diff --git a/.vendor/ggml-upstream.json b/.vendor/ggml-upstream.json index 40f5d3b..827825d 100644 --- a/.vendor/ggml-upstream.json +++ b/.vendor/ggml-upstream.json @@ -6,16 +6,16 @@ "subdirectory": "ggml" }, "current": { - "ref": "b8833", - "commit": "45cac7ca703fb9085eae62b9121fca01d20177f6", - "imported_in": "f5bb1543e899bacc11b90330632adfdf6cffd811", - "date": "2026-04-25", - "notes": "Exact upstream ref recorded." + "ref": "b11913", + "commit": "7aca0d2ea8d92b56911233b822db01f9fbb1ae50", + "imported_in": "ggml-b11913", + "date": "2026-10-27", + "notes": "Exact upstream ref recorded; local overlay (GTE ops, repack tweaks, llamafile fallback fix, WebGPU GTE kernels) re-applied with a 3-way merge against pristine b8833." }, "latest_checked": { - "date": "2026-04-24", - "ref": "b8833", - "commit": "45cac7c", + "date": "2026-10-27", + "ref": "b11913", + "commit": "7aca0d2", "source": "https://github.com/ggml-org/llama.cpp/releases" }, "local_policy": { diff --git a/README.md b/README.md index 67c2980..f61cc2a 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,21 @@ cmake --build build --target quantize .\build\Release\quantize.exe .\models\bge-m3.fp16.gguf .\models\bge-m3.q4_k.gguf q4_k ``` +Mixed quantization: `EMBEDDINGS_CPP_SKIP_QUANT_PATTERNS` (comma-separated +substrings of tensor names) keeps matching tensors as they are, and already +quantized tensors are never re-quantized, so two passes give a mix. For +bge-m3, q4_k MLP and vocabulary with q8_0 attention matches q8_0 retrieval +quality and is ~40% faster on x86 (AVX2), 384 MB instead of 612 MB: + +```bash +EMBEDDINGS_CPP_SKIP_QUANT_PATTERNS='attention.,position_embeddings' \ + ./build/quantize models/bge-m3.fp16.gguf models/bge-m3.q4_k_mlp_emb.tmp.gguf q4_k +./build/quantize models/bge-m3.q4_k_mlp_emb.tmp.gguf models/bge-m3.q4_k_mlp_emb_q8_attn.gguf q8_0 +``` + +On Apple silicon q8_0 stays faster on CPU (i8mm repack); with Metal both run +at the same speed. + ### Supported Quantization Types - `q4_k`: 4-bit quantization with K-means clustering (good balance of size and quality) @@ -441,8 +456,11 @@ Endpoints: was cut to `--max-tokens` has `"truncated": true`. - `POST /embed`: TEI-compatible (`inputs`, `normalize`, `truncate`; with `"truncate": false` an over-long input is rejected with 413). -- `GET /health`, `GET /ready`, `GET /info` (model, architecture, dimensions, - pooling, max tokens, limits), `GET /v1/models`. +- `GET /health`, `GET /ready`, `GET /info` (model, architecture, ftype, + `weight_types`, dimensions, pooling, max tokens, limits), `GET /v1/models`. + `weight_types` lists matrix weight types with counts (`"q4_K:49,q8_0:97"`); + unlike the file-level `ftype` it tells mixed quantizations apart, so clients + can use it to decide whether vectors from two servers are comparable. One model instance serves all requests; inference is serialized (ggml parallelizes inside a batch). Errors use the OpenAI error shape. diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 6b65ecd..35252a8 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -1,17 +1,11 @@ cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit target directories. -# ref: https://cmake.org/cmake/help/latest/policy/CMP0194.html -# MSVC is not a valid assembler for the ASM language. -# Set to NEW to avoid a warning on CMake 4.1+ with MSVC. -if (POLICY CMP0194) - cmake_policy(SET CMP0194 NEW) -endif() project("ggml" C CXX ASM) ### GGML Version set(GGML_VERSION_MAJOR 0) -set(GGML_VERSION_MINOR 9) -set(GGML_VERSION_PATCH 11) +set(GGML_VERSION_MINOR 27) +set(GGML_VERSION_PATCH 0) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") @@ -203,15 +197,16 @@ set(GGML_BLAS_VENDOR ${GGML_BLAS_VENDOR_DEFAULT} CACHE STRING option(GGML_LLAMAFILE "ggml: use LLAMAFILE" ${GGML_LLAMAFILE_DEFAULT}) option(GGML_CUDA "ggml: use CUDA" OFF) +set (GGML_CUDA_CCCL_VERSION "" CACHE STRING "ggml: CCCL git tag to fetch, empty to use the version bundled with the installed CUDA Toolkit") option(GGML_MUSA "ggml: use MUSA" OFF) option(GGML_CUDA_FORCE_MMQ "ggml: use mmq kernels instead of cuBLAS" OFF) option(GGML_CUDA_FORCE_CUBLAS "ggml: always use cuBLAS instead of mmq kernels" OFF) -set (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING - "ggml: max. batch size for using peer access") option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copies" OFF) option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF) option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON) option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF) +set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16" CACHE STRING + "ggml: FlashAttention K-V type combinations to compile, \"all\" or a list such as \"q8_0-q8_0;q8_0-q4_0\"") option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT}) option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON) set (GGML_CUDA_COMPRESSION_MODE "size" CACHE STRING @@ -219,10 +214,9 @@ set (GGML_CUDA_COMPRESSION_MODE "size" CACHE STRING set_property(CACHE GGML_CUDA_COMPRESSION_MODE PROPERTY STRINGS "none;speed;balance;size") option(GGML_HIP "ggml: use HIP" OFF) -option(GGML_HIP_GRAPHS "ggml: use HIP graph, experimental, slow" OFF) +option(GGML_HIP_GRAPHS "ggml: use HIP graph" ON) option(GGML_HIP_RCCL "ggml: use ROCm Collective Comm. Library" OFF) option(GGML_HIP_NO_VMM "ggml: do not try to use HIP VMM" ON) -option(GGML_HIP_ROCWMMA_FATTN "ggml: enable rocWMMA for FlashAttention" OFF) option(GGML_HIP_MMQ_MFMA "ggml: enable MFMA MMA for CDNA in MMQ" ON) option(GGML_HIP_EXPORT_METRICS "ggml: enable kernel perf metrics output" OFF) option(GGML_MUSA_GRAPHS "ggml: use MUSA graph, experimental, unstable" OFF) @@ -249,12 +243,16 @@ option(GGML_METAL_EMBED_LIBRARY "ggml: embed Metal library" set (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING "ggml: metal minimum macOS version") set (GGML_METAL_STD "" CACHE STRING "ggml: metal standard version (-std flag)") +set (GGML_METAL_TARGET_OS "macos" CACHE STRING + "ggml: metal -mtargetos OS name (macos, ios, xros, tvos)") option(GGML_OPENMP "ggml: use OpenMP" ON) +option(GGML_OPENMP_FETCH "ggml: fetch LLVM OpenMP" OFF) option(GGML_RPC "ggml: use RPC" OFF) option(GGML_SYCL "ggml: use SYCL" OFF) option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF) option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON) option(GGML_SYCL_HOST_MEM_FALLBACK "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON) +option(GGML_SYCL_SUPPORT_LEVEL_ZERO_API "ggml: use Level Zero API in SYCL backend" ON) option(GGML_SYCL_DNN "ggml: enable oneDNN in the SYCL backend" ON) set (GGML_SYCL_TARGET "INTEL" CACHE STRING "ggml: sycl target device") @@ -262,6 +260,8 @@ set (GGML_SYCL_DEVICE_ARCH "" CACHE STRING "ggml: sycl device architecture") option(GGML_OPENVINO "ggml: use OPENVINO" OFF) +option(GGML_ET "ggml: use ET backend" OFF) +option(GGML_ET_SYSEMU "ggml: use ET backend via sysemu" OFF) option(GGML_OPENCL "ggml: use OpenCL" OFF) option(GGML_OPENCL_PROFILING "ggml: use OpenCL profiling (increases overhead)" OFF) @@ -271,7 +271,6 @@ set (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING "ggml: OpenCL API version to target") option(GGML_HEXAGON "ggml: enable Hexagon backend" OFF) -set(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING "ggml: quantize group size (32, 64, or 128)") # toolchain for vulkan-shaders-gen set (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen") @@ -346,9 +345,6 @@ set(GGML_PUBLIC_HEADERS include/gguf.h) set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}") -#if (GGML_METAL) -# set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal") -#endif() install(TARGETS ggml LIBRARY PUBLIC_HEADER) install(TARGETS ggml-base LIBRARY) @@ -358,7 +354,7 @@ if (GGML_STANDALONE) @ONLY) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc - DESTINATION share/pkgconfig) + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) endif() # @@ -407,19 +403,15 @@ configure_package_config_file( GGML_BIN_INSTALL_DIR) write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake + ${CMAKE_CURRENT_BINARY_DIR}/ggml-config-version.cmake VERSION ${GGML_INSTALL_VERSION} COMPATIBILITY SameMajorVersion) -target_compile_definitions(ggml-base PRIVATE - GGML_VERSION="${GGML_INSTALL_VERSION}" - GGML_COMMIT="${GGML_BUILD_COMMIT}" -) message(STATUS "ggml version: ${GGML_INSTALL_VERSION}") message(STATUS "ggml commit: ${GGML_BUILD_COMMIT}") install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake + ${CMAKE_CURRENT_BINARY_DIR}/ggml-config-version.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml) if (MSVC) diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake index cb66388..42b65a5 100644 --- a/ggml/cmake/common.cmake +++ b/ggml/cmake/common.cmake @@ -2,10 +2,16 @@ function(ggml_get_flags CCID CCVER) set(C_FLAGS "") set(CXX_FLAGS "") - if (CCID MATCHES "Clang") + if (CCID MATCHES "Clang" OR CCID MATCHES "IntelLLVM") set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return) set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi) + if (CCID STREQUAL "IntelLLVM") + # for Intel DPC++ and LLVM SYCL, using debug does not disable optimization + list(APPEND C_FLAGS -Rno-debug-disables-optimization) + list(APPEND CXX_FLAGS -Rno-debug-disables-optimization) + endif() + if ( (CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0) @@ -28,6 +34,7 @@ endfunction() function(ggml_get_system_arch) if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR + (CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64ec" AND MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") OR (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$")) set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE) @@ -48,3 +55,74 @@ function(ggml_get_system_arch) set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE) endif() endfunction() + +# Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS. +function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS) + set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16) + + string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS) + string(STRIP "${FA_QUANTS}" FA_QUANTS) + if (GGML_CUDA_FA_ALL_QUANTS) + message(WARNING "GGML_CUDA_FA_ALL_QUANTS is deprecated, use GGML_CUDA_FA_QUANTS=all instead") + set(FA_QUANTS all) + endif() + if (NOT FA_QUANTS) + message(FATAL_ERROR "GGML_CUDA_FA_QUANTS must not be empty") + endif() + + if (FA_QUANTS STREQUAL "all") + set(FA_COMBINATIONS "") + foreach (TYPE_V IN LISTS FA_TYPES) + foreach (TYPE_K IN LISTS FA_TYPES) + list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V}) + endforeach() + endforeach() + else() + set(FA_COMBINATIONS f16-f16) + + string(REPLACE "," ";" FA_SELECTED "${FA_QUANTS}") + foreach (COMBINATION IN LISTS FA_SELECTED) + string(STRIP "${COMBINATION}" COMBINATION) + if (NOT COMBINATION MATCHES "^([a-z0-9_]+)-([a-z0-9_]+)$") + message(FATAL_ERROR "GGML_CUDA_FA_QUANTS: \"${COMBINATION}\" is not \"all\" or a - combination") + endif() + set(TYPE_K ${CMAKE_MATCH_1}) + set(TYPE_V ${CMAKE_MATCH_2}) + foreach (TYPE ${TYPE_K} ${TYPE_V}) + if (NOT TYPE IN_LIST FA_TYPES) + message(FATAL_ERROR + "GGML_CUDA_FA_QUANTS: unknown type \"${TYPE}\" in \"${COMBINATION}\", must be one of: ${FA_TYPES}") + endif() + endforeach() + list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V}) + endforeach() + endif() + list(REMOVE_DUPLICATES FA_COMBINATIONS) + + string(REPLACE ";" "," FA_QUANTS_DEFINE "${FA_QUANTS}") + add_compile_definitions(GGML_CUDA_FA_QUANTS="${FA_QUANTS_DEFINE}") + foreach (TYPE_V IN LISTS FA_TYPES) + foreach (TYPE_K IN LISTS FA_TYPES) + if ("${TYPE_K}-${TYPE_V}" IN_LIST FA_COMBINATIONS) + set(COMPILED 1) + else() + set(COMPILED 0) + endif() + string(TOUPPER "GGML_CUDA_FA_${TYPE_K}_${TYPE_V}" COMBINATION_DEF) + add_compile_definitions(${COMBINATION_DEF}=${COMPILED}) + endforeach() + endforeach() + + message(STATUS "FlashAttention K-V type combinations: ${FA_COMBINATIONS}") + + set(SRCS "") + foreach (COMBINATION IN LISTS FA_COMBINATIONS) + set(SRC "${DIR}/template-instances/fattn-vec-instance-${COMBINATION}.cu") + if (NOT EXISTS "${SRC}") + message(FATAL_ERROR "FlashAttention template instance \"${SRC}\" does not exist") + endif() + list(APPEND SRCS "${SRC}") + endforeach() + + set(${OUT_SRCS} ${SRCS} PARENT_SCOPE) +endfunction() diff --git a/ggml/cmake/ggml-config.cmake.in b/ggml/cmake/ggml-config.cmake.in index 91c9d5c..a28e49e 100644 --- a/ggml/cmake/ggml-config.cmake.in +++ b/ggml/cmake/ggml-config.cmake.in @@ -6,6 +6,7 @@ include(CMakeFindDependencyMacro) find_dependency(Threads) if (NOT GGML_SHARED_LIB) + set(GGML_BASE_INTERFACE_LINK_LIBRARIES "") set(GGML_CPU_INTERFACE_LINK_LIBRARIES "") set(GGML_CPU_INTERFACE_LINK_OPTIONS "") @@ -20,7 +21,15 @@ if (NOT GGML_SHARED_LIB) if (GGML_OPENMP_ENABLED) find_dependency(OpenMP) - list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_C OpenMP::OpenMP_CXX) + set(GGML_OPENMP_INTERFACE_LINK_LIBRARIES "") + if (TARGET OpenMP::OpenMP_C) + list(APPEND GGML_OPENMP_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_C) + endif() + if (TARGET OpenMP::OpenMP_CXX) + list(APPEND GGML_OPENMP_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_CXX) + endif() + list(APPEND GGML_BASE_INTERFACE_LINK_LIBRARIES ${GGML_OPENMP_INTERFACE_LINK_LIBRARIES}) + list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${GGML_OPENMP_INTERFACE_LINK_LIBRARIES}) endif() if (GGML_CPU_HBM) @@ -101,9 +110,20 @@ set_and_check(GGML_INCLUDE_DIR "@PACKAGE_GGML_INCLUDE_INSTALL_DIR@") set_and_check(GGML_LIB_DIR "@PACKAGE_GGML_LIB_INSTALL_DIR@") #set_and_check(GGML_BIN_DIR "@PACKAGE_GGML_BIN_INSTALL_DIR@") +if (NOT GGML_SHARED_LIB AND GGML_CPU_KLEIDIAI) + unset(KLEIDIAI_LIBRARY CACHE) + unset(KLEIDIAI_LIBRARY) + find_library(KLEIDIAI_LIBRARY kleidiai + REQUIRED + HINTS ${GGML_LIB_DIR} + NO_CMAKE_FIND_ROOT_PATH) + list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${KLEIDIAI_LIBRARY}) +endif() + if(NOT TARGET ggml::ggml) find_package(Threads REQUIRED) + unset(GGML_LIBRARY CACHE) find_library(GGML_LIBRARY ggml REQUIRED HINTS ${GGML_LIB_DIR} @@ -112,8 +132,10 @@ if(NOT TARGET ggml::ggml) add_library(ggml::ggml UNKNOWN IMPORTED) set_target_properties(ggml::ggml PROPERTIES - IMPORTED_LOCATION "${GGML_LIBRARY}") + IMPORTED_LOCATION "${GGML_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}") + unset(GGML_BASE_LIBRARY CACHE) find_library(GGML_BASE_LIBRARY ggml-base REQUIRED HINTS ${GGML_LIB_DIR} @@ -122,7 +144,9 @@ if(NOT TARGET ggml::ggml) add_library(ggml::ggml-base UNKNOWN IMPORTED) set_target_properties(ggml::ggml-base PROPERTIES - IMPORTED_LOCATION "${GGML_BASE_LIBRARY}") + IMPORTED_LOCATION "${GGML_BASE_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${GGML_BASE_INTERFACE_LINK_LIBRARIES}") set(_ggml_all_targets "") if (NOT GGML_BACKEND_DL) @@ -130,6 +154,7 @@ if(NOT TARGET ggml::ggml) string(REPLACE "-" "_" _ggml_backend_pfx "${_ggml_backend}") string(TOUPPER "${_ggml_backend_pfx}" _ggml_backend_pfx) + unset(${_ggml_backend_pfx}_LIBRARY CACHE) find_library(${_ggml_backend_pfx}_LIBRARY ${_ggml_backend} REQUIRED HINTS ${GGML_LIB_DIR} diff --git a/ggml/include/ggml-alloc.h b/ggml/include/ggml-alloc.h index 78aa059..81cce2e 100644 --- a/ggml/include/ggml-alloc.h +++ b/ggml/include/ggml-alloc.h @@ -75,8 +75,10 @@ GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_i // Utils // Create a buffer and allocate all the tensors in a ggml_context -// ggml_backend_alloc_ctx_tensors_from_buft_size returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft + +// returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft. returns 0 on failure GGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +// returns NULL on failure or if all tensors in ctx are already allocated or zero-sized GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend); diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index d0c7e5a..7a65d81 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -34,13 +34,15 @@ extern "C" { // Backend buffer type // - GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); - GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); - GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); - GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft); - GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); - GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); - GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft); + GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); + GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n (ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors); + GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); + GGML_API size_t ggml_backend_buft_get_alloc_size_n(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors); + GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft); // // Backend buffer @@ -154,6 +156,8 @@ extern "C" { bool buffer_from_host_ptr; // event synchronization bool events; + // mmap is supported for loading + bool mmap_support; }; // all the device properties @@ -169,7 +173,7 @@ extern "C" { // device type enum ggml_backend_dev_type type; // device id - // for PCI devices, this should be the PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:01:00.0") + // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0") // if the id is unknown, this should be NULL const char * device_id; // device capabilities @@ -313,6 +317,23 @@ extern "C" { // typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); + // Callback while copying input weights of a split + // if the user returns false the scheduler simply copies the entire weight + // the callback is called only for input weights in host buffers + // the callback is called after all non-weight inputs of the split have been copied + // `src` is the tensor in the previous split + // `dst` is the copy of `src` in the split + // `graph` is the compute graph of the split + typedef bool (*ggml_backend_sched_copy_callback)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst, struct ggml_cgraph * graph, void * user_data); + + // Callback when the buffers reserved for a graph are not sufficient and have to be reserved again + // it happens on the graph allocation path and costs a synchronization of the backends + // `sched` is the scheduler + // `graph` is the compute graph being allocated + // `backend_ids_changed` is true if the backend assignment of the graph nodes changed since the last allocation + // `graph_size_changed` is true if the graph size changed since the last allocation + typedef void (*ggml_backend_sched_realloc_callback)(ggml_backend_sched_t sched, struct ggml_cgraph * graph, bool backend_ids_changed, bool graph_size_changed, void * user_data); + // Initialize a backend scheduler, backends with low index are given priority over backends with high index GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload); GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); @@ -351,6 +372,12 @@ extern "C" { // Set a callback to be called for each resulting node during graph compute GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + // Set a callback to be called when the inputs weights of a split are being copied + GGML_API void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data); + + // Set a callback to be called when the compute buffers of a graph are being reallocated + GGML_API void ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data); + // // Meta backend // @@ -381,11 +408,16 @@ extern "C" { // - most tensors have n_segments == 1 and a contiguous slice of the tensor data // - some tensors have an inhomogenenous data layout along the split axis, // those tensors are divided into segments which are each individually split across devices - // - ne has one entry per segment and device that add up to ggml_tensor::ne for that axis, - // the outer/inner loops are over segments/devices like [seg0_dev0, seg0_dev1, seg1_dev0, seg1_dev1], + // (this usually happens when multiple tensors are fused into a single one) + // - ne has one entry per segment and device and that segment repeats nr times, + // in total when accounting for repetitions the segments add up to ggml_tensor::ne for that axis, + // the outer/inner loops are over segments/devices like [seg0_dev0_r0, seg0_dev1_r0, seg0_dev0_r1, seg0_dev1_r1, seg1_dev0_r0, seg1_dev1_r0], // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments - // that each need to be split individually across devices so that each device gets a slice of Q, K, and V + // that each need to be split individually across devices so that each device gets a slice of Q, K, and V, + // the Q matrix can be larger than the K and V matrices so this can either be expressed as 3 segments or as 2 segments + // where the segment for K/V repeats twice int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES]; + uint32_t nr[16]; uint32_t n_segments; }; diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.h index e3e067c..dc6453c 100644 --- a/ggml/include/ggml-cpu.h +++ b/ggml/include/ggml-cpu.h @@ -100,6 +100,7 @@ extern "C" { GGML_BACKEND_API int ggml_cpu_has_sve (void); GGML_BACKEND_API int ggml_cpu_get_sve_cnt (void); // sve vector length in bytes GGML_BACKEND_API int ggml_cpu_has_sme (void); + GGML_BACKEND_API int ggml_cpu_has_sme2 (void); // other GGML_BACKEND_API int ggml_cpu_has_riscv_v (void); GGML_BACKEND_API int ggml_cpu_get_rvv_vlen (void); // risc-v vector length in bytes diff --git a/ggml/include/ggml-cuda.h b/ggml/include/ggml-cuda.h index 5436c7e..1cd81ee 100644 --- a/ggml/include/ggml-cuda.h +++ b/ggml/include/ggml-cuda.h @@ -30,9 +30,6 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int de // conduct allreduce operation between devices GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends); -// split tensor buffer that splits matrices by rows across multiple devices -GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split); - // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void); diff --git a/ggml/include/ggml-et.h b/ggml/include/ggml-et.h new file mode 100644 index 0000000..8b78f39 --- /dev/null +++ b/ggml/include/ggml-et.h @@ -0,0 +1,28 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GGML_ET_NAME "ET" + +// backend API +GGML_BACKEND_API ggml_guid_t ggml_backend_et_guid(void); +GGML_BACKEND_API ggml_backend_t ggml_backend_et_init(size_t devidx); + +GGML_BACKEND_API bool ggml_backend_is_et(ggml_backend_t backend); +GGML_BACKEND_API int ggml_backend_et_get_device_count(void); +GGML_BACKEND_API void ggml_backend_et_get_device_description(int devidx, char * description, size_t description_size); +GGML_BACKEND_API void ggml_backend_et_get_device_memory(int devidx, size_t * free, size_t * total); + +GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_et_buffer_type(size_t dev_num); +GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_et_host_buffer_type(void); + +GGML_BACKEND_API ggml_backend_reg_t ggml_backend_et_reg(void); + +#ifdef __cplusplus +} +#endif diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h index 9fa4369..930d053 100644 --- a/ggml/include/ggml-rpc.h +++ b/ggml/include/ggml-rpc.h @@ -6,12 +6,12 @@ extern "C" { #endif -#define RPC_PROTO_MAJOR_VERSION 4 +#define RPC_PROTO_MAJOR_VERSION 8 #define RPC_PROTO_MINOR_VERSION 0 -#define RPC_PROTO_PATCH_VERSION 5 +#define RPC_PROTO_PATCH_VERSION 1 #ifdef __cplusplus -static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); +static_assert(GGML_OP_COUNT == 108, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); #endif #define GGML_RPC_MAX_SERVERS 16 diff --git a/ggml/include/ggml-sycl.h b/ggml/include/ggml-sycl.h index 5ce349a..1e353ff 100644 --- a/ggml/include/ggml-sycl.h +++ b/ggml/include/ggml-sycl.h @@ -25,9 +25,19 @@ GGML_BACKEND_API bool ggml_backend_is_sycl(ggml_backend_t backend); GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device); // split tensor buffer that splits matrices by rows across multiple devices -GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split); +GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(int main_device, const float * tensor_split); + +// Tensor parallelism (--split-mode tensor): comm_init/free/allreduce_tensor +// trio queried by the meta-backend via ggml_backend_reg_get_proc_address. +// See typedefs in ggml/include/ggml-backend.h. Mirrors the CUDA backend's +// pattern (ggml_backend_cuda_comm_*). +GGML_BACKEND_API void * ggml_backend_sycl_comm_init(ggml_backend_t * backends, size_t n_backends); +GGML_BACKEND_API void ggml_backend_sycl_comm_free(void * comm_ctx); +GGML_BACKEND_API bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx, struct ggml_tensor ** tensors); // pinned host buffer for use with the CPU backend for faster copies between CPU and GPU +// pins on device 0 - a copy between another device and this memory can fail, +// use ggml_backend_dev_host_buffer_type to pin on the device that does the copy GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type(void); GGML_BACKEND_API void ggml_backend_sycl_print_sycl_devices(void); @@ -35,7 +45,7 @@ GGML_BACKEND_API void ggml_backend_sycl_get_gpu_list(int *id_list, int max_len); GGML_BACKEND_API void ggml_backend_sycl_get_device_description(int device, char *description, size_t description_size); -GGML_BACKEND_API int ggml_backend_sycl_get_device_count(); +GGML_BACKEND_API int ggml_backend_sycl_get_device_count(void); GGML_BACKEND_API void ggml_backend_sycl_get_device_memory(int device, size_t *free, size_t *total); // SYCL doesn't support registering host memory, keep here for reference diff --git a/ggml/include/ggml-zdnn.h b/ggml/include/ggml-zdnn.h index fbf45b6..e6970ad 100644 --- a/ggml/include/ggml-zdnn.h +++ b/ggml/include/ggml-zdnn.h @@ -10,6 +10,8 @@ extern "C" { // device buffer GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_zdnn_buffer_type(void); +GGML_BACKEND_API bool ggml_backend_is_zdnn(ggml_backend_t backend); + GGML_BACKEND_API ggml_backend_reg_t ggml_backend_zdnn_reg(void); #ifdef __cplusplus diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 7950983..5288481 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -429,13 +429,31 @@ extern "C" { GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, - GGML_TYPE_COUNT = 42, + GGML_TYPE_Q2_0 = 42, + GGML_TYPE_COUNT = 43, }; - // precision + // [TAG_GGML_PREC] + // this enum is used to declare the allowed numerical precision/data-types types that can be used during the compute of an op + // the declared types can be: + // - result accumulation type + // - source tensor data representation type + // - etc. + // the precision parameters are stored as ggml_tensor.op_params to the respective ops enum ggml_prec { - GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default - GGML_PREC_F32 = 10, + GGML_PREC_UNDEFINED = 0, + GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED + GGML_PREC_F32 = 10, + GGML_PREC_BF16 = 15, + GGML_PREC_F16 = 20, + GGML_PREC_Q8 = 30, + GGML_PREC_Q4 = 40, + }; + + // op hint + enum ggml_op_hint { + GGML_HINT_NONE = 0, + GGML_HINT_SRC0_IS_HADAMARD = 1, }; // model file types @@ -467,6 +485,7 @@ extern "C" { GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors }; // available tensor operations: @@ -535,6 +554,7 @@ extern "C" { GGML_OP_IM2COL, GGML_OP_IM2COL_BACK, GGML_OP_IM2COL_3D, + GGML_OP_COL2IM_1D, GGML_OP_CONV_2D, GGML_OP_CONV_3D, GGML_OP_CONV_2D_DW, @@ -567,6 +587,11 @@ extern "C" { GGML_OP_RWKV_WKV7, GGML_OP_SOLVE_TRI, GGML_OP_GATED_DELTA_NET, + GGML_OP_LIGHTNING_INDEXER, + GGML_OP_DSV4_HC_COMB, + GGML_OP_DSV4_HC_PRE, + GGML_OP_DSV4_HC_POST, + GGML_OP_TOP_K_MASK, GGML_OP_UNARY, @@ -620,6 +645,7 @@ extern "C" { GGML_GLU_OP_SWIGLU_OAI, GGML_GLU_OP_GEGLU_ERF, GGML_GLU_OP_GEGLU_QUICK, + GGML_GLU_OP_SWIGLU_CLAMP, GGML_GLU_OP_COUNT, }; @@ -776,6 +802,10 @@ extern "C" { GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1 GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2 + GGML_API bool ggml_is_contiguous_to_1(const struct ggml_tensor * tensor); // contiguous for dims < 1 + GGML_API bool ggml_is_contiguous_to_2(const struct ggml_tensor * tensor); // contiguous for dims < 2 + GGML_API bool ggml_is_contiguous_to_3(const struct ggml_tensor * tensor); // contiguous for dims < 3 + // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok) GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor); @@ -902,6 +932,7 @@ extern "C" { enum ggml_type type); // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]] + // an id of -1 adds nothing: dst[i0, i1, i2] = a[i0, i1, i2] GGML_API struct ggml_tensor * ggml_add_id( struct ggml_context * ctx, struct ggml_tensor * a, @@ -1189,8 +1220,8 @@ extern "C" { struct ggml_context * ctx, struct ggml_tensor * a); - // a - x - // b - dy + // a - dy + // b - x GGML_API struct ggml_tensor * ggml_silu_back( struct ggml_context * ctx, struct ggml_tensor * a, @@ -1356,6 +1387,12 @@ extern "C" { float alpha, float limit); + GGML_API struct ggml_tensor * ggml_swiglu_clamp( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float limit); + // normalize along rows GGML_API struct ggml_tensor * ggml_norm( struct ggml_context * ctx, @@ -1411,6 +1448,52 @@ extern "C" { struct ggml_tensor * b, float eps); + // [TAG_GGML_PREC] + // set the minimum required accumulator type for the implementation to use during the compute + // for example: + // - GGML_PREC_F32 - requires accumulation of the results in F32 + // - GGML_PREC_BF16 - can accumulate the results in BF16, F32 + // - GGML_PREC_F16 - can accumulate the results in F16, F32 + // - GGML_PREC_Q8 - not allowed + // - GGML_PREC_Q4 - not allowed + // + // return false on faliure + GGML_API bool ggml_prec_set_acc( + struct ggml_tensor * a, + enum ggml_prec prec); + + // [TAG_GGML_PREC] + // set the smallest rank that the implementation can use to internally convert the src[idx] data to + // ranks in decreasing order: + // - GGML_PREC_F32 - GGML_TYPE_F32 + // - GGML_PREC_BF16 - GGML_TYPE_BF16 + // - GGML_PREC_F16 - GGML_TYPE_F16, + // - GGML_PREC_Q8 - GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, GGML_TYPE_Q8_K, etc. + // - GGML_PREC_Q4 - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_NVFP4, GGML_TYPE_MXFP4, etc. + // + // for example: + // - ggml_prec_set_src(a, GGML_PREC_Q8, 1): + // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to GGML_TYPE_Q8_0 + // - cannot quantize it down to GGML_TYPE_Q4_0 or GGML_TYPE_NVFP4 + // - ggml_prec_set_src(a, GGML_PREC_Q4, 1): + // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to 4-bit datatypes such as GGML_TYPE_Q4_K, GGML_TYPE_NVFP4 etc. + // + // return false on faliure + GGML_API bool ggml_prec_set_src( + struct ggml_tensor * a, + enum ggml_prec prec, + int idx); + + // [TAG_GGML_PREC] + // get the accumulator / src[idx] precision set by the functions above + // return GGML_PREC_UNDEFINED when the op does not carry the requested precision + GGML_API enum ggml_prec ggml_prec_get_acc( + const struct ggml_tensor * a); + + GGML_API enum ggml_prec ggml_prec_get_src( + const struct ggml_tensor * a, + int idx); + // A: k columns, n rows => [ne03, ne02, n, k] // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] @@ -1421,11 +1504,23 @@ extern "C" { // change the precision of a matrix multiplication // set to GGML_PREC_F32 for higher precision (useful for phi-2) - GGML_API void ggml_mul_mat_set_prec( + GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec( struct ggml_tensor * a, - enum ggml_prec prec); + enum ggml_prec prec), + "use ggml_prec_set_acc() instead"); + + // change the hint of a matrix multiplication + GGML_API void ggml_mul_mat_set_hint( + struct ggml_tensor * a, + enum ggml_op_hint hint); // indirect matrix multiplication + // an id of -1 skips the slot and zeroes the matching row of the result + // + // [TAG_MUL_MAT_ID_DUPLICATE_ID] + // note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b` + // (i.e. duplicate experts per token are not guaranteed to be supported in all backends) + // TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533 GGML_API struct ggml_tensor * ggml_mul_mat_id( struct ggml_context * ctx, struct ggml_tensor * as, @@ -1708,6 +1803,19 @@ extern "C" { struct ggml_tensor * a, int n_past); + GGML_API struct ggml_tensor * ggml_clamp( + struct ggml_context * ctx, + struct ggml_tensor * a, + float min, + float max); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_clamp_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float min, + float max); + GGML_API struct ggml_tensor * ggml_soft_max( struct ggml_context * ctx, struct ggml_tensor * a); @@ -2013,13 +2121,14 @@ extern "C" { struct ggml_tensor * bias); - // clamp - // in-place, returns view(a) - GGML_API struct ggml_tensor * ggml_clamp( - struct ggml_context * ctx, + // set the offset dims for RoPE + // a must be GGML_OP_ROPE or GGML_OP_ROPE_BACK + // vision RoPE is not supported + // example: (marking: x = rotated, 0 = unrotated) + // n_embd = 10, n_dims = 4, offset = 2 --> [00xxxx0000] + GGML_API struct ggml_tensor * ggml_rope_set_offset( struct ggml_tensor * a, - float min, - float max); + int n_offs); // im2col // converts data into a format that effectively results in a convolution when combined with matrix multiplication @@ -2049,6 +2158,16 @@ extern "C" { int d1, // dilation dimension 1 bool is_2D); + // col2im_1d: scatter-add GEMM columns back to 1D signal + // a: [K*OC, T_in] (columns from matmul, K = a->ne[0]/OC) + // result: [T_out, OC] where T_out = (T_in - 1)*s0 + K - 2*p0 + GGML_API struct ggml_tensor * ggml_col2im_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // columns [K*OC, T_in] + int s0, // stride + int oc, // output channels + int p0); // padding to crop from both sides + GGML_API struct ggml_tensor * ggml_conv_1d( struct ggml_context * ctx, struct ggml_tensor * a, // convolution kernel @@ -2447,13 +2566,20 @@ extern "C" { float max_bias, float logit_softcap); - GGML_API void ggml_flash_attn_ext_set_prec( + GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec( struct ggml_tensor * a, - enum ggml_prec prec); + enum ggml_prec prec), + "use ggml_prec_set_acc() instead"); GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( const struct ggml_tensor * a); + // Use finite mask entries as a sparse K/V set. Set 0 to disable. + // n_kv_max must bound the number of finite entries in every mask row. + GGML_API void ggml_flash_attn_ext_set_n_kv_max( + struct ggml_tensor * a, + int32_t n_kv_max); + GGML_API void ggml_flash_attn_ext_add_sinks( struct ggml_tensor * a, struct ggml_tensor * sinks); @@ -2480,7 +2606,8 @@ extern "C" { struct ggml_tensor * A, struct ggml_tensor * B, struct ggml_tensor * C, - struct ggml_tensor * ids); + struct ggml_tensor * ids, + int64_t K); // partition into non-overlapping windows with padding if needed // example: @@ -2583,6 +2710,18 @@ extern "C" { // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST] // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306 + // + // tensor shapes (S_k == S_v, H_v % H_k == 0): + // q, k : [S_k, H_k, n_tokens, n_seqs] + // v : [S_v, H_v, n_tokens, n_seqs] + // g : [1, H_v, n_tokens, n_seqs] (scalar gate) or [S_v, H_v, n_tokens, n_seqs] (KDA) + // beta : [1, H_v, n_tokens, n_seqs] + // state : [S_v, S_v, H_v, n_seqs] -- initial recurrent state s0 + // + // the output packs the attention scores [S_v, H_v, n_tokens, n_seqs] followed by K state + // snapshots, most-recent first (slot 0 = final state, slot s = state s tokens back). K == 1 + // keeps only the final state; when n_tokens < K only slots 0..n_tokens-1 are written. + // when n_tokens > K only slot 0 is guaranteed; backends may leave slots 1..K-1 unwritten. GGML_API struct ggml_tensor * ggml_gated_delta_net( struct ggml_context * ctx, struct ggml_tensor * q, @@ -2590,7 +2729,95 @@ extern "C" { struct ggml_tensor * v, struct ggml_tensor * g, struct ggml_tensor * beta, - struct ggml_tensor * state); + struct ggml_tensor * state, + int64_t K); + + // DSA lightning indexer + // + // q: [n_embd_idx, n_head_idx, n_batch, ne3 ] + // k: [n_embd_idx, 1, n_kv, ne3 ] + // weights: [n_head_idx, n_batch, 1, ne3 ] !! prescaled !! + // mask: [n_kv, n_batch, 1, ne33] !! f16 !! + // res: [n_kv, n_batch, 1, ne3 ] + // + // broadcast: + // ne3 % ne33 == 0 + // + GGML_API struct ggml_tensor * ggml_lightning_indexer( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * weights, + struct ggml_tensor * mask); + + // DeepSeek V4 hyper-connections (ref. https://arxiv.org/pdf/2512.24880) + // In short these operations are replacements for the original residual connection (x = transformer(x) + x) + // using a richer representation through streams. + // + // hc_comb: mixes [(2 + hc)*hc, n_tokens], scale [3], base [(2 + hc)*hc] + // -> [dst_hc, src_hc, n_tokens] + // logits[dst, src, t] = mixes[2*hc + dst + hc*src, t]*scale[2] + // + base[2*hc + dst + hc*src] + // Softmax over dst, add eps, normalize over src, then repeat normalization + // over dst followed by src for iterations 1 through n_iter - 1. + GGML_API struct ggml_tensor * ggml_dsv4_hc_comb( + struct ggml_context * ctx, + struct ggml_tensor * mixes, + struct ggml_tensor * scale, + struct ggml_tensor * base, + float eps, + int32_t n_iter); + + // hc_pre: x [n_embd, hc, n_tokens], weights [hc, n_tokens] -> [n_embd, n_tokens] + // result[i, t] = sum_h x[i, h, t]*weights[h, t] + // + GGML_API struct ggml_tensor * ggml_dsv4_hc_pre( + struct ggml_context * ctx, + struct ggml_tensor * x, + struct ggml_tensor * weights); + + // hc_pre with a per-element gate (Qwen3.8-Flash-Next): gate [n_embd, hc, n_tokens] + // result[i, t] = scale*sum_h x[i, h, t]*sigmoid(gate[i, h, t]) + // + GGML_API struct ggml_tensor * ggml_dsv4_hc_pre_gated( + struct ggml_context * ctx, + struct ggml_tensor * x, + struct ggml_tensor * gate, + float scale); + + // hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens], + // post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens] + // -> [n_embd, hc, n_tokens] + // result[i, dst, t] = x[i, t]*post[dst, t] + // + sum_src residual[i, src, t]*comb[dst, src, t] + // comb == NULL uses the identity: result[i, dst, t] = x[i, t]*post[dst, t] + residual[i, dst, t] + // + GGML_API struct ggml_tensor * ggml_dsv4_hc_post( + struct ggml_context * ctx, + struct ggml_tensor * x, + struct ggml_tensor * residual, + struct ggml_tensor * post, + struct ggml_tensor * comb); + + // top-k sparse attention mask: keep the mask cells of the live picks and the extra cells, set the rest to -inf + // + // mask: [n_kv, m1, m2, m3] f16 or f32, broadcast to the rows of top_k + // top_k: [n_top, ne1, ne2, ne3] i32, picks of each row + // score: [n_pick, ne1, ne2, ne3] f32, can be NULL: a pick is live when its score is above -inf, else every pick is live + // blocks: [n_blk, n_pick] i32, can be NULL: the member cells of each pick, else a pick is a cell + // extra: [n_extra, ne1, ne2, ne3] i32, can be NULL: cells of each row kept without a pick + // res: [n_kv, ne1, ne2, ne3] same type as mask + // + // res[c, i] = mask[c, i % m] when c is a cell of a live pick of row i or an extra cell of row i, else -inf + // picks and cells out of range are skipped + // + GGML_API struct ggml_tensor * ggml_top_k_mask( + struct ggml_context * ctx, + struct ggml_tensor * mask, + struct ggml_tensor * top_k, + struct ggml_tensor * score, + struct ggml_tensor * blocks, + struct ggml_tensor * extra); // custom operators @@ -2740,6 +2967,12 @@ extern "C" { struct ggml_cgraph * cgraph, struct ggml_tensor * tensor); + // add the tensor and its parents to the graph without marking them for compute + // the flag is set later, when the tensor is reached from a node that computes + GGML_API void ggml_build_forward_order( + struct ggml_cgraph * cgraph, + struct ggml_tensor * tensor); + GGML_API void ggml_build_backward_expand( struct ggml_context * ctx, // context for gradient computation struct ggml_cgraph * cgraph, diff --git a/ggml/include/gguf.h b/ggml/include/gguf.h index 02d5f22..b3a1e12 100644 --- a/ggml/include/gguf.h +++ b/ggml/include/gguf.h @@ -76,10 +76,16 @@ extern "C" { struct ggml_context ** ctx; }; + // callback to simulate or wrap a FILE pointer - read up to `len` bytes at `offset` into `output` and return the number of bytes read + typedef size_t (*gguf_reader_callback_t)(void * userdata, void * output, uint64_t offset, size_t len); + GGML_API struct gguf_context * gguf_init_empty(void); GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); - //GGML_API struct gguf_context * gguf_init_from_buffer(..); + GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); + + // max_chunk_read is the maximum number of bytes that the GGUF code will read at once from the callback, a value of 0 means no limit + GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params); GGML_API void gguf_free(struct gguf_context * ctx); @@ -87,7 +93,7 @@ extern "C" { GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx); GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx); - GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); + GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); // padded to gguf_get_alignment if and only if the gguf_context contains at least one tensor GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx); GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found @@ -119,12 +125,13 @@ extern "C" { // get ith C string from array with given key_id GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i); - GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx); - GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found - GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id); - GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id); - GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id); - GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx); + GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found + GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id); + GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API const int64_t * gguf_get_tensor_ne (const struct gguf_context * ctx, int64_t tensor_id); // returns ne, an array of GGML_MAX_DIMS elements; ne[dim] is 1 for dim >= n_dims + GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id); // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist) GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key); diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 48fbe20..fe001a0 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -213,7 +213,9 @@ set_target_properties(ggml-base PROPERTIES SOVERSION ${GGML_VERSION_MAJOR} ) -target_include_directories(ggml-base PRIVATE .) +configure_file(ggml-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.h @ONLY) + +target_include_directories(ggml-base PRIVATE . ${CMAKE_CURRENT_BINARY_DIR}) if (GGML_BACKEND_DL) target_compile_definitions(ggml-base PUBLIC GGML_BACKEND_DL) endif() @@ -222,6 +224,137 @@ if (GGML_SCHED_NO_REALLOC) target_compile_definitions(ggml-base PUBLIC GGML_SCHED_NO_REALLOC) endif() +if (GGML_OPENMP_FETCH) + if (NOT GGML_OPENMP) + message(FATAL_ERROR "GGML_OPENMP_FETCH requires GGML_OPENMP") + elseif (NOT WIN32 OR NOT (CMAKE_C_COMPILER_ID MATCHES "Clang")) + message(FATAL_ERROR "GGML_OPENMP_FETCH currently requires Clang on Windows") + endif() + + set(GGML_OPENMP_LLVM_VERSION "20.1.8") + string(REGEX MATCH "^[0-9]+" GGML_OPENMP_LLVM_VERSION_MAJOR "${GGML_OPENMP_LLVM_VERSION}") + string(REGEX MATCH "^[0-9]+" GGML_OPENMP_COMPILER_VERSION_MAJOR "${CMAKE_C_COMPILER_VERSION}") + if (NOT GGML_OPENMP_COMPILER_VERSION_MAJOR STREQUAL GGML_OPENMP_LLVM_VERSION_MAJOR) + message(FATAL_ERROR "LLVM OpenMP ${GGML_OPENMP_LLVM_VERSION} requires Clang ${GGML_OPENMP_LLVM_VERSION_MAJOR}.x") + endif() + + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" GGML_OPENMP_SYSTEM_PROCESSOR) + if (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(amd64|x86_64)$") + set(GGML_OPENMP_ARCH "x64") + set(GGML_OPENMP_INSTALLER_SUFFIX "win64") + set(GGML_OPENMP_INSTALLER_SHA256 "3197846a2b19063687dd56e93e34cd941e3548d907f23a6131571321bdf9fe7b") + elseif (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + set(GGML_OPENMP_ARCH "arm64") + set(GGML_OPENMP_INSTALLER_SUFFIX "woa64") + set(GGML_OPENMP_INSTALLER_SHA256 "7c4ac97eb2ae6b960ca5f9caf3ff6124c8d2a18cc07a7840a4d2ea15537bad8e") + else() + message(FATAL_ERROR "GGML_OPENMP_FETCH does not support ${CMAKE_SYSTEM_PROCESSOR}") + endif() + + set(GGML_OPENMP_CACHE_DIR "${CMAKE_BINARY_DIR}/_deps") + set(GGML_OPENMP_ROOT "${GGML_OPENMP_CACHE_DIR}/llvm-openmp-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_ARCH}") + set(GGML_OPENMP_LIBRARY "${GGML_OPENMP_ROOT}/lib/libomp.lib") + set(GGML_OPENMP_RUNTIME "${GGML_OPENMP_ROOT}/bin/libomp.dll") + set(GGML_OPENMP_HEADER "${GGML_OPENMP_ROOT}/include/omp.h") + set(GGML_OPENMP_LICENSE "${GGML_OPENMP_ROOT}/LICENSE.TXT") + set(GGML_OPENMP_LICENSE_SHA256 "fdad1758a9e1f9d5a81e18879b3406772115edc92c24bfa36b70c654f325e8e4") + + if (NOT EXISTS "${GGML_OPENMP_LIBRARY}" OR NOT EXISTS "${GGML_OPENMP_RUNTIME}" OR NOT EXISTS "${GGML_OPENMP_HEADER}") + find_program(GGML_OPENMP_7Z NAMES 7z 7zz 7za) + if (NOT GGML_OPENMP_7Z) + message(FATAL_ERROR "GGML_OPENMP_FETCH requires 7-Zip to extract the LLVM installer") + endif() + + set(GGML_OPENMP_INSTALLER "${GGML_OPENMP_ROOT}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe") + set(GGML_OPENMP_EXTRACT_DIR "${GGML_OPENMP_ROOT}/extract") + set(GGML_OPENMP_INSTALLER_URL "https://github.com/llvm/llvm-project/releases/download/llvmorg-${GGML_OPENMP_LLVM_VERSION}/LLVM-${GGML_OPENMP_LLVM_VERSION}-${GGML_OPENMP_INSTALLER_SUFFIX}.exe") + + file(MAKE_DIRECTORY "${GGML_OPENMP_EXTRACT_DIR}") + file(DOWNLOAD "${GGML_OPENMP_INSTALLER_URL}" "${GGML_OPENMP_INSTALLER}" + EXPECTED_HASH "SHA256=${GGML_OPENMP_INSTALLER_SHA256}" + SHOW_PROGRESS + STATUS GGML_OPENMP_DOWNLOAD_STATUS) + list(GET GGML_OPENMP_DOWNLOAD_STATUS 0 GGML_OPENMP_DOWNLOAD_RESULT) + if (NOT GGML_OPENMP_DOWNLOAD_RESULT EQUAL 0) + list(GET GGML_OPENMP_DOWNLOAD_STATUS 1 GGML_OPENMP_DOWNLOAD_ERROR) + message(FATAL_ERROR "Failed to download LLVM OpenMP: ${GGML_OPENMP_DOWNLOAD_ERROR}") + endif() + + execute_process( + COMMAND "${GGML_OPENMP_7Z}" e -y "-o${GGML_OPENMP_EXTRACT_DIR}" "${GGML_OPENMP_INSTALLER}" -r libomp.lib libomp.dll omp.h + RESULT_VARIABLE GGML_OPENMP_EXTRACT_RESULT + OUTPUT_QUIET) + if (NOT GGML_OPENMP_EXTRACT_RESULT EQUAL 0 OR + NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" OR + NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" OR + NOT EXISTS "${GGML_OPENMP_EXTRACT_DIR}/omp.h") + message(FATAL_ERROR "Failed to extract libomp from ${GGML_OPENMP_INSTALLER}") + endif() + + file(MAKE_DIRECTORY "${GGML_OPENMP_ROOT}/lib" "${GGML_OPENMP_ROOT}/bin" "${GGML_OPENMP_ROOT}/include") + file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.lib" DESTINATION "${GGML_OPENMP_ROOT}/lib") + file(COPY "${GGML_OPENMP_EXTRACT_DIR}/libomp.dll" DESTINATION "${GGML_OPENMP_ROOT}/bin") + file(COPY "${GGML_OPENMP_EXTRACT_DIR}/omp.h" DESTINATION "${GGML_OPENMP_ROOT}/include") + file(REMOVE_RECURSE "${GGML_OPENMP_INSTALLER}" "${GGML_OPENMP_EXTRACT_DIR}") + endif() + + # The NSIS installer embeds LLVM's general license in its UI but does not install it as a file; use OpenMP's license to include its additional notices. + if (EXISTS "${GGML_OPENMP_LICENSE}") + file(SHA256 "${GGML_OPENMP_LICENSE}" GGML_OPENMP_LICENSE_ACTUAL_SHA256) + endif() + if (NOT GGML_OPENMP_LICENSE_ACTUAL_SHA256 STREQUAL GGML_OPENMP_LICENSE_SHA256) + file(DOWNLOAD "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-${GGML_OPENMP_LLVM_VERSION}/openmp/LICENSE.TXT" "${GGML_OPENMP_LICENSE}" + EXPECTED_HASH "SHA256=${GGML_OPENMP_LICENSE_SHA256}") + endif() + + if (COMMAND license_add_file) + license_add_file("LLVM OpenMP" "${GGML_OPENMP_LICENSE}") + endif() + + add_library(ggml-openmp-c INTERFACE) + target_compile_options(ggml-openmp-c INTERFACE "$<$:-fopenmp=libomp>") + target_include_directories(ggml-openmp-c SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include") + target_link_libraries(ggml-openmp-c INTERFACE "${GGML_OPENMP_LIBRARY}") + + add_library(ggml-openmp-cxx INTERFACE) + target_compile_options(ggml-openmp-cxx INTERFACE "$<$:-fopenmp=libomp>") + target_include_directories(ggml-openmp-cxx SYSTEM INTERFACE "${GGML_OPENMP_ROOT}/include") + target_link_libraries(ggml-openmp-cxx INTERFACE "${GGML_OPENMP_LIBRARY}") + + set(GGML_OPENMP_RUNTIME_OUTPUT_DIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}") + if (CMAKE_CONFIGURATION_TYPES) + string(APPEND GGML_OPENMP_RUNTIME_OUTPUT_DIR "/$") + endif() + add_custom_target(ggml-openmp-runtime ALL + COMMAND ${CMAKE_COMMAND} -E make_directory "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_RUNTIME}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/libomp.dll" + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${GGML_OPENMP_LICENSE}" "${GGML_OPENMP_RUNTIME_OUTPUT_DIR}/LICENSE-LLVM-OpenMP") + add_dependencies(ggml-base ggml-openmp-runtime) + install(FILES "${GGML_OPENMP_RUNTIME}" DESTINATION ${CMAKE_INSTALL_BINDIR}) + install(FILES "${GGML_OPENMP_LICENSE}" DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME LICENSE-LLVM-OpenMP) + + set(GGML_OPENMP_TARGET_C ggml-openmp-c) + set(GGML_OPENMP_TARGET_CXX ggml-openmp-cxx) + set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "") +elseif (GGML_OPENMP) + find_package(OpenMP) + if (OpenMP_FOUND) + set(GGML_OPENMP_TARGET_C OpenMP::OpenMP_C) + set(GGML_OPENMP_TARGET_CXX OpenMP::OpenMP_CXX) + set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "") + else() + set(GGML_OPENMP_ENABLED "OFF" CACHE INTERNAL "") + message(WARNING "OpenMP not found") + endif() +else() + set(GGML_OPENMP_ENABLED "OFF" CACHE INTERNAL "") +endif() + +if (GGML_OPENMP_ENABLED) + target_compile_definitions(ggml-base PRIVATE GGML_USE_OPENMP) + target_link_libraries(ggml-base PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX}) +endif() + add_library(ggml ggml-backend-dl.cpp ggml-backend-reg.cpp) @@ -329,6 +462,8 @@ function(ggml_add_cpu_backend_variant tag_name) set(GGML_INTERNAL_${feat} ON) endforeach() elseif (GGML_SYSTEM_ARCH STREQUAL "s390x") + set(GGML_NATIVE OFF) + foreach (feat VXE2 NNPA) set(GGML_INTERNAL_${feat} OFF) endforeach() @@ -413,7 +548,7 @@ if (GGML_CPU_ALL_VARIANTS) message(FATAL_ERROR "Unsupported ARM target OS: ${CMAKE_SYSTEM_NAME}") endif() elseif (GGML_SYSTEM_ARCH STREQUAL "PowerPC") - if (CMAKE_SYSTEM_NAME MATCHES "Linux") + if (CMAKE_SYSTEM_NAME MATCHES "Linux|AIX") ggml_add_cpu_backend_variant(power0) ggml_add_cpu_backend_variant(power7_1 POWER7) ggml_add_cpu_backend_variant(power7_2 POWER7 VSX) @@ -421,7 +556,14 @@ if (GGML_CPU_ALL_VARIANTS) ggml_add_cpu_backend_variant(power8_2 POWER8 VSX) ggml_add_cpu_backend_variant(power9 POWER9 VSX) ggml_add_cpu_backend_variant(power10 POWER10 VSX) - ggml_add_cpu_backend_variant(power11 POWER11 VSX) + # POWER11 backend: only if compiler supports -mcpu=power11 + check_cxx_compiler_flag("-mcpu=power11" GGML_CXX_SUPPORTS_POWER11) + if (GGML_CXX_SUPPORTS_POWER11) + message(STATUS "Compiler supports -mcpu=power11, enabling POWER11 backend") + ggml_add_cpu_backend_variant(power11 POWER11 VSX) + else() + message(STATUS "Skipping POWER11 backend: compiler does not support -mcpu=power11") + endif() else() message(FATAL_ERROR "Unsupported PowerPC target OS: ${CMAKE_SYSTEM_NAME}") endif() @@ -429,6 +571,14 @@ if (GGML_CPU_ALL_VARIANTS) if (CMAKE_SYSTEM_NAME MATCHES "Linux") ggml_add_cpu_backend_variant(z15 Z15 VXE2) ggml_add_cpu_backend_variant(z16 Z16 VXE2 NNPA) + + # check if compiler supports "-march=z17" codename + check_cxx_compiler_flag("-march=arch15" GGML_CXX_SUPPORTS_Z17) + if (GGML_CXX_SUPPORTS_Z17) + ggml_add_cpu_backend_variant(arch15 Z17 VXE2 NNPA) + else() + message(WARNING "Skipping z17 target: compiler must be GCC 15.1 and later") + endif() else() message(FATAL_ERROR "Unsupported s390x target OS: ${CMAKE_SYSTEM_NAME}") endif() @@ -449,6 +599,7 @@ endif() ggml_add_backend(BLAS) ggml_add_backend(CANN) ggml_add_backend(CUDA) +ggml_add_backend(ET) ggml_add_backend(HIP) ggml_add_backend(METAL) ggml_add_backend(MUSA) @@ -470,11 +621,10 @@ endforeach() target_link_libraries(ggml-base PRIVATE Threads::Threads) -find_library(MATH_LIBRARY m) -if (MATH_LIBRARY) - if (NOT WIN32 OR NOT DEFINED ENV{ONEAPI_ROOT}) - target_link_libraries(ggml-base PRIVATE m) - endif() +if (DEFINED MATH_LIBRARY) + target_link_libraries(ggml-base PRIVATE ${MATH_LIBRARY}) +elseif (NOT WIN32 AND NOT DEFINED ENV{ONEAPI_ROOT}) + target_link_libraries(ggml-base PRIVATE m) endif() if (CMAKE_SYSTEM_NAME MATCHES "Android") @@ -487,7 +637,11 @@ endif() if (BUILD_SHARED_LIBS) foreach (target ggml-base ggml) - set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON) + set_target_properties(${target} PROPERTIES + POSITION_INDEPENDENT_CODE ON + C_VISIBILITY_PRESET hidden + CXX_VISIBILITY_PRESET hidden + ) target_compile_definitions(${target} PRIVATE GGML_BUILD) target_compile_definitions(${target} PUBLIC GGML_SHARED) endforeach() diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c index a4b01cc..b0c067b 100644 --- a/ggml/src/ggml-alloc.c +++ b/ggml/src/ggml-alloc.c @@ -40,6 +40,7 @@ bool ggml_op_can_inplace(enum ggml_op op) { case GGML_OP_SILU_BACK: case GGML_OP_RMS_NORM: case GGML_OP_RMS_NORM_BACK: + case GGML_OP_CLAMP: case GGML_OP_SOFT_MAX: case GGML_OP_SOFT_MAX_BACK: return true; @@ -150,7 +151,7 @@ static void ggml_dyn_tallocr_insert_block(struct tallocr_chunk * chunk, size_t o static void ggml_dyn_tallocr_remove_block(struct tallocr_chunk * chunk, int idx) { // shift all elements after idx by 1 to the left, overwriting the element at idx - for (int i = idx; i < chunk->n_free_blocks; i++) { + for (int i = idx; i < chunk->n_free_blocks - 1; i++) { chunk->free_blocks[i] = chunk->free_blocks[i+1]; } chunk->n_free_blocks--; @@ -1116,131 +1117,55 @@ size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) { // utils -static void free_buffers(ggml_backend_buffer_t ** buffers, const size_t * n_buffers) { - for (size_t i = 0; i < *n_buffers; i++) { - ggml_backend_buffer_free((*buffers)[i]); +static struct ggml_tensor ** ggml_backend_alloc_ctx_tensors_from_buft_collect( + struct ggml_context * ctx, int * n_tensors) { + int n = 0; + for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + n++; } - free(*buffers); -} - -static bool alloc_tensor_range(struct ggml_context * ctx, - struct ggml_tensor * first, struct ggml_tensor * last, - ggml_backend_buffer_type_t buft, size_t size, - ggml_backend_buffer_t ** buffers, size_t * n_buffers) { - - ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size); - if (buffer == NULL) { - GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), size); - free_buffers(buffers, n_buffers); - return false; + *n_tensors = n; + if (n == 0) { + return NULL; } - *buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1)); - (*buffers)[(*n_buffers)++] = buffer; - - struct ggml_tallocr tallocr = ggml_tallocr_new(buffer); - - for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) { - enum ggml_status status = GGML_STATUS_SUCCESS; - if (t->data == NULL) { - if (t->view_src == NULL) { - status = ggml_tallocr_alloc(&tallocr, t); - } else if (t->buffer == NULL) { - status = ggml_backend_view_init(t); - } - } else { - if (t->view_src != NULL && t->buffer == NULL) { - // view of a pre-allocated tensor - status = ggml_backend_view_init(t); - } - } - if (status != GGML_STATUS_SUCCESS) { - GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t->name); - free_buffers(buffers, n_buffers); - return false; - } + struct ggml_tensor ** tensors = (struct ggml_tensor **) malloc(n * sizeof(struct ggml_tensor *)); + if (tensors == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, n * sizeof(struct ggml_tensor *)); + return NULL; } - - return true; + int i = 0; + for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + tensors[i++] = t; + } + return tensors; } -static ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft_impl( - struct ggml_context * ctx, ggml_backend_buffer_type_t buft, size_t * nbytes_total, bool no_alloc) { +ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) { GGML_ASSERT(ggml_get_no_alloc(ctx) == true); - size_t alignment = ggml_backend_buft_get_alignment(buft); - size_t max_size = ggml_backend_buft_get_max_size(buft); - - ggml_backend_buffer_t * buffers = NULL; - size_t n_buffers = 0; - *nbytes_total = 0; - - size_t cur_buf_size = 0; - struct ggml_tensor * first = ggml_get_first_tensor(ctx); - for (struct ggml_tensor * t = first; t != NULL; t = ggml_get_next_tensor(ctx, t)) { - size_t this_size = 0; - if (t->data == NULL && t->view_src == NULL) { - this_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment); - } - - if (cur_buf_size > 0 && (cur_buf_size + this_size) > max_size) { - // allocate tensors in the current buffer - if (!no_alloc && !alloc_tensor_range(ctx, first, t, buft, cur_buf_size, &buffers, &n_buffers)) { - return NULL; - } - first = t; - *nbytes_total += cur_buf_size; - cur_buf_size = this_size; - } else { - cur_buf_size += this_size; - } - } - - // allocate remaining tensors - if (cur_buf_size > 0) { - *nbytes_total += cur_buf_size; - if (!no_alloc && !alloc_tensor_range(ctx, first, NULL, buft, cur_buf_size, &buffers, &n_buffers)) { - return NULL; - } - } - - if (no_alloc) { - return NULL; - } - - if (n_buffers == 0) { -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: all tensors in the context are already allocated\n", __func__); -#endif - GGML_ASSERT(!buffers); + int n_tensors = 0; + struct ggml_tensor ** tensors = ggml_backend_alloc_ctx_tensors_from_buft_collect(ctx, &n_tensors); + if (tensors == NULL) { return NULL; } - ggml_backend_buffer_t buffer; - if (n_buffers == 1) { - buffer = buffers[0]; - } else { - buffer = ggml_backend_multi_buffer_alloc_buffer(buffers, n_buffers); - } - if (buffers) { - free(buffers); // can be NULL if context is empty or no_alloc - } + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer_n(buft, tensors, n_tensors); + free(tensors); return buffer; } size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) { - size_t nbytes_total = 0; - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft_impl(ctx, buft, &nbytes_total, /*no_alloc=*/ true); - GGML_ASSERT(!buf); - return nbytes_total; -} + GGML_ASSERT(ggml_get_no_alloc(ctx) == true); -ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) { - size_t nbytes_total = 0; - if (ggml_backend_buft_is_meta(buft)) { - return ggml_backend_meta_alloc_ctx_tensors_from_buft(ctx, buft); + int n_tensors = 0; + struct ggml_tensor ** tensors = ggml_backend_alloc_ctx_tensors_from_buft_collect(ctx, &n_tensors); + if (tensors == NULL) { + return 0; } - return ggml_backend_alloc_ctx_tensors_from_buft_impl(ctx, buft, &nbytes_total, /*no_alloc =*/ false); + + size_t nbytes_total = ggml_backend_buft_get_alloc_size_n(buft, tensors, n_tensors); + free(tensors); + return nbytes_total; } ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend) { diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index 9c56ec3..dc7367d 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -8,24 +8,28 @@ extern "C" { #endif - #define GGML_BACKEND_API_VERSION 2 + #define GGML_BACKEND_API_VERSION 3 // // Backend buffer type // struct ggml_backend_buffer_type_i { - const char * (*get_name) (ggml_backend_buffer_type_t buft); + const char * (*get_name) (ggml_backend_buffer_type_t buft); // allocate a buffer of this type - ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size); + ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size); + // (optional) allocate tensors from a list into a buffer of this type (defaults to alloc_buffer + linear allocator) + ggml_backend_buffer_t (*alloc_buffer_n) (ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors); // tensor alignment - size_t (*get_alignment) (ggml_backend_buffer_type_t buft); + size_t (*get_alignment) (ggml_backend_buffer_type_t buft); // (optional) max buffer size that can be allocated (defaults to SIZE_MAX) - size_t (*get_max_size) (ggml_backend_buffer_type_t buft); + size_t (*get_max_size) (ggml_backend_buffer_type_t buft); // (optional) data size needed to allocate the tensor, including padding (defaults to ggml_nbytes) - size_t (*get_alloc_size)(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); + size_t (*get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); + // (optional) total data size needed to allocate the given tensors, including padding and splitting (defaults to per-tensor get_alloc_size) + size_t (*get_alloc_size_n)(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors); // (optional) check if tensor data is in host memory and uses standard ggml tensor layout (defaults to false) - bool (*is_host) (ggml_backend_buffer_type_t buft); + bool (*is_host) (ggml_backend_buffer_type_t buft); }; struct ggml_backend_buffer_type { @@ -34,6 +38,11 @@ extern "C" { void * context; }; + // [TAG_ALLOC_SIZE_EXPAND] + // returns true for ops that may require additional memory for fleeting data on some backends, + // i.e. the backend buffer type's get_alloc_size may return more than ggml_nbytes for the output tensor + GGML_API bool ggml_op_alloc_size_may_expand(enum ggml_op op); + // // Backend buffer // @@ -83,6 +92,7 @@ extern "C" { GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers); GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer); GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); + GGML_API void ggml_backend_meta_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); // // Backend (meta) @@ -95,13 +105,20 @@ extern "C" { GGML_API size_t ggml_backend_meta_n_backends (ggml_backend_t meta_backend); GGML_API ggml_backend_t ggml_backend_meta_simple_backend(ggml_backend_t meta_backend, size_t index); - // temporary workaround to statically allocate tensors from a context in a deduplicated way: - GGML_API struct ggml_backend_buffer * ggml_backend_meta_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); - // // Backend (stream) // + // passed to graph_optimize so the backend can add allocation dependencies: + // if the backend executes parts of the graph out of order (e.g. on concurrent streams), + // it must keep the affected tensors allocated until a node where execution is known to have joined + struct ggml_backend_graph_optimize_params { + // keep `tensor` allocated at least until `until` (a node of the same graph) has been computed + // can be called multiple times for the same tensor: the longest lifetime applies + void (*add_alloc_dep)(void * user_data, struct ggml_tensor * tensor, struct ggml_tensor * until); + void * user_data; + }; + struct ggml_backend_i { const char * (*get_name)(ggml_backend_t backend); @@ -136,7 +153,7 @@ extern "C" { void (*event_wait) (ggml_backend_t backend, ggml_backend_event_t event); // (optional) sort/optimize the nodes in the graph - void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph); + void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params); }; struct ggml_backend { diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 1ee3eeb..fb35be3 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -131,6 +132,7 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back /* .host_buffer = */ false, // Not implemented. /* .buffer_from_host_ptr = */ false, // Not implemented. /* .events = */ false, // Not implemented. + /* .mmap_support = */ true, }; for (ggml_backend_dev_t simple_dev : meta_dev_ctx->simple_devs) { ggml_backend_dev_props tmp_props; @@ -139,6 +141,7 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back props->caps.host_buffer = props->caps.host_buffer && tmp_props.caps.host_buffer; props->caps.buffer_from_host_ptr = props->caps.buffer_from_host_ptr && tmp_props.caps.buffer_from_host_ptr; props->caps.events = props->caps.events && tmp_props.caps.events; + props->caps.mmap_support = props->caps.mmap_support && tmp_props.caps.mmap_support; } } @@ -287,6 +290,8 @@ static ggml_backend_buffer_type_t ggml_backend_meta_buft_simple_buft(ggml_backen static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size); +static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors); + static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft); size_t max_alignment = 1; @@ -328,12 +333,14 @@ static bool ggml_backend_meta_buffer_type_is_host(ggml_backend_buffer_type_t buf } static const struct ggml_backend_buffer_type_i ggml_backend_meta_buffer_type_iface = { - /* .get_name = */ ggml_backend_meta_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_meta_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_meta_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_meta_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_meta_buffer_type_get_alloc_size, - /* .is_host = */ ggml_backend_meta_buffer_type_is_host, + /* .get_name = */ ggml_backend_meta_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_meta_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ ggml_backend_meta_buffer_type_alloc_buffer_n, + /* .get_alignment = */ ggml_backend_meta_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_meta_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_meta_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_meta_buffer_type_is_host, }; bool ggml_backend_buft_is_meta(ggml_backend_buffer_type_t buft) { @@ -392,64 +399,103 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_host_buffer_type( // meta backend buffer // +// Container to hold the tensor slices per simple ggml backend buffer. +struct ggml_backend_meta_simple_tensor_container { + std::vector ctxs; + std::map> simple_tensors; + + ggml_backend_meta_simple_tensor_container(const ggml_init_params & params, const int n_simple) { + ctxs.reserve(n_simple); + for (int i = 0; i < n_simple; i++) { + ctxs.emplace_back(ggml_init(params)); + } + } + ggml_backend_meta_simple_tensor_container() {} +}; + struct ggml_backend_meta_buffer_context { + // FIXME + // Most tensors can simply be stored statically in their own buffer. + // Externally created views however also need a mapping to simple tensors but they use the buffer of the view source. + // If external views are simply using that buffer they will slowly deplete its memory. + // Current solution: rotating set of 2 "compute" containers to hold external views, works correctly for llama.cpp. + // Long-term: tie the lifetime of external views to the meta backend executing the graph instead, + // currently not possible due to graph-external operations in the backend scheduler. + ggml_backend_meta_simple_tensor_container stc_static; + ggml_backend_meta_simple_tensor_container stc_compute[2]; + int stc_compute_index = 0; + int stc_compute_index_next = 0; + std::vector bufs; + + // FIXME + // The size of the split state cache is unbounded and can theoretically grow infinitely large. + // However, it is also expensive to build and clearing it on every rebuild in ggml_backend_meta_graph_compute is too expensive. static constexpr size_t nbtc = GGML_TENSOR_SIZE - sizeof(ggml_tensor::padding); - std::map, std::pair> split_state_cache; - std::map< const ggml_tensor *, std::vector> simple_tensors; - - struct buffer_config { - ggml_context * ctx; - ggml_backend_buffer_t buf; - - buffer_config(ggml_context * ctx, ggml_backend_buffer_t buf) : ctx(ctx), buf(buf) {} - }; - std::vector buf_configs; int debug; - ggml_backend_meta_buffer_context() { + ggml_backend_meta_buffer_context( + ggml_backend_meta_simple_tensor_container & stc_static, + ggml_backend_meta_simple_tensor_container & stc_compute_0, + ggml_backend_meta_simple_tensor_container & stc_compute_1, + const std::vector & bufs) + : stc_static(std::move(stc_static)), stc_compute{std::move(stc_compute_0), std::move(stc_compute_1)} { + this->bufs.reserve(bufs.size()); + for (ggml_backend_buffer_t buf : bufs) { + this->bufs.emplace_back(buf); + } const char * GGML_META_DEBUG = getenv("GGML_META_DEBUG"); debug = GGML_META_DEBUG ? atoi(GGML_META_DEBUG) : 0; } + + ggml_backend_meta_simple_tensor_container & get_simple_tensor_container(const ggml_tensor * tensor) { + if (stc_static.simple_tensors.find(tensor) != stc_static.simple_tensors.end()) { + return stc_static; + } + return stc_compute[stc_compute_index]; + } }; static void ggml_backend_meta_buffer_free_buffer(ggml_backend_buffer_t buffer) { GGML_ASSERT(ggml_backend_buffer_is_meta(buffer)); ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context; - for (auto & [ctx, buf] : buf_ctx->buf_configs) { - ggml_backend_buffer_free(buf); - ggml_free(ctx); - } delete buf_ctx; } static size_t ggml_backend_meta_buffer_n_bufs(ggml_backend_buffer_t meta_buf) { GGML_ASSERT(ggml_backend_buffer_is_meta(meta_buf)); ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) meta_buf->context; - return buf_ctx->buf_configs.size(); + return buf_ctx->bufs.size(); } static ggml_backend_buffer_t ggml_backend_meta_buffer_simple_buffer(ggml_backend_buffer_t meta_buf, size_t index) { GGML_ASSERT(ggml_backend_buffer_is_meta(meta_buf)); ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) meta_buf->context; - GGML_ASSERT(index < buf_ctx->buf_configs.size()); - return buf_ctx->buf_configs[index].buf; + GGML_ASSERT(index < buf_ctx->bufs.size()); + return buf_ctx->bufs[index].get(); } static struct ggml_tensor * ggml_backend_meta_buffer_simple_tensor(const struct ggml_tensor * tensor, size_t index) { GGML_ASSERT(ggml_backend_buffer_is_meta(tensor->buffer)); ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context; - GGML_ASSERT(index < buf_ctx->buf_configs.size()); + GGML_ASSERT(index < buf_ctx->bufs.size()); - auto it = buf_ctx->simple_tensors.find(tensor); - if (it == buf_ctx->simple_tensors.end()) { + ggml_backend_meta_simple_tensor_container & stc = buf_ctx->get_simple_tensor_container(tensor); + auto it = stc.simple_tensors.find(tensor); + if (it == stc.simple_tensors.end()) { return nullptr; } return it->second[index]; } -static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync) { +static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync); + +static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( + ggml_backend_meta_simple_tensor_container & stc, const struct ggml_tensor * tensor, bool assume_sync) { + // FIXME Currently this function preserves/erases the information in n_segments and nr in an inconsistent way. + // Since the operations in question are developed specifically for llama.cpp this currently does not manifest as a bug there. + // However, in a broader ggml context with arbitrary ggml graphs this can lead to unexpected results. const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(tensor->buffer); ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context; @@ -460,11 +506,11 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co for (size_t j = 0; j < n_bufs; j++) { int64_t sum_a = 0; for (size_t s = 0; s < a.n_segments; s++) { - sum_a += a.ne[s*n_bufs + j]; + sum_a += a.ne[s*n_bufs + j] * a.nr[s]; } int64_t sum_b = 0; for (size_t s = 0; s < b.n_segments; s++) { - sum_b += b.ne[s*n_bufs + j]; + sum_b += b.ne[s*n_bufs + j] * b.nr[s]; } if (sum_a != sum_b) { return false; @@ -474,7 +520,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co }; auto handle_generic = [&](const std::vector & src_ss, bool scalar_only) -> ggml_backend_meta_split_state { - ggml_backend_meta_split_state ret = {GGML_BACKEND_SPLIT_AXIS_NONE, {0}, 1}; + ggml_backend_meta_split_state ret = {GGML_BACKEND_SPLIT_AXIS_NONE, {0}, {1}, 1}; for (size_t i = 0; i < GGML_MAX_SRC; i++) { if (tensor->src[i] == nullptr || tensor->src[i] == tensor) { continue; @@ -482,15 +528,15 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co if (ret.axis == GGML_BACKEND_SPLIT_AXIS_NONE) { ret = src_ss[i]; } else if (!split_states_equal(src_ss[i], ret)) { - ret = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + ret = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; break; } } if (ret.axis == GGML_BACKEND_SPLIT_AXIS_NONE) { - ret = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + ret = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; } if (scalar_only && ret.axis >= 0 && ret.axis < GGML_MAX_DIMS) { - ret = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + ret = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; } GGML_ASSERT(ret.axis != GGML_BACKEND_SPLIT_AXIS_UNKNOWN); return ret; @@ -534,42 +580,35 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co auto handle_mul_mat = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { - return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, 1}; + return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; } if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_1 && src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { ggml_backend_meta_split_state ret = src_ss[0]; ret.axis = GGML_BACKEND_SPLIT_AXIS_0; + ret.nr[0] = 1; ret.n_segments = 1; return ret; } if (src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_1 && src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { - ggml_backend_meta_split_state ret = src_ss[1]; - ret.n_segments = 1; - return ret; + return src_ss[1]; } if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_0 && src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_0) { GGML_ASSERT(split_states_equal(src_ss[0], src_ss[1])); - return {assume_sync ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_PARTIAL, {0}, 1}; + return {assume_sync ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_PARTIAL, {0}, {1}, 1}; } - GGML_ABORT("fatal error"); - //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; - }; - - auto handle_cpy = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { - if (src_ss[0].axis >= 0 && src_ss[0].axis < GGML_MAX_DIMS) { - int64_t ne_split_src = tensor->src[0]->ne[0]; - for (int dim = 1; dim <= src_ss[0].axis; dim++) { - ne_split_src *= tensor->src[0]->ne[dim]; - } - int64_t ne_split_dst = 1; - for (int dim = 0; dim < GGML_MAX_DIMS; dim++) { - ne_split_dst *= tensor->ne[dim]; - if (ne_split_dst == ne_split_src) { - return {ggml_backend_meta_split_axis(dim), {0}, 1}; - } - } + if (src_ss[0].axis == src_ss[1].axis && src_ss[0].axis >= GGML_BACKEND_SPLIT_AXIS_2 && + src_ss[0].axis < GGML_MAX_DIMS) { + GGML_ASSERT(split_states_equal(src_ss[0], src_ss[1])); + return src_ss[0]; } - return handle_generic(src_ss, /*scalar_only =*/ false); + // batched matmul with the batches split across devices and a replicated activation + if (src_ss[0].axis >= GGML_BACKEND_SPLIT_AXIS_2 && src_ss[0].axis < GGML_MAX_DIMS && + src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { + return src_ss[0]; + } + GGML_ABORT("unsupported mul_mat split states: node=%s src0=%s axis=%d src1=%s axis=%d", + tensor->name, tensor->src[0]->name, (int) src_ss[0].axis, tensor->src[1]->name, (int) src_ss[1].axis); + //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; }; auto handle_reshape = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { @@ -578,35 +617,40 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co case GGML_BACKEND_SPLIT_AXIS_1: case GGML_BACKEND_SPLIT_AXIS_2: case GGML_BACKEND_SPLIT_AXIS_3: { - GGML_ASSERT(!ggml_is_permuted(tensor) && !ggml_is_permuted(tensor->src[0])); - if (src_ss[0].axis == ggml_n_dims(tensor->src[0]) - 1) { - return {ggml_backend_meta_split_axis(ggml_n_dims(tensor) - 1), {0}, 1}; + int64_t base_ne_in = 1; + for (int dim = 0; dim <= src_ss[0].axis; dim++) { + base_ne_in *= tensor->src[0]->ne[dim]; } - std::vector base_ne_in; - base_ne_in.reserve(GGML_MAX_DIMS - src_ss[0].axis); - { - base_ne_in.push_back(1); - int dim = 0; - for (; dim <= src_ss[0].axis; dim++) { - base_ne_in[0] *= tensor->src[0]->ne[dim]; + if (src_ss[0].n_segments == 1) { + base_ne_in /= src_ss[0].nr[0]; + if (src_ss[0].axis == ggml_n_dims(tensor->src[0]) - 1 && src_ss[0].nr[0] == 1) { + return {ggml_backend_meta_split_axis(ggml_n_dims(tensor) - 1), {0}, {1}, 1}; } - for (; dim <= GGML_MAX_DIMS; dim++) { - base_ne_in.push_back(base_ne_in.back() * tensor->src[0]->ne[dim]); + if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_0 && tensor->ne[0] == tensor->src[0]->ne[0] && + tensor->ne[1] == 1 && src_ss[0].nr[0] == 1) { + bool complete_rows = true; + for (size_t j = 0; j < n_bufs; j++) { + const int64_t ne = src_ss[0].ne[j]; + complete_rows = complete_rows && (ne == 0 || ne == tensor->src[0]->ne[0]); + } + if (complete_rows) { + // Move a complete dim-0 split to the following singleton dimension. + return {GGML_BACKEND_SPLIT_AXIS_1, {0}, {1}, 1}; + } } } + // Reshape outputs use one segment; split-state propagation merges source segments. int64_t base_ne_out = 1; for (int dim = 0; dim < GGML_MAX_DIMS; dim++) { - const int64_t base_ne_out_next = base_ne_out *= tensor->ne[dim]; - for (const int64_t & bni : base_ne_in) { - if (bni == base_ne_out_next) { - return {ggml_backend_meta_split_axis(dim), {0}, 1}; - } + base_ne_out *= tensor->ne[dim]; + if (base_ne_out % base_ne_in == 0) { + return {ggml_backend_meta_split_axis(dim), {0}, {uint32_t(base_ne_out/base_ne_in)}, 1}; } - if (base_ne_out_next > base_ne_in[0]) { - GGML_ASSERT(dim + 1 < GGML_MAX_DIMS); - return {ggml_backend_meta_split_axis(dim + 1), {0}, 1}; + if (base_ne_out > base_ne_in) { + GGML_ASSERT(src_ss[0].n_segments == 1); + GGML_ASSERT(src_ss[0].nr[0] == 1); + return {ggml_backend_meta_split_axis(dim), {0}, {1}, 1}; } - base_ne_out = base_ne_out_next; } GGML_ABORT("shape mismatch for %s", ggml_op_name(tensor->op)); } @@ -616,11 +660,18 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co } default: { GGML_ABORT("fatal error"); - //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; } } }; + auto handle_cpy = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { + if (src_ss[0].axis >= 0 && src_ss[0].axis < GGML_MAX_DIMS) { + return handle_reshape(src_ss); + } + return handle_generic(src_ss, /*scalar_only =*/ false); + }; + auto handle_view = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { if (ggml_is_contiguous(tensor) && ggml_is_contiguous(tensor->src[0])) { return handle_reshape(src_ss); @@ -644,7 +695,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co if (!ggml_is_permuted(tensor) && !ggml_is_permuted(tensor->src[0]) && axis >= 0 && axis < GGML_MAX_DIMS-1) { for (int dim = 0; dim < GGML_MAX_DIMS-1; dim++) { if (tensor->nb[dim+1] == tensor->src[0]->nb[axis+1]) { - return {ggml_backend_meta_split_axis(dim), {0}, 1}; + return {ggml_backend_meta_split_axis(dim), {0}, {1}, 1}; } } GGML_ABORT("fatal error"); @@ -653,7 +704,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co return src_ss[0]; } GGML_ABORT("view of permuted tensor not implemented"); - //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; }; auto handle_permute = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { @@ -662,7 +713,8 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co case GGML_BACKEND_SPLIT_AXIS_1: case GGML_BACKEND_SPLIT_AXIS_2: case GGML_BACKEND_SPLIT_AXIS_3: { - return {ggml_backend_meta_split_axis(tensor->op_params[src_ss[0].axis]), {0}, 1}; + GGML_ASSERT(src_ss[0].n_segments == 1 || src_ss[0].nr[0] == 1); + return {ggml_backend_meta_split_axis(tensor->op_params[src_ss[0].axis]), {0}, {src_ss[0].nr[0]}, 1}; } case GGML_BACKEND_SPLIT_AXIS_MIRRORED: case GGML_BACKEND_SPLIT_AXIS_PARTIAL: { @@ -670,7 +722,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co } default: { GGML_ABORT("fatal error"); - //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; } } }; @@ -679,7 +731,8 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co switch (src_ss[0].axis) { case GGML_BACKEND_SPLIT_AXIS_0: case GGML_BACKEND_SPLIT_AXIS_1: { - return {ggml_backend_meta_split_axis(int(src_ss[0].axis) ^ 1), {0}, 1}; + GGML_ASSERT(src_ss[0].n_segments == 1 || src_ss[0].nr[0] == 1); + return {ggml_backend_meta_split_axis(int(src_ss[0].axis) ^ 1), {0}, {src_ss[0].nr[0]}, 1}; } case GGML_BACKEND_SPLIT_AXIS_2: case GGML_BACKEND_SPLIT_AXIS_3: @@ -689,7 +742,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co } default: { GGML_ABORT("fatal error"); - //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; } } }; @@ -722,21 +775,40 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co }; auto handle_flash_attn_ext = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { - GGML_ASSERT( src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_2); - GGML_ASSERT( src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_2); - GGML_ASSERT( src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_2); - GGML_ASSERT(tensor->src[4] == nullptr || src_ss[3].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); + GGML_ASSERT(tensor->src[3] == nullptr || src_ss[3].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); + + if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { + GGML_ASSERT(src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); + GGML_ASSERT(src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); + GGML_ASSERT(tensor->src[4] == nullptr || src_ss[4].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); + return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; + } + + GGML_ASSERT(src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_2); + const bool kv_split = src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_2 && + src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_2; + const bool kv_mirrored = src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && + src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED; + GGML_ASSERT(kv_split || kv_mirrored); GGML_ASSERT(tensor->src[4] == nullptr || src_ss[4].axis == GGML_BACKEND_SPLIT_AXIS_0); - return {GGML_BACKEND_SPLIT_AXIS_1, {0}, 1}; + return {GGML_BACKEND_SPLIT_AXIS_1, {0}, {1}, 1}; + }; + + auto handle_lightning_indexer = [&]( + const std::vector & src_ss) -> ggml_backend_meta_split_state { + for (size_t i = 0; i < 4; i++) { + GGML_ASSERT(src_ss[i].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); + } + return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; }; auto handle_ssm_conv = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { if (src_ss[0].axis == src_ss[1].axis) { if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_0) { - return {GGML_BACKEND_SPLIT_AXIS_1, {0}, 1}; + return {GGML_BACKEND_SPLIT_AXIS_1, {0}, {1}, 1}; } if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_1) { - return {GGML_BACKEND_SPLIT_AXIS_0, {0}, 1}; + return {GGML_BACKEND_SPLIT_AXIS_0, {0}, {1}, 1}; } } return handle_generic(src_ss, /*scalar_only =*/ false); @@ -744,8 +816,8 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co auto handle_gated_delta_net = [&](const std::vector & src_ss) -> ggml_backend_meta_split_state { if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && src_ss[1].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && - src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && src_ss[3].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && - src_ss[4].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && src_ss[5].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { + src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && src_ss[3].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && + src_ss[4].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED && src_ss[5].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { return src_ss[0]; } GGML_ASSERT(src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_1); @@ -753,44 +825,56 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co GGML_ASSERT(src_ss[2].axis == GGML_BACKEND_SPLIT_AXIS_1); GGML_ASSERT(src_ss[3].axis == GGML_BACKEND_SPLIT_AXIS_1); GGML_ASSERT(src_ss[4].axis == GGML_BACKEND_SPLIT_AXIS_1); - GGML_ASSERT(src_ss[5].axis == GGML_BACKEND_SPLIT_AXIS_2); - return {GGML_BACKEND_SPLIT_AXIS_0, {0}, 1}; + // state shape is [S_v, S_v, H_v, n_seqs] (s0 only); the heads dim is its own axis 2, + // so a head-aligned split on the input cache lands on axis 2 here. + GGML_ASSERT(src_ss[5].axis == GGML_BACKEND_SPLIT_AXIS_2 || src_ss[5].axis == GGML_BACKEND_SPLIT_AXIS_1 || src_ss[5].axis == GGML_BACKEND_SPLIT_AXIS_0); + return {GGML_BACKEND_SPLIT_AXIS_0, {0}, {1}, 1}; }; auto calculate_split_state = [&]() -> ggml_backend_meta_split_state { if (ggml_nelements(tensor) == 0) { - return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; } if (ggml_backend_buffer_get_usage(tensor->buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE && tensor->view_src == nullptr) { ggml_backend_dev_t dev = ggml_backend_buft_get_device(ggml_backend_buffer_get_type(tensor->buffer)); const ggml_backend_meta_device_context * dev_ctx = (const ggml_backend_meta_device_context *) dev->context; ggml_backend_meta_split_state ret = dev_ctx->get_split_state(tensor, dev_ctx->get_split_state_ud); - if (ret.axis >= 0 && ret.axis <= GGML_MAX_DIMS) { + if (ret.axis >= 0 && ret.axis < GGML_MAX_DIMS) { const int64_t granularity = ret.axis == GGML_BACKEND_SPLIT_AXIS_0 ? ggml_blck_size(tensor->type) : 1; int64_t ne_sum = 0; - for (size_t sj = 0; sj < ret.n_segments*n_bufs; sj++) { - GGML_ASSERT(ret.ne[sj] % granularity == 0); - ne_sum += ret.ne[sj]; + for (size_t s = 0; s < ret.n_segments; s++) { + for (size_t j = 0; j < n_bufs; j++) { + GGML_ASSERT(ret.ne[s*n_bufs + j] % granularity == 0); + ne_sum += ret.ne[s*n_bufs + j] * ret.nr[s]; + } } GGML_ASSERT(ne_sum == tensor->ne[ret.axis]); + } else if (ret.axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) { + GGML_ASSERT(ret.n_segments == 1); + GGML_ASSERT(ret.nr[0] == 1); } return ret; } - std::vector src_ss(GGML_MAX_SRC, {GGML_BACKEND_SPLIT_AXIS_NONE, {0}, 1}); + std::vector src_ss(GGML_MAX_SRC, {GGML_BACKEND_SPLIT_AXIS_NONE, {0}, {1}, 1}); for (size_t i = 0; i < GGML_MAX_SRC; i++) { if (tensor->src[i] == nullptr || tensor->src[i] == tensor) { - src_ss[i] = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + src_ss[i] = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; continue; } - src_ss[i] = ggml_backend_meta_get_split_state(tensor->src[i], /*assume_sync =*/ true); + src_ss[i] = ggml_backend_meta_get_split_state(stc, tensor->src[i], /*assume_sync =*/ true); GGML_ASSERT(src_ss[i].axis != GGML_BACKEND_SPLIT_AXIS_UNKNOWN); } ggml_backend_meta_split_state split_state; switch (tensor->op) { case GGML_OP_NONE: { - split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, 1}; + if (tensor->view_src != nullptr) { + // full-tensor view created with ggml_view_tensor, transparent for the split state + split_state = ggml_backend_meta_get_split_state(stc, tensor->view_src, assume_sync); + } else { + split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; + } } break; case GGML_OP_DUP: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); @@ -893,7 +977,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co split_state = handle_rope(src_ss); } break; case GGML_OP_ROPE_BACK: { - split_state = handle_generic(src_ss, /*scalar_only =*/ true); + split_state = handle_rope(src_ss); } break; case GGML_OP_CLAMP: { split_state = handle_generic(src_ss, /*scalar_only =*/ false); @@ -957,6 +1041,15 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co case GGML_OP_GATED_DELTA_NET: { split_state = handle_gated_delta_net(src_ss); } break; + case GGML_OP_LIGHTNING_INDEXER: { + split_state = handle_lightning_indexer(src_ss); + } break; + case GGML_OP_DSV4_HC_COMB: + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + case GGML_OP_TOP_K_MASK: { + split_state = handle_generic(src_ss, /*scalar_only =*/ true); + } break; case GGML_OP_UNARY: { split_state = handle_generic(src_ss, /*scalar_only =*/ false); } break; @@ -977,7 +1070,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co } break; default: { GGML_ABORT("ggml op not implemented: %s", ggml_op_name(tensor->op)); - split_state = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, 1}; + split_state = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; } break; } if (split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS) { @@ -995,23 +1088,25 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co split_state.ne[s*n_bufs + j] = 0; } for (size_t s = 0; s < src_ss[i].n_segments; s++) { - split_state.ne[j] += src_ss[i].ne[s*n_bufs + j]; + split_state.ne[j] += src_ss[i].ne[s*n_bufs + j] * src_ss[i].nr[s]; } split_state.ne[j] *= tensor->ne[split_state.axis]; if (split_state.ne[j] != 0 || tensor->src[i]->ne[src_ss[i].axis] != 0) { - GGML_ASSERT(split_state.ne[j] % tensor->src[i]->ne[src_ss[i].axis] == 0); - split_state.ne[j] /= tensor->src[i]->ne[src_ss[i].axis]; + const int64_t div = tensor->src[i]->ne[src_ss[i].axis] * split_state.nr[0]; + GGML_ASSERT(split_state.ne[j] % div == 0); + split_state.ne[j] /= div; } } } else { + GGML_ASSERT(split_state.n_segments == 1); for (size_t j = 0; j < n_bufs; j++) { + // Assert that ratio is consistent: int64_t sum = 0; for (size_t s = 0; s < src_ss[i].n_segments; s++) { - sum += src_ss[i].ne[s*n_bufs + j]; + sum += src_ss[i].ne[s*n_bufs + j] * src_ss[i].nr[s]; } - // Assert that ratio is consistent: - GGML_ASSERT(split_state.ne[j] * tensor->src[i]->ne[src_ss[i].axis] - == sum * tensor->ne[split_state.axis]); + GGML_ASSERT(split_state.ne[j]*split_state.nr[0] * tensor->src[i]->ne[src_ss[i].axis] + == sum * tensor->ne[split_state.axis]); } } first_src_split_by_axis = false; @@ -1034,20 +1129,22 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co if (buf_ctx->debug > 0) { std::string srcs_info; for (size_t i = 0; i < GGML_MAX_SRC; i++) { - if (tensor->src[i] == nullptr) { + if (tensor->src[i] == nullptr || tensor->src[i] == tensor) { continue; } if (!srcs_info.empty()) { srcs_info += ", "; } - const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor->src[0], true); + const ggml_backend_meta_split_state split_state = + ggml_backend_meta_get_split_state(tensor->src[i], true); + GGML_ASSERT(split_state.n_segments == 1); const char * axis_name = ggml_backend_meta_split_axis_name(split_state.axis); std::string ne_info; for (size_t j = 0; j < n_bufs; j++) { if (!ne_info.empty()) { ne_info += ", "; } - ne_info += std::to_string(split_state.ne[j]); + ne_info += std::to_string(split_state.ne[j]) + "x" + std::to_string(split_state.nr[0]); } srcs_info += std::string(tensor->src[i]->name) + "[" + ggml_op_name(tensor->src[i]->op) + ", " + axis_name + ", {" + ne_info + "}]"; } @@ -1056,7 +1153,8 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co if (!ne_info.empty()) { ne_info += ", "; } - ne_info += std::to_string(buf_ctx->split_state_cache[key].first.ne[j]); + const ggml_backend_meta_split_state & ss = buf_ctx->split_state_cache[key].first; + ne_info += std::to_string(ss.ne[j]) + "x" + std::to_string(ss.nr[0]); } GGML_LOG_DEBUG("SPLIT_STATE: {%s} -> %s[%s, %s, {%s}]\n", srcs_info.c_str(), tensor->name, ggml_op_name(tensor->op), ggml_backend_meta_split_axis_name(buf_ctx->split_state_cache[key].first.axis), ne_info.c_str()); @@ -1068,8 +1166,10 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co #ifndef NDEBUG if (ret.axis >= 0 && ret.axis < GGML_MAX_DIMS) { int64_t ne_ret = 0; - for (size_t sj = 0; sj < ret.n_segments*n_bufs; sj++) { - ne_ret += ret.ne[sj]; + for (size_t s = 0; s < ret.n_segments; s++) { + for (size_t j = 0; j < n_bufs; j++) { + ne_ret += ret.ne[s*n_bufs + j] * ret.nr[s]; + } } assert(ne_ret == tensor->ne[int(ret.axis)]); } @@ -1077,17 +1177,37 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co return ret; } +static bool ggml_backend_meta_is_host_view(const struct ggml_tensor * tensor) { + return ggml_is_view(tensor) && ggml_backend_buffer_is_host(tensor->view_src->buffer); +} + +static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync) { + // [TAG_META_HOST_VIEWS] + // TODO: technically, this check should not be needed if the backend scheduler correctly prevents assigning + // such host-buffer views to the meta backend. figure out how to update the scheduler logic to achieve that + // ref: https://github.com/ggml-org/llama.cpp/pull/30217 + if (!ggml_backend_buffer_is_meta(tensor->buffer)) { + GGML_ASSERT(ggml_backend_meta_is_host_view(tensor)); + + // the view is not allocated in the meta buffer, it is not split across the sub-devices + return { GGML_BACKEND_SPLIT_AXIS_NONE, {0}, {1}, 1 }; + } + + ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context; + return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync); +} + static void * ggml_backend_meta_buffer_get_base(ggml_backend_buffer_t buffer) { GGML_UNUSED(buffer); return (void *) 0x1000000000000000; // FIXME } -static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - GGML_ASSERT(ggml_backend_buffer_is_meta(buffer)); - ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context; - const size_t n_simple_bufs = ggml_backend_meta_buffer_n_bufs(buffer); +static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_meta_simple_tensor_container & stc, ggml_tensor * tensor) { + GGML_ASSERT(ggml_backend_buffer_is_meta(tensor->buffer)); + ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context; + const size_t n_simple_bufs = ggml_backend_meta_buffer_n_bufs(tensor->buffer); - const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ true); + const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(stc, tensor, /*assume_sync =*/ true); GGML_ASSERT(ggml_nelements(tensor) == 0 || split_state.axis != GGML_BACKEND_SPLIT_AXIS_UNKNOWN); GGML_ASSERT(split_state.n_segments <= 16); @@ -1102,15 +1222,15 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer std::vector simple_tensors; simple_tensors.reserve(n_simple_bufs); for (size_t j = 0; j < n_simple_bufs; j++) { - ggml_context * simple_ctx = buf_ctx->buf_configs[j].ctx; - ggml_backend_buffer_t simple_buf = buf_ctx->buf_configs[j].buf; + ggml_context * simple_ctx = stc.ctxs[j].get(); + ggml_backend_buffer_t simple_buf = buf_ctx->bufs[j].get(); if (split_dim >= 0 && split_dim < GGML_MAX_DIMS) { // TODO: the following assert fails for llama-parallel even though the results are correct: // GGML_ASSERT(ggml_is_contiguously_allocated(tensor)); ne[split_dim] = 0; for (size_t s = 0; s < split_state.n_segments; s++) { - ne[split_dim] += split_state.ne[s*n_simple_bufs + j]; + ne[split_dim] += split_state.ne[s*n_simple_bufs + j] * split_state.nr[s]; } for (int i = 0; i < GGML_MAX_DIMS; i++) { if (tensor->nb[i] > tensor->nb[split_dim]) { @@ -1133,7 +1253,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer if (t_ij->view_src != nullptr && ggml_backend_buffer_is_meta(t_ij->view_src->buffer)) { t_ij->view_src = ggml_backend_meta_buffer_simple_tensor(tensor->view_src, j); if (t_ij->view_offs > 0 && split_dim >= 0 && split_dim < GGML_MAX_DIMS) { - GGML_ASSERT(ne[split_dim] != 0 && tensor->ne[split_dim] != 0); + GGML_ASSERT(tensor->ne[split_dim] != 0); const int split_dim_view_src = ggml_backend_meta_get_split_state(tensor->view_src, /*assume_sync =*/ true).axis; GGML_ASSERT(split_dim_view_src >= 0 && split_dim_view_src < GGML_MAX_DIMS); @@ -1152,13 +1272,31 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer } } } + // TODO: revisit once the graph allocator has been refactored, see https://github.com/ggml-org/llama.cpp/pull/25051#issuecomment-4842873396 + ggml_backend_buffer_t init_buf = simple_buf; if (t_ij->view_src != nullptr) { t_ij->data = (char *) t_ij->view_src->data + t_ij->view_offs; + // views inherit the source slice's concrete sub-buffer (issue 22197) + if (tensor->view_src != nullptr && ggml_backend_buffer_is_meta(tensor->view_src->buffer) + && t_ij->view_src->buffer != nullptr) { + t_ij->buffer = t_ij->view_src->buffer; + init_buf = t_ij->view_src->buffer; + } } else if (simple_buf != nullptr) { + if (ggml_backend_buffer_is_multi_buffer(simple_buf)) { + GGML_ABORT("multi buffers are not supported by the meta backend"); + } t_ij->data = (char *) ggml_backend_buffer_get_base(simple_buf) - + size_t(tensor->data) - size_t(ggml_backend_buffer_get_base(buffer)); + + size_t(tensor->data) - size_t(ggml_backend_buffer_get_base(tensor->buffer)); + } + + if (init_buf) { + // the backend that owns the buffer will set .extra + ggml_backend_buffer_init_tensor(init_buf, t_ij); + } else { + t_ij->extra = tensor->extra; } - t_ij->extra = tensor->extra; + for (int i = 0; i < GGML_MAX_SRC; i++) { t_ij->src[i] = tensor->src[i]; if (tensor->src[i] == tensor) { @@ -1170,53 +1308,205 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer simple_tensors.push_back(t_ij); } - buf_ctx->simple_tensors[tensor] = simple_tensors; + + // If one of the sources has a zero-sized slice, disable the computation: + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (tensor->src[i] == nullptr || !ggml_backend_buffer_is_meta(tensor->src[i]->buffer)) { + continue; + } + + const ggml_backend_meta_split_state split_state_src = ggml_backend_meta_get_split_state(tensor->src[i], /*assume_sync =*/ true); + if (split_state_src.axis < 0 || split_state_src.axis >= GGML_MAX_DIMS) { + continue; + } + for (size_t j = 0; j < n_simple_bufs; j++) { + int64_t ne_sum = 0; + for (size_t s = 0; s < split_state_src.n_segments; s++) { + ne_sum += split_state_src.ne[s*n_simple_bufs + j] * split_state_src.nr[s]; + } + if (ne_sum == 0) { + simple_tensors[j]->flags &= ~GGML_TENSOR_FLAG_COMPUTE; + } + } + } + + stc.simple_tensors[tensor] = simple_tensors; return GGML_STATUS_SUCCESS; } -static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer); - GGML_ASSERT(ggml_is_contiguous(tensor)); +static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + GGML_ASSERT(ggml_backend_buffer_is_meta(buffer)); + ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context; + buf_ctx->stc_compute_index = buf_ctx->stc_compute_index_next; + return ggml_backend_meta_buffer_init_tensor_impl(buf_ctx->get_simple_tensor_container(tensor), tensor); +} - const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false); +static void ggml_backend_meta_buffer_memset_tensor( + ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { + const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer); + const ggml_backend_meta_split_state split_state = + ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false); + GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); - if (split_state.n_segments != 1) { + if (split_state.n_segments != 1 || split_state.nr[0] != 1) { GGML_ASSERT(split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS); - GGML_ASSERT(offset == 0); - GGML_ASSERT(size == ggml_nbytes(tensor)); + GGML_ASSERT(split_state.nr[0] != 0); GGML_ASSERT(tensor->ne[3] == 1); - size_t offset_data = 0; + std::vector simple_offsets(n_bufs, 0); if (split_state.axis == GGML_BACKEND_SPLIT_AXIS_0) { GGML_ASSERT(tensor->ne[2] == 1); + + const size_t row_stride = tensor->nb[1]; + GGML_ASSERT(offset % row_stride == 0); + GGML_ASSERT(size % row_stride == 0); + const int64_t row_start = offset / row_stride; + const int64_t row_count = size / row_stride; + GGML_ASSERT(row_start + row_count <= tensor->ne[1]); + const int64_t blck_size = ggml_blck_size(tensor->type); for (size_t s = 0; s < split_state.n_segments; s++) { + for (size_t r = 0; r < split_state.nr[s]; r++) { + for (size_t j = 0; j < n_bufs; j++) { + ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); + GGML_ASSERT(split_state.ne[s*n_bufs + j] % blck_size == 0); + const size_t nbytes = split_state.ne[s*n_bufs + j]/blck_size * tensor->nb[0]; + for (int64_t row = 0; row < row_count; row++) { + ggml_backend_tensor_memset(simple_tensor, value, + simple_offsets[j] + (row_start + row)*simple_tensor->nb[1], nbytes); + } + simple_offsets[j] += nbytes; + } + } + } + return; + } + + GGML_ASSERT(split_state.axis == GGML_BACKEND_SPLIT_AXIS_1); + + const size_t row_stride = tensor->nb[2]; + GGML_ASSERT(offset % row_stride == 0); + GGML_ASSERT(size % row_stride == 0); + const int64_t row_start = offset / row_stride; + const int64_t row_count = size / row_stride; + GGML_ASSERT(row_start + row_count <= tensor->ne[2]); + + for (size_t s = 0; s < split_state.n_segments; s++) { + for (size_t r = 0; r < split_state.nr[s]; r++) { for (size_t j = 0; j < n_bufs; j++) { ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); - GGML_ASSERT(split_state.ne[s*n_bufs + j] % blck_size == 0); - const size_t nbytes = split_state.ne[s*n_bufs + j]/blck_size * tensor->nb[0]; - ggml_backend_tensor_set_2d(simple_tensor, (const char *) data + offset_data, simple_offsets[j], nbytes, - tensor->ne[1], simple_tensor->nb[1], tensor->nb[1]); - offset_data += nbytes; + const size_t nbytes = split_state.ne[s*n_bufs + j] * tensor->nb[1]; + for (int64_t row = 0; row < row_count; row++) { + ggml_backend_tensor_memset(simple_tensor, value, + simple_offsets[j] + (row_start + row)*simple_tensor->nb[2], nbytes); + } simple_offsets[j] += nbytes; } } - GGML_ASSERT(offset_data*tensor->ne[1] == size); + } + return; + } + + switch (split_state.axis) { + case GGML_BACKEND_SPLIT_AXIS_0: + case GGML_BACKEND_SPLIT_AXIS_1: + case GGML_BACKEND_SPLIT_AXIS_2: { + const size_t chunk_size_full = tensor->nb[split_state.axis + 1]; + GGML_ASSERT(offset % chunk_size_full == 0); + GGML_ASSERT(size % chunk_size_full == 0); + const int64_t i_start = offset / chunk_size_full; + const int64_t i_stop = (offset + size) / chunk_size_full; + for (size_t j = 0; j < n_bufs; j++) { + ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); + const size_t chunk_size = simple_tensor->nb[split_state.axis + 1]; + if (chunk_size == 0) { + continue; + } + for (int64_t i = i_start; i < i_stop; i++) { + ggml_backend_tensor_memset(simple_tensor, value, i*chunk_size, chunk_size); + } + } + } break; + case GGML_BACKEND_SPLIT_AXIS_PARTIAL: { + GGML_ASSERT(value == 0); + [[fallthrough]]; + } + case GGML_BACKEND_SPLIT_AXIS_MIRRORED: { + for (size_t j = 0; j < n_bufs; j++) { + ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); + ggml_backend_tensor_memset(simple_tensor, value, offset, size); + } + } break; + default: { + GGML_ABORT("fatal error"); + } + } +} + +static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer); + const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false); + GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); + + if (split_state.n_segments != 1 || split_state.nr[0] != 1) { + GGML_ASSERT(split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS); + GGML_ASSERT(split_state.nr[0] != 0); + GGML_ASSERT(tensor->ne[3] == 1); + + size_t offset_data = 0; + std::vector simple_offsets(n_bufs, 0); + if (split_state.axis == GGML_BACKEND_SPLIT_AXIS_0) { + GGML_ASSERT(tensor->ne[2] == 1); + + const size_t row_stride = tensor->nb[1]; + GGML_ASSERT(offset % row_stride == 0); + GGML_ASSERT(size % row_stride == 0); + const int64_t row_start = offset / row_stride; + const int64_t row_count = size / row_stride; + GGML_ASSERT(row_start + row_count <= tensor->ne[1]); + + const int64_t blck_size = ggml_blck_size(tensor->type); + for (size_t s = 0; s < split_state.n_segments; s++) { + for (size_t r = 0; r < split_state.nr[s]; r++) { + for (size_t j = 0; j < n_bufs; j++) { + ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); + GGML_ASSERT(split_state.ne[s*n_bufs + j] % blck_size == 0); + const size_t nbytes = split_state.ne[s*n_bufs + j]/blck_size * tensor->nb[0]; + ggml_backend_tensor_set_2d(simple_tensor, (const char *) data + offset_data, + simple_offsets[j] + row_start * simple_tensor->nb[1], nbytes, + row_count, simple_tensor->nb[1], tensor->nb[1]); + offset_data += nbytes; + simple_offsets[j] += nbytes; + } + } + } + GGML_ASSERT(offset_data*row_count == size); return; } GGML_ASSERT(split_state.axis == GGML_BACKEND_SPLIT_AXIS_1); + + const size_t row_stride = tensor->nb[2]; + GGML_ASSERT(offset % row_stride == 0); + GGML_ASSERT(size % row_stride == 0); + const int64_t row_start = offset / row_stride; + const int64_t row_count = size / row_stride; + GGML_ASSERT(row_start + row_count <= tensor->ne[2]); + for (size_t s = 0; s < split_state.n_segments; s++) { - for (size_t j = 0; j < n_bufs; j++) { - ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); - const size_t nbytes = split_state.ne[s*n_bufs + j] * tensor->nb[1]; - ggml_backend_tensor_set_2d(simple_tensor, (const char *) data + offset_data, simple_offsets[j], nbytes, - tensor->ne[2], simple_tensor->nb[2], tensor->nb[2]); - offset_data += nbytes; - simple_offsets[j] += nbytes; + for (size_t r = 0; r < split_state.nr[s]; r++) { + for (size_t j = 0; j < n_bufs; j++) { + ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); + const size_t nbytes = split_state.ne[s*n_bufs + j] * tensor->nb[1]; + ggml_backend_tensor_set_2d(simple_tensor, (const char *) data + offset_data, + simple_offsets[j] + row_start * simple_tensor->nb[2], nbytes, + row_count, simple_tensor->nb[2], tensor->nb[2]); + offset_data += nbytes; + simple_offsets[j] += nbytes; + } } } - GGML_ASSERT(offset_data*tensor->ne[2] == size); + GGML_ASSERT(offset_data*row_count == size); return; } @@ -1234,6 +1524,9 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg for (size_t j = 0; j < n_bufs; j++) { ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); const size_t chunk_size_j = simple_tensor->nb[split_state.axis + 1]; + if (chunk_size_j == 0) { + continue; + } const size_t simple_offset = i_start * chunk_size_j; ggml_backend_tensor_set_2d(simple_tensor, (const char *) data + offset_j, simple_offset, chunk_size_j, i_stop - i_start, chunk_size_j, chunk_size_full); offset_j += chunk_size_j; @@ -1248,15 +1541,29 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg } break; case GGML_BACKEND_SPLIT_AXIS_PARTIAL: { GGML_ASSERT(tensor->type == GGML_TYPE_F32); - const int64_t ne = ggml_nelements(tensor); - std::vector tmp; - tmp.reserve(ne); - for (int64_t i = 0; i < ne; i++) { - tmp.push_back(((const float *) data)[i] / n_bufs); + GGML_ASSERT(offset % sizeof(float) == 0); + GGML_ASSERT(size % sizeof(float) == 0); + const size_t n_values = size / sizeof(float); + size_t n_contributors = 0; + for (size_t j = 0; j < n_bufs; j++) { + n_contributors += split_state.ne[j] != 0; + } + const bool has_contributor_mask = n_contributors != 0; + if (!has_contributor_mask) { + n_contributors = n_bufs; + } + std::vector tmp(n_values); + for (size_t i = 0; i < n_values; i++) { + tmp[i] = ((const float *) data)[i] / n_contributors; + } + std::vector zero; + if (has_contributor_mask) { + zero.resize(n_values, 0.0f); } for (size_t j = 0; j < n_bufs; j++) { ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); - ggml_backend_tensor_set(simple_tensor, tmp.data(), offset, size); + const float * partial = has_contributor_mask && split_state.ne[j] == 0 ? zero.data() : tmp.data(); + ggml_backend_tensor_set(simple_tensor, partial, offset, size); } } break; default: { @@ -1267,10 +1574,69 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer); - GGML_ASSERT(ggml_is_contiguous(tensor)); - const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false); - GGML_ASSERT(split_state.n_segments == 1); + GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED); + + if (split_state.n_segments != 1 || split_state.nr[0] != 1) { + GGML_ASSERT(split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS); + GGML_ASSERT(split_state.nr[0] != 0); + GGML_ASSERT(tensor->ne[3] == 1); + + size_t offset_data = 0; + std::vector simple_offsets(n_bufs, 0); + if (split_state.axis == GGML_BACKEND_SPLIT_AXIS_0) { + GGML_ASSERT(tensor->ne[2] == 1); + + const size_t row_stride = tensor->nb[1]; + GGML_ASSERT(offset % row_stride == 0); + GGML_ASSERT(size % row_stride == 0); + const int64_t row_start = offset / row_stride; + const int64_t row_count = size / row_stride; + GGML_ASSERT(row_start + row_count <= tensor->ne[1]); + + const int64_t blck_size = ggml_blck_size(tensor->type); + for (size_t s = 0; s < split_state.n_segments; s++) { + for (size_t r = 0; r < split_state.nr[s]; r++) { + for (size_t j = 0; j < n_bufs; j++) { + const ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); + GGML_ASSERT(split_state.ne[s*n_bufs + j] % blck_size == 0); + const size_t nbytes = split_state.ne[s*n_bufs + j]/blck_size * tensor->nb[0]; + ggml_backend_tensor_get_2d(simple_tensor, (char *) data + offset_data, + simple_offsets[j] + row_start * simple_tensor->nb[1], nbytes, + row_count, simple_tensor->nb[1], tensor->nb[1]); + offset_data += nbytes; + simple_offsets[j] += nbytes; + } + } + } + GGML_ASSERT(offset_data*row_count == size); + return; + } + GGML_ASSERT(split_state.axis == GGML_BACKEND_SPLIT_AXIS_1); + + const size_t row_stride = tensor->nb[2]; + GGML_ASSERT(offset % row_stride == 0); + GGML_ASSERT(size % row_stride == 0); + const int64_t row_start = offset / row_stride; + const int64_t row_count = size / row_stride; + GGML_ASSERT(row_start + row_count <= tensor->ne[2]); + + for (size_t s = 0; s < split_state.n_segments; s++) { + for (size_t r = 0; r < split_state.nr[s]; r++) { + for (size_t j = 0; j < n_bufs; j++) { + const ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); + const size_t nbytes = split_state.ne[s*n_bufs + j] * tensor->nb[1]; + ggml_backend_tensor_get_2d(simple_tensor, (char *) data + offset_data, + simple_offsets[j] + row_start * simple_tensor->nb[2], nbytes, + row_count, simple_tensor->nb[2], tensor->nb[2]); + offset_data += nbytes; + simple_offsets[j] += nbytes; + } + } + } + GGML_ASSERT(offset_data*row_count == size); + return; + } switch (split_state.axis) { case GGML_BACKEND_SPLIT_AXIS_0: @@ -1286,6 +1652,9 @@ static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, co for (size_t j = 0; j < n_bufs; j++){ const ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); const size_t chunk_size_j = simple_tensor->nb[split_state.axis + 1]; + if (chunk_size_j == 0) { + continue; + } const size_t simple_offset = i_start * chunk_size_j; ggml_backend_tensor_get_2d(simple_tensor, (char *) data + offset_j, simple_offset, chunk_size_j, i_stop - i_start, chunk_size_j, chunk_size_full); offset_j += chunk_size_j; @@ -1311,8 +1680,9 @@ static void ggml_backend_meta_buffer_clear(ggml_backend_buffer_t buffer, uint8_t } static void ggml_backend_meta_buffer_reset(ggml_backend_buffer_t buffer) { - const size_t n_buffers = ggml_backend_meta_buffer_n_bufs(buffer); - for (size_t i = 0; i < n_buffers; i++) { + GGML_ASSERT(ggml_backend_buffer_is_meta(buffer)); + ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context; + for (size_t i = 0; i < buf_ctx->bufs.size(); i++) { ggml_backend_buffer_reset(ggml_backend_meta_buffer_simple_buffer(buffer, i)); } } @@ -1321,7 +1691,7 @@ static const ggml_backend_buffer_i ggml_backend_meta_buffer_iface = { /* .free_buffer = */ ggml_backend_meta_buffer_free_buffer, /* .get_base = */ ggml_backend_meta_buffer_get_base, /* .init_tensor = */ ggml_backend_meta_buffer_init_tensor, - /* .memset_tensor = */ nullptr, // TODO implement + /* .memset_tensor = */ ggml_backend_meta_buffer_memset_tensor, /* .set_tensor = */ ggml_backend_meta_buffer_set_tensor, /* .get_tensor = */ ggml_backend_meta_buffer_get_tensor, /* .set_tensor_2d = */ nullptr, @@ -1335,52 +1705,92 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) { return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer; } +void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) { + GGML_ASSERT(ggml_backend_buffer_is_meta(buffer)); + ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context; + for (size_t i = 0; i < buf_ctx->bufs.size(); i++) { + if (buf_ctx->bufs[i]) { + ggml_backend_buffer_set_usage(buf_ctx->bufs[i].get(), usage); + } + } +} + static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft); - ggml_init_params params = { - /*.mem_size =*/ 1024*1024*1024, // FIXME + const ggml_init_params params = { + /*.mem_size =*/ 1024*1024*ggml_tensor_overhead(), // FIXME /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; + ggml_backend_meta_simple_tensor_container stc_static; + ggml_backend_meta_simple_tensor_container stc_compute_0(params, n_simple_bufts); + ggml_backend_meta_simple_tensor_container stc_compute_1(params, n_simple_bufts); - ggml_backend_meta_buffer_context * buf_ctx = new ggml_backend_meta_buffer_context(); size_t max_size = 0; - buf_ctx->buf_configs.reserve(n_simple_bufts); + std::vector bufs; + bufs.reserve(n_simple_bufts); for (size_t i = 0; i < n_simple_bufts; i++) { - ggml_backend_buffer_t simple_buf = ggml_backend_buft_alloc_buffer(ggml_backend_meta_buft_simple_buft(buft, i), size); - max_size = std::max(max_size, ggml_backend_buffer_get_size(simple_buf)); - buf_ctx->buf_configs.emplace_back(ggml_init(params), simple_buf); + bufs.push_back(ggml_backend_buft_alloc_buffer(ggml_backend_meta_buft_simple_buft(buft, i), size)); + GGML_ASSERT(bufs.back() != nullptr); + max_size = std::max(max_size, ggml_backend_buffer_get_size(bufs.back())); } + ggml_backend_meta_buffer_context * buf_ctx = new ggml_backend_meta_buffer_context(stc_static, stc_compute_0, stc_compute_1, bufs); return ggml_backend_buffer_init(buft, ggml_backend_meta_buffer_iface, buf_ctx, max_size); } -struct ggml_backend_buffer * ggml_backend_meta_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) { +static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors) { const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft); - ggml_init_params params = { - /*.mem_size =*/ 1024*1024*1024, // FIXME + constexpr size_t compute_headroom = 16; // Maximum number of views per statically allocated tensor that can be created between evals. + const ggml_init_params params_static = { + /*.mem_size =*/ n_tensors * ggml_tensor_overhead(), /*.mem_buffer =*/ nullptr, /*.no_alloc =*/ true, }; + const ggml_init_params params_compute = { + /*.mem_size =*/ compute_headroom * n_tensors * ggml_tensor_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ggml_backend_meta_simple_tensor_container stc_static (params_static, n_simple_bufts); + ggml_backend_meta_simple_tensor_container stc_compute_0(params_compute, n_simple_bufts); + ggml_backend_meta_simple_tensor_container stc_compute_1(params_compute, n_simple_bufts); - ggml_backend_meta_buffer_context * meta_buf_ctx = new ggml_backend_meta_buffer_context(); - meta_buf_ctx->buf_configs.reserve(n_simple_bufts); - for (size_t i = 0; i < n_simple_bufts; i++) { - meta_buf_ctx->buf_configs.emplace_back(ggml_init(params), nullptr); - } + std::vector bufs(n_simple_bufts, nullptr); + ggml_backend_meta_buffer_context * meta_buf_ctx = new ggml_backend_meta_buffer_context(stc_static, stc_compute_0, stc_compute_1, bufs); ggml_backend_buffer_t meta_buf = ggml_backend_buffer_init(buft, ggml_backend_meta_buffer_iface, meta_buf_ctx, 0); - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + for (int i = 0; i < n_tensors; i++) { + ggml_tensor * t = tensors[i]; t->buffer = meta_buf; - ggml_backend_meta_buffer_init_tensor(meta_buf, t); + ggml_backend_meta_buffer_init_tensor_impl(meta_buf_ctx->stc_static, t); t->data = (void *) 0x2000000000000000; // FIXME } for (size_t i = 0; i < n_simple_bufts; i++) { - meta_buf_ctx->buf_configs[i].buf = ggml_backend_alloc_ctx_tensors_from_buft( - meta_buf_ctx->buf_configs[i].ctx, ggml_backend_meta_buft_simple_buft(buft, i)); - meta_buf->size = std::max(meta_buf->size, ggml_backend_buffer_get_size(meta_buf_ctx->buf_configs[i].buf)); + ggml_context * ctx = meta_buf_ctx->stc_static.ctxs[i].get(); + ggml_backend_buffer_type_t simple_buft = ggml_backend_meta_buft_simple_buft(buft, i); + + // If a ggml_context only has zero-sized tensors, ggml_backend_alloc_ctx_tensors_from_buft returns NULL. + // For those edge cases, allocate a dummy buffer instead. + bool any_nonzero_slice = false; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (ggml_nelements(t) != 0) { + any_nonzero_slice = true; + break; + } + } + if (any_nonzero_slice) { + meta_buf_ctx->bufs[i].reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx, simple_buft)); + } else { + meta_buf_ctx->bufs[i].reset(ggml_backend_buft_alloc_buffer(simple_buft, 0)); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + t->buffer = meta_buf_ctx->bufs[i].get(); + } + } + GGML_ASSERT(meta_buf_ctx->bufs[i]); + meta_buf->size = std::max(meta_buf->size, ggml_backend_buffer_get_size(meta_buf_ctx->bufs[i].get())); } return meta_buf; } @@ -1404,26 +1814,32 @@ struct ggml_backend_meta_context { struct backend_config { ggml_backend_t backend; - std::vector cgraphs; - std::vector nodes; - ggml_backend_buffer_ptr buf; + std::vector cgraphs; + std::vector nodes; + std::vector bufs; - backend_config(ggml_backend_t backend) : backend(backend) {} + backend_config(ggml_backend_t backend, const size_t n_reduce_steps) : backend(backend) { + bufs.resize(n_reduce_steps); + } }; std::string name; std::vector backend_configs; ggml_context_ptr ctx; std::vector cgraphs_aux; std::vector nodes_aux; + size_t n_reduce_steps; int max_nnodes = 0; size_t max_tmp_size = 0; size_t max_subgraphs = 0; + size_t n_subgraphs = 0; + uint64_t uid = 0; void * comm_ctx = nullptr; ggml_backend_comm_allreduce_tensor_t comm_allreduce = nullptr; ggml_backend_meta_context(ggml_backend_dev_t meta_dev, const char * params) { const size_t n_devs = ggml_backend_meta_dev_n_devs(meta_dev); + n_reduce_steps = std::ceil(std::log2(n_devs)); name = "Meta("; std::vector simple_backends; backend_configs.reserve(n_devs); @@ -1435,7 +1851,7 @@ struct ggml_backend_meta_context { } name += ggml_backend_dev_name(simple_dev); simple_backends.push_back(ggml_backend_dev_init(simple_dev, params)); - backend_configs.emplace_back(simple_backends.back()); + backend_configs.emplace_back(simple_backends.back(), n_reduce_steps); } name += ")"; @@ -1465,10 +1881,6 @@ struct ggml_backend_meta_context { ggml_backend_free(bc.backend); } } - - size_t n_reduce_steps() const { - return std::ceil(std::log2(backend_configs.size())); - } }; static const char * ggml_backend_meta_get_name(ggml_backend_t backend) { @@ -1491,6 +1903,7 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false); GGML_ASSERT(split_state.n_segments == 1); + GGML_ASSERT(split_state.nr[0] == 1); switch (split_state.axis) { case GGML_BACKEND_SPLIT_AXIS_0: @@ -1507,6 +1920,9 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens ggml_backend_t simple_backend = ggml_backend_meta_simple_backend(backend, j); ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); const size_t chunk_size_j = simple_tensor->nb[split_state.axis + 1]; + if (chunk_size_j == 0) { + continue; + } ggml_backend_tensor_set_2d_async(simple_backend, simple_tensor, (const char *) data + offset_j, offset, chunk_size_j, i_stop - i_start, chunk_size_j, chunk_size_full); offset_j += chunk_size_j; @@ -1532,6 +1948,7 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false); GGML_ASSERT(split_state.n_segments == 1); + GGML_ASSERT(split_state.nr[0] == 1); switch (split_state.axis) { case GGML_BACKEND_SPLIT_AXIS_0: @@ -1548,6 +1965,9 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm ggml_backend_t simple_backend = ggml_backend_meta_simple_backend(backend, j); const ggml_tensor * simple_tensor = ggml_backend_meta_buffer_simple_tensor(tensor, j); const size_t chunk_size_j = simple_tensor->nb[split_state.axis + 1]; + if (chunk_size_j == 0) { + continue; + } ggml_backend_tensor_get_2d_async(simple_backend, simple_tensor, (char *) data + offset_j, offset, chunk_size_j, i_stop - i_start, chunk_size_j, chunk_size_full); offset_j += chunk_size_j; @@ -1578,6 +1998,9 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, const size_t n_backends = ggml_backend_meta_n_backends(backend); ggml_backend_meta_context * backend_ctx = (ggml_backend_meta_context *) backend->context; + // If the previous cgraph had a defined UID it can be used to skip rebuilding the subgraphs per simple backend. + const bool needs_rebuild = (cgraph->uid == 0) || (cgraph->uid != backend_ctx->uid); + bool max_nnodes_raised = false; if (cgraph->n_nodes > backend_ctx->max_nnodes) { for (size_t j = 0; j < n_backends; j++) { @@ -1587,173 +2010,311 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, } backend_ctx->max_nnodes = cgraph->n_nodes; max_nnodes_raised = true; + assert(needs_rebuild); } - for (size_t j = 0; j < n_backends; j++) { - auto & bcj = backend_ctx->backend_configs[j]; + if (needs_rebuild) { + std::set used_buffers; + for (int i = 0; i < cgraph->n_leafs; i++) { + if (ggml_backend_buffer_is_meta(cgraph->leafs[i]->buffer)) { + used_buffers.emplace(cgraph->leafs[i]->buffer); + } + } for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - if (node->view_src != nullptr && node->view_src->op == GGML_OP_NONE && ggml_backend_buffer_is_host(node->view_src->buffer)) { - // FIXME s_copy_main is on the CPU and its view seems to be incorrectly added to the graph nodes. - // For regular usage this doesn't matter since it's a noop but trying to call ggml_backend_meta_buffer_simple_tensor results in a crash. - bcj.nodes[i] = node; - continue; + if (ggml_backend_buffer_is_meta(cgraph->nodes[i]->buffer)) { + used_buffers.emplace(cgraph->nodes[i]->buffer); } - bcj.nodes[i] = ggml_backend_meta_buffer_simple_tensor(node, j); - GGML_ASSERT(bcj.nodes[i]); } - } + for (ggml_backend_buffer_t buf : used_buffers) { + ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buf->context; + buf_ctx->stc_compute_index_next = buf_ctx->stc_compute_index ^ 1; + ggml_backend_meta_simple_tensor_container & stc = buf_ctx->stc_compute[buf_ctx->stc_compute_index_next]; + for (ggml_context_ptr & ctx : stc.ctxs) { + ggml_reset(ctx.get()); + } + stc.simple_tensors.clear(); + } + size_t n_subgraphs = 0; + size_t max_tmp_size = 0; - size_t n_subgraphs = 0; - size_t max_tmp_size = 0; - { - // For MoE models it may make sense to delay the AllReduce in order to reduce I/O: - auto get_i_delayed = [&](const int i) -> int { - int id = i; // i_delayed - int idr = i; // i_delayed return, last safe return value - - ggml_tensor * node = cgraph->nodes[id]; - int32_t n_used = ggml_node_get_use_count(cgraph, id); - if (id + 1 >= cgraph->n_nodes) { - return idr; + for (size_t j = 0; j < n_backends; j++) { + auto & bcj = backend_ctx->backend_configs[j]; + + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + if (!ggml_backend_buffer_is_meta(node->buffer)) { + // [TAG_META_HOST_VIEWS] + GGML_ASSERT(ggml_backend_meta_is_host_view(node)); + + // keep the node as is, mapping it to a simple tensor is not possible + bcj.nodes[i] = node; + continue; + } + bcj.nodes[i] = ggml_backend_meta_buffer_simple_tensor(node, j); + GGML_ASSERT(bcj.nodes[i]); } - { - ggml_tensor * next = cgraph->nodes[id+1]; - if (next->op == GGML_OP_ADD_ID && next->src[0] == node && - ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL && - ggml_backend_meta_get_split_state(next->src[2], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { - node = next; + } + + { + // For MoE models it may make sense to delay the AllReduce in order to reduce I/O: + auto get_i_delayed_branch = [&](const int i) -> int { + int id = i; // i_delayed + int idr = i; // i_delayed return, last safe return value + + ggml_tensor * node = cgraph->nodes[id]; + int32_t n_used = ggml_node_get_use_count(cgraph, id); + + // Skip MIRRORED nodes that don't consume node + auto skip_unrelated = [&]() { + while (id + 1 < cgraph->n_nodes) { + ggml_tensor * next = cgraph->nodes[id+1]; + if (ggml_backend_meta_get_split_state(next, false).axis != GGML_BACKEND_SPLIT_AXIS_MIRRORED) { + break; + } + bool safe = true; + for (int s = 0; s < GGML_MAX_SRC; s++) { + if (next->src[s] == nullptr) { + continue; + } + if (next->src[s] == node) { + safe = false; + break; + } + if (ggml_backend_meta_get_split_state(next->src[s], false).axis != GGML_BACKEND_SPLIT_AXIS_MIRRORED) { + safe = false; + break; + } + } + if (!safe) { + break; + } + id++; + } + }; + + skip_unrelated(); + if (id + 1 >= cgraph->n_nodes) { + return idr; + } + { + ggml_tensor * next = cgraph->nodes[id+1]; + if (next->op == GGML_OP_ADD_ID && next->src[0] == node && + ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL && + ggml_backend_meta_get_split_state(next->src[2], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { + node = next; + id++; + idr = id; + n_used = ggml_node_get_use_count(cgraph, id); + } + } + // Chain of MULs with MIRRORED src[1] + while (true) { + skip_unrelated(); + if (id + 1 >= cgraph->n_nodes) { + return idr; + } + ggml_tensor * next = cgraph->nodes[id+1]; + if (next->op == GGML_OP_MUL && next->src[0] == node && + ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { + node = next; + id++; + idr = id; + n_used = ggml_node_get_use_count(cgraph, id); + } else { + break; + } + } + + if (n_used != node->ne[1] || id + 2*n_used-1 >= cgraph->n_nodes) { + return idr; + } + for (int32_t k = 0; k < n_used; k++) { + ggml_tensor * next = cgraph->nodes[id+1]; + if (next->op != GGML_OP_VIEW || next->view_src != node || next->view_offs != k*node->nb[1] || + next->ne[0] != node->ne[0] || next->ne[1] != node->ne[2] || next->nb[1] != node->nb[2] || + ggml_node_get_use_count(cgraph, id+1) != 1) { + return idr; + } id++; - idr = id; - n_used = ggml_node_get_use_count(cgraph, id); } - } - if (id + 1 >= cgraph->n_nodes) { - return idr; - } - { - ggml_tensor * next = cgraph->nodes[id+1]; - if (next->op == GGML_OP_MUL && next->src[0] == node && - ggml_backend_meta_get_split_state(next->src[1], false).axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED) { - node = next; + { + ggml_tensor * next = cgraph->nodes[id+1]; + if (next->op != GGML_OP_ADD || next->src[0] != cgraph->nodes[id - (n_used-1)] || + next->src[1] != cgraph->nodes[id - (n_used-2)] || ggml_node_get_use_count(cgraph, id+1) != 1) { + return idr; + } id++; - idr = id; - n_used = ggml_node_get_use_count(cgraph, id); } - } - - if (n_used != node->ne[1] || id + 2*n_used-1 >= cgraph->n_nodes) { + for (int32_t k = 0; k < n_used - 2; k++) { + ggml_tensor * next = cgraph->nodes[id+1]; + if (next->op != GGML_OP_ADD || next->src[0] != cgraph->nodes[id] || + next->src[1] != cgraph->nodes[id - (n_used-2)] || ggml_node_get_use_count(cgraph, id+1) != 1) { + return idr; + } + id++; + } + idr = id; return idr; - } - for (int32_t k = 0; k < n_used; k++) { - ggml_tensor * next = cgraph->nodes[id+1]; - if (next->op != GGML_OP_VIEW || next->view_src != node || next->view_offs != k*node->nb[1] || - next->ne[0] != node->ne[0] || next->ne[1] != node->ne[2] || next->nb[1] != node->nb[2] || - ggml_node_get_use_count(cgraph, id+1) != 1) { - return idr; + }; + + // AllReduce(a) + AllReduce(b) == AllReduce(a + b) for independent partial branches. + auto get_i_delayed = [&](const int i) -> int { + const int i_delayed = get_i_delayed_branch(i); + ggml_tensor * node = cgraph->nodes[i_delayed]; + + if (ggml_node_get_use_count(cgraph, i_delayed) != 1) { + return i_delayed; } - id++; - } - { - ggml_tensor * next = cgraph->nodes[id+1]; - if (next->op != GGML_OP_ADD || next->src[0] != cgraph->nodes[id - (n_used-1)] || - next->src[1] != cgraph->nodes[id - (n_used-2)] || ggml_node_get_use_count(cgraph, id+1) != 1) { - return idr; + + for (int id = i_delayed + 1; id < cgraph->n_nodes; id++) { + ggml_tensor * next = cgraph->nodes[id]; + if (next->view_src == node) { + return i_delayed; + } + for (int s = 0; s < GGML_MAX_SRC; s++) { + if (next->src[s] == node) { + return i_delayed; + } + } + + if (next->view_src != nullptr && next->view_src->op == GGML_OP_NONE && ggml_backend_buffer_is_host(next->view_src->buffer)) { + continue; + } + if (ggml_backend_meta_get_split_state(next, false).axis != GGML_BACKEND_SPLIT_AXIS_PARTIAL) { + continue; + } + + const int i_other = id; + const int i_other_delayed = get_i_delayed_branch(i_other); + ggml_tensor * other = cgraph->nodes[i_other_delayed]; + if (ggml_node_get_use_count(cgraph, i_other_delayed) != 1 || i_other_delayed + 1 >= cgraph->n_nodes) { + return i_delayed; + } + + ggml_tensor * sum = cgraph->nodes[i_other_delayed + 1]; + if (sum->op != GGML_OP_ADD || + !ggml_are_same_shape(node, other) || node->type != other->type || sum->type != node->type || + !((sum->src[0] == node && sum->src[1] == other) || + (sum->src[0] == other && sum->src[1] == node)) || + ggml_backend_meta_get_split_state(sum, false).axis != GGML_BACKEND_SPLIT_AXIS_MIRRORED) { + return i_delayed; + } + + for (size_t j = 0; j < n_backends; j++) { + auto & bcj = backend_ctx->backend_configs[j]; + const bool compute = bcj.nodes[i]->flags & GGML_TENSOR_FLAG_COMPUTE; + const bool compute_other = bcj.nodes[i_other]->flags & GGML_TENSOR_FLAG_COMPUTE; + if (compute != compute_other) { + return i_delayed; + } + } + return i_other_delayed + 1; } - id++; - } - for (int32_t k = 0; k < n_used - 2; k++) { - ggml_tensor * next = cgraph->nodes[id+1]; - if (next->op != GGML_OP_ADD || next->src[0] != cgraph->nodes[id] || - next->src[1] != cgraph->nodes[id - (n_used-2)] || ggml_node_get_use_count(cgraph, id+1) != 1) { - return idr; + return i_delayed; + }; + + int i_start = 0; + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + if (node->view_src != nullptr && node->view_src->op == GGML_OP_NONE && ggml_backend_buffer_is_host(node->view_src->buffer)) { + continue; + } + const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(node, /*assume_sync =*/ false); + if (split_state.axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) { + max_tmp_size = std::max(max_tmp_size, ggml_nbytes(node)); + } + const bool new_subgraph = i + 1 == cgraph->n_nodes || split_state.axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL; + if (!new_subgraph) { + continue; } - id++; - } - idr = id; - return idr; - }; - int i_start = 0; - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - if (node->view_src != nullptr && node->view_src->op == GGML_OP_NONE && ggml_backend_buffer_is_host(node->view_src->buffer)) { - continue; - } - const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(node, /*assume_sync =*/ false); - if (split_state.axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) { - max_tmp_size = std::max(max_tmp_size, ggml_nbytes(node)); - } - const bool new_subgraph = i + 1 == cgraph->n_nodes || split_state.axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL; - if (!new_subgraph) { - continue; + const int i_delayed = get_i_delayed(i); + + // If we can delay the AllReduce we need to consider the interaction with zero-sized tensor slices. + // A backend with such a slice would normally have valid data after participating in the AllReduce with a node that has + // its compute flag disabled and thus gets its data zeroed out. + // If the AllReduce is delayed then the nodes until that point also need to have their compute flag disabled. + if (i_delayed > i) { + for (size_t j = 0; j < n_backends; j++) { + auto & bcj = backend_ctx->backend_configs[j]; + if ((bcj.nodes[i]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + for (int ii = i + 1; ii <= i_delayed; ii++) { + bcj.nodes[ii]->flags &= ~GGML_TENSOR_FLAG_COMPUTE; + } + } + } + } + + i = i_delayed; + + for (size_t j = 0; j < n_backends; j++) { + auto & bcj = backend_ctx->backend_configs[j]; + bcj.cgraphs[n_subgraphs].offset = i_start; + } + n_subgraphs++; + i_start = i + 1; } + GGML_ASSERT(i_start == cgraph->n_nodes); + } - i = get_i_delayed(i); + backend_ctx->uid = cgraph->uid; + backend_ctx->n_subgraphs = n_subgraphs; + if (max_tmp_size > backend_ctx->max_tmp_size) { for (size_t j = 0; j < n_backends; j++) { auto & bcj = backend_ctx->backend_configs[j]; - bcj.cgraphs[n_subgraphs].offset = i_start; + for (size_t i = 0; i < backend_ctx->n_reduce_steps; i++) { + bcj.bufs[i].reset(ggml_backend_alloc_buffer(bcj.backend, max_tmp_size)); + } + } + backend_ctx->max_tmp_size = max_tmp_size; + } + + if (max_nnodes_raised || n_subgraphs > backend_ctx->max_subgraphs) { + backend_ctx->max_subgraphs = std::max(backend_ctx->max_subgraphs, n_subgraphs); + const size_t n_nodes_per_device = 3 * backend_ctx->n_reduce_steps; // tmp + ADD (+zeroing) graph per step and device + const size_t n_cgraphs_per_device = 2 * backend_ctx->n_reduce_steps; // ADD ( + zeroing) graph per step and device + const size_t mem_per_device_graphs_main = backend_ctx->max_subgraphs*ggml_graph_overhead_custom(backend_ctx->max_nnodes, cgraph->grads); + const size_t mem_per_device_graphs_aux = n_cgraphs_per_device*backend_ctx->max_subgraphs*ggml_graph_overhead_custom(1, cgraph->grads); + const size_t mem_per_device_nodes_aux = n_nodes_per_device*backend_ctx->max_subgraphs*ggml_tensor_overhead(); + const ggml_init_params params = { + /*.mem_size =*/ n_backends * (mem_per_device_graphs_main + mem_per_device_graphs_aux + mem_per_device_nodes_aux), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + backend_ctx->ctx.reset(ggml_init(params)); + for (size_t j = 0; j < n_backends; j++) { + auto & bcj = backend_ctx->backend_configs[j]; + for (size_t i = 0; i < backend_ctx->max_subgraphs; i++) { + bcj.cgraphs[i].cgraph_main = ggml_new_graph_custom(backend_ctx->ctx.get(), backend_ctx->max_nnodes, /*grads =*/ false); + } + } + backend_ctx->cgraphs_aux.resize(n_backends*n_cgraphs_per_device*backend_ctx->max_subgraphs); + for (size_t k = 0; k < backend_ctx->cgraphs_aux.size(); k++) { + backend_ctx->cgraphs_aux[k] = ggml_new_graph_custom(backend_ctx->ctx.get(), 1, cgraph->grads); + } + backend_ctx->nodes_aux.resize(n_backends*n_nodes_per_device*backend_ctx->max_subgraphs); + for (size_t k = 0; k < backend_ctx->nodes_aux.size(); k++) { + backend_ctx->nodes_aux[k] = ggml_new_tensor_1d(backend_ctx->ctx.get(), GGML_TYPE_F32, 1); } - n_subgraphs++; - i_start = i + 1; - } - GGML_ASSERT(i_start == cgraph->n_nodes); - } - - if (max_tmp_size > backend_ctx->max_tmp_size) { - for (size_t j = 0; j < n_backends; j++) { - auto & bcj = backend_ctx->backend_configs[j]; - bcj.buf.reset(ggml_backend_alloc_buffer(bcj.backend, max_tmp_size)); } - backend_ctx->max_tmp_size = max_tmp_size; - } - - if (max_nnodes_raised || n_subgraphs > backend_ctx->max_subgraphs) { - backend_ctx->max_subgraphs = std::max(backend_ctx->max_subgraphs, n_subgraphs); - const size_t n_reduce_steps = backend_ctx->n_reduce_steps(); - const size_t n_nodes_per_device = 2 * n_reduce_steps; // tmp + ADD per step - const size_t n_cgraphs_per_device = n_reduce_steps; // 1 ADD graph per step - const size_t mem_per_device_graphs_main = backend_ctx->max_subgraphs*ggml_graph_overhead_custom(backend_ctx->max_nnodes, cgraph->grads); - const size_t mem_per_device_graphs_aux = n_cgraphs_per_device*backend_ctx->max_subgraphs*ggml_graph_overhead_custom(1, cgraph->grads); - const size_t mem_per_device_nodes_aux = n_nodes_per_device*backend_ctx->max_subgraphs*ggml_tensor_overhead(); - ggml_init_params params = { - /*.mem_size =*/ n_backends * (mem_per_device_graphs_main + mem_per_device_graphs_aux + mem_per_device_nodes_aux), - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - backend_ctx->ctx.reset(ggml_init(params)); for (size_t j = 0; j < n_backends; j++) { auto & bcj = backend_ctx->backend_configs[j]; - for (size_t i = 0; i < n_subgraphs; i++) { - bcj.cgraphs[i].cgraph_main = ggml_new_graph_custom(backend_ctx->ctx.get(), cgraph->n_nodes, /*grads =*/ false); - } - } - backend_ctx->cgraphs_aux.resize(n_backends*n_cgraphs_per_device*backend_ctx->max_subgraphs); - for (size_t k = 0; k < backend_ctx->cgraphs_aux.size(); k++) { - backend_ctx->cgraphs_aux[k] = ggml_new_graph_custom(backend_ctx->ctx.get(), 1, cgraph->grads); - } - backend_ctx->nodes_aux.resize(n_backends*n_nodes_per_device*backend_ctx->max_subgraphs); - for (size_t k = 0; k < backend_ctx->nodes_aux.size(); k++) { - backend_ctx->nodes_aux[k] = ggml_new_tensor_1d(backend_ctx->ctx.get(), GGML_TYPE_F32, 1); - } - } - - for (size_t j = 0; j < n_backends; j++) { - auto & bcj = backend_ctx->backend_configs[j]; - for (size_t i_graph = 0; i_graph < n_subgraphs; i_graph++) { - ggml_cgraph * cgraph_ij = bcj.cgraphs[i_graph].cgraph_main; - const size_t i_node_start = bcj.cgraphs[i_graph].offset; - const size_t i_node_stop = i_graph + 1 < n_subgraphs ? bcj.cgraphs[i_graph + 1].offset : cgraph->n_nodes; - cgraph_ij->n_nodes = i_node_stop - i_node_start; - ggml_hash_set_reset(&cgraph_ij->visited_hash_set); - for (size_t i_node = i_node_start; i_node < i_node_stop; i_node++) { - ggml_tensor * node_ij = bcj.nodes[i_node]; - cgraph_ij->nodes[i_node - i_node_start] = node_ij; - const size_t hash_pos_orig = ggml_hash_find(&cgraph->visited_hash_set, cgraph->nodes[i_node]); - const size_t hash_pos_ij = ggml_hash_insert(&cgraph_ij->visited_hash_set, node_ij); - cgraph_ij->use_counts[hash_pos_ij] = cgraph->use_counts[hash_pos_orig]; + for (size_t i_graph = 0; i_graph < n_subgraphs; i_graph++) { + ggml_cgraph * cgraph_ij = bcj.cgraphs[i_graph].cgraph_main; + const size_t i_node_start = bcj.cgraphs[i_graph].offset; + const size_t i_node_stop = i_graph + 1 < n_subgraphs ? bcj.cgraphs[i_graph + 1].offset : cgraph->n_nodes; + cgraph_ij->n_nodes = i_node_stop - i_node_start; + ggml_hash_set_reset(&cgraph_ij->visited_hash_set); + for (size_t i_node = i_node_start; i_node < i_node_stop; i_node++) { + ggml_tensor * node_ij = bcj.nodes[i_node]; + cgraph_ij->nodes[i_node - i_node_start] = node_ij; + const size_t hash_pos_orig = ggml_hash_find(&cgraph->visited_hash_set, cgraph->nodes[i_node]); + const size_t hash_pos_ij = ggml_hash_insert(&cgraph_ij->visited_hash_set, node_ij); + cgraph_ij->use_counts[hash_pos_ij] = cgraph->use_counts[hash_pos_orig]; + } + cgraph_ij->uid = ggml_graph_next_uid(); } } } @@ -1761,11 +2322,6 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, size_t iga = 0; // i graph aux size_t ina = 0; // i node aux - // FIXME usage_counts - auto get_cgraph_aux = [&]() -> ggml_cgraph * { - ggml_cgraph * ret = backend_ctx->cgraphs_aux[iga++]; - return ret; - }; auto get_node_aux = [&](ggml_tensor * t) -> ggml_tensor * { ggml_tensor * ret = backend_ctx->nodes_aux[ina++]; memset(ret, 0, sizeof(ggml_tensor)); @@ -1777,75 +2333,113 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, } return ret; }; + auto set_tmp_data = [&](ggml_tensor * tensor, const size_t j, const size_t i_buf) { + auto & bcj = backend_ctx->backend_configs[j]; + ggml_backend_buffer_ptr & buf_ptr = bcj.bufs[i_buf]; + if (!buf_ptr || ggml_backend_buffer_get_size(buf_ptr.get()) < backend_ctx->max_tmp_size) { + buf_ptr.reset(ggml_backend_alloc_buffer(bcj.backend, backend_ctx->max_tmp_size)); + } + tensor->buffer = buf_ptr.get(); + tensor->data = ggml_backend_buffer_get_base(buf_ptr.get()); + }; + // FIXME usage_counts + auto get_cgraph_aux = [&]() -> ggml_cgraph * { + ggml_cgraph * ret = backend_ctx->cgraphs_aux[iga++]; + return ret; + }; // Preferentially use backend-specific allreduce_tensor_async (e.g. NCCL for CUDA), use a generic fallback if unavailable: auto allreduce_fallback = [&](size_t i) -> ggml_status { std::vector step_cgraphs(n_backends, nullptr); - for (size_t offset_j = 1; offset_j < n_backends; offset_j *= 2) { + // Zero out nodes that were disabled due to having a zero-sized slice: + for (size_t j = 0; j < n_backends; j++) { + auto & bcj = backend_ctx->backend_configs[j]; + ggml_tensor * node = bcj.cgraphs[i].cgraph_main->nodes[bcj.cgraphs[i].cgraph_main->n_nodes - 1]; + if (node->flags & GGML_TENSOR_FLAG_COMPUTE) { + continue; + } + if (ggml_nelements(node) > 0) { + ggml_tensor * node_zero = get_node_aux(node); + node_zero->op = GGML_OP_FILL; + node_zero->src[0] = node; // only used for the shape, the data is not read + ggml_set_op_params_f32(node_zero, 0, 0.0f); + node_zero->data = node->data; + node_zero->buffer = node->buffer; + node_zero->flags |= GGML_TENSOR_FLAG_COMPUTE; + + step_cgraphs[j] = get_cgraph_aux(); + step_cgraphs[j]->nodes[0] = node_zero; + step_cgraphs[j]->n_nodes = 1; + const ggml_status status = ggml_backend_graph_compute_async(bcj.backend, step_cgraphs[j]); + if (status != GGML_STATUS_SUCCESS) { + return status; + } + } + } + std::fill(step_cgraphs.begin(), step_cgraphs.end(), nullptr); + + auto push_data = [&](const size_t j_src, const size_t j_dst, const size_t i_buf) { + assert(step_cgraphs[j_dst] == nullptr); + auto & bcj_src = backend_ctx->backend_configs[j_src]; + auto & bcj_dst = backend_ctx->backend_configs[j_dst]; + + ggml_tensor * node_src = bcj_src.cgraphs[i].cgraph_main->nodes[bcj_src.cgraphs[i].cgraph_main->n_nodes - 1]; + ggml_tensor * node_dst = bcj_dst.cgraphs[i].cgraph_main->nodes[bcj_dst.cgraphs[i].cgraph_main->n_nodes - 1]; + GGML_ASSERT(ggml_is_contiguous(node_src)); + GGML_ASSERT(ggml_is_contiguous(node_dst)); + + ggml_tensor * node_tmp = get_node_aux(node_dst); + set_tmp_data(node_tmp, j_dst, i_buf); + + ggml_backend_tensor_copy_async(bcj_src.backend, bcj_dst.backend, node_src, node_tmp); + + ggml_tensor * node_red = get_node_aux(node_dst); + node_red->view_src = node_dst->view_src == nullptr ? node_dst : node_dst->view_src; + node_red->view_offs = node_dst->view_offs; + node_red->op = GGML_OP_ADD; + node_red->src[0] = node_dst; + node_red->src[1] = node_tmp; + node_red->flags |= GGML_TENSOR_FLAG_COMPUTE; + ggml_backend_view_init(node_red); + + ggml_cgraph * cgraph_aux = get_cgraph_aux(); + cgraph_aux->nodes[0] = node_red; + cgraph_aux->n_nodes = 1; + step_cgraphs[j_dst] = cgraph_aux; + }; + + size_t offset_j = n_backends/2; + while ((offset_j & (offset_j - 1)) != 0) { + offset_j--; + } + const size_t offset_j_max = offset_j; + size_t i_buf = 0; + + // If n_backends is not a power of 2, fold in the excess prior to butterfly reduction: + for (size_t j_src = 2*offset_j_max; j_src < n_backends; j_src++) { + const size_t j_dst = j_src - 2*offset_j_max; + push_data(j_src, j_dst, i_buf); + const ggml_status status = ggml_backend_graph_compute_async(backend_ctx->backend_configs[j_dst].backend, step_cgraphs[j_dst]); + if (status != GGML_STATUS_SUCCESS) { + return status; + } + i_buf = 1; + } + + // Butterfly reduction: + for (; offset_j >= 1; offset_j /= 2) { std::fill(step_cgraphs.begin(), step_cgraphs.end(), nullptr); - for (size_t j = 0; j < n_backends; j++) { + for (size_t j = 0; j < 2*offset_j_max; j++) { const size_t j_other = j ^ offset_j; - if (j_other > j) { + if (j_other >= n_backends) { continue; } + push_data(j, j_other, i_buf); + } - auto & bcj1 = backend_ctx->backend_configs[j]; - auto & bcj2 = backend_ctx->backend_configs[j_other]; - - ggml_tensor * node1 = bcj1.cgraphs[i].cgraph_main->nodes[bcj1.cgraphs[i].cgraph_main->n_nodes - 1]; - ggml_tensor * node2 = bcj2.cgraphs[i].cgraph_main->nodes[bcj2.cgraphs[i].cgraph_main->n_nodes - 1]; - GGML_ASSERT(ggml_is_contiguous(node1)); - GGML_ASSERT(ggml_is_contiguous(node2)); - - // Tmp tensors to receive P2P copies - ggml_tensor * node_tmp_1 = get_node_aux(node1); - node_tmp_1->buffer = bcj1.buf.get(); - node_tmp_1->data = ggml_backend_buffer_get_base(bcj1.buf.get()); - - ggml_tensor * node_tmp_2 = get_node_aux(node2); - node_tmp_2->buffer = bcj2.buf.get(); - node_tmp_2->data = ggml_backend_buffer_get_base(bcj2.buf.get()); - - // 2 P2P copies: exchange full buffers - ggml_backend_tensor_copy_async(bcj1.backend, bcj2.backend, node1, node_tmp_2); - ggml_backend_tensor_copy_async(bcj2.backend, bcj1.backend, node2, node_tmp_1); - - // Local ADD: node1 += tmp1 (in-place via view) - ggml_tensor * node_red_1 = get_node_aux(node1); - node_red_1->view_src = node1->view_src == nullptr ? node1 : node1->view_src; - node_red_1->view_offs = node1->view_offs; - node_red_1->op = GGML_OP_ADD; - node_red_1->src[0] = node1; - node_red_1->src[1] = node_tmp_1; - node_red_1->flags |= GGML_TENSOR_FLAG_COMPUTE; - ggml_backend_view_init(node_red_1); - - // Local ADD: node2 += tmp2 (in-place via view) - ggml_tensor * node_red_2 = get_node_aux(node2); - node_red_2->view_src = node2->view_src == nullptr ? node2 : node2->view_src; - node_red_2->view_offs = node2->view_offs; - node_red_2->op = GGML_OP_ADD; - node_red_2->src[0] = node2; - node_red_2->src[1] = node_tmp_2; - node_red_2->flags |= GGML_TENSOR_FLAG_COMPUTE; - ggml_backend_view_init(node_red_2); - - // Build 1-node cgraphs for the ADD ops - ggml_cgraph * cgraph_aux_1 = get_cgraph_aux(); - cgraph_aux_1->nodes[0] = node_red_1; - cgraph_aux_1->n_nodes = 1; - step_cgraphs[j] = cgraph_aux_1; - - ggml_cgraph * cgraph_aux_2 = get_cgraph_aux(); - cgraph_aux_2->nodes[0] = node_red_2; - cgraph_aux_2->n_nodes = 1; - step_cgraphs[j_other] = cgraph_aux_2; - } - - // Execute local ADDs for this step - for (size_t j = 0; j < n_backends; j++) { + for (size_t j = 0; j < 2*offset_j_max; j++) { if (step_cgraphs[j] == nullptr) { continue; } @@ -1855,12 +2449,25 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, return status; } } + i_buf++; + } + assert(i_buf == backend_ctx->n_reduce_steps); + + // If n_backends is not a power of 2, copy back the reduced tensors to the excess: + for (size_t j = 2*offset_j_max; j < n_backends; j++) { + auto & bcj_src = backend_ctx->backend_configs[j - 2*offset_j_max]; + auto & bcj_dst = backend_ctx->backend_configs[j]; + + ggml_tensor * node_src = bcj_src.cgraphs[i].cgraph_main->nodes[bcj_src.cgraphs[i].cgraph_main->n_nodes - 1]; + ggml_tensor * node_dst = bcj_dst.cgraphs[i].cgraph_main->nodes[bcj_dst.cgraphs[i].cgraph_main->n_nodes - 1]; + ggml_backend_tensor_copy_async(bcj_src.backend, bcj_dst.backend, node_src, node_dst); } + return GGML_STATUS_SUCCESS; }; - for (size_t i = 0; i < n_subgraphs; i++) { + for (size_t i = 0; i < backend_ctx->n_subgraphs; i++) { for (size_t j = 0; j < n_backends; j++) { auto & bcj = backend_ctx->backend_configs[j]; const ggml_status status = ggml_backend_graph_compute_async(bcj.backend, bcj.cgraphs[i].cgraph_main); @@ -1869,7 +2476,7 @@ static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, } } - if (n_backends > 1 && i < n_subgraphs - 1) { + if (n_backends > 1 && i < backend_ctx->n_subgraphs - 1) { bool backend_allreduce_success = false; if (backend_ctx->comm_ctx) { std::vector nodes; @@ -1898,8 +2505,8 @@ static const ggml_backend_i ggml_backend_meta_i = { /* .free = */ ggml_backend_meta_free, /* .set_tensor_async = */ ggml_backend_meta_set_tensor_async, /* .get_tensor_async = */ ggml_backend_meta_get_tensor_async, - /* .get_tensor_2d_async = */ nullptr, /* .set_tensor_2d_async = */ nullptr, + /* .get_tensor_2d_async = */ nullptr, /* .cpy_tensor_async = */ nullptr, /* .synchronize = */ ggml_backend_meta_synchronize, /* .graph_plan_create = */ nullptr, @@ -1938,4 +2545,3 @@ ggml_backend_t ggml_backend_meta_simple_backend(ggml_backend_t meta_backend, siz const ggml_backend_meta_context * backend_ctx = (const ggml_backend_meta_context *) meta_backend->context; return backend_ctx->backend_configs[index].backend; } - diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp index 0587109..4e9681c 100644 --- a/ggml/src/ggml-backend-reg.cpp +++ b/ggml/src/ggml-backend-reg.cpp @@ -86,6 +86,10 @@ #include "ggml-openvino.h" #endif +#ifdef GGML_USE_ET +#include "ggml-et.h" +#endif + namespace fs = std::filesystem; static std::string path_str(const fs::path & path) { @@ -161,6 +165,9 @@ struct ggml_backend_registry { #ifdef GGML_USE_OPENVINO register_backend(ggml_backend_openvino_reg()); #endif +#ifdef GGML_USE_ET + register_backend(ggml_backend_et_reg()); +#endif #ifdef GGML_USE_CPU register_backend(ggml_backend_cpu_reg()); #endif @@ -181,6 +188,12 @@ struct ggml_backend_registry { return; } + for (auto & entry : backends) { + if (entry.reg == reg) { + return; + } + } + #ifndef NDEBUG GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n", __func__, ggml_backend_reg_name(reg), ggml_backend_reg_dev_count(reg)); @@ -192,6 +205,12 @@ struct ggml_backend_registry { } void register_device(ggml_backend_dev_t device) { + for (auto & dev : devices) { + if (dev == device) { + return; + } + } + #ifndef NDEBUG GGML_LOG_DEBUG("%s: registered device %s (%s)\n", __func__, ggml_backend_dev_name(device), ggml_backend_dev_description(device)); #endif @@ -471,7 +490,13 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, #endif // default search paths: executable directory, current directory search_paths.push_back(get_executable_path()); - search_paths.push_back(fs::current_path()); + std::error_code cwd_ec; + const fs::path cwd = fs::current_path(cwd_ec); + if (cwd_ec) { + GGML_LOG_DEBUG("%s: failed to get current directory: %s\n", __func__, cwd_ec.message().c_str()); + } else { + search_paths.push_back(cwd); + } } else { search_paths.push_back(fs::u8path(user_search_path)); } @@ -481,16 +506,11 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, std::error_code ec; for (const auto & search_path : search_paths) { - if (!fs::exists(search_path, ec)) { - if (ec) { - GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(search_path).c_str(), ec.message().c_str()); - } else { - GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str()); - } - continue; - } - fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied); - for (const auto & entry : dir_it) { + std::error_code dir_ec; + fs::directory_iterator dir_it(search_path, dir_ec); + + for (const fs::directory_iterator end; dir_it != end; dir_it.increment(dir_ec)) { + const auto & entry = *dir_it; if (entry.is_regular_file(ec)) { auto filename = entry.path().filename(); auto ext = entry.path().extension(); @@ -519,6 +539,10 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, } } } + + if (dir_ec) { + GGML_LOG_DEBUG("%s: failed to enumerate %s: %s\n", __func__, path_str(search_path).c_str(), dir_ec.message().c_str()); + } } if (best_score == 0) { diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index d9f8aae..22e5fd9 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #ifdef __APPLE__ @@ -44,6 +45,138 @@ ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t return buft->iface.alloc_buffer(buft, size); } +// shared planning logic for allocating a list of tensors into one or more buffers of the given type +struct ggml_backend_buft_alloc_buffer_n_plan_item { + size_t size; // total bytes for this buffer + int first; // first tensor index (inclusive) + int last; // last tensor index (exclusive) +}; + +using ggml_backend_buft_alloc_buffer_n_plan_t = std::vector; + +static ggml_backend_buft_alloc_buffer_n_plan_t ggml_backend_buft_alloc_buffer_n_plan( + ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + ggml_backend_buft_alloc_buffer_n_plan_t plan; + + const size_t alignment = ggml_backend_buft_get_alignment(buft); + const size_t max_size = ggml_backend_buft_get_max_size(buft); + + size_t cur_buf_size = 0; + int first = 0; + + for (int i = 0; i < n_tensors; i++) { + size_t this_size = 0; + struct ggml_tensor * t = tensors[i]; + if (t->data == NULL && t->view_src == NULL) { + this_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment); + } + + // flush the current buffer if adding this tensor would exceed max_size + if (cur_buf_size > 0 && (cur_buf_size + this_size) > max_size) { + plan.push_back({ cur_buf_size, first, i }); + cur_buf_size = this_size; + first = i; + } else { + cur_buf_size += this_size; + } + } + + if (cur_buf_size > 0) { + plan.push_back({ cur_buf_size, first, n_tensors }); + } + + return plan; +} + +// default implementation of alloc_buffer_n +// allocates tensors from a list into one or more buffers of the given type +static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + const ggml_backend_buft_alloc_buffer_n_plan_t plan = ggml_backend_buft_alloc_buffer_n_plan(buft, tensors, n_tensors); + + std::vector buffers; + buffers.reserve(plan.size()); + + for (const ggml_backend_buft_alloc_buffer_n_plan_item & item : plan) { + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, item.size); + if (buffer == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), item.size); + for (ggml_backend_buffer_t b : buffers) { + ggml_backend_buffer_free(b); + } + return NULL; + } + + struct ggml_tallocr tallocr = ggml_tallocr_new(buffer); + + // allocate tensors in the current buffer + struct ggml_tensor * t_failed = NULL; + for (int j = item.first; j < item.last; j++) { + struct ggml_tensor * t = tensors[j]; + if (t->data == NULL) { + if (t->view_src == NULL) { + if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) { + t_failed = t; + break; + } + } else if (t->buffer == NULL) { + if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) { + t_failed = t; + break; + } + } + } else { + if (t->view_src != NULL && t->buffer == NULL) { + // view of a pre-allocated tensor + if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) { + t_failed = t; + break; + } + } + } + } + if (t_failed != NULL) { + GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name); + for (ggml_backend_buffer_t b : buffers) { + ggml_backend_buffer_free(b); + } + ggml_backend_buffer_free(buffer); + return NULL; + } + + buffers.push_back(buffer); + } + + if (buffers.empty()) { + return NULL; + } + + if (buffers.size() == 1) { + return buffers[0]; + } + + return ggml_backend_multi_buffer_alloc_buffer(buffers.data(), buffers.size()); +} + +// default implementation of get_alloc_size_n +// returns the total size that alloc_buffer_n_default would allocate for the given tensors +static size_t ggml_backend_buft_get_alloc_size_n_default(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + const ggml_backend_buft_alloc_buffer_n_plan_t plan = ggml_backend_buft_alloc_buffer_n_plan(buft, tensors, n_tensors); + + size_t total = 0; + for (const ggml_backend_buft_alloc_buffer_n_plan_item & item : plan) { + total += item.size; + } + return total; +} + +ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + GGML_ASSERT(buft); + if (buft->iface.alloc_buffer_n) { + return buft->iface.alloc_buffer_n(buft, tensors, n_tensors); + } + return ggml_backend_buft_alloc_buffer_n_default(buft, tensors, n_tensors); +} + size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) { GGML_ASSERT(buft); return buft->iface.get_alignment(buft); @@ -64,11 +197,27 @@ size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const s if (buft->iface.get_alloc_size) { size_t size = buft->iface.get_alloc_size(buft, tensor); assert(size >= ggml_nbytes(tensor)); + + // [TAG_ALLOC_SIZE_EXPAND] + // if you hit this assert, update ggml_backend_op_alloc_size_may_expand() accordingly + GGML_ASSERT(size <= ggml_nbytes(tensor) || + ggml_op_is_empty(tensor->op) || + ggml_is_quantized(tensor->type) || // [TAG_ALLOC_SIZE_EXPAND] + ggml_op_alloc_size_may_expand(tensor->op)); + return size; } return ggml_nbytes(tensor); } +size_t ggml_backend_buft_get_alloc_size_n(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + GGML_ASSERT(buft); + if (buft->iface.get_alloc_size_n) { + return buft->iface.get_alloc_size_n(buft, tensors, n_tensors); + } + return ggml_backend_buft_get_alloc_size_n_default(buft, tensors, n_tensors); +} + bool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) { GGML_ASSERT(buft); if (buft->iface.is_host) { @@ -182,6 +331,8 @@ void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backe // FIXME: add a generic callback to the buffer interface if (ggml_backend_buffer_is_multi_buffer(buffer)) { ggml_backend_multi_buffer_set_usage(buffer, usage); + } else if (ggml_backend_buffer_is_meta(buffer)) { + ggml_backend_meta_buffer_set_usage(buffer, usage); } } @@ -306,7 +457,7 @@ void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_ GGML_ASSERT(tensor); GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); - if (n_copies <= 1 || backend->iface.set_tensor_2d_async == NULL) { + if (n_copies <= 1 || backend->iface.get_tensor_2d_async == NULL) { for (size_t i = 0; i < n_copies; i++) { ggml_backend_tensor_get_async(backend, tensor, (char *) data + i*stride_data, offset + i*stride_tensor, size); } @@ -317,7 +468,7 @@ void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_ } GGML_ASSERT(tensor->data != NULL && "tensor not allocated"); - GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor write out of bounds"); + GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor read out of bounds"); backend->iface.get_tensor_2d_async(backend, tensor, data, offset, size, n_copies, stride_tensor, stride_data); } @@ -379,7 +530,7 @@ void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; GGML_ASSERT(buf != NULL && "tensor buffer not set"); - if (n_copies <= 1 || buf->iface.set_tensor_2d == NULL) { + if (n_copies <= 1 || buf->iface.get_tensor_2d == NULL) { for (size_t i = 0; i < n_copies; i++) { ggml_backend_tensor_get(tensor, (char *) data + i*stride_data, offset + i*stride_tensor, size); } @@ -556,10 +707,10 @@ void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) backend->iface.event_wait(backend, event); } -static void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph) { +static void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params) { GGML_ASSERT(backend); if (backend->iface.graph_optimize != NULL) { - backend->iface.graph_optimize(backend, cgraph); + backend->iface.graph_optimize(backend, cgraph, params); } } @@ -765,8 +916,9 @@ struct ggml_backend_sched_split { int backend_id; int i_start; int i_end; - struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS]; + struct ggml_tensor ** inputs; int n_inputs; + int inputs_capacity; // graph view of this split struct ggml_cgraph graph; }; @@ -805,14 +957,21 @@ struct ggml_backend_sched { int cur_copy; int next_copy; ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES]; - struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS]; + struct ggml_tensor ** graph_inputs; int n_graph_inputs; + int graph_inputs_capacity; struct ggml_context * ctx; ggml_backend_sched_eval_callback callback_eval; void * callback_eval_user_data; + ggml_backend_sched_copy_callback callback_copy; + void * callback_copy_user_data; + + ggml_backend_sched_realloc_callback callback_realloc; + void * callback_realloc_user_data; + char * context_buffer; size_t context_buffer_size; @@ -820,11 +979,13 @@ struct ggml_backend_sched { int debug; - // used for debugging graph reallocations [GGML_SCHED_DEBUG_REALLOC] + // graph size of the current and the previous scheduling, used by the realloc callback and for debugging + // graph reallocations [GGML_SCHED_DEBUG_REALLOC] // ref: https://github.com/ggml-org/llama.cpp/pull/17617 + int graph_size; + int prev_graph_size; + int debug_realloc; - int debug_graph_size; - int debug_prev_graph_size; }; #define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor) @@ -832,6 +993,36 @@ struct ggml_backend_sched { #define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)] #define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id) +static void ggml_backend_sched_split_inputs_grow(struct ggml_backend_sched_split * split) { + int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS; + if (split->inputs_capacity > 0) { + new_cap = 2*split->inputs_capacity; + GGML_LOG_DEBUG("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap); + } + auto * pnew = (struct ggml_tensor **) realloc((void *) split->inputs, new_cap * sizeof(struct ggml_tensor *)); + if (pnew == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *)); + GGML_ABORT("failed to grow split inputs container"); + } + split->inputs = pnew; + split->inputs_capacity = new_cap; +} + +static void ggml_backend_sched_graph_inputs_grow(ggml_backend_sched_t sched) { + int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS; + if (sched->graph_inputs_capacity > 0) { + new_cap = 2*sched->graph_inputs_capacity; + GGML_LOG_DEBUG("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap); + } + auto * pnew = (struct ggml_tensor **) realloc((void *) sched->graph_inputs, new_cap * sizeof(struct ggml_tensor *)); + if (pnew == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *)); + GGML_ABORT("failed to grow graph inputs container"); + } + sched->graph_inputs = pnew; + sched->graph_inputs_capacity = new_cap; +} + // returns the priority of the backend, lower id is higher priority static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) { for (int i = 0; i < sched->n_backends; i++) { @@ -906,26 +1097,35 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, st } // operations with weights are preferably run on the same backend as the weights - for (int i = 0; i < GGML_MAX_SRC; i++) { - const struct ggml_tensor * src = tensor->src[i]; - if (src == NULL) { - continue; - } - // skip ROPE since the rope freqs tensor is too small to choose a backend based on it - // not an ideal solution - if (tensor->op != GGML_OP_ROPE && src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor); - // check if a backend with higher prio wants to offload the op - if (sched->op_offload && src_backend_id == sched->n_backends - 1 && ggml_backend_buffer_is_host(src->buffer)) { - for (int b = 0; b < src_backend_id; b++) { - if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) { - SET_CAUSE(tensor, "1.off"); - return b; + // TODO: there are exceptions (see below) - not an ideal solution + bool allow = true; + + // skip ROPE since the rope freqs tensor is too small to choose a backend based on it + allow = allow && tensor->op != GGML_OP_ROPE; + + // skip FLASH_ATTN_EXT since the sinks tensor is too small to choose a based based on it + allow = allow && tensor->op != GGML_OP_FLASH_ATTN_EXT; + + if (allow) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + const struct ggml_tensor * src = tensor->src[i]; + if (src == NULL) { + continue; + } + if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor); + // check if a backend with higher prio wants to offload the op + if (sched->op_offload && src_backend_id == sched->n_backends - 1 && ggml_backend_buffer_is_host(src->buffer)) { + for (int b = 0; b < src_backend_id; b++) { + if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) { + SET_CAUSE(tensor, "1.off"); + return b; + } } } + SET_CAUSE(tensor, "1.wgt%d", i); + return src_backend_id; } - SET_CAUSE(tensor, "1.wgt%d", i); - return src_backend_id; } } @@ -965,7 +1165,7 @@ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, str } if (sched->debug > 1) { ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node); - GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:", i, ggml_op_name(node->op), node->name, + GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:", i, ggml_op_desc(node), node->name, fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node), graph->use_counts[ggml_hash_find(&graph->visited_hash_set, node)], node->flags & GGML_TENSOR_FLAG_COMPUTE ? 1 : 0); for (int j = 0; j < GGML_MAX_SRC; j++) { @@ -1286,17 +1486,6 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra break; } } - // check if the split has too many inputs - // FIXME: count the number of inputs instead of only checking when full - if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) { - const size_t id = hash_id(src); - int src_backend_id = sched->hv_tensor_backend_ids[id]; - bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id); - if (src_backend_id != cur_backend_id && tensor_id_copy(id, cur_backend_id, 0) == NULL && !supported) { - need_new_split = true; - break; - } - } } } @@ -1304,10 +1493,14 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra split->i_end = i; i_split++; if (i_split >= sched->splits_capacity) { + int old_cap = sched->splits_capacity; sched->splits_capacity *= 2; sched->splits = (ggml_backend_sched_split *) realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split)); GGML_ASSERT(sched->splits != NULL); + for (int k = old_cap; k < sched->splits_capacity; k++) { + memset(&sched->splits[k], 0, sizeof(struct ggml_backend_sched_split)); + } } split = &sched->splits[i_split]; split->backend_id = node_backend_id; @@ -1327,28 +1520,6 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra const int src_backend_id = sched->hv_tensor_backend_ids[src_id]; GGML_ASSERT(src_backend_id != -1); // all inputs should be assigned by now - if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) { - if (tensor_id_copy(src_id, src_backend_id, 0) == NULL) { - ggml_backend_t backend = sched->backends[src_backend_id]; - for (int c = 0; c < sched->n_copies; c++) { - struct ggml_tensor * tensor_copy; - if (c == sched->cur_copy) { - tensor_copy = src; // use the original tensor as the current copy - } else { - tensor_copy = ggml_dup_tensor_layout(sched->ctx, src); - ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c); - } - ggml_set_input(tensor_copy); - ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor - tensor_id_copy(src_id, src_backend_id, c) = tensor_copy; - SET_CAUSE(tensor_copy, "4.cpy"); - } - int n_graph_inputs = sched->n_graph_inputs++; - GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS); - sched->graph_inputs[n_graph_inputs] = src; - } - } - if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) { // create a copy of the input in the split's backend if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) { @@ -1364,7 +1535,9 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra SET_CAUSE(tensor_copy, "4.cpy"); } int n_inputs = split->n_inputs++; - GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS); + if (n_inputs >= split->inputs_capacity) { + ggml_backend_sched_split_inputs_grow(split); + } split->inputs[n_inputs] = src; } node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy); @@ -1379,6 +1552,46 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra ggml_backend_sched_print_assignments(sched, graph); } + // pass 6: collect all input tensors into graph_inputs + // this includes inputs not consumed by any node (e.g. the embeddings input of a text-only batch) so that + // the graph composition does not depend on which inputs are used, which would otherwise cause graph + // reallocations when switching between different types of batches [GGML_SCHED_DEBUG_REALLOC] + if (sched->n_copies > 1) { + for (int i = 0; i < graph->n_leafs; i++) { + struct ggml_tensor * leaf = graph->leafs[i]; + if ((leaf->flags & GGML_TENSOR_FLAG_INPUT) == 0) { + continue; + } + + const size_t leaf_id = hash_id(leaf); + const int leaf_backend_id = tensor_backend_id(leaf); + GGML_ASSERT(leaf_backend_id != -1); // all leafs should be assigned by now + + if (tensor_id_copy(leaf_id, leaf_backend_id, 0) == NULL) { + ggml_backend_t backend = sched->backends[leaf_backend_id]; + for (int c = 0; c < sched->n_copies; c++) { + struct ggml_tensor * tensor_copy; + if (c == sched->cur_copy) { + tensor_copy = leaf; // use the original tensor as the current copy + } else { + tensor_copy = ggml_dup_tensor_layout(sched->ctx, leaf); + ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), leaf->name, c); + } + ggml_set_input(tensor_copy); + ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor + tensor_id_copy(leaf_id, leaf_backend_id, c) = tensor_copy; + SET_CAUSE(tensor_copy, "6.cpy"); + } + } + + int n_graph_inputs = sched->n_graph_inputs++; + if (n_graph_inputs >= sched->graph_inputs_capacity) { + ggml_backend_sched_graph_inputs_grow(sched); + } + sched->graph_inputs[n_graph_inputs] = leaf; + } + } + // swap node_backend_ids and leaf _backend_ids with prevs { int * tmp = sched->node_backend_ids; @@ -1390,11 +1603,44 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra sched->prev_leaf_backend_ids = tmp; } - int graph_size = std::max(graph->n_nodes, graph->n_leafs) + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sched->n_copies; + // optimize the split graphs and collect the allocation dependencies added by the backends + // this needs to happen before we make graph_copy, so they are in sync + // TODO: this may create many small allocations in the scheduler, restructure to use a flat array + std::unordered_map> alloc_deps; + + struct ggml_backend_graph_optimize_params opt_params = { + /* .add_alloc_dep = */ [](void * user_data, ggml_tensor * tensor, ggml_tensor * until) { + auto & deps = *(std::unordered_map> *) user_data; + std::vector & keep = deps[until]; + if (std::find(keep.begin(), keep.end(), tensor) == keep.end()) { + keep.push_back(tensor); + } + }, + /* .user_data = */ &alloc_deps, + }; + + for (int i = 0; i < sched->n_splits; i++) { + struct ggml_backend_sched_split * split = &sched->splits[i]; + split->graph = ggml_graph_view(graph, split->i_start, split->i_end); - // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC] - sched->debug_prev_graph_size = sched->debug_graph_size; - sched->debug_graph_size = graph_size; + ggml_backend_graph_optimize(sched->backends[split->backend_id], &split->graph, &opt_params); + } + + // each dep is added to graph_copy as a GGML_OP_NONE node with the kept tensors as srcs + int n_dep_nodes = 0; + for (const auto & it : alloc_deps) { + n_dep_nodes += (it.second.size() + GGML_MAX_SRC - 1) / GGML_MAX_SRC; + } + + int total_inputs = sched->n_graph_inputs; + for (int i = 0; i < sched->n_splits; i++) { + total_inputs += sched->splits[i].n_inputs; + } + int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies + n_dep_nodes; + + // remember the actual graph_size for performing reallocation checks later + sched->prev_graph_size = sched->graph_size; + sched->graph_size = graph_size; if (sched->graph.size < graph_size) { sched->graph.size = graph_size; @@ -1408,13 +1654,10 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra struct ggml_cgraph * graph_copy = &sched->graph; + int n_dep_nodes_added = 0; + for (int i = 0; i < sched->n_splits; i++) { struct ggml_backend_sched_split * split = &sched->splits[i]; - split->graph = ggml_graph_view(graph, split->i_start, split->i_end); - - // Optimize this split of the graph. This needs to happen before we make graph_copy, - // so they are in sync. - ggml_backend_graph_optimize(sched->backends[split->backend_id], &split->graph); // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split for (int j = 0; j < split->n_inputs; j++) { @@ -1439,9 +1682,32 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra assert(graph_copy->size > graph_copy->n_nodes); sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]); graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j]; + + if (alloc_deps.empty()) { + continue; + } + + // add a dependency node so that the kept tensors are not freed before this node is computed + auto it = alloc_deps.find(graph->nodes[j]); + if (it != alloc_deps.end()) { + const std::vector & keep = it->second; + for (size_t k = 0; k < keep.size(); k += GGML_MAX_SRC) { + struct ggml_tensor * dep = ggml_view_tensor(sched->ctx, keep[k]); + for (size_t s = 0; s < GGML_MAX_SRC && k + s < keep.size(); s++) { + dep->src[s] = keep[k + s]; + } + assert(graph_copy->size > graph_copy->n_nodes); + sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id; + graph_copy->nodes[graph_copy->n_nodes++] = dep; + n_dep_nodes_added++; + } + } } } + // a mismatch means a backend added a dep with an `until` tensor that is not a node of the optimized graph + GGML_ASSERT(n_dep_nodes_added == n_dep_nodes); + if (sched->n_copies > 1) { // add input copies as leafs so that they are allocated first for (int i = 0; i < sched->n_graph_inputs; i++) { @@ -1511,14 +1777,21 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { GGML_LOG_DEBUG("%s: failed to allocate graph, reserving (backend_ids_changed = %d)\n", __func__, backend_ids_changed); #endif + // a reallocation with an unchanged graph size means that the previous reservation was not sufficient + const bool graph_size_changed = sched->prev_graph_size != sched->graph_size; + + if (sched->callback_realloc != NULL) { + sched->callback_realloc(sched, &sched->graph, backend_ids_changed, graph_size_changed, sched->callback_realloc_user_data); + } + if (sched->debug_realloc > 0) { // we are interested only in situations where the graph was reallocated even though its size remained the same [GGML_SCHED_DEBUG_REALLOC] // example: https://github.com/ggml-org/llama.cpp/pull/17143 - const bool unexpected = !backend_ids_changed && sched->debug_prev_graph_size == sched->debug_graph_size; + const bool unexpected = !backend_ids_changed && !graph_size_changed; if (unexpected || sched->debug_realloc > 1) { GGML_ABORT("%s: unexpected graph reallocation (graph size = %d, nodes = %d, leafs = %d), debug_realloc = %d\n", __func__, - sched->debug_graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc); + sched->graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc); } } @@ -1528,7 +1801,10 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { ggml_backend_synchronize(sched->backends[i]); } - ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids); + if (!ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) { + GGML_LOG_ERROR("%s: failed to reserve graph buffers\n", __func__); + return false; + } if (!ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) { GGML_LOG_ERROR("%s: failed to allocate graph\n", __func__); return false; @@ -1538,139 +1814,85 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { return true; } +static bool ggml_backend_sched_is_host_weight(const struct ggml_tensor * t) { + return t->buffer != NULL && + ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + ggml_backend_buffer_is_host(t->buffer); +} + +static void ggml_backend_sched_copy_input(ggml_backend_sched_t sched, struct ggml_backend_sched_split * split, struct ggml_tensor * input) { + const int split_backend_id = split->backend_id; + ggml_backend_t split_backend = sched->backends[split_backend_id]; + ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, input); + struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy); + + if (input->flags & GGML_TENSOR_FLAG_INPUT) { + // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(split_backend); + } + ggml_backend_tensor_copy(input, input_cpy); + return; + } + + // wait for the split backend to finish using the input before overwriting it + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(split_backend); + } + + if (sched->callback_copy != NULL && ggml_backend_sched_is_host_weight(input) && + sched->callback_copy(split_backend, input, input_cpy, &split->graph, sched->callback_copy_user_data)) { + return; + } + + // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events + // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface + if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) { + ggml_backend_synchronize(input_backend); + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); + } else { + ggml_backend_synchronize(split_backend); + } + ggml_backend_tensor_copy(input, input_cpy); + } +} + static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { GGML_ASSERT(sched); struct ggml_backend_sched_split * splits = sched->splits; - ggml_tensor * prev_ids_tensor = nullptr; - std::vector ids; - std::vector used_ids; + int prev_backend_id = -1; for (int split_id = 0; split_id < sched->n_splits; split_id++) { struct ggml_backend_sched_split * split = &splits[split_id]; int split_backend_id = split->backend_id; ggml_backend_t split_backend = sched->backends[split_backend_id]; - // copy the input tensors to the split backend - for (int input_id = 0; input_id < split->n_inputs; input_id++) { - ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); - struct ggml_tensor * input = split->inputs[input_id]; - struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy); - - if (input->flags & GGML_TENSOR_FLAG_INPUT) { - // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); - } else { - ggml_backend_synchronize(split_backend); - } - ggml_backend_tensor_copy(input, input_cpy); + // ensure the previous split's async work has completed before we start + // this split, the allocator may have reused buffer regions across splits + if (split->n_inputs == 0 && prev_backend_id >= 0 && prev_backend_id != split_backend_id) { + if (sched->events[prev_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_synchronize(sched->events[prev_backend_id][sched->cur_copy]); } else { - // wait for the split backend to finish using the input before overwriting it - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]); - } else { - ggml_backend_synchronize(split_backend); - } - - // when offloading MoE weights, we can reduce the amount of data copied by copying only the experts that are used - ggml_tensor * node = split->graph.nodes[0]; - if (split->graph.n_nodes > 0 && - ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && - ggml_backend_buffer_is_host(input->buffer) && ( - (node->src[0] == input_cpy && node->op == GGML_OP_MUL_MAT_ID) - //|| (node->src[1] == input_cpy && node->op == GGML_OP_ADD_ID) /* GGML_OP_ADD_ID weights are small and not worth splitting */ - )) { - - const int64_t n_expert = node->op == GGML_OP_MUL_MAT_ID ? input->ne[2] : input->ne[1]; - const size_t expert_size = node->op == GGML_OP_MUL_MAT_ID ? input->nb[2] : input->nb[1]; - - ggml_backend_synchronize(input_backend); - - // get the ids - ggml_tensor * ids_tensor = node->src[2]; - ggml_backend_t ids_backend = split_backend; - - // if the ids tensor is also an input of the split, it may not have been copied yet to the split backend - // in that case, we use the original ids tensor - for (int i = input_id + 1; i < split->n_inputs; i++) { - if (ids_tensor == tensor_copy(split->inputs[i], split_backend_id, sched->cur_copy)) { - ids_tensor = split->inputs[i]; - ids_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[i]); - break; - } - } - - if (ids_tensor != prev_ids_tensor) { - ids.resize(ggml_nbytes(ids_tensor) / sizeof(int32_t)); - ggml_backend_tensor_get_async(ids_backend, ids_tensor, ids.data(), 0, ggml_nbytes(ids_tensor)); - ggml_backend_synchronize(ids_backend); - - // find the used experts - used_ids.clear(); - used_ids.resize(ggml_bitset_size(n_expert)); - for (int64_t i1 = 0; i1 < ids_tensor->ne[1]; i1++) { - for (int64_t i0 = 0; i0 < ids_tensor->ne[0]; i0++) { - int32_t id = ids[i1 * ids_tensor->nb[1]/sizeof(int32_t) + i0 * ids_tensor->nb[0]/sizeof(int32_t)]; - GGML_ASSERT(id >= 0 && id < n_expert); - ggml_bitset_set(used_ids.data(), id); - } - } - - prev_ids_tensor = ids_tensor; - } - - // group consecutive experts and copy them together - auto copy_experts = [&](int32_t first_id, int32_t last_id) { - const size_t expert_offset = first_id * expert_size; - const size_t expert_size_copy = (last_id - first_id + 1) * expert_size; - const size_t padding = std::min(expert_size, 512); - const size_t padding_end = last_id < n_expert - 1 ? padding : 0; - - ggml_backend_tensor_set_async(split_backend, - input_cpy, - (const uint8_t *)input->data + expert_offset, expert_offset, - // copy a bit extra at the to ensure there are no NaNs in the padding of the last expert - // this is necessary for MMQ in the CUDA backend - expert_size_copy + padding_end); - }; - - int id = 0; - while (!ggml_bitset_get(used_ids.data(), id)) { - id++; - } - int32_t first_id = id; - int32_t last_id = first_id; - - for (++id; id < n_expert; ++id) { - if (!ggml_bitset_get(used_ids.data(), id)) { - continue; - } - - if (id == last_id + 1) { - last_id = id; - continue; - } - - copy_experts(first_id, last_id); + ggml_backend_synchronize(sched->backends[prev_backend_id]); + } + } - first_id = id; - last_id = id; - } - copy_experts(first_id, last_id); - } else { - // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events - // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface - if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) { - ggml_backend_synchronize(input_backend); - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]); - } else { - ggml_backend_synchronize(split_backend); - } - ggml_backend_tensor_copy(input, input_cpy); - } - } + // copy the input tensors to the split backend + // the weights in host memory are copied last, so that the copy callback can read the other inputs of the split + for (int input_id = 0; input_id < split->n_inputs; input_id++) { + if (!ggml_backend_sched_is_host_weight(split->inputs[input_id])) { + ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]); + } + } + for (int input_id = 0; input_id < split->n_inputs; input_id++) { + if (ggml_backend_sched_is_host_weight(split->inputs[input_id])) { + ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]); } } @@ -1713,12 +1935,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } } - // record the event of this copy - if (split->n_inputs > 0) { - if (sched->events[split_backend_id][sched->cur_copy] != NULL) { - ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend); - } + // record the event of this split + if (sched->events[split_backend_id][sched->cur_copy] != NULL) { + ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend); } + + prev_backend_id = split_backend_id; } return GGML_STATUS_SUCCESS; @@ -1763,8 +1985,8 @@ ggml_backend_sched_t ggml_backend_sched_new( sched->prev_node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0])); sched->prev_leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0])); - sched->debug_graph_size = 0; - sched->debug_prev_graph_size = 0; + sched->graph_size = 0; + sched->prev_graph_size = 0; sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false); sched->context_buffer = (char *) malloc(sched->context_buffer_size); @@ -1773,6 +1995,9 @@ ggml_backend_sched_t ggml_backend_sched_new( sched->splits = (ggml_backend_sched_split *) calloc(initial_splits_capacity, sizeof(sched->splits[0])); sched->splits_capacity = initial_splits_capacity; + sched->graph_inputs_capacity = GGML_SCHED_MAX_SPLIT_INPUTS; + sched->graph_inputs = (struct ggml_tensor **) calloc(sched->graph_inputs_capacity, sizeof(struct ggml_tensor *)); + for (int b = 0; b < n_backends; b++) { sched->backends[b] = backends[b]; sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]); @@ -1805,7 +2030,11 @@ void ggml_backend_sched_free(ggml_backend_sched_t sched) { ggml_gallocr_free(sched->galloc); ggml_free(sched->ctx); ggml_hash_set_free(&sched->hash_set); + for (int i = 0; i < sched->splits_capacity; i++) { + free(sched->splits[i].inputs); + } free(sched->splits); + free(sched->graph_inputs); free(sched->hv_tensor_backend_ids); free(sched->hv_tensor_copies); free(sched->node_backend_ids); @@ -1920,6 +2149,18 @@ void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backe sched->callback_eval_user_data = user_data; } +void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data) { + GGML_ASSERT(sched); + sched->callback_copy = callback; + sched->callback_copy_user_data = user_data; +} + +void ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data) { + GGML_ASSERT(sched); + sched->callback_realloc = callback; + sched->callback_realloc_user_data = user_data; +} + int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) { GGML_ASSERT(sched); return sched->n_splits; @@ -1977,6 +2218,21 @@ ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, // utils +bool ggml_op_alloc_size_may_expand(enum ggml_op op) { + switch (op) { + case GGML_OP_FLASH_ATTN_EXT: + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + case GGML_OP_CUMSUM: + case GGML_OP_ARGSORT: + case GGML_OP_TOP_K: + case GGML_OP_ARGMAX: + return true; + default: + return false; + } +} + enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor) { GGML_ASSERT(tensor); GGML_ASSERT(tensor->buffer == NULL); @@ -2328,12 +2584,14 @@ static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) { static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = { /* .iface = */ { - /* .get_name = */ ggml_backend_cpu_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes - /* .is_host = */ ggml_backend_cpu_buffer_type_is_host, + /* .get_name = */ ggml_backend_cpu_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_cpu_buffer_type_is_host, }, /* .device = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0), /* .context = */ NULL, @@ -2351,12 +2609,14 @@ static const char * ggml_backend_cpu_buffer_from_ptr_type_get_name(ggml_backend_ static ggml_backend_buffer_type_t ggml_backend_cpu_buffer_from_ptr_type(void) { static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = { /* .iface = */ { - /* .get_name = */ ggml_backend_cpu_buffer_from_ptr_type_get_name, - /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes - /* .is_host = */ ggml_backend_cpu_buffer_type_is_host, + /* .get_name = */ ggml_backend_cpu_buffer_from_ptr_type_get_name, + /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_cpu_buffer_type_is_host, }, /* .device = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0), /* .context = */ NULL, diff --git a/ggml/src/ggml-blas/CMakeLists.txt b/ggml/src/ggml-blas/CMakeLists.txt index c27dc17..5b46b79 100644 --- a/ggml/src/ggml-blas/CMakeLists.txt +++ b/ggml/src/ggml-blas/CMakeLists.txt @@ -84,10 +84,17 @@ if (BLAS_FOUND) add_compile_definitions(GGML_BLAS_USE_OPENBLAS) endif() + # Vendors with the BLIS API take GGML_BLAS_USE_BLIS (blis.h, bli_thread_set_num_threads). + # AOCL also takes GGML_BLAS_USE_AOCL so its device label is AOCL-BLAS, not BLIS. + # Neither flag selects the linked library; FindBLAS does that from GGML_BLAS_VENDOR. if ("${GGML_BLAS_VENDOR}" MATCHES "FLAME" OR "${GGML_BLAS_VENDOR}" MATCHES "AOCL" OR "${GGML_BLAS_VENDOR}" MATCHES "AOCL_mt") add_compile_definitions(GGML_BLAS_USE_BLIS) endif() + if ("${GGML_BLAS_VENDOR}" MATCHES "AOCL" OR "${GGML_BLAS_VENDOR}" MATCHES "AOCL_mt") + add_compile_definitions(GGML_BLAS_USE_AOCL) + endif() + if ("${GGML_BLAS_VENDOR}" MATCHES "NVPL") add_compile_definitions(GGML_BLAS_USE_NVPL) endif() diff --git a/ggml/src/ggml-blas/ggml-blas.cpp b/ggml/src/ggml-blas/ggml-blas.cpp index 05245b6..6feb370 100644 --- a/ggml/src/ggml-blas/ggml-blas.cpp +++ b/ggml/src/ggml-blas/ggml-blas.cpp @@ -1,3 +1,4 @@ +#include "ggml.h" #include "ggml-impl.h" #include "ggml-blas.h" #include "ggml-backend-impl.h" @@ -262,9 +263,9 @@ static struct ggml_backend_i blas_backend_i = { /* .get_name = */ ggml_backend_blas_get_name, /* .free = */ ggml_backend_blas_free, /* .set_tensor_async = */ NULL, - /* .get_tensor_2d_async = */ NULL, - /* .set_tensor_2d_async = */ NULL, /* .get_tensor_async = */ NULL, + /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, /* .cpy_tensor_async = */ NULL, /* .synchronize = */ NULL, /* .graph_plan_create = */ NULL, @@ -329,6 +330,8 @@ static const char * ggml_backend_blas_device_get_description(ggml_backend_dev_t return "Accelerate"; #elif defined(GGML_BLAS_USE_MKL) return "MKL"; + #elif defined(GGML_BLAS_USE_AOCL) + return "AOCL-BLAS"; #elif defined(GGML_BLAS_USE_BLIS) return "BLIS"; #elif defined(GGML_BLAS_USE_NVPL) @@ -366,6 +369,7 @@ static void ggml_backend_blas_device_get_props(ggml_backend_dev_t dev, struct gg /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ true, /* .events = */ false, + /* .mmap_support = */ true, }; } @@ -415,6 +419,12 @@ static bool ggml_backend_blas_device_supports_op(ggml_backend_dev_t dev, const s // TODO: find the optimal value const int64_t min_batch = 32; + // default back to CPU fast path + // see: https://github.com/ggml-org/llama.cpp/issues/25565 + if (ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD) { + return false; + } + return ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && src1->type == GGML_TYPE_F32 && diff --git a/ggml/src/ggml-cann/aclnn_ops.cpp b/ggml/src/ggml-cann/aclnn_ops.cpp index a950475..902d2ed 100644 --- a/ggml/src/ggml-cann/aclnn_ops.cpp +++ b/ggml/src/ggml-cann/aclnn_ops.cpp @@ -25,6 +25,7 @@ #include "ggml-impl.h" #include "ggml.h" + #include #include #include @@ -45,7 +46,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -62,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -69,11 +73,15 @@ #include #include #include +#include #include #include #include #include +#include #include +#include +#include #include #include #include @@ -151,6 +159,151 @@ void ggml_cann_op_unary_gated(std::functionsrc[1] != nullptr || swapped != 0) { + ggml_cann_op_unary_gated(silu_fn, ctx, dst); + return; + } + + // aclnnSwiGlu requires the split dim (src->ne[0]) to be even; fall back otherwise. + if (dst->src[0]->ne[0] % 2 != 0) { + ggml_cann_op_unary_gated(silu_fn, ctx, dst); + return; + } + + ggml_tensor * src0 = dst->src[0]; + size_t elem_size = ggml_element_size(src0); + + // src0 GGML: [2*ne0, ne1, ne2, ne3] → 3D view [2*ne0, ne1, ne2*ne3] + // CANN reversed: [ne2*ne3, ne1, 2*ne0], split along CANN dim 2 (last). + int64_t ne0_x2 = src0->ne[0]; + int64_t ne1 = src0->ne[1]; + int64_t ne23 = src0->ne[2] * src0->ne[3]; + int64_t src3d_ne[] = { ne0_x2, ne1, ne23 }; + size_t src3d_nb[] = { (size_t)src0->nb[0], (size_t)src0->nb[1], (size_t)src0->nb[2] }; + acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0->data, ggml_cann_type_mapping(src0->type), + elem_size, src3d_ne, src3d_nb, 3); + + // dst GGML: [ne0, ne1, ne2, ne3] → 3D view [ne0, ne1, ne2*ne3] + int64_t ne0 = dst->ne[0]; + int64_t dst3d_ne[] = { ne0, ne1, ne23 }; + size_t dst3d_nb[] = { (size_t)dst->nb[0], (size_t)dst->nb[1], (size_t)dst->nb[2] }; + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst->data, ggml_cann_type_mapping(dst->type), + elem_size, dst3d_ne, dst3d_nb, 3); + + // CANN tensor [ne23, ne1, 2*ne0]: split along CANN dim 2 (last) = 2*ne0. + GGML_CANN_CALL_ACLNN_OP(ctx, SwiGlu, acl_src.get(), (int64_t)2, acl_dst.get()); +} + +void ggml_cann_swiglu_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst) { + ggml_tensor * src0 = dst->src[0]; + ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(ggml_is_contiguous_1(dst)); + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + acl_tensor_ptr acl_gate; + acl_tensor_ptr acl_up; + if (src1) { + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src0->type == src1->type); + acl_gate = ggml_cann_create_tensor(src0); + acl_up = ggml_cann_create_tensor(src1); + } else { + int64_t ne[] = { src0->ne[0] / 2, src0->ne[1], src0->ne[2], src0->ne[3] }; + size_t nb[] = { src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3] }; + acl_gate = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, 0); + acl_up = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, ne[0] * ggml_element_size(src0)); + if (swapped) { + std::swap(acl_gate, acl_up); + } + } + + ggml_cann_pool_alloc temp_alloc(ctx.pool(), ggml_nbytes(dst)); + acl_tensor_ptr acl_temp = ggml_cann_create_tensor(temp_alloc.get(), ggml_cann_type_mapping(dst->type), + ggml_element_size(dst), dst->ne, dst->nb, GGML_MAX_DIMS); + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); + + const float limit = ggml_get_op_params_f32(dst, 3); + float min_gate = -INFINITY; + float min_up = -limit; + float max_value = limit; + acl_scalar_ptr acl_min_gate = ggml_cann_create_scalar(&min_gate, ACL_FLOAT); + acl_scalar_ptr acl_min_up = ggml_cann_create_scalar(&min_up, ACL_FLOAT); + acl_scalar_ptr acl_limit = ggml_cann_create_scalar(&max_value, ACL_FLOAT); + + GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_gate.get(), acl_min_gate.get(), acl_limit.get(), acl_temp.get()); + GGML_CANN_CALL_ACLNN_OP(ctx, Silu, acl_temp.get(), acl_dst.get()); + GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_up.get(), acl_min_up.get(), acl_limit.get(), acl_temp.get()); + GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, acl_dst.get(), acl_temp.get()); +} + +// Fused GeGLU using aclnnGeGluV3: splits input along ne[0] (CANN last dim), +// activates the LEFT half with GELU, multiplies by right half. +// approximate: 0=tanh, 1=none(erf). activateLeft=true matches GGML convention. +// outGelu is a required-but-discard output buffer. +// +// Falls back to the generic two-kernel path when src[1] != nullptr (two +// independent halves) or swapped != 0 (reversed activation order), as +// aclnnGeGluV3 only handles the single interleaved tensor in standard order. +void ggml_cann_geglu(ggml_backend_cann_context & ctx, ggml_tensor * dst, int64_t approximate) { + auto gelu_fn = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) { + GGML_CANN_CALL_ACLNN_OP(ctx, Gelu, acl_src, acl_dst); + }; + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + if (dst->src[1] != nullptr || swapped != 0) { + ggml_cann_op_unary_gated(gelu_fn, ctx, dst); + return; + } + + // aclnnGeGluV3 requires the split dim (src->ne[0]) to be even; fall back otherwise. + if (dst->src[0]->ne[0] % 2 != 0) { + ggml_cann_op_unary_gated(gelu_fn, ctx, dst); + return; + } + + ggml_tensor * src0 = dst->src[0]; + acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0); + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); + + // Allocate a temporary buffer for the required outGelu output (same shape as dst). + // Build contiguous strides since the pool allocation is a fresh buffer. + size_t elem_size = ggml_element_size(dst); + int64_t ne[GGML_MAX_DIMS] = { dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3] }; + size_t nb[GGML_MAX_DIMS]; + nb[0] = elem_size; + for (int i = 1; i < GGML_MAX_DIMS; i++) { + nb[i] = nb[i - 1] * ne[i - 1]; + } + size_t gelu_out_size = nb[GGML_MAX_DIMS - 1] * ne[GGML_MAX_DIMS - 1]; + ggml_cann_pool_alloc gelu_out_alloc(ctx.pool(), gelu_out_size); + + acl_tensor_ptr acl_gelu_out = ggml_cann_create_tensor( + gelu_out_alloc.get(), ggml_cann_type_mapping(dst->type), elem_size, ne, nb, GGML_MAX_DIMS); + // V3 adds activateLeft param; true → Gelu(left)*right, matching GGML convention. + // GGML dim 0 → CANN last dim (index GGML_MAX_DIMS-1 = 3 for 4D tensor). + GGML_CANN_CALL_ACLNN_OP(ctx, GeGluV3, acl_src.get(), (int64_t)(GGML_MAX_DIMS - 1), approximate, true, + acl_dst.get(), acl_gelu_out.get()); +} + /** * @brief Repeats elements of a tensor along each dimension according to the * specified repeat array. @@ -445,28 +598,33 @@ void ggml_cann_l2_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) { ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), n_bytes); void * buffer = temp_buffer_allocator.get(); - int64_t div_ne[] = { 1, src->ne[1], src->ne[2], src->ne[3] }; - size_t div_nb[GGML_MAX_DIMS]; - div_nb[0] = sizeof(float); + int64_t norm_ne[] = { 1, src->ne[1], src->ne[2], src->ne[3] }; + size_t norm_nb[GGML_MAX_DIMS]; + norm_nb[0] = sizeof(float); for (int i = 1; i < GGML_MAX_DIMS; ++i) { - div_nb[i] = div_nb[i - 1] * div_ne[i - 1]; + norm_nb[i] = norm_nb[i - 1] * norm_ne[i - 1]; } - acl_tensor_ptr acl_div = ggml_cann_create_tensor(buffer, ACL_FLOAT, type_size, div_ne, div_nb, GGML_MAX_DIMS); + acl_tensor_ptr acl_norm = ggml_cann_create_tensor(buffer, ACL_FLOAT, sizeof(float), norm_ne, norm_nb, GGML_MAX_DIMS); std::vector norm_dims = { 3 }; acl_int_array_ptr dims_array = ggml_cann_create_int_array(norm_dims.data(), norm_dims.size()); float p_value = 2.0f; acl_scalar_ptr p_scalar = ggml_cann_create_scalar(&p_value, aclDataType::ACL_FLOAT); - GGML_CANN_CALL_ACLNN_OP(ctx, Norm, acl_src.get(), p_scalar.get(), dims_array.get(), true, acl_div.get()); + GGML_CANN_CALL_ACLNN_OP(ctx, Norm, acl_src.get(), p_scalar.get(), dims_array.get(), true, acl_norm.get()); - // Clamp norm to at least eps: scale = 1/fmaxf(norm, eps) - acl_scalar_ptr acl_min = ggml_cann_create_scalar(&eps, aclDataType::ACL_FLOAT); - float flt_max = FLT_MAX; - acl_scalar_ptr acl_max = ggml_cann_create_scalar(&flt_max, aclDataType::ACL_FLOAT); - GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_div.get(), acl_min.get(), acl_max.get(), acl_div.get()); + ggml_cann_pool_alloc clamp_buffer_allocator(ctx.pool()); + acl_tensor_ptr acl_clamped; - GGML_CANN_CALL_ACLNN_OP(ctx, Div, acl_src.get(), acl_div.get(), acl_dst.get()); + if (eps > 0.0f) { + void * clamp_buf = clamp_buffer_allocator.alloc(n_bytes); + acl_clamped = ggml_cann_create_tensor(clamp_buf, ACL_FLOAT, sizeof(float), norm_ne, norm_nb, GGML_MAX_DIMS); + acl_scalar_ptr eps_scalar = ggml_cann_create_scalar(&eps, aclDataType::ACL_FLOAT); + GGML_CANN_CALL_ACLNN_OP(ctx, ClampMin, acl_norm.get(), eps_scalar.get(), acl_clamped.get()); + } + + aclTensor * acl_div_input = acl_clamped ? acl_clamped.get() : acl_norm.get(); + GGML_CANN_CALL_ACLNN_OP(ctx, Div, acl_src.get(), acl_div_input, acl_dst.get()); } void ggml_cann_cross_entropy_loss(ggml_backend_cann_context & ctx, ggml_tensor * dst) { @@ -482,56 +640,30 @@ void ggml_cann_cross_entropy_loss(ggml_backend_cann_context & ctx, ggml_tensor * logits_nb[1] = logits_nb[0] * logits_ne[0]; acl_tensor_ptr acl_logits = ggml_cann_create_tensor(src0->data, ACL_FLOAT, sizeof(float), logits_ne, logits_nb, 2); - size_t log_softmax_type_size = sizeof(float); - int64_t log_softmax_n_bytes = nr * nc * log_softmax_type_size; - ggml_cann_pool_alloc log_softmax_allocator(ctx.pool(), log_softmax_n_bytes); - void * log_softmax_buffer = log_softmax_allocator.get(); - - int64_t log_softmax_ne[] = { nc, nr }; - size_t log_softmax_nb[2]; - log_softmax_nb[0] = log_softmax_type_size; - log_softmax_nb[1] = log_softmax_nb[0] * log_softmax_ne[0]; - acl_tensor_ptr acl_log_softmax = ggml_cann_create_tensor(log_softmax_buffer, ACL_FLOAT, log_softmax_type_size, - log_softmax_ne, log_softmax_nb, 2); - - GGML_CANN_CALL_ACLNN_OP(ctx, LogSoftmax, acl_logits.get(), 1, acl_log_softmax.get()); - int64_t labels_ne[] = { nc, nr }; size_t labels_nb[2]; labels_nb[0] = ggml_type_size(src1->type); labels_nb[1] = labels_nb[0] * labels_ne[0]; acl_tensor_ptr acl_labels = ggml_cann_create_tensor(src1->data, ACL_FLOAT, sizeof(float), labels_ne, labels_nb, 2); - size_t mul_type_size = sizeof(float); - int64_t mul_n_bytes = nr * nc * mul_type_size; - ggml_cann_pool_alloc mul_allocator(ctx.pool(), mul_n_bytes); - void * mul_buffer = mul_allocator.get(); - - int64_t mul_ne[] = { nc, nr }; - size_t mul_nb[2]; - mul_nb[0] = mul_type_size; - mul_nb[1] = mul_nb[0] * mul_ne[0]; - acl_tensor_ptr acl_mul_result = ggml_cann_create_tensor(mul_buffer, ACL_FLOAT, mul_type_size, mul_ne, mul_nb, 2); + size_t loss_per_sample_type_size = sizeof(float); + int64_t loss_per_sample_n_bytes = nr * loss_per_sample_type_size; + ggml_cann_pool_alloc loss_per_sample_allocator(ctx.pool(), loss_per_sample_n_bytes); + void * loss_per_sample_buffer = loss_per_sample_allocator.get(); - GGML_CANN_CALL_ACLNN_OP(ctx, Mul, acl_log_softmax.get(), acl_labels.get(), acl_mul_result.get()); + int64_t loss_per_sample_ne[] = { nr }; + size_t loss_per_sample_nb[1]; + loss_per_sample_nb[0] = loss_per_sample_type_size; + acl_tensor_ptr acl_loss_per_sample = ggml_cann_create_tensor( + loss_per_sample_buffer, ACL_FLOAT, loss_per_sample_type_size, loss_per_sample_ne, loss_per_sample_nb, 1); - size_t sum_per_sample_type_size = sizeof(float); - int64_t sum_per_sample_n_bytes = nr * sum_per_sample_type_size; - ggml_cann_pool_alloc sum_per_sample_allocator(ctx.pool(), sum_per_sample_n_bytes); - void * sum_per_sample_buffer = sum_per_sample_allocator.get(); + size_t backprop_n_bytes = nr * nc * sizeof(float); + ggml_cann_pool_alloc backprop_allocator(ctx.pool(), backprop_n_bytes); + void * backprop_buffer = backprop_allocator.get(); + acl_tensor_ptr acl_backprop = ggml_cann_create_tensor(backprop_buffer, ACL_FLOAT, sizeof(float), logits_ne, logits_nb, 2); - int64_t sum_per_sample_ne[] = { nr }; - size_t sum_per_sample_nb[1]; - sum_per_sample_nb[0] = sum_per_sample_type_size; - acl_tensor_ptr acl_sum_per_sample = ggml_cann_create_tensor( - sum_per_sample_buffer, ACL_FLOAT, sum_per_sample_type_size, sum_per_sample_ne, sum_per_sample_nb, 1); - - std::vector sum_dims = { 1 }; - acl_int_array_ptr dims_array = ggml_cann_create_int_array(sum_dims.data(), sum_dims.size()); - bool keep_dims = false; - - GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_mul_result.get(), dims_array.get(), keep_dims, ACL_FLOAT, - acl_sum_per_sample.get()); + GGML_CANN_CALL_ACLNN_OP(ctx, SoftmaxCrossEntropyWithLogits, acl_logits.get(), acl_labels.get(), + acl_loss_per_sample.get(), acl_backprop.get()); size_t total_sum_type_size = sizeof(float); int64_t total_sum_n_bytes = 1 * total_sum_type_size; @@ -547,11 +679,12 @@ void ggml_cann_cross_entropy_loss(ggml_backend_cann_context & ctx, ggml_tensor * std::vector total_sum_dims = { 0 }; acl_int_array_ptr total_sum_dims_array = ggml_cann_create_int_array(total_sum_dims.data(), total_sum_dims.size()); + bool keep_dims = false; - GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_sum_per_sample.get(), total_sum_dims_array.get(), keep_dims, ACL_FLOAT, + GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_loss_per_sample.get(), total_sum_dims_array.get(), keep_dims, ACL_FLOAT, acl_total_sum.get()); - float value = -1.0f / static_cast(nr); + float value = 1.0f / static_cast(nr); acl_scalar_ptr scale_factor = ggml_cann_create_scalar(&value, aclDataType::ACL_FLOAT); acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst->data, ACL_FLOAT, sizeof(float), total_sum_ne, total_sum_nb, 1); @@ -589,6 +722,33 @@ void ggml_cann_group_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) { acl_mean_out.get(), acl_rstd_out.get()); } +void ggml_cann_set(ggml_backend_cann_context & ctx, ggml_tensor * dst) { + ggml_tensor * src0 = dst->src[0]; + ggml_tensor * src1 = dst->src[1]; + + size_t nb1 = ((int32_t *) dst->op_params)[0]; + size_t nb2 = ((int32_t *) dst->op_params)[1]; + size_t nb3 = ((int32_t *) dst->op_params)[2]; + size_t offset = ((int32_t *) dst->op_params)[3]; + bool inplace = (bool) ((int32_t *) dst->op_params)[4]; + + size_t param_nb[] = { ggml_element_size(src0), nb1, nb2, nb3 }; + + // Create a view of dst at the target offset with src1's dimensions + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, src1->ne, param_nb, GGML_MAX_DIMS, ACL_FORMAT_ND, offset); + acl_tensor_ptr acl_src1 = ggml_cann_create_tensor(src1); + + if (!inplace) { + // First copy src0 to dst entirely + size_t cpy_size = ggml_nbytes(dst); + ACL_CHECK( + aclrtMemcpyAsync(dst->data, cpy_size, src0->data, cpy_size, ACL_MEMCPY_DEVICE_TO_DEVICE, ctx.stream())); + } + + // Copy src1 into the target region of dst + GGML_CANN_CALL_ACLNN_OP(ctx, InplaceCopy, acl_dst.get(), acl_src1.get()); +} + void ggml_cann_acc(ggml_backend_cann_context & ctx, ggml_tensor * dst) { ggml_tensor * src0 = dst->src[0]; ggml_tensor * src1 = dst->src[1]; @@ -652,6 +812,113 @@ void ggml_cann_sum(ggml_backend_cann_context & ctx, ggml_tensor * dst) { aclnn_reduce_sum(ctx, dst, reduce_dims, 4); } +void ggml_cann_cumsum(ggml_backend_cann_context & ctx, ggml_tensor * dst) { + ggml_tensor * src = dst->src[0]; + acl_tensor_ptr acl_src = ggml_cann_create_tensor(src); + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); + // GGML cumsum operates along dim 0 (innermost / ne[0]). + // ggml_cann_create_tensor reverses dimensions to [ne3,ne2,ne1,ne0], + // so GGML dim 0 maps to CANN dim 3 (the last dim of the 4-D tensor). + GGML_CANN_CALL_ACLNN_OP(ctx, Cumsum, acl_src.get(), (int64_t)3, + ggml_cann_type_mapping(dst->type), acl_dst.get()); +} + +void ggml_cann_solve_tri(ggml_backend_cann_context & ctx, ggml_tensor * dst) { + ggml_tensor * src0 = dst->src[0]; // A: [N, N, B2, B3] lower triangular + ggml_tensor * src1 = dst->src[1]; // B: [K, N, B2, B3] + + acl_tensor_ptr acl_a = ggml_cann_create_tensor(src0); + acl_tensor_ptr acl_b = ggml_cann_create_tensor(src1); + acl_tensor_ptr acl_x = ggml_cann_create_tensor(dst); + + // mOut: triangular copy of A (required output), same shape as A. + const size_t a_bytes = ggml_nbytes(src0); + ggml_cann_pool_alloc m_alloc(ctx.pool(), a_bytes); + acl_tensor_ptr acl_m = ggml_cann_create_tensor( + m_alloc.get(), ggml_cann_type_mapping(src0->type), + ggml_type_size(src0->type), src0->ne, src0->nb, GGML_MAX_DIMS); + + // Solve AX = B: upper=false (lower tri), transpose=false, unitriangular=false. + GGML_CANN_CALL_ACLNN_OP(ctx, TriangularSolve, + acl_b.get(), acl_a.get(), false, false, false, + acl_x.get(), acl_m.get()); +} + +void ggml_cann_diag(ggml_backend_cann_context & ctx, ggml_tensor * dst) { + ggml_tensor * src = dst->src[0]; + + GGML_ASSERT(src->ne[1] == 1); + + const int64_t N = src->ne[0]; + const int64_t n_batch = src->ne[2] * src->ne[3]; + const size_t nb_f32 = sizeof(float); + + // Fill dst with zeros. + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); + { + float zero = 0.0f; + acl_scalar_ptr acl_zero = ggml_cann_create_scalar(&zero, ACL_FLOAT); + GGML_CANN_CALL_ACLNN_OP(ctx, InplaceFillScalar, acl_dst.get(), acl_zero.get()); + } + + // Copy src vector onto the diagonal of dst via strided views. + // src viewed as [N, n_batch], contiguous strides. + int64_t ne_vec[2] = { N, n_batch }; + size_t nb_src_vec[2] = { nb_f32, N * nb_f32 }; + // dst diagonal view: stride (N+1)*4 steps along the diagonal. + size_t nb_dst_diag[2] = { (N + 1) * nb_f32, N * N * nb_f32 }; + + acl_tensor_ptr acl_src_vec = ggml_cann_create_tensor(src->data, ACL_FLOAT, nb_f32, ne_vec, nb_src_vec, 2); + acl_tensor_ptr acl_dst_diag = ggml_cann_create_tensor(dst->data, ACL_FLOAT, nb_f32, ne_vec, nb_dst_diag, 2); + + GGML_CANN_CALL_ACLNN_OP(ctx, InplaceCopy, acl_dst_diag.get(), acl_src_vec.get()); +} + +void ggml_cann_fill(ggml_backend_cann_context & ctx, ggml_tensor * dst) { + float c = ggml_get_op_params_f32(dst, 0); + + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); + acl_scalar_ptr acl_c = ggml_cann_create_scalar(&c, ACL_FLOAT); + GGML_CANN_CALL_ACLNN_OP(ctx, InplaceFillScalar, acl_dst.get(), acl_c.get()); +} + +void ggml_cann_tri(ggml_backend_cann_context & ctx, ggml_tensor * dst) { + ggml_tensor * src = dst->src[0]; + + const int64_t S = src->ne[0]; + const int64_t n_batch = src->ne[2] * src->ne[3]; + const size_t nb_f32 = sizeof(float); + + int64_t ne3d[3] = { S, S, n_batch }; + size_t nb3d[3] = { nb_f32, S * nb_f32, S * S * nb_f32 }; + + const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0); + + acl_tensor_ptr acl_src = ggml_cann_create_tensor(src->data, ACL_FLOAT, nb_f32, ne3d, nb3d, 3); + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst->data, ACL_FLOAT, nb_f32, ne3d, nb3d, 3); + + switch (ttype) { + case GGML_TRI_TYPE_LOWER: + // Tril(-1): preserve row > col (strict lower), zero upper + diagonal. + GGML_CANN_CALL_ACLNN_OP(ctx, Tril, acl_src.get(), (int64_t)-1, acl_dst.get()); + break; + case GGML_TRI_TYPE_UPPER_DIAG: + // Triu(0): preserve row <= col (upper + diagonal), zero strict lower. + GGML_CANN_CALL_ACLNN_OP(ctx, Triu, acl_src.get(), (int64_t)0, acl_dst.get()); + break; + case GGML_TRI_TYPE_UPPER: + // Triu(1): preserve row < col (strict upper), zero lower + diagonal. + GGML_CANN_CALL_ACLNN_OP(ctx, Triu, acl_src.get(), (int64_t)1, acl_dst.get()); + break; + case GGML_TRI_TYPE_LOWER_DIAG: + // Tril(0): preserve row >= col (lower + diagonal), zero strict upper. + GGML_CANN_CALL_ACLNN_OP(ctx, Tril, acl_src.get(), (int64_t)0, acl_dst.get()); + break; + default: + GGML_ABORT("unsupported tri type"); + } +} + void ggml_cann_upsample_nearest2d(ggml_backend_cann_context & ctx, ggml_tensor * dst) { ggml_tensor * src = dst->src[0]; acl_tensor_ptr acl_src = ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW); @@ -1695,152 +1962,90 @@ void ggml_cann_softmax(ggml_backend_cann_context & ctx, ggml_tensor * dst) { aclnn_softmax(ctx, softmax_tensor.get(), 3, acl_dst.get()); } -/** - * @brief Performs index select operation on a 4D tensor using the CANN backend. - * - * This function applies the `IndexSelect` operation along a specific dimension - * of the source tensor (`src_buffer`) using the indices from the index tensor (`index`). - * It iterates over the last two dimensions of the source tensor, creates the corresponding - * CANN tensors for the source, index, and output slices, and executes the `IndexSelect` - * operation for each slice. - * - * @param ctx The context for CANN backend operations. - * @param src_buffer The source buffer containing the 4D input tensor data. - * @param src_ne The dimensions of the source tensor. - * @param src_nb The strides (byte offsets) of the source tensor. - * @param dst_buffer The destination buffer where the output tensor data will be written. - * @param dst_ne The dimensions of the destination tensor. - * @param dst_nb The strides (byte offsets) of the destination tensor. - * @param index The index tensor specifying the indices to select from the source tensor. - * @param type The data type of the source and destination tensors. - */ -static void aclnn_index_select_4d(ggml_backend_cann_context & ctx, - void * src_buffer, - int64_t * src_ne, - size_t * src_nb, - void * dst_buffer, - int64_t * dst_ne, - size_t * dst_nb, - ggml_tensor * index, - ggml_type type) { - for (int64_t i = 0; i < src_ne[3]; i++) { - for (int64_t j = 0; j < src_ne[2]; j++) { - // src - acl_tensor_ptr acl_src_tensor = - ggml_cann_create_tensor((char *) src_buffer + i * src_nb[3] + j * src_nb[2], - ggml_cann_type_mapping(type), ggml_type_size(type), src_ne, src_nb, 2); - - // index - acl_tensor_ptr acl_index = ggml_cann_create_tensor( - (char *) index->data + (i % index->ne[2]) * index->nb[2] + (j % index->ne[1]) * index->nb[1], - ggml_cann_type_mapping(index->type), ggml_element_size(index), index->ne, index->nb, 1); - - // out - acl_tensor_ptr acl_out = - ggml_cann_create_tensor((char *) dst_buffer + i * dst_nb[3] + j * dst_nb[2], - ggml_cann_type_mapping(type), ggml_type_size(type), dst_ne, dst_nb, 2); - GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, acl_src_tensor.get(), 0, acl_index.get(), acl_out.get()); - } - } -} - -/** - * @brief Performs inplace index copy operation on a 4D tensor using the CANN backend. - * - * This function applies the `IndexCopy` operation along a specific dimension of the - * destination tensor (`dst_buffer`) by copying elements from the source tensor (`src_buffer`) - * to positions specified by the index tensor (`index`). - * It iterates over the last two dimensions of the tensors, creates the corresponding - * CANN tensors for source, index, and destination slices, and performs the index copy - * operation for each slice. - * - * @param ctx The context for CANN backend operations. - * @param src_buffer The source buffer containing the 4D input tensor data to be copied. - * @param src_ne The dimensions of the source tensor. - * @param src_nb The strides (byte offsets) of the source tensor. - * @param dst_buffer The destination buffer where values will be copied to. - * @param dst_ne The dimensions of the destination tensor. - * @param dst_nb The strides (byte offsets) of the destination tensor. - * @param index The index tensor specifying target positions in the destination tensor. - * @param type The data type of the source and destination tensors. - */ -static void aclnn_index_copy_4d(ggml_backend_cann_context & ctx, - void * src_buffer, - int64_t * src_ne, - size_t * src_nb, - void * dst_buffer, - int64_t * dst_ne, - size_t * dst_nb, - ggml_tensor * index, - ggml_type type) { - for (int64_t i = 0; i < src_ne[3]; i++) { - for (int64_t j = 0; j < src_ne[2]; j++) { - // src - acl_tensor_ptr acl_src_tensor = - ggml_cann_create_tensor((char *) src_buffer + i * src_nb[3] + j * src_nb[2], - ggml_cann_type_mapping(type), ggml_type_size(type), src_ne, src_nb, 2); - - // index - acl_tensor_ptr acl_index = ggml_cann_create_tensor( - (char *) index->data + (i % index->ne[2]) * index->nb[2] + (j % index->ne[1]) * index->nb[1], - ggml_cann_type_mapping(index->type), ggml_element_size(index), index->ne, index->nb, 1); - - // out - acl_tensor_ptr acl_out = - ggml_cann_create_tensor((char *) dst_buffer + i * dst_nb[3] + j * dst_nb[2], - ggml_cann_type_mapping(type), ggml_type_size(type), dst_ne, dst_nb, 2); - GGML_CANN_CALL_ACLNN_OP(ctx, InplaceIndexCopy, acl_out.get(), 0, acl_index.get(), acl_src_tensor.get()); - } - } -} void ggml_cann_get_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) { - ggml_tensor * src0 = dst->src[0]; // src + ggml_tensor * src0 = dst->src[0]; // weight ggml_tensor * src1 = dst->src[1]; // index GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16); + // n_idx: number of row indices per (i2, i3) batch slice. + // ggml guarantees: src0->ne[2] == src1->ne[1], src0->ne[3] == src1->ne[2], src1->ne[3] == 1. + const int64_t n_idx = src1->ne[0]; + + // Gather all (i2, i3) batch slices from src into dst. + // ggml_cann_create_tensor reverses dims, so ACL sees [ne1, ne0]. + // GatherV2 with dim=0 gathers along ACL dim-0 == ggml ne[1] (the vocabulary / row axis). + // nb: the 4 strides of the source buffer (nb[0..1] for the 2D slice shape, + // nb[2..3] for computing per-batch-slice base pointer offsets). + auto gather_batched = [&](void * src_base, aclDataType acl_type, size_t type_size, + const size_t * nb) { + int64_t src_ne[2] = { src0->ne[0], src0->ne[1] }; + size_t src_nb_2d[2] = { nb[0], nb[1] }; + int64_t dst_ne[2] = { src0->ne[0], n_idx }; + size_t dst_nb_2d[2] = { dst->nb[0], dst->nb[1] }; + int64_t idx_ne[1] = { n_idx }; + size_t idx_nb[1] = { (size_t)ggml_element_size(src1) }; + + for (int64_t i3 = 0; i3 < src0->ne[3]; i3++) { + for (int64_t i2 = 0; i2 < src0->ne[2]; i2++) { + acl_tensor_ptr acl_src = ggml_cann_create_tensor( + (char *)src_base + i3 * nb[3] + i2 * nb[2], + acl_type, type_size, src_ne, src_nb_2d, 2); + acl_tensor_ptr acl_idx = ggml_cann_create_tensor( + (char *)src1->data + i3 * src1->nb[2] + i2 * src1->nb[1], + ggml_cann_type_mapping(src1->type), (size_t)ggml_element_size(src1), + idx_ne, idx_nb, 1); + acl_tensor_ptr acl_dst = ggml_cann_create_tensor( + (char *)dst->data + i3 * dst->nb[3] + i2 * dst->nb[2], + acl_type, type_size, dst_ne, dst_nb_2d, 2); + GGML_CANN_CALL_ACLNN_OP(ctx, GatherV2, acl_src.get(), 0, acl_idx.get(), acl_dst.get()); + } + } + }; + switch (src0->type) { case GGML_TYPE_BF16: case GGML_TYPE_F16: case GGML_TYPE_F32: if (src0->type == dst->type) { - aclnn_index_select_4d(ctx, src0->data, src0->ne, src0->nb, dst->data, dst->ne, dst->nb, src1, - dst->type); + gather_batched(src0->data, + ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type), + src0->nb); } else { - acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0); - ggml_cann_pool_alloc src_buffer_allocator(ctx.pool(), ggml_nelements(src0) * ggml_element_size(dst)); - void * src_trans_buffer = src_buffer_allocator.get(); - size_t src_trans_nb[GGML_MAX_DIMS]; - src_trans_nb[0] = dst->nb[0]; + // Cast src0 to dst type, then gather. + ggml_cann_pool_alloc src_cast_allocator(ctx.pool(), + ggml_nelements(src0) * ggml_element_size(dst)); + size_t src_cast_nb[GGML_MAX_DIMS]; + src_cast_nb[0] = ggml_type_size(dst->type); for (int i = 1; i < GGML_MAX_DIMS; i++) { - src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1]; + src_cast_nb[i] = src_cast_nb[i - 1] * src0->ne[i - 1]; } - acl_tensor_ptr src_trans_tensor = - ggml_cann_create_tensor(src_trans_buffer, ggml_cann_type_mapping(dst->type), - ggml_type_size(dst->type), src0->ne, src_trans_nb, GGML_MAX_DIMS); - aclnn_cast(ctx, acl_src0.get(), src_trans_tensor.get(), ggml_cann_type_mapping(dst->type)); - aclnn_index_select_4d(ctx, src_trans_buffer, src0->ne, src_trans_nb, dst->data, dst->ne, dst->nb, src1, - dst->type); + acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0); + acl_tensor_ptr acl_src_cast = ggml_cann_create_tensor( + src_cast_allocator.get(), ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type), + src0->ne, src_cast_nb, GGML_MAX_DIMS); + aclnn_cast(ctx, acl_src0.get(), acl_src_cast.get(), ggml_cann_type_mapping(dst->type)); + + gather_batched(src_cast_allocator.get(), + ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type), + src_cast_nb); } break; case GGML_TYPE_Q8_0: { - // add 1 dim for bcast mul. + // Dequantize Q8_0 to dst type, then gather. size_t weight_nb[GGML_MAX_DIMS + 1], scale_nb[GGML_MAX_DIMS + 1], dequant_nb[GGML_MAX_DIMS + 1]; int64_t weight_ne[GGML_MAX_DIMS + 1], scale_ne[GGML_MAX_DIMS + 1], *dequant_ne; - int64_t scale_offset = 0; - // [3,4,5,64] -> [3,4,5,2,32] - weight_ne[0] = QK8_0; - weight_ne[1] = src0->ne[0] / QK8_0; - weight_nb[0] = sizeof(int8_t); - weight_nb[1] = weight_nb[0] * weight_ne[0]; + weight_ne[0] = QK8_0; + weight_ne[1] = src0->ne[0] / QK8_0; + weight_nb[0] = sizeof(int8_t); + weight_nb[1] = weight_nb[0] * weight_ne[0]; for (int i = 2; i < GGML_MAX_DIMS + 1; i++) { weight_ne[i] = src0->ne[i - 1]; weight_nb[i] = weight_nb[i - 1] * weight_ne[i - 1]; } - // [3,4,5,64] -> [3,4,5,2,1] scale_ne[0] = 1; scale_ne[1] = src0->ne[0] / QK8_0; scale_nb[0] = sizeof(uint16_t); @@ -1849,31 +2054,33 @@ void ggml_cann_get_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) { scale_ne[i] = src0->ne[i - 1]; scale_nb[i] = scale_nb[i - 1] * scale_ne[i - 1]; } - // [3,4,5,64] -> [3,4,5,2,32] dequant_ne = weight_ne; dequant_nb[0] = ggml_type_size(dst->type); for (int i = 1; i < GGML_MAX_DIMS + 1; i++) { dequant_nb[i] = dequant_nb[i - 1] * dequant_ne[i - 1]; } - scale_offset = ggml_nelements(src0) * sizeof(int8_t); - ggml_cann_pool_alloc dequant_buffer_allocator(ctx.pool(), - ggml_nelements(src0) * ggml_type_size(dst->type)); - acl_tensor_ptr acl_weight_tensor = ggml_cann_create_tensor(src0->data, ACL_INT8, sizeof(int8_t), - weight_ne, weight_nb, GGML_MAX_DIMS + 1); - acl_tensor_ptr acl_scale_tensor = - ggml_cann_create_tensor(src0->data, ACL_FLOAT16, sizeof(uint16_t), scale_ne, scale_nb, - GGML_MAX_DIMS + 1, ACL_FORMAT_ND, scale_offset); - acl_tensor_ptr dequant_tensor = - ggml_cann_create_tensor(dequant_buffer_allocator.get(), ggml_cann_type_mapping(dst->type), - ggml_type_size(dst->type), dequant_ne, dequant_nb, GGML_MAX_DIMS + 1); - aclnn_mul(ctx, acl_weight_tensor.get(), acl_scale_tensor.get(), dequant_tensor.get()); - dequant_nb[0] = ggml_type_size(dst->type); + const int64_t scale_offset = ggml_nelements(src0) * sizeof(int8_t); + ggml_cann_pool_alloc dequant_allocator(ctx.pool(), + ggml_nelements(src0) * ggml_type_size(dst->type)); + acl_tensor_ptr acl_weight = ggml_cann_create_tensor(src0->data, ACL_INT8, sizeof(int8_t), + weight_ne, weight_nb, GGML_MAX_DIMS + 1); + acl_tensor_ptr acl_scale = ggml_cann_create_tensor( + src0->data, ACL_FLOAT16, sizeof(uint16_t), scale_ne, scale_nb, + GGML_MAX_DIMS + 1, ACL_FORMAT_ND, scale_offset); + acl_tensor_ptr acl_dequant = ggml_cann_create_tensor( + dequant_allocator.get(), ggml_cann_type_mapping(dst->type), + ggml_type_size(dst->type), dequant_ne, dequant_nb, GGML_MAX_DIMS + 1); + aclnn_mul(ctx, acl_weight.get(), acl_scale.get(), acl_dequant.get()); + + // Reinterpret dequant buffer as 4D [src0->ne] with contiguous strides. dequant_ne = src0->ne; + dequant_nb[0] = ggml_type_size(dst->type); for (int i = 1; i < GGML_MAX_DIMS; i++) { dequant_nb[i] = dequant_nb[i - 1] * src0->ne[i - 1]; } - aclnn_index_select_4d(ctx, dequant_buffer_allocator.get(), dequant_ne, dequant_nb, dst->data, dst->ne, - dst->nb, src1, dst->type); + gather_batched(dequant_allocator.get(), + ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type), + dequant_nb); break; } default: @@ -1883,31 +2090,70 @@ void ggml_cann_get_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) { } void ggml_cann_set_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) { - ggml_tensor * src0 = dst->src[0]; // src - ggml_tensor * src1 = dst->src[1]; // index + ggml_tensor * src0 = dst->src[0]; // source values + ggml_tensor * src1 = dst->src[1]; // row indices + + // n_idx: number of source rows to scatter per batch slice. + // ggml guarantees: src0->ne[1] == src1->ne[0]. + const int64_t n_idx = src1->ne[0]; + + // Copy n_idx rows of src [ne0, n_idx] into dst [ne0, ne1] at positions given by a 1D index. + // ggml_cann_create_tensor reverses dims, so ACL sees [ne1, ne0] for dst. + // InplaceIndexCopy with dim=0 copies along ACL dim-0 == ggml ne[1] (the row axis). + // src_nb: the 4 strides of the source buffer (nb[0..1] for the 2D slice shape, + // nb[2..3] for computing per-batch-slice base pointer offsets). + auto scatter_batched = [&](void * src_base, aclDataType acl_type, size_t type_size, + const size_t * src_nb) { + int64_t d_ne[2] = { dst->ne[0], dst->ne[1] }; + size_t d_nb[2] = { dst->nb[0], dst->nb[1] }; + int64_t s_ne[2] = { dst->ne[0], n_idx }; + size_t s_nb_2d[2] = { src_nb[0], src_nb[1] }; + int64_t i_ne[1] = { n_idx }; + size_t i_nb[1] = { (size_t)ggml_element_size(src1) }; + + for (int64_t i3 = 0; i3 < dst->ne[3]; i3++) { + for (int64_t i2 = 0; i2 < dst->ne[2]; i2++) { + acl_tensor_ptr acl_dst = ggml_cann_create_tensor( + (char *)dst->data + i3 * dst->nb[3] + i2 * dst->nb[2], + acl_type, type_size, d_ne, d_nb, 2); + acl_tensor_ptr acl_idx = ggml_cann_create_tensor( + (char *)src1->data + (i3 % src1->ne[2]) * src1->nb[2] + (i2 % src1->ne[1]) * src1->nb[1], + ggml_cann_type_mapping(src1->type), (size_t)ggml_element_size(src1), + i_ne, i_nb, 1); + acl_tensor_ptr acl_src = ggml_cann_create_tensor( + (char *)src_base + i3 * src_nb[3] + i2 * src_nb[2], + acl_type, type_size, s_ne, s_nb_2d, 2); + GGML_CANN_CALL_ACLNN_OP(ctx, InplaceIndexCopy, acl_dst.get(), 0, acl_idx.get(), acl_src.get()); + } + } + }; switch (dst->type) { case GGML_TYPE_F32: - { - aclnn_index_copy_4d(ctx, src0->data, src0->ne, src0->nb, dst->data, dst->ne, dst->nb, src1, dst->type); - break; - } + scatter_batched(src0->data, + ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type), + src0->nb); + break; case GGML_TYPE_F16: case GGML_TYPE_BF16: { - acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0); - ggml_cann_pool_alloc src_buffer_allocator(ctx.pool(), ggml_nelements(src0) * sizeof(uint16_t)); - void * src_trans_buffer = src_buffer_allocator.get(); - size_t src_trans_nb[GGML_MAX_DIMS]; - src_trans_nb[0] = sizeof(uint16_t); + // Cast src0 (F32) to dst type first. + ggml_cann_pool_alloc src_cast_allocator(ctx.pool(), + ggml_nelements(src0) * ggml_type_size(dst->type)); + size_t src_cast_nb[GGML_MAX_DIMS]; + src_cast_nb[0] = ggml_type_size(dst->type); for (int i = 1; i < GGML_MAX_DIMS; i++) { - src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1]; + src_cast_nb[i] = src_cast_nb[i - 1] * src0->ne[i - 1]; } - acl_tensor_ptr src_trans_tensor = ggml_cann_create_tensor( - src_trans_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type), src0->ne, src_trans_nb, GGML_MAX_DIMS); - aclnn_cast(ctx, acl_src0.get(), src_trans_tensor.get(), ggml_cann_type_mapping(dst->type)); - aclnn_index_copy_4d(ctx, src_trans_buffer, src0->ne, src_trans_nb, dst->data, dst->ne, dst->nb, src1, - dst->type); + acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0); + acl_tensor_ptr acl_src_cast = ggml_cann_create_tensor( + src_cast_allocator.get(), ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type), + src0->ne, src_cast_nb, GGML_MAX_DIMS); + aclnn_cast(ctx, acl_src0.get(), acl_src_cast.get(), ggml_cann_type_mapping(dst->type)); + + scatter_batched(src_cast_allocator.get(), + ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type), + src_cast_nb); break; } default: @@ -3268,29 +3514,50 @@ void ggml_cann_pad_reflect_1d(ggml_backend_cann_context & ctx, ggml_tensor * dst int64_t paddingsArray[2] = { opts[0], opts[1] }; acl_int_array_ptr paddings = ggml_cann_create_int_array(paddingsArray, 2); - for (int64_t i = 0; i < src0->ne[3]; i++) { - acl_tensor_ptr acl_src = - ggml_cann_create_tensor((char *) src0->data + i * src0->ne[3], ggml_cann_type_mapping(src0->type), - ggml_element_size(src0), src0->ne, src0->nb, 3); + // Collapsing ne[2]*ne[3] into a single batch dimension requires that dim3 + // is contiguous with respect to dim2 in both src and dst. + GGML_ASSERT(src0->nb[3] == src0->nb[2] * src0->ne[2]); + GGML_ASSERT(dst->nb[3] == dst->nb[2] * dst->ne[2]); - acl_tensor_ptr acl_dst = - ggml_cann_create_tensor((char *) dst->data + i * src0->ne[3], ggml_cann_type_mapping(dst->type), - ggml_element_size(dst), dst->ne, dst->nb, 3); + int64_t src_ne_3d[3] = { src0->ne[0], src0->ne[1], src0->ne[2] * src0->ne[3] }; + int64_t dst_ne_3d[3] = { dst->ne[0], dst->ne[1], dst->ne[2] * dst->ne[3] }; - GGML_CANN_CALL_ACLNN_OP(ctx, ReflectionPad1d, acl_src.get(), paddings.get(), acl_dst.get()); - } + acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0->data, ggml_cann_type_mapping(src0->type), + ggml_element_size(src0), src_ne_3d, src0->nb, 3); + + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst->data, ggml_cann_type_mapping(dst->type), + ggml_element_size(dst), dst_ne_3d, dst->nb, 3); + + GGML_CANN_CALL_ACLNN_OP(ctx, ReflectionPad1d, acl_src.get(), paddings.get(), acl_dst.get()); } void ggml_cann_count_equal(ggml_backend_cann_context & ctx, ggml_tensor * dst) { ggml_tensor * src0 = dst->src[0]; ggml_tensor * src1 = dst->src[1]; + // Write element-wise equality (0 or 1) into a temporary buffer to avoid + // modifying src0 in-place. Use the same type as src0 so ReduceSum can + // consume it directly without a type cast. + ggml_cann_pool_alloc eq_alloc(ctx.pool(), ggml_nelements(src0) * ggml_element_size(src0)); + size_t eq_nb[GGML_MAX_DIMS]; + eq_nb[0] = ggml_element_size(src0); + for (int i = 1; i < GGML_MAX_DIMS; i++) { + eq_nb[i] = eq_nb[i - 1] * src0->ne[i - 1]; + } + acl_tensor_ptr acl_eq = ggml_cann_create_tensor( + eq_alloc.get(), ggml_cann_type_mapping(src0->type), ggml_element_size(src0), + src0->ne, eq_nb, GGML_MAX_DIMS); + acl_tensor_ptr acl_self = ggml_cann_create_tensor(src0); acl_tensor_ptr acl_other = ggml_cann_create_tensor(src1); - - GGML_CANN_CALL_ACLNN_OP(ctx, InplaceEqTensor, acl_self.get(), acl_other.get()); - - ggml_cann_sum(ctx, dst); + GGML_CANN_CALL_ACLNN_OP(ctx, EqTensor, acl_self.get(), acl_other.get(), acl_eq.get()); + + // Sum the 0/1 values into dst. + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); + int64_t dims[4] = { 0, 1, 2, 3 }; + acl_int_array_ptr dims_arr = ggml_cann_create_int_array(dims, 4); + GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_eq.get(), dims_arr.get(), true, + ggml_cann_type_mapping(dst->type), acl_dst.get()); } void ggml_cann_step(ggml_backend_cann_context & ctx, ggml_tensor * dst) { @@ -3306,6 +3573,27 @@ void ggml_cann_step(ggml_backend_cann_context & ctx, ggml_tensor * dst) { GGML_CANN_CALL_ACLNN_OP(ctx, GtScalar, acl_src.get(), alpha.get(), acl_dst.get()); } +void ggml_cann_softplus(ggml_backend_cann_context & ctx, ggml_tensor * dst) { + ggml_tensor * src0 = dst->src[0]; + + acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0); + acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); + + float beta_val = 1.0f; + float threshold_val = 20.0f; + acl_scalar_ptr beta = ggml_cann_create_scalar(&beta_val, ACL_FLOAT); + acl_scalar_ptr threshold = ggml_cann_create_scalar(&threshold_val, ACL_FLOAT); + + GGML_CANN_CALL_ACLNN_OP(ctx, Softplus, acl_src.get(), beta.get(), threshold.get(), acl_dst.get()); +} + +void ggml_cann_geglu_quick(ggml_backend_cann_context & ctx, ggml_tensor * dst) { + auto gelu_quick_fn = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) { + GGML_CANN_CALL_ACLNN_OP(ctx, GeluV2, acl_src, 0, acl_dst); + }; + ggml_cann_op_unary_gated(gelu_quick_fn, ctx, dst); +} + /** * @brief Performs expert-specific matrix multiplication (MoE) with * floating-point precision using the CANN backend. @@ -3892,46 +4180,65 @@ void ggml_cann_flash_attn_ext(ggml_backend_cann_context & ctx, ggml_tensor * dst } static void ggml_cann_out_prod_fp(ggml_backend_cann_context & ctx, ggml_tensor * dst) { - ggml_tensor * src0 = dst->src[0]; // weight - ggml_tensor * src1 = dst->src[1]; // input + ggml_tensor * src0 = dst->src[0]; // weight [ne00=m, ne01=K, ne02, ne03] + ggml_tensor * src1 = dst->src[1]; // input [ne10=n, ne11=K, ne12, ne13] GGML_TENSOR_BINARY_OP_LOCALS - acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst); - GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, acl_dst.get()); + // dst[i,j] = sum_k src0[i,k] * src1[j,k] i.e. dst = src0 @ src1^T. + // + // ggml_cann_create_tensor reverses dimension order, so ACL sees: + // acl_src0 slice: ggml[m,K] -> ACL[K,m] + // acl_src1 slice: ggml[n,K] -> ACL[K,n] + // acl_dst slice: ggml[m,n] -> ACL[n,m] + // + // Build a transposed view of src1 by swapping ne[0]/ne[1]: + // src1_t: ggml[K,n] (swapped strides) -> ACL[n,K] + // + // Matmul(src1_t [n,K], src0 [K,m]) = [n,m] = acl_dst ✓ + // + // The outer batch loop is kept because src0 may have fewer batch slices than + // dst (ne02 <= ne2, ne03 <= ne3): this is a strided-broadcast not supported + // by standard CANN Matmul broadcasting. + + const aclDataType src0_acl_type = ggml_cann_type_mapping(src0->type); + const aclDataType src1_acl_type = ggml_cann_type_mapping(src1->type); + const aclDataType dst_acl_type = ggml_cann_type_mapping(dst->type); + const size_t src0_type_sz = ggml_type_size(src0->type); + const size_t src1_type_sz = ggml_type_size(src1->type); + const size_t dst_type_sz = ggml_type_size(dst->type); const int64_t dps2 = ne2 / ne02; const int64_t dps3 = ne3 / ne03; + for (int64_t i3 = 0; i3 < ne3; i3++) { for (int64_t i2 = 0; i2 < ne2; i2++) { const int64_t i02 = i2 / dps2; const int64_t i03 = i3 / dps3; - const int64_t i12 = i2; - const int64_t i13 = i3; - acl_tensor_ptr accumulator = - ggml_cann_create_tensor((char *) dst->data + i2 * nb2 + i3 * nb3, ggml_cann_type_mapping(dst->type), - ggml_type_size(dst->type), dst->ne, dst->nb, 2); - - // The outer product needs to be accumulated in this dimension. - for (int64_t i1 = 0; i1 < ne11; i1++) { - acl_tensor_ptr acl_input = ggml_cann_create_tensor( - (char *) src1->data + i1 * nb11 + i12 * nb12 + i13 * nb13, ggml_cann_type_mapping(src0->type), - ggml_type_size(src0->type), src1->ne, src1->nb, 1); - - acl_tensor_ptr acl_weight = ggml_cann_create_tensor( - (char *) src0->data + i1 * nb01 + i02 * nb02 + i03 * nb03, ggml_cann_type_mapping(src0->type), - ggml_type_size(src0->type), src0->ne, src0->nb, 1); - - ggml_cann_pool_alloc output_allocator(ctx.pool()); - void * output_buffer = output_allocator.alloc(ggml_nbytes(dst)); - acl_tensor_ptr acl_out = ggml_cann_create_tensor(output_buffer, ggml_cann_type_mapping(dst->type), - ggml_type_size(dst->type), dst->ne, dst->nb, 2); - - GGML_CANN_CALL_ACLNN_OP(ctx, Ger, acl_input.get(), acl_weight.get(), acl_out.get()); - float alpha_value = 1.0f; - aclScalar * alpha = aclCreateScalar(&alpha_value, ACL_FLOAT); - GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, accumulator.get(), acl_out.get(), alpha); - } + // src0 2D slice at [i02, i03]: ggml [m, K] -> ACL [K, m] + int64_t src0_ne[2] = { ne00, ne01 }; + size_t src0_nb[2] = { nb00, nb01 }; + acl_tensor_ptr acl_src0_s = ggml_cann_create_tensor( + (char *) src0->data + i02 * nb02 + i03 * nb03, + src0_acl_type, src0_type_sz, src0_ne, src0_nb, 2); + + // src1 transposed 2D slice at [i2, i3]: swap ne/nb -> ggml[K,n] -> ACL[n,K] + int64_t src1_t_ne[2] = { ne11, ne10 }; + size_t src1_t_nb[2] = { nb11, nb10 }; + acl_tensor_ptr acl_src1_t = ggml_cann_create_tensor( + (char *) src1->data + i2 * nb12 + i3 * nb13, + src1_acl_type, src1_type_sz, src1_t_ne, src1_t_nb, 2); + + // dst 2D slice at [i2, i3]: ggml [m, n] -> ACL [n, m] + int64_t dst_ne[2] = { ne0, ne1 }; + size_t dst_nb[2] = { nb0, nb1 }; + acl_tensor_ptr acl_dst_s = ggml_cann_create_tensor( + (char *) dst->data + i2 * nb2 + i3 * nb3, + dst_acl_type, dst_type_sz, dst_ne, dst_nb, 2); + + // Matmul(src1_t [n,K], src0 [K,m]) = [n,m] = acl_dst_s ✓ + GGML_CANN_CALL_ACLNN_OP(ctx, Matmul, + acl_src1_t.get(), acl_src0_s.get(), acl_dst_s.get(), (int8_t) 1); } } } diff --git a/ggml/src/ggml-cann/aclnn_ops.h b/ggml/src/ggml-cann/aclnn_ops.h index 7f5ba4d..678f4d6 100644 --- a/ggml/src/ggml-cann/aclnn_ops.h +++ b/ggml/src/ggml-cann/aclnn_ops.h @@ -32,6 +32,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -47,6 +50,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -69,6 +75,10 @@ */ void ggml_cann_repeat(ggml_backend_cann_context & ctx, ggml_tensor * dst); +void ggml_cann_swiglu(ggml_backend_cann_context & ctx, ggml_tensor * dst); +void ggml_cann_swiglu_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst); +void ggml_cann_geglu(ggml_backend_cann_context & ctx, ggml_tensor * dst, int64_t approximate); + /** * @brief Applies the Leaky ReLU activation function to a tensor using the CANN * backend. @@ -325,6 +335,48 @@ void ggml_cann_sum_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst); void ggml_cann_sum(ggml_backend_cann_context & ctx, ggml_tensor * dst); +/** + * @brief Computes the cumulative sum of a ggml tensor along dim 0 using the + * CANN backend. + * + * @param ctx The CANN context used for operations. + * @param dst The destination tensor. dst->op is `GGML_OP_CUMSUM`. + */ +void ggml_cann_cumsum(ggml_backend_cann_context & ctx, ggml_tensor * dst); + +/** + * @brief Computes a triangular mask (tril/triu) of a square ggml tensor + * using the CANN backend. + * + * @param ctx The CANN context used for operations. + * @param dst The destination tensor. dst->op is `GGML_OP_TRI`. + */ +void ggml_cann_tri(ggml_backend_cann_context & ctx, ggml_tensor * dst); + +/** + * @brief Solves a triangular linear system AX=B using the CANN backend. + * + * @param ctx The CANN context used for operations. + * @param dst The destination tensor. dst->op is `GGML_OP_SOLVE_TRI`. + */ +void ggml_cann_solve_tri(ggml_backend_cann_context & ctx, ggml_tensor * dst); + +/** + * @brief Creates a diagonal matrix from a vector using the CANN backend. + * + * @param ctx The CANN context used for operations. + * @param dst The destination tensor. dst->op is `GGML_OP_DIAG`. + */ +void ggml_cann_diag(ggml_backend_cann_context & ctx, ggml_tensor * dst); + +/** + * @brief Fills a tensor with a constant scalar value using the CANN backend. + * + * @param ctx The CANN context used for operations. + * @param dst The destination tensor. dst->op is `GGML_OP_FILL`. + */ +void ggml_cann_fill(ggml_backend_cann_context & ctx, ggml_tensor * dst); + /** * @brief Upsamples a ggml tensor using nearest neighbor interpolation using * the CANN backend. @@ -461,6 +513,9 @@ void ggml_cann_timestep_embedding(ggml_backend_cann_context & ctx, ggml_tensor * // @see ggml_cann_dup. void ggml_cann_cpy(ggml_backend_cann_context & ctx, ggml_tensor * dst); +// @see ggml_cann_acc, but copies src1 into dst instead of adding. +void ggml_cann_set(ggml_backend_cann_context & ctx, ggml_tensor * dst); + /** * @brief Computes the softmax activation with optional masking. * @@ -813,6 +868,8 @@ void ggml_cann_count_equal(ggml_backend_cann_context & ctx, ggml_tensor * dst); * dst->op is expected to be `GGML_OP_STEP`. */ void ggml_cann_step(ggml_backend_cann_context & ctx, ggml_tensor * dst); +void ggml_cann_softplus(ggml_backend_cann_context & ctx, ggml_tensor * dst); +void ggml_cann_geglu_quick(ggml_backend_cann_context & ctx, ggml_tensor * dst); /** * @brief Performs the Flash Attention extended operator using the CANN backend. diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index 5fc484b..2c81543 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -920,7 +920,7 @@ static void * ggml_backend_cann_buffer_get_base(ggml_backend_buffer_t buffer) { * @param dst Pointer to the destination buffer where transformed data will be * stored. */ -static void ggml_backend_cann_transform_q4_0(ggml_tensor * tensor, const void * src, void * dst) { +static void ggml_backend_cann_transform_q4_0(const ggml_tensor * tensor, const void * src, void * dst) { int64_t n_elems = ggml_nelements(tensor); int64_t groups = n_elems / QK4_0; size_t quant_bytes = n_elems * sizeof(uint8_t) / 2; @@ -1014,7 +1014,7 @@ static void ggml_backend_cann_transform_back_q4_0(const ggml_tensor * tensor, vo * @param dst Pointer to the destination buffer where transformed data will be * stored. */ -static void ggml_backend_cann_transform_q8_0(ggml_tensor * tensor, const void * src, void * dst) { +static void ggml_backend_cann_transform_q8_0(const ggml_tensor * tensor, const void * src, void * dst) { int64_t n_elems = ggml_nelements(tensor); int64_t groups = n_elems / QK8_0; size_t quant_bytes = n_elems * sizeof(uint8_t); @@ -1075,7 +1075,7 @@ static void ggml_backend_cann_transform_back_q8_0(const ggml_tensor * tensor, co * @param dst Pointer to the destination buffer where transformed data will be * stored. */ -static void ggml_backend_cann_transform(ggml_tensor * tensor, const void * src, void * dst) { +static void ggml_backend_cann_transform(const ggml_tensor * tensor, const void * src, void * dst) { switch (tensor->type) { case GGML_TYPE_Q4_0: ggml_backend_cann_transform_q4_0(tensor, src, dst); @@ -1428,6 +1428,22 @@ static bool ggml_backend_cann_buffer_cpy_tensor(ggml_backend_buffer_t buffer, return false; } +/** + * @brief Set a region of a tensor's device memory to a specified value. + * + * @param buffer The CANN buffer containing the tensor. + * @param tensor Pointer to the tensor whose memory will be set. + * @param value The value to which each byte in the region will be set. + * @param offset Byte offset within the tensor's data to start setting. + * @param size Number of bytes to set. + */ +static void ggml_backend_cann_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { + ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context; + + ggml_cann_set_device(ctx->device); + ACL_CHECK(aclrtMemset((char *) tensor->data + offset, size, value, size)); +} + /** * @brief Clear a CANN buffer by setting all its memory to a specified value. * @@ -1454,7 +1470,7 @@ static const ggml_backend_buffer_i ggml_backend_cann_buffer_interface = { /* .free_buffer = */ ggml_backend_cann_buffer_free_buffer, /* .get_base = */ ggml_backend_cann_buffer_get_base, /* .init_tensor = */ ggml_backend_cann_buffer_init_tensor, - /* .memset_tensor = */ NULL, + /* .memset_tensor = */ ggml_backend_cann_buffer_memset_tensor, /* .set_tensor = */ ggml_backend_cann_buffer_set_tensor, /* .get_tensor = */ ggml_backend_cann_buffer_get_tensor, /* .set_tensor_2d = */ NULL, @@ -1579,12 +1595,14 @@ static bool ggml_backend_cann_buffer_type_is_host(ggml_backend_buffer_type_t buf * memory for CANN buffer types in the GGML backend. */ static const ggml_backend_buffer_type_i ggml_backend_cann_buffer_type_interface = { - /* .get_name = */ ggml_backend_cann_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_cann_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cann_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cann_buffer_type_get_alloc_size, - /* .is_host = */ ggml_backend_cann_buffer_type_is_host, + /* .get_name = */ ggml_backend_cann_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_cann_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_cann_buffer_type_get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_cann_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_cann_buffer_type_is_host, }; /** @@ -1726,12 +1744,14 @@ static ggml_backend_buffer_t ggml_backend_cann_host_buffer_type_alloc_buffer(ggm ggml_backend_buffer_type_t ggml_backend_cann_host_buffer_type() { static struct ggml_backend_buffer_type ggml_backend_cann_buffer_type_host = { /* .iface = */ { - /* .get_name = */ ggml_backend_cann_host_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_cann_host_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, - /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, + /* .get_name = */ ggml_backend_cann_host_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_cann_host_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, }, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cann_reg(), 0), @@ -1835,6 +1855,9 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct gg case GGML_UNARY_OP_STEP: ggml_cann_step(ctx, dst); break; + case GGML_UNARY_OP_SOFTPLUS: + ggml_cann_softplus(ctx, dst); + break; default: return false; } @@ -1845,20 +1868,19 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct gg GGML_CANN_CALL_OP_UNARY_GATED(Relu); break; case GGML_GLU_OP_GEGLU: + ggml_cann_geglu(ctx, dst, 0); // approximate=0 → tanh + break; case GGML_GLU_OP_GEGLU_ERF: - // aclnnGelu internally uses the erf-based approximation. - GGML_CANN_CALL_OP_UNARY_GATED(Gelu); + ggml_cann_geglu(ctx, dst, 1); // approximate=1 → erf break; case GGML_GLU_OP_SWIGLU: - GGML_CANN_CALL_OP_UNARY_GATED(Silu); + ggml_cann_swiglu(ctx, dst); + break; + case GGML_GLU_OP_SWIGLU_CLAMP: + ggml_cann_swiglu_clamp(ctx, dst); break; case GGML_GLU_OP_GEGLU_QUICK: - { - auto lambda = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) { - GGML_CANN_CALL_ACLNN_OP(ctx, GeluV2, acl_src, 0, acl_dst); - }; - ggml_cann_op_unary_gated(lambda, ctx, dst); - } + ggml_cann_geglu_quick(ctx, dst); break; default: return false; @@ -1920,6 +1942,9 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct gg case GGML_OP_CPY: ggml_cann_cpy(ctx, dst); break; + case GGML_OP_SET: + ggml_cann_set(ctx, dst); + break; case GGML_OP_CONT: ggml_cann_dup(ctx, dst); break; @@ -1989,6 +2014,21 @@ static bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct gg case GGML_OP_SSM_CONV: ggml_cann_ssm_conv(ctx, dst); break; + case GGML_OP_CUMSUM: + ggml_cann_cumsum(ctx, dst); + break; + case GGML_OP_TRI: + ggml_cann_tri(ctx, dst); + break; + case GGML_OP_FILL: + ggml_cann_fill(ctx, dst); + break; + case GGML_OP_DIAG: + ggml_cann_diag(ctx, dst); + break; + case GGML_OP_SOLVE_TRI: + ggml_cann_solve_tri(ctx, dst); + break; default: return false; } @@ -2324,6 +2364,7 @@ static enum ggml_status ggml_backend_cann_graph_compute(ggml_backend_t backend, if (use_cann_graph) { // If no matching graph is found, the graph needs to be recaptured. graph_capture_required = !cann_ctx->graph_lru_cache.find_and_move_to_front(cgraph); + if (graph_capture_required) { // If no matching graph is found, add a new ACL graph. ggml_cann_graph * new_graph = ggml_cann_graph::create_from_cgraph(cgraph); @@ -2382,6 +2423,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten case GGML_UNARY_OP_SGN: case GGML_UNARY_OP_STEP: case GGML_UNARY_OP_GELU_ERF: + case GGML_UNARY_OP_SOFTPLUS: return true; default: return false; @@ -2393,6 +2435,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten case GGML_GLU_OP_SWIGLU: case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: + case GGML_GLU_OP_SWIGLU_CLAMP: return true; default: return false; @@ -2412,9 +2455,10 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten #ifdef ASCEND_310P // Q4 && Q8 per group is not support on 310p device return false; -#endif +#else // only support contiguous for quantized types. return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); +#endif default: return false; } @@ -2429,9 +2473,10 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten #ifdef ASCEND_310P // Q4 && Q8 per group is not support on 310p device return false; -#endif +#else // only support contiguous for quantized types. return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); +#endif default: return false; } @@ -2499,6 +2544,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten } case GGML_OP_ROPE: { + if (((const int32_t *) op->op_params)[15] != 0) { + return false; // FIXME: support ggml_rope_set_offset + } if (op->src[0]->ne[0] > 896) { return false; } @@ -2572,6 +2620,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten case GGML_OP_SUM_ROWS: case GGML_OP_ARGSORT: case GGML_OP_ACC: + case GGML_OP_SET: case GGML_OP_GROUP_NORM: return true; case GGML_OP_PAD: @@ -2594,7 +2643,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten #ifdef ASCEND_310P // Ger is not supported on 310p device return false; -#endif +#else switch (op->src[0]->type) { case GGML_TYPE_F16: case GGML_TYPE_F32: @@ -2602,6 +2651,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten default: return false; } +#endif } case GGML_OP_CONV_TRANSPOSE_1D: return true; @@ -2620,7 +2670,7 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten #ifdef ASCEND_310P // FA not support on 310p device return false; -#endif +#else // derived from [ggml-cuda.cu] if (op->src[1]->type != GGML_TYPE_F16 || op->src[2]->type != GGML_TYPE_F16) { return false; @@ -2646,9 +2696,20 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten return false; } return true; +#endif } case GGML_OP_SSM_CONV: return true; + case GGML_OP_CUMSUM: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_TRI: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_FILL: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_DIAG: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_SOLVE_TRI: + return op->src[0]->type == GGML_TYPE_F32; default: return false; } @@ -2700,8 +2761,8 @@ static const ggml_backend_i ggml_backend_cann_interface = { /* .free = */ ggml_backend_cann_free, /* .set_tensor_async = */ ggml_backend_cann_set_tensor_async, /* .get_tensor_async = */ ggml_backend_cann_get_tensor_async, - /* .get_tensor_2d_async = */ NULL, /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, /* .cpy_tensor_async = */ ggml_backend_cann_cpy_tensor_async, /* .synchronize = */ ggml_backend_cann_synchronize, /* .graph_plan_create = */ NULL, @@ -2769,6 +2830,7 @@ static void ggml_backend_cann_device_get_props(ggml_backend_dev_t dev, ggml_back /* .host_buffer = */ host_buffer, /* .buffer_from_host_ptr = */ false, /* .events = */ true, + /* .mmap_support = */ true, }; } diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index f05683b..1dbbe32 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -96,6 +96,9 @@ typedef sycl::half2 ggml_half2; #define QI1_0 (QK1_0 / 32) #define QR1_0 1 +#define QI2_0 (QK2_0 / 32) +#define QR2_0 1 + #define QI4_0 (QK4_0 / (4 * QR4_0)) #define QR4_0 2 @@ -181,6 +184,13 @@ typedef struct { } block_q1_0; static_assert(sizeof(block_q1_0) == sizeof(ggml_half) + QK1_0 / 8, "wrong q1_0 block size/padding"); +#define QK2_0 64 +typedef struct { + ggml_half d; // delta (scale) + uint8_t qs[QK2_0 / 4]; // 2 bits per element +} block_q2_0; +static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding"); + #define QK4_0 32 typedef struct { ggml_half d; // delta @@ -1111,16 +1121,17 @@ GGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16) -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, GGML_TABLE_END() -// e2m1 values (doubled) +// e2m1 values (doubled), shared by MXFP4 and NVFP4 // ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf -GGML_TABLE_BEGIN(int8_t, kvalues_mxfp4, 16) +GGML_TABLE_BEGIN(int8_t, kvalues_fp4, 16) 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12, GGML_TABLE_END() +#define kvalues_mxfp4 kvalues_fp4 #define NGRID_IQ1S 2048 #define IQ1S_DELTA 0.125f #define IQ1M_DELTA 0.125f -#if defined(GGML_COMMON_IMPL_C) +#if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP) GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S) 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff, 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff, diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index beebc47..8327c52 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -41,6 +41,10 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/amx/amx.h ggml-cpu/amx/mmq.cpp ggml-cpu/amx/mmq.h + ggml-cpu/tiled/tiled.cpp + ggml-cpu/tiled/tiled.h + ggml-cpu/tiled/tiled-kernel.cpp + ggml-cpu/tiled/tiled-kernel.h ggml-cpu/ggml-cpu-impl.h ggml-cpu/common.h ggml-cpu/binary-ops.h @@ -72,17 +76,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name) endif() endif() - if (GGML_OPENMP) - find_package(OpenMP) - if (OpenMP_FOUND) - set(GGML_OPENMP_ENABLED "ON" CACHE INTERNAL "") - target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP) - - target_link_libraries(${GGML_CPU_NAME} PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX) - else() - set(GGML_OPENMP_ENABLED "OFF" CACHE INTERNAL "") - message(WARNING "OpenMP not found") - endif() + if (GGML_OPENMP_ENABLED) + target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP) + target_link_libraries(${GGML_CPU_NAME} PRIVATE ${GGML_OPENMP_TARGET_C} ${GGML_OPENMP_TARGET_CXX}) endif() if (GGML_LLAMAFILE) @@ -110,54 +106,79 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/arch/arm/repack.cpp ) + # MSVC proper (not clang-cl) has no -mcpu/-march for ARM: features can only be + # selected by probing the host at configure time and forwarding -D__ARM_FEATURE_* if (MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") - message(FATAL_ERROR "MSVC is not supported for ARM, use clang") + set(GGML_ARM_MSVC ON) + else() + set(GGML_ARM_MSVC OFF) + endif() + + if (GGML_ARM_MSVC AND NOT GGML_NATIVE) + message(FATAL_ERROR "MSVC on ARM requires GGML_NATIVE=ON, use clang for GGML_CPU_ARM_ARCH / GGML_CPU_ALL_VARIANTS") else() - check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E) - if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "") - list(APPEND ARCH_FLAGS -mfp16-format=ieee) + if (NOT GGML_ARM_MSVC) + check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E) + if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "") + list(APPEND ARCH_FLAGS -mfp16-format=ieee) + endif() endif() if (GGML_NATIVE) # -mcpu=native does not always enable all the features in some compilers, # so we check for them manually and enable them if available - execute_process( - COMMAND ${CMAKE_C_COMPILER} -mcpu=native -E -v - - INPUT_FILE "/dev/null" - OUTPUT_QUIET - ERROR_VARIABLE ARM_MCPU - RESULT_VARIABLE ARM_MCPU_RESULT - ) - if (NOT ARM_MCPU_RESULT) - string(REGEX MATCH "-mcpu=[^ ']+" ARM_MCPU_FLAG "${ARM_MCPU}") - string(REGEX MATCH "-march=[^ ']+" ARM_MARCH_FLAG "${ARM_MCPU}") - - # on some old GCC we need to read -march= - if (ARM_MARCH_FLAG AND NOT "${ARM_MARCH_FLAG}" STREQUAL "-march=native") - set(ARM_NATIVE_FLAG "${ARM_MARCH_FLAG}") - elseif(ARM_MCPU_FLAG AND NOT "${ARM_MCPU_FLAG}" STREQUAL "-mcpu=native") - set(ARM_NATIVE_FLAG "${ARM_MCPU_FLAG}") + # cl.exe cannot be queried this way: no /dev/null, and it rejects the flags. + # ARM_NATIVE_FLAG stays empty for MSVC and is never appended below. + if (NOT GGML_ARM_MSVC) + execute_process( + COMMAND ${CMAKE_C_COMPILER} -mcpu=native -E -v - + INPUT_FILE "/dev/null" + OUTPUT_QUIET + ERROR_VARIABLE ARM_MCPU + RESULT_VARIABLE ARM_MCPU_RESULT + ) + if (NOT ARM_MCPU_RESULT) + string(REGEX MATCH "-mcpu=[^ ']+" ARM_MCPU_FLAG "${ARM_MCPU}") + string(REGEX MATCH "-march=[^ ']+" ARM_MARCH_FLAG "${ARM_MCPU}") + + # on some old GCC we need to read -march= + if (ARM_MARCH_FLAG AND NOT "${ARM_MARCH_FLAG}" STREQUAL "-march=native") + set(ARM_NATIVE_FLAG "${ARM_MARCH_FLAG}") + elseif(ARM_MCPU_FLAG AND NOT "${ARM_MCPU_FLAG}" STREQUAL "-mcpu=native") + set(ARM_NATIVE_FLAG "${ARM_MCPU_FLAG}") + endif() endif() - endif() - if ("${ARM_NATIVE_FLAG}" STREQUAL "") - set(ARM_NATIVE_FLAG -mcpu=native) - message(WARNING "ARM -march/-mcpu not found, -mcpu=native will be used") - else() - message(STATUS "ARM detected flags: ${ARM_NATIVE_FLAG}") + if ("${ARM_NATIVE_FLAG}" STREQUAL "") + set(ARM_NATIVE_FLAG -mcpu=native) + message(WARNING "ARM -march/-mcpu not found, -mcpu=native will be used") + else() + message(STATUS "ARM detected flags: ${ARM_NATIVE_FLAG}") + endif() endif() include(CheckCXXSourceRuns) macro(check_arm_feature tag feature code) set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) - set(CMAKE_REQUIRED_FLAGS "${ARM_NATIVE_FLAG}+${tag}") + if (GGML_ARM_MSVC) + set(ARM_PROBE_FLAG "") + set(ARM_PROBE_NOFLAG "") + else() + set(ARM_PROBE_FLAG "${ARM_NATIVE_FLAG}+${tag}") + set(ARM_PROBE_NOFLAG "${ARM_NATIVE_FLAG}+no${tag}") + endif() + set(CMAKE_REQUIRED_FLAGS "${ARM_PROBE_FLAG}") check_cxx_source_runs("${code}" GGML_MACHINE_SUPPORTS_${tag}) if (GGML_MACHINE_SUPPORTS_${tag}) set(ARM_NATIVE_FLAG_FIX "${ARM_NATIVE_FLAG_FIX}+${tag}") + if (GGML_ARM_MSVC) + # forward runtime probing decisions for MSVC + list(APPEND ARCH_FLAGS "-D__ARM_FEATURE_${feature}=1") + endif() else() - set(CMAKE_REQUIRED_FLAGS "${ARM_NATIVE_FLAG}+no${tag}") + set(CMAKE_REQUIRED_FLAGS "${ARM_PROBE_NOFLAG}") check_cxx_source_compiles("int main() { return 0; }" GGML_MACHINE_SUPPORTS_no${tag}) if (GGML_MACHINE_SUPPORTS_no${tag}) set(ARM_NATIVE_FLAG_FIX "${ARM_NATIVE_FLAG_FIX}+no${tag}") @@ -167,12 +188,24 @@ function(ggml_add_cpu_backend_variant_impl tag_name) set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) endmacro() - check_arm_feature(dotprod DOTPROD "#include \nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }") - check_arm_feature(i8mm MATMUL_INT8 "#include \nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vmmlaq_s32(_s, _a, _b); return 0; }") - check_arm_feature(sve SVE "#include \nint main() { svfloat32_t _a, _b; volatile svfloat32_t _c = svadd_f32_z(svptrue_b8(), _a, _b); return 0; }") + if (GGML_ARM_MSVC) + # drop volatile + check_arm_feature(dotprod DOTPROD "#include \nint main() { int8x16_t _a = vdupq_n_s8(0), _b = vdupq_n_s8(0); int32x4_t _s = vdupq_n_s32(0); _s = vdotq_s32(_s, _a, _b); return 0; }") + check_arm_feature(i8mm MATMUL_INT8 "#include \nint main() { int8x16_t _a = vdupq_n_s8(0), _b = vdupq_n_s8(0); int32x4_t _s = vdupq_n_s32(0); _s = vmmlaq_s32(_s, _a, _b); return 0; }") + check_arm_feature(sve SVE "#include \nint main() { const svbool_t pg = svptrue_b8(); const svint8_t a = svdup_n_s8(0), b = svdup_n_s8(0); svint32_t s = svdup_n_s32(0); s = svdot_s32(s, a, b); (void) pg; (void) s; return 0; }") + # FMA is mandatory on ARMv8-A but MSVC never defines __ARM_FEATURE_FMA, + # so probe it like the others and forward the define + check_arm_feature(fma FMA "#include \nint main() { float32x4_t _a = vdupq_n_f32(0), _b = vdupq_n_f32(0), _c = vdupq_n_f32(0); _a = vfmaq_f32(_a, _b, _c); return 0; }") + else() + check_arm_feature(dotprod DOTPROD "#include \nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }") + check_arm_feature(i8mm MATMUL_INT8 "#include \nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vmmlaq_s32(_s, _a, _b); return 0; }") + check_arm_feature(sve SVE "#include \nint main() { svfloat32_t _a, _b; volatile svfloat32_t _c = svadd_f32_z(svptrue_b8(), _a, _b); return 0; }") + endif() check_arm_feature(sme SME "#include \n__arm_locally_streaming int main() { __asm__ volatile(\"smstart; smstop;\"); return 0; }") - list(APPEND ARCH_FLAGS "${ARM_NATIVE_FLAG}${ARM_NATIVE_FLAG_FIX}") + if (NOT GGML_ARM_MSVC) + list(APPEND ARCH_FLAGS "${ARM_NATIVE_FLAG}${ARM_NATIVE_FLAG_FIX}") + endif() else() if (GGML_CPU_ARM_ARCH) list(APPEND ARCH_FLAGS -march=${GGML_CPU_ARM_ARCH}) @@ -397,7 +430,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) string(REGEX MATCHALL "POWER *([0-9]+)" MATCHED_STRING "${POWER10_M_UPPER}") string(REGEX REPLACE "POWER *([0-9]+)" "\\1" EXTRACTED_NUMBER "${MATCHED_STRING}") - if (EXTRACTED_NUMBER GREATER_EQUAL 10) + if (EXTRACTED_NUMBER EQUAL 10 OR EXTRACTED_NUMBER EQUAL 11) list(APPEND ARCH_FLAGS -mcpu=power10) elseif (EXTRACTED_NUMBER EQUAL 9) list(APPEND ARCH_FLAGS -mcpu=power9) @@ -450,13 +483,27 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/arch/riscv/repack.cpp ) if (GGML_CPU_RISCV64_SPACEMIT) + include(ggml-cpu/cmake/FindSMTIME.cmake) target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_RISCV64_SPACEMIT ${RISCV64_SPACEMIT_IME_SPEC}) list(APPEND GGML_CPU_SOURCES ggml-cpu/spacemit/ime.cpp ggml-cpu/spacemit/ime.h - ggml-cpu/spacemit/ime1_kernels.cpp + ggml-cpu/spacemit/spine_mem_pool.cpp + ggml-cpu/spacemit/spine_mem_pool.h + ggml-cpu/spacemit/repack.cpp + ggml-cpu/spacemit/repack.h + ggml-cpu/spacemit/ime_env.cpp + ggml-cpu/spacemit/ime_env.h ggml-cpu/spacemit/ime_kernels.h + ggml-cpu/spacemit/rvv_kernels.cpp + ggml-cpu/spacemit/rvv_kernels.h ) + if ("RISCV64_SPACEMIT_IME1" IN_LIST RISCV64_SPACEMIT_IME_SPEC) + list(APPEND GGML_CPU_SOURCES ggml-cpu/spacemit/ime1_kernels.cpp) + endif() + if ("RISCV64_SPACEMIT_IME2" IN_LIST RISCV64_SPACEMIT_IME_SPEC) + list(APPEND GGML_CPU_SOURCES ggml-cpu/spacemit/ime2_kernels.cpp) + endif() endif() if(NOT GGML_CPU_ALL_VARIANTS) set(MARCH_STR "rv64gc") @@ -485,6 +532,16 @@ function(ggml_add_cpu_backend_variant_impl tag_name) if (GGML_RV_ZIHINTPAUSE) string(APPEND MARCH_STR "_zihintpause") endif() + if (GGML_RV_ZBA) + string(APPEND MARCH_STR "_zba") + endif() + if (GGML_CPU_RISCV64_SPACEMIT) + # `xsmtvdotii' is only required for GCC >= 15. + if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND + CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 15) + string(APPEND MARCH_STR "_xsmtvdotii") + endif() + endif() list(APPEND ARCH_FLAGS "-march=${MARCH_STR}" -mabi=lp64d) else() @@ -502,7 +559,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name) elseif (GGML_SYSTEM_ARCH STREQUAL "s390x") message(STATUS "s390x detected") list(APPEND GGML_CPU_SOURCES - ggml-cpu/arch/s390/quants.c) + ggml-cpu/arch/s390/quants.c + ggml-cpu/arch/s390/repack.cpp + ) # for native compilation if (GGML_NATIVE) @@ -534,7 +593,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name) foreach (ZHW RANGE 15 17) if(DEFINED GGML_INTERNAL_Z${ZHW}) message(STATUS "z${ZHW} cross-compile target") - list(APPEND ARCH_FLAGS -march=z${ZHW}) + if (ZHW EQUAL 17) + # z17 is an alias of arch15, use the arch level for wider toolchain support + list(APPEND ARCH_FLAGS -march=arch15) + else() + list(APPEND ARCH_FLAGS -march=z${ZHW}) + endif() endif() endforeach() endif() @@ -564,50 +628,83 @@ function(ggml_add_cpu_backend_variant_impl tag_name) endif() if (GGML_CPU_KLEIDIAI) - message(STATUS "Using KleidiAI optimized kernels if applicable") + # upstream repo requires at least cmake 3.16 + if (CMAKE_VERSION VERSION_LESS 3.16) + message(FATAL_ERROR "GGML_CPU_KLEIDIAI requires CMake >= 3.16") + endif() - # Disable the KleidiAI tests - set(KLEIDIAI_BUILD_TESTS OFF) + set(GGML_CPU_KLEIDIAI_AARCH64 OFF) + if (GGML_SYSTEM_ARCH STREQUAL "ARM" AND + (APPLE OR WIN32 OR CMAKE_SYSTEM_NAME MATCHES "^(Linux|Android)$") AND + (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64|arm64-v8a)$" OR + CMAKE_OSX_ARCHITECTURES MATCHES "arm64" OR + CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR + CMAKE_ANDROID_ARCH_ABI STREQUAL "arm64-v8a")) + set(GGML_CPU_KLEIDIAI_AARCH64 ON) + endif() + if (NOT GGML_CPU_KLEIDIAI_AARCH64) + message(FATAL_ERROR "GGML_CPU_KLEIDIAI requires a Linux, Android, Apple, or Windows AArch64/arm64 target") + endif() + + message(STATUS "Using KleidiAI optimized kernels if applicable") # Fetch KleidiAI sources: include(FetchContent) - set(KLEIDIAI_COMMIT_TAG "v1.22.0") - set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/archive/refs/tags/${KLEIDIAI_COMMIT_TAG}.tar.gz") - set(KLEIDIAI_ARCHIVE_MD5 "54049037570ab0ee0a0d126b2ba5ece1") + set(KLEIDIAI_COMMIT_TAG "v1.32.0") + set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz") + set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "05f1953428ffe8e8c7f585dc6d508887") set(KLEIDIAI_FETCH_ARGS URL ${KLEIDIAI_DOWNLOAD_URL} - URL_HASH MD5=${KLEIDIAI_ARCHIVE_MD5} + URL_HASH MD5=${KLEIDIAI_RELEASE_ARCHIVE_MD5} ) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") list(APPEND KLEIDIAI_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP NEW) endif() - if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.28") - FetchContent_Declare(KleidiAI_Download - ${KLEIDIAI_FETCH_ARGS} - EXCLUDE_FROM_ALL - ) + FetchContent_Declare(kleidiai + ${KLEIDIAI_FETCH_ARGS} + ) - FetchContent_MakeAvailable(KleidiAI_Download) - FetchContent_GetProperties(KleidiAI_Download SOURCE_DIR KLEIDIAI_SRC) - else() - FetchContent_Declare(KleidiAI_Download - ${KLEIDIAI_FETCH_ARGS} - ) + # Disable tests and benchmark building + set(KLEIDIAI_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(KLEIDIAI_BUILD_BENCHMARK OFF CACHE BOOL "" FORCE) - FetchContent_GetProperties(KleidiAI_Download + # Use the Populate/add_subdirectory flow for compatibility with CMake 3.16. + FetchContent_GetProperties(kleidiai + SOURCE_DIR KLEIDIAI_SRC + BINARY_DIR KLEIDIAI_BIN + POPULATED KLEIDIAI_POPULATED + ) + if (NOT KLEIDIAI_POPULATED) + FetchContent_Populate(kleidiai) + FetchContent_GetProperties(kleidiai SOURCE_DIR KLEIDIAI_SRC - POPULATED KLEIDIAI_POPULATED + BINARY_DIR KLEIDIAI_BIN ) + endif() - if (NOT KLEIDIAI_POPULATED) - FetchContent_Populate(KleidiAI_Download) - FetchContent_GetProperties(KleidiAI_Download SOURCE_DIR KLEIDIAI_SRC) + if (NOT TARGET kleidiai) + add_subdirectory( + "${CMAKE_CURRENT_SOURCE_DIR}/ggml-cpu/kleidiai" + "${CMAKE_CURRENT_BINARY_DIR}/kleidiai-wrapper" + EXCLUDE_FROM_ALL + ) + if (NOT CMAKE_SKIP_INSTALL_RULES AND + (NOT DEFINED BUILD_SHARED_LIBS OR NOT BUILD_SHARED_LIBS)) + install(TARGETS kleidiai ARCHIVE) endif() endif() - add_compile_definitions(GGML_USE_CPU_KLEIDIAI) + if (NOT TARGET kleidiai) + message(FATAL_ERROR "KleidiAI target was not created") + endif() + + set_target_properties(kleidiai PROPERTIES POSITION_INDEPENDENT_CODE ON) + + target_link_libraries(${GGML_CPU_NAME} PRIVATE kleidiai) + + target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_KLEIDIAI) list(APPEND GGML_CPU_SOURCES ggml-cpu/kleidiai/kleidiai.cpp @@ -615,84 +712,6 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/kleidiai/kleidiai.h ggml-cpu/kleidiai/kernels.h ) - - # KleidiAI - include_directories( - ${KLEIDIAI_SRC}/ - ${KLEIDIAI_SRC}/kai/ - ${KLEIDIAI_SRC}/kai/ukernels/ - ${KLEIDIAI_SRC}/kai/ukernels/matmul/ - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/ - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/ - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/ - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/ - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/) - - set(ARCH_FLAGS_TEMP "${ARCH_FLAGS}") - if (NOT ARCH_FLAGS_TEMP) - string(REGEX MATCH "-march=[^ ]+" ARCH_FLAGS_TEMP "${CMAKE_C_FLAGS}") - endif() - string(FIND "${ARCH_FLAGS_TEMP}" "+dotprod" DOTPROD_ENABLED) - string(FIND "${ARCH_FLAGS_TEMP}" "+i8mm" I8MM_ENABLED) - string(FIND "${ARCH_FLAGS_TEMP}" "+sme" SME_ENABLED) - string(FIND "${ARCH_FLAGS_TEMP}" "+sve" SVE_ENABLED) - - set(PRIVATE_ARCH_FLAGS ${ARCH_FLAGS_TEMP}) - - list(APPEND GGML_KLEIDIAI_SOURCES - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.c) - - if (NOT DOTPROD_ENABLED MATCHES -1) - list(APPEND GGML_KLEIDIAI_SOURCES - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.c) - endif() - - if (NOT I8MM_ENABLED MATCHES -1) - list(APPEND GGML_KLEIDIAI_SOURCES - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.c) - endif() - - if (NOT SME_ENABLED MATCHES -1) - list(APPEND GGML_KLEIDIAI_SOURCES - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa_asm.S - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot_asm.S - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa_asm.S - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa_asm.S - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_bf16p2vlx2_f32_sme.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_f16pmrx2_f32_neon.c - ${KLEIDIAI_SRC}/kai/kai_common_sme_asm.S) - set(PRIVATE_ARCH_FLAGS "-fno-tree-vectorize;${PRIVATE_ARCH_FLAGS}+sve+sve2+sme2+fp16") - endif() - - if (NOT SVE_ENABLED MATCHES -1) - list(APPEND GGML_KLEIDIAI_SOURCES - ${KLEIDIAI_SRC}/kai/kai_common_sve_asm.S - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod_asm.S - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.c - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm_asm.S - ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.c) - endif() - - set_source_files_properties(${GGML_KLEIDIAI_SOURCES} PROPERTIES COMPILE_OPTIONS "${PRIVATE_ARCH_FLAGS}") - list(APPEND GGML_CPU_SOURCES ${GGML_KLEIDIAI_SOURCES}) endif() message(STATUS "Adding CPU backend variant ${GGML_CPU_NAME}: ${ARCH_FLAGS} ${ARCH_DEFINITIONS}") @@ -704,8 +723,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name) set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS "-msimd128") endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - # The compiler automatically enables "-ffast-math" which can cause NaNs in tests due to "-fassociative-math" - target_compile_options(${GGML_CPU_NAME} PRIVATE "-fno-associative-math") - endif() + if (CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") + # The compiler automatically enables "-ffast-math" which can cause NaNs in tests due to "-fassociative-math" + target_compile_options(${GGML_CPU_NAME} PRIVATE "$<$,$>:$<$:/clang:>-fno-associative-math>") + endif() + endfunction() diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp index 1118f71..e90c279 100644 --- a/ggml/src/ggml-cpu/amx/amx.cpp +++ b/ggml/src/ggml-cpu/amx/amx.cpp @@ -228,12 +228,14 @@ static bool ggml_amx_init() { ggml_backend_buffer_type_t ggml_backend_amx_buffer_type() { static struct ggml_backend_buffer_type ggml_backend_buffer_type_amx = { /* .iface = */ { - /* .get_name = */ ggml_backend_amx_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_amx_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_amx_buffer_type_get_alignment, - /* .get_max_size = */ nullptr, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_amx_buffer_type_get_alloc_size, - /* .is_host = */ nullptr, + /* .get_name = */ ggml_backend_amx_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_amx_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ nullptr, + /* .get_alignment = */ ggml_backend_amx_buffer_type_get_alignment, + /* .get_max_size = */ nullptr, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_amx_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ nullptr, }, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0), /* .context = */ new ggml::cpu::amx::extra_buffer_type(), diff --git a/ggml/src/ggml-cpu/amx/mmq.cpp b/ggml/src/ggml-cpu/amx/mmq.cpp index 93a6d39..9f3a744 100644 --- a/ggml/src/ggml-cpu/amx/mmq.cpp +++ b/ggml/src/ggml-cpu/amx/mmq.cpp @@ -2005,12 +2005,12 @@ void tinygemm_kernel_amx(int M, int N, int KB, const void * RESTRICT _A, const v const int lda = KB * sizeof(TA); //const int ldb = KB * sizeof(TB); - static thread_local packed_B_t Tile0[TILE_N * TILE_K]; - static thread_local packed_B_t Tile1[TILE_N * TILE_K]; - static thread_local int8_t Tile23[TILE_M * TILE_K]; + alignas(64) static thread_local packed_B_t Tile0[TILE_N * TILE_K]; + alignas(64) static thread_local packed_B_t Tile1[TILE_N * TILE_K]; + alignas(64) static thread_local int8_t Tile23[TILE_M * TILE_K]; - static thread_local int32_t TileC0[TILE_M * TILE_N * 4]; - static thread_local int32_t TileC1[TILE_M * TILE_N * 4]; + alignas(64) static thread_local int32_t TileC0[TILE_M * TILE_N * 4]; + alignas(64) static thread_local int32_t TileC1[TILE_M * TILE_N * 4]; // double buffering C to interleave avx512 and amx int32_t * C_cur = TileC0; @@ -2187,21 +2187,21 @@ void tinygemm_kernel_amx(int M, int N, int KB, const void * RESTRICT _A, const v const int m1 = std::max(M - TILE_M, 0); //const int lda = KB * sizeof(TA); - static thread_local int8_t Tile0[TILE_N * TILE_K]; - static thread_local int8_t Tile1[TILE_N * TILE_K]; - static thread_local int8_t Tile23[TILE_M * TILE_K]; + alignas(64) static thread_local int8_t Tile0[TILE_N * TILE_K]; + alignas(64) static thread_local int8_t Tile1[TILE_N * TILE_K]; + alignas(64) static thread_local int8_t Tile23[TILE_M * TILE_K]; // mat mul result for each group - static thread_local int32_t Tile4[TILE_M * TILE_N]; - static thread_local int32_t Tile5[TILE_M * TILE_N]; - static thread_local int32_t Tile6[TILE_M * TILE_N]; - static thread_local int32_t Tile7[TILE_M * TILE_N]; + alignas(64) static thread_local int32_t Tile4[TILE_M * TILE_N]; + alignas(64) static thread_local int32_t Tile5[TILE_M * TILE_N]; + alignas(64) static thread_local int32_t Tile6[TILE_M * TILE_N]; + alignas(64) static thread_local int32_t Tile7[TILE_M * TILE_N]; // sum of each QK_K block, contains 8 groups, int32 - static thread_local int32_t Sumi4[TILE_M * TILE_N]; - static thread_local int32_t Sumi5[TILE_M * TILE_N]; - static thread_local int32_t Sumi6[TILE_M * TILE_N]; - static thread_local int32_t Sumi7[TILE_M * TILE_N]; + alignas(64) static thread_local int32_t Sumi4[TILE_M * TILE_N]; + alignas(64) static thread_local int32_t Sumi5[TILE_M * TILE_N]; + alignas(64) static thread_local int32_t Sumi6[TILE_M * TILE_N]; + alignas(64) static thread_local int32_t Sumi7[TILE_M * TILE_N]; const int k_group_size = std::is_same::value ? 16 : 32; for (int i = 0; i < KB; ++i) { @@ -2417,15 +2417,14 @@ void ggml_backend_amx_mul_mat(const ggml_compute_params * params, struct ggml_te // Q4_K, Q5_K, Q6_K, IQ4_XS handles 8 TILE_K per blck_size GGML_ASSERT(TILE_K == blck_size || TILE_K * 8 == blck_size); - parallel_for_ggml(params, n_batch, [&](int begin, int end) { - for (int batch_idx = begin; batch_idx < end; ++batch_idx) { + parallel_for_ggml(params, n_batch * M, [&](int begin, int end) { + for (int idx = begin; idx < end; ++idx) { + int batch_idx = idx / M; + int m = idx % M; int64_t src1_offset = ggml_batch_offset(src1, batch_idx, ne2); const float * A_data = (const float *)((const char *)src1->data + src1_offset); char * wdata_batch = (char *)wdata + batch_idx * M * row_size_A; - - for (int m = 0; m < M; ++m) { - from_float(A_data + m * K, wdata_batch + m * row_size_A, K); - } + from_float(A_data + m * K, wdata_batch + m * row_size_A, K); } }); }); diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index c589a21..4e59d0a 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -17,6 +17,7 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K @@ -38,6 +39,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -54,6 +57,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -70,7 +75,7 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 -#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) +#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 @@ -81,12 +86,11 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) -// quants.c -#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 -#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K @@ -98,6 +102,8 @@ #define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K @@ -115,6 +121,7 @@ #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K @@ -123,6 +130,8 @@ #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -139,6 +148,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -164,11 +175,14 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -185,6 +199,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 @@ -204,13 +220,15 @@ #elif defined(__riscv) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 -#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1 #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_K_4x1_generic ggml_quantize_mat_q8_K_4x1 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K @@ -226,6 +244,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K @@ -245,7 +265,7 @@ // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 -#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K @@ -257,11 +277,11 @@ #define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K // repack.cpp -#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 -#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 #define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K @@ -277,7 +297,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 -#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K @@ -295,7 +316,6 @@ #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__wasm__) // quants.c -#define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1 #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K @@ -310,11 +330,14 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 +#define ggml_gemv_q1_0_4x4_q8_0_generic ggml_gemv_q1_0_4x4_q8_0 +#define ggml_gemv_q1_0_4x8_q8_0_generic ggml_gemv_q1_0_4x8_q8_0 #define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0 #define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0 #define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0 @@ -331,6 +354,8 @@ #define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0 #define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0 #define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0 +#define ggml_gemm_q1_0_4x4_q8_0_generic ggml_gemm_q1_0_4x4_q8_0 +#define ggml_gemm_q1_0_4x8_q8_0_generic ggml_gemm_q1_0_4x8_q8_0 #define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0 #define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0 #define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0 diff --git a/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp b/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp index c460c54..84a11ea 100644 --- a/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +++ b/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp @@ -1,76 +1,19 @@ #include "ggml-backend-impl.h" +#include "ggml-feats.h" -#if defined(__aarch64__) - -#if defined(__linux__) -#include -#elif defined(__APPLE__) -#include -#endif - -#if !defined(HWCAP2_SVE2) -#define HWCAP2_SVE2 (1 << 1) -#endif - -#if !defined(HWCAP2_I8MM) -#define HWCAP2_I8MM (1 << 13) -#endif - -#if !defined(HWCAP2_SME) -#define HWCAP2_SME (1 << 23) -#endif - -struct aarch64_features { - // has_neon not needed, aarch64 has NEON guaranteed - bool has_dotprod = false; - bool has_fp16_va = false; - bool has_sve = false; - bool has_sve2 = false; - bool has_i8mm = false; - bool has_sme = false; - - aarch64_features() { -#if defined(__linux__) - uint32_t hwcap = getauxval(AT_HWCAP); - uint32_t hwcap2 = getauxval(AT_HWCAP2); - - has_dotprod = !!(hwcap & HWCAP_ASIMDDP); - has_fp16_va = !!(hwcap & HWCAP_FPHP); - has_sve = !!(hwcap & HWCAP_SVE); - has_sve2 = !!(hwcap2 & HWCAP2_SVE2); - has_i8mm = !!(hwcap2 & HWCAP2_I8MM); - has_sme = !!(hwcap2 & HWCAP2_SME); -#elif defined(__APPLE__) - int oldp = 0; - size_t size = sizeof(oldp); - - if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, NULL, 0) == 0) { - has_dotprod = static_cast(oldp); - } - - if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, NULL, 0) == 0) { - has_i8mm = static_cast(oldp); - } - - if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, NULL, 0) == 0) { - has_sme = static_cast(oldp); - } - - // Apple apparently does not implement SVE yet -#endif - } -}; +#if defined(__aarch64__) || defined(_M_ARM64) static int ggml_backend_cpu_aarch64_score() { int score = 1; - aarch64_features af; + const ggml_feats_arch64_runtime_t af = ggml_feats_get_arch64_runtime(); + GGML_UNUSED(af); #ifdef GGML_USE_DOTPROD if (!af.has_dotprod) { return 0; } score += 1<<1; #endif #ifdef GGML_USE_FP16_VECTOR_ARITHMETIC - if (!af.has_fp16_va) { return 0; } + if (!af.has_fp16) { return 0; } score += 1<<2; #endif #ifdef GGML_USE_SVE @@ -95,4 +38,4 @@ static int ggml_backend_cpu_aarch64_score() { GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_aarch64_score) -# endif // defined(__aarch64__) +# endif // defined(__aarch64__) || defined(_M_ARM64) diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index 64d811f..f16505b 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -151,8 +151,6 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi const block_q1_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; - float sumf = 0.0f; - #if defined(__ARM_NEON) float32x4_t sumv = vdupq_n_f32(0.0f); @@ -212,34 +210,93 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi } } - sumf = vaddvq_f32(sumv); + *s = vaddvq_f32(sumv); #else - // Scalar fallback + UNUSED(nb); + UNUSED(x); + UNUSED(y); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + +void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK2_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + +#if defined(__ARM_NEON) + // Replicate pattern: each byte repeated 4 times + static const uint8_t tbl_idx_lo[16] = {0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3}; + static const uint8_t tbl_idx_hi[16] = {4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7}; + // Right-shift amounts: 0,2,4,6 repeated for each group of 4 + static const int8_t shift_vals[16] = {0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6}; + + const uint8x16_t idx_lo = vld1q_u8(tbl_idx_lo); + const uint8x16_t idx_hi = vld1q_u8(tbl_idx_hi); + const int8x16_t shifts = vld1q_s8(shift_vals); + const uint8x16_t mask2 = vdupq_n_u8(0x03); + const int8x16_t one = vdupq_n_s8(1); + + float32x4_t sumv = vdupq_n_f32(0.0f); + for (int i = 0; i < nb; i++) { - const float d0 = GGML_FP16_TO_FP32(x[i].d); + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); - // Process 4 Q8_0 blocks - for (int k = 0; k < 4; k++) { - const float d1 = GGML_FP16_TO_FP32(y[i*4 + k].d); + // group 64: one Q2_0 block (64 weights) maps to two Q8_0 blocks (2 * 32 = 64) + for (int k = 0; k < 2; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 2 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); - int sumi = 0; - for (int j = 0; j < QK8_0; j++) { - const int bit_index = k * QK8_0 + j; - const int byte_index = bit_index / 8; - const int bit_offset = bit_index % 8; + // Load 8 bytes of packed 2-bit values + const uint8x8_t raw = vld1_u8(&x[i].qs[k * 8]); + const uint8x16_t raw16 = vcombine_u8(raw, raw); - const int xi = ((x[i].qs[byte_index] >> bit_offset) & 1) ? 1 : -1; - sumi += xi * y[i*4 + k].qs[j]; - } - sumf += d0 * d1 * sumi; + // First 16 elements: replicate bytes 0-3, shift, mask, subtract 1 + uint8x16_t bytes0 = ggml_vqtbl1q_u8(raw16, idx_lo); + int8x16_t qv0 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes0, shifts), mask2)), + one); + + // Second 16 elements: replicate bytes 4-7, shift, mask, subtract 1 + uint8x16_t bytes1 = ggml_vqtbl1q_u8(raw16, idx_hi); + int8x16_t qv1 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshlq_u8(bytes1, shifts), mask2)), + one); + + // Load Q8_0 values and dot product + const int8x16_t y0 = vld1q_s8(yb->qs); + const int8x16_t y1 = vld1q_s8(yb->qs + 16); + + int32x4_t p0 = ggml_vdotq_s32(vdupq_n_s32(0), qv0, y0); + int32x4_t p1 = ggml_vdotq_s32(p0, qv1, y1); + + sumv = vmlaq_n_f32(sumv, vcvtq_f32_s32(p1), d0 * d1); } } + + sumf = vaddvq_f32(sumv); +#else + UNUSED(nb); + UNUSED(x); + UNUSED(y); + ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + return; #endif *s = sumf; } - void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -821,23 +878,23 @@ void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo const int8x8_t q8_3_lo = vld1_s8(y[2*ib+1].qs + 16); const int8x8_t q8_3_hi = vld1_s8(y[2*ib+1].qs + 24); - const int32x4_t sumi = (int32x4_t){ + const int32x4_t sumi = vld1q_s32(((const int32_t[4]) { vaddvq_s32(ggml_nvfp4_dot8(q4_0_lo, q8_0_lo, q4_0_hi, q8_0_hi)), vaddvq_s32(ggml_nvfp4_dot8(q4_1_lo, q8_1_lo, q4_1_hi, q8_1_hi)), vaddvq_s32(ggml_nvfp4_dot8(q4_2_lo, q8_2_lo, q4_2_hi, q8_2_hi)), vaddvq_s32(ggml_nvfp4_dot8(q4_3_lo, q8_3_lo, q4_3_hi, q8_3_hi)), - }; + })); #endif const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); - const float32x4_t nvsc = { - ggml_ue4m3_to_fp32(x[ib].d[0]), - ggml_ue4m3_to_fp32(x[ib].d[1]), - ggml_ue4m3_to_fp32(x[ib].d[2]), - ggml_ue4m3_to_fp32(x[ib].d[3]) - }; - const float32x4_t scales = vmulq_f32(nvsc, (float32x4_t){dy0, dy0, dy1, dy1}); + const float32x4_t nvsc = vld1q_f32(((const float[4]) { + GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]), + GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]), + GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]), + GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) + })); + const float32x4_t scales = vmulq_f32(nvsc, vld1q_f32(((const float[4]) {dy0, dy0, dy1, dy1}))); acc = vfmaq_f32(acc, vcvtq_f32_s32(sumi), scales); } @@ -2537,7 +2594,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi memcpy(scales_mins, x0->scales, 12); const uint32_t mins_0_3 = scales_mins[1] & kmask1; const uint32_t mins_4_7 = ((scales_mins[2] >> 4) & kmask2) | (((scales_mins[1] >> 6) & kmask3) << 4); - const uint32x2_t mins = {mins_0_3, mins_4_7}; + const uint32x2_t mins = vcreate_u32((uint64_t) mins_0_3 | ((uint64_t) mins_4_7 << 32)); x0_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins))); uint32_t scales[2]; scales[0] = scales_mins[0] & kmask1; // scales 0~3 @@ -2549,7 +2606,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi memcpy(scales_mins, x1->scales, 12); const uint32_t mins_0_3 = scales_mins[1] & kmask1; const uint32_t mins_4_7 = ((scales_mins[2] >> 4) & kmask2) | (((scales_mins[1] >> 6) & kmask3) << 4); - const uint32x2_t mins = {mins_0_3, mins_4_7}; + const uint32x2_t mins = vcreate_u32((uint64_t) mins_0_3 | ((uint64_t) mins_4_7 << 32)); x1_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins))); uint32_t scales[2]; scales[0] = scales_mins[0] & kmask1; // scales 0~3 @@ -2557,7 +2614,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi memcpy(x1_scales, scales, 8); } - int32x4_t visum = {0}; + int32x4_t visum = vdupq_n_s32(0); // process 64 data points per iteration, totally 256 data points for (int j = 0; j < QK_K / 64; ++j, qx0 += 32, qx1 += 32, qy0 += 64, qy1 += 64) { @@ -2583,14 +2640,14 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi // process 32 data points (share same block scale) per iteration for (int k = 0; k < 2; ++k) { const int blk = j * 2 + k; - const int32x4_t block_scale = { + const int32x4_t block_scale = vld1q_s32(((const int32_t[4]) { x0_scales[blk], x0_scales[blk], x1_scales[blk], x1_scales[blk], - }; + })); - int32x4_t vr = {0}; + int32x4_t vr = vdupq_n_s32(0); for (int l = 0; l < 2; ++l) { const int idx = k * 2 + l; const int64x2_t vx0_s64 = vreinterpretq_s64_s8(vx0[idx]); @@ -2624,20 +2681,22 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi vmull_s16(vget_high_s16(y0_sums), vget_high_s16(x1_mins)))); bias[3] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y1_sums), vget_low_s16(x1_mins)), vmull_s16(vget_high_s16(y1_sums), vget_high_s16(x1_mins)))); - const float32x4_t dmins = { + // note: the parentheses around the compound literal are required, vld1q_f32() is a + // function-like macro on MSVC and would otherwise see four separate arguments + const float32x4_t dmins = vld1q_f32(((const float[4]) { GGML_CPU_FP16_TO_FP32(x0->dmin) * y0->d, GGML_CPU_FP16_TO_FP32(x0->dmin) * y1->d, GGML_CPU_FP16_TO_FP32(x1->dmin) * y0->d, GGML_CPU_FP16_TO_FP32(x1->dmin) * y1->d, - }; + })); vfsum = vmlsq_f32(vfsum, vcvtq_f32_s32(vld1q_s32(bias)), dmins); - const float32x4_t superblock_scale = { + const float32x4_t superblock_scale = vld1q_f32(((const float[4]) { GGML_CPU_FP16_TO_FP32(x0->d) * y0->d, GGML_CPU_FP16_TO_FP32(x0->d) * y1->d, GGML_CPU_FP16_TO_FP32(x1->d) * y0->d, GGML_CPU_FP16_TO_FP32(x1->d) * y1->d, - }; + })); vfsum = vmlaq_f32(vfsum, vcvtq_f32_s32(visum), superblock_scale); } } @@ -3207,12 +3266,8 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi qy0 += 16; qy1 += 16; - const int32x4_t block_scale = { - x0->scales[blk], - x0->scales[blk], - x1->scales[blk], - x1->scales[blk], - }; + const int32x4_t block_scale = + vcombine_s32(vdup_n_s32(x0->scales[blk]), vdup_n_s32(x1->scales[blk])); // calculate four results at once with outer product const int8x16_t vx_l = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(vx0[k]), vreinterpretq_s64_s8(vx1[k]))); @@ -3265,12 +3320,14 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi const int32x4_t vibias = vmulq_n_s32(vld1q_s32(bias), 32); - const float32x4_t superblock_scale = { + // note: the parentheses around the compound literal are required, vld1q_f32() is a + // function-like macro on MSVC and would otherwise see four separate arguments + const float32x4_t superblock_scale = vld1q_f32(((const float[4]) { GGML_CPU_FP16_TO_FP32(x0->d) * y0->d, GGML_CPU_FP16_TO_FP32(x0->d) * y1->d, GGML_CPU_FP16_TO_FP32(x1->d) * y0->d, GGML_CPU_FP16_TO_FP32(x1->d) * y1->d, - }; + })); visum = vsubq_s32(visum, vibias); vfsum = vmlaq_f32(vfsum, vcvtq_f32_s32(visum), superblock_scale); diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 80ff5ce..ad0e5cc 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -48,6 +48,24 @@ static inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t } #endif +#if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_FEATURE_DOTPROD) || defined(__ARM_FEATURE_MATMUL_INT8)) +#define B1(c,s,n) 0x ## n ## c , 0x ## n ## s +#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s) +#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s) +#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s) +#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s) +#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s) +#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s) +#define B8(c,s ) B7(c,s, c), B7(c,s, s) + +static const uint64_t table_q1_signs[256] = { B8(ff, 01) }; + +static inline int8x16_t ggml_q1_0_unpack_pair(uint8_t bits0, uint8_t bits1) { + return vreinterpretq_s8_u8(vcombine_u8(vcreate_u8(table_q1_signs[bits0]), + vcreate_u8(table_q1_signs[bits1]))); +} +#endif + void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { assert(QK8_0 == 32); assert(k % QK8_0 == 0); @@ -1823,6 +1841,132 @@ void ggml_gemv_q8_0_4x8_q8_0(int n, ggml_gemv_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } +void ggml_gemv_q1_0_4x4_q8_0(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(nb); + UNUSED(ncols_interleaved); + +#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) + for (int c = 0; c < nc; c += ncols_interleaved) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (c / ncols_interleaved) * nb; + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + float32x4_t acc = vdupq_n_f32(0); + + for (int l = 0; l < nb; l++) { + const float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d)); + float32x4_t accb = vdupq_n_f32(0); + + for (int k = 0; k < 4; k++) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * 4 + k; + const float ad = GGML_CPU_FP16_TO_FP32(a_blk->d); + int32x4_t ret = vdupq_n_s32(0); + + for (int tile = 0; tile < 8; tile += 4) { + const int8x16_t signs0 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 0) + 0], + b_ptr[l].qs[k * 16 + 2 * (tile + 0) + 1]); + const int8x16_t signs1 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 1) + 0], + b_ptr[l].qs[k * 16 + 2 * (tile + 1) + 1]); + const int8x16_t signs2 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 2) + 0], + b_ptr[l].qs[k * 16 + 2 * (tile + 2) + 1]); + const int8x16_t signs3 = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * (tile + 3) + 0], + b_ptr[l].qs[k * 16 + 2 * (tile + 3) + 1]); + const int8x16_t q_tiles = vld1q_s8(a_blk->qs + tile * 4); + + ret = vdotq_laneq_s32(ret, signs0, q_tiles, 0); + ret = vdotq_laneq_s32(ret, signs1, q_tiles, 1); + ret = vdotq_laneq_s32(ret, signs2, q_tiles, 2); + ret = vdotq_laneq_s32(ret, signs3, q_tiles, 3); + } + + accb = vfmaq_n_f32(accb, vcvtq_f32_s32(ret), ad); + } + acc = vfmaq_f32(acc, accb, b_d); + } + vst1q_f32(s, acc); + s += ncols_interleaved; + } + return; +#endif + ggml_gemv_q1_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemv_q1_0_4x8_q8_0(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(nb); + UNUSED(ncols_interleaved); + +#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) + for (int c = 0; c < nc; c += ncols_interleaved) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (c / ncols_interleaved) * nb; + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + float32x4_t acc = vdupq_n_f32(0); + + for (int l = 0; l < nb; l++) { + const float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d)); + float32x4_t accb = vdupq_n_f32(0); + + for (int k = 0; k < 4; ++k) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * 4 + k; + const uint8_t * GGML_RESTRICT b_qs = (const uint8_t *) b_ptr[l].qs + k * 16; + const float ad = GGML_CPU_FP16_TO_FP32(a_blk->d); + + int8x8x4_t a_chunks = vld1_s8_x4(a_blk->qs); + int8x16_t a0 = vcombine_s8(a_chunks.val[0], a_chunks.val[0]); + int8x16_t a1 = vcombine_s8(a_chunks.val[1], a_chunks.val[1]); + int8x16_t a2 = vcombine_s8(a_chunks.val[2], a_chunks.val[2]); + int8x16_t a3 = vcombine_s8(a_chunks.val[3], a_chunks.val[3]); + + int32x4_t ret0 = vdupq_n_s32(0); + int32x4_t ret1 = vdupq_n_s32(0); + + ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[0], b_qs[1]), a0); + ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[2], b_qs[3]), a0); + ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[4], b_qs[5]), a1); + ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[6], b_qs[7]), a1); + ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[8], b_qs[9]), a2); + ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[10], b_qs[11]), a2); + ret0 = vdotq_s32(ret0, ggml_q1_0_unpack_pair(b_qs[12], b_qs[13]), a3); + ret1 = vdotq_s32(ret1, ggml_q1_0_unpack_pair(b_qs[14], b_qs[15]), a3); + + accb = vfmaq_n_f32(accb, vcvtq_f32_s32(vpaddq_s32(ret0, ret1)), ad); + } + + acc = vfmaq_f32(acc, accb, b_d); + } + + vst1q_f32(s, acc); + s += ncols_interleaved; + } + return; +#endif + + ggml_gemv_q1_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { const int qk = QK8_0; const int nb = n / qk; @@ -5023,6 +5167,71 @@ void ggml_gemm_q8_0_4x8_q8_0(int n, UNUSED(ncols_interleaved); UNUSED(blocklen); +#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) + if (svcntb() * 8 == 256) { + const block_q8_0x4 * b_ptr_base = (const block_q8_0x4 *) vx; + + static const uint32_t idx_arr[8] = {0, 1, 4, 5, 2, 3, 6, 7}; + svuint32_t idx = svld1(svptrue_b32(), idx_arr); + static const uint32_t idx_arr1[8] = {0, 1, 2, 3, 1, 2, 3, 0}; + svuint32_t idx_sc1 = svld1(svptrue_b32(), idx_arr1); + static const uint32_t idx_arr2[8] = {0, 1, 2, 3, 0, 1, 2, 3}; + svuint32_t idx_sc2 = svld1(svptrue_b32(), idx_arr2); + + for (int y = 0; y < nr; y += 4) { + const block_q8_0x4 * a_ptr_base = (const block_q8_0x4 *) vy + (y / 4) * nb; + + for (int x = 0; x < nc; x += ncols_interleaved) { + const block_q8_0x4 * b_ptr = b_ptr_base + (x / 4) * nb; + const block_q8_0x4 * a_ptr = a_ptr_base; + + svfloat32_t acc_f32_01 = svdup_f32(0); + svfloat32_t acc_f32_23 = svdup_f32(0); + + for (int b = 0; b < nb; b++) { + + svint32_t acc_01 = svdup_s32(0); + svint32_t acc_23 = svdup_s32(0); + + // Process 4 chunks of 8 positions each + for (int chunk = 0; chunk < 4; chunk++) { + svint8_t s_a01 = svld1rq_s8(svptrue_b8(), a_ptr->qs + chunk * 32); + svint8_t s_a23 = svld1rq_s8(svptrue_b8(), a_ptr->qs + chunk * 32 + 16); + svint8_t s_b0123 = svld1_s8(svptrue_b8(), b_ptr->qs + chunk * 32); + + acc_01 = svmmla_s32(acc_01, s_a01, s_b0123); + acc_23 = svmmla_s32(acc_23, s_a23, s_b0123); + } + + // Reorder outputs from 2×2 tiles to row-major + // acc[01] = [r0c0, r0c1, r1c0, r1c1, r0c2, r0c3, r1c2, r1c3] + // acc[23] = [r2c0, r2c1, r3c0, r3c1, r2c2, r2c3, r3c2, r3c3] + + svint32_t row01 = svtbl_s32(acc_01, idx); + svint32_t row23 = svtbl_s32(acc_23, idx); + + svfloat16_t temp1 = svld1_f16(svptrue_pat_b16(SV_VL4), (const __fp16 *) a_ptr->d); + svfloat16_t temp2 = svld1_f16(svptrue_pat_b16(SV_VL4), (const __fp16 *) b_ptr->d); + svfloat32_t sv_a_d = svtbl_f32(svcvt_f32_f16_x(svptrue_b32(), svzip1_f16(temp1, temp1)), idx_sc1); + svfloat32_t sv_b_d = svtbl_f32(svcvt_f32_f16_x(svptrue_b32(), svzip1_f16(temp2, temp2)), idx_sc2); + + acc_f32_01 = svmla_f32_x(svptrue_b32(), acc_f32_01, svcvt_f32_s32_x(svptrue_b32(), row01), svmul_lane_f32(sv_b_d, sv_a_d, 0)); + acc_f32_23 = svmla_f32_x(svptrue_b32(), acc_f32_23, svcvt_f32_s32_x(svptrue_b32(), row23), svmul_lane_f32(sv_b_d, sv_a_d, 2)); + a_ptr++; + b_ptr++; + } + + svbool_t pg4 = svptrue_pat_b32(SV_VL4); + svst1_f32(pg4, s + (y+0) * bs + x, acc_f32_01); + svst1_f32(pg4, s + (y+1) * bs + x, svext_f32(acc_f32_01, acc_f32_01, 4)); + svst1_f32(pg4, s + (y+2) * bs + x, acc_f32_23); + svst1_f32(pg4, s + (y+3) * bs + x, svext_f32(acc_f32_23, acc_f32_23, 4)); + } + } + return; + } +#endif // SVE compile-time end + #if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) const block_q8_0x4 * b_ptr_base = (const block_q8_0x4 *) vx; @@ -5089,3 +5298,168 @@ void ggml_gemm_q8_0_4x8_q8_0(int n, #endif // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) ggml_gemm_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); } + +void ggml_gemm_q1_0_4x4_q8_0(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(nb); + UNUSED(ncols_interleaved); + +#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + float32x4_t sumf[4]; + for (int m = 0; m < 4; m++) { + sumf[m] = vdupq_n_f32(0); + } + + for (int l = 0; l < nb; l++) { + float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d)); + float32x4_t blockf_0 = vdupq_n_f32(0); + float32x4_t blockf_1 = vdupq_n_f32(0); + float32x4_t blockf_2 = vdupq_n_f32(0); + float32x4_t blockf_3 = vdupq_n_f32(0); + + for (int k = 0; k < 4; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k; + float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *) a_blk->d)); + + int32x4_t sumi_0 = vdupq_n_s32(0); + int32x4_t sumi_1 = vdupq_n_s32(0); + int32x4_t sumi_2 = vdupq_n_s32(0); + int32x4_t sumi_3 = vdupq_n_s32(0); + + for (int tile = 0; tile < 8; ++tile) { + const int8x16_t signs = ggml_q1_0_unpack_pair(b_ptr[l].qs[k * 16 + 2 * tile + 0], + b_ptr[l].qs[k * 16 + 2 * tile + 1]); + const int8x16_t a_tile = vld1q_s8(a_blk->qs + tile * 16); + + sumi_0 = vdotq_laneq_s32(sumi_0, signs, a_tile, 0); + sumi_1 = vdotq_laneq_s32(sumi_1, signs, a_tile, 1); + sumi_2 = vdotq_laneq_s32(sumi_2, signs, a_tile, 2); + sumi_3 = vdotq_laneq_s32(sumi_3, signs, a_tile, 3); + } + + blockf_0 = vfmaq_laneq_f32(blockf_0, vcvtq_f32_s32(sumi_0), a_d, 0); + blockf_1 = vfmaq_laneq_f32(blockf_1, vcvtq_f32_s32(sumi_1), a_d, 1); + blockf_2 = vfmaq_laneq_f32(blockf_2, vcvtq_f32_s32(sumi_2), a_d, 2); + blockf_3 = vfmaq_laneq_f32(blockf_3, vcvtq_f32_s32(sumi_3), a_d, 3); + } + + sumf[0] = vfmaq_f32(sumf[0], blockf_0, b_d); + sumf[1] = vfmaq_f32(sumf[1], blockf_1, b_d); + sumf[2] = vfmaq_f32(sumf[2], blockf_2, b_d); + sumf[3] = vfmaq_f32(sumf[3], blockf_3, b_d); + } + + for (int m = 0; m < 4; m++) { + vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]); + } + } + } + return; +#endif + ggml_gemm_q1_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} + +void ggml_gemm_q1_0_4x8_q8_0(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(nb); + UNUSED(ncols_interleaved); + +#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + float32x4_t sumf[4]; + for (int m = 0; m < 4; ++m) { + sumf[m] = vdupq_n_f32(0); + } + + for (int l = 0; l < nb; l++) { + const float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d)); + float32x4_t blockf[4]; + for (int m = 0; m < 4; ++m) { + blockf[m] = vdupq_n_f32(0); + } + + for (int k = 0; k < 4; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k; + const uint8_t * GGML_RESTRICT b_qs = (const uint8_t *) b_ptr[l].qs + k * 16; + + int32x4_t acc[4]; + for (int i = 0; i < 4; ++i) { + acc[i] = vdupq_n_s32(0); + } + + for (int chunk = 0; chunk < 4; ++chunk) { + const int8x16_t a01 = vld1q_s8(a_blk->qs + chunk * 32); + const int8x16_t a23 = vld1q_s8(a_blk->qs + chunk * 32 + 16); + const int8x16_t b01 = ggml_q1_0_unpack_pair(b_qs[chunk * 4 + 0], b_qs[chunk * 4 + 1]); + const int8x16_t b23 = ggml_q1_0_unpack_pair(b_qs[chunk * 4 + 2], b_qs[chunk * 4 + 3]); + + acc[0] = vmmlaq_s32(acc[0], a01, b01); + acc[1] = vmmlaq_s32(acc[1], a01, b23); + acc[2] = vmmlaq_s32(acc[2], a23, b01); + acc[3] = vmmlaq_s32(acc[3], a23, b23); + } + + const int32x4_t row0 = vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])); + const int32x4_t row1 = vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])); + const int32x4_t row2 = vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])); + const int32x4_t row3 = vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])); + const float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *) a_blk->d)); + + blockf[0] = vfmaq_laneq_f32(blockf[0], vcvtq_f32_s32(row0), a_d, 0); + blockf[1] = vfmaq_laneq_f32(blockf[1], vcvtq_f32_s32(row1), a_d, 1); + blockf[2] = vfmaq_laneq_f32(blockf[2], vcvtq_f32_s32(row2), a_d, 2); + blockf[3] = vfmaq_laneq_f32(blockf[3], vcvtq_f32_s32(row3), a_d, 3); + } + + sumf[0] = vfmaq_f32(sumf[0], blockf[0], b_d); + sumf[1] = vfmaq_f32(sumf[1], blockf[1], b_d); + sumf[2] = vfmaq_f32(sumf[2], blockf[2], b_d); + sumf[3] = vfmaq_f32(sumf[3], blockf[3], b_d); + } + + for (int m = 0; m < 4; ++m) { + vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]); + } + } + } + return; +#endif + + ggml_gemm_q1_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); +} diff --git a/ggml/src/ggml-cpu/arch/loongarch/quants.c b/ggml/src/ggml-cpu/arch/loongarch/quants.c index 74e0c08..9c43da6 100644 --- a/ggml/src/ggml-cpu/arch/loongarch/quants.c +++ b/ggml/src/ggml-cpu/arch/loongarch/quants.c @@ -977,6 +977,35 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi sumf = hsum_float_8(acc); *s = sumf; + +#elif defined(__loongarch_sx) + + __m128 acc = (__m128)__lsx_vldi(0); + + for (; ib < nb; ++ib) { + const float d = GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d); + const __m128i qx_0 = __lsx_vld((const __m128i *)x[ib].qs, 0); + const __m128i qx_1 = __lsx_vld((const __m128i *)x[ib].qs + 1, 0); + const __m128i qy_0 = __lsx_vld((const __m128i *)y[ib].qs, 0); + const __m128i qy_1 = __lsx_vld((const __m128i *)y[ib].qs + 1, 0); + + const __m128i p16_0 = lsx_maddubs_h(qx_0, qy_0); + const __m128i p16_1 = lsx_maddubs_h(qx_1, qy_1); + + // Sum int16 pairs → int32 + const __m128i s_0 = __lsx_vaddwev_w_h(p16_0, p16_1); + const __m128i s_1 = __lsx_vaddwod_w_h(p16_0, p16_1); + + const __m128 q = __lsx_vffint_s_w(__lsx_vadd_w(s_0, s_1)); + acc = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(d), q, acc); + } + + __m128 res = lsx_hadd_s(acc, acc); + res = lsx_hadd_s(res, res); + sumf = ((v4f32)res)[0]; + + *s = sumf; + #else UNUSED(nb); UNUSED(ib); @@ -1443,6 +1472,99 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = hsum_float_8(acc); +#elif defined(__loongarch_sx) + + const __m128i m32s = __lsx_vreplgr2vr_b(32); + + __m128 acc_0 = (__m128)__lsx_vldi(0); + __m128 acc_1 = (__m128)__lsx_vldi(0); + + for (int i = 0; i < nb; ++i) { + + const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); + + const uint8_t * GGML_RESTRICT q4 = x[i].ql; + const uint8_t * GGML_RESTRICT qh = x[i].qh; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + const __m128i scale_i8 = __lsx_vld(x[i].scales, 0); + const __m128i scales_lo = __lsx_vsllwil_h_b(scale_i8, 0); + const __m128i scales_hi = __lsx_vsllwil_h_b(__lsx_vbsrl_v(scale_i8, 8), 0); + + __m128i sumi_0 = __lsx_vldi(0); + __m128i sumi_1 = __lsx_vldi(0); + + for (int j = 0; j < QK_K/128; ++j) { + + const __m128i q4bitsH_0 = __lsx_vld((const __m128i*)qh, 0); qh += 16; + const __m128i q4bitsH_1 = __lsx_vld((const __m128i*)qh, 0); qh += 16; + + const __m128i q4h_0 = __lsx_vslli_b(__lsx_vandi_b(q4bitsH_0, 3), 4); + const __m128i q4h_1 = __lsx_vslli_b(__lsx_vandi_b(q4bitsH_1, 3), 4); + const __m128i q4h_2 = __lsx_vslli_b(__lsx_vandi_b(q4bitsH_0, 3 << 2), 2); + const __m128i q4h_3 = __lsx_vslli_b(__lsx_vandi_b(q4bitsH_1, 3 << 2), 2); + const __m128i q4h_4 = __lsx_vandi_b(q4bitsH_0, 3 << 4); + const __m128i q4h_5 = __lsx_vandi_b(q4bitsH_1, 3 << 4); + const __m128i q4h_6 = __lsx_vsrli_b(__lsx_vandi_b(q4bitsH_0, 3 << 6), 2); + const __m128i q4h_7 = __lsx_vsrli_b(__lsx_vandi_b(q4bitsH_1, 3 << 6), 2); + + const __m128i q4bits1_0 = __lsx_vld((const __m128i*)q4, 0); q4 += 16; + const __m128i q4bits1_1 = __lsx_vld((const __m128i*)q4, 0); q4 += 16; + const __m128i q4bits2_0 = __lsx_vld((const __m128i*)q4, 0); q4 += 16; + const __m128i q4bits2_1 = __lsx_vld((const __m128i*)q4, 0); q4 += 16; + + const __m128i q4_0 = __lsx_vor_v(__lsx_vandi_b(q4bits1_0, 0xf), q4h_0); + const __m128i q4_1 = __lsx_vor_v(__lsx_vandi_b(q4bits1_1, 0xf), q4h_1); + const __m128i q4_2 = __lsx_vor_v(__lsx_vandi_b(q4bits2_0, 0xf), q4h_2); + const __m128i q4_3 = __lsx_vor_v(__lsx_vandi_b(q4bits2_1, 0xf), q4h_3); + const __m128i q4_4 = __lsx_vor_v(__lsx_vsrli_b(q4bits1_0, 4), q4h_4); + const __m128i q4_5 = __lsx_vor_v(__lsx_vsrli_b(q4bits1_1, 4), q4h_5); + const __m128i q4_6 = __lsx_vor_v(__lsx_vsrli_b(q4bits2_0, 4), q4h_6); + const __m128i q4_7 = __lsx_vor_v(__lsx_vsrli_b(q4bits2_1, 4), q4h_7); + + const __m128i q8_0 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + const __m128i q8_1 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + const __m128i q8_2 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + const __m128i q8_3 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + const __m128i q8_4 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + const __m128i q8_5 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + const __m128i q8_6 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + const __m128i q8_7 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + + __m128i p16_0 = lsx_maddubs_h(__lsx_vsub_b(q4_0, m32s), q8_0); + __m128i p16_1 = lsx_maddubs_h(__lsx_vsub_b(q4_1, m32s), q8_1); + __m128i p16_2 = lsx_maddubs_h(__lsx_vsub_b(q4_2, m32s), q8_2); + __m128i p16_3 = lsx_maddubs_h(__lsx_vsub_b(q4_3, m32s), q8_3); + __m128i p16_4 = lsx_maddubs_h(__lsx_vsub_b(q4_4, m32s), q8_4); + __m128i p16_5 = lsx_maddubs_h(__lsx_vsub_b(q4_5, m32s), q8_5); + __m128i p16_6 = lsx_maddubs_h(__lsx_vsub_b(q4_6, m32s), q8_6); + __m128i p16_7 = lsx_maddubs_h(__lsx_vsub_b(q4_7, m32s), q8_7); + + const __m128i sc_vec = j == 0 ? scales_lo : scales_hi; + + p16_0 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 0), p16_0); + p16_1 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 1), p16_1); + p16_2 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 2), p16_2); + p16_3 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 3), p16_3); + p16_4 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 4), p16_4); + p16_5 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 5), p16_5); + p16_6 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 6), p16_6); + p16_7 = lsx_madd_h(__lsx_vreplvei_h(sc_vec, 7), p16_7); + + sumi_0 = __lsx_vadd_w(sumi_0, __lsx_vadd_w(p16_0, p16_2)); + sumi_1 = __lsx_vadd_w(sumi_1, __lsx_vadd_w(p16_1, p16_3)); + sumi_0 = __lsx_vadd_w(sumi_0, __lsx_vadd_w(p16_4, p16_6)); + sumi_1 = __lsx_vadd_w(sumi_1, __lsx_vadd_w(p16_5, p16_7)); + } + + __m128 p_0 = __lsx_vfmul_s(__lsx_vreplfr2vr_s(d), __lsx_vffint_s_w(sumi_0)); + __m128 p_1 = __lsx_vfmul_s(__lsx_vreplfr2vr_s(d), __lsx_vffint_s_w(sumi_1)); + acc_0 = __lsx_vfadd_s(p_0, acc_0); + acc_1 = __lsx_vfadd_s(p_1, acc_1); + } + + *s = hsum_float_4x4(acc_0, acc_1, (__m128)__lsx_vldi(0), (__m128)__lsx_vldi(0)); + #else UNUSED(x); UNUSED(y); @@ -2149,6 +2271,35 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v *s = hsum_float_8(accum); +#elif defined(__loongarch_sx) + + const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0); + + __m128 accum = (__m128)__lsx_vldi(0); + for (int ibl = 0; ibl < nb; ++ibl) { + const uint8_t * qs = x[ibl].qs; + const int8_t * q8 = y[ibl].qs; + uint16_t sh = x[ibl].scales_h; + __m128i sumi = __lsx_vldi(0); + for (int ib = 0; ib < QK_K/32; ++ib) { + const __m128i q4bits = __lsx_vld((const __m128i*)qs, 0); qs += 16; + const __m128i q8b_0 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + const __m128i q8b_1 = __lsx_vld((const __m128i*)q8, 0); q8 += 16; + const __m128i q4b_0 = __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits, 0xf)); + const __m128i q4b_1 = __lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits, 4)); + const __m128i p16_0 = lsx_maddubs_h(q4b_0, q8b_0); + const __m128i p16_1 = lsx_maddubs_h(q4b_1, q8b_1); + const int16_t ls = (((x[ibl].scales_l[ib/2] >> ((ib & 1) * 4)) & 0xf) | ((sh & 0x3) << 4)) - 32; + sh >>= 2; + sumi = __lsx_vadd_w(lsx_madd_h(p16_0, __lsx_vreplgr2vr_h(ls)), sumi); + sumi = __lsx_vadd_w(lsx_madd_h(p16_1, __lsx_vreplgr2vr_h(ls)), sumi); + } + const float ds = GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d; + accum = __lsx_vfadd_s(__lsx_vfmul_s(__lsx_vreplfr2vr_s(ds), __lsx_vffint_s_w(sumi)), accum); + } + + *s = ((v4f32)lsx_hadd_s(lsx_hadd_s(accum, accum), lsx_hadd_s(accum, accum)))[0]; + #else UNUSED(x); UNUSED(y); diff --git a/ggml/src/ggml-cpu/arch/riscv/quants.c b/ggml/src/ggml-cpu/arch/riscv/quants.c index d3278d6..47e9180 100644 --- a/ggml/src/ggml-cpu/arch/riscv/quants.c +++ b/ggml/src/ggml-cpu/arch/riscv/quants.c @@ -123,7 +123,7 @@ void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in assert(k % QK_K == 0); size_t nb = k / QK_K; -#if defined __riscv_v_intrinsic +#if defined __riscv_v block_q8_K * y_blocks = (block_q8_K *)y; const size_t vlmax_f32m8 = __riscv_vsetvlmax_e32m8(); @@ -480,7 +480,106 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } -void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined(__riscv_v) +static NOINLINE void ggml_vec_dot_q1_0_q8_0_vl256(const int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy) { + const int qk = QK1_0; + const int nb = n / qk; + assert(n % qk == 0); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + //LMUL = 1, VLMAX = 32 + const size_t vl32 = __riscv_vsetvl_e8m1(32); + assert(vl32 == 32); + + const vint16m1_t zero = __riscv_vmv_v_x_i16m1(0, 1); + + float sumf = 0; + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); + + float acc = 0; + + for (int k = 0; k < 4; ++k) { + const block_q8_0 * GGML_RESTRICT yb = &y[ib * 4 + k]; + const vbool8_t is_not_zero = __riscv_vlm_v_b8(x[ib].qs + 4 * k, vl32); + + const vint8m1_t qy = __riscv_vle8_v_i8m1(yb->qs, vl32); + const vint8m1_t neg_qy = __riscv_vneg_v_i8m1(qy, vl32); + const vint8m1_t sy = __riscv_vmerge_vvm_i8m1(neg_qy, qy, is_not_zero, vl32); + + const vint16m1_t red = __riscv_vwredsum_vs_i8m1_i16m1(sy, zero, vl32); + acc += GGML_CPU_FP16_TO_FP32(yb->d) * (float)__riscv_vmv_x_s_i16m1_i16(red); + } + + sumf += d0 * acc; + } + + *s = sumf; +} + +static NOINLINE void ggml_vec_dot_q1_0_q8_0_vl128(const int n, float * GGML_RESTRICT s, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy) { + const int qk = QK1_0; + const int nb = n / qk; + assert(n % qk == 0); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + //LMUL = 2, VLMAX = 32 + const size_t vl32 = __riscv_vsetvl_e8m2(32); + assert(vl32 == 32); + + const vint16m1_t zero = __riscv_vmv_v_x_i16m1(0, 1); + + float sumf = 0; + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); + + float acc = 0; + + for (int k = 0; k < 4; ++k) { + const block_q8_0 * GGML_RESTRICT yb = &y[ib * 4 + k]; + const vbool4_t is_not_zero = __riscv_vlm_v_b4(x[ib].qs + 4 * k, vl32); + + const vint8m2_t qy = __riscv_vle8_v_i8m2(yb->qs, vl32); + const vint8m2_t neg_qy =__riscv_vneg_v_i8m2(qy, vl32); + const vint8m2_t sy = __riscv_vmerge_vvm_i8m2(neg_qy, qy, is_not_zero, vl32); + + const vint16m1_t red = __riscv_vwredsum_vs_i8m2_i16m1(sy, zero, vl32); + acc += GGML_CPU_FP16_TO_FP32(yb->d) * (float)__riscv_vmv_x_s_i16m1_i16(red); + } + + sumf += d0 * acc; + } + + *s = sumf; +} +#endif + +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined(__riscv_v) + assert(nrc == 1); + + const size_t vlen_bits = __riscv_vlenb() * 8; + + if (vlen_bits >= 256) { + ggml_vec_dot_q1_0_q8_0_vl256(n, s, vx, vy); + } else if (vlen_bits >= 128) { + ggml_vec_dot_q1_0_q8_0_vl128(n, s, vx, vy); + } else { + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + } +#else + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + +#if defined __riscv_xtheadvector +void ggml_vec_dot_q2_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -492,8 +591,6 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi const int nb = n / QK_K; -#if defined __riscv_xtheadvector - float sumf = 0; uint8_t atmp[16]; @@ -588,246 +685,281 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi } *s = sumf; +} +#endif -#elif defined __riscv_v +#if defined __riscv_v +void ggml_vec_dot_q2_K_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; float sumf = 0; uint8_t atmp[16]; - const int vector_length = __riscv_vlenb() * 8; uint8_t temp_01[32] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - switch (vector_length) { - case 256: - for (int i = 0; i < nb; ++i) { - const uint8_t * q2 = x[i].qs; - const int8_t * q8 = y[i].qs; - const uint8_t * sc = x[i].scales; + for (int i = 0; i < nb; ++i) { + const uint8_t * q2 = x[i].qs; + const int8_t * q8 = y[i].qs; + const uint8_t * sc = x[i].scales; + const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); + const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); + uint8_t *patmp = atmp; + int vsums; + int tmp, t1, t2, t3, t4, t5, t6, t7; + __asm__ __volatile__( + "vsetivli zero, 16, e8, m1\n\t" + "vmv.v.x v8, zero\n\t" + "lb zero, 15(%[sc])\n\t" + "vle8.v v1, (%[sc])\n\t" + "vle8.v v2, (%[bsums])\n\t" + "addi %[tmp], %[bsums], 16\n\t" + "vand.vi v0, v1, 0xF\n\t" + "vsrl.vi v1, v1, 4\n\t" + "vle8.v v3, (%[tmp])\n\t" + "vse8.v v0, (%[scale])\n\t" + "vsetivli zero, 16, e16, m2\n\t" + "vzext.vf2 v0, v1\n\t" + "vwmul.vv v4, v0, v2\n\t" + "vsetivli zero, 16, e32, m4\n\t" + "vredsum.vs v8, v4, v8\n\t" + "vmv.x.s %[vsums], v8" + : [tmp] "=&r" (tmp), [vsums] "=&r" (vsums) + : [sc] "r" (sc), [scale] "r" (atmp), [bsums] "r" (y[i].bsums) + : "memory" + , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" + , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" + ); + sumf += dmin * vsums; + int isum = 0; + + for (int j = 0; j < QK_K/128; ++j) { + __asm__ __volatile__( + "lb zero, 31(%[q2])\n\t" + "addi %[tmp], %[q2], 16\n\t" + "addi %[t1], %[q8], 16\n\t" + "vsetivli zero, 16, e8, m1\n\t" + "vle8.v v0, (%[q2])\n\t" + "vle8.v v1, (%[tmp])\n\t" + "vsrl.vi v2, v0, 2\n\t" + "vsrl.vi v3, v1, 2\n\t" + "vsrl.vi v4, v0, 4\n\t" + "addi %[tmp], %[q8], 32\n\t" + "vle8.v v8, (%[q8])\n\t" + "vle8.v v9, (%[t1])\n\t" + "addi %[t1], %[t1], 32\n\t" + "vsrl.vi v5, v1, 4\n\t" + "vsrl.vi v6, v0, 6\n\t" + "vsrl.vi v7, v1, 6\n\t" + "vle8.v v10, (%[tmp])\n\t" + "vle8.v v11, (%[t1])\n\t" + "addi %[tmp], %[tmp], 32\n\t" + "addi %[t1], %[t1], 32\n\t" + "vand.vi v0, v0, 0x3\n\t" + "vand.vi v1, v1, 0x3\n\t" + "vand.vi v2, v2, 0x3\n\t" + "vle8.v v12, (%[tmp])\n\t" + "vle8.v v13, (%[t1])\n\t" + "addi %[tmp], %[tmp], 32\n\t" + "addi %[t1], %[t1], 32\n\t" + "vand.vi v3, v3, 0x3\n\t" + "vand.vi v4, v4, 0x3\n\t" + "vand.vi v5, v5, 0x3\n\t" + "vle8.v v14, (%[tmp])\n\t" + "vle8.v v15, (%[t1])\n\t" + "vwmul.vv v16, v0, v8\n\t" + "vwmul.vv v18, v1, v9\n\t" + "vwmul.vv v20, v2, v10\n\t" + "vwmul.vv v22, v3, v11\n\t" + "vwmul.vv v24, v4, v12\n\t" + "vwmul.vv v26, v5, v13\n\t" + "vwmul.vv v28, v6, v14\n\t" + "vwmul.vv v30, v7, v15\n\t" + "vsetivli zero, 8, e16, m1\n\t" + "vmv.v.x v0, zero\n\t" + "lbu %[tmp], 0(%[scale])\n\t" + "vwredsum.vs v8, v16, v0\n\t" + "vwredsum.vs v9, v18, v0\n\t" + "lbu %[t1], 1(%[scale])\n\t" + "vwredsum.vs v10, v20, v0\n\t" + "vwredsum.vs v11, v22, v0\n\t" + "lbu %[t2], 2(%[scale])\n\t" + "vwredsum.vs v12, v24, v0\n\t" + "vwredsum.vs v13, v26, v0\n\t" + "lbu %[t3], 3(%[scale])\n\t" + "vwredsum.vs v14, v28, v0\n\t" + "vwredsum.vs v15, v30, v0\n\t" + "lbu %[t4], 4(%[scale])\n\t" + "vwredsum.vs v8, v17, v8\n\t" + "vwredsum.vs v9, v19, v9\n\t" + "lbu %[t5], 5(%[scale])\n\t" + "vwredsum.vs v10, v21, v10\n\t" + "vwredsum.vs v11, v23, v11\n\t" + "lbu %[t6], 6(%[scale])\n\t" + "vwredsum.vs v12, v25, v12\n\t" + "vwredsum.vs v13, v27, v13\n\t" + "lbu %[t7], 7(%[scale])\n\t" + "vwredsum.vs v14, v29, v14\n\t" + "vwredsum.vs v15, v31, v15\n\t" + "vsetivli zero, 4, e32, m1\n\t" + "vmul.vx v0, v8, %[tmp]\n\t" + "vmul.vx v1, v9, %[t1]\n\t" + "vmacc.vx v0, %[t2], v10\n\t" + "vmacc.vx v1, %[t3], v11\n\t" + "vmacc.vx v0, %[t4], v12\n\t" + "vmacc.vx v1, %[t5], v13\n\t" + "vmacc.vx v0, %[t6], v14\n\t" + "vmacc.vx v1, %[t7], v15\n\t" + "vmv.x.s %[tmp], v0\n\t" + "vmv.x.s %[t1], v1\n\t" + "add %[isum], %[isum], %[tmp]\n\t" + "add %[isum], %[isum], %[t1]" + : [tmp] "=&r" (tmp), [t1] "=&r" (t1), [t2] "=&r" (t2), [t3] "=&r" (t3) + , [t4] "=&r" (t4), [t5] "=&r" (t5), [t6] "=&r" (t6), [t7] "=&r" (t7) + , [isum] "+&r" (isum) + : [q2] "r" (q2), [scale] "r" (patmp), [q8] "r" (q8) + : "memory" + , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" + , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" + ); + q2 += 32; q8 += 128; patmp += 8; + } + + sumf += dall * isum; + } - const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); - const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); + *s = sumf; +} - size_t vl = 16; +void ggml_vec_dot_q2_K_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); - vuint8m1_t scales = __riscv_vle8_v_u8m1(sc, vl); - vuint8m1_t aux = __riscv_vand_vx_u8m1(scales, 0x0F, vl); + const block_q2_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; - vint16m1_t q8sums = __riscv_vle16_v_i16m1(y[i].bsums, vl); + const int nb = n / QK_K; - vuint8mf2_t scales_2 = __riscv_vle8_v_u8mf2(sc, vl); - vuint8mf2_t mins8 = __riscv_vsrl_vx_u8mf2(scales_2, 0x4, vl); - vint16m1_t mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl)); - vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, mins, vl); - vint32m1_t vsums = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl); + float sumf = 0; + uint8_t atmp[16]; - sumf += dmin * __riscv_vmv_x_s_i32m1_i32(vsums); + uint8_t temp_01[32] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - vl = 32; + for (int i = 0; i < nb; ++i) { + const uint8_t * q2 = x[i].qs; + const int8_t * q8 = y[i].qs; + const uint8_t * sc = x[i].scales; - vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); - vuint8m1_t v_b = __riscv_vle8_v_u8m1(temp_01, vl); + const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); + const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); - uint8_t is = 0; - int isum = 0; + size_t vl = 16; - for (int j = 0; j < QK_K / 128; ++j) { - // load Q2 - vuint8m1_t q2_x = __riscv_vle8_v_u8m1(q2, vl); + vuint8m1_t scales = __riscv_vle8_v_u8m1(sc, vl); + vuint8m1_t aux = __riscv_vand_vx_u8m1(scales, 0x0F, vl); - vuint8m1_t q2_0 = __riscv_vand_vx_u8m1(q2_x, 0x03, vl); - vuint8m1_t q2_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x2, vl), 0x03, vl); - vuint8m1_t q2_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x4, vl), 0x03, vl); - vuint8m1_t q2_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x6, vl), 0x03, vl); + vint16m1_t q8sums = __riscv_vle16_v_i16m1(y[i].bsums, vl); - // duplicate scale elements for product - vuint8m1_t sc0 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 0 + is, vl), vl); - vuint8m1_t sc1 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 2 + is, vl), vl); - vuint8m1_t sc2 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 4 + is, vl), vl); - vuint8m1_t sc3 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 6 + is, vl), vl); + vuint8mf2_t scales_2 = __riscv_vle8_v_u8mf2(sc, vl); + vuint8mf2_t mins8 = __riscv_vsrl_vx_u8mf2(scales_2, 0x4, vl); + vint16m1_t mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl)); + vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, mins, vl); + vint32m1_t vsums = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl); - vint16m2_t p0 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_0, sc0, vl)); - vint16m2_t p1 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_1, sc1, vl)); - vint16m2_t p2 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_2, sc2, vl)); - vint16m2_t p3 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_3, sc3, vl)); + sumf += dmin * __riscv_vmv_x_s_i32m1_i32(vsums); - // load Q8 - vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl); - vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8 + 32, vl); - vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8 + 64, vl); - vint8m1_t q8_3 = __riscv_vle8_v_i8m1(q8 + 96, vl); + vl = 32; - vint32m4_t s0 = __riscv_vwmul_vv_i32m4(p0, __riscv_vwcvt_x_x_v_i16m2(q8_0, vl), vl); - vint32m4_t s1 = __riscv_vwmul_vv_i32m4(p1, __riscv_vwcvt_x_x_v_i16m2(q8_1, vl), vl); - vint32m4_t s2 = __riscv_vwmul_vv_i32m4(p2, __riscv_vwcvt_x_x_v_i16m2(q8_2, vl), vl); - vint32m4_t s3 = __riscv_vwmul_vv_i32m4(p3, __riscv_vwcvt_x_x_v_i16m2(q8_3, vl), vl); + vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); + vuint8m1_t v_b = __riscv_vle8_v_u8m1(temp_01, vl); - vint32m1_t isum0 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s0, s1, vl), vzero, vl); - vint32m1_t isum1 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s2, s3, vl), isum0, vl); + uint8_t is = 0; + int isum = 0; - isum += __riscv_vmv_x_s_i32m1_i32(isum1); + for (int j = 0; j < QK_K / 128; ++j) { + // load Q2 + vuint8m1_t q2_x = __riscv_vle8_v_u8m1(q2, vl); - q2 += 32; - q8 += 128; - is = 8; - } + vuint8m1_t q2_0 = __riscv_vand_vx_u8m1(q2_x, 0x03, vl); + vuint8m1_t q2_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x2, vl), 0x03, vl); + vuint8m1_t q2_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x4, vl), 0x03, vl); + vuint8m1_t q2_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x6, vl), 0x03, vl); - sumf += dall * isum; - } - break; - case 128: - for (int i = 0; i < nb; ++i) { - const uint8_t * q2 = x[i].qs; - const int8_t * q8 = y[i].qs; - const uint8_t * sc = x[i].scales; - const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); - const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); - uint8_t *patmp = atmp; - int vsums; - int tmp, t1, t2, t3, t4, t5, t6, t7; - __asm__ __volatile__( - "vsetivli zero, 16, e8, m1\n\t" - "vmv.v.x v8, zero\n\t" - "lb zero, 15(%[sc])\n\t" - "vle8.v v1, (%[sc])\n\t" - "vle8.v v2, (%[bsums])\n\t" - "addi %[tmp], %[bsums], 16\n\t" - "vand.vi v0, v1, 0xF\n\t" - "vsrl.vi v1, v1, 4\n\t" - "vle8.v v3, (%[tmp])\n\t" - "vse8.v v0, (%[scale])\n\t" - "vsetivli zero, 16, e16, m2\n\t" - "vzext.vf2 v0, v1\n\t" - "vwmul.vv v4, v0, v2\n\t" - "vsetivli zero, 16, e32, m4\n\t" - "vredsum.vs v8, v4, v8\n\t" - "vmv.x.s %[vsums], v8" - : [tmp] "=&r" (tmp), [vsums] "=&r" (vsums) - : [sc] "r" (sc), [scale] "r" (atmp), [bsums] "r" (y[i].bsums) - : "memory" - , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" - , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" - , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" - , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" - ); - sumf += dmin * vsums; - int isum = 0; - - for (int j = 0; j < QK_K/128; ++j) { - __asm__ __volatile__( - "lb zero, 31(%[q2])\n\t" - "addi %[tmp], %[q2], 16\n\t" - "addi %[t1], %[q8], 16\n\t" - "vsetivli zero, 16, e8, m1\n\t" - "vle8.v v0, (%[q2])\n\t" - "vle8.v v1, (%[tmp])\n\t" - "vsrl.vi v2, v0, 2\n\t" - "vsrl.vi v3, v1, 2\n\t" - "vsrl.vi v4, v0, 4\n\t" - "addi %[tmp], %[q8], 32\n\t" - "vle8.v v8, (%[q8])\n\t" - "vle8.v v9, (%[t1])\n\t" - "addi %[t1], %[t1], 32\n\t" - "vsrl.vi v5, v1, 4\n\t" - "vsrl.vi v6, v0, 6\n\t" - "vsrl.vi v7, v1, 6\n\t" - "vle8.v v10, (%[tmp])\n\t" - "vle8.v v11, (%[t1])\n\t" - "addi %[tmp], %[tmp], 32\n\t" - "addi %[t1], %[t1], 32\n\t" - "vand.vi v0, v0, 0x3\n\t" - "vand.vi v1, v1, 0x3\n\t" - "vand.vi v2, v2, 0x3\n\t" - "vle8.v v12, (%[tmp])\n\t" - "vle8.v v13, (%[t1])\n\t" - "addi %[tmp], %[tmp], 32\n\t" - "addi %[t1], %[t1], 32\n\t" - "vand.vi v3, v3, 0x3\n\t" - "vand.vi v4, v4, 0x3\n\t" - "vand.vi v5, v5, 0x3\n\t" - "vle8.v v14, (%[tmp])\n\t" - "vle8.v v15, (%[t1])\n\t" - "vwmul.vv v16, v0, v8\n\t" - "vwmul.vv v18, v1, v9\n\t" - "vwmul.vv v20, v2, v10\n\t" - "vwmul.vv v22, v3, v11\n\t" - "vwmul.vv v24, v4, v12\n\t" - "vwmul.vv v26, v5, v13\n\t" - "vwmul.vv v28, v6, v14\n\t" - "vwmul.vv v30, v7, v15\n\t" - "vsetivli zero, 8, e16, m1\n\t" - "vmv.v.x v0, zero\n\t" - "lbu %[tmp], 0(%[scale])\n\t" - "vwredsum.vs v8, v16, v0\n\t" - "vwredsum.vs v9, v18, v0\n\t" - "lbu %[t1], 1(%[scale])\n\t" - "vwredsum.vs v10, v20, v0\n\t" - "vwredsum.vs v11, v22, v0\n\t" - "lbu %[t2], 2(%[scale])\n\t" - "vwredsum.vs v12, v24, v0\n\t" - "vwredsum.vs v13, v26, v0\n\t" - "lbu %[t3], 3(%[scale])\n\t" - "vwredsum.vs v14, v28, v0\n\t" - "vwredsum.vs v15, v30, v0\n\t" - "lbu %[t4], 4(%[scale])\n\t" - "vwredsum.vs v8, v17, v8\n\t" - "vwredsum.vs v9, v19, v9\n\t" - "lbu %[t5], 5(%[scale])\n\t" - "vwredsum.vs v10, v21, v10\n\t" - "vwredsum.vs v11, v23, v11\n\t" - "lbu %[t6], 6(%[scale])\n\t" - "vwredsum.vs v12, v25, v12\n\t" - "vwredsum.vs v13, v27, v13\n\t" - "lbu %[t7], 7(%[scale])\n\t" - "vwredsum.vs v14, v29, v14\n\t" - "vwredsum.vs v15, v31, v15\n\t" - "vsetivli zero, 4, e32, m1\n\t" - "vmul.vx v0, v8, %[tmp]\n\t" - "vmul.vx v1, v9, %[t1]\n\t" - "vmacc.vx v0, %[t2], v10\n\t" - "vmacc.vx v1, %[t3], v11\n\t" - "vmacc.vx v0, %[t4], v12\n\t" - "vmacc.vx v1, %[t5], v13\n\t" - "vmacc.vx v0, %[t6], v14\n\t" - "vmacc.vx v1, %[t7], v15\n\t" - "vmv.x.s %[tmp], v0\n\t" - "vmv.x.s %[t1], v1\n\t" - "add %[isum], %[isum], %[tmp]\n\t" - "add %[isum], %[isum], %[t1]" - : [tmp] "=&r" (tmp), [t1] "=&r" (t1), [t2] "=&r" (t2), [t3] "=&r" (t3) - , [t4] "=&r" (t4), [t5] "=&r" (t5), [t6] "=&r" (t6), [t7] "=&r" (t7) - , [isum] "+&r" (isum) - : [q2] "r" (q2), [scale] "r" (patmp), [q8] "r" (q8) - : "memory" - , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" - , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" - , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" - , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" - ); - q2 += 32; q8 += 128; patmp += 8; - } + // duplicate scale elements for product + vuint8m1_t sc0 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 0 + is, vl), vl); + vuint8m1_t sc1 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 2 + is, vl), vl); + vuint8m1_t sc2 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 4 + is, vl), vl); + vuint8m1_t sc3 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 6 + is, vl), vl); + + vint16m2_t p0 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_0, sc0, vl)); + vint16m2_t p1 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_1, sc1, vl)); + vint16m2_t p2 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_2, sc2, vl)); + vint16m2_t p3 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_3, sc3, vl)); + + // load Q8 + vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl); + vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8 + 32, vl); + vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8 + 64, vl); + vint8m1_t q8_3 = __riscv_vle8_v_i8m1(q8 + 96, vl); + + vint32m4_t s0 = __riscv_vwmul_vv_i32m4(p0, __riscv_vwcvt_x_x_v_i16m2(q8_0, vl), vl); + vint32m4_t s1 = __riscv_vwmul_vv_i32m4(p1, __riscv_vwcvt_x_x_v_i16m2(q8_1, vl), vl); + vint32m4_t s2 = __riscv_vwmul_vv_i32m4(p2, __riscv_vwcvt_x_x_v_i16m2(q8_2, vl), vl); + vint32m4_t s3 = __riscv_vwmul_vv_i32m4(p3, __riscv_vwcvt_x_x_v_i16m2(q8_3, vl), vl); - sumf += dall * isum; + vint32m1_t isum0 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s0, s1, vl), vzero, vl); + vint32m1_t isum1 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s2, s3, vl), isum0, vl); + + isum += __riscv_vmv_x_s_i32m1_i32(isum1); + + q2 += 32; + q8 += 128; + is = 8; } - break; - default: - assert(false && "Unsupported vector length"); - break; + + sumf += dall * isum; } *s = sumf; +} +#endif +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined __riscv_xtheadvector + ggml_vec_dot_q2_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc); +#elif defined __riscv_v + switch (__riscv_vlenb() * 8) { + case 128: + ggml_vec_dot_q2_K_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + break; + default: + ggml_vec_dot_q2_K_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + break; + } #else - - UNUSED(x); - UNUSED(y); - UNUSED(nb); - ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); #endif } -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined __riscv_xtheadvector +void ggml_vec_dot_q3_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -843,8 +975,6 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi const int nb = n / QK_K; -#if defined __riscv_xtheadvector - uint32_t utmp[4]; float sumf = 0; @@ -970,257 +1100,538 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi } *s = sumf; +} +#endif -#elif defined __riscv_v +#if defined __riscv_v +void ggml_vec_dot_q3_K_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; + + const block_q3_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; uint32_t utmp[4]; float sumf = 0; uint32_t aux[3]; - const int vector_length = __riscv_vlenb() * 8; - switch (vector_length) { - case 256: - for (int i = 0; i < nb; ++i) { + for (int i = 0; i < nb; ++i) { + const uint8_t * restrict q3 = x[i].qs; + const uint8_t * restrict qh = x[i].hmask; + const int8_t * restrict q8 = y[i].qs; - const uint8_t * GGML_RESTRICT q3 = x[i].qs; - const uint8_t * GGML_RESTRICT qh = x[i].hmask; - const int8_t * GGML_RESTRICT q8 = y[i].qs; + int8_t * scale = (int8_t *)utmp; + int tmp, t1, t2, t3, t4, t5, t6, t7; + __asm__ __volatile__( + "vsetivli zero, 12, e8, m1\n\t" + "vle8.v v0, (%[s6b])\n\t" + "vmv1r.v v2, v0\n\t" + "vsetivli zero, 2, e64, m1\n\t" + "vmv.v.x v9, %[sh]\n\t"\ + "vslidedown.vi v1, v0, 1\n\t" + "vslide1up.vx v8, v9, zero\n\t" // {0, 0, 4, 4} + "vslideup.vi v0, v2, 1\n\t" // {aux[0], aux[1], aux[0], aux[1]} + "vsetivli zero, 4, e32, m1\n\t" + "vid.v v9\n\t" + "vmv.x.s %[tmp], v1\n\t" + "vsll.vi v9, v9, 1\n\t" // {0, 2, 4, 6} + "vmv.v.x v1, %[tmp]\n\t" // {aux[2], aux[2], aux[2], aux[2]} + "vsrl.vv v4, v1, v9\n\t" + "vsrl.vv v2, v0, v8\n\t" + "vand.vx v5, v4, %[kmask1]\n\t" + "vand.vx v3, v2, %[kmask2]\n\t" + "vsll.vi v6, v5, 4\n\t" + "vor.vv v7, v6, v3\n\t" + "vsetivli zero, 16, e8, m1\n\t" + "vsub.vx v0, v7, %[c]\n\t" + "vse8.v v0, (%[scale])" + : [tmp] "=&r" (tmp) + : [sh] "r" (0x0000000400000004), [s6b] "r" (x[i].scales), [c] "r" (32) + , [scale] "r" (scale), [kmask1] "r" (kmask1), [kmask2] "r" (kmask2) + : "memory" + , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" + , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" + ); - memcpy(aux, x[i].scales, 12); - utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4); - utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4); - utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4); - utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4); + uint8_t m = 1; + int isum = 0; + for (int j = 0; j < QK_K; j += 128) { + __asm__ __volatile__( + "lb zero, 31(%[q3])\n\t" + "vsetvli zero, %[vl32], e8, m2, ta, mu\n\t" + "vle8.v v8, (%[q3])\n\t" + "vsrl.vi v10, v8, 2\n\t" + "vsrl.vi v12, v8, 4\n\t" + "vsrl.vi v14, v8, 6\n\t" + "lb zero, 64(%[q8])\n\t" + "vand.vi v8, v8, 3\n\t" + "vand.vi v10, v10, 3\n\t" + "vand.vi v12, v12, 3\n\t" + "vle8.v v2, (%[qh])\n\t" + "lb zero, 127(%[q8])\n\t" + "vand.vx v4, v2, %[m]\n\t" + "slli %[m], %[m], 1\n\t" + "vmseq.vx v0, v4, zero\n\t" + "vadd.vi v8, v8, -4, v0.t\n\t" + "lb zero, 0(%[q8])\n\t" + "vand.vx v4, v2, %[m]\n\t" + "slli %[m], %[m], 1\n\t" + "vmseq.vx v0, v4, zero\n\t" + "vadd.vi v10, v10, -4, v0.t\n\t" + "vand.vx v4, v2, %[m]\n\t" + "slli %[m], %[m], 1\n\t" + "vmseq.vx v0, v4, zero\n\t" + "vadd.vi v12, v12, -4, v0.t\n\t" + "vand.vx v4, v2, %[m]\n\t" + "slli %[m], %[m], 1\n\t" + "vmseq.vx v0, v4, zero\n\t" + "vadd.vi v14, v14, -4, v0.t\n\t" + "vsetvli zero, %[vl128], e8, m8\n\t" + "vle8.v v0, (%[q8])\n\t" + "lb %[tmp], 0(%[scale])\n\t" + "lb %[t1], 1(%[scale])\n\t" + "lb %[t2], 2(%[scale])\n\t" + "lb %[t3], 3(%[scale])\n\t" + "vsetvli zero, %[vl64], e8, m4\n\t" + "vwmul.vv v16, v0, v8\n\t" + "vwmul.vv v24, v4, v12\n\t" + "vsetivli zero, 16, e16, m2\n\t" + "vmv.v.x v0, zero\n\t" + "vwredsum.vs v8, v16, v0\n\t" + "lb %[t4], 4(%[scale])\n\t" + "lb %[t5], 5(%[scale])\n\t" + "vwredsum.vs v9, v18, v0\n\t" + "vwredsum.vs v10, v20, v0\n\t" + "vwredsum.vs v11, v22, v0\n\t" + "vwredsum.vs v12, v24, v0\n\t" + "lb %[t6], 6(%[scale])\n\t" + "lb %[t7], 7(%[scale])\n\t" + "vwredsum.vs v13, v26, v0\n\t" + "vwredsum.vs v14, v28, v0\n\t" + "vwredsum.vs v15, v30, v0\n\t" + "vsetivli zero, 4, e32, m1\n\t" + "vmul.vx v0, v8, %[tmp]\n\t" + "vmul.vx v1, v9, %[t1]\n\t" + "vmacc.vx v0, %[t2], v10\n\t" + "vmacc.vx v1, %[t3], v11\n\t" + "vmacc.vx v0, %[t4], v12\n\t" + "vmacc.vx v1, %[t5], v13\n\t" + "vmacc.vx v0, %[t6], v14\n\t" + "vmacc.vx v1, %[t7], v15\n\t" + "vmv.x.s %[tmp], v0\n\t" + "vmv.x.s %[t1], v1\n\t" + "add %[isum], %[isum], %[tmp]\n\t" + "add %[isum], %[isum], %[t1]" + : [tmp] "=&r" (tmp), [t1] "=&r" (t1), [t2] "=&r" (t2), [t3] "=&r" (t3) + , [t4] "=&r" (t4), [t5] "=&r" (t5), [t6] "=&r" (t6), [t7] "=&r" (t7) + , [m] "+&r" (m), [isum] "+&r" (isum) + : [vl128] "r" (128), [vl64] "r" (64), [vl32] "r" (32) + , [q3] "r" (q3), [qh] "r" (qh), [scale] "r" (scale), [q8] "r" (q8) + : "memory" + , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" + , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" + ); + q3 += 32; q8 += 128; scale += 8; + } - int8_t * scale = (int8_t *)utmp; - for (int j = 0; j < 16; ++j) scale[j] -= 32; + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + sumf += d * isum; + } + *s = sumf; +} - size_t vl = 32; - uint8_t m = 1; +void ggml_vec_dot_q3_K_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); - vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); - vuint8m1_t vqh = __riscv_vle8_v_u8m1(qh, vl); + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; - int sum_t = 0; + const block_q3_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; - for (int j = 0; j < QK_K; j += 128) { + const int nb = n / QK_K; + uint32_t utmp[4]; + float sumf = 0; + uint32_t aux[3]; - vl = 32; + for (int i = 0; i < nb; ++i) { + const uint8_t * GGML_RESTRICT q3 = x[i].qs; + const uint8_t * GGML_RESTRICT qh = x[i].hmask; + const int8_t * GGML_RESTRICT q8 = y[i].qs; - // load Q3 - vuint8m1_t q3_x = __riscv_vle8_v_u8m1(q3, vl); + memcpy(aux, x[i].scales, 12); + utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4); + utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4); + utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4); + utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4); - vint8m1_t q3_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q3_x, 0x03, vl)); - vint8m1_t q3_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x2, vl), 0x03 , vl)); - vint8m1_t q3_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x4, vl), 0x03 , vl)); - vint8m1_t q3_3 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x6, vl), 0x03 , vl)); + int8_t * scale = (int8_t *)utmp; + for (int j = 0; j < 16; ++j) scale[j] -= 32; - // compute mask for subtraction - vuint8m1_t qh_m0 = __riscv_vand_vx_u8m1(vqh, m, vl); - vbool8_t vmask_0 = __riscv_vmseq_vx_u8m1_b8(qh_m0, 0, vl); - vint8m1_t q3_m0 = __riscv_vsub_vx_i8m1_mu(vmask_0, q3_0, q3_0, 0x4, vl); - m <<= 1; - vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl); - vbool8_t vmask_1 = __riscv_vmseq_vx_u8m1_b8(qh_m1, 0, vl); - vint8m1_t q3_m1 = __riscv_vsub_vx_i8m1_mu(vmask_1, q3_1, q3_1, 0x4, vl); - m <<= 1; + size_t vl = 32; + uint8_t m = 1; - vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl); - vbool8_t vmask_2 = __riscv_vmseq_vx_u8m1_b8(qh_m2, 0, vl); - vint8m1_t q3_m2 = __riscv_vsub_vx_i8m1_mu(vmask_2, q3_2, q3_2, 0x4, vl); - m <<= 1; + vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); + vuint8m1_t vqh = __riscv_vle8_v_u8m1(qh, vl); - vuint8m1_t qh_m3 = __riscv_vand_vx_u8m1(vqh, m, vl); - vbool8_t vmask_3 = __riscv_vmseq_vx_u8m1_b8(qh_m3, 0, vl); - vint8m1_t q3_m3 = __riscv_vsub_vx_i8m1_mu(vmask_3, q3_3, q3_3, 0x4, vl); - m <<= 1; + int sum_t = 0; - // load Q8 and take product with Q3 - vint16m2_t a0 = __riscv_vwmul_vv_i16m2(q3_m0, __riscv_vle8_v_i8m1(q8, vl), vl); - vint16m2_t a1 = __riscv_vwmul_vv_i16m2(q3_m1, __riscv_vle8_v_i8m1(q8+32, vl), vl); - vint16m2_t a2 = __riscv_vwmul_vv_i16m2(q3_m2, __riscv_vle8_v_i8m1(q8+64, vl), vl); - vint16m2_t a3 = __riscv_vwmul_vv_i16m2(q3_m3, __riscv_vle8_v_i8m1(q8+96, vl), vl); + for (int j = 0; j < QK_K; j += 128) { - vl = 16; + vl = 32; - // retrieve lane to multiply with scale - vint32m2_t aux0_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 0), (scale[0]), vl); - vint32m2_t aux0_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 1), (scale[1]), vl); - vint32m2_t aux1_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 0), (scale[2]), vl); - vint32m2_t aux1_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 1), (scale[3]), vl); - vint32m2_t aux2_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 0), (scale[4]), vl); - vint32m2_t aux2_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 1), (scale[5]), vl); - vint32m2_t aux3_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 0), (scale[6]), vl); - vint32m2_t aux3_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 1), (scale[7]), vl); + // load Q3 + vuint8m1_t q3_x = __riscv_vle8_v_u8m1(q3, vl); - vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux0_0, aux0_1, vl), vzero, vl); - vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux1_0, aux1_1, vl), isum0, vl); - vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux2_0, aux2_1, vl), isum1, vl); - vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux3_0, aux3_1, vl), isum2, vl); + vint8m1_t q3_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q3_x, 0x03, vl)); + vint8m1_t q3_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x2, vl), 0x03 , vl)); + vint8m1_t q3_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x4, vl), 0x03 , vl)); + vint8m1_t q3_3 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x6, vl), 0x03 , vl)); - sum_t += __riscv_vmv_x_s_i32m1_i32(isum3); + // compute mask for subtraction + vuint8m1_t qh_m0 = __riscv_vand_vx_u8m1(vqh, m, vl); + vbool8_t vmask_0 = __riscv_vmseq_vx_u8m1_b8(qh_m0, 0, vl); + vint8m1_t q3_m0 = __riscv_vsub_vx_i8m1_mu(vmask_0, q3_0, q3_0, 0x4, vl); + m <<= 1; - q3 += 32; q8 += 128; scale += 8; + vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl); + vbool8_t vmask_1 = __riscv_vmseq_vx_u8m1_b8(qh_m1, 0, vl); + vint8m1_t q3_m1 = __riscv_vsub_vx_i8m1_mu(vmask_1, q3_1, q3_1, 0x4, vl); + m <<= 1; - } + vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl); + vbool8_t vmask_2 = __riscv_vmseq_vx_u8m1_b8(qh_m2, 0, vl); + vint8m1_t q3_m2 = __riscv_vsub_vx_i8m1_mu(vmask_2, q3_2, q3_2, 0x4, vl); + m <<= 1; + + vuint8m1_t qh_m3 = __riscv_vand_vx_u8m1(vqh, m, vl); + vbool8_t vmask_3 = __riscv_vmseq_vx_u8m1_b8(qh_m3, 0, vl); + vint8m1_t q3_m3 = __riscv_vsub_vx_i8m1_mu(vmask_3, q3_3, q3_3, 0x4, vl); + m <<= 1; + + // load Q8 and take product with Q3 + vint16m2_t a0 = __riscv_vwmul_vv_i16m2(q3_m0, __riscv_vle8_v_i8m1(q8, vl), vl); + vint16m2_t a1 = __riscv_vwmul_vv_i16m2(q3_m1, __riscv_vle8_v_i8m1(q8+32, vl), vl); + vint16m2_t a2 = __riscv_vwmul_vv_i16m2(q3_m2, __riscv_vle8_v_i8m1(q8+64, vl), vl); + vint16m2_t a3 = __riscv_vwmul_vv_i16m2(q3_m3, __riscv_vle8_v_i8m1(q8+96, vl), vl); + + vl = 16; + + // retrieve lane to multiply with scale + vint32m2_t aux0_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 0), (scale[0]), vl); + vint32m2_t aux0_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 1), (scale[1]), vl); + vint32m2_t aux1_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 0), (scale[2]), vl); + vint32m2_t aux1_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 1), (scale[3]), vl); + vint32m2_t aux2_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 0), (scale[4]), vl); + vint32m2_t aux2_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 1), (scale[5]), vl); + vint32m2_t aux3_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 0), (scale[6]), vl); + vint32m2_t aux3_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 1), (scale[7]), vl); - const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux0_0, aux0_1, vl), vzero, vl); + vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux1_0, aux1_1, vl), isum0, vl); + vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux2_0, aux2_1, vl), isum1, vl); + vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux3_0, aux3_1, vl), isum2, vl); - sumf += d*sum_t; + sum_t += __riscv_vmv_x_s_i32m1_i32(isum3); + + q3 += 32; q8 += 128; scale += 8; } - break; - case 128: - for (int i = 0; i < nb; ++i) { - const uint8_t * restrict q3 = x[i].qs; - const uint8_t * restrict qh = x[i].hmask; - const int8_t * restrict q8 = y[i].qs; - - int8_t * scale = (int8_t *)utmp; - int tmp, t1, t2, t3, t4, t5, t6, t7; - __asm__ __volatile__( - "vsetivli zero, 12, e8, m1\n\t" - "vle8.v v0, (%[s6b])\n\t" - "vmv1r.v v2, v0\n\t" - "vsetivli zero, 2, e64, m1\n\t" - "vmv.v.x v9, %[sh]\n\t"\ - "vslidedown.vi v1, v0, 1\n\t" - "vslide1up.vx v8, v9, zero\n\t" // {0, 0, 4, 4} - "vslideup.vi v0, v2, 1\n\t" // {aux[0], aux[1], aux[0], aux[1]} - "vsetivli zero, 4, e32, m1\n\t" - "vid.v v9\n\t" - "vmv.x.s %[tmp], v1\n\t" - "vsll.vi v9, v9, 1\n\t" // {0, 2, 4, 6} - "vmv.v.x v1, %[tmp]\n\t" // {aux[2], aux[2], aux[2], aux[2]} - "vsrl.vv v4, v1, v9\n\t" - "vsrl.vv v2, v0, v8\n\t" - "vand.vx v5, v4, %[kmask1]\n\t" - "vand.vx v3, v2, %[kmask2]\n\t" - "vsll.vi v6, v5, 4\n\t" - "vor.vv v7, v6, v3\n\t" - "vsetivli zero, 16, e8, m1\n\t" - "vsub.vx v0, v7, %[c]\n\t" - "vse8.v v0, (%[scale])" - : [tmp] "=&r" (tmp) - : [sh] "r" (0x0000000400000004), [s6b] "r" (x[i].scales), [c] "r" (32) - , [scale] "r" (scale), [kmask1] "r" (kmask1), [kmask2] "r" (kmask2) - : "memory" - , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" - , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" - , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" - , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" - ); - uint8_t m = 1; - int isum = 0; - for (int j = 0; j < QK_K; j += 128) { - __asm__ __volatile__( - "lb zero, 31(%[q3])\n\t" - "vsetvli zero, %[vl32], e8, m2, ta, mu\n\t" - "vle8.v v8, (%[q3])\n\t" - "vsrl.vi v10, v8, 2\n\t" - "vsrl.vi v12, v8, 4\n\t" - "vsrl.vi v14, v8, 6\n\t" - "lb zero, 64(%[q8])\n\t" - "vand.vi v8, v8, 3\n\t" - "vand.vi v10, v10, 3\n\t" - "vand.vi v12, v12, 3\n\t" - "vle8.v v2, (%[qh])\n\t" - "lb zero, 127(%[q8])\n\t" - "vand.vx v4, v2, %[m]\n\t" - "slli %[m], %[m], 1\n\t" - "vmseq.vx v0, v4, zero\n\t" - "vadd.vi v8, v8, -4, v0.t\n\t" - "lb zero, 0(%[q8])\n\t" - "vand.vx v4, v2, %[m]\n\t" - "slli %[m], %[m], 1\n\t" - "vmseq.vx v0, v4, zero\n\t" - "vadd.vi v10, v10, -4, v0.t\n\t" - "vand.vx v4, v2, %[m]\n\t" - "slli %[m], %[m], 1\n\t" - "vmseq.vx v0, v4, zero\n\t" - "vadd.vi v12, v12, -4, v0.t\n\t" - "vand.vx v4, v2, %[m]\n\t" - "slli %[m], %[m], 1\n\t" - "vmseq.vx v0, v4, zero\n\t" - "vadd.vi v14, v14, -4, v0.t\n\t" - "vsetvli zero, %[vl128], e8, m8\n\t" - "vle8.v v0, (%[q8])\n\t" - "lb %[tmp], 0(%[scale])\n\t" - "lb %[t1], 1(%[scale])\n\t" - "lb %[t2], 2(%[scale])\n\t" - "lb %[t3], 3(%[scale])\n\t" - "vsetvli zero, %[vl64], e8, m4\n\t" - "vwmul.vv v16, v0, v8\n\t" - "vwmul.vv v24, v4, v12\n\t" - "vsetivli zero, 16, e16, m2\n\t" - "vmv.v.x v0, zero\n\t" - "vwredsum.vs v8, v16, v0\n\t" - "lb %[t4], 4(%[scale])\n\t" - "lb %[t5], 5(%[scale])\n\t" - "vwredsum.vs v9, v18, v0\n\t" - "vwredsum.vs v10, v20, v0\n\t" - "vwredsum.vs v11, v22, v0\n\t" - "vwredsum.vs v12, v24, v0\n\t" - "lb %[t6], 6(%[scale])\n\t" - "lb %[t7], 7(%[scale])\n\t" - "vwredsum.vs v13, v26, v0\n\t" - "vwredsum.vs v14, v28, v0\n\t" - "vwredsum.vs v15, v30, v0\n\t" - "vsetivli zero, 4, e32, m1\n\t" - "vmul.vx v0, v8, %[tmp]\n\t" - "vmul.vx v1, v9, %[t1]\n\t" - "vmacc.vx v0, %[t2], v10\n\t" - "vmacc.vx v1, %[t3], v11\n\t" - "vmacc.vx v0, %[t4], v12\n\t" - "vmacc.vx v1, %[t5], v13\n\t" - "vmacc.vx v0, %[t6], v14\n\t" - "vmacc.vx v1, %[t7], v15\n\t" - "vmv.x.s %[tmp], v0\n\t" - "vmv.x.s %[t1], v1\n\t" - "add %[isum], %[isum], %[tmp]\n\t" - "add %[isum], %[isum], %[t1]" - : [tmp] "=&r" (tmp), [t1] "=&r" (t1), [t2] "=&r" (t2), [t3] "=&r" (t3) - , [t4] "=&r" (t4), [t5] "=&r" (t5), [t6] "=&r" (t6), [t7] "=&r" (t7) - , [m] "+&r" (m), [isum] "+&r" (isum) - : [vl128] "r" (128), [vl64] "r" (64), [vl32] "r" (32) - , [q3] "r" (q3), [qh] "r" (qh), [scale] "r" (scale), [q8] "r" (q8) - : "memory" - , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" - , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" - , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" - , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" - ); - q3 += 32; q8 += 128; scale += 8; - } + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + sumf += d*sum_t; + + } + + *s = sumf; +} + +void ggml_vec_dot_q3_K_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; + + const block_q3_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + // mask for processing 16 elements per prod register + const vuint16m1_t va_index = __riscv_vid_v_u16m1(32); + const vbool16_t va_mask = __riscv_vmsgtu_vx_u16m1_b16(va_index, 15, 32); + + uint32_t utmp[4]; + float sumf = 0; + uint32_t aux[3]; + + for (int i = 0; i < nb; ++i) { + const uint8_t * GGML_RESTRICT q3 = x[i].qs; + const uint8_t * GGML_RESTRICT qh = x[i].hmask; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + memcpy(aux, x[i].scales, 12); + utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4); + utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4); + utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4); + utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4); + + int8_t * scale = (int8_t *)utmp; + for (int j = 0; j < 16; ++j) scale[j] -= 32; + + + size_t vl = 32; + uint8_t m = 1; - const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; - sumf += d * isum; + vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); + vuint8mf2_t vqh = __riscv_vle8_v_u8mf2(qh, vl); + + int sum_t = 0; + + vint32m2_t vaux_0 = __riscv_vmv_v_x_i32m2(0, vl); + vint32m2_t vaux_1 = __riscv_vmv_v_x_i32m2(0, vl); + vint32m2_t vaux_2 = __riscv_vmv_v_x_i32m2(0, vl); + vint32m2_t vaux_3 = __riscv_vmv_v_x_i32m2(0, vl); + + for (int j = 0; j < QK_K; j += 128) { + + vl = 32; + + // load Q3 + vuint8mf2_t q3_x = __riscv_vle8_v_u8mf2(q3, vl); + + vint8mf2_t q3_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q3_x, 0x03, vl)); + vint8mf2_t q3_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 0x2, vl), 0x03 , vl)); + vint8mf2_t q3_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 0x4, vl), 0x03 , vl)); + vint8mf2_t q3_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 0x6, vl), 0x03 , vl)); + + // compute mask for subtraction + vuint8mf2_t qh_m0 = __riscv_vand_vx_u8mf2(vqh, m, vl); + vbool16_t vmask_0 = __riscv_vmseq_vx_u8mf2_b16(qh_m0, 0, vl); + vint8mf2_t q3_m0 = __riscv_vsub_vx_i8mf2_mu(vmask_0, q3_0, q3_0, 0x4, vl); + m <<= 1; + + vuint8mf2_t qh_m1 = __riscv_vand_vx_u8mf2(vqh, m, vl); + vbool16_t vmask_1 = __riscv_vmseq_vx_u8mf2_b16(qh_m1, 0, vl); + vint8mf2_t q3_m1 = __riscv_vsub_vx_i8mf2_mu(vmask_1, q3_1, q3_1, 0x4, vl); + m <<= 1; + + vuint8mf2_t qh_m2 = __riscv_vand_vx_u8mf2(vqh, m, vl); + vbool16_t vmask_2 = __riscv_vmseq_vx_u8mf2_b16(qh_m2, 0, vl); + vint8mf2_t q3_m2 = __riscv_vsub_vx_i8mf2_mu(vmask_2, q3_2, q3_2, 0x4, vl); + m <<= 1; + + vuint8mf2_t qh_m3 = __riscv_vand_vx_u8mf2(vqh, m, vl); + vbool16_t vmask_3 = __riscv_vmseq_vx_u8mf2_b16(qh_m3, 0, vl); + vint8mf2_t q3_m3 = __riscv_vsub_vx_i8mf2_mu(vmask_3, q3_3, q3_3, 0x4, vl); + m <<= 1; + + // load Q8 and take product + vint16m1_t va_q_0 = __riscv_vwmul_vv_i16m1(q3_m0, __riscv_vle8_v_i8mf2(q8, vl), vl); + vint16m1_t va_q_1 = __riscv_vwmul_vv_i16m1(q3_m1, __riscv_vle8_v_i8mf2(q8+32, vl), vl); + vint16m1_t va_q_2 = __riscv_vwmul_vv_i16m1(q3_m2, __riscv_vle8_v_i8mf2(q8+64, vl), vl); + vint16m1_t va_q_3 = __riscv_vwmul_vv_i16m1(q3_m3, __riscv_vle8_v_i8mf2(q8+96, vl), vl); + + // accumulate + vaux_0 = __riscv_vwmacc_vx_i32m2(vaux_0, scale[0], va_q_0, 16); + vaux_1 = __riscv_vwmacc_vx_i32m2(vaux_1, scale[2], va_q_1, 16); + vaux_2 = __riscv_vwmacc_vx_i32m2(vaux_2, scale[4], va_q_2, 16); + vaux_3 = __riscv_vwmacc_vx_i32m2(vaux_3, scale[6], va_q_3, 16); + // + vaux_0 = __riscv_vwmacc_vx_i32m2_m(va_mask, vaux_0, scale[1], va_q_0, vl); + vaux_1 = __riscv_vwmacc_vx_i32m2_m(va_mask, vaux_1, scale[3], va_q_1, vl); + vaux_2 = __riscv_vwmacc_vx_i32m2_m(va_mask, vaux_2, scale[5], va_q_2, vl); + vaux_3 = __riscv_vwmacc_vx_i32m2_m(va_mask, vaux_3, scale[7], va_q_3, vl); + + q3 += 32; q8 += 128; scale += 8; } - break; - default: - assert(false && "Unsupported vector length"); - break; + + vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl); + vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl); + + sum_t += __riscv_vmv_x_s_i32m1_i32(isum1); + + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + sumf += d*sum_t; } *s = sumf; +} -#else +void ggml_vec_dot_q3_K_q8_K_vl1024(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); - UNUSED(kmask1); - UNUSED(kmask2); - UNUSED(x); - UNUSED(y); - UNUSED(nb); + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; - ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + const block_q3_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + // mask for processing 16 elements per prod register + const vuint16mf2_t va_index = __riscv_vid_v_u16mf2(32); + const vbool32_t va_mask = __riscv_vmsgtu_vx_u16mf2_b32(va_index, 15, 32); + + uint32_t utmp[4]; + float sumf = 0; + uint32_t aux[3]; + + for (int i = 0; i < nb; ++i) { + const uint8_t * GGML_RESTRICT q3 = x[i].qs; + const uint8_t * GGML_RESTRICT qh = x[i].hmask; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + memcpy(aux, x[i].scales, 12); + utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4); + utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4); + utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4); + utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4); + + int8_t * scale = (int8_t *)utmp; + for (int j = 0; j < 16; ++j) scale[j] -= 32; + + + size_t vl = 32; + uint8_t m = 1; + + vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); + vuint8mf4_t vqh = __riscv_vle8_v_u8mf4(qh, vl); + + int sum_t = 0; + + vint32m1_t vaux_0 = __riscv_vmv_v_x_i32m1(0, vl); + vint32m1_t vaux_1 = __riscv_vmv_v_x_i32m1(0, vl); + vint32m1_t vaux_2 = __riscv_vmv_v_x_i32m1(0, vl); + vint32m1_t vaux_3 = __riscv_vmv_v_x_i32m1(0, vl); + + for (int j = 0; j < QK_K; j += 128) { + + vl = 32; + + // load Q3 + vuint8mf4_t q3_x = __riscv_vle8_v_u8mf4(q3, vl); + + vint8mf4_t q3_0 = __riscv_vreinterpret_v_u8mf4_i8mf4(__riscv_vand_vx_u8mf4(q3_x, 0x03, vl)); + vint8mf4_t q3_1 = __riscv_vreinterpret_v_u8mf4_i8mf4(__riscv_vand_vx_u8mf4(__riscv_vsrl_vx_u8mf4(q3_x, 0x2, vl), 0x03 , vl)); + vint8mf4_t q3_2 = __riscv_vreinterpret_v_u8mf4_i8mf4(__riscv_vand_vx_u8mf4(__riscv_vsrl_vx_u8mf4(q3_x, 0x4, vl), 0x03 , vl)); + vint8mf4_t q3_3 = __riscv_vreinterpret_v_u8mf4_i8mf4(__riscv_vand_vx_u8mf4(__riscv_vsrl_vx_u8mf4(q3_x, 0x6, vl), 0x03 , vl)); + + // compute mask for subtraction + vuint8mf4_t qh_m0 = __riscv_vand_vx_u8mf4(vqh, m, vl); + vbool32_t vmask_0 = __riscv_vmseq_vx_u8mf4_b32(qh_m0, 0, vl); + vint8mf4_t q3_m0 = __riscv_vsub_vx_i8mf4_mu(vmask_0, q3_0, q3_0, 0x4, vl); + m <<= 1; + + vuint8mf4_t qh_m1 = __riscv_vand_vx_u8mf4(vqh, m, vl); + vbool32_t vmask_1 = __riscv_vmseq_vx_u8mf4_b32(qh_m1, 0, vl); + vint8mf4_t q3_m1 = __riscv_vsub_vx_i8mf4_mu(vmask_1, q3_1, q3_1, 0x4, vl); + m <<= 1; + + vuint8mf4_t qh_m2 = __riscv_vand_vx_u8mf4(vqh, m, vl); + vbool32_t vmask_2 = __riscv_vmseq_vx_u8mf4_b32(qh_m2, 0, vl); + vint8mf4_t q3_m2 = __riscv_vsub_vx_i8mf4_mu(vmask_2, q3_2, q3_2, 0x4, vl); + m <<= 1; + + vuint8mf4_t qh_m3 = __riscv_vand_vx_u8mf4(vqh, m, vl); + vbool32_t vmask_3 = __riscv_vmseq_vx_u8mf4_b32(qh_m3, 0, vl); + vint8mf4_t q3_m3 = __riscv_vsub_vx_i8mf4_mu(vmask_3, q3_3, q3_3, 0x4, vl); + m <<= 1; + + // load Q8 and take product + vint16mf2_t va_q_0 = __riscv_vwmul_vv_i16mf2(q3_m0, __riscv_vle8_v_i8mf4(q8, vl), vl); + vint16mf2_t va_q_1 = __riscv_vwmul_vv_i16mf2(q3_m1, __riscv_vle8_v_i8mf4(q8+32, vl), vl); + vint16mf2_t va_q_2 = __riscv_vwmul_vv_i16mf2(q3_m2, __riscv_vle8_v_i8mf4(q8+64, vl), vl); + vint16mf2_t va_q_3 = __riscv_vwmul_vv_i16mf2(q3_m3, __riscv_vle8_v_i8mf4(q8+96, vl), vl); + + // accumulate + vaux_0 = __riscv_vwmacc_vx_i32m1(vaux_0, scale[0], va_q_0, 16); + vaux_1 = __riscv_vwmacc_vx_i32m1(vaux_1, scale[2], va_q_1, 16); + vaux_2 = __riscv_vwmacc_vx_i32m1(vaux_2, scale[4], va_q_2, 16); + vaux_3 = __riscv_vwmacc_vx_i32m1(vaux_3, scale[6], va_q_3, 16); + // + vaux_0 = __riscv_vwmacc_vx_i32m1_m(va_mask, vaux_0, scale[1], va_q_0, vl); + vaux_1 = __riscv_vwmacc_vx_i32m1_m(va_mask, vaux_1, scale[3], va_q_1, vl); + vaux_2 = __riscv_vwmacc_vx_i32m1_m(va_mask, vaux_2, scale[5], va_q_2, vl); + vaux_3 = __riscv_vwmacc_vx_i32m1_m(va_mask, vaux_3, scale[7], va_q_3, vl); + + q3 += 32; q8 += 128; scale += 8; + } + + vint32m1_t isum0 = __riscv_vredsum_vs_i32m1_i32m1(__riscv_vadd_vv_i32m1(vaux_0, vaux_1, vl), vzero, vl); + vint32m1_t isum1 = __riscv_vredsum_vs_i32m1_i32m1(__riscv_vadd_vv_i32m1(vaux_2, vaux_3, vl), isum0, vl); + + sum_t += __riscv_vmv_x_s_i32m1_i32(isum1); + + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + sumf += d*sum_t; + } + + *s = sumf; +} #endif +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined __riscv_xtheadvector + ggml_vec_dot_q3_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc); +#elif defined __riscv_v + switch (__riscv_vlenb() * 8) { + case 128: + ggml_vec_dot_q3_K_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + break; + case 256: + ggml_vec_dot_q3_K_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + break; + case 512: + ggml_vec_dot_q3_K_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); + break; + case 1024: + ggml_vec_dot_q3_K_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); + break; + default: + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + break; + } +#else + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif } -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined __riscv_xtheadvector +static NOINLINE void ggml_vec_dot_q4_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -1239,8 +1650,6 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi uint32_t utmp[4]; -#if defined __riscv_xtheadvector - const uint8_t * scales = (const uint8_t*)&utmp[0]; const uint8_t * mins = (const uint8_t*)&utmp[2]; @@ -1354,277 +1763,317 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi } *s = sumf; +} +#endif -#elif defined __riscv_v +#if defined __riscv_v +static NOINLINE void ggml_vec_dot_q4_K_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q4_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + static const uint32_t kmask1 = 0x3f3f3f3f; + static const uint32_t kmask2 = 0x0f0f0f0f; + static const uint32_t kmask3 = 0x03030303; + + uint32_t utmp[4]; const uint8_t * scales = (const uint8_t*)&utmp[0]; const uint8_t * mins = (const uint8_t*)&utmp[2]; float sumf = 0; - const int vector_length = __riscv_vlenb() * 8; + for (int i = 0; i < nb; ++i) { + const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); + const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); - switch (vector_length) { - case 256: - for (int i = 0; i < nb; ++i) { + float ftmp, ft2; + const uint8_t * restrict q40; + const uint8_t * restrict q41; + const uint8_t * restrict q42; + const uint8_t * restrict q43; + const int8_t * restrict q80; + const int8_t * restrict q81; + const int8_t * restrict q82; + const int8_t * restrict q83; + int s0, s1, s2, s3; - size_t vl = 8; + __asm__ __volatile__( + "li %[s1], 8\n\t" + "vsetivli zero, 4, e32, m1, ta, ma\n\t" + "vle32.v v1, (%[s6b])\n\t" + "vslide1down.vx v1, v1, zero\n\t" + "vmv.v.x v16, zero\n\t" + "vslidedown.vi v2, v1, 2\n\t" + "vmv1r.v v3, v2\n\t" + "vslideup.vi v2, v3, 1\n\t" // {aux[2], aux[2]} + "vsetivli zero, 2, e32, m1, ta, ma\n\t" + "vmv.v.i v4, 4\n\t" + "vand.vx v8, v1, %[kmask1]\n\t" + "vslide1up.vx v5, v4, zero\n\t" // {0, 4} + "vsrl.vi v6, v1, 6\n\t" + "vsrl.vv v7, v2, v5\n\t" + "vsse32.v v8, (%[utmp]), %[s1]\n\t" + "vand.vx v0, v6, %[kmask3]\n\t" + "vand.vx v2, v7, %[kmask2]\n\t" + "vsll.vi v6, v0, 4\n\t" + "addi %[s0], %[utmp], 4\n\t" + "vor.vv v1, v6, v2\n\t" + "vsse32.v v1, (%[s0]), %[s1]\n\t" + "vsetivli zero, 8, e16, m1, ta, ma\n\t" + "vle32.v v2, (%[bsums])\n\t" + "vnsrl.wi v0, v2, 0\n\t" + "vnsrl.wi v1, v2, 16\n\t" + "vadd.vv v2, v0, v1\n\t" + "vle8.v v3, (%[mins])\n\t" + "vzext.vf2 v4, v3\n\t" + "vwmul.vv v6, v4, v2\n\t" + "vsetivli zero, 4, e32, m1, ta, ma\n\t" + "vredsum.vs v0, v6, v16\n\t" + "vredsum.vs v0, v7, v0\n\t" + "vfcvt.f.x.v v0, v0\n\t" + "vfmv.f.s %[ftmp], v0\n\t" + "vsetivli zero, 16, e8, m1, ta, ma\n\t" + "vle8.v v0, (%[xs])\n\t" + "fnmsub.s %[sumf], %[dmin], %[ftmp], %[sumf]\n\t" + "addi %[q40], %[xs], 64\n\t" + "addi %[q41], %[xs], 16\n\t" + "addi %[q42], %[xs], 32\n\t" + "addi %[q43], %[xs], 48\n\t" + "addi %[q80], %[ys], 64\n\t" + "vle8.v v1, (%[q41])\n\t" + "vle8.v v2, (%[q42])\n\t" + "addi %[q81], %[ys], 16\n\t" + "addi %[q41], %[q41], 64\n\t" + "addi %[q82], %[ys], 32\n\t" + "vle8.v v3, (%[q43])\n\t" + "vle8.v v8, (%[ys])\n\t" + "addi %[q42], %[q42], 64\n\t" + "addi %[q83], %[ys], 48\n\t" + "addi %[q43], %[q43], 64\n\t" + "vsrl.vi v4, v0, 4\n\t" + "vle8.v v9, (%[q81])\n\t" + "vle8.v v10, (%[q82])\n\t" + "vand.vi v0, v0, 0xF\n\t" + "addi %[q81], %[q81], 64\n\t" + "vsrl.vi v5, v1, 4\n\t" + "addi %[q82], %[q82], 64\n\t" + "vle8.v v11, (%[q83])\n\t" + "vle8.v v12, (%[q80])\n\t" + "vand.vi v1, v1, 0xF\n\t" + "addi %[q83], %[q83], 64\n\t" + "vsrl.vi v6, v2, 4\n\t" + "addi %[q80], %[q80], 64\n\t" + "vle8.v v13, (%[q81])\n\t" + "vle8.v v14, (%[q82])\n\t" + "vand.vi v2, v2, 0xF\n\t" + "addi %[q81], %[q81], 64\n\t" + "vsrl.vi v7, v3, 4\n\t" + "addi %[q82], %[q82], 64\n\t" + "vwmul.vv v16, v0, v8\n\t" + "vle8.v v15, (%[q83])\n\t" + "vle8.v v0, (%[q40])\n\t" + "vand.vi v3, v3, 0xF\n\t" + "addi %[q83], %[q83], 64\n\t" + "vwmul.vv v24, v2, v12\n\t" + "vwmul.vv v20, v4, v10\n\t" + "vwmul.vv v28, v6, v14\n\t" + "vwmacc.vv v16, v1, v9\n\t" + "vle8.v v1, (%[q41])\n\t" + "vle8.v v2, (%[q42])\n\t" + "vwmacc.vv v24, v3, v13\n\t" + "vwmacc.vv v20, v5, v11\n\t" + "vwmacc.vv v28, v7, v15\n\t" + "addi %[q40], %[q80], 64\n\t" + "addi %[q41], %[q81], 64\n\t" + "vle8.v v3, (%[q43])\n\t" + "vle8.v v8, (%[q80])\n\t" + "addi %[q42], %[q82], 64\n\t" + "addi %[q43], %[q83], 64\n\t" + "vsrl.vi v4, v0, 4\n\t" + "vle8.v v9, (%[q81])\n\t" + "vle8.v v10, (%[q82])\n\t" + "vand.vi v0, v0, 0xF\n\t" + "vsrl.vi v5, v1, 4\n\t" + "vsrl.vi v7, v3, 4\n\t" + "vand.vi v3, v3, 0xF\n\t" + "vle8.v v11, (%[q83])\n\t" + "vle8.v v12, (%[q40])\n\t" + "vand.vi v1, v1, 0xF\n\t" + "vsrl.vi v6, v2, 4\n\t" + "vand.vi v2, v2, 0xF\n\t" + "vwmul.vv v18, v0, v8\n\t" + "vle8.v v13, (%[q41])\n\t" + "vle8.v v14, (%[q42])\n\t" + "vwmul.vv v26, v2, v12\n\t" + "vwmul.vv v22, v4, v10\n\t" + "vwmul.vv v30, v6, v14\n\t" + "vwmacc.vv v18, v1, v9\n\t" + "vle8.v v15, (%[q43])\n\t" + "vwmacc.vv v26, v3, v13\n\t" + "vwmacc.vv v22, v5, v11\n\t" + "vwmacc.vv v30, v7, v15\n\t" + "vmv.v.x v0, zero\n\t" + "vsetivli zero, 16, e16, m2, ta, ma\n\t" + "vwredsum.vs v4, v16, v0\n\t" + "lbu %[s0], 0(%[scale])\n\t" + "vwredsum.vs v5, v20, v0\n\t" + "lbu %[s1], 1(%[scale])\n\t" + "vwredsum.vs v6, v24, v0\n\t" + "lbu %[s2], 2(%[scale])\n\t" + "vwredsum.vs v7, v28, v0\n\t" + "lbu %[s3], 3(%[scale])\n\t" + "vwredsum.vs v8, v18, v0\n\t" + "lbu %[q40], 4(%[scale])\n\t" + "vwredsum.vs v9, v22, v0\n\t" + "lbu %[q41], 5(%[scale])\n\t" + "vwredsum.vs v10, v26, v0\n\t" + "lbu %[q42], 6(%[scale])\n\t" + "vwredsum.vs v11, v30, v0\n\t" + "lbu %[q43], 7(%[scale])\n\t" + "vsetivli zero, 4, e32, m1, ta, ma\n\t" + "vmul.vx v0, v4, %[s0]\n\t" + "vmul.vx v1, v8, %[q40]\n\t" + "vmacc.vx v0, %[s1], v5\n\t" + "vmacc.vx v1, %[q41], v9\n\t" + "vmacc.vx v0, %[s2], v6\n\t" + "vmacc.vx v1, %[q42], v10\n\t" + "vmacc.vx v0, %[s3], v7\n\t" + "vmacc.vx v1, %[q43], v11\n\t" + "vfcvt.f.x.v v0, v0\n\t" + "vfcvt.f.x.v v1, v1\n\t" + "vfmv.f.s %[ft2], v0\n\t" + "vfmv.f.s %[ftmp], v1\n\t" + "fadd.s %[ft2], %[ft2], %[ftmp]\n\t" + "fmadd.s %[sumf], %[d], %[ft2], %[sumf]" + : [ftmp] "=&f" (ftmp), [sumf] "+&f" (sumf), [ft2] "=&f" (ft2) + , [s0] "=&r" (s0), [s1] "=&r" (s1), [s2] "=&r" (s2), [s3] "=&r" (s3) + , [q40] "=&r" (q40), [q41] "=&r" (q41), [q42] "=&r" (q42), [q43] "=&r" (q43) + , [q80] "=&r" (q80), [q81] "=&r" (q81), [q82] "=&r" (q82), [q83] "=&r" (q83) + : [d] "f" (d), [ys] "r" (y[i].qs), [xs] "r" (x[i].qs), [scale] "r" (scales) + , [bsums] "r" (y[i].bsums), [mins] "r" (mins), [utmp] "r" (utmp) + , [s6b] "r" (&x[i]), [kmask1] "r" (kmask1), [dmin] "f" (dmin) + , [kmask2] "r" (kmask2), [kmask3] "r" (kmask3) + : "memory" + , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" + , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" + ); + } - const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); - const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); + *s = sumf; +} - vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl); - vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl); - vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl); +static NOINLINE void ggml_vec_dot_q4_K_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); - memcpy(utmp, x[i].scales, 12); - utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4); - const uint32_t uaux = utmp[1] & kmask1; - utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); - utmp[2] = uaux; - utmp[0] &= kmask1; + const block_q4_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; - vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl); - vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl)); - vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl); + const int nb = n / QK_K; - vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl); - sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi); + static const uint32_t kmask1 = 0x3f3f3f3f; + static const uint32_t kmask2 = 0x0f0f0f0f; + static const uint32_t kmask3 = 0x03030303; - const uint8_t * GGML_RESTRICT q4 = x[i].qs; - const int8_t * GGML_RESTRICT q8 = y[i].qs; + uint32_t utmp[4]; - vl = 32; + const uint8_t * scales = (const uint8_t*)&utmp[0]; + const uint8_t * mins = (const uint8_t*)&utmp[2]; - int32_t sum_1 = 0; - int32_t sum_2 = 0; + float sumf = 0; + for (int i = 0; i < nb; ++i) { + size_t vl = 8; - vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1); + const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); + const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); - for (int j = 0; j < QK_K/64; ++j) { - // load Q4 - vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl); + vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl); + vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl); + vint16mf2_t q8sums = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl); - // load Q8 and multiply it with lower Q4 nibble - vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl); - vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl)); - vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl); - vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl); + memcpy(utmp, x[i].scales, 12); + utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4); + const uint32_t uaux = utmp[1] & kmask1; + utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); + utmp[2] = uaux; + utmp[0] &= kmask1; - sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0]; + vuint8mf4_t mins8 = __riscv_vle8_v_u8mf4(mins, vl); + vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl)); + vint32m1_t prod = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl); - // load Q8 and multiply it with upper Q4 nibble - vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl); - vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl)); - vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl); - vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl); + vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl); + sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi); - sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1]; + const uint8_t * GGML_RESTRICT q4 = x[i].qs; + const int8_t * GGML_RESTRICT q8 = y[i].qs; - q4 += 32; q8 += 64; + vl = 32; - } + int32_t sum_1 = 0; + int32_t sum_2 = 0; - sumf += d*(sum_1 + sum_2); + vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1); - } - break; - case 128: - for (int i = 0; i < nb; ++i) { - const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); - const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); - - float ftmp, ft2; - const uint8_t * restrict q40; - const uint8_t * restrict q41; - const uint8_t * restrict q42; - const uint8_t * restrict q43; - const int8_t * restrict q80; - const int8_t * restrict q81; - const int8_t * restrict q82; - const int8_t * restrict q83; - int s0, s1, s2, s3; + for (int j = 0; j < QK_K/64; ++j) { + // load Q4 + vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl); + + // load Q8 and multiply it with lower Q4 nibble + vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl); + vint8m1_t q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl)); + vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl); + vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl); + + sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0]; + + // load Q8 and multiply it with upper Q4 nibble + vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8+32, vl); + vint8m1_t q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl)); + vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl); + vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl); + + sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1]; + + q4 += 32; q8 += 64; - __asm__ __volatile__( - "li %[s1], 8\n\t" - "vsetivli zero, 4, e32, m1, ta, ma\n\t" - "vle32.v v1, (%[s6b])\n\t" - "vslide1down.vx v1, v1, zero\n\t" - "vmv.v.x v16, zero\n\t" - "vslidedown.vi v2, v1, 2\n\t" - "vmv1r.v v3, v2\n\t" - "vslideup.vi v2, v3, 1\n\t" // {aux[2], aux[2]} - "vsetivli zero, 2, e32, m1, ta, ma\n\t" - "vmv.v.i v4, 4\n\t" - "vand.vx v8, v1, %[kmask1]\n\t" - "vslide1up.vx v5, v4, zero\n\t" // {0, 4} - "vsrl.vi v6, v1, 6\n\t" - "vsrl.vv v7, v2, v5\n\t" - "vsse32.v v8, (%[utmp]), %[s1]\n\t" - "vand.vx v0, v6, %[kmask3]\n\t" - "vand.vx v2, v7, %[kmask2]\n\t" - "vsll.vi v6, v0, 4\n\t" - "addi %[s0], %[utmp], 4\n\t" - "vor.vv v1, v6, v2\n\t" - "vsse32.v v1, (%[s0]), %[s1]\n\t" - "vsetivli zero, 8, e16, m1, ta, ma\n\t" - "vle32.v v2, (%[bsums])\n\t" - "vnsrl.wi v0, v2, 0\n\t" - "vnsrl.wi v1, v2, 16\n\t" - "vadd.vv v2, v0, v1\n\t" - "vle8.v v3, (%[mins])\n\t" - "vzext.vf2 v4, v3\n\t" - "vwmul.vv v6, v4, v2\n\t" - "vsetivli zero, 4, e32, m1, ta, ma\n\t" - "vredsum.vs v0, v6, v16\n\t" - "vredsum.vs v0, v7, v0\n\t" - "vfcvt.f.x.v v0, v0\n\t" - "vfmv.f.s %[ftmp], v0\n\t" - "vsetivli zero, 16, e8, m1, ta, ma\n\t" - "vle8.v v0, (%[xs])\n\t" - "fnmsub.s %[sumf], %[dmin], %[ftmp], %[sumf]\n\t" - "addi %[q40], %[xs], 64\n\t" - "addi %[q41], %[xs], 16\n\t" - "addi %[q42], %[xs], 32\n\t" - "addi %[q43], %[xs], 48\n\t" - "addi %[q80], %[ys], 64\n\t" - "vle8.v v1, (%[q41])\n\t" - "vle8.v v2, (%[q42])\n\t" - "addi %[q81], %[ys], 16\n\t" - "addi %[q41], %[q41], 64\n\t" - "addi %[q82], %[ys], 32\n\t" - "vle8.v v3, (%[q43])\n\t" - "vle8.v v8, (%[ys])\n\t" - "addi %[q42], %[q42], 64\n\t" - "addi %[q83], %[ys], 48\n\t" - "addi %[q43], %[q43], 64\n\t" - "vsrl.vi v4, v0, 4\n\t" - "vle8.v v9, (%[q81])\n\t" - "vle8.v v10, (%[q82])\n\t" - "vand.vi v0, v0, 0xF\n\t" - "addi %[q81], %[q81], 64\n\t" - "vsrl.vi v5, v1, 4\n\t" - "addi %[q82], %[q82], 64\n\t" - "vle8.v v11, (%[q83])\n\t" - "vle8.v v12, (%[q80])\n\t" - "vand.vi v1, v1, 0xF\n\t" - "addi %[q83], %[q83], 64\n\t" - "vsrl.vi v6, v2, 4\n\t" - "addi %[q80], %[q80], 64\n\t" - "vle8.v v13, (%[q81])\n\t" - "vle8.v v14, (%[q82])\n\t" - "vand.vi v2, v2, 0xF\n\t" - "addi %[q81], %[q81], 64\n\t" - "vsrl.vi v7, v3, 4\n\t" - "addi %[q82], %[q82], 64\n\t" - "vwmul.vv v16, v0, v8\n\t" - "vle8.v v15, (%[q83])\n\t" - "vle8.v v0, (%[q40])\n\t" - "vand.vi v3, v3, 0xF\n\t" - "addi %[q83], %[q83], 64\n\t" - "vwmul.vv v24, v2, v12\n\t" - "vwmul.vv v20, v4, v10\n\t" - "vwmul.vv v28, v6, v14\n\t" - "vwmacc.vv v16, v1, v9\n\t" - "vle8.v v1, (%[q41])\n\t" - "vle8.v v2, (%[q42])\n\t" - "vwmacc.vv v24, v3, v13\n\t" - "vwmacc.vv v20, v5, v11\n\t" - "vwmacc.vv v28, v7, v15\n\t" - "addi %[q40], %[q80], 64\n\t" - "addi %[q41], %[q81], 64\n\t" - "vle8.v v3, (%[q43])\n\t" - "vle8.v v8, (%[q80])\n\t" - "addi %[q42], %[q82], 64\n\t" - "addi %[q43], %[q83], 64\n\t" - "vsrl.vi v4, v0, 4\n\t" - "vle8.v v9, (%[q81])\n\t" - "vle8.v v10, (%[q82])\n\t" - "vand.vi v0, v0, 0xF\n\t" - "vsrl.vi v5, v1, 4\n\t" - "vsrl.vi v7, v3, 4\n\t" - "vand.vi v3, v3, 0xF\n\t" - "vle8.v v11, (%[q83])\n\t" - "vle8.v v12, (%[q40])\n\t" - "vand.vi v1, v1, 0xF\n\t" - "vsrl.vi v6, v2, 4\n\t" - "vand.vi v2, v2, 0xF\n\t" - "vwmul.vv v18, v0, v8\n\t" - "vle8.v v13, (%[q41])\n\t" - "vle8.v v14, (%[q42])\n\t" - "vwmul.vv v26, v2, v12\n\t" - "vwmul.vv v22, v4, v10\n\t" - "vwmul.vv v30, v6, v14\n\t" - "vwmacc.vv v18, v1, v9\n\t" - "vle8.v v15, (%[q43])\n\t" - "vwmacc.vv v26, v3, v13\n\t" - "vwmacc.vv v22, v5, v11\n\t" - "vwmacc.vv v30, v7, v15\n\t" - "vmv.v.x v0, zero\n\t" - "vsetivli zero, 16, e16, m2, ta, ma\n\t" - "vwredsum.vs v4, v16, v0\n\t" - "lbu %[s0], 0(%[scale])\n\t" - "vwredsum.vs v5, v20, v0\n\t" - "lbu %[s1], 1(%[scale])\n\t" - "vwredsum.vs v6, v24, v0\n\t" - "lbu %[s2], 2(%[scale])\n\t" - "vwredsum.vs v7, v28, v0\n\t" - "lbu %[s3], 3(%[scale])\n\t" - "vwredsum.vs v8, v18, v0\n\t" - "lbu %[q40], 4(%[scale])\n\t" - "vwredsum.vs v9, v22, v0\n\t" - "lbu %[q41], 5(%[scale])\n\t" - "vwredsum.vs v10, v26, v0\n\t" - "lbu %[q42], 6(%[scale])\n\t" - "vwredsum.vs v11, v30, v0\n\t" - "lbu %[q43], 7(%[scale])\n\t" - "vsetivli zero, 4, e32, m1, ta, ma\n\t" - "vmul.vx v0, v4, %[s0]\n\t" - "vmul.vx v1, v8, %[q40]\n\t" - "vmacc.vx v0, %[s1], v5\n\t" - "vmacc.vx v1, %[q41], v9\n\t" - "vmacc.vx v0, %[s2], v6\n\t" - "vmacc.vx v1, %[q42], v10\n\t" - "vmacc.vx v0, %[s3], v7\n\t" - "vmacc.vx v1, %[q43], v11\n\t" - "vfcvt.f.x.v v0, v0\n\t" - "vfcvt.f.x.v v1, v1\n\t" - "vfmv.f.s %[ft2], v0\n\t" - "vfmv.f.s %[ftmp], v1\n\t" - "fadd.s %[ft2], %[ft2], %[ftmp]\n\t" - "fmadd.s %[sumf], %[d], %[ft2], %[sumf]" - : [ftmp] "=&f" (ftmp), [sumf] "+&f" (sumf), [ft2] "=&f" (ft2) - , [s0] "=&r" (s0), [s1] "=&r" (s1), [s2] "=&r" (s2), [s3] "=&r" (s3) - , [q40] "=&r" (q40), [q41] "=&r" (q41), [q42] "=&r" (q42), [q43] "=&r" (q43) - , [q80] "=&r" (q80), [q81] "=&r" (q81), [q82] "=&r" (q82), [q83] "=&r" (q83) - : [d] "f" (d), [ys] "r" (y[i].qs), [xs] "r" (x[i].qs), [scale] "r" (scales) - , [bsums] "r" (y[i].bsums), [mins] "r" (mins), [utmp] "r" (utmp) - , [s6b] "r" (&x[i]), [kmask1] "r" (kmask1), [dmin] "f" (dmin) - , [kmask2] "r" (kmask2), [kmask3] "r" (kmask3) - : "memory" - , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" - , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" - , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" - , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" - ); } - break; - default: - assert(false && "Unsupported vector length"); - break; + + sumf += d*(sum_1 + sum_2); + } *s = sumf; +} +#endif +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined __riscv_xtheadvector + ggml_vec_dot_q4_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc); +#elif defined __riscv_v + switch (__riscv_vlenb() * 8) { + case 128: + ggml_vec_dot_q4_K_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + break; + default: // 256 and above + ggml_vec_dot_q4_K_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + break; + } #else - - UNUSED(x); - UNUSED(y); - UNUSED(kmask1); - UNUSED(kmask2); - UNUSED(kmask3); - UNUSED(nb); - UNUSED(utmp); - ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); #endif } @@ -1725,7 +2174,6 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi aux32 += __riscv_vmv_x_s_i32m1_i32(vacc2); q5 += 32; q8 += 64; - } sums += aux32 * d; @@ -1748,7 +2196,8 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined __riscv_xtheadvector +static NOINLINE void ggml_vec_dot_q6_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -1761,8 +2210,6 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi const int nb = n / QK_K; -#if defined __riscv_xtheadvector - float sumf = 0; for (int i = 0; i < nb; ++i) { @@ -1841,224 +2288,462 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi } *s = sumf; +} +#endif -#elif defined __riscv_v +#if defined __riscv_v +static NOINLINE void ggml_vec_dot_q6_K_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q6_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + float sumf = 0.0f; + for (int i = 0; i < nb; ++i) { + __builtin_prefetch(&x[i + 1].d, 0, 1); + + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint8_t * restrict q6 = x[i].ql; + const uint8_t * restrict qh = x[i].qh; + const int8_t * restrict q8 = y[i].qs; + + const int8_t * restrict scale = x[i].scales; + + int q6h; + float ftmp; + + for (int j = 0; j < QK_K/128; ++j) { + __asm__ __volatile__( + "addi %[q6h], %[q6], 32\n\t" + "ld t0, 0(%[scale])\n\t" + "addi %[scale], %[scale], 8\n\t" + "slli t6, t0, 1 * 8\n\t" + "lb zero, 0(%[q6])\n\t" + "slli t5, t0, 2 * 8\n\t" + "slli t4, t0, 3 * 8\n\t" + "lb zero, 0(%[q6h])\n\t" + "slli t3, t0, 4 * 8\n\t" + "slli t2, t0, 5 * 8\n\t" + "lb zero, 0(%[qh])\n\t" + "lb zero, 31(%[q6h])\n\t" + "slli t1, t0, 6 * 8\n\t" + "srai a7, t0, 56\n\t" + "vsetvli zero, %[vl32], e8, m2\n\t" + "vle8.v v8, (%[q6])\n\t" + "srai t6, t6, 56\n\t" + "srai t5, t5, 56\n\t" + "srai t4, t4, 56\n\t" + "srai t3, t3, 56\n\t" + "vle8.v v10, (%[q6h])\n\t" + "addi %[q6], %[q6], 64\n\t" + "slli t0, t0, 7 * 8\n\t" + "srai t2, t2, 56\n\t" + "srai t1, t1, 56\n\t" + "srai t0, t0, 56\n\t" + "vle8.v v4, (%[qh])\n\t" + "vsrl.vi v12, v8, 4\n\t" + "vsrl.vi v14, v10, 4\n\t" + "lb zero, 0(%[q8])\n\t" + "vand.vi v8, v8, 0xF\n\t" + "vand.vi v10, v10, 0xF\n\t" + "lb zero, 32(%[q8])\n\t" + "vsll.vi v0, v4, 4\n\t" + "vsll.vi v2, v4, 2\n\t" + "lb zero, 64(%[q8])\n\t" + "vsrl.vi v6, v4, 2\n\t" + "vand.vx v0, v0, %[mask]\n\t" + "lb zero, 96(%[q8])\n\t" + "vand.vx v2, v2, %[mask]\n\t" + "vand.vx v4, v4, %[mask]\n\t" + "vand.vx v6, v6, %[mask]\n\t" + "vor.vv v8, v8, v0\n\t" + "lb zero, 127(%[q8])\n\t" + "vor.vv v10, v10, v2\n\t" + "vor.vv v12, v12, v4\n\t" + "vor.vv v14, v14, v6\n\t" + "vsetvli zero, %[vl128], e8, m8\n\t" + "vle8.v v0, (%[q8])\n\t" + "vsub.vx v8, v8, %[vl32]\n\t" + "vsetvli zero, %[vl64], e8, m4\n\t" + "vwmul.vv v16, v0, v8\n\t" + "vwmul.vv v24, v4, v12\n\t" + "vsetivli zero, 16, e16, m2\n\t" + "vmv.v.x v0, zero\n\t" + "vwredsum.vs v10, v16, v0\n\t" + "vwredsum.vs v9, v18, v0\n\t" + "vwredsum.vs v8, v20, v0\n\t" + "vwredsum.vs v7, v22, v0\n\t" + "vwredsum.vs v11, v24, v0\n\t" + "vwredsum.vs v12, v26, v0\n\t" + "vwredsum.vs v13, v28, v0\n\t" + "vwredsum.vs v14, v30, v0\n\t" + "vsetivli zero, 4, e32, m1\n\t" + "vmul.vx v0, v10, t0\n\t" + "vmul.vx v1, v9, t1\n\t" + "vmacc.vx v0, t2, v8\n\t" + "vmacc.vx v1, t3, v7\n\t" + "vmacc.vx v0, t4, v11\n\t" + "vmacc.vx v1, t5, v12\n\t" + "vmacc.vx v0, t6, v13\n\t" + "vmacc.vx v1, a7, v14\n\t" + "vadd.vv v0, v0, v1\n\t" + "vfcvt.f.x.v v0, v0\n\t" + "vfmv.f.s %[ftmp], v0\n\t" + "fmadd.s %[sumf], %[d], %[ftmp], %[sumf]" + : [q6] "+&r" (q6), [q6h] "=&r" (q6h) + , [scale] "+&r" (scale) + , [sumf] "+&f" (sumf), [ftmp] "=&f" (ftmp) + : [qh] "r" (qh), [q8] "r" (q8) + , [vl32] "r" (32), [vl64] "r" (64), [vl128] "r" (128) + , [mask] "r" (0x30), [d] "f" (d) + : "memory" + , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" + , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" + , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" + , "t0", "t1", "t2", "t3", "t4", "t5", "t6", "a7" + , "a6", "a5", "a4", "a3" + ); + qh += 32; q8 += 128; + } + } + + *s = sumf; +} + +static NOINLINE void ggml_vec_dot_q6_K_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q6_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + float sumf = 0; + for (int i = 0; i < nb; ++i) { + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint8_t * GGML_RESTRICT q6 = x[i].ql; + const uint8_t * GGML_RESTRICT qh = x[i].qh; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + const int8_t * GGML_RESTRICT scale = x[i].scales; + + size_t vl; + + vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); + + int sum_t = 0; + int is = 0; + + for (int j = 0; j < QK_K/128; ++j) { + + vl = 32; + + // load qh + vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl); + + // load Q6 + vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl); + vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl); + + vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl); + vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl); + vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl); + vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl); + + vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl); + vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl); + vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl); + vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl); + + vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl); + vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl); + vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl); + vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl); + + vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl); + vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl); + vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl); + vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl); + + // load Q8 and take product + vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl); + vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl); + vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl); + vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl); + + vl = 16; + + vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl); + vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl); + vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl); + vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl); + vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl); + vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl); + vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl); + vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl); + + vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl); + vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl); + vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl); + vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl); + + sum_t += __riscv_vmv_x_s_i32m1_i32(isum3); + + q6 += 64; qh += 32; q8 += 128; is=8; + + } + + sumf += d * sum_t; + + } + + *s = sumf; +} + +static NOINLINE void ggml_vec_dot_q6_K_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q6_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + // mask for processing 16 elements per prod register + const vuint16m1_t va_index = __riscv_vid_v_u16m1(32); + const vbool16_t va_mask = __riscv_vmsgtu_vx_u16m1_b16(va_index, 15, 32); float sumf = 0; - const int vector_length = __riscv_vlenb() * 8; - switch (vector_length) { - case 256: - for (int i = 0; i < nb; ++i) { + for (int i = 0; i < nb; ++i) { + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; - const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + const uint8_t * GGML_RESTRICT q6 = x[i].ql; + const uint8_t * GGML_RESTRICT qh = x[i].qh; + const int8_t * GGML_RESTRICT q8 = y[i].qs; - const uint8_t * GGML_RESTRICT q6 = x[i].ql; - const uint8_t * GGML_RESTRICT qh = x[i].qh; - const int8_t * GGML_RESTRICT q8 = y[i].qs; + const int8_t * GGML_RESTRICT scale = x[i].scales; - const int8_t * GGML_RESTRICT scale = x[i].scales; + size_t vl = 32; - size_t vl; + vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); + + int sum_t = 0; + int is = 0; + + vint32m2_t vaux_0 = __riscv_vmv_v_x_i32m2(0, vl); + vint32m2_t vaux_1 = __riscv_vmv_v_x_i32m2(0, vl); + vint32m2_t vaux_2 = __riscv_vmv_v_x_i32m2(0, vl); + vint32m2_t vaux_3 = __riscv_vmv_v_x_i32m2(0, vl); + + for (int j = 0; j < QK_K/128; ++j) { + // load qh + vuint8mf2_t qh_x = __riscv_vle8_v_u8mf2(qh, vl); + + // load Q6 + vuint8mf2_t q6_0 = __riscv_vle8_v_u8mf2(q6, vl); + vuint8mf2_t q6_1 = __riscv_vle8_v_u8mf2(q6+32, vl); + + vuint8mf2_t q6a_0 = __riscv_vand_vx_u8mf2(q6_0, 0x0F, vl); + vuint8mf2_t q6a_1 = __riscv_vand_vx_u8mf2(q6_1, 0x0F, vl); + vuint8mf2_t q6s_0 = __riscv_vsrl_vx_u8mf2(q6_0, 0x04, vl); + vuint8mf2_t q6s_1 = __riscv_vsrl_vx_u8mf2(q6_1, 0x04, vl); + + vuint8mf2_t qh_0 = __riscv_vand_vx_u8mf2(qh_x, 0x03, vl); + vuint8mf2_t qh_1 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x2, vl), 0x03 , vl); + vuint8mf2_t qh_2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), 0x03 , vl); + vuint8mf2_t qh_3 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x6, vl), 0x03 , vl); + + vuint8mf2_t qhi_0 = __riscv_vor_vv_u8mf2(q6a_0, __riscv_vsll_vx_u8mf2(qh_0, 0x04, vl), vl); + vuint8mf2_t qhi_1 = __riscv_vor_vv_u8mf2(q6a_1, __riscv_vsll_vx_u8mf2(qh_1, 0x04, vl), vl); + vuint8mf2_t qhi_2 = __riscv_vor_vv_u8mf2(q6s_0, __riscv_vsll_vx_u8mf2(qh_2, 0x04, vl), vl); + vuint8mf2_t qhi_3 = __riscv_vor_vv_u8mf2(q6s_1, __riscv_vsll_vx_u8mf2(qh_3, 0x04, vl), vl); + + vint8mf2_t a_0 = __riscv_vsub_vx_i8mf2(__riscv_vreinterpret_v_u8mf2_i8mf2(qhi_0), 32, vl); + vint8mf2_t a_1 = __riscv_vsub_vx_i8mf2(__riscv_vreinterpret_v_u8mf2_i8mf2(qhi_1), 32, vl); + vint8mf2_t a_2 = __riscv_vsub_vx_i8mf2(__riscv_vreinterpret_v_u8mf2_i8mf2(qhi_2), 32, vl); + vint8mf2_t a_3 = __riscv_vsub_vx_i8mf2(__riscv_vreinterpret_v_u8mf2_i8mf2(qhi_3), 32, vl); + + // load Q8 and take product + vint16m1_t va_q_0 = __riscv_vwmul_vv_i16m1(a_0, __riscv_vle8_v_i8mf2(q8, vl), vl); + vint16m1_t va_q_1 = __riscv_vwmul_vv_i16m1(a_1, __riscv_vle8_v_i8mf2(q8+32, vl), vl); + vint16m1_t va_q_2 = __riscv_vwmul_vv_i16m1(a_2, __riscv_vle8_v_i8mf2(q8+64, vl), vl); + vint16m1_t va_q_3 = __riscv_vwmul_vv_i16m1(a_3, __riscv_vle8_v_i8mf2(q8+96, vl), vl); + + // accumulate + vaux_0 = __riscv_vwmacc_vx_i32m2(vaux_0, scale[is+0], va_q_0, 16); + vaux_1 = __riscv_vwmacc_vx_i32m2(vaux_1, scale[is+2], va_q_1, 16); + vaux_2 = __riscv_vwmacc_vx_i32m2(vaux_2, scale[is+4], va_q_2, 16); + vaux_3 = __riscv_vwmacc_vx_i32m2(vaux_3, scale[is+6], va_q_3, 16); + // + vaux_0 = __riscv_vwmacc_vx_i32m2_m(va_mask, vaux_0, scale[is+1], va_q_0, vl); + vaux_1 = __riscv_vwmacc_vx_i32m2_m(va_mask, vaux_1, scale[is+3], va_q_1, vl); + vaux_2 = __riscv_vwmacc_vx_i32m2_m(va_mask, vaux_2, scale[is+5], va_q_2, vl); + vaux_3 = __riscv_vwmacc_vx_i32m2_m(va_mask, vaux_3, scale[is+7], va_q_3, vl); + + q6 += 64; qh += 32; q8 += 128; is=8; + } + + vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl); + vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl); + + sum_t += __riscv_vmv_x_s_i32m1_i32(isum1); + + sumf += d * sum_t; - vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); + } - int sum_t = 0; - int is = 0; + *s = sumf; +} + +static NOINLINE void ggml_vec_dot_q6_K_q8_K_vl1024(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q6_K * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; - for (int j = 0; j < QK_K/128; ++j) { + const int nb = n / QK_K; - vl = 32; + // mask for processing 16 elements per prod register + const vuint16mf2_t va_index = __riscv_vid_v_u16mf2(32); + const vbool32_t va_mask = __riscv_vmsgtu_vx_u16mf2_b32(va_index, 15, 32); - // load qh - vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl); + float sumf = 0; - // load Q6 - vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl); - vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl); + for (int i = 0; i < nb; ++i) { + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; - vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl); - vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl); - vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl); - vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl); + const uint8_t * GGML_RESTRICT q6 = x[i].ql; + const uint8_t * GGML_RESTRICT qh = x[i].qh; + const int8_t * GGML_RESTRICT q8 = y[i].qs; - vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl); - vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl); - vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl); - vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl); + const int8_t * GGML_RESTRICT scale = x[i].scales; - vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl); - vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl); - vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl); - vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl); + size_t vl = 32; - vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl); - vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl); - vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl); - vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl); + vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); - // load Q8 and take product - vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl); - vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl); - vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl); - vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl); + int sum_t = 0; + int is = 0; - vl = 16; + vint32m1_t vaux_0 = __riscv_vmv_v_x_i32m1(0, vl); + vint32m1_t vaux_1 = __riscv_vmv_v_x_i32m1(0, vl); + vint32m1_t vaux_2 = __riscv_vmv_v_x_i32m1(0, vl); + vint32m1_t vaux_3 = __riscv_vmv_v_x_i32m1(0, vl); - vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl); - vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl); - vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl); - vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl); - vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl); - vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl); - vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl); - vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl); + for (int j = 0; j < QK_K/128; ++j) { + // load qh + vuint8mf4_t qh_x = __riscv_vle8_v_u8mf4(qh, vl); + + // load Q6 + vuint8mf4_t q6_0 = __riscv_vle8_v_u8mf4(q6, vl); + vuint8mf4_t q6_1 = __riscv_vle8_v_u8mf4(q6+32, vl); + + vuint8mf4_t q6a_0 = __riscv_vand_vx_u8mf4(q6_0, 0x0F, vl); + vuint8mf4_t q6a_1 = __riscv_vand_vx_u8mf4(q6_1, 0x0F, vl); + vuint8mf4_t q6s_0 = __riscv_vsrl_vx_u8mf4(q6_0, 0x04, vl); + vuint8mf4_t q6s_1 = __riscv_vsrl_vx_u8mf4(q6_1, 0x04, vl); + + vuint8mf4_t qh_0 = __riscv_vand_vx_u8mf4(qh_x, 0x03, vl); + vuint8mf4_t qh_1 = __riscv_vand_vx_u8mf4(__riscv_vsrl_vx_u8mf4(qh_x, 0x2, vl), 0x03 , vl); + vuint8mf4_t qh_2 = __riscv_vand_vx_u8mf4(__riscv_vsrl_vx_u8mf4(qh_x, 0x4, vl), 0x03 , vl); + vuint8mf4_t qh_3 = __riscv_vand_vx_u8mf4(__riscv_vsrl_vx_u8mf4(qh_x, 0x6, vl), 0x03 , vl); + + vuint8mf4_t qhi_0 = __riscv_vor_vv_u8mf4(q6a_0, __riscv_vsll_vx_u8mf4(qh_0, 0x04, vl), vl); + vuint8mf4_t qhi_1 = __riscv_vor_vv_u8mf4(q6a_1, __riscv_vsll_vx_u8mf4(qh_1, 0x04, vl), vl); + vuint8mf4_t qhi_2 = __riscv_vor_vv_u8mf4(q6s_0, __riscv_vsll_vx_u8mf4(qh_2, 0x04, vl), vl); + vuint8mf4_t qhi_3 = __riscv_vor_vv_u8mf4(q6s_1, __riscv_vsll_vx_u8mf4(qh_3, 0x04, vl), vl); + + vint8mf4_t a_0 = __riscv_vsub_vx_i8mf4(__riscv_vreinterpret_v_u8mf4_i8mf4(qhi_0), 32, vl); + vint8mf4_t a_1 = __riscv_vsub_vx_i8mf4(__riscv_vreinterpret_v_u8mf4_i8mf4(qhi_1), 32, vl); + vint8mf4_t a_2 = __riscv_vsub_vx_i8mf4(__riscv_vreinterpret_v_u8mf4_i8mf4(qhi_2), 32, vl); + vint8mf4_t a_3 = __riscv_vsub_vx_i8mf4(__riscv_vreinterpret_v_u8mf4_i8mf4(qhi_3), 32, vl); + + // load Q8 and take product + vint16mf2_t va_q_0 = __riscv_vwmul_vv_i16mf2(a_0, __riscv_vle8_v_i8mf4(q8, vl), vl); + vint16mf2_t va_q_1 = __riscv_vwmul_vv_i16mf2(a_1, __riscv_vle8_v_i8mf4(q8+32, vl), vl); + vint16mf2_t va_q_2 = __riscv_vwmul_vv_i16mf2(a_2, __riscv_vle8_v_i8mf4(q8+64, vl), vl); + vint16mf2_t va_q_3 = __riscv_vwmul_vv_i16mf2(a_3, __riscv_vle8_v_i8mf4(q8+96, vl), vl); + + // accumulate + vaux_0 = __riscv_vwmacc_vx_i32m1(vaux_0, scale[is+0], va_q_0, 16); + vaux_1 = __riscv_vwmacc_vx_i32m1(vaux_1, scale[is+2], va_q_1, 16); + vaux_2 = __riscv_vwmacc_vx_i32m1(vaux_2, scale[is+4], va_q_2, 16); + vaux_3 = __riscv_vwmacc_vx_i32m1(vaux_3, scale[is+6], va_q_3, 16); + // + vaux_0 = __riscv_vwmacc_vx_i32m1_m(va_mask, vaux_0, scale[is+1], va_q_0, vl); + vaux_1 = __riscv_vwmacc_vx_i32m1_m(va_mask, vaux_1, scale[is+3], va_q_1, vl); + vaux_2 = __riscv_vwmacc_vx_i32m1_m(va_mask, vaux_2, scale[is+5], va_q_2, vl); + vaux_3 = __riscv_vwmacc_vx_i32m1_m(va_mask, vaux_3, scale[is+7], va_q_3, vl); - vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl); - vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl); - vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl); - vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl); + q6 += 64; qh += 32; q8 += 128; is=8; - sum_t += __riscv_vmv_x_s_i32m1_i32(isum3); + } - q6 += 64; qh += 32; q8 += 128; is=8; + vint32m1_t isum0 = __riscv_vredsum_vs_i32m1_i32m1(__riscv_vadd_vv_i32m1(vaux_0, vaux_1, vl), vzero, vl); + vint32m1_t isum1 = __riscv_vredsum_vs_i32m1_i32m1(__riscv_vadd_vv_i32m1(vaux_2, vaux_3, vl), isum0, vl); - } + sum_t += __riscv_vmv_x_s_i32m1_i32(isum1); - sumf += d * sum_t; + sumf += d * sum_t; - } - break; - case 128: - for (int i = 0; i < nb; ++i) { - - __builtin_prefetch(&x[i + 1].d, 0, 1); - - const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; - - const uint8_t * restrict q6 = x[i].ql; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - - const int8_t * restrict scale = x[i].scales; - - int q6h; - float ftmp; - - for (int j = 0; j < QK_K/128; ++j) { - __asm__ __volatile__( - "addi %[q6h], %[q6], 32\n\t" - "ld t0, 0(%[scale])\n\t" - "addi %[scale], %[scale], 8\n\t" - "slli t6, t0, 1 * 8\n\t" - "lb zero, 0(%[q6])\n\t" - "slli t5, t0, 2 * 8\n\t" - "slli t4, t0, 3 * 8\n\t" - "lb zero, 0(%[q6h])\n\t" - "slli t3, t0, 4 * 8\n\t" - "slli t2, t0, 5 * 8\n\t" - "lb zero, 0(%[qh])\n\t" - "lb zero, 31(%[q6h])\n\t" - "slli t1, t0, 6 * 8\n\t" - "srai a7, t0, 56\n\t" - "vsetvli zero, %[vl32], e8, m2\n\t" - "vle8.v v8, (%[q6])\n\t" - "srai t6, t6, 56\n\t" - "srai t5, t5, 56\n\t" - "srai t4, t4, 56\n\t" - "srai t3, t3, 56\n\t" - "vle8.v v10, (%[q6h])\n\t" - "addi %[q6], %[q6], 64\n\t" - "slli t0, t0, 7 * 8\n\t" - "srai t2, t2, 56\n\t" - "srai t1, t1, 56\n\t" - "srai t0, t0, 56\n\t" - "vle8.v v4, (%[qh])\n\t" - "vsrl.vi v12, v8, 4\n\t" - "vsrl.vi v14, v10, 4\n\t" - "lb zero, 0(%[q8])\n\t" - "vand.vi v8, v8, 0xF\n\t" - "vand.vi v10, v10, 0xF\n\t" - "lb zero, 32(%[q8])\n\t" - "vsll.vi v0, v4, 4\n\t" - "vsll.vi v2, v4, 2\n\t" - "lb zero, 64(%[q8])\n\t" - "vsrl.vi v6, v4, 2\n\t" - "vand.vx v0, v0, %[mask]\n\t" - "lb zero, 96(%[q8])\n\t" - "vand.vx v2, v2, %[mask]\n\t" - "vand.vx v4, v4, %[mask]\n\t" - "vand.vx v6, v6, %[mask]\n\t" - "vor.vv v8, v8, v0\n\t" - "lb zero, 127(%[q8])\n\t" - "vor.vv v10, v10, v2\n\t" - "vor.vv v12, v12, v4\n\t" - "vor.vv v14, v14, v6\n\t" - "vsetvli zero, %[vl128], e8, m8\n\t" - "vle8.v v0, (%[q8])\n\t" - "vsub.vx v8, v8, %[vl32]\n\t" - "vsetvli zero, %[vl64], e8, m4\n\t" - "vwmul.vv v16, v0, v8\n\t" - "vwmul.vv v24, v4, v12\n\t" - "vsetivli zero, 16, e16, m2\n\t" - "vmv.v.x v0, zero\n\t" - "vwredsum.vs v10, v16, v0\n\t" - "vwredsum.vs v9, v18, v0\n\t" - "vwredsum.vs v8, v20, v0\n\t" - "vwredsum.vs v7, v22, v0\n\t" - "vwredsum.vs v11, v24, v0\n\t" - "vwredsum.vs v12, v26, v0\n\t" - "vwredsum.vs v13, v28, v0\n\t" - "vwredsum.vs v14, v30, v0\n\t" - "vsetivli zero, 4, e32, m1\n\t" - "vmul.vx v0, v10, t0\n\t" - "vmul.vx v1, v9, t1\n\t" - "vmacc.vx v0, t2, v8\n\t" - "vmacc.vx v1, t3, v7\n\t" - "vmacc.vx v0, t4, v11\n\t" - "vmacc.vx v1, t5, v12\n\t" - "vmacc.vx v0, t6, v13\n\t" - "vmacc.vx v1, a7, v14\n\t" - "vadd.vv v0, v0, v1\n\t" - "vfcvt.f.x.v v0, v0\n\t" - "vfmv.f.s %[ftmp], v0\n\t" - "fmadd.s %[sumf], %[d], %[ftmp], %[sumf]" - : [q6] "+&r" (q6), [q6h] "=&r" (q6h) - , [scale] "+&r" (scale) - , [sumf] "+&f" (sumf), [ftmp] "=&f" (ftmp) - : [qh] "r" (qh), [q8] "r" (q8) - , [vl32] "r" (32), [vl64] "r" (64), [vl128] "r" (128) - , [mask] "r" (0x30), [d] "f" (d) - : "memory" - , "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" - , "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15" - , "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23" - , "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" - , "t0", "t1", "t2", "t3", "t4", "t5", "t6", "a7" - , "a6", "a5", "a4", "a3" - ); - qh += 32; q8 += 128; - } - } - break; - default: - assert(false && "Unsupported vector length"); - break; } *s = sumf; +} +#endif +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined __riscv_xtheadvector + ggml_vec_dot_q6_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc); +#elif defined __riscv_v + switch (__riscv_vlenb() * 8) { + case 128: + ggml_vec_dot_q6_K_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + break; + case 256: + ggml_vec_dot_q6_K_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + break; + case 512: + ggml_vec_dot_q6_K_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); + break; + case 1024: + ggml_vec_dot_q6_K_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); + break; + default: + ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + break; + } #else - - UNUSED(x); - UNUSED(y); - UNUSED(nb); - ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -2266,10 +2951,190 @@ static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl256(int n, float * GGML_RESTRICT *s = sumf; } + +static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq1_s * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + float sumf = 0; + for (int i = 0; i < nb; ++i) { + // Load qh once for the entire superblock. + vuint16mf4_t qh = __riscv_vle16_v_u16mf4(x[i].qh, 8); + + // Calculate ls. + vuint16mf4_t temp = __riscv_vsrl_vx_u16mf4(qh, 12, 8); + temp = __riscv_vand_vx_u16mf4(temp, 7, 8); + vint32mf2_t ls = __riscv_vreinterpret_v_u32mf2_i32mf2(__riscv_vwmulu_vx_u32mf2(temp, 2, 8)); + ls = __riscv_vadd_vx_i32mf2(ls, 1, 8); + + // Calculate delta. + vbool64_t mask = __riscv_vmseq_vx_u16mf4_b64(__riscv_vand_vx_u16mf4(qh, 0x8000, 8), 0, 8); + vint32mf2_t delta_neg = __riscv_vmv_v_x_i32mf2(-1, 8); + vint32mf2_t delta_pos = __riscv_vmv_v_x_i32mf2(1, 8); + vint32mf2_t delta = __riscv_vmerge_vvm_i32mf2(delta_neg, delta_pos, mask, 8); + + // Load qs. + vuint8mf2_t qs = __riscv_vle8_v_u8mf2(x[i].qs, 32); + + // Prepare the indices. + const uint64_t shift = 0x0009000600030000; + vuint16m1_t qh_shift = __riscv_vreinterpret_v_u64m1_u16m1(__riscv_vmv_v_x_u64m1(shift, 8)); + vuint16m1_t qh_gather_index = __riscv_vreinterpret_v_i16m1_u16m1( + __riscv_vdiv_vx_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vid_v_u16m1(32)), 4, 32)); + vuint16m1_t qh_ext = __riscv_vlmul_ext_v_u16mf2_u16m1(__riscv_vlmul_ext_v_u16mf4_u16mf2(qh)); + vuint16m1_t qh_index = __riscv_vrgather_vv_u16m1(qh_ext, qh_gather_index, 32); + qh_index = __riscv_vsrl_vv_u16m1(qh_index, qh_shift, 32); + qh_index = __riscv_vand_vx_u16m1(qh_index, 7, 32); + qh_index = __riscv_vsll_vx_u16m1(qh_index, 8, 32); + qh_index = __riscv_vor_vv_u16m1(qh_index, __riscv_vzext_vf2_u16m1(qs, 32), 32); + vuint16m1_t index = __riscv_vsll_vx_u16m1(qh_index, 3, 32); + + // Final lsums. + int32_t lsums_s[8]; + vint32m1_t one_scalar = __riscv_vmv_v_x_i32m1(0, 1); + + // Sub-blocks 1-8 + { + vint8m4_t grid0 = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vluxei16_v_i64m4((const int64_t*)iq1s_grid, index, 32)); + vint8m4_t q80 = __riscv_vle8_v_i8m4(y[i].qs, 256); + vint16m8_t lsum0 = __riscv_vwmul_vv_i16m8(grid0, q80, 256); + lsums_s[0] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(lsum0, 0), one_scalar, 32)); + lsums_s[1] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(lsum0, 1), one_scalar, 32)); + lsums_s[2] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(lsum0, 2), one_scalar, 32)); + lsums_s[3] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(lsum0, 3), one_scalar, 32)); + lsums_s[4] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(lsum0, 4), one_scalar, 32)); + lsums_s[5] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(lsum0, 5), one_scalar, 32)); + lsums_s[6] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(lsum0, 6), one_scalar, 32)); + lsums_s[7] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(lsum0, 7), one_scalar, 32)); + } + __asm__ __volatile__("" ::: "memory"); + vint32mf2_t lsums = __riscv_vle32_v_i32mf2(&lsums_s[0], 8); + + // Calculate the bsums. + vint16mf2_t bsums_0 = __riscv_vle16_v_i16mf2(y[i].bsums, 16); + const vuint32mf2_t bsums_i32 = __riscv_vreinterpret_v_u16mf2_u32mf2(__riscv_vreinterpret_v_i16mf2_u16mf2(bsums_0)); + const vint16mf4_t bsums_i32_0 = __riscv_vreinterpret_v_u16mf4_i16mf4(__riscv_vnsrl_wx_u16mf4(bsums_i32, 0, 8)); + const vint16mf4_t bsums_i32_1 = __riscv_vreinterpret_v_u16mf4_i16mf4(__riscv_vnsrl_wx_u16mf4(bsums_i32, 16, 8)); + const vint32mf2_t bsums = __riscv_vwadd_vv_i32mf2(bsums_i32_0, bsums_i32_1, 8); + + // Accumulation. + vint32mf2_t sumi_v = __riscv_vmul_vv_i32mf2(ls, lsums, 8); + vint32mf2_t sumi1_v = __riscv_vmul_vv_i32mf2(__riscv_vmul_vv_i32mf2(ls, delta, 8), bsums, 8); + + // Update sumf. + int sumi = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32mf2_i32m1(sumi_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8)); + int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32mf2_i32m1(sumi1_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8)); + sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1); + } + + *s = sumf; +} + +static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl1024(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq1_s * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + // Mask for processing 32 elements per lsum register. + vuint16m1_t l_index = __riscv_vid_v_u16m1(64); + vbool16_t l_mask = __riscv_vmsgtu_vx_u16m1_b16(l_index, 31, 64); + + float sumf = 0; + for (int i = 0; i < nb; ++i) { + // Load qh once for the entire superblock. + vuint16mf4_t qh = __riscv_vle16_v_u16mf4(x[i].qh, 8); + + // Calculate ls. + vuint16mf4_t temp = __riscv_vsrl_vx_u16mf4(qh, 12, 8); + temp = __riscv_vand_vx_u16mf4(temp, 7, 8); + vint32mf2_t ls = __riscv_vreinterpret_v_u32mf2_i32mf2(__riscv_vwmulu_vx_u32mf2(temp, 2, 8)); + ls = __riscv_vadd_vx_i32mf2(ls, 1, 8); + + // Calculate delta. + vbool64_t mask = __riscv_vmseq_vx_u16mf4_b64(__riscv_vand_vx_u16mf4(qh, 0x8000, 8), 0, 8); + vint32mf2_t delta_neg = __riscv_vmv_v_x_i32mf2(-1, 8); + vint32mf2_t delta_pos = __riscv_vmv_v_x_i32mf2(1, 8); + vint32mf2_t delta = __riscv_vmerge_vvm_i32mf2(delta_neg, delta_pos, mask, 8); + + // Load qs. + vuint8mf2_t qs = __riscv_vle8_v_u8mf2(x[i].qs, 32); + + // Prepare the indices. + const uint64_t shift = 0x0009000600030000; + vuint16m1_t qh_shift = __riscv_vreinterpret_v_u64m1_u16m1(__riscv_vmv_v_x_u64m1(shift, 8)); + vuint16m1_t qh_gather_index = __riscv_vreinterpret_v_i16m1_u16m1( + __riscv_vdiv_vx_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vid_v_u16m1(32)), 4, 32)); + vuint16m1_t qh_ext = __riscv_vlmul_ext_v_u16mf2_u16m1(__riscv_vlmul_ext_v_u16mf4_u16mf2(qh)); + vuint16m1_t qh_index = __riscv_vrgather_vv_u16m1(qh_ext, qh_gather_index, 32); + qh_index = __riscv_vsrl_vv_u16m1(qh_index, qh_shift, 32); + qh_index = __riscv_vand_vx_u16m1(qh_index, 7, 32); + qh_index = __riscv_vsll_vx_u16m1(qh_index, 8, 32); + qh_index = __riscv_vor_vv_u16m1(qh_index, __riscv_vzext_vf2_u16m1(qs, 32), 32); + vuint16mf2_t index = __riscv_vlmul_trunc_v_u16m1_u16mf2(__riscv_vsll_vx_u16m1(qh_index, 3, 32)); + + // Final lsums. + int32_t lsums_s[8]; + vint32m1_t one_scalar = __riscv_vmv_v_x_i32m1(0, 1); + + // Sub-blocks 1-8 + { + vint8m2_t grid0 = __riscv_vreinterpret_v_i64m2_i8m2(__riscv_vluxei16_v_i64m2((const int64_t*)iq1s_grid, index, 32)); + vint8m2_t q80 = __riscv_vle8_v_i8m2(y[i].qs, 256); + vint16m4_t lsum0 = __riscv_vwmul_vv_i16m4(grid0, q80, 256); + + // Reduce. + lsums_s[0] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( __riscv_vget_v_i16m4_i16m1(lsum0, 0), one_scalar, 32)); + lsums_s[1] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(l_mask, __riscv_vget_v_i16m4_i16m1(lsum0, 0), one_scalar, 64)); + lsums_s[2] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( __riscv_vget_v_i16m4_i16m1(lsum0, 1), one_scalar, 32)); + lsums_s[3] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(l_mask, __riscv_vget_v_i16m4_i16m1(lsum0, 1), one_scalar, 64)); + lsums_s[4] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( __riscv_vget_v_i16m4_i16m1(lsum0, 2), one_scalar, 32)); + lsums_s[5] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(l_mask, __riscv_vget_v_i16m4_i16m1(lsum0, 2), one_scalar, 64)); + lsums_s[6] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( __riscv_vget_v_i16m4_i16m1(lsum0, 3), one_scalar, 32)); + lsums_s[7] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(l_mask, __riscv_vget_v_i16m4_i16m1(lsum0, 3), one_scalar, 64)); + } + __asm__ __volatile__("" ::: "memory"); + vint32mf2_t lsums = __riscv_vle32_v_i32mf2(&lsums_s[0], 8); + + // Calculate the bsums. + vint16mf2_t bsums_0 = __riscv_vle16_v_i16mf2(y[i].bsums, 16); + const vuint32mf2_t bsums_i32 = __riscv_vreinterpret_v_u16mf2_u32mf2(__riscv_vreinterpret_v_i16mf2_u16mf2(bsums_0)); + const vint16mf4_t bsums_i32_0 = __riscv_vreinterpret_v_u16mf4_i16mf4(__riscv_vnsrl_wx_u16mf4(bsums_i32, 0, 8)); + const vint16mf4_t bsums_i32_1 = __riscv_vreinterpret_v_u16mf4_i16mf4(__riscv_vnsrl_wx_u16mf4(bsums_i32, 16, 8)); + const vint32mf2_t bsums = __riscv_vwadd_vv_i32mf2(bsums_i32_0, bsums_i32_1, 8); + + // Accumulation. + vint32mf2_t sumi_v = __riscv_vmul_vv_i32mf2(ls, lsums, 8); + vint32mf2_t sumi1_v = __riscv_vmul_vv_i32mf2(__riscv_vmul_vv_i32mf2(ls, delta, 8), bsums, 8); + + // Update sumf. + int sumi = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32mf2_i32m1(sumi_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8)); + int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32mf2_i32m1(sumi1_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8)); + sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1); + } + + *s = sumf; +} #endif void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_iq1_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); @@ -2277,6 +3142,12 @@ void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo case 256: ggml_vec_dot_iq1_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; + case 512: + ggml_vec_dot_iq1_s_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); + break; + case 1024: + ggml_vec_dot_iq1_s_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); + break; default: ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); break; @@ -2286,7 +3157,7 @@ void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -2566,10 +3437,287 @@ static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl256(int n, float * GGML_RESTRICT *s = sumf; } + +static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq1_m * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + iq1m_scale_t scale; + + // Mask for processing 16 elements per lsum register. + const vuint16m1_t l_index = __riscv_vid_v_u16m1(32); + const vbool16_t l_mask = __riscv_vmsgtu_vx_u16m1_b16(l_index, 15, 32); + + float sumf = 0.0f; + for (int i = 0; i < nb; ++i) { + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint8_t * qh = x[i].qh; + const uint16_t * sc = (const uint16_t *)x[i].scales; + + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); + + // Accumulators. + vint32m2_t acc1 = __riscv_vmv_v_x_i32m2(0, 32); + vint32m2_t acc2 = __riscv_vmv_v_x_i32m2(0, 32); + + // We process all the sub-blocks together. + #pragma GCC unroll 1 + for (int ib = 0; ib < QK_K/256; ib++) { + // Load qh for all 16 sub-blocks. + const vuint8mf4_t qh_8 = __riscv_vle8_v_u8mf4(qh, 16); + const vuint16mf2_t qh_16_lo = __riscv_vzext_vf2_u16mf2(qh_8, 16); + const vuint16mf2_t qh_16_hi = __riscv_vsll_vx_u16mf2(qh_16_lo, 8, 16); + const vuint16m1_t qhb = __riscv_vzext_vf2_u16m1( + __riscv_vreinterpret_v_u16mf2_u8mf2(__riscv_vor_vv_u16mf2(qh_16_lo, qh_16_hi, 16)), 32); + __asm__ __volatile__("" ::: "memory"); + + // Prepare grid indices. + const vuint16m1_t qsb = __riscv_vzext_vf2_u16m1(__riscv_vle8_v_u8mf2(&qs[0], 32), 32); + const vuint16m1_t shift = __riscv_vreinterpret_v_u32m1_u16m1(__riscv_vmv_v_x_u32m1(0x00040008, 16)); + vuint16m1_t index = __riscv_vor_vv_u16m1(qsb, __riscv_vand_vx_u16m1(__riscv_vsll_vv_u16m1(qhb, shift, 32), 0x700, 32), 32); + index = __riscv_vsll_vx_u16m1(index, 3, 32); + __asm__ __volatile__("" ::: "memory"); + + // Load the grid. + const vint8m4_t iq1b = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vreinterpret_v_u64m4_i64m4( + __riscv_vluxei16_v_u64m4(iq1s_grid, index, 32))); + + // Prepare the deltas. + const vbool16_t mask = __riscv_vmsgtu_vx_u16m1_b16( + __riscv_vand_vv_u16m1(qhb, __riscv_vreinterpret_v_u32m1_u16m1(__riscv_vmv_v_x_u32m1(0x00800008, 16)), 32), 0, 32); + const vint64m4_t delta_pos = __riscv_vmv_v_x_i64m4(0x0101010101010101, 32); + const vint8m4_t delta = __riscv_vreinterpret_v_i64m4_i8m4( + __riscv_vmerge_vxm_i64m4(delta_pos, 0xffffffffffffffff, mask, 32)); + + // Load q8 for sub-blocks. + const vint8m4_t q8b = __riscv_vle8_v_i8m4(q8, 256); + + // Calculate the lsums. + const vint16m8_t lsum1 = __riscv_vwmul_vv_i16m8(iq1b, q8b, 256); + const vint16m8_t lsum2 = __riscv_vwmul_vv_i16m8(delta, q8b, 256); + + // Prepare the scales. + const int16_t ls_0 = 2*((sc[0] >> 0) & 0x7) + 1; + const int16_t ls_1 = 2*((sc[0] >> 3) & 0x7) + 1; + const int16_t ls_2 = 2*((sc[0] >> 6) & 0x7) + 1; + const int16_t ls_3 = 2*((sc[0] >> 9) & 0x7) + 1; + const int16_t ls_4 = 2*((sc[1] >> 0) & 0x7) + 1; + const int16_t ls_5 = 2*((sc[1] >> 3) & 0x7) + 1; + const int16_t ls_6 = 2*((sc[1] >> 6) & 0x7) + 1; + const int16_t ls_7 = 2*((sc[1] >> 9) & 0x7) + 1; + const int16_t ls_8 = 2*((sc[2] >> 0) & 0x7) + 1; + const int16_t ls_9 = 2*((sc[2] >> 3) & 0x7) + 1; + const int16_t ls_10 = 2*((sc[2] >> 6) & 0x7) + 1; + const int16_t ls_11 = 2*((sc[2] >> 9) & 0x7) + 1; + const int16_t ls_12 = 2*((sc[3] >> 0) & 0x7) + 1; + const int16_t ls_13 = 2*((sc[3] >> 3) & 0x7) + 1; + const int16_t ls_14 = 2*((sc[3] >> 6) & 0x7) + 1; + const int16_t ls_15 = 2*((sc[3] >> 9) & 0x7) + 1; + + // Accumulate in acc0 and acc1 for each sub-block. + acc1 = __riscv_vwmacc_vx_i32m2( acc1, ls_0, __riscv_vget_v_i16m8_i16m1(lsum1, 0), 16); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc1, ls_1, __riscv_vget_v_i16m8_i16m1(lsum1, 0), 32); + acc2 = __riscv_vwmacc_vx_i32m2( acc2, ls_0, __riscv_vget_v_i16m8_i16m1(lsum2, 0), 16); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc2, ls_1, __riscv_vget_v_i16m8_i16m1(lsum2, 0), 32); + // + acc1 = __riscv_vwmacc_vx_i32m2( acc1, ls_2, __riscv_vget_v_i16m8_i16m1(lsum1, 1), 16); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc1, ls_3, __riscv_vget_v_i16m8_i16m1(lsum1, 1), 32); + acc2 = __riscv_vwmacc_vx_i32m2( acc2, ls_2, __riscv_vget_v_i16m8_i16m1(lsum2, 1), 16); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc2, ls_3, __riscv_vget_v_i16m8_i16m1(lsum2, 1), 32); + // + acc1 = __riscv_vwmacc_vx_i32m2( acc1, ls_4, __riscv_vget_v_i16m8_i16m1(lsum1, 2), 16); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc1, ls_5, __riscv_vget_v_i16m8_i16m1(lsum1, 2), 32); + acc2 = __riscv_vwmacc_vx_i32m2( acc2, ls_4, __riscv_vget_v_i16m8_i16m1(lsum2, 2), 16); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc2, ls_5, __riscv_vget_v_i16m8_i16m1(lsum2, 2), 32); + // + acc1 = __riscv_vwmacc_vx_i32m2( acc1, ls_6, __riscv_vget_v_i16m8_i16m1(lsum1, 3), 16); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc1, ls_7, __riscv_vget_v_i16m8_i16m1(lsum1, 3), 32); + acc2 = __riscv_vwmacc_vx_i32m2( acc2, ls_6, __riscv_vget_v_i16m8_i16m1(lsum2, 3), 16); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc2, ls_7, __riscv_vget_v_i16m8_i16m1(lsum2, 3), 32); + // + acc1 = __riscv_vwmacc_vx_i32m2( acc1, ls_8, __riscv_vget_v_i16m8_i16m1(lsum1, 4), 16); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc1, ls_9, __riscv_vget_v_i16m8_i16m1(lsum1, 4), 32); + acc2 = __riscv_vwmacc_vx_i32m2( acc2, ls_8, __riscv_vget_v_i16m8_i16m1(lsum2, 4), 16); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc2, ls_9, __riscv_vget_v_i16m8_i16m1(lsum2, 4), 32); + // + acc1 = __riscv_vwmacc_vx_i32m2( acc1, ls_10, __riscv_vget_v_i16m8_i16m1(lsum1, 5), 16); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc1, ls_11, __riscv_vget_v_i16m8_i16m1(lsum1, 5), 32); + acc2 = __riscv_vwmacc_vx_i32m2( acc2, ls_10, __riscv_vget_v_i16m8_i16m1(lsum2, 5), 16); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc2, ls_11, __riscv_vget_v_i16m8_i16m1(lsum2, 5), 32); + // + acc1 = __riscv_vwmacc_vx_i32m2( acc1, ls_12, __riscv_vget_v_i16m8_i16m1(lsum1, 6), 16); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc1, ls_13, __riscv_vget_v_i16m8_i16m1(lsum1, 6), 32); + acc2 = __riscv_vwmacc_vx_i32m2( acc2, ls_12, __riscv_vget_v_i16m8_i16m1(lsum2, 6), 16); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc2, ls_13, __riscv_vget_v_i16m8_i16m1(lsum2, 6), 32); + // + acc1 = __riscv_vwmacc_vx_i32m2( acc1, ls_14, __riscv_vget_v_i16m8_i16m1(lsum1, 7), 16); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc1, ls_15, __riscv_vget_v_i16m8_i16m1(lsum1, 7), 32); + acc2 = __riscv_vwmacc_vx_i32m2( acc2, ls_14, __riscv_vget_v_i16m8_i16m1(lsum2, 7), 16); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask, acc2, ls_15, __riscv_vget_v_i16m8_i16m1(lsum2, 7), 32); + + __asm__ __volatile__("" ::: "memory"); + } + + // Reduce and accumulate in `sumf`. + vint32m1_t one = __riscv_vmv_v_x_i32m1(0, 1); + int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc1, one, 32)); + int sumi2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc2, one, 32)); + sumf += y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16) * (sumi1 + IQ1M_DELTA * sumi2); + } + + *s = sumf; +} + +static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl1024(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq1_m * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + iq1m_scale_t scale; + float sumf = 0.0f; + for (int i = 0; i < nb; ++i) { + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint8_t * qh = x[i].qh; + const uint16_t * sc = (const uint16_t *)x[i].scales; + + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); + + // Accumulators. + vint32m2_t acc1 = __riscv_vmv_v_x_i32m2(0, 64); + vint32m2_t acc2 = __riscv_vmv_v_x_i32m2(0, 64); + + // We process all the sub-blocks together. + #pragma GCC unroll 1 + for (int ib = 0; ib < QK_K/256; ib++) { + // Load qh for all 16 sub-blocks. + const vuint8mf8_t qh_8 = __riscv_vle8_v_u8mf8(qh, 16); + const vuint16mf4_t qh_16_lo = __riscv_vzext_vf2_u16mf4(qh_8, 16); + const vuint16mf4_t qh_16_hi = __riscv_vsll_vx_u16mf4(qh_16_lo, 8, 16); + const vuint16mf2_t qhb = __riscv_vzext_vf2_u16mf2( + __riscv_vreinterpret_v_u16mf4_u8mf4(__riscv_vor_vv_u16mf4(qh_16_lo, qh_16_hi, 16)), 32); + __asm__ __volatile__("" ::: "memory"); + + // Prepare grid indices. + const vuint16mf2_t qsb = __riscv_vzext_vf2_u16mf2(__riscv_vle8_v_u8mf4(&qs[0], 32), 32); + const vuint16mf2_t shift = __riscv_vreinterpret_v_u32mf2_u16mf2(__riscv_vmv_v_x_u32mf2(0x00040008, 16)); + vuint16mf2_t index = __riscv_vor_vv_u16mf2(qsb, __riscv_vand_vx_u16mf2(__riscv_vsll_vv_u16mf2(qhb, shift, 32), 0x700, 32), 32); + index = __riscv_vsll_vx_u16mf2(index, 3, 32); + __asm__ __volatile__("" ::: "memory"); + + // Load the grid. + const vint8m2_t iq1b = __riscv_vreinterpret_v_i64m2_i8m2(__riscv_vreinterpret_v_u64m2_i64m2( + __riscv_vluxei16_v_u64m2(iq1s_grid, index, 32))); + + // Prepare the deltas. + const vbool32_t mask = __riscv_vmsgtu_vx_u16mf2_b32( + __riscv_vand_vv_u16mf2(qhb, __riscv_vreinterpret_v_u32mf2_u16mf2(__riscv_vmv_v_x_u32mf2(0x00800008, 16)), 32), 0, 32); + const vint64m2_t delta_pos = __riscv_vmv_v_x_i64m2(0x0101010101010101, 32); + const vint8m2_t delta = __riscv_vreinterpret_v_i64m2_i8m2( + __riscv_vmerge_vxm_i64m2(delta_pos, 0xffffffffffffffff, mask, 32)); + + // Load q8 for sub-blocks. + const vint8m2_t q8b = __riscv_vle8_v_i8m2(q8, 256); + + // Calculate the lsums. + const vint16m4_t lsum1 = __riscv_vwmul_vv_i16m4(iq1b, q8b, 256); + const vint16m4_t lsum2 = __riscv_vwmul_vv_i16m4(delta, q8b, 256); + + // Prepare the scales. + const int16_t ls_0 = 2*((sc[0] >> 0) & 0x7) + 1; + const int16_t ls_1 = 2*((sc[0] >> 3) & 0x7) + 1; + const int16_t ls_2 = 2*((sc[0] >> 6) & 0x7) + 1; + const int16_t ls_3 = 2*((sc[0] >> 9) & 0x7) + 1; + const int16_t ls_4 = 2*((sc[1] >> 0) & 0x7) + 1; + const int16_t ls_5 = 2*((sc[1] >> 3) & 0x7) + 1; + const int16_t ls_6 = 2*((sc[1] >> 6) & 0x7) + 1; + const int16_t ls_7 = 2*((sc[1] >> 9) & 0x7) + 1; + const int16_t ls_8 = 2*((sc[2] >> 0) & 0x7) + 1; + const int16_t ls_9 = 2*((sc[2] >> 3) & 0x7) + 1; + const int16_t ls_10 = 2*((sc[2] >> 6) & 0x7) + 1; + const int16_t ls_11 = 2*((sc[2] >> 9) & 0x7) + 1; + const int16_t ls_12 = 2*((sc[3] >> 0) & 0x7) + 1; + const int16_t ls_13 = 2*((sc[3] >> 3) & 0x7) + 1; + const int16_t ls_14 = 2*((sc[3] >> 6) & 0x7) + 1; + const int16_t ls_15 = 2*((sc[3] >> 9) & 0x7) + 1; + + // Mask for processing 16 elements per lsum register. + const vuint16m1_t l_index = __riscv_vid_v_u16m1(64); + + // Accumulate in acc1 and acc2 for each sub-block. + acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_0, __riscv_vget_v_i16m4_i16m1(lsum1, 0), 16); + acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_0, __riscv_vget_v_i16m4_i16m1(lsum2, 0), 16); + acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_4, __riscv_vget_v_i16m4_i16m1(lsum1, 1), 16); + acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_4, __riscv_vget_v_i16m4_i16m1(lsum2, 1), 16); + acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_8, __riscv_vget_v_i16m4_i16m1(lsum1, 2), 16); + acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_8, __riscv_vget_v_i16m4_i16m1(lsum2, 2), 16); + acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_12, __riscv_vget_v_i16m4_i16m1(lsum1, 3), 16); + acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_12, __riscv_vget_v_i16m4_i16m1(lsum2, 3), 16); + // + const vbool16_t l_mask_16_32 = __riscv_vmsgtu_vx_u16m1_b16(l_index, 15, 64); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_16_32, acc1, ls_1, __riscv_vget_v_i16m4_i16m1(lsum1, 0), 32); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_16_32, acc2, ls_1, __riscv_vget_v_i16m4_i16m1(lsum2, 0), 32); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_16_32, acc1, ls_5, __riscv_vget_v_i16m4_i16m1(lsum1, 1), 32); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_16_32, acc2, ls_5, __riscv_vget_v_i16m4_i16m1(lsum2, 1), 32); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_16_32, acc1, ls_9, __riscv_vget_v_i16m4_i16m1(lsum1, 2), 32); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_16_32, acc2, ls_9, __riscv_vget_v_i16m4_i16m1(lsum2, 2), 32); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_16_32, acc1, ls_13, __riscv_vget_v_i16m4_i16m1(lsum1, 3), 32); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_16_32, acc2, ls_13, __riscv_vget_v_i16m4_i16m1(lsum2, 3), 32); + // + const vbool16_t l_mask_32_48 = __riscv_vmsgtu_vx_u16m1_b16(l_index, 31, 64); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_32_48, acc1, ls_2, __riscv_vget_v_i16m4_i16m1(lsum1, 0), 48); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_32_48, acc2, ls_2, __riscv_vget_v_i16m4_i16m1(lsum2, 0), 48); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_32_48, acc1, ls_6, __riscv_vget_v_i16m4_i16m1(lsum1, 1), 48); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_32_48, acc2, ls_6, __riscv_vget_v_i16m4_i16m1(lsum2, 1), 48); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_32_48, acc1, ls_10, __riscv_vget_v_i16m4_i16m1(lsum1, 2), 48); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_32_48, acc2, ls_10, __riscv_vget_v_i16m4_i16m1(lsum2, 2), 48); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_32_48, acc1, ls_14, __riscv_vget_v_i16m4_i16m1(lsum1, 3), 48); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_32_48, acc2, ls_14, __riscv_vget_v_i16m4_i16m1(lsum2, 3), 48); + // + const vbool16_t l_mask_48_64 = __riscv_vmsgtu_vx_u16m1_b16(l_index, 47, 64); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_48_64, acc1, ls_3, __riscv_vget_v_i16m4_i16m1(lsum1, 0), 64); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_48_64, acc2, ls_3, __riscv_vget_v_i16m4_i16m1(lsum2, 0), 64); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_48_64, acc1, ls_7, __riscv_vget_v_i16m4_i16m1(lsum1, 1), 64); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_48_64, acc2, ls_7, __riscv_vget_v_i16m4_i16m1(lsum2, 1), 64); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_48_64, acc1, ls_11, __riscv_vget_v_i16m4_i16m1(lsum1, 2), 64); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_48_64, acc2, ls_11, __riscv_vget_v_i16m4_i16m1(lsum2, 2), 64); + acc1 = __riscv_vwmacc_vx_i32m2_m(l_mask_48_64, acc1, ls_15, __riscv_vget_v_i16m4_i16m1(lsum1, 3), 64); + acc2 = __riscv_vwmacc_vx_i32m2_m(l_mask_48_64, acc2, ls_15, __riscv_vget_v_i16m4_i16m1(lsum2, 3), 64); + + __asm__ __volatile__("" ::: "memory"); + } + + // Reduce and accumulate in `sumf`. + vint32m1_t one = __riscv_vmv_v_x_i32m1(0, 1); + int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc1, one, 64)); + int sumi2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc2, one, 64)); + sumf += y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16) * (sumi1 + IQ1M_DELTA * sumi2); + } + + *s = sumf; +} #endif void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_iq1_m_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); @@ -2577,6 +3725,12 @@ void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo case 256: ggml_vec_dot_iq1_m_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; + case 512: + ggml_vec_dot_iq1_m_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); + break; + case 1024: + ggml_vec_dot_iq1_m_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); + break; default: ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); break; @@ -2586,7 +3740,7 @@ void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static const uint8_t sign_gather_indices_arr[64] = { 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 5,5,5,5,5,5,5,5, 6,6,6,6,6,6,6,6, 7,7,7,7,7,7,7,7 @@ -2789,10 +3943,275 @@ static NOINLINE void ggml_vec_dot_iq2_s_q8_K_vl256(int n, float * GGML_RESTRICT } *s = 0.125f * sumf; } + +static NOINLINE void ggml_vec_dot_iq2_s_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + + const block_iq2_s * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + const uint64_t * grid64 = (const uint64_t *)iq2s_grid; + + vuint8m2_t v_ids = __riscv_vid_v_u8m2(128); + vuint8m2_t v_sign_gather_indices = __riscv_vsrl_vx_u8m2(v_ids, 3, 128); + + vuint8m2_t v_ones = __riscv_vmv_v_x_u8m2(1, 128); + vuint8m2_t v_shift_amts = __riscv_vand_vx_u8m2(v_ids, 7, 128); + vuint8m2_t v_sign_masks = __riscv_vsll_vv_u8m2(v_ones, v_shift_amts, 128); + + uint16_t gather_qh_arr[16] = {0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3}; + vuint16mf2_t v_gather_qh = __riscv_vle16_v_u16mf2(gather_qh_arr, 16); + + uint16_t shift_qh_arr[16] = {11, 9, 7, 5, 11, 9, 7, 5, 11, 9, 7, 5, 11, 9, 7, 5}; + vuint16mf2_t v_shift_qh = __riscv_vle16_v_u16mf2(shift_qh_arr, 16); + + // Masks for selecting lower/upper 16 lanes within a 32-lane i16m1 register + vuint16m1_t v_ids16 = __riscv_vid_v_u16m1(32); + vbool16_t m_hi16 = __riscv_vmsgeu_vx_u16m1_b16(v_ids16, 16, 32); + float sumf = 0.0f; + + for (int i = 0; i < nb; ++i) { + const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint8_t * GGML_RESTRICT qs = x[i].qs; + const uint8_t * GGML_RESTRICT qh = x[i].qh; + const uint8_t * GGML_RESTRICT scales = x[i].scales; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + const uint8_t * signs_ptr = qs + 32; + + float sum_block = 0.0f; + + for (int ib = 0; ib < 2; ++ib) { + vuint8mf4_t v_qs_u8 = __riscv_vle8_v_u8mf4(qs, 16); + qs += 16; + + vuint8mf8_t v_qh_raw = __riscv_vle8_v_u8mf8(qh, 4); + qh += 4; + + vuint16mf4_t v_qh_u16 = __riscv_vwcvtu_x_x_v_u16mf4(v_qh_raw, 4); + vuint16mf2_t v_qh_u16_ext = __riscv_vlmul_ext_v_u16mf4_u16mf2(v_qh_u16); + vuint16mf2_t v_qh_expanded = __riscv_vrgather_vv_u16mf2(v_qh_u16_ext, v_gather_qh, 16); + v_qh_expanded = __riscv_vsll_vv_u16mf2(v_qh_expanded, v_shift_qh, 16); + v_qh_expanded = __riscv_vand_vx_u16mf2(v_qh_expanded, 0x1800, 16); + + vuint16mf2_t v_qs_u16 = __riscv_vwcvtu_x_x_v_u16mf2(v_qs_u8, 16); + v_qs_u16 = __riscv_vsll_vx_u16mf2(v_qs_u16, 3, 16); + + vuint16mf2_t v_grid_offsets = __riscv_vor_vv_u16mf2(v_qs_u16, v_qh_expanded, 16); + vuint64m2_t v_grid_vals = __riscv_vluxei16_v_u64m2(grid64, v_grid_offsets, 16); + vuint8m2_t v_grid_u8 = __riscv_vreinterpret_v_u64m2_u8m2(v_grid_vals); + vint8m2_t v_grid_i8 = __riscv_vreinterpret_v_u8m2_i8m2(v_grid_u8); + + vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs_ptr, 16); + signs_ptr += 16; + + vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw); + vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 128); + vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 128); + vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 128); + vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 128); + q8 += 128; + + vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative, v_q8, v_q8, 0, 128); + vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_grid_i8, v_q8_signed, 128); + + vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1); + vint16m1_t v0 = __riscv_vget_v_i16m4_i16m1(v_dot, 0); + vint16m1_t v1 = __riscv_vget_v_i16m4_i16m1(v_dot, 1); + vint16m1_t v2 = __riscv_vget_v_i16m4_i16m1(v_dot, 2); + vint16m1_t v3 = __riscv_vget_v_i16m4_i16m1(v_dot, 3); + + int32_t s0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(v0, v_zero, 16)); + int32_t s1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(m_hi16, v0, v_zero, 32)); + int32_t s2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(v1, v_zero, 16)); + int32_t s3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(m_hi16, v1, v_zero, 32)); + int32_t s4 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(v2, v_zero, 16)); + int32_t s5 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(m_hi16, v2, v_zero, 32)); + int32_t s6 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( v3, v_zero, 16)); + int32_t s7 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(m_hi16, v3, v_zero, 32)); + + uint8_t sc0 = scales[0]; + uint8_t sc1 = scales[1]; + uint8_t sc2 = scales[2]; + uint8_t sc3 = scales[3]; + scales += 4; + + sum_block += s0 * (2 * (sc0 & 0xF) + 1); + sum_block += s1 * (2 * (sc0 >> 4) + 1); + sum_block += s2 * (2 * (sc1 & 0xF) + 1); + sum_block += s3 * (2 * (sc1 >> 4) + 1); + sum_block += s4 * (2 * (sc2 & 0xF) + 1); + sum_block += s5 * (2 * (sc2 >> 4) + 1); + sum_block += s6 * (2 * (sc3 & 0xF) + 1); + sum_block += s7 * (2 * (sc3 >> 4) + 1); + } + + sumf += sum_block * combined_scale; + } + *s = 0.125f * sumf; +} + +static NOINLINE void ggml_vec_dot_iq2_s_q8_K_vl1024(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + + const block_iq2_s * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + const uint64_t * grid64 = (const uint64_t *)iq2s_grid; + vuint8m2_t v_ids = __riscv_vid_v_u8m2(256); + vuint8m2_t v_sign_gather_indices = __riscv_vsrl_vx_u8m2(v_ids, 3, 256); + + vuint8m2_t v_ones = __riscv_vmv_v_x_u8m2(1, 256); + vuint8m2_t v_shift_amts = __riscv_vand_vx_u8m2(v_ids, 7, 256); + vuint8m2_t v_sign_masks = __riscv_vsll_vv_u8m2(v_ones, v_shift_amts, 256); + + uint16_t gather_qh_arr[32] = { + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7 + }; + vuint16mf2_t v_gather_qh = __riscv_vle16_v_u16mf2(gather_qh_arr, 32); + + uint16_t shift_qh_arr[32] = { + 11, 9, 7, 5, 11, 9, 7, 5, 11, 9, 7, 5, 11, 9, 7, 5, + 11, 9, 7, 5, 11, 9, 7, 5, 11, 9, 7, 5, 11, 9, 7, 5 + }; + vuint16mf2_t v_shift_qh = __riscv_vle16_v_u16mf2(shift_qh_arr, 32); + + // Masks for 4 groups of 16 lanes within a 64-lane i16m4 chunk + vuint16m4_t v_ids64 = __riscv_vid_v_u16m4(64); + vbool4_t m_g0 = __riscv_vmsltu_vx_u16m4_b4(v_ids64, 16, 64); + vbool4_t m_g1 = __riscv_vmand_mm_b4( + __riscv_vmsgeu_vx_u16m4_b4(v_ids64, 16, 64), + __riscv_vmsltu_vx_u16m4_b4(v_ids64, 32, 64), 64); + vbool4_t m_g2 = __riscv_vmand_mm_b4( + __riscv_vmsgeu_vx_u16m4_b4(v_ids64, 32, 64), + __riscv_vmsltu_vx_u16m4_b4(v_ids64, 48, 64), 64); + vbool4_t m_g3 = __riscv_vmsgeu_vx_u16m4_b4(v_ids64, 48, 64); + + float sumf = 0.0f; + + for (int i = 0; i < nb; ++i) { + const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint8_t * GGML_RESTRICT qs = x[i].qs; + const uint8_t * GGML_RESTRICT qh = x[i].qh; + const uint8_t * GGML_RESTRICT scales = x[i].scales; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + const uint8_t * signs_ptr = qs + 32; + + float sum_block = 0.0f; + + vuint8mf4_t v_qs_u8 = __riscv_vle8_v_u8mf4(qs, 32); + qs += 32; + + vuint8mf8_t v_qh_raw = __riscv_vle8_v_u8mf8(qh, 8); + qh += 8; + + vuint16mf4_t v_qh_u16 = __riscv_vwcvtu_x_x_v_u16mf4(v_qh_raw, 8); + vuint16mf2_t v_qh_u16_ext = __riscv_vlmul_ext_v_u16mf4_u16mf2(v_qh_u16); + vuint16mf2_t v_qh_expanded = __riscv_vrgather_vv_u16mf2(v_qh_u16_ext, v_gather_qh, 32); + v_qh_expanded = __riscv_vsll_vv_u16mf2(v_qh_expanded, v_shift_qh, 32); + v_qh_expanded = __riscv_vand_vx_u16mf2(v_qh_expanded, 0x1800, 32); + + vuint16mf2_t v_qs_u16 = __riscv_vwcvtu_x_x_v_u16mf2(v_qs_u8, 32); + v_qs_u16 = __riscv_vsll_vx_u16mf2(v_qs_u16, 3, 32); + + vuint16mf2_t v_grid_offsets = __riscv_vor_vv_u16mf2(v_qs_u16, v_qh_expanded, 32); + vuint64m2_t v_grid_vals = __riscv_vluxei16_v_u64m2(grid64, v_grid_offsets, 32); + vuint8m2_t v_grid_u8 = __riscv_vreinterpret_v_u64m2_u8m2(v_grid_vals); + vint8m2_t v_grid_i8 = __riscv_vreinterpret_v_u8m2_i8m2(v_grid_u8); + + //loading signs + vuint8mf2_t v_signs_raw = __riscv_vle8_v_u8mf2(signs_ptr, 32); + signs_ptr += 32; + + vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf2_u8m2(v_signs_raw); + vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 256); + vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 256); + vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 256); + + vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 256); + q8 += 256; + + vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative, v_q8, v_q8, 0, 256); + vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_grid_i8, v_q8_signed, 256); + + vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1); + + vint16m4_t c = v_dot; + + int32_t s0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g0, c, v_zero, 64)); + int32_t s1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g1, c, v_zero, 64)); + int32_t s2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g2, c, v_zero, 64)); + int32_t s3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g3, c, v_zero, 64)); + + c = __riscv_vslidedown_vx_i16m4(c, 64, 256); + int32_t s4 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g0, c, v_zero, 64)); + int32_t s5 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g1, c, v_zero, 64)); + int32_t s6 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g2, c, v_zero, 64)); + int32_t s7 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g3, c, v_zero, 64)); + + c = __riscv_vslidedown_vx_i16m4(c, 64, 256); + int32_t s8 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g0, c, v_zero, 64)); + int32_t s9 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g1, c, v_zero, 64)); + int32_t s10 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g2, c, v_zero, 64)); + int32_t s11 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g3, c, v_zero, 64)); + + c = __riscv_vslidedown_vx_i16m4(c, 64, 256); + int32_t s12 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g0, c, v_zero, 64)); + int32_t s13 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g1, c, v_zero, 64)); + int32_t s14 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g2, c, v_zero, 64)); + int32_t s15 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m4_i32m1_m(m_g3, c, v_zero, 64)); + + int32_t sums_arr[16] = { s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15 }; + + // Load 8 scale bytes and split into 16 nibbles + vuint8mf2_t v_sc8 = __riscv_vle8_v_u8mf2(scales, 8); + scales += 8; + + vuint8mf2_t v_lo8 = __riscv_vand_vx_u8mf2(v_sc8, 0x0F, 8); + vuint8mf2_t v_hi8 = __riscv_vsrl_vx_u8mf2(v_sc8, 4, 8); + + vuint8m1_t v_idx16 = __riscv_vid_v_u8m1(16); + vuint8m1_t v_half = __riscv_vsrl_vx_u8m1(v_idx16, 1, 16); + vbool8_t m_even = __riscv_vmseq_vx_u8m1_b8(__riscv_vand_vx_u8m1(v_idx16, 1, 16), 0, 16); + + vuint8m1_t v_lo_ext = __riscv_vlmul_ext_v_u8mf2_u8m1(v_lo8); + vuint8m1_t v_hi_ext = __riscv_vlmul_ext_v_u8mf2_u8m1(v_hi8); + vuint8m1_t v_lo_g = __riscv_vrgather_vv_u8m1(v_lo_ext, v_half, 16); + vuint8m1_t v_hi_g = __riscv_vrgather_vv_u8m1(v_hi_ext, v_half, 16); + vuint8m1_t v_nib = __riscv_vmerge_vvm_u8m1(v_lo_g, v_hi_g, m_even, 16); + + static const uint8_t iq2s_scale_lut_16_local[16] = { + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + }; + vuint8m1_t v_lut = __riscv_vle8_v_u8m1(iq2s_scale_lut_16_local, 16); + vuint8m1_t v_sc8v = __riscv_vrgather_vv_u8m1(v_lut, v_nib, 16); + + vint32m4_t v_sums = __riscv_vle32_v_i32m4(sums_arr, 16); + vuint16m2_t v_sc16 = __riscv_vwcvtu_x_x_v_u16m2(v_sc8v, 16); + vuint32m4_t v_sc32u = __riscv_vwcvtu_x_x_v_u32m4(v_sc16, 16); + vint32m4_t v_sc32 = __riscv_vreinterpret_v_u32m4_i32m4(v_sc32u); + vint32m4_t v_prod = __riscv_vmul_vv_i32m4(v_sums, v_sc32, 16); + + vint32m1_t v_zero32 = __riscv_vmv_v_x_i32m1(0, 1); + int32_t sum_part = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m4_i32m1(v_prod, v_zero32, 16)); + sum_block += sum_part; + + sumf += sum_block * combined_scale; + } + *s = 0.125f * sumf; +} #endif void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_iq2_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); @@ -2800,8 +4219,11 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo case 256: ggml_vec_dot_iq2_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; + case 512: + ggml_vec_dot_iq2_s_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); + break; default: - ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_s_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); break; } #else @@ -2809,7 +4231,7 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static const int8_t keven_signs_q2xs[1024] = { 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1, @@ -2947,59 +4369,140 @@ static NOINLINE void ggml_vec_dot_iq2_xs_q8_K_vl256(int n, float * GGML_RESTRICT int32_t sum_int = 0; - // Loop over 4 subblocks of 64 elements (QK_K = 256) - for (int ib64 = 0; ib64 < QK_K / 64; ++ib64) { - // Load 8 uint16 indices (controls 64 values) - vuint16mf2_t v_qs = __riscv_vle16_v_u16mf2(qs, 8); - qs += 8; + for (int ib128 = 0; ib128 < 2; ++ib128) { + + vuint16m1_t v_qs = __riscv_vle16_v_u16m1(qs, 16); + qs += 16; - // Extract indices for grid (low 9 bits) and signs (high 7 bits) - // Multiply by 8 (<< 3) for byte offsets into the uint64 tables - vuint16mf2_t vidx_grid = __riscv_vsll_vx_u16mf2(__riscv_vand_vx_u16mf2(v_qs, 511, 8), 3, 8); - vuint16mf2_t vidx_sign = __riscv_vsll_vx_u16mf2(__riscv_vsrl_vx_u16mf2(v_qs, 9, 8), 3, 8); + // Prepare offsets for grid and signs + vuint16m1_t vidx_grid = __riscv_vsll_vx_u16m1(__riscv_vand_vx_u16m1(v_qs, 511, 16), 3, 16); + vuint16m1_t vidx_sign = __riscv_vsll_vx_u16m1(__riscv_vsrl_vx_u16m1(v_qs, 9, 16), 3, 16); - vuint64m2_t vq2_64 = __riscv_vluxei16_v_u64m2(grid64, vidx_grid, 8); - vuint64m2_t vs2_64 = __riscv_vluxei16_v_u64m2(signs64, vidx_sign, 8); + // Indexed load 128 weights (16 x 8-byte chunks) + vuint64m4_t vq2_64 = __riscv_vluxei16_v_u64m4(grid64, vidx_grid, 16); + vuint64m4_t vs2_64 = __riscv_vluxei16_v_u64m4(signs64, vidx_sign, 16); - vint8m2_t q2u = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vq2_64)); - vint8m2_t q2s = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vs2_64)); + vint8m4_t q2u = __riscv_vreinterpret_v_u8m4_i8m4(__riscv_vreinterpret_v_u64m4_u8m4(vq2_64)); + vint8m4_t q2s = __riscv_vreinterpret_v_u8m4_i8m4(__riscv_vreinterpret_v_u64m4_u8m4(vs2_64)); - vint8m2_t q2_final = __riscv_vmul_vv_i8m2(q2u, q2s, 64); + // Apply signs to get dequantized IQ2 values + vint8m4_t q2_final = __riscv_vmul_vv_i8m4(q2u, q2s, 128); + asm volatile("" ::: "memory"); - vint8m2_t q8v = __riscv_vle8_v_i8m2(q8, 64); - q8 += 64; + // Load corresponding Q8 weights + vint8m4_t q8v = __riscv_vle8_v_i8m4(q8, 128); + q8 += 128; + + vint16m8_t prod = __riscv_vwmul_vv_i16m8(q2_final, q8v, 128); + asm volatile("" ::: "memory"); - vint16m4_t prod = __riscv_vwmul_vv_i16m4(q2_final, q8v, 64); + uint8_t sc0 = scales[0]; + uint8_t sc1 = scales[1]; + uint8_t sc2 = scales[2]; + uint8_t sc3 = scales[3]; + scales += 4; vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1); - int32_t sum0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( - __riscv_vget_v_i16m4_i16m1(prod, 0), zero_vec, 16)); - int32_t sum1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( - __riscv_vget_v_i16m4_i16m1(prod, 1), zero_vec, 16)); - int32_t sum2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( - __riscv_vget_v_i16m4_i16m1(prod, 2), zero_vec, 16)); - int32_t sum3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( - __riscv_vget_v_i16m4_i16m1(prod, 3), zero_vec, 16)); + // 9. Reduce each 16-element chunk and apply corresponding nibble scale - const uint8_t scale_byte_1 = scales[0]; - const uint8_t scale_byte_2 = scales[1]; - scales += 2; + int32_t s0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 0), zero_vec, 16)); + sum_int += s0 * ((sc0 & 0x0F) * 2 + 1); - sum_int += sum0 * ((scale_byte_1 & 0x0F) * 2 + 1); - sum_int += sum1 * ((scale_byte_1 >> 4) * 2 + 1); - sum_int += sum2 * ((scale_byte_2 & 0x0F) * 2 + 1); - sum_int += sum3 * ((scale_byte_2 >> 4) * 2 + 1); + int32_t s1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 1), zero_vec, 16)); + sum_int += s1 * ((sc0 >> 4) * 2 + 1); + + int32_t s2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 2), zero_vec, 16)); + sum_int += s2 * ((sc1 & 0x0F) * 2 + 1); + + int32_t s3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 3), zero_vec, 16)); + sum_int += s3 * ((sc1 >> 4) * 2 + 1); + + int32_t s4 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 4), zero_vec, 16)); + sum_int += s4 * ((sc2 & 0x0F) * 2 + 1); + + int32_t s5 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 5), zero_vec, 16)); + sum_int += s5 * ((sc2 >> 4) * 2 + 1); + + int32_t s6 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 6), zero_vec, 16)); + sum_int += s6 * ((sc3 & 0x0F) * 2 + 1); + + int32_t s7 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 7), zero_vec, 16)); + sum_int += s7 * ((sc3 >> 4) * 2 + 1); } - sumf += d * sum_int; + sumf += d * (float)sum_int; + } + *s = 0.125f * sumf; +} + +static NOINLINE void ggml_vec_dot_iq2_xs_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + + const block_iq2_xs * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + const uint64_t * grid64 = (const uint64_t *)iq2xs_grid; + + float sumf = 0.0f; + for (int i = 0; i < nb; ++i) { + const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint16_t * GGML_RESTRICT qs = x[i].qs; + const uint8_t * GGML_RESTRICT scales = x[i].scales; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + vint8m4_t q8_all = __riscv_vle8_v_i8m4(q8, 256); + + // Load indices --- + vuint16m1_t v_qs = __riscv_vle16_v_u16m1(qs, 32); + + // Extract low 9 bits and multiply by 8 (shift left 3) for byte offset into uint64 table + vuint16m1_t vidx_grid = __riscv_vsll_vx_u16m1(__riscv_vand_vx_u16m1(v_qs, 511, 32), 3, 32); + + // Extract high 7 bits (shift right 9) and multiply by 8 (shift left 3) for byte offset + vuint16m1_t vidx_sign = __riscv_vsll_vx_u16m1(__riscv_vsrl_vx_u16m1(v_qs, 9, 32), 3, 32); + + vuint64m4_t vq2_64 = __riscv_vluxei16_v_u64m4(grid64, vidx_grid, 32); + vuint64m4_t vs2_64 = __riscv_vluxei16_v_u64m4(signs64, vidx_sign, 32); + + vint8m4_t q2_all = __riscv_vreinterpret_v_u8m4_i8m4(__riscv_vreinterpret_v_u64m4_u8m4(vq2_64)); + vint8m4_t s2_all = __riscv_vreinterpret_v_u8m4_i8m4(__riscv_vreinterpret_v_u64m4_u8m4(vs2_64)); + + vint8m4_t q2_signed = __riscv_vmul_vv_i8m4(q2_all, s2_all, 256); + vint16m8_t dot_all = __riscv_vwmul_vv_i16m8(q2_signed, q8_all, 256); + float sum = 0.0f; + vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1); + +#pragma GCC unroll 1 + for (int j = 0; j < 8; ++j) { + uint8_t sc = scales[j]; + int16_t sc_lo = 2 * (sc & 0x0F) + 1; + int16_t sc_hi = 2 * (sc >> 4) + 1; + + vint32m1_t sum_v0 = __riscv_vwredsum_vs_i16m8_i32m1( + __riscv_vslidedown_vx_i16m8(dot_all, j * 32, 16), zero_vec, 16); + int32_t isum0 = __riscv_vmv_x_s_i32m1_i32(sum_v0); + + vint32m1_t sum_v1 = __riscv_vwredsum_vs_i16m8_i32m1( + __riscv_vslidedown_vx_i16m8(dot_all, j * 32 + 16, 16), zero_vec, 16); + int32_t isum1 = __riscv_vmv_x_s_i32m1_i32(sum_v1); + + sum += (float)isum0 * sc_lo + (float)isum1 * sc_hi; + } + + sumf += sum * combined_scale; } *s = 0.125f * sumf; } #endif void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_iq2_xs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); @@ -3007,8 +4510,8 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v case 256: ggml_vec_dot_iq2_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; - default: - ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + default: // 512 and above + ggml_vec_dot_iq2_xs_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); break; } #else @@ -3016,7 +4519,7 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_iq2_xxs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -3201,24 +4704,99 @@ static NOINLINE void ggml_vec_dot_iq2_xxs_q8_K_vl256(int n, float * GGML_RESTRIC } *s = 0.125f * sumf; } + +static NOINLINE void ggml_vec_dot_iq2_xxs_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + + const block_iq2_xxs * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + const uint64_t * grid64 = (const uint64_t *)iq2xxs_grid; + // Shift pattern {0,7,14,21} repeated 8 times for all 8 sub-blocks + uint8_t shift_arr[32] = { + 0, 7, 14, 21, 0, 7, 14, 21, 0, 7, 14, 21, 0, 7, 14, 21, + 0, 7, 14, 21, 0, 7, 14, 21, 0, 7, 14, 21, 0, 7, 14, 21 + }; + vuint8mf2_t v_shifts = __riscv_vle8_v_u8mf2(shift_arr, 32); + + // Gather pattern to broadcast the 8 sub-block scales across the 32 lookup slots + uint8_t gather_arr[32] = { + 0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3, + 4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7 + }; + vuint8mf2_t v_sign_gather_idx = __riscv_vle8_v_u8mf2(gather_arr, 32); + + float sumf = 0.0f; + for (int i = 0; i < nb; ++i) { + const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint8_t * GGML_RESTRICT q2_ptr = (const uint8_t *) x[i].qs; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + vint8m4_t q8_all = __riscv_vle8_v_i8m4(q8, 256); + + // De-interleave all 8 Index/Scale pairs for the 8x32-element sub-blocks + vuint32mf2x2_t tuple = __riscv_vlseg2e32_v_u32mf2x2((const uint32_t*)q2_ptr, 8); + vuint32mf2_t v_ind32 = __riscv_vget_v_u32mf2x2_u32mf2(tuple, 0); + vuint32mf2_t v_sc32 = __riscv_vget_v_u32mf2x2_u32mf2(tuple, 1); + + vuint8mf2_t v_raw_q2 = __riscv_vreinterpret_v_u32mf2_u8mf2(v_ind32); + vuint16m1_t vidx_q2 = __riscv_vwcvtu_x_x_v_u16m1(v_raw_q2, 32); + vidx_q2 = __riscv_vsll_vx_u16m1(vidx_q2, 3, 32); + + vuint32m2_t v_s = __riscv_vrgatherei16_vv_u32m2(__riscv_vlmul_ext_v_u32mf2_u32m2(v_sc32), __riscv_vwcvtu_x_x_v_u16m1(v_sign_gather_idx,32), 32); + v_s = __riscv_vsrl_vv_u32m2(v_s, __riscv_vwcvtu_x_x_v_u32m2(__riscv_vwcvtu_x_x_v_u16m1(v_shifts,32),32), 32); + v_s = __riscv_vand_vx_u32m2(v_s, 127, 32); + vuint16m1_t vidx_s2 = __riscv_vsll_vx_u16m1(__riscv_vncvt_x_x_w_u16m1(v_s, 32), 3, 32); + + vuint64m4_t vq2_64 = __riscv_vluxei16_v_u64m4(grid64, vidx_q2, 32); + vuint64m4_t vs2_64 = __riscv_vluxei16_v_u64m4(signs64, vidx_s2, 32); + vint8m4_t q2_all = __riscv_vreinterpret_v_u8m4_i8m4(__riscv_vreinterpret_v_u64m4_u8m4(vq2_64)); + vint8m4_t s2_all = __riscv_vreinterpret_v_u8m4_i8m4(__riscv_vreinterpret_v_u64m4_u8m4(vs2_64)); + + vint8m4_t q8s_all = __riscv_vmul_vv_i8m4(q8_all, s2_all, 256); + vint16m8_t dot_all = __riscv_vwmul_vv_i16m8(q8s_all, q2_all, 256); + + float sum = 0.0f; + vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1); + + for (int j = 0; j < 8; ++j) { + uint32_t s_p = __riscv_vmv_x_s_u32mf2_u32(__riscv_vslidedown_vx_u32mf2(v_sc32, j, 8)); + int16_t sc = 2 * ((s_p >> 28) & 0xF) + 1; + dot_all=__riscv_vslidedown_vx_i16m8(dot_all,j*32,32); + vint32m1_t sum_v = __riscv_vwredsum_vs_i16m8_i32m1(dot_all, zero_vec, 32); + int32_t isum = __riscv_vmv_x_s_i32m1_i32(sum_v); + sum += (float)isum * sc; + } + + sumf += sum * combined_scale; + } + *s = 0.125f * sumf; +} #endif void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_iq2_xxs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); break; - default: // 256 and above + case 256: ggml_vec_dot_iq2_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; + default: // 512 and above + ggml_vec_dot_iq2_xxs_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); + break; } #else ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_iq3_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -3408,19 +4986,108 @@ static NOINLINE void ggml_vec_dot_iq3_s_q8_K_vl256(int n, float * GGML_RESTRICT } *s = sumf; } + +static NOINLINE void ggml_vec_dot_iq3_s_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + const block_iq3_s * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + const uint32_t * grid32 = (const uint32_t *)iq3s_grid; + + // Generate Constants + vuint8mf2_t v_id_32 = __riscv_vid_v_u8mf2(32); + vuint8mf2_t v_qh_gather = __riscv_vsrl_vx_u8mf2(v_id_32, 3, 32); + vuint8mf2_t v_qh_shifts = __riscv_vand_vx_u8mf2(v_id_32, 7, 32); + vuint8m2_t v_id_128 = __riscv_vid_v_u8m2(128); + vuint8m2_t v_sign_gather = __riscv_vsrl_vx_u8m2(v_id_128, 3, 128); // byte index + vuint8m2_t v_sign_shift_amts = __riscv_vand_vx_u8m2(v_id_128, 7, 128); // bit shift + vuint8m2_t v_one_128 = __riscv_vmv_v_x_u8m2(1, 128); + vuint8m2_t v_sign_masks = __riscv_vsll_vv_u8m2(v_one_128, v_sign_shift_amts, 128); + vuint8m2_t v_scale_indices = __riscv_vsrl_vx_u8m2(v_id_128, 5, 128); + + float sumf = 0.0f; + + for (int i = 0; i < nb; ++i) { + const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint8_t * GGML_RESTRICT qs = x[i].qs; + const uint8_t * GGML_RESTRICT qh = x[i].qh; + const uint8_t * GGML_RESTRICT scales = x[i].scales; + const uint8_t * GGML_RESTRICT signs = x[i].signs; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + float sum_block = 0.0f; + for (int ib = 0; ib < 2; ++ib) { + vuint8mf2_t v_qs_u8 = __riscv_vle8_v_u8mf2(qs, 32); + qs += 32; + vuint8mf2_t v_qh_loaded = __riscv_vle8_v_u8mf2(qh, 4); + qh += 4; + vuint8mf2_t v_qh_expanded = __riscv_vrgather_vv_u8mf2(v_qh_loaded, v_qh_gather, 32); + v_qh_expanded = __riscv_vsrl_vv_u8mf2(v_qh_expanded, v_qh_shifts, 32); + v_qh_expanded = __riscv_vand_vx_u8mf2(v_qh_expanded, 1, 32); + vuint16m1_t v_qs_u16 = __riscv_vwcvtu_x_x_v_u16m1(v_qs_u8, 32); + v_qs_u16 = __riscv_vsll_vx_u16m1(v_qs_u16, 2, 32); // * 4 + + vuint16m1_t v_qh_u16 = __riscv_vwcvtu_x_x_v_u16m1(v_qh_expanded, 32); + v_qh_u16 = __riscv_vsll_vx_u16m1(v_qh_u16, 10, 32); // * 256 * 4 + + vuint16m1_t v_grid_offsets = __riscv_vor_vv_u16m1(v_qs_u16, v_qh_u16, 32); + vuint32m2_t v_grid_packed = __riscv_vluxei16_v_u32m2(grid32, v_grid_offsets, 32); + vuint8m2_t v_grid_u8 = __riscv_vreinterpret_v_u32m2_u8m2(v_grid_packed); + vuint8mf2_t v_signs_raw = __riscv_vle8_v_u8mf2(signs, 16); + signs += 16; + + vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf2_u8m2(v_signs_raw); + vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather, 128); + vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 128); + vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 128); + + vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 128); + q8 += 128; + + vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative, v_q8, v_q8, 0, 128); + vint16m4_t v_dot = __riscv_vwmulsu_vv_i16m4(v_q8_signed, v_grid_u8, 128); + uint16_t sc_raw; + memcpy(&sc_raw, scales, 2); + scales += 2; // Advance 2 bytes + + uint8_t sc_unpacked[4]; + sc_unpacked[0] = (sc_raw & 0xF); + sc_unpacked[1] = (sc_raw >> 4) & 0xF; + sc_unpacked[2] = (sc_raw >> 8) & 0xF; + sc_unpacked[3] = (sc_raw >> 12) & 0xF; + + vuint8mf2_t v_sc_4 = __riscv_vle8_v_u8mf2(sc_unpacked, 4); + v_sc_4 = __riscv_vmul_vx_u8mf2(v_sc_4, 2, 4); + v_sc_4 = __riscv_vadd_vx_u8mf2(v_sc_4, 1, 4); + vuint8m2_t v_sc_4_expanded = __riscv_vlmul_ext_v_u8mf2_u8m2(v_sc_4); + vuint8m2_t v_scales_bcast = __riscv_vrgather_vv_u8m2(v_sc_4_expanded, v_scale_indices, 128); + vint16m4_t v_scales_i16 = __riscv_vreinterpret_v_u16m4_i16m4(__riscv_vwcvtu_x_x_v_u16m4(v_scales_bcast, 128)); + vint32m8_t v_weighted_sum = __riscv_vwmul_vv_i32m8(v_dot, v_scales_i16, 128); + vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1); + int32_t s_val = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m8_i32m1(v_weighted_sum, v_zero, 128)); + + sum_block += s_val; + } + sumf += sum_block * combined_scale; + } + *s = sumf; +} #endif void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq3_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); break; case 256: ggml_vec_dot_iq3_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; - default: - ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + default: // 512 and above + ggml_vec_dot_iq3_s_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); break; } #else @@ -3428,7 +5095,7 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -3614,10 +5281,181 @@ static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl256(int n, float * GGML_RESTRIC } *s = 0.25f * sumf; } + +static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + const block_iq3_xxs * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + const int nb = n / QK_K; + + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + const uint32_t * grid32 = (const uint32_t *)iq3xxs_grid; + + // generate constants for unpacking metadata words into sign indices + vuint32m1_t v_shifts; + { + vuint32m1_t v_base = __riscv_vid_v_u32m1(16); + vuint32m1_t v_mod4 = __riscv_vand_vx_u32m1(v_base, 3, 16); + v_shifts = __riscv_vmul_vx_u32m1(v_mod4, 7, 16); + } + + vuint16mf2_t v_gather_idx; + { + vuint16mf2_t v_idx = __riscv_vid_v_u16mf2(16); + v_gather_idx = __riscv_vsrl_vx_u16mf2(v_idx, 2, 16); + } + + float sumf = 0.0f; + + for (int i = 0; i < nb; ++i) { + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint8_t * GGML_RESTRICT q3_indices = x[i].qs; + const uint8_t * GGML_RESTRICT metadata = x[i].qs + QK_K/4; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + float block_sum = 0.0f; + for (int ib128 = 0; ib128 < 2; ++ib128) { + + vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 128); + q8 += 128; + vuint8mf2_t v_q3_idx_u8 = __riscv_vle8_v_u8mf2(q3_indices, 32); + q3_indices += 32; + + vuint16m1_t v_q3_idx_u16 = __riscv_vwmulu_vx_u16m1(v_q3_idx_u8, 4, 32); + vuint32m2_t v_q3_mag_u32 = __riscv_vluxei16_v_u32m2(grid32, v_q3_idx_u16, 32); + vint8m2_t v_q3_magnitudes = __riscv_vreinterpret_v_u8m2_i8m2( + __riscv_vreinterpret_v_u32m2_u8m2(v_q3_mag_u32)); + vuint32m1_t v_aux = __riscv_vreinterpret_v_u8m1_u32m1(__riscv_vle8_v_u8m1(metadata, 16)); + metadata += 4 * sizeof(uint32_t); + + vuint32m1_t v_aux_expanded = __riscv_vrgatherei16_vv_u32m1(v_aux, v_gather_idx, 16); + + vuint32m1_t v_s_raw = __riscv_vand_vx_u32m1( + __riscv_vsrl_vv_u32m1(v_aux_expanded, v_shifts, 16), 127, 16); + vuint16mf2_t sign_byte_offset = __riscv_vsll_vx_u16mf2( + __riscv_vncvt_x_x_w_u16mf2(v_s_raw, 16), 3, 16); + vuint64m2_t v_s_u64 = __riscv_vluxei16_v_u64m2(signs64, sign_byte_offset, 16); + vint8m2_t v_signs = __riscv_vreinterpret_v_u8m2_i8m2( + __riscv_vreinterpret_v_u64m2_u8m2(v_s_u64)); + vint8m2_t v_q3_signed = __riscv_vmul_vv_i8m2(v_q3_magnitudes, v_signs, 128); + vint16m4_t prod = __riscv_vwmul_vv_i16m4(v_q3_signed, v_q8, 128); + + vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1); + int32_t group0_sum = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( + __riscv_vget_v_i16m4_i16m1(prod, 0), zero_vec, 32)); + int32_t group1_sum = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( + __riscv_vget_v_i16m4_i16m1(prod, 1), zero_vec, 32)); + int32_t group2_sum = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( + __riscv_vget_v_i16m4_i16m1(prod, 2), zero_vec, 32)); + int32_t group3_sum = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( + __riscv_vget_v_i16m4_i16m1(prod, 3), zero_vec, 32)); + + vuint32m1_t v_scales_raw = __riscv_vsrl_vx_u32m1(v_aux, 28, 4); + vuint32m1_t v_scales = __riscv_vadd_vx_u32m1( + __riscv_vsll_vx_u32m1(v_scales_raw, 1, 4), + 1, 4); + int32_t scale0 = (int32_t)__riscv_vmv_x_s_u32m1_u32(v_scales); + int32_t scale1 = (int32_t)__riscv_vmv_x_s_u32m1_u32(__riscv_vslidedown_vx_u32m1(v_scales, 1, 4)); + int32_t scale2 = (int32_t)__riscv_vmv_x_s_u32m1_u32(__riscv_vslidedown_vx_u32m1(v_scales, 2, 4)); + int32_t scale3 = (int32_t)__riscv_vmv_x_s_u32m1_u32(__riscv_vslidedown_vx_u32m1(v_scales, 3, 4)); + + block_sum += (float)(group0_sum * scale0 + group1_sum * scale1 + + group2_sum * scale2 + group3_sum * scale3); + } + + sumf += d * block_sum; + } + *s = 0.25f * sumf; +} + +static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl1024(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + + const block_iq3_xxs * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + const int nb = n / QK_K; + + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + const uint32_t * grid32 = (const uint32_t *)iq3xxs_grid; + + vuint32m1_t v_shifts; + { + vuint32m1_t v_id = __riscv_vid_v_u32m1(32); + vuint32m1_t v_mod4 = __riscv_vand_vx_u32m1(v_id, 3, 32); + v_shifts = __riscv_vmul_vx_u32m1(v_mod4, 7, 32); + } + vuint16mf2_t v_gather_idx; + { + vuint16mf2_t v_id_16 = __riscv_vid_v_u16mf2(32); + v_gather_idx = __riscv_vsrl_vx_u16mf2(v_id_16, 2, 32); + } + + float sumf = 0.0f; + uint32_t aux32[8]; // Buffer for block metadata + + for (int i = 0; i < nb; ++i) { + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d; + + const uint8_t * GGML_RESTRICT q3_indices = x[i].qs; + const uint8_t * GGML_RESTRICT metadata = x[i].qs + QK_K/4; + const int8_t * GGML_RESTRICT q8 = y[i].qs; + + vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 256); + vuint8mf2_t v_q3_idx_raw = __riscv_vle8_v_u8mf2(q3_indices, 64); + vuint16m1_t v_q3_idx_u16 = __riscv_vwmulu_vx_u16m1(v_q3_idx_raw, 4, 64); + + vuint32m2_t v_q3_grid_vals = __riscv_vluxei16_v_u32m2(grid32, v_q3_idx_u16, 64); + + vint8m2_t v_q3_mags = __riscv_vreinterpret_v_u8m2_i8m2( + __riscv_vreinterpret_v_u32m2_u8m2(v_q3_grid_vals)); + + memcpy(aux32, metadata, 8 * sizeof(uint32_t)); + vuint32m1_t v_aux_8 = __riscv_vle32_v_u32m1(aux32, 8); + + vuint32m1_t v_aux_32 = __riscv_vrgatherei16_vv_u32m1(v_aux_8, v_gather_idx, 32); + + vuint32m1_t v_sign_idx_raw = __riscv_vand_vx_u32m1( + __riscv_vsrl_vv_u32m1(v_aux_32, v_shifts, 32), 127, 32); + + vuint16mf2_t v_sign_offsets = __riscv_vsll_vx_u16mf2( + __riscv_vncvt_x_x_w_u16mf2(v_sign_idx_raw, 32), 3, 32); + + vuint64m2_t v_signs_u64 = __riscv_vluxei16_v_u64m2(signs64, v_sign_offsets, 32); + + vint8m2_t v_signs = __riscv_vreinterpret_v_u8m2_i8m2( + __riscv_vreinterpret_v_u64m2_u8m2(v_signs_u64)); + + vint8m2_t v_q3_final = __riscv_vmul_vv_i8m2(v_q3_mags, v_signs, 256); + + vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_q8, v_q3_final, 256); + float block_sum = 0.0f; + vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1); + vint16m4_t v_accum = v_dot; + + for (int j = 0; j < 8; ++j) { + float scale = (float)(2 * (aux32[j] >> 28) + 1); + + vint32m1_t v_partial_sum = __riscv_vwredsum_vs_i16m4_i32m1(v_accum, v_zero, 32); + + int32_t partial_sum_i = __riscv_vmv_x_s_i32m1_i32(v_partial_sum); + block_sum += partial_sum_i * scale; + v_accum = __riscv_vslidedown_vx_i16m4(v_accum, 32, 32); + + } + + sumf += d * block_sum; + } + *s = 0.25f * sumf; +} #endif void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_iq3_xxs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); @@ -3625,8 +5463,11 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const case 256: ggml_vec_dot_iq3_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; - default: - ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + case 512: + ggml_vec_dot_iq3_xxs_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); + break; + default: // 1024 and above + ggml_vec_dot_iq3_xxs_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); break; } #else @@ -3634,7 +5475,7 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_iq4_nl_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(nrc == 1); UNUSED(nrc); @@ -3749,7 +5590,7 @@ static NOINLINE void ggml_vec_dot_iq4_nl_q8_0_vl256(int n, float * GGML_RESTRICT #endif void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_iq4_nl_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc); @@ -3763,7 +5604,7 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(nrc == 1); UNUSED(nrc); @@ -3909,10 +5750,205 @@ static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl256(int n, float * GGML_RESTRICT *s = sumf; } + +static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK_K == 0); + + const block_iq4_xs * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + const vint8m4_t values = __riscv_vle8_v_i8m4(kvalues_iq4nl, 16); + float sumf = 0; + + // Indices for re-ordering IQ4 data. + const uint16_t index[32] = { + 0, 1, 16, 17, + 2, 3, 18, 19, + 4, 5,20, 21, + 6, 7, 22, 23, + 8, 9, 24, 25, + 10, 11, 26, 27, + 12, 13,28, 29, + 14, 15, 30, 31, + }; + const vuint16m1_t i_vec = __riscv_vle16_v_u16m1(index, 32); + + for (int ibl = 0; ibl < nb; ++ibl) { + const int8_t * q8 = y[ibl].qs; + const uint8_t * iq4 = x[ibl].qs; + uint16_t h = x[ibl].scales_h; + + int sumi = 0; + + #pragma GCC unroll 1 + // Process the entire super-block together. + for (int ib = 0; ib < QK_K / 256; ++ib) { + // Weights and activations. + const vuint8m2_t iq4_packed = __riscv_vle8_v_u8m2(iq4, 128); + iq4 += 128; + + // Unpack the weight blocks. + const vuint8m2_t iq4bits_lo = __riscv_vand_vx_u8m2(iq4_packed, 0xf, 128); + const vuint8m2_t iq4bits_hi = __riscv_vsrl_vx_u8m2(iq4_packed, 4, 128); + const vuint8m4_t iq4bits = __riscv_vcreate_v_u8m2_u8m4(iq4bits_lo, iq4bits_hi); + const vuint8m4_t iq4bits_reorder = __riscv_vreinterpret_v_u64m4_u8m4(__riscv_vrgatherei16_vv_u64m4(__riscv_vreinterpret_v_u8m4_u64m4(iq4bits), i_vec, 32)); + const vint8m4_t iq4b = __riscv_vrgather_vv_i8m4(values, iq4bits_reorder, 256); + + __asm__ __volatile__("" ::: "memory"); + + // Multiply with activations. + const vint8m4_t q8b = __riscv_vle8_v_i8m4(q8, 256); + const vint16m8_t prod = __riscv_vwmul_vv_i16m8(iq4b, q8b, 256); + q8 += 256; + + // Reduce separately. + const int acc0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 0), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 1), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 2), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 3), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc4 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 4), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc5 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 5), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc6 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 6), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc7 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(__riscv_vget_v_i16m8_i16m1(prod, 7), __riscv_vmv_v_x_i32m1(0, 1), 32)); + + + const int ls0 = ((x[ibl].scales_l[0] & 0xf) | ((h << 4) & 0x30)) - 32; + const int ls1 = ((x[ibl].scales_l[0] >> 4) | ((h << 2) & 0x30)) - 32; + const int ls2 = ((x[ibl].scales_l[1] & 0xf) | ((h << 0) & 0x30)) - 32; + const int ls3 = ((x[ibl].scales_l[1] >> 4) | ((h >> 2) & 0x30)) - 32; + h >>= 8; + const int ls4 = ((x[ibl].scales_l[2] & 0xf) | ((h << 4) & 0x30)) - 32; + const int ls5 = ((x[ibl].scales_l[2] >> 4) | ((h << 2) & 0x30)) - 32; + const int ls6 = ((x[ibl].scales_l[3] & 0xf) | ((h << 0) & 0x30)) - 32; + const int ls7 = ((x[ibl].scales_l[3] >> 4) | ((h >> 2) & 0x30)) - 32; + + sumi += acc0 * ls0; + sumi += acc1 * ls1; + sumi += acc2 * ls2; + sumi += acc3 * ls3; + sumi += acc4 * ls4; + sumi += acc5 * ls5; + sumi += acc6 * ls6; + sumi += acc7 * ls7; + + __asm__ __volatile__("" ::: "memory"); + } + + sumf += GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi); + } + + *s = sumf; +} + +static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl1024(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK_K == 0); + + const block_iq4_xs * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + const vint8m2_t values = __riscv_vle8_v_i8m2(kvalues_iq4nl, 16); + float sumf = 0; + + // Indices for re-ordering IQ4 data. + const uint16_t index[32] = { + 0, 1, 16, 17, + 2, 3, 18, 19, + 4, 5,20, 21, + 6, 7, 22, 23, + 8, 9, 24, 25, + 10, 11, 26, 27, + 12, 13,28, 29, + 14, 15, 30, 31, + }; + const vuint16mf2_t i_vec = __riscv_vle16_v_u16mf2(index, 32); + + for (int ibl = 0; ibl < nb; ++ibl) { + const int8_t * q8 = y[ibl].qs; + const uint8_t * iq4 = x[ibl].qs; + uint16_t h = x[ibl].scales_h; + + int sumi = 0; + + #pragma GCC unroll 1 + // Process the entire super-block together. + for (int ib = 0; ib < QK_K / 256; ++ib) { + // Weights and activations. + const vuint8m1_t iq4_packed = __riscv_vle8_v_u8m1(iq4, 128); + iq4 += 128; + + // Unpack the weight blocks. + const vuint8m1_t iq4bits_lo = __riscv_vand_vx_u8m1(iq4_packed, 0xf, 128); + const vuint8m1_t iq4bits_hi = __riscv_vsrl_vx_u8m1(iq4_packed, 4, 128); + const vuint8m2_t iq4bits = __riscv_vcreate_v_u8m1_u8m2(iq4bits_lo, iq4bits_hi); + const vuint8m2_t iq4bits_reorder = __riscv_vreinterpret_v_u64m2_u8m2(__riscv_vrgatherei16_vv_u64m2(__riscv_vreinterpret_v_u8m2_u64m2(iq4bits), i_vec, 32)); + const vint8m2_t iq4b = __riscv_vrgather_vv_i8m2(values, iq4bits_reorder, 256); + + __asm__ __volatile__("" ::: "memory"); + + // Multiply with activations. + const vint8m2_t q8b = __riscv_vle8_v_i8m2(q8, 256); + const vint16m4_t prod = __riscv_vwmul_vv_i16m4(iq4b, q8b, 256); + q8 += 256; + + // Mask for processing 32 elements per prod register. + const vuint16m1_t p_index = __riscv_vid_v_u16m1(64); + const vbool16_t p_mask = __riscv_vmsgtu_vx_u16m1_b16(p_index, 31, 64); + + // Reduce separately. + const int acc0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( __riscv_vget_v_i16m4_i16m1(prod, 0), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(p_mask, __riscv_vget_v_i16m4_i16m1(prod, 0), __riscv_vmv_v_x_i32m1(0, 1), 64)); + const int acc2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( __riscv_vget_v_i16m4_i16m1(prod, 1), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(p_mask, __riscv_vget_v_i16m4_i16m1(prod, 1), __riscv_vmv_v_x_i32m1(0, 1), 64)); + const int acc4 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( __riscv_vget_v_i16m4_i16m1(prod, 2), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc5 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(p_mask, __riscv_vget_v_i16m4_i16m1(prod, 2), __riscv_vmv_v_x_i32m1(0, 1), 64)); + const int acc6 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1( __riscv_vget_v_i16m4_i16m1(prod, 3), __riscv_vmv_v_x_i32m1(0, 1), 32)); + const int acc7 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1_m(p_mask, __riscv_vget_v_i16m4_i16m1(prod, 3), __riscv_vmv_v_x_i32m1(0, 1), 64)); + + const int ls0 = ((x[ibl].scales_l[0] & 0xf) | ((h << 4) & 0x30)) - 32; + const int ls1 = ((x[ibl].scales_l[0] >> 4) | ((h << 2) & 0x30)) - 32; + const int ls2 = ((x[ibl].scales_l[1] & 0xf) | ((h << 0) & 0x30)) - 32; + const int ls3 = ((x[ibl].scales_l[1] >> 4) | ((h >> 2) & 0x30)) - 32; + h >>= 8; + const int ls4 = ((x[ibl].scales_l[2] & 0xf) | ((h << 4) & 0x30)) - 32; + const int ls5 = ((x[ibl].scales_l[2] >> 4) | ((h << 2) & 0x30)) - 32; + const int ls6 = ((x[ibl].scales_l[3] & 0xf) | ((h << 0) & 0x30)) - 32; + const int ls7 = ((x[ibl].scales_l[3] >> 4) | ((h >> 2) & 0x30)) - 32; + + sumi += acc0 * ls0; + sumi += acc1 * ls1; + sumi += acc2 * ls2; + sumi += acc3 * ls3; + sumi += acc4 * ls4; + sumi += acc5 * ls5; + sumi += acc6 * ls6; + sumi += acc7 * ls7; + + __asm__ __volatile__("" ::: "memory"); + } + + sumf += GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi); + } + + *s = sumf; +} #endif void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_iq4_xs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); @@ -3920,6 +5956,12 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v case 256: ggml_vec_dot_iq4_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; + case 512: + ggml_vec_dot_iq4_xs_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); + break; + case 1024: + ggml_vec_dot_iq4_xs_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); + break; default: ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); break; @@ -3929,7 +5971,7 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_tq1_0_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(nrc == 1); UNUSED(nrc); @@ -4132,10 +6174,112 @@ static NOINLINE void ggml_vec_dot_tq1_0_q8_K_vl256(int n, float * GGML_RESTRICT *s = sumf; } + +static NOINLINE void ggml_vec_dot_tq1_0_q8_K_vl512(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_tq1_0 * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + float sumf = 0.0f; + uint8_t pow[16] = {1, 1, 1, 1, 3, 3, 3, 3, 9, 9, 9, 9, 27, 27, 27, 27}; + + for (int i = 0; i < nb; i++) { + // First loop. + vint16m1_t suml1; + { + const int vl = 32; + vuint8mf2_t tq = __riscv_vle8_v_u8mf2(x[i].qs, vl); + + vuint16m1_t tq0 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(tq, 3, vl), 8, vl); + vuint16m1_t tq1 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 3, vl), 3, vl), 8, vl); + vuint16m1_t tq2 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 9, vl), 3, vl), 8, vl); + vuint16m1_t tq3 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 27, vl), 3, vl), 8, vl); + vuint16m1_t tq4 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 81, vl), 3, vl), 8, vl); + + vint16m1_t q80 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 0, vl), vl); + vint16m1_t q81 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 32, vl), vl); + vint16m1_t q82 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 64, vl), vl); + vint16m1_t q83 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 96, vl), vl); + vint16m1_t q84 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 128, vl), vl); + + vint16m1_t sum0 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq0, 1, vl)), q80, vl); + vint16m1_t sum1 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq1, 1, vl)), q81, vl); + vint16m1_t sum2 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq2, 1, vl)), q82, vl); + vint16m1_t sum3 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq3, 1, vl)), q83, vl); + vint16m1_t sum4 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq4, 1, vl)), q84, vl); + + vint16m1_t sumi0 = __riscv_vadd_vv_i16m1(sum0, sum1, vl); + vint16m1_t sumi1 = __riscv_vadd_vv_i16m1(sum2, sum3, vl); + suml1 = __riscv_vadd_vv_i16m1(sum4, __riscv_vadd_vv_i16m1(sumi0, sumi1, vl), vl); + } + + // Second loop. + vint16mf2_t suml2; + { + const int vl = 16; + vuint8mf4_t tq = __riscv_vle8_v_u8mf4(x[i].qs + 32, vl); + + vuint16mf2_t tq0 = __riscv_vsrl_vx_u16mf2(__riscv_vwmulu_vx_u16mf2(tq, 3 * 1, vl), 8, vl); + vuint16mf2_t tq1 = __riscv_vsrl_vx_u16mf2(__riscv_vwmulu_vx_u16mf2(__riscv_vmul_vx_u8mf4(tq, 3, vl), 3, vl), 8, vl); + vuint16mf2_t tq2 = __riscv_vsrl_vx_u16mf2(__riscv_vwmulu_vx_u16mf2(__riscv_vmul_vx_u8mf4(tq, 9, vl), 3, vl), 8, vl); + vuint16mf2_t tq3 = __riscv_vsrl_vx_u16mf2(__riscv_vwmulu_vx_u16mf2(__riscv_vmul_vx_u8mf4(tq, 27, vl), 3, vl), 8, vl); + vuint16mf2_t tq4 = __riscv_vsrl_vx_u16mf2(__riscv_vwmulu_vx_u16mf2(__riscv_vmul_vx_u8mf4(tq, 81, vl), 3, vl), 8, vl); + + vint16mf2_t q80 = __riscv_vwcvt_x_x_v_i16mf2(__riscv_vle8_v_i8mf4(y[i].qs + 160, vl), vl); + vint16mf2_t q81 = __riscv_vwcvt_x_x_v_i16mf2(__riscv_vle8_v_i8mf4(y[i].qs + 176, vl), vl); + vint16mf2_t q82 = __riscv_vwcvt_x_x_v_i16mf2(__riscv_vle8_v_i8mf4(y[i].qs + 192, vl), vl); + vint16mf2_t q83 = __riscv_vwcvt_x_x_v_i16mf2(__riscv_vle8_v_i8mf4(y[i].qs + 208, vl), vl); + vint16mf2_t q84 = __riscv_vwcvt_x_x_v_i16mf2(__riscv_vle8_v_i8mf4(y[i].qs + 224, vl), vl); + + vint16mf2_t sum0 = __riscv_vmul_vv_i16mf2(__riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vsub_vx_u16mf2(tq0, 1, vl)), q80, vl); + vint16mf2_t sum1 = __riscv_vmul_vv_i16mf2(__riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vsub_vx_u16mf2(tq1, 1, vl)), q81, vl); + vint16mf2_t sum2 = __riscv_vmul_vv_i16mf2(__riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vsub_vx_u16mf2(tq2, 1, vl)), q82, vl); + vint16mf2_t sum3 = __riscv_vmul_vv_i16mf2(__riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vsub_vx_u16mf2(tq3, 1, vl)), q83, vl); + vint16mf2_t sum4 = __riscv_vmul_vv_i16mf2(__riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vsub_vx_u16mf2(tq4, 1, vl)), q84, vl); + + vint16mf2_t sumi0 = __riscv_vadd_vv_i16mf2(sum0, sum1, vl); + vint16mf2_t sumi1 = __riscv_vadd_vv_i16mf2(sum2, sum3, vl); + suml2 = __riscv_vadd_vv_i16mf2(sum4, __riscv_vadd_vv_i16mf2(sumi0, sumi1, vl), vl); + } + + // Third loop. + vint16mf2_t suml3; + { + const int vl = 16; + + uint32_t qh; + memcpy(&qh, &x[i].qh[0], 4); + // Prevent fusion with vmv. + __asm__ __volatile__("" : "+r"(qh)); + vuint8mf4_t tq = __riscv_vlmul_trunc_v_u8mf2_u8mf4(__riscv_vreinterpret_v_u32mf2_u8mf2(__riscv_vmv_v_x_u32mf2(qh, vl / 4))); + + vuint8mf4_t p = __riscv_vle8_v_u8mf4(pow, vl); + + vuint16mf2_t tq0 = __riscv_vsrl_vx_u16mf2(__riscv_vwmulu_vx_u16mf2(__riscv_vmul_vv_u8mf4(tq, p, vl), 3, vl), 8, vl); + + vint16mf2_t q80 = __riscv_vwcvt_x_x_v_i16mf2(__riscv_vle8_v_i8mf4(y[i].qs + 240, vl), vl); + + suml3 = __riscv_vmul_vv_i16mf2(__riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vsub_vx_u16mf2(tq0, 1, vl)), q80, vl); + } + + vint32m1_t sum = __riscv_vwredsum_vs_i16m1_i32m1(suml1, __riscv_vmv_v_x_i32m1(0, 1), 32); + sum = __riscv_vwredsum_vs_i16mf2_i32m1(__riscv_vadd_vv_i16mf2(suml2, suml3, 16), sum, 16); + sumf += __riscv_vmv_x_s_i32m1_i32(sum) * y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); + } + + *s = sumf; +} #endif void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_tq1_0_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); @@ -4143,8 +6287,8 @@ void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo case 256: ggml_vec_dot_tq1_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; - default: - ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + default: // 512 and above + ggml_vec_dot_tq1_0_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); break; } #else @@ -4152,7 +6296,7 @@ void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_tq2_0_q8_K_vl128(const int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -4308,24 +6452,21 @@ static NOINLINE void ggml_vec_dot_tq2_0_q8_K_vl256(int n, float * GGML_RESTRICT #endif void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_tq2_0_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); break; - case 256: + default: // 256 and above ggml_vec_dot_tq2_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); break; - default: - ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); - break; } #else ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); #endif } -#if defined __riscv_v_intrinsic +#if defined __riscv_v static NOINLINE void ggml_vec_dot_mxfp4_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(nrc == 1); UNUSED(nrc); @@ -4440,7 +6581,7 @@ static NOINLINE void ggml_vec_dot_mxfp4_q8_0_vl256(int n, float * GGML_RESTRICT #endif void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { -#if defined __riscv_v_intrinsic +#if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: ggml_vec_dot_mxfp4_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc); diff --git a/ggml/src/ggml-cpu/arch/s390/quants.c b/ggml/src/ggml-cpu/arch/s390/quants.c index 5008575..22b8b83 100644 --- a/ggml/src/ggml-cpu/arch/s390/quants.c +++ b/ggml/src/ggml-cpu/arch/s390/quants.c @@ -146,6 +146,74 @@ void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, i //===================================== Dot products ================================= +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK1_0; // 128 + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + +#if defined(__VXE__) || defined(__VXE2__) + float32x4_t v_sumf = vec_splats(0.0f); + + const uint8x16_t v_zero = vec_splats((uint8_t)0x00); // zero + const uint8x16_t v_bias = vec_splats((uint8_t)0x80); // bias from signed to unsigned + // v ^ 0x80 == v + 128 + + const uint8x16_t v_idx = (const uint8x16_t){ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 }; + const uint8x16_t v_bit = (const uint8x16_t){ 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128 }; + + for (int i = 0; i < nb; ++i) { + const uint8x16_t v_x = vec_xl(0, (const uint8_t *)x[i].qs); + const float32x4_t v_xd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d)); + + for (int k = 0; k < 4; ++k) { + // sub-block k holds elements 32k .. 32k+31 + const block_q8_0 * GGML_RESTRICT yb = &y[i*4 + k]; + const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(yb->d)); + + const uint8x16_t v_xrl = vec_perm(v_x, v_x, vec_add(v_idx, vec_splats((uint8_t)(k*4 + 0)))); + const uint8x16_t v_xrh = vec_perm(v_x, v_x, vec_add(v_idx, vec_splats((uint8_t)(k*4 + 2)))); + + // isolate each lane's bit, then set all ones where that bit is clear, the -d case + const int8x16_t v_ml = (int8x16_t)vec_cmpeq(vec_and(v_xrl, v_bit), v_zero); + const int8x16_t v_mh = (int8x16_t)vec_cmpeq(vec_and(v_xrh, v_bit), v_zero); + + const int8x16_t v_yl = vec_xl(0, (const int8_t *)yb->qs); + const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)yb->qs); + + // weights are only +1 or -1, so negate y + const int8x16_t v_ysl = vec_sub(vec_xor(v_yl, v_ml), v_ml); + const int8x16_t v_ysh = vec_sub(vec_xor(v_yh, v_mh), v_mh); + + // bias to unsigned, then vec_sum4 adds each group of 4 bytes into one word + const uint32x4_t v_p = vec_add(vec_sum4(vec_xor((uint8x16_t)v_ysl, v_bias), v_zero), + vec_sum4(vec_xor((uint8x16_t)v_ysh, v_bias), v_zero)); + + // each word summed 8 biased bytes, so take back 8 * 128 + const int32x4_t v_xy = vec_sub((int32x4_t)v_p, vec_splats((int32_t)1024)); + + // apply both block scales and add into the running total + v_sumf = vec_madd(vec_float(v_xy), vec_mul(v_xd, v_yd), v_sumf); + } + } + + *s = vec_hsum_f32x4(v_sumf); +#else + UNUSED(nb); + UNUSED(x); + UNUSED(y); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -242,7 +310,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_yl = vec_xl(0 , y[ib].qs); const int8x16_t v_yh = vec_xl(QK8_1/2, y[ib].qs); - const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const int32x4_t v_xy_ = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh); const float32x4_t v_xy = vec_float(v_xy_); const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d)); @@ -311,8 +379,8 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo const int8x16_t v_y1l = vec_xl(0, y1->qs); const int8x16_t v_y1h = vec_xl(QK8_0/2, y1->qs); - const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h); - const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h); + const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h); + const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h); const float32x4_t v_xy0f = vec_float(v_xy0); const float32x4_t v_xy1f = vec_float(v_xy1); @@ -339,7 +407,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo const int8x16_t v_yl = vec_xl(0, y0->qs); const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs); - const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh); const float32x4_t v_xyf = vec_float(v_xy); const float32x4_t v_d = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d)); @@ -349,6 +417,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo sumf = vec_hsum_f32x4(v_acc); *s = sumf; #else + UNUSED(nb); UNUSED(x); UNUSED(y); UNUSED(ib); @@ -433,8 +502,8 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_y1l = vec_xl(0, (const int8_t *)y1->qs); const int8x16_t v_y1h = vec_xl(QK8_0/2, (const int8_t *)y1->qs); - const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h); - const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h); + const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h); + const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h); const float32x4_t v_xy0f = vec_float(v_xy0); const float32x4_t v_xy1f = vec_float(v_xy1); @@ -479,7 +548,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_yl = vec_xl(0, (const int8_t *)y0->qs); const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)y0->qs); - const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh); + const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh); const float32x4_t v_xyf = vec_float(v_xy); const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d)); @@ -584,8 +653,8 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_y1l = vec_xl(0 , y1->qs); const int8x16_t v_y1h = vec_xl(QK8_1/2, y1->qs); - const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h); - const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h); + const int32x4_t v_xy0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h); + const int32x4_t v_xy1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h); const float32x4_t v_xy0f = vec_float(v_xy0); const float32x4_t v_xy1f = vec_float(v_xy1); @@ -632,11 +701,11 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_yl = vec_xl(0 , y0->qs); const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs); - const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh); + const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh); const float32x4_t v_xyf = vec_float(v_xy); const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d)); - const float32x4_t v_acc = vec_madd(v_xyf, v_d, v_acc); + const float32x4_t v_acc = vec_madd(v_xyf, v_d, vec_splats(0.0f)); sumf += vec_hsum_f32x4(v_acc) + summs; } @@ -682,7 +751,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi const int8x16_t v_yl = vec_xl(0 , y[ib].qs); const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs); - const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const int32x4_t v_xy_ = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh); const float32x4_t v_xy = vec_float(v_xy_); const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d)); @@ -786,10 +855,10 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 2), v_3m), (int8x16_t)q3h[2]); q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 2), v_3m), (int8x16_t)q3h[3]); - isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[0]); - isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[1]); - isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[2]); - isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[3]); + isum0 = ggml_vec_dot_s32(v_z, q3bytes[0], q8bytes[0]); + isum1 = ggml_vec_dot_s32(v_z, q3bytes[1], q8bytes[1]); + isum2 = ggml_vec_dot_s32(v_z, q3bytes[2], q8bytes[2]); + isum3 = ggml_vec_dot_s32(v_z, q3bytes[3], q8bytes[3]); isum += (isum0[0] + isum0[1] + isum0[2] + isum0[3]) * scale[0]; isum += (isum1[0] + isum1[1] + isum1[2] + isum1[3]) * scale[1]; @@ -808,10 +877,10 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 6), v_3m), (int8x16_t)q3h[2]); q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 6), v_3m), (int8x16_t)q3h[3]); - isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[4]); - isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[5]); - isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[6]); - isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[7]); + isum0 = ggml_vec_dot_s32(v_z, q3bytes[0], q8bytes[4]); + isum1 = ggml_vec_dot_s32(v_z, q3bytes[1], q8bytes[5]); + isum2 = ggml_vec_dot_s32(v_z, q3bytes[2], q8bytes[6]); + isum3 = ggml_vec_dot_s32(v_z, q3bytes[3], q8bytes[7]); isum += vec_hsum_i32x4(isum0) * scale[0]; isum += vec_hsum_i32x4(isum1) * scale[1]; @@ -912,7 +981,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi v_xl[0] = (int8x16_t)vec_and(v_x[0], v_lm); v_xl[1] = (int8x16_t)vec_and(v_x[1], v_lm); - const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); + const int32x4_t p1 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); sumi1 += vec_hsum_i32x4(p1) * scales[2*j+0]; v_y[0] = vec_xl(0 , y0); @@ -922,7 +991,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi v_xl[0] = (int8x16_t)vec_sr(v_x[0], 4); v_xl[1] = (int8x16_t)vec_sr(v_x[1], 4); - const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); + const int32x4_t p2 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); sumi2 += vec_hsum_i32x4(p2) * scales[2*j+1]; } @@ -1038,8 +1107,8 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q5b[2] = (int8x16_t)vec_or(vec_sr(v_xl[0], 4), q5h[2]); q5b[3] = (int8x16_t)vec_or(vec_sr(v_xl[1], 4), q5h[3]); - int32x4_t sumi0 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]); - int32x4_t sumi1 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]); + int32x4_t sumi0 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]); + int32x4_t sumi1 = ggml_vec_dot_s32(ggml_vec_dot_s32(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]); sumi += vec_hsum_i32x4(sumi0) * *scales++; sumi += vec_hsum_i32x4(sumi1) * *scales++; @@ -1148,10 +1217,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q6b[2] = (int8x16_t)(vec_or(vec_and(v_xl[2], v_lm), q6h[2])); q6b[3] = (int8x16_t)(vec_or(vec_and(v_xl[3], v_lm), q6h[3])); - int32x4_t summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]); - int32x4_t summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]); - int32x4_t summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]); - int32x4_t summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]); + int32x4_t summs0 = ggml_vec_dot_s32(v_z, q6b[0], v_y[0]); + int32x4_t summs1 = ggml_vec_dot_s32(v_z, q6b[1], v_y[1]); + int32x4_t summs2 = ggml_vec_dot_s32(v_z, q6b[2], v_y[2]); + int32x4_t summs3 = ggml_vec_dot_s32(v_z, q6b[3], v_y[3]); isum += vec_hsum_i32x4(summs0) * scale[0] + vec_hsum_i32x4(summs1) * scale[1] + @@ -1182,10 +1251,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi q6b[2] = (int8x16_t)(vec_or(vec_sr(v_xl[2], 4), q6h[2])); q6b[3] = (int8x16_t)(vec_or(vec_sr(v_xl[3], 4), q6h[3])); - summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]); - summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]); - summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]); - summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]); + summs0 = ggml_vec_dot_s32(v_z, q6b[0], v_y[0]); + summs1 = ggml_vec_dot_s32(v_z, q6b[1], v_y[1]); + summs2 = ggml_vec_dot_s32(v_z, q6b[2], v_y[2]); + summs3 = ggml_vec_dot_s32(v_z, q6b[3], v_y[3]); isum += vec_hsum_i32x4(summs0) * scale[0] + vec_hsum_i32x4(summs1) * scale[1] + @@ -1298,8 +1367,8 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi // q2u2 = vec_mul(q2u2, q2s2); // q2u3 = vec_mul(q2u3, q2s3); -// const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1); -// const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3); +// const int32x4_t p1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1); +// const int32x4_t p2 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3); // sumf1 += (p1[0] + p1[1] + p1[2] + p1[3]) * (0.5f + (aux32[1] >> 28)); // sumf2 += (p2[0] + p2[1] + p2[2] + p2[3]) * (0.5f + (aux32[3] >> 28)); @@ -1376,7 +1445,7 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v const int8x16_t v_yl = vec_xl(0 , y0->qs); const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs); - const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const int32x4_t v_xy = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_xl, v_yl), v_xh, v_yh); sumf += GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d) * vec_hsum_i32x4(v_xy); } @@ -1439,8 +1508,8 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v const int8x16_t v_y3 = vec_xl(48, q8); q8 += 64; - int32x4_t vsumi0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1); - int32x4_t vsumi1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3); + int32x4_t vsumi0 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1); + int32x4_t vsumi1 = ggml_vec_dot_s32(ggml_vec_dot_s32(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3); int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32; int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32; diff --git a/ggml/src/ggml-cpu/arch/s390/repack.cpp b/ggml/src/ggml-cpu/arch/s390/repack.cpp new file mode 100644 index 0000000..c12fe21 --- /dev/null +++ b/ggml/src/ggml-cpu/arch/s390/repack.cpp @@ -0,0 +1,208 @@ +#define GGML_COMMON_IMPL_CPP +#define GGML_COMMON_DECL_CPP +#include "ggml-common.h" +#include "ggml-backend-impl.h" + +#include "ggml-impl.h" +#include "ggml-cpu.h" +#include "ggml-cpu-impl.h" +#include "simd-mappings.h" +#include "traits.h" + +#include +#include +#include + +#define GGML_CPU_CLANG_WORKAROUND +#include "../../repack.h" + +#define UNUSED GGML_UNUSED + +void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { + assert(QK8_0 == 32); + assert(k % QK8_0 == 0); + const int nb = k / QK8_0; + + block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; + +#if defined(__VXE__) || defined(__VXE2__) + float32x4_t v_src[4][8]; + float id[4]; + + for (int i = 0; i < nb; i++) { + float32x4_t v_asrc[8]; + float32x4_t v_amax[8]; + + for (int row_iter = 0; row_iter < 4; row_iter++) { + for (int j = 0; j < 8; j++) v_src[row_iter][j] = vec_xl(0, x + row_iter * k + i * 32 + 4 * j); + for (int j = 0; j < 8; j++) v_asrc[j] = vec_abs(v_src[row_iter][j]); + + for (int j = 0; j < 4; j++) v_amax[2 * j] = vec_max(v_asrc[2 * j], v_asrc[2 * j + 1]); + for (int j = 0; j < 2; j++) v_amax[4 * j] = vec_max(v_amax[4 * j], v_amax[4 * j + 2]); + for (int j = 0; j < 1; j++) v_amax[8 * j] = vec_max(v_amax[8 * j], v_amax[8 * j + 4]); + + const float amax = MAX(MAX(vec_extract(v_amax[0], 0), vec_extract(v_amax[0], 1)), + MAX(vec_extract(v_amax[0], 2), vec_extract(v_amax[0], 3))); + + const float d = amax / ((1 << 7) - 1); + id[row_iter] = d ? 1.0f / d : 0.0f; + + y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d); + } + + for (int j = 0; j < 8; j++) { + /* Uses non-default rounding for vec_signed or vec_round */ + const int32x4_t v_qs0 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[0][j], id[0]), 4, 1)); + const int32x4_t v_qs1 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[1][j], id[1]), 4, 1)); + const int32x4_t v_qs2 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[2][j], id[2]), 4, 1)); + const int32x4_t v_qs3 = vec_signed(__builtin_s390_vfisb(vec_mul(v_src[3][j], id[3]), 4, 1)); + + const int16x8_t v_qs01 = vec_packs(v_qs0, v_qs1); + const int16x8_t v_qs23 = vec_packs(v_qs2, v_qs3); + + vec_xst(vec_packs(v_qs01, v_qs23), 0, y[i].qs + 16 * j); + } + } +#else + UNUSED(nb); + UNUSED(y); + ggml_quantize_mat_q8_0_4x4_generic(x, vy, k); +#endif +} + +void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK8_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + +#if defined(__VXE__) || defined(__VXE2__) + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + float * res_ptr = s; + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb); + + float32x4_t v_sumf = vec_splats(0.0f); + + for (int l = 0; l < nb; l++) { + const int8_t * x_qs = b_ptr[l].qs; + + const int8x16_t v_x0 = vec_xl( 0, x_qs); + const int8x16_t v_x1 = vec_xl(16, x_qs); + const int8x16_t v_x2 = vec_xl(32, x_qs); + const int8x16_t v_x3 = vec_xl(48, x_qs); + + const int8x16_t v_x0l = vec_sra(vec_sl(v_x0, 4), 4); + const int8x16_t v_x1l = vec_sra(vec_sl(v_x1, 4), 4); + const int8x16_t v_x2l = vec_sra(vec_sl(v_x2, 4), 4); + const int8x16_t v_x3l = vec_sra(vec_sl(v_x3, 4), 4); + + const int8x16_t v_x0h = vec_sra(v_x0, 4); + const int8x16_t v_x1h = vec_sra(v_x1, 4); + const int8x16_t v_x2h = vec_sra(v_x2, 4); + const int8x16_t v_x3h = vec_sra(v_x3, 4); + + const int8_t * y_lo = a_ptr[l].qs; + const int8_t * y_hi = y_lo + qk / 2; + + int16x8_t v_sumi = vec_splats((int16_t)0); + + v_sumi = ggml_vec_dot_s16(v_sumi, v_x0l, ggml_vec_xl_s8x4_splat(y_lo + 0)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x1l, ggml_vec_xl_s8x4_splat(y_lo + 4)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x2l, ggml_vec_xl_s8x4_splat(y_lo + 8)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x3l, ggml_vec_xl_s8x4_splat(y_lo + 12)); + + v_sumi = ggml_vec_dot_s16(v_sumi, v_x0h, ggml_vec_xl_s8x4_splat(y_hi + 0)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x1h, ggml_vec_xl_s8x4_splat(y_hi + 4)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x2h, ggml_vec_xl_s8x4_splat(y_hi + 8)); + v_sumi = ggml_vec_dot_s16(v_sumi, v_x3h, ggml_vec_xl_s8x4_splat(y_hi + 12)); + + const float32x4_t v_yd = vec_splats(GGML_CPU_FP16_TO_FP32(a_ptr[l].d)); + const float32x4_t v_xd = ggml_vec_xl_f16x4_finite(b_ptr[l].d); + const float32x4_t v_d = vec_mul(v_yd, v_xd); + + v_sumf = vec_madd(vec_float(vec_paddl_s16(v_sumi)), v_d, v_sumf); + } + + vec_xst(v_sumf, 0, res_ptr + x * ncols_interleaved); + } +#else + UNUSED(nb); + UNUSED(ncols_interleaved); + ggml_gemv_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +#endif +} + +void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { + const int qk = QK8_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(nr % 4 == 0); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + +#if defined(__VXE__) || defined(__VXE2__) + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb); + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb); + + float32x4_t v_sumf[4]; + for (int m = 0; m < 4; m++) { + v_sumf[m] = vec_splats(0.0f); + } + + for (int l = 0; l < nb; l++) { + int16x8_t v_sumi0 = vec_splats((int16_t)0); + int16x8_t v_sumi1 = vec_splats((int16_t)0); + int16x8_t v_sumi2 = vec_splats((int16_t)0); + int16x8_t v_sumi3 = vec_splats((int16_t)0); + + for (int k = 0; k < 4; k++) { + const int8x16_t v_x = vec_xl(0, b_ptr[l].qs + 16 * k); + const int8x16_t v_xl = vec_sra(vec_sl(v_x, 4), 4); + const int8x16_t v_xh = vec_sra(v_x, 4); + + const int8_t * y_lo = a_ptr[l].qs + 16 * k; + const int8_t * y_hi = y_lo + qk / 2 * 4; + + v_sumi0 = ggml_vec_dot_s16(v_sumi0, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 0)); + v_sumi1 = ggml_vec_dot_s16(v_sumi1, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 4)); + v_sumi2 = ggml_vec_dot_s16(v_sumi2, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 8)); + v_sumi3 = ggml_vec_dot_s16(v_sumi3, v_xl, ggml_vec_xl_s8x4_splat(y_lo + 12)); + + v_sumi0 = ggml_vec_dot_s16(v_sumi0, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 0)); + v_sumi1 = ggml_vec_dot_s16(v_sumi1, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 4)); + v_sumi2 = ggml_vec_dot_s16(v_sumi2, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 8)); + v_sumi3 = ggml_vec_dot_s16(v_sumi3, v_xh, ggml_vec_xl_s8x4_splat(y_hi + 12)); + } + + const float32x4_t v_yd = __lzs_f16cx4_load(a_ptr[l].d); + const float32x4_t v_xd = ggml_vec_xl_f16x4_finite(b_ptr[l].d); + + v_sumf[0] = vec_madd(vec_float(vec_paddl_s16(v_sumi0)), vec_mul(v_xd, vec_splat(v_yd, 0)), v_sumf[0]); + v_sumf[1] = vec_madd(vec_float(vec_paddl_s16(v_sumi1)), vec_mul(v_xd, vec_splat(v_yd, 1)), v_sumf[1]); + v_sumf[2] = vec_madd(vec_float(vec_paddl_s16(v_sumi2)), vec_mul(v_xd, vec_splat(v_yd, 2)), v_sumf[2]); + v_sumf[3] = vec_madd(vec_float(vec_paddl_s16(v_sumi3)), vec_mul(v_xd, vec_splat(v_yd, 3)), v_sumf[3]); + } + + for (int m = 0; m < 4; m++) { + vec_xst(v_sumf[m], 0, s + (y * 4 + m) * bs + x * ncols_interleaved); + } + } + } +#else + UNUSED(nb); + UNUSED(ncols_interleaved); + ggml_gemm_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc); +#endif +} diff --git a/ggml/src/ggml-cpu/arch/wasm/quants.c b/ggml/src/ggml-cpu/arch/wasm/quants.c index 648c6fc..0a7119b 100644 --- a/ggml/src/ggml-cpu/arch/wasm/quants.c +++ b/ggml/src/ggml-cpu/arch/wasm/quants.c @@ -355,6 +355,78 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK8_1; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q4_1 * GGML_RESTRICT x = vx; + const block_q8_1 * GGML_RESTRICT y = vy; + + float sumf = 0; + +#if defined __wasm_simd128__ + v128_t sumv = wasm_f32x4_splat(0.0f); + float summs = 0.0f; + + for (int ib = 0; ib < nb; ++ib) { + const block_q4_1 * GGML_RESTRICT x0 = &x[ib]; + const block_q8_1 * GGML_RESTRICT y0 = &y[ib]; + + summs += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s); + + const v128_t raw = wasm_v128_load(x0->qs); + const v128_t v0s = wasm_v128_and(raw, wasm_i8x16_splat(0x0F)); + const v128_t v1s = wasm_u8x16_shr(raw, 4); + + const v128_t ys_lo = wasm_v128_load(y0->qs); + const v128_t ys_hi = wasm_v128_load(y0->qs + 16); + + const v128_t v0s_l = wasm_u16x8_extend_low_u8x16(v0s); + const v128_t v0s_h = wasm_u16x8_extend_high_u8x16(v0s); + const v128_t ylo_l = wasm_i16x8_extend_low_i8x16(ys_lo); + const v128_t ylo_h = wasm_i16x8_extend_high_i8x16(ys_lo); + const v128_t v1s_l = wasm_u16x8_extend_low_u8x16(v1s); + const v128_t v1s_h = wasm_u16x8_extend_high_u8x16(v1s); + const v128_t yhi_l = wasm_i16x8_extend_low_i8x16(ys_hi); + const v128_t yhi_h = wasm_i16x8_extend_high_i8x16(ys_hi); + + const v128_t acc = wasm_i32x4_add( + wasm_i32x4_add( + wasm_i32x4_dot_i16x8(v0s_l, ylo_l), + wasm_i32x4_dot_i16x8(v0s_h, ylo_h)), + wasm_i32x4_add( + wasm_i32x4_dot_i16x8(v1s_l, yhi_l), + wasm_i32x4_dot_i16x8(v1s_h, yhi_h))); + + sumv = wasm_f32x4_add(sumv, + wasm_f32x4_mul( + wasm_f32x4_convert_i32x4(acc), + wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d)))); + } + + sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) + + wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs; + + *s = sumf; + +#else + UNUSED(nb); + UNUSED(x); + UNUSED(y); + UNUSED(sumf); + + ggml_vec_dot_q4_1_q8_1_generic( + n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index 74d699f..21cd631 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -274,6 +274,18 @@ static inline __m256 quad_mx_delta_float(const uint8_t x0, const float y0, const } #endif #elif defined(__SSSE3__) +static inline __m128i bytes_from_bits_16(const uint8_t * x) { + uint16_t x16; + memcpy(&x16, x, sizeof(uint16_t)); + + const __m128i shuf_mask = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000); + __m128i bytes = _mm_shuffle_epi8(_mm_set1_epi16((short) x16), shuf_mask); + const __m128i bit_mask = _mm_set_epi64x(0x7fbfdfeff7fbfdfe, 0x7fbfdfeff7fbfdfe); + bytes = _mm_or_si128(bytes, bit_mask); + + return _mm_cmpeq_epi8(bytes, _mm_set1_epi64x(-1)); +} + // horizontally add 4x4 floats static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) { __m128 res_0 =_mm_hadd_ps(a, b); @@ -540,6 +552,216 @@ static inline __m128i get_scale_shuffle(int i) { } #endif +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK1_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + +#if defined(__AVX2__) + const __m256i ones_8 = _mm256_set1_epi8(1); + const __m256i ones_16 = _mm256_set1_epi16(1); + const __m256i byte_shuf = _mm256_setr_epi8( + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3); + const __m256i bit_masks = _mm256_setr_epi8( + 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128, + 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128); + const __m256i zero = _mm256_setzero_si256(); + __m256 acc = _mm256_setzero_ps(); + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); + const uint32_t * GGML_RESTRICT qs32 = (const uint32_t *) x[ib].qs; + const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; + + __m256 acc_block; + { + const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[0].qs); + const __m256i sm = _mm256_cmpeq_epi8( + _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[0]), byte_shuf), bit_masks), zero); + const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm); + const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); + acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), _mm256_cvtepi32_ps(s32)); + } + for (int K = 1; K < 4; ++K) { + const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[K].qs); + const __m256i sm = _mm256_cmpeq_epi8( + _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[K]), byte_shuf), bit_masks), zero); + const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm); + const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); + acc_block = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[K].d)), _mm256_cvtepi32_ps(s32), acc_block); + } + acc = _mm256_fmadd_ps(_mm256_set1_ps(d0), acc_block, acc); + } + + *s = hsum_float_8(acc); +#elif defined(__AVX__) + const __m128i ones_8 = _mm_set1_epi8(1); + const __m128i ones_16 = _mm_set1_epi16(1); + const __m128i zero = _mm_setzero_si128(); + __m256 acc = _mm256_setzero_ps(); + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); + const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; + __m256 acc_block; + { + const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[0]); + const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask); + const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1); + const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[0]); + const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[16]); + const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); + const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); + const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); + const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); + const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0); + const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1); + const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16); + const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16); + const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0)); + acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), q); + } + for(int K = 1; K < 4; ++K) { + const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[(K) * 4]); + const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask); + const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1); + const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[0]); + const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[16]); + const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); + const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); + const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); + const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); + const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0); + const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1); + const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16); + const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16); + const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0)); + acc_block = _mm256_add_ps(acc_block, _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(K)].d)), q)); + } +#undef Q1_AVX_BLOCK + + acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_set1_ps(d0), acc_block)); + } + + *s = hsum_float_8(acc); +#elif defined(__SSSE3__) + const __m128i ones_8 = _mm_set1_epi8(1); + const __m128i ones_16 = _mm_set1_epi16(1); + const __m128i zero = _mm_setzero_si128(); + __m128 acc_0 = _mm_setzero_ps(); + __m128 acc_1 = _mm_setzero_ps(); + __m128 acc_2 = _mm_setzero_ps(); + __m128 acc_3 = _mm_setzero_ps(); + + for (int ib = 0; ib < nb; ++ib) { + const __m128 d0 = _mm_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d)); + const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; + +#define Q1_SSSE3_BLOCK(QS_OFF, Y_IDX, ACC) \ + { \ + const __m128i bit_mask_0 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 0]); \ + const __m128i bit_mask_1 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 2]); \ + const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[0]); \ + const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[16]); \ + const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); \ + const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); \ + const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); \ + const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); \ + const __m128i sum_0 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_0), ones_16); \ + const __m128i sum_1 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_1), ones_16); \ + const __m128 q = _mm_cvtepi32_ps(_mm_add_epi32(sum_0, sum_1)); \ + (ACC) = _mm_add_ps((ACC), _mm_mul_ps(_mm_mul_ps(d0, _mm_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(Y_IDX)].d))), q)); \ + } + Q1_SSSE3_BLOCK(0, 0, acc_0) + Q1_SSSE3_BLOCK(4, 1, acc_1) + Q1_SSSE3_BLOCK(8, 2, acc_2) + Q1_SSSE3_BLOCK(12, 3, acc_3) +#undef Q1_SSSE3_BLOCK + } + + *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3); +#else + UNUSED(nb); + UNUSED(x); + UNUSED(y); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + +#if defined(__AVX2__) +// four consecutive Q2_0 weights per byte: byte b becomes b & 3, (b >> 2) & 3, (b >> 4) & 3, (b >> 6) & 3 (OR, never add: the overlapping copies only touch masked-off bits) +static inline __m256i q2_0_unpack_32_avx2(const uint8_t * qs) { + __m256i v = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *) qs)); + v = _mm256_or_si256(v, _mm256_slli_epi32(v, 12)); + v = _mm256_or_si256(v, _mm256_slli_epi32(v, 6)); + return _mm256_and_si256(v, _mm256_set1_epi32(0x03030303)); +} + +// 32 weights against one Q8_0 block: sum((v - 1) * y) = sum(v * y) - sum(y), exact for every int8 y +static inline __m256 q2_0_dot_32_avx2(const uint8_t * qs, const int8_t * qy) { + const __m256i v = q2_0_unpack_32_avx2(qs); + const __m256i y = _mm256_loadu_si256((const __m256i *) qy); + const __m256i p = _mm256_sub_epi16(_mm256_maddubs_epi16(v, y), _mm256_maddubs_epi16(_mm256_set1_epi8(1), y)); + return _mm256_cvtepi32_ps(_mm256_madd_epi16(p, _mm256_set1_epi16(1))); +} + +static inline __m256 q2_0_acc_block_avx2(const block_q2_0 * x, const block_q8_0 * y, __m256 acc) { + const float dx = GGML_CPU_FP16_TO_FP32(x->d); + const __m256 d0 = _mm256_set1_ps(dx * GGML_CPU_FP16_TO_FP32(y[0].d)); + const __m256 d1 = _mm256_set1_ps(dx * GGML_CPU_FP16_TO_FP32(y[1].d)); + const __m256 p0 = q2_0_dot_32_avx2(x->qs + 0, y[0].qs); + const __m256 p1 = q2_0_dot_32_avx2(x->qs + 8, y[1].qs); +#if defined(__FMA__) + acc = _mm256_fmadd_ps(d0, p0, acc); + return _mm256_fmadd_ps(d1, p1, acc); +#else + acc = _mm256_add_ps(acc, _mm256_mul_ps(d0, p0)); + return _mm256_add_ps(acc, _mm256_mul_ps(d1, p1)); +#endif +} +#endif + +void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +#if defined(__AVX2__) + assert(n % QK2_0 == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + const int nb = n / QK2_0; + + // two Q2_0 blocks per step, each into its own accumulator (two FMA chains) + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + int i = 0; + for (; i + 1 < nb; i += 2) { + acc0 = q2_0_acc_block_avx2(x + i + 0, y + 2*i + 0, acc0); + acc1 = q2_0_acc_block_avx2(x + i + 1, y + 2*i + 2, acc1); + } + if (i < nb) { + acc0 = q2_0_acc_block_avx2(x + i, y + 2*i, acc0); + } + + *s = hsum_float_8(_mm256_add_ps(acc0, acc1)); +#else + ggml_vec_dot_q2_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -776,7 +998,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo #if defined __AVX2__ - const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4); + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); const __m128i m4b = _mm_set1_epi8(0x0f); const __m256i mone = _mm256_set1_epi16(1); @@ -805,7 +1027,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo sumf = hsum_float_8(_mm256_add_ps(accum1, accum2)); #elif defined __AVX__ - const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4); + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); const __m128i m4b = _mm_set1_epi8(0x0f); __m256 accum = _mm256_setzero_ps(); @@ -835,14 +1057,152 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo int sumi1 = 0; int sumi2 = 0; for (int j = 0; j < QK_MXFP4/2; ++j) { - sumi1 += y[ib].qs[j + 0] * kvalues_mxfp4[x[ib].qs[j] & 0xf]; - sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >> 4]; + sumi1 += y[ib].qs[j + 0] * kvalues_fp4[x[ib].qs[j] & 0xf]; + sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_fp4[x[ib].qs[j] >> 4]; } sumf += d * (sumi1 + sumi2); } *s = sumf; } +void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK_NVFP4 == 0); + + const block_nvfp4 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + const int nb = n / QK_NVFP4; + int ib = 0; + float sumf = 0; + +#if defined(__AVX2__) + + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); + const __m128i m4b = _mm_set1_epi8(0x0f); + const __m256i mone = _mm256_set1_epi16(1); + + __m256 accum = _mm256_setzero_ps(); + for(; ib < nb; ib++){ + + const __m128i q4bits_01 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 0)); + const __m128i q4bits_23 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 16)); + + const __m256i q8_01 = _mm256_loadu_si256((const __m256i *)y[2*ib + 0].qs); + const __m256i q8_23 = _mm256_loadu_si256((const __m256i *)y[2*ib + 1].qs); + + const __m128i q4_01_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_01, m4b)); + const __m128i q4_01_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_01, 4), m4b)); + const __m128i q4_23_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_23, m4b)); + const __m128i q4_23_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_23, 4), m4b)); + + //reordering + const __m256i q4_01 = MM256_SET_M128I(_mm_unpackhi_epi64(q4_01_lo,q4_01_hi), _mm_unpacklo_epi64(q4_01_lo,q4_01_hi)); + const __m256i q4_23 = MM256_SET_M128I(_mm_unpackhi_epi64(q4_23_lo,q4_23_hi),_mm_unpacklo_epi64(q4_23_lo,q4_23_hi)); + + const __m256i p01 = mul_add_epi8(q4_01,q8_01); + const __m256i p_1 = _mm256_madd_epi16(p01, mone); + + const __m256i p23 = mul_add_epi8(q4_23,q8_23); + const __m256i p_2 = _mm256_madd_epi16(p23, mone); + + const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); + const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); + + const float s0 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]) * dy0; + const float s1 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]) * dy0; + const float s2 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]) * dy1; + const float s3 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) * dy1; + + const __m256 scales01 = _mm256_set_m128(_mm_set1_ps(s1), _mm_set1_ps(s0)); + const __m256 scales23 = _mm256_set_m128(_mm_set1_ps(s3), _mm_set1_ps(s2)); + + accum = _mm256_fmadd_ps(scales01, _mm256_cvtepi32_ps(p_1), accum); + accum = _mm256_fmadd_ps(scales23, _mm256_cvtepi32_ps(p_2), accum); + } + sumf = hsum_float_8(accum); + +#elif defined(__AVX__) + + const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_fp4); + const __m128i m4b = _mm_set1_epi8(0x0f); + + __m256 accum = _mm256_setzero_ps(); + for(; ib < nb; ib++){ + + const __m128i q4bits_01 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 0)); + const __m128i q4bits_23 = _mm_loadu_si128((const __m128i *)(x[ib].qs + 16)); + + const __m128i q8_0 = _mm_loadu_si128((const __m128i *)(y[2*ib + 0].qs + 0)); + const __m128i q8_1 = _mm_loadu_si128((const __m128i *)(y[2*ib + 0].qs + 16)); + const __m128i q8_2 = _mm_loadu_si128((const __m128i *)(y[2*ib + 1].qs + 0)); + const __m128i q8_3 = _mm_loadu_si128((const __m128i *)(y[2*ib + 1].qs + 16)); + + const __m128i q4_01_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_01, m4b)); + const __m128i q4_01_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_01, 4), m4b)); + const __m128i q4_23_lo = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_23, m4b)); + const __m128i q4_23_hi = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_23, 4), m4b)); + + const __m128i q4_0 = _mm_unpacklo_epi64(q4_01_lo, q4_01_hi); + const __m128i q4_1 = _mm_unpackhi_epi64(q4_01_lo, q4_01_hi); + const __m128i q4_2 = _mm_unpacklo_epi64(q4_23_lo, q4_23_hi); + const __m128i q4_3 = _mm_unpackhi_epi64(q4_23_lo, q4_23_hi); + + const __m128i p0_i32 = mul_sum_i8_pairs(q4_0, q8_0); + const __m128i p1_i32 = mul_sum_i8_pairs(q4_1, q8_1); + const __m128i p2_i32 = mul_sum_i8_pairs(q4_2, q8_2); + const __m128i p3_i32 = mul_sum_i8_pairs(q4_3, q8_3); + + const __m128 p0 = _mm_cvtepi32_ps(p0_i32); + const __m128 p1 = _mm_cvtepi32_ps(p1_i32); + const __m128 p2 = _mm_cvtepi32_ps(p2_i32); + const __m128 p3 = _mm_cvtepi32_ps(p3_i32); + + const __m256 p01 = _mm256_set_m128(p1, p0); + const __m256 p23 = _mm256_set_m128(p3, p2); + + const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); + const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); + + const float s0 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]) * dy0; + const float s1 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]) * dy0; + const float s2 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]) * dy1; + const float s3 = GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) * dy1; + + const __m256 scales01 = _mm256_set_m128(_mm_set1_ps(s1), _mm_set1_ps(s0)); + const __m256 scales23 = _mm256_set_m128(_mm_set1_ps(s3), _mm_set1_ps(s2)); + + accum = _mm256_add_ps(accum, _mm256_mul_ps(p01, scales01)); + accum = _mm256_add_ps(accum, _mm256_mul_ps(p23, scales23)); + } + sumf = hsum_float_8(accum); + +#endif + + for (;ib < nb; ++ib) { + for (int s_idx = 0; s_idx < 4; ++s_idx) { + const float d = GGML_CPU_UE4M3_TO_FP32(x[ib].d[s_idx]); + const int q8_block = s_idx / 2; + const int q8_off = (s_idx % 2) * QK_NVFP4_SUB; + const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8_block].d); + + int sumi_lo = 0, sumi_hi = 0; + for (int j = 0; j < QK_NVFP4_SUB/2; ++j) { + const uint8_t qv = x[ib].qs[s_idx*(QK_NVFP4_SUB/2) + j]; + sumi_lo += y[2*ib + q8_block].qs[q8_off + j + 0] * kvalues_fp4[qv & 0xf]; + sumi_hi += y[2*ib + q8_block].qs[q8_off + j + QK_NVFP4_SUB/2] * kvalues_fp4[qv >> 4]; + } + + sumf += dy * d * (sumi_lo + sumi_hi); + } + } + *s = sumf; +} + void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; @@ -2142,9 +2502,8 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi #if defined __AVX2__ - const __m256i m4 = _mm256_set1_epi8(0xF); - const __m256i m2 = _mm256_set1_epi8(3); - const __m256i m32s = _mm256_set1_epi8(32); + const __m256i m3 = _mm256_set1_epi8(3); + const __m256i m15 = _mm256_set1_epi8(15); __m256 acc = _mm256_setzero_ps(); @@ -2156,53 +2515,45 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi const uint8_t * GGML_RESTRICT qh = x[i].qh; const int8_t * GGML_RESTRICT q8 = y[i].qs; + const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums); const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales); + const __m256i scales_16 = _mm256_cvtepi8_epi16(scales); + const __m256i q8sclsub = _mm256_slli_epi32(_mm256_madd_epi16(q8sums, scales_16), 5); __m256i sumi = _mm256_setzero_si256(); int is = 0; for (int j = 0; j < QK_K/128; ++j) { - - const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0)); - const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1)); - const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2)); - const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3)); - is += 4; - const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32; const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32; const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32; - const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4); - const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4); - const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4); - const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4); + const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m3), 4); + const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, _mm256_set1_epi8(12)), 2); + const __m256i q4h_2 = _mm256_and_si256(q4bitsH, _mm256_set1_epi8(48)); + const __m256i q4h_3 = _mm256_srli_epi16(_mm256_and_si256(q4bitsH, _mm256_set1_epi8(-64)), 2); - const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0); - const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1); - const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2); - const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3); + const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m15), q4h_0); + const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m15), q4h_1); + const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m15), q4h_2); + const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m15), q4h_3); const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32; const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32; const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32; const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32; - __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0); - __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1); - __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2); - __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3); - __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0); __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1); __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2); __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3); - p16_0 = _mm256_sub_epi16(p16_0, q8s_0); - p16_1 = _mm256_sub_epi16(p16_1, q8s_1); - p16_2 = _mm256_sub_epi16(p16_2, q8s_2); - p16_3 = _mm256_sub_epi16(p16_3, q8s_3); + const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0)); + const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1)); + const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2)); + const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3)); + is += 4; p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0); p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1); @@ -2214,6 +2565,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi } + sumi = _mm256_sub_epi32(sumi, q8sclsub); acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); } diff --git a/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake b/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake new file mode 100644 index 0000000..c8a4d4b --- /dev/null +++ b/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake @@ -0,0 +1,32 @@ +include(CheckCSourceRuns) + +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv)" AND GGML_CPU_RISCV64_SPACEMIT) + set(SMT_MARCH_STR "-march=rv64gcv_zfh_zvfh_zba_zicbop") + if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND + CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 15) + string(APPEND SMT_MARCH_STR "_xsmtvdotii") + endif() + set(CMAKE_REQUIRED_FLAGS "${SMT_MARCH_STR}") + + check_c_source_compiles("int main() {__asm__ volatile(\"vmadot v2, v0, v1\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_IME1) + check_c_source_compiles("int main() {__asm__ volatile(\"vmadot v2, v0, v1, i4\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VMADOT_S4) + check_c_source_compiles("int main() {__asm__ volatile(\"vmadot v2, v0, v1, i8\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VMADOT_S8) + check_c_source_compiles("int main() {__asm__ volatile(\"vfwmadot v2, v0, v1, fp16\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VFWMADOT_FP16) + check_c_source_compiles("int main() {__asm__ volatile(\"vmadot.hp v2, v0, v1, v0, 0, i4\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VFMADOT_S4) + check_c_source_compiles("int main() {__asm__ volatile(\"vmadot.hp v2, v0, v1, v0, 0, i8\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VFMADOT_S8) + check_c_source_compiles("int main() {__asm__ volatile(\"vmadot1 v2, v0, v1\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VMADOTN) + check_c_source_compiles("int main() {__asm__ volatile(\"vpack.vv v2, v0, v1, 2\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VPACK) + check_c_source_compiles("int main() {__asm__ volatile(\"vnspack.vv v2, v0, v1, 2\");}" SPACEMIT_RISCV_COMPILER_SUPPORT_VNPACK) + unset(CMAKE_REQUIRED_FLAGS) + + list(APPEND RISCV64_SPACEMIT_IME_SPEC "") + if (SPACEMIT_RISCV_COMPILER_SUPPORT_IME1) + set(RISCV64_SPACEMIT_IME_SPEC "RISCV64_SPACEMIT_IME1") + endif() + + if (SPACEMIT_RISCV_COMPILER_SUPPORT_VMADOT_S4 AND SPACEMIT_RISCV_COMPILER_SUPPORT_VPACK AND SPACEMIT_RISCV_COMPILER_SUPPORT_VNPACK) + list(APPEND RISCV64_SPACEMIT_IME_SPEC "RISCV64_SPACEMIT_IME2") + endif() + + message("RISCV64_SPACEMIT_IME_SPEC: ${RISCV64_SPACEMIT_IME_SPEC}") +endif() diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 5d1ca5f..0d48c9a 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -78,7 +78,7 @@ struct ggml_compute_params { #if defined(__ARM_NEON) // ref: https://github.com/ggml-org/llama.cpp/pull/5404 -#ifdef _MSC_VER +#if defined(_MSC_VER) && !defined(__clang__) #define ggml_vld1q_u32(w,x,y,z) { ((w) + ((uint64_t)(x) << 32)), ((y) + ((uint64_t)(z) << 32)) } #else #define ggml_vld1q_u32(w,x,y,z) { (w), (x), (y), (z) } @@ -390,17 +390,16 @@ typedef unsigned char uchar8x16_t __attribute__((vector_size(16))); typedef int8_t int8x16_t __attribute__((vector_size(16))); typedef int16_t int16x8_t __attribute__((vector_size(16))); typedef int32_t int32x4_t __attribute__((vector_size(16))); +typedef int64_t int64x2_t __attribute__((vector_size(16))); typedef uint8_t uint8x16_t __attribute__((vector_size(16))); typedef uint16_t uint16x8_t __attribute__((vector_size(16))); typedef uint32_t uint32x4_t __attribute__((vector_size(16))); +typedef uint64_t uint64x2_t __attribute__((vector_size(16))); typedef float float32x4_t __attribute__((vector_size(16))); typedef double double64x2_t __attribute__((vector_size(16))); -typedef signed long long long64x2_t __attribute__((vector_size(16))); -typedef unsigned long long ulong64x2_t __attribute__((vector_size(16))); - typedef struct ggml_uint8x16x2_t { uint8x16_t val[2]; } ggml_uint8x16x2_t; @@ -444,6 +443,13 @@ inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) { return res; } +// load 4 bytes and replicate them into all 4 word lanes +inline static int8x16_t ggml_vec_xl_s8x4_splat(const int8_t * ptr) { + uint32_t tmp; + memcpy(&tmp, ptr, sizeof(tmp)); + return (int8x16_t)vec_splats(tmp); +} + typedef struct ggml_int16x8x2_t { int16x8_t val[2]; } ggml_int16x8x2_t; @@ -457,31 +463,17 @@ inline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) { return res; } -/* - ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs - ! or iq4_nl for example implementation. -*/ -inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { - int8x16_t res; - - res[ 0] = a[b[ 0]]; - res[ 1] = a[b[ 1]]; - res[ 2] = a[b[ 2]]; - res[ 3] = a[b[ 3]]; - res[ 4] = a[b[ 4]]; - res[ 5] = a[b[ 5]]; - res[ 6] = a[b[ 6]]; - res[ 7] = a[b[ 7]]; - res[ 8] = a[b[ 8]]; - res[ 9] = a[b[ 9]]; - res[10] = a[b[10]]; - res[11] = a[b[11]]; - res[12] = a[b[12]]; - res[13] = a[b[13]]; - res[14] = a[b[14]]; - res[15] = a[b[15]]; - - return res; +// faster fp16 -> fp32 conversion using vector intrinsics +// WARNING: fp16 inf converts into 65536.0f. use this only for finite values. +inline static float32x4_t ggml_vec_xl_f16x4_finite(const ggml_fp16_t * x) { + uint64_t tmp; + memcpy(&tmp, x, sizeof(tmp)); + + const uint64x2_t v_x = { tmp, tmp }; + const uint32x4_t v_xh = vec_unpackh((uint16x8_t)v_x); + const uint32x4_t v_bits = vec_sl(vec_and(v_xh, vec_splats((uint32_t)0x7FFF)), 13); + const float32x4_t v_abs = vec_mul((float32x4_t)v_bits, vec_splats(0x1.0p+112f)); + return (float32x4_t)vec_or((uint32x4_t)v_abs, vec_sl(vec_and(v_xh, vec_splats((uint32_t)0x8000)), 16)); } inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) { @@ -493,6 +485,11 @@ inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) { return v_abo + v_abe; } +inline static int32x4_t vec_paddl_s16(int16x8_t a) { + const int16x8_t v_ones = vec_splats((int16_t)1); + return vec_mule(a, v_ones) + vec_mulo(a, v_ones); +} + /** * @see https://github.com/ggml-org/llama.cpp/pull/14037 */ @@ -506,7 +503,11 @@ inline static int32_t vec_hsum_i32x4(int32x4_t v) { return v_temp[0] + v_temp[1]; } -inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) { +inline static int16x8_t ggml_vec_dot_s16(int16x8_t acc, int8x16_t a, int8x16_t b) { + return vec_meadd(a, b, vec_moadd(a, b, acc)); +} + +inline static int32x4_t ggml_vec_dot_s32(int32x4_t acc, int8x16_t a, int8x16_t b) { const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b); return acc + (vec_unpackh(p) + vec_unpackl(p)); } diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 6d9212f..75f32c0 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -14,6 +14,7 @@ #include "ops.h" #include "ggml.h" #include "common.h" +#include "tiled/tiled.h" #if defined(_MSC_VER) || defined(__MINGW32__) #include // using malloc.h with MSC/MINGW @@ -50,6 +51,10 @@ #include "llamafile/sgemm.h" #endif +#ifdef GGML_USE_CPU_RISCV64_SPACEMIT +# include "spacemit/ime.h" +#endif + // Note: once we move threading into a separate C++ file // will use std::hardware_destructive_interference_size instead of hardcoding it here // and we'll use C++ attribute syntax. @@ -78,6 +83,9 @@ float ggml_table_f32_f16[1 << 16]; // precomputed f32 table for e8m0 half (1 KB) (simd-mappings.h) float ggml_table_f32_e8m0_half[1 << 8]; +// precomputed f32 table for ue4m3 (1 KB) (simd-mappings.h) +float ggml_table_f32_ue4m3[1 << 8]; + #if defined(__ARM_ARCH) struct ggml_arm_arch_features_type { int sve_cnt; @@ -223,6 +231,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, + [GGML_TYPE_Q2_0] = { + .from_float = quantize_row_q2_0, + .vec_dot = ggml_vec_dot_q2_0_q8_0, + .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, + }, [GGML_TYPE_Q4_0] = { .from_float = quantize_row_q4_0, .vec_dot = ggml_vec_dot_q4_0_q8_0, @@ -445,7 +459,7 @@ typedef pthread_mutex_t ggml_mutex_t; #define ggml_lock_init(x) UNUSED(x) #define ggml_lock_destroy(x) UNUSED(x) -#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64)) +#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64) && !defined(_M_ARM64EC)) #define ggml_lock_lock(x) _mm_pause() #else #define ggml_lock_lock(x) UNUSED(x) @@ -1245,6 +1259,17 @@ void ggml_compute_forward_mul_mat( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; + const int32_t hint = ggml_get_op_params_i32(dst, 1); + if (hint == GGML_HINT_SRC0_IS_HADAMARD && !params->use_ref) { + ggml_compute_forward_fwht(params, dst); + return; + } + + // If tiled is supported, it will execute the full op here and we return + if (ggml_compute_forward_mul_mat_tiled(params, dst)) { + return; + } + GGML_TENSOR_BINARY_OP_LOCALS const int ith = params->ith; @@ -1280,7 +1305,7 @@ void ggml_compute_forward_mul_mat( const bool src1_cont = ggml_is_contiguous(src1); - if (src1_cont) { + if (!params->use_ref && src1_cont) { bool llamafile_all_ok = true; for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) @@ -1313,7 +1338,11 @@ UseGgmlGemm1:; const size_t nbw3 = nbw2*ne12; assert(params->wsize >= ne13*nbw3); - GGML_ASSERT(src1->type == GGML_TYPE_F32); + // src1 is either packed from F32 into vec_dot_type, or widened from F16 or BF16 into the F32 work buffer + const bool widen = src1->type != GGML_TYPE_F32; + + GGML_ASSERT(!widen || vec_dot_type == GGML_TYPE_F32); + GGML_ASSERT(!widen || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16); #if 0 for (int64_t i13 = 0; i13 < ne13; ++i13) { @@ -1326,15 +1355,25 @@ UseGgmlGemm1:; } } #else + const int64_t bs = ggml_blck_size(vec_dot_type); + for (int64_t i13 = 0; i13 < ne13; ++i13) { for (int64_t i12 = 0; i12 < ne12; ++i12) { for (int64_t i11 = 0; i11 < ne11; ++i11) { - size_t bs = ggml_blck_size(vec_dot_type); int64_t ne10_block_start = (ith * ne10/bs) / nth; int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth; - from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10), - (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0), - (ne10_block_end - ne10_block_start) * bs); + const void * src1_row = (const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10; + void * wdata_row = wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0; + + const int64_t ne10_block_size = (ne10_block_end - ne10_block_start) * bs; + + if (src1->type == GGML_TYPE_F16) { + ggml_cpu_fp16_to_fp32((const ggml_fp16_t *) src1_row, (float *) wdata_row, ne10_block_size); + } else if (src1->type == GGML_TYPE_BF16) { + ggml_cpu_bf16_to_fp32((const ggml_bf16_t *) src1_row, (float *) wdata_row, ne10_block_size); + } else { + from_float((const float *) src1_row, wdata_row, ne10_block_size); + } } } } @@ -1349,7 +1388,7 @@ UseGgmlGemm1:; ggml_barrier(params->threadpool); #if GGML_USE_LLAMAFILE - if (src1->type != vec_dot_type) { + if (!params->use_ref && src1->type != vec_dot_type) { const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); @@ -1565,6 +1604,10 @@ static void ggml_compute_forward_mul_mat_id( char (*atomic_current_chunk)[CACHE_LINE_SIZE] = // [n_as] incr_ptr_aligned(&wdata_cur, CACHE_LINE_SIZE * n_as, CACHE_LINE_SIZE); + // Tiled matmul (see tiled.h); per-thread work buffers, 0 bytes when disabled. The + // reservation is unconditional, the per expert eligibility is decided at dispatch time + char * tiled_scratch = incr_ptr_aligned(&wdata_cur, ggml_tiled_wdata_size(nth, dst), 64); + GGML_ASSERT(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata)); if (src1->type != vec_dot_type) { @@ -1613,7 +1656,12 @@ static void ggml_compute_forward_mul_mat_id( for (int id = 0; id < n_ids; ++id) { const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]); - assert(i02 >= 0 && i02 < n_as); + assert(i02 == -1 || (i02 >= 0 && i02 < n_as)); + + if (i02 == -1) { + memset((char *) dst->data + id*nb1 + iid1*nb2, 0, ne0*sizeof(float)); + continue; + } MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) {id, iid1}; matrix_row_counts[i02] += 1; @@ -1636,6 +1684,11 @@ static void ggml_compute_forward_mul_mat_id( continue; } + // tiled takes over if profitable for this expert (see tiled.h) + if (ggml_compute_forward_mul_mat_id_tiled(params, dst, cur_a, cne1, (const int32_t *) &MMID_MATRIX_ROW(cur_a, 0), tiled_scratch)) { + continue; + } + const char * src0_cur = (const char *) src0->data + cur_a * nb02; const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); @@ -1930,6 +1983,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_im2col_3d(params, tensor); } break; + case GGML_OP_COL2IM_1D: + { + ggml_compute_forward_col2im_1d(params, tensor); + } break; case GGML_OP_CONV_2D: { ggml_compute_forward_conv_2d(params, tensor); @@ -2065,6 +2122,26 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_gated_delta_net(params, tensor); } break; + case GGML_OP_LIGHTNING_INDEXER: + { + ggml_compute_forward_lightning_indexer(params, tensor); + } break; + case GGML_OP_DSV4_HC_COMB: + { + ggml_compute_forward_dsv4_hc_comb(params, tensor); + } break; + case GGML_OP_DSV4_HC_PRE: + { + ggml_compute_forward_dsv4_hc_pre(params, tensor); + } break; + case GGML_OP_DSV4_HC_POST: + { + ggml_compute_forward_dsv4_hc_post(params, tensor); + } break; + case GGML_OP_TOP_K_MASK: + { + ggml_compute_forward_top_k_mask(params, tensor); + } break; case GGML_OP_MAP_CUSTOM1: { ggml_compute_forward_map_custom1(params, tensor); @@ -2245,6 +2322,10 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_COUNT_EQUAL: case GGML_OP_SOLVE_TRI: case GGML_OP_GATED_DELTA_NET: + case GGML_OP_DSV4_HC_COMB: + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + case GGML_OP_TOP_K_MASK: { n_tasks = n_threads; } break; @@ -2297,6 +2378,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_GLU_OP_SWIGLU_OAI: case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: + case GGML_GLU_OP_SWIGLU_CLAMP: { n_tasks = n_threads; } break; @@ -2316,11 +2398,11 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: case GGML_OP_OUT_PROD: + case GGML_OP_SET_ROWS: { n_tasks = n_threads; } break; case GGML_OP_GET_ROWS: - case GGML_OP_SET_ROWS: { // FIXME: get_rows can use additional threads, but the cost of launching additional threads // decreases performance with GPU offloading @@ -2361,6 +2443,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_CONV_2D: case GGML_OP_CONV_3D: case GGML_OP_CONV_2D_DW: + case GGML_OP_COL2IM_1D: case GGML_OP_CONV_TRANSPOSE_1D: case GGML_OP_CONV_TRANSPOSE_2D: { @@ -2384,6 +2467,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_FLASH_ATTN_BACK: case GGML_OP_SSM_CONV: case GGML_OP_SSM_SCAN: + case GGML_OP_LIGHTNING_INDEXER: { n_tasks = n_threads; } break; @@ -2614,7 +2698,7 @@ static bool ggml_thread_apply_priority(int32_t prio) { return true; } -#elif defined(__gnu_linux__) +#elif defined(__linux__) // TODO: this may not work on BSD, to be verified static bool ggml_thread_apply_affinity(const bool * mask) { @@ -2801,6 +2885,11 @@ struct ggml_cplan ggml_graph_plan( n_threads = 1; #endif +#if defined(__wasi__) + // WASI doesn't support parallelism yet + n_threads = 1; +#endif + size_t work_size = 0; struct ggml_cplan cplan; @@ -2854,10 +2943,12 @@ struct ggml_cplan ggml_graph_plan( case GGML_OP_MUL_MAT: { const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type; - if (node->src[1]->type != vec_dot_type) { cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1])); } + // Workspace for tiled (see tiled.h) + cur = GGML_PAD(cur, 64); + cur += ggml_tiled_wdata_size(n_tasks, node); } break; case GGML_OP_MUL_MAT_ID: { @@ -2877,10 +2968,20 @@ struct ggml_cplan ggml_graph_plan( cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t); // atomic_current_chunk cur += CACHE_LINE_SIZE*n_as + CACHE_LINE_SIZE; + // Workspace for tiled (see tiled.h) + cur = GGML_PAD(cur, 64); + cur += ggml_tiled_wdata_size(n_tasks, node); } break; case GGML_OP_OUT_PROD: { - if (ggml_is_quantized(node->src[0]->type)) { + if (ggml_is_quantized(node->src[0]->type) || + node->src[0]->type == GGML_TYPE_F16) { + cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; + } + } break; + case GGML_OP_SET_ROWS: + { + if (node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) { cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; } } break; @@ -2930,12 +3031,13 @@ struct ggml_cplan ggml_graph_plan( const int64_t ne10 = node->src[1]->ne[0]; // W const int64_t ne11 = node->src[1]->ne[1]; // H const int64_t ne12 = node->src[1]->ne[2]; // Channels In + const int64_t ne13 = node->src[1]->ne[3]; // Batch GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32); cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03; - cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12; + cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12 * ne13; } break; case GGML_OP_TOP_K: @@ -2983,12 +3085,20 @@ struct ggml_cplan ggml_graph_plan( case GGML_OP_GATED_DELTA_NET: { const int64_t S_v = node->src[2]->ne[0]; - cur = S_v * sizeof(float) * n_tasks; + const int64_t K = ggml_get_op_params_i32(node, 0); + const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0); + cur = per_thread * sizeof(float) * n_tasks; } break; case GGML_OP_COUNT: { GGML_ABORT("fatal error"); } + case GGML_OP_LIGHTNING_INDEXER: + { + // temp buffer for dequantizing lightning indexer keys + const int64_t ne10 = node->src[1]->ne[0]; + cur += sizeof(float)*ne10*n_tasks; + } break; default: break; } @@ -3009,6 +3119,45 @@ struct ggml_cplan ggml_graph_plan( return cplan; } + +// Try to fuse the current node with subsequent nodes for better performance. +// Returns the number of nodes skipped by fusion (>=1), or 0 if no fusion was applied. +static bool ggml_cpu_disable_fusion = false; // initialized once in ggml_cpu_init(), read-only afterwards + +static int ggml_cpu_try_fuse_ops( + const struct ggml_cgraph * cgraph, + const int node_n, + const struct ggml_compute_params * params, + const struct ggml_cplan * cplan) { + + if (ggml_cpu_disable_fusion || cplan->use_ref) { + return 0; + } + + struct ggml_tensor * node = cgraph->nodes[node_n]; + + if (node->op == GGML_OP_RMS_NORM) { + // RMS_NORM + MUL fusion + const enum ggml_op fuse_ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL }; + if (ggml_can_fuse(cgraph, node_n, fuse_ops, 2)) { + struct ggml_tensor * mul_node = cgraph->nodes[node_n + 1]; + const struct ggml_tensor * mul_w = (mul_node->src[0] == node) + ? mul_node->src[1] : mul_node->src[0]; + if (node->src[0]->type == GGML_TYPE_F32 && + mul_node->type == GGML_TYPE_F32 && + mul_w->type == GGML_TYPE_F32 && + mul_w->ne[0] == node->ne[0] && + mul_w->nb[0] == sizeof(float)) { + + ggml_compute_forward_rms_norm_mul_fused(params, node, mul_node); + return 1; + } + } + } + + return 0; +} + static thread_ret_t ggml_graph_compute_thread(void * data) { struct ggml_compute_state * state = (struct ggml_compute_state *) data; struct ggml_threadpool * tp = state->threadpool; @@ -3016,7 +3165,11 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { const struct ggml_cgraph * cgraph = tp->cgraph; const struct ggml_cplan * cplan = tp->cplan; +#ifdef GGML_USE_CPU_RISCV64_SPACEMIT + ggml_backend_cpu_riscv64_spacemit_set_numa_thread_affinity(state->ith); +#else set_numa_thread_affinity(state->ith); +#endif struct ggml_compute_params params = { /*.ith =*/ state->ith, @@ -3045,7 +3198,14 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { continue; } - ggml_compute_forward(¶ms, node); + // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time + // Try fused ops, fall back to normal compute + const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, ¶ms, cplan); + if (n_fused > 0) { + node_n += n_fused; + } else { + ggml_compute_forward(¶ms, node); + } if (state->ith == 0 && cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) { @@ -3066,6 +3226,10 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { ggml_barrier(state->threadpool); +#ifdef GGML_USE_CPU_RISCV64_SPACEMIT + ggml_backend_cpu_riscv64_spacemit_clear_numa_thread_affinity_threaded(state->ith); +#endif + return 0; } @@ -3400,6 +3564,12 @@ void ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) { vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl); __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl); } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 7 < n; i += 8) { + const uint32x4_t v_yl = __lzs_f32cx4_to_f16(vec_xl(0, x + i + 0)); + const uint32x4_t v_yh = __lzs_f32cx4_to_f16(vec_xl(0, x + i + 4)); + vec_xst(vec_pack(v_yl, v_yh), 0, (uint16_t *)(y + i)); + } #endif for (; i < n; ++i) { y[i] = GGML_CPU_FP32_TO_FP16(x[i]); @@ -3744,6 +3914,14 @@ int ggml_cpu_has_sme(void) { #endif } +int ggml_cpu_has_sme2(void) { +#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME2) + return 1; +#else + return 0; +#endif +} + void ggml_cpu_init(void) { // needed to initialize ggml_time { @@ -3777,6 +3955,11 @@ void ggml_cpu_init(void) { ggml_table_f32_e8m0_half[i] = GGML_E8M0_TO_FP32_HALF(i); } + // initialize UE4M3 table (256 entries) + for (int i = 0; i < (1 << 8); ++i) { + ggml_table_f32_ue4m3[i] = ggml_ue4m3_to_fp32(i); + } + const uint64_t t_end = ggml_time_us(); UNUSED(t_end); GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0); @@ -3807,6 +3990,11 @@ void ggml_cpu_init(void) { ggml_init_riscv_arch_features(); #endif + { + const char * env = getenv("GGML_CPU_DISABLE_FUSION"); + ggml_cpu_disable_fusion = (env != NULL && atoi(env) == 1); + } + is_first_call = false; } diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 49f840b..81ff5d7 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -195,8 +195,8 @@ static const struct ggml_backend_i ggml_backend_cpu_i = { /* .free = */ ggml_backend_cpu_free, /* .set_tensor_async = */ NULL, /* .get_tensor_async = */ NULL, - /* .get_tensor_2d_async = */ NULL, /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, /* .cpy_tensor_async = */ NULL, /* .synchronize = */ NULL, /* .graph_plan_create = */ ggml_backend_cpu_graph_plan_create, @@ -397,6 +397,7 @@ static void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggm /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ true, /* .events = */ false, + /* .mmap_support = */ true, }; } @@ -450,7 +451,14 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st op->type != GGML_TYPE_IQ1_S && op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float case GGML_OP_MUL_MAT: - return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type; + if (ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD && + src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) { + return src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16; + } + // BF16 in src1 is widened into the F32 work buffer + return src1->type == GGML_TYPE_F32 || + src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type || + (src1->type == GGML_TYPE_BF16 && ggml_get_type_traits_cpu(src0->type)->vec_dot_type == GGML_TYPE_F32); case GGML_OP_SOFT_MAX_BACK: { if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) { return false; @@ -462,12 +470,17 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st return max_bias == 0.0f; } case GGML_OP_IM2COL_BACK: - return src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32; + return src0->type == GGML_TYPE_F32 && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); case GGML_OP_GET_ROWS_BACK: return src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16; case GGML_OP_OUT_PROD: - return (src0->type == GGML_TYPE_F32 || (ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) && + return (src0->type == GGML_TYPE_F32 || + ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) && src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_CONV_2D: + return ggml_is_contiguous(op->src[0]); + case GGML_OP_SSM_SCAN: + return ggml_get_op_params_i32(op, 0) == 1 || op->src[3]->ne[0] == 1; default: return true; } @@ -594,6 +607,9 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r if (ggml_cpu_has_sme()) { features.push_back({ "SME", "1" }); } + if (ggml_cpu_has_sme2()) { + features.push_back({ "SME2", "1" }); + } if (ggml_cpu_has_riscv_v()) { features.push_back({ "RISCV_V", "1" }); } diff --git a/ggml/src/ggml-cpu/hbm.cpp b/ggml/src/ggml-cpu/hbm.cpp index a4073c1..0ca5c86 100644 --- a/ggml/src/ggml-cpu/hbm.cpp +++ b/ggml/src/ggml-cpu/hbm.cpp @@ -40,12 +40,14 @@ static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_ ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) { static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_hbm = { /* .iface = */ { - /* .get_name = */ ggml_backend_cpu_hbm_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment, - /* .get_max_size = */ nullptr, // defaults to SIZE_MAX - /* .get_alloc_size = */ nullptr, // defaults to ggml_nbytes - /* .is_host = */ ggml_backend_cpu_buffer_type_is_host, + /* .get_name = */ ggml_backend_cpu_hbm_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ nullptr, + /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment, + /* .get_max_size = */ nullptr, // defaults to SIZE_MAX + /* .get_alloc_size = */ nullptr, // defaults to ggml_nbytes + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_cpu_buffer_type_is_host, }, /* .context = */ nullptr, }; diff --git a/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt b/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt new file mode 100644 index 0000000..b36cb6d --- /dev/null +++ b/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt @@ -0,0 +1,14 @@ +set(BUILD_SHARED_LIBS OFF) +set(CMAKE_SKIP_INSTALL_RULES TRUE) + +add_subdirectory("${KLEIDIAI_SRC}" "${KLEIDIAI_BIN}" EXCLUDE_FROM_ALL) + +if (NOT TARGET kleidiai) + message(FATAL_ERROR "KleidiAI target was not created") +endif() + +if (MSVC) + target_compile_options(kleidiai PRIVATE $<$:/WX->) +else() + target_compile_options(kleidiai PRIVATE $<$:-Wno-error>) +endif() diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cpp index 8c4d7bc..24aa282 100644 --- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp @@ -3,39 +3,50 @@ // // KleidiAI micro-kernels -#include "kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h" -#include "kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h" -#include "kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h" -#include "kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.h" -#include "kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.h" -#include "kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.h" -#include "kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.h" -#include "kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.h" -#include "kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.h" -#include "kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.h" -#include "kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.h" -#include "kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.h" -#include "kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.h" -#include "kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.h" -#include "kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.h" -#include "kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h" -#include "kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h" - -#include "kai_lhs_pack_bf16p2vlx2_f32_sme.h" -#include "kai_lhs_quant_pack_qsi8d32p_f32.h" -#include "kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.h" -#include "kai_lhs_quant_pack_qsi8d32p_f32_neon.h" -#include "kai_lhs_quant_pack_qai8dxp_f32.h" - -#include "kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h" -#include "kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h" -#include "kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.h" -#include "kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.h" -#include "kai_lhs_pack_f16pmrx2_f32_neon.h" - -#include "kai_common.h" - -#include "simd-mappings.h" +#include + +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.h" +#include "kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_f32_f32p/kai_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_f32p_f32p/kai_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa.h" + +#include "kai/ukernels/matmul/pack/kai_lhs_pack_bf16p2vlx2_f32_sme.h" +#include "kai/ukernels/matmul/pack/kai_lhs_pack_f32p2vlx1_f32_sme.h" +#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32.h" +#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.h" +#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.h" +#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.h" + +#include "kai/ukernels/matmul/kai_matmul.h" +#include "kai/ukernels/matmul/kai_matmul_pack_lhs.h" +#include "kai/ukernels/matmul/kai_matmul_pack_rhs.h" + +#include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h" +#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.h" +#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h" +#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.h" +#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.h" +#include "kai/ukernels/matmul/pack/kai_lhs_pack_f16pmrx2_f32_neon.h" + +#include "kai/kai_common.h" #define GGML_COMMON_DECL_CPP #include "ggml-common.h" @@ -70,6 +81,21 @@ static inline void kernel_run_fn10(size_t m, size_t n, size_t k, size_t /*bl*/, Fn(m, n, k, lhs, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max); } +template +static inline void kernel_run_lhs_stride_fn10(size_t m, + size_t n, + size_t k, + size_t lhs_stride, + const void * lhs, + const void * rhs, + void * dst, + size_t dst_stride_row, + size_t dst_stride_col, + float clamp_min, + float clamp_max) { + Fn(m, n, k, lhs, lhs_stride, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max); +} + template static inline void kernel_run_float_fn10(size_t m, size_t n, size_t k, size_t /*bl*/, const void* lhs, const void* rhs, void* dst, @@ -132,16 +158,6 @@ static inline size_t rhs_ps_fn2(size_t n, size_t k, size_t /*nr*/, size_t /*kr*/ return Fn(n, k); } -template -static inline size_t rhs_stride_fn4(size_t k, size_t nr, size_t kr, size_t bl) { - return Fn(k, nr, kr, bl); -} - -template -static inline size_t rhs_stride_fn1(size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) { - return Fn(k); -} - template static inline void rhs_pack_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl, size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* /*scale*/, @@ -172,143 +188,242 @@ static inline void rhs_pack_fn13(size_t num_groups, size_t n, size_t k, size_t n Fn(num_groups, n, k, nr, kr, sr, rhs_stride, rhs, bias, scale, rhs_packed, extra_bytes, params); } -static const size_t INT4_PER_BYTE = 2; -static const size_t INT4_BITS = 4; -static const int Q4_0_ZERO_POINT = 8; -const size_t INT4_PER_UINT16 = 4; - -static void dequantize_row_qsi4c32pscalef16( - const void *packed_data, - int32_t row_idx, - int64_t nc, - float *out, - size_t nr_pack, - size_t packed_row_stride, - size_t kr, - size_t bl, - size_t num_bytes_multiplier -) { - size_t group_idx = row_idx / nr_pack; - size_t row_in_group = row_idx % nr_pack; - const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride; - size_t num_blocks = nc / bl; - const uint8_t *block_ptr = packed_group; - - for (size_t b = 0; b < num_blocks; ++b) { - uint16_t scale_f16 = *((const uint16_t *)(block_ptr + row_in_group * num_bytes_multiplier)); - float scale = GGML_CPU_FP16_TO_FP32(scale_f16); - - const uint8_t *segment_ptr = block_ptr + nr_pack * num_bytes_multiplier; - size_t num_segments = bl / kr; - size_t num_bytes_per_segment = kr / INT4_PER_BYTE; - - for (size_t s = 0; s < num_segments; ++s) { - const uint8_t *seg_base = segment_ptr + s * nr_pack * num_bytes_per_segment; - const uint8_t *qbytes = seg_base + row_in_group * num_bytes_per_segment; - for (size_t k = 0; k < num_bytes_per_segment; ++k) { - uint8_t byte = qbytes[k] ^ 0x88; - int x0 = (byte & 0x0F) - Q4_0_ZERO_POINT; - int x1 = (byte >> INT4_BITS) - Q4_0_ZERO_POINT; - out[b * bl + s * num_bytes_per_segment + k] = x0 * scale; - out[b * bl + s * num_bytes_per_segment + k + bl/2] = x1 * scale; - } - } - block_ptr += nr_pack * num_bytes_multiplier + num_segments * nr_pack * num_bytes_per_segment; - } +static constexpr size_t KLEIDIAI_Q4_K_BLOCK_LENGTH = QK_K / 8; + +static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks"); +static_assert(sizeof(block_q4_K) == 144, "Unexpected ggml Q4_K block layout"); +static_assert(offsetof(block_q4_K, scales) == 4, "Unexpected ggml Q4_K metadata offset"); +static_assert(offsetof(block_q4_K, qs) == 16, "Unexpected ggml Q4_K data offset"); + +static struct kai_matmul_uker_config q4_k_matmul_config() { + struct kai_matmul_uker_config config = {}; + config.format.bl = KLEIDIAI_Q4_K_BLOCK_LENGTH; + return config; } -static void dequantize_row_qsi4c32ps1s0scalef16( - const void *packed_data, - int32_t row_idx, - int64_t k, - float *out, - size_t nr, - size_t packed_row_stride, - size_t kr, - size_t bl, - size_t num_bytes_multiplier -) { - const size_t num_blocks = k / bl; - const size_t bl4 = bl / INT4_PER_UINT16; - - size_t group_idx = row_idx / nr; - size_t row_in_group = row_idx % nr; - - const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride; - const uint16_t *qdata = (const uint16_t *)packed_group; - const uint16_t *scales = (const uint16_t *)(packed_group + packed_row_stride - (nr * num_blocks * num_bytes_multiplier)); - - for (size_t block_idx = 0; block_idx < num_blocks; ++block_idx) { - uint16_t scale_f16 = scales[row_in_group + block_idx * nr]; - float scale = GGML_CPU_FP16_TO_FP32(scale_f16); - - for (size_t bl4_idx = 0; bl4_idx < bl4; ++bl4_idx) { - uint16_t q = qdata[(block_idx * bl4 + bl4_idx) * nr + row_in_group]; - - for (size_t qidx = 0; qidx < INT4_PER_UINT16; ++qidx) { - int v = ((q >> (qidx * 4)) & 0xF) - Q4_0_ZERO_POINT; - out[block_idx * bl + bl4_idx * INT4_BITS + qidx] = v * scale; - } +static struct kai_matmul_uker_config default_matmul_config() { + return {}; +} + +template< + struct kai_matmul_uker_api (*GetApi)(void), + struct kai_matmul_uker_config (*GetConfig)(void), + size_t MR, size_t NR, size_t KR, size_t SR> +struct matmul_uker_adapter { + static size_t get_m_step() { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + return api.get_step(&config).m; + } + + static size_t get_n_step() { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + return api.get_step(&config).n; + } + + static size_t get_mr() { + if constexpr (MR == 0) { + return get_m_step(); } + return MR; } - GGML_UNUSED(kr); -} - -static void dequantize_row_qsi8cxp( - const void *packed_data, - int32_t row_idx, - int64_t k, - float *out, - size_t nr, - size_t packed_row_stride, - size_t kr, - size_t bl, - size_t num_bytes_multiplier -) { - GGML_UNUSED(bl); - GGML_UNUSED(num_bytes_multiplier); - - const size_t k_internal = ((size_t) k + QK8_0 - 1) / QK8_0 * QK8_0; - const size_t group_idx = row_idx / nr; - const size_t row_in_group = row_idx % nr; - - const uint8_t * group_ptr = static_cast(packed_data) + group_idx * packed_row_stride; - const int8_t * data_base = reinterpret_cast(group_ptr); - - const size_t num_blocks = k_internal / kr; - - for (size_t block = 0; block < num_blocks; ++block) { - const int8_t * block_ptr = data_base + (block * nr + row_in_group) * kr; - for (size_t i = 0; i < kr; ++i) { - const size_t k_idx = block * kr + i; - if (k_idx < (size_t) k) { - out[k_idx] = static_cast(block_ptr[i]); - } + + static size_t get_nr() { + if constexpr (NR == 0) { + return get_n_step(); } + return NR; } - const uint8_t * sums_ptr = group_ptr + nr * k_internal; - GGML_UNUSED(sums_ptr); + static size_t get_kr() { + return KR; + } - const float * scale_ptr = reinterpret_cast(sums_ptr + nr * sizeof(int32_t)); - const float scale = scale_ptr[row_in_group]; + static size_t get_sr() { + return SR; + } - if (scale == 0.0f) { - for (size_t i = 0; i < (size_t) k; ++i) { - out[i] = 0.0f; - } - return; + static size_t get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx }; + const struct kai_matmul_uker_dst_stride_args stride = { dst_stride }; + return api.get_dst_offset(&config, &index, &stride); + } + + static size_t get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + GGML_ASSERT(bl == config.format.bl); + const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k }; + const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 }; + const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape); + return api.get_lhs_offset(&config, &index, &stride); } - for (size_t i = 0; i < (size_t) k; ++i) { - out[i] *= scale; + static size_t get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + GGML_ASSERT(bl == config.format.bl); + const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k }; + const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 }; + const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape); + return api.get_rhs_offset(&config, &index, &stride); } + + static void run( + size_t m, size_t n, size_t k, size_t bl, + const void * lhs_packed, const void * rhs_packed, + void * dst, size_t dst_stride_row, size_t dst_stride_col, + float clamp_min, float clamp_max) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = GetConfig(); + GGML_ASSERT(bl == config.format.bl); + GGML_ASSERT(dst_stride_col == sizeof(float)); + const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k }; + const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k }; + + struct kai_matmul_uker_args args = {}; + args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP; + args.shape = { m, n, k }; + args.operand.lhs.ptr = lhs_packed; + args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape); + args.operand.rhs.ptr = rhs_packed; + args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape); + args.operand.dst.ptr = dst; + args.operand.dst.stride.m = dst_stride_row; + args.activation.clamp.min_ptr = &clamp_min; + args.activation.clamp.max_ptr = &clamp_max; + api.run(&config, &args); + } +}; + +using q4_k_gemm_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa, + q4_k_matmul_config, 0, 0, 4, 2>; +using q4_k_gemv_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot, + q4_k_matmul_config, 0, 0, 4, 2>; +// GEMM and GEMV share the RHS layout only at SVE256 (NR=8, KR=8, SR=1). +using sve_q8_gemm_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_qai8dxp4x8sf32_qsi8cxp4vsx8sf32bf32_16x4vs_sve_i8mm, + default_matmul_config, 4, 0, 8, 1>; +using sve_q8_gemv_adapter = matmul_uker_adapter< + kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp8x8_1x8_sve_dot, + default_matmul_config, 1, 8, 8, 1>; + +static struct kai_matmul_pack_lhs_uker_config q4_k_lhs_pack_config() { + struct kai_matmul_pack_lhs_uker_config config = {}; + return config; +} + +static size_t q4_k_lhs_get_offset(size_t m_idx, size_t lhs_stride) { + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_dim_args index = { m_idx, 0 }; + const struct kai_matmul_pack_lhs_uker_lhs_stride_args stride = { lhs_stride }; + return api.get_lhs_offset(&config, &index, &stride); +} + +static size_t q4_k_lhs_get_packed_offset( + size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(mr == 1 && kr == 4 && sr == 2); + + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m_idx, k }; + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args index = { m_idx, 0 }; + const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape); + return api.get_lhs_packed_offset(&config, &index, &stride); +} + +static size_t q4_k_lhs_get_packed_size( + size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(mr == 1 && kr == 4 && sr == 2); + + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m, k }; + const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape); + return api.get_lhs_packed_size(&config, &shape, &stride); +} + +static void q4_k_lhs_pack( + size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr, + size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(mr == 1 && kr == 4 && sr == 2); + GGML_ASSERT(m_idx_start == 0); + + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args packed_shape = { m, k }; + + struct kai_matmul_pack_lhs_uker_args args = {}; + args.flags = 0; + args.shape = { m, k }; + args.operand.lhs.ptr = lhs; + args.operand.lhs.stride.m = lhs_stride; + args.operand.lhs_packed.ptr = lhs_packed; + args.operand.lhs_packed.stride = api.get_lhs_packed_stride(&config, &packed_shape); + api.run(&config, &args); +} + +static struct kai_matmul_pack_rhs_uker_config q4_k_rhs_pack_config() { + struct kai_matmul_pack_rhs_uker_config config = {}; + return config; +} + +static size_t q4_k_rhs_get_packed_size(size_t n, size_t k, size_t nr, size_t /*kr*/, size_t bl) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + + const struct kai_matmul_pack_rhs_uker_api api = + kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme(); + const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config(); + GGML_ASSERT(api.get_step(&config).n == nr); + const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args shape = { n, k }; + const struct kai_matmul_pack_rhs_uker_rhs_packed_stride_args stride = api.get_rhs_packed_stride(&config, &shape); + return api.get_rhs_packed_size(&config, &shape, &stride); +} + +static void q4_k_rhs_pack( + size_t num_groups, size_t n, size_t k, size_t nr, size_t /*kr*/, size_t /*sr*/, size_t bl, + size_t rhs_stride, const void * rhs, const void * bias, const void * scale, + void * rhs_packed, size_t extra_bytes, const void * params) { + GGML_ASSERT(num_groups == 1); + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(bias == nullptr); + GGML_ASSERT(scale == nullptr); + GGML_ASSERT(extra_bytes == 0); + GGML_ASSERT(params == nullptr); + + const struct kai_matmul_pack_rhs_uker_api api = + kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme(); + const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config(); + GGML_ASSERT(api.get_step(&config).n == nr); + const struct kai_matmul_pack_rhs_uker_rhs_dim_args rhs_shape = { n, k }; + const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args packed_shape = { n, k }; + + struct kai_matmul_pack_rhs_uker_args args = {}; + args.flags = 0; + args.shape = { n, k }; + args.operand.rhs.ptr = rhs; + args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape); + if (rhs_stride != 0) { + GGML_ASSERT(rhs_stride >= args.operand.rhs.stride.n); + args.operand.rhs.stride.n = rhs_stride; + } + args.operand.rhs_packed.ptr = rhs_packed; + args.operand.rhs_packed.stride = api.get_rhs_packed_stride(&config, &packed_shape); + api.run(&config, &args); } static ggml_kleidiai_kernels gemm_gemv_kernels[] = { -#if defined(__ARM_FEATURE_SME) { - /* SME GEMM */ + /* SME2 GEMM */ /* .kern_info = */ { /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, @@ -317,7 +432,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -329,7 +443,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .packed_size_ex = */ &lhs_ps_fn6, /* .pack_func_ex = */ &lhs_pack_void_fn10, }, - /* SME GEMV */ + /* SME2 GEMV */ /* .kern_info = */ { /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, @@ -338,7 +452,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -350,19 +463,16 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon, - /* .to_float = */ dequantize_row_qsi4c32ps1s0scalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, - /* .required_cpu = */ CPU_FEATURE_SME, + /* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16, /* .lhs_type = */ GGML_TYPE_F32, /* .rhs_type = */ GGML_TYPE_Q4_0, /* .op_type = */ GGML_TYPE_F32, }, { - /* SME GEMM */ + /* SME2 GEMM */ /* .kern_info = */ { /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, @@ -371,7 +481,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_fn10, @@ -382,7 +491,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .packed_size_ex = */ &lhs_ps_fn5, /* .pack_func_ex = */ &lhs_pack_void_fn9, }, - /* SME GEMV */ + /* SME2 GEMV */ /* .kern_info = */ { /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, @@ -391,7 +500,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_lhs_offset_ex = */ nullptr, /* .get_rhs_packed_offset_ex = */ nullptr, /* .run_kernel_ex = */ nullptr, @@ -403,20 +511,15 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_void_fn9, }, /* .rhs_info = */ { - /* .packed_stride = */ nullptr, - /* .to_float = */ nullptr, /* .packed_size_ex = */ &rhs_ps_fn2, - /* .packed_stride_ex = */ &rhs_stride_fn1, /* .pack_func_ex = */ &rhs_pack_fn13, }, - /* .required_cpu = */ CPU_FEATURE_SME, + /* .required_cpu = */ CPU_FEATURE_SME2, /* .lhs_type = */ GGML_TYPE_F32, /* .rhs_type = */ GGML_TYPE_F16, /* .op_type = */ GGML_TYPE_F32, }, -#endif #if defined(__APPLE__) -#if defined(__ARM_FEATURE_DOTPROD) { /* DOTPROD GEMM */ /* .kern_info = */ { @@ -427,7 +530,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -447,7 +549,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -459,10 +560,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, /* .required_cpu = */ CPU_FEATURE_DOTPROD, @@ -470,8 +568,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .rhs_type = */ GGML_TYPE_Q4_0, /* .op_type = */ GGML_TYPE_F32, }, -#endif -#if defined(__ARM_FEATURE_MATMUL_INT8) { /* i8mm GEMM */ /* .kern_info = */ { @@ -482,7 +578,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -493,7 +588,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .packed_size_ex = */ &lhs_ps_fn6, /* .pack_func_ex = */ &lhs_pack_float_fn10, }, - /* i8mm GEMV */ + /* DOTPROD GEMV */ /* .kern_info = */ { /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, @@ -502,7 +597,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -514,20 +608,15 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, - /* .required_cpu = */ CPU_FEATURE_I8MM, + /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, /* .lhs_type = */ GGML_TYPE_F32, /* .rhs_type = */ GGML_TYPE_Q4_0, /* .op_type = */ GGML_TYPE_F32, }, -#endif #else -#if defined(__ARM_FEATURE_SVE) { /* SVE i8mm GEMM */ /* .kern_info = */ { @@ -538,7 +627,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -558,7 +646,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -570,19 +657,14 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, - /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, + /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD, /* .lhs_type = */ GGML_TYPE_F32, /* .rhs_type = */ GGML_TYPE_Q4_0, /* .op_type = */ GGML_TYPE_F32, }, -#endif -#if defined(__ARM_FEATURE_MATMUL_INT8) { /* i8mm GEMM */ /* .kern_info = */ { @@ -593,7 +675,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -604,7 +685,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .packed_size_ex = */ &lhs_ps_fn6, /* .pack_func_ex = */ &lhs_pack_float_fn10, }, - /* i8mm GEMV */ + /* DOTPROD GEMV */ /* .kern_info = */ { /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, @@ -613,7 +694,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -625,19 +705,14 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, - /* .required_cpu = */ CPU_FEATURE_I8MM, + /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, /* .lhs_type = */ GGML_TYPE_F32, /* .rhs_type = */ GGML_TYPE_Q4_0, /* .op_type = */ GGML_TYPE_F32, }, -#endif // __ARM_FEATURE_MATMUL_INT8 -#if defined(__ARM_FEATURE_DOTPROD) { /* DOTPROD GEMM */ /* .kern_info = */ { @@ -648,7 +723,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -668,7 +742,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -680,10 +753,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, /* .required_cpu = */ CPU_FEATURE_DOTPROD, @@ -692,14 +762,65 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .op_type = */ GGML_TYPE_F32, }, #endif -#endif + { /* Sentinel */ } +}; + +static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = { + { + /* SME2 GEMM */ + { + /* .get_m_step = */ q4_k_gemm_adapter::get_m_step, + /* .get_n_step = */ q4_k_gemm_adapter::get_n_step, + /* .get_mr = */ q4_k_gemm_adapter::get_mr, + /* .get_nr = */ q4_k_gemm_adapter::get_nr, + /* .get_kr = */ q4_k_gemm_adapter::get_kr, + /* .get_sr = */ q4_k_gemm_adapter::get_sr, + /* .get_dst_offset = */ q4_k_gemm_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ q4_k_gemm_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ q4_k_gemm_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ q4_k_gemm_adapter::run, + }, + { + /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon, + /* .get_packed_offset_ex = */ &lhs_offs_fn6, + /* .packed_size_ex = */ &lhs_ps_fn6, + /* .pack_func_ex = */ &lhs_pack_void_fn10, + }, + /* SME2 GEMV */ + { + /* .get_m_step = */ q4_k_gemv_adapter::get_m_step, + /* .get_n_step = */ q4_k_gemv_adapter::get_n_step, + /* .get_mr = */ q4_k_gemv_adapter::get_mr, + /* .get_nr = */ q4_k_gemv_adapter::get_nr, + /* .get_kr = */ q4_k_gemv_adapter::get_kr, + /* .get_sr = */ q4_k_gemv_adapter::get_sr, + /* .get_dst_offset = */ q4_k_gemv_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ q4_k_gemv_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ q4_k_gemv_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ q4_k_gemv_adapter::run, + }, + { + /* .get_offset = */ q4_k_lhs_get_offset, + /* .get_packed_offset_ex = */ q4_k_lhs_get_packed_offset, + /* .packed_size_ex = */ q4_k_lhs_get_packed_size, + /* .pack_func_ex = */ q4_k_lhs_pack, + }, + { + /* .packed_size_ex = */ q4_k_rhs_get_packed_size, + /* .pack_func_ex = */ q4_k_rhs_pack, + /* .repack_mode = */ RHS_REPACK_SINGLE_ONLY, + }, + /* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16, + /* .lhs_type = */ GGML_TYPE_F32, + /* .rhs_type = */ GGML_TYPE_Q4_K, + /* .op_type = */ GGML_TYPE_F32, + }, { /* Sentinel */ } }; static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { -#if defined(__ARM_FEATURE_SME) { - /* SME GEMM */ + /* SME2 GEMM */ { /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, @@ -708,7 +829,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -719,7 +839,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .packed_size_ex = */ &lhs_ps_fn5, /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, }, - /* SME GEMV */ + /* SME2 GEMV */ { /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, @@ -728,7 +848,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -740,10 +859,55 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, + /* .pack_func_ex = */ &rhs_pack_scale_fn12, + }, + /* .required_cpu = */ CPU_FEATURE_SME2, + /* .lhs_type = */ GGML_TYPE_F32, + /* .rhs_type = */ GGML_TYPE_Q8_0, + /* .op_type = */ GGML_TYPE_F32, + }, + { + /* SME GEMM (pure SME, no SME2 required) */ + { + /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, + /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, + /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, + /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, + /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, + /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, + /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, + /* .get_lhs_offset_ex = */ &kernel_offs_fn2, + /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, + /* .run_kernel_ex = */ &kernel_run_float_fn10, + }, + /* .gemm_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32, + /* .get_packed_offset_ex = */ &lhs_offs_fn5, + /* .packed_size_ex = */ &lhs_ps_fn5, + /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, + }, + /* SME GEMV (pure SME, no SME2 required) */ + { + /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, + /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, + /* .get_mr = */ kai_get_mr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, + /* .get_nr = */ kai_get_nr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, + /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, + /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, + /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, + /* .get_lhs_offset_ex = */ &kernel_offs_fn2, + /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, + /* .run_kernel_ex = */ &kernel_run_float_fn10, + }, + /* .gemv_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32, + /* .get_packed_offset_ex = */ &lhs_offs_fn5, + /* .packed_size_ex = */ &lhs_ps_fn5, + /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, + }, + /* .rhs_info = */ { + /* .packed_size_ex = */ &rhs_ps_fn5, /* .pack_func_ex = */ &rhs_pack_scale_fn12, }, /* .required_cpu = */ CPU_FEATURE_SME, @@ -751,8 +915,54 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .rhs_type = */ GGML_TYPE_Q8_0, /* .op_type = */ GGML_TYPE_F32, }, -#endif -#if defined(__ARM_FEATURE_MATMUL_INT8) + { + /* SVE I8MM GEMM */ + { + /* .get_m_step = */ sve_q8_gemm_adapter::get_m_step, + /* .get_n_step = */ sve_q8_gemm_adapter::get_n_step, + /* .get_mr = */ sve_q8_gemm_adapter::get_mr, + /* .get_nr = */ sve_q8_gemm_adapter::get_nr, + /* .get_kr = */ sve_q8_gemm_adapter::get_kr, + /* .get_sr = */ sve_q8_gemm_adapter::get_sr, + /* .get_dst_offset = */ sve_q8_gemm_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ sve_q8_gemm_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ sve_q8_gemm_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ sve_q8_gemm_adapter::run, + }, + /* .gemm_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32, + /* .get_packed_offset_ex = */ &lhs_offs_fn5, + /* .packed_size_ex = */ &lhs_ps_fn5, + /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, + }, + /* SVE DOT GEMV */ + { + /* .get_m_step = */ sve_q8_gemv_adapter::get_m_step, + /* .get_n_step = */ sve_q8_gemv_adapter::get_n_step, + /* .get_mr = */ sve_q8_gemv_adapter::get_mr, + /* .get_nr = */ sve_q8_gemv_adapter::get_nr, + /* .get_kr = */ sve_q8_gemv_adapter::get_kr, + /* .get_sr = */ sve_q8_gemv_adapter::get_sr, + /* .get_dst_offset = */ sve_q8_gemv_adapter::get_dst_offset, + /* .get_lhs_offset_ex = */ sve_q8_gemv_adapter::get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ sve_q8_gemv_adapter::get_rhs_packed_offset, + /* .run_kernel_ex = */ sve_q8_gemv_adapter::run, + }, + /* .gemv_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32, + /* .get_packed_offset_ex = */ &lhs_offs_fn5, + /* .packed_size_ex = */ &lhs_ps_fn5, + /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, + }, + /* .rhs_info = */ { + /* .packed_size_ex = */ &rhs_ps_fn5, + /* .pack_func_ex = */ &rhs_pack_scale_fn12, + }, + /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD, + /* .lhs_type = */ GGML_TYPE_F32, + /* .rhs_type = */ GGML_TYPE_Q8_0, + /* .op_type = */ GGML_TYPE_F32, + }, { /* I8MM GEMM */ { @@ -763,7 +973,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -783,7 +992,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -795,19 +1003,14 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_scale_fn12, }, - /* .required_cpu = */ CPU_FEATURE_I8MM, + /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, /* .lhs_type = */ GGML_TYPE_F32, /* .rhs_type = */ GGML_TYPE_Q8_0, /* .op_type = */ GGML_TYPE_F32, }, -#endif -#if defined(__ARM_FEATURE_DOTPROD) { /* DOTPROD GEMM */ { @@ -818,7 +1021,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -838,7 +1040,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -850,10 +1051,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_scale_fn12, }, /* .required_cpu = */ CPU_FEATURE_DOTPROD, @@ -861,7 +1059,106 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .rhs_type = */ GGML_TYPE_Q8_0, /* .op_type = */ GGML_TYPE_F32, }, -#endif + { /* Sentinel */ } +}; + +static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { + { + /* SME2 GEMM */ + { + /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, + /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, + /* .get_mr = */ kai_get_mr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, + /* .get_nr = */ kai_get_nr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, + /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, + /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, + /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, + /* .get_lhs_offset_ex = */ &kernel_offs_fn2, + /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, + /* .run_kernel_ex = */ &kernel_run_fn10, + }, + /* .gemm_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f32p2vlx1_f32_sme, + /* .get_packed_offset_ex = */ &lhs_offs_fn5, + /* .packed_size_ex = */ &lhs_ps_fn5, + /* .pack_func_ex = */ &lhs_pack_void_fn9, + }, + /* SME2 GEMV */ + { + /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_mr = */ kai_get_m_step_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_nr = */ kai_get_nr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, + /* .get_lhs_offset_ex = */ &kernel_offs_fn2, + /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, + /* .run_kernel_ex = */ &kernel_run_lhs_stride_fn10, + }, + /* .gemv_lhs_info = */ { + /* .get_offset = */ nullptr, + /* .get_packed_offset_ex = */ nullptr, + /* .packed_size_ex = */ nullptr, + /* .pack_func_ex = */ nullptr, + }, + /* .rhs_info = */ { + /* .packed_size_ex = */ &rhs_ps_fn2, + /* .pack_func_ex = */ &rhs_pack_fn13, + }, + /* .required_cpu = */ CPU_FEATURE_SME2, + /* .lhs_type = */ GGML_TYPE_F32, + /* .rhs_type = */ GGML_TYPE_F32, + /* .op_type = */ GGML_TYPE_F32, + }, + { + /* SME GEMM */ + { + /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_mr = */ kai_get_mr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_nr = */ kai_get_nr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_lhs_offset_ex = */ &kernel_offs_fn2, + /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, + /* .run_kernel_ex = */ &kernel_run_fn10, + }, + /* .gemm_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f32p2vlx1_f32_sme, + /* .get_packed_offset_ex = */ &lhs_offs_fn5, + /* .packed_size_ex = */ &lhs_ps_fn5, + /* .pack_func_ex = */ &lhs_pack_void_fn9, + }, + /* SME GEMV */ + { + /* .get_m_step = */ kai_get_m_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_n_step = */ kai_get_n_step_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_mr = */ kai_get_mr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_nr = */ kai_get_nr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, + /* .get_lhs_offset_ex = */ nullptr, + /* .get_rhs_packed_offset_ex = */ nullptr, + /* .run_kernel_ex = */ nullptr, + }, + /* .gemv_lhs_info = */ { + /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f32p2vlx1_f32_sme, + /* .get_packed_offset_ex = */ &lhs_offs_fn5, + /* .packed_size_ex = */ &lhs_ps_fn5, + /* .pack_func_ex = */ &lhs_pack_void_fn9, + }, + /* .rhs_info = */ { + /* .packed_size_ex = */ &rhs_ps_fn2, + /* .pack_func_ex = */ &rhs_pack_fn13, + }, + /* .required_cpu = */ CPU_FEATURE_SME, + /* .lhs_type = */ GGML_TYPE_F32, + /* .rhs_type = */ GGML_TYPE_F32, + /* .op_type = */ GGML_TYPE_F32, + }, { /* Sentinel */ } }; @@ -869,10 +1166,6 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, c ggml_kleidiai_kernels * kernel = nullptr; if (tensor->op == GGML_OP_MUL_MAT && tensor->src[0] != nullptr && tensor->src[1] != nullptr) { -#if defined(__ARM_FEATURE_SME) || \ - defined(__ARM_FEATURE_DOTPROD) || \ - defined(__ARM_FEATURE_MATMUL_INT8) || \ - defined(__ARM_FEATURE_SVE) auto try_table = [&](auto & table) { for (size_t i = 0; i < NELEMS(table) - 1; ++i) { if ((cpu_features & table[i].required_cpu) == table[i].required_cpu && @@ -886,16 +1179,28 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, c return false; }; - if (tensor->src[0]->type == GGML_TYPE_Q8_0) { - try_table(gemm_gemv_kernels_q8); + if (tensor->src[0]->type == GGML_TYPE_Q4_K) { + try_table(gemm_gemv_kernels_q4_k); + } else if (tensor->src[0]->type == GGML_TYPE_Q8_0 || tensor->src[0]->type == GGML_TYPE_Q6_K) { + const ggml_type original_type = tensor->src[0]->type; + if (original_type == GGML_TYPE_Q6_K) { + for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) { + if ((cpu_features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu && + gemm_gemv_kernels_q8[i].lhs_type == tensor->src[1]->type && + gemm_gemv_kernels_q8[i].rhs_type == GGML_TYPE_Q8_0 && + gemm_gemv_kernels_q8[i].op_type == tensor->type) { + kernel = &gemm_gemv_kernels_q8[i]; + break; + } + } + } else { + try_table(gemm_gemv_kernels_q8); + } + } else if (tensor->src[0]->type == GGML_TYPE_F32) { + try_table(ggml_kleidiai_kernels_f32); } else { try_table(gemm_gemv_kernels); } -#else - GGML_UNUSED(gemm_gemv_kernels); - GGML_UNUSED(gemm_gemv_kernels_q8); - GGML_UNUSED(cpu_features); -#endif } return kernel; @@ -904,19 +1209,26 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, c ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features) { ggml_kleidiai_kernels * kernels = nullptr; -#if defined(__ARM_FEATURE_SME) || \ - defined(__ARM_FEATURE_DOTPROD) || \ - defined(__ARM_FEATURE_MATMUL_INT8) || \ - defined(__ARM_FEATURE_SVE) for (size_t i = 0; i < NELEMS(gemm_gemv_kernels) - 1; ++i) { - if ((features & gemm_gemv_kernels[i].required_cpu) == gemm_gemv_kernels[i].required_cpu) { + if ((features & gemm_gemv_kernels[i].required_cpu) == gemm_gemv_kernels[i].required_cpu && + gemm_gemv_kernels[i].rhs_type == GGML_TYPE_Q4_0) { kernels = &gemm_gemv_kernels[i]; break; } } -#else - GGML_UNUSED(features); -#endif + + return kernels; +} + +ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features) { + ggml_kleidiai_kernels * kernels = nullptr; + + for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q4_k) - 1; ++i) { + if ((features & gemm_gemv_kernels_q4_k[i].required_cpu) == gemm_gemv_kernels_q4_k[i].required_cpu) { + kernels = &gemm_gemv_kernels_q4_k[i]; + break; + } + } return kernels; } @@ -924,16 +1236,24 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features) ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features) { ggml_kleidiai_kernels * kernels = nullptr; -#if defined(__ARM_FEATURE_SME) || defined(__ARM_FEATURE_DOTPROD) || defined(__ARM_FEATURE_MATMUL_INT8) for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) { if ((features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu) { kernels = &gemm_gemv_kernels_q8[i]; break; } } -#else - GGML_UNUSED(features); -#endif return kernels; } + +ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_f32(cpu_feature features) { + ggml_kleidiai_kernels * kernels = nullptr; + + for (size_t i = 0; i < NELEMS(ggml_kleidiai_kernels_f32) - 1; ++i) { + if ((features & ggml_kleidiai_kernels_f32[i].required_cpu) == ggml_kleidiai_kernels_f32[i].required_cpu) { + kernels = &ggml_kleidiai_kernels_f32[i]; + break; + } + } + return kernels; +} diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.h b/ggml/src/ggml-cpu/kleidiai/kernels.h index 1292454..ac5d5d3 100644 --- a/ggml/src/ggml-cpu/kleidiai/kernels.h +++ b/ggml/src/ggml-cpu/kleidiai/kernels.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates +// SPDX-FileCopyrightText: Copyright 2025-2026 Arm Limited and/or its affiliates // SPDX-License-Identifier: MIT // @@ -11,7 +11,10 @@ enum cpu_feature { CPU_FEATURE_DOTPROD = 1, CPU_FEATURE_I8MM = 2, CPU_FEATURE_SVE = 4, - CPU_FEATURE_SME = 8 + CPU_FEATURE_SME = 8, + CPU_FEATURE_SME2 = 16, + CPU_FEATURE_FP16 = 32, + CPU_FEATURE_SVEI8MM = 64 }; inline cpu_feature& operator|=(cpu_feature& lhs, cpu_feature rhs) { @@ -31,7 +34,6 @@ struct kernel_info { size_t (*get_sr)(void); size_t (*get_dst_offset)(size_t m_idx, size_t n_idx, size_t stride); - size_t (*get_dst_size)(size_t m, size_t n); size_t (*get_lhs_offset_ex)(size_t m_idx, size_t k, size_t bl); @@ -55,19 +57,18 @@ struct lhs_packing_info { size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed); }; -struct rhs_packing_info { - size_t (*packed_stride)(size_t k, size_t nr, size_t kr, size_t bl); - - void (*to_float)(const void *packed_data, int32_t row_idx, int64_t nc, float *out, - size_t nr_pack, size_t packed_row_stride, size_t kr, size_t bl, - size_t num_bytes_multiplier); +enum rhs_repack_mode { + RHS_REPACK_PER_KERNEL, + RHS_REPACK_SINGLE_ONLY, +}; +struct rhs_packing_info { size_t (*packed_size_ex)(size_t n, size_t k, size_t nr, size_t kr, size_t bl); - size_t (*packed_stride_ex)(size_t k, size_t nr, size_t kr, size_t bl); - void (*pack_func_ex)(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl, size_t rhs_stride, const void * rhs, const void * bias, const void * scale, void * rhs_packed, size_t extra_bytes, const void * params); + + rhs_repack_mode repack_mode = RHS_REPACK_PER_KERNEL; }; struct ggml_kleidiai_kernels { @@ -87,4 +88,6 @@ struct ggml_kleidiai_kernels { ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor); ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features); +ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features); ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features); +ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_f32(cpu_feature features); diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp index 0ecf7ae..57a4a56 100644 --- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp @@ -2,10 +2,12 @@ // SPDX-License-Identifier: MIT // #include -#include -#include +#include +#include +#include #include #include +#include #include #include #include @@ -17,35 +19,37 @@ #include #include #include -#include +#include #include #include +#include +#include #if defined(__linux__) #include +#include #include #include #include #include #elif defined(__APPLE__) -#include #include #include -#elif defined(_WIN32) -#include -#include #endif #include "kleidiai.h" #include "ggml-cpu.h" +#include "ggml-cpu-impl.h" #include "ggml-impl.h" +#include "ggml-feats.h" #include "ggml-backend-impl.h" +#include "ggml-quants.h" #include "ggml-threading.h" #include "traits.h" #include "kernels.h" -#include "kai_common.h" +#include "kai/kai_common.h" #define GGML_COMMON_DECL_CPP #include "ggml-common.h" @@ -54,18 +58,30 @@ static constexpr int GGML_KLEIDIAI_MAX_KERNEL_SLOTS = 2; static constexpr uint32_t GGML_KLEIDIAI_PACK_MAGIC = 0x4b4c4149; // "KLAI" static constexpr uint16_t GGML_KLEIDIAI_PACK_VERSION = 1; static constexpr size_t GGML_KLEIDIAI_PACK_ALIGN = 64; +static constexpr size_t GGML_KLEIDIAI_Q4_K_BL = QK_K / 8; + +static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks"); struct ggml_kleidiai_context { cpu_feature features; ggml_kleidiai_kernels * kernels_q4; + ggml_kleidiai_kernels * kernels_q4k; ggml_kleidiai_kernels * kernels_q8; - int sme_thread_cap; // <= 0 means “SME disabled/unknown”; - int thread_hint; // <= 0 means “no hint” -} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, 0, -1 }; + ggml_kleidiai_kernels * kernels_f32; + int sme_thread_cap; // <= 0 means "SME disabled/unknown" + int thread_hint; // <= 0 means "no hint" + int chunk_multiplier; +} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, nullptr, 0, -1, 4 }; + +static inline bool is_sme_family(cpu_feature f) { + return (f & (CPU_FEATURE_SME | CPU_FEATURE_SME2)) != CPU_FEATURE_NONE; +} static const char* cpu_feature_to_string(cpu_feature f) { if (f == CPU_FEATURE_NONE) { return "NONE"; + } else if ((f & CPU_FEATURE_SME2) == CPU_FEATURE_SME2) { + return "SME2"; } else if ((f & CPU_FEATURE_SME) == CPU_FEATURE_SME) { return "SME"; } else if ((f & CPU_FEATURE_SVE) == CPU_FEATURE_SVE) { @@ -81,24 +97,117 @@ static const char* cpu_feature_to_string(cpu_feature f) { } } +#if defined(__linux__) && defined(__aarch64__) +static bool parse_cpu_dir_name(const char* name, size_t* cpu) { + if (strncmp(name, "cpu", 3) != 0 || + name[3] < '0' || name[3] > '9') { + return false; + } + + const char* first = name + 3; + const char* last = name + strlen(name); + + size_t value = 0; + const auto [end, ec] = std::from_chars(first, last, value, 10); + + if (ec != std::errc{} || end != last) { + return false; + } + + *cpu = value; + return true; +} + +static std::vector detect_cpu_ids() { + std::vector cpus; + + DIR * dir = opendir("/sys/devices/system/cpu"); + if (dir == nullptr) { + return cpus; + } + + while (dirent * entry = readdir(dir)) { + size_t cpu = 0; + if (parse_cpu_dir_name(entry->d_name, &cpu)) { + cpus.push_back(cpu); + } + } + closedir(dir); + + std::sort(cpus.begin(), cpus.end()); + cpus.erase(std::unique(cpus.begin(), cpus.end()), cpus.end()); + return cpus; +} +#endif + +#if defined(__APPLE__) && defined(__aarch64__) +static bool apple_sme_counted_perf_level(std::string name) { + for (std::string::size_type i = 0; i < name.size(); ++i) { + name[i] = (char) std::tolower((unsigned char) name[i]); + } + + // Conservative ceiling: only count perf-level names observed to provide full SME throughput. + // Future names should be calibrated here before they raise the automatic SME thread cap. + return name.find("super") != std::string::npos || + name.find("performance") != std::string::npos; +} +#endif + +static void add_smcus_from_smidr(uint64_t smidr, size_t & num_private, std::map & shared_counts) { + // Arm ARM: SMIDR_EL1. SH==0 is implementation-defined; keep the existing + // conservative policy and only treat zero affinity as private. + const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3); + const uint32_t nsmc = (uint32_t)((smidr >> 56) & 0xF); + const size_t shared_count = nsmc == 0xF ? 1 : (size_t)nsmc + 1; + const uint32_t affinity = (uint32_t)(smidr & 0xFFFu); + const uint32_t affinity2 = (uint32_t)((smidr >> 32) & 0xFFFFFu); + const uint32_t id = (affinity2 << 12) | affinity; + + if (nsmc == 0xF) { + GGML_LOG_WARN("kleidiai: NSMC detected as 0xF indicating reseved value, setting min safe shared SMCU count to 1"); + } + + switch (sh) { + case 2: // private SMCU + ++num_private; + break; + case 3: // shared SMCU + if (shared_counts[id] < shared_count) { + shared_counts[id] = shared_count; + } + break; + case 0: + if (id == 0) { + ++num_private; + } else if (shared_counts[id] < shared_count) { + shared_counts[id] = shared_count; + } + break; + default: + break; + } +} + static size_t detect_num_smcus() { - if (!ggml_cpu_has_sme()) { + const auto runtime_feat = ggml_feats_get_arch64_runtime(); + if (!runtime_feat.has_sme) { return 0; } #if defined(__linux__) && defined(__aarch64__) // Linux/aarch64: Best-effort count of Streaming Mode Compute Units (SMCUs) via SMIDR_EL1 sysfs. size_t num_private = 0; - std::set shared_ids; + std::map shared_counts; - for (size_t cpu = 0;; ++cpu) { + const std::vector cpus = detect_cpu_ids(); + for (const size_t cpu : cpus) { const std::string path = "/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/regs/identification/smidr_el1"; std::ifstream file(path); if (!file.is_open()) { - break; + continue; } uint64_t smidr = 0; @@ -106,58 +215,73 @@ static size_t detect_num_smcus() { continue; } - // Arm ARM: SMIDR_EL1 - const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3); - // Build an "affinity-like" identifier for shared SMCUs. - // Keep the original packing logic, but isolate it here. - const uint32_t id = (uint32_t)((smidr & 0xFFFu) | ((smidr >> 20) & 0xFFFFF000u)); - - switch (sh) { - case 0b10: // private SMCU - ++num_private; - break; - case 0b11: // shared SMCU - shared_ids.emplace(id); - break; - case 0b00: - // Ambiguous / implementation-defined. Be conservative: - // treat id==0 as private, otherwise as shared. - if (id == 0) ++num_private; - else shared_ids.emplace(id); - break; - default: - break; - } + add_smcus_from_smidr(smidr, num_private, shared_counts); } - return num_private + shared_ids.size(); + size_t total = num_private; + for (const auto & entry : shared_counts) { + total += entry.second; + } + return total; #elif defined(__APPLE__) && defined(__aarch64__) - // table for known M4 variants. Users can override via GGML_KLEIDIAI_SME=. - char chip_name[256] = {}; - size_t size = sizeof(chip_name); - - if (sysctlbyname("machdep.cpu.brand_string", chip_name, &size, nullptr, 0) == 0) { - const std::string brand(chip_name); - - struct ModelSMCU { const char *match; size_t smcus; }; - static const ModelSMCU table[] = { - { "M4 Ultra", 2 }, - { "M4 Max", 2 }, - { "M4 Pro", 2 }, - { "M4", 1 }, - }; + int perf_levels = 0; + size_t size = sizeof(perf_levels); + if (sysctlbyname("hw.nperflevels", &perf_levels, &size, nullptr, 0) != 0 || + size != sizeof(perf_levels) || perf_levels <= 0) { + return 0; + } - for (const auto &e : table) { - if (brand.find(e.match) != std::string::npos) { - return e.smcus; - } + size_t units = 0; + for (int i = 0; i < perf_levels; ++i) { + char key[64] = {}; + int physical_cpus = 0; + int cpus_per_l2 = 0; + + snprintf(key, sizeof(key), "hw.perflevel%d.physicalcpu", i); + size = sizeof(physical_cpus); + if (sysctlbyname(key, &physical_cpus, &size, nullptr, 0) != 0 || + size != sizeof(physical_cpus) || physical_cpus <= 0) { + continue; + } + + snprintf(key, sizeof(key), "hw.perflevel%d.cpusperl2", i); + size = sizeof(cpus_per_l2); + if (sysctlbyname(key, &cpus_per_l2, &size, nullptr, 0) != 0 || + size != sizeof(cpus_per_l2) || cpus_per_l2 <= 0) { + continue; + } + + snprintf(key, sizeof(key), "hw.perflevel%d.name", i); + size = 0; + if (sysctlbyname(key, nullptr, &size, nullptr, 0) != 0 || size == 0) { + continue; + } + + std::string name(size, '\0'); + if (sysctlbyname(key, &name[0], &size, nullptr, 0) != 0) { + continue; + } + name.resize(size); + while (!name.empty() && name.back() == '\0') { + name.pop_back(); + } + + if (apple_sme_counted_perf_level(name)) { + units += (size_t) ((physical_cpus + cpus_per_l2 - 1) / cpus_per_l2); } } - return 1; + + return units; + +#elif defined(_WIN32) && (defined(_M_ARM64) || defined(__aarch64__)) + // No verified Windows arm64 SMCU detection path yet. Return unknown and use + // GGML_KLEIDIAI_SME=N as a diagnostics/debug override for SME thread cap + // calibration until a detection mechanism is verified on real hardware. + return 0; #else - return 1; + return 0; #endif } @@ -186,15 +310,21 @@ static void init_kleidiai_context(void) { if (!initialized) { initialized = true; - const char *env_sme = getenv("GGML_KLEIDIAI_SME"); - const char *env_threads = getenv("GGML_TOTAL_THREADS"); + // Optional diagnostics/debug overrides; production defaults come from runtime detection. + const char *env_sme = getenv("GGML_KLEIDIAI_SME"); + const char *env_threads = getenv("GGML_TOTAL_THREADS"); + const char *env_chunk_mult = getenv("GGML_KLEIDIAI_CHUNK_MULTIPLIER"); + + const auto runtime_feat = ggml_feats_get_arch64_runtime(); - const bool cpu_has_sme = ggml_cpu_has_sme(); size_t detected_smcus = 0; - ctx.features = (ggml_cpu_has_dotprod() ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) | - (ggml_cpu_has_matmul_int8() ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) | - ((ggml_cpu_has_sve() && ggml_cpu_get_sve_cnt() == QK8_0) ? CPU_FEATURE_SVE : CPU_FEATURE_NONE); + // The fixed-width SVE kernels require 32-byte vectors, including the Q8 shared RHS layout. + ctx.features = (runtime_feat.has_dotprod ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) | + (runtime_feat.has_i8mm ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) | + (runtime_feat.has_svei8mm ? CPU_FEATURE_SVEI8MM : CPU_FEATURE_NONE) | + (runtime_feat.has_fp16 ? CPU_FEATURE_FP16 : CPU_FEATURE_NONE) | + (runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE); if (env_threads) { bool ok = false; @@ -204,57 +334,69 @@ static void init_kleidiai_context(void) { } } - // SME policy: - // - If CPU doesn't support SME: SME always off. - // - Else: - // - env unset => auto-detect cores; enable if detected > 0. - // - env=0 => force off. - // - env>0 => force N cores (skip detection). + if (env_chunk_mult) { + bool ok = false; + int multiplier = parse_uint_env(env_chunk_mult, "GGML_KLEIDIAI_CHUNK_MULTIPLIER", &ok); + if (ok && multiplier > 0) { + ctx.chunk_multiplier = multiplier; + } + } + int sme_cores = 0; bool sme_env_ok = false; bool sme_env_set = (env_sme != nullptr); - if (!cpu_has_sme) { - if (sme_env_set) { - bool ok = false; - int req = parse_uint_env(env_sme, "GGML_KLEIDIAI_SME", &ok); - if (ok && req > 0) { - GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but SME is not supported on this CPU; disabling SME\n", req); - } + const bool has_supported_sme_family = runtime_feat.has_sme; + bool sme_cap_detected = false; + + if (has_supported_sme_family) { + detected_smcus = detect_num_smcus(); + sme_cap_detected = detected_smcus > 0; + // Some platforms expose SME without exposing a calibrated SMCU count. + // Use one SME thread as the conservative default; add platform SMCU detection to raise it. + sme_cores = sme_cap_detected ? (int)detected_smcus : 1; + + if (!sme_env_set && !sme_cap_detected) { + GGML_LOG_INFO("kleidiai: SME detected; SMCU count unavailable, using conservative SME thread cap=1\n"); } - sme_cores = 0; - } else { - if (sme_env_set) { - bool ok = false; - int v = parse_uint_env(env_sme, "GGML_KLEIDIAI_SME", &ok); - sme_env_ok = ok; - - if (!ok) { - GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; falling back to runtime SME-core detection\n"); - detected_smcus = detect_num_smcus(); - sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0; - } else if (v == 0) { - sme_cores = 0; - } else { + } + + // Runtime-detect SME support and available SMCUs first. The detected SMCU + // count is used as the SME thread cap, and GGML_KLEIDIAI_SME can debug-override that: + // - unset: use runtime detection. + // - 0: disable SME-family kernels. + // - N > 0: use N as the SME thread cap, if an SME-family kernel is selectable. + if (sme_env_set) { + bool ok = false; + int v = parse_uint_env(env_sme, "GGML_KLEIDIAI_SME", &ok); + sme_env_ok = ok; + + if (ok) { + if (has_supported_sme_family) { sme_cores = v; + } else { + if (v > 0) { + GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but SME is not supported on this CPU; disabling SME-family kernels\n", v); + } + sme_cores = 0; } } else { - detected_smcus = detect_num_smcus(); - sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0; - } - - if (!sme_env_set && sme_cores == 0) { - GGML_LOG_WARN("kleidiai: SME supported but runtime SME-core detection returned 0; falling back to NEON\n"); + GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; using automatic SME thread cap\n"); } + } - if (sme_cores > 0) { - ctx.features |= CPU_FEATURE_SME; + if (sme_cores > 0 && has_supported_sme_family) { + ctx.features |= CPU_FEATURE_SME; + if (runtime_feat.has_sme2) { + ctx.features |= CPU_FEATURE_SME2; } } // Kernel selection - ctx.kernels_q4 = ggml_kleidiai_select_kernels_q4_0(ctx.features); - ctx.kernels_q8 = ggml_kleidiai_select_kernels_q8_0(ctx.features); + ctx.kernels_q4 = ggml_kleidiai_select_kernels_q4_0(ctx.features); + ctx.kernels_q4k = ggml_kleidiai_select_kernels_q4_k(ctx.features); + ctx.kernels_q8 = ggml_kleidiai_select_kernels_q8_0(ctx.features); + ctx.kernels_f32 = ggml_kleidiai_select_kernels_f32(ctx.features); if (!ctx.kernels_q4) { GGML_LOG_INFO("kleidiai: no compatible q4 kernels found for CPU features mask %d\n", (int)ctx.features); @@ -268,13 +410,32 @@ static void init_kleidiai_context(void) { GGML_LOG_INFO("kleidiai: primary q8 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q8->required_cpu)); } - ctx.sme_thread_cap = (ctx.features & CPU_FEATURE_SME) ? sme_cores : 0; + if (!ctx.kernels_q4k) { + GGML_LOG_INFO("kleidiai: no compatible q4_k kernels found for CPU features mask %d\n", (int) ctx.features); + } else { + GGML_LOG_INFO("kleidiai: primary q4_k kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q4k->required_cpu)); + } + + if (!ctx.kernels_f32) { + GGML_LOG_INFO("kleidiai: no compatible f32 kernels found for CPU features mask %d\n", (int)ctx.features); + } else { + GGML_LOG_INFO("kleidiai: primary f32 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_f32->required_cpu)); + } + + const bool has_selected_sme_family_kernel = + (ctx.kernels_q4 && is_sme_family(ctx.kernels_q4->required_cpu)) || + (ctx.kernels_q4k && is_sme_family(ctx.kernels_q4k->required_cpu)) || + (ctx.kernels_q8 && is_sme_family(ctx.kernels_q8->required_cpu)) || + (ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu)); + ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0; - if (ctx.features & CPU_FEATURE_SME) { + if (has_selected_sme_family_kernel) { if (sme_env_set && sme_env_ok && sme_cores > 0) { - GGML_LOG_INFO("kleidiai: SME enabled (GGML_KLEIDIAI_SME=%d override)\n", sme_cores); + GGML_LOG_INFO("kleidiai: SME enabled (GGML_KLEIDIAI_SME=%d debug override)\n", sme_cores); + } else if (sme_cap_detected) { + GGML_LOG_INFO("kleidiai: SME enabled (runtime-detected SME thread cap=%d)\n", sme_cores); } else { - GGML_LOG_INFO("kleidiai: SME enabled (runtime-detected SME cores=%d)\n", sme_cores); + GGML_LOG_INFO("kleidiai: SME enabled (runtime SME detected, conservative thread cap=%d)\n", sme_cores); } } else { GGML_LOG_INFO("kleidiai: SME disabled\n"); @@ -296,6 +457,59 @@ static inline size_t align_up(size_t value, size_t alignment) { return remainder == 0 ? value : value + (alignment - remainder); } +static inline size_t gcd_size(size_t a, size_t b) { + while (b != 0) { + const size_t t = a % b; + a = b; + b = t; + } + return a; +} + +static inline bool lcm_size(size_t a, size_t b, size_t & result) { + if (a == 0 || b == 0) { + result = 0; + return false; + } + const size_t g = gcd_size(a, b); + const size_t q = a / g; + if (q > SIZE_MAX / b) { + return false; + } + result = q * b; + return true; +} + +static inline size_t ceil_div_size(size_t a, size_t b) { + return b == 0 ? 0 : (a + b - 1) / b; +} + +static inline size_t kleidiai_chunk_cols(size_t n, int nth_total, bool disable_chunking, size_t n_step) { + const size_t multiplier = (nth_total == 1 || disable_chunking) ? 1 : std::max(1, (size_t) ctx.chunk_multiplier); + const size_t divisor = std::max(1, (size_t) nth_total * multiplier); + const size_t chunk_cols = align_up(std::max(1, ceil_div_size(n, divisor)), n_step); + return chunk_cols ? chunk_cols : n_step; +} + +struct kleidiai_block_args { + size_t lhs_bl; + size_t rhs_bl; + size_t pack_bl; +}; + +static inline kleidiai_block_args kleidiai_get_block_args(ggml_type rhs_type) { + switch (rhs_type) { + case GGML_TYPE_Q4_0: + return { QK4_0, QK4_0, QK4_0 }; + case GGML_TYPE_Q4_K: + return { GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL }; + case GGML_TYPE_Q8_0: + return { 0, 0, QK8_0 }; + default: + return { 0, 0, 0 }; + } +} + static inline bool kleidiai_pack_fallback_allowed() { if (ctx.sme_thread_cap <= 0) { return false; @@ -359,10 +573,18 @@ static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4() { return ctx.kernels_q4; } +static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4k() { + return ctx.kernels_q4k; +} + static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q8() { return ctx.kernels_q8; } +static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_f32() { + return ctx.kernels_f32; +} + template static int kleidiai_collect_kernel_chain_common( ggml_kleidiai_kernels * primary, @@ -375,11 +597,16 @@ static int kleidiai_collect_kernel_chain_common( } out[count++] = primary; - if ((primary->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) { - const cpu_feature fallback_mask = static_cast(features & ~CPU_FEATURE_SME); + if (primary->rhs_info.repack_mode == RHS_REPACK_SINGLE_ONLY) { + return count; + } + + if (is_sme_family(primary->required_cpu)) { + const cpu_feature fallback_mask = static_cast(features & ~(CPU_FEATURE_SME | CPU_FEATURE_SME2)); if (fallback_mask != CPU_FEATURE_NONE) { ggml_kleidiai_kernels * fallback = select_fallback(fallback_mask); if (fallback && fallback != primary && + fallback->rhs_info.repack_mode != RHS_REPACK_SINGLE_ONLY && fallback->lhs_type == primary->lhs_type && fallback->rhs_type == primary->rhs_type && fallback->op_type == primary->op_type) { @@ -404,12 +631,24 @@ static int kleidiai_collect_q4_chain(std::array & out) { + ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q4k(); + return kleidiai_collect_kernel_chain_common(primary, ctx.features, out, + [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_k(mask); }); +} + static int kleidiai_collect_q8_chain(std::array & out) { ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q8(); return kleidiai_collect_kernel_chain_common(primary, ctx.features, out, [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q8_0(mask); }); } +static int kleidiai_collect_f32_chain(std::array & out) { + ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_f32(); + return kleidiai_collect_kernel_chain_common(primary, ctx.features, out, + [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_f32(mask); }); +} + static inline int64_t ggml_ne(const ggml_tensor * tensor, int dim) { GGML_ASSERT(dim >= 0 && dim < GGML_MAX_DIMS); return tensor->ne[dim]; @@ -450,8 +689,10 @@ class tensor_traits : public ggml::cpu::tensor_traits { const size_t n = op->src[0]->ne[1]; const size_t m = op->src[1]->ne[1]; - if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) { - const size_t qk = (op->src[0]->type == GGML_TYPE_Q4_0) ? QK4_0 : QK8_0; + if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K) { + const kleidiai_block_args block_args = kleidiai_get_block_args(kernel_chain[0]->rhs_type); + const size_t qk = block_args.pack_bl; size_t cursor = 0; bool any_slot = false; @@ -484,6 +725,45 @@ class tensor_traits : public ggml::cpu::tensor_traits { return true; } + if (op->src[0]->type == GGML_TYPE_F32) { + ggml_kleidiai_kernels * primary = kernel_chain[0]; + kernel_info * gemv_kernel = primary ? &primary->gemv : nullptr; + if (is_gemv && op->src[1]->nb[0] == (int64_t) sizeof(float) && gemv_kernel && + gemv_kernel->get_lhs_offset_ex && gemv_kernel->get_rhs_packed_offset_ex && + gemv_kernel->run_kernel_ex && gemv_kernel->get_dst_offset) { + size = 0; + return true; + } + + size_t cursor = 0; + bool any_slot = false; + + for (int slot = 0; slot < slot_count; ++slot) { + ggml_kleidiai_kernels * kernels = kernel_chain[slot]; + lhs_packing_info * lhs_info = &kernels->gemm_lhs_info; + kernel_info * kernel = &kernels->gemm; + + if (!lhs_info || !lhs_info->packed_size_ex || !kernel) { + return false; + } + + const size_t mr = kernel->get_mr(); + const size_t kr = kernel->get_kr(); + const size_t sr = kernel->get_sr(); + + cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN); + cursor += lhs_info->packed_size_ex(m, k, 0, mr, kr, sr); + any_slot = true; + } + + if (!any_slot) { + return false; + } + + size = cursor; + return true; + } + if (op->src[0]->type == GGML_TYPE_F16) { const int64_t lhs_batch_size0 = op->src[1]->ne[2]; const int64_t rhs_batch_size0 = op->src[0]->ne[2]; @@ -538,19 +818,172 @@ class tensor_traits : public ggml::cpu::tensor_traits { bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * dst) override { if (dst->op == GGML_OP_MUL_MAT) { - if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) { + if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K || + dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K) { return compute_forward_qx(params, dst); + } else if (dst->src[0]->type == GGML_TYPE_F32) { + return compute_forward_f32(params, dst); } else if (dst->src[0]->type == GGML_TYPE_F16) { return compute_forward_fp16(params, dst); } - } else if (dst->op == GGML_OP_GET_ROWS) { - if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) { - return compute_forward_get_rows(params, dst); - } } return false; } + bool compute_forward_f32(ggml_compute_params * params, struct ggml_tensor * dst) { + GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32); + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + GGML_TENSOR_BINARY_OP_LOCALS + + if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + ggml_kleidiai_kernels * kernels = kleidiai_primary_kernel_f32(); + if (!kernels) { + return false; + } + + const size_t k = ne00; + const size_t m = ne11; + const size_t n = ne01; + const bool use_gemv = m == 1 && src1->nb[0] == (int64_t) sizeof(float) && + kernels->gemv.get_lhs_offset_ex && + kernels->gemv.get_rhs_packed_offset_ex && + kernels->gemv.run_kernel_ex && + kernels->gemv.get_dst_offset; + + kernel_info * kernel = use_gemv ? &kernels->gemv : &kernels->gemm; + lhs_packing_info * lhs_info = &kernels->gemm_lhs_info; + + if (!kernel || !kernel->get_lhs_offset_ex || + !kernel->get_rhs_packed_offset_ex || !kernel->run_kernel_ex || !kernel->get_dst_offset) { + return false; + } + + if (!use_gemv && (!lhs_info || !lhs_info->get_offset || !lhs_info->get_packed_offset_ex || + !lhs_info->packed_size_ex || !lhs_info->pack_func_ex)) { + return false; + } + + const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data); + const bool has_header = kleidiai_is_weight_header_valid(header); + + const uint8_t * rhs_base = has_header ? kleidiai_weight_slot_ptr(header, 0) + : static_cast(src0->data); + if (!rhs_base) { + return false; + } + + const int nth = params->nth > 0 ? params->nth : 1; + const int ith = params->ith; + + const size_t mr = kernel->get_mr(); + const size_t kr = kernel->get_kr(); + const size_t sr = kernel->get_sr(); + + const size_t lhs_packed_size = use_gemv ? 0 : lhs_info->packed_size_ex(m, k, 0, mr, kr, sr); + if (!use_gemv) { + GGML_ASSERT(lhs_packed_size <= params->wsize); + } + + uint8_t * lhs_packed = static_cast(params->wdata); + const size_t dst_stride = dst->nb[1]; + const size_t n_step = kernel->get_n_step() ? kernel->get_n_step() : 1; + const bool disable_chunking = ggml_is_numa(); + GGML_ASSERT(n <= (size_t) INT_MAX); + + const int64_t batch_size = ne12 * ne13; + for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) { + const int64_t i12 = batch_idx % ne12; + const int64_t i13 = batch_idx / ne12; + const uint8_t * lhs_batch_base = static_cast(src1->data) + i12 * src1->nb[2] + i13 * src1->nb[3]; + uint8_t * dst_batch_base = static_cast(dst->data) + i12 * dst->nb[2] + i13 * dst->nb[3]; + + if (!use_gemv) { + const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)mr); + int64_t max_threads = mr ? (m_roundup_mr / (int64_t)mr) : nth; + max_threads = std::max(1, max_threads); + const int64_t use_threads = std::min(nth, max_threads); + + if (ith < use_threads) { + const int64_t num_m_per_thread0 = round_down((size_t)(m_roundup_mr / use_threads), mr); + const int64_t num_m_per_threadN_1 = (int64_t)m - (use_threads - 1) * num_m_per_thread0; + + const int64_t m_start = (int64_t)ith * num_m_per_thread0; + const int64_t m_count = (ith == use_threads - 1) ? num_m_per_threadN_1 : num_m_per_thread0; + + const size_t base_packed_off = lhs_info->get_packed_offset_ex(m_start, k, 0, mr, kr, sr); + const size_t next_block_off = lhs_info->get_packed_offset_ex(m_start + mr, k, 0, mr, kr, sr); + const size_t row_stride_bytes = mr ? (next_block_off - base_packed_off) / mr : 0; + + int64_t remaining = m_count; + int64_t cur = m_start; + + while (remaining > 0) { + const int64_t take = std::min((int64_t)m - cur, remaining); + const size_t src_off = lhs_info->get_offset(cur, src1->nb[1]); + const void * src_ptr = lhs_batch_base + src_off; + const size_t dst_off = base_packed_off + (size_t)(cur - m_start) * row_stride_bytes; + void * dst_ptr = lhs_packed + dst_off; + + lhs_info->pack_func_ex(take, k, 0, mr, kr, sr, 0, src_ptr, src1->nb[1], dst_ptr); + + cur += take; + remaining -= take; + } + } + } + + if (ith == 0) { + ggml_threadpool_chunk_set(params->threadpool, 0); + } + + ggml_barrier(params->threadpool); + + const size_t chunk_cols = kleidiai_chunk_cols(n, nth, disable_chunking, n_step); + GGML_ASSERT(chunk_cols <= (size_t) INT_MAX); + + int current_col = ggml_threadpool_chunk_add(params->threadpool, (int) chunk_cols); + while ((size_t) current_col < n) { + const size_t n_start = (size_t) current_col; + const size_t n_to_process = std::min(chunk_cols, n - n_start); + + if (n_to_process > 0) { + const size_t lhs_offset = use_gemv ? kernel->get_lhs_offset_ex(0, k, 0) + : lhs_info->get_packed_offset_ex(0, k, 0, mr, kr, sr); + const size_t rhs_packed_offset = kernel->get_rhs_packed_offset_ex(n_start, k, 0); + const size_t dst_offset = kernel->get_dst_offset(0, n_start, dst_stride); + + const void * lhs_ptr = use_gemv ? lhs_batch_base + lhs_offset + : lhs_packed + lhs_offset; + const void * rhs_ptr = rhs_base + rhs_packed_offset; + float * dst_ptr = reinterpret_cast(dst_batch_base + dst_offset); + + kernel->run_kernel_ex(m, n_to_process, k, use_gemv ? src1->nb[1] : 0, + lhs_ptr, + rhs_ptr, + dst_ptr, + dst_stride, + sizeof(float), + -FLT_MAX, + FLT_MAX); + } + + current_col = ggml_threadpool_chunk_add(params->threadpool, (int) chunk_cols); + } + + if (batch_idx != batch_size - 1) { + ggml_barrier(params->threadpool); + } + } + + return true; + } + bool compute_forward_fp16(ggml_compute_params * params, struct ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; @@ -703,7 +1136,8 @@ class tensor_traits : public ggml::cpu::tensor_traits { } bool compute_forward_qx(struct ggml_compute_params * params, struct ggml_tensor * dst) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0); + GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K || + dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K); const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; @@ -746,8 +1180,10 @@ class tensor_traits : public ggml::cpu::tensor_traits { size_t n_step; size_t lhs_packed_size; size_t lhs_offset; - size_t n_offset; - size_t n_cols; + size_t lhs_bl; + size_t rhs_bl; + size_t pack_bl; + size_t lhs_packed_offset0; int assigned_threads; int thread_begin; int thread_end; @@ -772,6 +1208,8 @@ class tensor_traits : public ggml::cpu::tensor_traits { continue; } + const kleidiai_block_args block_args = kleidiai_get_block_args(kernels->rhs_type); + runtime[runtime_count] = { slot, kernels, @@ -784,7 +1222,9 @@ class tensor_traits : public ggml::cpu::tensor_traits { kinfo->get_n_step(), 0, 0, - 0, + block_args.lhs_bl, + block_args.rhs_bl, + block_args.pack_bl, 0, 0, 0, @@ -795,58 +1235,29 @@ class tensor_traits : public ggml::cpu::tensor_traits { } if (runtime_count == 0) { - ggml_kleidiai_kernels * fallback = ggml_kleidiai_select_kernels(ctx.features, dst); - if (!fallback) { - return false; - } - kernel_info * kinfo = is_gemv ? &fallback->gemv : &fallback->gemm; - lhs_packing_info * linfo = is_gemv ? &fallback->gemv_lhs_info : &fallback->gemm_lhs_info; - rhs_packing_info * rinfo = &fallback->rhs_info; - if (!kinfo || !linfo || !linfo->packed_size_ex || !linfo->pack_func_ex || - !kinfo->get_rhs_packed_offset_ex || !kinfo->run_kernel_ex || !kinfo->get_dst_offset || - !rinfo || !rinfo->pack_func_ex || !rinfo->packed_size_ex) { - return false; - } - kernel_chain[0] = fallback; - runtime[0] = { - 0, - fallback, - kinfo, - linfo, - kinfo->get_mr(), - kinfo->get_nr(), - kinfo->get_kr(), - kinfo->get_sr(), - kinfo->get_n_step(), - 0, - 0, - 0, - 0, - 0, - 0, - 0, - nullptr - }; - size_t rhs_size_fallback = 0; - const uint8_t * rhs_base = weight_for_slot(0, rhs_size_fallback); - if (!rhs_base) { - rhs_base = static_cast(src0->data); - } - runtime[0].rhs_base = rhs_base; - runtime_count = 1; + GGML_LOG_WARN("kleidiai: no runtime kernel slot available for supported op %s\n", dst->name); + return false; } const int nth_total = params->nth > 0 ? params->nth : 1; const int ith_total = params->ith; int sme_slot = -1; + int non_sme_slot = -1; for (int i = 0; i < runtime_count; ++i) { - if ((runtime[i].kernels->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) { + if (is_sme_family(runtime[i].kernels->required_cpu)) { sme_slot = i; break; } } + for (int i = 0; i < runtime_count; ++i) { + if (!is_sme_family(runtime[i].kernels->required_cpu)) { + non_sme_slot = i; + break; + } + } + const int sme_cap_limit = ctx.sme_thread_cap; const bool use_hybrid = sme_cap_limit > 0 && runtime_count > 1 && @@ -864,24 +1275,29 @@ class tensor_traits : public ggml::cpu::tensor_traits { if (!hybrid_enabled) { int chosen_slot = 0; if (too_small_for_hybrid && sme_slot != -1) { - chosen_slot = sme_slot; + chosen_slot = nth_total > sme_cap_limit && non_sme_slot != -1 ? non_sme_slot : sme_slot; } else if (runtime_count > 1 && ctx.sme_thread_cap > 0 && nth_total > ctx.sme_thread_cap) { chosen_slot = 1; } if (chosen_slot != 0 && chosen_slot < runtime_count) { runtime[0] = runtime[chosen_slot]; + runtime[0].assigned_threads = 0; + runtime[0].thread_begin = 0; + runtime[0].thread_end = 0; } runtime_count = runtime_count > 0 ? 1 : 0; // Recompute SME slot based on the collapsed runtime[0] sme_slot = -1; if (runtime_count > 0 && - (runtime[0].kernels->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) { + is_sme_family(runtime[0].kernels->required_cpu)) { sme_slot = 0; } } - int sme_cap = kleidiai_sme_thread_cap(); + const bool sme_single_only = sme_slot != -1 && + runtime[sme_slot].kernels->rhs_info.repack_mode == RHS_REPACK_SINGLE_ONLY; + int sme_cap = sme_single_only ? nth_total : kleidiai_sme_thread_cap(); if (sme_cap < 0) { sme_cap = nth_total; } @@ -896,6 +1312,8 @@ class tensor_traits : public ggml::cpu::tensor_traits { int fallback_indices[GGML_KLEIDIAI_MAX_KERNEL_SLOTS]; int fallback_count = 0; + // The current hybrid chain is bounded to SME + one non-SME fallback slot. + GGML_ASSERT(GGML_KLEIDIAI_MAX_KERNEL_SLOTS == 2); for (int i = 0; i < runtime_count; ++i) { if (i == sme_slot) { continue; @@ -915,23 +1333,14 @@ class tensor_traits : public ggml::cpu::tensor_traits { threads_remaining -= share; } - if (threads_remaining > 0) { - const int fallback_slot = (sme_slot != -1) ? sme_slot : 0; - runtime[fallback_slot].assigned_threads += threads_remaining; - threads_remaining = 0; - } - int thread_cursor = 0; for (int i = 0; i < runtime_count; ++i) { runtime[i].thread_begin = thread_cursor; thread_cursor += runtime[i].assigned_threads; runtime[i].thread_end = thread_cursor; } - - if (thread_cursor < nth_total && runtime_count > 0) { - runtime[runtime_count - 1].assigned_threads += nth_total - thread_cursor; - runtime[runtime_count - 1].thread_end = nth_total; - } + GGML_ASSERT(thread_cursor > 0); + const int active_threads = thread_cursor; int local_slot = -1; int local_ith = 0; @@ -942,83 +1351,73 @@ class tensor_traits : public ggml::cpu::tensor_traits { break; } } - if (local_slot == -1) { - return false; - } - const size_t k = ne00; const size_t m = ne11; const size_t n = ne01; size_t cursor = 0; for (int i = 0; i < runtime_count; ++i) { - const ggml_type slot_rhs_type = runtime[i].kernels->rhs_type; - const size_t slot_pack_size_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : - slot_rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : 0; - runtime[i].lhs_packed_size = runtime[i].lhs_info->packed_size_ex(m, k, slot_pack_size_arg, runtime[i].mr, runtime[i].kr, runtime[i].sr); + runtime[i].lhs_packed_size = runtime[i].lhs_info->packed_size_ex(m, k, runtime[i].pack_bl, runtime[i].mr, runtime[i].kr, runtime[i].sr); cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN); runtime[i].lhs_offset = cursor; + runtime[i].lhs_packed_offset0 = runtime[i].lhs_info->get_packed_offset_ex(0, k, runtime[i].lhs_bl, runtime[i].mr, runtime[i].kr, runtime[i].sr); cursor += runtime[i].lhs_packed_size; } GGML_ASSERT(cursor <= params->wsize); uint8_t * scratch = static_cast(params->wdata); - size_t assigned_cols = 0; - uint64_t weighted_total = 0; - if (runtime_count > 1 && sme_slot != -1) { - for (int i = 0; i < runtime_count; ++i) { - const uint64_t weight = (i == sme_slot) ? (sme_cap << 1) : 1; - weighted_total += (uint64_t)runtime[i].assigned_threads * weight; - } - } + size_t common_step = 1; for (int i = 0; i < runtime_count; ++i) { - runtime[i].n_offset = assigned_cols; if (runtime[i].assigned_threads == 0) { - runtime[i].n_cols = 0; - continue; - } - const size_t remaining_cols = n - assigned_cols; - if (remaining_cols == 0) { - runtime[i].n_cols = 0; continue; } - const size_t step = runtime[i].n_step ? runtime[i].n_step : 1; - size_t target = 0; - if (weighted_total > 0) { - const uint64_t weight = (i == sme_slot) ? (sme_cap << 1) : 1; - target = (size_t)(((uint64_t)n * runtime[i].assigned_threads * weight) / weighted_total); - } else { - target = (size_t)(((uint64_t)n * runtime[i].assigned_threads) / nth_total); - } - target = std::min(target, remaining_cols); - size_t aligned = round_down(target, step); - if (aligned == 0 && remaining_cols >= step) { - aligned = step; + size_t next_step = 0; + if (!lcm_size(common_step, runtime[i].n_step ? runtime[i].n_step : 1, next_step)) { + return false; } - runtime[i].n_cols = aligned; - assigned_cols += aligned; + common_step = next_step; } - - if (assigned_cols < n) { - for (int i = runtime_count - 1; i >= 0; --i) { - if (runtime[i].assigned_threads > 0) { - runtime[i].n_cols += n - assigned_cols; - break; - } - } + GGML_ASSERT(common_step > 0); + + const bool disable_chunking = ggml_is_numa(); + const size_t chunk_multiplier = std::max(1, ctx.chunk_multiplier); + const size_t chunk_divisor = (active_threads == 1 || disable_chunking) ? (size_t) active_threads : (size_t) active_threads * chunk_multiplier; + size_t chunk_cols = align_up(std::max(1, ceil_div_size(n, chunk_divisor)), common_step); + if (chunk_cols == 0) { + chunk_cols = common_step; } + // If common_step is larger than n, the loop below runs one valid tail chunk + // with cols == n. + const size_t nchunk_size = std::max(1, ceil_div_size(n, chunk_cols)); + GGML_ASSERT(nchunk_size <= (size_t)INT_MAX); + const int nchunk = (int)nchunk_size; const size_t dst_stride = dst->nb[1]; + auto run_chunk = [&](runtime_slot & slot, size_t global_start, size_t cols, uint8_t * dst_batch_base) { + const size_t rhs_packed_offset = slot.kernel->get_rhs_packed_offset_ex(global_start, k, slot.rhs_bl); + const size_t dst_offset = slot.kernel->get_dst_offset(0, global_start, dst_stride); + + const uint8_t * lhs_ptr = scratch + slot.lhs_offset + slot.lhs_packed_offset0; + const uint8_t * rhs_ptr = slot.rhs_base + rhs_packed_offset; + float * dst_ptr = reinterpret_cast(dst_batch_base + dst_offset); + + slot.kernel->run_kernel_ex(m, cols, k, slot.rhs_bl, + lhs_ptr, + rhs_ptr, + dst_ptr, + dst_stride, + sizeof(float), + -FLT_MAX, + FLT_MAX); + }; + for (int64_t batch_idx = 0; batch_idx < ne12; ++batch_idx) { const uint8_t * lhs_batch_base = static_cast(src1->data) + batch_idx * src1->nb[2]; uint8_t * dst_batch_base = static_cast(dst->data) + batch_idx * dst->nb[2]; - if (runtime[local_slot].assigned_threads > 0) { + if (local_slot != -1) { runtime_slot & slot = runtime[local_slot]; - const ggml_type slot_rhs_type = slot.kernels->rhs_type; - const size_t slot_lhs_exec_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : - slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0; const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)slot.mr); int64_t max_threads = slot.mr ? (m_roundup_mr / (int64_t)slot.mr) : slot.assigned_threads; max_threads = std::max(1, max_threads); @@ -1031,8 +1430,8 @@ class tensor_traits : public ggml::cpu::tensor_traits { const int64_t m_start = (int64_t)local_ith * num_m_per_thread0; const int64_t m_count = (local_ith == use_threads - 1) ? num_m_per_threadN_1 : num_m_per_thread0; - const size_t base_packed_off = slot.lhs_info->get_packed_offset_ex(m_start, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr); - const size_t next_block_off = slot.lhs_info->get_packed_offset_ex(m_start + slot.mr, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr); + const size_t base_packed_off = slot.lhs_info->get_packed_offset_ex(m_start, k, slot.lhs_bl, slot.mr, slot.kr, slot.sr); + const size_t next_block_off = slot.lhs_info->get_packed_offset_ex(m_start + slot.mr, k, slot.lhs_bl, slot.mr, slot.kr, slot.sr); const size_t row_stride_bytes = slot.mr ? (next_block_off - base_packed_off) / slot.mr : 0; int64_t remaining = m_count; @@ -1049,7 +1448,7 @@ class tensor_traits : public ggml::cpu::tensor_traits { const size_t dst_off = base_packed_off + (size_t)(cur - m_start) * row_stride_bytes; void * dst_ptr = lhs_packed + dst_off; - slot.lhs_info->pack_func_ex(take, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr, 0, src_ptr, src1->nb[1], dst_ptr); + slot.lhs_info->pack_func_ex(take, k, slot.lhs_bl, slot.mr, slot.kr, slot.sr, 0, src_ptr, src1->nb[1], dst_ptr); cur += take; remaining -= take; @@ -1057,48 +1456,30 @@ class tensor_traits : public ggml::cpu::tensor_traits { } } - ggml_barrier(params->threadpool); + if (ith_total == 0) { + ggml_threadpool_chunk_set(params->threadpool, active_threads); + } - runtime_slot & slot = runtime[local_slot]; - if (slot.n_cols > 0 && slot.assigned_threads > 0) { - int64_t active_threads = slot.assigned_threads; - const int64_t max_threads = slot.n_step ? (slot.n_cols / slot.n_step) : slot.assigned_threads; - if (max_threads > 0) { - active_threads = std::min(active_threads, std::max(1, max_threads)); - } - active_threads = std::max(1, active_threads); + // Publishes both LHS packing and the initialized dynamic chunk queue. + ggml_barrier(params->threadpool); - if (local_ith < active_threads) { - const size_t step = slot.n_step ? slot.n_step : 1; - const size_t chunk0 = round_down((size_t)(slot.n_cols / active_threads), step); - const size_t chunkN = slot.n_cols - (active_threads - 1) * chunk0; - const size_t local_start = (size_t)local_ith * chunk0; - const size_t cols = (local_ith == active_threads - 1) ? chunkN : chunk0; + if (local_slot != -1) { + runtime_slot & slot = runtime[local_slot]; + int current_chunk = ith_total; + while (current_chunk < nchunk) { + const size_t global_start = (size_t)current_chunk * chunk_cols; + if (global_start >= n) { + break; + } + const size_t cols = std::min(chunk_cols, n - global_start); if (cols > 0) { - const ggml_type slot_rhs_type = slot.kernels->rhs_type; - const size_t slot_lhs_exec_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : - slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0; - const size_t slot_rhs_block_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : - slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0; - const size_t global_start = slot.n_offset + local_start; - const size_t lhs_packed_offset = slot.lhs_info->get_packed_offset_ex(0, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr); - const size_t rhs_packed_offset = slot.kernel->get_rhs_packed_offset_ex(global_start, k, slot_rhs_block_arg); - const size_t dst_offset = slot.kernel->get_dst_offset(0, global_start, dst_stride); - - const uint8_t * lhs_ptr = scratch + slot.lhs_offset + lhs_packed_offset; - const uint8_t * rhs_ptr = slot.rhs_base + rhs_packed_offset; - float * dst_ptr = reinterpret_cast(dst_batch_base + dst_offset); - - slot.kernel->run_kernel_ex(m, cols, k, slot_rhs_block_arg, - lhs_ptr, - rhs_ptr, - dst_ptr, - dst_stride, - sizeof(float), - -FLT_MAX, - FLT_MAX); + // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths; + // only non-tail chunks are guaranteed to be n_step-aligned. + run_chunk(slot, global_start, cols, dst_batch_base); } + + current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1); } } @@ -1110,101 +1491,11 @@ class tensor_traits : public ggml::cpu::tensor_traits { return true; } - bool compute_forward_get_rows(struct ggml_compute_params * params, struct ggml_tensor * dst) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - GGML_TENSOR_BINARY_OP_LOCALS - - const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data); - const bool has_header = kleidiai_is_weight_header_valid(header); - - std::array kernel_chain; - const bool want_q8 = src0->type == GGML_TYPE_Q8_0; - const int chain_count = want_q8 ? kleidiai_collect_q8_chain(kernel_chain) - : kleidiai_collect_q4_chain(kernel_chain); - - ggml_kleidiai_kernels * kernels = nullptr; - const uint8_t * packed_base = static_cast(src0->data); - - if (has_header && chain_count > 0) { - int select_slot = 0; - if (select_slot >= header->slot_count) { - select_slot = header->slot_count - 1; - } - if (select_slot >= 0 && select_slot < chain_count) { - kernels = kernel_chain[select_slot]; - const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, select_slot); - if (slot_ptr) { - packed_base = slot_ptr; - } - } - } - - if (!kernels && chain_count > 0) { - kernels = kernel_chain[0]; - if (has_header) { - const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, 0); - if (slot_ptr) { - packed_base = slot_ptr; - } - } - } - - if (!kernels) { - return false; - } - - rhs_packing_info * rhs_info = &kernels->rhs_info; - kernel_info * kernel = &kernels->gemm; - if (!rhs_info->to_float || !kernel->get_nr) { - return false; - } - - const int64_t nc = ne00; - const int64_t nr = ggml_nelements(src1); - - const ggml_type rhs_type = kernels->rhs_type; - size_t block_len = 0; - size_t num_bytes_multiplier = 0; - if (rhs_type == GGML_TYPE_Q4_0) { - block_len = QK4_0; - num_bytes_multiplier = sizeof(uint16_t); - } else if (rhs_type == GGML_TYPE_Q8_0) { - block_len = QK8_0; - num_bytes_multiplier = sizeof(float); - } else { - return false; - } - - const size_t block_rows = kernel->get_nr(); - const size_t kr = kernel->get_kr(); - - const size_t packed_stride = rhs_info->packed_stride(nc, block_rows, kr, block_len); - - const int ith = params->ith; - const int nth = params->nth; - - const int dr = (nr + nth - 1) / nth; - const int ir0 = dr * ith; - const int ir1 = MIN(ir0 + dr, nr); - - for (int64_t i = ir0; i < ir1; ++i) { - GGML_ASSERT(src1->type == GGML_TYPE_I32); - int64_t row_idx = ((const int32_t *)src1->data)[i]; - GGML_ASSERT(row_idx >= 0 && row_idx < src0->ne[1]); - - float *out = (float *)((char *)dst->data + i * nb1); - rhs_info->to_float(packed_base, row_idx, nc, out, block_rows, packed_stride, kr, block_len, num_bytes_multiplier); - } - - return true; - } - public: int repack(struct ggml_tensor * tensor, const void * data, size_t data_size) { - GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0); + GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q4_K || + tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K || + tensor->type == GGML_TYPE_F32); const size_t n = tensor->ne[1]; const size_t k = tensor->ne[0]; @@ -1222,13 +1513,19 @@ class tensor_traits : public ggml::cpu::tensor_traits { cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN); std::array kernel_chain; - const bool want_q8 = tensor->type == GGML_TYPE_Q8_0; - const int slot_total = want_q8 ? kleidiai_collect_q8_chain(kernel_chain) - : kleidiai_collect_q4_chain(kernel_chain); + const bool want_q4k = tensor->type == GGML_TYPE_Q4_K; + const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K; + const bool want_f32 = tensor->type == GGML_TYPE_F32; + const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain) + : want_q8 ? kleidiai_collect_q8_chain(kernel_chain) + : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain) + : kleidiai_collect_q4_chain(kernel_chain); const bool allow_fallback = kleidiai_pack_fallback_allowed(); std::vector qdata; std::vector scales; + std::vector bias; + std::vector row_f32; if (want_q8 && slot_total > 0) { qdata.resize(n * k, 0); @@ -1237,21 +1534,38 @@ class tensor_traits : public ggml::cpu::tensor_traits { const size_t row_stride = tensor->nb[1]; const size_t k_blocks = (k + QK8_0 - 1) / QK8_0; + if (tensor->type == GGML_TYPE_Q6_K) { + if (k % QK_K != 0 || k % QK8_0 != 0) { + return -1; + } + row_f32.resize(k); + } + for (size_t row = 0; row < n; ++row) { - const auto * row_blocks = reinterpret_cast( - static_cast(data) + row * row_stride); + const uint8_t * row_data = static_cast(data) + row * row_stride; + + if (tensor->type == GGML_TYPE_Q6_K) { + dequantize_row_q6_K(reinterpret_cast(row_data), row_f32.data(), k); + } float max_abs = 0.0f; - for (size_t block = 0; block < k_blocks; ++block) { - const block_q8_0 & blk = row_blocks[block]; - const float d = GGML_FP16_TO_FP32(blk.d); - for (size_t l = 0; l < QK8_0; ++l) { - const size_t linear_idx = block * QK8_0 + l; - if (linear_idx >= k) { - break; + if (tensor->type == GGML_TYPE_Q6_K) { + for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) { + max_abs = std::max(max_abs, std::fabs(row_f32[linear_idx])); + } + } else { + const block_q8_0 * row_blocks = reinterpret_cast(row_data); + for (size_t block = 0; block < k_blocks; ++block) { + const block_q8_0 & blk = row_blocks[block]; + const float d = GGML_FP16_TO_FP32(blk.d); + for (size_t l = 0; l < QK8_0; ++l) { + const size_t linear_idx = block * QK8_0 + l; + if (linear_idx >= k) { + break; + } + const float value = d * static_cast(blk.qs[l]); + max_abs = std::max(max_abs, std::fabs(value)); } - const float value = d * static_cast(blk.qs[l]); - max_abs = std::max(max_abs, std::fabs(value)); } } @@ -1259,23 +1573,36 @@ class tensor_traits : public ggml::cpu::tensor_traits { scales[row] = scale; const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; - for (size_t block = 0; block < k_blocks; ++block) { - const block_q8_0 & blk = row_blocks[block]; - const float d = GGML_FP16_TO_FP32(blk.d); - for (size_t l = 0; l < QK8_0; ++l) { - const size_t linear_idx = block * QK8_0 + l; - if (linear_idx >= k) { - break; - } - const float value = d * static_cast(blk.qs[l]); - int32_t q = scale > 0.0f ? static_cast(std::lround(value * inv_scale)) : 0; + if (tensor->type == GGML_TYPE_Q6_K) { + for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) { + int32_t q = scale > 0.0f ? static_cast(std::lround(row_f32[linear_idx] * inv_scale)) : 0; q = std::clamp(q, -127, 127); qdata[row * k + linear_idx] = static_cast(q); } + } else { + const block_q8_0 * row_blocks = reinterpret_cast(row_data); + for (size_t block = 0; block < k_blocks; ++block) { + const block_q8_0 & blk = row_blocks[block]; + const float d = GGML_FP16_TO_FP32(blk.d); + for (size_t l = 0; l < QK8_0; ++l) { + const size_t linear_idx = block * QK8_0 + l; + if (linear_idx >= k) { + break; + } + const float value = d * static_cast(blk.qs[l]); + int32_t q = scale > 0.0f ? static_cast(std::lround(value * inv_scale)) : 0; + q = std::clamp(q, -127, 127); + qdata[row * k + linear_idx] = static_cast(q); + } + } } } } + if (want_f32 && slot_total > 0) { + bias.resize(n, 0.0f); + } + for (int slot = 0; slot < slot_total && slot < GGML_KLEIDIAI_MAX_KERNEL_SLOTS; ++slot) { if (!allow_fallback && slot > 0) { break; @@ -1291,9 +1618,16 @@ class tensor_traits : public ggml::cpu::tensor_traits { const size_t kr = kernel->get_kr(); const size_t sr = kernel->get_sr(); const ggml_type rhs_type = kernels->rhs_type; + if (rhs_type == GGML_TYPE_Q8_0 && (kernels->required_cpu & CPU_FEATURE_SVE)) { + GGML_ASSERT(nr == kernels->gemv.get_nr()); + GGML_ASSERT(kr == kernels->gemv.get_kr()); + GGML_ASSERT(sr == kernels->gemv.get_sr()); + } const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : - rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : 0; - if (block_len == 0) { + rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : + rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL : + rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX; + if (block_len == SIZE_MAX) { continue; } @@ -1309,6 +1643,9 @@ class tensor_traits : public ggml::cpu::tensor_traits { rhs_info->pack_func_ex(1, n, k, nr, kr, sr, QK4_0, 0, static_cast(data), nullptr, nullptr, dst_ptr, 0, ¶ms); + } else if (rhs_type == GGML_TYPE_Q4_K) { + rhs_info->pack_func_ex(1, n, k, nr, kr, sr, GGML_KLEIDIAI_Q4_K_BL, tensor->nb[1], + data, nullptr, nullptr, dst_ptr, 0, nullptr); } else if (rhs_type == GGML_TYPE_Q8_0) { struct kai_rhs_pack_qsi8cx_params params; params.lhs_zero_point = 1; @@ -1316,6 +1653,10 @@ class tensor_traits : public ggml::cpu::tensor_traits { rhs_info->pack_func_ex(1, n, k, nr, kr, sr, 0, 0, qdata.data(), nullptr, scales.data(), dst_ptr, 0, ¶ms); + } else if (rhs_type == GGML_TYPE_F32) { + rhs_info->pack_func_ex(1, n, k, nr, kr, sr, 0, tensor->nb[1], + data, bias.data(), nullptr, + dst_ptr, 0, nullptr); } else { continue; } @@ -1347,7 +1688,6 @@ static enum ggml_status ggml_backend_cpu_kleidiai_buffer_init_tensor(ggml_backen tensor->extra = (void *) ggml::cpu::kleidiai::get_tensor_traits(buffer, tensor); return GGML_STATUS_SUCCESS; - GGML_UNUSED(buffer); } static void ggml_backend_cpu_kleidiai_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, @@ -1390,7 +1730,9 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alignment(ggml_backend_b static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) { GGML_UNUSED(buft); - if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q8_0) { + if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q4_K && + tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_Q6_K && + tensor->type != GGML_TYPE_F32) { return ggml_nbytes(tensor); } @@ -1401,9 +1743,13 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_ cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN); std::array kernel_chain; - const bool want_q8 = tensor->type == GGML_TYPE_Q8_0; - const int slot_total = want_q8 ? kleidiai_collect_q8_chain(kernel_chain) - : kleidiai_collect_q4_chain(kernel_chain); + const bool want_q4k = tensor->type == GGML_TYPE_Q4_K; + const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K; + const bool want_f32 = tensor->type == GGML_TYPE_F32; + const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain) + : want_q8 ? kleidiai_collect_q8_chain(kernel_chain) + : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain) + : kleidiai_collect_q4_chain(kernel_chain); const bool allow_fallback = kleidiai_pack_fallback_allowed(); size_t slot_count = 0; @@ -1423,8 +1769,10 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_ const ggml_type rhs_type = kernels->rhs_type; const size_t block_len = rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : - rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : 0; - if (block_len == 0) { + rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL : + rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : + rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX; + if (block_len == SIZE_MAX) { continue; } @@ -1443,36 +1791,90 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_ namespace ggml::cpu::kleidiai { class extra_buffer_type : ggml::cpu::extra_buffer_type { bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override { + // GET_ROWS needs the original weights in a native buffer. + if (op->op != GGML_OP_MUL_MAT) { + return false; + } + std::array kernel_chain; const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain); - if ((op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) && - (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) && + const bool src0_is_kleidiai = op->src[0]->buffer && (ggml_n_dims(op->src[0]) == 2) && - op->src[0]->buffer->buft == ggml_backend_cpu_kleidiai_buffer_type() && - slot_total > 0) { + op->src[0]->buffer->buft->context == this && + slot_total > 0; + + if ((op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K || + op->src[0]->type == GGML_TYPE_F32) && + src0_is_kleidiai) { if (op->src[0]->type == GGML_TYPE_Q4_0 && ctx.kernels_q4 == nullptr) { return false; } if (op->src[0]->type == GGML_TYPE_Q8_0 && ctx.kernels_q8 == nullptr) { return false; } + if (op->src[0]->type == GGML_TYPE_Q6_K && ctx.kernels_q8 == nullptr) { + return false; + } + if (op->src[0]->type == GGML_TYPE_Q4_K && ctx.kernels_q4k == nullptr) { + return false; + } + if (op->src[0]->type == GGML_TYPE_F32 && ctx.kernels_f32 == nullptr) { + return false; + } if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) { return false; } - if ((op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_I32) && - ggml_ne(op->src[1], 3) == 1) { - return true; + + if (op->src[0]->type == GGML_TYPE_Q4_K) { + if (op->op != GGML_OP_MUL_MAT || op->src[1]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) { + return false; + } + return ggml_ne(op->src[1], 2) == 1 && ggml_ne(op->src[1], 3) == 1 && + op->src[1]->nb[0] == sizeof(float) && + (op->src[1]->ne[1] == 1 || op->src[1]->nb[1] == op->src[1]->ne[0] * sizeof(float)); } + + if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || + op->src[0]->type == GGML_TYPE_Q6_K) { + if (op->src[1]->type == GGML_TYPE_F32 && + ggml_ne(op->src[1], 3) == 1) { + return true; + } + return false; + } + + if (op->op != GGML_OP_MUL_MAT || op->src[1]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) { + return false; + } + + return true; } + return false; } ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override { - if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) { - if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_kleidiai_buffer_type()) { + if (op->op == GGML_OP_MUL_MAT) { + if (op->src[0]->buffer && op->src[0]->buffer->buft->context == this) { return (ggml::cpu::tensor_traits *) op->src[0]->extra; } else { + // KleidiAI only has kernels for Q4_0, Q4_K, Q8_0 and Q6_K. For a quantized weight of any + // other type (K-quants, IQ) it declines the op and returns nullptr below, so + // KleidiAI does not accelerate it. Another CPU backend may still take the op, + // and this can run during graph planning, so the message says what KleidiAI + // did rather than what ends up executing. Warn once per process. + if (ggml_is_quantized(op->src[0]->type) && + op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q4_K && + op->src[0]->type != GGML_TYPE_Q8_0 && op->src[0]->type != GGML_TYPE_Q6_K) { + static std::atomic warned(false); + if (!warned.exchange(true)) { + GGML_LOG_WARN("kleidiai: no kernel for tensor type %s, not accelerated by KleidiAI " + "(kernels available for Q4_0, Q4_K, Q8_0 and Q6_K)\n", + ggml_type_name(op->src[0]->type)); + } + } if (op->src[0]->type != GGML_TYPE_F16) { return nullptr; } @@ -1496,12 +1898,14 @@ ggml_backend_buffer_type_t ggml_backend_cpu_kleidiai_buffer_type(void) { static ggml::cpu::kleidiai::extra_buffer_type ctx; static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_kleidiai = { /* .iface = */ { - /* .get_name = */ ggml_backend_cpu_kleidiai_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_cpu_kleidiai_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_kleidiai_buffer_type_get_alignment, - /* .get_max_size = */ nullptr, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size, - /* .is_host = */ nullptr, + /* .get_name = */ ggml_backend_cpu_kleidiai_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_cpu_kleidiai_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ nullptr, + /* .get_alignment = */ ggml_backend_cpu_kleidiai_buffer_type_get_alignment, + /* .get_max_size = */ nullptr, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ nullptr, }, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0), /* .context = */ &ctx, diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 34e320e..d28840a 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -228,7 +228,7 @@ template <> inline vfloat32m8_t madd(vbfloat16m4_t a, vbfloat16m4_t b, vfloat32m //////////////////////////////////////////////////////////////////////////////////////////////////// // VECTORIZED HORIZONTAL SUM -#if defined(__ARM_NEON) +#if defined(__ARM_NEON) || defined(_M_ARM64) || defined(_M_ARM64EC) inline float hsum(float32x4_t x) { return vaddvq_f32(x); } @@ -384,6 +384,80 @@ template <> inline __m256bh load(const float *p) { } #endif +#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) +template T load_partial(const U *, int); +template T load_partial_u16(const void *, int); + +template <> inline __m128i load_partial_u16(const void *p, int n) { +#if defined(__AVX512BW__) && defined(__AVX512VL__) + return _mm_maskz_loadu_epi16((1u << n) - 1, p); +#else + const __m128i index = _mm_setr_epi32(0, 1, 2, 3); + const __m128i pairs = _mm_set1_epi32(n / 2); + __m128i v = _mm_castps_si128(_mm_maskload_ps((const float *)p, _mm_cmpgt_epi32(pairs, index))); + if (n & 1) { + uint16_t last; + memcpy(&last, (const char *)p + 2*(n - 1), sizeof(last)); + v = _mm_or_si128(v, _mm_and_si128(_mm_cmpeq_epi32(pairs, index), _mm_set1_epi32(last))); + } + return v; +#endif +} + +template <> inline __m256 load_partial(const float *p, int n) { + const __m256 index = _mm256_setr_ps(0, 1, 2, 3, 4, 5, 6, 7); + return _mm256_maskload_ps(p, _mm256_castps_si256(_mm256_cmp_ps(index, _mm256_set1_ps(n), _CMP_LT_OQ))); +} + +#if defined(__F16C__) +template <> inline __m256 load_partial(const ggml_fp16_t *p, int n) { + return _mm256_cvtph_ps(load_partial_u16<__m128i>(p, n)); +} +#endif + +#if defined(__AVX2__) || defined(__AVX512F__) +template <> inline __m256 load_partial(const ggml_bf16_t *p, int n) { + return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(load_partial_u16<__m128i>(p, n)), 16)); +} +#endif + +#if defined(__AVX512F__) +template <> inline __m256i load_partial_u16(const void *p, int n) { +#if defined(__AVX512BW__) && defined(__AVX512VL__) + return _mm256_maskz_loadu_epi16((1u << n) - 1, p); +#else + const __m256i index = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7); + const __m256i pairs = _mm256_set1_epi32(n / 2); + __m256i v = _mm256_maskload_epi32((const int *)p, _mm256_cmpgt_epi32(pairs, index)); + if (n & 1) { + uint16_t last; + memcpy(&last, (const char *)p + 2*(n - 1), sizeof(last)); + v = _mm256_or_si256(v, _mm256_and_si256(_mm256_cmpeq_epi32(pairs, index), _mm256_set1_epi32(last))); + } + return v; +#endif +} + +template <> inline __m512 load_partial(const float *p, int n) { + return _mm512_maskz_loadu_ps((1u << n) - 1, p); +} + +template <> inline __m512 load_partial(const ggml_fp16_t *p, int n) { + return _mm512_cvtph_ps(load_partial_u16<__m256i>(p, n)); +} + +template <> inline __m512 load_partial(const ggml_bf16_t *p, int n) { + return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(load_partial_u16<__m256i>(p, n)), 16)); +} +#endif + +#if defined(__AVX512BF16__) +template <> inline __m512bh load_partial(const ggml_bf16_t *p, int n) { + return (__m512bh) _mm512_maskz_loadu_epi16((uint64_t(1) << n) - 1, p); +} +#endif +#endif + #if defined(__riscv_v_intrinsic) template <> inline vfloat32m1_t load(const float *p) { return __riscv_vle32_v_f32m1(p, __riscv_vsetvlmax_e32m1()); @@ -492,8 +566,10 @@ class tinyBLAS { } bool matmul(int64_t m, int64_t n) { +#if !defined(__AVX__) && !defined(__AVX2__) && !defined(__AVX512F__) if (k % KN != 0) return false; +#endif // compute RM for only need tile with size RM&RM-1 #if VECTOR_REGISTERS == 32 if (m % 16 == 0 && (m/16 >= params->nth)) { @@ -548,7 +624,7 @@ class tinyBLAS { template inline void gemm_bloc(int64_t ii, int64_t jj) { D Cv[RN][RM] = {}; - for (int64_t l = 0; l < k; l += KN) { + for (int64_t l = 0; l + KN <= k; l += KN) { // help compiler for op order. if constexpr (RM <= RN) { V Av[RM]; @@ -574,6 +650,21 @@ class tinyBLAS { } } } +#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) + const int64_t rem = k % KN; + if (rem != 0) { + V Av[RM]; + for (int64_t i = 0; i < RM; ++i) { + Av[i] = load_partial(A + lda * (ii + i) + k - rem, rem); + } + for (int64_t j = 0; j < RN; ++j) { + V Bv = load_partial(B + ldb * (jj + j) + k - rem, rem); + for (int64_t i = 0; i < RM; ++i) { + Cv[j][i] = madd(Av[i], Bv, Cv[j][i]); + } + } + } +#endif for (int64_t j = 0; j < RN; ++j) for (int64_t i = 0; i < RM; ++i) C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]); @@ -1797,14 +1888,6 @@ class tinyBLAS_Q0_AVX { //PPC Implementation #if defined(__MMA__) -#define SAVE_ACC(ACC, ii, jj) \ - __builtin_mma_disassemble_acc(vec_C, ACC); \ - for (int I = 0; I < 4; I++) { \ - for (int J = 0; J < 4; J++) { \ - *((float*)(C+ii+((jj+J)*ldc)+I)) = *((float*)&vec_C[I]+J); \ - } \ - } \ - template struct mma_instr; @@ -1834,10 +1917,49 @@ class tinyBLAS_HP16_PPC { } void matmul(int64_t m, int64_t n) { - mnpack(0, m, 0, n); + int64_t mc = 256; + int64_t nc = 256; + int64_t kc = 256; + #if defined(_AIX) || defined(__BIG_ENDIAN__) + mc = 128; + nc = 128; + kc = 128; + #endif + if (k < kc) { + kc = k; + } + bool can_use_tiled = (m % mc == 0) && (n % nc == 0) && (k % kc == 0); + if (can_use_tiled) { + matmul_tiled(m, n, mc, nc, kc); + } else { + mnpack(0, m, 0, n); + } } private: + __attribute__((always_inline)) + inline void save_acc(acc_t * ACC, int64_t ii, int64_t jj) { + vec_t vec_C[4]; + __builtin_mma_disassemble_acc(vec_C, ACC); + for (int I = 0; I < 4; I++) { + for (int J = 0; J < 4; J++) { + *((float *)(C+ii+((jj+J)*ldc)+I)) = *((float *)&vec_C[I]+J); + } + } + } + + __attribute__((always_inline)) + inline void add_save_acc(acc_t * ACC, int64_t ii, int64_t jj) { + vec_t vec_C[4]; + __builtin_mma_disassemble_acc(vec_C, ACC); + for (int I = 0; I < 4; I++) { + for (int J = 0; J < 4; J++) { + float * c_ptr = (float *)(C+ii+((jj+J)*ldc)+I); + *c_ptr += *((float *)&vec_C[I]+J); + } + } + } + void vector_permute_store(vec_t *c, int numVec, unsigned char *vecOffset) { vec_t t[8], s[8]; vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}; @@ -1896,6 +2018,7 @@ class tinyBLAS_HP16_PPC { j = (rows >> 3); if (j > 0) { do { + aoffsets[0] = aoffset; if (cols == 4) { aoffsets[0] = aoffset; for (int it = 1; it < 4; ++it) @@ -1910,17 +2033,17 @@ class tinyBLAS_HP16_PPC { } i = (cols >> 3); if (i > 0) { - aoffsets[0] = aoffset; for (int it = 1; it < 8; ++it) { aoffsets[it] = aoffsets[it-1] + lda; } aoffset += 8 * lda; + do { for (int it = 0; it < 8; ++it) c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]); vector_permute_store(c_arr, 8, vecOffset); for (int it = 0; it < 8; ++it) - aoffsets[it] = aoffsets[it] + 8*lda; + aoffsets[it] = aoffsets[it] + 8; vecOffset += 128; i--; } while(i > 0); @@ -2147,8 +2270,8 @@ class tinyBLAS_HP16_PPC { mma_instr::outer_product(&acc_1, vec_A[x], vec_B[x+4]); } } - SAVE_ACC(&acc_0, ii, jj); - SAVE_ACC(&acc_1, ii, jj+4); + save_acc(&acc_0, ii, jj); + save_acc(&acc_1, ii, jj+4); } void KERNEL_8x4(int64_t ii, int64_t jj) { @@ -2164,8 +2287,8 @@ class tinyBLAS_HP16_PPC { mma_instr::outer_product(&acc_1, vec_A[x+4], vec_B[x]); } } - SAVE_ACC(&acc_0, ii, jj); - SAVE_ACC(&acc_1, ii+4, jj); + save_acc(&acc_0, ii, jj); + save_acc(&acc_1, ii+4, jj); } @@ -2186,13 +2309,64 @@ class tinyBLAS_HP16_PPC { mma_instr::outer_product(&acc_3, vec_A[x+4], vec_B[x+4]); } } - - SAVE_ACC(&acc_0, ii, jj); - SAVE_ACC(&acc_1, ii, jj+4); - SAVE_ACC(&acc_2, ii+4, jj); - SAVE_ACC(&acc_3, ii+4, jj+4); + save_acc(&acc_0, ii, jj); + save_acc(&acc_1, ii, jj+4); + save_acc(&acc_2, ii+4, jj); + save_acc(&acc_3, ii+4, jj+4); } + inline void MMA_16x8(vec_t * vec_A0, vec_t * vec_A1, vec_t * vec_B, acc_t * acc) { + for (int x = 0; x < 4; x ++) { + mma_instr::outer_product(&acc[0], vec_A0[x], vec_B[x]); + mma_instr::outer_product(&acc[1], vec_A0[x], vec_B[x+4]); + mma_instr::outer_product(&acc[2], vec_A0[x+4], vec_B[x]); + mma_instr::outer_product(&acc[3], vec_A0[x+4], vec_B[x+4]); + mma_instr::outer_product(&acc[4], vec_A1[x], vec_B[x]); + mma_instr::outer_product(&acc[5], vec_A1[x], vec_B[x+4]); + mma_instr::outer_product(&acc[6], vec_A1[x+4], vec_B[x]); + mma_instr::outer_product(&acc[7], vec_A1[x+4], vec_B[x+4]); + } + } + void KERNEL(int64_t ii, int64_t jj, int64_t mc, int64_t nc, int64_t kc, vec_t * vec_A, vec_t * vec_B, int64_t kk) { + for (int64_t i = 0; i < mc; i += 16) { + int A_base_addr = (mc / 8) * (i / 8) * 8; + for (int64_t j = 0; j < nc; j += 8) { + int B_base_addr = (nc / 8) * (j / 8) * 8; + acc_t acc[8]; + vec_t A0_block[8]; vec_t A1_block[8]; + for (int x = 0; x < 8; x++) + __builtin_mma_xxsetaccz(&acc[x]); + for (int64_t l = 0; l < kc; l += 8) { + int A0_block_idx = A_base_addr + (l / 8) * 8; + int A1_block_idx = A0_block_idx + (mc / 8) * 8; + int B_block_idx = B_base_addr + (l / 8) * 8; + vec_t* A0_block = &vec_A[A0_block_idx]; + vec_t* A1_block = &vec_A[A1_block_idx]; + vec_t* B_block = &vec_B[B_block_idx]; + MMA_16x8(A0_block, A1_block, B_block, acc); + } + if (kk == 0) { + save_acc(&acc[0], ii + i, jj + j); + save_acc(&acc[1], ii + i, jj + j + 4); + save_acc(&acc[2], ii + i + 4, jj + j); + save_acc(&acc[3], ii + i + 4, jj + j + 4); + save_acc(&acc[4], ii + i + 8, jj + j); + save_acc(&acc[5], ii + i + 8, jj + j + 4); + save_acc(&acc[6], ii + i + 12, jj + j); + save_acc(&acc[7], ii + i + 12, jj + j + 4); + } else { + add_save_acc(&acc[0], ii + i, jj + j); + add_save_acc(&acc[1], ii + i, jj + j + 4); + add_save_acc(&acc[2], ii + i + 4, jj + j); + add_save_acc(&acc[3], ii + i + 4, jj + j + 4); + add_save_acc(&acc[4], ii + i + 8, jj + j); + add_save_acc(&acc[5], ii + i + 8, jj + j + 4); + add_save_acc(&acc[6], ii + i + 12, jj + j); + add_save_acc(&acc[7], ii + i + 12, jj + j + 4); + } + } + } + } template void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n) { int64_t ytiles = (m - m0) / RM; @@ -2281,6 +2455,29 @@ class tinyBLAS_HP16_PPC { } } + void matmul_tiled(int64_t m, int64_t n, int64_t mc, int64_t nc, int64_t kc) { + int64_t ytiles = m / mc; + int64_t xtiles = n / nc; + int64_t tiles = xtiles * ytiles; + int64_t duty = (tiles + nth - 1) / nth; + int64_t start = duty * ith; + int64_t end = start + duty; + if (end > tiles) { + end = tiles; + } + for (int64_t job = start; job < end; ++job) { + int64_t ii = (job / xtiles) * mc; + int64_t jj = (job % xtiles) * nc; + for (int64_t kk = 0; kk < k; kk += kc) { + vec_t A_pack[kc * mc / 8]; + vec_t B_pack[kc * nc / 8]; + packNormal(A + (ii * lda) + kk, lda, kc, mc, (uint8_t *)A_pack); + packNormal(B + (jj * ldb) + kk, ldb, kc, nc, (uint8_t *)B_pack); + KERNEL(ii, jj, mc, nc, kc, A_pack, B_pack, kk); + } + } + } + template NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) { int64_t ytiles = (m - m0) / RM; @@ -2321,29 +2518,35 @@ class tinyBLAS_Q0_PPC { } void matmul(int64_t m, int64_t n) { - const int64_t mc = 64; - const int64_t kc = 64; + int64_t mc = 64; int64_t nc = 64; + int64_t kc = 64; + int64_t n_chunk = 64; + #if defined(_AIX) || defined(__BIG_ENDIAN__) + mc = 32; + nc = 32; + kc = 32; + n_chunk = 32; + #endif int64_t n_aligned = 0; - if (n % 64 == 0) { + if (n % n_chunk == 0) { n_aligned = n; } else if (n == 4) { n_aligned = 4; - } else if (n < 64) { + } else if (n < n_chunk) { n_aligned = (n / 8) * 8; } else { - n_aligned = (n / 64) * 64; + n_aligned = (n / n_chunk) * n_chunk; } - if (n_aligned > 0) { - if (n_aligned % 64 == 0) nc = 64; + if (n_aligned % n_chunk == 0) nc = n_chunk; else if (n_aligned == n) nc = n; else if (n_aligned % 32 == 0) nc = 32; else if (n_aligned % 24 == 0) nc = 24; else if (n_aligned % 16 == 0) nc = 16; else nc = 8; } - bool can_use_tiled = n_aligned > 0 && (m % mc == 0) && (k % kc == 0); + bool can_use_tiled = n_aligned > 0 && (m % mc == 0); if (can_use_tiled) { matmul_tiled(m, n_aligned, mc, nc, kc); if (n > n_aligned) { @@ -3060,13 +3263,14 @@ class tinyBLAS_Q0_PPC { int64_t ii = (job / xtiles) * mc; int64_t jj = (job % xtiles) * nc; for (int64_t kk = 0; kk < k; kk += kc) { + int64_t k_cur = MIN(kc, k - kk); if constexpr(is_Ablock_q4) { - packNormal_q4_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack); + packNormal_q4_fp16(A + ii * lda + kk, lda, mc, k_cur, (uint8_t *)A_pack); } else { - packNormal_q8_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack); + packNormal_q8_fp16(A + ii * lda + kk, lda, mc, k_cur, (uint8_t *)A_pack); } - packNormal_q8_fp16(B + jj * ldb + kk, ldb, nc, kc, (uint8_t *)B_pack); - KERNEL_Q0(ii, jj, mc, nc, kc, kk, A_pack, B_pack); + packNormal_q8_fp16(B + jj * ldb + kk, ldb, nc, k_cur, (uint8_t *)B_pack); + KERNEL_Q0(ii, jj, mc, nc, k_cur, kk, A_pack, B_pack); } } } @@ -3191,7 +3395,14 @@ class tinyBLAS_PPC { } void matmul(int64_t m, int64_t n) { - int64_t mc = 256; int64_t nc = 256; int64_t kc = 256; + int64_t mc = 256; + int64_t nc = 256; + int64_t kc = 256; + #if defined(_AIX) || defined(__BIG_ENDIAN__) + mc = 128; + nc = 128; + kc = 128; + #endif if (m % mc == 0 && n % nc == 0 && k % kc == 0) { matmul_tiled(m, n, mc, nc, kc); } else { diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index efb06b5..c2ec691 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -665,6 +665,7 @@ void ggml_compute_forward_add( ggml_compute_forward_add_non_quantized(params, dst); } break; case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -742,12 +743,17 @@ static void ggml_compute_forward_add_id_f32( // src1 indices const int i11 = *(int32_t *) ((char *) src2->data + i1*nb20 + i2*nb21); - GGML_ASSERT(i11 >= 0 && i11 < ne11); + GGML_ASSERT(i11 == -1 || (i11 >= 0 && i11 < ne11)); - ggml_vec_add_f32(ne0, - (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ), - (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), - (float *) ((char *) src1->data + i11*nb11)); + float * dst_row = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ); + const float * src0_row = (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01); + + if (i11 == -1) { + memcpy(dst_row, src0_row, ne0*sizeof(float)); + continue; + } + + ggml_vec_add_f32(ne0, dst_row, src0_row, (float *) ((char *) src1->data + i11*nb11)); } } @@ -1115,6 +1121,7 @@ void ggml_compute_forward_add1( } } break; case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1245,6 +1252,7 @@ void ggml_compute_forward_acc( case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1289,24 +1297,37 @@ static void ggml_compute_forward_sum_f32( } assert(ggml_is_scalar(dst)); - assert(src0->nb[0] == sizeof(float)); GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) - ggml_float sum = 0; - ggml_float row_sum = 0; + ggml_float sum = 0; - for (int64_t i03 = 0; i03 < ne03; i03++) { - for (int64_t i02 = 0; i02 < ne02; i02++) { - for (int64_t i01 = 0; i01 < ne01; i01++) { - ggml_vec_sum_f32_ggf(ne00, - &row_sum, - (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03)); - sum += row_sum; + if (nb00 == sizeof(float)) { + ggml_float row_sum = 0; + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + for (int64_t i01 = 0; i01 < ne01; i01++) { + ggml_vec_sum_f32_ggf(ne00, + &row_sum, + (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03)); + sum += row_sum; + } + } + } + } else { + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + for (int64_t i01 = 0; i01 < ne01; i01++) { + const char * row = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + for (int64_t i00 = 0; i00 < ne00; i00++) { + sum += (ggml_float) *(const float *) (row + i00*nb00); + } + } } } } + ((float *) dst->data)[0] = sum; } @@ -1894,204 +1915,126 @@ void ggml_compute_forward_repeat_back( // ggml_compute_forward_concat -static void ggml_compute_forward_concat_any( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - const size_t len = ggml_type_size(src0->type); +// rows of dst copied together, so that a strided source is read along neighboring rows +static constexpr int64_t GGML_CONCAT_TILE = 32; - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; - - const char * x; - - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03; - } else { - x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13; - } +// copy n_rows rows of n elements of es bytes from a source with element stride nb0, element by element +template +static void ggml_concat_copy_strided(char * y, size_t nby, const char * x, size_t nbx, size_t nb0, int64_t n, int64_t n_rows) { + for (int64_t i0 = 0; i0 < n; i0++) { + for (int64_t r = 0; r < n_rows; r++) { + memcpy(y + r*nby + i0*es, x + r*nbx + i0*nb0, es); + } + } +} - char * y = (char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3; +// copy n_rows rows of n elements of src into dst, rows are nbx bytes apart in src and nby bytes apart in dst +static void ggml_concat_copy_rows(char * y, size_t nby, const ggml_tensor * src, const char * x, size_t nbx, int64_t n, int64_t n_rows) { + const size_t es = ggml_type_size(src->type); - memcpy(y, x, len); - } - } + if (src->nb[0] == es) { + const size_t row_size = ggml_row_size(src->type, n); + for (int64_t r = 0; r < n_rows; r++) { + memcpy(y + r*nby, x + r*nbx, row_size); } + return; + } + + switch (es) { + case 1: ggml_concat_copy_strided<1>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + case 2: ggml_concat_copy_strided<2>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + case 4: ggml_concat_copy_strided<4>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + case 8: ggml_concat_copy_strided<8>(y, nby, x, nbx, src->nb[0], n, n_rows); break; + default: GGML_ABORT("unsupported element size"); } } -static void ggml_compute_forward_concat_i8( +// each dst row comes from src0 or src1, or for dim 0 from the row of src0 followed by the row of src1 +void ggml_compute_forward_concat( const ggml_compute_params * params, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t)); - - const int ith = params->ith; - const int nth = params->nth; - GGML_TENSOR_BINARY_OP_LOCALS const int32_t dim = ggml_get_op_params_i32(dst, 0); GGML_ASSERT(dim >= 0 && dim < 4); + GGML_ASSERT(nb0 == ggml_type_size(dst->type)); + GGML_ASSERT(!ggml_is_quantized(dst->type) || (nb00 == nb0 && nb10 == nb0)); + GGML_ASSERT(dim != 0 || (ne00 % ggml_blck_size(dst->type) == 0 && ne10 % ggml_blck_size(dst->type) == 0)); - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; + const size_t y1 = dim == 0 ? ggml_row_size(dst->type, ne00) : 0; - const int8_t * x; + const int64_t nr = ne1*ne2*ne3; + const int64_t dr = (nr + params->nth - 1)/params->nth; - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const int8_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); - } else { - x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); - } + const int64_t ir0 = dr*params->ith; + const int64_t ir1 = MIN(ir0 + dr, nr); - int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + for (int64_t ir = ir0; ir < ir1; ) { + int64_t i[4] = { 0, ir % ne1, (ir/ne1) % ne2, ir/(ne1*ne2) }; - *y = *x; - } - } + // a tile of consecutive rows sharing i2, i3 and their source + int64_t nt = MIN(MIN(ir1 - ir, ne1 - i[1]), GGML_CONCAT_TILE); + if (dim == 1 && i[1] < ne01) { + nt = MIN(nt, ne01 - i[1]); } - } -} - -static void ggml_compute_forward_concat_f16( - const ggml_compute_params * params, - ggml_tensor * dst) { - - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t)); - - const int ith = params->ith; - const int nth = params->nth; - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; - - const ggml_fp16_t * x; - - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const ggml_fp16_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); - } else { - x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); - } + char * y = (char *) dst->data + i[1]*nb1 + i[2]*nb2 + i[3]*nb3; - ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + if (dim == 0 || i[dim] < src0->ne[dim]) { + ggml_concat_copy_rows(y, nb1, src0, (const char *) src0->data + i[1]*nb01 + i[2]*nb02 + i[3]*nb03, nb01, ne00, nt); + } - *y = *x; - } - } + if (dim == 0 || i[dim] >= src0->ne[dim]) { + i[dim] -= src0->ne[dim]; // i[0] does not address rows + ggml_concat_copy_rows(y + y1, nb1, src1, (const char *) src1->data + i[1]*nb11 + i[2]*nb12 + i[3]*nb13, nb11, ne10, nt); } + + ir += nt; } } -static void ggml_compute_forward_concat_f32( - const ggml_compute_params * params, - ggml_tensor * dst) { +// row loop shared by the BF16 unary ops +template +static void ggml_compute_forward_unary_bf16_rows( + const ggml_compute_params * params, + ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float)); + assert(ggml_is_contiguous_rows(src0)); + assert(ggml_are_same_shape(src0, dst)); + + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) const int ith = params->ith; const int nth = params->nth; - GGML_TENSOR_BINARY_OP_LOCALS - - const int32_t dim = ggml_get_op_params_i32(dst, 0); - - GGML_ASSERT(dim >= 0 && dim < 4); - - int64_t o[4] = {0, 0, 0, 0}; - o[dim] = src0->ne[dim]; - - const float * x; - - // TODO: smarter multi-theading - for (int i3 = 0; i3 < ne3; i3++) { - for (int i2 = ith; i2 < ne2; i2 += nth) { - for (int i1 = 0; i1 < ne1; i1++) { - for (int i0 = 0; i0 < ne0; i0++) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const float *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03); - } else { - x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13); - } - - float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + const int nc = src0->ne[0]; + const int nr = ggml_nrows(src0); - *y = *x; - } - } - } - } -} + // rows per thread + const int dr = (nr + nth - 1)/nth; -void ggml_compute_forward_concat( - const ggml_compute_params * params, - ggml_tensor * dst) { + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); - const ggml_tensor * src0 = dst->src[0]; + for (int ir = ir0; ir < ir1; ++ir) { + const int i3 = ir/(ne02*ne01); + const int i2 = (ir - i3*ne02*ne01)/ne01; + const int i1 = (ir - i3*ne02*ne01 - i2*ne01); - switch (src0->type) { - case GGML_TYPE_F16: - case GGML_TYPE_BF16: - case GGML_TYPE_I16: - { - ggml_compute_forward_concat_f16(params, dst); - } break; - case GGML_TYPE_I8: - { - ggml_compute_forward_concat_i8(params, dst); - } break; - case GGML_TYPE_F32: - case GGML_TYPE_I32: - { - ggml_compute_forward_concat_f32(params, dst); - } break; - default: - { - ggml_compute_forward_concat_any(params, dst); - } + vec_op(nc, + (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1), + (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01)); } } @@ -2192,6 +2135,12 @@ static void ggml_compute_forward_gelu_f16( } } +static void ggml_compute_forward_gelu_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + ggml_compute_forward_unary_bf16_rows(params, dst); +} + static void ggml_compute_forward_gelu( const ggml_compute_params * params, ggml_tensor * dst) { @@ -2207,6 +2156,10 @@ static void ggml_compute_forward_gelu( { ggml_compute_forward_gelu_f16(params, dst); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_gelu_bf16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -2235,8 +2188,42 @@ static void ggml_compute_forward_fill_f32(const ggml_compute_params * params, gg } } +static void ggml_compute_forward_fill_f16(const ggml_compute_params * params, ggml_tensor * dst) { + const ggml_fp16_t c = GGML_CPU_FP32_TO_FP16(ggml_get_op_params_f32(dst, 0)); + + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne); + GGML_TENSOR_LOCALS(size_t, nb, dst, nb); + + const auto [ir0, ir1] = get_thread_range(params, dst); + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t i03 = ir/(ne2*ne1); + const int64_t i02 = (ir - i03*ne2*ne1)/ne1; + const int64_t i01 = (ir - i03*ne2*ne1 - i02*ne1); + + ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1); + + ggml_vec_set_f16(ne0, dst_ptr, c); + } +} + void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor * dst) { - ggml_compute_forward_fill_f32(params, dst); + const ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_fill_f32(params, dst); + } break; + case GGML_TYPE_F16: + { + ggml_compute_forward_fill_f16(params, dst); + } break; + default: + { + GGML_ABORT("unsupported type for ggml_compute_forward_fill: %s", ggml_type_name(src0->type)); + } + } } // ggml_compute_tri @@ -2388,6 +2375,12 @@ static void ggml_compute_forward_gelu_erf_f16( } } +static void ggml_compute_forward_gelu_erf_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + ggml_compute_forward_unary_bf16_rows(params, dst); +} + static void ggml_compute_forward_gelu_erf( const ggml_compute_params * params, ggml_tensor * dst) { @@ -2403,6 +2396,10 @@ static void ggml_compute_forward_gelu_erf( { ggml_compute_forward_gelu_erf_f16(params, dst); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_gelu_erf_bf16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -2507,6 +2504,12 @@ static void ggml_compute_forward_gelu_quick_f16( } } +static void ggml_compute_forward_gelu_quick_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + ggml_compute_forward_unary_bf16_rows(params, dst); +} + static void ggml_compute_forward_gelu_quick( const ggml_compute_params * params, ggml_tensor * dst) { @@ -2522,6 +2525,10 @@ static void ggml_compute_forward_gelu_quick( { ggml_compute_forward_gelu_quick_f16(params, dst); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_gelu_quick_bf16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -2626,6 +2633,12 @@ static void ggml_compute_forward_silu_f16( } } +static void ggml_compute_forward_silu_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + ggml_compute_forward_unary_bf16_rows(params, dst); +} + static void ggml_compute_forward_silu( const ggml_compute_params * params, ggml_tensor * dst) { @@ -2641,6 +2654,10 @@ static void ggml_compute_forward_silu( { ggml_compute_forward_silu_f16(params, dst); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_silu_bf16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -2844,7 +2861,9 @@ void ggml_compute_forward_silu_back( // ggml_compute_forward_reglu -static void ggml_compute_forward_reglu_f32( +// row loop shared by the BF16 GLU ops +template +static void ggml_compute_forward_glu_bf16_rows( const ggml_compute_params * params, ggml_tensor * dst) { @@ -2882,30 +2901,22 @@ static void ggml_compute_forward_reglu_f32( const int ir1 = MIN(ir0 + dr, nr); for (int i1 = ir0; i1 < ir1; i1++) { - float * src0_p = (float *) (src0_d + i1*src0_o); - float * src1_p = (float *) (src1_d + i1*src1_o); + const ggml_bf16_t * src0_p = (const ggml_bf16_t *) (src0_d + i1*src0_o); + const ggml_bf16_t * src1_p = (const ggml_bf16_t *) (src1_d + i1*src1_o); + ggml_bf16_t * dst_p = (ggml_bf16_t *) ((char *) dst->data + i1*(dst->nb[1])); if (!src1) { src0_p += swapped ? nc : 0; src1_p += swapped ? 0 : nc; } - ggml_vec_reglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p); - -#ifndef NDEBUG - for (int k = 0; k < nc; k++) { - const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k]; - GGML_UNUSED(x); - assert(!isnan(x)); - assert(!isinf(x)); - } -#endif // NDEBUG + vec_op(nc, dst_p, src0_p, src1_p); } } -static void ggml_compute_forward_reglu_f16( - const ggml_compute_params * params, - ggml_tensor * dst) { +static void ggml_compute_forward_reglu_f32( + const ggml_compute_params * params, + ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; @@ -2941,36 +2952,101 @@ static void ggml_compute_forward_reglu_f16( const int ir1 = MIN(ir0 + dr, nr); for (int i1 = ir0; i1 < ir1; i1++) { - ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o); - ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o); + float * src0_p = (float *) (src0_d + i1*src0_o); + float * src1_p = (float *) (src1_d + i1*src1_o); if (!src1) { src0_p += swapped ? nc : 0; src1_p += swapped ? 0 : nc; } - ggml_vec_reglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p); + ggml_vec_reglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p); #ifndef NDEBUG for (int k = 0; k < nc; k++) { - const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k]; - const float v = GGML_FP16_TO_FP32(x); - GGML_UNUSED(v); - assert(!isnan(v)); - assert(!isinf(v)); + const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k]; + GGML_UNUSED(x); + assert(!isnan(x)); + assert(!isinf(x)); } #endif // NDEBUG } } -static void ggml_compute_forward_reglu( - const ggml_compute_params * params, - ggml_tensor * dst) { +static void ggml_compute_forward_reglu_f16( + const ggml_compute_params * params, + ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; - - switch (src0->type) { - case GGML_TYPE_F32: + const ggml_tensor * src1 = dst->src[1]; + char * src0_d = (char *) src0->data; + char * src1_d = (char *) (src1 ? src1->data : src0->data); + const size_t src0_o = src0->nb[1]; + const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(ggml_is_contiguous_1(dst)); + + if (src1) { + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src0->type == src1->type); + } + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2; + const int nr = ggml_nrows(src0); + + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == nr); + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int i1 = ir0; i1 < ir1; i1++) { + ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o); + ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o); + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + ggml_vec_reglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p); + +#ifndef NDEBUG + for (int k = 0; k < nc; k++) { + const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k]; + const float v = GGML_FP16_TO_FP32(x); + GGML_UNUSED(v); + assert(!isnan(v)); + assert(!isinf(v)); + } +#endif // NDEBUG + } +} + +static void ggml_compute_forward_reglu_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + ggml_compute_forward_glu_bf16_rows(params, dst); +} + +static void ggml_compute_forward_reglu( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: { ggml_compute_forward_reglu_f32(params, dst); } break; @@ -2978,6 +3054,10 @@ static void ggml_compute_forward_reglu( { ggml_compute_forward_reglu_f16(params, dst); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_reglu_bf16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -3106,6 +3186,12 @@ static void ggml_compute_forward_geglu_f16( } } +static void ggml_compute_forward_geglu_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + ggml_compute_forward_glu_bf16_rows(params, dst); +} + static void ggml_compute_forward_geglu( const ggml_compute_params * params, ggml_tensor * dst) { @@ -3121,6 +3207,10 @@ static void ggml_compute_forward_geglu( { ggml_compute_forward_geglu_f16(params, dst); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_geglu_bf16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -3249,6 +3339,12 @@ static void ggml_compute_forward_swiglu_f16( } } +static void ggml_compute_forward_swiglu_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + ggml_compute_forward_glu_bf16_rows(params, dst); +} + static void ggml_compute_forward_swiglu( const ggml_compute_params * params, ggml_tensor * dst) { @@ -3264,6 +3360,10 @@ static void ggml_compute_forward_swiglu( { ggml_compute_forward_swiglu_f16(params, dst); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_swiglu_bf16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -3340,6 +3440,73 @@ static void ggml_compute_forward_swiglu_oai_f32( } } +static void ggml_compute_forward_swiglu_oai_f16( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + char * src0_d = (char *) src0->data; + char * src1_d = (char *) (src1 ? src1->data : src0->data); + const size_t src0_o = src0->nb[1]; + const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(ggml_is_contiguous_1(dst)); + + if (src1) { + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src0->type == src1->type); + } + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2; + const int nr = ggml_nrows(src0); + + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == nr); + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + const float alpha = ggml_get_op_params_f32(dst, 2); + const float limit = ggml_get_op_params_f32(dst, 3); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int i1 = ir0; i1 < ir1; i1++) { + ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o); + ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o); + ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])); + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + for (int k = 0; k < nc; k++) { + const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit); + const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit); + const float out_glu = x / (1.f + expf(alpha * (-x))); + dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f)); + } + +#ifndef NDEBUG + for (int k = 0; k < nc; k++) { + const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]); + GGML_UNUSED(x); + assert(!isnan(x)); + assert(!isinf(x)); + } +#endif // NDEBUG + } +} + static void ggml_compute_forward_swiglu_oai( const ggml_compute_params * params, ggml_tensor * dst) { @@ -3351,6 +3518,10 @@ static void ggml_compute_forward_swiglu_oai( { ggml_compute_forward_swiglu_oai_f32(params, dst); } break; + case GGML_TYPE_F16: + { + ggml_compute_forward_swiglu_oai_f16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -3358,6 +3529,139 @@ static void ggml_compute_forward_swiglu_oai( } } +// ggml_compute_forward_swiglu_clamp + +static void ggml_compute_forward_swiglu_clamp_f32(const ggml_compute_params * params, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + char * src0_d = (char *) src0->data; + char * src1_d = (char *) (src1 ? src1->data : src0->data); + const size_t src0_o = src0->nb[1]; + const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(ggml_is_contiguous_1(dst)); + + if (src1) { + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src0->type == src1->type); + } + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2; + const int nr = ggml_nrows(src0); + + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == nr); + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + const float limit = ggml_get_op_params_f32(dst, 3); + + const int dr = (nr + nth - 1) / nth; + const int ir0 = dr * ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int i1 = ir0; i1 < ir1; i1++) { + float * src0_p = (float *) (src0_d + i1 * src0_o); + float * src1_p = (float *) (src1_d + i1 * src1_o); + float * dst_p = (float *) ((char *) dst->data + i1 * (dst->nb[1])); + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + for (int k = 0; k < nc; k++) { + const float gate = std::min(src0_p[k], limit); + const float up = std::clamp(src1_p[k], -limit, limit); + dst_p[k] = gate / (1.f + expf(-gate)) * up; + } + +#ifndef NDEBUG + for (int k = 0; k < nc; k++) { + const float x = dst_p[k]; + GGML_UNUSED(x); + assert(!isnan(x)); + assert(!isinf(x)); + } +#endif // NDEBUG + } +} + +static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * params, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + char * src0_d = (char *) src0->data; + char * src1_d = (char *) (src1 ? src1->data : src0->data); + const size_t src0_o = src0->nb[1]; + const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(ggml_is_contiguous_1(dst)); + + if (src1) { + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src0->type == src1->type); + } + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2; + const int nr = ggml_nrows(src0); + + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == nr); + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + const float limit = ggml_get_op_params_f32(dst, 3); + + const int dr = (nr + nth - 1) / nth; + const int ir0 = dr * ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int i1 = ir0; i1 < ir1; i1++) { + ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1 * src0_o); + ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1 * src1_o); + ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1 * (dst->nb[1])); + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + for (int k = 0; k < nc; k++) { + const float gate = std::min(GGML_FP16_TO_FP32(src0_p[k]), limit); + const float up = std::clamp(GGML_FP16_TO_FP32(src1_p[k]), -limit, limit); + dst_p[k] = GGML_FP32_TO_FP16(gate / (1.f + expf(-gate)) * up); + } + +#ifndef NDEBUG + for (int k = 0; k < nc; k++) { + const float x = GGML_FP16_TO_FP32(dst_p[k]); + GGML_UNUSED(x); + assert(!isnan(x)); + assert(!isinf(x)); + } +#endif // NDEBUG + } +} + +static void ggml_compute_forward_swiglu_clamp(const ggml_compute_params * params, ggml_tensor * dst) { + switch (dst->src[0]->type) { + case GGML_TYPE_F32: + ggml_compute_forward_swiglu_clamp_f32(params, dst); + break; + case GGML_TYPE_F16: + ggml_compute_forward_swiglu_clamp_f16(params, dst); + break; + default: + GGML_ABORT("fatal error"); + } +} + // ggml_compute_forward_geglu_erf static void ggml_compute_forward_geglu_erf_f32( @@ -3479,6 +3783,12 @@ static void ggml_compute_forward_geglu_erf_f16( } } +static void ggml_compute_forward_geglu_erf_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + ggml_compute_forward_glu_bf16_rows(params, dst); +} + static void ggml_compute_forward_geglu_erf( const ggml_compute_params * params, ggml_tensor * dst) { @@ -3494,6 +3804,10 @@ static void ggml_compute_forward_geglu_erf( { ggml_compute_forward_geglu_erf_f16(params, dst); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_geglu_erf_bf16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -3622,6 +3936,12 @@ static void ggml_compute_forward_geglu_quick_f16( } } +static void ggml_compute_forward_geglu_quick_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + ggml_compute_forward_glu_bf16_rows(params, dst); +} + static void ggml_compute_forward_geglu_quick( const ggml_compute_params * params, ggml_tensor * dst) { @@ -3637,7 +3957,11 @@ static void ggml_compute_forward_geglu_quick( { ggml_compute_forward_geglu_quick_f16(params, dst); } break; - default: + case GGML_TYPE_BF16: + { + ggml_compute_forward_geglu_quick_bf16(params, dst); + } break; + default: { GGML_ABORT("fatal error"); } @@ -3654,8 +3978,6 @@ static void ggml_compute_forward_norm_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); - GGML_ASSERT(src0->nb[0] == sizeof(float)); - const int ith = params->ith; const int nth = params->nth; @@ -3669,25 +3991,49 @@ static void ggml_compute_forward_norm_f32( for (int64_t i03 = 0; i03 < ne03; i03++) { for (int64_t i02 = 0; i02 < ne02; i02++) { for (int64_t i01 = ith; i01 < ne01; i01 += nth) { - const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03); + const char * x = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + char * y = (char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - float sum = 0.0; - ggml_vec_sum_f32(ne00, &sum, x); - float mean = sum/ne00; + if (nb00 == sizeof(float) && nb0 == sizeof(float)) { + const float * xf = (const float *) x; - float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); - float variance = 0; + float sum = 0.0; + ggml_vec_sum_f32(ne00, &sum, xf); + float mean = sum/ne00; + + float * yf = (float *) y; + float variance = 0; #ifdef GGML_USE_ACCELERATE - mean = -mean; - vDSP_vsadd(x, 1, &mean, y, 1, ne00); - vDSP_measqv(y, 1, &variance, ne00); + mean = -mean; + vDSP_vsadd(xf, 1, &mean, yf, 1, ne00); + vDSP_measqv(yf, 1, &variance, ne00); #else - variance = ggml_vec_cvar_f32(ne00, y, x, mean); + variance = ggml_vec_cvar_f32(ne00, yf, xf, mean); #endif //GGML_USE_ACCELERATE - const float scale = 1.0f/sqrtf(variance + eps); - ggml_vec_scale_f32(ne00, y, scale); + const float scale = 1.0f/sqrtf(variance + eps); + ggml_vec_scale_f32(ne00, yf, scale); + } else { + float sum = 0.0; + for (int64_t i00 = 0; i00 < ne00; i00++) { + sum += *(const float *) (x + i00*nb00); + } + const float mean = sum/ne00; + + float variance = 0.0f; + for (int64_t i00 = 0; i00 < ne00; i00++) { + const float v = *(const float *) (x + i00*nb00) - mean; + *(float *) (y + i00*nb0) = v; + variance += v * v; + } + variance /= ne00; + + const float scale = 1.0f/sqrtf(variance + eps); + for (int64_t i00 = 0; i00 < ne00; i00++) { + *(float *) (y + i00*nb0) *= scale; + } + } } } } @@ -3713,11 +4059,27 @@ void ggml_compute_forward_norm( // ggml_compute_forward_group_rms_norm +// fusion kinds that can be combined with the rms_norm computation in a single pass. +// extend this enum when adding new fused variants (e.g. FUSE_ADD, FUSE_MUL_ADD, ...). +enum ggml_rms_norm_fuse_op { + GGML_RMS_NORM_FUSE_OP_NONE, + GGML_RMS_NORM_FUSE_OP_MUL, +}; + +template static void ggml_compute_forward_rms_norm_f32( const ggml_compute_params * params, - ggml_tensor * dst) { + ggml_tensor * dst_rms_norm, + ggml_tensor * dst_fused = nullptr) { - const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src0 = dst_rms_norm->src[0]; + const ggml_tensor * src1 = nullptr; + ggml_tensor * dst = dst_rms_norm; + + if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_OP_MUL) { + src1 = (dst_fused->src[0] == dst_rms_norm) ? dst_fused->src[1] : dst_fused->src[0]; + dst = dst_fused; + } GGML_ASSERT(ggml_are_same_shape(src0, dst)); @@ -3726,11 +4088,10 @@ static void ggml_compute_forward_rms_norm_f32( const int ith = params->ith; const int nth = params->nth; - GGML_TENSOR_UNARY_OP_LOCALS + GGML_TENSOR_BINARY_OP_LOCALS float eps; - memcpy(&eps, dst->op_params, sizeof(float)); - + memcpy(&eps, dst_rms_norm->op_params, sizeof(float)); GGML_ASSERT(eps >= 0.0f); // TODO: optimize @@ -3740,25 +4101,32 @@ static void ggml_compute_forward_rms_norm_f32( const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03); ggml_float sum = 0.0; + // worth switching to explicit SIMD? for (int64_t i00 = 0; i00 < ne00; i00++) { sum += (ggml_float)(x[i00] * x[i00]); } - const float mean = sum/ne00; - - float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); - - memcpy(y, x, ne00 * sizeof(float)); - // for (int i00 = 0; i00 < ne00; i00++) { - // y[i00] = x[i00]; - // } - + const float mean = sum/ne00; const float scale = 1.0f/sqrtf(mean + eps); // if you hit this, likely you got an inf somewhere earlier assert(scale > 0.0f); - ggml_vec_scale_f32(ne00, y, scale); + float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); + + if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_OP_MUL) { + const int64_t i11 = i01 % ne11; + const int64_t i12 = i02 % ne12; + const int64_t i13 = i03 % ne13; + const float * w = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13); + + for (int64_t i00 = 0; i00 < ne00; i00++) { + y[i00] = x[i00] * scale * w[i00]; + } + } else { + memcpy(y, x, ne00 * sizeof(float)); + ggml_vec_scale_f32(ne00, y, scale); + } } } } @@ -3773,7 +4141,31 @@ void ggml_compute_forward_rms_norm( switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_rms_norm_f32(params, dst); + ggml_compute_forward_rms_norm_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + +// Fused RMS_NORM + MUL: computes dst = rms_norm(src0) * src1 in a single pass. +// This avoids materializing the intermediate rms_norm result in memory. +void ggml_compute_forward_rms_norm_mul_fused( + const ggml_compute_params * params, + ggml_tensor * dst_rms_norm, + ggml_tensor * dst_mul) { + + GGML_ASSERT(dst_mul != nullptr); + GGML_ASSERT(dst_mul->src[0] == dst_rms_norm || dst_mul->src[1] == dst_rms_norm); + + const ggml_tensor * src0 = dst_rms_norm->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_rms_norm_f32(params, dst_rms_norm, dst_mul); } break; default: { @@ -3928,12 +4320,12 @@ static void ggml_compute_forward_rms_norm_back_f32( // dx := scale(dx, rrms) float * dx = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); - // dx[i00] = (x*(-sum_xdz/sum_eps) + dz) / sqrtf(mean_eps) - ggml_vec_cpy_f32 (ne00, dx, x); - // ggml_vec_scale_f32(ne00, dx, -mean_xdz/mean_eps); - ggml_vec_scale_f32(ne00, dx, (float)(-sum_xdz)/sum_eps); - ggml_vec_acc_f32 (ne00, dx, dz); - ggml_vec_scale_f32(ne00, dx, rrms); + // dx[i00] = (dz + x*(-sum_xdz/sum_eps)) * rrms + // note: https://github.com/ggml-org/ggml/issues/1491 + const float scale_x = (float) (-sum_xdz) / sum_eps; + for (int64_t i00 = 0; i00 < ne00; i00++) { + dx[i00] = (dz[i00] + x[i00] * scale_x) * rrms; + } } } } @@ -4062,8 +4454,6 @@ static void ggml_compute_forward_l2_norm_f32( GGML_ASSERT(ggml_are_same_shape(src0, dst)); - GGML_ASSERT(src0->nb[0] == sizeof(float)); - const int ith = params->ith; const int nth = params->nth; @@ -4078,20 +4468,27 @@ static void ggml_compute_forward_l2_norm_f32( for (int64_t i03 = 0; i03 < ne03; i03++) { for (int64_t i02 = 0; i02 < ne02; i02++) { for (int64_t i01 = ith; i01 < ne01; i01 += nth) { - const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03); + const char * x = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; ggml_float sum = 0.0; for (int64_t i00 = 0; i00 < ne00; i00++) { - sum += (ggml_float)(x[i00] * x[i00]); + const float xi = *(const float *) (x + i00*nb00); + sum += (ggml_float)(xi * xi); } - float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); - - memcpy(y, x, ne00 * sizeof(float)); - const float scale = 1.0f/fmaxf(sqrtf(sum), eps); - ggml_vec_scale_f32(ne00, y, scale); + char * y = (char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3; + + if (nb00 == sizeof(float) && nb0 == sizeof(float)) { + memcpy(y, x, ne00 * sizeof(float)); + ggml_vec_scale_f32(ne00, (float *) y, scale); + } else { + for (int64_t i00 = 0; i00 < ne00; i00++) { + const float xi = *(const float *) (x + i00*nb00); + *(float *) (y + i00*nb0) = xi * scale; + } + } } } } @@ -4327,6 +4724,70 @@ static void ggml_compute_forward_out_prod_q_f32( } } +static void ggml_compute_forward_out_prod_f16_f32( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + GGML_TENSOR_BINARY_OP_LOCALS; + + const int ith = params->ith; + const int nth = params->nth; + + GGML_ASSERT(src0->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_ASSERT(ne02 == ne12); + GGML_ASSERT(ne03 == ne13); + GGML_ASSERT(ne2 == ne12); + GGML_ASSERT(ne3 == ne13); + + GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); + GGML_ASSERT(nb0 == sizeof(float)); + + GGML_ASSERT(ne0 == ne00); + GGML_ASSERT(ne1 == ne10); + GGML_ASSERT(ne2 == ne02); + GGML_ASSERT(ne3 == ne03); + + if (ith == 0) { + ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0); + } + ggml_barrier(params->threadpool); + + const int64_t nr = ne1*ne2*ne3; + const int64_t dr = (nr + nth - 1)/nth; + const int64_t ir0 = dr*ith; + const int64_t ir1 = MIN(ir0 + dr, nr); + + float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith; + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t i3 = ir/(ne2*ne1); + const int64_t i2 = (ir - i3*ne2*ne1)/ne1; + const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1); + + const int64_t i02 = i2; + const int64_t i03 = i3; + + const int64_t i12 = i2; + const int64_t i13 = i3; + + float * d = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)); + + for (int64_t i01 = 0; i01 < ne01; ++i01) { + const int64_t i11 = i01; + ggml_fp16_t * s0 = (ggml_fp16_t *) ((char *) src0->data + (i01*nb01 + i02*nb02 + i03*nb03)); + float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13)); + ggml_fp16_to_fp32_row(s0, wdata, ne0); + ggml_vec_mad_f32(ne0, d, wdata, *s1); + } + } +} + void ggml_compute_forward_out_prod( const ggml_compute_params * params, ggml_tensor * dst) { @@ -4335,6 +4796,7 @@ void ggml_compute_forward_out_prod( switch (src0->type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4363,9 +4825,8 @@ void ggml_compute_forward_out_prod( } break; case GGML_TYPE_F16: { - GGML_ABORT("fatal error"); // todo - // ggml_compute_forward_out_prod_f16_f32(params, dst); - } + ggml_compute_forward_out_prod_f16_f32(params, dst); + } break; case GGML_TYPE_F32: { ggml_compute_forward_out_prod_f32(params, dst); @@ -4431,6 +4892,44 @@ static void ggml_compute_forward_scale_f32( } } +static void ggml_compute_forward_scale_bf16( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * src0 = dst->src[0]; + + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_are_same_shape(src0, dst)); + + float s; // scale factor + float b; // bias + + memcpy(&s, (float *) dst->op_params + 0, sizeof(float)); + memcpy(&b, (float *) dst->op_params + 1, sizeof(float)); + + const int ith = params->ith; + const int nth = params->nth; + + const int nc = src0->ne[0]; + const int nr = ggml_nrows(src0); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int i1 = ir0; i1 < ir1; i1++) { + const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*src0->nb[1]); + ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*dst->nb[1]); + for (int k = 0; k < nc; k++) { + y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b); + } + } +} + void ggml_compute_forward_scale( const ggml_compute_params * params, ggml_tensor * dst) { @@ -4442,6 +4941,10 @@ void ggml_compute_forward_scale( { ggml_compute_forward_scale_f32(params, dst); } break; + case GGML_TYPE_BF16: + { + ggml_compute_forward_scale_bf16(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -4611,6 +5114,7 @@ void ggml_compute_forward_set( case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4835,6 +5339,7 @@ void ggml_compute_forward_get_rows( switch (src0->type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4900,8 +5405,8 @@ void ggml_compute_forward_get_rows( //} } -template -static void ggml_compute_forward_set_rows_f32( +template +static void ggml_compute_forward_set_rows_impl( const ggml_compute_params * params, ggml_tensor * dst) { @@ -4916,7 +5421,7 @@ static void ggml_compute_forward_set_rows_f32( assert(ne0 == nc); assert(ne2 == ne02); assert(ne3 == ne03); - assert(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); assert(ne02 % ne11 == 0); assert(ne03 % ne12 == 0); @@ -4930,6 +5435,8 @@ static void ggml_compute_forward_set_rows_f32( const int64_t ir0 = dr*ith; const int64_t ir1 = std::min(ir0 + dr, nr); + const size_t rs = ggml_row_size(src0->type, nc); + ggml_from_float_t const from_float = ggml_get_type_traits_cpu(dst->type)->from_float; for (int64_t i03 = 0; i03 < ne03; ++i03) { @@ -4943,9 +5450,27 @@ static void ggml_compute_forward_set_rows_f32( GGML_ASSERT(i1 >= 0 && i1 < ne1); - from_float( - (const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), - ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); + if constexpr (std::is_same_v) { + from_float( + (const float *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), + ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); + } else if constexpr (std::is_same_v) { + if (dst->type == GGML_TYPE_F16) { + memcpy( + ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), + ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), + rs); + } else { + float * wdata = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith; + ggml_fp16_to_fp32_row( + (const ggml_fp16_t *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03), + wdata, nc); + from_float(wdata, + ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc); + } + } else { + GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type)); + } } } } @@ -4962,9 +5487,19 @@ void ggml_compute_forward_set_rows( case GGML_TYPE_F32: { if (src1->type == GGML_TYPE_I64) { - ggml_compute_forward_set_rows_f32(params, dst); + ggml_compute_forward_set_rows_impl(params, dst); + } else if (src1->type == GGML_TYPE_I32) { + ggml_compute_forward_set_rows_impl(params, dst); + } else { + GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type)); + } + } break; + case GGML_TYPE_F16: + { + if (src1->type == GGML_TYPE_I64) { + ggml_compute_forward_set_rows_impl(params, dst); } else if (src1->type == GGML_TYPE_I32) { - ggml_compute_forward_set_rows_f32(params, dst); + ggml_compute_forward_set_rows_impl(params, dst); } else { GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type)); } @@ -5004,6 +5539,8 @@ static void ggml_compute_forward_get_rows_back_f32_f16( for (int i = 0; i < nr; ++i) { const int r = ((int32_t *) src1->data)[i]; + GGML_ASSERT(r >= 0 && r < dst->ne[1]); + for (int j = 0; j < nc; ++j) { ggml_fp16_t v = ((ggml_fp16_t *) ((char *) src0->data + i*src0->nb[1]))[j]; ((float *) ((char *) dst->data + r*dst->nb[1]))[j] += GGML_CPU_FP16_TO_FP32(v); @@ -5037,6 +5574,8 @@ static void ggml_compute_forward_get_rows_back_f32( for (int i = 0; i < nr; ++i) { const int r = ((int32_t *) src1->data)[i]; + GGML_ASSERT(r >= 0 && r < dst->ne[1]); + ggml_vec_add_f32(nc, (float *) ((char *) dst->data + r*dst->nb[1]), (float *) ((char *) dst->data + r*dst->nb[1]), @@ -5436,11 +5975,10 @@ static void ggml_compute_forward_soft_max_ext_back_f32( // linear runtime, no additional memory float dot_y_dy = 0; - ggml_vec_dot_f32 (nc, &dot_y_dy, 0, y, 0, dy, 0, 1); - ggml_vec_cpy_f32 (nc, dx, dy); - ggml_vec_acc1_f32 (nc, dx, -dot_y_dy); - ggml_vec_mul_f32 (nc, dx, dx, y); - ggml_vec_scale_f32(nc, dx, scale); + ggml_vec_dot_f32(nc, &dot_y_dy, 0, y, 0, dy, 0, 1); + for (int i = 0; i < nc; i++) { + dx[i] = scale * (dy[i] - dot_y_dy) * y[i]; + } #ifndef NDEBUG for (int i = 0; i < nc; ++i) { @@ -5488,18 +6026,18 @@ static void ggml_compute_forward_clamp_f32( const int n = ggml_nrows(src0); const int nc = src0->ne[0]; - const size_t nb00 = src0->nb[0]; - const size_t nb01 = src0->nb[1]; - - const size_t nb0 = dst->nb[0]; - const size_t nb1 = dst->nb[1]; + GGML_TENSOR_UNARY_OP_LOCALS GGML_ASSERT( nb0 == sizeof(float)); GGML_ASSERT(nb00 == sizeof(float)); for (int j = ith; j < n; j += nth) { - float * dst_ptr = (float *) ((char *) dst->data + j*nb1); - float * src0_ptr = (float *) ((char *) src0->data + j*nb01); + const int64_t i1 = j % ne01; + const int64_t i2 = (j / ne01) % ne02; + const int64_t i3 = j / (ne01*ne02); + + float * dst_ptr = (float *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); + float * src0_ptr = (float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); for (int i = 0; i < nc; i++) { dst_ptr[i] = MAX(MIN(src0_ptr[i], max), min); @@ -5524,18 +6062,18 @@ static void ggml_compute_forward_clamp_f16( const int n = ggml_nrows(src0); const int nc = src0->ne[0]; - const size_t nb00 = src0->nb[0]; - const size_t nb01 = src0->nb[1]; - - const size_t nb0 = dst->nb[0]; - const size_t nb1 = dst->nb[1]; + GGML_TENSOR_UNARY_OP_LOCALS GGML_ASSERT( nb0 == sizeof(ggml_fp16_t)); GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); for (int j = ith; j < n; j += nth) { - ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + j*nb1); - ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + j*nb01); + const int64_t i1 = j % ne01; + const int64_t i2 = (j / ne01) % ne02; + const int64_t i3 = j / (ne01*ne02); + + ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3); + ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03); for (int i = 0; i < nc; i++) { float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i]); @@ -5561,6 +6099,7 @@ void ggml_compute_forward_clamp( } break; case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -5760,6 +6299,8 @@ static void ggml_compute_forward_rope_flt( memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float)); memcpy(§ions, (int32_t *) dst->op_params + 11, sizeof(int)*4); + const int n_offs = ((int32_t *) dst->op_params)[15]; + GGML_TENSOR_UNARY_OP_LOCALS //printf("ne0: %d, ne1: %d, ne2: %d, ne3: %d\n", ne0, ne1, ne2, ne3); @@ -5776,6 +6317,10 @@ static void ggml_compute_forward_rope_flt( GGML_ASSERT(n_dims <= ne0); GGML_ASSERT(n_dims % 2 == 0); + GGML_ASSERT(n_offs >= 0); + GGML_ASSERT(n_offs % 2 == 0); + GGML_ASSERT(n_offs + n_dims <= ne0); + // rows per thread const int dr = (nr + nth - 1)/nth; @@ -5801,6 +6346,7 @@ static void ggml_compute_forward_rope_flt( if (is_vision) { GGML_ASSERT(n_dims == ne0/2); + GGML_ASSERT(n_offs == 0); } const float * freq_factors = NULL; @@ -5849,12 +6395,12 @@ static void ggml_compute_forward_rope_flt( switch (mode) { case GGML_ROPE_TYPE_NORMAL: - rotate_pairs(n_dims, 1, cache, src, dst_data, 1); + rotate_pairs(n_dims, 1, cache, src + n_offs, dst_data + n_offs, 1); break; case GGML_ROPE_TYPE_NEOX: case GGML_ROPE_TYPE_MROPE: case GGML_ROPE_TYPE_IMROPE: - rotate_pairs(n_dims, n_dims/2, cache, src, dst_data); + rotate_pairs(n_dims, n_dims/2, cache, src + n_offs, dst_data + n_offs); break; case GGML_ROPE_TYPE_VISION: rotate_pairs(ne0, n_dims, cache, src, dst_data); @@ -5865,7 +6411,11 @@ static void ggml_compute_forward_rope_flt( if (!is_vision) { // fill the remain channels with data from src tensor - for (int64_t i0 = n_dims; i0 < ne0; i0 += 2) { + for (int64_t i0 = 0; i0 < ne0; i0 += 2) { + if (i0 == n_offs) { + i0 += n_dims - 2; // skip the rotated channels + continue; + } const T * const src = (T *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); T * dst_data = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); @@ -6567,7 +7117,6 @@ static void ggml_compute_forward_im2col_f16( const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F16); @@ -6598,7 +7147,6 @@ static void ggml_compute_forward_im2col_f16( int ofs0 = is_2D ? nb13 : nb12; int ofs1 = is_2D ? nb12 : nb11; - GGML_ASSERT(nb00 == sizeof(ggml_fp16_t)); GGML_ASSERT(nb10 == ggml_type_size(src1->type)); // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW] @@ -6671,7 +7219,7 @@ void ggml_compute_forward_im2col_back_f32( const ggml_tensor * src1 = dst->src[1]; // convolution kernel GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); GGML_ASSERT( dst->type == GGML_TYPE_F32); GGML_TENSOR_BINARY_OP_LOCALS; @@ -6768,7 +7316,6 @@ static void ggml_compute_forward_im2col_3d_f16( const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F16); @@ -7006,6 +7553,78 @@ static inline int64_t ggml_wrap_around(int64_t coord, int64_t size) { return (coord + size) % size; // adding size avoids negative number weirdness } +// ggml_compute_forward_col2im_1d +// +// Scatter-add columns [K*OC, T_in] -> signal [T_out, OC] +// where T_out = (T_in - 1)*s + K - 2*p. Gather approach: each output reads ceil(K/s) inputs. +// Parallelized over the time axis so the split stays balanced whatever OC is. +// Supports F32, F16, BF16 input/output (same type), F32 accumulator. + +template +static void ggml_compute_forward_col2im_1d_impl( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * src = dst->src[0]; // [K*OC, T_in] + + GGML_ASSERT(ggml_is_contiguous(src)); + GGML_ASSERT(ggml_is_contiguous(dst)); + + const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; + const int32_t OC = ((const int32_t *)(dst->op_params))[1]; + const int32_t p0 = ((const int32_t *)(dst->op_params))[2]; + + const int64_t K_OC = src->ne[0]; + const int64_t T_in = src->ne[1]; + const int64_t K = K_OC / OC; + const int64_t T_out = dst->ne[0]; + + const elem_t * col_data = (const elem_t *) src->data; + elem_t * dst_data = (elem_t *) dst->data; + + const int ith = params->ith; + const int nth = params->nth; + + // Parallelize over the time axis: the split stays balanced whatever OC is, + // down to OC = 1 for mono audio, and threads read disjoint column bands + const int64_t dr = (T_out + nth - 1) / nth; + const int64_t it0 = dr * ith; + const int64_t it1 = it0 + dr < T_out ? it0 + dr : T_out; + + for (int64_t oc = 0; oc < OC; oc++) { + for (int64_t t_out = it0; t_out < it1; t_out++) { + const int64_t t_abs = t_out + p0; // absolute position in uncropped signal + // Gather: find all (t_in, k) where t_in * s + k == t_abs, 0 <= k < K + int64_t t_in_min = (t_abs - K + 1 + s0 - 1) / s0; // ceil((t_abs-K+1)/s) + if (t_in_min < 0) t_in_min = 0; + int64_t t_in_max = t_abs / s0; + if (t_in_max >= T_in) t_in_max = T_in - 1; + + float sum = 0.0f; + for (int64_t t_in = t_in_min; t_in <= t_in_max; t_in++) { + int64_t k = t_abs - t_in * s0; + if (k >= 0 && k < K) { + // col layout: [K*OC, T_in], element (oc*K+k, t_in) + sum += type_conversion_table::to_f32(col_data[(oc * K + k) + t_in * K_OC]); + } + } + // dst layout: [T_out, OC], element (t_out, oc) + dst_data[t_out + oc * T_out] = type_conversion_table::from_f32(sum); + } + } +} + +void ggml_compute_forward_col2im_1d( + const ggml_compute_params * params, + ggml_tensor * dst) { + switch (dst->src[0]->type) { + case GGML_TYPE_F32: ggml_compute_forward_col2im_1d_impl (params, dst); break; + case GGML_TYPE_F16: ggml_compute_forward_col2im_1d_impl(params, dst); break; + case GGML_TYPE_BF16: ggml_compute_forward_col2im_1d_impl(params, dst); break; + default: GGML_ABORT("col2im_1d: unsupported type %d", dst->src[0]->type); + } +} + // ggml_compute_forward_conv_2d @@ -7328,18 +7947,21 @@ static void ggml_compute_forward_conv_transpose_2d_impl( } } - // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh) + // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches { kernel_t * const wdata = (kernel_t *) params->wdata + nk; - for (int i12 = 0; i12 < ne12; i12++) { - for (int i11 = 0; i11 < ne11; i11++) { - const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11); - kernel_t * dst_data = wdata + i11*ne10*ne12; - for (int i10 = 0; i10 < ne10; i10++) { - if constexpr (std::is_same_v) { - dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]); - } else { - dst_data[i10*ne12 + i12] = src[i10]; + for (int i13 = 0; i13 < ne13; i13++) { + kernel_t * const wdata_b = wdata + i13*ne10*ne11*ne12; + for (int i12 = 0; i12 < ne12; i12++) { + for (int i11 = 0; i11 < ne11; i11++) { + const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11); + kernel_t * dst_data = wdata_b + i11*ne10*ne12; + for (int i10 = 0; i10 < ne10; i10++) { + if constexpr (std::is_same_v) { + dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]); + } else { + dst_data[i10*ne12 + i12] = src[i10]; + } } } } @@ -7366,24 +7988,27 @@ static void ggml_compute_forward_conv_transpose_2d_impl( kernel_t * const wdata_src = wdata + nk; for (int i2 = ip0; i2 < ip1; i2++) { // Cout - float * dst_data = (float *)((char *) dst->data + i2*nb2); kernel_t * wdata_kernel = wdata + i2*ne01*ne00*ne03; - for (int i11 = 0; i11 < ne11; i11++) { - for (int i10 = 0; i10 < ne10; i10++) { - const int i1n = i11*ne10*ne12 + i10*ne12; - for (int i01 = 0; i01 < ne01; i01++) { - for (int i00 = 0; i00 < ne00; i00++) { - float v = 0; - if constexpr (std::is_same_v) { - ggml_vec_dot_f16(ne03, &v, 0, - wdata_src + i1n, 0, - wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); - } else { - ggml_vec_dot_f32(ne03, &v, 0, - wdata_src + i1n, 0, - wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); + for (int i3 = 0; i3 < ne3; i3++) { // batch + float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2); + kernel_t * wdata_src_b = wdata_src + i3*ne10*ne11*ne12; + for (int i11 = 0; i11 < ne11; i11++) { + for (int i10 = 0; i10 < ne10; i10++) { + const int i1n = i11*ne10*ne12 + i10*ne12; + for (int i01 = 0; i01 < ne01; i01++) { + for (int i00 = 0; i00 < ne00; i00++) { + float v = 0; + if constexpr (std::is_same_v) { + ggml_vec_dot_f16(ne03, &v, 0, + wdata_src_b + i1n, 0, + wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); + } else { + ggml_vec_dot_f32(ne03, &v, 0, + wdata_src_b + i1n, 0, + wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); + } + dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v; } - dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v; } } } @@ -7432,6 +8057,13 @@ struct ggml_conv_2d_dw_params { int dilation_y; }; +static inline float ggml_conv_2d_dw_knl_f32(const char * data, int64_t i, ggml_type type) { + if (type == GGML_TYPE_F16) { + return GGML_FP16_TO_FP32(((const ggml_fp16_t *)data)[i]); + } + return ((const float *)data)[i]; +} + static void ggml_compute_forward_conv_2d_dw_cwhn( const ggml_compute_params * params, const ggml_tensor * src, @@ -7440,7 +8072,8 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( const ggml_conv_2d_dw_params & p) { const int64_t c = p.channels; - const float * knl_data = (const float *)kernel->data; + const char * knl_data = (const char *)kernel->data; + const ggml_type knl_type = kernel->type; const int64_t rows_total = p.dst_h * p.batch; const int64_t rows_per_thread = (rows_total + params->nth - 1) / params->nth; @@ -7448,13 +8081,16 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( const int64_t row_end = MIN(row_start + rows_per_thread, rows_total); #ifdef GGML_SIMD + int64_t c_pkg_end = 0; + int64_t pkg_size = GGML_F32_EPR; + if (knl_type == GGML_TYPE_F32) { #if defined(__ARM_FEATURE_SVE) - const int64_t pkg_size = svcntw(); + pkg_size = svcntw(); #else - const int64_t pkg_size = GGML_F32_EPR; + pkg_size = GGML_F32_EPR; #endif - const int64_t pkg_count = c / pkg_size; - const int64_t c_pkg_end = pkg_count * pkg_size; + c_pkg_end = (c / pkg_size) * pkg_size; + } #else const int64_t c_pkg_end = 0; #endif @@ -7468,7 +8104,6 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( const int64_t src_x_base = dst_x * p.stride_x - p.pad_x; #ifdef GGML_SIMD - // Vectorized loop for (int64_t c_i = 0; c_i < c_pkg_end; c_i += pkg_size) { GGML_F32_VEC sum = GGML_F32_VEC_ZERO; for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) { @@ -7481,7 +8116,8 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( if (src_x < 0 || src_x >= p.src_w) { continue; } - GGML_F32_VEC k = GGML_F32_VEC_LOAD(knl_data + (knl_y * p.knl_w + knl_x) * c + c_i); + const float * kp = (const float *)knl_data + (knl_y * p.knl_w + knl_x) * c + c_i; + GGML_F32_VEC k = GGML_F32_VEC_LOAD(kp); GGML_F32_VEC s = GGML_F32_VEC_LOAD(src_data + (src_y * p.src_w + src_x) * c + c_i); sum = GGML_F32_VEC_FMA(sum, k, s); } @@ -7489,7 +8125,6 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( GGML_F32_VEC_STORE(dst_data + c_i, sum); } #endif - // Scalar loop for (int64_t c_i = c_pkg_end; c_i < c; ++c_i) { float sum = 0.0f; for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) { @@ -7502,7 +8137,7 @@ static void ggml_compute_forward_conv_2d_dw_cwhn( if (src_x < 0 || src_x >= p.src_w) { continue; } - sum += knl_data[(knl_y * p.knl_w + knl_x) * c + c_i] + sum += ggml_conv_2d_dw_knl_f32(knl_data, (knl_y * p.knl_w + knl_x) * c + c_i, knl_type) * src_data[(src_y * p.src_w + src_x) * c + c_i]; } } @@ -7523,9 +8158,11 @@ static void ggml_compute_forward_conv_2d_dw_whcn( const int64_t per_thread = (n + params->nth - 1) / params->nth; const int64_t start = params->ith * per_thread; const int64_t end = MIN(start + per_thread, n); + const char * knl_base = (const char *)kernel->data; + const ggml_type knl_type = kernel->type; for (int64_t i = start; i < end; ++i) { - const float * knl_data = (const float *)kernel->data + (i % p.channels) * p.knl_w * p.knl_h; + const int64_t knl_offset = (i % p.channels) * p.knl_w * p.knl_h; const float * src_data = (const float *)src->data + i * p.src_w * p.src_h; float * dst_data = (float *)dst->data + i * p.dst_w * p.dst_h; @@ -7543,7 +8180,7 @@ static void ggml_compute_forward_conv_2d_dw_whcn( if (src_x < 0 || src_x >= p.src_w) { continue; } - sum += knl_data[knl_y * p.knl_w + knl_x] + sum += ggml_conv_2d_dw_knl_f32(knl_base, knl_offset + knl_y * p.knl_w + knl_x, knl_type) * src_data[src_y * p.src_w + src_x]; } } @@ -7575,13 +8212,13 @@ void ggml_compute_forward_conv_2d_dw( p.dilation_x = dst->op_params[4]; p.dilation_y = dst->op_params[5]; + GGML_ASSERT(kernel->type == GGML_TYPE_F32 || kernel->type == GGML_TYPE_F16); GGML_ASSERT(kernel->ne[3] == p.channels); GGML_ASSERT(dst->ne[3] == p.batch); if (ggml_is_contiguous(src)) { ggml_compute_forward_conv_2d_dw_whcn(params, src, kernel, dst, p); } else if (ggml_is_contiguous_channels(src)) { - // kernel should also have channels most contiguous in memory GGML_ASSERT(kernel->nb[0] >= kernel->nb[2] && kernel->nb[1] >= kernel->nb[0]); ggml_compute_forward_conv_2d_dw_cwhn(params, src, kernel, dst, p); } else { @@ -8126,41 +8763,48 @@ static void ggml_compute_forward_pad_f32( const int32_t lp3 = ggml_get_op_params_i32(dst, 6); const int32_t rp3 = ggml_get_op_params_i32(dst, 7); - // TODO: optimize + const int64_t nrows = ne1 * ne2 * ne3; + const bool src_row_contig = src0->nb[0] == sizeof(float); + + for (int64_t row = ith; row < nrows; row += nth) { + const int64_t i1 = row % ne1; + const int64_t i2 = (row / ne1) % ne2; + const int64_t i3 = row / (ne1 * ne2); + + float * dst_row = dst_ptr + row * ne0; + + if constexpr (circular_t) { + // circular padding wraps around on a torus + const int64_t src_i1 = ggml_wrap_around(i1 - lp1, ne01); + const int64_t src_i2 = ggml_wrap_around(i2 - lp2, ne02); + const int64_t src_i3 = ggml_wrap_around(i3 - lp3, ne03); - for (int64_t i2 = 0; i2 < ne2; ++i2) { - for (int64_t i1 = ith; i1 < ne1; i1 += nth) { for (int64_t i0 = 0; i0 < ne0; ++i0) { - for (int64_t i3 = 0; i3 < ne3; ++i3) { - // circular means wrap around on a torus, so x and y loop around - if constexpr (circular_t) { - const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0; - const int64_t src_i0 = ggml_wrap_around(i0 - lp0, ne00); - const int64_t src_i1 = ggml_wrap_around(i1 - lp1, ne01); - const int64_t src_i2 = ggml_wrap_around(i2 - lp2, ne02); - const int64_t src_i3 = ggml_wrap_around(i3 - lp3, ne03); - - const int64_t src_idx = - src_i3*nb03 + - src_i2*nb02 + - src_i1*nb01 + - src_i0*nb00; - - const float * src_ptr = (const float *)((char *) src0->data + src_idx); - dst_ptr[dst_idx] = *src_ptr; - } else { - const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0; - if ((i0 >= lp0 && i0 < ne0 - rp0) \ - && (i1 >= lp1 && i1 < ne1 - rp1) \ - && (i2 >= lp2 && i2 < ne2 - rp2) \ - && (i3 >= lp3 && i3 < ne3 - rp3)) { - const int64_t src_idx = (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01 + (i0 - lp0)*nb00; - const float * src_ptr = (const float *)((char *) src0->data + src_idx); - dst_ptr[dst_idx] = *src_ptr; - } else { - dst_ptr[dst_idx] = 0; - } - } + const int64_t src_i0 = ggml_wrap_around(i0 - lp0, ne00); + dst_row[i0] = *(const float *)((const char *) src0->data + src_i3*nb03 + src_i2*nb02 + src_i1*nb01 + src_i0*nb00); + } + } else { + const bool interior = (i1 >= lp1 && i1 < ne1 - rp1) + && (i2 >= lp2 && i2 < ne2 - rp2) + && (i3 >= lp3 && i3 < ne3 - rp3); + if (!interior) { + memset(dst_row, 0, ne0 * sizeof(float)); + continue; + } + + const char * src_row = (const char *) src0->data + (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01; + + if (src_row_contig && lp0 >= 0 && rp0 >= 0) { + memset(dst_row, 0, lp0 * sizeof(float)); + memcpy(dst_row + lp0, src_row, ne00 * sizeof(float)); + memset(dst_row + lp0 + ne00, 0, rp0 * sizeof(float)); + } else { + for (int64_t i0 = 0; i0 < ne0; ++i0) { + if (i0 >= lp0 && i0 < ne0 - rp0) { + dst_row[i0] = *(const float *)(src_row + (i0 - lp0)*nb00); + } else { + dst_row[i0] = 0.0f; + } } } } @@ -8947,6 +9591,11 @@ static void ggml_compute_forward_flash_attn_ext_tiled( simd_gemm(KQ, (const float *)Q_q, K_f32, Q_TILE_SZ, DK, KV_TILE_SZ); ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale); + if (logit_softcap != 0.0f) { + ggml_vec_tanh_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, KQ); + ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, logit_softcap); + } + // Set padded KQ entries to -inf so softmax gives them zero weight if (kv_tile < KV_TILE_SZ) { for (int tq = 0; tq < Q_TILE_SZ; tq++) { @@ -8956,11 +9605,6 @@ static void ggml_compute_forward_flash_attn_ext_tiled( } } - if (logit_softcap != 0.0f) { - ggml_vec_tanh_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, KQ); - ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, logit_softcap); - } - if (mask) { ggml_vec_add_f32(tile_rows * KV_TILE_SZ, KQ, KQ, mask32); } @@ -8997,7 +9641,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled( for (int tk = 0; tk < kv_tile; tk++) { const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3; if (kv_type == GGML_TYPE_F16) { - ggml_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV); + ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV); } else { memcpy(V32 + tk * DV, v_data, DV * sizeof(float)); } @@ -9230,8 +9874,13 @@ static void ggml_compute_forward_flash_attn_ext_f16( kv_is_f32_or_f16 && k->type == v->type && neq1 >= Q_TILE_SZ); -#ifdef GGML_SIMD - use_tiled &= (DV % GGML_F32_EPR == 0); +#if defined(GGML_SIMD) && !defined(__x86_64__) && !defined(_M_X64) +#if defined(__ARM_FEATURE_SVE) + const int64_t f32_epr = svcntw(); +#else + const int64_t f32_epr = GGML_F32_EPR; +#endif + use_tiled &= (DV % f32_epr == 0); #endif int current_chunk = ith; @@ -9695,11 +10344,13 @@ static void ggml_compute_forward_ssm_scan_f32( const int64_t ng = src4->ne[1]; const int64_t nt = src1->ne[2]; // number of tokens per sequence const int64_t ns = src1->ne[3]; // number of sequences in the batch + const int64_t K = ggml_get_op_params_i32(dst, 0); // can't use ggml_nbytes because src1 is not necessarily contiguous const int64_t s_off = ggml_nelements(src1) * ggml_element_size(src1); - GGML_ASSERT(ggml_nelements(src1) + nc*nr*nh*ns == ggml_nelements(dst)); + GGML_ASSERT(K >= 1); + GGML_ASSERT(ggml_nelements(src1) + K*nc*nr*nh*ns == ggml_nelements(dst)); GGML_ASSERT(src0->nb[0] == sizeof(float)); GGML_ASSERT(src1->nb[0] == sizeof(float)); GGML_ASSERT(src2->nb[0] == sizeof(float)); @@ -9708,6 +10359,7 @@ static void ggml_compute_forward_ssm_scan_f32( GGML_ASSERT(src5->nb[0] == sizeof(float)); GGML_ASSERT(src6->nb[0] == sizeof(int32_t)); GGML_ASSERT(nh % ng == 0); + GGML_ASSERT(src3->ne[0] == 1 || K == 1); // heads per thread const int dh = (nh + nth - 1)/nth; @@ -9882,6 +10534,13 @@ static void ggml_compute_forward_ssm_scan_f32( } } } + const int64_t slot = nt - 1 - i2; + if (K > 1 && slot > 0 && slot < K) { + float * s_snapshot = (float *) ((char *) dst->data + s_off + (slot*ns + i3)*(src0->nb[3])); + for (int h = ih0; h < ih1; ++h) { + memcpy((char *) s_snapshot + h*src0->nb[2], (char *) s + h*src0->nb[2], src0->nb[2]); + } + } // use the output as the source when it's not the first token-wise iteration s0 = s; } @@ -10164,6 +10823,10 @@ void ggml_compute_forward_glu( { ggml_compute_forward_geglu_quick(params, dst); } break; + case GGML_GLU_OP_SWIGLU_CLAMP: + { + ggml_compute_forward_swiglu_clamp(params, dst); + } break; default: { GGML_ABORT("fatal error"); @@ -10823,19 +11486,29 @@ static void ggml_compute_forward_gated_delta_net_one_chunk( const bool kda = (neg0 == S_v); - // scratch layout per thread: [delta(S_v)] - const int64_t scratch_per_thread = S_v; + // K (snapshot slot count) is an op param; state holds s0 only [S_v, S_v, H, n_seqs]. + const int64_t K = ggml_get_op_params_i32(dst, 0); + GGML_ASSERT(K >= 1); + // per-seq stride in floats (seq s starts at state + s * seq_stride) + const int64_t state_seq_stride = src_state->nb[3] / sizeof(float); + + const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0); const int ith = params->ith; - float * delta = (float *)params->wdata + ith * scratch_per_thread + CACHE_LINE_SIZE_F32; + float * delta = (float *)params->wdata + ith * per_thread + CACHE_LINE_SIZE_F32; + float * state_work = K > 1 ? (delta + S_v) : nullptr; // output layout: [attn_scores | new_states] - // attn_scores: S_v * H * n_tokens * n_seqs floats - // new_states: S_v * S_v * H * n_seqs floats - const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs; + // attn_scores: S_v * H * n_tokens * n_seqs floats + // new_states: S_v * S_v * H * n_seqs * K floats (K snapshot slots; last min(n_tokens, K)) + const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs; + const int64_t state_size_per_snap = S_v * S_v * H * n_seqs; float * attn_out_base = (float *)dst->data; float * state_out_base = (float *)dst->data + attn_score_elems; + // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. + // When n_tokens < K only slots 0..n_tokens-1 are written; older slots are caller-owned. + const float * state_in_base = (const float *)src_state->data; //const int64_t rq1 = nev1 / neq1; @@ -10855,10 +11528,15 @@ static void ggml_compute_forward_gated_delta_net_one_chunk( const int64_t iq3 = iv3 / rq3; const int64_t ik3 = iv3 / rk3; - float * s_out = state_out_base + (iv3 * H + iv1) * S_v * S_v; + // For K=1, write directly to the single output slot to avoid an extra memcpy at the end. + // For K>1, work in scratch and copy out per-token when the slot is in range. + float * s_out = (K > 1) + ? state_work + : state_out_base + (iv3 * H + iv1) * S_v * S_v; - // copy input state into output buffer and operate in-place - const float * s_in = state_in_base + (iv3 * H + iv1) * S_v * S_v; + // copy input state into the working buffer and operate in-place + // state layout [S_v, S_v, H, n_seqs]: seq iv3 starts at iv3 * state_seq_stride. + const float * s_in = state_in_base + iv3 * state_seq_stride + iv1 * S_v * S_v; memcpy(s_out, s_in, S_v * S_v * sizeof(float)); // attn output pointer for first token of this (head, seq) @@ -10908,6 +11586,15 @@ static void ggml_compute_forward_gated_delta_net_one_chunk( } attn_data += S_v * H; // advance to next token + + if (K > 1) { + const int64_t target_slot = n_tokens - 1 - t; + if (target_slot >= 0 && target_slot < K) { + float * curr_state_o = state_out_base + target_slot * state_size_per_snap + + (iv3 * H + iv1) * S_v * S_v; + memcpy(curr_state_o, s_out, S_v * S_v * sizeof(float)); + } + } } } } @@ -10971,6 +11658,405 @@ void ggml_compute_forward_gated_delta_net( } } + +// ggml_compute_forward_dsv4_hc_comb + +static void ggml_dsv4_hc_comb_norm_cols(float * comb, float eps) { + constexpr int64_t hc = 4; + + for (int64_t idst = 0; idst < hc; ++idst) { + float sum = eps; + for (int64_t isrc = 0; isrc < hc; ++isrc) { + sum += comb[idst + hc*isrc]; + } + + const float inv_sum = 1.0f / sum; + for (int64_t isrc = 0; isrc < hc; ++isrc) { + comb[idst + hc*isrc] *= inv_sum; + } + } +} + +static void ggml_dsv4_hc_comb_norm_rows(float * comb, float eps) { + constexpr int64_t hc = 4; + + for (int64_t isrc = 0; isrc < hc; ++isrc) { + float sum = eps; + for (int64_t idst = 0; idst < hc; ++idst) { + sum += comb[idst + hc*isrc]; + } + + const float inv_sum = 1.0f / sum; + for (int64_t idst = 0; idst < hc; ++idst) { + comb[idst + hc*isrc] *= inv_sum; + } + } +} + +static void ggml_compute_forward_dsv4_hc_comb_f32( + const ggml_compute_params * params, + ggml_tensor * dst) { + const ggml_tensor * mixes = dst->src[0]; + const ggml_tensor * scale = dst->src[1]; + const ggml_tensor * base = dst->src[2]; + + GGML_ASSERT(mixes->type == GGML_TYPE_F32); + GGML_ASSERT(scale->type == GGML_TYPE_F32); + GGML_ASSERT(base->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + constexpr int64_t hc = 4; + constexpr int64_t comb_offset = 2*hc; + constexpr int64_t hc_mix_dim = (2 + hc)*hc; + + const int64_t n_tokens = mixes->ne[1]; + + GGML_ASSERT(mixes->ne[0] == hc_mix_dim); + GGML_ASSERT(dst->ne[0] == hc); + GGML_ASSERT(dst->ne[1] == hc); + GGML_ASSERT(dst->ne[2] == n_tokens); + GGML_ASSERT(scale->ne[0] >= 3); + GGML_ASSERT(base->ne[0] == hc_mix_dim); + + GGML_TENSOR_LOCALS(size_t, nbm, mixes, nb); + GGML_TENSOR_LOCALS(size_t, nbs, scale, nb); + GGML_TENSOR_LOCALS(size_t, nbb, base, nb); + GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const float eps = ggml_get_op_params_f32(dst, 0); + const int32_t n_iter = ggml_get_op_params_i32(dst, 1); + GGML_ASSERT(n_iter > 0); + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t dr = (n_tokens + nth - 1) / nth; + const int64_t it0 = dr * ith; + const int64_t it1 = MIN(it0 + dr, n_tokens); + + const float scale_comb = *(const float *) ((const char *) scale->data + 2*nbs0); + + for (int64_t it = it0; it < it1; ++it) { + float comb[hc*hc]; + + for (int64_t isrc = 0; isrc < hc; ++isrc) { + float max = -INFINITY; + for (int64_t idst = 0; idst < hc; ++idst) { + const int64_t idx = idst + hc*isrc; + const float xv = *(const float *) ((const char *) mixes->data + (comb_offset + idx)*nbm0 + it*nbm1); + const float bv = *(const float *) ((const char *) base->data + (comb_offset + idx)*nbb0); + const float v = xv * scale_comb + bv; + comb[idx] = v; + max = MAX(max, v); + } + + float sum = 0.0f; + for (int64_t idst = 0; idst < hc; ++idst) { + const int64_t idx = idst + hc*isrc; + const float v = expf(comb[idx] - max); + comb[idx] = v; + sum += v; + } + + const float inv_sum = 1.0f / sum; + for (int64_t idst = 0; idst < hc; ++idst) { + const int64_t idx = idst + hc*isrc; + comb[idx] = comb[idx] * inv_sum + eps; + } + } + + ggml_dsv4_hc_comb_norm_cols(comb, eps); + for (int32_t i = 1; i < n_iter; ++i) { + ggml_dsv4_hc_comb_norm_rows(comb, eps); + ggml_dsv4_hc_comb_norm_cols(comb, eps); + } + + for (int64_t isrc = 0; isrc < hc; ++isrc) { + for (int64_t idst = 0; idst < hc; ++idst) { + const int64_t idx = idst + hc*isrc; + *(float *) ((char *) dst->data + idst*nbd0 + isrc*nbd1 + it*nbd2) = comb[idx]; + } + } + } +} + +void ggml_compute_forward_dsv4_hc_comb( + const ggml_compute_params * params, + ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_dsv4_hc_comb_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + +// ggml_compute_forward_dsv4_hc_pre + +static void ggml_compute_forward_dsv4_hc_pre_f32( + const ggml_compute_params * params, + ggml_tensor * dst) { + const ggml_tensor * x = dst->src[0]; + const ggml_tensor * weights = dst->src[1]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(weights->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + const int64_t n_embd = x->ne[0]; + const int64_t hc = x->ne[1]; + const int64_t n_tokens = x->ne[2]; + + const float scale = ggml_get_op_params_f32(dst, 0); + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + + GGML_ASSERT(dst->ne[0] == n_embd); + GGML_ASSERT(dst->ne[1] == n_tokens); + if (gated) { + GGML_ASSERT(weights->ne[0] == n_embd); + GGML_ASSERT(weights->ne[1] == hc); + GGML_ASSERT(weights->ne[2] == n_tokens); + } else { + GGML_ASSERT(weights->ne[0] == hc); + GGML_ASSERT(weights->ne[1] == n_tokens); + } + + GGML_TENSOR_LOCALS(size_t, nbx, x, nb); + GGML_TENSOR_LOCALS(size_t, nbw, weights, nb); + GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t nr = n_embd * n_tokens; + const int64_t dr = (nr + nth - 1) / nth; + const int64_t ir0 = dr * ith; + const int64_t ir1 = MIN(ir0 + dr, nr); + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t i0 = ir % n_embd; + const int64_t it = ir / n_embd; + + float sum = 0.0f; + for (int64_t ih = 0; ih < hc; ++ih) { + const float xv = *(const float *) ((const char *) x->data + i0*nbx0 + ih*nbx1 + it*nbx2); + float wv; + if (gated) { + const float gv = *(const float *) ((const char *) weights->data + i0*nbw0 + ih*nbw1 + it*nbw2); + wv = 1.0f / (1.0f + expf(-gv)); + } else { + wv = *(const float *) ((const char *) weights->data + ih*nbw0 + it*nbw1); + } + sum += xv * wv; + } + + *(float *) ((char *) dst->data + i0*nbd0 + it*nbd1) = scale * sum; + } +} + +void ggml_compute_forward_dsv4_hc_pre( + const ggml_compute_params * params, + ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_dsv4_hc_pre_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + +// ggml_compute_forward_dsv4_hc_post + +static void ggml_compute_forward_dsv4_hc_post_f32( + const ggml_compute_params * params, + ggml_tensor * dst) { + const ggml_tensor * x = dst->src[0]; + const ggml_tensor * residual = dst->src[1]; + const ggml_tensor * post = dst->src[2]; + const ggml_tensor * comb = dst->src[3]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(residual->type == GGML_TYPE_F32); + GGML_ASSERT(post->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + const int64_t n_embd = x->ne[0]; + const int64_t n_tokens = x->ne[1]; + const int64_t hc = residual->ne[1]; + + GGML_ASSERT(dst->ne[0] == n_embd); + GGML_ASSERT(dst->ne[1] == hc); + GGML_ASSERT(dst->ne[2] == n_tokens); + GGML_ASSERT(residual->ne[0] == n_embd); + GGML_ASSERT(residual->ne[2] == n_tokens); + GGML_ASSERT(post->ne[0] == hc); + GGML_ASSERT(post->ne[1] == n_tokens); + + // comb == NULL: identity mixing, each stream keeps its own residual + size_t nbc0 = 0; + size_t nbc1 = 0; + size_t nbc2 = 0; + if (comb) { + GGML_ASSERT(comb->type == GGML_TYPE_F32); + GGML_ASSERT(comb->ne[0] == hc); + GGML_ASSERT(comb->ne[1] == hc); + GGML_ASSERT(comb->ne[2] == n_tokens); + nbc0 = comb->nb[0]; + nbc1 = comb->nb[1]; + nbc2 = comb->nb[2]; + } + + GGML_TENSOR_LOCALS(size_t, nbx, x, nb); + GGML_TENSOR_LOCALS(size_t, nbr, residual, nb); + GGML_TENSOR_LOCALS(size_t, nbp, post, nb); + GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t nr = n_embd * hc * n_tokens; + const int64_t dr = (nr + nth - 1) / nth; + const int64_t ir0 = dr * ith; + const int64_t ir1 = MIN(ir0 + dr, nr); + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t i0 = ir % n_embd; + const int64_t idst = (ir / n_embd) % hc; + const int64_t it = ir / (n_embd * hc); + + const float xv = *(const float *) ((const char *) x->data + i0*nbx0 + it*nbx1); + const float pv = *(const float *) ((const char *) post->data + idst*nbp0 + it*nbp1); + + float sum = xv * pv; + if (comb) { + for (int64_t isrc = 0; isrc < hc; ++isrc) { + const float rv = *(const float *) ((const char *) residual->data + i0*nbr0 + isrc*nbr1 + it*nbr2); + const float cv = *(const float *) ((const char *) comb->data + idst*nbc0 + isrc*nbc1 + it*nbc2); + sum += rv * cv; + } + } else { + sum += *(const float *) ((const char *) residual->data + i0*nbr0 + idst*nbr1 + it*nbr2); + } + + *(float *) ((char *) dst->data + i0*nbd0 + idst*nbd1 + it*nbd2) = sum; + } +} + +void ggml_compute_forward_dsv4_hc_post( + const ggml_compute_params * params, + ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_dsv4_hc_post_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + +// ggml_compute_forward_top_k_mask + +void ggml_compute_forward_top_k_mask( + const ggml_compute_params * params, + ggml_tensor * dst) { + const ggml_tensor * mask = dst->src[0]; + const ggml_tensor * top_k = dst->src[1]; + const ggml_tensor * score = dst->src[2]; + const ggml_tensor * blocks = dst->src[3]; + const ggml_tensor * extra = dst->src[4]; + + GGML_ASSERT(dst->type == mask->type); + GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type)); + + const int64_t n_kv = mask->ne[0]; + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t n_rows = ggml_nrows(dst); + const int64_t n_top = top_k->ne[0]; + const int64_t n_blk = blocks ? blocks->ne[0] : 1; + const int64_t n_pick = blocks ? blocks->ne[1] : score ? score->ne[0] : n_kv; + const int64_t n_extra = extra ? extra->ne[0] : 0; + + const size_t ts = ggml_type_size(dst->type); + + const int ith = params->ith; + const int nth = params->nth; + + for (int64_t ir = ith; ir < n_rows; ir += nth) { + const int64_t i1 = ir % ne1; + const int64_t i2 = (ir / ne1) % ne2; + const int64_t i3 = ir / (ne1*ne2); + + const char * mask_row = (const char *) mask->data + + (i1 % mask->ne[1])*mask->nb[1] + (i2 % mask->ne[2])*mask->nb[2] + (i3 % mask->ne[3])*mask->nb[3]; + char * dst_row = (char *) dst->data + i1*dst->nb[1] + i2*dst->nb[2] + i3*dst->nb[3]; + + if (dst->type == GGML_TYPE_F16) { + const ggml_fp16_t ninf = GGML_CPU_FP32_TO_FP16(-INFINITY); + for (int64_t c = 0; c < n_kv; ++c) { + ((ggml_fp16_t *) dst_row)[c] = ninf; + } + } else { + for (int64_t c = 0; c < n_kv; ++c) { + ((float *) dst_row)[c] = -INFINITY; + } + } + + auto keep = [&](int32_t c) { + if (c >= 0 && c < n_kv) { + memcpy(dst_row + c*ts, mask_row + c*ts, ts); + } + }; + + const char * top_k_row = (const char *) top_k->data + i1*top_k->nb[1] + i2*top_k->nb[2] + i3*top_k->nb[3]; + + for (int64_t j = 0; j < n_top; ++j) { + const int32_t p = *(const int32_t *) (top_k_row + j*top_k->nb[0]); + if (p < 0 || p >= n_pick) { + continue; + } + if (score) { + const float s = *(const float *) ((const char *) score->data + + p*score->nb[0] + i1*score->nb[1] + i2*score->nb[2] + i3*score->nb[3]); + if (!(s > -INFINITY)) { + continue; + } + } + if (!blocks) { + keep(p); + continue; + } + for (int64_t k = 0; k < n_blk; ++k) { + keep(*(const int32_t *) ((const char *) blocks->data + k*blocks->nb[0] + p*blocks->nb[1])); + } + } + + if (extra) { + const char * extra_row = (const char *) extra->data + i1*extra->nb[1] + i2*extra->nb[2] + i3*extra->nb[3]; + for (int64_t j = 0; j < n_extra; ++j) { + keep(*(const int32_t *) (extra_row + j*extra->nb[0])); + } + } + } +} + // ggml_compute_forward_rwkv_wkv7 static void ggml_compute_forward_rwkv_wkv7_f32( @@ -11568,3 +12654,193 @@ void ggml_compute_forward_opt_step_sgd(const ggml_compute_params * params, ggml_ } } } + +static inline float ggml_fwht_load(const float value) { + return value; +} + +static inline float ggml_fwht_load(const ggml_fp16_t value) { + return ggml_fp16_to_fp32(value); +} + +template +static void ggml_compute_forward_fwht_impl(const ggml_compute_params * params, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(src1->type == (std::is_same_v ? GGML_TYPE_F32 : GGML_TYPE_F16)); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_TENSOR_BINARY_OP_LOCALS + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t n = ne10; + GGML_ASSERT((n & (n - 1)) == 0); // must be power of 2 + + const int64_t nr = ne11 * ne12 * ne13; + const int64_t rows_per_thread = (nr + nth - 1) / nth; + const int64_t start_row = ith * rows_per_thread; + const int64_t end_row = MIN(start_row + rows_per_thread, nr); + + const float scale = 1.0f / sqrtf((float)n); + +#if defined(GGML_SIMD) + const GGML_F32_VEC v_minus_one = GGML_F32_VEC_SET1(-1.0f); +#endif + + for (int64_t r = start_row; r < end_row; r++) { + const int64_t i13 = r / (ne11 * ne12); + const int64_t i12 = (r - i13 * ne11 * ne12) / ne11; + const int64_t i11 = r - i13 * ne11 * ne12 - i12 * ne11; + + const src_t * src_row = (const src_t *) ((const char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13); + float * dst_row = (float *) ((char *) dst->data + i11 * nb1 + i12 * nb2 + i13 * nb3); + + for (int64_t j = 0; j < n; j++) { + dst_row[j] = ggml_fwht_load(src_row[j]) * scale; + } + + // Scalar passes +#if defined(GGML_SIMD) +#if defined(__ARM_FEATURE_SVE) + const int step = svcntw(); +#else + const int step = GGML_F32_EPR; +#endif +#else + const int step = n; +#endif + for (int64_t len = 1; len < step && len < n; len <<= 1) { + for (int64_t i = 0; i < n; i += 2 * len) { + for (int64_t j = 0; j < len; j++) { + float u = dst_row[i + j]; + float v = dst_row[i + len + j]; + dst_row[i + j] = u + v; + dst_row[i + len + j] = u - v; + } + } + } + + // SIMD passes using GGML_F32_VEC_* macros for multi-architecture support +#if defined(GGML_SIMD) + for (int64_t len = step; len < n; len <<= 1) { + for (int64_t i = 0; i < n; i += 2 * len) { + for (int64_t j = 0; j < len; j += step) { + GGML_F32_VEC u = GGML_F32_VEC_LOAD(dst_row + i + j); + GGML_F32_VEC v = GGML_F32_VEC_LOAD(dst_row + i + len + j); + + GGML_F32_VEC_STORE(dst_row + i + j, GGML_F32_VEC_ADD(u, v)); + GGML_F32_VEC_STORE(dst_row + i + len + j, GGML_F32_VEC_FMA(u, v, v_minus_one)); + } + } + } +#endif + } +} + +void ggml_compute_forward_fwht(const ggml_compute_params * params, ggml_tensor * dst) { + const ggml_tensor * src1 = dst->src[1]; + + switch (src1->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_fwht_impl(params, dst); + } + break; + case GGML_TYPE_F16: + { + ggml_compute_forward_fwht_impl(params, dst); + } + break; + default: + { + GGML_ABORT("fatal error - fwht supports F32 and F16 input"); + } + } +} + +// ggml_compute_forward_lightning_indexer + +void ggml_compute_forward_lightning_indexer( + const ggml_compute_params * params, + ggml_tensor * dst) { + + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * w = dst->src[2]; // weights + const ggml_tensor * m = dst->src[3]; // mask + + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT( q->type == GGML_TYPE_F32); + GGML_ASSERT( w->type == GGML_TYPE_F32); + GGML_ASSERT( m->type == GGML_TYPE_F16); + + GGML_TENSOR_LOCALS(int64_t, neq, q, ne) + GGML_TENSOR_LOCALS(size_t, nbq, q, nb) + GGML_TENSOR_LOCALS(int64_t, nek, k, ne) + GGML_TENSOR_LOCALS(size_t, nbk, k, nb) + GGML_TENSOR_LOCALS(int64_t, new, w, ne) + GGML_TENSOR_LOCALS(size_t, nbw, w, nb) + GGML_TENSOR_LOCALS(int64_t, nem, m, ne) + GGML_TENSOR_LOCALS(size_t, nbm, m, nb) + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + + GGML_ASSERT( nb0 == ggml_type_size(dst->type)); + GGML_ASSERT(nbq0 == ggml_type_size( q->type)); + GGML_ASSERT(nbk0 == ggml_type_size( k->type)); + GGML_ASSERT(nbw0 == ggml_type_size( w->type)); + GGML_ASSERT(nbm0 == ggml_type_size( m->type)); + + const int n_embd = q->ne[0]; + const int n_head = q->ne[1]; + const int n_tokens = q->ne[2]; + const int n_stream = q->ne[3]; + const int n_kv = k->ne[2]; + + ggml_to_float_t const k_to_float = ggml_get_type_traits(k->type)->to_float; + GGML_ASSERT((k->type == GGML_TYPE_F32 || k_to_float) && "lightning indexer: unsupported K-type"); + + const int nr = n_kv; + const int ith = params->ith; + const int nth = params->nth; + + // (temporary) buffer for K converted to float + float * k_row_f32 = (float *) params->wdata + ith*(1*n_embd + CACHE_LINE_SIZE_F32); + + // rows per thread + const int dr = (nr + nth - 1)/nth; + + // row range for this thread + const int ir0 = dr*ith; + const int ir1 = MIN(ir0 + dr, nr); + + for (int s = 0; s < n_stream; ++s) { + for (int t = 0; t < n_tokens; ++t) { + const float * w_row = (float *) ((char *) w->data + t*nbw1 + s*nbw3); + const ggml_fp16_t * m_row = (ggml_fp16_t *) ((char *) m->data + t*nbm1 + (s%nem3)*nbm3); + float * dst_row = (float *) ((char *) dst->data + t*nb1 + s*nb3 ); + for (int ik = ir0; ik < ir1; ++ik) { + char * k_row = (char *) k->data + ik*nbk2 + s*nbk3; + if (k_to_float) { + k_to_float(k_row, k_row_f32, n_embd); + } else { + k_row_f32 = (float *) k_row; + } + float score = 0.0f; + for (int h = 0; h < n_head; ++h) { + // dot product of q and k for head h + float qk = 0.0f; + const float * q_row = (float *) ((char *) q->data + h*nbq1 + t*nbq2 + s*nbq3); + ggml_vec_dot_f32(n_embd, &qk, 0, q_row, 0, k_row_f32, 0, 1); + // ReLU and weights (prescaled) + score += MAX(qk, 0.0f) * w_row[h]; + } + // apply mask + dst_row[ik] = score + GGML_CPU_FP16_TO_FP32(m_row[ik]); + } + } + } +} diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index 5eb7e69..88cbb77 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -5,10 +5,10 @@ // // cache line // - -#if defined(__cpp_lib_hardware_interference_size) -#define CACHE_LINE_SIZE std::hardware_destructive_interference_size -#else +// TODO: rework CACHE_LINE_SIZE so std::hardware_destructive_interference_size +// can be used consistently between C and C++ TUs; the previous macro form +// diverged based on include order and undersized the work buffer. +// ref: https://github.com/ggml-org/llama.cpp/pull/28882 #if defined(__POWER9_VECTOR__) #define CACHE_LINE_SIZE 128 #elif defined(__VXE__) || defined(__VXE2__) @@ -16,7 +16,6 @@ #else #define CACHE_LINE_SIZE 64 #endif -#endif static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float); @@ -44,6 +43,7 @@ void ggml_compute_forward_concat(const struct ggml_compute_params * params, stru void ggml_compute_forward_silu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_rms_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_rms_norm_mul_fused(const struct ggml_compute_params * params, struct ggml_tensor * dst_rms_norm, struct ggml_tensor * dst_mul); void ggml_compute_forward_rms_norm_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_group_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_l2_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); @@ -73,6 +73,7 @@ void ggml_compute_forward_conv_transpose_1d(const struct ggml_compute_params * p void ggml_compute_forward_im2col(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_im2col_back_f32(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_im2col_3d(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_col2im_1d(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_conv_2d(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_conv_3d(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_conv_transpose_2d(const struct ggml_compute_params * params, struct ggml_tensor * dst); @@ -109,6 +110,11 @@ void ggml_compute_forward_rwkv_wkv7(const struct ggml_compute_params * params, s void ggml_compute_forward_solve_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_gla(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_gated_delta_net(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_lightning_indexer(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_dsv4_hc_comb(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_dsv4_hc_pre(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_dsv4_hc_post(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_top_k_mask(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_map_custom1(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_map_custom2(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_map_custom3(const struct ggml_compute_params * params, struct ggml_tensor * dst); @@ -117,6 +123,7 @@ void ggml_compute_forward_cross_entropy_loss(const struct ggml_compute_params * void ggml_compute_forward_cross_entropy_loss_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_opt_step_adamw(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_mul_mat(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_fwht(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_opt_step_sgd(const struct ggml_compute_params * params, struct ggml_tensor * dst); #ifdef __cplusplus } diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index f66127c..5e36459 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -26,6 +26,10 @@ void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in quantize_row_q1_0_ref(x, y, k); } +void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + quantize_row_q2_0_ref(x, y, k); +} + void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { quantize_row_q4_0_ref(x, y, k); } @@ -137,22 +141,76 @@ void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c float sumf = 0.0; for (int i = 0; i < nb; i++) { - const float d0 = GGML_FP16_TO_FP32(x[i].d); + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); float sumi = 0.0f; for (int k = 0; k < 4; k++) { - const float d1 = GGML_FP16_TO_FP32(y[i*4 + k].d); + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); + int sumi_block = 0; + + const uint8_t * GGML_RESTRICT bits = &x[i].qs[k * 4]; + const int8_t * GGML_RESTRICT qy = yb->qs; + + for (int b = 0; b < 4; ++b, qy += 8) { + const unsigned mask = bits[b]; + sumi_block += ((mask & 0x01) ? qy[0] : -qy[0]) + + ((mask & 0x02) ? qy[1] : -qy[1]) + + ((mask & 0x04) ? qy[2] : -qy[2]) + + ((mask & 0x08) ? qy[3] : -qy[3]) + + ((mask & 0x10) ? qy[4] : -qy[4]) + + ((mask & 0x20) ? qy[5] : -qy[5]) + + ((mask & 0x40) ? qy[6] : -qy[6]) + + ((mask & 0x80) ? qy[7] : -qy[7]); + } + + sumi += d1 * sumi_block; + } + + sumf += d0 * sumi; + } + + *s = sumf; +} + +void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK2_0; + const int nb = n / qk; + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + + for (int i = 0; i < nb; i++) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + + float sumi = 0.0f; + + // group 64: one Q2_0 block (64 weights) maps to two Q8_0 blocks (2 * 32 = 64) + for (int k = 0; k < 2; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 2 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); int sumi_block = 0; - for (int j = 0; j < QK8_0; j++) { - const int bit_index = k * QK8_0 + j; - const int byte_index = bit_index / 8; - const int bit_offset = bit_index % 8; + const uint8_t * GGML_RESTRICT qs = &x[i].qs[k * 8]; + const int8_t * GGML_RESTRICT qy = yb->qs; - const int xi = ((x[i].qs[byte_index] >> bit_offset) & 1) ? 1 : -1; - sumi_block += xi * y[i*4 + k].qs[j]; + for (int b = 0; b < 8; ++b) { + const uint8_t byte = qs[b]; + // Extract 4 two-bit values, map {0,1,2,3} -> {-1,0,1,2} + sumi_block += ((int)((byte >> 0) & 3) - 1) * qy[b*4 + 0]; + sumi_block += ((int)((byte >> 2) & 3) - 1) * qy[b*4 + 1]; + sumi_block += ((int)((byte >> 4) & 3) - 1) * qy[b*4 + 2]; + sumi_block += ((int)((byte >> 6) & 3) - 1) * qy[b*4 + 3]; } sumi += d1 * sumi_block; @@ -164,7 +222,6 @@ void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } - void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { const int qk = QK8_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index d4bc87a..93ea7ee 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -13,6 +13,7 @@ extern "C" { // Quantization void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); @@ -38,6 +39,7 @@ void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, // Dot product void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -71,6 +73,7 @@ void quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRI void quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index 5ef30e3..5fb5270 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -1365,6 +1365,133 @@ void ggml_gemv_q8_0_4x8_q8_0_generic(int n, } } +void ggml_gemv_q1_0_4x4_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + + float sumf[4]; + + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + for (int j = 0; j < ncols_interleaved; j++) { + sumf[j] = 0.0; + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < QK1_0 / QK8_0; ++k) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * (QK1_0 / QK8_0) + k; + const float d1 = GGML_CPU_FP16_TO_FP32(a_blk->d); + const float scale[4] = { d0[0] * d1, d0[1] * d1, d0[2] * d1, d0[3] * d1 }; + + for (int tile = 0; tile < QK8_0 / 4; ++tile) { + const uint8_t bits_lo = b_ptr[l].qs[k * 16 + 2 * tile + 0]; + const uint8_t bits_hi = b_ptr[l].qs[k * 16 + 2 * tile + 1]; + + for (int p = 0; p < 4; ++p) { + const float q = (float) a_blk->qs[tile * 4 + p]; + + sumf[0] += ((bits_lo & (1u << p)) ? scale[0] : -scale[0]) * q; + sumf[1] += ((bits_lo & (1u << (4 + p))) ? scale[1] : -scale[1]) * q; + sumf[2] += ((bits_hi & (1u << p)) ? scale[2] : -scale[2]) * q; + sumf[3] += ((bits_hi & (1u << (4 + p))) ? scale[3] : -scale[3]) * q; + } + } + } + } + + for (int j = 0; j < ncols_interleaved; j++) { + s[x * ncols_interleaved + j] = sumf[j]; + } + } +} + +void ggml_gemv_q1_0_4x8_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + const int blocklen = 8; + + assert(nr == 1); + assert(n % qk == 0); + assert(nc % ncols_interleaved == 0); + + UNUSED(bs); + UNUSED(nr); + + float sumf[4]; + + const block_q8_0 * a_ptr = (const block_q8_0 *) vy; + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + for (int j = 0; j < ncols_interleaved; j++) { + sumf[j] = 0.0f; + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < qk / blocklen; ++k) { + const block_q8_0 * GGML_RESTRICT a_blk = a_ptr + l * (qk / QK8_0) + k / (QK8_0 / blocklen); + const float d1 = GGML_CPU_FP16_TO_FP32(a_blk->d); + const float scale[4] = { d0[0] * d1, d0[1] * d1, d0[2] * d1, d0[3] * d1 }; + const uint8_t bits0 = b_ptr[l].qs[k * ncols_interleaved + 0]; + const uint8_t bits1 = b_ptr[l].qs[k * ncols_interleaved + 1]; + const uint8_t bits2 = b_ptr[l].qs[k * ncols_interleaved + 2]; + const uint8_t bits3 = b_ptr[l].qs[k * ncols_interleaved + 3]; + const int q_offset = (k % (QK8_0 / blocklen)) * blocklen; + + for (int p = 0; p < blocklen; ++p) { + const float q = (float) a_blk->qs[q_offset + p]; + + sumf[0] += ((bits0 & (1u << p)) ? scale[0] : -scale[0]) * q; + sumf[1] += ((bits1 & (1u << p)) ? scale[1] : -scale[1]) * q; + sumf[2] += ((bits2 & (1u << p)) ? scale[2] : -scale[2]) * q; + sumf[3] += ((bits3 & (1u << p)) ? scale[3] : -scale[3]) * q; + } + } + } + + for (int j = 0; j < ncols_interleaved; j++) { + s[x * ncols_interleaved + j] = sumf[j]; + } + } +} + // Only enable these for RISC-V. #if defined __riscv_zvfh void ggml_gemv_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { @@ -2383,6 +2510,176 @@ void ggml_gemm_q8_0_4x8_q8_0_generic(int n, } } +void ggml_gemm_q1_0_4x4_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % ncols_interleaved == 0); + + float sumf[4][4]; + + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + sumf[m][j] = 0.0; + } + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < QK1_0 / QK8_0; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k; + const float a_d[4] = { + GGML_CPU_FP16_TO_FP32(a_blk->d[0]), + GGML_CPU_FP16_TO_FP32(a_blk->d[1]), + GGML_CPU_FP16_TO_FP32(a_blk->d[2]), + GGML_CPU_FP16_TO_FP32(a_blk->d[3]), + }; + + for (int tile = 0; tile < QK8_0 / 4; ++tile) { + const uint8_t bits_lo = b_ptr[l].qs[k * 16 + 2 * tile + 0]; + const uint8_t bits_hi = b_ptr[l].qs[k * 16 + 2 * tile + 1]; + const int tile_offset = tile * 16; + + for (int p = 0; p < 4; ++p) { + const int8_t q_row[4] = { + a_blk->qs[tile_offset + 0 * 4 + p], + a_blk->qs[tile_offset + 1 * 4 + p], + a_blk->qs[tile_offset + 2 * 4 + p], + a_blk->qs[tile_offset + 3 * 4 + p], + }; + const int sign[4] = { + (bits_lo & (1u << p)) ? 1 : -1, + (bits_lo & (1u << (4 + p))) ? 1 : -1, + (bits_hi & (1u << p)) ? 1 : -1, + (bits_hi & (1u << (4 + p))) ? 1 : -1, + }; + + for (int m = 0; m < 4; ++m) { + const float row_scale = a_d[m]; + sumf[m][0] += sign[0] * q_row[m] * d0[0] * row_scale; + sumf[m][1] += sign[1] * q_row[m] * d0[1] * row_scale; + sumf[m][2] += sign[2] * q_row[m] * d0[2] * row_scale; + sumf[m][3] += sign[3] * q_row[m] * d0[3] * row_scale; + } + } + } + } + } + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j]; + } + } + } + } +} + +void ggml_gemm_q1_0_4x8_q8_0_generic(int n, + float * GGML_RESTRICT s, + size_t bs, + const void * GGML_RESTRICT vx, + const void * GGML_RESTRICT vy, + int nr, + int nc) { + const int qk = QK1_0; + const int nb = n / qk; + const int ncols_interleaved = 4; + const int blocklen = 8; + + assert(n % qk == 0); + assert(nr % 4 == 0); + assert(nc % ncols_interleaved == 0); + + float sumf[4][4]; + + for (int y = 0; y < nr / 4; y++) { + const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (4 * y * nb); + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q1_0x4 * b_ptr = (const block_q1_0x4 *) vx + (x * nb); + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + sumf[m][j] = 0.0f; + } + } + + for (int l = 0; l < nb; l++) { + const float d0[4] = { + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]), + GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]), + }; + + for (int k = 0; k < qk / blocklen; ++k) { + const block_q8_0x4 * GGML_RESTRICT a_blk = a_ptr + 4 * l + k / (QK8_0 / blocklen); + const float a_d[4] = { + GGML_CPU_FP16_TO_FP32(a_blk->d[0]), + GGML_CPU_FP16_TO_FP32(a_blk->d[1]), + GGML_CPU_FP16_TO_FP32(a_blk->d[2]), + GGML_CPU_FP16_TO_FP32(a_blk->d[3]), + }; + const uint8_t bits0 = b_ptr[l].qs[k * ncols_interleaved + 0]; + const uint8_t bits1 = b_ptr[l].qs[k * ncols_interleaved + 1]; + const uint8_t bits2 = b_ptr[l].qs[k * ncols_interleaved + 2]; + const uint8_t bits3 = b_ptr[l].qs[k * ncols_interleaved + 3]; + const int q_offset = (k % (QK8_0 / blocklen)) * 4 * blocklen; + + for (int p = 0; p < blocklen; ++p) { + const int8_t q_row[4] = { + a_blk->qs[q_offset + 0 * blocklen + p], + a_blk->qs[q_offset + 1 * blocklen + p], + a_blk->qs[q_offset + 2 * blocklen + p], + a_blk->qs[q_offset + 3 * blocklen + p], + }; + const int sign[4] = { + (bits0 & (1u << p)) ? 1 : -1, + (bits1 & (1u << p)) ? 1 : -1, + (bits2 & (1u << p)) ? 1 : -1, + (bits3 & (1u << p)) ? 1 : -1, + }; + + for (int m = 0; m < 4; ++m) { + const float row_scale = a_d[m]; + sumf[m][0] += sign[0] * q_row[m] * d0[0] * row_scale; + sumf[m][1] += sign[1] * q_row[m] * d0[1] * row_scale; + sumf[m][2] += sign[2] * q_row[m] * d0[2] * row_scale; + sumf[m][3] += sign[3] * q_row[m] * d0[3] * row_scale; + } + } + } + } + + for (int m = 0; m < 4; m++) { + for (int j = 0; j < ncols_interleaved; j++) { + s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j]; + } + } + } + } +} + // Only enable these for RISC-V. #if defined __riscv_zvfh void ggml_gemm_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) { @@ -2739,7 +3036,51 @@ static block_q8_0x4 make_block_q8_0x4(block_q8_0 * in, unsigned int blck_size_in return out; } -static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave) { +static block_q1_0x4 make_block_q1_0x4(block_q1_0 * in, unsigned int blck_size_interleave) { + block_q1_0x4 out; + + for (int i = 0; i < 4; i++) { + out.d[i] = in[i].d; + } + + GGML_ASSERT(blck_size_interleave == 4 || blck_size_interleave == 8); + + if (blck_size_interleave == 4) { + for (int k = 0; k < QK1_0 / QK8_0; ++k) { + for (int tile = 0; tile < QK8_0 / 4; ++tile) { + uint8_t packed_lo = 0; + uint8_t packed_hi = 0; + + const int weight_base = k * QK8_0 + tile * 4; + for (int pos = 0; pos < 4; ++pos) { + const int weight_idx = weight_base + pos; + const int byte_idx = weight_idx / 8; + const int bit_idx = weight_idx % 8; + + packed_lo |= ((in[0].qs[byte_idx] >> bit_idx) & 1u) << pos; + packed_lo |= ((in[1].qs[byte_idx] >> bit_idx) & 1u) << (4 + pos); + packed_hi |= ((in[2].qs[byte_idx] >> bit_idx) & 1u) << pos; + packed_hi |= ((in[3].qs[byte_idx] >> bit_idx) & 1u) << (4 + pos); + } + + out.qs[k * 16 + 2 * tile + 0] = packed_lo; + out.qs[k * 16 + 2 * tile + 1] = packed_hi; + } + } + return out; + } + + for (int byte_idx = 0; byte_idx < QK1_0 / 8; ++byte_idx) { + out.qs[byte_idx * 4 + 0] = in[0].qs[byte_idx]; + out.qs[byte_idx * 4 + 1] = in[1].qs[byte_idx]; + out.qs[byte_idx * 4 + 2] = in[2].qs[byte_idx]; + out.qs[byte_idx * 4 + 3] = in[3].qs[byte_idx]; + } + + return out; +} + +static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, int blck_size_interleave) { block_q4_0x4 out; for (int i = 0; i < 4; i++) { @@ -3509,6 +3850,38 @@ static int repack_q8_0_to_q8_0_4_bl(struct ggml_tensor * t, return 0; } +static int repack_q1_0_to_q1_0_4_bl(struct ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q1_0); + GGML_ASSERT(interleave_block == 4 || interleave_block == 8); + constexpr int nrows_interleaved = 4; + + block_q1_0x4 * dst = (block_q1_0x4 *) t->data; + const block_q1_0 * src = (const block_q1_0 *) data; + block_q1_0 dst_tmp[4]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK1_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q1_0)); + + if (t->ne[1] % nrows_interleaved != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q1_0x4(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; +} + static block_q8_0x16 make_block_q8_0x16(block_q8_0 * in, unsigned int blck_size_interleave) { block_q8_0x16 out; @@ -3865,6 +4238,14 @@ template int repack(struct ggml_tensor *, const void *, size_t); // TODO: generalise. +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_q1_0_to_q1_0_4_bl(t, 4, data, data_size); +} + +template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { + return repack_q1_0_to_q1_0_4_bl(t, 8, data, data_size); +} + template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { return repack_q4_0_to_q4_0_4_bl(t, 4, data, data_size); } @@ -3960,6 +4341,14 @@ template <> int repack(struct ggml_tensor * t, const void * d template void gemv(int, float *, size_t, const void *, const void *, int, int); +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_q1_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemv_q1_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); +} + template <> void gemv(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemv_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc); } @@ -4057,6 +4446,14 @@ template <> void gemv(int n, float * s, size_ template void gemm(int, float *, size_t, const void *, const void *, int, int); +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_q1_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc); +} + +template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { + ggml_gemm_q1_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc); +} + template <> void gemm(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) { ggml_gemm_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc); } @@ -4458,7 +4855,12 @@ template data + iid1 * ids->nb[1] + id * ids->nb[0]); - GGML_ASSERT(i02 >= 0 && i02 < n_as); + GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as)); + + if (i02 == -1) { + memset((char *) dst->data + id*nb1 + iid1*nb2, 0, ne0*sizeof(float)); + continue; + } MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = { id, iid1 }; matrix_row_counts[i02] += 1; @@ -4530,6 +4932,10 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons return nullptr; } + // instance for Q1_0 + static const ggml::cpu::repack::tensor_traits q1_0_4x4_q8_0; + static const ggml::cpu::repack::tensor_traits q1_0_4x8_q8_0; + // instance for Q4 static const ggml::cpu::repack::tensor_traits q4_0_4x4_q8_0; static const ggml::cpu::repack::tensor_traits q4_0_4x8_q8_0; @@ -4590,6 +4996,11 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons return &q4_0_4x4_q8_0; } } + if (ggml_cpu_has_vxe()) { + if (cur->ne[1] % 4 == 0) { + return &q4_0_4x4_q8_0; + } + } if (ggml_cpu_has_riscv_v()) { #if defined __riscv_zvfh switch (__riscv_vlenb() * 8) { @@ -4728,6 +5139,17 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons } #endif } + } else if (cur->type == GGML_TYPE_Q1_0) { + if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) { + if (cur->ne[1] % 4 == 0) { + return &q1_0_4x8_q8_0; + } + } + if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) { + if (cur->ne[1] % 4 == 0) { + return &q1_0_4x4_q8_0; + } + } } return nullptr; @@ -4836,12 +5258,14 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { ggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void) { static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_repack = { /* .iface = */ { - /* .get_name = */ ggml_backend_cpu_repack_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_cpu_repack_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_repack_buffer_type_get_alignment, - /* .get_max_size = */ nullptr, // defaults to SIZE_MAX - /* .get_alloc_size = */ nullptr, // defaults to ggml_nbytes - /* .is_host = */ nullptr, + /* .get_name = */ ggml_backend_cpu_repack_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_cpu_repack_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ nullptr, + /* .get_alignment = */ ggml_backend_cpu_repack_buffer_type_get_alignment, + /* .get_max_size = */ nullptr, // defaults to SIZE_MAX + /* .get_alloc_size = */ nullptr, // defaults to ggml_nbytes + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ nullptr, }, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0), /* .context = */ new ggml::cpu::repack::extra_buffer_type(), diff --git a/ggml/src/ggml-cpu/repack.h b/ggml/src/ggml-cpu/repack.h index cb21edf..fc6715c 100644 --- a/ggml/src/ggml-cpu/repack.h +++ b/ggml/src/ggml-cpu/repack.h @@ -11,6 +11,9 @@ ggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void); template constexpr int QK_0() { + if constexpr (K == 1) { + return QK1_0; + } if constexpr (K == 4) { return QK4_0; } @@ -26,6 +29,7 @@ template struct block { }; // control size +static_assert(sizeof(block<1, 4>) == 4 * sizeof(ggml_half) + QK1_0 / 2, "wrong block<1,4> size/padding"); static_assert(sizeof(block<4, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 2, "wrong block<4,4> size/padding"); static_assert(sizeof(block<4, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 4, "wrong block<4,8> size/padding"); static_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 8, "wrong block<4,16> size/padding"); @@ -33,6 +37,7 @@ static_assert(sizeof(block<8, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 4, "wrong b static_assert(sizeof(block<8, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong block<8,8> size/padding"); static_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 16, "wrong block<8,16> size/padding"); +using block_q1_0x4 = block<1, 4>; using block_q4_0x4 = block<4, 4>; using block_q4_0x8 = block<4, 8>; using block_q4_0x16 = block<4, 16>; @@ -141,6 +146,8 @@ void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTR void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_gemv_q1_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -157,6 +164,8 @@ void ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v void ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -193,6 +202,8 @@ void ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GG void ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); +void ggml_gemv_q1_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemv_q1_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); @@ -209,6 +220,8 @@ void ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemv_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); +void ggml_gemm_q1_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); void ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc); diff --git a/ggml/src/ggml-cpu/simd-gemm.h b/ggml/src/ggml-cpu/simd-gemm.h index 4119d04..4b9396d 100644 --- a/ggml/src/ggml-cpu/simd-gemm.h +++ b/ggml/src/ggml-cpu/simd-gemm.h @@ -56,6 +56,56 @@ static inline void simd_gemm_ukernel( } } +template +static inline void simd_gemm_ukernel_tail( + float * GGML_RESTRICT C, + const float * GGML_RESTRICT A, + const float * GGML_RESTRICT B, + int K, int N, int cols) +{ +#if defined(__AVX512F__) + const __mmask16 mask = (1u << cols) - 1; + __m512 acc[RM]; + for (int64_t i = 0; i < RM; i++) { + acc[i] = _mm512_maskz_loadu_ps(mask, C + i * N); + } + for (int64_t kk = 0; kk < K; kk++) { + const __m512 b = _mm512_maskz_loadu_ps(mask, B + kk * N); + for (int64_t i = 0; i < RM; i++) { + acc[i] = _mm512_mask3_fmadd_ps(_mm512_set1_ps(A[i * K + kk]), b, acc[i], mask); + } + } + for (int64_t i = 0; i < RM; i++) { + _mm512_mask_storeu_ps(C + i * N, mask, acc[i]); + } +#elif defined(__AVX2__) + const __m256i mask = _mm256_cmpgt_epi32(_mm256_set1_epi32(cols), _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7)); + __m256 acc[RM]; + for (int64_t i = 0; i < RM; i++) { + acc[i] = _mm256_maskload_ps(C + i * N, mask); + } + for (int64_t kk = 0; kk < K; kk++) { + const __m256 b = _mm256_maskload_ps(B + kk * N, mask); + for (int64_t i = 0; i < RM; i++) { + acc[i] = GGML_F32_VEC_FMA(acc[i], b, _mm256_set1_ps(A[i * K + kk])); + } + } + for (int64_t i = 0; i < RM; i++) { + _mm256_maskstore_ps(C + i * N, mask, acc[i]); + } +#else + for (int64_t j = 0; j < cols; j++) { + for (int64_t i = 0; i < RM; i++) { + float a = C[i * N + j]; + for (int64_t kk = 0; kk < K; kk++) { + a += A[i * K + kk] * B[kk * N + j]; + } + C[i * N + j] = a; + } + } +#endif +} + // C[M x N] += A[M x K] * B[K x N] static void simd_gemm( float * GGML_RESTRICT C, @@ -74,14 +124,8 @@ static void simd_gemm( for (; jj + KN <= N; jj += KN) { simd_gemm_ukernel(C + jj, A, B + jj, K, N); } - for (; jj < N; jj++) { - for (int64_t i = 0; i < GEMM_RM; i++) { - float a = C[i * N + jj]; - for (int64_t kk = 0; kk < K; kk++) { - a += A[i + kk] * B[kk * N + jj]; - } - C[i * N + jj] = a; - } + if (jj < N) { + simd_gemm_ukernel_tail(C + jj, A, B + jj, K, N, N - jj); } A += GEMM_RM * K; @@ -97,12 +141,8 @@ static void simd_gemm( for (; jj + KN <= N; jj += KN) { simd_gemm_ukernel<1, 1>(C + jj, A, B + jj, K, N); } - for (; jj < N; jj++) { - float a = C[jj]; - for (int64_t kk = 0; kk < K; kk++) { - a += A[kk] * B[kk * N + jj]; - } - C[jj] = a; + if (jj < N) { + simd_gemm_ukernel_tail<1>(C + jj, A, B + jj, K, N, N - jj); } A += K; diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h index 0deda93..b7b390d 100644 --- a/ggml/src/ggml-cpu/simd-mappings.h +++ b/ggml/src/ggml-cpu/simd-mappings.h @@ -29,17 +29,20 @@ extern "C" { // FP16 to FP32 conversion // 16-bit float -// on Arm, we use __fp16 +// on Arm, we use __fp16, which requires the IEEE fp16 format: implied on +// AArch64, selected by -mfp16-format=ieee on 32 bit Arm, where the compiler +// may otherwise reject the type // on x86, we use uint16_t // // for old CUDA compilers (<= 11), we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/10616 // for MUSA compilers , we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/11843 // -#if defined(__ARM_NEON) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11) && !defined(__MUSACC__) +#if defined(__ARM_NEON) && defined(__ARM_FP16_FORMAT_IEEE) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11) && !defined(__MUSACC__) #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) neon_compute_fp16_to_fp32(x) #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) neon_compute_fp32_to_fp16(x) #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x) + #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x) static inline float neon_compute_fp16_to_fp32(ggml_fp16_t h) { __fp16 tmp; @@ -120,6 +123,10 @@ extern float ggml_table_f32_f16[1 << 16]; // defined in ggml-cpu.c, initialized in ggml_cpu_init() extern float ggml_table_f32_e8m0_half[1 << 8]; +// precomputed f32 table for ue4m3 (1 KB) +// defined in ggml-cpu.c, initialized in ggml_cpu_init() +extern float ggml_table_f32_ue4m3[1 << 8]; + // Use lookup table for E8M0 on x86 (faster than bit manipulation) #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) #define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)] @@ -127,9 +134,32 @@ extern float ggml_table_f32_e8m0_half[1 << 8]; #define GGML_CPU_E8M0_TO_FP32_HALF(x) GGML_E8M0_TO_FP32_HALF(x) #endif -// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32, -// so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON. -// This is also true for POWER9. +// Use lookup table for UE4M3 on x86 and ARM (faster than bit manipulation) +#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__ARM_NEON) +#define GGML_CPU_UE4M3_TO_FP32(x) ggml_table_f32_ue4m3[(uint8_t)(x)] +#else +#define GGML_CPU_UE4M3_TO_FP32(x) ggml_ue4m3_to_fp32(x) +#endif + +// Pre AVX512 CPUs (AVX2 with F16C, such as i5-8th gen or i7-4790) +// do not produce codegen as checks below don't test for __F16C__ +// but this check does. +#if defined(__F16C__) +// force FP16 -> FP32 conversion to use LUT instead +// see: https://github.com/ggml-org/llama.cpp/issues/30342 +//#define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x) +#define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x) +#endif + +// a branch that defines a GGML_CPU_COMPUTE_* macro must also select it, or the fallback takes over silently +#if defined(GGML_CPU_COMPUTE_FP16_TO_FP32) && !defined(GGML_CPU_FP16_TO_FP32) && !defined(__F16C__) // see: https://github.com/ggml-org/llama.cpp/issues/30342 +#error "a branch defined GGML_CPU_COMPUTE_FP16_TO_FP32 without selecting it as GGML_CPU_FP16_TO_FP32" +#endif +#if defined(GGML_CPU_COMPUTE_FP32_TO_FP16) && !defined(GGML_CPU_FP32_TO_FP16) +#error "a branch defined GGML_CPU_COMPUTE_FP32_TO_FP16 without selecting it as GGML_CPU_FP32_TO_FP16" +#endif + +// fall back to the lookup table / the portable software conversion for whichever direction no branch selected #if !defined(GGML_CPU_FP16_TO_FP32) inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) { uint16_t s; @@ -315,7 +345,7 @@ inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) { #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE #endif -#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) +#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) && defined(__ARM_FP16_FORMAT_IEEE) #define GGML_SIMD @@ -1125,25 +1155,12 @@ static inline void __lasx_f32cx8_store(ggml_fp16_t * x, __m256 y) { #define GGML_F16_EPR 4 static inline __m128 __lsx_f16x4_load(const ggml_fp16_t * x) { - float tmp[4]; - - tmp[0] = GGML_CPU_FP16_TO_FP32(x[0]); - tmp[1] = GGML_CPU_FP16_TO_FP32(x[1]); - tmp[2] = GGML_CPU_FP16_TO_FP32(x[2]); - tmp[3] = GGML_CPU_FP16_TO_FP32(x[3]); - - return (__m128)__lsx_vld(tmp, 0); + return __lsx_vfcvtl_s_h(__lsx_vld((const void *)x, 0)); } static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { - float arr[4]; - - __lsx_vst(y, arr, 0); - - x[0] = GGML_CPU_FP32_TO_FP16(arr[0]); - x[1] = GGML_CPU_FP32_TO_FP16(arr[1]); - x[2] = GGML_CPU_FP32_TO_FP16(arr[2]); - x[3] = GGML_CPU_FP32_TO_FP16(arr[3]); + __m128i a = __lsx_vfcvt_h_s(y, y); + memcpy(x, &a, sizeof(ggml_fp16_t) * 4); } #define GGML_F32Cx4 __m128 @@ -1223,6 +1240,24 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { #define GGML_F16_STEP GGML_F32_STEP #define GGML_F16_EPR GGML_F32_EPR +static inline uint32x4_t __lzs_f32cx4_to_f16(float32x4_t v_f) { + float32x4_t v_base = vec_mul(vec_mul(vec_abs(v_f), vec_splats(0x1.0p+112f)), vec_splats(0x1.0p-110f)); + + const uint32x4_t v_w = (uint32x4_t)v_f; + const uint32x4_t v_shl1_w = vec_add(v_w, v_w); + const uint32x4_t v_sign = vec_and(v_w, vec_splats(UINT32_C(0x80000000))); + const uint32x4_t v_bias = vec_max(vec_and(v_shl1_w, vec_splats(UINT32_C(0xFF000000))), vec_splats(UINT32_C(0x71000000))); + + v_base = vec_add((float32x4_t)vec_add(vec_sr(v_bias, 1), vec_splats(UINT32_C(0x07800000))), v_base); + + const uint32x4_t v_bits = (uint32x4_t)v_base; + const uint32x4_t v_nonsign = vec_add(vec_and(vec_sr(v_bits, 13), vec_splats(UINT32_C(0x00007C00))), + vec_and(v_bits, vec_splats(UINT32_C(0x00000FFF)))); + const uint32x4_t v_is_nan = (uint32x4_t)vec_cmpgt(v_shl1_w, vec_splats(UINT32_C(0xFF000000))); + + return vec_or(vec_sr(v_sign, 16), vec_sel(v_nonsign, vec_splats(UINT32_C(0x7E00)), v_is_nan)); +} + static inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) { float tmp[4]; @@ -1236,15 +1271,9 @@ static inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) { } static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) { - float arr[4]; - - // note: keep type-cast here to prevent compiler bugs - // see: https://github.com/ggml-org/llama.cpp/issues/12846 - vec_xst(v_y, 0, (float *)(arr)); - - for (int i = 0; i < 4; i++) { - x[i] = GGML_CPU_FP32_TO_FP16(arr[i]); - } + const uint32x4_t v_h = __lzs_f32cx4_to_f16(v_y); + const uint64_t tmp = ((uint64x2_t)vec_pack(v_h, v_h))[0]; + memcpy(x, &tmp, sizeof(tmp)); } #define GGML_F16_VEC GGML_F32x4 @@ -1314,6 +1343,33 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) { #define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) #endif +// GGML_F16_DOT_* +// like GGML_F16_* but for dot products which need F32 accumulation on AVX512-FP16 + +#if defined(__AVX512FP16__) + +#define GGML_F16_DOT_STEP GGML_F32_STEP +#define GGML_F16_DOT_EPR GGML_F32_EPR +#define GGML_F16_DOT_ARR GGML_F32_ARR +#define GGML_F16_DOT_VEC GGML_F32x16 +#define GGML_F16_DOT_VEC_ZERO GGML_F32x16_ZERO +#define GGML_F16_DOT_VEC_LOAD(p, i) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p))) +#define GGML_F16_DOT_VEC_FMA GGML_F32x16_FMA +#define GGML_F16_DOT_VEC_REDUCE GGML_F32x16_REDUCE + +#else + +#define GGML_F16_DOT_STEP GGML_F16_STEP +#define GGML_F16_DOT_EPR GGML_F16_EPR +#define GGML_F16_DOT_ARR GGML_F16_ARR +#define GGML_F16_DOT_VEC GGML_F16_VEC +#define GGML_F16_DOT_VEC_ZERO GGML_F16_VEC_ZERO +#define GGML_F16_DOT_VEC_LOAD GGML_F16_VEC_LOAD +#define GGML_F16_DOT_VEC_FMA GGML_F16_VEC_FMA +#define GGML_F16_DOT_VEC_REDUCE GGML_F16_VEC_REDUCE + +#endif // defined(__AVX512FP16__) + #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-cpu/spacemit/ime.cpp b/ggml/src/ggml-cpu/spacemit/ime.cpp index 91fe192..7b80127 100644 --- a/ggml/src/ggml-cpu/spacemit/ime.cpp +++ b/ggml/src/ggml-cpu/spacemit/ime.cpp @@ -3,19 +3,32 @@ #include "ime.h" +#include "binary-ops.h" +#include "common.h" #include "ggml-backend-impl.h" #include "ggml-common.h" #include "ggml-cpu.h" +#include "ime_env.h" #include "ime_kernels.h" +#include "ops.h" +#include "repack.h" +#include "rvv_kernels.h" +#include "spine_mem_pool.h" #include "traits.h" +#include "vec.h" + +#include +#include +#include #include +#include #include +#include #include #include // for GGML_ASSERT #include #include - // clang-format off #if defined(__riscv) @@ -25,13 +38,17 @@ #include #endif -#if !defined(__riscv_zfh) -#error "riscv zfh extension not enabled" +#if !defined(__riscv_zfh) || !defined(__riscv_zvfh) +#error "riscv zfh extension not enabled, GGML_RV_ZFH and GGML_RV_ZVFH must be defined to 1" #endif -#if defined(RISCV64_SPACEMIT_IME1) +#if !defined(__riscv_zba) +#error "riscv zba extension not enabled, GGML_RV_ZBA must be defined to 1" +#endif + +#if defined(RISCV64_SPACEMIT_IME1) || defined(RISCV64_SPACEMIT_IME2) #else -#error "RISCV64_SPACEMIT_IME1 not defined" +#error "RISCV64_SPACEMIT_IME1 or RISCV64_SPACEMIT_IME2 not defined" #endif #else @@ -46,382 +63,495 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #endif -#if defined(RISCV64_SPACEMIT_IME1) -#define QGEMM_STRIDEN_THREAD_ALIGN 16 -#else -#define QGEMM_STRIDEN_THREAD_ALIGN 32 -#endif - // clang-format on -struct qnbitgemm_spacemit_ime_args { - const float * a_ptr = nullptr; - size_t lda = 0; - const std::byte * packed_quant_b_data = nullptr; - const float * quant_b_scale = nullptr; - const void * quant_b_zp = nullptr; - const float * quant_b_blksum = nullptr; - const float * bias = nullptr; - float * c_ptr = nullptr; - size_t ldc = 0; -}; - -constexpr size_t div_round_up(size_t up, size_t down) { - return (up + down - 1) / down; -} - -constexpr size_t q8_blk_size(size_t blk_len) { - const size_t blk_size = sizeof(float) + blk_len * sizeof(int8_t); - // Currently, the strictest alignment requirement of a block is for a float. - // Ensure contiguous blocks are suitably aligned. - assert(blk_size % alignof(float) == 0); - return blk_size; +extern "C" { +extern void ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value); +extern int ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value); } namespace ggml::cpu::riscv64_spacemit { -const int num_ai_cores = std::thread::hardware_concurrency() / 2; - -} // namespace ggml::cpu::riscv64_spacemit +struct TLSContext { + int cpu_id{ -1 }; + cpu_set_t cpuset; + void * tcm_buffer{ nullptr }; + size_t tcm_buffer_size{ 0 }; +}; -static void sqnbitgemm_spacemit_ime_i8i4(const size_t blk_len, - const size_t gemm_k, - const qnbitgemm_spacemit_ime_args * gemm_args, - void * const per_gemm_ws, - const size_t m_start, - const size_t m_count, - const size_t n_start, - const size_t n_count) { - constexpr size_t scale_stride = sizeof(uint16_t); - constexpr size_t blk_bitwidth = 4; +thread_local TLSContext tls_context; + +template constexpr size_t get_repacked_block_type_size() { + if constexpr (std::is_same_v || std::is_same_v) { + return sizeof(block_q8_0); + } else if constexpr (std::is_same_v) { + return sizeof(block_q4_0) * INTER_SIZE / QK4_0; + } else if constexpr (std::is_same_v || std::is_same_v) { + return (sizeof(block_q4_0) + sizeof(uint8_t)) * INTER_SIZE / QK4_1; + } else if constexpr (std::is_same_v) { + return sizeof(spacemit_kernels::nrow_block_q2_k<1>); + } else if constexpr (std::is_same_v) { + return sizeof(spacemit_kernels::nrow_block_q3_k<1>); + } else if constexpr (std::is_same_v) { + return sizeof(spacemit_kernels::nrow_block_mxfp4<1>); + } else if constexpr (std::is_same_v || std::is_same_v) { + return sizeof(spacemit_kernels::nrow_block_q5_1<1>); + } else if constexpr (std::is_same_v) { + return sizeof(spacemit_kernels::nrow_block_q5_0<1>); + } else { + assert(false); + return 0; + } +} - const size_t k_blks = div_round_up(gemm_k, blk_len); +template constexpr bool block_type_has_zp() { + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v) { + return false; + } else if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || + std::is_same_v) { + return true; + } else { + assert(false); + return false; + } +} - const size_t lda = k_blks * q8_blk_size(blk_len); - const size_t ldc = gemm_args->ldc; - const size_t ldb = k_blks * (blk_len * blk_bitwidth / 8); - const std::byte * quant_a_ptr = static_cast(per_gemm_ws) + m_start * lda; +class tensor_traits_base : public ggml::cpu::tensor_traits { + public: + virtual int repack(ggml_tensor * t, const void * data, size_t data_size) = 0; +}; - const size_t zero_point_stride = gemm_args->quant_b_zp != nullptr ? sizeof(uint8_t) : 0; - const size_t packed_b_stride = ldb + k_blks * (scale_stride + zero_point_stride); - const std::byte * packed_quant_b_data = gemm_args->packed_quant_b_data + n_start * packed_b_stride; +template class tensor_traits : public tensor_traits_base { + bool work_size(int /* n_threads */, const ggml_tensor * op, size_t & size) override { + switch (op->op) { + case GGML_OP_MUL_MAT: + { + int64_t src1_nelements = ggml_nelements(op->src[1]); + + if constexpr (std::is_same_v || std::is_same_v) { + size = + spacemit_kernels::div_round_up(src1_nelements, QK_K) * spacemit_kernels::q8k_blk_size(QK_K); + } else if constexpr (INTER_SIZE == QK4_0) { + size = spacemit_kernels::div_round_up(src1_nelements, QK4_0) * + spacemit_kernels::q8_blk_size(QK4_0, true); + } else if constexpr (INTER_SIZE == 256) { + size = spacemit_kernels::div_round_up(src1_nelements, 256) * + spacemit_kernels::q8_hp_blk_size(256, true, true); + } else { + GGML_ABORT("unsupported block type"); + } - float * c_ptr = gemm_args->c_ptr + m_start * ldc + n_start; + size = GGML_PAD(size, sizeof(int64_t)); - size_t count_n = 0; - const size_t compute_block_count_n = m_count == 1 ? n_count : 16; - for (size_t n = 0; n < n_count; n += count_n) { - count_n = std::min(n_count - n, compute_block_count_n); + return true; + } + case GGML_OP_MUL_MAT_ID: + { + int64_t src1_nelements = ggml_nelements(op->src[1]); + + if constexpr (std::is_same_v || std::is_same_v) { + size = + spacemit_kernels::div_round_up(src1_nelements, QK_K) * spacemit_kernels::q8k_blk_size(QK_K); + } else if constexpr (INTER_SIZE == QK4_0) { + size = spacemit_kernels::div_round_up(src1_nelements, QK4_0) * + spacemit_kernels::q8_blk_size(QK4_0, true); + } else if constexpr (INTER_SIZE == 256) { + size = spacemit_kernels::div_round_up(src1_nelements, 256) * + spacemit_kernels::q8_hp_blk_size(256, true, true); + } else { + GGML_ABORT("unsupported block type"); + } - const std::byte * a_row = quant_a_ptr; - const std::byte * b_col = packed_quant_b_data + n * packed_b_stride; - const std::byte * b_col_zp = (zero_point_stride != 0) ? b_col : nullptr; - float * c_blk = c_ptr + n; + size = GGML_PAD(size, sizeof(int64_t)); - int32_t rows_remaining = m_count; + const int64_t ne02 = op->src[0]->ne[2]; // n_as, n_expert + const int64_t ne12 = op->src[1]->ne[2]; // n_tokens - while (rows_remaining > 0) { - const auto rows_handled = sqnbitgemm_spacemit_ime::ime1::gemm_kernel_i8i4( - blk_len, a_row, b_col, nullptr, b_col_zp, c_blk, rows_remaining, count_n, gemm_k, k_blks, ldc, nullptr, - scale_stride); + const size_t sizeof_mmid_row_mapping = sizeof(int64_t); + size += sizeof_mmid_row_mapping * ne02 * (ne12 + 1) + (ne02 + 1) * sizeof(int64_t); - c_blk += rows_handled * ldc; - a_row += rows_handled * lda; + size = GGML_PAD(size, sizeof(int64_t)); - rows_remaining -= rows_handled; + return true; + } + default: + // GGML_ABORT("fatal error"); + break; } + return false; } -} -template constexpr int QK_0() { - if constexpr (K == 4) { - return QK4_0; - } - if constexpr (K == 8) { - return QK8_0; + bool compute_forward(ggml_compute_params * params, ggml_tensor * op) override { + switch (op->op) { + case GGML_OP_MUL_MAT: + switch (op->src[0]->type) { + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q5_K: + //case GGML_TYPE_MXFP4: + forward_mul_mat(params, op); + return true; + default: + // GGML_ABORT("fatal error: unsupported type for src0 in MUL_MAT"); + return false; + } + break; + case GGML_OP_MUL_MAT_ID: + switch (op->src[0]->type) { + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q5_K: + //case GGML_TYPE_MXFP4: + forward_mul_mat_id(params, op); + return true; + default: + // GGML_ABORT("fatal error: unsupported type for src0 in MUL_MAT_ID"); + return false; + } + break; + default: + // GGML_ABORT("fatal error"); + break; + } + return false; } - return -1; -} -template struct block { - ggml_half d[N]; // deltas for N qK_0 blocks - uint8_t qs[(QK_0() * N * K) / 8]; // quants for N qK_0 blocks -}; + void forward_mul_mat(ggml_compute_params * params, ggml_tensor * op) { + constexpr size_t a_blk_len = INTER_SIZE; + constexpr size_t b_blk_len = INTER_SIZE; -template struct block_with_zp { - ggml_half d[N]; // deltas for N qK_1 blocks - uint8_t zp[N]; // zero points for N qK_1 blocks - uint8_t qs[(QK_0() * N * K) / 8]; // quants for N qK_1 blocks -}; + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; -// control size -static_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 8, "wrong block<4,16> size/padding"); -static_assert(sizeof(block_with_zp<4, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 8 + 16 * sizeof(uint8_t), - "wrong block_with_zp<4,16> size/padding"); -static_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 16, "wrong block<8,16> size/padding"); + GGML_TENSOR_BINARY_OP_LOCALS -using block_q4_0x16 = block<4, 16>; -using block_q4_1x16 = block_with_zp<4, 16>; -using block_q8_0x16 = block<8, 16>; + int ith = params->ith; + int nth = params->nth; -static block_q4_0x16 make_block_q4_0x16(block_q4_0 * in, unsigned int blck_size_interleave) { - block_q4_0x16 out; - GGML_ASSERT(QK4_0 / blck_size_interleave == 2); + [[maybe_unused]] const enum ggml_type type = src0->type; - for (int i = 0; i < 16; i++) { - out.d[i] = in[i].d; - } + void * w_data = (void *) src0->data; + const float * feature = (const float *) src1->data; + float * output = (float *) dst->data; - for (int i = 0; i < 16; i++) { - // [0, 15], in.d & 0x0F - for (int j = 0; j < QK4_0 / 4; j++) { - //src [b0 b16] ......... [b8 b24] ......... [b15 b31] - //dst [b0 b8] ......... [b7 b15] - out.qs[i * QK4_0 / 4 + j] = (in[i].qs[j] & 0x0F) | ((in[i].qs[j + QK4_0 / 4] & 0x0F) << 4); + const int64_t gemm_m = ne11 * ne12 * ne13; + const int64_t gemm_k = ne10; + const int64_t gemm_n = ne01; + + spacemit_kernels::quantize_a_row_def quantize_a_row_i8; + spacemit_kernels::quantize_a_row_def quantize_a_4row_i8; + spacemit_kernels::gemm_kernel_quantize_def gemm_kernel; + bool set_kernel_impl = false; + + int64_t block_stride_a = spacemit_kernels::q8_blk_size(a_blk_len); + +#if defined(RISCV64_SPACEMIT_IME2) + if (!set_kernel_impl && (global_spine_env_info.use_ime2)) { + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8; + quantize_a_4row_i8 = spacemit_kernels::rvv::quantize_a_4row_i8; + block_stride_a = spacemit_kernels::q8_blk_size(a_blk_len, true); + + if constexpr (std::is_same_v || std::is_same_v) { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i8; + set_kernel_impl = true; + } else if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if constexpr (INTER_SIZE == 256) { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i4_hp; + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8_hp; + quantize_a_4row_i8 = spacemit_kernels::rvv::quantize_a_4row_i8_hp; + block_stride_a = spacemit_kernels::q8_hp_blk_size(a_blk_len, true, true); + set_kernel_impl = true; + } else { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i4; + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8; + quantize_a_4row_i8 = spacemit_kernels::rvv::quantize_a_4row_i8; + block_stride_a = spacemit_kernels::q8_blk_size(a_blk_len, true); + set_kernel_impl = true; + } + } else if constexpr (std::is_same_v) { + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8k; + quantize_a_4row_i8 = spacemit_kernels::rvv::quantize_a_4row_i8k; + block_stride_a = spacemit_kernels::q8k_blk_size(a_blk_len); + + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i2k; + set_kernel_impl = true; + } else if constexpr (std::is_same_v) { + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8k; + quantize_a_4row_i8 = spacemit_kernels::rvv::quantize_a_4row_i8k; + block_stride_a = spacemit_kernels::q8k_blk_size(a_blk_len); + + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i3k; + set_kernel_impl = true; + } else if constexpr (std::is_same_v) { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8mxfp4; + set_kernel_impl = true; + } else if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i5; + set_kernel_impl = true; + } } - } +#endif - for (int i = 0; i < 16; i++) { - // [16, 31], in.d & 0xF0 - for (int j = 0; j < QK4_0 / 4; j++) { - //src [b0 b16] ......... [b8 b24] ......... [b15 b31] - //dst [b16 b24] ......... [b23 b31] - out.qs[4 * QK4_0 + i * QK4_0 / 4 + j] = ((in[i].qs[j] & 0xF0) >> 4) | (in[i].qs[j + QK4_0 / 4] & 0xF0); +#if defined(RISCV64_SPACEMIT_IME1) + if (!set_kernel_impl && (global_spine_env_info.use_ime1)) { + quantize_a_row_i8 = spacemit_kernels::ime1::quantize_a_row_i8; + quantize_a_4row_i8 = spacemit_kernels::ime1::quantize_a_4row_i8; + + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + gemm_kernel = spacemit_kernels::ime1::gemm_kernel_i8i4; + set_kernel_impl = true; + } else if constexpr (std::is_same_v) { + gemm_kernel = spacemit_kernels::ime1::gemm_kernel_i8i8; + set_kernel_impl = true; + } + } +#endif + if (!set_kernel_impl) { + GGML_ABORT("no kernel implementation found for the block type"); } - } - return out; -} + const int64_t a_k_blks = spacemit_kernels::div_round_up(gemm_k, a_blk_len); + const int64_t b_k_blks = spacemit_kernels::div_round_up(gemm_k, b_blk_len); -static block_q4_1x16 make_block_q4_1x16(block_q4_1 * in, unsigned int blck_size_interleave) { - block_q4_1x16 out; - GGML_ASSERT(QK4_1 / blck_size_interleave == 2); - - for (int i = 0; i < 16; i++) { - float d = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d); - float m = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m); - float mid = -std::nearbyintf(m / d); - mid = std::min(15.0f, std::max(0.0f, mid)); - out.d[i] = GGML_FP32_TO_FP16(d); - out.zp[i] = static_cast(mid); - } + const int64_t row_stride_a = a_k_blks * block_stride_a; + const int64_t gemm_workspace_size = GGML_PAD(gemm_m * row_stride_a, alignof(int64_t)); - for (int i = 0; i < 16; i++) { - // [0, 15], in.d & 0x0F - for (int j = 0; j < QK4_1 / 4; j++) { - //src [b0 b16] ......... [b8 b24] ......... [b15 b31] - //dst [b0 b8] ......... [b7 b15] - out.qs[i * QK4_1 / 4 + j] = (in[i].qs[j] & 0x0F) | ((in[i].qs[j + QK4_1 / 4] & 0x0F) << 4); + if (ith == 0 && params->wsize < gemm_workspace_size) { + GGML_ABORT("wsize less than gemm_workspace_size"); } - } - for (int i = 0; i < 16; i++) { - // [16, 31], in.d & 0xF0 - for (int j = 0; j < QK4_1 / 4; j++) { - //src [b0 b16] ......... [b8 b24] ......... [b15 b31] - //dst [b16 b24] ......... [b23 b31] - out.qs[4 * QK4_1 + i * QK4_1 / 4 + j] = ((in[i].qs[j] & 0xF0) >> 4) | (in[i].qs[j + QK4_1 / 4] & 0xF0); - } - } + uintptr_t ws_ptr = reinterpret_cast(params->wdata); - return out; -} + void * tcm_buffer = ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer; + const int64_t tcm_buffer_size = ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer_size; -static int repack_q4_0_to_q4_0_16_bl(struct ggml_tensor * t, - int interleave_block, - const void * GGML_RESTRICT data, - size_t data_size) { - GGML_ASSERT(t->type == GGML_TYPE_Q4_0); - GGML_ASSERT(interleave_block == 16); + auto * quant_a_buffer = reinterpret_cast(ws_ptr); - constexpr int nrows_interleaved = 16; + constexpr int64_t row_align = 4; + const int64_t row_blks = spacemit_kernels::div_round_up(gemm_m, row_align); - block_q4_0x16 * dst = (block_q4_0x16 *) t->data; - const block_q4_0 * src = (const block_q4_0 *) data; - block_q4_0 dst_tmp[16]; - int nrow = ggml_nrows(t); - int nblocks = t->ne[0] / QK4_0; + const int64_t row_stride_b = b_k_blks * get_repacked_block_type_size(); + const int64_t per_mb_rows_wsize = row_align * row_stride_a; + const int64_t per_nb_cols_wsize = NB_COLS * row_stride_b; - GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0)); + const int64_t barrier_idx = static_cast(ith / 2); - if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_0 != 0) { - return -1; - } + GGML_ASSERT(global_spine_env_info.init_barrier != nullptr); + GGML_ASSERT(barrier_idx < spine_init_barrier_count); + spine_barrier_t * cur_barrier = &global_spine_env_info.init_barrier[barrier_idx]; - for (int b = 0; b < nrow; b += nrows_interleaved) { - for (int64_t x = 0; x < nblocks; x++) { - for (int i = 0; i < nrows_interleaved; i++) { - dst_tmp[i] = src[x + i * nblocks]; + if (gemm_m == 1) { + int task_per_thread = spacemit_kernels::div_round_up(a_k_blks, nth); + int a_blk_start = ith * task_per_thread; + int a_blk_end = std::min(a_blk_start + task_per_thread, (int) a_k_blks); + if (a_blk_start < a_blk_end) { + quantize_a_row_i8(a_blk_len, feature + a_blk_start * a_blk_len, (a_blk_end - a_blk_start) * a_blk_len, + quant_a_buffer + a_blk_start * block_stride_a); + } + } else { + int task_per_thread = spacemit_kernels::div_round_up(row_blks, nth); + int m_row_blk_start = ith * task_per_thread; + int m_row_blk_end = std::min(m_row_blk_start + task_per_thread, (int) row_blks); + for (int m_row_blk = m_row_blk_start; m_row_blk < m_row_blk_end; m_row_blk++) { + int m_idx = m_row_blk * row_align; + int rows_tobe_handled = (gemm_m - m_idx) > row_align ? row_align : (gemm_m - m_idx); + + if (rows_tobe_handled == row_align && quantize_a_4row_i8 != nullptr) { + const float * a_row_ptr = feature + m_idx * gemm_k; + auto * quant_a_row_ptr = quant_a_buffer + m_idx * row_stride_a; + quantize_a_4row_i8(a_blk_len, a_row_ptr, gemm_k, quant_a_row_ptr); + } else { + while (rows_tobe_handled) { + const float * a_row_ptr = feature + m_idx * gemm_k; + auto * quant_a_row_ptr = quant_a_buffer + m_idx * row_stride_a; + quantize_a_row_i8(a_blk_len, a_row_ptr, gemm_k, quant_a_row_ptr); + rows_tobe_handled -= 1; + m_idx += 1; + } + } } - *dst++ = make_block_q4_0x16(dst_tmp, interleave_block); } - src += nrows_interleaved * nblocks; - } - return 0; - GGML_UNUSED(data_size); -} + ggml_barrier(params->threadpool); -static int repack_q4_1_to_q4_1_16_bl(struct ggml_tensor * t, - int interleave_block, - const void * GGML_RESTRICT data, - size_t data_size) { - GGML_ASSERT(t->type == GGML_TYPE_Q4_1); - GGML_ASSERT(interleave_block == 16); + const int64_t gemm_m_stride = gemm_n / gemm_m > 64 ? gemm_m : 16; + const int64_t gemm_m_blocked = spacemit_kernels::div_round_up(gemm_m, gemm_m_stride); + const int64_t max_gemm_n_stride = spacemit_kernels::div_round_up(gemm_n * gemm_m_blocked, nth); - constexpr int nrows_interleaved = 16; + int64_t gemm_n_stride = gemm_n; + if (max_gemm_n_stride < gemm_n) { + gemm_n_stride = + std::min(gemm_n_stride, spacemit_kernels::div_round_up(max_gemm_n_stride, NB_COLS) * NB_COLS); + } - block_q4_1x16 * dst = (block_q4_1x16 *) t->data; - const block_q4_1 * src = (const block_q4_1 *) data; - block_q4_1 dst_tmp[16]; - int nrow = ggml_nrows(t); - int nblocks = t->ne[0] / QK4_1; + if (gemm_n_stride == gemm_n && tcm_buffer != nullptr && per_mb_rows_wsize <= tcm_buffer_size) { + for (int64_t m_start = ith * row_align; m_start < gemm_m; m_start += row_align * nth) { + uint8_t * b_col = reinterpret_cast(w_data); + uint8_t * b_col_zp = block_type_has_zp() ? b_col : nullptr; - GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_1)); + int64_t m_row_real = std::min(gemm_m - m_start, row_align); - if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_1 != 0) { - return -1; - } + spacemit_kernels::rvv::memcpy1d(tcm_buffer, quant_a_buffer + m_start * row_stride_a, + m_row_real * row_stride_a); + + int64_t n_blk_real = 0; + for (int64_t ni = 0; ni < gemm_n; ni += n_blk_real, b_col += n_blk_real * row_stride_b) { + n_blk_real = std::min(gemm_n - ni, (int64_t) NB_COLS); - for (int b = 0; b < nrow; b += nrows_interleaved) { - for (int64_t x = 0; x < nblocks; x++) { - for (int i = 0; i < nrows_interleaved; i++) { - dst_tmp[i] = src[x + i * nblocks]; + uint8_t * a_row_ptr = (uint8_t *) tcm_buffer; + float * c_blk = output + m_start * gemm_n + ni; + + int32_t rows_remaining = m_row_real; + + while (rows_remaining > 0) { + auto rows_handled = gemm_kernel(b_blk_len, a_row_ptr, b_col, b_col_zp, c_blk, rows_remaining, + n_blk_real, b_k_blks, gemm_n); + + c_blk += rows_handled * gemm_n; + a_row_ptr += rows_handled * row_stride_a; + + rows_remaining -= rows_handled; + } + } } - *dst++ = make_block_q4_1x16(dst_tmp, interleave_block); - } - src += nrows_interleaved * nblocks; - } - return 0; + } else if (tcm_buffer != nullptr && per_nb_cols_wsize <= tcm_buffer_size) { + uint8_t * a_row = quant_a_buffer; + uint8_t * b_col = reinterpret_cast(tcm_buffer); + if ((gemm_workspace_size + per_nb_cols_wsize) <= tcm_buffer_size) { + a_row = (uint8_t *) tcm_buffer; + b_col = reinterpret_cast(tcm_buffer) + gemm_workspace_size; + } + uint8_t * b_col_zp = block_type_has_zp() ? b_col : nullptr; - GGML_UNUSED(data_size); -} + int64_t ni = ith * NB_COLS; + int64_t nb_real = std::min(gemm_n - ni, NB_COLS); -static inline void get_scale_min_k4(int j, - const uint8_t * GGML_RESTRICT q, - uint8_t * GGML_RESTRICT d, - uint8_t * GGML_RESTRICT m) { - if (j < 4) { - *d = q[j] & 63; - *m = q[j + 4] & 63; - } else { - *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); - *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); - } -} + if (ith % 2 == 0 && nb_real > 0) { + spacemit_kernels::rvv::memcpy1d(b_col, reinterpret_cast(w_data) + ni * row_stride_b, + nb_real * row_stride_b); + if (a_row != quant_a_buffer) { + spacemit_kernels::rvv::memcpy1d(a_row, quant_a_buffer, gemm_workspace_size); + } + } -static int repack_q4_k_to_q4_1_16_bl(struct ggml_tensor * t, - int interleave_block, - const void * GGML_RESTRICT data, - size_t data_size) { - GGML_ASSERT(t->type == GGML_TYPE_Q4_K); - GGML_ASSERT(interleave_block == 16); - GGML_ASSERT(QK_K / QK4_1 == 8); + spine_barrier_wait(cur_barrier); - constexpr int nrows_interleaved = 16; + if (ith % 2 != 0 && nb_real > 0) { + if (a_row != quant_a_buffer) { + spacemit_kernels::rvv::memcpy1d(a_row, quant_a_buffer, gemm_workspace_size); + } + spacemit_kernels::rvv::memcpy1d(b_col, reinterpret_cast(w_data) + ni * row_stride_b, + nb_real * row_stride_b); + } - block_q4_1x16 * dst = (block_q4_1x16 *) t->data; - const block_q4_K * src = (const block_q4_K *) data; - block_q4_1 dst_tmp[16]; - int nrow = ggml_nrows(t); - int nblocks = t->ne[0] / QK_K; + for (; ni < gemm_n; ni += NB_COLS * nth) { + int64_t rows_remaining = gemm_m; + float * c_blk = output + ni; + auto * a_row_cur = a_row; - if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_K != 0) { - return -1; - } + if (ith % 2 != 0) { + spine_barrier_wait(cur_barrier); + } - for (int b = 0; b < nrow; b += nrows_interleaved) { - for (int64_t x = 0; x < nblocks; x++) { - for (int j = 0; j < 8; j++) { - for (int i = 0; i < nrows_interleaved; i++) { - uint8_t sc, m; - const float d = GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d); - const float min = - GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin); - get_scale_min_k4(j, src[x + i * nblocks].scales, &sc, &m); - const float d1 = d * sc; - const float m1 = min * m; - - dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d = GGML_FP32_TO_FP16(d1); - dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m = GGML_FP32_TO_FP16(-m1); - // src -> [b0, b32] [b1, b33] ... [b31, b63] - // dst -> [b0, b16] [b1, b17] ... [b15, b31] [b32, b48] [b33, b49] ... [b47, b63] - const uint8_t * q = src[x + i * nblocks].qs + (j / 2) * QK4_1; - if (j % 2 == 0) { - for (int ii = 0; ii < 16; ii++) { - dst_tmp[i].qs[ii] = (q[ii] & 0x0F) | ((q[ii + 16] & 0x0F) << 4); - } - } else { - for (int ii = 0; ii < 16; ii++) { - dst_tmp[i].qs[ii] = ((q[ii] & 0xF0) >> 4) | (q[ii + 16] & 0xF0); - } - } + while (rows_remaining > 0) { + auto rows_handled = gemm_kernel(b_blk_len, a_row_cur, b_col, b_col_zp, c_blk, rows_remaining, + nb_real, b_k_blks, gemm_n); + + c_blk += rows_handled * gemm_n; + a_row_cur += rows_handled * row_stride_a; + + rows_remaining -= rows_handled; + } + + if (ith % 2 == 0) { + spine_barrier_wait(cur_barrier); + } + + const int64_t next_ni = ni + NB_COLS * nth; + if (next_ni < gemm_n) { + nb_real = std::min(gemm_n - next_ni, NB_COLS); + spacemit_kernels::rvv::memcpy1d(b_col, reinterpret_cast(w_data) + next_ni * row_stride_b, + nb_real * row_stride_b); } - *dst++ = make_block_q4_1x16(dst_tmp, interleave_block); } - } - src += nrows_interleaved * nblocks; - } - return 0; + } else { + const int64_t task_count_m = spacemit_kernels::div_round_up(gemm_m, gemm_m_stride); + const int64_t task_count_n = spacemit_kernels::div_round_up(gemm_n, gemm_n_stride); - GGML_UNUSED(data_size); -} + int64_t task_count = task_count_m * task_count_n; + int64_t task_per_thread = (task_count + nth - 1) / nth; + int64_t start = ith * task_per_thread; + int64_t end = std::min((ith + 1) * task_per_thread, task_count); + for (int64_t compute_idx = start; compute_idx < end; compute_idx++) { + const auto tid_n = compute_idx / task_count_m; + const auto tid_m = compute_idx % task_count_m; -namespace ggml::cpu::riscv64_spacemit { + const int64_t m_start = tid_m * gemm_m_stride; + const int64_t m_count = std::min(gemm_m - m_start, (int64_t) gemm_m_stride); -template -int repack(struct ggml_tensor *, const void *, size_t); + const int64_t n_start = tid_n * gemm_n_stride; + const int64_t n_count = std::min(gemm_n - n_start, (int64_t) gemm_n_stride); -template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { - return repack_q4_0_to_q4_0_16_bl(t, 16, data, data_size); -} + const int64_t n_blk = m_count == 1 ? n_count : NB_COLS; -template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { - return repack_q4_1_to_q4_1_16_bl(t, 16, data, data_size); -} + uint8_t * b_col = reinterpret_cast(w_data) + n_start * row_stride_b; + uint8_t * b_col_zp = block_type_has_zp() ? b_col : nullptr; -template <> int repack(struct ggml_tensor * t, const void * data, size_t data_size) { - return repack_q4_k_to_q4_1_16_bl(t, 16, data, data_size); -} + int64_t n_blk_real = 0; + for (int64_t ni = 0; ni < n_count; ni += n_blk_real, b_col += n_blk_real * row_stride_b) { + n_blk_real = std::min(n_count - ni, n_blk); -class tensor_traits_base : public ggml::cpu::tensor_traits { - public: - virtual int repack(struct ggml_tensor * t, const void * data, size_t data_size) = 0; -}; + uint8_t * a_row = quant_a_buffer + m_start * row_stride_a; -template class tensor_traits : public tensor_traits_base { - bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override { - switch (op->op) { - case GGML_OP_MUL_MAT: - size = ggml_row_size(GGML_TYPE_Q8_0, ggml_nelements(op->src[1])) * 4; - size = ((size + QK4_0 - 1) / QK4_0) * (QK4_0 * sizeof(float) + sizeof(float)); - return true; - default: - // GGML_ABORT("fatal error"); - break; - } - return false; - } + float * c_blk = output + m_start * gemm_n + n_start + ni; - bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override { - switch (op->op) { - case GGML_OP_MUL_MAT: - if (op->src[0]->type == GGML_TYPE_Q4_0 || // - op->src[0]->type == GGML_TYPE_Q4_1 || // - op->src[0]->type == GGML_TYPE_Q4_K) { - forward_mul_mat_q4(params, op); - return true; + int64_t rows_remaining = m_count; + + uint8_t * b_col_cur = b_col; + uint8_t * b_col_zp_cur = b_col_zp; + + while (rows_remaining > 0) { + auto rows_handled = gemm_kernel(b_blk_len, a_row, b_col_cur, b_col_zp_cur, c_blk, + rows_remaining, n_blk_real, b_k_blks, gemm_n); + + c_blk += rows_handled * gemm_n; + a_row += rows_handled * row_stride_a; + + rows_remaining -= rows_handled; + } } - default: - // GGML_ABORT("fatal error"); - break; + } } - return false; } - void forward_mul_mat_q4(ggml_compute_params * params, ggml_tensor * op) { + void forward_mul_mat_id(ggml_compute_params * params, ggml_tensor * op) { + constexpr size_t a_blk_len = INTER_SIZE; + constexpr size_t b_blk_len = INTER_SIZE; + const ggml_tensor * src0 = op->src[0]; const ggml_tensor * src1 = op->src[1]; + const ggml_tensor * ids = op->src[2]; ggml_tensor * dst = op; GGML_TENSOR_BINARY_OP_LOCALS @@ -429,133 +559,389 @@ template class tensor_ int ith = params->ith; int nth = params->nth; - [[maybe_unused]] const enum ggml_type type = src0->type; + // row groups + const int n_ids = ids->ne[0]; // n_expert_used + const int n_as = ne02; // n_expert + + struct mmid_row_mapping { + int32_t i1; + int32_t i2; + }; + + spacemit_kernels::quantize_a_row_def quantize_a_row_i8; + spacemit_kernels::gemm_kernel_quantize_def gemm_kernel; + spacemit_kernels::moe_gemm_kernel_quantize_def moe_gemm_kernel_m2; + bool set_kernel_impl = false; + size_t block_stride_a = spacemit_kernels::q8_blk_size(QK4_0); + +#if defined(RISCV64_SPACEMIT_IME2) + if (!set_kernel_impl && (global_spine_env_info.use_ime2)) { + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8; + block_stride_a = spacemit_kernels::q8_blk_size(QK4_0, true); + + if constexpr (std::is_same_v || std::is_same_v) { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i8; + set_kernel_impl = true; + } else if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if constexpr (INTER_SIZE == 256) { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i4_hp; + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8_hp; + block_stride_a = spacemit_kernels::q8_hp_blk_size(a_blk_len, true, true); + set_kernel_impl = true; + } else { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i4; + moe_gemm_kernel_m2 = spacemit_kernels::ime2::moe_m2_gemm_kernel_i8i4; + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8; + block_stride_a = spacemit_kernels::q8_blk_size(a_blk_len, true); + set_kernel_impl = true; + } + } else if constexpr (std::is_same_v) { + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8k; + block_stride_a = spacemit_kernels::q8k_blk_size(a_blk_len); + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i2k; + set_kernel_impl = true; + } else if constexpr (std::is_same_v) { + quantize_a_row_i8 = spacemit_kernels::rvv::quantize_a_row_i8k; + block_stride_a = spacemit_kernels::q8k_blk_size(a_blk_len); + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i3k; + set_kernel_impl = true; + } else if constexpr (std::is_same_v) { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8mxfp4; + moe_gemm_kernel_m2 = spacemit_kernels::ime2::moe_m2_gemm_kernel_i8mxfp4; + set_kernel_impl = true; + } else if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + gemm_kernel = spacemit_kernels::ime2::gemm_kernel_i8i5; + moe_gemm_kernel_m2 = spacemit_kernels::ime2::moe_m2_gemm_kernel_i8i5; + set_kernel_impl = true; + } + } +#endif - void * w_data = (void *) src0->data; - const float * feature = (const float *) src1->data; - float * output = (float *) dst->data; +#if defined(RISCV64_SPACEMIT_IME1) + if (!set_kernel_impl && (global_spine_env_info.use_ime1)) { + quantize_a_row_i8 = spacemit_kernels::ime1::quantize_a_row_i8; + + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + gemm_kernel = spacemit_kernels::ime1::gemm_kernel_i8i4; + set_kernel_impl = true; + } else if constexpr (std::is_same_v) { + gemm_kernel = spacemit_kernels::ime1::gemm_kernel_i8i8; + set_kernel_impl = true; + } + } +#endif + if (!set_kernel_impl) { + GGML_ABORT("no kernel implementation found for the block type"); + } - const size_t batch_feature = ne12 * ne13; - [[maybe_unused]] const size_t batch_weight = ne02 * ne03; - const size_t gemm_m = ne11; - const size_t gemm_k = ne10; - const size_t gemm_n = ne01; + const size_t a_k_blks = spacemit_kernels::div_round_up(ne10, a_blk_len); + const size_t b_k_blks = spacemit_kernels::div_round_up(ne10, b_blk_len); - GGML_ASSERT(batch_weight == 1); + const size_t nbw1 = a_k_blks * block_stride_a; + const size_t nbw2 = ne11 * nbw1; + const size_t nbw3 = nbw2 * ne12; + const size_t gemm_workspace_size = GGML_PAD(nbw3, alignof(int64_t)); - const size_t block_count_k = div_round_up(gemm_k, QK4_0); - const size_t per_gemm_workspace_size = gemm_m * block_count_k * q8_blk_size(QK4_0); - const size_t per_gemm_workspace_stride = - div_round_up(per_gemm_workspace_size, alignof(uint64_t)) * alignof(uint64_t); - const size_t gemm_workspace_size = batch_feature * per_gemm_workspace_stride; - const size_t desired_wsize = gemm_workspace_size + alignof(uint64_t) - 1; + const uintptr_t ws_ptr = reinterpret_cast(params->wdata); + auto * quant_a_buffer = reinterpret_cast(ws_ptr); - if (ith == 0 && params->wsize < desired_wsize) { - throw std::runtime_error("wsize less than desired_wsize"); + if (ne11 == 1) { + for (int64_t ii = ith; ii < ne12 * a_k_blks; ii += nth) { + int64_t i12 = ii / a_k_blks; + int64_t ak_blk_id = ii % a_k_blks; + quantize_a_row_i8(a_blk_len, (float *) ((char *) src1->data + i12 * nb12) + ak_blk_id * a_blk_len, + a_blk_len, quant_a_buffer + i12 * nbw2 + ak_blk_id * block_stride_a); + } + } else { + for (int64_t ii = ith; ii < ne12 * ne11; ii += nth) { + int64_t i12 = ii / ne11; + int64_t i11 = ii % ne11; + quantize_a_row_i8(a_blk_len, (float *) ((char *) src1->data + i12 * nb12 + i11 * nb11), ne10, + quant_a_buffer + i12 * nbw2 + i11 * nbw1); + } } - std::vector qnbitgemm_args(batch_feature); +#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) *ne12 + (i1)] - for (size_t i = 0; i < batch_feature; i++) { - qnbitgemm_args[i].a_ptr = feature + gemm_m * gemm_k * i; - qnbitgemm_args[i].lda = gemm_k; - qnbitgemm_args[i].packed_quant_b_data = (const std::byte *) w_data; - qnbitgemm_args[i].quant_b_scale = nullptr; + int64_t * matrix_row_counts = (int64_t *) (ws_ptr + gemm_workspace_size); + int32_t * valid_ep_count = (int32_t *) (matrix_row_counts + n_as); + int32_t * valid_act_count = (int32_t *) (valid_ep_count + 1); + int64_t * valid_matrix_row_counts = (int64_t *) (valid_act_count + 1); + mmid_row_mapping * matrix_rows = (mmid_row_mapping *) (valid_matrix_row_counts + n_as); - if constexpr (std::is_same_v) { - qnbitgemm_args[i].quant_b_zp = nullptr; - } else { - qnbitgemm_args[i].quant_b_zp = w_data; - } + if (ith == 0) { + // initialize matrix_row_counts + memset(matrix_row_counts, 0, n_as * sizeof(int64_t)); - qnbitgemm_args[i].bias = nullptr; - qnbitgemm_args[i].c_ptr = output + gemm_m * gemm_n * i; - qnbitgemm_args[i].ldc = gemm_n; - } + // group rows by src0 matrix + for (int32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { + for (int32_t id = 0; id < n_ids; ++id) { + const int32_t i02 = + *(const int32_t *) ((const char *) ids->data + iid1 * ids->nb[1] + id * ids->nb[0]); - const uintptr_t ws_ptr = reinterpret_cast(params->wdata); - void * ws = reinterpret_cast((ws_ptr + alignof(uint64_t) - 1) & (~(alignof(uint64_t) - 1))); - const size_t quant_a_stride = block_count_k * q8_blk_size(QK4_0); + GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as)); - { - constexpr size_t block_size_m = 4; - size_t per_gemm_block_count_m = div_round_up(gemm_m, block_size_m); - int32_t task_count = batch_feature * per_gemm_block_count_m; - int32_t task_per_thread = (task_count + nth - 1) / nth; - int32_t start = ith * task_per_thread; - int32_t end = std::min((ith + 1) * task_per_thread, task_count); - for (int32_t compute_idx = start; compute_idx < end; compute_idx++) { - int32_t gemm_idx = compute_idx / per_gemm_block_count_m; - int32_t block_idx_in_gemm = compute_idx % per_gemm_block_count_m; - int32_t m_idx = block_idx_in_gemm * block_size_m; - const qnbitgemm_spacemit_ime_args & data = qnbitgemm_args[gemm_idx]; - int32_t rows_tobe_handled = (gemm_m - m_idx) > block_size_m ? block_size_m : (gemm_m - m_idx); - - if (rows_tobe_handled == block_size_m) { - const float * a_row_ptr = data.a_ptr + m_idx * data.lda; - std::byte * quant_a_row_ptr = - static_cast(ws) + gemm_idx * per_gemm_workspace_stride + m_idx * quant_a_stride; - sqnbitgemm_spacemit_ime::ime1::quantize_a_4row_i8(QK4_0, a_row_ptr, gemm_k, quant_a_row_ptr); - } else { - while (rows_tobe_handled) { - const float * a_row_ptr = data.a_ptr + m_idx * data.lda; - std::byte * quant_a_row_ptr = static_cast(ws) + - gemm_idx * per_gemm_workspace_stride + m_idx * quant_a_stride; - sqnbitgemm_spacemit_ime::ime1::quantize_a_row_i8(QK4_0, a_row_ptr, gemm_k, quant_a_row_ptr); - rows_tobe_handled -= 1; - m_idx += 1; + if (i02 == -1) { + memset((char *) dst->data + id * nb1 + iid1 * nb2, 0, ne0 * sizeof(float)); + continue; } + + MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = { id, iid1 }; + matrix_row_counts[i02] += 1; } } + + int32_t valid_ep_count_t = 0; + int32_t valid_act_count_t = 0; + for (int cur_a = 0; cur_a < n_as; ++cur_a) { + const int64_t cne1 = matrix_row_counts[cur_a]; + if (cne1 == 0) { + continue; + } + valid_matrix_row_counts[valid_ep_count_t] = cur_a; + valid_act_count_t += cne1; + valid_ep_count_t += 1; + } + valid_ep_count[0] = valid_ep_count_t; + valid_act_count[0] = valid_act_count_t; } + const int64_t barrier_idx = static_cast(ith / 2); + + GGML_ASSERT(global_spine_env_info.init_barrier != nullptr); + GGML_ASSERT(barrier_idx < spine_init_barrier_count); + spine_barrier_t * cur_barrier = &global_spine_env_info.init_barrier[barrier_idx]; + ggml_barrier(params->threadpool); - if (ith >= ggml::cpu::riscv64_spacemit::num_ai_cores) { - return; - } - nth = std::min(nth, int{ ggml::cpu::riscv64_spacemit::num_ai_cores }); - - size_t threads_per_gemm = nth / batch_feature; - constexpr size_t gemm_m_stride = 128; - size_t nc = gemm_n; - const size_t gemm_m_blocked = div_round_up(gemm_m, gemm_m_stride); - const size_t max_nc = div_round_up(gemm_n * gemm_m_blocked, threads_per_gemm); - if (max_nc < nc) { - nc = std::min(nc, div_round_up(max_nc, QGEMM_STRIDEN_THREAD_ALIGN) * QGEMM_STRIDEN_THREAD_ALIGN); - } - const size_t gemm_n_stride = nc; - const size_t thread_count_m = div_round_up(gemm_m, gemm_m_stride); - const size_t thread_count_n = div_round_up(gemm_n, gemm_n_stride); - threads_per_gemm = thread_count_m * thread_count_n; + const size_t row_stride_b = b_k_blks * get_repacked_block_type_size(); + const size_t expert_b_stride = ne01 * row_stride_b; + const size_t per_nb_cols_wsize = NB_COLS * row_stride_b; - { - int task_count = batch_feature * threads_per_gemm; - int task_per_thread = (task_count + nth - 1) / nth; - int start = ith * task_per_thread; - int end = std::min((ith + 1) * task_per_thread, task_count); - for (int compute_idx = start; compute_idx < end; compute_idx++) { - const auto gemm_i = compute_idx / threads_per_gemm; - const auto blk_i = compute_idx % threads_per_gemm; - const auto * data = &qnbitgemm_args[gemm_i]; + std::array src_workspaces; + std::array dst_workspaces; + + auto * tcm_buffer = ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer; + const auto tcm_buffer_size = ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer_size; + + const auto valid_ep_count_t = valid_ep_count[0]; + const auto valid_act_count_t = valid_act_count[0]; + + int nth_es = 1; + int nth_n = nth; - const auto tid_n = blk_i / thread_count_m; - const auto tid_m = blk_i % thread_count_m; + int ith_es = ith % nth_es; + int ith_n = (ith / nth_es) % nth_n; - const size_t m_start = tid_m * gemm_m_stride; - const size_t m_count = std::min(gemm_m - m_start, (size_t) gemm_m_stride); + if (valid_ep_count_t % nth == 0 && tcm_buffer != nullptr && valid_ep_count_t == n_as && + valid_act_count_t == n_as && per_nb_cols_wsize <= tcm_buffer_size) { + for (int64_t valid_id = ith; valid_id < valid_ep_count_t; valid_id += nth) { + const int64_t cur_a = valid_matrix_row_counts[valid_id]; - const size_t n_start = tid_n * gemm_n_stride; - const size_t n_count = std::min(gemm_n - n_start, (size_t) gemm_n_stride); + auto * src0_cur = (uint8_t *) src0->data + cur_a * expert_b_stride; - void * per_gemm_ws = reinterpret_cast(ws) + gemm_i * per_gemm_workspace_stride; + mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, 0); + const int id = row_mapping.i1; + const int64_t i11 = id % ne11; + const int64_t i12 = row_mapping.i2; + const int64_t i1 = id; + const int64_t i2 = i12; + + auto * src1_col = quant_a_buffer + (i11 * nbw1 + i12 * nbw2); + float * c_blk = (float *) ((char *) dst->data + (i1 * nb1 + i2 * nb2)); + + uint8_t * a_row = src1_col; + uint8_t * b_col = reinterpret_cast(tcm_buffer); + if ((nbw1 + per_nb_cols_wsize) <= tcm_buffer_size) { + a_row = (uint8_t *) tcm_buffer; + b_col = reinterpret_cast(tcm_buffer) + nbw1; + } + uint8_t * b_col_zp = block_type_has_zp() ? b_col : nullptr; - sqnbitgemm_spacemit_ime_i8i4(QK4_0, gemm_k, data, per_gemm_ws, m_start, m_count, n_start, n_count); + if (ith % 2 == 0) { + spacemit_kernels::rvv::memcpy1d(b_col, reinterpret_cast(src0_cur), per_nb_cols_wsize); + + if (a_row != src1_col) { + spacemit_kernels::rvv::memcpy1d(a_row, src1_col, nbw1); + } + } + + spine_barrier_wait(cur_barrier); + + if (ith % 2 != 0) { + if (a_row != src1_col) { + spacemit_kernels::rvv::memcpy1d(a_row, src1_col, nbw1); + } + + spacemit_kernels::rvv::memcpy1d(b_col, reinterpret_cast(src0_cur), per_nb_cols_wsize); + } + + int64_t nb_real = std::min(ne01, NB_COLS); + for (int64_t ni = 0; ni < ne01; ni += NB_COLS) { + if (ith % 2 != 0) { + spine_barrier_wait(cur_barrier); + } + + gemm_kernel(b_blk_len, a_row, b_col, b_col_zp, c_blk + ni, 1, nb_real, b_k_blks, ne01); + + if (ith % 2 == 0) { + spine_barrier_wait(cur_barrier); + } + + const int64_t next_ni = ni + NB_COLS; + if (next_ni < ne01) { + nb_real = std::min(ne01 - next_ni, NB_COLS); + spacemit_kernels::rvv::memcpy1d( + b_col, reinterpret_cast(src0_cur) + next_ni * row_stride_b, per_nb_cols_wsize); + } + } + } + } else { + for (int64_t valid_id = ith_es; valid_id < valid_ep_count_t; valid_id += nth_es) { + const int64_t cur_a = valid_matrix_row_counts[valid_id]; + const int64_t cne1 = matrix_row_counts[cur_a]; + + int64_t src1_cur_start = 0; + int64_t src1_cur_end = cne1; + + int64_t src0_cur_start = (ith_n * ne01) / nth_n; + int64_t src0_cur_end = MIN(((ith_n + 1) * ne01) / nth_n, ne01); + + if (src1_cur_start >= src1_cur_end || src0_cur_start >= src0_cur_end) { + continue; + } + + src0_cur_start = + (src0_cur_start % NB_COLS) ? src0_cur_start + NB_COLS - (src0_cur_start % NB_COLS) : src0_cur_start; + src0_cur_end = + (src0_cur_end % NB_COLS) ? src0_cur_end + NB_COLS - (src0_cur_end % NB_COLS) : src0_cur_end; + + auto * src0_cur = (uint8_t *) src0->data + cur_a * expert_b_stride + src0_cur_start * row_stride_b; + uint8_t * b_col_zp = block_type_has_zp() ? src0_cur : nullptr; + + size_t extra_tcm_buffer_size = tcm_buffer_size; + void * extra_tcm_buffer = tcm_buffer; + if (tcm_buffer != nullptr && (src1_cur_end - src1_cur_start) >= 4 && + (src0_cur_end - src0_cur_start) * row_stride_b <= tcm_buffer_size) { + spacemit_kernels::rvv::memcpy1d(tcm_buffer, src0_cur, + (src0_cur_end - src0_cur_start) * row_stride_b); + src0_cur = reinterpret_cast(tcm_buffer); + b_col_zp = block_type_has_zp() ? src0_cur : nullptr; + extra_tcm_buffer_size -= (src0_cur_end - src0_cur_start) * row_stride_b; + extra_tcm_buffer = reinterpret_cast(reinterpret_cast(tcm_buffer) + + (src0_cur_end - src0_cur_start) * row_stride_b); + } + + int ir1 = src1_cur_start; + + if (extra_tcm_buffer_size >= nbw1 && extra_tcm_buffer != nullptr) { + int64_t quant_a_tile_size = extra_tcm_buffer_size / nbw1; + do { + quant_a_tile_size = MIN(quant_a_tile_size, src1_cur_end - ir1); + + uint8_t * quant_a_tile_buffer = reinterpret_cast(extra_tcm_buffer); + + int iir1 = ir1; + for (; iir1 < (ir1 + quant_a_tile_size); ++iir1) { + mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, iir1); + + const int id = row_mapping.i1; // selected expert index + + const int64_t i11 = id % ne11; + const int64_t i12 = row_mapping.i2; // row index in src1 + + auto * src1_col = quant_a_buffer + (i11 * nbw1 + i12 * nbw2); + spacemit_kernels::rvv::memcpy1d(quant_a_tile_buffer, src1_col, nbw1); + quant_a_tile_buffer = quant_a_tile_buffer + nbw1; + } + + quant_a_tile_buffer = reinterpret_cast(extra_tcm_buffer); + iir1 = ir1; + + if (moe_gemm_kernel_m2 != nullptr) { + for (; iir1 < (ir1 + quant_a_tile_size - 1); iir1 += 2, quant_a_tile_buffer += 2 * nbw1) { + mmid_row_mapping row_mapping_0 = MMID_MATRIX_ROW(cur_a, iir1); + mmid_row_mapping row_mapping_1 = MMID_MATRIX_ROW(cur_a, iir1 + 1); + + src_workspaces[0] = quant_a_tile_buffer; + src_workspaces[1] = quant_a_tile_buffer + nbw1; + + dst_workspaces[0] = + (float *) ((char *) dst->data + (row_mapping_0.i1 * nb1 + row_mapping_0.i2 * nb2)) + + src0_cur_start; + dst_workspaces[1] = (float *) ((char *) dst->data + + ((row_mapping_1.i1) * nb1 + (row_mapping_1.i2) * nb2)) + + src0_cur_start; + moe_gemm_kernel_m2(b_blk_len, src_workspaces.data(), src0_cur, b_col_zp, + dst_workspaces.data(), 1, src0_cur_end - src0_cur_start, b_k_blks, + ne01); + } + } + + for (; iir1 < (ir1 + quant_a_tile_size); iir1++, quant_a_tile_buffer += nbw1) { + mmid_row_mapping row_mapping_0 = MMID_MATRIX_ROW(cur_a, iir1); + + gemm_kernel( + b_blk_len, quant_a_tile_buffer, src0_cur, b_col_zp, + (float *) ((char *) dst->data + (row_mapping_0.i1 * nb1 + row_mapping_0.i2 * nb2)) + + src0_cur_start, + 1, src0_cur_end - src0_cur_start, b_k_blks, ne01); + } + + ir1 += quant_a_tile_size; + } while (ir1 < src1_cur_end); + } else { + if (moe_gemm_kernel_m2 != nullptr) { + for (; ir1 < src1_cur_end - 1; ir1 += 2) { + for (int iir1 = 0; iir1 < 2; ++iir1) { + mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, ir1 + iir1); + + const int id = row_mapping.i1; // selected expert index + + const int64_t i11 = id % ne11; + const int64_t i12 = row_mapping.i2; // row index in src1 + + const int64_t i1 = id; // selected expert index + const int64_t i2 = i12; // row + + src_workspaces[iir1] = quant_a_buffer + (i11 * nbw1 + i12 * nbw2); + + dst_workspaces[iir1] = + (float *) ((char *) dst->data + (i1 * nb1 + i2 * nb2)) + src0_cur_start; + } + + moe_gemm_kernel_m2(b_blk_len, src_workspaces.data(), src0_cur, b_col_zp, + dst_workspaces.data(), 1, src0_cur_end - src0_cur_start, b_k_blks, ne01); + } + } + + for (; ir1 < src1_cur_end; ir1++) { + mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, ir1); + + const int id = row_mapping.i1; // selected expert index + + const int64_t i11 = id % ne11; + const int64_t i12 = row_mapping.i2; // row index in src1 + + const int64_t i1 = id; // selected expert index + const int64_t i2 = i12; // row + + auto * src1_col = quant_a_buffer + (i11 * nbw1 + i12 * nbw2); + + gemm_kernel(b_blk_len, src1_col, src0_cur, b_col_zp, + (float *) ((char *) dst->data + (i1 * nb1 + i2 * nb2)) + src0_cur_start, 1, + src0_cur_end - src0_cur_start, b_k_blks, ne01); + } + } } } +#undef MMID_MATRIX_ROW } - int repack(struct ggml_tensor * t, const void * data, size_t data_size) override { + int repack(ggml_tensor * t, const void * data, size_t data_size) override { GGML_LOG_DEBUG("%s: repack tensor %s with %s_%dx%d\n", __func__, t->name, ggml_type_name(t->type), (int) NB_COLS, (int) INTER_SIZE); return ggml::cpu::riscv64_spacemit::repack(t, data, data_size); @@ -563,309 +949,471 @@ template class tensor_ }; class tensor_traits_common : public tensor_traits_base { - bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override { + bool work_size(int n_threads, const ggml_tensor * op, size_t & size) override { switch (op->op) { - case GGML_OP_NORM: - case GGML_OP_RMS_NORM: - size = 0; + case GGML_OP_FLASH_ATTN_EXT: + { + const int n_tasks = n_threads; + const int64_t neq2 = op->src[0]->ne[2]; // number of query heads + const int64_t DK = op->src[1]->ne[0]; + const int64_t DV = op->src[2]->ne[0]; // DV + + // Tiled flash attention scratch (tile sizes defined in common.h) + // Per-thread: Q_q + KQ + mask + VKQ32 + V32 + K_f32 + padding + size_t prefill = sizeof(float) * + (GGML_FA_TILE_Q * DK + 2 * GGML_FA_TILE_Q * GGML_FA_TILE_KV + GGML_FA_TILE_Q * DV + + GGML_FA_TILE_KV * DV + GGML_FA_TILE_KV * DK) * + n_tasks; + + // Decode path: n_kv_chunks = n_tasks (one chunk per thread) + // Per-thread: VKQ accmulator (DV), partial M, partial S + intra-thread scratch for V, Q and VKQ + size_t n_chunks = n_tasks; + size_t decode = sizeof(float) * (neq2 * n_chunks * (2 + DV) + n_tasks * (DK + 2 * DV)); + + size = MAX(prefill, decode); + } return true; default: - // GGML_ABORT("fatal error"); break; } return false; } - bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override { + bool compute_forward(ggml_compute_params * params, ggml_tensor * op) override { switch (op->op) { case GGML_OP_NORM: - forward_norm_f32(params, op); - return true; + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_norm_f32(params, op); + return true; + default: + GGML_ABORT("fatal error"); + } case GGML_OP_RMS_NORM: - forward_rms_norm_f32(params, op); + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_rms_norm_f32(params, op); + return true; + default: + GGML_ABORT("fatal error"); + } + case GGML_OP_ADD: + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_binary(params, op); + return true; + case GGML_TYPE_F16: + spacemit_kernels::rvv::forward_binary(params, op); + return true; + default: + ggml_compute_forward_add(params, op); + return true; + } + case GGML_OP_SUB: + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_binary(params, op); + return true; + case GGML_TYPE_F16: + spacemit_kernels::rvv::forward_binary(params, op); + return true; + default: + ggml_compute_forward_sub(params, op); + return true; + } + case GGML_OP_MUL: + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_binary(params, op); + return true; + case GGML_TYPE_F16: + spacemit_kernels::rvv::forward_binary(params, op); + return true; + default: + ggml_compute_forward_mul(params, op); + return true; + } + case GGML_OP_DIV: + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_binary(params, op); + return true; + case GGML_TYPE_F16: + spacemit_kernels::rvv::forward_binary(params, op); + return true; + default: + ggml_compute_forward_div(params, op); + return true; + } + case GGML_OP_FLASH_ATTN_EXT: + forward_flash_attn_ext_f16(params, op); return true; + case GGML_OP_CONT: + { + const ggml_tensor * src0 = op->src[0]; + if (op->type == src0->type && op->nb[0] != src0->nb[0] && op->nb[0] == src0->nb[1] && + op->ne[3] * op->ne[2] * op->nb[2] == src0->ne[3] * src0->ne[2] * src0->nb[2]) { + spacemit_kernels::rvv::forward_cont_with_permute(params, op); + } else { + ggml_compute_forward_cont(params, op); + } + return true; + } + case GGML_OP_CPY: + { + const ggml_tensor * src0 = op->src[0]; + if (op->type == src0->type && op->nb[0] == src0->nb[1] && src0->nb[0] != src0->nb[1] && + ggml_nelements(src0) == ggml_nelements(op)) { + spacemit_kernels::rvv::forward_cpy_with_permute(params, op); + } else { + ggml_compute_forward_cpy(params, op); + } + return true; + } + case GGML_OP_REPEAT: + { + const bool rows_equal = ggml_nrows(op->src[0]) == ggml_nrows(op); + const bool broadcast_or_equal = op->src[0]->ne[0] == 1 || op->src[0]->ne[0] == op->ne[0]; + + if (rows_equal && broadcast_or_equal) { + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_repeat_nrows(params, op); + return true; + case GGML_TYPE_F16: + spacemit_kernels::rvv::forward_repeat_nrows(params, op); + return true; + default: + break; + } + } + + if (op->src[0]->ne[1] == 1 && op->src[0]->ne[0] == op->ne[0]) { + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_repeat_dim1(params, op); + return true; + case GGML_TYPE_F16: + spacemit_kernels::rvv::forward_repeat_dim1(params, op); + return true; + default: + break; + } + } + + ggml_compute_forward_repeat(params, op); + } + return true; + case GGML_OP_SUM_ROWS: + { + if (op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) { + spacemit_kernels::rvv::forward_sum_rows(params, op); + } else { + ggml_compute_forward_sum_rows(params, op); + } + } + return true; + case GGML_OP_GET_ROWS: + { + if (op->src[0]->type == op->type) { + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_get_rows(params, op); + return true; + case GGML_TYPE_F16: + spacemit_kernels::rvv::forward_get_rows(params, op); + return true; + default: + break; + } + } + + ggml_compute_forward_get_rows(params, op); + } + return true; + case GGML_OP_CONCAT: + { + const int32_t dim = ggml_get_op_params_i32(op, 0); + if (dim == 0 && op->type == op->src[0]->type) { + switch (op->src[0]->type) { + case GGML_TYPE_F32: + spacemit_kernels::rvv::forward_concat(params, op); + return true; + case GGML_TYPE_F16: + spacemit_kernels::rvv::forward_concat(params, op); + return true; + default: + break; + } + } + + ggml_compute_forward_concat(params, op); + } + return true; + // TODO For GGML_OP_GATED_DELTA_NET + // case GGML_OP_GATED_DELTA_NET: + // return true; default: - // GGML_ABORT("fatal error"); break; } return false; } - void forward_norm_f32(ggml_compute_params * params, ggml_tensor * op) { - const ggml_tensor * src0 = op->src[0]; - ggml_tensor * dst = op; - GGML_ASSERT(ggml_are_same_shape(src0, dst)); - GGML_ASSERT(src0->nb[0] == sizeof(float)); + void forward_flash_attn_ext_f16(const ggml_compute_params * params, ggml_tensor * dst) { + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * v = dst->src[2]; + + GGML_TENSOR_LOCALS(int64_t, neq, q, ne) + GGML_TENSOR_LOCALS(size_t, nbq, q, nb) + GGML_TENSOR_LOCALS(int64_t, nek, k, ne) + GGML_TENSOR_LOCALS(size_t, nbk, k, nb) + GGML_TENSOR_LOCALS(int64_t, nev, v, ne) + GGML_TENSOR_LOCALS(size_t, nbv, v, nb) + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + + const int64_t DK = nek0; + const int64_t DV = nev0; + + const bool supported_prec = (dst->op_params[3] == GGML_PREC_F32 || dst->op_params[3] == GGML_PREC_DEFAULT); + const bool supported_types = (q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16); + const bool supported_shape = (DK > 0 && DK <= 128 && DV > 0 && DV <= 128); + const bool supported_vlen = (__riscv_vlenb() == 128); + + if (!(supported_prec && supported_types && supported_shape && supported_vlen)) { + ggml_compute_forward_flash_attn_ext(params, dst); + return; + } + + // total rows in q + const int64_t nr = neq1 * neq2 * neq3; + // rows per thread const int ith = params->ith; const int nth = params->nth; - GGML_TENSOR_UNARY_OP_LOCALS + static constexpr int64_t Q_TILE_SZ = ggml_fa_tile_config::Q; + const bool use_tiled = !params->use_ref && (neq1 >= Q_TILE_SZ); - float epsilon; - memcpy(&epsilon, dst->op_params, sizeof(float)); + // 4x chunks per thread + // int nth_scaled = nth * 4; + // int64_t chunk_size = (nr + nth_scaled - 1) / nth_scaled; + // int64_t nchunk = (nr + chunk_size - 1) / chunk_size; - GGML_ASSERT(epsilon > 0.0f); + // if (nth == 1 || nchunk < nth) { + // nchunk = nth; + // } - auto * input = (float *) src0->data; - auto * output = (float *) dst->data; + int64_t nchunk = nth; - const auto hidden_size = ne00; - const auto task_count = ne01 * ne02 * ne03; - const auto task_per_thread = (task_count + nth - 1) / nth; - - const auto task_begin = ith * task_per_thread; - const auto task_end = std::min((ith + 1) * task_per_thread, task_count); + if (ith == 0) { + // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start. + ggml_threadpool_chunk_set(params->threadpool, nth); + } - for (auto task_idx = task_begin; task_idx < task_end; task_idx++) { - auto offset = task_idx * hidden_size; - auto * p_input = const_cast(input + offset); + ggml_barrier(params->threadpool); - auto * p_output = output + offset; - auto * p_temp_output = p_output; - auto * p_gamma_data = (const float *) nullptr; - auto * p_beta_data = (const float *) nullptr; - size_t gvl = __riscv_vsetvlmax_e32m4(); - vfloat32m4_t sum = __riscv_vfmv_v_f_f32m4(0.f, gvl); - vfloat32m4_t sum_sq = __riscv_vfmv_v_f_f32m4(0.f, gvl); - int64_t length = hidden_size; - while (length > 0) { - gvl = __riscv_vsetvl_e32m4(length); - // load data - vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_input, gvl); + // The number of elements in each chunk + const int64_t dr = (nr + nchunk - 1) / nchunk; - sum = __riscv_vfadd_vv_f32m4(sum, src_data, gvl); - sum_sq = __riscv_vfmacc_vv_f32m4(sum_sq, src_data, src_data, gvl); + // The first chunk comes from our thread_id, the rest will get auto-assigned. + int current_chunk = ith; - __riscv_vse32_v_f32m4(p_temp_output, src_data, gvl); + while (current_chunk < nchunk) { + const int64_t ir0 = dr * current_chunk; + const int64_t ir1 = MIN(ir0 + dr, nr); - p_input += gvl; - p_temp_output += gvl; - length -= gvl; + if (use_tiled) { + spacemit_kernels::rvv::forward_flash_attn_ext_f16_tiled_vlen1024_vf16( + params, dst, ir0, ir1, ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer, + ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer_size); + } else { + spacemit_kernels::rvv::forward_flash_attn_ext_f16_one_chunk_vlen1024_vf16( + params, dst, ir0, ir1, ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer, + ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer_size); } - gvl = __riscv_vsetvlmax_e32m1(); - - float mean = 0.f; - vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.f, gvl); - vfloat32m1_t mean_v = - __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum, 0), __riscv_vget_v_f32m4_f32m1(sum, 1), gvl); - mean_v = __riscv_vfadd_vv_f32m1(mean_v, __riscv_vget_v_f32m4_f32m1(sum, 2), gvl); - mean_v = __riscv_vfadd_vv_f32m1(mean_v, __riscv_vget_v_f32m4_f32m1(sum, 3), gvl); - mean_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_v, zero_v, gvl); - mean = __riscv_vfmv_f_s_f32m1_f32(mean_v); - mean /= hidden_size; - - vfloat32m1_t mean_square_v = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum_sq, 0), - __riscv_vget_v_f32m4_f32m1(sum_sq, 1), gvl); - mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 2), gvl); - mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 3), gvl); - mean_square_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_square_v, zero_v, gvl); - - float mean_square = __riscv_vfmv_f_s_f32m1_f32(mean_square_v); - mean_square /= hidden_size; - mean_square = sqrt(mean_square - mean * mean + epsilon); - - mean_square = 1.0f / mean_square; - length = hidden_size; - p_temp_output = p_output; - - if (p_gamma_data == nullptr && p_beta_data == nullptr) { - while (length > 0) { - gvl = __riscv_vsetvl_e32m4(length); - vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl); - src_data = __riscv_vfsub_vf_f32m4(src_data, mean, gvl); - src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl); - __riscv_vse32_v_f32m4(p_output, src_data, gvl); - p_temp_output += gvl; - p_output += gvl; - length -= gvl; - } - } else if (p_beta_data == nullptr) { - while (length > 0) { - gvl = __riscv_vsetvl_e32m4(length); - vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl); - vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl); - src_data = __riscv_vfsub_vf_f32m4(src_data, mean, gvl); - src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl); - src_data = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl); - __riscv_vse32_v_f32m4(p_output, src_data, gvl); - p_temp_output += gvl; - p_output += gvl; - p_gamma_data += gvl; - length -= gvl; - } - } else if (p_gamma_data != nullptr) { - while (length > 0) { - gvl = __riscv_vsetvl_e32m4(length); - vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl); - vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl); - src_data = __riscv_vfsub_vf_f32m4(src_data, mean, gvl); - src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl); - src_data = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl); - vfloat32m4_t beta_data_v = __riscv_vle32_v_f32m4(p_beta_data, gvl); - src_data = __riscv_vfadd_vv_f32m4(src_data, beta_data_v, gvl); - p_beta_data += gvl; - __riscv_vse32_v_f32m4(p_output, src_data, gvl); - p_temp_output += gvl; - p_output += gvl; - p_gamma_data += gvl; - length -= gvl; - } - } + current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1); } } - void forward_rms_norm_f32(ggml_compute_params * params, ggml_tensor * op) { - const ggml_tensor * src0 = op->src[0]; - ggml_tensor * dst = op; - GGML_ASSERT(ggml_are_same_shape(src0, dst)); - GGML_ASSERT(src0->nb[0] == sizeof(float)); - - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_UNARY_OP_LOCALS - - float epsilon; - memcpy(&epsilon, dst->op_params, sizeof(float)); - - GGML_ASSERT(epsilon > 0.0f); - - auto * input = (float *) src0->data; - auto * output = (float *) dst->data; - - const auto hidden_size = ne00; - const auto task_count = ne01 * ne02 * ne03; - const auto task_per_thread = (task_count + nth - 1) / nth; - - const auto task_begin = ith * task_per_thread; - const auto task_end = std::min((ith + 1) * task_per_thread, task_count); - - for (auto task_idx = task_begin; task_idx < task_end; task_idx++) { - auto offset = task_idx * hidden_size; - auto * p_input = const_cast(input + offset); - auto * p_output = output + offset; - auto * p_temp_output = p_output; - auto * p_gamma_data = (const float *) nullptr; - auto * p_beta_data = (const float *) nullptr; - - size_t gvl = __riscv_vsetvlmax_e32m4(); - // vfloat32m4_t sum = __riscv_vfmv_v_f_f32m4(0.f, gvl); - vfloat32m4_t sum_sq = __riscv_vfmv_v_f_f32m4(0.f, gvl); - int64_t length = hidden_size; - while (length > 0) { - gvl = __riscv_vsetvl_e32m4(length); - // load data - vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_input, gvl); + int repack(ggml_tensor * t, const void * data, size_t data_size) override { + memcpy(t->data, data, data_size); + return 0; + } +}; - sum_sq = __riscv_vfmacc_vv_f32m4(sum_sq, src_data, src_data, gvl); +// Impl By IME1 +static const tensor_traits q4_0_16x32_q8_0; +static const tensor_traits q4_1_16x32_q8_0; +static const tensor_traits q4_k_16x32_q8_0; +static const tensor_traits q8_0_16x32_q8_0; +// Impl By IME2 +static const tensor_traits q2_k_32x256_q8_0; +static const tensor_traits q3_k_32x256_q8_0; +static const tensor_traits q4_0_32x32_q8_0; +static const tensor_traits q4_1_32x32_q8_0; +static const tensor_traits q4_0_32x256_q8_0; +static const tensor_traits q4_1_32x256_q8_0; +static const tensor_traits q4_k_32x32_q8_0; +static const tensor_traits q6_k_32x32_q8_0; +static const tensor_traits q8_0_32x32_q8_0; +static const tensor_traits mxfp4_32x32_q8_0; +static const tensor_traits q5_k_32x32_q8_0; +static const tensor_traits q5_1_32x32_q8_0; +static const tensor_traits q5_0_32x32_q8_0; +// Impl By RVV +static const tensor_traits_common rvv_impl; - __riscv_vse32_v_f32m4(p_temp_output, src_data, gvl); +} // namespace ggml::cpu::riscv64_spacemit - p_input += gvl; - p_temp_output += gvl; - length -= gvl; +static const ggml::cpu::tensor_traits * ggml_riscv64_spacemit_get_optimal_repack_type(const ggml_tensor * cur) { + switch (cur->type) { + case GGML_TYPE_Q2_K: + { +#if defined(RISCV64_SPACEMIT_IME2) + if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q2_k_32x256_q8_0; + } +#endif } + break; + case GGML_TYPE_Q3_K: + { +#if defined(RISCV64_SPACEMIT_IME2) + if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q3_k_32x256_q8_0; + } +#endif + } + break; + case GGML_TYPE_Q4_0: + { +#if defined(RISCV64_SPACEMIT_IME2) + if (cur->ne[1] % 32 == 0 && cur->ne[0] % 256 == 0 && + (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q4_0_32x256_q8_0; + } - gvl = __riscv_vsetvlmax_e32m1(); - - // float mean = 0.f; - vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.f, gvl); - - vfloat32m1_t mean_square_v = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum_sq, 0), - __riscv_vget_v_f32m4_f32m1(sum_sq, 1), gvl); - mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 2), gvl); - mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 3), gvl); - mean_square_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_square_v, zero_v, gvl); + if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q4_0_32x32_q8_0; + } +#endif - float mean_square = __riscv_vfmv_f_s_f32m1_f32(mean_square_v); - mean_square /= hidden_size; +#if defined(RISCV64_SPACEMIT_IME1) + if (cur->ne[1] % 16 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime1)) { + return &ggml::cpu::riscv64_spacemit::q4_0_16x32_q8_0; + } +#endif + } + break; + case GGML_TYPE_Q4_1: + { +#if defined(RISCV64_SPACEMIT_IME2) + // TODO + // if (cur->ne[1] % 32 == 0 && cur->ne[0] % 256 == 0 && + // (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + // return &ggml::cpu::riscv64_spacemit::q4_1_32x256_q8_0; + // } + + if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q4_1_32x32_q8_0; + } +#endif - mean_square = sqrt(mean_square + epsilon); +#if defined(RISCV64_SPACEMIT_IME1) + if (cur->ne[1] % 16 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime1)) { + return &ggml::cpu::riscv64_spacemit::q4_1_16x32_q8_0; + } +#endif + } + break; + case GGML_TYPE_Q4_K: + { +#if defined(RISCV64_SPACEMIT_IME2) + if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q4_k_32x32_q8_0; + } +#endif - mean_square = 1.0f / mean_square; - length = hidden_size; - p_temp_output = p_output; +#if defined(RISCV64_SPACEMIT_IME1) + if (cur->ne[1] % 16 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime1)) { + return &ggml::cpu::riscv64_spacemit::q4_k_16x32_q8_0; + } +#endif + } + break; + case GGML_TYPE_Q6_K: + { +#if defined(RISCV64_SPACEMIT_IME2) + if ((ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q6_k_32x32_q8_0; + } +#endif + } + break; + case GGML_TYPE_Q8_0: + { +#if defined(RISCV64_SPACEMIT_IME2) + if ((ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q8_0_32x32_q8_0; + } +#endif - if (p_gamma_data == nullptr && p_beta_data == nullptr) { - while (length > 0) { - gvl = __riscv_vsetvl_e32m4(length); - vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl); - src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl); - __riscv_vse32_v_f32m4(p_output, src_data, gvl); - p_temp_output += gvl; - p_output += gvl; - length -= gvl; +#if defined(RISCV64_SPACEMIT_IME1) + if (cur->ne[1] % 16 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime1)) { + return &ggml::cpu::riscv64_spacemit::q8_0_16x32_q8_0; } - } else if (p_beta_data == nullptr) { - while (length > 0) { - gvl = __riscv_vsetvl_e32m4(length); - vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl); - vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl); - src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl); - src_data = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl); - __riscv_vse32_v_f32m4(p_output, src_data, gvl); - p_temp_output += gvl; - p_output += gvl; - p_gamma_data += gvl; - length -= gvl; +#endif + } + break; + case GGML_TYPE_MXFP4: + { +#if defined(RISCV64_SPACEMIT_IME2) + // TODO + // if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + // return &ggml::cpu::riscv64_spacemit::mxfp4_32x32_q8_0; + // } +#endif + } + break; + case GGML_TYPE_Q5_K: + { +#if defined(RISCV64_SPACEMIT_IME2) + if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q5_k_32x32_q8_0; } - } else if (p_gamma_data != nullptr) { - while (length > 0) { - gvl = __riscv_vsetvl_e32m4(length); - vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl); - vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl); - src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl); - src_data = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl); - vfloat32m4_t beta_data_v = __riscv_vle32_v_f32m4(p_beta_data, gvl); - src_data = __riscv_vfadd_vv_f32m4(src_data, beta_data_v, gvl); - p_beta_data += gvl; - __riscv_vse32_v_f32m4(p_output, src_data, gvl); - p_temp_output += gvl; - p_output += gvl; - p_gamma_data += gvl; - length -= gvl; +#endif + } + break; + case GGML_TYPE_Q5_1: + { +#if defined(RISCV64_SPACEMIT_IME2) + if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q5_1_32x32_q8_0; } +#endif } - } - } - - int repack(struct ggml_tensor * t, const void * data, size_t data_size) override { - memcpy(t->data, data, data_size); - return 0; - } -}; - -static const tensor_traits q4_0_16x8_q8_0; -static const tensor_traits q4_1_16x8_q8_0; -static const tensor_traits q4_k_16x8_q8_0; -static const tensor_traits_common rvv_impl; - -} // namespace ggml::cpu::riscv64_spacemit - -static const ggml::cpu::tensor_traits * ggml_riscv64_spacemit_get_optimal_repack_type(const struct ggml_tensor * cur) { - if (cur->type == GGML_TYPE_Q4_0) { - if (cur->ne[1] % 16 == 0) { - return &ggml::cpu::riscv64_spacemit::q4_0_16x8_q8_0; - } - } else if (cur->type == GGML_TYPE_Q4_1) { - if (cur->ne[1] % 16 == 0) { - return &ggml::cpu::riscv64_spacemit::q4_1_16x8_q8_0; - } - } else if (cur->type == GGML_TYPE_Q4_K) { - if (cur->ne[1] % 16 == 0) { - return &ggml::cpu::riscv64_spacemit::q4_k_16x8_q8_0; - } - } else if (cur->type == GGML_TYPE_F32) { - return &ggml::cpu::riscv64_spacemit::rvv_impl; + break; + case GGML_TYPE_Q5_0: + { +#if defined(RISCV64_SPACEMIT_IME2) + if (cur->ne[1] % 32 == 0 && (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2)) { + return &ggml::cpu::riscv64_spacemit::q5_0_32x32_q8_0; + } +#endif + } + break; + default: + break; } return nullptr; } static enum ggml_status ggml_backend_riscv64_spacemit_buffer_init_tensor(ggml_backend_buffer_t buffer, - struct ggml_tensor * tensor) { + ggml_tensor * tensor) { tensor->extra = (void *) const_cast(ggml_riscv64_spacemit_get_optimal_repack_type(tensor)); @@ -874,8 +1422,46 @@ static enum ggml_status ggml_backend_riscv64_spacemit_buffer_init_tensor(ggml_ba return GGML_STATUS_SUCCESS; } +static void ggml_backend_riscv64_spacemit_buffer_free_buffer(ggml_backend_buffer_t buffer) { + GGML_ASSERT(buffer); + + void * base = buffer->context; + if (base == nullptr) { + return; + } + + ggml::cpu::riscv64_spacemit::spine_mem_pool_free(base); +} + +static void * ggml_backend_riscv64_spacemit_buffer_get_base(ggml_backend_buffer_t buffer) { + GGML_ASSERT(buffer); + + void * base = buffer->context; + GGML_ASSERT(base != nullptr); + return base; +} + +static void ggml_backend_riscv64_spacemit_buffer_memset_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + uint8_t value, + size_t offset, + size_t size) { + GGML_ASSERT(tensor); + memset((char *) tensor->data + offset, value, size); + + GGML_UNUSED(buffer); +} + +static void ggml_backend_riscv64_spacemit_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + GGML_ASSERT(buffer); + + void * base = buffer->context; + GGML_ASSERT(base != nullptr); + memset(base, value, buffer->size); +} + static void ggml_backend_riscv64_spacemit_buffer_set_tensor(ggml_backend_buffer_t buffer, - struct ggml_tensor * tensor, + ggml_tensor * tensor, const void * data, size_t offset, size_t size) { @@ -891,6 +1477,20 @@ static void ggml_backend_riscv64_spacemit_buffer_set_tensor(ggml_backend_buffer_ GGML_UNUSED(buffer); } +static const ggml_backend_buffer_i ggml_backend_riscv64_spacemit_buffer_i = { + /* .free_buffer = */ ggml_backend_riscv64_spacemit_buffer_free_buffer, + /* .get_base = */ ggml_backend_riscv64_spacemit_buffer_get_base, + /* .init_tensor = */ ggml_backend_riscv64_spacemit_buffer_init_tensor, + /* .memset_tensor = */ ggml_backend_riscv64_spacemit_buffer_memset_tensor, + /* .set_tensor = */ ggml_backend_riscv64_spacemit_buffer_set_tensor, + /* .get_tensor = */ nullptr, + /* .set_tensor_2d = */ nullptr, + /* .get_tensor_2d = */ nullptr, + /* .cpy_tensor = */ nullptr, + /* .clear = */ ggml_backend_riscv64_spacemit_buffer_clear, + /* .reset = */ nullptr, +}; + static const char * ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name(ggml_backend_buffer_type_t buft) { return "CPU_RISCV64_SPACEMIT"; @@ -899,18 +1499,12 @@ static const char * ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name(ggml_ static ggml_backend_buffer_t ggml_backend_cpu_riscv64_spacemit_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { - ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); - - if (buffer == nullptr) { + void * base = ggml::cpu::riscv64_spacemit::spine_mem_pool_alloc(size, 64); + if (base == nullptr) { return nullptr; } - buffer->buft = buft; - buffer->iface.init_tensor = ggml_backend_riscv64_spacemit_buffer_init_tensor; - buffer->iface.set_tensor = ggml_backend_riscv64_spacemit_buffer_set_tensor; - buffer->iface.get_tensor = nullptr; - buffer->iface.cpy_tensor = nullptr; - return buffer; + return ggml_backend_buffer_init(buft, ggml_backend_riscv64_spacemit_buffer_i, base, size); } static size_t ggml_backend_cpu_riscv64_spacemit_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { @@ -919,44 +1513,91 @@ static size_t ggml_backend_cpu_riscv64_spacemit_buffer_type_get_alignment(ggml_b GGML_UNUSED(buft); } -static size_t ggml_backend_cpu_riscv64_spacemit_nbytes(ggml_backend_buffer_type_t buft, - const struct ggml_tensor * tensor) { +static size_t ggml_backend_cpu_riscv64_spacemit_nbytes(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { for (int i = 0; i < GGML_MAX_DIMS; ++i) { if (tensor->ne[i] <= 0) { return 0; } } - size_t nbytes; + GGML_UNUSED(buft); + + const auto plain_nbytes = [&]() { + size_t total = ggml_type_size(tensor->type); + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + total += (tensor->ne[i] - 1) * tensor->nb[i]; + } + return total; + }; + const size_t blck_size = ggml_blck_size(tensor->type); if (blck_size == 1) { - nbytes = ggml_type_size(tensor->type); - for (int i = 0; i < GGML_MAX_DIMS; ++i) { - nbytes += (tensor->ne[i] - 1) * tensor->nb[i]; + return plain_nbytes(); + } + + const size_t row_nbytes = tensor->ne[0] * tensor->nb[0] / blck_size; + + const auto add_strided_nbytes = [&](size_t total, size_t src_block_size, size_t dst_block_size) { + for (int i = 1; i < GGML_MAX_DIMS; ++i) { + total += (tensor->ne[i] - 1) * (tensor->nb[i] / src_block_size) * dst_block_size; } - } else { - nbytes = tensor->ne[0] * tensor->nb[0] / blck_size; - if (tensor->type == GGML_TYPE_Q4_K) { - GGML_ASSERT(nbytes % sizeof(block_q4_K) == 0); - nbytes = (nbytes / sizeof(block_q4_K)) * sizeof(block_q4_1) * 8; - for (int i = 1; i < GGML_MAX_DIMS; ++i) { - nbytes += (tensor->ne[i] - 1) * (tensor->nb[i] / sizeof(block_q4_K)) * sizeof(block_q4_1) * 8; - } - } else { - for (int i = 1; i < GGML_MAX_DIMS; ++i) { - nbytes += (tensor->ne[i] - 1) * tensor->nb[i]; - } + return total; + }; + + const auto remap_block_nbytes = [&](size_t src_block_size, size_t dst_block_size, int64_t padded_rows = 0) { + GGML_ASSERT(row_nbytes % src_block_size == 0); + + size_t total = + add_strided_nbytes((row_nbytes / src_block_size) * dst_block_size, src_block_size, dst_block_size); + + if (padded_rows > 0 && tensor->ne[1] % padded_rows != 0) { + total += (padded_rows - tensor->ne[1] % padded_rows) * (tensor->nb[1] / src_block_size) * dst_block_size; } + + return total; + }; + + size_t nbytes = row_nbytes; + switch (tensor->type) { + case GGML_TYPE_Q4_K: + nbytes = remap_block_nbytes(sizeof(block_q4_K), sizeof(block_q4_1) * 8); + break; + case GGML_TYPE_Q6_K: + nbytes = remap_block_nbytes(sizeof(block_q6_K), sizeof(block_q8_0) * 8, 32); + break; + case GGML_TYPE_Q8_0: + nbytes = remap_block_nbytes(sizeof(block_q8_0), sizeof(block_q8_0), 32); + break; + case GGML_TYPE_Q2_K: + nbytes = remap_block_nbytes(sizeof(block_q2_K), sizeof(spacemit_kernels::nrow_block_q2_k<1>)); + break; + case GGML_TYPE_Q3_K: + nbytes = remap_block_nbytes(sizeof(block_q3_K), sizeof(spacemit_kernels::nrow_block_q3_k<1>)); + break; + case GGML_TYPE_MXFP4: + nbytes = remap_block_nbytes(sizeof(block_mxfp4), sizeof(spacemit_kernels::nrow_block_mxfp4<1>)); + break; + case GGML_TYPE_Q5_K: + nbytes = remap_block_nbytes(sizeof(block_q5_K), sizeof(spacemit_kernels::nrow_block_q5_1<1>) * 8); + break; + case GGML_TYPE_Q5_1: + nbytes = remap_block_nbytes(sizeof(block_q5_1), sizeof(spacemit_kernels::nrow_block_q5_1<1>)); + break; + case GGML_TYPE_Q5_0: + nbytes = remap_block_nbytes(sizeof(block_q5_0), sizeof(spacemit_kernels::nrow_block_q5_0<1>)); + break; + default: + nbytes = add_strided_nbytes(row_nbytes, 1, 1); + break; } - GGML_UNUSED(buft); return nbytes; } namespace ggml::cpu::riscv64_spacemit { class extra_buffer_type : ggml::cpu::extra_buffer_type { - bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override { + bool supports_op(ggml_backend_dev_t, const ggml_tensor * op) override { switch (op->op) { case GGML_OP_MUL_MAT: if (op->src[0]->buffer && (ggml_n_dims(op->src[0]) == 2) && @@ -970,10 +1611,16 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { } } break; - case GGML_OP_NORM: - case GGML_OP_RMS_NORM: - if (op->src[0]->type == GGML_TYPE_F32) { - return true; + case GGML_OP_MUL_MAT_ID: + if (op->src[0]->buffer && (ggml_n_dims(op->src[0]) == 3) && + op->src[0]->buffer->buft == ggml_backend_cpu_riscv64_spacemit_buffer_type() && + ggml_riscv64_spacemit_get_optimal_repack_type(op->src[0])) { + if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) { + return false; + } + if (op->src[1]->type == GGML_TYPE_F32) { + return true; + } } break; default: @@ -983,15 +1630,28 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { return false; } - ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override { + ggml::cpu::tensor_traits * get_tensor_traits(const ggml_tensor * op) override { switch (op->op) { case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_riscv64_spacemit_buffer_type()) { return (ggml::cpu::tensor_traits *) op->src[0]->extra; } break; case GGML_OP_NORM: case GGML_OP_RMS_NORM: + case GGML_OP_ADD: + case GGML_OP_SUB: + case GGML_OP_MUL: + case GGML_OP_DIV: + case GGML_OP_FLASH_ATTN_EXT: + case GGML_OP_CONT: + case GGML_OP_CPY: + case GGML_OP_REPEAT: + case GGML_OP_SUM_ROWS: + case GGML_OP_GET_ROWS: + case GGML_OP_CONCAT: + // case GGML_OP_GATED_DELTA_NET: return (ggml::cpu::tensor_traits *) (&ggml::cpu::riscv64_spacemit::rvv_impl); default: // GGML_ABORT("fatal error"); @@ -1005,15 +1665,17 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { } // namespace ggml::cpu::riscv64_spacemit ggml_backend_buffer_type_t ggml_backend_cpu_riscv64_spacemit_buffer_type(void) { - static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_riscv64_spacemit = { + static ggml_backend_buffer_type ggml_backend_cpu_buffer_type_riscv64_spacemit = { /* .iface = */ { - /* .get_name = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_get_alignment, - /* .get_max_size = */ nullptr, - /* .get_alloc_size = */ ggml_backend_cpu_riscv64_spacemit_nbytes, - /* .is_host = */ nullptr, + /* .get_name = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_get_alignment, + /* .get_max_size = */ nullptr, + /* .get_alloc_size = */ ggml_backend_cpu_riscv64_spacemit_nbytes, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ nullptr, }, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0), @@ -1023,3 +1685,78 @@ ggml_backend_buffer_type_t ggml_backend_cpu_riscv64_spacemit_buffer_type(void) { return &ggml_backend_cpu_buffer_type_riscv64_spacemit; } + +extern "C" { +static int bind_ai_thread() { + int fd, bytes; + char str[32]; + + fd = open("/proc/set_ai_thread", O_WRONLY); + if (fd < 0) { + GGML_LOG_ERROR("try open /proc/set_ai_thread failed\n"); + return -1; + } + + snprintf(str, 16, "%d", 0); + bytes = write(fd, str, strlen(str)); + if (bytes < 0) { + GGML_LOG_ERROR("try write /proc/set_ai_thread failed\n"); + close(fd); + return -1; + } + + close(fd); + return 0; +} + +void ggml_backend_cpu_riscv64_spacemit_set_numa_thread_affinity(int thread_n) { + int cpu_id = sched_getcpu(); + if (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_ime2 && + !((1 << cpu_id) & ggml::cpu::riscv64_spacemit::global_spine_env_info.cpu_mask)) { + GGML_PRINT_DEBUG("bind_ai_thread for thread %d, pid %d\n", thread_n, getpid()); + bind_ai_thread(); + } + + if (ggml::cpu::riscv64_spacemit::global_spine_env_info.use_tcm && + ggml::cpu::riscv64_spacemit::tls_context.cpu_id == -1) { + CPU_ZERO(&(ggml::cpu::riscv64_spacemit::tls_context.cpuset)); + pthread_t main_thread = pthread_self(); + const auto & perfer_core_ids = ggml::cpu::riscv64_spacemit::global_spine_env_info.perfer_core_ids; + if (thread_n < 0 || static_cast(thread_n) >= perfer_core_ids.size()) { + GGML_ABORT("thread_n %d exceeds perfer_core_ids size %zu\n", thread_n, perfer_core_ids.size()); + } + auto perfer_cpu_id = perfer_core_ids[static_cast(thread_n)]; + CPU_SET(perfer_cpu_id, &(ggml::cpu::riscv64_spacemit::tls_context.cpuset)); + int s = + pthread_setaffinity_np(main_thread, sizeof(cpu_set_t), &(ggml::cpu::riscv64_spacemit::tls_context.cpuset)); + if (s != 0) { + GGML_ABORT("set thread affinity error for thread_n %d, cpu_id %d\n", thread_n, perfer_cpu_id); + } + + int ai_cpu_id = perfer_cpu_id - ggml::cpu::riscv64_spacemit::global_spine_env_info.aicpu_id_offset; + ggml::cpu::riscv64_spacemit::tls_context.cpu_id = ai_cpu_id; + ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer = + ggml::cpu::riscv64_spacemit::spine_mem_pool_tcm_mem_get(ai_cpu_id); + ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer_size = + ggml::cpu::riscv64_spacemit::global_spine_env_info.tcm_blk_size; + } + + if (ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer != nullptr) { + void * rt = + ggml::cpu::riscv64_spacemit::spine_mem_pool_tcm_mem_wait(ggml::cpu::riscv64_spacemit::tls_context.cpu_id); + if (rt == nullptr) { + GGML_ABORT("wait tcm buffer failed for cpu_id: %d", ggml::cpu::riscv64_spacemit::tls_context.cpu_id); + } + } +} + +void ggml_backend_cpu_riscv64_spacemit_clear_numa_thread_affinity_threaded(int thread_n) { + if (ggml::cpu::riscv64_spacemit::tls_context.tcm_buffer != nullptr) { + auto rt = ggml::cpu::riscv64_spacemit::spine_mem_pool_tcm_mem_release( + ggml::cpu::riscv64_spacemit::tls_context.cpu_id); + if (rt != 0) { + GGML_ABORT("release tcm buffer failed for cpu_id: %d", ggml::cpu::riscv64_spacemit::tls_context.cpu_id); + } + } +} +} diff --git a/ggml/src/ggml-cpu/spacemit/ime.h b/ggml/src/ggml-cpu/spacemit/ime.h index 800d91a..6849dd9 100644 --- a/ggml/src/ggml-cpu/spacemit/ime.h +++ b/ggml/src/ggml-cpu/spacemit/ime.h @@ -8,6 +8,14 @@ extern "C" { ggml_backend_buffer_type_t ggml_backend_cpu_riscv64_spacemit_buffer_type(void); +void ggml_backend_cpu_riscv64_spacemit_set_numa_thread_affinity(int thread_n); + +void ggml_backend_cpu_riscv64_spacemit_clear_numa_thread_affinity_threaded(int thread_n); + +void * ggml_backend_cpu_riscv64_spacemit_alloc_shared(size_t size, size_t alignment); + +void ggml_backend_cpu_riscv64_spacemit_free_shared(void * ptr); + #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp b/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp index cbbb6cd..6c974f6 100644 --- a/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +++ b/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp @@ -1,8 +1,26 @@ +#include "ggml-impl.h" #include "ggml.h" #include "ime_kernels.h" +#include "rvv_kernels.h" #include #include +#include + +#if !defined(__riscv_v) || !defined(__riscv_v_intrinsic) +# error "riscv v extension or v_intrinsic not enabled" +#else +# include +#endif + +#if !defined(__riscv_zfh) +# error "riscv zfh extension not enabled" +#endif + +#if defined(RISCV64_SPACEMIT_IME1) +#else +# error "RISCV64_SPACEMIT_IME1 not defined" +#endif // clang-format off #if defined(__GNUC__) @@ -11,7 +29,7 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #endif // clang-format on -namespace sqnbitgemm_spacemit_ime { +namespace spacemit_kernels { #define QUANTIZEM4ROW_KERNEL \ "vmv.s.x v16, zero \n\t" \ @@ -76,1093 +94,208 @@ namespace sqnbitgemm_spacemit_ime { "vse8.v v31, (s1) \n\t" namespace ime1 { -void quantize_a_4row_i8(size_t BlkLen, const float * A, size_t CountK, std::byte * QuantA) { +void quantize_a_4row_i8(size_t BlkLen, const float * A, size_t CountK, uint8_t * QuantA) { constexpr float range_max_reciprocal = 1.0f / ((1 << 7) - 1); const float fone = 1.0f; - if (BlkLen == 16 || BlkLen == 32 || BlkLen == 64) { - for (size_t row_index = 0; row_index < 4; ++row_index) { - const float * SRC = A + row_index * CountK; - std::byte * DST = QuantA + row_index * sizeof(float); - - const size_t offset = (4 - row_index) * 4 + row_index * 8; - const size_t stride = 4 * (sizeof(float) + BlkLen); - __asm__ volatile( - "vsetvli t0, zero, e32, m8 \n\t" - "addi t2, %[CountK], 0 \n\t" - "addi a1, %[DST], 0 \n\t" - "blt t2, %[BlkLen], TAIL%= \n\t" - - "LOOP%=: \n\t" - "vsetvli t0, %[BlkLen], e32, m8 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "sub t2, t2, t0 \n\t" - "slli t1, t0, 2 \n\t" - "add %[SRC], %[SRC], t1 \n\t" - "add s1, a1, %[OFFSET] \n\t" - - QUANTIZEM4ROW_KERNEL QUANTIZEM4ROW_STORE - - "add a1, a1, %[STRIDE] \n\t" - "bge t2, %[BlkLen], LOOP%= \n\t" - - "TAIL%=: \n\t" - "blez t2, QUIT%= \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v24, v24, v24 \n\t" - "vsetvli t0, t2, e32, m8 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "add s1, a1, %[OFFSET] \n\t" - - QUANTIZEM4ROW_KERNEL - - "addi t3, %[BlkLen], 0 \n\t" - "addi s2, s1, 0 \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vxor.vv v8, v8, v8 \n\t" - "SET_ZERO%=: \n\t" - "vse8.v v8, (s2) \n\t" - "addi s2, s2, 32 \n\t" - "addi t3, t3, -8 \n\t" - "bnez t3, SET_ZERO%= \n\t" - - QUANTIZEM4ROW_STORE - - "QUIT%=: \n\t" - : [SRC] "+r"(SRC) - : [DST] "r"(DST), [BlkLen] "r"(BlkLen), [OFFSET] "r"(offset), [STRIDE] "r"(stride), - [CountK] "r"(CountK), [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal) - : "cc", "t0", "t1", "t2", "t3", "a1", "s1", "s2", "f10", "f11"); - } - } else if (BlkLen == 128) { - for (size_t row_index = 0; row_index < 4; ++row_index) { - const float * SRC = A + row_index * CountK; - std::byte * DST = QuantA + row_index * sizeof(float); + for (size_t row_index = 0; row_index < 4; ++row_index) { + const float * SRC = A + row_index * CountK; + uint8_t * DST = QuantA + row_index * sizeof(float); - const size_t offset = (4 - row_index) * 4 + row_index * 8; - const size_t stride = 4 * (sizeof(float) + BlkLen); - __asm__ volatile( - "vsetvli t0, zero, e32, m8 \n\t" - "li t6, 32 \n\t" - "addi t2, %[CountK], 0 \n\t" - "addi a1, %[DST], 0 \n\t" - "add s1, a1, %[OFFSET] \n\t" - "blt t2, %[BlkLen], TAIL%= \n\t" - - "LOOP%=: \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v8, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "addi t2, t2, -128 \n\t" - - "QUANTIZE%=: \n\t" - "add s1, a1, %[OFFSET] \n\t" - "vfabs.v v16, v0 \n\t" - "vfabs.v v24, v8 \n\t" - "vfmax.vv v16, v24, v16 \n\t" - "vfredmax.vs v24, v16, v24 \n\t" - "vfmv.f.s f10, v24 \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (a1) \n\t" - "fdiv.s f11, %[FONE], f10 \n\t" - "vfmul.vf v16, v0, f11 \n\t" - "vfmul.vf v24, v8, f11 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vfcvt.x.f.v v24, v24 \n\t" - "vsetvli t0, zero, e16, m4 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vnclip.wx v20, v24, zero \n\t" - "vsetvli t0, zero, e8, m4 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vsetvli t0, zero, e64, m4 \n\t" - "vsse64.v v16, (s1), t6 \n\t" - "add a1, a1, %[STRIDE] \n\t" - "bge t2, %[BlkLen], LOOP%= \n\t" - - "TAIL%=: \n\t" - "blez t2, QUIT%= \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v0, v0, v0 \n\t" - "vxor.vv v8, v8, v8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v24, v24, v24 \n\t" - "vsetvli t0, t2, e32, m8 \n\t" - "sub t2, t2, t0 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vsetvli t0, t2, e32, m8 \n\t" - "vle32.v v8, (%[SRC]) \n\t" - "sub t2, t2, t2 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "jal x0, QUANTIZE%= \n\t" - - "QUIT%=: \n\t" - : [SRC] "+r"(SRC) - : [DST] "r"(DST), [BlkLen] "r"(BlkLen), [OFFSET] "r"(offset), [STRIDE] "r"(stride), - [CountK] "r"(CountK), [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal) - : "cc", "t0", "t1", "t2", "t6", "a1", "s1", "s2", "f10", "f11"); - } - } else if (BlkLen == 256) { - for (size_t row_index = 0; row_index < 4; ++row_index) { - const float * SRC = A + row_index * CountK; - std::byte * DST = QuantA + row_index * sizeof(float); - const size_t offset = (4 - row_index) * 4 + row_index * 8; - const size_t stride = 4 * (sizeof(float) + BlkLen); - __asm__ volatile( - "vsetvli t0, zero, e32, m8 \n\t" - "li t6, 32 \n\t" - "addi t2, %[CountK], 0 \n\t" - "addi a1, %[DST], 0 \n\t" - "add s1, a1, %[OFFSET] \n\t" - "blt t2, %[BlkLen], TAIL%= \n\t" - - "LOOP%=: \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v8, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v16, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v24, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], -768 \n\t" - "addi t2, t2, -256 \n\t" - "vfabs.v v0, v0 \n\t" - "vfabs.v v8, v8 \n\t" - "vfabs.v v16, v16 \n\t" - "vfabs.v v24, v24 \n\t" - "vfmax.vv v8, v0, v8 \n\t" - "vfmax.vv v24, v24, v16 \n\t" - "vfmax.vv v8, v8, v24 \n\t" - "vfredmax.vs v24, v8, v24 \n\t" - "vfmv.f.s f10, v24 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v8, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v16, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v24, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - - "QUANTIZE%=: \n\t" - "add s1, a1, %[OFFSET] \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (a1) \n\t" - "fdiv.s f11, %[FONE], f10 \n\t" - "vfmul.vf v0, v0, f11 \n\t" - "vfmul.vf v8, v8, f11 \n\t" - "vfmul.vf v16, v16, f11 \n\t" - "vfmul.vf v24, v24, f11 \n\t" - "vfcvt.x.f.v v0, v0 \n\t" - "vfcvt.x.f.v v8, v8 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vfcvt.x.f.v v24, v24 \n\t" - "vsetvli t0, zero, e16, m4 \n\t" - "vnclip.wx v0, v0, zero \n\t" - "vnclip.wx v4, v8, zero \n\t" - "vnclip.wx v8, v16, zero \n\t" - "vnclip.wx v12, v24, zero \n\t" - "vsetvli t0, zero, e8, m4 \n\t" - "vnclip.wx v0, v0, zero \n\t" - "vnclip.wx v4, v8, zero \n\t" - "vsetvli t0, zero, e64, m8 \n\t" - "vsse64.v v0, (s1), t6 \n\t" - "add a1, a1, %[STRIDE] \n\t" - "bge t2, %[BlkLen], LOOP%= \n\t" - - "TAIL%=: \n\t" - "blez t2, QUIT%= \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v0, v0, v0 \n\t" - "vxor.vv v8, v8, v8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v24, v24, v24 \n\t" - "addi t1, t2, 0 \n\t" - "vsetvli t0, t1, e32, m8 \n\t" - "sub t1, t1, t0 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vsetvli t0, t1, e32, m8 \n\t" - "sub t1, t1, t0 \n\t" - "vle32.v v8, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vsetvli t0, t1, e32, m8 \n\t" - "sub t1, t1, t0 \n\t" - "vle32.v v16, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vsetvli t0, t1, e32, m8 \n\t" - "vle32.v v24, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], -768 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vfabs.v v0, v0 \n\t" - "vfabs.v v8, v8 \n\t" - "vfabs.v v16, v16 \n\t" - "vfabs.v v24, v24 \n\t" - "vfmax.vv v8, v0, v8 \n\t" - "vfmax.vv v24, v16, v24 \n\t" - "vfmax.vv v8, v8, v24 \n\t" - "vfredmax.vs v24, v8, v24 \n\t" - "vfmv.f.s f10, v24 \n\t" - "add s1, a1, %[OFFSET] \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (a1) \n\t" - "fdiv.s f11, %[FONE], f10 \n\t" - "vsetvli t0, zero, e64, m8 \n\t" - "vxor.vv v0, v0, v0 \n\t" - "vsse64.v v0, (s1), t6 \n\t" - - "TAIL_LOOP%=: \n\t" - "vsetvli t0, zero, e32, m4 \n\t" - "vxor.vv v0, v0, v0 \n\t" - "vsetvli t0, t2, e32, m1 \n\t" - "sub t2, t2, t0 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 32 \n\t" - "vfmul.vf v1, v0, f11 \n\t" - "vfcvt.x.f.v v2, v1 \n\t" - "vsetvli t0, zero, e16, mf2 \n\t" - "vnclip.wx v3, v2, zero \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vnclip.wx v3, v3, zero \n\t" - "vse8.v v3, (s1) \n\t" - "addi s1, s1, 32 \n\t" - "bnez t2, TAIL_LOOP%= \n\t" - - "QUIT%=: \n\t" - : [SRC] "+r"(SRC) - : [DST] "r"(DST), [BlkLen] "r"(BlkLen), [OFFSET] "r"(offset), [STRIDE] "r"(stride), - [CountK] "r"(CountK), [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal) - : "cc", "t0", "t1", "t2", "t6", "a1", "s1", "s2", "f10", "f11"); - } + const size_t offset = (4 - row_index) * 4 + row_index * 8; + const size_t stride = 4 * (sizeof(float) + BlkLen); + __asm__ volatile( + "vsetvli t0, zero, e32, m8 \n\t" + "addi t2, %[CountK], 0 \n\t" + "addi a1, %[DST], 0 \n\t" + "blt t2, %[BlkLen], TAIL%= \n\t" + + "LOOP%=: \n\t" + "vsetvli t0, %[BlkLen], e32, m8 \n\t" + "vle32.v v0, (%[SRC]) \n\t" + "sub t2, t2, t0 \n\t" + "slli t1, t0, 2 \n\t" + "add %[SRC], %[SRC], t1 \n\t" + "add s1, a1, %[OFFSET] \n\t" + + QUANTIZEM4ROW_KERNEL QUANTIZEM4ROW_STORE + + "add a1, a1, %[STRIDE] \n\t" + "bge t2, %[BlkLen], LOOP%= \n\t" + + "TAIL%=: \n\t" + "blez t2, QUIT%= \n\t" + "vsetvli t0, zero, e32, m8 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v24, v24, v24 \n\t" + "vsetvli t0, t2, e32, m8 \n\t" + "vle32.v v0, (%[SRC]) \n\t" + "add s1, a1, %[OFFSET] \n\t" + + QUANTIZEM4ROW_KERNEL + + "addi t3, %[BlkLen], 0 \n\t" + "addi s2, s1, 0 \n\t" + "vsetvli t0, zero, e8, mf4 \n\t" + "vxor.vv v8, v8, v8 \n\t" + "SET_ZERO%=: \n\t" + "vse8.v v8, (s2) \n\t" + "addi s2, s2, 32 \n\t" + "addi t3, t3, -8 \n\t" + "bnez t3, SET_ZERO%= \n\t" + + QUANTIZEM4ROW_STORE + + "QUIT%=: \n\t" + : [SRC] "+r"(SRC) + : [DST] "r"(DST), [BlkLen] "r"(BlkLen), [OFFSET] "r"(offset), [STRIDE] "r"(stride), [CountK] "r"(CountK), + [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal) + : "cc", "t0", "t1", "t2", "t3", "a1", "s1", "s2", "f10", "f11"); } } -void quantize_a_row_i8(size_t BlkLen, const float * A, size_t CountK, std::byte * QuantA) { +void quantize_a_row_i8(size_t BlkLen, const float * A, size_t CountK, uint8_t * QuantA) { const float * SRC = A; - std::byte * DST = QuantA; + uint8_t * DST = QuantA; constexpr float range_max_reciprocal = 1.0f / ((1 << 7) - 1); const float fone = 1.0f; - std::byte * QuantA_offset = QuantA + CountK + 4 * ((CountK + BlkLen - 1) / BlkLen); + uint8_t * QuantA_offset = QuantA + CountK + 4 * ((CountK + BlkLen - 1) / BlkLen); size_t offset = (CountK + BlkLen - 1) / BlkLen * BlkLen - CountK; - if (CountK <= BlkLen) { - float max_abs_A = 0.0f; - for (size_t k = 0; k < CountK; k++) { - max_abs_A = std::max(max_abs_A, fabsf(A[k])); - } - float scale_A = max_abs_A * range_max_reciprocal; - - ((float *) QuantA)[0] = scale_A; - - auto * QuantAData_offset = (int8_t *) (QuantA + sizeof(float)); - - for (size_t k = 0; k < CountK; k++) { - QuantAData_offset[k] = - (int8_t) std::clamp(roundf(A[k] / scale_A), (float) std::numeric_limits::lowest(), - (float) std::numeric_limits::max()); - } - for (size_t k = CountK; k < BlkLen; k++) { - QuantAData_offset[k] = 0; - } - - return; - } - - if (BlkLen != 32 || BlkLen != 64 || BlkLen != 128) { - __asm__ volatile( - "vsetvli t0, zero, e8, m8 \n\t" - "vxor.vv v24, v24, v24 \n\t" - "LOOP%=: \n\t" - "vsetvli t0, %[CNT], e8, m8 \n\t" - "vse8.v v24, (%[DST]) \n\t" - "addi %[DST], %[DST], 128 \n\t" - "sub %[CNT], %[CNT], t0 \n\t" - "bnez %[CNT], LOOP%= \n\t" - : [DST] "+r"(QuantA_offset), [CNT] "+r"(offset) - : - : "cc", "t0"); - } - if (BlkLen == 16) { - float buffer[64] = { 0.0f }; - __asm__ volatile( - "addi t3, zero, 16*8 \n\t" - "addi t2, zero, 16 \n\t" - "blt %[K], t3, LOOP_K%= \n\t" - "blt %[K], t2, TAIL%= \n\t" - "LOOP_MAIN%=: \n\t" - "vsetvli t1, zero, e32, m2 \n\t" - "addi %[K], %[K], -128 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 64 \n\t" - "vle32.v v2, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 64 \n\t" - "vle32.v v4, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 64 \n\t" - "vle32.v v6, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 64 \n\t" - "vle32.v v8, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 64 \n\t" - "vle32.v v10, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 64 \n\t" - "vle32.v v12, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 64 \n\t" - "vle32.v v14, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 64 \n\t" - "addi a1, %[BUFFER], 0 \n\t" - "vfabs.v v16, v0 \n\t" - "vfabs.v v18, v2 \n\t" - "vfabs.v v20, v4 \n\t" - "vfabs.v v22, v6 \n\t" - "vfabs.v v24, v8 \n\t" - "vfabs.v v26, v10 \n\t" - "vfabs.v v28, v12 \n\t" - "vfabs.v v30, v14 \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vfmax.vv v16, v16, v17 \n\t" - "vfmax.vv v18, v18, v19 \n\t" - "vfmax.vv v20, v20, v21 \n\t" - "vfmax.vv v22, v22, v23 \n\t" - "vfmax.vv v24, v24, v25 \n\t" - "vfmax.vv v26, v26, v27 \n\t" - "vfmax.vv v28, v28, v29 \n\t" - "vfmax.vv v30, v30, v31 \n\t" - "vse32.v v16, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vse32.v v18, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vse32.v v20, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vse32.v v22, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vse32.v v24, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vse32.v v26, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vse32.v v28, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vse32.v v30, (a1) \n\t" - "addi a1, %[BUFFER], 0 \n\t" - "flw f0, (a1) \n\t" - "flw f1, 4(a1) \n\t" - "flw f2, 8(a1) \n\t" - "flw f3, 12(a1) \n\t" - "flw f4, 16(a1) \n\t" - "flw f5, 20(a1) \n\t" - "flw f6, 24(a1) \n\t" - "flw f7, 28(a1) \n\t" - "addi a1, a1, 32 \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f10, f3, f7 \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "fdiv.s f10, %[FONE], f10 \n\t" - "flw f0, (a1) \n\t" - "flw f1, 4(a1) \n\t" - "flw f2, 8(a1) \n\t" - "flw f3, 12(a1) \n\t" - "flw f4, 16(a1) \n\t" - "flw f5, 20(a1) \n\t" - "flw f6, 24(a1) \n\t" - "flw f7, 28(a1) \n\t" - "addi a1, a1, 32 \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f11, f3, f7 \n\t" - "fmul.s f11, f11, %[RMAXREC] \n\t" - "fsw f11, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "fdiv.s f11, %[FONE], f11 \n\t" - "flw f0, (a1) \n\t" - "flw f1, 4(a1) \n\t" - "flw f2, 8(a1) \n\t" - "flw f3, 12(a1) \n\t" - "flw f4, 16(a1) \n\t" - "flw f5, 20(a1) \n\t" - "flw f6, 24(a1) \n\t" - "flw f7, 28(a1) \n\t" - "addi a1, a1, 32 \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f12, f3, f7 \n\t" - "fmul.s f12, f12, %[RMAXREC] \n\t" - "fsw f12, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "fdiv.s f12, %[FONE], f12 \n\t" - "flw f0, (a1) \n\t" - "flw f1, 4(a1) \n\t" - "flw f2, 8(a1) \n\t" - "flw f3, 12(a1) \n\t" - "flw f4, 16(a1) \n\t" - "flw f5, 20(a1) \n\t" - "flw f6, 24(a1) \n\t" - "flw f7, 28(a1) \n\t" - "addi a1, a1, 32 \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f13, f3, f7 \n\t" - "fmul.s f13, f13, %[RMAXREC] \n\t" - "fsw f13, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "fdiv.s f13, %[FONE], f13 \n\t" - "flw f0, (a1) \n\t" - "flw f1, 4(a1) \n\t" - "flw f2, 8(a1) \n\t" - "flw f3, 12(a1) \n\t" - "flw f4, 16(a1) \n\t" - "flw f5, 20(a1) \n\t" - "flw f6, 24(a1) \n\t" - "flw f7, 28(a1) \n\t" - "addi a1, a1, 32 \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f14, f3, f7 \n\t" - "fmul.s f14, f14, %[RMAXREC] \n\t" - "fsw f14, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "fdiv.s f14, %[FONE], f14 \n\t" - "flw f0, (a1) \n\t" - "flw f1, 4(a1) \n\t" - "flw f2, 8(a1) \n\t" - "flw f3, 12(a1) \n\t" - "flw f4, 16(a1) \n\t" - "flw f5, 20(a1) \n\t" - "flw f6, 24(a1) \n\t" - "flw f7, 28(a1) \n\t" - "addi a1, a1, 32 \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f15, f3, f7 \n\t" - "fmul.s f15, f15, %[RMAXREC] \n\t" - "fsw f15, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "fdiv.s f15, %[FONE], f15 \n\t" - "flw f0, (a1) \n\t" - "flw f1, 4(a1) \n\t" - "flw f2, 8(a1) \n\t" - "flw f3, 12(a1) \n\t" - "flw f4, 16(a1) \n\t" - "flw f5, 20(a1) \n\t" - "flw f6, 24(a1) \n\t" - "flw f7, 28(a1) \n\t" - "addi a1, a1, 32 \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f16, f3, f7 \n\t" - "fmul.s f16, f16, %[RMAXREC] \n\t" - "fsw f16, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "fdiv.s f16, %[FONE], f16 \n\t" - "flw f0, (a1) \n\t" - "flw f1, 4(a1) \n\t" - "flw f2, 8(a1) \n\t" - "flw f3, 12(a1) \n\t" - "flw f4, 16(a1) \n\t" - "flw f5, 20(a1) \n\t" - "flw f6, 24(a1) \n\t" - "flw f7, 28(a1) \n\t" - "addi a1, a1, 32 \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f17, f3, f7 \n\t" - "fmul.s f17, f17, %[RMAXREC] \n\t" - "fsw f17, (%[DST]) \n\t" - "addi %[DST], %[DST], -136 \n\t" - "fdiv.s f17, %[FONE], f17 \n\t" - "vsetvli t0, zero, e32, m2 \n\t" - "vfmul.vf v16, v0, f10 \n\t" - "vfmul.vf v18, v2, f11 \n\t" - "vfmul.vf v20, v4, f12 \n\t" - "vfmul.vf v22, v6, f13 \n\t" - "vfmul.vf v24, v8, f14 \n\t" - "vfmul.vf v26, v10, f15 \n\t" - "vfmul.vf v28, v12, f16 \n\t" - "vfmul.vf v30, v14, f17 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vfcvt.x.f.v v18, v18 \n\t" - "vfcvt.x.f.v v20, v20 \n\t" - "vfcvt.x.f.v v22, v22 \n\t" - "vfcvt.x.f.v v24, v24 \n\t" - "vfcvt.x.f.v v26, v26 \n\t" - "vfcvt.x.f.v v28, v28 \n\t" - "vfcvt.x.f.v v30, v30 \n\t" - "vsetvli t0, zero, e16, m1 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vnclip.wx v18, v18, zero \n\t" - "vnclip.wx v20, v20, zero \n\t" - "vnclip.wx v22, v22, zero \n\t" - "vnclip.wx v24, v24, zero \n\t" - "vnclip.wx v26, v26, zero \n\t" - "vnclip.wx v28, v28, zero \n\t" - "vnclip.wx v30, v30, zero \n\t" - "vsetvli t0, t1, e8, mf2 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vnclip.wx v18, v18, zero \n\t" - "vnclip.wx v20, v20, zero \n\t" - "vnclip.wx v22, v22, zero \n\t" - "vnclip.wx v24, v24, zero \n\t" - "vnclip.wx v26, v26, zero \n\t" - "vnclip.wx v28, v28, zero \n\t" - "vnclip.wx v30, v30, zero \n\t" - "vse8.v v16, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "vse8.v v18, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "vse8.v v20, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "vse8.v v22, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "vse8.v v24, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "vse8.v v26, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "vse8.v v28, (%[DST]) \n\t" - "addi %[DST], %[DST], 20 \n\t" - "vse8.v v30, (%[DST]) \n\t" - "addi %[DST], %[DST], 16 \n\t" - "bge %[K], t3, LOOP_MAIN%= \n\t" - "blt %[K], t2, TAIL%= \n\t" - "LOOP_K%=: \n\t" - "vsetvli t1, %[K], e32, m2 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 64 \n\t" - "sub %[K], %[K], t1 \n\t" - "vfabs.v v16, v0 \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vfmax.vv v16, v16, v17 \n\t" - "vse32.v v16, (%[BUFFER]) \n\t" - "flw f0, (%[BUFFER]) \n\t" - "flw f1, 4(%[BUFFER]) \n\t" - "flw f2, 8(%[BUFFER]) \n\t" - "flw f3, 12(%[BUFFER]) \n\t" - "flw f4, 16(%[BUFFER]) \n\t" - "flw f5, 20(%[BUFFER]) \n\t" - "flw f6, 24(%[BUFFER]) \n\t" - "flw f7, 28(%[BUFFER]) \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f10, f3, f7 \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (%[DST]) \n\t" - "addi %[DST], %[DST], 4 \n\t" - "fdiv.s f11, %[FONE], f10 \n\t" - "vsetvli t0, zero, e32, m2 \n\t" - "vfmul.vf v16, v0, f11 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vsetvli t0, zero, e16, m1 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vsetvli t0, t1, e8, mf2 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vse8.v v16, (%[DST]) \n\t" - "addi %[DST], %[DST], 16 \n\t" - "bge %[K], t2, LOOP_K%= \n\t" - "TAIL%=: \n\t" - "blez %[K], END%= \n\t" - "vsetvli t0, t3, e32, m2 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "jal x0, LOOP_K%= \n\t" - "END%=: \n\t" - : [SRC] "+r"(SRC), [DST] "+r"(DST), [K] "+r"(CountK) - : [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal), [BUFFER] "r"(buffer) - : "cc", "t3", "t2", "t1", "t0", "a1", "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f10", "f11", "f12", - "f13", "f14", "f15", "f16", "f17"); - } else if (BlkLen == 32) { - __asm__ volatile( - "addi t3, zero, 32*4 \n\t" - "addi t2, zero, 32 \n\t" - - "addi a1, %[SRC], 0 \n\t" - "addi a2, %[SRC], 128 \n\t" - "addi a3, %[SRC], 256 \n\t" - "addi a4, %[SRC], 384 \n\t" - - "addi s1, %[DST], 0 \n\t" - "addi s2, %[DST], 36 \n\t" - "addi s3, %[DST], 72 \n\t" - "addi s4, %[DST], 108 \n\t" - "blt %[K], t3, LOOP_K%= \n\t" - "blt %[K], t2, TAIL%= \n\t" - - "LOOP_MAIN%=: \n\t" - "vsetvli t1, zero, e32, m4 \n\t" - "addi %[K], %[K], -128 \n\t" - "vle32.v v0, (a1) \n\t" - "addi a1, a1, 512 \n\t" - "vle32.v v4, (a2) \n\t" - "addi a2, a2, 512 \n\t" - "vle32.v v8, (a3) \n\t" - "addi a3, a3, 512 \n\t" - "vle32.v v12, (a4) \n\t" - "addi a4, a4, 512 \n\t" - "vfabs.v v16, v0 \n\t" - "vfabs.v v20, v4 \n\t" - "vfabs.v v24, v8 \n\t" - "vfabs.v v28, v12 \n\t" - "vsetvli t0, zero, e32, m2 \n\t" - "vfmax.vv v16, v16, v18 \n\t" - "vfmax.vv v20, v20, v22 \n\t" - "vfmax.vv v24, v24, v26 \n\t" - "vfmax.vv v28, v28, v30 \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vfmax.vv v16, v16, v17 \n\t" - "vfmax.vv v20, v20, v21 \n\t" - "vfmax.vv v24, v24, v25 \n\t" - "vfmax.vv v28, v28, v29 \n\t" - - "vfredmax.vs v17, v16, v17 \n\t" - "vfredmax.vs v21, v20, v21 \n\t" - "vfredmax.vs v25, v24, v25 \n\t" - "vfredmax.vs v29, v28, v29 \n\t" - "vfmv.f.s f10, v17 \n\t" - "vfmv.f.s f11, v21 \n\t" - "vfmv.f.s f12, v25 \n\t" - "vfmv.f.s f13, v29 \n\t" - - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fmul.s f11, f11, %[RMAXREC] \n\t" - "fmul.s f12, f12, %[RMAXREC] \n\t" - "fmul.s f13, f13, %[RMAXREC] \n\t" - "fsw f10, (s1) \n\t" - "addi s1, s1, 4 \n\t" - - "fsw f11, (s2) \n\t" - "addi s2, s2, 4 \n\t" - "fsw f12, (s3) \n\t" - "addi s3, s3, 4 \n\t" - "fsw f13, (s4) \n\t" - "addi s4, s4, 4 \n\t" - "fdiv.s f10, %[FONE], f10 \n\t" - "fdiv.s f11, %[FONE], f11 \n\t" - "fdiv.s f12, %[FONE], f12 \n\t" - "fdiv.s f13, %[FONE], f13 \n\t" - "vsetvli t0, zero, e32, m4 \n\t" - "vfmul.vf v16, v0, f10 \n\t" - "vfmul.vf v20, v4, f11 \n\t" - "vfmul.vf v24, v8, f12 \n\t" - "vfmul.vf v28, v12, f13 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vfcvt.x.f.v v20, v20 \n\t" - "vfcvt.x.f.v v24, v24 \n\t" - "vfcvt.x.f.v v28, v28 \n\t" - "vsetvli t0, zero, e16, m2 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vnclip.wx v20, v20, zero \n\t" - "vnclip.wx v24, v24, zero \n\t" - "vnclip.wx v28, v28, zero \n\t" - "vsetvli t0, t1, e8, m1 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vnclip.wx v20, v20, zero \n\t" - "vnclip.wx v24, v24, zero \n\t" - "vnclip.wx v28, v28, zero \n\t" - "vse8.v v16, (s1) \n\t" - "addi s1, s1, 140 \n\t" - "vse8.v v20, (s2) \n\t" - "addi s2, s2, 140 \n\t" - "vse8.v v24, (s3) \n\t" - "addi s3, s3, 140 \n\t" - "vse8.v v28, (s4) \n\t" - "addi s4, s4, 140 \n\t" - "bge %[K], t3, LOOP_MAIN%= \n\t" - "blt %[K], t2, TAIL%= \n\t" - "LOOP_K%=: \n\t" - "vsetvli t1, %[K], e32, m4 \n\t" - "vle32.v v0, (a1) \n\t" - "addi a1, a1, 128 \n\t" - "sub %[K], %[K], t1 \n\t" - "vfabs.v v16, v0 \n\t" - "vsetvli t0, zero, e32, m2 \n\t" - "vfmax.vv v16, v16, v18 \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vfmax.vv v16, v16, v17 \n\t" - "vfredmax.vs v17, v16, v17 \n\t" - "vfmv.f.s f10, v17 \n\t" - - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (s1) \n\t" - "addi s1, s1, 4 \n\t" - "fdiv.s f11, %[FONE], f10 \n\t" - "vsetvli t0, zero, e32, m4 \n\t" - "vfmul.vf v16, v0, f11 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vsetvli t0, zero, e16, m2 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vse8.v v16, (s1) \n\t" - "addi s1, s1, 32 \n\t" - "bge %[K], t2, LOOP_K%= \n\t" - "TAIL%=: \n\t" - "blez %[K], END%= \n\t" - "vsetvli t0, t3, e32, m4 \n\t" - "vxor.vv v0, v0, v0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "jal x0, LOOP_K%= \n\t" - "END%=: \n\t" - : [K] "+r"(CountK) - : [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal), [SRC] "r"(SRC), [DST] "r"(DST) - : "cc", "t3", "t2", "t1", "t0", "a1", "a2", "a3", "a4", "s1", "s2", "s3", "s4", "f10", "f11", "f12", "f13"); - } else if (BlkLen == 64) { - __asm__ volatile( - "addi t3, zero, 64*2 \n\t" - "addi t2, zero, 64 \n\t" - "addi a1, %[SRC], 0 \n\t" - "addi a2, %[SRC], 256 \n\t" - "addi s1, %[DST], 0 \n\t" - "addi s2, %[DST], 68 \n\t" - "blt %[K], t3, LOOP_K%= \n\t" - "blt %[K], t2, TAIL%= \n\t" - "LOOP_MAIN%=: \n\t" - "vsetvli t1, zero, e32, m8 \n\t" - "addi %[K], %[K], -128 \n\t" - "vle32.v v0, (a1) \n\t" - "addi a1, a1, 512 \n\t" - "vle32.v v8, (a2) \n\t" - "addi a2, a2, 512 \n\t" - "vfabs.v v16, v0 \n\t" - "vfabs.v v24, v8 \n\t" - "vsetvli t0, zero, e32, m4 \n\t" - "vfmax.vv v16, v16, v20 \n\t" - "vfmax.vv v24, v24, v28 \n\t" - "vsetvli t0, zero, e32, m2 \n\t" - "vfmax.vv v16, v16, v18 \n\t" - "vfmax.vv v24, v24, v26 \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vfmax.vv v16, v16, v17 \n\t" - "vfmax.vv v24, v24, v25 \n\t" - "vfredmax.vs v17, v16, v17 \n\t" - "vfredmax.vs v25, v24, v25 \n\t" - "vfmv.f.s f10, v17 \n\t" - "vfmv.f.s f11, v25 \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fmul.s f11, f11, %[RMAXREC] \n\t" - "fsw f10, (s1) \n\t" - "addi s1, s1, 4 \n\t" - "fsw f11, (s2) \n\t" - "addi s2, s2, 4 \n\t" - "fdiv.s f10, %[FONE], f10 \n\t" - "fdiv.s f11, %[FONE], f11 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vfmul.vf v16, v0, f10 \n\t" - "vfmul.vf v24, v8, f11 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vfcvt.x.f.v v24, v24 \n\t" - "vsetvli t0, zero, e16, m4 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vnclip.wx v24, v24, zero \n\t" - "vsetvli t0, t1, e8, m2 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vnclip.wx v24, v24, zero \n\t" - "vse8.v v16, (s1) \n\t" - "addi s1, s1, 132 \n\t" - "vse8.v v24, (s2) \n\t" - "addi s2, s2, 132 \n\t" - "bge %[K], t3, LOOP_MAIN%= \n\t" - "blt %[K], t2, TAIL%= \n\t" - "LOOP_K%=: \n\t" - "vsetvli t1, %[K], e32, m8 \n\t" - "vle32.v v0, (a1) \n\t" - "addi a1, a1, 256 \n\t" - "sub %[K], %[K], t1 \n\t" - "vfabs.v v16, v0 \n\t" - "vsetvli t0, zero, e32, m4 \n\t" - "vfmax.vv v16, v16, v20 \n\t" - "vsetvli t0, zero, e32, m2 \n\t" - "vfmax.vv v16, v16, v18 \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vfmax.vv v16, v16, v17 \n\t" - "vfredmax.vs v17, v16, v17 \n\t" - "vfmv.f.s f10, v17 \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (s1) \n\t" - "addi s1, s1, 4 \n\t" - "fdiv.s f11, %[FONE], f10 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vfmul.vf v16, v0, f11 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vsetvli t0, zero, e16, m4 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vsetvli t0, zero, e8, m2 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vse8.v v16, (s1) \n\t" - "addi s1, s1, 64 \n\t" - "bge %[K], t2, LOOP_K%= \n\t" - "TAIL%=: \n\t" - "blez %[K], END%= \n\t" - "vsetvli t0, t3, e32, m8 \n\t" - "vxor.vv v0, v0, v0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "jal x0, LOOP_K%= \n\t" - "END%=: \n\t" - : [K] "+r"(CountK) - : [SRC] "r"(SRC), [DST] "r"(DST), [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal) - : "cc", "t3", "t2", "t1", "t0", "a1", "a2", "s1", "s2", "f10", "f11"); - } else if (BlkLen == 128) { - __asm__ volatile( - "addi t2, zero, 128 \n\t" - "addi a1, %[SRC], 0 \n\t" - "addi a2, %[SRC], 256 \n\t" - "blt %[K], t2, TAIL%= \n\t" - "LOOP_K%=: \n\t" - "vsetvli t1, zero, e32, m8 \n\t" - "vle32.v v0, (a1) \n\t" - "addi a1, a1, 512 \n\t" - "vle32.v v8, (a2) \n\t" - "addi a2, a2, 512 \n\t" - "sub %[K], %[K], t2 \n\t" - "QUANT%=: \n\t" - "vfabs.v v16, v0 \n\t" - "vfabs.v v24, v8 \n\t" - "vfmax.vv v24, v16, v24 \n\t" - "vsetvli t1, zero, e32, m4 \n\t" - "vfmax.vv v28, v24, v28 \n\t" - "vsetvli t0, zero, e32, m2 \n\t" - "vfmax.vv v30, v28, v30 \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vfmax.vv v30, v30, v31 \n\t" - "vfredmax.vs v31, v30, v31 \n\t" - "vfmv.f.s f10, v31 \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (%[DST]) \n\t" - "addi %[DST], %[DST], 4 \n\t" - "fdiv.s f11, %[FONE], f10 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vfmul.vf v16, v0, f11 \n\t" - "vfmul.vf v24, v8, f11 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vfcvt.x.f.v v24, v24 \n\t" - "vsetvli t0, zero, e16, m4 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vnclip.wx v20, v24, zero \n\t" - "vsetvli t0, zero, e8, m4 \n\t" - "vnclip.wx v16, v16, zero \n\t" - "vse8.v v16, (%[DST]) \n\t" - "addi %[DST], %[DST], 128 \n\t" - "bge %[K], t2, LOOP_K%= \n\t" - "TAIL%=: \n\t" - "blez %[K], END%= \n\t" - "vsetvli t1, zero, e32, m8 \n\t" - "vxor.vv v0, v0, v0 \n\t" - "vxor.vv v8, v8, v8 \n\t" - "vsetvli t0, %[K], e32, m8 \n\t" - "vle32.v v0, (a1) \n\t" - "sub %[K], %[K], t0 \n\t" - "vsetvli t0, %[K], e32, m8 \n\t" - "vle32.v v8, (a2) \n\t" - "sub %[K], %[K], t0 \n\t" - "vsetvli t1, zero, e32, m8 \n\t" - "jal x0, QUANT%= \n\t" - "END%=: \n\t" - - : [DST] "+r"(DST), [K] "+r"(CountK) - : [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal), [SRC] "r"(SRC) - : "cc", "t2", "t1", "t0", "a1", "a2", "f10", "f11"); - } else { - float buffer[8] = { 0.0f }; - size_t cnt = BlkLen / 256; - - __asm__ volatile( - "slli t3, %[BLK], 2 \n\t" - "blt %[K], %[BLK], LOOP_TAIL%= \n\t" - "LOOP_MAIN%=: \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vxor.vv v31, v31, v31 \n\t" - "vse32.v v31, (%[BUFFER]) \n\t" - "addi t6, %[CNT], 0 \n\t" - "LOOP_CMP%=: \n\t" - "addi t6, t6, -1 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v8, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v16, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v24, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vfabs.v v0, v0 \n\t" - "vfabs.v v8, v8 \n\t" - "vfabs.v v16, v16 \n\t" - "vfabs.v v24, v24 \n\t" - "vfmax.vv v8, v0, v8 \n\t" - "vfmax.vv v16, v16, v24 \n\t" - "vfmax.vv v0, v0, v16 \n\t" - "vsetvli t0, zero, e32, m4 \n\t" - "vfmax.vv v0, v0, v4 \n\t" - "vsetvli t0, zero, e32, m2 \n\t" - "vfmax.vv v0, v0, v2 \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vfmax.vv v0, v0, v1 \n\t" - "vle32.v v30, (%[BUFFER]) \n\t" - "vfmax.vv v31, v30, v0 \n\t" - "vse32.v v31, (%[BUFFER]) \n\t" - "bnez t6, LOOP_CMP%= \n\t" - "sub %[SRC], %[SRC], t3 \n\t" - "addi t6, %[CNT], 0 \n\t" - "flw f0, (%[BUFFER]) \n\t" - "flw f1, 4(%[BUFFER]) \n\t" - "flw f2, 8(%[BUFFER]) \n\t" - "flw f3, 12(%[BUFFER]) \n\t" - "flw f4, 16(%[BUFFER]) \n\t" - "flw f5, 20(%[BUFFER]) \n\t" - "flw f6, 24(%[BUFFER]) \n\t" - "flw f7, 28(%[BUFFER]) \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f10, f3, f7 \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (%[DST]) \n\t" - "addi %[DST], %[DST], 4 \n\t" - "fdiv.s f11, %[FONE], f10 \n\t" - "addi t6, %[CNT], 0 \n\t" - "LOOP_QUANT%=: \n\t" - "addi t6, t6, -1 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v8, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v16, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vle32.v v24, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vfmul.vf v0, v0, f11 \n\t" - "vfmul.vf v8, v8, f11 \n\t" - "vfmul.vf v16, v16, f11 \n\t" - "vfmul.vf v24, v24, f11 \n\t" - "vfcvt.x.f.v v0, v0 \n\t" - "vfcvt.x.f.v v8, v8 \n\t" - "vfcvt.x.f.v v16, v16 \n\t" - "vfcvt.x.f.v v24, v24 \n\t" - "vsetvli t0, zero, e16, m4 \n\t" - "vnclip.wx v0, v0, zero \n\t" - "vnclip.wx v4, v8, zero \n\t" - "vnclip.wx v8, v16, zero \n\t" - "vnclip.wx v12, v24, zero \n\t" - "vsetvli t0, zero, e8, m4 \n\t" - "vnclip.wx v0, v0, zero \n\t" - "vnclip.wx v4, v8, zero \n\t" - "vse8.v v0, (%[DST]) \n\t" - "addi %[DST], %[DST], 128 \n\t" - "vse8.v v4, (%[DST]) \n\t" - "addi %[DST], %[DST], 128 \n\t" - "bnez t6, LOOP_QUANT%= \n\t" - "sub %[K], %[K], %[BLK] \n\t" - "bge %[K], %[BLK], LOOP_MAIN%= \n\t" - "blez %[K], END%= \n\t" - "LOOP_TAIL%=: \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vxor.vv v31, v31, v31 \n\t" - "vse32.v v31, (%[BUFFER]) \n\t" - "addi t6, %[K], 0 \n\t" - "addi s1, %[SRC], 0 \n\t" - "TAIL_CMP%=: \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v0, v0, v0 \n\t" - "vsetvli t0, t6, e32, m8 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi %[SRC], %[SRC], 256 \n\t" - "sub t6, t6, t0 \n\t" - "vfabs.v v0, v0 \n\t" - "vsetvli t0, zero, e32, m4 \n\t" - "vfmax.vv v0, v0, v4 \n\t" - "vsetvli t0, zero, e32, m2 \n\t" - "vfmax.vv v0, v0, v2 \n\t" - "vsetvli t0, zero, e32, m1 \n\t" - "vfmax.vv v0, v0, v1 \n\t" - "vle32.v v30, (%[BUFFER]) \n\t" - "vfmax.vv v31, v30, v0 \n\t" - "vse32.v v31, (%[BUFFER]) \n\t" - "bnez t6, TAIL_CMP%= \n\t" - "addi t6, %[K], 0 \n\t" - "flw f0, (%[BUFFER]) \n\t" - "flw f1, 4(%[BUFFER]) \n\t" - "flw f2, 8(%[BUFFER]) \n\t" - "flw f3, 12(%[BUFFER]) \n\t" - "flw f4, 16(%[BUFFER]) \n\t" - "flw f5, 20(%[BUFFER]) \n\t" - "flw f6, 24(%[BUFFER]) \n\t" - "flw f7, 28(%[BUFFER]) \n\t" - "fmax.s f1, f0, f1 \n\t" - "fmax.s f3, f2, f3 \n\t" - "fmax.s f5, f4, f5 \n\t" - "fmax.s f7, f6, f7 \n\t" - "fmax.s f3, f1, f3 \n\t" - "fmax.s f7, f5, f7 \n\t" - "fmax.s f10, f3, f7 \n\t" - "fmul.s f10, f10, %[RMAXREC] \n\t" - "fsw f10, (%[DST]) \n\t" - "addi %[DST], %[DST], 4 \n\t" - "fdiv.s f11, %[FONE], f10 \n\t" - "addi t6, %[K], 0 \n\t" - "TAIL_QUANT%=: \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v0, v0, v0 \n\t" - "vsetvli t1, t6, e32, m8 \n\t" - "vle32.v v0, (s1) \n\t" - "addi s1, s1, 256 \n\t" - "sub t6, t6, t1 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vfmul.vf v0, v0, f11 \n\t" - "vfcvt.x.f.v v0, v0 \n\t" - "vsetvli t0, zero, e16, m4 \n\t" - "vnclip.wx v0, v0, zero \n\t" - "vsetvli t0, t1, e8, m2 \n\t" - "vnclip.wx v0, v0, zero \n\t" - "vse8.v v0, (%[DST]) \n\t" - "addi %[DST], %[DST], 64 \n\t" - "bnez t6, TAIL_QUANT%= \n\t" - "END%=: \n\t" - : [SRC] "+r"(SRC), [DST] "+r"(DST), [K] "+r"(CountK) - : [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal), [BLK] "r"(BlkLen), [BUFFER] "r"(buffer), - [CNT] "r"(cnt) - : "cc", "t1", "t0", "t6", "s1", "f0", "f1", "f2", "f3", "f4", "f5", "f6"); - } + __asm__ volatile( + "addi t3, zero, 32*4 \n\t" + "addi t2, zero, 32 \n\t" + + "addi a1, %[SRC], 0 \n\t" + "addi a2, %[SRC], 128 \n\t" + "addi a3, %[SRC], 256 \n\t" + "addi a4, %[SRC], 384 \n\t" + + "addi s1, %[DST], 0 \n\t" + "addi s2, %[DST], 36 \n\t" + "addi s3, %[DST], 72 \n\t" + "addi s4, %[DST], 108 \n\t" + "blt %[K], t3, LOOP_K%= \n\t" + "blt %[K], t2, TAIL%= \n\t" + + "LOOP_MAIN%=: \n\t" + "vsetvli t1, zero, e32, m4 \n\t" + "addi %[K], %[K], -128 \n\t" + "vle32.v v0, (a1) \n\t" + "addi a1, a1, 512 \n\t" + "vle32.v v4, (a2) \n\t" + "addi a2, a2, 512 \n\t" + "vle32.v v8, (a3) \n\t" + "addi a3, a3, 512 \n\t" + "vle32.v v12, (a4) \n\t" + "addi a4, a4, 512 \n\t" + "vfabs.v v16, v0 \n\t" + "vfabs.v v20, v4 \n\t" + "vfabs.v v24, v8 \n\t" + "vfabs.v v28, v12 \n\t" + "vsetvli t0, zero, e32, m2 \n\t" + "vfmax.vv v16, v16, v18 \n\t" + "vfmax.vv v20, v20, v22 \n\t" + "vfmax.vv v24, v24, v26 \n\t" + "vfmax.vv v28, v28, v30 \n\t" + "vsetvli t0, zero, e32, m1 \n\t" + "vfmax.vv v16, v16, v17 \n\t" + "vfmax.vv v20, v20, v21 \n\t" + "vfmax.vv v24, v24, v25 \n\t" + "vfmax.vv v28, v28, v29 \n\t" + + "vfredmax.vs v17, v16, v17 \n\t" + "vfredmax.vs v21, v20, v21 \n\t" + "vfredmax.vs v25, v24, v25 \n\t" + "vfredmax.vs v29, v28, v29 \n\t" + "vfmv.f.s f10, v17 \n\t" + "vfmv.f.s f11, v21 \n\t" + "vfmv.f.s f12, v25 \n\t" + "vfmv.f.s f13, v29 \n\t" + + "fmul.s f10, f10, %[RMAXREC] \n\t" + "fmul.s f11, f11, %[RMAXREC] \n\t" + "fmul.s f12, f12, %[RMAXREC] \n\t" + "fmul.s f13, f13, %[RMAXREC] \n\t" + "fsw f10, (s1) \n\t" + "addi s1, s1, 4 \n\t" + + "fsw f11, (s2) \n\t" + "addi s2, s2, 4 \n\t" + "fsw f12, (s3) \n\t" + "addi s3, s3, 4 \n\t" + "fsw f13, (s4) \n\t" + "addi s4, s4, 4 \n\t" + "fdiv.s f10, %[FONE], f10 \n\t" + "fdiv.s f11, %[FONE], f11 \n\t" + "fdiv.s f12, %[FONE], f12 \n\t" + "fdiv.s f13, %[FONE], f13 \n\t" + "vsetvli t0, zero, e32, m4 \n\t" + "vfmul.vf v16, v0, f10 \n\t" + "vfmul.vf v20, v4, f11 \n\t" + "vfmul.vf v24, v8, f12 \n\t" + "vfmul.vf v28, v12, f13 \n\t" + "vfcvt.x.f.v v16, v16 \n\t" + "vfcvt.x.f.v v20, v20 \n\t" + "vfcvt.x.f.v v24, v24 \n\t" + "vfcvt.x.f.v v28, v28 \n\t" + "vsetvli t0, zero, e16, m2 \n\t" + "vnclip.wx v16, v16, zero \n\t" + "vnclip.wx v20, v20, zero \n\t" + "vnclip.wx v24, v24, zero \n\t" + "vnclip.wx v28, v28, zero \n\t" + "vsetvli t0, t1, e8, m1 \n\t" + "vnclip.wx v16, v16, zero \n\t" + "vnclip.wx v20, v20, zero \n\t" + "vnclip.wx v24, v24, zero \n\t" + "vnclip.wx v28, v28, zero \n\t" + "vse8.v v16, (s1) \n\t" + "addi s1, s1, 140 \n\t" + "vse8.v v20, (s2) \n\t" + "addi s2, s2, 140 \n\t" + "vse8.v v24, (s3) \n\t" + "addi s3, s3, 140 \n\t" + "vse8.v v28, (s4) \n\t" + "addi s4, s4, 140 \n\t" + "bge %[K], t3, LOOP_MAIN%= \n\t" + "blt %[K], t2, TAIL%= \n\t" + "LOOP_K%=: \n\t" + "vsetvli t1, %[K], e32, m4 \n\t" + "vle32.v v0, (a1) \n\t" + "addi a1, a1, 128 \n\t" + "sub %[K], %[K], t1 \n\t" + "vfabs.v v16, v0 \n\t" + "vsetvli t0, zero, e32, m2 \n\t" + "vfmax.vv v16, v16, v18 \n\t" + "vsetvli t0, zero, e32, m1 \n\t" + "vfmax.vv v16, v16, v17 \n\t" + "vfredmax.vs v17, v16, v17 \n\t" + "vfmv.f.s f10, v17 \n\t" + + "fmul.s f10, f10, %[RMAXREC] \n\t" + "fsw f10, (s1) \n\t" + "addi s1, s1, 4 \n\t" + "fdiv.s f11, %[FONE], f10 \n\t" + "vsetvli t0, zero, e32, m4 \n\t" + "vfmul.vf v16, v0, f11 \n\t" + "vfcvt.x.f.v v16, v16 \n\t" + "vsetvli t0, zero, e16, m2 \n\t" + "vnclip.wx v16, v16, zero \n\t" + "vsetvli t0, zero, e8, m1 \n\t" + "vnclip.wx v16, v16, zero \n\t" + "vse8.v v16, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "bge %[K], t2, LOOP_K%= \n\t" + "TAIL%=: \n\t" + "blez %[K], END%= \n\t" + "vsetvli t0, t3, e32, m4 \n\t" + "vxor.vv v0, v0, v0 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "jal x0, LOOP_K%= \n\t" + "END%=: \n\t" + : [K] "+r"(CountK) + : [FONE] "f"(fone), [RMAXREC] "f"(range_max_reciprocal), [SRC] "r"(SRC), [DST] "r"(DST) + : "cc", "t3", "t2", "t1", "t0", "a1", "a2", "a3", "a4", "s1", "s2", "s3", "s4", "f10", "f11", "f12", "f13"); } } // namespace ime1 @@ -1451,1746 +584,681 @@ namespace { "vadd.vi v1, v1, -12 \n\t" template -void SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl(size_t BlkLen, - const std::byte * QuantA, - const std::byte * QuantBData, - const float * QuantBScale, - const std::byte * QuantBZeroPoint, - float * C, - size_t CountN, - size_t BlockCountK, - const float * Bias, - const size_t ldc) { - GGML_UNUSED(QuantBScale); - GGML_UNUSED(QuantBZeroPoint); +void SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl(size_t BlkLen, + const uint8_t * QuantA, + const uint8_t * QuantBData, + float * C, + size_t CountN, + size_t BlockCountK, + const size_t ldc) { size_t LDC = ldc * sizeof(float); const size_t INNER = BlkLen / 16; float tmp[4 * 16]; if constexpr (HasZeroPoint) { for (size_t n = 0; n < CountN; n += 16) { - size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n; - std::byte * QuantBDataPtr = (std::byte *) QuantBData + // - n * BlockCountK * BlkLen / 2 + // b data - n * BlockCountK * sizeof(uint8_t) + // zp - n * BlockCountK * sizeof(_Float16); // scale - float * CPtr = C + n; - if (NBLKS < 16) { - CPtr = tmp; - LDC = 16 * sizeof(float); - } - if (Bias != nullptr) { - const float * bias = Bias + n; - if (NBLKS < 16) { - __asm__ volatile( - "vsetvli t0, %[N], e32, m2 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "vse32.v v0, (%[DST]) \n\t" - : - : [SRC] "r"(bias), [DST] "r"(tmp), [N] "r"(NBLKS) - : "cc", "t0"); - bias = tmp; - } - __asm__ volatile(LOAD_BIAS - - "addi t3, %[BlockCountK], 0 \n\t" - - "vsetvli t0, zero, e8, m1 \n\t" - "li s1, 24 \n\t" - "vmv.v.i v1, 3 \n\t" - "vsetvli t0, s1, e8, m1 \n\t" - "vmv.v.i v1, 2 \n\t" - "vsetvli t0, zero, e8, mf2 \n\t" - "vmv.v.i v1, 1 \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vmv.v.i v1, 0 \n\t" - - "addi a1, %[A], 0 \n\t" - "addi s1, %[B], 0 \n\t" - - "BLOCK_COUNTK_LOOP%=: \n\t" - // scale offset - "addi s5, s1, 0 \n\t" - // zp offset - "addi s6, s1, 32 \n\t" - "addi s1, s6, 16 \n\t" - "addi s2, s1, 32 \n\t" - "addi s3, s1, 32*2 \n\t" - "addi s4, s1, 32*3 \n\t" - - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - // load a scale - "flw f1, (a1) \n\t" - "flw f2, 4(a1) \n\t" - "flw f3, 8(a1) \n\t" - "flw f4, 12(a1) \n\t" - "addi a1, a1, 16 \n\t" - "addi t2, %[INNER], 0 \n\t" - - SQ4BIT_KERNEL_LOAD_ZP_16X1_v2 - - "BLOCK_INNER_LOOP%=: \n\t" - - LOAD_B_16x8x2 - - "vle8.v v10, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vle8.v v11, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vsub.vv v2, v2, v12 \n\t" - "vsub.vv v6, v6, v12 \n\t" - "vsub.vv v3, v3, v13 \n\t" - "vsub.vv v7, v7, v13 \n\t" - "vsub.vv v4, v4, v14 \n\t" - "vsub.vv v8, v8, v14 \n\t" - "vsub.vv v5, v5, v15 \n\t" - "vsub.vv v9, v9, v15 \n\t" - - SQ4BIT_KERNEL_COMP_4x16x16 - - "addi t2, t2, -1 \n\t" - "bnez t2, BLOCK_INNER_LOOP%= \n\t" - - LOAD_SCALE_4x16_FP16 - - "vsetvli t0, zero, e32, m8 \n\t" - "vfcvt.f.x.v v16, v16 \n\t" - "vfmacc.vv v24, v16, v8 \n\t" - "addi t3, t3, -1 \n\t" - "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" - - "RESULT_SAVE%=: \n\t" - - SAVE_RESULT_4x16 - - : - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), - [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr), [BIAS] "r"(bias) - : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", - "s2", "s3", "s4", "s5", "s6"); - - } else { - __asm__ volatile( - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v24, v24, v24 \n\t" - "addi t3, %[BlockCountK], 0 \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - "li s1, 24 \n\t" - "vmv.v.i v1, 3 \n\t" - "vsetvli t0, s1, e8, m1 \n\t" - "vmv.v.i v1, 2 \n\t" - "vsetvli t0, zero, e8, mf2 \n\t" - "vmv.v.i v1, 1 \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vmv.v.i v1, 0 \n\t" - "addi a1, %[A], 0 \n\t" - "addi s1, %[B], 0 \n\t" - "BLOCK_COUNTK_LOOP%=: \n\t" - // scale offset - "addi s5, s1, 0 \n\t" - // zp offset - "addi s6, s1, 32 \n\t" - "addi s1, s6, 16 \n\t" - "addi s2, s1, 32 \n\t" - "addi s3, s1, 32*2 \n\t" - "addi s4, s1, 32*3 \n\t" - - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - // load a scale - "flw f1, (a1) \n\t" - "flw f2, 4(a1) \n\t" - "flw f3, 8(a1) \n\t" - "flw f4, 12(a1) \n\t" - "addi a1, a1, 16 \n\t" - "addi t2, %[INNER], 0 \n\t" - - SQ4BIT_KERNEL_LOAD_ZP_16X1_v2 - - "BLOCK_INNER_LOOP%=: \n\t" - - LOAD_B_16x8x2 - - "vle8.v v10, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vle8.v v11, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vsub.vv v2, v2, v12 \n\t" - "vsub.vv v6, v6, v12 \n\t" - "vsub.vv v3, v3, v13 \n\t" - "vsub.vv v7, v7, v13 \n\t" - "vsub.vv v4, v4, v14 \n\t" - "vsub.vv v8, v8, v14 \n\t" - "vsub.vv v5, v5, v15 \n\t" - "vsub.vv v9, v9, v15 \n\t" - - SQ4BIT_KERNEL_COMP_4x16x16 - - "addi t2, t2, -1 \n\t" - "bnez t2, BLOCK_INNER_LOOP%= \n\t" - - LOAD_SCALE_4x16_FP16 - - "vsetvli t0, zero, e32, m8 \n\t" - "vfcvt.f.x.v v16, v16 \n\t" - "vfmacc.vv v24, v16, v8 \n\t" - "addi t3, t3, -1 \n\t" - "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" - - "RESULT_SAVE%=: \n\t" - - SAVE_RESULT_4x16 - - : - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), - [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr) - : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", "s2", "s3", - "s4", "s5", "s6"); - } - } - } else { - for (size_t n = 0; n < CountN; n += 16) { - size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n; - std::byte * QuantBDataPtr = (std::byte *) QuantBData + // - n * BlockCountK * BlkLen / 2 + // b data - n * BlockCountK * sizeof(_Float16); // scale + size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n; + uint8_t * QuantBDataPtr = (uint8_t *) QuantBData + // + n * BlockCountK * BlkLen / 2 + // b data + n * BlockCountK * sizeof(uint8_t) + // zp + n * BlockCountK * sizeof(_Float16); // scale float * CPtr = C + n; if (NBLKS < 16) { CPtr = tmp; LDC = 16 * sizeof(float); } - if (Bias != nullptr) { - const float * bias = Bias + n; - if (NBLKS < 16) { - __asm__ volatile( - "vsetvli t0, %[N], e32, m2 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "vse32.v v0, (%[DST]) \n\t" - : - : [SRC] "r"(bias), [DST] "r"(tmp), [N] "r"(NBLKS) - : "cc", "t0"); - bias = tmp; - } - __asm__ volatile(LOAD_BIAS - - "addi t3, %[BlockCountK], 0 \n\t" - "addi a1, %[A], 0 \n\t" - "addi s1, %[B], 0 \n\t" - "BLOCK_COUNTK_LOOP%=: \n\t" - "addi s5, s1, 0 \n\t" - "addi s1, s5, 32 \n\t" - "addi s2, s1, 32 \n\t" - "addi s3, s1, 32*2 \n\t" - "addi s4, s1, 32*3 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - // load a scale - "flw f1, (a1) \n\t" - "flw f2, 4(a1) \n\t" - "flw f3, 8(a1) \n\t" - "flw f4, 12(a1) \n\t" - "addi a1, a1, 16 \n\t" - "addi t2, %[INNER], 0 \n\t" - "BLOCK_INNER_LOOP%=: \n\t" - - LOAD_B_16x8x2 - - "vsetvli t0, zero, e8, m1 \n\t" - "vle8.v v10, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vle8.v v11, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vadd.vi v2, v2, -8 \n\t" - "vadd.vi v3, v3, -8 \n\t" - "vadd.vi v4, v4, -8 \n\t" - "vadd.vi v5, v5, -8 \n\t" - "vadd.vi v6, v6, -8 \n\t" - "vadd.vi v7, v7, -8 \n\t" - "vadd.vi v8, v8, -8 \n\t" - "vadd.vi v9, v9, -8 \n\t" - - SQ4BIT_KERNEL_COMP_4x16x16 - - "addi t2, t2, -1 \n\t" - "bnez t2, BLOCK_INNER_LOOP%= \n\t" - - LOAD_SCALE_4x16_FP16 - - "vsetvli t0, zero, e32, m8 \n\t" - "vfcvt.f.x.v v16, v16 \n\t" - "vfmacc.vv v24, v16, v8 \n\t" - "addi t3, t3, -1 \n\t" - "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" - "RESULT_SAVE%=: \n\t" - - SAVE_RESULT_4x16 - - : - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), - [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr), [BIAS] "r"(bias) - : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", - "s2", "s3", "s4", "s5", "s6"); - - } else { - __asm__ volatile( - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v24, v24, v24 \n\t" - "addi t3, %[BlockCountK], 0 \n\t" - "addi a1, %[A], 0 \n\t" - "addi s1, %[B], 0 \n\t" - "BLOCK_COUNTK_LOOP%=: \n\t" - "addi s5, s1, 0 \n\t" - "addi s1, s5, 32 \n\t" - "addi s2, s1, 32 \n\t" - "addi s3, s1, 32*2 \n\t" - "addi s4, s1, 32*3 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - // load a scale - "flw f1, (a1) \n\t" - "flw f2, 4(a1) \n\t" - "flw f3, 8(a1) \n\t" - "flw f4, 12(a1) \n\t" - "addi a1, a1, 16 \n\t" - "addi t2, %[INNER], 0 \n\t" - "BLOCK_INNER_LOOP%=: \n\t" - - LOAD_B_16x8x2 - - "vsetvli t0, zero, e8, m1 \n\t" - "vle8.v v10, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vle8.v v11, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vadd.vi v2, v2, -8 \n\t" - "vadd.vi v3, v3, -8 \n\t" - "vadd.vi v4, v4, -8 \n\t" - "vadd.vi v5, v5, -8 \n\t" - "vadd.vi v6, v6, -8 \n\t" - "vadd.vi v7, v7, -8 \n\t" - "vadd.vi v8, v8, -8 \n\t" - "vadd.vi v9, v9, -8 \n\t" - - SQ4BIT_KERNEL_COMP_4x16x16 - - "addi t2, t2, -1 \n\t" - "bnez t2, BLOCK_INNER_LOOP%= \n\t" - - LOAD_SCALE_4x16_FP16 - - "vsetvli t0, zero, e32, m8 \n\t" - "vfcvt.f.x.v v16, v16 \n\t" - "vfmacc.vv v24, v16, v8 \n\t" - "addi t3, t3, -1 \n\t" - "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" - "RESULT_SAVE%=: \n\t" - - SAVE_RESULT_4x16 - - : - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), - [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr) - : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", "s2", "s3", - "s4", "s5", "s6"); - } - } - } - if (CountN % 16 != 0) { - // stroe output from tmp to C when NBLKS less than 16. - float * CPtr = C + CountN / 16 * 16; - const size_t N = CountN % 16; - LDC = ldc * sizeof(float); - __asm__ volatile( - "vsetvli t0, %[N], e32, m2 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi s2, %[SRC], 64 \n\t" - "addi s3, %[SRC], 64*2 \n\t" - "addi s4, %[SRC], 64*3 \n\t" - "vle32.v v2, (s2) \n\t" - "vle32.v v4, (s3) \n\t" - "vle32.v v6, (s4) \n\t" - "add t2, %[DST], %[LDC] \n\t" - "add t3, t2, %[LDC] \n\t" - "add t4, t3, %[LDC] \n\t" - "vse32.v v0, (%[DST]) \n\t" - "vse32.v v2, (t2) \n\t" - "vse32.v v4, (t3) \n\t" - "vse32.v v6, (t4) \n\t" - : - : [N] "r"(N), [SRC] "r"(tmp), [DST] "r"(CPtr), [LDC] "r"(LDC) - : "cc", "t0", "t2", "t3", "t4", "s2", "s3", "s4"); - } -} - -template -void SQ4BitGemmM4Kernel_CompInt8_Impl(size_t BlkLen, - const std::byte * QuantA, - const std::byte * QuantBData, - const float * QuantBScale, - const std::byte * QuantBZeroPoint, - float * C, - size_t CountN, - size_t BlockCountK, - const float * Bias, - const size_t ldc) { - GGML_UNUSED(QuantBScale); - GGML_UNUSED(QuantBZeroPoint); - size_t LDC = ldc * sizeof(float); - const size_t INNER = BlkLen / 16; - float tmp[4 * 16]; - if constexpr (HasZeroPoint) { - for (size_t n = 0; n < CountN; n += 16) { - size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n; - std::byte * QuantBDataPtr = (std::byte *) QuantBData + // - n * BlockCountK * BlkLen / 2 + // b data - n * BlockCountK * sizeof(uint8_t) + // zp - n * BlockCountK * sizeof(float); // scale - float * CPtr = C + n; - if (NBLKS < 16) { - CPtr = tmp; - LDC = 16 * sizeof(float); - } - if (Bias != nullptr) { - const float * bias = Bias + n; - if (NBLKS < 16) { - __asm__ volatile( - "vsetvli t0, %[N], e32, m2 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "vse32.v v0, (%[DST]) \n\t" - : - : [SRC] "r"(bias), [DST] "r"(tmp), [N] "r"(NBLKS) - : "cc", "t0"); - bias = tmp; - } - - __asm__ volatile(LOAD_BIAS - "addi t3, %[BlockCountK], 0 \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - "li s1, 24 \n\t" - "vmv.v.i v1, 3 \n\t" - "vsetvli t0, s1, e8, m1 \n\t" - "vmv.v.i v1, 2 \n\t" - "vsetvli t0, zero, e8, mf2 \n\t" - "vmv.v.i v1, 1 \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vmv.v.i v1, 0 \n\t" - "addi a1, %[A], 0 \n\t" - "addi s1, %[B], 0 \n\t" - "BLOCK_COUNTK_LOOP%=: \n\t" - // scale offset - "addi s5, s1, 0 \n\t" - // zp offset - "addi s6, s1, 64 \n\t" - "addi s1, s6, 16 \n\t" - "addi s2, s1, 32 \n\t" - "addi s3, s1, 32*2 \n\t" - "addi s4, s1, 32*3 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - // load a scale - "flw f1, (a1) \n\t" - "flw f2, 4(a1) \n\t" - "flw f3, 8(a1) \n\t" - "flw f4, 12(a1) \n\t" - "addi a1, a1, 16 \n\t" - "addi t2, %[INNER], 0 \n\t" - - SQ4BIT_KERNEL_LOAD_ZP_16X1_v2 - - "BLOCK_INNER_LOOP%=: \n\t" - - LOAD_B_16x8x2 - - "vle8.v v10, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vle8.v v11, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vsub.vv v2, v2, v12 \n\t" - "vsub.vv v6, v6, v12 \n\t" - "vsub.vv v3, v3, v13 \n\t" - "vsub.vv v7, v7, v13 \n\t" - "vsub.vv v4, v4, v14 \n\t" - "vsub.vv v8, v8, v14 \n\t" - "vsub.vv v5, v5, v15 \n\t" - "vsub.vv v9, v9, v15 \n\t" - - SQ4BIT_KERNEL_COMP_4x16x16 - - "addi t2, t2, -1 \n\t" - "bnez t2, BLOCK_INNER_LOOP%= \n\t" - - LOAD_SCALE_4x16 - - "vsetvli t0, zero, e32, m8 \n\t" - "vfcvt.f.x.v v16, v16 \n\t" - "vfmacc.vv v24, v16, v8 \n\t" - "addi t3, t3, -1 \n\t" - "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" - - "RESULT_SAVE%=: \n\t" - - SAVE_RESULT_4x16 - - : - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), - [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr), [BIAS] "r"(bias) - : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", - "s2", "s3", "s4", "s5", "s6"); - - } else { - __asm__ volatile( - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v24, v24, v24 \n\t" - "addi t3, %[BlockCountK], 0 \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - "li s1, 24 \n\t" - "vmv.v.i v1, 3 \n\t" - "vsetvli t0, s1, e8, m1 \n\t" - "vmv.v.i v1, 2 \n\t" - "vsetvli t0, zero, e8, mf2 \n\t" - "vmv.v.i v1, 1 \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vmv.v.i v1, 0 \n\t" - "addi a1, %[A], 0 \n\t" - "addi s1, %[B], 0 \n\t" - "BLOCK_COUNTK_LOOP%=: \n\t" - // scale offset - "addi s5, s1, 0 \n\t" - // zp offset - "addi s6, s1, 64 \n\t" - "addi s1, s6, 16 \n\t" - "addi s2, s1, 32 \n\t" - "addi s3, s1, 32*2 \n\t" - "addi s4, s1, 32*3 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - // load a scale - // load a scale - "flw f1, (a1) \n\t" - "flw f2, 4(a1) \n\t" - "flw f3, 8(a1) \n\t" - "flw f4, 12(a1) \n\t" - "addi a1, a1, 16 \n\t" - "addi t2, %[INNER], 0 \n\t" - - SQ4BIT_KERNEL_LOAD_ZP_16X1_v2 - - "BLOCK_INNER_LOOP%=: \n\t" - - LOAD_B_16x8x2 - - "vle8.v v10, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vle8.v v11, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vsub.vv v2, v2, v12 \n\t" - "vsub.vv v6, v6, v12 \n\t" - "vsub.vv v3, v3, v13 \n\t" - "vsub.vv v7, v7, v13 \n\t" - "vsub.vv v4, v4, v14 \n\t" - "vsub.vv v8, v8, v14 \n\t" - "vsub.vv v5, v5, v15 \n\t" - "vsub.vv v9, v9, v15 \n\t" - - SQ4BIT_KERNEL_COMP_4x16x16 - - "addi t2, t2, -1 \n\t" - "bnez t2, BLOCK_INNER_LOOP%= \n\t" - - LOAD_SCALE_4x16 - - "vsetvli t0, zero, e32, m8 \n\t" - "vfcvt.f.x.v v16, v16 \n\t" - "vfmacc.vv v24, v16, v8 \n\t" - "addi t3, t3, -1 \n\t" - "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" - - "RESULT_SAVE%=: \n\t" - - SAVE_RESULT_4x16 - - : - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), - [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr) - : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", "s2", "s3", - "s4", "s5", "s6"); - } + __asm__ volatile( + "vsetvli t0, zero, e32, m8 \n\t" + "vxor.vv v24, v24, v24 \n\t" + "addi t3, %[BlockCountK], 0 \n\t" + "vsetvli t0, zero, e8, m1 \n\t" + "li s1, 24 \n\t" + "vmv.v.i v1, 3 \n\t" + "vsetvli t0, s1, e8, m1 \n\t" + "vmv.v.i v1, 2 \n\t" + "vsetvli t0, zero, e8, mf2 \n\t" + "vmv.v.i v1, 1 \n\t" + "vsetvli t0, zero, e8, mf4 \n\t" + "vmv.v.i v1, 0 \n\t" + "addi a1, %[A], 0 \n\t" + "addi s1, %[B], 0 \n\t" + "BLOCK_COUNTK_LOOP%=: \n\t" + // scale offset + "addi s5, s1, 0 \n\t" + // zp offset + "addi s6, s1, 32 \n\t" + "addi s1, s6, 16 \n\t" + "addi s2, s1, 32 \n\t" + "addi s3, s1, 32*2 \n\t" + "addi s4, s1, 32*3 \n\t" + + "vsetvli t0, zero, e32, m8 \n\t" + "vxor.vv v16, v16, v16 \n\t" + // load a scale + "flw f1, (a1) \n\t" + "flw f2, 4(a1) \n\t" + "flw f3, 8(a1) \n\t" + "flw f4, 12(a1) \n\t" + "addi a1, a1, 16 \n\t" + "addi t2, %[INNER], 0 \n\t" + + SQ4BIT_KERNEL_LOAD_ZP_16X1_v2 + + "BLOCK_INNER_LOOP%=: \n\t" + + LOAD_B_16x8x2 + + "vle8.v v10, (a1) \n\t" + "addi a1, a1, 32 \n\t" + "vle8.v v11, (a1) \n\t" + "addi a1, a1, 32 \n\t" + "vsub.vv v2, v2, v12 \n\t" + "vsub.vv v6, v6, v12 \n\t" + "vsub.vv v3, v3, v13 \n\t" + "vsub.vv v7, v7, v13 \n\t" + "vsub.vv v4, v4, v14 \n\t" + "vsub.vv v8, v8, v14 \n\t" + "vsub.vv v5, v5, v15 \n\t" + "vsub.vv v9, v9, v15 \n\t" + + SQ4BIT_KERNEL_COMP_4x16x16 + + "addi t2, t2, -1 \n\t" + "bnez t2, BLOCK_INNER_LOOP%= \n\t" + + LOAD_SCALE_4x16_FP16 + + "vsetvli t0, zero, e32, m8 \n\t" + "vfcvt.f.x.v v16, v16 \n\t" + "vfmacc.vv v24, v16, v8 \n\t" + "addi t3, t3, -1 \n\t" + "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" + + "RESULT_SAVE%=: \n\t" + + SAVE_RESULT_4x16 + + : + : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), + [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr) + : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", "s2", "s3", "s4", + "s5", "s6"); } } else { for (size_t n = 0; n < CountN; n += 16) { - size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n; - std::byte * QuantBDataPtr = (std::byte *) QuantBData + // - n * BlockCountK * BlkLen / 2 + // b data - n * BlockCountK * sizeof(float); // scale + size_t NBLKS = (CountN - n) > 16 ? 16 : CountN - n; + uint8_t * QuantBDataPtr = (uint8_t *) QuantBData + // + n * BlockCountK * BlkLen / 2 + // b data + n * BlockCountK * sizeof(_Float16); // scale float * CPtr = C + n; if (NBLKS < 16) { CPtr = tmp; LDC = 16 * sizeof(float); } - if (Bias != nullptr) { - const float * bias = Bias + n; - if (NBLKS < 16) { - __asm__ volatile( - "vsetvli t0, %[N], e32, m2 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "vse32.v v0, (%[DST]) \n\t" - : - : [SRC] "r"(bias), [DST] "r"(tmp), [N] "r"(NBLKS) - : "cc", "t0"); - bias = tmp; - } - __asm__ volatile(LOAD_BIAS - "addi t3, %[BlockCountK], 0 \n\t" - "addi a1, %[A], 0 \n\t" - "addi s1, %[B], 0 \n\t" - "BLOCK_COUNTK_LOOP%=: \n\t" - "addi s5, s1, 0 \n\t" - "addi s1, s5, 64 \n\t" - "addi s2, s1, 32 \n\t" - "addi s3, s1, 32*2 \n\t" - "addi s4, s1, 32*3 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - // load a scale - "flw f1, (a1) \n\t" - "flw f2, 4(a1) \n\t" - "flw f3, 8(a1) \n\t" - "flw f4, 12(a1) \n\t" - "addi a1, a1, 16 \n\t" - "addi t2, %[INNER], 0 \n\t" - "BLOCK_INNER_LOOP%=: \n\t" - - LOAD_B_16x8x2 - - "vsetvli t0, zero, e8, m1 \n\t" - "vle8.v v10, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vle8.v v11, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vadd.vi v2, v2, -8 \n\t" - "vadd.vi v3, v3, -8 \n\t" - "vadd.vi v4, v4, -8 \n\t" - "vadd.vi v5, v5, -8 \n\t" - "vadd.vi v6, v6, -8 \n\t" - "vadd.vi v7, v7, -8 \n\t" - "vadd.vi v8, v8, -8 \n\t" - "vadd.vi v9, v9, -8 \n\t" - - SQ4BIT_KERNEL_COMP_4x16x16 - - "addi t2, t2, -1 \n\t" - "bnez t2, BLOCK_INNER_LOOP%= \n\t" - - LOAD_SCALE_4x16 - - "vsetvli t0, zero, e32, m8 \n\t" - "vfcvt.f.x.v v16, v16 \n\t" - "vfmacc.vv v24, v16, v8 \n\t" - "addi t3, t3, -1 \n\t" - "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" - - "RESULT_SAVE%=: \n\t" - - SAVE_RESULT_4x16 - - : - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), - [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr), [BIAS] "r"(bias) - : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", - "s2", "s3", "s4", "s5", "s6"); - - } else { - __asm__ volatile( - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v24, v24, v24 \n\t" - "addi t3, %[BlockCountK], 0 \n\t" - "addi a1, %[A], 0 \n\t" - "addi s1, %[B], 0 \n\t" - "BLOCK_COUNTK_LOOP%=: \n\t" - "addi s5, s1, 0 \n\t" - "addi s1, s5, 64 \n\t" - "addi s2, s1, 32 \n\t" - "addi s3, s1, 32*2 \n\t" - "addi s4, s1, 32*3 \n\t" - "vsetvli t0, zero, e32, m8 \n\t" - "vxor.vv v16, v16, v16 \n\t" - // load a scale - "flw f1, (a1) \n\t" - "flw f2, 4(a1) \n\t" - "flw f3, 8(a1) \n\t" - "flw f4, 12(a1) \n\t" - "addi a1, a1, 16 \n\t" - "addi t2, %[INNER], 0 \n\t" - "BLOCK_INNER_LOOP%=: \n\t" - - LOAD_B_16x8x2 - - "vsetvli t0, zero, e8, m1 \n\t" - "vle8.v v10, (a1) \n\t" - - "addi a1, a1, 32 \n\t" - "vle8.v v11, (a1) \n\t" - "addi a1, a1, 32 \n\t" - "vadd.vi v2, v2, -8 \n\t" - "vadd.vi v3, v3, -8 \n\t" - "vadd.vi v4, v4, -8 \n\t" - "vadd.vi v5, v5, -8 \n\t" - "vadd.vi v6, v6, -8 \n\t" - "vadd.vi v7, v7, -8 \n\t" - "vadd.vi v8, v8, -8 \n\t" - "vadd.vi v9, v9, -8 \n\t" - - SQ4BIT_KERNEL_COMP_4x16x16 - - "addi t2, t2, -1 \n\t" - "bnez t2, BLOCK_INNER_LOOP%= \n\t" - - LOAD_SCALE_4x16 - - "vsetvli t0, zero, e32, m8 \n\t" - "vfcvt.f.x.v v16, v16 \n\t" - "vfmacc.vv v24, v16, v8 \n\t" - "addi t3, t3, -1 \n\t" - "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" - - "RESULT_SAVE%=: \n\t" - - SAVE_RESULT_4x16 - - : - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), - [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr) - : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", "s2", "s3", - "s4", "s5", "s6"); - } + + __asm__ volatile( + "vsetvli t0, zero, e32, m8 \n\t" + "vxor.vv v24, v24, v24 \n\t" + "addi t3, %[BlockCountK], 0 \n\t" + "addi a1, %[A], 0 \n\t" + "addi s1, %[B], 0 \n\t" + "BLOCK_COUNTK_LOOP%=: \n\t" + "addi s5, s1, 0 \n\t" + "addi s1, s5, 32 \n\t" + "addi s2, s1, 32 \n\t" + "addi s3, s1, 32*2 \n\t" + "addi s4, s1, 32*3 \n\t" + "vsetvli t0, zero, e32, m8 \n\t" + "vxor.vv v16, v16, v16 \n\t" + // load a scale + "flw f1, (a1) \n\t" + "flw f2, 4(a1) \n\t" + "flw f3, 8(a1) \n\t" + "flw f4, 12(a1) \n\t" + "addi a1, a1, 16 \n\t" + "addi t2, %[INNER], 0 \n\t" + "BLOCK_INNER_LOOP%=: \n\t" + + LOAD_B_16x8x2 + + "vsetvli t0, zero, e8, m1 \n\t" + "vle8.v v10, (a1) \n\t" + "addi a1, a1, 32 \n\t" + "vle8.v v11, (a1) \n\t" + "addi a1, a1, 32 \n\t" + "vadd.vi v2, v2, -8 \n\t" + "vadd.vi v3, v3, -8 \n\t" + "vadd.vi v4, v4, -8 \n\t" + "vadd.vi v5, v5, -8 \n\t" + "vadd.vi v6, v6, -8 \n\t" + "vadd.vi v7, v7, -8 \n\t" + "vadd.vi v8, v8, -8 \n\t" + "vadd.vi v9, v9, -8 \n\t" + + SQ4BIT_KERNEL_COMP_4x16x16 + + "addi t2, t2, -1 \n\t" + "bnez t2, BLOCK_INNER_LOOP%= \n\t" + + LOAD_SCALE_4x16_FP16 + + "vsetvli t0, zero, e32, m8 \n\t" + "vfcvt.f.x.v v16, v16 \n\t" + "vfmacc.vv v24, v16, v8 \n\t" + "addi t3, t3, -1 \n\t" + "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" + "RESULT_SAVE%=: \n\t" + + SAVE_RESULT_4x16 + + : + : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), + [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr) + : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", "s1", "s2", "s3", "s4", + "s5", "s6"); } } - if (CountN % 16 != 0) { - // stroe output from tmp to C when NBLKS less than 16. - float * CPtr = C + CountN / 16 * 16; - const size_t N = CountN % 16; - LDC = ldc * sizeof(float); - __asm__ volatile( - "vsetvli t0, %[N], e32, m2 \n\t" - "vle32.v v0, (%[SRC]) \n\t" - "addi s2, %[SRC], 64 \n\t" - "addi s3, %[SRC], 64*2 \n\t" - "addi s4, %[SRC], 64*3 \n\t" - "vle32.v v2, (s2) \n\t" - "vle32.v v4, (s3) \n\t" - "vle32.v v6, (s4) \n\t" - "add t2, %[DST], %[LDC] \n\t" - "add t3, t2, %[LDC] \n\t" - "add t4, t3, %[LDC] \n\t" - "vse32.v v0, (%[DST]) \n\t" - "vse32.v v2, (t2) \n\t" - "vse32.v v4, (t3) \n\t" - "vse32.v v6, (t4) \n\t" - : - : [N] "r"(N), [SRC] "r"(tmp), [DST] "r"(CPtr), [LDC] "r"(LDC) - : "cc", "t0", "t2", "t3", "t4", "s2", "s3", "s4"); - } } template -void SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(size_t BlkLen, - const std::byte * QuantA, - const std::byte * QuantBData, - const float * QuantBScale, - const std::byte * QuantBZeroPoint, - float * C, - size_t CountN, - size_t BlockCountK, - const float * Bias) { - GGML_UNUSED(QuantBScale); - GGML_UNUSED(QuantBZeroPoint); +void SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(size_t BlkLen, + const uint8_t * QuantA, + const uint8_t * QuantBData, + float * C, + size_t CountN, + size_t BlockCountK, + const size_t ldc) { + GGML_UNUSED(ldc); size_t INNER = BlkLen / 16; if constexpr (HasZeroPoint) { for (size_t n = 0; n < CountN; n += 16) { - size_t nblks = (CountN - n) > 16 ? 16 : CountN - n; - std::byte * QuantBDataPtr = (std::byte *) QuantBData + // - n * BlockCountK * BlkLen / 2 + // b data - n * BlockCountK * sizeof(uint8_t) + // zp - n * BlockCountK * sizeof(_Float16); // scale + size_t nblks = (CountN - n) > 16 ? 16 : CountN - n; + uint8_t * QuantBDataPtr = (uint8_t *) QuantBData + // + n * BlockCountK * BlkLen / 2 + // b data + n * BlockCountK * sizeof(uint8_t) + // zp + n * BlockCountK * sizeof(_Float16); // scale float * CPtr = C + n; size_t cnt = BlockCountK; - if (Bias != nullptr) { - const float * bias = Bias + n; - __asm__ volatile( - "addi t3, %[NBLKS], 0 \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - - "vmv.v.i v13, 3 \n\t" - "li s1, 24 \n\t" - "vsetvli t0, s1, e8, m1 \n\t" - "vmv.v.i v13, 2 \n\t" - "vsetvli t0, zero, e8, mf2 \n\t" - "vmv.v.i v13, 1 \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vmv.v.i v13, 0 \n\t" - "addi s1, %[B], 0 \n\t" - "addi s2, %[B], 8 \n\t" - "addi s3, %[B], 16 \n\t" - "addi s4, %[B], 24 \n\t" - // zp offset - "addi s7, %[B], 32 \n\t" - // a offset - "addi s5, %[A], 0 \n\t" - "addi s6, %[A], 12 \n\t" - - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v28, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v29, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v30, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v31, (%[BIAS]) \n\t" - - "LOOP_K%=: \n\t" - "vsetvli t0, zero, e16, mf4 \n\t" - - "vle16.v v4, (s1) \n\t" - "addi s1, s1, 48 \n\t" - "vle16.v v5, (s2) \n\t" - "addi s2, s2, 72 \n\t" - "vle16.v v6, (s3) \n\t" - "addi s3, s3, 96 \n\t" - "vle16.v v7, (s4) \n\t" - "addi s4, s4, 120 \n\t" - "flw f1, (s5) \n\t" - "addi s5, s5, 4 \n\t" - "vfwcvt.f.f.v v8, v4 \n\t" - "vfwcvt.f.f.v v9, v5 \n\t" - "vfwcvt.f.f.v v10, v6 \n\t" - "vfwcvt.f.f.v v11, v7 \n\t" - - "vsetvli t0, zero, e32, mf2 \n\t" - "addi t5, %[INNER], 0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v18, v18, v18 \n\t" - "vxor.vv v20, v20, v20 \n\t" - "vxor.vv v22, v22, v22 \n\t" - "vfmul.vf v24, v8, f1 \n\t" - "vfmul.vf v25, v9, f1 \n\t" - "vfmul.vf v26, v10, f1 \n\t" - "vfmul.vf v27, v11, f1 \n\t" - "addi %[CNT], %[CNT], -1 \n\t" - - SQ4BIT_KERNEL_LOAD_ZP_16X1 - - "LOOP_INNER%=: \n\t" - - SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 - - "vsub.vv v0, v0, v8 \n\t" - "vsub.vv v4, v4, v8 \n\t" - "vsub.vv v1, v1, v9 \n\t" - "vsub.vv v5, v5, v9 \n\t" - "vsub.vv v2, v2, v10 \n\t" - "vsub.vv v6, v6, v10 \n\t" - "vsub.vv v3, v3, v11 \n\t" - "vsub.vv v7, v7, v11 \n\t" - - SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 - - "bnez t5, LOOP_INNER%= \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - SQ4BIT_KERNEL_ACC_F16_1X4X4 - "addi s7, s1, 32 \n\t" - - "bnez %[CNT], LOOP_K%= \n\t" - "addi t3, zero, 16 \n\t" - "addi s1, %[C], 16 \n\t" - "addi s2, %[C], 32 \n\t" - "addi s3, %[C], 48 \n\t" - "blt %[NBLKS], t3, ST_TAIL%= \n\t" - "vse32.v v28, (%[C]) \n\t" - "vse32.v v29, (s1) \n\t" - "vse32.v v30, (s2) \n\t" - "vse32.v v31, (s3) \n\t" - "jal x0, END%= \n\t" - - "ST_TAIL%=: \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v28, (%[C]) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v29, (s1) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v30, (s2) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v31, (s3) \n\t" - "END%=: \n\t" - - : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks), [BIAS] "+r"(bias) - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) - : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6", "s7"); - } else { - __asm__ volatile( - "vsetvli t0, zero, e32, m4 \n\t" - "vxor.vv v28, v28, v28 \n\t" - - "vsetvli t0, zero, e8, m1 \n\t" - "vmv.v.i v13, 3 \n\t" - "li s1, 24 \n\t" - "vsetvli t0, s1, e8, m1 \n\t" - "vmv.v.i v13, 2 \n\t" - "vsetvli t0, zero, e8, mf2 \n\t" - "vmv.v.i v13, 1 \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vmv.v.i v13, 0 \n\t" - - "addi s1, %[B], 0 \n\t" - "addi s2, %[B], 8 \n\t" - "addi s3, %[B], 16 \n\t" - "addi s4, %[B], 24 \n\t" - - "addi s7, %[B], 32 \n\t" - - "addi s5, %[A], 0 \n\t" - "addi s6, %[A], 12 \n\t" - "LOOP_K%=: \n\t" - "vsetvli t0, zero, e16, mf4 \n\t" - "vle16.v v4, (s1) \n\t" - "addi s1, s1, 48 \n\t" - "vle16.v v5, (s2) \n\t" - "addi s2, s2, 72 \n\t" - "vle16.v v6, (s3) \n\t" - "addi s3, s3, 96 \n\t" - "vle16.v v7, (s4) \n\t" - "addi s4, s4, 120 \n\t" - "flw f1, (s5) \n\t" - "addi s5, s5, 4 \n\t" - - "vfwcvt.f.f.v v8, v4 \n\t" - "vfwcvt.f.f.v v9, v5 \n\t" - "vfwcvt.f.f.v v10, v6 \n\t" - "vfwcvt.f.f.v v11, v7 \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - "addi t5, %[INNER], 0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v18, v18, v18 \n\t" - "vxor.vv v20, v20, v20 \n\t" - "vxor.vv v22, v22, v22 \n\t" - "vfmul.vf v24, v8, f1 \n\t" - "vfmul.vf v25, v9, f1 \n\t" - "vfmul.vf v26, v10, f1 \n\t" - "vfmul.vf v27, v11, f1 \n\t" - "addi %[CNT], %[CNT], -1 \n\t" - - SQ4BIT_KERNEL_LOAD_ZP_16X1 - - "LOOP_INNER%=: \n\t" - - SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 - - "vsub.vv v0, v0, v8 \n\t" - "vsub.vv v4, v4, v8 \n\t" - "vsub.vv v1, v1, v9 \n\t" - "vsub.vv v5, v5, v9 \n\t" - "vsub.vv v2, v2, v10 \n\t" - "vsub.vv v6, v6, v10 \n\t" - "vsub.vv v3, v3, v11 \n\t" - "vsub.vv v7, v7, v11 \n\t" - - SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 - - "bnez t5, LOOP_INNER%= \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - SQ4BIT_KERNEL_ACC_F16_1X4X4 - "addi s7, s1, 32 \n\t" - - "bnez %[CNT], LOOP_K%= \n\t" - "addi t3, zero, 16 \n\t" - "addi s1, %[C], 16 \n\t" - "addi s2, %[C], 32 \n\t" - "addi s3, %[C], 48 \n\t" - "blt %[NBLKS], t3, ST_TAIL%= \n\t" - "vse32.v v28, (%[C]) \n\t" - "vse32.v v29, (s1) \n\t" - "vse32.v v30, (s2) \n\t" - "vse32.v v31, (s3) \n\t" - "jal x0, END%= \n\t" - - "ST_TAIL%=: \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v28, (%[C]) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v29, (s1) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v30, (s2) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v31, (s3) \n\t" - "END%=: \n\t" - - : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks) - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) - : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6", "s7"); - } + + __asm__ volatile( + "vsetvli t0, zero, e32, m4 \n\t" + "vxor.vv v28, v28, v28 \n\t" + + "vsetvli t0, zero, e8, m1 \n\t" + "vmv.v.i v13, 3 \n\t" + "li s1, 24 \n\t" + "vsetvli t0, s1, e8, m1 \n\t" + "vmv.v.i v13, 2 \n\t" + "vsetvli t0, zero, e8, mf2 \n\t" + "vmv.v.i v13, 1 \n\t" + "vsetvli t0, zero, e8, mf4 \n\t" + "vmv.v.i v13, 0 \n\t" + + "addi s1, %[B], 0 \n\t" + "addi s2, %[B], 8 \n\t" + "addi s3, %[B], 16 \n\t" + "addi s4, %[B], 24 \n\t" + + "addi s7, %[B], 32 \n\t" + + "addi s5, %[A], 0 \n\t" + "addi s6, %[A], 12 \n\t" + "LOOP_K%=: \n\t" + "vsetvli t0, zero, e16, mf4 \n\t" + "vle16.v v4, (s1) \n\t" + "addi s1, s1, 48 \n\t" + "vle16.v v5, (s2) \n\t" + "addi s2, s2, 72 \n\t" + "vle16.v v6, (s3) \n\t" + "addi s3, s3, 96 \n\t" + "vle16.v v7, (s4) \n\t" + "addi s4, s4, 120 \n\t" + "flw f1, (s5) \n\t" + "addi s5, s5, 4 \n\t" + + "vfwcvt.f.f.v v8, v4 \n\t" + "vfwcvt.f.f.v v9, v5 \n\t" + "vfwcvt.f.f.v v10, v6 \n\t" + "vfwcvt.f.f.v v11, v7 \n\t" + "vsetvli t0, zero, e32, mf2 \n\t" + + "addi t5, %[INNER], 0 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v18, v18 \n\t" + "vxor.vv v20, v20, v20 \n\t" + "vxor.vv v22, v22, v22 \n\t" + "vfmul.vf v24, v8, f1 \n\t" + "vfmul.vf v25, v9, f1 \n\t" + "vfmul.vf v26, v10, f1 \n\t" + "vfmul.vf v27, v11, f1 \n\t" + "addi %[CNT], %[CNT], -1 \n\t" + + SQ4BIT_KERNEL_LOAD_ZP_16X1 + + "LOOP_INNER%=: \n\t" + + SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 + + "vsub.vv v0, v0, v8 \n\t" + "vsub.vv v4, v4, v8 \n\t" + "vsub.vv v1, v1, v9 \n\t" + "vsub.vv v5, v5, v9 \n\t" + "vsub.vv v2, v2, v10 \n\t" + "vsub.vv v6, v6, v10 \n\t" + "vsub.vv v3, v3, v11 \n\t" + "vsub.vv v7, v7, v11 \n\t" + + SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 + + "bnez t5, LOOP_INNER%= \n\t" + "vsetvli t0, zero, e32, mf2 \n\t" + + SQ4BIT_KERNEL_ACC_F16_1X4X4 + "addi s7, s1, 32 \n\t" + + "bnez %[CNT], LOOP_K%= \n\t" + "addi t3, zero, 16 \n\t" + "addi s1, %[C], 16 \n\t" + "addi s2, %[C], 32 \n\t" + "addi s3, %[C], 48 \n\t" + "blt %[NBLKS], t3, ST_TAIL%= \n\t" + "vse32.v v28, (%[C]) \n\t" + "vse32.v v29, (s1) \n\t" + "vse32.v v30, (s2) \n\t" + "vse32.v v31, (s3) \n\t" + "jal x0, END%= \n\t" + + "ST_TAIL%=: \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v28, (%[C]) \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v29, (s1) \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v30, (s2) \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v31, (s3) \n\t" + "END%=: \n\t" + + : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks) + : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) + : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6", "s7"); } } else { for (size_t n = 0; n < CountN; n += 16) { - size_t nblks = (CountN - n) > 16 ? 16 : CountN - n; - std::byte * QuantBDataPtr = (std::byte *) QuantBData + // - n * BlockCountK * BlkLen / 2 + // b data - n * BlockCountK * sizeof(_Float16); // scale + size_t nblks = (CountN - n) > 16 ? 16 : CountN - n; + uint8_t * QuantBDataPtr = (uint8_t *) QuantBData + // + n * BlockCountK * BlkLen / 2 + // b data + n * BlockCountK * sizeof(_Float16); // scale float * CPtr = C + n; size_t cnt = BlockCountK; - if (Bias != nullptr) { - const float * bias = Bias + n; - __asm__ volatile( - "addi t3, %[NBLKS], 0 \n\t" - "addi s1, %[B], 0 \n\t" - "addi s2, %[B], 8 \n\t" - "addi s3, %[B], 16 \n\t" - "addi s4, %[B], 24 \n\t" - "addi s5, %[A], 0 \n\t" - "addi s6, %[A], 12 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v28, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v29, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v30, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v31, (%[BIAS]) \n\t" - - "LOOP_K%=: \n\t" - "vsetvli t0, zero, e16, mf4 \n\t" - - "vle16.v v4, (s1) \n\t" - "addi s1, s1, 32 \n\t" - "vle16.v v5, (s2) \n\t" - "addi s2, s2, 56 \n\t" - "vle16.v v6, (s3) \n\t" - "addi s3, s3, 80 \n\t" - "vle16.v v7, (s4) \n\t" - "addi s4, s4, 104 \n\t" - "flw f1, (s5) \n\t" - "addi s5, s5, 4 \n\t" - "vfwcvt.f.f.v v8, v4 \n\t" - "vfwcvt.f.f.v v9, v5 \n\t" - "vfwcvt.f.f.v v10, v6 \n\t" - "vfwcvt.f.f.v v11, v7 \n\t" - - "vsetvli t0, zero, e32, mf2 \n\t" - "addi t5, %[INNER], 0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v18, v18, v18 \n\t" - "vxor.vv v20, v20, v20 \n\t" - "vxor.vv v22, v22, v22 \n\t" - "vfmul.vf v24, v8, f1 \n\t" - "vfmul.vf v25, v9, f1 \n\t" - "vfmul.vf v26, v10, f1 \n\t" - "vfmul.vf v27, v11, f1 \n\t" - "addi %[CNT], %[CNT], -1 \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - "LOOP_INNER%=: \n\t" - - SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 - - "vadd.vi v0, v0, -8 \n\t" - "vadd.vi v1, v1, -8 \n\t" - "vadd.vi v2, v2, -8 \n\t" - "vadd.vi v3, v3, -8 \n\t" - "vadd.vi v4, v4, -8 \n\t" - "vadd.vi v5, v5, -8 \n\t" - "vadd.vi v6, v6, -8 \n\t" - "vadd.vi v7, v7, -8 \n\t" - - SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 - - "bnez t5, LOOP_INNER%= \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - SQ4BIT_KERNEL_ACC_F16_1X4X4 - - "bnez %[CNT], LOOP_K%= \n\t" - "addi t3, zero, 16 \n\t" - "addi s1, %[C], 16 \n\t" - "addi s2, %[C], 32 \n\t" - "addi s3, %[C], 48 \n\t" - "blt %[NBLKS], t3, ST_TAIL%= \n\t" - "vse32.v v28, (%[C]) \n\t" - "vse32.v v29, (s1) \n\t" - "vse32.v v30, (s2) \n\t" - "vse32.v v31, (s3) \n\t" - "jal x0, END%= \n\t" - - "ST_TAIL%=: \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v28, (%[C]) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v29, (s1) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v30, (s2) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v31, (s3) \n\t" - "END%=: \n\t" - - : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks), [BIAS] "+r"(bias) - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) - : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6"); - } else { - __asm__ volatile( - "vsetvli t0, zero, e32, m4 \n\t" - "vxor.vv v28, v28, v28 \n\t" - "addi s1, %[B], 0 \n\t" - "addi s2, %[B], 8 \n\t" - "addi s3, %[B], 16 \n\t" - "addi s4, %[B], 24 \n\t" - - "addi s5, %[A], 0 \n\t" - "addi s6, %[A], 12 \n\t" - "LOOP_K%=: \n\t" - "vsetvli t0, zero, e16, mf4 \n\t" - "vle16.v v4, (s1) \n\t" - "addi s1, s1, 32 \n\t" - "vle16.v v5, (s2) \n\t" - "addi s2, s2, 56 \n\t" - "vle16.v v6, (s3) \n\t" - "addi s3, s3, 80 \n\t" - "vle16.v v7, (s4) \n\t" - "addi s4, s4, 104 \n\t" - "flw f1, (s5) \n\t" - "addi s5, s5, 4 \n\t" - - "vfwcvt.f.f.v v8, v4 \n\t" - "vfwcvt.f.f.v v9, v5 \n\t" - "vfwcvt.f.f.v v10, v6 \n\t" - "vfwcvt.f.f.v v11, v7 \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - "addi t5, %[INNER], 0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v18, v18, v18 \n\t" - "vxor.vv v20, v20, v20 \n\t" - "vxor.vv v22, v22, v22 \n\t" - "vfmul.vf v24, v8, f1 \n\t" - "vfmul.vf v25, v9, f1 \n\t" - "vfmul.vf v26, v10, f1 \n\t" - "vfmul.vf v27, v11, f1 \n\t" - "addi %[CNT], %[CNT], -1 \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - "LOOP_INNER%=: \n\t" - - SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 - - "vadd.vi v0, v0, -8 \n\t" - "vadd.vi v1, v1, -8 \n\t" - "vadd.vi v2, v2, -8 \n\t" - "vadd.vi v3, v3, -8 \n\t" - "vadd.vi v4, v4, -8 \n\t" - "vadd.vi v5, v5, -8 \n\t" - "vadd.vi v6, v6, -8 \n\t" - "vadd.vi v7, v7, -8 \n\t" - - SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 - - "bnez t5, LOOP_INNER%= \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - SQ4BIT_KERNEL_ACC_F16_1X4X4 - - "bnez %[CNT], LOOP_K%= \n\t" - "addi t3, zero, 16 \n\t" - "addi s1, %[C], 16 \n\t" - "addi s2, %[C], 32 \n\t" - "addi s3, %[C], 48 \n\t" - "blt %[NBLKS], t3, ST_TAIL%= \n\t" - "vse32.v v28, (%[C]) \n\t" - "vse32.v v29, (s1) \n\t" - "vse32.v v30, (s2) \n\t" - "vse32.v v31, (s3) \n\t" - "jal x0, END%= \n\t" - - "ST_TAIL%=: \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v28, (%[C]) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v29, (s1) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v30, (s2) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v31, (s3) \n\t" - "END%=: \n\t" - - : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks) - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) - : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6"); - } + + __asm__ volatile( + "vsetvli t0, zero, e32, m4 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "addi s1, %[B], 0 \n\t" + "addi s2, %[B], 8 \n\t" + "addi s3, %[B], 16 \n\t" + "addi s4, %[B], 24 \n\t" + + "addi s5, %[A], 0 \n\t" + "addi s6, %[A], 12 \n\t" + "LOOP_K%=: \n\t" + "vsetvli t0, zero, e16, mf4 \n\t" + "vle16.v v4, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle16.v v5, (s2) \n\t" + "addi s2, s2, 56 \n\t" + "vle16.v v6, (s3) \n\t" + "addi s3, s3, 80 \n\t" + "vle16.v v7, (s4) \n\t" + "addi s4, s4, 104 \n\t" + "flw f1, (s5) \n\t" + "addi s5, s5, 4 \n\t" + + "vfwcvt.f.f.v v8, v4 \n\t" + "vfwcvt.f.f.v v9, v5 \n\t" + "vfwcvt.f.f.v v10, v6 \n\t" + "vfwcvt.f.f.v v11, v7 \n\t" + "vsetvli t0, zero, e32, mf2 \n\t" + + "addi t5, %[INNER], 0 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v18, v18 \n\t" + "vxor.vv v20, v20, v20 \n\t" + "vxor.vv v22, v22, v22 \n\t" + "vfmul.vf v24, v8, f1 \n\t" + "vfmul.vf v25, v9, f1 \n\t" + "vfmul.vf v26, v10, f1 \n\t" + "vfmul.vf v27, v11, f1 \n\t" + "addi %[CNT], %[CNT], -1 \n\t" + "vsetvli t0, zero, e8, m1 \n\t" + "LOOP_INNER%=: \n\t" + + SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 + + "vadd.vi v0, v0, -8 \n\t" + "vadd.vi v1, v1, -8 \n\t" + "vadd.vi v2, v2, -8 \n\t" + "vadd.vi v3, v3, -8 \n\t" + "vadd.vi v4, v4, -8 \n\t" + "vadd.vi v5, v5, -8 \n\t" + "vadd.vi v6, v6, -8 \n\t" + "vadd.vi v7, v7, -8 \n\t" + + SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 + + "bnez t5, LOOP_INNER%= \n\t" + "vsetvli t0, zero, e32, mf2 \n\t" + + SQ4BIT_KERNEL_ACC_F16_1X4X4 + + "bnez %[CNT], LOOP_K%= \n\t" + "addi t3, zero, 16 \n\t" + "addi s1, %[C], 16 \n\t" + "addi s2, %[C], 32 \n\t" + "addi s3, %[C], 48 \n\t" + "blt %[NBLKS], t3, ST_TAIL%= \n\t" + "vse32.v v28, (%[C]) \n\t" + "vse32.v v29, (s1) \n\t" + "vse32.v v30, (s2) \n\t" + "vse32.v v31, (s3) \n\t" + "jal x0, END%= \n\t" + + "ST_TAIL%=: \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v28, (%[C]) \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v29, (s1) \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v30, (s2) \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v31, (s3) \n\t" + "END%=: \n\t" + + : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks) + : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) + : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6"); } } } -template -void SQ4BitGemmM1Kernel_CompInt8_Impl(size_t BlkLen, - const std::byte * QuantA, - const std::byte * QuantBData, - const float * QuantBScale, - const std::byte * QuantBZeroPoint, - float * C, - size_t CountN, - size_t BlockCountK, - const float * Bias) { - GGML_UNUSED(QuantBScale); - GGML_UNUSED(QuantBZeroPoint); + +// ---- Q8_0 IME1 int8xint8 M4 kernel ---------------------------------------- +// Handles 4 rows at once using the 4-row interleaved A produced by quantize_a_4row_i8: +// per K-block (stride 144B) the first 16B are the four fp32 row scales and the following +// 128B are four 32B chunks, each holding 8 K-values for each of the 4 rows. That maps +// directly onto vmadot's 4-lane row dimension, so the A side is identical to Q4_0. +// B is block_q8_0x16 as in the M1 kernel: 32B of fp16 scales then 512B of interleaved +// int8, loaded with 8 plain vle8 into v2..v9 (no nibble unpack). +static void SQ8BitGemmM4Kernel_CompInt8_ScaleFp16_Impl(size_t BlkLen, + const uint8_t * QuantA, + const uint8_t * QuantBData, + float * C, + size_t CountN, + size_t BlockCountK, + const size_t ldc) { + // Same invariant as the Q4_0 M4 kernel: SAVE_RESULT_4x16 stores a full 4x16 tile with no tail + // handling. Q8_0 is only admitted to the IME1 path when ne[1] % 16 == 0 (ime.cpp), + // and the n-tiling step is NB_COLS == 16, so a partial tile never reaches here. + GGML_ASSERT(CountN % 16 == 0); + const size_t INNER = BlkLen / 16; - if constexpr (HasZeroPoint) { - for (size_t n = 0; n < CountN; n += 16) { - size_t nblks = (CountN - n) > 16 ? 16 : CountN - n; - std::byte * QuantBDataPtr = (std::byte *) QuantBData + // - n * BlockCountK * BlkLen / 2 + // b data - n * BlockCountK * sizeof(uint8_t) + // zp - n * BlockCountK * sizeof(float); // scale - float * CPtr = C + n; - size_t cnt = BlockCountK; - if (Bias != nullptr) { - const float * bias = Bias + n; - __asm__ volatile( - "addi t3, %[NBLKS], 0 \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - "vmv.v.i v13, 3 \n\t" - "li s1, 24 \n\t" - "vsetvli t0, s1, e8, m1 \n\t" - "vmv.v.i v13, 2 \n\t" - "vsetvli t0, zero, e8, mf2 \n\t" - "vmv.v.i v13, 1 \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vmv.v.i v13, 0 \n\t" - "vsetvli t0, zero, e32, m4 \n\t" - "vxor.vv v28, v28, v28 \n\t" - - // scale offset, scale0.0, scale1.0, scale2.0, scale3.0....scale15.0 - "addi s1, %[B], 0 \n\t" - "addi s2, %[B], 16 \n\t" - "addi s3, %[B], 32 \n\t" - "addi s4, %[B], 48 \n\t" - // zp offset - "addi s7, %[B], 64 \n\t" - // a offset - "addi s5, %[A], 0 \n\t" - "addi s6, %[A], 12 \n\t" - - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v28, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v29, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v30, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v31, (%[BIAS]) \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - "LOOP_K%=: \n\t" - - // load scale - "vle32.v v8, (s1) \n\t" - "addi s1, s1, 80 \n\t" - "vle32.v v9, (s2) \n\t" - "addi s2, s2, 96 \n\t" - "vle32.v v10, (s3) \n\t" - "addi s3, s3, 112 \n\t" - "vle32.v v11, (s4) \n\t" - "addi s4, s4, 128 \n\t" - - // load a scale - "flw f1, (s5) \n\t" - "addi s5, s5, 4 \n\t" - - "addi t5, %[INNER], 0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v18, v18, v18 \n\t" - "vxor.vv v20, v20, v20 \n\t" - "vxor.vv v22, v22, v22 \n\t" - - // a scale * b scale - "vfmul.vf v24, v8, f1 \n\t" - "vfmul.vf v25, v9, f1 \n\t" - "vfmul.vf v26, v10, f1 \n\t" - "vfmul.vf v27, v11, f1 \n\t" - "addi %[CNT], %[CNT], -1 \n\t" - - SQ4BIT_KERNEL_LOAD_ZP_16X1 - - "LOOP_INNER%=: \n\t" - - SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 - - "vsub.vv v0, v0, v8 \n\t" - "vsub.vv v4, v4, v8 \n\t" - "vsub.vv v1, v1, v9 \n\t" - "vsub.vv v5, v5, v9 \n\t" - "vsub.vv v2, v2, v10 \n\t" - "vsub.vv v6, v6, v10 \n\t" - "vsub.vv v3, v3, v11 \n\t" - "vsub.vv v7, v7, v11 \n\t" - - SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 - - "bnez t5, LOOP_INNER%= \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - SQ4BIT_KERNEL_ACC_1X4X4 - "addi s7, s1, 64 \n\t" - - "bnez %[CNT], LOOP_K%= \n\t" - - "addi t3, zero, 16 \n\t" - "addi s1, %[C], 16 \n\t" - "addi s2, %[C], 32 \n\t" - "addi s3, %[C], 48 \n\t" - "blt %[NBLKS], t3, ST_TAIL%= \n\t" - "vse32.v v28, (%[C]) \n\t" - "vse32.v v29, (s1) \n\t" - "vse32.v v30, (s2) \n\t" - "vse32.v v31, (s3) \n\t" - "jal x0, END%= \n\t" - - "ST_TAIL%=: \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v28, (%[C]) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v29, (s1) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v30, (s2) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v31, (s3) \n\t" - "END%=: \n\t" - - : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks), [BIAS] "+r"(bias) - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) - : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6", "s7"); - } else { - __asm__ volatile( - "vsetvli t0, zero, e32, m4 \n\t" - "vxor.vv v28, v28, v28 \n\t" - - "vsetvli t0, zero, e8, m1 \n\t" - "vmv.v.i v13, 3 \n\t" - "li s1, 24 \n\t" - "vsetvli t0, s1, e8, m1 \n\t" - "vmv.v.i v13, 2 \n\t" - "vsetvli t0, zero, e8, mf2 \n\t" - "vmv.v.i v13, 1 \n\t" - "vsetvli t0, zero, e8, mf4 \n\t" - "vmv.v.i v13, 0 \n\t" - "addi s1, %[B], 0 \n\t" - "addi s2, %[B], 16 \n\t" - "addi s3, %[B], 32 \n\t" - "addi s4, %[B], 48 \n\t" - - "addi s7, %[B], 64 \n\t" - - "addi s5, %[A], 0 \n\t" - "addi s6, %[A], 12 \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - "LOOP_K%=: \n\t" - "vle32.v v8, (s1) \n\t" - "addi s1, s1, 80 \n\t" - "vle32.v v9, (s2) \n\t" - "addi s2, s2, 96 \n\t" - "vle32.v v10, (s3) \n\t" - "addi s3, s3, 112 \n\t" - "vle32.v v11, (s4) \n\t" - "addi s4, s4, 128 \n\t" - - "flw f1, (s5) \n\t" - "addi s5, s5, 4 \n\t" - - "addi t5, %[INNER], 0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v18, v18, v18 \n\t" - "vxor.vv v20, v20, v20 \n\t" - "vxor.vv v22, v22, v22 \n\t" - - "vfmul.vf v24, v8, f1 \n\t" - "vfmul.vf v25, v9, f1 \n\t" - "vfmul.vf v26, v10, f1 \n\t" - "vfmul.vf v27, v11, f1 \n\t" - "addi %[CNT], %[CNT], -1 \n\t" - - SQ4BIT_KERNEL_LOAD_ZP_16X1 - - "LOOP_INNER%=: \n\t" - - SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 - - "vsub.vv v0, v0, v8 \n\t" - "vsub.vv v4, v4, v8 \n\t" - "vsub.vv v1, v1, v9 \n\t" - "vsub.vv v5, v5, v9 \n\t" - "vsub.vv v2, v2, v10 \n\t" - "vsub.vv v6, v6, v10 \n\t" - "vsub.vv v3, v3, v11 \n\t" - "vsub.vv v7, v7, v11 \n\t" - - SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 - - "bnez t5, LOOP_INNER%= \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - SQ4BIT_KERNEL_ACC_1X4X4 - "addi s7, s1, 64 \n\t" - - "bnez %[CNT], LOOP_K%= \n\t" - - "addi t3, zero, 16 \n\t" - "addi s1, %[C], 16 \n\t" - "addi s2, %[C], 32 \n\t" - "addi s3, %[C], 48 \n\t" - "blt %[NBLKS], t3, ST_TAIL%= \n\t" - "vse32.v v28, (%[C]) \n\t" - "vse32.v v29, (s1) \n\t" - "vse32.v v30, (s2) \n\t" - "vse32.v v31, (s3) \n\t" - "jal x0, END%= \n\t" - - "ST_TAIL%=: \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v28, (%[C]) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v29, (s1) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v30, (s2) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v31, (s3) \n\t" - "END%=: \n\t" - - : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks) - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) - : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6", "s7"); - } - } - } else { - for (size_t n = 0; n < CountN; n += 16) { - size_t nblks = (CountN - n) > 16 ? 16 : CountN - n; - std::byte * QuantBDataPtr = (std::byte *) QuantBData + // - n * BlockCountK * BlkLen / 2 + // b data - n * BlockCountK * sizeof(float); // scale - float * CPtr = C + n; - size_t cnt = BlockCountK; - if (Bias != nullptr) { - const float * bias = Bias + n; - __asm__ volatile( - "addi t3, %[NBLKS], 0 \n\t" - "addi s1, %[B], 0 \n\t" - "addi s2, %[B], 16 \n\t" - "addi s3, %[B], 32 \n\t" - "addi s4, %[B], 48 \n\t" - "addi s5, %[A], 0 \n\t" - "addi s6, %[A], 12 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v28, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v29, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v30, (%[BIAS]) \n\t" - "sub t3, t3, t0 \n\t" - "addi %[BIAS], %[BIAS], 16 \n\t" - "vsetvli t0, t3, e32, mf2 \n\t" - "vle32.v v31, (%[BIAS]) \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - "LOOP_K%=: \n\t" - "vle32.v v8, (s1) \n\t" - "addi s1, s1, 64 \n\t" - "vle32.v v9, (s2) \n\t" - "addi s2, s2, 80 \n\t" - "vle32.v v10, (s3) \n\t" - "addi s3, s3, 96 \n\t" - "vle32.v v11, (s4) \n\t" - "addi s4, s4, 112 \n\t" - "flw f1, (s5) \n\t" - "addi s5, s5, 4 \n\t" - - "addi t5, %[INNER], 0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v18, v18, v18 \n\t" - "vxor.vv v20, v20, v20 \n\t" - "vxor.vv v22, v22, v22 \n\t" - "vfmul.vf v24, v8, f1 \n\t" - "vfmul.vf v25, v9, f1 \n\t" - "vfmul.vf v26, v10, f1 \n\t" - "vfmul.vf v27, v11, f1 \n\t" - "addi %[CNT], %[CNT], -1 \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - "LOOP_INNER%=: \n\t" - - SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 - - "vadd.vi v0, v0, -8 \n\t" - "vadd.vi v1, v1, -8 \n\t" - "vadd.vi v2, v2, -8 \n\t" - "vadd.vi v3, v3, -8 \n\t" - "vadd.vi v4, v4, -8 \n\t" - "vadd.vi v5, v5, -8 \n\t" - "vadd.vi v6, v6, -8 \n\t" - "vadd.vi v7, v7, -8 \n\t" - - SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 - - "bnez t5, LOOP_INNER%= \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - SQ4BIT_KERNEL_ACC_1X4X4 - - "bnez %[CNT], LOOP_K%= \n\t" - "addi t3, zero, 16 \n\t" - "addi s1, %[C], 16 \n\t" - "addi s2, %[C], 32 \n\t" - "addi s3, %[C], 48 \n\t" - "blt %[NBLKS], t3, ST_TAIL%= \n\t" - "vse32.v v28, (%[C]) \n\t" - "vse32.v v29, (s1) \n\t" - "vse32.v v30, (s2) \n\t" - "vse32.v v31, (s3) \n\t" - "jal x0, END%= \n\t" - - "ST_TAIL%=: \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v28, (%[C]) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v29, (s1) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v30, (s2) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v31, (s3) \n\t" - "END%=: \n\t" - - : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks), [BIAS] "+r"(bias) - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) - : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6"); - } else { - __asm__ volatile( - "vsetvli t0, zero, e32, m4 \n\t" - "vxor.vv v28, v28, v28 \n\t" - "addi s1, %[B], 0 \n\t" - "addi s2, %[B], 16 \n\t" - "addi s3, %[B], 32 \n\t" - "addi s4, %[B], 48 \n\t" - - "addi s5, %[A], 0 \n\t" - "addi s6, %[A], 12 \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - "LOOP_K%=: \n\t" - "vle32.v v8, (s1) \n\t" - "addi s1, s1, 64 \n\t" - "vle32.v v9, (s2) \n\t" - "addi s2, s2, 80 \n\t" - "vle32.v v10, (s3) \n\t" - "addi s3, s3, 96 \n\t" - "vle32.v v11, (s4) \n\t" - "addi s4, s4, 112 \n\t" - "flw f1, (s5) \n\t" - "addi s5, s5, 4 \n\t" - - "addi t5, %[INNER], 0 \n\t" - "vxor.vv v16, v16, v16 \n\t" - "vxor.vv v18, v18, v18 \n\t" - "vxor.vv v20, v20, v20 \n\t" - "vxor.vv v22, v22, v22 \n\t" - "vfmul.vf v24, v8, f1 \n\t" - "vfmul.vf v25, v9, f1 \n\t" - "vfmul.vf v26, v10, f1 \n\t" - "vfmul.vf v27, v11, f1 \n\t" - "addi %[CNT], %[CNT], -1 \n\t" - "vsetvli t0, zero, e8, m1 \n\t" - "LOOP_INNER%=: \n\t" - - SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4 - - "vadd.vi v0, v0, -8 \n\t" - "vadd.vi v1, v1, -8 \n\t" - "vadd.vi v2, v2, -8 \n\t" - "vadd.vi v3, v3, -8 \n\t" - "vadd.vi v4, v4, -8 \n\t" - "vadd.vi v5, v5, -8 \n\t" - "vadd.vi v6, v6, -8 \n\t" - "vadd.vi v7, v7, -8 \n\t" - - SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 - - "bnez t5, LOOP_INNER%= \n\t" - "vsetvli t0, zero, e32, mf2 \n\t" - - SQ4BIT_KERNEL_ACC_1X4X4 - - "bnez %[CNT], LOOP_K%= \n\t" - "addi t3, zero, 16 \n\t" - "addi s1, %[C], 16 \n\t" - "addi s2, %[C], 32 \n\t" - "addi s3, %[C], 48 \n\t" - "blt %[NBLKS], t3, ST_TAIL%= \n\t" - "vse32.v v28, (%[C]) \n\t" - "vse32.v v29, (s1) \n\t" - "vse32.v v30, (s2) \n\t" - "vse32.v v31, (s3) \n\t" - "jal x0, END%= \n\t" - - "ST_TAIL%=: \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v28, (%[C]) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v29, (s1) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v30, (s2) \n\t" - "vsetvli t0, %[NBLKS], e32, mf2 \n\t" - "sub %[NBLKS], %[NBLKS], t0 \n\t" - "vse32.v v31, (s3) \n\t" - "END%=: \n\t" - - : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks) - : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) - : "cc", "t0", "t5", "t3", "f1", "s1", "s2", "s3", "s4", "s5", "s6"); - } - } - } -} + const size_t LDC = ldc * sizeof(float); -template -inline void SQ4BitGemmM4Kernel_CompInt8_DispatchOnBlkLen(size_t BlkLen, - const std::byte * QuantA, - const std::byte * QuantBData, - const float * QuantBScale, - const std::byte * QuantBZeroPoint, - float * C, - size_t CountM, - size_t CountN, - size_t BlockStrideQuantB, - const float * Bias, - const size_t ldc, - const size_t scalestride) { - if (scalestride == 4) { - SQ4BitGemmM4Kernel_CompInt8_Impl(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, C, - CountN, BlockStrideQuantB, Bias, ldc); - - } else if (scalestride == 2) { - SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl( - BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, C, CountN, BlockStrideQuantB, Bias, ldc); + for (size_t n = 0; n < CountN; n += 16) { + uint8_t * QuantBDataPtr = + (uint8_t *) QuantBData + (n / 16) * BlockCountK * (16 * sizeof(_Float16) + 512); + float * CPtr = C + n; + + __asm__ volatile( + "vsetvli t0, zero, e32, m8 \n\t" + "vxor.vv v24, v24, v24 \n\t" + "addi t3, %[BlockCountK], 0 \n\t" + "addi a1, %[A], 0 \n\t" + "addi s1, %[B], 0 \n\t" + + "BLOCK_COUNTK_LOOP%=: \n\t" + "addi s5, s1, 0 \n\t" + "addi s1, s5, 32 \n\t" + "vsetvli t0, zero, e32, m8 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "flw f1, (a1) \n\t" + "flw f2, 4(a1) \n\t" + "flw f3, 8(a1) \n\t" + "flw f4, 12(a1) \n\t" + "addi a1, a1, 16 \n\t" + "addi t2, %[INNER], 0 \n\t" + + "BLOCK_INNER_LOOP%=: \n\t" + "vsetvli t0, zero, e8, m1 \n\t" + "vle8.v v2, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v3, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v4, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v5, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v6, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v7, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v8, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v9, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v10, (a1) \n\t" + "addi a1, a1, 32 \n\t" + "vle8.v v11, (a1) \n\t" + "addi a1, a1, 32 \n\t" + + SQ4BIT_KERNEL_COMP_4x16x16 + + "addi t2, t2, -1 \n\t" + "bnez t2, BLOCK_INNER_LOOP%= \n\t" + + LOAD_SCALE_4x16_FP16 + + "vsetvli t0, zero, e32, m8 \n\t" + "vfcvt.f.x.v v16, v16 \n\t" + "vfmacc.vv v24, v16, v8 \n\t" + "addi t3, t3, -1 \n\t" + "bnez t3, BLOCK_COUNTK_LOOP%= \n\t" + + "RESULT_SAVE%=: \n\t" + + SAVE_RESULT_4x16 + + : + : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [LDC] "r"(LDC), + [BlockCountK] "r"(BlockCountK), [C] "r"(CPtr) + : "cc", "t0", "t1", "t2", "t3", "a1", "a2", "a3", "a4", "f1", "f2", "f3", "f4", + "s1", "s2", "s3", "s4", "s5", "s6"); } } -template -inline void SQ4BitGemmM1Kernel_CompInt8_DispatchOnBlkLen(size_t BlkLen, - const std::byte * QuantA, - const std::byte * QuantBData, - const float * QuantBScale, - const std::byte * QuantBZeroPoint, - float * C, - size_t CountM, - size_t CountN, - size_t BlockStrideQuantB, - const float * Bias, - const size_t ldc, - const size_t scalestride) { - if (scalestride == 4) { - SQ4BitGemmM1Kernel_CompInt8_Impl(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, C, - CountN, BlockStrideQuantB, Bias); - } else if (scalestride == 2) { - SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(BlkLen, QuantA, QuantBData, QuantBScale, - QuantBZeroPoint, C, CountN, BlockStrideQuantB, Bias); +// ---- Q8_0 IME1 int8xint8 M1 kernel ---------------------------------------- +// B comes from block_q8_0x16 (make_block_q8_0x16): 16 fp16 scales (32B) then 512B of interleaved +// int8 weights laid out as [INNER0: reg0..reg7][INNER1: reg0..reg7], each reg 32B = 4 columns x +// (even=K-first-half / odd=K-second-half). A comes from quantize_a_row_i8 (same as Q4_0 path). +// Reuses the vmadot COMP macro and the ACC_F16 dequant tail from the Q4_0 kernel. +static void SQ8BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(size_t BlkLen, + const uint8_t * QuantA, + const uint8_t * QuantBData, + float * C, + size_t CountN, + size_t BlockCountK, + const size_t ldc) { + GGML_UNUSED(ldc); + const size_t INNER = BlkLen / 16; // = 2 for QK8_0=32 + + for (size_t n = 0; n < CountN; n += 16) { + size_t nblks = (CountN - n) > 16 ? 16 : CountN - n; + // Each x16 K-block is {16 fp16 scales (32B), 512B interleaved int8}; stride = 544B. + uint8_t * QuantBDataPtr = (uint8_t *) QuantBData + (n / 16) * BlockCountK * (16 * sizeof(_Float16) + 512); + float * CPtr = C + n; + size_t cnt = BlockCountK; + + __asm__ volatile( + "vsetvli t0, zero, e32, m4 \n\t" + "vxor.vv v28, v28, v28 \n\t" + // s7 = per-K-block base (scale@+0, data@+32, block stride 544) + "addi s7, %[B], 0 \n\t" + "addi s5, %[A], 0 \n\t" // A scale (fp32) + "addi s6, %[A], 12 \n\t" // A data (int8), offset like Q4_0 M1 + "LOOP_K%=: \n\t" + "addi s1, s7, 32 \n\t" // data base for this K-block + // B scales: d[0..15] fp16 at block start. Load in 4 groups of 4 (d[0-3]/[4-7]/[8-11]/[12-15]) + // matching the 4 accumulators (each covers columns [g*4 .. g*4+3]). + "addi s2, s7, 8 \n\t" + "addi s3, s7, 16 \n\t" + "addi s4, s7, 24 \n\t" + "vsetvli t0, zero, e16, mf4 \n\t" + "vle16.v v4, (s7) \n\t" + "vle16.v v5, (s2) \n\t" + "vle16.v v6, (s3) \n\t" + "vle16.v v7, (s4) \n\t" + "addi s7, s7, 544 \n\t" // advance to next K-block (32 scale + 512 data) + "flw f1, (s5) \n\t" + "addi s5, s5, 4 \n\t" + "vfwcvt.f.f.v v8, v4 \n\t" + "vfwcvt.f.f.v v9, v5 \n\t" + "vfwcvt.f.f.v v10, v6 \n\t" + "vfwcvt.f.f.v v11, v7 \n\t" + "vsetvli t0, zero, e32, mf2 \n\t" + "addi t5, %[INNER], 0 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v18, v18 \n\t" + "vxor.vv v20, v20, v20 \n\t" + "vxor.vv v22, v22, v22 \n\t" + // combined scale (A_scale * B_scale) -> v24..v27 (one per accumulator) + "vfmul.vf v24, v8, f1 \n\t" + "vfmul.vf v25, v9, f1 \n\t" + "vfmul.vf v26, v10, f1 \n\t" + "vfmul.vf v27, v11, f1 \n\t" + "addi %[CNT], %[CNT], -1 \n\t" + "vsetvli t0, zero, e8, m1 \n\t" + "LOOP_INNER%=: \n\t" + // load 8 B data regs (v0..v7) directly (int8, no nibble unpack) + "vle8.v v0, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v1, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v2, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v3, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v4, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v5, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v6, (s1) \n\t" + "addi s1, s1, 32 \n\t" + "vle8.v v7, (s1) \n\t" + "addi s1, s1, 32 \n\t" + // load A (2 halves): v14 from s5, v15 from s6 (matches Q4_0 M1 A packing) + "vsetvli t0, zero, e8, mf4 \n\t" + "vle8.v v14, (s5) \n\t" + "addi s5, s5, 16 \n\t" + "vle8.v v15, (s6) \n\t" + "addi s6, s6, 16 \n\t" + "addi t5, t5, -1 \n\t" + "vsetvli t0, zero, e8, m1 \n\t" + SQ4BIT_KERNEL_COMP_1x8x2_4X8X4 + "bnez t5, LOOP_INNER%= \n\t" + "vsetvli t0, zero, e32, mf2 \n\t" + SQ4BIT_KERNEL_ACC_F16_1X4X4 + "bnez %[CNT], LOOP_K%= \n\t" + "addi t3, zero, 16 \n\t" + "addi s1, %[C], 16 \n\t" + "addi s2, %[C], 32 \n\t" + "addi s3, %[C], 48 \n\t" + "blt %[NBLKS], t3, ST_TAIL%= \n\t" + "vse32.v v28, (%[C]) \n\t" + "vse32.v v29, (s1) \n\t" + "vse32.v v30, (s2) \n\t" + "vse32.v v31, (s3) \n\t" + "jal x0, END%= \n\t" + "ST_TAIL%=: \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v28, (%[C]) \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v29, (s1) \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v30, (s2) \n\t" + "vsetvli t0, %[NBLKS], e32, mf2 \n\t" + "sub %[NBLKS], %[NBLKS], t0 \n\t" + "vse32.v v31, (s3) \n\t" + "END%=: \n\t" + : [CNT] "+r"(cnt), [NBLKS] "+r"(nblks) + : [INNER] "r"(INNER), [A] "r"(QuantA), [B] "r"(QuantBDataPtr), [C] "r"(CPtr) + : "cc", "t0", "t3", "t5", "f1", "s1", "s2", "s3", "s4", "s5", "s6", "s7"); } } } // namespace namespace ime1 { -size_t gemm_kernel_i8i4(size_t BlkLen, - const std::byte * QuantA, - const std::byte * QuantBData, - const float * QuantBScale, - const std::byte * QuantBZeroPoint, - float * C, - size_t CountM, - size_t CountN, - size_t CountK, - size_t BlockCountK, - size_t ldc, - const float * Bias, - const size_t ScaleStride) { - GGML_UNUSED(CountM); - GGML_UNUSED(CountK); - GGML_UNUSED(ldc); - if (CountM >= 4) { - if (QuantBZeroPoint != nullptr) { - SQ4BitGemmM4Kernel_CompInt8_DispatchOnBlkLen(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, - C, CountM, CountN, BlockCountK, Bias, ldc, ScaleStride); +size_t gemm_kernel_i8i4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + if (count_m >= 4) { + if (quant_b_zp != nullptr) { + SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_n, k_blks, + ldc); } else { - SQ4BitGemmM4Kernel_CompInt8_DispatchOnBlkLen(BlkLen, QuantA, QuantBData, QuantBScale, - QuantBZeroPoint, C, CountM, CountN, BlockCountK, Bias, - ldc, ScaleStride); + SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_n, + k_blks, ldc); } return 4; } else { - if (QuantBZeroPoint != nullptr) { - SQ4BitGemmM1Kernel_CompInt8_DispatchOnBlkLen(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, - C, CountM, CountN, BlockCountK, Bias, ldc, ScaleStride); + if (quant_b_zp != nullptr) { + SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_n, k_blks, + ldc); } else { - SQ4BitGemmM1Kernel_CompInt8_DispatchOnBlkLen(BlkLen, QuantA, QuantBData, QuantBScale, - QuantBZeroPoint, C, CountM, CountN, BlockCountK, Bias, - ldc, ScaleStride); + SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_n, + k_blks, ldc); } return 1; } } + +size_t gemm_kernel_i8i8(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + GGML_UNUSED(quant_b_zp); + if (count_m >= 4) { + SQ8BitGemmM4Kernel_CompInt8_ScaleFp16_Impl(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_n, k_blks, + ldc); + return 4; + } + SQ8BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_n, k_blks, ldc); + return 1; +} } // namespace ime1 -} // namespace sqnbitgemm_spacemit_ime +} // namespace spacemit_kernels diff --git a/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp b/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp new file mode 100644 index 0000000..0c7a036 --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp @@ -0,0 +1,5768 @@ +#include "ggml-impl.h" +#include "ggml.h" +#include "ime_kernels.h" +#include "rvv_kernels.h" +#include "string.h" + +#include +#include +#include + +#if !defined(__riscv_v) || !defined(__riscv_v_intrinsic) +# error "riscv v extension or v_intrinsic not enabled" +#else +# include +#endif + +#if !defined(__riscv_zfh) +# error "riscv zfh extension not enabled" +#endif + +#if defined(RISCV64_SPACEMIT_IME2) +#else +# error "RISCV64_SPACEMIT_IME2 not defined" +#endif + +#if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Woverlength-strings" +# pragma GCC diagnostic ignored "-Wcast-qual" +# pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +namespace spacemit_kernels { +namespace ime2 { + +template +void gemm_kernel_i8i2k_mrow_ref(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + using blk_type = nrow_block_q2_k; + constexpr float refactor_scale = 16.0f; + constexpr float factor_scale = 1.0f / refactor_scale; + + int64_t a_blk_stride = q8k_blk_size(256); + int64_t a_nrow_block_stride = a_blk_stride * MB_ROWS; + int64_t b_ncol_block_stride = sizeof(blk_type); + + float output[MB_ROWS * NB_COLS] = { 0 }; + _Float16 output_f16[MB_ROWS * NB_COLS] = { 0 }; + blk_type * quant_b_blk_data = (blk_type *) (quant_b_data); + + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + + int8_t * a_data = (int8_t *) quant_a_ptr + sizeof(float) * MB_ROWS + sizeof(int16_t) * MB_ROWS * 16; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] = 0; + } + } + + for (size_t ki = 0; ki < k_blks; ki++, quant_b_blk_data++, a_data += a_nrow_block_stride) { + uint8_t * b_data = quant_b_blk_data->qs; + uint8_t * scales = quant_b_blk_data->scales; + uint8_t * scales16 = (uint8_t *) (quant_b_blk_data->scales16); + uint8_t * zeros16 = (uint8_t *) (quant_b_blk_data->zeros16); + + _Float16 * scales_fp16 = (_Float16 *) scales16; + _Float16 * zeros_fp16 = (_Float16 *) zeros16; + + float * a_scale_row = (float *) (a_data - sizeof(float) * MB_ROWS - sizeof(int16_t) * MB_ROWS * 16); + int16_t * a_sum_row = (int16_t *) (a_data - sizeof(int16_t) * MB_ROWS * 16); + + memset(output_f16, 0, sizeof(output_f16)); + + uint8_t * scales_temp = scales; + uint8_t * zps_temp = scales; + for (size_t kii = 0; kii < 16; kii++, scales_temp += NB_COLS, zps_temp++) { + size_t b_shift = (kii % 4) * 2; + + uint8_t * b_data_col = b_data + (kii / 4) * NB_COLS * 16; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + int16_t a_sum = a_sum_row[mi * 16 + kii]; + for (size_t ci = 0; ci < NB_COLS; ci++) { + _Float16 acc_0 = 0.0; + + uint8_t b_zp = zps_temp[ci * 16] >> 4; + uint8_t b_scale = scales_temp[ci] & 0x0F; + for (size_t bi = 0; bi < 16; bi++) { + int8_t a0 = a_data[mi * 256 + bi + kii * 16]; + uint8_t b0 = b_data_col[ci * 16 + bi]; + acc_0 += static_cast(a0) * static_cast((b0 >> b_shift) & 0x03); + } + + _Float16 scale_item = + static_cast<_Float16>(b_scale) * static_cast<_Float16>(factor_scale) * scales_fp16[ci]; + + output_f16[ci + mi * NB_COLS] += acc_0 * scale_item; + output[ci + mi * NB_COLS] += b_zp * a_sum * a_scale_row[mi] * zeros_fp16[ci]; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + auto a_scale = a_scale_row[mi] * refactor_scale; + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] += output_f16[ci + mi * NB_COLS] * a_scale; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < nb_real; ci++) { + c_ptr[mi * ldc + ci] = output[mi * NB_COLS + ci]; + } + } + } +} + +template +void gemm_kernel_i8i3k_mrow_ref(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + using blk_type = nrow_block_q2_k; + constexpr float refactor_scale = 16.0f; + constexpr float factor_scale = 1.0f / refactor_scale; + + int64_t a_blk_stride = q8k_blk_size(256); + int64_t a_nrow_block_stride = a_blk_stride * MB_ROWS; + int64_t b_ncol_block_stride = sizeof(blk_type); + + float output[MB_ROWS * NB_COLS] = { 0 }; + _Float16 output_f16[MB_ROWS * NB_COLS] = { 0 }; + + blk_type * quant_b_blk_data = (blk_type *) (quant_b_data); + + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + + int8_t * a_data = (int8_t *) quant_a_ptr + sizeof(float) * MB_ROWS + sizeof(int16_t) * MB_ROWS * 16; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] = 0; + } + } + + for (size_t ki = 0; ki < k_blks; ki++, quant_b_blk_data++, a_data += a_nrow_block_stride) { + uint8_t * b_data = quant_b_blk_data->qs; + uint8_t * b_hmask = quant_b_blk_data->hmask; + int8_t * scales = quant_b_blk_data->scales; + uint8_t * scales16 = (uint8_t *) (quant_b_blk_data->scales16); + + _Float16 * scales_fp16 = (_Float16 *) scales16; + + float * a_scale_row = (float *) (a_data - sizeof(float) * MB_ROWS - sizeof(int16_t) * MB_ROWS * 16); + int16_t * a_sum_row = (int16_t *) (a_data - sizeof(int16_t) * MB_ROWS * 16); + + memset(output_f16, 0, sizeof(output_f16)); + + int8_t * scales_temp = scales; + uint16_t * b_mask_col = (uint16_t *) b_hmask; + + float acc_0_max = 0.0f; + for (size_t kii = 0; kii < 16; kii++, scales_temp += NB_COLS, b_mask_col += NB_COLS) { + size_t b_shift = (kii % 4) * 2; + + uint8_t * b_data_col = b_data + (kii / 4) * NB_COLS * 16; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + _Float16 acc_0 = 0; + // blk 2 * kii + 0 + uint16_t b_shift_mask = 1; + for (size_t bi = 0; bi < 16; bi++, b_shift_mask <<= 1) { + int8_t a0 = a_data[mi * 256 + bi + kii * 16]; + int8_t b0 = static_cast((b_data_col[ci * 16 + bi] >> b_shift) & 0x03); + b0 -= b_mask_col[ci] & b_shift_mask ? 0 : 4; + acc_0 += static_cast(a0) * static_cast(b0); + } + + _Float16 scale_item = static_cast<_Float16>(scales_temp[ci]) * scales_fp16[ci] * + static_cast<_Float16>(factor_scale); + + output_f16[ci + mi * NB_COLS] += acc_0 * scale_item; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + auto a_scale = a_scale_row[mi] * refactor_scale; + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] += output_f16[ci + mi * NB_COLS] * a_scale; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < nb_real; ci++) { + c_ptr[mi * ldc + ci] = output[mi * NB_COLS + ci]; + } + } + } +} + +template +void gemm_kernel_i8i4_mrow_ref(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t kblks_per_blk = 16; + GGML_ASSERT(k_blks % kblks_per_blk == 0); + + int64_t b_blk_stride = (sizeof(_Float16) + (blk_len / 2) + (quant_b_zp ? sizeof(uint8_t) : 0)); + int64_t b_stride = k_blks * b_blk_stride; + int64_t a_blk_stride = q8_blk_size(blk_len, true); + int64_t a_nrow_block_stride = a_blk_stride * MB_ROWS; + int64_t b_ncol_block_stride = b_blk_stride * NB_COLS; + + float output[MB_ROWS * NB_COLS] = { 0 }; + _Float16 output_f16[MB_ROWS * NB_COLS] = { 0 }; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + uint8_t * b_data = (uint8_t *) quant_b_data + ni * b_stride + NB_COLS * sizeof(_Float16); + if (quant_b_zp) { + b_data += NB_COLS * sizeof(uint8_t); + } + + int8_t * a_data = (int8_t *) quant_a_ptr + sizeof(float) * MB_ROWS + sizeof(int16_t) * MB_ROWS; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] = 0.0f; + output_f16[ci + mi * NB_COLS] = static_cast<_Float16>(0.0f); + } + } + + size_t kii = 0; + for (size_t ki = 0; ki < k_blks; ki++, a_data += a_nrow_block_stride, b_data += b_ncol_block_stride) { + _Float16 * b_scale_fp16 = (_Float16 *) (b_data - NB_COLS * sizeof(_Float16)); + uint8_t * b_zp = nullptr; + if (quant_b_zp) { + b_scale_fp16 = (_Float16 *) (b_data - NB_COLS * sizeof(_Float16) - NB_COLS * sizeof(uint8_t)); + b_zp = (uint8_t *) (b_data - NB_COLS * sizeof(uint8_t)); + } + + float * a_scale_row = (float *) (a_data - sizeof(float) * MB_ROWS - sizeof(int16_t) * MB_ROWS); + int16_t * a_sum_row = (int16_t *) (a_data - sizeof(int16_t) * MB_ROWS); + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + _Float16 a_scale = a_scale_row[mi]; + int16_t a_sum = a_sum_row[mi]; + + for (size_t ci = 0; ci < NB_COLS; ci++) { + _Float16 b_scale = b_scale_fp16[ci]; + int32_t acc = 0; + if (b_zp) { + acc += a_sum * b_zp[ci]; + } else { + acc += a_sum * 8; + } + for (size_t bi = 0; bi < blk_len / 2; bi++) { + int8_t a0 = a_data[mi * blk_len + 2 * bi]; + int8_t a1 = a_data[mi * blk_len + 2 * bi + 1]; + uint8_t b = b_data[ci * blk_len / 2 + bi]; + int8_t b0 = static_cast(b & 0x0F); + int8_t b1 = static_cast((b & 0xF0) >> 4); + acc += static_cast(a0) * static_cast(b0) + + static_cast(a1) * static_cast(b1); + } + output_f16[ci + mi * NB_COLS] += + static_cast(acc) * static_cast(a_scale) * static_cast(b_scale); + } + } + + if (kii == kblks_per_blk - 1) { + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] += static_cast(output_f16[ci + mi * NB_COLS]); + output_f16[ci + mi * NB_COLS] = 0.0f; + } + } + kii = 0; + } else { + kii++; + } + } + + if (kii == kblks_per_blk - 1) { + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] += static_cast(output_f16[ci + mi * NB_COLS]); + output_f16[ci + mi * NB_COLS] = 0.0f; + } + } + kii = 0; + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < nb_real; ci++) { + c_ptr[mi * ldc + ci] = output[mi * NB_COLS + ci]; + } + } + } +} + +template +void gemm_kernel_i8i4_hp_mrow_ref(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t k_subblks_per_superblk = 8; + + struct block_q4_0x32_layout { + _Float16 d[NB_COLS]; + uint8_t qs[16 * NB_COLS]; + }; + + GGML_ASSERT(blk_len == 256); + + const size_t b_superblk_stride = sizeof(block_q4_0x32_layout) * k_subblks_per_superblk + + (quant_b_zp ? NB_COLS * k_subblks_per_superblk * sizeof(uint8_t) : 0); + const size_t b_tile_stride = k_blks * b_superblk_stride; + + const size_t a_nrow_block_stride = q8_hp_blk_size(blk_len, true, true) * MB_ROWS; + const size_t a_subblk_stride = q8_hp_blk_size(32, false, false) * MB_ROWS; + + float output[MB_ROWS * NB_COLS] = { 0 }; + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + const uint8_t * b_tile_base = quant_b_data + (ni / NB_COLS) * b_tile_stride; + int8_t * a_data = (int8_t *) quant_a_ptr; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] = 0.0f; + } + } + + for (size_t ki = 0; ki < k_blks; ki++, a_data += a_nrow_block_stride) { + _Float16 output_f16[MB_ROWS * NB_COLS] = { 0 }; + + const uint8_t * b_superblk_ptr = b_tile_base + ki * b_superblk_stride; + const block_q4_0x32_layout * b_blocks = reinterpret_cast(b_superblk_ptr); + const uint8_t * b_zps = + quant_b_zp ? b_superblk_ptr + sizeof(block_q4_0x32_layout) * k_subblks_per_superblk : nullptr; + + _Float16 * a_sum_row = (_Float16 *) (a_data + a_subblk_stride * k_subblks_per_superblk); + _Float16 * a_scale_avg_row = (_Float16 *) (a_data + a_nrow_block_stride - sizeof(_Float16) * MB_ROWS); + _Float16 scale_factor = a_scale_avg_row[0]; + + for (size_t ksi = 0; ksi < k_subblks_per_superblk; ++ksi) { + const _Float16 * a_scale_row = reinterpret_cast(a_data + a_subblk_stride * ksi); + int8_t * a_subblk = a_data + a_subblk_stride * ksi + MB_ROWS * sizeof(_Float16); + const _Float16 a_scale = a_scale_row[0]; + const block_q4_0x32_layout & b_block = b_blocks[ksi]; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + const uint8_t * b_qs = b_block.qs + ci * 16; + _Float16 b_scale = b_block.d[ci] * a_scale; + + int16_t acc = 0; + for (size_t bi = 0; bi < 16; bi++) { + uint8_t b = b_qs[bi]; + int8_t b0 = static_cast(b & 0x0F); + int8_t b1 = static_cast((b & 0xF0) >> 4); + + acc += static_cast(a_subblk[mi * 32 + 2 * bi]) * static_cast(b0) + + static_cast(a_subblk[mi * 32 + 2 * bi + 1]) * static_cast(b1); + } + + const _Float16 scaled_acc = static_cast<_Float16>(acc) * b_scale; + output_f16[ci + mi * NB_COLS] += scaled_acc; + } + } + } + + for (size_t ksi = 0; ksi < k_subblks_per_superblk; ++ksi) { + const _Float16 * a_scale_row = reinterpret_cast(a_data + a_subblk_stride * ksi); + const block_q4_0x32_layout & b_block = b_blocks[ksi]; + const uint8_t * b_zp_row = b_zps ? b_zps + ksi * NB_COLS : nullptr; + const _Float16 a_scale = a_scale_row[0]; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + const _Float16 a_sum = a_sum_row[mi * k_subblks_per_superblk + ksi]; + for (size_t ci = 0; ci < NB_COLS; ci++) { + _Float16 b_scale = b_block.d[ci] * a_scale; + _Float16 a_sum_bzp = a_sum; + if (b_zp_row) { + a_sum_bzp = a_sum * static_cast<_Float16>(0.125f) * static_cast<_Float16>(b_zp_row[ci]); + } + + const _Float16 scaled_acc = a_sum_bzp * b_scale; + output[ci + mi * NB_COLS] += scaled_acc * scale_factor; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + auto val = static_cast(output_f16[ci + mi * NB_COLS]) * static_cast(scale_factor); + output[ci + mi * NB_COLS] += val; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < nb_real; ci++) { + c_ptr[mi * ldc + ci] = output[mi * NB_COLS + ci]; + } + } + } +} + +template +void moe_gemm_kernel_i8i4_mrow_ref(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + int64_t b_blk_stride = (sizeof(ggml_fp16_t) + (blk_len / 2) + (quant_b_zp ? sizeof(uint8_t) : 0)); + int64_t b_stride = k_blks * b_blk_stride; + int64_t a_blk_stride = q8_blk_size(blk_len, true); + int64_t b_ncol_block_stride = b_blk_stride * NB_COLS; + + float output[MB_ROWS * NB_COLS] = { 0 }; + std::array a_data; + std::array c_data; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + c_data[mi] = c_ptr[mi]; + } + + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + uint8_t * b_data = (uint8_t *) quant_b_data + ni * b_stride + NB_COLS * sizeof(ggml_fp16_t); + if (quant_b_zp) { + b_data += NB_COLS * sizeof(uint8_t); + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + a_data[mi] = (int8_t *) quant_a_ptr[mi] + sizeof(float) + sizeof(int16_t); + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] = 0; + } + } + + for (size_t ki = 0; ki < k_blks; ki++, b_data += b_ncol_block_stride) { + ggml_fp16_t * b_scale_fp16 = (ggml_fp16_t *) (b_data - NB_COLS * sizeof(ggml_fp16_t)); + uint8_t * b_zp = nullptr; + if (quant_b_zp) { + b_scale_fp16 = (ggml_fp16_t *) (b_data - NB_COLS * sizeof(ggml_fp16_t) - NB_COLS * sizeof(uint8_t)); + b_zp = (uint8_t *) (b_data - NB_COLS * sizeof(uint8_t)); + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + float * a_scale_row = (float *) (a_data[mi] - sizeof(float) - sizeof(int16_t)); + int16_t * a_sum_row = (int16_t *) (a_data[mi] - sizeof(int16_t)); + + float a_scale = *a_scale_row; + int16_t a_sum = *a_sum_row; + + for (size_t ci = 0; ci < NB_COLS; ci++) { + float b_scale = ggml_fp16_to_fp32(b_scale_fp16[ci]); + int32_t acc = 0; + if (b_zp) { + acc += a_sum * b_zp[ci]; + } else { + acc += a_sum * 8; + } + for (size_t bi = 0; bi < blk_len / 2; bi++) { + int8_t a0 = (a_data[mi])[2 * bi]; + int8_t a1 = (a_data[mi])[2 * bi + 1]; + uint8_t b = b_data[ci * blk_len / 2 + bi]; + int8_t b0 = static_cast(b & 0x0F); + int8_t b1 = static_cast((b & 0xF0) >> 4); + acc += static_cast(a0) * static_cast(b0) + + static_cast(a1) * static_cast(b1); + } + output[ci + mi * NB_COLS] += static_cast(acc) * a_scale * b_scale; + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + a_data[mi] += a_blk_stride; + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < nb_real; ci++) { + (c_data[mi])[ci] = output[mi * NB_COLS + ci]; + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + c_data[mi] += NB_COLS; + } + } +} + +template +void moe_gemm_kernel_i8i5_mrow_ref(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + GGML_UNUSED(count_m); + GGML_UNUSED(ldc); + + // blk_len is expected to be 32 for Q5 types. + int64_t a_blk_stride = q8_blk_size(blk_len, true); + + float output[MB_ROWS * NB_COLS] = { 0 }; + std::array a_data; + std::array c_data; + + for (size_t mi = 0; mi < MB_ROWS; ++mi) { + c_data[mi] = c_ptr[mi]; + } + + if (quant_b_zp) { + using blk_type = nrow_block_q5_1; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + blk_type * quant_b_blk_data = (blk_type *) quant_b_data + (ni / NB_COLS) * k_blks; + + for (size_t mi = 0; mi < MB_ROWS; ++mi) { + a_data[mi] = (int8_t *) quant_a_ptr[mi] + sizeof(float) + sizeof(int16_t); + } + + for (size_t mi = 0; mi < MB_ROWS; ++mi) { + for (size_t ci = 0; ci < NB_COLS; ++ci) { + output[ci + mi * NB_COLS] = 0; + } + } + + for (size_t ki = 0; ki < k_blks; ++ki, ++quant_b_blk_data) { + for (size_t mi = 0; mi < MB_ROWS; ++mi) { + float * a_scale_row = (float *) (a_data[mi] - sizeof(float) - sizeof(int16_t)); + int16_t * a_sum_row = (int16_t *) (a_data[mi] - sizeof(int16_t)); + float a_scale = *a_scale_row; + int16_t a_sum = *a_sum_row; + + for (size_t ci = 0; ci < NB_COLS; ++ci) { + float b_scale = ggml_fp16_to_fp32(quant_b_blk_data->scales16[ci]); + uint8_t b_zp_val = quant_b_blk_data->zp[ci]; + int32_t acc = a_sum * static_cast(b_zp_val); + + for (size_t bi = 0; bi < blk_len / 2; ++bi) { + int8_t a0 = a_data[mi][2 * bi]; + int8_t a1 = a_data[mi][2 * bi + 1]; + uint8_t qs_byte = quant_b_blk_data->qs[ci * (blk_len / 2) + bi]; + int8_t b0 = static_cast(qs_byte & 0x0F); + int8_t b1 = static_cast((qs_byte >> 4) & 0x0F); + uint8_t qh_byte0 = quant_b_blk_data->qh[ci * 4 + (2 * bi) / 8]; + uint8_t qh_byte1 = quant_b_blk_data->qh[ci * 4 + (2 * bi + 1) / 8]; + uint8_t h0 = (qh_byte0 >> ((2 * bi) % 8)) & 1; + uint8_t h1 = (qh_byte1 >> ((2 * bi + 1) % 8)) & 1; + + b0 |= (h0 << 4); + b1 |= (h1 << 4); + + acc += static_cast(a0) * static_cast(b0) + + static_cast(a1) * static_cast(b1); + } + + output[ci + mi * NB_COLS] += static_cast(acc) * a_scale * b_scale; + } + + a_data[mi] += a_blk_stride; + } + } + + for (size_t mi = 0; mi < MB_ROWS; ++mi) { + for (size_t ci = 0; ci < nb_real; ++ci) { + c_data[mi][ci] = output[mi * NB_COLS + ci]; + } + c_data[mi] += NB_COLS; + } + } + } else { + using blk_type = nrow_block_q5_0; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + blk_type * quant_b_blk_data = (blk_type *) quant_b_data + (ni / NB_COLS) * k_blks; + + for (size_t mi = 0; mi < MB_ROWS; ++mi) { + a_data[mi] = (int8_t *) quant_a_ptr[mi] + sizeof(float) + sizeof(int16_t); + } + + for (size_t mi = 0; mi < MB_ROWS; ++mi) { + for (size_t ci = 0; ci < NB_COLS; ++ci) { + output[ci + mi * NB_COLS] = 0; + } + } + + for (size_t ki = 0; ki < k_blks; ++ki, ++quant_b_blk_data) { + for (size_t mi = 0; mi < MB_ROWS; ++mi) { + float * a_scale_row = (float *) (a_data[mi] - sizeof(float) - sizeof(int16_t)); + int16_t * a_sum_row = (int16_t *) (a_data[mi] - sizeof(int16_t)); + float a_scale = *a_scale_row; + int16_t a_sum = *a_sum_row; + + for (size_t ci = 0; ci < NB_COLS; ++ci) { + float b_scale = ggml_fp16_to_fp32(quant_b_blk_data->scales16[ci]); + int32_t acc = a_sum * 16; + + for (size_t bi = 0; bi < blk_len / 2; ++bi) { + int8_t a0 = a_data[mi][2 * bi]; + int8_t a1 = a_data[mi][2 * bi + 1]; + uint8_t qs_byte = quant_b_blk_data->qs[ci * (blk_len / 2) + bi]; + int8_t b0 = static_cast(qs_byte & 0x0F); + int8_t b1 = static_cast((qs_byte >> 4) & 0x0F); + uint8_t qh_byte0 = quant_b_blk_data->qh[ci * 4 + (2 * bi) / 8]; + uint8_t qh_byte1 = quant_b_blk_data->qh[ci * 4 + (2 * bi + 1) / 8]; + uint8_t h0 = (qh_byte0 >> ((2 * bi) % 8)) & 1; + uint8_t h1 = (qh_byte1 >> ((2 * bi + 1) % 8)) & 1; + + b0 |= (h0 << 4); + b1 |= (h1 << 4); + + acc += static_cast(a0) * static_cast(b0) + + static_cast(a1) * static_cast(b1); + } + + output[ci + mi * NB_COLS] += static_cast(acc) * a_scale * b_scale; + } + + a_data[mi] += a_blk_stride; + } + } + + for (size_t mi = 0; mi < MB_ROWS; ++mi) { + for (size_t ci = 0; ci < nb_real; ++ci) { + c_data[mi][ci] = output[mi * NB_COLS + ci]; + } + c_data[mi] += NB_COLS; + } + } + } +} + +template +void gemm_kernel_i8i8_mrow_ref(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + int64_t b_blk_stride = (sizeof(ggml_fp16_t) + blk_len); + int64_t b_stride = k_blks * b_blk_stride; + int64_t a_blk_stride = q8_blk_size(blk_len, true); + int64_t a_nrow_block_stride = a_blk_stride * MB_ROWS; + int64_t b_ncol_block_stride = b_blk_stride * NB_COLS; + + float output[MB_ROWS * NB_COLS] = { 0 }; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + int8_t * b_data = (int8_t *) quant_b_data + ni * b_stride + NB_COLS * sizeof(ggml_fp16_t); + + int8_t * a_data = (int8_t *) quant_a_ptr + sizeof(float) * MB_ROWS + sizeof(int16_t) * MB_ROWS; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] = 0; + } + } + + for (size_t ki = 0; ki < k_blks; ki++, a_data += a_nrow_block_stride, b_data += b_ncol_block_stride) { + ggml_fp16_t * b_scale_fp16 = (ggml_fp16_t *) (b_data - NB_COLS * sizeof(ggml_fp16_t)); + + float * a_scale_row = (float *) (a_data - sizeof(float) * MB_ROWS - sizeof(int16_t) * MB_ROWS); + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + float a_scale = a_scale_row[mi]; + for (size_t ci = 0; ci < NB_COLS; ci++) { + float b_scale = ggml_fp16_to_fp32(b_scale_fp16[ci]); + int32_t acc = 0; + for (size_t bi = 0; bi < blk_len; bi++) { + int8_t a0 = a_data[mi * blk_len + bi]; + int8_t b0 = b_data[ci * blk_len + bi]; + acc += static_cast(a0) * static_cast(b0); + } + output[ci + mi * NB_COLS] += static_cast(acc) * a_scale * b_scale; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < nb_real; ci++) { + c_ptr[mi * ldc + ci] = output[mi * NB_COLS + ci]; + } + } + } +} + +template +void gemm_kernel_i8i5_mrow_ref(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + // blk_len is expected to be 32 for Q5 types + // quant_b_zp != nullptr => nrow_block_q5_1 (has zp) + // quant_b_zp == nullptr => nrow_block_q5_0 (no zp) + + int64_t a_blk_stride = q8_blk_size(blk_len, true); + int64_t a_nrow_block_stride = a_blk_stride * MB_ROWS; + + float output[MB_ROWS * NB_COLS] = { 0 }; + + if (quant_b_zp) { + // nrow_block_q5_1: scales16[NB_COLS] + zp[NB_COLS] + qh[4*NB_COLS] + qs[16*NB_COLS] + using blk_type = nrow_block_q5_1; + int64_t b_ncol_block_stride = sizeof(blk_type); + blk_type * quant_b_blk_data = (blk_type *) quant_b_data; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + + int8_t * a_data = (int8_t *) quant_a_ptr + sizeof(float) * MB_ROWS + sizeof(int16_t) * MB_ROWS; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] = 0; + } + } + + for (size_t ki = 0; ki < k_blks; ki++, quant_b_blk_data++, a_data += a_nrow_block_stride) { + float * a_scale_row = (float *) (a_data - sizeof(float) * MB_ROWS - sizeof(int16_t) * MB_ROWS); + int16_t * a_sum_row = (int16_t *) (a_data - sizeof(int16_t) * MB_ROWS); + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + float a_scale = a_scale_row[mi]; + int16_t a_sum = a_sum_row[mi]; + + for (size_t ci = 0; ci < NB_COLS; ci++) { + float b_scale = ggml_fp16_to_fp32(quant_b_blk_data->scales16[ci]); + uint8_t b_zp_val = quant_b_blk_data->zp[ci]; + int32_t acc = a_sum * static_cast(b_zp_val); + + for (size_t bi = 0; bi < blk_len / 2; bi++) { + int8_t a0 = a_data[mi * blk_len + 2 * bi]; + int8_t a1 = a_data[mi * blk_len + 2 * bi + 1]; + uint8_t qs_byte = quant_b_blk_data->qs[ci * (blk_len / 2) + bi]; + int8_t b0 = static_cast(qs_byte & 0x0F); + int8_t b1 = static_cast((qs_byte >> 4) & 0x0F); + + // Extract high bits from qh + // qh is packed as 4 bytes per column (32 bits for 32 elements) + uint8_t qh_byte0 = quant_b_blk_data->qh[ci * 4 + (2 * bi) / 8]; + uint8_t qh_byte1 = quant_b_blk_data->qh[ci * 4 + (2 * bi + 1) / 8]; + uint8_t h0 = (qh_byte0 >> ((2 * bi) % 8)) & 1; + uint8_t h1 = (qh_byte1 >> ((2 * bi + 1) % 8)) & 1; + + b0 |= (h0 << 4); + b1 |= (h1 << 4); + + acc += static_cast(a0) * static_cast(b0) + + static_cast(a1) * static_cast(b1); + } + output[ci + mi * NB_COLS] += static_cast(acc) * a_scale * b_scale; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < nb_real; ci++) { + c_ptr[mi * ldc + ci] = output[mi * NB_COLS + ci]; + } + } + } + } else { + // nrow_block_q5_0: scales16[NB_COLS] + qh[4*NB_COLS] + qs[16*NB_COLS] + using blk_type = nrow_block_q5_0; + int64_t b_ncol_block_stride = sizeof(blk_type); + blk_type * quant_b_blk_data = (blk_type *) quant_b_data; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + + int8_t * a_data = (int8_t *) quant_a_ptr + sizeof(float) * MB_ROWS + sizeof(int16_t) * MB_ROWS; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] = 0; + } + } + + for (size_t ki = 0; ki < k_blks; ki++, quant_b_blk_data++, a_data += a_nrow_block_stride) { + float * a_scale_row = (float *) (a_data - sizeof(float) * MB_ROWS - sizeof(int16_t) * MB_ROWS); + int16_t * a_sum_row = (int16_t *) (a_data - sizeof(int16_t) * MB_ROWS); + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + float a_scale = a_scale_row[mi]; + int16_t a_sum = a_sum_row[mi]; + + for (size_t ci = 0; ci < NB_COLS; ci++) { + float b_scale = ggml_fp16_to_fp32(quant_b_blk_data->scales16[ci]); + // Q5_0 has no zp, use default offset 16 (midpoint of 5-bit unsigned range) + int32_t acc = a_sum * 16; + + for (size_t bi = 0; bi < blk_len / 2; bi++) { + int8_t a0 = a_data[mi * blk_len + 2 * bi]; + int8_t a1 = a_data[mi * blk_len + 2 * bi + 1]; + uint8_t qs_byte = quant_b_blk_data->qs[ci * (blk_len / 2) + bi]; + int8_t b0 = static_cast(qs_byte & 0x0F); + int8_t b1 = static_cast((qs_byte >> 4) & 0x0F); + + // Extract high bits from qh + uint8_t qh_byte0 = quant_b_blk_data->qh[ci * 4 + (2 * bi) / 8]; + uint8_t qh_byte1 = quant_b_blk_data->qh[ci * 4 + (2 * bi + 1) / 8]; + uint8_t h0 = (qh_byte0 >> ((2 * bi) % 8)) & 1; + uint8_t h1 = (qh_byte1 >> ((2 * bi + 1) % 8)) & 1; + + b0 |= (h0 << 4); + b1 |= (h1 << 4); + + acc += static_cast(a0) * static_cast(b0) + + static_cast(a1) * static_cast(b1); + } + output[ci + mi * NB_COLS] += static_cast(acc) * a_scale * b_scale; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < nb_real; ci++) { + c_ptr[mi * ldc + ci] = output[mi * NB_COLS + ci]; + } + } + } + } +} + +template +void gemm_kernel_i8mxfp4_mrow_ref(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + // blk_len is expected to be 32 (QK_MXFP4) + // quant_b_zp is unused for MXFP4 (symmetric quantization) + GGML_UNUSED(quant_b_zp); + + int64_t a_blk_stride = q8_blk_size(blk_len, true); + int64_t a_nrow_block_stride = a_blk_stride * MB_ROWS; + + float output[MB_ROWS * NB_COLS] = { 0 }; + + using blk_type = nrow_block_mxfp4; + blk_type * quant_b_blk_data = (blk_type *) quant_b_data; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + + int8_t * a_data = (int8_t *) quant_a_ptr + sizeof(float) * MB_ROWS + sizeof(int16_t) * MB_ROWS; + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < NB_COLS; ci++) { + output[ci + mi * NB_COLS] = 0; + } + } + + for (size_t ki = 0; ki < k_blks; ki++, quant_b_blk_data++, a_data += a_nrow_block_stride) { + float * a_scale_row = (float *) (a_data - sizeof(float) * MB_ROWS - sizeof(int16_t) * MB_ROWS); + int16_t * a_sum_row = (int16_t *) (a_data - sizeof(int16_t) * MB_ROWS); + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + float a_scale = a_scale_row[mi]; + + for (size_t ci = 0; ci < NB_COLS; ci++) { + float b_scale = GGML_E8M0_TO_FP32_HALF(quant_b_blk_data->e[ci]); + + // Read 32 sign bits for this column + uint32_t sign_bits; + memcpy(&sign_bits, &quant_b_blk_data->qh[ci * 4], 4); + + int32_t acc = 0; + for (size_t bi = 0; bi < blk_len / 2; bi++) { + int8_t a0 = a_data[mi * blk_len + 2 * bi]; + int8_t a1 = a_data[mi * blk_len + 2 * bi + 1]; + + // qs[ci*16 + bi] stores abs(vals[bi*2]) in low 4 bits + // and abs(vals[bi*2+1]) in high 4 bits + uint8_t qs_byte = quant_b_blk_data->qs[ci * 16 + bi]; + int8_t b_abs0 = static_cast(qs_byte & 0x0F); + int8_t b_abs1 = static_cast((qs_byte >> 4) & 0x0F); + + // Extract sign bits: bit (2*bi) for vals[2*bi], bit (2*bi+1) for vals[2*bi+1] + int8_t b0 = (sign_bits >> (2 * bi)) & 1 ? -b_abs0 : b_abs0; + int8_t b1 = (sign_bits >> (2 * bi + 1)) & 1 ? -b_abs1 : b_abs1; + + acc += static_cast(a0) * static_cast(b0) + + static_cast(a1) * static_cast(b1); + } + output[ci + mi * NB_COLS] += static_cast(acc) * a_scale * b_scale; + } + } + } + + for (size_t mi = 0; mi < MB_ROWS; mi++) { + for (size_t ci = 0; ci < nb_real; ci++) { + c_ptr[mi * ldc + ci] = output[mi * NB_COLS + ci]; + } + } + } +} + +void gemm_kernel_i8i2k_m1(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t NB_COLS = 32; + using blk_type = nrow_block_q2_k; + + int64_t b_ncol_block_stride = sizeof(blk_type) * k_blks; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * b_ncol_block_stride; + int8_t * a_data = (int8_t *) quant_a_ptr; + float * dst_c = (float *) c_ptr + ni; + + asm volatile( + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v31, v31, v31 \n\t" + "mv s1, %[BK] \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // load scale A + "flw fa0, (%[A]) \n\t" + "addi %[A], %[A], 4 \n\t" + + "li t1, 4 \n\t" + "addi t2, %[B], 512 \n\t" // B data addr + "addi t3, %[A], 32 \n\t" // A data addr + "addi s3, %[B], 0 \n\t" + "vxor.vv v30, v29, v29 \n\t" // tmp result + + "INNER_K_LOOP%=: \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vxor.vv v2, v2, v2 \n\t" + "vxor.vv v3, v3, v3 \n\t" + "vxor.vv v4, v4, v4 \n\t" + "vxor.vv v5, v5, v5 \n\t" + "vxor.vv v6, v6, v6 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v29, v29, v29 \n\t" + + // load scale B + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v0, (%[B]) \n\t" + "addi %[B], %[B], 128 \n\t" + + // A data, 1x64@i8 + "vsetivli t0, 16, e8, mf4 \n\t" + "vle8.v v2, (t3) \n\t" + "addi t3, t3, 16 \n\t" + + "vsetivli t0, 16, e8, mf4 \n\t" + "vle8.v v4, (t3) \n\t" + "addi t3, t3, 16 \n\t" + + "vsetivli t0, 16, e8, mf4 \n\t" + "vle8.v v5, (t3) \n\t" + "addi t3, t3, 16 \n\t" + + "vsetivli t0, 16, e8, mf4 \n\t" + "vle8.v v6, (t3) \n\t" + "addi t3, t3, 16 \n\t" + + "vsetvli t0, x0, e64, mf2 \n\t" + "vslideup.vi v3, v4, 2 \n\t" + "vslideup.vi v28, v5, 4 \n\t" + "vslideup.vi v29, v6, 6 \n\t" + + // init the accumu to zero + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v20, v18, v18 \n\t" + "vxor.vv v22, v18, v18 \n\t" + "vxor.vv v24, v18, v18 \n\t" + "vxor.vv v26, v18, v18 \n\t" + + // B data, 32x64@i2 + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v4, (t2) \n\t" + "addi t2, t2, 512 \n\t" + "vand.vi v8, v4, 0x3 \n\t" // 0-15 + "vsrl.vi v9, v4, 2 \n\t" + "vsrl.vi v10, v4, 4 \n\t" + "vsrl.vi v11, v4, 6 \n\t" // 48-63 + "vand.vi v9, v9, 0x3 \n\t" // 16-31 + "vand.vi v10, v10, 0x3 \n\t" // 32-47 + + "vand.vi v12, v5, 0x3 \n\t" // 0-15 + "vsrl.vi v13, v5, 2 \n\t" + "vsrl.vi v14, v5, 4 \n\t" + "vsrl.vi v15, v5, 6 \n\t" // 48-63 + "vand.vi v13, v13, 0x3 \n\t" // 16-31 + "vand.vi v14, v14, 0x3 \n\t" // 32-47 + + "vand.vi v16, v6, 0x3 \n\t" // 0-15 + "vsrl.vi v17, v6, 2 \n\t" + "vsrl.vi v18, v6, 4 \n\t" + "vsrl.vi v19, v6, 6 \n\t" // 48-63 + "vand.vi v17, v17, 0x3 \n\t" // 16-31 + "vand.vi v18, v18, 0x3 \n\t" // 32-47 + + "vand.vi v4, v7, 0x3 \n\t" // 0-15 + "vsrl.vi v5, v7, 2 \n\t" + "vsrl.vi v6, v7, 4 \n\t" + "vsrl.vi v7, v7, 6 \n\t" // 48-63 + "vand.vi v5, v5, 0x3 \n\t" // 16-31 + "vand.vi v6, v6, 0x3 \n\t" // 32-47 + + // i2 * i8 vmadot + "vsetvli t0, x0, e8, m1 \n\t" + "vmadotsu v20, v2, v8, i8 \n\t" + "vmadotsu v22, v2, v12, i8 \n\t" + "vmadotsu v24, v2, v16, i8 \n\t" + "vmadotsu v26, v2, v4, i8 \n\t" + + "vmadotsu v20, v3, v9, i8 \n\t" + "vmadotsu v22, v3, v13, i8 \n\t" + "vmadotsu v24, v3, v17, i8 \n\t" + "vmadotsu v26, v3, v5, i8 \n\t" + + "vmadotsu v20, v28, v10, i8 \n\t" + "vmadotsu v22, v28, v14, i8 \n\t" + "vmadotsu v24, v28, v18, i8 \n\t" + "vmadotsu v26, v28, v6, i8 \n\t" + + "vmadotsu v20, v29, v11, i8 \n\t" + "vmadotsu v22, v29, v15, i8 \n\t" + "vmadotsu v24, v29, v19, i8 \n\t" + "vmadotsu v26, v29, v7, i8 \n\t" + + "vand.vi v10, v0, 0xf \n\t" // scale + "vwadd.vx v12, v10, x0 \n\t" + "vsetvli t0, x0, e16, m2 \n\t" + "vwadd.vx v16, v12, x0 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vpack.vv v2, v20, v22, 2 \n\t" + "vpack.vv v4, v24, v26, 2 \n\t" + "vpack.vv v6, v2, v4, 3 \n\t" // 0,1 + "vpack.vv v8, v3, v5, 3 \n\t" // 2,3 + + // mul scale + "vmacc.vv v30, v6, v16 \n\t" + "vmacc.vv v30, v7, v17 \n\t" + "vmacc.vv v30, v8, v18 \n\t" + "vmacc.vv v30, v9, v19 \n\t" + + "addi t1, t1, -1 \n\t" + "bgtz t1, INNER_K_LOOP%= \n\t" + + // load zp B + "vsetvli t0, x0, e8, m4 \n\t" + "vle8.v v4, (s3) \n\t" + "vsrl.vi v8, v4, 4 \n\t" // zp + + // asum * zp + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v20, v20, v20 \n\t" + "vxor.vv v22, v22, v22 \n\t" + "vxor.vv v24, v24, v24 \n\t" + "vxor.vv v26, v26, v26 \n\t" + + "vsetvli t0, x0, e16, mf4 \n\t" + "vle16.v v2, (%[A]) \n\t" + "vsetvli t0, x0, e8, mf4 \n\t" + "vnsrl.wi v12, v2, 0 \n\t" // low 8 + "vnsra.wi v13, v2, 8 \n\t" // high 8 + + "vsetvli t0, x0, e32, m1 \n\t" + "vmadotsu v20, v13, v8, i8 \n\t" + "vmadotsu v22, v13, v9, i8 \n\t" + "vmadotsu v24, v13, v10, i8 \n\t" + "vmadotsu v26, v13, v11, i8 \n\t" + + "vsll.vi v20, v20, 8 \n\t" + "vsll.vi v22, v22, 8 \n\t" + "vsll.vi v24, v24, 8 \n\t" + "vsll.vi v26, v26, 8 \n\t" + + "vmadotu v20, v12, v8, i8 \n\t" + "vmadotu v22, v12, v9, i8 \n\t" + "vmadotu v24, v12, v10, i8 \n\t" + "vmadotu v26, v12, v11, i8 \n\t" + + "vpack.vv v2, v20, v22, 2 \n\t" + "vpack.vv v4, v24, v26, 2 \n\t" + "vpack.vv v28, v2, v4, 3 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v0, (t2) \n\t" // scale16 + "addi t2, t2, 64 \n\t" + "vle16.v v1, (t2) \n\t" // zero16 + "vfwcvt.f.f.v v2, v0 \n\t" + "vfwcvt.f.f.v v4, v1 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v30, v30 \n\t" + "vfcvt.f.x.v v28, v28 \n\t" + "addi %[B], t2, 64 \n\t" + "mv %[A], t3 \n\t" + + "vfmul.vv v30, v30, v2 \n\t" // mul scale16 + "vfmacc.vv v30, v28, v4 \n\t" // + mul zero16 + "vfmacc.vf v31, fa0, v30 \n\t" + "addi s1, s1, -1 \n\t" + "bgtz s1, BLK_LOOP%= \n\t" + + // save + "vsetvli t0, x0, e32, m1 \n\t" + "vse32.v v31, (%[DST]) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data) + : [DST] "r"(dst_c), [BK] "r"(k_blks) + : "t0", "t1", "t2", "t3", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30", "v31", "fa0", "t4", "t5", "t6", "s1", "s2", "s3"); + } +} + +void gemm_kernel_i8i2k_m4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t NB_COLS = 32; + using blk_type = nrow_block_q2_k; + + int64_t b_ncol_block_stride = sizeof(blk_type) * k_blks; + _Float16 scale = 0.0625f; + _Float16 scale_1 = 16.0f; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * b_ncol_block_stride; + int8_t * a_data = (int8_t *) quant_a_ptr; + float * dst_c = (float *) c_ptr + ni; + + asm volatile( + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v28, v31, v31 \n\t" // init result + "vxor.vv v29, v31, v31 \n\t" + "vxor.vv v30, v31, v31 \n\t" + "vxor.vv v31, v31, v31 \n\t" + "mv s1, %[BK] \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // load scale A + "flw fa0, (%[A]) \n\t" + "flw fa1, 4(%[A]) \n\t" + "flw fa2, 8(%[A]) \n\t" + "flw fa3, 12(%[A]) \n\t" + "addi %[A], %[A], 16 \n\t" + + "li t1, 4 \n\t" + "addi t2, %[B], 512 \n\t" // B data addr + "addi t3, %[A], 128 \n\t" // A data addr + "addi s4, t2, 1024 \n\t" // scale16 addr + "addi s4, s4, 1024 \n\t" // TODO + "addi s3, %[B], 0 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v1, (s4) \n\t" // load scale16 + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v22, v1, v1, 3 \n\t" + + "addi s4, t3, 256 \n\t" // addr 1 + "addi s5, t3, 512 \n\t" // addr 2 + "addi s6, t3, 768 \n\t" // addr 3 + + // init the accu to 0 + "vxor.vv v24, v24, v24 \n\t" + "vxor.vv v25, v25, v25 \n\t" + "vxor.vv v26, v26, v26 \n\t" + "vxor.vv v27, v27, v27 \n\t" + + "INNER_K_LOOP%=: \n\t" + // load scale B + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v1, (%[B]) \n\t" + "addi %[B], %[B], 128 \n\t" + "vand.vi v1, v1, 0xf \n\t" + + "vfwcvt.f.x.v v20, v1 \n\t" // f16 scale B + "vsetvli t0, x0, e16, m1 \n\t" + "vfmul.vv v0, v20, v22 \n\t" // mul scale16 + "vfmul.vv v1, v21, v22 \n\t" // mul scale16 + "vfmul.vf v0, v0, %[SCALE] \n\t" // mul magic + "vfmul.vf v1, v1, %[SCALE] \n\t" // mul magic + + // A data, 4x64@i8 + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v2, (t3) \n\t" + "addi t3, t3, 64 \n\t" + "vle8.v v3, (s4) \n\t" + "addi s4, s4, 64 \n\t" + "vle8.v v4, (s5) \n\t" + "addi s5, s5, 64 \n\t" + "vle8.v v5, (s6) \n\t" + "addi s6, s6, 64 \n\t" + + // 4x64 => 4x16x4 + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v6, v2, v3, 1 \n\t" + "vpack.vv v8, v4, v5, 1 \n\t" + "vpack.vv v2, v6, v8, 2 \n\t" // 0, 2 + + "vpack.vv v20, v2, v2, 3 \n\t" // 1 + "vor.vv v23, v21, v21 \n\t" + "vpack.vv v20, v3, v3, 3 \n\t" // 3 + + // B data, 32x64@i2 + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v4, (t2) \n\t" + "addi t2, t2, 512 \n\t" + "vand.vi v8, v4, 0x3 \n\t" // 0-15 + "vsrl.vi v9, v4, 2 \n\t" + "vsrl.vi v10, v4, 4 \n\t" + "vsrl.vi v11, v4, 6 \n\t" // 48-63 + "vand.vi v9, v9, 0x3 \n\t" // 16-31 + "vand.vi v10, v10, 0x3 \n\t" // 32-47 + + "vand.vi v12, v5, 0x3 \n\t" // 0-15 + "vsrl.vi v13, v5, 2 \n\t" + "vsrl.vi v14, v5, 4 \n\t" + "vsrl.vi v15, v5, 6 \n\t" // 48-63 + "vand.vi v13, v13, 0x3 \n\t" // 16-31 + "vand.vi v14, v14, 0x3 \n\t" // 32-47 + + "vand.vi v16, v6, 0x3 \n\t" // 0-15 + "vsrl.vi v17, v6, 2 \n\t" + "vsrl.vi v18, v6, 4 \n\t" + "vsrl.vi v19, v6, 6 \n\t" // 48-63 + "vand.vi v17, v17, 0x3 \n\t" // 16-31 + "vand.vi v18, v18, 0x3 \n\t" // 32-47 + + "vand.vi v4, v7, 0x3 \n\t" // 0-15 + "vsrl.vi v5, v7, 2 \n\t" + "vsrl.vi v6, v7, 4 \n\t" + "vsrl.vi v7, v7, 6 \n\t" // 48-63 + "vand.vi v5, v5, 0x3 \n\t" // 16-31 + "vand.vi v6, v6, 0x3 \n\t" // 32-47 + + // i2 * i8 vmadot + "vsetvli t0, x0, e8, m1 \n\t" + "vmadotsu.hp v24, v2, v8, v0, 0, i8 \n\t" + "vmadotsu.hp v25, v2, v12, v0, 1, i8 \n\t" + "vmadotsu.hp v26, v2, v16, v0, 2, i8 \n\t" + "vmadotsu.hp v27, v2, v4, v0, 3, i8 \n\t" + + "vmadotsu.hp v24, v23, v9, v0, 4, i8 \n\t" + "vmadotsu.hp v25, v23, v13, v0, 5, i8\n\t" + "vmadotsu.hp v26, v23, v17, v0, 6, i8\n\t" + "vmadotsu.hp v27, v23, v5, v0, 7, i8 \n\t" + + "vmadotsu.hp v24, v3, v10, v1, 0, i8 \n\t" + "vmadotsu.hp v25, v3, v14, v1, 1, i8 \n\t" + "vmadotsu.hp v26, v3, v18, v1, 2, i8 \n\t" + "vmadotsu.hp v27, v3, v6, v1, 3, i8 \n\t" + + "vmadotsu.hp v24, v21, v11, v1, 4, i8\n\t" + "vmadotsu.hp v25, v21, v15, v1, 5, i8\n\t" + "vmadotsu.hp v26, v21, v19, v1, 6, i8\n\t" + "vmadotsu.hp v27, v21, v7, v1, 7, i8 \n\t" + + "addi t1, t1, -1 \n\t" + "bgtz t1, INNER_K_LOOP%= \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v2, v24, v25, 1 \n\t" + "vpack.vv v4, v26, v27, 1 \n\t" + "vpack.vv v6, v2, v4, 2 \n\t" // 0,1,2,3 + + "vxor.vv v18, v18, v18 \n\t" + "vxor.vv v20, v20, v20 \n\t" + "vxor.vv v22, v22, v22 \n\t" + "vxor.vv v24, v24, v24 \n\t" + // load zp B, 16x8x4@int4 + "vsetvli t0, x0, e8, m4 \n\t" + "vle8.v v0, (s3) \n\t" + "vsrl.vi v0, v0, 4 \n\t" // zp + + // 4x16@int16 + "vsetvli t0, x0, e16, m1 \n\t" // a sum + "vle16.v v12, (%[A]) \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vnsrl.wi v10, v12, 0 \n\t" // low 8 + "vnsra.wi v11, v12, 8 \n\t" // high 8 + + // asum * zp + "vsetvli t0, x0, e32, m1 \n\t" + "vmadotsu v18, v11, v0, i8 \n\t" + "vmadotsu v20, v11, v1, i8 \n\t" + "vmadotsu v22, v11, v2, i8 \n\t" + "vmadotsu v24, v11, v3, i8 \n\t" + "vsll.vi v18, v18, 8 \n\t" + "vsll.vi v20, v20, 8 \n\t" + "vsll.vi v22, v22, 8 \n\t" + "vsll.vi v24, v24, 8 \n\t" + "vmadotu v18, v10, v0, i8 \n\t" + "vmadotu v20, v10, v1, i8 \n\t" + "vmadotu v22, v10, v2, i8 \n\t" + "vmadotu v24, v10, v3, i8 \n\t" + + "vpack.vv v10, v18, v20, 2 \n\t" + "vpack.vv v12, v22, v24, 2 \n\t" + "vpack.vv v14, v10, v12, 3 \n\t" + "vpack.vv v16, v11, v13, 3 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "addi t2, t2, 64 \n\t" + "vle16.v v20, (t2) \n\t" // zero16 + "vfwcvt.f.f.v v22, v20 \n\t" + + // mul 1/magic + "vsetvli t0, x0, e16, m1 \n\t" + "vfwmul.vf v0, v6, %[SCALE_1] \n\t" + "vfwmul.vf v2, v7, %[SCALE_1] \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v14, v14 \n\t" + "vfcvt.f.x.v v15, v15 \n\t" + "vfcvt.f.x.v v16, v16 \n\t" + "vfcvt.f.x.v v17, v17 \n\t" + + "addi %[B], t2, 64 \n\t" + "mv %[A], s6 \n\t" + + "vfmacc.vv v0, v14, v22 \n\t" // + mul zero16 + "vfmacc.vv v1, v15, v22 \n\t" + "vfmacc.vv v2, v16, v22 \n\t" + "vfmacc.vv v3, v17, v22 \n\t" + + "vfmacc.vf v28, fa0, v0 \n\t" // mul a scale + "vfmacc.vf v29, fa1, v1 \n\t" + "vfmacc.vf v30, fa2, v2 \n\t" + "vfmacc.vf v31, fa3, v3 \n\t" + + "addi s1, s1, -1 \n\t" + "bgtz s1, BLK_LOOP%= \n\t" + + // save + "vsetvli t0, x0, e32, m1 \n\t" + "add t1, %[LDC], %[DST] \n\t" + "vse32.v v28, (%[DST]) \n\t" + "vse32.v v29, (t1) \n\t" + "add t1, t1, %[LDC] \n\t" + "vse32.v v30, (t1) \n\t" + "add t1, t1, %[LDC] \n\t" + "vse32.v v31, (t1) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data) + : [DST] "r"(dst_c), [BK] "r"(k_blks), [LDC] "r"(ldc * 4), [SCALE] "f"(scale), [SCALE_1] "f"(scale_1) + : "t0", "t1", "t2", "t3", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30", "v31", "fa0", "t4", "t5", "t6", "s1", "s2", "s3", "s4", "s5", "s6"); + } +} + +void gemm_kernel_i8i3k_m1(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t NB_COLS = 32; //only support 32 in ASM + using blk_type = nrow_block_q3_k; + + const blk_type * b_base = reinterpret_cast(quant_b_data); + + int64_t a_blk_stride = q8k_blk_size(256); + int64_t a_nrow_block_stride = a_blk_stride; + int64_t b_ncol_block_stride = sizeof(blk_type); + + // Constants used by q3_k scaling in HP branch: + // - k_q3k_scale_step: per-nibble scale factor (1/16). + // - k_a_scale_post_mul: A_scale needs an extra *16 at the end (pairs with 1/16 above). + const _Float16 k_q3k_scale_step = (_Float16) 0.0625f; // 1 / 16 + const float k_a_scale_post_mul = 16.0f; + + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + const blk_type * quant_b_blk_data = b_base + (ni / NB_COLS) * k_blks; +#if 0 + //------------------------------------------------------------------------------ + // A format + // Ascale fp32 * 1 32bit + // Asum int16 * 16 256bit + // A M1K256 int8 2048bit + //------------------------------------------------------------------------------ + // B format + // B_scl uint8*N32*16 4096bit + // B_Hmask N32K16*16 1bit 8192bit + // B_Qs N32K16*16 2bit 16384bit + // B scl16 fp16 * N32 512bit; + //------------------------------------------------------------------------------ + //bias always be nullptr + __asm__ volatile( + // t2 = k_blks (each is K256 superblock) + "mv t2, %[KBLKS] \n\t" + // t3 = 256/64 = 4 (K64 iterations per superblock) + "li t3, 4 \n\t" + "mv s2, %[pA] \n\t" // s2 = pASCL + "addi s3, %[pA], 4+32 \n\t" // s3 = pAData, (pA+AScl+ASum) + + // B block layout for nrow_block_q3_k<32>: + // scales: 512B, hmask: 1024B, qs: 2048B, scales16: 64B + "addi s5, %[pB], 32*16 \n\t" // s5 = pB_hmask + "mv s4, %[pB] \n\t" // s4 = pB_scales + "addi s6, s5, 1024 \n\t" // s6 = pB_qs + "mv s7, %[pB] \n\t" // s7 = pB_base + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v31, v0, v0 \n\t" // clear acc + "vxor.vv v30, v0, v0 \n\t" // clear acc of K256 + + // ordinary vmadot: vle*10 vecIns*78 vmadot*16 + ".align 4 \n\t" + "BLK_LPST%=: \n\t" + "K64_LPST%=: \n\t" + + // K0-15 + // load B scales (32 bytes per K16, 16 times => 512B) + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v2, (s4) \n\t" + "addi s4, s4, 128 \n\t" + + // load B qs chunk (128B per K16, 16 times => 2048B) + "vle8.v v4, (s6) \n\t" + "addi s6, s6, 128 \n\t" + "vle8.v v5, (s6) \n\t" + "addi s6, s6, 128 \n\t" + "vle8.v v6, (s6) \n\t" + "addi s6, s6, 128 \n\t" + "vle8.v v7, (s6) \n\t" + "addi s6, s6, 128 \n\t" + + // load B hmask chunk (64B per K16, 16 times => 1024B) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // load A data (16 bytes per K16, 16 times => 256B) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v1, (s3) \n\t" + "addi s3, s3, 64 \n\t" + + // unpack 2-bit qs + hmask -> signed values + "vsetvli t0, x0, e8, m1 \n\t" + "vnot.v v0, v0 \n\t" + "vand.vi v12, v4, 0x3 \n\t" + "vand.vi v13, v5, 0x3 \n\t" + "vand.vi v14, v6, 0x3 \n\t" + "vand.vi v15, v7, 0x3 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v12, v12, -4, v0.t \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadot v16, v1, v12, i8 \n\t" + "vmadot v18, v1, v13, i8 \n\t" + "vmadot v20, v1, v14, i8 \n\t" + "vmadot v22, v1, v15, i8 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v24, v16, v18, 2 \n\t" + "vpack.vv v26, v20, v22, 2 \n\t" + "vpack.vv v16, v24, v26, 3 \n\t" // N0-N31 in v16 + + // apply B int8 scales (-32 bias has been applyed) + "vsetvli t0, x0, e8, mf4 \n\t" + "vwadd.vx v18, v2, x0 \n\t" // int8 -> int16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v19, v18, x0 \n\t" // int8 -> int16 + + // static_cast(qsum) * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vmacc.vv v30, v16, v19 \n\t" + + //K16-31 + // load B scales (32 bytes per K16, 16 times => 512B) + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v2, v2, 4 \n\t" + + // load B hmask chunk (64B per K16, 16 times => 1024B) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // load A data (16 bytes per K16, 16 times => 256B) + "vsetvli t0, x0, e64, mf2 \n\t" + "vslidedown.vi v1, v1, 2 \n\t" + + // unpack 2-bit qs + hmask -> signed values + "vsetvli t0, x0, e8, m1 \n\t" + "vsll.vi v8, v4, 4 \n\t" + "vsll.vi v9, v5, 4 \n\t" + "vsll.vi v10, v6, 4 \n\t" + "vsll.vi v11, v7, 4 \n\t" + "vnot.v v0, v0 \n\t" + + "vsrl.vi v12, v8, 6 \n\t" + "vsrl.vi v13, v9, 6 \n\t" + "vsrl.vi v14, v10, 6 \n\t" + "vsrl.vi v15, v11, 6 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v12, v12, -4, v0.t \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadot v16, v1, v12, i8 \n\t" + "vmadot v18, v1, v13, i8 \n\t" + "vmadot v20, v1, v14, i8 \n\t" + "vmadot v22, v1, v15, i8 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v24, v16, v18, 2 \n\t" + "vpack.vv v26, v20, v22, 2 \n\t" + "vpack.vv v16, v24, v26, 3 \n\t" // N0-N31 in v16 + + // apply B int8 scales (-32 bias has been applyed) + "vsetvli t0, x0, e8, mf4 \n\t" + "vwadd.vx v18, v2, x0 \n\t" // int8 -> int16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v19, v18, x0 \n\t" // int8 -> int16 + + // static_cast(qsum) * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vmacc.vv v30, v16, v19 \n\t" + + //K32-47 + // load B scales (32 bytes per K16, 16 times => 512B) + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v2, v2, 4 \n\t" + + // load B hmask chunk (64B per K16, 16 times => 1024B) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // load A data (16 bytes per K16, 16 times => 256B) + "vsetvli t0, x0, e64, mf2 \n\t" + "vslidedown.vi v1, v1, 2 \n\t" + + // unpack 2-bit qs + hmask -> signed values + "vsetvli t0, x0, e8, m1 \n\t" + "vsll.vi v8, v4, 2 \n\t" + "vsll.vi v9, v5, 2 \n\t" + "vsll.vi v10, v6, 2 \n\t" + "vsll.vi v11, v7, 2 \n\t" + "vnot.v v0, v0 \n\t" + + "vsrl.vi v12, v8, 6 \n\t" + "vsrl.vi v13, v9, 6 \n\t" + "vsrl.vi v14, v10, 6 \n\t" + "vsrl.vi v15, v11, 6 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v12, v12, -4, v0.t \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadot v16, v1, v12, i8 \n\t" + "vmadot v18, v1, v13, i8 \n\t" + "vmadot v20, v1, v14, i8 \n\t" + "vmadot v22, v1, v15, i8 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v24, v16, v18, 2 \n\t" + "vpack.vv v26, v20, v22, 2 \n\t" + "vpack.vv v16, v24, v26, 3 \n\t" + + // apply B int8 scales (-32 bias has been applyed) + "vsetvli t0, x0, e8, mf4 \n\t" + "vwadd.vx v18, v2, x0 \n\t" // int8 -> int16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v19, v18, x0 \n\t" // int8 -> int16 + + // static_cast(qsum) * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vmacc.vv v30, v16, v19 \n\t" + + // K48-63 + // load B scales (32 bytes per K16, 16 times => 512B) + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v2, v2, 4 \n\t" + + // load B hmask chunk (64B per K16, 16 times => 1024B) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // load A data (16 bytes per K16, 16 times => 256B) + "vsetvli t0, x0, e64, mf2 \n\t" + "vslidedown.vi v1, v1, 2 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vnot.v v0, v0 \n\t" + "vsrl.vi v12, v4, 6 \n\t" + "vsrl.vi v13, v5, 6 \n\t" + "vsrl.vi v14, v6, 6 \n\t" + "vsrl.vi v15, v7, 6 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v12, v12, -4, v0.t \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadot v16, v1, v12, i8 \n\t" + "vmadot v18, v1, v13, i8 \n\t" + "vmadot v20, v1, v14, i8 \n\t" + "vmadot v22, v1, v15, i8 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v24, v16, v18, 2 \n\t" + "vpack.vv v26, v20, v22, 2 \n\t" + "vpack.vv v16, v24, v26, 3 \n\t" + + // apply B int8 scales (-32 bias has been applyed) + "vsetvli t0, x0, e8, mf4 \n\t" + "vwadd.vx v18, v2, x0 \n\t" // int8 -> int16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v19, v18, x0 \n\t" // int8 -> int16 + + // static_cast(qsum) * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vmacc.vv v30, v16, v19 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, K64_LPST%= \n\t" + "K64_LPND%=: \n\t" + + // load A scale (fp32) and advance A to next superblock + "flw f0, (s2) \n\t" + "addi s2, s2, 4+32+256 \n\t" + "add t4, s7, %[B_STR] \n\t" // t4 = next B blk base + "addi s3, s2, 4+32 \n\t" + + // load B scales16[32] (fp16) at end of qs region + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v2, (s6) \n\t" + + // pointer modify + "addi s5, t4, 32*16 \n\t" + "mv s4, t4 \n\t" + "addi s6, s5, 32*32 \n\t" + "addi s7, t4, 0 \n\t" + + // b_scale fp16 -> fp32 + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v24, v2 \n\t" + + // a_scale * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v26, v30 \n\t" + "vfmul.vf v1, v24, f0 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + // static_cast(qsum) * a_scale * b_scale; + "vfmacc.vv v31, v1, v26 \n\t" + + // next K-superblock + "addi t2, t2, -1 \n\t" + "vxor.vv v30, v0, v0 \n\t" // clear acc of K256 + "li t3, 4 \n\t" + "bgtz t2, BLK_LPST%= \n\t" + + "BLK_LPND%=: \n\t" + "vsetvli t0, %[NBLKS], e32, m1 \n\t" + "vse32.v v31, (%[pC]) \n\t" + "FUNC_END%=: \n\t" + + : + : [KBLKS] "r"(k_blks), [NBLKS] "r"(nb_real), [pA] "r"(quant_a_ptr), [pB] "r"(quant_b_blk_data), + [pC] "r"(c_ptr), [B_STR] "r"(b_ncol_block_stride) + : "cc", "memory", "t0", "t2", "t3", "t4", "t5", "f0", "s2", "s3", "s4", "s5", "s6", "s7"); +#else + + __asm__ volatile( + // ========================= + // Kernel overview (M1 x N32) + // ========================= + // Process one output row (M=1) and 32 columns (N=32) per call. + // + // Loop structure: + // - Outer loop: K superblocks of size K=256 (k_blks times) + // - Each K256 superblock is broken into 4 x K64 + // - Each K64 is processed as 4 x K16 "sub-blocks" (via unpack+dot) + // + // Data layout (high level): + // A (q8k K=256, per superblock): + // [ fp32 a_scale ][ int16 a_sum[16] ][ int8 a_qs[256] ] + // B (nrow_block_q3_k<32>, per superblock): + // [ int8 scales[32*16] ][ hmask[1024] ][ qs[2048] ][ fp16 scales16[32] ] + // + // Registers/pointers: + // s2: pA (points at A superblock header; used to load fp32 a_scale) + // s3: pA_qs (points at A int8 data within the current superblock) + // s4: pB_scales (points at B int8 per-K16 scales) + // s5: pB_hmask (points at B sign mask area) + // s6: pB_qs (points at B 2-bit packed qs area) + // s8: pB_scales16 (points at B fp16 scales16[32] at the end of block) + // s7: pB_base (base pointer to current B block; used for block-to-block stride) + + // t2 = number of K256 superblocks + "mv t2, %[KBLKS] \n\t" + // t3 = number of K64 chunks per K256 superblock (256 / 64) + "li t3, 4 \n\t" + + // A pointers + "mv s2, %[pA] \n\t" // s2 = pA_superblock (a_scale at +0) + "addi s3, %[pA], 4+32 \n\t" // s3 = pA_qs (skip a_scale + a_sum[16]) + + // B pointers for nrow_block_q3_k<32> + "addi s5, %[pB], 32*16 \n\t" // s5 = pB_hmask (skip scales[32*16]) + "mv s4, %[pB] \n\t" // s4 = pB_scales + "addi s6, s5, 1024 \n\t" // s6 = pB_qs (skip hmask) + // scales16 is at the end of the block: qs(2048) after hmask + "addi s8, s6, 1024 \n\t" + "addi s8, s8, 1024 \n\t" // s8 = pB_scales16 (fp16 scales16[32]) + "mv s7, %[pB] \n\t" // s7 = pB_base (for next-block address calc) + + // v31: final FP32 accumulator for N=32 + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v31, v0, v0 \n\t" + + // ---- Preload B scales16[32] and build FP16 scale vector used by vmadot.hp ---- + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v1, (s8) \n\t" // load fp16 scales16[32] + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v26, v1, v1, 3 \n\t" // broadcast/pack to match lanes + "vmv.v.v v17, v26 \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vfmul.vf v30, v17, %[q3_step] \n\t" // v30 = scales16 * (1/16) + + // v24-v27: fp16 partial accumulators for a K64 chunk (vmadot.hp outputs) + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v16, v16 \n\t" + "vxor.vv v25, v16, v16 \n\t" + "vxor.vv v26, v16, v16 \n\t" + "vxor.vv v27, v16, v16 \n\t" + + // HP vmadot: vle*10 vecIns*38 vmadot.hp*16 + ".align 4 \n\t" + "BLK_LPST%=: \n\t" // loop over K256 superblocks + "K64_LPST%=: \n\t" // loop over 4 x K64 chunks + + // ------------------------------------------------------------ + // K0-15: load B scales + {hmask, qs} + A data; unpack and dot + // ------------------------------------------------------------ + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v2, (s4) \n\t" // B int8 scales for this K16 + "addi s4, s4, 128 \n\t" + + "vle8.v v4, (s6) \n\t" + "addi s6, s6, 128 \n\t" + "vle8.v v5, (s6) \n\t" + "addi s6, s6, 128 \n\t" + "vle8.v v6, (s6) \n\t" + "addi s6, s6, 128 \n\t" + "vle8.v v7, (s6) \n\t" + "addi s6, s6, 128 \n\t" + + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" // B hmask for this K16 + "addi s5, s5, 64 \n\t" + + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v3, (s3) \n\t" // A int8 data for this K16 + "addi s3, s3, 64 \n\t" + + // Convert B int8 scales to FP16 and apply scales16*(1/16) + "vsetvli t0, x0, e8, m1 \n\t" + "vfwcvt.f.x.v v28, v2 \n\t" // int8 -> fp16 + "vsetvli t0, x0, e16, m1 \n\t" + "vfmul.vv v1, v28, v30 \n\t" // v1: FP16 scale vector for vmadot.hp + "vfmul.vv v29, v29, v30 \n\t" + + // Unpack B 2-bit qs + hmask -> signed int8 in v12..v15 + "vsetvli t0, x0, e8, m1 \n\t" + "vnot.v v0, v0 \n\t" + "vand.vi v12, v4, 0x3 \n\t" + "vand.vi v13, v5, 0x3 \n\t" + "vand.vi v14, v6, 0x3 \n\t" + "vand.vi v15, v7, 0x3 \n\t" + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v12, v12, -4, v0.t \n\t" + + // (Next K16 unpack path uses a fresh hmask load) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // Prepare another group from packed qs (bit shifts) + apply sign from hmask + "vsetvli t0, x0, e8, m1 \n\t" + "vsll.vi v8, v4, 4 \n\t" + "vsll.vi v9, v5, 4 \n\t" + "vsll.vi v10, v6, 4 \n\t" + "vsll.vi v11, v7, 4 \n\t" + "vsrl.vi v16, v8, 6 \n\t" + "vsrl.vi v17, v9, 6 \n\t" + "vnot.v v0, v0 \n\t" + "vsrl.vi v18, v10, 6 \n\t" + "vsrl.vi v19, v11, 6 \n\t" + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v16, v16, -4, v0.t \n\t" + + // A shift for the second dot within this K64 + "vsetvli t0, x0, e64, mf2 \n\t" + "vslidedown.vi v2, v3, 2 \n\t" + + // Dot products with FP16 scaling (accumulate into v24..v27) + "vsetvli t0, x0, e32, m1 \n\t" + "vmadot.hp v24, v3, v12, v1, 0, i8 \n\t" + "vmadot.hp v25, v3, v13, v1, 1, i8 \n\t" + "vmadot.hp v26, v3, v14, v1, 2, i8 \n\t" + "vmadot.hp v27, v3, v15, v1, 3, i8 \n\t" + "vmadot.hp v24, v2, v16, v1, 4, i8 \n\t" + "vmadot.hp v25, v2, v17, v1, 5, i8 \n\t" + "vmadot.hp v26, v2, v18, v1, 6, i8 \n\t" + "vmadot.hp v27, v2, v19, v1, 7, i8 \n\t" + + // (K32-47 / K48-63 blocks continue unchanged...) + // load B scales (32 bytes per K16, 16 times => 512B) + "vsetvli t0, x0, e64, m1 \n\t" + "vmv.v.v v1, v29 \n\t" + + // load B hmask chunk (64B per K16, 16 times => 1024B) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // load A data (16 bytes per K16, 16 times => 256B) + "vsetvli t0, x0, e64, mf2 \n\t" + "vslidedown.vi v3, v3, 4 \n\t" + + // unpack 2-bit qs + hmask -> signed values + "vsetvli t0, x0, e8, m1 \n\t" + "vsll.vi v8, v4, 2 \n\t" + "vsll.vi v9, v5, 2 \n\t" + "vsll.vi v10, v6, 2 \n\t" + "vsll.vi v11, v7, 2 \n\t" + + "vsrl.vi v20, v8, 6 \n\t" + "vsrl.vi v21, v9, 6 \n\t" + "vnot.v v0, v0 \n\t" + "vsrl.vi v22, v10, 6 \n\t" + "vsrl.vi v23, v11, 6 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v20, v20, -4, v0.t \n\t" + + // K48-63 + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vsrl.vi v8, v4, 6 \n\t" + "vsrl.vi v9, v5, 6 \n\t" + "vnot.v v0, v0 \n\t" + "vsrl.vi v10, v6, 6 \n\t" + "vsrl.vi v11, v7, 6 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v8, v8, -4, v0.t \n\t" + + // load A data (16 bytes per K16, 16 times => 256B) + "vsetvli t0, x0, e64, mf2 \n\t" + "vslidedown.vi v2, v3, 2 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vmadot.hp v24, v3, v20, v1, 0, i8 \n\t" + "vmadot.hp v25, v3, v21, v1, 1, i8 \n\t" + "vmadot.hp v26, v3, v22, v1, 2, i8 \n\t" + "vmadot.hp v27, v3, v23, v1, 3, i8 \n\t" + "vmadot.hp v24, v2, v8, v1, 4, i8 \n\t" + "vmadot.hp v25, v2, v9, v1, 5, i8 \n\t" + "vmadot.hp v26, v2, v10, v1, 6, i8 \n\t" + "vmadot.hp v27, v2, v11, v1, 7, i8 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, K64_LPST%= \n\t" + "K64_LPND%=: \n\t" + + // ---- End of K64 chunk: reduce fp16 accumulators -> fp32 and scale by A ---- + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v12, v24, v25, 1 \n\t" + "vpack.vv v14, v26, v27, 1 \n\t" + "vpack.vv v16, v12, v14, 2 \n\t" + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v26, v16 \n\t" // fp16 -> fp32 vector (qsum * b_scales) + + // Load A scale and advance A pointer to next K256 superblock + "flw f0, (s2) \n\t" + "addi s2, s2, 4+32+256 \n\t" + "add t4, s7, %[B_STR] \n\t" // next B block base + "addi s3, s2, 4+32 \n\t" // reset A data pointer for next block + + // Advance B pointers to next K256 superblock + "addi s5, t4, 32*16 \n\t" + "mv s4, t4 \n\t" + "addi s6, s5, 32*32 \n\t" + "addi s8, s6, 1024 \n\t" + "addi s8, s8, 1024 \n\t" + "addi s7, t4, 0 \n\t" + "addi t2, t2, -1 \n\t" + + // Final per-block scaling: a_scale * 16.0f + "fmul.s f0, f0, %[a_post_mul] \n\t" + // acc += (qsum * b_scales) * (a_scale*16) + "vsetvli t0, x0, e32, m1 \n\t" + "vfmacc.vf v31, f0, v26 \n\t" + + "beqz t2, BLK_LPND%= \n\t" + + // Preload next block's scales16 and rebuild v30 for vmadot.hp + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v1, (s8) \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v26, v1, v1, 3 \n\t" + "vmv.v.v v17, v26 \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vfmul.vf v30, v17, %[q3_step] \n\t" + + // Reset fp16 partial accumulators for next K64 loop(s) + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v16, v16 \n\t" + "vxor.vv v25, v16, v16 \n\t" + "vxor.vv v26, v16, v16 \n\t" + "vxor.vv v27, v16, v16 \n\t" + + "li t3, 4 \n\t" + "bgtz t2, BLK_LPST%= \n\t" + + "BLK_LPND%=: \n\t" + "vsetvli t0, %[NBLKS], e32, m1 \n\t" + "vse32.v v31, (%[pC]) \n\t" + + : + : [KBLKS] "r"(k_blks), [NBLKS] "r"(nb_real), [pA] "r"(quant_a_ptr), [pB] "r"(quant_b_blk_data), + [pC] "r"(c_ptr), [B_STR] "r"(b_ncol_block_stride), [q3_step] "f"(k_q3k_scale_step), + [a_post_mul] "f"(k_a_scale_post_mul) + : "cc", "memory", "t0", "t2", "t3", "t4", "t5", "f0", "f1", "s2", "s3", "s4", "s5", "s6", "s7", "s8"); +#endif + } +} + +void gemm_kernel_i8i3k_m4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + using blk_type = nrow_block_q3_k<32>; + constexpr size_t NB_COLS = 32; //only support 32 in ASM + + const blk_type * b_base = reinterpret_cast(quant_b_data); + + int64_t a_blk_stride = q8k_blk_size(256); + int64_t a_nrow_block_stride = a_blk_stride * 4; + int64_t b_ncol_block_stride = sizeof(blk_type); + + for (size_t ni = 0; ni < count_n; ni += NB_COLS, c_ptr += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + const blk_type * quant_b_blk_data = b_base + (ni / NB_COLS) * k_blks; + + //------------------------------------------------------------------------------ + // A format + // Ascale fp32 * 1* 4row 128bit + // Asum int16 * 16 4row 1024bit + // A M1K256 int8 4row 8192bit + //------------------------------------------------------------------------------ + // B format + // B_scl uint8*N32*16 4096bit + // B_Hmask N32K16*16 1bit 8192bit + // B_Qs N32K16*16 2bit 16384bit + // B scl16 fp16 * N32 512bit; + //------------------------------------------------------------------------------ + //bias always be nullptr + __asm__ volatile( + // t2 = k_blks (each is K256 superblock) + "mv t2, %[KBLKS] \n\t" + // t3 = 256/64 = 4 (K64 iterations per superblock) + "li t3, 4 \n\t" + "mv s2, %[pA] \n\t" // s2 = pASCL + "addi s3, %[pA], 16+128 \n\t" // s3 = pAData, (pA+AScl+ASum) + + // B block layout for nrow_block_q3_k<32>: + // scales: 512B, hmask: 1024B, qs: 2048B, scales16: 64B + "addi s5, %[pB], 32*16 \n\t" // s5 = pB_hmask (skip scales) + "mv s4, %[pB] \n\t" // s4 = pB_scales + "addi s6, s5, 1024 \n\t" // s6 = pB_qs (skip hmask) + "mv s7, %[pB] \n\t" // s7 = pB_base + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v0, v0 \n\t" // v24-v27: K256 temp accumulator + "vxor.vv v25, v0, v0 \n\t" + "vxor.vv v26, v0, v0 \n\t" + "vxor.vv v27, v0, v0 \n\t" + "vxor.vv v28, v0, v0 \n\t" // v28-v31: final accumulator + "vxor.vv v29, v0, v0 \n\t" + "vxor.vv v30, v0, v0 \n\t" + "vxor.vv v31, v0, v0 \n\t" + + // ordinary vmadot: vle*13 vecIns*96 vmadot*16 + ".align 4 \n\t" + "BLK_LPST%=: \n\t" + "K64_LPST%=: \n\t" + + // ========== K0-15: First K16 sub-block ========== + // Load B INT8 scale factors (32 cols × 16 K16 blocks) + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v8, (s4) \n\t" + "addi s4, s4, 128 \n\t" + + // Load B quantized data (32 cols × 16 elements × 2bit, stored in 4 groups) + "vle8.v v4, (s6) \n\t" + "addi s6, s6, 128 \n\t" + "vle8.v v5, (s6) \n\t" + "addi s6, s6, 128 \n\t" + "vle8.v v6, (s6) \n\t" + "addi s6, s6, 128 \n\t" + "vle8.v v7, (s6) \n\t" + "addi s6, s6, 128 \n\t" + + // Load B hmask (32 cols × 16bit sign mask) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // Load A data (4 rows × 16 elements × INT8) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v12, (s3) \n\t" + "addi s3, s3, 256 \n\t" // Jump to next row + "vle8.v v13, (s3) \n\t" + "addi s3, s3, 256 \n\t" + "vle8.v v14, (s3) \n\t" + "addi s3, s3, 256 \n\t" + "vle8.v v15, (s3) \n\t" + "addi s3, s3, -768+64 \n\t" // Back to first row, advance 16 elements + + // Pack A data: merge 4 rows into 2 vectors + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v16, v12, v13, 1 \n\t" + "vpack.vv v18, v14, v15, 1 \n\t" + "vpack.vv v2, v16, v18, 2 \n\t" + + // unpack 2-bit qs + hmask -> signed values + "vsetvli t0, x0, e8, m1 \n\t" + "vnot.v v0, v0 \n\t" + "vand.vi v12, v4, 0x3 \n\t" + "vand.vi v13, v5, 0x3 \n\t" + "vand.vi v14, v6, 0x3 \n\t" + "vand.vi v15, v7, 0x3 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v12, v12, -4, v0.t \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadot v16, v2, v12, i8 \n\t" // 4 rows × cols 0-7 + "vmadot v18, v2, v13, i8 \n\t" // 4 rows × cols 8-15 + "vmadot v20, v2, v14, i8 \n\t" // 4 rows × cols 16-23 + "vmadot v22, v2, v15, i8 \n\t" // 4 rows × cols 24-31 + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v12, v16, v18, 2 \n\t" // Merge cols 0-15 + "vpack.vv v14, v20, v22, 2 \n\t" // Merge cols 16-31 + "vpack.vv v16, v12, v14, 3 \n\t" // Inter-row results (INT16) + "vpack.vv v18, v13, v15, 3 \n\t" + + // apply B int8 scales (-32 bias has been applyed) + "vsetvli t0, x0, e8, mf4 \n\t" + "vwadd.vx v21, v8, x0 \n\t" // INT8 → INT16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v23, v21, x0 \n\t" // INT16 → INT32 + + // Accumulate to K256 accumulator: qsum * b_scale + "vsetvli t0, x0, e32, m1 \n\t" + "vmacc.vv v24, v16, v23 \n\t" // Row 0 + "vmacc.vv v25, v17, v23 \n\t" // Row 1 + "vmacc.vv v26, v18, v23 \n\t" // Row 2 + "vmacc.vv v27, v19, v23 \n\t" + + // ========== K16-31, K32-47, K48-63: Similar processing ========== + // load B scales (32 bytes per K16, 16 times => 512B) + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v8, v8, 4 \n\t" + + // load B hmask chunk (64B per K16, 16 times => 1024B) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // load A data (16 bytes per K16, 16 times => 256B) + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v2, v2, 8 \n\t" + + // unpack 2-bit qs + hmask -> signed values + "vsetvli t0, x0, e8, m1 \n\t" + "vsll.vi v12, v4, 4 \n\t" + "vsll.vi v13, v5, 4 \n\t" + "vsll.vi v14, v6, 4 \n\t" + "vsll.vi v15, v7, 4 \n\t" + "vnot.v v0, v0 \n\t" + + "vsrl.vi v12, v12, 6 \n\t" + "vsrl.vi v13, v13, 6 \n\t" + "vsrl.vi v14, v14, 6 \n\t" + "vsrl.vi v15, v15, 6 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v12, v12, -4, v0.t \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadot v16, v2, v12, i8 \n\t" + "vmadot v18, v2, v13, i8 \n\t" + "vmadot v20, v2, v14, i8 \n\t" + "vmadot v22, v2, v15, i8 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v12, v16, v18, 2 \n\t" + "vpack.vv v14, v20, v22, 2 \n\t" + "vpack.vv v16, v12, v14, 3 \n\t" // N0-N31 in v16 + "vpack.vv v18, v13, v15, 3 \n\t" + + // apply B int8 scales (-32 bias has been applyed) + "vsetvli t0, x0, e8, mf4 \n\t" + "vwadd.vx v21, v8, x0 \n\t" // int8 -> int16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v23, v21, x0 \n\t" // int8 -> int16 + + // static_cast(qsum) * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vmacc.vv v24, v16, v23 \n\t" + "vmacc.vv v25, v17, v23 \n\t" + "vmacc.vv v26, v18, v23 \n\t" + "vmacc.vv v27, v19, v23 \n\t" + + //K32-47 + // load B scales (32 bytes per K16, 16 times => 512B) + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v8, v8, 4 \n\t" + + // load B hmask chunk (64B per K16, 16 times => 1024B) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // load A data (16 bytes per K16, 16 times => 256B) + + // unpack 2-bit qs + hmask -> signed values + "vsetvli t0, x0, e8, m1 \n\t" + "vsll.vi v12, v4, 2 \n\t" + "vsll.vi v13, v5, 2 \n\t" + "vsll.vi v14, v6, 2 \n\t" + "vsll.vi v15, v7, 2 \n\t" + "vnot.v v0, v0 \n\t" + + "vsrl.vi v12, v12, 6 \n\t" + "vsrl.vi v13, v13, 6 \n\t" + "vsrl.vi v14, v14, 6 \n\t" + "vsrl.vi v15, v15, 6 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v12, v12, -4, v0.t \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadot v16, v3, v12, i8 \n\t" + "vmadot v18, v3, v13, i8 \n\t" + "vmadot v20, v3, v14, i8 \n\t" + "vmadot v22, v3, v15, i8 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v12, v16, v18, 2 \n\t" + "vpack.vv v14, v20, v22, 2 \n\t" + "vpack.vv v16, v12, v14, 3 \n\t" // N0-N31 in v16 + "vpack.vv v18, v13, v15, 3 \n\t" + + // apply B int8 scales (-32 bias has been applyed) + "vsetvli t0, x0, e8, mf4 \n\t" + "vwadd.vx v21, v8, x0 \n\t" // int8 -> int16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v23, v21, x0 \n\t" // int8 -> int16 + + // static_cast(qsum) * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vmacc.vv v24, v16, v23 \n\t" + "vmacc.vv v25, v17, v23 \n\t" + "vmacc.vv v26, v18, v23 \n\t" + "vmacc.vv v27, v19, v23 \n\t" + + // K48-63 + // load B scales (32 bytes per K16, 16 times => 512B) + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v8, v8, 4 \n\t" + + // load B hmask chunk (64B per K16, 16 times => 1024B) + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s5) \n\t" + "addi s5, s5, 64 \n\t" + + // load A data (16 bytes per K16, 16 times => 256B) + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v3, v3, 8 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vnot.v v0, v0 \n\t" + "vsrl.vi v12, v4, 6 \n\t" + "vsrl.vi v13, v5, 6 \n\t" + "vsrl.vi v14, v6, 6 \n\t" + "vsrl.vi v15, v7, 6 \n\t" + + "vsetvli t0, x0, e8, m4 \n\t" + "vadd.vi v12, v12, -4, v0.t \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadot v16, v3, v12, i8 \n\t" + "vmadot v18, v3, v13, i8 \n\t" + "vmadot v20, v3, v14, i8 \n\t" + "vmadot v22, v3, v15, i8 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v12, v16, v18, 2 \n\t" + "vpack.vv v14, v20, v22, 2 \n\t" + "vpack.vv v16, v12, v14, 3 \n\t" // N0-N31 in v16 + "vpack.vv v18, v13, v15, 3 \n\t" + + // apply B int8 scales (-32 bias has been applyed) + "vsetvli t0, x0, e8, mf4 \n\t" + "vwadd.vx v21, v8, x0 \n\t" // int8 -> int16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v23, v21, x0 \n\t" // int8 -> int16 + + // static_cast(qsum) * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vmacc.vv v24, v16, v23 \n\t" + "vmacc.vv v25, v17, v23 \n\t" + "vmacc.vv v26, v18, v23 \n\t" + "vmacc.vv v27, v19, v23 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, K64_LPST%= \n\t" + "K64_LPND%=: \n\t" + + // ========== K256 superblock complete, apply scale factors ========== + // Load A's 4 row scale factors (FP32) + "flw f0, (s2) \n\t" + "flw f1, 4(s2) \n\t" + "flw f2, 8(s2) \n\t" + "flw f3, 12(s2) \n\t" + "add s2, s2, %[A_STR] \n\t" // Advance to next superblock + "add t4, s7, %[B_STR] \n\t" // t4 = next B block address + "addi s3, s2, (4+32)*4 \n\t" + + // Load B FP16 global scale factors (32 cols) + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v8, (s6) \n\t" + + // Update B pointers to next block + "addi s5, t4, 32*16 \n\t" + "mv s4, t4 \n\t" + "addi s6, s5, 32*32 \n\t" + "addi s7, t4, 0 \n\t" + + // ========== Type conversion and final scaling ========== + // FP16 → FP32 + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v9, v8 \n\t" + + // INT32 → FP32 + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v24, v24 \n\t" + "vfcvt.f.x.v v25, v25 \n\t" + "vfcvt.f.x.v v26, v26 \n\t" + "vfcvt.f.x.v v27, v27 \n\t" + + // Compute a_scale * b_scale (4 rows) + "vfmul.vf v12, v9, f0 \n\t" + "vfmul.vf v13, v9, f1 \n\t" + "vfmul.vf v14, v9, f2 \n\t" + "vfmul.vf v15, v9, f3 \n\t" + + // Final accumulation: result += qsum * a_scale * b_scale + "vsetvli t0, x0, e32, m1 \n\t" + "vfmacc.vv v28, v12, v24 \n\t" + "vfmacc.vv v29, v13, v25 \n\t" + "vfmacc.vv v30, v14, v26 \n\t" + "vfmacc.vv v31, v15, v27 \n\t" + + // Prepare for next K superblock + "addi t2, t2, -1 \n\t" + "vxor.vv v24, v0, v0 \n\t" // Clear K256 accumulator + "vxor.vv v25, v0, v0 \n\t" + "vxor.vv v26, v0, v0 \n\t" + "vxor.vv v27, v0, v0 \n\t" + "li t3, 4 \n\t" + "bgtz t2, BLK_LPST%= \n\t" + + "BLK_LPND%=: \n\t" + + // ========== Store results (4 rows × 32 cols) ========== + "mv t5, %[pC] \n\t" + "vsetvli t0, %[NBLKS], e32, m1 \n\t" + "vse32.v v28, (%[pC]) \n\t" + "add t5, t5, %[LDC] \n\t" + "vse32.v v29, (t5) \n\t" + "add t5, t5, %[LDC] \n\t" + "vse32.v v30, (t5) \n\t" + "add t5, t5, %[LDC] \n\t" + "vse32.v v31, (t5) \n\t" + "add t5, t5, %[LDC] \n\t" + "FUNC_END%=: \n\t" + + : + : [KBLKS] "r"(k_blks), [NBLKS] "r"(nb_real), [pA] "r"(quant_a_ptr), [pB] "r"(quant_b_blk_data), + [pC] "r"(c_ptr), [B_STR] "r"(b_ncol_block_stride), [A_STR] "r"(a_nrow_block_stride), [LDC] "r"(ldc * 4) + : "cc", "memory", "t0", "t2", "t3", "t4", "t5", "f0", "f1", "f2", "f3", "s2", "s3", "s4", "s5", "s6", "s7"); + } +} + +void gemm_kernel_i8i4_m1(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + if (quant_b_zp == NULL) { + for (size_t n = 0; n < count_n; n += 32) { + size_t nblks = (count_n - n) > 32 ? 32 : count_n - n; + uint8_t * QuantBDataPtr = (uint8_t *) quant_b_data + // + n * k_blks * blk_len / 2 + // b data + n * k_blks * sizeof(_Float16); // scale + float * CPtr = c_ptr + n; + size_t cnt = k_blks; + + // A format Version_1 (FP32 SCALE FOR Normal VMADOTins of IME2) + // A M1K32 int8 256bit + // Ascale fp32 * 1 32bit + // || scl*1(fp32) | Asum(int16) | blk0 || scl*1(fp32) | Asum(int16) | blk0 || ... + // || Element || Element || ... + // B format + // B N8K32 int4 1024bit + // 4VRF, N32K32, 4096bit + // Bscale fp16 * N32 512bit; + // || scl*32..(fp16) | blk0 blk1 ... blk31 || scl*32..(fp16) | blk0 blk1 ... blk31 || ... + // || Element || Element || ... +#if 0 + //bias always be nullptr + __asm__ volatile( + + // t3 = k/32 + "mv t3, %[BCK] \n\t" + "mv t4, %[NBLKS] \n\t" + "mv s2, %[pA] \n\t" // s2 = pASCL + "addi s3, %[pA], 4+2 \n\t" // s3 = pAData, (pA+AScl+ASum) + "mv s4, %[pB] \n\t" // s4 = pBSCL + "addi s5, %[pB], 32*2 \n\t" // s5 = pBdata; + "mv s6, %[pC] \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v2, v0, v0 \n\t" // clear acc + + // ordinary vmadot: vle*6 flw*1 vecIns*21 vmadot*8 + ".align 4 \n\t" + "_K_LPST%=: \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v4, (s5) \n\t" // B Data 4VRF * 8Row * 32 + "addi s5, s5, 128*4+64 \n\t" // 1024bit + + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s4) \n\t" // B Scale 4VRF*8Row*FP16 = 512bit + "addi s4, s4, 64+128*4 \n\t" + + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v3, (s3) \n\t" // A Data M1*K32*int8 = 256bit + "addi s3, s3, 32+6 \n\t" + + "flw f0, (s2) \n\t" // A Scale fp32 + "lh t2, 4(s2) \n\t" // A sum of int16 + "addi s2, s2, 6+32 \n\t" + + "vsetvli t0, zero, e8, m1 \n\t" + "vsrl.vi v24, v3, 4 \n\t" + + "vnpack4.vv v8, v3, v3, 3 \n\t" // lo4 of A + "vnpack4.vv v10, v24, v24, 3 \n\t" // hi4 of A + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadotsu v16, v10, v4, i4 \n\t" // M0 N0 - N7 INT32(256bit) + "vmadotsu v18, v10, v5, i4 \n\t" // M0 N8 - N15 + "vmadotsu v20, v10, v6, i4 \n\t" // M0 N16 - N23 + "vmadotsu v22, v10, v7, i4 \n\t" // M0 N24 - N31 + + "vsll.vi v16, v16, 4 \n\t" + "vsll.vi v18, v18, 4 \n\t" + "vsll.vi v20, v20, 4 \n\t" + "vsll.vi v22, v22, 4 \n\t" + + "vmadotu v16, v8, v4, i4 \n\t" + "vmadotu v18, v8, v5, i4 \n\t" + "vmadotu v20, v8, v6, i4 \n\t" + "vmadotu v22, v8, v7, i4 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vmv.v.i v28, 8 \n\t" + "vpack.vv v24, v16, v18, 2 \n\t" + "vpack.vv v26, v20, v22, 2 \n\t" + "vpack.vv v16, v24, v26, 3 \n\t" + + "vwmul.vx v24, v28, t2 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vadd.vv v16, v16, v24 \n\t" + + // b_scale fp16 -> fp32 + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v24, v0 \n\t" + // mac result i32 -> fp32 + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v26, v16 \n\t" + // a_scale * b_scale; + "vfmul.vf v1, v24, f0 \n\t" + // static_cast(qsum) * a_scale * b_scale; + "vfmacc.vv v2, v1, v26 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, _K_LPST%= \n\t" + "_K_LPND%=: \n\t" + + //----------------------------------------- + // STORE Equal 32N------------------------- + "_ST32%=: \n\t" + "vsetvli t0, t4, e32, m1 \n\t" + "vse32.v v2, (s6) \n\t" // M0 [N0 : N32]; FP32(1024bit) + + "_FUNC_END%=: \n\t" + + : + : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) + : "cc", "t0", "t2", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6"); +#else + __asm__ volatile( + + // t3 = k/32 + "mv t3, %[BCK] \n\t" + "mv t4, %[NBLKS] \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vmv.v.i v0, 1 \n\t" // init the scale + "mv s2, %[pA] \n\t" // s2 = pASCL + "addi s3, %[pA], 4+2 \n\t" // s3 = pAData, (pA+AScl+ASum) + "mv s4, %[pB] \n\t" // s4 = pBSCL + "addi s5, %[pB], 32*2 \n\t" // s5 = pBdata; + "mv s6, %[pC] \n\t" + + "vsll.vi v1, v0, 4 \n\t" + "vxor.vv v2, v0, v0 \n\t" // clear acc + "vfcvt.f.x.v v0, v0 \n\t" + "vfcvt.f.x.v v1, v1 \n\t" + + // vmadot hp: vle*7 flw*1 vecIns*14 vmadot*8 + ".align 4 \n\t" + "_K_LPST%=: \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v4, (s5) \n\t" // B Data 4VRF * 8Row * 32 + "addi s5, s5, 128*4+64 \n\t" // 1024bit + + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v30, (s4) \n\t" // B Scale 4VRF*8Row*FP16 = 512bit + "addi s4, s4, 64+128*4 \n\t" + + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v3, (s3) \n\t" // A Data M1*K32*int8 = 256bit + "addi s3, s3, 32+6 \n\t" + + "flw f0, (s2) \n\t" // A Scale fp32 + "lh t2, 4(s2) \n\t" // A sum of int16 + "addi s2, s2, 6+32 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vmv.v.i v28, 8 \n\t" // Bzp u8 -> u16 + "vsetvli t0, x0, e8, m1 \n\t" + "vsrl.vi v24, v3, 4 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vmul.vx v26, v28, t2 \n\t" // asum*zp i16*i16 + "vnpack4.vv v8, v3, v3, 3 \n\t" // lo4 of A + "vnpack4.vv v10, v24, v24, 3 \n\t" // hi4 of A + + "vfcvt.f.x.v v16, v26 \n\t" // zp i16 -> fp16 + "vadd.vi v18, v16, 0 \n\t" + "vadd.vi v20, v16, 0 \n\t" + "vadd.vi v22, v16, 0 \n\t" + + "vmadotsu.hp v16, v10, v4, v1, 0, i4 \n\t" // high 4 + "vmadotsu.hp v18, v10, v5, v1, 0, i4 \n\t" + "vmadotsu.hp v20, v10, v6, v1, 0, i4 \n\t" + "vmadotsu.hp v22, v10, v7, v1, 0, i4 \n\t" + "vmadotu.hp v16, v8, v4, v0, 0, i4 \n\t" // low 4 + "vmadotu.hp v18, v8, v5, v0, 0, i4 \n\t" + "vmadotu.hp v20, v8, v6, v0, 0, i4 \n\t" + "vmadotu.hp v22, v8, v7, v0, 0, i4 \n\t" + + "vpack.vv v24, v16, v18, 1 \n\t" + "vpack.vv v26, v20, v22, 1 \n\t" + "vpack.vv v16, v24, v26, 2 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + // mac result * b_scale; f16*f16->f32 + "vfwmul.vv v31, v30, v16 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + // static_cast(qsum * b_scale) * a_scale; + "vfmacc.vf v2, f0, v31 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, _K_LPST%= \n\t" + "_K_LPND%=: \n\t" + + //----------------------------------------- + // STORE Equal 32N------------------------- + "_ST32%=: \n\t" + "vsetvli t0, t4, e32, m1 \n\t" + "vse32.v v2, (s6) \n\t" // M0 [N0 : N32]; FP32(1024bit) + + "_FUNC_END%=: \n\t" + + : + : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) + : "cc", "t0", "t2", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6"); + +#endif + } + } else { + for (size_t n = 0; n < count_n; n += 32) { + size_t nblks = (count_n - n) > 32 ? 32 : count_n - n; + uint8_t * QuantBDataPtr = (uint8_t *) quant_b_data + // + n * k_blks * blk_len / 2 + // b data + n * k_blks * sizeof(uint8_t) + // b zp + n * k_blks * sizeof(_Float16); // scale + float * CPtr = c_ptr + n; + size_t cnt = k_blks; + + // A format Version_1 (FP32 SCALE FOR Normal VMADOTins of IME2) + // A M1K32 int8 256bit + // Ascale fp32 * 1 32bit + // || scl*1(fp32) | Asum(int16) | blk0 || scl*1(fp32) | Asum(int16) | blk0 || ... + // || Element || Element || ... + // B format + // B N8K32 int4 1024bit + // 4VRF, N32K32, 4096bit + // Bscale fp16 * N32 512bit; + // Bzp uint8_t * N32 256bit; + // || scl*32..(fp16) | zp*32(uint8) | blk0 blk1 ... blk31 || scl*32..(fp16) ... + // || Element || Element ... + + //bias always be nullptr +#if 0 + __asm__ volatile( + + // t3 = k/32 + "mv t3, %[BCK] \n\t" + "mv t4, %[NBLKS] \n\t" + "mv s2, %[pA] \n\t" // s2 = pASCL + "addi s3, %[pA], 4+2 \n\t" // s3 = pAData, (pA+AScl+ASum) + "mv s4, %[pB] \n\t" // s4 = pBSCL + "addi s5, %[pB], 32*3 \n\t" // s5 = pBdata, (pB+BScl+Bzp) + "mv s6, %[pC] \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v2, v0, v0 \n\t" // clear acc + + // ordinary vmadot: vle*6 flw*1 vecIns*21 vmadot*8 + ".align 4 \n\t" + "_K_LPST%=: \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v4, (s5) \n\t" // B Data 4VRF * 8Row * 32 + "addi s5, s5, 128*4+96 \n\t" // 1024bit + + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s4) \n\t" // B Scale 4VRF*8Row*FP16 = 512bit + "addi s4, s4, 64 \n\t" + + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v3, (s3) \n\t" // A Data M1*K32*int8 = 256bit + "addi s3, s3, 32+6 \n\t" + + "flw f0, (s2) \n\t" // A Scale fp32 + "lh t2, 4(s2) \n\t" // A sum of int16 + "addi s2, s2, 6+32 \n\t" + + "vsetvli t0, zero, e8, m1 \n\t" + "vsrl.vi v24, v3, 4 \n\t" + + "vnpack4.vv v8, v3, v3, 3 \n\t" // lo4 of A + "vnpack4.vv v10, v24, v24, 3 \n\t" // hi4 of A + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadotsu v16, v10, v4, i4 \n\t" // M0 N0 - N7 INT32(256bit) + "vmadotsu v18, v10, v5, i4 \n\t" // M0 N8 - N15 + "vmadotsu v20, v10, v6, i4 \n\t" // M0 N16 - N23 + "vmadotsu v22, v10, v7, i4 \n\t" // M0 N24 - N31 + + "vsll.vi v16, v16, 4 \n\t" + "vsll.vi v18, v18, 4 \n\t" + "vsll.vi v20, v20, 4 \n\t" + "vsll.vi v22, v22, 4 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v1, (s4) \n\t" // Bzp + "addi s4, s4, 32+128*4 \n\t" + + "vmadotu v16, v8, v4, i4 \n\t" + "vmadotu v18, v8, v5, i4 \n\t" + "vmadotu v20, v8, v6, i4 \n\t" + "vmadotu v22, v8, v7, i4 \n\t" + + "vwaddu.vx v28, v1, x0 \n\t" // uint8 -> uint16 + "vpack.vv v24, v16, v18, 2 \n\t" + "vpack.vv v26, v20, v22, 2 \n\t" + "vpack.vv v16, v24, v26, 3 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vwmul.vx v24, v28, t2 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vadd.vv v16, v16, v24 \n\t" + + // b_scale fp16 -> fp32 + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v24, v0 \n\t" + // mac result i32 -> fp32 + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v26, v16 \n\t" + // a_scale * b_scale; + "vfmul.vf v1, v24, f0 \n\t" + // static_cast(qsum) * a_scale * b_scale; + "vfmacc.vv v2, v1, v26 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, _K_LPST%= \n\t" + "_K_LPND%=: \n\t" + + //----------------------------------------- + // STORE Equal 32N------------------------- + "_ST32%=: \n\t" + "vsetvli t0, t4, e32, m1 \n\t" + "vse32.v v2, (s6) \n\t" // M0 [N0 : N32]; FP32(1024bit) + + "_FUNC_END%=: \n\t" + + : + : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) + : "cc", "t0", "t2", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6"); +#else + __asm__ volatile( + + // t3 = k/32 + "mv t3, %[BCK] \n\t" + "mv t4, %[NBLKS] \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vmv.v.i v0, 1 \n\t" // init the scale + "mv s2, %[pA] \n\t" // s2 = pASCL + "addi s3, %[pA], 4+2 \n\t" // s3 = pAData, (pA+AScl+ASum) + "mv s4, %[pB] \n\t" // s4 = pBSCL + "addi s5, %[pB], 32*3 \n\t" // s5 = pBdata, (pB+BScl+Bzp) + "mv s6, %[pC] \n\t" + + "vsll.vi v1, v0, 4 \n\t" + "vxor.vv v2, v0, v0 \n\t" // clear acc + "vfcvt.f.x.v v0, v0 \n\t" + "vfcvt.f.x.v v1, v1 \n\t" + + // vmadot hp: vle*6 flw*1 vecIns*14 vmadot*8 + ".align 4 \n\t" + "_K_LPST%=: \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v4, (s5) \n\t" // B Data 4VRF * 8Row * 32 + "addi s5, s5, 128*4+96 \n\t" // 1024bit + + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v30, (s4) \n\t" // B Scale 4VRF*8Row*FP16 = 512bit + "addi s4, s4, 64 \n\t" + + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v31, (s4) \n\t" // B zp 32Row*uint8 = 256bit + "addi s4, s4, 32+128*4 \n\t" + + "vle8.v v3, (s3) \n\t" // A Data M1*K32*int8 = 256bit + "addi s3, s3, 32+6 \n\t" + + "flw f0, (s2) \n\t" // A Scale fp32 + "lh t2, 4(s2) \n\t" // A sum of int16 + "addi s2, s2, 6+32 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vsrl.vi v24, v3, 4 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vnpack4.vv v8, v3, v3, 3 \n\t" // lo4 of A + "vnpack4.vv v10, v24, v24, 3 \n\t" // hi4 of A + + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadotsu.hp v16, v10, v4, v1, 0, i4 \n\t" // high 4 + "vmadotsu.hp v18, v10, v5, v1, 0, i4 \n\t" + "vmadotsu.hp v20, v10, v6, v1, 0, i4 \n\t" + "vmadotsu.hp v22, v10, v7, v1, 0, i4 \n\t" + "vmadotu.hp v16, v8, v4, v0, 0, i4 \n\t" // low 4 + "vmadotu.hp v18, v8, v5, v0, 0, i4 \n\t" + "vmadotu.hp v20, v8, v6, v0, 0, i4 \n\t" + "vmadotu.hp v22, v8, v7, v0, 0, i4 \n\t" + + "vsetvli t0, x0, e8, mf4 \n\t" + "vwaddu.vx v28, v31, x0 \n\t" // Bzp u8 -> u16 + + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v24, v16, v18, 1 \n\t" + "vpack.vv v26, v20, v22, 1 \n\t" + "vpack.vv v16, v24, v26, 2 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "vmul.vx v26, v28, t2 \n\t" // asum*zp i16*i16 + "vfwcvt.f.f.v v22, v30 \n\t" // b_scale fp16 -> fp32 + "vfcvt.f.x.v v18, v26 \n\t" // zp i16 -> fp16 + "vsetvli t0, x0, e16, m1 \n\t" + "vfwadd.vv v20, v18, v16 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + // mac result * b_scale; f32*f32->f32 + "vfmul.vv v31, v22, v20 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + // static_cast(qsum * b_scale) * a_scale; + "vfmacc.vf v2, f0, v31 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, _K_LPST%= \n\t" + "_K_LPND%=: \n\t" + + //----------------------------------------- + // STORE Equal 32N------------------------- + "_ST32%=: \n\t" + "vsetvli t0, t4, e32, m1 \n\t" + "vse32.v v2, (s6) \n\t" // M0 [N0 : N32]; FP32(1024bit) + + "_FUNC_END%=: \n\t" + + : + : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) + : "cc", "t0", "t2", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6"); +#endif + } + } +} + +void gemm_kernel_i8i4_hp_m1(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t NB_COLS = 32; + constexpr size_t k_subblks_per_superblk = 8; + + struct block_q4_0x32_layout { + _Float16 d[NB_COLS]; + uint8_t qs[16 * NB_COLS]; + }; + + GGML_ASSERT(blk_len == 256); + + const size_t b_superblk_stride = sizeof(block_q4_0x32_layout) * k_subblks_per_superblk + + (quant_b_zp ? NB_COLS * k_subblks_per_superblk * sizeof(uint8_t) : 0); + const size_t b_tile_stride = k_blks * b_superblk_stride; + + if (quant_b_zp == NULL) { + for (size_t ni = 0; ni < count_n; ni += 32) { + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * b_tile_stride; + int8_t * a_data = (int8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + + asm volatile( + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v31, v31, v31 \n\t" // init acc to zero + "mv t4, %[BK] \n\t" + "li t0, 0x4c00 \n\t" // 16 in fp16 + "fmv.h.x fa0, t0 \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + "li t5, 8 \n\t" + "addi t6, %[A], 288 \n\t" // point to blk scale + "flh ft1, (t6) \n\t" + "addi t6, %[A], 272 \n\t" // point to asum + + // init the acc fp16 + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v16, v18, v18 \n\t" + "vxor.vv v17, v18, v18 \n\t" + "vxor.vv v18, v18, v18 \n\t" + "vxor.vv v19, v18, v18 \n\t" + + "INNER_BLK_LOOP%=: \n\t" + // load a sum and scale + "flh fa1, (t6) \n\t" + "addi t6, t6, 2 \n\t" + "flh ft0, (%[A]) \n\t" + "addi %[A], %[A], 2 \n\t" + // load A + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v3, (%[A]) \n\t" // 1x32@i8 + "addi %[A], %[A], 32 \n\t" + + // load scale B and B + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v8, (%[B]) \n\t" // b_scale fp16 + "addi %[B], %[B], 64 \n\t" + "vl4r.v v4, (%[B]) \n\t" // 32*32@i4 + "addi %[B], %[B], 512 \n\t" + "vfmul.vf v8, v8, ft0 \n\t" // scale b * scale a + "vfmul.vf v9, v8, fa0 \n\t" + "vfmul.vf v10, v8, fa1 \n\t" // scale b * scale a * asm + "vfwmacc.vf v31, ft1, v10 \n\t" // asum * scale a * scale b * blk scale + + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v0, v8, v9, 3 \n\t" + "vsrl.vi v28, v3, 4 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vnpack4.vv v2, v3, v3, 3 \n\t" // lo4 of A + "vnpack4.vv v3, v28, v28, 3 \n\t" // hi4 of A + + // i4 * i4 vmadot + "vsetvli t0, x0, e16, m1 \n\t" + "vmadotsu.hp v16, v3, v4, v0, 4, i4 \n\t" // high 4 + "vmadotsu.hp v17, v3, v5, v0, 5, i4 \n\t" + "vmadotsu.hp v18, v3, v6, v0, 6, i4 \n\t" + "vmadotsu.hp v19, v3, v7, v0, 7, i4 \n\t" + "vmadotu.hp v16, v2, v4, v0, 0, i4 \n\t" // low 4 + "vmadotu.hp v17, v2, v5, v0, 1, i4 \n\t" + "vmadotu.hp v18, v2, v6, v0, 2, i4 \n\t" + "vmadotu.hp v19, v2, v7, v0, 3, i4 \n\t" + + "addi t5, t5, -1 \n\t" + "bgtz t5, INNER_BLK_LOOP%= \n\t" + + "vpack.vv v8, v16, v17, 1 \n\t" + "vpack.vv v12, v18, v19, 1 \n\t" + "vpack.vv v20, v8, v12, 2 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "addi t4, t4, -1 \n\t" + "vfwmacc.vf v31, ft1, v20 \n\t" + //"vsetvli t0, x0, e32, m1 \n\t" + //"vfmul.vf v31, v31, ft1 \n\t" // blk scale + + // update A ptr + "addi %[A], t6, 2 \n\t" + + "bgtz t4, BLK_LOOP%= \n\t" + + // save + "vsetvli t0, x0, e32, m1 \n\t" + "vse32.v v31, (%[DST]) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data) + : [DST] "r"(dst_c), [BK] "r"(k_blks) + : "t0", "t1", "t2", "t3", "t4", "t5", "t6", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", + "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "ft0", "ft1"); + } + } else { + // TODO: support quant_b_zp for i8i4 hp kernel + GGML_ABORT("gemm_kernel_i8i4_hp_m1 with quant_b_zp is not supported yet"); + } +} + +void gemm_kernel_i8i4_m4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + int64_t b_data_stride = + k_blks * (sizeof(ggml_fp16_t) + 16 * sizeof(int8_t) + (quant_b_zp != NULL ? sizeof(int8_t) : 0)); + if (quant_b_zp == NULL) { + for (size_t ni = 0; ni < count_n; ni += 32) { + uint8_t * b_data = (uint8_t *) quant_b_data + ni * b_data_stride; + int8_t * a_data = (int8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; +#if 0 + asm volatile( + "li t1, 8 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v29, v29, v29 \n\t" + "vxor.vv v30, v30, v30 \n\t" + "vxor.vv v31, v31, v31 \n\t" + "mv t4, %[BK] \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // load scale A + "flw fa0, (%[A]) \n\t" + "flw fa1, 4(%[A]) \n\t" + "flw fa2, 8(%[A]) \n\t" + "flw fa3, 12(%[A]) \n\t" + "addi %[A], %[A], 16 \n\t" + + // load scale B + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v12, (%[B]) \n\t" + "addi %[B], %[B], 64 \n\t" + "vfwcvt.f.f.v v14, v12 \n\t" + + "vsetivli t0, 4, e16, mf2 \n\t" + "vle16.v v8, (%[A]) \n\t" // asum + "addi %[A], %[A], 8 \n\t" + "vwmul.vx v10, v8, t1 \n\t" // 8*asum + + "vsetvli t0, x0, e8, m1 \n\t" + "vl1r.v v0, (%[A]) \n\t" + "addi %[A], %[A], 128 \n\t" // 4*32@i8 + "vl4r.v v4, (%[B]) \n\t" // 32*32@i4 + "addi %[B], %[B], 512 \n\t" + "vsrl.vi v1, v0, 4 \n\t" + "vnpack4.vv v12, v0, v1, 3 \n\t" // A low u4 + "vupack.vv v2, v12, v12, 2 \n\t" + + // init the accumu to asum * zp + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + // i4 * i4 vmadot + "vsetvli t0, x0, e32, m1 \n\t" + "vmadotsu v16, v3, v4, i4 \n\t" // high 4 + "vmadotsu v18, v3, v5, i4 \n\t" + "vmadotsu v20, v3, v6, i4 \n\t" + "vmadotsu v22, v3, v7, i4 \n\t" + "vsll.vi v16, v16, 4 \n\t" + "vsll.vi v18, v18, 4 \n\t" + "vsll.vi v20, v20, 4 \n\t" + "vsll.vi v22, v22, 4 \n\t" + "vmadotu v16, v2, v4, i4 \n\t" // low 4 + "vmadotu v18, v2, v5, i4 \n\t" + "vmadotu v20, v2, v6, i4 \n\t" + "vmadotu v22, v2, v7, i4 \n\t" + + "vpack.vv v0, v16, v18, 2 \n\t" + "vpack.vv v2, v20, v22, 2 \n\t" + "vpack.vv v16, v0, v2, 3 \n\t" + "vpack.vv v18, v1, v3, 3 \n\t" + + "vrgather.vi v0, v10, 0 \n\t" + "vrgather.vi v1, v10, 1 \n\t" + "vrgather.vi v2, v10, 2 \n\t" + "vrgather.vi v3, v10, 3 \n\t" + + "vadd.vv v16, v16, v0 \n\t" + "vadd.vv v17, v17, v1 \n\t" + "vadd.vv v18, v18, v2 \n\t" + "vadd.vv v19, v19, v3 \n\t" + + "vfcvt.f.x.v v16, v16 \n\t" + "vfcvt.f.x.v v17, v17 \n\t" + "vfcvt.f.x.v v18, v18 \n\t" + "vfcvt.f.x.v v19, v19 \n\t" + + // mul scale + "vfmul.vv v16, v16, v14 \n\t" + "vfmul.vv v17, v17, v14 \n\t" + "vfmul.vv v18, v18, v14 \n\t" + "vfmul.vv v19, v19, v14 \n\t" + + "addi t4, t4, -1 \n\t" + "vfmacc.vf v28, fa0, v16 \n\t" + "vfmacc.vf v29, fa1, v17 \n\t" + "vfmacc.vf v30, fa2, v18 \n\t" + "vfmacc.vf v31, fa3, v19 \n\t" + + "bgtz t4, BLK_LOOP%= \n\t" + + // save + "vsetvli t0, x0, e32, m1 \n\t" + "add t2, %[LDC], %[DST] \n\t" + "vse32.v v28, (%[DST]) \n\t" + "add t3, %[LDC], t2 \n\t" + "vse32.v v29, (t2) \n\t" + "add t2, %[LDC], t3 \n\t" + "vse32.v v30, (t3) \n\t" + "vse32.v v31, (t2) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data) + : [DST] "r"(dst_c), [LDC] "r"(ldc*4), [BK] "r"(k_blks) + : "t0", "t1", "t2", "t3", "t4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", + "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", + "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "fa2", "fa3"); +#else + asm volatile( + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v29, v29, v29 \n\t" + "vxor.vv v30, v30, v30 \n\t" + "vxor.vv v31, v31, v31 \n\t" + "vmv.v.i v0, 1 \n\t" // init the scale + "vsll.vi v1, v0, 4 \n\t" + "vfcvt.f.x.v v0, v0 \n\t" + "vfcvt.f.x.v v1, v1 \n\t" + "mv t4, %[BK] \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // load scale A + "flw fa0, (%[A]) \n\t" + "flw fa1, 4(%[A]) \n\t" + "flw fa2, 8(%[A]) \n\t" + "flw fa3, 12(%[A]) \n\t" + "addi %[A], %[A], 16 \n\t" + + // load scale B + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v12, (%[B]) \n\t" + "addi %[B], %[B], 64 \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v14, v12, v12, 3 \n\t" + + "vsetivli t0, 4, e16, mf2 \n\t" + "vle16.v v8, (%[A]) \n\t" // asum + "addi %[A], %[A], 8 \n\t" + "vsll.vi v8, v8, 3 \n\t" // asum * 8 + "vfcvt.f.x.v v9, v8 \n\t" + "vsetvli t0, x0, e64, m1 \n\t" + "vrgather.vi v10, v9, 0 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vl1r.v v16, (%[A]) \n\t" + "addi %[A], %[A], 128 \n\t" // 4*32@i8 + "vl4r.v v4, (%[B]) \n\t" // 32*32@i4 + "addi %[B], %[B], 512 \n\t" + "vsrl.vi v17, v16, 4 \n\t" + "vnpack4.vv v12, v16, v17, 3 \n\t" // A low u4 + "vupack.vv v2, v12, v12, 2 \n\t" + + // init the accumu to asum * zp + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v16, v10, v10,0 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vpack.vv v20, v16, v16,0 \n\t" + "vsetvli t0, x0, e64, m1 \n\t" + "vpack.vv v18, v20, v20, 0 \n\t" + "vor.vv v20, v18, v18 \n\t" + "vor.vv v21, v18, v18 \n\t" + + // i4 * i4 vmadot + "vsetvli t0, x0, e16, m1 \n\t" + "vmadotsu.hp v18, v3, v4, v1, 0, i4 \n\t" // high 4 + "vmadotsu.hp v19, v3, v5, v1, 0, i4 \n\t" + "vmadotsu.hp v20, v3, v6, v1, 0, i4 \n\t" + "vmadotsu.hp v21, v3, v7, v1, 0, i4 \n\t" + "vmadotu.hp v18, v2, v4, v0, 0, i4 \n\t" // low 4 + "vmadotu.hp v19, v2, v5, v0, 0, i4 \n\t" + "vmadotu.hp v20, v2, v6, v0, 0, i4 \n\t" + "vmadotu.hp v21, v2, v7, v0, 0, i4 \n\t" + + "vpack.vv v8, v18, v19, 1 \n\t" + "vpack.vv v12, v20, v21, 1 \n\t" + "vpack.vv v20, v8, v12, 2 \n\t" + + "vfwmul.vv v16, v20, v14 \n\t" + "vfwmul.vv v18, v21, v14 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + + "addi t4, t4, -1 \n\t" + "vfmacc.vf v28, fa0, v16 \n\t" + "vfmacc.vf v29, fa1, v17 \n\t" + "vfmacc.vf v30, fa2, v18 \n\t" + "vfmacc.vf v31, fa3, v19 \n\t" + + "bgtz t4, BLK_LOOP%= \n\t" + + // save + "vsetvli t0, x0, e32, m1 \n\t" + "add t2, %[LDC], %[DST] \n\t" + "vse32.v v28, (%[DST]) \n\t" + "add t3, %[LDC], t2 \n\t" + "vse32.v v29, (t2) \n\t" + "add t2, %[LDC], t3 \n\t" + "vse32.v v30, (t3) \n\t" + "vse32.v v31, (t2) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data) + : [DST] "r"(dst_c), [LDC] "r"(ldc * 4), [BK] "r"(k_blks) + : "t0", "t1", "t2", "t3", "t4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", + "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "fa2", "fa3"); +#endif + } + } else { + for (size_t ni = 0; ni < count_n; ni += 32) { + uint8_t * b_data = (uint8_t *) quant_b_data + ni * b_data_stride; + int8_t * a_data = (int8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + + asm volatile( + "li t1, 8 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v29, v29, v29 \n\t" + "vxor.vv v30, v30, v30 \n\t" + "vxor.vv v31, v31, v31 \n\t" + "mv t4, %[BK] \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // load scale A + "flw fa0, (%[A]) \n\t" + "flw fa1, 4(%[A]) \n\t" + "flw fa2, 8(%[A]) \n\t" + "flw fa3, 12(%[A]) \n\t" + "addi %[A], %[A], 16 \n\t" + + // load scale B + "vsetvli t0, x0, e16, mf2\n\t" + "vle16.v v12, (%[B]) \n\t" + "addi %[B], %[B], 64 \n\t" + "vfwcvt.f.f.v v14, v12 \n\t" + + // load zp + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v8, (%[B]) \n\t" + "addi %[B], %[B], 32 \n\t" + "vwaddu.vx v10, v8, x0 \n\t" + + // load a sum + "lh s1, (%[A]) \n\t" + "lh s2, 2(%[A]) \n\t" + "lh s3, 4(%[A]) \n\t" + "lh s4, 6(%[A]) \n\t" + "addi %[A], %[A], 8 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vl1r.v v0, (%[A]) \n\t" + "addi %[A], %[A], 128 \n\t" // 4*32@i8 + "vl4r.v v4, (%[B]) \n\t" // 32*32@i4 + "addi %[B], %[B], 512 \n\t" + "vsrl.vi v1, v0, 4 \n\t" + "vnpack4.vv v12, v0, v1, 3 \n\t" // A low u4 + "vupack.vv v2, v12, v12, 2 \n\t" + + // init the accumu to asum * zp + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + // i4 * i4 vmadot + "vsetvli t0, x0, e32, m1 \n\t" + "vmadotsu v16, v3, v4, i4 \n\t" // high 4 + "vmadotsu v18, v3, v5, i4 \n\t" + "vmadotsu v20, v3, v6, i4 \n\t" + "vmadotsu v22, v3, v7, i4 \n\t" + "vsll.vi v16, v16, 4 \n\t" + "vsll.vi v18, v18, 4 \n\t" + "vsll.vi v20, v20, 4 \n\t" + "vsll.vi v22, v22, 4 \n\t" + "vmadotu v16, v2, v4, i4 \n\t" // low 4 + "vmadotu v18, v2, v5, i4 \n\t" + "vmadotu v20, v2, v6, i4 \n\t" + "vmadotu v22, v2, v7, i4 \n\t" + + "vpack.vv v0, v16, v18, 2 \n\t" + "vpack.vv v2, v20, v22, 2 \n\t" + "vpack.vv v16, v0, v2, 3 \n\t" + "vpack.vv v18, v1, v3, 3 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vwmul.vx v0, v10, s1 \n\t" + "vwmul.vx v2, v10, s2 \n\t" + "vwmul.vx v4, v10, s3 \n\t" + "vwmul.vx v6, v10, s4 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vadd.vv v16, v16, v0 \n\t" + "vadd.vv v17, v17, v2 \n\t" + "vadd.vv v18, v18, v4 \n\t" + "vadd.vv v19, v19, v6 \n\t" + + "vfcvt.f.x.v v16, v16 \n\t" + "vfcvt.f.x.v v17, v17 \n\t" + "vfcvt.f.x.v v18, v18 \n\t" + "vfcvt.f.x.v v19, v19 \n\t" + + // mul scale + "vfmul.vv v16, v16, v14 \n\t" + "vfmul.vv v17, v17, v14 \n\t" + "vfmul.vv v18, v18, v14 \n\t" + "vfmul.vv v19, v19, v14 \n\t" + + "addi t4, t4, -1 \n\t" + "vfmacc.vf v28, fa0, v16 \n\t" + "vfmacc.vf v29, fa1, v17 \n\t" + "vfmacc.vf v30, fa2, v18 \n\t" + "vfmacc.vf v31, fa3, v19 \n\t" + + "bgtz t4, BLK_LOOP%= \n\t" + + // save + "vsetvli t0, x0, e32, m1 \n\t" + "add t2, %[LDC], %[DST]\n\t" + "vse32.v v28, (%[DST]) \n\t" + "add t3, %[LDC], t2 \n\t" + "vse32.v v29, (t2) \n\t" + "add t2, %[LDC], t3 \n\t" + "vse32.v v30, (t3) \n\t" + "vse32.v v31, (t2) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data) + : [DST] "r"(dst_c), [LDC] "r"(ldc * 4), [BK] "r"(k_blks) + : "t0", "t1", "t2", "t3", "t4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", + "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", + "v25", "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "fa2", "fa3", "s1", "s2", "s3", "s4"); + } + } +} + +void gemm_kernel_i8i4_hp_m4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t NB_COLS = 32; + constexpr size_t K_SUBBLKS_PER_SUPERBLK = 8; + constexpr size_t K_SUBBLK_LEN = 32; + + struct block_q4_0x32_layout { + _Float16 d[NB_COLS]; + uint8_t qs[16 * NB_COLS]; + }; + + GGML_ASSERT(blk_len == 256); + GGML_ASSERT(count_m >= 4); + + // Contract: + // - computes a 4-row x 32-col tile per inner invocation + // - A is q8 HP packed in m4 layout, one logical K256 block at a time + // - B is q4 HP packed in N32 tiles, optionally with a separate zp area + // - tail-N is currently not handled here; the caller must provide full N32 tiles + + const size_t b_superblk_stride = sizeof(block_q4_0x32_layout) * K_SUBBLKS_PER_SUPERBLK + + (quant_b_zp ? NB_COLS * K_SUBBLKS_PER_SUPERBLK * sizeof(uint8_t) : 0); + const size_t b_tile_stride = k_blks * b_superblk_stride; + const size_t a_nrow_block_stride = q8_hp_blk_size(blk_len, true, true) * 4; + const size_t a_subblk_stride = q8_hp_blk_size(K_SUBBLK_LEN, false, false) * 4; + + if (quant_b_zp != nullptr) { + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + const size_t nb_real = std::min(NB_COLS, count_n - ni); + if (nb_real != NB_COLS) { + break; + } + + uint8_t * b_tile_base = (uint8_t *) quant_b_data + (ni / NB_COLS) * b_tile_stride; + uint8_t * a_block = (uint8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + + // Data layout summary for the with-zp path. + // + // A: M4 x K256 q8 HP block + // - split into 8 x K32 subblocks + // - each K32 subblock is 136B: + // 8B = 4 x fp16 row scales + // 128B = 4 x int8[32] row payloads + // - trailer after 8 subblocks is 72B: + // 4 rows x fp16[8] a_sum values, indexed as [row][ksi] + // 4 rows x fp16 scale_avg tail + // + // B: N32 x K256 q4 HP block with explicit zp area + // - each K32 subblock is 576B: + // 64B = fp16 scale[32] + // 512B = packed q4 payload for 32 columns x 32 k-elements + // - zp is stored separately, not interleaved with the 576B payload block + // - one K256 superblock is laid out as: + // 8 x (scale + qs) blocks = 4608B + // 8 x zp[32] = 256B + // + // C: 4 rows x 32 fp32 outputs + // + // ASM pointer convention: + // - t6: current A K32 subblock base + // - t2: current A a_sum base for this ksi + // row1/row2/row3 are at +16/+32/+48 bytes + // - s5: current B (scale + qs) K32 subblock base + // - s6: current B zp[32] base for this ksi + // + // Loop progression: + // - per ksi: A += 136, a_sum += 2, B_data += 576, B_zp += 32 + // - per ki : skip the 72B A trailer and advance B to the next 4864B superblock + + const _Float16 hp_scale_16 = (_Float16) 16.0f; + const _Float16 hp_scale_1 = (_Float16) 1.0f; + const _Float16 hp_scale_0125 = (_Float16) 0.125f; + + // VPR grouping used below: + // - v4-v7 : B q4 payload for N32 split as 4 x N8 groups + // - v8/v10 : zp u8 / widened fp16 + // - v12 : B fp16 scale[32] + // - v14-v15 : packed (Bscale * Ascale) for rows [0,1] / [2,3] + // - v16-v19 : temporary per-row scaled B scales + // - v28-v31 : final fp32 accumulators for rows 0..3 + + asm volatile( + "mv t5, %[BK] \n\t" + "mv t6, %[A] \n\t" + "mv s5, %[B] \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v29, v29, v29 \n\t" + "vxor.vv v30, v30, v30 \n\t" + "vxor.vv v31, v31, v31 \n\t" + "li t4, 8 \n\t" + "li t1, 4608 \n\t" + "addi t2, t6, 1088 \n\t" // 8 * 136B A K32 subblocks, a_sum trailer starts here + "add s6, s5, t1 \n\t" // 8 * 576B B(scale+qs), zp area starts here + + ".align 4 \n\t" + "_BLK_LPST%=: \n\t" + "flh fa1, 64(t2) \n\t" // a_scale_avg_row[0] + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v18, v30, v30 \n\t" + "vxor.vv v19, v31, v31 \n\t" + "vxor.vv v20, v30, v30 \n\t" + "vxor.vv v21, v31, v31 \n\t" + "_KsubBLK_LPST%=: \n\t" + // load first subblock scales for 4 rows + "flh fa0, 0(t6) \n\t" // ascale_fp16 + + // load B fp16 scales[32] + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v12, (s5) \n\t" + + // load Bzp[32] for the current ksi from the dedicated zp area + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v8, (s6) \n\t" + + "fmul.h fa2, fa0, %[HP16] \n\t" + "vfwcvt.f.xu.v v10, v8 \n\t" // uint8 -> fp16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vfmul.vf v16, v12, fa0 \n\t" // row0: Bscale * Ascale + "vfmul.vf v17, v12, fa2 \n\t" + + // load a_sum[row][ksi] from the trailer; t2 points to row0[ksi] + "flh ft1, 0(t2) \n\t" + "flh ft2, 16(t2) \n\t" + "flh ft3, 32(t2) \n\t" + "flh ft4, 48(t2) \n\t" + + "fmul.h ft1, ft1, %[HP0125] \n\t" + "fmul.h ft2, ft2, %[HP0125] \n\t" + "fmul.h ft3, ft3, %[HP0125] \n\t" + "fmul.h ft4, ft4, %[HP0125] \n\t" + + // load A payload from current K32 subblock and B q4 payload from current 576B block + "addi t3, t6, 8 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vl1r.v v0, (t3) \n\t" //A + "addi t3, s5, 64 \n\t" + "vl4r.v v4, (t3) \n\t" //B + + "vsetvli t0, x0, e8, m1 \n\t" + "vsrl.vi v1, v0, 4 \n\t" + "vnpack4.vv v12, v0, v1, 3 \n\t" + "vpack.vv v0, v17, v16, 3 \n\t" + "vupack.vv v2, v12, v12, 2 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" // mf2 -> mf2 + "vfmul.vv v10, v10, v16 \n\t" // zp * ascale * bscale; fp16*fp16 + + "vsetvli t0, x0, e16, mf2 \n\t" // mf2 -> m1 + "vfmul.vf v12, v10, ft1 \n\t" // zp(1:n)* abscale * asum_m0; fp16*fp16 + "vfmul.vf v13, v10, ft2 \n\t" // zp(1:n)* abscale * asum_m1; fp16*fp16 + "vfmul.vf v24, v10, ft3 \n\t" // zp(1:n)* abscale * asum_m2; fp16*fp16 + "vfmul.vf v25, v10, ft4 \n\t" // zp(1:n)* abscale * asum_m3; fp16*fp16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwmacc.vf v28, fa1, v12 \n\t" // row0/1 accum += dot * packed scale + "vfwmacc.vf v29, fa1, v13 \n\t" + "vfwmacc.vf v30, fa1, v24 \n\t" + "vfwmacc.vf v31, fa1, v25 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vmadotsu.hp v18, v3, v4, v0, 0, i4 \n\t" //lo4;n0n7 + "vmadotsu.hp v19, v3, v5, v0, 1, i4 \n\t" //lo4;n8n15 + "vmadotsu.hp v20, v3, v6, v0, 2, i4 \n\t" //lo4;n16n23 + "vmadotsu.hp v21, v3, v7, v0, 3, i4 \n\t" //lo4;n24n31 + "vmadotu.hp v18, v2, v4, v0, 4, i4 \n\t" //hi4;n0n7 + "vmadotu.hp v19, v2, v5, v0, 5, i4 \n\t" //hi4;n8n15 + "vmadotu.hp v20, v2, v6, v0, 6, i4 \n\t" //hi4;n16n23 + "vmadotu.hp v21, v2, v7, v0, 7, i4 \n\t" //hi4;n24n31 + + "addi t4, t4, -1 \n\t" + "addi t6, t6, 8+128 \n\t" // next A K32 subblock + "addi t2, t2, 2 \n\t" // next ksi entry in each a_sum row + "addi s5, s5, 64+512 \n\t" // next B (scale + qs) K32 block + "addi s6, s6, 32 \n\t" // next zp[32] + "bgtz t4, _KsubBLK_LPST%= \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v8, v18, v19, 1 \n\t" // 128(16*8)->256(16*16) + "vpack.vv v12, v20, v21, 1 \n\t" + "vpack.vv v26, v8, v12, 2 \n\t" // 256(16*16)->512(16*32) + + "vsetvli t0, x0, e16, m1 \n\t" + "vfwmacc.vf v28, fa1, v26 \n\t" // row0/1 accum += dot * packed scale + "vfwmacc.vf v30, fa1, v27 \n\t" + + "li t4, 8 \n\t" + "addi t5, t5, -1 \n\t" + "addi t6, t6, 72 \n\t" // skip A trailer after 8 subblocks and scale_avg tail + "mv s5, s6 \n\t" // s6 already points to next B superblock base + "addi t2, t6, 1088 \n\t" // 8 * 136B A K32 subblocks, a_sum trailer starts here + "add s6, s5, t1 \n\t" // 8 * 576B B(scale+qs), zp area starts here + "bgtz t5, _BLK_LPST%= \n\t" + + "_BLK_LPND%=: \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "add t2, %[LDC], %[DST] \n\t" + "vse32.v v28, (%[DST]) \n\t" + "add t3, %[LDC], t2 \n\t" + "vse32.v v29, (t2) \n\t" + "add t2, %[LDC], t3 \n\t" + "vse32.v v30, (t3) \n\t" + "vse32.v v31, (t2) \n\t" + : [A] "+r"(a_block), [B] "+r"(b_tile_base) + : [DST] "r"(dst_c), [LDC] "r"(ldc * 4), [BK] "r"(k_blks), [HP16] "f"(hp_scale_16), + [HP1] "f"(hp_scale_1), [HP0125] "f"(hp_scale_0125) + : "t0", "t1", "t2", "t3", "t4", "t5", "t6", "s5", "s6", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v10", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v24", + "v25", "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "fa2", "ft1", "ft2", "ft3", "ft4", + "memory"); + } + return; + } else { + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + const size_t nb_real = std::min(NB_COLS, count_n - ni); + if (nb_real != NB_COLS) { + break; + } + + uint8_t * b_tile_base = (uint8_t *) quant_b_data + (ni / NB_COLS) * b_tile_stride; + uint8_t * a_block = (uint8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + + // Data layout summary for the no-zp path. + // + // A layout is identical to the with-zp branch. + // + // B: N32 x K256 q4 HP block without explicit zp storage + // - each K32 subblock is still 576B: + // 64B = fp16 scale[32] + // 512B = packed q4 payload + // - zp is implicit and treated as a constant value 8 in the kernel + // - one K256 superblock therefore contains only: + // 8 x (scale + qs) blocks = 4608B + // + // C: 4 rows x 32 fp32 outputs + // + // ASM pointer convention: + // - t6: current A K32 subblock base + // - t2: current A a_sum base for this ksi + // - s5: current B (scale + qs) K32 subblock base + // + // Loop progression: + // - per ksi: A += 136, a_sum += 2, B_data += 576 + // - per ki : skip the 72B A trailer and advance B to the next 4608B superblock + + const _Float16 hp_scale_16 = (_Float16) 16.0f; + const _Float16 hp_scale_1 = (_Float16) 1.0f; + + // VPR grouping used below matches the with-zp path: + // - v4-v7 : B q4 payload for N32 split as 4 x N8 groups + // - v8/v10 : implicit zp lane / widened fp16 + // - v12 : B fp16 scale[32] + // - v14-v15 : packed (Bscale * Ascale) for rows [0,1] / [2,3] + // - v16-v19 : temporary per-row scaled B scales + // - v28-v31 : final fp32 accumulators for rows 0..3 + + asm volatile( + "mv t5, %[BK] \n\t" + "mv t6, %[A] \n\t" + "mv s5, %[B] \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v29, v29, v29 \n\t" + "vxor.vv v30, v30, v30 \n\t" + "vxor.vv v31, v31, v31 \n\t" + "li t4, 8 \n\t" + "addi t2, t6, 1088 \n\t" // 8 * 136B A K32 subblocks, a_sum trailer starts here + + ".align 4 \n\t" + "_BLK_LPST%=: \n\t" + "flh fa1, 64(t2) \n\t" // a_scale_avg_row[0] + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v18, v30, v30 \n\t" + "vxor.vv v19, v31, v31 \n\t" + "vxor.vv v20, v30, v30 \n\t" + "vxor.vv v21, v31, v31 \n\t" + "_KsubBLK_LPST%=: \n\t" + // load first subblock scales for 4 rows + "flh fa0, 0(t6) \n\t" // ascale_fp16 + + // load B fp16 scales[32] + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v12, (s5) \n\t" + + "fmul.h fa2, fa0, %[HP16] \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "vfmul.vf v16, v12, fa0 \n\t" // row0: Bscale * Ascale + "vfmul.vf v17, v12, fa2 \n\t" + + // load a_sum[row][ksi] from the trailer; t2 points to row0[ksi] + "flh ft1, 0(t2) \n\t" + "flh ft2, 16(t2) \n\t" + "flh ft3, 32(t2) \n\t" + "flh ft4, 48(t2) \n\t" + + // load A payload from current K32 subblock and B q4 payload from current 576B block + "addi t3, t6, 8 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vl1r.v v0, (t3) \n\t" //A + "addi t3, s5, 64 \n\t" + "vl4r.v v4, (t3) \n\t" //B + + "vsetvli t0, x0, e8, m1 \n\t" + "vsrl.vi v1, v0, 4 \n\t" + "vnpack4.vv v12, v0, v1, 3 \n\t" + "vpack.vv v0, v17, v16, 3 \n\t" + "vupack.vv v2, v12, v12, 2 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" // mf2 -> m1 + "vfmul.vf v12, v16, ft1 \n\t" // zp(1:n)* abscale * asum_m0; fp16*fp16 + "vfmul.vf v13, v16, ft2 \n\t" // zp(1:n)* abscale * asum_m1; fp16*fp16 + "vfmul.vf v24, v16, ft3 \n\t" // zp(1:n)* abscale * asum_m2; fp16*fp16 + "vfmul.vf v25, v16, ft4 \n\t" // zp(1:n)* abscale * asum_m3; fp16*fp16 + + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwmacc.vf v28, fa1, v12 \n\t" + "vfwmacc.vf v29, fa1, v13 \n\t" + "vfwmacc.vf v30, fa1, v24 \n\t" + "vfwmacc.vf v31, fa1, v25 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vmadotsu.hp v18, v3, v4, v0, 0, i4 \n\t" //lo4;n0n7 + "vmadotsu.hp v19, v3, v5, v0, 1, i4 \n\t" //lo4;n8n15 + "vmadotsu.hp v20, v3, v6, v0, 2, i4 \n\t" //lo4;n16n23 + "vmadotsu.hp v21, v3, v7, v0, 3, i4 \n\t" //lo4;n24n31 + "vmadotu.hp v18, v2, v4, v0, 4, i4 \n\t" //hi4;n0n7 + "vmadotu.hp v19, v2, v5, v0, 5, i4 \n\t" //hi4;n8n15 + "vmadotu.hp v20, v2, v6, v0, 6, i4 \n\t" //hi4;n16n23 + "vmadotu.hp v21, v2, v7, v0, 7, i4 \n\t" //hi4;n24n31 + + "addi t4, t4, -1 \n\t" + + "addi t6, t6, 8+128 \n\t" // next A K32 subblock + "addi t2, t2, 2 \n\t" // next ksi entry in each a_sum row + "addi s5, s5, 64+512 \n\t" // next B (scale + qs) K32 block + "bgtz t4, _KsubBLK_LPST%= \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" //N32in1register + "vpack.vv v8, v18, v19, 1 \n\t" // 128(16*8)->256(16*16) + "vpack.vv v12, v20, v21, 1 \n\t" + "vpack.vv v26, v8, v12, 2 \n\t" // 256(16*16)->512(16*32) + + "vsetvli t0, x0, e16, m1 \n\t" + "vfwmacc.vf v28, fa1, v26 \n\t" // row0/1 accum += dot * packed scale + "vfwmacc.vf v30, fa1, v27 \n\t" + + "li t4, 8 \n\t" + "addi t5, t5, -1 \n\t" + "addi t6, t6, 72 \n\t" // skip A trailer after 8 subblocks and scale_avg tail + // s5 already points to next B superblock base + "addi t2, t6, 1088 \n\t" // 8 * 136B A K32 subblocks, a_sum trailer starts here + "bgtz t5, _BLK_LPST%= \n\t" + + "_BLK_LPND%=: \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "add t2, %[LDC], %[DST] \n\t" + "vse32.v v28, (%[DST]) \n\t" + "add t3, %[LDC], t2 \n\t" + "vse32.v v29, (t2) \n\t" + "add t2, %[LDC], t3 \n\t" + "vse32.v v30, (t3) \n\t" + "vse32.v v31, (t2) \n\t" + : [A] "+r"(a_block), [B] "+r"(b_tile_base) + : [DST] "r"(dst_c), [LDC] "r"(ldc * 4), [BK] "r"(k_blks), [HP16] "f"(hp_scale_16), [HP1] "f"(hp_scale_1) + : "t0", "t2", "t3", "t4", "t5", "t6", "s5", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v10", + "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v24", "v25", "v26", + "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "fa2", "ft1", "ft2", "ft3", "ft4", "memory"); + } + return; + } +} + +void gemm_kernel_i8mxfp4_m1(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t NB_COLS = 32; + constexpr size_t K_TILE = 32; + using blk_type = nrow_block_mxfp4; + + GGML_ASSERT(blk_len == K_TILE); + GGML_ASSERT(count_m == 1); + GGML_UNUSED(quant_b_zp); + + const size_t a_blk_stride = q8_blk_size(blk_len, true); + const size_t b_blk_stride = sizeof(blk_type); + const size_t b_tile_stride = k_blks * b_blk_stride; + + if (quant_b_zp == NULL) { + for (size_t n = 0; n < count_n; n += 32) { + size_t nblks = (count_n - n) > 32 ? 32 : count_n - n; + // MXFP4 no-zp: per column per k-block stride = scale_e8m0(1B) + qs(16B) + qh(4B) = 21B + uint8_t * QuantBDataPtr = (uint8_t *) quant_b_data + // + n * k_blks * (blk_len / 8) + // qh sign/high-bit mask: n×k_blks×4 + n * k_blks * blk_len / 2 + // qs packed 4-bit magnitudes: n×k_blks×16 + n * k_blks * sizeof(uint8_t); // scale: n×k_blks×1 + float * CPtr = c_ptr + n; + size_t cnt = k_blks; + + // A format (q8 block with per-block scale and stored sum field): + // || scl(fp32,4B) | asum(int16,2B) | data(int8,32B) || × k_blks + // + // Register map: + // t3 = k_blks loop counter t4 = nblks (tail) + // f0 = A scale (fp32) + // s2 = pA (scale/asum) s3 = pA data + // s4 = pB scales (u8×32) + // s5 = pB qh (sign/high-bit mask, 128B) + // s6 = pB qs (packed 4-bit magnitudes, 512B) + // s7 = pC + // v3 = fp32 accumulator (N32) + // v2 = B scales u8 (loaded as bytes; later widened) + // v0 = qh mask bytes (also used as v0.t mask after load) + // v1 = A int8 (K32) + // v8..v15 / v16..v23 = qs unpack/pack temporaries (build signed vmadot lanes) + // v24/v26/v28/v30 = int32 dot accumulators & packing temps + + __asm__ volatile( + "mv t3, %[BCK] \n\t" // t3 = k_blks + "mv t4, %[NBLKS] \n\t" // t4 = nblks (tail guard) + + // ---- pre-loop: init fp16 constants in e16 m1 context ---- + "vsetvli t0, x0, e16, m1 \n\t" + "vmv.v.i v0, 1 \n\t" // v0 = int16(1) + "vfcvt.f.x.v v0, v0 \n\t" // v0 = 1.0_fp16 + "vxor.vv v3, v16, v16 \n\t" + + // ---- pointer setup ---- + "mv s2, %[pA] \n\t" // s2 = pA (scale, fp32) + "addi s3, %[pA], 4+2 \n\t" // s3 = pA data (skip scale+asum) + "mv s4, %[pB] \n\t" // s4 = pBSCL + "addi s5, %[pB], 32 \n\t" // s5 = pBh (pB + 32B scale) + "addi s6, %[pB], 32+128 \n\t" // s6 = pBs (pB + 32 + 128 = pB+192) + "mv s7, %[pC] \n\t" // s7 = pC + + // ===================================================================== + // K-block loop: each iteration processes one N32×K32 block + // Stride per k-block = 672B = 32(scl) + 512(Bs) + 128(Bh) + // ===================================================================== + ".align 4 \n\t" + "BLK_LPST%=: \n\t" + + // ---- load qs (512B = 4 VRF) from s6, advance s6 by 672 ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v8, (s6) \n\t" // v8..v11 = qs N32K32 packed 4-bit magnitudes + "addi s6, s6, 128*4+128+32 \n\t" // s6 += 672 (512+128+32) + + // ---- load B scale (32B = 32×u8) from s4, advance s4 by 672 ---- + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v2, (s4) \n\t" // v2 = scale_u8 × 32 + "addi s4, s4, 32+128*4+128 \n\t" // s4 += 672 (32+512+128) + + // ---- load qh (128B = 1 VRF) from s5, advance s5 by 672 ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v0, (s5) \n\t" // v0 = qh N32K32 sign/high-bit packed + "addi s5, s5, 128+32+128*4 \n\t" // s5 += 672 (128+32+512) + + // ---- load A data (32B = K32 int8) from s3 ---- + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v1, (s3) \n\t" // v1 = A M1K32 int8 + "addi s3, s3, 32+6 \n\t" // s3 += 38 (data + scl + asum) + + // ---- load A scale (fp32) and asum (int16) from s2 ---- + "flw f0, (s2) \n\t" // f0 = A scale (fp32) + "addi s2, s2, 6+32 \n\t" // s2 += 38 + + // ---- Decode packed MXFP4 payload into a vmadot-friendly signed-lane layout ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vand.vi v12, v8, 0xF \n\t" //8bit(lo4) //[8*32] + "vand.vi v13, v9, 0xF \n\t" + "vand.vi v14, v10, 0xF \n\t" + "vand.vi v15, v11, 0xF \n\t" + "vsrl.vi v8, v8, 4 \n\t" //8bit(hi4) + "vsrl.vi v9, v9, 4 \n\t" + "vsrl.vi v10, v10, 4 \n\t" + "vsrl.vi v11, v11, 4 \n\t" + + // [4*32]*2 + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v16, v12, v8, 0 \n\t" + "vpack.vv v18, v13, v9, 0 \n\t" + "vpack.vv v20, v14, v10, 0 \n\t" + "vpack.vv v22, v15, v11, 0 \n\t" + + "vsetvli t0, x0, e8, m8 \n\t" + "vrsub.vi v16, v16, 0, v0.t \n\t" + + // [4*32]*2 -> [8*16] + "vsetvli t0, x0, e8, m1 \n\t" + "vupack.vv v8, v16, v17, 1 \n\t" + "vupack.vv v10, v18, v19, 1 \n\t" + "vupack.vv v12, v20, v21, 1 \n\t" + "vupack.vv v14, v22, v23, 1 \n\t" + + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v16, v1, 2 \n\t" + + // init the accumu to 0 + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v16, v16 \n\t" + "vxor.vv v26, v16, v16 \n\t" + "vxor.vv v28, v16, v16 \n\t" + "vxor.vv v30, v16, v16 \n\t" + + // ---- int8 dot products over the decoded MXFP4 lane groups ---- + "vmadot v24, v1, v8, i8 \n\t" // N0..7 + "vmadot v26, v1, v10, i8 \n\t" // N8..15 + "vmadot v28, v1, v12, i8 \n\t" // N16..23 + "vmadot v30, v1, v14, i8 \n\t" // N24..31 + "vmadot v24, v16, v9, i8 \n\t" // N0..7 + "vmadot v26, v16, v11, i8 \n\t" // N8..15 + "vmadot v28, v16, v13, i8 \n\t" // N16..23 + "vmadot v30, v16, v15, i8 \n\t" // N24..31 + + "vsetvli t0, x0, e32, m1 \n\t" + "vpack.vv v16, v24, v26, 2 \n\t" // v16 = N0..15 + "vpack.vv v18, v28, v30, 2 \n\t" // v18 = N16..31 + "vpack.vv v24, v16, v18, 3 \n\t" // v24 = N0..31 + + "lui t1, 0x00200 \n\t" + "vmv.v.x v30, t1 \n\t" + // b_scale e8m0 -> fp32 + "vsetvli t0, x0, e8, mf4 \n\t" + "vwaddu.vx v28, v2, x0 \n\t" + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v2, v28, x0 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vmsle.vi v0, v2, 1 \n\t" + "vadd.vi v28, v2, -1 \n\t" + "vsll.vi v28, v28, 23 \n\t" + "vsll.vv v28, v30, v2, v0.t \n\t" + + // a_scale * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v26, v24 \n\t" + "vfmul.vf v30, v28, f0 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + // static_cast(qsum) * a_scale * b_scale; + "vfmacc.vv v3, v30, v26 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, BLK_LPST%= \n\t" + "BLK_LPND%=: \n\t" + "vsetvli t0, %[NBLKS], e32, m1 \n\t" + "vse32.v v3, (%[pC]) \n\t" + "FUNC_END%=: \n\t" + + : + : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) + : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6", "s7", "v0", "v1", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v18", "v19", + "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + } +} + +void gemm_kernel_i8mxfp4_m4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t NB_COLS = 32; + constexpr size_t K_TILE = 32; + using blk_type = nrow_block_mxfp4; + + GGML_ASSERT(blk_len == K_TILE); + GGML_ASSERT(count_m == 4); + GGML_UNUSED(quant_b_zp); + + const size_t a_blk_stride = q8_blk_size(blk_len, true); + const size_t b_blk_stride = sizeof(blk_type); + const size_t b_tile_stride = k_blks * b_blk_stride; + + if (quant_b_zp == NULL) { + // MXFP4 block layout per K32/N32 tile: + // [scale_e8m0 x 32][qh sign/high-bit mask x 128B][qs packed 4-bit magnitudes x 512B] + // There is no explicit zp stream; qh is combined with qs to reconstruct signed MXFP4 values. + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * b_tile_stride; + uint8_t * a_data = (uint8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + size_t cnt = k_blks; + + asm volatile( + // v4-v7 are the fp32 accumulators for rows 0..3 of the current N32 tile. + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v4, v4, v4 \n\t" + "vxor.vv v5, v5, v5 \n\t" + "vxor.vv v6, v6, v6 \n\t" + "vxor.vv v7, v7, v7 \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // Load the 4 A-row scales for this K32 block and build row data pointers. + "flw fa0, 0(%[A]) \n\t" + "flw fa1, 4(%[A]) \n\t" + "flw fa2, 8(%[A]) \n\t" + "flw fa3, 12(%[A]) \n\t" + "addi t3, %[A], 24 \n\t" + "addi t4, t3, 32 \n\t" + "addi t5, t3, 64 \n\t" + "addi t6, t3, 96 \n\t" + "addi %[A], %[A], 152 \n\t" + + // B-side pointers: + // t1 -> qh bitmask stream, t2 -> qs low-nibble stream. + "addi t1, %[B], 32 \n\t" + "addi t2, %[B], 160 \n\t" + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v2, (%[B]) \n\t" + "addi %[B], %[B], 672 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v0, (t1) \n\t" + "vl4r.v v8, (t2) \n\t" + + // Decode the packed MXFP4 payload once for the whole tile and expand it + // into a vmadot-friendly layout. + "vand.vi v12, v8, 0xF \n\t" + "vand.vi v13, v9, 0xF \n\t" + "vand.vi v14, v10, 0xF \n\t" + "vand.vi v15, v11, 0xF \n\t" + "vsrl.vi v8, v8, 4 \n\t" + "vsrl.vi v9, v9, 4 \n\t" + "vsrl.vi v10, v10, 4 \n\t" + "vsrl.vi v11, v11, 4 \n\t" + + "vpack.vv v16, v12, v8, 0 \n\t" + "vpack.vv v18, v13, v9, 0 \n\t" + "vpack.vv v20, v14, v10, 0 \n\t" + "vpack.vv v22, v15, v11, 0 \n\t" + + "vsetvli t0, x0, e8, m8 \n\t" + "vrsub.vi v16, v16, 0, v0.t \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vupack.vv v8, v16, v17, 1 \n\t" + "vupack.vv v10, v18, v19, 1 \n\t" + "vupack.vv v12, v20, v21, 1 \n\t" + "vupack.vv v14, v22, v23, 1 \n\t" + + "lui t1, 0x00200 \n\t" + "vmv.v.x v30, t1 \n\t" + // b_scale e8m0 -> fp32 + "vsetvli t0, x0, e8, mf4 \n\t" + "vwaddu.vx v28, v2, x0 \n\t" + "vsetvli t0, x0, e16, mf2 \n\t" + "vwadd.vx v26, v28, x0 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vmsle.vi v0, v26, 1 \n\t" + "vadd.vi v24, v26, -1 \n\t" + "vsll.vi v18, v24, 23 \n\t" + "vsll.vv v18, v30, v26, v0.t \n\t" + + // Row 0: dot(A0, decoded MXFP4 lane groups), accumulate in int32 and + // then apply A/B scaling. + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v1, (t3) \n\t" + "vsetvli t0, x0, e64, m1 \n\t" + "vupack.vv v16, v1, v2, 1 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v24, v24 \n\t" + "vxor.vv v26, v26, v26 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v30, v30, v30 \n\t" + "vmadot v24, v16, v8, i8 \n\t" + "vmadot v26, v16, v10, i8 \n\t" + "vmadot v28, v16, v12, i8 \n\t" + "vmadot v30, v16, v14, i8 \n\t" + "vmadot v24, v17, v9, i8 \n\t" + "vmadot v26, v17, v11, i8 \n\t" + "vmadot v28, v17, v13, i8 \n\t" + "vmadot v30, v17, v15, i8 \n\t" + "vpack.vv v16, v24, v26, 2 \n\t" + "vpack.vv v20, v28, v30, 2 \n\t" + "vpack.vv v24, v16, v20, 3 \n\t" + "vpack.vv v26, v17, v21, 3 \n\t" + "vfcvt.f.x.v v24, v24 \n\t" + "vfcvt.f.x.v v25, v25 \n\t" + "vfcvt.f.x.v v26, v26 \n\t" + "vfcvt.f.x.v v27, v27 \n\t" + "vfmul.vv v24, v24, v18 \n\t" + "vfmul.vv v25, v25, v18 \n\t" + "vfmul.vv v26, v26, v18 \n\t" + "vfmul.vv v27, v27, v18 \n\t" + "vfmacc.vf v4, fa0, v24 \n\t" + "vfmacc.vf v5, fa1, v25 \n\t" + "vfmacc.vf v6, fa2, v26 \n\t" + "vfmacc.vf v7, fa3, v27 \n\t" + + "addi %[BK], %[BK], -1 \n\t" + "bgtz %[BK], BLK_LOOP%= \n\t" + + // Tail-aware store for the final N tile (`nb_real` may be < 32). + "vsetvli t0, %[NBLKS], e32, m1 \n\t" + "add t1, %[LDC], %[DST] \n\t" + "vse32.v v4, (%[DST]) \n\t" + "vse32.v v5, (t1) \n\t" + "add t2, t1, %[LDC] \n\t" + "vse32.v v6, (t2) \n\t" + "add t3, t2, %[LDC] \n\t" + "vse32.v v7, (t3) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data), [BK] "+r"(cnt) + : [DST] "r"(dst_c), [LDC] "r"(ldc * 4), [NBLKS] "r"(nb_real) + : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "s1", "s2", "s3", "s4", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", + "fa0", "fa1", "fa2", "fa3"); + } + } +} + +void gemm_kernel_i8i5_m1(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + // ========================================================================= + // i8i5: 8-bit activation × 5-bit weight (4-bit low + 1-bit high mask) + // + // B layout per N32K32 k-block (no-zp): + // [0 .. 63 ] : scale_fp16 × 32 (64B) + // [64 .. 191] : Bh i1-high-bit × 32N × 32K (128B = 1 VRF) + // [192.. 703] : Bs i4-low-nibble × 32N × 32K (512B = 4 VRF) + // Total: 704B per k-block stride + // + // B layout per N32K32 k-block (with-zp): + // [0 .. 63 ] : scale_fp16 × 32 (64B) + // [64 .. 95 ] : zp_uint8 × 32 (32B) + // [96 .. 223] : Bh i1-high-bit × 32N × 32K (128B = 1 VRF) + // [224.. 735] : Bs i4-low-nibble × 32N × 32K (512B = 4 VRF) + // Total: 736B per k-block stride + // + // Bh format per N8K32 sub-block (32B): + // K rows × N cols × 1bit packed as bytes (8 cols per byte, K groups of 4B) + // Byte k gives 8 mask bits for columns N7..N0 at k-th K-element. + // + // Computation: + // B5bit_signed = (Bs | (Bh << 4)) - zp + // dot(A, B5) = dot(A, Bs_u4) + 16*dot(A, Bh_u1) - zp*asum + // No-zp: implicit zp = 16 (unsigned [0..31] centered at 16) + // With-zp: explicit zp from data + // + // ========================================================================= + + if (quant_b_zp == NULL) { + for (size_t n = 0; n < count_n; n += 32) { + size_t nblks = (count_n - n) > 32 ? 32 : count_n - n; + // i8i5 no-zp: per column per k-block stride = fp16(2B) + i4(16B) + i1(4B) = 22B + uint8_t * QuantBDataPtr = (uint8_t *) quant_b_data + // + n * k_blks * (blk_len / 8) + // Bh i1 mask: n×k_blks×4 + n * k_blks * blk_len / 2 + // Bs i4 data: n×k_blks×16 + n * k_blks * sizeof(_Float16); // scale: n×k_blks×2 + float * CPtr = c_ptr + n; + size_t cnt = k_blks; + + // A format (same as i8i4): + // || scl(fp32,4B) | asum(int16,2B) | data(int8,32B) || × k_blks + // + // Register map: + // t3 = k_blks loop counter t4 = nblks (tail) + // t2 = A asum (int16) << 4 f0 = A scale (fp32) + // s2 = pA (scale/asum) s3 = pA data + // s4 = pB scales (fp16×32) + // s5 = pB Bh (i1 mask, 128B) + // s6 = pB Bs (i4 packed, 512B) + // s7 = pC + // v3 = fp32 accumulator (N32) + // v2 = B scales fp16 (loaded as bytes; later widened) + // v0 = Bh mask bytes (also used as v0.t mask after load) + // v1 = A int8 (K32) + // v8..v15 / v16..v23 = Bs unpack/pack temporaries (build b5bit bytes) + // v24/v26/v28/v30 = int32 dot accumulators & packing temps + + __asm__ volatile( + "mv t3, %[BCK] \n\t" // t3 = k_blks + "mv t4, %[NBLKS] \n\t" // t4 = nblks (tail guard) + + // ---- pre-loop: init fp16 constants in e16 m1 context ---- + "vsetvli t0, x0, e16, m1 \n\t" + "vmv.v.i v0, 1 \n\t" // v0 = int16(1) + "vfcvt.f.x.v v0, v0 \n\t" // v0 = 1.0_fp16 + "vxor.vv v3, v16, v16 \n\t" + + // ---- pointer setup ---- + "mv s2, %[pA] \n\t" // s2 = pA (scale, fp32) + "addi s3, %[pA], 4+2 \n\t" // s3 = pA data (skip scale+asum) + "mv s4, %[pB] \n\t" // s4 = pBSCL + "addi s5, %[pB], 32*2 \n\t" // s5 = pBh (pB + 64B scale) + "addi s6, %[pB], 32*2+128 \n\t" // s6 = pBs (pB + 64 + 128 = pB+192) + "mv s7, %[pC] \n\t" // s7 = pC + + // ===================================================================== + // K-block loop: each iteration processes one N32×K32 block + // Stride per k-block = 704B = 64(scl) + 512(Bs) + 128(Bh) + // ===================================================================== + ".align 4 \n\t" + "BLK_LPST%=: \n\t" + + // ---- load Bs (512B = 4 VRF) from s6, advance s6 by 704 ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v8, (s6) \n\t" // v8..v11 = Bs N32K32 i4 + "addi s6, s6, 128*4+128+64 \n\t" // s6 += 704 (512+128+64) + + // ---- load B scale (64B = 32×fp16) from s4, advance s4 by 704 ---- + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v2, (s4) \n\t" // v2 = scale_fp16 × 32 + "addi s4, s4, 64+128*4+128 \n\t" // s4 += 704 (64+512+128) + + // ---- load Bh (128B = 1 VRF) from s5, advance s5 by 704 ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v0, (s5) \n\t" // v0 = Bh N32K32 1-bit packed + "addi s5, s5, 128+64+128*4 \n\t" // s5 += 704 (128+64+512) + + // ---- load A data (32B = K32 int8) from s3 ---- + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v1, (s3) \n\t" // v1 = A M1K32 int8 + "addi s3, s3, 32+6 \n\t" // s3 += 38 (data + scl + asum) + + // ---- load A scale (fp32) and asum (int16) from s2 ---- + "flw f0, (s2) \n\t" // f0 = A scale (fp32) + "lh t2, 4(s2) \n\t" // t2 = A asum (int16) + "addi s2, s2, 6+32 \n\t" // s2 += 38 + + //// ---- A nibble unpacking ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vand.vi v12, v8, 0xF \n\t" //8bit(lo4) //[8*32] + "vand.vi v13, v9, 0xF \n\t" + "vand.vi v14, v10, 0xF \n\t" + "vand.vi v15, v11, 0xF \n\t" + "vsrl.vi v8, v8, 4 \n\t" //8bit(hi4) + "vsrl.vi v9, v9, 4 \n\t" + "vsrl.vi v10, v10, 4 \n\t" + "vsrl.vi v11, v11, 4 \n\t" + + "slli t2, t2, 4 \n\t" // a_sum * 16; + // [4*32]*2 + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v16, v12, v8, 0 \n\t" + "vpack.vv v18, v13, v9, 0 \n\t" + "vpack.vv v20, v14, v10, 0 \n\t" + "vpack.vv v22, v15, v11, 0 \n\t" + + "li t1, 16 \n\t" + "vsetvli t0, x0, e8, m8 \n\t" + "vadd.vx v16, v16, t1, v0.t \n\t" + + // [4*32]*2 -> [8*16] + "vsetvli t0, x0, e8, m1 \n\t" + "vupack.vv v8, v16, v17, 1 \n\t" + "vupack.vv v10, v18, v19, 1 \n\t" + "vupack.vv v12, v20, v21, 1 \n\t" + "vupack.vv v14, v22, v23, 1 \n\t" + + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v16, v1, 2 \n\t" + + // init the accumu to asum * zp + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v16, v16 \n\t" + "vxor.vv v26, v16, v16 \n\t" + "vxor.vv v28, v16, v16 \n\t" + "vxor.vv v30, v16, v16 \n\t" + + // ---- i8 main dot products ---- + // vmadot: A × unsigned Bh × 16 → fp16 accumulate + "vmadot v24, v1, v8, i8 \n\t" // N0..7 + "vmadot v26, v1, v10, i8 \n\t" // N8..15 + "vmadot v28, v1, v12, i8 \n\t" // N16..23 + "vmadot v30, v1, v14, i8 \n\t" // N24..31 + //// vmadot: A × unsigned Bh × 1 → fp16 accumulate + "vmadot v24, v16, v9, i8 \n\t" // N0..7 + "vmadot v26, v16, v11, i8 \n\t" // N8..15 + "vmadot v28, v16, v13, i8 \n\t" // N16..23 + "vmadot v30, v16, v15, i8 \n\t" // N24..31 + + "vsetvli t0, x0, e32, m1 \n\t" + "vpack.vv v16, v24, v26, 2 \n\t" // v16 = N0..15 + "vpack.vv v18, v28, v30, 2 \n\t" // v18 = N16..31 + "vpack.vv v24, v16, v18, 3 \n\t" // v24 = N0..31 + + "vadd.vx v24, v24, t2 \n\t" + // b_scale fp16 -> fp32 + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v28, v2 \n\t" + + // a_scale * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v26, v24 \n\t" + "vfmul.vf v30, v28, f0 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + // static_cast(qsum) * a_scale * b_scale; + "vfmacc.vv v3, v30, v26 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, BLK_LPST%= \n\t" + "BLK_LPND%=: \n\t" + "vsetvli t0, %[NBLKS], e32, m1 \n\t" + "vse32.v v3, (%[pC]) \n\t" + "FUNC_END%=: \n\t" + + : + : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) + : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6", "s7", "v0", "v1", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v18", "v19", + "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + } else { + for (size_t n = 0; n < count_n; n += 32) { + size_t nblks = (count_n - n) > 32 ? 32 : count_n - n; + // i8i5 with-zp: per column per k-block stride = fp16(2B)+zp(1B)+i4(16B)+i1(4B)=23B + uint8_t * QuantBDataPtr = (uint8_t *) quant_b_data + // + n * k_blks * blk_len / 2 + // Bs i4: n×k_blks×16 + n * k_blks * (blk_len / 8) + // Bh i1: n×k_blks×4 + n * k_blks * sizeof(uint8_t) + // zp: n×k_blks×1 + n * k_blks * sizeof(_Float16); // scale: n×k_blks×2 + float * CPtr = c_ptr + n; + size_t cnt = k_blks; + + // A format (same as i8i4): + // || scl(fp32,4B) | asum(int16,2B) | data(int8,32B) || × k_blks + // + // Register map: + // t3 = k_blks loop counter t4 = nblks (tail) + // t2 = A asum (int16) << 4 f0 = A scale (fp32) + // s2 = pA (scale/asum) s3 = pA data + // s4 = pB scales (fp16×32); 每个 k-block 先 +64 指向 zp，再 +672 到下一个 block + // s5 = pB Bh (i1 mask, 128B) (offset +96) + // s6 = pB Bs (i4 packed, 512B) (offset +224) + // s7 = pC + // v3 = fp32 accumulator (N32) + // v2 = B scales fp16 (loaded as bytes; later widened) + // v0 = Bh mask bytes (also used as v0.t mask after load) + // v1 = A int8 (K32) / later reused to hold Bzp bytes + // v8..v15 / v16..v23 = Bs unpack/pack temporaries (build b5bit bytes) + // v24/v26/v28/v30 = int32 dot accumulators & packing temps + + __asm__ volatile( + "mv t3, %[BCK] \n\t" // t3 = k_blks + "mv t4, %[NBLKS] \n\t" // t4 = nblks (tail guard) + + // ---- pre-loop: init fp16 constants in e16 m1 context ---- + "vsetvli t0, x0, e16, m1 \n\t" + "vmv.v.i v0, 1 \n\t" // v0 = int16(1) + "vfcvt.f.x.v v0, v0 \n\t" // v0 = 1.0_fp16 + "vxor.vv v3, v16, v16 \n\t" + + // ---- pointer setup ---- + "mv s2, %[pA] \n\t" // s2 = pA (scale, fp32) + "addi s3, %[pA], 4+2 \n\t" // s3 = pA data (skip scale+asum) + "mv s4, %[pB] \n\t" // s4 = pBSCL + "addi s5, %[pB], 32*3 \n\t" // s5 = pBh (pB + 64B scale + 32B zp = pB+96) + "addi s6, %[pB], 32*3+128 \n\t" // s6 = pBs (pB + 96 + 128 = pB+224) + "mv s7, %[pC] \n\t" // s7 = pC + + // ===================================================================== + // K-block loop: each iteration processes one N32×K32 block + // Stride per k-block = 736B = 64(scale) + 32(zp) + 128(Bh) + 512(Bs) + // ===================================================================== + ".align 4 \n\t" + "BLK_LPST%=: \n\t" + + // ---- load Bs (512B = 4 VRF) from s6, advance s6 by 736 ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v8, (s6) \n\t" // v8..v11 = Bs N32K32 i4 + "addi s6, s6, 128*4+128+96 \n\t" // s6 += 736 (512+128+96) + + // ---- load B scale (64B = 32×fp16) from s4; then s4 points to zp[32] ---- + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v2, (s4) \n\t" // v2 = scale_fp16 × 32 + "addi s4, s4, 64 \n\t" // s4 += 64 (now points to zp) + + // ---- load Bh (128B = 1 VRF) from s5, advance s5 by 736 ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v0, (s5) \n\t" // v0 = Bh N32K32 1-bit packed + "addi s5, s5, 128+96+128*4 \n\t" // s5 += 736 (128+96+512) + + // ---- load A data (32B = K32 int8) from s3 ---- + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v1, (s3) \n\t" // v1 = A M1K32 int8 + "addi s3, s3, 32+6 \n\t" // s3 += 38 (data + scl + asum) + + // ---- load A scale (fp32) and asum (int16) from s2 ---- + "flw f0, (s2) \n\t" // f0 = A scale (fp32) + "lh t2, 4(s2) \n\t" // t2 = A asum (int16) + "addi s2, s2, 6+32 \n\t" // s2 += 38 + + //// ---- A nibble unpacking ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vand.vi v12, v8, 0xF \n\t" //8bit(lo4) //[8*32] + "vand.vi v13, v9, 0xF \n\t" + "vand.vi v14, v10, 0xF \n\t" + "vand.vi v15, v11, 0xF \n\t" + "vsrl.vi v8, v8, 4 \n\t" //8bit(hi4) + "vsrl.vi v9, v9, 4 \n\t" + "vsrl.vi v10, v10, 4 \n\t" + "vsrl.vi v11, v11, 4 \n\t" + + // [4*32]*2 + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v16, v12, v8, 0 \n\t" + "vpack.vv v18, v13, v9, 0 \n\t" + "vpack.vv v20, v14, v10, 0 \n\t" + "vpack.vv v22, v15, v11, 0 \n\t" + + "li t1, 16 \n\t" + "vsetvli t0, x0, e8, m8 \n\t" + "vadd.vx v16, v16, t1, v0.t \n\t" + + // [4*32]*2 -> [8*16] + "vsetvli t0, x0, e8, m1 \n\t" + "vupack.vv v8, v16, v17, 1 \n\t" + "vupack.vv v10, v18, v19, 1 \n\t" + "vupack.vv v12, v20, v21, 1 \n\t" + "vupack.vv v14, v22, v23, 1 \n\t" + + "vsetvli t0, x0, e64, m1 \n\t" + "vslidedown.vi v16, v1, 2 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v16, v16 \n\t" + "vxor.vv v26, v16, v16 \n\t" + "vxor.vv v28, v16, v16 \n\t" + "vxor.vv v30, v16, v16 \n\t" + + // ---- i8 main dot products ---- + // vmadot: A × unsigned Bh × 16 → fp16 accumulate + "vmadot v24, v1, v8, i8 \n\t" // N0..7 + "vmadot v26, v1, v10, i8 \n\t" // N8..15 + "vmadot v28, v1, v12, i8 \n\t" // N16..23 + "vmadot v30, v1, v14, i8 \n\t" // N24..31 + // vmadot: A × unsigned Bh × 1 → fp16 accumulate + "vmadot v24, v16, v9, i8 \n\t" // N0..7 + "vmadot v26, v16, v11, i8 \n\t" // N8..15 + "vmadot v28, v16, v13, i8 \n\t" // N16..23 + "vmadot v30, v16, v15, i8 \n\t" // N24..31 + + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v1, (s4) \n\t" // Bzp + "addi s4, s4, 32+128*4+128 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v16, v24, v26, 2 \n\t" // v16 = N0..15 + "vpack.vv v18, v28, v30, 2 \n\t" // v18 = N16..31 + "vpack.vv v24, v16, v18, 3 \n\t" // v24 = N0..31 + + "vwaddu.vx v28, v1, x0 \n\t" // uint8 -> uint16 + + "vsetvli t0, x0, e16, m1 \n\t" + "vwmul.vx v30, v28, t2 \n\t" + + // b_scale fp16 -> fp32 + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v28, v2 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vadd.vv v24, v24, v30 \n\t" + + // a_scale * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vfmul.vf v30, v28, f0 \n\t" + "vfcvt.f.x.v v26, v24 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + // static_cast(qsum) * a_scale * b_scale; + "vfmacc.vv v3, v30, v26 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, BLK_LPST%= \n\t" + "BLK_LPND%=: \n\t" + "vsetvli t0, %[NBLKS], e32, m1 \n\t" + "vse32.v v3, (%[pC]) \n\t" + "FUNC_END%=: \n\t" + : + : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) + : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6", "s7", "v0", "v1", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v16", "v17", "v18", "v19", + "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + } + } +} + +void gemm_kernel_i8i5_m4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t NB_COLS = 32; + + GGML_UNUSED(count_m); + GGML_UNUSED(blk_len); + + // This kernel computes a 4x32 output tile. For each K32 block we decode the + // packed Q5 weights once and reuse the decoded vectors across the 4 A rows. + constexpr size_t B_Q50_BLK_STRIDE = sizeof(nrow_block_q5_0); + constexpr size_t B_Q51_BLK_STRIDE = sizeof(nrow_block_q5_1); + + if (quant_b_zp) { + // Q5_1 block layout per K32/N32 tile: + // [scale_fp16 x 32][zp_u8 x 32][qh high-bit mask x 128B][qs low nibbles x 512B] + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * k_blks * B_Q51_BLK_STRIDE; + uint8_t * a_data = (uint8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + size_t cnt = k_blks; + + asm volatile( + // v4-v7 are the fp32 accumulators for rows 0..3 of the current N32 tile. + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v4, v4, v4 \n\t" + "vxor.vv v5, v5, v5 \n\t" + "vxor.vv v6, v6, v6 \n\t" + "vxor.vv v7, v7, v7 \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // Load the 4 A-row scales/sums for this K32 block and build row data pointers. + "flw fa0, 0(%[A]) \n\t" + "flw fa1, 4(%[A]) \n\t" + "flw fa2, 8(%[A]) \n\t" + "flw fa3, 12(%[A]) \n\t" + "lh s1, 16(%[A]) \n\t" + "lh s2, 18(%[A]) \n\t" + "lh s3, 20(%[A]) \n\t" + "lh s4, 22(%[A]) \n\t" + "addi t3, %[A], 24 \n\t" + "addi t4, t3, 32 \n\t" + "addi t5, t3, 64 \n\t" + "addi t6, t3, 96 \n\t" + "addi %[A], %[A], 152 \n\t" + + // B-side pointers: + // t1 -> zp stream, t2 -> qh bitmask stream, s5 -> qs low-nibble stream. + "addi t1, %[B], 64 \n\t" + "addi t2, %[B], 96 \n\t" + "addi s5, %[B], 224 \n\t" + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v2, (%[B]) \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v0, (t2) \n\t" + "vl4r.v v8, (s5) \n\t" + "addi %[B], %[B], 736 \n\t" + + // Decode Q5 payload once for the whole tile: + // 1) split `qs` low/high nibbles, + // 2) repack into bytes, + // 3) use the `qh` mask to inject bit4 (+16) where needed, + // 4) expand into the vmadot-friendly layout reused by all 4 rows. + "vand.vi v12, v8, 0xF \n\t" + "vand.vi v13, v9, 0xF \n\t" + "vand.vi v14, v10, 0xF \n\t" + "vand.vi v15, v11, 0xF \n\t" + "vsrl.vi v8, v8, 4 \n\t" + "vsrl.vi v9, v9, 4 \n\t" + "vsrl.vi v10, v10, 4 \n\t" + "vsrl.vi v11, v11, 4 \n\t" + + "vpack.vv v16, v12, v8, 0 \n\t" + "vpack.vv v18, v13, v9, 0 \n\t" + "li t2, 16 \n\t" + "vpack.vv v20, v14, v10, 0 \n\t" + "vpack.vv v22, v15, v11, 0 \n\t" + + "vsetvli t0, x0, e8, m8 \n\t" + "vadd.vx v16, v16, t2, v0.t \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vupack.vv v8, v16, v17, 1 \n\t" + "vupack.vv v10, v18, v19, 1 \n\t" + "vupack.vv v12, v20, v21, 1 \n\t" + "vupack.vv v14, v22, v23, 1 \n\t" + + // Convert per-column fp16 scales once; the same scale vector is shared by all 4 rows. + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v18, v2 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v3, (t1) \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + + // Row 0: dot(A0, decoded_q5) + a_sum0 * zp, then scale by A/B scales. + // The widen/mul correction sequence intentionally matches the proven m1 Q5_1 path. + "vle8.v v1, (t3) \n\t" + "vsetvli t0, x0, e64, m1 \n\t" + "vupack.vv v16, v1, v2, 1 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v24, v24 \n\t" + "vxor.vv v26, v26, v26 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v30, v30, v30 \n\t" + "vmadot v24, v16, v8, i8 \n\t" + "vmadot v26, v16, v10, i8 \n\t" + "vmadot v28, v16, v12, i8 \n\t" + "vmadot v30, v16, v14, i8 \n\t" + "vmadot v24, v17, v9, i8 \n\t" + "vmadot v26, v17, v11, i8 \n\t" + "vmadot v28, v17, v13, i8 \n\t" + "vmadot v30, v17, v15, i8 \n\t" + "vpack.vv v16, v24, v26, 2 \n\t" + "vpack.vv v20, v28, v30, 2 \n\t" + "vpack.vv v24, v16, v20, 3 \n\t" + "vpack.vv v26, v17, v21, 3 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vwaddu.vx v28, v3, x0 \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vwmul.vx v12, v28, s1 \n\t" + "vwmul.vx v14, v28, s2 \n\t" + "vwmul.vx v20, v28, s3 \n\t" + "vwmul.vx v22, v28, s4 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vadd.vv v24, v24, v12 \n\t" + "vadd.vv v25, v25, v14 \n\t" + "vadd.vv v26, v26, v20 \n\t" + "vadd.vv v27, v27, v22 \n\t" + "vfcvt.f.x.v v12, v24 \n\t" + "vfcvt.f.x.v v14, v25 \n\t" + "vfcvt.f.x.v v20, v26 \n\t" + "vfcvt.f.x.v v22, v27 \n\t" + "vfmul.vv v12, v12, v18 \n\t" + "vfmul.vv v14, v14, v18 \n\t" + "vfmul.vv v20, v20, v18 \n\t" + "vfmul.vv v22, v22, v18 \n\t" + "vfmacc.vf v4, fa0, v12 \n\t" + "vfmacc.vf v5, fa1, v14 \n\t" + "vfmacc.vf v6, fa2, v20 \n\t" + "vfmacc.vf v7, fa3, v22 \n\t" + + "addi %[BK], %[BK], -1 \n\t" + "bgtz %[BK], BLK_LOOP%= \n\t" + + // Tail-aware store for the final N tile (`nb_real` may be < 32). + "vsetvli t0, %[NBLKS], e32, m1 \n\t" + "add t1, %[LDC], %[DST] \n\t" + "vse32.v v4, (%[DST]) \n\t" + "vse32.v v5, (t1) \n\t" + "add t2, t1, %[LDC] \n\t" + "vse32.v v6, (t2) \n\t" + "add t3, t2, %[LDC] \n\t" + "vse32.v v7, (t3) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data), [BK] "+r"(cnt) + : [DST] "r"(dst_c), [LDC] "r"(ldc * 4), [NBLKS] "r"(nb_real) + : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "s1", "s2", "s3", "s4", "s5", "v0", "v1", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31", "fa0", "fa1", "fa2", "fa3"); + } + } else { + // Q5_0 block layout per K32/N32 tile: + // [scale_fp16 x 32][qh high-bit mask x 128B][qs low nibbles x 512B] + // There is no explicit zp stream; the implicit midpoint correction is +16. + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * k_blks * B_Q50_BLK_STRIDE; + uint8_t * a_data = (uint8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + size_t cnt = k_blks; + + asm volatile( + // v4-v7 are the fp32 accumulators for rows 0..3 of the current N32 tile. + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v4, v4, v4 \n\t" + "vxor.vv v5, v5, v5 \n\t" + "vxor.vv v6, v6, v6 \n\t" + "vxor.vv v7, v7, v7 \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // Load the 4 A-row scales/sums for this K32 block and build row data pointers. + "flw fa0, 0(%[A]) \n\t" + "flw fa1, 4(%[A]) \n\t" + "flw fa2, 8(%[A]) \n\t" + "flw fa3, 12(%[A]) \n\t" + "lh s1, 16(%[A]) \n\t" + "lh s2, 18(%[A]) \n\t" + "lh s3, 20(%[A]) \n\t" + "lh s4, 22(%[A]) \n\t" + "addi t3, %[A], 24 \n\t" + "addi t4, t3, 32 \n\t" + "addi t5, t3, 64 \n\t" + "addi t6, t3, 96 \n\t" + "addi %[A], %[A], 152 \n\t" + + // B-side pointers: + // t1 -> qh bitmask stream, t2 -> qs low-nibble stream. + "addi t1, %[B], 64 \n\t" + "addi t2, %[B], 192 \n\t" + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v2, (%[B]) \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v0, (t1) \n\t" + "vl4r.v v8, (t2) \n\t" + "addi %[B], %[B], 704 \n\t" + + // Decode Q5 payload once for the whole tile and expand it into the vmadot layout. + "vand.vi v12, v8, 0xF \n\t" + "vand.vi v13, v9, 0xF \n\t" + "vand.vi v14, v10, 0xF \n\t" + "vand.vi v15, v11, 0xF \n\t" + "vsrl.vi v8, v8, 4 \n\t" + "vsrl.vi v9, v9, 4 \n\t" + "vsrl.vi v10, v10, 4 \n\t" + "vsrl.vi v11, v11, 4 \n\t" + + "vpack.vv v16, v12, v8, 0 \n\t" + "vpack.vv v18, v13, v9, 0 \n\t" + "li t2, 16 \n\t" + "vpack.vv v20, v14, v10, 0 \n\t" + "vpack.vv v22, v15, v11, 0 \n\t" + + "vsetvli t0, x0, e8, m8 \n\t" + "vadd.vx v16, v16, t2, v0.t \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vupack.vv v8, v16, v17, 1 \n\t" + "vupack.vv v10, v18, v19, 1 \n\t" + "vupack.vv v12, v20, v21, 1 \n\t" + "vupack.vv v14, v22, v23, 1 \n\t" + + // Convert per-column fp16 scales once; the same scale vector is shared by all 4 rows. + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v18, v2 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + + // Row 0: dot(A0, decoded_q5) + a_sum0 * 16 (implicit Q5_0 midpoint correction). + "vle8.v v1, (t3) \n\t" + "vsetvli t0, x0, e64, m1 \n\t" + "vupack.vv v16, v1, v2, 1 \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v24, v24 \n\t" + "vxor.vv v26, v26, v26 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v30, v30, v30 \n\t" + "vmadot v24, v16, v8, i8 \n\t" + "vmadot v26, v16, v10, i8 \n\t" + "vmadot v28, v16, v12, i8 \n\t" + "vmadot v30, v16, v14, i8 \n\t" + "vmadot v24, v17, v9, i8 \n\t" + "vmadot v26, v17, v11, i8 \n\t" + "vmadot v28, v17, v13, i8 \n\t" + "vmadot v30, v17, v15, i8 \n\t" + "vpack.vv v16, v24, v26, 2 \n\t" + "slli s1, s1, 4 \n\t" + "vpack.vv v20, v28, v30, 2 \n\t" + "slli s2, s2, 4 \n\t" + "vpack.vv v24, v16, v20, 3 \n\t" + "slli s3, s3, 4 \n\t" + "vpack.vv v26, v17, v21, 3 \n\t" + "slli s4, s4, 4 \n\t" + "vadd.vx v24, v24, s1 \n\t" + "vadd.vx v25, v25, s2 \n\t" + "vadd.vx v26, v26, s3 \n\t" + "vadd.vx v27, v27, s4 \n\t" + "vfcvt.f.x.v v24, v24 \n\t" + "vfcvt.f.x.v v25, v25 \n\t" + "vfcvt.f.x.v v26, v26 \n\t" + "vfcvt.f.x.v v27, v27 \n\t" + "vfmul.vv v24, v24, v18 \n\t" + "vfmul.vv v25, v25, v18 \n\t" + "vfmul.vv v26, v26, v18 \n\t" + "vfmul.vv v27, v27, v18 \n\t" + "vfmacc.vf v4, fa0, v24 \n\t" + "vfmacc.vf v5, fa1, v25 \n\t" + "vfmacc.vf v6, fa2, v26 \n\t" + "vfmacc.vf v7, fa3, v27 \n\t" + + "addi %[BK], %[BK], -1 \n\t" + "bgtz %[BK], BLK_LOOP%= \n\t" + + // Tail-aware store for the final N tile (`nb_real` may be < 32). + "vsetvli t0, %[NBLKS], e32, m1 \n\t" + "add t1, %[LDC], %[DST] \n\t" + "vse32.v v4, (%[DST]) \n\t" + "vse32.v v5, (t1) \n\t" + "add t2, t1, %[LDC] \n\t" + "vse32.v v6, (t2) \n\t" + "add t3, t2, %[LDC] \n\t" + "vse32.v v7, (t3) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data), [BK] "+r"(cnt) + : [DST] "r"(dst_c), [LDC] "r"(ldc * 4), [NBLKS] "r"(nb_real) + : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "s1", "s2", "s3", "s4", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", + "fa0", "fa1", "fa2", "fa3"); + } + } +} + +void gemm_kernel_i8i8_m1(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + for (size_t n = 0; n < count_n; n += 32) { + size_t nblks = (count_n - n) > 32 ? 32 : count_n - n; + uint8_t * QuantBDataPtr = (uint8_t *) quant_b_data + // + n * k_blks * blk_len + // b data + n * k_blks * sizeof(_Float16); // scale + float * CPtr = c_ptr + n; + size_t cnt = k_blks; + + // A format Version_1 (FP32 SCALE FOR Normal VMADOTins of IME2) + // A M1K32 int8 256bit + // Ascale fp32 * 1 32bit + // || scl*1(fp32) | Asum(int16) | blk0 || scl*1(fp32) | Asum(int16) | blk0 || ... + // || Element || Element || ... + // B format + // B N8K32 int4 2048bit + // 4VRF, N32K32, 8192bit + // Bscale fp16 * N32 512bit; + // || scl*32..(fp16) | blk0 blk1 ... blk31 || scl*32..(fp16) | blk0 blk1 ... blk31 || ... + // || Element || Element || ... + + //bias always be nullptr + __asm__ volatile( + + // t3 = k/32 + "mv t3, %[BCK] \n\t" + "mv t4, %[NBLKS] \n\t" + "mv s2, %[pA] \n\t" // s2 = pASCL + "addi s3, %[pA], 4+2 \n\t" // s3 = pAData, (pA+AScl+ASum) + "mv s4, %[pB] \n\t" // s4 = pBSCL + "addi s5, %[pB], 32*2 \n\t" // s5 = pBdata; + "mv s6, %[pC] \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v2, v0, v0 \n\t" // clear acc + + // ordinary vmadot: vle*6 flw*1 vecIns*64 vmadot*8 + ".align 4 \n\t" + "_K_LPST%=: \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vl4r.v v4, (s5) \n\t" // B Data 4VRF * 8Row * 32 + "addi s5, s5, 128*4 \n\t" + "vl4r.v v8, (s5) \n\t" // B Data 4VRF * 8Row * 32 + "addi s5, s5, 128*4+64 \n\t" + + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v0, (s4) \n\t" // B Scale 4VRF*8Row*FP16 = 512bit + "addi s4, s4, 64+128*8 \n\t" + + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v3, (s3) \n\t" // A Data M1*K32*int8 = 256bit + "addi s3, s3, 32+6 \n\t" + + "flw f0, (s2) \n\t" // A Scale fp32 + "addi s2, s2, 6+32 \n\t" // AScale + Asum(FP32+i16) + + "vsetvli t0, zero, e32, m1 \n\t" + "vupack.vv v24, v4, v5, 1 \n\t" + "vupack.vv v26, v6, v7, 1 \n\t" + "vupack.vv v28, v8, v9, 1 \n\t" + "vupack.vv v30, v10, v11, 1 \n\t" + + "vslidedown.vi v4, v3, 4 \n\t" + + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + "vmadot v16, v3, v24, i8 \n\t" // M0 N0 - N7 INT32(256bit) + "vmadot v18, v3, v26, i8 \n\t" // M0 N8 - N15 + "vmadot v20, v3, v28, i8 \n\t" // M0 N16 - N23 + "vmadot v22, v3, v30, i8 \n\t" // M0 N24 - N31 + + "vmadot v16, v4, v25, i8 \n\t" + "vmadot v18, v4, v27, i8 \n\t" + "vmadot v20, v4, v29, i8 \n\t" + "vmadot v22, v4, v31, i8 \n\t" + + "vpack.vv v24, v16, v18, 2 \n\t" + "vpack.vv v26, v20, v22, 2 \n\t" + "vpack.vv v16, v24, v26, 3 \n\t" + + // b_scale fp16 -> fp32 + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v24, v0 \n\t" + // mac result i32 -> fp32 + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v26, v16 \n\t" + // a_scale * b_scale; + "vfmul.vf v1, v24, f0 \n\t" + // static_cast(qsum) * a_scale * b_scale; + "vfmacc.vv v2, v1, v26 \n\t" + + "addi t3, t3, -1 \n\t" + "bgtz t3, _K_LPST%= \n\t" + "_K_LPND%=: \n\t" + + //----------------------------------------- + // STORE Equal 32N------------------------- + "_ST32%=: \n\t" + "vsetvli t0, t4, e32, m1 \n\t" + "vse32.v v2, (s6) \n\t" // M0 [N0 : N32]; FP32(1024bit) + + "_FUNC_END%=: \n\t" + + : + : [BCK] "r"(cnt), [NBLKS] "r"(nblks), [pA] "r"(quant_a_ptr), [pB] "r"(QuantBDataPtr), [pC] "r"(CPtr) + : "cc", "t0", "t3", "t4", "f0", "s2", "s3", "s4", "s5", "s6"); + } +} + +void gemm_kernel_i8i8_m4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + int64_t b_data_stride = k_blks * sizeof(ggml_fp16_t) + k_blks * blk_len; + for (size_t ni = 0; ni < count_n; ni += 32) { + uint8_t * b_data = (uint8_t *) quant_b_data + ni * b_data_stride; + int8_t * a_data = (int8_t *) quant_a_ptr; + float * dst_c = c_ptr + ni; + + asm volatile( + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v29, v29, v29 \n\t" + "vxor.vv v30, v30, v30 \n\t" + "vxor.vv v31, v31, v31 \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // load scale A + "flw fa0, (%[A]) \n\t" + "flw fa1, 4(%[A]) \n\t" + "flw fa2, 8(%[A]) \n\t" + "flw fa3, 12(%[A]) \n\t" + "addi %[A], %[A], 16+8 \n\t" // Ascl+Asum; FP32*4+i16*4 + + // load scale B + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v12, (%[B]) \n\t" + "addi %[B], %[B], 64 \n\t" + "vfwcvt.f.f.v v14, v12 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vl1r.v v0, (%[A]) \n\t" + "addi %[A], %[A], 128 \n\t" // 4*32@i8 + "vl4r.v v4, (%[B]) \n\t" // 32*32@i8 + "addi %[B], %[B], 512 \n\t" + "vl4r.v v8, (%[B]) \n\t" // 32*32@i8 + "addi %[B], %[B], 512 \n\t" + + "vsetvli t0, zero, e32, m1 \n\t" + "vupack.vv v2, v0, v0, 1 \n\t" + + "vupack.vv v24, v4, v5, 1 \n\t" + "vupack.vv v26, v6, v7, 1 \n\t" + "vupack.vv v4, v8, v9, 1 \n\t" + "vupack.vv v6, v10, v11, 1 \n\t" + + // init the accumu to asum * zp + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v16, v16, v16 \n\t" + "vxor.vv v18, v16, v16 \n\t" + "vxor.vv v20, v16, v16 \n\t" + "vxor.vv v22, v16, v16 \n\t" + + // i4 * i4 vmadot + "vsetvli t0, x0, e32, m1 \n\t" + "vmadot v16, v2, v24, i8 \n\t" + "vmadot v18, v2, v26, i8 \n\t" + "vmadot v20, v2, v4, i8 \n\t" + "vmadot v22, v2, v6, i8 \n\t" + "vmadot v16, v3, v25, i8 \n\t" + "vmadot v18, v3, v27, i8 \n\t" + "vmadot v20, v3, v5, i8 \n\t" + "vmadot v22, v3, v7, i8 \n\t" + + "vpack.vv v0, v16, v18, 2 \n\t" + "vpack.vv v2, v20, v22, 2 \n\t" + "vpack.vv v16, v0, v2, 3 \n\t" + "vpack.vv v18, v1, v3, 3 \n\t" + + "vfcvt.f.x.v v16, v16 \n\t" + "vfcvt.f.x.v v17, v17 \n\t" + "vfcvt.f.x.v v18, v18 \n\t" + "vfcvt.f.x.v v19, v19 \n\t" + + // mul scale + "vfmul.vv v16, v16, v14 \n\t" + "vfmul.vv v17, v17, v14 \n\t" + "vfmul.vv v18, v18, v14 \n\t" + "vfmul.vv v19, v19, v14 \n\t" + + "addi %[BK], %[BK], -1 \n\t" + "vfmacc.vf v28, fa0, v16 \n\t" + "vfmacc.vf v29, fa1, v17 \n\t" + "vfmacc.vf v30, fa2, v18 \n\t" + "vfmacc.vf v31, fa3, v19 \n\t" + + "bgtz %[BK], BLK_LOOP%= \n\t" + + // save + "vsetvli t0, x0, e32, m1 \n\t" + "add t2, %[LDC], %[DST] \n\t" + "vse32.v v28, (%[DST]) \n\t" + "add t3, %[LDC], t2 \n\t" + "vse32.v v29, (t2) \n\t" + "add t2, %[LDC], t3 \n\t" + "vse32.v v30, (t3) \n\t" + "vse32.v v31, (t2) \n\t" + : [A] "+r"(a_data), [B] "+r"(b_data) + : [DST] "r"(dst_c), [LDC] "r"(ldc * 4), [BK] "r"(k_blks) + : "t0", "t1", "t2", "t3", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30", "v31", "fa0", "fa1", "fa2", "fa3"); + } +} + +void moe_m2_gemm_kernel_i8i4_impl(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { +#if 0 + moe_gemm_kernel_i8i4_mrow_ref<2, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, + ldc); +#else + int64_t b_data_stride = + k_blks * (sizeof(ggml_fp16_t) + 16 * sizeof(int8_t) + (quant_b_zp != NULL ? sizeof(int8_t) : 0)); + if (quant_b_zp == NULL) { + for (size_t ni = 0; ni < count_n; ni += 32) { + uint8_t * b_data = (uint8_t *) quant_b_data + ni * b_data_stride; + int8_t * a_data0 = (int8_t *) quant_a_ptr[0]; + int8_t * a_data1 = (int8_t *) quant_a_ptr[1]; + float * dst_c0 = (float *) c_ptr[0] + ni; + float * dst_c1 = (float *) c_ptr[1] + ni; + + asm volatile( + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v29, v29, v29 \n\t" + "vmv.v.i v0, 1 \n\t" // init the scale + "vsll.vi v1, v0, 4 \n\t" + "vfcvt.f.x.v v0, v0 \n\t" + "vfcvt.f.x.v v1, v1 \n\t" + "mv t3, %[BK] \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // load scale A0 + "flw fa0, (%[A0]) \n\t" // A0 scale + "lh t1, 4(%[A0]) \n\t" // A0 asum + "addi %[A0], %[A0], 6 \n\t" + + // load scale B + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v12, (%[B]) \n\t" + "addi %[B], %[B], 64 \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v14, v12, v12, 3 \n\t" + + // load scale A1 + "flw fa1, (%[A1]) \n\t" // A1 scale + "lh t2, 4(%[A1]) \n\t" // A1 asum + "addi %[A1], %[A1], 6 \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vmv.v.x v10, t1 \n\t" + "vmv.v.x v11, t2 \n\t" + + "vpack.vv v18, v10, v11, 1 \n\t" + "vsll.vi v18, v18, 3 \n\t" // mul 8 + "vfcvt.f.x.v v18, v18 \n\t" + + "vsetvli t0, x0, e8, mf4 \n\t" // A0 data + "vle8.v v16, (%[A0]) \n\t" + "addi %[A0], %[A0], 32 \n\t" // 1*32@i8 + "vle8.v v20, (%[A1]) \n\t" + "addi %[A1], %[A1], 32 \n\t" // 1*32@i8 + + "vl4r.v v4, (%[B]) \n\t" // 32*32@i4 + "addi %[B], %[B], 512 \n\t" + + "vsrl.vi v17, v16, 4 \n\t" + "vsrl.vi v21, v20, 4 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vnpack4.vv v2, v16, v20, 2 \n\t" // low u4 + "vnpack4.vv v3, v17, v21, 2 \n\t" // high s4 + + // init the accumu to asum * zp + "vsetvli t0, x0, e16, m1 \n\t" + "vor.vv v19, v18, v18 \n\t" + "vor.vv v20, v18, v18 \n\t" + "vor.vv v21, v18, v18 \n\t" + + // i4 * i4 vmadot + "vsetvli t0, x0, e16, m1 \n\t" + "vmadotsu.hp v18, v3, v4, v1, 0, i4 \n\t" // high 4 + "vmadotsu.hp v19, v3, v5, v1, 0, i4 \n\t" + "vmadotsu.hp v20, v3, v6, v1, 0, i4 \n\t" + "vmadotsu.hp v21, v3, v7, v1, 0, i4 \n\t" + "vmadotu.hp v18, v2, v4, v0, 0, i4 \n\t" // low 4 + "vmadotu.hp v19, v2, v5, v0, 0, i4 \n\t" + "vmadotu.hp v20, v2, v6, v0, 0, i4 \n\t" + "vmadotu.hp v21, v2, v7, v0, 0, i4 \n\t" + + "vpack.vv v8, v18, v19, 1 \n\t" + "vpack.vv v12, v20, v21, 1 \n\t" + "vpack.vv v20, v8, v12, 2 \n\t" + + "vfwmul.vv v16, v20, v14 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + + "addi t3, t3, -1 \n\t" + "vfmacc.vf v28, fa0, v16 \n\t" + "vfmacc.vf v29, fa1, v17 \n\t" + + "bgtz t3, BLK_LOOP%= \n\t" + + // save + "vsetvli t0, x0, e32, m1 \n\t" + "vse32.v v28, (%[DST0]) \n\t" + "vse32.v v29, (%[DST1]) \n\t" + : [A0] "+r"(a_data0), [A1] "+r"(a_data1), [B] "+r"(b_data) + : [DST0] "r"(dst_c0), [DST1] "r"(dst_c1), [BK] "r"(k_blks) + : "t0", "t1", "t2", "t3", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", + "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", + "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "fa2", "fa3"); + } + } else { +# if 0 + moe_gemm_kernel_i8i4_mrow_ref<2, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +# else + for (size_t ni = 0; ni < count_n; ni += 32) { + uint8_t * b_data = (uint8_t *) quant_b_data + ni * b_data_stride; + int8_t * a_data0 = (int8_t *) quant_a_ptr[0]; + int8_t * a_data1 = (int8_t *) quant_a_ptr[1]; + float * dst_c0 = (float *) c_ptr[0] + ni; + float * dst_c1 = (float *) c_ptr[1] + ni; + + asm volatile( + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v28, v28, v28 \n\t" + "vxor.vv v29, v29, v29 \n\t" + "vmv.v.i v0, 1 \n\t" // init the scale + "vsll.vi v1, v0, 4 \n\t" + "vfcvt.f.x.v v0, v0 \n\t" + "vfcvt.f.x.v v1, v1 \n\t" + "mv t3, %[BK] \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // load scale A0 + "flw fa0, (%[A0]) \n\t" // A0 scale + "lh t1, 4(%[A0]) \n\t" // A0 asum + "addi %[A0], %[A0], 6 \n\t" + + // load scale B + "vsetvli t0, x0, e16, mf2 \n\t" + "vle16.v v12, (%[B]) \n\t" + "addi %[B], %[B], 64 \n\t" + "vsetvli t0, x0, e16, m1 \n\t" + "vpack.vv v14, v12, v12, 3 \n\t" + + // load scale A1 + "flw fa1, (%[A1]) \n\t" // A1 scale + "lh t2, 4(%[A1]) \n\t" // A1 asum + "addi %[A1], %[A1], 6 \n\t" + + // load zp + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v8, (%[B]) \n\t" + "addi %[B], %[B], 32 \n\t" + "vwaddu.vx v10, v8, x0 \n\t" + + "vsetvli t0, x0, e8, mf4 \n\t" // A0 data + "vle8.v v16, (%[A0]) \n\t" + "addi %[A0], %[A0], 32 \n\t" // 1*32@i8 + "vle8.v v20, (%[A1]) \n\t" + "addi %[A1], %[A1], 32 \n\t" // 1*32@i8 + + "vl4r.v v4, (%[B]) \n\t" // 32*32@i4 + "addi %[B], %[B], 512 \n\t" + + "vsrl.vi v17, v16, 4 \n\t" + "vsrl.vi v21, v20, 4 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vnpack4.vv v2, v16, v20, 2 \n\t" // low u4 + "vnpack4.vv v3, v17, v21, 2 \n\t" // high s4 + + // init the accumu to asum * zp + "vsetvli t0, x0, e16, m1 \n\t" + "vxor.vv v18, v18, v18 \n\t" + "vxor.vv v19, v19, v19 \n\t" + "vxor.vv v20, v20, v20 \n\t" + "vxor.vv v21, v21, v21 \n\t" + + // i4 * i4 vmadot + "vsetvli t0, x0, e16, m1 \n\t" + "vmadotsu.hp v18, v3, v4, v1, 0, i4 \n\t" // high 4 + "vmadotsu.hp v19, v3, v5, v1, 0, i4 \n\t" + "vmadotsu.hp v20, v3, v6, v1, 0, i4 \n\t" + "vmadotsu.hp v21, v3, v7, v1, 0, i4 \n\t" + "vmadotu.hp v18, v2, v4, v0, 0, i4 \n\t" // low 4 + "vmadotu.hp v19, v2, v5, v0, 0, i4 \n\t" + "vmadotu.hp v20, v2, v6, v0, 0, i4 \n\t" + "vmadotu.hp v21, v2, v7, v0, 0, i4 \n\t" + + "vpack.vv v8, v18, v19, 1 \n\t" + "vpack.vv v12, v20, v21, 1 \n\t" + "vpack.vv v20, v8, v12, 2 \n\t" + // asum*zp + "vsetvli t0, x0, e16, mf2 \n\t" + "vwmul.vx v2, v10, t1 \n\t" + "vwmul.vx v4, v10, t2 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + + "vfcvt.f.x.v v2, v2 \n\t" + "vfcvt.f.x.v v4, v4 \n\t" + + "vsetvli t0, x0, e16, m1 \n\t" + "vfwcvt.f.f.v v16, v20 \n\t" + + "vfwcvt.f.f.v v18, v14 \n\t" + + // +asum*zp + "vsetvli t0, x0, e32, m1 \n\t" + "vfadd.vv v16, v16, v2 \n\t" + "vfadd.vv v17, v17, v4 \n\t" + "vfmul.vv v16, v16, v18 \n\t" + "vfmul.vv v17, v17, v18 \n\t" + + "addi t3, t3, -1 \n\t" + "vfmacc.vf v28, fa0, v16 \n\t" + "vfmacc.vf v29, fa1, v17 \n\t" + + "bgtz t3, BLK_LOOP%= \n\t" + + // save + "vsetvli t0, x0, e32, m1 \n\t" + "vse32.v v28, (%[DST0]) \n\t" + "vse32.v v29, (%[DST1]) \n\t" + : [A0] "+r"(a_data0), [A1] "+r"(a_data1), [B] "+r"(b_data) + : [DST0] "r"(dst_c0), [DST1] "r"(dst_c1), [BK] "r"(k_blks) + : "t0", "t1", "t2", "t3", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", + "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", + "v26", "v27", "v28", "v29", "v30", "v31", "fa0", "fa1", "fa2", "fa3"); + } +# endif + } +#endif +} + +void moe_m2_gemm_kernel_i8i5_impl(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + constexpr size_t NB_COLS = 32; + constexpr size_t B_Q50_BLK_STRIDE = sizeof(nrow_block_q5_0); + constexpr size_t B_Q51_BLK_STRIDE = sizeof(nrow_block_q5_1); + + GGML_UNUSED(blk_len); + GGML_UNUSED(count_m); + GGML_UNUSED(ldc); + + if (quant_b_zp == NULL) { + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * k_blks * B_Q50_BLK_STRIDE; + int8_t * a_data0 = (int8_t *) quant_a_ptr[0]; + int8_t * a_data1 = (int8_t *) quant_a_ptr[1]; + float * dst_c0 = (float *) c_ptr[0] + ni; + float * dst_c1 = (float *) c_ptr[1] + ni; + + asm volatile( + "mv t4, %[BK] \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v2, v0, v0 \n\t" + "vxor.vv v3, v0, v0 \n\t" + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // ---- load B scale/Bh/Bs and advance to the next q5_0 k-block ---- + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v1, (%[B]) \n\t" // v1 = scale_fp16 × 32 + "addi %[B], %[B], 64 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v0, (%[B]) \n\t" // v0 = Bh N32K32 1-bit packed + "addi %[B], %[B], 128 \n\t" + "vl4r.v v8, (%[B]) \n\t" // v8..v11 = Bs N32K32 i4 + "addi %[B], %[B], 512 \n\t" + + // ---- load A0/A1 header then payload, each block stride = 38B ---- + "flw f0, (%[A0]) \n\t" // f0 = A0 scale (fp32) + "lh t2, 4(%[A0]) \n\t" // t2 = A0 asum (int16) + "addi %[A0], %[A0], 6 \n\t" + "flw f1, (%[A1]) \n\t" // f1 = A1 scale (fp32) + "lh t3, 4(%[A1]) \n\t" // t3 = A1 asum (int16) + "addi %[A1], %[A1], 6 \n\t" + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v4, (%[A0]) \n\t" // v4 = A0 M1K32 int8 + "addi %[A0], %[A0], 32 \n\t" + "vle8.v v5, (%[A1]) \n\t" // v5 = A1 M1K32 int8 + "addi %[A1], %[A1], 32 \n\t" + + //// ---- A nibble unpacking ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vand.vi v12, v8, 0xF \n\t" //8bit(lo4) //[8*32] + "vand.vi v13, v9, 0xF \n\t" + "vand.vi v14, v10, 0xF \n\t" + "vand.vi v15, v11, 0xF \n\t" + "vsrl.vi v8, v8, 4 \n\t" //8bit(hi4) + "vsrl.vi v9, v9, 4 \n\t" + "vsrl.vi v10, v10, 4 \n\t" + "vsrl.vi v11, v11, 4 \n\t" + + "slli t2, t2, 4 \n\t" // a_sum * 16; + "slli t3, t3, 4 \n\t" + // [4*32]*2 + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v16, v12, v8, 0 \n\t" + "vpack.vv v18, v13, v9, 0 \n\t" + "vpack.vv v20, v14, v10, 0 \n\t" + "vpack.vv v22, v15, v11, 0 \n\t" + + "li t1, 16 \n\t" + "vsetvli t0, x0, e8, m8 \n\t" + "vadd.vx v16, v16, t1, v0.t \n\t" + + // [4*32]*2 -> [8*16] + "vsetvli t0, x0, e8, m1 \n\t" + "vupack.vv v8, v16, v17, 1 \n\t" + "vupack.vv v10, v18, v19, 1 \n\t" + "vupack.vv v12, v20, v21, 1 \n\t" + "vupack.vv v14, v22, v23, 1 \n\t" + + "vpack.vv v6, v4, v5, 2 \n\t" + + // init the accumu to asum * zp + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v16, v16 \n\t" + "vxor.vv v26, v16, v16 \n\t" + "vupack.vv v4, v6, v7, 1 \n\t" + "vxor.vv v28, v16, v16 \n\t" + "vxor.vv v30, v16, v16 \n\t" + + // ---- i8 main dot products ---- + // vmadot: A × unsigned Bh × 16 → fp16 accumulate + "vmadot v24, v4, v8, i8 \n\t" // N0..7 + "vmadot v26, v4, v10, i8 \n\t" // N8..15 + "vmadot v28, v4, v12, i8 \n\t" // N16..23 + "vmadot v30, v4, v14, i8 \n\t" // N24..31 + // vmadot: A × unsigned Bh × 1 → fp16 accumulate + "vmadot v24, v5, v9, i8 \n\t" // N0..7 + "vmadot v26, v5, v11, i8 \n\t" // N8..15 + "vmadot v28, v5, v13, i8 \n\t" // N16..23 + "vmadot v30, v5, v15, i8 \n\t" // N24..31 + + "vpack.vv v16, v24, v26, 2 \n\t" // v16 = N0..15 + "vpack.vv v18, v28, v30, 2 \n\t" // v18 = N16..31 + "vpack.vv v24, v16, v18, 3 \n\t" // v24 = N0..31 + + "vadd.vx v24, v24, t2 \n\t" + "vadd.vx v25, v25, t3 \n\t" + // b_scale fp16 -> fp32 + "vsetvli t0, x0, e16, mf2 \n\t" + "vfwcvt.f.f.v v28, v1 \n\t" + + // a_scale * b_scale; + "vsetvli t0, x0, e32, m1 \n\t" + "vfcvt.f.x.v v26, v24 \n\t" + "vfcvt.f.x.v v27, v25 \n\t" + "vfmul.vf v30, v28, f0 \n\t" + "vfmul.vf v31, v28, f1 \n\t" + // static_cast(qsum) * a_scale * b_scale; + "vfmacc.vv v2, v30, v26 \n\t" + "vfmacc.vv v3, v31, v27 \n\t" + + "addi t4, t4, -1 \n\t" + "bgtz t4, BLK_LOOP%= \n\t" + + "vsetvli t0, %[NR], e32, m1 \n\t" + "vse32.v v2, (%[DST0]) \n\t" + "vse32.v v3, (%[DST1]) \n\t" + : [A0] "+r"(a_data0), [A1] "+r"(a_data1), [B] "+r"(b_data) + : [DST0] "r"(dst_c0), [DST1] "r"(dst_c1), [BK] "r"(k_blks), [NR] "r"(nb_real) + : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", + "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "f0", "f1"); + } + } else { + for (size_t ni = 0; ni < count_n; ni += NB_COLS) { + size_t nb_real = std::min(NB_COLS, count_n - ni); + uint8_t * b_data = (uint8_t *) quant_b_data + (ni / NB_COLS) * k_blks * B_Q51_BLK_STRIDE; + int8_t * a_data0 = (int8_t *) quant_a_ptr[0]; + int8_t * a_data1 = (int8_t *) quant_a_ptr[1]; + float * dst_c0 = (float *) c_ptr[0] + ni; + float * dst_c1 = (float *) c_ptr[1] + ni; + + asm volatile( + "mv t4, %[BK] \n\t" + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v2, v0, v0 \n\t" + "vxor.vv v3, v0, v0 \n\t" + "addi t5, %[B], 64 \n\t" // t5 = zp (32B) + "addi t6, %[B], 96 \n\t" // t6 = qh (128B) + "addi s1, %[B], 224 \n\t" // s1 = qs (512B) + + ".align 4 \n\t" + "BLK_LOOP%=: \n\t" + // ---- load B scale/zp/Bh/Bs and advance to the next q5_1 k-block ---- + "vsetvli t0, x0, e8, mf2 \n\t" + "vle8.v v1, (%[B]) \n\t" // v1 = scale_fp16 × 32 + "addi %[B], %[B], 736 \n\t" + "vsetvli t0, x0, e8, m1 \n\t" + "vle8.v v0, (t6) \n\t" // v0 = Bh N32K32 1-bit packed + "addi t6, t6, 736 \n\t" + "vl4r.v v8, (s1) \n\t" // v8..v11 = Bs N32K32 i4 + "addi s1, s1, 736 \n\t" + + // ---- load A0/A1 header then payload, each block stride = 38B ---- + "flw f0, (%[A0]) \n\t" // f0 = A0 scale (fp32) + "lh t2, 4(%[A0]) \n\t" // t2 = A0 asum (int16) + "addi %[A0], %[A0], 6 \n\t" + "flw f1, (%[A1]) \n\t" // f1 = A1 scale (fp32) + "lh t3, 4(%[A1]) \n\t" // t3 = A1 asum (int16) + "addi %[A1], %[A1], 6 \n\t" + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v4, (%[A0]) \n\t" // v4 = A0 M1K32 int8 + "addi %[A0], %[A0], 32 \n\t" + "vle8.v v5, (%[A1]) \n\t" // v5 = A1 M1K32 int8 + "addi %[A1], %[A1], 32 \n\t" + + //// ---- A nibble unpacking ---- + "vsetvli t0, x0, e8, m1 \n\t" + "vand.vi v12, v8, 0xF \n\t" //8bit(lo4) //[8*32] + "vand.vi v13, v9, 0xF \n\t" + "vand.vi v14, v10, 0xF \n\t" + "vand.vi v15, v11, 0xF \n\t" + "vsrl.vi v8, v8, 4 \n\t" //8bit(hi4) + "vsrl.vi v9, v9, 4 \n\t" + "vsrl.vi v10, v10, 4 \n\t" + "vsrl.vi v11, v11, 4 \n\t" + + // q5_1 uses explicit zp, so keep a_sum unshifted here. + // [4*32]*2 + "vpack.vv v16, v12, v8, 0 \n\t" + "vpack.vv v18, v13, v9, 0 \n\t" + "vpack.vv v20, v14, v10, 0 \n\t" + "vpack.vv v22, v15, v11, 0 \n\t" + + "li t1, 16 \n\t" + "vsetvli t0, x0, e8, m8 \n\t" + "vadd.vx v16, v16, t1, v0.t \n\t" + + // [4*32]*2 -> [8*16] + "vsetvli t0, x0, e8, m1 \n\t" + "vupack.vv v8, v16, v17, 1 \n\t" + "vupack.vv v10, v18, v19, 1 \n\t" + "vupack.vv v12, v20, v21, 1 \n\t" + "vupack.vv v14, v22, v23, 1 \n\t" + + "vpack.vv v6, v4, v5, 2 \n\t" + + // init the accumu to asum * zp + "vsetvli t0, x0, e32, m1 \n\t" + "vxor.vv v24, v16, v16 \n\t" + "vxor.vv v26, v16, v16 \n\t" + "vupack.vv v4, v6, v7, 1 \n\t" + "vxor.vv v28, v16, v16 \n\t" + "vxor.vv v30, v16, v16 \n\t" + + // ---- i8 main dot products ---- + // vmadot: A × unsigned Bh × 16 → fp16 accumulate + "vmadot v24, v4, v8, i8 \n\t" // N0..7 + "vmadot v26, v4, v10, i8 \n\t" // N8..15 + "vmadot v28, v4, v12, i8 \n\t" // N16..23 + "vmadot v30, v4, v14, i8 \n\t" // N24..31 + // vmadot: A × unsigned Bh × 1 → fp16 accumulate + "vmadot v24, v5, v9, i8 \n\t" // N0..7 + "vmadot v26, v5, v11, i8 \n\t" // N8..15 + "vmadot v28, v5, v13, i8 \n\t" // N16..23 + "vmadot v30, v5, v15, i8 \n\t" // N24..31 + + "vsetvli t0, x0, e8, mf4 \n\t" + "vle8.v v4, (t5) \n\t" // v4 = Bzp N32 uint8 + "addi t5, t5, 736 \n\t" + + "vsetvli t0, x0, e8, m1 \n\t" + "vpack.vv v16, v24, v26, 2 \n\t" // v16 = N0..15 + "vpack.vv v18, v28, v30, 2 \n\t" // v18 = N16..31 + "vpack.vv v24, v16, v18, 3 \n\t" // v24 = N0..31 + + "vsetvli t0, x0, e8, mf4 \n\t" + "vwaddu.vx v28, v4, x0 \n\t" + + "vsetvli t0, x0, e16, mf2 \n\t" + "vwmul.vx v30, v28, t2 \n\t" + "vwmul.vx v31, v28, t3 \n\t" + + // b_scale fp16 -> fp32 + "vfwcvt.f.f.v v28, v1 \n\t" + + "vsetvli t0, x0, e32, m1 \n\t" + "vadd.vv v24, v24, v30 \n\t" + "vadd.vv v25, v25, v31 \n\t" + + // a_scale * b_scale; + "vfcvt.f.x.v v26, v24 \n\t" + "vfcvt.f.x.v v27, v25 \n\t" + "vfmul.vf v30, v28, f0 \n\t" + "vfmul.vf v31, v28, f1 \n\t" + // static_cast(qsum) * a_scale * b_scale; + "vfmacc.vv v2, v30, v26 \n\t" + "vfmacc.vv v3, v31, v27 \n\t" + + "addi t4, t4, -1 \n\t" + "bgtz t4, BLK_LOOP%= \n\t" + + "vsetvli t0, %[NR], e32, m1 \n\t" + "vse32.v v2, (%[DST0]) \n\t" + "vse32.v v3, (%[DST1]) \n\t" + : [A0] "+r"(a_data0), [A1] "+r"(a_data1), [B] "+r"(b_data) + : [DST0] "r"(dst_c0), [DST1] "r"(dst_c1), [BK] "r"(k_blks), [NR] "r"(nb_real) + : "cc", "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "s1", "v0", "v1", "v2", "v3", "v4", "v5", + "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", + "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "f0", "f1"); + } + } +} + +size_t gemm_kernel_i8i2k(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + if (count_m >= 4) { +#if 0 + gemm_kernel_i8i2k_mrow_ref<4, 32>(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_m, count_n, k_blks, ldc); +#else + gemm_kernel_i8i2k_m4(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 4; + } else { +#if 0 + gemm_kernel_i8i2k_mrow_ref<1, 32>(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_m, count_n, k_blks, + ldc); +#else + gemm_kernel_i8i2k_m1(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 1; + } +} + +size_t gemm_kernel_i8i3k(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + if (count_m >= 4) { +#if 0 + gemm_kernel_i8i3k_mrow_ref<4, 32>(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_m, count_n, k_blks, ldc); +#else + gemm_kernel_i8i3k_m4(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 4; + } else { +#if 0 + gemm_kernel_i8i3k_mrow_ref<1, 32>(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_m, count_n, k_blks, ldc); +#else + gemm_kernel_i8i3k_m1(blk_len, quant_a_ptr, quant_b_data, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 1; + } +} + +size_t gemm_kernel_i8i4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + if (count_m >= 4) { +#if 0 + gemm_kernel_i8i4_mrow_ref<4, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8i4_m4(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 4; + } else { +#if 0 + gemm_kernel_i8i4_mrow_ref<1, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8i4_m1(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 1; + } +} + +size_t gemm_kernel_i8i4_hp(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + if (count_m >= 4) { +#if 0 + gemm_kernel_i8i4_hp_mrow_ref<4, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8i4_hp_m4(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 4; + } else { +#if 0 + gemm_kernel_i8i4_hp_mrow_ref<1, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8i4_hp_m1(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 1; + } +} + +size_t moe_m2_gemm_kernel_i8i4(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + moe_m2_gemm_kernel_i8i4_impl(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); + return 2; +} + +size_t gemm_kernel_i8i8(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + if (count_m >= 4) { +#if 0 + gemm_kernel_i8i8_mrow_ref<4, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8i8_m4(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 4; + } else { +#if 0 + gemm_kernel_i8i8_mrow_ref<1, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8i8_m1(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 1; + } +} + +size_t gemm_kernel_i8mxfp4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + if (count_m >= 4) { +#if 1 + gemm_kernel_i8mxfp4_mrow_ref<4, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8mxfp4_m4(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 4; + } else { +#if 1 + gemm_kernel_i8mxfp4_mrow_ref<1, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8mxfp4_m1(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 1; + } +} + +size_t moe_m2_gemm_kernel_i8mxfp4(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + //moe_m2_gemm_kernel_i8mxfp4_impl(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); + return 2; +} + +size_t gemm_kernel_i8i5(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { + if (count_m >= 4) { +#if 0 + gemm_kernel_i8i5_mrow_ref<4, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8i5_m4(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 4; + } else { +#if 0 + gemm_kernel_i8i5_mrow_ref<1, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + gemm_kernel_i8i5_m1(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 1; + } +} + +size_t moe_m2_gemm_kernel_i8i5(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc) { +#if 0 + moe_gemm_kernel_i8i5_mrow_ref<2, 32>(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, + k_blks, ldc); +#else + moe_m2_gemm_kernel_i8i5_impl(blk_len, quant_a_ptr, quant_b_data, quant_b_zp, c_ptr, count_m, count_n, k_blks, ldc); +#endif + return 2; +} + +} // namespace ime2 +} // namespace spacemit_kernels diff --git a/ggml/src/ggml-cpu/spacemit/ime_env.cpp b/ggml/src/ggml-cpu/spacemit/ime_env.cpp new file mode 100644 index 0000000..a13ba39 --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/ime_env.cpp @@ -0,0 +1,320 @@ +#include "ime_env.h" + +#include "ggml-impl.h" +#include "spine_mem_pool.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace ggml::cpu::riscv64_spacemit { +bool spine_core_info::get_spine_core_info(std::vector & result) { + static std::unordered_map spine_march_mapping_ = { + {0x8000000058000001, spine_core_arch_id::core_arch_x60 }, + { 0x8000000041000001, spine_core_arch_id::core_arch_a60 }, + { 0x8000000058000002, spine_core_arch_id::core_arch_x100}, + { 0x8000000041000002, spine_core_arch_id::core_arch_a100}, + }; + + result.clear(); + std::ifstream file("/proc/cpuinfo"); + std::string line; + + std::vector> cpu_info_list; + + uint64_t current_processor = spine_invalid_core_id; + uint64_t current_marchid = 0; + bool has_processor = false; + bool has_marchid = false; + + if (!file.is_open()) { + return false; + } + + while (std::getline(file, line)) { + if (line.substr(0, 9) == "processor") { + if (has_processor && has_marchid) { + cpu_info_list.push_back({ current_processor, current_marchid }); + } + + size_t colon_pos = line.find(':'); + if (colon_pos != std::string::npos) { + current_processor = std::stoi(line.substr(colon_pos + 1)); + has_processor = true; + } + + has_marchid = false; + } else if (line.substr(0, 7) == "marchid") { + size_t colon_pos = line.find(':'); + if (colon_pos != std::string::npos) { + std::string marchid_str = line.substr(colon_pos + 1); + marchid_str.erase(std::remove_if(marchid_str.begin(), marchid_str.end(), isspace), marchid_str.end()); + current_marchid = std::stoull(marchid_str, nullptr, 16); + has_marchid = true; + } + } + } + + if (has_processor && has_marchid) { + cpu_info_list.push_back({ current_processor, current_marchid }); + } + + if (has_processor && has_marchid) { + for (auto & cpu_info : cpu_info_list) { + if (cpu_info[0] != spine_invalid_core_id && + spine_march_mapping_.find(cpu_info[1]) != spine_march_mapping_.end()) { + auto core_info = spine_core_info(); + core_info.core_id = cpu_info[0]; + core_info.arch_id = spine_core_arch_id(spine_march_mapping_[cpu_info[1]]); + + result.push_back(core_info); + } + } + } + + return has_processor && has_marchid; +} + +namespace { +uint16_t hex_string_to_u16(const std::string & hex_str) { + try { + size_t pos = 0; + if (hex_str.substr(0, 2) == "0x" || hex_str.substr(0, 2) == "0X") { + pos = 2; + } + unsigned long result = std::stoul(hex_str.substr(pos), nullptr, 16); + if (result > std::numeric_limits::max()) { + throw std::out_of_range("Converted value is out of range for uint16_t"); + } + return static_cast(result); + } catch (const std::invalid_argument & e) { + throw std::invalid_argument("Invalid hexadecimal string"); + } catch (const std::out_of_range & e) { + throw; + } +} + +const char * spine_mem_pool_backend_to_string(spine_mem_pool_backend backend) { + switch (backend) { + case spine_mem_pool_backend::none: + return "NONE"; + case spine_mem_pool_backend::posix_memalign: + return "POSIX"; + case spine_mem_pool_backend::transparent_hugepage: + return "HPAGE"; + case spine_mem_pool_backend::hugetlb_1g: + return "HPAGE1GB"; + } + + return "unknown"; +} + +spine_mem_pool_backend parse_mem_backend(const char * mem_backend_str) { + if (mem_backend_str == nullptr || mem_backend_str[0] == '\0') { + return spine_mem_pool_backend::transparent_hugepage; + } + + std::string value(mem_backend_str); + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + + if (value == "none") { + return spine_mem_pool_backend::none; + } + + if (value == "posix") { + return spine_mem_pool_backend::posix_memalign; + } + + if (value == "hpage") { + return spine_mem_pool_backend::transparent_hugepage; + } + + if (value == "hpage1gb") { + return spine_mem_pool_backend::hugetlb_1g; + } + + throw std::runtime_error("invalid SPACEMIT_MEM_BACKEND: " + value + ", expected NONE, POSIX, HPAGE or HPAGE1GB"); +} +} // namespace + +spine_env_info::spine_env_info() { + num_cores = static_cast(std::thread::hardware_concurrency()); + spine_core_info::get_spine_core_info(core_info_list); + + // special for x60 K1 + if (core_info_list.size() == 8 && core_info_list[0].arch_id == spine_core_arch_id::core_arch_x60) { + for (int i = 0; i < 4; i++) { + core_info_list[i].arch_id = spine_core_arch_id::core_arch_a60; + } + } + + // special for qemu + if (core_info_list.size() == 0) { + char * spine_core_arch_str = getenv("SPACEMIT_CORE_ARCH"); + if (spine_core_arch_str != nullptr) { + auto arch_id = hex_string_to_u16(spine_core_arch_str); + for (int i = 0; i < num_cores; i++) { + auto core_info = spine_core_info(); + core_info.core_id = i; + core_info.arch_id = spine_core_arch_id{ arch_id }; + core_info_list.push_back(core_info); + } + } + } + + if (core_info_list.size() == 0) { + throw std::runtime_error( + "Failed to get SPACEMIT_CORE_ARCH from environment or failed to parse it from /proc/cpuinfo"); + } + + char * spine_perfer_core_arch_str = getenv("SPACEMIT_PERFER_CORE_ARCH"); + if (spine_perfer_core_arch_str != nullptr && spine_perfer_core_arch_str != "") { + perfer_core_arch_id = spine_core_arch_id{ hex_string_to_u16(spine_perfer_core_arch_str) }; + } + + char * spine_perfer_core_id_str = getenv("SPACEMIT_PERFER_CORE_ID"); + std::vector perfer_core_id_vec; + if (spine_perfer_core_id_str != nullptr && spine_perfer_core_id_str != "") { + std::string perfer_core_id_str(spine_perfer_core_id_str); + size_t start = 0; + size_t end = 0; + while ((end = perfer_core_id_str.find(',', start)) != std::string::npos) { + std::string core_id_substr = perfer_core_id_str.substr(start, end - start); + perfer_core_id_vec.push_back(std::stoi(core_id_substr)); + start = end + 1; + } + std::string core_id_substr = perfer_core_id_str.substr(start); + perfer_core_id_vec.push_back(std::stoi(core_id_substr)); + } + + perfer_core_ids.reserve(num_cores); + if (perfer_core_arch_id == spine_core_arch_id::core_arch_none) { + for (auto & core_info : core_info_list) { + auto core_arch_id = core_info.arch_id; + auto core_arch_head = (uint16_t) (core_arch_id) >> 12; + if (core_arch_head == 0xA) { + num_perfer_cores++; + perfer_core_arch_id = core_arch_id; + cpu_mask |= (1ULL << core_info.core_id); + perfer_core_ids.push_back(core_info.core_id); + } + } + } else { + for (auto & core_info : core_info_list) { + auto core_arch_id = core_info.arch_id; + if (core_arch_id == perfer_core_arch_id) { + num_perfer_cores++; + cpu_mask |= (1ULL << core_info.core_id); + + auto core_arch_head = (uint16_t) (core_arch_id) >> 12; + if (core_arch_head == 0xA) { + perfer_core_ids.push_back(core_info.core_id); + } + } + } + if (num_perfer_cores == 0) { + GGML_ABORT("can not find core with arch id %x for SPACEMIT_PERFER_CORE_ARCH in core info list\n", + (uint16_t) perfer_core_arch_id); + } + } + + if (perfer_core_id_vec.size() > 0) { + perfer_core_ids.clear(); + cpu_mask = 0; + num_perfer_cores = 0; + for (int core_id : perfer_core_id_vec) { + if (core_id < 0 || core_id >= num_cores) { + GGML_ABORT("invalid core id in SPACEMIT_PERFER_CORE_ID: %d, should be between 0 and %d\n", core_id, + num_cores - 1); + } + auto core_info = core_info_list[core_id]; + auto core_arch_id = core_info.arch_id; + if (core_arch_id == perfer_core_arch_id) { + cpu_mask |= (1ULL << core_id); + perfer_core_ids.push_back(core_id); + } else { + GGML_ABORT( + "core id %d in SPACEMIT_PERFER_CORE_ID has arch id %x which does not match " + "SPACEMIT_PERFER_CORE_ARCH %x\n", + core_id, (uint16_t) core_arch_id, (uint16_t) perfer_core_arch_id); + } + } + std::string perfer_core_id_vec_str; + for (int core_id : perfer_core_id_vec) { + perfer_core_id_vec_str += std::to_string(core_id) + ","; + } + perfer_core_id_vec_str.pop_back(); + GGML_LOG_DEBUG("SPACEMIT_PERFER_CORE_ID is set, perferred core ids: %s\n", perfer_core_id_vec_str.c_str()); + num_perfer_cores = static_cast(perfer_core_id_vec.size()); + } + + use_ime1 = perfer_core_arch_id == spine_core_arch_id::core_arch_a60 || + perfer_core_arch_id == spine_core_arch_id::core_arch_x100; + + use_ime2 = perfer_core_arch_id == spine_core_arch_id::core_arch_a100; + + mem_backend = parse_mem_backend(getenv("SPACEMIT_MEM_BACKEND")); + char * spine_disable_tcm_str = getenv("SPACEMIT_DISABLE_TCM"); + auto user_disable_tcm = spine_disable_tcm_str != nullptr && strcmp(spine_disable_tcm_str, "0") != 0; + + if (!user_disable_tcm) { + spine_mem_pool_tcm_info tcm_info; + if (spine_mem_pool_tcm_init(&tcm_info)) { + use_tcm = tcm_info.available; + tcm_blk_size = tcm_info.blk_size; + GGML_LOG_DEBUG("CPU_RISCV64_SPACEMIT: tcm is available, blk_size: %zu, blk_num: %zu, is_fake_tcm: %d\n", + tcm_info.blk_size, tcm_info.blk_num, tcm_info.is_fake_tcm); + + for (auto & core_info : core_info_list) { + auto core_arch_head = (uint16_t) (core_info.arch_id) >> 12; + if (core_arch_head != 0xA) { + aicpu_id_offset++; + } else { + break; + } + } + } + } + + GGML_LOG_DEBUG( + "CPU_RISCV64_SPACEMIT: num_cores: %d, num_perfer_cores: %d, perfer_core_arch_id: %x, exclude_main_thread: %d, " + "use_ime1: %d, use_ime2: %d, mem_backend: %s, cpu_mask: %lx, aicpu_id_offset: %d\n", + num_cores, num_perfer_cores, (uint16_t) perfer_core_arch_id, exclude_main_thread, use_ime1, use_ime2, + spine_mem_pool_backend_to_string(mem_backend), cpu_mask, aicpu_id_offset); + + const size_t init_barrier_size = sizeof(spine_barrier_t) * spine_init_barrier_count; + init_barrier = + static_cast(spine_mem_pool_shared_mem_alloc(init_barrier_size, alignof(spine_barrier_t))); + if (init_barrier != nullptr) { + init_barrier_is_shared_mem = true; + } else { + GGML_LOG_WARN("CPU_RISCV64_SPACEMIT: failed to allocate init_barrier from shared mem, falling back to heap\n", + __func__); + init_barrier = new spine_barrier_t[spine_init_barrier_count]; + } + + spine_barrier_init(init_barrier, spine_init_barrier_count, 2); +} + +spine_env_info::~spine_env_info() { + if (init_barrier_is_shared_mem) { + spine_mem_pool_shared_mem_free(init_barrier); + } else { + delete[] init_barrier; + } + + init_barrier = nullptr; + init_barrier_is_shared_mem = false; +} + +spine_env_info global_spine_env_info; + +} // namespace ggml::cpu::riscv64_spacemit diff --git a/ggml/src/ggml-cpu/spacemit/ime_env.h b/ggml/src/ggml-cpu/spacemit/ime_env.h new file mode 100644 index 0000000..a6ca06d --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/ime_env.h @@ -0,0 +1,55 @@ +#pragma once + +#include "spine_barrier.h" +#include "spine_mem_pool.h" + +#include +#include +#include + +namespace ggml::cpu::riscv64_spacemit { + +constexpr uint64_t spine_invalid_core_id = 0xFFFFFFFF; +constexpr size_t spine_init_barrier_count = 16; + +enum class spine_core_arch_id : uint16_t { + core_arch_none = 0, + core_arch_x60 = 0x503C, + core_arch_x100 = 0x5064, + core_arch_x200 = 0x50C8, + core_arch_a60 = 0xA03C, + core_arch_a100 = 0xA064, + core_arch_a200 = 0xA0C8, +}; + +struct spine_core_info { + uint64_t core_id{ spine_invalid_core_id }; + spine_core_arch_id arch_id{ spine_core_arch_id::core_arch_none }; + + static bool get_spine_core_info(std::vector & result); +}; + +struct spine_env_info { + std::vector core_info_list; + std::vector perfer_core_ids; + int aicpu_id_offset{ 0 }; + int num_cores{ 0 }; + int num_perfer_cores{ 0 }; + spine_core_arch_id perfer_core_arch_id{ spine_core_arch_id::core_arch_none }; + bool exclude_main_thread{ false }; + bool use_ime2{ false }; + bool use_ime1{ false }; + bool use_tcm{ false }; + spine_mem_pool_backend mem_backend{ spine_mem_pool_backend::transparent_hugepage }; + uint64_t tcm_blk_size{ 0 }; + uint64_t cpu_mask{ 0 }; + spine_barrier_t * init_barrier{ nullptr }; + bool init_barrier_is_shared_mem{ false }; + + spine_env_info(); + ~spine_env_info(); +}; + +extern spine_env_info global_spine_env_info; + +} // namespace ggml::cpu::riscv64_spacemit diff --git a/ggml/src/ggml-cpu/spacemit/ime_kernels.h b/ggml/src/ggml-cpu/spacemit/ime_kernels.h index 7570634..476a0e7 100644 --- a/ggml/src/ggml-cpu/spacemit/ime_kernels.h +++ b/ggml/src/ggml-cpu/spacemit/ime_kernels.h @@ -1,26 +1,199 @@ #pragma once +#include #include +#include + +namespace spacemit_kernels { + +#define BLOCK_QNK_LEN 256 + +template struct nrow_block_q2_k { + // [4bit scale + 4bit zp] * N * 16 + uint8_t scales[N * BLOCK_QNK_LEN / 16]; + // [b0, b16, b32, b48] [b1, b17, b33, b49] ... [b15, b31, b47, b63] + // [b64, b80, b96, b112] ...[b79, b95, b111, b127] + // [b128, b144, b160, b176] ...[b143, b159, b175, b191] + // [b192, b208, b224, b240] ...[b207, b223, b239, b255] + uint8_t qs[N * BLOCK_QNK_LEN / 4]; + uint16_t scales16[N]; + uint16_t zeros16[N]; +}; + +template struct nrow_block_q3_k { + // [8bit scale] * N * 16 + int8_t scales[N * 16]; + // [b0, b1, b2, b3, b4, b5, b6, b7] ... [b248, b249, b250, b251, b252, b253, b254, b255] + uint8_t hmask[N * BLOCK_QNK_LEN / 8]; + // [b0, b16, b32, b48] [b1, b17, b33, b49] ... [b15, b31, b47, b63] + // [b64, b80, b96, b112] ...[b79, b95, b111, b127] + // [b128, b144, b160, b176] ...[b143, b159, b175, b191] + // [b192, b208, b224, b240] ...[b207, b223, b239, b255] + uint8_t qs[N * BLOCK_QNK_LEN / 4]; + uint16_t scales16[N]; +}; + +template struct nrow_block_mxfp4 { + uint8_t e[N]; + uint8_t qh[4 * N]; + uint8_t qs[16 * N]; +}; + +template struct __attribute__((packed)) nrow_block_q5_1 { + uint16_t scales16[N]; + uint8_t zp[N]; + // n0 [bh0, bh1, bh2, bh3, bh4, bh5, bh6, bh7] .... + uint8_t qh[4 * N]; + // n0 [b0, b1], [b2, b3] .... [b30, b31] + // n1 [b0, b1], [b2, b3] .... [b30, b31] + uint8_t qs[16 * N]; +}; + +static_assert(sizeof(nrow_block_q5_1<1>) == sizeof(uint8_t) + 22, "wrong nrow_block_q5_1 block size/padding"); + +template struct __attribute__((packed)) nrow_block_q5_0 { + uint16_t scales16[N]; + // n0 [bh0, bh1, bh2, bh3, bh4, bh5, bh6, bh7] .... + uint8_t qh[4 * N]; + // n0 [b0, b1], [b2, b3] .... [b30, b31] + // n1 [b0, b1], [b2, b3] .... [b30, b31] + uint8_t qs[16 * N]; +}; + +static_assert(sizeof(nrow_block_q5_0<1>) == 22, "wrong nrow_block_q5_0 block size/padding"); + +using gemm_kernel_quantize_def = std::function< + size_t(size_t, const uint8_t *, const uint8_t *, const uint8_t *, float *, size_t, size_t, size_t, size_t)>; + +using moe_gemm_kernel_quantize_def = std::function< + size_t(size_t, const uint8_t **, const uint8_t *, const uint8_t *, float **, size_t, size_t, size_t, size_t)>; -namespace sqnbitgemm_spacemit_ime { namespace ime1 { -size_t gemm_kernel_i8i4(size_t blk_len, - const std::byte * quant_a_ptr, - const std::byte * quant_b_data, - const float * quant_b_scale, - const std::byte * quant_b_zp, - float * c_ptr, - size_t count_m, - size_t count_n, - size_t count_k, - size_t block_count_k, - size_t ldc, - const float * bias, - const size_t scale_stride); - -void quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, std::byte * quant_a_ptr); - -void quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, std::byte * quant_a_ptr); +size_t gemm_kernel_i8i4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t gemm_kernel_i8i8(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +void quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr); + +void quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr); } // namespace ime1 -} // namespace sqnbitgemm_spacemit_ime + +namespace ime2 { +size_t gemm_kernel_i8i2k(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t gemm_kernel_i8i3k(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t gemm_kernel_i8i4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t gemm_kernel_i8i4_hp(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t moe_m2_gemm_kernel_i8i4(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t gemm_kernel_i8i8(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t gemm_kernel_i8mxfp4(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t moe_m2_gemm_kernel_i8mxfp4(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t gemm_kernel_i8i5(size_t blk_len, + const uint8_t * quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float * c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); + +size_t moe_m2_gemm_kernel_i8i5(size_t blk_len, + const uint8_t ** quant_a_ptr, + const uint8_t * quant_b_data, + const uint8_t * quant_b_zp, + float ** c_ptr, + size_t count_m, + size_t count_n, + size_t k_blks, + size_t ldc); +} // namespace ime2 +} // namespace spacemit_kernels diff --git a/ggml/src/ggml-cpu/spacemit/repack.cpp b/ggml/src/ggml-cpu/spacemit/repack.cpp new file mode 100644 index 0000000..2039056 --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/repack.cpp @@ -0,0 +1,1867 @@ +#define GGML_COMMON_IMPL_CPP +#define GGML_COMMON_DECL_CPP + +#include "repack.h" + +#include "ggml-common.h" +#include "ggml-cpu.h" +#include "ggml-impl.h" +#include "ime_kernels.h" + +#include +#include +#include +#include + +// clang-format off +#if defined(__riscv) + +#if !defined(__riscv_v) || !defined(__riscv_v_intrinsic) +#error "riscv v extension or v_intrinsic not enabled" +#else +#include +#endif + +#if !defined(__riscv_zfh) +#error "riscv zfh extension not enabled" +#endif + +#else +#error "riscv not enabled in this build" +#endif + +#if defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wcast-qual" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +// clang-format on + +template constexpr int QK_0() { + if constexpr (K == 4) { + return QK4_0; + } + if constexpr (K == 8) { + return QK8_0; + } + return -1; +} + +template struct block { + ggml_half d[N]; // deltas for N qK_0 blocks + uint8_t qs[(QK_0() * N * K) / 8]; // quants for N qK_0 blocks +}; + +template struct block_with_zp { + ggml_half d[N]; // deltas for N qK_1 blocks + uint8_t zp[N]; // zero points for N qK_1 blocks + uint8_t qs[(QK_0() * N * K) / 8]; // quants for N qK_1 blocks +}; + +// control size +static_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 8, "wrong block<4,16> size/padding"); +static_assert(sizeof(block_with_zp<4, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 8 + 16 * sizeof(uint8_t), + "wrong block_with_zp<4,16> size/padding"); + +static_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 16, "wrong block<8,16> size/padding"); + +static_assert(sizeof(block<4, 32>) == 32 * sizeof(ggml_half) + QK4_0 * 16, "wrong block<4,32> size/padding"); +static_assert(sizeof(block_with_zp<4, 32>) == 32 * sizeof(ggml_half) + QK4_0 * 16 + 32 * sizeof(uint8_t), + "wrong block_with_zp<4,32> size/padding"); + +using block_q4_0x16 = block<4, 16>; +using block_q4_1x16 = block_with_zp<4, 16>; +using block_q8_0x16 = block<8, 16>; + +using block_q4_0x32 = block<4, 32>; +using block_q4_1x32 = block_with_zp<4, 32>; +using block_q8_0x32 = block<8, 32>; + +struct block_q4_0x32x256 { + block_q4_0x32 blocks[8]; // [f16 * 32 | i4 * 32 * 32] * 8 +}; + +struct block_q4_1x32x256 { + block_q4_0x32 blocks[8]; + uint8_t zps[32 * 8]; +}; + +static block_q4_0x16 make_block_q4_0x16(block_q4_0 * in, unsigned int blck_size_interleave) { + block_q4_0x16 out; + GGML_ASSERT(QK4_0 / blck_size_interleave == 2); + + for (int i = 0; i < 16; i++) { + out.d[i] = in[i].d; + } + + for (int i = 0; i < 16; i++) { + // [0, 15], in.d & 0x0F + for (int j = 0; j < QK4_0 / 4; j++) { + //src [b0 b16] ......... [b8 b24] ......... [b15 b31] + //dst [b0 b8] ......... [b7 b15] + out.qs[i * QK4_0 / 4 + j] = (in[i].qs[j] & 0x0F) | ((in[i].qs[j + QK4_0 / 4] & 0x0F) << 4); + } + } + + for (int i = 0; i < 16; i++) { + // [16, 31], in.d & 0xF0 + for (int j = 0; j < QK4_0 / 4; j++) { + //src [b0 b16] ......... [b8 b24] ......... [b15 b31] + //dst [b16 b24] ......... [b23 b31] + out.qs[4 * QK4_0 + i * QK4_0 / 4 + j] = ((in[i].qs[j] & 0xF0) >> 4) | (in[i].qs[j + QK4_0 / 4] & 0xF0); + } + } + + return out; +} + +static block_q4_1x16 make_block_q4_1x16(block_q4_1 * in, unsigned int blck_size_interleave) { + block_q4_1x16 out; + GGML_ASSERT(QK4_1 / blck_size_interleave == 2); + + for (int i = 0; i < 16; i++) { + float d = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d); + float m = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m); + float mid = -std::nearbyintf(m / d); + mid = std::min(15.0f, std::max(0.0f, mid)); + out.d[i] = GGML_FP32_TO_FP16(d); + out.zp[i] = static_cast(mid); + } + + for (int i = 0; i < 16; i++) { + // [0, 15], in.d & 0x0F + for (int j = 0; j < QK4_1 / 4; j++) { + //src [b0 b16] ......... [b8 b24] ......... [b15 b31] + //dst [b0 b8] ......... [b7 b15] + out.qs[i * QK4_1 / 4 + j] = (in[i].qs[j] & 0x0F) | ((in[i].qs[j + QK4_1 / 4] & 0x0F) << 4); + } + } + + for (int i = 0; i < 16; i++) { + // [16, 31], in.d & 0xF0 + for (int j = 0; j < QK4_1 / 4; j++) { + //src [b0 b16] ......... [b8 b24] ......... [b15 b31] + //dst [b16 b24] ......... [b23 b31] + out.qs[4 * QK4_1 + i * QK4_1 / 4 + j] = ((in[i].qs[j] & 0xF0) >> 4) | (in[i].qs[j + QK4_1 / 4] & 0xF0); + } + } + + return out; +} + +static int repack_q4_0_to_q4_0_16_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_0); + GGML_ASSERT(interleave_block == 16); + + constexpr int nrows_interleaved = 16; + + block_q4_0x16 * dst = (block_q4_0x16 *) t->data; + const block_q4_0 * src = (const block_q4_0 *) data; + block_q4_0 dst_tmp[16]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK4_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_0 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q4_0x16(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static int repack_q4_1_to_q4_1_16_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_1); + GGML_ASSERT(interleave_block == 16); + + constexpr int nrows_interleaved = 16; + + block_q4_1x16 * dst = (block_q4_1x16 *) t->data; + const block_q4_1 * src = (const block_q4_1 *) data; + block_q4_1 dst_tmp[16]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK4_1; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_1)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_1 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q4_1x16(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static inline void get_scale_min_k4(int j, + const uint8_t * GGML_RESTRICT q, + uint8_t * GGML_RESTRICT d, + uint8_t * GGML_RESTRICT m) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j + 4] & 63; + } else { + *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); + } +} + +static int repack_q4_k_to_q4_1_16_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_K); + GGML_ASSERT(interleave_block == 16); + GGML_ASSERT(QK_K / QK4_1 == 8); + + constexpr int nrows_interleaved = 16; + + block_q4_1x16 * dst = (block_q4_1x16 *) t->data; + const block_q4_K * src = (const block_q4_K *) data; + block_q4_1 dst_tmp[16]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_K; + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_K != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int j = 0; j < 8; j++) { + for (int i = 0; i < nrows_interleaved; i++) { + uint8_t sc, m; + const float d = GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d); + const float min = + GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin); + get_scale_min_k4(j, src[x + i * nblocks].scales, &sc, &m); + const float d1 = d * sc; + const float m1 = min * m; + + dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d = GGML_FP32_TO_FP16(d1); + dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m = GGML_FP32_TO_FP16(-m1); + // src -> [b0, b32] [b1, b33] ... [b31, b63] + // dst -> [b0, b16] [b1, b17] ... [b15, b31] [b32, b48] [b33, b49] ... [b47, b63] + const uint8_t * q = src[x + i * nblocks].qs + (j / 2) * QK4_1; + if (j % 2 == 0) { + for (int ii = 0; ii < 16; ii++) { + dst_tmp[i].qs[ii] = (q[ii] & 0x0F) | ((q[ii + 16] & 0x0F) << 4); + } + } else { + for (int ii = 0; ii < 16; ii++) { + dst_tmp[i].qs[ii] = ((q[ii] & 0xF0) >> 4) | (q[ii + 16] & 0xF0); + } + } + } + *dst++ = make_block_q4_1x16(dst_tmp, interleave_block); + } + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static block_q4_0x32 make_block_q4_0x32(block_q4_0 * in, unsigned int blck_size_interleave) { + block_q4_0x32 out; + assert(QK4_0 / blck_size_interleave == 1); + GGML_UNUSED(blck_size_interleave); + + for (int i = 0; i < 32; i++) { + out.d[i] = in[i].d; + } + + for (int i = 0; i < 32; i++) { + // [0, 15], in.d & 0x0F + for (int j = 0; j < QK4_0 / 4; j++) { + //src [b0 b16] ......... [b8 b24] ......... [b15 b31] + //dst [b0 b1] ......... [b14 b15] + out.qs[i * QK4_0 / 2 + j] = (in[i].qs[j * 2] & 0x0F) | ((in[i].qs[j * 2 + 1] & 0x0F) << 4); + } + } + + for (int i = 0; i < 32; i++) { + // [16, 31], in.d & 0xF0 + for (int j = 0; j < QK4_0 / 4; j++) { + //src [b0 b16] ......... [b8 b24] ......... [b15 b31] + //dst [b16 b17] ......... [b30 b31] + out.qs[i * QK4_0 / 2 + QK4_0 / 4 + j] = ((in[i].qs[j * 2] & 0xF0) >> 4) | (in[i].qs[j * 2 + 1] & 0xF0); + } + } + + return out; +} + +static block_q4_1x32 make_block_q4_1x32(block_q4_1 * in, unsigned int blck_size_interleave) { + block_q4_1x32 out; + GGML_ASSERT(QK4_1 / blck_size_interleave == 1); + GGML_UNUSED(blck_size_interleave); + + for (int i = 0; i < 32; i++) { + float d = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d); + float m = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m); + float mid = -std::nearbyintf(m / d); + mid = std::min(15.0f, std::max(0.0f, mid)); + out.d[i] = GGML_FP32_TO_FP16(d); + out.zp[i] = static_cast(mid); + } + + for (int i = 0; i < 32; i++) { + // [0, 15], in.d & 0x0F + for (int j = 0; j < QK4_1 / 4; j++) { + //src [b0 b16] ......... [b8 b24] ......... [b15 b31] + //dst [b0 b1] ......... [b14 b15] + out.qs[i * QK4_1 / 2 + j] = (in[i].qs[j * 2] & 0x0F) | ((in[i].qs[j * 2 + 1] & 0x0F) << 4); + } + } + + for (int i = 0; i < 32; i++) { + // [16, 31], in.d & 0xF0 + for (int j = 0; j < QK4_1 / 4; j++) { + //src [b0 b16] ......... [b8 b24] ......... [b15 b31] + //dst [b16 b24] ......... [b23 b31] + out.qs[i * QK4_1 / 2 + QK4_1 / 4 + j] = ((in[i].qs[j * 2] & 0xF0) >> 4) | (in[i].qs[j * 2 + 1] & 0xF0); + } + } + + return out; +} + +static block_q8_0x32 make_block_q8_0x32(block_q8_0 * in, unsigned int blck_size_interleave) { + block_q8_0x32 out; + GGML_ASSERT(QK8_0 / blck_size_interleave == 1); + GGML_UNUSED(blck_size_interleave); + + for (int i = 0; i < 32; i++) { + out.d[i] = in[i].d; + } + + for (int i = 0; i < 32; i++) { + memcpy(out.qs + i * QK8_0, in[i].qs, QK8_0); + } + + return out; +} + +// IME1: interleave 16 q8_0 rows so a plain vle8 sequence in the i8i8 kernel lands weights in the +// vmadot group/parity layout. Mirrors make_block_q4_0x16 but stores full int8 (no nibble packing). +// qs (512B) = [INNER step 0: reg0..reg7][INNER step 1: reg0..reg7], each reg 32B holding 4 columns +// x (even=K-first-half / odd=K-second-half). Column col -> acc=col/4, cgrp=col%4. +static block_q8_0x16 make_block_q8_0x16(block_q8_0 * in, unsigned int blck_size_interleave) { + block_q8_0x16 out; + GGML_ASSERT(QK8_0 / blck_size_interleave == 2); + GGML_UNUSED(blck_size_interleave); + + for (int i = 0; i < 16; i++) { + out.d[i] = in[i].d; + } + + memset(out.qs, 0, sizeof(out.qs)); + for (int col = 0; col < 16; col++) { + const int acc = col / 4; + const int cgrp = col % 4; + const int8_t * q = in[col].qs; + for (int s = 0; s < 2; s++) { + const int base = s * 16; + uint8_t * reg_lo = out.qs + (s * 8 + acc) * 32; + uint8_t * reg_hi = out.qs + (s * 8 + acc + 4) * 32; + for (int i = 0; i < 4; i++) { + reg_lo[(2 * cgrp) * 4 + i] = (uint8_t) q[base + 0 + i]; + reg_lo[(2 * cgrp + 1) * 4 + i] = (uint8_t) q[base + 4 + i]; + reg_hi[(2 * cgrp) * 4 + i] = (uint8_t) q[base + 8 + i]; + reg_hi[(2 * cgrp + 1) * 4 + i] = (uint8_t) q[base + 12 + i]; + } + } + } + return out; +} + +static int repack_q8_0_to_q8_0_16_bl_ref(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q8_0); + GGML_ASSERT(interleave_block == 16); + + constexpr int nrows_interleaved = 16; + + block_q8_0x16 * dst = (block_q8_0x16 *) t->data; + const block_q8_0 * src = (const block_q8_0 *) data; + block_q8_0 dst_tmp[16]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK8_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q8_0)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK8_0 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q8_0x16(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static int repack_q2_k_to_q2_k_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q2_K); + GGML_ASSERT(interleave_block == 32); + GGML_ASSERT(QK_K == 256); + + constexpr int nrows_interleaved = 32; + + const block_q2_K * src = (const block_q2_K *) data; + + auto * dst = (spacemit_kernels::nrow_block_q2_k<32> *) t->data; + + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_K; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q2_K)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_K != 0) { + return -1; + } + + uint8_t qs_aux[256] = { 0 }; + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + const block_q2_K * src_block = &src[(b + i) * nblocks + x]; + + // scale for [16, N] + for (int j = 0; j < 16; j++) { + auto zp_aux = (dst->scales[j * nrows_interleaved + i]) & 0xF0; + + dst->scales[j * nrows_interleaved + i] = (src_block->scales[j] & 0x0F) | zp_aux; + } + + // zp for [N, 16] + for (int j = 0; j < 16; j++) { + auto scale_aux = (dst->scales[16 * i + j]) & 0x0F; + + dst->scales[16 * i + j] = (src_block->scales[j] & 0xF0) | scale_aux; + } + + for (int k = 0; k < 4; k++) { + for (int j = 0; j < 32; j++) { + qs_aux[k * 32 + j] = (src_block->qs[j] >> (2 * k)) & 0x03; + } + } + + for (int k = 0; k < 4; k++) { + for (int j = 0; j < 32; j++) { + qs_aux[k * 32 + j + 128] = (src_block->qs[j + 32] >> (2 * k)) & 0x03; + } + } + + // from nrows_interleaved * [2 * 32byte] + // to 4 * [nrows_interleaved * 16byte] + for (int k = 0; k < 4; k++) { + for (int j = 0; j < 16; j++) { + uint8_t qs0 = qs_aux[j + k * 64]; + uint8_t qs16 = qs_aux[j + 16 + k * 64]; + uint8_t qs32 = qs_aux[j + 32 + k * 64]; + uint8_t qs48 = qs_aux[j + 48 + k * 64]; + + dst->qs[(k * nrows_interleaved + i) * 16 + j] = + (qs0 & 0x03) | ((qs16 & 0x03) << 2) | ((qs32 & 0x03) << 4) | ((qs48 & 0x03) << 6); + } + } + + dst->scales16[i] = src_block->GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d; + dst->zeros16[i] = src_block->GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin; + } + dst++; + } + } + + return 0; +} + +static int repack_q3_k_to_q3_k_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q3_K); + GGML_ASSERT(interleave_block == 32); + GGML_ASSERT(QK_K == 256); + + constexpr int nrows_interleaved = 32; + + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; + + const block_q3_K * src = (const block_q3_K *) data; + + auto * dst = (spacemit_kernels::nrow_block_q3_k<32> *) t->data; + + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_K; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q3_K)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_K != 0) { + return -1; + } + + uint32_t b_scale_aux[4] = { 0 }; + uint8_t qs_aux[256] = { 0 }; + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + const block_q3_K * src_block = &src[(b + i) * nblocks + x]; + + uint32_t * auxs = b_scale_aux; + int8_t * scale = (int8_t *) auxs; + memcpy(auxs, src_block->scales, 12); + + uint32_t tmp = auxs[2]; + auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); + auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); + auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); + auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); + + for (int j = 0; j < 16; j++) { + dst->scales[j * nrows_interleaved + i] = scale[j] - 32; + } + + for (int k = 0; k < 4; k++) { + for (int j = 0; j < 32; j++) { + qs_aux[k * 32 + j] = (src_block->qs[j] >> (2 * k)) & 0x03; + } + } + + for (int k = 0; k < 4; k++) { + for (int j = 0; j < 32; j++) { + qs_aux[k * 32 + j + 128] = (src_block->qs[j + 32] >> (2 * k)) & 0x03; + } + } + + // from nrows_interleaved * [2 * 32byte] + // to 4 * [nrows_interleaved * 16byte] + for (int k = 0; k < 4; k++) { + for (int j = 0; j < 16; j++) { + uint8_t qs0 = qs_aux[j + k * 64]; + uint8_t qs16 = qs_aux[j + 16 + k * 64]; + uint8_t qs32 = qs_aux[j + 32 + k * 64]; + uint8_t qs48 = qs_aux[j + 48 + k * 64]; + + dst->qs[(k * nrows_interleaved + i) * 16 + j] = + (qs0 & 0x03) | ((qs16 & 0x03) << 2) | ((qs32 & 0x03) << 4) | ((qs48 & 0x03) << 6); + } + } + + //memcpy(dst->hmask + i * 32, src_block->hmask, 32); + + // from nrows_interleaved * [32byte] + // to 16 * [nrows_interleaved * uint16_t] + uint16_t * dst_mask = ((uint16_t *) dst->hmask) + i; + for (int j = 0; j < 16; j++, dst_mask += nrows_interleaved) { + uint8_t b_shift = j / 2; + uint8_t * b_mask_col = (uint8_t *) (src_block->hmask + (j % 2) * 16); + // b0 - b15 + uint16_t msk_out_0 = 0; + + for (int k = 0; k < 8; k++) { + msk_out_0 |= (uint16_t) ((b_mask_col[k] >> b_shift) & 0x01) << k; + } + for (int k = 8; k < 16; k++) { + msk_out_0 |= (uint16_t) ((b_mask_col[k] >> b_shift) & 0x01) << k; + } + + dst_mask[0] = msk_out_0; + } + + dst->scales16[i] = src_block->d; + } + + dst++; + } + } + + return 0; +} + +static int repack_q4_0_to_q4_0_32_bl_ref(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_0); + GGML_ASSERT(interleave_block == 32); // unused + + constexpr int nrows_interleaved = 32; + + block_q4_0x32 * dst = (block_q4_0x32 *) t->data; + const block_q4_0 * src = (const block_q4_0 *) data; + block_q4_0 dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK4_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_0 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q4_0x32(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static int repack_q4_0_to_q4_0_256_32_bl_ref(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_0); + GGML_ASSERT(interleave_block == 32); // unused + + constexpr int nrows_interleaved = 32; + + block_q4_0x32x256 * dst = (block_q4_0x32x256 *) t->data; + const block_q4_0 * src = (const block_q4_0 *) data; + block_q4_0 dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK4_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0)); + GGML_ASSERT(nblocks % 8 == 0); // for 256-block interleaving + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_0 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x += 8) { + for (int j = 0; j < 8; j++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + j + i * nblocks]; + } + dst->blocks[j] = make_block_q4_0x32(dst_tmp, interleave_block); + } + dst++; + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static int repack_q4_0_to_q4_1_256_32_bl_ref(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_1); + GGML_ASSERT(interleave_block == 32); // unused + + constexpr int nrows_interleaved = 32; + + block_q4_1x32x256 * dst = (block_q4_1x32x256 *) t->data; + const block_q4_1 * src = (const block_q4_1 *) data; + block_q4_1 dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK4_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_1)); + GGML_ASSERT(nblocks % 8 == 0); // for 256-block interleaving + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_0 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x += 8) { + for (int j = 0; j < 8; j++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + j + i * nblocks]; + } + + block_q4_0x32 * dst_block = &dst->blocks[j]; + uint8_t * dst_zp = dst->zps + j * nrows_interleaved; + + for (int i = 0; i < nrows_interleaved; i++) { + float d = GGML_FP16_TO_FP32(dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d); + float m = GGML_FP16_TO_FP32(dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m); + float mid = -std::nearbyintf(m / d); + mid = std::min(15.0f, std::max(0.0f, mid)); + + dst_block->d[i] = GGML_FP32_TO_FP16(d); + dst_zp[i] = static_cast(mid); + } + + for (int i = 0; i < nrows_interleaved; i++) { + for (int k = 0; k < QK4_1 / 4; k++) { + dst_block->qs[i * QK4_1 / 2 + k] = + (dst_tmp[i].qs[k * 2] & 0x0F) | ((dst_tmp[i].qs[k * 2 + 1] & 0x0F) << 4); + } + } + + for (int i = 0; i < nrows_interleaved; i++) { + for (int k = 0; k < QK4_1 / 4; k++) { + dst_block->qs[i * QK4_1 / 2 + QK4_1 / 4 + k] = + ((dst_tmp[i].qs[k * 2] & 0xF0) >> 4) | (dst_tmp[i].qs[k * 2 + 1] & 0xF0); + } + } + } + dst++; + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +// RVV optimized version of repack_q4_0_to_q4_0_32_bl +// Eliminates the intermediate dst_tmp buffer and vectorizes nibble repack. +static int repack_q4_0_to_q4_0_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_0); + GGML_ASSERT(interleave_block == 32); + + constexpr int nrows_interleaved = 32; + constexpr int qs_bytes = QK4_0 / 2; // 16 + + block_q4_0x32 * dst = (block_q4_0x32 *) t->data; + const block_q4_0 * src = (const block_q4_0 *) data; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK4_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_0 != 0) { + return -1; + } + + const ptrdiff_t row_stride = (ptrdiff_t) nblocks * sizeof(block_q4_0); + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + const block_q4_0 * col_src = src + x; + + // --- 1) Gather 32 scale values (ggml_half d) with stride load --- + // d is at offset 0 of each block_q4_0, stride between rows = row_stride + { + const uint8_t * d_base = (const uint8_t *) &col_src->d; + ggml_half * d_dst = dst->d; + size_t remaining = 32; + size_t offset = 0; + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e16m1(remaining); + vuint16m1_t vd = + __riscv_vlse16_v_u16m1((const uint16_t *) (d_base + offset * row_stride), row_stride, vl); + __riscv_vse16_v_u16m1((uint16_t *) (d_dst + offset), vd, vl); + offset += vl; + remaining -= vl; + } + } + + // --- 2) Nibble repack qs for each of the 32 rows --- + // For each row i: + // src qs[16]: [b0|b16] [b1|b17] ... [b15|b31] (lo nibble = b_j, hi nibble = b_{j+16}) + // dst qs low 8B: (qs[2j] & 0x0F) | ((qs[2j+1] & 0x0F) << 4) for j=0..7 + // dst qs high 8B: ((qs[2j] >> 4)) | (qs[2j+1] & 0xF0) for j=0..7 + { + const size_t vl8 = __riscv_vsetvl_e8m1(8); + for (int i = 0; i < 32; i++) { + const uint8_t * sq = col_src[i * nblocks].qs; + uint8_t * dq = dst->qs + i * qs_bytes; + + // stride-2 load to separate even/odd bytes + vuint8m1_t v_even = __riscv_vlse8_v_u8m1(sq, 2, vl8); // qs[0], qs[2], ..., qs[14] + vuint8m1_t v_odd = __riscv_vlse8_v_u8m1(sq + 1, 2, vl8); // qs[1], qs[3], ..., qs[15] + + // low nibble part: (even & 0x0F) | ((odd & 0x0F) << 4) + vuint8m1_t v_even_lo = __riscv_vand_vx_u8m1(v_even, 0x0F, vl8); + vuint8m1_t v_odd_lo = __riscv_vand_vx_u8m1(v_odd, 0x0F, vl8); + vuint8m1_t v_lo = __riscv_vor_vv_u8m1(v_even_lo, __riscv_vsll_vx_u8m1(v_odd_lo, 4, vl8), vl8); + + // high nibble part: (even >> 4) | (odd & 0xF0) + vuint8m1_t v_even_hi = __riscv_vsrl_vx_u8m1(v_even, 4, vl8); + vuint8m1_t v_odd_hi = __riscv_vand_vx_u8m1(v_odd, 0xF0, vl8); + vuint8m1_t v_hi = __riscv_vor_vv_u8m1(v_even_hi, v_odd_hi, vl8); + + __riscv_vse8_v_u8m1(dq, v_lo, vl8); + __riscv_vse8_v_u8m1(dq + 8, v_hi, vl8); + } + } + + dst++; + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static int repack_q4_1_to_q4_1_32_bl_ref(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_1); + GGML_ASSERT(interleave_block == 32); // unused + + constexpr int nrows_interleaved = 32; + + block_q4_1x32 * dst = (block_q4_1x32 *) t->data; + const block_q4_1 * src = (const block_q4_1 *) data; + block_q4_1 dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK4_1; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_1)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_1 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + dst_tmp[i] = src[x + i * nblocks]; + } + *dst++ = make_block_q4_1x32(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +// RVV optimized version of repack_q4_1_to_q4_1_32_bl +// Eliminates the intermediate dst_tmp buffer and vectorizes nibble repack + zp computation. +static int repack_q4_1_to_q4_1_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_1); + GGML_ASSERT(interleave_block == 32); + + constexpr int nrows_interleaved = 32; + constexpr int qs_bytes = QK4_1 / 2; // 16 + + block_q4_1x32 * dst = (block_q4_1x32 *) t->data; + const block_q4_1 * src = (const block_q4_1 *) data; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK4_1; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_1)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_1 != 0) { + return -1; + } + + const ptrdiff_t row_stride = (ptrdiff_t) nblocks * sizeof(block_q4_1); + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + const block_q4_1 * col_src = src + x; + + // --- 1) Gather d and m, compute zp = clamp(nearbyint(-m/d), 0, 15) --- + // block_q4_1 layout: [d(f16), m(f16), qs[16]] + // d is at byte offset 0, m is at byte offset 2 from each block start + { + const uint8_t * dm_base = (const uint8_t *) &col_src->GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d; + ggml_half * d_dst = dst->d; + uint8_t * zp_dst = dst->zp; + size_t remaining = 32; + size_t offset = 0; + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e16m1(remaining); + + // stride load d (f16) from each row + vuint16m1_t vd_raw = + __riscv_vlse16_v_u16m1((const uint16_t *) (dm_base + offset * row_stride), row_stride, vl); + __riscv_vse16_v_u16m1((uint16_t *) (d_dst + offset), vd_raw, vl); + + // stride load m (f16) from each row (offset +2 bytes from d) + vuint16m1_t vm_raw = + __riscv_vlse16_v_u16m1((const uint16_t *) (dm_base + 2 + offset * row_stride), row_stride, vl); + + // convert to f32 for zp computation: zp = nearbyint(-m / d) + vfloat16m1_t vd_f16 = __riscv_vreinterpret_v_u16m1_f16m1(vd_raw); + vfloat16m1_t vm_f16 = __riscv_vreinterpret_v_u16m1_f16m1(vm_raw); + + // -m / d in f16 directly (SpaceMIT X60 supports f16 arithmetic) + vfloat16m1_t v_neg_m = __riscv_vfneg_v_f16m1(vm_f16, vl); + vfloat16m1_t v_ratio = __riscv_vfdiv_vv_f16m1(v_neg_m, vd_f16, vl); + + // Convert to f32 for nearbyint, then clamp + vfloat32m2_t v_ratio_f32 = __riscv_vfwcvt_f_f_v_f32m2(v_ratio, vl); + + // Use integer rounding: convert f32 -> int (rounds to nearest) + vint32m2_t v_zp_i32 = __riscv_vfcvt_x_f_v_i32m2(v_ratio_f32, vl); + + // clamp to [0, 15] + v_zp_i32 = __riscv_vmax_vx_i32m2(v_zp_i32, 0, vl); + v_zp_i32 = __riscv_vmin_vx_i32m2(v_zp_i32, 15, vl); + + // narrow i32 -> u8 + vint16m1_t v_zp_i16 = __riscv_vncvt_x_x_w_i16m1(v_zp_i32, vl); + vint8mf2_t v_zp_i8 = __riscv_vncvt_x_x_w_i8mf2(v_zp_i16, vl); + vuint8mf2_t v_zp_u8 = __riscv_vreinterpret_v_i8mf2_u8mf2(v_zp_i8); + __riscv_vse8_v_u8mf2(zp_dst + offset, v_zp_u8, vl); + + offset += vl; + remaining -= vl; + } + } + + // --- 2) Nibble repack qs for each of the 32 rows --- + { + const size_t vl8 = __riscv_vsetvl_e8m1(8); + for (int i = 0; i < 32; i++) { + const uint8_t * sq = col_src[i * nblocks].qs; + uint8_t * dq = dst->qs + i * qs_bytes; + + // stride-2 load to separate even/odd bytes + vuint8m1_t v_even = __riscv_vlse8_v_u8m1(sq, 2, vl8); + vuint8m1_t v_odd = __riscv_vlse8_v_u8m1(sq + 1, 2, vl8); + + // low nibble part: (even & 0x0F) | ((odd & 0x0F) << 4) + vuint8m1_t v_even_lo = __riscv_vand_vx_u8m1(v_even, 0x0F, vl8); + vuint8m1_t v_odd_lo = __riscv_vand_vx_u8m1(v_odd, 0x0F, vl8); + vuint8m1_t v_lo = __riscv_vor_vv_u8m1(v_even_lo, __riscv_vsll_vx_u8m1(v_odd_lo, 4, vl8), vl8); + + // high nibble part: (even >> 4) | (odd & 0xF0) + vuint8m1_t v_even_hi = __riscv_vsrl_vx_u8m1(v_even, 4, vl8); + vuint8m1_t v_odd_hi = __riscv_vand_vx_u8m1(v_odd, 0xF0, vl8); + vuint8m1_t v_hi = __riscv_vor_vv_u8m1(v_even_hi, v_odd_hi, vl8); + + __riscv_vse8_v_u8m1(dq, v_lo, vl8); + __riscv_vse8_v_u8m1(dq + 8, v_hi, vl8); + } + } + + dst++; + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static int repack_q4_k_to_q4_1_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q4_K); + GGML_ASSERT(interleave_block == 32); + GGML_ASSERT(QK_K / QK4_1 == 8); + + constexpr int nrows_interleaved = 32; + + block_q4_1x32 * dst = (block_q4_1x32 *) t->data; + const block_q4_K * src = (const block_q4_K *) data; + block_q4_1 dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_K; + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_K != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int j = 0; j < 8; j++) { + for (int i = 0; i < nrows_interleaved; i++) { + uint8_t sc, m; + const float d = GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d); + const float min = + GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin); + get_scale_min_k4(j, src[x + i * nblocks].scales, &sc, &m); + const float d1 = d * sc; + const float m1 = min * m; + + dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d = GGML_FP32_TO_FP16(d1); + dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m = GGML_FP32_TO_FP16(-m1); + // src -> [b0, b32] [b1, b33] ... [b31, b63] + // dst -> [b0, b16] [b1, b17] ... [b15, b31] [b32, b48] [b33, b49] ... [b47, b63] + const uint8_t * q = src[x + i * nblocks].qs + (j / 2) * QK4_1; + if (j % 2 == 0) { + for (int ii = 0; ii < 16; ii++) { + dst_tmp[i].qs[ii] = (q[ii] & 0x0F) | ((q[ii + 16] & 0x0F) << 4); + } + } else { + for (int ii = 0; ii < 16; ii++) { + dst_tmp[i].qs[ii] = ((q[ii] & 0xF0) >> 4) | (q[ii + 16] & 0xF0); + } + } + } + *dst++ = make_block_q4_1x32(dst_tmp, interleave_block); + } + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static int repack_q6_k_to_q8_0_32_bl_ref(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q6_K); + GGML_ASSERT(interleave_block == 32); + GGML_ASSERT(QK_K / QK4_1 == 8); + + constexpr int nrows_interleaved = 32; + + block_q8_0x32 * dst = (block_q8_0x32 *) t->data; + const block_q6_K * src = (const block_q6_K *) data; + block_q8_0 dst_tmp[32]; + int8_t aux8[QK4_1]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_K; + + if (t->ne[0] % QK_K != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + int64_t nrow_real = std::min((int64_t) nrow - b, (int64_t) nrows_interleaved); + for (int64_t x = 0; x < nblocks; x++) { + for (int bi = 0; bi < 8; bi++) { + int i = 0; + for (; i < nrow_real; i++) { + const uint8_t * q4 = src[x + i * nblocks].ql; + const uint8_t * qh = src[x + i * nblocks].qh; + const int8_t * scales = src[x + i * nblocks].scales; + float d = GGML_FP16_TO_FP32(src[x + i * nblocks].d); + + q4 += 64 * (bi / 4); + qh += 32 * (bi / 4); + int8_t * GGML_RESTRICT a = aux8; + + int8_t bi_idx = bi % 4; + + if (bi_idx == 0) { + for (int l = 0; l < 32; ++l) { + a[l] = (int8_t) ((q4[l] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32; + } + } else if (bi_idx == 1) { + for (int l = 0; l < 32; ++l) { + a[l] = (int8_t) ((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32; + } + } else if (bi_idx == 2) { + for (int l = 0; l < 32; ++l) { + a[l] = (int8_t) ((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32; + } + } else if (bi_idx == 3) { + for (int l = 0; l < 32; ++l) { + a[l] = (int8_t) ((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32; + } + } + a = aux8; + + float a_max_abs = 0.0f; + float scale_0 = scales[bi * 2 + 0] * d; + float scale_1 = scales[bi * 2 + 1] * d; + for (int l = 0; l < 16; ++l) { + a_max_abs = std::max(a_max_abs, std::abs(a[l] * scale_0)); + } + + for (int l = 16; l < 32; ++l) { + a_max_abs = std::max(a_max_abs, std::abs(a[l] * scale_1)); + } + + float reflect_scale = a_max_abs / ((1 << 7) - 1); + float reflect_scale_0 = scale_0 / reflect_scale; + float reflect_scale_1 = scale_1 / reflect_scale; + + for (int l = 0; l < 16; ++l) { + float a_temp = std::clamp(std::nearbyintf(a[l] * reflect_scale_0), -128.0f, 127.0f); + a[l] = (int8_t) (a_temp); + } + + for (int l = 16; l < 32; ++l) { + float a_temp = std::clamp(std::nearbyintf(a[l] * reflect_scale_1), -128.0f, 127.0f); + a[l] = (int8_t) (a_temp); + } + + dst_tmp[i].d = GGML_FP32_TO_FP16(reflect_scale); + + memcpy(dst_tmp[i].qs, a, 32 * sizeof(int8_t)); + } + + for (; i < nrows_interleaved; i++) { + memset(&dst_tmp[i], 0, sizeof(block_q8_0)); + } + + *dst++ = make_block_q8_0x32(dst_tmp, interleave_block); + } + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +// RVV optimized version of repack_q6_k_to_q8_0_32_bl +// Vectorizes the Q6_K dequant -> requant pipeline using RVV intrinsics. +// For each sub-block (bi), dequant 32 Q6_K values to int6 -> apply two sub-block scales -> +// find max abs -> compute reflect_scale -> requant to int8 -> gather d with stride load. +static int repack_q6_k_to_q8_0_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q6_K); + GGML_ASSERT(interleave_block == 32); + GGML_ASSERT(QK_K / QK4_1 == 8); + + constexpr int nrows_interleaved = 32; + + block_q8_0x32 * dst = (block_q8_0x32 *) t->data; + const block_q6_K * src = (const block_q6_K *) data; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_K; + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_K != 0) { + return -1; + } + + const ptrdiff_t row_stride = (ptrdiff_t) nblocks * sizeof(block_q6_K); + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int bi = 0; bi < 8; bi++) { + // --- 1) Gather 32 d values with stride load --- + // We need to compute reflect_scale per row first, so gather d later. + // Process each row: dequant Q6_K sub-block -> requant to Q8_0 + for (int i = 0; i < nrows_interleaved; i++) { + const block_q6_K * src_blk = &src[x + i * nblocks]; + const uint8_t * q4 = src_blk->ql + 64 * (bi / 4); + const uint8_t * qh = src_blk->qh + 32 * (bi / 4); + const int8_t * scales = src_blk->scales; + float d = GGML_FP16_TO_FP32(src_blk->d); + + int8_t bi_idx = bi % 4; + + // --- Dequant 32 Q6_K values to int6 (range [-32, 31]) using RVV --- + // vl = 32 for e8m2 (VLEN=256) or loop for smaller VLEN + const size_t vl16 = __riscv_vsetvl_e8m1(16); + + vint8m1_t va_lo, va_hi; // 16 elements each + + if (bi_idx == 0) { + // a[l] = (q4[l] & 0xF) | (((qh[l] >> 0) & 3) << 4) - 32 + vuint8m1_t vq4_lo = __riscv_vle8_v_u8m1(q4, vl16); + vuint8m1_t vq4_hi = __riscv_vle8_v_u8m1(q4 + 16, vl16); + vuint8m1_t vqh_lo = __riscv_vle8_v_u8m1(qh, vl16); + vuint8m1_t vqh_hi = __riscv_vle8_v_u8m1(qh + 16, vl16); + + vuint8m1_t vlo4_lo = __riscv_vand_vx_u8m1(vq4_lo, 0x0F, vl16); + vuint8m1_t vlo4_hi = __riscv_vand_vx_u8m1(vq4_hi, 0x0F, vl16); + vuint8m1_t vh_lo = __riscv_vsll_vx_u8m1(__riscv_vand_vx_u8m1(vqh_lo, 0x03, vl16), 4, vl16); + vuint8m1_t vh_hi = __riscv_vsll_vx_u8m1(__riscv_vand_vx_u8m1(vqh_hi, 0x03, vl16), 4, vl16); + + vuint8m1_t vcomb_lo = __riscv_vor_vv_u8m1(vlo4_lo, vh_lo, vl16); + vuint8m1_t vcomb_hi = __riscv_vor_vv_u8m1(vlo4_hi, vh_hi, vl16); + + va_lo = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(vcomb_lo), 32, vl16); + va_hi = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(vcomb_hi), 32, vl16); + } else if (bi_idx == 1) { + // a[l] = (q4[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4) - 32 + vuint8m1_t vq4_lo = __riscv_vle8_v_u8m1(q4 + 32, vl16); + vuint8m1_t vq4_hi = __riscv_vle8_v_u8m1(q4 + 48, vl16); + vuint8m1_t vqh_lo = __riscv_vle8_v_u8m1(qh, vl16); + vuint8m1_t vqh_hi = __riscv_vle8_v_u8m1(qh + 16, vl16); + + vuint8m1_t vlo4_lo = __riscv_vand_vx_u8m1(vq4_lo, 0x0F, vl16); + vuint8m1_t vlo4_hi = __riscv_vand_vx_u8m1(vq4_hi, 0x0F, vl16); + vuint8m1_t vh_lo = __riscv_vsll_vx_u8m1( + __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vqh_lo, 2, vl16), 0x03, vl16), 4, vl16); + vuint8m1_t vh_hi = __riscv_vsll_vx_u8m1( + __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vqh_hi, 2, vl16), 0x03, vl16), 4, vl16); + + vuint8m1_t vcomb_lo = __riscv_vor_vv_u8m1(vlo4_lo, vh_lo, vl16); + vuint8m1_t vcomb_hi = __riscv_vor_vv_u8m1(vlo4_hi, vh_hi, vl16); + + va_lo = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(vcomb_lo), 32, vl16); + va_hi = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(vcomb_hi), 32, vl16); + } else if (bi_idx == 2) { + // a[l] = (q4[l] >> 4) | (((qh[l] >> 4) & 3) << 4) - 32 + vuint8m1_t vq4_lo = __riscv_vle8_v_u8m1(q4, vl16); + vuint8m1_t vq4_hi = __riscv_vle8_v_u8m1(q4 + 16, vl16); + vuint8m1_t vqh_lo = __riscv_vle8_v_u8m1(qh, vl16); + vuint8m1_t vqh_hi = __riscv_vle8_v_u8m1(qh + 16, vl16); + + vuint8m1_t vhi4_lo = __riscv_vsrl_vx_u8m1(vq4_lo, 4, vl16); + vuint8m1_t vhi4_hi = __riscv_vsrl_vx_u8m1(vq4_hi, 4, vl16); + vuint8m1_t vh_lo = __riscv_vsll_vx_u8m1( + __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vqh_lo, 4, vl16), 0x03, vl16), 4, vl16); + vuint8m1_t vh_hi = __riscv_vsll_vx_u8m1( + __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vqh_hi, 4, vl16), 0x03, vl16), 4, vl16); + + vuint8m1_t vcomb_lo = __riscv_vor_vv_u8m1(vhi4_lo, vh_lo, vl16); + vuint8m1_t vcomb_hi = __riscv_vor_vv_u8m1(vhi4_hi, vh_hi, vl16); + + va_lo = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(vcomb_lo), 32, vl16); + va_hi = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(vcomb_hi), 32, vl16); + } else { // bi_idx == 3 + // a[l] = (q4[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4) - 32 + vuint8m1_t vq4_lo = __riscv_vle8_v_u8m1(q4 + 32, vl16); + vuint8m1_t vq4_hi = __riscv_vle8_v_u8m1(q4 + 48, vl16); + vuint8m1_t vqh_lo = __riscv_vle8_v_u8m1(qh, vl16); + vuint8m1_t vqh_hi = __riscv_vle8_v_u8m1(qh + 16, vl16); + + vuint8m1_t vhi4_lo = __riscv_vsrl_vx_u8m1(vq4_lo, 4, vl16); + vuint8m1_t vhi4_hi = __riscv_vsrl_vx_u8m1(vq4_hi, 4, vl16); + vuint8m1_t vh_lo = __riscv_vsll_vx_u8m1( + __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vqh_lo, 6, vl16), 0x03, vl16), 4, vl16); + vuint8m1_t vh_hi = __riscv_vsll_vx_u8m1( + __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vqh_hi, 6, vl16), 0x03, vl16), 4, vl16); + + vuint8m1_t vcomb_lo = __riscv_vor_vv_u8m1(vhi4_lo, vh_lo, vl16); + vuint8m1_t vcomb_hi = __riscv_vor_vv_u8m1(vhi4_hi, vh_hi, vl16); + + va_lo = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(vcomb_lo), 32, vl16); + va_hi = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(vcomb_hi), 32, vl16); + } + + // --- Widen to i16 for scaled abs computation --- + float scale_0 = scales[bi * 2 + 0] * d; + float scale_1 = scales[bi * 2 + 1] * d; + + // Widen i8 -> i16 -> f32 for abs*scale computation + vint16m2_t va_lo_w = __riscv_vsext_vf2_i16m2(va_lo, vl16); + vint16m2_t va_hi_w = __riscv_vsext_vf2_i16m2(va_hi, vl16); + + // Compute |a[l] * scale_0| for lo half, |a[l] * scale_1| for hi half + vfloat32m4_t vf_lo = __riscv_vfcvt_f_x_v_f32m4(__riscv_vsext_vf2_i32m4(va_lo_w, vl16), vl16); + vfloat32m4_t vf_hi = __riscv_vfcvt_f_x_v_f32m4(__riscv_vsext_vf2_i32m4(va_hi_w, vl16), vl16); + + vfloat32m4_t vabs_lo = __riscv_vfabs_v_f32m4(__riscv_vfmul_vf_f32m4(vf_lo, scale_0, vl16), vl16); + vfloat32m4_t vabs_hi = __riscv_vfabs_v_f32m4(__riscv_vfmul_vf_f32m4(vf_hi, scale_1, vl16), vl16); + + // Find max abs across both halves + vfloat32m4_t vabs_max = __riscv_vfmax_vv_f32m4(vabs_lo, vabs_hi, vl16); + + // Reduce to scalar max + vfloat32m1_t vzero = __riscv_vfmv_v_f_f32m1(0.0f, 1); + vfloat32m1_t vmax_red = __riscv_vfredmax_vs_f32m4_f32m1(vabs_max, vzero, vl16); + float a_max_abs = __riscv_vfmv_f_s_f32m1_f32(vmax_red); + + float reflect_scale = a_max_abs / 127.0f; + float reflect_scale_0 = scale_0 / reflect_scale; + float reflect_scale_1 = scale_1 / reflect_scale; + + // --- Requant: a[l] = clamp(nearbyint(a[l] * reflect_scale_x), -128, 127) --- + vfloat32m4_t vscaled_lo = __riscv_vfmul_vf_f32m4(vf_lo, reflect_scale_0, vl16); + vfloat32m4_t vscaled_hi = __riscv_vfmul_vf_f32m4(vf_hi, reflect_scale_1, vl16); + + // fcvt.x rounds to nearest (using current rounding mode) + vint32m4_t vi_lo = __riscv_vfcvt_x_f_v_i32m4(vscaled_lo, vl16); + vint32m4_t vi_hi = __riscv_vfcvt_x_f_v_i32m4(vscaled_hi, vl16); + + // Clamp to [-128, 127] + vi_lo = __riscv_vmax_vx_i32m4(vi_lo, -128, vl16); + vi_lo = __riscv_vmin_vx_i32m4(vi_lo, 127, vl16); + vi_hi = __riscv_vmax_vx_i32m4(vi_hi, -128, vl16); + vi_hi = __riscv_vmin_vx_i32m4(vi_hi, 127, vl16); + + // Narrow i32 -> i16 -> i8 + vint16m2_t vi16_lo = __riscv_vncvt_x_x_w_i16m2(vi_lo, vl16); + vint16m2_t vi16_hi = __riscv_vncvt_x_x_w_i16m2(vi_hi, vl16); + vint8m1_t vi8_lo = __riscv_vncvt_x_x_w_i8m1(vi16_lo, vl16); + vint8m1_t vi8_hi = __riscv_vncvt_x_x_w_i8m1(vi16_hi, vl16); + + // Store d and qs directly into dst block + dst->d[i] = GGML_FP32_TO_FP16(reflect_scale); + int8_t * dq = (int8_t *) dst->qs + i * QK8_0; + __riscv_vse8_v_i8m1(dq, vi8_lo, vl16); + __riscv_vse8_v_i8m1(dq + 16, vi8_hi, vl16); + } + dst++; + } + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static int repack_q8_0_to_q8_0_32_bl_ref(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q8_0); + GGML_ASSERT(interleave_block == 32); // unused + + constexpr int nrows_interleaved = 32; + + block_q8_0x32 * dst = (block_q8_0x32 *) t->data; + const block_q8_0 * src = (const block_q8_0 *) data; + block_q8_0 dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK8_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q8_0)); + + if (t->ne[0] % QK8_0 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + int64_t nrows_real = std::min((int64_t) nrow - b, (int64_t) nrows_interleaved); + for (int64_t x = 0; x < nblocks; x++) { + int i = 0; + for (; i < nrows_real; i++) { + dst_tmp[i] = src[x + i * nblocks]; + } + for (; i < nrows_interleaved; i++) { + memset(&dst_tmp[i], 0, sizeof(block_q8_0)); + } + *dst++ = make_block_q8_0x32(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +// RVV optimized version of repack_q8_0_to_q8_0_32_bl +// Eliminates the intermediate dst_tmp buffer and vectorizes scale gather + qs copy. +static int repack_q8_0_to_q8_0_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q8_0); + GGML_ASSERT(interleave_block == 32); + + constexpr int nrows_interleaved = 32; + + block_q8_0x32 * dst = (block_q8_0x32 *) t->data; + const block_q8_0 * src = (const block_q8_0 *) data; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK8_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q8_0)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK8_0 != 0) { + return -1; + } + + const ptrdiff_t row_stride = (ptrdiff_t) nblocks * sizeof(block_q8_0); + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + const block_q8_0 * col_src = src + x; + + // --- 1) Gather 32 scale values (ggml_half d) with stride load --- + { + const uint8_t * d_base = (const uint8_t *) &col_src->d; + ggml_half * d_dst = dst->d; + size_t remaining = 32; + size_t offset = 0; + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e16m1(remaining); + vuint16m1_t vd = + __riscv_vlse16_v_u16m1((const uint16_t *) (d_base + offset * row_stride), row_stride, vl); + __riscv_vse16_v_u16m1((uint16_t *) (d_dst + offset), vd, vl); + offset += vl; + remaining -= vl; + } + } + + // --- 2) Copy qs for each of the 32 rows (32 bytes per row) --- + { + for (int i = 0; i < 32; i++) { + const int8_t * sq = col_src[i * nblocks].qs; + int8_t * dq = (int8_t *) dst->qs + i * QK8_0; + + size_t len = QK8_0; + size_t idx = 0; + while (len > 0) { + size_t vl = __riscv_vsetvl_e8m2(len); + vint8m2_t vs = __riscv_vle8_v_i8m2(sq + idx, vl); + __riscv_vse8_v_i8m2(dq + idx, vs, vl); + idx += vl; + len -= vl; + } + } + } + + dst++; + } + src += nrows_interleaved * nblocks; + } + return 0; + + GGML_UNUSED(data_size); +} + +static void convert_mxfp4_to_5bit(const block_mxfp4 & src, spacemit_kernels::nrow_block_mxfp4<1> & dst) { + dst.e[0] = src.e; + + // Decode all 32 mxfp4 values to signed integers via kvalues_mxfp4 + int8_t vals[32]; + for (int j = 0; j < QK_MXFP4 / 2; j++) { + vals[j] = kvalues_mxfp4[src.qs[j] & 0xF]; + vals[j + QK_MXFP4 / 2] = kvalues_mxfp4[src.qs[j] >> 4]; + } + + // vals [b0, b1, b2, b3, ..., b30, b31] + // Pack abs into qs with reorder: [b0,b1]..[b14,b15]..[b30,b31] + for (int j = 0; j < QK_MXFP4 / 2; j++) { + uint8_t lo0 = static_cast(std::abs(vals[j * 2])); + uint8_t lo1 = static_cast(std::abs(vals[j * 2 + 1])); + dst.qs[j] = (lo0 & 0x0F) | ((lo1 & 0x0F) << 4); + } + + // Pack sign bits into qh[4] (32 bits total, 1 bit per weight) + // reorder: [0,1,2,...,15,16,17,...,31] after the qs reorder above + uint32_t sign_bits = 0; + for (int j = 0; j < 32; j++) { + if (vals[j] < 0) { + sign_bits |= (1u << j); + } + } + memcpy(dst.qh, &sign_bits, 4); +} + +static spacemit_kernels::nrow_block_mxfp4<32> make_block_mxfp4x32(spacemit_kernels::nrow_block_mxfp4<1> * in, + unsigned int blck_size_interleave) { + spacemit_kernels::nrow_block_mxfp4<32> out; + GGML_ASSERT(QK_MXFP4 / blck_size_interleave == 1); + GGML_UNUSED(blck_size_interleave); + + for (int i = 0; i < 32; i++) { + out.e[i] = in[i].e[0]; + } + + // qs: copy per-row 16 bytes + for (int i = 0; i < 32; i++) { + memcpy(out.qs + i * 16, in[i].qs, 16); + } + + // qh: copy per-row 4 bytes + for (int i = 0; i < 32; i++) { + memcpy(out.qh + i * 4, in[i].qh, 4); + } + + return out; +} + +static int repack_mxfp4_to_mxfp4_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_MXFP4); + GGML_ASSERT(interleave_block == 32); + + constexpr int nrows_interleaved = 32; + + spacemit_kernels::nrow_block_mxfp4<32> * dst = (spacemit_kernels::nrow_block_mxfp4<32> *) t->data; + const block_mxfp4 * src = (const block_mxfp4 *) data; + spacemit_kernels::nrow_block_mxfp4<1> dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_MXFP4; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_mxfp4)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_MXFP4 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + convert_mxfp4_to_5bit(src[x + i * nblocks], dst_tmp[i]); + } + *dst++ = make_block_mxfp4x32(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; +} + +static spacemit_kernels::nrow_block_q5_1<32> make_block_q5_1x32(spacemit_kernels::nrow_block_q5_1<1> * in, + unsigned int blck_size_interleave) { + spacemit_kernels::nrow_block_q5_1<32> out; + GGML_ASSERT(QK5_1 / blck_size_interleave == 1); + GGML_UNUSED(blck_size_interleave); + + for (int i = 0; i < 32; i++) { + out.scales16[i] = in[i].scales16[0]; + out.zp[i] = in[i].zp[0]; + } + + // qs: low 4 bits, reorder from [b0,b16],[b1,b17]... to [b0,b1]...[b14,b15] and [b16,b17]...[b30,b31] + for (int i = 0; i < 32; i++) { + // low half [0..15] + for (int j = 0; j < QK5_1 / 4; j++) { + out.qs[i * QK5_1 / 2 + j] = (in[i].qs[j * 2] & 0x0F) | ((in[i].qs[j * 2 + 1] & 0x0F) << 4); + } + // high half [16..31] + for (int j = 0; j < QK5_1 / 4; j++) { + out.qs[i * QK5_1 / 2 + QK5_1 / 4 + j] = ((in[i].qs[j * 2] & 0xF0) >> 4) | (in[i].qs[j * 2 + 1] & 0xF0); + } + } + + // qh: 5th bit, copy directly + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 4; j++) { + out.qh[i * 4 + j] = in[i].qh[j]; + } + } + + return out; +} + +static spacemit_kernels::nrow_block_q5_0<32> make_block_q5_0x32(spacemit_kernels::nrow_block_q5_0<1> * in, + unsigned int blck_size_interleave) { + spacemit_kernels::nrow_block_q5_0<32> out; + GGML_ASSERT(QK5_0 / blck_size_interleave == 1); + GGML_UNUSED(blck_size_interleave); + + for (int i = 0; i < 32; i++) { + out.scales16[i] = in[i].scales16[0]; + } + + // qs: low 4 bits, reorder from [b0,b16],[b1,b17]... to [b0,b1]...[b14,b15] and [b16,b17]...[b30,b31] + for (int i = 0; i < 32; i++) { + // low half [0..15] + for (int j = 0; j < QK5_0 / 4; j++) { + out.qs[i * QK5_0 / 2 + j] = (in[i].qs[j * 2] & 0x0F) | ((in[i].qs[j * 2 + 1] & 0x0F) << 4); + } + // high half [16..31] + for (int j = 0; j < QK5_0 / 4; j++) { + out.qs[i * QK5_0 / 2 + QK5_0 / 4 + j] = ((in[i].qs[j * 2] & 0xF0) >> 4) | (in[i].qs[j * 2 + 1] & 0xF0); + } + } + + // qh: 5th bit, copy directly + for (int i = 0; i < 32; i++) { + for (int j = 0; j < 4; j++) { + out.qh[i * 4 + j] = in[i].qh[j]; + } + } + + return out; +} + +static int repack_q5_0_to_q5_0_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q5_0); + GGML_ASSERT(interleave_block == 32); // unused + + constexpr int nrows_interleaved = 32; + + spacemit_kernels::nrow_block_q5_0<32> * dst = (spacemit_kernels::nrow_block_q5_0<32> *) t->data; + const block_q5_0 * src = (const block_q5_0 *) data; + spacemit_kernels::nrow_block_q5_0<1> dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK5_0; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q5_0)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK5_0 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + const block_q5_0 & s = src[x + i * nblocks]; + + dst_tmp[i].scales16[0] = s.d; + memcpy(dst_tmp[i].qs, s.qs, sizeof(dst_tmp[i].qs)); + memcpy(dst_tmp[i].qh, s.qh, sizeof(dst_tmp[i].qh)); + } + *dst++ = make_block_q5_0x32(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; +} + +static int repack_q5_1_to_q5_1_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q5_1); + GGML_ASSERT(interleave_block == 32); // unused + + constexpr int nrows_interleaved = 32; + + spacemit_kernels::nrow_block_q5_1<32> * dst = (spacemit_kernels::nrow_block_q5_1<32> *) t->data; + const block_q5_1 * src = (const block_q5_1 *) data; + spacemit_kernels::nrow_block_q5_1<1> dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK5_1; + + GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q5_1)); + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK5_1 != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int i = 0; i < nrows_interleaved; i++) { + const block_q5_1 & s = src[x + i * nblocks]; + + float d = GGML_FP16_TO_FP32(s.GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d); + float m = GGML_FP16_TO_FP32(s.GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m); + + if (d == 0.0f) { + dst_tmp[i].scales16[0] = GGML_FP32_TO_FP16(std::fabs(m)); + dst_tmp[i].zp[0] = m < 0.0f ? 1 : 0; + memset(dst_tmp[i].qh, 0, sizeof(dst_tmp[i].qh)); + memset(dst_tmp[i].qs, m > 0.0f ? 0x11 : 0x00, sizeof(dst_tmp[i].qs)); + continue; + } + + float mid = std::nearbyintf(-m / d); + mid = std::min(31.0f, std::max(0.0f, mid)); + + dst_tmp[i].scales16[0] = GGML_FP32_TO_FP16(d); + dst_tmp[i].zp[0] = static_cast(mid); + + // qs: copy low 4 bits directly (same nibble packing) + memcpy(dst_tmp[i].qs, s.qs, QK5_1 / 2); + + // qh: copy 5th bit directly + memcpy(dst_tmp[i].qh, s.qh, 4); + } + *dst++ = make_block_q5_1x32(dst_tmp, interleave_block); + } + src += nrows_interleaved * nblocks; + } + return 0; +} + +static int repack_q5_k_to_q5_1_32_bl(ggml_tensor * t, + int interleave_block, + const void * GGML_RESTRICT data, + size_t data_size) { + GGML_ASSERT(t->type == GGML_TYPE_Q5_K); + GGML_ASSERT(interleave_block == 32); + GGML_ASSERT(QK_K / QK5_1 == 8); + + constexpr int nrows_interleaved = 32; + + spacemit_kernels::nrow_block_q5_1<32> * dst = (spacemit_kernels::nrow_block_q5_1<32> *) t->data; + const block_q5_K * src = (const block_q5_K *) data; + spacemit_kernels::nrow_block_q5_1<1> dst_tmp[32]; + int nrow = ggml_nrows(t); + int nblocks = t->ne[0] / QK_K; + + if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_K != 0) { + return -1; + } + + for (int b = 0; b < nrow; b += nrows_interleaved) { + for (int64_t x = 0; x < nblocks; x++) { + for (int j = 0; j < 8; j++) { + for (int i = 0; i < nrows_interleaved; i++) { + uint8_t sc, m; + const float d = GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d); + const float min = + GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin); + get_scale_min_k4(j, src[x + i * nblocks].scales, &sc, &m); + + float d1 = d * sc; + float m1 = min * m; + + float mid = std::nearbyintf(m1 / d1); + mid = std::min(31.0f, std::max(0.0f, mid)); + dst_tmp[i].scales16[0] = GGML_FP32_TO_FP16(d1); + dst_tmp[i].zp[0] = static_cast(mid); + + // src -> [b0, b32] [b1, b33] ... [b31, b63] + // dst -> [b0, b16] [b1, b17] ... [b15, b31] [b32, b48] [b33, b49] ... [b47, b63] + const uint8_t * q = src[x + i * nblocks].qs + (j / 2) * QK5_1; + if (j % 2 == 0) { + for (int ii = 0; ii < 16; ii++) { + dst_tmp[i].qs[ii] = (q[ii] & 0x0F) | ((q[ii + 16] & 0x0F) << 4); + } + } else { + for (int ii = 0; ii < 16; ii++) { + dst_tmp[i].qs[ii] = ((q[ii] & 0xF0) >> 4) | (q[ii + 16] & 0xF0); + } + } + + // Extract the 5th bit (qh) for this sub-block + // block_q5_K.qh[32]: for sub-block j, the 5th bit is at bit position j in qh[l] + // qs was reordered: dst_qs maps to src weights [0,16,1,17,...,15,31] + // So qh must follow the same reorder to stay aligned with qs + // dst qh[4] = 32 bits for 32 weights in the reordered layout: + // byte 0: weights 0..7 (from src_qh[0..7]) + // byte 1: weights 8..15 (from src_qh[8..15]) + // byte 2: weights 16..23 (from src_qh[16..23]) + // byte 3: weights 24..31 (from src_qh[24..31]) + const uint8_t * src_qh = src[x + i * nblocks].qh; + for (int bi = 0; bi < 4; bi++) { + uint8_t qh_byte = 0; + for (int k = 0; k < 8; k++) { + int src_idx = bi * 8 + k; + qh_byte |= ((src_qh[src_idx] >> j) & 1) << k; + } + dst_tmp[i].qh[bi] = qh_byte; + } + } + *dst++ = make_block_q5_1x32(dst_tmp, interleave_block); + } + } + src += nrows_interleaved * nblocks; + } + return 0; +} + +namespace ggml::cpu::riscv64_spacemit { + +template int repack(ggml_tensor *, const void *, size_t); + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q4_0_to_q4_0_16_bl(t, 16, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q4_1_to_q4_1_16_bl(t, 16, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q4_k_to_q4_1_16_bl(t, 16, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q2_k_to_q2_k_32_bl(t, 32, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q3_k_to_q3_k_32_bl(t, 32, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { +#if 0 + return repack_q4_0_to_q4_0_32_bl_ref(t, 32, data, data_size); +#else + return repack_q4_0_to_q4_0_32_bl(t, 32, data, data_size); +#endif +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { +#if 1 + return repack_q4_0_to_q4_0_256_32_bl_ref(t, 32, data, data_size); +#else + //return repack_q4_0_to_q4_0_256_32_bl(t, 32, data, data_size); +#endif +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { +#if 0 + return repack_q4_1_to_q4_1_32_bl_ref(t, 32, data, data_size); +#else + return repack_q4_1_to_q4_1_32_bl(t, 32, data, data_size); +#endif +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { +#if 1 + return repack_q4_0_to_q4_1_256_32_bl_ref(t, 32, data, data_size); +#else + return repack_q4_1_to_q4_1_256_32_bl(t, 32, data, data_size); +#endif +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q4_k_to_q4_1_32_bl(t, 32, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { +#if 1 + return repack_q6_k_to_q8_0_32_bl_ref(t, 32, data, data_size); +#else + return repack_q6_k_to_q8_0_32_bl(t, 32, data, data_size); +#endif +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q8_0_to_q8_0_16_bl_ref(t, 16, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { +#if 1 + return repack_q8_0_to_q8_0_32_bl_ref(t, 32, data, data_size); +#else + return repack_q8_0_to_q8_0_32_bl(t, 32, data, data_size); +#endif +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_mxfp4_to_mxfp4_32_bl(t, 32, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q5_0_to_q5_0_32_bl(t, 32, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q5_1_to_q5_1_32_bl(t, 32, data, data_size); +} + +template <> int repack(ggml_tensor * t, const void * data, size_t data_size) { + return repack_q5_k_to_q5_1_32_bl(t, 32, data, data_size); +} + +} // namespace ggml::cpu::riscv64_spacemit diff --git a/ggml/src/ggml-cpu/spacemit/repack.h b/ggml/src/ggml-cpu/spacemit/repack.h new file mode 100644 index 0000000..950cbde --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/repack.h @@ -0,0 +1,14 @@ +#pragma once + +#include "ggml-common.h" +#include "ggml.h" + +#include +#include + +namespace ggml::cpu::riscv64_spacemit { + +template +int repack(ggml_tensor * t, const void * data, size_t data_size); + +} // namespace ggml::cpu::riscv64_spacemit diff --git a/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp b/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp new file mode 100644 index 0000000..13b84dc --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp @@ -0,0 +1,3178 @@ +#include "rvv_kernels.h" + +#include "common.h" +#include "ggml.h" +#include "ops.h" +#include "string.h" + +#include +#include +#include +#include + +#if !defined(__riscv_v) || !defined(__riscv_v_intrinsic) +# error "riscv v extension or v_intrinsic not enabled" +#else +# include +#endif + +#if !defined(__riscv_zfh) +# error "riscv zfh extension not enabled" +#endif + +#if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Woverlength-strings" +# pragma GCC diagnostic ignored "-Wcast-qual" +# pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +namespace spacemit_kernels::rvv { + +namespace { + +auto align_up(size_t value, size_t alignment) { + return (value + alignment - 1) / alignment * alignment; +} + +static inline bool flash_attn_ext_supported_d_vlen1024_vf16(int64_t d) { + return d > 0 && d <= 128; +} + +static inline bool flash_attn_ext_supported_shape_vlen1024_vf16(int64_t DK, int64_t DV) { + return flash_attn_ext_supported_d_vlen1024_vf16(DK) && flash_attn_ext_supported_d_vlen1024_vf16(DV); +} + +static inline float reduce_sum_f32m4_vlen1024(vfloat32m4_t v, size_t vl) { + vfloat32m1_t s_v = __riscv_vfmv_v_f_f32m1(0.0f, 1); + s_v = __riscv_vfredusum_vs_f32m4_f32m1(v, s_v, vl); + return __riscv_vfmv_f_s_f32m1_f32(s_v); +} + +static inline float reduce_sum_f32m2_vlen1024(vfloat32m2_t v, size_t vl) { + vfloat32m1_t s_v = __riscv_vfmv_v_f_f32m1(0.0f, 1); + s_v = __riscv_vfredusum_vs_f32m2_f32m1(v, s_v, vl); + return __riscv_vfmv_f_s_f32m1_f32(s_v); +} + +// Adapted from ggml_v_expf_m2 in vec.h. This is accurate enough for softmax. +static inline vfloat32m2_t rvv_expf_approx_f32m2(vfloat32m2_t x, size_t vl) { + const vfloat32m2_t r = __riscv_vfmv_v_f_f32m2(0x1.8p23f, vl); + const vfloat32m2_t z = __riscv_vfmacc_vf_f32m2(r, 0x1.715476p+0f, x, vl); + const vfloat32m2_t n = __riscv_vfsub_vv_f32m2(z, r, vl); + const vfloat32m2_t b = + __riscv_vfnmsac_vf_f32m2(__riscv_vfnmsac_vf_f32m2(x, 0x1.62e4p-1f, n, vl), 0x1.7f7d1cp-20f, n, vl); + const vuint32m2_t e = __riscv_vsll_vx_u32m2(__riscv_vreinterpret_v_f32m2_u32m2(z), 23, vl); + const vfloat32m2_t k = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vadd_vx_u32m2(e, 0x3f800000, vl)); + const vbool16_t c = __riscv_vmfgt_vf_f32m2_b16(__riscv_vfabs_v_f32m2(n, vl), 126.0f, vl); + const vfloat32m2_t u = __riscv_vfmul_vv_f32m2(b, b, vl); + const vfloat32m2_t j = __riscv_vfmacc_vv_f32m2( + __riscv_vfmul_vf_f32m2(b, 0x1.ffffecp-1f, vl), + __riscv_vfmacc_vv_f32m2( + __riscv_vfmacc_vf_f32m2(__riscv_vfmv_v_f_f32m2(0x1.fffdb6p-2f, vl), 0x1.555e66p-3f, b, vl), + __riscv_vfmacc_vf_f32m2(__riscv_vfmv_v_f_f32m2(0x1.573e2ep-5f, vl), 0x1.0e4020p-7f, b, vl), u, vl), + u, vl); + + if (!__riscv_vcpop_m_b16(c, vl)) { + return __riscv_vfmacc_vv_f32m2(k, j, k, vl); + } + + const vbool16_t dm = __riscv_vmfle_vf_f32m2_b16(n, 0.0f, vl); + const vuint32m2_t d = __riscv_vmerge_vxm_u32m2(__riscv_vmv_v_x_u32m2(0, vl), 0x82000000, dm, vl); + const vfloat32m2_t s1 = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vadd_vx_u32m2(d, 0x7f000000, vl)); + const vfloat32m2_t s2 = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vsub_vv_u32m2(e, d, vl)); + const vfloat32m2_t r1 = + __riscv_vmerge_vvm_f32m2(__riscv_vfmacc_vv_f32m2(k, k, j, vl), + __riscv_vfmul_vv_f32m2(__riscv_vfmacc_vv_f32m2(s2, s2, j, vl), s1, vl), c, vl); + return __riscv_vmerge_vvm_f32m2(r1, __riscv_vfmul_vv_f32m2(s1, s1, vl), + __riscv_vmfgt_vf_f32m2_b16(__riscv_vfabs_v_f32m2(n, vl), 192.0f, vl), vl); +} + +static inline vfloat32m2_t rvv_tanh_approx_f32m2(vfloat32m2_t x, size_t vl) { + const vfloat32m2_t abs_x = __riscv_vfabs_v_f32m2(x, vl); + const vfloat32m2_t neg_2_abs = __riscv_vfmul_vf_f32m2(abs_x, -2.0f, vl); + const vfloat32m2_t exp_term = rvv_expf_approx_f32m2(neg_2_abs, vl); + const vfloat32m2_t numerator = __riscv_vfsub_vf_f32m2(exp_term, 1.0f, vl); + const vfloat32m2_t denominator = __riscv_vfadd_vf_f32m2(exp_term, 1.0f, vl); + const vfloat32m2_t tanh_abs = __riscv_vfneg_v_f32m2(__riscv_vfdiv_vv_f32m2(numerator, denominator, vl), vl); + const vbool16_t neg_mask = __riscv_vmflt_vf_f32m2_b16(x, 0.0f, vl); + const vfloat32m2_t tanh_neg = __riscv_vfneg_v_f32m2(tanh_abs, vl); + return __riscv_vmerge_vvm_f32m2(tanh_abs, tanh_neg, neg_mask, vl); +} + +static void rvv_softcap_tanh_inplace_f32(float * dst, int64_t dst_stride, int64_t tile_rows, int64_t n, float softcap) { + for (int tq = 0; tq < tile_rows; ++tq, dst += dst_stride) { + float * dst_row = dst; + int64_t remaining = n; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m2(remaining); + vfloat32m2_t v = __riscv_vle32_v_f32m2(dst_row, vl); + v = rvv_tanh_approx_f32m2(v, vl); + v = __riscv_vfmul_vf_f32m2(v, softcap, vl); + __riscv_vse32_v_f32m2(dst_row, v, vl); + dst_row += vl; + remaining -= vl; + } + } +} + +static inline float rvv_softmax_exp_inplace_f32(float * dst, int64_t n, float max_value) { + float row_sum = 0.0f; + while (n > 0) { + const size_t vl = __riscv_vsetvl_e32m2(n); + vfloat32m2_t v = __riscv_vle32_v_f32m2(dst, vl); + v = __riscv_vfsub_vf_f32m2(v, max_value, vl); + v = rvv_expf_approx_f32m2(v, vl); + __riscv_vse32_v_f32m2(dst, v, vl); + row_sum += reduce_sum_f32m2_vlen1024(v, vl); + dst += vl; + n -= vl; + } + return row_sum; +} + +static inline float rvv_add_max_inplace_f32(float * dst, const float * src, int64_t n) { + float max_val = -INFINITY; + while (n > 0) { + const size_t vl = __riscv_vsetvl_e32m4(n); + vfloat32m4_t vdst = __riscv_vle32_v_f32m4(dst, vl); + vfloat32m4_t vsrc = __riscv_vle32_v_f32m4(src, vl); + vdst = __riscv_vfadd_vv_f32m4(vdst, vsrc, vl); + __riscv_vse32_v_f32m4(dst, vdst, vl); + + vfloat32m1_t seed = __riscv_vfmv_v_f_f32m1(max_val, 1); + seed = __riscv_vfredmax_vs_f32m4_f32m1(vdst, seed, vl); + max_val = __riscv_vfmv_f_s_f32m1_f32(seed); + + dst += vl; + src += vl; + n -= vl; + } + return max_val; +} + +static inline float rvv_softcap_add_max_inplace_f32(float * dst, const float * src, int64_t n, float softcap) { + if (softcap == 0.0f) { + return rvv_add_max_inplace_f32(dst, src, n); + } + + float max_val = -INFINITY; + while (n > 0) { + const size_t vl = __riscv_vsetvl_e32m2(n); + vfloat32m2_t vdst = __riscv_vle32_v_f32m2(dst, vl); + vfloat32m2_t vsrc = __riscv_vle32_v_f32m2(src, vl); + vdst = rvv_tanh_approx_f32m2(vdst, vl); + vdst = __riscv_vfmul_vf_f32m2(vdst, softcap, vl); + vdst = __riscv_vfadd_vv_f32m2(vdst, vsrc, vl); + __riscv_vse32_v_f32m2(dst, vdst, vl); + + vfloat32m1_t seed = __riscv_vfmv_v_f_f32m1(max_val, 1); + seed = __riscv_vfredmax_vs_f32m2_f32m1(vdst, seed, vl); + max_val = __riscv_vfmv_f_s_f32m1_f32(seed); + + dst += vl; + src += vl; + n -= vl; + } + return max_val; +} + +static inline void rvv_zero_f32(float * dst, int64_t n) { + while (n > 0) { + const size_t vl = __riscv_vsetvl_e32m4(n); + const vfloat32m4_t z = __riscv_vfmv_v_f_f32m4(0.0f, vl); + __riscv_vse32_v_f32m4(dst, z, vl); + dst += vl; + n -= vl; + } +} + +static inline void rvv_scale_f32(float * dst, float scale, int64_t n) { + while (n > 0) { + const size_t vl = __riscv_vsetvl_e32m4(n); + vfloat32m4_t v = __riscv_vle32_v_f32m4(dst, vl); + v = __riscv_vfmul_vf_f32m4(v, scale, vl); + __riscv_vse32_v_f32m4(dst, v, vl); + dst += vl; + n -= vl; + } +} + +static inline void rvv_add_inplace_f32(float * dst, + int64_t dst_stride, + const float * src, + int64_t src_stride, + int64_t tile_rows, + int64_t n) { + for (int tq = 0; tq < tile_rows; ++tq, dst += dst_stride, src += src_stride) { + int64_t remaining = n; + float * dst_row = dst; + const float * src_row = src; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m4(remaining); + vfloat32m4_t vdst = __riscv_vle32_v_f32m4(dst_row, vl); + vfloat32m4_t vsrc = __riscv_vle32_v_f32m4(src_row, vl); + vdst = __riscv_vfadd_vv_f32m4(vdst, vsrc, vl); + __riscv_vse32_v_f32m4(dst_row, vdst, vl); + dst_row += vl; + src_row += vl; + remaining -= vl; + } + } +} + +static inline float rvv_max_f32(const float * src, int64_t n) { + float max_val = -INFINITY; + while (n > 0) { + const size_t vl = __riscv_vsetvl_e32m4(n); + const vfloat32m4_t v = __riscv_vle32_v_f32m4(src, vl); + vfloat32m1_t seed = __riscv_vfmv_v_f_f32m1(max_val, 1); + seed = __riscv_vfredmax_vs_f32m4_f32m1(v, seed, vl); + max_val = __riscv_vfmv_f_s_f32m1_f32(seed); + src += vl; + n -= vl; + } + return max_val; +} + +static void rvv_pack_f32_as_scaled_f16(void * dst, + int64_t dst_row_stride, + const void * src, + int64_t src_row_stride, + int64_t tile_rows, + int64_t n, + float scale) { + for (int tq = 0; tq < tile_rows; ++tq) { + const float * row_ptr = (const float *) ((const char *) src + tq * src_row_stride); + _Float16 * dst_row_ptr = (_Float16 *) ((char *) dst + tq * dst_row_stride); + int64_t remaining = n; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m4(remaining); + vfloat32m4_t v32 = __riscv_vle32_v_f32m4(row_ptr, vl); + v32 = __riscv_vfmul_vf_f32m4(v32, scale, vl); + const vfloat16m2_t v16 = __riscv_vfncvt_f_f_w_f16m2(v32, vl); + __riscv_vse16_v_f16m2(dst_row_ptr, v16, vl); + dst_row_ptr += vl; + row_ptr += vl; + remaining -= vl; + } + } +} + +static void rvv_pack_scaled_f16_as_f32(void * dst, + int64_t dst_row_stride, + const void * src, + int64_t src_row_stride, + int64_t tile_rows, + int64_t n, + float scale) { + for (int tq = 0; tq < tile_rows; ++tq) { + const _Float16 * row_ptr = (const _Float16 *) ((const char *) src + tq * src_row_stride); + float * dst_row_ptr = (float *) ((char *) dst + tq * dst_row_stride); + int64_t remaining = n; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e16m2(remaining); + const vfloat16m2_t v16 = __riscv_vle16_v_f16m2(row_ptr, vl); + vfloat32m4_t v32 = __riscv_vfwcvt_f_f_v_f32m4(v16, vl); + v32 = __riscv_vfmul_vf_f32m4(v32, scale, vl); + __riscv_vse32_v_f32m4(dst_row_ptr, v32, vl); + dst_row_ptr += vl; + row_ptr += vl; + remaining -= vl; + } + } +} + +static void rvv_pack_scaled_f32_as_f32(void * dst, + int64_t dst_row_stride, + const void * src, + int64_t src_row_stride, + int64_t tile_rows, + int64_t n, + float * scale) { + for (int tq = 0; tq < tile_rows; ++tq) { + const float * row_ptr = (const float *) ((const char *) src + tq * src_row_stride); + float * dst_row_ptr = (float *) ((char *) dst + tq * dst_row_stride); + int64_t remaining = n; + while (remaining > 0) { + const size_t vl = __riscv_vsetvl_e32m4(remaining); + vfloat32m4_t v32 = __riscv_vle32_v_f32m4(row_ptr, vl); + v32 = __riscv_vfmul_vf_f32m4(v32, scale[tq], vl); + __riscv_vse32_v_f32m4(dst_row_ptr, v32, vl); + dst_row_ptr += vl; + row_ptr += vl; + remaining -= vl; + } + } +} + +static inline void rvv_transposed_s32_mn_to_nm(int8_t * dst, + int64_t n_dst_stride, + int8_t * src, + int64_t m_src_stride, + int64_t m, + int64_t n) { + int8_t * in = src; + int8_t * out = dst; + + __asm__ volatile( + "vsetvli t0, zero, e32, m1, tu, mu \n\t" + "mul t3, t0, %[os0] \n\t" + "srli t2, %[isz0], 3 \n\t" + "blez t2, M1%= \n\t" + + "LOOP_M8%=: \n\t" + "addi a1, %[dst], 0 \n\t" + "addi s1, %[src], 0 \n\t" + "add s2, %[src], %[is0] \n\t" + "add s3, s2, %[is0] \n\t" + "add s4, s3, %[is0] \n\t" + "add s5, s4, %[is0] \n\t" + "add s6, s5, %[is0] \n\t" + "add s7, s6, %[is0] \n\t" + "add s8, s7, %[is0] \n\t" + "addi t1, %[isz1], 0 \n\t" + + "LOOP_M8N%=: \n\t" + "vsetvli t0, t1, e32, m1, tu, mu \n\t" + "sub t1, t1, t0 \n\t" + "vle32.v v0, (s1) \n\t" + "sh2add s1, t0, s1 \n\t" + "vle32.v v1, (s2) \n\t" + "sh2add s2, t0, s2 \n\t" + "vle32.v v2, (s3) \n\t" + "sh2add s3, t0, s3 \n\t" + "vle32.v v3, (s4) \n\t" + "sh2add s4, t0, s4 \n\t" + "vle32.v v4, (s5) \n\t" + "sh2add s5, t0, s5 \n\t" + "vle32.v v5, (s6) \n\t" + "sh2add s6, t0, s6 \n\t" + "vle32.v v6, (s7) \n\t" + "sh2add s7, t0, s7 \n\t" + "vle32.v v7, (s8) \n\t" + "sh2add s8, t0, s8 \n\t" + "vssseg8e32.v v0, (a1), %[os0] \n\t" + "add a1, a1, t3 \n\t" + "bnez t1, LOOP_M8N%= \n\t" + "sh3add %[src], %[is0], %[src] \n\t" + "addi %[dst], %[dst], 32 \n\t" + "addi t2, t2, -1 \n\t" + "bnez t2, LOOP_M8%= \n\t" + + "M1%=: \n\t" + "andi t2, %[isz0], 7 \n\t" + "blez t2, END%= \n\t" + + "LOOP_M1%=: \n\t" + "addi a1, %[dst], 0 \n\t" + "addi s1, %[src], 0 \n\t" + "addi t1, %[isz1], 0 \n\t" + + "LOOP_M1N%=: \n\t" + "vsetvli t0, t1, e32, m1, tu, mu \n\t" + "sub t1, t1, t0 \n\t" + "vle32.v v0, (s1) \n\t" + "sh2add s1, t0, s1 \n\t" + "vsse32.v v0, (a1), %[os0] \n\t" + "add a1, a1, t3 \n\t" + "bnez t1, LOOP_M1N%= \n\t" + "add %[src], %[is0], %[src] \n\t" + "addi %[dst], %[dst], 4 \n\t" + "addi t2, t2, -1 \n\t" + "bnez t2, LOOP_M1%= \n\t" + "END%=: \n\t" + + : [src] "+r"(in), [dst] "+r"(out), [isz0] "+r"(m) + : [isz1] "r"(n), [is0] "r"(m_src_stride), [os0] "r"(n_dst_stride) + : "cc", "t0", "t1", "t2", "t3", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "a1"); +} + +static inline void rvv_transposed_s16_mn_to_nm(int8_t * dst, + int64_t n_dst_stride, + int8_t * src, + int64_t m_src_stride, + int64_t m, + int64_t n) { + int8_t * in = src; + int8_t * out = dst; + + __asm__ volatile( + "vsetvli t0, zero, e16, m1, tu, mu \n\t" + "mul t3, t0, %[os0] \n\t" + "srli t2, %[isz0], 3 \n\t" + "blez t2, M1%= \n\t" + + "LOOP_M8%=: \n\t" + "addi a1, %[dst], 0 \n\t" + "addi s1, %[src], 0 \n\t" + "add s2, %[src], %[is0] \n\t" + "add s3, s2, %[is0] \n\t" + "add s4, s3, %[is0] \n\t" + "add s5, s4, %[is0] \n\t" + "add s6, s5, %[is0] \n\t" + "add s7, s6, %[is0] \n\t" + "add s8, s7, %[is0] \n\t" + "addi t1, %[isz1], 0 \n\t" + + "LOOP_M8N%=: \n\t" + "vsetvli t0, t1, e16, m1, tu, mu \n\t" + "sub t1, t1, t0 \n\t" + "vle16.v v0, (s1) \n\t" + "sh1add s1, t0, s1 \n\t" + "vle16.v v1, (s2) \n\t" + "sh1add s2, t0, s2 \n\t" + "vle16.v v2, (s3) \n\t" + "sh1add s3, t0, s3 \n\t" + "vle16.v v3, (s4) \n\t" + "sh1add s4, t0, s4 \n\t" + "vle16.v v4, (s5) \n\t" + "sh1add s5, t0, s5 \n\t" + "vle16.v v5, (s6) \n\t" + "sh1add s6, t0, s6 \n\t" + "vle16.v v6, (s7) \n\t" + "sh1add s7, t0, s7 \n\t" + "vle16.v v7, (s8) \n\t" + "sh1add s8, t0, s8 \n\t" + "vssseg8e16.v v0, (a1), %[os0] \n\t" + "add a1, a1, t3 \n\t" + "bnez t1, LOOP_M8N%= \n\t" + "sh3add %[src], %[is0], %[src] \n\t" + "addi %[dst], %[dst], 16 \n\t" + "addi t2, t2, -1 \n\t" + "bnez t2, LOOP_M8%= \n\t" + + "M1%=: \n\t" + "andi t2, %[isz0], 7 \n\t" + "blez t2, END%= \n\t" + + "LOOP_M1%=: \n\t" + "addi a1, %[dst], 0 \n\t" + "addi s1, %[src], 0 \n\t" + "addi t1, %[isz1], 0 \n\t" + + "LOOP_M1N%=: \n\t" + "vsetvli t0, t1, e16, m1, tu, mu \n\t" + "sub t1, t1, t0 \n\t" + "vle16.v v0, (s1) \n\t" + "sh1add s1, t0, s1 \n\t" + "vsse16.v v0, (a1), %[os0] \n\t" + "add a1, a1, t3 \n\t" + "bnez t1, LOOP_M1N%= \n\t" + "add %[src], %[is0], %[src] \n\t" + "addi %[dst], %[dst], 2 \n\t" + "addi t2, t2, -1 \n\t" + "bnez t2, LOOP_M1%= \n\t" + "END%=: \n\t" + + : [src] "+r"(in), [dst] "+r"(out), [isz0] "+r"(m) + : [isz1] "r"(n), [is0] "r"(m_src_stride), [os0] "r"(n_dst_stride) + : "cc", "t0", "t1", "t2", "t3", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "a1"); +} + +static inline void rvv_qk_dot_tile_f16_x1(float * dst, + const _Float16 * q_row, + const _Float16 * k_pack, + int64_t dk, + int64_t kv_tile) { + const size_t vl = __riscv_vsetvl_e16m1(kv_tile); + vfloat32m2_t acc = __riscv_vfmv_v_f_f32m2(0.0f, vl); + + for (int64_t d = 0; d < dk; ++d) { + const vfloat16m1_t k_vec = __riscv_vle16_v_f16m1(k_pack + d * ggml_fa_tile_config::KV, vl); + acc = __riscv_vfwmacc_vf_f32m2(acc, q_row[d], k_vec, vl); + } + + __riscv_vse32_v_f32m2(dst, acc, vl); +} + +static inline void rvv_qk_dot_tile_f16_x4(float * dst0, + float * dst1, + float * dst2, + float * dst3, + const _Float16 * q0, + const _Float16 * q1, + const _Float16 * q2, + const _Float16 * q3, + const _Float16 * k_pack, + int64_t dk, + int64_t kv_tile) { + const size_t vl = __riscv_vsetvl_e16m1(kv_tile); + vfloat32m2_t acc0 = __riscv_vfmv_v_f_f32m2(0.0f, vl); + vfloat32m2_t acc1 = __riscv_vfmv_v_f_f32m2(0.0f, vl); + vfloat32m2_t acc2 = __riscv_vfmv_v_f_f32m2(0.0f, vl); + vfloat32m2_t acc3 = __riscv_vfmv_v_f_f32m2(0.0f, vl); + + for (int64_t d = 0; d < dk; ++d) { + const vfloat16m1_t k_vec = __riscv_vle16_v_f16m1(k_pack + d * ggml_fa_tile_config::KV, vl); + acc0 = __riscv_vfwmacc_vf_f32m2(acc0, q0[d], k_vec, vl); + acc1 = __riscv_vfwmacc_vf_f32m2(acc1, q1[d], k_vec, vl); + acc2 = __riscv_vfwmacc_vf_f32m2(acc2, q2[d], k_vec, vl); + acc3 = __riscv_vfwmacc_vf_f32m2(acc3, q3[d], k_vec, vl); + } + + __riscv_vse32_v_f32m2(dst0, acc0, vl); + __riscv_vse32_v_f32m2(dst1, acc1, vl); + __riscv_vse32_v_f32m2(dst2, acc2, vl); + __riscv_vse32_v_f32m2(dst3, acc3, vl); +} + +static inline void rvv_pv_accumulate_f16_x1(float * dst, + const float * prob, + const _Float16 * v_pack, + int64_t kv_tile, + int64_t dv) { + int64_t d_left = dv; + int64_t d_off = 0; + + while (d_left > 0) { + const size_t vl = __riscv_vsetvl_e16m2(d_left); + vfloat32m4_t acc = __riscv_vle32_v_f32m4(dst + d_off, vl); + + for (int64_t tk = 0; tk < kv_tile; ++tk) { + const vfloat16m2_t v16 = __riscv_vle16_v_f16m2(v_pack + tk * dv + d_off, vl); + const vfloat32m4_t v32 = __riscv_vfwcvt_f_f_v_f32m4(v16, vl); + acc = __riscv_vfmacc_vf_f32m4(acc, prob[tk], v32, vl); + } + + __riscv_vse32_v_f32m4(dst + d_off, acc, vl); + d_left -= vl; + d_off += vl; + } +} + +static inline void rvv_pv_accumulate_f16_x4(float * dst0, + float * dst1, + float * dst2, + float * dst3, + const float * prob0, + const float * prob1, + const float * prob2, + const float * prob3, + const _Float16 * v_pack, + int64_t kv_tile, + int64_t dv) { + int64_t d_left = dv; + int64_t d_off = 0; + + while (d_left > 0) { + const size_t vl = __riscv_vsetvl_e16m2(d_left); + vfloat32m4_t acc0 = __riscv_vle32_v_f32m4(dst0 + d_off, vl); + vfloat32m4_t acc1 = __riscv_vle32_v_f32m4(dst1 + d_off, vl); + vfloat32m4_t acc2 = __riscv_vle32_v_f32m4(dst2 + d_off, vl); + vfloat32m4_t acc3 = __riscv_vle32_v_f32m4(dst3 + d_off, vl); + + for (int64_t tk = 0; tk < kv_tile; ++tk) { + const vfloat16m2_t v16 = __riscv_vle16_v_f16m2(v_pack + tk * dv + d_off, vl); + const vfloat32m4_t v32 = __riscv_vfwcvt_f_f_v_f32m4(v16, vl); + acc0 = __riscv_vfmacc_vf_f32m4(acc0, prob0[tk], v32, vl); + acc1 = __riscv_vfmacc_vf_f32m4(acc1, prob1[tk], v32, vl); + acc2 = __riscv_vfmacc_vf_f32m4(acc2, prob2[tk], v32, vl); + acc3 = __riscv_vfmacc_vf_f32m4(acc3, prob3[tk], v32, vl); + } + + __riscv_vse32_v_f32m4(dst0 + d_off, acc0, vl); + __riscv_vse32_v_f32m4(dst1 + d_off, acc1, vl); + __riscv_vse32_v_f32m4(dst2 + d_off, acc2, vl); + __riscv_vse32_v_f32m4(dst3 + d_off, acc3, vl); + d_left -= vl; + d_off += vl; + } +} + +static inline void rvv_qk_dot_tile(float * dst, + const float * q_row, + const float * k_pack, + int64_t dk, + int64_t kv_tile, + float scale) { + const size_t vl = __riscv_vsetvl_e32m4(kv_tile); + vfloat32m4_t acc = __riscv_vfmv_v_f_f32m4(0.0f, vl); + + for (int64_t d = 0; d < dk; ++d) { + const vfloat32m4_t k_vec = __riscv_vle32_v_f32m4(k_pack + d * kv_tile, vl); + acc = __riscv_vfmacc_vf_f32m4(acc, q_row[d] * scale, k_vec, vl); + } + + __riscv_vse32_v_f32m4(dst, acc, vl); +} + +static inline void rvv_pv_accumulate(float * dst, + const float * prob, + const float * v_pack, + int64_t kv_tile, + int64_t dv) { + int64_t d_left = dv; + int64_t d_off = 0; + + while (d_left > 0) { + const size_t vl = __riscv_vsetvl_e32m4(d_left); + vfloat32m4_t acc = __riscv_vle32_v_f32m4(dst + d_off, vl); + + for (int64_t tk = 0; tk < kv_tile; ++tk) { + const vfloat32m4_t v_vec = __riscv_vle32_v_f32m4(v_pack + tk * dv + d_off, vl); + acc = __riscv_vfmacc_vf_f32m4(acc, prob[tk], v_vec, vl); + } + + __riscv_vse32_v_f32m4(dst + d_off, acc, vl); + d_left -= vl; + d_off += vl; + } +} + +static void permute_transpose_impl(const ggml_tensor * src0, + ggml_tensor * dst, + int64_t batch, + int64_t m, + int64_t n, + int64_t batch_stride, + int64_t m_src_stride, + int64_t n_src_stride, + int64_t n_dst_stride, + int ith, + int nth) { + GGML_ASSERT(n_src_stride == sizeof(int32_t) || n_src_stride == sizeof(int16_t)); + + if (n_src_stride == sizeof(int32_t)) { + for (int64_t bi = ith; bi < batch; bi += nth) { + rvv_transposed_s32_mn_to_nm((int8_t *) ((char *) dst->data + bi * batch_stride), n_dst_stride, + (int8_t *) ((char *) src0->data + bi * batch_stride), m_src_stride, m, n); + } + } else if (n_src_stride == sizeof(int16_t)) { + for (int64_t bi = ith; bi < batch; bi += nth) { + rvv_transposed_s16_mn_to_nm((int8_t *) ((char *) dst->data + bi * batch_stride), n_dst_stride, + (int8_t *) ((char *) src0->data + bi * batch_stride), m_src_stride, m, n); + } + } else { + GGML_ABORT("not implemented"); + } +} + +template +static void flash_attn_ext_f16_one_chunk_inner_vlen1024_vf16_mrow(float ** pq, + const char * k_data_row, + const char * v_data_row, + const ggml_fp16_t * mp, + float ** sinks, + float ** dst, + float scale, + float logit_softcap, + float slope, + int64_t nek1, + int64_t nbk1, + int64_t nbv1, + int64_t DV, + int64_t DK, + void * tcm_buffer, + size_t tcm_buffer_size) { + GGML_ASSERT(flash_attn_ext_supported_shape_vlen1024_vf16(DK, DV)); + float S[QLEN] = { 0.0f }; // sum + float M[QLEN] = { -INFINITY }; // maximum KQ value + + _Float16 * kq16_buffer = (_Float16 *) tcm_buffer; + _Float16 * qv_buffer = kq16_buffer + QLEN * DV; + const size_t qkv_temp_buffer_size = (QLEN * DV + QLEN * DK) * sizeof(_Float16); + char * kv_tile_buffer = (char *) (qv_buffer + QLEN * DK); + + { + vfloat16m2_t VKQ16_v = __riscv_vfmv_v_f_f16m2(0.0f, DV); + for (int64_t i = 0; i < QLEN; ++i) { + __riscv_vse16_v_f16m2(kq16_buffer + i * DV, VKQ16_v, DV); + vfloat16m2_t Q_q_v = __riscv_vfncvt_f_f_w_f16m2(__riscv_vle32_v_f32m4(pq[i], DK), DK); + __riscv_vse16_v_f16m2(qv_buffer + i * DK, Q_q_v, DK); + } + } + + const uintptr_t scratch_addr = reinterpret_cast(kv_tile_buffer); + const size_t scratch_size = tcm_buffer_size > qkv_temp_buffer_size ? tcm_buffer_size - qkv_temp_buffer_size : 0; + const uintptr_t kq_tile_addr = align_up(scratch_addr, alignof(float)); + const size_t scratch_prefix = kq_tile_addr - scratch_addr; + const size_t packed_tile_size = + QLEN * sizeof(float) + DK * sizeof(_Float16) + DV * sizeof(_Float16) + sizeof(float); + const int64_t max_ic_tile_step = ((int64_t) __riscv_vsetvlmax_e16m1()) & ~((int64_t) 7); + const int64_t max_fit_by_tcm = + scratch_size > scratch_prefix ? (int64_t) ((scratch_size - scratch_prefix) / packed_tile_size) : 0; + const int64_t ic_tile_step = std::min(max_ic_tile_step, max_fit_by_tcm) & ~((int64_t) 7); + + const uintptr_t k_tile_addr = kq_tile_addr + QLEN * ic_tile_step * sizeof(float); + const uintptr_t v_tile_addr = k_tile_addr + DK * ic_tile_step * sizeof(_Float16); + const uintptr_t mv_tile_addr = v_tile_addr + ic_tile_step * DV * sizeof(_Float16); + + if (ic_tile_step >= 8) { + float * kq_tile_buffer = reinterpret_cast(kq_tile_addr); + _Float16 * k_tile_pack = reinterpret_cast<_Float16 *>(k_tile_addr); + _Float16 * v_tile_pack = reinterpret_cast<_Float16 *>(v_tile_addr); + float * mv_tile_pack = reinterpret_cast(mv_tile_addr); + + const int64_t k_tile_byte_stride = ic_tile_step * (int64_t) sizeof(_Float16); + + int64_t ic_step = 0; + for (int64_t ic = 0; ic < nek1; ++ic) { + const float mv = mp ? slope * ((_Float16 *) mp)[ic] : 0.0f; + + if (mv != -INFINITY) { + const _Float16 * k_data = (const _Float16 *) (k_data_row + ic * nbk1); + const _Float16 * v_data = (const _Float16 *) (v_data_row + ic * nbv1); + + const vfloat16m2_t k_data_v = __riscv_vle16_v_f16m2(k_data, DK); + const vfloat16m2_t v_data_v = __riscv_vle16_v_f16m2(v_data, DV); + __riscv_vsse16_v_f16m2(k_tile_pack + ic_step, k_tile_byte_stride, k_data_v, DK); + __riscv_vse16_v_f16m2(v_tile_pack + ic_step * DV, v_data_v, DV); + mv_tile_pack[ic_step] = mv; + ic_step++; + } + + if (ic_step > 0 && (ic_step == ic_tile_step || ic == (nek1 - 1))) { + if constexpr (QLEN == 4) { + const size_t qk_vl = __riscv_vsetvl_e16m1(ic_step); + vfloat32m2_t qk_acc0 = __riscv_vfmv_v_f_f32m2(0.0f, qk_vl); + vfloat32m2_t qk_acc1 = __riscv_vfmv_v_f_f32m2(0.0f, qk_vl); + vfloat32m2_t qk_acc2 = __riscv_vfmv_v_f_f32m2(0.0f, qk_vl); + vfloat32m2_t qk_acc3 = __riscv_vfmv_v_f_f32m2(0.0f, qk_vl); + + for (int64_t d = 0; d < DK; ++d) { + const vfloat16m1_t k_vec = __riscv_vle16_v_f16m1(k_tile_pack + d * ic_tile_step, qk_vl); + qk_acc0 = __riscv_vfwmacc_vf_f32m2(qk_acc0, qv_buffer[0 * DK + d], k_vec, qk_vl); + qk_acc1 = __riscv_vfwmacc_vf_f32m2(qk_acc1, qv_buffer[1 * DK + d], k_vec, qk_vl); + qk_acc2 = __riscv_vfwmacc_vf_f32m2(qk_acc2, qv_buffer[2 * DK + d], k_vec, qk_vl); + qk_acc3 = __riscv_vfwmacc_vf_f32m2(qk_acc3, qv_buffer[3 * DK + d], k_vec, qk_vl); + } + + qk_acc0 = __riscv_vfmul_vf_f32m2(qk_acc0, scale, qk_vl); + qk_acc1 = __riscv_vfmul_vf_f32m2(qk_acc1, scale, qk_vl); + qk_acc2 = __riscv_vfmul_vf_f32m2(qk_acc2, scale, qk_vl); + qk_acc3 = __riscv_vfmul_vf_f32m2(qk_acc3, scale, qk_vl); + + __riscv_vse32_v_f32m2(kq_tile_buffer + 0 * ic_tile_step, qk_acc0, qk_vl); + __riscv_vse32_v_f32m2(kq_tile_buffer + 1 * ic_tile_step, qk_acc1, qk_vl); + __riscv_vse32_v_f32m2(kq_tile_buffer + 2 * ic_tile_step, qk_acc2, qk_vl); + __riscv_vse32_v_f32m2(kq_tile_buffer + 3 * ic_tile_step, qk_acc3, qk_vl); + } else { + static_assert(QLEN == 2, "unsupported QLEN"); + + const size_t qk_vl = __riscv_vsetvl_e16m1(ic_step); + vfloat32m2_t qk_acc0 = __riscv_vfmv_v_f_f32m2(0.0f, qk_vl); + vfloat32m2_t qk_acc1 = __riscv_vfmv_v_f_f32m2(0.0f, qk_vl); + + for (int64_t d = 0; d < DK; ++d) { + const vfloat16m1_t k_vec = __riscv_vle16_v_f16m1(k_tile_pack + d * ic_tile_step, qk_vl); + qk_acc0 = __riscv_vfwmacc_vf_f32m2(qk_acc0, qv_buffer[0 * DK + d], k_vec, qk_vl); + qk_acc1 = __riscv_vfwmacc_vf_f32m2(qk_acc1, qv_buffer[1 * DK + d], k_vec, qk_vl); + } + + qk_acc0 = __riscv_vfmul_vf_f32m2(qk_acc0, scale, qk_vl); + qk_acc1 = __riscv_vfmul_vf_f32m2(qk_acc1, scale, qk_vl); + + __riscv_vse32_v_f32m2(kq_tile_buffer + 0 * ic_tile_step, qk_acc0, qk_vl); + __riscv_vse32_v_f32m2(kq_tile_buffer + 1 * ic_tile_step, qk_acc1, qk_vl); + } + + for (int i = 0; i < QLEN; ++i) { + float * row_ptr = kq_tile_buffer + i * ic_tile_step; + const float tile_max = + rvv_softcap_add_max_inplace_f32(row_ptr, mv_tile_pack, ic_step, logit_softcap); + + const float Mold = M[i]; + + if (tile_max > Mold) { + const float ms = expf(Mold - tile_max); + M[i] = tile_max; + S[i] *= ms; + + vfloat16m2_t VKQ16_v = __riscv_vle16_v_f16m2(kq16_buffer + i * DV, DV); + VKQ16_v = __riscv_vfmul_vf_f16m2(VKQ16_v, (_Float16) ms, DV); + __riscv_vse16_v_f16m2(kq16_buffer + i * DV, VKQ16_v, DV); + } + + S[i] += rvv_softmax_exp_inplace_f32(row_ptr, ic_step, M[i]); + } + + if constexpr (QLEN == 4) { + vfloat16m2_t pv_acc0 = __riscv_vle16_v_f16m2(kq16_buffer + 0 * DV, DV); + vfloat16m2_t pv_acc1 = __riscv_vle16_v_f16m2(kq16_buffer + 1 * DV, DV); + vfloat16m2_t pv_acc2 = __riscv_vle16_v_f16m2(kq16_buffer + 2 * DV, DV); + vfloat16m2_t pv_acc3 = __riscv_vle16_v_f16m2(kq16_buffer + 3 * DV, DV); + + for (int64_t tk = 0; tk < ic_step; ++tk) { + const vfloat16m2_t v16 = __riscv_vle16_v_f16m2(v_tile_pack + tk * DV, DV); + pv_acc0 = + __riscv_vfmacc_vf_f16m2(pv_acc0, (_Float16) kq_tile_buffer[0 * ic_tile_step + tk], v16, DV); + pv_acc1 = + __riscv_vfmacc_vf_f16m2(pv_acc1, (_Float16) kq_tile_buffer[1 * ic_tile_step + tk], v16, DV); + pv_acc2 = + __riscv_vfmacc_vf_f16m2(pv_acc2, (_Float16) kq_tile_buffer[2 * ic_tile_step + tk], v16, DV); + pv_acc3 = + __riscv_vfmacc_vf_f16m2(pv_acc3, (_Float16) kq_tile_buffer[3 * ic_tile_step + tk], v16, DV); + } + + __riscv_vse16_v_f16m2(kq16_buffer + 0 * DV, pv_acc0, DV); + __riscv_vse16_v_f16m2(kq16_buffer + 1 * DV, pv_acc1, DV); + __riscv_vse16_v_f16m2(kq16_buffer + 2 * DV, pv_acc2, DV); + __riscv_vse16_v_f16m2(kq16_buffer + 3 * DV, pv_acc3, DV); + } else { + static_assert(QLEN == 2, "unsupported QLEN"); + vfloat16m2_t pv_acc0 = __riscv_vle16_v_f16m2(kq16_buffer + 0 * DV, DV); + vfloat16m2_t pv_acc1 = __riscv_vle16_v_f16m2(kq16_buffer + 1 * DV, DV); + + for (int64_t tk = 0; tk < ic_step; ++tk) { + const vfloat16m2_t v16 = __riscv_vle16_v_f16m2(v_tile_pack + tk * DV, DV); + pv_acc0 = + __riscv_vfmacc_vf_f16m2(pv_acc0, (_Float16) kq_tile_buffer[0 * ic_tile_step + tk], v16, DV); + pv_acc1 = + __riscv_vfmacc_vf_f16m2(pv_acc1, (_Float16) kq_tile_buffer[1 * ic_tile_step + tk], v16, DV); + } + + __riscv_vse16_v_f16m2(kq16_buffer + 0 * DV, pv_acc0, DV); + __riscv_vse16_v_f16m2(kq16_buffer + 1 * DV, pv_acc1, DV); + } + + ic_step = 0; + } + } + } else { + for (int64_t ic = 0; ic < nek1; ++ic) { + const float mv = mp ? slope * ((_Float16 *) mp)[ic] : 0.0f; + + const char * k_data = k_data_row + ic * nbk1; + const char * v_data = v_data_row + ic * nbv1; + + vfloat16m2_t k_data_v; + vfloat16m2_t v_data_v; + + if (mv != -INFINITY) { + k_data_v = __riscv_vle16_v_f16m2((_Float16 *) k_data, DK); + v_data_v = __riscv_vle16_v_f16m2((_Float16 *) v_data, DV); + } else { + continue; + } + + for (int i = 0; i < QLEN; ++i) { + vfloat16m2_t Q_q_v = __riscv_vle16_v_f16m2(qv_buffer + i * DK, DK); + vfloat32m4_t qk_acc_v = __riscv_vfwmul_vv_f32m4(k_data_v, Q_q_v, DK); + float s = reduce_sum_f32m4_vlen1024(qk_acc_v, DK); + s = s * scale; + if (logit_softcap != 0.0f) { + s = logit_softcap * tanhf(s); + } + s += mv; + + const float Mold = M[i]; + + float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value + float vs = 1.0f; // post-softmax KQ value, expf(s - M) + + vfloat16m2_t VKQ16_v = __riscv_vle16_v_f16m2(kq16_buffer + i * DV, DV); + if (s > M[i]) { + // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f + M[i] = s; + ms = expf(Mold - M[i]); + + // V = V*expf(Mold - M) + VKQ16_v = __riscv_vfmul_vf_f16m2(VKQ16_v, ms, DV); + } else { + // no new maximum, ms == 1.0f, vs != 1.0f + vs = expf(s - M[i]); + } + VKQ16_v = __riscv_vfmacc_vf_f16m2(VKQ16_v, vs, v_data_v, DV); + __riscv_vse16_v_f16m2(kq16_buffer + i * DV, VKQ16_v, DV); + S[i] = S[i] * ms + vs; // scale and increment sum with partial sum + } + } + } + + for (int i = 0; i < QLEN; ++i) { + vfloat16m2_t VKQ16_v = __riscv_vle16_v_f16m2(kq16_buffer + i * DV, DV); + vfloat32m4_t VKQ32_v = __riscv_vfwcvt_f_f_v_f32m4(VKQ16_v, DV); + + // sinks + if (sinks[i]) { + const float s = *(sinks[i]); + + float ms = 1.0f; + float vs = 1.0f; + + if (s > M[i]) { + ms = expf(M[i] - s); + M[i] = s; + VKQ32_v = __riscv_vfmul_vf_f32m4(VKQ32_v, ms, DV); + } else { + vs = expf(s - M[i]); + } + + S[i] = S[i] * ms + vs; + } + + // V /= S + const float S_inv = S[i] == 0.0f ? 0.0f : 1.0f / S[i]; + + VKQ32_v = __riscv_vfmul_vf_f32m4(VKQ32_v, S_inv, DV); + + __riscv_vse32_v_f32m4(dst[i], VKQ32_v, DV); + } +} + +static void flash_attn_ext_f16_one_chunk_inner_vlen1024_vf16_m1(const float * pq, + const char * k_data_row, + const char * v_data_row, + const ggml_fp16_t * mp, + const float * sinks, + float * dst, + float scale, + float logit_softcap, + float slope, + int64_t nek1, + int64_t nbk1, + int64_t nbv1, + int64_t DV, + int64_t DK) { + GGML_ASSERT(flash_attn_ext_supported_shape_vlen1024_vf16(DK, DV)); + + float S = 0.0f; // sum + float M = -INFINITY; // maximum KQ value + + vfloat16m2_t VKQ16_v = __riscv_vfmv_v_f_f16m2(0.0f, DV); + + vfloat16m2_t Q_q_v = __riscv_vfncvt_f_f_w_f16m2(__riscv_vle32_v_f32m4(pq, DK), DK); + + for (int64_t ic = 0; ic < nek1; ++ic) { + const float mv = mp ? slope * ((_Float16 *) mp)[ic] : 0.0f; + if (mv == -INFINITY) { + continue; + } + + const char * k_data = k_data_row + ic * nbk1; + + vfloat16m2_t k_data_v = __riscv_vle16_v_f16m2((_Float16 *) k_data, DK); + + vfloat32m4_t qk_acc_v = __riscv_vfwmul_vv_f32m4(k_data_v, Q_q_v, DK); + float s = reduce_sum_f32m4_vlen1024(qk_acc_v, DK); + + s = s * scale; // scale KQ value + + if (logit_softcap != 0.0f) { + s = logit_softcap * tanhf(s); + } + + s += mv; // apply mask + + const float Mold = M; + + float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value + float vs = 1.0f; // post-softmax KQ value, expf(s - M) + + const char * v_data = v_data_row + ic * nbv1; + + vfloat16m2_t v_data_v = __riscv_vle16_v_f16m2((_Float16 *) v_data, DV); + + if (s > M) { + // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f + M = s; + ms = expf(Mold - M); + + // V = V*expf(Mold - M) + VKQ16_v = __riscv_vfmul_vf_f16m2(VKQ16_v, ms, DV); + } else { + // no new maximum, ms == 1.0f, vs != 1.0f + vs = expf(s - M); + } + + VKQ16_v = __riscv_vfmacc_vf_f16m2(VKQ16_v, vs, v_data_v, DV); + + S = S * ms + vs; // scale and increment sum with partial sum + } + + vfloat32m4_t VKQ32_v = __riscv_vfwcvt_f_f_v_f32m4(VKQ16_v, DV); + + // sinks + if (sinks) { + const float s = *sinks; + + float ms = 1.0f; + float vs = 1.0f; + + if (s > M) { + ms = expf(M - s); + M = s; + VKQ32_v = __riscv_vfmul_vf_f32m4(VKQ32_v, ms, DV); + } else { + vs = expf(s - M); + } + + S = S * ms + vs; + } + + // V /= S + const float S_inv = S == 0.0f ? 0.0f : 1.0f / S; + + VKQ32_v = __riscv_vfmul_vf_f32m4(VKQ32_v, S_inv, DV); + + __riscv_vse32_v_f32m4(dst, VKQ32_v, DV); +} + +} // namespace + +void memcpy1d(void * dst, const void * src, int64_t size) { + size_t byte_size_all = size; + size_t vlen = __riscv_vlenb() * 8; + if (vlen == 256) { + // 1024 bytes + __asm__ volatile( + // + "srli t0, %[size], 10 \n\t" + "blez t0, memcpy_tail%= \n\t" + "vsetvli t1, x0, e8, m8, tu, mu \n\t" + "memcpy_main_loop%=: \n\t" + "addi t0, t0, -1 \n\t" + "vle8.v v0, (%[s]) \n\t" + "addi %[s], %[s], 256 \n\t" + "vle8.v v8, (%[s]) \n\t" + "addi %[s], %[s], 256 \n\t" + "vle8.v v16, (%[s]) \n\t" + "addi %[s], %[s], 256 \n\t" + "vle8.v v24, (%[s]) \n\t" + "addi %[s], %[s], 256 \n\t" + // + "vse8.v v0, (%[d]) \n\t" + "addi %[d], %[d], 256 \n\t" + "vse8.v v8, (%[d]) \n\t" + "addi %[d], %[d], 256 \n\t" + "vse8.v v16, (%[d]) \n\t" + "addi %[d], %[d], 256 \n\t" + "vse8.v v24, (%[d]) \n\t" + "addi %[d], %[d], 256 \n\t" + // + "bnez t0, memcpy_main_loop%= \n\t" + "memcpy_tail%=: \n\t" + "andi t1, %[size], 1023 \n\t" + "blez t1, out%= \n\t" + "memcpy_tail_loop%=: \n\t" + "vsetvli t0, t1, e8, m8, tu, mu \n\t" + "sub t1, t1, t0 \n\t" + "vle8.v v0, (%[s]) \n\t" + "add %[s], %[s], t0 \n\t" + "vse8.v v0, (%[d]) \n\t" + "add %[d], %[d], t0 \n\t" + "bnez t1, memcpy_tail_loop%= \n\t" + "out%=: \n\t" + : [s] "+r"(src), [d] "+r"(dst) + : [size] "r"(byte_size_all) + : "cc", "t0", "t1"); + } else if (vlen == 1024) { + // 2048 bytes + __asm__ volatile( + // + "srli t0, %[size], 11 \n\t" + "blez t0, memcpy_tail%= \n\t" + "vsetvli t1, x0, e8, m8, tu, mu \n\t" + "addi t2, %[s], 1024 \n\t" + "addi t3, %[d], 1024 \n\t" + "li t5, 2048 \n\t" + "memcpy_main_loop%=: \n\t" + "addi t0, t0, -1 \n\t" + "vle8.v v0, (%[s]) \n\t" + "add %[s], %[s], t5 \n\t" + "vle8.v v8, (t2) \n\t" + "add t2, t2, t5 \n\t" + // + "vse8.v v0, (%[d]) \n\t" + "add %[d], %[d], t5 \n\t" + "vse8.v v8, (t3) \n\t" + "add t3, t3, t5 \n\t" + // + "bnez t0, memcpy_main_loop%= \n\t" + "memcpy_tail%=: \n\t" + "andi t1, %[size], 2047 \n\t" + "blez t1, out%= \n\t" + "memcpy_tail_loop%=: \n\t" + "vsetvli t0, t1, e8, m2, tu, mu \n\t" + "sub t1, t1, t0 \n\t" + "vle8.v v0, (%[s]) \n\t" + "add %[s], %[s], t0 \n\t" + "vse8.v v0, (%[d]) \n\t" + "add %[d], %[d], t0 \n\t" + "bnez t1, memcpy_tail_loop%= \n\t" + "out%=: \n\t" + : [s] "+r"(src), [d] "+r"(dst) + : [size] "r"(byte_size_all) + : "cc", "t0", "t1", "t2", "t3", "t5"); + } else { + __asm__ volatile( + // + "add t1, %[size], zero \n\t" + "memcpy_tail_loop%=: \n\t" + "vsetvli t0, t1, e8, m8, tu, mu \n\t" + "sub t1, t1, t0 \n\t" + "vle8.v v0, (%[s]) \n\t" + "add %[s], %[s], t0 \n\t" + "vse8.v v0, (%[d]) \n\t" + "add %[d], %[d], t0 \n\t" + "bnez t1, memcpy_tail_loop%= \n\t" + : [s] "+r"(src), [d] "+r"(dst) + : [size] "r"(byte_size_all) + : "cc", "t0", "t1", "t2", "t4", "t3"); + } +} + +void memcpy2d(void * dst, int64_t dst_stride, const void * src, int64_t src_stride, int64_t tile_rows, int64_t size) { + for (int64_t i = 0; i < tile_rows; ++i) { + memcpy1d((char *) dst + i * dst_stride, (const char *) src + i * src_stride, size); + } +} + +void forward_flash_attn_ext_f16_one_chunk_vlen1024_vf16(const ggml_compute_params * params, + ggml_tensor * dst, + int ir0, + int ir1, + void * tcm_buffer, + size_t tcm_buffer_size) { + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + + GGML_TENSOR_LOCALS(int64_t, neq, q, ne) + GGML_TENSOR_LOCALS(size_t, nbq, q, nb) + GGML_TENSOR_LOCALS(int64_t, nek, k, ne) + GGML_TENSOR_LOCALS(size_t, nbk, k, nb) + GGML_TENSOR_LOCALS(int64_t, nev, v, ne) + GGML_TENSOR_LOCALS(size_t, nbv, v, nb) + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + + const int64_t DK = nek0; + const int64_t DV = nev0; + const int64_t N = neq1; + + GGML_ASSERT(flash_attn_ext_supported_shape_vlen1024_vf16(DK, DV)); + + // broadcast factors + const int64_t rk2 = neq2 / nek2; + const int64_t rk3 = neq3 / nek3; + + const int64_t rv2 = neq2 / nev2; + const int64_t rv3 = neq3 / nev3; + + // parallelize by q rows using ggml_vec_dot_f32 + + float scale = *((float *) dst->op_params + 0); + float max_bias = *((float *) dst->op_params + 1); + float logit_softcap = *((float *) dst->op_params + 2); + + if (logit_softcap != 0) { + scale /= logit_softcap; + } + + const uint32_t n_head = neq2; + const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); + + const float m0 = powf(2.0f, -(max_bias) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + const int KV_row_size = DK * sizeof(_Float16) + DV * sizeof(_Float16); + + int ith = params->ith; + int ir_step = 1; + for (int ir = ir0; ir < ir1; ir += ir_step) { + // q indices + const int iq3 = ir / (neq2 * neq1); + const int iq2 = (ir - iq3 * neq2 * neq1) / neq1; + const int iq1 = (ir - iq3 * neq2 * neq1 - iq2 * neq1); + + const int iq3_1 = (ir + 1) / (neq2 * neq1); + const int iq2_1 = (ir + 1 - iq3_1 * neq2 * neq1) / neq1; + const int iq1_1 = (ir + 1 - iq3_1 * neq2 * neq1 - iq2_1 * neq1); + + const int iq3_2 = (ir + 2) / (neq2 * neq1); + const int iq2_2 = (ir + 2 - iq3_2 * neq2 * neq1) / neq1; + const int iq1_2 = (ir + 2 - iq3_2 * neq2 * neq1 - iq2_2 * neq1); + + const int iq3_3 = (ir + 3) / (neq2 * neq1); + const int iq2_3 = (ir + 3 - iq3_3 * neq2 * neq1) / neq1; + const int iq1_3 = (ir + 3 - iq3_3 * neq2 * neq1 - iq2_3 * neq1); + + const uint32_t h = iq2; // head index + const float slope = + (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2 * (h - n_head_log2) + 1) : 1.0f; + + const ggml_fp16_t * mp = + mask ? (ggml_fp16_t *) ((char *) mask->data + iq1 * mask->nb[1] + (iq2 % mask->ne[2]) * mask->nb[2] + + (iq3 % mask->ne[3]) * mask->nb[3]) : + NULL; + + const bool mp_equal_2 = iq1_1 == iq1 && (iq2 % mask->ne[2]) == (iq2_1 % mask->ne[2]) && + (iq3 % mask->ne[3]) == (iq3_1 % mask->ne[3]); + + const bool mp_equal_4 = mp_equal_2 && iq1_2 == iq1 && (iq2 % mask->ne[2]) == (iq2_2 % mask->ne[2]) && + (iq3 % mask->ne[3]) == (iq3_2 % mask->ne[3]) && iq1_3 == iq1 && + (iq2 % mask->ne[2]) == (iq2_3 % mask->ne[2]) && + (iq3 % mask->ne[3]) == (iq3_3 % mask->ne[3]); + + // k indices + const int ik3 = iq3 / rk3; + const int ik2 = iq2 / rk2; + + const int ik3_1 = iq3_1 / rk3; + const int ik2_1 = iq2_1 / rk2; + + const int ik3_2 = iq3_2 / rk3; + const int ik2_2 = iq2_2 / rk2; + + const int ik3_3 = iq3_3 / rk3; + const int ik2_3 = iq2_3 / rk2; + + // v indices + const int iv3 = iq3 / rv3; + const int iv2 = iq2 / rv2; + + const int iv3_1 = iq3_1 / rv3; + const int iv2_1 = iq2_1 / rv2; + + const int iv3_2 = iq3_2 / rv3; + const int iv2_2 = iq2_2 / rv2; + + const int iv3_3 = iq3_3 / rv3; + const int iv2_3 = iq2_3 / rv2; + + const float * pq = (const float *) ((char *) q->data + (iq1 * nbq1 + iq2 * nbq2 + iq3 * nbq3)); + + std::array pq_buffer; + std::array sinks_buffer; + std::array dst_buffer; + + if (tcm_buffer != nullptr && 4 * KV_row_size < tcm_buffer_size && ir < (ir1 - 3) && mp_equal_4 && + ik3_3 == ik3 && ik2_3 == ik2 && iv3_3 == iv3 && iv2_3 == iv2 && ik3_2 == ik3 && ik2_2 == ik2 && + iv3_2 == iv3 && iv2_2 == iv2 && ik3_1 == ik3 && ik2_1 == ik2 && iv3_1 == iv3 && iv2_1 == iv2) { + ir_step = 4; + + pq_buffer[0] = (float *) ((char *) q->data + (iq1 * nbq1 + iq2 * nbq2 + iq3 * nbq3)); + pq_buffer[1] = (float *) ((char *) q->data + (iq1_1 * nbq1 + iq2_1 * nbq2 + iq3_1 * nbq3)); + pq_buffer[2] = (float *) ((char *) q->data + (iq1_2 * nbq1 + iq2_2 * nbq2 + iq3_2 * nbq3)); + pq_buffer[3] = (float *) ((char *) q->data + (iq1_3 * nbq1 + iq2_3 * nbq2 + iq3_3 * nbq3)); + + sinks_buffer[0] = sinks ? ((float *) ((char *) sinks->data)) + iq2 : nullptr; + sinks_buffer[1] = sinks ? ((float *) ((char *) sinks->data)) + iq2_1 : nullptr; + sinks_buffer[2] = sinks ? ((float *) ((char *) sinks->data)) + iq2_2 : nullptr; + sinks_buffer[3] = sinks ? ((float *) ((char *) sinks->data)) + iq2_3 : nullptr; + + dst_buffer[0] = (float *) ((char *) dst->data + (iq3 * ne2 * ne1 + iq2 + iq1 * ne1) * nb1); + dst_buffer[1] = (float *) ((char *) dst->data + (iq3_1 * ne2 * ne1 + iq2_1 + iq1_1 * ne1) * nb1); + dst_buffer[2] = (float *) ((char *) dst->data + (iq3_2 * ne2 * ne1 + iq2_2 + iq1_2 * ne1) * nb1); + dst_buffer[3] = (float *) ((char *) dst->data + (iq3_3 * ne2 * ne1 + iq2_3 + iq1_3 * ne1) * nb1); + + flash_attn_ext_f16_one_chunk_inner_vlen1024_vf16_mrow<4>( // + pq_buffer.data(), // + (const char *) k->data + (ik2 * nbk2 + ik3 * nbk3), // + (const char *) v->data + (iv2 * nbv2 + iv3 * nbv3), // + mp, // + sinks_buffer.data(), // + dst_buffer.data(), // + scale, logit_softcap, slope, nek1, nbk1, nbv1, DV, DK, tcm_buffer, tcm_buffer_size); + } else if (tcm_buffer != nullptr && 2 * KV_row_size < tcm_buffer_size && ir < (ir1 - 1) && mp_equal_2 && + ik3_1 == ik3 && ik2_1 == ik2 && iv3_1 == iv3 && iv2_1 == iv2) { + ir_step = 2; + + pq_buffer[0] = (float *) ((char *) q->data + (iq1 * nbq1 + iq2 * nbq2 + iq3 * nbq3)); + pq_buffer[1] = (float *) ((char *) q->data + (iq1_1 * nbq1 + iq2_1 * nbq2 + iq3_1 * nbq3)); + + sinks_buffer[0] = sinks ? ((float *) ((char *) sinks->data)) + iq2 : nullptr; + sinks_buffer[1] = sinks ? ((float *) ((char *) sinks->data)) + iq2_1 : nullptr; + + dst_buffer[0] = (float *) ((char *) dst->data + (iq3 * ne2 * ne1 + iq2 + iq1 * ne1) * nb1); + dst_buffer[1] = (float *) ((char *) dst->data + (iq3_1 * ne2 * ne1 + iq2_1 + iq1_1 * ne1) * nb1); + + flash_attn_ext_f16_one_chunk_inner_vlen1024_vf16_mrow<2>( // + pq_buffer.data(), // + (const char *) k->data + (ik2 * nbk2 + ik3 * nbk3), // + (const char *) v->data + (iv2 * nbv2 + iv3 * nbv3), // + mp, // + sinks_buffer.data(), // + dst_buffer.data(), // + scale, logit_softcap, slope, nek1, nbk1, nbv1, DV, DK, tcm_buffer, tcm_buffer_size); + } else { + ir_step = 1; + flash_attn_ext_f16_one_chunk_inner_vlen1024_vf16_m1( // + pq, // + (const char *) k->data + (ik2 * nbk2 + ik3 * nbk3), // + (const char *) v->data + (iv2 * nbv2 + iv3 * nbv3), // + mp, // + sinks ? ((float *) ((char *) sinks->data)) + h : nullptr, // + (float *) ((char *) dst->data + (iq3 * ne2 * ne1 + iq2 + iq1 * ne1) * nb1), // + scale, logit_softcap, slope, nek1, nbk1, nbv1, DV, DK); + } + } +} + +void forward_flash_attn_ext_f16_tiled_vlen1024_vf16(const ggml_compute_params * params, + ggml_tensor * dst, + int ir0, + int ir1, + void * tcm_buffer, + size_t tcm_buffer_size) { + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + + GGML_TENSOR_LOCALS(int64_t, neq, q, ne) + GGML_TENSOR_LOCALS(size_t, nbq, q, nb) + GGML_TENSOR_LOCALS(int64_t, nek, k, ne) + GGML_TENSOR_LOCALS(size_t, nbk, k, nb) + GGML_TENSOR_LOCALS(int64_t, nev, v, ne) + GGML_TENSOR_LOCALS(size_t, nbv, v, nb) + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + + const int64_t DK = nek0; + const int64_t DV = nev0; + const int64_t N = neq1; + + GGML_ASSERT(flash_attn_ext_supported_shape_vlen1024_vf16(DK, DV)); + + GGML_ASSERT(ne0 == DV); + GGML_ASSERT(ne2 == N); + + // input tensor rows must be contiguous + GGML_ASSERT(nbq0 == ggml_type_size(q->type)); + GGML_ASSERT(nbk0 == ggml_type_size(k->type)); + GGML_ASSERT(nbv0 == ggml_type_size(v->type)); + + GGML_ASSERT(neq0 == DK); + GGML_ASSERT(nek0 == DK); + GGML_ASSERT(nev0 == DV); + + GGML_ASSERT(neq1 == N); + + // dst cannot be transposed or permuted + GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(nb0 <= nb1); + GGML_ASSERT(nb1 <= nb2); + GGML_ASSERT(nb2 <= nb3); + + GGML_ASSERT(k->type == v->type); + const ggml_type kv_type = k->type; + + // broadcast factors + const int64_t rk2 = neq2 / nek2; + const int64_t rk3 = neq3 / nek3; + + const int64_t rv2 = neq2 / nev2; + const int64_t rv3 = neq3 / nev3; + + float * param_list = (float *) dst->op_params; + float scale = param_list[0]; + float max_bias = param_list[1]; + float logit_softcap = param_list[2]; + + if (logit_softcap != 0) { + scale /= logit_softcap; + } + + const uint32_t n_head = neq2; + const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); + + const float m0 = powf(2.0f, -(max_bias) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + int ith = params->ith; + + static constexpr int Q_TILE_SZ = ggml_fa_tile_config::Q; + static constexpr int KV_TILE_SZ = ggml_fa_tile_config::KV; + + // Per-thread scratch layout: + // Q_f32: Q_TILE_SZ * DK + // KQ: Q_TILE_SZ * KV_TILE_SZ + // mask32: Q_TILE_SZ * KV_TILE_SZ + // VKQ32: Q_TILE_SZ * DV + // V32: KV_TILE_SZ * DV + // K_f32: DK * KV_TILE_SZ (transposed K tile) + float * base = (float *) params->wdata + ith * (Q_TILE_SZ * DK + 2 * Q_TILE_SZ * KV_TILE_SZ + Q_TILE_SZ * DV + + KV_TILE_SZ * DV + KV_TILE_SZ * DK + CACHE_LINE_SIZE_F32); + const size_t base_size = + (Q_TILE_SZ * DK + 2 * Q_TILE_SZ * KV_TILE_SZ + Q_TILE_SZ * DV + KV_TILE_SZ * DV + KV_TILE_SZ * DK) * + sizeof(float) + + CACHE_LINE_SIZE_F32; + + if (base_size <= tcm_buffer_size && tcm_buffer != nullptr) { + base = (float *) tcm_buffer; + } + + float S_M_Buf[Q_TILE_SZ * 2]; // buffer to hold S, M, bias for one tile to reduce register pressure in main loop + float * S = S_M_Buf; + float * M = S_M_Buf + Q_TILE_SZ; + + int ir = ir0; + while (ir < ir1) { + // q indices for the start of this tile + const int iq3 = ir / (neq2 * neq1); + const int iq2 = (ir - iq3 * neq2 * neq1) / neq1; + const int iq1 = (ir - iq3 * neq2 * neq1 - iq2 * neq1); + + // Number of valid rows in this tile: + // - limited by tile size (Q_TILE_SZ) + // - limited by chunk boundary (ir1 - ir) + // - limited by head boundary (neq1 - iq1) to avoid crossing into next head + const int tile_rows = MIN(Q_TILE_SZ, MIN((int) (ir1 - ir), (int) (neq1 - iq1))); + GGML_ASSERT(tile_rows > 0); + + const uint32_t h = iq2; // head index + const float slope = + (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2 * (h - n_head_log2) + 1) : 1.0f; + + for (int i = 0; i < Q_TILE_SZ; ++i) { + S[i] = 0.; + M[i] = -INFINITY; + } + + float * Q_f32 = base; + float * KQ = (float *) ((char *) base + Q_TILE_SZ * DK * sizeof(float)); + float * mask32 = KQ + Q_TILE_SZ * KV_TILE_SZ; + float * VKQ32 = mask32 + Q_TILE_SZ * KV_TILE_SZ; + float * V32 = VKQ32 + Q_TILE_SZ * DV; + float * K_f32 = V32 + KV_TILE_SZ * DV; + _Float16 * Q_f16 = (_Float16 *) Q_f32; + _Float16 * V_f16 = (_Float16 *) V32; + _Float16 * K_f16 = (_Float16 *) K_f32; + + rvv_zero_f32(VKQ32, Q_TILE_SZ * DV); + + // k indices + const int ik3 = iq3 / rk3; + const int ik2 = iq2 / rk2; + + // v indices + const int iv3 = iq3 / rv3; + const int iv2 = iq2 / rv2; + + const float * pq = (const float *) ((char *) q->data + (iq1 * nbq1 + iq2 * nbq2 + iq3 * nbq3)); + if (kv_type == GGML_TYPE_F16) { + rvv_pack_f32_as_scaled_f16((uint8_t *) Q_f16, DK * sizeof(_Float16), (uint8_t *) pq, nbq1, tile_rows, DK, + scale); + } else { + memcpy2d(Q_f32, DK * sizeof(float), pq, nbq1, tile_rows, DK * sizeof(float)); + } + + for (int64_t ic = 0; ic < nek1; ic += KV_TILE_SZ) { + const int kv_tile = (int) std::min((int64_t) KV_TILE_SZ, nek1 - ic); + + rvv_zero_f32(K_f32, DK * KV_TILE_SZ); + rvv_zero_f32(V32, KV_TILE_SZ * DV); + + // skip the tile entirely if all the masks are -inf + if (mask) { + bool can_skip = true; + const ggml_fp16_t * mp_row = + (const ggml_fp16_t *) ((const char *) mask->data + iq1 * mask->nb[1] + + (iq2 % mask->ne[2]) * mask->nb[2] + (iq3 % mask->ne[3]) * mask->nb[3]); + rvv_pack_scaled_f16_as_f32(mask32, KV_TILE_SZ * sizeof(float), mp_row + ic, mask->nb[1], tile_rows, + kv_tile, slope); + + for (int tq = 0; tq < tile_rows; tq++) { + for (int tk = 0; tk < kv_tile; tk++) { + if (mask32[tq * KV_TILE_SZ + tk] != -INFINITY) { + can_skip = false; + } + } + // Pad remaining mask entries with -inf + for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) { + mask32[tq * KV_TILE_SZ + tk] = -INFINITY; + } + } + + if (can_skip) { + continue; + } + } + + if (kv_type == GGML_TYPE_F16) { + rvv_transposed_s16_mn_to_nm((int8_t *) K_f16, KV_TILE_SZ * sizeof(_Float16), + (int8_t *) k->data + ic * nbk1 + ik2 * nbk2 + ik3 * nbk3, nbk1, kv_tile, + DK); + + int tq = 0; + for (; tq + 3 < tile_rows; tq += 4) { + rvv_qk_dot_tile_f16_x4(KQ + (tq + 0) * KV_TILE_SZ, KQ + (tq + 1) * KV_TILE_SZ, + KQ + (tq + 2) * KV_TILE_SZ, KQ + (tq + 3) * KV_TILE_SZ, + Q_f16 + (tq + 0) * DK, Q_f16 + (tq + 1) * DK, Q_f16 + (tq + 2) * DK, + Q_f16 + (tq + 3) * DK, K_f16, DK, kv_tile); + } + for (; tq < tile_rows; ++tq) { + rvv_qk_dot_tile_f16_x1(KQ + tq * KV_TILE_SZ, Q_f16 + tq * DK, K_f16, DK, kv_tile); + } + } else { + for (int tk = 0; tk < kv_tile; tk++) { + const char * k_data = (const char *) k->data + (ic + tk) * nbk1 + ik2 * nbk2 + ik3 * nbk3; + float * k_col = K_f32 + tk; + const float * k_src = (const float *) k_data; + for (int64_t dk = 0; dk < DK; ++dk) { + k_col[dk * KV_TILE_SZ] = k_src[dk]; + } + } + + for (int tq = 0; tq < tile_rows; ++tq) { + rvv_qk_dot_tile(KQ + tq * KV_TILE_SZ, Q_f32 + tq * DK, K_f32, DK, KV_TILE_SZ, scale); + } + } + + // Set padded KQ entries to -inf so softmax gives them zero weight + if (kv_tile < KV_TILE_SZ) { + for (int tq = 0; tq < tile_rows; tq++) { + for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) { + KQ[tq * KV_TILE_SZ + tk] = -INFINITY; + } + } + } + + if (logit_softcap != 0.0f) { + rvv_softcap_tanh_inplace_f32(KQ, KV_TILE_SZ, tile_rows, KV_TILE_SZ, logit_softcap); + } + + if (mask) { + rvv_add_inplace_f32(KQ, KV_TILE_SZ, mask32, KV_TILE_SZ, tile_rows, KV_TILE_SZ); + } + + bool skip[Q_TILE_SZ] = {}; + + for (int tq = 0; tq < tile_rows; tq++) { + float * kq_row = KQ + tq * KV_TILE_SZ; + + const float tile_max = rvv_max_f32(kq_row, KV_TILE_SZ); + + if (tile_max == -INFINITY) { + skip[tq] = true; + continue; + } + + const float Mold = M[tq]; + const float Mnew = fmaxf(Mold, tile_max); + + if (Mnew > Mold) { + const float ms = expf(Mold - Mnew); + rvv_scale_f32(VKQ32 + tq * DV, ms, DV); + S[tq] *= ms; + } + M[tq] = Mnew; + + S[tq] += rvv_softmax_exp_inplace_f32(kq_row, KV_TILE_SZ, Mnew); + } + + // Pack V as contiguous [KV_TILE_SZ][DV]. + if (kv_type == GGML_TYPE_F16) { + const char * v_data = (const char *) v->data + ic * nbv1 + iv2 * nbv2 + iv3 * nbv3; + memcpy2d(V_f16, DV * sizeof(_Float16), v_data, nbv1, kv_tile, DV * sizeof(_Float16)); + + int tq = 0; + for (; tq + 3 < tile_rows; tq += 4) { + if (skip[tq + 0] || skip[tq + 1] || skip[tq + 2] || skip[tq + 3]) { + for (int i = 0; i < 4; ++i) { + if (!skip[tq + i]) { + rvv_pv_accumulate_f16_x1(VKQ32 + (tq + i) * DV, KQ + (tq + i) * KV_TILE_SZ, V_f16, + KV_TILE_SZ, DV); + } + } + continue; + } + + rvv_pv_accumulate_f16_x4(VKQ32 + (tq + 0) * DV, VKQ32 + (tq + 1) * DV, VKQ32 + (tq + 2) * DV, + VKQ32 + (tq + 3) * DV, KQ + (tq + 0) * KV_TILE_SZ, + KQ + (tq + 1) * KV_TILE_SZ, KQ + (tq + 2) * KV_TILE_SZ, + KQ + (tq + 3) * KV_TILE_SZ, V_f16, KV_TILE_SZ, DV); + } + for (; tq < tile_rows; ++tq) { + if (!skip[tq]) { + rvv_pv_accumulate_f16_x1(VKQ32 + tq * DV, KQ + tq * KV_TILE_SZ, V_f16, KV_TILE_SZ, DV); + } + } + } else { + const char * v_data = (const char *) v->data + ic * nbv1 + iv2 * nbv2 + iv3 * nbv3; + memcpy2d(V32, DV * sizeof(float), v_data, nbv1, kv_tile, DV * sizeof(float)); + + for (int tq = 0; tq < tile_rows; ++tq) { + if (!skip[tq]) { + rvv_pv_accumulate(VKQ32 + tq * DV, KQ + tq * KV_TILE_SZ, V32, KV_TILE_SZ, DV); + } + } + } + } + + // sinks (apply only to valid rows in the tile) + if (sinks) { + const float s = ((float *) ((char *) sinks->data))[h]; + + for (int tq = 0; tq < tile_rows; tq++) { + float ms = 1.0f; + float vs = 1.0f; + + if (s > M[tq]) { + ms = expf(M[tq] - s); + rvv_scale_f32(VKQ32 + tq * DV, ms, DV); + } else { + vs = expf(s - M[tq]); + } + + float S_temp = S[tq] * ms + vs; + S[tq] = S_temp == 0.0f ? 0.0f : 1.0f / S_temp; + } + } else { + for (int tq = 0; tq < tile_rows; tq++) { + const float S_inv = S[tq] == 0.0f ? 0.0f : 1.0f / S[tq]; + S[tq] = S_inv; + } + } + + float * dst_ptr = (float *) ((char *) dst->data + (iq3 * ne2 * ne1 + iq2 + (iq1) *ne1) * nb1); + rvv_pack_scaled_f32_as_f32(dst_ptr, nb1 * ne1, VKQ32, DV * sizeof(float), tile_rows, DV, S); + + ir += tile_rows; + } +} + +void forward_rms_norm_f32(ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + ggml_tensor * dst = op; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); + GGML_ASSERT(src0->nb[0] == sizeof(float)); + + int ith = params->ith; + int nth = params->nth; + + GGML_TENSOR_UNARY_OP_LOCALS + + float epsilon = *((float *) dst->op_params); + + GGML_ASSERT(epsilon > 0.0f); + + auto * input = (char *) src0->data; + auto * output = (char *) dst->data; + + const auto hidden_size = ne00; + const auto task_count = ne01 * ne02 * ne03; + const auto task_per_thread = (task_count + nth - 1) / nth; + + const auto task_begin = ith * task_per_thread; + const auto task_end = std::min((ith + 1) * task_per_thread, task_count); + + for (auto task_idx = task_begin; task_idx < task_end; task_idx++) { + int64_t i03 = task_idx / (ne02 * ne01); + int64_t i02 = (task_idx - i03 * ne02 * ne01) / ne01; + int64_t i01 = (task_idx - i03 * ne02 * ne01 - i02 * ne01); + + auto * p_input = (float *) (input + i01 * nb01 + i02 * nb02 + i03 * nb03); + auto * p_output = (float *) (output + i01 * nb1 + i02 * nb2 + i03 * nb3); + auto * p_temp_output = p_output; + + size_t gvl = __riscv_vsetvlmax_e32m4(); + vfloat32m4_t sum_sq = __riscv_vfmv_v_f_f32m4(0.f, gvl); + int64_t length = hidden_size; + while (length > 0) { + gvl = __riscv_vsetvl_e32m4(length); + vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_input, gvl); + sum_sq = __riscv_vfmacc_vv_f32m4(sum_sq, src_data, src_data, gvl); + __riscv_vse32_v_f32m4(p_temp_output, src_data, gvl); + + p_input += gvl; + p_temp_output += gvl; + length -= gvl; + } + + gvl = __riscv_vsetvlmax_e32m1(); + vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.f, gvl); + vfloat32m1_t mean_square_v = + __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum_sq, 0), __riscv_vget_v_f32m4_f32m1(sum_sq, 1), gvl); + + mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 2), gvl); + mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 3), gvl); + mean_square_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_square_v, zero_v, gvl); + + float mean_square = __riscv_vfmv_f_s_f32m1_f32(mean_square_v); + mean_square /= hidden_size; + + mean_square = sqrt(mean_square + epsilon); + + mean_square = 1.0f / mean_square; + length = hidden_size; + p_temp_output = p_output; + + while (length > 0) { + gvl = __riscv_vsetvl_e32m4(length); + vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl); + src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl); + __riscv_vse32_v_f32m4(p_output, src_data, gvl); + p_temp_output += gvl; + p_output += gvl; + length -= gvl; + } + } +} + +template +void quantize_a_nrow_i8_ref(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr) { + int64_t a_blk_stride = q8_blk_size(blk_len, true); + int64_t a_nrow_block_stride = a_blk_stride * MB_ROWS; + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_nrow_block_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float) * MB_ROWS); + int8_t * quant_a_blk = + reinterpret_cast(quant_a_ptr + sizeof(float) * MB_ROWS + sizeof(int16_t) * MB_ROWS); + + for (size_t row = 0; row < MB_ROWS; row++) { + float max_abs_a = 0.0f; + for (size_t bk = 0; bk < blk_len; bk++) { + max_abs_a = std::max(max_abs_a, std::abs(a_ptr[row * count_k + k + bk])); + } + + float rep_scale_a = ((1 << 7) - 1) / max_abs_a; + scale_a_ptr[row] = 1 / rep_scale_a; + + int16_t a_sum = 0; + for (size_t bk = 0; bk < blk_len; bk++) { + const int8_t quantized = static_cast( + std::clamp(std::nearbyintf(a_ptr[row * count_k + k + bk] * rep_scale_a), -128.0f, 127.0f)); + quant_a_blk[row * blk_len + bk] = quantized; + a_sum += quantized; + } + a_sum_ptr[row] = -a_sum; + } + } +} + +template +void quantize_a_nrow_i8_hp_ref(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr) { + constexpr size_t k_subblk_len = 32; + const size_t subblk_count = blk_len / k_subblk_len; + + GGML_ASSERT(blk_len == 256); + + float scale_temp[8] = { 0.0f }; + int64_t a_blk_stride = q8_hp_blk_size(blk_len, true, true); + int64_t a_nrow_block_stride = a_blk_stride * MB_ROWS; + int64_t a_subblk_stride = q8_hp_blk_size(k_subblk_len, false, false) * MB_ROWS; + + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_nrow_block_stride) { + _Float16 * a_sum_ptr = reinterpret_cast<_Float16 *>(quant_a_ptr + a_subblk_stride * subblk_count); + + float scale_avg = 0.0f; + for (size_t kk = 0; kk < subblk_count; kk++) { + float max_abs_a = 0.0f; + for (size_t row = 0; row < MB_ROWS; row++) { + for (size_t bk = 0; bk < k_subblk_len; bk++) { + max_abs_a = std::max(max_abs_a, std::abs(a_ptr[row * count_k + k + bk + kk * k_subblk_len])); + } + } + scale_temp[kk] = max_abs_a / ((1 << 7) - 1); + scale_avg += scale_temp[kk]; + } + + scale_avg /= subblk_count; + float scale_factor = 1.0f / scale_avg; + + _Float16 * scale_avg_ptr = + reinterpret_cast<_Float16 *>(quant_a_ptr + a_nrow_block_stride - sizeof(_Float16) * MB_ROWS); + scale_avg_ptr[0] = scale_avg; + + for (size_t kk = 0; kk < subblk_count; kk++) { + uint8_t * a_subblk_base = quant_a_ptr + kk * a_subblk_stride; + _Float16 * scale_a_ptr = reinterpret_cast<_Float16 *>(a_subblk_base); + int8_t * quant_a_blk = reinterpret_cast(a_subblk_base + sizeof(_Float16) * MB_ROWS); + + scale_a_ptr[0] = static_cast<_Float16>(scale_temp[kk] * scale_factor); + + const float rep_scale_a = 1.0f / scale_temp[kk]; + + for (size_t row = 0; row < MB_ROWS; row++) { + int16_t a_sum = 0; + for (size_t bk = 0; bk < k_subblk_len; bk++) { + const int8_t quantized = static_cast( + std::clamp(std::nearbyintf(a_ptr[row * count_k + k + bk + kk * k_subblk_len] * rep_scale_a), + -128.0f, 127.0f)); + quant_a_blk[row * k_subblk_len + bk] = quantized; + a_sum += quantized; + } + a_sum_ptr[row * subblk_count + kk] = static_cast<_Float16>(-a_sum) * static_cast<_Float16>(8.0f); + } + } + } +} + +template +void quantize_a_nrow_i8k_ref(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr) { + int64_t a_blk_stride = q8k_blk_size(256); + int64_t a_nrow_block_stride = a_blk_stride * MB_ROWS; + int64_t a_sum_size = 256 / 16; + + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_nrow_block_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float) * MB_ROWS); + int8_t * quant_a_blk = + reinterpret_cast(quant_a_ptr + sizeof(float) * MB_ROWS + sizeof(int16_t) * a_sum_size * MB_ROWS); + + for (size_t row = 0; row < MB_ROWS; row++) { + float max_a = 0.0f; + float max_abs_a = 0.0f; + for (size_t bk = 0; bk < blk_len; bk++) { + float ax = std::abs(a_ptr[row * count_k + k + bk]); + if (ax > max_abs_a) { + max_abs_a = ax; + max_a = a_ptr[row * count_k + k + bk]; + } + } + + if (!max_abs_a) { + scale_a_ptr[row] = 0; + for (size_t bki = 0; bki < a_sum_size; bki++) { + for (size_t bk = bki * 16; bk < (bki + 1) * 16; bk++) { + quant_a_blk[row * blk_len + bk] = 0; + } + a_sum_ptr[row * a_sum_size + bki] = 0; + } + continue; + } + + float rep_scale_a = ((1 << 7) - 1) / max_abs_a; + scale_a_ptr[row] = 1 / rep_scale_a; + + for (size_t bki = 0; bki < a_sum_size; bki++) { + int16_t a_sum = 0; + for (size_t bk = bki * 16; bk < (bki + 1) * 16; bk++) { + const int8_t quantized = static_cast( + std::clamp(std::nearbyintf(a_ptr[row * count_k + k + bk] * rep_scale_a), -128.0f, 127.0f)); + quant_a_blk[row * blk_len + bk] = quantized; + a_sum += quantized; + } + a_sum_ptr[row * a_sum_size + bki] = -a_sum; + } + } + } +} + +void quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr) { + GGML_ASSERT(blk_len == 32); + int64_t a_blk_stride = q8_blk_size(blk_len, true); + size_t vlenb = __riscv_vlenb(); + + if (vlenb == 128) { + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_blk_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float)); + int8_t * quant_a_blk = reinterpret_cast(quant_a_ptr + sizeof(float) + sizeof(int16_t)); + + size_t vl = __riscv_vsetvl_e32m1(blk_len); + vfloat32m1_t v_a = __riscv_vle32_v_f32m1(a_ptr + k, vl); + vfloat32m1_t v_a_abs = __riscv_vfabs_v_f32m1(v_a, vl); + + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_a_max = __riscv_vfredmax_vs_f32m1_f32m1(v_a_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_a_max); + + float scale_a = max_abs_a / ((1 << 7) - 1); + float rep_scale_a = scale_a ? 1.0f / scale_a : 0.0f; + scale_a_ptr[0] = scale_a; + + vfloat32m1_t v_a_scale = __riscv_vfmul_vf_f32m1(v_a, rep_scale_a, vl); + vint16mf2_t v_a_quant = __riscv_vfncvt_x_f_w_i16mf2(v_a_scale, vl); + vint8mf4_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8mf4(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8mf4_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[0] = -a_sum; + + __riscv_vse8_v_i8mf4(quant_a_blk, v_a_quant_i8, vl); + } + } else if (vlenb == 32) { + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_blk_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float)); + int8_t * quant_a_blk = reinterpret_cast(quant_a_ptr + sizeof(float) + sizeof(int16_t)); + + size_t vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t v_a = __riscv_vle32_v_f32m4(a_ptr + k, vl); + vfloat32m4_t v_a_abs = __riscv_vfabs_v_f32m4(v_a, vl); + + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_a_max = __riscv_vfredmax_vs_f32m4_f32m1(v_a_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_a_max); + + float scale_a = max_abs_a / ((1 << 7) - 1); + float rep_scale_a = scale_a ? 1.0f / scale_a : 0.0f; + scale_a_ptr[0] = scale_a; + + vfloat32m4_t v_a_scale = __riscv_vfmul_vf_f32m4(v_a, rep_scale_a, vl); + vint16m2_t v_a_quant = __riscv_vfncvt_x_f_w_i16m2(v_a_scale, vl); + vint8m1_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8m1(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8m1_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[0] = -a_sum; + + __riscv_vse8_v_i8m1(quant_a_blk, v_a_quant_i8, vl); + } + } else { + quantize_a_nrow_i8_ref<1>(blk_len, a_ptr, count_k, quant_a_ptr); + } +} + +void quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr) { + GGML_ASSERT(blk_len == 32); + int64_t a_blk_stride = q8_blk_size(blk_len, true); + int64_t a_nrow_block_stride = a_blk_stride * 4; + size_t vlenb = __riscv_vlenb(); + + if (vlenb == 128) { + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_nrow_block_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float) * 4); + int8_t * quant_a_blk = reinterpret_cast(quant_a_ptr + sizeof(float) * 4 + sizeof(int16_t) * 4); + + for (size_t mi = 0; mi < 4; mi++) { + size_t vl = __riscv_vsetvl_e32m1(blk_len); + vfloat32m1_t v_a = __riscv_vle32_v_f32m1(a_ptr + mi * count_k + k, vl); + vfloat32m1_t v_a_abs = __riscv_vfabs_v_f32m1(v_a, vl); + + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_a_max = __riscv_vfredmax_vs_f32m1_f32m1(v_a_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_a_max); + + float scale_a = max_abs_a / ((1 << 7) - 1); + float rep_scale_a = scale_a ? 1.0f / scale_a : 0.0f; + scale_a_ptr[mi] = scale_a; + + vfloat32m1_t v_a_scale = __riscv_vfmul_vf_f32m1(v_a, rep_scale_a, vl); + vint16mf2_t v_a_quant = __riscv_vfncvt_x_f_w_i16mf2(v_a_scale, vl); + vint8mf4_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8mf4(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8mf4_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[mi] = -a_sum; + + __riscv_vse8_v_i8mf4(quant_a_blk + mi * blk_len, v_a_quant_i8, vl); + } + } + } else if (vlenb == 32) { + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_nrow_block_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float) * 4); + int8_t * quant_a_blk = reinterpret_cast(quant_a_ptr + sizeof(float) * 4 + sizeof(int16_t) * 4); + + for (size_t mi = 0; mi < 4; mi++) { + size_t vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t v_a = __riscv_vle32_v_f32m4(a_ptr + mi * count_k + k, vl); + vfloat32m4_t v_a_abs = __riscv_vfabs_v_f32m4(v_a, vl); + + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_a_max = __riscv_vfredmax_vs_f32m4_f32m1(v_a_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_a_max); + + float scale_a = max_abs_a / ((1 << 7) - 1); + float rep_scale_a = scale_a ? 1.0f / scale_a : 0.0f; + scale_a_ptr[mi] = scale_a; + + vfloat32m4_t v_a_scale = __riscv_vfmul_vf_f32m4(v_a, rep_scale_a, vl); + vint16m2_t v_a_quant = __riscv_vfncvt_x_f_w_i16m2(v_a_scale, vl); + vint8m1_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8m1(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8m1_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[mi] = -a_sum; + + __riscv_vse8_v_i8m1(quant_a_blk + mi * blk_len, v_a_quant_i8, vl); + } + } + } else { + quantize_a_nrow_i8_ref<4>(blk_len, a_ptr, count_k, quant_a_ptr); + } +} + +void quantize_a_row_i8_hp(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr) { + constexpr size_t k_subblk_len = 32; + GGML_ASSERT(blk_len == 256); + + constexpr size_t subblk_count = 256 / k_subblk_len; + int64_t a_blk_stride = q8_hp_blk_size(blk_len, true, true); + int64_t a_subblk_stride = q8_hp_blk_size(k_subblk_len, false, false); + size_t vlenb = __riscv_vlenb(); + float scale_temp[subblk_count] = { 0.0f }; + + if (vlenb == 128) { + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_blk_stride) { + _Float16 * a_sum_ptr = reinterpret_cast<_Float16 *>(quant_a_ptr + a_subblk_stride * subblk_count); + _Float16 * scale_avg_ptr = reinterpret_cast<_Float16 *>(quant_a_ptr + a_blk_stride - sizeof(_Float16)); + float scale_avg = 0.0f; + + for (size_t kk = 0; kk < subblk_count; ++kk) { + const float * a_src_ptr = a_ptr + k + kk * k_subblk_len; + + size_t vl = __riscv_vsetvl_e32m1(k_subblk_len); + vfloat32m1_t v_a = __riscv_vle32_v_f32m1(a_src_ptr, vl); + vfloat32m1_t v_a_abs = __riscv_vfabs_v_f32m1(v_a, vl); + + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_a_max = __riscv_vfredmax_vs_f32m1_f32m1(v_a_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_a_max); + + scale_temp[kk] = max_abs_a / ((1 << 7) - 1); + scale_avg += scale_temp[kk]; + } + + scale_avg /= subblk_count; + const float scale_factor = scale_avg ? 1.0f / scale_avg : 0.0f; + scale_avg_ptr[0] = static_cast<_Float16>(scale_avg); + + for (size_t kk = 0; kk < subblk_count; ++kk) { + uint8_t * a_subblk_base = quant_a_ptr + kk * a_subblk_stride; + _Float16 * scale_a_ptr = reinterpret_cast<_Float16 *>(a_subblk_base); + int8_t * quant_a_blk = reinterpret_cast(a_subblk_base + sizeof(_Float16)); + const float * a_src_ptr = a_ptr + k + kk * k_subblk_len; + + size_t vl = __riscv_vsetvl_e32m1(k_subblk_len); + vfloat32m1_t v_a = __riscv_vle32_v_f32m1(a_src_ptr, vl); + float rep_scale_a = scale_temp[kk] ? 1.0f / scale_temp[kk] : 0.0f; + scale_a_ptr[0] = static_cast<_Float16>(scale_temp[kk] * scale_factor); + + vfloat32m1_t v_a_scale = __riscv_vfmul_vf_f32m1(v_a, rep_scale_a, vl); + vint16mf2_t v_a_quant = __riscv_vfncvt_x_f_w_i16mf2(v_a_scale, vl); + vint8mf4_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8mf4(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8mf4_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[kk] = static_cast<_Float16>(-a_sum) * static_cast<_Float16>(8.0f); + + __riscv_vse8_v_i8mf4(quant_a_blk, v_a_quant_i8, vl); + } + } + } else if (vlenb == 32) { + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_blk_stride) { + _Float16 * a_sum_ptr = reinterpret_cast<_Float16 *>(quant_a_ptr + a_subblk_stride * subblk_count); + _Float16 * scale_avg_ptr = reinterpret_cast<_Float16 *>(quant_a_ptr + a_blk_stride - sizeof(_Float16)); + float scale_avg = 0.0f; + + for (size_t kk = 0; kk < subblk_count; ++kk) { + const float * a_src_ptr = a_ptr + k + kk * k_subblk_len; + + size_t vl = __riscv_vsetvl_e32m4(k_subblk_len); + vfloat32m4_t v_a = __riscv_vle32_v_f32m4(a_src_ptr, vl); + vfloat32m4_t v_a_abs = __riscv_vfabs_v_f32m4(v_a, vl); + + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_a_max = __riscv_vfredmax_vs_f32m4_f32m1(v_a_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_a_max); + + scale_temp[kk] = max_abs_a / ((1 << 7) - 1); + scale_avg += scale_temp[kk]; + } + + scale_avg /= subblk_count; + const float scale_factor = scale_avg ? 1.0f / scale_avg : 0.0f; + scale_avg_ptr[0] = static_cast<_Float16>(scale_avg); + + for (size_t kk = 0; kk < subblk_count; ++kk) { + uint8_t * a_subblk_base = quant_a_ptr + kk * a_subblk_stride; + _Float16 * scale_a_ptr = reinterpret_cast<_Float16 *>(a_subblk_base); + int8_t * quant_a_blk = reinterpret_cast(a_subblk_base + sizeof(_Float16)); + const float * a_src_ptr = a_ptr + k + kk * k_subblk_len; + + size_t vl = __riscv_vsetvl_e32m4(k_subblk_len); + vfloat32m4_t v_a = __riscv_vle32_v_f32m4(a_src_ptr, vl); + float rep_scale_a = scale_temp[kk] ? 1.0f / scale_temp[kk] : 0.0f; + scale_a_ptr[0] = static_cast<_Float16>(scale_temp[kk] * scale_factor); + + vfloat32m4_t v_a_scale = __riscv_vfmul_vf_f32m4(v_a, rep_scale_a, vl); + vint16m2_t v_a_quant = __riscv_vfncvt_x_f_w_i16m2(v_a_scale, vl); + vint8m1_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8m1(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8m1_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[kk] = static_cast<_Float16>(-a_sum) * static_cast<_Float16>(8.0f); + + __riscv_vse8_v_i8m1(quant_a_blk, v_a_quant_i8, vl); + } + } + } else { + quantize_a_nrow_i8_hp_ref<1>(blk_len, a_ptr, count_k, quant_a_ptr); + } +} + +void quantize_a_4row_i8_hp(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr) { + constexpr size_t k_subblk_len = 32; + GGML_ASSERT(blk_len == 256); + + constexpr size_t subblk_count = 256 / k_subblk_len; + int64_t a_blk_stride = q8_hp_blk_size(blk_len, true, true); + int64_t a_nrow_block_stride = a_blk_stride * 4; + int64_t a_subblk_stride = q8_hp_blk_size(k_subblk_len, false, false) * 4; + size_t vlenb = __riscv_vlenb(); + float scale_temp[subblk_count] = { 0.0f }; + + if (vlenb == 128) { + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_nrow_block_stride) { + _Float16 * a_sum_ptr = reinterpret_cast<_Float16 *>(quant_a_ptr + a_subblk_stride * subblk_count); + _Float16 * scale_avg_ptr = + reinterpret_cast<_Float16 *>(quant_a_ptr + a_nrow_block_stride - sizeof(_Float16) * 4); + float scale_avg = 0.0f; + + for (size_t kk = 0; kk < subblk_count; ++kk) { + const float * a_src_ptr0 = a_ptr + 0 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr1 = a_ptr + 1 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr2 = a_ptr + 2 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr3 = a_ptr + 3 * count_k + k + kk * k_subblk_len; + + size_t vl = __riscv_vsetvl_e32m1(k_subblk_len); + vfloat32m1_t v_a0 = __riscv_vle32_v_f32m1(a_src_ptr0, vl); + vfloat32m1_t v_a1 = __riscv_vle32_v_f32m1(a_src_ptr1, vl); + vfloat32m1_t v_a2 = __riscv_vle32_v_f32m1(a_src_ptr2, vl); + vfloat32m1_t v_a3 = __riscv_vle32_v_f32m1(a_src_ptr3, vl); + vfloat32m1_t v_a0_abs = __riscv_vfabs_v_f32m1(v_a0, vl); + vfloat32m1_t v_a1_abs = __riscv_vfabs_v_f32m1(v_a1, vl); + vfloat32m1_t v_a2_abs = __riscv_vfabs_v_f32m1(v_a2, vl); + vfloat32m1_t v_a3_abs = __riscv_vfabs_v_f32m1(v_a3, vl); + + vfloat32m1_t v_max_abs = __riscv_vfmax_vv_f32m1(v_a0_abs, v_a1_abs, vl); + v_max_abs = __riscv_vfmax_vv_f32m1(v_max_abs, v_a2_abs, vl); + v_max_abs = __riscv_vfmax_vv_f32m1(v_max_abs, v_a3_abs, vl); + + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_a_max = __riscv_vfredmax_vs_f32m1_f32m1(v_max_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_a_max); + + scale_temp[kk] = max_abs_a / ((1 << 7) - 1); + scale_avg += scale_temp[kk]; + } + + scale_avg /= subblk_count; + const float scale_factor = scale_avg ? 1.0f / scale_avg : 0.0f; + scale_avg_ptr[0] = static_cast<_Float16>(scale_avg); + + for (size_t kk = 0; kk < subblk_count; ++kk) { + uint8_t * a_subblk_base = quant_a_ptr + kk * a_subblk_stride; + _Float16 * scale_a_ptr = reinterpret_cast<_Float16 *>(a_subblk_base); + int8_t * quant_a_blk = reinterpret_cast(a_subblk_base + sizeof(_Float16) * 4); + const float * a_src_ptr0 = a_ptr + 0 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr1 = a_ptr + 1 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr2 = a_ptr + 2 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr3 = a_ptr + 3 * count_k + k + kk * k_subblk_len; + + size_t vl = __riscv_vsetvl_e32m1(k_subblk_len); + vfloat32m1_t v_a0 = __riscv_vle32_v_f32m1(a_src_ptr0, vl); + vfloat32m1_t v_a1 = __riscv_vle32_v_f32m1(a_src_ptr1, vl); + vfloat32m1_t v_a2 = __riscv_vle32_v_f32m1(a_src_ptr2, vl); + vfloat32m1_t v_a3 = __riscv_vle32_v_f32m1(a_src_ptr3, vl); + + float rep_scale_a = scale_temp[kk] ? 1.0f / scale_temp[kk] : 0.0f; + scale_a_ptr[0] = static_cast<_Float16>(scale_temp[kk] * scale_factor); + + vfloat32m1_t v_a0_scale = __riscv_vfmul_vf_f32m1(v_a0, rep_scale_a, vl); + vfloat32m1_t v_a1_scale = __riscv_vfmul_vf_f32m1(v_a1, rep_scale_a, vl); + vfloat32m1_t v_a2_scale = __riscv_vfmul_vf_f32m1(v_a2, rep_scale_a, vl); + vfloat32m1_t v_a3_scale = __riscv_vfmul_vf_f32m1(v_a3, rep_scale_a, vl); + vint16mf2_t v_a0_quant = __riscv_vfncvt_x_f_w_i16mf2(v_a0_scale, vl); + vint16mf2_t v_a1_quant = __riscv_vfncvt_x_f_w_i16mf2(v_a1_scale, vl); + vint16mf2_t v_a2_quant = __riscv_vfncvt_x_f_w_i16mf2(v_a2_scale, vl); + vint16mf2_t v_a3_quant = __riscv_vfncvt_x_f_w_i16mf2(v_a3_scale, vl); + vint8mf4_t v_a0_quant_i8 = __riscv_vncvt_x_x_w_i8mf4(v_a0_quant, vl); + vint8mf4_t v_a1_quant_i8 = __riscv_vncvt_x_x_w_i8mf4(v_a1_quant, vl); + vint8mf4_t v_a2_quant_i8 = __riscv_vncvt_x_x_w_i8mf4(v_a2_quant, vl); + vint8mf4_t v_a3_quant_i8 = __riscv_vncvt_x_x_w_i8mf4(v_a3_quant, vl); + + vint16m1_t tmp_sum0 = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t tmp_sum1 = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t tmp_sum2 = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t tmp_sum3 = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a0_sum = __riscv_vwredsum_vs_i8mf4_i16m1(v_a0_quant_i8, tmp_sum0, vl); + vint16m1_t v_a1_sum = __riscv_vwredsum_vs_i8mf4_i16m1(v_a1_quant_i8, tmp_sum1, vl); + vint16m1_t v_a2_sum = __riscv_vwredsum_vs_i8mf4_i16m1(v_a2_quant_i8, tmp_sum2, vl); + vint16m1_t v_a3_sum = __riscv_vwredsum_vs_i8mf4_i16m1(v_a3_quant_i8, tmp_sum3, vl); + + a_sum_ptr[0 * subblk_count + kk] = + static_cast<_Float16>(-__riscv_vmv_x_s_i16m1_i16(v_a0_sum)) * static_cast<_Float16>(8.0f); + a_sum_ptr[1 * subblk_count + kk] = + static_cast<_Float16>(-__riscv_vmv_x_s_i16m1_i16(v_a1_sum)) * static_cast<_Float16>(8.0f); + a_sum_ptr[2 * subblk_count + kk] = + static_cast<_Float16>(-__riscv_vmv_x_s_i16m1_i16(v_a2_sum)) * static_cast<_Float16>(8.0f); + a_sum_ptr[3 * subblk_count + kk] = + static_cast<_Float16>(-__riscv_vmv_x_s_i16m1_i16(v_a3_sum)) * static_cast<_Float16>(8.0f); + + __riscv_vse8_v_i8mf4(quant_a_blk + 0 * k_subblk_len, v_a0_quant_i8, vl); + __riscv_vse8_v_i8mf4(quant_a_blk + 1 * k_subblk_len, v_a1_quant_i8, vl); + __riscv_vse8_v_i8mf4(quant_a_blk + 2 * k_subblk_len, v_a2_quant_i8, vl); + __riscv_vse8_v_i8mf4(quant_a_blk + 3 * k_subblk_len, v_a3_quant_i8, vl); + } + } + } else if (vlenb == 32) { + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_nrow_block_stride) { + _Float16 * a_sum_ptr = reinterpret_cast<_Float16 *>(quant_a_ptr + a_subblk_stride * subblk_count); + _Float16 * scale_avg_ptr = + reinterpret_cast<_Float16 *>(quant_a_ptr + a_nrow_block_stride - sizeof(_Float16) * 4); + float scale_avg = 0.0f; + + for (size_t kk = 0; kk < subblk_count; ++kk) { + const float * a_src_ptr0 = a_ptr + 0 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr1 = a_ptr + 1 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr2 = a_ptr + 2 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr3 = a_ptr + 3 * count_k + k + kk * k_subblk_len; + + size_t vl = __riscv_vsetvl_e32m4(k_subblk_len); + vfloat32m4_t v_a0 = __riscv_vle32_v_f32m4(a_src_ptr0, vl); + vfloat32m4_t v_a1 = __riscv_vle32_v_f32m4(a_src_ptr1, vl); + vfloat32m4_t v_a2 = __riscv_vle32_v_f32m4(a_src_ptr2, vl); + vfloat32m4_t v_a3 = __riscv_vle32_v_f32m4(a_src_ptr3, vl); + + vfloat32m4_t v_a0_abs = __riscv_vfabs_v_f32m4(v_a0, vl); + vfloat32m4_t v_a1_abs = __riscv_vfabs_v_f32m4(v_a1, vl); + vfloat32m4_t v_a2_abs = __riscv_vfabs_v_f32m4(v_a2, vl); + vfloat32m4_t v_a3_abs = __riscv_vfabs_v_f32m4(v_a3, vl); + + vfloat32m4_t v_max_abs = __riscv_vfmax_vv_f32m4(v_a0_abs, v_a1_abs, vl); + v_max_abs = __riscv_vfmax_vv_f32m4(v_max_abs, v_a2_abs, vl); + v_max_abs = __riscv_vfmax_vv_f32m4(v_max_abs, v_a3_abs, vl); + + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_a_max = __riscv_vfredmax_vs_f32m4_f32m1(v_max_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_a_max); + + scale_temp[kk] = max_abs_a / ((1 << 7) - 1); + scale_avg += scale_temp[kk]; + } + + scale_avg /= subblk_count; + const float scale_factor = scale_avg ? 1.0f / scale_avg : 0.0f; + scale_avg_ptr[0] = static_cast<_Float16>(scale_avg); + + for (size_t kk = 0; kk < subblk_count; ++kk) { + uint8_t * a_subblk_base = quant_a_ptr + kk * a_subblk_stride; + _Float16 * scale_a_ptr = reinterpret_cast<_Float16 *>(a_subblk_base); + int8_t * quant_a_blk = reinterpret_cast(a_subblk_base + sizeof(_Float16) * 4); + const float * a_src_ptr0 = a_ptr + 0 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr1 = a_ptr + 1 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr2 = a_ptr + 2 * count_k + k + kk * k_subblk_len; + const float * a_src_ptr3 = a_ptr + 3 * count_k + k + kk * k_subblk_len; + + size_t vl = __riscv_vsetvl_e32m4(k_subblk_len); + vfloat32m4_t v_a0 = __riscv_vle32_v_f32m4(a_src_ptr0, vl); + vfloat32m4_t v_a1 = __riscv_vle32_v_f32m4(a_src_ptr1, vl); + vfloat32m4_t v_a2 = __riscv_vle32_v_f32m4(a_src_ptr2, vl); + vfloat32m4_t v_a3 = __riscv_vle32_v_f32m4(a_src_ptr3, vl); + + float rep_scale_a = scale_temp[kk] ? 1.0f / scale_temp[kk] : 0.0f; + scale_a_ptr[0] = static_cast<_Float16>(scale_temp[kk] * scale_factor); + + vfloat32m4_t v_a0_scale = __riscv_vfmul_vf_f32m4(v_a0, rep_scale_a, vl); + vfloat32m4_t v_a1_scale = __riscv_vfmul_vf_f32m4(v_a1, rep_scale_a, vl); + vfloat32m4_t v_a2_scale = __riscv_vfmul_vf_f32m4(v_a2, rep_scale_a, vl); + vfloat32m4_t v_a3_scale = __riscv_vfmul_vf_f32m4(v_a3, rep_scale_a, vl); + vint16m2_t v_a0_quant = __riscv_vfncvt_x_f_w_i16m2(v_a0_scale, vl); + vint16m2_t v_a1_quant = __riscv_vfncvt_x_f_w_i16m2(v_a1_scale, vl); + vint16m2_t v_a2_quant = __riscv_vfncvt_x_f_w_i16m2(v_a2_scale, vl); + vint16m2_t v_a3_quant = __riscv_vfncvt_x_f_w_i16m2(v_a3_scale, vl); + vint8m1_t v_a0_quant_i8 = __riscv_vncvt_x_x_w_i8m1(v_a0_quant, vl); + vint8m1_t v_a1_quant_i8 = __riscv_vncvt_x_x_w_i8m1(v_a1_quant, vl); + vint8m1_t v_a2_quant_i8 = __riscv_vncvt_x_x_w_i8m1(v_a2_quant, vl); + vint8m1_t v_a3_quant_i8 = __riscv_vncvt_x_x_w_i8m1(v_a3_quant, vl); + + vint16m1_t tmp_sum0 = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t tmp_sum1 = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t tmp_sum2 = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t tmp_sum3 = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a0_sum = __riscv_vwredsum_vs_i8m1_i16m1(v_a0_quant_i8, tmp_sum0, vl); + vint16m1_t v_a1_sum = __riscv_vwredsum_vs_i8m1_i16m1(v_a1_quant_i8, tmp_sum1, vl); + vint16m1_t v_a2_sum = __riscv_vwredsum_vs_i8m1_i16m1(v_a2_quant_i8, tmp_sum2, vl); + vint16m1_t v_a3_sum = __riscv_vwredsum_vs_i8m1_i16m1(v_a3_quant_i8, tmp_sum3, vl); + + a_sum_ptr[0 * subblk_count + kk] = + static_cast<_Float16>(-__riscv_vmv_x_s_i16m1_i16(v_a0_sum)) * static_cast<_Float16>(8.0f); + a_sum_ptr[1 * subblk_count + kk] = + static_cast<_Float16>(-__riscv_vmv_x_s_i16m1_i16(v_a1_sum)) * static_cast<_Float16>(8.0f); + a_sum_ptr[2 * subblk_count + kk] = + static_cast<_Float16>(-__riscv_vmv_x_s_i16m1_i16(v_a2_sum)) * static_cast<_Float16>(8.0f); + a_sum_ptr[3 * subblk_count + kk] = + static_cast<_Float16>(-__riscv_vmv_x_s_i16m1_i16(v_a3_sum)) * static_cast<_Float16>(8.0f); + + __riscv_vse8_v_i8m1(quant_a_blk + 0 * k_subblk_len, v_a0_quant_i8, vl); + __riscv_vse8_v_i8m1(quant_a_blk + 1 * k_subblk_len, v_a1_quant_i8, vl); + __riscv_vse8_v_i8m1(quant_a_blk + 2 * k_subblk_len, v_a2_quant_i8, vl); + __riscv_vse8_v_i8m1(quant_a_blk + 3 * k_subblk_len, v_a3_quant_i8, vl); + } + } + } else { + quantize_a_nrow_i8_hp_ref<4>(blk_len, a_ptr, count_k, quant_a_ptr); + } +} + +void quantize_a_row_i8k(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr) { + GGML_ASSERT(blk_len == 256); + constexpr int64_t a_blk_stride = q8k_blk_size(256); + constexpr int64_t a_sum_size = 256 / 16; + size_t vlenb = __riscv_vlenb(); + + if (vlenb == 128) { + // vlen = 1024 bits, can process 32 float32 elements with m1 + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_blk_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float)); + int8_t * quant_a_blk = + reinterpret_cast(quant_a_ptr + sizeof(float) + sizeof(int16_t) * a_sum_size); + + // Find max absolute value across all 256 elements + size_t vl = __riscv_vsetvl_e32m1(16); + vfloat32m1_t v_max_abs = __riscv_vfmv_v_f_f32m1(0.0f, vl); + + for (size_t bki = 0; bki < a_sum_size; bki++) { + vfloat32m1_t v_a = __riscv_vle32_v_f32m1(a_ptr + k + bki * 16, vl); + vfloat32m1_t v_a_abs = __riscv_vfabs_v_f32m1(v_a, vl); + v_max_abs = __riscv_vfmax_vv_f32m1(v_a_abs, v_max_abs, vl); + } + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_local_max = __riscv_vfredmax_vs_f32m1_f32m1(v_max_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_local_max); + + float scale_a = max_abs_a / ((1 << 7) - 1); + float rep_scale_a = scale_a ? 1.0f / scale_a : 0.0f; + scale_a_ptr[0] = scale_a; + + // Quantize and compute sums for each 16-element group + for (size_t bki = 0; bki < a_sum_size; bki++) { + vfloat32m1_t v_a = __riscv_vle32_v_f32m1(a_ptr + k + bki * 16, vl); + vfloat32m1_t v_a_scale = __riscv_vfmul_vf_f32m1(v_a, rep_scale_a, vl); + vint16mf2_t v_a_quant = __riscv_vfncvt_x_f_w_i16mf2(v_a_scale, vl); + vint8mf4_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8mf4(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8mf4_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[bki] = -a_sum; + + __riscv_vse8_v_i8mf4(quant_a_blk + bki * 16, v_a_quant_i8, vl); + } + } + } else if (vlenb == 32) { + // vlen = 256 bits, can process 8 float32 elements with m1 + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_blk_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float)); + int8_t * quant_a_blk = + reinterpret_cast(quant_a_ptr + sizeof(float) + sizeof(int16_t) * a_sum_size); + + // Find max absolute value across all 256 elements + size_t vl = __riscv_vsetvl_e32m2(16); + vfloat32m2_t v_max_abs = __riscv_vfmv_v_f_f32m2(0.0f, vl); + + for (size_t bki = 0; bki < a_sum_size; bki++) { + vfloat32m2_t v_a = __riscv_vle32_v_f32m2(a_ptr + k + bki * 16, vl); + vfloat32m2_t v_a_abs = __riscv_vfabs_v_f32m2(v_a, vl); + v_max_abs = __riscv_vfmax_vv_f32m2(v_a_abs, v_max_abs, vl); + } + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_local_max = __riscv_vfredmax_vs_f32m2_f32m1(v_max_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_local_max); + + float scale_a = max_abs_a / ((1 << 7) - 1); + float rep_scale_a = scale_a ? 1.0f / scale_a : 0.0f; + scale_a_ptr[0] = scale_a; + + // Quantize and compute sums for each 16-element group + for (size_t bki = 0; bki < a_sum_size; bki++) { + vfloat32m2_t v_a = __riscv_vle32_v_f32m2(a_ptr + k + bki * 16, vl); + vfloat32m2_t v_a_scale = __riscv_vfmul_vf_f32m2(v_a, rep_scale_a, vl); + vint16m1_t v_a_quant = __riscv_vfncvt_x_f_w_i16m1(v_a_scale, vl); + vint8mf2_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8mf2(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8mf2_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[bki] = -a_sum; + + __riscv_vse8_v_i8mf2(quant_a_blk + bki * 16, v_a_quant_i8, vl); + } + } + } else { + quantize_a_nrow_i8k_ref<1>(blk_len, a_ptr, count_k, quant_a_ptr); + } +} + +void quantize_a_4row_i8k(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr) { + GGML_ASSERT(blk_len == 256); + constexpr int64_t a_blk_stride = q8k_blk_size(256); + constexpr int64_t a_nrow_block_stride = a_blk_stride * 4; + constexpr int64_t a_sum_size = 256 / 16; + size_t vlenb = __riscv_vlenb(); + + if (vlenb == 128) { + // vlen = 1024 bits + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_nrow_block_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float) * 4); + int8_t * quant_a_blk = + reinterpret_cast(quant_a_ptr + sizeof(float) * 4 + sizeof(int16_t) * a_sum_size * 4); + + for (size_t mi = 0; mi < 4; mi++) { + // Find max absolute value across all 256 elements for this row + size_t vl = __riscv_vsetvl_e32m1(16); + vfloat32m1_t v_max_abs = __riscv_vfmv_v_f_f32m1(0.0f, vl); + + for (size_t bki = 0; bki < a_sum_size; bki++) { + vfloat32m1_t v_a = __riscv_vle32_v_f32m1(a_ptr + mi * count_k + k + bki * 16, vl); + vfloat32m1_t v_a_abs = __riscv_vfabs_v_f32m1(v_a, vl); + v_max_abs = __riscv_vfmax_vv_f32m1(v_a_abs, v_max_abs, vl); + } + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_local_max = __riscv_vfredmax_vs_f32m1_f32m1(v_max_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_local_max); + + float scale_a = max_abs_a / ((1 << 7) - 1); + float rep_scale_a = scale_a ? 1.0f / scale_a : 0.0f; + scale_a_ptr[mi] = scale_a; + + // Quantize and compute sums for each 16-element group + for (size_t bki = 0; bki < a_sum_size; bki++) { + vfloat32m1_t v_a = __riscv_vle32_v_f32m1(a_ptr + mi * count_k + k + bki * 16, vl); + vfloat32m1_t v_a_scale = __riscv_vfmul_vf_f32m1(v_a, rep_scale_a, vl); + vint16mf2_t v_a_quant = __riscv_vfncvt_x_f_w_i16mf2(v_a_scale, vl); + vint8mf4_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8mf4(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8mf4_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[mi * a_sum_size + bki] = -a_sum; + + __riscv_vse8_v_i8mf4(quant_a_blk + mi * blk_len + bki * 16, v_a_quant_i8, vl); + } + } + } + } else if (vlenb == 32) { + // vlen = 256 bits + for (size_t k = 0; k < count_k; k += blk_len, quant_a_ptr += a_nrow_block_stride) { + float * scale_a_ptr = reinterpret_cast(quant_a_ptr); + int16_t * a_sum_ptr = reinterpret_cast(quant_a_ptr + sizeof(float) * 4); + int8_t * quant_a_blk = + reinterpret_cast(quant_a_ptr + sizeof(float) * 4 + sizeof(int16_t) * a_sum_size * 4); + + for (size_t mi = 0; mi < 4; mi++) { + // Find max absolute value across all 256 elements for this row + size_t vl = __riscv_vsetvl_e32m2(16); + vfloat32m2_t v_max_abs = __riscv_vfmv_v_f_f32m2(0.0f, vl); + + for (size_t bki = 0; bki < a_sum_size; bki++) { + vfloat32m2_t v_a = __riscv_vle32_v_f32m2(a_ptr + mi * count_k + k + bki * 16, vl); + vfloat32m2_t v_a_abs = __riscv_vfabs_v_f32m2(v_a, vl); + v_max_abs = __riscv_vfmax_vv_f32m2(v_a_abs, v_max_abs, vl); + } + vfloat32m1_t tmp = __riscv_vfmv_v_f_f32m1(0.0f, vl); + vfloat32m1_t v_local_max = __riscv_vfredmax_vs_f32m2_f32m1(v_max_abs, tmp, vl); + float max_abs_a = __riscv_vfmv_f_s_f32m1_f32(v_local_max); + + float scale_a = max_abs_a / ((1 << 7) - 1); + float rep_scale_a = scale_a ? 1.0f / scale_a : 0.0f; + scale_a_ptr[mi] = scale_a; + + // Quantize and compute sums for each 16-element group + for (size_t bki = 0; bki < a_sum_size; bki++) { + vfloat32m2_t v_a = __riscv_vle32_v_f32m2(a_ptr + mi * count_k + k + bki * 16, vl); + vfloat32m2_t v_a_scale = __riscv_vfmul_vf_f32m2(v_a, rep_scale_a, vl); + vint16m1_t v_a_quant = __riscv_vfncvt_x_f_w_i16m1(v_a_scale, vl); + vint8mf2_t v_a_quant_i8 = __riscv_vncvt_x_x_w_i8mf2(v_a_quant, vl); + + vint16m1_t tmp_sum = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t v_a_sum = __riscv_vwredsum_vs_i8mf2_i16m1(v_a_quant_i8, tmp_sum, vl); + int16_t a_sum = __riscv_vmv_x_s_i16m1_i16(v_a_sum); + a_sum_ptr[mi * a_sum_size + bki] = -a_sum; + + __riscv_vse8_v_i8mf2(quant_a_blk + mi * blk_len + bki * 16, v_a_quant_i8, vl); + } + } + } + } else { + quantize_a_nrow_i8k_ref<4>(blk_len, a_ptr, count_k, quant_a_ptr); + } +} + +void forward_cpy_with_permute(ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + ggml_tensor * dst = op; + const int ith = params->ith; + const int nth = params->nth; + + // [batch, m, n] -> [batch, n, m] + int64_t batch = src0->ne[2] * src0->ne[3]; + int64_t m = src0->ne[1]; + int64_t n = src0->ne[0]; + + int64_t batch_stride = src0->nb[2]; + int64_t m_src_stride = src0->nb[0]; + int64_t n_src_stride = src0->nb[1]; + int64_t n_dst_stride = n_src_stride * m; + + permute_transpose_impl(src0, dst, batch, m, n, batch_stride, m_src_stride, n_src_stride, n_dst_stride, ith, nth); +} + +void forward_cont_with_permute(ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + ggml_tensor * dst = op; + const int ith = params->ith; + const int nth = params->nth; + + // [batch, m, n] -> [batch, n, m] + int64_t batch = dst->ne[2] * dst->ne[3]; + int64_t n = dst->ne[1]; + int64_t m = dst->ne[0]; + + int64_t batch_stride = dst->nb[2]; + int64_t m_src_stride = src0->nb[0]; + int64_t n_src_stride = src0->nb[1]; + int64_t n_dst_stride = dst->nb[1]; + + permute_transpose_impl(src0, dst, batch, m, n, batch_stride, m_src_stride, n_src_stride, n_dst_stride, ith, nth); +} + +void forward_norm_f32(ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + ggml_tensor * dst = op; + GGML_ASSERT(ggml_are_same_shape(src0, dst)); + GGML_ASSERT(src0->nb[0] == sizeof(float)); + + int ith = params->ith; + int nth = params->nth; + + GGML_TENSOR_UNARY_OP_LOCALS + + float epsilon = *((float *) dst->op_params); + + GGML_ASSERT(epsilon > 0.0f); + + auto * input = (char *) src0->data; + auto * output = (char *) dst->data; + + const auto hidden_size = ne00; + const auto task_count = ne01 * ne02 * ne03; + const auto task_per_thread = (task_count + nth - 1) / nth; + + const auto task_begin = ith * task_per_thread; + const auto task_end = std::min((ith + 1) * task_per_thread, task_count); + + for (auto task_idx = task_begin; task_idx < task_end; task_idx++) { + int64_t i03 = task_idx / (ne02 * ne01); + int64_t i02 = (task_idx - i03 * ne02 * ne01) / ne01; + int64_t i01 = (task_idx - i03 * ne02 * ne01 - i02 * ne01); + + auto * p_input = (float *) (input + i01 * nb01 + i02 * nb02 + i03 * nb03); + auto * p_output = (float *) (output + i01 * nb1 + i02 * nb2 + i03 * nb3); + auto * p_temp_output = p_output; + + size_t gvl = __riscv_vsetvlmax_e32m4(); + vfloat32m4_t sum = __riscv_vfmv_v_f_f32m4(0.f, gvl); + vfloat32m4_t sum_sq = __riscv_vfmv_v_f_f32m4(0.f, gvl); + int64_t length = hidden_size; + while (length > 0) { + gvl = __riscv_vsetvl_e32m4(length); + // load data + vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_input, gvl); + + sum = __riscv_vfadd_vv_f32m4(sum, src_data, gvl); + sum_sq = __riscv_vfmacc_vv_f32m4(sum_sq, src_data, src_data, gvl); + + __riscv_vse32_v_f32m4(p_temp_output, src_data, gvl); + + p_input += gvl; + p_temp_output += gvl; + length -= gvl; + } + + gvl = __riscv_vsetvlmax_e32m1(); + + float mean = 0.f; + vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.f, gvl); + vfloat32m1_t mean_v = + __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum, 0), __riscv_vget_v_f32m4_f32m1(sum, 1), gvl); + mean_v = __riscv_vfadd_vv_f32m1(mean_v, __riscv_vget_v_f32m4_f32m1(sum, 2), gvl); + mean_v = __riscv_vfadd_vv_f32m1(mean_v, __riscv_vget_v_f32m4_f32m1(sum, 3), gvl); + mean_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_v, zero_v, gvl); + mean = __riscv_vfmv_f_s_f32m1_f32(mean_v); + mean /= hidden_size; + + vfloat32m1_t mean_square_v = + __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum_sq, 0), __riscv_vget_v_f32m4_f32m1(sum_sq, 1), gvl); + mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 2), gvl); + mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 3), gvl); + mean_square_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_square_v, zero_v, gvl); + + float mean_square = __riscv_vfmv_f_s_f32m1_f32(mean_square_v); + mean_square /= hidden_size; + mean_square = sqrt(mean_square - mean * mean + epsilon); + + mean_square = 1.0f / mean_square; + length = hidden_size; + p_temp_output = p_output; + + while (length > 0) { + gvl = __riscv_vsetvl_e32m4(length); + vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl); + src_data = __riscv_vfsub_vf_f32m4(src_data, mean, gvl); + src_data = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl); + __riscv_vse32_v_f32m4(p_output, src_data, gvl); + p_temp_output += gvl; + p_output += gvl; + length -= gvl; + } + } +} + +template void forward_binary(ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); + + auto src0_rows = ggml_nrows(src0); + auto src1_rows = ggml_nrows(src1); + + int ith = params->ith; + int nth = params->nth; + + GGML_TENSOR_BINARY_OP_LOCALS + + GGML_ASSERT(nb0 == sizeof(T)); + GGML_ASSERT(nb00 == sizeof(T)); + + const auto [ir0, ir1] = get_thread_range(params, src0); + + auto compute_func_vv = [&](int64_t blk_len, int64_t r, T * src0_ptr, T * src1_ptr, T * dst_ptr) { + int64_t idx = 0; + if constexpr (op_type == GGML_OP_ADD) { + if constexpr (std::is_same_v) { + for (size_t vl; blk_len > 0; blk_len -= vl, idx += vl) { + vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t lhs = __riscv_vle32_v_f32m4(src0_ptr + idx + r, vl); + vfloat32m4_t rhs = __riscv_vle32_v_f32m4(src1_ptr + idx, vl); + vfloat32m4_t res = __riscv_vfadd_vv_f32m4(lhs, rhs, vl); + __riscv_vse32_v_f32m4(dst_ptr + idx + r, res, vl); + } + } else if constexpr (std::is_same_v) { + for (size_t vl; blk_len > 0; blk_len -= vl, idx += vl) { + vl = __riscv_vsetvl_e16m4(blk_len); + vfloat16m4_t lhs = __riscv_vle16_v_f16m4((src0_ptr + idx + r), vl); + vfloat16m4_t rhs = __riscv_vle16_v_f16m4((src1_ptr + idx), vl); + vfloat16m4_t res = __riscv_vfadd_vv_f16m4(lhs, rhs, vl); + __riscv_vse16_v_f16m4((dst_ptr + idx + r), res, vl); + } + } else { + GGML_ABORT("fatal error"); + } + } else if constexpr (op_type == GGML_OP_SUB) { + if constexpr (std::is_same_v) { + for (size_t vl; blk_len > 0; blk_len -= vl, idx += vl) { + vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t lhs = __riscv_vle32_v_f32m4(src0_ptr + idx + r, vl); + vfloat32m4_t rhs = __riscv_vle32_v_f32m4(src1_ptr + idx, vl); + vfloat32m4_t res = __riscv_vfsub_vv_f32m4(lhs, rhs, vl); + __riscv_vse32_v_f32m4(dst_ptr + idx + r, res, vl); + } + } else if constexpr (std::is_same_v) { + for (size_t vl; blk_len > 0; blk_len -= vl, idx += vl) { + vl = __riscv_vsetvl_e16m4(blk_len); + vfloat16m4_t lhs = __riscv_vle16_v_f16m4((src0_ptr + idx + r), vl); + vfloat16m4_t rhs = __riscv_vle16_v_f16m4((src1_ptr + idx), vl); + vfloat16m4_t res = __riscv_vfsub_vv_f16m4(lhs, rhs, vl); + __riscv_vse16_v_f16m4((dst_ptr + idx + r), res, vl); + } + } else { + GGML_ABORT("fatal error"); + } + } else if constexpr (op_type == GGML_OP_MUL) { + if constexpr (std::is_same_v) { + for (size_t vl; blk_len > 0; blk_len -= vl, idx += vl) { + vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t lhs = __riscv_vle32_v_f32m4(src0_ptr + idx + r, vl); + vfloat32m4_t rhs = __riscv_vle32_v_f32m4(src1_ptr + idx, vl); + vfloat32m4_t res = __riscv_vfmul_vv_f32m4(lhs, rhs, vl); + __riscv_vse32_v_f32m4(dst_ptr + idx + r, res, vl); + } + } else if constexpr (std::is_same_v) { + for (size_t vl; blk_len > 0; blk_len -= vl, idx += vl) { + vl = __riscv_vsetvl_e16m4(blk_len); + vfloat16m4_t lhs = __riscv_vle16_v_f16m4((src0_ptr + idx + r), vl); + vfloat16m4_t rhs = __riscv_vle16_v_f16m4((src1_ptr + idx), vl); + vfloat16m4_t res = __riscv_vfmul_vv_f16m4(lhs, rhs, vl); + __riscv_vse16_v_f16m4((dst_ptr + idx + r), res, vl); + } + } else { + GGML_ABORT("fatal error"); + } + } else if constexpr (op_type == GGML_OP_DIV) { + if constexpr (std::is_same_v) { + for (size_t vl; blk_len > 0; blk_len -= vl, idx += vl) { + vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t lhs = __riscv_vle32_v_f32m4(src0_ptr + idx + r, vl); + vfloat32m4_t rhs = __riscv_vle32_v_f32m4(src1_ptr + idx, vl); + vfloat32m4_t res = __riscv_vfdiv_vv_f32m4(lhs, rhs, vl); + __riscv_vse32_v_f32m4(dst_ptr + idx + r, res, vl); + } + } else if constexpr (std::is_same_v) { + for (size_t vl; blk_len > 0; blk_len -= vl, idx += vl) { + vl = __riscv_vsetvl_e16m4(blk_len); + vfloat16m4_t lhs = __riscv_vle16_v_f16m4((src0_ptr + idx + r), vl); + vfloat16m4_t rhs = __riscv_vle16_v_f16m4((src1_ptr + idx), vl); + vfloat16m4_t res = __riscv_vfdiv_vv_f16m4(lhs, rhs, vl); + __riscv_vse16_v_f16m4((dst_ptr + idx + r), res, vl); + } + } else { + GGML_ABORT("fatal error"); + } + } else { + GGML_ABORT("fatal error"); + } + }; + + if (src0_rows == src1_rows && src0_rows == 1 && ne00 == ne10) { + int64_t task_per_thread = (ne00 + nth - 1) / nth; + int64_t task_begin = ith * task_per_thread; + int64_t task_end = std::min((ith + 1) * task_per_thread, ne00); + + T * dst_ptr = ((T *) dst->data) + task_begin; + T * src0_ptr = ((T *) src0->data) + task_begin; + T * src1_ptr = ((T *) src1->data) + task_begin; + + compute_func_vv(task_end - task_begin, 0, src0_ptr, src1_ptr, dst_ptr); + } else if (ne10 > 1) { + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t i03 = ir / (ne02 * ne01); + const int64_t i02 = (ir - i03 * ne02 * ne01) / ne01; + const int64_t i01 = (ir - i03 * ne02 * ne01 - i02 * ne01); + + const int64_t i13 = i03 % ne13; + const int64_t i12 = i02 % ne12; + const int64_t i11 = i01 % ne11; + + T * dst_ptr = (T *) ((char *) dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1); + T * src0_ptr = (T *) ((char *) src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01); + T * src1_ptr = (T *) ((char *) src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11); + + // src1 is broadcastable across src0 and dst in i1, i2, i3 + for (int64_t r = 0; r < ne00; r += ne10) { + compute_func_vv(ne10, r, src0_ptr, src1_ptr, dst_ptr); + } + } + } else { + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t i03 = ir / (ne02 * ne01); + const int64_t i02 = (ir - i03 * ne02 * ne01) / ne01; + const int64_t i01 = (ir - i03 * ne02 * ne01 - i02 * ne01); + + const int64_t i13 = i03 % ne13; + const int64_t i12 = i02 % ne12; + const int64_t i11 = i01 % ne11; + + T * dst_ptr = (T *) ((char *) dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1); + T * src0_ptr = (T *) ((char *) src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01); + T * src1_ptr = (T *) ((char *) src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11); + + T rhs_scalar = src1_ptr[0]; + int64_t blk_len = ne00; + int64_t r = 0; + + for (size_t vl; blk_len > 0; blk_len -= vl, r += vl) { + if constexpr (op_type == GGML_OP_ADD) { + if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t lhs = __riscv_vle32_v_f32m4(src0_ptr + r, vl); + vfloat32m4_t res = __riscv_vfadd_vf_f32m4(lhs, rhs_scalar, vl); + __riscv_vse32_v_f32m4(dst_ptr + r, res, vl); + } else if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e16m4(blk_len); + vfloat16m4_t lhs = __riscv_vle16_v_f16m4((src0_ptr + r), vl); + vfloat16m4_t res = __riscv_vfadd_vf_f16m4(lhs, rhs_scalar, vl); + __riscv_vse16_v_f16m4((dst_ptr + r), res, vl); + } else { + GGML_ABORT("fatal error"); + } + } else if constexpr (op_type == GGML_OP_SUB) { + if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t lhs = __riscv_vle32_v_f32m4(src0_ptr + r, vl); + vfloat32m4_t res = __riscv_vfsub_vf_f32m4(lhs, rhs_scalar, vl); + __riscv_vse32_v_f32m4(dst_ptr + r, res, vl); + } else if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e16m4(blk_len); + vfloat16m4_t lhs = __riscv_vle16_v_f16m4((src0_ptr + r), vl); + vfloat16m4_t res = __riscv_vfsub_vf_f16m4(lhs, rhs_scalar, vl); + __riscv_vse16_v_f16m4((dst_ptr + r), res, vl); + } else { + GGML_ABORT("fatal error"); + } + } else if constexpr (op_type == GGML_OP_MUL) { + if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t lhs = __riscv_vle32_v_f32m4(src0_ptr + r, vl); + vfloat32m4_t res = __riscv_vfmul_vf_f32m4(lhs, rhs_scalar, vl); + __riscv_vse32_v_f32m4(dst_ptr + r, res, vl); + } else if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e16m4(blk_len); + vfloat16m4_t lhs = __riscv_vle16_v_f16m4((src0_ptr + r), vl); + vfloat16m4_t res = __riscv_vfmul_vf_f16m4(lhs, rhs_scalar, vl); + __riscv_vse16_v_f16m4((dst_ptr + r), res, vl); + } else { + GGML_ABORT("fatal error"); + } + } else if constexpr (op_type == GGML_OP_DIV) { + if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e32m4(blk_len); + vfloat32m4_t lhs = __riscv_vle32_v_f32m4(src0_ptr + r, vl); + vfloat32m4_t res = __riscv_vfdiv_vf_f32m4(lhs, rhs_scalar, vl); + __riscv_vse32_v_f32m4(dst_ptr + r, res, vl); + } else if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e16m4(blk_len); + vfloat16m4_t lhs = __riscv_vle16_v_f16m4((src0_ptr + r), vl); + vfloat16m4_t res = __riscv_vfdiv_vf_f16m4(lhs, rhs_scalar, vl); + __riscv_vse16_v_f16m4((dst_ptr + r), res, vl); + } else { + GGML_ABORT("fatal error"); + } + } else { + GGML_ABORT("fatal error"); + } + } + } + } +} + +template void forward_sum_rows(const ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + ggml_tensor * dst = op; + + const int ith = params->ith; + const int nth = params->nth; + + GGML_TENSOR_UNARY_OP_LOCALS + + GGML_ASSERT(ne0 == 1); + GGML_ASSERT(ne1 == ne01); + GGML_ASSERT(ne2 == ne02); + GGML_ASSERT(ne3 == ne03); + + int64_t n_task = ne01 * ne02 * ne03; + int64_t task_per_thread = (n_task + nth - 1) / nth; + int64_t ir_start = ith * task_per_thread; + int64_t ir_end = std::min(ir_start + task_per_thread, n_task); + + for (int64_t ir = ir_start; ir < ir_end; ir++) { + const int64_t i3 = ir / (ne02 * ne01); + const int64_t i2 = (ir - i3 * ne02 * ne01) / ne01; + const int64_t i1 = (ir - i3 * ne02 * ne01 - i2 * ne01); + + T * src_row = (T *) ((char *) src0->data + i1 * nb01 + i2 * nb02 + i3 * nb03); + T * dst_row = (T *) ((char *) op->data + i1 * nb1 + i2 * nb2 + i3 * nb3); + + float row_sum = 0; + + if constexpr (std::is_same_v) { + size_t gvl = __riscv_vsetvlmax_e32m4(); + vfloat32m4_t acc_vec = __riscv_vfmv_v_f_f32m4(0.0f, gvl); + int64_t length = ne00; + const float * p_data = src_row; + + while (length > 0) { + size_t vl = __riscv_vsetvl_e32m4(length); + vfloat32m4_t vec = __riscv_vle32_v_f32m4(p_data, vl); + acc_vec = __riscv_vfadd_vv_f32m4(acc_vec, vec, vl); + p_data += vl; + length -= vl; + } + + gvl = __riscv_vsetvlmax_e32m1(); + vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.0f, gvl); + vfloat32m1_t sum_v = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(acc_vec, 0), + __riscv_vget_v_f32m4_f32m1(acc_vec, 1), gvl); + sum_v = __riscv_vfadd_vv_f32m1(sum_v, __riscv_vget_v_f32m4_f32m1(acc_vec, 2), gvl); + sum_v = __riscv_vfadd_vv_f32m1(sum_v, __riscv_vget_v_f32m4_f32m1(acc_vec, 3), gvl); + sum_v = __riscv_vfredusum_vs_f32m1_f32m1(sum_v, zero_v, gvl); + row_sum = __riscv_vfmv_f_s_f32m1_f32(sum_v); + } else if constexpr (std::is_same_v) { + size_t gvl = __riscv_vsetvlmax_e16m2(); + vfloat32m4_t acc_vec = __riscv_vfmv_v_f_f32m4(0.0f, gvl); + int64_t length = ne00; + const _Float16 * p_data = src_row; + + while (length > 0) { + size_t vl = __riscv_vsetvl_e16m2(length); + vfloat16m2_t vec_f16 = __riscv_vle16_v_f16m2(p_data, vl); + vfloat32m4_t vec_f32 = __riscv_vfwcvt_f_f_v_f32m4(vec_f16, vl); + acc_vec = __riscv_vfadd_vv_f32m4(acc_vec, vec_f32, vl); + p_data += vl; + length -= vl; + } + + gvl = __riscv_vsetvlmax_e32m1(); + vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.0f, gvl); + vfloat32m1_t sum_v = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(acc_vec, 0), + __riscv_vget_v_f32m4_f32m1(acc_vec, 1), gvl); + sum_v = __riscv_vfadd_vv_f32m1(sum_v, __riscv_vget_v_f32m4_f32m1(acc_vec, 2), gvl); + sum_v = __riscv_vfadd_vv_f32m1(sum_v, __riscv_vget_v_f32m4_f32m1(acc_vec, 3), gvl); + sum_v = __riscv_vfredusum_vs_f32m1_f32m1(sum_v, zero_v, gvl); + row_sum = __riscv_vfmv_f_s_f32m1_f32(sum_v); + } else { + GGML_ABORT("fatal error"); + } + + dst_row[0] = row_sum; + } +} + +template void forward_repeat_nrows(ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + ggml_tensor * dst = op; + + const int ith = params->ith; + const int nth = params->nth; + + int64_t nrows = ggml_nrows(src0); + int64_t nrows_per_thread = (nrows + nth - 1) / nth; + int64_t ir_start = ith * nrows_per_thread; + int64_t ir_end = std::min(ir_start + nrows_per_thread, nrows); + + if (src0->ne[0] == 1) { + for (int64_t ir = ir_start; ir < ir_end; ir++) { + T * src_row = (T *) ((char *) src0->data + ir * src0->nb[1]); + T * dst_row = (T *) ((char *) dst->data + ir * dst->nb[1]); + + T src_scalar = src_row[0]; + + int64_t length = dst->ne[0]; + int64_t idx = 0; + size_t vl = 0; + + while (length > 0) { + if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e32m4(length); + vint32m4_t vec = __riscv_vmv_v_x_i32m4(src_scalar, vl); + __riscv_vse32_v_i32m4(dst_row + idx, vec, vl); + } else if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e16m4(length); + vint16m4_t vec = __riscv_vmv_v_x_i16m4(src_scalar, vl); + __riscv_vse16_v_i16m4((dst_row + idx), vec, vl); + } else { + GGML_ABORT("fatal error"); + } + idx += vl; + length -= vl; + } + } + } else if (src0->ne[0] == dst->ne[0]) { + for (int64_t ir = ir_start; ir < ir_end; ir++) { + T * src_row = (T *) ((char *) src0->data + ir * src0->nb[1]); + T * dst_row = (T *) ((char *) dst->data + ir * dst->nb[1]); + + int64_t length = dst->ne[0]; + int64_t idx = 0; + size_t vl = 0; + + while (length > 0) { + if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e32m4(length); + vint32m4_t vec = __riscv_vle32_v_i32m4(src_row + idx, vl); + __riscv_vse32_v_i32m4(dst_row + idx, vec, vl); + } else if constexpr (std::is_same_v) { + vl = __riscv_vsetvl_e16m4(length); + vint16m4_t vec = __riscv_vle16_v_i16m4((src_row + idx), vl); + __riscv_vse16_v_i16m4((dst_row + idx), vec, vl); + } else { + GGML_ABORT("fatal error"); + } + idx += vl; + length -= vl; + } + } + } else { + GGML_ABORT("fatal error"); + } +} + +template void forward_repeat_dim1(ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + ggml_tensor * dst = op; + + const int ith = params->ith; + const int nth = params->nth; + + const int64_t ne0 = dst->ne[0]; + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t ne3 = dst->ne[3]; + + const int64_t total_batches = ne2 * ne3; + const int64_t batches_per_thread = (total_batches + nth - 1) / nth; + const int64_t batch_start = ith * batches_per_thread; + const int64_t batch_end = std::min(batch_start + batches_per_thread, total_batches); + + for (int64_t b = batch_start; b < batch_end; b++) { + const int64_t i3 = b / ne2; + const int64_t i2 = b % ne2; + + T * src_base = (T *) ((char *) src0->data + i2 * src0->nb[2] + i3 * src0->nb[3]); + T * dst_batch = (T *) ((char *) dst->data + i2 * dst->nb[2] + i3 * dst->nb[3]); + + for (int64_t i1 = 0; i1 < ne1; i1++) { + T * dst_ptr = (T *) ((char *) dst_batch + i1 * dst->nb[1]); + int64_t length = ne0; + int64_t idx = 0; + + while (length > 0) { + if constexpr (std::is_same_v) { + size_t vl = __riscv_vsetvl_e32m4(length); + vint32m4_t vec = __riscv_vle32_v_i32m4(src_base + idx, vl); + __riscv_vse32_v_i32m4(dst_ptr + idx, vec, vl); + idx += vl; + length -= vl; + } else if constexpr (std::is_same_v) { + size_t vl = __riscv_vsetvl_e16m4(length); + vint16m4_t vec = __riscv_vle16_v_i16m4((src_base + idx), vl); + __riscv_vse16_v_i16m4((dst_ptr + idx), vec, vl); + idx += vl; + length -= vl; + } else { + GGML_ABORT("fatal error"); + } + } + } + } +} + +template void forward_get_rows(ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGML_TENSOR_BINARY_OP_LOCALS + + const int64_t nc = ne00; + const int64_t nr = ggml_nelements(src1); + + assert(ne0 == nc); + assert(ne02 == ne11); + assert(nb00 == sizeof(float)); + assert(ggml_nrows(op) == nr); + + const int ith = params->ith; + const int nth = params->nth; + + int rows_nth = nth; + int cols_nth = 1; + + if (nr == 1) { + rows_nth = 1; + cols_nth = nth; + } + + // rows per thread + const int dr = (nr + rows_nth - 1) / rows_nth; + const int dc = (nc + cols_nth - 1) / cols_nth; + + int rows_ith = ith % rows_nth; + int cols_ith = ith % cols_nth; + + // row range for this thread + const int ir0 = dr * rows_ith; + const int ir1 = MIN(ir0 + dr, nr); + + const int cr0 = dc * cols_ith; + const int cr1 = MIN(cr0 + dc, nc); + + for (int64_t i = ir0; i < ir1; ++i) { + const int64_t i12 = i / (ne11 * ne10); + const int64_t i11 = (i - i12 * ne11 * ne10) / ne10; + const int64_t i10 = (i - i12 * ne11 * ne10 - i11 * ne10); + const int64_t i01 = *(int32_t *) ((char *) src1->data + i10 * nb10 + i11 * nb11 + i12 * nb12); + + GGML_ASSERT(i01 >= 0 && i01 < ne01); + + memcpy1d(((char *) dst->data + i10 * nb1 + i11 * nb2 + i12 * nb3) + cr0 * sizeof(T), + ((char *) src0->data + i01 * nb01 + i11 * nb02 + i12 * nb03) + cr0 * sizeof(T), + (cr1 - cr0) * sizeof(T)); + } +} + +template void forward_concat(ggml_compute_params * params, ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + ggml_tensor * dst = op; + + GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float)); + + GGML_TENSOR_BINARY_OP_LOCALS + + const int32_t dim = ggml_get_op_params_i32(dst, 0); + + GGML_ASSERT(dim == 0 && nb0 == sizeof(float) && nb1 == sizeof(float) * (ne00 + ne10)); + + const int64_t nr = ggml_nrows(dst); + const int64_t nc = ne0; + + const int ith = params->ith; + const int nth = params->nth; + + int rows_nth = nth; + int cols_nth = 1; + + if (nr == 1) { + rows_nth = 1; + cols_nth = nth; + } + + const int dr = (nr + rows_nth - 1) / rows_nth; + const int dc = (nc + cols_nth - 1) / cols_nth; + + int rows_ith = ith % rows_nth; + int cols_ith = ith % cols_nth; + + // row range for this thread + const int ir0 = dr * rows_ith; + const int ir1 = MIN(ir0 + dr, nr); + + const int cr0 = dc * cols_ith; + const int cr1 = MIN(cr0 + dc, nc); + + int64_t o[4] = { 0, 0, 0, 0 }; + o[dim] = src0->ne[dim]; + const float * x; + + for (int64_t i = ir0; i < ir1; ++i) { + const int64_t i3 = i / (ne02 * ne01); + const int64_t i2 = (i - i3 * ne02 * ne01) / ne01; + const int64_t i1 = (i - i3 * ne02 * ne01 - i2 * ne01); + + for (int i0 = cr0; i0 < cr1; i0++) { + if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { + x = (const float *) ((const char *) src0->data + (i0) *nb00 + (i1) *nb01 + (i2) *nb02 + (i3) *nb03); + } else { + x = (const float *) ((const char *) src1->data + (i0 - o[0]) * nb10 + (i1 - o[1]) * nb11 + + (i2 - o[2]) * nb12 + (i3 - o[3]) * nb13); + } + + float * y = (float *) ((char *) dst->data + i0 * nb0 + i1 * nb1 + i2 * nb2 + i3 * nb3); + + *y = *x; + } + } +} + +template void forward_binary(ggml_compute_params * params, ggml_tensor * op); +template void forward_binary(ggml_compute_params * params, ggml_tensor * op); +template void forward_binary(ggml_compute_params * params, ggml_tensor * op); +template void forward_binary(ggml_compute_params * params, ggml_tensor * op); +template void forward_binary(ggml_compute_params * params, ggml_tensor * op); +template void forward_binary(ggml_compute_params * params, ggml_tensor * op); +template void forward_binary(ggml_compute_params * params, ggml_tensor * op); +template void forward_binary(ggml_compute_params * params, ggml_tensor * op); +template void forward_sum_rows(const ggml_compute_params * params, ggml_tensor * op); +template void forward_sum_rows<_Float16>(const ggml_compute_params * params, ggml_tensor * op); +template void forward_repeat_nrows(ggml_compute_params * params, ggml_tensor * op); +template void forward_repeat_nrows(ggml_compute_params * params, ggml_tensor * op); +template void forward_repeat_dim1(ggml_compute_params * params, ggml_tensor * op); +template void forward_repeat_dim1(ggml_compute_params * params, ggml_tensor * op); +template void forward_get_rows(ggml_compute_params * params, ggml_tensor * op); +template void forward_get_rows(ggml_compute_params * params, ggml_tensor * op); +template void forward_concat(ggml_compute_params * params, ggml_tensor * op); +template void forward_concat(ggml_compute_params * params, ggml_tensor * op); + +} // namespace spacemit_kernels::rvv diff --git a/ggml/src/ggml-cpu/spacemit/rvv_kernels.h b/ggml/src/ggml-cpu/spacemit/rvv_kernels.h new file mode 100644 index 0000000..edddf95 --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/rvv_kernels.h @@ -0,0 +1,95 @@ +#pragma once + +#include "ggml-cpu-impl.h" + +#include +#include +#include +#include + +namespace spacemit_kernels { + +constexpr auto div_round_up(auto up, auto down) { + return (up + down - 1) / down; +} + +// Q8 Blk [f32] [s16] [int8 * blk_len] +// Q8 Blk N [f32 * N] [s16 * N] [int8 * blk_len * N] +constexpr size_t q8_blk_size(size_t blk_len, bool with_blk_sum = false) { + const size_t blk_size = sizeof(float) + blk_len * sizeof(int8_t) + (with_blk_sum ? sizeof(int16_t) : 0); + return blk_size; +} + +// Q8 HP row block: K is split into K32 subblocks. +// Each subblock stores [f32 scale] [int8 * 32], with an optional fp16 sum trailer per subblock. +constexpr size_t q8_hp_blk_size(size_t blk_len, bool with_blk_sum = false, bool with_blk_scale = false) { + const size_t subblk_count = div_round_up(blk_len, size_t(32)); + const size_t blk_size = blk_len * sizeof(int8_t) + subblk_count * sizeof(_Float16) + + (with_blk_sum ? subblk_count * sizeof(_Float16) : 0) + + (with_blk_scale ? sizeof(_Float16) : 0); + return blk_size; +} + +// Q8K Blk [f32] [s16 * (blk_len / 16)] [int8 * blk_len] +// Q8K Blk N [f32 * N] [s16 * (blk_len / 16) * N] [int8 * blk_len * N] +constexpr size_t q8k_blk_size(size_t blk_len) { + const size_t blk_size = sizeof(float) + blk_len * sizeof(int8_t) + sizeof(int16_t) * blk_len / 16; + return blk_size; +} + +using quantize_a_row_def = std::function; + +namespace rvv { +void memcpy1d(void * dst, const void * src, int64_t size); + +void memcpy2d(void * dst, int64_t dst_stride, const void * src, int64_t src_stride, int64_t tile_rows, int64_t size); + +void forward_flash_attn_ext_f16_one_chunk_vlen1024_vf16(const ggml_compute_params * params, + ggml_tensor * dst, + int ir0, + int ir1, + void * tcm_buffer, + size_t tcm_buffer_size); + +void forward_flash_attn_ext_f16_tiled_vlen1024_vf16(const ggml_compute_params * params, + ggml_tensor * dst, + int ir0, + int ir1, + void * tcm_buffer, + size_t tcm_buffer_size); + +void forward_rms_norm_f32(ggml_compute_params * params, ggml_tensor * op); + +void forward_norm_f32(ggml_compute_params * params, ggml_tensor * op); + +void forward_cont_with_permute(ggml_compute_params * params, ggml_tensor * op); + +void forward_cpy_with_permute(ggml_compute_params * params, ggml_tensor * op); + +template void forward_get_rows(ggml_compute_params * params, ggml_tensor * op); + +template void forward_concat(ggml_compute_params * params, ggml_tensor * op); + +template void forward_binary(ggml_compute_params * params, ggml_tensor * op); + +template void forward_sum_rows(const ggml_compute_params * params, ggml_tensor * op); + +template void forward_repeat_nrows(ggml_compute_params * params, ggml_tensor * op); + +template void forward_repeat_dim1(ggml_compute_params * params, ggml_tensor * op); + +void quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr); + +void quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr); + +void quantize_a_row_i8_hp(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr); + +void quantize_a_4row_i8_hp(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr); + +void quantize_a_row_i8k(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr); + +void quantize_a_4row_i8k(size_t blk_len, const float * a_ptr, size_t count_k, uint8_t * quant_a_ptr); + +} // namespace rvv + +} // namespace spacemit_kernels diff --git a/ggml/src/ggml-cpu/spacemit/spine_barrier.h b/ggml/src/ggml-cpu/spacemit/spine_barrier.h new file mode 100644 index 0000000..f897dad --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/spine_barrier.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#define SPINE_CACHE_LINE 64 +#define SPINE_CACHE_ALIGN __attribute__((aligned(SPINE_CACHE_LINE))) + +struct spine_barrier_t { + SPINE_CACHE_ALIGN std::atomic pending_; + SPINE_CACHE_ALIGN std::atomic rounds_; + SPINE_CACHE_ALIGN int64_t total_; +}; + +inline void spine_barrier_wait(spine_barrier_t * b) { + auto cur_round = b->rounds_.load(std::memory_order_acquire); + auto cnt = --b->pending_; + if (cnt == 0) { + b->pending_.store(b->total_); + b->rounds_.store(cur_round + 1); + } else { + while (cur_round == b->rounds_.load(std::memory_order_relaxed)) { + __asm__ volatile("pause " ::: "memory"); + } + } +} + +inline void spine_barrier_init(spine_barrier_t * b, int num_barriers, uint64_t thread_count) { + for (int i = 0; i < num_barriers; i++) { + b[i].total_ = thread_count; + b[i].pending_.store(thread_count); + b[i].rounds_.store(0); + } +} diff --git a/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp b/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp new file mode 100644 index 0000000..1409423 --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp @@ -0,0 +1,760 @@ +#include "spine_mem_pool.h" + +#include "common.h" +#include "ime_env.h" +#include "spine_tcm.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ggml::cpu::riscv64_spacemit { +namespace { + +constexpr size_t SPINE_MEM_POOL_CHUNK_SIZE = 512ull * 1024ull * 1024ull; +constexpr size_t SPINE_SHARE_MEM_POOL_CHUNK_SIZE = 512ull * 1024ull; +constexpr size_t SPINE_MEM_POOL_1G_REGION_SIZE = 1ull << 30; +constexpr uint64_t HUGETLB_1G_FLAG_REQUIRE_PUD = 1ull << 0; +constexpr char SPINE_MEM_POOL_HUGETLB_1G_DEV[] = "/dev/hugetlb_1g"; +constexpr char SPINE_MEM_POOL_TCM_SYNC_MEM_DEV[] = "/dev/tcm_sync_mem"; + +struct hugetlb_1g_region { + uint64_t size{ 0 }; + uint64_t dma_addr{ 0 }; + uint64_t flags{ 0 }; + uint64_t reserved{ 0 }; +}; + +#define HUGETLB_1G_IOC_MAGIC 'M' +#define HUGETLB_1G_IOC_ALLOC _IOWR(HUGETLB_1G_IOC_MAGIC, 0x00, struct hugetlb_1g_region) +#define HUGETLB_1G_IOC_FREE _IO(HUGETLB_1G_IOC_MAGIC, 0x01) + +struct free_block { + size_t offset{ 0 }; + size_t size{ 0 }; +}; + +struct pool_chunk { + uint8_t * base{ nullptr }; + size_t size{ 0 }; + int fd{ -1 }; + std::vector free_blocks; +}; + +struct pool_allocation { + void * chunk_base{ nullptr }; + size_t chunk_size{ 0 }; + void * base{ nullptr }; + size_t size{ 0 }; +}; + +bool is_power_of_two(size_t value) { + return value != 0 && (value & (value - 1)) == 0; +} + +bool align_up(size_t value, size_t alignment, size_t * aligned_value) { + if (aligned_value == nullptr || alignment == 0) { + return false; + } + + const size_t remainder = value % alignment; + if (remainder == 0) { + *aligned_value = value; + return true; + } + + const size_t padding = alignment - remainder; + if (value > std::numeric_limits::max() - padding) { + return false; + } + + *aligned_value = value + padding; + return true; +} + +bool align_up_uintptr(uintptr_t value, size_t alignment, uintptr_t * aligned_value) { + if (aligned_value == nullptr || alignment == 0) { + return false; + } + + const uintptr_t remainder = value % alignment; + if (remainder == 0) { + *aligned_value = value; + return true; + } + + const uintptr_t padding = alignment - remainder; + if (value > std::numeric_limits::max() - padding) { + return false; + } + + *aligned_value = value + padding; + return true; +} + +class spine_mem_pool_manager { + public: + explicit spine_mem_pool_manager(size_t default_chunk_size) : default_chunk_size_(default_chunk_size) {} + + virtual ~spine_mem_pool_manager() = default; + + void * alloc(size_t size, size_t alignment) { + if (size == 0 || !is_power_of_two(alignment)) { + return nullptr; + } + + size_t aligned_size = 0; + if (!align_up(size, alignment, &aligned_size)) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: align_up failed for size %zu alignment %zu\n", __func__, size, + alignment); + return nullptr; + } + + pool_allocation allocation; + + std::lock_guard lock(mutex_); + + if (!try_alloc_locked(aligned_size, alignment, &allocation)) { + if (!add_chunk_locked(aligned_size, alignment)) { + return nullptr; + } + + if (!try_alloc_locked(aligned_size, alignment, &allocation)) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: allocation retry failed for size %zu alignment %zu\n", + __func__, aligned_size, alignment); + return nullptr; + } + } + + try { + const auto [allocation_it, inserted] = allocations_.emplace(allocation.base, allocation); + if (!inserted) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: duplicate allocation key %p\n", __func__, allocation.base); + rollback_allocation_locked(allocation); + return nullptr; + } + } catch (const std::bad_alloc &) { + rollback_allocation_locked(allocation); + throw; + } + + return allocation.base; + } + + void free(void * base) { + if (base == nullptr) { + return; + } + + std::lock_guard lock(mutex_); + + auto allocation_it = allocations_.find(base); + if (allocation_it == allocations_.end()) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: unknown allocation %p\n", __func__, base); + return; + } + + pool_allocation allocation = allocation_it->second; + allocations_.erase(allocation_it); + + auto chunk_it = find_chunk_locked(allocation); + if (chunk_it == chunks_.end()) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: unknown chunk for allocation %p size %zu\n", __func__, + allocation.base, allocation.size); + return; + } + + auto * chunk_base = chunk_it->base; + auto * alloc_base = static_cast(allocation.base); + if (alloc_base < chunk_base || alloc_base >= chunk_base + chunk_it->size) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: allocation %p out of chunk range %p..%p\n", __func__, + allocation.base, chunk_base, chunk_base + chunk_it->size); + return; + } + + const size_t offset = static_cast(alloc_base - chunk_base); + if (offset > chunk_it->size || allocation.size > chunk_it->size - offset) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: allocation %p size %zu exceeds chunk size %zu\n", __func__, + allocation.base, allocation.size, chunk_it->size); + return; + } + + insert_free_block_locked(*chunk_it, { offset, allocation.size }); + maybe_release_empty_chunk_locked(chunk_it); + } + + protected: + void release_chunks() { + std::lock_guard lock(mutex_); + + allocations_.clear(); + for (auto & chunk : chunks_) { + dealloc_chunk(&chunk); + } + chunks_.clear(); + } + + size_t default_chunk_size() const { return default_chunk_size_; } + + static void clear_chunk(pool_chunk * chunk) { + chunk->base = nullptr; + chunk->size = 0; + chunk->fd = -1; + chunk->free_blocks.clear(); + } + + virtual bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) = 0; + virtual void dealloc_chunk(pool_chunk * chunk) = 0; + + private: + struct alloc_candidate { + size_t chunk_index{ 0 }; + size_t block_index{ 0 }; + size_t aligned_offset{ 0 }; + uintptr_t address{ std::numeric_limits::max() }; + bool valid{ false }; + }; + + std::vector::iterator find_chunk_locked(const pool_allocation & allocation) { + return std::find_if(chunks_.begin(), chunks_.end(), [&](const pool_chunk & chunk) { + return chunk.base == allocation.chunk_base && chunk.size == allocation.chunk_size; + }); + } + + bool add_chunk_locked(size_t min_size, size_t alignment) { + pool_chunk chunk; + const size_t chunk_request = default_chunk_size_ == 0 ? min_size : std::max(min_size, default_chunk_size_); + void * hint_addr = nullptr; + + for (const auto & existing_chunk : chunks_) { + auto * chunk_end = existing_chunk.base + existing_chunk.size; + if (hint_addr == nullptr || chunk_end > hint_addr) { + hint_addr = chunk_end; + } + } + + if (!alloc_chunk(chunk_request, alignment, hint_addr, &chunk)) { + return false; + } + + if (chunk.base == nullptr || chunk.size < min_size) { + GGML_LOG_ERROR( + "CPU_RISCV64_SPACEMIT: %s: invalid chunk returned for request size %zu, chunk_base=%p chunk_size=%zu\n", + __func__, min_size, chunk.base, chunk.size); + dealloc_chunk(&chunk); + return false; + } + + try { + chunk.free_blocks.push_back({ 0, chunk.size }); + chunks_.push_back(std::move(chunk)); + } catch (const std::bad_alloc &) { + dealloc_chunk(&chunk); + throw; + } + + return true; + } + + void rollback_allocation_locked(const pool_allocation & allocation) { + auto chunk_it = find_chunk_locked(allocation); + if (chunk_it == chunks_.end()) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to rollback allocation %p, owning chunk not found\n", + __func__, allocation.base); + return; + } + + auto * chunk_base = chunk_it->base; + auto * alloc_base = static_cast(allocation.base); + if (alloc_base < chunk_base || alloc_base >= chunk_base + chunk_it->size) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to rollback allocation %p, chunk range is invalid\n", + __func__, allocation.base); + return; + } + + const size_t offset = static_cast(alloc_base - chunk_base); + if (offset > chunk_it->size || allocation.size > chunk_it->size - offset) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to rollback allocation %p size %zu\n", __func__, + allocation.base, allocation.size); + return; + } + + insert_free_block_locked(*chunk_it, { offset, allocation.size }); + maybe_release_empty_chunk_locked(chunk_it); + } + + bool try_alloc_locked(size_t size, size_t alignment, pool_allocation * allocation) { + alloc_candidate best; + + for (size_t chunk_index = 0; chunk_index < chunks_.size(); ++chunk_index) { + const auto & chunk = chunks_[chunk_index]; + for (size_t block_index = 0; block_index < chunk.free_blocks.size(); ++block_index) { + const auto & block = chunk.free_blocks[block_index]; + + uintptr_t aligned_addr = 0; + const auto block_addr = reinterpret_cast(chunk.base + block.offset); + if (!align_up_uintptr(block_addr, alignment, &aligned_addr)) { + continue; + } + + if (aligned_addr < block_addr) { + continue; + } + + const size_t aligned_offset = block.offset + static_cast(aligned_addr - block_addr); + const size_t padding = aligned_offset - block.offset; + if (padding > block.size || size > block.size - padding) { + continue; + } + + if (!best.valid || aligned_addr < best.address) { + best.chunk_index = chunk_index; + best.block_index = block_index; + best.aligned_offset = aligned_offset; + best.address = aligned_addr; + best.valid = true; + } + } + } + + if (!best.valid) { + return false; + } + + auto & chunk = chunks_[best.chunk_index]; + const free_block block = chunk.free_blocks[best.block_index]; + const size_t padding = best.aligned_offset - block.offset; + const size_t alloc_end = best.aligned_offset + size; + const size_t block_end = block.offset + block.size; + + chunk.free_blocks.erase(chunk.free_blocks.begin() + best.block_index); + auto insert_it = chunk.free_blocks.begin() + best.block_index; + if (padding != 0) { + insert_it = chunk.free_blocks.insert(insert_it, { block.offset, padding }); + ++insert_it; + } + if (alloc_end < block_end) { + chunk.free_blocks.insert(insert_it, { alloc_end, block_end - alloc_end }); + } + + allocation->chunk_base = chunk.base; + allocation->chunk_size = chunk.size; + allocation->base = chunk.base + best.aligned_offset; + allocation->size = size; + return true; + } + + void maybe_release_empty_chunk_locked(std::vector::iterator chunk_it) { + if (chunk_it->free_blocks.size() != 1) { + return; + } + + const auto & block = chunk_it->free_blocks.front(); + if (block.offset != 0 || block.size != chunk_it->size) { + return; + } + + dealloc_chunk(&*chunk_it); + chunks_.erase(chunk_it); + } + + void insert_free_block_locked(pool_chunk & chunk, free_block block) { + auto it = chunk.free_blocks.begin(); + while (it != chunk.free_blocks.end() && it->offset < block.offset) { + ++it; + } + + if (it != chunk.free_blocks.begin()) { + const auto & prev = *(it - 1); + if (prev.offset + prev.size > block.offset) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: overlapping free block at offset %zu size %zu\n", __func__, + block.offset, block.size); + return; + } + } + + if (it != chunk.free_blocks.end() && block.offset + block.size > it->offset) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: overlapping next free block at offset %zu size %zu\n", __func__, + block.offset, block.size); + return; + } + + it = chunk.free_blocks.insert(it, block); + + if (it != chunk.free_blocks.begin()) { + auto prev = it - 1; + if (prev->offset + prev->size == it->offset) { + it->offset = prev->offset; + it->size += prev->size; + it = chunk.free_blocks.erase(prev); + } + } + + if (it + 1 != chunk.free_blocks.end() && it->offset + it->size == (it + 1)->offset) { + it->size += (it + 1)->size; + chunk.free_blocks.erase(it + 1); + } + } + + std::mutex mutex_; + std::vector chunks_; + std::unordered_map allocations_; + size_t default_chunk_size_{ 0 }; +}; + +class spine_mem_pool_posix final : public spine_mem_pool_manager { + public: + spine_mem_pool_posix() : spine_mem_pool_manager(0) {} + + ~spine_mem_pool_posix() override { release_chunks(); } + + private: + bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) override { + (void) hint_addr; + + const size_t alloc_alignment = std::max(alignment, sizeof(void *)); + void * base = nullptr; + const int rc = posix_memalign(&base, alloc_alignment, min_size); + if (rc != 0) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: posix_memalign failed for size %zu alignment %zu, rc=%d\n", + __func__, min_size, alloc_alignment, rc); + return false; + } + + chunk->base = static_cast(base); + chunk->size = min_size; + chunk->fd = -1; + return true; + } + + void dealloc_chunk(pool_chunk * chunk) override { + std::free(chunk->base); + clear_chunk(chunk); + } +}; + +class spine_mem_pool_transparent_hugepage final : public spine_mem_pool_manager { + public: + spine_mem_pool_transparent_hugepage() : spine_mem_pool_manager(SPINE_MEM_POOL_CHUNK_SIZE) {} + + ~spine_mem_pool_transparent_hugepage() override { release_chunks(); } + + private: + bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) override { + (void) alignment; + + size_t chunk_size = 0; + if (!align_up(min_size, default_chunk_size(), &chunk_size)) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to round chunk size for %zu\n", __func__, min_size); + return false; + } + + void * map_addr = mmap(hint_addr, chunk_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map_addr == MAP_FAILED) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: mmap failed for chunk size %zu, errno=%d\n", __func__, chunk_size, + errno); + return false; + } + + if (madvise(map_addr, chunk_size, MADV_HUGEPAGE) != 0) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: madvise(MADV_HUGEPAGE) failed for chunk size %zu, errno=%d\n", + __func__, chunk_size, errno); + munmap(map_addr, chunk_size); + return false; + } + + chunk->base = static_cast(map_addr); + chunk->size = chunk_size; + chunk->fd = -1; + return true; + } + + void dealloc_chunk(pool_chunk * chunk) override { + if (chunk->base != nullptr && chunk->size != 0 && munmap(chunk->base, chunk->size) != 0) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: munmap failed for chunk %p size %zu, errno=%d\n", __func__, + chunk->base, chunk->size, errno); + } + + clear_chunk(chunk); + } +}; + +class spine_mem_pool_hugetlb_1g final : public spine_mem_pool_manager { + public: + spine_mem_pool_hugetlb_1g() : spine_mem_pool_manager(SPINE_MEM_POOL_1G_REGION_SIZE) {} + + ~spine_mem_pool_hugetlb_1g() override { release_chunks(); } + + private: + bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) override { + (void) alignment; + (void) hint_addr; + + size_t region_size = 0; + if (!align_up(min_size, SPINE_MEM_POOL_1G_REGION_SIZE, ®ion_size)) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to round hugetlb_1g size for %zu\n", __func__, min_size); + return false; + } + + const int fd = open(SPINE_MEM_POOL_HUGETLB_1G_DEV, O_RDWR); + if (fd < 0) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: open(%s) failed, errno=%d\n", __func__, + SPINE_MEM_POOL_HUGETLB_1G_DEV, errno); + return false; + } + + hugetlb_1g_region region; + region.size = region_size; + region.flags = HUGETLB_1G_FLAG_REQUIRE_PUD; + if (ioctl(fd, HUGETLB_1G_IOC_ALLOC, ®ion) < 0) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: HUGETLB_1G_IOC_ALLOC failed for size %zu, errno=%d\n", __func__, + region_size, errno); + close(fd); + return false; + } + + void * map_addr = mmap(nullptr, region.size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (map_addr == MAP_FAILED) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: mmap failed for hugetlb_1g size %llu, errno=%d\n", __func__, + static_cast(region.size), errno); + ioctl(fd, HUGETLB_1G_IOC_FREE); + close(fd); + return false; + } + + chunk->base = static_cast(map_addr); + chunk->size = region.size; + chunk->fd = fd; + return true; + } + + void dealloc_chunk(pool_chunk * chunk) override { + if (chunk->base != nullptr && chunk->size != 0 && munmap(chunk->base, chunk->size) != 0) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: munmap failed for hugetlb_1g chunk %p size %zu, errno=%d\n", + __func__, chunk->base, chunk->size, errno); + } + + if (chunk->fd >= 0) { + if (ioctl(chunk->fd, HUGETLB_1G_IOC_FREE) < 0) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: HUGETLB_1G_IOC_FREE failed for chunk %p, errno=%d\n", + __func__, chunk->base, errno); + } + + close(chunk->fd); + } + + clear_chunk(chunk); + } +}; + +class spine_mem_pool_shared_mem final : public spine_mem_pool_manager { + public: + spine_mem_pool_shared_mem() : spine_mem_pool_manager(SPINE_SHARE_MEM_POOL_CHUNK_SIZE) {} + + ~spine_mem_pool_shared_mem() override { release_chunks(); } + + private: + bool alloc_chunk(size_t min_size, size_t alignment, void * hint_addr, pool_chunk * chunk) override { + (void) alignment; + + if (hint_addr != nullptr) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: shared_mem does not support multiple active chunks\n", __func__); + return false; + } + + if (min_size > default_chunk_size()) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: shared_mem request %zu exceeds chunk size %zu\n", __func__, + min_size, default_chunk_size()); + return false; + } + + const int fd = open(SPINE_MEM_POOL_TCM_SYNC_MEM_DEV, O_RDWR | O_SYNC); + if (fd < 0) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: open(%s) failed, errno=%d\n", __func__, + SPINE_MEM_POOL_TCM_SYNC_MEM_DEV, errno); + return false; + } + + void * map_addr = mmap(nullptr, default_chunk_size(), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (map_addr == MAP_FAILED) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: mmap failed for %s size %zu, errno=%d\n", __func__, + SPINE_MEM_POOL_TCM_SYNC_MEM_DEV, default_chunk_size(), errno); + close(fd); + return false; + } + + chunk->base = static_cast(map_addr); + chunk->size = default_chunk_size(); + chunk->fd = fd; + return true; + } + + void dealloc_chunk(pool_chunk * chunk) override { + if (chunk->base != nullptr && chunk->size != 0 && munmap(chunk->base, chunk->size) != 0) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: munmap failed for shared_mem chunk %p size %zu, errno=%d\n", + __func__, chunk->base, chunk->size, errno); + } + + if (chunk->fd >= 0) { + close(chunk->fd); + } + + clear_chunk(chunk); + } +}; + +spine_mem_pool_manager & get_spine_mem_pool_manager() { + static std::once_flag pool_once; + static std::unique_ptr selected_pool; + static spine_mem_pool_backend selected_backend = spine_mem_pool_backend::none; + + spine_mem_pool_backend backend = global_spine_env_info.mem_backend; + if (backend == spine_mem_pool_backend::none) { + backend = spine_mem_pool_backend::transparent_hugepage; + } + + std::call_once(pool_once, [&]() { + selected_backend = backend; + + switch (selected_backend) { + case spine_mem_pool_backend::posix_memalign: + selected_pool = std::make_unique(); + break; + case spine_mem_pool_backend::transparent_hugepage: + selected_pool = std::make_unique(); + break; + case spine_mem_pool_backend::hugetlb_1g: + selected_pool = std::make_unique(); + break; + case spine_mem_pool_backend::none: + selected_backend = spine_mem_pool_backend::transparent_hugepage; + selected_pool = std::make_unique(); + break; + } + }); + + if (backend != selected_backend) { + GGML_LOG_ERROR( + "CPU_RISCV64_SPACEMIT: %s: mem pool backend is process-global and mutually exclusive, requested=%d but " + "selected=%d\n", + __func__, static_cast(backend), static_cast(selected_backend)); + } + + if (selected_pool) { + return *selected_pool; + } + + throw std::bad_alloc(); +} + +spine_mem_pool_manager & get_spine_mem_pool_shared_mem_manager() { + static std::once_flag shared_mem_pool_once; + static std::unique_ptr shared_mem_pool; + + std::call_once(shared_mem_pool_once, [&]() { shared_mem_pool = std::make_unique(); }); + + if (shared_mem_pool) { + return *shared_mem_pool; + } + + throw std::bad_alloc(); +} + +} // namespace + +bool spine_mem_pool_tcm_init(spine_mem_pool_tcm_info * info) noexcept { + if (info == nullptr) { + return false; + } + + *info = {}; + + if (spine_tcm_open_handle(NULL) != 0 || !spine_tcm_is_available()) { + return false; + } + + spine_tcm_mem_info_t mem_info; + if (spine_tcm_mem_info(&mem_info) != 0) { + return false; + } + + info->available = true; + info->blk_size = mem_info.blk_size; + info->blk_num = mem_info.blk_num; + info->is_fake_tcm = mem_info.is_fake_tcm != 0; + return true; +} + +void * spine_mem_pool_tcm_mem_get(int cpu_id) noexcept { + return spine_tcm_mem_get(cpu_id); +} + +void * spine_mem_pool_tcm_mem_wait(int cpu_id) noexcept { + return spine_tcm_mem_try_wait(cpu_id, 1000 * 1000); +} + +int spine_mem_pool_tcm_mem_release(int cpu_id) noexcept { + return spine_tcm_mem_release(cpu_id); +} + +void * spine_mem_pool_alloc(size_t size, size_t alignment) noexcept { + try { + return get_spine_mem_pool_manager().alloc(size, alignment); + } catch (const std::bad_alloc &) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: bad_alloc while allocating size %zu\n", __func__, size); + return nullptr; + } +} + +void * spine_mem_pool_shared_mem_alloc(size_t size, size_t alignment) noexcept { + try { + return get_spine_mem_pool_shared_mem_manager().alloc(size, alignment); + } catch (const std::bad_alloc &) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: bad_alloc while allocating shared memory size %zu\n", __func__, size); + return nullptr; + } +} + +void spine_mem_pool_free(void * base) noexcept { + try { + get_spine_mem_pool_manager().free(base); + } catch (const std::bad_alloc &) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: bad_alloc while freeing allocation %p\n", __func__, base); + } +} + +void spine_mem_pool_shared_mem_free(void * base) noexcept { + try { + get_spine_mem_pool_shared_mem_manager().free(base); + } catch (const std::bad_alloc &) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: bad_alloc while freeing shared allocation %p\n", __func__, base); + } +} + +} // namespace ggml::cpu::riscv64_spacemit + +extern "C" { +void * ggml_backend_cpu_riscv64_spacemit_alloc_shared(size_t size, size_t alignment) { + void * result = ggml::cpu::riscv64_spacemit::spine_mem_pool_shared_mem_alloc(size, alignment); + if (result == nullptr) { + GGML_LOG_ERROR("CPU_RISCV64_SPACEMIT: %s: failed to allocate shared memory size %zu alignment %zu\n", __func__, + size, alignment); + } + return result; +} + +void ggml_backend_cpu_riscv64_spacemit_free_shared(void * ptr) { + ggml::cpu::riscv64_spacemit::spine_mem_pool_shared_mem_free(ptr); +} +} diff --git a/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h b/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h new file mode 100644 index 0000000..8740d2c --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +namespace ggml::cpu::riscv64_spacemit { + +enum class spine_mem_pool_backend : uint8_t { + none, + posix_memalign, + transparent_hugepage, + hugetlb_1g, +}; + +struct spine_mem_pool_tcm_info { + bool available{ false }; + size_t blk_size{ 0 }; + size_t blk_num{ 0 }; + bool is_fake_tcm{ false }; +}; + +bool spine_mem_pool_tcm_init(spine_mem_pool_tcm_info * info) noexcept; +void * spine_mem_pool_tcm_mem_get(int cpu_id) noexcept; +void * spine_mem_pool_tcm_mem_wait(int cpu_id) noexcept; +int spine_mem_pool_tcm_mem_release(int cpu_id) noexcept; + +void * spine_mem_pool_alloc(size_t size, size_t alignment) noexcept; +void * spine_mem_pool_shared_mem_alloc(size_t size, size_t alignment) noexcept; +void spine_mem_pool_free(void * base) noexcept; +void spine_mem_pool_shared_mem_free(void * base) noexcept; + +} // namespace ggml::cpu::riscv64_spacemit diff --git a/ggml/src/ggml-cpu/spacemit/spine_tcm.h b/ggml/src/ggml-cpu/spacemit/spine_tcm.h new file mode 100644 index 0000000..f300d7d --- /dev/null +++ b/ggml/src/ggml-cpu/spacemit/spine_tcm.h @@ -0,0 +1,409 @@ +#ifndef SPINE_TCM_PUBLIC_H_ +#define SPINE_TCM_PUBLIC_H_ + +/* + * spine_tcm public API + * + * Usage: + * 1. Direct link mode + * Define SPINE_TCM_DIRECT_LINK and link against libspine_tcm.so. + * + * if (spine_tcm_is_available()) { + * void *buffer = spine_tcm_mem_get(0); + * spine_tcm_mem_free(0); + * } + * + * 2. Header-only loader mode + * Include this header without linking libspine_tcm.so. The loader first + * tries to reuse a process-global spine_tcm instance and falls back to + * dlopen("libspine_tcm.so") when needed. + * + * spine_tcm_open_handle(NULL); // optional pre-bind + * if (spine_tcm_is_available()) { + * void *buffer = spine_tcm_mem_get(0); + * spine_tcm_mem_free(0); + * } + */ + +#include +#include +#include + +#if !defined(SPINE_TCM_BUILD_SHARED) && !defined(SPINE_TCM_DIRECT_LINK) +# include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_WIN32) +# if defined(SPINE_TCM_BUILD_SHARED) +# define SPINE_TCM_API __declspec(dllexport) +# else +# define SPINE_TCM_API __declspec(dllimport) +# endif +#else +# define SPINE_TCM_API __attribute__((visibility("default"))) +#endif + +typedef struct spine_tcm_mem_info { + size_t blk_size; + size_t blk_num; + int is_fake_tcm; +} spine_tcm_mem_info_t; + +typedef struct spine_tcm_block_info { + int id; + void * va; + size_t size; + uint64_t phys_addr; + uint64_t cpu_affinity_mask; + int owner_tid; + int is_acquired; +} spine_tcm_block_info_t; + +/* Shared-library runtime ABI exported by libspine_tcm.so. */ +SPINE_TCM_API const char * spine_tcm_runtime_version(void); +SPINE_TCM_API int spine_tcm_runtime_is_available(void); +SPINE_TCM_API int spine_tcm_runtime_layout_info(spine_tcm_mem_info_t * info); +SPINE_TCM_API int spine_tcm_runtime_mem_info(int id, spine_tcm_block_info_t * info); +SPINE_TCM_API void * spine_tcm_runtime_mem_get(int id); +SPINE_TCM_API int spine_tcm_runtime_mem_free(int id); +SPINE_TCM_API void * spine_tcm_runtime_mem_try_wait(int id, size_t timeout_us); +SPINE_TCM_API int spine_tcm_runtime_mem_release(int id); +SPINE_TCM_API int spine_tcm_runtime_mem_force_release(int id); +SPINE_TCM_API int spine_tcm_runtime_mem_query(int id); + +#if defined(SPINE_TCM_DIRECT_LINK) +/* Optional no-op in direct-link mode. */ +static inline int spine_tcm_open_handle(const char * so_path) { + (void) so_path; + return 0; +} + +static inline const char * spine_tcm_version(void) { + return spine_tcm_runtime_version(); +} + +/* Returns 1 when the runtime driver is available, otherwise 0. */ +static inline int spine_tcm_is_available(void) { + return spine_tcm_runtime_is_available(); +} + +/* Returns runtime memory geometry and whether the current backend is fake TCM. */ +static inline int spine_tcm_mem_info(spine_tcm_mem_info_t * info) { + return spine_tcm_runtime_layout_info(info); +} + +/* Returns per-block runtime metadata for the given TCM id. */ +static inline int spine_tcm_block_info(int id, spine_tcm_block_info_t * info) { + return spine_tcm_runtime_mem_info(id, info); +} + +/* Returns a cached buffer for the given TCM id, or NULL on failure. */ +static inline void * spine_tcm_mem_get(int id) { + return spine_tcm_runtime_mem_get(id); +} + +/* Releases one reference acquired by spine_tcm_mem_get(id). */ +static inline int spine_tcm_mem_free(int id) { + return spine_tcm_runtime_mem_free(id); +} + +/* Waits for a TCM block handoff and returns the driver-owned buffer when available. */ +static inline void * spine_tcm_mem_try_wait(int id, size_t over_time) { + return spine_tcm_runtime_mem_try_wait(id, over_time); +} + +/* Releases a buffer acquired by spine_tcm_mem_try_wait(id, over_time). */ +static inline int spine_tcm_mem_release(int id) { + return spine_tcm_runtime_mem_release(id); +} + +/* Forces a release for the given TCM id when the backend supports it. */ +static inline int spine_tcm_mem_force_release(int id) { + return spine_tcm_runtime_mem_force_release(id); +} + +/* Returns whether the given TCM id is currently acquired. */ +static inline int spine_tcm_mem_query(int id) { + return spine_tcm_runtime_mem_query(id); +} +#elif !defined(SPINE_TCM_BUILD_SHARED) +typedef struct spine_tcm_handle { + void * module_handle; + int use_global_scope; + int owns_module_handle; + const char * (*runtime_version)(void); + int (*runtime_is_available)(void); + int (*runtime_layout_info)(spine_tcm_mem_info_t * info); + int (*runtime_mem_info)(int id, spine_tcm_block_info_t * info); + void * (*runtime_mem_get)(int id); + int (*runtime_mem_free)(int id); + void * (*runtime_mem_try_wait)(int id, size_t over_time); + int (*runtime_mem_release)(int id); + int (*runtime_mem_force_release)(int id); + int (*runtime_mem_query)(int id); +} spine_tcm_handle_t; + +static inline spine_tcm_handle_t * spine_tcm_default_handle(void) { + static spine_tcm_handle_t handle = { 0 }; + return &handle; +} + +static inline void spine_tcm_handle_reset(spine_tcm_handle_t * handle) { + if (handle != NULL) { + memset(handle, 0, sizeof(*handle)); + } +} + +static inline int spine_tcm_handle_bind(spine_tcm_handle_t * handle) { + void * symbol_scope = handle->use_global_scope ? RTLD_DEFAULT : handle->module_handle; + + handle->runtime_version = (const char * (*) (void) ) dlsym(symbol_scope, "spine_tcm_runtime_version"); + handle->runtime_is_available = (int (*)(void)) dlsym(symbol_scope, "spine_tcm_runtime_is_available"); + handle->runtime_layout_info = + (int (*)(spine_tcm_mem_info_t *)) dlsym(symbol_scope, "spine_tcm_runtime_layout_info"); + handle->runtime_mem_info = + (int (*)(int, spine_tcm_block_info_t *)) dlsym(symbol_scope, "spine_tcm_runtime_mem_info"); + handle->runtime_mem_get = (void * (*) (int) ) dlsym(symbol_scope, "spine_tcm_runtime_mem_get"); + handle->runtime_mem_free = (int (*)(int)) dlsym(symbol_scope, "spine_tcm_runtime_mem_free"); + handle->runtime_mem_try_wait = (void * (*) (int, size_t)) dlsym(symbol_scope, "spine_tcm_runtime_mem_try_wait"); + handle->runtime_mem_release = (int (*)(int)) dlsym(symbol_scope, "spine_tcm_runtime_mem_release"); + handle->runtime_mem_force_release = (int (*)(int)) dlsym(symbol_scope, "spine_tcm_runtime_mem_force_release"); + handle->runtime_mem_query = (int (*)(int)) dlsym(symbol_scope, "spine_tcm_runtime_mem_query"); + + return handle->runtime_version != NULL && handle->runtime_is_available != NULL && + handle->runtime_layout_info != NULL && handle->runtime_mem_info != NULL && + handle->runtime_mem_get != NULL && handle->runtime_mem_free != NULL && + handle->runtime_mem_try_wait != NULL && handle->runtime_mem_release != NULL && + handle->runtime_mem_force_release != NULL && handle->runtime_mem_query != NULL ? + 0 : + -1; +} + +/* + * Try to bind against an already-loaded process-global spine_tcm instance. + * The shared library exports spine_tcm_runtime_marker only for this probe. + */ +static inline int spine_tcm_try_bind_global(spine_tcm_handle_t * handle) { + if (dlsym(RTLD_DEFAULT, "spine_tcm_runtime_marker") == NULL) { + return -1; + } + + handle->use_global_scope = 1; + return spine_tcm_handle_bind(handle); +} + +/* + * Optional pre-bind entry point. + * + * Behavior: + * - Reuses an already-loaded global spine_tcm instance when available. + * - Otherwise loads the shared library from so_path or the default soname. + * - Repeated calls are safe and return 0 after the first successful bind. + */ +static inline int spine_tcm_open_handle(const char * so_path) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + const char * library = (so_path != NULL && so_path[0] != '\0') ? so_path : "libspine_tcm.so"; + + if (resolved->module_handle != NULL || resolved->use_global_scope) { + return 0; + } + + if (spine_tcm_try_bind_global(resolved) == 0) { + return 0; + } + + spine_tcm_handle_reset(resolved); + + resolved->module_handle = dlopen(library, RTLD_LAZY | RTLD_GLOBAL); + resolved->owns_module_handle = resolved->module_handle != NULL ? 1 : 0; + + if (resolved->module_handle == NULL) { + spine_tcm_handle_reset(resolved); + return -1; + } + + if (spine_tcm_handle_bind(resolved) != 0) { + if (resolved->owns_module_handle) { + dlclose(resolved->module_handle); + } + spine_tcm_handle_reset(resolved); + return -1; + } + + return 0; +} + +/* Returns 1 when the runtime driver is available, otherwise 0. */ +static inline int spine_tcm_is_available(void) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_is_available == NULL) { + return 0; + } + + return resolved->runtime_is_available(); +} + +/* Returns runtime memory geometry and whether the current backend is fake TCM. */ +static inline int spine_tcm_mem_info(spine_tcm_mem_info_t * info) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_layout_info == NULL) { + return -1; + } + + return resolved->runtime_layout_info(info); +} + +static inline const char * spine_tcm_version(void) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_version == NULL) { + return "unknown"; + } + + return resolved->runtime_version(); +} + +/* Returns per-block runtime metadata for the given TCM id. */ +static inline int spine_tcm_block_info(int id, spine_tcm_block_info_t * info) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_mem_info == NULL) { + return -1; + } + + return resolved->runtime_mem_info(id, info); +} + +/* Returns a cached buffer for the given TCM id, or NULL on failure. */ +static inline void * spine_tcm_mem_get(int id) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + return NULL; + } + + if (resolved->runtime_mem_get == NULL) { + return NULL; + } + + return resolved->runtime_mem_get(id); +} + +/* Releases one reference acquired by spine_tcm_mem_get(id). */ +static inline int spine_tcm_mem_free(int id) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_mem_free == NULL) { + return -1; + } + + return resolved->runtime_mem_free(id); +} + +/* Waits for a TCM block handoff and returns the driver-owned buffer when available. */ +static inline void * spine_tcm_mem_try_wait(int id, size_t over_time) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + return NULL; + } + + if (resolved->runtime_mem_try_wait == NULL) { + return NULL; + } + + return resolved->runtime_mem_try_wait(id, over_time); +} + +/* Releases a buffer acquired by spine_tcm_mem_try_wait(id, over_time). */ +static inline int spine_tcm_mem_release(int id) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_mem_release == NULL) { + return -1; + } + + return resolved->runtime_mem_release(id); +} + +/* Forces a release for the given TCM id when the backend supports it. */ +static inline int spine_tcm_mem_force_release(int id) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if ((resolved->module_handle == NULL && !resolved->use_global_scope) || + resolved->runtime_mem_force_release == NULL) { + return -1; + } + + return resolved->runtime_mem_force_release(id); +} + +/* Returns whether the given TCM id is currently acquired. */ +static inline int spine_tcm_mem_query(int id) { + spine_tcm_handle_t * resolved = spine_tcm_default_handle(); + + if (resolved->module_handle == NULL && !resolved->use_global_scope) { + (void) spine_tcm_open_handle(NULL); + } + + if ((resolved->module_handle == NULL && !resolved->use_global_scope) || resolved->runtime_mem_query == NULL) { + return -1; + } + + return resolved->runtime_mem_query(id); +} +#else +static inline const char * spine_tcm_version(void) { + return spine_tcm_runtime_version(); +} +#endif + +#define SPINE_TCM_VERSION (spine_tcm_version()) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ggml/src/ggml-cpu/tiled/tiled-kernel.cpp b/ggml/src/ggml-cpu/tiled/tiled-kernel.cpp new file mode 100644 index 0000000..5819ac0 --- /dev/null +++ b/ggml/src/ggml-cpu/tiled/tiled-kernel.cpp @@ -0,0 +1,714 @@ +// mulmat microtile kernels + +#include "tiled-kernel.h" + +#include "ggml.h" + +#include + +#if defined(__AVX512VNNI__) || defined(__AVX2__) || defined(__AVX__) +#include +#endif + + +// Reference implementation, slower than existing vec_dot approach +template +static void tiled_run_microtile_scalar(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride) { + constexpr int NB = TILED_TILE_K / SUBBLK; // subblocks per 256-K block + constexpr int NS = SUBBLK / 16; // per-16 bsums per subblock + + // num_k/slab: the narrow path holds num_k slabs at row stride num_k*256 (num_k=1, slab=0 = standard) + // NK: 1 = standard single-slab (offsets 0, strides compile-time); 0 = runtime + // (num_k/slab from the args). The narrow path is memory-bound, so one runtime + // version serves all of it. + const int nkr = (NK > 0) ? NK : num_k; + const int seff = (NK > 0) ? 0 : slab; + const int qk_stride = nkr * TILED_TILE_K; + const int qk_off = seff * TILED_TILE_K; + const int nb_stride = NB * nkr; + const int nb_off = seff * NB; + const int bs_stride = (nkr == 1) ? TILED_TILE_ROWS : TILED_MICRO; + const int bs_off = seff * TILED_MICRO; + + float acc[TILED_MICRO][TILED_MICRO]; + memset(acc, 0, sizeof(acc)); + + // subdots at subblock granularity over the 256-K block, exact integer math + for (int s = 0; s < NB; s++) { + for (int j = 0; j < TILED_MICRO; j++) { + const int br = j0 + j; + const int8_t * q1 = &src1.q[br * qk_stride + qk_off + s * SUBBLK]; + int32_t bsum = 0; + for (int u = 0; u < NS; u++) { + bsum += src1.bsums[(bs_off + s * NS + u) * bs_stride + br]; + } + + for (int i = 0; i < TILED_MICRO; i++) { + const int ar = i0 + i; + const int d_off = seff * TILED_MICRO + ar; + const uint8_t * q0 = &src0.q[ar * qk_stride + qk_off + s * SUBBLK]; + + int32_t raw = 0; + for (int e = 0; e < SUBBLK; e++) { + raw += (int32_t) q0[e] * (int32_t) q1[e]; + } + + // BIAS: subtract BIAS*bsum (src1's per-subblock code sum) from the exact int raw + int32_t corr = raw; + if constexpr (BIAS != 0) { + corr -= BIAS * bsum; + } + const int32_t scales_raw = (int32_t) src0.scales[ar * nb_stride + nb_off + s] * corr; + // d is NOT applied here: it is constant over the s-loop, so we apply it last before write-out + if constexpr (HAS_MIN) { + const int32_t mins_bsum = (int32_t) src0.mins[ar * nb_stride + nb_off + s] * bsum; + acc[i][j] += (float) src0.d[d_off] * (float) scales_raw + - (float) src0.dmin[d_off] * (float) mins_bsum; + } else { + acc[i][j] += (float) src0.d[d_off] * (float) scales_raw; + } + } + } + } + + // Apply d and write out to buf + for (int i = 0; i < TILED_MICRO; i++) { + for (int j = 0; j < TILED_MICRO; j++) { + buf[(i0 + i) * buf_stride + (j0 + j)] += src1.d[seff * TILED_MICRO + j0 + j] * acc[i][j]; + } + } +} + +#if defined(__AVX512VNNI__) && defined(__AVX512VL__) && defined(__AVX512DQ__) + + +// VNNI microkernel +// 8x16 band pass: 8 src0 rows (i0, i0+8) x 16 src1 cols (j0, j0+16) +// Math: s1_acc += scales_s*raw_s, s2_acc += mins_s*bsums_s per subblock; the row result +// is d0*s1_acc - dmin*s2_acc (f32, exact in range), applied once per row. +// +// Register pressure: the band pass holds acc16 + s1_acc = 16 zmm for the +// 8-row band (s2_acc has no dpbusd dependency, so it is computed after the +// band pass and adds no register pressure); +template +static void tiled_run_micro_vnni_8x16(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride) { + constexpr int NB = TILED_TILE_K / SUBBLK; + constexpr int NS = SUBBLK / 16; + constexpr int NG = SUBBLK / 4; + + // band width, see the register-pressure note above + constexpr int NUM_ROWS = 8; + + // num_k = K-blocks per row (the narrow path holds num_k slabs at row stride num_k*256 so + // each weight row is one long stream); slab = this call's slab. num_k=1, slab=0 = standard. + // NK: 1 = standard single-slab (offsets 0, strides compile-time); 0 = runtime + // (num_k/slab from the args). The narrow path is memory-bound, so one runtime + // version serves all of it. + const int nkr = (NK > 0) ? NK : num_k; + const int seff = (NK > 0) ? 0 : slab; + const int qk_stride = nkr * TILED_TILE_K; + const int qk_off = seff * TILED_TILE_K; + const int nb_stride = NB * nkr; + const int nb_off = seff * NB; + const int bs_stride = (nkr == 1) ? TILED_TILE_ROWS : TILED_MICRO; + const int bs_off = seff * TILED_MICRO; + + const __m512 d1_vec = _mm512_load_ps(&src1.d[seff * TILED_MICRO + j0]); + + __m512i s1_acc[NUM_ROWS]; + for (int t = 0; t < NUM_ROWS; t++) { s1_acc[t] = _mm512_setzero_si512(); } + __m512i s2_acc[NUM_ROWS]; + if constexpr (HAS_MIN) { + for (int t = 0; t < NUM_ROWS; t++) { s2_acc[t] = _mm512_setzero_si512(); } + } + + const int32_t * bsums = src1.bsums; // int32 per-16 sums; NS > 1 combines NS lanes per subblock + for (int s = 0; s < NB; s++) { + __m512i bsums32 = _mm512_load_si512((const __m512i *) &bsums[(bs_off + s * NS) * bs_stride + j0]); + for (int u = 1; u < NS; u++) { + bsums32 = _mm512_add_epi32(bsums32, _mm512_load_si512((const __m512i *) &bsums[(bs_off + s * NS + u) * bs_stride + j0])); + } + + __m512i bias32 = _mm512_setzero_si512(); + if constexpr (BIAS != 0) { + bias32 = _mm512_mullo_epi32(bsums32, _mm512_set1_epi32(BIAS)); + } + + __m512i acc16[NUM_ROWS]; + for (int t = 0; t < NUM_ROWS; t++) { acc16[t] = _mm512_setzero_si512(); } + +#ifdef __GNUC__ + #pragma GCC unroll 8 // pragma unrolled justified by measuing with/without +#endif + for (int g = 0; g < NG; g++) { + const int kg = s * NG + g; + // in-place interleave layout: [kg%16 @ TILED_TILE_K][kg/16 @ 64][row @ 4] + const __m512i codes = _mm512_load_si512((const __m512i *) &src1.q[(j0 / TILED_MICRO) * (TILED_MICRO * qk_stride) + + qk_off + (kg % TILED_MICRO) * qk_stride + (kg / TILED_MICRO) * (TILED_MICRO * 4)]); + for (int t = 0; t < NUM_ROWS; t++) { + const uint32_t u4 = *(const uint32_t *) &src0.q[(i0 + t) * qk_stride + qk_off + kg * 4]; + const __m512i u4b = _mm512_set1_epi32((int) u4); + acc16[t] = _mm512_dpbusd_epi32(acc16[t], u4b, codes); + } + } + + // int correction: s1_acc += scales*(raw-BIAS*bsums) + for (int t = 0; t < NUM_ROWS; t++) { + const int ar = i0 + t; + __m512i rawi = acc16[t]; + if constexpr (BIAS != 0) { + rawi = _mm512_sub_epi32(rawi, bias32); + } + s1_acc[t] = _mm512_add_epi32(s1_acc[t], + _mm512_mullo_epi32(rawi, _mm512_set1_epi32(src0.scales[ar * nb_stride + nb_off + s]))); + } + } + + // s2_acc += mins*bsums; independent of the dpbusd results, so it runs here instead of in the + // band pass: the band pass keeps the register budget for the 8-row band + if constexpr (HAS_MIN) { + for (int s = 0; s < NB; s++) { + __m512i bsums32 = _mm512_load_si512((const __m512i *) &bsums[(bs_off + s * NS) * bs_stride + j0]); + for (int u = 1; u < NS; u++) { + bsums32 = _mm512_add_epi32(bsums32, _mm512_load_si512((const __m512i *) &bsums[(bs_off + s * NS + u) * bs_stride + j0])); + } + for (int t = 0; t < NUM_ROWS; t++) { + s2_acc[t] = _mm512_add_epi32(s2_acc[t], + _mm512_mullo_epi32(bsums32, _mm512_set1_epi32(src0.mins[(i0 + t) * nb_stride + nb_off + s]))); + } + } + } + + // epilogue: int->float, apply per-row scales, store to buf + for (int t = 0; t < NUM_ROWS; t++) { + const int ar = i0 + t; + const int d_off = seff * TILED_MICRO + ar; + __m512 f1 = _mm512_cvtepi32_ps(s1_acc[t]); + __m512 result = _mm512_mul_ps(f1, _mm512_set1_ps(src0.d[d_off])); + if constexpr (HAS_MIN) { + __m512 f2 = _mm512_cvtepi32_ps(s2_acc[t]); + result = _mm512_fnmadd_ps(_mm512_set1_ps(src0.dmin[d_off]), f2, result); + } + float * p = &buf[(i0 + t) * buf_stride + j0]; + _mm512_store_ps(p, _mm512_add_ps(_mm512_load_ps(p), _mm512_mul_ps(result, d1_vec))); + } +} + +// 16x16 microtile as two explicit 8x16 band passes +template +static void tiled_run_microtile_vnni(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride) { + tiled_run_micro_vnni_8x16(src0, src1, i0, j0, num_k, slab, buf, buf_stride); + tiled_run_micro_vnni_8x16(src0, src1, i0 + 8, j0, num_k, slab, buf, buf_stride); +} + +#endif // __AVX512VNNI__ && __AVX512VL__ + +#if defined(__AVX2__) + +// AVX2 kernel. +// We're effectively applying the existing vec_dot algorithms to an 8x16 block here. +// Different paths based on subblock size as it affects when/where we multiply in scales and apply mins +// SUBBLK=32: one 256-bit maddubs per (s, column), a 256-bit set1 scale, +// one 8-lane i32 accumulator per column. +// SUBBLK=16: two subblocks per 32B load. The 256-bit maddubs product +// still gives 16 i16 lanes (0..7 = subblock sp, 8..15 = sp+1), but the +// scale is applied per 128-bit half. +// With BIAS != 0 the q0 codes are biased (up to 241) and a maddubs i16 pair +// overflows, so the weight is debiased (w = q0 - BIAS, small). Two MACs: +// ACTBIAS: activation pre-biased (+128) by repack, one maddubs(q1b, w); the +128 is +// corrected out in the epilogue (128*sum_s scales[s]*w_bsum[s], precomputed in repack_src0). +// Fits only when the debiased weight magnitude Wmax <= 64 (2*255*Wmax < 32767). +// else (iq4_xs, Wmax = 127): the signed-int8 sign trick, ax = |w|, sy = q1*sign(w), +// one maddubs (2*127*127 < 32767) so it always fits. +template +static void tiled_run_microtile_avx2(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride) { + constexpr int NB = TILED_TILE_K / SUBBLK; + constexpr int GROUP = 8; // Process 8 src1 columns concurrently (1 YMM vector) + + static_assert(SUBBLK == 16 || SUBBLK == 32, "unsupported SUBBLK"); + + // num_k/slab: the narrow path holds num_k slabs at row stride num_k*256 (num_k=1, slab=0 = standard) + // NK: 1 = standard single-slab (offsets 0, strides compile-time); 0 = runtime + // (num_k/slab from the args). The narrow path is memory-bound, so one runtime + // version serves all of it. + const int nkr = (NK > 0) ? NK : num_k; + const int seff = (NK > 0) ? 0 : slab; + const int qk_stride = nkr * TILED_TILE_K; + const int qk_off = seff * TILED_TILE_K; + const int nb_stride = NB * nkr; + const int nb_off = seff * NB; + const int bs_stride = (nkr == 1) ? TILED_TILE_ROWS : TILED_MICRO; + const int bs_off = seff * TILED_MICRO; + + for (int i = 0; i < TILED_MICRO; i++) { + const int ar = i0 + i; + const int d_off = seff * TILED_MICRO + ar; + const float d0 = src0.d[d_off]; + const float dmin0 = src0.dmin[d_off]; + const uint8_t * q0 = &src0.q[ar * qk_stride + qk_off]; + const int32_t * scales_row = &src0.scales[ar * nb_stride + nb_off]; + const int32_t * mins_row = &src0.mins[ar * nb_stride + nb_off]; + + for (int g = 0; g < TILED_MICRO; g += GROUP) { + // Pre-calculate weight pointers for this column group + const int8_t * q1_ptr[GROUP]; + for (int t = 0; t < GROUP; t++) { + q1_ptr[t] = &src1.q[(j0 + g + t) * qk_stride + qk_off]; + } + + __m256i s2 = _mm256_setzero_si256(); + + __m256i acc[GROUP]; + for (int t = 0; t < GROUP; t++) { + acc[t] = _mm256_setzero_si256(); + } + + if constexpr (SUBBLK == 32) { + for (int s = 0; s < NB; s++) { + // OPTIMIZATION 1: Use aligned loads (_mm256_load_si256) + const __m256i q0_32 = _mm256_load_si256((const __m256i *) &q0[s * SUBBLK]); + const __m256i scales16 = _mm256_set1_epi16(scales_row[s]); + + if constexpr (BIAS != 0) { + const __m256i w = _mm256_sub_epi8(q0_32, _mm256_set1_epi8((int8_t) BIAS)); // debiased weight + if constexpr (ACTBIAS) { + // activation pre-biased (+128) by repack; q1b unsigned, w signed + #pragma GCC unroll 8 + for (int t = 0; t < GROUP; t++) { + const __m256i q1b_32 = _mm256_load_si256((const __m256i *) &q1_ptr[t][s * SUBBLK]); + acc[t] = _mm256_add_epi32(acc[t], + _mm256_madd_epi16(scales16, _mm256_maddubs_epi16(q1b_32, w))); + } + } else { + // iq4_xs (Wmax > 64 overflows act-bias): sign trick, both operands <= 127 + const __m256i ax = _mm256_sign_epi8(w, w); + #pragma GCC unroll 8 + for (int t = 0; t < GROUP; t++) { + const __m256i q1_32 = _mm256_load_si256((const __m256i *) &q1_ptr[t][s * SUBBLK]); + acc[t] = _mm256_add_epi32(acc[t], + _mm256_madd_epi16(scales16, _mm256_maddubs_epi16(ax, _mm256_sign_epi8(q1_32, w)))); + } + } + } else { + #pragma GCC unroll 8 + for (int t = 0; t < GROUP; t++) { + const __m256i q1_32 = _mm256_load_si256((const __m256i *) &q1_ptr[t][s * SUBBLK]); + acc[t] = _mm256_add_epi32(acc[t], + _mm256_madd_epi16(scales16, _mm256_maddubs_epi16(q0_32, q1_32))); + } + } + + if constexpr (HAS_MIN) { + const __m256i bsums_v = _mm256_add_epi32( + _mm256_load_si256((const __m256i *) &src1.bsums[(bs_off + s * 2) * bs_stride + j0 + g]), + _mm256_load_si256((const __m256i *) &src1.bsums[(bs_off + s * 2 + 1) * bs_stride + j0 + g])); + s2 = _mm256_add_epi32(s2, _mm256_mullo_epi32(bsums_v, _mm256_set1_epi32(mins_row[s]))); + } + } + } else { // SUBBLK == 16 + for (int sp = 0; sp < NB; sp += 2) { + const __m256i q0_32 = _mm256_load_si256((const __m256i *) &q0[sp * SUBBLK]); + const __m256i scalesv = _mm256_set_m128i(_mm_set1_epi16(scales_row[sp + 1]), _mm_set1_epi16(scales_row[sp])); + if constexpr (BIAS != 0) { + const __m256i w = _mm256_sub_epi8(q0_32, _mm256_set1_epi8((int8_t) BIAS)); // debiased weight + if constexpr (ACTBIAS) { + // activation pre-biased (+128) by repack; q1b unsigned, w signed + #pragma GCC unroll 8 + for (int t = 0; t < GROUP; t++) { + const __m256i q1b_32 = _mm256_load_si256((const __m256i *) &q1_ptr[t][sp * SUBBLK]); + acc[t] = _mm256_add_epi32(acc[t], _mm256_madd_epi16( + scalesv, _mm256_maddubs_epi16(q1b_32, w))); + } + } else { + // iq4_xs (Wmax > 64 overflows act-bias): sign trick, both operands <= 127 + const __m256i ax = _mm256_sign_epi8(w, w); + #pragma GCC unroll 8 + for (int t = 0; t < GROUP; t++) { + const __m256i q1_32 = _mm256_load_si256((const __m256i *) &q1_ptr[t][sp * SUBBLK]); + acc[t] = _mm256_add_epi32(acc[t], _mm256_madd_epi16( + scalesv, _mm256_maddubs_epi16(ax, _mm256_sign_epi8(q1_32, w)))); + } + } + } else { + #pragma GCC unroll 8 + for (int t = 0; t < GROUP; t++) { + const __m256i q1_32 = _mm256_load_si256((const __m256i *) &q1_ptr[t][sp * SUBBLK]); + acc[t] = _mm256_add_epi32(acc[t], _mm256_madd_epi16( + scalesv, _mm256_maddubs_epi16(q0_32, q1_32))); + } + } + + if constexpr (HAS_MIN) { + const __m256i bsums0_v = _mm256_load_si256((const __m256i *) &src1.bsums[(bs_off + sp) * bs_stride + j0 + g]); + const __m256i bsums1_v = _mm256_load_si256((const __m256i *) &src1.bsums[(bs_off + sp + 1) * bs_stride + j0 + g]); + s2 = _mm256_add_epi32(s2, _mm256_add_epi32( + _mm256_mullo_epi32(bsums0_v, _mm256_set1_epi32(mins_row[sp])), + _mm256_mullo_epi32(bsums1_v, _mm256_set1_epi32(mins_row[sp + 1])))); + } + } + } + + // Reduce the eight accumulators to column sums in acc[0]..acc[7] order, with additions modulo 2^32. + const __m256i ab = _mm256_hadd_epi32(acc[0], acc[1]); + const __m256i cd = _mm256_hadd_epi32(acc[2], acc[3]); + const __m256i ef = _mm256_hadd_epi32(acc[4], acc[5]); + const __m256i gh = _mm256_hadd_epi32(acc[6], acc[7]); + + // Each 128-bit half now holds four column sums over input lanes 0..3 or 4..7. + const __m256i abcd = _mm256_hadd_epi32(ab, cd); + const __m256i efgh = _mm256_hadd_epi32(ef, gh); + const __m256i lo = _mm256_permute2x128_si256(abcd, efgh, 0x20); + const __m256i hi = _mm256_permute2x128_si256(abcd, efgh, 0x31); + __m256i s1_vec = _mm256_add_epi32(lo, hi); + + // OPTIMIZATION 2: Vectorized Epilogue with FMA3 gating + if constexpr (BIAS != 0 && ACTBIAS) { + // act-bias correction: raw = desired + 128*sum_s scales[s]*w_bsum[s]; corr precomputed in repack_src0 + s1_vec = _mm256_sub_epi32(s1_vec, _mm256_set1_epi32(mins_row[0])); + } + + const __m256 src1_d_vec = _mm256_load_ps(&src1.d[seff * TILED_MICRO + j0 + g]); + __m256 res_vec = _mm256_mul_ps(_mm256_set1_ps(d0), _mm256_cvtepi32_ps(s1_vec)); + + if constexpr (HAS_MIN) { +#if defined(__FMA__) + // FMA3 implementation: res_vec = res_vec - (dmin0 * s2) + res_vec = _mm256_fnmadd_ps(_mm256_set1_ps(dmin0), _mm256_cvtepi32_ps(s2), res_vec); +#else + // Non-FMA fallback + res_vec = _mm256_sub_ps(res_vec, _mm256_mul_ps(_mm256_set1_ps(dmin0), _mm256_cvtepi32_ps(s2))); +#endif + } + + float * buf_ptr = &buf[ar * buf_stride + j0 + g]; + __m256 current_buf = _mm256_loadu_ps(buf_ptr); + +#if defined(__FMA__) + // FMA3 accumulation: current_buf + (res_vec * src1_d_vec) + __m256 updated_buf = _mm256_fmadd_ps(res_vec, src1_d_vec, current_buf); +#else + __m256 updated_buf = _mm256_add_ps(current_buf, _mm256_mul_ps(res_vec, src1_d_vec)); +#endif + + _mm256_storeu_ps(buf_ptr, updated_buf); + } + } +} + +#endif // __AVX2__ + +#if defined(__AVX__) && !defined(__AVX2__) +template +static void tiled_run_microtile_avx(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride) { + constexpr int NB = TILED_TILE_K / SUBBLK; + constexpr int NS = SUBBLK / 16; + constexpr int GROUP = 4; // src1 columns per group: one acc32 per column + + // num_k/slab: the narrow path holds num_k slabs at row stride num_k*256 (num_k=1, slab=0 = standard) + // NK: 1 = standard single-slab (offsets 0, strides compile-time); 0 = runtime + // (num_k/slab from the args). The narrow path is memory-bound, so one runtime + // version serves all of it. + const int nkr = (NK > 0) ? NK : num_k; + const int seff = (NK > 0) ? 0 : slab; + const int qk_stride = nkr * TILED_TILE_K; + const int qk_off = seff * TILED_TILE_K; + const int nb_stride = NB * nkr; + const int nb_off = seff * NB; + const int bs_stride = (nkr == 1) ? TILED_TILE_ROWS : TILED_MICRO; + const int bs_off = seff * TILED_MICRO; + + for (int i = 0; i < TILED_MICRO; i++) { + const int ar = i0 + i; + const int d_off = seff * TILED_MICRO + ar; + const float d0 = src0.d[d_off]; + const float dmin0 = src0.dmin[d_off]; + const uint8_t * q0 = &src0.q[ar * qk_stride + qk_off]; + const int32_t * scales_row = &src0.scales[ar * nb_stride + nb_off]; + const int32_t * mins_row = &src0.mins[ar * nb_stride + nb_off]; + + for (int g = 0; g < TILED_MICRO; g += GROUP) { + const int8_t * q1g[GROUP]; + __m128i acc[GROUP]; + __m128i s2 = _mm_setzero_si128(); // sum_s mins_s * bsums_s + for (int t = 0; t < GROUP; t++) { + q1g[t] = &src1.q[(j0 + g + t) * qk_stride + qk_off]; + acc[t] = _mm_setzero_si128(); + } + + for (int s = 0; s < NB; s++) { + const __m128i scales16 = _mm_set1_epi16(scales_row[s]); + if constexpr (HAS_MIN) { + __m128i bsums_v = _mm_setzero_si128(); // per-col bsums as a 4-lane i32 vector + for (int u = 0; u < NS; u++) { + bsums_v = _mm_add_epi32(bsums_v, _mm_loadu_si128( + (const __m128i *) &src1.bsums[(bs_off + s * NS + u) * bs_stride + j0 + g])); + } + s2 = _mm_add_epi32(s2, _mm_mullo_epi32(bsums_v, _mm_set1_epi32(mins_row[s]))); + } + for (int u = 0; u < NS; u++) { + const __m128i a16 = _mm_loadu_si128((const __m128i *) &q0[s * SUBBLK + u * 16]); + if constexpr (BIAS != 0) { + // debiased weight w = a16 - BIAS; signed dot via the sign trick (see header) + const __m128i a16s = _mm_sub_epi8(a16, _mm_set1_epi8((int8_t) BIAS)); + const __m128i ax = _mm_sign_epi8(a16s, a16s); + for (int t = 0; t < GROUP; t++) { + const __m128i q1_16 = _mm_loadu_si128((const __m128i *) &q1g[t][s * SUBBLK + u * 16]); + acc[t] = _mm_add_epi32(acc[t], + _mm_madd_epi16(scales16, _mm_maddubs_epi16(ax, _mm_sign_epi8(q1_16, a16s)))); + } + } else { + for (int t = 0; t < GROUP; t++) { + acc[t] = _mm_add_epi32(acc[t], _mm_madd_epi16(scales16, + _mm_maddubs_epi16(a16, + _mm_loadu_si128((const __m128i *) &q1g[t][s * SUBBLK + u * 16])))); + } + } + } + } + + int32_t s2_s[GROUP] = { 0, 0, 0, 0 }; + if constexpr (HAS_MIN) { + _mm_storeu_si128((__m128i *) s2_s, s2); + } + + for (int t = 0; t < GROUP; t++) { + // 4 i32 lanes -> scalar + __m128i v = _mm_shuffle_epi32(acc[t], _MM_SHUFFLE(2, 3, 0, 1)); + acc[t] = _mm_add_epi32(acc[t], v); + v = _mm_shuffle_epi32(acc[t], _MM_SHUFFLE(1, 0, 3, 2)); + acc[t] = _mm_add_epi32(acc[t], v); + const int32_t s1 = _mm_cvtsi128_si32(acc[t]); + + float res = d0 * (float) s1; + if constexpr (HAS_MIN) { + res -= dmin0 * (float) s2_s[t]; + } + buf[ar * buf_stride + j0 + g + t] += src1.d[seff * TILED_MICRO + j0 + g + t] * res; + } + } + } +} + +#endif // __AVX__ && !__AVX2__ + +// main microtile entry point. num_k = K-blocks per row (the narrow path reads num_k slabs at row +// stride num_k*256 so each weight row is one long stream); slab = this call's slab. num_k=1, +// slab=0 is the standard single-slab path. +// num_k==1 dispatches to the NK=1 instantiation, whose strides/offsets are compile-time +// constants (shifts/scaled-leas). num_k>1 (the narrow path) uses the NK=0 runtime-strided +// version; it is memory-bound, so one version serves all of it. +template +void tiled_run_microtile(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride) { + const bool standard = (num_k == 1); +#if defined(__AVX512VNNI__) && defined(__AVX512VL__) && defined(__AVX512DQ__) + if (standard) { + tiled_run_microtile_vnni(src0, src1, i0, j0, num_k, slab, buf, buf_stride); + } else { + tiled_run_microtile_vnni(src0, src1, i0, j0, num_k, slab, buf, buf_stride); + } +#elif defined(__AVX2__) + if (standard) { + tiled_run_microtile_avx2(src0, src1, i0, j0, num_k, slab, buf, buf_stride); + } else { + tiled_run_microtile_avx2(src0, src1, i0, j0, num_k, slab, buf, buf_stride); + } +#elif defined(__AVX__) + if (standard) { + tiled_run_microtile_avx(src0, src1, i0, j0, num_k, slab, buf, buf_stride); + } else { + tiled_run_microtile_avx(src0, src1, i0, j0, num_k, slab, buf, buf_stride); + } +#else + if (standard) { + tiled_run_microtile_scalar(src0, src1, i0, j0, num_k, slab, buf, buf_stride); + } else { + tiled_run_microtile_scalar(src0, src1, i0, j0, num_k, slab, buf, buf_stride); + } +#endif +} + +// explicit instantiations for the in-use formats (q4_K and q5_K share the constants) +// ACTBIAS selects the act-bias MAC (Wmax <= 64); iq4_xs (Wmax 127) keeps the sign trick +// q4_K / q5_K: BIAS = 0 + template void tiled_run_microtile<32, true, 0, false>(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride); +// iq4_xs: sign trick (Wmax 127 overflows act-bias) +template void tiled_run_microtile<32, false, 128, false>(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride); +// iq2_xxs, iq3_xxs, iq3_s, iq1_s: act-bias (Wmax <= 62) +template void tiled_run_microtile<32, false, 128, true>(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride); +// iq2_xs, iq2_s, iq1_m: act-bias +template void tiled_run_microtile<16, false, 128, true>(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride); +// q6_K: act-bias (Wmax 32) +template void tiled_run_microtile<16, false, 32, true>(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride); +// q3_K: act-bias (Wmax 4) +template void tiled_run_microtile<16, false, 4, true>(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride); +// q2_K: BIAS = 0 +template void tiled_run_microtile<16, true, 0, false>(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride); + + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +// block_q8_K in 4-byte words, for the int32 gather indices +static_assert(sizeof(block_q8_K) == 292 && offsetof(block_q8_K, qs) == 4, + "block_q8_K layout changed, fix the src1 repack"); + +// VNNI: interleave the natural [row][k] codes in-place into the group-local +// [kg%16][kg/16][row][4] layout. base points at row 0 (row r at base + r*row_stride); +// n_tiles 16x16 int32 tiles are transposed to the right (tile t occupies bytes +// [t*64, t*64+64) per row). Each 16-row group is self-contained in row_stride*16 bytes. +#if defined(__AVX512VNNI__) && defined(__AVX512VL__) && defined(__AVX512DQ__) +void tiled_repack_src1(tiled_tile_src1 * src1, int row0, int num_k, bool bias) { + GGML_UNUSED(bias); + const int n_tiles = num_k * (TILED_TILE_K / (TILED_MICRO * 4)); + const int row_stride = num_k * TILED_TILE_K; + uint8_t * base = (uint8_t *) src1->q + row0 * row_stride; + for (int t = 0; t < n_tiles; t++) { + uint8_t * cbase = base + t * (TILED_MICRO * 4); + __m512i v[16]; + for (int r = 0; r < 16; r++) { + v[r] = _mm512_load_si512((const __m512i *) (cbase + r * row_stride)); + } + + // 16x16 int32 transpose, 4 butterfly phases + // Phase 1: 1-element interleave, pairs (0,1), (2,3), ..., (14,15) + { + const __m512i idx_a = _mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + const __m512i idx_b = _mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + for (int i = 0; i < 16; i += 2) { + __m512i a = v[i], b = v[i+1]; + v[i] = _mm512_permutex2var_epi32(a, idx_a, b); + v[i+1] = _mm512_permutex2var_epi32(a, idx_b, b); + } + } + // Phase 2: 2-element interleave, pairs (0,2), (1,3), (4,6), (5,7), ... + { + const __m512i idx_a = _mm512_setr_epi32(0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29); + const __m512i idx_b = _mm512_setr_epi32(2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31); + for (int i = 0; i < 16; i += 4) { + __m512i a = v[i], b = v[i+2]; + v[i] = _mm512_permutex2var_epi32(a, idx_a, b); + v[i+2] = _mm512_permutex2var_epi32(a, idx_b, b); + a = v[i+1], b = v[i+3]; + v[i+1] = _mm512_permutex2var_epi32(a, idx_a, b); + v[i+3] = _mm512_permutex2var_epi32(a, idx_b, b); + } + } + // Phase 3: 4-element interleave, pairs (0,4), (1,5), ..., (7,11), (8,12), ... + { + const __m512i idx_a = _mm512_setr_epi32(0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23); + const __m512i idx_b = _mm512_setr_epi32(8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31); + for (int i = 0; i < 16; i += 8) { + for (int j = 0; j < 4; j++) { + __m512i a = v[i+j], b = v[i+4+j]; + v[i+j] = _mm512_permutex2var_epi32(a, idx_a, b); + v[i+4+j] = _mm512_permutex2var_epi32(a, idx_b, b); + } + } + } + // Phase 4: 8-element interleave, pairs (0,8), (1,9), ..., (7,15) + { + const __m512i idx_a = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23); + const __m512i idx_b = _mm512_setr_epi32(8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31); + for (int i = 0; i < 8; i++) { + __m512i a = v[i], b = v[i+8]; + v[i] = _mm512_permutex2var_epi32(a, idx_a, b); + v[i+8] = _mm512_permutex2var_epi32(a, idx_b, b); + } + } + + // store: v[g] holds 16 int32s for k-group col_order[g], rows 0..15 + static const int col_order[16] = {0, 8, 1, 9, 4, 12, 5, 13, 2, 10, 3, 11, 6, 14, 7, 15}; + for (int g = 0; g < 16; g++) { + _mm512_store_si512((void *) (cbase + col_order[g] * row_stride), v[g]); + } + } +} +#elif defined(__AVX2__) +// act-bias: pre-bias the activation in place (+128) so the MAC's maddubs(q1b, w) sees unsigned bytes; +// no transpose on AVX2 (natural layout). 16 rows x n_tiles*64 bytes, row r at base + r*row_stride +void tiled_repack_src1(tiled_tile_src1 * src1, int row0, int num_k, bool bias) { + if (!bias) { + return; + } + const int row_stride = num_k * TILED_TILE_K; + uint8_t * base = (uint8_t *) src1->q + row0 * row_stride; + const int nbytes = row_stride; // full row width + const __m256i b128 = _mm256_set1_epi8((int8_t) 0x80); + for (int r = 0; r < TILED_MICRO; r++) { + uint8_t * row = base + r * row_stride; + for (int off = 0; off < nbytes; off += 32) { + const __m256i v = _mm256_loadu_si256((const __m256i *) (row + off)); + _mm256_storeu_si256((__m256i *) (row + off), _mm256_xor_si256(v, b128)); + } + } +} +#else +void tiled_repack_src1(tiled_tile_src1 * src1, int row0, int num_k, bool bias) { + GGML_UNUSED(src1); GGML_UNUSED(row0); GGML_UNUSED(num_k); GGML_UNUSED(bias); +} +#endif + +#if defined(__AVX2__) +// sum 32 unsigned bytes to int32 (via maddubs -> int16 pairs -> hsum) +static int32_t tiled_byte_sum_32(const uint8_t * p) { + const __m256i dot = _mm256_maddubs_epi16(_mm256_loadu_si256((const __m256i *) p), _mm256_set1_epi8(1)); + const __m256i sum = _mm256_madd_epi16(_mm256_set1_epi16(1), dot); // 8 int32 + // reduce 8 lanes: lo+hi -> 4 lanes, hadd -> 2, hadd -> 1 (a single hadd pair only folds 4 lanes) + const __m128i t = _mm_add_epi32(_mm256_castsi256_si128(sum), _mm256_extracti128_si256(sum, 1)); + const __m128i h = _mm_hadd_epi32(t, t); + return _mm_cvtsi128_si32(_mm_hadd_epi32(h, h)); +} +// sum 16 unsigned bytes to int32 +static int32_t tiled_byte_sum_16(const uint8_t * p) { + const __m128i dot = _mm_maddubs_epi16(_mm_loadu_si128((const __m128i *) p), _mm_set1_epi8(1)); + const __m128i sum = _mm_madd_epi16(_mm_set1_epi16(1), dot); // 4 int32 + const __m128i h = _mm_hadd_epi32(sum, sum); + return _mm_cvtsi128_si32(_mm_hadd_epi32(h, h)); +} +#endif + +// act-bias: precompute the per-weight-row correction corr = 128*sum_s scales[s]*w_bsum[s] where +// w_bsum[s] = sum of the debiased weight bytes in subblock s. Stored in mins[r][0] (unused for +// HAS_MIN = false, which is exactly the act-bias case). Reused across all act bands and weight groups. +// No-op off AVX2 (the act-bias MAC is AVX2-only) and when corr is false (iq4_xs / BIAS = 0). +template +void tiled_repack_src0(tiled_tile_src0 * tile, int n_rows, int num_k, int BIAS, bool corr) { +#if defined(__AVX2__) + if (!corr) { + return; + } + constexpr int NB = TILED_TILE_K / SUBBLK; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const uint8_t * q = &tile->q[r * (num_k * TILED_TILE_K) + slab * TILED_TILE_K]; + const int32_t * scales = &tile->scales[r * nb_stride + slab * NB]; + int32_t corr_val = 0; + for (int s = 0; s < NB; s++) { + const int32_t qsum = (SUBBLK == 32) ? tiled_byte_sum_32(&q[s * SUBBLK]) : tiled_byte_sum_16(&q[s * SUBBLK]); + corr_val += 128 * scales[s] * (qsum - SUBBLK * BIAS); + } + tile->mins[r * nb_stride + slab * NB] = corr_val; + } + } +#else + GGML_UNUSED(tile); GGML_UNUSED(n_rows); GGML_UNUSED(num_k); GGML_UNUSED(BIAS); GGML_UNUSED(corr); +#endif +} + +template void tiled_repack_src0<16>(tiled_tile_src0 * tile, int n_rows, int num_k, int BIAS, bool corr); +template void tiled_repack_src0<32>(tiled_tile_src0 * tile, int n_rows, int num_k, int BIAS, bool corr); + + diff --git a/ggml/src/ggml-cpu/tiled/tiled-kernel.h b/ggml/src/ggml-cpu/tiled/tiled-kernel.h new file mode 100644 index 0000000..1a97f40 --- /dev/null +++ b/ggml/src/ggml-cpu/tiled/tiled-kernel.h @@ -0,0 +1,180 @@ +#pragma once + +// Tiled matmul kernel API: tile structs, kernel definitions + +// Currently only optimized for x86, new architectures should implement: +// tiled_run_microtile: 16x16 microkernel +// tiled_repack_src0: Optional repack/recalculation of src0, per macrotile +// tiled_repack_src1: Optional repack/recalculation of src1, per microtile-band +// bit unpacking routines: tiled_unpk_nib4, tiled_unpk_2bit, tiled_unpk_or +// LUT value expansion routines: tiled_lut8, tiled_unpk_sign32, tiled_unpk_tern8 + +#define GGML_COMMON_DECL_CPP +#include "ggml-common.h" + +#include +#include + +#if defined(__AVX2__) +#include +#endif + +#define TILED_TILE_K 256 // one QK_K block +#define TILED_TILE_ROWS 256 // max window rows, ragged at edges +#define TILED_MICRO 16 // microtile edge (also the bsums code-sum granularity) +#define TILED_WS_SLOT (512 * 1024) // per-thread workspace slot, a clean 512KB (multiple of 64B) + +// src0 tile: weight side, shared by all formats. +// scales/mins are sized for the max subblock count (SUBBLK=16); +// SUBBLK=32 formats index at stride 8 and leave the slack unused. +struct tiled_tile_src0 { + static constexpr int NB_MAX = TILED_TILE_K / 16; // max subblocks per 256-elem block + + alignas(64) uint8_t q[TILED_TILE_ROWS * TILED_TILE_K]; // unsigned quants, widened to uint8 + float d[TILED_TILE_ROWS]; // One d from each input block, widened to f32 + float dmin[TILED_TILE_ROWS]; // dmin from each input block (if applicable), widened to F32 + int32_t scales[TILED_TILE_ROWS * NB_MAX]; // per-subblock scale, stored as int32_t + int32_t mins[TILED_TILE_ROWS * NB_MAX]; // per-subblock min, used when HAS_MIN +}; + +// src1 tile: built from q8_K (wdata) +struct tiled_tile_src1 { + // q8 codes, one byte per element. Note for VNNI these are reshaped + transposed to be suitable for dpbusd. + alignas(64) int8_t q[TILED_TILE_ROWS * TILED_TILE_K]; + // per-16 code sums from q8_k (int16), widened to int32 so the kernels load them directly, no per-use cvt + alignas(64) int32_t bsums[(TILED_TILE_K / 16) * TILED_TILE_ROWS]; + // f32 (not f16): q8_k stores fp16, the unpack converts once + float d[TILED_TILE_ROWS]; +}; + +// per-thread workspace: all tiled state lives here, allocated in wdata (one slot per thread) +struct tiled_ws { + tiled_tile_src0 src0; + tiled_tile_src1 src1; + alignas(64) float acc[TILED_TILE_ROWS * TILED_TILE_ROWS]; +}; + +static_assert(sizeof(tiled_ws) <= TILED_WS_SLOT, "tiled workspace exceeds the 512KB per-thread slot"); +// the slot base is 64B-aligned and TILED_WS_SLOT is a multiple of 64B, so each per-thread slot +// is 64B-aligned; these pin the tile fields and the acc buffer at aligned offsets within a slot +static_assert(offsetof(tiled_ws, src0) % 64 == 0, "src0 not 64B-aligned in the workspace"); +static_assert(offsetof(tiled_ws, src1) % 64 == 0, "src1 not 64B-aligned in the workspace"); +static_assert(offsetof(tiled_ws, acc) % 64 == 0, "acc not 64B-aligned in the workspace"); + +// unpack primitives for reading quants, defined as inline here to keep arch-specific code in kernel.h/.cpp +// If this section gets too hairy later, we can break up into separate includes. +#if defined(__AVX2__) +// packed 4-bit codes -> low nibbles (lo) + high nibbles (hi) +inline void tiled_unpk_nib4(const uint8_t * src, uint8_t * lo, uint8_t * hi) { + const __m256i v = _mm256_loadu_si256((const __m256i *) src); + // mask before the lane shift so bits do not cross byte boundaries + _mm256_storeu_si256((__m256i *) lo, _mm256_and_si256(v, _mm256_set1_epi8(0x0F))); + _mm256_storeu_si256((__m256i *) hi, _mm256_srli_epi32(_mm256_and_si256(v, _mm256_set1_epi8((int8_t) 0xF0)), 4)); +} +// 2-bit values at bit offset S +template inline void tiled_unpk_2bit(const uint8_t * src, uint8_t * dst) { + _mm256_storeu_si256((__m256i *) dst, _mm256_and_si256( + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i *) src), S), _mm256_set1_epi8(0x03))); +} +// OR the M-bit value at bit offset S of src into bit offset D of dst +template +inline void tiled_unpk_or(uint8_t * dst, const uint8_t * src) { + const __m256i v = _mm256_slli_epi32(_mm256_and_si256( + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i *) src), S), _mm256_set1_epi8((uint8_t) M)), D); + _mm256_storeu_si256((__m256i *) dst, _mm256_or_si256(_mm256_loadu_si256((const __m256i *) dst), v)); +} + + +// Unpacking kernels for IQ quants + +// LUT value expansion for the LUT-based formats (iq4_xs, iq grids): the bit unpackers +// above give the indices, these expand 8/16 of them to widened codes in one pass +// 16-entry byte LUT: dst[j] = lut[src[j]] (16 bytes) +inline void tiled_lut8(const uint8_t * lut, const uint8_t * src, uint8_t * dst) { + _mm_storeu_si128((__m128i *) dst, _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) lut), + _mm_loadu_si128((const __m128i *) src))); +} + + +// 32 grid magnitudes (4 x 64-bit groups g0..g3, 8 values each) + 4 sign bytes +// (byte l signs values 8*l .. 8*l+7) -> codes stored as (value + 128) +inline void tiled_unpk_sign32(uint64_t g0, uint64_t g1, uint64_t g2, uint64_t g3, + const uint8_t signs[4], uint8_t * dst32) { + const __m256i v = _mm256_set_epi64x((int64_t) g3, (int64_t) g2, (int64_t) g1, (int64_t) g0); + const __m256i sv = _mm256_shuffle_epi8(_mm256_set1_epi32((int32_t) (signs[0] | signs[1] << 8 | signs[2] << 16 | signs[3] << 24)), + _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3)); + const __m256i sel = _mm256_set1_epi64x((int64_t) 0x8040201008040201ULL); + // 0xFF in every lane whose sign bit is set; GFNI does the and+compare in one instruction +#ifdef __GFNI__ + const __m256i mask = _mm256_gf2p8affine_epi64_epi8(sel, sv, 0); +#else + const __m256i mask = _mm256_cmpeq_epi8(_mm256_and_si256(sv, sel), sel); +#endif + // v ^ mask - mask negates the signed lanes (v elsewhere); + 128 gives the biased code + const __m256i sgn = _mm256_sub_epi8(_mm256_xor_si256(v, mask), mask); + _mm256_storeu_si256((__m256i *) dst32, _mm256_add_epi8(sgn, _mm256_set1_epi8((int8_t) 128))); +} +// 8 ternary grid bytes (0 = 0, 1 = +1, 0xFF = -1): dst[j] = 128 + delta + 8 * (int8_t) src[j] +inline void tiled_unpk_tern8(const uint8_t * src, int8_t delta, uint8_t * dst) { + const __m128i v = _mm_cvtepi8_epi16(_mm_loadl_epi64((const __m128i *) src)); + const __m128i p = _mm_add_epi16(_mm_slli_epi16(v, 3), _mm_set1_epi16(128 + (int) delta)); + _mm_storel_epi64((__m128i *) dst, _mm_packus_epi16(p, _mm_setzero_si128())); +} +#else + +// Scalar definitions for unpackers. + + +inline void tiled_unpk_nib4(const uint8_t * src, uint8_t * lo, uint8_t * hi) { + for (int l = 0; l < 32; l++) { lo[l] = (uint8_t) (src[l] & 0xF); hi[l] = (uint8_t) (src[l] >> 4); } +} +template +inline void tiled_unpk_2bit(const uint8_t * src, uint8_t * dst) { + for (int l = 0; l < 32; l++) { dst[l] = (uint8_t) ((src[l] >> S) & 3); } +} +template +inline void tiled_unpk_or(uint8_t * dst, const uint8_t * src) { + for (int l = 0; l < 32; l++) { dst[l] = (uint8_t) (dst[l] | (((src[l] >> S) & M) << D)); } +} +inline void tiled_lut8(const uint8_t * lut, const uint8_t * src, uint8_t * dst) { + for (int j = 0; j < 16; j++) { dst[j] = lut[src[j]]; } +} +inline void tiled_unpk_sign32(uint64_t g0, uint64_t g1, uint64_t g2, uint64_t g3, + const uint8_t signs[4], uint8_t * dst32) { + const uint64_t g[4] = { g0, g1, g2, g3 }; + for (int l = 0; l < 4; l++) { + const uint8_t * v = (const uint8_t *) &g[l]; + const uint8_t s = signs[l]; + for (int j = 0; j < 8; j++) { + dst32[8 * l + j] = (s & (1 << j)) ? (uint8_t) (128 - v[j]) : (uint8_t) (128 + v[j]); + } + } +} +// 8 ternary grid bytes (0 = 0, 1 = +1, 0xFF = -1): dst[j] = 128 + delta + 8 * (int8_t) src[j] +inline void tiled_unpk_tern8(const uint8_t * src, int8_t delta, uint8_t * dst) { + for (int j = 0; j < 8; j++) { dst[j] = (uint8_t) (128 + (int) delta + 8 * (int8_t) src[j]); } +} +#endif + +// Accumulate one 16x16 microtile (src0 rows [i0, i0+16), src1 cols [j0, j0+16)) +// over one 256-K slab held in the tiles into a j-major float buffer +// (row width buf_stride): buf[i*buf_stride + j] += partial. +// SUBBLK/HAS_MIN/BIAS are the src0 format constants (see tiled_tile_src0). +// ACTBIAS (AVX2 only): the activation is pre-biased +128 by tiled_repack_src1, +// num_k = K-blocks per row: the tile holds num_k slabs at row stride num_k*256 +// (Default case is num_k=1, 256x256 tiles, we go to longer num_k to improve memory bandwidth when num_rows is small) +template +void tiled_run_microtile(const tiled_tile_src0 & src0, const tiled_tile_src1 & src1, + int i0, int j0, int num_k, int slab, float * buf, int buf_stride); + +// Optional repack, if profitable for the kernel. +// Repacks one 16-row band of src1 codes, called by driver as we reach each 16-row band in outer loop +void tiled_repack_src1(tiled_tile_src1 * src1, int row0, int num_k, bool bias); + +// Optional repack, if profitable for the kernel +// Repack the entire src0 panel in place, called by the driver immediately after dequant +template +void tiled_repack_src0(tiled_tile_src0 * tile, int n_rows, int num_k, int BIAS, bool corr); + + diff --git a/ggml/src/ggml-cpu/tiled/tiled.cpp b/ggml/src/ggml-cpu/tiled/tiled.cpp new file mode 100644 index 0000000..77e1a99 --- /dev/null +++ b/ggml/src/ggml-cpu/tiled/tiled.cpp @@ -0,0 +1,1245 @@ +#include "tiled.h" +#include "tiled-kernel.h" + +#include "ggml-cpu-impl.h" +#include "ggml-cpu.h" +#include "ggml.h" + +// kvalues table (impl section) for the iq4_xs unpack +#define GGML_COMMON_IMPL_CPP +#include "ggml-common.h" + +#include +#include +#include + +#include + +#define UNUSED GGML_UNUSED + +// unpack routines for various quant types src0 +static void tiled_unpack_src0(const block_q4_K * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 8; // 32-wide subblocks + // 12-byte packed scale/min decode, same extraction as the reference kernels + static const uint32_t kmask1 = 0x3f3f3f3f; + static const uint32_t kmask2 = 0x0f0f0f0f; + static const uint32_t kmask3 = 0x03030303; + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_q4_K & x = rows[r * row_stride + slab]; + + tile->d[d_off] = ggml_fp16_to_fp32(x.data.data.d); + tile->dmin[d_off] = ggml_fp16_to_fp32(x.data.data.dmin); + + uint32_t utmp[4]; + memcpy(utmp, x.scales, 12); + utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4); + const uint32_t uaux = utmp[1] & kmask1; + utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); + utmp[2] = uaux; + utmp[0] &= kmask1; + + const uint8_t * scales = (const uint8_t *) &utmp[0]; + const uint8_t * mins = (const uint8_t *) &utmp[2]; + for (int s = 0; s < NB; s++) { + tile->scales[s_off + s] = (int32_t) scales[s]; + tile->mins[s_off + s] = (int32_t) mins[s]; + } + + // extract the 4-bit codes (low 4 + high 4), same extraction as the reference kernels + uint8_t * q = &tile->q[q_off]; + tiled_unpk_nib4(x.qs + 0, q + 0, q + 32); + tiled_unpk_nib4(x.qs + 32, q + 64, q + 96); + tiled_unpk_nib4(x.qs + 64, q + 128, q + 160); + tiled_unpk_nib4(x.qs + 96, q + 192, q + 224); + } + } +} + +static void tiled_unpack_src0(const block_q5_K * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 8; // 32-wide subblocks + // 12-byte packed scale/min decode, same extraction as the reference kernels + static const uint32_t kmask1 = 0x3f3f3f3f; + static const uint32_t kmask2 = 0x0f0f0f0f; + static const uint32_t kmask3 = 0x03030303; + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_q5_K & x = rows[r * row_stride + slab]; + + tile->d[d_off] = ggml_fp16_to_fp32(x.data.data.d); + tile->dmin[d_off] = ggml_fp16_to_fp32(x.data.data.dmin); + + uint32_t utmp[4]; + memcpy(utmp, x.scales, 12); + utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4); + const uint32_t uaux = utmp[1] & kmask1; + utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); + utmp[2] = uaux; + utmp[0] &= kmask1; + + const uint8_t * scales = (const uint8_t *) &utmp[0]; + const uint8_t * mins = (const uint8_t *) &utmp[2]; + for (int s = 0; s < NB; s++) { + tile->scales[s_off + s] = (int32_t) scales[s]; + tile->mins[s_off + s] = (int32_t) mins[s]; + } + + // extract the 5-bit codes (4 low bits + 1 high bit), same as the generic kernels: + // 64-element chunk j uses qh bits 2j (low 32) and 2j+1 (high 32); OR adds the 5th bit + // (no overlap with the 4-bit codes, identical to the reference ADD) + uint8_t * q = &tile->q[q_off]; + tiled_unpk_nib4(x.qs + 0, q + 0, q + 32); + tiled_unpk_nib4(x.qs + 32, q + 64, q + 96); + tiled_unpk_nib4(x.qs + 64, q + 128, q + 160); + tiled_unpk_nib4(x.qs + 96, q + 192, q + 224); + tiled_unpk_or<0, 4, 1>(q + 0, x.qh); + tiled_unpk_or<1, 4, 1>(q + 32, x.qh); + tiled_unpk_or<2, 4, 1>(q + 64, x.qh); + tiled_unpk_or<3, 4, 1>(q + 96, x.qh); + tiled_unpk_or<4, 4, 1>(q + 128, x.qh); + tiled_unpk_or<5, 4, 1>(q + 160, x.qh); + tiled_unpk_or<6, 4, 1>(q + 192, x.qh); + tiled_unpk_or<7, 4, 1>(q + 224, x.qh); + } + } +} + +static void tiled_unpack_src0(const block_q6_K * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 16; // 16-wide subblocks + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_q6_K & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.d); + + // 6-bit code = 4 low bits (ql) | 2 high bits (qh); see ggml_vec_dot_q6_K_q8_K_generic + // per half the lanes are [ql lo(0:32)] [ql lo(32:64)] [ql hi(0:32)] [ql hi(32:64)] + uint8_t * q = &tile->q[q_off]; + for (int half = 0; half < 2; half++) { + uint8_t * out = q + 128 * half; + tiled_unpk_nib4(x.ql + 64 * half + 0, out + 0, out + 64); + tiled_unpk_nib4(x.ql + 64 * half + 32, out + 32, out + 96); + tiled_unpk_or<0, 4, 3>(out + 0, x.qh + 32 * half); + tiled_unpk_or<2, 4, 3>(out + 32, x.qh + 32 * half); + tiled_unpk_or<4, 4, 3>(out + 64, x.qh + 32 * half); + tiled_unpk_or<6, 4, 3>(out + 96, x.qh + 32 * half); + } + // scale is a plain int8 per 16-element subblock (16 per 256-K) + for (int s = 0; s < NB; s++) { tile->scales[s_off + s] = (int32_t) (int8_t) x.scales[s]; } + } + } +} + +static void tiled_unpack_src0(const block_q3_K * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 16; // 16-wide subblocks + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_q3_K & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.d); + + // 3-bit code = 2 low bits (qs) | (1 high bit from hmask << 2) + // element e (0..255): half=e>>7, el=e&127, group=el>>5, l=el&31 + // low2 = (qs[half*32 + l] >> 2*group) & 3 + // high = (hmask[l] >> (half*4 + group)) & 1 + // see ggml_vec_dot_q3_K_q8_K_generic + uint8_t * q = &tile->q[q_off]; + const uint8_t * s0 = x.qs; + const uint8_t * s1 = x.qs + 32; + uint8_t * o0 = q; + uint8_t * o1 = q + 128; + tiled_unpk_2bit<0>(s0, o0); tiled_unpk_or<0, 2, 1>(o0, x.hmask); + tiled_unpk_2bit<2>(s0, o0 + 32); tiled_unpk_or<1, 2, 1>(o0 + 32, x.hmask); + tiled_unpk_2bit<4>(s0, o0 + 64); tiled_unpk_or<2, 2, 1>(o0 + 64, x.hmask); + tiled_unpk_2bit<6>(s0, o0 + 96); tiled_unpk_or<3, 2, 1>(o0 + 96, x.hmask); + tiled_unpk_2bit<0>(s1, o1); tiled_unpk_or<4, 2, 1>(o1, x.hmask); + tiled_unpk_2bit<2>(s1, o1 + 32); tiled_unpk_or<5, 2, 1>(o1 + 32, x.hmask); + tiled_unpk_2bit<4>(s1, o1 + 64); tiled_unpk_or<6, 2, 1>(o1 + 64, x.hmask); + tiled_unpk_2bit<6>(s1, o1 + 96); tiled_unpk_or<7, 2, 1>(o1 + 96, x.hmask); + // 6-bit scale decode (same kmask trick as the reference), stored as (scales - 32) + static const uint32_t kmask1 = 0x03030303; + static const uint32_t kmask2 = 0x0f0f0f0f; + uint32_t auxs[4]; + memcpy(auxs, x.scales, 12); + const uint32_t tmp = auxs[2]; + auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); + auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); + auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); + auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); + const int8_t * scales = (const int8_t *) &auxs[0]; + for (int s = 0; s < NB; s++) { tile->scales[s_off + s] = (int32_t) scales[s] - 32; } + } + } +} + +static void tiled_unpack_src0(const block_q2_K * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 16; // 16-wide subblocks + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_q2_K & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.data.data.d); + tile->dmin[d_off] = ggml_fp16_to_fp32(x.data.data.dmin); + + // 2-bit code: element e -> half=e>>7, el=e&127 + // byte = half*32 + (el & 31), shift = 2*(el >> 5) + // see ggml_vec_dot_q2_K_q8_K_generic + uint8_t * q = &tile->q[q_off]; + for (int half = 0; half < 2; half++) { + const uint8_t * s = x.qs + 32 * half; + uint8_t * out = q + 128 * half; + tiled_unpk_2bit<0>(s, out + 0); + tiled_unpk_2bit<2>(s, out + 32); + tiled_unpk_2bit<4>(s, out + 64); + tiled_unpk_2bit<6>(s, out + 96); + } + // scale/min packed in one byte per 16-element subblock: low 4 bits = scale, high 4 = min + for (int s = 0; s < NB; s++) { + tile->scales[s_off + s] = (int32_t) (x.scales[s] & 0xF); + tile->mins[s_off + s] = (int32_t) (x.scales[s] >> 4); + } + } + } +} + +// iq4_xs: 4-bit codes through the kvalues_iq4nl LUT, 6-bit scales per 32, no min. +// LUT values are stored as kvalues + 128 so the +128 shift the kernel applies to +// the activations cancels against the bsums correction (BIAS = 128); the +128 is +// folded into the table up front so the expansion is a plain lookup +static void tiled_unpack_src0(const block_iq4_xs * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 8; // 32-wide subblocks + + static const uint8_t lut[16] = { + (uint8_t) (kvalues_iq4nl[0] + 128), (uint8_t) (kvalues_iq4nl[1] + 128), + (uint8_t) (kvalues_iq4nl[2] + 128), (uint8_t) (kvalues_iq4nl[3] + 128), + (uint8_t) (kvalues_iq4nl[4] + 128), (uint8_t) (kvalues_iq4nl[5] + 128), + (uint8_t) (kvalues_iq4nl[6] + 128), (uint8_t) (kvalues_iq4nl[7] + 128), + (uint8_t) (kvalues_iq4nl[8] + 128), (uint8_t) (kvalues_iq4nl[9] + 128), + (uint8_t) (kvalues_iq4nl[10] + 128), (uint8_t) (kvalues_iq4nl[11] + 128), + (uint8_t) (kvalues_iq4nl[12] + 128), (uint8_t) (kvalues_iq4nl[13] + 128), + (uint8_t) (kvalues_iq4nl[14] + 128), (uint8_t) (kvalues_iq4nl[15] + 128), + }; + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_iq4_xs & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.d); + + // 6-bit scale per 32, stored as (ls - 32); same extraction as dequantize_row_iq4_xs + for (int s = 0; s < NB; s++) { + const int ls = ((x.scales_l[s / 2] >> 4 * (s % 2)) & 0xf) | (((x.scales_h >> 2 * s) & 3) << 4); + tile->scales[s_off + s] = (int32_t) ls - 32; + } + + // 4-bit codes through the LUT: low nibbles of a byte pair come first + uint8_t * q = &tile->q[q_off]; + uint8_t lo[32], hi[32]; + for (int u = 0; u < 4; u++) { + tiled_unpk_nib4(x.qs + 32 * u, lo, hi); + tiled_lut8(lut, lo + 0, q + 64 * u + 0); + tiled_lut8(lut, hi + 0, q + 64 * u + 16); + tiled_lut8(lut, lo + 16, q + 64 * u + 32); + tiled_lut8(lut, hi + 16, q + 64 * u + 48); + } + } + } +} + +// iq2_xxs: 2-bit grids through the iq2xxs_grid LUT, 4-bit scale per 32, no min +// codes stored as (value + 128), matching BIAS = 128 +static void tiled_unpack_src0(const block_iq2_xxs * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 8; // 32-wide subblocks + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_iq2_xxs & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.d) * 0.125f; + + uint32_t aux32[2]; + const uint8_t * aux8 = (const uint8_t *) aux32; + uint64_t g[4]; + uint8_t signs4[4]; + for (int ib32 = 0; ib32 < NB; ib32++) { + memcpy(aux32, x.qs + 4 * ib32, 2 * sizeof(uint32_t)); + tile->scales[s_off + ib32] = (int32_t) (2 * (aux32[1] >> 28) + 1); + for (int l = 0; l < 4; l++) { + g[l] = iq2xxs_grid[aux8[l]]; + signs4[l] = ksigns_iq2xs[(aux32[1] >> (7 * l)) & 127]; + } + tiled_unpk_sign32(g[0], g[1], g[2], g[3], signs4, &tile->q[q_off + 32 * ib32]); + } + } + } +} + +// iq2_xs: 2-bit grids through the iq2xs_grid LUT, 4-bit scale per 16 (two per 32), no min +// codes stored as (value + 128), matching BIAS = 128 +static void tiled_unpack_src0(const block_iq2_xs * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 16; // 16-wide subblocks + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_iq2_xs & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.d) * 0.125f; + + uint64_t g[4]; + uint8_t signs4[4]; + for (int ib32 = 0; ib32 < QK_K / 32; ib32++) { + tile->scales[s_off + 2 * ib32 + 0] = (int32_t) (2 * (x.scales[ib32] & 0xf) + 1); + tile->scales[s_off + 2 * ib32 + 1] = (int32_t) (2 * (x.scales[ib32] >> 4) + 1); + const uint16_t * q = x.qs + 4 * ib32; + for (int l = 0; l < 4; l++) { + g[l] = iq2xs_grid[q[l] & 511]; + signs4[l] = ksigns_iq2xs[q[l] >> 9]; + } + tiled_unpk_sign32(g[0], g[1], g[2], g[3], signs4, &tile->q[q_off + 32 * ib32]); + } + } + } +} + +// iq2_s: 2-bit grids through the iq2s_grid LUT, 4-bit scale per 16 (two per 32), no min +// codes stored as (value + 128), matching BIAS = 128 +static void tiled_unpack_src0(const block_iq2_s * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 16; // 16-wide subblocks + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_iq2_s & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.d) * 0.125f; + + const uint8_t * qs = x.qs; + const uint8_t * signs = x.qs + QK_K / 8; // packed sign bytes share the qs array, same as the reference + uint64_t g[4]; + for (int ib32 = 0; ib32 < QK_K / 32; ib32++) { + tile->scales[s_off + 2 * ib32 + 0] = (int32_t) (2 * (x.scales[ib32] & 0xf) + 1); + tile->scales[s_off + 2 * ib32 + 1] = (int32_t) (2 * (x.scales[ib32] >> 4) + 1); + for (int l = 0; l < 4; l++) { + g[l] = iq2s_grid[qs[l] | (x.qh[ib32] << (8 - 2 * l) & 0x300)]; + } + tiled_unpk_sign32(g[0], g[1], g[2], g[3], signs, &tile->q[q_off + 32 * ib32]); + qs += 4; + signs += 4; + } + } + } +} + +// iq3_xxs: 3-bit grids through the iq3xxs_grid LUT, 4-bit scale per 32, no min +// codes stored as (value + 128), matching BIAS = 128 +static void tiled_unpack_src0(const block_iq3_xxs * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 8; // 32-wide subblocks + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_iq3_xxs & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.d) * 0.25f; + + const uint8_t * qs = x.qs; + const uint8_t * scales_and_signs = x.qs + QK_K / 4; // 4 bytes per 32: code bits in the top nibble, signs in 7-bit chunks + uint64_t g[4]; + uint8_t signs4[4]; + for (int ib32 = 0; ib32 < QK_K / 32; ib32++) { + const uint32_t aux32 = *(const uint32_t *) (scales_and_signs + 4 * ib32); + tile->scales[s_off + ib32] = (int32_t) (2 * (aux32 >> 28) + 1); + for (int l = 0; l < 4; l++) { + // bytes 8*l .. 8*l+7: low entry e1 (4 values) then high entry e2 (4 values) + g[l] = (uint64_t) iq3xxs_grid[qs[2 * l + 1]] << 32 | iq3xxs_grid[qs[2 * l + 0]]; + signs4[l] = ksigns_iq2xs[(aux32 >> (7 * l)) & 127]; + } + tiled_unpk_sign32(g[0], g[1], g[2], g[3], signs4, &tile->q[q_off + 32 * ib32]); + qs += 8; + } + } + } +} + +// iq3_s: 3-bit grids through the iq3s_grid LUT, 4-bit scale per 32 (two per scale byte), no min +// codes stored as (value + 128), matching BIAS = 128 +static void tiled_unpack_src0(const block_iq3_s * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 8; // 32-wide subblocks + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_iq3_s & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.d); + + const uint8_t * qs = x.qs; + const uint8_t * qh = x.qh; + const uint8_t * signs = x.signs; + for (int ib32 = 0; ib32 < QK_K / 32; ib32 += 2) { + tile->scales[s_off + ib32 + 0] = (int32_t) (1 + 2 * (x.scales[ib32 / 2] & 0xf)); + tile->scales[s_off + ib32 + 1] = (int32_t) (1 + 2 * (x.scales[ib32 / 2] >> 4)); + for (int h = 0; h < 2; h++) { + uint64_t g[4]; + for (int l = 0; l < 4; l++) { + // bytes 8*l .. 8*l+7: low entry e1 (4 values) then high entry e2 (4 values) + g[l] = (uint64_t) iq3s_grid[qs[2 * l + 1] | ((qh[h] << (7 - 2 * l)) & 256)] << 32 + | iq3s_grid[qs[2 * l + 0] | ((qh[h] << (8 - 2 * l)) & 256)]; + } + tiled_unpk_sign32(g[0], g[1], g[2], g[3], signs, &tile->q[q_off + (ib32 + h) * 32]); + qs += 8; + signs += 4; + } + qh += 2; + } + } + } +} + +// iq1_s: ternary grid (values +-1/0) scaled by 8 to leave room for the +-1 delta offset, +// the /8 folds into d; 3-bit scale per 32, no min +// codes stored as (8 * grid + delta + 128), matching BIAS = 128 +static void tiled_unpack_src0(const block_iq1_s * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 8; // 32-wide subblocks + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_iq1_s & x = rows[r * row_stride + slab]; + tile->d[d_off] = ggml_fp16_to_fp32(x.d) * 0.125f; + + const uint8_t * qs = x.qs; + for (int ib = 0; ib < QK_K / 32; ib++) { + const uint16_t hw = x.qh[ib]; + tile->scales[s_off + ib] = (int32_t) (2 * ((hw >> 12) & 7) + 1); + const int8_t delta = (hw & 0x8000) ? -1 : 1; + for (int l = 0; l < 4; l++) { + const uint64_t entry = iq1s_grid[qs[l] | (((hw >> (3 * l)) & 7) << 8)]; + tiled_unpk_tern8((const uint8_t *) &entry, delta, &tile->q[q_off + 32 * ib + 8 * l]); + } + qs += 4; + } + } + } +} + +// iq1_m: like iq1_s but the fp16 scale is packed across the 4 scale bytes (no d field) and the +// delta offset is per 8, giving one scale per 16; 3-bit scale per 16, no min +// codes stored as (8 * grid + delta + 128), matching BIAS = 128 +static void tiled_unpack_src0(const block_iq1_m * rows, int64_t row_stride, int n_rows, tiled_tile_src0 * tile, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + constexpr int NB = 16; // 16-wide subblocks + + const int qk_stride = num_k * TILED_TILE_K; + const int nb_stride = NB * num_k; + for (int slab = 0; slab < num_k; slab++) { + for (int r = 0; r < n_rows; r++) { + const int d_off = slab * TILED_MICRO + r; + const int q_off = r * qk_stride + slab * TILED_TILE_K; + const int s_off = r * nb_stride + slab * NB; + const block_iq1_m & x = rows[r * row_stride + slab]; + + const uint16_t * sc = (const uint16_t *) x.scales; + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); + tile->d[d_off] = ggml_fp16_to_fp32(scale.f16) * 0.125f; + + const uint8_t * qs = x.qs; + const uint8_t * qh = x.qh; + for (int ib = 0; ib < QK_K / 32; ib++) { + const uint16_t hw = sc[ib / 2]; + const int sh = 6 * (ib % 2); + tile->scales[s_off + 2 * ib + 0] = (int32_t) (2 * ((hw >> sh) & 7) + 1); + tile->scales[s_off + 2 * ib + 1] = (int32_t) (2 * ((hw >> (sh + 3)) & 7) + 1); + + const uint16_t idx[4] = { + (uint16_t) (qs[0] | ((qh[0] << 8) & 0x700)), + (uint16_t) (qs[1] | ((qh[0] << 4) & 0x700)), + (uint16_t) (qs[2] | ((qh[1] << 8) & 0x700)), + (uint16_t) (qs[3] | ((qh[1] << 4) & 0x700)), + }; + const int8_t delta[4] = { + (int8_t) ((qh[0] & 0x08) ? -1 : 1), (int8_t) ((qh[0] & 0x80) ? -1 : 1), + (int8_t) ((qh[1] & 0x08) ? -1 : 1), (int8_t) ((qh[1] & 0x80) ? -1 : 1), + }; + for (int l = 0; l < 4; l++) { + const uint64_t entry = iq1s_grid[idx[l]]; + tiled_unpk_tern8((const uint8_t *) &entry, delta[l], &tile->q[q_off + 32 * ib + 8 * l]); + } + qs += 4; + qh += 2; + } + } + } +} + +// unpack src1 tile from q8_K rows. num_k = K-blocks per row: each row decodes num_k consecutive +// source blocks (rows[r][kblk + slab]) into the tile at row stride num_k*256, so each activation +// row is one long stream. num_k=1 is the standard single-slab path (byte-identical to before). +static void tiled_unpack_src1_q8_K(const block_q8_K * const * rows, int n_rows, tiled_tile_src1 * tile, + int kblk, int num_k) { + GGML_ASSERT(n_rows <= TILED_TILE_ROWS); + const int n_padded = (n_rows + TILED_MICRO - 1) & ~(TILED_MICRO - 1); + const int bs_stride = (num_k == 1) ? TILED_TILE_ROWS : TILED_MICRO; + const int n16 = TILED_TILE_K / TILED_MICRO; + // natural [row][k] fill, zero-pad ragged tail; codes stay natural here, the driver + // repacks them just-in-time via tiled_repack_src1 (per 16-row band on the standard path, + // the whole chunk on the narrow path) so each repacked region is L1-hot for its uses + for (int slab = 0; slab < num_k; slab++) { + const int q_off = slab * TILED_TILE_K; + for (int r = 0; r < n_padded; r++) { + if (r < n_rows) { + memcpy(&tile->q[r * (num_k * TILED_TILE_K) + q_off], rows[r][kblk + slab].qs, TILED_TILE_K); + } else { + memset(&tile->q[r * (num_k * TILED_TILE_K) + q_off], 0, TILED_TILE_K); + } + } + // d and bsums (ISA-independent) + for (int r = 0; r < n_padded; r++) { + if (r < n_rows) { + const block_q8_K & x = rows[r][kblk + slab]; + for (int s = 0; s < n16; s++) { tile->bsums[(slab * n16 + s) * bs_stride + r] = (int32_t) x.bsums[s]; } + tile->d[slab * TILED_MICRO + r] = x.d; + } else { + for (int s = 0; s < n16; s++) { tile->bsums[(slab * n16 + s) * bs_stride + r] = 0; } + tile->d[slab * TILED_MICRO + r] = 0.0f; + } + } + } +} + +// GGML_CPU_TILED_MM: master switch, on by default. If off, we fast return false and normal vec_dot mul_mat resumes +static bool ggml_tiled_matmul_enabled(void) { + static bool enabled = true; + static std::once_flag flag; + std::call_once(flag, []() { + const char * env = getenv("GGML_CPU_TILED_MM"); + enabled = env == NULL || atoi(env) != 0; + }); + return enabled; +} + +// GGML_CPU_TILED_MM_FORCE: test/bench only, take the tiled path even when unprofitable +static bool ggml_tiled_matmul_forced(void) { + static bool forced = false; + static std::once_flag flag; + std::call_once(flag, []() { + const char * env = getenv("GGML_CPU_TILED_MM_FORCE"); + forced = env != NULL && atoi(env) == 1; + }); + return forced; +} + +// hard constraints shared by the MUL_MAT and MUL_MAT_ID entries; the src0 type gate is the +// entries' type switch + +#if !defined(__AVX512VNNI__) && !defined(__AVX2__) && !defined(__AVX__) +static bool ggml_tiled_supported(const struct ggml_tensor * src0, + const struct ggml_tensor * src1) { + UNUSED(src0); + UNUSED(src1); + return false; +} +#else +static bool ggml_tiled_supported(const struct ggml_tensor * src0, + const struct ggml_tensor * src1) { + if (!ggml_tiled_matmul_enabled()) { + return false; + } + + // repack-buffer weights hold a repacked layout, let that kernel handle + if (src0->extra != NULL) { + return false; + } + // Supported quant types for src0 + switch (src0->type) { + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q2_K: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + return true; + default: + return false; + } + if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_Q8_K) { + return false; + } + + if (src1->type == GGML_TYPE_Q8_K && !ggml_is_contiguous(src1)) { + // We can handle noncontiguous floats because we're repacking to q8_k anyways + return false; + } + return true; +} +#endif + +// per-thread workspace slot size (0 when tiled is disabled or unsupported on this arch) +static size_t ggml_tiled_ws_size(void) { + if (!ggml_tiled_matmul_enabled()) { + return 0; + } + return TILED_WS_SLOT; // clean 512KB slot, rounded up from sizeof(tiled_ws) +} + +size_t ggml_tiled_wdata_size(int n_tasks, struct ggml_tensor * dst) { + if (! ggml_tiled_supported(dst->src[0], dst->src[1])) { + return 0; // unsupported, don't allocate + } + return 64 + n_tasks * ggml_tiled_ws_size(); // 64 for alignment plus one 512KB slot per thread +} + + +// narrow-path K chunk: how many K's worth of weights to read contiguously into long/skinny tiles +// Assumes 32kb L1 cache budget +static int ggml_tiled_narrow_k_extent(int64_t n_rows, int64_t ne00) { + int l1 = 31 * 1024; // fit weights in 31k to leave room for scales + + int rows = (int) n_rows; + if (rows < TILED_MICRO) rows = TILED_MICRO; + int ke = l1 / (TILED_MICRO + rows); + ke &= ~(TILED_TILE_K - 1); // floor to a multiple of 256 + const int ke_max = TILED_TILE_ROWS * TILED_TILE_K / TILED_MICRO; // buffer ceiling + if (ke > ke_max) ke = ke_max; + while (ke >= TILED_TILE_K && ne00 % ke != 0) { + ke -= TILED_TILE_K; + } + return ke < TILED_TILE_K ? 0 : ke; +} + +// Writeback of the 256x256 window: buf is j-major (row stride buf_stride), dst is i-major (column stride dst_stride). +static void tiled_store_window(const float * buf, int n_src0, int n_src1, int buf_stride, + float * dst, size_t dst_stride) { + int ri = 0; + for (; ri + 16 <= n_src0; ri += 16) { + int rj = 0; + for (; rj + 8 <= n_src1; rj += 8) { + float r[16][8]; + for (int t = 0; t < 16; t++) { + for (int u = 0; u < 8; u++) { + r[t][u] = buf[(ri + t) * buf_stride + rj + u]; + } + } + for (int u = 0; u < 8; u++) { + for (int t = 0; t < 16; t++) { + dst[(ri + t) + (size_t) (rj + u) * dst_stride] = r[t][u]; + } + } + } + // ragged j tail + for (; rj < n_src1; rj++) { + for (int t = 0; t < 16; t++) { + dst[(ri + t) + (size_t) rj * dst_stride] = buf[(ri + t) * buf_stride + rj]; + } + } + } + // ragged i tail + for (; ri < n_src0; ri++) { + for (int j = 0; j < n_src1; j++) { + dst[ri + (size_t) j * dst_stride] = buf[ri * buf_stride + j]; + } + } +} + +// MUL_MAT_ID (MoE): src1 rows are gathered per output row (expert dispatch) and dst rows are +// scattered back. The expert's cne1 gathered q8_K rows are staged once per expert into +// thread-local scratch (contiguous rows, plus the [k/4][row][4] interleave on VNNI) so the +// unpack and microtile are reused unchanged from mul_mat. + +// src0 rows per g group; finer than the TILED_TILE_K kernel tile so all threads stay busy at small ne01 +#define TILED_MMID_GROUP 64 + +// like tiled_store_window, but the dst columns are not contiguous: col_ptrs[j] points at the +// start of dst column j, whose rows are contiguous (dim 0, nb0 == 4 bytes). +static void tiled_store_window_scatter(const float * buf, int n_src0, int n_src1, int buf_stride, + float * const * col_ptrs) { + int ri = 0; + for (; ri + 16 <= n_src0; ri += 16) { + int rj = 0; + for (; rj + 8 <= n_src1; rj += 8) { + float r[16][8]; + for (int t = 0; t < 16; t++) { + for (int u = 0; u < 8; u++) { + r[t][u] = buf[(ri + t) * buf_stride + rj + u]; + } + } + for (int u = 0; u < 8; u++) { + for (int t = 0; t < 16; t++) { + col_ptrs[rj + u][ri + t] = r[t][u]; + } + } + } + // ragged j tail + for (; rj < n_src1; rj++) { + for (int t = 0; t < 16; t++) { + col_ptrs[rj][ri + t] = buf[(ri + t) * buf_stride + rj]; + } + } + } + // ragged i tail + for (; ri < n_src0; ri++) { + for (int j = 0; j < n_src1; j++) { + col_ptrs[j][ri] = buf[ri * buf_stride + j]; + } + } +} + + +// one (g, k) macrotile: zero the acc window, sweep K in 256 element slabs, scatter the result rows +// rows points at this k window's routed src1 rows, row r at its base block +template +static void tiled_mmid_gemm_window(struct ggml_tensor * dst, const struct ggml_tensor * src0, + const char * src0_cur, int64_t r, int64_t k, int64_t nrows, + const int32_t * expert_rows, + const block_q8_K * const * rows, + tiled_ws * ws) { + const int64_t ne00 = src0->ne[0]; + const int64_t ne01 = src0->ne[1]; + + const int64_t r_end = MIN(r + TILED_MMID_GROUP, ne01); + const int n_src0 = (int) (r_end - r); + + const size_t src0_bs = ggml_type_size(src0->type); + const int64_t src0_stride = src0->nb[1] / src0_bs; + + // the window is at most TILED_MMID_GROUP x TILED_TILE_K, so zero only that region of acc + for (int64_t i = 0; i < n_src0; i++) { + memset(&ws->acc[i * TILED_TILE_ROWS], 0, nrows * sizeof(float)); + } + + // scattered writeback: column m goes to its routed dst row; r * nb[0] is the window row offset + float * col_ptrs[TILED_TILE_ROWS]; + for (int64_t m = 0; m < nrows; m++) { + col_ptrs[m] = (float *) ((char *) dst->data + r * dst->nb[0] + expert_rows[2 * (k + m) + 0] * dst->nb[1] + + expert_rows[2 * (k + m) + 1] * dst->nb[2]); + } + + // narrow path (small nrows): we're memory bound in this case so do longer, skinny tiled in order + // to have contiguous reads and improve memory bandwidth + if (nrows <= TILED_MICRO) { + const int k_extent = ggml_tiled_narrow_k_extent(nrows, ne00); + const int num_k = k_extent / TILED_TILE_K; + for (int64_t k0 = 0; k0 < ne00; k0 += k_extent) { + const int kstart = (int) (k0 / TILED_TILE_K); + // activation: one 16-row band (nrows <= 16). Unpack the whole k_extent chunk (all + // slabs) so each activation row is a long stream, then repack each slab in place + tiled_unpack_src1_q8_K(rows, nrows, &ws->src1, kstart, num_k); + // repack the whole k_extent chunk in place: num_k slabs x 4 tiles, 16 rows, row stride num_k*256 + tiled_repack_src1(&ws->src1, 0, num_k, ACTBIAS); + // weight groups (16 at a time): unpack the k_extent chunk of the group (the long per-row + // read), then one standard MAC per slab accumulating into the same acc rows + for (int64_t ir0 = r; ir0 < r_end; ir0 += TILED_MICRO) { + const int n0 = (int) MIN(TILED_MICRO, r_end - ir0); + const B * wbase = (const B *) (src0_cur + ir0 * src0->nb[1] + kstart * src0_bs); + tiled_unpack_src0(wbase, src0_stride, n0, &ws->src0, num_k); + tiled_repack_src0(&ws->src0, n0, num_k, BIAS, ACTBIAS); + float * buf = ws->acc + (ir0 - r) * TILED_TILE_ROWS; + for (int slab = 0; slab < num_k; slab++) { + tiled_run_microtile(ws->src0, ws->src1, 0, 0, num_k, slab, buf, TILED_TILE_ROWS); + } + } + } + } else { + // standard per-slab path: K is stepped in 256-element slabs + for (int64_t ib = 0; ib < ne00; ib += TILED_TILE_K) { + const int kblk = (int) (ib / TILED_TILE_K); + tiled_unpack_src0((const B *) (src0_cur + r * src0->nb[1] + kblk * src0_bs), src0_stride, n_src0, &ws->src0, 1); + tiled_repack_src0(&ws->src0, n_src0, 1, BIAS, ACTBIAS); + tiled_unpack_src1_q8_K(rows, nrows, &ws->src1, kblk, 1); + // 16x16 microtiles sweeping the window; repack each src1 band just-in-time + // (j0-outer) so only the bands actually used are repacked and each is L1-hot + for (int64_t ir1 = 0; ir1 < nrows; ir1 += TILED_MICRO) { + tiled_repack_src1(&ws->src1, (int) ir1, 1, ACTBIAS); + for (int64_t ir0 = r; ir0 < r_end; ir0 += TILED_MICRO) { + tiled_run_microtile(ws->src0, ws->src1, + (int) (ir0 - r), (int) ir1, 1, 0, + ws->acc, TILED_TILE_ROWS); + } + } + } + } + + tiled_store_window_scatter(ws->acc, n_src0, nrows, TILED_TILE_ROWS, col_ptrs); +} + +// one expert of MUL_MAT_ID: each k window's dispatched rows are pointed at by a per-window +// row pointer list and swept over the thread's row windows, the dst rows are scattered back +// per window +template +static void ggml_compute_forward_mul_mat_id_tiled_one_expert( + const struct ggml_compute_params * params, + struct ggml_tensor * dst, + int64_t cur_a, + int64_t cne1, + const int32_t * expert_rows, + char * scratch) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + GGML_TENSOR_BINARY_OP_LOCALS + + const int ith = params->ith; + const int nth = params->nth; + + const enum ggml_type vec_dot_type = ggml_get_type_traits_cpu(src0->type)->vec_dot_type; + + const size_t nbw1 = (src1->type == vec_dot_type) ? nb11 : ggml_row_size(vec_dot_type, ne10); + const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + + const char * src0_cur = (const char *) src0->data + cur_a * nb02; + + // this thread's workspace + tiled_ws * ws = (tiled_ws *) (scratch + (size_t) ith * ggml_tiled_ws_size()); + + // groups of TILED_MMID_GROUP rows; rounded up, the window tail is clamped in the gemm + const int64_t ngroups = (ne01 + TILED_MMID_GROUP - 1) / TILED_MMID_GROUP; + + const int64_t g0 = (ngroups * ith) / nth; + const int64_t g1 = (ngroups * (ith + 1)) / nth; + + // no rows for this thread; nothing to do + if (g0 >= g1) { + return; + } + + // MUL_MAT_ID scatters the src1 rows per routed row: point each k window's rows + // at the routed src1 rows, then sweep the row windows + for (int64_t k = 0; k < cne1; k += TILED_TILE_K) { + const int64_t nrows = MIN(TILED_TILE_K, cne1 - k); + + const block_q8_K * rows[TILED_TILE_ROWS]; + for (int64_t i = 0; i < nrows; i++) { + const int64_t i11 = expert_rows[2 * (k + i) + 0] % ne11; + const int64_t i12 = expert_rows[2 * (k + i) + 1]; + rows[i] = (const block_q8_K *) ((const char *) wdata + (i11 + i12 * ne11) * nbw1); + } + + for (int64_t g = g0; g < g1; g++) { + const int64_t r = g * TILED_MMID_GROUP; + + tiled_mmid_gemm_window(dst, src0, src0_cur, r, k, nrows, expert_rows, rows, ws); + } + } +} + +template +static void ggml_compute_forward_mul_mat_tiled_one_chunk( + const struct ggml_compute_params * params, + struct ggml_tensor * dst, + const int64_t ir0_start, + const int64_t ir0_end, + const int64_t ir1_start, + const int64_t ir1_end, + tiled_ws * ws) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + GGML_TENSOR_BINARY_OP_LOCALS + + const enum ggml_type vec_dot_type = ggml_get_type_traits_cpu(src0->type)->vec_dot_type; + + // broadcast factors + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + if (ir0_start >= ir0_end || ir1_start >= ir1_end) { + return; + } + + const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + const size_t row_size = ggml_row_size(vec_dot_type, ne10); + const size_t src0_bs = ggml_type_size(src0->type); + const size_t src1_bs = ggml_type_size(vec_dot_type); + + GGML_ASSERT(ne00 % 256 == 0); + assert(ne12 % ne02 == 0); + assert(ne13 % ne03 == 0); + + const int64_t src0_stride = nb01 / src0_bs; // blocks between src0 rows + const int64_t src1_stride = (src1->type == vec_dot_type ? src1->nb[1] : row_size) / src1_bs; + + const int64_t TILE = 256; + const int64_t MICRO = 16; + + // 256-wide windows over the chunk. The iir1 window is additionally clamped at the + // src1 batch (ne11) boundary: the tiles require a constant batch index (i12/i13) + // within a window, so advance by the clamped end, not a fixed 256. + for (int64_t iir1 = ir1_start; iir1 < ir1_end; ) { + int64_t iir1_end = MIN(iir1 + TILE, ir1_end); + const int64_t bnd = (iir1 / ne11 + 1) * ne11; + if (bnd < iir1_end) { + iir1_end = bnd; + } + + const int n_src1 = (int) (iir1_end - iir1); + + // batch coords, constant within the clamped window + const int64_t i13 = iir1 / (ne12 * ne11); + const int64_t i12 = (iir1 - i13 * ne12 * ne11) / ne11; + // within-batch row; dst_col below holds the i12/i13 batch offset, so the store + // applies i11 * nb1 (not the flattened iir1, which spans all batch dims) + const int64_t i11 = iir1 - i13 * ne12 * ne11 - i12 * ne11; + + // dst batches == src1 batches (ggml_mul_mat), so the loop batch coords are in + // src1 space; src0 batches are broadcast over them, map down into src0's batch + const int64_t i02 = i12 / r2; + const int64_t i03 = i13 / r3; + + const char * src0_row = (const char *) src0->data + i02 * src0->nb[2] + i03 * src0->nb[3]; + char * dst_col = (char *) dst->data + i12 * nb2 + i13 * nb3; + + // rows[r] is the window's row r base block; the k-slab offset is applied in the unpack + const block_q8_K * rows[TILED_TILE_ROWS]; + for (int r = 0; r < n_src1; r++) { + rows[r] = (const block_q8_K *) ((const char *) wdata + (iir1 + r) * src1_stride * src1_bs); + } + + for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += TILE) { + int64_t iir0_end = MIN(iir0 + TILE, ir0_end); + const int n_src0 = (int) (iir0_end - iir0); + + // result buffer zeroed once per macrotile + memset(ws->acc, 0, (size_t)TILED_TILE_ROWS * TILED_TILE_ROWS * sizeof(float)); + + if (n_src1 <= TILED_MICRO) { + // narrow path: if we have few enough rows, we're memory bound + // Do longer, skinner tiles so we have longer contiguous reads and improve bandwidth + const int k_extent = ggml_tiled_narrow_k_extent(n_src1, ne00); + const int num_k = k_extent / TILED_TILE_K; + for (int64_t k0 = 0; k0 < ne00; k0 += k_extent) { + const int kstart = (int) (k0 / TILED_TILE_K); + // activation: one 16-row band (n_src1 <= 16). Unpack the whole k_extent chunk + // (all slabs) so each activation row is a long stream, then repack each slab in place + tiled_unpack_src1_q8_K(rows, n_src1, &ws->src1, kstart, num_k); + // repack the whole k_extent chunk in place, if kernel wants to: num_k slabs x 4 tiles, 16 rows, row stride num_k*256 + tiled_repack_src1(&ws->src1, 0, num_k, ACTBIAS); + // weight groups: unpack the k_extent chunk of each 16-row group (the long per-row + // read), then one standard MAC per slab accumulating into the same acc rows + for (int64_t ir0 = iir0; ir0 < iir0_end; ir0 += MICRO) { + const int n0 = (int) MIN(MICRO, iir0_end - ir0); + const B * wbase = (const B *) (src0_row + ir0 * nb01 + kstart * src0_bs); + tiled_unpack_src0(wbase, src0_stride, n0, &ws->src0, num_k); + tiled_repack_src0(&ws->src0, n0, num_k, BIAS, ACTBIAS); + float * buf = ws->acc + (ir0 - iir0) * TILED_TILE_ROWS; + for (int slab = 0; slab < num_k; slab++) { + tiled_run_microtile(ws->src0, ws->src1, 0, 0, num_k, slab, buf, TILED_TILE_ROWS); + } + } + } + } else { + // standard per-slab path: K is stepped in 256-element chunks + for (int64_t ib = 0; ib < ne00; ib += TILE) { + const int kblk = (int) (ib / TILE); + tiled_unpack_src0((const B *) (src0_row + iir0 * nb01 + kblk * src0_bs), src0_stride, n_src0, &ws->src0, 1); + tiled_repack_src0(&ws->src0, n_src0, 1, BIAS, ACTBIAS); + tiled_unpack_src1_q8_K(rows, n_src1, &ws->src1, kblk, 1); + + // 16x16 microtiles sweeping the window; repack each src1 band before first use + for (int64_t ir1 = iir1; ir1 < iir1_end; ir1 += MICRO) { + tiled_repack_src1(&ws->src1, (int) (ir1 - iir1), 1, ACTBIAS); + for (int64_t ir0 = iir0; ir0 < iir0_end; ir0 += MICRO) { + tiled_run_microtile(ws->src0, ws->src1, + (int) (ir0 - iir0), (int) (ir1 - iir1), 1, 0, + ws->acc, TILED_TILE_ROWS); + } + } + } + } + // write acc back out from L2 to main memory + tiled_store_window(ws->acc, n_src0, n_src1, TILED_TILE_ROWS, + (float *) (dst_col + iir0 * nb0 + i11 * nb1), nb1 / nb0); + } + iir1 = iir1_end; + } +} + +template +static void ggml_compute_forward_mul_mat_tiled_driver( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + GGML_TENSOR_BINARY_OP_LOCALS + + const int ith = params->ith; + const int nth = params->nth; + + enum ggml_type const vec_dot_type = ggml_get_type_traits_cpu(src0->type)->vec_dot_type; + ggml_from_float_t const from_float = ggml_get_type_traits_cpu(vec_dot_type)->from_float; + + GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne1 == ne11); + GGML_ASSERT(ne2 == ne12); + GGML_ASSERT(ne3 == ne13); + + // we don't support permuted src0 or src1 + GGML_ASSERT(nb10 == ggml_type_size(src1->type)); + + // dst cannot be transposed or permuted + GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(nb0 <= nb1); + GGML_ASSERT(nb1 <= nb2); + GGML_ASSERT(nb2 <= nb3); + + if (src1->type != vec_dot_type) { + char * wdata = (char *) params->wdata; + + const size_t nbw0 = ggml_type_size(vec_dot_type); + const size_t nbw1 = ggml_row_size(vec_dot_type, ne10); + const size_t nbw2 = nbw1*ne11; + const size_t nbw3 = nbw2*ne12; + + assert(params->wsize >= ne13*nbw3); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + + for (int64_t i13 = 0; i13 < ne13; ++i13) { + for (int64_t i12 = 0; i12 < ne12; ++i12) { + for (int64_t i11 = 0; i11 < ne11; ++i11) { + size_t bs = ggml_blck_size(vec_dot_type); + int64_t ne10_block_start = (ith * ne10/bs) / nth; + int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth; + from_float((float *)((char *) src1->data + i13*src1->nb[3] + i12*src1->nb[2] + i11*src1->nb[1] + ne10_block_start*bs*src1->nb[0]), + (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0), + (ne10_block_end - ne10_block_start) * bs); + } + } + } + } + + if (ith == 0) { + // Every thread starts at ith, so the first unprocessed chunk is nth. This saves a bit of coordination right at the start. + ggml_threadpool_chunk_set(params->threadpool, nth); + } + + ggml_barrier(params->threadpool); + + // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) + const int64_t nr0 = ne0; + + // This is the size of the rest of the dimensions of the result + const int64_t nr1 = ne1 * ne2 * ne3; + + // Now select a reasonable chunk size. + int chunk_size = 256; //TILED_TILE_ROWS; + + // distribute the work across the inner or outer loop based on which one is larger + // The number of chunks in the 0/1 dim. CEIL(nr/chunk_size) + int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; + int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; + + // Step down chunk size if too few chunks to saturate cores, minimum is microtile size + while (nchunk0 * nchunk1 < nth * 4 && chunk_size > 16) { + chunk_size = chunk_size / 2; + nchunk0 = (nr0 + chunk_size - 1) / chunk_size; + nchunk1 = (nr1 + chunk_size - 1) / chunk_size; + } + + // The number of elements in each chunk + const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; + const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; + + // The first chunk comes from our thread_id, the rest will get auto-assigned. + int current_chunk = ith; + + // per-thread workspace: after any converted src1 data in wdata, aligned to 64 + char * ws_base = (char *) params->wdata; + if (src1->type != vec_dot_type) { + ws_base += GGML_PAD(ggml_row_size(vec_dot_type, ggml_nelements(src1)), 64); + } + ws_base = (char *) (((uintptr_t) ws_base + 63) & ~(uintptr_t) 63); + tiled_ws * ws = (tiled_ws *) (ws_base + (size_t) ith * ggml_tiled_ws_size()); + + // TODO: if we KNOW we're on a machine where all cores are equal, we could skip the coordination/work-stealing and just assign chunks deterministically + while (current_chunk < nchunk0 * nchunk1) { + const int64_t ith0 = current_chunk % nchunk0; + const int64_t ith1 = current_chunk / nchunk0; + + const int64_t ir0_start = dr0 * ith0; + const int64_t ir0_end = MIN(ir0_start + dr0, nr0); + + const int64_t ir1_start = dr1 * ith1; + const int64_t ir1_end = MIN(ir1_start + dr1, nr1); + + ggml_compute_forward_mul_mat_tiled_one_chunk(params, dst, ir0_start, ir0_end, ir1_start, ir1_end, ws); + + if (nth >= nchunk0 * nchunk1) { + break; + } + + current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1); + } +} + +// src0 type dispatch, shared by the MUL_MAT and MUL_MAT_ID entries: one expert for +// MUL_MAT_ID (expert_rows != NULL), the full op for MUL_MAT +template +static bool tiled_matmul_dispatch(const struct ggml_compute_params * params, + struct ggml_tensor * dst, + const int32_t * expert_rows, + int64_t cur_a, + int64_t cne1, + char * scratch) { + if (expert_rows == NULL) { + ggml_compute_forward_mul_mat_tiled_driver(params, dst); + } else { + ggml_compute_forward_mul_mat_id_tiled_one_expert(params, dst, cur_a, cne1, expert_rows, scratch); + } + return true; +} + +// the supported src0 types, one list for both ops; false to fall through to vec_dot +static bool ggml_tiled_matmul_type_dispatch(const struct ggml_compute_params * params, + struct ggml_tensor * dst, + const int32_t * expert_rows = NULL, + int64_t cur_a = 0, + int64_t cne1 = 0, + char * scratch = NULL) { + switch (dst->src[0]->type) { + case GGML_TYPE_Q6_K: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_Q5_K: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_Q4_K: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_Q3_K: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_Q2_K: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_IQ4_XS: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_IQ2_XXS: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_IQ2_XS: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_IQ2_S: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_IQ3_XXS: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_IQ3_S: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_IQ1_S: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + case GGML_TYPE_IQ1_M: + return tiled_matmul_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); + default: + return false; + } +} + +static bool ggml_tiled_min_batch(int64_t rows) { + // Profitable at rows >= 8, take even when unprofitable if we're forced + return rows >= 8 || ggml_tiled_matmul_forced(); +} + +// tiled K-quant matmul; returns true if the op was computed here, +// false to fall through to the stock path +bool ggml_compute_forward_mul_mat_tiled( + const struct ggml_compute_params * params, + struct ggml_tensor * dst) { + // --use-ref means bail out and go back to vec_dot reference impl + if (params->use_ref) { + return false; + } + if (!ggml_tiled_supported(dst->src[0], dst->src[1])) { + return false; + } + if (!ggml_tiled_min_batch(dst->src[1]->ne[1])) { + return false; + } + return ggml_tiled_matmul_type_dispatch(params, dst); +} + +// MUL_MAT_ID (MoE), one expert; returns true if the expert was computed here, +// per expert eligibility (type gate, batch floor) is decided here +bool ggml_compute_forward_mul_mat_id_tiled( + const struct ggml_compute_params * params, + struct ggml_tensor * dst, + int64_t cur_a, + int64_t cne1, + const int32_t * expert_rows, + char * scratch) { + if (params->use_ref) { + return false; + } + if (!ggml_tiled_supported(dst->src[0], dst->src[1])) { + return false; + } + // profitability is per expert: the rows routed to this expert + if (!ggml_tiled_min_batch(cne1)) { + return false; + } + return ggml_tiled_matmul_type_dispatch(params, dst, expert_rows, cur_a, cne1, scratch); +} diff --git a/ggml/src/ggml-cpu/tiled/tiled.h b/ggml/src/ggml-cpu/tiled/tiled.h new file mode 100644 index 0000000..6f3e8dc --- /dev/null +++ b/ggml/src/ggml-cpu/tiled/tiled.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Amount of wdata to reserve for tiled workspaces +size_t ggml_tiled_wdata_size(int n_tasks, struct ggml_tensor * dst); + +// tiled K-quant matmul; returns true if the op was computed here, +// false to fall through to the stock path +bool ggml_compute_forward_mul_mat_tiled(const struct ggml_compute_params * params, + struct ggml_tensor * dst); + +// MUL_MAT_ID (MoE) path, one expert; returns true if the expert was computed here, per expert +// eligibility (type gate, batch floor) is decided inside. expert_rows points at the expert's +// row of the matrix_rows table of (expert slot, batch row) int32 pairs; scratch is the +// per-thread tiled_ws region reserved in wdata, n_tasks of ggml_tiled_ws_size() bytes +bool ggml_compute_forward_mul_mat_id_tiled(const struct ggml_compute_params * params, + struct ggml_tensor * dst, + int64_t cur_a, + int64_t cne1, + const int32_t * expert_rows, + char * scratch); + +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-cpu/unary-ops.cpp b/ggml/src/ggml-cpu/unary-ops.cpp index 1d83444..f5d57d6 100644 --- a/ggml/src/ggml-cpu/unary-ops.cpp +++ b/ggml/src/ggml-cpu/unary-ops.cpp @@ -78,7 +78,7 @@ static inline float op_expm1(float x) { } static inline float op_softplus(float x) { - return (x > 20.0f) ? x : logf(1.0f + expf(x)); + return (x > 20.0f) ? x : log1pf(expf(x)); } static inline float op_floor(float x) { diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp index d0e4001..8f0bf49 100644 --- a/ggml/src/ggml-cpu/vec.cpp +++ b/ggml/src/ggml-cpu/vec.cpp @@ -75,12 +75,12 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G ay1 = GGML_F32_VEC_LOAD(y + i); sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1); } - // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only + // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmla on available elements only if (np2 < n) { svbool_t pg = svwhilelt_b32(np2, n); ax1 = svld1_f32(pg, x + np2); ay1 = svld1_f32(pg, y + np2); - sum1 = svmad_f32_m(pg, ax1, ay1, sum1); + sum1 = svmla_f32_m(pg, sum1, ax1, ay1); } // reduce sum1,sum2 to sum1 GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8); @@ -273,67 +273,51 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G #if defined(GGML_SIMD) #if defined(__ARM_FEATURE_SVE) - const int sve_register_length = svcntb() * 8; //get vector length - const int ggml_f16_epr = sve_register_length / 16; // running when 16 - const int ggml_f16_step = 8 * ggml_f16_epr; // choose 8 SVE registers - - const int np= (n & ~(ggml_f16_step - 1)); - svfloat16_t sum1 = svdup_n_f16(0.0f); - svfloat16_t sum2 = svdup_n_f16(0.0f); - svfloat16_t sum3 = svdup_n_f16(0.0f); - svfloat16_t sum4 = svdup_n_f16(0.0f); - - svfloat16_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8; - svfloat16_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8; - for (int i = 0; i < np; i += ggml_f16_step) { - ax1 = GGML_F16x_VEC_LOAD(x + i + 0 * ggml_f16_epr, 0); - ay1 = GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0); - sum1 = GGML_F16x_VEC_FMA(sum1, ax1, ay1); - - ax2 = GGML_F16x_VEC_LOAD(x + i + 1 * ggml_f16_epr, 1); - ay2 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1); - sum2 = GGML_F16x_VEC_FMA(sum2, ax2, ay2); - - ax3 = GGML_F16x_VEC_LOAD(x + i + 2 * ggml_f16_epr, 2); - ay3 = GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2); - sum3 = GGML_F16x_VEC_FMA(sum3, ax3, ay3); - - ax4 = GGML_F16x_VEC_LOAD(x + i + 3 * ggml_f16_epr, 3); - ay4 = GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3); - sum4 = GGML_F16x_VEC_FMA(sum4, ax4, ay4); - - ax5 = GGML_F16x_VEC_LOAD(x + i + 4 * ggml_f16_epr, 4); - ay5 = GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4); - sum1 = GGML_F16x_VEC_FMA(sum1, ax5, ay5); - - ax6 = GGML_F16x_VEC_LOAD(x + i + 5 * ggml_f16_epr, 5); - ay6 = GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5); - sum2 = GGML_F16x_VEC_FMA(sum2, ax6, ay6); + const int ggml_f16_epr = svcnth(); + const int ggml_f16_step = 8 * ggml_f16_epr; + const int np = n - (n % ggml_f16_step); + const int np2 = n - (n % ggml_f16_epr); + + svfloat32_t sum1_lo = svdup_n_f32(0.0f); + svfloat32_t sum1_hi = svdup_n_f32(0.0f); + svfloat32_t sum2_lo = svdup_n_f32(0.0f); + svfloat32_t sum2_hi = svdup_n_f32(0.0f); + svfloat32_t sum3_lo = svdup_n_f32(0.0f); + svfloat32_t sum3_hi = svdup_n_f32(0.0f); + svfloat32_t sum4_lo = svdup_n_f32(0.0f); + svfloat32_t sum4_hi = svdup_n_f32(0.0f); - ax7 = GGML_F16x_VEC_LOAD(x + i + 6 * ggml_f16_epr, 6); - ay7 = GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6); - sum3 = GGML_F16x_VEC_FMA(sum3, ax7, ay7); - - ax8 = GGML_F16x_VEC_LOAD(x + i + 7 * ggml_f16_epr, 7); - ay8 = GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7); - sum4 = GGML_F16x_VEC_FMA(sum4, ax8, ay8); + for (int i = 0; i < np; i += ggml_f16_step) { + ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, GGML_F16x_VEC_LOAD(x + i + 0 * ggml_f16_epr, 0), GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0)); + ggml_sve_f16_fma_widened(&sum2_lo, &sum2_hi, GGML_F16x_VEC_LOAD(x + i + 1 * ggml_f16_epr, 1), GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1)); + ggml_sve_f16_fma_widened(&sum3_lo, &sum3_hi, GGML_F16x_VEC_LOAD(x + i + 2 * ggml_f16_epr, 2), GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2)); + ggml_sve_f16_fma_widened(&sum4_lo, &sum4_hi, GGML_F16x_VEC_LOAD(x + i + 3 * ggml_f16_epr, 3), GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3)); + ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, GGML_F16x_VEC_LOAD(x + i + 4 * ggml_f16_epr, 4), GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4)); + ggml_sve_f16_fma_widened(&sum2_lo, &sum2_hi, GGML_F16x_VEC_LOAD(x + i + 5 * ggml_f16_epr, 5), GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5)); + ggml_sve_f16_fma_widened(&sum3_lo, &sum3_hi, GGML_F16x_VEC_LOAD(x + i + 6 * ggml_f16_epr, 6), GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6)); + ggml_sve_f16_fma_widened(&sum4_lo, &sum4_hi, GGML_F16x_VEC_LOAD(x + i + 7 * ggml_f16_epr, 7), GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7)); } - const int np2 = (n & ~(ggml_f16_epr - 1)); // round down to multiple of 8 - for (int k = np; k < np2; k += ggml_f16_epr) { - svfloat16_t rx = GGML_F16x_VEC_LOAD(x + k, 0); - svfloat16_t ry = GGML_F16x_VEC_LOAD(y + k, 0); - sum1 = GGML_F16x_VEC_FMA(sum1, rx, ry); + for (int i = np; i < np2; i += ggml_f16_epr) { + ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, GGML_F16x_VEC_LOAD(x + i, 0), GGML_F16x_VEC_LOAD(y + i, 0)); } if (np2 < n) { - svbool_t pg = svwhilelt_b16(np2, n); - svfloat16_t hx = svld1_f16(pg, (const __fp16 *)(x + np2)); - svfloat16_t hy = svld1_f16(pg, (const __fp16 *)(y + np2)); + const svbool_t pg = svwhilelt_b16(np2, n); + const svfloat16_t rx = svld1_f16(pg, (const __fp16 *)(x + np2)); + const svfloat16_t ry = svld1_f16(pg, (const __fp16 *)(y + np2)); - sum1 = svmad_f16_x(pg, hx, hy, sum1); + ggml_sve_f16_fma_widened(&sum1_lo, &sum1_hi, rx, ry); } - GGML_F16x_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4); + + sum1_lo = svadd_f32_m(DEFAULT_PG32, sum1_lo, sum2_lo); + sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum2_hi); + sum3_lo = svadd_f32_m(DEFAULT_PG32, sum3_lo, sum4_lo); + sum3_hi = svadd_f32_m(DEFAULT_PG32, sum3_hi, sum4_hi); + sum1_lo = svadd_f32_m(DEFAULT_PG32, sum1_lo, sum3_lo); + sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum3_hi); + + sumf = ggml_sve_sum_f32x2(sum1_lo, sum1_hi); #elif defined(__riscv_v_intrinsic) #if defined(__riscv_zvfh) int vl = __riscv_vsetvlmax_e32m2(); @@ -358,24 +342,24 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G } #endif // __riscv_zvfh #else - const int np = (n & ~(GGML_F16_STEP - 1)); + const int np = (n & ~(GGML_F16_DOT_STEP - 1)); - GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO }; + GGML_F16_DOT_VEC sum[GGML_F16_DOT_ARR] = { GGML_F16_DOT_VEC_ZERO }; - GGML_F16_VEC ax[GGML_F16_ARR]; - GGML_F16_VEC ay[GGML_F16_ARR]; + GGML_F16_DOT_VEC ax[GGML_F16_DOT_ARR]; + GGML_F16_DOT_VEC ay[GGML_F16_DOT_ARR]; - for (int i = 0; i < np; i += GGML_F16_STEP) { - for (int j = 0; j < GGML_F16_ARR; j++) { - ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j); - ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j); + for (int i = 0; i < np; i += GGML_F16_DOT_STEP) { + for (int j = 0; j < GGML_F16_DOT_ARR; j++) { + ax[j] = GGML_F16_DOT_VEC_LOAD(x + i + j*GGML_F16_DOT_EPR, j); + ay[j] = GGML_F16_DOT_VEC_LOAD(y + i + j*GGML_F16_DOT_EPR, j); - sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]); + sum[j] = GGML_F16_DOT_VEC_FMA(sum[j], ax[j], ay[j]); } } // reduce sum0..sum3 to sum0 - GGML_F16_VEC_REDUCE(sumf, sum); + GGML_F16_DOT_VEC_REDUCE(sumf, sum); // leftovers for (int i = np; i < n; ++i) { @@ -417,6 +401,10 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) { for (; i + 3 < n; i += 4) { vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i))); } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(ggml_v_silu(vec_xl(0, x + i)), 0, y + i); + } #elif defined(__riscv_v_intrinsic) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); @@ -454,6 +442,10 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * for (; i + 3 < n; i += 4) { vst1q_f32(y + i, vmulq_f32(ggml_v_silu(vld1q_f32(x + i)), vld1q_f32(g + i))); } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(vec_mul(ggml_v_silu(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i); + } #elif defined(__riscv_v_intrinsic) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); @@ -468,6 +460,92 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * } } +#ifndef GGML_GELU_QUICK_FP16 +// computed in f32 without the fp16 table, which loses precision on f32 input +void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) { + int i = 0; +#if defined(__AVX512F__) && defined(__AVX512DQ__) + for (; i + 15 < n; i += 16) { + _mm512_storeu_ps(y + i, ggml_v_gelu_quick(_mm512_loadu_ps(x + i))); + } +#elif defined(__AVX2__) && defined(__FMA__) + for (; i + 7 < n; i += 8) { + _mm256_storeu_ps(y + i, ggml_v_gelu_quick(_mm256_loadu_ps(x + i))); + } +#elif defined(__SSE2__) + for (; i + 3 < n; i += 4) { + _mm_storeu_ps(y + i, ggml_v_gelu_quick(_mm_loadu_ps(x + i))); + } +#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__) + const int vlen = svcntw(); + for (; i < n; i += vlen) { + const svbool_t pg = svwhilelt_b32_s32(i, n); + svst1_f32(pg, y + i, ggml_v_gelu_quick(pg, svld1_f32(pg, x + i))); + } +#elif defined(__ARM_NEON) && defined(__aarch64__) + for (; i + 3 < n; i += 4) { + vst1q_f32(y + i, ggml_v_gelu_quick(vld1q_f32(x + i))); + } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(ggml_v_gelu_quick(vec_xl(0, x + i)), 0, y + i); + } +#elif defined(__riscv_v_intrinsic) + for (int vl; i < n; i += vl) { + vl = __riscv_vsetvl_e32m2(n - i); + vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); + vfloat32m2_t vy = ggml_v_gelu_quick_m2(vx, vl); + __riscv_vse32_v_f32m2(&y[i], vy, vl); + } +#endif + for (; i < n; ++i) { + y[i] = ggml_gelu_quick_f32(x[i]); + } +} + +void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) { + int i = 0; +#if defined(__AVX512F__) && defined(__AVX512DQ__) + for (; i + 15 < n; i += 16) { + _mm512_storeu_ps(y + i, _mm512_mul_ps(ggml_v_gelu_quick(_mm512_loadu_ps(x + i)), _mm512_loadu_ps(g + i))); + } +#elif defined(__AVX2__) && defined(__FMA__) + for (; i + 7 < n; i += 8) { + _mm256_storeu_ps(y + i, _mm256_mul_ps(ggml_v_gelu_quick(_mm256_loadu_ps(x + i)), _mm256_loadu_ps(g + i))); + } +#elif defined(__SSE2__) + for (; i + 3 < n; i += 4) { + _mm_storeu_ps(y + i, _mm_mul_ps(ggml_v_gelu_quick(_mm_loadu_ps(x + i)), _mm_loadu_ps(g + i))); + } +#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__) + const int vlen = svcntw(); + for (; i < n; i += vlen) { + const svbool_t pg = svwhilelt_b32_s32(i, n); + svst1_f32(pg, y + i, svmul_f32_x(pg, ggml_v_gelu_quick(pg, svld1_f32(pg, x + i)), svld1_f32(pg, g + i))); + } +#elif defined(__ARM_NEON) && defined(__aarch64__) + for (; i + 3 < n; i += 4) { + vst1q_f32(y + i, vmulq_f32(ggml_v_gelu_quick(vld1q_f32(x + i)), vld1q_f32(g + i))); + } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + vec_xst(vec_mul(ggml_v_gelu_quick(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i); + } +#elif defined(__riscv_v_intrinsic) + for (int vl; i < n; i += vl) { + vl = __riscv_vsetvl_e32m2(n - i); + vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); + vfloat32m2_t vg = __riscv_vle32_v_f32m2(&g[i], vl); + vfloat32m2_t vy = __riscv_vfmul_vv_f32m2(ggml_v_gelu_quick_m2(vx, vl), vg, vl); + __riscv_vse32_v_f32m2(&y[i], vy, vl); + } +#endif + for (; i < n; ++i) { + y[i] = ggml_gelu_quick_f32(x[i]) * g[i]; + } +} +#endif + ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean) { int i = 0; ggml_float sum = 0; @@ -597,6 +675,12 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float vst1q_f32(y + i, val); sum += (ggml_float)vaddvq_f32(val); } +#elif defined(__VXE__) || defined(__VXE2__) + for (; i + 3 < n; i += 4) { + float32x4_t v_val = ggml_v_expf(vec_sub(vec_xl(0, x + i), vec_splats(max))); + vec_xst(v_val, 0, y + i); + sum += (ggml_float)vec_hsum_f32x4(v_val); + } #elif defined(__riscv_v_intrinsic) vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1); for (int avl; i < n; i += avl) { diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index a0375a2..f24fa00 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -14,8 +14,36 @@ // floating point type used to accumulate sums typedef double ggml_float; +#if defined(__ARM_FEATURE_SVE) +inline static void ggml_sve_f16_fma_widened( + svfloat32_t * acc_lo, + svfloat32_t * acc_hi, + svfloat16_t x, + svfloat16_t y) { +#if defined(__ARM_FEATURE_SVE2) + *acc_lo = svmlalb_f32(*acc_lo, x, y); + *acc_hi = svmlalt_f32(*acc_hi, x, y); +#else + // Plain SVE fallback path if SVE2 instructions not available + svfloat16_t x_even = svtrn1_f16(x, x); + svfloat16_t x_odd = svtrn2_f16(x, x); + + svfloat16_t y_even = svtrn1_f16(y, y); + svfloat16_t y_odd = svtrn2_f16(y, y); + + svbool_t pg = svptrue_b32(); + + *acc_lo = svmla_f32_x(pg, *acc_lo, svcvt_f32_f16_x(pg, x_even), svcvt_f32_f16_x(pg, y_even)); + *acc_hi = svmla_f32_x(pg, *acc_hi, svcvt_f32_f16_x(pg, x_odd), svcvt_f32_f16_x(pg, y_odd)); +#endif +} + +inline static ggml_float ggml_sve_sum_f32x2(svfloat32_t sum_lo, svfloat32_t sum_hi) { + return (ggml_float) (svaddv_f32(svptrue_b32(), sum_lo) + svaddv_f32(svptrue_b32(), sum_hi)); +} +#endif + #define GGML_GELU_FP16 -#define GGML_GELU_QUICK_FP16 #define GGML_SOFT_MAX_UNROLL 4 #define GGML_VEC_DOT_UNROLL 2 @@ -122,108 +150,61 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GG #if defined(GGML_SIMD) #if defined(__ARM_FEATURE_SVE) - const int sve_register_length = svcntb() * 8; - const int ggml_f16_epr = sve_register_length / 16; // running when 16 - const int ggml_f16_step = 8 * ggml_f16_epr; // choose 8 SVE registers - - int np = (n & ~(ggml_f16_step - 1)); - - svfloat16_t sum_00 = svdup_n_f16(0.0f); - svfloat16_t sum_01 = svdup_n_f16(0.0f); - svfloat16_t sum_02 = svdup_n_f16(0.0f); - svfloat16_t sum_03 = svdup_n_f16(0.0f); + const int ggml_f16_epr = svcnth(); + const int ggml_f16_step = 2 * ggml_f16_epr; + int np = n - (n % ggml_f16_step); + int np2 = n - (n % ggml_f16_epr); - svfloat16_t sum_10 = svdup_n_f16(0.0f); - svfloat16_t sum_11 = svdup_n_f16(0.0f); - svfloat16_t sum_12 = svdup_n_f16(0.0f); - svfloat16_t sum_13 = svdup_n_f16(0.0f); - - svfloat16_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8; - svfloat16_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8; + svfloat32_t sum_0_0_lo = svdup_n_f32(0.0f); + svfloat32_t sum_0_0_hi = svdup_n_f32(0.0f); + svfloat32_t sum_0_1_lo = svdup_n_f32(0.0f); + svfloat32_t sum_0_1_hi = svdup_n_f32(0.0f); + svfloat32_t sum_1_0_lo = svdup_n_f32(0.0f); + svfloat32_t sum_1_0_hi = svdup_n_f32(0.0f); + svfloat32_t sum_1_1_lo = svdup_n_f32(0.0f); + svfloat32_t sum_1_1_hi = svdup_n_f32(0.0f); for (int i = 0; i < np; i += ggml_f16_step) { - ay1 = GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0); // 8 elements - - ax1 = GGML_F16x_VEC_LOAD(x[0] + i + 0*ggml_f16_epr, 0); // 8 elements - sum_00 = GGML_F16x_VEC_FMA(sum_00, ax1, ay1); // sum_00 = sum_00+ax1*ay1 - ax1 = GGML_F16x_VEC_LOAD(x[1] + i + 0*ggml_f16_epr, 0); // 8 elements - sum_10 = GGML_F16x_VEC_FMA(sum_10, ax1, ay1); - - ay2 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1); // next 8 elements - - ax2 = GGML_F16x_VEC_LOAD(x[0] + i + 1*ggml_f16_epr, 1); // next 8 elements - sum_01 = GGML_F16x_VEC_FMA(sum_01, ax2, ay2); - ax2 = GGML_F16x_VEC_LOAD(x[1] + i + 1*ggml_f16_epr, 1); - sum_11 = GGML_F16x_VEC_FMA(sum_11, ax2, ay2); - - ay3 = GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2); - - ax3 = GGML_F16x_VEC_LOAD(x[0] + i + 2*ggml_f16_epr, 2); - sum_02 = GGML_F16x_VEC_FMA(sum_02, ax3, ay3); - ax3 = GGML_F16x_VEC_LOAD(x[1] + i + 2*ggml_f16_epr, 2); - sum_12 = GGML_F16x_VEC_FMA(sum_12, ax3, ay3); - - ay4 = GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3); - - ax4 = GGML_F16x_VEC_LOAD(x[0] + i + 3*ggml_f16_epr, 3); - sum_03 = GGML_F16x_VEC_FMA(sum_03, ax4, ay4); - ax4 = GGML_F16x_VEC_LOAD(x[1] + i + 3*ggml_f16_epr, 3); - sum_13 = GGML_F16x_VEC_FMA(sum_13, ax4, ay4); - - ay5 = GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4); - - ax5 = GGML_F16x_VEC_LOAD(x[0] + i + 4*ggml_f16_epr, 4); - - sum_00 = GGML_F16x_VEC_FMA(sum_00, ax5, ay5); - ax5 = GGML_F16x_VEC_LOAD(x[1] + i + 4*ggml_f16_epr, 4); - sum_10 = GGML_F16x_VEC_FMA(sum_10, ax5, ay5); - - ay6 = GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5); - - ax6 = GGML_F16x_VEC_LOAD(x[0] + i + 5*ggml_f16_epr, 5); - - sum_01 = GGML_F16x_VEC_FMA(sum_01, ax6, ay6); - ax6 = GGML_F16x_VEC_LOAD(x[1] + i + 5*ggml_f16_epr, 5); - sum_11 = GGML_F16x_VEC_FMA(sum_11, ax6, ay6); - - ay7 = GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6); - - ax7 = GGML_F16x_VEC_LOAD(x[0] + i + 6*ggml_f16_epr, 6); - - sum_02 = GGML_F16x_VEC_FMA(sum_02, ax7, ay7); - ax7 = GGML_F16x_VEC_LOAD(x[1] + i + 6*ggml_f16_epr, 6); - sum_12 = GGML_F16x_VEC_FMA(sum_12, ax7, ay7); + const svfloat16_t ay0 = GGML_F16x_VEC_LOAD(y + i, 0); + const svfloat16_t ax00 = GGML_F16x_VEC_LOAD(x[0] + i, 0); + const svfloat16_t ax01 = GGML_F16x_VEC_LOAD(x[1] + i, 0); - ay8 = GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7); + ggml_sve_f16_fma_widened(&sum_0_0_lo, &sum_0_0_hi, ax00, ay0); + ggml_sve_f16_fma_widened(&sum_1_0_lo, &sum_1_0_hi, ax01, ay0); - ax8 = GGML_F16x_VEC_LOAD(x[0] + i + 7*ggml_f16_epr, 7); + const svfloat16_t ay1 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 0); + const svfloat16_t ax10 = GGML_F16x_VEC_LOAD(x[0] + i + 1 * ggml_f16_epr, 0); + const svfloat16_t ax11 = GGML_F16x_VEC_LOAD(x[1] + i + 1 * ggml_f16_epr, 0); - sum_03 = GGML_F16x_VEC_FMA(sum_03, ax8, ay8); - ax8 = GGML_F16x_VEC_LOAD(x[1] + i + 7*ggml_f16_epr, 7); - sum_13 = GGML_F16x_VEC_FMA(sum_13, ax8, ay8); + ggml_sve_f16_fma_widened(&sum_0_1_lo, &sum_0_1_hi, ax10, ay1); + ggml_sve_f16_fma_widened(&sum_1_1_lo, &sum_1_1_hi, ax11, ay1); } - const int np2 = (n & ~(ggml_f16_epr - 1)); - for (int k = np; k < np2; k += ggml_f16_epr) { - svfloat16_t ry = GGML_F16x_VEC_LOAD(y + k, 0); + for (int i = np; i < np2; i += ggml_f16_epr) { + const svfloat16_t ry = GGML_F16x_VEC_LOAD(y + i, 0); + const svfloat16_t rx0 = GGML_F16x_VEC_LOAD(x[0] + i, 0); + const svfloat16_t rx1 = GGML_F16x_VEC_LOAD(x[1] + i, 0); - svfloat16_t rx = GGML_F16x_VEC_LOAD(x[0] + k, 0); - sum_00 = GGML_F16x_VEC_FMA(sum_00, rx, ry); - rx = GGML_F16x_VEC_LOAD(x[1] + k, 0); - sum_10 = GGML_F16x_VEC_FMA(sum_10, rx, ry); + ggml_sve_f16_fma_widened(&sum_0_0_lo, &sum_0_0_hi, rx0, ry); + ggml_sve_f16_fma_widened(&sum_1_0_lo, &sum_1_0_hi, rx1, ry); } if (np2 < n) { - svbool_t pg = svwhilelt_b16(np2, n); - svfloat16_t hx_0 = svld1_f16(pg, (const __fp16 *)(x[0] + np2)); - svfloat16_t hx_1 = svld1_f16(pg, (const __fp16 *)(x[1] + np2)); - svfloat16_t hy = svld1_f16(pg, (const __fp16 *)(y + np2)); + const svbool_t pg = svwhilelt_b16(np2, n); + const svfloat16_t ay = svld1_f16(pg, (const __fp16 *)(y + np2)); + const svfloat16_t ax0 = svld1_f16(pg, (const __fp16 *)(x[0] + np2)); + const svfloat16_t ax1 = svld1_f16(pg, (const __fp16 *)(x[1] + np2)); - sum_00 = svmad_f16_x(pg, hx_0, hy, sum_00); - sum_10 = svmad_f16_x(pg, hx_1, hy, sum_10); + ggml_sve_f16_fma_widened(&sum_0_0_lo, &sum_0_0_hi, ax0, ay); + ggml_sve_f16_fma_widened(&sum_1_0_lo, &sum_1_0_hi, ax1, ay); } - GGML_F16x_VEC_REDUCE(sumf[0], sum_00, sum_01, sum_02, sum_03); - GGML_F16x_VEC_REDUCE(sumf[1], sum_10, sum_11, sum_12, sum_13); + + svfloat32_t sum_0_lo = svadd_f32_x(DEFAULT_PG32, sum_0_0_lo, sum_0_1_lo); + svfloat32_t sum_0_hi = svadd_f32_x(DEFAULT_PG32, sum_0_0_hi, sum_0_1_hi); + svfloat32_t sum_1_lo = svadd_f32_x(DEFAULT_PG32, sum_1_0_lo, sum_1_1_lo); + svfloat32_t sum_1_hi = svadd_f32_x(DEFAULT_PG32, sum_1_0_hi, sum_1_1_hi); + sumf[0] = ggml_sve_sum_f32x2(sum_0_lo, sum_0_hi); + sumf[1] = ggml_sve_sum_f32x2(sum_1_lo, sum_1_hi); np = n; #elif defined(__riscv_v_intrinsic) #if defined(__riscv_zvfh) @@ -294,28 +275,28 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GG const int np = 0; #endif #else - const int np = (n & ~(GGML_F16_STEP - 1)); + const int np = (n & ~(GGML_F16_DOT_STEP - 1)); - GGML_F16_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } }; + GGML_F16_DOT_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_DOT_ARR] = { { GGML_F16_DOT_VEC_ZERO } }; - GGML_F16_VEC ax[GGML_F16_ARR]; - GGML_F16_VEC ay[GGML_F16_ARR]; + GGML_F16_DOT_VEC ax[GGML_F16_DOT_ARR]; + GGML_F16_DOT_VEC ay[GGML_F16_DOT_ARR]; - for (int i = 0; i < np; i += GGML_F16_STEP) { - for (int j = 0; j < GGML_F16_ARR; j++) { - ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j); + for (int i = 0; i < np; i += GGML_F16_DOT_STEP) { + for (int j = 0; j < GGML_F16_DOT_ARR; j++) { + ay[j] = GGML_F16_DOT_VEC_LOAD(y + i + j*GGML_F16_DOT_EPR, j); for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) { - ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j); + ax[j] = GGML_F16_DOT_VEC_LOAD(x[k] + i + j*GGML_F16_DOT_EPR, j); - sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]); + sum[k][j] = GGML_F16_DOT_VEC_FMA(sum[k][j], ax[j], ay[j]); } } } // reduce sum0..sum3 to sum0 for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) { - GGML_F16_VEC_REDUCE(sumf[k], sum[k]); + GGML_F16_DOT_VEC_REDUCE(sumf[k], sum[k]); } #endif #else @@ -994,6 +975,12 @@ inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp } } +inline static void ggml_vec_gelu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) { + for (int i = 0; i < n; ++i) { + y[i] = GGML_FP32_TO_BF16(ggml_gelu_f32(GGML_BF16_TO_FP32(x[i]))); + } +} + inline static void ggml_vec_gelu_erf_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { for (int i = 0; i < n; ++i) { float xi = GGML_CPU_FP16_TO_FP32(x[i]); @@ -1002,6 +989,13 @@ inline static void ggml_vec_gelu_erf_f16(const int n, ggml_fp16_t * y, const ggm } } +inline static void ggml_vec_gelu_erf_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) { + for (int i = 0; i < n; ++i) { + float xi = GGML_BF16_TO_FP32(x[i]); + y[i] = GGML_FP32_TO_BF16(0.5f*xi*(1.0f + erff(xi*SQRT_2_INV))); + } +} + #ifdef GGML_GELU_FP16 inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) { uint16_t t; @@ -1036,12 +1030,18 @@ inline static float ggml_gelu_quick_f32(float x) { return x*(1.0f/(1.0f+expf(GELU_QUICK_COEF*x))); } -//inline static void ggml_vec_gelu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { -// const uint16_t * i16 = (const uint16_t *) x; -// for (int i = 0; i < n; ++i) { -// y[i] = ggml_table_gelu_quick_f16[i16[i]]; -// } -//} +inline static void ggml_vec_gelu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { + const uint16_t * i16 = (const uint16_t *) x; + for (int i = 0; i < n; ++i) { + y[i] = ggml_table_gelu_quick_f16[i16[i]]; + } +} + +inline static void ggml_vec_gelu_quick_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) { + for (int i = 0; i < n; ++i) { + y[i] = GGML_FP32_TO_BF16(ggml_gelu_quick_f32(GGML_BF16_TO_FP32(x[i]))); + } +} #ifdef GGML_GELU_QUICK_FP16 inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) { @@ -1053,20 +1053,9 @@ inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * } } #else -inline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) { - for (int i = 0; i < n; ++i) { - y[i] = ggml_gelu_quick_f32(x[i]); - } -} +void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x); #endif -inline static void ggml_vec_gelu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { - for (int i = 0; i < n; ++i) { - float v = GGML_CPU_FP16_TO_FP32(x[i]); - y[i] = GGML_CPU_FP32_TO_FP16(v*(1.0f/(1.0f+expf(GELU_QUICK_COEF*v)))); - } -} - // Sigmoid Linear Unit (SiLU) function inline static float ggml_silu_f32(float x) { return x/(1.0f + expf(-x)); @@ -1149,6 +1138,13 @@ inline static svfloat32_t ggml_v_silu(svbool_t pg, svfloat32_t x) { return svdiv_f32_x(pg, x, one_plus_exp_neg_x); } +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static svfloat32_t ggml_v_gelu_quick(svbool_t pg, svfloat32_t x) { + const svfloat32_t one = svdup_n_f32_x(pg, 1.0f); + const svfloat32_t exp_cx = ggml_v_expf(pg, svmul_n_f32_x(pg, x, GELU_QUICK_COEF)); + return svdiv_f32_x(pg, x, svadd_f32_x(pg, one, exp_cx)); +} + #elif defined(__ARM_NEON) && defined(__aarch64__) // adapted from arm limited optimized routine @@ -1188,6 +1184,13 @@ inline static float32x4_t ggml_v_silu(float32x4_t x) { return vdivq_f32(x, one_plus_exp_neg_x); } +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static float32x4_t ggml_v_gelu_quick(float32x4_t x) { + const float32x4_t one = vdupq_n_f32(1.0f); + const float32x4_t exp_cx = ggml_v_expf(vmulq_n_f32(x, GELU_QUICK_COEF)); + return vdivq_f32(x, vaddq_f32(one, exp_cx)); +} + #elif defined(__AVX512F__) && defined(__AVX512DQ__) // adapted from arm limited optimized routine @@ -1231,6 +1234,13 @@ inline static __m512 ggml_v_silu(__m512 x) { return _mm512_div_ps(x, one_plus_exp_neg_x); } +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static __m512 ggml_v_gelu_quick(__m512 x) { + const __m512 one = _mm512_set1_ps(1); + const __m512 exp_cx = ggml_v_expf(_mm512_mul_ps(x, _mm512_set1_ps(GELU_QUICK_COEF))); + return _mm512_div_ps(x, _mm512_add_ps(one, exp_cx)); +} + #elif defined(__AVX2__) && defined(__FMA__) // adapted from arm limited optimized routine @@ -1286,6 +1296,13 @@ inline static __m256 ggml_v_silu(__m256 x) { return _mm256_div_ps(x, one_plus_exp_neg_x); } +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static __m256 ggml_v_gelu_quick(__m256 x) { + const __m256 one = _mm256_set1_ps(1); + const __m256 exp_cx = ggml_v_expf(_mm256_mul_ps(x, _mm256_set1_ps(GELU_QUICK_COEF))); + return _mm256_div_ps(x, _mm256_add_ps(one, exp_cx)); +} + #elif defined(__SSE2__) // __AVX2__ / __ARM_NEON #if defined(__FMA__) @@ -1340,6 +1357,60 @@ inline static __m128 ggml_v_silu(__m128 x) { return _mm_div_ps(x, one_plus_exp_neg_x); } +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static __m128 ggml_v_gelu_quick(__m128 x) { + const __m128 one = _mm_set1_ps(1); + const __m128 exp_cx = ggml_v_expf(_mm_mul_ps(x, _mm_set1_ps(GELU_QUICK_COEF))); + return _mm_div_ps(x, _mm_add_ps(one, exp_cx)); +} + +#elif defined(__VXE__) || defined(__VXE2__) + +// adapted from arm limited optimized routine +// the maximum error is 1.45358 plus 0.5 ulps +// numbers above 88.38 will flush to infinity +// numbers beneath -103.97 will flush to zero +inline static float32x4_t ggml_v_expf(float32x4_t v_x) { + const float32x4_t v_r = vec_splats(0x1.8p23f); + const float32x4_t v_z = vec_madd(v_x, vec_splats(0x1.715476p+0f), v_r); + const float32x4_t v_n = vec_sub(v_z, v_r); + const float32x4_t v_b = vec_nmsub(v_n, vec_splats(0x1.7f7d1cp-20f), + vec_nmsub(v_n, vec_splats(0x1.62e4p-1f), v_x)); + const uint32x4_t v_e = vec_sl((uint32x4_t)v_z, 23); + const float32x4_t v_k = (float32x4_t)vec_add(v_e, (uint32x4_t)vec_splats(1.0f)); + const uint32x4_t v_c = (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(126.0f)); + const float32x4_t v_u = vec_mul(v_b, v_b); + const float32x4_t v_j = vec_madd( + vec_madd(vec_madd(vec_splats(0x1.0e4020p-7f), v_b, vec_splats(0x1.573e2ep-5f)), v_u, + vec_madd(vec_splats(0x1.555e66p-3f), v_b, vec_splats(0x1.fffdb6p-2f))), + v_u, vec_mul(vec_splats(0x1.ffffecp-1f), v_b)); + if (vec_all_eq(v_c, vec_splats(UINT32_C(0)))) { + return vec_madd(v_j, v_k, v_k); + } + const uint32x4_t v_d = vec_and((uint32x4_t)vec_cmple(v_n, vec_splats(0.0f)), vec_splats(UINT32_C(0x82000000))); + const float32x4_t v_s1 = (float32x4_t)vec_add(v_d, vec_splats(UINT32_C(0x7f000000))); + const float32x4_t v_s2 = (float32x4_t)vec_sub(v_e, v_d); + return vec_sel(vec_sel(vec_madd(v_k, v_j, v_k), vec_mul(vec_madd(v_s2, v_j, v_s2), v_s1), v_c), + vec_mul(v_s1, v_s1), (uint32x4_t)vec_cmpgt(vec_abs(v_n), vec_splats(192.0f))); +} + +// computes silu x/(1+exp(-x)) in single precision vector +inline static float32x4_t ggml_v_silu(float32x4_t v_x) { + const float32x4_t v_one = vec_splats(1.0f); + const float32x4_t v_zero = vec_splats(0.0f); + const float32x4_t v_neg_x = vec_sub(v_zero, v_x); + const float32x4_t v_exp_neg_x = ggml_v_expf(v_neg_x); + const float32x4_t v_one_plus_exp_neg_x = vec_add(v_one, v_exp_neg_x); + return vec_div(v_x, v_one_plus_exp_neg_x); +} + +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static float32x4_t ggml_v_gelu_quick(float32x4_t x) { + const float32x4_t one = vec_splats(1.0f); + const float32x4_t exp_cx = ggml_v_expf(vec_mul(x, vec_splats(GELU_QUICK_COEF))); + return vec_div(x, vec_add(one, exp_cx)); +} + #elif defined(__riscv_v_intrinsic) // adapted from arm limited optimized routine @@ -1392,7 +1463,13 @@ inline static vfloat32m2_t ggml_v_silu_m2(vfloat32m2_t x, int vl) { return __riscv_vfdiv_vv_f32m2(x, one_plus_exp_neg_x, vl); } -#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __riscv_v_intrinsic +// gelu_quick(x) = x / (1 + exp(-1.702 x)); x stays the numerator so it cannot overflow +inline static vfloat32m2_t ggml_v_gelu_quick_m2(vfloat32m2_t x, int vl) { + const vfloat32m2_t exp_cx = ggml_v_expf_m2(__riscv_vfmul_vf_f32m2(x, GELU_QUICK_COEF, vl), vl); + return __riscv_vfdiv_vv_f32m2(x, __riscv_vfadd_vf_f32m2(exp_cx, 1.0f, vl), vl); +} + +#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v_intrinsic inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { for (int i = 0; i < n; ++i) { @@ -1400,6 +1477,12 @@ inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp } } +inline static void ggml_vec_silu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) { + for (int i = 0; i < n; ++i) { + y[i] = GGML_FP32_TO_BF16(ggml_silu_f32(GGML_BF16_TO_FP32(x[i]))); + } +} + inline static float ggml_silu_backward_f32(float x, float dy) { const float s = 1.0f/(1.0f + expf(-x)); return dy*s*(1.0f + x*(1.0f - s)); @@ -1436,6 +1519,13 @@ inline static void ggml_vec_reglu_f16 (const int n, ggml_fp16_t * y, const ggml_ } } +inline static void ggml_vec_reglu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) { + for (int i = 0; i < n; ++i) { + float v = GGML_BF16_TO_FP32(x[i]); + y[i] = GGML_FP32_TO_BF16((v > 0.f) ? v * GGML_BF16_TO_FP32(g[i]) : 0.f); + } +} + #ifdef GGML_GELU_FP16 inline static void ggml_vec_geglu_f32(const int n, float * y, const float * x, const float * g) { uint16_t t; @@ -1467,6 +1557,12 @@ inline static void ggml_vec_geglu_f16(const int n, ggml_fp16_t * y, const ggml_f } } +inline static void ggml_vec_geglu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) { + for (int i = 0; i < n; ++i) { + y[i] = GGML_FP32_TO_BF16(ggml_gelu_f32(GGML_BF16_TO_FP32(x[i])) * GGML_BF16_TO_FP32(g[i])); + } +} + void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * g); inline static void ggml_vec_swiglu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) { @@ -1477,6 +1573,14 @@ inline static void ggml_vec_swiglu_f16(const int n, ggml_fp16_t * y, const ggml_ } } +inline static void ggml_vec_swiglu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) { + for (int i = 0; i < n; ++i) { + float xi = GGML_BF16_TO_FP32(x[i]); + float gi = GGML_BF16_TO_FP32(g[i]); + y[i] = GGML_FP32_TO_BF16((xi/(1.0f + expf(-xi))) * gi); + } +} + inline static void ggml_vec_geglu_erf_f32(const int n, float * y, const float * x, const float * g) { for (int i = 0; i < n; ++i) { float xi = x[i]; @@ -1492,6 +1596,14 @@ inline static void ggml_vec_geglu_erf_f16(const int n, ggml_fp16_t * y, const gg } } +inline static void ggml_vec_geglu_erf_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) { + for (int i = 0; i < n; ++i) { + float xi = GGML_BF16_TO_FP32(x[i]); + float gi = GGML_BF16_TO_FP32(g[i]); + y[i] = GGML_FP32_TO_BF16(0.5f * xi * (1.0f + erff(xi*SQRT_2_INV)) * gi); + } +} + #ifdef GGML_GELU_QUICK_FP16 inline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) { uint16_t t; @@ -1502,11 +1614,7 @@ inline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float } } #else -inline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) { - for (int i = 0; i < n; ++i) { - y[i] = ggml_gelu_quick_f32(x[i]) * g[i]; - } -} +void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g); #endif inline static void ggml_vec_geglu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) { @@ -1517,6 +1625,12 @@ inline static void ggml_vec_geglu_quick_f16(const int n, ggml_fp16_t * y, const } } +inline static void ggml_vec_geglu_quick_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x, const ggml_bf16_t * g) { + for (int i = 0; i < n; ++i) { + y[i] = GGML_FP32_TO_BF16(ggml_gelu_quick_f32(GGML_BF16_TO_FP32(x[i])) * GGML_BF16_TO_FP32(g[i])); + } +} + inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) { #ifndef GGML_USE_ACCELERATE ggml_float sum = 0.0; diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt index b54d4a6..297d9ff 100644 --- a/ggml/src/ggml-cuda/CMakeLists.txt +++ b/ggml/src/ggml-cuda/CMakeLists.txt @@ -1,5 +1,18 @@ cmake_minimum_required(VERSION 3.18) # for CMAKE_CUDA_ARCHITECTURES +# ARM64EC uses the x64 ABI, so it needs the x64 import libraries. The arm64 ones +# hold native ARM64 symbols and cannot satisfy EC references at link time. +# FindCUDAToolkit picks the library dir from the host arch, not the target, so +# anchor its search at lib/x64 here. It derives everything else from CUDA_CUDART. +string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" GGML_CUDA_PLATFORM_LWR) +if (MSVC AND GGML_CUDA_PLATFORM_LWR STREQUAL "arm64ec") + find_library(CUDA_CUDART + NAMES cudart + HINTS ${CUDAToolkit_ROOT} ENV CUDA_PATH + PATH_SUFFIXES lib/x64 + ) +endif() + find_package(CUDAToolkit) if (CUDAToolkit_FOUND) @@ -15,6 +28,7 @@ if (CUDAToolkit_FOUND) # 80 == Ampere, asynchronous data loading, faster tensor core instructions # 86 == RTX 3000, needs CUDA v11.1 # 89 == RTX 4000, needs CUDA v11.8 + # 90 == Hopper H100/200, needs CUDA v11.8 # 120 == Blackwell, needs CUDA v12.8, FP4 tensor cores # # XX-virtual == compile CUDA code as PTX, do JIT compilation to binary code on first run @@ -33,7 +47,7 @@ if (CUDAToolkit_FOUND) list(APPEND CMAKE_CUDA_ARCHITECTURES 75-virtual 80-virtual 86-real) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "11.8") - list(APPEND CMAKE_CUDA_ARCHITECTURES 89-real) + list(APPEND CMAKE_CUDA_ARCHITECTURES 89-real 90-virtual) endif() if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.8") @@ -57,14 +71,13 @@ if (CUDAToolkit_FOUND) enable_language(CUDA) - # TODO: Remove once CCCL 3.2 has been released and bundled with CUDA Toolkit - if (GGML_CUDA_CUB_3DOT2) + if (GGML_CUDA_CCCL_VERSION) include(FetchContent) FetchContent_Declare( CCCL GIT_REPOSITORY https://github.com/nvidia/cccl.git - GIT_TAG v3.2.0 + GIT_TAG "${GGML_CUDA_CCCL_VERSION}" GIT_SHALLOW TRUE ) @@ -111,25 +124,14 @@ if (CUDAToolkit_FOUND) file(GLOB SRCS "template-instances/mmf*.cu") list(APPEND GGML_SOURCES_CUDA ${SRCS}) - if (GGML_CUDA_FA_ALL_QUANTS) - file(GLOB SRCS "template-instances/fattn-vec*.cu") - list(APPEND GGML_SOURCES_CUDA ${SRCS}) - add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS) - else() - list(APPEND GGML_SOURCES_CUDA - template-instances/fattn-vec-instance-f16-f16.cu - template-instances/fattn-vec-instance-q4_0-q4_0.cu - template-instances/fattn-vec-instance-q8_0-q8_0.cu - template-instances/fattn-vec-instance-bf16-bf16.cu) - endif() + ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR} SRCS) + list(APPEND GGML_SOURCES_CUDA ${SRCS}) ggml_add_backend_library(ggml-cuda ${GGML_HEADERS_CUDA} ${GGML_SOURCES_CUDA} ) - add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE}) - if (GGML_CUDA_GRAPHS) add_compile_definitions(GGML_CUDA_USE_GRAPHS) endif() @@ -154,14 +156,15 @@ if (CUDAToolkit_FOUND) add_compile_definitions(GGML_CUDA_NO_PEER_COPY) endif() + if (GGML_CUDA_CCCL_VERSION) + target_link_libraries(ggml-cuda PRIVATE CCCL::CCCL) + endif() + if (GGML_STATIC) if (WIN32) # As of 12.3.1 CUDA Toolkit for Windows does not offer a static cublas library target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas) else () - if (GGML_CUDA_CUB_3DOT2) - target_link_libraries(ggml-cuda PRIVATE CCCL::CCCL) - endif() if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "10.1") target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static) else() @@ -169,9 +172,6 @@ if (CUDAToolkit_FOUND) endif() endif() else() - if (GGML_CUDA_CUB_3DOT2) - target_link_libraries(ggml-cuda PRIVATE CCCL::CCCL) - endif() target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas) endif() diff --git a/ggml/src/ggml-cuda/allreduce.cu b/ggml/src/ggml-cuda/allreduce.cu new file mode 100644 index 0000000..39b23be --- /dev/null +++ b/ggml/src/ggml-cuda/allreduce.cu @@ -0,0 +1,977 @@ +#include "allreduce.cuh" + +#if !defined(GGML_USE_MUSA) + +#include "convert.cuh" +#include "ggml-impl.h" + +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// AllReduce for tensor-parallel inference across two GPUs (CUDA or +// ROCm/HIP). +// +// Provides an in-place sum reduction over matching tensors on two GPUs +// in the same process. Used by the tensor-split path alongside NCCL; +// targets setups without NVLink/xGMI, where data is exchanged between the +// GPUs by staging it through pinned host memory over PCIe. +// +// Two reduction strategies are selected per call by tensor size: +// +// * Chunked kernel path (small reductions): a single CUDA kernel both +// stages data through pinned host memory and performs the local sum. +// Cross-GPU synchronization happens *inside the kernel* (busy-wait on +// a host-memory flag), which keeps launch overhead low for the +// latency-sensitive token-generation case. +// +// * Copy-engine path (large reductions): the transfer is split into +// D2H + H2D cudaMemcpyAsync chunks driven by the GPU's copy engine, +// followed by a small device-side add kernel. Cross-GPU +// synchronization happens *outside the kernel*, via CUDA events +// between streams. This keeps the compute engine free while large +// transfers are in flight, which matters for prefill-sized tensors. +// Reductions larger than the per-call inner cap are processed by an +// outer chunker that issues sequential inner calls. +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Cross-GPU signal mechanism +// +// One int per (slot, rank) pair in pinned host memory. Each AR call writes a +// strictly increasing token (= the AR call number) into its own arrival int. +// The peer spins until its read of the other's arrival int equals the token +// it expects for this call -- a mismatch means the peer hasn't arrived yet. +// Tokens never repeat over realistic call rates (32-bit int wraps in tens of +// days at thousands of ARs/sec), so arrival ints don't need to be reset +// between calls; we initialize once at pipeline init and let the values +// accumulate. +// +// There is exactly one writer (the owning GPU) and one reader (the peer), so +// we don't need atomics. A volatile store paired with __threadfence_system() +// provides the release ordering that makes the D2H writes visible system-wide +// before the arrival token is observed. +// +// atomicAdd_system() requires hostNativeAtomicSupported, which is unavailable +// on PCIe-attached consumer GPUs without NVLink, so the volatile path is the +// portable choice. +// --------------------------------------------------------------------------- + +static __device__ __forceinline__ void ggml_cuda_ar_signal_set(int * p, int token) { + *(volatile int *)p = token; +} +static __device__ __forceinline__ int ggml_cuda_ar_signal_get(const int * p) { + return *(const volatile int *)p; +} + +// Byte spacing between adjacent arrival ints. 64 bytes (one cache line) +// ensures each GPU/block's arrival slot lives on its own line, preventing +// false-sharing stalls on the polling GPU. +static constexpr size_t GGML_CUDA_AR_ARRIVAL_STRIDE = 64; + +// Number of blocks the chunked kernel launches with. Each block stripes a +// disjoint slice of the data and synchronizes through its own arrival-token +// slot so multiple SMs can pump PCIe stores in parallel. +static constexpr int GGML_CUDA_AR_KERNEL_BLOCKS = 8; + +// --------------------------------------------------------------------------- +// Chunked kernel AllReduce -- 2 GPUs, supports float, half, and bfloat16. +// +// Both GPUs run this kernel simultaneously on independent streams. sendbuf +// and recvbuf live in T_dst (the caller's tensor type); host_mine / host_other +// carry data in T_wire (the on-wire type, possibly narrower than T_dst -- e.g. +// T_dst=F32 with T_wire=BF16 halves the bytes pushed across PCIe). When +// T_dst == T_wire the casts below are no-ops. +// +// Each GPU runs three phases: +// +// Phase 1 (all threads): cast sendbuf (T_dst) -> T_wire and store as +// single-instruction-width vectors into host_mine. +// __threadfence_system() commits these writes to host +// memory. +// Phase 2 (thread 0): write token to arrival_mine; spin until +// arrival_other == token. +// Phase 3 (all threads): read T_wire vectors from host_other, cast +// each element to T_dst, and sum with the local +// sendbuf value (also rounded through T_wire so that +// both GPUs truncate identically -- this guarantees +// bit-equivalent results across the two devices). +// +// Multi-block: blocks stripe vectors across (gridDim.x * blockDim.x) global +// threads to keep multiple SMs issuing PCIe stores in parallel. Each block +// has its own arrival-token slot (offset by blockIdx.x * ARRIVAL_STRIDE); +// thread 0 of each block signals/spins on that slot independently of other +// blocks. Tail elements (the leftover < ELEMS_PER_VEC at the end) are +// handled only by block 0 to avoid cross-block writes to the same slots. +// --------------------------------------------------------------------------- +template +static __global__ void ggml_cuda_ar_kernel( + const T_dst * sendbuf, + T_dst * recvbuf, + T_wire * __restrict__ host_mine, + const T_wire * __restrict__ host_other, + int count, + int * arrival_mine, + int * arrival_other, + int token) { + + // Vector unit for the wire type, sized to the arch's widest single-instruction + // copy (16 B on Volta+). Each phase-1 iter writes one vector to host memory; + // each phase-3 iter reads one and produces ELEMS_PER_VEC sums. + constexpr int ELEMS_PER_VEC = ggml_cuda_get_max_cpy_bytes() / sizeof(T_wire); + constexpr int ARRIVAL_INTS = (int)(GGML_CUDA_AR_ARRIVAL_STRIDE / sizeof(int)); + + const int tid = threadIdx.x; + const int nt = blockDim.x; + const int bid = blockIdx.x; + const int gtid = bid * nt + tid; + const int gnt = gridDim.x * nt; + const int count_vec = count / ELEMS_PER_VEC; + const int tail = count_vec * ELEMS_PER_VEC; + + // Phase 1: cast sendbuf (T_dst) -> host_mine (T_wire) and store as vectors. + { + for (int i = gtid; i < count_vec; i += gnt) { + const int off = i * ELEMS_PER_VEC; + T_wire wire[ELEMS_PER_VEC]; + #pragma unroll + for (int k = 0; k < ELEMS_PER_VEC; ++k) { + wire[k] = ggml_cuda_cast(sendbuf[off + k]); + } + ggml_cuda_memcpy_1(&host_mine[off], wire); + } + if (bid == 0 && tid < count - tail) { + host_mine[tail + tid] = ggml_cuda_cast(sendbuf[tail + tid]); + } + } + + // Commit this block's host writes before signalling. + __threadfence_system(); + __syncthreads(); + + // Phase 2: thread 0 of each block signals on its own arrival slot, then + // spins for the matching slot from peer. Per-block tokens mean blocks + // proceed independently -- no inter-block barrier needed. + if (tid == 0) { + int * my_slot = arrival_mine + bid * ARRIVAL_INTS; + const int * other_slot = arrival_other + bid * ARRIVAL_INTS; + + ggml_cuda_ar_signal_set(my_slot, token); + __threadfence_system(); // make our signal visible system-wide + + while (ggml_cuda_ar_signal_get(other_slot) != token) { +#ifdef GGML_USE_HIP + // Equals ~100ns at 2500 MHz (sleeps for n * [1,64] clock cycles) + __builtin_amdgcn_s_sleep(4); +#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA + __nanosleep(100); +#else + NO_DEVICE_CODE; +#endif // GGML_USE_HIP + } + } + + __syncthreads(); + + // Acquire peer's host_other writes (this block's stripe of them). + __threadfence_system(); + + // Phase 3: read peer's T_wire vector, cast both sides through T_wire for + // bit-equivalence, sum in T_dst precision, and write back to recvbuf. + { + for (int i = gtid; i < count_vec; i += gnt) { + const int off = i * ELEMS_PER_VEC; + T_wire wire[ELEMS_PER_VEC]; + ggml_cuda_memcpy_1(wire, &host_other[off]); + #pragma unroll + for (int k = 0; k < ELEMS_PER_VEC; ++k) { + const T_wire d_low = ggml_cuda_cast(sendbuf[off + k]); + recvbuf[off + k] = ggml_cuda_cast( + ggml_cuda_cast(d_low) + ggml_cuda_cast(wire[k])); + } + } + if (bid == 0 && tid < count - tail) { + const T_wire d_low = ggml_cuda_cast(sendbuf[tail + tid]); + recvbuf[tail + tid] = ggml_cuda_cast( + ggml_cuda_cast(d_low) + + ggml_cuda_cast(host_other[tail + tid])); + } + } +} + +// Combined load-convert-add kernel. The peer's contribution arrives as T_src +// (which may be a lower-precision type than T_dst when the BF16 round-trip is +// active). For bit-equivalence between the two GPUs, dst is first rounded +// through T_src's precision via ggml_cuda_cast -- peer already truncated its +// own value the same way before sending -- so both sides perform identical +// arithmetic. When T_dst == T_src the round-trip cast is a no-op. +template +static __global__ void ggml_cuda_ar_add_kernel( + T_dst * __restrict__ dst, + const T_src * __restrict__ src, + int count) { + const int tid = blockIdx.x * blockDim.x + threadIdx.x; + const int nt = gridDim.x * blockDim.x; + for (int i = tid; i < count; i += nt) { + const T_src d_low = ggml_cuda_cast(dst[i]); + dst[i] = ggml_cuda_cast( + ggml_cuda_cast(d_low) + ggml_cuda_cast(src[i])); + } +} + +// --------------------------------------------------------------------------- +// Pipeline structure +// --------------------------------------------------------------------------- + +// Number of slots in the event / arrival ring. Two slots is sufficient: +// lockstep guarantees the two GPUs are at most one AR (or chunk) apart, so +// slot[N%2] is always safe to reuse -- peer has already consumed slot[N%2] +// from AR N-2 by the time we get to AR N. acquire_slot's +// cudaEventSynchronize on ev.ker for both devices makes that consumption +// explicit before we overwrite host_buf[slot] for the new AR. +static constexpr int GGML_CUDA_AR_POOL_SIZE = 2; + +// Maximum chunk size (bytes per GPU) handled by one chunked kernel launch. +// Larger tensors are reduced by issuing multiple chunked launches. +static constexpr size_t GGML_CUDA_AR_MAX_BYTES = 1024 * 1024; // 1 MB + +// Copy-engine path: largest tensor accepted on this path; sets host_large / +// dev_tmp allocation size. +static constexpr size_t GGML_CUDA_AR_COPY_MAX_BYTES = 32 * 1024 * 1024; // 32 MB + +// AR wire size at which the copy-engine path takes over from the chunked- +// kernel path. Override via GGML_CUDA_AR_COPY_THRESHOLD. +static constexpr size_t GGML_CUDA_AR_COPY_THRESHOLD_DEFAULT = 1024 * 1024; // 1 MB +// Per-call CE chunk-size heuristic: chunk_bytes = clamp(nbytes / 4, MIN, MAX). +// The /4 keeps ~4 chunks in flight at any moment (good D2H/H2D overlap with +// the peer); the clamps cover the cases where nbytes/4 is too small (per- +// memcpy fixed cost dominates) or too large (chunk-level pipelining stalls). +// Env var GGML_CUDA_AR_COPY_CHUNK_BYTES can override with a fixed value. +static constexpr size_t GGML_CUDA_AR_COPY_CHUNK_BYTES_HEURISTIC_MIN = 512 * 1024; // 512 KB +static constexpr size_t GGML_CUDA_AR_COPY_CHUNK_BYTES_HEURISTIC_MAX = 2 * 1024 * 1024; // 2 MB +// Absolute floor that an env-var override is allowed to set; this caps the +// per-slot copy-event array. 256 KB -> up to 128 chunks per 32 MB tensor. +static constexpr size_t GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN = 256 * 1024; +static constexpr int GGML_CUDA_AR_COPY_MAX_CHUNKS = + static_cast((GGML_CUDA_AR_COPY_MAX_BYTES + GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN - 1) / + GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN); + +struct ggml_cuda_ar_event_slot { + cudaEvent_t app = nullptr; // upstream computation complete + cudaEvent_t cpy[GGML_CUDA_AR_COPY_MAX_CHUNKS] = {}; // copy-engine D2H chunks complete + cudaEvent_t h2d = nullptr; // copy-engine H2Ds complete (handoff AR stream -> compute stream) + cudaEvent_t ker = nullptr; // AllReduce kernel complete +}; + +// Mapped pinned host allocation: cudaHostAlloc + cudaHostGetDevicePointer +// in one place, with the host handle preserved for cudaFreeHost. Used where +// the CPU never touches the buffer -- only the device reads/writes via the +// mapped device pointer. Required on systems where cudaDevAttrCanUseHost- +// PointerForRegisteredMem is 0 and the host pointer can't be used as a +// device pointer. +struct ggml_cuda_ar_host_mapping { + uint8_t * host = nullptr; // cudaFreeHost handle; also the H-side ptr for cudaMemcpyAsync + uint8_t * dev = nullptr; // device-side pointer for kernels / cudaMemset + + cudaError_t alloc(size_t bytes) { + cudaError_t rc = cudaHostAlloc(reinterpret_cast(&host), bytes, + cudaHostAllocPortable | cudaHostAllocMapped); + if (rc != cudaSuccess) { + host = nullptr; + return rc; + } + rc = cudaHostGetDevicePointer(reinterpret_cast(&dev), host, 0); + if (rc != cudaSuccess) { + CUDA_CHECK(cudaFreeHost(host)); + host = nullptr; + dev = nullptr; + } + return rc; + } + + void free() { + if (host) { + CUDA_CHECK(cudaFreeHost(host)); + host = nullptr; + dev = nullptr; + } + } +}; + +struct ggml_cuda_ar_pipeline { + int n_devices; + int devices[GGML_CUDA_MAX_DEVICES]; + size_t buf_bytes; // bytes per device in host_buf[] + size_t copy_bytes; // bytes per device in host_large[] / dev_tmp[] + size_t copy_threshold; + size_t copy_chunk_bytes; + size_t bf16_threshold; // tensors >= this size (bytes) are reduced via FP32->BF16 round-trip; 0 disables + uint64_t call_count; + + // Per-device resources. + ggml_cuda_ar_host_mapping host_buf[GGML_CUDA_MAX_DEVICES]; // pinned staging (chunked kernel) + ggml_cuda_ar_host_mapping host_large[GGML_CUDA_MAX_DEVICES]; // pinned staging (copy-engine) + char * dev_tmp[GGML_CUDA_MAX_DEVICES]; // device scratch for copy-engine path + cudaStream_t streams[GGML_CUDA_MAX_DEVICES]; // non-blocking + ggml_cuda_ar_event_slot ev_pool[GGML_CUDA_MAX_DEVICES][GGML_CUDA_AR_POOL_SIZE]; + + // Copy-engine: per-device "I finished reading my peer's host_large" + // event. Indexed by RECORDER device. Recorded same-device on streams[i] + // after stage 2's last H2D from host_large[peer]. Waited cross-device + // by peer's stage-1 stream before the next AR overwrites host_large[peer]. + cudaEvent_t host_large_read_done[GGML_CUDA_MAX_DEVICES]; + bool host_large_read_done_valid; + + // Copy-engine: per-device "my add_kernel is done with dev_tmp" event. + // Recorded on the compute stream after each add_kernel; the AR stream + // waits on it before the next copy_impl's H2D overwrites dev_tmp. Lets us + // single-buffer dev_tmp despite add_kernel running on a separate stream. + cudaEvent_t dev_tmp_kernel_done[GGML_CUDA_MAX_DEVICES]; + bool dev_tmp_kernel_done_valid; + + // Arrival ring: ARRIVAL_STRIDE bytes between adjacent ints. Mapped pinned + // memory; CPU never reads/writes -- only the kernel and cudaMemset. + // Use ggml_cuda_ar_arrival_ptr() to index. + ggml_cuda_ar_host_mapping arrival; +}; + +// Base pointer for the (slot, rank) per-block token block. The kernel adds +// blockIdx.x * (ARRIVAL_STRIDE/sizeof(int)) internally to land on its own slot. +static int * ggml_cuda_ar_arrival_ptr(const ggml_cuda_ar_pipeline * p, int slot, int rank) { + const size_t offset = ((size_t)slot * p->n_devices + rank) * + GGML_CUDA_AR_KERNEL_BLOCKS * GGML_CUDA_AR_ARRIVAL_STRIDE; + return reinterpret_cast(p->arrival.dev + offset); +} + +static uint64_t ggml_cuda_ar_env_u64(const char * name, uint64_t default_value) { + const char * value = getenv(name); + if (value == nullptr || value[0] == '\0') { + return default_value; + } + + char * end = nullptr; + const unsigned long long parsed = strtoull(value, &end, 10); + return end != value ? (uint64_t) parsed : default_value; +} + +struct ggml_cuda_ar_slot_info { + int slot; + int token; +}; + +static ggml_cuda_ar_slot_info ggml_cuda_ar_acquire_slot(ggml_cuda_ar_pipeline * p) { + const int slot = static_cast(p->call_count % GGML_CUDA_AR_POOL_SIZE); + const bool pool_lapped = p->call_count >= GGML_CUDA_AR_POOL_SIZE; + p->call_count++; + + if (pool_lapped) { + for (int i = 0; i < p->n_devices; ++i) { + ggml_cuda_set_device(p->devices[i]); + CUDA_CHECK(cudaEventSynchronize(p->ev_pool[i][slot].ker)); + } + } + + return { slot, (int) p->call_count }; +} + +// Per-AR copy-engine chunk size: env-var override if set, else heuristic +// (clamp(nbytes/4, HEURISTIC_MIN, HEURISTIC_MAX)). +static size_t ggml_cuda_ar_chunk_bytes(const ggml_cuda_ar_pipeline * p, size_t nbytes) { + if (p->copy_chunk_bytes > 0) { + return p->copy_chunk_bytes; + } + return std::min(GGML_CUDA_AR_COPY_CHUNK_BYTES_HEURISTIC_MAX, + std::max(GGML_CUDA_AR_COPY_CHUNK_BYTES_HEURISTIC_MIN, nbytes / 4)); +} + +static void ggml_cuda_ar_wait_for_compute( + ggml_cuda_ar_pipeline * p, ggml_backend_cuda_context * cuda_ctx, int rank, int slot) { + ggml_cuda_ar_event_slot & ev = p->ev_pool[rank][slot]; + CUDA_CHECK(cudaEventRecord(ev.app, cuda_ctx->stream())); + CUDA_CHECK(cudaStreamWaitEvent(p->streams[rank], ev.app)); +} + +// --------------------------------------------------------------------------- +// Init / free +// --------------------------------------------------------------------------- + +ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(const int * devices, size_t n_devices) { + + if (n_devices != 2) { + GGML_LOG_DEBUG("%s: internal AllReduce only supports n_devices=2 (got %zu); " + "falling back\n", __func__, n_devices); + return nullptr; + } + + // The chunked kernel uses __nanosleep (NVIDIA, sm70+) or + // __builtin_amdgcn_s_sleep (AMD). + for (size_t i = 0; i < n_devices; ++i) { + const int cc = ggml_cuda_info().devices[devices[i]].cc; + if (cc < GGML_CUDA_CC_VOLTA) { + GGML_LOG_DEBUG("%s: internal AllReduce requires compute capability >= %d " + "(device %d has cc=%d); falling back\n", + __func__, GGML_CUDA_CC_VOLTA, devices[i], cc); + return nullptr; + } + } + + auto * p = new ggml_cuda_ar_pipeline{}; + p->n_devices = n_devices; + p->copy_bytes = GGML_CUDA_AR_COPY_MAX_BYTES; + p->copy_threshold = ggml_cuda_ar_env_u64("GGML_CUDA_AR_COPY_THRESHOLD", GGML_CUDA_AR_COPY_THRESHOLD_DEFAULT); + // 0 = use the per-call heuristic (default). Non-zero env value forces a + // fixed chunk size for diagnostics, with a floor at COPY_CHUNK_BYTES_MIN. + p->copy_chunk_bytes = ggml_cuda_ar_env_u64("GGML_CUDA_AR_COPY_CHUNK_BYTES", 0); + if (p->copy_chunk_bytes > 0 && p->copy_chunk_bytes < GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN) { + GGML_LOG_WARN("%s: GGML_CUDA_AR_COPY_CHUNK_BYTES=%zu below minimum %zu; clamping\n", + __func__, p->copy_chunk_bytes, GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN); + p->copy_chunk_bytes = GGML_CUDA_AR_COPY_CHUNK_BYTES_MIN; + } + // Default 1: BF16 round-trip is always on for F32 inputs (any non-zero + // ne). Set GGML_CUDA_AR_BF16_THRESHOLD=0 to disable, or to a larger + // byte threshold to opt out for small tensors. + p->bf16_threshold = ggml_cuda_ar_env_u64("GGML_CUDA_AR_BF16_THRESHOLD", 1); + for (size_t i = 0; i < n_devices; ++i) { + p->devices[i] = devices[i]; + } + + // Per-device streams and event pools. + for (size_t i = 0; i < n_devices; ++i) { + ggml_cuda_set_device(p->devices[i]); + + cudaStream_t stream = nullptr; + if (cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking) != cudaSuccess) { + GGML_LOG_ERROR("%s: cudaStreamCreateWithFlags failed for device %d\n", + __func__, p->devices[i]); + ggml_cuda_ar_pipeline_free(p); + return nullptr; + } + p->streams[i] = stream; + + for (int s = 0; s < GGML_CUDA_AR_POOL_SIZE; ++s) { + bool ok = + cudaEventCreateWithFlags(&p->ev_pool[i][s].app, cudaEventDisableTiming) == cudaSuccess && + cudaEventCreateWithFlags(&p->ev_pool[i][s].h2d, cudaEventDisableTiming) == cudaSuccess && + cudaEventCreateWithFlags(&p->ev_pool[i][s].ker, cudaEventDisableTiming) == cudaSuccess; + for (int c = 0; ok && c < GGML_CUDA_AR_COPY_MAX_CHUNKS; ++c) { + ok = cudaEventCreateWithFlags(&p->ev_pool[i][s].cpy[c], cudaEventDisableTiming) == cudaSuccess; + } + if (!ok) { + GGML_LOG_ERROR("%s: cudaEventCreate failed for device %d slot %d\n", + __func__, p->devices[i], s); + ggml_cuda_ar_pipeline_free(p); + return nullptr; + } + } + + if (cudaEventCreateWithFlags(&p->host_large_read_done[i], cudaEventDisableTiming) != cudaSuccess) { + GGML_LOG_ERROR("%s: cudaEventCreate for host_large_read_done failed for device %d\n", + __func__, p->devices[i]); + ggml_cuda_ar_pipeline_free(p); + return nullptr; + } + if (cudaEventCreateWithFlags(&p->dev_tmp_kernel_done[i], cudaEventDisableTiming) != cudaSuccess) { + GGML_LOG_ERROR("%s: cudaEventCreate for dev_tmp_kernel_done failed for device %d\n", + __func__, p->devices[i]); + ggml_cuda_ar_pipeline_free(p); + return nullptr; + } + } + + // Arrival ring: cache-line padded so each GPU's int is on its own line. + const size_t arrival_bytes = + (size_t)GGML_CUDA_AR_POOL_SIZE * n_devices * + GGML_CUDA_AR_KERNEL_BLOCKS * GGML_CUDA_AR_ARRIVAL_STRIDE; + if (p->arrival.alloc(arrival_bytes) != cudaSuccess) { + GGML_LOG_ERROR("%s: alloc for arrival ring failed (%zu bytes)\n", + __func__, arrival_bytes); + ggml_cuda_ar_pipeline_free(p); + return nullptr; + } + ggml_cuda_set_device(p->devices[0]); + if (cudaMemset(p->arrival.dev, 0, arrival_bytes) != cudaSuccess) { + GGML_LOG_ERROR("%s: cudaMemset for arrival ring failed (%zu bytes)\n", + __func__, arrival_bytes); + ggml_cuda_ar_pipeline_free(p); + return nullptr; + } + + // Per-device pinned staging buffers -- POOL_SIZE-deep ring so the chunked- + // kernel can write the next slot's data while the peer is still reading + // the previous slot's. Indexed by (slot * buf_bytes) at the call site. + p->buf_bytes = GGML_CUDA_AR_MAX_BYTES; + const size_t host_buf_total = (size_t) GGML_CUDA_AR_POOL_SIZE * p->buf_bytes; + for (size_t i = 0; i < n_devices; ++i) { + if (p->host_buf[i].alloc(host_buf_total) != cudaSuccess) { + GGML_LOG_ERROR("%s: alloc for staging failed (%zu bytes)\n", + __func__, host_buf_total); + ggml_cuda_ar_pipeline_free(p); + return nullptr; + } + } + + // Copy-engine path: pinned host staging + device scratch, sized for the + // largest tensor we accept on this path (GGML_CUDA_AR_COPY_MAX_BYTES). + // dev_tmp is single-buffered; cross-AR safety is enforced by an explicit + // cross-stream wait in copy_impl on the prior AR's add_kernel-done event. + for (size_t i = 0; i < n_devices; ++i) { + ggml_cuda_set_device(p->devices[i]); + if (p->host_large[i].alloc(p->copy_bytes) != cudaSuccess) { + GGML_LOG_ERROR("%s: alloc for large staging failed (%zu bytes)\n", + __func__, p->copy_bytes); + ggml_cuda_ar_pipeline_free(p); + return nullptr; + } + if (cudaMalloc(reinterpret_cast(&p->dev_tmp[i]), p->copy_bytes) != cudaSuccess) { + GGML_LOG_ERROR("%s: cudaMalloc for copy scratch failed (%zu bytes) on device %d\n", + __func__, p->copy_bytes, p->devices[i]); + ggml_cuda_ar_pipeline_free(p); + return nullptr; + } + } + + GGML_LOG_INFO("%s: initialized AllReduce pipeline: %zu GPUs, " + "%zu KB chunked kernel staging + %zu MB copy-engine staging per GPU\n", + __func__, n_devices, p->buf_bytes >> 10, p->copy_bytes >> 20); + + return p; +} + +void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline * p) { + if (!p) { + return; + } + + // Drain all in-flight kernels before tearing down resources. + for (int i = 0; i < p->n_devices; ++i) { + if (p->streams[i]) { + ggml_cuda_set_device(p->devices[i]); + CUDA_CHECK(cudaStreamSynchronize(p->streams[i])); + } + } + + for (int i = 0; i < p->n_devices; ++i) { + p->host_buf[i].free(); + p->host_large[i].free(); + if (p->dev_tmp[i]) { + ggml_cuda_set_device(p->devices[i]); + CUDA_CHECK(cudaFree(p->dev_tmp[i])); + } + ggml_cuda_set_device(p->devices[i]); + for (int s = 0; s < GGML_CUDA_AR_POOL_SIZE; ++s) { + if (p->ev_pool[i][s].app) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].app)); } + for (int c = 0; c < GGML_CUDA_AR_COPY_MAX_CHUNKS; ++c) { + if (p->ev_pool[i][s].cpy[c]) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].cpy[c])); } + } + if (p->ev_pool[i][s].h2d) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].h2d)); } + if (p->ev_pool[i][s].ker) { CUDA_CHECK(cudaEventDestroy(p->ev_pool[i][s].ker)); } + } + if (p->host_large_read_done[i]) { + ggml_cuda_set_device(p->devices[i]); + CUDA_CHECK(cudaEventDestroy(p->host_large_read_done[i])); + } + if (p->dev_tmp_kernel_done[i]) { + ggml_cuda_set_device(p->devices[i]); + CUDA_CHECK(cudaEventDestroy(p->dev_tmp_kernel_done[i])); + } + if (p->streams[i]) { + ggml_cuda_set_device(p->devices[i]); + CUDA_CHECK(cudaStreamDestroy(p->streams[i])); + } + } + p->arrival.free(); + delete p; +} + +// --------------------------------------------------------------------------- +// Dispatch +// --------------------------------------------------------------------------- + +// Asymmetric copy_impl: data sent over PCIe in T_src precision (one element of +// nbytes per ne element); accumulated locally into a T_dst buffer. When +// T_src == T_dst this is the original homogeneous reduction. When they differ +// (e.g. BF16 wire / F32 accumulator) the add kernel rounds dst through T_src +// for bit-equivalence between GPUs and we skip the otherwise-needed +// post-conversion entirely. +template +static bool ggml_cuda_ar_allreduce_copy_impl( + ggml_cuda_ar_pipeline * p, + ggml_backend_t * backends, + T_src * const src_buf[GGML_CUDA_MAX_DEVICES], + T_dst * const dst_buf[GGML_CUDA_MAX_DEVICES], + const bool compute[GGML_CUDA_MAX_DEVICES], + int64_t ne, + size_t nbytes) { + GGML_ASSERT(p->n_devices == 2); + GGML_ASSERT(nbytes <= p->copy_bytes); + GGML_ASSERT(ne <= std::numeric_limits::max()); + + const size_t chunk_bytes = ggml_cuda_ar_chunk_bytes(p, nbytes); + GGML_ASSERT(chunk_bytes > 0); + + const int slot = ggml_cuda_ar_acquire_slot(p).slot; + const size_t copy_chunks = (nbytes + chunk_bytes - 1) / chunk_bytes; + GGML_ASSERT(copy_chunks <= GGML_CUDA_AR_COPY_MAX_CHUNKS); + + ggml_backend_cuda_context * cuda_ctx[2] = {}; + + // Stage 1: both GPUs copy their local contribution to pinned host memory. + for (int i = 0; i < 2; ++i) { + ggml_cuda_set_device(p->devices[i]); + cuda_ctx[i] = static_cast(backends[i]->context); + GGML_ASSERT(cuda_ctx[i]->device == p->devices[i]); + + ggml_cuda_ar_wait_for_compute(p, cuda_ctx[i], i, slot); + + // Wait for peer's H2D from our host_large[i] (recorded in the + // previous AR's stage 2) to complete before we overwrite host_large[i]. + // host_large_read_done[peer] = peer finished reading host_large[i]. + // No-op on the first AR -- no prior record exists. + if (p->host_large_read_done_valid) { + const int peer = 1 - i; + CUDA_CHECK(cudaStreamWaitEvent(p->streams[i], p->host_large_read_done[peer])); + } + + if (!compute[i]) { + CUDA_CHECK(cudaMemsetAsync(src_buf[i], 0, nbytes, p->streams[i])); + } + + for (size_t c = 0; c < copy_chunks; ++c) { + const size_t offset = c * chunk_bytes; + const size_t this_bytes = (nbytes - offset) < chunk_bytes ? + (nbytes - offset) : chunk_bytes; + + CUDA_CHECK(cudaMemcpyAsync( + p->host_large[i].host + offset, reinterpret_cast(src_buf[i]) + offset, this_bytes, + cudaMemcpyDeviceToHost, p->streams[i])); + CUDA_CHECK(cudaEventRecord(p->ev_pool[i][slot].cpy[c], p->streams[i])); + } + } + + // Stage 2: each GPU waits for each peer D2H chunk, pulls that chunk back to + // local device scratch (dev_tmp), then performs one device-local add over + // the assembled peer tensor. The H2Ds run on the AR stream (copy engine) + // and the add_kernel runs on the caller's compute stream, so the AR stream + // stays pure-copy and avoids an in-stream copy->compute engine switch every + // AR. dev_tmp is single-buffered: the AR stream waits cross-stream on the + // prior AR's add_kernel-done event before overwriting it. + for (int i = 0; i < 2; ++i) { + const int peer = 1 - i; + ggml_cuda_set_device(p->devices[i]); + + // Wait for the previous AR's add_kernel (on the compute stream) to + // finish reading dev_tmp before our H2D overwrites it. No-op on the + // first copy_impl call. + if (p->dev_tmp_kernel_done_valid) { + CUDA_CHECK(cudaStreamWaitEvent(p->streams[i], p->dev_tmp_kernel_done[i])); + } + + for (size_t c = 0; c < copy_chunks; ++c) { + const size_t offset = c * chunk_bytes; + const size_t this_bytes = (nbytes - offset) < chunk_bytes ? + (nbytes - offset) : chunk_bytes; + + CUDA_CHECK(cudaStreamWaitEvent(p->streams[i], p->ev_pool[peer][slot].cpy[c])); + CUDA_CHECK(cudaMemcpyAsync( + p->dev_tmp[i] + offset, p->host_large[peer].host + offset, this_bytes, + cudaMemcpyHostToDevice, p->streams[i])); + } + + // Mark our reads of host_large[peer] complete so peer's next AR can + // safely overwrite it. + CUDA_CHECK(cudaEventRecord(p->host_large_read_done[i], p->streams[i])); + + // Hand off from AR stream (copy engine) to compute stream: compute + // stream waits for all H2Ds to finish, then runs the add_kernel. + CUDA_CHECK(cudaEventRecord(p->ev_pool[i][slot].h2d, p->streams[i])); + CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx[i]->stream(), p->ev_pool[i][slot].h2d)); + + const int block_size = 256; + int n_blocks = (int) ((ne + block_size - 1) / block_size); + if (n_blocks > 1024) { + n_blocks = 1024; + } + ggml_cuda_ar_add_kernel<<stream()>>>( + dst_buf[i], + reinterpret_cast(p->dev_tmp[i]), + (int) ne); + CUDA_CHECK(cudaGetLastError()); + + // Record dev_tmp-released on the compute stream so the next copy_impl + // can wait for the kernel to finish before overwriting dev_tmp. Also + // record AR-done as ev.ker for acquire_slot's pool-wraparound sync. + CUDA_CHECK(cudaEventRecord(p->dev_tmp_kernel_done[i], cuda_ctx[i]->stream())); + CUDA_CHECK(cudaEventRecord(p->ev_pool[i][slot].ker, cuda_ctx[i]->stream())); + } + p->host_large_read_done_valid = true; + p->dev_tmp_kernel_done_valid = true; + + return true; +} + +// Outer-level chunker: copy_impl handles up to copy_bytes per call (limited by +// the host_large / dev_tmp allocation size). When the full AR exceeds that, +// slice the tensor into copy_bytes-sized pieces and call copy_impl repeatedly. +// Each slice goes through its own stage 1 -> stage 2 cycle and acquires its own +// slot, so cross-AR fences and pool wraparound work the same way as for any +// other sequence of small ARs. +template +static bool ggml_cuda_ar_allreduce_copy_outer( + ggml_cuda_ar_pipeline * p, + ggml_backend_t * backends, + T_src * const src_buf[GGML_CUDA_MAX_DEVICES], + T_dst * const dst_buf[GGML_CUDA_MAX_DEVICES], + const bool compute[GGML_CUDA_MAX_DEVICES], + int64_t ne) { + const int64_t outer_max_elems = (int64_t) (p->copy_bytes / sizeof(T_src)); + GGML_ASSERT(outer_max_elems > 0); + + bool ok = true; + for (int64_t outer_start = 0; outer_start < ne && ok; outer_start += outer_max_elems) { + const int64_t outer_ne = std::min(outer_max_elems, ne - outer_start); + const size_t outer_nbytes = (size_t) outer_ne * sizeof(T_src); + + T_src * src[GGML_CUDA_MAX_DEVICES] = {}; + T_dst * dst[GGML_CUDA_MAX_DEVICES] = {}; + for (int i = 0; i < p->n_devices; ++i) { + src[i] = src_buf[i] + outer_start; + dst[i] = dst_buf[i] + outer_start; + } + ok = ggml_cuda_ar_allreduce_copy_impl( + p, backends, src, dst, compute, outer_ne, outer_nbytes); + } + return ok; +} + +bool ggml_cuda_ar_allreduce( + ggml_cuda_ar_pipeline * p, + ggml_backend_t * backends, + ggml_tensor ** tensors) { + GGML_ASSERT(p != nullptr); + + const int n = p->n_devices; + GGML_ASSERT(n == 2); + + const ggml_type input_type = tensors[0]->type; + GGML_ASSERT(input_type == GGML_TYPE_F32 || input_type == GGML_TYPE_F16 || input_type == GGML_TYPE_BF16); + + const int64_t ne = ggml_nelements(tensors[0]); + GGML_ASSERT(ne > 0); + + const size_t input_nbytes = ggml_nbytes(tensors[0]); + + // BF16 round-trip: F32 inputs >= bf16_threshold are converted to BF16 for + // the reduction (chunked or copy-engine), halving on-wire bytes. Matches + // NCCL's behaviour. The pre-conversion zeroes inactive shards so the + // inner paths see them as already-prepared compute tensors. + const bool use_bf16 = + input_type == GGML_TYPE_F32 && + p->bf16_threshold > 0 && + input_nbytes >= p->bf16_threshold; + + const ggml_type kernel_type = use_bf16 ? GGML_TYPE_BF16 : input_type; + const size_t type_size = ggml_type_size(kernel_type); + GGML_ASSERT(p->buf_bytes >= type_size); + const size_t nbytes = (size_t) ne * type_size; + + bool compute_flag[GGML_CUDA_MAX_DEVICES] = {}; + for (int i = 0; i < n; ++i) { + compute_flag[i] = (tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) != 0; + } + + // Decide between copy-engine and chunked kernel paths based on the working + // type's actual byte count. No upper bound: copy_outer slices reductions + // larger than copy_bytes into copy_bytes-sized pieces. + const bool use_copy_engine = + p->copy_threshold > 0 && + nbytes >= p->copy_threshold; + + // BF16 inactive-shard zeroing: when use_bf16 is on, the combined kernel + // (chunked kernel path) and the combined add kernel (copy_engine path) + // both accumulate into the F32 tensor data directly, so an inactive + // shard's accumulator must start at zero. + if (use_bf16) { + for (int i = 0; i < n; ++i) { + if (!compute_flag[i]) { + auto * cuda_ctx = static_cast(backends[i]->context); + GGML_ASSERT(cuda_ctx->device == p->devices[i]); + ggml_cuda_set_device(p->devices[i]); + CUDA_CHECK(cudaMemsetAsync(tensors[i]->data, 0, (size_t) ne * sizeof(float), cuda_ctx->stream())); + } + } + } + + // Pre-convert F32 -> BF16 into bf16_tmp ONLY for the copy_engine + use_bf16 + // path; the chunked kernel path's combined kernel does the conversion + // inline as it writes to host_buf. + ggml_cuda_pool_alloc bf16_tmp[GGML_CUDA_MAX_DEVICES]; + void * copy_src_ptr[GGML_CUDA_MAX_DEVICES] = {}; + + if (use_copy_engine && use_bf16) { + to_bf16_cuda_t to_bf16 = ggml_get_to_bf16_cuda(GGML_TYPE_F32); + for (int i = 0; i < n; ++i) { + auto * cuda_ctx = static_cast(backends[i]->context); + GGML_ASSERT(cuda_ctx->device == p->devices[i]); + bf16_tmp[i].pool = &cuda_ctx->pool(); + bf16_tmp[i].alloc(ne); + ggml_cuda_set_device(p->devices[i]); + if (compute_flag[i]) { + to_bf16(tensors[i]->data, bf16_tmp[i].get(), ne, cuda_ctx->stream()); + CUDA_CHECK(cudaGetLastError()); + } else { + CUDA_CHECK(cudaMemsetAsync(bf16_tmp[i].get(), 0, nbytes, cuda_ctx->stream())); + } + copy_src_ptr[i] = bf16_tmp[i].get(); + } + } + + bool ok = true; + if (use_copy_engine) { + // After up-front BF16 conversion, the tmp buffers already hold the + // (possibly zeroed-for-inactive) data, so the inner path can treat + // every shard as compute. + bool inner_compute[GGML_CUDA_MAX_DEVICES]; + for (int i = 0; i < n; ++i) { + inner_compute[i] = use_bf16 ? true : compute_flag[i]; + } + + // Dispatch into copy_impl with explicit src/dst types. When use_bf16 + // is on, the wire type is BF16 (src = bf16_tmp) and the accumulator + // is F32 (dst = tensors[i]->data); the combined add kernel rounds dst + // through BF16 for bit-equivalence and writes F32 directly, so no + // post-conversion is needed. Otherwise src == dst (same native type). + if (use_bf16) { + GGML_ASSERT(kernel_type == GGML_TYPE_BF16); + nv_bfloat16 * src[GGML_CUDA_MAX_DEVICES] = {}; + float * dst[GGML_CUDA_MAX_DEVICES] = {}; + for (int i = 0; i < n; ++i) { + src[i] = static_cast(copy_src_ptr[i]); + dst[i] = static_cast(tensors[i]->data); + } + ok = ggml_cuda_ar_allreduce_copy_outer( + p, backends, src, dst, inner_compute, ne); + } else { + switch (kernel_type) { + case GGML_TYPE_F32: { + float * buf[GGML_CUDA_MAX_DEVICES] = {}; + for (int i = 0; i < n; ++i) { + buf[i] = static_cast(tensors[i]->data); + } + ok = ggml_cuda_ar_allreduce_copy_outer( + p, backends, buf, buf, inner_compute, ne); + break; + } + case GGML_TYPE_BF16: { + nv_bfloat16 * buf[GGML_CUDA_MAX_DEVICES] = {}; + for (int i = 0; i < n; ++i) { + buf[i] = static_cast(tensors[i]->data); + } + ok = ggml_cuda_ar_allreduce_copy_outer( + p, backends, buf, buf, inner_compute, ne); + break; + } + case GGML_TYPE_F16: { + half * buf[GGML_CUDA_MAX_DEVICES] = {}; + for (int i = 0; i < n; ++i) { + buf[i] = static_cast(tensors[i]->data); + } + ok = ggml_cuda_ar_allreduce_copy_outer( + p, backends, buf, buf, inner_compute, ne); + break; + } + default: + GGML_ASSERT(false); + } + } + } else { + // host_buf carries T_wire-typed data; max_chunk_elems is the count that + // fits in one host_buf at the wire size. + const size_t max_chunk_elems = p->buf_bytes / type_size; + const size_t input_type_size = ggml_type_size(input_type); + + // Chunked kernel path runs entirely on the caller's compute stream: + // since AR is a barrier here, same-stream ordering subsumes any + // cross-stream event handshake that the copy-engine path needs, and + // skips the cross-stream scheduling overhead that was hurting the + // small-tensor (tg) latency on the AR-stream variant. Only ev.ker is + // still recorded at end-of-AR for acquire_slot's pool-wraparound check. + for (int64_t chunk_start = 0; chunk_start < ne; chunk_start += (int64_t) max_chunk_elems) { + const size_t remaining_elems = (size_t) (ne - chunk_start); + const size_t chunk_elems = remaining_elems < max_chunk_elems ? remaining_elems : max_chunk_elems; + const size_t chunk_dst_bytes = chunk_elems * input_type_size; + + const auto [slot, token] = ggml_cuda_ar_acquire_slot(p); + const bool last_chunk = chunk_start + (int64_t) chunk_elems == ne; + + for (int i = 0; i < n; ++i) { + const int peer = 1 - i; // valid for n == 2 only + ggml_cuda_set_device(p->devices[i]); + auto * cuda_ctx = static_cast(backends[i]->context); + GGML_ASSERT(cuda_ctx->device == p->devices[i]); + cudaStream_t stream = cuda_ctx->stream(); + + char * data = static_cast(tensors[i]->data) + chunk_start * (int64_t) input_type_size; + + // Match NCCL/meta-backend semantics: inactive shards contribute + // zeros. On the BF16 path the F32 tensor data was already + // zeroed up-front (above), so per-chunk zeroing isn't needed. + if (!compute_flag[i] && !use_bf16) { + CUDA_CHECK(cudaMemsetAsync(data, 0, chunk_dst_bytes, stream)); + } + +#define LAUNCH_AR_KERNEL(T_dst, T_wire) \ + ggml_cuda_ar_kernel<<>>( \ + reinterpret_cast(data), \ + reinterpret_cast(data), \ + reinterpret_cast(p->host_buf[i].dev + (size_t) slot * p->buf_bytes), \ + reinterpret_cast(p->host_buf[peer].dev + (size_t) slot * p->buf_bytes), \ + static_cast(chunk_elems), \ + ggml_cuda_ar_arrival_ptr(p, slot, i), \ + ggml_cuda_ar_arrival_ptr(p, slot, peer), \ + token) + + if (use_bf16) { + GGML_ASSERT(input_type == GGML_TYPE_F32); + LAUNCH_AR_KERNEL(float, nv_bfloat16); + } else { + switch (input_type) { + case GGML_TYPE_F32: LAUNCH_AR_KERNEL(float, float); break; + case GGML_TYPE_F16: LAUNCH_AR_KERNEL(half, half); break; + case GGML_TYPE_BF16: LAUNCH_AR_KERNEL(nv_bfloat16, nv_bfloat16); break; + default: GGML_ASSERT(false); + } + } + +#undef LAUNCH_AR_KERNEL + CUDA_CHECK(cudaGetLastError()); + + if (last_chunk) { + CUDA_CHECK(cudaEventRecord(p->ev_pool[i][slot].ker, stream)); + } + } + } + } + + return ok; +} + +#else // defined(GGML_USE_MUSA) + +// MUSA lacks the host-mapped pinned-memory APIs (cudaHostAllocPortable +// / cudaHostAllocMapped / cudaHostGetDevicePointer) and a device-side +// sleep intrinsic that this implementation relies on, so the internal +// AllReduce is unavailable there. The dispatcher in ggml-cuda.cu treats +// a nullptr pipeline as "init failed" and silently falls back to the meta +// backend's generic AllReduce. +ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init(const int *, size_t) { + return nullptr; +} +void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline *) { +} +bool ggml_cuda_ar_allreduce(ggml_cuda_ar_pipeline *, ggml_backend_t *, ggml_tensor **) { + return false; +} + +#endif // !defined(GGML_USE_MUSA) diff --git a/ggml/src/ggml-cuda/allreduce.cuh b/ggml/src/ggml-cuda/allreduce.cuh new file mode 100644 index 0000000..76205d3 --- /dev/null +++ b/ggml/src/ggml-cuda/allreduce.cuh @@ -0,0 +1,29 @@ +#pragma once + +#include "common.cuh" +#include "ggml-backend-impl.h" + +#include + +// Opaque pipeline context -- owns all pinned buffers, streams, and events. +struct ggml_cuda_ar_pipeline; + +// Allocate a pipeline for n_devices GPUs. +// devices[] holds the GPU device IDs in rank order. +// Returns nullptr on allocation failure. +ggml_cuda_ar_pipeline * ggml_cuda_ar_pipeline_init( + const int * devices, size_t n_devices); + +// Release all resources owned by the pipeline. +void ggml_cuda_ar_pipeline_free(ggml_cuda_ar_pipeline * pipeline); + +// Execute an in-place AllReduce (sum) across tensors[0..n_devices-1]. +// tensors[i] must live on the device managed by backends[i] and be +// contiguous F32, F16, or BF16. +// Preconditions are checked by the CUDA comm dispatcher before calling this. +// Returns true once the reduction work has been enqueued successfully. +bool ggml_cuda_ar_allreduce( + ggml_cuda_ar_pipeline * pipeline, + ggml_backend_t * backends, + ggml_tensor ** tensors); + diff --git a/ggml/src/ggml-cuda/argsort.cu b/ggml/src/ggml-cuda/argsort.cu index 0f3f017..101afeb 100644 --- a/ggml/src/ggml-cuda/argsort.cu +++ b/ggml/src/ggml-cuda/argsort.cu @@ -2,8 +2,10 @@ #ifdef GGML_CUDA_USE_CUB # include -# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1) + // strided_iterator was added in CCCL 3.1 +# if (CCCL_MAJOR_VERSION > 3 || (CCCL_MAJOR_VERSION == 3 && CCCL_MINOR_VERSION >= 1)) # define STRIDED_ITERATOR_AVAILABLE +# include # endif using namespace cub; #endif // GGML_CUDA_USE_CUB @@ -26,7 +28,21 @@ static __global__ void init_offsets(int * offsets, const int ncols, const int nr } #endif // STRIDED_ITERATOR_AVAILABLE +// returns the suggested maximum number of rows to process at once, given the temporary buffer bytes per row +int ggml_cuda_chunk_nrows(const size_t row_bytes, const int64_t nrows) { + // process rows in chunks up to approximately this size (currently 64MB) + // to avoid excessive temporary buffers memory usage + const int chunk_bytes = 1 << 26; + + // calculate how many rows will fit in one chunk (must be at least one) + const int chunk_nrows = std::max((int) (chunk_bytes / row_bytes), 1); + + // limit the resulting amount to total nrows + return std::min((int64_t) chunk_nrows, nrows); +} + #ifdef GGML_CUDA_USE_CUB + void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, const float * x, int * dst, @@ -36,9 +52,12 @@ void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, cudaStream_t stream) { ggml_cuda_pool_alloc temp_indices_alloc(pool, ncols * nrows); ggml_cuda_pool_alloc temp_keys_alloc(pool, ncols * nrows); + // Device*Sort algorithms currently do not allow for in-place sorting/aliasing of input/outputs + ggml_cuda_pool_alloc temp_keys_out_alloc(pool, ncols * nrows); int * temp_indices = temp_indices_alloc.get(); float * temp_keys = temp_keys_alloc.get(); + float * temp_keys_out = temp_keys_out_alloc.get(); static const int block_size = 256; const dim3 grid_size((ncols + block_size - 1) / block_size, nrows); @@ -70,18 +89,18 @@ void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, if (order == GGML_SORT_ORDER_ASC) { if (nrows == 1) { - CUDA_CHECK(DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, temp_keys, // keys (in-place) + CUDA_CHECK(DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, temp_keys_out, // keys in, keys out temp_indices, dst, // values (indices) ncols, 0, sizeof(float) * 8, stream)); } else if (is_capturing) { CUDA_CHECK(DeviceSegmentedRadixSort::SortPairs( - nullptr, temp_storage_bytes, temp_keys, temp_keys, // keys (in-place) + nullptr, temp_storage_bytes, temp_keys, temp_keys_out, // keys in, keys out temp_indices, dst, // values (indices) ncols * nrows, nrows, // num items, num segments offset_iterator, offset_iterator + 1, 0, sizeof(float) * 8, stream)); } else { CUDA_CHECK(DeviceSegmentedSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, - temp_keys, // keys (in-place) + temp_keys_out, // keys out temp_indices, dst, // values (indices) ncols * nrows, nrows, // num items, num segments offset_iterator, offset_iterator + 1, stream)); @@ -89,15 +108,15 @@ void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, } else { if (nrows == 1) { CUDA_CHECK(DeviceRadixSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, - temp_keys, // keys (in-place) + temp_keys_out, // keys out temp_indices, dst, // values (indices) ncols, 0, sizeof(float) * 8, stream)); } else if (is_capturing) { CUDA_CHECK(DeviceSegmentedRadixSort::SortPairsDescending( - nullptr, temp_storage_bytes, temp_keys, temp_keys, temp_indices, dst, ncols * nrows, nrows, + nullptr, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, 0, sizeof(float) * 8, stream)); } else { - CUDA_CHECK(DeviceSegmentedSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, temp_keys, + CUDA_CHECK(DeviceSegmentedSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, stream)); } @@ -109,31 +128,31 @@ void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, if (order == GGML_SORT_ORDER_ASC) { if (nrows == 1) { CUDA_CHECK(DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, - temp_keys, // keys (in-place) + temp_keys_out, // keys out temp_indices, dst, // values (indices) ncols, 0, sizeof(float) * 8, stream)); } else if (is_capturing) { - CUDA_CHECK(DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, + CUDA_CHECK(DeviceSegmentedRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, 0, sizeof(float) * 8, stream)); } else { - CUDA_CHECK(DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, + CUDA_CHECK(DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, stream)); } } else { if (nrows == 1) { CUDA_CHECK(DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, temp_keys, - temp_keys, // keys (in-place) + temp_keys_out, // keys out temp_indices, dst, // values (indices) ncols, 0, sizeof(float) * 8, stream)); } else if (is_capturing) { CUDA_CHECK(DeviceSegmentedRadixSort::SortPairsDescending( - d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, temp_indices, dst, ncols * nrows, nrows, + d_temp_storage, temp_storage_bytes, temp_keys, temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, 0, sizeof(float) * 8, stream)); } else { CUDA_CHECK(DeviceSegmentedSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, temp_keys, - temp_keys, temp_indices, dst, ncols * nrows, nrows, + temp_keys_out, temp_indices, dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1, stream)); } } @@ -148,52 +167,62 @@ static inline __device__ void ggml_cuda_swap(T & a, T & b) { b = tmp; } +// One compare-exchange of the bitonic network at (k, j) for column col. template -static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ncols_pad) { - // bitonic sort - int col = threadIdx.x; - int row = blockIdx.x; - - if (col >= ncols_pad) { +static inline __device__ void bitonic_step(const float * x_row, int * dst_row, const int ncols, const int col, const int k, const int j) { + const int ixj = col ^ j; + if (ixj <= col) { return; } + if ((col & k) == 0) { + if (dst_row[col] >= ncols || + (dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ? + x_row[dst_row[col]] > x_row[dst_row[ixj]] : + x_row[dst_row[col]] < x_row[dst_row[ixj]])) + ) { + ggml_cuda_swap(dst_row[col], dst_row[ixj]); + } + } else { + if (dst_row[ixj] >= ncols || + (dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ? + x_row[dst_row[col]] < x_row[dst_row[ixj]] : + x_row[dst_row[col]] > x_row[dst_row[ixj]])) + ) { + ggml_cuda_swap(dst_row[col], dst_row[ixj]); + } + } +} + +// Bitonic sort of one row per block. Each thread owns the columns +// threadIdx.x + i * blockDim.x, so rows wider than the block (up to the +// shared memory limit) sort with several columns per thread. Every +// (k, j) stage runs all owned columns before the barrier; a pair +// (col, col ^ j) is exchanged by the owner of its lower index only. +template +static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ncols_pad) { + const int row = blockIdx.x; const float * x_row = x + row * ncols; extern __shared__ int dst_row[]; // initialize indices - dst_row[col] = col; + for (int col = threadIdx.x; col < ncols_pad; col += blockDim.x) { + dst_row[col] = col; + } __syncthreads(); for (int k = 2; k <= ncols_pad; k *= 2) { for (int j = k / 2; j > 0; j /= 2) { - int ixj = col ^ j; - if (ixj > col) { - if ((col & k) == 0) { - if (dst_row[col] >= ncols || - (dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ? - x_row[dst_row[col]] > x_row[dst_row[ixj]] : - x_row[dst_row[col]] < x_row[dst_row[ixj]])) - ) { - ggml_cuda_swap(dst_row[col], dst_row[ixj]); - } - } else { - if (dst_row[ixj] >= ncols || - (dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ? - x_row[dst_row[col]] < x_row[dst_row[ixj]] : - x_row[dst_row[col]] > x_row[dst_row[ixj]])) - ) { - ggml_cuda_swap(dst_row[col], dst_row[ixj]); - } - } + for (int col = threadIdx.x; col < ncols_pad; col += blockDim.x) { + bitonic_step(x_row, dst_row, ncols, col, k, j); } __syncthreads(); } } // copy the result to dst without the padding - if (col < ncols) { + for (int col = threadIdx.x; col < ncols; col += blockDim.x) { dst[row * ncols + col] = dst_row[col]; } } @@ -215,7 +244,9 @@ void argsort_f32_i32_cuda_bitonic(const float * x, // bitonic sort requires ncols to be power of 2 const int ncols_pad = next_power_of_2(ncols); - const dim3 block_dims(ncols_pad, 1, 1); + // one thread per column up to the block limit, several columns per + // thread beyond it; shared memory is the remaining bound + const dim3 block_dims(ncols_pad < CUDA_ARGSORT_BLOCK_SIZE ? ncols_pad : CUDA_ARGSORT_BLOCK_SIZE, 1, 1); const dim3 block_nums(nrows, 1, 1); const size_t shared_mem = ncols_pad * sizeof(int); @@ -253,11 +284,23 @@ void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const size_t shared_mem = ncols_pad * sizeof(int); const size_t max_shared_mem = ggml_cuda_info().devices[ggml_cuda_get_device()].smpb; - if (shared_mem > max_shared_mem || ncols > 1024) { - ggml_cuda_pool & pool = ctx.pool(); - argsort_f32_i32_cuda_cub(pool, src0_d, (int *) dst_d, ncols, nrows, order, stream); - } else { + // early return if we can use bitonic argsort + if (shared_mem <= max_shared_mem && ncols <= 1024) { argsort_f32_i32_cuda_bitonic(src0_d, (int *) dst_d, ncols, nrows, order, stream); + return; + } + + const int chunk_nrows = ggml_cuda_chunk_nrows(src0->nb[1], nrows); + + ggml_cuda_pool & pool = ctx.pool(); + + for (int64_t i = 0; i < nrows; i += chunk_nrows) { + int iter_nrows = std::min((int64_t) chunk_nrows, nrows - i); + + argsort_f32_i32_cuda_cub(pool, src0_d, (int *) dst_d, ncols, iter_nrows, order, stream); + + src0_d += ncols * iter_nrows; + dst_d += ncols * iter_nrows; } #else argsort_f32_i32_cuda_bitonic(src0_d, (int *) dst_d, ncols, nrows, order, stream); diff --git a/ggml/src/ggml-cuda/argsort.cuh b/ggml/src/ggml-cuda/argsort.cuh index 22b7306..86df5fd 100644 --- a/ggml/src/ggml-cuda/argsort.cuh +++ b/ggml/src/ggml-cuda/argsort.cuh @@ -1,7 +1,11 @@ #include "common.cuh" +#define CUDA_ARGSORT_BLOCK_SIZE 1024 + void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +int ggml_cuda_chunk_nrows(const size_t row_bytes, const int64_t nrows); + #ifdef GGML_CUDA_USE_CUB void argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool, const float * x, diff --git a/ggml/src/ggml-cuda/binbcast.cu b/ggml/src/ggml-cuda/binbcast.cu index adb4d5f..f7c3a73 100644 --- a/ggml/src/ggml-cuda/binbcast.cu +++ b/ggml/src/ggml-cuda/binbcast.cu @@ -1,7 +1,11 @@ #include "binbcast.cuh" +#include "convert.cuh" #include #include +template +using type_for_index = T; + static __device__ __forceinline__ float op_repeat(const float a, const float b) { return b; GGML_UNUSED(a); @@ -31,33 +35,34 @@ template = (uint32_t)ne0 || i1 >= (uint32_t)ne1 || i2 >= (uint32_t)ne2 || i3 >= ne3.z) { + if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3.z) { return; } @@ -65,24 +70,32 @@ static __global__ void k_bin_bcast(const src0_t * src0, const uint32_t i12 = fastmodulo(i2, ne12); const uint32_t i13 = fastmodulo(i3, ne13); - const size_t i_src0 = i3*s03 + i2*s02 + i1*s01; - const size_t i_src1 = i13*s13 + i12*s12 + i11*s11; - const size_t i_dst = i3*s3 + i2*s2 + i1*s1; + const size_t i_src0 = size_t( i3)*s03 + size_t( i2)*s02 + size_t( i1)*s01; + const size_t i_src1 = size_t(i13)*s13 + size_t(i12)*s12 + size_t(i11)*s11; + const size_t i_dst = size_t( i3)*s3 + size_t( i2)*s2 + size_t( i1)*s1; const src0_t * src0_row = src0 ? (src0 + i_src0) : nullptr; dst_t * dst_row = dst + i_dst; - for (int i0 = i0s; i0 < ne0; i0 += blockDim.x * gridDim.x) { + const uint32_t s0 = blockDim.x * gridDim.x; + + ggml_cuda_pdl_sync(); + for (uint32_t i0 = i0s; i0 < ne0; i0 += s0) { const uint32_t i10 = fastmodulo(i0, ne10); - float result = src0_row ? (float) src0_row[i0*s00] : 0.0f; + float result = src0_row ? ggml_cuda_cast(src0_row[size_t(i0)*s00]) : 0.0f; if constexpr (sizeof...(src1_ptrs) > 0) { - result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10*s10]))); + result = (..., (result = bin_op(result, ggml_cuda_cast(src1s[i_src1 + size_t(i10)*s10])))); } else { - result = bin_op(result, (float)src1[i_src1 + i10*s10]); + result = bin_op(result, ggml_cuda_cast(src1[i_src1 + size_t(i10)*s10])); } - dst_row[i0] = (dst_t) result; + dst_row[i0] = ggml_cuda_cast(result); + + // protect i0 from overflow + if (ne0 - i0 <= s0) { + break; + } } } @@ -105,19 +118,19 @@ static __global__ void k_bin_bcast_unravel(const src0_t * src0, const uint3 ne12, const uint3 ne13, /*const int s0,*/ - const int s1, - const int s2, - const int s3, - const int s00, - const int s01, - const int s02, - const int s03, - const int s10, - const int s11, - const int s12, - const int s13, + const uint32_t s1, + const uint32_t s2, + const uint32_t s3, + const uint32_t s00, + const uint32_t s01, + const uint32_t s02, + const uint32_t s03, + const uint32_t s10, + const uint32_t s11, + const uint32_t s12, + const uint32_t s13, src1_ptrs... src1s) { - const int i = blockDim.x*blockIdx.x + threadIdx.x; + const uint32_t i = blockDim.x*blockIdx.x + threadIdx.x; const uint32_t i3 = fastdiv(i, prod_012); const uint32_t i2 = fastdiv(i - i3 * prod_012.z, prod_01); @@ -128,27 +141,28 @@ static __global__ void k_bin_bcast_unravel(const src0_t * src0, return; } - const int i11 = fastmodulo(i1, ne11); - const int i12 = fastmodulo(i2, ne12); - const int i13 = fastmodulo(i3, ne13); + const uint32_t i11 = fastmodulo(i1, ne11); + const uint32_t i12 = fastmodulo(i2, ne12); + const uint32_t i13 = fastmodulo(i3, ne13); - const size_t i_src0 = i3*s03 + i2*s02 + i1*s01; - const size_t i_src1 = i13*s13 + i12*s12 + i11*s11; - const size_t i_dst = i3*s3 + i2*s2 + i1*s1; + const size_t i_src0 = size_t( i3)*s03 + size_t( i2)*s02 + size_t( i1)*s01; + const size_t i_src1 = size_t(i13)*s13 + size_t(i12)*s12 + size_t(i11)*s11; + const size_t i_dst = size_t( i3)*s3 + size_t( i2)*s2 + size_t( i1)*s1; const src0_t * src0_row = src0 ? (src0 + i_src0) : nullptr; dst_t * dst_row = dst + i_dst; - const int i10 = fastmodulo(i0, ne10); + const uint32_t i10 = fastmodulo(i0, ne10); - float result = src0_row ? (float) src0_row[i0*s00] : 0.0f; + ggml_cuda_pdl_sync(); + float result = src0_row ? ggml_cuda_cast(src0_row[size_t(i0)*s00]) : 0.0f; if constexpr (sizeof...(src1_ptrs) > 0) { - result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10*s10]))); + result = (..., (result = bin_op(result, ggml_cuda_cast(src1s[i_src1 + size_t(i10)*s10])))); } else { - result = bin_op(result, (float)src1[i_src1 + i10*s10]); + result = bin_op(result, ggml_cuda_cast(src1[i_src1 + size_t(i10)*s10])); } - dst_row[i0] = (dst_t) result; + dst_row[i0] = ggml_cuda_cast(result); } template @@ -242,6 +256,31 @@ static void launch_bin_bcast_pack(const ggml_tensor * src0, const ggml_tensor * size_t s02 = nb02 / sizeof(src0_t); size_t s03 = nb03 / sizeof(src0_t); + GGML_ASSERT(ne0 <= std::numeric_limits::max()); + GGML_ASSERT(ne1 <= std::numeric_limits::max()); + GGML_ASSERT(ne2 <= std::numeric_limits::max()); + GGML_ASSERT(ne3 <= std::numeric_limits::max()); + + //GGML_ASSERT(s0 <= std::numeric_limits::max()); + GGML_ASSERT(s1 <= std::numeric_limits::max()); + GGML_ASSERT(s2 <= std::numeric_limits::max()); + GGML_ASSERT(s3 <= std::numeric_limits::max()); + + GGML_ASSERT(s00 <= std::numeric_limits::max()); + GGML_ASSERT(s01 <= std::numeric_limits::max()); + GGML_ASSERT(s02 <= std::numeric_limits::max()); + GGML_ASSERT(s03 <= std::numeric_limits::max()); + + GGML_ASSERT(s10 <= std::numeric_limits::max()); + GGML_ASSERT(s11 <= std::numeric_limits::max()); + GGML_ASSERT(s12 <= std::numeric_limits::max()); + GGML_ASSERT(s13 <= std::numeric_limits::max()); + + GGML_ASSERT(cne1[0] <= std::numeric_limits::max()); + GGML_ASSERT(cne1[1] <= std::numeric_limits::max()); + GGML_ASSERT(cne1[2] <= std::numeric_limits::max()); + GGML_ASSERT(cne1[3] <= std::numeric_limits::max()); + GGML_ASSERT(nb0 % sizeof(dst_t) == 0); GGML_ASSERT(nb1 % sizeof(dst_t) == 0); GGML_ASSERT(nb2 % sizeof(dst_t) == 0); @@ -257,6 +296,8 @@ static void launch_bin_bcast_pack(const ggml_tensor * src0, const ggml_tensor * GGML_ASSERT(nb12 % sizeof(src1_t) == 0); GGML_ASSERT(nb13 % sizeof(src1_t) == 0); + GGML_ASSERT(ne2 * ne3 <= std::numeric_limits::max()); + const int block_size = 128; int64_t hne0 = std::max(ne0 / 2LL, 1LL); @@ -275,42 +316,41 @@ static void launch_bin_bcast_pack(const ggml_tensor * src0, const ggml_tensor * const uint3 ne13 = init_fastdiv_values((uint32_t) cne1[3]); if (block_nums.z > 65535 || block_nums.y > 65535) { - int block_num = (ne0 * ne1 * ne2 * ne3 + block_size - 1) / block_size; + int64_t block_num = (ne0 * ne1 * ne2 * ne3 + block_size - 1) / block_size; + + GGML_ASSERT(block_num <= std::numeric_limits::max()); + GGML_ASSERT(block_num * block_size <= std::numeric_limits::max()); + GGML_ASSERT(ne0 * ne1 <= std::numeric_limits::max()); + GGML_ASSERT(ne0 * ne1 * ne2 <= std::numeric_limits::max()); + const uint3 prod_012 = init_fastdiv_values((uint32_t) (ne0 * ne1 * ne2)); const uint3 prod_01 = init_fastdiv_values((uint32_t) (ne0 * ne1)); const uint3 ne0_fastdiv = init_fastdiv_values((uint32_t) ne0); const uint3 ne1_fastdiv = init_fastdiv_values((uint32_t) ne1); const uint3 ne2_fastdiv = init_fastdiv_values((uint32_t) ne2); - if constexpr (sizeof...(I) > 0) { - k_bin_bcast_unravel<<>>( + { + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)block_num, block_size, 0, stream); + ggml_cuda_kernel_launch(k_bin_bcast_unravel...>, launch_params, src0_dd, src1_dd, dst_dd, ne0_fastdiv, ne1_fastdiv, ne2_fastdiv, ne3, prod_012, prod_01, ne10, ne11, ne12, ne13, /*s0,*/ s1, s2, s3, s00, s01, s02, s03, s10, s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...); - } else { - k_bin_bcast_unravel - <<>>(src0_dd, src1_dd, dst_dd, ne0_fastdiv, ne1_fastdiv, - ne2_fastdiv, ne3, prod_012, prod_01, ne10, ne11, ne12, ne13, - /*s0,*/ s1, s2, s3, - s00, s01, s02, s03, - s10, s11, s12, s13); } } else { + GGML_ASSERT(int64_t(block_nums.x) * block_dims.x <= std::numeric_limits::max()); + GGML_ASSERT(int64_t(block_nums.y) * block_dims.y <= std::numeric_limits::max()); + GGML_ASSERT(int64_t(block_nums.z) * block_dims.z <= std::numeric_limits::max()); + const uint3 ne3_fastdiv = init_fastdiv_values((uint32_t) ne3); - if constexpr (sizeof...(I) > 0) { - k_bin_bcast<<>>( + { + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + ggml_cuda_kernel_launch(k_bin_bcast...>, launch_params, src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3_fastdiv, ne10, ne11, ne12, ne13, /*s0,*/ s1, s2, s3, - s00 ,s01, s02, s03, - s10, s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...); - } else { - k_bin_bcast<<>>( - src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3_fastdiv, ne10, ne11, ne12, ne13, - /*s0,*/ s1, s2, s3, s00, s01, s02, s03, - s10, s11, s12, s13); + s10, s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...); } } } @@ -333,6 +373,7 @@ static __global__ void k_repeat_back( } T sum = 0; + ggml_cuda_pdl_sync(); for (int64_t i3 = tid3; i3 < ne03; i3 += ne3) { for (int64_t i2 = tid2; i2 < ne02; i2 += ne2) { for (int64_t i1 = tid1; i1 < ne01; i1 += ne1) { @@ -373,10 +414,14 @@ static void ggml_cuda_op_bin_bcast( const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const void * src0_dd, const void * src1_dd, void * dst_dd, cudaStream_t stream) { - GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16); - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + if (src0->type == GGML_TYPE_F32 && src1->type != GGML_TYPE_BF16 && dst->type == GGML_TYPE_F32) { op()(src0, src1, dst, (const float *)src0_dd, (const float *)src1_dd, (float *)dst_dd, stream); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { + op()(src0, src1, dst, (const nv_bfloat16 *) src0_dd, (const nv_bfloat16 *) src1_dd, (nv_bfloat16 *) dst_dd, stream); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_BF16) { + op()(src0, src1, dst, (const nv_bfloat16 *) src0_dd, (const float *) src1_dd, (nv_bfloat16 *) dst_dd, stream); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { op()(src0, src1, dst, (const half *) src0_dd, (const half *)src1_dd, (half *) dst_dd, stream); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) { @@ -417,10 +462,18 @@ static void ggml_cuda_op_fused_binbcast_impl(ggml_backend_cuda_context & ctx, gg const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + if (src0->type == GGML_TYPE_F32 && src1->type != GGML_TYPE_BF16 && dst->type == GGML_TYPE_F32) { launch_bin_bcast_pack(src0, src1, dst, (const float *) src0->data, (const float *) src1->data, (float *) dst->data, stream, std::make_index_sequence{}); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { + launch_bin_bcast_pack(src0, src1, dst, + (const nv_bfloat16 *) src0->data, (const nv_bfloat16 *) src1->data, (nv_bfloat16 *) dst->data, + stream, std::make_index_sequence{}); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_BF16) { + launch_bin_bcast_pack(src0, src1, dst, + (const nv_bfloat16 *) src0->data, (const float *) src1->data, (nv_bfloat16 *) dst->data, + stream, std::make_index_sequence{}); } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { launch_bin_bcast_pack(src0, src1, dst, (const half *) src0->data, (const half *) src1->data, (half *) dst->data, @@ -527,8 +580,7 @@ void ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst float * dst_d = (float *) dst->data; repeat_back_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s00, s01, s02, s03, ne0, ne1, ne2, ne3, stream); } break; - default: { + default: GGML_ASSERT(false); - } break; } } diff --git a/ggml/src/ggml-cuda/clamp.cu b/ggml/src/ggml-cuda/clamp.cu index fe415e7..2727b7e 100644 --- a/ggml/src/ggml-cuda/clamp.cu +++ b/ggml/src/ggml-cuda/clamp.cu @@ -4,21 +4,42 @@ static __device__ __forceinline__ float op_clamp(float x, float min, float max) return fminf(fmaxf(x, min), max); } +// src and dst may be views: rows are contiguous, dims 1..3 follow the strides (in elements). template -static __global__ void op_clamp_kernel(const T * x, T * dst, const T min, const T max, const int k) { - const int i = blockDim.x*blockIdx.x + threadIdx.x; +static __global__ void op_clamp_kernel(const T * x, T * dst, const T min, const T max, const uint32_t k, + const uint3 ne0, const uint3 ne1, const uint3 ne2, + const uint32_t s01, const uint32_t s02, const uint32_t s03, + const uint32_t s1, const uint32_t s2, const uint32_t s3) { + const uint32_t i = blockDim.x*blockIdx.x + threadIdx.x; if (i >= k) { return; } - dst[i] = (T)op_clamp((float)x[i], (float)min, (float)max); + const uint2 d0 = fast_div_modulo(i, ne0); // + const uint2 d1 = fast_div_modulo(d0.x, ne1); // + const uint2 d2 = fast_div_modulo(d1.x, ne2); // + + const size_t i_src = d0.y + size_t(d1.y)*s01 + size_t(d2.y)*s02 + size_t(d2.x)*s03; + const size_t i_dst = d0.y + size_t(d1.y)*s1 + size_t(d2.y)*s2 + size_t(d2.x)*s3; + + dst[i_dst] = (T)op_clamp((float)x[i_src], (float)min, (float)max); } template -static void clamp_cuda(const T * x, T * dst, const T min, const T max, const int k, cudaStream_t stream) { - const int num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE; - op_clamp_kernel<<>>(x, dst, min, max, k); +static void clamp_cuda(const T * x, T * dst, const T min, const T max, const ggml_tensor * src0, const ggml_tensor * t, cudaStream_t stream) { + const int64_t k = ggml_nelements(src0); + const size_t ts = sizeof(T); + GGML_ASSERT(k <= std::numeric_limits::max()); + + const uint3 ne0 = init_fastdiv_values(src0->ne[0]); + const uint3 ne1 = init_fastdiv_values(src0->ne[1]); + const uint3 ne2 = init_fastdiv_values(src0->ne[2]); + + const int64_t num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE; + op_clamp_kernel<<>>(x, dst, min, max, (uint32_t) k, ne0, ne1, ne2, + src0->nb[1]/ts, src0->nb[2]/ts, src0->nb[3]/ts, + t->nb[1]/ts, t->nb[2]/ts, t->nb[3]/ts); } @@ -31,6 +52,7 @@ void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst)); float min; float max; @@ -38,8 +60,8 @@ void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { memcpy(&max, (float *) dst->op_params + 1, sizeof(float)); if (src0->type == GGML_TYPE_F16) { - clamp_cuda((const half *)src0_d, (half *)dst_d, (half)min, (half)max, ggml_nelements(src0), stream); + clamp_cuda((const half *)src0_d, (half *)dst_d, (half)min, (half)max, src0, dst, stream); } else { - clamp_cuda((const float *)src0_d, (float *)dst_d, (float)min, (float)max, ggml_nelements(src0), stream); + clamp_cuda((const float *)src0_d, (float *)dst_d, (float)min, (float)max, src0, dst, stream); } } diff --git a/ggml/src/ggml-cuda/col2im-1d.cu b/ggml/src/ggml-cuda/col2im-1d.cu new file mode 100644 index 0000000..fecd4c6 --- /dev/null +++ b/ggml/src/ggml-cuda/col2im-1d.cu @@ -0,0 +1,81 @@ +#include "col2im-1d.cuh" +#include "convert.cuh" + +// col2im_1d: scatter-add GEMM columns to 1D signal (gather approach) +// columns: [K*OC, T_in] -> output: [T_out, OC] +// Supports F32, F16, BF16 data with F32 accumulator. + +template +static __global__ void col2im_1d_kernel( + const T * __restrict__ col, + T * __restrict__ dst, + const int T_in, const uint3 T_out_fd, + const int OC, const int K, const int K_OC, + const int s0, const int p0, const int total) { + + const int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx >= total) return; + + // dst layout: [T_out, OC], ne[0]=T_out fastest + const uint2 qr = fast_div_modulo((uint32_t)idx, T_out_fd); // qr.x = idx / T_out, qr.y = idx % T_out + const int oc = (int)qr.x; + const int t_out = (int)qr.y; + const int t_abs = t_out + p0; // absolute position in uncropped signal + + // Gather: find all (t_in, k) where t_in*s + k == t_abs, 0 <= k < K + int t_in_min = (t_abs - K + s0) / s0; // ceil((t_abs - K + 1) / s) + if (t_in_min < 0) t_in_min = 0; + int t_in_max = t_abs / s0; + if (t_in_max >= T_in) t_in_max = T_in - 1; + + float sum = 0.0f; + for (int t_in = t_in_min; t_in <= t_in_max; t_in++) { + const int k = t_abs - t_in * s0; + // col layout: [K*OC, T_in], column index = oc * K + k + sum += ggml_cuda_cast(col[(oc * K + k) + t_in * K_OC]); + } + + dst[idx] = ggml_cuda_cast(sum); +} + +void ggml_cuda_op_col2im_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(ggml_is_contiguous(src0)); + + const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; + const int32_t OC = ((const int32_t *)(dst->op_params))[1]; + const int32_t p0 = ((const int32_t *)(dst->op_params))[2]; + + const int K_OC = (int) src0->ne[0]; + const int T_in = (int) src0->ne[1]; + const int K = K_OC / OC; + const int T_out = (int) dst->ne[0]; + + const uint3 T_out_fd = init_fastdiv_values((uint32_t)T_out); + + const int total = T_out * OC; + const int block_size = 256; + const int num_blocks = (total + block_size - 1) / block_size; + + switch (src0->type) { + case GGML_TYPE_F32: { + col2im_1d_kernel<<>>( + (const float *)src0->data, (float *)dst->data, + T_in, T_out_fd, OC, K, K_OC, s0, p0, total); + } break; + case GGML_TYPE_F16: { + col2im_1d_kernel<<>>( + (const half *)src0->data, (half *)dst->data, + T_in, T_out_fd, OC, K, K_OC, s0, p0, total); + } break; + case GGML_TYPE_BF16: { + col2im_1d_kernel<<>>( + (const nv_bfloat16 *)src0->data, (nv_bfloat16 *)dst->data, + T_in, T_out_fd, OC, K, K_OC, s0, p0, total); + } break; + default: + GGML_ABORT("col2im_1d: unsupported type"); + } +} diff --git a/ggml/src/ggml-cuda/col2im-1d.cuh b/ggml/src/ggml-cuda/col2im-1d.cuh new file mode 100644 index 0000000..efc3313 --- /dev/null +++ b/ggml/src/ggml-cuda/col2im-1d.cuh @@ -0,0 +1,3 @@ +#include "common.cuh" + +void ggml_cuda_op_col2im_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index ddf50ba..6b36ca9 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -5,7 +5,9 @@ #include "ggml-cuda.h" #include +#include #include +#include #if defined(GGML_USE_HIP) #define GGML_COMMON_DECL_HIP @@ -27,6 +29,7 @@ #include #include #include +#include #include #if defined(GGML_USE_HIP) @@ -49,7 +52,10 @@ #define GGML_CUDA_CC_VOLTA 700 #define GGML_CUDA_CC_TURING 750 #define GGML_CUDA_CC_AMPERE 800 +#define GGML_CUDA_CC_AMPERE_GA10X 860 +#define GGML_CUDA_CC_ORIN 870 #define GGML_CUDA_CC_ADA_LOVELACE 890 +#define GGML_CUDA_CC_HOPPER 900 // While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see // https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms #define GGML_CUDA_CC_BLACKWELL 1200 @@ -64,6 +70,8 @@ #define GGML_CUDA_CC_GCN4 (GGML_CUDA_CC_OFFSET_AMD + 0x803) // Tonga, Fiji, Polaris, minimum for fast fp16 #define GGML_CUDA_CC_VEGA (GGML_CUDA_CC_OFFSET_AMD + 0x900) // Vega56/64, minimum for fp16 dual issue #define GGML_CUDA_CC_VEGA20 (GGML_CUDA_CC_OFFSET_AMD + 0x906) // MI50/Radeon VII, minimum for dp4a +#define GGML_CUDA_CC_GFX909 (GGML_CUDA_CC_OFFSET_AMD + 0x909) // GCN APU +#define GGML_CUDA_CC_GFX90C (GGML_CUDA_CC_OFFSET_AMD + 0x90c) // GCN APU #define GGML_CUDA_CC_CDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x908) // MI100, minimum for MFMA, acc registers #define GGML_CUDA_CC_CDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x90a) // MI210 (gfx90a), minimum acc register renaming #define GGML_CUDA_CC_CDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x942) // MI300 @@ -84,12 +92,13 @@ #define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4) #define GGML_CUDA_CC_IS_RDNA3(cc) (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) #define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4) -#define GGML_CUDA_CC_IS_GCN(cc) (cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) -#define GGML_CUDA_CC_IS_CDNA(cc) (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1) -#define GGML_CUDA_CC_IS_CDNA1(cc) (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2) -#define GGML_CUDA_CC_IS_CDNA2(cc) (cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3) -#define GGML_CUDA_CC_IS_CDNA3(cc) (cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_CDNA4) -#define GGML_CUDA_CC_IS_CDNA4(cc) (cc >= GGML_CUDA_CC_CDNA4 && cc < GGML_CUDA_CC_RDNA1) +#define GGML_CUDA_CC_IS_GCN_APU(cc) ((cc) == GGML_CUDA_CC_GFX909 || (cc) == GGML_CUDA_CC_GFX90C) +#define GGML_CUDA_CC_IS_GCN(cc) ((cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) || GGML_CUDA_CC_IS_GCN_APU(cc)) +#define GGML_CUDA_CC_IS_CDNA(cc) (!GGML_CUDA_CC_IS_GCN_APU(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1) +#define GGML_CUDA_CC_IS_CDNA1(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2) +#define GGML_CUDA_CC_IS_CDNA2(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3) +#define GGML_CUDA_CC_IS_CDNA3(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_CDNA4) +#define GGML_CUDA_CC_IS_CDNA4(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA4 && cc < GGML_CUDA_CC_RDNA1) // Moore Threads #define MUSART_HMASK 40300 // MUSA rc4.3, min. ver. for half2 -> uint mask comparisons @@ -103,9 +112,36 @@ #define GGML_CUDA_CC_IS_QY2(cc) (cc >= GGML_CUDA_CC_QY2 && cc < GGML_CUDA_CC_PH1) #define GGML_CUDA_CC_IS_PH1(cc) (cc >= GGML_CUDA_CC_PH1) -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070 +#if !defined(GGML_USE_HIP) && (defined(GGML_USE_MUSA) || CUDART_VERSION >= 11070) # define GGML_CUDA_USE_CUB -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070 +#endif // !defined(GGML_USE_HIP) && (defined(GGML_USE_MUSA) || CUDART_VERSION >= 11070) + +// PDL host-side support (cudaLaunchKernelEx) requires CUDART >= 11.8. +// However, this has been bugged in CTK < 12.3 for MSVC builds, see +// https://github.com/ggml-org/llama.cpp/pull/22522#discussion_r3302393293 +// __CUDA_ARCH__ is undefined in host passes; GPU arch check happens in device-side code. +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && \ + (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080)) +# define GGML_CUDA_USE_PDL +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080)) + +static __device__ __forceinline__ void ggml_cuda_syncwarp() { +#ifndef GGML_USE_HIP + __syncwarp(); +#endif // GGML_USE_HIP +} + +static __device__ __forceinline__ void ggml_cuda_pdl_sync() { +#if defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER + cudaGridDependencySynchronize(); +#endif // defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER +} + +static __device__ __forceinline__ void ggml_cuda_pdl_lc() { +#if defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER + cudaTriggerProgrammaticLaunchCompletion(); +#endif // defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER +} #ifdef __CUDA_ARCH_LIST__ constexpr bool ggml_cuda_has_arch_impl(int) { @@ -165,6 +201,7 @@ void ggml_cuda_error(const char * stmt, const char * func, const char * file, in #define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString) + #if CUDART_VERSION >= 12000 || defined(GGML_USE_MUSA) static const char * cublas_get_error_str(const cublasStatus_t err) { return cublasGetStatusString(err); @@ -201,7 +238,7 @@ static const char * cu_get_error_str(CUresult err) { #define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str) #endif -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#if !defined(GGML_USE_HIP) # define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \ do { \ static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = { false }; \ @@ -216,7 +253,7 @@ static const char * cu_get_error_str(CUresult err) { do { \ GGML_UNUSED(nbytes); \ } while (0) -#endif // !(defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#endif // !defined(GGML_USE_HIP) #if CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA) #define GGML_CUDA_ASSUME(x) __builtin_assume(x) @@ -249,30 +286,26 @@ static const char * cu_get_error_str(CUresult err) { #define VOLTA_MMA_AVAILABLE #endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA -#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING #define TURING_MMA_AVAILABLE -#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING -#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE #define AMPERE_MMA_AVAILABLE -#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE #if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN # define BLACKWELL_MMA_AVAILABLE #endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL -#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE #define CP_ASYNC_AVAILABLE -#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE #if !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220) #define FLASH_ATTN_AVAILABLE #endif // !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220) -#if defined(TURING_MMA_AVAILABLE) -#define LDMATRIX_TRANS_AVAILABLE -#endif // defined(TURING_MMA_AVAILABLE) - static bool fp16_available(const int cc) { return ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_PASCAL || (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1); @@ -297,6 +330,12 @@ static bool fp16_mma_hardware_available(const int cc) { (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2); } +// To be used for feature selection of external libraries, e.g. cuBLAS. +static bool fast_bf16_hardware_available(const int cc) { + return (GGML_CUDA_CC_IS_AMD(cc) && (cc >= GGML_CUDA_CC_RDNA3 || GGML_CUDA_CC_IS_CDNA(cc))) + || (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE); +} + static bool bf16_mma_hardware_available(const int cc) { return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE) || GGML_CUDA_CC_IS_CDNA(cc) || cc >= GGML_CUDA_CC_RDNA3 || @@ -340,6 +379,15 @@ static bool blackwell_mma_available(const int cc) { ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN; } +// Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes. +static bool ggml_cuda_is_aligned(const ggml_tensor * tensor, const size_t alignment) { + GGML_ASSERT(tensor != nullptr); + return (reinterpret_cast(tensor->data) % alignment) == 0 && + tensor->nb[1] % alignment == 0 && + tensor->nb[2] % alignment == 0 && + tensor->nb[3] % alignment == 0; +} + static constexpr __device__ int ggml_cuda_get_physical_warp_size() { #if defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__)) return 64; @@ -350,7 +398,7 @@ static constexpr __device__ int ggml_cuda_get_physical_warp_size() { // Maximum number of bytes that can be copied in a single instruction. static constexpr __device__ int ggml_cuda_get_max_cpy_bytes() { -#ifdef GGML_USE_HIP +#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) return 16; #else #if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA @@ -358,7 +406,7 @@ static constexpr __device__ int ggml_cuda_get_max_cpy_bytes() { #else return 8; #endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA -#endif // GGML_USE_HIP +#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) } @@ -377,10 +425,6 @@ static __device__ void no_device_code( __trap(); GGML_UNUSED(no_device_code); // suppress unused function warning - -#if defined(GGML_USE_MUSA) - __builtin_unreachable(); -#endif // defined(GGML_USE_MUSA) } #ifdef __CUDA_ARCH__ @@ -410,7 +454,7 @@ struct ggml_cuda_unroll<1> { template static __device__ __forceinline__ int warp_reduce_sum(int x) { -#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE return __reduce_add_sync(0xffffffff, x); #else #pragma unroll @@ -418,7 +462,7 @@ static __device__ __forceinline__ int warp_reduce_sum(int x) { x += __shfl_xor_sync(0xffffffff, x, offset, width); } return x; -#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE } template @@ -596,7 +640,8 @@ template struct block_reduce_policy { }; template -static __device__ T block_reduce(T val, T * shared_vals) { +static __device__ T block_reduce(T val, [[maybe_unused]] T * shared_vals) { + // for multi-warp reductions, callers must not reuse shared_vals until all reads from this invocation have completed val = block_reduce_policy::reduce(val); const unsigned int block_size = block_size_template == 0 ? blockDim.x : block_size_template; if (block_size > WARP_SIZE) { @@ -648,16 +693,11 @@ static __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const hal template static __device__ __forceinline__ half2 warp_reduce_max(half2 x) { -#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL || defined(GGML_USE_HIP) #pragma unroll for (int offset = width/2; offset > 0; offset >>= 1) { x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, offset, width)); } return x; -#else - GGML_UNUSED(x); - NO_DEVICE_CODE; -#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL || defined(GGML_USE_HIP) } #if (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || \ @@ -834,6 +874,19 @@ static __device__ __forceinline__ float ggml_cuda_ue4m3_to_fp32(uint8_t x) { #endif // defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000 } +static __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_ue4m3(float x) { +#if defined(BLACKWELL_MMA_AVAILABLE) // This is used for NVFP4 subblock scale quantizations only + if (!(x > 0.0f)) { + return 0; + } + const __nv_fp8_e4m3 xf(x); + return xf.__x; +#else + GGML_UNUSED(x); + NO_DEVICE_CODE; // Used only for NVFP4 Scales for Activations, only for Blackwell +#endif // defined(BLACKWELL_MMA_AVAILABLE) +} + __device__ __forceinline__ uint8_t ggml_cuda_float_to_fp4_e2m1(float x, float e) { const uint8_t sign_bit = (x < 0.0f) << 3; float ax = fabsf(x) * e; @@ -903,6 +956,9 @@ static __device__ __forceinline__ uint2 fast_div_modulo(uint32_t n, const uint3 typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, float2 & v); +template +using dequantize_kq_t = void (*)(const void * vx, const int64_t ib, dst_t * y, const int tid); + static __device__ __forceinline__ float get_alibi_slope( const float max_bias, const uint32_t h, const uint32_t n_head_log2, const float m0, const float m1 ) { @@ -922,6 +978,7 @@ template<> struct ggml_cuda_type_traits { static constexpr int qk = 1; static constexpr int qr = 1; + static constexpr int bs = sizeof(ggml_half); }; template<> @@ -929,6 +986,15 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK1_0; static constexpr int qr = QR1_0; static constexpr int qi = QI1_0; + static constexpr int bs = sizeof(block_q1_0); +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK2_0; + static constexpr int qr = QR2_0; + static constexpr int qi = QI2_0; + static constexpr int bs = sizeof(block_q2_0); }; template<> @@ -936,6 +1002,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK4_0; static constexpr int qr = QR4_0; static constexpr int qi = QI4_0; + static constexpr int bs = sizeof(block_q4_0); }; template<> @@ -943,6 +1010,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK4_1; static constexpr int qr = QR4_1; static constexpr int qi = QI4_1; + static constexpr int bs = sizeof(block_q4_1); }; template<> @@ -950,6 +1018,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK5_0; static constexpr int qr = QR5_0; static constexpr int qi = QI5_0; + static constexpr int bs = sizeof(block_q5_0); }; template<> @@ -957,6 +1026,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK5_1; static constexpr int qr = QR5_1; static constexpr int qi = QI5_1; + static constexpr int bs = sizeof(block_q5_1); }; template<> @@ -964,6 +1034,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK8_0; static constexpr int qr = QR8_0; static constexpr int qi = QI8_0; + static constexpr int bs = sizeof(block_q8_0); }; template<> @@ -971,6 +1042,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_MXFP4; static constexpr int qr = QR_MXFP4; static constexpr int qi = QI_MXFP4; + static constexpr int bs = sizeof(block_mxfp4); }; template<> @@ -978,6 +1050,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_NVFP4; static constexpr int qr = QR_NVFP4; static constexpr int qi = QI_NVFP4; + static constexpr int bs = sizeof(block_nvfp4); }; template<> @@ -985,6 +1058,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR2_K; static constexpr int qi = QI2_K; + static constexpr int bs = sizeof(block_q2_K); }; template<> @@ -992,6 +1066,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR3_K; static constexpr int qi = QI3_K; + static constexpr int bs = sizeof(block_q3_K); }; template<> @@ -999,6 +1074,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR4_K; static constexpr int qi = QI4_K; + static constexpr int bs = sizeof(block_q4_K); }; template<> @@ -1006,6 +1082,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR5_K; static constexpr int qi = QI5_K; + static constexpr int bs = sizeof(block_q5_K); }; template<> @@ -1013,6 +1090,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR6_K; static constexpr int qi = QI6_K; + static constexpr int bs = sizeof(block_q6_K); }; template<> @@ -1020,6 +1098,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR2_XXS; static constexpr int qi = QI2_XXS; + static constexpr int bs = sizeof(block_iq2_xxs); }; template<> @@ -1027,6 +1106,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR2_XS; static constexpr int qi = QI2_XS; + static constexpr int bs = sizeof(block_iq2_xs); }; template<> @@ -1034,6 +1114,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR2_S; static constexpr int qi = QI2_S; + static constexpr int bs = sizeof(block_iq2_s); }; template<> @@ -1041,6 +1122,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR3_XXS; static constexpr int qi = QI3_XXS; + static constexpr int bs = sizeof(block_iq3_xxs); }; template<> @@ -1048,6 +1130,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR1_S; static constexpr int qi = QI1_S; + static constexpr int bs = sizeof(block_iq1_s); }; template<> @@ -1055,6 +1138,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR1_M; static constexpr int qi = QI1_M; + static constexpr int bs = sizeof(block_iq1_m); }; template<> @@ -1062,6 +1146,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK4_NL; static constexpr int qr = QR4_NL; static constexpr int qi = QI4_NL; + static constexpr int bs = sizeof(block_iq4_nl); }; template<> @@ -1069,6 +1154,7 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR4_XS; static constexpr int qi = QI4_XS; + static constexpr int bs = sizeof(block_iq4_xs); }; template<> @@ -1076,12 +1162,14 @@ struct ggml_cuda_type_traits { static constexpr int qk = QK_K; static constexpr int qr = QR3_S; static constexpr int qi = QI3_S; + static constexpr int bs = sizeof(block_iq3_s); }; ////////////////////// struct ggml_cuda_device_info { - int device_count; + int device_count; // number of (possibly virtual) devices exposed to the rest of ggml + int physical_device_count; // number of physical CUDA devices actually present struct cuda_device_info { int cc; // compute capability @@ -1094,6 +1182,9 @@ struct ggml_cuda_device_info { size_t total_vram; int warp_size; // Number of threads in a dispatch bool supports_cooperative_launch; // whether cooperative launch is supported + int physical_device; // backing physical CUDA device for this (virtual) device + int physical_share_count; // number of (virtual) devices sharing this device's physical GPU + int virtual_index; // index of this (virtual) device among those sharing its physical GPU }; cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {}; @@ -1173,15 +1264,12 @@ struct ggml_tensor_extra_gpu { struct ggml_cuda_graph { #ifdef USE_CUDA_GRAPH ~ggml_cuda_graph() { - if (instance != nullptr) { + for (cudaGraphExec_t instance : instances) { CUDA_CHECK(cudaGraphExecDestroy(instance)); } - if (graph != nullptr) { - CUDA_CHECK(cudaGraphDestroy(graph)); - } } - cudaGraph_t graph = nullptr; - cudaGraphExec_t instance = nullptr; + // the ggml graph is captured as a series of CUDA graphs, so the GPU can run one chunk while the next is enqueued + std::vector instances; size_t num_nodes = 0; std::vector nodes; bool disable_due_to_gpu_arch = false; @@ -1359,8 +1447,10 @@ struct ggml_backend_cuda_context { std::string name; cudaEvent_t copy_event = nullptr; - cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } }; - cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr}; + cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {}; + cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {}; + void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {}; + size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {}; int curr_stream_no = 0; @@ -1408,7 +1498,7 @@ struct ggml_backend_cuda_context { // Check if any CUDA graph has an instance for this context bool any_cuda_graph_has_instance() const { for (const auto & [key, graph] : cuda_graphs) { - if (graph && graph->instance != nullptr) { + if (graph && !graph->instances.empty()) { return true; } } @@ -1437,27 +1527,32 @@ struct ggml_backend_cuda_context { ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; } - cublasHandle_t cublas_handle(int device) { - if (cublas_handles[device] == nullptr) { + cublasHandle_t cublas_handle() { + if (cublas_handles[device][curr_stream_no] == nullptr) { ggml_cuda_set_device(device); - CUBLAS_CHECK(cublasCreate(&cublas_handles[device])); - CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no])); + CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream())); +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2)) + if (cublas_workspace_sizes[device] == 0) { + const int cc = ggml_cuda_info().devices[device].cc; + cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024; + } + CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); + CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device])); +#endif } - return cublas_handles[device]; - } - - cublasHandle_t cublas_handle() { - return cublas_handle(device); + return cublas_handles[device][curr_stream_no]; } // pool std::unique_ptr pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; - static std::unique_ptr new_pool_for_device(int device, int stream_no); + static std::unique_ptr new_pool_for_device(ggml_backend_cuda_context & ctx, int device, int stream_no); ggml_cuda_pool & pool(int device) { if (pools[device][curr_stream_no] == nullptr) { - pools[device][curr_stream_no] = new_pool_for_device(device, curr_stream_no); + pools[device][curr_stream_no] = new_pool_for_device(*this, device, curr_stream_no); } return *pools[device][curr_stream_no]; } @@ -1471,11 +1566,149 @@ struct ggml_cuda_mm_fusion_args_host { const ggml_tensor * x_bias = nullptr; const ggml_tensor * gate = nullptr; const ggml_tensor * gate_bias = nullptr; + const ggml_tensor * x_scale = nullptr; + const ggml_tensor * gate_scale = nullptr; ggml_glu_op glu_op; + float glu_limit = 0.0f; + const ggml_tensor * shared_up = nullptr; + const ggml_tensor * shared_gate = nullptr; + ggml_tensor * shared_dst = nullptr; }; struct ggml_cuda_mm_fusion_args_device { const void * x_bias = nullptr; const void * gate = nullptr; const void * gate_bias = nullptr; + const void * x_scale = nullptr; + const void * gate_scale = nullptr; ggml_glu_op glu_op; + float glu_limit = 0.0f; + const void * shared_up = nullptr; + const void * shared_gate = nullptr; + float * shared_dst = nullptr; + uint32_t shared_stride_col_dst = 0; +}; + +struct ggml_cuda_kernel_launch_params { + dim3 block_nums; + dim3 block_dims; + size_t shmem; + cudaStream_t stream; + + // size_t shmem + ggml_cuda_kernel_launch_params(const dim3& block_nums_, const dim3& block_dims_, const size_t shmem_, const cudaStream_t stream_) + : block_nums(block_nums_), block_dims(block_dims_), shmem(shmem_), stream(stream_) {} + + // Some call sites pass ints instead of the required size_t. This 2nd constructor casts int->size_t to avoid these -Wnarrowing warnings. + ggml_cuda_kernel_launch_params(const dim3& block_nums_, const dim3& block_dims_, const int shmem_, const cudaStream_t stream_) + : block_nums(block_nums_), block_dims(block_dims_), shmem((size_t)shmem_), stream(stream_) {} }; + +#if defined(GGML_CUDA_USE_PDL) +struct ggml_cuda_pdl_config { + cudaLaunchAttribute attr; + cudaLaunchConfig_t cfg; + + ggml_cuda_pdl_config(const ggml_cuda_kernel_launch_params & params) { + attr.id = cudaLaunchAttributeProgrammaticStreamSerialization; + attr.val.programmaticStreamSerializationAllowed = 1; + + cfg = {}; + cfg.gridDim = params.block_nums; + cfg.blockDim = params.block_dims; + cfg.dynamicSmemBytes = params.shmem; + cfg.stream = params.stream; + cfg.attrs = &attr; + cfg.numAttrs = 1; + } + + // Delete due to &attr + ggml_cuda_pdl_config(const ggml_cuda_pdl_config&) = delete; + ggml_cuda_pdl_config& operator=(const ggml_cuda_pdl_config&) = delete; + ggml_cuda_pdl_config& operator=(ggml_cuda_pdl_config&&) = delete; + +}; + +static bool ggml_cuda_kernel_can_use_pdl(const void * kernel) { + const int device = ggml_cuda_get_device(); + + struct cache_key { + int device; + const void * kernel; + + bool operator==(const cache_key & other) const { return device == other.device && kernel == other.kernel; } + }; + + struct cache_key_hash { + // MurmurHash3 mixing function for better hash distribution (vs. just std::hash which in some implementations simply returns the identity) + static size_t hash_mix(size_t x) { + std::uint64_t y = x; + const std::uint64_t m = 0xe9846af9b1a615d; + + y ^= y >> 32; + y *= m; + y ^= y >> 32; + y *= m; + y ^= y >> 28; + + return static_cast(y); + } + + size_t operator()(const cache_key & key) const { + // Use a nonzero seed to avoid mapping all-zero keys to zero + size_t h = 42; + h = hash_mix(h + key.device); + h = hash_mix(h + reinterpret_cast(key.kernel)); + return h; + } + }; + + static std::mutex cache_mutex; + static std::unordered_map cache; + + const cache_key key = { device, kernel }; + std::lock_guard lock(cache_mutex); + const auto it = cache.find(key); + if (it != cache.end()) { + return it->second; + } + + cudaFuncAttributes attr = {}; + CUDA_CHECK(cudaFuncGetAttributes(&attr, kernel)); + + // PDL device-side primitives are emitted only for PTX versions >= 90. + // We have to guard on a loaded kernel's PTX version so a kernel forward-JIT'ed + // from pre-Hopper PTX to a Hopper-or-newer GPU does not opt into PDL. + const bool can_use_pdl = attr.ptxVersion >= 90; + cache.emplace(key, can_use_pdl); + return can_use_pdl; +} + +#endif //defined(GGML_CUDA_USE_PDL) + +// PDL and __restrict__ need to be mutually exclusive, see https://github.com/ggml-org/llama.cpp/pull/24030 +# if (defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER) +# define GGML_CUDA_RESTRICT +# else +# define GGML_CUDA_RESTRICT __restrict__ +# endif // defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER + +template +static __inline__ void ggml_cuda_kernel_launch(Kernel kernel, const ggml_cuda_kernel_launch_params & launch_params, Args&&... args) { +#if defined(GGML_CUDA_USE_PDL) + + static const bool env_pdl_enabled = []() { + const char * env = getenv("GGML_CUDA_PDL"); + return env == nullptr || std::atoi(env) != 0; + }(); + + if (env_pdl_enabled && ggml_cuda_kernel_can_use_pdl(reinterpret_cast(kernel))) { + auto pdl_cfg = ggml_cuda_pdl_config(launch_params); + + CUDA_CHECK(cudaLaunchKernelEx(&pdl_cfg.cfg, kernel, std::forward(args)... )); + return; + } +#endif //defined(GGML_CUDA_USE_PDL) + + kernel<<>>(std::forward(args)... ); + CUDA_CHECK(cudaGetLastError()); +} diff --git a/ggml/src/ggml-cuda/concat.cu b/ggml/src/ggml-cuda/concat.cu index e9ffd27..f8dcd95 100644 --- a/ggml/src/ggml-cuda/concat.cu +++ b/ggml/src/ggml-cuda/concat.cu @@ -1,102 +1,88 @@ #include "concat.cuh" -// contiguous kernels -static __global__ void concat_f32_dim0(const float * x, const float * y, float * dst, const int ne0, const int ne00) { - int nidx = threadIdx.x + blockIdx.x * blockDim.x; - if (nidx >= ne0) { - return; - } - - int offset_dst = - nidx + - blockIdx.y * ne0 + - blockIdx.z * ne0 * gridDim.y; - - if (nidx < ne00) { // src0 - int offset_src = - nidx + - blockIdx.y * ne00 + - blockIdx.z * ne00 * gridDim.y; - dst[offset_dst] = x[offset_src]; - } else { - int offset_src = - (nidx - ne00) + - blockIdx.y * (ne0 - ne00) + - blockIdx.z * (ne0 - ne00) * gridDim.y; - dst[offset_dst] = y[offset_src]; - } -} +#include -static __global__ void concat_f32_dim1(const float * x, const float * y, float * dst, const int ne0, const int ne01) { - int nidx = threadIdx.x + blockIdx.x * blockDim.x; - if (nidx >= ne0) { - return; - } +// contiguous kernels +template +static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) concat_cont(const T * x, + const T * y, + T * dst, + int64_t ne00, + int64_t ne01, + int64_t ne02, + int64_t ne0, + int64_t ne1, + int64_t ne2) { + static_assert(dim >= 0 && dim <= 2, "dim must be in [0, 2]"); + + const int64_t n = ne0 * ne1 * ne2; + + ggml_cuda_pdl_sync(); + for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) blockDim.x * gridDim.x) { + if constexpr (dim == 0) { + const int64_t row = i / ne0; + const int64_t i0 = i - row * ne0; + + if (i0 < ne00) { + dst[i] = x[row * ne00 + i0]; + } else { + dst[i] = y[row * (ne0 - ne00) + (i0 - ne00)]; + } + } else if constexpr (dim == 1) { + const int64_t dst_plane = ne0 * ne1; + const int64_t src0_plane = ne0 * ne01; + const int64_t src1_plane = dst_plane - src0_plane; + const int64_t i2 = i / dst_plane; + const int64_t i01 = i - i2 * dst_plane; + + if (i01 < src0_plane) { + dst[i] = x[i2 * src0_plane + i01]; + } else { + dst[i] = y[i2 * src1_plane + (i01 - src0_plane)]; + } + } else { + const int64_t src0_size = ne0 * ne1 * ne02; - int offset_dst = - nidx + - blockIdx.y * ne0 + - blockIdx.z * ne0 * gridDim.y; - - if (blockIdx.y < (unsigned)ne01) { // src0 - int offset_src = - nidx + - blockIdx.y * ne0 + - blockIdx.z * ne0 * ne01; - dst[offset_dst] = x[offset_src]; - } else { - int offset_src = - nidx + - (blockIdx.y - ne01) * ne0 + - blockIdx.z * ne0 * (gridDim.y - ne01); - dst[offset_dst] = y[offset_src]; + if (i < src0_size) { + dst[i] = x[i]; + } else { + dst[i] = y[i - src0_size]; + } + } } } -static __global__ void concat_f32_dim2(const float * x, const float * y, float * dst, const int ne0, const int ne02) { - int nidx = threadIdx.x + blockIdx.x * blockDim.x; - if (nidx >= ne0) { - return; - } - - int offset_dst = - nidx + - blockIdx.y * ne0 + - blockIdx.z * ne0 * gridDim.y; - - if (blockIdx.z < (unsigned)ne02) { // src0 - int offset_src = - nidx + - blockIdx.y * ne0 + - blockIdx.z * ne0 * gridDim.y; - dst[offset_dst] = x[offset_src]; - } else { - int offset_src = - nidx + - blockIdx.y * ne0 + - (blockIdx.z - ne02) * ne0 * gridDim.y; - dst[offset_dst] = y[offset_src]; - } -} +template +static void concat_cont_cuda(const T * x, + const T * y, + T * dst, + int64_t ne00, + int64_t ne01, + int64_t ne02, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int dim, + cudaStream_t stream) { + const int64_t n = ne0 * ne1 * ne2; + const int num_blocks = (n + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE; -static void concat_f32_cuda(const float * x, const float * y, float * dst, int ne00, int ne01, int ne02, int ne0, int ne1, int ne2, int dim, cudaStream_t stream) { - int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE; - dim3 gridDim(num_blocks, ne1, ne2); if (dim == 0) { - concat_f32_dim0<<>>(x, y, dst, ne0, ne00); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, CUDA_CONCAT_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(concat_cont, launch_params, x, y, dst, ne00, ne01, ne02, ne0, ne1, ne2); return; } if (dim == 1) { - concat_f32_dim1<<>>(x, y, dst, ne0, ne01); + concat_cont<<>>(x, y, dst, ne00, ne01, ne02, ne0, ne1, ne2); return; } - concat_f32_dim2<<>>(x, y, dst, ne0, ne02); + concat_cont<<>>(x, y, dst, ne00, ne01, ne02, ne0, ne1, ne2); } // non-contiguous kernel (slow) -template +template static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) - concat_f32_non_cont( + concat_non_cont( const char * src0, const char * src1, char * dst, @@ -123,75 +109,63 @@ static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) uint64_t nb0, uint64_t nb1, uint64_t nb2, - uint64_t nb3){ + uint64_t nb3) { static_assert(dim >= 0 && dim <= 3, "dim must be in [0, 3]"); const int64_t i3 = blockIdx.z; const int64_t i2 = blockIdx.y; const int64_t i1 = blockIdx.x; - const float * x; + const T * x; for (int64_t i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) { if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); + x = (const T *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); } else { if constexpr (dim == 0) { - x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + i1 * nb11 + (i0 - ne00) * nb10); + x = (const T *)(src1 + i3*nb13 + i2*nb12 + i1*nb11 + (i0 - ne00)*nb10); } else if constexpr (dim == 1) { - x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + (i1 - ne01) * nb11 + i0 * nb10); + x = (const T *)(src1 + i3*nb13 + i2*nb12 + (i1 - ne01)*nb11 + i0*nb10); } else if constexpr (dim == 2) { - x = (const float *) (src1 + i3 * nb13 + (i2 - ne02) * nb12 + i1 * nb11 + i0 * nb10); + x = (const T *)(src1 + i3*nb13 + (i2 - ne02)*nb12 + i1*nb11 + i0*nb10); } else if constexpr (dim == 3) { - x = (const float *) (src1 + (i3 - ne03) * nb13 + i2 * nb12 + i1 * nb11 + i0 * nb10); + x = (const T *)(src1 + (i3 - ne03)*nb13 + i2*nb12 + i1*nb11 + i0*nb10); } } - float * y = (float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + T * y = (T *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); *y = *x; } } - -void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - cudaStream_t stream = ctx.stream(); - - const int32_t dim = ((int32_t *) dst->op_params)[0]; - - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); - - if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { - const float * src0_d = (const float *)src0->data; - const float * src1_d = (const float *)src1->data; - - float * dst_d = (float *)dst->data; - - if (dim != 3) { - for (int i3 = 0; i3 < dst->ne[3]; i3++) { - concat_f32_cuda( - src0_d + i3 * (src0->nb[3] / 4), - src1_d + i3 * (src1->nb[3] / 4), - dst_d + i3 * ( dst->nb[3] / 4), - src0->ne[0], src0->ne[1], src0->ne[2], - dst->ne[0], dst->ne[1], dst->ne[2], dim, stream); - } - } else { - const size_t size0 = ggml_nbytes(src0); - const size_t size1 = ggml_nbytes(src1); - - CUDA_CHECK(cudaMemcpyAsync(dst_d, src0_d, size0, cudaMemcpyDeviceToDevice, stream)); - CUDA_CHECK(cudaMemcpyAsync(dst_d + size0/4, src1_d, size1, cudaMemcpyDeviceToDevice, stream)); +template +static void concat_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, int dim, cudaStream_t stream) { + if (dim != 3 && ggml_is_contiguous_to_3(src0) && ggml_is_contiguous_to_3(src1)) { + const T * src0_d = (const T *) src0->data; + const T * src1_d = (const T *) src1->data; + T * dst_d = (T *) dst->data; + + for (int64_t i3 = 0; i3 < dst->ne[3]; i3++) { + concat_cont_cuda( + src0_d + i3*(src0->nb[3] / sizeof(T)), + src1_d + i3*(src1->nb[3] / sizeof(T)), + dst_d + i3*( dst->nb[3] / sizeof(T)), + ggml_row_size(src0->type, src0->ne[0])/sizeof(T), src0->ne[1], src0->ne[2], + ggml_row_size(dst->type, dst->ne[0])/sizeof(T), dst->ne[1], dst->ne[2], dim, stream); } + } else if (dim == 3 && ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { + const size_t size0 = ggml_nbytes(src0); + const size_t size1 = ggml_nbytes(src1); + + CUDA_CHECK(cudaMemcpyAsync((char *) dst->data, src0->data, size0, cudaMemcpyDeviceToDevice, stream)); + CUDA_CHECK(cudaMemcpyAsync((char *) dst->data + size0, src1->data, size1, cudaMemcpyDeviceToDevice, stream)); } else { + GGML_ASSERT(!ggml_is_quantized(src0->type)); + dim3 grid_dim(dst->ne[1], dst->ne[2], dst->ne[3]); auto launch_kernel = [&](auto dim) { - concat_f32_non_cont<<>>( + concat_non_cont<<>>( (const char *) src0->data, (const char *) src1->data, (char *) dst->data, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], @@ -215,7 +189,52 @@ void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { break; default: GGML_ABORT("Invalid dim: %d", dim); + } + } +} + +void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + cudaStream_t stream = ctx.stream(); + + const int32_t dim = ((int32_t *) dst->op_params)[0]; + + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(dst->type == src0->type); + + if (ggml_is_quantized(src0->type)) { + if (dim == 3) { + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); + } else { + GGML_ASSERT(ggml_is_contiguous_to_3(src0)); + GGML_ASSERT(ggml_is_contiguous_to_3(src1)); + } + GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0); + GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0); + + // if first 3 dimensions are contiguous and ne[0] is multiple of the block size we can concat both tensors as byte tensors + concat_cuda(src0, src1, dst, dim, stream); + } else { + GGML_ASSERT(ggml_blck_size(src0->type) == 1); + + switch (ggml_type_size(src0->type)) { + case 1: + concat_cuda(src0, src1, dst, dim, stream); + break; + case 2: + concat_cuda(src0, src1, dst, dim, stream); break; + case 4: + concat_cuda(src0, src1, dst, dim, stream); + break; + case 8: + concat_cuda(src0, src1, dst, dim, stream); + break; + default: + GGML_ABORT("Unsupported type size: %zu", ggml_type_size(src0->type)); } } } diff --git a/ggml/src/ggml-cuda/conv-transpose-1d.cu b/ggml/src/ggml-cuda/conv-transpose-1d.cu index 8418ba6..ebf2aa8 100644 --- a/ggml/src/ggml-cuda/conv-transpose-1d.cu +++ b/ggml/src/ggml-cuda/conv-transpose-1d.cu @@ -11,30 +11,32 @@ static __global__ void conv_transpose_1d_kernel( return; } - int out_index = global_index / dst_ne0; + int out_t = global_index % dst_ne0; + int out_ch = (global_index / dst_ne0) % dst_ne1; + int plane = global_index / (dst_ne0 * dst_ne1); float accumulator = 0; for (int c = 0; c < src0_ne2; c++) { - int idx = global_index % dst_ne0; + int kernel_offset = src0_ne0 * (out_ch + src0_ne1 * c); + int input_offset = src1_ne0 * (c + src1_ne1 * plane); - int kernel_offset = (src0_ne0 * src0_ne1 * c) + (out_index * src0_ne0); - int input_offset = src1_ne0 * c; - - for (int i = 0; i < src1_ne0; i++) { - if (!(idx >= i*s0 && idx < i*s0 + src0_ne0)) { + for (int k = 0; k < src0_ne0; k++) { + int input_numer = out_t + p0 - k*d0; + if (input_numer < 0 || input_numer % s0 != 0) { continue; } - int weight_idx = idx - i*s0; - float kernel_weight = src0[kernel_offset + weight_idx]; - float input_value = src1[input_offset+i]; + int input_t = input_numer / s0; + if (input_t >= src1_ne0) { + continue; + } - accumulator += kernel_weight * input_value; + accumulator += src0[kernel_offset + k] * src1[input_offset + input_t]; } } dst[global_index] = accumulator; - GGML_UNUSED_VARS(p0, d0, src0_ne3, src1_ne3, dst_ne3, src1_ne1, dst_ne1, src1_ne2, dst_ne2); + GGML_UNUSED_VARS(src0_ne3, src1_ne2, src1_ne3, dst_ne2, dst_ne3); } static void conv_transpose_1d_f32_f32_cuda( diff --git a/ggml/src/ggml-cuda/conv2d-dw.cu b/ggml/src/ggml-cuda/conv2d-dw.cu index 7583233..6371d0a 100644 --- a/ggml/src/ggml-cuda/conv2d-dw.cu +++ b/ggml/src/ggml-cuda/conv2d-dw.cu @@ -1,4 +1,5 @@ #include "conv2d-dw.cuh" +#include "convert.cuh" struct conv_params { int in_w, in_h; @@ -79,7 +80,7 @@ struct cwhn_layout { }; template -__global__ void conv2d_dw_kernel(const T * __restrict__ input, const T * __restrict__ kernel, T * __restrict__ output, +__global__ void conv2d_dw_kernel(const float * __restrict__ input, const T * __restrict__ kernel, float * __restrict__ output, const int in_w, const int in_h, const int out_w, const int out_h, const int kernel_w, const int kernel_h, const int stride_x, const int stride_y, const int padding_x, const int padding_y, const int dilation_x, const int dilation_y, @@ -97,7 +98,7 @@ __global__ void conv2d_dw_kernel(const T * __restrict__ input, const T * __restr int batch_idx, channel_idx, out_y_idx, out_x_idx; Layout::unpack_indices(global_idx, params, batch_idx, channel_idx, out_y_idx, out_x_idx); - T accumulator = 0; + float accumulator = 0.0f; kernel_bounds bounds = calculate_kernel_bounds(out_x_idx, out_y_idx, params); for (int kern_y = bounds.y_min; kern_y < bounds.y_max; ++kern_y) { @@ -106,10 +107,10 @@ __global__ void conv2d_dw_kernel(const T * __restrict__ input, const T * __restr for (int kern_x = bounds.x_min; kern_x < bounds.x_max; ++kern_x) { int in_x_idx = calculate_input_coord(out_x_idx, kern_x, params.stride_x, params.dilation_x, params.padding_x); - const T input_val = input[Layout::input_index(batch_idx, channel_idx, in_y_idx, in_x_idx, params)]; - const T kernel_val = kernel[Layout::kernel_index(channel_idx, kern_y, kern_x, params)]; + const float input_val = input[Layout::input_index(batch_idx, channel_idx, in_y_idx, in_x_idx, params)]; + const T kernel_val = kernel[Layout::kernel_index(channel_idx, kern_y, kern_x, params)]; - accumulator += input_val * kernel_val; + accumulator += input_val * ggml_cuda_cast(kernel_val); } } @@ -120,8 +121,9 @@ void ggml_cuda_op_conv2d_dw(ggml_backend_cuda_context & ctx, ggml_tensor * dst) const ggml_tensor * kernel = dst->src[0]; const ggml_tensor * input = dst->src[1]; - GGML_ASSERT(kernel->type == GGML_TYPE_F32 && input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); - const float * w_d = (const float *) kernel->data; + GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); + GGML_ASSERT(input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + const void * w_d = kernel->data; const float * x_d = (const float *) input->data; float * y_d = (float *) dst->data; @@ -148,13 +150,25 @@ void ggml_cuda_op_conv2d_dw(ggml_backend_cuda_context & ctx, ggml_tensor * dst) const int blocks = (total + CUDA_CONV2D_DW_BLOCK_SIZE - 1) / CUDA_CONV2D_DW_BLOCK_SIZE; if (ggml_is_contiguous(input)) { - conv2d_dw_kernel<<>>( - x_d, w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, padding_x, padding_y, - dilation_x, dilation_y, channels, batches); + if (kernel->type == GGML_TYPE_F16) { + conv2d_dw_kernel<<>>( + x_d, (const half *) w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, + padding_x, padding_y, dilation_x, dilation_y, channels, batches); + } else { + conv2d_dw_kernel<<>>( + x_d, (const float *) w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, + padding_x, padding_y, dilation_x, dilation_y, channels, batches); + } } else if (ggml_is_contiguous_channels(input)) { - conv2d_dw_kernel<<>>( - x_d, w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, padding_x, padding_y, - dilation_x, dilation_y, channels, batches); + if (kernel->type == GGML_TYPE_F16) { + conv2d_dw_kernel<<>>( + x_d, (const half *) w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, + padding_x, padding_y, dilation_x, dilation_y, channels, batches); + } else { + conv2d_dw_kernel<<>>( + x_d, (const float *) w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, + padding_x, padding_y, dilation_x, dilation_y, channels, batches); + } } else { GGML_ABORT("Unsupported memory layout for conv_2d_dw"); } diff --git a/ggml/src/ggml-cuda/conv2d.cu b/ggml/src/ggml-cuda/conv2d.cu index 142dd66..9636db2 100644 --- a/ggml/src/ggml-cuda/conv2d.cu +++ b/ggml/src/ggml-cuda/conv2d.cu @@ -1,5 +1,6 @@ #include "conv2d.cuh" #include "convert.cuh" +#include "mma.cuh" struct conv_params { const int64_t IW, IH; @@ -111,6 +112,220 @@ static void conv2d_cuda(const float * X_D, const T * K_D, float * Y_D, const con conv2d_kernel<<>>(X_D, K_D, Y_D, P); } +static __global__ void +conv2d_pad_f16(const float * input, half * output, int iw, int ih, int pw, int ph, int px, int py, int total) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= total) { + return; + } + const int x = i % pw - px, y = i / pw % ph - py, nc = i / (pw * ph); + output[i] = __float2half( + (unsigned) x < (unsigned) iw && (unsigned) y < (unsigned) ih ? input[(nc * ih + y) * iw + x] : 0.0f); +} + +template +static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input, + const half * __restrict__ weight, + float * __restrict__ output, + const conv_params P, + const int split_k) { + using namespace ggml_cuda_mma; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nthreads = 4 * warp_size; + constexpr int BM = 64, BN = 64, BK = 64; + constexpr int AS = BK / 2 + 4; + constexpr int BS = BN / 2 + 4; + __shared__ __align__(16) half2 a_s[BM][AS]; + __shared__ __align__(16) half2 b_s[BK][BS]; + + const int tid = threadIdx.y * warp_size + threadIdx.x; + const int iw = int(P.IW), ih = int(P.IH), ow = int(P.OW), oh = int(P.OH); + const int kw = KW ? KW : int(P.KW), kh = KH ? KH : int(P.KH); + const int ic = int(P.IC), oc = int(P.OC); + const int sx = int(P.ST_X), sy = int(P.ST_Y); + const int dx = int(P.DL_X), dy = int(P.DL_Y); + const int n = blockIdx.z / split_k, split = blockIdx.z % split_k; + const int m0 = blockIdx.y * BM, n0 = blockIdx.x * BN; + + const int k_total = ic * kw * kh; + const int load_lane = warp_size == 32 ? threadIdx.x : threadIdx.x % (BN / 2); + const int load_row = threadIdx.y * (warp_size / (BN / 2)) + (warp_size == 32 ? 0 : threadIdx.x / (BN / 2)); + const int spatial = n0 + 2 * load_lane; + const int spatial0 = min(spatial, ow * oh - 1), spatial1 = min(spatial + 1, ow * oh - 1); + const int y0 = spatial0 / ow, x0 = spatial0 % ow; + const int y1 = spatial1 / ow, x1 = spatial1 % ow; + const int pos0 = y0 * sy * iw + x0 * sx, pos1 = y1 * sy * iw + x1 * sx; + + [[maybe_unused]] const int wm = threadIdx.y / 2 * 32, wn = threadIdx.y % 2 * 32; +#if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + using tile_ab = tile<16, 8, half2, get_input_data_layout()>; +# if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + // AMD accumulator fragments transpose the input fragment's row/column mapping. + using tile_c = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>; +# else + using tile_c = tile<16, 16, float>; +# endif + [[maybe_unused]] tile_c c[2][2]; +#else + if constexpr (use_mma) { + NO_DEVICE_CODE; + return; + } +#endif + constexpr int RM = 4, RN = BM * BN / (nthreads * RM); + [[maybe_unused]] const int simt_m = tid / (BN / RN) * RM, simt_n = tid % (BN / RN) * RN; + [[maybe_unused]] float c_simt[RM][RN] = {}; + const int tiles = (k_total + BK - 1) / BK; + const int begin = int(int64_t(tiles) * split / split_k) * BK; + const int end = int(int64_t(tiles) * (split + 1) / split_k) * BK; + for (int k0 = begin; k0 < end; k0 += BK) { + if (k_total % 8 == 0 && uintptr_t(weight) % 16 == 0) { +#pragma unroll + for (int i = tid; i < BM * BK / 8; i += nthreads) { + const int row = i / (BK / 8), col = 8 * (i % (BK / 8)); + const int4 v = m0 + row < oc && k0 + col < k_total ? + ((const int4 *) weight)[((m0 + row) * k_total + k0 + col) / 8] : + make_int4(0, 0, 0, 0); + *(int4 *) &a_s[row][col / 2] = v; + } + } else { +#pragma unroll + for (int i = tid; i < BM * BK / 2; i += nthreads) { + const int row = i / (BK / 2), col = 2 * (i % (BK / 2)); + half lo = __float2half(0.0f), hi = lo; + if (m0 + row < oc && k0 + col < k_total) { + lo = weight[(m0 + row) * k_total + k0 + col]; + if (k0 + col + 1 < k_total) { + hi = weight[(m0 + row) * k_total + k0 + col + 1]; + } + } + a_s[row][col / 2] = __halves2half2(lo, hi); + } + } +#pragma unroll + for (int k = load_row; k < BK; k += nthreads / (BN / 2)) { + const int ki = k0 + k; + const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw; + const int offset = ki < k_total ? (n * ic + ci) * ih * iw + ky * dy * iw + kx * dx : 0; + half lo = __float2half(0.0f), hi = lo; + if (ki < k_total && spatial < ow * oh) { + lo = input[offset + pos0]; + } + if (ki < k_total && spatial + 1 < ow * oh) { + hi = input[offset + pos1]; + } + b_s[k][load_lane] = __halves2half2(lo, hi); + } + __syncthreads(); + if constexpr (use_mma) { +#if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) +# pragma unroll + for (int k = 0; k < BK; k += 16) { + tile_ab a[2], b[2]; +# pragma unroll + for (int i = 0; i < 2; ++i) { + load_ldmatrix(a[i], &a_s[wm + 16 * i][k / 2], AS); + load_ldmatrix_trans(b[i], &b_s[k][(wn + 16 * i) / 2], BS); + } +# pragma unroll + for (int i = 0; i < 2; ++i) { +# pragma unroll + for (int j = 0; j < 2; ++j) { + mma(c[i][j], a[i], b[j]); + } + } + } +#endif + } else { +#pragma unroll 4 + for (int k = 0; k < BK; ++k) { + float a[RM], b[RN]; +#pragma unroll + for (int i = 0; i < RM; ++i) { + a[i] = __half2float(((const half *) a_s[simt_m + i])[k]); + } +#pragma unroll + for (int j = 0; j < RN; ++j) { + b[j] = __half2float(((const half *) b_s[k])[simt_n + j]); + } +#pragma unroll + for (int i = 0; i < RM; ++i) { +#pragma unroll + for (int j = 0; j < RN; ++j) { + c_simt[i][j] += a[i] * b[j]; + } + } + } + } + __syncthreads(); + } + if constexpr (use_mma) { +#if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) +# pragma unroll + for (int i = 0; i < 2; ++i) { +# pragma unroll + for (int j = 0; j < 2; ++j) { +# pragma unroll + for (int l = 0; l < c[i][j].ne; ++l) { + const int co = m0 + wm + 16 * i + c[i][j].get_i(l); + const int pos = n0 + wn + 16 * j + c[i][j].get_j(l); + if (co < oc && pos < ow * oh) { + output[(int64_t(blockIdx.z) * oc + co) * ow * oh + pos] = c[i][j].x[l]; + } + } + } + } +#endif + } else { +#pragma unroll + for (int i = 0; i < RM; ++i) { +#pragma unroll + for (int j = 0; j < RN; ++j) { + const int co = m0 + simt_m + i, pos = n0 + simt_n + j; + if (co < oc && pos < ow * oh) { + output[(int64_t(blockIdx.z) * oc + co) * ow * oh + pos] = c_simt[i][j]; + } + } + } + } +} + +static __global__ void conv2d_reduce_split_k(const float * __restrict__ partial, + float * __restrict__ output, + const int total, + const int per_batch, + const int split_k) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= total) { + return; + } + const int n = i / per_batch; + const float * src = partial + int64_t(n) * (split_k - 1) * per_batch + i; + float sum = 0.0f; + for (int k = 0; k < split_k; ++k) { + sum += src[int64_t(k) * per_batch]; + } + output[i] = sum; +} + +template +static void conv2d_launch_implicit_gemm(const half * input, + const half * weight, + float * output, + const conv_params & params, + int split_k, + dim3 grid, + dim3 block, + cudaStream_t stream) { + if (params.KW == 3 && params.KH == 3) { + conv2d_implicit_gemm_f16<3, 3, use_mma><<>>(input, weight, output, params, split_k); + } else if (params.KW == 1 && params.KH == 1) { + conv2d_implicit_gemm_f16<1, 1, use_mma><<>>(input, weight, output, params, split_k); + } else { + conv2d_implicit_gemm_f16<0, 0, use_mma><<>>(input, weight, output, params, split_k); + } +} + static void conv2d_cuda_f16(const float * X_D, const half * K_D, float * Y_D, const conv_params P, cudaStream_t st) { conv2d_cuda(X_D, K_D, Y_D, P, st); } @@ -126,6 +341,8 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const float * X_D = (const float *) input->data; float * Y_D = (float *) dst->data; + GGML_ASSERT(input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(input)); GGML_ASSERT(ggml_is_contiguous(kernel)); GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); @@ -145,19 +362,86 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { // No cwhn GGML_ASSERT(p[6] == false); - const int IW = input->ne[0]; // input_w - const int IH = input->ne[1]; // input_h - const int OW = dst->ne[0]; // output_w - const int OH = dst->ne[1]; // output_h - const int KW = kernel->ne[0]; // kernel_w - const int KH = kernel->ne[1]; // kernel_h - const int IC = input->ne[2]; // input_channels - const int OC = kernel->ne[3]; // ouptut_chanles - const int B = input->ne[3]; // n_batches + const int64_t IW = input->ne[0]; // input_w + const int64_t IH = input->ne[1]; // input_h + const int64_t OW = dst->ne[0]; // output_w + const int64_t OH = dst->ne[1]; // output_h + const int64_t KW = kernel->ne[0]; // kernel_w + const int64_t KH = kernel->ne[1]; // kernel_h + const int64_t IC = input->ne[2]; // input_channels + const int64_t OC = kernel->ne[3]; // ouptut_chanles + const int64_t B = input->ne[3]; // n_batches const int64_t total = B * OC * OH * OW; conv_params params = { IW, IH, OW, OH, KW, KH, ST_X, ST_Y, PD_X, PD_Y, DL_X, DL_Y, IC, OC, B, total }; + const auto & device = ggml_cuda_info().devices[ctx.device]; + const bool use_mma = + turing_mma_available(device.cc) || amd_wmma_available(device.cc) || amd_mfma_available(device.cc); + // Devices without a native fragment implementation in mma.cuh can share the tiling. + const bool use_simt = GGML_CUDA_CC_IS_MTHREADS(device.cc) || (GGML_CUDA_CC_IS_AMD(device.cc) && !use_mma && IC * KW * KH >= 64); + const bool pointwise = KW == 1 && KH == 1 && ST_X == 1 && ST_Y == 1 && PD_X == 0 && PD_Y == 0; + const bool use_blas = pointwise && fast_fp16_hardware_available(device.cc); + // Short reductions on small maps do not amortize conversion and launch costs. + const bool small_conv = IC * KW * KH < 64 && OW * OH < 512; + + const int64_t limit = INT_MAX - 256; + const int64_t padded_w = IW + 2 * int64_t(PD_X), padded_h = IH + 2 * int64_t(PD_Y); + const bool padded_fits = padded_w > 0 && padded_w <= limit && padded_h > 0 && padded_h <= limit && + padded_w * padded_h <= limit && IC * B <= limit / (padded_w * padded_h); + if (kernel->type == GGML_TYPE_F16 && (use_mma || use_blas || use_simt) && (use_blas || !small_conv) && + ggml_nelements(input) <= limit && ggml_nelements(kernel) <= limit && total <= limit && padded_fits && + PD_X >= 0 && PD_Y >= 0 && ST_X > 0 && ST_Y > 0 && DL_X > 0 && DL_Y > 0 && + (OW - 1) * ST_X + (KW - 1) * DL_X < padded_w && (OH - 1) * ST_Y + (KH - 1) * DL_Y < padded_h && + (OC + 63) / 64 <= 65535 && B <= 65535) { + const int pw = int(padded_w), ph = int(padded_h); + const int padded_total = int(padded_w * padded_h * IC * B); + + ggml_cuda_pool_alloc x_half(ctx.pool(), padded_total); + // Match im2col's F16 input precision, but expand patches only in shared memory and accumulate in F32. + if (PD_X == 0 && PD_Y == 0) { + ggml_get_to_fp16_cuda(input->type)(X_D, x_half.get(), padded_total, st); + } else { + conv2d_pad_f16<<<(padded_total + 255) / 256, 256, 0, st>>>(X_D, x_half.get(), int(IW), int(IH), pw, ph, + PD_X, PD_Y, padded_total); + } + const conv_params padded_params = { pw, ph, OW, OH, KW, KH, ST_X, ST_Y, 0, 0, DL_X, DL_Y, IC, OC, B, total }; + if (use_blas) { + const float alpha = 1.0f, beta = 0.0f; + const int positions = int(OW * OH); + cublasHandle_t cublas_h = ctx.cublas_handle(); + for (int n = 0; n < B; ++n) { + CUBLAS_CHECK(cublasGemmEx(cublas_h, CUBLAS_OP_N, CUBLAS_OP_N, positions, int(OC), int(IC), &alpha, + x_half.get() + int64_t(n) * IC * positions, CUDA_R_16F, positions, K_D, + CUDA_R_16F, int(IC), &beta, Y_D + int64_t(n) * OC * positions, CUDA_R_32F, + positions, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + } + return; + } + const int64_t blocks = ((OW * OH + 63) / 64) * ((OC + 63) / 64) * B; + const int target = 8 * ggml_cuda_info().devices[ctx.device].nsm; + // Split long reductions so small spatial maps still occupy the GPU. + const int split_k = int(std::min({ int64_t(32), int64_t(65535) / B, (IC * KW * KH + 63) / 64, + std::max(int64_t(1), (target + blocks - 1) / blocks) })); + + ggml_cuda_pool_alloc partial(ctx.pool()); + float * result = split_k == 1 ? Y_D : partial.alloc(total * split_k); + const dim3 block(device.warp_size, 4); + const dim3 grid(unsigned((OW * OH + 63) / 64), unsigned((OC + 63) / 64), unsigned(B * split_k)); + if (use_mma) { + conv2d_launch_implicit_gemm(x_half.get(), (const half *) K_D, result, padded_params, split_k, grid, + block, st); + } else { + conv2d_launch_implicit_gemm(x_half.get(), (const half *) K_D, result, padded_params, split_k, grid, + block, st); + } + if (split_k > 1) { + conv2d_reduce_split_k<<<(total + 255) / 256, 256, 0, st>>>(result, Y_D, int(total), int(OC * OW * OH), + split_k); + } + return; + } + if (kernel->type == GGML_TYPE_F16) { conv2d_cuda_f16(X_D, (half *) K_D, Y_D, params, st); } else { diff --git a/ggml/src/ggml-cuda/conv3d.cu b/ggml/src/ggml-cuda/conv3d.cu new file mode 100644 index 0000000..7e4bd57 --- /dev/null +++ b/ggml/src/ggml-cuda/conv3d.cu @@ -0,0 +1,362 @@ +#include "conv3d.cuh" +#include "convert.cuh" +#include "mma.cuh" + +struct conv3d_params { + int64_t IW, IH, ID; + int64_t OW, OH, OD; + int64_t KW, KH, KD; + int64_t ST_X, ST_Y, ST_Z; + int64_t PD_X, PD_Y, PD_Z; + int64_t DL_X, DL_Y, DL_Z; + int64_t IC, OC, B, TOTAL; +}; + +template +static __global__ void conv3d_kernel(const float * input, const T * weight, float * output, const conv3d_params P) { + const int64_t spatial = P.OW * P.OH * P.OD; + for (int64_t i = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; i < P.TOTAL; + i += int64_t(gridDim.x) * blockDim.x) { + const int64_t x = i % P.OW, y = i / P.OW % P.OH, z = i / (P.OW * P.OH) % P.OD; + const int64_t co = i / spatial % P.OC, n = i / (spatial * P.OC); + float sum = 0.0f; + for (int64_t ci = 0; ci < P.IC; ++ci) { + for (int64_t kz = 0; kz < P.KD; ++kz) { + const int64_t iz = z * P.ST_Z + kz * P.DL_Z - P.PD_Z; + if (iz < 0 || iz >= P.ID) { + continue; + } + for (int64_t ky = 0; ky < P.KH; ++ky) { + const int64_t iy = y * P.ST_Y + ky * P.DL_Y - P.PD_Y; + if (iy < 0 || iy >= P.IH) { + continue; + } + for (int64_t kx = 0; kx < P.KW; ++kx) { + const int64_t ix = x * P.ST_X + kx * P.DL_X - P.PD_X; + if (ix >= 0 && ix < P.IW) { + const int64_t xi = (((n * P.IC + ci) * P.ID + iz) * P.IH + iy) * P.IW + ix; + const int64_t wi = (((co * P.IC + ci) * P.KD + kz) * P.KH + ky) * P.KW + kx; + sum += input[xi] * ggml_cuda_cast(weight[wi]); + } + } + } + } + } + output[i] = sum; + } +} + +static __global__ void conv3d_pad_f16(const float * input, + half * output, + int iw, + int ih, + int id, + int pw, + int ph, + int pd, + int px, + int py, + int pz, + int total) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= total) { + return; + } + const int x = i % pw - px, y = i / pw % ph - py, z = i / (pw * ph) % pd - pz; + const int nc = i / (pw * ph * pd); + output[i] = + __float2half((unsigned) x < (unsigned) iw && (unsigned) y < (unsigned) ih && (unsigned) z < (unsigned) id ? + input[((nc * id + z) * ih + y) * iw + x] : + 0.0f); +} + +template +static __global__ void conv3d_implicit_gemm_f16(const half * __restrict__ input, + const half * __restrict__ weight, + float * __restrict__ output, + const conv3d_params P, + const int split_k, + const bool aligned_weights) { + using namespace ggml_cuda_mma; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nthreads = 4 * warp_size; + constexpr int BM = 64, BN = 64, BK = 64; + constexpr int AS = BK / 2 + 4; + constexpr int BS = BN / 2 + 4; + static_assert(AS * sizeof(half2) % sizeof(int4) == 0, "shared weight rows must be 16-byte aligned"); + __shared__ __align__(16) half2 a_s[BM][AS]; + __shared__ __align__(16) half2 b_s[BK][BS]; + const int tid = threadIdx.y * warp_size + threadIdx.x; + const int iw = int(P.IW), ih = int(P.IH), id = int(P.ID); + const int ow = int(P.OW), oh = int(P.OH), od = int(P.OD); + const int kw = KW ? KW : int(P.KW), kh = KH ? KH : int(P.KH), kd = KD ? KD : int(P.KD); + const int ic = int(P.IC), oc = int(P.OC); + const int sx = int(P.ST_X), sy = int(P.ST_Y), sz = int(P.ST_Z); + const int dx = int(P.DL_X), dy = int(P.DL_Y), dz = int(P.DL_Z); + const int n = blockIdx.z / split_k, split = blockIdx.z % split_k; + const int m0 = blockIdx.y * BM, n0 = blockIdx.x * BN; + const int k_total = ic * kw * kh * kd; + const int load_lane = warp_size == 32 ? threadIdx.x : threadIdx.x % (BN / 2); + const int load_row = threadIdx.y * (warp_size / (BN / 2)) + (warp_size == 32 ? 0 : threadIdx.x / (BN / 2)); + const int spatial = n0 + 2 * load_lane; + const int spatial0 = min(spatial, ow * oh * od - 1), spatial1 = min(spatial + 1, ow * oh * od - 1); + const int z0 = spatial0 / (ow * oh), y0 = spatial0 / ow % oh, x0 = spatial0 % ow; + const int z1 = spatial1 / (ow * oh), y1 = spatial1 / ow % oh, x1 = spatial1 % ow; + const int pos0 = (z0 * sz * ih + y0 * sy) * iw + x0 * sx; + const int pos1 = (z1 * sz * ih + y1 * sy) * iw + x1 * sx; + [[maybe_unused]] const int wm = threadIdx.y / 2 * 32, wn = threadIdx.y % 2 * 32; + using tile_ab = tile<16, 8, half2, get_input_data_layout()>; +#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) + // AMD accumulator fragments transpose the input fragment's row/column mapping. + using tile_c = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>; +#else + using tile_c = tile<16, 16, float>; +#endif + [[maybe_unused]] tile_c c[2][2]; + constexpr int RM = 4, RN = BM * BN / (nthreads * RM); + [[maybe_unused]] const int simt_m = tid / (BN / RN) * RM, simt_n = tid % (BN / RN) * RN; + [[maybe_unused]] float c_simt[RM][RN] = {}; + const int tiles = (k_total + BK - 1) / BK; + const int begin = int(int64_t(tiles) * split / split_k) * BK; + const int end = int(int64_t(tiles) * (split + 1) / split_k) * BK; + for (int k0 = begin; k0 < end; k0 += BK) { + if (aligned_weights) { +#pragma unroll + for (int i0 = 0; i0 < BM * BK / 8; i0 += nthreads) { + const int i = i0 + tid; + const int row = i / (BK / 8), col = 8 * (i % (BK / 8)); + const int4 v = m0 + row < oc && k0 + col < k_total ? + ((const int4 *) weight)[((m0 + row) * k_total + k0 + col) / 8] : + make_int4(0, 0, 0, 0); + *(int4 *) &a_s[row][col / 2] = v; + } + } else { +#pragma unroll + for (int i0 = 0; i0 < BM * BK / 2; i0 += nthreads) { + const int i = i0 + tid; + const int row = i / (BK / 2), col = 2 * (i % (BK / 2)); + half lo = __float2half(0.0f), hi = lo; + if (m0 + row < oc && k0 + col < k_total) { + lo = weight[(m0 + row) * k_total + k0 + col]; + if (k0 + col + 1 < k_total) { + hi = weight[(m0 + row) * k_total + k0 + col + 1]; + } + } + a_s[row][col / 2] = __halves2half2(lo, hi); + } + } +#pragma unroll + for (int kb = 0; kb < BK; kb += nthreads / (BN / 2)) { + const int k = kb + load_row; + const int ki = k0 + k; + const int ci = ki / (kw * kh * kd), kz = ki / (kw * kh) % kd, ky = ki / kw % kh, kx = ki % kw; + const int offset = ki < k_total ? ((n * ic + ci) * id + kz * dz) * ih * iw + ky * dy * iw + kx * dx : 0; + half lo = __float2half(0.0f), hi = lo; + if (ki < k_total && spatial < ow * oh * od) { + lo = input[offset + pos0]; + } + if (ki < k_total && spatial + 1 < ow * oh * od) { + hi = input[offset + pos1]; + } + b_s[k][load_lane] = __halves2half2(lo, hi); + } + __syncthreads(); + if constexpr (use_mma) { +#pragma unroll + for (int k = 0; k < BK; k += 16) { + tile_ab a[2], b[2]; +#pragma unroll + for (int i = 0; i < 2; ++i) { + load_ldmatrix(a[i], &a_s[wm + 16 * i][k / 2], AS); + load_ldmatrix_trans(b[i], &b_s[k][(wn + 16 * i) / 2], BS); + } +#pragma unroll + for (int i = 0; i < 2; ++i) { +#pragma unroll + for (int j = 0; j < 2; ++j) { + mma(c[i][j], a[i], b[j]); + } + } + } + } else { +#pragma unroll 4 + for (int k = 0; k < BK; ++k) { + float a[RM], b[RN]; +#pragma unroll + for (int i = 0; i < RM; ++i) { + a[i] = __half2float(((const half *) a_s[simt_m + i])[k]); + } +#pragma unroll + for (int j = 0; j < RN; ++j) { + b[j] = __half2float(((const half *) b_s[k])[simt_n + j]); + } +#pragma unroll + for (int i = 0; i < RM; ++i) { +#pragma unroll + for (int j = 0; j < RN; ++j) { + c_simt[i][j] += a[i] * b[j]; + } + } + } + } + __syncthreads(); + } + if constexpr (use_mma) { +#pragma unroll + for (int i = 0; i < 2; ++i) { +#pragma unroll + for (int j = 0; j < 2; ++j) { +#pragma unroll + for (int l = 0; l < c[i][j].ne; ++l) { + const int co = m0 + wm + 16 * i + c[i][j].get_i(l); + const int pos = n0 + wn + 16 * j + c[i][j].get_j(l); + if (co < oc && pos < ow * oh * od) { + output[(int64_t(blockIdx.z) * oc + co) * ow * oh * od + pos] = c[i][j].x[l]; + } + } + } + } + } else { +#pragma unroll + for (int i = 0; i < RM; ++i) { +#pragma unroll + for (int j = 0; j < RN; ++j) { + const int co = m0 + simt_m + i, pos = n0 + simt_n + j; + if (co < oc && pos < ow * oh * od) { + output[(int64_t(blockIdx.z) * oc + co) * ow * oh * od + pos] = c_simt[i][j]; + } + } + } + } +} + +static __global__ void conv3d_reduce_split_k(const float * __restrict__ partial, + float * __restrict__ output, + const int total, + const int per_batch, + const int split_k) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= total) { + return; + } + const int n = i / per_batch; + // Partial slices are ordered as [batch, split, output channel, spatial position]. + const float * src = partial + int64_t(n) * (split_k - 1) * per_batch + i; + float sum = 0.0f; + for (int k = 0; k < split_k; ++k) { + sum += src[int64_t(k) * per_batch]; + } + output[i] = sum; +} + +template +static void conv3d_launch_implicit_gemm(const half * input, + const half * weight, + float * output, + const conv3d_params & params, + int split_k, + dim3 grid, + dim3 block, + cudaStream_t stream) { + // Vector loads require both the base pointer and each weight row to be 16-byte aligned. + const bool aligned_weights = uintptr_t(weight) % sizeof(int4) == 0 && + (params.IC * params.KW * params.KH * params.KD) % (sizeof(int4) / sizeof(half)) == 0; + if (params.KW == 3 && params.KH == 3 && params.KD == 3) { + conv3d_implicit_gemm_f16<3, 3, 3, use_mma> + <<>>(input, weight, output, params, split_k, aligned_weights); + } else if (params.KW == 1 && params.KH == 1 && params.KD == 3) { + conv3d_implicit_gemm_f16<1, 1, 3, use_mma> + <<>>(input, weight, output, params, split_k, aligned_weights); + } else if (params.KW == 1 && params.KH == 1 && params.KD == 1) { + conv3d_implicit_gemm_f16<1, 1, 1, use_mma> + <<>>(input, weight, output, params, split_k, aligned_weights); + } else { + conv3d_implicit_gemm_f16<0, 0, 0, use_mma> + <<>>(input, weight, output, params, split_k, aligned_weights); + } +} + +void ggml_cuda_op_conv3d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * kernel = dst->src[0]; + const ggml_tensor * input = dst->src[1]; + GGML_ASSERT(input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(input) && ggml_is_contiguous(kernel) && ggml_is_contiguous(dst)); + + const int32_t * p = dst->op_params; + const int64_t IW = input->ne[0], IH = input->ne[1], ID = input->ne[2]; + const int64_t OW = dst->ne[0], OH = dst->ne[1], OD = dst->ne[2]; + const int64_t KW = kernel->ne[0], KH = kernel->ne[1], KD = kernel->ne[2]; + const int64_t IC = p[9], B = p[10], OC = p[11]; + GGML_ASSERT(IC > 0 && B > 0 && OC > 0 && input->ne[3] == IC * B && kernel->ne[3] == IC * OC); + GGML_ASSERT(dst->ne[3] == OC * B && p[0] > 0 && p[1] > 0 && p[2] > 0 && p[6] > 0 && p[7] > 0 && p[8] > 0); + const int64_t total = ggml_nelements(dst); + const conv3d_params params = { IW, IH, ID, OW, OH, OD, KW, KH, KD, p[0], p[1], + p[2], p[3], p[4], p[5], p[6], p[7], p[8], IC, OC, B, total }; + const float * x = (const float *) input->data; + const half * w = (const half *) kernel->data; + float * y = (float *) dst->data; + cudaStream_t stream = ctx.stream(); + const auto & device = ggml_cuda_info().devices[ctx.device]; + const bool use_mma = + turing_mma_available(device.cc) || amd_wmma_available(device.cc) || amd_mfma_available(device.cc); + const bool pointwise = + KW == 1 && KH == 1 && KD == 1 && p[0] == 1 && p[1] == 1 && p[2] == 1 && p[3] == 0 && p[4] == 0 && p[5] == 0; + const bool use_blas = pointwise && fast_fp16_hardware_available(device.cc); + const int64_t limit = INT_MAX - 256; + const int64_t pw = IW + 2 * int64_t(p[3]), ph = IH + 2 * int64_t(p[4]), pd = ID + 2 * int64_t(p[5]); + const bool padded_fits = pw > 0 && pw <= limit && ph > 0 && ph <= limit && pd > 0 && pd <= limit && + pw * ph <= limit / pd && IC * B <= limit / (pw * ph * pd); + if (kernel->type == GGML_TYPE_F16 && KW > 0 && KH > 0 && KD > 0 && ggml_nelements(input) <= limit && + ggml_nelements(kernel) <= limit && total <= limit && padded_fits && p[3] >= 0 && p[4] >= 0 && p[5] >= 0 && + (OW - 1) * p[0] + (KW - 1) * p[6] < pw && (OH - 1) * p[1] + (KH - 1) * p[7] < ph && + (OD - 1) * p[2] + (KD - 1) * p[8] < pd && (OC + 63) / 64 <= 65535 && B <= 65535) { + const int padded_total = int(pw * ph * pd * IC * B); + ggml_cuda_pool_alloc x_half(ctx.pool(), padded_total); + // Match im2col's F16 input precision without materializing all patches in global memory. + if (p[3] == 0 && p[4] == 0 && p[5] == 0) { + ggml_get_to_fp16_cuda(input->type)(x, x_half.get(), padded_total, stream); + } else { + conv3d_pad_f16<<<(padded_total + 255) / 256, 256, 0, stream>>>( + x, x_half.get(), int(IW), int(IH), int(ID), int(pw), int(ph), int(pd), p[3], p[4], p[5], padded_total); + } + const conv3d_params padded_params = { pw, ph, pd, OW, OH, OD, KW, KH, KD, p[0], p[1], + p[2], 0, 0, 0, p[6], p[7], p[8], IC, OC, B, total }; + const int positions = int(OW * OH * OD); + if (use_blas) { + const float alpha = 1.0f, beta = 0.0f; + cublasHandle_t cublas_h = ctx.cublas_handle(); + for (int n = 0; n < B; ++n) { + CUBLAS_CHECK(cublasGemmEx(cublas_h, CUBLAS_OP_N, CUBLAS_OP_N, positions, int(OC), int(IC), &alpha, + x_half.get() + int64_t(n) * IC * positions, CUDA_R_16F, positions, w, + CUDA_R_16F, int(IC), &beta, y + int64_t(n) * OC * positions, CUDA_R_32F, + positions, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + } + return; + } + const int64_t blocks = ((positions + 63) / 64) * ((OC + 63) / 64) * B; + const int target = 8 * device.nsm; + const int split_k = int(std::min({ int64_t(32), int64_t(65535) / B, (IC * KW * KH * KD + 63) / 64, + std::max(int64_t(1), (target + blocks - 1) / blocks) })); + ggml_cuda_pool_alloc partial(ctx.pool()); + float * result = split_k == 1 ? y : partial.alloc(total * split_k); + const dim3 block(device.warp_size, 4); + const dim3 grid(unsigned((positions + 63) / 64), unsigned((OC + 63) / 64), unsigned(B * split_k)); + if (use_mma) { + conv3d_launch_implicit_gemm(x_half.get(), w, result, padded_params, split_k, grid, block, stream); + } else { + conv3d_launch_implicit_gemm(x_half.get(), w, result, padded_params, split_k, grid, block, stream); + } + if (split_k > 1) { + conv3d_reduce_split_k<<>>(result, y, int(total), + int(OC * positions), split_k); + } + return; + } + const int blocks = int(std::min(int64_t(65535), (total + 255) / 256)); + if (kernel->type == GGML_TYPE_F16) { + conv3d_kernel<<>>(x, w, y, params); + } else { + conv3d_kernel<<>>(x, (const float *) kernel->data, y, params); + } +} diff --git a/ggml/src/ggml-cuda/conv3d.cuh b/ggml/src/ggml-cuda/conv3d.cuh new file mode 100644 index 0000000..b321dcc --- /dev/null +++ b/ggml/src/ggml-cuda/conv3d.cuh @@ -0,0 +1,8 @@ +#ifndef GGML_CUDA_CONV3D_CUH +#define GGML_CUDA_CONV3D_CUH + +#include "common.cuh" + +void ggml_cuda_op_conv3d(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +#endif diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 61630a3..77fd288 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -104,8 +104,8 @@ static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t const uint8_t * q = x->qs + 4*il; for (int l = 0; l < 4; ++l) { - y[l+ 0] = d * (q[l] & 0xF) + dm; - y[l+16] = d * (q[l] >> 4) + dm; + y[l+ 0] = ggml_cuda_cast(d * (q[l] & 0xF) + dm); + y[l+16] = ggml_cuda_cast(d * (q[l] >> 4) + dm); } } @@ -131,8 +131,8 @@ static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t const uint8_t * q = x->qs + 4*il; for (int l = 0; l < 4; ++l) { - y[l+ 0] = d.x * (q[l] & 0xF) + d.y; - y[l+16] = d.x * (q[l] >> 4) + d.y; + y[l+ 0] = ggml_cuda_cast(d.x * (q[l] & 0xF) + d.y); + y[l+16] = ggml_cuda_cast(d.x * (q[l] >> 4) + d.y); } } @@ -140,350 +140,112 @@ static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t template static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; - const int64_t i = blockIdx.x; - const block_q2_K * x = (const block_q2_K *) vx; - - const int64_t tid = threadIdx.x; - const int64_t n = tid/32; - const int64_t l = tid - 32*n; - const int64_t is = 8*n + l/16; - - const uint8_t q = x[i].qs[32*n + l]; - dst_t * y = yy + i*QK_K + 128*n; - - float dall = __low2half(x[i].dm); - float dmin = __high2half(x[i].dm); - y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4); - y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4); - y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4); - y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4); + dequantize_q2_K(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) { - const int64_t i = blockIdx.x; - const block_q3_K * x = (const block_q3_K *) vx; - - const int64_t r = threadIdx.x/4; - const int64_t tid = r/2; - const int64_t is0 = r%2; - const int64_t l0 = 16*is0 + 4*(threadIdx.x%4); - const int64_t n = tid / 4; - const int64_t j = tid - 4*n; - - uint8_t m = 1 << (4*n + j); - int64_t is = 8*n + 2*j + is0; - int shift = 2*j; - - int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) : - is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) : - is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) : - (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4); - float d_all = x[i].d; - float dl = d_all * (us - 32); - dst_t * y = yy + i*QK_K + 128*n + 32*j; - const uint8_t * q = x[i].qs + 32*n; - const uint8_t * hm = x[i].hmask; - - for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)); -} - -static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) { - if (j < 4) { - d = q[j] & 63; m = q[j + 4] & 63; - } else { - d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4); - m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); - } + dequantize_q3_K(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) { - const block_q4_K * x = (const block_q4_K *) vx; - const int64_t i = blockIdx.x; - // assume 32 threads - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; - const int64_t ir = tid%8; - const int64_t is = 2*il; - const int64_t n = 4; - - dst_t * y = yy + i*QK_K + 64*il + n*ir; - - const float dall = __low2half(x[i].dm); - const float dmin = __high2half(x[i].dm); - - const uint8_t * q = x[i].qs + 32*il + n*ir; - - uint8_t sc, m; - get_scale_min_k4(is + 0, x[i].scales, sc, m); - const float d1 = dall * sc; const float m1 = dmin * m; - get_scale_min_k4(is + 1, x[i].scales, sc, m); - const float d2 = dall * sc; const float m2 = dmin * m; - for (int l = 0; l < n; ++l) { - y[l + 0] = d1 * (q[l] & 0xF) - m1; - y[l +32] = d2 * (q[l] >> 4) - m2; - } + dequantize_q4_K(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) { - const block_q5_K * x = (const block_q5_K *) vx; - const int64_t i = blockIdx.x; - // assume 64 threads - this is very slightly better than the one below - const int64_t tid = threadIdx.x; - const int64_t il = tid/16; // il is in 0...3 - const int64_t ir = tid%16; // ir is in 0...15 - const int64_t is = 2*il; // is is in 0...6 - - dst_t * y = yy + i*QK_K + 64*il + 2*ir; - - const float dall = __low2half(x[i].dm); - const float dmin = __high2half(x[i].dm); - - const uint8_t * ql = x[i].qs + 32*il + 2*ir; - const uint8_t * qh = x[i].qh + 2*ir; - - uint8_t sc, m; - get_scale_min_k4(is + 0, x[i].scales, sc, m); - const float d1 = dall * sc; const float m1 = dmin * m; - get_scale_min_k4(is + 1, x[i].scales, sc, m); - const float d2 = dall * sc; const float m2 = dmin * m; - - uint8_t hm = 1 << (2*il); - y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1; - y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1; - hm <<= 1; - y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2; - y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2; + dequantize_q5_K(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) { - const block_q6_K * x = (const block_q6_K *) vx; - const int64_t i = blockIdx.x; - // assume 64 threads - this is very slightly better than the one below - const int64_t tid = threadIdx.x; - const int64_t ip = tid/32; // ip is 0 or 1 - const int64_t il = tid - 32*ip; // 0...32 - const int64_t is = 8*ip + il/16; - - dst_t * y = yy + i*QK_K + 128*ip + il; - - const float d = x[i].d; - - const uint8_t * ql = x[i].ql + 64*ip + il; - const uint8_t qh = x[i].qh[32*ip + il]; - const int8_t * sc = x[i].scales + is; - - y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32); - y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32); - y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32); - y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32); + dequantize_q6_K(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; - const int64_t i = blockIdx.x; - const block_iq2_xxs * x = (const block_iq2_xxs *) vx; - - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const uint16_t * q2 = x[i].qs + 4*ib; - const uint8_t * aux8 = (const uint8_t *)q2; - const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]); - const uint32_t aux32 = q2[2] | (q2[3] << 16); - const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f; - const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127]; - for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + dequantize_iq2_xxs(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; - const int64_t i = blockIdx.x; - const block_iq2_xs * x = (const block_iq2_xs *) vx; - - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const uint16_t * q2 = x[i].qs + 4*ib; - const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511)); - const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; - const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; - for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + dequantize_iq2_xs(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; - const int64_t i = blockIdx.x; - const block_iq2_s * x = (const block_iq2_s *) vx; - - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300))); - const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; - const uint8_t signs = x[i].qs[QK_K/8+4*ib+il]; - for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + dequantize_iq2_s(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; - const int64_t i = blockIdx.x; - const block_iq3_xxs * x = (const block_iq3_xxs *) vx; - - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const uint8_t * q3 = x[i].qs + 8*ib; - const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib; - const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]); - const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]); - const uint32_t aux32 = gas[0] | (gas[1] << 16); - const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f; - const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127]; - for (int j = 0; j < 4; ++j) { - y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); - y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); - } + dequantize_iq3_xxs(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; - const int64_t i = blockIdx.x; - const block_iq3_s * x = (const block_iq3_s *) vx; - - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const uint8_t * qs = x[i].qs + 8*ib; - const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256))); - const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256))); - const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)); - const uint8_t signs = x[i].signs[4*ib + il]; - for (int j = 0; j < 4; ++j) { - y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); - y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); - } + dequantize_iq3_s(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; - const int64_t i = blockIdx.x; - const block_iq1_s * x = (const block_iq1_s *) vx; - - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA; - const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1); - uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32; - grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)]; - grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; - grid32[0] &= 0x0f0f0f0f; - for (int j = 0; j < 8; ++j) { - y[j] = d * (q[j] + delta); - } + dequantize_iq1_s(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; - const int64_t i = blockIdx.x; - const block_iq1_m * x = (const block_iq1_m *) vx; - - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 8*il; - const uint16_t * sc = (const uint16_t *)x[i].scales; - iq1m_scale_t scale; - scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); - const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4); - const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1); - const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA; - uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32; - grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)]; - grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; - grid32[0] &= 0x0f0f0f0f; - for (int j = 0; j < 8; ++j) { - y[j] = d * (q[j] + delta); - } + dequantize_iq1_m(vx, i, yy + i*QK_K, threadIdx.x); } template -static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) { - - const int64_t i = blockIdx.x; - const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL); +static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) { + const int64_t i = blockIdx.x; - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 4*il; - const uint8_t * q4 = x[ib].qs + 4*il; - const float d = (float)x[ib].d; - for (int j = 0; j < 4; ++j) { - y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf]; - y[j+16] = d * kvalues_iq4nl[q4[j] >> 4]; + const int64_t ib = 8*i + threadIdx.x%8; + if (ib >= nb32) { + return; } + + dequantize_iq4_nl(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) { - const int64_t i = blockIdx.x; - const block_iq4_xs * x = (const block_iq4_xs *)vx; + const int64_t i = blockIdx.x; - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 4*il; - const uint8_t * q4 = x[i].qs + 16*ib + 4*il; - const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32); - for (int j = 0; j < 4; ++j) { - y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf]; - y[j+16] = d * kvalues_iq4nl[q4[j] >> 4]; - } + dequantize_iq4_xs(vx, i, yy + i*QK_K, threadIdx.x); } template static __global__ void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; - const int64_t i = blockIdx.x; - const block_mxfp4 * x = (const block_mxfp4 *) vx + i*(QK_K/QK_MXFP4); - - const int64_t tid = threadIdx.x; - const int64_t il = tid/8; // 0...3 - const int64_t ib = tid%8; // 0...7 - dst_t * y = yy + i*QK_K + 32*ib + 4*il; - const uint8_t * q4 = x[ib].qs + 4*il; - const float d = ggml_cuda_e8m0_to_fp32(x[ib].e); - for (int j = 0; j < 4; ++j) { - y[j+ 0] = d * kvalues_mxfp4[q4[j] & 0xf]*0.5f; - y[j+16] = d * kvalues_mxfp4[q4[j] >> 4]*0.5f; - } + dequantize_mxfp4(vx, i, yy + i*QK_K, threadIdx.x); } template @@ -595,8 +357,9 @@ static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t template static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb32 = k / QK4_NL; const int nb = (k + QK_K - 1) / QK_K; - dequantize_block_iq4_nl<<>>(vx, y); + dequantize_block_iq4_nl<<>>(vx, y, nb32); } template @@ -682,6 +445,29 @@ static __global__ void convert_unary( } } +template struct alignas(sizeof(T)*4) cvt_vec4 { T v[4]; }; + +// four elements per thread, so a warp moves 512B (RDNA) / 1k (CDNA) per load +template +static __global__ void convert_unary_cont_vec4( + const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k4) { + const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x; + + if (i >= k4) { + return; + } + + const cvt_vec4 xv = ((const cvt_vec4 *) vx)[i]; + + cvt_vec4 yv; +#pragma unroll + for (int j = 0; j < 4; ++j) { + yv.v[j] = ggml_cuda_cast(xv.v[j]); + } + + ((cvt_vec4 *) y)[i] = yv; +} + template static void convert_unary_cuda(const void * vx, dst_t * y, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03, @@ -695,11 +481,66 @@ static void convert_unary_cuda(const void * vx, dst_t * y, template static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + if (k % 4 == 0 && + (uintptr_t) vx % alignof(cvt_vec4) == 0 && + (uintptr_t) y % alignof(cvt_vec4) == 0) { + const int64_t k4 = k/4; + const int64_t num_blocks = (k4 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE; + convert_unary_cont_vec4<<>>(vx, y, k4); + return; + } + convert_unary_cuda(vx, y, k, 1, 1, 1, k, k, k, stream); } to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q4_0: + return dequantize_row_q4_0_cuda; + case GGML_TYPE_Q4_1: + return dequantize_row_q4_1_cuda; + case GGML_TYPE_Q5_0: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q5_1: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q8_0: + return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_K: + return dequantize_row_q2_K_cuda; + case GGML_TYPE_Q3_K: + return dequantize_row_q3_K_cuda; + case GGML_TYPE_Q4_K: + return dequantize_row_q4_K_cuda; + case GGML_TYPE_Q5_K: + return dequantize_row_q5_K_cuda; + case GGML_TYPE_Q6_K: + return dequantize_row_q6_K_cuda; + case GGML_TYPE_IQ2_XXS: + return dequantize_row_iq2_xxs_cuda; + case GGML_TYPE_IQ2_XS: + return dequantize_row_iq2_xs_cuda; + case GGML_TYPE_IQ2_S: + return dequantize_row_iq2_s_cuda; + case GGML_TYPE_IQ3_XXS: + return dequantize_row_iq3_xxs_cuda; + case GGML_TYPE_IQ1_S: + return dequantize_row_iq1_s_cuda; + case GGML_TYPE_IQ1_M: + return dequantize_row_iq1_m_cuda; + case GGML_TYPE_IQ4_NL: + return dequantize_row_iq4_nl_cuda; + case GGML_TYPE_IQ4_XS: + return dequantize_row_iq4_xs_cuda; + case GGML_TYPE_IQ3_S: + return dequantize_row_iq3_s_cuda; + case GGML_TYPE_MXFP4: + return dequantize_row_mxfp4_cuda; + case GGML_TYPE_NVFP4: + return dequantize_row_nvfp4_cuda; case GGML_TYPE_F32: return convert_unary_cont_cuda; case GGML_TYPE_F16: @@ -713,6 +554,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cont_cuda; case GGML_TYPE_Q4_0: return dequantize_row_q4_0_cuda; case GGML_TYPE_Q4_1: @@ -771,6 +614,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cont_cuda; case GGML_TYPE_Q4_0: return dequantize_row_q4_0_cuda; case GGML_TYPE_Q4_1: @@ -828,6 +673,8 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) { return convert_unary_cuda; case GGML_TYPE_Q1_0: return dequantize_block_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cuda; case GGML_TYPE_Q4_0: return dequantize_block_cuda; case GGML_TYPE_Q4_1: @@ -851,6 +698,8 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) { return convert_unary_cuda; case GGML_TYPE_Q1_0: return dequantize_block_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cuda; case GGML_TYPE_Q4_0: return dequantize_block_cuda; case GGML_TYPE_Q4_1: @@ -874,6 +723,8 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) { return convert_unary_cuda; case GGML_TYPE_Q1_0: return dequantize_block_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cuda; case GGML_TYPE_Q4_0: return dequantize_block_cuda; case GGML_TYPE_Q4_1: diff --git a/ggml/src/ggml-cuda/convert.cuh b/ggml/src/ggml-cuda/convert.cuh index f5d37c7..91255cc 100644 --- a/ggml/src/ggml-cuda/convert.cuh +++ b/ggml/src/ggml-cuda/convert.cuh @@ -45,11 +45,11 @@ template #ifdef GGML_USE_HIP return make_float2(__bfloat162float(__low2bfloat16(x)), __bfloat162float(__high2bfloat16(x))); #else -#if __CUDA_ARCH__ >= 800 +#if !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= 800 return __bfloat1622float2(x); #else return make_float2(__bfloat162float(x.x), __bfloat162float(x.y)); -#endif // __CUDA_ARCH__ >= 800 +#endif // !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= 800 #endif // GGML_USE_HIP } else if constexpr(std::is_same_v && std::is_same_v) { // bypass compile error on cuda 12.0.1 diff --git a/ggml/src/ggml-cuda/count-equal.cu b/ggml/src/ggml-cuda/count-equal.cu index 0889811..411d08c 100644 --- a/ggml/src/ggml-cuda/count-equal.cu +++ b/ggml/src/ggml-cuda/count-equal.cu @@ -59,6 +59,5 @@ void ggml_cuda_count_equal(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { } break; default: GGML_ASSERT(false); - break; } } diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu index d208acf..90a06b9 100644 --- a/ggml/src/ggml-cuda/cpy.cu +++ b/ggml/src/ggml-cuda/cpy.cu @@ -16,6 +16,7 @@ static __global__ void cpy_scalar(const char * cx, char * cdst, const int64_t ne const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13) { + ggml_cuda_pdl_lc(); const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x; if (i >= ne) { @@ -36,6 +37,7 @@ static __global__ void cpy_scalar(const char * cx, char * cdst, const int64_t ne const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10; const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13; + ggml_cuda_pdl_sync(); cpy_1(cx + x_offset, cdst + dst_offset); } @@ -51,14 +53,15 @@ static __global__ void cpy_scalar_transpose(const char * cx, char * cdst, const const int64_t nmat = ne / (ne00 * ne01); const int64_t n = ne00 * ne01; - const int x = blockIdx.x * CUDA_CPY_TILE_DIM_2D + threadIdx.x; - const int y = blockIdx.y * CUDA_CPY_TILE_DIM_2D + threadIdx.y; - const int tx = blockIdx.y * CUDA_CPY_TILE_DIM_2D + threadIdx.x; // transpose block offset - const int ty = blockIdx.x * CUDA_CPY_TILE_DIM_2D + threadIdx.y; + const int64_t x = (int64_t) blockIdx.x * CUDA_CPY_TILE_DIM_2D + threadIdx.x; + const int64_t y = (int64_t) blockIdx.y * CUDA_CPY_TILE_DIM_2D + threadIdx.y; + const int64_t tx = (int64_t) blockIdx.y * CUDA_CPY_TILE_DIM_2D + threadIdx.x; // transpose block offset + const int64_t ty = (int64_t) blockIdx.x * CUDA_CPY_TILE_DIM_2D + threadIdx.y; __shared__ float tile[2][CUDA_CPY_TILE_DIM_2D][CUDA_CPY_TILE_DIM_2D+1]; int cur_tile_buf = 0; + ggml_cuda_pdl_sync(); #pragma unroll for (int i = 0; i < CUDA_CPY_BLOCK_NM; ++i) { @@ -142,6 +145,7 @@ static __global__ void cpy_f32_q(const char * cx, char * cdst, const int64_t ne, const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10; const int64_t dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13; + ggml_cuda_pdl_sync(); cpy_blck(cx + x_offset, cdst + dst_offset); } @@ -168,6 +172,7 @@ static __global__ void cpy_q_f32(const char * cx, char * cdst, const int64_t ne, const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10; const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13; + ggml_cuda_pdl_sync(); cpy_blck(cx + x_offset, cdst + dst_offset); } @@ -182,6 +187,7 @@ static __global__ void cpy_scalar_contiguous(const char * cx, char * cdst, const const src_t * x = (const src_t *) cx; dst_t * dst = (dst_t *) cdst; + ggml_cuda_pdl_sync(); dst[i] = ggml_cuda_cast(x[i]); } @@ -191,9 +197,9 @@ static void ggml_cpy_scalar_contiguous_cuda( cudaStream_t stream) { const int64_t num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_scalar_contiguous<<>> - (cx, cdst, ne); + GGML_ASSERT(num_blocks <= INT_MAX); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(cpy_scalar_contiguous, launch_params, cx, cdst, ne); } template @@ -202,6 +208,14 @@ static void ggml_cpy_scalar_cuda( const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { + const auto launch_scalar_generic = [&]() { + const int64_t num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(cpy_scalar>, launch_params, + cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); + }; + if (transposed) { GGML_ASSERT(ne == ne00*ne01*ne02); // ne[3] is 1 assumed int64_t ne00n, ne01n, ne02n; @@ -218,18 +232,18 @@ static void ggml_cpy_scalar_cuda( int64_t grid_x = (ne01n + CUDA_CPY_TILE_DIM_2D - 1) / CUDA_CPY_TILE_DIM_2D; int64_t grid_y = (ne00n + CUDA_CPY_TILE_DIM_2D - 1) / CUDA_CPY_TILE_DIM_2D; int64_t grid_z = (ne/(ne01n*ne00n) + CUDA_CPY_BLOCK_NM - 1) / CUDA_CPY_BLOCK_NM; - GGML_ASSERT(grid_x < UINT_MAX); - GGML_ASSERT(grid_y < USHRT_MAX); - GGML_ASSERT(grid_z < USHRT_MAX); - dim3 dimGrid(grid_x, grid_y, grid_z); - dim3 dimBlock(CUDA_CPY_TILE_DIM_2D, CUDA_CPY_BLOCK_ROWS, 1); - cpy_scalar_transpose<<>> - (cx, cdst, ne, ne00n, ne01n, ne02n, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); + GGML_ASSERT(grid_x <= INT_MAX); + if (grid_y > USHRT_MAX || grid_z > USHRT_MAX) { + launch_scalar_generic(); + } else { + dim3 dimGrid(grid_x, grid_y, grid_z); + dim3 dimBlock(CUDA_CPY_TILE_DIM_2D, CUDA_CPY_BLOCK_ROWS, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(dimGrid, dimBlock, 0, stream); + ggml_cuda_kernel_launch(cpy_scalar_transpose, launch_params, + cx, cdst, ne, ne00n, ne01n, ne02n, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); + } } else { - const int64_t num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_scalar><<>> - (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); + launch_scalar_generic(); } } @@ -239,9 +253,9 @@ static void ggml_cpy_f32_q8_0_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { GGML_ASSERT(ne % QK8_0 == 0); - const int64_t num_blocks = ne / QK8_0; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_f32_q<<>> + const int64_t num_blocks = (ne/QK8_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_f32_q<<>> (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -250,9 +264,9 @@ static void ggml_cpy_q8_0_f32_cuda( const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { - const int64_t num_blocks = ne; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_q_f32<<>> + const int64_t num_blocks = (ne/QK8_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_q_f32<<>> (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -262,9 +276,9 @@ static void ggml_cpy_f32_q4_0_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { GGML_ASSERT(ne % QK4_0 == 0); - const int64_t num_blocks = ne / QK4_0; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_f32_q<<>> + const int64_t num_blocks = (ne/QK4_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_f32_q<<>> (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -275,9 +289,9 @@ static void ggml_cpy_q4_0_f32_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { - const int64_t num_blocks = ne; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_q_f32, QK4_0><<>>( + const int64_t num_blocks = (ne/QK4_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_q_f32, QK4_0><<>>( cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -288,9 +302,9 @@ static void ggml_cpy_f32_q4_1_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { GGML_ASSERT(ne % QK4_1 == 0); - const int64_t num_blocks = ne / QK4_1; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_f32_q<<>> + const int64_t num_blocks = (ne/QK4_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_f32_q<<>> (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -301,9 +315,9 @@ static void ggml_cpy_q4_1_f32_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { - const int64_t num_blocks = ne; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_q_f32, QK4_1><<>>( + const int64_t num_blocks = (ne/QK4_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_q_f32, QK4_1><<>>( cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -314,9 +328,9 @@ static void ggml_cpy_f32_q5_0_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { GGML_ASSERT(ne % QK5_0 == 0); - const int64_t num_blocks = ne / QK5_0; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_f32_q<<>> + const int64_t num_blocks = (ne/QK5_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_f32_q<<>> (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -327,9 +341,9 @@ static void ggml_cpy_q5_0_f32_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { - const int64_t num_blocks = ne; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_q_f32, QK5_0><<>>( + const int64_t num_blocks = (ne/QK5_0 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_q_f32, QK5_0><<>>( cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -340,9 +354,9 @@ static void ggml_cpy_f32_q5_1_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { GGML_ASSERT(ne % QK5_1 == 0); - const int64_t num_blocks = ne / QK5_1; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_f32_q<<>> + const int64_t num_blocks = (ne/QK5_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_f32_q<<>> (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -353,9 +367,9 @@ static void ggml_cpy_q5_1_f32_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { - const int64_t num_blocks = ne; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_q_f32, QK5_1><<>>( + const int64_t num_blocks = (ne/QK5_1 + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_q_f32, QK5_1><<>>( cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } @@ -366,12 +380,52 @@ static void ggml_cpy_f32_iq4_nl_cuda( const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) { GGML_ASSERT(ne % QK4_NL == 0); - const int64_t num_blocks = ne / QK4_NL; - GGML_ASSERT(num_blocks < UINT_MAX); - cpy_f32_q<<>> + const int64_t num_blocks = (ne/QK4_NL + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE; + GGML_ASSERT(num_blocks <= INT_MAX); + cpy_f32_q<<>> (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13); } +// check if a same-type copy reduces to a 2D strided copy (height rows of width +// contiguous bytes), so it can use cudaMemcpy2DAsync instead of the scalar kernel +static bool ggml_cuda_cpy_as_memcpy_2d(const ggml_tensor * src0, const ggml_tensor * src1, + size_t & width, size_t & height, size_t & spitch, size_t & dpitch) { + // require matching shape: a reshaped copy maps elements by flat order, which the + // prefix walk below does not handle + if (src0->type != src1->type || !ggml_are_same_shape(src0, src1)) { + return false; + } + + // grow the contiguous prefix block shared by both tensors + size_t block_nb = ggml_element_size(src0); + int d = 0; + for (; d < GGML_MAX_DIMS; ++d) { + if (src0->nb[d] != block_nb || src1->nb[d] != block_nb) { + break; + } + block_nb *= src0->ne[d]; + } + + // d == 0: nothing contiguous; d == GGML_MAX_DIMS: fully contiguous (handled by memcpy) + if (d == 0 || d == GGML_MAX_DIMS) { + return false; + } + + // dim d carries the rows; everything above it must be a single element + for (int i = d + 1; i < GGML_MAX_DIMS; ++i) { + if (src0->ne[i] != 1) { + return false; + } + } + + width = block_nb; + height = src0->ne[d]; + spitch = src0->nb[d]; + dpitch = src1->nb[d]; + + return spitch >= width && dpitch >= width; +} + void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) { const int64_t ne = ggml_nelements(src0); GGML_ASSERT(ne == ggml_nelements(src1)); @@ -405,7 +459,10 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg const bool contiguous_srcs = ggml_is_contiguous(src0) && ggml_is_contiguous(src1); const bool can_be_transposed = nb01 == (int64_t)ggml_element_size(src0) && - src0->ne[3] == 1 && nb02 == ne00 * ne01 * (int64_t)ggml_element_size(src0); + src0->ne[3] == 1 && nb02 == ne00 * ne01 * (int64_t)ggml_element_size(src0) && + ggml_is_contiguous(src1); + + size_t mc_width = 0, mc_height = 0, mc_spitch = 0, mc_dpitch = 0; if (src0->type == src1->type && contiguous_srcs) { GGML_ASSERT(ggml_nbytes(src0) == ggml_nbytes(src1)); @@ -417,6 +474,9 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg { CUDA_CHECK(cudaMemcpyAsync(src1_ddc, src0_ddc, ggml_nbytes(src0), cudaMemcpyDeviceToDevice, main_stream)); } + } else if (ggml_cuda_cpy_as_memcpy_2d(src0, src1, mc_width, mc_height, mc_spitch, mc_dpitch)) { + CUDA_CHECK(cudaMemcpy2DAsync(src1_ddc, mc_dpitch, src0_ddc, mc_spitch, + mc_width, mc_height, cudaMemcpyDeviceToDevice, main_stream)); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { if (can_be_transposed) { ggml_cpy_scalar_cuda @@ -530,6 +590,14 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg ggml_cpy_scalar_cuda (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } + } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) { + if (can_be_transposed) { + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else { + ggml_cpy_scalar_cuda + (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32) { if (contiguous_srcs) { ggml_cpy_scalar_contiguous_cuda diff --git a/ggml/src/ggml-cuda/dequantize.cuh b/ggml/src/ggml-cuda/dequantize.cuh index 9ae1342..651524b 100644 --- a/ggml/src/ggml-cuda/dequantize.cuh +++ b/ggml/src/ggml-cuda/dequantize.cuh @@ -1,4 +1,5 @@ #include "common.cuh" +#include "convert.cuh" static __device__ __forceinline__ void dequantize_q1_0(const void * vx, const int64_t ib, const int iqs, float2 & v){ const block_q1_0 * x = (const block_q1_0 *) vx; @@ -22,6 +23,26 @@ static __device__ __forceinline__ void dequantize_q1_0(const void * vx, const in v.y = (2*bit_1 - 1) * d; } +static __device__ __forceinline__ void dequantize_q2_0(const void * vx, const int64_t ib, const int iqs, float2 & v){ + const block_q2_0 * x = (const block_q2_0 *) vx; + + const float d = x[ib].d; + + // Q2_0: 2 bits per element, 4 elements per byte. + // Stored code c in {0,1,2,3} maps to symbol s = c - 1 in {-1, 0, +1, +2}. + const int byte_index_0 = iqs / 4; + const int bit_offset_0 = (iqs % 4) * 2; + + const int byte_index_1 = (iqs + 1) / 4; + const int bit_offset_1 = ((iqs + 1) % 4) * 2; + + const int c0 = (x[ib].qs[byte_index_0] >> bit_offset_0) & 0x3; + const int c1 = (x[ib].qs[byte_index_1] >> bit_offset_1) & 0x3; + + v.x = (c0 - 1) * d; + v.y = (c1 - 1) * d; +} + static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){ const block_q4_0 * x = (const block_q4_0 *) vx; @@ -97,3 +118,335 @@ static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const in v.x *= d; v.y *= d; } + +//================================== k-quants + +// Each call dequantizes one super-block of QK_K values into y using the +// thread layout of the caller: 32 threads for q4_K, 64 threads otherwise. + +template +static __device__ __forceinline__ void dequantize_q2_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) { + const block_q2_K * x = (const block_q2_K *) vx; + + const int64_t n = tid/32; + const int64_t l = tid - 32*n; + const int64_t is = 8*n + l/16; + + const uint8_t q = x[ib].qs[32*n + l]; + dst_t * y = yy + 128*n; + + float dall = __low2half(x[ib].dm); + float dmin = __high2half(x[ib].dm); + y[l+ 0] = ggml_cuda_cast(dall * (x[ib].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[ib].scales[is+0] >> 4)); + y[l+32] = ggml_cuda_cast(dall * (x[ib].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[ib].scales[is+2] >> 4)); + y[l+64] = ggml_cuda_cast(dall * (x[ib].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[ib].scales[is+4] >> 4)); + y[l+96] = ggml_cuda_cast(dall * (x[ib].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[ib].scales[is+6] >> 4)); +} + +template +static __device__ __forceinline__ void dequantize_q3_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) { + const block_q3_K * x = (const block_q3_K *) vx; + + const int64_t r = tid/4; + const int64_t t = r/2; + const int64_t is0 = r%2; + const int64_t l0 = 16*is0 + 4*(tid%4); + const int64_t n = t / 4; + const int64_t j = t - 4*n; + + uint8_t m = 1 << (4*n + j); + int64_t is = 8*n + 2*j + is0; + int shift = 2*j; + + int8_t us = is < 4 ? (x[ib].scales[is-0] & 0xF) | (((x[ib].scales[is+8] >> 0) & 3) << 4) : + is < 8 ? (x[ib].scales[is-0] & 0xF) | (((x[ib].scales[is+4] >> 2) & 3) << 4) : + is < 12 ? (x[ib].scales[is-8] >> 4) | (((x[ib].scales[is+0] >> 4) & 3) << 4) : + (x[ib].scales[is-8] >> 4) | (((x[ib].scales[is-4] >> 6) & 3) << 4); + float d_all = x[ib].d; + float dl = d_all * (us - 32); + + dst_t * y = yy + 128*n + 32*j; + const uint8_t * q = x[ib].qs + 32*n; + const uint8_t * hm = x[ib].hmask; + + for (int l = l0; l < l0+4; ++l) { + y[l] = ggml_cuda_cast(dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4))); + } +} + +static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) { + if (j < 4) { + d = q[j] & 63; m = q[j + 4] & 63; + } else { + d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4); + m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); + } +} + +template +static __device__ __forceinline__ void dequantize_q4_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) { + const block_q4_K * x = (const block_q4_K *) vx; + + // assume 32 threads + const int64_t il = tid/8; + const int64_t ir = tid%8; + const int64_t is = 2*il; + const int64_t n = 4; + + dst_t * y = yy + 64*il + n*ir; + + const float dall = __low2half(x[ib].dm); + const float dmin = __high2half(x[ib].dm); + + const uint8_t * q = x[ib].qs + 32*il + n*ir; + + uint8_t sc, m; + get_scale_min_k4(is + 0, x[ib].scales, sc, m); + const float d1 = dall * sc; const float m1 = dmin * m; + get_scale_min_k4(is + 1, x[ib].scales, sc, m); + const float d2 = dall * sc; const float m2 = dmin * m; + for (int l = 0; l < n; ++l) { + y[l + 0] = ggml_cuda_cast(d1 * (q[l] & 0xF) - m1); + y[l +32] = ggml_cuda_cast(d2 * (q[l] >> 4) - m2); + } +} + +template +static __device__ __forceinline__ void dequantize_q5_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) { + const block_q5_K * x = (const block_q5_K *) vx; + + // assume 64 threads - this is very slightly better than the one below + const int64_t il = tid/16; // il is in 0...3 + const int64_t ir = tid%16; // ir is in 0...15 + const int64_t is = 2*il; // is is in 0...6 + + dst_t * y = yy + 64*il + 2*ir; + + const float dall = __low2half(x[ib].dm); + const float dmin = __high2half(x[ib].dm); + + const uint8_t * ql = x[ib].qs + 32*il + 2*ir; + const uint8_t * qh = x[ib].qh + 2*ir; + + uint8_t sc, m; + get_scale_min_k4(is + 0, x[ib].scales, sc, m); + const float d1 = dall * sc; const float m1 = dmin * m; + get_scale_min_k4(is + 1, x[ib].scales, sc, m); + const float d2 = dall * sc; const float m2 = dmin * m; + + uint8_t hm = 1 << (2*il); + y[ 0] = ggml_cuda_cast(d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1); + y[ 1] = ggml_cuda_cast(d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1); + hm <<= 1; + y[32] = ggml_cuda_cast(d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2); + y[33] = ggml_cuda_cast(d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2); +} + +template +static __device__ __forceinline__ void dequantize_q6_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) { + const block_q6_K * x = (const block_q6_K *) vx; + + // assume 64 threads - this is very slightly better than the one below + const int64_t ip = tid/32; // ip is 0 or 1 + const int64_t il = tid - 32*ip; // 0...32 + const int64_t is = 8*ip + il/16; + + dst_t * y = yy + 128*ip + il; + + const float d = x[ib].d; + + const uint8_t * ql = x[ib].ql + 64*ip + il; + const uint8_t qh = x[ib].qh[32*ip + il]; + const int8_t * sc = x[ib].scales + is; + + y[ 0] = ggml_cuda_cast(d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32)); + y[32] = ggml_cuda_cast(d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32)); + y[64] = ggml_cuda_cast(d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32)); + y[96] = ggml_cuda_cast(d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32)); +} + +//================================== i-quants + +// Each call dequantizes one super-block of QK_K values into y with 32 +// threads; iq4_nl packs QK_K/QK4_NL sub-blocks per super-block. + +template +static __device__ __forceinline__ void dequantize_iq2_xxs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + + const block_iq2_xxs * x = (const block_iq2_xxs *) vx; + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 8*il; + const uint16_t * q2 = x[ibs].qs + 4*ib; + const uint8_t * aux8 = (const uint8_t *)q2; + const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]); + const uint32_t aux32 = q2[2] | (q2[3] << 16); + const float d = (float)x[ibs].d * (0.5f + (aux32 >> 28)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127]; + for (int j = 0; j < 8; ++j) { + y[j] = ggml_cuda_cast(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f)); + } +} + +template +static __device__ __forceinline__ void dequantize_iq2_xs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + + const block_iq2_xs * x = (const block_iq2_xs *) vx; + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 8*il; + const uint16_t * q2 = x[ibs].qs + 4*ib; + const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511)); + const float d = (float)x[ibs].d * (0.5f + ((x[ibs].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; + for (int j = 0; j < 8; ++j) { + y[j] = ggml_cuda_cast(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f)); + } +} + +template +static __device__ __forceinline__ void dequantize_iq2_s(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + + const block_iq2_s * x = (const block_iq2_s *) vx; + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 8*il; + const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[ibs].qs[4*ib+il] | ((x[ibs].qh[ib] << (8-2*il)) & 0x300))); + const float d = (float)x[ibs].d * (0.5f + ((x[ibs].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; + const uint8_t signs = x[ibs].qs[QK_K/8+4*ib+il]; + for (int j = 0; j < 8; ++j) { + y[j] = ggml_cuda_cast(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f)); + } +} + +template +static __device__ __forceinline__ void dequantize_iq3_xxs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + + const block_iq3_xxs * x = (const block_iq3_xxs *) vx; + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 8*il; + const uint8_t * q3 = x[ibs].qs + 8*ib; + const uint16_t * gas = (const uint16_t *)(x[ibs].qs + QK_K/4) + 2*ib; + const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]); + const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]); + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float d = (float)x[ibs].d * (0.5f + (aux32 >> 28)) * 0.5f; + const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127]; + for (int j = 0; j < 4; ++j) { + y[j+0] = ggml_cuda_cast(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f)); + y[j+4] = ggml_cuda_cast(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f)); + } +} + +template +static __device__ __forceinline__ void dequantize_iq3_s(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + + const block_iq3_s * x = (const block_iq3_s *) vx; + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 8*il; + const uint8_t * qs = x[ibs].qs + 8*ib; + const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[ibs].qh[ib] << (8-2*il)) & 256))); + const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[ibs].qh[ib] << (7-2*il)) & 256))); + const float d = (float)x[ibs].d * (1 + 2*((x[ibs].scales[ib/2] >> 4*(ib%2)) & 0xf)); + const uint8_t signs = x[ibs].signs[4*ib + il]; + for (int j = 0; j < 4; ++j) { + y[j+0] = ggml_cuda_cast(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f)); + y[j+4] = ggml_cuda_cast(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f)); + } +} + +template +static __device__ __forceinline__ void dequantize_iq1_s(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + + const block_iq1_s * x = (const block_iq1_s *) vx; + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 8*il; + const float delta = x[ibs].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA; + const float d = (float)x[ibs].d * (2*((x[ibs].qh[ib] >> 12) & 7) + 1); + uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32; + grid32[0] = iq1s_grid_gpu[x[ibs].qs[4*ib+il] | (((x[ibs].qh[ib] >> 3*il) & 7) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + for (int j = 0; j < 8; ++j) { + y[j] = ggml_cuda_cast(d * (q[j] + delta)); + } +} + +template +static __device__ __forceinline__ void dequantize_iq1_m(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + + const block_iq1_m * x = (const block_iq1_m *) vx; + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 8*il; + const uint16_t * sc = (const uint16_t *)x[ibs].scales; + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); + const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4); + const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1); + const float delta = x[ibs].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA; + uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32; + grid32[0] = iq1s_grid_gpu[x[ibs].qs[4*ib+il] | (((x[ibs].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + for (int j = 0; j < 8; ++j) { + y[j] = ggml_cuda_cast(d * (q[j] + delta)); + } +} + +template +static __device__ __forceinline__ void dequantize_iq4_nl(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + + const block_iq4_nl * x = (const block_iq4_nl *) vx + ibs*(QK_K/QK4_NL); + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 4*il; + const uint8_t * q4 = x[ib].qs + 4*il; + const float d = (float)x[ib].d; + for (int j = 0; j < 4; ++j) { + y[j+ 0] = ggml_cuda_cast(d * kvalues_iq4nl[q4[j] & 0xf]); + y[j+16] = ggml_cuda_cast(d * kvalues_iq4nl[q4[j] >> 4]); + } +} + +template +static __device__ __forceinline__ void dequantize_iq4_xs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + const block_iq4_xs * x = (const block_iq4_xs *)vx; + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 4*il; + const uint8_t * q4 = x[ibs].qs + 16*ib + 4*il; + const float d = (float)x[ibs].d * ((((x[ibs].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[ibs].scales_h >> 2*ib) & 3) << 4)) - 32); + for (int j = 0; j < 4; ++j) { + y[j+ 0] = ggml_cuda_cast(d * kvalues_iq4nl[q4[j] & 0xf]); + y[j+16] = ggml_cuda_cast(d * kvalues_iq4nl[q4[j] >> 4]); + } +} + +template +static __device__ __forceinline__ void dequantize_mxfp4(const void * vx, const int64_t ibs, dst_t * yy, const int tid) { + + const block_mxfp4 * x = (const block_mxfp4 *) vx + ibs*(QK_K/QK_MXFP4); + + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + 32*ib + 4*il; + const uint8_t * q4 = x[ib].qs + 4*il; + const float d = ggml_cuda_e8m0_to_fp32(x[ib].e); + for (int j = 0; j < 4; ++j) { + y[j+ 0] = ggml_cuda_cast(d * kvalues_mxfp4[q4[j] & 0xf]*0.5f); + y[j+16] = ggml_cuda_cast(d * kvalues_mxfp4[q4[j] >> 4]*0.5f); + } +} diff --git a/ggml/src/ggml-cuda/dsv4-hc.cu b/ggml/src/ggml-cuda/dsv4-hc.cu new file mode 100644 index 0000000..2d0b3cf --- /dev/null +++ b/ggml/src/ggml-cuda/dsv4-hc.cu @@ -0,0 +1,333 @@ +#include "common.cuh" +#include "dsv4-hc.cuh" + + +static constexpr int DSV4_HC = 4; + + +static __device__ void dsv4_hc_comb_norm_cols(float * comb, float eps) { + for (int idst = 0; idst < DSV4_HC; ++idst) { + float sum = eps; + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + sum += comb[idst + DSV4_HC*isrc]; + } + + const float inv_sum = 1.0f / sum; + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + comb[idst + DSV4_HC*isrc] *= inv_sum; + } + } +} + +static __device__ void dsv4_hc_comb_norm_rows(float * comb, float eps) { + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + float sum = eps; + for (int idst = 0; idst < DSV4_HC; ++idst) { + sum += comb[idst + DSV4_HC*isrc]; + } + + const float inv_sum = 1.0f / sum; + for (int idst = 0; idst < DSV4_HC; ++idst) { + comb[idst + DSV4_HC*isrc] *= inv_sum; + } + } +} + +static __global__ void dsv4_hc_comb_f32( + const float * mixes, + const float * scale, + const float * base, + float * dst, + int64_t n_tokens, + int64_t sm0, + int64_t sm1, + int64_t ss0, + int64_t sb0, + int64_t sd0, + int64_t sd1, + int64_t sd2, + float eps, + int32_t n_iter) { + constexpr int comb_offset = 2*DSV4_HC; + + ggml_cuda_pdl_lc(); + const int64_t it = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; + + if (it >= n_tokens) { + return; + } + + ggml_cuda_pdl_sync(); + + const float scale_comb = scale[2*ss0]; + float comb[DSV4_HC*DSV4_HC]; + + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + float max = -INFINITY; + for (int idst = 0; idst < DSV4_HC; ++idst) { + const int idx = idst + DSV4_HC*isrc; + const float v = mixes[(comb_offset + idx)*sm0 + it*sm1] * scale_comb + base[(comb_offset + idx)*sb0]; + comb[idx] = v; + max = fmaxf(max, v); + } + + float sum = 0.0f; + for (int idst = 0; idst < DSV4_HC; ++idst) { + const int idx = idst + DSV4_HC*isrc; + const float v = expf(comb[idx] - max); + comb[idx] = v; + sum += v; + } + + const float inv_sum = 1.0f / sum; + for (int idst = 0; idst < DSV4_HC; ++idst) { + const int idx = idst + DSV4_HC*isrc; + comb[idx] = comb[idx] * inv_sum + eps; + } + } + + dsv4_hc_comb_norm_cols(comb, eps); + for (int32_t i = 1; i < n_iter; ++i) { + dsv4_hc_comb_norm_rows(comb, eps); + dsv4_hc_comb_norm_cols(comb, eps); + } + + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + for (int idst = 0; idst < DSV4_HC; ++idst) { + const int idx = idst + DSV4_HC*isrc; + dst[idst*sd0 + isrc*sd1 + it*sd2] = comb[idx]; + } + } +} + +template +static __global__ void dsv4_hc_pre_f32( + const float * x, + const float * weights, + float * dst, + int64_t n_embd, + int64_t hc, + int64_t n_tokens, + int64_t sx0, + int64_t sx1, + int64_t sx2, + int64_t sw0, + int64_t sw1, + int64_t sw2, + int64_t sd0, + int64_t sd1, + float scale) { + ggml_cuda_pdl_lc(); + const int64_t ir = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; + const int64_t nr = n_embd * n_tokens; + + if (ir >= nr) { + return; + } + + ggml_cuda_pdl_sync(); + + const int64_t i0 = ir % n_embd; + const int64_t it = ir / n_embd; + + float sum = 0.0f; + for (int64_t ih = 0; ih < hc; ++ih) { + const float xv = x[i0*sx0 + ih*sx1 + it*sx2]; + float wv; + if constexpr (gated) { + wv = 1.0f / (1.0f + expf(-weights[i0*sw0 + ih*sw1 + it*sw2])); + } else { + wv = weights[ih*sw0 + it*sw1]; + } + sum += xv * wv; + } + + dst[i0*sd0 + it*sd1] = scale * sum; +} + +// gated: post = gate_scale_out*sigmoid(gate_scale_in*post), the fused SCALE -> SIGMOID -> SCALE chain of qwen4exp +template +static __global__ void dsv4_hc_post_f32( + const float * x, + const float * residual, + const float * post, + const float * comb, + float * dst, + int64_t n_embd, + int64_t hc, + int64_t n_tokens, + int64_t sx0, + int64_t sx1, + int64_t sr0, + int64_t sr1, + int64_t sr2, + int64_t sp0, + int64_t sp1, + int64_t sc0, + int64_t sc1, + int64_t sc2, + int64_t sd0, + int64_t sd1, + int64_t sd2, + float gate_scale_in, + float gate_scale_out) { + ggml_cuda_pdl_lc(); + const int64_t ir = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; + const int64_t nr = n_embd * hc * n_tokens; + + if (ir >= nr) { + return; + } + + ggml_cuda_pdl_sync(); + + const int64_t i0 = ir % n_embd; + const int64_t idst = (ir / n_embd) % hc; + const int64_t it = ir / (n_embd * hc); + + float p = post[idst*sp0 + it*sp1]; + if constexpr (gated) { + p = gate_scale_out * (1.0f / (1.0f + expf(-(gate_scale_in * p)))); + } + + float sum = x[i0*sx0 + it*sx1] * p; + if constexpr (has_comb) { + for (int64_t isrc = 0; isrc < hc; ++isrc) { + sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2]; + } + } else { + sum += residual[i0*sr0 + idst*sr1 + it*sr2]; + } + + dst[i0*sd0 + idst*sd1 + it*sd2] = sum; +} + +void ggml_cuda_op_dsv4_hc_comb(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * mixes = dst->src[0]; + const ggml_tensor * scale = dst->src[1]; + const ggml_tensor * base = dst->src[2]; + + GGML_ASSERT(mixes->type == GGML_TYPE_F32); + GGML_ASSERT(scale->type == GGML_TYPE_F32); + GGML_ASSERT(base->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + constexpr int64_t hc_mix_dim = (2 + DSV4_HC)*DSV4_HC; + + GGML_ASSERT(mixes->ne[0] == hc_mix_dim); + GGML_ASSERT(dst->ne[0] == DSV4_HC); + GGML_ASSERT(dst->ne[1] == DSV4_HC); + GGML_ASSERT(dst->ne[2] == mixes->ne[1]); + GGML_ASSERT(scale->ne[0] >= 3); + GGML_ASSERT(base->ne[0] == hc_mix_dim); + + GGML_TENSOR_LOCALS(size_t, nbm, mixes, nb); + GGML_TENSOR_LOCALS(size_t, nbs, scale, nb); + GGML_TENSOR_LOCALS(size_t, nbb, base, nb); + GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const int64_t n_tokens = mixes->ne[1]; + const float eps = ggml_get_op_params_f32(dst, 0); + const int32_t n_iter = ggml_get_op_params_i32(dst, 1); + + const int block_size = 256; + const dim3 block_dims(block_size, 1, 1); + const dim3 grid_dims((n_tokens + block_size - 1) / block_size, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream()); + + ggml_cuda_kernel_launch(dsv4_hc_comb_f32, launch_params, + (const float *) mixes->data, (const float *) scale->data, (const float *) base->data, (float *) dst->data, + n_tokens, + nbm0 / sizeof(float), nbm1 / sizeof(float), + nbs0 / sizeof(float), + nbb0 / sizeof(float), + nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float), + eps, n_iter); +} + +void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * x = dst->src[0]; + const ggml_tensor * weights = dst->src[1]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(weights->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_TENSOR_LOCALS(size_t, nbx, x, nb); + GGML_TENSOR_LOCALS(size_t, nbw, weights, nb); + GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const int64_t n_embd = x->ne[0]; + const int64_t hc = x->ne[1]; + const int64_t n_tokens = x->ne[2]; + + const float scale = ggml_get_op_params_f32(dst, 0); + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + + const int block_size = 256; + const int64_t nr = n_embd * n_tokens; + const dim3 block_dims(block_size, 1, 1); + const dim3 grid_dims((nr + block_size - 1) / block_size, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream()); + + auto kernel = gated ? dsv4_hc_pre_f32 : dsv4_hc_pre_f32; + ggml_cuda_kernel_launch(kernel, launch_params, + (const float *) x->data, (const float *) weights->data, (float *) dst->data, + n_embd, hc, n_tokens, + nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float), + nbw0 / sizeof(float), nbw1 / sizeof(float), nbw2 / sizeof(float), + nbd0 / sizeof(float), nbd1 / sizeof(float), + scale); +} + +void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + const ggml_tensor * gate_scale_in, const ggml_tensor * gate_scale_out) { + const ggml_tensor * x = dst->src[0]; + const ggml_tensor * residual = dst->src[1]; + const ggml_tensor * comb = dst->src[3]; + + // when fused, read the raw post weights from before the SCALE -> SIGMOID -> SCALE chain + const bool gated = gate_scale_in != nullptr; + const ggml_tensor * post = gated ? gate_scale_in->src[0] : dst->src[2]; + + const float gate_in = gated ? ggml_get_op_params_f32(gate_scale_in, 0) : 1.0f; + const float gate_out = gated ? ggml_get_op_params_f32(gate_scale_out, 0) : 1.0f; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(residual->type == GGML_TYPE_F32); + GGML_ASSERT(post->type == GGML_TYPE_F32); + GGML_ASSERT(comb == nullptr || comb->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_TENSOR_LOCALS(size_t, nbx, x, nb); + GGML_TENSOR_LOCALS(size_t, nbr, residual, nb); + GGML_TENSOR_LOCALS(size_t, nbp, post, nb); + GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const size_t nbc0 = comb ? comb->nb[0] : 0; + const size_t nbc1 = comb ? comb->nb[1] : 0; + const size_t nbc2 = comb ? comb->nb[2] : 0; + + const int64_t n_embd = x->ne[0]; + const int64_t n_tokens = x->ne[1]; + const int64_t hc = residual->ne[1]; + + const int block_size = 256; + const int64_t nr = n_embd * hc * n_tokens; + const dim3 block_dims(block_size, 1, 1); + const dim3 grid_dims((nr + block_size - 1) / block_size, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream()); + + auto kernel = comb ? (gated ? dsv4_hc_post_f32 : dsv4_hc_post_f32) + : (gated ? dsv4_hc_post_f32 : dsv4_hc_post_f32); + ggml_cuda_kernel_launch(kernel, launch_params, + (const float *) x->data, (const float *) residual->data, + (const float *) post->data, comb ? (const float *) comb->data : nullptr, (float *) dst->data, + n_embd, hc, n_tokens, + nbx0 / sizeof(float), nbx1 / sizeof(float), + nbr0 / sizeof(float), nbr1 / sizeof(float), nbr2 / sizeof(float), + nbp0 / sizeof(float), nbp1 / sizeof(float), + nbc0 / sizeof(float), nbc1 / sizeof(float), nbc2 / sizeof(float), + nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float), + gate_in, gate_out); +} diff --git a/ggml/src/ggml-cuda/dsv4-hc.cuh b/ggml/src/ggml-cuda/dsv4-hc.cuh new file mode 100644 index 0000000..1885e76 --- /dev/null +++ b/ggml/src/ggml-cuda/dsv4-hc.cuh @@ -0,0 +1,8 @@ +#include "common.cuh" +#include "ggml.h" + +void ggml_cuda_op_dsv4_hc_comb(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +void ggml_cuda_op_dsv4_hc_pre(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +// gate_scale_in/gate_scale_out: the SCALE nodes of a fused SCALE -> SIGMOID -> SCALE chain feeding src[2] +void ggml_cuda_op_dsv4_hc_post(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + const ggml_tensor * gate_scale_in = nullptr, const ggml_tensor * gate_scale_out = nullptr); diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index beeb523..41d9483 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -44,6 +44,46 @@ typedef void (* fattn_kernel_t)( typedef float (*vec_dot_KQ_t)( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds); +struct ggml_cuda_flash_attn_ext_f16_extra_data { + uintptr_t K; + uintptr_t V; + uintptr_t end; +}; + +static inline ggml_cuda_flash_attn_ext_f16_extra_data ggml_cuda_flash_attn_ext_get_f16_extra_data( + const ggml_tensor * dst, const bool need_f16_K, const bool need_f16_V) { + GGML_ASSERT(dst->op == GGML_OP_FLASH_ATTN_EXT); + + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + GGML_ASSERT(K != nullptr); + GGML_ASSERT(V != nullptr); + + const bool V_is_K_view = V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)); + + ggml_cuda_flash_attn_ext_f16_extra_data data = {}; + data.end = (uintptr_t) dst->data + ggml_nbytes(dst); + + if (need_f16_K && K->type != GGML_TYPE_F16) { + data.end = GGML_PAD(data.end, 128); + data.K = data.end; + data.end += ggml_nelements(K)*ggml_type_size(GGML_TYPE_F16); + } + + if (need_f16_V && V->type != GGML_TYPE_F16) { + if (V_is_K_view) { + data.V = data.K; + } else { + data.end = GGML_PAD(data.end, 128); + data.V = data.end; + data.end += ggml_nelements(V)*ggml_type_size(GGML_TYPE_F16); + } + } + + return data; +} + template static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_f16( const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds_v) { @@ -376,9 +416,14 @@ static __device__ __forceinline__ void dequantize_V_q4_0(const void * __restrict int q; static_assert(ne == 2 || ne == 4, "bad ne"); ggml_cuda_memcpy_1(&q, x[ib].qs + iqs); +#if defined(GGML_USE_HIP) + // Keep this VMEM read close to its packed-byte dequantization. Hoisting it too far + // increases VGPR pressure substantially in some FlashAttention vector kernels. + __builtin_amdgcn_sched_group_barrier(0x20, 1, 0); +#endif // defined(GGML_USE_HIP) q >>= 4*shift; q &= 0x0F0F0F0F; - q = __vsubss4(q, 0x08080808); + q = __vsub4(q, 0x08080808); const int8_t * q8 = (const int8_t *) &q; @@ -468,7 +513,7 @@ static __device__ __forceinline__ void dequantize_V_q5_0(const void * __restrict } } - q = __vsubss4(q, 0x10101010); + q = __vsub4(q, 0x10101010); const int8_t * q8 = (const int8_t *) &q; @@ -624,7 +669,10 @@ constexpr __device__ dequantize_V_t get_dequantize_V() { template __launch_bounds__(FATTN_KQ_STRIDE/2, 1) static __global__ void flash_attn_mask_to_KV_max( - const half2 * __restrict__ mask, int * __restrict__ KV_max, const int ne30, const int s31, const int s33) { + const half2 * mask_ptr, int * KV_max_ptr, const int ne30, const int64_t s31, const int64_t s33) { + const half2 * GGML_CUDA_RESTRICT mask = mask_ptr; + int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr; + const int ne31 = gridDim.x; const int tid = threadIdx.x; const int sequence = blockIdx.y; @@ -636,6 +684,7 @@ static __global__ void flash_attn_mask_to_KV_max( if (tid < WARP_SIZE) { buf_iw[tid] = 1; } + ggml_cuda_pdl_sync(); __syncthreads(); int KV_max_sj = (ne30 - 1) * FATTN_KQ_STRIDE; @@ -674,11 +723,14 @@ static __global__ void flash_attn_mask_to_KV_max( KV_max[sequence*ne31 + jt] = KV_max_sj; } +void ggml_cuda_flash_attn_ext_compact_mask( + const ggml_tensor * mask, int32_t * indices, int32_t * counts, int32_t n_queries, int32_t ncols1, int32_t n_kv_max, cudaStream_t stream); + template // D == head size __launch_bounds__(D, 1) static __global__ void flash_attn_stream_k_fixup_uniform( - float * __restrict__ dst, - const float2 * __restrict__ dst_fixup, + float * dst_ptr, + const float2 * dst_fixup_ptr, const int ne01, const int ne02, const int ne12, const int nblocks_stream_k, const int gqa_ratio, @@ -687,6 +739,9 @@ static __global__ void flash_attn_stream_k_fixup_uniform( const uint3 fd_iter_j_z, const uint3 fd_iter_j) { constexpr int ncols = ncols1*ncols2; + ggml_cuda_pdl_lc(); + float * GGML_CUDA_RESTRICT dst = dst_ptr; + const float2 * GGML_CUDA_RESTRICT dst_fixup = dst_fixup_ptr; const int tile_idx = blockIdx.x; // One block per output tile. const int j = blockIdx.y; @@ -718,6 +773,7 @@ static __global__ void flash_attn_stream_k_fixup_uniform( dst += sequence*ne02*ne01*D + jt*ne02*(ncols1*D) + zt_Q*D + (j*ne02 + c)*D + tid; + ggml_cuda_pdl_sync(); // Load the partial result that needs a fixup float dst_val = *dst; float max_val; @@ -757,8 +813,8 @@ static __global__ void flash_attn_stream_k_fixup_uniform( template // D == head size __launch_bounds__(D, 1) static __global__ void flash_attn_stream_k_fixup_general( - float * __restrict__ dst, - const float2 * __restrict__ dst_fixup, + float * dst_ptr, + const float2 * dst_fixup_ptr, const int ne01, const int ne02, const int gqa_ratio, const int total_work, @@ -766,6 +822,8 @@ static __global__ void flash_attn_stream_k_fixup_general( const uint3 fd_iter_k_j_z, const uint3 fd_iter_k_j, const uint3 fd_iter_k) { + float * GGML_CUDA_RESTRICT dst = dst_ptr; + const float2 * GGML_CUDA_RESTRICT dst_fixup = dst_fixup_ptr; constexpr int ncols = ncols1*ncols2; const int bidx0 = blockIdx.x; @@ -809,6 +867,7 @@ static __global__ void flash_attn_stream_k_fixup_general( float dst_val = 0.0f; float max_val = 0.0f; float rowsum = 0.0f; + ggml_cuda_pdl_sync(); { dst_val = *dst; @@ -863,10 +922,14 @@ static __global__ void flash_attn_stream_k_fixup_general( template // D == head size __launch_bounds__(D, 1) static __global__ void flash_attn_combine_results( - const float * __restrict__ VKQ_parts, - const float2 * __restrict__ VKQ_meta, - float * __restrict__ dst, + const float * VKQ_parts_ptr, + const float2 * VKQ_meta_ptr, + float * dst_ptr, const int parallel_blocks) { + ggml_cuda_pdl_lc(); + const float * GGML_CUDA_RESTRICT VKQ_parts = VKQ_parts_ptr; + const float2 * GGML_CUDA_RESTRICT VKQ_meta = VKQ_meta_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; // Dimension 0: threadIdx.x // Dimension 1: blockIdx.x // Dimension 2: blockIdx.y @@ -890,6 +953,7 @@ static __global__ void flash_attn_combine_results( __builtin_assume(tid < D); extern __shared__ float2 meta[]; + ggml_cuda_pdl_sync(); for (int i = tid; i < 2*parallel_blocks; i += D) { ((float *) meta)[i] = ((const float *)VKQ_meta) [i]; } @@ -916,7 +980,8 @@ static __global__ void flash_attn_combine_results( template void launch_fattn( ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared, - const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const int warp_size = WARP_SIZE + const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const bool use_sparse, + const int warp_size = WARP_SIZE, const bool async_kv_preload = false ) { constexpr int ncols = ncols1 * ncols2; @@ -946,8 +1011,9 @@ void launch_fattn( const int cc = ggml_cuda_info().devices[id].cc; const int nsm = ggml_cuda_info().devices[id].nsm; - ggml_cuda_pool_alloc K_f16(pool); - ggml_cuda_pool_alloc V_f16(pool); + const ggml_cuda_flash_attn_ext_f16_extra_data f16_extra = + ggml_cuda_flash_attn_ext_get_f16_extra_data(KQV, need_f16_K, need_f16_V); + ggml_cuda_pool_alloc KV_max(pool); ggml_cuda_pool_alloc dst_tmp(pool); ggml_cuda_pool_alloc dst_tmp_meta(pool); @@ -966,10 +1032,11 @@ void launch_fattn( const size_t bs = ggml_blck_size(K->type); const size_t ts = ggml_type_size(K->type); - K_f16.alloc(ggml_nelements(K)); + GGML_ASSERT(f16_extra.K != 0); + half * K_f16 = (half *) f16_extra.K; if (ggml_is_contiguously_allocated(K)) { to_fp16_cuda_t to_fp16 = ggml_get_to_fp16_cuda(K->type); - to_fp16(K_data, K_f16.ptr, ggml_nelements(K), main_stream); + to_fp16(K_data, K_f16, ggml_nelements(K), main_stream); nb11 = nb11*bs*sizeof(half)/ts; nb12 = nb12*bs*sizeof(half)/ts; @@ -980,13 +1047,13 @@ void launch_fattn( const int64_t s01 = nb11 / ts; const int64_t s02 = nb12 / ts; const int64_t s03 = nb13 / ts; - to_fp16(K_data, K_f16.ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], s01, s02, s03, main_stream); + to_fp16(K_data, K_f16, K->ne[0], K->ne[1], K->ne[2], K->ne[3], s01, s02, s03, main_stream); nb11 = K->ne[0] * sizeof(half); nb12 = K->ne[1] * nb11; nb13 = K->ne[2] * nb12; } - K_data = (char *) K_f16.ptr; + K_data = (char *) K_f16; } if (need_f16_V && V->type != GGML_TYPE_F16) { @@ -999,11 +1066,12 @@ void launch_fattn( const size_t bs = ggml_blck_size(V->type); const size_t ts = ggml_type_size(V->type); - V_f16.alloc(ggml_nelements(V)); + GGML_ASSERT(f16_extra.V != 0); + half * V_f16 = (half *) f16_extra.V; if (ggml_is_contiguously_allocated(V)) { to_fp16_cuda_t to_fp16 = ggml_get_to_fp16_cuda(V->type); - to_fp16(V_data, V_f16.ptr, ggml_nelements(V), main_stream); - V_data = (char *) V_f16.ptr; + to_fp16(V_data, V_f16, ggml_nelements(V), main_stream); + V_data = (char *) V_f16; nb21 = nb21*bs*sizeof(half)/ts; nb22 = nb22*bs*sizeof(half)/ts; @@ -1014,13 +1082,13 @@ void launch_fattn( const int64_t s01 = nb21 / ts; const int64_t s02 = nb22 / ts; const int64_t s03 = nb23 / ts; - to_fp16(V_data, V_f16.ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], s01, s02, s03, main_stream); + to_fp16(V_data, V_f16, V->ne[0], V->ne[1], V->ne[2], V->ne[3], s01, s02, s03, main_stream); nb21 = V->ne[0] * sizeof(half); nb22 = V->ne[1] * nb21; nb23 = V->ne[2] * nb22; } - V_data = (char *) V_f16.ptr; + V_data = (char *) V_f16; } } @@ -1029,12 +1097,27 @@ void launch_fattn( const int ntiles_z_gqa = ((gqa_ratio + ncols2 - 1) / ncols2); const int ntiles_dst = ntiles_x * ntiles_z_gqa * K->ne[2] * Q->ne[3]; + // sparse: a query tile of ncols1 queries shares one index list, the union of the queries' visible columns + int32_t n_kv_max = 0; + if (use_sparse) { + GGML_ASSERT(mask != nullptr); + const int32_t n_kv_max_query = ggml_get_op_params_i32(KQV, 4); + GGML_ASSERT(n_kv_max_query > 0); + n_kv_max = std::min(K->ne[1], int64_t(ncols1)*n_kv_max_query); + + const size_t n_lists = size_t(ntiles_x) * mask->ne[3]; + + KV_max.alloc(size_t(n_kv_max)*n_lists + n_lists); + ggml_cuda_flash_attn_ext_compact_mask(mask, KV_max.ptr, KV_max.ptr + size_t(n_kv_max)*n_lists, Q->ne[1], ncols1, n_kv_max, main_stream); + } + // Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped. // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or // multiple sequences of possibly different lengths. - if (mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) { - const int s31 = mask->nb[1] / sizeof(half2); - const int s33 = mask->nb[3] / sizeof(half2); + const bool scan_mask = !use_sparse && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1); + if (scan_mask) { + const int64_t s31 = mask->nb[1] / sizeof(half2); + const int64_t s33 = mask->nb[3] / sizeof(half2); const dim3 blocks_num_KV_max(ntiles_x, Q->ne[3], 1); const dim3 block_dim_KV_max(FATTN_KQ_STRIDE/2, 1, 1); @@ -1043,8 +1126,9 @@ void launch_fattn( const int iter_k = K->ne[1] / FATTN_KQ_STRIDE; KV_max.alloc(ne_KV_max); - flash_attn_mask_to_KV_max<<>> - ((const half2 *) mask->data, KV_max.ptr, iter_k, s31, s33); + ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_KV_max, block_dim_KV_max, 0, main_stream); + ggml_cuda_kernel_launch(flash_attn_mask_to_KV_max, launch_params, + (const half2 *) mask->data, KV_max.ptr, iter_k, s31, s33); CUDA_CHECK(cudaGetLastError()); } @@ -1054,16 +1138,32 @@ void launch_fattn( GGML_ASSERT(max_blocks_per_sm > 0); int parallel_blocks = max_blocks_per_sm; - const int ntiles_KV = (K->ne[1] + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length. + const int64_t n_kv = use_sparse ? n_kv_max : K->ne[1]; + const int ntiles_KV = (n_kv + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length. dim3 blocks_num; if (stream_k) { - // For short contexts it can be faster to have the SMs work on whole tiles because this lets us skip the fixup. - const int max_blocks = max_blocks_per_sm*nsm; - const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; - const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); + // Stream-K splits the work before the mask scan is applied, so skipped KV tiles make the blocks uneven. + const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload && scan_mask; + + auto should_use_stream_k = [prefer_whole_tiles](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) { + const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks; + const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves); + + if (prefer_whole_tiles && tiles_efficiency_percent >= 75) { + return false; + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE) { + return true; + } + if (amd_wmma_available(cc) && DKQ == 64) { + return true; // TODO better configuration + } + return tiles_efficiency_percent < 75; + }; - const bool use_stream_k = cc >= GGML_CUDA_CC_ADA_LOVELACE || amd_wmma_available(cc) || tiles_efficiency_percent < 75; + const int max_blocks = max_blocks_per_sm*nsm; + const bool use_stream_k = should_use_stream_k(cc, ntiles_dst, max_blocks, Q->ne[0]); blocks_num.x = ntiles_dst; blocks_num.y = 1; @@ -1146,7 +1246,9 @@ void launch_fattn( const uint3 ne01 = init_fastdiv_values(Q->ne[1]); GGML_ASSERT(block_dim.x % warp_size == 0); - fattn_kernel<<>>( + + ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num, block_dim, nbytes_shared, main_stream); + ggml_cuda_kernel_launch(fattn_kernel, launch_params, (const char *) Q->data, K_data, V_data, @@ -1156,7 +1258,7 @@ void launch_fattn( !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], - K->ne[0], K->ne[1], K->ne[2], K->ne[3], nb11, nb12, nb13, + K->ne[0], n_kv, K->ne[2], K->ne[3], nb11, nb12, nb13, nb21, nb22, nb23, mask ? mask->ne[1] : 0, mask ? mask->ne[2] : 0, mask ? mask->ne[3] : 0, mask ? mask->nb[1] : 0, mask ? mask->nb[2] : 0, mask ? mask->nb[3] : 0 @@ -1176,9 +1278,9 @@ void launch_fattn( const dim3 block_dim_combine(DV, 1, 1); const dim3 blocks_num_combine = {(unsigned)ntiles_dst, ncols1, ncols2}; - flash_attn_stream_k_fixup_uniform - <<>> - ((float *) KQV->data, dst_tmp_meta.ptr, + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_combine, block_dim_combine, 0, main_stream); + ggml_cuda_kernel_launch(flash_attn_stream_k_fixup_uniform, launch_params, + (float *) KQV->data, dst_tmp_meta.ptr, Q->ne[1], Q->ne[2], K->ne[2], nblocks_sk, gqa_ratio, bpt, fd0, fd1, fd2); } else if (ntiles_dst % blocks_num.x != 0) { @@ -1193,9 +1295,9 @@ void launch_fattn( const dim3 block_dim_combine(DV, 1, 1); const dim3 blocks_num_combine = {blocks_num.x, ncols1, ncols2}; - flash_attn_stream_k_fixup_general - <<>> - ((float *) KQV->data, dst_tmp_meta.ptr, + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_combine, block_dim_combine, 0, main_stream); + ggml_cuda_kernel_launch(flash_attn_stream_k_fixup_general, launch_params, + (float *) KQV->data, dst_tmp_meta.ptr, Q->ne[1], Q->ne[2], gqa_ratio, total_work, fd_k_j_z_ne12, fd_k_j_z, fd_k_j, fd_k); } @@ -1204,9 +1306,9 @@ void launch_fattn( const dim3 blocks_num_combine(Q->ne[1], Q->ne[2], Q->ne[3]); const size_t nbytes_shared_combine = parallel_blocks*sizeof(float2); - flash_attn_combine_results - <<>> - (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data, parallel_blocks); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_combine, block_dim_combine, nbytes_shared_combine, main_stream); + ggml_cuda_kernel_launch(flash_attn_combine_results, launch_params, + dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data, parallel_blocks); } CUDA_CHECK(cudaGetLastError()); } diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index b613ae6..296aa2b 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -61,14 +61,22 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2, 64, 64, 64, 64, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 128, 2, 64, 64, 64, 64, 2, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 4, 64, 128, 128, 128, 2, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 4, 32, 128, 128, 128, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 8, 64, 4, 64, 96, 64, 64, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 16, 64, 4, 32, 96, 64, 64, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 128, 2, 32, 96, 64, 64, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 64, 128, 2, 32, 96, 64, 64, 2, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 128, 2, 64, 128, 128, 128, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 256, 1, 64, 128, 128, 128, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 32, 128, 128, 128, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 32, 128, 128, 128, 2, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 8, 64, 4, 32, 256, 256, 128, 1, false); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 64, 4, 32, 256, 256, 128, 1, false); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 128, 2, 32, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 256, 1, 32, 128, 128, 128, 1, false); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 8, 128, 2, 64, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 256, 1, 64, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 256, 1, 32, 128, 128, 128, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 64, 256, 1, 32, 128, 128, 128, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 8, 64, 4, 32, 288, 256, 128, 1, false); @@ -85,6 +93,9 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 2, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 256, 1, 32, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 8, 64, 4, 32, 96, 64, 128, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 64, 4, 32, 96, 64, 128, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 128, 2, 32, 128, 128, 128, 1, false); @@ -99,6 +110,9 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co } static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_volta(const int DKQ, const int DV, const int ncols) { + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 128, 128, 64, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 256, 1, 32, 128, 128, 64, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 8, 64, 4, 32, 256, 256, 64, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 64, 4, 32, 256, 256, 64, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 128, 2, 32, 128, 128, 64, 1, false); @@ -114,58 +128,107 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co } static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_rdna(const int DKQ, const int DV, const int ncols) { - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 128, 2, 64, 128, 128, 128, 2, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 2, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 2, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 2, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 128, 2, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 128, 2, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 128, 2, 64, 32, 32, 32, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 64, 4, 32, 128, 128, 128, 1, false); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 128, 2, 32, 128, 128, 128, 1, false); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 64, 256, 1, 32, 128, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 64, 2, 32, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 64, 2, 32, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 128, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 64, 128, 2, 64, 40, 40, 40, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16, 64, 4, 32, 96, 64, 128, 1, false); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 128, 1, false); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 128, 1, false); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 8, 64, 2, 32, 48, 48, 48, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 16, 64, 2, 32, 48, 48, 48, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 32, 128, 2, 64, 48, 48, 48, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 64, 128, 2, 64, 48, 48, 48, 1, true); - // TODO tune specifically for RDNA - return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 8, 64, 2, 32, 56, 56, 56, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16, 64, 2, 32, 56, 56, 56, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 128, 2, 64, 56, 56, 56, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 64, 128, 2, 64, 56, 56, 56, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 8, 64, 2, 32, 64, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16, 64, 2, 32, 64, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2, 64, 64, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 128, 2, 64, 64, 64, 64, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 8, 64, 2, 32, 96, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 16, 64, 2, 32, 96, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 128, 2, 64, 96, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 64, 128, 2, 64, 96, 64, 64, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 2, 32, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 2, 32, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 256, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2, 64, 128, 128, 64, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 160, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2, 32, 160, 128, 128, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 8, 128, 3, 64, 96, 64, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 128, 3, 64, 96, 64, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 128, 2, 32, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 64, 128, 2, 32, 128, 128, 128, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 8, 128, 3, 64, 96, 64, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16, 128, 3, 64, 96, 64, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 128, 2, 32, 160, 128, 128, 1, true); + + return fattn_mma_config(32, 1, 0, 0, 0, 0, 0, false); } static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_cdna(const int DKQ, const int DV, const int ncols) { - // Conservative configs for CDNA (MI100+): 64KB LDS, wavefront64, nstages=1 (no cp.async). - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 2, 128, 32, 32, 32, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 128, 2, 64, 32, 32, 32, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 128, 2, 64, 32, 32, 32, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 2, 64, 32, 32, 32, 1, true); - - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 128, 2, 128, 40, 40, 40, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 128, 2, 64, 40, 40, 40, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 128, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 1, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 32, 32, 32, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 3, 64, 32, 32, 32, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40, 40, 40, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 32, 256, 2, 64, 40, 40, 40, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 64, 256, 2, 64, 40, 40, 40, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 8, 128, 2, 128, 48, 48, 48, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 16, 128, 2, 64, 48, 48, 48, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 32, 128, 2, 64, 48, 48, 48, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 8, 256, 2, 64, 48, 48, 48, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 16, 256, 2, 64, 48, 48, 48, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 32, 256, 2, 64, 48, 48, 48, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96, 96, 64, 256, 2, 64, 48, 48, 48, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 8, 128, 2, 128, 56, 56, 56, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16, 128, 2, 64, 56, 56, 56, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 128, 2, 64, 56, 56, 56, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 8, 256, 2, 64, 56, 56, 56, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16, 256, 2, 64, 56, 56, 56, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 256, 2, 64, 56, 56, 56, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 64, 256, 2, 64, 56, 56, 56, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 8, 128, 2, 128, 64, 64, 64, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16, 128, 2, 64, 64, 64, 64, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2, 64, 64, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 8, 256, 2, 64, 64, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16, 256, 2, 64, 64, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64, 64, 64, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 256, 2, 64, 64, 64, 64, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 4, 64, 128, 128, 128, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 4, 32, 128, 128, 128, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 32, 128, 128, 128, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2, 32, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 8, 256, 1, 64, 64, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 16, 256, 1, 64, 64, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 256, 1, 64, 64, 64, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 64, 512, 1, 64, 64, 64, 64, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 256, 1, 64, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 256, 1, 64, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 256, 1, 64, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 512, 1, 64, 128, 128, 64, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 256, 1, 64, 160, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 256, 1, 64, 160, 128, 128, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 8, 256, 1, 64, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 16, 256, 1, 64, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 32, 256, 1, 64, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(512, 512, 64, 256, 1, 64, 128, 128, 128, 1, true); + + GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 8, 256, 1, 64, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16, 256, 1, 64, 128, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 256, 1, 64, 160, 128, 128, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 64, 160, 128, 128, 1, true); - // Fallback for unsupported DKQ values (e.g. 576). Must return non-zero values to satisfy - // compile-time static_asserts even though the kernel guard prevents runtime execution. - // nthreads=256 gives nwarps=4 (warp_size=64) or 8 (warp_size=32), nbatch_fa=128 satisfies np*16 divisibility. - return fattn_mma_config(256, 1, 128, 4, 4, 4, 1, false); + return fattn_mma_config(32, 1, 0, 0, 0, 0, 0, false); } static __host__ fattn_mma_config ggml_cuda_fattn_mma_get_config(const int DKQ, const int DV, const int ncols, const int cc) { @@ -283,34 +346,54 @@ static __host__ int get_cols_per_warp(const int cc) { } } +static __host__ bool ggml_cuda_fattn_mma_get_swizzled(const int DKQ, const int DV, const int ncols, const int cc) { + return turing_mma_available(cc) && + ggml_cuda_fattn_mma_get_nbatch_K2(DKQ, DV, ncols, cc) % 32 == 0 && ggml_cuda_fattn_mma_get_nbatch_V2(DKQ, DV, ncols, cc) % 32 == 0; +} + +static constexpr __device__ bool ggml_cuda_fattn_mma_get_swizzled(const int DKQ, const int DV, const int ncols) { +#ifdef TURING_MMA_AVAILABLE + return ggml_cuda_fattn_mma_get_nbatch_K2(DKQ, DV, ncols) % 32 == 0 && ggml_cuda_fattn_mma_get_nbatch_V2(DKQ, DV, ncols) % 32 == 0; +#else + GGML_UNUSED_VARS(DKQ, DV, ncols); + return false; +#endif // TURING_MMA_AVAILABLE +} + // ------------------------------------------------------------------------------------------------------------------ static __host__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV, const int ncols1, const int ncols2, const int cc) { return cp_async_available(cc) && ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2, cc) : 0; } -static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV, const int ncols1, const int ncols2) { +static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages( + const int DKQ, const int DV, const int ncols1, const int ncols2, const bool use_sparse) { #ifdef CP_ASYNC_AVAILABLE - return ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2) : 0; + const int nstages_target = ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2) : 0; + // sparse gather is not implemented for multi-stage loading + return use_sparse && nstages_target > 1 ? 1 : nstages_target; #else - GGML_UNUSED_VARS(DKQ, DV, ncols1, ncols2); + GGML_UNUSED_VARS(DKQ, DV, ncols1, ncols2, use_sparse); return 0; #endif // CP_ASYNC_AVAILABLE } // ------------------------------------------------------------------------------------------------------------------ -template +template static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( - const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int D2, const int stride_KV, const int i_sup) { + const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int D2, const int stride_KV, + const int k_VKQ_0, const int i_sup, const int32_t * const __restrict__ indices) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); // K/V data is loaded with decreasing granularity for D for better memory bandwidth. - // The minimum granularity with cp.async is 16 bytes, with synchronous data loading it's 4 bytes. + // The minimum granularity is 16 bytes. + constexpr int chunk_size = 16; + constexpr int h2_per_chunk = chunk_size / sizeof(half2); + const int chunks_per_row = D2 / h2_per_chunk; if constexpr (use_cp_async) { - static_assert(!oob_check, "OOB check not compatible with cp_async"); + static_assert(warp_size == 32, "bad warp_size"); + static_assert(!oob_check || use_sparse, "OOB check not compatible with cp_async"); constexpr int preload = 64; - constexpr int h2_per_chunk = 16/sizeof(half2); - const int chunks_per_row = D2 / h2_per_chunk; const unsigned int tile_KV_32 = ggml_cuda_cvta_generic_to_shared(tile_KV); @@ -332,11 +415,20 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( break; } + int64_t i_KV; + if constexpr (use_sparse) { + // padded slots gather row 0, the -inf mask removes their contribution + const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : 0; + i_KV = index >= 0 ? index : 0; + } else { + i_KV = k_VKQ_0 + i; + } + #pragma unroll for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); - cp_async_cg_16(tile_KV_32 + i*(stride_tile*sizeof(half2)) + k*16, KV + i*stride_KV + k*h2_per_chunk); + cp_async_cg_16(tile_KV_32 + swizzle(i*stride_tile*sizeof(half2) + k*chunk_size, i), KV + i_KV*stride_KV + k*h2_per_chunk); } } }; @@ -348,11 +440,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( // 6: max 1*16= 16 bytes, 8 half ggml_cuda_unroll<6>{}(load); } else { - // TODO use ggml_cuda_memcpy_1 + const half2 zero[4] = {{0.0f, 0.0f}, {0.0f, 0.0f}, {0.0f, 0.0f}, {0.0f, 0.0f}}; auto load = [&] __device__ (const int n) { - const int stride_k = warp_size >> n; - const int k0_start = stride_k == warp_size ? 0 : D2 - D2 % (2*stride_k); - const int k0_stop = D2 - D2 % (1*stride_k); + const int stride_k = 32 >> n; + const int k0_start = stride_k == 32 ? 0 : chunks_per_row - chunks_per_row % (2*stride_k); + const int k0_stop = chunks_per_row - chunks_per_row % (1*stride_k); const int stride_i = warp_size / stride_k; if (k0_start == k0_stop) { @@ -371,26 +463,37 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile( for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) { const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k); - tile_KV[i*stride_tile + k] = !oob_check || i < i_sup ? KV[i*stride_KV + k] : make_half2(0.0f, 0.0f); + const half2 * src; + if constexpr (use_sparse) { + const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : -1; + src = index >= 0 ? KV + int64_t(index)*stride_KV + k*h2_per_chunk : zero; + } else { + src = !oob_check || i < i_sup ? KV + int64_t(k_VKQ_0 + i)*stride_KV + k*h2_per_chunk : zero; + } + ggml_cuda_memcpy_1<16>(swizzle(tile_KV, i*stride_tile + k*h2_per_chunk, i), src); } } }; - // 1: max 32* 4=128 bytes, 64 half - // 2: max 16* 4= 64 bytes, 32 half - // 3: max 8* 4= 32 bytes, 16 half - // 4: max 4* 4= 16 bytes, 8 half - ggml_cuda_unroll<4>{}(load); + // 1: max 32*16=512 bytes, 256 half + // 2: max 16*16=256 bytes, 128 half + // 3: max 8*16=128 bytes, 64 half + // 4: max 4*16= 64 bytes, 32 half + // 5: max 2*16= 32 bytes, 16 half + // 6: max 1*16= 16 bytes, 8 half + ggml_cuda_unroll<6>{}(load); } } -template +template static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( const half * const __restrict__ mask_h, half * const __restrict__ tile_mask, - const int stride_mask, const int i_sup, const int j0, const uint3 ne01) { + const int stride_mask, const int k_VKQ_0, const int i_sup, const int j0, const uint3 ne01, + const int32_t * const __restrict__ indices) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); if constexpr (use_cp_async) { static_assert(nbatch_fa <= 8*warp_size && nbatch_fa % 8 == 0, "bad nbatch_fa"); static_assert(!oob_check, "OOB check incompatible with cp_async"); + static_assert(!use_sparse, "sparse gather incompatible with cp_async"); constexpr int preload = nbatch_fa >= 32 ? nbatch_fa * sizeof(half) : 64; constexpr int cols_per_warp = 8*warp_size/nbatch_fa; constexpr int stride_j = nwarps * cols_per_warp; @@ -408,9 +511,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( const int i = 8 * (threadIdx.x % (nbatch_fa/8)); - cp_async_cg_16(tile_mask_32 + j_sram*(nbatch_fa*sizeof(half) + 16) + i*sizeof(half), mask_h + j_vram*stride_mask + i); + cp_async_cg_16(tile_mask_32 + j_sram*(nbatch_fa*sizeof(half) + 16) + i*sizeof(half), mask_h + int64_t(j_vram)*stride_mask + k_VKQ_0 + i); } - } else if constexpr (oob_check) { + } else if constexpr (oob_check || use_sparse) { #pragma unroll for (int j1 = 0; j1 < ncols1; j1 += nwarps) { const int j_sram = j1 + threadIdx.y; @@ -424,7 +527,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( for (int i0 = 0; i0 < nbatch_fa; i0 += warp_size) { const int i = i0 + threadIdx.x; - tile_mask[j_sram*(nbatch_fa + 8) + i] = i < i_sup ? mask_h[j_vram*stride_mask + i] : half(0.0f); + if constexpr (use_sparse) { + const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : -1; + tile_mask[j_sram*(nbatch_fa + 8) + i] = index >= 0 ? mask_h[int64_t(j_vram)*stride_mask + index] : half(-INFINITY); + } else { + tile_mask[j_sram*(nbatch_fa + 8) + i] = i < i_sup ? mask_h[int64_t(j_vram)*stride_mask + k_VKQ_0 + i] : half(0.0f); + } } } } else if constexpr (nbatch_fa < 2*warp_size) { @@ -441,7 +549,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( const int i = threadIdx.x % (warp_size/cols_per_warp); - ggml_cuda_memcpy_1(tile_mask + j_sram*(nbatch_fa + 8) + 2*i, mask_h + j_vram*stride_mask + 2*i); + ggml_cuda_memcpy_1(tile_mask + j_sram*(nbatch_fa + 8) + 2*i, mask_h + int64_t(j_vram)*stride_mask + k_VKQ_0 + 2*i); } } else { #pragma unroll @@ -457,20 +565,21 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask( for (int i0 = 0; i0 < nbatch_fa; i0 += 2*warp_size) { const int i = i0 + 2*threadIdx.x; - ggml_cuda_memcpy_1(tile_mask + j_sram*(nbatch_fa + 8) + i, mask_h + j_vram*stride_mask + i); + ggml_cuda_memcpy_1(tile_mask + j_sram*(nbatch_fa + 8) + i, mask_h + int64_t(j_vram)*stride_mask + k_VKQ_0 + i); } } } } template static __device__ __forceinline__ void flash_attn_ext_f16_iter( const float2 * const __restrict__ Q_f2, const half2 * const __restrict__ K_h2, const half2 * const __restrict__ V_h2, const half * const __restrict__ mask_h, + const int32_t * const __restrict__ indices, float2 * const __restrict__ dstk, float2 * const __restrict__ dstk_fixup, const float scale, @@ -492,7 +601,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const int jt, const int kb0, const int k_VKQ_sup) { -#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE) +#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int ncols = ncols1 * ncols2; constexpr int cols_per_warp = T_B_KQ::I; @@ -502,12 +611,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( constexpr int nbatch_K2 = ggml_cuda_fattn_mma_get_nbatch_K2(DKQ, DV, ncols); constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2(DKQ, DV, ncols); constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); - constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2); + constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse); - constexpr int stride_tile_Q = DKQ/2 + 4; - constexpr int stride_tile_K = nbatch_K2 + 4; - - constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4; + constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols); + constexpr int stride_tile_K = swz ? nbatch_K2 : nbatch_K2 + 4; + constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : (swz ? nbatch_V2 : nbatch_V2 + 4); const int k_VKQ_0 = kb0 * nbatch_fa; #if defined(TURING_MMA_AVAILABLE) @@ -525,13 +633,14 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( constexpr bool use_cp_async = true; cp_async_wait_all(); __syncthreads(); - flash_attn_ext_f16_load_tile - (V_h2 + int64_t(k_VKQ_0)*stride_V, tile_V, nbatch_V2, stride_V, k_VKQ_sup); + flash_attn_ext_f16_load_tile + (V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, nullptr); } else { - constexpr bool use_cp_async = nstages == 1; + // the sparse mask values are gathered per element, always load them synchronously + constexpr bool use_cp_async = nstages == 1 && !use_sparse; if (ncols2 > 1 || mask_h) { - flash_attn_ext_f16_load_mask - (mask_h + k_VKQ_0, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01); + flash_attn_ext_f16_load_mask + (mask_h, tile_mask, stride_mask, k_VKQ_0, k_VKQ_sup, jt*ncols1, ne01, indices); } } @@ -540,12 +649,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( #pragma unroll for (int k0_start = (DKQ/2-1) - (DKQ/2-1) % nbatch_K2; k0_start >= 0; k0_start -= nbatch_K2) { const int k0_stop = k0_start + nbatch_K2 < DKQ/2 ? k0_start + nbatch_K2 : DKQ/2; - const int k0_diff = k0_stop - k0_start; if constexpr (nstages <= 1) { + const int k0_diff = k0_stop - k0_start; constexpr bool use_cp_async = nstages == 1; - flash_attn_ext_f16_load_tile - (K_h2 + int64_t(k_VKQ_0)*stride_K + k0_start, tile_K, k0_diff, stride_K, k_VKQ_sup); + flash_attn_ext_f16_load_tile + (K_h2 + k0_start, tile_K, k0_diff, stride_K, k_VKQ_0, k_VKQ_sup, indices); if (use_cp_async) { cp_async_wait_all(); } @@ -560,7 +669,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( #pragma unroll for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) { T_A_KQ K_A; - load_ldmatrix(K_A, tile_K + i_KQ_0*stride_tile_K + (k_KQ_0 - k0_start), stride_tile_K); + load_ldmatrix_swizzled(K_A, tile_K, i_KQ_0*stride_tile_K + k_KQ_0-k0_start); if constexpr (cols_per_warp == 8) { mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[k_KQ_0/T_A_KQ::J]); } else { @@ -576,6 +685,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } } } else { + constexpr int stride_tile_Q = DKQ/2 + 4; #pragma unroll for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) { load_ldmatrix(Q_B[0], tile_Q + (threadIdx.y / np)*(T_B_KQ::I*stride_tile_Q) + k_KQ_0, stride_tile_Q); @@ -585,7 +695,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const int i_KQ_0 = i_KQ_00 + (threadIdx.y % np)*T_A_KQ::I; T_A_KQ K_A; - load_ldmatrix(K_A, tile_K + i_KQ_0*stride_tile_K + (k_KQ_0 - k0_start), stride_tile_K); + load_ldmatrix_swizzled(K_A, tile_K, i_KQ_0*stride_tile_K + k_KQ_0-k0_start); if constexpr (cols_per_warp == 8) { mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[0]); @@ -694,6 +804,18 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( #pragma unroll for (int i00 = 0; i00 < nbatch_fa; i00 += np*T_C_KQ::J) { const int i0 = i00 + (threadIdx.y % np)*T_C_KQ::J; + + // The mask is stored as 16 bit half values, loading them as 32 bit half2 values is preferred in terms of speed. + // However, this is not possible for RDNA3 where 2 consecutive l indices are not consecutive in the mask memory layout. +#ifdef RDNA3 +#pragma unroll + for (int l = 0; l < T_C_KQ::ne; ++l) { + const int i = i0 + T_C_KQ::get_j(l); + const int j = ((threadIdx.y / np)*cols_per_warp + T_C_KQ::get_i(l)) / ncols2; + + KQ_C[i00/(np*T_C_KQ::J)].x[l] += __half2float(tile_mask[j*(nbatch_fa + 8) + i]); + } +#else #pragma unroll for (int l0 = 0; l0 < T_C_KQ::ne; l0 += 2) { const int i = (i0 + T_C_KQ::get_j(l0)) / 2; @@ -703,6 +825,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( KQ_C[i00/(np*T_C_KQ::J)].x[l0 + 0] += slope*tmp.x; KQ_C[i00/(np*T_C_KQ::J)].x[l0 + 1] += slope*tmp.y; } +#endif // RDNA3 } } @@ -809,13 +932,23 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } } #elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) - const half2 KQ_max_scale_h2 = make_half2( - KQ_max_scale[0], KQ_max_scale[0]); + if constexpr (std::is_same_v) { + const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { #pragma unroll - for (int l = 0; l < T_C_VKQ::ne; ++l) { - VKQ_C[i].x[l] *= KQ_max_scale_h2; + for (int l = 0; l < T_C_VKQ::ne; ++l) { + VKQ_C[i].x[l] *= KQ_max_scale_h2; + } + } + } else { + static_assert(std::is_same_v, "bad VKQ type"); +#pragma unroll + for (int i = 0; i < DV/T_C_VKQ::J; ++i) { +#pragma unroll + for (int l = 0; l < T_C_VKQ::ne; ++l) { + VKQ_C[i].x[l] *= KQ_max_scale[0]; + } } } #else // Volta @@ -846,6 +979,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } if constexpr (nstages > 1) { + static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading"); static_assert(!V_is_K_view, "K data reuse not implemented multi-stage loading"); // Preload K tile for next iteration: constexpr bool use_cp_async = true; @@ -853,83 +987,55 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( __syncthreads(); if (!last_iter) { if (ncols2 > 1 || mask_h) { - flash_attn_ext_f16_load_mask - (mask_h + k_VKQ_0 + nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01); + flash_attn_ext_f16_load_mask + (mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr); } - flash_attn_ext_f16_load_tile - (K_h2 + int64_t(k_VKQ_0 + nbatch_fa)*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup); + flash_attn_ext_f16_load_tile + (K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, nullptr); } } -#if defined(AMD_WMMA_AVAILABLE) && !defined(LDMATRIX_TRANS_AVAILABLE) - T_A_VKQ A_identity; - make_identity_mat(A_identity); -#endif // defined(AMD_WMMA_AVAILABLE) && !defined(LDMATRIX_TRANS_AVAILABLE) - // Calculate VKQ tile, need to use logical rather than physical elements for i0 due to transposition of V: #pragma unroll for (int i0_start = 0; i0_start < DV; i0_start += 2*nbatch_V2) { static_assert(DV % (2*nbatch_V2) == 0, "bad loop size"); const int i0_stop = i0_start + 2*nbatch_V2; - const int i0_diff = i0_stop - i0_start; if constexpr (nstages <= 1) { + const int i0_diff = i0_stop - i0_start; if (!V_is_K_view || i0_stop > 2*nbatch_K2) { constexpr bool use_cp_async = nstages == 1; - flash_attn_ext_f16_load_tile - (V_h2 + int64_t(k_VKQ_0)*stride_V + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_sup); + flash_attn_ext_f16_load_tile + (V_h2 + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_0, k_VKQ_sup, indices); if (use_cp_async) { cp_async_wait_all(); } __syncthreads(); } } - const half2 * tile_V_i = !V_is_K_view || i0_stop > 2*nbatch_K2 ? tile_V : tile_V + i0_start/2; + const int tile_V_offset_i = !V_is_K_view || i0_stop > 2*nbatch_K2 ? 0 : i0_start/2; #if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) - constexpr int i0_stride = cols_per_warp == 8 ? T_C_VKQ::I : 2*T_C_VKQ::J; #pragma unroll - for (int i_VKQ_0 = i0_start; i_VKQ_0 < i0_stop; i_VKQ_0 += i0_stride) { + for (int i_VKQ_0 = i0_start; i_VKQ_0 < i0_stop; i_VKQ_0 += T_A_VKQ::I) { static_assert((nbatch_fa/2) % (np*T_A_VKQ::J) == 0, "bad loop size"); #pragma unroll for (int k00 = 0; k00 < nbatch_fa/2; k00 += np*T_A_VKQ::J) { const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::J; T_A_VKQ A; // Transposed in SRAM but not in registers, gets transposed on load. -#if defined(LDMATRIX_TRANS_AVAILABLE) - load_ldmatrix_trans(A, tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V); -#elif defined(AMD_MFMA_AVAILABLE) - // MFMA A register layout: A_mat[i=lane%16][k=4*(lane/16)+reg]. - // Normal load gives A_mat[seq][dv] but we need A_mat[dv][seq] = V^T. - // Load with transposed addressing: 4 strided half loads. - { - const half2 * xs0 = tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2; - const half * xs0_h = (const half *) xs0; - const int stride_h = stride_tile_V * 2; // stride in half units - half * A_h = (half *) A.x; -#pragma unroll - for (int l = 0; l < 4; ++l) { - A_h[l] = xs0_h[(4*(threadIdx.x / 16) + l) * stride_h + threadIdx.x % 16]; - } - } -#else - // TODO: Try to transpose tile_V when loading gmem to smem. - // Use mma to transpose T_A_VKQ for RDNA. - T_A_VKQ A_trans; - load_ldmatrix(A_trans, tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V); - mma(A, A_trans, A_identity); -#endif // defined(LDMATRIX_TRANS_AVAILABLE) + load_ldmatrix_trans_swizzled(A, tile_V, tile_V_offset_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2); if constexpr (T_B_KQ::I == 8) { - mma(VKQ_C[i_VKQ_0/i0_stride], A, B[k00/(np*T_A_VKQ::J)]); + mma(VKQ_C[i_VKQ_0/T_A_VKQ::I], A, B[k00/(np*T_A_VKQ::J)]); } else { // Wide version of VKQ_C is column-major. #if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) // AMD matrix C is column-major. - mma(VKQ_C[i_VKQ_0/i0_stride], A, B[k00/(np*T_A_VKQ::J)]); + mma(VKQ_C[i_VKQ_0/T_A_VKQ::I], A, B[k00/(np*T_A_VKQ::J)]); #else // swap A and B for CUDA. - mma(VKQ_C[i_VKQ_0/i0_stride], B[k00/(np*T_A_VKQ::J)], A); + mma(VKQ_C[i_VKQ_0/T_A_VKQ::I], B[k00/(np*T_A_VKQ::J)], A); #endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) } } @@ -945,7 +1051,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::I; T_A_VKQ A; // Transposed in both SRAM and registers, load normally. - load_ldmatrix(A, tile_V_i + k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V); + load_ldmatrix_swizzled(A, tile_V, tile_V_offset_i + k0*stride_tile_V + (i_VKQ_0 - i0_start)/2); mma(VKQ_C[i_VKQ_0/i0_stride], B[k00/(np*T_A_VKQ::I)], A); } } @@ -956,17 +1062,17 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter( } } #else - GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, + GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap, ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0); NO_DEVICE_CODE; -#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE) +#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) } #if defined(TURING_MMA_AVAILABLE) -template struct mma_tile_sizes { +template struct mma_tile_sizes { using T_A_KQ = tile<16, 8, half2>; // row-major using T_B_KQ = tile<16, 8, half2>; // column-major using T_C_KQ = tile<16, 16, float>; // column-major @@ -974,7 +1080,8 @@ template struct mma_tile_sizes { using T_B_VKQ = tile<16, 8, half2>; // column-major using T_C_VKQ = tile<16, 8, half2>; // column-major }; -template<> struct mma_tile_sizes<8> { +// If there are only 8 columns, use thinner B tiles to avoid wasting compute: +template struct mma_tile_sizes { using T_A_KQ = tile<16, 8, half2>; // row-major using T_B_KQ = tile< 8, 8, half2>; // column-major using T_C_KQ = tile<16, 8, float>; // row-major @@ -982,8 +1089,67 @@ template<> struct mma_tile_sizes<8> { using T_B_VKQ = tile< 8, 8, half2>; // column-major using T_C_VKQ = tile<16, 4, half2>; // row-major }; -#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) -template struct mma_tile_sizes { +// For very large head sizes, use thinner B tiles to reduce register pressure: +template<> struct mma_tile_sizes<256, 16> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile< 8, 8, half2>; // column-major + using T_C_KQ = tile<16, 8, float>; // row-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile< 8, 8, half2>; // column-major + using T_C_VKQ = tile<16, 4, half2>; // row-major +}; +template<> struct mma_tile_sizes<512, 16> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile< 8, 8, half2>; // column-major + using T_C_KQ = tile<16, 8, float>; // row-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile< 8, 8, half2>; // column-major + using T_C_VKQ = tile<16, 4, half2>; // row-major +}; +template<> struct mma_tile_sizes<512, 32> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile< 8, 8, half2>; // column-major + using T_C_KQ = tile<16, 8, float>; // row-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile< 8, 8, half2>; // column-major + using T_C_VKQ = tile<16, 4, half2>; // row-major +}; +#elif defined(AMD_WMMA_AVAILABLE) +#ifdef RDNA3 +template struct mma_tile_sizes { + using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major + using T_A_VKQ = tile<32, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_VKQ = tile<16, 16, half2, DATA_LAYOUT_I_MAJOR>; // column-major +}; +template struct mma_tile_sizes<80, ncols> { + using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major + using T_A_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major +}; +template struct mma_tile_sizes<112, ncols> { + using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major + using T_A_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major + using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // column-major + using T_C_VKQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major +}; +#else +template struct mma_tile_sizes { + using T_A_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR>; // row-major + using T_B_KQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR>; // column-major + using T_C_KQ = tile<16, 16, float, DATA_LAYOUT_I_MAJOR>; // column-major + using T_A_VKQ = tile<32, 8, half2, DATA_LAYOUT_I_MAJOR>; // row-major + using T_B_VKQ = tile<16, 8, half2, DATA_LAYOUT_I_MAJOR>; // column-major + using T_C_VKQ = tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED>; // column-major +}; +template struct mma_tile_sizes<80, ncols> { using T_A_KQ = tile<16, 8, half2>; // row-major using T_B_KQ = tile<16, 8, half2>; // column-major using T_C_KQ = tile<16, 16, float>; // column-major @@ -991,8 +1157,26 @@ template struct mma_tile_sizes { using T_B_VKQ = tile<16, 8, half2>; // column-major using T_C_VKQ = tile<16, 8, half2>; // column-major }; +template struct mma_tile_sizes<112, ncols> { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile<16, 8, half2>; // column-major + using T_C_KQ = tile<16, 16, float>; // column-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile<16, 8, half2>; // column-major + using T_C_VKQ = tile<16, 8, half2>; // column-major +}; +#endif // RDNA3 +#elif defined(AMD_MFMA_AVAILABLE) +template struct mma_tile_sizes { + using T_A_KQ = tile<16, 8, half2>; // row-major + using T_B_KQ = tile<16, 8, half2>; // column-major + using T_C_KQ = tile<16, 16, float>; // column-major + using T_A_VKQ = tile<16, 8, half2>; // row-major + using T_B_VKQ = tile<16, 8, half2>; // column-major + using T_C_VKQ = tile<16, 16, float>; // column-major +}; #else // Volta -template struct mma_tile_sizes { +template struct mma_tile_sizes { using T_A_KQ = tile< 8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major using T_B_KQ = tile<32, 4, half2, DATA_LAYOUT_I_MAJOR>; // column-major using T_C_KQ = tile<32, 8, float, DATA_LAYOUT_I_MAJOR>; // column-major @@ -1002,12 +1186,13 @@ template struct mma_tile_sizes { }; #endif // defined(TURING_MMA_AVAILABLE) -template +template static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const float2 * const __restrict__ Q_f2, const half2 * const __restrict__ K_h2, const half2 * const __restrict__ V_h2, const half * const __restrict__ mask_h, + const int32_t * const __restrict__ indices, const float * const __restrict__ sinks_f, float2 * const __restrict__ dstk, float2 * const __restrict__ dstk_fixup, @@ -1027,17 +1212,17 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const int zt_gqa, const int kb0_start, const int kb0_stop) { -#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE) +#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) //In this kernel Q, K, V are matrices while i, j, k are matrix indices. constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int ncols = ncols1 * ncols2; - using T_A_KQ = typename mma_tile_sizes::T_A_KQ; - using T_B_KQ = typename mma_tile_sizes::T_B_KQ; - using T_C_KQ = typename mma_tile_sizes::T_C_KQ; - using T_A_VKQ = typename mma_tile_sizes::T_A_VKQ; - using T_B_VKQ = typename mma_tile_sizes::T_B_VKQ; - using T_C_VKQ = typename mma_tile_sizes::T_C_VKQ; + using T_A_KQ = typename mma_tile_sizes::T_A_KQ; + using T_B_KQ = typename mma_tile_sizes::T_B_KQ; + using T_C_KQ = typename mma_tile_sizes::T_C_KQ; + using T_A_VKQ = typename mma_tile_sizes::T_A_VKQ; + using T_B_VKQ = typename mma_tile_sizes::T_B_VKQ; + using T_C_VKQ = typename mma_tile_sizes::T_C_VKQ; constexpr int cols_per_warp = T_B_KQ::I; constexpr int cols_per_thread = get_cols_per_thread(); @@ -1047,7 +1232,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2 (DKQ, DV, ncols); constexpr int nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols); constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols); - constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2); + constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse); if (cols_per_warp > ncols) { NO_DEVICE_CODE; @@ -1056,10 +1241,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( static_assert(nwarps * (cols_per_warp/ncols2) % ncols1 == 0, "bad nwarps"); - constexpr int stride_tile_Q = DKQ/2 + 4; - constexpr int stride_tile_K = nbatch_K2 + 4; - - constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4; + constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols); + constexpr int stride_tile_Q = DKQ/2 + 4; + constexpr int stride_tile_K = swz ? nbatch_K2 : nbatch_K2 + 4; + constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : (swz ? nbatch_V2 : nbatch_V2 + 4); constexpr int stride_tile_KV_max = stride_tile_K > stride_tile_V ? stride_tile_K : stride_tile_V; extern __shared__ half2 tile_Q[]; @@ -1070,7 +1255,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( T_B_KQ Q_B[(Q_in_reg ? DKQ/(2*T_B_KQ::J) : 1)]; #if defined(TURING_MMA_AVAILABLE) T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)]; -#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) +#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) + T_C_VKQ VKQ_C[DV % 32 != 0 ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)]; +#elif defined(AMD_MFMA_AVAILABLE) + T_C_VKQ VKQ_C[ DV/T_C_VKQ::J]; +#elif defined(AMD_WMMA_AVAILABLE) T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; #else // Volta T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)]; @@ -1144,37 +1333,38 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( // Preload mask and K data for first iteration when using cp_async with multiple stages: if constexpr (nstages > 1) { + static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading"); static_assert(nbatch_K2 == DKQ/2, "batching not implemented for multi-stage pipeline"); constexpr bool use_cp_async = true; constexpr bool oob_check = false; constexpr int k_VKQ_sup = nbatch_fa; if (ncols2 > 1 || mask_h) { - flash_attn_ext_f16_load_mask - (mask_h + kb0*nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01); + flash_attn_ext_f16_load_mask + (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr); } - flash_attn_ext_f16_load_tile - (K_h2 + int64_t(kb0)*nbatch_fa*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup); + flash_attn_ext_f16_load_tile + (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr); } // kb0_start is always < kb0_stop so the last iter can be executed unconditionally. - if constexpr (ncols2 == 1) { + if constexpr (ncols2 == 1 || use_sparse) { constexpr bool oob_check = true; for (; kb0 < kb0_stop-1; ++kb0) { constexpr bool last_iter = false; constexpr int k_VKQ_sup = nbatch_fa; flash_attn_ext_f16_iter - - (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap, ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup); } constexpr bool last_iter = true; const int k_VKQ_sup = ne11 - kb0*nbatch_fa; flash_attn_ext_f16_iter - - (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap, ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup); } else { @@ -1183,18 +1373,18 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( constexpr bool last_iter = false; constexpr int k_VKQ_sup = nbatch_fa; flash_attn_ext_f16_iter - - (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap, ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup); } constexpr bool last_iter = true; constexpr int k_VKQ_sup = nbatch_fa; flash_attn_ext_f16_iter - - (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap, + (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap, ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C, KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup); } @@ -1240,7 +1430,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( float KQ_max_scale[cols_per_thread]; #pragma unroll for (int col = 0; col < cols_per_thread; ++col) { - const int jc = cols_per_warp == 8 ? T_C_KQ::get_j(col) : T_C_KQ::get_i(2*col); + const int jc = (threadIdx.y/np)*cols_per_warp + (cols_per_warp == 8 ? T_C_KQ::get_j(col) : T_C_KQ::get_i(2*col)); const float sink = sinks_f[jc % ncols2]; const float KQ_max_new = fmaxf(KQ_max[col], sink); @@ -1278,12 +1468,23 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } } #elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) - const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]); + if constexpr (std::is_same_v) { + const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]); #pragma unroll - for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { + for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) { #pragma unroll - for (int l = 0; l < T_C_VKQ::ne; ++l) { - VKQ_C[i].x[l] *= KQ_max_scale_h2; + for (int l = 0; l < T_C_VKQ::ne; ++l) { + VKQ_C[i].x[l] *= KQ_max_scale_h2; + } + } + } else { + static_assert(std::is_same_v, "bad VKQ type"); +#pragma unroll + for (int i = 0; i < DV/T_C_VKQ::J; ++i) { +#pragma unroll + for (int l = 0; l < T_C_VKQ::ne; ++l) { + VKQ_C[i].x[l] *= KQ_max_scale[0]; + } } } #else // Volta @@ -1311,6 +1512,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const int jc_cwm = threadIdx.y*(2*T_C_VKQ::J) + 2*T_C_VKQ::get_j(-1) + jc_cwmo; // jc combine write meta const float2 KQ_cmr = make_float2(KQ_max[jc_cwmo], KQ_rowsum[jc_cwmo]); // KQ combine max rowsum + if constexpr (swz) { + __syncthreads(); + } + if (((!needs_fixup && !is_fixup) || np > 1) && threadIdx.x < 2*T_C_VKQ::J) { // Use the 16 bytes of padding in each row to store the meta data: KQ max, KQ rowsum, KQ max scale. ((float2 *) tile_Q)[jc_cwm*(tile_stride/2) + nbatch_combine/2] = KQ_cmr; @@ -1347,6 +1552,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( const bool thread_should_write = T_C_KQ::J == 8 || T_C_KQ::get_j(threadIdx.x & 2) < 8; #endif // defined(TURING_MMA_AVAILABLE) + if constexpr (swz) { + __syncthreads(); + } + if (((!needs_fixup && !is_fixup) || np > 1) && thread_should_write) { ((float2 *) tile_Q)[jc_cwm*(tile_stride/2) + nbatch_combine/2] = KQ_cmr; } @@ -1366,82 +1575,83 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } } - if (np > 1 && threadIdx.y % np == 0) { - // Combine the meta data for parallel warps via shared memory. - // Warps with threadIdx.y % np != 0 must NOT return early. - // All threads must return simultaneously to avoid race conditions with work on the next tile. - + if (np > 1) { constexpr int nmeta = np*cols_per_warp >= warp_size ? np*cols_per_warp/warp_size : 1; + float KQ_cmn; + float KQ_cms[nmeta]; + float KQ_crs; + const int jc_meta = threadIdx.y*cols_per_warp + (np*cols_per_warp < warp_size ? threadIdx.x % (np*cols_per_warp) : threadIdx.x); float2 * const meta_ptr = ((float2 *) tile_Q) + jc_meta*(tile_stride/2) + nbatch_combine/2; - float2 meta[nmeta]; + + if (threadIdx.y % np == 0) { + // Combine the meta data for parallel warps via shared memory. + float2 meta[nmeta]; #pragma unroll - for (int imeta = 0; imeta < nmeta; ++imeta) { - meta[imeta] = meta_ptr[imeta * warp_size * tile_stride/2]; - } + for (int imeta = 0; imeta < nmeta; ++imeta) { + meta[imeta] = meta_ptr[imeta * warp_size * tile_stride/2]; + } - float KQ_cmn = meta[0].x; // KQ combine max new, max between all parallel warps. + KQ_cmn = meta[0].x; // KQ combine max new, max between all parallel warps. #pragma unroll - for (int imeta = 1; imeta < nmeta; ++imeta) { - KQ_cmn = fmaxf(KQ_cmn, meta[imeta].x); - } + for (int imeta = 1; imeta < nmeta; ++imeta) { + KQ_cmn = fmaxf(KQ_cmn, meta[imeta].x); + } #pragma unroll - for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { - if (offset < warp_size) { - KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, warp_size)); + for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { + if (offset < warp_size) { + KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, warp_size)); + } } - } - float KQ_cms[nmeta]; // KQ combine max scale per warp. #pragma unroll - for (int imeta = 0; imeta < nmeta; ++imeta) { - KQ_cms[imeta] = expf(meta[imeta].x - KQ_cmn); - } + for (int imeta = 0; imeta < nmeta; ++imeta) { + KQ_cms[imeta] = expf(meta[imeta].x - KQ_cmn); + } - float KQ_crs = KQ_cms[0]*meta[0].y; // KQ combine rowsum, scaled sum of all parallel warps. + KQ_crs = KQ_cms[0]*meta[0].y; // KQ combine rowsum, scaled sum of all parallel warps. #pragma unroll - for (int imeta = 1; imeta < nmeta; ++imeta) { - KQ_crs += KQ_cms[imeta]*meta[imeta].y; - } + for (int imeta = 1; imeta < nmeta; ++imeta) { + KQ_crs += KQ_cms[imeta]*meta[imeta].y; + } #pragma unroll - for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { - if (offset < warp_size) { - KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, warp_size); + for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) { + if (offset < warp_size) { + KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, warp_size); + } } } __syncthreads(); - // Write back combined meta data: + if (threadIdx.y % np == 0) { + // Write back combined meta data: #pragma unroll - for (int imeta = 0; imeta < nmeta; ++imeta) { - if (np*cols_per_warp >= warp_size || threadIdx.x < np*cols_per_warp) { - // Combined KQ max scale + rowsum. - meta_ptr[imeta * warp_size * tile_stride/2] = make_float2(KQ_cms[imeta], KQ_crs); + for (int imeta = 0; imeta < nmeta; ++imeta) { + if (np*cols_per_warp >= warp_size || threadIdx.x < np*cols_per_warp) { + // Combined KQ max scale + rowsum. + meta_ptr[imeta * warp_size * tile_stride/2] = make_float2(KQ_cms[imeta], KQ_crs); + } } - } - // Combined KQ max + rowsum. - static_assert(cols_per_warp <= warp_size); - if (needs_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) { - float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols; - dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); - } - if (is_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) { - float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols; - dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); + // Combined KQ max + rowsum. + static_assert(cols_per_warp <= warp_size); + if (needs_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) { + float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols; + dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); + } + if (is_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) { + float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols; + dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs); + } } - } else if (np > 1) { - // Warps with threadIdx.y % np == 0 execute a __syncthreads() in the if branch. - // Therefore, all other warps also need to execute a __syncthreads(). - // Otherwise the points at which warps synchronize with each other would become misaligned. - __syncthreads(); } #pragma unroll for (int k00 = 0; k00 < DV/2; k00 += nbatch_combine) { if constexpr (cols_per_warp == 8) { + static_assert(std::is_same_v, "bad VKQ type"); const int jc_cwd = threadIdx.y*T_B_KQ::I + T_B_KQ::get_i(-1); // jc combine write data #pragma unroll for (int k1 = 0; k1 < nbatch_combine; k1 += T_B_KQ::J) { @@ -1456,14 +1666,45 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } } else { const int j0 = threadIdx.y*cols_per_warp; + if constexpr (std::is_same_v) { + if constexpr (T_C_VKQ::dl == DATA_LAYOUT_I_MAJOR) { #pragma unroll - for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J) { + for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J) { #pragma unroll - for (int l = 0; l < T_C_VKQ::ne; ++l) { - const int j = j0 + T_C_VKQ::get_i(l); - const int k = k1 + T_C_VKQ::get_j(l); + for (int l = 0; l < T_C_VKQ::ne; ++l) { + const int j = j0 + T_C_VKQ::get_i(l); + const int k = k1 + T_C_VKQ::get_j(l); + + tile_Q[j*tile_stride + k] = VKQ_C[(k00 + k1)/T_C_VKQ::J].x[l]; + } + } + } else { + static_assert(T_C_VKQ::dl == DATA_LAYOUT_I_MAJOR_SCRAMBLED, "bad T_C_VKQ data layout"); + using T_C_VKQ_us = tile; // us == unscrambled +#pragma unroll + for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J) { + const T_C_VKQ_us VKQ_C_us = unscramble(VKQ_C[(k00 + k1)/T_C_VKQ::J]); +#pragma unroll + for (int l = 0; l < T_C_VKQ_us::ne; ++l) { + const int j = j0 + T_C_VKQ_us::get_i(l); + const int k = k1 + T_C_VKQ_us::get_j(l); + + tile_Q[j*tile_stride + k] = VKQ_C_us.x[l]; + } + } + } + } else { + static_assert(std::is_same_v, "bad VKQ type"); + half * tile_Q_h = (half *) tile_Q; +#pragma unroll + for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J/2) { +#pragma unroll + for (int l = 0; l < T_C_VKQ::ne; ++l) { + const int j = j0 + T_C_VKQ::get_i(l); + const int k = 2*k1 + T_C_VKQ::get_j(l); - tile_Q[j*tile_stride + k] = VKQ_C[(k00 + k1)/T_C_VKQ::J].x[l]; + tile_Q_h[j*(2*tile_stride) + k] = VKQ_C[(k00 + k1)/(T_C_VKQ::J/2)].x[l]; + } } } } @@ -1531,30 +1772,38 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } } } - if (np > 1) { + if (np > 1 || nbatch_combine != DKQ/2) { __syncthreads(); } } #else - GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dstk_fixup, + GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dstk_fixup, scale, slope, logit_softcap, ne01, ne02, gqa_ratio, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, kb0_start, kb0_stop); NO_DEVICE_CODE; -#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE) +#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) } -template +static constexpr __host__ __device__ bool ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse( + const int DKQ, const int DV, const int ncols1, const int ncols2) { + return (DKQ == 512 && DV == 512 && ncols1 == 1 && ncols2 == 8) || + (DKQ == 576 && DV == 512 && ncols1 == 1 && ncols2 == 16) || + (DKQ == 256 && DV == 256 && ncols1 == 1 && ncols2 == 8) || + (DKQ == 256 && DV == 256 && ncols1 == 8 && ncols2 == 8); +} + +template __launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2)) static __global__ void flash_attn_ext_f16( - const char * __restrict__ Q, - const char * __restrict__ K, - const char * __restrict__ V, - const char * __restrict__ mask, - const char * __restrict__ sinks, - const int * __restrict__ KV_max, - float * __restrict__ dst, - float2 * __restrict__ dst_meta, + const char * Q_ptr, + const char * K_ptr, + const char * V_ptr, + const char * mask_ptr, + const char * sinks_ptr, + const int * KV_max_ptr, + float * dst_ptr, + float2 * dst_meta_ptr, const float scale, const float max_bias, const float m0, @@ -1568,13 +1817,33 @@ static __global__ void flash_attn_ext_f16( const int32_t nb21, const int32_t nb22, const int64_t nb23, const int32_t ne31, const int32_t ne32, const int32_t ne33, const int32_t nb31, const int32_t nb32, const int64_t nb33) { -#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)) + ggml_cuda_pdl_sync(); // TODO optimize placement +#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)) + const char * GGML_CUDA_RESTRICT Q = Q_ptr; + const char * GGML_CUDA_RESTRICT K = K_ptr; + const char * GGML_CUDA_RESTRICT V = V_ptr; + const char * GGML_CUDA_RESTRICT mask = mask_ptr; + const char * GGML_CUDA_RESTRICT sinks = sinks_ptr; + // sparse: one index list per (sequence, query tile), the live count of each list follows the lists + const int * GGML_CUDA_RESTRICT sparse_indices = use_sparse ? KV_max_ptr : nullptr; + const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; + float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr; // Skip unused kernel variants for faster compilation: if (use_logit_softcap && !(DKQ == 128 || DKQ == 256 || DKQ == 512)) { NO_DEVICE_CODE; return; } + if (DKQ == 192 && ncols2 != 8 && ncols2 != 16) { + NO_DEVICE_CODE; + return; + } + + if (!ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2) && use_sparse) { + NO_DEVICE_CODE; + return; + } #ifdef VOLTA_MMA_AVAILABLE if (ncols1*ncols2 < 32) { NO_DEVICE_CODE; @@ -1590,14 +1859,14 @@ static __global__ void flash_attn_ext_f16( #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING #if defined(AMD_WMMA_AVAILABLE) - if (ncols1*ncols2 > 32 || ncols1*ncols2 < 16 || DKQ > 128 || ncols2 == 1) { + if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) { NO_DEVICE_CODE; return; } #endif // defined(AMD_WMMA_AVAILABLE) #if defined(AMD_MFMA_AVAILABLE) - if (DKQ != 64 && DKQ != 80 && DKQ != 96 && DKQ != 112 && DKQ != 128) { + if (ncols1*ncols2 < 16 || (DKQ > 256 && ncols1*ncols2 < 32)) { NO_DEVICE_CODE; return; } @@ -1622,6 +1891,10 @@ static __global__ void flash_attn_ext_f16( const int iter_j = (ne01.z + (ncols1 - 1)) / ncols1; const int iter_z_gqa = (gqa_ratio + (ncols2 - 1)) / ncols2; + if (use_sparse) { + KV_max = KV_max_ptr + int64_t(iter_j)*ne33*ne11; + } + // kbc == k block continuous, current index in continuous ijk space. int kbc = int64_t(blockIdx.x + 0)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x; const int kbc_stop = int64_t(blockIdx.x + 1)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x; @@ -1651,22 +1924,25 @@ static __global__ void flash_attn_ext_f16( const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV); const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr; + const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr; const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f; - if (KV_max) { + if (use_sparse) { + kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa); + } else if (KV_max) { kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa); } constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. if (kb0_start == 0) { constexpr bool needs_fixup = false; // CUDA block is working on an entire tile. - flash_attn_ext_f16_process_tile - (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, + flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop); } else { constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile. - flash_attn_ext_f16_process_tile - (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, + flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop); } @@ -1697,20 +1973,23 @@ static __global__ void flash_attn_ext_f16( const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV); const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr; + const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr; const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f; - if (KV_max) { + if (use_sparse) { + kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa); + } else if (KV_max) { kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa); } constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. constexpr bool needs_fixup = false; - flash_attn_ext_f16_process_tile - (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, + flash_attn_ext_f16_process_tile + (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap, ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop); #else - GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, + GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, ne00, ne01, ne02, ne03, nb01, nb02, nb03, @@ -1720,9 +1999,11 @@ static __global__ void flash_attn_ext_f16( ne31, ne32, ne33, nb31, nb32, nb33); NO_DEVICE_CODE; -#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)) +#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)) } +bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_tensor * dst, const int ncols1, const int ncols2); + template void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * KQV = dst; @@ -1745,8 +2026,12 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml constexpr bool V_is_K_view = DKQ == 576; // Guaranteed by the kernel selection logic in fattn.cu - const size_t nbytes_shared_KV_1stage = nbatch_fa * std::max(nbatch_K2 + 4, nbatch_V2 + 4) * sizeof(half2); - const size_t nbytes_shared_KV_2stage = nbatch_fa * (nbatch_K2 + 4 + nbatch_V2 + 4) * sizeof(half2); + // KV tile strides must match flash_attn_ext_f16_iter / _process_tile. + const bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols, cc); + const int stride_tile_K = swz ? nbatch_K2 : nbatch_K2 + 4; + const int stride_tile_V = V_is_K_view ? stride_tile_K : (swz ? nbatch_V2 : nbatch_V2 + 4); + const size_t nbytes_shared_KV_1stage = nbatch_fa * std::max(stride_tile_K, stride_tile_V) * sizeof(half2); + const size_t nbytes_shared_KV_2stage = nbatch_fa * (stride_tile_K + stride_tile_V) * sizeof(half2); const size_t nbytes_shared_Q = ncols * (DKQ/2 + 4) * sizeof(half2); const size_t nbytes_shared_mask = ncols1 * (nbatch_fa/2 + 4) * sizeof(half2); const size_t nbytes_shared_combine = nwarps*cols_per_warp * (nbatch_combine + 4) * sizeof(half2); @@ -1766,32 +2051,58 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml using fattn_kernel_ptr_t = fattn_kernel_t; #endif // defined(GGML_USE_HIP) fattn_kernel_t fattn_kernel; + bool use_sparse = false; if (logit_softcap == 0.0f) { constexpr bool use_logit_softcap = false; - fattn_kernel = flash_attn_ext_f16; +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2)) { + if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, ncols1, ncols2)) { + constexpr bool use_sparse_kernel = true; + fattn_kernel = flash_attn_ext_f16; + use_sparse = true; + + static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false}; + if (!shared_memory_limit_raised[id]) { + CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total)); + shared_memory_limit_raised[id] = true; + } + } else { + constexpr bool use_sparse_kernel = false; + fattn_kernel = flash_attn_ext_f16; -#if !defined(GGML_USE_MUSA) - static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false}; - if (!shared_memory_limit_raised[id]) { - CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total)); - shared_memory_limit_raised[id] = true; + static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false}; + if (!shared_memory_limit_raised[id]) { + CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total)); + shared_memory_limit_raised[id] = true; + } + } + } else +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + { + constexpr bool use_sparse_kernel = false; + fattn_kernel = flash_attn_ext_f16; + + static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false}; + if (!shared_memory_limit_raised[id]) { + CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total)); + shared_memory_limit_raised[id] = true; + } } -#endif // !defined(GGML_USE_MUSA) } else { constexpr bool use_logit_softcap = true; - fattn_kernel = flash_attn_ext_f16; + constexpr bool use_sparse_kernel = false; + fattn_kernel = flash_attn_ext_f16; -#if !defined(GGML_USE_MUSA) static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false}; if (!shared_memory_limit_raised[id]) { CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total)); shared_memory_limit_raised[id] = true; } -#endif // !defined(GGML_USE_MUSA) } + const bool async_kv_preload = nstages == 2 && !use_sparse; launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, warp_size_host); + (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host, async_kv_preload); } @@ -1834,6 +2145,10 @@ DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128, 64) DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256, 64) +extern DECL_FATTN_MMA_F16_CASE(512, 512, 4, 2); +extern DECL_FATTN_MMA_F16_CASE(512, 512, 8, 2); +extern DECL_FATTN_MMA_F16_CASE(512, 512, 16, 2); +extern DECL_FATTN_MMA_F16_CASE(512, 512, 32, 2); extern DECL_FATTN_MMA_F16_CASE(512, 512, 2, 4); extern DECL_FATTN_MMA_F16_CASE(512, 512, 4, 4); extern DECL_FATTN_MMA_F16_CASE(512, 512, 8, 4); @@ -1848,6 +2163,10 @@ extern DECL_FATTN_MMA_F16_CASE(576, 512, 1, 16); extern DECL_FATTN_MMA_F16_CASE(576, 512, 2, 16); extern DECL_FATTN_MMA_F16_CASE(576, 512, 4, 16); +// Mistral Small 4 (DKQ=320, DV=256), GQA=32-only build: +extern DECL_FATTN_MMA_F16_CASE(320, 256, 1, 32); +extern DECL_FATTN_MMA_F16_CASE(320, 256, 2, 32); + // For GLM 4.7 Flash extern DECL_FATTN_MMA_F16_CASE(576, 512, 4, 4); extern DECL_FATTN_MMA_F16_CASE(576, 512, 8, 4); diff --git a/ggml/src/ggml-cuda/fattn-tile.cu b/ggml/src/ggml-cuda/fattn-tile.cu index 25b16e8..accef63 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cu +++ b/ggml/src/ggml-cuda/fattn-tile.cu @@ -1,6 +1,5 @@ #include "common.cuh" #include "fattn-tile.cuh" -#include "fattn-wmma-f16.cuh" void ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * K = dst->src[1]; @@ -34,10 +33,18 @@ void ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor GGML_ASSERT(V->ne[0] == K->ne[0]); ggml_cuda_flash_attn_ext_tile_case<128, 128>(ctx, dst); } break; + case 192: { + GGML_ASSERT(V->ne[0] == 128); + ggml_cuda_flash_attn_ext_tile_case<192, 128>(ctx, dst); + } break; case 256: { GGML_ASSERT(V->ne[0] == K->ne[0]); ggml_cuda_flash_attn_ext_tile_case<256, 256>(ctx, dst); } break; + case 320: { + GGML_ASSERT(V->ne[0] == 256); + ggml_cuda_flash_attn_ext_tile_case<320, 256>(ctx, dst); + } break; case 512: { GGML_ASSERT(V->ne[0] == K->ne[0]); ggml_cuda_flash_attn_ext_tile_case<512, 512>(ctx, dst); @@ -46,8 +53,7 @@ void ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor GGML_ASSERT(V->ne[0] == 512); ggml_cuda_flash_attn_ext_tile_case<576, 512>(ctx, dst); } break; - default: { + default: GGML_ABORT("Unsupported head size"); - } break; } } diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh index 26721cc..bd4a771 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cuh +++ b/ggml/src/ggml-cuda/fattn-tile.cuh @@ -1,6 +1,5 @@ #include "common.cuh" #include "fattn-common.cuh" -#include "fattn-wmma-f16.cuh" // nbatch_fa == number of KQ rows to process per iteration // nbatch_K == number of K columns to load in parallel for KQ calculation @@ -20,41 +19,41 @@ } \ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nvidia_fp16(const int DKQ, const int DV, const int ncols) { - GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 64, 2, 64, 40) - GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 64, 40) - GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 256, 2, 64, 40) + GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 2, 128, 3, 128, 40) + GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 4, 128, 2, 128, 40) + GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 8, 128, 3, 128, 40) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 16, 256, 2, 64, 40) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 40, 40, 32, 256, 2, 64, 40) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 4, 128, 2, 64, 64) - GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 8, 256, 3, 128, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 64) - GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 64, 2, 64, 72) - GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 2, 64, 72) + GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 2, 128, 2, 64, 72) + GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 4, 128, 3, 128, 72) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 8, 256, 2, 64, 72) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 16, 256, 2, 64, 72) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 72, 72, 32, 256, 2, 64, 72) - GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 64, 2, 64, 40) + GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 2, 128, 2, 64, 40) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 4, 128, 2, 64, 40) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40) - GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 64, 40) + GGML_CUDA_FATTN_TILE_CONFIG_CASE( 80, 80, 32, 256, 2, 128, 40) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 2, 64, 2, 64, 48) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 4, 128, 2, 64, 48) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 8, 256, 2, 64, 48) GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 16, 256, 2, 64, 48) - GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 64, 48) + GGML_CUDA_FATTN_TILE_CONFIG_CASE( 96, 96, 32, 256, 2, 128, 24) - GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 2, 64, 2, 64, 56) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 2, 128, 3, 64, 56) GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 4, 128, 2, 64, 56) - GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 2, 64, 56) - GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 64, 56) - GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 64, 56) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 8, 256, 3, 64, 112) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 16, 256, 2, 32, 112) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(112, 112, 32, 256, 2, 128, 56) GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 4, 128, 2, 64, 64) @@ -62,12 +61,21 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 2, 64, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 4, 128, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 8, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 16, 256, 2, 64, 64) + + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 2, 64, 64) @@ -122,12 +130,21 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 16, 128, 3, 32, 128) GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 2, 128, 3, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 4, 128, 3, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 8, 256, 2, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 2, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 128, 3, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 3, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 32, 256) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 32, 128) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 16, 256, 2, 32, 64) + + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 2, 32, 64) @@ -189,12 +206,21 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 2, 64, 32) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 2, 256, 2, 128, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 4, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 8, 256, 2, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 2, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 256, 2, 128, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 256, 2, 64, 128) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 64, 128) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 32, 128) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 32, 128) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 32, 512, 1, 128, 64) + + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 2, 64, 64) @@ -258,12 +284,21 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 32, 256, 3, 128, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(128, 128, 64, 256, 3, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 2, 64, 8, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 4, 128, 6, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 8, 128, 6, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 16, 256, 5, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 3, 64, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 64, 8, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 6, 32, 256) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 128, 6, 32, 256) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 5, 32, 256) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 3, 64, 128) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(320, 256, 32, 256, 2, 128, 64) + + GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 2, 64, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 4, 128, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 8, 256, 2, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(512, 512, 16, 256, 4, 64, 64) @@ -756,14 +791,14 @@ static __device__ __forceinline__ void flash_attn_tile_iter( template // D == head size __launch_bounds__(ggml_cuda_fattn_tile_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_tile_get_occupancy(DKQ, DV, ncols1*ncols2)) static __global__ void flash_attn_tile( - const char * __restrict__ Q, - const char * __restrict__ K, - const char * __restrict__ V, - const char * __restrict__ mask, - const char * __restrict__ sinks, - const int * __restrict__ KV_max, - float * __restrict__ dst, - float2 * __restrict__ dst_meta, + const char * Q_ptr, + const char * K_ptr, + const char * V_ptr, + const char * mask_ptr, + const char * sinks_ptr, + const int * KV_max_ptr, + float * dst_ptr, + float2 * dst_meta_ptr, const float scale, const float max_bias, const float m0, @@ -778,15 +813,18 @@ static __global__ void flash_attn_tile( const int32_t ne31, const int32_t ne32, const int32_t ne33, const int32_t nb31, const int32_t nb32, const int64_t nb33) { #ifdef FLASH_ATTN_AVAILABLE + const char * GGML_CUDA_RESTRICT Q = Q_ptr; + const char * GGML_CUDA_RESTRICT K = K_ptr; + const char * GGML_CUDA_RESTRICT V = V_ptr; + const char * GGML_CUDA_RESTRICT mask = mask_ptr; + const char * GGML_CUDA_RESTRICT sinks = sinks_ptr; + const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; + float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr; // Skip unused kernel variants for faster compilation: - if ( -#ifdef GGML_USE_WMMA_FATTN - (ncols2 != 1 && DV != 40 && DV != 72 && DV != 512) || -#endif // GGML_USE_WMMA_FATTN - (use_logit_softcap && !(DV == 128 || DV == 256 || DV == 512)) - ) { + if ((use_logit_softcap && !(DV == 128 || DV == 256 || DV == 512))) { GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, max_bias, m0, m1, n_head_log2, logit_softcap, ne00, ne01, ne02, ne03, @@ -862,6 +900,8 @@ static __global__ void flash_attn_tile( } float KQ_sum[cpw] = {0.0f}; + ggml_cuda_pdl_sync(); + // Load Q data, convert to FP16 if fast: #pragma unroll for (int jc0 = 0; jc0 < cpw; ++jc0) { @@ -915,7 +955,7 @@ static __global__ void flash_attn_tile( if (ncols2 == 1) { // Branch with out-of-bounds checks. int k_VKQ_0 = blockIdx.y*nbatch_fa; - while (k_VKQ_0 < k_VKQ_max - nbatch_fa) { + while (k_VKQ_0 <= k_VKQ_max - nbatch_fa) { constexpr bool oob_check = false; flash_attn_tile_iter (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, @@ -1092,7 +1132,7 @@ static __global__ void flash_attn_tile( } } #else - GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, + GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, ne00, ne01, ne02, ne03, nb01, nb02, nb03, @@ -1116,21 +1156,21 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm constexpr size_t nbytes_shared = 0; #ifdef GGML_USE_HIP - if constexpr (DV <= 128) { + if constexpr (DKQ <= 128) { if (Q->ne[1] > 32/ncols2) { constexpr int cols_per_block = 64; const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size; const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); fattn_kernel_t fattn_kernel = flash_attn_tile; launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size); return; } } #endif // GGML_USE_HIP #ifndef GGML_USE_HIP - if constexpr (DV <= 256) + if constexpr (DKQ <= 256) #endif // GGML_USE_HIP { if (Q->ne[1] > 16/ncols2) { @@ -1139,19 +1179,21 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); fattn_kernel_t fattn_kernel = flash_attn_tile; launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size); return; } } - if (Q->ne[1] > 8/ncols2) { - constexpr int cols_per_block = 16; - const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size; - const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); - fattn_kernel_t fattn_kernel = flash_attn_tile; - launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); - return; + if constexpr (ncols2 <= 16) { + if (Q->ne[1] > 8/ncols2) { + constexpr int cols_per_block = 16; + const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size; + const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); + fattn_kernel_t fattn_kernel = flash_attn_tile; + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size); + return; + } } if constexpr (ncols2 <= 8) { @@ -1161,7 +1203,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); fattn_kernel_t fattn_kernel = flash_attn_tile; launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size); return; } } @@ -1173,7 +1215,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); fattn_kernel_t fattn_kernel = flash_attn_tile; launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size); return; } } @@ -1184,7 +1226,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); fattn_kernel_t fattn_kernel = flash_attn_tile; launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size); return; } @@ -1210,6 +1252,25 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm const int gqa_limit = nvidia && gqa_ratio <= 4 && DV <= 256 ? 16 : INT_MAX; const bool use_gqa_opt = mask && max_bias == 0.0f && Q->ne[1] <= gqa_limit && K->ne[1] % FATTN_KQ_STRIDE == 0; + if constexpr (DKQ == 320) { + // This branch is only used for Mistral Small 4 which has a GQA ratio of 32. + // On AMD, simply use that GQA ratio with 32 columns / block since we always have enough SRAM. + // On NVIDIA however, the tile kernel is only used for GPUs that can't use the mma kernel (Pascal and older). + // Therefore, use a GQA ratio of 16 with 16 columns / block to stay below 48 kiB of SRAM / block. +#ifdef GGML_USE_HIP + if (use_gqa_opt && gqa_ratio % 32 == 0) { + launch_fattn_tile_switch_ncols1(ctx, dst); + return; + } +#else + if (use_gqa_opt && gqa_ratio % 16 == 0) { + launch_fattn_tile_switch_ncols1(ctx, dst); + return; + } +#endif // GGML_USE_HIP + GGML_ABORT("flash-attn tile (320/256): expected GQA ratio multiple of 32"); + } + if constexpr (DKQ == 576) { if (use_gqa_opt && gqa_ratio % 16 == 0) { launch_fattn_tile_switch_ncols1(ctx, dst); @@ -1221,7 +1282,20 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm } } - if constexpr (DKQ <= 512) { + if constexpr (DKQ == 192) { + // MiMo-V2.5 / V2.5-Pro / V2-Flash: gqa_ratio is 8 (SWA) or 16 (full attn) + if (use_gqa_opt && gqa_ratio % 16 == 0) { + launch_fattn_tile_switch_ncols1(ctx, dst); + return; + } + if (use_gqa_opt && gqa_ratio % 8 == 0) { + launch_fattn_tile_switch_ncols1(ctx, dst); + return; + } + GGML_ABORT("flash-attn tile (192/128): expected GQA ratio multiple of 8"); + } + + if constexpr (DKQ <= 512 && DKQ != 320 && DKQ != 192) { if (use_gqa_opt && gqa_ratio % 8 == 0) { launch_fattn_tile_switch_ncols1(ctx, dst); return; @@ -1232,12 +1306,12 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm return; } - if constexpr (DV <= 256) { - if (use_gqa_opt && gqa_ratio % 2 == 0) { - launch_fattn_tile_switch_ncols1(ctx, dst); - return; - } + if (use_gqa_opt && gqa_ratio % 2 == 0) { + launch_fattn_tile_switch_ncols1(ctx, dst); + return; + } + if constexpr (DV <= 256) { launch_fattn_tile_switch_ncols1(ctx, dst); return; } @@ -1274,6 +1348,8 @@ extern DECL_FATTN_TILE_CASE( 80, 80); extern DECL_FATTN_TILE_CASE( 96, 96); extern DECL_FATTN_TILE_CASE(112, 112); extern DECL_FATTN_TILE_CASE(128, 128); +extern DECL_FATTN_TILE_CASE(192, 128); extern DECL_FATTN_TILE_CASE(256, 256); +extern DECL_FATTN_TILE_CASE(320, 256); extern DECL_FATTN_TILE_CASE(512, 512); extern DECL_FATTN_TILE_CASE(576, 512); diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuh index f0bd42a..74f2820 100644 --- a/ggml/src/ggml-cuda/fattn-vec.cuh +++ b/ggml/src/ggml-cuda/fattn-vec.cuh @@ -19,14 +19,14 @@ static constexpr __device__ int ggml_cuda_fattn_vec_get_nthreads_device() { template // D == head size __launch_bounds__(ggml_cuda_fattn_vec_get_nthreads_device(), 1) static __global__ void flash_attn_ext_vec( - const char * __restrict__ Q, - const char * __restrict__ K, - const char * __restrict__ V, - const char * __restrict__ mask, - const char * __restrict__ sinks, - const int * __restrict__ KV_max, - float * __restrict__ dst, - float2 * __restrict__ dst_meta, + const char * Q_ptr, + const char * K_ptr, + const char * V_ptr, + const char * mask_ptr, + const char * sinks_ptr, + const int * KV_max_ptr, + float * dst_ptr, + float2 * dst_meta_ptr, const float scale, const float max_bias, const float m0, @@ -40,7 +40,16 @@ static __global__ void flash_attn_ext_vec( const int32_t nb21, const int32_t nb22, const int64_t nb23, const int32_t ne31, const int32_t ne32, const int32_t ne33, const int32_t nb31, const int32_t nb32, const int64_t nb33) { + ggml_cuda_pdl_lc(); #ifdef FLASH_ATTN_AVAILABLE + const char * GGML_CUDA_RESTRICT Q = Q_ptr; + const char * GGML_CUDA_RESTRICT K = K_ptr; + const char * GGML_CUDA_RESTRICT V = V_ptr; + const char * GGML_CUDA_RESTRICT mask = mask_ptr; + const char * GGML_CUDA_RESTRICT sinks = sinks_ptr; + const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; + float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr; // Skip unused kernel variants for faster compilation: if (use_logit_softcap && !(D == 128 || D == 256)) { @@ -136,6 +145,8 @@ static __global__ void flash_attn_ext_vec( #endif // V_DOT2_F32_F16_AVAILABLE int Q_i32[ncols][1 > D/(sizeof(int)*nthreads_KQ) ? 1 : D/(sizeof(int)*nthreads_KQ)]; float2 Q_ds[ncols][1 > D/(sizeof(int)*nthreads_KQ) ? 1 : D/(sizeof(int)*nthreads_KQ)]; + + ggml_cuda_pdl_sync(); if constexpr (Q_q8_1) { #pragma unroll for (int j0 = 0; j0 < ncols; j0 += nwarps) { @@ -306,9 +317,7 @@ static __global__ void flash_attn_ext_vec( #endif // V_DOT2_F32_F16_AVAILABLE } -#ifndef GGML_USE_HIP - __syncwarp(); -#endif // GGML_USE_HIP + ggml_cuda_syncwarp(); // Needed for KQ synchronization. #pragma unroll for (int k0 = 0; k0 < WARP_SIZE; k0 += V_cols_per_iter) { @@ -365,6 +374,8 @@ static __global__ void flash_attn_ext_vec( } #endif // V_DOT2_F32_F16_AVAILABLE } + + ggml_cuda_syncwarp(); // Needed for KQ synchronization. } if (sinks && blockIdx.y == 0) { @@ -503,7 +514,7 @@ static __global__ void flash_attn_ext_vec( dst_meta[((sequence*int(ne01.z) + ic0 + tid)*ne02 + head)*gridDim.y + blockIdx.y] = make_float2(KQ_max[tid], KQ_sum[tid]); } #else - GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, + GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, ne00, ne01, ne02, ne03, nb01, nb02, nb03, @@ -529,7 +540,7 @@ void ggml_cuda_flash_attn_ext_vec_case_impl(ggml_backend_cuda_context & ctx, ggm const bool need_f16_K = type_K == GGML_TYPE_F16; const bool need_f16_V = type_V == GGML_TYPE_F16; constexpr size_t nbytes_shared = 0; - launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false); + launch_fattn(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false, false); } template diff --git a/ggml/src/ggml-cuda/fattn-wmma-f16.cu b/ggml/src/ggml-cuda/fattn-wmma-f16.cu deleted file mode 100644 index f19defb..0000000 --- a/ggml/src/ggml-cuda/fattn-wmma-f16.cu +++ /dev/null @@ -1,696 +0,0 @@ -// Old and deprecated WMMA FlashAttention implementation. -// It is still needed for Volta since the memory layout of NVIDIA tensor cores changed with Turing. -// Long-term the WMMA code should be replaced with a dedicated Volta implementation. - -#include "common.cuh" -#include "fattn-common.cuh" -#include "fattn-wmma-f16.cuh" - -#ifdef GGML_USE_WMMA_FATTN -#if !defined(GGML_USE_HIP) -#include -#if defined(GGML_USE_MUSA) -namespace wmma = mtmusa::wmma; -#else // GGML_USE_MUSA -namespace wmma = nvcuda::wmma; -#endif // GGML_USE_MUSA -#elif defined(GGML_USE_HIP) -#include -namespace wmma = rocwmma; -#endif // !defined(GGML_USE_HIP) -#endif // GGML_USE_WMMA_FATTN - -// D == head size, VKQ_stride == num VKQ rows calculated in parallel: -template -__launch_bounds__(nwarps*ggml_cuda_get_physical_warp_size(), 1) -static __global__ void flash_attn_ext_f16( - const char * __restrict__ Q, - const char * __restrict__ K, - const char * __restrict__ V, - const char * __restrict__ mask, - const char * __restrict__ sinks, - const int * __restrict__ KV_max, - float * __restrict__ dst, - float2 * __restrict__ dst_meta, - const float scale, - const float max_bias, - const float m0, - const float m1, - const uint32_t n_head_log2, - const float logit_softcap, - const int32_t ne00, const uint3 ne01, const int32_t ne02, const int32_t ne03, - const int32_t nb01, const int32_t nb02, const int32_t nb03, - const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13, - const int32_t nb11, const int32_t nb12, const int64_t nb13, - const int32_t nb21, const int32_t nb22, const int64_t nb23, - const int32_t ne31, const int32_t ne32, const int32_t ne33, - const int32_t nb31, const int32_t nb32, const int64_t nb33) { -#if defined(FLASH_ATTN_AVAILABLE) && (defined(GGML_HIP_ROCWMMA_FATTN) && defined(GGML_USE_WMMA_FATTN)) - // Skip unused kernel variants for faster compilation: - if (use_logit_softcap && !(D == 128 || D == 256)) { - NO_DEVICE_CODE; - return; - } - - //In this kernel Q, K, V are matrices while i, j, k are matrix indices. - - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - const int ic0 = ncols*blockIdx.x; // Index of the first Q/QKV column to work on. - - static_assert(D <= FATTN_KQ_STRIDE, "D must be <= FATTN_KQ_STRIDE."); - static_assert(ncols == 8 || ncols % 16 == 0, "ncols must be 8 or a multiple of 16."); - constexpr int frag_m = ncols == 8 ? 32 : 16; - constexpr int frag_n = ncols == 8 ? 8 : 16; - static_assert(D % frag_m == 0, "If ncols == 8 then D % frag_m must be 0."); -#if defined(GGML_USE_HIP) && HIP_VERSION >= 60500000 - typedef wmma::fragment frag_a_K; - typedef wmma::fragment frag_a_V; - typedef wmma::fragment frag_b; - typedef wmma::fragment frag_c_KQ; - typedef wmma::fragment frag_c_VKQ; -#else - typedef wmma::fragment frag_a_K; - typedef wmma::fragment frag_a_V; - typedef wmma::fragment frag_b; - typedef wmma::fragment frag_c_KQ; - typedef wmma::fragment frag_c_VKQ; -#endif - - constexpr int KQ_stride_tc = nwarps*frag_m; // Number of KQ rows calculated in parallel. - constexpr int VKQ_ratio = KQ_stride_tc/VKQ_stride; // Number of parallel VKQ accumulators needed to keep all warps busy. - static_assert(VKQ_ratio <= nwarps, "VKQ_ratio must be <= nwarps."); - - // Pad internal representation of KQ, KQV to reduce shared memory bank conflicts: - constexpr int D_padded = D + 8; - constexpr int kqs_padded = FATTN_KQ_STRIDE + 8; - constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half); - - const int sequence = blockIdx.z / ne02; - const int head = blockIdx.z - sequence*ne02; - const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const float * Q_f = (const float *) (Q + nb03* sequence + nb02* head + nb01*ic0); - const half * K_h = (const half *) (K + nb13* sequence + nb12*(head / gqa_ratio)); - const half * V_h = (const half *) (V + nb13* sequence + nb12*(head / gqa_ratio)); // K and V have same shape - const half * maskh = (const half *) (mask + nb33*(sequence % ne33) + nb31*ic0); - const half2 * mask2 = (const half2 *) maskh; - const float * sinksf = (const float *) sinks; - - const int stride_Q = nb01 / sizeof(float); - const int stride_KV = nb11 / sizeof(half); - - const float slopef = get_alibi_slope(max_bias, head, n_head_log2, m0, m1); - const half slopeh = __float2half(slopef); - const half2 slope2 = make_half2(slopef, slopef); - - const half2 logit_softcap_2 = make_half2(logit_softcap, logit_softcap); - - frag_b Q_b[D/16][ncols/frag_n]; - - // A single buffer for temporarily holding tiles of KQ and VKQ parts: - constexpr int mem_KQ = ncols*kqs_padded*kqar; - constexpr int mem_VKQ_parts = VKQ_ratio*ncols*D_padded; - __shared__ half KQ[mem_KQ >= mem_VKQ_parts ? mem_KQ : mem_VKQ_parts]; - float * KQ_f = (float *) KQ; - half2 * KQ2 = (half2 *) KQ; - - float KQ_rowsum_f[ncols/nwarps] = {0.0f}; - float KQ_max_f[ncols/nwarps]; - float KQ_max_scale_f[ncols/nwarps] = {0.0f}; - -#pragma unroll - for (int j = 0; j < ncols/nwarps; ++j) { - KQ_max_f[j] = -FLT_MAX/2.0f; - } - - half2 KQ_rowsum_h2[ncols/nwarps] = {{0.0f, 0.0f}}; - half2 KQ_max_h2[ncols/nwarps]; - half2 KQ_max_scale_h2[ncols/nwarps] = {{0.0f, 0.0f}}; - -#pragma unroll - for (int j = 0; j < ncols/nwarps; ++j) { - KQ_max_h2[j] = make_half2(-HALF_MAX_HALF, -HALF_MAX_HALF); - } - - __shared__ half VKQ[ncols*D_padded]; // Accumulator for final VKQ slice. - half2 * VKQ2 = (half2 *) VKQ; - -#if defined(GGML_USE_HIP) && HIP_VERSION >= 60500000 - const _Float16 * K_h_f16 = reinterpret_cast(K_h); - const _Float16 * V_h_f16 = reinterpret_cast(V_h); - _Float16 * KQ_f16 = reinterpret_cast<_Float16 *>(KQ); - _Float16 * VKQ_f16 = reinterpret_cast<_Float16 *>(VKQ); -#else - const half * K_h_f16 = K_h; - const half * V_h_f16 = V_h; - half * KQ_f16 = KQ; - half * VKQ_f16 = VKQ; -#endif - -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += nwarps) { - const int j = j0 + threadIdx.y; -#pragma unroll - for (int i0 = 0; i0 < D/2; i0 += warp_size) { - const int i = i0 + threadIdx.x; - if (i0 + warp_size > D/2 && i >= D/2) { - break; - } - VKQ2[j*(D_padded/2) + i] = make_half2(0.0f, 0.0f); - } - } - - // Convert Q to half and apply scale, temporarily store in KQ: -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += nwarps) { - const int j = j0 + threadIdx.y; -#pragma unroll - for (int i0 = 0; i0 < D; i0 += warp_size) { - const int i = i0 + threadIdx.x; - if (i0 + warp_size > D && i >= D) { - break; - } - KQ[j*D_padded + i] = ic0 + j < int(ne01.z) ? Q_f[j*stride_Q + i] * scale : 0.0f; - } - } - - __syncthreads(); - - // Load Q into tensor core fragments/registers since it will be used frequently: -#pragma unroll - for (int i0 = 0; i0 < D; i0 += 16) { -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += frag_n) { - wmma::load_matrix_sync(Q_b[i0/16][j0/frag_n], KQ_f16 + j0*D_padded + i0, D_padded); - } - } - - __syncthreads(); - - // Iterate over ne11 == previous tokens: - const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11; - for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE) { - // Calculate tile of KQ: -#pragma unroll - for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) { - frag_c_KQ KQ_c[ncols/frag_n]; -#pragma unroll - for (int j = 0; j < ncols/frag_n; ++j) { - wmma::fill_fragment(KQ_c[j], static_cast(0.0f)); - } -#pragma unroll - for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 16) { - frag_a_K K_a; - wmma::load_matrix_sync(K_a, K_h_f16 + int64_t(k_VKQ_0 + i_KQ_0 + frag_m*threadIdx.y)*stride_KV + k_KQ_0, stride_KV); -#pragma unroll - for (int j = 0; j < ncols/frag_n; ++j) { - wmma::mma_sync(KQ_c[j], K_a, Q_b[k_KQ_0/16][j], KQ_c[j]); - } - } -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += frag_n) { - wmma::store_matrix_sync((KQ_acc_t *) KQ + j0*kqs_padded + i_KQ_0 + frag_m*threadIdx.y, KQ_c[j0/frag_n], kqs_padded, wmma::mem_col_major); - } - } - - __syncthreads(); - - // Calculate softmax for each KQ column using the current max. value. - // The divisor is stored in KQ_rowsum and will be applied at the end. -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += nwarps) { - const int j = j0 + threadIdx.y; - - if (std::is_same::value) { - float KQ_f_tmp[FATTN_KQ_STRIDE / warp_size]; -#pragma unroll - for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += warp_size) { - const int k = k0 + threadIdx.x; - - KQ_f_tmp[k0/warp_size] = KQ_f[j*kqs_padded + k]; - - if (use_logit_softcap) { - KQ_f_tmp[k0/warp_size] = logit_softcap*tanhf(KQ_f_tmp[k0/warp_size]); - } - } - - float KQ_max_new = KQ_max_f[j0/nwarps]; -#pragma unroll - for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += warp_size) { - const int k = k0 + threadIdx.x; - - KQ_f_tmp[k0/warp_size] += mask && ic0 + j < int(ne01.z) ? - __half2float(slopeh*maskh[j*(nb31/sizeof(half)) + k_VKQ_0 + k]) : 0.0f; - KQ_max_new = max(KQ_max_new, KQ_f_tmp[k0/warp_size] + FATTN_KQ_MAX_OFFSET); - } - KQ_max_new = warp_reduce_max(KQ_max_new); - - const float diff = KQ_max_f[j0/nwarps] - KQ_max_new; - KQ_max_scale_f[j0/nwarps] = expf(diff); - if (diff <= SOFTMAX_FTZ_THRESHOLD) { - KQ_max_scale_f[j0/nwarps] = 0.0f; - } - KQ_max_f[j0/nwarps] = KQ_max_new; - - float KQ_rowsum_add = 0.0f; -#pragma unroll - for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += warp_size) { - const int k = k0 + threadIdx.x; - - const float diff = KQ_f_tmp[k0/warp_size] - KQ_max_f[j0/nwarps]; - KQ_f_tmp[k0/warp_size] = expf(diff); - if (diff <= SOFTMAX_FTZ_THRESHOLD) { - KQ_f_tmp[k0/warp_size] = 0.0f; - } - KQ_rowsum_add += KQ_f_tmp[k0/warp_size]; - KQ[j*(kqar*kqs_padded) + k] = KQ_f_tmp[k0/warp_size]; - } - KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add); - - // Scale previous KQ_rowsum to account for a potential increase in KQ_max: - KQ_rowsum_f[j0/nwarps] = KQ_max_scale_f[j0/nwarps]*KQ_rowsum_f[j0/nwarps] + KQ_rowsum_add; - } else { - half2 KQ2_tmp[FATTN_KQ_STRIDE/(2*warp_size)]; -#pragma unroll - for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += warp_size) { - const int k = k0 + threadIdx.x; - - KQ2_tmp[k0/warp_size] = KQ2[j*(kqs_padded/2) + k]; - - if (use_logit_softcap) { - // There is no dedicated tangens hyperbolicus function for half2. - KQ2_tmp[k0/warp_size] = h2exp(KQ2_tmp[k0/warp_size]*make_half2(2.0f, 2.0f)); - KQ2_tmp[k0/warp_size] = (KQ2_tmp[k0/warp_size] - make_half2(1.0f, 1.0f)) - /(KQ2_tmp[k0/warp_size] + make_half2(1.0f, 1.0f)); - - KQ2_tmp[k0/warp_size] *= logit_softcap_2; - } - } - - half2 KQ_max_new = KQ_max_h2[j0/nwarps]; -#pragma unroll - for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += warp_size) { - const int k = k0 + threadIdx.x; - - KQ2_tmp[k0/warp_size] += mask && ic0 + j < int(ne01.z) ? slope2*mask2[(j*ne11 + k_VKQ_0)/2 + k] : make_half2(0.0f, 0.0f); - KQ_max_new = ggml_cuda_hmax2(KQ_max_new, KQ2_tmp[k0/warp_size]); - } - KQ_max_new = __half2half2(warp_reduce_max(ggml_cuda_hmax(__low2half(KQ_max_new), __high2half(KQ_max_new)))); - const half2 diff = KQ_max_h2[j0/nwarps] - KQ_max_new; - KQ_max_scale_h2[j0/nwarps] = h2exp(diff); - const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD)); - *((uint32_t *) &KQ_max_scale_h2[j0/nwarps]) &= ftz_mask; - KQ_max_h2[j0/nwarps] = KQ_max_new; - - half2 KQ_rowsum_add = make_half2(0.0f, 0.0f); -#pragma unroll - for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += warp_size) { - const int k = k0 + threadIdx.x; - - const half2 diff = KQ2_tmp[k0/warp_size] - KQ_max_h2[j0/nwarps]; - KQ2_tmp[k0/warp_size] = h2exp(diff); - const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD)); - *((uint32_t *) &KQ2_tmp[k0/warp_size]) &= ftz_mask; - KQ_rowsum_add += KQ2_tmp[k0/warp_size]; - KQ2[j*(kqs_padded/2) + k] = KQ2_tmp[k0/warp_size]; - } - KQ_rowsum_add = warp_reduce_sum(KQ_rowsum_add); - - // Scale previous KQ_rowsum to account for a potential increase in KQ_max: - KQ_rowsum_h2[j0/nwarps] = KQ_max_scale_h2[j0/nwarps]*KQ_rowsum_h2[j0/nwarps] + KQ_rowsum_add; - } - } - - __syncthreads(); - - frag_b KQ_b[FATTN_KQ_STRIDE/(VKQ_ratio*16)][ncols/frag_n]; -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += frag_n) { -#pragma unroll - for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) { - const int k = k0 + (threadIdx.y % VKQ_ratio)*16; - wmma::load_matrix_sync( - KQ_b[k0/(VKQ_ratio*16)][j0/frag_n], - KQ_f16 + j0*(kqar*kqs_padded) + k, - kqar*kqs_padded); - } - } - - frag_c_VKQ VKQ_c[D/VKQ_stride][ncols/frag_n]; -#pragma unroll - for (int i_VKQ_0 = 0; i_VKQ_0 < D; i_VKQ_0 += VKQ_stride) { -#pragma unroll - for (int j = 0; j < ncols/frag_n; ++j) { - wmma::fill_fragment(VKQ_c[i_VKQ_0/VKQ_stride][j], static_cast(0.0f)); - } - -#pragma unroll - for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) { - const int k = k0 + (threadIdx.y % VKQ_ratio)*16; - - frag_a_V v_a; - wmma::load_matrix_sync(v_a, V_h_f16 + int64_t(k_VKQ_0 + k)*stride_KV + i_VKQ_0 + frag_m*(threadIdx.y/VKQ_ratio), stride_KV); -#pragma unroll - for (int j = 0; j < ncols/frag_n; ++j) { - wmma::mma_sync(VKQ_c[i_VKQ_0/VKQ_stride][j], v_a, KQ_b[k0/(VKQ_ratio*16)][j], VKQ_c[i_VKQ_0/VKQ_stride][j]); - } - } - } - - __syncthreads(); - - const int offset_k = (threadIdx.y % VKQ_ratio) * (ncols*D_padded); -#pragma unroll - for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += VKQ_stride) { -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += frag_n) { - wmma::store_matrix_sync( - KQ_f16 + offset_k + j0*D_padded + i_KQ_0 + frag_m*(threadIdx.y/VKQ_ratio), - VKQ_c[i_KQ_0/VKQ_stride][j0/frag_n], - D_padded, wmma::mem_col_major); - } - } - - __syncthreads(); - -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += nwarps) { - const int j = j0 + threadIdx.y; - - half2 VKQ_scale; - if (std::is_same::value) { - VKQ_scale = make_half2(KQ_max_scale_f[j0/nwarps], KQ_max_scale_f[j0/nwarps]); - } else { - VKQ_scale = KQ_max_scale_h2[j0/nwarps]; - } - -#pragma unroll - for (int i0 = 0; i0 < D/2; i0 += warp_size) { - const int i = i0 + threadIdx.x; - if (i0 + warp_size > D/2 && i >= D/2) { - break; - } - - half2 VKQ_add = make_half2(0.0f, 0.0f); -#pragma unroll - for (int l = 0; l < VKQ_ratio; ++l) { - VKQ_add += KQ2[l*(ncols*D_padded/2) + j*(D_padded/2) + i]; - } - VKQ2[j*(D_padded/2) + i] = VKQ_scale*VKQ2[j*(D_padded/2) + i] + VKQ_add; - } - } - - __syncthreads(); - } - - // Apply attention sinks - if (sinksf && blockIdx.y == 0) { - const float sinkf = sinksf[head]; - const half sinkh = __float2half(sinkf); - -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += nwarps) { - const int j = j0 + threadIdx.y; - - if (std::is_same::value) { - float kqmax_new = fmaxf(KQ_max_f[j0/nwarps], sinkf); - - const float KQ_max_scale = expf(KQ_max_f[j0/nwarps] - kqmax_new); - KQ_max_f[j0/nwarps] = kqmax_new; - - KQ_rowsum_f[j0/nwarps] = KQ_rowsum_f[j0/nwarps] * KQ_max_scale + expf(sinkf - KQ_max_f[j0/nwarps]); - - const half2 scale_h2 = make_half2(KQ_max_scale, KQ_max_scale); -#pragma unroll - for (int i0 = 0; i0 < D/2; i0 += warp_size) { - const int i = i0 + threadIdx.x; - if (i0 + warp_size > D/2 && i >= D/2) break; - VKQ2[j*(D_padded/2) + i] *= scale_h2; - } - } else { - half kqmax_old = __low2half(KQ_max_h2[j0/nwarps]); - half kqmax_new = fmaxf(kqmax_old, sinkh); - KQ_max_h2[j0/nwarps] = __half2half2(kqmax_new); - - const half KQ_max_scale_h = hexp(kqmax_old - kqmax_new); - const half2 KQ_max_scale = __half2half2(KQ_max_scale_h); - - KQ_rowsum_h2[j0/nwarps] = KQ_rowsum_h2[j0/nwarps] * KQ_max_scale; - const half val = hexp(sinkh - kqmax_new); - KQ_rowsum_h2[j0/nwarps].x = __hadd(KQ_rowsum_h2[j0/nwarps].x, val); - -#pragma unroll - for (int i0 = 0; i0 < D/2; i0 += warp_size) { - const int i = i0 + threadIdx.x; - if (i0 + warp_size > D/2 && i >= D/2) break; - VKQ2[j*(D_padded/2) + i] *= KQ_max_scale; - } - } - } - - __syncthreads(); - } -#pragma unroll - for (int j0 = 0; j0 < ncols; j0 += nwarps) { - const int j_VKQ = j0 + threadIdx.y; - if (ic0 + j_VKQ >= int(ne01.z)) { - return; - } - - float KQ_rowsum_j; - if (std::is_same::value) { - KQ_rowsum_j = KQ_rowsum_f[j0/nwarps]; - } else { - KQ_rowsum_j = __low2float(KQ_rowsum_h2[j0/nwarps]) + __high2float(KQ_rowsum_h2[j0/nwarps]); - } - - const int j_dst_unrolled = ((sequence*int(ne01.z) + ic0 + j_VKQ)*ne02 + head)*gridDim.y + blockIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < D; i0 += warp_size) { - const int i = i0 + threadIdx.x; - if (i0 + warp_size > D && i >= D) { - break; - } - float dst_val = VKQ[j_VKQ*D_padded + i]; - if (gridDim.y == 1) { - dst_val /= KQ_rowsum_j; - } - dst[j_dst_unrolled*D + i] = dst_val; - } - - if (gridDim.y == 1 || threadIdx.x != 0) { - continue; - } - - float2 dst_meta_val; - if (std::is_same::value) { - dst_meta_val.x = KQ_max_f[j0/nwarps]; - } else { - dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]); - } - dst_meta_val.y = KQ_rowsum_j; - dst_meta[j_dst_unrolled] = dst_meta_val; - } -#else - GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, - max_bias, m0, m1, n_head_log2, logit_softcap, - ne00, ne01, ne02, ne03, - nb01, nb02, nb03, - ne10, ne11, ne12, ne13, - nb11, nb12, nb13, - nb21, nb22, nb23, - ne31, ne32, ne33, - nb31, nb32, nb33); - NO_DEVICE_CODE; -#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(GGML_HIP_ROCWMMA_FATTN) && defined(GGML_USE_WMMA_FATTN)) -} - -constexpr int get_max_power_of_2(int x) { - return x % 2 == 0 ? 2*get_max_power_of_2(x/2) : 1; -} - -static_assert(get_max_power_of_2(1) == 1, "Test failed."); -static_assert(get_max_power_of_2(2) == 2, "Test failed."); -static_assert(get_max_power_of_2(4) == 4, "Test failed."); -static_assert(get_max_power_of_2(6) == 2, "Test failed."); - -// Number of VKQ rows calculated in parallel: -constexpr int get_VKQ_stride(int D, int nwarps, int frag_m) { - return (get_max_power_of_2(D/frag_m) < nwarps ? get_max_power_of_2(D/frag_m) : nwarps)*frag_m; -} - -static_assert(get_VKQ_stride(128, 1, 32) == 32, "Test failed."); -static_assert(get_VKQ_stride(128, 2, 32) == 64, "Test failed."); -static_assert(get_VKQ_stride(128, 4, 32) == 128, "Test failed."); -static_assert(get_VKQ_stride( 64, 1, 32) == 32, "Test failed."); -static_assert(get_VKQ_stride( 64, 2, 32) == 64, "Test failed."); -static_assert(get_VKQ_stride( 64, 4, 32) == 64, "Test failed."); -static_assert(get_VKQ_stride( 80, 1, 16) == 16, "Test failed."); -static_assert(get_VKQ_stride( 80, 2, 16) == 16, "Test failed."); -static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed."); - -template -void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * KQV = dst; - - constexpr int nwarps = 4; - - constexpr int frag_m = cols_per_block == 8 && D % 32 == 0 ? 32 : 16; - const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; - - float logit_softcap; - memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); - - fattn_kernel_t fattn_kernel; - if (logit_softcap == 0.0f) { - constexpr bool use_logit_softcap = false; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>; - } else { - constexpr bool use_logit_softcap = true; - fattn_kernel = flash_attn_ext_f16< - D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>; - } - launch_fattn(ctx, dst, fattn_kernel, nwarps, 0, FATTN_KQ_STRIDE, true, true, false, warp_size); -} - -void ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * KQV = dst; - const ggml_tensor * Q = dst->src[0]; - - const enum ggml_prec prec = ggml_flash_attn_ext_get_prec(KQV); - const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - - if (prec != GGML_PREC_DEFAULT) { - if (Q->ne[1] <= 32 || Q->ne[0] > 128) { - constexpr int cols_per_block = 16; - switch (Q->ne[0]) { - case 64: - ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, float>(ctx, dst); - break; - case 80: - ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, float>(ctx, dst); - break; - case 96: - ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, float>(ctx, dst); - break; - case 112: - ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, float>(ctx, dst); - break; - case 128: - ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, float>(ctx, dst); - break; - case 256: - ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, float>(ctx, dst); - break; - default: - GGML_ABORT("fatal error"); - break; - } - } else { - constexpr int cols_per_block = 32; - switch (Q->ne[0]) { - case 64: - ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, float>(ctx, dst); - break; - case 80: - ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, float>(ctx, dst); - break; - case 96: - ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, float>(ctx, dst); - break; - case 112: - ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, float>(ctx, dst); - break; - case 128: - ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, float>(ctx, dst); - break; - // case 256: - // ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, float>(ctx, dst); - // break; - default: - GGML_ABORT("fatal error"); - break; - } - } - return; - } - -#if !defined(GGML_USE_HIP) - if (Q->ne[1] <= 8 && Q->ne[0] % warp_size == 0) { - constexpr int cols_per_block = 8; - switch (Q->ne[0]) { - case 64: - ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst); - break; - case 96: - ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst); - break; - case 128: - ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst); - break; - case 256: - ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst); - break; - default: - GGML_ABORT("fatal error"); - break; - } - return; - } -#endif // !defined(GGML_USE_HIP) - - if (Q->ne[1] <= 32) { - constexpr int cols_per_block = 16; - switch (Q->ne[0]) { - case 64: - ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst); - break; - case 80: - ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, half>(ctx, dst); - break; - case 96: - ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst); - break; - case 112: - ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, half>(ctx, dst); - break; - case 128: - ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst); - break; - case 256: - ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst); - break; - default: - GGML_ABORT("fatal error"); - break; - } - return; - } - - constexpr int cols_per_block = 32; - switch (Q->ne[0]) { - case 64: - ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst); - break; - case 80: - ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, half>(ctx, dst); - break; - case 96: - ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst); - break; - case 112: - ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, half>(ctx, dst); - break; - case 128: - ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst); - break; - case 256: - ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst); - break; - default: - GGML_ABORT("fatal error"); - break; - } -} diff --git a/ggml/src/ggml-cuda/fattn-wmma-f16.cuh b/ggml/src/ggml-cuda/fattn-wmma-f16.cuh deleted file mode 100644 index aaf711a..0000000 --- a/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include "common.cuh" - -#if defined(GGML_USE_MUSA) -#define GGML_USE_WMMA_FATTN -#endif // defined(GGML_USE_MUSA) - -#if defined(GGML_HIP_ROCWMMA_FATTN) -#if defined(CDNA) && (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0) -#define GGML_USE_WMMA_FATTN -#elif defined(CDNA) -#warning "rocwmma fattn on CDNA is broken on rocwmma v2.0.0, expect degraded performance" -#endif // defined(CDNA) && (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0) -#if defined(RDNA3) -#define GGML_USE_WMMA_FATTN -#endif // defined(RDNA3) -#if defined(RDNA4) && ROCWMMA_VERSION_MAJOR > 1 -#define GGML_USE_WMMA_FATTN -#elif defined(RDNA4) -#warning "rocwmma fattn is not supported on RDNA4 on rocwmma < v2.0.0, expect degraded performance" -#endif // defined(RDNA4) && ROCWMMA_VERSION_MAJOR > 1 -#endif // defined(GGML_HIP_ROCWMMA_FATTN) - -// WMMA flash attention requires FP16 matrix instructions to be available for ggml code. -static bool ggml_cuda_should_use_wmma_fattn(const int cc) { -#if defined(GGML_USE_HIP) && !defined(GGML_HIP_ROCWMMA_FATTN) - return false; -#else - if ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_VOLTA) || - GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_MTHREADS(cc)) { - return true; - } else if (GGML_CUDA_CC_IS_CDNA(cc)){ -#if defined(GGML_HIP_ROCWMMA_FATTN) && (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0) - return true; -#else - return false; -#endif // defined(GGML_HIP_ROCWMMA_FATTN) (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0) - } else if (GGML_CUDA_CC_IS_RDNA4(cc)) { -#if defined(GGML_HIP_ROCWMMA_FATTN) && ROCWMMA_VERSION_MAJOR > 1 - return true; -#else - return false; -#endif // defined(GGML_HIP_ROCWMMA_FATTN) && ROCWMMA_VERSION_MAJOR > 1 - } else { - return false; - } -#endif // defined(GGML_USE_HIP) && !defined(GGML_HIP_ROCWMMA_FATTN) -} - -void ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index ea6607c..1515572 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -3,14 +3,170 @@ #include "fattn-mma-f16.cuh" #include "fattn-tile.cuh" #include "fattn-vec.cuh" -#include "fattn-wmma-f16.cuh" #include "fattn.cuh" +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +// one list per group of ncols1 queries: a column is selected if any query of the group can see it +template +__launch_bounds__(256, 1) +static __global__ void flash_attn_mask_to_sparse_indices( + const half * mask_ptr, int32_t * indices_ptr, int32_t * counts_ptr, const int ne30, const int n_queries, + const int n_kv_max, const int64_t s31, const int64_t s33) { + ggml_cuda_pdl_sync(); + + constexpr int values_per_lane = 8; + const int tid = threadIdx.x; + const int warp = tid / WARP_SIZE; + const int lane = tid % WARP_SIZE; + const int sequence = blockIdx.y; + const int group = blockIdx.x; + + const int q0 = group*ncols1; + const int q1 = min(q0 + ncols1, n_queries); + + const half * mask = mask_ptr + sequence*s33 + q0*s31; + int32_t * indices = indices_ptr + (int64_t(sequence)*gridDim.x + group)*n_kv_max; + + __shared__ int warp_offsets[256/WARP_SIZE]; + __shared__ int row_count; + __shared__ int chunk_count; + + if (tid == 0) { + row_count = 0; + } + __syncthreads(); + + for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) { + uint32_t selected_warp[values_per_lane]; + int warp_count = 0; +#pragma unroll + for (int item = 0; item < values_per_lane; ++item) { + const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane; + bool selected = false; + if (i < ne30) { +#pragma unroll + for (int q = 0; q < ncols1; ++q) { + selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i])); + } + } + selected_warp[item] = __ballot_sync(0xFFFFFFFF, selected); + warp_count += __popc(selected_warp[item]); + } + + if (lane == 0) { + warp_offsets[warp] = warp_count; + } + __syncthreads(); + + if (tid == 0) { + int offset = 0; +#pragma unroll + for (int iw = 0; iw < 256/WARP_SIZE; ++iw) { + const int count = warp_offsets[iw]; + warp_offsets[iw] = offset; + offset += count; + } + chunk_count = offset; + } + __syncthreads(); + + const uint32_t lane_mask = lane == 0 ? 0 : (1u << lane) - 1; + int warp_item_offset = 0; +#pragma unroll + for (int item = 0; item < values_per_lane; ++item) { + const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane; + const int dst = row_count + warp_offsets[warp] + warp_item_offset + __popc(selected_warp[item] & lane_mask); + if ((selected_warp[item] & (uint32_t(1) << lane)) && dst < n_kv_max) { + indices[dst] = i; + } + warp_item_offset += __popc(selected_warp[item]); + } + __syncthreads(); + + if (tid == 0) { + row_count += chunk_count; + } + __syncthreads(); + } + + const int count = min(row_count, n_kv_max); + for (int i = count + tid; i < n_kv_max; i += blockDim.x) { + indices[i] = -1; + } + if (tid == 0) { + counts_ptr[int64_t(sequence)*gridDim.x + group] = count; + } + __syncthreads(); + + // the dependent grid reads indices, signal once the row is complete + ggml_cuda_pdl_lc(); +} +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +void ggml_cuda_flash_attn_ext_compact_mask( + const ggml_tensor * mask, int32_t * indices, int32_t * counts, int32_t n_queries, int32_t ncols1, int32_t n_kv_max, cudaStream_t stream) { +#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) + GGML_UNUSED_VARS(mask, indices, counts, n_queries, ncols1, n_kv_max, stream); + GGML_ABORT("sparse flash attention is only supported on NVIDIA CUDA"); +#else + const int64_t s31 = mask->nb[1] / sizeof(half); + const int64_t s33 = mask->nb[3] / sizeof(half); + const dim3 blocks_num((n_queries + ncols1 - 1)/ncols1, mask->ne[3], 1); + const dim3 block_dim(256, 1, 1); + const ggml_cuda_kernel_launch_params launch_params(blocks_num, block_dim, 0, stream); + // the last group of queries is partial only if ncols1 does not divide n_queries + GGML_ASSERT(ncols1 == 1 || ncols1 == 8); + const auto kernel = ncols1 == 1 ? flash_attn_mask_to_sparse_indices<1, false> : + n_queries % 8 != 0 ? flash_attn_mask_to_sparse_indices<8, true> : + flash_attn_mask_to_sparse_indices<8, false>; + ggml_cuda_kernel_launch(kernel, launch_params, + (const half *) mask->data, indices, counts, int(mask->ne[0]), n_queries, n_kv_max, s31, s33); + CUDA_CHECK(cudaGetLastError()); +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +} + +bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_tensor * dst, const int ncols1, const int ncols2) { +#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) + GGML_UNUSED_VARS(cc, dst, ncols1, ncols2); + return false; +#else + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * mask = dst->src[3]; + + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + + const int32_t n_kv_max = ggml_get_op_params_i32(dst, 4); + + // the dense kernel handles up to 64/ncols2 queries per K/V pass, the single-query gather has to beat that + const int64_t n_gather = (ncols1 == 1 ? std::min(Q->ne[1], 64/ncols2) : ncols1) * (int64_t) n_kv_max; + + return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && + mask != nullptr && n_kv_max > 0 && max_bias == 0.0f && logit_softcap == 0.0f && + mask->ne[0] == K->ne[1] && mask->ne[1] >= Q->ne[1] && mask->ne[2] == 1 && + K->ne[1] >= std::max(4096, 2*n_gather); +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +} + template static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const ggml_tensor * Q = dst->src[0]; +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 1, ncols2)) { + // a sparse variant at the full tile width gathers the union of its queries once, prefer it for large batches + constexpr bool has_wide_sparse = ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 64/ncols2, ncols2); + if (!(has_wide_sparse && Q->ne[1] > 32/ncols2) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, ncols2)) { + ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); + return; + } + } +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + if constexpr (ncols2 <= 8) { if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) { ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); @@ -19,13 +175,14 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con } if constexpr (ncols2 <= 16) { - if ((turing_mma_available(cc) || amd_wmma_available(cc)) && Q->ne[1] <= 16/ncols2) { + if (Q->ne[1] <= 16/ncols2) { ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); return; } } - if (ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING || amd_wmma_available(cc) || Q->ne[1] <= 32/ncols2) { + if (Q->ne[1] <= 32/ncols2 || (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING) || + (GGML_CUDA_CC_IS_AMD(cc) && DKQ > 256)) { ggml_cuda_flash_attn_ext_mma_f16_case(ctx, dst); return; } @@ -88,6 +245,24 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con } } + // On RDNA it is preferable to minimize wasted compute vs. duplicate I/O for the mask. + if (amd_wmma_available(cc)) { + if (use_gqa_opt && gqa_ratio % 8 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 4 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + + if (use_gqa_opt && gqa_ratio % 2 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + } + if (use_gqa_opt && gqa_ratio > 4) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); return; @@ -98,12 +273,12 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con return; } - if constexpr (DKQ <= 256) { - if (use_gqa_opt && gqa_ratio > 1) { - ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); - return; - } + if (use_gqa_opt && gqa_ratio > 1) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); + return; + } + if constexpr (DKQ <= 256) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ctx, dst); } else { GGML_ABORT("fatal error"); @@ -139,10 +314,42 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg GGML_ASSERT(V->ne[0] == 128); ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<128, 128>(ctx, dst); break; + case 192: { + // MiMo-V2.5 / V2.5-Pro / V2-Flash: gqa_ratio is 8 (SWA) or 16 (full attn) + GGML_ASSERT(V->ne[0] == 128); + float max_bias = 0.0f; + memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); + const bool use_gqa_opt = mask && max_bias == 0.0f; + GGML_ASSERT(use_gqa_opt); + GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); + const int gqa_ratio = Q->ne[2] / K->ne[2]; + if (gqa_ratio % 16 == 0) { + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<192, 128, 16>(ctx, dst); + } else { + GGML_ASSERT(gqa_ratio % 8 == 0); + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<192, 128, 8>(ctx, dst); + } + } break; case 256: GGML_ASSERT(V->ne[0] == 256); ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<256, 256>(ctx, dst); break; + case 320: + // For Mistral Small 4, go straight to the ncols1 switch (ncols2=32-only build). + GGML_ASSERT(V->ne[0] == 256); + { + float max_bias = 0.0f; + memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); + + const bool use_gqa_opt = mask && max_bias == 0.0f; + GGML_ASSERT(use_gqa_opt); + GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); + const int gqa_ratio = Q->ne[2] / K->ne[2]; + GGML_ASSERT(gqa_ratio % 32 == 0); + + ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<320, 256, 32>(ctx, dst); + } + break; case 512: GGML_ASSERT(V->ne[0] == 512); ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2<512, 512>(ctx, dst); @@ -158,7 +365,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); const int gqa_ratio = Q->ne[2] / K->ne[2]; - if (gqa_ratio == 20) { // GLM 4.7 Flash + if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash if (cc >= GGML_CUDA_CC_DGX_SPARK) { if (Q->ne[1] <= 8) { ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst); @@ -205,105 +412,131 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg } break; default: GGML_ABORT("fatal error"); - break; } } -#define FATTN_VEC_CASE(D, type_K, type_V) \ - { \ - const bool type_K_okay = K->type == (type_K) || (K->type == GGML_TYPE_F32 && (type_K) == GGML_TYPE_F16); \ - const bool type_V_okay = V->type == (type_V) || (V->type == GGML_TYPE_F32 && (type_V) == GGML_TYPE_F16); \ - if (Q->ne[0] == (D) && type_K_okay && type_V_okay) { \ - ggml_cuda_flash_attn_ext_vec_case(ctx, dst); \ - return; \ - } \ - } \ - -#define FATTN_VEC_CASES_ALL_D(type_K, type_V) \ - FATTN_VEC_CASE( 64, type_K, type_V) \ - FATTN_VEC_CASE(128, type_K, type_V) \ - FATTN_VEC_CASE(256, type_K, type_V) \ +#define FATTN_VEC_CASE(D, type_K_case, type_V_case) \ + if constexpr (GGML_CUDA_FA_##type_K_case##_##type_V_case) { \ + if (head_size == (D) && type_K == GGML_TYPE_##type_K_case && type_V == GGML_TYPE_##type_V_case) { \ + return ggml_cuda_flash_attn_ext_vec_case; \ + } \ + } \ + +#define FATTN_VEC_CASES_ALL_D(type_K_case, type_V_case) \ + FATTN_VEC_CASE( 64, type_K_case, type_V_case) \ + FATTN_VEC_CASE(128, type_K_case, type_V_case) \ + FATTN_VEC_CASE(256, type_K_case, type_V_case) \ + +typedef void (* fattn_vec_case_t)(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// Vector kernel for the given head size and K/V types, nullptr if its template instance was not compiled: +static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K_in, const ggml_type type_V_in) { + const ggml_type type_K = type_K_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_K_in; + const ggml_type type_V = type_V_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_V_in; + FATTN_VEC_CASES_ALL_D(F16, F16) + FATTN_VEC_CASES_ALL_D(Q4_0, F16) + FATTN_VEC_CASES_ALL_D(Q4_1, F16) + FATTN_VEC_CASES_ALL_D(Q5_0, F16) + FATTN_VEC_CASES_ALL_D(Q5_1, F16) + FATTN_VEC_CASES_ALL_D(Q8_0, F16) + FATTN_VEC_CASES_ALL_D(BF16, F16) + + FATTN_VEC_CASES_ALL_D(F16, Q4_0) + FATTN_VEC_CASES_ALL_D(Q4_0, Q4_0) + FATTN_VEC_CASES_ALL_D(Q4_1, Q4_0) + FATTN_VEC_CASES_ALL_D(Q5_0, Q4_0) + FATTN_VEC_CASES_ALL_D(Q5_1, Q4_0) + FATTN_VEC_CASES_ALL_D(Q8_0, Q4_0) + FATTN_VEC_CASES_ALL_D(BF16, Q4_0) + + FATTN_VEC_CASES_ALL_D(F16, Q4_1) + FATTN_VEC_CASES_ALL_D(Q4_0, Q4_1) + FATTN_VEC_CASES_ALL_D(Q4_1, Q4_1) + FATTN_VEC_CASES_ALL_D(Q5_0, Q4_1) + FATTN_VEC_CASES_ALL_D(Q5_1, Q4_1) + FATTN_VEC_CASES_ALL_D(Q8_0, Q4_1) + FATTN_VEC_CASES_ALL_D(BF16, Q4_1) + + FATTN_VEC_CASES_ALL_D(F16, Q5_0) + FATTN_VEC_CASES_ALL_D(Q4_0, Q5_0) + FATTN_VEC_CASES_ALL_D(Q4_1, Q5_0) + FATTN_VEC_CASES_ALL_D(Q5_0, Q5_0) + FATTN_VEC_CASES_ALL_D(Q5_1, Q5_0) + FATTN_VEC_CASES_ALL_D(Q8_0, Q5_0) + FATTN_VEC_CASES_ALL_D(BF16, Q5_0) + + FATTN_VEC_CASES_ALL_D(F16, Q5_1) + FATTN_VEC_CASES_ALL_D(Q4_0, Q5_1) + FATTN_VEC_CASES_ALL_D(Q4_1, Q5_1) + FATTN_VEC_CASES_ALL_D(Q5_0, Q5_1) + FATTN_VEC_CASES_ALL_D(Q5_1, Q5_1) + FATTN_VEC_CASES_ALL_D(Q8_0, Q5_1) + FATTN_VEC_CASES_ALL_D(BF16, Q5_1) + + FATTN_VEC_CASES_ALL_D(F16, Q8_0) + FATTN_VEC_CASES_ALL_D(Q4_0, Q8_0) + FATTN_VEC_CASES_ALL_D(Q4_1, Q8_0) + FATTN_VEC_CASES_ALL_D(Q5_0, Q8_0) + FATTN_VEC_CASES_ALL_D(Q5_1, Q8_0) + FATTN_VEC_CASES_ALL_D(Q8_0, Q8_0) + FATTN_VEC_CASES_ALL_D(BF16, Q8_0) + + FATTN_VEC_CASES_ALL_D(F16, BF16) + FATTN_VEC_CASES_ALL_D(Q4_0, BF16) + FATTN_VEC_CASES_ALL_D(Q4_1, BF16) + FATTN_VEC_CASES_ALL_D(Q5_0, BF16) + FATTN_VEC_CASES_ALL_D(Q5_1, BF16) + FATTN_VEC_CASES_ALL_D(Q8_0, BF16) + FATTN_VEC_CASES_ALL_D(BF16, BF16) + + return nullptr; +} static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - ggml_tensor * Q = dst->src[0]; - ggml_tensor * K = dst->src[1]; - ggml_tensor * V = dst->src[2]; - -#ifdef GGML_CUDA_FA_ALL_QUANTS - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_F16) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q4_0) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q4_1) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q5_0) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_1) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q5_1) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_Q8_0) - - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_BF16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_BF16) -#else - FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) - FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_BF16) -#endif // GGML_CUDA_FA_ALL_QUANTS - - GGML_ABORT("fatal error"); + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + fattn_vec_case_t vec_case = ggml_cuda_get_fattn_vec_case(Q->ne[0], K->type, V->type); + if (vec_case == nullptr) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("%s: no FlashAttention vector kernel compiled for K/V types %s-%s, converting K and V to f16 instead (slow). " + "Add \"%s-%s\" to GGML_CUDA_FA_QUANTS to compile it.\n", + __func__, ggml_type_name(K->type), ggml_type_name(V->type), ggml_type_name(K->type), ggml_type_name(V->type)); + warned = true; + } + vec_case = ggml_cuda_get_fattn_vec_case(Q->ne[0], GGML_TYPE_F16, GGML_TYPE_F16); + } + GGML_ASSERT(vec_case != nullptr); + vec_case(ctx, dst); } // Best FlashAttention kernel for a specific GPU: enum best_fattn_kernel { - BEST_FATTN_KERNEL_NONE = 0, - BEST_FATTN_KERNEL_TILE = 200, - BEST_FATTN_KERNEL_VEC = 100, - BEST_FATTN_KERNEL_WMMA_F16 = 300, - BEST_FATTN_KERNEL_MMA_F16 = 400, + BEST_FATTN_KERNEL_NONE = 0, + BEST_FATTN_KERNEL_TILE = 200, + BEST_FATTN_KERNEL_VEC = 100, + BEST_FATTN_KERNEL_MMA_F16 = 400, }; +// K/V types for which there is a vector kernel template instance, other kernels convert these to f16: +static bool ggml_cuda_fattn_kv_type_supported(const ggml_type type) { + switch (type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return true; + default: + return false; + } +} + static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const ggml_tensor * dst) { #ifndef FLASH_ATTN_AVAILABLE GGML_UNUSED(device); GGML_UNUSED(dst); @@ -352,6 +585,22 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_NONE; } break; + case 192: + if (V->ne[0] != 128 || !gqa_opt_applies) { + return BEST_FATTN_KERNEL_NONE; + } + if (gqa_ratio % 8 != 0) { + return BEST_FATTN_KERNEL_NONE; + } + break; + case 320: + if (V->ne[0] != 256 || !gqa_opt_applies) { + return BEST_FATTN_KERNEL_NONE; + } + if (gqa_ratio % 32 != 0) { + return BEST_FATTN_KERNEL_NONE; + } + break; case 512: if (V->ne[0] != K->ne[0]) { return BEST_FATTN_KERNEL_NONE; @@ -372,42 +621,28 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_NONE; } -#ifndef GGML_CUDA_FA_ALL_QUANTS - if (K->type != V->type) { + if (!ggml_cuda_fattn_kv_type_supported(K->type) || !ggml_cuda_fattn_kv_type_supported(V->type)) { return BEST_FATTN_KERNEL_NONE; } -#endif // GGML_CUDA_FA_ALL_QUANTS - - switch (K->type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - break; - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: -#ifndef GGML_CUDA_FA_ALL_QUANTS - return BEST_FATTN_KERNEL_NONE; -#endif // GGML_CUDA_FA_ALL_QUANTS - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q8_0: - case GGML_TYPE_BF16: - break; - default: - return BEST_FATTN_KERNEL_NONE; - } if (mask && mask->ne[2] != 1) { return BEST_FATTN_KERNEL_NONE; } // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes: - const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0; + // 192 satisfies % 64 == 0 but has no vec instance (DKQ != DV); force it onto the MMA path. + const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0; // If Turing tensor cores are available, use them: if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { if (can_use_vector_kernel) { if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { - if (cc >= GGML_CUDA_CC_ADA_LOVELACE && Q->ne[1] == 1 && Q->ne[3] == 1 && !(gqa_ratio > 4 && K->ne[1] >= 8192)) { + // the sparse gather exists only in the MMA kernel: (DKQ, DV, 1, 8) with GQA > 4 + const bool sparse_decode = gqa_opt_applies && gqa_ratio > 4 && + ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(K->ne[0], V->ne[0], 1, 8) && + ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 8); + if (!sparse_decode && cc >= GGML_CUDA_CC_ADA_LOVELACE && Q->ne[1] == 1 && Q->ne[3] == 1 && + !(gqa_ratio > 4 && (Q->ne[0] >= 256 || K->ne[1] >= 8192))) { return BEST_FATTN_KERNEL_VEC; } } else { @@ -428,12 +663,13 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_MMA_F16; } + const int ncols2_max = Q->ne[0] == 320 ? 32 : ((Q->ne[0] == 576 || Q->ne[0] == 192) ? 16 : 8); + int gqa_ratio_eff = 1; + while (max_bias == 0.0f && gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) { + gqa_ratio_eff *= 2; + } + if (volta_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { - int gqa_ratio_eff = 1; - const int ncols2_max = Q->ne[0] == 576 ? 16 : 8; - while (gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) { - gqa_ratio_eff *= 2; - } if (can_use_vector_kernel && Q->ne[1] * gqa_ratio_eff <= 2) { return BEST_FATTN_KERNEL_VEC; } @@ -443,49 +679,26 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_MMA_F16; } - // Use the WMMA kernel if possible: - if (ggml_cuda_should_use_wmma_fattn(cc) && K->ne[1] % FATTN_KQ_STRIDE == 0 && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 512 && Q->ne[0] != 576) { - if (can_use_vector_kernel && Q->ne[1] <= 2) { - return BEST_FATTN_KERNEL_VEC; + // AMD MFMA needs a certain minimum batch size to outscale the tile kernel for large head sizes. + if (amd_mfma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { + if ((Q->ne[0] <= 64 && Q->ne[1] * gqa_ratio_eff > 8)) { + return BEST_FATTN_KERNEL_MMA_F16; } - return BEST_FATTN_KERNEL_WMMA_F16; - } - - if (amd_wmma_available(cc) && GGML_CUDA_CC_IS_RDNA4(cc) && gqa_opt_applies && Q->ne[0] <= 128 && Q->ne[0] != 40 && Q->ne[0] != 72) { - if (can_use_vector_kernel) { - if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { - if (Q->ne[1] == 1) { - if (!gqa_opt_applies) { - return BEST_FATTN_KERNEL_VEC; - } - } - } else { - if (Q->ne[1] <= 2) { - return BEST_FATTN_KERNEL_VEC; - } - } + if ((Q->ne[0] <= 128 && Q->ne[1] * gqa_ratio_eff > 16)) { + return BEST_FATTN_KERNEL_MMA_F16; } - int gqa_ratio_eff = 1; - const int ncols2_max = Q->ne[0] == 576 ? 16 : 8; - while (gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) { - gqa_ratio_eff *= 2; + if ((Q->ne[0] <= 256 && Q->ne[1] * gqa_ratio_eff > 64)) { + return BEST_FATTN_KERNEL_MMA_F16; } - if (Q->ne[1] * gqa_ratio_eff <= 8) { - return BEST_FATTN_KERNEL_TILE; // AMD WMMA is only faster if the full tile width of 16 can be utilized. + if (Q->ne[0] > 256 && gqa_opt_applies && Q->ne[1] * gqa_ratio_eff > 128) { + return BEST_FATTN_KERNEL_MMA_F16; } - return BEST_FATTN_KERNEL_MMA_F16; } - // Use MFMA flash attention for CDNA (MI100+): - if (amd_mfma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 256 && Q->ne[0] != 512 && Q->ne[0] != 576) { - const int64_t eff_nq = Q->ne[1] * (gqa_opt_applies ? gqa_ratio : 1); - // MMA vs tile crossover benchmarked on MI300X @ d32768: - // hsk=64 (gqa=4): MMA wins at eff >= 128 (+11%) - // hsk=128 (gqa=4): MMA wins at eff >= 128 (+4%) - if (eff_nq >= (GGML_CUDA_CC_IS_CDNA1(cc) && Q->ne[0] == 64 ? 64 : 128)) { - return BEST_FATTN_KERNEL_MMA_F16; - } - // Fall through to tile kernel for small effective batch sizes. + // AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized. + if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 && + Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) { + return BEST_FATTN_KERNEL_MMA_F16; } // If there are no tensor cores available, use the generic tile kernel: @@ -505,6 +718,42 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_TILE; } +size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * dst) { + GGML_ASSERT(dst->op == GGML_OP_FLASH_ATTN_EXT); + + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + GGML_ASSERT(K != nullptr); + GGML_ASSERT(V != nullptr); + + const best_fattn_kernel kernel = ggml_cuda_get_best_fattn_kernel(device, dst); + + bool need_f16_K = false; + bool need_f16_V = false; + + switch (kernel) { + case BEST_FATTN_KERNEL_TILE: + case BEST_FATTN_KERNEL_MMA_F16: + need_f16_K = true; + need_f16_V = true; + break; + case BEST_FATTN_KERNEL_VEC: { + const bool f16_fallback = ggml_cuda_get_fattn_vec_case(Q->ne[0], K->type, V->type) == nullptr; + need_f16_K = K->type == GGML_TYPE_F32 || f16_fallback; + need_f16_V = V->type == GGML_TYPE_F32 || f16_fallback; + } break; + case BEST_FATTN_KERNEL_NONE: + break; + } + + const ggml_cuda_flash_attn_ext_f16_extra_data f16_extra = + ggml_cuda_flash_attn_ext_get_f16_extra_data(dst, need_f16_K, need_f16_V); + + return f16_extra.end - (uintptr_t) dst->data; +} + void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ggml_cuda_set_device(ctx.device); switch (ggml_cuda_get_best_fattn_kernel(ggml_cuda_get_device(), dst)) { @@ -516,9 +765,6 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst case BEST_FATTN_KERNEL_VEC: ggml_cuda_flash_attn_ext_vec(ctx, dst); break; - case BEST_FATTN_KERNEL_WMMA_F16: - ggml_cuda_flash_attn_ext_wmma_f16(ctx, dst); - break; case BEST_FATTN_KERNEL_MMA_F16: ggml_cuda_flash_attn_ext_mma_f16(ctx, dst); break; diff --git a/ggml/src/ggml-cuda/fattn.cuh b/ggml/src/ggml-cuda/fattn.cuh index 78705d5..f9a7e15 100644 --- a/ggml/src/ggml-cuda/fattn.cuh +++ b/ggml/src/ggml-cuda/fattn.cuh @@ -3,3 +3,5 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst); bool ggml_cuda_flash_attn_ext_supported(int device, const ggml_tensor * dst); + +size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/fwht.cu b/ggml/src/ggml-cuda/fwht.cu new file mode 100644 index 0000000..78768ea --- /dev/null +++ b/ggml/src/ggml-cuda/fwht.cu @@ -0,0 +1,242 @@ +#include "common.cuh" +#include "convert.cuh" +#include "fwht.cuh" + +// wide FWHT blocks use one row per thread block with this many threads +#define GGML_CUDA_FWHT_BLOCK_NT 256 + +template +__launch_bounds__(4*ggml_cuda_get_physical_warp_size(), 1) +__global__ void fwht_cuda(const T * src, float * dst, const int64_t n_rows, const float scale) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + + const int64_t r = (int64_t) blockIdx.x * blockDim.y + threadIdx.y; + + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + static constexpr int el_w = N / warp_size; + float reg[el_w]; + const int lane = threadIdx.x; + + ggml_cuda_pdl_sync(); +#pragma unroll + for (int i = 0; i < el_w; ++i) { + reg[i] = ggml_cuda_cast(src[i * warp_size + lane]) * scale; + } + +#pragma unroll + for (int h = 1; h < warp_size; h *= 2) { +#pragma unroll + for (int j = 0; j < el_w; j++) { + const float val = reg[j]; + const float val2 = __shfl_xor_sync(0xFFFFFFFF, val, h, warp_size); + + reg[j] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + +#pragma unroll + for (int h = warp_size; h < N; h *= 2) { + const int step = h / warp_size; +#pragma unroll + for (int j = 0; j < el_w; j += 2 * step) { +#pragma unroll + for (int k = 0; k < step; k++) { + const float x = reg[j + k]; + const float y = reg[j + k + step]; + + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + dst[i * warp_size + lane] = reg[i]; + } +} + +// Wide blocks: one row per thread block instead of per warp, so each thread keeps N/NT +// values rather than N/32. Stages below the warp width still shuffle, those up to the +// block width go through shared memory, and the rest stay in registers. +template +__launch_bounds__(NT, 1) +__global__ void fwht_cuda_block(const T * src, float * dst, const int64_t n_rows, const float scale) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int NE = N / NT; + static_assert(NE >= 1 && N % NT == 0 && NT % warp_size == 0, "bad FWHT block shape"); + + __shared__ float s[N]; + + const int64_t r = blockIdx.x; + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + const int tid = threadIdx.x; + const int lane = tid % warp_size; + + ggml_cuda_pdl_sync(); + + float reg[NE]; +#pragma unroll + for (int i = 0; i < NE; ++i) { + reg[i] = ggml_cuda_cast(src[i * NT + tid]) * scale; + } + + // stages within a warp: partner differs in the lane bits +#pragma unroll + for (int h = 1; h < warp_size; h *= 2) { +#pragma unroll + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = __shfl_xor_sync(0xFFFFFFFF, val, h, warp_size); + reg[j] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + + // stages across warps: partner differs in the thread-index bits above the lane +#pragma unroll + for (int h = warp_size; h < NT; h *= 2) { +#pragma unroll + for (int j = 0; j < NE; j++) { + s[j * NT + tid] = reg[j]; + } + __syncthreads(); +#pragma unroll + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = s[j * NT + (tid ^ h)]; + reg[j] = (tid & h) == 0 ? val + val2 : val2 - val; + } + __syncthreads(); + } + + // stages above the block width: partner is another register of the same thread +#pragma unroll + for (int h = NT; h < N; h *= 2) { + const int step = h / NT; +#pragma unroll + for (int j = 0; j < NE; j += 2 * step) { +#pragma unroll + for (int k = 0; k < step; k++) { + const float x = reg[j + k]; + const float y = reg[j + k + step]; + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + +#pragma unroll + for (int i = 0; i < NE; ++i) { + dst[i * NT + tid] = reg[i]; + } +} + +template +static bool ggml_cuda_op_fwht_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { + const int n = src->ne[0]; + const int64_t rows = ggml_nrows(src); + + const T * src_d = (const T *) src->data; + float * dst_d = (float *) dst->data; + + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; + const int rows_per_block = 4; + + const int64_t num_blocks = (rows + rows_per_block - 1) / rows_per_block; + + cudaStream_t stream = ctx.stream(); + dim3 grid_dims(num_blocks, 1, 1); + dim3 block_dims(warp_size, rows_per_block, 1); + const ggml_cuda_kernel_launch_params launch_params = + ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); + + const float scale = 1 / sqrtf(n); + + switch (n) { + case 64: + ggml_cuda_kernel_launch(fwht_cuda<64, T>, launch_params, src_d, dst_d, rows, scale); + return true; + case 128: + ggml_cuda_kernel_launch(fwht_cuda<128, T>, launch_params, src_d, dst_d, rows, scale); + return true; + case 256: + ggml_cuda_kernel_launch(fwht_cuda<256, T>, launch_params, src_d, dst_d, rows, scale); + return true; + case 512: + ggml_cuda_kernel_launch(fwht_cuda<512, T>, launch_params, src_d, dst_d, rows, scale); + return true; + default: + break; + } + + // wide blocks: one row per thread block + { + constexpr int nt = GGML_CUDA_FWHT_BLOCK_NT; + + dim3 grid_dims_w(rows, 1, 1); + dim3 block_dims_w(nt, 1, 1); + const ggml_cuda_kernel_launch_params launch_params_w = + ggml_cuda_kernel_launch_params(grid_dims_w, block_dims_w, 0, stream); + + switch (n) { + case 1024: + ggml_cuda_kernel_launch(fwht_cuda_block<1024, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; + case 2048: + ggml_cuda_kernel_launch(fwht_cuda_block<2048, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; + case 4096: + ggml_cuda_kernel_launch(fwht_cuda_block<4096, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; +#if !defined(GGML_USE_MUSA) + // 32 KB of shared memory, above the MUSA limit; falls back there + case 8192: + ggml_cuda_kernel_launch(fwht_cuda_block<8192, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; +#endif // !defined(GGML_USE_MUSA) + default: + return false; + } + } +} + +bool ggml_cuda_op_mul_mat_use_fwht(const struct ggml_tensor * op) { + const struct ggml_tensor * a = op->src[0]; + const struct ggml_tensor * b = op->src[1]; + + return op->op == GGML_OP_MUL_MAT && ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD && + a->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + (b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16) && ggml_is_contiguous(b) && ggml_is_contiguous(op) && + ggml_are_same_shape(b, op); +} + +bool ggml_cuda_op_fwht(ggml_backend_cuda_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { + GGML_ASSERT(ggml_are_same_shape(src, dst)); + if (!ggml_is_contiguous(src) || !ggml_is_contiguous(dst)) { + return false; + } + if (dst->type != GGML_TYPE_F32) { + return false; + } + + switch (src->type) { + case GGML_TYPE_F32: + return ggml_cuda_op_fwht_impl(ctx, src, dst); + case GGML_TYPE_F16: + return ggml_cuda_op_fwht_impl(ctx, src, dst); + default: + return false; + } +} diff --git a/ggml/src/ggml-cuda/fwht.cuh b/ggml/src/ggml-cuda/fwht.cuh new file mode 100644 index 0000000..e7ad534 --- /dev/null +++ b/ggml/src/ggml-cuda/fwht.cuh @@ -0,0 +1,6 @@ +#include "common.cuh" + +bool ggml_cuda_op_mul_mat_use_fwht(const struct ggml_tensor * op); + +// Returns whether the Fast Walsh-Hadamard transform could be used. +bool ggml_cuda_op_fwht(ggml_backend_cuda_context & ctx, const ggml_tensor * src, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/gated-delta-net-mma.cu b/ggml/src/ggml-cuda/gated-delta-net-mma.cu new file mode 100644 index 0000000..0684adc --- /dev/null +++ b/ggml/src/ggml-cuda/gated-delta-net-mma.cu @@ -0,0 +1,443 @@ +#include "gated-delta-net-mma.cuh" +#include "mma.cuh" + +#include +#include + +namespace ggml_cuda_gated_delta_net_mma { +constexpr int D = 128; +using bf16 = nv_bfloat16; + +template struct matrix { + bf16 hi[R * C], lo[R * C]; + + static __device__ __forceinline__ int index(int r, int c) { +#ifdef GGML_USE_HIP + return r * C + c; +#else + return 2 * ggml_cuda_mma::swizzle(r * (C / 2) + c / 2, r) + c % 2; +#endif // GGML_USE_HIP + } + + __device__ __forceinline__ void store2(int r, int c, float2 x) { +#ifdef GGML_USE_MUSA + store(r, c, x.x); + store(r, c + 1, x.y); +#else + const int i = index(r, c); + const auto h = __float22bfloat162_rn(x); + const float2 rounded = __bfloat1622float2(h); + *reinterpret_cast(hi + i) = h; + *reinterpret_cast(lo + i) = + __float22bfloat162_rn(make_float2(x.x - rounded.x, x.y - rounded.y)); +#endif // GGML_USE_MUSA + } + + __device__ __forceinline__ void store(int r, int c, float x) { + const int i = index(r, c); + const bf16 h = __float2bfloat16(x); + hi[i] = h; + lo[i] = __float2bfloat16(x - __bfloat162float(h)); + } +}; + +template struct alignas(128) shared { + matrix q, k; + matrix p, inverse; + matrix delta; + + union { + matrix state; + matrix solved; + } scratch; + + float lower[C * C], prefix[C], beta[C]; + + static __device__ __forceinline__ int lower_index(int r, int c) { return r * C + c; } +}; + +#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) +using namespace ggml_cuda_mma; +#ifdef GGML_USE_HIP +using acc = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>; +using a_tile = tile<16, 8, nv_bfloat162, DATA_LAYOUT_I_MAJOR_MIRRORED>; +using b_tile = a_tile; +#else +using acc = tile<16, 8, float>; +using a_tile = tile<16, 8, nv_bfloat162>; +using b_tile = tile<8, 8, nv_bfloat162>; +#endif // GGML_USE_HIP + +template +__device__ __forceinline__ tile_t load(const matrix & m, int row, int col, bool low) { + tile_t t; + const bf16 * data = low ? m.lo : m.hi; +#ifdef GGML_USE_HIP +#pragma unroll + for (int i = 0; i < tile_t::ne; ++i) { + const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i); + const bf16 x = data[TRANS ? m.index(c, r) : m.index(r, c)]; + const bf16 y = data[TRANS ? m.index(c + 1, r) : m.index(r, c + 1)]; + t.x[i] = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y))); + } +#else + const auto * packed = reinterpret_cast(data); + if constexpr (TRANS) { + load_ldmatrix_trans_swizzled(t, packed, col * (C / 2) + row / 2); + } else { + load_ldmatrix_swizzled(t, packed, row * (C / 2) + col / 2); + } +#endif // GGML_USE_HIP + return t; +} + +// B is addressed as B-transpose. Preserve both BF16 first-order residual terms. +template +__device__ __forceinline__ void product(acc & c, const matrix & a, const matrix & b, int row, int col) { +#pragma unroll + for (int k = 0; k < K; k += 16) { + const auto ah = load(a, row, k, false); + const auto al = load(a, row, k, true); + const auto bh = load(b, col, k, false); + const auto bl = load(b, col, k, true); + mma(c, al, bh); + mma(c, ah, bl); + mma(c, ah, bh); + } +} + +template __device__ __forceinline__ void store(matrix & m, const acc & x, int r, int c) { +#pragma unroll +#ifdef GGML_USE_HIP + for (int i = 0; i < acc::ne; ++i) { + m.store(r + acc::get_i(i), c + acc::get_j(i), x.x[i]); + } +#else + for (int i = 0; i < acc::ne; i += 2) { + m.store2(r + acc::get_i(i), c + acc::get_j(i), make_float2(x.x[i], x.x[i + 1])); + } +#endif // GGML_USE_HIP +} +#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) + +// Each block owns V independent value columns and advances by C tokens. +// Keep recurrent state in FP32; pack high/residual BF16 operands inside the block. +template +static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_mma_args a) { +#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) + static_assert(ggml_cuda_get_physical_warp_size() == 32, "GDN MMA requires 32-lane warps"); + constexpr int WARPS = 8; +#ifdef GGML_USE_HIP + constexpr int N = 16; +#else + constexpr int N = 8; +#endif // GGML_USE_HIP + static_assert(C >= 16 && (C & (C - 1)) == 0 && V % N == 0 && D % V == 0, "invalid GDN tile"); + extern __shared__ __align__(128) unsigned char bytes[]; + auto & s = *reinterpret_cast *>(bytes); + const int lane = threadIdx.x, warp = threadIdx.y, tid = warp * 32 + lane; + const int slice = blockIdx.x % (D / V); + const int h = (blockIdx.x / (D / V)) % a.H; + const int seq = blockIdx.x / ((D / V) * a.H); + const int v0 = slice * V; + const int64_t soff = ((int64_t) seq * a.H + h) * D * D; + const int64_t qoff = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1; + const int64_t voff = seq * a.sv3 + h * a.sv1 + v0; + const int64_t goff = seq * a.sb3 + h * a.sb1; + float * out = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0; + constexpr int STATE_TILES = (D / 16) * (V / N); + constexpr int PER_WARP = STATE_TILES / WARPS; + static_assert(STATE_TILES % WARPS == 0, "state must divide across warps"); + acc state[PER_WARP]; +#pragma unroll + for (int j = 0; j < PER_WARP; ++j) { + const int tile_index = warp + j * WARPS, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N; +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + state[j].x[i] = a.state[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)]; + } + } + auto read_qk = [&](int64_t t, float4 & q, float4 & k) { + t += warp; + q = k = make_float4(0, 0, 0, 0); + if (t < a.n_tokens) { + const int64_t off = qoff + t * a.sq2 + 4 * lane; + if ((((uintptr_t) (a.q + off) | (uintptr_t) (a.k + off)) & 15) == 0) { + q = *reinterpret_cast(a.q + off); + k = *reinterpret_cast(a.k + off); + } else { + q = make_float4(a.q[off], a.q[off + 1], a.q[off + 2], a.q[off + 3]); + k = make_float4(a.k[off], a.k[off + 1], a.k[off + 2], a.k[off + 3]); + } + } + }; + // Carry the first eight input rows across chunks to hide global-load latency. + constexpr bool PREFETCH = C == 16 && V == 32; + float4 next_q, next_k; + if constexpr (PREFETCH) { + read_qk(0, next_q, next_k); + } + for (int64_t t0 = 0; t0 < a.n_tokens; t0 += C) { + const int valid = min((int64_t) C, a.n_tokens - t0); +#pragma unroll + for (int j = 0; j < C / 8; ++j) { + const int t = warp + 8 * j, d = 4 * lane; + float4 q, k; + if (PREFETCH && j == 0) { + q = next_q; + k = next_k; + } else { + read_qk(t0 + 8 * j, q, k); + } + s.q.store2(t, d, make_float2(q.x * a.scale, q.y * a.scale)); + s.q.store2(t, d + 2, make_float2(q.z * a.scale, q.w * a.scale)); + s.k.store2(t, d, make_float2(k.x, k.y)); + s.k.store2(t, d + 2, make_float2(k.z, k.w)); + } + // Load gates in parallel before the FP32 prefix scan. + if (warp == 0) { + float carry = 0.f; + for (int base = 0; base < C; base += 32) { + const int t = base + lane; + float g = t < valid ? a.g[goff + (t0 + t) * a.sb2] : 0.f; + const float beta = t < valid ? a.beta[goff + (t0 + t) * a.sb2] : 0.f; +#pragma unroll + for (int offset = 1; offset < 32; offset *= 2) { + const float previous = __shfl_up_sync(0xffffffff, g, offset, 32); + if (lane >= offset) { + g += previous; + } + } + g += carry; + if (t < C) { + s.prefix[t] = g; + s.beta[t] = beta; + } + carry = __shfl_sync(0xffffffff, g, 31, 32); + } + } +#pragma unroll + for (int j = 0; j < PER_WARP; ++j) { + const int ti = warp + j * WARPS; + store(s.scratch.state, state[j], (ti / (V / N)) * 16, (ti % (V / N)) * N); + } + __syncthreads(); + if constexpr (PREFETCH) { + if (t0 + C < a.n_tokens) { + read_qk(t0 + C, next_q, next_k); + } + } + // V32 overlaps Gram work with one output tile per projection warp. + constexpr bool SPLIT_PROJECTION = C == 16 && V == 32; + acc base_output[((C / 16) * (V / N) + WARPS - 1) / WARPS]; + // delta = beta * (v - decay * K S), base output = decay * Q S + auto project = [&]() { + if constexpr (V == 32) { + if (warp < 4) { + return; + } + } + for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : WARPS)) { + const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; + acc qs, ks; + product(qs, s.q, s.scratch.state, r, c); + product(ks, s.k, s.scratch.state, r, c); +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + const int t = r + acc::get_i(i), v = c + acc::get_j(i); + const float decay = expf(s.prefix[t]); + const float vv = t < valid ? a.v[voff + (t0 + t) * a.sv2 + v] : 0.f; + s.delta.store(t, v, s.beta[t] * (vv - decay * ks.x[i])); + base_output[ti / WARPS].x[i] = decay * qs.x[i]; + } + } + }; + // A = beta * strict_tril(K K^T * decay), P = tril(Q K^T * decay) + for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : WARPS)) { + const bool gram = ti < (C / 16) * (C / N); + const int t = ti % ((C / 16) * (C / N)), r = (t / (C / N)) * 16, c = (t % (C / N)) * N; + acc x; + if (gram) { + product(x, s.k, s.k, r, c); + } else { + product(x, s.q, s.k, r, c); + } +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + const int rr = r + acc::get_i(i), cc = c + acc::get_j(i); + const float v = rr < valid && cc <= rr ? x.x[i] * expf(s.prefix[rr] - s.prefix[cc]) : 0.f; + if (gram) { + if (cc < rr) { + s.lower[s.lower_index(rr, cc)] = v * s.beta[rr]; + } + } else { + s.p.store(rr, cc, v); + } + } + } + if constexpr (V == 32) { + project(); + } + __syncthreads(); + // inverse = (I + A)^-1 by forward substitution + for (int row = warp; row < C; row += WARPS) { + float x[(C + 31) / 32]; +#pragma unroll + for (int j = 0; j < (C + 31) / 32; ++j) { + const int col = lane + 32 * j; + x[j] = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f); + } + for (int pivot = row - 1; pivot > 0; --pivot) { + const float xp = __shfl_sync(0xffffffff, x[pivot / 32], pivot % 32, 32); +#pragma unroll + for (int j = 0; j < (C + 31) / 32; ++j) { + const int col = lane + 32 * j; + if (col < pivot) { + x[j] -= xp * s.lower[s.lower_index(pivot, col)]; + } + } + } +#pragma unroll + for (int j = 0; j < (C + 31) / 32; ++j) { + if (lane + 32 * j < C) { + s.inverse.store(row, lane + 32 * j, x[j]); + } + } + } + if constexpr (V != 32) { + project(); + } + __syncthreads(); + // solved = inverse * delta + for (int ti = warp; ti < (C / 16) * (V / N); ti += WARPS) { + const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; + acc x; + product(x, s.inverse, s.delta, r, c); + store(s.scratch.solved, x, r, c); + } + __syncthreads(); + // out = base output + P * solved + for (int ti = (SPLIT_PROJECTION ? warp - 4 : warp); ti < (C / 16) * (V / N) && (!SPLIT_PROJECTION || warp >= 4); + ti += WARPS) { + const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; + acc x; + product(x, s.p, s.scratch.solved, r, c); +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + const int t = r + acc::get_i(i), v = c + acc::get_j(i); + if (t < valid) { + out[(t0 + t) * a.H * D + v] = base_output[ti / WARPS].x[i] + x.x[i]; + } + } + } + // S = decay(chunk) * S + K^T * (solved * decay to chunk end) + for (int i = tid; i < C * V; i += 256) { + const int t = i / V, v = i % V, j = s.scratch.solved.index(t, v); + const float x = __bfloat162float(s.scratch.solved.hi[j]) + __bfloat162float(s.scratch.solved.lo[j]); + s.delta.store(t, v, x * expf(s.prefix[valid - 1] - s.prefix[t])); + } + __syncthreads(); + const float decay = expf(s.prefix[valid - 1]); +#pragma unroll + for (int j = 0; j < PER_WARP; ++j) { + const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + state[j].x[i] *= decay; + } + product(state[j], s.k, s.delta, r, c); + } + __syncthreads(); + } +#pragma unroll + for (int j = 0; j < PER_WARP; ++j) { + const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N; +#pragma unroll + for (int i = 0; i < acc::ne; ++i) { + a.state_out[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)] = state[j].x[i]; + } + } +#else + GGML_UNUSED_VARS(a); + NO_DEVICE_CODE; +#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) +} + +template bool init(int device) { + static std::once_flag once[GGML_CUDA_MAX_DEVICES]; + static bool available[GGML_CUDA_MAX_DEVICES] = {}; + std::call_once(once[device], [device] { + ggml_cuda_set_device(device); + if (sizeof(shared) > ggml_cuda_info().devices[device].smpbo) { + return; + } + const auto status = cudaFuncSetAttribute((const void *) gdn_single, + cudaFuncAttributeMaxDynamicSharedMemorySize, sizeof(shared)); + available[device] = status == cudaSuccess; + if (status != cudaSuccess) { + (void) cudaGetLastError(); + } + }); + return available[device]; +} + +enum class gdn_path { ar, slice32_one_block, slice32_two_blocks, slice64, whole_head }; + +static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a) { + GGML_ASSERT(device >= 0 && device < GGML_CUDA_MAX_DEVICES); + if (!a.eligible || a.H_k != 16 || (a.H != 16 && a.H != 32 && a.H != 48 && a.H != 64) || a.n_tokens < 64 || + a.n_tokens > INT64_MAX - 32 || a.rq3 <= 0 || a.n_seqs <= 0 || a.n_seqs > INT_MAX / (a.H * 4)) { + return gdn_path::ar; + } + const auto & info = ggml_cuda_info().devices[device]; +#ifdef GGML_USE_HIP + if (!GGML_CUDA_CC_IS_RDNA3_5(info.cc) || a.n_tokens < 2048) { + return gdn_path::ar; + } + return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar; +#elif defined(GGML_USE_MUSA) + GGML_UNUSED_VARS(info); + return gdn_path::ar; +#else + // Enable only architectures benchmarked for this schedule. MMA support alone does not imply a speedup. + if (!ampere_mma_available(info.cc) || + (info.cc != GGML_CUDA_CC_AMPERE_GA10X && info.cc != GGML_CUDA_CC_ADA_LOVELACE && + info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_DGX_SPARK)) { + return gdn_path::ar; + } + // Value splitting pays off when whole heads leave at least half the SMs idle. + if (a.H * a.n_seqs * 2 <= info.nsm) { + // A grid that already fits in one wave benefits from a larger register budget. + if (a.H * a.n_seqs * 4 <= info.nsm) { + return init<16, 32, 1>(device) ? gdn_path::slice32_one_block : gdn_path::ar; + } + return init<16, 32, 2>(device) ? gdn_path::slice32_two_blocks : gdn_path::ar; + } + return init<16, 128>(device) ? gdn_path::whole_head : gdn_path::ar; +#endif // GGML_USE_HIP +} +} // namespace +using namespace ggml_cuda_gated_delta_net_mma; + +bool ggml_cuda_gdn_mma_launch(int device, const ggml_cuda_gdn_mma_args & a, cudaStream_t stream) { + if (!a.state_out) { + return false; + } + const auto path = select_gdn_mma_path(device, a); + if (path == gdn_path::ar) { + return false; + } +#ifdef GGML_USE_HIP + gdn_single<16, 64><<), stream>>>(a); +#else + if (path == gdn_path::slice32_two_blocks) { + gdn_single<16, 32, 2><<), stream>>>(a); + } else if (path == gdn_path::slice32_one_block) { + gdn_single<16, 32, 1><<), stream>>>(a); + } else { + gdn_single<16, 128><<), stream>>>(a); + } +#endif // GGML_USE_HIP + CUDA_CHECK(cudaGetLastError()); + return true; +} diff --git a/ggml/src/ggml-cuda/gated-delta-net-mma.cuh b/ggml/src/ggml-cuda/gated-delta-net-mma.cuh new file mode 100644 index 0000000..8eb7c4f --- /dev/null +++ b/ggml/src/ggml-cuda/gated-delta-net-mma.cuh @@ -0,0 +1,35 @@ +#pragma once + +#include "common.cuh" + +#include + +struct ggml_cuda_gdn_mma_args { + const float * q; + const float * k; + const float * v; + const float * g; + const float * beta; + const float * state; + float * dst; + float * state_out; + + int64_t H; + int64_t H_k; + int64_t n_tokens; + int64_t n_seqs; + int64_t rq3; + int64_t sq1; + int64_t sq2; + int64_t sq3; + int64_t sv1; + int64_t sv2; + int64_t sv3; + int64_t sb1; + int64_t sb2; + int64_t sb3; + float scale; + bool eligible; +}; + +bool ggml_cuda_gdn_mma_launch(int device, const ggml_cuda_gdn_mma_args & args, cudaStream_t stream); diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index 6b44bec..ae2374c 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -1,6 +1,12 @@ #include "gated_delta_net.cuh" +#include "gated-delta-net-mma.cuh" +#include "ggml-cuda/common.cuh" -template +#include + +constexpr int gdn_cols_per_warp = 4; + +template __global__ void __launch_bounds__((ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v) * 4, 2) gated_delta_net_cuda(const float * q, const float * k, @@ -9,6 +15,7 @@ gated_delta_net_cuda(const float * q, const float * beta, const float * curr_state, float * dst, + float * state, int64_t H, int64_t n_tokens, int64_t n_seqs, @@ -23,34 +30,45 @@ gated_delta_net_cuda(const float * q, int64_t sb3, const uint3 neqk1_magic, const uint3 rq3_magic, - float scale) { + float scale, + int64_t state_slot_stride, + int K) { const uint32_t h_idx = blockIdx.x; const uint32_t sequence = blockIdx.y; - // each warp owns one column, using warp-level primitives to reduce across rows - const int lane = threadIdx.x; - const int col = blockIdx.z * blockDim.y + threadIdx.y; + + constexpr int warp_size = ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v; + static_assert(S_v % warp_size == 0, "S_v must be a multiple of warp_size"); + // the warp is split into cols_per_warp segments of lanes_per_col lanes; each segment owns + // one state column and reduces within itself + constexpr int lanes_per_col = warp_size / cols_per_warp; + constexpr int rows_per_lane = S_v / lanes_per_col; + static_assert(S_v % lanes_per_col == 0, "S_v must be a multiple of lanes_per_col"); + + const int lane = threadIdx.x; + const int col_in_warp = lane / lanes_per_col; // column slot within the warp + const int lane_in_col = lane - col_in_warp * lanes_per_col; // lane within the column's reduction segment + const int col = (blockIdx.z * blockDim.y + threadIdx.y) * cols_per_warp + col_in_warp; const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic); const uint32_t iq3 = fastdiv(sequence, rq3_magic); - const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs; float * attn_data = dst; - float * state = dst + attn_score_elems; - const int64_t state_offset = (sequence * H + h_idx) * S_v * S_v; - state += state_offset; - curr_state += state_offset + col * S_v; + // input state holds s0 only: [S_v, S_v, H, n_seqs] — seq stride is D = H * S_v * S_v. + // output state layout (per-slot D * n_seqs) — same per-(seq,head) offset as before. + const int64_t state_in_offset = sequence * H * S_v * S_v + h_idx * S_v * S_v; + const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v; + state += state_out_offset; + curr_state += state_in_offset + col * S_v; attn_data += (sequence * n_tokens * H + h_idx) * S_v; - constexpr int warp_size = ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v; - static_assert(S_v % warp_size == 0, "S_v must be a multiple of warp_size"); - constexpr int rows_per_lane = (S_v + warp_size - 1) / warp_size; float s_shard[rows_per_lane]; // state is stored transposed: M[col][i] = S[i][col], row col is contiguous + ggml_cuda_pdl_sync(); #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; s_shard[r] = curr_state[i]; } @@ -70,7 +88,7 @@ gated_delta_net_cuda(const float * q, float q_reg[rows_per_lane]; #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; k_reg[r] = k_t[i]; q_reg[r] = q_t[i]; } @@ -84,7 +102,7 @@ gated_delta_net_cuda(const float * q, for (int r = 0; r < rows_per_lane; r++) { kv_shard += s_shard[r] * k_reg[r]; } - float kv_col = warp_reduce_sum(kv_shard); + float kv_col = warp_reduce_sum(kv_shard); // delta[col] = (v[col] - g * kv[col]) * beta float delta_col = (v_t[col] - g_val * kv_col) * beta_val; @@ -98,9 +116,9 @@ gated_delta_net_cuda(const float * q, attn_partial += s_shard[r] * q_reg[r]; } - float attn_col = warp_reduce_sum(attn_partial); + float attn_col = warp_reduce_sum(attn_partial); - if (lane == 0) { + if (lane_in_col == 0) { attn_data[col] = attn_col * scale; } } else { @@ -108,11 +126,11 @@ gated_delta_net_cuda(const float * q, float kv_shard = 0.0f; #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; kv_shard += expf(g_t[i]) * s_shard[r] * k_reg[r]; } - float kv_col = warp_reduce_sum(kv_shard); + float kv_col = warp_reduce_sum(kv_shard); // delta[col] = (v[col] - kv[col]) * beta float delta_col = (v_t[col] - kv_col) * beta_val; @@ -122,85 +140,145 @@ gated_delta_net_cuda(const float * q, float attn_partial = 0.0f; #pragma unroll for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; + const int i = r * lanes_per_col + lane_in_col; s_shard[r] = expf(g_t[i]) * s_shard[r] + k_reg[r] * delta_col; attn_partial += s_shard[r] * q_reg[r]; } - float attn_col = warp_reduce_sum(attn_partial); + float attn_col = warp_reduce_sum(attn_partial); - if (lane == 0) { + if (lane_in_col == 0) { attn_data[col] = attn_col * scale; } } attn_data += S_v * H; + + if constexpr (keep_rs_t) { + // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. + // When n_tokens < K only slots 0..n_tokens-1 are written; older slots are caller-owned. + const int target_slot = (int) n_tokens - 1 - t; + if (target_slot >= 0 && target_slot < K) { + float * curr_state = state + target_slot * state_slot_stride; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * lanes_per_col + lane_in_col; + curr_state[col * S_v + i] = s_shard[r]; + } + } + } } - // Write state back to global memory (transposed layout) + if constexpr (!keep_rs_t) { #pragma unroll - for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; - state[col * S_v + i] = s_shard[r]; + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * lanes_per_col + lane_in_col; + state[col * S_v + i] = s_shard[r]; + } } } -template +template static void launch_gated_delta_net( const float * q_d, const float * k_d, const float * v_d, const float * g_d, const float * b_d, const float * s_d, - float * dst_d, + float * dst_d, float * state_d, int64_t S_v, int64_t H, int64_t n_tokens, int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, int64_t sv2, int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, int64_t rq3, - float scale, cudaStream_t stream) { + float scale, int64_t state_slot_stride, int K, cudaStream_t stream) { //TODO: Add chunked kernel for even faster pre-fill const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; - const int num_warps = 4; - dim3 grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps); + // four columns per warp (see the kernel); shrink the CTA when the wider CTA would leave + // SMs without a CTA, so small head counts keep the device filled + const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; + const int cols_per_warp = gdn_cols_per_warp; + int num_warps = 4; + while (num_warps > 1 && H*n_seqs*(S_v / (cols_per_warp * num_warps)) < nsm) { + num_warps /= 2; + } + // one CTA covers cols_per_warp*num_warps columns (see the kernel) + dim3 grid_dims(H, n_seqs, (S_v + cols_per_warp * num_warps - 1) / (cols_per_warp * num_warps)); dim3 block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1); const uint3 neqk1_magic = init_fastdiv_values(neqk1); const uint3 rq3_magic = init_fastdiv_values(rq3); - int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); switch (S_v) { case 16: - gated_delta_net_cuda<16, KDA><<>>( - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + ggml_cuda_kernel_launch(gated_delta_net_cuda<16, KDA, keep_rs_t>, launch_params, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; case 32: - gated_delta_net_cuda<32, KDA><<>>( - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + ggml_cuda_kernel_launch(gated_delta_net_cuda<32, KDA, keep_rs_t>, launch_params, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; case 64: { - gated_delta_net_cuda<64, KDA><<>>( - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + ggml_cuda_kernel_launch(gated_delta_net_cuda<64, KDA, keep_rs_t>, launch_params, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; } case 128: { - gated_delta_net_cuda<128, KDA><<>>( - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, + ggml_cuda_kernel_launch(gated_delta_net_cuda<128, KDA, keep_rs_t>, launch_params, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1_magic, rq3_magic, scale); + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); break; } default: GGML_ABORT("fatal error"); - break; } } -void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { +static ggml_cuda_gdn_mma_args ggml_cuda_gdn_mma_args_from_tensor(const ggml_tensor * dst) { + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * g = dst->src[3]; + const ggml_tensor * beta = dst->src[4]; + const ggml_tensor * state = dst->src[5]; + + const int64_t S_v = v->ne[0]; + const int64_t H = v->ne[1]; + return { + (const float *) q->data, + (const float *) k->data, + (const float *) v->data, + (const float *) g->data, + (const float *) beta->data, + (const float *) state->data, + (float *) dst->data, + nullptr, + H, + q->ne[1], + v->ne[2], + v->ne[3], + v->ne[3] / q->ne[3], + (int64_t) (q->nb[1] / sizeof(float)), + (int64_t) (q->nb[2] / sizeof(float)), + (int64_t) (q->nb[3] / sizeof(float)), + (int64_t) (v->nb[1] / sizeof(float)), + (int64_t) (v->nb[2] / sizeof(float)), + (int64_t) (v->nb[3] / sizeof(float)), + (int64_t) (beta->nb[1] / sizeof(float)), + (int64_t) (beta->nb[2] / sizeof(float)), + (int64_t) (beta->nb[3] / sizeof(float)), + 1.0f / sqrtf((float) S_v), + g->ne[0] == 1 && ggml_get_op_params_i32(dst, 0) < v->ne[2] && S_v == 128 && q->ne[0] == 128, + }; +} + +static void ggml_cuda_op_gated_delta_net_impl( + ggml_backend_cuda_context & ctx, ggml_tensor * dst, const ggml_cuda_gated_delta_net_fused_cache * cache) { ggml_tensor * src_q = dst->src[0]; ggml_tensor * src_k = dst->src[1]; ggml_tensor * src_v = dst->src[2]; @@ -226,6 +304,8 @@ void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_ASSERT(neq1 == nek1); const int64_t neqk1 = neq1; + GGML_ASSERT(neq3 > 0 && nev3 >= neq3 && nev3 % neq3 == 0); + GGML_ASSERT(nek3 == neq3); const int64_t rq3 = nev3 / neq3; const float * q_d = (const float *) src_q->data; @@ -261,13 +341,53 @@ void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * cudaStream_t stream = ctx.stream(); + // K (snapshot slot count) is an op param; state holds s0 only [S_v, S_v, H, n_seqs]. + const int K = ggml_get_op_params_i32(dst, 0); + const bool keep_rs = K > 1; + + // recurrent state -> gdn_out tail (after attention scores), or the cache when fusing + float * state_d = dst_d + S_v * H * n_tokens * n_seqs; + int64_t state_slot_stride = S_v * S_v * H * n_seqs; + if (cache != nullptr) { + state_d = cache->data; + state_slot_stride = cache->slot_stride; + } + + { + ggml_cuda_gdn_mma_args args = ggml_cuda_gdn_mma_args_from_tensor(dst); + args.state_out = state_d; + if (ggml_cuda_gdn_mma_launch(ctx.device, args, stream)) { + return; + } + } if (kda) { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, - S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, stream); + if (keep_rs) { + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, + S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + } else { + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, + S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + } } else { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, - S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, stream); + if (keep_rs) { + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, + S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + } else { + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, + S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + } } } + +void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + ggml_cuda_op_gated_delta_net_impl(ctx, dst, nullptr); +} + +void ggml_cuda_op_gated_delta_net_fused_cache( + ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_cuda_gated_delta_net_fused_cache cache) { + ggml_cuda_op_gated_delta_net_impl(ctx, dst, &cache); +} diff --git a/ggml/src/ggml-cuda/gated_delta_net.cuh b/ggml/src/ggml-cuda/gated_delta_net.cuh index 7375e81..f9bf437 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cuh +++ b/ggml/src/ggml-cuda/gated_delta_net.cuh @@ -1,4 +1,14 @@ #include "common.cuh" #include "ggml.h" +// fused-kernel recurrent-state output; strides in elements (per-seq stride is always D, set in-kernel) +struct ggml_cuda_gated_delta_net_fused_cache { + float * data; // rollback slot 0 + int64_t slot_stride; // between rollback slots (0 when K==1) +}; + void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// same op, but writes the snapshot(s) into the cache instead of dst (see ggml_cuda_try_gdn_cache_fusion) +void ggml_cuda_op_gated_delta_net_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + ggml_cuda_gated_delta_net_fused_cache cache); diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu index e99cba6..d97e8fd 100644 --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -6,17 +6,19 @@ template static __global__ void k_get_rows( const void * __restrict__ src0, const int32_t * __restrict__ src1, dst_t * __restrict__ dst, const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/ - /*const int64_t ne10,*/ const int64_t ne11, const int64_t ne12, /*const int64_t ne13,*/ + /*const int64_t ne10,*/ const int64_t ne11, const uint3 ne12_fdv, /*const int64_t ne13,*/ /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3, /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03, const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) { - for (int64_t z = blockIdx.z; z < ne11*ne12; z += gridDim.z) { + ggml_cuda_pdl_sync(); + for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) { for (int64_t i00 = 2*(blockIdx.y*blockDim.x + threadIdx.x); i00 < ne00; i00 += gridDim.y*blockDim.x) { // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher. const int i10 = blockIdx.x; - const int i11 = z / ne12; // TODO fastdiv - const int i12 = z % ne12; + const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv); + const int i11 = dm.x; + const int i12 = dm.y; const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; @@ -38,57 +40,120 @@ static __global__ void k_get_rows( } } +template dequantize_kq> +static __global__ void k_get_rows_kq( + const void * __restrict__ src0, const int32_t * __restrict__ src1, dst_t * __restrict__ dst, + const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/ + /*const int64_t ne10,*/ const int64_t ne11, const uint3 ne12_fdv, /*const int64_t ne13,*/ + /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3, + /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03, + const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) { + + ggml_cuda_pdl_sync(); + const int64_t nsb = ne00/QK_K; // super-blocks per row + for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) { + // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher. + const int i10 = blockIdx.x; + const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv); + const int i11 = dm.x; + const int i12 = dm.y; + + const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; + + dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; + const void * src0_row = (const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03; + + for (int64_t ib = blockIdx.y; ib < nsb; ib += gridDim.y) { + dequantize_kq(src0_row, ib, dst_row + ib*QK_K, threadIdx.x); + } + } +} + template static __global__ void k_get_rows_float( - const src0_t * __restrict__ src0, const int32_t * __restrict__ src1, dst_t * __restrict__ dst, + const src0_t * src0_ptr, const int32_t * src1_ptr, dst_t * dst_ptr, const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/ - /*const int64_t ne10,*/ const int64_t ne11, const int64_t ne12, /*const int64_t ne13,*/ + /*const int64_t ne10,*/ const int64_t ne11, const uint3 ne12_fdv, /*const int64_t ne13,*/ /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3, /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03, const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) { - for (int64_t z = blockIdx.z; z < ne11*ne12; z += gridDim.z) { - for (int64_t i00 = blockIdx.y*blockDim.x + threadIdx.x; i00 < ne00; i00 += gridDim.y*blockDim.x) { - // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher. - const int i10 = blockIdx.x; - const int i11 = z / ne12; // TODO fastdiv - const int i12 = z % ne12; - - if (i00 >= ne00) { - return; - } + ggml_cuda_pdl_lc(); + const src0_t * GGML_CUDA_RESTRICT src0 = src0_ptr; + const int32_t * GGML_CUDA_RESTRICT src1 = src1_ptr; + dst_t * GGML_CUDA_RESTRICT dst = dst_ptr; + ggml_cuda_pdl_sync(); + for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) { + // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher. + const int i10 = blockIdx.x; + const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv); + const int i11 = dm.x; + const int i12 = dm.y; - const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; + const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; - dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; - const src0_t * src0_row = (const src0_t *)((const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03); + dst_t * GGML_CUDA_RESTRICT dst_row = dst + i10*s1 + i11*s2 + i12*s3; + const src0_t * GGML_CUDA_RESTRICT src0_row = (const src0_t *)((const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03); + for (int64_t i00 = blockIdx.y*blockDim.x + threadIdx.x; i00 < ne00; i00 += gridDim.y*blockDim.x) { dst_row[i00] = ggml_cuda_cast(src0_row[i00]); } } } +template +static __global__ void k_get_rows_float_vec( + const dst_t * src0_ptr, const int32_t * src1_ptr, dst_t * dst_ptr, + const int64_t ne00v, + const int64_t ne11, const uint3 ne12_fdv, + const size_t s1, const size_t s2, const size_t s3, + const size_t nb01, const size_t nb02, const size_t nb03, + const size_t s10, const size_t s11, const size_t s12) { + + ggml_cuda_pdl_lc(); + ggml_cuda_pdl_sync(); + for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) { + const int i10 = blockIdx.x; + const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv); + const int i11 = dm.x; + const int i12 = dm.y; + + const int i01 = src1_ptr[i10*s10 + i11*s11 + i12*s12]; + + int4 * GGML_CUDA_RESTRICT dst_row = (int4 *) (dst_ptr + i10*s1 + i11*s2 + i12*s3); + const int4 * GGML_CUDA_RESTRICT src0_row = (const int4 *)((const char *) src0_ptr + i01*nb01 + i11*nb02 + i12*nb03); + + for (int64_t i = blockIdx.y*blockDim.x + threadIdx.x; i < ne00v; i += gridDim.y*blockDim.x) { + dst_row[i] = src0_row[i]; + } + } +} + template static __global__ void k_get_rows_back_float( - const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst, const int64_t ncols, const int64_t nrows_grad) { + const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst, + const int64_t ncols, const int64_t nrows_grad, const int64_t nrows_dst) { const int col = blockIdx.x*blockDim.x + threadIdx.x; if (col >= ncols) { return; } - const int dst_row = blockIdx.y*blockDim.y + threadIdx.y; + ggml_cuda_pdl_sync(); - float sum = 0.0f; + // grid.y is clamped to the CUDA grid limit, so stride over the destination rows + for (int64_t dst_row = blockIdx.y; dst_row < nrows_dst; dst_row += gridDim.y) { + float sum = 0.0f; - for (int64_t i = 0; i < nrows_grad; ++i) { - if (rows[i] != dst_row) { - continue; + for (int64_t i = 0; i < nrows_grad; ++i) { + if (rows[i] != dst_row) { + continue; + } + sum += grad[i*ncols + col]; } - sum += grad[i*ncols + col]; - } - dst[dst_row*ncols + col] = sum; + dst[dst_row*ncols + col] = sum; + } } template @@ -115,10 +180,51 @@ static void get_rows_cuda_q( GGML_ASSERT(ne00 % 2 == 0); + GGML_ASSERT(ne12 > 0); + GGML_ASSERT(ne11 <= std::numeric_limits::max() / ne12); + const uint3 ne12_fdv = init_fastdiv_values(ne12); + k_get_rows<<>>( src0_d, src1_d, dst_d, ne00, /*ne01, ne02, ne03,*/ - /*ne10,*/ ne11, ne12, /*ne13,*/ + /*ne10,*/ ne11, ne12_fdv, /*ne13,*/ + /* s0,*/ s1, s2, s3, + /* nb00,*/ nb01, nb02, nb03, + s10, s11, s12/*, s13*/); +} + +template dequantize_kq> +static void get_rows_cuda_kq( + const void * src0_d, const int32_t * src1_d, dst_t * dst_d, + const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12, + const size_t nb1, const size_t nb2, const size_t nb3, + cudaStream_t stream) { + GGML_ASSERT(ne00 % QK_K == 0); + const int64_t nsb = ne00/QK_K; + + const dim3 block_dims(block_dim, 1, 1); + const dim3 block_nums(ne10, MIN(nsb, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX)); + + // strides in elements + // const size_t s0 = nb0 / sizeof(dst_t); + const size_t s1 = nb1 / sizeof(dst_t); + const size_t s2 = nb2 / sizeof(dst_t); + const size_t s3 = nb3 / sizeof(dst_t); + + const size_t s10 = nb10 / sizeof(int32_t); + const size_t s11 = nb11 / sizeof(int32_t); + const size_t s12 = nb12 / sizeof(int32_t); + // const size_t s13 = nb13 / sizeof(int32_t); + + GGML_ASSERT(ne12 > 0); + GGML_ASSERT(ne11 <= std::numeric_limits::max() / ne12); + const uint3 ne12_fdv = init_fastdiv_values(ne12); + + k_get_rows_kq<<>>( + src0_d, src1_d, dst_d, + ne00, /*ne01, ne02, ne03,*/ + /*ne10,*/ ne11, ne12_fdv, /*ne13,*/ /* s0,*/ s1, s2, s3, /* nb00,*/ nb01, nb02, nb03, s10, s11, s12/*, s13*/); @@ -132,8 +238,6 @@ static void get_rows_cuda_float( const size_t nb1, const size_t nb2, const size_t nb3, cudaStream_t stream) { const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1); - const int block_num_y = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE; - const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX)); // strides in elements // const size_t s0 = nb0 / sizeof(dst_t); @@ -146,10 +250,43 @@ static void get_rows_cuda_float( const size_t s12 = nb12 / sizeof(int32_t); // const size_t s13 = nb13 / sizeof(int32_t); - k_get_rows_float<<>>( + GGML_ASSERT(ne12 > 0); + GGML_ASSERT(ne11 <= std::numeric_limits::max() / ne12); + const uint3 ne12_fdv = init_fastdiv_values(ne12); + + if constexpr (std::is_same::value) { + constexpr int VEC = 16 / sizeof(dst_t); + const int64_t ne00v = ne00 / VEC; + const int64_t vec_block_num_y = (ne00v + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE; + const bool enough_blocks = vec_block_num_y * ne10 * ne11 * ne12 >= 128; + const bool can_vec = VEC > 1 && enough_blocks && + (ne00 % VEC == 0) && + (nb01 % 16 == 0) && (nb02 % 16 == 0) && (nb03 % 16 == 0) && + (nb1 % 16 == 0) && (nb2 % 16 == 0) && (nb3 % 16 == 0) && + (((uintptr_t) src0_d) % 16 == 0) && (((uintptr_t) dst_d) % 16 == 0); + + if (can_vec) { + const int block_num_y = vec_block_num_y; + const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX)); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{block_nums, block_dims, 0, stream}; + ggml_cuda_kernel_launch(k_get_rows_float_vec, launch_params, + (const dst_t *) src0_d, src1_d, dst_d, + ne00v, ne11, ne12_fdv, + s1, s2, s3, + nb01, nb02, nb03, + s10, s11, s12); + return; + } + } + + const int block_num_y = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE; + const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX)); + + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{block_nums, block_dims, 0, stream}; + ggml_cuda_kernel_launch(k_get_rows_float, launch_params, src0_d, src1_d, dst_d, ne00, /*ne01, ne02, ne03,*/ - /*ne10,*/ ne11, ne12, /*ne13,*/ + /*ne10,*/ ne11, ne12_fdv, /*ne13,*/ /* s0,*/ s1, s2, s3, /* nb00,*/ nb01, nb02, nb03, s10, s11, s12/*, s13*/); @@ -183,6 +320,10 @@ static void ggml_cuda_get_rows_switch_src0_type( get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_Q2_0: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; case GGML_TYPE_Q4_0: get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); @@ -203,10 +344,68 @@ static void ggml_cuda_get_rows_switch_src0_type( get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_Q2_K: + get_rows_cuda_kq<64, dst_t, dequantize_q2_K>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_Q3_K: + get_rows_cuda_kq<64, dst_t, dequantize_q3_K>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_Q4_K: + get_rows_cuda_kq<32, dst_t, dequantize_q4_K>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_Q5_K: + get_rows_cuda_kq<64, dst_t, dequantize_q5_K>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_Q6_K: + get_rows_cuda_kq<64, dst_t, dequantize_q6_K>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_IQ2_XXS: + get_rows_cuda_kq<32, dst_t, dequantize_iq2_xxs>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_IQ2_XS: + get_rows_cuda_kq<32, dst_t, dequantize_iq2_xs>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_IQ2_S: + get_rows_cuda_kq<32, dst_t, dequantize_iq2_s>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_IQ3_XXS: + get_rows_cuda_kq<32, dst_t, dequantize_iq3_xxs>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_IQ3_S: + get_rows_cuda_kq<32, dst_t, dequantize_iq3_s>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_IQ1_S: + get_rows_cuda_kq<32, dst_t, dequantize_iq1_s>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_IQ1_M: + get_rows_cuda_kq<32, dst_t, dequantize_iq1_m>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_IQ4_NL: + get_rows_cuda_kq<32, dst_t, dequantize_iq4_nl>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_IQ4_XS: + get_rows_cuda_kq<32, dst_t, dequantize_iq4_xs>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; + case GGML_TYPE_MXFP4: + get_rows_cuda_kq<32, dst_t, dequantize_mxfp4>(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; default: - // TODO: k-quants GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0_type)); - break; } } @@ -235,7 +434,6 @@ void get_rows_cuda( break; default: GGML_ABORT("%s: unsupported dst type: %s\n", __func__, ggml_type_name(dst_type)); - break; } } @@ -284,7 +482,7 @@ void ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * d const dim3 block_dims(CUDA_GET_ROWS_BACK_BLOCK_SIZE, 1, 1); const int block_num_x = (ne00 + CUDA_GET_ROWS_BACK_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BACK_BLOCK_SIZE; - const dim3 block_nums(block_num_x, ne1, 1); + const dim3 block_nums(block_num_x, MIN(ne1, (int64_t)UINT16_MAX), 1); - k_get_rows_back_float<<>>(src0_d, src1_d, dst_d, ne00, ne10); + k_get_rows_back_float<<>>(src0_d, src1_d, dst_d, ne00, ne10, ne1); } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index de579d2..b795951 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2,6 +2,7 @@ #include "ggml-impl.h" #include "ggml-backend-impl.h" +#include "ggml-cuda/allreduce.cuh" #include "ggml-cuda/common.cuh" #include "ggml-cuda/acc.cuh" #include "ggml-cuda/add-id.cuh" @@ -10,11 +11,13 @@ #include "ggml-cuda/argsort.cuh" #include "ggml-cuda/binbcast.cuh" #include "ggml-cuda/clamp.cuh" +#include "ggml-cuda/col2im-1d.cuh" #include "ggml-cuda/concat.cuh" #include "ggml-cuda/conv-transpose-1d.cuh" #include "ggml-cuda/conv2d.cuh" #include "ggml-cuda/conv2d-dw.cuh" #include "ggml-cuda/conv2d-transpose.cuh" +#include "ggml-cuda/conv3d.cuh" #include "ggml-cuda/convert.cuh" #include "ggml-cuda/count-equal.cuh" #include "ggml-cuda/cpy.cuh" @@ -23,22 +26,26 @@ #include "ggml-cuda/diagmask.cuh" #include "ggml-cuda/diag.cuh" #include "ggml-cuda/fattn.cuh" +#include "ggml-cuda/fwht.cuh" #include "ggml-cuda/getrows.cuh" #include "ggml-cuda/im2col.cuh" #include "ggml-cuda/mmf.cuh" #include "ggml-cuda/mmq.cuh" #include "ggml-cuda/mmvf.cuh" #include "ggml-cuda/mmvq.cuh" +#include "ggml-cuda/moe-weighted-reduction.cuh" #include "ggml-cuda/norm.cuh" #include "ggml-cuda/opt-step-adamw.cuh" #include "ggml-cuda/opt-step-sgd.cuh" #include "ggml-cuda/out-prod.cuh" #include "ggml-cuda/pad.cuh" #include "ggml-cuda/pool2d.cuh" +#include "ggml-cuda/pool1d.cuh" #include "ggml-cuda/quantize.cuh" #include "ggml-cuda/rope.cuh" #include "ggml-cuda/roll.cuh" #include "ggml-cuda/scale.cuh" +#include "ggml-cuda/snake.cuh" #include "ggml-cuda/softcap.cuh" #include "ggml-cuda/softmax.cuh" #include "ggml-cuda/ssm-conv.cuh" @@ -54,6 +61,7 @@ #include "ggml-cuda/wkv.cuh" #include "ggml-cuda/gla.cuh" #include "ggml-cuda/gated_delta_net.cuh" +#include "ggml-cuda/dsv4-hc.cuh" #include "ggml-cuda/set.cuh" #include "ggml-cuda/set-rows.cuh" #include "ggml-cuda/pad_reflect_1d.cuh" @@ -61,6 +69,8 @@ #include "ggml-cuda/tri.cuh" #include "ggml-cuda/cumsum.cuh" #include "ggml-cuda/fill.cuh" +#include "ggml-cuda/lightning-indexer.cuh" +#include "ggml-cuda/top-k-mask.cuh" #include "ggml.h" #include @@ -85,6 +95,9 @@ static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size"); +#define GGML_LOG_WARN_ONCE(str) \ + { static std::once_flag warn_flag; std::call_once(warn_flag, []() { GGML_LOG_WARN(str); }); } + [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) { int id = -1; // in case cudaGetDevice fails @@ -97,17 +110,27 @@ void ggml_cuda_error(const char * stmt, const char * func, const char * file, in GGML_ABORT(GGML_CUDA_NAME " error"); } +// map a (possibly virtual) device id to the physical CUDA device that backs it +static int ggml_cuda_get_physical_device(int device) { + const ggml_cuda_device_info & info = ggml_cuda_info(); + GGML_ASSERT(device >= 0 && device < info.device_count); + return info.devices[device].physical_device; +} + // this is faster on Windows // probably because the Windows CUDA libraries forget to make this check before invoking the drivers void ggml_cuda_set_device(int device) { + // translate the (possibly virtual) device id to the physical CUDA device that backs it + const int physical_device = ggml_cuda_get_physical_device(device); + int current_device; CUDA_CHECK(cudaGetDevice(¤t_device)); - if (device == current_device) { + if (physical_device == current_device) { return; } - CUDA_CHECK(cudaSetDevice(device)); + CUDA_CHECK(cudaSetDevice(physical_device)); } int ggml_cuda_get_device() { @@ -116,10 +139,54 @@ int ggml_cuda_get_device() { return id; } +static bool ggml_cuda_env_flag(const char * name) { + const char * env = getenv(name); + return env != nullptr && std::atoi(env) != 0; +} + +static int ggml_cuda_env_int(const char * name, int default_value) { + const char * env = getenv(name); + return env != nullptr ? std::atoi(env) : default_value; +} + +static bool ggml_cuda_env_set(const char * name) { + return getenv(name) != nullptr; +} + +static bool ggml_cuda_unified_memory_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_ENABLE_UNIFIED_MEMORY"); + return enabled; +} + +static bool ggml_cuda_pinned_memory_disabled() { + static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_NO_PINNED"); + return disabled; +} + +static bool ggml_cuda_register_host_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_REGISTER_HOST"); + return enabled; +} + +static bool ggml_cuda_fusion_disabled() { + static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_DISABLE_FUSION"); + return disabled; +} + +static bool ggml_cuda_p2p_enabled() { + static const bool enabled = ggml_cuda_env_set("GGML_CUDA_P2P"); + return enabled; +} + +static int ggml_cuda_op_offload_min_batch() { + static const int min_batch = ggml_cuda_env_int("GGML_OP_OFFLOAD_MIN_BATCH", 32); + return min_batch; +} + static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) { ggml_cuda_set_device(device); cudaError_t err; - if (getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr) { + if (ggml_cuda_unified_memory_enabled()) { err = cudaMallocManaged(ptr, size); #if defined(GGML_USE_HIP) if (err == hipSuccess) { @@ -147,111 +214,134 @@ static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) } #if defined(GGML_USE_HIP) -static int ggml_cuda_parse_id(char devName[]) { +static int ggml_cuda_parse_id(const char * devName) { // A list of possible Target IDs can be found under the rocclr/clr repo in device.cpp // these values are not stable so this is susceptible to breakage // https://github.com/ROCm/clr/blob/amd-staging/rocclr/device/device.cpp - int archMajor = 0x0; - int archMinor = 0x0; - int archNum = GGML_CUDA_CC_OFFSET_AMD; - int archLen = strlen(devName); - char archName[archLen + 1]; + // Extract the base architecture model ("gfx908:sramecc+:xnack-" -> "908") + std::string archName = devName; + archName.erase(0, 3); + archName = archName.substr(0, archName.find(':')); - // strip leading 'gfx' while copying into our buffer - if (archLen > 3) { - strcpy(archName, &devName[3]); - archLen -= 3; - } + const auto hex = [](const std::string & s) { return (int) strtoul(s.c_str(), nullptr, 16); }; - // trim trailing :xnack- or :sramecc- statuses - archLen = strcspn(archName, ":"); - archName[archLen] = '\0'; + int cc = 0; // tease out the version information - if (archLen > 8) { + if (archName.size() > 8) { // versions labeled generic use '-' as delimiter - // strip the trailing "-generic" then iterate through what remains - if ((strstr(archName, "-generic"))) { - archName[archLen - 8] = '\0'; - char * pch; - if ((pch = strtok(archName, "-"))) { - archMajor = (int)strtoul(pch, 0, 16); - if ((pch = strtok(NULL, "-"))) { - archMinor = 0x10 * (int)strtoul(pch, 0, 16); - } + if (archName.compare(archName.size() - 8, 8, "-generic") == 0) { + archName.resize(archName.size() - 8); + const size_t sep = archName.find('-'); + cc = 0x100 * hex(archName.substr(0, sep)); + if (sep != std::string::npos) { + cc += 0x10 * hex(archName.substr(sep + 1)); } } - } else if (archLen >= 3) { - // last two digits should be the minor * 0x10 + stepping - archMinor = (int)strtoul(&archName[archLen - 2], 0, 16); - archName[archLen - 2] = '\0'; - - // only the major version remains - archMajor = (int)strtoul(archName, 0, 16); + } else if (archName.size() >= 3) { + // ordinary IDs are the version in hex + cc = hex(archName); } - archNum += archMajor * 0x100; - archNum += archMinor; - return archNum; + + return GGML_CUDA_CC_OFFSET_AMD + cc; } #endif // defined(GGML_USE_HIP) static ggml_cuda_device_info ggml_cuda_init() { ggml_cuda_device_info info = {}; - cudaError_t err = cudaGetDeviceCount(&info.device_count); + cudaError_t err = cudaGetDeviceCount(&info.physical_device_count); if (err != cudaSuccess) { GGML_LOG_ERROR("%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err)); return info; } - GGML_ASSERT(info.device_count <= GGML_CUDA_MAX_DEVICES); + GGML_ASSERT(info.physical_device_count <= GGML_CUDA_MAX_DEVICES); - int64_t total_vram = 0; + // by default expose exactly the physical devices; GGML_CUDA_DEVICES can request a different + // number of (virtual) devices to emulate multi-GPU systems on a machine with fewer GPUs + info.device_count = info.physical_device_count; + + const char * devices_env = getenv("GGML_CUDA_DEVICES"); + if (devices_env != nullptr && info.physical_device_count > 0) { + const int requested = atoi(devices_env); + if (requested > 0) { + info.device_count = requested; + } else { + GGML_LOG_WARN("%s: ignoring invalid GGML_CUDA_DEVICES=\"%s\"\n", __func__, devices_env); + } + } + + if (info.device_count > GGML_CUDA_MAX_DEVICES) { + GGML_LOG_WARN("%s: requested %d devices, clamping to GGML_CUDA_MAX_DEVICES=%d\n", + __func__, info.device_count, GGML_CUDA_MAX_DEVICES); + info.device_count = GGML_CUDA_MAX_DEVICES; + } + + // map each (virtual) device to a backing physical device (round-robin), assign each its index + // among the (virtual) devices sharing that physical GPU, and store the per-physical share count + int physical_share_count[GGML_CUDA_MAX_DEVICES] = {}; + GGML_ASSERT(info.device_count == 0 || info.physical_device_count > 0); for (int id = 0; id < info.device_count; ++id) { + info.devices[id].physical_device = id % info.physical_device_count; + info.devices[id].virtual_index = physical_share_count[info.devices[id].physical_device]++; + } + + int64_t total_vram = 0; + for (int id = 0; id < info.physical_device_count; ++id) { cudaDeviceProp prop; CUDA_CHECK(cudaGetDeviceProperties(&prop, id)); total_vram += prop.totalGlobalMem; } GGML_LOG_INFO("%s: found %d " GGML_CUDA_NAME " devices (Total VRAM: %zu MiB):\n", - __func__, info.device_count, (size_t)(total_vram / (1024 * 1024))); + __func__, info.physical_device_count, (size_t)(total_vram / (1024 * 1024))); + if (info.device_count != info.physical_device_count) { + GGML_LOG_INFO("%s: emulating %d virtual device(s) on %d physical device(s) (GGML_CUDA_DEVICES)\n", + __func__, info.device_count, info.physical_device_count); + } total_vram = 0; std::vector> turing_devices_without_mma; for (int id = 0; id < info.device_count; ++id) { + const int physical_id = info.devices[id].physical_device; + int device_vmm = 0; #if defined(GGML_USE_VMM) CUdevice device; - CU_CHECK(cuDeviceGet(&device, id)); + CU_CHECK(cuDeviceGet(&device, physical_id)); CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device)); if (device_vmm) { CUmemAllocationProp alloc_prop = {}; alloc_prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; alloc_prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - alloc_prop.location.id = id; + alloc_prop.location.id = physical_id; CU_CHECK(cuMemGetAllocationGranularity(&info.devices[id].vmm_granularity, &alloc_prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED)); } #endif // defined(GGML_USE_VMM) info.devices[id].vmm = !!device_vmm; cudaDeviceProp prop; - CUDA_CHECK(cudaGetDeviceProperties(&prop, id)); + CUDA_CHECK(cudaGetDeviceProperties(&prop, physical_id)); + + // a virtual device owns only a share of its physical GPU's memory; report that share so the + // logged per-device VRAM sums to the physical total above. + GGML_ASSERT(physical_share_count[physical_id] > 0); + info.devices[id].physical_share_count = physical_share_count[physical_id]; + const size_t device_vram = prop.totalGlobalMem / info.devices[id].physical_share_count; + const size_t device_vram_mib = device_vram / (1024 * 1024); info.default_tensor_split[id] = total_vram; - total_vram += prop.totalGlobalMem; + total_vram += device_vram; info.devices[id].integrated = false; // Temporarily disabled due to issues with corrupted output (e.g. #15034) info.devices[id].nsm = prop.multiProcessorCount; info.devices[id].smpb = prop.sharedMemPerBlock; info.devices[id].warp_size = prop.warpSize; -#ifndef GGML_USE_MUSA int supports_coop_launch = 0; - CUDA_CHECK(cudaDeviceGetAttribute(&supports_coop_launch, cudaDevAttrCooperativeLaunch, id)); + CUDA_CHECK(cudaDeviceGetAttribute(&supports_coop_launch, cudaDevAttrCooperativeLaunch, physical_id)); info.devices[id].supports_cooperative_launch = !!supports_coop_launch; -#else - info.devices[id].supports_cooperative_launch = false; -#endif // !(GGML_USE_MUSA) #if defined(GGML_USE_HIP) info.devices[id].smpbo = prop.sharedMemPerBlock; @@ -270,22 +360,20 @@ static ggml_cuda_device_info ggml_cuda_init() { GGML_LOG_INFO(" Device %d: %s, %s (0x%x), VMM: %s, Wave Size: %d, VRAM: %zu MiB\n", id, prop.name, prop.gcnArchName, info.devices[id].cc & 0xffff, device_vmm ? "yes" : "no", prop.warpSize, - (size_t)(prop.totalGlobalMem / (1024 * 1024))); + device_vram_mib); #elif defined(GGML_USE_MUSA) - // FIXME: Ensure compatibility with varying warp sizes across different MUSA archs. - info.devices[id].warp_size = 32; info.devices[id].smpbo = prop.sharedMemPerBlockOptin; info.devices[id].cc = GGML_CUDA_CC_OFFSET_MTHREADS + prop.major * 0x100; info.devices[id].cc += prop.minor * 0x10; GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no", - (size_t)(prop.totalGlobalMem / (1024 * 1024))); + device_vram_mib); #else info.devices[id].smpbo = prop.sharedMemPerBlockOptin; info.devices[id].cc = 100*prop.major + 10*prop.minor; GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no", - (size_t)(prop.totalGlobalMem / (1024 * 1024))); + device_vram_mib); std::string device_name(prop.name); if (device_name == "NVIDIA GeForce MX450") { turing_devices_without_mma.push_back({ id, device_name }); @@ -300,7 +388,7 @@ static ggml_cuda_device_info ggml_cuda_init() { // TODO: Check for future drivers the default scheduling strategy and // remove this call again when cudaDeviceScheduleSpin is default. if (prop.major == 12 && prop.minor == 1) { - CUDA_CHECK(cudaSetDevice(id)); + CUDA_CHECK(cudaSetDevice(physical_id)); CUDA_CHECK(cudaSetDeviceFlags(cudaDeviceScheduleSpin)); } @@ -324,10 +412,10 @@ static ggml_cuda_device_info ggml_cuda_init() { // configure logging to stdout // CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr)); - if (getenv("GGML_CUDA_P2P") != nullptr) { - for (int id = 0; id < info.device_count; ++id) { - ggml_cuda_set_device(id); - for (int id_other = 0; id_other < info.device_count; ++id_other) { + if (ggml_cuda_p2p_enabled()) { + for (int id = 0; id < info.physical_device_count; ++id) { + CUDA_CHECK(cudaSetDevice(id)); + for (int id_other = 0; id_other < info.physical_device_count; ++id_other) { if (id == id_other) { continue; } @@ -354,6 +442,7 @@ const ggml_cuda_device_info & ggml_cuda_info() { struct ggml_cuda_pool_leg : public ggml_cuda_pool { static const int MAX_BUFFERS = 256; + ggml_backend_cuda_context & ctx; int device; struct ggml_cuda_buffer { void * ptr = nullptr; @@ -363,20 +452,41 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool { ggml_cuda_buffer buffer_pool[MAX_BUFFERS] = {}; size_t pool_size = 0; - explicit ggml_cuda_pool_leg(int device) : + ggml_cuda_pool_leg(ggml_backend_cuda_context & ctx, int device) : + ctx(ctx), device(device) { } ~ggml_cuda_pool_leg() { + clear_pool(); + GGML_ASSERT(pool_size == 0); + } + + void clear_pool() { ggml_cuda_set_device(device); for (int i = 0; i < MAX_BUFFERS; ++i) { ggml_cuda_buffer & b = buffer_pool[i]; if (b.ptr != nullptr) { CUDA_CHECK(cudaFree(b.ptr)); pool_size -= b.size; + b.ptr = nullptr; + b.size = 0; } } - GGML_ASSERT(pool_size == 0); + } + + // captured CUDA graphs can still use freed buffers, so they must be captured again + void invalidate_cuda_graphs() { +#ifdef USE_CUDA_GRAPH + int n_captured = 0; + for (const auto & [key, graph] : ctx.cuda_graphs) { + n_captured += graph->instances.size(); + } + if (n_captured > 0) { + GGML_LOG_DEBUG(GGML_CUDA_NAME " pool[%d]: memory freed, %d captured CUDA graphs must be captured again\n", device, n_captured); + } + ctx.cuda_graphs.clear(); +#endif // USE_CUDA_GRAPH } void * alloc(size_t size, size_t * actual_size) override { @@ -421,7 +531,21 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool { size_t look_ahead_size = (size_t) (1.05 * size); look_ahead_size = 256 * ((look_ahead_size + 255)/256); ggml_cuda_set_device(device); - CUDA_CHECK(ggml_cuda_device_malloc(&ptr, look_ahead_size, device)); + cudaError_t err = ggml_cuda_device_malloc(&ptr, look_ahead_size, device); + if (err == cudaErrorMemoryAllocation) { + (void)cudaGetLastError(); + const size_t cached_bytes = pool_size; + GGML_LOG_DEBUG(GGML_CUDA_NAME " pool[%d]: alloc of %.2f MiB failed, flushing %.2f MiB of cached buffers and retrying\n", + device, look_ahead_size/1024.0/1024.0, cached_bytes/1024.0/1024.0); + CUDA_CHECK(cudaDeviceSynchronize()); + clear_pool(); + invalidate_cuda_graphs(); + err = ggml_cuda_device_malloc(&ptr, look_ahead_size, device); + if (err == cudaSuccess) { + GGML_LOG_DEBUG(GGML_CUDA_NAME " pool[%d]: retry succeeded\n", device); + } + } + CUDA_CHECK(err); *actual_size = look_ahead_size; pool_size += look_ahead_size; #ifdef DEBUG_CUDA_MALLOC @@ -444,6 +568,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool { ggml_cuda_set_device(device); CUDA_CHECK(cudaFree(ptr)); pool_size -= size; + invalidate_cuda_graphs(); } }; @@ -453,6 +578,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB int device; + int physical_device; CUdeviceptr pool_addr = 0; size_t pool_used = 0; size_t pool_size = 0; @@ -463,6 +589,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { explicit ggml_cuda_pool_vmm(int device) : device(device), + physical_device(ggml_cuda_get_physical_device(device)), granularity(ggml_cuda_info().devices[device].vmm_granularity) { } @@ -498,7 +625,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { CUmemAllocationProp prop = {}; prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - prop.location.id = device; + prop.location.id = physical_device; CUmemGenericAllocationHandle handle; CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0)); @@ -517,12 +644,50 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { // the memory allocation handle is no longer needed after mapping CU_CHECK(cuMemRelease(handle)); - // set access - CUmemAccessDesc access = {}; - access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; - access.location.id = device; - access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; - CU_CHECK(cuMemSetAccess((CUdeviceptr)((char *)(pool_addr) + pool_size), reserve_size, &access, 1)); + // VMM Bug fix for P2P access if GGML_CUDA_P2P is set, or if NCCL build + bool use_peer_access = ggml_cuda_p2p_enabled(); +#if defined(GGML_USE_NCCL) + use_peer_access = true; +#endif // defined(GGML_USE_NCCL) + + if (use_peer_access) { + // NCCL implicitly enables peer access (cudaDeviceEnablePeerAccess), and + // GGML_CUDA_P2P enables it explicitly. Unlike cudaMalloc buffers, VMM + // allocations do not become peer-accessible from that alone, so access + // must be granted explicitly here. With virtual devices, grant access + // on the backing *physical* devices (deduplicated, since several + // virtual devices can map to the same physical GPU). + std::vector access_descs; + bool physical_seen[GGML_CUDA_MAX_DEVICES] = {}; + const int device_count = ggml_cuda_info().device_count; + for (int id = 0; id < device_count; ++id) { + const int id_physical = ggml_cuda_get_physical_device(id); + if (id_physical != physical_device) { + int can_access_peer = 0; + CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id_physical, physical_device)); + if (!can_access_peer) { + continue; + } + } + if (physical_seen[id_physical]) { + continue; + } + physical_seen[id_physical] = true; + CUmemAccessDesc access = {}; + access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + access.location.id = id_physical; + access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + access_descs.push_back(access); + } + CU_CHECK(cuMemSetAccess(start_ptr, reserve_size, access_descs.data(), access_descs.size())); + } else { + // set access for non P2P + CUmemAccessDesc access = {}; + access.location.type = CU_MEM_LOCATION_TYPE_DEVICE; + access.location.id = physical_device; + access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; + CU_CHECK(cuMemSetAccess(start_ptr, reserve_size, &access, 1)); + } // add to the pool pool_size += reserve_size; @@ -558,14 +723,15 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool { }; #endif // defined(GGML_USE_VMM) -std::unique_ptr ggml_backend_cuda_context::new_pool_for_device(int device, - [[maybe_unused]] int stream_no) { +std::unique_ptr ggml_backend_cuda_context::new_pool_for_device(ggml_backend_cuda_context & ctx, + int device, + [[maybe_unused]] int stream_no) { #if defined(GGML_USE_VMM) if (ggml_cuda_info().devices[device].vmm) { return std::unique_ptr(new ggml_cuda_pool_vmm(device)); } #endif // defined(GGML_USE_VMM) - return std::unique_ptr(new ggml_cuda_pool_leg(device)); + return std::unique_ptr(new ggml_cuda_pool_leg(ctx, device)); } // destroying a cuBLAS handle while a graph is being captured in a different thread can result in a CUDA error @@ -587,9 +753,12 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { if (streams[i][j] != nullptr) { CUDA_CHECK(cudaStreamDestroy(streams[i][j])); } - } - if (cublas_handles[i] != nullptr) { - CUBLAS_CHECK(cublasDestroy(cublas_handles[i])); + if (cublas_handles[i][j] != nullptr) { + CUBLAS_CHECK(cublasDestroy(cublas_handles[i][j])); + } + if (cublas_workspaces[i][j] != nullptr) { + CUDA_CHECK(cudaFree(cublas_workspaces[i][j])); + } } } } @@ -641,7 +810,8 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer if (padded_size > original_size) { ggml_cuda_set_device(ctx->device); - CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size)); + CUDA_CHECK(cudaMemsetAsync((char *)tensor->data + original_size, 0, padded_size - original_size, cudaStreamPerThread)); + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); } } return GGML_STATUS_SUCCESS; @@ -695,13 +865,17 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co if (ggml_backend_buffer_is_cuda(src->buffer)) { ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context; ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context; - if (src_ctx->device == dst_ctx->device) { + // compare the backing physical devices: distinct virtual devices may share one physical GPU, + // in which case a same-device copy (not a peer copy) is required + const int src_physical = ggml_cuda_get_physical_device(src_ctx->device); + const int dst_physical = ggml_cuda_get_physical_device(dst_ctx->device); + if (src_physical == dst_physical) { CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice, cudaStreamPerThread)); } else { #ifdef GGML_CUDA_NO_PEER_COPY return false; #else - CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_ctx->device, src->data, src_ctx->device, ggml_nbytes(src), cudaStreamPerThread)); + CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_physical, src->data, src_physical, ggml_nbytes(src), cudaStreamPerThread)); #endif } CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); @@ -776,9 +950,14 @@ static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_ty } static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { - size_t size = ggml_nbytes(tensor); + ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *) buft->context; + + size_t size = tensor->op == GGML_OP_FLASH_ATTN_EXT + ? ggml_cuda_flash_attn_ext_get_alloc_size(buft_ctx->device, tensor) + : ggml_nbytes(tensor); int64_t ne0 = tensor->ne[0]; + // [TAG_ALLOC_SIZE_EXPAND] if (ggml_is_quantized(tensor->type)) { if (ne0 % MATRIX_ROW_PADDING != 0) { GGML_ASSERT(tensor->nb[0] == ggml_element_size(tensor)); @@ -787,17 +966,17 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t } return size; - - GGML_UNUSED(buft); } static const ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = { - /* .get_name = */ ggml_backend_cuda_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size, - /* .is_host = */ NULL, + /* .get_name = */ ggml_backend_cuda_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ NULL, }; ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { @@ -826,363 +1005,46 @@ ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { return &ggml_backend_cuda_buffer_types[device]; } -// cuda split buffer - -static int64_t get_row_rounding(const std::array & tensor_split) { - int64_t row_rounding = 0; - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) { - continue; - } - - const int cc = ggml_cuda_info().devices[id].cc; - row_rounding = std::max(row_rounding, (int64_t)get_mmq_y_host(cc)); - } - return row_rounding; -} - -static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array & tensor_split, int id) { - const int64_t nrows = ggml_nrows(tensor); - const int64_t rounding = get_row_rounding(tensor_split); - - *row_low = id == 0 ? 0 : nrows*tensor_split[id]; - *row_low -= *row_low % rounding; - - if (id == ggml_backend_cuda_get_device_count() - 1) { - *row_high = nrows; - } else { - *row_high = nrows*tensor_split[id + 1]; - *row_high -= *row_high % rounding; - } -} - -static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) { - static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); - - return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]); -} - -struct ggml_backend_cuda_split_buffer_type_context { - int main_device; - std::array tensor_split; - std::string name; -}; - -struct ggml_backend_cuda_split_buffer_context { - ~ggml_backend_cuda_split_buffer_context() { - for (ggml_tensor_extra_gpu * extra : tensor_extras) { - for (int id = 0; id < GGML_CUDA_MAX_DEVICES; ++id) { - for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) { - if (extra->events[id][is] != nullptr) { - CUDA_CHECK(cudaEventDestroy(extra->events[id][is])); - } - } - if (extra->data_device[id] != nullptr) { - CUDA_CHECK(cudaFree(extra->data_device[id])); - } - } - delete extra; - } - } - - std::vector tensor_extras; -}; - - -static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; - delete ctx; -} - -static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) { - // the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced - return (void *)0x1000; - - GGML_UNUSED(buffer); -} - -static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported - GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors"); - - ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context; - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; - - const int64_t ne0 = tensor->ne[0]; - - ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; - ctx->tensor_extras.push_back(extra); - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - // FIXME: do not crash if cudaMalloc fails - // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first - ggml_cuda_set_device(id); - char * buf; - CUDA_CHECK(ggml_cuda_device_malloc((void**)&buf, size, id)); - - // set padding to 0 to avoid possible NaN values - if (size > original_size) { - CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size)); - } - - extra->data_device[id] = buf; - - for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) { - CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming)); - } - } - tensor->extra = extra; - return GGML_STATUS_SUCCESS; -} - -static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - // split tensors must always be set in their entirety at once - GGML_ASSERT(offset == 0); - GGML_ASSERT(size == ggml_nbytes(tensor)); - GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors"); - - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; - - const int64_t ne0 = tensor->ne[0]; - const size_t nb1 = tensor->nb[1]; - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - const size_t offset_split = row_low*nb1; - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - const char * buf_host = (const char *)data + offset_split; - CUDA_CHECK(cudaMemcpyAsync(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice, cudaStreamPerThread)); - } - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); - } -} - -static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - // split tensors must always be set in their entirety at once - GGML_ASSERT(offset == 0); - GGML_ASSERT(size == ggml_nbytes(tensor)); - GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors"); - - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context; - - const int64_t ne0 = tensor->ne[0]; - const size_t nb1 = tensor->nb[1]; - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - const size_t offset_split = row_low*nb1; - size_t size = ggml_nbytes_split(tensor, nrows_split); - const size_t original_size = size; - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - - char * buf_host = (char *)data + offset_split; - CUDA_CHECK(cudaMemcpyAsync(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost, cudaStreamPerThread)); - } - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); - } -} - -static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - GGML_UNUSED(buffer); - GGML_UNUSED(value); -} - -static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = { - /* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer, - /* .get_base = */ ggml_backend_cuda_split_buffer_get_base, - /* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor, - /* .memset_tensor = */ NULL, - /* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, - /* .get_tensor_2d = */ NULL, - /* .cpy_tensor = */ NULL, - /* .clear = */ ggml_backend_cuda_split_buffer_clear, - /* .reset = */ NULL, -}; - -// cuda split buffer type - -static const char * ggml_backend_cuda_split_buffer_type_get_name(ggml_backend_buffer_type_t buft) { - ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context; - - return ctx->name.c_str(); -} - -static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) { - return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_get_name; -} - -static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { - // since we don't know the exact split after rounding, we cannot allocate the device buffers at this point - // instead, we allocate them for each tensor separately in init_tensor - // however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated, - // as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct. - ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context(); - - return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size); -} - -static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return 128; - - GGML_UNUSED(buft); -} - -static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { - ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context; - GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors"); - - size_t total_size = 0; - - const int64_t ne0 = tensor->ne[0]; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - int64_t row_low, row_high; - get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id); - - int64_t nrows_split = row_high - row_low; - if (nrows_split == 0) { - continue; - } - - total_size += ggml_nbytes_split(tensor, nrows_split); - - // pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses - if (ne0 % MATRIX_ROW_PADDING != 0) { - total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); - } - } - - return total_size; -} +// Communication context for multi-GPU AllReduce during tensor parallelism. +// +// Created once per meta backend instance. Resources for the selected mode +// (NCCL communicators or the internal AllReduce pipeline) are initialised +// eagerly during comm_init so any init failure surfaces at startup rather +// than mid-run. +struct ggml_backend_cuda_comm_context { + using try_allreduce_fn = bool(*)(ggml_backend_cuda_comm_context *, struct ggml_tensor **); -static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - return false; + std::vector backends; + std::vector dev_ids; - GGML_UNUSED(buft); -} + // Set by the init chain (comm_init_{nccl, internal, none}) to one of + // try_allreduce_{nccl, internal, butterfly}. nccl needs `comms`, + // internal needs `ar_pipeline`, butterfly needs nothing. Per-call + // failures return false; the meta backend's generic implementation then + // handles that call. + try_allreduce_fn try_allreduce = nullptr; -static const ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = { - /* .get_name = */ ggml_backend_cuda_split_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size, - /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host, -}; + ggml_cuda_ar_pipeline * ar_pipeline = nullptr; #ifdef GGML_USE_NCCL -struct ggml_backend_cuda_comm_context { - std::vector backends; - std::vector comms; + std::vector comms; +#endif // GGML_USE_NCCL ~ggml_backend_cuda_comm_context() { +#ifdef GGML_USE_NCCL for (ncclComm_t comm : comms) { NCCL_CHECK(ncclCommDestroy(comm)); } - } -}; -#endif // GGML_USE_NCCL - -static void ggml_backend_cuda_comm_free(void * comm_ctx_v) { -#ifdef GGML_USE_NCCL - if (comm_ctx_v == nullptr) { - return; - } - ggml_backend_cuda_comm_context * comm_ctx = (ggml_backend_cuda_comm_context *) comm_ctx_v; - delete comm_ctx; -#else - GGML_UNUSED(comm_ctx_v); #endif // GGML_USE_NCCL -} - -static void * ggml_backend_cuda_comm_init(ggml_backend_t * backends, size_t n_backends) { -#ifdef GGML_USE_NCCL - for (size_t i = 0; i < n_backends; i++) { - if (!ggml_backend_is_cuda(backends[i])) { - return nullptr; - } - } - ggml_backend_cuda_comm_context * ret = new ggml_backend_cuda_comm_context; - std::vector dev_ids; - ret->backends.reserve(n_backends); - dev_ids.reserve(n_backends); - for (size_t i = 0; i < n_backends; i++) { - ret->backends.push_back(backends[i]); - ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backends[i]->context; - dev_ids.push_back(cuda_ctx->device); + ggml_cuda_ar_pipeline_free(ar_pipeline); } +}; - ret->comms.resize(n_backends); - NCCL_CHECK(ncclCommInitAll(ret->comms.data(), n_backends, dev_ids.data())); - return ret; -#else - // If NCCL is installed it is used by default for optimal performance. - // However, NVIDIA does not distribute NCCL with CUDA so users may be unwittingly missing this package. - // RCCL is disabled by default, users are explicitly opting in. - // Therefore print no warning for RCCL. -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - static bool warning_printed = false; - if (!warning_printed) { - GGML_LOG_WARN("%s: NVIDIA Collective Communications Library (NCCL) is unavailable, multi GPU performance will be suboptimal\n", __func__); - warning_printed = true; - } -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - GGML_UNUSED_VARS(backends, n_backends); - return nullptr; -#endif // GGML_USE_NCCL -} - -static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { #ifdef GGML_USE_NCCL +// AllReduce via NCCL. Reduces as FP32 for small tensors and BF16 for large +// tensors (bandwidth-bound), then converts back to FP32. +static bool ggml_backend_cuda_comm_allreduce_nccl( + ggml_backend_cuda_comm_context * comm_ctx, struct ggml_tensor ** tensors) { const int64_t ne = ggml_nelements(tensors[0]); // FIXME the input of llm_graph_context::build_in_out_ids can produce a tensor with 0 elements if n_outputs == 0 // This then causes a crash in this function @@ -1190,8 +1052,6 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg return true; } - GGML_ASSERT(comm_ctx_v != nullptr); - ggml_backend_cuda_comm_context * comm_ctx = (ggml_backend_cuda_comm_context *) comm_ctx_v; const size_t n_backends = comm_ctx->backends.size(); for (size_t i = 0; i < n_backends; ++i) { @@ -1203,13 +1063,19 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg // For small tensors, simply reduce them as FP32. // The following heuristic for how "small" a tensor should be is based on RTX 4090s connected via 16x PCIe 4.0. if ((n_backends <= 2 && ne < 32768) || (n_backends == 3 && ne < 131072) || (n_backends >= 4 && ne < 262144)) { + for (size_t i = 0; i < n_backends; ++i) { + if ((tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) comm_ctx->backends[i]->context; + ggml_cuda_set_device(cuda_ctx->device); + CUDA_CHECK(cudaMemsetAsync(tensors[i]->data, 0, ggml_nbytes(tensors[i]), cuda_ctx->stream())); + } + } NCCL_CHECK(ncclGroupStart()); for (size_t i = 0; i < n_backends; ++i) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) comm_ctx->backends[i]->context; NCCL_CHECK(ncclAllReduce(tensors[i]->data, tensors[i]->data, ne, ncclFloat, ncclSum, comm_ctx->comms[i], cuda_ctx->stream())); } NCCL_CHECK(ncclGroupEnd()); - return true; } @@ -1224,7 +1090,11 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg tmp[i].alloc(ne); ggml_cuda_set_device(cuda_ctx->device); - to_bf16(tensors[i]->data, tmp[i].get(), ne, cuda_ctx->stream()); + if (tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) { + to_bf16(tensors[i]->data, tmp[i].get(), ne, cuda_ctx->stream()); + } else { + CUDA_CHECK(cudaMemsetAsync(tmp[i].get(), 0, ne * sizeof(nv_bfloat16), cuda_ctx->stream())); + } CUDA_CHECK(cudaGetLastError()); } @@ -1244,52 +1114,193 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg } return true; -#else - GGML_UNUSED_VARS(comm_ctx_v, tensors); - return false; -#endif // GGML_USE_NCCL } +#endif // GGML_USE_NCCL -ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split) { - static std::mutex mutex; - std::lock_guard lock(mutex); +// Run the internal AR pipeline. Returns false on unsupported / failed input +// -- the caller decides whether to abort (env-forced) or fall back silently. +static bool ggml_backend_cuda_comm_allreduce_internal( + ggml_backend_cuda_comm_context * comm_ctx, struct ggml_tensor ** tensors) { + GGML_ASSERT(comm_ctx->ar_pipeline != nullptr); - static std::map>, struct ggml_backend_buffer_type> buft_map; + const size_t n_backends = comm_ctx->backends.size(); + GGML_ASSERT(n_backends == 2); + GGML_ASSERT(tensors[0] != nullptr); - std::array tensor_split_arr = {}; + const int64_t ne = ggml_nelements(tensors[0]); + const ggml_type type = tensors[0]->type; - bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; }); - if (all_zero) { - tensor_split_arr = ggml_cuda_info().default_tensor_split; - } else { - float split_sum = 0.0f; - for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) { - tensor_split_arr[i] = split_sum; - split_sum += tensor_split[i]; - } - for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) { - tensor_split_arr[i] /= split_sum; - } + if (type != GGML_TYPE_F32 && type != GGML_TYPE_F16 && type != GGML_TYPE_BF16) { + GGML_LOG_DEBUG("%s: internal unsupported: type=%d\n", __func__, (int) type); + return false; } - auto it = buft_map.find({main_device, tensor_split_arr}); - if (it != buft_map.end()) { - return &it->second; + if (ne == 0) { + return true; } - auto * ctx = new ggml_backend_cuda_split_buffer_type_context{ - main_device, - tensor_split_arr, - GGML_CUDA_NAME + std::to_string(main_device) + "_Split", - }; - struct ggml_backend_buffer_type buft { - /* .iface = */ ggml_backend_cuda_split_buffer_type_interface, - /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cuda_reg(), main_device), - /* .context = */ ctx, - }; + for (size_t i = 0; i < n_backends; ++i) { + if (tensors[i] == nullptr) { + GGML_LOG_ERROR("%s: internal failed: tensor[%zu] is null\n", __func__, i); + return false; + } + if (ggml_nelements(tensors[i]) != ne || tensors[i]->type != type) { + GGML_LOG_ERROR("%s: internal failed: tensor[%zu] ne=%" PRId64 " type=%d expected ne=%" PRId64 " type=%d\n", + __func__, i, ggml_nelements(tensors[i]), (int) tensors[i]->type, ne, (int) type); + return false; + } + if (!ggml_is_contiguously_allocated(tensors[i])) { + GGML_LOG_DEBUG("%s: internal unsupported: tensor[%zu] is not contiguously allocated: ne=%" PRId64 " nbytes=%zu packed=%zu type=%d\n", + __func__, i, ne, ggml_nbytes(tensors[i]), + (size_t) ne * ggml_type_size(type) / ggml_blck_size(type), (int) type); + return false; + } + if (((uintptr_t) tensors[i]->data & 0xF) != 0) { + GGML_LOG_DEBUG("%s: internal unsupported: tensor[%zu] data pointer is not 16-byte aligned: %p type=%d ne=%" PRId64 "\n", + __func__, i, tensors[i]->data, (int) type, ne); + return false; + } + GGML_ASSERT((ggml_nbytes(tensors[i]) & 0xF) == 0); + } + + return ggml_cuda_ar_allreduce(comm_ctx->ar_pipeline, comm_ctx->backends.data(), tensors); +} + +// --------------------------------------------------------------------------- +// Per-call dispatch -- three variants, one per backend. Each is set as +// comm_ctx->try_allreduce by the matching init step. Per-call failure +// returns false; the meta backend's generic implementation handles that call. +// --------------------------------------------------------------------------- + +#ifdef GGML_USE_NCCL +static bool ggml_backend_cuda_comm_try_allreduce_nccl( + ggml_backend_cuda_comm_context * comm_ctx, struct ggml_tensor ** tensors) { + return ggml_backend_cuda_comm_allreduce_nccl(comm_ctx, tensors); +} +#endif // GGML_USE_NCCL + +static bool ggml_backend_cuda_comm_try_allreduce_internal( + ggml_backend_cuda_comm_context * comm_ctx, struct ggml_tensor ** tensors) { + return ggml_backend_cuda_comm_allreduce_internal(comm_ctx, tensors); +} + +static bool ggml_backend_cuda_comm_try_allreduce_butterfly( + ggml_backend_cuda_comm_context *, struct ggml_tensor **) { + return false; +} + +static void ggml_backend_cuda_comm_free(void * comm_ctx_v) { + if (comm_ctx_v == nullptr) { + return; + } + delete static_cast(comm_ctx_v); +} + +// --------------------------------------------------------------------------- +// Init -- chained nccl -> internal -> none. Each step tries to bring up its +// resource; on failure it warns and recurses into the next step. +// --------------------------------------------------------------------------- +static void ggml_backend_cuda_comm_init_none(ggml_backend_cuda_comm_context * ret) { + ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_butterfly; +} + +static void ggml_backend_cuda_comm_init_internal(ggml_backend_cuda_comm_context * ret) { + ret->ar_pipeline = ggml_cuda_ar_pipeline_init(ret->dev_ids.data(), ret->dev_ids.size()); + if (ret->ar_pipeline) { + ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_internal; + return; + } + + // Clear sticky CUDA error from the failed init. + (void) cudaGetLastError(); + GGML_LOG_WARN("internal AllReduce init failed (n_devices != 2?); " + "falling back to meta-backend butterfly\n"); + ggml_backend_cuda_comm_init_none(ret); +} + +static void ggml_backend_cuda_comm_init_nccl(ggml_backend_cuda_comm_context * ret) { +#ifdef GGML_USE_NCCL + // Disabling NCCL path when CUDA virtual devices are in use since NCCL requires one distinct physical GPU per rank. + const ggml_cuda_device_info & info = ggml_cuda_info(); + if (info.device_count > info.physical_device_count) { + GGML_LOG_WARN("NCCL disabled: virtual devices in use; " + "falling back to internal AllReduce\n"); + ggml_backend_cuda_comm_init_internal(ret); + return; + } + + const size_t n = ret->dev_ids.size(); + ret->comms.resize(n); + ncclResult_t rc = ncclCommInitAll(ret->comms.data(), (int) n, ret->dev_ids.data()); + if (rc == ncclSuccess) { + ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_nccl; + return; + } + + ret->comms.clear(); + GGML_LOG_WARN("NCCL init failed (%s); falling back to internal AllReduce\n", + ncclGetErrorString(rc)); +#else // GGML_USE_NCCL +#ifndef GGML_USE_HIP + GGML_LOG_WARN("NCCL not compiled in; falling back to internal AllReduce. " + "Recompile with -DGGML_CUDA_NCCL=ON for best multi-GPU performance.\n"); +#endif // !GGML_USE_HIP +#endif // GGML_USE_NCCL + + ggml_backend_cuda_comm_init_internal(ret); +} + +// Top-level init. Picks one of the three init paths based on +// GGML_CUDA_ALLREDUCE (or the platform default) and lets the chain handle +// any fallback. Unrecognised env values warn and fall through to the +// platform default. +static void * ggml_backend_cuda_comm_init(ggml_backend_t * backends, size_t n_backends) { + for (size_t i = 0; i < n_backends; i++) { + if (!ggml_backend_is_cuda(backends[i])) { + return nullptr; + } + } + + auto * ret = new ggml_backend_cuda_comm_context; + ret->backends.assign(backends, backends + n_backends); + ret->dev_ids.reserve(n_backends); + for (size_t i = 0; i < n_backends; i++) { + ret->dev_ids.push_back(static_cast(backends[i]->context)->device); + } + + const char * env = getenv("GGML_CUDA_ALLREDUCE"); + if (!env) { + // Platform default: Linux uses NCCL, otherwise (generally Windows) internal +#if defined(__linux__) + ggml_backend_cuda_comm_init_nccl(ret); +#else + ggml_backend_cuda_comm_init_internal(ret); +#endif // defined(__linux__) + } else { + std::string env_str(env); + if (env_str == "nccl") { + ggml_backend_cuda_comm_init_nccl(ret); + } else if (env_str == "internal") { + ggml_backend_cuda_comm_init_internal(ret); + } else if (env_str == "none") { + ggml_backend_cuda_comm_init_none(ret); + } else { + GGML_LOG_WARN("unknown GGML_CUDA_ALLREDUCE value: %s\n", env); + ggml_backend_cuda_comm_init_none(ret); + } + } + + return ret; +} - auto result = buft_map.emplace(std::make_pair(main_device, tensor_split_arr), buft); - return &result.first->second; +// Top-level dispatch -- calls the function pointer chosen by comm_init. +// Returns false to let the meta-backend's butterfly run. +static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { + if (comm_ctx_v == nullptr) { + return false; + } + auto * comm_ctx = static_cast(comm_ctx_v); + return comm_ctx->try_allreduce(comm_ctx, tensors); } // host buffer type @@ -1309,7 +1320,7 @@ static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buff } static void * ggml_cuda_host_malloc(size_t size) { - if (getenv("GGML_CUDA_NO_PINNED") != nullptr) { + if (ggml_cuda_pinned_memory_disabled()) { return nullptr; } @@ -1341,15 +1352,23 @@ static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggm return buffer; } +static size_t ggml_backend_cuda_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + // very large pinned allocations can fail (e.g. on Windows) and fall back to pageable memory, smaller ones succeed + return 4ull*1024*1024*1024; +} + ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() { static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = { /* .iface = */ { - /* .get_name = */ ggml_backend_cuda_host_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, - /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, + /* .get_name = */ ggml_backend_cuda_host_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, + /* .get_max_size = */ ggml_backend_cuda_host_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, }, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cuda_reg(), 0), /* .context = */ nullptr, @@ -1370,572 +1389,6 @@ typedef void (*ggml_cuda_op_mul_mat_t)( const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream); -#ifndef GGML_CUDA_PEER_MAX_BATCH_SIZE -#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128 -#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE - -#define MUL_MAT_SRC1_COL_STRIDE 128 - -static cudaError_t ggml_cuda_cpy_tensor_2d( - void * dst, const struct ggml_tensor * src, int64_t i3, int64_t i2, int64_t i1_low, int64_t i1_high, cudaStream_t stream) { - - const char * src_ptr = (const char *) src->data; - char * dst_ptr = (char *) dst; - - const int64_t ne0 = src->ne[0]; - const int64_t nb0 = src->nb[0]; - const int64_t nb1 = src->nb[1]; - const int64_t nb2 = src->nb[2]; - const int64_t nb3 = src->nb[3]; - const enum ggml_type type = src->type; - const int64_t ts = ggml_type_size(type); - const int64_t bs = ggml_blck_size(type); - const int64_t i1_diff = i1_high - i1_low; - - const char * x = src_ptr + i1_low*nb1 + i2*nb2 + i3*nb3; - if (nb0 == ts && nb1 == ts*ne0/bs) { - return cudaMemcpyAsync(dst_ptr, x, i1_diff*nb1, cudaMemcpyDeviceToDevice, stream); - } else if (nb0 == ts) { - return cudaMemcpy2DAsync(dst_ptr, ts*ne0/bs, x, nb1, ts*ne0/bs, i1_diff, cudaMemcpyDeviceToDevice, stream); - } else { - for (int64_t i1 = 0; i1 < i1_diff; i1++) { - const void * rx = (const void *) ((const char *) x + i1*nb1); - void * rd = (void *) (dst_ptr + i1*ts*ne0/bs); - // pretend the row is a matrix with cols=1 - cudaError_t r = cudaMemcpy2DAsync(rd, ts/bs, rx, nb0, ts/bs, ne0, cudaMemcpyDeviceToDevice, stream); - if (r != cudaSuccess) { - return r; - } - } - return cudaSuccess; - } -} - -struct cublas_force_compute_type { - bool fp32 = false; - bool fp16 = false; -}; - -static const cublas_force_compute_type & ggml_cuda_cublas_get_force_compute_type() { - static const cublas_force_compute_type compute_type = [] { - cublas_force_compute_type result; - - const bool ggml_cuda_force_cublas_compute_32f_env = getenv("GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F") != nullptr; - const bool ggml_cuda_force_cublas_compute_16f_env = getenv("GGML_CUDA_FORCE_CUBLAS_COMPUTE_16F") != nullptr; - - GGML_ASSERT(ggml_cuda_force_cublas_compute_16f_env == false || ggml_cuda_force_cublas_compute_32f_env == false); - - if (ggml_cuda_force_cublas_compute_32f_env) { - GGML_LOG_INFO("Detected GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F\n"); - result.fp32 = true; - } else if (ggml_cuda_force_cublas_compute_16f_env) { - GGML_LOG_INFO("Detected GGML_CUDA_FORCE_CUBLAS_COMPUTE_16F\n"); - result.fp16 = true; - } - - return result; - }(); - - return compute_type; -} - -static void ggml_cuda_op_mul_mat_cublas( - ggml_backend_cuda_context & ctx, - const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, - const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, - const int64_t src1_padded_row_size, cudaStream_t stream) { - - GGML_ASSERT(src0_dd_i != nullptr); - GGML_ASSERT(src1_ddf_i != nullptr); - GGML_ASSERT(dst_dd_i != nullptr); - - const int64_t ne00 = src0->ne[0]; - const int64_t ne10 = src1->ne[0]; - - const int64_t ne0 = dst->ne[0]; - - const int64_t row_diff = row_high - row_low; - - int id = ggml_cuda_get_device(); - - // the main device has a larger memory buffer to hold the results from all GPUs - // ldc == nrows of the matrix that cuBLAS writes into - int64_t ldc = id == ctx.device ? ne0 : row_diff; - - const int cc = ggml_cuda_info().devices[id].cc; - - const bool supports_bf16 = GGML_CUDA_CC_IS_NVIDIA(cc) || GGML_CUDA_CC_IS_AMD(cc) || - (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2); - - const bool use_fp16 = - src0->type != GGML_TYPE_NVFP4 && - (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && - ggml_is_contiguous(src0) && - row_diff == src0->ne[1] && - dst->op_params[0] == GGML_PREC_DEFAULT; - - if (supports_bf16 && src0->type == GGML_TYPE_BF16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1]) { - ggml_cuda_pool_alloc src1_as_bf16(ctx.pool(id)); - if (src1->type != GGML_TYPE_BF16) { - const to_bf16_cuda_t to_bf16_cuda = ggml_get_to_bf16_cuda(src1->type); - GGML_ASSERT(to_bf16_cuda != nullptr); - size_t ne = src1_ncols*ne10; - src1_as_bf16.alloc(ne); - to_bf16_cuda(src1_ddf_i, src1_as_bf16.get(), ne, stream); - } - const nv_bfloat16 * src1_ptr = src1->type == GGML_TYPE_BF16 ? (const nv_bfloat16 *) src1_ddf_i : src1_as_bf16.get(); - const nv_bfloat16 * src0_ptr = (const nv_bfloat16 *)src0_dd_i; - ggml_cuda_pool_alloc dst_bf16(ctx.pool(id), row_diff*src1_ncols); - - const float alpha_f32 = 1.0f; - const float beta_f32 = 0.0f; - - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - CUBLAS_CHECK( - cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, - row_diff, src1_ncols, ne10, - &alpha_f32, src0_ptr, CUDA_R_16BF, ne00, - src1_ptr, CUDA_R_16BF, ne10, - &beta_f32, dst_bf16.get(), CUDA_R_16BF, ldc, - CUBLAS_COMPUTE_32F, - CUBLAS_GEMM_DEFAULT_TENSOR_OP)); - - const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_BF16); - to_fp32_cuda(dst_bf16.get(), dst_dd_i, row_diff*src1_ncols, stream); - } else if (fast_fp16_hardware_available(cc) && use_fp16) { - // convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32 - ggml_cuda_pool_alloc src0_as_f16(ctx.pool(id)); - if (src0->type != GGML_TYPE_F16) { - const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src0->type); - GGML_ASSERT(to_fp16_cuda != nullptr); - size_t ne = row_diff*ne00; - src0_as_f16.alloc(ne); - to_fp16_cuda(src0_dd_i, src0_as_f16.get(), ne, stream); - } - const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16.get(); - - ggml_cuda_pool_alloc src1_as_f16(ctx.pool(id)); - if (src1->type != GGML_TYPE_F16) { - const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type); - GGML_ASSERT(to_fp16_cuda != nullptr); - size_t ne = src1_ncols*ne10; - src1_as_f16.alloc(ne); - to_fp16_cuda(src1_ddf_i, src1_as_f16.get(), ne, stream); - } - const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16.get(); - - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - - const auto & force_compute_type = ggml_cuda_cublas_get_force_compute_type(); - - if (!force_compute_type.fp16 && (GGML_CUDA_CC_IS_CDNA(cc) - || GGML_CUDA_CC_IS_RDNA4(cc) - || cc == GGML_CUDA_CC_VOLTA - || force_compute_type.fp32)) - { - const float alpha = 1.0f; - const float beta = 0.0f; - CUBLAS_CHECK( - cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, - row_diff, src1_ncols, ne10, - &alpha, src0_ptr, CUDA_R_16F, ne00, - src1_ptr, CUDA_R_16F, ne10, - &beta, dst_dd_i, CUDA_R_32F, ldc, - CUBLAS_COMPUTE_32F, - CUBLAS_GEMM_DEFAULT_TENSOR_OP)); - } else { - ggml_cuda_pool_alloc dst_f16(ctx.pool(id), row_diff*src1_ncols); - - const half alpha_f16 = 1.0f; - const half beta_f16 = 0.0f; - - CUBLAS_CHECK( - cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, - row_diff, src1_ncols, ne10, - &alpha_f16, src0_ptr, CUDA_R_16F, ne00, - src1_ptr, CUDA_R_16F, ne10, - &beta_f16, dst_f16.get(), CUDA_R_16F, ldc, - CUBLAS_COMPUTE_16F, - CUBLAS_GEMM_DEFAULT_TENSOR_OP)); - - const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16); - to_fp32_cuda(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream); - } - } else { - ggml_cuda_pool_alloc src0_ddq_as_f32(ctx.pool(id)); - ggml_cuda_pool_alloc src1_ddq_as_f32(ctx.pool(id)); - - if (src0->type != GGML_TYPE_F32) { - const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src0->type); - GGML_ASSERT(to_fp32_cuda != nullptr); - src0_ddq_as_f32.alloc(row_diff*ne00); - to_fp32_cuda(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream); - } - if (src1->type != GGML_TYPE_F32) { - const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src1->type); - GGML_ASSERT(to_fp32_cuda != nullptr); - src1_ddq_as_f32.alloc(src1_ncols*ne10); - to_fp32_cuda(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream); - } - - const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get(); - const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get(); - - const float alpha = 1.0f; - const float beta = 0.0f; - - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - CUBLAS_CHECK( - cublasSgemm(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N, - row_diff, src1_ncols, ne10, - &alpha, src0_ddf_i, ne00, - src1_ddf1_i, ne10, - &beta, dst_dd_i, ldc)); - } - - GGML_UNUSED_VARS(dst, src1_ddq_i, src1_padded_row_size); -} - -static cudaError_t ggml_cuda_Memcpy2DPeerAsync( - void * dst, int dstDevice, size_t dpitch, void * src, int srcDevice, size_t spitch, size_t width, size_t height, cudaStream_t stream) { - -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices - cudaMemcpy3DPeerParms p = {}; - p.dstDevice = dstDevice; - p.dstPtr = make_cudaPitchedPtr(dst, dpitch, dpitch, height); - p.srcDevice = srcDevice; - p.srcPtr = make_cudaPitchedPtr(src, spitch, spitch, height); - p.extent = make_cudaExtent(width, height, 1); - return cudaMemcpy3DPeerAsync(&p, stream); -#else - // HIP does not support cudaMemcpy3DPeerAsync or vmm pools - GGML_UNUSED(dstDevice); - GGML_UNUSED(srcDevice); - return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, stream); -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) -} - -static void ggml_cuda_op_mul_mat( - ggml_backend_cuda_context & ctx, - const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op, - quantize_cuda_t quantize_src1) { - - const int64_t ne00 = src0->ne[0]; - const int64_t ne01 = src0->ne[1]; - const int64_t ne02 = src0->ne[2]; - const int64_t ne03 = src0->ne[3]; - - const int64_t ne10 = src1->ne[0]; - const int64_t ne11 = src1->ne[1]; - const int64_t ne12 = src1->ne[2]; - const int64_t ne13 = src1->ne[3]; - const int64_t nrows1 = ggml_nrows(src1); - - const int64_t ne0 = dst->ne[0]; - const int64_t ne1 = dst->ne[1]; - - // const int64_t nb10 = src1->nb[0]; - const int64_t nb11 = src1->nb[1]; - const int64_t nb12 = src1->nb[2]; - const int64_t nb13 = src1->nb[3]; - - const int64_t nb2 = dst->nb[2]; - const int64_t nb3 = dst->nb[3]; - - ggml_backend_cuda_buffer_context * src1_ctx = (ggml_backend_cuda_buffer_context *) src1->buffer->context; - ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *) dst->buffer->context; - - GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1)); - - GGML_ASSERT(ne12 % ne02 == 0); - GGML_ASSERT(ne13 % ne03 == 0); - - const int64_t i02_divisor = ne12 / ne02; - const int64_t i03_divisor = ne13 / ne03; - - const size_t src0_ts = ggml_type_size(src0->type); - const size_t src0_bs = ggml_blck_size(src0->type); - const size_t q8_1_ts = sizeof(block_q8_1); - const size_t q8_1_bs = QK8_1; - - const bool src0_is_contiguous = ggml_is_contiguous(src0); - const bool src1_is_contiguous = ggml_is_contiguous(src1); - - const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING); - - const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft); - GGML_ASSERT(!(split && ne02 > 1)); - GGML_ASSERT(!(split && ne03 > 1)); - GGML_ASSERT(!(split && ne02 < ne12)); - GGML_ASSERT(!(split && ne03 < ne13)); - - ggml_tensor_extra_gpu * src0_extra = split ? (ggml_tensor_extra_gpu *) src0->extra : nullptr; - - - std::array tensor_split; - if (split) { - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; - tensor_split = buft_ctx->tensor_split; - } - - struct dev_data { - int cc; - - ggml_cuda_pool_alloc src0_dd_alloc; - ggml_cuda_pool_alloc src1_ddf_alloc; - ggml_cuda_pool_alloc src1_ddq_alloc; - ggml_cuda_pool_alloc dst_dd_alloc; - - char * src0_dd = nullptr; - float * src1_ddf = nullptr; // float - char * src1_ddq = nullptr; // q8_1 - float * dst_dd = nullptr; - - int64_t row_low; - int64_t row_high; - }; - - dev_data dev[GGML_CUDA_MAX_DEVICES]; - - int used_devices = 0; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - dev[id].cc = ggml_cuda_info().devices[id].cc; - - // by default, use all rows - dev[id].row_low = 0; - dev[id].row_high = ne01; - - // for multi GPU, get the row boundaries from tensor split - // and round to mul_mat_q tile sizes - if (split) { - const int64_t rounding = get_row_rounding(tensor_split); - - if (id != 0) { - dev[id].row_low = ne01*tensor_split[id]; - if (dev[id].row_low < ne01) { - dev[id].row_low -= dev[id].row_low % rounding; - } - } - - if (id != ggml_backend_cuda_get_device_count() - 1) { - dev[id].row_high = ne01*tensor_split[id + 1]; - if (dev[id].row_high < ne01) { - dev[id].row_high -= dev[id].row_high % rounding; - } - } - } - } - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) { - continue; - } - - used_devices++; - - const bool src1_on_device = id == src1_ctx->device; - const bool dst_on_device = id == dst_ctx->device; - - ggml_cuda_set_device(id); - cudaStream_t stream = ctx.stream(id, 0); - - if (src0_is_contiguous) { - dev[id].src0_dd = split ? (char *) src0_extra->data_device[id] : (char *) src0->data; - } else { - // If src0 is not contiguous it will be copied to a temporary buffer. - // This buffer needs to be cleared entirely because multiple regions will function as padding. - const size_t nbytes_data = ggml_nbytes(src0); - const size_t nbytes_padding = ggml_row_size(src0->type, MATRIX_ROW_PADDING - ne00 % MATRIX_ROW_PADDING); - dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ctx.pool(id), nbytes_data + nbytes_padding); - CUDA_CHECK(cudaMemsetAsync(dev[id].src0_dd, 0, nbytes_data + nbytes_padding, stream)); - } - - // If src0 is on a temporary compute buffer (partial offloading) there may be some padding that needs to be cleared: - if (ne00 % MATRIX_ROW_PADDING != 0 && ggml_is_quantized(src0->type) && ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && src0->view_src == nullptr) { - GGML_ASSERT(ggml_is_contiguously_allocated(src0)); - GGML_ASSERT(!src0->view_src); - const size_t nbytes_data = ggml_row_size(src0->type, (dev[id].row_high - dev[id].row_low)*ne00); - const size_t nbytes_padding = ggml_row_size(src0->type, MATRIX_ROW_PADDING - ne00 % MATRIX_ROW_PADDING); - CUDA_CHECK(cudaMemsetAsync(dev[id].src0_dd + nbytes_data, 0, nbytes_padding, stream)); - } - - if (src1_on_device && src1_is_contiguous) { - dev[id].src1_ddf = (float *) src1->data; - } else { - dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ctx.pool(id), ggml_nelements(src1)); - } - - if (quantize_src1) { - size_t src_1_ddq_size = nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs; - if (quantize_src1 == quantize_mmq_q8_1_cuda) { - src_1_ddq_size += get_mmq_x_max_host(dev[id].cc)*sizeof(block_q8_1_mmq); - } - dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(ctx.pool(id), src_1_ddq_size); - - if (src1_on_device && src1_is_contiguous) { - quantize_src1( - dev[id].src1_ddf, nullptr, dev[id].src1_ddq, src0->type, ne10, - nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float), - src1_padded_col_size, ne11, ne12, ne13, stream); - CUDA_CHECK(cudaGetLastError()); - } - } - - if (dst_on_device) { - dev[id].dst_dd = (float *) dst->data; - } else { - const size_t size_dst_ddf = split ? (dev[id].row_high - dev[id].row_low)*ne1 : ggml_nelements(dst); - dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(ctx.pool(id), size_dst_ddf); - } - } - - // if multiple devices are used they need to wait for the main device - // here an event is recorded that signals that the main device has finished calculating the input data - if (split && used_devices > 1) { - ggml_cuda_set_device(ctx.device); - CUDA_CHECK(cudaEventRecord(src0_extra->events[ctx.device][0], ctx.stream())); - } - - const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11; - for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) { - const int64_t is = split ? (src1_col_0/src1_col_stride) % GGML_CUDA_MAX_STREAMS : 0; - const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride; - - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) { - continue; - } - - const bool src1_on_device = id == src1_ctx->device; - const bool dst_on_device = id == dst_ctx->device; - const int64_t row_diff = dev[id].row_high - dev[id].row_low; - - ggml_cuda_set_device(id); - cudaStream_t stream = ctx.stream(id, is); - - // wait for main GPU data if necessary - if (split && (id != ctx.device || is != 0)) { - CUDA_CHECK(cudaStreamWaitEvent(stream, src0_extra->events[ctx.device][0], 0)); - } - - for (int64_t i0 = 0; i0 < ne13*ne12; ++i0) { - const int64_t i03 = i0 / ne12; - const int64_t i02 = i0 % ne12; - - size_t src1_ddq_i_offset = i0*ne11 * src1_padded_col_size*q8_1_ts/q8_1_bs; - if (quantize_src1 == quantize_mmq_q8_1_cuda) { - src1_ddq_i_offset += src1_col_0 * sizeof(block_q8_1_mmq); - } else { - src1_ddq_i_offset += src1_col_0 * src1_padded_col_size*q8_1_ts/q8_1_bs; - } - - // for split tensors the data begins at i0 == i0_offset_low - const size_t nbytes_src0_matrix = ne01*ne00*src0_ts / src0_bs; - char * src0_dd_i = dev[id].src0_dd + ((i03/i03_divisor)*ne02 + (i02/i02_divisor)) * nbytes_src0_matrix; - float * src1_ddf_i = dev[id].src1_ddf + (i0*ne11 + src1_col_0) * ne10; - char * src1_ddq_i = dev[id].src1_ddq + src1_ddq_i_offset; - float * dst_dd_i = dev[id].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff); - - // the main device memory buffer can be on VRAM scratch, with space for all partial results - // in that case an offset on dst_ddf_i is needed - if (id == ctx.device) { - dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split - } - - // copy src0, src1 to device if necessary - if (src1_is_contiguous) { - if (id != ctx.device) { - if (quantize_src1) { - char * src1_ddq_i_source = dev[ctx.device].src1_ddq + src1_ddq_i_offset; - if (quantize_src1 == quantize_mmq_q8_1_cuda) { - const size_t pitch = ne11*sizeof(block_q8_1_mmq); - const size_t width = src1_ncols*sizeof(block_q8_1_mmq); - const size_t height = src1_padded_col_size/(4*QK8_1); - CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(src1_ddq_i, id, pitch, src1_ddq_i_source, ctx.device, pitch, width, height, stream)); - } else { - CUDA_CHECK(cudaMemcpyPeerAsync( - src1_ddq_i, id, src1_ddq_i_source, ctx.device, src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream)); - } - } else { - float * src1_ddf_i_source = (float *) src1->data; - src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10; - CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, ctx.device, - src1_ncols*ne10*sizeof(float), stream)); - } - } - } else if (src1_on_device && !src1_is_contiguous) { - CUDA_CHECK(ggml_cuda_cpy_tensor_2d( - src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream)); - } else { - GGML_ABORT("fatal error"); - } - - if (quantize_src1 && !src1_is_contiguous) { - quantize_src1( - src1_ddf_i, nullptr, src1_ddq_i, src0->type, ne10, ne10, ne11*ne10, ne12*ne11*ne10, - src1_padded_col_size, src1_ncols, 1, 1, stream); - CUDA_CHECK(cudaGetLastError()); - } - - if (src1_col_0 == 0 && !src0_is_contiguous && i03 % i03_divisor == 0 && i02 % i02_divisor == 0) { - CUDA_CHECK(ggml_cuda_cpy_tensor_2d( - src0_dd_i, src0, i03/i03_divisor, i02/i02_divisor, dev[id].row_low, dev[id].row_high, stream)); - } - - // do the computation - op(ctx, src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i, - dev[id].row_low, dev[id].row_high, src1_ncols, src1_padded_col_size, stream); - CUDA_CHECK(cudaGetLastError()); - - // copy dst to host or other device if necessary - if (!dst_on_device) { - void * dst_off_device = dst->data; - if (split) { - // src0 = weight matrix is saved as a transposed matrix for better memory layout. - // dst is NOT transposed. - // The outputs of matrix matrix multiplications can therefore NOT simply be concatenated for >1 GPU. - // Instead they need to be copied to the correct slice in ne0 = dst row index. - // If dst is a vector with ne0 == 1 then you don't have to do this but it still produces correct results. - float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3); - GGML_ASSERT(dst->nb[1] == ne0*sizeof(float)); - dhf_dst_i += src1_col_0*ne0 + dev[id].row_low; - CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync( - dhf_dst_i, ctx.device, ne0*sizeof(float), dst_dd_i, id, row_diff*sizeof(float), row_diff*sizeof(float), src1_ncols, stream)); - } else { - float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3); - GGML_ASSERT(dst->nb[1] == ne0*sizeof(float)); - dhf_dst_i += src1_col_0*ne0; - CUDA_CHECK(cudaMemcpyAsync(dhf_dst_i, dst_dd_i, src1_ncols*ne0*sizeof(float), cudaMemcpyDeviceToDevice, stream)); - } - } - - // add event for the main device to wait on until other device is done - if (split && (id != ctx.device || is != 0)) { - CUDA_CHECK(cudaEventRecord(src0_extra->events[id][is], stream)); - } - } - } - } - - // main device waits for all other devices to be finished - if (split && ggml_backend_cuda_get_device_count() > 1) { - int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE; - is_max = is_max <= GGML_CUDA_MAX_STREAMS ? is_max : GGML_CUDA_MAX_STREAMS; - - ggml_cuda_set_device(ctx.device); - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - if (dev[id].row_low == dev[id].row_high) { - continue; - } - for (int64_t is = 0; is < is_max; ++is) { - CUDA_CHECK(cudaStreamWaitEvent(ctx.stream(), src0_extra->events[id][is], 0)); - } - } - } -} - static __global__ void k_compute_batched_ptrs( const void * src0_as_f16, const void * src1_as_f16, char * dst, const void ** ptrs_src, void ** ptrs_dst, @@ -1974,7 +1427,8 @@ struct batched_mul_mat_traits { static inline const float beta = 0.0f; static inline const void* get_alpha() { static const float val = alpha; return &val; } static inline const void* get_beta() { static const float val = beta; return &val; } - static inline auto get_nc_converter(ggml_type src_type) { return ggml_get_to_fp32_nc_cuda(src_type); } + static inline auto convert(ggml_type src_type) { return ggml_get_to_fp32_cuda(src_type); } + static inline auto convert_nc(ggml_type src_type) { return ggml_get_to_fp32_nc_cuda(src_type); } }; template<> @@ -1987,7 +1441,8 @@ struct batched_mul_mat_traits { static inline const float beta = 0.0f; static inline const void* get_alpha() { static const float val = alpha; return &val; } static inline const void* get_beta() { static const float val = beta; return &val; } - static inline auto get_nc_converter(ggml_type src_type) { return ggml_get_to_bf16_nc_cuda(src_type); } + static inline auto convert(ggml_type src_type) { return ggml_get_to_bf16_cuda(src_type); } + static inline auto convert_nc(ggml_type src_type) { return ggml_get_to_bf16_nc_cuda(src_type); } }; template<> @@ -2000,35 +1455,39 @@ struct batched_mul_mat_traits { static inline const half beta = 0.0; static inline const void* get_alpha() { static const half val = alpha; return &val; } static inline const void* get_beta() { static const half val = beta; return &val; } - static inline auto get_nc_converter(ggml_type src_type) { return ggml_get_to_fp16_nc_cuda(src_type); } + static inline auto convert(ggml_type src_type) { return ggml_get_to_fp16_cuda(src_type); } + static inline auto convert_nc(ggml_type src_type) { return ggml_get_to_fp16_nc_cuda(src_type); } }; -template -static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - using traits = batched_mul_mat_traits; +template +static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + using traits = batched_mul_mat_traits; using cuda_t = typename traits::cuda_type; - GGML_ASSERT(!ggml_is_transposed(src0)); - GGML_ASSERT(!ggml_is_transposed(src1)); - GGML_ASSERT(!ggml_backend_buft_is_cuda_split(src0->buffer->buft)); - GGML_ASSERT(src0->type == src0_type); - GGML_ASSERT(ggml_is_contiguous(dst)); - - // Byte offsets and tensor dimensions are currently used in an inconsistent way for dst. - // As long as dst is contiguous this does not matter though. + // F32 chunks can have padding between rows to preserve the original destination stride. + GGML_ASSERT(ggml_is_contiguous(dst) || + (compute_type == GGML_TYPE_F32 && ggml_is_contiguous_1(dst))); GGML_TENSOR_BINARY_OP_LOCALS + const int64_t ldc = nb1 / sizeof(float); const int64_t ne_dst = ggml_nelements(dst); cudaStream_t main_stream = ctx.stream(); - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream)); + cublasHandle_t cublas_h = ctx.cublas_handle(); + + const size_t src0_ts = ggml_type_size(src0->type); + GGML_ASSERT(nb00 == src0_ts); + int64_t s01 = nb01 / src0_ts; + int64_t s02 = nb02 / src0_ts; + int64_t s03 = nb03 / src0_ts; + + const size_t src1_ts = ggml_type_size(src1->type); + GGML_ASSERT(nb10 == src1_ts); + int64_t s11 = nb11 / src1_ts; + int64_t s12 = nb12 / src1_ts; + int64_t s13 = nb13 / src1_ts; float * dst_ddf = (float *) dst->data; - const size_t ts_src1 = ggml_type_size(src1->type); - GGML_ASSERT(nb10 == ts_src1); - int64_t s11 = nb11 / ts_src1; - int64_t s12 = nb12 / ts_src1; - int64_t s13 = nb13 / ts_src1; const cuda_t * src0_ptr = nullptr; const cuda_t * src1_ptr = nullptr; @@ -2039,31 +1498,58 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct bool is_src0_cont_2 = ggml_is_contiguous_2(src0); bool is_src1_cont_2 = ggml_is_contiguous_2(src1); - // Handle src0 - src0_ptr = (const cuda_t *) src0->data; + if (src0->type == compute_type) { + src0_ptr = (const cuda_t *) src0->data; + } else { + src0_alloc.alloc(ggml_nelements(src0)); + + if (ggml_is_contiguously_allocated(src0)) { + const auto convert_func = traits::convert(src0->type); + GGML_ASSERT(convert_func != nullptr); + convert_func(src0->data, src0_alloc.get(), ggml_nelements(src0), main_stream); + const size_t src0_bs = ggml_blck_size(src0->type); + s01 *= src0_bs; + s02 *= src0_bs; + s03 *= src0_bs; + } else { + const auto convert_func = traits::convert_nc(src0->type); + GGML_ASSERT(convert_func != nullptr); + convert_func(src0->data, src0_alloc.get(), ne00, ne01, ne02, ne03, s01, s02, s03, main_stream); + s01 = ne00; + s02 = ne01*s01; + s03 = ne02*s02; + is_src0_cont_2 = true; + } + src0_ptr = src0_alloc.get(); + } - // Handle src1 - convert if necessary - if (src1->type == src0_type) { + if (src1->type == compute_type) { src1_ptr = (const cuda_t *) src1->data; } else { - // Convert src1 to target type using traits conversion functions - const int64_t ne_src1 = ggml_nelements(src1); - src1_alloc.alloc(ne_src1); - - const auto convert_func = traits::get_nc_converter(src1->type); - GGML_ASSERT(convert_func != nullptr); - convert_func(src1->data, src1_alloc.get(), ne10, ne11, ne12, ne13, s11, s12, s13, main_stream); + src1_alloc.alloc(ggml_nelements(src1)); + + if (ggml_is_contiguously_allocated(src1)) { + const auto convert_func = traits::convert(src1->type); + GGML_ASSERT(convert_func != nullptr); + convert_func(src1->data, src1_alloc.get(), ggml_nelements(src1), main_stream); + const size_t src1_bs = ggml_blck_size(src1->type); + s11 *= src1_bs; + s12 *= src1_bs; + s13 *= src1_bs; + } else { + const auto convert_func = traits::convert_nc(src1->type); + GGML_ASSERT(convert_func != nullptr); + convert_func(src1->data, src1_alloc.get(), ne10, ne11, ne12, ne13, s11, s12, s13, main_stream); + s11 = ne10; + s12 = ne11*s11; + s13 = ne12*s12; + is_src1_cont_2 = true; + } src1_ptr = src1_alloc.get(); - s11 = ne10; - s12 = ne11*s11; - s13 = ne12*s12; - - is_src1_cont_2 = true; } - // Setup destination buffer ggml_cuda_pool_alloc dst_temp(ctx.pool()); - char * dst_t; + char * dst_ptr; size_t nbd2 = dst->nb[2]; size_t nbd3 = dst->nb[3]; @@ -2074,35 +1560,28 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct const void * alpha = traits::get_alpha(); const void * beta = traits::get_beta(); - const auto & force_compute_type = ggml_cuda_cublas_get_force_compute_type(); - - int id = ggml_cuda_get_device(); - const int cc = ggml_cuda_info().devices[id].cc; - static constexpr bool is_src0_type_f16 = src0_type == GGML_TYPE_F16; - - // bf16 and fp32 are already being computed in fp32 (ensure it using static_assert), - // so checking necessity of forced fp32 only for fp16 src0_type - static_assert(is_src0_type_f16 || traits::compute_type == CUBLAS_COMPUTE_32F); - - const bool need_compute_32f = is_src0_type_f16 && !force_compute_type.fp16 && (GGML_CUDA_CC_IS_CDNA(cc) - || GGML_CUDA_CC_IS_RDNA4(cc) - || cc == GGML_CUDA_CC_VOLTA - || force_compute_type.fp32); + const int cc = ggml_cuda_info().devices[ctx.device].cc; + bool prefer_f32_output = false; + if (compute_type == GGML_TYPE_F16) { + prefer_f32_output = cc == GGML_CUDA_CC_VOLTA || GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_CDNA(cc); + } else if (compute_type == GGML_TYPE_BF16) { + prefer_f32_output = !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_CDNA(cc); + } - if (dst->op_params[0] == GGML_PREC_DEFAULT && !need_compute_32f) { - if constexpr (src0_type == GGML_TYPE_F32) { - dst_t = (char *) dst_ddf; // Direct F32 output - } else { - dst_t = (char *) dst_temp.alloc(ne_dst); - nbd2 /= sizeof(float) / sizeof(cuda_t); - nbd3 /= sizeof(float) / sizeof(cuda_t); - } - } else { - dst_t = (char *) dst_ddf; + if (prefer_f32_output) { + dst_ptr = (char *) dst_ddf; cu_compute_type = batched_mul_mat_traits::compute_type; cu_data_type = batched_mul_mat_traits::data_type; alpha = batched_mul_mat_traits::get_alpha(); beta = batched_mul_mat_traits::get_beta(); + } else { + if constexpr (compute_type == GGML_TYPE_F32) { + dst_ptr = (char *) dst_ddf; // Direct F32 output + } else { + dst_ptr = (char *) dst_temp.alloc(ne_dst); + nbd2 /= sizeof(float) / sizeof(cuda_t); + nbd3 /= sizeof(float) / sizeof(cuda_t); + } } GGML_ASSERT(ne12 % ne02 == 0); @@ -2112,19 +1591,38 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct const int64_t r2 = ne12/ne02; const int64_t r3 = ne13/ne03; - if (r2 == 1 && r3 == 1 && is_src0_cont_2 && is_src1_cont_2) { + // Theoretically cublasGemmStridedBatchedEx would always work, even for a single matrix. + // However, for some old NVIDIA and AMD GPUs the strided/Ex GEMM is much slower, + // probably because the internal kernel selection logic is suboptimal. + if (compute_type == GGML_TYPE_F32 && ne12 == 1 && ne13 == 1) { + CUBLAS_CHECK( + cublasSgemm(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, + ne01, ne11, ne10, + (const float *) alpha, (const float *) src0_ptr, s01, + (const float *) src1_ptr, s11, + (const float *) beta, (float *) dst_ptr, ldc)); + } else if (ne12 == 1 && ne13 == 1) { + CUBLAS_CHECK( + cublasGemmEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, + ne01, ne11, ne10, + alpha, src0_ptr, cu_data_type_a, s01, + src1_ptr, cu_data_type_b, s11, + beta, dst_ptr, cu_data_type, ldc, + cu_compute_type, + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + } else if (r2 == 1 && r3 == 1 && is_src0_cont_2 && is_src1_cont_2) { // with a [0, 2, 1, 3] perm. and ne02==1 the matrix strides need to be determined from dim 3: - const int64_t sma = ne02 == 1 ? nb03/nb00 : nb02/nb00; - const int64_t smb = ne12 == 1 ? s13 : s12; + const int64_t sma = ne02 == 1 ? s03 : s02; + const int64_t smb = ne12 == 1 ? s13 : s12; // there is no broadcast and src0, src1 are contiguous across dims 2, 3 // use cublasGemmStridedBatchedEx CUBLAS_CHECK( - cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmStridedBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, - alpha, src0_ptr, cu_data_type_a, nb01/nb00, sma, // strideA - src1_ptr, cu_data_type_b, s11, smb, // strideB - beta, dst_t, cu_data_type, ne0, ne1*ne0, // strideC + alpha, src0_ptr, cu_data_type_a, s01, sma, // strideA + src1_ptr, cu_data_type_b, s11, smb, // strideB + beta, dst_ptr, cu_data_type, ldc, ne1*ldc, // strideC ne12*ne13, cu_compute_type, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); @@ -2135,62 +1633,124 @@ static void ggml_cuda_mul_mat_batched_cublas_impl(ggml_backend_cuda_context & ct ggml_cuda_pool_alloc ptrs_src(ctx.pool(), 2*ne23); ggml_cuda_pool_alloc< void *> ptrs_dst(ctx.pool(), 1*ne23); - size_t src1_stride_size = sizeof(cuda_t); + const size_t src_type_size = sizeof(cuda_t); const int threads_x = 16; const int threads_y = 16; - dim3 block_dims(threads_x, threads_y); + const dim3 block_dims(threads_x, threads_y); - dim3 grid_dims( + const dim3 grid_dims( (ne13 + threads_x - 1) / threads_x, (ne12 + threads_y - 1) / threads_y ); k_compute_batched_ptrs<<>>( - src0_ptr, src1_ptr, dst_t, + src0_ptr, src1_ptr, dst_ptr, ptrs_src.get(), ptrs_dst.get(), ne12, ne13, ne23, - nb02, nb03, - (src1->type == src0_type) ? nb12 : s12*src1_stride_size, - (src1->type == src0_type) ? nb13 : s13*src1_stride_size, + s02*src_type_size, s03*src_type_size, + s12*src_type_size, s13*src_type_size, nbd2, nbd3, r2, r3); CUDA_CHECK(cudaGetLastError()); CUBLAS_CHECK( - cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, + cublasGemmBatchedEx(cublas_h, CUBLAS_OP_T, CUBLAS_OP_N, ne01, ne11, ne10, - alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, nb01/nb00, + alpha, (const void **) (ptrs_src.get() + 0*ne23), cu_data_type_a, s01, (const void **) (ptrs_src.get() + 1*ne23), cu_data_type_b, s11, - beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ne0, + beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ldc, ne23, cu_compute_type, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); } // Convert output back to F32 if needed - if (dst->op_params[0] == GGML_PREC_DEFAULT && cu_data_type != CUDA_R_32F) { + if (cu_data_type != CUDA_R_32F) { const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(traits::ggml_type_val); to_fp32_cuda(dst_temp.get(), dst_ddf, ne_dst, main_stream); } } -static void ggml_cuda_mul_mat_batched_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 || src0->type == GGML_TYPE_F32); +static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const int cc = ggml_cuda_info().devices[ctx.device].cc; + const ggml_prec prec = (ggml_prec) ggml_get_op_params_i32(dst, 0); + ggml_type compute_type = src0->type; + if (ggml_is_quantized(compute_type)) { + compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32; + } else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) { + compute_type = GGML_TYPE_F32; + } else if (compute_type == GGML_TYPE_BF16 && !fast_bf16_hardware_available(cc)) { + if (GGML_CUDA_CC_IS_AMD(cc) && src1->ne[1] > 32) { + compute_type = GGML_TYPE_F32; + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && src1->ne[1] > (cc >= GGML_CUDA_CC_VOLTA ? 8 : 128)) { + compute_type = GGML_TYPE_F32; + } + } + // F16 is the only compute type that can not satisfy a request for BF16 + if (prec == GGML_PREC_BF16 && compute_type == GGML_TYPE_F16) { + compute_type = fast_bf16_hardware_available(cc) ? GGML_TYPE_BF16 : GGML_TYPE_F32; + } else if (prec == GGML_PREC_F32) { + compute_type = GGML_TYPE_F32; + } + + const char * env_c = getenv("GGML_CUDA_CUBLAS_COMPUTE_TYPE"); + if (env_c != nullptr) { + std::string env_cpp = env_c; + for (char & c : env_cpp) { + c = std::tolower(c); + } + if (env_cpp == "f32" || env_cpp == "fp32") { + compute_type = GGML_TYPE_F32; + } else if (env_cpp == "f16" || env_cpp == "fp16") { + compute_type = GGML_TYPE_F16; + } else if (env_cpp == "bf16") { + compute_type = GGML_TYPE_BF16; + } else if (env_cpp != "auto") { + GGML_LOG_WARN("%s: unknown value for GGML_CUDA_CUBLAS_COMPUTE_TYPE: %s", __func__, env_cpp.c_str()); + } + } + + constexpr size_t max_src0_convert_size = 512ull * 1024 * 1024; + const size_t src0_f32_size = ggml_nelements(src0) * sizeof(float); + + if (compute_type == GGML_TYPE_F32 && + (src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) && + src0_f32_size > max_src0_convert_size) { + const size_t f32_row_size = src0_f32_size / src0->ne[1]; + const int64_t rows_per_chunk = std::max(1, (int64_t) (max_src0_convert_size / f32_row_size)); + + if (rows_per_chunk < src0->ne[1]) { + ggml_tensor src0_chunk = *src0; + ggml_tensor dst_chunk = *dst; - switch (src0->type) { + for (int64_t i01 = 0; i01 < src0->ne[1]; i01 += rows_per_chunk) { + src0_chunk.ne[1] = std::min(rows_per_chunk, src0->ne[1] - i01); + src0_chunk.data = (char *) src0->data + i01*src0->nb[1]; + + dst_chunk.ne[0] = src0_chunk.ne[1]; + dst_chunk.data = (char *) dst->data + i01*dst->nb[0]; + + ggml_cuda_mul_mat_cublas_impl(ctx, &src0_chunk, src1, &dst_chunk); + } + return; + } + } + + switch (compute_type) { case GGML_TYPE_F32: - ggml_cuda_mul_mat_batched_cublas_impl(ctx, src0, src1, dst); + ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); break; case GGML_TYPE_BF16: - ggml_cuda_mul_mat_batched_cublas_impl(ctx, src0, src1, dst); + ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); break; case GGML_TYPE_F16: - ggml_cuda_mul_mat_batched_cublas_impl(ctx, src0, src1, dst); + ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst); break; default: - GGML_ABORT("Unsupported type"); + GGML_ABORT("fatal error"); } } @@ -2198,12 +1758,18 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, const ggml_tensor * ffn_gate, const ggml_tensor * glu, const ggml_tensor * ffn_up_bias = nullptr, - const ggml_tensor * ffn_gate_bias = nullptr) { + const ggml_tensor * ffn_gate_bias = nullptr, + const ggml_tensor * ffn_up_scale = nullptr, + const ggml_tensor * ffn_gate_scale = nullptr) { const bool has_bias = ffn_up_bias != nullptr || ffn_gate_bias != nullptr; + const bool has_scale = ffn_up_scale != nullptr || ffn_gate_scale != nullptr; if (has_bias && (!ffn_up_bias || !ffn_gate_bias)) { return false; } + if (has_scale && (!ffn_up_scale || !ffn_gate_scale)) { + return false; + } const bool is_mul_mat = ffn_up->op == GGML_OP_MUL_MAT && ffn_gate->op == GGML_OP_MUL_MAT && glu->op == GGML_OP_GLU; const bool is_mul_mat_id = ffn_up->op == GGML_OP_MUL_MAT_ID && ffn_gate->op == GGML_OP_MUL_MAT_ID && glu->op == GGML_OP_GLU; @@ -2215,34 +1781,45 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, } const ggml_op expected_bias_op = is_mul_mat ? GGML_OP_ADD : GGML_OP_ADD_ID; + const ggml_tensor * ffn_up_bias_src = has_scale ? ffn_up_scale : ffn_up; + const ggml_tensor * ffn_gate_bias_src = has_scale ? ffn_gate_scale : ffn_gate; + const ggml_tensor * ffn_up_out = has_bias ? ffn_up_bias : ffn_up_bias_src; + const ggml_tensor * ffn_gate_out = has_bias ? ffn_gate_bias : ffn_gate_bias_src; - if (has_bias) { - if (ffn_up_bias->op != expected_bias_op || ffn_gate_bias->op != expected_bias_op) { + if (glu->src[0] != ffn_gate_out || glu->src[1] != ffn_up_out) { + return false; + } + + if (has_scale) { + if (ffn_up_scale->op != GGML_OP_MUL || ffn_gate_scale->op != GGML_OP_MUL) { + return false; + } + const bool up_has_mm = ffn_up_scale->src[0] == ffn_up || ffn_up_scale->src[1] == ffn_up; + const bool gate_has_mm = ffn_gate_scale->src[0] == ffn_gate || ffn_gate_scale->src[1] == ffn_gate; + if (!up_has_mm || !gate_has_mm) { return false; } + } - if (glu->src[0] != ffn_gate_bias || glu->src[1] != ffn_up_bias) { + if (has_bias) { + if (ffn_up_bias->op != expected_bias_op || ffn_gate_bias->op != expected_bias_op) { return false; } if (expected_bias_op == GGML_OP_ADD) { - const bool up_has_mul = ffn_up_bias->src[0] == ffn_up || ffn_up_bias->src[1] == ffn_up; - const bool gate_has_mul = ffn_gate_bias->src[0] == ffn_gate || ffn_gate_bias->src[1] == ffn_gate; + const bool up_has_mul = ffn_up_bias->src[0] == ffn_up_bias_src || ffn_up_bias->src[1] == ffn_up_bias_src; + const bool gate_has_mul = ffn_gate_bias->src[0] == ffn_gate_bias_src || ffn_gate_bias->src[1] == ffn_gate_bias_src; if (!up_has_mul || !gate_has_mul) { return false; } } else { // GGML_OP_ADD_ID - if (ffn_up_bias->src[0] != ffn_up || ffn_gate_bias->src[0] != ffn_gate) { + if (ffn_up_bias->src[0] != ffn_up_bias_src || ffn_gate_bias->src[0] != ffn_gate_bias_src) { return false; } if (ffn_up_bias->src[2] != ffn_up->src[2] || ffn_gate_bias->src[2] != ffn_gate->src[2]) { return false; } } - } else { - if (glu->src[0] != ffn_gate && glu->src[1] != ffn_up) { - return false; - } } if (ffn_up->src[0]->type != ffn_gate->src[0]->type || !ggml_are_same_shape(ffn_up->src[0], ffn_gate->src[0]) || @@ -2254,11 +1831,11 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, return false; } - if (ffn_up->src[2] && (ffn_up->src[2] != ffn_gate->src[2])) { + if (is_mul_mat_id && ffn_up->src[2] != ffn_gate->src[2]) { return false; } - static constexpr std::array valid_glu_ops = { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_OAI }; + static constexpr std::array valid_glu_ops = { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU, GGML_GLU_OP_SWIGLU_OAI, GGML_GLU_OP_SWIGLU_CLAMP }; if (std::find(valid_glu_ops.begin(), valid_glu_ops.end(), ggml_get_glu_op(glu)) == valid_glu_ops.end()) { return false; @@ -2268,14 +1845,6 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, return false; } - const bool split = ggml_backend_buft_is_cuda_split(ffn_up->src[0]->buffer->buft) || - ggml_backend_buft_is_cuda_split(ffn_gate->src[0]->buffer->buft); - - //TODO: add support for fusion for split buffers - if (split) { - return false; - } - return true; } @@ -2290,16 +1859,9 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) { (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]); - - const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft) || - ggml_backend_buft_is_cuda_split(src1->buffer->buft); - - //TODO: add support for fusion for split buffers - if (split) { - return false; - } + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; + use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]); //we only support fusion for ncols_dst = 1 if (tensor->op == GGML_OP_MUL_MAT && dst->ne[1] != 1) { @@ -2336,105 +1898,206 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) { return false; } - if (tensor->op == GGML_OP_MUL_MAT_ID && dst->ne[2] != 1) { + if (tensor->op == GGML_OP_MUL_MAT_ID && dst->ne[2] > get_mmvq_mmid_max_batch(src0->type, cc)) { + return false; + } + + return use_mul_mat_vec_q; +} + +static bool ggml_cuda_match_shared_expert(const ggml_cgraph * graph, int routed_idx, int shared_idx) { + if (routed_idx + 2 >= graph->n_nodes || shared_idx + 2 >= graph->n_nodes || shared_idx < routed_idx + 3) { + return false; + } + const int nodes[] = { routed_idx, routed_idx + 1, routed_idx + 2, shared_idx, shared_idx + 1, shared_idx + 2 }; + const ggml_op ops[] = { GGML_OP_MUL_MAT_ID, GGML_OP_MUL_MAT_ID, GGML_OP_GLU, + GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }; + const int outputs[] = { routed_idx + 2, shared_idx + 2 }; + if (!ggml_can_fuse_subgraph_ext(graph, nodes, 6, ops, outputs, 2)) { + return false; + } + + const ggml_tensor * routed = graph->nodes[routed_idx + 2]; + const ggml_tensor * shared = graph->nodes[shared_idx + 2]; + const ggml_tensor * gate = routed->src[0]; + const ggml_tensor * up = routed->src[1]; + const ggml_tensor * shared_gate = shared->src[0]; + const ggml_tensor * shared_up = shared->src[1]; + const auto is_pair = [&](const ggml_tensor * a, const ggml_tensor * b, int idx) { + return (a == graph->nodes[idx] && b == graph->nodes[idx + 1]) || + (b == graph->nodes[idx] && a == graph->nodes[idx + 1]); + }; + // only batch-size independent checks here: graph_optimize must produce the same graph topology for every ubatch + // size, otherwise ggml-alloc has to re-reserve (and the scheduler to synchronize) at runtime. + // the MMVQ batch size check is done in ggml_cuda_try_fuse + if (!is_pair(gate, up, routed_idx) || !is_pair(shared_gate, shared_up, shared_idx) || + !ggml_cuda_should_fuse_mul_mat(up, gate, routed) || + !ggml_cuda_should_fuse_mul_mat(shared_up, shared_gate, shared) || + !up->src[0]->buffer || + !ggml_is_quantized(up->src[0]->type)) { + return false; + } + const ggml_tensor * input = up->src[1]; + const ggml_tensor * weight = up->src[0]; + const ggml_tensor * shared_weight = shared_up->src[0]; + if (input->op != GGML_OP_RESHAPE || input->src[0] != shared_up->src[1] || + input->ne[1] != 1 || input->ne[3] != 1 || !ggml_is_contiguous(input) || + !ggml_is_contiguous(shared_up->src[1]) || !ggml_is_matrix(shared_up->src[1]) || + weight->type != shared_weight->type || weight->ne[0] != shared_weight->ne[0] || + weight->ne[1] != shared_weight->ne[1] || weight->nb[1] != shared_weight->nb[1] || weight->ne[3] != 1 || + !ggml_is_matrix(shared_weight) || !ggml_is_contiguous(shared_weight) || + !ggml_is_contiguous(shared_gate->src[0]) || !ggml_is_contiguous(routed) || !ggml_is_contiguous(shared)) { + return false; + } + if (shared_weight->op != GGML_OP_NONE || shared_gate->src[0]->op != GGML_OP_NONE || + ggml_get_glu_op(routed) != ggml_get_glu_op(shared) || + ggml_get_op_params_f32(routed, 3) != ggml_get_op_params_f32(shared, 3)) { return false; } + return true; +} +// Buffer- and batch-size independent, so graph_optimize produces the same graph topology for every ubatch size (see #29986). +static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu, const int device) { + const ggml_tensor * x_up = up->src[0]; + const ggml_tensor * y = up->src[1]; - const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft) || - ggml_backend_buft_is_cuda_split(src1->buffer->buft); + if (up->op != GGML_OP_MUL_MAT || !ggml_cuda_mmq_fusion_supported(x_up->type) || y->type != GGML_TYPE_F32) { + return false; + } - //TODO: add support for fusion for split buffers - if (split) { + // The fused kernel is only compiled for weights whose rows fill whole tiles (no fallback). + if (ggml_cuda_mmq_needs_fallback(x_up->ne[1])) { return false; } - return use_mul_mat_vec_q; + if (!ggml_cuda_should_fuse_mul_mat(up, gate, glu)) { + return false; + } + + // The fused write-back applies SWIGLU_OAI with the default alpha and limit of ggml_cuda_op_swiglu_oai_single. + if (ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU_OAI && (ggml_get_op_params_f32(glu, 2) != 1.702f || ggml_get_op_params_f32(glu, 3) != 7.0f)) { + return false; + } + + // NVIDIA only: the fused write-back relies on the MMA accumulator layout. + return turing_mma_available(ggml_cuda_info().devices[device].cc); +} + +static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_tensor * gate, const ggml_tensor * glu) { + // A hinted matmul can be fused, but its unfused path may be faster (e.g. FWHT for GGML_HINT_SRC0_IS_HADAMARD). + if (ggml_get_op_params_i32(up, 1) != GGML_HINT_NONE || ggml_get_op_params_i32(gate, 1) != GGML_HINT_NONE) { + return false; + } + + // The padding of a compute-buffer weight that is a view cannot be cleared, see ggml_cuda_mul_mat. + const auto bad_padding_clear = [](const ggml_tensor * x) { + return ggml_backend_buffer_get_usage(x->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && + ggml_nbytes(x) != ggml_backend_buffer_get_alloc_size(x->buffer, x) && x->view_src; + }; + const int device = ggml_cuda_get_device(); + if (!ggml_cuda_mul_mat_q_fusion_matches(up, gate, glu, device) || bad_padding_clear(up->src[0]) || bad_padding_clear(gate->src[0])) { + return false; + } + + // The batch size checks are done here, not in ggml_cuda_mul_mat_q_fusion_matches. + const ggml_tensor * x_up = up->src[0]; + const ggml_tensor * y = up->src[1]; + const int cc = ggml_cuda_info().devices[device].cc; + if (ggml_cuda_should_use_mmvq(x_up->type, cc, y->ne[1]) || !ggml_cuda_should_use_mmq(x_up->type, cc, y->ne[1], 0)) { + return false; + } + + // The fused kernel needs one block per tile (no stream-k split), so only fuse if the tiling is efficient. + return ggml_cuda_mmq_fusion_is_efficient(x_up->type, x_up->ne[1], y->ne[1], y->ne[2]*y->ne[3], device); } static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - const bool split = ggml_backend_buft_is_cuda_split(src0->buffer->buft); + GGML_TENSOR_BINARY_OP_LOCALS + + if (ggml_cuda_op_mul_mat_use_fwht(dst) && ggml_cuda_op_fwht(ctx, src1, dst)) { + return; + } // If src0 is a temporary compute buffer it may have some padding that needs to be cleared for mul_mat_vec_q or mul_mat_q. // But if src0 is also a view of another tensor then this cannot be done safely because it may overwrite valid tensor data. // Therefore, in such cases use cuBLAS. const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && src0->view_src; + if (bad_padding_clear || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); + return; + } - bool use_mul_mat_vec_f = (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; - bool use_mul_mat_f = !ggml_is_quantized(src0->type) - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; - bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && !bad_padding_clear - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 - && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE; - bool use_mul_mat_q = ggml_is_quantized(src0->type) && !bad_padding_clear - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; - - bool any_gpus_with_slow_fp16 = false; - - if (split) { - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context; - auto & tensor_split = buft_ctx->tensor_split; - for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) { - // skip devices that are not going to do any work: - if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) { - continue; - } + const int cc = ggml_cuda_info().devices[ctx.device].cc; + const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - const int cc = ggml_cuda_info().devices[id].cc; - const int warp_size = ggml_cuda_info().devices[id].warp_size; - use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0); - use_mul_mat_f = use_mul_mat_f && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false); - use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]); - any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available(cc); - } - } else { - const int cc = ggml_cuda_info().devices[ctx.device].cc; - const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - use_mul_mat_q = use_mul_mat_q && ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[1], /*n_experts=*/0); - use_mul_mat_f = use_mul_mat_f && ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[1], /*mul_mat_id=*/false); - use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, src1->ne[1]); - any_gpus_with_slow_fp16 = any_gpus_with_slow_fp16 || !fast_fp16_hardware_available(cc); - } - - // debug helpers - //printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]); - //printf(" %8d %8d %8d %8d\n", src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]); - //printf("src1: %8d %8d %8d %8d\n", src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]); - //printf(" %8d %8d %8d %8d\n", src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]); - //printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name); - //printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name); - - //TODO update for generic tensor parallelism - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - bool use_batched_cublas_f16 = src0->type == GGML_TYPE_F16 && (src1->type == GGML_TYPE_F16 || !any_gpus_with_slow_fp16); - bool use_batched_cublas_bf16 = src0->type == GGML_TYPE_BF16 && bf16_mma_hardware_available(cc); - bool use_batched_cublas_f32 = src0->type == GGML_TYPE_F32; - - if (!split && use_mul_mat_vec_f) { - // the custom F16 vector kernel can be used over batched cuBLAS GEMM - // but this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention) + if (ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11)) { + // The custom F16 vector kernel can be used over batched cuBLAS GEMM. + // But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention) ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); - } else if (!split && use_mul_mat_f) { + return; + } + // A transposed vector can still use MMVQ (i.e. ne01 == 1) + if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1 + && src0->type == GGML_TYPE_F32 + && ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) + && ggml_cuda_should_use_mmvf(src1->type, cc, warp_size, src1->ne, src1->nb, /*ne11 =*/ 1)) { + ggml_tensor dst_vec = *dst; + dst_vec.ne[0] = ne11; + dst_vec.ne[1] = 1; + dst_vec.nb[1] = dst_vec.nb[0]*ne11; + dst_vec.nb[2] = dst_vec.nb[1]; + dst_vec.nb[3] = dst_vec.nb[1]; + ggml_cuda_mul_mat_vec_f(ctx, src1, src0, nullptr, &dst_vec); + return; + } + if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11, /*mul_mat_id =*/ false)) { ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst); - } else if (!split && use_mul_mat_vec_q) { + return; + } + if (ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); - } else if (!split && use_mul_mat_q) { + return; + } + if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); - } else if (!split && (use_batched_cublas_f16 || use_batched_cublas_bf16 || use_batched_cublas_f32) - && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) { - // general KQ + KQV multi-batch without FlashAttention - ggml_cuda_mul_mat_batched_cublas(ctx, src0, src1, dst); - } else if (use_mul_mat_vec_f) { - ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_f, nullptr); - } else if (use_mul_mat_vec_q) { - ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, quantize_row_q8_1_cuda); - } else if (use_mul_mat_q) { - ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, quantize_mmq_q8_1_cuda); - } else { - ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, nullptr); + return; + } + ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); +} + +// returns true when ggml_cuda_mul_mat_id takes the fallback path that requires stream synchronization +// [TAG_MUL_MAT_ID_CUDA_GRAPHS] +static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int cc) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; + + if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return true; + } + + if (dst->ne[2] <= MMVQ_MAX_BATCH_SIZE) { + if (ggml_is_quantized(src0->type)) { + if (dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc)) { + return false; + } + } else if (GGML_CUDA_CC_IS_AMD(cc)) { + return false; + } } + + if (ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[2], /*n_experts=*/src0->ne[2])) { + return false; + } + + if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { + return false; + } + + return true; } static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { @@ -2444,11 +2107,11 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); - GGML_ASSERT(!ggml_backend_buft_is_cuda_split(src0->buffer->buft) && "mul_mat_id does not support split buffers"); GGML_TENSOR_BINARY_OP_LOCALS const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; // [TAG_MUL_MAT_ID_CUDA_GRAPHS] if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { @@ -2473,14 +2136,14 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * return; } - if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { + if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { ggml_cuda_mul_mat_f(ctx, src0, src1, ids, dst); return; } } // note: this path should not be reached when recording CUDA graphs, because it requires stream synchronization - // TODO: add asserts to verify this. should work with CUDA, HIP, etc. + GGML_ASSERT(ggml_cuda_mul_mat_id_needs_sync(dst, cc)); cudaStream_t stream = ctx.stream(); GGML_ASSERT(nb12 % nb11 == 0); @@ -2570,6 +2233,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * ggml_tensor dst_slice; memset(&dst_slice, 0, sizeof(dst_slice)); + memcpy(dst_slice.op_params, dst->op_params, sizeof(dst_slice.op_params)); dst_slice.buffer = dst->buffer; dst_slice.type = type_dst_sorted; dst_slice.ne[0] = ne0; @@ -2741,6 +2405,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_GLU_OP_GEGLU_QUICK: ggml_cuda_op_geglu_quick(ctx, dst); break; + case GGML_GLU_OP_SWIGLU_CLAMP: + ggml_cuda_op_swiglu_clamp(ctx, dst); + break; default: return false; } @@ -2850,6 +2517,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_CONV_2D: ggml_cuda_op_conv2d(ctx, dst); break; + case GGML_OP_CONV_3D: + ggml_cuda_op_conv3d(ctx, dst); + break; case GGML_OP_CONV_2D_DW: ggml_cuda_op_conv2d_dw(ctx, dst); break; @@ -2859,9 +2529,15 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_CONV_TRANSPOSE_1D: ggml_cuda_op_conv_transpose_1d(ctx,dst); break; + case GGML_OP_COL2IM_1D: + ggml_cuda_op_col2im_1d(ctx, dst); + break; case GGML_OP_POOL_2D: ggml_cuda_op_pool2d(ctx, dst); break; + case GGML_OP_POOL_1D: + ggml_cuda_op_pool1d(ctx, dst); + break; case GGML_OP_SUM: ggml_cuda_op_sum(ctx, dst); break; @@ -2904,6 +2580,15 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_GATED_DELTA_NET: ggml_cuda_op_gated_delta_net(ctx, dst); break; + case GGML_OP_DSV4_HC_COMB: + ggml_cuda_op_dsv4_hc_comb(ctx, dst); + break; + case GGML_OP_DSV4_HC_PRE: + ggml_cuda_op_dsv4_hc_pre(ctx, dst); + break; + case GGML_OP_DSV4_HC_POST: + ggml_cuda_op_dsv4_hc_post(ctx, dst); + break; case GGML_OP_RWKV_WKV7: ggml_cuda_op_rwkv_wkv7(ctx, dst); break; @@ -2922,6 +2607,12 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_FILL: ggml_cuda_op_fill(ctx, dst); break; + case GGML_OP_LIGHTNING_INDEXER: + ggml_cuda_lightning_indexer(ctx, dst); + break; + case GGML_OP_TOP_K_MASK: + ggml_cuda_op_top_k_mask(ctx, dst); + break; default: return false; } @@ -3020,13 +2711,17 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_ if (backend_src != backend_dst) { // copy on src stream - if (cuda_ctx_src->device == cuda_ctx_dst->device) { + // compare the backing physical devices: distinct virtual devices may share one physical GPU, + // in which case a same-device copy (not a peer copy) is required + const int src_physical = ggml_cuda_get_physical_device(cuda_ctx_src->device); + const int dst_physical = ggml_cuda_get_physical_device(cuda_ctx_dst->device); + if (src_physical == dst_physical) { CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_src->stream())); } else { #ifdef GGML_CUDA_NO_PEER_COPY return false; #else - CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), cuda_ctx_src->stream())); + CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_physical, src->data, src_physical, ggml_nbytes(dst), cuda_ctx_src->stream())); #endif // GGML_CUDA_NO_PEER_COPY } @@ -3055,7 +2750,17 @@ static void ggml_backend_cuda_synchronize(ggml_backend_t backend) { GGML_UNUSED(backend); } +static bool ggml_cuda_is_view_or_noop(const ggml_tensor * t) { + return ggml_is_empty(t) || t->op == GGML_OP_RESHAPE || t->op == GGML_OP_TRANSPOSE || + t->op == GGML_OP_VIEW || t->op == GGML_OP_PERMUTE || t->op == GGML_OP_NONE; +} + #ifdef USE_CUDA_GRAPH +static bool ggml_cuda_graph_debug_enabled() { + static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_GRAPH_DEBUG"); + return enabled; +} + static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { bool use_cuda_graph = true; @@ -3064,29 +2769,20 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_cuda_is_view_or_noop(node)) { continue; } - if (node->src[0] && node->src[0]->buffer && ggml_backend_buft_is_cuda_split(node->src[0]->buffer->buft)) { - use_cuda_graph = false; // Split buffers are not supported by CUDA graph capture -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: disabling CUDA graphs due to split buffer\n", __func__); -#endif - } - // [TAG_MUL_MAT_ID_CUDA_GRAPHS] if (node->op == GGML_OP_MUL_MAT_ID) { const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - const int mmvq_mmid_max = get_mmvq_mmid_max_batch(node->src[0]->type, cc); - if (!ggml_is_quantized(node->src[0]->type) || node->ne[2] > mmvq_mmid_max) { - // under these conditions, the mul_mat_id operation will need to synchronize the stream, so we cannot use CUDA graphs - // TODO: figure out a way to enable for larger batch sizes, without hurting performance + if (ggml_cuda_mul_mat_id_needs_sync(node, cc)) { + // the mul_mat_id fallback path synchronizes the stream, so we cannot use CUDA graphs // ref: https://github.com/ggml-org/llama.cpp/pull/18958 use_cuda_graph = false; -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__); -#endif + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__); + } } } @@ -3110,7 +2806,9 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx if (cgraph->uid != 0 && cgraph->uid == graph->uid) { - GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid); + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid); + } GGML_ASSERT((int)graph->node_props.size() == cgraph->n_nodes); return false; } @@ -3144,33 +2842,57 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx return res; } -static void ggml_cuda_graph_update_executable(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) { - ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); - +static void ggml_cuda_graph_update_executable(cudaGraphExec_t & instance, cudaGraph_t cuda_graph) { #if CUDART_VERSION >= 12000 cudaGraphExecUpdateResultInfo result_info; - cudaError_t stat = cudaGraphExecUpdate(graph->instance, graph->graph, &result_info); + cudaError_t stat = cudaGraphExecUpdate(instance, cuda_graph, &result_info); #else cudaGraphNode_t errorNode; cudaGraphExecUpdateResult result_info; - cudaError_t stat = cudaGraphExecUpdate(graph->instance, graph->graph, &errorNode, &result_info); + cudaError_t stat = cudaGraphExecUpdate(instance, cuda_graph, &errorNode, &result_info); #endif // CUDART_VERSION >= 12000 if (stat == cudaErrorGraphExecUpdateFailure) { -#ifndef NDEBUG - GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__); -#endif + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__); + } // The pre-existing graph exec cannot be updated due to violated constraints // so instead clear error and re-instantiate (void)cudaGetLastError(); - CUDA_CHECK(cudaGraphExecDestroy(graph->instance)); - graph->instance = nullptr; - CUDA_CHECK(cudaGraphInstantiate(&graph->instance, graph->graph, NULL, NULL, 0)); + CUDA_CHECK(cudaGraphExecDestroy(instance)); + instance = nullptr; + CUDA_CHECK(cudaGraphInstantiate(&instance, cuda_graph, nullptr, nullptr, 0)); } else { GGML_ASSERT(stat == cudaSuccess); } } + +// number of ggml nodes per CUDA graph chunk, 0 captures the whole ggml graph as one CUDA graph +static int ggml_cuda_graph_chunk_size() { + static const int chunk_size = [] { + const char * env = getenv("GGML_CUDA_GRAPH_CHUNK_SIZE"); + return env != nullptr ? atoi(env) : 512; + }(); + return chunk_size; +} + +// end the capture of chunk `idx`, update or create its executable and launch it +static void ggml_cuda_graph_launch_chunk(ggml_backend_cuda_context * cuda_ctx, ggml_cuda_graph * graph, size_t idx) { + cudaGraph_t cuda_graph = nullptr; + CUDA_CHECK(cudaStreamEndCapture(cuda_ctx->stream(), &cuda_graph)); + + if (idx < graph->instances.size()) { + ggml_cuda_graph_update_executable(graph->instances[idx], cuda_graph); + } else { + GGML_ASSERT(idx == graph->instances.size()); + graph->instances.emplace_back(nullptr); + CUDA_CHECK(cudaGraphInstantiate(&graph->instances[idx], cuda_graph, nullptr, nullptr, 0)); + } + CUDA_CHECK(cudaGraphDestroy(cuda_graph)); + + CUDA_CHECK(cudaGraphLaunch(graph->instances[idx], cuda_ctx->stream())); +} #endif // USE_CUDA_GRAPH static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope, @@ -3198,97 +2920,363 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope, return false; } - // Only norm/neox shaders have the fusion code + // Only norm/neox/multi shaders have the fusion code const int mode = ((const int32_t *) rope->op_params)[2]; - if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX) { + if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && + mode != GGML_ROPE_TYPE_MROPE && mode != GGML_ROPE_TYPE_IMROPE) { return false; } return true; } -static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) { - args.sigmoid = false; - args.softmax = false; - args.delayed_softmax = false; - args.prob_bias = false; - args.norm = false; +static bool ggml_cuda_should_fuse_rms_norm_mul_rope(const ggml_tensor * rms_norm, + const ggml_tensor * mul, + const ggml_tensor * rope) { + if (rms_norm->op != GGML_OP_RMS_NORM || mul->op != GGML_OP_MUL || rope->op != GGML_OP_ROPE) { + return false; + } - const int n_nodes = cgraph->n_nodes; - ggml_tensor ** nodes = cgraph->nodes; + if (rms_norm->src[0]->type != GGML_TYPE_F32 || rms_norm->type != GGML_TYPE_F32 || + mul->src[0]->type != GGML_TYPE_F32 || mul->src[1]->type != GGML_TYPE_F32 || + mul->type != GGML_TYPE_F32 || rope->type != GGML_TYPE_F32) { + return false; + } - if (nodes[node_idx]->op == GGML_OP_SOFT_MAX) { - args.softmax = true; + if (rope->src[0] != mul) { + return false; } - if (nodes[node_idx]->op == GGML_OP_UNARY) { - if (ggml_get_unary_op(nodes[node_idx]) != GGML_UNARY_OP_SIGMOID) { - return false; - } - args.sigmoid = true; + //if rms norm is the B operand, then we don't handle broadcast + if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) { + return false; } - if (nodes[node_idx]->op == GGML_OP_ARGSORT) { - args.delayed_softmax = true; + if (!ggml_are_same_shape(rms_norm, mul)) { + return false; } - node_idx++; + //rms_norm kernel assumes contiguous rows + if (!ggml_is_contiguous_rows(rms_norm->src[0]) || + !ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) { + return false; + } - if (args.sigmoid || args.softmax) { - // SOFTMAX -> RESHAPE - if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_RESHAPE || - nodes[node_idx]->src[0] != nodes[node_idx - 1]) { - return false; - } - ggml_tensor * probs_reshaped = nodes[node_idx]; - node_idx++; + // the fused kernel handles the norm/neox rope modes only + const int mode = ((const int32_t *) rope->op_params)[2]; + if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX) { + return false; + } - if (node_idx >= n_nodes) { - return false; - } + const int n_dims = ((const int32_t *) rope->op_params)[1]; + if (n_dims % 2 != 0 || rope->src[0]->ne[0] % 2 != 0) { + return false; + } - // src of bias add is the unreshaped probs (-2 instead of -1) - if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == nodes[node_idx - 2]) { - args.prob_bias = true; - node_idx++; + // ggml_rope_set_offset is not yet supported in the fused kernel + const int n_offs = ((const int32_t *) rope->op_params)[15]; + if (n_offs != 0) { + return false; + } + + return true; +} + +// match gated_delta_net + the strided cpy that scatters its state snapshots into the cache +// (slot i -> rollback group i, slot 0 newest), so the kernel can write them and skip the cpy. +static int ggml_cuda_try_gdn_cache_fusion( + const ggml_cgraph * cgraph, int node_idx, ggml_cuda_gated_delta_net_fused_cache & fused_state_cpy) { + const ggml_tensor * gdn = cgraph->nodes[node_idx]; + // the kernel skips the snapshot tail, so the gdn output must not be a graph output + if (gdn->op != GGML_OP_GATED_DELTA_NET || gdn->type != GGML_TYPE_F32 || + (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + + const ggml_tensor * src_v = gdn->src[2]; + const int64_t S_v = src_v->ne[0]; + const int64_t H = src_v->ne[1]; + const int64_t n_tokens = src_v->ne[2]; + const int64_t n_seqs = src_v->ne[3]; + const int64_t D = S_v * S_v * H; + const int64_t K = ggml_get_op_params_i32(gdn, 0); // snapshot slot count + const int64_t n_written = std::min(n_tokens, K); // newest n_written slots are written + + // snapshot tail starts right after the attention scores + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + // snapshot cpy is the first real node after the gdn (skip views/no-ops) + const ggml_tensor * cpy = nullptr; + int skip = 0; + for (int j = node_idx + 1; j < cgraph->n_nodes && cpy == nullptr; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_cuda_is_view_or_noop(n)) { + continue; } - // RESHAPE/ADD -> ARGSORT - if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT) { - return false; + if (n->op != GGML_OP_CPY || (n->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; } + cpy = n; + skip = j - node_idx; + } + if (cpy == nullptr) { + return 0; + } - if (args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 1]) { - return false; - } else if (!args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 2]) { - return false; - } + const ggml_tensor * src = cpy->src[0]; // view of the gdn snapshot tail + const ggml_tensor * dst = cpy->src[1]; // cache view the kernel writes to - node_idx++; + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != gdn || src->view_offs != tail_off || + !ggml_is_contiguous(src)) { + return 0; + } - // ARGSORT-> VIEW - if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_VIEW || - nodes[node_idx]->src[0] != nodes[node_idx - 1]) { - return false; - } - node_idx++; + // dst is the [D, n_seqs, n_written] cache view; require nb[1] == D (the per-seq stride the kernel + // assumes). ggml_cpy pins src to the same element count. + const std::array expected_ne = { D, n_seqs, n_written, 1 }; + if (dst->op != GGML_OP_VIEW || dst->type != GGML_TYPE_F32 || dst->data == nullptr || + !std::equal(expected_ne.begin(), expected_ne.end(), dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || dst->nb[1] != (size_t) ggml_row_size(GGML_TYPE_F32, D)) { + return 0; + } - if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_GET_ROWS) { - return false; - } + fused_state_cpy.data = (float *) dst->data; // rollback group 0 (newest) + fused_state_cpy.slot_stride = K > 1 ? (int64_t) (dst->nb[2] / sizeof(float)) : 0; + return skip; +} - // GET_ROWS - if (nodes[node_idx]->src[0] != probs_reshaped || nodes[node_idx]->src[1] != nodes[node_idx - 1]) { - return false; - } - node_idx++; - } else if (args.delayed_softmax) { - if (node_idx - 2 < 0) { - return false; - } - ggml_tensor * probs_reshaped = nodes[node_idx - 2]; +// match ssm_scan + the strided cpy that scatters its state snapshots into the cache, so the kernel writes them and skips the cpy +static int ggml_cuda_try_ssm_scan_cache_fusion( + const ggml_cgraph * cgraph, int node_idx, ggml_cuda_ssm_scan_fused_cache & fused_state_cpy) { + const ggml_tensor * ssm = cgraph->nodes[node_idx]; + // the kernel skips the snapshot tail, so the scan output must not be a graph output + if (ssm->op != GGML_OP_SSM_SCAN || ssm->type != GGML_TYPE_F32 || (ssm->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } - // VIEW->ARGSORT - if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_VIEW || + const int64_t K = ggml_get_op_params_i32(ssm, 0); // snapshot slot count + + const ggml_tensor * s = ssm->src[0]; + const ggml_tensor * x = ssm->src[1]; + const ggml_tensor * A = ssm->src[3]; + + const int64_t d_state = s->ne[0]; + const int64_t D = d_state * s->ne[1] * x->ne[1]; // d_state * head_dim * n_head + const int64_t n_tok = x->ne[2]; + const int64_t n_seqs = x->ne[3]; + + // only the mamba-2 kernels (group scan and SSD) write to the cache; mamba-1 still uses the cpy + if (A->nb[1] != sizeof(float) || (d_state != 96 && d_state != 128 && d_state != 256)) { + return 0; + } + + // the scan reads its input rows from the cache (picked by ids), so with more than one seq a seq can read a row that another seq writes in the same launch + if (n_seqs != 1) { + return 0; + } + + const int64_t n_written = std::min(n_tok, K); + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, ggml_nelements(x)); + + // snapshot cpy is the first real node after the scan (skip views/no-ops) + const ggml_tensor * cpy = nullptr; + int skip = 0; + for (int j = node_idx + 1; j < cgraph->n_nodes && cpy == nullptr; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_cuda_is_view_or_noop(n)) { + continue; + } + if (n->op != GGML_OP_CPY || (n->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + cpy = n; + skip = j - node_idx; + } + if (cpy == nullptr) { + return 0; + } + + const ggml_tensor * src = cpy->src[0]; // view of the scan snapshot tail + const ggml_tensor * dst = cpy->src[1]; // cache view the kernel writes to + + // src must be this scan's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != ssm || src->view_offs != tail_off || + !ggml_is_contiguous(src)) { + return 0; + } + + // dst is the [D, n_seqs, n_written] cache view; require nb[1] == D, the per-seq stride the kernel takes from src0->nb[3] + const std::array expected_ne = { D, n_seqs, n_written, 1 }; + if (dst->op != GGML_OP_VIEW || dst->type != GGML_TYPE_F32 || dst->data == nullptr || + !std::equal(expected_ne.begin(), expected_ne.end(), dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || dst->nb[1] != (size_t) ggml_row_size(GGML_TYPE_F32, D)) { + return 0; + } + + fused_state_cpy.data = (float *) dst->data; // rollback slot 0 (newest) + fused_state_cpy.slot_stride = K > 1 ? (int64_t) (dst->nb[2] / sizeof(float)) : 0; + return skip; +} + +// expert group selection from build_moe_ffn (n_expert_groups > 1) +// src0/src1 index into this table, -1 is the (biased) probs, -2 is not checked +static const struct { + ggml_op op; + int src0; + int src1; +} topk_moe_group_pattern[] = { + { GGML_OP_RESHAPE, -1, -2 }, // 0: [n_exp_per_group, n_expert_groups, n_tokens] + { GGML_OP_RESHAPE, 0, -2 }, // 1: [1, n_exp_per_group, n_expert_groups, n_tokens] + { GGML_OP_ARGSORT, 0, -2 }, // 2 + { GGML_OP_VIEW, 2, -2 }, // 3: top-2 per group + { GGML_OP_GET_ROWS, 1, 3 }, // 4 + { GGML_OP_RESHAPE, 4, -2 }, // 5 + { GGML_OP_SUM_ROWS, 5, -2 }, // 6: group scores + { GGML_OP_RESHAPE, 6, -2 }, // 7 + { GGML_OP_ARGSORT, 7, -2 }, // 8 + { GGML_OP_VIEW, 8, -2 }, // 9: top n_group_used groups + { GGML_OP_GET_ROWS, 0, 9 }, // 10 + { GGML_OP_FILL, 0, -2 }, // 11 + { GGML_OP_SET_ROWS, 10, 9 }, // 12: unused groups -> -inf + { GGML_OP_RESHAPE, 12, -2 }, // 13: [n_expert, n_tokens] +}; + +static bool ggml_cuda_topk_moe_group_fusion(const struct ggml_cgraph * cgraph, int node_idx, const ggml_tensor * selection, ggml_cuda_topk_moe_args & args) { +#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000 + GGML_UNUSED_VARS(cgraph, node_idx, selection, args); + return false; +#else + const int n_ops = (int) std::size(topk_moe_group_pattern); + if (node_idx + n_ops > cgraph->n_nodes) { + return false; + } + + ggml_tensor ** nodes = cgraph->nodes + node_idx; + auto src_ok = [&](const ggml_tensor * src, int idx) { + return idx == -2 || src == (idx == -1 ? selection : nodes[idx]); + }; + + for (int i = 0; i < n_ops; i++) { + const auto & p = topk_moe_group_pattern[i]; + if (nodes[i]->op != p.op || !src_ok(nodes[i]->src[0], p.src0) || !src_ok(nodes[i]->src[1], p.src1)) { + return false; + } + } + + const int n_groups = nodes[0]->ne[1]; + const int n_group_used = nodes[9]->ne[0]; + + if (nodes[3]->ne[0] != 2 || nodes[12]->src[2] != nodes[11] || ggml_get_op_params_f32(nodes[11], 0) != -INFINITY || + n_groups > WARP_SIZE) { + return false; + } + + args.n_groups = n_groups; + args.n_group_used = n_group_used; + return true; +#endif // defined(GGML_USE_HIP) && HIP_VERSION < 60300000 +} + +static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) { + args.sigmoid = false; + args.sqrt_softplus = false; + args.softmax = false; + args.delayed_softmax = false; + args.prob_bias = false; + args.norm = false; + args.n_groups = 0; + args.n_group_used = 0; + + const int n_nodes = cgraph->n_nodes; + ggml_tensor ** nodes = cgraph->nodes; + + if (nodes[node_idx]->op == GGML_OP_SOFT_MAX) { + args.softmax = true; + } + + if (nodes[node_idx]->op == GGML_OP_UNARY) { + const ggml_unary_op unary_op = ggml_get_unary_op(nodes[node_idx]); + if (unary_op == GGML_UNARY_OP_SIGMOID) { + args.sigmoid = true; + } else if (unary_op == GGML_UNARY_OP_SOFTPLUS && node_idx + 1 < n_nodes && + nodes[node_idx + 1]->op == GGML_OP_SQRT && nodes[node_idx + 1]->src[0] == nodes[node_idx]) { + // sqrt(softplus(x)) scoring (DeepSeek-V4) + args.sqrt_softplus = true; + node_idx++; + } else { + return false; + } + } + + if (nodes[node_idx]->op == GGML_OP_ARGSORT) { + args.delayed_softmax = true; + } + + node_idx++; + + if (args.sigmoid || args.sqrt_softplus || args.softmax) { + // SOFTMAX -> RESHAPE + if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_RESHAPE || + nodes[node_idx]->src[0] != nodes[node_idx - 1]) { + return false; + } + ggml_tensor * probs_reshaped = nodes[node_idx]; + node_idx++; + + if (node_idx >= n_nodes) { + return false; + } + + // src of bias add is the unreshaped probs (-2 instead of -1) + const ggml_tensor * selection = nodes[node_idx - 2]; + if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == selection) { + args.prob_bias = true; + selection = nodes[node_idx]; + node_idx++; + } + + if (ggml_cuda_topk_moe_group_fusion(cgraph, node_idx, selection, args)) { + node_idx += (int) std::size(topk_moe_group_pattern); + selection = nodes[node_idx - 1]; + } + + // RESHAPE/ADD/group mask -> ARGSORT + if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT || nodes[node_idx]->src[0] != selection) { + return false; + } + + node_idx++; + + // ARGSORT-> VIEW + if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_VIEW || + nodes[node_idx]->src[0] != nodes[node_idx - 1]) { + return false; + } + + // kept groups must hold all used experts + if (args.n_groups > 0 && nodes[node_idx]->ne[0] > args.n_group_used * (probs_reshaped->ne[1] / args.n_groups)) { + return false; + } + node_idx++; + + if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_GET_ROWS) { + return false; + } + + // GET_ROWS + if (nodes[node_idx]->src[0] != probs_reshaped || nodes[node_idx]->src[1] != nodes[node_idx - 1]) { + return false; + } + node_idx++; + } else if (args.delayed_softmax) { + if (node_idx - 2 < 0) { + return false; + } + ggml_tensor * probs_reshaped = nodes[node_idx - 2]; + + // VIEW->ARGSORT + if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_VIEW || nodes[node_idx]->src[0] != nodes[node_idx - 1]) { return false; } @@ -3375,9 +3363,10 @@ static bool ggml_cuda_check_fusion_memory_ranges(const ggml_cgraph * cgraph, }; bool is_ok = true; - // exception for topk-moe, as each row is read entirely before writing - if (ggml_nrows(cgraph->nodes[node_idx]) == 1 && is_topk_moe) { - return true; + // one block reads all logits before it writes, so logits may alias the out nodes + const ggml_tensor * logits_may_alias = nullptr; + if (is_topk_moe && ggml_nrows(cgraph->nodes[node_idx]) <= TOPK_MOE_ROWS_PER_BLOCK) { + logits_may_alias = cgraph->nodes[node_idx]->src[0]; } for (int i = 0; i < out_count; ++i) { @@ -3391,7 +3380,7 @@ static bool ggml_cuda_check_fusion_memory_ranges(const ggml_cgraph * cgraph, for (int src_idx = 0; src_idx < GGML_MAX_SRC; ++src_idx) { const ggml_tensor * src = cgraph->nodes[j]->src[src_idx]; - if (!src || src->op == GGML_OP_NONE) { + if (!src || src->op == GGML_OP_NONE || src == logits_may_alias) { continue; } @@ -3417,6 +3406,150 @@ static bool ggml_cuda_check_fusion_memory_ranges(const ggml_cgraph * cgraph, return is_ok; } +// The long form spans 2*k + 1 nodes. ggml_can_fuse_subgraph() accepts at most +// 31 nodes, so k <= 15; larger values use the per-operation path. +static constexpr int MOE_WEIGHTED_REDUCTION_MAX_EXPERTS = 15; + +struct ggml_cuda_moe_weighted_reduction_match { + const ggml_tensor * experts = nullptr; + const ggml_tensor * expert_scale = nullptr; + const ggml_tensor * weights = nullptr; + ggml_tensor * dst = nullptr; + int node_count = 0; +}; + +static bool ggml_cuda_match_moe_weighted_reduction( + const ggml_cgraph * cgraph, + int node_idx, + ggml_cuda_moe_weighted_reduction_match & match) { + const ggml_tensor * first = cgraph->nodes[node_idx]; + if (first->op != GGML_OP_MUL || first->type != GGML_TYPE_F32 || !ggml_is_contiguous(first)) { + return false; + } + + auto split_mul = [](const ggml_tensor * mul, const ggml_tensor *& full, const ggml_tensor *& broadcast) { + auto is_weights = [mul](const ggml_tensor * tensor) { + return tensor && tensor->type == GGML_TYPE_F32 && ggml_is_contiguous(tensor) && tensor->ne[0] == 1 && + tensor->ne[1] == mul->ne[1] && tensor->ne[2] == mul->ne[2] && tensor->ne[3] == mul->ne[3]; + }; + auto is_experts = [mul](const ggml_tensor * tensor) { + return tensor && tensor->type == GGML_TYPE_F32 && ggml_is_contiguous(tensor) && + ggml_are_same_shape(tensor, mul); + }; + + if (is_experts(mul->src[0]) && is_weights(mul->src[1])) { + full = mul->src[0]; + broadcast = mul->src[1]; + return true; + } + if (is_experts(mul->src[1]) && is_weights(mul->src[0])) { + full = mul->src[1]; + broadcast = mul->src[0]; + return true; + } + return false; + }; + + const ggml_tensor * weighted = first; + const ggml_tensor * experts = nullptr; + const ggml_tensor * expert_scale = nullptr; + const ggml_tensor * weights = nullptr; + int mul_count = 1; + + // Match both structural forms: + // (experts * expert_scale) * router_weight + // experts * router_weight + // The matcher does not depend on the model or quantization type. + if (node_idx + 1 < cgraph->n_nodes) { + const ggml_tensor * second = cgraph->nodes[node_idx + 1]; + const ggml_tensor * scaled = nullptr; + const ggml_tensor * route = nullptr; + const ggml_tensor * raw = nullptr; + const ggml_tensor * scale = nullptr; + if (second->op == GGML_OP_MUL && second->type == GGML_TYPE_F32 && ggml_is_contiguous(second) && + split_mul(second, scaled, route) && scaled == first && split_mul(first, raw, scale)) { + weighted = second; + experts = raw; + expert_scale = scale; + weights = route; + mul_count = 2; + } + } + + if (experts == nullptr && !split_mul(first, experts, weights)) { + return false; + } + + const int n_expert_used = (int) weighted->ne[1]; + const int64_t n_tokens = weighted->ne[2] * weighted->ne[3]; + if (n_expert_used < 2 || n_expert_used > MOE_WEIGHTED_REDUCTION_MAX_EXPERTS) { + return false; + } + + const int node_count = 2 * n_expert_used + mul_count - 1; + if (node_idx + node_count > cgraph->n_nodes) { + return false; + } + + std::vector ops(node_count, GGML_OP_VIEW); + ops[0] = GGML_OP_MUL; + if (mul_count == 2) { + ops[1] = GGML_OP_MUL; + } + std::vector views; + views.reserve(n_expert_used); + const ggml_tensor * previous = nullptr; + int n_adds = 0; + for (int offset = mul_count; offset < node_count; ++offset) { + const ggml_tensor * candidate = cgraph->nodes[node_idx + offset]; + ops[offset] = candidate->op; + + if (candidate->op == GGML_OP_VIEW) { + const int expert = (int) views.size(); + if (expert >= n_expert_used || candidate->src[0] != weighted || candidate->view_src != weighted || + candidate->type != GGML_TYPE_F32 || candidate->ne[0] != weighted->ne[0] || + candidate->ne[1] != n_tokens || candidate->ne[2] != 1 || candidate->ne[3] != 1 || + candidate->nb[0] != weighted->nb[0] || candidate->nb[1] != weighted->nb[2] || + candidate->view_offs != (size_t) expert * weighted->nb[1]) { + return false; + } + views.push_back(candidate); + continue; + } + + if (candidate->op != GGML_OP_ADD || views.size() < 2 || n_adds + 1 >= (int) views.size()) { + return false; + } + const ggml_tensor * lhs = n_adds == 0 ? views[0] : previous; + const ggml_tensor * rhs = views[n_adds + 1]; + if (candidate->src[0] != lhs || candidate->src[1] != rhs || candidate->type != GGML_TYPE_F32) { + return false; + } + previous = candidate; + ++n_adds; + } + + if ((int) views.size() != n_expert_used || n_adds != n_expert_used - 1 || previous == nullptr) { + return false; + } + if (!ggml_is_contiguous(previous) || previous->ne[0] != weighted->ne[0] || + previous->ne[1] != n_tokens || previous->ne[2] != 1 || previous->ne[3] != 1) { + return false; + } + + const int output_idx = node_idx + node_count - 1; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, node_count, ops.data(), &output_idx, 1)) { + return false; + } + + match.experts = experts; + match.expert_scale = expert_scale; + match.weights = weights; + match.dst = cgraph->nodes[output_idx]; + match.node_count = node_count; + return true; +} + static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, @@ -3464,6 +3597,36 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, } } + std::initializer_list rms_norm_mul_rope_ops = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE }; + std::initializer_list rms_norm_mul_rope_set_rows_ops = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }; + + if (is_equal(rms_norm_mul_rope_set_rows_ops, ops) && ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 4 })) { + const ggml_tensor * rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + const ggml_tensor * rope = cgraph->nodes[node_idx + 2]; + const ggml_tensor * view = cgraph->nodes[node_idx + 3]; + const ggml_tensor * set_rows = cgraph->nodes[node_idx + 4]; + + if (ggml_check_edges(cgraph, node_idx, {{1, 0, 0}, {2, 0, 1}, {3, 0, 2}, {4, 0, 3}}) && + ggml_cuda_should_fuse_rms_norm_mul_rope(rms_norm, mul, rope) && + ggml_cuda_should_fuse_rope_set_rows(rope, view, set_rows)) { + int out_nodes[] = { node_idx + 4 }; + return ggml_cuda_check_fusion_memory_ranges(cgraph, node_idx, (int)ops.size(), out_nodes, 1); + } + } + + if (is_equal(rms_norm_mul_rope_ops, ops) && ggml_can_fuse(cgraph, node_idx, ops)) { + const ggml_tensor * rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + const ggml_tensor * rope = cgraph->nodes[node_idx + 2]; + + if (ggml_cuda_should_fuse_rms_norm_mul_rope(rms_norm, mul, rope)) { + int out_nodes[] = { node_idx + 2 }; + return ggml_cuda_check_fusion_memory_ranges(cgraph, node_idx, (int)ops.size(), out_nodes, 1); + } + return false; + } + std::initializer_list rope_set_rows_ops = { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }; if (is_equal(rope_set_rows_ops, ops) && ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 2 })) { @@ -3472,10 +3635,31 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, const ggml_tensor * set_rows = cgraph->nodes[node_idx + 2]; if (ggml_cuda_should_fuse_rope_set_rows(rope, view, set_rows)) { - return true; + int out_nodes[] = { node_idx + 2 }; + return ggml_cuda_check_fusion_memory_ranges(cgraph, node_idx, (int)ops.size(), out_nodes, 1); } } + // scale_out*sigmoid(scale_in*post) as the hc_post weights (qwen4exp hc_combine) + std::initializer_list hc_post_gate_ops = { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_DSV4_HC_POST }; + + if (is_equal(hc_post_gate_ops, ops) && ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 3 })) { + const ggml_tensor * scale_in = cgraph->nodes[node_idx]; + const ggml_tensor * sigmoid = cgraph->nodes[node_idx + 1]; + const ggml_tensor * scale_out = cgraph->nodes[node_idx + 2]; + + // the chain must feed hc_post->src[2]; a bias on either scale is not handled + if (ggml_check_edges(cgraph, node_idx, {{1, 0, 0}, {2, 0, 1}, {3, 2, 2}}) && + ggml_get_unary_op(sigmoid) == GGML_UNARY_OP_SIGMOID && + ggml_get_op_params_f32(scale_in, 1) == 0.0f && ggml_get_op_params_f32(scale_out, 1) == 0.0f && + scale_in->src[0]->type == GGML_TYPE_F32 && scale_out->type == GGML_TYPE_F32 && + ggml_are_same_shape(scale_in->src[0], scale_out)) { + int out_nodes[] = { node_idx + 3 }; + return ggml_cuda_check_fusion_memory_ranges(cgraph, node_idx, (int)ops.size(), out_nodes, 1); + } + return false; + } + if (!ggml_can_fuse(cgraph, node_idx, ops)) { return false; } @@ -3522,10 +3706,26 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, return true; } + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_SCALE) { + const ggml_tensor * rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor * scale = cgraph->nodes[node_idx+1]; + + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + + float bias; + memcpy(&bias, (const float *) scale->op_params + 1, sizeof(float)); + + return bias == 0.0f && scale->type == GGML_TYPE_F32; + } + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_SSM_CONV && ops.begin()[1] == GGML_OP_UNARY && unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_SILU) { const ggml_tensor * ssm_conv = cgraph->nodes[node_idx]; const ggml_tensor * silu = cgraph->nodes[node_idx+1]; + if (ggml_get_unary_op(silu) != unary_ops.begin()[0]) { + return false; + } if (ssm_conv->type != GGML_TYPE_F32 || silu->type != GGML_TYPE_F32) { return false; @@ -3534,6 +3734,31 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, return true; } + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_SSM_CONV && ops.begin()[1] == GGML_OP_ADD + && ops.begin()[2] == GGML_OP_UNARY && unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_SILU) { + const ggml_tensor * ssm_conv = cgraph->nodes[node_idx]; + const ggml_tensor * add = cgraph->nodes[node_idx+1]; + const ggml_tensor * silu = cgraph->nodes[node_idx+2]; + if (ggml_get_unary_op(silu) != unary_ops.begin()[0]) { + return false; + } + + if (ssm_conv->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 || silu->type != GGML_TYPE_F32) { + return false; + } + + // ADD must consume ssm_conv's output and broadcast a 1-D channel-wise bias. + const ggml_tensor * bias = (add->src[0] == ssm_conv) ? add->src[1] : add->src[0]; + if (bias->type != GGML_TYPE_F32 || !ggml_is_contiguous(bias)) { + return false; + } + if (ggml_nelements(bias) != ssm_conv->ne[0] || bias->ne[0] != ssm_conv->ne[0]) { + return false; + } + + return true; + } + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL && unary_ops.size() == 1 && (unary_ops.begin()[0] == GGML_UNARY_OP_SILU || unary_ops.begin()[0] == GGML_UNARY_OP_SIGMOID || unary_ops.begin()[0] == GGML_UNARY_OP_SOFTPLUS)) { const ggml_tensor * unary = cgraph->nodes[node_idx]; @@ -3543,7 +3768,7 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, return false; } - if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) { + if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16 && unary->type != GGML_TYPE_BF16) { return false; } @@ -3562,28 +3787,865 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, return true; } + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_SQR + && unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_RELU) { + const ggml_tensor * unary = cgraph->nodes[node_idx]; + const ggml_tensor * sqr = cgraph->nodes[node_idx+1]; + + if (ggml_get_unary_op(unary) != GGML_UNARY_OP_RELU) { + return false; + } + + if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) { + return false; + } + + if (unary->type != sqr->type) { + return false; + } + + if (!ggml_is_contiguous(unary->src[0])) { + return false; + } + + return true; + } + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_SCALE && ops.begin()[1] == GGML_OP_UNARY && ops.begin()[2] == GGML_OP_SCALE && unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_TANH) { const ggml_tensor *scale = cgraph->nodes[node_idx]; const ggml_tensor *tanh = cgraph->nodes[node_idx+1]; const ggml_tensor *scale2 = cgraph->nodes[node_idx+2]; - GGML_ASSERT(scale->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT(scale->type == GGML_TYPE_F32); + // the fused softcap kernel is F32 only + if (scale->src[0]->type != GGML_TYPE_F32 || scale->type != GGML_TYPE_F32) { + return false; + } + + if (ggml_get_unary_op(tanh) != GGML_UNARY_OP_TANH) { + return false; + } + + // Check for bias + if (ggml_get_op_params_f32(scale, 1) != 0.0f || ggml_get_op_params_f32(scale2, 1) != 0.0f) { + return false; + } + + return true; + } + + return false; +} + +// try and fuse nodes and return the number of nodes to skip +static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { + + if (ggml_cuda_fusion_disabled()) { + return 0; + } + + ggml_tensor * node = cgraph->nodes[i]; + + if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() && + ggml_cuda_match_shared_expert(cgraph, i, i + 3) && + ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) { + const int outputs[] = { i + 2, i + 5 }; + if (ggml_cuda_check_fusion_memory_ranges(cgraph, i, 6, outputs, 2)) { + ggml_tensor * routed = cgraph->nodes[i + 2]; + ggml_tensor * shared = cgraph->nodes[i + 5]; + const ggml_tensor * up = routed->src[1]; + ggml_cuda_mm_fusion_args_host fusion{}; + fusion.gate = routed->src[0]->src[0]; + fusion.glu_op = ggml_get_glu_op(routed); + fusion.glu_limit = ggml_get_op_params_f32(routed, 3); + fusion.shared_up = shared->src[1]->src[0]; + fusion.shared_gate = shared->src[0]->src[0]; + fusion.shared_dst = shared; + ggml_cuda_mul_mat_vec_q(*cuda_ctx, up->src[0], up->src[1], up->src[2], routed, &fusion); + return 5; + } + } + + if (node->op == GGML_OP_MUL) { + ggml_cuda_moe_weighted_reduction_match match; + if (ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) { + const int output_idx = i + match.node_count - 1; + if (ggml_cuda_check_fusion_memory_ranges(cgraph, i, match.node_count, &output_idx, 1)) { + ggml_cuda_op_moe_weighted_reduction( + *cuda_ctx, match.experts, match.expert_scale, match.weights, match.dst); + return match.node_count - 1; + } + } + } + + // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache + if (node->op == GGML_OP_GATED_DELTA_NET) { + ggml_cuda_gated_delta_net_fused_cache fused_state_cpy; + const int nodes_to_skip = ggml_cuda_try_gdn_cache_fusion(cgraph, i, fused_state_cpy); + if (nodes_to_skip > 0) { +#ifdef GGML_CUDA_DEBUG + GGML_LOG_INFO("%s: fused gated_delta_net snapshot copies for %s (skipped %d nodes)\n", + __func__, node->name, nodes_to_skip); +#endif + ggml_cuda_op_gated_delta_net_fused_cache(*cuda_ctx, node, fused_state_cpy); + return nodes_to_skip; + } + } + + // ssm_scan -> cpy: scatter recurrent-state snapshots into the cache + if (node->op == GGML_OP_SSM_SCAN) { + ggml_cuda_ssm_scan_fused_cache fused_state_cpy; + const int nodes_to_skip = ggml_cuda_try_ssm_scan_cache_fusion(cgraph, i, fused_state_cpy); + if (nodes_to_skip > 0) { +#ifdef GGML_CUDA_DEBUG + GGML_LOG_INFO("%s: fused ssm_scan snapshot copies for %s (skipped %d nodes)\n", + __func__, node->name, nodes_to_skip); +#endif + ggml_cuda_op_ssm_scan_fused_cache(*cuda_ctx, node, fused_state_cpy); + return nodes_to_skip; + } + } + + //topk-moe + if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || + cgraph->nodes[i]->op == GGML_OP_ARGSORT) { + ggml_cuda_topk_moe_args args; + const bool can_fuse = ggml_cuda_topk_moe_fusion(cgraph, i, args); + std::vector ops; + ops.reserve(13); // max ops; avoids gcc -Wstringop-overflow false positive + + if (can_fuse) { + const ggml_tensor * logits = node->src[0]; + ggml_tensor * weights = nullptr; + ggml_tensor * ids = nullptr; + const ggml_tensor * bias = nullptr; + const ggml_tensor * clamp = nullptr; + const ggml_tensor * scale = nullptr; + + if (!args.delayed_softmax) { + int out_nodes[2]; // nodes which can't be elided + + if (args.sigmoid) { + ops.insert(ops.end(), { GGML_OP_UNARY }); + } else if (args.sqrt_softplus) { + ops.insert(ops.end(), { GGML_OP_UNARY, GGML_OP_SQRT }); + } else { + ops.insert(ops.end(), { GGML_OP_SOFT_MAX }); + } + ops.push_back(GGML_OP_RESHAPE); + if (args.prob_bias) { + ops.push_back(GGML_OP_ADD); + bias = cgraph->nodes[i + ops.size() - 1]->src[1]; + } + if (args.n_groups > 0) { + for (const auto & p : topk_moe_group_pattern) { + ops.push_back(p.op); + } + } + ops.insert(ops.end(), { GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }); + out_nodes[0] = i + ops.size() - 2; + ids = cgraph->nodes[out_nodes[0]]; + + if (args.norm) { + ops.insert(ops.end(), + { GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE }); + clamp = cgraph->nodes[i + ops.size() - 3]; + } + if (args.scale) { + ops.insert(ops.end(), { GGML_OP_SCALE }); + scale = cgraph->nodes[i + ops.size() - 1]; + } + + weights = cgraph->nodes[i + ops.size() - 1]; + out_nodes[1] = i + ops.size() - 1; + + if (ggml_can_fuse_subgraph(cgraph, i, ops.size(), ops.data(), out_nodes, 2) && + ggml_cuda_should_use_topk_moe(node, logits, weights, ids) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, ops.size(), out_nodes, 2, /*is_topk_moe=*/true)) { + ggml_cuda_op_topk_moe(*cuda_ctx, logits, weights, ids, clamp, scale, bias, args); + return ops.size() - 1; + } + } else if (!args.norm && !args.prob_bias) { + //special case gpt-oss, no norm, no bias. + ops.insert(ops.end(), { GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE, + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE }); + weights = cgraph->nodes[i + 5]; + ids = cgraph->nodes[i + 1]; + const ggml_tensor * softmax = cgraph->nodes[i + 4]; + + int out_nodes[2] = { i + 1, i + 5 }; + if (ggml_can_fuse_subgraph(cgraph, i, ops.size(), ops.data(), out_nodes, 2) && + ggml_cuda_should_use_topk_moe(softmax, logits, weights, ids) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, ops.size(), out_nodes, 2, /*is_topk_moe=*/true)) { + ggml_cuda_op_topk_moe(*cuda_ctx, logits, weights, ids, clamp, scale, bias, args); + return ops.size() - 1; + } + } + } + } + + //RoPE + view + set-rows + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }, {})) { + ggml_tensor * rope = cgraph->nodes[i]; + ggml_tensor * set_rows = cgraph->nodes[i + 2]; + + ggml_cuda_op_rope_fused(*cuda_ctx, rope, set_rows); + return 2; + } + + // Snake activation: y = x + sin(a*x)^2 * inv_b + // Naive 5-op decomposition emitted by frontends: mul -> sin -> sqr -> mul -> add + if (ggml_can_fuse_subgraph(cgraph, i, + { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }, + { i + 4 })) { + const ggml_tensor * mul0 = cgraph->nodes[i]; + const ggml_tensor * sqr = cgraph->nodes[i + 2]; + const ggml_tensor * mul1 = cgraph->nodes[i + 3]; + ggml_tensor * add = cgraph->nodes[i + 4]; + + // x carries the full activation shape, a is the broadcast operand + const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + + // mul1 reads sqr and inv_b in either operand order + const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + + // closure check: the trailing add must read the same x as the leading mul + const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; + + // Kernel iterates over total = T * C, so x and add must be 2D and + // a / inv_b must collapse to [1, C, 1, 1]. Higher dims are not handled. + const bool dim_ok = (x->ne[2] == 1 && x->ne[3] == 1) && + (add->ne[2] == 1 && add->ne[3] == 1) && + (a->ne[2] == 1 && a->ne[3] == 1); + const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; + + // x is in the supported whitelist and every chain intermediate shares + // x's type. launch_snake reads a and inv_b as const float *, so they + // stay F32. + const ggml_tensor * sin1 = cgraph->nodes[i + 1]; + const bool types_ok = (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && + (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && + (mul0->type == x->type) && (sin1->type == x->type) && + (sqr->type == x->type) && (mul1->type == x->type) && + (add->type == x->type); + + // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous + const bool contig_ok = ggml_is_contiguous(x) && ggml_is_contiguous(add) && + ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); + + if (types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x) { + ggml_cuda_op_snake_fused(*cuda_ctx, x, a, inv_b, add); + return 4; + } + } + + // multi-(add or mul) + if (node->op == GGML_OP_ADD || node->op == GGML_OP_MUL) { + int n_fuse = 0; + ggml_op ops[8]; + std::fill(ops, ops + 8, node->op); + + for (; n_fuse <= 6; ++n_fuse) { + if (!ggml_can_fuse(cgraph, i + n_fuse, ops + n_fuse, 2)) { + break; + } + if (cgraph->nodes[i + n_fuse] != cgraph->nodes[i + n_fuse + 1]->src[0]) { + break; + } + if (!ggml_are_same_layout(cgraph->nodes[i + n_fuse]->src[1], cgraph->nodes[i + n_fuse + 1]->src[1])) { + break; + } + } + + n_fuse++; + + if (n_fuse > 1) { + ggml_tensor fused_node; + memcpy(&fused_node, node, sizeof(ggml_tensor)); + for (int j = 0; j < n_fuse - 1; ++j) { + fused_node.src[j + 2] = cgraph->nodes[i + j + 1]->src[1]; + } + fused_node.data = cgraph->nodes[i + n_fuse - 1]->data; + if (node->op == GGML_OP_ADD) { + ggml_cuda_op_fused_add(*cuda_ctx, &fused_node, n_fuse); + } else { + ggml_cuda_op_fused_mul(*cuda_ctx, &fused_node, n_fuse); + } + return n_fuse - 1; + } + } + + bool fused_mul_mat_vec = false; + int fused_node_count = 0; + + auto get_mul_mat_scale = [](const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * { + const bool scale_lhs_mm = scale_node->src[0] == mm_node; + const bool scale_rhs_mm = scale_node->src[1] == mm_node; + if (!scale_lhs_mm && !scale_rhs_mm) { + return nullptr; + } + + const ggml_tensor * scale = scale_lhs_mm ? scale_node->src[1] : scale_node->src[0]; + if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != 1 || + !ggml_are_same_shape(scale_node, mm_node)) { + return nullptr; + } + + return scale; + }; + + auto get_mul_mat_id_scale = [](const ggml_tensor * reshape, const ggml_tensor * repeat, const ggml_tensor * getrows, + const ggml_tensor * scale_node, const ggml_tensor * mm_node) -> const ggml_tensor * { + if (repeat->src[0] != reshape || getrows->src[0] != repeat || getrows->src[1] != mm_node->src[2]) { + return nullptr; + } + if (!((scale_node->src[0] == mm_node && scale_node->src[1] == getrows) || + (scale_node->src[0] == getrows && scale_node->src[1] == mm_node))) { + return nullptr; + } + + const ggml_tensor * scale = reshape->src[0]; + if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != mm_node->src[0]->ne[2] || + !ggml_are_same_shape(scale_node, mm_node)) { + return nullptr; + } + + return scale; + }; + + auto get_bias_tensor = [](const ggml_tensor * bias_node, const ggml_tensor * mul_node, ggml_op op_bias) -> const ggml_tensor * { + if (op_bias == GGML_OP_ADD) { + if (bias_node->src[0] == mul_node) { + return bias_node->src[1]; + } + if (bias_node->src[1] == mul_node) { + return bias_node->src[0]; + } + return nullptr; + } + GGML_ASSERT(op_bias == GGML_OP_ADD_ID); + GGML_ASSERT(bias_node->src[0] == mul_node); + return bias_node->src[1]; + }; + + // gate + glu + up, with optional scale/bias on both lanes. + for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) { + const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID; + + if (op == GGML_OP_MUL_MAT) { + for (const bool with_bias : { false, true }) { + const int gate_idx = i; + const int gate_scale_idx = i + 1; + const int gate_bias_idx = with_bias ? i + 2 : -1; + const int up_idx = with_bias ? i + 3 : i + 2; + const int up_scale_idx = up_idx + 1; + const int up_bias_idx = with_bias ? up_idx + 2 : -1; + const int glu_idx = with_bias ? up_idx + 3 : up_idx + 2; + + const int out_nodes[] = { glu_idx }; + ggml_op ops[7]; + if (with_bias) { + ops[0] = op; + ops[1] = GGML_OP_MUL; + ops[2] = bias_op; + ops[3] = op; + ops[4] = GGML_OP_MUL; + ops[5] = bias_op; + ops[6] = GGML_OP_GLU; + } else { + ops[0] = op; + ops[1] = GGML_OP_MUL; + ops[2] = op; + ops[3] = GGML_OP_MUL; + ops[4] = GGML_OP_GLU; + } + const int n_ops = with_bias ? 7 : 5; + + if (!ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1) || + !ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) { + continue; + } + + ggml_tensor * gate_n = cgraph->nodes[gate_idx]; + ggml_tensor * gate_scale_n = cgraph->nodes[gate_scale_idx]; + ggml_tensor * gate_out_n = with_bias ? cgraph->nodes[gate_bias_idx] : gate_scale_n; + ggml_tensor * up_n = cgraph->nodes[up_idx]; + ggml_tensor * up_scale_n = cgraph->nodes[up_scale_idx]; + ggml_tensor * up_out_n = with_bias ? cgraph->nodes[up_bias_idx] : up_scale_n; + const ggml_tensor * glu = cgraph->nodes[glu_idx]; + + if (!ggml_cuda_should_fuse_mul_mat(up_n, gate_n, glu, + with_bias ? up_out_n : nullptr, with_bias ? gate_out_n : nullptr, up_scale_n, gate_scale_n)) { + continue; + } + + const ggml_tensor * gate_scale = get_mul_mat_scale(gate_scale_n, gate_n); + const ggml_tensor * up_scale = get_mul_mat_scale(up_scale_n, up_n); + if (!gate_scale || !up_scale) { + continue; + } + + const ggml_tensor * up_bias = with_bias ? get_bias_tensor(up_out_n, up_scale_n, bias_op) : nullptr; + const ggml_tensor * gate_bias = with_bias ? get_bias_tensor(gate_out_n, gate_scale_n, bias_op) : nullptr; + if (with_bias && (!ggml_are_same_shape(gate_out_n->src[0], gate_out_n->src[1]) || + !ggml_are_same_shape(up_out_n->src[0], up_out_n->src[1]))) { + continue; + } + + const ggml_tensor * src0 = up_n->src[0]; + const ggml_tensor * src1 = up_n->src[1]; + const ggml_tensor * ids = up_n->src[2]; + + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate_n->src[0]; + fusion_data.x_bias = up_bias; + fusion_data.gate_bias = gate_bias; + fusion_data.x_scale = up_scale; + fusion_data.gate_scale = gate_scale; + fusion_data.glu_op = ggml_get_glu_op(glu); + fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + } + + if (fused_mul_mat_vec) { + break; + } + } else { + for (const bool with_bias : { false, true }) { + const int gate_idx = i; + const int gate_scale_idx = i + 4; + const int gate_bias_idx = with_bias ? i + 5 : -1; + const int up_idx = with_bias ? i + 6 : i + 5; + const int up_scale_idx = up_idx + 4; + const int up_bias_idx = with_bias ? up_idx + 5 : -1; + const int glu_idx = with_bias ? up_idx + 6 : up_idx + 5; + + const int out_nodes[] = { glu_idx }; + ggml_op ops[13]; + if (with_bias) { + ops[0] = op; + ops[1] = GGML_OP_RESHAPE; + ops[2] = GGML_OP_REPEAT; + ops[3] = GGML_OP_GET_ROWS; + ops[4] = GGML_OP_MUL; + ops[5] = bias_op; + ops[6] = op; + ops[7] = GGML_OP_RESHAPE; + ops[8] = GGML_OP_REPEAT; + ops[9] = GGML_OP_GET_ROWS; + ops[10] = GGML_OP_MUL; + ops[11] = bias_op; + ops[12] = GGML_OP_GLU; + } else { + ops[0] = op; + ops[1] = GGML_OP_RESHAPE; + ops[2] = GGML_OP_REPEAT; + ops[3] = GGML_OP_GET_ROWS; + ops[4] = GGML_OP_MUL; + ops[5] = op; + ops[6] = GGML_OP_RESHAPE; + ops[7] = GGML_OP_REPEAT; + ops[8] = GGML_OP_GET_ROWS; + ops[9] = GGML_OP_MUL; + ops[10] = GGML_OP_GLU; + } + const int n_ops = with_bias ? 13 : 11; + + if (!ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1) || + !ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) { + continue; + } + + ggml_tensor * gate_n = cgraph->nodes[gate_idx]; + ggml_tensor * gate_scale_n = cgraph->nodes[gate_scale_idx]; + ggml_tensor * gate_out_n = with_bias ? cgraph->nodes[gate_bias_idx] : gate_scale_n; + ggml_tensor * up_n = cgraph->nodes[up_idx]; + ggml_tensor * up_scale_n = cgraph->nodes[up_scale_idx]; + ggml_tensor * up_out_n = with_bias ? cgraph->nodes[up_bias_idx] : up_scale_n; + const ggml_tensor * glu = cgraph->nodes[glu_idx]; + + if (!ggml_cuda_should_fuse_mul_mat(up_n, gate_n, glu, + with_bias ? up_out_n : nullptr, with_bias ? gate_out_n : nullptr, up_scale_n, gate_scale_n)) { + continue; + } + + const ggml_tensor * gate_scale = get_mul_mat_id_scale(cgraph->nodes[gate_idx + 1], cgraph->nodes[gate_idx + 2], + cgraph->nodes[gate_idx + 3], gate_scale_n, gate_n); + const ggml_tensor * up_scale = get_mul_mat_id_scale(cgraph->nodes[up_idx + 1], cgraph->nodes[up_idx + 2], + cgraph->nodes[up_idx + 3], up_scale_n, up_n); + if (!gate_scale || !up_scale) { + continue; + } + + const ggml_tensor * up_bias = with_bias ? get_bias_tensor(up_out_n, up_scale_n, bias_op) : nullptr; + const ggml_tensor * gate_bias = with_bias ? get_bias_tensor(gate_out_n, gate_scale_n, bias_op) : nullptr; + + const ggml_tensor * src0 = up_n->src[0]; + const ggml_tensor * src1 = up_n->src[1]; + const ggml_tensor * ids = up_n->src[2]; + + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate_n->src[0]; + fusion_data.x_bias = up_bias; + fusion_data.gate_bias = gate_bias; + fusion_data.x_scale = up_scale; + fusion_data.gate_scale = gate_scale; + fusion_data.glu_op = ggml_get_glu_op(glu); + fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + } + + if (fused_mul_mat_vec) { + break; + } + } + + if (ggml_cuda_can_fuse(cgraph, i, { op, bias_op, op, bias_op, GGML_OP_GLU }, {})) { + ggml_tensor * glu = cgraph->nodes[i + 4]; + ggml_tensor * gate_bias_n = glu->src[0]; + ggml_tensor * up_bias_n = glu->src[1]; + + //we don't assume the order for {gate, up}. Instead infer it from the bias tensor + ggml_tensor * gate_n = nullptr; + ggml_tensor * up_n = nullptr; + + if (gate_bias_n->src[0] == cgraph->nodes[i] || gate_bias_n->src[1] == cgraph->nodes[i]) { + gate_n = cgraph->nodes[i]; + up_n = cgraph->nodes[i + 2]; + } else if (gate_bias_n->src[0] == cgraph->nodes[i + 2] || gate_bias_n->src[1] == cgraph->nodes[i + 2]) { + gate_n = cgraph->nodes[i + 2]; + up_n = cgraph->nodes[i]; + } else { + continue; + } + + const ggml_tensor * up_bias_tensor = get_bias_tensor(up_bias_n, up_n, bias_op); + const ggml_tensor * gate_bias_tensor = get_bias_tensor(gate_bias_n, gate_n, bias_op); + + if (!up_bias_tensor || !gate_bias_tensor) { + continue; + } + + // we don't support repeating adds + if (bias_op == GGML_OP_ADD && (!ggml_are_same_shape(gate_bias_n->src[0], gate_bias_n->src[1]) || + !ggml_are_same_shape(up_bias_n->src[0], up_bias_n->src[1]))) { + continue; + } + + const ggml_tensor * src0 = up_n->src[0]; + const ggml_tensor * src1 = up_n->src[1]; + const ggml_tensor * ids = up_n->src[2]; + + if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n)) { + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate_n->src[0]; + fusion_data.x_bias = up_bias_tensor; + fusion_data.gate_bias = gate_bias_tensor; + fusion_data.glu_op = ggml_get_glu_op(glu); + fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + + ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = 5; + break; + } + + if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate_n->src[0]; + fusion_data.x_bias = up_bias_tensor; + fusion_data.gate_bias = gate_bias_tensor; + fusion_data.glu_op = ggml_get_glu_op(glu); + fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = 5; + break; + } + } else if (ggml_cuda_can_fuse(cgraph, i, { op, op, GGML_OP_GLU }, {})) { + ggml_tensor * glu = cgraph->nodes[i + 2]; + ggml_tensor * gate = glu->src[0]; + ggml_tensor * up = glu->src[1]; + + bool ok = (gate == cgraph->nodes[i] && up == cgraph->nodes[i + 1]) || + (gate == cgraph->nodes[i + 1] && up == cgraph->nodes[i]); + + if (!ok) { + continue; + } + + const ggml_tensor * src0 = up->src[0]; + const ggml_tensor * src1 = up->src[1]; + const ggml_tensor * ids = up->src[2]; + + if (ggml_cuda_should_fuse_mul_mat_vec_f(up)) { + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate->src[0]; + fusion_data.glu_op = ggml_get_glu_op(glu); + fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + + ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = 3; + break; + } + + if (ggml_cuda_should_fuse_mul_mat_vec_q(up)) { + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate->src[0]; + fusion_data.glu_op = ggml_get_glu_op(glu); + fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = 3; + break; + } + + if (ggml_cuda_should_fuse_mul_mat_q(up, gate, glu)) { + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.gate = gate->src[0]; + fusion_data.glu_op = ggml_get_glu_op(glu); + fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3); + + ggml_cuda_mul_mat_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = 3; + break; + } + } + } + + if (fused_mul_mat_vec) { + return fused_node_count - 1; + } + + fused_mul_mat_vec = false; + fused_node_count = 0; + + // mul_mat + scale + optional bias + for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) { + const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID; + + for (const bool with_bias : { false, true }) { + const int n_ops = op == GGML_OP_MUL_MAT ? (with_bias ? 3 : 2) : (with_bias ? 6 : 5); + const int out_nodes[] = { i + n_ops - 1 }; + ggml_op ops[6]; + if (op == GGML_OP_MUL_MAT) { + if (with_bias) { + ops[0] = op; + ops[1] = GGML_OP_MUL; + ops[2] = bias_op; + } else { + ops[0] = op; + ops[1] = GGML_OP_MUL; + } + } else { + if (with_bias) { + ops[0] = op; + ops[1] = GGML_OP_RESHAPE; + ops[2] = GGML_OP_REPEAT; + ops[3] = GGML_OP_GET_ROWS; + ops[4] = GGML_OP_MUL; + ops[5] = bias_op; + } else { + ops[0] = op; + ops[1] = GGML_OP_RESHAPE; + ops[2] = GGML_OP_REPEAT; + ops[3] = GGML_OP_GET_ROWS; + ops[4] = GGML_OP_MUL; + } + } + + if (!ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1) || + !ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) { + continue; + } + + ggml_tensor * mm_node = cgraph->nodes[i]; + ggml_tensor * scale_node = op == GGML_OP_MUL_MAT ? cgraph->nodes[i + 1] : cgraph->nodes[i + 4]; + ggml_tensor * out_node = with_bias ? cgraph->nodes[i + n_ops - 1] : scale_node; + + const ggml_tensor * scale = nullptr; + if (op == GGML_OP_MUL_MAT) { + scale = get_mul_mat_scale(scale_node, mm_node); + } else { + scale = get_mul_mat_id_scale(cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 3], scale_node, mm_node); + } + if (!scale) { + continue; + } + + const ggml_tensor * bias = with_bias ? get_bias_tensor(out_node, scale_node, bias_op) : nullptr; + if (with_bias && !bias) { + continue; + } + if (with_bias && bias_op == GGML_OP_ADD && !ggml_are_same_shape(out_node->src[0], out_node->src[1])) { + continue; + } + if (with_bias && bias_op == GGML_OP_ADD_ID && out_node->src[2] != mm_node->src[2]) { + continue; + } + + const ggml_tensor * src0 = mm_node->src[0]; + const ggml_tensor * src1 = mm_node->src[1]; + const ggml_tensor * ids = mm_node->src[2]; + + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.x_bias = bias; + fusion_data.x_scale = scale; + + if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) { + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = n_ops; + break; + } + } + if (fused_mul_mat_vec) { + break; + } + } + + if (fused_mul_mat_vec) { + return fused_node_count - 1; + } + + // mul_mat + add + for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) { + const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID; + + if (!ggml_can_fuse(cgraph, i, { op, bias_op })) { + continue; + } + + ggml_tensor * mm_node = cgraph->nodes[i]; + ggml_tensor * bias_node = cgraph->nodes[i + 1]; + + ggml_tensor * bias_tensor = nullptr; + if (bias_op == GGML_OP_ADD) { + if (bias_node->src[0] == mm_node) { + bias_tensor = bias_node->src[1]; + } else if (bias_node->src[1] == mm_node) { + bias_tensor = bias_node->src[0]; + } else { + continue; + } + } else { + if (bias_node->src[0] != mm_node) { + continue; + } + bias_tensor = bias_node->src[1]; + } + + const ggml_tensor * src0 = mm_node->src[0]; + const ggml_tensor * src1 = mm_node->src[1]; + const ggml_tensor * ids = mm_node->src[2]; + + if (bias_op == GGML_OP_ADD_ID && bias_node->src[2] != ids) { + continue; + } + + if (bias_op == GGML_OP_ADD && !ggml_are_same_shape(bias_node->src[0], bias_node->src[1])) { + continue; + } + + ggml_cuda_mm_fusion_args_host fusion_data{}; + fusion_data.x_bias = bias_tensor; + + if (ggml_cuda_should_fuse_mul_mat_vec_f(mm_node)) { + ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, bias_node, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = 2; + break; + } + + if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) { + ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, bias_node, &fusion_data); + fused_mul_mat_vec = true; + fused_node_count = 2; + break; + } + } + + if (fused_mul_mat_vec) { + return fused_node_count - 1; + } + + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }, {})) { + ggml_cuda_op_rms_norm_mul_rope_fused(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2], cgraph->nodes[i + 4]); + return 4; + } + + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE }, {})) { + ggml_cuda_op_rms_norm_mul_rope_fused(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2], nullptr); + return 2; + } + + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) { + ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]); + return 2; + } + + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL }, {})) { + ggml_cuda_op_rms_norm_fused(*cuda_ctx, node, cgraph->nodes[i + 1]); + return 1; + } + + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_SCALE }, {})) { + ggml_cuda_op_rms_norm_scale_fused(*cuda_ctx, node, cgraph->nodes[i + 1]); + return 1; + } + + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SSM_CONV, GGML_OP_ADD, GGML_OP_UNARY }, { GGML_UNARY_OP_SILU })) { + ggml_cuda_op_ssm_conv(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]); + return 2; + } - if (ggml_get_unary_op(tanh) != GGML_UNARY_OP_TANH) { - return false; - } + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SSM_CONV, GGML_OP_UNARY }, { GGML_UNARY_OP_SILU })) { + ggml_cuda_op_ssm_conv(*cuda_ctx, node, /*bias_add_node=*/ nullptr, cgraph->nodes[i + 1]); + return 1; + } - // Check for bias - if (ggml_get_op_params_f32(scale, 1) != 0.0f || ggml_get_op_params_f32(scale2, 1) != 0.0f) { - return false; - } + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SILU }) || + ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SIGMOID }) || + ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SOFTPLUS })) { + ggml_cuda_op_unary_mul(*cuda_ctx, node, cgraph->nodes[i + 1]); + return 1; + } - return true; + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_SQR }, { GGML_UNARY_OP_RELU })) { + ggml_cuda_op_relu_sqr(*cuda_ctx, node, cgraph->nodes[i + 1]); + return 1; } - return false; + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE, GGML_OP_DSV4_HC_POST }, { GGML_UNARY_OP_SIGMOID })) { + ggml_cuda_op_dsv4_hc_post(*cuda_ctx, cgraph->nodes[i + 3], node, cgraph->nodes[i + 2]); + return 3; + } + + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE }, { GGML_UNARY_OP_TANH })) { + ggml_cuda_op_softcap(*cuda_ctx, cgraph->nodes[i + 2], node); + return 2; + } + + return 0; } static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, const bool use_cuda_graph, const bool cuda_graph_update_required, const void * graph_key) { @@ -3597,6 +4659,13 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud ggml_cuda_concurrent_event * concurrent_event = nullptr; bool should_launch_concurrent_events = false; +#ifdef USE_CUDA_GRAPH + ggml_cuda_graph * graph = use_cuda_graph ? cuda_ctx->cuda_graph(graph_key) : nullptr; + const int chunk_size = ggml_cuda_graph_chunk_size(); + int chunk_end = chunk_size; + size_t n_chunks = 0; +#endif // USE_CUDA_GRAPH + const auto try_launch_concurrent_event = [&](const ggml_tensor * node) { if (stream_ctx.concurrent_events.find(node) != stream_ctx.concurrent_events.end()) { concurrent_event = &stream_ctx.concurrent_events[node]; @@ -3686,6 +4755,15 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; +#ifdef USE_CUDA_GRAPH + // fused nodes are skipped together, so a cut never splits a fusion + // a concurrent region must join before its capture can end + if (use_cuda_graph && chunk_size > 0 && i >= chunk_end && !is_concurrent_event_active) { + ggml_cuda_graph_launch_chunk(cuda_ctx, graph, n_chunks++); + CUDA_CHECK(cudaStreamBeginCapture(cuda_ctx->stream(), cudaStreamCaptureModeRelaxed)); + chunk_end = i + chunk_size; + } +#endif // USE_CUDA_GRAPH if (is_concurrent_event_active) { GGML_ASSERT(concurrent_event); @@ -3716,15 +4794,9 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud } } -#ifdef GGML_CUDA_DEBUG - const int nodes_fused = i - prev_i - 1; - if (nodes_fused > 0) { - GGML_LOG_INFO("nodes_fused: %d\n", nodes_fused); - } -#endif prev_i = i; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_cuda_is_view_or_noop(node)) { continue; } @@ -3732,357 +4804,29 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud continue; } - // start of fusion operations - static bool disable_fusion = (getenv("GGML_CUDA_DISABLE_FUSION") != nullptr); - if (!disable_fusion) { - ggml_cuda_topk_moe_args args; - - if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || - cgraph->nodes[i]->op == GGML_OP_ARGSORT) { - const bool can_fuse = ggml_cuda_topk_moe_fusion(cgraph, i, args); - - std::vector ops; - - if (can_fuse) { - const ggml_tensor * logits = node->src[0]; - ggml_tensor * weights = nullptr; - ggml_tensor * ids = nullptr; - const ggml_tensor * bias = nullptr; - const ggml_tensor * clamp = nullptr; - const ggml_tensor * scale = nullptr; - - if (!args.delayed_softmax) { - ggml_op gating_op = args.sigmoid ? GGML_OP_UNARY : GGML_OP_SOFT_MAX; - int out_nodes[2]; // nodes which can't be elided - - if (args.prob_bias) { - bias = cgraph->nodes[i + 2]->src[1]; - ops.insert(ops.end(), { gating_op, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, - GGML_OP_VIEW, GGML_OP_GET_ROWS }); - out_nodes[0] = i + 4; - ids = cgraph->nodes[i + 4]; - } else { - ops.insert(ops.end(), { gating_op, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, - GGML_OP_GET_ROWS }); - out_nodes[0] = i + 3; - ids = cgraph->nodes[i + 3]; - } - - if (args.norm) { - ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, - GGML_OP_DIV, GGML_OP_RESHAPE }); - clamp = cgraph->nodes[i + ops.size() - 3]; - } - if (args.scale) { - ops.insert(ops.end(), { GGML_OP_SCALE }); - scale = cgraph->nodes[i + ops.size() - 1]; - } - - weights = cgraph->nodes[i + ops.size() - 1]; - out_nodes[1] = i + ops.size() - 1; - - if (ggml_can_fuse_subgraph(cgraph, i, ops.size(), ops.data(), out_nodes, 2) && - ggml_cuda_should_use_topk_moe(node, logits, weights, ids) && - ggml_cuda_check_fusion_memory_ranges(cgraph, i, ops.size(), out_nodes, 2, /*is_topk_moe=*/ true)) { - ggml_cuda_op_topk_moe(*cuda_ctx, logits, weights, ids, clamp, scale, bias, args); - i += ops.size() - 1; - continue; - } - } else if (!args.norm && !args.prob_bias) { - //special case gpt-oss, no norm, no bias. - ops.insert(ops.end(), { GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, - GGML_OP_RESHAPE, GGML_OP_SOFT_MAX, GGML_OP_RESHAPE }); - weights = cgraph->nodes[i + 5]; - ids = cgraph->nodes[i + 1]; - const ggml_tensor * softmax = cgraph->nodes[i + 4]; - - int out_nodes[2] = { i + 1, i + 5 }; - if (ggml_can_fuse_subgraph(cgraph, i, ops.size(), ops.data(), out_nodes, 2) && - ggml_cuda_should_use_topk_moe(softmax, logits, weights, ids) && - ggml_cuda_check_fusion_memory_ranges(cgraph, i, ops.size(), out_nodes, 2, /*is_topk_moe=*/ true)) { - ggml_cuda_op_topk_moe(*cuda_ctx, logits, weights, ids, clamp, scale, bias, args); - i += ops.size() - 1; - continue; - } - } - } - } - - if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }, {})) { - ggml_tensor * rope = cgraph->nodes[i]; - ggml_tensor * set_rows = cgraph->nodes[i + 2]; - - ggml_cuda_op_rope_fused(*cuda_ctx, rope, set_rows); - i += 2; - continue; - } - - if (node->op == GGML_OP_ADD || node->op == GGML_OP_MUL) { - int n_fuse = 0; - ggml_op ops[8]; - std::fill(ops, ops + 8, node->op); - - for (; n_fuse <= 6; ++n_fuse){ - if (!ggml_can_fuse(cgraph, i + n_fuse, ops + n_fuse, 2)) { - break; - } - if (cgraph->nodes[i + n_fuse] != cgraph->nodes[i + n_fuse + 1]->src[0]) { - break; - } - if (!ggml_are_same_layout(cgraph->nodes[i + n_fuse]->src[1], cgraph->nodes[i + n_fuse + 1]->src[1])) { - break; - } - } - - n_fuse++; - - if (n_fuse > 1) { - ggml_tensor fused_node; - memcpy(&fused_node, node, sizeof(ggml_tensor)); - for (int j = 0; j < n_fuse - 1; ++j) { - fused_node.src[j + 2] = cgraph->nodes[i + j + 1]->src[1]; - } - fused_node.data = cgraph->nodes[i + n_fuse - 1]->data; - if (node->op == GGML_OP_ADD) { - ggml_cuda_op_fused_add(*cuda_ctx, &fused_node, n_fuse); - } else { - ggml_cuda_op_fused_mul(*cuda_ctx, &fused_node, n_fuse); - } - i += n_fuse - 1; - - continue; - } - } - - bool fused_mul_mat_vec = false; - int fused_node_count = 0; - - for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) { - const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID; - - if (ggml_cuda_can_fuse(cgraph, i, { op, bias_op, op, bias_op, GGML_OP_GLU }, {})) { - ggml_tensor * glu = cgraph->nodes[i + 4]; - ggml_tensor * gate_bias_n = glu->src[0]; - ggml_tensor * up_bias_n = glu->src[1]; - - //we don't assume the order for {gate, up}. Instead infer it from the bias tensor - ggml_tensor * gate_n = nullptr; - ggml_tensor * up_n = nullptr; - - if (gate_bias_n->src[0] == cgraph->nodes[i] || gate_bias_n->src[1] == cgraph->nodes[i]) { - gate_n = cgraph->nodes[i]; - up_n = cgraph->nodes[i + 2]; - } else if (gate_bias_n->src[0] == cgraph->nodes[i + 2] || gate_bias_n->src[1] == cgraph->nodes[i + 2]) { - gate_n = cgraph->nodes[i + 2]; - up_n = cgraph->nodes[i]; - } else { - continue; - } - - auto get_bias_tensor = [](const ggml_tensor * bias_node, const ggml_tensor * mul_node, ggml_op op_bias) { - if (op_bias == GGML_OP_ADD) { - if (bias_node->src[0] == mul_node) { - return bias_node->src[1]; - } - if (bias_node->src[1] == mul_node) { - return bias_node->src[0]; - } - return (ggml_tensor *) nullptr; - } - GGML_ASSERT(op_bias == GGML_OP_ADD_ID); - GGML_ASSERT(bias_node->src[0] == mul_node); - return bias_node->src[1]; - }; - - ggml_tensor * up_bias_tensor = get_bias_tensor(up_bias_n, up_n, bias_op); - ggml_tensor * gate_bias_tensor = get_bias_tensor(gate_bias_n, gate_n, bias_op); - - if (!up_bias_tensor || !gate_bias_tensor) { - continue; - } - - // we don't support repeating adds - if (bias_op == GGML_OP_ADD && - (!ggml_are_same_shape(gate_bias_n->src[0], gate_bias_n->src[1]) || - !ggml_are_same_shape(up_bias_n->src[0], up_bias_n->src[1]))) { - continue; - } - - const ggml_tensor * src0 = up_n->src[0]; - const ggml_tensor * src1 = up_n->src[1]; - const ggml_tensor * ids = up_n->src[2]; - - if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n)) { - ggml_cuda_mm_fusion_args_host fusion_data{}; - fusion_data.gate = gate_n->src[0]; - fusion_data.x_bias = up_bias_tensor; - fusion_data.gate_bias = gate_bias_tensor; - fusion_data.glu_op = ggml_get_glu_op(glu); - - ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data); - fused_mul_mat_vec = true; - fused_node_count = 5; - break; - } - - if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) { - ggml_cuda_mm_fusion_args_host fusion_data{}; - fusion_data.gate = gate_n->src[0]; - fusion_data.x_bias = up_bias_tensor; - fusion_data.gate_bias = gate_bias_tensor; - fusion_data.glu_op = ggml_get_glu_op(glu); - - ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); - fused_mul_mat_vec = true; - fused_node_count = 5; - break; - } - } else if (ggml_cuda_can_fuse(cgraph, i, { op, op, GGML_OP_GLU }, {})) { - ggml_tensor * glu = cgraph->nodes[i + 2]; - ggml_tensor * gate = glu->src[0]; - ggml_tensor * up = glu->src[1]; - - bool ok = (gate == cgraph->nodes[i] && up == cgraph->nodes[i + 1]) - || (gate == cgraph->nodes[i + 1] && up == cgraph->nodes[i]); - - if (!ok) continue; - - const ggml_tensor * src0 = up->src[0]; - const ggml_tensor * src1 = up->src[1]; - const ggml_tensor * ids = up->src[2]; - - if (ggml_cuda_should_fuse_mul_mat_vec_f(up)) { - ggml_cuda_mm_fusion_args_host fusion_data{}; - fusion_data.gate = gate->src[0]; - fusion_data.glu_op = ggml_get_glu_op(glu); - - ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, glu, &fusion_data); - fused_mul_mat_vec = true; - fused_node_count = 3; - break; - } - - if (ggml_cuda_should_fuse_mul_mat_vec_q(up)) { - ggml_cuda_mm_fusion_args_host fusion_data{}; - fusion_data.gate = gate->src[0]; - fusion_data.glu_op = ggml_get_glu_op(glu); - - ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, glu, &fusion_data); - fused_mul_mat_vec = true; - fused_node_count = 3; - break; - } - } - } - - if (fused_mul_mat_vec) { - i += fused_node_count - 1; - continue; - } - - fused_mul_mat_vec = false; - fused_node_count = 0; - - for (ggml_op op : { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT_ID }) { - const ggml_op bias_op = op == GGML_OP_MUL_MAT ? GGML_OP_ADD : GGML_OP_ADD_ID; - - if (!ggml_can_fuse(cgraph, i, { op, bias_op })) { - continue; - } - - ggml_tensor * mm_node = cgraph->nodes[i]; - ggml_tensor * bias_node = cgraph->nodes[i + 1]; - - ggml_tensor * bias_tensor = nullptr; - if (bias_op == GGML_OP_ADD) { - if (bias_node->src[0] == mm_node) { - bias_tensor = bias_node->src[1]; - } else if (bias_node->src[1] == mm_node) { - bias_tensor = bias_node->src[0]; - } else { - continue; - } - } else { - if (bias_node->src[0] != mm_node) { - continue; - } - bias_tensor = bias_node->src[1]; - } - - const ggml_tensor * src0 = mm_node->src[0]; - const ggml_tensor * src1 = mm_node->src[1]; - const ggml_tensor * ids = mm_node->src[2]; - - if (bias_op == GGML_OP_ADD_ID && bias_node->src[2] != ids) { - continue; - } - - if (bias_op == GGML_OP_ADD && !ggml_are_same_shape(bias_node->src[0], bias_node->src[1])) { - continue; - } - - ggml_cuda_mm_fusion_args_host fusion_data{}; - fusion_data.x_bias = bias_tensor; - - if (ggml_cuda_should_fuse_mul_mat_vec_f(mm_node)) { - ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, bias_node, &fusion_data); - fused_mul_mat_vec = true; - fused_node_count = 2; - break; - } - - if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) { - ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, bias_node, &fusion_data); - fused_mul_mat_vec = true; - fused_node_count = 2; - break; - } - } - - if (fused_mul_mat_vec) { - i += fused_node_count - 1; - continue; - } - - if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD}, {})) { - ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); - i += 2; - continue; - } - - if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL}, {})) { - ggml_cuda_op_rms_norm_fused(*cuda_ctx, node, cgraph->nodes[i+1]); - i++; - continue; - } - - if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SSM_CONV, GGML_OP_UNARY }, { GGML_UNARY_OP_SILU })) { - ggml_cuda_op_ssm_conv(*cuda_ctx, node, cgraph->nodes[i+1]); - i++; - continue; - } - - if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SILU }) || - ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SIGMOID }) || - ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SOFTPLUS })) { - ggml_cuda_op_unary_mul(*cuda_ctx, node, cgraph->nodes[i+1]); - i++; - continue; - } + int nodes_to_skip = ggml_cuda_try_fuse(cuda_ctx, cgraph, i); - if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_SCALE }, { GGML_UNARY_OP_TANH })) { - i += 2; - ggml_cuda_op_softcap(*cuda_ctx, cgraph->nodes[i], node); - continue; - } + if (nodes_to_skip != 0) { +#ifdef GGML_CUDA_DEBUG + const int last_fused = i + nodes_to_skip; + GGML_LOG_INFO("nodes_fused: %d, first: %s (%s), last: %s (%s)\n", + nodes_to_skip + 1, ggml_op_name(node->op), node->name, + ggml_op_name(cgraph->nodes[last_fused]->op), cgraph->nodes[last_fused]->name); +#endif + i += nodes_to_skip; + continue; } #ifndef NDEBUG - assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device)); + // On integrated GPUs (APUs, e.g. RDNA3.5) the scheduler may place a + // node's output on the host-visible buffer, which the compute path + // handles. Allow that here, mirroring the src-tensor check below. + assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || + (integrated && ggml_backend_buft_is_cuda_host(node->buffer->buft))); for (int j = 0; j < GGML_MAX_SRC; j++) { if (node->src[j] != nullptr) { assert(node->src[j]->buffer); assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || - ggml_backend_buft_is_cuda_split(node->src[j]->buffer->buft) || (integrated && ggml_backend_buft_is_cuda_host(node->src[j]->buffer->buft))); + (integrated && ggml_backend_buft_is_cuda_host(node->src[j]->buffer->buft))); } } #else @@ -4102,14 +4846,14 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud } #ifdef USE_CUDA_GRAPH - ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); if (use_cuda_graph && cuda_graph_update_required) { // End CUDA graph capture - if (graph->graph != nullptr) { - CUDA_CHECK(cudaGraphDestroy(graph->graph)); - graph->graph = nullptr; - } + ggml_cuda_graph_launch_chunk(cuda_ctx, graph, n_chunks++); - CUDA_CHECK(cudaStreamEndCapture(cuda_ctx->stream(), &graph->graph)); + // drop the chunks left over from a longer graph + for (size_t i = n_chunks; i < graph->instances.size(); ++i) { + CUDA_CHECK(cudaGraphExecDestroy(graph->instances[i])); + } + graph->instances.resize(n_chunks); graph_evaluated_or_captured = true; // CUDA graph has been captured std::lock_guard lock(ggml_cuda_lock); @@ -4121,16 +4865,11 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud } } - if (use_cuda_graph) { - ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); - if (graph->instance == nullptr) { // Create executable graph from captured graph. - CUDA_CHECK(cudaGraphInstantiate(&graph->instance, graph->graph, NULL, NULL, 0)); + if (use_cuda_graph && !cuda_graph_update_required) { + // the chunks were launched during capture, otherwise launch them here + for (cudaGraphExec_t instance : graph->instances) { + CUDA_CHECK(cudaGraphLaunch(instance, cuda_ctx->stream())); } - if (cuda_graph_update_required) { // Update graph executable - ggml_cuda_graph_update_executable(cuda_ctx, graph_key); - } - // Launch graph - CUDA_CHECK(cudaGraphLaunch(graph->instance, cuda_ctx->stream())); #else GGML_UNUSED(graph_key); graph_evaluated_or_captured = true; @@ -4142,9 +4881,9 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) { ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); - if (graph->graph == nullptr) { - if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_AMPERE) { - if (!graph->disable_due_to_gpu_arch) { + if (graph->instances.empty()) { + if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_VOLTA) { + if (!graph->disable_due_to_gpu_arch && ggml_cuda_graph_debug_enabled()) { GGML_LOG_DEBUG("%s: disabling CUDA graphs due to GPU architecture\n", __func__); } graph->disable_due_to_gpu_arch = true; @@ -4179,7 +4918,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, // Warmup: need at least 2 calls with no property change on the 2nd call if (!properties_changed) { graph->warmup_complete = true; - GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__); + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__); + } use_cuda_graph = true; cuda_graph_update_required = true; } @@ -4189,10 +4930,12 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, if (properties_changed) { // Properties changed - reset warmup, execute directly until stable again graph->warmup_complete = false; - GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__); + if (ggml_cuda_graph_debug_enabled()) { + GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__); + } } else { use_cuda_graph = true; - cuda_graph_update_required = graph->instance == nullptr; + cuda_graph_update_required = graph->instances.empty(); } } } @@ -4239,9 +4982,132 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev } } -static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) { +static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context; + const bool disable_fusion = ggml_cuda_fusion_disabled(); + + auto add_alloc_deps = [&](size_t start, size_t last_node) { + + for (size_t i = start; i < last_node; ++i) { + params->add_alloc_dep(params->user_data, cgraph->nodes[i], cgraph->nodes[last_node]); + + for (int j = 0; j < GGML_MAX_SRC; ++j) { + if (cgraph->nodes[i]->src[j]) { + params->add_alloc_dep(params->user_data, cgraph->nodes[i]->src[j], cgraph->nodes[last_node]); + } + } + } + }; + + if (!disable_fusion) { + // add alloc deps for performance positive fusions. This may increase the overall compute buffer size. + // TODO: consolidate fusion paths in graph_optimize and graph_compute + ggml_cuda_set_device(cuda_ctx->device); + for (int i = 0; i + 5 < cgraph->n_nodes; ++i) { + if (cgraph->nodes[i]->op != GGML_OP_MUL_MAT_ID) { + continue; + } + for (int j = i + 3; j + 2 < cgraph->n_nodes; ++j) { + if (cgraph->nodes[j]->op == GGML_OP_MUL_MAT_ID && cgraph->nodes[j + 1]->op == GGML_OP_MUL_MAT_ID) { + break; + } + if (cgraph->nodes[j]->op != GGML_OP_MUL_MAT || !ggml_cuda_match_shared_expert(cgraph, i, j)) { + continue; + } + // Group both outputs before allocation so the shared result cannot alias intervening nodes. + std::rotate(cgraph->nodes + i + 3, cgraph->nodes + j, cgraph->nodes + j + 3); + ggml_tensor * up = cgraph->nodes[i + 2]->src[1]; + params->add_alloc_dep(params->user_data, up->src[1], cgraph->nodes[i + 5]); + params->add_alloc_dep(params->user_data, up->src[2], cgraph->nodes[i + 5]); + i += 5; + break; + } + } + for (int i = 0; i < cgraph->n_nodes; ++i) { + // Keep the inputs of the fused gate/up/GLU MMQ kernel alive until the GLU so that they cannot alias its output (under op offload the weights are buffer copies). + if (i + 2 < cgraph->n_nodes && cgraph->nodes[i + 2]->op == GGML_OP_GLU) { + ggml_tensor * glu = cgraph->nodes[i + 2]; + ggml_tensor * gate = glu->src[0]; + ggml_tensor * up = glu->src[1]; + const bool ok = (gate == cgraph->nodes[i] && up == cgraph->nodes[i + 1]) || (gate == cgraph->nodes[i + 1] && up == cgraph->nodes[i]); + if (ok && ggml_cuda_mul_mat_q_fusion_matches(up, gate, glu, cuda_ctx->device)) { + add_alloc_deps(i, i + 2); + i += 2; + continue; + } + } + + ggml_cuda_moe_weighted_reduction_match match; + if (ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) { + params->add_alloc_dep(params->user_data, const_cast(match.experts), match.dst); + params->add_alloc_dep(params->user_data, const_cast(match.weights), match.dst); + if (match.expert_scale != nullptr) { + params->add_alloc_dep( + params->user_data, const_cast(match.expert_scale), match.dst); + } + i += match.node_count - 1; + } + + if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || + cgraph->nodes[i]->op == GGML_OP_ARGSORT) { + ggml_cuda_topk_moe_args args; + const bool can_fuse = ggml_cuda_topk_moe_fusion(cgraph, i, args); + std::vector ops; + ops.reserve(13); // max ops; avoids gcc -Wstringop-overflow false positive + + const ggml_tensor * node = cgraph->nodes[i]; + + if (can_fuse) { + const ggml_tensor * logits = node->src[0]; + ggml_tensor * weights = nullptr; + ggml_tensor * ids = nullptr; + + if (!args.delayed_softmax) { + int out_nodes[2]; // nodes which can't be elided + + if (args.sigmoid) { + ops.insert(ops.end(), { GGML_OP_UNARY }); + } else if (args.sqrt_softplus) { + ops.insert(ops.end(), { GGML_OP_UNARY, GGML_OP_SQRT }); + } else { + ops.insert(ops.end(), { GGML_OP_SOFT_MAX }); + } + const int i_probs = i + (int) ops.size() - 1; // last node of the gating activation + + if (args.prob_bias) { + ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, + GGML_OP_GET_ROWS }); + out_nodes[0] = i_probs + 4; + } else { + ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }); + out_nodes[0] = i_probs + 3; + } + ids = cgraph->nodes[out_nodes[0]]; + + if (args.norm) { + ops.insert(ops.end(), + { GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE }); + } + if (args.scale) { + ops.insert(ops.end(), { GGML_OP_SCALE }); + } + + weights = cgraph->nodes[i + ops.size() - 1]; + out_nodes[1] = i + ops.size() - 1; + + if (ggml_can_fuse_subgraph(cgraph, i, ops.size(), ops.data(), out_nodes, 2) && + ggml_cuda_should_use_topk_moe(node, logits, weights, ids)) { + + add_alloc_deps(i, i + ops.size()); + i += ops.size() - 1; + } + } + } + } + } + } + #ifdef USE_CUDA_GRAPH const void * graph_key = ggml_cuda_graph_get_key(cgraph); const bool use_cuda_graph = ggml_cuda_graph_set_enabled(cuda_ctx, graph_key); @@ -4263,10 +5129,12 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph ggml_cuda_stream_context & stream_context = cuda_ctx->stream_context(); stream_context.reset(); - if (!use_cuda_graph || ggml_backend_cuda_get_device_count() != 1) { + if (!use_cuda_graph) { return; } + ggml_cuda_set_device(cuda_ctx->device); + // number of out-degrees for a particular node std::unordered_map fan_out; // reverse mapping of node to index in the cgraph @@ -4488,8 +5356,8 @@ static const ggml_backend_i ggml_backend_cuda_interface = { /* .free = */ ggml_backend_cuda_free, /* .set_tensor_async = */ ggml_backend_cuda_set_tensor_async, /* .get_tensor_async = */ ggml_backend_cuda_get_tensor_async, - /* .get_tensor_2d_async = */ ggml_backend_cuda_set_tensor_2d_async, - /* .set_tensor_2d_async = */ ggml_backend_cuda_get_tensor_2d_async, + /* .set_tensor_2d_async = */ ggml_backend_cuda_set_tensor_2d_async, + /* .get_tensor_2d_async = */ ggml_backend_cuda_get_tensor_2d_async, /* .cpy_tensor_async = */ ggml_backend_cuda_cpy_tensor_async, /* .synchronize = */ ggml_backend_cuda_synchronize, /* .graph_plan_create = */ NULL, @@ -4515,20 +5383,42 @@ int ggml_backend_cuda_get_device_count() { return ggml_cuda_info().device_count; } -void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) { +static std::string ggml_cuda_device_description(int device) { cudaDeviceProp prop; - CUDA_CHECK(cudaGetDeviceProperties(&prop, device)); - snprintf(description, description_size, "%s", prop.name); + CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(device))); + + const ggml_cuda_device_info & info = ggml_cuda_info(); + std::string description = prop.name; + if (info.device_count > info.physical_device_count) { + description += " (dev p" + std::to_string(info.devices[device].physical_device) + + "/v" + std::to_string(info.devices[device].virtual_index) + ")"; + } + return description; +} + +void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) { + snprintf(description, description_size, "%s", ggml_cuda_device_description(device).c_str()); +} + +static int ggml_cuda_physical_device_share_count(int device) { + const ggml_cuda_device_info & info = ggml_cuda_info(); + GGML_ASSERT(device >= 0 && device < info.device_count); + return info.devices[device].physical_share_count; } void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) { ggml_cuda_set_device(device); CUDA_CHECK(cudaMemGetInfo(free, total)); + + // virtual devices sharing one physical GPU share its memory pool; split it between them + const int share_count = ggml_cuda_physical_device_share_count(device); + *free /= share_count; + *total /= share_count; } bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) { - if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) { + if (!ggml_cuda_register_host_enabled()) { return false; } @@ -4551,7 +5441,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) { } void ggml_backend_cuda_unregister_host_buffer(void * buffer) { - if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) { + if (!ggml_cuda_register_host_enabled()) { return; } @@ -4662,17 +5552,23 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; ggml_cuda_set_device(ctx->device); - CUDA_CHECK(cudaMemGetInfo(free, total)); + cudaError_t err = cudaMemGetInfo(free, total); + if (err != cudaSuccess) { + (void)cudaGetLastError(); + GGML_LOG_WARN("%s: cudaMemGetInfo failed (%s), returning 0/0\n", __func__, cudaGetErrorString(err)); + *free = 0; + *total = 0; + return; + } // ref: https://github.com/ggml-org/llama.cpp/pull/17368 -#if defined(__linux__) +#if defined(__linux__) && !defined(GGML_USE_HIP) // Check if this is a UMA (Unified Memory Architecture) system cudaDeviceProp prop; - CUDA_CHECK(cudaGetDeviceProperties(&prop, ctx->device)); + CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); // Check if UMA is explicitly enabled via environment variable - bool uma_env = getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr; - bool is_uma = prop.integrated > 0 || uma_env; + bool is_uma = prop.integrated > 0 || ggml_cuda_unified_memory_enabled(); if (is_uma) { // For UMA systems (like DGX Spark), use system memory info @@ -4685,13 +5581,23 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * GGML_LOG_ERROR("%s: /proc/meminfo reading failed, using cudaMemGetInfo\n", __func__); } } -#endif // defined(__linux__) +#endif // defined(__linux__) && !defined(GGML_USE_HIP) + // virtual devices sharing one physical GPU share its memory pool; split it between them + const int share_count = ggml_cuda_physical_device_share_count(ctx->device); + *free /= share_count; + *total /= share_count; } static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend_dev_t dev) { - GGML_UNUSED(dev); - return GGML_BACKEND_DEVICE_TYPE_GPU; + ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *) dev->context; + + cudaDeviceProp prop; + CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); + + return prop.integrated + ? GGML_BACKEND_DEVICE_TYPE_IGPU + : GGML_BACKEND_DEVICE_TYPE_GPU; } static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { @@ -4703,7 +5609,7 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back props->device_id = ctx->pci_bus_id.empty() ? nullptr : ctx->pci_bus_id.c_str(); ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total); - bool host_buffer = getenv("GGML_CUDA_NO_PINNED") == nullptr; + const bool host_buffer = !ggml_cuda_pinned_memory_disabled(); #ifdef GGML_CUDA_NO_PEER_COPY bool events = false; #else @@ -4715,6 +5621,7 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back /* .host_buffer = */ host_buffer, /* .buffer_from_host_ptr = */ false, /* .events = */ events, + /* .mmap_support = */ props->type != GGML_BACKEND_DEVICE_TYPE_IGPU, }; } @@ -4738,15 +5645,6 @@ static ggml_backend_buffer_type_t ggml_backend_cuda_device_get_host_buffer_type( static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context; - // split buffers can only be used with GGML_OP_MUL_MAT - if (op->op != GGML_OP_MUL_MAT) { - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (op->src[i] && op->src[i]->buffer && ggml_backend_buft_is_cuda_split(op->src[i]->buffer->buft)) { - return false; - } - } - } - // check if all the sources are allocated on this device for (int i = 0; i < GGML_MAX_SRC; i++) { if (op->src[i] && op->src[i]->buffer && ggml_backend_buft_is_cuda(op->src[i]->buffer->buft)) { @@ -4782,13 +5680,13 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_UNARY_OP_CEIL: case GGML_UNARY_OP_ROUND: case GGML_UNARY_OP_TRUNC: - // TODO: should become: - //return ggml_is_contiguous_rows(op->src[0]); - return ggml_is_contiguous(op->src[0]); + if (op->src[0]->type == GGML_TYPE_BF16 && ggml_get_unary_op(op) == GGML_UNARY_OP_XIELU) { + return false; + } + return op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16; default: return false; } - break; case GGML_OP_GLU: switch (ggml_get_glu_op(op)) { case GGML_GLU_OP_REGLU: @@ -4797,31 +5695,30 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_GLU_OP_SWIGLU_OAI: case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: + case GGML_GLU_OP_SWIGLU_CLAMP: + if (op->src[0]->type == GGML_TYPE_BF16 && + (ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_CLAMP)) { + return false; + } + if (op->src[0]->type == GGML_TYPE_F16 && ggml_get_glu_op(op) == GGML_GLU_OP_SWIGLU_OAI) { + return false; + } return ggml_is_contiguous_1(op->src[0]); default: return false; } - break; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: { struct ggml_tensor * a = op->src[0]; struct ggml_tensor * b = op->src[1]; - if (a->buffer && ggml_backend_buft_is_cuda_split(a->buffer->buft)) { - if (a->ne[2] > 1 || a->ne[3] > 1) { - return false; - } - // for small weight matrices the active device can end up without any rows, don't use row split in those cases - // this avoids some edge cases (and the performance would not be good anyways) - ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) a->buffer->buft->context; - int64_t row_low; - int64_t row_high; - get_row_split(&row_low, &row_high, a, buft_ctx->tensor_split, dev_ctx->device); - if (row_low == row_high) { - return false; - } + if (a->nb[0] != ggml_element_size(a) || b->nb[0] != ggml_element_size(b)) { + return false; // TODO this could in principle be implemented though currently there is no use case. } - if (b->type == GGML_TYPE_F16 && a->type != GGML_TYPE_F16) { + if (b->type == GGML_TYPE_F16 && a->type != GGML_TYPE_F16 && !ggml_cuda_op_mul_mat_use_fwht(op)) { + return false; + } + if (op->op == GGML_OP_MUL_MAT_ID && ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) { return false; } #ifdef GGML_USE_MUSA @@ -4841,6 +5738,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_F32: case GGML_TYPE_F16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4868,7 +5766,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g default: return false; } - } break; + } case GGML_OP_OUT_PROD: return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32; case GGML_OP_GET_ROWS: @@ -4879,35 +5777,58 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_BF16: case GGML_TYPE_I32: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_XS: return true; + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + // 32-value sub-blocks, the row size does not guarantee + // the QK_K super-blocks the get_rows kernel iterates on + return op->src[0]->ne[0] % QK_K == 0; default: return false; } - } break; + } case GGML_OP_GET_ROWS_BACK: - { - return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->ne[2] == 1 && op->ne[3] == 1; - } break; + return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->ne[2] == 1 && op->ne[3] == 1; case GGML_OP_SET_ROWS: { - return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || - op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || - op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) && - op->src[0]->type == GGML_TYPE_F32 && + return ( + ( + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || + op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || + op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) && + op->src[0]->type == GGML_TYPE_F32 + ) || ( + op->type == GGML_TYPE_F16 && op->src[0]->type == GGML_TYPE_F16 + ) + ) && (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); - } break; + } case GGML_OP_SET: { const ggml_type t = op->type; return (t == GGML_TYPE_F32 || t == GGML_TYPE_I32) && t == op->src[0]->type && t == op->src[1]->type; - } break; + } case GGML_OP_CPY: { ggml_type src0_type = op->src[0]->type; @@ -4963,29 +5884,55 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return true; } return false; - } break; + } case GGML_OP_DUP: - { - ggml_type src0_type = op->src[0]->type; - return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; - } break; + return true; case GGML_OP_ARGMAX: case GGML_OP_COUNT_EQUAL: - { - return true; - } break; + return true; case GGML_OP_REPEAT: { + // the CUDA REPEAT path only implements F32/F16; other types assert at runtime ggml_type src0_type = op->src[0]->type; - return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; - } break; + return src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_F16; + } case GGML_OP_REPEAT_BACK: return op->type == GGML_TYPE_F32 && (op->src[0]->ne[2]*op->src[0]->ne[3]) <= (1 << 15); case GGML_OP_CONCAT: { ggml_type src0_type = op->src[0]->type; - return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16; - } break; + ggml_type src1_type = op->src[1]->type; + const int32_t dim = op->op_params[0]; + return src0_type == src1_type && + src0_type == op->type && + ( + ( + ggml_is_quantized(src0_type) && + ( + ( + dim == 3 && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op->src[1]) + ) || ( + dim != 3 && + ggml_is_contiguous_to_3(op->src[0]) && + ggml_is_contiguous_to_3(op->src[1]) + ) + ) && + op->src[0]->ne[0] % ggml_blck_size(src0_type) == 0 && + op->src[1]->ne[0] % ggml_blck_size(src0_type) == 0 + ) || ( + !ggml_is_quantized(src0_type) && + ggml_blck_size(src0_type) == 1 && + ( + ggml_type_size(src0_type) == 1 || + ggml_type_size(src0_type) == 2 || + ggml_type_size(src0_type) == 4 || + ggml_type_size(src0_type) == 8 + ) + ) + ); + } case GGML_OP_CONV_TRANSPOSE_1D: { ggml_type src0_type = op->src[0]->type; @@ -4994,42 +5941,64 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return true; } return false; - } break; + } + case GGML_OP_COL2IM_1D: + { + ggml_type src0_type = op->src[0]->type; + return (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_F16 || src0_type == GGML_TYPE_BF16) && + op->type == src0_type && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op); + } case GGML_OP_SILU_BACK: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; - break; case GGML_OP_NORM: case GGML_OP_RMS_NORM: case GGML_OP_L2_NORM: - return true; + return ggml_is_contiguous_rows(op->src[0]); case GGML_OP_RMS_NORM_BACK: return ggml_is_contiguous(op->src[0]); - break; case GGML_OP_NONE: case GGML_OP_RESHAPE: case GGML_OP_VIEW: case GGML_OP_PERMUTE: case GGML_OP_TRANSPOSE: - case GGML_OP_ADD: case GGML_OP_ADD_ID: case GGML_OP_ADD1: - case GGML_OP_SUB: - case GGML_OP_MUL: - case GGML_OP_DIV: - case GGML_OP_SCALE: case GGML_OP_SQR: case GGML_OP_SQRT: case GGML_OP_SIN: case GGML_OP_COS: - case GGML_OP_CLAMP: case GGML_OP_LOG: return true; + case GGML_OP_SCALE: + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type; + case GGML_OP_CLAMP: + return ggml_is_contiguous_rows(op->src[0]); + case GGML_OP_ADD: + case GGML_OP_SUB: + case GGML_OP_MUL: + case GGML_OP_DIV: + if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_BF16) { + return op->src[0]->type == GGML_TYPE_BF16 && op->type == GGML_TYPE_BF16 && + (op->src[1]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_F32); + } + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); case GGML_OP_SSM_SCAN: { + const int32_t K = ggml_get_op_params_i32(op, 0); + if (op->src[3]->ne[0] == 1) { // Mamba2 - // (kernel only supports (d_state == 128 || d_state == 256) && d_head % 16 == 0) - return (op->src[0]->ne[0] == 128 || op->src[0]->ne[0] == 256) && op->src[0]->ne[1] % 16 == 0; + // (kernel only supports (d_state == 96 || d_state == 128 || d_state == 256) && d_head % 16 == 0) + const int64_t d_state = op->src[0]->ne[0]; + return (d_state == 96 || d_state == 128 || d_state == 256) && op->src[0]->ne[1] % 16 == 0; } else { + if (K > 1) { + return false; + } + // Mamba // (kernel only supports d_state == 16, d_head == 1, n_head % 128 == 0, n_group == 1) return op->src[0]->ne[0] == 16 && op->src[0]->ne[1] == 1 && op->src[0]->ne[2] % 128 == 0 && op->src[4]->ne[1] == 1; @@ -5062,8 +6031,16 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_IM2COL: case GGML_OP_IM2COL_3D: case GGML_OP_CONV_2D: + return (ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])); + case GGML_OP_CONV_3D: + return (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) && + op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && ggml_is_contiguous(op); case GGML_OP_CONV_2D_DW: + return (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) && + op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; case GGML_OP_CONV_TRANSPOSE_2D: + case GGML_OP_POOL_1D: case GGML_OP_POOL_2D: return true; case GGML_OP_ACC: @@ -5071,16 +6048,26 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g //return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]); return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); case GGML_OP_SUM: - return ggml_is_contiguous_rows(op->src[0]); + return ggml_is_contiguously_allocated(op->src[0]); case GGML_OP_TOP_K: + return op->src[0]->ne[0] <= INT_MAX; case GGML_OP_ARGSORT: #ifndef GGML_CUDA_USE_CUB - return op->src[0]->ne[0] <= 1024; + { + // bitonic path: the padded row must fit in shared memory + int64_t ncols_pad = 1; + while (ncols_pad < op->src[0]->ne[0]) { + ncols_pad *= 2; + } + return ncols_pad * sizeof(int) <= ggml_cuda_info().devices[dev_ctx->device].smpb; + } #else - return true; + return op->src[0]->ne[0] <= INT_MAX; #endif case GGML_OP_SUM_ROWS: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_MEAN: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]); case GGML_OP_GROUP_NORM: return ggml_is_contiguous(op->src[0]); case GGML_OP_PAD: @@ -5095,12 +6082,17 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_RWKV_WKV7: return true; case GGML_OP_GATED_DELTA_NET: - //TODO: enable once MUSA compiler is solved https://github.com/ggml-org/llama.cpp/pull/19504#issuecomment-4018634327 -#ifdef GGML_USE_MUSA - return false; -#else return true; -#endif // GGML_USE_MUSA + case GGML_OP_DSV4_HC_COMB: + return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_DSV4_HC_PRE: + return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32; + case GGML_OP_DSV4_HC_POST: + return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && (op->src[3] == nullptr || op->src[3]->type == GGML_TYPE_F32) && + op->type == GGML_TYPE_F32; case GGML_OP_FLASH_ATTN_EXT: return ggml_cuda_flash_attn_ext_supported(dev_ctx->device, op); case GGML_OP_CROSS_ENTROPY_LOSS: @@ -5113,6 +6105,10 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_DIAG: case GGML_OP_SOLVE_TRI: return true; + case GGML_OP_LIGHTNING_INDEXER: + return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op); + case GGML_OP_TOP_K_MASK: + return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32; default: return false; @@ -5122,7 +6118,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g static bool ggml_backend_cuda_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context; const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated; - return (((ggml_backend_buft_is_cuda(buft) || ggml_backend_buft_is_cuda_split(buft)) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft))); + return (ggml_backend_buft_is_cuda(buft) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft)); } static int64_t get_op_batch_size(const ggml_tensor * op) { @@ -5148,6 +6144,7 @@ static bool ggml_backend_cuda_device_offload_op(ggml_backend_dev_t dev, const gg static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_t dev) { #ifdef GGML_CUDA_NO_PEER_COPY + GGML_UNUSED(dev); return nullptr; #else ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *)dev->context; @@ -5246,12 +6243,8 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t features.push_back({ "USE_GRAPHS", "1" }); #endif - #ifdef GGML_CUDA_PEER_MAX_BATCH_SIZE - features.push_back({ "PEER_MAX_BATCH_SIZE", STRINGIFY(GGML_CUDA_PEER_MAX_BATCH_SIZE) }); - #endif - - #ifdef GGML_CUDA_FA_ALL_QUANTS - features.push_back({ "FA_ALL_QUANTS", "1" }); + #ifdef GGML_CUDA_FA_QUANTS + features.push_back({ "FA_QUANTS", GGML_CUDA_FA_QUANTS }); #endif { @@ -5288,9 +6281,6 @@ static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, con if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { return (void *)ggml_backend_cuda_comm_allreduce_tensor; } - if (strcmp(name, "ggml_backend_split_buffer_type") == 0) { - return (void *)ggml_backend_cuda_split_buffer_type; - } if (strcmp(name, "ggml_backend_register_host_buffer") == 0) { return (void *)ggml_backend_cuda_register_host_buffer; } @@ -5320,20 +6310,29 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { std::lock_guard lock(mutex); if (!initialized) { ggml_backend_cuda_reg_context * ctx = new ggml_backend_cuda_reg_context; - const int min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32; + const int min_batch_size = ggml_cuda_op_offload_min_batch(); + + const ggml_cuda_device_info & info = ggml_cuda_info(); + const bool virtual_devices = info.device_count > info.physical_device_count; + + for (int i = 0; i < info.device_count; i++) { + const int physical_id = info.devices[i].physical_device; - for (int i = 0; i < ggml_cuda_info().device_count; i++) { ggml_backend_cuda_device_context * dev_ctx = new ggml_backend_cuda_device_context; dev_ctx->device = i; dev_ctx->name = GGML_CUDA_NAME + std::to_string(i); + dev_ctx->description = ggml_cuda_device_description(i); - cudaDeviceProp prop; - CUDA_CHECK(cudaGetDeviceProperties(&prop, i)); - dev_ctx->description = prop.name; - - char pci_bus_id[16] = {}; - snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.0", prop.pciDomainID, prop.pciBusID, prop.pciDeviceID); + char pci_bus_id[32] = {}; + CUDA_CHECK(cudaDeviceGetPCIBusId(pci_bus_id, sizeof(pci_bus_id), physical_id)); dev_ctx->pci_bus_id = pci_bus_id; + if (virtual_devices) { + // make the pci bus id unique for virtual devices + dev_ctx->pci_bus_id += "-v" + std::to_string(i); + } + for (char & c : dev_ctx->pci_bus_id) { + c = std::tolower(c); + } dev_ctx->op_offload_min_batch_size = min_batch_size; ggml_backend_dev_t dev = new ggml_backend_device { diff --git a/ggml/src/ggml-cuda/im2col.cu b/ggml/src/ggml-cuda/im2col.cu index 56dc054..d377f28 100644 --- a/ggml/src/ggml-cuda/im2col.cu +++ b/ggml/src/ggml-cuda/im2col.cu @@ -1,44 +1,47 @@ #include "im2col.cuh" +#define MAX_GRIDDIM_Y 65535 #define MAX_GRIDDIM_Z 65535 template static __global__ void im2col_kernel( const float * x, T * dst, int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, - int64_t IC_IH_IW, int64_t IH_IW, int64_t N_OH, int64_t KH_KW, int64_t IC_KH_KW, + int64_t N, int64_t IC_IH_IW, int64_t IH_IW, int64_t N_OH, int64_t KH_KW, int64_t IC_KH_KW, int s0, int s1, int p0, int p1, int d0, int d1) { - const int64_t i = threadIdx.x + blockIdx.x * blockDim.x; - if (i >= IC_KH_KW) { - return; - } - - const int64_t iic = i / (KH_KW); - const int64_t rem = i - iic * KH_KW; - const int64_t ikh = rem / KW; - const int64_t ikw = rem - ikh * KW; - - const int64_t iow = blockIdx.y; - for (int64_t iz = blockIdx.z; iz < N_OH; iz+=MAX_GRIDDIM_Z) { - const int64_t in = iz / OH; - const int64_t ioh = iz - in * OH; - - const int64_t iiw = iow * s0 + ikw * d0 - p0; - const int64_t iih = ioh * s1 + ikh * d1 - p1; - - const int64_t offset_dst = - ((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw; - - if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { - dst[offset_dst] = 0.0f; - } else { - const int64_t offset_src = iic * IC_IH_IW + in * IH_IW; - dst[offset_dst] = x[offset_src + iih * IW + iiw]; + const int tid = threadIdx.x; + + const int64_t total_channels = IC * KH * KW; + const int threads_per_pos = blockDim.x; + const int64_t start_ch = tid; + const int64_t stride_ch = threads_per_pos; + + for (int64_t iow = blockIdx.x; iow < OW; iow += MAX_GRIDDIM_Y) { + for (int64_t iz = blockIdx.y; iz < N_OH; iz += MAX_GRIDDIM_Z) { + const int64_t in = iz / OH; + const int64_t ioh = iz - in * OH; + + for (int64_t iic_khw = start_ch; iic_khw < total_channels; iic_khw += stride_ch) { + const int64_t iic = iic_khw / KH_KW; + const int64_t rem = iic_khw - iic * KH_KW; + const int64_t ikh = rem / KW; + const int64_t ikw = rem - ikh * KW; + + const int64_t iiw = iow * s0 + ikw * d0 - p0; + const int64_t iih = ioh * s1 + ikh * d1 - p1; + + const int64_t offset_dst = + ((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw; + + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { + dst[offset_dst] = 0.0f; + } else { + const int64_t offset_src = iic * IC_IH_IW + in * IH_IW; + dst[offset_dst] = x[offset_src + iih * IW + iiw]; + } + } } } - - GGML_UNUSED(IC); - GGML_UNUSED(KH); } // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW] @@ -48,13 +51,15 @@ static void im2col_cuda(const float * x, T* dst, int64_t N, int64_t IC_IH_IW, int64_t IH_IW, int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) { const int64_t IC_KH_KW = IC * KH * KW; - const int64_t num_blocks = (IC_KH_KW + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE; const int64_t N_OH = N * OH; const int64_t KH_KW = KW*KH; - dim3 block_nums(num_blocks, OW, MIN(N_OH, MAX_GRIDDIM_Z)); - im2col_kernel<<>>(x, dst, IC, IW, IH, OH, OW, KW, KH, - IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW, - s0, s1, p0, p1, d0, d1); + const int threads_per_block = MIN((int)IC_KH_KW, CUDA_IM2COL_BLOCK_SIZE); + dim3 block_nums(MIN(OW, MAX_GRIDDIM_Y), MIN(N_OH, MAX_GRIDDIM_Z)); + + im2col_kernel<<>>( + x, dst, IC, IW, IH, OH, OW, KW, KH, + N, IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW, + s0, s1, p0, p1, d0, d1); } static void im2col_cuda_f16(const float * x, half * dst, @@ -136,23 +141,24 @@ static __global__ void im2col_3d_kernel( const int64_t ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / KW; const int64_t ikw = i % KW; - const int64_t iow = blockIdx.y; - for (int64_t iz = blockIdx.z; iz < N_OD_OH; iz+=MAX_GRIDDIM_Z) { - const int64_t in = iz / OD_OH; - const int64_t iod = (iz - in*OD_OH) / OH; - const int64_t ioh = iz % OH; + for (int64_t iow = blockIdx.y; iow < OW; iow += MAX_GRIDDIM_Y) { + for (int64_t iz = blockIdx.z; iz < N_OD_OH; iz += MAX_GRIDDIM_Z) { + const int64_t in = iz / OD_OH; + const int64_t iod = (iz - in*OD_OH) / OH; + const int64_t ioh = iz % OH; - const int64_t iiw = iow * s0 + ikw * d0 - p0; - const int64_t iih = ioh * s1 + ikh * d1 - p1; - const int64_t iid = iod * s2 + ikd * d2 - p2; + const int64_t iiw = iow * s0 + ikw * d0 - p0; + const int64_t iih = ioh * s1 + ikh * d1 - p1; + const int64_t iid = iod * s2 + ikd * d2 - p2; - const int64_t offset_dst = in*OD_OH_OW_IC_KD_KH_KW + iod*OH_OW_IC_KD_KH_KW + ioh*OW_IC_KD_KH_KW + iow*IC_KD_KH_KW + iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw; + const int64_t offset_dst = in*OD_OH_OW_IC_KD_KH_KW + iod*OH_OW_IC_KD_KH_KW + ioh*OW_IC_KD_KH_KW + iow*IC_KD_KH_KW + iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw; - if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) { - dst[offset_dst] = 0.0f; - } else { - const int64_t offset_src = ((in * IC + iic) * stride_q) + (iid * stride_z) + (iih * stride_y) + (iiw * stride_x); - dst[offset_dst] = src[offset_src]; + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) { + dst[offset_dst] = 0.0f; + } else { + const int64_t offset_src = ((in * IC + iic) * stride_q) + (iid * stride_z) + (iih * stride_y) + (iiw * stride_x); + dst[offset_dst] = src[offset_src]; + } } } } @@ -178,7 +184,7 @@ static void im2col_3d_cuda(const float * src, T* dst, const int64_t OH_OW_IC_KD_KH_KW = OH*OW*IC*KD*KH*KW; const int64_t OW_IC_KD_KH_KW = OW*IC*KD*KH*KW; const int64_t num_blocks = (IC_KD_KH_KW + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE; - dim3 block_nums(num_blocks, OW, MIN(N_OD_OH, MAX_GRIDDIM_Z)); + dim3 block_nums(num_blocks, MIN(OW, MAX_GRIDDIM_Y), MIN(N_OD_OH, MAX_GRIDDIM_Z)); im2col_3d_kernel<<>>(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW, OH_OW, KD_KH_KW, ID_IH_IW, KH_KW, IH_IW, IC_ID_IH_IW, IC_KD_KH_KW, OW_KD_KH_KW, OD_OH_OW_IC_KD_KH_KW, diff --git a/ggml/src/ggml-cuda/lightning-indexer.cu b/ggml/src/ggml-cuda/lightning-indexer.cu new file mode 100644 index 0000000..429d2e2 --- /dev/null +++ b/ggml/src/ggml-cuda/lightning-indexer.cu @@ -0,0 +1,795 @@ +#include "common.cuh" +#include "lightning-indexer.cuh" +#include "fattn-common.cuh" +#include "convert.cuh" + +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +#if defined(TURING_MMA_AVAILABLE) + +typedef union { + int2 i2; + half2 h2[2]; +} half4; + +// TODO add support for AMD cards via rocWMMA +#include +namespace wmma = nvcuda::wmma; + +template +static __global__ void lightning_indexer_kernel_wmma( + const float * Q, const char * K, const float * W, const half * M, float * dst, + int64_t n_stream, int64_t n_batch, int64_t n_kv, + size_t nb1, size_t nb2, size_t nb3, + size_t nbq1, size_t nbq2, size_t nbq3, + size_t nbk1, size_t nbk2, size_t nbk3, + size_t nbw1, size_t nbw2, size_t nbw3, + size_t nbm1, size_t nbm2, size_t nbm3, + int64_t nem3 + ) { + + constexpr int THREADS_PER_BLOCK = WARPS_PER_BLOCK * WARP_SIZE; + constexpr int HEADS_PER_INNER_LOOP = 8; + constexpr int K_EMBD_PER_INNER_LOOP = 16; + constexpr int N_EMBD_PADDED = N_EMBD + 8; + + const int i_batch = blockIdx.y; + const int i_stream = blockIdx.z; + const int i_warp = threadIdx.y; + const int i_lane = threadIdx.x; + const int tid = i_warp * WARP_SIZE + i_lane; + + // each block processes K_VECS_PER_BLOCK K vectors + const int start_kv = blockIdx.x * K_VECS_PER_BLOCK; + + const char * q_base = (const char *) Q + i_batch*nbq2 + i_stream*nbq3; + const float * w_base = (const float *) ((const char *) W + i_batch*nbw1 + i_stream*nbw3); + + // phase 1 - load weights and first Q tile to shared memory + + __shared__ float w_shared[N_HEAD]; + __shared__ int2 q_shared_h[HEADS_PER_INNER_LOOP][N_EMBD_PADDED / 4]; + + if (tid < N_HEAD) { + w_shared[tid] = w_base[tid]; + } + + // total number of half4 elements in HEADS_PER_INNER_LOOP x N_EMBD Q tile + constexpr int N_Q_TILE = HEADS_PER_INNER_LOOP * (N_EMBD / 4); + // number of registers needed in each thread to store Q tile in thread block + constexpr int N_Q_NEXT = (N_Q_TILE + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + +#pragma unroll + for (int i_q = tid; i_q < N_Q_TILE; i_q += THREADS_PER_BLOCK) { + const int i_head = i_q / (N_EMBD / 4); + const int i_embd = i_q % (N_EMBD / 4); + const float4 q = *(const float4 *) (q_base + i_head*nbq1 + i_embd*sizeof(float4)); + half4 q_packed; + q_packed.h2[0] = __float22half2_rn(make_float2(q.x, q.y)); + q_packed.h2[1] = __float22half2_rn(make_float2(q.z, q.w)); + q_shared_h[i_head][i_embd] = q_packed.i2; + } + + // phase 2 - load (and dequantize if needed) K to shared mem + + __shared__ half2 k_shared_h[K_VECS_PER_BLOCK][N_EMBD_PADDED / 4][2]; + + constexpr int n_k = K_VECS_PER_BLOCK * (N_EMBD / 4); + + if constexpr (TYPE_K == GGML_TYPE_F16) { +#pragma unroll + for (int i_k = tid; i_k < n_k; i_k += THREADS_PER_BLOCK) { + const int i_k_vec = i_k / (N_EMBD / 4); + const int i_embd = i_k % (N_EMBD / 4); + const int i_kv = start_kv + i_k_vec; + if (i_kv < n_kv) { + const int2 * k_base = (const int2 *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); + *(int2*) &k_shared_h[i_k_vec][i_embd] = k_base[i_embd]; + } else { + *(int2*) &k_shared_h[i_k_vec][i_embd] = make_int2(0, 0); + } + } + } else { + constexpr dequantize_V_t dequantize_k = get_dequantize_V(); +#pragma unroll + for (int i_k = tid; i_k < n_k; i_k += THREADS_PER_BLOCK) { + const int i_k_vec = i_k / (N_EMBD / 4); + const int i_embd = i_k % (N_EMBD / 4); + const int i_kv = start_kv + i_k_vec; + if (i_kv < n_kv) { + const void * k_base = (const void *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); + dequantize_k(k_base, &k_shared_h[i_k_vec][i_embd][0], i_embd * 4); + } else { + *(int2*) &k_shared_h[i_k_vec][i_embd] = make_int2(0, 0); + } + } + } + + __syncthreads(); + + // phase 3 - calculate lightning indexer scores + + __shared__ float qk_shared[WARPS_PER_BLOCK][HEADS_PER_INNER_LOOP][K_VECS_PER_BLOCK]; + + // load K fragment + wmma::fragment frag_k; + wmma::load_matrix_sync(frag_k, (half*) &k_shared_h[0][i_warp * K_EMBD_PER_INNER_LOOP / 4], N_EMBD_PADDED); + + float score_k = 0.0f; + + for (int i_head_0 = 0; i_head_0 < N_HEAD; i_head_0 += HEADS_PER_INNER_LOOP) { + const int i_head_next = i_head_0 + HEADS_PER_INNER_LOOP; + + // we don't use accumulator for anything, fill it with zeros + wmma::fragment frag_acc; + wmma::fill_fragment(frag_acc, 0.0f); + + // load Q fragment + wmma::fragment frag_q; + wmma::load_matrix_sync(frag_q, (half*) &q_shared_h[0][i_warp * K_EMBD_PER_INNER_LOOP / 4], N_EMBD_PADDED); + + // preload next Q tile to registers during matrix multiplication + float4 q_next[N_Q_NEXT]; + + if (i_head_next < N_HEAD) { +#pragma unroll + for (int i_q = tid, i_q_next = 0; i_q < N_Q_TILE; i_q += THREADS_PER_BLOCK) { + const int i_head = i_head_next + i_q / (N_EMBD / 4); + const int i_embd = i_q % (N_EMBD / 4); + q_next[i_q_next++] = *(const float4 *) (q_base + i_head*nbq1 + i_embd*sizeof(float4)); + } + } + + // perform matrix multiplication + wmma::mma_sync(frag_acc, frag_q, frag_k, frag_acc); + wmma::store_matrix_sync((float*) &qk_shared[i_warp][0][0], frag_acc, K_VECS_PER_BLOCK, wmma::mem_row_major); + + // make sure all threads finished using q_shared_h so we can store next tile + __syncthreads(); + + // write preloaded Q tile to shared memory + if (i_head_next < N_HEAD) { +#pragma unroll + for (int i_q = tid, i_q_next = 0; i_q < N_Q_TILE; i_q += THREADS_PER_BLOCK) { + const int i_head = i_q / (N_EMBD / 4); + const int i_embd = i_q % (N_EMBD / 4); + half4 q_packed; + q_packed.h2[0] = __float22half2_rn(make_float2(q_next[i_q_next].x, q_next[i_q_next].y)); + q_packed.h2[1] = __float22half2_rn(make_float2(q_next[i_q_next].z, q_next[i_q_next].w)); + q_shared_h[i_head][i_embd] = q_packed.i2; + ++i_q_next; + } + } + + // accumulate QK multiplication results from all block warps + // (there are 256 threads in block and 256 matmul outputs) + // TODO it will break if WARP_SIZE is not 32 + const int h = tid / K_VECS_PER_BLOCK; + const int k = tid % K_VECS_PER_BLOCK; + const float w_val = w_shared[i_head_0 + h]; + + float sum = 0.0f; +#pragma unroll + for (int w = 0; w < WARPS_PER_BLOCK; ++w) { + sum += qk_shared[w][h][k]; + } + + // ReLU, weight + sum = sum > 0.0f ? sum : 0.0f; + sum *= w_val; + + // wait until qk_shared[0] is no longer used + __syncthreads(); + + // reuse qk_shared[0] for storing partial results + qk_shared[0][h][k] = sum; + + // wait until all threads write their results + __syncthreads(); + + // accumulate result over heads + if (tid < K_VECS_PER_BLOCK) { +#pragma unroll + for (int i_head = 0; i_head < HEADS_PER_INNER_LOOP; ++i_head) { + score_k += qk_shared[0][i_head][tid]; + } + } + + // make sure all threads finished using qk_shared + __syncthreads(); + } + + // phase 4 - store output to VRAM + + if (tid < K_VECS_PER_BLOCK) { + const int i_kv = start_kv + tid; + if (i_kv < n_kv) { + const half * m_base = (const half *) ((const char *) M + i_batch*nbm1 + (i_stream%nem3)*nbm3); + float * dst_base = (float *) ((char *) dst + i_batch*nb1 + i_stream*nb3); + dst_base[i_kv] = score_k + __half2float(m_base[i_kv]); + } + } +} + +#else // defined(TURING_MMA_AVAILABLE) + +template +static __global__ void lightning_indexer_kernel_wmma( + const float * Q, const char * K, const float * W, const half * M, float * dst, + int64_t n_stream, int64_t n_batch, int64_t n_kv, + size_t nb1, size_t nb2, size_t nb3, + size_t nbq1, size_t nbq2, size_t nbq3, + size_t nbk1, size_t nbk2, size_t nbk3, + size_t nbw1, size_t nbw2, size_t nbw3, + size_t nbm1, size_t nbm2, size_t nbm3, + int64_t nem3 + ) { + GGML_UNUSED_VARS(Q, K, W, M, dst, + n_stream, n_batch, n_kv, + nb1, nb2, nb3, + nbq1, nbq2, nbq3, + nbk1, nbk2, nbk3, + nbw1, nbw2, nbw3, + nem3); + NO_DEVICE_CODE; +} + +#endif // defined(TURING_MMA_AVAILABLE) +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +// tokens scored per block by the tile kernel +#define LIGHTNING_INDEXER_TILE_TOKENS 8 + +// heads whose queries the tile kernel stages per pass, MUSA arch 21 caps static shared memory +// at 28 KB and the queries of four heads do not fit there next to the key tile +#if defined(GGML_USE_MUSA) && defined(__MUSA_ARCH__) && __MUSA_ARCH__ < 220 +#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 2 +#else +#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 4 +#endif + +// TODO there is one ugly assumption used in this kernel - that WARP_SIZE is equal to 32 +// thanks to that one warp operating on float4 processes whole indexer K/Q vectors +// 32 * 4 = 128 (N_EMBD) + +template +static __global__ void lightning_indexer_kernel_vec( + const float * Q, const char * K, const float * W, const half * M, float * dst, + int64_t n_stream, int64_t n_batch, int64_t n_kv, + size_t nb1, size_t nb2, size_t nb3, + size_t nbq1, size_t nbq2, size_t nbq3, + size_t nbk1, size_t nbk2, size_t nbk3, + size_t nbw1, size_t nbw2, size_t nbw3, + size_t nbm1, size_t nbm2, size_t nbm3, + int64_t nem3 + ) { + + constexpr int K_VECS_PER_WARP = K_VECS_PER_BLOCK / WARPS_PER_BLOCK; + constexpr int THREADS_PER_BLOCK = WARPS_PER_BLOCK * WARP_SIZE; + + const int i_batch = blockIdx.y; + const int i_stream = blockIdx.z; + const int i_warp = threadIdx.y; + const int i_lane = threadIdx.x; + const int tid = i_warp * WARP_SIZE + i_lane; + + // each warp processes K_VECS_PER_WARP K vectors + const int start_kv_block = blockIdx.x * K_VECS_PER_BLOCK; + const int start_kv = start_kv_block + i_warp * K_VECS_PER_WARP; + + const char * q_base = (const char *) Q + i_batch*nbq2 + i_stream*nbq3; + const float * w_base = (const float *) ((const char *) W + i_batch*nbw1 + i_stream*nbw3); + + // phase 1 - load (and dequantize if needed) K to registers + + float4 k_reg_f[K_VECS_PER_WARP]; + + if constexpr (TYPE_K == GGML_TYPE_F32) { + // direct copy of float4 +#pragma unroll + for (int k = 0; k < K_VECS_PER_WARP; ++k) { + int i_kv = start_kv + k; + if (i_kv < n_kv) { + const float4 * k_base = (const float4 *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); + k_reg_f[k] = k_base[i_lane]; + } else { + k_reg_f[k] = make_float4(0, 0, 0, 0); + } + } + } else { + // dequantize remaining types to float + constexpr dequantize_V_t dequantize_k = get_dequantize_V(); +#pragma unroll + for (int k = 0; k < K_VECS_PER_WARP; ++k) { + int i_kv = start_kv + k; + if (i_kv < n_kv) { + const void * k_base = (const void *) ((const char *) K + i_kv*nbk2 + i_stream*nbk3); + dequantize_k(k_base, &k_reg_f[k], i_lane * 4); + } else { + k_reg_f[k] = make_float4(0, 0, 0, 0); + } + } + } + + float score_k[K_VECS_PER_WARP] = { 0.0f }; + + // load weights and Q only for N_HEAD_INNER heads at once to reduce shared memory usage + constexpr int N_HEAD_INNER = N_HEAD / 4; + + for (int i_head_0 = 0; i_head_0 < N_HEAD; i_head_0 += N_HEAD_INNER) { + // phase 2 - load weights and Q to shared memory + + __shared__ float w_shared[N_HEAD_INNER]; + __shared__ float4 q_shared_f[N_HEAD_INNER][N_EMBD / 4]; + + if (tid < N_HEAD_INNER) { + w_shared[tid] = w_base[i_head_0 + tid]; + } + + constexpr int n_q = N_HEAD_INNER * (N_EMBD / 4); +#pragma unroll + for (int i_q = tid; i_q < n_q; i_q += THREADS_PER_BLOCK) { + const int i_head_inner = i_q / (N_EMBD / 4); + const int i_head = i_head_0 + i_head_inner; + const int i_embd = i_q % (N_EMBD / 4); + q_shared_f[i_head_inner][i_embd] = *(const float4 *) (q_base + i_head*nbq1 + i_embd*sizeof(float4)); + } + + __syncthreads(); + + // phase 3 - calculate lightning indexer scores + + for (int i_head_inner = 0; i_head_inner < N_HEAD_INNER; ++i_head_inner) { + const float w_val = w_shared[i_head_inner]; + float qk[K_VECS_PER_WARP] = { 0.0f }; + + // dot product of floats + const float4 q_vec = q_shared_f[i_head_inner][i_lane]; + +#pragma unroll + for (int k = 0; k < K_VECS_PER_WARP; ++k) { + ggml_cuda_mad(qk[k], q_vec.x, k_reg_f[k].x); + ggml_cuda_mad(qk[k], q_vec.y, k_reg_f[k].y); + ggml_cuda_mad(qk[k], q_vec.z, k_reg_f[k].z); + ggml_cuda_mad(qk[k], q_vec.w, k_reg_f[k].w); + } + +#pragma unroll + for (int k = 0; k < K_VECS_PER_WARP; ++k) { + float sum = warp_reduce_sum(qk[k]); + + // ReLU, weight + if (i_lane == 0) { + sum = (sum > 0.0f) ? sum : 0.0f; + score_k[k] += sum * w_val; + } + } + } + + __syncthreads(); + } + + // phase 4 - store outputs to shared memory + + __shared__ float dst_shared[K_VECS_PER_BLOCK]; + + if (i_lane == 0) { +#pragma unroll + for (int k = 0; k < K_VECS_PER_WARP; ++k) { + dst_shared[i_warp * K_VECS_PER_WARP + k] = score_k[k]; + } + } + + __syncthreads(); + + // phase 5 - write from shared memory to VRAM in coalesced manner + + if (tid < K_VECS_PER_BLOCK) { + int i_kv = start_kv_block + tid; + if (i_kv < n_kv) { + const half * m_base = (const half *) ((const char *) M + i_batch*nbm1 + (i_stream%nem3)*nbm3); + float * dst_base = (float *) ((char *) dst + i_batch*nb1 + i_stream*nb3); + dst_base[i_kv] = dst_shared[tid] + __half2float(m_base[i_kv]); + } + } +} + +// one block scores a tile of K_VECS_PER_BLOCK keys against TOKENS_PER_BLOCK tokens: the keys are +// staged in half precision and the queries of every head in float, each thread owns KEYS_PER_THREAD +// keys for one token, a warp shares its token so the query reads are broadcasts, and every key +// element is widened once for all heads, so no dot product needs a cross thread reduction +template +static __global__ void lightning_indexer_kernel_tile( + const float * Q, const char * K, const float * W, const half * M, float * dst, + int64_t n_stream, int64_t n_batch, int64_t n_kv, + size_t nb1, size_t nb2, size_t nb3, + size_t nbq1, size_t nbq2, size_t nbq3, + size_t nbk1, size_t nbk2, size_t nbk3, + size_t nbw1, size_t nbw2, size_t nbw3, + size_t nbm1, size_t nbm2, size_t nbm3, + int64_t nem3 + ) { + + constexpr int THREADS_PER_BLOCK = WARPS_PER_BLOCK * WARP_SIZE; + constexpr int TOKENS_PER_BLOCK = LIGHTNING_INDEXER_TILE_TOKENS; + constexpr int KEY_LANES = THREADS_PER_BLOCK / TOKENS_PER_BLOCK; + constexpr int KEYS_PER_THREAD = K_VECS_PER_BLOCK / KEY_LANES; + constexpr int N_EMBD_H2 = N_EMBD / 2; + constexpr int HEADS_PER_PASS = N_HEAD < LIGHTNING_INDEXER_TILE_HEADS_PER_PASS ? N_HEAD : LIGHTNING_INDEXER_TILE_HEADS_PER_PASS; + + static_assert(THREADS_PER_BLOCK % TOKENS_PER_BLOCK == 0, "threads must cover the token tile"); + static_assert(K_VECS_PER_BLOCK % KEY_LANES == 0, "key lanes must cover the key tile"); + static_assert(N_HEAD % HEADS_PER_PASS == 0, "head passes must cover the heads"); + + const int tid = threadIdx.y * WARP_SIZE + threadIdx.x; + const int start_kv = blockIdx.x * K_VECS_PER_BLOCK; + const int start_batch = blockIdx.y * TOKENS_PER_BLOCK; + const int i_stream = blockIdx.z; + + // the row padding keeps the keys of consecutive threads in distinct banks + __shared__ half2 k_shared[K_VECS_PER_BLOCK][N_EMBD_H2 + 1]; + __shared__ float2 q_shared[HEADS_PER_PASS][TOKENS_PER_BLOCK][N_EMBD_H2]; + __shared__ float w_shared[N_HEAD][TOKENS_PER_BLOCK]; + + // phase 1 - stage the key tile four elements at a time, rows past n_kv are zero + +#pragma unroll + for (int i = tid; i < K_VECS_PER_BLOCK * (N_EMBD / 4); i += THREADS_PER_BLOCK) { + const int r = i / (N_EMBD / 4); + const int c4 = i % (N_EMBD / 4); + + half2 lo = make_half2(0.0f, 0.0f); + half2 hi = lo; + if (start_kv + r < n_kv) { + const char * k_row = K + (start_kv + r)*nbk2 + i_stream*nbk3; + if constexpr (TYPE_K == GGML_TYPE_F16) { + lo = ((const half2 *) k_row)[2*c4 + 0]; + hi = ((const half2 *) k_row)[2*c4 + 1]; + } else { + float4 v; + if constexpr (TYPE_K == GGML_TYPE_F32) { + v = ((const float4 *) k_row)[c4]; + } else { + constexpr dequantize_V_t dequantize_k = get_dequantize_V(); + dequantize_k(k_row, &v, c4 * 4); + } + lo = make_half2(v.x, v.y); + hi = make_half2(v.z, v.w); + } + } + + k_shared[r][2*c4 + 0] = lo; + k_shared[r][2*c4 + 1] = hi; + } + + // phase 2 - stage the weights of every head, tokens past n_batch are zero + + if (tid < N_HEAD * TOKENS_PER_BLOCK) { + const int h = tid / TOKENS_PER_BLOCK; + const int r = tid % TOKENS_PER_BLOCK; + w_shared[h][r] = start_batch + r < n_batch ? + ((const float *) ((const char *) W + (start_batch + r)*nbw1 + i_stream*nbw3))[h] : 0.0f; + } + + const int kl = tid % KEY_LANES; + const int tl = tid / KEY_LANES; + + float qk[N_HEAD][KEYS_PER_THREAD] = { { 0.0f } }; + +#pragma unroll + for (int h0 = 0; h0 < N_HEAD; h0 += HEADS_PER_PASS) { + // the previous pass is fully consumed before its queries are replaced + if (h0 > 0) { + __syncthreads(); + } + + // phase 3 - stage the queries of the heads of this pass, tokens past n_batch are zero + +#pragma unroll + for (int i = tid; i < HEADS_PER_PASS * TOKENS_PER_BLOCK * (N_EMBD / 4); i += THREADS_PER_BLOCK) { + const int h = i / (TOKENS_PER_BLOCK * (N_EMBD / 4)); + const int r = i / (N_EMBD / 4) % TOKENS_PER_BLOCK; + const int c4 = i % (N_EMBD / 4); + + float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f); + if (start_batch + r < n_batch) { + v = *(const float4 *) ((const char *) Q + (h0 + h)*nbq1 + (start_batch + r)*nbq2 + i_stream*nbq3 + c4*sizeof(float4)); + } + + q_shared[h][r][2*c4 + 0] = make_float2(v.x, v.y); + q_shared[h][r][2*c4 + 1] = make_float2(v.z, v.w); + } + + __syncthreads(); + + // phase 4 - float products of the widened keys for the heads of this pass + +#pragma unroll 8 + for (int c = 0; c < N_EMBD_H2; ++c) { + float2 k_val[KEYS_PER_THREAD]; +#pragma unroll + for (int j = 0; j < KEYS_PER_THREAD; ++j) { + k_val[j] = __half22float2(k_shared[kl + j*KEY_LANES][c]); + } +#pragma unroll + for (int h = 0; h < HEADS_PER_PASS; ++h) { + const float2 q_val = q_shared[h][tl][c]; +#pragma unroll + for (int j = 0; j < KEYS_PER_THREAD; ++j) { + qk[h0 + h][j] = fmaf(k_val[j].x, q_val.x, qk[h0 + h][j]); + qk[h0 + h][j] = fmaf(k_val[j].y, q_val.y, qk[h0 + h][j]); + } + } + } + } + + // phase 5 - ReLU, weight, add the mask and write, consecutive threads write consecutive keys + + float score[KEYS_PER_THREAD] = { 0.0f }; + +#pragma unroll + for (int h = 0; h < N_HEAD; ++h) { +#pragma unroll + for (int j = 0; j < KEYS_PER_THREAD; ++j) { + score[j] += fmaxf(qk[h][j], 0.0f) * w_shared[h][tl]; + } + } + + const int i_batch = start_batch + tl; + if (i_batch >= n_batch) { + return; + } + + const half * m_base = (const half *) ((const char *) M + i_batch*nbm1 + (i_stream%nem3)*nbm3); + float * dst_base = (float *) ((char *) dst + i_batch*nb1 + i_stream*nb3); + +#pragma unroll + for (int j = 0; j < KEYS_PER_THREAD; ++j) { + const int i_kv = start_kv + kl + j*KEY_LANES; + if (i_kv < n_kv) { + dst_base[i_kv] = score[j] + __half2float(m_base[i_kv]); + } + } +} + +#define LIGHTNING_INDEXER_CASE(lightning_indexer_kernel, n_embd, n_head, K, type_K) \ + if (K->type == (type_K)) { \ + lightning_indexer_kernel \ + <<>>( \ + q_d, k_d, w_d, m_d, dst_d, \ + n_stream, n_batch, n_kv, \ + nb1, nb2, nb3, \ + nbq1, nbq2, nbq3, \ + nbk1, nbk2, nbk3, \ + nbw1, nbw2, nbw3, \ + nbm1, nbm2, nbm3, \ + nem3 \ + ); \ + } else + +void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * w = dst->src[2]; // weights + const ggml_tensor * m = dst->src[3]; // mask + + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT( q->type == GGML_TYPE_F32); + GGML_ASSERT( w->type == GGML_TYPE_F32); + GGML_ASSERT( m->type == GGML_TYPE_F16); + + GGML_TENSOR_LOCALS(int64_t, neq, q, ne) + GGML_TENSOR_LOCALS(size_t, nbq, q, nb) + GGML_TENSOR_LOCALS(int64_t, nek, k, ne) + GGML_TENSOR_LOCALS(size_t, nbk, k, nb) + GGML_TENSOR_LOCALS(int64_t, new, w, ne) + GGML_TENSOR_LOCALS(size_t, nbw, w, nb) + GGML_TENSOR_LOCALS(int64_t, nem, m, ne) + GGML_TENSOR_LOCALS(size_t, nbm, m, nb) + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + + // input tensor rows must be contiguous + GGML_ASSERT(nbq0 == ggml_type_size(q->type)); + GGML_ASSERT(nbk0 == ggml_type_size(k->type)); + GGML_ASSERT(nbw0 == ggml_type_size(w->type)); + GGML_ASSERT(nbm0 == ggml_type_size(m->type)); + + // dst cannot be transposed or permuted + GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(nb0 <= nb1); + GGML_ASSERT(nb1 <= nb2); + GGML_ASSERT(nb2 <= nb3); + + const int n_embd = q->ne[0]; + const int n_head = q->ne[1]; + const int n_batch = q->ne[2]; + const int n_stream = q->ne[3]; + const int n_kv = k->ne[2]; + + const float * q_d = (const float *) q->data; + const char * k_d = (const char *) k->data; + const float * w_d = (const float *) w->data; + const half * m_d = (const half *) m->data; + float * dst_d = ( float *) dst->data; + + const int device = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[device].cc; + + if (n_embd == 128 && n_head == 64) { +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + if (GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { + // use wmma kernel + constexpr int K_VECS_PER_BLOCK = 32; + constexpr int WARPS_PER_BLOCK = 8; + + dim3 block(32, WARPS_PER_BLOCK); + int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); + dim3 grid(num_kv_blocks, n_batch, n_stream); + + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 64, k, GGML_TYPE_Q8_0) + GGML_ABORT("fatal error"); + } else { +#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + { +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + // use vector kernel + constexpr int K_VECS_PER_WARP = 8; + constexpr int WARPS_PER_BLOCK = 8; + constexpr int K_VECS_PER_BLOCK = K_VECS_PER_WARP * WARPS_PER_BLOCK; + + dim3 block(32, WARPS_PER_BLOCK); + int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); + dim3 grid(num_kv_blocks, n_batch, n_stream); + + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 64, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 64, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 64, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 64, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 64, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 64, k, GGML_TYPE_Q8_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 64, k, GGML_TYPE_BF16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 64, k, GGML_TYPE_F32) + GGML_ABORT("fatal error"); + } + } else if (n_embd == 128 && n_head == 32) { +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + if (GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_BF16) { + // use wmma kernel + constexpr int K_VECS_PER_BLOCK = 32; + constexpr int WARPS_PER_BLOCK = 8; + + dim3 block(32, WARPS_PER_BLOCK); + int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); + dim3 grid(num_kv_blocks, n_batch, n_stream); + + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_wmma, 128, 32, k, GGML_TYPE_Q8_0) + GGML_ABORT("fatal error"); + } else { +#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + { +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + // use vector kernel + constexpr int K_VECS_PER_WARP = 8; + constexpr int WARPS_PER_BLOCK = 8; + constexpr int K_VECS_PER_BLOCK = K_VECS_PER_WARP * WARPS_PER_BLOCK; + + dim3 block(32, WARPS_PER_BLOCK); + int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); + dim3 grid(num_kv_blocks, n_batch, n_stream); + + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_Q8_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_BF16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_F32) + GGML_ABORT("fatal error"); + } + } else if (n_embd == 128 && n_head == 4 && n_batch >= LIGHTNING_INDEXER_TILE_TOKENS) { + // too few heads for a wmma tile, the tile kernel shares the keys across the tokens + constexpr int WARPS_PER_BLOCK = 8; + constexpr int K_VECS_PER_BLOCK = 64; + + dim3 block(32, WARPS_PER_BLOCK); + int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); + int num_batch_blocks = (n_batch + LIGHTNING_INDEXER_TILE_TOKENS - 1) / LIGHTNING_INDEXER_TILE_TOKENS; + dim3 grid(num_kv_blocks, num_batch_blocks, n_stream); + + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_Q8_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_BF16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_F32) + GGML_ABORT("fatal error"); + } else if (n_embd == 128 && n_head == 4) { + // a batch smaller than a token tile, use vector kernel + constexpr int K_VECS_PER_WARP = 8; + constexpr int WARPS_PER_BLOCK = 8; + constexpr int K_VECS_PER_BLOCK = K_VECS_PER_WARP * WARPS_PER_BLOCK; + + dim3 block(32, WARPS_PER_BLOCK); + int num_kv_blocks = (n_kv + (K_VECS_PER_BLOCK) - 1) / (K_VECS_PER_BLOCK); + dim3 grid(num_kv_blocks, n_batch, n_stream); + + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 4, k, GGML_TYPE_F16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 4, k, GGML_TYPE_Q4_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 4, k, GGML_TYPE_Q4_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 4, k, GGML_TYPE_Q5_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 4, k, GGML_TYPE_Q5_1) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 4, k, GGML_TYPE_Q8_0) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 4, k, GGML_TYPE_BF16) + LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 4, k, GGML_TYPE_F32) + GGML_ABORT("fatal error"); + } else { + GGML_ABORT("fatal error"); + } +} + +bool ggml_cuda_lightning_indexer_supported(int device, const ggml_tensor * dst) { + GGML_UNUSED(device); + + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * w = dst->src[2]; // weights + const ggml_tensor * m = dst->src[3]; // mask + + GGML_TENSOR_LOCALS(int64_t, neq, q, ne) + GGML_TENSOR_LOCALS(size_t, nbq, q, nb) + GGML_TENSOR_LOCALS(int64_t, nek, k, ne) + GGML_TENSOR_LOCALS(size_t, nbk, k, nb) + GGML_TENSOR_LOCALS(int64_t, new, w, ne) + GGML_TENSOR_LOCALS(size_t, nbw, w, nb) + GGML_TENSOR_LOCALS(int64_t, nem, m, ne) + GGML_TENSOR_LOCALS(size_t, nbm, m, nb) + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + + if (neq0 != 128) { + return false; + } + + if (neq1 != 64 && neq1 != 32 && neq1 != 4) { + return false; + } + + // alignment checks + for (const ggml_tensor * t : {q, k}) { + if (ggml_is_quantized(t->type)) { + continue; + } + for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { + if (t->nb[i] % 16 != 0) { + return false; + } + } + } + + switch(k->type) { + case GGML_TYPE_F32: + case GGML_TYPE_BF16: + case GGML_TYPE_F16: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q4_0: + return true; + default: + return false; + } +} diff --git a/ggml/src/ggml-cuda/lightning-indexer.cuh b/ggml/src/ggml-cuda/lightning-indexer.cuh new file mode 100644 index 0000000..f2fc951 --- /dev/null +++ b/ggml/src/ggml-cuda/lightning-indexer.cuh @@ -0,0 +1,4 @@ +#include "common.cuh" + +void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +bool ggml_cuda_lightning_indexer_supported(int device, const ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/mean.cu b/ggml/src/ggml-cuda/mean.cu index 49af538..64ad7e1 100644 --- a/ggml/src/ggml-cuda/mean.cu +++ b/ggml/src/ggml-cuda/mean.cu @@ -18,7 +18,7 @@ void ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); const int64_t ncols = src0->ne[0]; const int64_t nrows = ggml_nrows(src0); @@ -65,11 +65,20 @@ void ggml_cuda_op_mean(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { // Heuristic for block size selection to optimize occupancy. // See discussion in: https://github.com/ggml-org/llama.cpp/pull/15132 + dim3 block_dims; if ((nrows / nsm) < 2) { - const dim3 block_dims(512, 1, 1); - reduce_rows_f32<<>>(src0_d, dst_d, ncols); + block_dims = dim3(512, 1, 1); } else { - const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1); - reduce_rows_f32<<>>(src0_d, dst_d, ncols); + block_dims = dim3(ncols < 1024 ? 32 : 128, 1, 1); } + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + + if (ggml_is_contiguous(src0)) { + ggml_cuda_kernel_launch(reduce_rows_f32, launch_params, src0_d, dst_d, ncols); + return; + } + + const char * src0_d_bytes = (const char *) src0->data; + ggml_cuda_kernel_launch(reduce_rows_f32_strided, launch_params, src0_d_bytes, dst_d, ncols, + src0->ne[1], src0->ne[2], src0->nb[1], src0->nb[2], src0->nb[3]); } diff --git a/ggml/src/ggml-cuda/mma.cuh b/ggml/src/ggml-cuda/mma.cuh index c91dd2d..c72f121 100644 --- a/ggml/src/ggml-cuda/mma.cuh +++ b/ggml/src/ggml-cuda/mma.cuh @@ -80,23 +80,19 @@ namespace ggml_cuda_mma { DATA_LAYOUT_J_MAJOR = 10, // Matrix C for CDNA and RDNA4, int and float matrix C for RDNA3. DATA_LAYOUT_I_MAJOR_MIRRORED = 20, // Volta, matrix A&B for RDNA3. DATA_LAYOUT_J_MAJOR_MIRRORED = 30, + DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (RDNA4/CDNA), convert to float to unscramble. }; // Implemented mma combinations are: // - (I_MAJOR, I_MAJOR) -> I_MAJOR // - (I_MAJOR, I_MAJOR_MIRRORED) -> I_MAJOR // - (I_MAJOR, J_MAJOR_MIRRORED) -> I_MAJOR - static constexpr bool is_i_major(const data_layout dl) { - return dl == DATA_LAYOUT_I_MAJOR || - dl == DATA_LAYOUT_I_MAJOR_MIRRORED; - } - static constexpr __device__ data_layout get_input_data_layout() { -#if defined(RDNA3) || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#if defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE) return DATA_LAYOUT_I_MAJOR_MIRRORED; #else return DATA_LAYOUT_I_MAJOR; -#endif // defined(RDNA3) || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#endif // defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE) } template @@ -113,7 +109,6 @@ namespace ggml_cuda_mma { T x[ne] = {0}; static constexpr __device__ bool supported() { - if (I == 64 && J == 2) return true; if (I == 16 && J == 8) return true; if (I == 32 && J == 4) return true; if (I == 16 && J == 16) return true; @@ -122,7 +117,7 @@ namespace ggml_cuda_mma { } static __device__ __forceinline__ int get_i(const int l) { - if constexpr (I == 64 && J == 2) { // Special tile size to load <16, 4> as <16, 8> + if constexpr (I == 16 && J == 4) { return threadIdx.x % 16; } else if constexpr (I == 16 && J == 8) { return threadIdx.x % 16; @@ -139,8 +134,8 @@ namespace ggml_cuda_mma { } static __device__ __forceinline__ int get_j(const int l) { - if constexpr (I == 64 && J == 2) { // Special tile size to load <16, 4> as <16, 8> - return (2 * ((threadIdx.x / 16) % 2) + l); + if constexpr (I == 16 && J == 4) { + return threadIdx.x / 16; } else if constexpr (I == 16 && J == 8) { return 2 * (threadIdx.x / 16) + l; } else if constexpr (I == 32 && J == 4) { @@ -154,7 +149,7 @@ namespace ggml_cuda_mma { return -1; } } -#elif __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#elif defined(VOLTA_MMA_AVAILABLE) static constexpr int ne = I * J / 32; T x[ne] = {0}; @@ -283,7 +278,7 @@ namespace ggml_cuda_mma { static constexpr int J = J_; static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR; -#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#if defined(VOLTA_MMA_AVAILABLE) static constexpr int ne = I * J / WARP_SIZE; half2 x[ne] = {{0.0f, 0.0f}}; @@ -318,13 +313,19 @@ namespace ggml_cuda_mma { half2 x[ne] = {{0.0f, 0.0f}}; static constexpr __device__ bool supported() { - if (I == 16 && J == 8) return true; + if (I == 16 && J == 8) return true; + if (I == 16 && J == 16) return true; + if (I == 32 && J == 8) return true; return false; } static __device__ __forceinline__ int get_i(const int l) { if constexpr (I == 16 && J == 8) { return threadIdx.x % 16; + } else if constexpr (I == 16 && J == 16) { + return threadIdx.x % 16; + } else if constexpr (I == 32 && J == 8) { + return (threadIdx.x % 16) * 2 + l / (ne/2); } else { NO_DEVICE_CODE; return -1; @@ -333,7 +334,15 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ int get_j(const int l) { if constexpr (I == 16 && J == 8) { - return ne * (threadIdx.x / 16) + l; + return (threadIdx.x / 16) * ne + l; + } else if constexpr (I == 16 && J == 16) { +#ifdef RDNA3 + return l*2 + (threadIdx.x / 16); +#else + return (threadIdx.x / 16) * ne + l; +#endif // RDNA3 + } else if constexpr (I == 32 && J == 8) { + return (threadIdx.x / 16) * (ne/2) + l % (ne/2); } else { NO_DEVICE_CODE; return -1; @@ -344,13 +353,19 @@ namespace ggml_cuda_mma { half2 x[ne] = {{0.0f, 0.0f}}; static constexpr __device__ bool supported() { - if (I == 16 && J == 8) return true; + if (I == 16 && J == 8) return true; + if (I == 16 && J == 16) return true; + if (I == 32 && J == 8) return true; return false; } static __device__ __forceinline__ int get_i(const int l) { if constexpr (I == 16 && J == 8) { return threadIdx.x % 16; + } else if constexpr (I == 16 && J == 16) { + return threadIdx.x % 16; + } else if constexpr (I == 32 && J == 8) { + return (threadIdx.x % 16) * 2 + l / (ne/2); } else { NO_DEVICE_CODE; return -1; @@ -359,7 +374,11 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ int get_j(const int l) { if constexpr (I == 16 && J == 8) { - return ne * (threadIdx.x / 16) + l; + return (threadIdx.x / 16) * ne + l; + } else if constexpr (I == 16 && J == 16) { + return (threadIdx.x / 16) * ne + l; + } else if constexpr (I == 32 && J == 8) { + return (threadIdx.x / 16) * (ne/2) + l % (ne/2); } else { NO_DEVICE_CODE; return -1; @@ -407,7 +426,7 @@ namespace ggml_cuda_mma { return -1; } } -#endif // __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#endif // defined(VOLTA_MMA_AVAILABLE) }; template @@ -522,12 +541,15 @@ namespace ggml_cuda_mma { if (I == 16 && J == 16) return true; if (I == 16 && J == 8) return true; if (I == 16 && J == 4) return true; + if (I == 32 && J == 8) return true; return false; } - static __device__ __forceinline__ int get_i(const int /*l*/) { - if constexpr (supported()) { + static __device__ __forceinline__ int get_i(const int l) { + if constexpr (I == 16) { return threadIdx.x % 16; + } else if constexpr (I == 32) { + return (threadIdx.x % 16) * 2 + l / (ne/2); } else { NO_DEVICE_CODE; return -1; @@ -535,8 +557,10 @@ namespace ggml_cuda_mma { } static __device__ __forceinline__ int get_j(const int l) { - if constexpr (supported()) { + if constexpr (I == 16) { return l; + } else if constexpr (I == 32) { + return l % (ne/2); } else { NO_DEVICE_CODE; return -1; @@ -650,6 +674,40 @@ namespace ggml_cuda_mma { } }; + template + struct tile { + static constexpr int I = I_; + static constexpr int J = J_; + static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_SCRAMBLED; + + static constexpr int ne = I * J / ggml_cuda_get_physical_warp_size(); + half2 x[ne] = {{0.0f, 0.0f}}; + + static constexpr __device__ bool supported() { + if (I == 16 && J == 16) return true; + return false; + } + + static __device__ __forceinline__ int get_i(const int l) { + return tile::get_i(l); + } + }; + + static __device__ __forceinline__ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> unscramble(const tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & t) { +#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) + tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> ret; +#pragma unroll + for (int l0 = 0; l0 < t.ne/2; ++l0) { + ret.x[2*l0 + 0] = __lows2half2(t.x[l0], t.x[l0 + t.ne/2]); + ret.x[2*l0 + 1] = __highs2half2(t.x[l0], t.x[l0 + t.ne/2]); + } + return ret; +#else + NO_DEVICE_CODE; + GGML_UNUSED(t); +#endif // defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) + } + #if defined(TURING_MMA_AVAILABLE) template static __device__ __forceinline__ tile get_half2(const tile & tile_float) { @@ -666,6 +724,21 @@ namespace ggml_cuda_mma { ret.x[0] = ggml_cuda_movmatrix(t.x[0]); ret.x[1] = ggml_cuda_movmatrix(t.x[1]); + return ret; + } +#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) + static __device__ __forceinline__ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> get_half2( + const tile<16, 16, float, DATA_LAYOUT_I_MAJOR> & tile_float) { + tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> ret; +#pragma unroll + for (int l = 0; l < tile_float.ne; ++l) { + float tmp[2]; + int i = threadIdx.x / 16; + tmp[i] = tile_float.x[l]; + i ^= 1; + tmp[i] = __shfl_xor_sync(0xFFFFFFFF, tile_float.x[l], 16, WARP_SIZE); + ret.x[l] = make_half2(tmp[0], tmp[1]); + } return ret; } #elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) @@ -701,57 +774,35 @@ namespace ggml_cuda_mma { } #endif // defined(TURING_MMA_AVAILABLE) - static __device__ __forceinline__ void make_identity_mat(tile<16, 8, half2> & t) { -#if defined(RDNA4) - const int row = t.get_i(0); - const int left_right = t.get_j(0) / 4; - const int up_down = row / 8; - const int idx = row % 8; - reinterpret_cast(t.x)[idx] = left_right == up_down ? 1.0f : 0.0f; -#else - GGML_UNUSED_VARS(t); - NO_DEVICE_CODE; -#endif // defined(RDNA4) - } - template static __device__ __forceinline__ void load_generic(tile & t, const T * __restrict__ xs0, const int stride) { -#if defined(AMD_MFMA_AVAILABLE) - if constexpr (I == 64 && J == 2) { // Special tile size to load <16, 4> as <16, 8> -#pragma unroll - for (int l = 0; l < t.ne; ++l) { - t.x[l] = xs0[t.get_i(l)*stride + t.get_j(l)]; - } - } else { - ggml_cuda_memcpy_1(t.x, xs0 + t.get_i(0) * stride + t.get_j(0)); - } -#elif defined(AMD_WMMA_AVAILABLE) - // All wmma layout has contiguous data when i-major. - if constexpr (is_i_major(dl)) { - // the data must be aligned to 16 bytes when bigger than ggml_cuda_get_max_cpy_bytes() - constexpr int aligned_copy_bytes = ggml_cuda_get_max_cpy_bytes(); - if constexpr (sizeof(t.x) > aligned_copy_bytes) { - static_assert(sizeof(t.x) % aligned_copy_bytes == 0, "bad type size"); - constexpr int aligned_copy_count = sizeof(t.x)/aligned_copy_bytes; -#pragma unroll - for (int i = 0; i < aligned_copy_count; ++i) { - ggml_cuda_memcpy_1(t.x + t.ne/aligned_copy_count*i, xs0 + t.get_i(0) * stride + t.get_j(t.ne/aligned_copy_count*i)); - } - } else { - ggml_cuda_memcpy_1(t.x, xs0 + t.get_i(0) * stride + t.get_j(0)); - } - } else { -#pragma unroll - for (int l = 0; l < t.ne; ++l) { - t.x[l] = xs0[t.get_i(l)*stride + t.get_j(l)]; - } - } -#else #pragma unroll for (int l = 0; l < t.ne; ++l) { t.x[l] = xs0[t.get_i(l)*stride + t.get_j(l)]; } -#endif // defined(AMD_MFMA_AVAILABLE) + } + + template + static __device__ __forceinline__ uint32_t swizzle(const uint32_t offset, const uint32_t i) { + static_assert(sizeof(T) <= 4, "unsupported type size"); + constexpr int stride_bytes = stride*sizeof(T); + static_assert(stride_bytes % 16 == 0, "bad stride"); + constexpr uint32_t shift = sizeof(T) == 1 ? 4 : (sizeof(T) == 2 ? 3 : 2); + if (stride_bytes % 32 != 0) { + return offset; // Equivalent to padding with 16 bytes. + } + if (stride_bytes % 64 != 0) { + return offset ^ (((i / 4) % 2) << shift); + } + if (stride_bytes % 128 != 0) { + return offset ^ (((i / 2) % 4) << shift); + } + return offset ^ ((i % 8) << shift); + } + + template + static __device__ __forceinline__ T * swizzle(T * ptr, const uint32_t offset, const uint32_t i) { + return ptr + swizzle(offset, i); } template @@ -764,26 +815,37 @@ namespace ggml_cuda_mma { : "=r"(xi[0]), "=r"(xi[1]) : "l"(xs)); #else - load_generic(t, xs0, stride); + GGML_UNUSED_VARS(t, xs0, stride); + NO_DEVICE_CODE; #endif // TURING_MMA_AVAILABLE } - template + template static __device__ __forceinline__ void load_ldmatrix( - tile<16, 4, T> & t, const T * __restrict__ xs0, const int stride) { + tile<16, 4, T, dl> & t, const T * __restrict__ xs0, const int stride) { #ifdef TURING_MMA_AVAILABLE int * xi = (int *) t.x; const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride; asm volatile("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];" : "=r"(xi[0]), "=r"(xi[1]) : "l"(xs)); +#elif defined(AMD_WMMA_AVAILABLE) +#ifdef RDNA3 + static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout"); + static_assert(sizeof(t.x) == 16, "bad ne"); + ggml_cuda_memcpy_1<8>(t.x + 0, xs0 + t.get_i(0)*stride + 0); + ggml_cuda_memcpy_1<8>(t.x + 2, xs0 + t.get_i(0)*stride + 2); +#else + static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); + static_assert(sizeof(t.x) == 8, "bad ne"); + ggml_cuda_memcpy_1<8>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0)); +#endif // RDNA3 +#elif defined(AMD_MFMA_AVAILABLE) + static_assert(sizeof(t.x) == 4, "bad ne"); + ggml_cuda_memcpy_1<4>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0)); #else -#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA GGML_UNUSED_VARS(t, xs0, stride); NO_DEVICE_CODE; -#else - load_generic(t, xs0, stride); -#endif // __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA #endif // TURING_MMA_AVAILABLE } @@ -796,19 +858,26 @@ namespace ggml_cuda_mma { asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];" : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3]) : "l"(xs)); -#else -#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA -#if 1 - // TODO: more generic handling - static_assert(sizeof(T) == 4, "bad type size"); +#elif defined(VOLTA_MMA_AVAILABLE) ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 0, xs0 + t.get_i(0)*stride + 0); ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 4, xs0 + t.get_i(4)*stride + 4); +#elif defined(AMD_WMMA_AVAILABLE) +#ifdef RDNA3 + static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout"); + static_assert(sizeof(t.x) == 32, "bad ne"); + ggml_cuda_memcpy_1<16>(t.x + 0, xs0 + t.get_i(0)*stride + 0); + ggml_cuda_memcpy_1<16>(t.x + 4, xs0 + t.get_i(0)*stride + 4); #else - load_generic(t, xs0, stride); -#endif // 1 + static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); + static_assert(sizeof(t.x) == 16, "bad ne"); + ggml_cuda_memcpy_1<16>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0)); +#endif // RDNA3 +#elif defined(AMD_MFMA_AVAILABLE) + static_assert(sizeof(t.x) == 8, "bad ne"); + ggml_cuda_memcpy_1<8>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0)); #else - load_generic(t, xs0, stride); -#endif // __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA + GGML_UNUSED_VARS(t, xs0, stride); + NO_DEVICE_CODE; #endif // TURING_MMA_AVAILABLE } @@ -819,37 +888,185 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ void load_ldmatrix( tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) { +#ifdef VOLTA_MMA_AVAILABLE #pragma unroll for (int l0 = 0; l0 < t.ne; l0 += 2) { ggml_cuda_memcpy_1<2*sizeof(half2)>(t.x + l0, xs0 + t.get_i(l0)*stride + t.get_j(l0)); } +#else + GGML_UNUSED_VARS(t, xs0, stride); + NO_DEVICE_CODE; +#endif // VOLTA_MMA_AVAILABLE } static __device__ __forceinline__ void load_ldmatrix( tile<32, 4, half2> & t, const half2 * __restrict__ xs0, const int stride) { -#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#if defined(VOLTA_MMA_AVAILABLE) ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride); #else GGML_UNUSED_VARS(t, xs0, stride); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#endif // defined(VOLTA_MMA_AVAILABLE) } - template + template static __device__ __forceinline__ void load_ldmatrix_trans( - tile<16, 8, T> & t, const T * __restrict__ xs0, const int stride) { + tile & t, const T * __restrict__ xs0, const int stride) { #ifdef TURING_MMA_AVAILABLE - int * xi = (int * ) t.x; + static_assert(I == 16, "bad tile width"); + static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); + int * xi = (int *) t.x; const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2); asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];" : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3]) : "l"(xs)); +#elif defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + static_assert(dl == DATA_LAYOUT_I_MAJOR || dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout"); + if constexpr (I == 32) { +#pragma unroll + for (int l0 = 0; l0 < t.ne/2; ++l0) { + const half2 tmp0 = xs0[(2*t.get_j(l0) + 0)*stride + t.get_i(l0)/2]; + const half2 tmp1 = xs0[(2*t.get_j(l0) + 1)*stride + t.get_i(l0)/2]; + + t.x[l0] = __lows2half2(tmp0, tmp1); + t.x[l0 + t.ne/2] = __highs2half2(tmp0, tmp1); + } + } else { + half * xh = (half *) t.x; +#pragma unroll + for (int l = 0; l < t.ne; ++l) { + xh[2*l + 0] = ((const half *) xs0)[(2*t.get_j(l) + 0)*(2*stride) + t.get_i(l)]; + xh[2*l + 1] = ((const half *) xs0)[(2*t.get_j(l) + 1)*(2*stride) + t.get_i(l)]; + } + } #else GGML_UNUSED_VARS(t, xs0, stride); NO_DEVICE_CODE; #endif // TURING_MMA_AVAILABLE } + template + static __device__ __forceinline__ void load_ldmatrix_swizzled( + tile & t, const T * __restrict__ xs0, const int offset) { +#if defined(TURING_MMA_AVAILABLE) + static_assert(I == 8 || I == 16, "bad tile width"); + static_assert(J == 8, "bad tile height"); + const int i = threadIdx.x % t.I; + const int j = ((threadIdx.x / t.I) * (t.J / 2)) % t.J; + int offset_ij = offset + i * stride + j; + offset_ij = swizzle(offset_ij, i); + int * xi = (int *) t.x; + if constexpr (I == 16) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3]) + : "l"(xs0 + offset_ij)); + } else { + asm volatile("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];" + : "=r"(xi[0]), "=r"(xi[1]) + : "l"(xs0 + offset_ij)); + } +#elif defined(VOLTA_MMA_AVAILABLE) +#pragma unroll + for (int o = 0; o < t.ne; o += 4) { + const int offset_ij = offset + t.get_i(o) * stride + o; + ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + o, swizzle(xs0, offset_ij, t.get_i(o))); + } +#elif defined(AMD_WMMA_AVAILABLE) +#ifdef RDNA3 + static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout"); + static_assert(sizeof(t.x) == 32, "bad ne"); + static_assert(I == 16, "bad tile width"); + static_assert(J == 8, "bad tile height"); +#pragma unroll + for (int o = 0; o < 8; o += 4) { + const int offset_ij = offset + t.get_i(0) * stride + o; + ggml_cuda_memcpy_1<16>(t.x + o, swizzle(xs0, offset_ij, t.get_i(0))); + } +#else + static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); + static_assert(sizeof(t.x) == 16, "bad ne"); + const int offset_ij = offset + t.get_i(0)*stride + t.get_j(0); + ggml_cuda_memcpy_1<16>(t.x, swizzle(xs0, offset_ij, t.get_i(0))); +#endif // RDNA3 +#elif defined(AMD_MFMA_AVAILABLE) + static_assert(sizeof(t.x) == 8, "bad ne"); + const int offset_ij = offset + t.get_i(0)*stride + t.get_j(0); + ggml_cuda_memcpy_1<8>(t.x, swizzle(xs0, offset_ij, t.get_i(0))); +#else + GGML_UNUSED_VARS(t, xs0, offset); + NO_DEVICE_CODE; +#endif // defined(TURING_MMA_AVAILABLE) + } + + template + static __device__ __forceinline__ void load_ldmatrix_swizzled( + tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int offset) { +#ifdef VOLTA_MMA_AVAILABLE +#pragma unroll + for (int l0 = 0; l0 < t.ne; l0 += 2) { + const int offset_ij = offset + t.get_i(l0)*stride + t.get_j(l0); + ggml_cuda_memcpy_1<2*sizeof(half2)>(t.x + l0, swizzle(xs0, offset_ij, t.get_i(l0))); + } +#else + GGML_UNUSED_VARS(t, xs0, offset); + NO_DEVICE_CODE; +#endif // VOLTA_MMA_AVAILABLE + } + + template + static __device__ __forceinline__ void load_ldmatrix_trans_swizzled( + tile & t, const T * __restrict__ xs0, const int offset) { +#if defined(TURING_MMA_AVAILABLE) + static_assert(I == 8 || I == 16, "bad tile width"); + static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout"); + const int i = threadIdx.x % 16; + const int j = I == 16 ? (threadIdx.x / 16) * (t.J / 2) : 0; + int offset_ij = offset + i * stride + j; + offset_ij = swizzle(offset_ij, i); + int * xi = (int *) t.x; + if constexpr (I == 16) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];" + : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3]) + : "l"(xs0 + offset_ij)); + } else { + asm volatile("ldmatrix.sync.aligned.m8n8.x2.trans.b16 {%0, %1}, [%2];" + : "=r"(xi[0]), "=r"(xi[1]) + : "l"(xs0 + offset_ij)); + } +#elif defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + static_assert(dl == DATA_LAYOUT_I_MAJOR || dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout"); + if constexpr (I == 32) { +#pragma unroll + for (int l0 = 0; l0 < t.ne/2; ++l0) { + half2 tmp[2]; +#pragma unroll + for (int o = 0; o < 2; ++o) { + const int j = 2*t.get_j(l0) + o; + int offset_ij = offset + j*stride + t.get_i(l0)/2; + offset_ij = swizzle(offset_ij, j); + tmp[o] = xs0[offset_ij]; + } + + t.x[l0] = __lows2half2(tmp[0], tmp[1]); + t.x[l0 + t.ne/2] = __highs2half2(tmp[0], tmp[1]); + } + } else { + half * xh = (half *) t.x; +#pragma unroll + for (int l = 0; l < t.ne; ++l) { +#pragma unroll + for (int o = 0; o < 2; ++o) { + const int j = 2*t.get_j(l) + o; + xh[2*l + o] = ((const half *) xs0)[swizzle<2*stride, half>(2*offset + j*(2*stride) + t.get_i(l), j)]; + } + } + } +#else + GGML_UNUSED_VARS(t, xs0, offset); + NO_DEVICE_CODE; +#endif // defined(TURING_MMA_AVAILABLE) + } + static __device__ __forceinline__ void mma( tile<16, 8, int> & D, const tile<16, 4, int> & A, const tile<8, 4, int> & B) { #ifdef TURING_MMA_AVAILABLE @@ -998,6 +1215,20 @@ namespace ggml_cuda_mma { #endif // TURING_MMA_AVAILABLE } + static __device__ __forceinline__ void mma( + tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR> & A, + const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR> & B) { +#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) + tile<16, 8, half2> * D16 = (tile<16, 8, half2> *) &D; + const tile<16, 8, half2> * A16 = (const tile<16, 8, half2> *) &A; + mma(D16[0], A16[0], B); + mma(D16[1], A16[1], B); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) && defined(RDNA4) + } + template static __device__ __forceinline__ void mma( tile<16, 8, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<8, 8, float, dl_ab> & B) { @@ -1041,25 +1272,35 @@ namespace ggml_cuda_mma { #endif // AMD_MFMA_AVAILABLE } - static __device__ __forceinline__ void mma_block_scaled(tile<16, 8, float> & D, - const tile<16, 8, int> & A, - const tile<8, 8, int> & B, - uint32_t a_scale, - uint32_t b_scale) { + template + static __device__ __forceinline__ void mma_block_scaled_fp4(tile<16, 8, float> & D, + const tile<16, 8, int> & A, + const tile<8, 8, int> & B, + uint32_t a_scale, + uint32_t b_scale) { #ifdef BLACKWELL_MMA_AVAILABLE const int * Axi = (const int *) A.x; const int * Bxi = (const int *) B.x; float * Dxi = (float *) D.x; - asm volatile( - "mma.sync.aligned.kind::mxf4.block_scale.scale_vec::2X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue8m0 " - "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, " - "%10, {0, 0}, %11, {0, 0};" - : "+f"(Dxi[0]), "+f"(Dxi[1]), "+f"(Dxi[2]), "+f"(Dxi[3]) - : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]), "r"(a_scale), "r"(b_scale)); + if constexpr (type == GGML_TYPE_MXFP4) { + asm volatile( + "mma.sync.aligned.kind::mxf4.block_scale.scale_vec::2X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue8m0 " + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, " + "%10, {0, 0}, %11, {0, 0};" + : "+f"(Dxi[0]), "+f"(Dxi[1]), "+f"(Dxi[2]), "+f"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]), "r"(a_scale), "r"(b_scale)); + } else { + asm volatile( + "mma.sync.aligned.kind::mxf4nvf4.block_scale.scale_vec::4X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue4m3 " + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, " + "%10, {0, 0}, %11, {0, 0};" + : "+f"(Dxi[0]), "+f"(Dxi[1]), "+f"(Dxi[2]), "+f"(Dxi[3]) + : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]), "r"(a_scale), "r"(b_scale)); + } #else GGML_UNUSED_VARS(D, A, B, a_scale, b_scale); -#endif // BLACKWELL_MMA_AVAILABLE +#endif // BLACKWELL_MMA_AVAILABLE } static __device__ __forceinline__ void mma( @@ -1168,14 +1409,14 @@ namespace ggml_cuda_mma { tile<16, 16, float, dl_d> & D, const tile<16, 8, nv_bfloat162, dl_ab> & A, const tile<16, 8, nv_bfloat162, dl_ab> & B) { #if defined(AMD_WMMA_AVAILABLE) #if defined(RDNA4) - using bf16x8_t = __attribute__((ext_vector_type(8))) __bf16; + using bf16x8_t = __attribute__((ext_vector_type(8))) short; using floatx8_t = __attribute__((ext_vector_type(8))) float; floatx8_t& acc_frag = reinterpret_cast(D.x[0]); const bf16x8_t& a_frag = reinterpret_cast(A.x[0]); const bf16x8_t& b_frag = reinterpret_cast(B.x[0]); acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12(a_frag, b_frag, acc_frag); #elif defined(RDNA3) - using bf16x16_t = __attribute__((ext_vector_type(16))) __bf16; + using bf16x16_t = __attribute__((ext_vector_type(16))) short; using floatx8_t = __attribute__((ext_vector_type(8))) float; floatx8_t& acc_frag = reinterpret_cast(D.x[0]); const bf16x16_t& a_frag = reinterpret_cast(A.x[0]); @@ -1218,73 +1459,27 @@ namespace ggml_cuda_mma { using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; int32x4_t * acc = (int32x4_t *) D.x; #if defined(CDNA4) || defined(CDNA3) - acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(((int64_t *) A.x)[0], - ((int64_t *) B.x)[0], - acc[0], - 0, 0, 0); + acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(((int64_t *) A.x)[0], ((int64_t *) B.x)[0], acc[0], 0, 0, 0); #elif defined(CDNA2) || defined(CDNA1) - acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0], - B.x[0], - acc[0], - 0, 0, 0); - acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[1], - B.x[1], - acc[0], - 0, 0, 0); + acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0], B.x[0], acc[0], 0, 0, 0); + acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[1], B.x[1], acc[0], 0, 0, 0); #endif // defined(CDNA4) || defined(CDNA3) - #elif defined(AMD_WMMA_AVAILABLE) - using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int; int32x8_t * acc = (int32x8_t *) D.x; - #if defined(RDNA4) using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int; int32x2_t * a_vec = (int32x2_t *) A.x; int32x2_t * b_vec = (int32x2_t *) B.x; - - acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12( - true, - a_vec[0], - true, - b_vec[0], - acc[0], - true - ); - - acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12( - true, - a_vec[1], - true, - b_vec[1], - acc[0], - true - ); - + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], true); + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[1], true, b_vec[1], acc[0], true); #elif defined(RDNA3) using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; int32x4_t * a_vec = (int32x4_t *) A.x; int32x4_t * b_vec = (int32x4_t *) B.x; - - acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32( - true, - a_vec[0], - true, - b_vec[0], - acc[0], - true - ); - - acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32( - true, - a_vec[1], - true, - b_vec[1], - acc[0], - true - ); + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], true); + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[1], true, b_vec[1], acc[0], true); #endif // RDNA4 - #else GGML_UNUSED_VARS(D, A, B); NO_DEVICE_CODE; @@ -1297,19 +1492,10 @@ namespace ggml_cuda_mma { using int32x16_t = __attribute__((__vector_size__(16 * sizeof(int)))) int; int32x16_t * acc = (int32x16_t *) D.x; #if defined(CDNA4) || defined(CDNA3) - acc[0] = __builtin_amdgcn_mfma_i32_32x32x16_i8(((int64_t *) A.x)[0], - ((int64_t *) B.x)[0], - acc[0], - 0, 0, 0); + acc[0] = __builtin_amdgcn_mfma_i32_32x32x16_i8(((int64_t *) A.x)[0], ((int64_t *) B.x)[0], acc[0], 0, 0, 0); #elif defined(CDNA2) || defined(CDNA1) - acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[0], - B.x[0], - acc[0], - 0, 0, 0); - acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[1], - B.x[1], - acc[0], - 0, 0, 0); + acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[0], B.x[0], acc[0], 0, 0, 0); + acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[1], B.x[1], acc[0], 0, 0, 0); #endif // defined(CDNA4) || defined(CDNA3) #else @@ -1329,7 +1515,7 @@ namespace ggml_cuda_mma { static __device__ __forceinline__ void mma( tile<32, 8, float> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) { -#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#if defined(VOLTA_MMA_AVAILABLE) const int * Axi = (const int *) A.x; const int * Bxi = (const int *) B.x; int * Dxi = (int *) D.x; @@ -1344,12 +1530,12 @@ namespace ggml_cuda_mma { #else GGML_UNUSED_VARS(D, A, B); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA +#endif // defined(VOLTA_MMA_AVAILABLE) } static __device__ __forceinline__ void mma( tile<32, 4, half2> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & B) { -#if __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA +#if defined(VOLTA_MMA_AVAILABLE) const int * Axi = (const int *) A.x; const int * Bxi = (const int *) B.x; int * Dxi = (int *) D.x; @@ -1364,41 +1550,51 @@ namespace ggml_cuda_mma { #else GGML_UNUSED_VARS(D, A, B); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA +#endif // defined(VOLTA_MMA_AVAILABLE) + } + + static __device__ __forceinline__ void mma( + tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & A, + const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) { +#if defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) + using halfx16_t = __attribute__((ext_vector_type(16))) _Float16; + halfx16_t * xD = (halfx16_t *) D.x; + const halfx16_t * xA = (const halfx16_t *) A.x; + const halfx16_t * xB = (const halfx16_t *) B.x; + xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[0], xB[0], xD[0], /*opsel =*/ 0); + xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[1], xB[0], xD[0], /*opsel =*/ 1); +#else + GGML_UNUSED_VARS(D, A, B); + NO_DEVICE_CODE; +#endif // TURING_MMA_AVAILABLE } template static __device__ __forceinline__ void mma( tile<16, 16, int, dl_d> & D, const tile<16, 4, int, dl_ab> & A, const tile<16, 4, int, dl_ab> & B) { -#if defined(AMD_WMMA_AVAILABLE) +#if defined(AMD_MFMA_AVAILABLE) + using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; + int32x4_t * acc = (int32x4_t *) D.x; +#if defined(CDNA4) || defined(CDNA3) + const int64_t xA = uint32_t(A.x[0]); + const int64_t xB = uint32_t(B.x[0]); + acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(xA, xB, acc[0], 0, 0, 0); +#elif defined(CDNA2) || defined(CDNA1) + acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0], B.x[0], acc[0], 0, 0, 0); +#endif // defined(CDNA4) || defined(CDNA3) +#elif defined(AMD_WMMA_AVAILABLE) using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int; int32x8_t * acc = (int32x8_t *) D.x; #if defined(RDNA4) using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int; int32x2_t * a_vec = (int32x2_t *) A.x; int32x2_t * b_vec = (int32x2_t *) B.x; - - acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12( - true, - a_vec[0], - true, - b_vec[0], - acc[0], - false - ); + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], false); #elif defined(RDNA3) using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; int32x4_t * a_vec = (int32x4_t *) A.x; int32x4_t * b_vec = (int32x4_t *) B.x; - - acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32( - true, - a_vec[0], - true, - b_vec[0], - acc[0], - false - ); + acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], false); #endif // RDNA4 #else GGML_UNUSED(D); diff --git a/ggml/src/ggml-cuda/mmf.cu b/ggml/src/ggml-cuda/mmf.cu index aad4c34..646a589 100644 --- a/ggml/src/ggml-cuda/mmf.cu +++ b/ggml/src/ggml-cuda/mmf.cu @@ -85,7 +85,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr GGML_ASSERT(sis1 > 0); ggml_cuda_launch_mm_ids_helper(ids_d, ids_src_compact_dev.get(), ids_dst_compact_dev.get(), expert_bounds_dev.get(), - static_cast(n_experts), static_cast(n_tokens), static_cast(n_expert_used), static_cast(ne11), si1, sis1, ctx.stream()); + static_cast(n_experts), static_cast(n_tokens), static_cast(n_expert_used), static_cast(ne11), si1, sis1, /*write_inverse =*/ false, ctx.stream()); CUDA_CHECK(cudaGetLastError()); ids_info.ids_src_compact = ids_src_compact_dev.get(); diff --git a/ggml/src/ggml-cuda/mmf.cuh b/ggml/src/ggml-cuda/mmf.cuh index c2a8d54..429c887 100644 --- a/ggml/src/ggml-cuda/mmf.cuh +++ b/ggml/src/ggml-cuda/mmf.cuh @@ -91,7 +91,7 @@ static __global__ void mul_mat_f( const int row0 = blockIdx.x * rows_per_block; int expert_idx = 0; - int col_base = 0; + [[maybe_unused]] int col_base = 0; const int channel_dst = has_ids ? 0 : blockIdx.y; @@ -122,12 +122,12 @@ static __global__ void mul_mat_f( ids += col_offset * stride_row_id; } - const float2 * y2 = (const float2 *) y; + [[maybe_unused]] const float2 * y2 = (const float2 *) y; extern __shared__ char data_mmv[]; char * shmem_base = data_mmv; - int * slot_map = (int *) shmem_base; + [[maybe_unused]] int * slot_map = (int *) shmem_base; char * compute_base = has_ids ? (shmem_base + GGML_PAD(cols_per_block, 16) * sizeof(int)) : shmem_base; tile_C C[ntA][ntB]; @@ -143,6 +143,7 @@ static __global__ void mul_mat_f( if (threadIdx.x == 0) { slot_map[j] = -1; } + ggml_cuda_syncwarp(); if (col_base + j >= ncols_dst_total) { continue; @@ -171,10 +172,12 @@ static __global__ void mul_mat_f( tile_A A[ntA][warp_size / tile_A::J]; #pragma unroll for (int itA = 0; itA < ntA; ++itA) { + ggml_cuda_syncwarp(); #pragma unroll for (int i = 0; i < tile_A::I; ++i) { tile_xy[i*tile_k_padded + threadIdx.x] = x[(itA*tile_A::I + i)*stride_row + col]; } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_A::J) { load_ldmatrix(A[itA][k0/tile_A::J], tile_xy + k0, tile_k_padded); @@ -183,6 +186,7 @@ static __global__ void mul_mat_f( #pragma unroll for (int itB = 0; itB < ntB; ++itB) { + ggml_cuda_syncwarp(); if constexpr (std::is_same_v) { #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { @@ -212,6 +216,7 @@ static __global__ void mul_mat_f( } else { static_assert(std::is_same_v, "unsupported type"); } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) { tile_B B; @@ -229,6 +234,8 @@ static __global__ void mul_mat_f( if (nwarps > 1) { __syncthreads(); + } else { + ggml_cuda_syncwarp(); } #pragma unroll for (int itB = 0; itB < ntB; ++itB) { @@ -245,6 +252,8 @@ static __global__ void mul_mat_f( if (nwarps > 1) { __syncthreads(); + } else { + ggml_cuda_syncwarp(); } #pragma unroll @@ -382,10 +391,12 @@ static __global__ void mul_mat_f_ids( tile_A A[ntA][warp_size / tile_A::J]; #pragma unroll for (int itA = 0; itA < ntA; ++itA) { + ggml_cuda_syncwarp(); #pragma unroll for (int i = 0; i < tile_A::I; ++i) { tile_xy[i*tile_k_padded + threadIdx.x] = x[(itA*tile_A::I + i)*stride_row + col]; } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_A::J) { load_ldmatrix(A[itA][k0/tile_A::J], tile_xy + k0, tile_k_padded); @@ -419,6 +430,7 @@ static __global__ void mul_mat_f_ids( int next_buf = 1; #pragma unroll for (int itB = 0; itB < ntB; ++itB) { + ggml_cuda_syncwarp(); #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { tile_xy[j0*tile_k_padded + threadIdx.x] = vals_buf[curr_buf][j0]; @@ -428,6 +440,7 @@ static __global__ void mul_mat_f_ids( gather_tile(itB + 1, vals_buf[next_buf]); } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) { tile_B B; @@ -472,6 +485,7 @@ static __global__ void mul_mat_f_ids( int next_buf = 1; #pragma unroll for (int itB = 0; itB < ntB; ++itB) { + ggml_cuda_syncwarp(); #pragma unroll for (int j0 = 0; j0 < tile_B::I; ++j0) { const float2 tmp = vals_buf[curr_buf][j0]; @@ -482,6 +496,7 @@ static __global__ void mul_mat_f_ids( gather_tile(itB + 1, vals_buf[next_buf]); } + ggml_cuda_syncwarp(); #pragma unroll for (int k0 = 0; k0 < warp_size; k0 += tile_B::J) { tile_B B; @@ -507,6 +522,8 @@ static __global__ void mul_mat_f_ids( if (nwarps > 1) { __syncthreads(); + } else { + ggml_cuda_syncwarp(); } #pragma unroll for (int itB = 0; itB < ntB; ++itB) { @@ -523,6 +540,8 @@ static __global__ void mul_mat_f_ids( if (nwarps > 1) { __syncthreads(); + } else { + ggml_cuda_syncwarp(); } #pragma unroll @@ -722,9 +741,8 @@ void mul_mat_f_cuda( sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream, ids_data); } break; - default: { + default: GGML_ABORT("fatal error"); - } break; } GGML_UNUSED_VARS(nchannels_y); @@ -826,9 +844,8 @@ static void mul_mat_f_switch_cols_per_block( stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data); } break; - default: { + default: GGML_ABORT("fatal error"); - } break; } } diff --git a/ggml/src/ggml-cuda/mmid.cu b/ggml/src/ggml-cuda/mmid.cu index 3c61e45..12ae14c 100644 --- a/ggml/src/ggml-cuda/mmid.cu +++ b/ggml/src/ggml-cuda/mmid.cu @@ -19,6 +19,11 @@ struct mm_ids_helper_store { }; static_assert(sizeof(mm_ids_helper_store) == 4, "unexpected size for mm_ids_helper_store"); +// the generic path passes 0, which needs no padding since it never groups lanes by token +template struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2<(n + 1)/2>::value; }; +template <> struct mm_ids_pow2<1> { static constexpr int value = 1; }; +template <> struct mm_ids_pow2<0> { static constexpr int value = 1; }; + // Helper function for mul_mat_id, converts ids to a more convenient format. // ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert. // ids_dst describes the same mapping but for the dst tensor. @@ -27,7 +32,7 @@ template __launch_bounds__(ggml_cuda_get_physical_warp_size(), 1) static __global__ void mm_ids_helper( const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, - const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1) { + const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template; const int expert = blockIdx.x; @@ -60,8 +65,9 @@ static __global__ void mm_ids_helper( } } else { // Implementation optimized for specific numbers of experts used: - static_assert(n_expert_used == 6 || warp_size % n_expert_used == 0, "bad n_expert_used"); - const int neu_padded = n_expert_used == 6 ? 8 : n_expert_used; // Padded to next higher power of 2. + // a warp holds a whole number of token slots, so the slot count is padded to a power of 2 + constexpr int neu_padded = mm_ids_pow2::value; + static_assert(neu_padded <= warp_size && warp_size % neu_padded == 0, "bad n_expert_used"); for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) { const int it = it0 + threadIdx.x / neu_padded; @@ -93,13 +99,19 @@ static __global__ void mm_ids_helper( } } nex_prev = warp_reduce_sum(nex_prev); + ggml_cuda_syncwarp(); for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) { const mm_ids_helper_store store_it = store[itc]; const int it = store_it.it(); const int iex_used = store_it.iex_used(); - ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y; - ids_dst [nex_prev + itc] = it*n_expert_used + iex_used; + ids_dst[nex_prev + itc] = it*n_expert_used + iex_used; + // ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup + if (write_inverse) { + ids_src1[it*n_expert_used + iex_used] = nex_prev + itc; + } else { + ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y; + } } if (threadIdx.x != 0) { @@ -118,7 +130,7 @@ static __global__ void mm_ids_helper( template static void launch_mm_ids_helper( const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, - const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, cudaStream_t stream) { + const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { GGML_ASSERT(n_tokens < (1 << 22) && "too few bits in mm_ids_helper_store"); GGML_ASSERT(n_expert_used_var < (1 << 10) && "too few bits in mm_ids_helper_store"); @@ -132,33 +144,36 @@ static void launch_mm_ids_helper( const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store); GGML_ASSERT(nbytes_shared <= smpbo); mm_ids_helper<<>> - (ids, ids_src1, ids_dst, expert_bounds, n_tokens, n_expert_used_var, nchannels_y, si1, sis1); + (ids, ids_src1, ids_dst, expert_bounds, n_tokens, n_expert_used_var, nchannels_y, si1, sis1, write_inverse); } void ggml_cuda_launch_mm_ids_helper( const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, - const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, cudaStream_t stream) { + const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { switch (n_expert_used) { case 2: - launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream); + launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); break; case 4: - launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream); + launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); break; case 6: - launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream); + launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); break; case 8: - launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream); + launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); + break; + case 10: + launch_mm_ids_helper<10>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); break; case 16: - launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream); + launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); break; case 32: - launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream); + launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); break; default: - launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, stream); + launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); break; } } diff --git a/ggml/src/ggml-cuda/mmid.cuh b/ggml/src/ggml-cuda/mmid.cuh index ac090ae..74c2db4 100644 --- a/ggml/src/ggml-cuda/mmid.cuh +++ b/ggml/src/ggml-cuda/mmid.cuh @@ -2,4 +2,4 @@ void ggml_cuda_launch_mm_ids_helper( const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds, - int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, cudaStream_t stream); + int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream); diff --git a/ggml/src/ggml-cuda/mmq-config-ampere.cuh b/ggml/src/ggml-cuda/mmq-config-ampere.cuh new file mode 100644 index 0000000..9f9fd19 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-ampere.cuh @@ -0,0 +1,383 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_ampere(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-blackwell.cuh b/ggml/src/ggml-cuda/mmq-config-blackwell.cuh new file mode 100644 index 0000000..9fbe32b --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-blackwell.cuh @@ -0,0 +1,37 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_blackwell(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false); + + return ggml_cuda_mmq_get_config_ampere(type, J, fallback); +} diff --git a/ggml/src/ggml-cuda/mmq-config-cdna.cuh b/ggml/src/ggml-cuda/mmq-config-cdna.cuh new file mode 100644 index 0000000..4a8d89f --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-cdna.cuh @@ -0,0 +1,185 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_cdna(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-gcn.cuh b/ggml/src/ggml-cuda/mmq-config-gcn.cuh new file mode 100644 index 0000000..24af2ef --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-gcn.cuh @@ -0,0 +1,281 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_gcn(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh b/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh new file mode 100644 index 0000000..83eb7c1 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh @@ -0,0 +1,273 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_pascal_dp4a(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh b/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh new file mode 100644 index 0000000..2a8dc9e --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh @@ -0,0 +1,273 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_pascal_older(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-rdna2.cuh b/ggml/src/ggml-cuda/mmq-config-rdna2.cuh new file mode 100644 index 0000000..8324d9e --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-rdna2.cuh @@ -0,0 +1,273 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna2(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh new file mode 100644 index 0000000..ead3825 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh @@ -0,0 +1,290 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3_5(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3.cuh new file mode 100644 index 0000000..3a3ef7b --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-rdna3.cuh @@ -0,0 +1,274 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 4, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 4, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 4, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 4, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 4, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-rdna4.cuh b/ggml/src/ggml-cuda/mmq-config-rdna4.cuh new file mode 100644 index 0000000..9293d9d --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-rdna4.cuh @@ -0,0 +1,290 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna4(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh new file mode 100644 index 0000000..4951db7 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -0,0 +1,1826 @@ +#pragma once + +#include "vecdotq.cuh" + +#include "mmq.cuh" + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q1_0( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int blocks_per_iter = MMQ_ITER_K / QK1_0; + constexpr int threads_per_row = blocks_per_iter * QI1_0; + constexpr int nrows = warp_size / threads_per_row; + constexpr int scale_entries_per_block = QK1_0 / QK8_1; + constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block; + + const int txi = threadIdx.x % threads_per_row; + const int kbx = txi / QI1_0; + const int kqsx = txi % QI1_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + kbx; + const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 2; + + const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0; +#pragma unroll + for (int j = 0; j < 2; ++j) { + const int q = qxi[j]; + +#if defined(GGML_USE_HIP) + const uint32_t q_bits = (uint16_t) q; + + // Expand each group of four Q1_0 bits into byte selectors for v_perm_b32. + const uint32_t q0_bits = q_bits >> 0; + const uint32_t q1_bits = q_bits >> 4; + const uint32_t q2_bits = q_bits >> 8; + const uint32_t q3_bits = q_bits >> 12; + + const uint32_t q0_indices = (q0_bits & 0x01) | ((q0_bits & 0x02) << 7) | ((q0_bits & 0x04) << 14) | ((q0_bits & 0x08) << 21); + const uint32_t q1_indices = (q1_bits & 0x01) | ((q1_bits & 0x02) << 7) | ((q1_bits & 0x04) << 14) | ((q1_bits & 0x08) << 21); + const uint32_t q2_indices = (q2_bits & 0x01) | ((q2_bits & 0x02) << 7) | ((q2_bits & 0x04) << 14) | ((q2_bits & 0x08) << 21); + const uint32_t q3_indices = (q3_bits & 0x01) | ((q3_bits & 0x02) << 7) | ((q3_bits & 0x04) << 14) | ((q3_bits & 0x08) << 21); + + const int v0 = ggml_hip_perm(0x000001FF, 0x000001FF, q0_indices); + const int v1 = ggml_hip_perm(0x000001FF, 0x000001FF, q1_indices); + const int v2 = ggml_hip_perm(0x000001FF, 0x000001FF, q2_indices); + const int v3 = ggml_hip_perm(0x000001FF, 0x000001FF, q3_indices); +#else + // unpack crumbs into nibble indices + const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13] + const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15] + // unpack nibbles into byte values + const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >> 0); + const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >> 0); + const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16); + const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16); + // unshuffle values + const int v0 = __byte_perm(s0, s1, 0x5410); + const int v1 = __byte_perm(s0, s1, 0x7632); + const int v2 = __byte_perm(s2, s3, 0x5410); + const int v3 = __byte_perm(s2, s3, 0x7632); +#endif // defined(GGML_USE_HIP) + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + dst_offset + j*4+0] = v0; + x_qs[i*sram_stride + dst_offset + j*4+1] = v1; + x_qs[i*sram_stride + dst_offset + j*4+2] = v2; + x_qs[i*sram_stride + dst_offset + j*4+3] = v3; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+0] = v0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+1] = v1; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+2] = v2; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+3] = v3; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } + + const int ksx = threadIdx.x % scale_entries_per_row; + const int scale_block = ksx / scale_entries_per_block; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps) { + int i = i0 + threadIdx.y; + + if (fallback) { + i = min(i, i_max); + } + + const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + scale_block; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + ksx] = bxi->d; +#else + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int blocks_per_iter = MMQ_ITER_K / QK2_0; + constexpr int threads_per_row = blocks_per_iter * QI2_0; + constexpr int nrows = warp_size / threads_per_row; + constexpr int scale_entries_per_block = QK2_0 / QK8_1; + constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block; + + const int txi = threadIdx.x % threads_per_row; + const int kbx = txi / QI2_0; + const int kqsx = txi % QI2_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + kbx; + const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 4; + + const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0; + +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int q = qxi[j]; + +#if defined(GGML_USE_HIP) + const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18); + const uint32_t qy_bits = q >> 8; + const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18); + const int qx = ggml_hip_perm(0x020100FF, 0x020100FF, qx_indices); + const int qy = ggml_hip_perm(0x020100FF, 0x020100FF, qy_indices); +#else + // unpack even and odd crumbs into byte values + const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); + const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2); + // unshuffle values + const int qx = __byte_perm(qe, qo, 0x5140); + const int qy = __byte_perm(qe, qo, 0x7362); +#endif // defined(GGML_USE_HIP) + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + dst_offset + j*2+0] = qx; + x_qs[i*sram_stride + dst_offset + j*2+1] = qy; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+0] = qx; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+1] = qy; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } + + const int ksx = threadIdx.x % scale_entries_per_row; + const int scale_block = ksx / scale_entries_per_block; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps) { + int i = i0 + threadIdx.y; + + if (fallback) { + i = min(i, i_max); + } + + const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + scale_block; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + ksx] = bxi->d; +#else + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_0); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI4_0; + const int kqsx = txi % QI4_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbx; + const int qs0 = get_int_b2(bxi->qs, kqsx); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + 0] = __vsub4((qs0 >> 0) & 0x0F0F0F0F, 0x08080808); + x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + QI4_0] = __vsub4((qs0 >> 4) & 0x0F0F0F0F, 0x08080808); +#else + x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_0; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = bxi->d; +#else + x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + kbxd] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_1( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_1); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI4_1; + const int kqsx = txi % QI4_1; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbx; + const int qs0 = get_int_b4(bxi->qs, kqsx); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + 0] = (qs0 >> 0) & 0x0F0F0F0F; + x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + QI4_1] = (qs0 >> 4) & 0x0F0F0F0F; +#else + x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_1; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_dm[i*sram_stride + kbxd] = bxi->dm; +#else + x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + kbxd] = bxi->dm; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_0( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_0); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI5_0; + const int kqsx = txi % QI5_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbx; + + const int ql = get_int_b2(bxi->qs, kqsx); + const int qh = get_int_b2(bxi->qh, 0) >> (4 * kqsx); + + int qs0 = (ql >> 0) & 0x0F0F0F0F; + qs0 |= (qh << 4) & 0x00000010; // 0 -> 4 + qs0 |= (qh << 11) & 0x00001000; // 1 -> 12 + qs0 |= (qh << 18) & 0x00100000; // 2 -> 20 + qs0 |= (qh << 25) & 0x10000000; // 3 -> 28 + qs0 = __vsub4(qs0, 0x10101010); // subtract 16 + + int qs1 = (ql >> 4) & 0x0F0F0F0F; + qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4 + qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12 + qs1 |= (qh << 2) & 0x00100000; // 18 -> 20 + qs1 |= (qh << 9) & 0x10000000; // 19 -> 28 + qs1 = __vsub4(qs1, 0x10101010); // subtract 16 + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + 0] = qs0; + x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + 0] = qs0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_0; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = bxi->d; +#else + x_df[i*(MMQ_TILE_NE_K/QI5_0) + i/QI5_0 + kbxd] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_1( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_1); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI5_1; + const int kqsx = txi % QI5_1; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbx; + + const int ql = get_int_b4(bxi->qs, kqsx); + const int qh = get_int_b4(bxi->qh, 0) >> (4 * kqsx); + + int qs0 = (ql >> 0) & 0x0F0F0F0F; + qs0 |= (qh << 4) & 0x00000010; // 0 -> 4 + qs0 |= (qh << 11) & 0x00001000; // 1 -> 12 + qs0 |= (qh << 18) & 0x00100000; // 2 -> 20 + qs0 |= (qh << 25) & 0x10000000; // 3 -> 28 + + int qs1 = (ql >> 4) & 0x0F0F0F0F; + qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4 + qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12 + qs1 |= (qh << 2) & 0x00100000; // 18 -> 20 + qs1 |= (qh << 9) & 0x10000000; // 19 -> 28 + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + 0] = qs0; + x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + 0] = qs0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_1; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_dm[i*sram_stride + kbxd] = bxi->dm; +#else + x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + kbxd] = bxi->dm; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + // MMQ_ITER_K / (4 * QR8_0) == 64 required. but NV has only 32 threads per warp + constexpr int threads_per_row = 32; + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI8_0; + const int kqsx = txi % QI8_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q8_0 * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride) + kbx; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 0 + txi] = get_int_b2(bxi[0].qs, kqsx); + x_qs[i*sram_stride + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx); +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 0 + txi] = get_int_b2(bxi[0].qs, kqsx); + x_qs[i*(2*MMQ_TILE_NE_K + 1) + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q8_0 * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride) + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = bxi->d; +#else + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + kbxd] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +// --------------------------------------------------------------------------------------------- + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR2_K); + constexpr int nrows = ggml_cuda_get_physical_warp_size() / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q2_K * bxi = (const block_q2_K *) x + kbx0 + i*stride; + + const int x_ql_0 = get_int_b2(bxi->qs, kqsx); + +#pragma unroll + for (int l = 0; l < QR2_K; ++l) { + const int k = (kqsx/8)*32 + l*8 + kqsx % 8; + + const int x_qs_k = (x_ql_0 >> (2*l)) & 0x03030303; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k] = x_qs_k; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int sc_m = bxi->scales[kqsx]; +#ifdef FAST_FP16_AVAILABLE + const half2 x_dm_ik = __hmul2(bxi->dm, make_half2(sc_m & 0x0F, sc_m >> 4)); +#else + const float2 bxi_dmf = __half22float2(bxi->dm); + const half2 x_dm_ik = make_half2(bxi_dmf.x*(sc_m & 0x0F), bxi_dmf.y*(sc_m >> 4)); +#endif // FAST_FP16_AVAILABLE + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_dm[i*sram_stride + kqsx] = x_dm_ik; +#else + x_dm[i*(MMQ_TILE_NE_K + 1) + kqsx] = x_dm_ik; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); + int * x_sc = (int *) (x_df + txs.dm); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR3_K); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride; + + const int x_ql_0 = get_int_b2(bxi->qs, kqsx); + const int x_qh_0 = get_int_b2(bxi->hmask, kqsx % (QI3_K/2)) >> (4 * (kqsx / (QI3_K/2))); + +#pragma unroll + for (int l = 0; l < QR3_K; ++l) { + const int k = (kqsx/8)*32 + l*8 + kqsx % 8; + + const int x_ql_k = (x_ql_0 >> (2*l)) & 0x03030303; + const int x_qh_k = ((x_qh_0 >> l) << 2) & 0x04040404; + + const int x_qs_k = __vsub4(x_ql_k | x_qh_k, 0x04040404); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k] = x_qs_k; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } + + constexpr int rows_per_warp = warp_size / 4; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { + int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/4; + + if (fallback) { + i = min(i, i_max); + } + + const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride; + + const int ksc = threadIdx.x % 4; + + const int ksc_low = ksc % (QI3_K/8); + const int shift_low = 4 * (ksc / (QI3_K/8)); + const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F; + + const int ksc_high = QI3_K/8; + const int shift_high = 2 * ksc; + const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030; + + const int sc = __vsub4(sc_low | sc_high, 0x20202020); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + const int8_t * sc8 = (const int8_t *) ≻ + const float d = bxi->d; + +#pragma unroll + for (int l = 0; l < int(sizeof(int)); ++l) { + x_df[i*sram_stride + sizeof(int)*ksc + l] = d*sc8[l]; + } +#else + x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = sc; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + +#if !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)) +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) { + int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride; + + x_df[i] = bxi->d; + } +#endif // !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)) || defined(AMD_WMMA_AVAILABLE) +} + +static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, const int ksc) { + // scale arrangement after the following two lines: + // - ksc == 0: sc0, sc1, sc2, sc3 + // - ksc == 1: sc4, sc5, sc6, sc7 + // - ksc == 2: m0, m1, m2, m3 + // - ksc == 3: m4, m5, m6, m7 + return ((scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F) | // lower 4 bits + ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); + int * x_sc = (int *) (x_dm + txs.dm); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_K); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + const int qs0 = get_int_b4(bxi->qs, txi); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F; + x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 8] = (qs0 >> 4) & 0x0F0F0F0F; +#else + x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int rows_per_warp = warp_size / 2; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + // Need if on AMD instead of % because warp_size == 64 + // This causes double work and throughput loss (MI300X) + // H100 loses about 100 t/s with 'if' condition over '%' + int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2; + if (i < I) { +#else + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I; + { +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + if (fallback) { + i = min(i, i_max); + } + + const block_q4_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + + const int * scales = (const int *) bxi->scales; + const int ksc = threadIdx.x % 2; + + const int sc32 = unpack_scales_q45_K(scales, ksc + 0); + const int m32 = unpack_scales_q45_K(scales, ksc + 2); + + const uint8_t * sc8 = (const uint8_t *) &sc32; + const uint8_t * m8 = (const uint8_t *) &m32; + + const half2 dm = bxi->dm * make_half2(1.0f, -1.0f); + + #pragma unroll + for (int l = 0; l < int(sizeof(int)); ++l) { + x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]); + } + } + } +#else +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) { + int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + + x_dm[i] = bxi->dm; + } + constexpr int rows_per_warp = warp_size / 4; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q4_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride) + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8); + + const int * scales = (const int *) bxi->scales; + + const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8); + const int scales8 = unpack_scales_q45_K(scales, ksc); + + x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8; + } +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I); + int * x_qs = (int *) x_tile; + half2 * x_dm = (half2 *) (x_qs + txs.qs); + int * x_sc = (int *) (x_dm + txs.dm); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_K); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + const int ky = QR5_K*txi; + + const int ql = get_int_b4(bxi->qs, txi); + const int ql0 = (ql >> 0) & 0x0F0F0F0F; + const int ql1 = (ql >> 4) & 0x0F0F0F0F; + + const int qh = get_int_b4(bxi->qh, txi % (QI5_K/4)); + const int qh0 = ((qh >> (2 * (txi / (QI5_K/4)) + 0)) << 4) & 0x10101010; + const int qh1 = ((qh >> (2 * (txi / (QI5_K/4)) + 1)) << 4) & 0x10101010; + + const int kq0 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + 0; + const int kq1 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + QI5_K/4; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kq0] = ql0 | qh0; + x_qs[i*sram_stride + kq1] = ql1 | qh1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = ql0 | qh0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = ql1 | qh1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int rows_per_warp = warp_size / 2; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { +#if defined(AMD_MFMA_AVAILABLE) + // Need if on AMD instead of % because warp_size == 64 + // This causes double work and throughput loss (MI300X) + // H100 loses about 100 t/s with 'if' condition over '%' + int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2; + if (i < I) { +#else + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I; + { +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + if (fallback) { + i = min(i, i_max); + } + + const block_q5_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + + const int * scales = (const int *) bxi->scales; + const int ksc = threadIdx.x % 2; + + const int sc32 = unpack_scales_q45_K(scales, ksc + 0); + const int m32 = unpack_scales_q45_K(scales, ksc + 2); + + const uint8_t * sc8 = (const uint8_t *) &sc32; + const uint8_t * m8 = (const uint8_t *) &m32; + + const half2 dm = bxi->dm * make_half2(1.0f, -1.0f); + +#pragma unroll + for (int l = 0; l < int(sizeof(int)); ++l) { + x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]); + } + } + } +#else +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) { + int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + + x_dm[i] = bxi->dm; + } + + constexpr int rows_per_warp = warp_size / 4; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q5_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + + const int * scales = (const int *) bxi->scales; + + const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8); + const int scales8 = unpack_scales_q45_K(scales, ksc); + + x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8; + } +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); + int * x_sc = (int *) (x_df + MMQ_TILE_NE_K/QI6_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); + int * x_sc = (int *) (x_df + txs.dm); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR6_K); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q6_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + + const int ql = get_int_b2(bxi->ql, txi); + const int ql0 = (ql >> 0) & 0x0F0F0F0F; + const int ql1 = (ql >> 4) & 0x0F0F0F0F; + + const int qh = get_int_b2(bxi->qh, (QI6_K/4) * (txi / (QI6_K/2)) + txi % (QI6_K/4)); + const int qh0 = ((qh >> ((txi & 0x08) >> 2)) << 4) & 0x30303030; + const int qh1 = (qh >> ((txi & 0x08) >> 2)) & 0x30303030; + + const int kq0 = 2*txi - txi % (QI6_K/2) + 0; + const int kq1 = 2*txi - txi % (QI6_K/2) + QI6_K/2; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kq0] = __vsub4(ql0 | qh0, 0x20202020); + x_qs[i*sram_stride + kq1] = __vsub4(ql1 | qh1, 0x20202020); +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = __vsub4(ql0 | qh0, 0x20202020); + x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = __vsub4(ql1 | qh1, 0x20202020); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) { + int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q6_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride] = bxi->d; +#else + x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int rows_per_warp = warp_size / 4; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) { + int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I; + + if (fallback) { + i = min(i, i_max); + } + + const block_q6_K * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride) + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_sc[i*sram_stride + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8)); +#else + x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + threadIdx.x%(MMQ_TILE_NE_K/8)] = get_int_b2(bxi->scales, threadIdx.x%(QI6_K/8)); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +// --------------------------------------------------------------------------------------------- + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_s( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I); + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_S); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq1_s * bxi = (const block_iq1_s *) x + kbx0 + i*stride; + + const int qs_packed = get_int_b2(bxi->qs, kqsx); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bxi->qh[kqsx]; + + #pragma unroll + for (int l = 0; l < QR1_S/2; ++l) { + const int grid = iq1s_grid_gpu[qs[l] | (((qh >> (3*l)) & 0x07) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const float d1q = __half2float(bxi->d) * (((qh >> 11) & 0x0E) + 1); + const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_ds[i*sram_stride + kqsx] = make_half2(d1q, d1q*delta); +#else + x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XXS, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XXS)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq2_xxs * bxi = (const block_iq2_xxs *) x + kbx0 + i*stride; + + const int q2 = get_int_b2(bxi->qs, 2*kqsx+0); + const uint8_t * aux8 = (const uint8_t *) &q2; + const uint32_t aux32 = get_int_b2(bxi->qs, 2*kqsx+1); + +#pragma unroll + for (int l = 0; l < QR2_XXS; ++l) { + const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[l]]; + const uint32_t signs = unpack_ksigns(aux32 >> (7 * l)); + + const int signs0 = __vcmpne4(signs & 0x08040201, 0); + const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0); + + const int signs1 = __vcmpne4(signs & 0x80402010, 0); + const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0; + x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = aux32 >> 27 | 1; // (scale * 2 + 1) + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4 +#else + x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4 +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xs( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XS, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XS)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq2_xs * bxi = (const block_iq2_xs *) x + kbx0 + i*stride; + + const int2 q2_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1)); + const uint16_t * q2 = (const uint16_t *) &q2_packed; + + #pragma unroll + for (int l = 0; l < QR2_XS; ++l) { + const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l] & 0x1FF]; + const uint32_t signs = unpack_ksigns(q2[l] >> 9); + + const int signs0 = __vcmpne4(signs & 0x08040201, 0); + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + + const int signs1 = __vcmpne4(signs & 0x80402010, 0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = bxi->scales[kqsx]; + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; + x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; +#else + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_s( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_S, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_S)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq2_s * bxi = (const block_iq2_s *) x + kbx0 + i*stride; + + const int qs_packed = get_int_b2(bxi->qs, kqsx); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bxi->qh[kqsx]; + + const int signs_packed_32 = get_int_b2(bxi->qs, QK_K/32 + kqsx); + const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; + +#pragma unroll + for (int l = 0; l < QR2_S; ++l) { + const int * grid_pos = (const int *)(iq2s_grid + (qs[l] | ((qh << (8-2*l)) & 0x300))); + + const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000); + const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000); + + const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0); + const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = bxi->scales[kqsx]; + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; + x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; +#else + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_xxs( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_XXS, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_XXS)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq3_xxs * bxi = (const block_iq3_xxs *) x + kbx0 + i*stride; + + const int2 q3_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1)); + const uint8_t * q3 = (const uint8_t *) &q3_packed; + const uint32_t aux32 = get_int_b2(bxi->qs, QK_K/16 + kqsx); + +#pragma unroll + for (int l = 0; l < QR3_XXS; ++l) { + const int2 grid_pos = make_int2(iq3xxs_grid[q3[2*l+0]], iq3xxs_grid[q3[2*l+1]]); + const uint32_t signs = unpack_ksigns(aux32 >> (7*l)); + + const int signs0 = __vcmpne4(signs & 0x08040201, 0); + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + + const int signs1 = __vcmpne4(signs & 0x80402010, 0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = aux32 >> 28; + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kqsx] = (ls*d + d/2)/2; +#else + x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = (ls*d + d/2)/2; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_s( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_S)) / 2; + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq3_s * bxi = (const block_iq3_s *) x + kbx0 + i*stride; + + const int2 qs_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1)); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bxi->qh[kqsx]; + + const int signs_packed_32 = get_int_b2(bxi->signs, kqsx); + const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; + +#pragma unroll + for (int l = 0; l < QR3_S; ++l) { + const int2 grid_pos = make_int2( + iq3s_grid[qs[2*l+0] | ((qh << (8 - 2*l)) & 0x100)], + iq3s_grid[qs[2*l+1] | ((qh << (7 - 2*l)) & 0x100)]); + + const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000); + const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000); + + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid_l; + x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid_h; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid_l; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid_h; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const int ls = 1 + 2*((bxi->scales[kqsx/2] >> (((2*kqsx) << 1) & 0x04)) & 0x0F); + const float d = bxi->d; +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kqsx] = ls*d; +#else + x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = ls*d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_XS, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_XS); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_iq4_xs * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + + const int aux_q4 = get_int_b4(bxi->qs, kqsx); + const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl); + const int k0 = 8 * (kqsx / 4) + kqsx % 4; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k0 + 0] = v.x; + x_qs[i*sram_stride + k0 + 4] = v.y; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 4] = v.y; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int rows_per_warp = warp_size / 8; +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / (MMQ_TILE_NE_K/4); + + if (fallback) { + i = min(i, i_max); + } + + const block_iq4_xs * bxi = ggml_cuda_mmq_x_row(x, x_gate, kbx0, i, stride); + + const float d = __half2float(bxi->d); + + const int ls = ((bxi->scales_l[(threadIdx.x % 8)/2] >> (4*(threadIdx.x % 2))) & 0x0F) + | (((bxi->scales_h >> (2*(threadIdx.x % 8))) & 0x03) << 4); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + threadIdx.x % 8] = d * (ls - 32); +#else + x_df[i*(MMQ_TILE_NE_K/4) + i/4 + threadIdx.x % 8] = d * (ls - 32); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_nl( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_NL, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_NL); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI4_NL; + const int kqsx = txi % QI4_NL; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbx; + + const int aux_q4 = get_int_b2(bxi->qs, kqsx); + const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl); + const int k0 = kbx * (2 * QI4_NL) + kqsx; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k0 + 0] = v.x; + x_qs[i*sram_stride + k0 + QI4_NL] = v.y; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI4_NL] = v.y; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_NL; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = __half2float(bxi->d); +#else + x_df[i*(MMQ_TILE_NE_K/QI4_NL) + i/QI4_NL + kbxd] = __half2float(bxi->d); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +// --------------------------------------------------------------------------------------------- + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_MXFP4, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR_MXFP4); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI_MXFP4; + const int kqsx = txi % QI_MXFP4; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbx; + + const int aux_q4 = get_int_b1(bxi->qs, kqsx); + const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4); + const int k0 = kbx * (2 * QI_MXFP4) + kqsx; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + k0 + 0] = v.x; + x_qs[i*sram_stride + k0 + QI_MXFP4] = v.y; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI_MXFP4] = v.y; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI_MXFP4; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f; +#else + x_df[i*(MMQ_TILE_NE_K/QI_MXFP4) + i/QI_MXFP4 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4_fp4( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4); + + int * x_qs = (int *) x_tile; + uint32_t * x_sc = (uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K); + + const int txi = threadIdx.x; + + constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback, GGML_PREC_Q4); + + constexpr int threads_per_row = iter_k / QK_MXFP4; // each thread processes 1 block + constexpr int rows_per_warp = warp_size / threads_per_row; + const int kbx = txi % threads_per_row; + const int row_in_warp = txi / threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) { + int i = i0 + threadIdx.y * rows_per_warp + row_in_warp; + + if constexpr (fallback) { + i = min(i, i_max); + } + + const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i * stride + kbx; + + // quantize_mxfp4_mmq permutes nibbles to match the quantized format + const int k0 = kbx * 4; + memcpy(x_qs + i*sram_stride + k0, bxi->qs, 16); + + // Load E8M0 scales: pack 2 consecutive scales into one uint32 + if (kbx % 2 == 0) { + uint32_t e = bxi->e; + e |= ((bxi + 1)->e << 8); + x_sc[i*sram_stride + kbx / 2] = e; + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_NVFP4, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / QK_NVFP4; + constexpr int rows_per_warp = warp_size / threads_per_row; + const int kbx = threadIdx.x % threads_per_row; + const int row_in_warp = threadIdx.x / threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) { + int i = i0 + threadIdx.y * rows_per_warp + row_in_warp; + + if constexpr (fallback) { + i = min(i, i_max); + } + + const block_nvfp4 * bxi = (const block_nvfp4 *) x + kb0 + i * stride + kbx; + const uint32_t * __restrict__ src_qs = reinterpret_cast(bxi->qs); + const int kqs = 16 * kbx; + const int ksc = 4 * kbx; + +#pragma unroll + for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) { + const int2 q0 = get_int_from_table_16(src_qs[2 * sub + 0], kvalues_mxfp4); + const int2 q1 = get_int_from_table_16(src_qs[2 * sub + 1], kvalues_mxfp4); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + kqs + 4 * sub + 0] = q0.x; + x_qs[i*sram_stride + kqs + 4 * sub + 1] = q1.x; + x_qs[i*sram_stride + kqs + 4 * sub + 2] = q0.y; + x_qs[i*sram_stride + kqs + 4 * sub + 3] = q1.y; + x_df[i*sram_stride + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]); +#else + x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 0] = q0.x; + x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 1] = q1.x; + x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 2] = q0.y; + x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 3] = q1.y; + x_df[i * (2 * MMQ_TILE_NE_K * 2 / QI_NVFP4) + i / (QK_NVFP4_SUB / QI_NVFP4) + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4); + constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback, GGML_PREC_Q4); + constexpr int threads_per_row = iter_k / QK_NVFP4; // each thread processes 1 block + constexpr int rows_per_warp = warp_size / threads_per_row; + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4); + + uint32_t * x_u32 = (uint32_t *) x_tile; + + const int txi = threadIdx.x; + const int kbx = txi % threads_per_row; + const int row_in_warp = txi / threads_per_row; + + const block_nvfp4 * bxi_base = (const block_nvfp4 *) x + kbx0 + kbx; + uint32_t * x_u32_scale = x_u32 + 64 + kbx; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) { + int i = i0 + threadIdx.y * rows_per_warp + row_in_warp; + + if constexpr (fallback) { + i = min(i, i_max); + } + + const block_nvfp4 * bxi = bxi_base + i * stride; + + const uint32_t * src_qs = reinterpret_cast(bxi->qs); + +#pragma unroll + for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) { + x_u32[i*sram_stride + 8*kbx + 2 * sub + 0] = src_qs[2 * sub + 0]; + x_u32[i*sram_stride + 8*kbx + 2 * sub + 1] = src_qs[2 * sub + 1]; + } + + x_u32_scale[i*sram_stride] = get_int_b4(bxi->d, 0); + } +} diff --git a/ggml/src/ggml-cuda/mmq-vec-dot.cuh b/ggml/src/ggml-cuda/mmq-vec-dot.cuh new file mode 100644 index 0000000..a75dd04 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-vec-dot.cuh @@ -0,0 +1,1237 @@ +#pragma once + +#include "vecdotq.cuh" +#include "mma.cuh" + +using namespace ggml_cuda_mma; + +#include "mmq.cuh" + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_0*VDR_Q4_0_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2); + + int u[2*VDR_Q4_0_Q8_1_MMQ]; + + constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes(); + constexpr int mcpy_int = max_cpy / sizeof(int); + static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ"); + + int tmp0[4], tmp1[4]; + + #pragma unroll + for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) { + ggml_cuda_memcpy_1(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] ); + ggml_cuda_memcpy_1(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_0 + l0 * mcpy_int]); + } + + u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3]; + u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3]; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_0_q8_1_impl + (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_0], u, + x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + k0/(QR4_0*QI4_0)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_1*VDR_Q4_1_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2); + + int u[2*VDR_Q4_1_Q8_1_MMQ]; + + constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes(); + constexpr int mcpy_int = max_cpy / sizeof(int); + static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ"); + + int tmp0[4], tmp1[4]; + + #pragma unroll + for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) { + ggml_cuda_memcpy_1(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] ); + ggml_cuda_memcpy_1(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_1 + l0 * mcpy_int]); + } + + u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3]; + u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3]; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_1_q8_1_impl + (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_1], u, + x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + k0/(QR4_1*QI4_1)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_q8_1_impl + (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k0 % MMQ_TILE_NE_K], + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + k0/QI8_0], y_df[j*MMQ_TILE_Y_K + (k0/QI8_1) % (MMQ_TILE_NE_K/QI8_1)]); + } + } + } +} + +template +static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 8, int, input_layout> tile_A; + typedef tile<16, 8, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + const half2 * y_ds = (const half2 *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + float dB; + const int j = j0 + tile_C::get_j(0); + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + } else { + dB = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_A::I + tile_C::get_i(l); + const float dA = x_df[i*sram_stride + k0/QI8_0]; + sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA*dB; + } + } + } + } +#else + typedef tile<16, 8, int> tile_A; + typedef tile< 8, 8, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + const half2 * y_ds = (const half2 *) y; + + tile_A A[ntx][MMQ_TILE_NE_K/QI8_0]; + float dA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_0]; + + const int i0 = (threadIdx.y/ntx)*rows_per_warp; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + const int k0 = k00 + k01; + + load_ldmatrix(A[n][k01/QI8_0], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_A::I + tile_C::get_i(2*l); + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + const int k0 = k00 + k01; + + dA[n][l][k01/QI8_0] = x_df[i*sram_stride + k0/QI8_0]; + } + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + tile_B B; + float dB[tile_C::ne/2]; + + load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + } else { + dB[l] = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n][k01/QI8_0], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA[n][l/2][k01/QI8_0]*dB[l%2]; + } + } + } + } +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +} + + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_1_q8_1_impl + (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], + x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + k0/QI8_1], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 8, int, input_layout> tile_A; + typedef tile<16, 8, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K; + const int * y_qs = (const int *) y + 4; + const half2 * y_dm = (const half2 *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + const int j = j0 + tile_C::get_j(0); + const float2 dsB = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]); + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_A::I + tile_C::get_i(l); + float2 dmA = __half22float2(x_dm[i*sram_stride + k0/QI8_1]); + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.x*dsB.x*C.x[l]; + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.y*dsB.y; + } + } + } + } +#else + typedef tile<16, 8, int> tile_A; + typedef tile< 8, 8, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K; + const int * y_qs = (const int *) y + 4; + const half2 * y_dm = (const half2 *) y; + + tile_A A[ntx][MMQ_TILE_NE_K/QI8_1]; + float2 dmA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_1]; + + const int i0 = (threadIdx.y/ntx)*rows_per_warp; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + const int k0 = k00 + k01; + + load_ldmatrix(A[n][k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_A::I + tile_C::get_i(2*l); + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + const int k0 = k00 + k01; + + dmA[n][l][k01/QI8_1] = __half22float2(x_dm[i*sram_stride + k0/QI8_1]); + } + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + tile_B B; + float2 dsB[tile_C::ne/2]; + + load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + dsB[l] = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n][k01/QI8_1], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].x*dsB[l%2].x*C.x[l]; + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].y*dsB[l%2].y; + } + } + } + } +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +} + +// Used for NVFP4, Q3_K, IQ2_S, and IQ2_XS +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_16_q8_1_impl( + &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], + &y_qs[j*MMQ_TILE_Y_K + k01], + &x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + k0/(QI8_0/2)], + y_df[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +// Used for Q3_K, IQ2_S, and IQ2_XS: +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 4, int, input_layout> tile_A; + typedef tile<16, 4, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + const int j = j0 + tile_C::get_j(0); + const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(l); + sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*sram_stride + k0/4] * dB; + } + } + } + } +#elif defined(TURING_MMA_AVAILABLE) + + typedef tile<16, 4, int> tile_A; + typedef tile<16, 8, int> tile_A_8; + typedef tile< 8, 4, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + + const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I); + + tile_A A[ntx][8]; + float dA[ntx][tile_C::ne/2][8]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) { + const int k0 = k00 + k01; + + load_ldmatrix(((tile_A_8 *) A[n])[k01/8], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { + const int k0 = k00 + k01; + + dA[n][l][k01/4] = x_df[i*sram_stride + k0/4]; + } + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) { + tile_B B[2]; + float dB[tile_C::ne/2]; + + // Here load_generic is faster than load_ldmatrix. + load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K); + load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K); + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C[2]; + mma(C[0], A[n][k01/4 + 0], B[0]); + mma(C[1], A[n][k01/4 + 1], B[1]); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] += dB[l%2]*(C[0].x[l]*dA[n][l/2][k01/4 + 0] + C[1].x[l]*dA[n][l/2][k01/4 + 1]); + } + } + } + } +#else + GGML_UNUSED_VARS(x, y, sum, k00); + NO_DEVICE_CODE; +#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + + float2 y_df[J/nwarps]; +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + y_df[j0/nwarps] = __half22float2(y_ds[j*MMQ_TILE_Y_K]); + } + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K/2; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + constexpr int ns = 2; + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq( + &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], + &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y, + &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]); + } + } + } + + // Some compilers fail to unroll the loop over k01 if there is a conditional statement for ns in the inner loop. + // As a workaround 2 separate loops are used instead. +#pragma unroll + for (int k01 = MMQ_TILE_NE_K/2; k01 < MMQ_TILE_NE_K; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + constexpr int ns = 1; + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq( + &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], + &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y, + &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 4, int, input_layout> tile_A; + typedef tile<16, 4, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + const int j = j0 + tile_C::get_j(0); + const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y; + const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0 + : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y + : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x); + + tile_C Cm; + if (k01 >= MMQ_TILE_NE_K * 3/4) { + tile_A A1; +#pragma unroll + for (int l = 0; l < tile_A::ne; ++l) { + A1.x[l] = 0x01010101; + } + mma(Cm, A1, B); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C Cd; + mma(Cd, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(l); + const float2 dm = __half22float2(x_dm[i*sram_stride + k0/4]); + float tmp = Cd.x[l]*dm.x; + if (k01 >= MMQ_TILE_NE_K * 3/4) { + tmp -= Cm.x[l]*dm.y; + } + sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB; + sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB; + } + } + } + } +#elif defined(TURING_MMA_AVAILABLE) + + typedef tile<16, 4, int> tile_A; + typedef tile<16, 8, int> tile_A_8; + typedef tile< 8, 4, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + + const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I); + + tile_A A[ntx][8]; + float dA[ntx][tile_C::ne/2][8]; + float mA[ntx][tile_C::ne/2][8]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + const int k0 = k00 + k01; + + load_ldmatrix(((tile_A_8 *) A[n])[k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1/2) { + const int k0 = k00 + k01; + + const float2 dm = __half22float2(x_dm[i*sram_stride + k0/(QI8_1/2)]); + + dA[n][l][k01/(QI8_1/2)] = dm.x; + mA[n][l][k01/(QI8_1/2)] = dm.y; + } + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + float2 dB[tile_C::ne/2]; + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + dB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K]); + } + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { + tile_B B[2]; + + // Here load_generic is faster than load_ldmatrix. + load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K); + load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K); + + tile_C Cm[2]; + if (k01 >= MMQ_TILE_NE_K * 3/4) { + tile_A A1; + A1.x[0] = 0x01010101; + A1.x[1] = 0x01010101; + mma(Cm[0], A1, B[0]); + mma(Cm[1], A1, B[1]); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C Cd[2]; + + mma(Cd[0], A[n][k01/4 + 0], B[0]); + mma(Cd[1], A[n][k01/4 + 1], B[1]); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + float tmp = Cd[0].x[l]*dA[n][l/2][k01/4 + 0] + Cd[1].x[l]*dA[n][l/2][k01/4 + 1]; + if (k01 >= MMQ_TILE_NE_K * 3/4) { + tmp -= Cm[0].x[l]*mA[n][l/2][k01/4 + 0] + Cm[1].x[l]*mA[n][l/2][k01/4 + 1]; + } + sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*(k01 < MMQ_TILE_NE_K/2 ? dB[l%2].x : dB[l%2].y); + } + } + } + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K * 3/4; k01 += QI8_1) { + float2 sB[tile_C::ne/2]; + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + sB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]); + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 0]*sB[l%2].x; + sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 1]*sB[l%2].y; + } + } + } + } +#else + GGML_UNUSED_VARS(x, y, sum, k00); + NO_DEVICE_CODE; +#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * x_sc = (const int *) x_df + txs.dm; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + const int8_t * scales = ((const int8_t *) (x_sc + i*(MMQ_TILE_NE_K/8) + i/8)) + k0/4; + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q3_K_q8_1_impl_mmq( + &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], scales, + x_df[i], y_df[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * x_sc = (const int *) x_dm + txs.dm; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_K*VDR_Q4_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + const uint8_t * sc = (const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/32] + 2*(k01/16); + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_K_q8_1_impl_mmq( + &x_qs[i*(MMQ_TILE_NE_K + 1) + k0/2], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8, + x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I); + const int * x_qs = (const int *) x; + const half2 * x_dm = (const half2 *) x_qs + txs.qs; + const int * x_sc = (const int *) x_dm + txs.dm; + const int * y_qs = (const int *) y + 4; + const half2 * y_ds = (const half2 *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR5_K*VDR_Q5_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + const uint8_t * sc = ((const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k00/32]) + 2*(k01/16); + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q5_K_q8_1_impl_mmq( + &x_qs[i*(QR5_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8, + x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I); + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + txs.qs; + const int * x_sc = (const int *) x_df + txs.dm; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + +// #pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR6_K*VDR_Q6_K_Q8_1_MMQ) { + const int k0 = k00 + k01; + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += warp_size) { + const int i = i0 + threadIdx.x; + + const int8_t * sc = ((const int8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/16]); + + sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q6_K_q8_1_impl_mmq( + &x_qs[i*(QR6_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, + x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K], &y_df[j*MMQ_TILE_Y_K + k01/QI8_1]); + } + } + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr data_layout input_layout = get_input_data_layout(); + typedef tile<16, 4, int, input_layout> tile_A; + typedef tile<16, 4, int, input_layout> tile_B; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; + const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { + const int k0 = k00 + k01; + + tile_A A[ntx]; +#pragma unroll + for (int n = 0; n < ntx; ++n) { + load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride); + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + tile_B B; + load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); + + const int j = j0 + tile_C::get_j(0); + const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C; + mma(C, A[n], B); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(l); + const int8_t * sc = (const int8_t *) (x_sc + i*sram_stride + k00/16); + sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*sram_stride] * dB; + } + } + } + } +#elif defined(TURING_MMA_AVAILABLE) + + typedef tile<16, 4, int> tile_A; + typedef tile< 8, 4, int> tile_B; + typedef tile<16, 8, int> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + + y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; + const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K; + const int * y_qs = (const int *) y + 4; + const float * y_df = (const float *) y; + + const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I); + + tile_A A[ntx][8]; + int scA[ntx][tile_C::ne/2][8]; + float dA[ntx][tile_C::ne/2]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) { + const int k0 = k00 + k01; + + load_ldmatrix(A[n][k01/4 + 0], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + 0), sram_stride); + load_ldmatrix(A[n][k01/4 + 1], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + tile_A::J), sram_stride); + } + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 16) { + const int k0 = k00 + k01; + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); + + const int sc_packed = x_sc[i*sram_stride + k0/16]; + const int8_t * sc = (const int8_t *) &sc_packed; + +#pragma unroll + for (int ksc = 0; ksc < sizeof(int); ++ksc) { + scA[n][l][k01/4 + ksc] = sc[ksc]; + } + } + } + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); + + dA[n][l] = x_df[i*sram_stride]; + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + float tmp[ntx][tile_C::ne] = {{0.0f}}; + +#pragma unroll + for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) { + tile_B B[2]; + float dB[tile_C::ne/2]; + + // Here load_generic is faster than load_ldmatrix. + load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + 0 + k01, MMQ_TILE_Y_K); + load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + tile_B::J + k01, MMQ_TILE_Y_K); + +#pragma unroll + for (int l = 0; l < tile_C::ne/2; ++l) { + const int j = j0 + tile_C::get_j(l); + + dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + tile_C C[2]; + mma(C[0], A[n][k01/4 + 0], B[0]); + mma(C[1], A[n][k01/4 + 1], B[1]); + +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + tmp[n][l] += (C[0].x[l]*scA[n][l/2][k01/4 + 0] + C[1].x[l]*scA[n][l/2][k01/4 + 1])*dB[l%2]; + } + } + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int l = 0; l < tile_C::ne; ++l) { + sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp[n][l]*dA[n][l/2]; + } + } + } +#else + GGML_UNUSED_VARS(x, y, sum, k00); + NO_DEVICE_CODE; +#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE +} + +// --------------------------------------------------------------------------------------------- + +// Shared MMA kernel for MXFP4 and NVFP4 on Blackwell. +// Both quantizations encode values as e2m1 (FP4) and produce one uint32 scale per +// m16n8k64 MMA call; only the PTX kind (scale_vec::2X ue8m0 vs scale_vec::4X ue4m3) +// and the per-type stride constant differ. +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_fp4_fp4_mma( + const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { + + typedef tile<16, 8, int> tile_A; + typedef tile<8, 8, int> tile_B; + typedef tile<16, 8, float> tile_C; + + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q4); + constexpr int ntx = rows_per_warp / tile_C::I; + constexpr int nfrags = MMQ_TILE_NE_K / tile_A::J; + + y += (threadIdx.y % ntx) * (tile_C::J * MMQ_TILE_Y_K); + + const int * x_qs = (const int *) x; + const uint32_t * x_sc = (const uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K); + const int * y_qs = (const int *) y + 4; + const uint32_t * y_sc = (const uint32_t *) y; + + // 2 threads per quad supply the packed scale register to the block_scale MMA, + // see https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-block-scaling + const int tidx_A = threadIdx.x / 4 + (threadIdx.x % 2) * 8; + const int tidx_B = threadIdx.x / 4; + const int i0 = (threadIdx.y / ntx) * rows_per_warp; + + tile_A A[ntx][nfrags]; + uint32_t scaleA[ntx][nfrags]; + +#pragma unroll + for (int n = 0; n < ntx; ++n) { +#pragma unroll + for (int frag = 0; frag < nfrags; ++frag) { + const int k0 = k00 + frag * tile_A::J; + load_ldmatrix(A[n][frag], x_qs + (i0 + n * tile_A::I) * sram_stride + k0, sram_stride); + scaleA[n][frag] = x_sc[(i0 + n * tile_A::I + tidx_A) * sram_stride + k0 / tile_A::J]; + } + } + +#pragma unroll + for (int j0 = 0; j0 < J; j0 += ntx * tile_C::J) { + tile_B B[nfrags]; + uint32_t scaleB[nfrags]; + +#pragma unroll + for (int frag = 0; frag < nfrags; ++frag) { + const int k0 = frag * tile_B::J; + load_generic(B[frag], y_qs + j0 * MMQ_TILE_Y_K + k0, MMQ_TILE_Y_K); + scaleB[frag] = y_sc[(j0 + tidx_B) * MMQ_TILE_Y_K + frag]; + } + +#pragma unroll + for (int n = 0; n < ntx; ++n) { + // accumulate in place into the output sum array + tile_C & C = *reinterpret_cast(sum + (j0 / tile_C::J + n) * tile_C::ne); +#pragma unroll + for (int frag = 0; frag < nfrags; ++frag) { + mma_block_scaled_fp4(C, A[n][frag], B[frag], scaleA[n][frag], scaleB[frag]); + } + } + } +} + diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 3f01ff5..27558ec 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -3,87 +3,186 @@ #include "quantize.cuh" #include "mmid.cuh" -static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { +#include + +static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream, const ggml_prec prec_src1) { switch (args.type_x) { case GGML_TYPE_Q1_0: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); + break; + case GGML_TYPE_Q2_0: + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q4_0: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q4_1: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q5_0: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q5_1: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q8_0: - mul_mat_q_case(ctx, args, stream); - break; - case GGML_TYPE_MXFP4: - mul_mat_q_case(ctx, args, stream); - break; - case GGML_TYPE_NVFP4: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; +// ----------------------------------------------------------------------- case GGML_TYPE_Q2_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q3_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q4_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q5_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q6_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); + break; +// ----------------------------------------------------------------------- + case GGML_TYPE_IQ1_S: + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ2_XXS: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ2_XS: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ2_S: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ3_XXS: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ3_S: - mul_mat_q_case(ctx, args, stream); - break; - case GGML_TYPE_IQ1_S: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ4_XS: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ4_NL: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); + break; +// ----------------------------------------------------------------------- + case GGML_TYPE_MXFP4: + // src1 at Q4 uses the native FP4 instructions, which are Blackwell-only + if (prec_src1 == GGML_PREC_Q4) { + mul_mat_q_case(ctx, args, stream); + break; + } + mul_mat_q_case(ctx, args, stream); + break; + case GGML_TYPE_NVFP4: + if (prec_src1 == GGML_PREC_Q4) { + mul_mat_q_case(ctx, args, stream); + break; + } + mul_mat_q_case(ctx, args, stream); break; default: GGML_ABORT("fatal error"); - break; } } +// overrides the src1 precision requested by the graph, "auto" keeps the requested one +static ggml_prec ggml_cuda_mmq_get_prec_env() { + const char * env_c = getenv("GGML_CUDA_MMQ_PREC"); + if (env_c == nullptr) { + return GGML_PREC_UNDEFINED; + } + std::string env_cpp = env_c; + for (char & c : env_cpp) { + c = std::tolower(c); + } + if (env_cpp == "q4") { + return GGML_PREC_Q4; + } + if (env_cpp == "q8") { + return GGML_PREC_Q8; + } + if (env_cpp != "auto") { + GGML_LOG_WARN("%s: Unknown value for GGML_CUDA_MMQ_PREC: '%s'. Available: 'q4', 'q8', 'auto'.\n", __func__, env_cpp.c_str()); + } + return GGML_PREC_UNDEFINED; +} + +// src1 is quantized to Q8_1 unless the FP4 types can use 4-bit activations, in which case they +// default to the native W4A4 instructions on Blackwell. +static ggml_prec ggml_cuda_mmq_get_prec_src1(const ggml_tensor * src0, const ggml_tensor * dst, const int cc) { + static const ggml_prec prec_env = ggml_cuda_mmq_get_prec_env(); + + ggml_prec prec = prec_env; + if (prec == GGML_PREC_UNDEFINED) { + prec = (ggml_prec) ggml_get_op_params_i32(dst, 3); + } + + // Q4 only for the FP4 types on Blackwell + GGML_ASSERT(prec == GGML_PREC_UNDEFINED || prec == GGML_PREC_Q8 || prec == GGML_PREC_Q4); + const bool can_use_q4 = (src0->type == GGML_TYPE_NVFP4 || src0->type == GGML_TYPE_MXFP4) && blackwell_mma_available(cc); + if (prec == GGML_PREC_Q8 || !can_use_q4) { + return GGML_PREC_Q8; + } + return GGML_PREC_Q4; +} + +// Tile width in ne11 (dense) / ne12 (MoE) direction with the fewest tiles that fits into shared memory, 0 if there is none. +static int ggml_cuda_mmq_get_J_best( + const ggml_type type, const int64_t ncols, const bool fallback, const int cc, const size_t smpbo, const ggml_prec prec_src1, + int * nthreads_best = nullptr) { + int J_best = 0; + int ntiles_J_best = INT_MAX; + + for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) { + const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1); + if (config.type == GGML_TYPE_COUNT) { + continue; + } + + if (mmq_get_nbytes_shared(config, cc) > smpbo) { + continue; + } + + const int ntiles_x = (ncols + config.J - 1) / config.J; + + if (ntiles_x < ntiles_J_best) { + J_best = J; + if (nthreads_best) { + *nthreads_best = config.nthreads; + } + ntiles_J_best = ntiles_x; + } + } + return J_best; +} + void ggml_cuda_mul_mat_q( - ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, + const ggml_cuda_mm_fusion_args_host * fusion) { GGML_ASSERT( src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID. + // Fused gate/up/GLU: src0 is the up weight, dst the GLU output. + const ggml_tensor * gate = fusion ? fusion->gate : nullptr; + if (fusion) { + // TODO MoE support + GGML_ASSERT(gate && !ids && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale); + GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0)); + } + GGML_TENSOR_BINARY_OP_LOCALS; cudaStream_t stream = ctx.stream(); - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + const size_t smpbo = ggml_cuda_info().devices[id].smpbo; const size_t ts_src0 = ggml_type_size(src0->type); const size_t ts_src1 = ggml_type_size(src1->type); @@ -98,14 +197,16 @@ void ggml_cuda_mul_mat_q( const float * src1_d = (const float *) src1->data; float * dst_d = (float *) dst->data; - // If src0 is a temporary compute buffer, clear any potential padding. - if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { - const size_t size_data = ggml_nbytes(src0); - const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0); - if (size_alloc > size_data) { - GGML_ASSERT(ggml_is_contiguously_allocated(src0)); - GGML_ASSERT(!src0->view_src); - CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream)); + // If src0 (or the fused gate weight) is a temporary compute buffer, clear any potential padding. + for (const ggml_tensor * t : { src0, gate }) { + if (t && ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + const size_t size_data = ggml_nbytes(t); + const size_t size_alloc = ggml_backend_buffer_get_alloc_size(t->buffer, t); + if (size_alloc > size_data) { + GGML_ASSERT(ggml_is_contiguously_allocated(t)); + GGML_ASSERT(!t->view_src); + CUDA_CHECK(cudaMemsetAsync((char *) t->data + size_data, 0, size_alloc - size_data, stream)); + } } } @@ -118,24 +219,58 @@ void ggml_cuda_mul_mat_q( const int64_t s03 = src0->nb[3] / ts_src0; const int64_t s3 = dst->nb[3] / ts_dst; - const bool use_stream_k = (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) - || GGML_CUDA_CC_IS_CDNA(cc); + const bool fallback = ggml_cuda_mmq_needs_fallback(ne01); + + // With fusion dst is the GLU output, not a MUL_MAT; the fused types always quantize src1 to Q8_1. + // TODO support for Q4 precision if both tensors allow it + const ggml_prec prec_src1 = gate ? GGML_PREC_Q8 : ggml_cuda_mmq_get_prec_src1(src0, dst, cc); + + const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4; + const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq); + const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ; - // TODO: tighter pool buffer size vs q8 path - const bool use_native_mxfp4 = blackwell_mma_available(cc) && src0->type == GGML_TYPE_MXFP4; + int J_best = 0; + int nthreads_best = 0; + { + int64_t ncols_opt = ne11; + if (ids) { + const int64_t n_expert_used = ids->ne[0]; + ncols_opt = ne12; + + // Each expert only sees ne12*n_expert_used/ne02 tokens on average. + // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12. + if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) { + ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02; + } + } + + J_best = ggml_cuda_mmq_get_J_best(src0->type, ncols_opt, fallback, cc, smpbo, prec_src1, &nthreads_best); + } + GGML_ASSERT(J_best > 0); + + // A tile of size J can read in at most J - 1 extra columns. + // For simplicity, round up the padding of a full tile to a multiple of the number of bytes that nthreads can load in parallel. + const size_t src1_load_chunk_size = nthreads_best * sizeof(int); + const size_t src1_q8_1_padding = ((J_best * sizeof(block_q8_1_mmq) + src1_load_chunk_size - 1) / src1_load_chunk_size) + * src1_load_chunk_size; if (!ids) { - const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1 + - get_mmq_x_max_host(cc)*sizeof(block_q8_1_mmq); + const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding; ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), nbytes_src1_q8_1); + ggml_cuda_pool_alloc src1_scale(ctx.pool()); + if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) { + src1_scale.alloc(ne13*ne12*ne11); + } { const int64_t s11 = src1->nb[1] / ts_src1; const int64_t s12 = src1->nb[2] / ts_src1; const int64_t s13 = src1->nb[3] / ts_src1; - if (use_native_mxfp4) { + if (use_native_fp4) { + static constexpr size_t align_float8 = 32; + const bool use_aligned_float8 = ggml_cuda_is_aligned(src1, align_float8); static_assert(sizeof(block_fp4_mmq) == 4 * sizeof(block_q8_1)); - quantize_mmq_mxfp4_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, + quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); } else { @@ -146,20 +281,26 @@ void ggml_cuda_mul_mat_q( } // Stride depends on quantization format - const int64_t s12 = use_native_mxfp4 ? - ne11 * ne10_padded * sizeof(block_fp4_mmq) / - (8 * QK_MXFP4 * sizeof(int)) // block_fp4_mmq holds 256 values (8 blocks of 32) - : + const int64_t s12 = use_native_fp4 ? + ne11 * ne10_padded * sizeof(block_fp4_mmq) / (QK_FP4_MMQ * sizeof(int)) : ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); const int64_t s13 = ne12*s12; + ggml_cuda_mm_fusion_args_device fusion_device = {}; + if (gate) { + fusion_device.gate = gate->data; + fusion_device.glu_op = fusion->glu_op; + fusion_device.glu_limit = fusion->glu_limit; + } + const mmq_args args = { src0_d, src0->type, (const int *) src1_q8_1.ptr, nullptr, nullptr, dst_d, + src0->type == GGML_TYPE_NVFP4 && use_native_fp4 ? src1_scale.ptr : nullptr, ne00, ne01, ne1, s01, ne11, s1, ne02, ne12, s02, s12, s2, ne03, ne13, s03, s13, s3, - use_stream_k, ne1}; - ggml_cuda_mul_mat_q_switch_type(ctx, args, stream); + ne1, J_best, fusion_device}; + ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1); return; } @@ -172,22 +313,29 @@ void ggml_cuda_mul_mat_q( GGML_ASSERT(ne1 == n_expert_used); ggml_cuda_pool_alloc ids_src1(ctx.pool(), ne_get_rows); - ggml_cuda_pool_alloc ids_dst(ctx.pool(), ne_get_rows); + ggml_cuda_pool_alloc ids_dst(ctx.pool(), ne_get_rows + J_best-1); // Needs to be padded for unconditional memory access. ggml_cuda_pool_alloc expert_bounds(ctx.pool(), ne02 + 1); + // gate/up activations are broadcast across experts (ne11 == 1): quantize each token once and + // scatter to its slots. ids_src1 then holds the inverse map (token slot -> compact row). + const bool dedup_bcast = ne11 == 1 && n_expert_used > 1; + { GGML_ASSERT(ids->nb[0] == ggml_element_size(ids)); const int si1 = ids->nb[1] / ggml_element_size(ids); const int sis1 = nb12 / nb11; ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(), - ne02, ne12, n_expert_used, ne11, si1, sis1, stream); + ne02, ne12, n_expert_used, ne11, si1, sis1, /*write_inverse =*/ dedup_bcast, stream); CUDA_CHECK(cudaGetLastError()); } - const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * sizeof(block_q8_1)/QK8_1 + - get_mmq_x_max_host(cc)*sizeof(block_q8_1_mmq); + const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding; ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), nbytes_src1_q8_1); + ggml_cuda_pool_alloc src1_scale(ctx.pool()); + if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) { + src1_scale.alloc(ne12*n_expert_used); + } const int64_t ne11_flat = ne12*n_expert_used; const int64_t ne12_flat = 1; @@ -198,9 +346,19 @@ void ggml_cuda_mul_mat_q( const int64_t s12 = src1->nb[2] / ts_src1; const int64_t s13 = src1->nb[3] / ts_src1; - if (use_native_mxfp4) { - quantize_mmq_mxfp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13, - ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream); + if (use_native_fp4) { + static constexpr size_t align_float8 = 32; + const bool use_aligned_float8 = ggml_cuda_is_aligned(src1, align_float8); + if (dedup_bcast) { + quantize_scatter_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, + /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream); + } else { + quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, + ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream); + } + } else if (dedup_bcast) { + quantize_scatter_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, + /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream); } else { quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream); @@ -208,61 +366,21 @@ void ggml_cuda_mul_mat_q( CUDA_CHECK(cudaGetLastError()); } - const int64_t s12 = use_native_mxfp4 ? ne11 * ne10_padded * sizeof(block_fp4_mmq) / (8 * QK_MXFP4 * sizeof(int)) : - ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); + static_assert(QK_FP4_MMQ == 8 * QK_MXFP4, "QK_FP4_MMQ needs to be 8 * QK_MXFP4"); + const int64_t s12 = use_native_fp4 ? ne11 * ne10_padded * sizeof(block_fp4_mmq) / (QK_FP4_MMQ * sizeof(int)) : + ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); const int64_t s13 = ne12*s12; // Note that ne02 is used instead of ne12 because the number of y channels determines the z dimension of the CUDA grid. const mmq_args args = { src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), dst_d, + src1_scale.ptr, ne00, ne01, ne_get_rows, s01, ne_get_rows, s1, ne02, ne02, s02, s12, s2, ne03, ne13, s03, s13, s3, - use_stream_k, ne12}; - - ggml_cuda_mul_mat_q_switch_type(ctx, args, stream); -} - -void ggml_cuda_op_mul_mat_q( - ggml_backend_cuda_context & ctx, - const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, - const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, - const int64_t src1_padded_row_size, cudaStream_t stream) { - - const int64_t ne00 = src0->ne[0]; - - const int64_t ne10 = src1->ne[0]; - const int64_t ne11 = src1->ne[1]; - GGML_ASSERT(ne10 % QK8_1 == 0); - - const int64_t ne0 = dst->ne[0]; - - const int64_t row_diff = row_high - row_low; - const int64_t stride01 = ne00 / ggml_blck_size(src0->type); - - const int id = ggml_cuda_get_device(); - const int cc = ggml_cuda_info().devices[id].cc; - - // the main device has a larger memory buffer to hold the results from all GPUs - // nrows_dst == nrows of the matrix that the kernel writes into - const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff; - - // The stream-k decomposition is only faster for recent NVIDIA GPUs. - // Also its fixup needs to allocate a temporary buffer in the memory pool. - // There are multiple parallel CUDA streams for src1_ncols != ne11 which would introduce a race condition for this buffer. - const bool use_stream_k = ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) - || GGML_CUDA_CC_IS_CDNA(cc)) - && src1_ncols == ne11; - const mmq_args args = { - src0_dd_i, src0->type, (const int *) src1_ddq_i, nullptr, nullptr, dst_dd_i, - ne00, row_diff, src1_ncols, stride01, ne11, nrows_dst, - 1, 1, 0, 0, 0, - 1, 1, 0, 0, 0, - use_stream_k, src1_ncols}; - - ggml_cuda_mul_mat_q_switch_type(ctx, args, stream); + ne12, J_best}; - GGML_UNUSED_VARS(src1, dst, src1_ddf_i, src1_padded_row_size); + ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1); } bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) { @@ -274,26 +392,30 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t switch (type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: - case GGML_TYPE_MXFP4: - case GGML_TYPE_NVFP4: +// ------------------------------------------------- case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: +// ------------------------------------------------- + case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: - case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ4_NL: +// ------------------------------------------------- + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: mmq_supported = true; break; default: @@ -305,12 +427,23 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t return false; } + // MMQ tiles require at least 48 KiB per-block shared memory; fall back to BLAS otherwise. + { + const int id = ggml_cuda_get_device(); + const size_t smpbo = ggml_cuda_info().devices[id].smpbo; + if (smpbo < 48 * 1024) { + return false; + } + } + if (turing_mma_available(cc)) { return true; } if (ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_DP4A) { - return false; + // for MoE, mmq is faster even without native dp4a + // TODO: check if cards older than pascal might benefit from this as well + return cc >= GGML_CUDA_CC_PASCAL && n_experts > 0; } #ifdef GGML_CUDA_FORCE_MMQ @@ -369,5 +502,36 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t return true; } + // gfx900 (Vega 10), gfx909, and gfx90c lack native dp4a, losing to dequant + hipBLAS + // for dense matrices; keep MMQ only for MoE, where the + // hipBLAS path is much slower. + if (cc == GGML_CUDA_CC_VEGA || GGML_CUDA_CC_IS_GCN_APU(cc)) { + return n_experts > 0; + } + + // MUSA: the MMQ kernels compute wrong values on PH1 (MTT S5000). + if (cc == GGML_CUDA_CC_PH1) { + return false; + } + return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE; } + +bool ggml_cuda_mmq_fusion_is_efficient(const ggml_type type, const int64_t nrows_x, const int64_t ncols, const int64_t nchannels, const int device) { + const int cc = ggml_cuda_info().devices[device].cc; + const int nsm = ggml_cuda_info().devices[device].nsm; + const size_t smpbo = ggml_cuda_info().devices[device].smpbo; + + // Same tile width as ggml_cuda_mul_mat_q; the fused kernel has no fallback, quantizes src1 to Q8_1 and writes I/2 output rows per tile. + const bool fallback = false; + const int J_best = ggml_cuda_mmq_get_J_best(type, ncols, fallback, cc, smpbo, GGML_PREC_Q8); + if (J_best == 0) { + return false; + } + const int64_t nrows_tile = ggml_cuda_mmq_get_config(type, J_best, fallback, cc, GGML_PREC_Q8).I / 2; + const int64_t ntiles = (ncols + J_best - 1)/J_best * ((nrows_x + nrows_tile - 1)/nrows_tile) * nchannels; + // Same condition as launch_mul_mat_q uses to run the stream-k kernel with one block per tile. + const int64_t tiles_nwaves = (ntiles + nsm - 1) / nsm; + const int64_t tiles_efficiency_percent = 100 * ntiles / (nsm*tiles_nwaves); + return tiles_efficiency_percent >= 90; +} diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index f89fdfc..565ed14 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -1,23 +1,22 @@ #pragma once #include "common.cuh" -#include "vecdotq.cuh" -#include "mma.cuh" +#include "unary.cuh" #include #include -using namespace ggml_cuda_mma; - #define MMQ_DP4A_MAX_BATCH_SIZE 64 // Max. batch size to use for dp4a MMQ kernels when FP16 tensor cores are available. -#define MMQ_ITER_K 256 -#define MMQ_ITER_K_MXFP4_FP4 512 -#define MMQ_NWARPS 8 +#define MMQ_ITER_K 256 +#define MMQ_ITER_K_FP4 512 +#define MMQ_NWARPS 8 -typedef void (*load_tiles_mmq_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride); -typedef void (*vec_dot_mmq_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00); -typedef void (*mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted, - float * __restrict__ dst, const int stride, const int i_max, const int j_max); +typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, const char * __restrict__ x_gate, int * x_tile, + const int kbx0, const int i_max, const int stride); +typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00); +typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted, + float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max, + const ggml_glu_op glu_op, const float glu_limit); enum mmq_q8_1_ds_layout { MMQ_Q8_1_DS_LAYOUT_D4, @@ -25,6 +24,9 @@ enum mmq_q8_1_ds_layout { MMQ_Q8_1_DS_LAYOUT_D2S6, }; +static constexpr int QK8_1_MMQ = 4*QK8_1; +static constexpr int QK_FP4_MMQ = 2*QK8_1_MMQ; + struct block_q8_1_mmq { // The y float data is converted to a data layout that can simply be copied to shared memory as a contiguous block. // The y float data is first grouped as blocks of 128 values. @@ -43,21 +45,25 @@ struct block_q8_1_mmq { half d2s6[8]; // 1 16 bit scale per 64 values + 1 16 bit partial sum per 16 values for the first 96 values, // stored as d0,d1,s1,s2,s3,s4,s5 }; - int8_t qs[4*QK8_1]; // 128 values quantized to 8 bit each + int8_t qs[QK8_1_MMQ]; }; +// this struct is used for fp4 data types (currently only used for Blackwell) +// mxfp4 has block size 32, each int32 of d4 contains 2 e8m0 scales in the lower 16 bits +// nvfp4 has block size 16, each int32 of d4 contains 4 ue4m3 scales struct block_fp4_mmq { - uint32_t d4[4]; // 8 E8M0 scales (1 per 32 values), 2 packed per uint32: d4[0]={s0,s1}, d4[1]={s2,s3}, etc. - int8_t qs[4 * 32]; // 256 FP4 values packed as 4-bit pairs (2 per byte), 8 blocks of 32 values + uint32_t d4[4]; + int8_t qs[QK_FP4_MMQ / 2]; }; -static_assert(sizeof(block_q8_1_mmq) == 4*QK8_1 + 4*sizeof(half2), "Unexpected block_q8_1_mmq size"); +static_assert(sizeof(block_q8_1_mmq) == QK8_1_MMQ + 4*sizeof(half2), "Unexpected block_q8_1_mmq size"); static_assert(sizeof(block_q8_1_mmq) == 4*sizeof(block_q8_1), "Unexpected block_q8_1_mmq size"); static_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), "Unexpected block_fp4_mmq size"); static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) { switch (type_x) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: return MMQ_Q8_1_DS_LAYOUT_D4; case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -93,7 +99,6 @@ static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) { return MMQ_Q8_1_DS_LAYOUT_D4; default: GGML_ABORT("fatal error"); - break; } } @@ -103,68 +108,6 @@ struct tile_x_sizes { int sc; }; -static int get_mmq_x_max_host(const int cc) { - return (amd_mfma_available(cc) || turing_mma_available(cc) || amd_wmma_available(cc)) ? 128 : - GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA ? -#ifdef GGML_CUDA_FORCE_MMQ - 128 : 64; -#else - MMQ_DP4A_MAX_BATCH_SIZE : 64; -#endif // GGML_CUDA_FORCE_MMQ -} - -static constexpr __device__ int get_mmq_x_max_device() { -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - return 128; -#else // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) - -#if defined(GGML_USE_HIP) - return 64; -#else // defined(GGML_USE_HIP) - -#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA -#ifdef GGML_CUDA_FORCE_MMQ - return 128; -#else // GGML_CUDA_FORCE_MMQ - return MMQ_DP4A_MAX_BATCH_SIZE; -#endif // GGML_CUDA_FORCE_MMQ -#else // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA - return 64; -#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA - -#endif // defined(GGML_USE_HIP) -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) -} - -static int get_mmq_y_host(const int cc) { - return GGML_CUDA_CC_IS_AMD(cc) ? (GGML_CUDA_CC_IS_RDNA1(cc) ? 64 : 128) : - ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) ? 128 : 64); -} - -static constexpr __device__ int get_iter_k([[maybe_unused]] const ggml_type type) { -#if defined(BLACKWELL_MMA_AVAILABLE) - return type == GGML_TYPE_MXFP4 ? MMQ_ITER_K_MXFP4_FP4 : MMQ_ITER_K; -#else - return MMQ_ITER_K; -#endif // defined(BLACKWELL_MMA_AVAILABLE) -} - -static constexpr __device__ int get_mmq_y_device() { -#if defined(GGML_USE_HIP) -#if defined(RDNA1) - return 64; -#else - return 128; -#endif // defined RDNA1 -#else -#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA - return 128; -#else - return 64; -#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA -#endif // defined(GGML_USE_HIP) -} - // Decouple shared memory tile sizes from WARP_SIZE to allow for different warp sizes. // The K dimension of the tiles has either, // 1*MMQ_TILE_NE_K==32 (always for TILE_Y_K) or 2*MMQ_TILE_NE_K==64 (typically for TILE_X_K), @@ -174,3134 +117,358 @@ static constexpr __device__ int get_mmq_y_device() { // in terms of 32 bit elements that means K % 2 == 1 for dp4a or K % 8 == 4 for mma. #define MMQ_TILE_NE_K 32 -#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{mmq_y*MMQ_TILE_NE_K + mmq_y, mmq_y*MMQ_TILE_NE_K/QI4_0 + mmq_y/QI4_0, 0} -#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{mmq_y*MMQ_TILE_NE_K + mmq_y, mmq_y*MMQ_TILE_NE_K/QI4_1 + mmq_y/QI4_1, 0} -#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K*2/QI8_0 + mmq_y/(QI8_0/2), 0} -#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K*4/QI8_0 + mmq_y/(QI8_0/4), 0} -#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K*2/QI8_1 + mmq_y/(QI8_1/2), 0} -#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K + mmq_y, 0} -#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8} -#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K + mmq_y, mmq_y*MMQ_TILE_NE_K/QI4_K, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8} -#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K/QI5_K + mmq_y/QI5_K, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8} -#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{mmq_y*MMQ_TILE_NE_K*2 + mmq_y, mmq_y*MMQ_TILE_NE_K/QI6_K + mmq_y/QI6_K, mmq_y*MMQ_TILE_NE_K/8 + mmq_y/8} - -static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int mmq_y) { - switch (type) { - case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0; - case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0; - case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1; - case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0; - case GGML_TYPE_Q5_1: return MMQ_DP4A_TXS_Q8_1; - case GGML_TYPE_Q8_0: return MMQ_DP4A_TXS_Q8_0; - case GGML_TYPE_MXFP4: return MMQ_DP4A_TXS_Q8_1; - case GGML_TYPE_NVFP4: return MMQ_DP4A_TXS_Q8_0_16; - case GGML_TYPE_Q2_K: return MMQ_DP4A_TXS_Q2_K; - case GGML_TYPE_Q3_K: return MMQ_DP4A_TXS_Q3_K; - case GGML_TYPE_Q4_K: return MMQ_DP4A_TXS_Q4_K; - case GGML_TYPE_Q5_K: return MMQ_DP4A_TXS_Q5_K; - case GGML_TYPE_Q6_K: return MMQ_DP4A_TXS_Q6_K; - case GGML_TYPE_IQ2_XXS: return MMQ_DP4A_TXS_Q8_0; - case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16; - case GGML_TYPE_IQ2_S: return MMQ_DP4A_TXS_Q8_0_16; - case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0; - case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0; - case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0; - case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0; - case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0; - default: return tile_x_sizes{0, 0, 0}; - } -} - -#define MMQ_MMA_TILE_X_K_Q8_0 (2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4) -#define MMQ_MMA_TILE_X_K_FP4 (2*MMQ_TILE_NE_K + 8 + 4) // MXFP4 -#define MMQ_MMA_TILE_X_K_NVFP4 (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4) // NVFP4 -#define MMQ_MMA_TILE_X_K_Q8_1 (2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4) -#define MMQ_MMA_TILE_X_K_Q2_K (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K + 4) -#define MMQ_MMA_TILE_X_K_Q3_K (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4) -#define MMQ_MMA_TILE_X_K_Q6_K (2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7) - -static_assert(MMQ_MMA_TILE_X_K_Q8_0 % 8 == 4, "Wrong padding."); -static_assert(MMQ_MMA_TILE_X_K_Q8_1 % 8 == 4, "Wrong padding."); -static_assert(MMQ_MMA_TILE_X_K_Q2_K % 8 == 4, "Wrong padding."); -static_assert(MMQ_MMA_TILE_X_K_Q3_K % 8 == 4, "Wrong padding."); -static_assert(MMQ_MMA_TILE_X_K_Q6_K % 8 == 4, "Wrong padding."); -static_assert(MMQ_MMA_TILE_X_K_FP4 % 8 == 4, "Wrong padding."); -static_assert(MMQ_MMA_TILE_X_K_FP4 == MMQ_MMA_TILE_X_K_Q8_1, "Wrong tile size for MXFP4"); -static_assert(MMQ_MMA_TILE_X_K_NVFP4 % 8 == 4, "Wrong padding."); - - -static constexpr __host__ __device__ int mmq_get_mma_tile_x_k(ggml_type type) { - switch (type) { - case GGML_TYPE_Q1_0: return MMQ_MMA_TILE_X_K_Q8_0; - case GGML_TYPE_Q4_0: return MMQ_MMA_TILE_X_K_Q8_0; - case GGML_TYPE_Q4_1: return MMQ_MMA_TILE_X_K_Q8_1; - case GGML_TYPE_Q5_0: return MMQ_MMA_TILE_X_K_Q8_0; - case GGML_TYPE_Q5_1: return MMQ_MMA_TILE_X_K_Q8_1; - case GGML_TYPE_Q8_0: return MMQ_MMA_TILE_X_K_Q8_0; - // tile sizes are the same for Q8_1 and FP4 for blackwell - case GGML_TYPE_MXFP4: return MMQ_MMA_TILE_X_K_Q8_1; - case GGML_TYPE_NVFP4: return MMQ_MMA_TILE_X_K_NVFP4; - case GGML_TYPE_Q2_K: return MMQ_MMA_TILE_X_K_Q2_K; - case GGML_TYPE_Q3_K: return MMQ_MMA_TILE_X_K_Q3_K; - case GGML_TYPE_Q4_K: return MMQ_MMA_TILE_X_K_Q8_1; - case GGML_TYPE_Q5_K: return MMQ_MMA_TILE_X_K_Q8_1; - case GGML_TYPE_Q6_K: return MMQ_MMA_TILE_X_K_Q6_K; - case GGML_TYPE_IQ2_XXS: return MMQ_MMA_TILE_X_K_Q8_0; - case GGML_TYPE_IQ2_XS: return MMQ_MMA_TILE_X_K_Q3_K; - case GGML_TYPE_IQ2_S: return MMQ_MMA_TILE_X_K_Q3_K; - case GGML_TYPE_IQ3_XXS: return MMQ_MMA_TILE_X_K_Q8_0; - case GGML_TYPE_IQ3_S: return MMQ_MMA_TILE_X_K_Q8_0; - case GGML_TYPE_IQ1_S: return MMQ_MMA_TILE_X_K_Q8_0; - case GGML_TYPE_IQ4_XS: return MMQ_MMA_TILE_X_K_Q8_0; - case GGML_TYPE_IQ4_NL: return MMQ_MMA_TILE_X_K_Q8_0; - default: return 0; - } -} - // block_q8_1_mmq has (128 8-bit ints == 32 32-bit ints + 4 32-bit scales) #define MMQ_TILE_Y_K (MMQ_TILE_NE_K + MMQ_TILE_NE_K / QI8_1) #define MMQ_TILE_Y_FP4_K MMQ_TILE_Y_K -static int mmq_get_granularity_host(const int mmq_x, const int cc) { - if (amd_mfma_available(cc) || amd_wmma_available(cc)) { - return mmq_x >= 128 ? 32 : 16; - } else if (turing_mma_available(cc) && mmq_x >= 48) { - return 16; - } else { - return 8; - } -} - -#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) -static constexpr __device__ int mmq_get_granularity_device(const int mmq_x) { - return mmq_x >= 128 ? 32 : 16; -} -#elif defined(TURING_MMA_AVAILABLE) -static constexpr __device__ int mmq_get_granularity_device(const int mmq_x) { - return mmq_x >= 48 ? 16 : 8; -} -#else -static constexpr __device__ int mmq_get_granularity_device(const int /*mmq_x*/) { - return 8; -} -#endif // AMD_MFMA_AVAILABLE - -#if defined(GGML_USE_HIP) -static int mmq_get_nwarps_host(const int cc, const int warp_size) { - return amd_mfma_available(cc) ? 8 : 256/warp_size; -} -#else -static int mmq_get_nwarps_host(const int /*cc*/, const int warp_size) { - return 256/warp_size; -} -#endif // (GGML_USE_HIP) - -static constexpr __device__ int mmq_get_nwarps_device() { -#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - return 8; -#else - return 256/ggml_cuda_get_physical_warp_size(); -#endif // AMD_MFMA_AVAILABLE -} - -// ------------------------------------------------------------ - -template static __device__ __forceinline__ void load_tiles_q1_0( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int blocks_per_iter = MMQ_ITER_K / QK1_0; - constexpr int threads_per_row = blocks_per_iter * QI1_0; - constexpr int nrows = warp_size / threads_per_row; - constexpr int scale_entries_per_block = QK1_0 / QK8_1; - constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block; - - const int txi = threadIdx.x % threads_per_row; - const int kbx = txi / QI1_0; - const int kqsx = txi % QI1_0; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + kbx; - const int qs_offset = 4*kqsx; - const int qs0 = bxi->qs[qs_offset + 0] | (bxi->qs[qs_offset + 1] << 8) | - (bxi->qs[qs_offset + 2] << 16) | (bxi->qs[qs_offset + 3] << 24); - - int unpacked_bytes[8]; -#pragma unroll - for (int j = 0; j < 8; ++j) { - const int shift = j * 4; - const int bits4 = (qs0 >> shift) & 0x0F; - const int b0 = (bits4 & 0x01) ? 1 : -1; - const int b1 = (bits4 & 0x02) ? 1 : -1; - const int b2 = (bits4 & 0x04) ? 1 : -1; - const int b3 = (bits4 & 0x08) ? 1 : -1; - unpacked_bytes[j] = (b0 & 0xFF) | ((b1 & 0xFF) << 8) | ((b2 & 0xFF) << 16) | ((b3 & 0xFF) << 24); - } - - const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0; -#pragma unroll - for (int j = 0; j < 8; ++j) { -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + dst_offset + j] = unpacked_bytes[j]; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j] = unpacked_bytes[j]; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - } - - const int ksx = threadIdx.x % scale_entries_per_row; - const int scale_block = ksx / scale_entries_per_block; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps) { - int i = i0 + threadIdx.y; - - if (need_check) { - i = min(i, i_max); - } - - const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + scale_block; +enum ggml_cuda_mmq_sram_layout { + GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, + GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, + GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, + GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, + GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, + GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, // MXFP4 and NVFP4 on Blackwell. + GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, // Generic NVFP4 +}; -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + ksx] = bxi->d; -#else - x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +static constexpr __host__ __device__ int ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_sram_layout sram_layout) { + switch (sram_layout) { + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0: + return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1: + return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_1 + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K: + return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K: + return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K: + return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7; + case GGML_CUDA_MMQ_SRAM_LAYOUT_FP4: + return 2*MMQ_TILE_NE_K + 8 + 4; + case GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4: + return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4; + default: + return -1; } } -template static __device__ __forceinline__ void load_tiles_q4_0( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_0); - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - const int kbx = txi / QI4_0; - const int kqsx = txi % QI4_0; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - - if (need_check) { - i = min(i, i_max); - } - - const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbx; - const int qs0 = get_int_b2(bxi->qs, kqsx); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI4_0) + kqsx + 0] = __vsubss4((qs0 >> 0) & 0x0F0F0F0F, 0x08080808); - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI4_0) + kqsx + QI4_0] = __vsubss4((qs0 >> 4) & 0x0F0F0F0F, 0x08080808); -#else - x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) - } - - constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_0; - constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; - const int kbxd = threadIdx.x % blocks_per_tile_x_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) { - int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbxd; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d; +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) % 8 == 4, "Wrong padding."); +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4) % 8 == 4, "Wrong padding."); + +static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) == ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1), "Wrong tile size for MXFP4"); + +// Config options for the MMQ kernel. +// Should not affect results, only speed/register pressure/shared memory use. +struct ggml_cuda_mmq_config { + ggml_type type; // src0->type + int nthreads; // Number of threads per CUDA block. + int occupancy; // Targeted occupancy for the MMA kernel. + int I; // SRAM tile width in src0->ne[1]/dst->ne[0] direction. + int J; // SRAM tile width in src1->ne[1]/dst->ne[1] direction. + ggml_cuda_mmq_sram_layout sram_layout; // SRAM tile length in src0->ne[0]/src1->ne[0] direction (physical 32 bit elements). + int K_vram; // VRAM tile length in src0->ne[0]/src1->ne[0] direction (logical elements). + bool stream_k; // Whether or not to use stream-k decomposition. + bool fallback; // Whether a fallback for out-of-bounds check in src0->ne[1] direction is needed. + + constexpr __host__ __device__ ggml_cuda_mmq_config( + ggml_type type, int nthreads, int occupancy, int I, int J, ggml_cuda_mmq_sram_layout sram_layout, int K_vram, bool stream_k, bool fallback) : + type(type), nthreads(nthreads), occupancy(occupancy), I(I), J(J), sram_layout(sram_layout), K_vram(K_vram), stream_k(stream_k), fallback(fallback) {} + + constexpr __device__ int rows_per_warp() const { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + return 16; #else - x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + kbxd] = bxi->d; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template -static __device__ __forceinline__ void vec_dot_q4_0_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, mmq_y); - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + txs.qs; - const int * y_qs = (const int *) y + 4; - const half2 * y_ds = (const half2 *) y; - -// #pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_0*VDR_Q4_0_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2); - - int u[2*VDR_Q4_0_Q8_1_MMQ]; - - constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes(); - constexpr int mcpy_int = max_cpy / sizeof(int); - static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ"); - - int tmp0[4], tmp1[4]; - - #pragma unroll - for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) { - ggml_cuda_memcpy_1(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] ); - ggml_cuda_memcpy_1(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_0 + l0 * mcpy_int]); - } - - u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3]; - u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3]; - - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q4_0_q8_1_impl - (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_0], u, - x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + k0/(QR4_0*QI4_0)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - } + return J >= 48 && J % 16 == 0 ? 32 : 16; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) } -} - -template static __device__ __forceinline__ void load_tiles_q4_1( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, mmq_y); - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_1); - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - const int kbx = txi / QI4_1; - const int kqsx = txi % QI4_1; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - - if (need_check) { - i = min(i, i_max); + // TODO transition all combinations of GPUs and quantizations to the MMA data layout. + __host__ int use_mma_data_layout(const int cc) const { + if (amd_mfma_available(cc) || amd_wmma_available(cc) || turing_mma_available(cc)) { + return true; } - - const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbx; - const int qs0 = get_int_b4(bxi->qs, kqsx); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI4_1) + kqsx + 0] = (qs0 >> 0) & 0x0F0F0F0F; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI4_1) + kqsx + QI4_1] = (qs0 >> 4) & 0x0F0F0F0F; -#else - x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + return false; } - constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_1; - constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; - const int kbxd = threadIdx.x % blocks_per_tile_x_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) { - int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbxd; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = bxi->dm; + constexpr __device__ bool use_mma_data_layout() const { +#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) + return true; #else - x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + kbxd] = bxi->dm; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template -static __device__ __forceinline__ void vec_dot_q4_1_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, mmq_y); - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + txs.qs; - const int * y_qs = (const int *) y + 4; - const half2 * y_ds = (const half2 *) y; - -// #pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_1*VDR_Q4_1_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2); - - int u[2*VDR_Q4_1_Q8_1_MMQ]; - - constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes(); - constexpr int mcpy_int = max_cpy / sizeof(int); - static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ"); - - int tmp0[4], tmp1[4]; - - #pragma unroll - for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) { - ggml_cuda_memcpy_1(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] ); - ggml_cuda_memcpy_1(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_1 + l0 * mcpy_int]); - } - - u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3]; - u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3]; - - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q4_1_q8_1_impl - (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_1], u, - x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + k0/(QR4_1*QI4_1)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - } + return false; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) } -} - -template static __device__ __forceinline__ void load_tiles_q5_0( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_0, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_0); - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - const int kbx = txi / QI5_0; - const int kqsx = txi % QI5_0; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - if (need_check) { - i = min(i, i_max); - } - - const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbx; - - const int ql = get_int_b2(bxi->qs, kqsx); - const int qh = get_int_b2(bxi->qh, 0) >> (4 * kqsx); - - int qs0 = (ql >> 0) & 0x0F0F0F0F; - qs0 |= (qh << 4) & 0x00000010; // 0 -> 4 - qs0 |= (qh << 11) & 0x00001000; // 1 -> 12 - qs0 |= (qh << 18) & 0x00100000; // 2 -> 20 - qs0 |= (qh << 25) & 0x10000000; // 3 -> 28 - qs0 = __vsubss4(qs0, 0x10101010); // subtract 16 +}; - int qs1 = (ql >> 4) & 0x0F0F0F0F; - qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4 - qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12 - qs1 |= (qh << 2) & 0x00100000; // 18 -> 20 - qs1 |= (qh << 9) & 0x10000000; // 19 -> 28 - qs1 = __vsubss4(qs1, 0x10101010); // subtract 16 +#define CASE(type_, nthreads_, occupancy_, I_, J_, sram_layout_, K_vram_, stream_k_, fallback_) \ + if (type == (type_) && J == (J_) && fallback == (fallback_)) { \ + static_assert((nthreads_) % 32 == 0 && (nthreads_) <= 512, "bad nthreads"); \ + static_assert( (occupancy_) <= 8, "bad occupancy"); \ + static_assert((I_) % 32 == 0, "bad I"); \ + static_assert((J_) % 8 == 0 && (J_) <= 128, "bad J"); \ + static_assert((K_vram_) % 256 == 0, "bad K_vram"); \ + return ggml_cuda_mmq_config((type_), (nthreads_), (occupancy_), (I_), (J_), (sram_layout_), (K_vram_), (stream_k_), (fallback_)); \ + } \ -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI5_0) + kqsx + 0] = qs0; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + 0] = qs0; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } +#include "mmq-config-pascal-older.cuh" +#include "mmq-config-pascal-dp4a.cuh" +#include "mmq-config-ampere.cuh" +#include "mmq-config-blackwell.cuh" - constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_0; - constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; - const int kbxd = threadIdx.x % blocks_per_tile_x_row; +#include "mmq-config-gcn.cuh" +#include "mmq-config-cdna.cuh" +#include "mmq-config-rdna2.cuh" +#include "mmq-config-rdna3.cuh" +#include "mmq-config-rdna3-5.cuh" +#include "mmq-config-rdna4.cuh" -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) { - int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; +#undef CASE - if (need_check) { - i = min(i, i_max); +static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + if (GGML_CUDA_CC_IS_AMD(cc)) { + if (GGML_CUDA_CC_IS_GCN(cc)) { + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); } - - const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbxd; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d; -#else - x_df[i*(MMQ_TILE_NE_K/QI5_0) + i/QI5_0 + kbxd] = bxi->d; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template static __device__ __forceinline__ void load_tiles_q5_1( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, mmq_y); - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_1); - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - const int kbx = txi / QI5_1; - const int kqsx = txi % QI5_1; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - - if (need_check) { - i = min(i, i_max); + if (GGML_CUDA_CC_IS_CDNA(cc)) { + return ggml_cuda_mmq_get_config_cdna(type, J, fallback); } - - const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbx; - - const int ql = get_int_b4(bxi->qs, kqsx); - const int qh = get_int_b4(bxi->qh, 0) >> (4 * kqsx); - - int qs0 = (ql >> 0) & 0x0F0F0F0F; - qs0 |= (qh << 4) & 0x00000010; // 0 -> 4 - qs0 |= (qh << 11) & 0x00001000; // 1 -> 12 - qs0 |= (qh << 18) & 0x00100000; // 2 -> 20 - qs0 |= (qh << 25) & 0x10000000; // 3 -> 28 - - int qs1 = (ql >> 4) & 0x0F0F0F0F; - qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4 - qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12 - qs1 |= (qh << 2) & 0x00100000; // 18 -> 20 - qs1 |= (qh << 9) & 0x10000000; // 19 -> 28 - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI5_1) + kqsx + 0] = qs0; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + 0] = qs0; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_1; - constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; - const int kbxd = threadIdx.x % blocks_per_tile_x_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) { - int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; - - if (need_check) { - i = min(i, i_max); + if (GGML_CUDA_CC_IS_RDNA4(cc)) { + return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); } - - const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbxd; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = bxi->dm; -#else - x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + kbxd] = bxi->dm; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template static __device__ __forceinline__ void load_tiles_q8_0( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - // MMQ_ITER_K / (4 * QR8_0) == 64 required. but NV has only 32 threads per warp - constexpr int threads_per_row = 32; - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - const int kbx = txi / QI8_0; - const int kqsx = txi % QI8_0; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - - if (need_check) { - i = min(i, i_max); + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { + return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); } - - const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 0 + txi] = get_int_b2(bxi[0].qs, kqsx); - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx); -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 0 + txi] = get_int_b2(bxi[0].qs, kqsx); - x_qs[i*(2*MMQ_TILE_NE_K + 1) + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0; - constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; - const int kbxd = threadIdx.x % blocks_per_tile_x_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) { - int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; - - if (need_check) { - i = min(i, i_max); + if (GGML_CUDA_CC_IS_RDNA3(cc)) { // covers RDNA 3.0 + return ggml_cuda_mmq_get_config_rdna3(type, J, fallback); } - - const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = bxi->d; -#else - x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + kbxd] = bxi->d; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + return ggml_cuda_mmq_get_config_rdna2(type, J, fallback); } -} - -template static __device__ __forceinline__ void load_tiles_mxfp4( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_MXFP4, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR_MXFP4); - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - const int kbx = txi / QI_MXFP4; - const int kqsx = txi % QI_MXFP4; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - - if (need_check) { - i = min(i, i_max); + if (blackwell_mma_available(cc)) { + // only src1 at Q4 uses the native FP4 config, higher precisions keep src1 at Q8_1 + if (prec_src1 != GGML_PREC_Q4 && (type == GGML_TYPE_NVFP4 || type == GGML_TYPE_MXFP4)) { + return ggml_cuda_mmq_get_config_ampere(type, J, fallback); } - - const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbx; - - const int aux_q4 = get_int_b1(bxi->qs, kqsx); - const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4); - const int k0 = kbx * (2 * QI_MXFP4) + kqsx; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + k0 + 0] = v.x; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + k0 + QI_MXFP4] = v.y; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI_MXFP4] = v.y; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + return ggml_cuda_mmq_get_config_blackwell(type, J, fallback); } + if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) { + return ggml_cuda_mmq_get_config_ampere(type, J, fallback); + } + if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_DP4A) { + return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback); + } + return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback); +} - constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI_MXFP4; - constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; - const int kbxd = threadIdx.x % blocks_per_tile_x_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) { - int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbxd; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_1 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f; -#else - x_df[i*(MMQ_TILE_NE_K/QI_MXFP4) + i/QI_MXFP4 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template -static __device__ __forceinline__ void load_tiles_mxfp4_fp4(const char * __restrict__ x, - int * __restrict__ x_tile, - const int kbx0, - const int i_max, - const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - int * x_qs = (int *) x_tile; - uint32_t * x_sc = (uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K); - - const int txi = threadIdx.x; - - constexpr int iter_k = get_iter_k(GGML_TYPE_MXFP4); - - constexpr int threads_per_row = iter_k / QK_MXFP4; // each thread processes 1 block - constexpr int rows_per_warp = warp_size / threads_per_row; - const int kbx = txi % threads_per_row; - const int row_in_warp = txi / threads_per_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += rows_per_warp * nwarps) { - int i = i0 + threadIdx.y * rows_per_warp + row_in_warp; - - if constexpr (need_check) { - i = min(i, i_max); - } - - const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i * stride + kbx; - - // quantize_mxfp4_mmq permutes nibbles to match the quantized format - const int k0 = kbx * 4; - memcpy(x_qs + i * MMQ_MMA_TILE_X_K_FP4 + k0, bxi->qs, 16); - - // Load E8M0 scales: pack 2 consecutive scales into one uint32 - if (kbx % 2 == 0) { - uint32_t e = bxi->e; - e |= ((bxi + 1)->e << 8); - x_sc[i * MMQ_MMA_TILE_X_K_FP4 + kbx / 2] = e; - } - } -} - - -template -static __device__ __forceinline__ void load_tiles_nvfp4(const char * __restrict__ x, - int * __restrict__ x_tile, - const int kb0, - const int i_max, - const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_NVFP4, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / QK_NVFP4; - constexpr int rows_per_warp = warp_size / threads_per_row; - const int kbx = threadIdx.x % threads_per_row; - const int row_in_warp = threadIdx.x / threads_per_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += rows_per_warp * nwarps) { - int i = i0 + threadIdx.y * rows_per_warp + row_in_warp; - - if constexpr (need_check) { - i = min(i, i_max); - } - - const block_nvfp4 * bxi = (const block_nvfp4 *) x + kb0 + i * stride + kbx; - const uint32_t * __restrict__ src_qs = reinterpret_cast(bxi->qs); - const int kqs = 16 * kbx; - const int ksc = 4 * kbx; - -#pragma unroll - for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) { - const int2 q0 = get_int_from_table_16(src_qs[2 * sub + 0], kvalues_mxfp4); - const int2 q1 = get_int_from_table_16(src_qs[2 * sub + 1], kvalues_mxfp4); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i * MMQ_MMA_TILE_X_K_NVFP4 + kqs + 4 * sub + 0] = q0.x; - x_qs[i * MMQ_MMA_TILE_X_K_NVFP4 + kqs + 4 * sub + 1] = q1.x; - x_qs[i * MMQ_MMA_TILE_X_K_NVFP4 + kqs + 4 * sub + 2] = q0.y; - x_qs[i * MMQ_MMA_TILE_X_K_NVFP4 + kqs + 4 * sub + 3] = q1.y; - x_df[i * MMQ_MMA_TILE_X_K_NVFP4 + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]); -#else - x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 0] = q0.x; - x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 1] = q1.x; - x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 2] = q0.y; - x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 3] = q1.y; - x_df[i * (2 * MMQ_TILE_NE_K * 2 / QI_NVFP4) + i / (QK_NVFP4_SUB / QI_NVFP4) + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - } -} - -template -static __device__ __forceinline__ void vec_dot_q8_0_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, mmq_y); - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + txs.qs; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - -// #pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q8_0_q8_1_impl - (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k0 % MMQ_TILE_NE_K], - x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + k0/QI8_0], y_df[j*MMQ_TILE_Y_K + (k0/QI8_1) % (MMQ_TILE_NE_K/QI8_1)]); - } - } - } -} - -template -static __device__ __forceinline__ void vec_dot_q8_0_q8_1_mma( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { -#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr data_layout input_layout = get_input_data_layout(); - typedef tile<16, 8, int, input_layout> tile_A; - typedef tile<16, 8, int, input_layout> tile_B; - typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - const half2 * y_ds = (const half2 *) y; - - const int i0 = (threadIdx.y / ntx) * rows_per_warp; - - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { - const int k0 = k00 + k01; - - tile_A A[ntx]; -#pragma unroll - for (int n = 0; n < ntx; ++n) { - load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_0 + k0, MMQ_MMA_TILE_X_K_Q8_0); - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - tile_B B; - load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); - - float dB; - const int j = j0 + tile_C::get_j(0); - if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { - dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; - } else { - dB = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C; - mma(C, A[n], B); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - const int i = i0 + n*tile_A::I + tile_C::get_i(l); - const float dA = x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + k0/QI8_0]; - sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA*dB; - } - } - } - } -#else - typedef tile<16, 8, int> tile_A; - typedef tile< 8, 8, int> tile_B; - typedef tile<16, 8, int> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = 2 * granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - const half2 * y_ds = (const half2 *) y; - - tile_A A[ntx][MMQ_TILE_NE_K/QI8_0]; - float dA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_0]; - - const int i0 = (threadIdx.y/ntx)*rows_per_warp; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { - const int k0 = k00 + k01; - - load_ldmatrix(A[n][k01/QI8_0], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_0 + k0, MMQ_MMA_TILE_X_K_Q8_0); - } - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int i = i0 + n*tile_A::I + tile_C::get_i(2*l); - -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { - const int k0 = k00 + k01; - - dA[n][l][k01/QI8_0] = x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + k0/QI8_0]; - } - } - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { - tile_B B; - float dB[tile_C::ne/2]; - - load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int j = j0 + tile_C::get_j(l); - - if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { - dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; - } else { - dB[l] = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C; - mma(C, A[n][k01/QI8_0], B); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA[n][l/2][k01/QI8_0]*dB[l%2]; - } - } - } - } -#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) -} - -template -static __device__ __forceinline__ void vec_dot_mxfp4_mxfp4_mma(const int * __restrict__ x, - const int * __restrict__ y, - float * __restrict__ sum, - const int k00) { - typedef tile<16, 8, int> tile_A; - typedef tile<8, 8, int> tile_B; - typedef tile<16, 8, float> tile_C; // Output is float for native scaled MMA - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = 2 * granularity; - constexpr int ntx = rows_per_warp / tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J * MMQ_TILE_Y_FP4_K); - - // Match layout from load_tiles_mxfp4_fp4 - const int * x_qs = (const int *) x; - const uint32_t * x_sc = (const uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K); - const int * y_qs = (const int *) y + 4; - const uint32_t * y_sc = (const uint32_t *) y; - - // tile_A has a length of 64 logical values vs. 32 values in block_mxfp4 - tile_A A[ntx][MMQ_TILE_NE_K / (2 * QI_MXFP4)]; - uint32_t scaleA[ntx][MMQ_TILE_NE_K / (2 * QI_MXFP4)]; - - // Block scale - // Each thread has to point to a 4 byte scale value - // https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-block-scaling - - const int i0 = (threadIdx.y / ntx) * rows_per_warp; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 2 * QI_MXFP4) { - const int k0 = k00 + k01; - - load_ldmatrix(A[n][k01 / (2 * QI_MXFP4)], x_qs + (i0 + n * tile_A::I) * MMQ_MMA_TILE_X_K_FP4 + k0, - MMQ_MMA_TILE_X_K_FP4); - - // based on block-scaling document, 2 threads in each quad need to supply to the scale value - const int tidx = threadIdx.x / 4 + (threadIdx.x % 2) * 8; - scaleA[n][k01 / (2 * QI_MXFP4)] = - *(x_sc + (i0 + n * tile_A::I + tidx) * MMQ_MMA_TILE_X_K_FP4 + k0 / (2 * QI_MXFP4)); - } - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx * tile_C::J) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 2 * QI_MXFP4) { - tile_B B; - uint32_t scaleB; // 2xN scales - - load_generic(B, y_qs + j0 * MMQ_TILE_Y_FP4_K + k01, MMQ_TILE_Y_FP4_K); - - scaleB = y_sc[(j0 + threadIdx.x / 4) * MMQ_TILE_Y_FP4_K + k01 / (2 * QI_MXFP4)]; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C; - - mma_block_scaled(C, A[n][k01 / (2 * QI_MXFP4)], B, scaleA[n][k01 / (2 * QI_MXFP4)], scaleB); -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - sum[(j0 / tile_C::J + n) * tile_C::ne + l] += C.x[l]; - } - } - } - } -} - -template -static __device__ __forceinline__ void vec_dot_q8_1_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, mmq_y); - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + txs.qs; - const int * y_qs = (const int *) y + 4; - const half2 * y_ds = (const half2 *) y; - -// #pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q8_1_q8_1_impl - (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], - x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + k0/QI8_1], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - } - } -} - -template -static __device__ __forceinline__ void vec_dot_q8_1_q8_1_mma( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { -#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr data_layout input_layout = get_input_data_layout(); - typedef tile<16, 8, int, input_layout> tile_A; - typedef tile<16, 8, int, input_layout> tile_B; - typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K; - const int * y_qs = (const int *) y + 4; - const half2 * y_dm = (const half2 *) y; - - const int i0 = (threadIdx.y / ntx) * rows_per_warp; - - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { - const int k0 = k00 + k01; - - tile_A A[ntx]; -#pragma unroll - for (int n = 0; n < ntx; ++n) { - load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_1 + k0, MMQ_MMA_TILE_X_K_Q8_1); - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - tile_B B; - load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); - - const int j = j0 + tile_C::get_j(0); - const float2 dsB = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]); - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C; - mma(C, A[n], B); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - const int i = i0 + n*tile_A::I + tile_C::get_i(l); - float2 dmA = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + k0/QI8_1]); - sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.x*dsB.x*C.x[l]; - sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.y*dsB.y; - } - } - } - } -#else - typedef tile<16, 8, int> tile_A; - typedef tile< 8, 8, int> tile_B; - typedef tile<16, 8, int> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = 2 * granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K; - const int * y_qs = (const int *) y + 4; - const half2 * y_dm = (const half2 *) y; - - tile_A A[ntx][MMQ_TILE_NE_K/QI8_1]; - float2 dmA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_1]; - - const int i0 = (threadIdx.y/ntx)*rows_per_warp; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { - const int k0 = k00 + k01; - - load_ldmatrix(A[n][k01/QI8_1], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q8_1 + k0, MMQ_MMA_TILE_X_K_Q8_1); - } - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int i = i0 + n*tile_A::I + tile_C::get_i(2*l); - -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { - const int k0 = k00 + k01; - - dmA[n][l][k01/QI8_1] = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + k0/QI8_1]); - } - } - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { - tile_B B; - float2 dsB[tile_C::ne/2]; - - load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int j = j0 + tile_C::get_j(l); - - dsB[l] = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C; - mma(C, A[n][k01/QI8_1], B); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].x*dsB[l%2].x*C.x[l]; - sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].y*dsB[l%2].y; - } - } - } - } -#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) -} - -// Used for NVFP4, Q3_K, IQ2_S, and IQ2_XS -template -static __device__ __forceinline__ void vec_dot_q8_0_16_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = MMQ_DP4A_TXS_Q8_0_16; - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + txs.qs; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - -// #pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q8_0_16_q8_1_impl( - &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], - &y_qs[j*MMQ_TILE_Y_K + k01], - &x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + k0/(QI8_0/2)], - y_df[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - } - } -} - -// Used for Q3_K, IQ2_S, and IQ2_XS: -template -static __device__ __forceinline__ void vec_dot_q8_0_16_q8_1_mma( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { -#if defined(AMD_MFMA_AVAILABLE) - constexpr data_layout input_layout = get_input_data_layout(); - typedef tile<16, 8, int, input_layout> tile_A; - typedef tile<16, 8, int, input_layout> tile_B; - typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - typedef tile<64, 2, int, input_layout> tile_load; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - - const int i0 = (threadIdx.y / ntx) * rows_per_warp; - - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { - const int k0 = k00 + k01; - - tile_A A[ntx]; -#pragma unroll - for (int n = 0; n < ntx; ++n) { - load_generic(((tile_load *) A)[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q3_K + k0, MMQ_MMA_TILE_X_K_Q3_K); - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - tile_B B[1]; - load_generic(((tile_load *) B)[0], y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); - - const int j = j0 + tile_C::get_j(0); - const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1] / 2; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C; - mma(C, A[n], B[0]); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(l); - sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*MMQ_MMA_TILE_X_K_Q3_K + k0/4] * dB; - } - } - } - } -#elif defined(AMD_WMMA_AVAILABLE) //wmma instructions can handle 16x4 tiles, does not require loading 64x2 tiles - constexpr data_layout input_layout = get_input_data_layout(); - typedef tile<16, 4, int, input_layout> tile_A; - typedef tile<16, 4, int, input_layout> tile_B; - typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - - const int i0 = (threadIdx.y / ntx) * rows_per_warp; - - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { - const int k0 = k00 + k01; - - tile_A A[ntx]; -#pragma unroll - for (int n = 0; n < ntx; ++n) { - load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q3_K + k0, MMQ_MMA_TILE_X_K_Q3_K); - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - tile_B B; - load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); - - const int j = j0 + tile_C::get_j(0); - const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C; - mma(C, A[n], B); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(l); - sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*MMQ_MMA_TILE_X_K_Q3_K + k0/4] * dB; - } - } - } - } -#elif defined(TURING_MMA_AVAILABLE) - - typedef tile<16, 4, int> tile_A; - typedef tile<16, 8, int> tile_A_8; - typedef tile< 8, 4, int> tile_B; - typedef tile<16, 8, int> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = 2 * granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - - const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I); - - tile_A A[ntx][8]; - float dA[ntx][tile_C::ne/2][8]; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) { - const int k0 = k00 + k01; - - load_ldmatrix(((tile_A_8 *) A[n])[k01/8], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q3_K + k0, MMQ_MMA_TILE_X_K_Q3_K); - } - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); - -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { - const int k0 = k00 + k01; - - dA[n][l][k01/4] = x_df[i*MMQ_MMA_TILE_X_K_Q3_K + k0/4]; - } - } - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) { - tile_B B[2]; - float dB[tile_C::ne/2]; - - // Here load_generic is faster than load_ldmatrix. - load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K); - load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K); - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int j = j0 + tile_C::get_j(l); - - dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C[2]; - mma(C[0], A[n][k01/4 + 0], B[0]); - mma(C[1], A[n][k01/4 + 1], B[1]); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - sum[(j0/tile_C::J + n)*tile_C::ne + l] += dB[l%2]*(C[0].x[l]*dA[n][l/2][k01/4 + 0] + C[1].x[l]*dA[n][l/2][k01/4 + 1]); - } - } - } - } -#else - GGML_UNUSED_VARS(x, y, sum, k00); - NO_DEVICE_CODE; -#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE -} - -template static __device__ __forceinline__ void load_tiles_q2_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, mmq_y); - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR2_K); - constexpr int nrows = ggml_cuda_get_physical_warp_size() / threads_per_row; - const int kqsx = threadIdx.x % threads_per_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_q2_K * bxi = (const block_q2_K *) x + kbx0 + i*stride; - - const int x_ql_0 = get_int_b2(bxi->qs, kqsx); - -#pragma unroll - for (int l = 0; l < QR2_K; ++l) { - const int k = (kqsx/8)*32 + l*8 + kqsx % 8; - - const int x_qs_k = (x_ql_0 >> (2*l)) & 0x03030303; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q2_K + k] = x_qs_k; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - const int sc_m = bxi->scales[kqsx]; -#ifdef FAST_FP16_AVAILABLE - const half2 x_dm_ik = __hmul2(bxi->dm, make_half2(sc_m & 0x0F, sc_m >> 4)); -#else - const float2 bxi_dmf = __half22float2(bxi->dm); - const half2 x_dm_ik = make_half2(bxi_dmf.x*(sc_m & 0x0F), bxi_dmf.y*(sc_m >> 4)); -#endif // FAST_FP16_AVAILABLE - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + kqsx] = x_dm_ik; -#else - x_dm[i*(MMQ_TILE_NE_K + 1) + kqsx] = x_dm_ik; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template -static __device__ __forceinline__ void vec_dot_q2_K_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, mmq_y); - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + txs.qs; - const int * y_qs = (const int *) y + 4; - const half2 * y_ds = (const half2 *) y; - - float2 y_df[mmq_x/nwarps]; -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - - y_df[j0/nwarps] = __half22float2(y_ds[j*MMQ_TILE_Y_K]); - } - -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K/2; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - constexpr int ns = 2; - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq( - &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], - &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y, - &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]); - } - } - } - - // Some compilers fail to unroll the loop over k01 if there is a conditional statement for ns in the inner loop. - // As a workaround 2 separate loops are used instead. -#pragma unroll - for (int k01 = MMQ_TILE_NE_K/2; k01 < MMQ_TILE_NE_K; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - constexpr int ns = 1; - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq( - &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], - &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y, - &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]); - } - } - } -} - -template -static __device__ __forceinline__ void vec_dot_q2_K_q8_1_mma( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { -#if defined(AMD_MFMA_AVAILABLE) - constexpr data_layout input_layout = get_input_data_layout(); - typedef tile<16, 8, int, input_layout> tile_A; - typedef tile<16, 8, int, input_layout> tile_B; - typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - typedef tile<64, 2, int, input_layout> tile_load; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2; - const int * y_qs = (const int *) y + 4; - const half2 * y_ds = (const half2 *) y; - - const int i0 = (threadIdx.y / ntx) * rows_per_warp; - - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { - const int k0 = k00 + k01; - - tile_A A[ntx]; -#pragma unroll - for (int n = 0; n < ntx; ++n) { - load_generic(((tile_load *) A)[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q2_K + k0, MMQ_MMA_TILE_X_K_Q2_K); - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - tile_B B[1]; - load_generic(((tile_load *) B)[0], y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); - - const int j = j0 + tile_C::get_j(0); - const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x/2 : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y/2; - const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0 - : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y - : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x); - - tile_C Cm; - if (k01 >= MMQ_TILE_NE_K * 3/4) { - tile_A A1; - A1.x[0] = 0x01010101; - A1.x[1] = 0x01010101; - mma(Cm, A1, B[0]); - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C Cd; - mma(Cd, A[n], B[0]); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(l); - const float2 dm = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + k0/4]); - float tmp = Cd.x[l]*dm.x; - if (k01 >= MMQ_TILE_NE_K * 3/4) { - tmp -= Cm.x[l]*dm.y; - } - sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB; - sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB; - } - } - } - } -#elif defined(AMD_WMMA_AVAILABLE) //wmma instructions can handle 16x4 tiles, does not require loading 64x2 tiles - constexpr data_layout input_layout = get_input_data_layout(); - typedef tile<16, 4, int, input_layout> tile_A; - typedef tile<16, 4, int, input_layout> tile_B; - typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2; - const int * y_qs = (const int *) y + 4; - const half2 * y_ds = (const half2 *) y; - - const int i0 = (threadIdx.y / ntx) * rows_per_warp; - - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { - const int k0 = k00 + k01; - - tile_A A[ntx]; -#pragma unroll - for (int n = 0; n < ntx; ++n) { - load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q2_K + k0, MMQ_MMA_TILE_X_K_Q2_K); - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - tile_B B; - load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); - - const int j = j0 + tile_C::get_j(0); - const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y; - const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0 - : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y - : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x); - - tile_C Cm; - if (k01 >= MMQ_TILE_NE_K * 3/4) { - tile_A A1; -#pragma unroll - for (int l = 0; l < tile_A::ne; ++l) { - A1.x[l] = 0x01010101; - } - mma(Cm, A1, B); - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C Cd; - mma(Cd, A[n], B); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(l); - const float2 dm = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + k0/4]); - float tmp = Cd.x[l]*dm.x; - if (k01 >= MMQ_TILE_NE_K * 3/4) { - tmp -= Cm.x[l]*dm.y; - } - sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB; - sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB; - } - } - } - } -#elif defined(TURING_MMA_AVAILABLE) - - typedef tile<16, 4, int> tile_A; - typedef tile<16, 8, int> tile_A_8; - typedef tile< 8, 4, int> tile_B; - typedef tile<16, 8, int> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = 2 * granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2; - const int * y_qs = (const int *) y + 4; - const half2 * y_ds = (const half2 *) y; - - const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I); - - tile_A A[ntx][8]; - float dA[ntx][tile_C::ne/2][8]; - float mA[ntx][tile_C::ne/2][8]; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { - const int k0 = k00 + k01; - - load_ldmatrix(((tile_A_8 *) A[n])[k01/QI8_1], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q2_K + k0, MMQ_MMA_TILE_X_K_Q2_K); - } - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); - -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1/2) { - const int k0 = k00 + k01; - - const float2 dm = __half22float2(x_dm[i*MMQ_MMA_TILE_X_K_Q2_K + k0/(QI8_1/2)]); - - dA[n][l][k01/(QI8_1/2)] = dm.x; - mA[n][l][k01/(QI8_1/2)] = dm.y; - } - } - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - float2 dB[tile_C::ne/2]; - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int j = j0 + tile_C::get_j(l); - - dB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K]); - } - -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) { - tile_B B[2]; - - // Here load_generic is faster than load_ldmatrix. - load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K); - load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K); - - tile_C Cm[2]; - if (k01 >= MMQ_TILE_NE_K * 3/4) { - tile_A A1; - A1.x[0] = 0x01010101; - A1.x[1] = 0x01010101; - mma(Cm[0], A1, B[0]); - mma(Cm[1], A1, B[1]); - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C Cd[2]; - - mma(Cd[0], A[n][k01/4 + 0], B[0]); - mma(Cd[1], A[n][k01/4 + 1], B[1]); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - float tmp = Cd[0].x[l]*dA[n][l/2][k01/4 + 0] + Cd[1].x[l]*dA[n][l/2][k01/4 + 1]; - if (k01 >= MMQ_TILE_NE_K * 3/4) { - tmp -= Cm[0].x[l]*mA[n][l/2][k01/4 + 0] + Cm[1].x[l]*mA[n][l/2][k01/4 + 1]; - } - sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*(k01 < MMQ_TILE_NE_K/2 ? dB[l%2].x : dB[l%2].y); - } - } - } - -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K * 3/4; k01 += QI8_1) { - float2 sB[tile_C::ne/2]; - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int j = j0 + tile_C::get_j(l); - - sB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]); - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 0]*sB[l%2].x; - sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 1]*sB[l%2].y; - } - } - } - } -#else - GGML_UNUSED_VARS(x, y, sum, k00); - NO_DEVICE_CODE; -#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE -} - -template static __device__ __forceinline__ void load_tiles_q3_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); - int * x_sc = (int *) (x_df + txs.dm); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR3_K); - constexpr int nrows = warp_size / threads_per_row; - const int kqsx = threadIdx.x % threads_per_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride; - - const int x_ql_0 = get_int_b2(bxi->qs, kqsx); - const int x_qh_0 = get_int_b2(bxi->hmask, kqsx % (QI3_K/2)) >> (4 * (kqsx / (QI3_K/2))); - -#pragma unroll - for (int l = 0; l < QR3_K; ++l) { - const int k = (kqsx/8)*32 + l*8 + kqsx % 8; - - const int x_ql_k = (x_ql_0 >> (2*l)) & 0x03030303; - const int x_qh_k = ((x_qh_0 >> l) << 2) & 0x04040404; - - const int x_qs_k = __vsubss4(x_ql_k | x_qh_k, 0x04040404); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + k] = x_qs_k; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - } - - constexpr int rows_per_warp = warp_size / 4; -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) { - int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/4; - - if (need_check) { - i = min(i, i_max); - } - - const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride; - - const int ksc = threadIdx.x % 4; - - const int ksc_low = ksc % (QI3_K/8); - const int shift_low = 4 * (ksc / (QI3_K/8)); - const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F; - - const int ksc_high = QI3_K/8; - const int shift_high = 2 * ksc; - const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030; - - const int sc = __vsubss4(sc_low | sc_high, 0x20202020); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - const int8_t * sc8 = (const int8_t *) ≻ - const float d = bxi->d; - -#pragma unroll - for (int l = 0; l < int(sizeof(int)); ++l) { - x_df[i*MMQ_MMA_TILE_X_K_Q3_K + sizeof(int)*ksc + l] = d*sc8[l]; - } -#else - x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = sc; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - -#if !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)) -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) { - int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y; - - if (need_check) { - i = min(i, i_max); - } - - const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride; - - x_df[i] = bxi->d; - } -#endif // !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)) || defined(AMD_WMMA_AVAILABLE) -} - -template -static __device__ __forceinline__ void vec_dot_q3_K_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, mmq_y); - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + txs.qs; - const int * x_sc = (const int *) x_df + txs.dm; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - -// #pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - const int8_t * scales = ((const int8_t *) (x_sc + i*(MMQ_TILE_NE_K/8) + i/8)) + k0/4; - - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q3_K_q8_1_impl_mmq( - &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], scales, - x_df[i], y_df[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - } - } -} - -static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, const int ksc) { - // scale arrangement after the following two lines: - // - ksc == 0: sc0, sc1, sc2, sc3 - // - ksc == 1: sc4, sc5, sc6, sc7 - // - ksc == 2: m0, m1, m2, m3 - // - ksc == 3: m4, m5, m6, m7 - return ((scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F) | // lower 4 bits - ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits -} - -template static __device__ __forceinline__ void load_tiles_q4_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, mmq_y); - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + txs.qs); - int * x_sc = (int *) (x_dm + txs.dm); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_K); - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - - if (need_check) { - i = min(i, i_max); - } - - const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride; - const int qs0 = get_int_b4(bxi->qs, txi); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 16*(txi/8) + txi % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 16*(txi/8) + txi % 8 + 8] = (qs0 >> 4) & 0x0F0F0F0F; -#else - x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr int rows_per_warp = warp_size / 2; -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) { -#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - // Need if on AMD instead of % because warp_size == 64 - // This causes double work and throughput loss (MI300X) - // H100 loses about 100 t/s with 'if' condition over '%' - int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2; - if (i < mmq_y) { -#else - int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % mmq_y; - { -#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - if (need_check) { - i = min(i, i_max); - } - - const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride; - - const int * scales = (const int *) bxi->scales; - const int ksc = threadIdx.x % 2; - - const int sc32 = unpack_scales_q45_K(scales, ksc + 0); - const int m32 = unpack_scales_q45_K(scales, ksc + 2); - - const uint8_t * sc8 = (const uint8_t *) &sc32; - const uint8_t * m8 = (const uint8_t *) &m32; - - const half2 dm = bxi->dm * make_half2(1.0f, -1.0f); - - #pragma unroll - for (int l = 0; l < sizeof(int); ++l) { - x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]); - } - } - } -#else -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) { - int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y; - - if (need_check) { - i = min(i, i_max); - } - - const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride; - - x_dm[i] = bxi->dm; - } - constexpr int rows_per_warp = warp_size / 4; -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) { - int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % mmq_y; - - if (need_check) { - i = min(i, i_max); - } - - const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8); - - const int * scales = (const int *) bxi->scales; - - const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8); - const int scales8 = unpack_scales_q45_K(scales, ksc); - - x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8; - } -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) -} - -template -static __device__ __forceinline__ void vec_dot_q4_K_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, mmq_y); - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + txs.qs; - const int * x_sc = (const int *) x_dm + txs.dm; - const int * y_qs = (const int *) y + 4; - const half2 * y_ds = (const half2 *) y; - -// #pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_K*VDR_Q4_K_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - const uint8_t * sc = (const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/32] + 2*(k01/16); - - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q4_K_q8_1_impl_mmq( - &x_qs[i*(MMQ_TILE_NE_K + 1) + k0/2], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8, - x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - } - } -} - -template static __device__ __forceinline__ void load_tiles_q5_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, mmq_y); - int * x_qs = (int *) x_tile; - half2 * x_dm = (half2 *) (x_qs + txs.qs); - int * x_sc = (int *) (x_dm + txs.dm); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_K); - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - - if (need_check) { - i = min(i, i_max); - } - - const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; - const int ky = QR5_K*txi; - - const int ql = get_int_b4(bxi->qs, txi); - const int ql0 = (ql >> 0) & 0x0F0F0F0F; - const int ql1 = (ql >> 4) & 0x0F0F0F0F; - - const int qh = get_int_b4(bxi->qh, txi % (QI5_K/4)); - const int qh0 = ((qh >> (2 * (txi / (QI5_K/4)) + 0)) << 4) & 0x10101010; - const int qh1 = ((qh >> (2 * (txi / (QI5_K/4)) + 1)) << 4) & 0x10101010; - - const int kq0 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + 0; - const int kq1 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + QI5_K/4; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kq0] = ql0 | qh0; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + kq1] = ql1 | qh1; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = ql0 | qh0; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = ql1 | qh1; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr int rows_per_warp = warp_size / 2; -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) { -#if defined(AMD_MFMA_AVAILABLE) - // Need if on AMD instead of % because warp_size == 64 - // This causes double work and throughput loss (MI300X) - // H100 loses about 100 t/s with 'if' condition over '%' - int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2; - if (i < mmq_y) { -#else - int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % mmq_y; - { -#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - if (need_check) { - i = min(i, i_max); - } - - const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; - - const int * scales = (const int *) bxi->scales; - const int ksc = threadIdx.x % 2; - - const int sc32 = unpack_scales_q45_K(scales, ksc + 0); - const int m32 = unpack_scales_q45_K(scales, ksc + 2); - - const uint8_t * sc8 = (const uint8_t *) &sc32; - const uint8_t * m8 = (const uint8_t *) &m32; - - const half2 dm = bxi->dm * make_half2(1.0f, -1.0f); - -#pragma unroll - for (int l = 0; l < int(sizeof(int)); ++l) { - x_dm[i*MMQ_MMA_TILE_X_K_Q8_1 + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]); - } - } - } -#else -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) { - int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y; - - if (need_check) { - i = min(i, i_max); - } - - const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; - - x_dm[i] = bxi->dm; - } - - constexpr int rows_per_warp = warp_size / 4; -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) { - int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % mmq_y; - - if (need_check) { - i = min(i, i_max); - } - - const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride; - - const int * scales = (const int *) bxi->scales; - - const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8); - const int scales8 = unpack_scales_q45_K(scales, ksc); - - x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8; - } -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) -} - -template -static __device__ __forceinline__ void vec_dot_q5_K_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, mmq_y); - const int * x_qs = (const int *) x; - const half2 * x_dm = (const half2 *) x_qs + txs.qs; - const int * x_sc = (const int *) x_dm + txs.dm; - const int * y_qs = (const int *) y + 4; - const half2 * y_ds = (const half2 *) y; - -// #pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR5_K*VDR_Q5_K_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - const uint8_t * sc = ((const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k00/32]) + 2*(k01/16); - - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q5_K_q8_1_impl_mmq( - &x_qs[i*(QR5_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8, - x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - } - } -} - -template static __device__ __forceinline__ void load_tiles_q6_K( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); - int * x_sc = (int *) (x_df + MMQ_TILE_NE_K/QI6_K); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); - int * x_sc = (int *) (x_df + txs.dm); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR6_K); - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - - if (need_check) { - i = min(i, i_max); - } - - const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride; - - const int ql = get_int_b2(bxi->ql, txi); - const int ql0 = (ql >> 0) & 0x0F0F0F0F; - const int ql1 = (ql >> 4) & 0x0F0F0F0F; - - const int qh = get_int_b2(bxi->qh, (QI6_K/4) * (txi / (QI6_K/2)) + txi % (QI6_K/4)); - const int qh0 = ((qh >> ((txi & 0x08) >> 2)) << 4) & 0x30303030; - const int qh1 = (qh >> ((txi & 0x08) >> 2)) & 0x30303030; - - const int kq0 = 2*txi - txi % (QI6_K/2) + 0; - const int kq1 = 2*txi - txi % (QI6_K/2) + QI6_K/2; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q6_K + kq0] = __vsubss4(ql0 | qh0, 0x20202020); - x_qs[i*MMQ_MMA_TILE_X_K_Q6_K + kq1] = __vsubss4(ql1 | qh1, 0x20202020); -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020); - x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*warp_size) { - int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % mmq_y; - - if (need_check) { - i = min(i, i_max); - } - - const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q6_K] = bxi->d; -#else - x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K] = bxi->d; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - constexpr int rows_per_warp = warp_size / 4; -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps*rows_per_warp) { - int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % mmq_y; - - if (need_check) { - i = min(i, i_max); - } - - const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_sc[i*MMQ_MMA_TILE_X_K_Q6_K + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8)); -#else - x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + threadIdx.x%(MMQ_TILE_NE_K/8)] = get_int_b2(bxi->scales, threadIdx.x%(QI6_K/8)); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template -static __device__ __forceinline__ void vec_dot_q6_K_q8_1_dp4a( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, mmq_y); - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + txs.qs; - const int * x_sc = (const int *) x_df + txs.dm; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - -// #pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR6_K*VDR_Q6_K_Q8_1_MMQ) { - const int k0 = k00 + k01; - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - const int8_t * sc = ((const int8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/16]); - - sum[j0/nwarps*mmq_y/warp_size + i0/warp_size] += vec_dot_q6_K_q8_1_impl_mmq( - &x_qs[i*(QR6_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, - x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K], &y_df[j*MMQ_TILE_Y_K + k01/QI8_1]); - } - } - } -} - -template -static __device__ __forceinline__ void vec_dot_q6_K_q8_1_mma( - const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { -#if defined(AMD_MFMA_AVAILABLE) - constexpr data_layout input_layout = get_input_data_layout(); - typedef tile<16, 8, int, input_layout> tile_A; - typedef tile<16, 8, int, input_layout> tile_B; - typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - typedef tile<64, 2, int, input_layout> tile_load; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; - const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - - const int i0 = (threadIdx.y / ntx) * rows_per_warp; - - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { - const int k0 = k00 + k01; - - tile_A A[ntx]; -#pragma unroll - for (int n = 0; n < ntx; ++n) { - load_generic(((tile_load *) A)[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + k0, MMQ_MMA_TILE_X_K_Q6_K); - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - tile_B B[1]; - load_generic(((tile_load *) B)[0], y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); - - const int j = j0 + tile_C::get_j(0); - const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1] / 2; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C; - mma(C, A[n], B[0]); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(l); - const int8_t * sc = (const int8_t *) (x_sc + i*MMQ_MMA_TILE_X_K_Q6_K + k00/16); - sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*MMQ_MMA_TILE_X_K_Q6_K] * dB; - } - } - } - } -#elif defined(AMD_WMMA_AVAILABLE) //wmma instructions can handle 16x4 tiles, does not require loading 64x2 tiles - constexpr data_layout input_layout = get_input_data_layout(); - typedef tile<16, 4, int, input_layout> tile_A; - typedef tile<16, 4, int, input_layout> tile_B; - typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; - const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - - const int i0 = (threadIdx.y / ntx) * rows_per_warp; - - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) { - const int k0 = k00 + k01; - - tile_A A[ntx]; -#pragma unroll - for (int n = 0; n < ntx; ++n) { - load_generic(A[n], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + k0, MMQ_MMA_TILE_X_K_Q6_K); - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - tile_B B; - load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); - - const int j = j0 + tile_C::get_j(0); - const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C; - mma(C, A[n], B); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(l); - const int8_t * sc = (const int8_t *) (x_sc + i*MMQ_MMA_TILE_X_K_Q6_K + k00/16); - sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*MMQ_MMA_TILE_X_K_Q6_K] * dB; - } - } - } - } -#elif defined(TURING_MMA_AVAILABLE) - - typedef tile<16, 4, int> tile_A; - typedef tile< 8, 4, int> tile_B; - typedef tile<16, 8, int> tile_C; - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int rows_per_warp = 2 * granularity; - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. - - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); - - const int * x_qs = (const int *) x; - const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2; - const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K; - const int * y_qs = (const int *) y + 4; - const float * y_df = (const float *) y; - - const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I); - - tile_A A[ntx][8]; - int scA[ntx][tile_C::ne/2][8]; - float dA[ntx][tile_C::ne/2]; - -#pragma unroll - for (int n = 0; n < ntx; ++n) { -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) { - const int k0 = k00 + k01; - - load_ldmatrix(A[n][k01/4 + 0], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + (k0 + 0), MMQ_MMA_TILE_X_K_Q6_K); - load_ldmatrix(A[n][k01/4 + 1], x_qs + (i0 + n*tile_A::I)*MMQ_MMA_TILE_X_K_Q6_K + (k0 + tile_A::J), MMQ_MMA_TILE_X_K_Q6_K); - } - -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 16) { - const int k0 = k00 + k01; - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); - - const int sc_packed = x_sc[i*MMQ_MMA_TILE_X_K_Q6_K + k0/16]; - const int8_t * sc = (const int8_t *) &sc_packed; - -#pragma unroll - for (int ksc = 0; ksc < sizeof(int); ++ksc) { - scA[n][l][k01/4 + ksc] = sc[ksc]; - } - } - } - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int i = i0 + n*tile_C::I + tile_C::get_i(2*l); - - dA[n][l] = x_df[i*MMQ_MMA_TILE_X_K_Q6_K]; - } - } - -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { - float tmp[ntx][tile_C::ne] = {{0.0f}}; - -#pragma unroll - for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) { - tile_B B[2]; - float dB[tile_C::ne/2]; - - // Here load_generic is faster than load_ldmatrix. - load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + 0 + k01, MMQ_TILE_Y_K); - load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + tile_B::J + k01, MMQ_TILE_Y_K); - -#pragma unroll - for (int l = 0; l < tile_C::ne/2; ++l) { - const int j = j0 + tile_C::get_j(l); - - dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1]; - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { - tile_C C[2]; - mma(C[0], A[n][k01/4 + 0], B[0]); - mma(C[1], A[n][k01/4 + 1], B[1]); - -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - tmp[n][l] += (C[0].x[l]*scA[n][l/2][k01/4 + 0] + C[1].x[l]*scA[n][l/2][k01/4 + 1])*dB[l%2]; - } - } - } - -#pragma unroll - for (int n = 0; n < ntx; ++n) { -#pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { - sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp[n][l]*dA[n][l/2]; - } - } - } -#else - GGML_UNUSED_VARS(x, y, sum, k00); - NO_DEVICE_CODE; -#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE -} - -template static __device__ __forceinline__ void load_tiles_iq4_nl( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_NL, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_NL); - constexpr int nrows = warp_size / threads_per_row; - const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - const int kbx = txi / QI4_NL; - const int kqsx = txi % QI4_NL; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); - - if (need_check) { - i = min(i, i_max); - } - - const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbx; - - const int aux_q4 = get_int_b2(bxi->qs, kqsx); - const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl); - const int k0 = kbx * (2 * QI4_NL) + kqsx; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 0] = v.x; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + QI4_NL] = v.y; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI4_NL] = v.y; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_NL; - constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; - const int kbxd = threadIdx.x % blocks_per_tile_x_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) { - int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbxd; - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = __half2float(bxi->d); -#else - x_df[i*(MMQ_TILE_NE_K/QI4_NL) + i/QI4_NL + kbxd] = __half2float(bxi->d); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template static __device__ __forceinline__ void load_tiles_iq2_xxs( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XXS, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XXS)) / 2; - constexpr int nrows = warp_size / threads_per_row; - const int kqsx = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) { - int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_iq2_xxs * bxi = (const block_iq2_xxs *) x + kbx0 + i*stride; - - const int q2 = get_int_b2(bxi->qs, 2*kqsx+0); - const uint8_t * aux8 = (const uint8_t *) &q2; - const uint32_t aux32 = get_int_b2(bxi->qs, 2*kqsx+1); - -#pragma unroll - for (int l = 0; l < QR2_XXS; ++l) { - const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[l]]; - const uint32_t signs = unpack_ksigns(aux32 >> (7 * l)); - - const int signs0 = __vcmpne4(signs & 0x08040201, 0); - const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0); - - const int signs1 = __vcmpne4(signs & 0x80402010, 0); - const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 0)] = grid0; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 1)] = grid1; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - const int ls = aux32 >> 27 | 1; // (scale * 2 + 1) - const float d = bxi->d; -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4 -#else - x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4 -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template static __device__ __forceinline__ void load_tiles_iq2_xs( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = MMQ_DP4A_TXS_Q8_0_16; - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XS)) / 2; - constexpr int nrows = warp_size / threads_per_row; - const int kqsx = threadIdx.x % threads_per_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) { - int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_iq2_xs * bxi = (const block_iq2_xs *) x + kbx0 + i*stride; - - const int2 q2_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1)); - const uint16_t * q2 = (const uint16_t *) &q2_packed; - - #pragma unroll - for (int l = 0; l < QR2_XS; ++l) { - const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l] & 0x1FF]; - const uint32_t signs = unpack_ksigns(q2[l] >> 9); - - const int signs0 = __vcmpne4(signs & 0x08040201, 0); - const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); - - const int signs1 = __vcmpne4(signs & 0x80402010, 0); - const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 0)] = grid_l; - x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 1)] = grid_h; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - const int ls = bxi->scales[kqsx]; - const float d = bxi->d; -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; - x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; -#else - x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; - x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template static __device__ __forceinline__ void load_tiles_iq2_s( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_S, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_S)) / 2; - constexpr int nrows = warp_size / threads_per_row; - const int kqsx = threadIdx.x % threads_per_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) { - int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_iq2_s * bxi = (const block_iq2_s *) x + kbx0 + i*stride; - - const int qs_packed = get_int_b2(bxi->qs, kqsx); - const uint8_t * qs = (const uint8_t *) &qs_packed; - - const int qh = bxi->qh[kqsx]; - - const int signs_packed_32 = get_int_b2(bxi->qs, QK_K/32 + kqsx); - const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; - -#pragma unroll - for (int l = 0; l < QR2_S; ++l) { - const int * grid_pos = (const int *)(iq2s_grid + (qs[l] | ((qh << (8-2*l)) & 0x300))); - - const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000); - const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000); - - const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0); - const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 0)] = grid_l; - x_qs[i*MMQ_MMA_TILE_X_K_Q3_K + 8*kqsx + (2*l + 1)] = grid_h; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - const int ls = bxi->scales[kqsx]; - const float d = bxi->d; -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; - x_df[i*MMQ_MMA_TILE_X_K_Q3_K + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; -#else - x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4; - x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template static __device__ __forceinline__ void load_tiles_iq3_xxs( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_XXS, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_XXS)) / 2; - constexpr int nrows = warp_size / threads_per_row; - const int kqsx = threadIdx.x % threads_per_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) { - int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_iq3_xxs * bxi = (const block_iq3_xxs *) x + kbx0 + i*stride; - - const int2 q3_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1)); - const uint8_t * q3 = (const uint8_t *) &q3_packed; - const uint32_t aux32 = get_int_b2(bxi->qs, QK_K/16 + kqsx); - -#pragma unroll - for (int l = 0; l < QR3_XXS; ++l) { - const int2 grid_pos = make_int2(iq3xxs_grid[q3[2*l+0]], iq3xxs_grid[q3[2*l+1]]); - const uint32_t signs = unpack_ksigns(aux32 >> (7*l)); - - const int signs0 = __vcmpne4(signs & 0x08040201, 0); - const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); - - const int signs1 = __vcmpne4(signs & 0x80402010, 0); - const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 0)] = grid_l; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l + 1)] = grid_h; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - const int ls = aux32 >> 28; - const float d = bxi->d; -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = (ls*d + d/2)/2; -#else - x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = (ls*d + d/2)/2; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } -} - -template static __device__ __forceinline__ void load_tiles_iq3_s( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - - constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_S)) / 2; - constexpr int nrows = warp_size / threads_per_row; - const int kqsx = threadIdx.x % threads_per_row; - -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) { - int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; - - if (need_check) { - i = min(i, i_max); - } - - const block_iq3_s * bxi = (const block_iq3_s *) x + kbx0 + i*stride; - - const int2 qs_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1)); - const uint8_t * qs = (const uint8_t *) &qs_packed; - - const int qh = bxi->qh[kqsx]; - - const int signs_packed_32 = get_int_b2(bxi->signs, kqsx); - const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; - -#pragma unroll - for (int l = 0; l < QR3_S; ++l) { - const int2 grid_pos = make_int2( - iq3s_grid[qs[2*l+0] | ((qh << (8 - 2*l)) & 0x100)], - iq3s_grid[qs[2*l+1] | ((qh << (7 - 2*l)) & 0x100)]); - - const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000); - const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000); - - const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); - const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l+0)] = grid_l; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + 8*kqsx + (2*l+1)] = grid_h; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid_l; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid_h; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } - - const int ls = 1 + 2*((bxi->scales[kqsx/2] >> (((2*kqsx) << 1) & 0x04)) & 0x0F); - const float d = bxi->d; -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kqsx] = ls*d; +static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { +#ifdef GGML_USE_HIP +#ifdef GCN + return ggml_cuda_mmq_get_config_gcn(type, J, fallback); +#elif defined(CDNA) + return ggml_cuda_mmq_get_config_cdna(type, J, fallback); +#elif defined(RDNA4) + return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); +#elif defined(RDNA3_5) + return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); +#elif defined(RDNA3) + return ggml_cuda_mmq_get_config_rdna3(type, J, fallback); +#else + return ggml_cuda_mmq_get_config_rdna2(type, J, fallback); +#endif // CDNA #else - x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = ls*d; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +#ifdef BLACKWELL_MMA_AVAILABLE + // only src1 at Q4 uses the native FP4 config, higher precisions keep src1 at Q8_1 + if (prec_src1 != GGML_PREC_Q4 && (type == GGML_TYPE_NVFP4 || type == GGML_TYPE_MXFP4)) { + return ggml_cuda_mmq_get_config_ampere(type, J, fallback); } + return ggml_cuda_mmq_get_config_blackwell(type, J, fallback); +#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA + return ggml_cuda_mmq_get_config_ampere(type, J, fallback); +#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A + return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback); +#else + return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback); +#endif // BLACKWELL_MMA_AVAILABLE +#endif // GGML_USE_HIP + GGML_UNUSED_VARS(type, J, fallback, prec_src1); } -template static __device__ __forceinline__ void load_tiles_iq1_s( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, mmq_y); - int * x_qs = (int *) x_tile; - half2 * x_ds = (half2 *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).type; +} - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_S); - constexpr int nrows = warp_size / threads_per_row; - const int kqsx = threadIdx.x % threads_per_row; +static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type; +} -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * nrows) { - int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; +static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).nthreads; +} - if (need_check) { - i = min(i, i_max); - } +static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).occupancy; +} - const block_iq1_s * bxi = (const block_iq1_s *) x + kbx0 + i*stride; +static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).I; +} - const int qs_packed = get_int_b2(bxi->qs, kqsx); - const uint8_t * qs = (const uint8_t *) &qs_packed; +static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).I; +} - const int qh = bxi->qh[kqsx]; +static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).J; +} - #pragma unroll - for (int l = 0; l < QR1_S/2; ++l) { - const int grid = iq1s_grid_gpu[qs[l] | (((qh >> (3*l)) & 0x07) << 8)]; +static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).J; +} - const int grid0 = (grid >> 0) & 0x0F0F0F0F; - const int grid1 = (grid >> 4) & 0x0F0F0F0F; +static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).sram_layout; +} -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 8*kqsx + (2*l+0)] = grid0; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_1 + 8*kqsx + (2*l+1)] = grid1; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } +static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).sram_layout; +} - const float d1q = __half2float(bxi->d) * (((qh >> 11) & 0x0E) + 1); - const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000); +static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).K_vram; +} -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_ds[i*MMQ_MMA_TILE_X_K_Q8_1 + kqsx] = make_half2(d1q, d1q*delta); -#else - x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } +static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).K_vram; } -template static __device__ __forceinline__ void load_tiles_iq4_xs( - const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); +static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).stream_k; +} -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); -#else - constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_XS, mmq_y); - int * x_qs = (int *) x_tile; - float * x_df = (float *) (x_qs + txs.qs); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).stream_k; +} - constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_XS); - constexpr int nrows = warp_size / threads_per_row; - const int kqsx = threadIdx.x % threads_per_row; +static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).fallback; +} -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { - int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); +static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).fallback; +} - if (need_check) { - i = min(i, i_max); - } +// --------------------------------------------------------------------------------------------- - const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride; +static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc, prec_src1)); +} - const int aux_q4 = get_int_b4(bxi->qs, kqsx); - const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl); - const int k0 = 8 * (kqsx / 4) + kqsx % 4; +static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, prec_src1)); +} -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 0] = v.x; - x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 4] = v.y; -#else - x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; - x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 4] = v.y; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - } +static __host__ bool ggml_cuda_mmq_needs_fallback(const int64_t nrows_x) { + return nrows_x % 128 != 0; +} - constexpr int rows_per_warp = warp_size / 8; -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) { - int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / (MMQ_TILE_NE_K/4); +static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).rows_per_warp(); +} - if (need_check) { - i = min(i, i_max); - } +#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_0 + I/QI4_0, 0} +#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_1 + I/QI4_1, 0} +#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_0 + I/(QI8_0/2), 0} +#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*4/QI8_0 + I/(QI8_0/4), 0} +#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_1 + I/(QI8_1/2), 0} +#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K + I, 0} +#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I, I*MMQ_TILE_NE_K/8 + I/8} +#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_K, I*MMQ_TILE_NE_K/8 + I/8} +#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI5_K + I/QI5_K, I*MMQ_TILE_NE_K/8 + I/8} +#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI6_K + I/QI6_K, I*MMQ_TILE_NE_K/8 + I/8} + +static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int I) { + switch (type) { + case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q2_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0; + case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1; + case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q5_1: return MMQ_DP4A_TXS_Q8_1; + case GGML_TYPE_Q8_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_MXFP4: return MMQ_DP4A_TXS_Q8_1; + case GGML_TYPE_NVFP4: return MMQ_DP4A_TXS_Q8_0_16; + case GGML_TYPE_Q2_K: return MMQ_DP4A_TXS_Q2_K; + case GGML_TYPE_Q3_K: return MMQ_DP4A_TXS_Q3_K; + case GGML_TYPE_Q4_K: return MMQ_DP4A_TXS_Q4_K; + case GGML_TYPE_Q5_K: return MMQ_DP4A_TXS_Q5_K; + case GGML_TYPE_Q6_K: return MMQ_DP4A_TXS_Q6_K; + case GGML_TYPE_IQ2_XXS: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16; + case GGML_TYPE_IQ2_S: return MMQ_DP4A_TXS_Q8_0_16; + case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0; + default: return tile_x_sizes{0, 0, 0}; + } +} - const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride; +// FIXME temporary until all combinations of data types and GPUs can use the MMA data layout +static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config & config, const int cc) { + if (config.use_mma_data_layout(cc)) { + return config.I * ggml_cuda_mmq_get_sram_stride(config.sram_layout) * 4; + } + const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(config.type, config.I); + return (txs.qs + txs.dm + txs.sc) * 4; +} - const float d = __half2float(bxi->d); +// ------------------------------------------------------------ - const int ls = ((bxi->scales_l[(threadIdx.x % 8)/2] >> (4*(threadIdx.x % 2))) & 0x0F) - | (((bxi->scales_h >> (2*(threadIdx.x % 8))) & 0x03) << 4); +// Weight types with a fused gate/up/GLU path (see ggml_cuda_mmq_get_util_funcs). +static constexpr __host__ __device__ bool ggml_cuda_mmq_fusion_supported(const ggml_type type) { + return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_XS; +} -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + threadIdx.x % 8] = d * (ls - 32); -#else - x_df[i*(MMQ_TILE_NE_K/4) + i/4 + threadIdx.x % 8] = d * (ls - 32); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) +// Fused gate/up/GLU: the weight tile alternates chunks of this many up and gate rows, so rows r and r + 8 of an MMA tile (same thread) pair up. +static constexpr int GGML_CUDA_MMQ_FUSION_ROWS = 8; + +// Source of weight tile row i; with has_fusion the up or gate row that the interleaved tile holds there. +template +static __device__ __forceinline__ const block_t * ggml_cuda_mmq_x_row( + const char * __restrict__ x, const char * __restrict__ x_gate, const int kbx0, const int i, const int stride) { + if constexpr (has_fusion) { + constexpr int R = GGML_CUDA_MMQ_FUSION_ROWS; + const char * x_src = (i / R) % 2 == 0 ? x : x_gate; + return (const block_t *) x_src + kbx0 + ((i / (2*R))*R + i % R)*stride; + } else { + GGML_UNUSED(x_gate); + return (const block_t *) x + kbx0 + i*stride; } } -template -static __device__ __forceinline__ void mmq_write_back_dp4a( +#include "mmq-load-tiles.cuh" +#include "mmq-vec-dot.cuh" + +template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst, - const int stride, const int i_max, const int j_max) { - constexpr int nwarps = mmq_get_nwarps_device(); + const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max, + const ggml_glu_op glu_op, const float glu_limit) { + GGML_UNUSED_VARS(glu_op, glu_limit); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); + static_assert(I % warp_size == 0, "DP4A MMQ tile height must be a multiple of the physical warp size"); + + const bool y_scale_used = y_scale != nullptr; #pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { + for (int j0 = 0; j0 < J; j0 += nwarps) { const int j = j0 + threadIdx.y; if (j > j_max) { @@ -3309,290 +476,482 @@ static __device__ __forceinline__ void mmq_write_back_dp4a( } #pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { + for (int i0 = 0; i0 < I; i0 += warp_size) { const int i = i0 + threadIdx.x; - if (need_check && i > i_max) { + if (fallback && i > i_max) { continue; } - dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (mmq_y/warp_size) + i0/warp_size]; + if constexpr (type == GGML_TYPE_NVFP4) { + if (y_scale_used) { + dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + } else { + dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + } + } else { + dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + GGML_UNUSED(y_scale_used); + } } } } -template -static __device__ __forceinline__ void mmq_write_back_mma( - const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst, - const int stride, const int i_max, const int j_max) { - - constexpr int granularity = mmq_get_granularity_device(mmq_x); - constexpr int nwarps = mmq_get_nwarps_device(); +// has_fusion: accumulators l and l + ne/2 hold the up and gate values of one output element, the GLU of the two is written. +template +static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( + const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst, + const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max, + const ggml_glu_op glu_op, const float glu_limit) { #if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr int tileC_IJ = mmq_get_granularity_device(0); - typedef tile tile_C; - constexpr int rows_per_warp = granularity; + typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; #else - typedef tile<16, 8, int> tile_C; - constexpr int rows_per_warp = 2 * granularity; -#endif // defined(AMD_MFMA_AVAILABLE) - constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + typedef tile<16, 8, int> tile_C; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + static_assert(!has_fusion || tile_C::I == 2*GGML_CUDA_MMQ_FUSION_ROWS, "each C tile must hold one chunk of up rows and one of gate rows"); + + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1); + constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I); -#if defined(TURING_MMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - static_assert(nwarps*tile_C::I == mmq_y, "nwarps*tile_C::I != mmq_y"); -#else - GGML_UNUSED(nwarps); -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + const bool y_scale_used = y_scale != nullptr; #pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += ntx*tile_C::J) { + for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { #pragma unroll for (int n = 0; n < ntx; ++n) { #pragma unroll - for (int l = 0; l < tile_C::ne; ++l) { + for (int l = 0; l < (has_fusion ? tile_C::ne/2 : tile_C::ne); ++l) { const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l); if (j > j_max) { continue; } - const int i = i0 + n*tile_C::I + tile_C::get_i(l); + const int i = (i0 + n*tile_C::I)/(has_fusion ? 2 : 1) + tile_C::get_i(l); - if (need_check && i > i_max) { + if (fallback && i > i_max) { continue; } - dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + if constexpr (has_fusion) { + const int sum_index = (j0/tile_C::J + n)*tile_C::ne + l; + dst[ids_dst[j]*stride + i] = ggml_cuda_op_glu_single(glu_op, sum[sum_index + tile_C::ne/2], sum[sum_index], glu_limit); + } else if constexpr (type == GGML_TYPE_NVFP4) { + if (y_scale_used) { + dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]; + } else { + dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + } + } else { + dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + GGML_UNUSED(y_scale_used); + } } } } + + GGML_UNUSED_VARS(y_scale_used, glu_op, glu_limit); } // ------------------------------------------------------------------------------------------------------------------------------------- -template -struct mmq_type_traits; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q1_0_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q1_0; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q4_0_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_0; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_0_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q4_1_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_1; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_1_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q5_0_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_0; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; -}; +// TODO remove this struct and use ggml_cuda_mmq_sram_layout instead. +struct ggml_cuda_mmq_util_funcs { + int vdr; + ggml_cuda_mmq_load_tiles_t load_tiles; + ggml_cuda_mmq_vec_dot_t vec_dot; + ggml_cuda_mmq_write_back_t write_back; -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q5_1_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_1; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_1_q8_1_dp4a; + constexpr __host__ __device__ ggml_cuda_mmq_util_funcs( + int vdr, ggml_cuda_mmq_load_tiles_t load_tiles, ggml_cuda_mmq_vec_dot_t vec_dot, ggml_cuda_mmq_write_back_t write_back) : + vdr(vdr), load_tiles(load_tiles), vec_dot(vec_dot), write_back(write_back) {} }; -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q8_0_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q8_0; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; -}; +template +static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() { + static_assert(!has_fusion || ggml_cuda_mmq_fusion_supported(type), "no fused loader for this type"); + if (!ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) { + switch (type) { + case GGML_TYPE_Q1_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q1_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q1_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q2_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q2_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q2_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q4_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q4_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q4_0, + ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q4_1: + return ggml_cuda_mmq_util_funcs( + VDR_Q4_1_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q4_1, + ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q5_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q5_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q5_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q5_1: + return ggml_cuda_mmq_util_funcs( + VDR_Q5_1_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q5_1, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q8_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q8_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q8_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_Q2_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q2_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q2_K, + ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q3_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q3_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q3_K, + ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q4_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q4_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q4_K, + ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q5_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q5_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q5_K, + ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q6_K: + return ggml_cuda_mmq_util_funcs( + VDR_Q6_K_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q6_K, + ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_IQ1_S: + return ggml_cuda_mmq_util_funcs( + VDR_IQ1_S_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq1_s, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ2_XXS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ2_XXS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq2_xxs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ2_XS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ2_XS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq2_xs, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ2_S: + return ggml_cuda_mmq_util_funcs( + VDR_IQ2_S_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq2_s, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ3_XXS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ3_XXS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq3_xxs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ3_S: + return ggml_cuda_mmq_util_funcs( + VDR_IQ3_S_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq3_s, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ4_XS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ4_XS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq4_xs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ4_NL: + return ggml_cuda_mmq_util_funcs( + VDR_IQ4_NL_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq4_nl, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_MXFP4: + return ggml_cuda_mmq_util_funcs( + VDR_MXFP4_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_mxfp4, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_NVFP4: + return ggml_cuda_mmq_util_funcs( + VDR_NVFP4_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_nvfp4, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + default: + return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr); + } + } + +// --------------------------------------------------------------------------------------------- -template -struct mmq_type_traits { - static constexpr int vdr = VDR_MXFP4_Q8_1_MMQ; #ifdef BLACKWELL_MMA_AVAILABLE - static constexpr load_tiles_mmq_t load_tiles = load_tiles_mxfp4_fp4; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_mxfp4_mxfp4_mma; -#else - static constexpr load_tiles_mmq_t load_tiles = load_tiles_mxfp4; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; + switch (type) { + case GGML_TYPE_MXFP4: + if (prec_src1 == GGML_PREC_Q4) { + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_mxfp4_fp4, + ggml_cuda_mmq_vec_dot_fp4_fp4_mma, + ggml_cuda_mmq_write_back_mma); + } + break; + case GGML_TYPE_NVFP4: + if (prec_src1 == GGML_PREC_Q4) { + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_nvfp4_nvfp4, + ggml_cuda_mmq_vec_dot_fp4_fp4_mma, + ggml_cuda_mmq_write_back_mma); + } + break; + default: + break; + } #endif // BLACKWELL_MMA_AVAILABLE - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; -}; -template -struct mmq_type_traits { - static constexpr int vdr = VDR_NVFP4_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_nvfp4; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q2_K_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q2_K; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q2_K_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q2_K_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q3_K_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q3_K; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q3_K_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q4_K_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_K; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q4_K_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q5_K_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q5_K; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q5_K_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_Q6_K_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_q6_K; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q6_K_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q6_K_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_IQ2_XXS_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_xxs; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; -}; - -template -struct mmq_type_traits { - static constexpr int vdr = VDR_IQ2_XS_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_xs; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a; -}; +// --------------------------------------------------------------------------------------------- -template -struct mmq_type_traits { - static constexpr int vdr = VDR_IQ2_S_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq2_s; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_16_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_16_q8_1_dp4a; -}; + switch (type) { + case GGML_TYPE_Q1_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q1_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q2_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q2_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q4_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q4_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q4_1: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q4_1, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q5_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q5_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q5_1: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q5_1, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q8_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q8_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_Q2_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q2_K, + ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q3_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q3_K, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q4_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q4_K, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q5_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q5_K, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q6_K: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q6_K, + ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma, + ggml_cuda_mmq_write_back_mma); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_IQ1_S: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq1_s, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ2_XXS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq2_xxs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ2_XS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq2_xs, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ2_S: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq2_s, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ3_XXS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq3_xxs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ3_S: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq3_s, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ4_XS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq4_xs, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ4_NL: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq4_nl, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); +// --------------------------------------------------------------------------------------------- + case GGML_TYPE_MXFP4: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_mxfp4, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_NVFP4: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_nvfp4, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + default: + return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr); + } +} -template -struct mmq_type_traits { - static constexpr int vdr = VDR_IQ3_XXS_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq3_xxs; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; -}; +template +static constexpr __device__ int ggml_cuda_mmq_get_vdr() { + return ggml_cuda_mmq_get_util_funcs().vdr; +} -template -struct mmq_type_traits { - static constexpr int vdr = VDR_IQ3_S_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq3_s; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; -}; +template +static constexpr __device__ ggml_cuda_mmq_load_tiles_t ggml_cuda_mmq_get_load_tiles() { + return ggml_cuda_mmq_get_util_funcs().load_tiles; +} -template -struct mmq_type_traits { - static constexpr int vdr = VDR_IQ1_S_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq1_s; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_1_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_1_q8_1_dp4a; -}; +template +static constexpr __device__ ggml_cuda_mmq_vec_dot_t ggml_cuda_mmq_get_vec_dot() { + return ggml_cuda_mmq_get_util_funcs().vec_dot; +} -template -struct mmq_type_traits { - static constexpr int vdr = VDR_IQ4_NL_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq4_nl; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; -}; +template +static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_back() { + return ggml_cuda_mmq_get_util_funcs().write_back; +} -template -struct mmq_type_traits { - static constexpr int vdr = VDR_IQ4_XS_Q8_1_MMQ; - static constexpr load_tiles_mmq_t load_tiles = load_tiles_iq4_xs; - static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; - static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; -}; +// --------------------------------------------------------------------------------------------- -template +// has_fusion: x and x_gate are the up and gate weights. +template static __device__ __forceinline__ void mul_mat_q_process_tile( - const char * __restrict__ x, const int offset_x, const int * __restrict__ y, + const char * __restrict__ x, const char * __restrict__ x_gate, const int offset_x, const int * __restrict__ y, const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup, + const float * __restrict__ y_scale, const ggml_glu_op glu_op, const float glu_limit, const int stride_row_x, const int ncols_y, const int stride_col_dst, const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) { + static_assert(!(fixup && has_fusion), "the fused gate/up path does not split K"); + static_assert(!(fallback && has_fusion), "the fused gate/up path needs nrows_x % 128 == 0"); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = mmq_get_nwarps_device(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; constexpr int qk = ggml_cuda_type_traits::qk; - constexpr int mmq_y = get_mmq_y_device(); - constexpr load_tiles_mmq_t load_tiles = mmq_type_traits::load_tiles; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); + constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles(); + constexpr ggml_cuda_mmq_vec_dot_t vec_dot = ggml_cuda_mmq_get_vec_dot(); + constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back(); extern __shared__ int data_mul_mat_q[]; - int * tile_y = data_mul_mat_q + mmq_x; - int * tile_x = tile_y + GGML_PAD(mmq_x*MMQ_TILE_Y_K, nwarps*warp_size); - -#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr vec_dot_mmq_t vec_dot = mmq_type_traits::vec_dot_mma; - constexpr mmq_write_back_t write_back = mmq_write_back_mma; -#else - constexpr vec_dot_mmq_t vec_dot = mmq_type_traits::vec_dot_dp4a; - constexpr mmq_write_back_t write_back = mmq_write_back_dp4a; -#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * tile_y = data_mul_mat_q + J; + int * tile_x = tile_y + GGML_PAD(J*MMQ_TILE_Y_K, nwarps*warp_size); #if defined(BLACKWELL_MMA_AVAILABLE) - // FP4 tile stores 8 blocks - constexpr int ne_block = (type == GGML_TYPE_MXFP4) ? 8 * QK_MXFP4 : 4 * QK8_1; + // FP4 tile stores 8 blocks. src1 above Q4 uses the generic + // Q8_1 tile layout instead of the packed FP4 tile. + constexpr int ne_block = ((type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4) && prec_src1 == GGML_PREC_Q4) ? + QK_FP4_MMQ : QK8_1_MMQ; #else - constexpr int ne_block = 4 * QK8_1; + constexpr int ne_block = QK8_1_MMQ; #endif // defined(BLACKWELL_MMA_AVAILABLE) - constexpr int ITER_K = get_iter_k(type); + constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1); constexpr int blocks_per_iter = ITER_K / qk; - float sum[mmq_x*mmq_y / (nwarps*warp_size)] = {0.0f}; + float sum[J*I / (nwarps*warp_size)] = {0.0f}; constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int); for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) { - load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x); + load_tiles(x, x_gate, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x); { const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz; #pragma unroll - for (int l0 = 0; l0 < mmq_x * MMQ_TILE_Y_K; l0 += nwarps * warp_size) { + for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) { int l = l0 + threadIdx.y*warp_size + threadIdx.x; tile_y[l] = by0[l]; @@ -3608,7 +967,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( { const int * by0 = y + ncols_y * ((kb0 * qk / ne_block) * sz + sz); #pragma unroll - for (int l0 = 0; l0 < mmq_x * MMQ_TILE_Y_K; l0 += nwarps * warp_size) { + for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) { int l = l0 + threadIdx.y*warp_size + threadIdx.x; tile_y[l] = by0[l]; @@ -3623,59 +982,54 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( } if (fixup) { - write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(mmq_x*mmq_y), mmq_y, mmq_y, mmq_x); + write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J, glu_op, glu_limit); } else { - write_back(sum, ids_dst, dst, stride_col_dst, tile_x_max_i, tile_y_max_j); + write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j, glu_op, glu_limit); } } // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598 -template -#if defined(GGML_USE_HIP) -#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN) - __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 2) -#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN) -#else -#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA - __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 1) -#else - __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 2) -#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA -#endif // defined(GGML_USE_HIP) +template +__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1)) static __global__ void mul_mat_q( - const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst, + const char * __restrict__ x, const char * __restrict__ x_gate, const int * __restrict__ y, const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup, - const int ncols_x, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst, - const int channel_ratio, const int nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, - const int sample_ratio, const int nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, - const int ncols_max) { + const float * __restrict__ y_scale, const ggml_glu_op glu_op, const float glu_limit, + const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst, + const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, + const uint3 sample_ratio, const uint3 nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, + const uint3 ntx) { + +#ifdef TURING_MMA_AVAILABLE + constexpr bool fusion_available = true; +#else + constexpr bool fusion_available = false; +#endif // TURING_MMA_AVAILABLE // Skip unused template specializations for faster compilation: - if (mmq_x > get_mmq_x_max_device() || mmq_x % mmq_get_granularity_device(mmq_x) != 0) { + if (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type == GGML_TYPE_COUNT || (has_fusion && !fusion_available)) { NO_DEVICE_CODE; return; } - constexpr int nwarps = mmq_get_nwarps_device(); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); - constexpr int qk = ggml_cuda_type_traits::qk; - constexpr int mmq_y = get_mmq_y_device(); - - const int ntx = (ncols_max + mmq_x - 1) / mmq_x; // Number of tiles x - const int nty = (nrows_x + mmq_y - 1) / mmq_y; // Number of tiles y + const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y // Initialize the ids for writing back data with just the index. // For regular matrix multiplications this is never changed. // For MoE the correct indices are loaded from ids_dst. extern __shared__ int ids_dst_shared[]; // Stored at beginning of shared memory. #pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) { + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { const int j = j0 + threadIdx.y*warp_size + threadIdx.x; - if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) { + if (j0 + nwarps*warp_size > J && j >= J) { break; } @@ -3683,20 +1037,29 @@ static __global__ void mul_mat_q( } __syncthreads(); - // On non-CDNA AMD or old CUDA the performance with stream-k was worse, use conventional tiling instead: -#if (defined(GGML_USE_HIP) && !defined(CDNA)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA - { - const int wt = blockIdx.z / nchannels_y; - const int zt = blockIdx.z - wt*nchannels_y; - const int jt = blockIdx.y; - const int it = blockIdx.x; + // The fused kernel needs the whole K range of a tile in one block, so it always uses tiling. + if constexpr (has_fusion || !ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) { + // The fused kernel writes the GLU of I/2 up rows and I/2 gate rows per tile. + constexpr int I_dst = has_fusion ? I/2 : I; + + const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y); + const int wt = tmp2.x; + const int zt = tmp2.y; + const int jt = blockIdx.x; + const int it = blockIdx.y; // Defaults for regular matrix multiplication: int col_low = 0; int col_high = ncols_dst; int col_diff = ncols_dst; - int offset_y = wt*stride_sample_y + zt*stride_channel_y; - int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst; + int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; + int offset_y_scale; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; + } else { + GGML_UNUSED(offset_y_scale); + } if (ids_dst) { col_low = expert_bounds[zt + 0]; @@ -3705,72 +1068,86 @@ static __global__ void mul_mat_q( offset_y = 0; offset_dst = 0; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = 0; + } - if (jt*mmq_x >= col_diff) { + if (jt*J >= col_diff) { return; } // __syncthreads(); // There is no previous tile that could cause a race condition. #pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) { + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { const int j = j0 + threadIdx.y*warp_size + threadIdx.x; - if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) { + if (j0 + nwarps*warp_size > J && j >= J) { break; } - ids_dst_shared[j] = ids_dst[col_low + jt*mmq_x + j]; + ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; } __syncthreads(); } - offset_y += (col_low + jt*mmq_x)*(sizeof(block_q8_1_mmq)/sizeof(int)); - offset_dst += it*mmq_y; + offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int)); + offset_dst += it*I_dst; + const float * y_scale_tile = nullptr; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale += col_low + jt*J; + y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; + } - const int tile_x_max_i = nrows_x - it*mmq_y - 1; - const int tile_y_max_j = col_diff - jt*mmq_x - 1; + const int tile_x_max_i = nrows_x - it*I_dst - 1; + const int tile_y_max_j = col_diff - jt*J - 1; - const int offset_x = (wt/sample_ratio)*stride_sample_x + (zt/channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x; + const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I_dst*stride_row_x; constexpr bool fixup = false; - mul_mat_q_process_tile - (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst, - tile_x_max_i, tile_y_max_j, 0, ncols_x/qk); + mul_mat_q_process_tile + (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst, + tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z); return; } -#endif // (defined(GGML_USE_HIP) && !defined(CDNA4) && !defined(CDNA3)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA - constexpr int ITER_K = get_iter_k(type); - - const int64_t blocks_per_ne00 = ncols_x / qk; - constexpr int blocks_per_iter = ITER_K / qk; + constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1); + constexpr int blocks_per_iter = ITER_K / qk; // kbc == k block continuous, current index in continuous ijk space. - int64_t kbc = (int64_t) blockIdx.x *nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x; - int64_t kbc_stop = (int64_t)(blockIdx.x + 1)*nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x; + int kbc = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; - kbc -= (kbc % blocks_per_ne00) % blocks_per_iter; - kbc_stop -= (kbc_stop % blocks_per_ne00) % blocks_per_iter; + kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter; + kbc_stop -= fastmodulo(kbc_stop, blocks_per_ne00) % blocks_per_iter; // kb0 == k index when doing the matrix multiplication for an output tile. - int kb0_start = kbc % blocks_per_ne00; - int kb0_stop = min(blocks_per_ne00, kb0_start + kbc_stop - kbc); - while (kbc < kbc_stop && kb0_stop == blocks_per_ne00) { - int tmp = kbc; - const int it = tmp / (nsamples_y*nchannels_y*ntx*blocks_per_ne00); - tmp -= it * (nsamples_y*nchannels_y*ntx*blocks_per_ne00); - const int wt = tmp / (nchannels_y*ntx*blocks_per_ne00); - tmp -= wt * (nchannels_y*ntx*blocks_per_ne00); - const int zt = tmp / (ntx*blocks_per_ne00); - tmp -= zt * (ntx*blocks_per_ne00); - const int jt = tmp / blocks_per_ne00; + int kb0_start = fastmodulo(kbc, blocks_per_ne00); + int kb0_stop = min(blocks_per_ne00.z, uint32_t(kb0_start + kbc_stop - kbc)); + while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) { + int tmp = fastdiv(kbc, blocks_per_ne00); + uint2 tmp2 = fast_div_modulo(tmp, ntx); + const int jt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nchannels_y); + const int zt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nsamples_y); + const int wt = tmp2.y; + const int it = tmp2.x; // Defaults for regular matrix multiplication: int col_low = 0; int col_high = ncols_dst; int col_diff = ncols_dst; - int offset_y = wt*stride_sample_y + zt*stride_channel_y; - int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst; + int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; + int offset_y_scale; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; + } else { + GGML_UNUSED(offset_y_scale); + } if (ids_dst) { col_low = expert_bounds[zt + 0]; @@ -3779,70 +1156,87 @@ static __global__ void mul_mat_q( offset_y = 0; offset_dst = 0; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = 0; + } - if (jt*mmq_x >= col_diff) { - kbc += blocks_per_ne00; - kbc -= kbc % blocks_per_ne00; + if (jt*J >= col_diff) { + kbc += blocks_per_ne00.z; + kbc -= fastmodulo(kbc, blocks_per_ne00); kb0_start = 0; - kb0_stop = min(blocks_per_ne00, kbc_stop - kbc); + kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc)); continue; } __syncthreads(); #pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) { + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { const int j = j0 + threadIdx.y*warp_size + threadIdx.x; - if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) { + if (j0 + nwarps*warp_size > J && j >= J) { break; } - ids_dst_shared[j] = ids_dst[col_low + jt*mmq_x + j]; + ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; } __syncthreads(); } - offset_y += (col_low + jt * mmq_x) * (sizeof(block_q8_1_mmq) / sizeof(int)); - offset_dst += it*mmq_y; + offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int)); + offset_dst += it*I; + const float * y_scale_tile = nullptr; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale += col_low + jt * J; + y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; + } - const int tile_x_max_i = nrows_x - it*mmq_y - 1; - const int tile_y_max_j = col_diff - jt*mmq_x - 1; + const int tile_x_max_i = nrows_x - it*I - 1; + const int tile_y_max_j = col_diff - jt*J - 1; - const int offset_x = (wt/sample_ratio)*stride_sample_x + (zt/channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x; + const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer. - mul_mat_q_process_tile - (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst, + mul_mat_q_process_tile + (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst, tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); - kbc += blocks_per_ne00; - kbc -= kbc % blocks_per_ne00; + kbc += blocks_per_ne00.z; + kbc -= fastmodulo(kbc, blocks_per_ne00); kb0_start = 0; - kb0_stop = min(blocks_per_ne00, kbc_stop - kbc); + kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc)); } if (kbc >= kbc_stop) { return; } - int tmp = kbc; - const int it = tmp / (nsamples_y*nchannels_y*ntx*blocks_per_ne00); - tmp -= it * (nsamples_y*nchannels_y*ntx*blocks_per_ne00); - const int wt = tmp / (nchannels_y*ntx*blocks_per_ne00); - tmp -= wt * (nchannels_y*ntx*blocks_per_ne00); - const int zt = tmp / (ntx*blocks_per_ne00); - tmp -= zt * (ntx*blocks_per_ne00); - const int jt = tmp / blocks_per_ne00; + int tmp = fastdiv(kbc, blocks_per_ne00); + uint2 tmp2 = fast_div_modulo(tmp, ntx); + const int jt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nchannels_y); + const int zt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nsamples_y); + const int wt = tmp2.y; + const int it = tmp2.x; // Defaults for regular matrix multiplication: int col_low = 0; int col_high = ncols_dst; int col_diff = ncols_dst; - int offset_y = wt*stride_sample_y + zt*stride_channel_y; - int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst; + int offset_y = wt*stride_sample_y + zt*stride_channel_y; + int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst; + int offset_y_scale; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y; + } else { + GGML_UNUSED(offset_y_scale); + } if (ids_dst) { col_low = expert_bounds[zt + 0]; @@ -3851,18 +1245,21 @@ static __global__ void mul_mat_q( offset_y = 0; offset_dst = 0; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale = 0; + } - if (jt*mmq_x >= col_diff) { + if (jt*J >= col_diff) { return; } // The memory layout for the fixup buffer is always contiguous, therefore reset ids: __syncthreads(); #pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps*warp_size) { + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { const int j = j0 + threadIdx.y*warp_size + threadIdx.x; - if (j0 + nwarps*warp_size > mmq_x && j >= mmq_x) { + if (j0 + nwarps*warp_size > J && j >= J) { break; } @@ -3871,61 +1268,57 @@ static __global__ void mul_mat_q( __syncthreads(); } - offset_y += (col_low + jt * mmq_x) * (sizeof(block_q8_1_mmq) / sizeof(int)); - offset_dst += it*mmq_y; + offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int)); + offset_dst += it*I; + const float * y_scale_tile = nullptr; + if constexpr (type == GGML_TYPE_NVFP4) { + offset_y_scale += col_low + jt * J; + y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr; + } - const int tile_x_max_i = nrows_x - it*mmq_y - 1; - const int tile_y_max_j = col_diff - jt*mmq_x - 1; + const int tile_x_max_i = nrows_x - it*I - 1; + const int tile_y_max_j = col_diff - jt*J - 1; - const int offset_x = (wt/sample_ratio)*stride_sample_x + (zt/channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x; + const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x; constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks. - mul_mat_q_process_tile - (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst, + mul_mat_q_process_tile + (x, x_gate, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile, + glu_op, glu_limit, stride_row_x, ncols_y, stride_col_dst, tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); } -template -static __global__ void mul_mat_q_stream_k_fixup(const int32_t * ids_dst, - const int32_t * expert_bounds, - float * __restrict__ dst, - const float * __restrict__ tmp_last_tile, - const int ncols_x, - const int nrows_x, - const int ncols_dst, - const size_t stride_col_dst, - const int nchannels_y, - const size_t stride_channel_dst, - const int nsamples_y, - const size_t stride_sample_dst, - const int ncols_max) { - constexpr int mmq_y = get_mmq_y_device(); - constexpr int qk = ggml_cuda_type_traits::qk; - constexpr int ITER_K = get_iter_k(type); - - constexpr int blocks_per_iter = ITER_K / qk; - const int64_t blocks_per_ne00 = ncols_x / qk; - - constexpr int nwarps = mmq_get_nwarps_device(); - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); +template +__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1)/2, 1) +static __global__ void mul_mat_q_stream_k_fixup( + const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, + float * __restrict__ tmp_last_tile, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, + const int stride_col_dst, const uint3 nchannels_y, const int stride_channel_dst, const uint3 nsamples_y, + const int stride_sample_dst, const uint3 ntx) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = (ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / 2) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1); + constexpr int blocks_per_iter = ITER_K / qk; - float sum[mmq_x*mmq_y / (nwarps*warp_size)] = {0.0f}; + float sum[J / nwarps] = {0.0f}; + const int i = blockIdx.y*warp_size + threadIdx.x; - const int ntx = (ncols_max + mmq_x - 1) / mmq_x; - const int nty = (nrows_x + mmq_y - 1) / mmq_y; + const int nty = (nrows_x + I - 1) / I; const int bidx0 = blockIdx.x; // kbc == k block continuous, current index in continuous ijk space. - int64_t kbc0 = (int64_t) bidx0 *nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x; - int64_t kbc0_stop = (int64_t)(bidx0 + 1)*nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x; + int kbc0 = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + int kbc0_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; - kbc0 -= (kbc0 % blocks_per_ne00) % blocks_per_iter; - kbc0_stop -= (kbc0_stop % blocks_per_ne00) % blocks_per_iter; + kbc0 -= fastmodulo(kbc0, blocks_per_ne00) % blocks_per_iter; + kbc0_stop -= fastmodulo(kbc0_stop, blocks_per_ne00) % blocks_per_iter; const bool did_not_have_any_data = kbc0 == kbc0_stop; - const bool wrote_beginning_of_tile = kbc0 % blocks_per_ne00 == 0; - const bool did_not_write_last = kbc0/blocks_per_ne00 == kbc0_stop/blocks_per_ne00 && kbc0_stop % blocks_per_ne00 != 0; + const bool wrote_beginning_of_tile = fastmodulo(kbc0, blocks_per_ne00) == 0; + const bool did_not_write_last = fastdiv(kbc0, blocks_per_ne00) == fastdiv(kbc0_stop, blocks_per_ne00) && fastmodulo(kbc0_stop, blocks_per_ne00) != 0; if (did_not_have_any_data || wrote_beginning_of_tile || did_not_write_last) { return; } @@ -3934,11 +1327,11 @@ static __global__ void mul_mat_q_stream_k_fixup(const int32_t * ids_dst, // Iterate over previous blocks and sum up partial sums written to fixup buffer. // All CUDA blocks that get here must have a previous block that needs a fixup. - int64_t bidx = bidx0 - 1; - int64_t kbc_stop = kbc0; + int bidx = bidx0 - 1; + int kbc_stop = kbc0; while(true) { - int64_t kbc = bidx*nsamples_y*nchannels_y*ntx*nty*blocks_per_ne00 / gridDim.x; - kbc -= (kbc % blocks_per_ne00) % blocks_per_iter; + int kbc = int64_t(bidx)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x; + kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter; if (kbc == kbc_stop) { // Did not have any data. bidx--; @@ -3948,20 +1341,16 @@ static __global__ void mul_mat_q_stream_k_fixup(const int32_t * ids_dst, any_fixup = true; -#pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { - const int j = j0 + threadIdx.y; #pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; + for (int j0 = 0; j0 < J; j0 += nwarps) { + const int j = j0 + threadIdx.y; - sum[(j0/nwarps) * (mmq_y/warp_size) + i0/warp_size] += tmp_last_tile[bidx*(mmq_x*mmq_y) + j*mmq_y + i]; - } + sum[j0/nwarps] += tmp_last_tile[bidx*(J*I) + j*I + i]; } // If this block started in a previous tile we are done and don't need to combine additional partial results. - if (kbc % blocks_per_ne00 == 0 || kbc/blocks_per_ne00 < kbc0/blocks_per_ne00) { + if (fastmodulo(kbc, blocks_per_ne00) == 0 || fastdiv(kbc, blocks_per_ne00) < fastdiv(kbc0, blocks_per_ne00)) { break; } bidx--; @@ -3972,310 +1361,299 @@ static __global__ void mul_mat_q_stream_k_fixup(const int32_t * ids_dst, return; } - int tmp = kbc0; - const int it = tmp / (nsamples_y*nchannels_y*ntx*blocks_per_ne00); - tmp -= it * (nsamples_y*nchannels_y*ntx*blocks_per_ne00); - const int wt = tmp / (nchannels_y*ntx*blocks_per_ne00); - tmp -= wt * (nchannels_y*ntx*blocks_per_ne00); - const int zt = tmp / (ntx*blocks_per_ne00); - tmp -= zt * (ntx*blocks_per_ne00); - const int jt = tmp / blocks_per_ne00; + int tmp = fastdiv(kbc0, blocks_per_ne00); + uint2 tmp2 = fast_div_modulo(tmp, ntx); + const int jt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nchannels_y); + const int zt = tmp2.y; + tmp = tmp2.x; + tmp2 = fast_div_modulo(tmp, nsamples_y); + const int wt = tmp2.y; + const int it = tmp2.x; if (!ids_dst) { - const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*mmq_x*stride_col_dst + it*mmq_y; + const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I; dst += offset_dst; - const int i_max = nrows_x - it*mmq_y - 1; - const int j_max = ncols_dst - jt*mmq_x - 1; + const int i_max = nrows_x - it*I - 1; + const int j_max = ncols_dst - jt*J - 1; + if (fallback && i > i_max) { + return; + } #pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { + for (int j0 = 0; j0 < J; j0 += nwarps) { const int j = j0 + threadIdx.y; if (j > j_max) { return; } -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - if (need_check && i > i_max) { - continue; - } - - dst[j*stride_col_dst + i] += sum[(j0/nwarps) * (mmq_y/warp_size) + i0/warp_size]; - } + dst[j*stride_col_dst + i] += sum[j0/nwarps]; } return; } - __shared__ int ids_dst_shared[mmq_x]; + __shared__ int ids_dst_shared[J]; const int col_low = expert_bounds[zt + 0]; const int col_high = expert_bounds[zt + 1]; const int col_diff = col_high - col_low; - for (int j = threadIdx.y*warp_size + threadIdx.x; j < mmq_x; j += nwarps*warp_size) { - ids_dst_shared[j] = ids_dst[col_low + jt*mmq_x + j]; + for (int j = threadIdx.y*warp_size + threadIdx.x; j < J; j += nwarps*warp_size) { + ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; } __syncthreads(); - const int offset_dst = it*mmq_y; + const int offset_dst = it*I; dst += offset_dst; - const int i_max = nrows_x - it*mmq_y - 1; - const int j_max = col_diff - jt*mmq_x - 1; + const int i_max = nrows_x - it*I - 1; + const int j_max = col_diff - jt*J - 1; + if (fallback && i > i_max) { + return; + } #pragma unroll - for (int j0 = 0; j0 < mmq_x; j0 += nwarps) { + for (int j0 = 0; j0 < J; j0 += nwarps) { const int j = j0 + threadIdx.y; if (j > j_max) { return; } -#pragma unroll - for (int i0 = 0; i0 < mmq_y; i0 += warp_size) { - const int i = i0 + threadIdx.x; - - if (need_check && i > i_max) { - continue; - } - - dst[ids_dst_shared[j]*stride_col_dst + i] += sum[(j0/nwarps) * (mmq_y/warp_size) + i0/warp_size]; - } + dst[ids_dst_shared[j]*stride_col_dst + i] += sum[j0/nwarps]; } } struct mmq_args { const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst; + const float * y_scale; int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst; int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst; int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst; - bool use_stream_k; int64_t ncols_max; + int64_t ncols_max; + int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance. + ggml_cuda_mm_fusion_args_device fusion = {}; // gate != nullptr selects the fused gate/up/GLU path }; -template -static size_t mmq_get_nbytes_shared(const int mmq_x, const int mmq_y, const int cc, const int warp_size, const int nwarps) { - const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, mmq_y); - const int mmq_tile_x_k = mmq_get_mma_tile_x_k(type); - const size_t nbs_ids = mmq_x*sizeof(int); - const size_t nbs_x = (turing_mma_available(cc) || amd_mfma_available(cc) || amd_wmma_available(cc)) ? mmq_y*mmq_tile_x_k*sizeof(int) : txs.qs*sizeof(int) + txs.dm*sizeof(half2) + txs.sc*sizeof(int); - const size_t nbs_y = mmq_x * (sizeof(block_q8_1_mmq)); - return nbs_ids + nbs_x + GGML_PAD(nbs_y, nwarps*warp_size*sizeof(int)); +static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) { + const size_t nbs_ids = config.J*sizeof(int); + const size_t nbs_x = ggml_cuda_mmq_get_nbytes_shared_x(config, cc); + const size_t nbs_y = config.J * (sizeof(block_q8_1_mmq)); + return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int)); } -template +template static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { const int id = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[id].cc; const int nsm = ggml_cuda_info().devices[id].nsm; const int warp_size = ggml_cuda_info().devices[id].warp_size; - const int nwarps = mmq_get_nwarps_host(cc, warp_size); - const int mmq_y = get_mmq_y_host(cc); - const dim3 block_dims(warp_size, nwarps, 1); + const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1); + GGML_ASSERT(config.nthreads % warp_size == 0); + const int nwarps = config.nthreads / warp_size; + const int nbytes_shared = mmq_get_nbytes_shared(config, cc); - const int nbytes_shared = mmq_get_nbytes_shared(mmq_x, mmq_y, cc, warp_size, nwarps); + const dim3 block_dims(warp_size, nwarps, 1); - CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); - CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + if constexpr (!has_fusion) { + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + } - const int nty = (args.nrows_x + mmq_y - 1) / mmq_y; - const int ntx = (args.ncols_max + mmq_x - 1) / mmq_x; + const int I_dst = has_fusion ? config.I/2 : config.I; // Rows of dst per tile, see mul_mat_q. + const int nty = (args.nrows_x + I_dst - 1) / I_dst; + const int ntx = (args.ncols_max + config.J - 1) / config.J; const int ntzw = args.nchannels_y * args.nsamples_y; - const dim3 block_nums_xy_tiling(nty, ntx, ntzw); + // Column tiles innermost so that the concurrently running blocks share the same tile of x and read it from L2. + const dim3 block_nums_xy_tiling(ntx, nty, ntzw); GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0); GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0); const int channel_ratio = args.nchannels_y / args.nchannels_x; const int sample_ratio = args.nsamples_y / args.nsamples_x; - if (!args.use_stream_k) { - if (args.nrows_x % mmq_y == 0) { - constexpr bool need_check = false; - mul_mat_q<<>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, - args.ncols_x, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, - channel_ratio, args.nchannels_y, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, - sample_ratio, args.nsamples_y, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - args.ncols_max); - } else { - constexpr bool need_check = true; - mul_mat_q<<>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, - args.ncols_x, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, - channel_ratio, args.nchannels_y, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, - sample_ratio, args.nsamples_y, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - args.ncols_max); - } + const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits::qk); + const uint3 ntx_fd = init_fastdiv_values(ntx); + const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y); + const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y); + const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio); + const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); + + if (has_fusion || !config.stream_k) { + GGML_ASSERT(nty <= 65535); // gridDim.y limit + mul_mat_q<<>> + (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, + blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, + channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, + sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, + ntx_fd); return; } - const dim3 block_nums_stream_k(nsm, 1, 1); - const bool fixup_needed = ntx*nty*ntzw % nsm != 0; + // For the stream-k kernel it is possible to run it with tiling by setting the number of CUDA blocks equal to the number of tiles. + // This is worthwhile if the efficiency of tiling is high and skipping the fixup kernel is more important. + const int ntiles_dst = ntx * nty * ntzw; + const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm; + const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves); + + GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow. + + const int nwork = ntiles_dst * ((args.ncols_x + config.K_vram - 1) / config.K_vram); // Maximum number of CUDA blocks that can do useful work. + const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : std::min(nsm, nwork), 1, 1); + + const bool fixup_needed = ntiles_dst % block_nums_stream_k.x != 0; ggml_cuda_pool & pool = ctx.pool(id); ggml_cuda_pool_alloc tmp_fixup(pool); if (fixup_needed) { - tmp_fixup.alloc(block_nums_stream_k.x * mmq_x*mmq_y); + tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I); } - if (args.nrows_x % mmq_y == 0) { - constexpr bool need_check = false; - mul_mat_q<<>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, - args.ncols_x, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, - channel_ratio, args.nchannels_y, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, - sample_ratio, args.nsamples_y, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - args.ncols_max); - - if (!fixup_needed) { - return; - } + const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1); + const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z); - mul_mat_q_stream_k_fixup<<>> - (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.ncols_x, args.nrows_x, args.ncols_dst, - args.nrows_dst, args.nchannels_y, args.stride_channel_dst, args.nsamples_y, args.stride_sample_dst, - args.ncols_max); - } else { - constexpr bool need_check = true; - mul_mat_q<<>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, - args.ncols_x, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, - channel_ratio, args.nchannels_y, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, - sample_ratio, args.nsamples_y, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - args.ncols_max); - - if (!fixup_needed) { - return; - } + mul_mat_q<<>> + (args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, + blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, + channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, + sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, + ntx_fd); - mul_mat_q_stream_k_fixup<<>> - (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.ncols_x, args.nrows_x, args.ncols_dst, - args.nrows_dst, args.nchannels_y, args.stride_channel_dst, args.nsamples_y, args.stride_sample_dst, - args.ncols_max); + if (!fixup_needed) { + return; } -} - -template -void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { - const int id = ggml_cuda_get_device(); - const int cc = ggml_cuda_info().devices[id].cc; - const size_t smpbo = ggml_cuda_info().devices[id].smpbo; - const int warp_size = ggml_cuda_info().devices[id].warp_size; - const int nwarps = mmq_get_nwarps_host(cc, warp_size); - - const int mmq_x_max = get_mmq_x_max_host(cc); - const int mmq_y = get_mmq_y_host(cc); - - int mmq_x_best = 0; - int ntiles_x_best = INT_MAX; - - for (int mmq_x = 8; mmq_x <= mmq_x_max && ntiles_x_best > 1; mmq_x += 8) { - const int granularity = mmq_get_granularity_host(mmq_x, cc); - - if (mmq_x % granularity != 0 || mmq_get_nbytes_shared(mmq_x, mmq_y, cc, warp_size, nwarps) > smpbo) { - continue; - } - const int ntiles_x = (args.ncols_max + mmq_x - 1) / mmq_x; - - if (ntiles_x < ntiles_x_best) { - mmq_x_best = mmq_x; - ntiles_x_best = ntiles_x; - } - } + CUDA_CHECK(cudaGetLastError()); + mul_mat_q_stream_k_fixup<<>> + (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, + args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, + ntx_fd); +} - switch (mmq_x_best) { +template +void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + switch (args.J_best) { case 8: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 16: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 24: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 32: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 40: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 48: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 56: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 64: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 72: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 80: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 88: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 96: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 104: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 112: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 120: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; case 128: - launch_mul_mat_q(ctx, args, stream); + launch_mul_mat_q(ctx, args, stream); break; default: - fprintf(stderr, "mmq_x_best=%d\n", mmq_x_best); + fprintf(stderr, "J_best=%d\n", args.J_best); GGML_ABORT("fatal error"); - break; + } +} + +template +static void mul_mat_q_switch_fusion(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + if constexpr (!fallback && ggml_cuda_mmq_fusion_supported(type)) { + if (args.fusion.gate) { + mul_mat_q_switch_J(ctx, args, stream); + return; + } + } + GGML_ASSERT(!args.fusion.gate); + mul_mat_q_switch_J(ctx, args, stream); +} + +template +void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + if (ggml_cuda_mmq_needs_fallback(args.nrows_x)) { + constexpr bool fallback = true; + mul_mat_q_switch_fusion(ctx, args, stream); + } else { + constexpr bool fallback = false; + mul_mat_q_switch_fusion(ctx, args, stream); } } #define DECL_MMQ_CASE(type) \ - template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ + template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ + +// FP4 variant: uses native FP4 MMA instead of keeping src1 at Q8_1. +#define DECL_MMQ_CASE_W4A4(type) \ + template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ +extern DECL_MMQ_CASE(GGML_TYPE_Q1_0); +extern DECL_MMQ_CASE(GGML_TYPE_Q2_0); extern DECL_MMQ_CASE(GGML_TYPE_Q4_0); extern DECL_MMQ_CASE(GGML_TYPE_Q4_1); extern DECL_MMQ_CASE(GGML_TYPE_Q5_0); extern DECL_MMQ_CASE(GGML_TYPE_Q5_1); extern DECL_MMQ_CASE(GGML_TYPE_Q8_0); -extern DECL_MMQ_CASE(GGML_TYPE_MXFP4); -extern DECL_MMQ_CASE(GGML_TYPE_NVFP4); +// ----------------------------------------- extern DECL_MMQ_CASE(GGML_TYPE_Q2_K); extern DECL_MMQ_CASE(GGML_TYPE_Q3_K); extern DECL_MMQ_CASE(GGML_TYPE_Q4_K); extern DECL_MMQ_CASE(GGML_TYPE_Q5_K); extern DECL_MMQ_CASE(GGML_TYPE_Q6_K); +// ----------------------------------------- +extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S); extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS); extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS); extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S); extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS); extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S); -extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S); extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL); extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS); +// ----------------------------------------- +extern DECL_MMQ_CASE(GGML_TYPE_MXFP4); +extern DECL_MMQ_CASE(GGML_TYPE_NVFP4); +extern DECL_MMQ_CASE_W4A4(GGML_TYPE_MXFP4); +extern DECL_MMQ_CASE_W4A4(GGML_TYPE_NVFP4); // ------------------------------------------------------------------------------------------------------------------------- void ggml_cuda_mul_mat_q( - ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst); - -void ggml_cuda_op_mul_mat_q( - ggml_backend_cuda_context & ctx, - const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, - const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, - const int64_t src1_padded_row_size, cudaStream_t stream); + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, + const ggml_cuda_mm_fusion_args_host * fusion = nullptr); bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts); + +bool ggml_cuda_mmq_fusion_is_efficient(ggml_type type, int64_t nrows_x, int64_t ncols, int64_t nchannels, int device); diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu index d914720..752d28c 100644 --- a/ggml/src/ggml-cuda/mmvf.cu +++ b/ggml/src/ggml-cuda/mmvf.cu @@ -2,15 +2,20 @@ #include "common.cuh" #include "unary.cuh" #include "mmvf.cuh" +#include "mmf.cuh" #include "convert.cuh" template static __global__ void mul_mat_vec_f( - const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, const ggml_cuda_mm_fusion_args_device fusion, float * __restrict__ dst, + const T * x_ptr, const float * y_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, const int ncols2, const uint3 nchannels_y, const int stride_row, const int stride_col_y2, const int stride_col_dst, const uint3 channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, const uint3 sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, const int ids_stride) { + const T * GGML_CUDA_RESTRICT x = x_ptr; + const float * GGML_CUDA_RESTRICT y = y_ptr; + const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; const int row = blockIdx.x; // for MUL_MAT_ID - blockIdx.y = n_expert_used, blockIdx.z = ncols_dst (tokens) const int channel_dst = blockIdx.y; @@ -21,6 +26,7 @@ static __global__ void mul_mat_vec_f( int channel_y; int sample_dst; + ggml_cuda_pdl_sync(); if constexpr (is_multi_token_id) { // Multi-token MUL_MAT_ID path, adding these in the normal path causes a perf regression for n_tokens=1 case token_idx = blockIdx.z; @@ -51,6 +57,7 @@ static __global__ void mul_mat_vec_f( bool use_bias = false; bool use_gate_bias = false; ggml_glu_op glu_op = ggml_glu_op::GGML_GLU_OP_SWIGLU; + float glu_limit = 0.0f; const T * gate_x = nullptr; const float * x_bias = nullptr; const float * gate_bias = nullptr; @@ -60,6 +67,7 @@ static __global__ void mul_mat_vec_f( use_bias = fusion.x_bias != nullptr; use_gate_bias = fusion.gate_bias != nullptr; glu_op = fusion.glu_op; + glu_limit = fusion.glu_limit; if (use_gate) { gate_x = static_cast(fusion.gate); @@ -79,9 +87,8 @@ static __global__ void mul_mat_vec_f( gate_x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row*stride_row; } - const int channel_bias = ids ? channel_x : channel_dst; - if constexpr (has_fusion) { + const int channel_bias = ids ? channel_x : channel_dst; if (use_bias) { x_bias += int64_t(sample_dst)*stride_sample_dst + channel_bias*stride_channel_dst; } @@ -94,7 +101,7 @@ static __global__ void mul_mat_vec_f( extern __shared__ char data_mmv[]; float * buf_iw = (float *) data_mmv; - float * buf_iw_gate = nullptr; + [[maybe_unused]] float * buf_iw_gate = nullptr; if constexpr (has_fusion) { buf_iw_gate = (float *) (data_mmv + warp_size*sizeof(float)); } @@ -122,7 +129,7 @@ static __global__ void mul_mat_vec_f( if constexpr (std::is_same_v) { const float2 * x2 = (const float2 *) x; - const float2 * gate_x2 = nullptr; + [[maybe_unused]] const float2 * gate_x2 = nullptr; if constexpr (has_fusion) { if (use_gate) { gate_x2 = (const float2 *) gate_x; @@ -154,7 +161,7 @@ static __global__ void mul_mat_vec_f( } } else if constexpr (std::is_same_v) { const half2 * x2 = (const half2 *) x; - const half2 * gate_x2 = nullptr; + [[maybe_unused]] const half2 * gate_x2 = nullptr; if constexpr (has_fusion) { if (use_gate) { gate_x2 = (const half2 *) gate_x; @@ -265,7 +272,7 @@ static __global__ void mul_mat_vec_f( } #else const nv_bfloat162 * x2 = (const nv_bfloat162 *) x; - const nv_bfloat162 * gate_x2 = nullptr; + [[maybe_unused]] const nv_bfloat162 * gate_x2 = nullptr; if constexpr (has_fusion) { if (use_gate) { gate_x2 = (const nv_bfloat162 *) gate_x; @@ -273,7 +280,7 @@ static __global__ void mul_mat_vec_f( } for (int col2 = tid; col2 < ncols2; col2 += block_size) { const nv_bfloat162 tmpx = x2[col2]; - nv_bfloat162 tmpx_gate; + [[maybe_unused]] nv_bfloat162 tmpx_gate; if constexpr (has_fusion) { if (use_gate) { tmpx_gate = gate_x2[col2]; @@ -298,6 +305,7 @@ static __global__ void mul_mat_vec_f( static_assert(std::is_same_v, "unsupported type"); } + ggml_cuda_pdl_lc(); #pragma unroll for (int j = 0; j < ncols_dst; ++j) { sumf[j] = warp_reduce_sum(sumf[j]); @@ -360,6 +368,9 @@ static __global__ void mul_mat_vec_f( value = ggml_cuda_op_swiglu_oai_single(gate_value, value); break; } + case GGML_GLU_OP_SWIGLU_CLAMP: + value = ggml_cuda_op_swiglu_clamp_single(gate_value, value, glu_limit); + break; default: break; } @@ -369,7 +380,7 @@ static __global__ void mul_mat_vec_f( dst[tid*stride_col_dst + row] = value; if constexpr (!has_fusion) { - GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, gate_x, x_bias, gate_bias, sumf_gate); + GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, glu_limit, gate_x, x_bias, gate_bias, sumf_gate); } } @@ -382,11 +393,13 @@ static void mul_mat_vec_f_switch_fusion( const uint3 sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, const dim3 & block_dims, const dim3 & block_nums, const int nbytes_shared, const int ids_stride, const cudaStream_t stream) { + const ggml_cuda_kernel_launch_params launch_params = {block_nums, block_dims, nbytes_shared, stream}; + const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; if constexpr (ncols_dst == 1) { if (has_fusion) { - mul_mat_vec_f<<>> - (x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst, + ggml_cuda_kernel_launch(mul_mat_vec_f, launch_params, + x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); return; @@ -395,8 +408,8 @@ static void mul_mat_vec_f_switch_fusion( GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1"); - mul_mat_vec_f<<>> - (x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst, + ggml_cuda_kernel_launch(mul_mat_vec_f, launch_params, + x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); @@ -491,9 +504,8 @@ void launch_mul_mat_vec_f_cuda( channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); } break; - default: { + default: GGML_ABORT("fatal error"); - } break; } } @@ -590,7 +602,6 @@ static void mul_mat_vec_f_cuda_switch_ncols_dst( break; default: GGML_ABORT("fatal error"); - break; } } @@ -668,6 +679,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor fusion_local.gate_bias = fusion->gate_bias->data; } fusion_local.glu_op = fusion->glu_op; + fusion_local.glu_limit = fusion->glu_limit; } const int64_t s01 = src0->nb[1] / ts_src0; @@ -776,7 +788,7 @@ void ggml_cuda_op_mul_mat_vec_f( GGML_UNUSED_VARS(ctx, src1, dst, src1_ddq_i, src1_ncols, src1_padded_row_size); } -bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) { +bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) { if (src0_ne[0] % 2 != 0) { return false; } @@ -807,12 +819,16 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0 if (fp32_mma_hardware_available(cc)) { return ne11 <= 3; } - return ne11 <= 8; + return ne11 <= MMVF_MAX_BATCH_SIZE; } - return ne11 <= 8; + return ne11 <= MMVF_MAX_BATCH_SIZE; case GGML_TYPE_F16: if (GGML_CUDA_CC_IS_NVIDIA(cc)) { const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1); + // MMF needs full row tiles, for other row counts MMVF still beats cuBLAS at small batch size + if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) { + return ne11 <= MMVF_MAX_BATCH_SIZE; + } if (ampere_mma_available(cc)) { return src0_small && ne11 == 1; } @@ -822,8 +838,11 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0 if (fp16_mma_hardware_available(cc)) { return src0_small && ne11 <= 3; } - return ne11 <= 8; + return ne11 <= MMVF_MAX_BATCH_SIZE; } else if (GGML_CUDA_CC_IS_AMD(cc)) { + if (GGML_CUDA_CC_IS_RDNA(cc) && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) { + return ne11 <= MMVF_MAX_BATCH_SIZE; + } if (fp16_mma_hardware_available(cc)) { if (GGML_CUDA_CC_IS_RDNA3(cc)) { return ne11 <= 3; @@ -833,12 +852,16 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0 } return ne11 <= 2; } - return ne11 <= 8; + return ne11 <= MMVF_MAX_BATCH_SIZE; } - return ne11 <= 8; + return ne11 <= MMVF_MAX_BATCH_SIZE; case GGML_TYPE_BF16: if (GGML_CUDA_CC_IS_NVIDIA(cc)) { const bool src0_small = (src0_ne[1] <= 512 || src0_ne[2]*src0_ne[3] == 1); + // MMF needs full row tiles, for other row counts MMVF still beats cuBLAS at small batch size + if (src0_small && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) { + return ne11 <= MMVF_MAX_BATCH_SIZE; + } if (ampere_mma_available(cc)) { return src0_small && ne11 == 1; } @@ -848,14 +871,17 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0 if (bf16_mma_hardware_available(cc)) { return src0_small && ne11 <= 3; } - return ne11 <= 8; + return ne11 <= MMVF_MAX_BATCH_SIZE; } else if (GGML_CUDA_CC_IS_AMD(cc)) { + if (GGML_CUDA_CC_IS_RDNA(cc) && !ggml_cuda_should_use_mmf(type, cc, warp_size, src0_ne, src0_nb, ne11, /*mul_mat_id =*/ false)) { + return ne11 <= MMVF_MAX_BATCH_SIZE; + } if (bf16_mma_hardware_available(cc)) { return ne11 <= 3; } - return ne11 <= 8; + return ne11 <= MMVF_MAX_BATCH_SIZE; } - return ne11 <= 8; + return ne11 <= MMVF_MAX_BATCH_SIZE; default: return false; } diff --git a/ggml/src/ggml-cuda/mmvf.cuh b/ggml/src/ggml-cuda/mmvf.cuh index a50f7c0..350b5cc 100644 --- a/ggml/src/ggml-cuda/mmvf.cuh +++ b/ggml/src/ggml-cuda/mmvf.cuh @@ -11,4 +11,4 @@ void ggml_cuda_op_mul_mat_vec_f( const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream); -bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11); +bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11); diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 8f55cac..930f1ad 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -4,12 +4,43 @@ #include "vecdotq.cuh" #include +#include + +// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel +// has little exposed latency left to hide and the extra requests cost more than they save. +// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031 +#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK +// returns true only for those quants that benefit from prefetch and false otherwise +static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) { + switch (type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q8_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return true; + default: + return false; + } +} + +static __device__ __forceinline__ void mmvq_prefetch_l2(const void * p) { + asm volatile("prefetch.global.L2 [%0];" :: "l"(p)); +} +#endif typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs); static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return vec_dot_q1_0_q8_1; + case GGML_TYPE_Q2_0: return vec_dot_q2_0_q8_1; case GGML_TYPE_Q4_0: return vec_dot_q4_0_q8_1; case GGML_TYPE_Q4_1: return vec_dot_q4_1_q8_1; case GGML_TYPE_Q5_0: return vec_dot_q5_0_q8_1; @@ -38,6 +69,7 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return VDR_Q1_0_Q8_1_MMVQ; + case GGML_TYPE_Q2_0: return VDR_Q2_0_Q8_1_MMVQ; case GGML_TYPE_Q4_0: return VDR_Q4_0_Q8_1_MMVQ; case GGML_TYPE_Q4_1: return VDR_Q4_1_Q8_1_MMVQ; case GGML_TYPE_Q5_0: return VDR_Q5_0_Q8_1_MMVQ; @@ -63,10 +95,12 @@ static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) { enum mmvq_parameter_table_id { MMVQ_PARAMETERS_GENERIC = 0, + MMVQ_PARAMETERS_TURING, MMVQ_PARAMETERS_GCN, MMVQ_PARAMETERS_RDNA2, MMVQ_PARAMETERS_RDNA3_0, - MMVQ_PARAMETERS_RDNA4 + MMVQ_PARAMETERS_RDNA4, + MMVQ_PARAMETERS_GB10 }; static constexpr __device__ mmvq_parameter_table_id get_device_table_id() { @@ -78,6 +112,10 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() { return MMVQ_PARAMETERS_RDNA2; #elif defined(GCN) || defined(CDNA) return MMVQ_PARAMETERS_GCN; +#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE + return MMVQ_PARAMETERS_TURING; +#elif __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK + return MMVQ_PARAMETERS_GB10; #else return MMVQ_PARAMETERS_GENERIC; #endif @@ -96,6 +134,12 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) { if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) { return MMVQ_PARAMETERS_GCN; } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) { + return MMVQ_PARAMETERS_TURING; + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) { + return MMVQ_PARAMETERS_GB10; + } return MMVQ_PARAMETERS_GENERIC; } @@ -115,6 +159,7 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_pascal_older(gg case GGML_TYPE_IQ4_NL: return 6; case GGML_TYPE_IQ4_XS: return 5; case GGML_TYPE_MXFP4: return 4; + case GGML_TYPE_NVFP4: return 4; case GGML_TYPE_Q2_K: return 4; case GGML_TYPE_Q3_K: return 4; case GGML_TYPE_Q4_0: return 6; @@ -135,6 +180,7 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_turing_plus(ggm case GGML_TYPE_IQ3_S: return 6; case GGML_TYPE_IQ3_XXS: return 7; case GGML_TYPE_MXFP4: return 7; + case GGML_TYPE_NVFP4: return 8; case GGML_TYPE_Q2_K: return 7; case GGML_TYPE_Q3_K: return 5; default: return MMVQ_MAX_BATCH_SIZE; @@ -221,6 +267,7 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type case GGML_TYPE_IQ4_NL: return 7; case GGML_TYPE_IQ4_XS: return 5; case GGML_TYPE_MXFP4: return 5; + case GGML_TYPE_NVFP4: return 5; case GGML_TYPE_Q3_K: return 4; case GGML_TYPE_Q4_0: return 7; case GGML_TYPE_Q4_1: return 7; @@ -236,9 +283,9 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type // Host function: returns the max batch size for the current arch+type at runtime. int get_mmvq_mmid_max_batch(ggml_type type, int cc) { - // NVIDIA: Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID. + // NVIDIA: P100, Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID. if (GGML_CUDA_CC_IS_NVIDIA(cc)) { - if (cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) { + if (cc == GGML_CUDA_CC_PASCAL || cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) { return MMVQ_MAX_BATCH_SIZE; } if (cc >= GGML_CUDA_CC_TURING) { @@ -268,6 +315,118 @@ int get_mmvq_mmid_max_batch(ggml_type type, int cc) { return MMVQ_MAX_BATCH_SIZE; } +bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { + if (!ggml_is_quantized(type)) { + return false; + } + // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner. + // Only list quant-types MMQ supports, others would fall back to cuBLAS. + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) { + switch (type) { // tuned on RTX 4090 + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q3_K: + return ne11 <= 6; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) { + switch (type) { // tuned on RTX 5090 + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + return ne11 <= 5; + case GGML_TYPE_Q5_K: + return ne11 <= 6; + case GGML_TYPE_Q6_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + switch (type) { // tuned on DGX Spark GB10 + case GGML_TYPE_Q2_K: + return ne11 <= 6; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ORIN) { + switch (type) { // tuned for Jetson Orin + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + return ne11 <= 1; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_VOLTA) { + switch (type) { + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q3_K: + return ne11 <= 6; + case GGML_TYPE_Q4_K: + return ne11 <= 5; + case GGML_TYPE_Q5_K: + return ne11 <= 6; + case GGML_TYPE_Q6_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_CDNA(cc)) { + if (GGML_CUDA_CC_IS_CDNA1(cc)) { + switch (type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + return ne11 <= 7; + case GGML_TYPE_Q5_1: + return ne11 <= 7; + case GGML_TYPE_Q8_0: + return ne11 <= 6; + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q3_K: + return ne11 <= 3; + case GGML_TYPE_Q4_K: + return ne11 <= 2; + case GGML_TYPE_Q5_K: + return ne11 <= 3; + case GGML_TYPE_Q6_K: + return ne11 <= 4; + case GGML_TYPE_IQ1_S: + return ne11 <= 5; + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_XS: + return ne11 <= 6; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + switch (type) { // tuned for CDNA2 + case GGML_TYPE_Q2_K: + return ne11 <= 5; + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 3; + case GGML_TYPE_Q6_K: + return ne11 <= 5; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + return ne11 <= MMVQ_MAX_BATCH_SIZE; +} + // Device constexpr: returns the max batch size for the current arch+type at compile time. template static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() { @@ -281,16 +440,16 @@ static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() { return get_mmvq_mmid_max_batch_cdna(type); #elif defined(GCN) return get_mmvq_mmid_max_batch_gcn(type); -#elif defined(__CUDA_ARCH__) && (__CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE) +#elif !defined(GGML_USE_MUSA) && (__CUDA_ARCH__ == GGML_CUDA_CC_PASCAL || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE) return MMVQ_MAX_BATCH_SIZE; -#elif defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING +#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING return get_mmvq_mmid_max_batch_turing_plus(type); #else return get_mmvq_mmid_max_batch_pascal_older(type); #endif } -static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_dst, mmvq_parameter_table_id table_id) { +static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_dst, mmvq_parameter_table_id table_id, bool small_k = false, bool halve_iters = false) { if (table_id == MMVQ_PARAMETERS_GENERIC) { switch (ncols_dst) { case 1: @@ -309,10 +468,12 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d } else if (table_id == MMVQ_PARAMETERS_GCN) { switch (ncols_dst) { case 1: + return 2; case 2: case 3: + return type == GGML_TYPE_Q3_K ? 1 : 2; case 4: - return 2; + return type == GGML_TYPE_Q6_K? 1 : 2; case 5: case 6: case 7: @@ -355,8 +516,9 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: - case GGML_TYPE_Q4_K: + return 8; case GGML_TYPE_Q6_K: + return 2; case GGML_TYPE_IQ4_NL: return 8; default: @@ -365,14 +527,82 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d } return 1; } + if (table_id == MMVQ_PARAMETERS_TURING) { + if (ncols_dst == 1) { + switch (type) { + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + return 2; + default: + return 4; + } + } + switch (ncols_dst) { + case 2: + case 3: + case 4: + return 4; + case 5: + case 6: + case 7: + case 8: + return 2; + default: + return 1; + } + } + if (table_id == MMVQ_PARAMETERS_GB10) { + const int generic = calc_nwarps(type, ncols_dst, MMVQ_PARAMETERS_GENERIC); + // Only worth the wider block when it actually retires the K loop in half the trips (Observation) + if (ncols_dst == 1 && !small_k && halve_iters) { + switch (type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ4_NL: + return 2 * generic; + default: + break; + } + } + return generic; + } return 1; } -static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) { - if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN) { +// Low bit types load few weight bytes per thread, a second row per block shares the q8_1 loads and adds memory parallelism. +static constexpr __host__ __device__ bool mmvq_two_rows_ncols1(ggml_type type) { + switch (type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + return true; + default: + return false; + } +} + +static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) { + if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) { switch (ncols_dst) { case 1: - return small_k ? nwarps : 1; + if (small_k) { + return nwarps; + } + if (table_id == MMVQ_PARAMETERS_GB10) { + return type == GGML_TYPE_IQ2_S || type == GGML_TYPE_IQ3_S ? 2 : 1; + } + return table_id == MMVQ_PARAMETERS_GENERIC && mmvq_two_rows_ncols1(type) ? 2 : 1; case 2: case 3: case 4: @@ -388,22 +618,26 @@ static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int return 1; } -template -__launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id())*ggml_cuda_get_physical_warp_size(), 1) +template +__launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1) static __global__ void mul_mat_vec_q( - const void * __restrict__ vx, const void * __restrict__ vy, const int32_t * __restrict__ ids, const ggml_cuda_mm_fusion_args_device fusion, float * __restrict__ dst, + const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y, - const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x, + uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio, const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst, const uint32_t ids_stride) { + const void * GGML_CUDA_RESTRICT vx = vx_ptr; + const void * GGML_CUDA_RESTRICT vy = vy_ptr; + const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; constexpr int qk = ggml_cuda_type_traits::qk; constexpr int qi = ggml_cuda_type_traits::qi; constexpr int vdr = get_vdr_mmvq(type); constexpr mmvq_parameter_table_id table_id = get_device_table_id(); - constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id); - constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps); + constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters); + constexpr int rows_per_cuda_block = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type); @@ -413,13 +647,20 @@ static __global__ void mul_mat_vec_q( const int blocks_per_row_x = ncols_x / qk; constexpr int blocks_per_iter = vdr * nwarps*warp_size / qi; - const uint32_t channel_dst = blockIdx.y; + const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1; + const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y; + if (shared_expert) { + vx = fusion.shared_up; + dst = fusion.shared_dst; + stride_col_dst = fusion.shared_stride_col_dst; + } uint32_t channel_x; uint32_t channel_y; uint32_t sample_dst; - channel_x = ncols_dst == 1 && ids ? ids[channel_dst] : fastdiv(channel_dst, channel_ratio); + ggml_cuda_pdl_sync(); + channel_x = shared_expert ? 0 : ncols_dst == 1 && ids ? ids[channel_dst] : fastdiv(channel_dst, channel_ratio); channel_y = ncols_dst == 1 && ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst; sample_dst = blockIdx.z; @@ -429,47 +670,66 @@ static __global__ void mul_mat_vec_q( bool use_gate = false; bool use_bias = false; bool use_gate_bias = false; - const void * vgate = nullptr; + bool use_scale = false; + bool use_gate_scale = false; + [[maybe_unused]] const void * vgate = nullptr; const float * x_bias = nullptr; const float * gate_bias = nullptr; + const float * x_scale = nullptr; + const float * gate_scale = nullptr; ggml_glu_op active_glu; + float glu_limit = 0.0f; if constexpr (has_fusion) { use_gate = fusion.gate != nullptr; use_bias = fusion.x_bias != nullptr; use_gate_bias = fusion.gate_bias != nullptr && use_gate; - vgate = fusion.gate; + vgate = shared_expert ? fusion.shared_gate : fusion.gate; x_bias = (const float *) fusion.x_bias; gate_bias = (const float *) fusion.gate_bias; active_glu = fusion.glu_op; + glu_limit = fusion.glu_limit; + if constexpr (type == GGML_TYPE_NVFP4) { + use_scale = fusion.x_scale != nullptr; + use_gate_scale = fusion.gate_scale != nullptr && use_gate; + x_scale = (const float *) fusion.x_scale; + gate_scale = (const float *) fusion.gate_scale; + } } - float x_biases[ncols_dst] = { 0.0f }; - float gate_biases[ncols_dst] = { 0.0f }; + [[maybe_unused]] float x_biases[ncols_dst] = { 0.0f }; + [[maybe_unused]] float gate_biases[ncols_dst] = { 0.0f }; + [[maybe_unused]] float x_scales = 1.0f; + [[maybe_unused]] float gate_scales = 1.0f; if constexpr (has_fusion) { + // 1. Hide latency by prefetching bias, gates and scales here + // 2. load only on threads that won't die after partial sum calculation const uint32_t channel_bias = ids ? channel_x : channel_dst; - if (use_bias) { - x_bias = x_bias + sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row0; - // 1. Hide latency by prefetching bias and gate here - // 2. load only on threads that won't die after partial sum calculation - if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 && - (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) { + if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 && + (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) { + if (use_bias) { + x_bias = x_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0; #pragma unroll for (int j = 0; j < ncols_dst; ++j) { x_biases[j] = x_bias[j * stride_col_dst + threadIdx.x]; } } - } - if (use_gate_bias) { - gate_bias = gate_bias + sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row0; - if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 && - (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) { + if (use_gate_bias) { + gate_bias = gate_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0; #pragma unroll for (int j = 0; j < ncols_dst; ++j) { gate_biases[j] = gate_bias[j * stride_col_dst + threadIdx.x]; } } + if constexpr (type == GGML_TYPE_NVFP4) { + if (use_scale) { + x_scales = x_scale[ids ? channel_x : 0]; + } + if (use_gate_scale) { + gate_scales = gate_scale[ids ? channel_x : 0]; + } + } } } @@ -486,6 +746,27 @@ static __global__ void mul_mat_vec_q( // x block quant index when casting the quants to int const int kqs = vdr * (tid % (qi/vdr)); +#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK + // start the next iterations' weight loads early + if constexpr (mmvq_should_prefetch(type)) { + // loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1 + constexpr int pf_dist = type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q6_K ? 1 : 2; + const int kbx_pf = kbx + pf_dist*blocks_per_iter; + if (kbx_pf < blocks_per_row_x) { +#pragma unroll + for (int i = 0; i < rows_per_cuda_block; ++i) { + const size_t off = (size_t)(kbx_offset + i*stride_row_x + kbx_pf) * ggml_cuda_type_traits::bs; + mmvq_prefetch_l2((const char *) vx + off); + if constexpr (has_fusion) { + if (use_gate) { + mmvq_prefetch_l2((const char *) vgate + off); + } + } + } + } + } +#endif + #pragma unroll for (int j = 0; j < ncols_dst; ++j) { #pragma unroll @@ -503,12 +784,7 @@ static __global__ void mul_mat_vec_q( } __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size]; - __shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size]; - if constexpr (!has_fusion) { - (void) tmp_shared_gate; - } else if (!use_gate) { - (void) tmp_shared_gate; - } + [[maybe_unused]] __shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size]; if (threadIdx.y > 0) { #pragma unroll @@ -551,42 +827,49 @@ static __global__ void mul_mat_vec_q( tmp_gate[j][i] = warp_reduce_sum(tmp_gate[j][i]); } } - } - if (threadIdx.x < rows_per_cuda_block && (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) { - float result = tmp[j][threadIdx.x]; - if constexpr (has_fusion) { - if (use_bias) { + if (threadIdx.x == i && (rows_per_cuda_block == 1 || uint32_t(row0 + i) < stride_col_dst)) { + float result = tmp[j][i]; + if constexpr (has_fusion) { + if constexpr (type == GGML_TYPE_NVFP4) { + result *= x_scales; + } result += x_biases[j]; - } - if (use_gate) { - float gate_value = tmp_gate[j][threadIdx.x]; - if (use_gate_bias) { + if (use_gate) { + float gate_value = tmp_gate[j][i]; + if constexpr (type == GGML_TYPE_NVFP4) { + gate_value *= gate_scales; + } gate_value += gate_biases[j]; - } - switch (active_glu) { - case GGML_GLU_OP_SWIGLU: - result *= ggml_cuda_op_silu_single(gate_value); - break; - case GGML_GLU_OP_GEGLU: - result *= ggml_cuda_op_gelu_single(gate_value); - break; - case GGML_GLU_OP_SWIGLU_OAI: { - result = ggml_cuda_op_swiglu_oai_single(gate_value, result); - break; + switch (active_glu) { + case GGML_GLU_OP_SWIGLU: + result *= ggml_cuda_op_silu_single(gate_value); + break; + case GGML_GLU_OP_GEGLU: + result *= ggml_cuda_op_gelu_single(gate_value); + break; + case GGML_GLU_OP_SWIGLU_OAI: + result = ggml_cuda_op_swiglu_oai_single(gate_value, result); + break; + case GGML_GLU_OP_SWIGLU_CLAMP: + result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit); + break; + default: + result = result * gate_value; + break; } - default: - result = result * gate_value; - break; } } + dst[j*stride_col_dst + i] = result; } - dst[j*stride_col_dst + threadIdx.x] = result; } } if constexpr (!has_fusion) { - GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, active_glu, gate_bias, x_bias, tmp_gate); + GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, glu_limit, gate_bias, x_bias, x_scale, gate_scale, tmp_gate); + } + if constexpr (type != GGML_TYPE_NVFP4) { + GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale, x_scales, gate_scales); } } @@ -594,15 +877,19 @@ static __global__ void mul_mat_vec_q( // Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst) // Block: (warp_size, ncols_dst) - each warp handles one token independently. // No shared memory reduction needed since each warp works alone. -template +template __launch_bounds__(get_mmvq_mmid_max_batch_for_device()*ggml_cuda_get_physical_warp_size(), 1) static __global__ void mul_mat_vec_q_moe( - const void * __restrict__ vx, const void * __restrict__ vy, const int32_t * __restrict__ ids, - float * __restrict__ dst, + const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, + float * dst_ptr, const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x, - const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst, + const uint32_t stride_row_x, const uint32_t stride_col_y, uint32_t stride_col_dst, const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint32_t ncols_dst, const uint32_t ids_stride) { + const void * GGML_CUDA_RESTRICT vx = vx_ptr; + const void * GGML_CUDA_RESTRICT vy = vy_ptr; + const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; constexpr int qk = ggml_cuda_type_traits::qk; constexpr int qi = ggml_cuda_type_traits::qi; @@ -611,18 +898,49 @@ static __global__ void mul_mat_vec_q_moe( constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type); + const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1; + if (shared_expert) { + vx = fusion.shared_up; + dst = fusion.shared_dst; + stride_col_dst = fusion.shared_stride_col_dst; + } + + // fuse gate, bias, scales, and glu_op into the up projection + bool use_gate = false; + const void * vgate = nullptr; + const float * x_bias = nullptr; + const float * gate_bias = nullptr; + const float * x_scale = nullptr; + const float * gate_scale = nullptr; + ggml_glu_op active_glu = GGML_GLU_OP_SWIGLU; + float glu_limit = 0.0f; + + if constexpr (has_fusion) { + use_gate = fusion.gate != nullptr; + vgate = shared_expert ? fusion.shared_gate : fusion.gate; + x_bias = (const float *) fusion.x_bias; + gate_bias = (const float *) fusion.gate_bias; + active_glu = fusion.glu_op; + glu_limit = fusion.glu_limit; + if constexpr (type == GGML_TYPE_NVFP4) { + x_scale = (const float *) fusion.x_scale; + gate_scale = (const float *) fusion.gate_scale; + } + } + const uint32_t token_idx = threadIdx.y; const int row0 = c_rows_per_block*blockIdx.x; const int blocks_per_row_x = ncols_x / qk; constexpr int blocks_per_iter = vdr * warp_size / qi; - const uint32_t channel_dst = blockIdx.y; + const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y; if (token_idx >= ncols_dst) { return; } - const uint32_t channel_x = ids[channel_dst + token_idx * ids_stride]; + ggml_cuda_pdl_sync(); + const uint32_t channel_x = shared_expert ? 0 : ids[channel_dst + token_idx * ids_stride]; const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y); const block_q8_1 * y = ((const block_q8_1 *) vy) + channel_y*stride_channel_y + token_idx*stride_col_y; @@ -630,6 +948,7 @@ static __global__ void mul_mat_vec_q_moe( // partial sum for each thread float tmp[c_rows_per_block] = {0.0f}; + float tmp_gate[c_rows_per_block] = {0.0f}; for (int kbx = threadIdx.x / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) { const int kby = kbx * (qk/QK8_1); @@ -638,34 +957,93 @@ static __global__ void mul_mat_vec_q_moe( #pragma unroll for (int i = 0; i < c_rows_per_block; ++i) { tmp[i] += vec_dot_q_cuda(vx, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs); + if constexpr (has_fusion) { + if (use_gate) { + tmp_gate[i] += vec_dot_q_cuda(vgate, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs); + } + } } } + ggml_cuda_pdl_lc(); + // Warp-level reduction only - no shared memory needed #pragma unroll for (int i = 0; i < c_rows_per_block; ++i) { tmp[i] = warp_reduce_sum(tmp[i]); + if constexpr (has_fusion) { + if (use_gate) { + tmp_gate[i] = warp_reduce_sum(tmp_gate[i]); + } + } } // Write results if (threadIdx.x < c_rows_per_block && (c_rows_per_block == 1 || uint32_t(row0 + threadIdx.x) < nrows_x)) { - dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = tmp[threadIdx.x]; + float result = tmp[threadIdx.x]; + if constexpr (has_fusion) { + const uint32_t bias_idx = channel_x*stride_channel_dst + row0 + threadIdx.x; + + if constexpr (type == GGML_TYPE_NVFP4) { + if (x_scale) { + result *= x_scale[channel_x]; + } + } + if (x_bias) { + result += x_bias[bias_idx]; + } + if (use_gate) { + float gate_value = tmp_gate[threadIdx.x]; + if constexpr (type == GGML_TYPE_NVFP4) { + if (gate_scale) { + gate_value *= gate_scale[channel_x]; + } + } + if (gate_bias) { + gate_value += gate_bias[bias_idx]; + } + switch (active_glu) { + case GGML_GLU_OP_SWIGLU: + result *= ggml_cuda_op_silu_single(gate_value); + break; + case GGML_GLU_OP_GEGLU: + result *= ggml_cuda_op_gelu_single(gate_value); + break; + case GGML_GLU_OP_SWIGLU_OAI: + result = ggml_cuda_op_swiglu_oai_single(gate_value, result); + break; + case GGML_GLU_OP_SWIGLU_CLAMP: + result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit); + break; + default: + result = result * gate_value; + break; + } + } + } + dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = result; + } + + if constexpr (!has_fusion) { + GGML_UNUSED_VARS(use_gate, tmp_gate, vgate, x_bias, gate_bias, active_glu, glu_limit, x_scale, gate_scale); + } else if constexpr (type != GGML_TYPE_NVFP4) { + GGML_UNUSED_VARS(x_scale, gate_scale); } } template static std::pair calc_launch_params( const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens, - const int warp_size, const mmvq_parameter_table_id table_id, const bool small_k = false) { - const int nwarps = calc_nwarps(type, ncols_dst, table_id); - const int rpb = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps); + const int warp_size, const mmvq_parameter_table_id table_id, const bool small_k = false, const bool halve_iters = false) { + const int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters); + const int rpb = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps); const int64_t nblocks = (nrows_x + rpb - 1) / rpb; const dim3 block_nums(nblocks, nchannels_dst, nsamples_or_ntokens); const dim3 block_dims(warp_size, nwarps, 1); return {block_nums, block_dims}; } -template +template static void mul_mat_vec_q_switch_fusion( const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y, @@ -675,11 +1053,13 @@ static void mul_mat_vec_q_switch_fusion( const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared, const uint32_t ids_stride, cudaStream_t stream) { - const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; + const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr || + fusion.x_scale != nullptr || fusion.gate_scale != nullptr; if constexpr (c_ncols_dst == 1) { if (has_fusion) { - mul_mat_vec_q<<>> - (vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst, + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream); + ggml_cuda_kernel_launch(mul_mat_vec_q, launch_params, + vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); return; @@ -688,15 +1068,16 @@ static void mul_mat_vec_q_switch_fusion( GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1"); - mul_mat_vec_q<<>> - (vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst, + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream); + ggml_cuda_kernel_launch(mul_mat_vec_q, launch_params, + vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); } template static void mul_mat_vec_q_moe_launch( - const void * vx, const void * vy, const int32_t * ids, float * dst, + const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x, const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst, const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, @@ -705,14 +1086,26 @@ static void mul_mat_vec_q_moe_launch( constexpr int rows_per_block = 2; // 2 gives best perf based on tuning const int64_t nblocks_rows = (nrows_x + rows_per_block - 1) / rows_per_block; - const dim3 block_nums(nblocks_rows, nchannels_dst); + const dim3 block_nums(nblocks_rows, nchannels_dst + (fusion.shared_up != nullptr)); const dim3 block_dims(warp_size, ncols_dst); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + + const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr || + fusion.x_scale != nullptr || fusion.gate_scale != nullptr; - mul_mat_vec_q_moe<<>>( - vx, vy, ids, dst, ncols_x, nchannels_y, nrows_x, - stride_row_x, stride_col_y, stride_col_dst, - stride_channel_x, stride_channel_y, stride_channel_dst, - ncols_dst, ids_stride); + if (has_fusion) { + ggml_cuda_kernel_launch(mul_mat_vec_q_moe, launch_params, + vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x, + stride_row_x, stride_col_y, stride_col_dst, + stride_channel_x, stride_channel_y, stride_channel_dst, + ncols_dst, ids_stride); + } else { + ggml_cuda_kernel_launch(mul_mat_vec_q_moe, launch_params, + vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x, + stride_row_x, stride_col_y, stride_col_dst, + stride_channel_x, stride_channel_y, stride_channel_dst, + ncols_dst, ids_stride); + } } template @@ -737,19 +1130,20 @@ static void mul_mat_vec_q_switch_ncols_dst( const int warp_size = ggml_cuda_info().devices[device].warp_size; const mmvq_parameter_table_id table_id = get_device_table_id(cc); - const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; const bool has_ids = ids != nullptr; + // How the K loop divides up at the baseline block width, both decisions below use these. + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int qi = ggml_cuda_type_traits::qi; + constexpr int vdr = get_vdr_mmvq(type); + const int blocks_per_row_x = ncols_x / qk; + const int blocks_per_iter_1warp = vdr * warp_size / qi; + const auto should_use_small_k = [&](int c_ncols_dst) { // When K is small, increase rows_per_block to match nwarps so each warp has more work to do // Trigger when the full thread block covers all K blocks in a single loop iteration and few threads remain idle. - constexpr int qk = ggml_cuda_type_traits::qk; - constexpr int qi = ggml_cuda_type_traits::qi; - constexpr int vdr = get_vdr_mmvq(type); - const int blocks_per_row_x = ncols_x / qk; - const int blocks_per_iter_1warp = vdr * warp_size / qi; - const int nwarps = calc_nwarps(type, c_ncols_dst, table_id); - bool use = nwarps > 1 && blocks_per_row_x < nwarps * blocks_per_iter_1warp; + const int nwarps = calc_nwarps(type, c_ncols_dst, table_id); + bool use = nwarps > 1 && blocks_per_row_x < nwarps * blocks_per_iter_1warp; constexpr std::array iq_slow_turing = { GGML_TYPE_IQ3_XXS, @@ -782,10 +1176,32 @@ static void mul_mat_vec_q_switch_ncols_dst( return use; }; + // Whether doubling nwarps pays off on the ncols_dst == 1 path, where K sets the K loop trip count. + const auto should_halve_iters = [&] { + if (table_id != MMVQ_PARAMETERS_GB10) { + return false; + } + + // Expert rows are gathered per token, so a wider block adds reduction work without reuse. + if (has_ids) { + return false; + } + + const int blocks_per_iter = calc_nwarps(type, 1, table_id) * blocks_per_iter_1warp; + const int iters = (blocks_per_row_x + blocks_per_iter - 1) / blocks_per_iter; + const int iters_wide = (blocks_per_row_x + blocks_per_iter * 2 - 1) / (blocks_per_iter * 2); + + // An odd trip count leaves half the wider block idle for its last iteration, that tail is + // only affordable once the loop is long enough to dilute it to an eighth of the work (observation). + const int idle = iters_wide * 2 - iters; + + return idle * 8 <= iters_wide * 2; + }; + if (has_ids && ncols_dst > 1) { // Multi-token MUL_MAT_ID path - dedicated MoE kernel mul_mat_vec_q_moe_launch( - vx, vy, ids, dst, ncols_x, nchannels_y_fd, nrows_x, + vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, nrows_x, stride_row_x, stride_col_y, stride_col_dst, stride_channel_x, stride_channel_y, stride_channel_dst, ncols_dst, ids_stride, warp_size, nchannels_dst, stream); @@ -794,26 +1210,34 @@ static void mul_mat_vec_q_switch_ncols_dst( switch (ncols_dst) { case 1: { - constexpr int c_ncols_dst = 1; - - bool use_small_k = should_use_small_k(c_ncols_dst); - - if (use_small_k) { - std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, - nsamples_dst, warp_size, table_id, true); - mul_mat_vec_q_switch_fusion( + // static, else MSVC lambda capture breaks the constexpr uses below + static constexpr int c_ncols_dst = 1; + + // Tag types keep the flags compile-time, so __launch_bounds__ matches what is launched. + const auto launch = [&](auto small_k_tag, auto halve_iters_tag) { + constexpr bool c_small_k = decltype(small_k_tag)::value; + // Types the table does not promote would compile a second, identical kernel. + constexpr bool c_promoted = + calc_nwarps(type, c_ncols_dst, MMVQ_PARAMETERS_GB10, false, true) != + calc_nwarps(type, c_ncols_dst, MMVQ_PARAMETERS_GB10, false, false); + + constexpr bool c_halve_iters = decltype(halve_iters_tag)::value && c_promoted; + + const std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst + (fusion.shared_up != nullptr), + nsamples_dst, warp_size, table_id, c_small_k, c_halve_iters); + mul_mat_vec_q_switch_fusion( vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, dims.first, dims.second, 0, ids_stride, stream); + }; + + if (should_use_small_k(c_ncols_dst)) { + launch(std::true_type{}, std::false_type{}); + } else if (should_halve_iters()) { + launch(std::false_type{}, std::true_type{}); } else { - std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, - nsamples_dst, warp_size, table_id); - mul_mat_vec_q_switch_fusion( - vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst, - channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, - stride_sample_x, stride_sample_y, stride_sample_dst, dims.first, dims.second, 0, ids_stride, - stream); + launch(std::false_type{}, std::false_type{}); } } break; case 2: { @@ -874,10 +1298,7 @@ static void mul_mat_vec_q_switch_ncols_dst( } break; default: GGML_ABORT("fatal error"); - break; } - - GGML_UNUSED(has_fusion); } static void mul_mat_vec_q_switch_type( const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, @@ -894,6 +1315,12 @@ static void mul_mat_vec_q_switch_type( nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); break; + case GGML_TYPE_Q2_0: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; case GGML_TYPE_Q4_0: mul_mat_vec_q_switch_ncols_dst (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, @@ -1022,7 +1449,6 @@ static void mul_mat_vec_q_switch_type( break; default: GGML_ABORT("fatal error"); - break; } } @@ -1055,8 +1481,29 @@ void ggml_cuda_mul_mat_vec_q( ggml_cuda_mm_fusion_args_device fusion_local{}; if (fusion) { - GGML_ASSERT( !ids || dst->ne[2] == 1); + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + GGML_ASSERT( !ids || dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc)); GGML_ASSERT( ids || dst->ne[1] == 1); + // Scale fusion is only allowed for NVFP4 currently as the cost of checking this at run-time in the prologue is + // non-negligible for some models such as gpt-oss-20b + GGML_ASSERT((fusion->x_scale == nullptr && fusion->gate_scale == nullptr) || src0->type == GGML_TYPE_NVFP4); + + if (fusion->shared_up) { + GGML_ASSERT(ids && fusion->gate && fusion->shared_gate && fusion->shared_dst); + GGML_ASSERT(!fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale); + GGML_ASSERT(ne11 == 1 && ne03 == 1 && ne13 == 1); + GGML_ASSERT(fusion->shared_up->type == src0->type && fusion->shared_gate->type == src0->type); + GGML_ASSERT(ggml_are_same_shape(fusion->shared_up, fusion->shared_gate)); + GGML_ASSERT(ggml_is_contiguous(fusion->shared_up) && ggml_is_contiguous(fusion->shared_gate)); + GGML_ASSERT(fusion->shared_up->ne[0] == ne00 && fusion->shared_up->ne[1] == ne01); + GGML_ASSERT(fusion->shared_up->nb[1] == nb01 && ggml_is_matrix(fusion->shared_up)); + GGML_ASSERT(fusion->shared_dst->type == GGML_TYPE_F32 && ggml_is_contiguous(fusion->shared_dst)); + GGML_ASSERT(fusion->shared_dst->ne[0] == ne0 && fusion->shared_dst->ne[1] == ne2); + fusion_local.shared_up = fusion->shared_up->data; + fusion_local.shared_gate = fusion->shared_gate->data; + fusion_local.shared_dst = (float *) fusion->shared_dst->data; + fusion_local.shared_stride_col_dst = fusion->shared_dst->nb[1] / ts_dst; + } if (fusion->x_bias) { GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32); @@ -1074,7 +1521,20 @@ void ggml_cuda_mul_mat_vec_q( GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]); fusion_local.gate_bias = fusion->gate_bias->data; } + if (fusion->x_scale) { + GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(fusion->x_scale)); + GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1)); + fusion_local.x_scale = fusion->x_scale->data; + } + if (fusion->gate_scale) { + GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale)); + GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1)); + fusion_local.gate_scale = fusion->gate_scale->data; + } fusion_local.glu_op = fusion->glu_op; + fusion_local.glu_limit = fusion->glu_limit; } // If src0 is a temporary compute buffer, clear any potential padding. diff --git a/ggml/src/ggml-cuda/mmvq.cuh b/ggml/src/ggml-cuda/mmvq.cuh index 6bf0a8e..5605bf7 100644 --- a/ggml/src/ggml-cuda/mmvq.cuh +++ b/ggml/src/ggml-cuda/mmvq.cuh @@ -2,6 +2,8 @@ #define MMVQ_MAX_BATCH_SIZE 8 // Max. batch size for which to use MMVQ kernels. +bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11); + // Returns the maximum batch size for which MMVQ should be used for MUL_MAT_ID, // based on the quantization type and GPU architecture (compute capability). int get_mmvq_mmid_max_batch(ggml_type type, int cc); diff --git a/ggml/src/ggml-cuda/moe-weighted-reduction.cu b/ggml/src/ggml-cuda/moe-weighted-reduction.cu new file mode 100644 index 0000000..11ec584 --- /dev/null +++ b/ggml/src/ggml-cuda/moe-weighted-reduction.cu @@ -0,0 +1,65 @@ +#include "moe-weighted-reduction.cuh" + +static __global__ void moe_weighted_reduction_f32(const float * __restrict__ experts, + const float * __restrict__ expert_scale, + const float * __restrict__ weights, + float * __restrict__ dst, + const int64_t n_embd, + const int n_expert_used) { + const int64_t token = blockIdx.x; + const int64_t col = (int64_t) blockIdx.y * blockDim.x + threadIdx.x; + if (col >= n_embd) { + return; + } + + const uint64_t first_row = (uint64_t) token * n_expert_used; + const float first_scale = expert_scale != nullptr ? expert_scale[first_row] : 1.0f; + float sum = (experts[first_row * n_embd + col] * first_scale) * weights[first_row]; + + for (int expert = 1; expert < n_expert_used; ++expert) { + const uint64_t row = first_row + expert; + const float scale = expert_scale != nullptr ? expert_scale[row] : 1.0f; + sum += (experts[row * n_embd + col] * scale) * weights[row]; + } + dst[token * n_embd + col] = sum; +} + +static void launch_moe_weighted_reduction(const float * experts, + const float * expert_scale, + const float * weights, + float * dst, + int64_t n_embd, + int64_t n_tokens, + int n_expert_used, + cudaStream_t stream) { + constexpr int threads = 256; + const dim3 blocks(n_tokens, (n_embd + threads - 1) / threads, 1); + moe_weighted_reduction_f32 + <<>>(experts, expert_scale, weights, dst, n_embd, n_expert_used); +} + +void ggml_cuda_op_moe_weighted_reduction(ggml_backend_cuda_context & ctx, + const ggml_tensor * experts, + const ggml_tensor * expert_scale, + const ggml_tensor * weights, + ggml_tensor * dst) { + GGML_ASSERT(experts->type == GGML_TYPE_F32); + GGML_ASSERT(weights->type == GGML_TYPE_F32); + GGML_ASSERT(expert_scale == nullptr || expert_scale->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(experts)); + GGML_ASSERT(ggml_is_contiguous(weights)); + GGML_ASSERT(expert_scale == nullptr || ggml_is_contiguous(expert_scale)); + GGML_ASSERT(ggml_is_contiguous(dst)); + + const int64_t n_embd = experts->ne[0]; + const int64_t n_expert_used = experts->ne[1]; + const int64_t n_tokens = experts->ne[2] * experts->ne[3]; + cudaStream_t stream = ctx.stream(); + + launch_moe_weighted_reduction((const float *) experts->data, + expert_scale ? (const float *) expert_scale->data : nullptr, + (const float *) weights->data, + (float *) dst->data, n_embd, n_tokens, (int) n_expert_used, stream); + CUDA_CHECK(cudaGetLastError()); +} diff --git a/ggml/src/ggml-cuda/moe-weighted-reduction.cuh b/ggml/src/ggml-cuda/moe-weighted-reduction.cuh new file mode 100644 index 0000000..b72f947 --- /dev/null +++ b/ggml/src/ggml-cuda/moe-weighted-reduction.cuh @@ -0,0 +1,7 @@ +#include "common.cuh" + +void ggml_cuda_op_moe_weighted_reduction(ggml_backend_cuda_context & ctx, + const ggml_tensor * experts, + const ggml_tensor * expert_scale, + const ggml_tensor * weights, + ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index ef98f67..3f0ea6f 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -3,37 +3,46 @@ template static __global__ void norm_f32( - const float * x, float * dst, const int ncols, const int64_t stride_row, const int64_t stride_channel, - const int64_t stride_sample, const float eps) { - const int nrows = gridDim.x; - const int nchannels = gridDim.y; + const float * x, float * dst, const int ncols, const int nchannels, const int nsamples, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps) { + const int nrows = gridDim.x; + const int row = blockIdx.x; + const int tid = threadIdx.x; - const int row = blockIdx.x; - const int channel = blockIdx.y; - const int sample = blockIdx.z; - const int tid = threadIdx.x; + extern __shared__ float2 s_sum2[]; - x += sample*stride_sample + channel*stride_channel + row*stride_row; - dst += ((sample*nchannels + channel)*nrows + row)*ncols; + ggml_cuda_pdl_sync(); - float2 mean_var = make_float2(0.0f, 0.0f); + // grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples + for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { + for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { + const float * xc = x + sample*stride_sample + channel*stride_channel + row*stride_row; + float * dstc = dst + ((sample*nchannels + channel)*nrows + row)*ncols; - for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; - mean_var.x += xi; - mean_var.y += xi * xi; - } + float2 mean_var = make_float2(0.0f, 0.0f); - // sum up partial sums - extern __shared__ float2 s_sum2[]; - mean_var = block_reduce(mean_var, s_sum2); + for (int col = tid; col < ncols; col += block_size) { + const float xi = xc[col]; + mean_var.x += xi; + mean_var.y += xi * xi; + } - const float mean = mean_var.x / ncols; - const float var = mean_var.y / ncols - mean * mean; - const float inv_std = rsqrtf(var + eps); + // sum up partial sums + mean_var = block_reduce(mean_var, s_sum2); - for (int col = tid; col < ncols; col += block_size) { - dst[col] = (x[col] - mean) * inv_std; + const float mean = mean_var.x / ncols; + const float var = mean_var.y / ncols - mean * mean; + const float inv_std = rsqrtf(var + eps); + + for (int col = tid; col < ncols; col += block_size) { + dstc[col] = (xc[col] - mean) * inv_std; + } + + if constexpr (block_size > WARP_SIZE) { + // sync is needed as we reuse s_sum2 across block_reduce invocations, see #26385 + __syncthreads(); + } + } } } @@ -46,6 +55,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr float tmp = 0.0f; // partial sum for thread in warp + ggml_cuda_pdl_sync(); for (int j = start; j < end; j += block_size) { tmp += x[j]; } @@ -62,7 +72,7 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr tmp += xi * xi; } - tmp = block_reduce(tmp, s_sum); + tmp = block_reduce(tmp, s_sum + 32); const float variance = tmp / group_size; const float scale = rsqrtf(variance + eps); @@ -71,10 +81,12 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr } } -template +template static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, + const int nchannels, + const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, @@ -94,58 +106,125 @@ static __global__ void rms_norm_f32(const float * x, const uint3 add_ncols_packed = make_uint3(0, 0, 0), const uint3 add_nrows_packed = make_uint3(0, 0, 0), const uint3 add_nchannels_packed = make_uint3(0, 0, 0), - const uint3 add_nsamples_packed = make_uint3(0, 0, 0)) { - const int nrows = gridDim.x; - const int nchannels = gridDim.y; - - const int row = blockIdx.x; - const int channel = blockIdx.y; - const int sample = blockIdx.z; - const int tid = threadIdx.x; + const uint3 add_nsamples_packed = make_uint3(0, 0, 0), + const float scale_out = 1.0f) { + ggml_cuda_pdl_lc(); + const int nrows = gridDim.x; + const int row = blockIdx.x; + const int tid = threadIdx.x; static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying"); + static_assert(!do_scale || !do_multiply, "fusing scale is not supported with multiplying"); - x += sample*stride_sample + channel*stride_channel + row*stride_row; - dst += ((sample*nchannels + channel)*nrows + row)*ncols; - - if constexpr (do_multiply) { - const uint32_t mul_row = fastmodulo(row, mul_nrows_packed); - const uint32_t mul_channel = fastmodulo(channel, mul_nchannels_packed); - const uint32_t mul_sample = fastmodulo(sample, mul_nsamples_packed); - mul += mul_sample * mul_stride_sample + mul_channel * mul_stride_channel + mul_row * mul_stride_row; - } - - if constexpr (do_add) { - const int add_row = fastmodulo(row, add_nrows_packed); - const int add_channel = fastmodulo(channel, add_nchannels_packed); - const int add_sample = fastmodulo(sample, add_nsamples_packed); - add += add_sample * add_stride_sample + add_channel * add_stride_channel + add_row * add_stride_row; - } - - float tmp = 0.0f; // partial sum for thread in warp - - for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; - tmp += xi * xi; - } - - // sum up partial sums extern __shared__ float s_sum[]; - tmp = block_reduce(tmp, s_sum); - - const float mean = tmp / ncols; - const float scale = rsqrtf(mean + eps); - for (int col = tid; col < ncols; col += block_size) { - if constexpr (do_multiply && do_add) { - const int mul_col = fastmodulo(col, mul_ncols_packed); - const int add_col = fastmodulo(col, add_ncols_packed); - dst[col] = scale * x[col] * mul[mul_col] + add[add_col]; - } else if constexpr (do_multiply) { - const int mul_col = fastmodulo(col, mul_ncols_packed); - dst[col] = scale * x[col] * mul[mul_col]; - } else { - dst[col] = scale * x[col]; + ggml_cuda_pdl_sync(); + + // grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples + for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { + for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { + const float * xc = x + sample*stride_sample + channel*stride_channel + row*stride_row; + float * dstc = dst + ((sample*nchannels + channel)*nrows + row)*ncols; + + [[maybe_unused]] const float * mulc = nullptr; + if constexpr (do_multiply) { + const uint32_t mul_row = fastmodulo(row, mul_nrows_packed); + const uint32_t mul_channel = fastmodulo(channel, mul_nchannels_packed); + const uint32_t mul_sample = fastmodulo(sample, mul_nsamples_packed); + mulc = mul + mul_sample * mul_stride_sample + mul_channel * mul_stride_channel + mul_row * mul_stride_row; + } + + [[maybe_unused]] const float * addc = nullptr; + if constexpr (do_add) { + const int add_row = fastmodulo(row, add_nrows_packed); + const int add_channel = fastmodulo(channel, add_nchannels_packed); + const int add_sample = fastmodulo(sample, add_nsamples_packed); + addc = add + add_sample * add_stride_sample + add_channel * add_stride_channel + add_row * add_stride_row; + } + + if constexpr (do_vec) { + const int ncols4 = ncols / 4; + const float4 * x4 = (const float4 *) xc; + const float4 * mul4 = (const float4 *) mulc; + const float4 * add4 = (const float4 *) addc; + float4 * dst4 = (float4 *) dstc; + + float4 xs[vpt]; + float tmp = 0.0f; + +#pragma unroll + for (int k = 0; k < vpt; k++) { + const int i = tid + k*block_size; + if (i < ncols4) { + xs[k] = x4[i]; + tmp += xs[k].x*xs[k].x + xs[k].y*xs[k].y + xs[k].z*xs[k].z + xs[k].w*xs[k].w; + } + } + tmp = block_reduce(tmp, s_sum); + const float scale = rsqrtf(tmp / ncols + eps); + +#pragma unroll + for (int k = 0; k < vpt; k++) { + const int i = tid + k*block_size; + if (i < ncols4) { + float4 r = make_float4(scale*xs[k].x, scale*xs[k].y, scale*xs[k].z, scale*xs[k].w); + if constexpr (do_multiply) { + const float4 m = mul4[i]; + r.x *= m.x; + r.y *= m.y; + r.z *= m.z; + r.w *= m.w; + } + if constexpr (do_add) { + const float4 a = add4[i]; + r.x += a.x; + r.y += a.y; + r.z += a.z; + r.w += a.w; + } + if constexpr (do_scale) { + r.x *= scale_out; + r.y *= scale_out; + r.z *= scale_out; + r.w *= scale_out; + } + dst4[i] = r; + } + } + } else { + float tmp = 0.0f; // partial sum for thread in warp + + for (int col = tid; col < ncols; col += block_size) { + const float xi = xc[col]; + tmp += xi * xi; + } + + // sum up partial sums + tmp = block_reduce(tmp, s_sum); + + const float mean = tmp / ncols; + const float scale = rsqrtf(mean + eps); + + for (int col = tid; col < ncols; col += block_size) { + if constexpr (do_multiply && do_add) { + const int mul_col = fastmodulo(col, mul_ncols_packed); + const int add_col = fastmodulo(col, add_ncols_packed); + dstc[col] = scale * xc[col] * mulc[mul_col] + addc[add_col]; + } else if constexpr (do_multiply) { + const int mul_col = fastmodulo(col, mul_ncols_packed); + dstc[col] = scale * xc[col] * mulc[mul_col]; + } else if constexpr (do_scale) { + dstc[col] = scale_out * (scale * xc[col]); + } else { + dstc[col] = scale * xc[col]; + } + } + } + + if constexpr (block_size > WARP_SIZE) { + // sync is needed as we reuse s_sum across block_reduce invocations, see #26385 + __syncthreads(); + } } } } @@ -163,6 +242,7 @@ static __global__ void rms_norm_back_f32( float sum_xx = 0.0f; // sum for squares of x, equivalent to forward pass float sum_xg = 0.0f; // sum for x * gradient, needed because RMS norm mixes inputs + ggml_cuda_pdl_sync(); for (int col = tid; col < ncols; col += block_size) { const float xfi = xf[col]; sum_xx += xfi * xfi; @@ -238,48 +318,57 @@ static __global__ void rms_norm_back_f32( template static __global__ void l2_norm_f32( - const float * x, float * dst, const int ncols, const int64_t stride_row, const int64_t stride_channel, - const int64_t stride_sample, const float eps) { - const int nrows = gridDim.x; - const int nchannels = gridDim.y; + const float * x, float * dst, const int ncols, const int nchannels, const int nsamples, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps) { + const int nrows = gridDim.x; + const int row = blockIdx.x; + const int tid = threadIdx.x; - const int row = blockIdx.x; - const int channel = blockIdx.y; - const int sample = blockIdx.z; - const int tid = threadIdx.x; + extern __shared__ float s_sum[]; - x += sample*stride_sample + channel*stride_channel + row*stride_row; - dst += ((sample*nchannels + channel)*nrows + row)*ncols; + ggml_cuda_pdl_sync(); - float tmp = 0.0f; // partial sum for thread in warp + // grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples + for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { + for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { + const float * xc = x + sample*stride_sample + channel*stride_channel + row*stride_row; + float * dstc = dst + ((sample*nchannels + channel)*nrows + row)*ncols; - for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col]; - tmp += xi * xi; - } + float tmp = 0.0f; // partial sum for thread in warp - // sum up partial sums - extern __shared__ float s_sum[]; - tmp = block_reduce(tmp, s_sum); + for (int col = tid; col < ncols; col += block_size) { + const float xi = xc[col]; + tmp += xi * xi; + } - // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html - const float scale = rsqrtf(fmaxf(tmp, eps * eps)); + // sum up partial sums + tmp = block_reduce(tmp, s_sum); - for (int col = tid; col < ncols; col += block_size) { - dst[col] = scale * x[col]; + // from https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html + const float scale = rsqrtf(fmaxf(tmp, eps * eps)); + + for (int col = tid; col < ncols; col += block_size) { + dstc[col] = scale * xc[col]; + } + + if constexpr (block_size > WARP_SIZE) { + // sync is needed as we reuse s_sum across block_reduce invocations, see #26385 + __syncthreads(); + } + } } } static void norm_f32_cuda( const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { - const dim3 blocks_num(nrows, nchannels, nsamples); + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); if (ncols < 1024) { const dim3 block_dims(WARP_SIZE, 1, 1); - norm_f32<<>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + norm_f32<<>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps); } else { const dim3 block_dims(1024, 1, 1); - norm_f32<1024><< WARP_SIZE ? 32 * sizeof(float2): 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + norm_f32<1024><< WARP_SIZE ? 32 * sizeof(float2): 0, stream>>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps); } } @@ -290,27 +379,93 @@ static void group_norm_f32_cuda( group_norm_f32<<>>(x, dst, group_size, ne_elements, eps); } else { const dim3 block_dims(1024, 1, 1); - group_norm_f32<1024><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, group_size, ne_elements, eps); + group_norm_f32<1024><< WARP_SIZE ? 2 * 32 * sizeof(float): 0, stream>>>(x, dst, group_size, ne_elements, eps); } } -static void rms_norm_f32_cuda( - const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, - const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { - const dim3 blocks_num(nrows, nchannels, nsamples); - if (ncols < 1024) { - const dim3 block_dims(256, 1, 1); - rms_norm_f32<256, false><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); +#define RMS_NORM_LAUNCH(BLOCK_SIZE, VPT, VEC) \ + do { \ + const dim3 block_dims(BLOCK_SIZE, 1, 1); \ + const ggml_cuda_kernel_launch_params launch_params{ blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float) : 0, stream }; \ + ggml_cuda_kernel_launch(rms_norm_f32, launch_params, \ + x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, \ + mul, mul_stride_row, mul_stride_channel, mul_stride_sample, \ + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, \ + add, add_stride_row, add_stride_channel, add_stride_sample, \ + add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out); \ + } while (0) + +template +static void rms_norm_f32_dispatch( + const dim3 blocks_num, + const ggml_tensor * x_t, float * dst, const int ncols, const int nchannels, const int nsamples, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, + const ggml_tensor * mul_t, const int64_t mul_stride_row, const int64_t mul_stride_channel, const int64_t mul_stride_sample, + const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed, + const ggml_tensor * add_t, const int64_t add_stride_row, const int64_t add_stride_channel, const int64_t add_stride_sample, + const uint3 add_ncols_packed, const uint3 add_nrows_packed, const uint3 add_nchannels_packed, const uint3 add_nsamples_packed, + cudaStream_t stream, const float scale_out = 1.0f) { + static_assert(!do_add || do_multiply, "fusing add is not supported without multiplying"); + static_assert(!do_scale || !do_multiply, "fusing scale is not supported with multiplying"); + + const float * x = (const float *) x_t->data; + const float * mul = mul_t ? (const float *) mul_t->data : nullptr; + const float * add = add_t ? (const float *) add_t->data : nullptr; + + // dst is written at row * ncols, i.e. always at a multiple of ncols, so only its base needs checking here + bool vec = ncols % 4 == 0 && ncols > 0 && ((uintptr_t) dst & 15) == 0; + vec = vec && ggml_cuda_is_aligned(x_t, 16); + if constexpr (do_multiply) { + vec = vec && mul_ncols_packed.z == (uint32_t) ncols + && ggml_cuda_is_aligned(mul_t, 16); + } + if constexpr (do_add) { + vec = vec && add_ncols_packed.z == (uint32_t) ncols + && ggml_cuda_is_aligned(add_t, 16); + } + + const int n4 = ncols / 4; + if (vec) { + if (n4 <= 32) { + RMS_NORM_LAUNCH(32, 1, true); + } else if (n4 <= 128) { + RMS_NORM_LAUNCH(128, 1, true); + } else if (n4 <= 256) { + RMS_NORM_LAUNCH(256, 1, true); + } else if (n4 <= 512) { + RMS_NORM_LAUNCH(512, 1, true); + } else if (n4 <= 1024) { + RMS_NORM_LAUNCH(1024, 1, true); + } else if (n4 <= 4096) { + RMS_NORM_LAUNCH(1024, 4, true); + } else { + RMS_NORM_LAUNCH(1024, 1, false); + } + } else if (ncols < 1024) { + RMS_NORM_LAUNCH(256, 1, false); } else { - const dim3 block_dims(1024, 1, 1); - rms_norm_f32<1024, false><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + RMS_NORM_LAUNCH(1024, 1, false); } } +#undef RMS_NORM_LAUNCH + +template +static void rms_norm_f32_cuda( + const ggml_tensor * x_t, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream, + const float scale_out = 1.0f) { + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); + rms_norm_f32_dispatch( + blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, + nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), + nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), + stream, scale_out); +} -static void rms_norm_mul_f32_cuda(const float * x, - const float * mul, - const float * add, - float * dst, +static void rms_norm_mul_f32_cuda(const ggml_tensor * x_t, + const ggml_tensor * mul_t, + const ggml_tensor * add_t, + float * dst, const int ncols, const int nrows, const int nchannels, @@ -334,52 +489,34 @@ static void rms_norm_mul_f32_cuda(const float * x, const uint32_t add_nsamples, const float eps, cudaStream_t stream) { - const dim3 blocks_num(nrows, nchannels, nsamples); - if (mul == nullptr) { - rms_norm_f32_cuda(x, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream); + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); + if (mul_t == nullptr) { + rms_norm_f32_cuda(x_t, dst, ncols, nrows, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, stream); return; } - if (add == nullptr) { - const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols); - const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows); - const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); - const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); - if (ncols < 1024) { - const dim3 block_dims(256, 1, 1); - rms_norm_f32<256, true><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>( - x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, - mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed); - } else { - const dim3 block_dims(1024, 1, 1); - rms_norm_f32<1024, true><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>( - x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, - mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed); - } + const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols); + const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows); + const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); + const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); + const uint3 zero_packed = make_uint3(0, 0, 0); + + if (add_t == nullptr) { + rms_norm_f32_dispatch( + blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, + mul_t, mul_stride_row, mul_stride_channel, mul_stride_sample, + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + nullptr, 0, 0, 0, zero_packed, zero_packed, zero_packed, zero_packed, stream); } else { - const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols); - const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows); - const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); - const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); - const uint3 add_ncols_packed = init_fastdiv_values(add_ncols); const uint3 add_nrows_packed = init_fastdiv_values(add_nrows); const uint3 add_nchannels_packed = init_fastdiv_values(add_nchannels); const uint3 add_nsamples_packed = init_fastdiv_values(add_nsamples); - if (ncols < 1024) { - const dim3 block_dims(256, 1, 1); - rms_norm_f32<256, true, true><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>( - x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, - mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add, - add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed, - add_nchannels_packed, add_nsamples_packed); - } else { - const dim3 block_dims(1024, 1, 1); - rms_norm_f32<1024, true, true><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>( - x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, - mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, add, - add_stride_row, add_stride_channel, add_stride_sample, add_ncols_packed, add_nrows_packed, - add_nchannels_packed, add_nsamples_packed); - } + rms_norm_f32_dispatch( + blocks_num, x_t, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, + mul_t, mul_stride_row, mul_stride_channel, mul_stride_sample, + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + add_t, add_stride_row, add_stride_channel, add_stride_sample, + add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, stream); } } @@ -396,13 +533,15 @@ static void rms_norm_back_f32_cuda(const float * grad, const float * xf, float * static void l2_norm_f32_cuda( const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { - const dim3 blocks_num(nrows, nchannels, nsamples); + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); if (ncols < 1024) { const dim3 block_dims(WARP_SIZE, 1, 1); - l2_norm_f32<<>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, 0, stream}; + ggml_cuda_kernel_launch(l2_norm_f32, launch_params, x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps); } else { const dim3 block_dims(1024, 1, 1); - l2_norm_f32<1024><< WARP_SIZE ? 32 * sizeof(float): 0, stream>>>(x, dst, ncols, stride_row, stride_channel, stride_sample, eps); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; + ggml_cuda_kernel_launch(l2_norm_f32<1024>, launch_params, x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps); } } @@ -451,7 +590,6 @@ void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; - const float * src0_d = (const float *) src0->data; float * dst_d = (float *) dst->data; cudaStream_t stream = ctx.stream(); @@ -470,7 +608,33 @@ void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int64_t s02 = nb02 / ts0; const int64_t s03 = nb03 / ts0; - rms_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream); + rms_norm_f32_cuda(src0, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream); +} + +void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * scale_tensor) { + const ggml_tensor * src0 = dst->src[0]; + float * dst_d = (float *) scale_tensor->data; + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(scale_tensor->type == GGML_TYPE_F32); + + GGML_TENSOR_UNARY_OP_LOCALS; + + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); + GGML_ASSERT(eps >= 0.0f); + + float scale; + memcpy(&scale, (const float *) scale_tensor->op_params + 0, sizeof(float)); + + const size_t ts0 = ggml_type_size(src0->type); + GGML_ASSERT(nb00 == ts0); + const int64_t s01 = nb01 / ts0; + const int64_t s02 = nb02 / ts0; + const int64_t s03 = nb03 / ts0; + + rms_norm_f32_cuda(src0, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, scale); } void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor) { @@ -479,15 +643,11 @@ void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * memcpy(&eps, dst->op_params, sizeof(float)); - const float * src0_d = (const float *) rms_norm_src->data; - const float * mul_d = nullptr; const ggml_tensor * mul_src = nullptr; if (mul_tensor->src[0] == dst) { - mul_d = (float *) mul_tensor->src[1]->data; mul_src = mul_tensor->src[1]; } else if(mul_tensor->src[1] == dst) { - mul_d = (float *) mul_tensor->src[0]->data; mul_src = mul_tensor->src[0]; } else { GGML_ASSERT(false); @@ -523,7 +683,7 @@ void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * const int mul_nchannels = mul_src->ne[2]; const int mul_nsamples = mul_src->ne[3]; - rms_norm_mul_f32_cuda(src0_d, mul_d, nullptr, dst_d, + rms_norm_mul_f32_cuda(rms_norm_src, mul_src, nullptr, dst_d, ne00, ne01, ne02, ne03, /*s00*/ s01, s02, s03, /*mul_s00*/ mul_s01, mul_s02, mul_s03, @@ -542,28 +702,21 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, memcpy(&eps, dst->op_params, sizeof(float)); - const float * src0_d = (const float *) rms_norm_src->data; - const float * mul_d = nullptr; const ggml_tensor * mul_src = nullptr; if (mul_tensor->src[0] == dst) { - mul_d = (float *) mul_tensor->src[1]->data; mul_src = mul_tensor->src[1]; } else if (mul_tensor->src[1] == dst) { - mul_d = (float *) mul_tensor->src[0]->data; mul_src = mul_tensor->src[0]; } else { GGML_ASSERT(false); } - const float * add_d = nullptr; const ggml_tensor * add_src = nullptr; if (add_tensor->src[0] == mul_tensor) { - add_d = (float *) add_tensor->src[1]->data; add_src = add_tensor->src[1]; } else if (add_tensor->src[1] == mul_tensor) { - add_d = (float *) add_tensor->src[0]->data; add_src = add_tensor->src[0]; } else { GGML_ASSERT(false); @@ -611,7 +764,7 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, const int add_nchannels = add_src->ne[2]; const int add_nsamples = add_src->ne[3]; - rms_norm_mul_f32_cuda(src0_d, mul_d,add_d,dst_d, + rms_norm_mul_f32_cuda(rms_norm_src, mul_src,add_src,dst_d, ne00,ne01, ne02, ne03, /*s00*/ s01, s02, s03, /*mul_s00*/ mul_s01, mul_s02, mul_s03, diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh index a74f637..95618df 100644 --- a/ggml/src/ggml-cuda/norm.cuh +++ b/ggml/src/ggml-cuda/norm.cuh @@ -8,6 +8,8 @@ void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor); +void ggml_cuda_op_rms_norm_scale_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * scale_tensor); + void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor, diff --git a/ggml/src/ggml-cuda/out-prod.cu b/ggml/src/ggml-cuda/out-prod.cu index c9b2b69..c46e045 100644 --- a/ggml/src/ggml-cuda/out-prod.cu +++ b/ggml/src/ggml-cuda/out-prod.cu @@ -2,6 +2,28 @@ #include +static __global__ void k_compute_out_prod_ptrs( + const float * src0_d, const float * src1_d, float * dst_d, + const float ** ptrs_a, const float ** ptrs_b, float ** ptrs_c, + const int64_t ne2, const int64_t ne3, + const int64_t dps2, const int64_t dps3, + const size_t s02, const size_t s03, + const size_t s12, const size_t s13, + const size_t s2, const size_t s3) { + const int64_t i2 = blockIdx.x*blockDim.x + threadIdx.x; + const int64_t i3 = blockIdx.y*blockDim.y + threadIdx.y; + + if (i2 >= ne2 || i3 >= ne3) { + return; + } + + const int64_t idx = i3*ne2 + i2; + + ptrs_a[idx] = src0_d + (i3/dps3)*s03 + (i2/dps2)*s02; + ptrs_b[idx] = src1_d + i3 *s13 + i2 *s12; + ptrs_c[idx] = dst_d + i3 *s3 + i2 *s2; +} + void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; @@ -32,8 +54,6 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const float alpha = 1.0f; const float beta = 0.0f; - CUBLAS_CHECK(cublasSetStream(handle, stream)); - const int64_t lda = nb01 / sizeof(float); const int64_t ldc = nb1 / sizeof(float); @@ -54,15 +74,52 @@ void ggml_cuda_out_prod(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int64_t dps2 = ne2 / ne02; const int64_t dps3 = ne3 / ne03; - // TODO batched matrix multiplication - for (int64_t i3 = 0; i3 < ne3; ++i3) { - for (int64_t i2 = 0; i2 < ne2; ++i2) { + if (dps2 == 1 && ne2 > 1) { + // src0 has uniform stride s02 along dim 2; batch the inner loop with a strided GEMM + GGML_ASSERT(ne2 <= std::numeric_limits::max()); + const int batch_count = (int) ne2; + for (int64_t i3 = 0; i3 < ne3; ++i3) { CUBLAS_CHECK( - cublasSgemm(handle, CUBLAS_OP_N, src1_cublas_op, + cublasSgemmStridedBatched(handle, CUBLAS_OP_N, src1_cublas_op, ne0, ne1, ne01, - &alpha, src0_d + (i3/dps3)*s03 + (i2/dps2)*s02, lda, - src1_d + i3 *s13 + i2 *s12, ldb, - &beta, dst_d + i3 *s3 + i2 *s2, ldc)); + &alpha, src0_d + (i3/dps3)*s03, lda, s02, + src1_d + i3 *s13, ldb, s12, + &beta, dst_d + i3 *s3, ldc, s2, + batch_count)); } + } else if (ne2 > 1 || ne3 > 1) { + // dps2 > 1 (src0 broadcast along dim 2 with non-uniform stride) or multiple GEMMs + // along dim 3: compute per-GEMM pointers on the device and use a single batched GEMM. + GGML_ASSERT(ne3 > 0); + GGML_ASSERT(ne2 <= (int64_t) std::numeric_limits::max() / ne3); + const int batch_count = (int) (ne2 * ne3); + + ggml_cuda_pool_alloc ptrs_a(ctx.pool(), batch_count); + ggml_cuda_pool_alloc ptrs_b(ctx.pool(), batch_count); + ggml_cuda_pool_alloc< float *> ptrs_c(ctx.pool(), batch_count); + + const dim3 block_dims(16, 16); + const dim3 grid_dims((ne2 + block_dims.x - 1)/block_dims.x, (ne3 + block_dims.y - 1)/block_dims.y); + k_compute_out_prod_ptrs<<>>( + src0_d, src1_d, dst_d, + ptrs_a.get(), ptrs_b.get(), ptrs_c.get(), + ne2, ne3, dps2, dps3, s02, s03, s12, s13, s2, s3); + CUDA_CHECK(cudaGetLastError()); + + CUBLAS_CHECK( + cublasSgemmBatched(handle, CUBLAS_OP_N, src1_cublas_op, + ne0, ne1, ne01, + &alpha, ptrs_a.get(), lda, + ptrs_b.get(), ldb, + &beta, ptrs_c.get(), ldc, + batch_count)); + } else { + // ne2 == 1 && ne3 == 1: single GEMM + CUBLAS_CHECK( + cublasSgemm(handle, CUBLAS_OP_N, src1_cublas_op, + ne0, ne1, ne01, + &alpha, src0_d, lda, + src1_d, ldb, + &beta, dst_d, ldc)); } } diff --git a/ggml/src/ggml-cuda/pad.cu b/ggml/src/ggml-cuda/pad.cu index 31cd00f..e39a6bf 100644 --- a/ggml/src/ggml-cuda/pad.cu +++ b/ggml/src/ggml-cuda/pad.cu @@ -15,49 +15,52 @@ static __global__ void pad_f32(const float * src, size_t s00, size_t s01, size_t // blockIdx.z: i3*ne2+i2 // blockIdx.y: i1 // blockIDx.x: i0 / CUDA_PAD_BLOCK_SIZE - // gridDim.y: ne1 + // gridDim.y and gridDim.z are capped at 65535, blocks stride over larger ne1 and ne2*ne3 int i0 = threadIdx.x + blockIdx.x * blockDim.x; - int i1 = blockIdx.y; - int i2 = blockIdx.z % ne2; - int i3 = blockIdx.z / ne2; - - if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) { + if (i0 >= ne0) { return; } - const int64_t dst_idx = i3 * (ne0 * ne1 * ne2) + i2 * (ne0 * ne1) + i1 * ne0 + i0; - - if (!circular) { - if ((i0 >= lp0 && i0 < ne0 - rp0) && (i1 >= lp1 && i1 < ne1 - rp1) && (i2 >= lp2 && i2 < ne2 - rp2) && - (i3 >= lp3 && i3 < ne3 - rp3)) { - const int64_t i00 = i0 - lp0; - const int64_t i01 = i1 - lp1; - const int64_t i02 = i2 - lp2; - const int64_t i03 = i3 - lp3; - - const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00; - - dst[dst_idx] = src[src_idx]; - } else { - dst[dst_idx] = 0.0f; + for (int i1 = blockIdx.y; i1 < ne1; i1 += gridDim.y) { + for (int i23 = blockIdx.z; i23 < ne2 * ne3; i23 += gridDim.z) { + int i2 = i23 % ne2; + int i3 = i23 / ne2; + + const int64_t dst_idx = i3 * (ne0 * ne1 * ne2) + i2 * (ne0 * ne1) + i1 * ne0 + i0; + + if (!circular) { + if ((i0 >= lp0 && i0 < ne0 - rp0) && (i1 >= lp1 && i1 < ne1 - rp1) && (i2 >= lp2 && i2 < ne2 - rp2) && + (i3 >= lp3 && i3 < ne3 - rp3)) { + const int64_t i00 = i0 - lp0; + const int64_t i01 = i1 - lp1; + const int64_t i02 = i2 - lp2; + const int64_t i03 = i3 - lp3; + + const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00; + + dst[dst_idx] = src[src_idx]; + } else { + dst[dst_idx] = 0.0f; + } + } + // circular means on a torus, so x and y wrap around + else { + const int64_t ne00 = ne0 - lp0 - rp0; + const int64_t ne01 = ne1 - lp1 - rp1; + const int64_t ne02 = ne2 - lp2 - rp2; + const int64_t ne03 = ne3 - lp3 - rp3; + + const int64_t i00 = wrap_around(i0 - lp0, ne00); + const int64_t i01 = wrap_around(i1 - lp1, ne01); + const int64_t i02 = wrap_around(i2 - lp2, ne02); + const int64_t i03 = wrap_around(i3 - lp3, ne03); + + const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00; + + dst[dst_idx] = src[src_idx]; + } } } - // circular means on a torus, so x and y wrap around - else { - const int64_t ne00 = ne0 - lp0 - rp0; - const int64_t ne01 = ne1 - lp1 - rp1; - const int64_t ne02 = ne2 - lp2 - rp2; - const int64_t ne03 = ne3 - lp3 - rp3; - - const int64_t i00 = wrap_around(i0 - lp0, ne00); - const int64_t i01 = wrap_around(i1 - lp1, ne01); - const int64_t i02 = wrap_around(i2 - lp2, ne02); - const int64_t i03 = wrap_around(i3 - lp3, ne03); - - const int64_t src_idx = i03 * s03 + i02 * s02 + i01 * s01 + i00 * s00; - - dst[dst_idx] = src[src_idx]; - } } @@ -67,7 +70,7 @@ static void pad_f32_cuda(const float * src, size_t s00, size_t s01, size_t s02, const int ne0, const int ne1, const int ne2, const int ne3, const bool circular, cudaStream_t stream) { int num_blocks = (ne0 + CUDA_PAD_BLOCK_SIZE - 1) / CUDA_PAD_BLOCK_SIZE; - dim3 gridDim(num_blocks, ne1, ne2 * ne3); + dim3 gridDim(num_blocks, std::min(ne1, 65535), std::min(ne2 * ne3, 65535)); pad_f32<<>>(src, s00, s01, s02, s03, dst, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3, ne0, ne1, ne2, ne3, circular); diff --git a/ggml/src/ggml-cuda/pool1d.cu b/ggml/src/ggml-cuda/pool1d.cu new file mode 100644 index 0000000..ac6fb0c --- /dev/null +++ b/ggml/src/ggml-cuda/pool1d.cu @@ -0,0 +1,85 @@ +#include "pool1d.cuh" + +static __global__ void pool1d_nchw_kernel( + const int iw, const int ow, + const int kw, const int sw, const int pw, + const int parallel_elements, + const float * src, float * dst, const enum ggml_op_pool op) { + const int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx >= parallel_elements) { + return; + } + + const int nc = idx / ow; + const int cur_ow = idx % ow; + + const float * i_ptr = src + nc * iw; + float * o_ptr = dst + nc * ow; + + const int start = cur_ow * sw - pw; + const int b = max(0, start); + const int e = min(iw, start + kw); + + float res; + switch (op) { + case GGML_OP_POOL_AVG: res = 0.0f; break; + case GGML_OP_POOL_MAX: res = -FLT_MAX; break; + default: return; + } + + int count = 0; + for (int i = b; i < e; i++) { +#if __CUDA_ARCH__ >= 350 + float cur = __ldg(i_ptr + i); +#else + float cur = i_ptr[i]; +#endif + switch (op) { + case GGML_OP_POOL_AVG: res += cur; break; + case GGML_OP_POOL_MAX: res = max(res, cur); break; + default: break; + } + count++; + } + + if (op == GGML_OP_POOL_AVG) { + res = (count > 0) ? (res / count) : 0.0f; + } + + o_ptr[cur_ow] = res; +} + +static void pool1d_nchw_kernel_f32_f32_cuda( + const int iw, const int ow, + const int kw, const int sw, const int pw, + const int parallel_elements, + const float * src, float * dst, const enum ggml_op_pool op, + cudaStream_t stream) { + const int num_blocks = (parallel_elements + CUDA_POOL1D_BLOCK_SIZE - 1) / CUDA_POOL1D_BLOCK_SIZE; + dim3 block_nums(num_blocks); + pool1d_nchw_kernel<<>>(iw, ow, kw, sw, pw, parallel_elements, src, dst, op); +} + +void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const float * src0_d = (const float *)src0->data; + float * dst_d = (float *)dst->data; + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + + const int32_t * opts = (const int32_t *)dst->op_params; + enum ggml_op_pool op = static_cast(opts[0]); + const int k0 = opts[1]; + const int s0 = opts[2]; + const int p0 = opts[3]; + + const int64_t IW = src0->ne[0]; + const int64_t OW = dst->ne[0]; + const int64_t nr = ggml_nrows(src0); + + const int parallel_elements = (int)(nr * OW); + + pool1d_nchw_kernel_f32_f32_cuda(IW, OW, k0, s0, p0, parallel_elements, src0_d, dst_d, op, stream); +} diff --git a/ggml/src/ggml-cuda/pool1d.cuh b/ggml/src/ggml-cuda/pool1d.cuh new file mode 100644 index 0000000..c79461d --- /dev/null +++ b/ggml/src/ggml-cuda/pool1d.cuh @@ -0,0 +1,5 @@ +#include "common.cuh" + +#define CUDA_POOL1D_BLOCK_SIZE 256 + +void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/pool2d.cu b/ggml/src/ggml-cuda/pool2d.cu index c6d51e4..83a4975 100644 --- a/ggml/src/ggml-cuda/pool2d.cu +++ b/ggml/src/ggml-cuda/pool2d.cu @@ -50,6 +50,65 @@ static __global__ void pool2d_nchw_kernel( o_ptr[cur_oh * ow + cur_ow] = res; } +template +static __global__ void pool2d_nchw_kernel_warp( + const int ih, const int iw, const int oh, const int ow, + const int kh, const int kw, const int sh, const int sw, + const int ph, const int pw, const int parallel_elements, + const Ti * __restrict__ src, To * __restrict__ dst, const enum ggml_op_pool op) { + const int warp_id = (threadIdx.x + blockIdx.x * blockDim.x) / WARP_SIZE; + const int lane = threadIdx.x % WARP_SIZE; + if (warp_id >= parallel_elements) { + return; + } + + const int I_HW = ih * iw; + const int O_HW = oh * ow; + const int nc = warp_id / O_HW; + const int cur_oh = warp_id % O_HW / ow; + const int cur_ow = warp_id % O_HW % ow; + const Ti* i_ptr = src + nc * I_HW; + + const int start_h = cur_oh * sh - ph; + const int bh = max(0, start_h); + const int eh = min(ih, start_h + kh); + const int start_w = cur_ow * sw - pw; + const int bw = max(0, start_w); + const int ew = min(iw, start_w + kw); + + const int win_w = ew - bw; + const int win_elems = (eh - bh) * win_w; + const To scale = 1. / (kh * kw); + + To res; + switch (op) { + case GGML_OP_POOL_AVG: res = 0; break; + case GGML_OP_POOL_MAX: res = -FLT_MAX; break; + default: res = 0; assert(false); + } + + for (int t = lane; t < win_elems; t += WARP_SIZE) { + const int i = bh + t / win_w; + const int j = bw + t % win_w; + const Ti cur = i_ptr[i * iw + j]; + switch (op) { + case GGML_OP_POOL_AVG: res += cur * scale; break; + case GGML_OP_POOL_MAX: res = max(res, (To)cur); break; + default: break; + } + } + +#pragma unroll + for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) { + const To other = __shfl_xor_sync(0xFFFFFFFF, res, offset, WARP_SIZE); + res = (op == GGML_OP_POOL_MAX) ? max(res, other) : res + other; + } + + if (lane == 0) { + dst[nc * O_HW + cur_oh * ow + cur_ow] = res; + } +} + static void pool2d_nchw_kernel_f32_f32_cuda( const int ih, const int iw, const int oh, const int ow, const int kh, const int kw, const int sh, const int sw, @@ -57,6 +116,13 @@ static void pool2d_nchw_kernel_f32_f32_cuda( const float * src, float * dst, const enum ggml_op_pool op, cudaStream_t stream) { + if (kh * kw >= CUDA_POOL2D_WARP_KERNEL_MIN_WINDOW) { + const int warps_per_block = CUDA_POOL2D_BLOCK_SIZE / WARP_SIZE; + const int num_blocks = (parallel_elements + warps_per_block - 1) / warps_per_block; + pool2d_nchw_kernel_warp<<>>(ih, iw, oh, ow, kh, kw, sh, sw, ph, pw, parallel_elements, src, dst, op); + return; + } + const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE; dim3 block_nums(num_blocks); pool2d_nchw_kernel<<>>(ih, iw, oh, ow, kh, kw, sh, sw, ph, pw, parallel_elements, src, dst, op); diff --git a/ggml/src/ggml-cuda/pool2d.cuh b/ggml/src/ggml-cuda/pool2d.cuh index 7841292..e2c7fb4 100644 --- a/ggml/src/ggml-cuda/pool2d.cuh +++ b/ggml/src/ggml-cuda/pool2d.cuh @@ -1,5 +1,6 @@ #include "common.cuh" #define CUDA_POOL2D_BLOCK_SIZE 256 +#define CUDA_POOL2D_WARP_KERNEL_MIN_WINDOW 32 void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu index 4300ffc..ea089a8 100644 --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -1,11 +1,63 @@ #include "quantize.cuh" #include +#if defined(BLACKWELL_MMA_AVAILABLE) +// this maps to 256-bit loads in PTX on supported devices, +// and otherwise falls back to 2 128-bit loads +struct __builtin_align__(32) float8 { + float x; float y; float z; float w; + float p; float q; float r; float s; +}; + +#if CUDART_VERSION >= 12080 +static __device__ __forceinline__ float nvfp4_native_scale_error( + const float vals[QK_NVFP4_SUB], const float inv_col_scale, const float inv_scale, const float scale) { + const float scale_dequant = 2.0f * scale; + float err = 0.0f; + +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; k += 4) { + const float v0 = vals[k + 0] * inv_col_scale; + const float v1 = vals[k + 1] * inv_col_scale; + const float v2 = vals[k + 2] * inv_col_scale; + const float v3 = vals[k + 3] * inv_col_scale; + + const __nv_fp4x4_e2m1 q(make_float4(v0 * inv_scale, v1 * inv_scale, v2 * inv_scale, v3 * inv_scale)); + const __nv_fp4x4_storage_t q_storage = q.__x; + const __nv_fp4x2_storage_t q_lo = static_cast<__nv_fp4x2_storage_t>(q_storage); + const __nv_fp4x2_storage_t q_hi = static_cast<__nv_fp4x2_storage_t>(q_storage >> 8U); + + const __half2_raw hraw2_lo = __nv_cvt_fp4x2_to_halfraw2(q_lo, __NV_E2M1); + const __half2_raw hraw2_hi = __nv_cvt_fp4x2_to_halfraw2(q_hi, __NV_E2M1); + const __half2 h2_lo = static_cast<__half2>(hraw2_lo); + const __half2 h2_hi = static_cast<__half2>(hraw2_hi); + const float2 dq_lo = __half22float2(h2_lo); + const float2 dq_hi = __half22float2(h2_hi); + + const float err0 = fabsf(v0) - fabsf(dq_lo.x) * scale_dequant; + const float err1 = fabsf(v1) - fabsf(dq_lo.y) * scale_dequant; + const float err2 = fabsf(v2) - fabsf(dq_hi.x) * scale_dequant; + const float err3 = fabsf(v3) - fabsf(dq_hi.y) * scale_dequant; + + err = fmaf(err0, err0, err); + err = fmaf(err1, err1, err); + err = fmaf(err2, err2, err); + err = fmaf(err3, err3, err); + } + + return err; +} +#endif // CUDART_VERSION >= 12080 +#endif // defined(BLACKWELL_MMA_AVAILABLE) + __launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1) static __global__ void quantize_q8_1( - const float * __restrict__ x, void * __restrict__ vy, + const float * x_ptr, void * vy_ptr, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne0, const uint32_t ne1, const uint3 ne2) { + ggml_cuda_pdl_lc(); + const float * GGML_CUDA_RESTRICT x = x_ptr; + void * GGML_CUDA_RESTRICT vy = vy_ptr; const int64_t i0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x; if (i0 >= ne0) { @@ -28,6 +80,7 @@ static __global__ void quantize_q8_1( const int64_t ib = i_cont / QK8_1; // block index const int64_t iqs = i_cont % QK8_1; // quant index + ggml_cuda_pdl_sync(); const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f; float amax = fabsf(xi); float sum = xi; @@ -70,8 +123,217 @@ __device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) { return static_cast(biased); } +// scatter: grid over tokens, quantize once, write to all the token's compact rows +template +static __global__ void quantize_mmq_nvfp4( + const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, float * __restrict__ scale, + const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int n_expert_used) { +#if defined(BLACKWELL_MMA_AVAILABLE) + + int64_t base_idx; + if constexpr (scatter) { + base_idx = (int64_t) blockIdx.x * s02; // one physical row per token + } else { + const int64_t i2 = blockIdx.y % ne2; + const int64_t i3 = blockIdx.y / ne2; + const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; + base_idx = i3 * s03 + i2 * s02 + i01 * s01; + } + const float * __restrict__ x_row = x + base_idx; + + float amax = 0.0f; + if constexpr (use_aligned_float8) { + for (int64_t i0 = 8 * threadIdx.x; i0 < ne00; i0 += 8 * blockDim.x) { + const float * x_base = x_row + i0; + const float8 v = reinterpret_cast(x_base)[0]; + amax = fmaxf(amax, fabsf(v.x)); + amax = fmaxf(amax, fabsf(v.y)); + amax = fmaxf(amax, fabsf(v.z)); + amax = fmaxf(amax, fabsf(v.w)); + amax = fmaxf(amax, fabsf(v.p)); + amax = fmaxf(amax, fabsf(v.q)); + amax = fmaxf(amax, fabsf(v.r)); + amax = fmaxf(amax, fabsf(v.s)); + } + } else { + for (int64_t i0 = threadIdx.x; i0 < ne00; i0 += blockDim.x) { + amax = fmaxf(amax, fabsf(x_row[i0])); + } + } + + amax = warp_reduce_max(amax); + + __shared__ float warp_amax[CUDA_QUANTIZE_BLOCK_SIZE_MMQ / WARP_SIZE]; + const int lane = threadIdx.x % WARP_SIZE; + const int warp = threadIdx.x / WARP_SIZE; + + if (lane == 0) { + warp_amax[warp] = amax; + } + __syncthreads(); + + if (warp == 0) { + amax = threadIdx.x < int(CUDA_QUANTIZE_BLOCK_SIZE_MMQ / WARP_SIZE) ? warp_amax[lane] : 0.0f; + amax = warp_reduce_max(amax); + if (lane == 0) { + warp_amax[0] = amax / (6.0f * 448.0f); + if constexpr (scatter) { +#pragma unroll + for (int slot = 0; slot < n_expert_used; ++slot) { + const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + scale[i] = warp_amax[0]; + } + } else { + scale[blockIdx.y * ne1 + blockIdx.x] = warp_amax[0]; + } + } + } + __syncthreads(); + + block_fp4_mmq * y = (block_fp4_mmq *) vy; + const int64_t n_subblocks = (ne0 + QK_NVFP4_SUB - 1) / QK_NVFP4_SUB; + + for (int64_t isb = threadIdx.x; isb < n_subblocks; isb += blockDim.x) { + const int64_t i0_base = isb * QK_NVFP4_SUB; + const int64_t k_block = i0_base / QK_FP4_MMQ; + const int sub = (i0_base % QK_FP4_MMQ) / QK_NVFP4_SUB; + + const float row_scale = warp_amax[0]; + const float inv_col_scale = row_scale > 0.0f ? 1.0f / row_scale : 0.0f; + + float vals[QK_NVFP4_SUB]; + if constexpr (use_aligned_float8) { + const float * x_base = x_row + i0_base; + const float8 v0 = i0_base + 7 < ne00 ? reinterpret_cast(x_base)[0] : float8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + const float8 v1 = i0_base + 15 < ne00 ? reinterpret_cast(x_base + 8)[0] : float8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + vals[0] = v0.x; vals[1] = v0.y; vals[2] = v0.z; vals[3] = v0.w; + vals[4] = v0.p; vals[5] = v0.q; vals[6] = v0.r; vals[7] = v0.s; + vals[8] = v1.x; vals[9] = v1.y; vals[10] = v1.z; vals[11] = v1.w; + vals[12] = v1.p; vals[13] = v1.q; vals[14] = v1.r; vals[15] = v1.s; + } else { +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; ++k) { + const int64_t i00 = i0_base + k; + vals[k] = i00 < ne00 ? x_row[i00] : 0.0f; + } + } + + uint32_t q0 = 0; + uint32_t q1 = 0; + + float amax_sub = 0.0f; +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; ++k) { + amax_sub = fmaxf(amax_sub, fabsf(vals[k] * inv_col_scale)); + } + + static constexpr int test_offsets[5] = { 0, -1, 1, -2, 2 }; + const int first_fp8_code = (int) ggml_cuda_fp32_to_ue4m3(amax_sub / 6.0f); + + uint8_t fp8_code = (uint8_t) first_fp8_code; + float subblock_scale = ggml_cuda_ue4m3_to_fp32(fp8_code); + float inv_scale_err = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f; +#if CUDART_VERSION >= 12080 + float best_err = nvfp4_native_scale_error(vals, inv_col_scale, inv_scale_err, subblock_scale); +#else + float best_err = 0.0f; +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; ++k) { + const float v = vals[k] * inv_col_scale; + const uint8_t q = ggml_cuda_float_to_fp4_e2m1(v, inv_scale_err); + const float err_diff = fabsf(v) - fabsf(kvalues_fp4[q & 0x7]) * subblock_scale; + best_err = fmaf(err_diff, err_diff, best_err); + } +#endif // CUDART_VERSION >= 12080 + +#pragma unroll + for (int i = 1; i < 5; ++i) { + const int test_code = first_fp8_code + test_offsets[i]; + if (test_code < 0 || test_code > 0x7e) { + continue; + } + + const float test_scale = ggml_cuda_ue4m3_to_fp32((uint8_t) test_code); + const float test_inv_scale = test_scale > 0.0f ? 0.5f / test_scale : 0.0f; +#if CUDART_VERSION >= 12080 + const float cur_err = nvfp4_native_scale_error(vals, inv_col_scale, test_inv_scale, test_scale); +#else + float cur_err = 0.0f; +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB; ++k) { + const float v = vals[k] * inv_col_scale; + const uint8_t q = ggml_cuda_float_to_fp4_e2m1(v, test_inv_scale); + const float err_diff = fabsf(v) - fabsf(kvalues_fp4[q & 0x7]) * test_scale; + cur_err = fmaf(err_diff, err_diff, cur_err); + } +#endif // CUDART_VERSION >= 12080 + + if (cur_err < best_err) { + best_err = cur_err; + fp8_code = (uint8_t) test_code; + subblock_scale = test_scale; + } + } +#if CUDART_VERSION >= 12080 + const float inv_scale = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f; + const float s = inv_col_scale * inv_scale; + + __nv_fp4x4_e2m1 q0_lo(make_float4(vals[0] * s, vals[8] * s, vals[1] * s, vals[9] * s)); + __nv_fp4x4_e2m1 q0_hi(make_float4(vals[2] * s, vals[10] * s, vals[3] * s, vals[11] * s)); + __nv_fp4x4_e2m1 q1_lo(make_float4(vals[4] * s, vals[12] * s, vals[5] * s, vals[13] * s)); + __nv_fp4x4_e2m1 q1_hi(make_float4(vals[6] * s, vals[14] * s, vals[7] * s, vals[15] * s)); + + const char2 q0_lo_c = *reinterpret_cast(&q0_lo); + const char2 q0_hi_c = *reinterpret_cast(&q0_hi); + const char2 q1_lo_c = *reinterpret_cast(&q1_lo); + const char2 q1_hi_c = *reinterpret_cast(&q1_hi); + + q0 = uint32_t(uint8_t(q0_lo_c.x)) | (uint32_t(uint8_t(q0_lo_c.y)) << 8) | + (uint32_t(uint8_t(q0_hi_c.x)) << 16) | (uint32_t(uint8_t(q0_hi_c.y)) << 24); + q1 = uint32_t(uint8_t(q1_lo_c.x)) | (uint32_t(uint8_t(q1_lo_c.y)) << 8) | + (uint32_t(uint8_t(q1_hi_c.x)) << 16) | (uint32_t(uint8_t(q1_hi_c.y)) << 24); +#else + const float inv_scale = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f; +#pragma unroll + for (int k = 0; k < QK_NVFP4_SUB / 4; ++k) { + q0 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 0] * inv_col_scale, inv_scale)) << (8 * k); + q0 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 8] * inv_col_scale, inv_scale)) << (8 * k + 4); + q1 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 4] * inv_col_scale, inv_scale)) << (8 * k); + q1 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 12] * inv_col_scale, inv_scale)) << (8 * k + 4); + } +#endif // CUDART_VERSION >= 12080 + + if constexpr (scatter) { +#pragma unroll + for (int slot = 0; slot < n_expert_used; ++slot) { + const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + block_fp4_mmq * yb = y + (k_block * ne1 + i); + uint32_t * yqs = reinterpret_cast(yb->qs); + yqs[2 * sub + 0] = q0; + yqs[2 * sub + 1] = q1; + reinterpret_cast(yb->d4)[sub] = fp8_code; + } + } else { + const int64_t blocks_per_col = (ne0 + QK_FP4_MMQ - 1) / QK_FP4_MMQ; + block_fp4_mmq * yb = y + (blockIdx.y * (blocks_per_col * ne1) + k_block * ne1 + blockIdx.x); + uint32_t * yqs = reinterpret_cast(yb->qs); + yqs[2 * sub + 0] = q0; + yqs[2 * sub + 1] = q1; + reinterpret_cast(yb->d4)[sub] = fp8_code; + } + } +#else + GGML_UNUSED_VARS(x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, n_expert_used); + NO_DEVICE_CODE; // This is for Blackwell NVFP4 activations only. +#endif // defined(BLACKWELL_MMA_AVAILABLE) + +} + // quantize values in the format mxfp4 is stored which is interleaved nibbles // i.e. a block a0-a31 is represented as a0a16,a1a17 ...a15a31 +// scatter: grid over tokens, quantize once, write to all the token's compact rows +template static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, @@ -81,7 +343,8 @@ static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x, const int64_t s03, const int64_t ne0, const int ne1, - const int ne2) { + const int ne2, + const int n_expert_used) { constexpr int vals_per_scale = 32; constexpr int vals_per_warp = 2 * vals_per_scale; // Each warp processes 2 blocks of 32 = 64 values @@ -96,29 +359,27 @@ static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x, return; } - const int64_t i1 = blockIdx.x; - const int64_t i2 = blockIdx.z % ne2; - const int64_t i3 = blockIdx.z / ne2; - - const int64_t i01 = ids ? ids[i1] : i1; - const int64_t i02 = i2; - const int64_t i03 = i3; - - block_fp4_mmq * y = (block_fp4_mmq *) vy; - - const int64_t block_fp4_mmq_size = 8 * QK_MXFP4; // 256 values - const int64_t ib0 = blockIdx.z * ((int64_t) ne1 * (ne0 / block_fp4_mmq_size)); - const int64_t ib = ib0 + (warp_start_offset / block_fp4_mmq_size) * ne1 + blockIdx.x; + const int64_t block_fp4_mmq_size = QK_FP4_MMQ; + const int64_t k_block = warp_start_offset / block_fp4_mmq_size; const int64_t quad_idx_in_block = (warp_start_offset % block_fp4_mmq_size) / vals_per_warp; const int group_id = lane_id_32 / 4; const int lane_in_group = lane_id_32 % 4; const int base = group_id * 2; - char2 * yqs2 = (char2 *) y[ib].qs; - const int64_t base_pos = i03 * s03 + i02 * s02 + i01 * s01; + ggml_cuda_pdl_sync(); + int64_t base_pos; + if constexpr (scatter) { + base_pos = (int64_t) blockIdx.x * s02; // one physical row per token + } else { + const int64_t i2 = blockIdx.z % ne2; + const int64_t i3 = blockIdx.z / ne2; + const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; + base_pos = i3 * s03 + i2 * s02 + i01 * s01; + } uint8_t scales[2]; + char2 packed[2]; #pragma unroll for (int b = 0; b < 2; ++b) { @@ -143,11 +404,8 @@ static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x, const float val2 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 1, WARP_SIZE); const float val3 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 17, WARP_SIZE); - if (lane_in_group == 0) { - __nv_fp4x4_e2m1 fp4_packed(make_float4(val0, val1, val2, val3)); - - yqs2[quad_idx_in_block * 16 + b * 8 + group_id] = *(char2 *) &fp4_packed; - } + __nv_fp4x4_e2m1 fp4_packed(make_float4(val0, val1, val2, val3)); + packed[b] = *(char2 *) &fp4_packed; #else // Fallback: manual FP4 conversion using LUT const uint8_t q_val = ggml_cuda_float_to_fp4_e2m1(xi, inv_s); @@ -157,26 +415,49 @@ static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x, const uint8_t q_hi_0 = __shfl_sync(0xFFFFFFFF, q_val, base + 16, WARP_SIZE); const uint8_t q_hi_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 17, WARP_SIZE); - if (lane_in_group == 0) { - char2 q; - q.x = (q_hi_0 << 4) | q_lo_0; - q.y = (q_hi_1 << 4) | q_lo_1; - yqs2[quad_idx_in_block * 16 + b * 8 + group_id] = q; - } + char2 q; + q.x = (q_hi_0 << 4) | q_lo_0; + q.y = (q_hi_1 << 4) | q_lo_1; + packed[b] = q; #endif // CUDART_VERSION >= 12080 } - if (lane_id_32 == 0) { - // Store 2 scales packed into 1 uint32 - y[ib].d4[quad_idx_in_block] = (scales[1] << 8) | scales[0]; + block_fp4_mmq * y = (block_fp4_mmq *) vy; + if constexpr (scatter) { +#pragma unroll + for (int slot = 0; slot < n_expert_used; ++slot) { + const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + block_fp4_mmq * yb = y + (k_block * ne1 + i); + char2 * yqs2 = (char2 *) yb->qs; + if (lane_in_group == 0) { + yqs2[quad_idx_in_block * 16 + 0 * 8 + group_id] = packed[0]; + yqs2[quad_idx_in_block * 16 + 1 * 8 + group_id] = packed[1]; + } + if (lane_id_32 == 0) { + yb->d4[quad_idx_in_block] = (scales[1] << 8) | scales[0]; + } + } + } else { + const int64_t ib0 = blockIdx.z * ((int64_t) ne1 * (ne0 / block_fp4_mmq_size)); + block_fp4_mmq * yb = y + (ib0 + k_block * ne1 + blockIdx.x); + char2 * yqs2 = (char2 *) yb->qs; + if (lane_in_group == 0) { + yqs2[quad_idx_in_block * 16 + 0 * 8 + group_id] = packed[0]; + yqs2[quad_idx_in_block * 16 + 1 * 8 + group_id] = packed[1]; + } + if (lane_id_32 == 0) { + yb->d4[quad_idx_in_block] = (scales[1] << 8) | scales[0]; + } } + GGML_UNUSED(n_expert_used); } -template +// scatter: grid over tokens, quantize once, write to all the token's compact rows +template static __global__ void quantize_mmq_q8_1( const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, - const int64_t ne0, const int ne1, const int ne2) { + const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) { constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32; constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32; @@ -187,25 +468,27 @@ static __global__ void quantize_mmq_q8_1( return; } - const int64_t i1 = blockIdx.x; - const int64_t i2 = blockIdx.z % ne2; - const int64_t i3 = blockIdx.z / ne2; - const int64_t i00 = i0; - const int64_t i01 = ids ? ids[i1] : i1; - const int64_t i02 = i2; - const int64_t i03 = i3; + ggml_cuda_pdl_sync(); - const float4 * x4 = (const float4 *) x; + int64_t base_idx; + if constexpr (scatter) { + base_idx = (int64_t) blockIdx.x * s02; // one physical row per token + } else { + const int64_t i2 = blockIdx.z % ne2; + const int64_t i3 = blockIdx.z / ne2; + const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; + base_idx = i3*s03 + i2*s02 + i01*s01; + } + const float4 * x4 = (const float4 *) x; block_q8_1_mmq * y = (block_q8_1_mmq *) vy; - const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel - const int64_t ib = ib0 + (i0 / (4*QK8_1))*ne1 + blockIdx.x; // block index in channel - const int64_t iqs = i0 % (4*QK8_1); // quant index in block + const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel + const int64_t iqs = i0 % QK8_1_MMQ; // quant index in block // Load 4 floats per thread and calculate max. abs. value between them: - const float4 xi = i0 < ne00 ? x4[(i03*s03 + i02*s02 + i01*s01 + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f); + const float4 xi = i0 < ne00 ? x4[(base_idx + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f); float amax = fabsf(xi.x); amax = fmaxf(amax, fabsf(xi.y)); amax = fmaxf(amax, fabsf(xi.z)); @@ -234,40 +517,41 @@ static __global__ void quantize_mmq_q8_1( q.y = roundf(xi.y*d_inv); q.z = roundf(xi.z*d_inv); q.w = roundf(xi.w*d_inv); + const float d = 1.0f / d_inv; - // Write back 4 int8 values as a single 32 bit value for better memory bandwidth: - char4 * yqs4 = (char4 *) y[ib].qs; - yqs4[iqs/4] = q; - - if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) { - if (iqs % 16 != 0 || iqs >= 96) { - return; + // write the block once (normal) or to each of the token's compact rows (scatter) + const int nwrite = scatter ? n_expert_used : 1; +#pragma unroll + for (int slot = 0; slot < nwrite; ++slot) { + int64_t ib; + if constexpr (scatter) { + const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + ib = k_block*ne1 + i; + } else { + const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel + ib = ib0 + k_block*ne1 + blockIdx.x; } - y[ib].d2s6[2 + iqs/16] = sum; - - if (iqs % 64 != 0) { - return; + // Write back 4 int8 values as a single 32 bit value for better memory bandwidth: + char4 * yqs4 = (char4 *) y[ib].qs; + yqs4[iqs/4] = q; + + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) { + if (iqs % 16 == 0 && iqs < 96) { + y[ib].d2s6[2 + iqs/16] = sum; + if (iqs % 64 == 0) { + y[ib].d2s6[iqs/64] = d; + } + } + } else if (iqs % 32 == 0) { + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) { + y[ib].ds4[iqs/32] = make_half2(d, sum); + } else { + y[ib].d4[iqs/32] = d; + } } - - const float d = 1.0f / d_inv; - - y[ib].d2s6[iqs/64] = d; - - return; - } - - if (iqs % 32 != 0) { - return; - } - - const float d = 1.0f / d_inv; - - if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) { - y[ib].ds4[iqs/32] = make_half2(d, sum); - } else { - y[ib].d4[iqs/32] = d; } + GGML_UNUSED(n_expert_used); } void quantize_row_q8_1_cuda( @@ -282,7 +566,8 @@ void quantize_row_q8_1_cuda( const int64_t block_num_x = (ne0 + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE; const dim3 num_blocks(block_num_x, ne1, ne2*ne3); const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1); - quantize_q8_1<<>>(x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream); + ggml_cuda_kernel_launch(quantize_q8_1, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv); GGML_UNUSED(type_src0); } @@ -291,7 +576,7 @@ void quantize_mmq_q8_1_cuda( const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { GGML_ASSERT(ne00 % 4 == 0); - GGML_ASSERT(ne0 % (4*QK8_1) == 0); + GGML_ASSERT(ne0 % QK8_1_MMQ == 0); // ne1 tends to assume the highest values, therefore use it as the "x" dimension of the CUDA grid: const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); @@ -299,45 +584,111 @@ void quantize_mmq_q8_1_cuda( const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); switch (mmq_get_q8_1_ds_layout(type_src0)) { case MMQ_Q8_1_DS_LAYOUT_D4: - quantize_mmq_q8_1 - <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2); + quantize_mmq_q8_1 + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); break; case MMQ_Q8_1_DS_LAYOUT_DS4: - quantize_mmq_q8_1 - <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2); + quantize_mmq_q8_1 + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); break; case MMQ_Q8_1_DS_LAYOUT_D2S6: - quantize_mmq_q8_1 - <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2); + quantize_mmq_q8_1 + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); break; default: GGML_ABORT("fatal error"); + } +} + +// scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row) +void quantize_scatter_mmq_q8_1_cuda( + const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0, + const int64_t ne00, const int64_t stride_token, const int64_t ne0, + const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) { + GGML_ASSERT(ne00 % 4 == 0); + GGML_ASSERT(ne0 % QK8_1_MMQ == 0); + + const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); + const dim3 num_blocks(n_tokens, block_num_y, 1); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + switch (mmq_get_q8_1_ds_layout(type_src0)) { + case MMQ_Q8_1_DS_LAYOUT_D4: + quantize_mmq_q8_1<<>>( + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); + break; + case MMQ_Q8_1_DS_LAYOUT_DS4: + quantize_mmq_q8_1<<>>( + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); + break; + case MMQ_Q8_1_DS_LAYOUT_D2S6: + quantize_mmq_q8_1<<>>( + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); break; + default: + GGML_ABORT("fatal error"); + } +} + +// scatter=true reuses the quant kernels: grid over tokens, ids = inverse map (token slot -> compact row) +void quantize_scatter_mmq_fp4_cuda( + const float * x, const int32_t * ids_src1_inv, void * vy, float * scale, const ggml_type type_src0, const bool use_aligned_float8, + const int64_t ne00, const int64_t stride_token, const int64_t ne0, + const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) { + GGML_ASSERT(ne0 > 0); + if (type_src0 == GGML_TYPE_NVFP4) { + GGML_ASSERT(scale); + GGML_ASSERT(ne00 % QK_NVFP4 == 0); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + const dim3 num_blocks(n_tokens, 1, 1); + if (use_aligned_float8) { + quantize_mmq_nvfp4<<>>( + x, ids_src1_inv, vy, scale, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/nrows_dst, /*ne2=*/1, n_expert_used); + } else { + quantize_mmq_nvfp4<<>>( + x, ids_src1_inv, vy, scale, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/nrows_dst, /*ne2=*/1, n_expert_used); + } + } else { + GGML_ASSERT(type_src0 == GGML_TYPE_MXFP4); + constexpr int nwarps = 8; + constexpr int vals_per_block = nwarps * 2 * QK_MXFP4; + const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block; + const dim3 block_size(WARP_SIZE, nwarps, 1); + const dim3 num_blocks(n_tokens, block_num_y, 1); + quantize_mmq_mxfp4<<>>( + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); } } -void quantize_mmq_mxfp4_cuda(const float * x, - const int32_t * ids, - void * vy, - [[maybe_unused]] const ggml_type type_src0, - const int64_t ne00, - const int64_t s01, - const int64_t s02, - const int64_t s03, - const int64_t ne0, - const int64_t ne1, - const int64_t ne2, - const int64_t ne3, - cudaStream_t stream) { - GGML_ASSERT(ne0 % (2 * QK_MXFP4) == 0); - - constexpr int nwarps = 8; - constexpr int vals_per_warp = 2 * QK_MXFP4; - constexpr int vals_per_block = nwarps * vals_per_warp; - - const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block; - const dim3 num_blocks(ne1, block_num_y, ne2 * ne3); - const dim3 block_size(WARP_SIZE, nwarps, 1); - - quantize_mmq_mxfp4<<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2); +void quantize_mmq_fp4_cuda( + const float * x, const int32_t * ids, void * vy, float * scale, const ggml_type type_src0, const bool use_aligned_float8, + const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { + GGML_ASSERT(type_src0 == GGML_TYPE_MXFP4 || type_src0 == GGML_TYPE_NVFP4); + GGML_ASSERT(ne0 > 0); + + if (type_src0 == GGML_TYPE_NVFP4) { + GGML_ASSERT(scale); + GGML_ASSERT(ne00 % QK_NVFP4 == 0); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + const dim3 num_blocks(ne1, ne2 * ne3, 1); + if (use_aligned_float8) { + quantize_mmq_nvfp4<<>>( + x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + } else { + quantize_mmq_nvfp4<<>>( + x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + } + } else { + GGML_ASSERT(ne0 % (2 * QK_MXFP4) == 0); + + constexpr int nwarps = 8; + constexpr int vals_per_warp = 2 * QK_MXFP4; + constexpr int vals_per_block = nwarps * vals_per_warp; + + const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block; + const dim3 num_blocks(ne1, block_num_y, ne2 * ne3); + const dim3 block_size(WARP_SIZE, nwarps, 1); + + quantize_mmq_mxfp4<<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); + } } diff --git a/ggml/src/ggml-cuda/quantize.cuh b/ggml/src/ggml-cuda/quantize.cuh index 6a91df6..5f08dcb 100644 --- a/ggml/src/ggml-cuda/quantize.cuh +++ b/ggml/src/ggml-cuda/quantize.cuh @@ -26,10 +26,12 @@ void quantize_mmq_q8_1_cuda( ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); -void quantize_mmq_mxfp4_cuda(const float * x, +void quantize_mmq_fp4_cuda(const float * x, const int32_t * ids, void * vy, + float * scale, ggml_type type_src0, + bool use_aligned_float8, int64_t ne00, int64_t s01, int64_t s02, @@ -39,3 +41,30 @@ void quantize_mmq_mxfp4_cuda(const float * x, int64_t ne2, int64_t ne3, cudaStream_t stream); + +// quantize each token once and scatter the block to its compact rows (via the inverse map) +void quantize_scatter_mmq_fp4_cuda(const float * x, + const int32_t * ids_src1_inv, + void * vy, + float * scale, + ggml_type type_src0, + bool use_aligned_float8, + int64_t ne00, + int64_t stride_token, + int64_t ne0, + int64_t n_tokens, + int64_t nrows_dst, + int n_expert_used, + cudaStream_t stream); + +void quantize_scatter_mmq_q8_1_cuda(const float * x, + const int32_t * ids_src1_inv, + void * vy, + ggml_type type_src0, + int64_t ne00, + int64_t stride_token, + int64_t ne0, + int64_t n_tokens, + int64_t nrows_dst, + int n_expert_used, + cudaStream_t stream); diff --git a/ggml/src/ggml-cuda/reduce_rows.cuh b/ggml/src/ggml-cuda/reduce_rows.cuh index de240fd..111fd83 100644 --- a/ggml/src/ggml-cuda/reduce_rows.cuh +++ b/ggml/src/ggml-cuda/reduce_rows.cuh @@ -1,19 +1,18 @@ #include "common.cuh" -// Row reduction kernel template - compute sum (norm=false) or mean (norm=true) -template -static __global__ void reduce_rows_f32(const float * __restrict__ x, float * __restrict__ dst, const int ncols) { - const int row = blockIdx.x; +static __device__ __forceinline__ float reduce_row_f32(const float * x, const int ncols) { const int col = threadIdx.x; float sum = 0.0f; const int num_unroll = 8; float temp[num_unroll]; float sum_temp[num_unroll] = { 0.0f }; + + ggml_cuda_pdl_sync(); for (int i = col; i < ncols;) { for (int j = 0; j < num_unroll; ++j) { if (i < ncols) { - temp[j] = x[row * ncols + i]; + temp[j] = x[i]; } else { temp[j] = 0; } @@ -31,6 +30,40 @@ static __global__ void reduce_rows_f32(const float * __restrict__ x, float * __r __shared__ float shared_vals[32]; sum = block_reduce(sum, shared_vals); + return sum; +} + +// Row reduction kernel template - compute sum (norm=false) or mean (norm=true) +template +static __global__ void reduce_rows_f32(const float * x_ptr, float * dst_ptr, const int ncols) { + float * GGML_CUDA_RESTRICT dst = dst_ptr; + const int64_t row = blockIdx.x; + const int col = threadIdx.x; + + const float * GGML_CUDA_RESTRICT x = x_ptr + row*ncols; + const float sum = reduce_row_f32(x, ncols); + + if (col != 0) { + return; + } + + dst[row] = norm ? sum / ncols : sum; +} + +template +static __global__ void reduce_rows_f32_strided(const char * x_ptr, float * dst_ptr, const int ncols, + const int64_t ne1, const int64_t ne2, const int64_t nb1, const int64_t nb2, const int64_t nb3) { + float * GGML_CUDA_RESTRICT dst = dst_ptr; + const int64_t row = blockIdx.x; + const int col = threadIdx.x; + + const int64_t i1 = row % ne1; + const int64_t i2 = (row / ne1) % ne2; + const int64_t i3 = row / (ne1 * ne2); + + const float * GGML_CUDA_RESTRICT x = (const float *) (x_ptr + i1*nb1 + i2*nb2 + i3*nb3); + const float sum = reduce_row_f32(x, ncols); + if (col != 0) { return; } diff --git a/ggml/src/ggml-cuda/roll.cu b/ggml/src/ggml-cuda/roll.cu index a339dfc..c9bbe76 100644 --- a/ggml/src/ggml-cuda/roll.cu +++ b/ggml/src/ggml-cuda/roll.cu @@ -17,6 +17,10 @@ static __global__ void roll_f32_cuda(const float * __restrict__ src, const int64_t ne01, const int64_t ne02, const int64_t ne03, + const int64_t nb00, + const int64_t nb01, + const int64_t nb02, + const int64_t nb03, const int s0, const int s1, const int s2, @@ -39,7 +43,7 @@ static __global__ void roll_f32_cuda(const float * __restrict__ src, const int64_t d3 = wrap_index(i3 - s3, ne03); dst[i3 * (ne00 * ne01 * ne02) + i2 * (ne01 * ne00) + i1 * ne00 + i0] = - src[d3 * (ne00 * ne01 * ne02) + d2 * (ne01 * ne00) + d1 * ne00 + d0]; + src[(d3 * nb03 + d2 * nb02 + d1 * nb01 + d0 * nb00) / sizeof(float)]; } void ggml_cuda_op_roll(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { @@ -63,5 +67,5 @@ void ggml_cuda_op_roll(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { int64_t num_blocks = (sz + CUDA_ROLL_BLOCK_SIZE - 1) / CUDA_ROLL_BLOCK_SIZE; roll_f32_cuda<<>>( - src0_d, dst_d, ne00, ne01, ne02, ne03, s0, s1, s2, s3); + src0_d, dst_d, ne00, ne01, ne02, ne03, nb00, nb01, nb02, nb03, s0, s1, s2, s3); } diff --git a/ggml/src/ggml-cuda/rope.cu b/ggml/src/ggml-cuda/rope.cu index 45a49a5..45e8134 100644 --- a/ggml/src/ggml-cuda/rope.cu +++ b/ggml/src/ggml-cuda/rope.cu @@ -53,6 +53,7 @@ static __global__ void rope_norm(const T * x, const int s2, const int s3, const int n_dims, + const int n_offs, const int32_t * pos, const float freq_scale, const float ext_factor, @@ -61,7 +62,8 @@ static __global__ void rope_norm(const T * x, const float theta_scale, const float * freq_factors, const int64_t * row_indices, - const int set_rows_stride) { + const int set_rows_stride, + const bool inplace) { const int i0 = 2*(blockDim.y*blockIdx.y + threadIdx.y); if (i0 >= ne00) { @@ -92,19 +94,24 @@ static __global__ void rope_norm(const T * x, ggml_cuda_memcpy_1<4>(dst + idst, &v); } }; - if (i0 >= n_dims) { + if (i0 < n_offs || i0 >= n_offs + n_dims) { + if (inplace) { + return; + } store_coaelsced(x[ix + 0], x[ix + 1]); return; } - const float theta_base = pos[i2]*powf(theta_scale, i0/2.0f); + const int iw = i0 - n_offs; // relative idx - const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f; + const float theta_base = pos[i2]*powf(theta_scale, iw/2.0f); + + const float freq_factor = has_ff ? freq_factors[iw/2] : 1.0f; float cos_theta; float sin_theta; - rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta); + rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta); const float x0 = x[ix + 0]; const float x1 = x[ix + 1]; @@ -125,6 +132,7 @@ static __global__ void rope_neox(const T * x, const int s2, const int s3, const int n_dims, + const int n_offs, const int32_t * pos, const float freq_scale, const float ext_factor, @@ -133,7 +141,9 @@ static __global__ void rope_neox(const T * x, const float theta_scale, const float * freq_factors, const int64_t * row_indices, - const int set_rows_stride) { + const int set_rows_stride, + const bool inplace) { + ggml_cuda_pdl_lc(); const int i0 = 2*(blockDim.y*blockIdx.y + threadIdx.y); if (i0 >= ne00) { @@ -148,6 +158,7 @@ static __global__ void rope_neox(const T * x, int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3; const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03; + ggml_cuda_pdl_sync(); // Fusion optimization: ROPE + VIEW + SET_ROWS. // The rope output is viewed as a 1D tensor and offset based on a row index in row_indices. @@ -156,32 +167,38 @@ static __global__ void rope_neox(const T * x, idst += row_indices[i2] * set_rows_stride; } - if (i0 >= n_dims) { + if (i0 < n_offs || i0 >= n_offs + n_dims) { + if (inplace) { + return; + } dst[idst + i0 / 2 + 0] = ggml_cuda_cast(x[ix + i0 / 2 + 0]); dst[idst + i0 / 2 + 1] = ggml_cuda_cast(x[ix + i0 / 2 + 1]); return; } - const float theta_base = pos[i2]*powf(theta_scale, i0/2.0f); + const int iw = i0 - n_offs; // relative idx - const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f; + const float theta_base = pos[i2]*powf(theta_scale, iw/2.0f); + + const float freq_factor = has_ff ? freq_factors[iw/2] : 1.0f; float cos_theta; float sin_theta; - rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta); + rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta); - const float x0 = x[ix + 0]; - const float x1 = x[ix + n_dims/2]; + // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2 + const float x0 = x[ix + n_offs/2 + 0]; + const float x1 = x[ix + n_offs/2 + n_dims/2]; - dst[idst + 0] = ggml_cuda_cast(x0 * cos_theta - x1 * sin_theta); - dst[idst + n_dims / 2] = ggml_cuda_cast(x0 * sin_theta + x1 * cos_theta); + dst[idst + n_offs/2 + 0] = ggml_cuda_cast(x0 * cos_theta - x1 * sin_theta); + dst[idst + n_offs/2 + n_dims / 2] = ggml_cuda_cast(x0 * sin_theta + x1 * cos_theta); } -template +template static __global__ void rope_multi(const T * x, - T * dst, + D * dst, const int ne00, const int ne01, const int ne02, @@ -192,6 +209,7 @@ static __global__ void rope_multi(const T * x, const int s2, const int s3, const int n_dims, + const int n_offs, const int32_t * pos, const float freq_scale, const float ext_factor, @@ -200,7 +218,10 @@ static __global__ void rope_multi(const T * x, const float theta_scale, const float * freq_factors, const mrope_sections sections, - const bool is_imrope) { + const bool is_imrope, + const int64_t * row_indices, + const int set_rows_stride, + const bool inplace) { const int i0 = 2 * (blockDim.y * blockIdx.y + threadIdx.y); if (i0 >= ne00) { @@ -216,52 +237,67 @@ static __global__ void rope_multi(const T * x, int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3; const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03; - if (i0 >= n_dims) { - dst[idst + i0/2 + 0] = x[ix + i0/2 + 0]; - dst[idst + i0/2 + 1] = x[ix + i0/2 + 1]; + ggml_cuda_pdl_sync(); + + // Fusion optimization: ROPE + VIEW + SET_ROWS. + // The rope output is viewed as a 1D tensor and offset based on a row index in row_indices. + if (set_rows_stride != 0) { + idst = i1 * s1 + i0 / 2; + idst += row_indices[i2] * set_rows_stride; + } + + if (i0 < n_offs || i0 >= n_offs + n_dims) { + if (inplace) { + return; + } + dst[idst + i0/2 + 0] = ggml_cuda_cast(x[ix + i0/2 + 0]); + dst[idst + i0/2 + 1] = ggml_cuda_cast(x[ix + i0/2 + 1]); return; } + const int iw = i0 - n_offs; // relative idx + const int sect_dims = sections.v[0] + sections.v[1] + sections.v[2] + sections.v[3]; const int sec_w = sections.v[1] + sections.v[0]; - const int sector = (i0 / 2) % sect_dims; + const int sector = (iw / 2) % sect_dims; float theta_base = 0.0; if (is_imrope) { if (sector % 3 == 1 && sector < 3 * sections.v[1]) { // h - theta_base = pos[i2 + ne02 * 1] * powf(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 1] * powf(theta_scale, iw / 2.0f); } else if (sector % 3 == 2 && sector < 3 * sections.v[2]) { // w - theta_base = pos[i2 + ne02 * 2] * powf(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 2] * powf(theta_scale, iw / 2.0f); } else if (sector % 3 == 0 && sector < 3 * sections.v[0]) { // t - theta_base = pos[i2] * powf(theta_scale, i0 / 2.0f); + theta_base = pos[i2] * powf(theta_scale, iw / 2.0f); } else { - theta_base = pos[i2 + ne02 * 3] * powf(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 3] * powf(theta_scale, iw / 2.0f); } } else { if (sector < sections.v[0]) { - theta_base = pos[i2] * powf(theta_scale, i0 / 2.0f); + theta_base = pos[i2] * powf(theta_scale, iw / 2.0f); } else if (sector >= sections.v[0] && sector < sec_w) { - theta_base = pos[i2 + ne02 * 1] * powf(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 1] * powf(theta_scale, iw / 2.0f); } else if (sector >= sec_w && sector < sec_w + sections.v[2]) { - theta_base = pos[i2 + ne02 * 2] * powf(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 2] * powf(theta_scale, iw / 2.0f); } else if (sector >= sec_w + sections.v[2]) { - theta_base = pos[i2 + ne02 * 3] * powf(theta_scale, i0 / 2.0f); + theta_base = pos[i2 + ne02 * 3] * powf(theta_scale, iw / 2.0f); } } - const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f; + const float freq_factor = has_ff ? freq_factors[iw/2] : 1.0f; float cos_theta; float sin_theta; - rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta); + rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta); - const float x0 = x[ix + 0]; - const float x1 = x[ix + n_dims/2]; + // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2 + const float x0 = x[ix + n_offs/2 + 0]; + const float x1 = x[ix + n_offs/2 + n_dims/2]; - dst[idst + 0] = x0*cos_theta - x1*sin_theta; - dst[idst + n_dims/2] = x0*sin_theta + x1*cos_theta; + dst[idst + n_offs/2 + 0] = ggml_cuda_cast(x0*cos_theta - x1*sin_theta); + dst[idst + n_offs/2 + n_dims/2] = ggml_cuda_cast(x0*sin_theta + x1*cos_theta); } template @@ -300,6 +336,7 @@ static __global__ void rope_vision(const T * x, int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3; const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03; + ggml_cuda_pdl_sync(); const int sect_dims = sections.v[0] + sections.v[1]; const int sec_w = sections.v[1] + sections.v[0]; const int sector = (i0 / 2) % sect_dims; @@ -340,6 +377,7 @@ static void rope_norm_cuda(const T * x, const int s2, const int s3, const int n_dims, + const int n_offs, const int nr, const int32_t * pos, const float freq_scale, @@ -350,6 +388,7 @@ static void rope_norm_cuda(const T * x, const float * freq_factors, const int64_t * row_indices, const int set_rows_stride, + const bool inplace, cudaStream_t stream) { GGML_ASSERT(ne00 % 2 == 0); const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1); @@ -360,12 +399,12 @@ static void rope_norm_cuda(const T * x, if (freq_factors == nullptr) { rope_norm<<>>( - x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor, - attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride); + x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor, + attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride, inplace); } else { rope_norm<<>>( - x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor, - attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride); + x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor, + attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride, inplace); } } @@ -382,6 +421,7 @@ static void rope_neox_cuda(const T * x, const int s2, const int s3, const int n_dims, + const int n_offs, const int nr, const int32_t * pos, const float freq_scale, @@ -392,6 +432,7 @@ static void rope_neox_cuda(const T * x, const float * freq_factors, const int64_t * row_indices, const int set_rows_stride, + const bool inplace, cudaStream_t stream) { GGML_ASSERT(ne00 % 2 == 0); const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1); @@ -399,21 +440,22 @@ static void rope_neox_cuda(const T * x, const dim3 block_nums(nr, n_blocks_x, 1); const float theta_scale = powf(freq_base, -2.0f / n_dims); + const ggml_cuda_kernel_launch_params launch_params = {block_nums, block_dims, 0, stream}; if (freq_factors == nullptr) { - rope_neox<<>>( - x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor, - attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride); + ggml_cuda_kernel_launch(rope_neox, launch_params, + x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor, + attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride, inplace); } else { - rope_neox<<>>( - x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor, - attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride); + ggml_cuda_kernel_launch(rope_neox, launch_params, + x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor, + attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride, inplace); } } -template +template static void rope_multi_cuda(const T * x, - T * dst, + D * dst, const int ne00, const int ne01, const int ne02, @@ -424,6 +466,7 @@ static void rope_multi_cuda(const T * x, const int s2, const int s3, const int n_dims, + const int n_offs, const int nr, const int32_t * pos, const float freq_scale, @@ -434,6 +477,9 @@ static void rope_multi_cuda(const T * x, const float * freq_factors, const mrope_sections sections, const bool is_imrope, + const int64_t * row_indices, + const int set_rows_stride, + const bool inplace, cudaStream_t stream) { GGML_ASSERT(ne00 % 2 == 0); const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1); @@ -443,13 +489,15 @@ static void rope_multi_cuda(const T * x, const float theta_scale = powf(freq_base, -2.0f / n_dims); if (freq_factors == nullptr) { - rope_multi<<>>( - x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor, - attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + ggml_cuda_kernel_launch(rope_multi, launch_params, + x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor, + attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, row_indices, set_rows_stride, inplace); } else { - rope_multi<<>>( - x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, pos, freq_scale, ext_factor, - attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + ggml_cuda_kernel_launch(rope_multi, launch_params, + x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, pos, freq_scale, ext_factor, + attn_factor, corr_dims, theta_scale, freq_factors, sections, is_imrope, row_indices, set_rows_stride, inplace); } } @@ -545,8 +593,12 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, const int mode = ((int32_t *) dst->op_params)[2]; //const int n_ctx = ((int32_t *) dst->op_params)[3]; const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; + const int n_offs = ((int32_t *) dst->op_params)[15]; mrope_sections sections; + // when dst aliases src0, the channels outside the rotated window already hold the correct data + const bool inplace = dst_d == src0->data; + // RoPE alteration for extended context float freq_base; float freq_scale; @@ -574,6 +626,7 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, if (is_vision) { GGML_ASSERT(n_dims == ne00/2); + GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row } const int32_t * pos = (const int32_t *) src1_d; @@ -590,31 +643,38 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, if (is_neox) { if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) { rope_neox_cuda((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02, - s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, - set_rows_stride, stream); + set_rows_stride, inplace, stream); } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) { rope_neox_cuda((const float *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, - s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, - set_rows_stride, stream); + set_rows_stride, inplace, stream); } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) { rope_neox_cuda((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, - s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, - set_rows_stride, stream); + set_rows_stride, inplace, stream); } else { GGML_ABORT("fatal error"); } } else if (is_mrope && !is_vision) { - if (src0->type == GGML_TYPE_F32) { - rope_multi_cuda((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1, - s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, - corr_dims, freq_factors, sections, is_imrope, stream); - } else if (src0->type == GGML_TYPE_F16) { - rope_multi_cuda((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1, - s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, - corr_dims, freq_factors, sections, is_imrope, stream); + if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) { + rope_multi_cuda((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, + row_indices, set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) { + rope_multi_cuda((const float *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, + row_indices, set_rows_stride, inplace, stream); + } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) { + rope_multi_cuda((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, + ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope, + row_indices, set_rows_stride, inplace, stream); } else { GGML_ABORT("fatal error"); } @@ -633,19 +693,19 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx, } else { if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) { rope_norm_cuda((const float *) src0_d, (float *) dst_d, ne00, ne01, ne02, s01, s02, - s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, - set_rows_stride, stream); + set_rows_stride, inplace, stream); } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) { rope_norm_cuda((const float *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, - s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, - set_rows_stride, stream); + set_rows_stride, inplace, stream); } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) { rope_norm_cuda((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, - s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base, + s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, row_indices, - set_rows_stride, stream); + set_rows_stride, inplace, stream); } else { GGML_ABORT("fatal error"); } @@ -663,3 +723,248 @@ void ggml_cuda_op_rope_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) void ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rope, ggml_tensor * set_rows) { ggml_cuda_op_rope_impl(ctx, rope, set_rows); } + +// fused RMS_NORM + MUL + ROPE (+ VIEW + SET_ROWS) +// one block per row: block_reduce gives the norm scale, then each thread applies mul and rope to the elements it owns +template +static __global__ void rms_norm_mul_rope_f32( + const float * x, D * dst, const int ncols, const int nchannels, const int nsamples, + const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t s1, const int64_t s2, const int64_t s3, + const float eps, + const float * mul, + const int64_t mul_s01, const int64_t mul_s02, const int64_t mul_s03, + const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, + const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed, + const int n_dims, const int32_t * pos, + const float freq_scale, const float ext_factor, const float attn_factor, + const rope_corr_dims corr_dims, const float theta_scale, + const float * freq_factors, + const int64_t * row_indices, const int set_rows_stride, + const bool is_neox) { + ggml_cuda_pdl_lc(); + const int row = blockIdx.x; + const int tid = threadIdx.x; + + extern __shared__ float s_sum[]; + + ggml_cuda_pdl_sync(); + + // grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples + for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { + for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { + const float * xc = x + sample*s03 + channel*s02 + row*s01; + + const uint32_t mul_row = fastmodulo(row, mul_nrows_packed); + const uint32_t mul_channel = fastmodulo(channel, mul_nchannels_packed); + const uint32_t mul_sample = fastmodulo(sample, mul_nsamples_packed); + const float * mulc = mul + mul_sample*mul_s03 + mul_channel*mul_s02 + mul_row*mul_s01; + + float tmp = 0.0f; + + for (int col = tid; col < ncols; col += block_size) { + const float xi = xc[col]; + tmp += xi * xi; + } + + tmp = block_reduce(tmp, s_sum); + + const float scale = rsqrtf(tmp/ncols + eps); + + int64_t idst = sample*s3 + channel*s2 + row*s1; + if (set_rows_stride != 0) { + idst = row*s1 + row_indices[channel]*set_rows_stride; + } + D * dstc = dst + idst; + + for (int i0 = 2*tid; i0 < ncols; i0 += 2*block_size) { + int ix0; + int ix1; + if (is_neox && i0 < n_dims) { + ix0 = i0/2; + ix1 = i0/2 + n_dims/2; + } else { + ix0 = i0 + 0; + ix1 = i0 + 1; + } + + const float x0 = scale * xc[ix0] * mulc[fastmodulo(ix0, mul_ncols_packed)]; + const float x1 = scale * xc[ix1] * mulc[fastmodulo(ix1, mul_ncols_packed)]; + + if (i0 >= n_dims) { + dstc[ix0] = ggml_cuda_cast(x0); + dstc[ix1] = ggml_cuda_cast(x1); + continue; + } + + const float theta_base = pos[channel]*powf(theta_scale, i0/2.0f); + const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f; + + float cos_theta; + float sin_theta; + rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta); + + dstc[ix0] = ggml_cuda_cast(x0*cos_theta - x1*sin_theta); + dstc[ix1] = ggml_cuda_cast(x0*sin_theta + x1*cos_theta); + } + + if constexpr (block_size > WARP_SIZE) { + // sync is needed as we reuse s_sum across block_reduce invocations, see #26385 + __syncthreads(); + } + } + } +} + +template +static void rms_norm_mul_rope_cuda( + const float * x, D * dst, + const int ncols, const int nrows, const int nchannels, const int nsamples, + const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t s1, const int64_t s2, const int64_t s3, + const float eps, + const float * mul, + const int64_t mul_s01, const int64_t mul_s02, const int64_t mul_s03, + const uint32_t mul_ncols, const uint32_t mul_nrows, + const uint32_t mul_nchannels, const uint32_t mul_nsamples, + const int n_dims, const int32_t * pos, + const float freq_scale, const float freq_base, const float ext_factor, const float attn_factor, + const rope_corr_dims corr_dims, + const float * freq_factors, + const int64_t * row_indices, const int set_rows_stride, + const bool is_neox, cudaStream_t stream) { + GGML_ASSERT(ncols % 2 == 0); + + const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); + + const float theta_scale = powf(freq_base, -2.0f/n_dims); + + const uint3 mul_ncols_packed = init_fastdiv_values(mul_ncols); + const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows); + const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); + const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); + + if (ncols < 1024) { + const dim3 block_dims(256, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream}; + if (freq_factors == nullptr) { + ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, false, D>, launch_params, + x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, + freq_factors, row_indices, set_rows_stride, is_neox); + } else { + ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, true, D>, launch_params, + x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, + freq_factors, row_indices, set_rows_stride, is_neox); + } + } else { + const dim3 block_dims(1024, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream}; + if (freq_factors == nullptr) { + ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<1024, false, D>, launch_params, + x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, + freq_factors, row_indices, set_rows_stride, is_neox); + } else { + ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<1024, true, D>, launch_params, + x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, + freq_factors, row_indices, set_rows_stride, is_neox); + } + } +} + +void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, + ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows) { + const ggml_tensor * x = rms_norm->src[0]; + const ggml_tensor * mul_src = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0]; + + float eps = 0.0f; + memcpy(&eps, rms_norm->op_params, sizeof(float)); + GGML_ASSERT(eps >= 0.0f); + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(mul_src->type == GGML_TYPE_F32); + GGML_ASSERT(rope->type == GGML_TYPE_F32); + + void * dst_d = rope->data; + ggml_type dst_type = rope->type; + const int64_t * row_indices = nullptr; + int set_rows_stride = 0; + + if (set_rows != nullptr) { + dst_d = set_rows->data; + dst_type = set_rows->type; + row_indices = (const int64_t *) set_rows->src[1]->data; + set_rows_stride = set_rows->nb[1] / ggml_type_size(set_rows->type); + } + + const int n_dims = ((const int32_t *) rope->op_params)[1]; + const int mode = ((const int32_t *) rope->op_params)[2]; + const int n_ctx_orig = ((const int32_t *) rope->op_params)[4]; + + float freq_base; + float freq_scale; + float ext_factor; + float attn_factor; + float beta_fast; + float beta_slow; + + memcpy(&freq_base, (const int32_t *) rope->op_params + 5, sizeof(float)); + memcpy(&freq_scale, (const int32_t *) rope->op_params + 6, sizeof(float)); + memcpy(&ext_factor, (const int32_t *) rope->op_params + 7, sizeof(float)); + memcpy(&attn_factor, (const int32_t *) rope->op_params + 8, sizeof(float)); + memcpy(&beta_fast, (const int32_t *) rope->op_params + 9, sizeof(float)); + memcpy(&beta_slow, (const int32_t *) rope->op_params + 10, sizeof(float)); + + const bool is_neox = mode & GGML_ROPE_TYPE_NEOX; + + const int32_t * pos = (const int32_t *) rope->src[1]->data; + + const float * freq_factors = rope->src[2] != nullptr ? (const float *) rope->src[2]->data : nullptr; + + rope_corr_dims corr_dims; + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims.v); + + const size_t ts0 = ggml_type_size(x->type); + GGML_ASSERT(x->nb[0] == ts0); + const int64_t s01 = x->nb[1] / ts0; + const int64_t s02 = x->nb[2] / ts0; + const int64_t s03 = x->nb[3] / ts0; + + const size_t ts_mul = ggml_type_size(mul_src->type); + GGML_ASSERT(mul_src->nb[0] == ts_mul); + const int64_t mul_s01 = mul_src->nb[1] / ts_mul; + const int64_t mul_s02 = mul_src->nb[2] / ts_mul; + const int64_t mul_s03 = mul_src->nb[3] / ts_mul; + + const size_t ts_dst = ggml_type_size(rope->type); + const int64_t s1 = rope->nb[1] / ts_dst; + const int64_t s2 = rope->nb[2] / ts_dst; + const int64_t s3 = rope->nb[3] / ts_dst; + + cudaStream_t stream = ctx.stream(); + + if (dst_type == GGML_TYPE_F32) { + rms_norm_mul_rope_cuda((const float *) x->data, (float *) dst_d, + x->ne[0], x->ne[1], x->ne[2], x->ne[3], s01, s02, s03, s1, s2, s3, eps, + (const float *) mul_src->data, mul_s01, mul_s02, mul_s03, + mul_src->ne[0], mul_src->ne[1], mul_src->ne[2], mul_src->ne[3], + n_dims, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, + freq_factors, row_indices, set_rows_stride, is_neox, stream); + } else if (dst_type == GGML_TYPE_F16) { + rms_norm_mul_rope_cuda((const float *) x->data, (half *) dst_d, + x->ne[0], x->ne[1], x->ne[2], x->ne[3], s01, s02, s03, s1, s2, s3, eps, + (const float *) mul_src->data, mul_s01, mul_s02, mul_s03, + mul_src->ne[0], mul_src->ne[1], mul_src->ne[2], mul_src->ne[3], + n_dims, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, + freq_factors, row_indices, set_rows_stride, is_neox, stream); + } else { + GGML_ABORT("fatal error"); + } +} diff --git a/ggml/src/ggml-cuda/rope.cuh b/ggml/src/ggml-cuda/rope.cuh index 72af086..7ce2d71 100644 --- a/ggml/src/ggml-cuda/rope.cuh +++ b/ggml/src/ggml-cuda/rope.cuh @@ -7,3 +7,5 @@ void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_rope_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * set_rows); + +void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows); diff --git a/ggml/src/ggml-cuda/scale.cu b/ggml/src/ggml-cuda/scale.cu index 0ddeff6..5648b92 100644 --- a/ggml/src/ggml-cuda/scale.cu +++ b/ggml/src/ggml-cuda/scale.cu @@ -1,34 +1,42 @@ #include "scale.cuh" +#include "convert.cuh" #define MAX_GRIDDIM_X 0x7FFFFFFF -static __global__ void scale_f32(const float * x, float * dst, const float scale, const float bias, const int64_t nelements) { +template +static __global__ void scale_f32(const T * x, T * dst, const float scale, const float bias, const int64_t nelements) { + ggml_cuda_pdl_lc(); int64_t tid = (int64_t)blockIdx.x * (int64_t)blockDim.x + (int64_t)threadIdx.x; int64_t stride = (int64_t)blockDim.x * (int64_t)gridDim.x; + ggml_cuda_pdl_sync(); for (int64_t i = tid; i < nelements; i += stride) { - dst[i] = scale * x[i] + bias; + dst[i] = ggml_cuda_cast(scale * ggml_cuda_cast(x[i]) + bias); } } -static void scale_f32_cuda(const float * x, float * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) { +template +static void scale_f32_cuda(const T * x, T * dst, const float scale, const float bias, const int64_t nelements, cudaStream_t stream) { const int64_t num_blocks = (nelements + CUDA_SCALE_BLOCK_SIZE - 1) / CUDA_SCALE_BLOCK_SIZE; - scale_f32<<>>(x, dst, scale, bias, nelements); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(MIN(MAX_GRIDDIM_X, num_blocks), CUDA_SCALE_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(scale_f32, launch_params, x, dst, scale, bias, nelements); } void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; - const float * src0_d = (const float *)src0->data; - float * dst_d = (float *)dst->data; cudaStream_t stream = ctx.stream(); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16); + GGML_ASSERT(dst->type == src0->type); float scale; float bias; memcpy(&scale, (float *) dst->op_params + 0, sizeof(float)); memcpy(&bias, (float *) dst->op_params + 1, sizeof(float)); - scale_f32_cuda(src0_d, dst_d, scale, bias, ggml_nelements(src0), stream); + if (src0->type == GGML_TYPE_BF16) { + scale_f32_cuda((const nv_bfloat16 *) src0->data, (nv_bfloat16 *) dst->data, scale, bias, ggml_nelements(src0), stream); + } else { + scale_f32_cuda((const float *) src0->data, (float *) dst->data, scale, bias, ggml_nelements(src0), stream); + } } diff --git a/ggml/src/ggml-cuda/set-rows.cu b/ggml/src/ggml-cuda/set-rows.cu index 631de7e..4659970 100644 --- a/ggml/src/ggml-cuda/set-rows.cu +++ b/ggml/src/ggml-cuda/set-rows.cu @@ -53,6 +53,7 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0, const int64_t i11 = fastmodulo((uint32_t) i02, ne11_fd); const int64_t i10 = i01; + ggml_cuda_pdl_sync(); const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12); const float * src0_row = src0 + i01*s01 + i02*s02 + i03*s03; @@ -110,9 +111,9 @@ static void set_rows_cuda_quant( } template -static __global__ void k_set_rows(const src_t * __restrict__ src0, - const idx_t * __restrict__ src1, - dst_t * __restrict__ dst, +static __global__ void k_set_rows(const src_t * src0_ptr, + const idx_t * src1_ptr, + dst_t * dst_ptr, const int64_t ne_total, const int64_t ne10, const int64_t ne11, @@ -132,6 +133,9 @@ static __global__ void k_set_rows(const src_t * __restrict__ src0, const uint3 ne02, const uint3 ne11_fd, const uint3 ne12_fd) { + const src_t * GGML_CUDA_RESTRICT src0 = src0_ptr; + const idx_t * GGML_CUDA_RESTRICT src1 = src1_ptr; + dst_t * GGML_CUDA_RESTRICT dst = dst_ptr; const int64_t i = int64_t(blockDim.x) * blockIdx.x + threadIdx.x; if (i >= ne_total) { @@ -157,7 +161,9 @@ static __global__ void k_set_rows(const src_t * __restrict__ src0, const int64_t i11 = fastmodulo((uint32_t) i02, ne11_fd); const int64_t i10 = i01; + ggml_cuda_pdl_sync(); const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12); + ggml_cuda_pdl_lc(); const src_t * src0_row = src0 + i01*s01 + i02*s02 + i03*s03; dst_t * dst_row_ptr = dst + dst_row*s1 + i02*s2 + i03*s3; @@ -203,9 +209,11 @@ static void set_rows_cuda( const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11); const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12); - k_set_rows<<>>(src0_d, src1_d, dst_d, ne_total, ne10, ne11, ne12, ne13, s01, - s02, s03, s10, s11, s12, s1, s2, s3, ne00_fd, ne01_fd, ne02_fd, - ne11_fd, ne12_fd); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_size, block_size, 0, stream); + ggml_cuda_kernel_launch(k_set_rows, launch_params, + src0_d, src1_d, dst_d, ne_total, ne10, ne11, ne12, ne13, s01, + s02, s03, s10, s11, s12, s1, s2, s3, ne00_fd, ne01_fd, ne02_fd, + ne11_fd, ne12_fd); } } @@ -314,17 +322,77 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s } } +template<> +void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const half * src0_d = (const half *)src0->data; + const int32_t * src1_d = (const int32_t *)src1->data; + + GGML_TENSOR_BINARY_OP_LOCALS + + cudaStream_t stream = ctx.stream(); + + + if (dst->type == GGML_TYPE_F16) { + set_rows_cuda( + src0_d, src1_d, (half*)dst->data, + ne00, ne01, ne02, ne03, + ne10, ne11, ne12, ne13, + nb01, nb02, nb03, + nb10, nb11, nb12, + nb1, nb2, nb3, + stream + ); + } else { + GGML_ABORT("unsupported type %s", ggml_type_name(dst->type)); + } +} + +template<> +void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + const half * src0_d = (const half *)src0->data; + const int64_t * src1_d = (const int64_t *)src1->data; + + GGML_TENSOR_BINARY_OP_LOCALS + + cudaStream_t stream = ctx.stream(); + + + if (dst->type == GGML_TYPE_F16) { + set_rows_cuda( + src0_d, src1_d, (half*)dst->data, + ne00, ne01, ne02, ne03, + ne10, ne11, ne12, ne13, + nb01, nb02, nb03, + nb10, nb11, nb12, + nb1, nb2, nb3, + stream + ); + } else { + GGML_ABORT("unsupported type %s", ggml_type_name(dst->type)); + } +} + void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16)); GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32); - if (src1->type == GGML_TYPE_I64) { - set_rows_cuda(ctx, src0, src1, dst); + if (src0->type == GGML_TYPE_F32) { + if (src1->type == GGML_TYPE_I64) { + set_rows_cuda(ctx, src0, src1, dst); + } else { + set_rows_cuda(ctx, src0, src1, dst); + } + } else if (src0->type == GGML_TYPE_F16) { + if (src1->type == GGML_TYPE_I64) { + set_rows_cuda(ctx, src0, src1, dst); + } else { + set_rows_cuda(ctx, src0, src1, dst); + } } else { - set_rows_cuda(ctx, src0, src1, dst); + GGML_ABORT("unsupported type %s", ggml_type_name(src0->type)); } } diff --git a/ggml/src/ggml-cuda/snake.cu b/ggml/src/ggml-cuda/snake.cu new file mode 100644 index 0000000..384638c --- /dev/null +++ b/ggml/src/ggml-cuda/snake.cu @@ -0,0 +1,72 @@ +#include "snake.cuh" +#include "convert.cuh" + +// Fused Snake activation: y = x + sin^2(a * x) * inv_b +// x: [T, C] (T contiguous), a: [1, C], inv_b: [1, C] +// Supports F32, F16, BF16 data with F32 compute. + +template +static __global__ void snake_kernel( + const T * __restrict__ x, + const float * __restrict__ a, + const float * __restrict__ inv_b, + T * __restrict__ dst, + const int total, + const uint3 T_len_fastdiv) { + const int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= total) return; + + const int c = (int) fastdiv((uint32_t) idx, T_len_fastdiv); + + const float xi = ggml_cuda_cast(x[idx]); + const float s = sinf(a[c] * xi); + dst[idx] = ggml_cuda_cast(xi + s * s * inv_b[c]); +} + +// Internal launcher with explicit x/a/inv_b/dst tensors. +// Shared by the public op (reads dst->src) and the fusion path (explicit args). +static void launch_snake(ggml_backend_cuda_context & ctx, + const ggml_tensor * x, + const ggml_tensor * a, + const ggml_tensor * inv_b, + ggml_tensor * dst) { + const float * a_d = (const float *)a->data; + const float * inv_b_d = (const float *)inv_b->data; + + const int T = (int)x->ne[0]; + const int C = (int)x->ne[1]; + const int total = T * C; + const uint3 T_len_fastdiv = init_fastdiv_values((uint64_t) T); + + const int block_size = 256; + const int grid_size = (total + block_size - 1) / block_size; + + cudaStream_t stream = ctx.stream(); + + switch (x->type) { + case GGML_TYPE_F32: { + snake_kernel<<>>( + (const float *)x->data, a_d, inv_b_d, (float *)dst->data, total, T_len_fastdiv); + } break; + case GGML_TYPE_F16: { + snake_kernel<<>>( + (const half *)x->data, a_d, inv_b_d, (half *)dst->data, total, T_len_fastdiv); + } break; + case GGML_TYPE_BF16: { + snake_kernel<<>>( + (const nv_bfloat16 *)x->data, a_d, inv_b_d, (nv_bfloat16 *)dst->data, total, T_len_fastdiv); + } break; + default: + GGML_ABORT("snake: unsupported type"); + } +} + +// Fusion entry: caller supplies x/a/inv_b explicitly from the matched +// mul -> sin -> sqr -> mul -> add pattern. The dst is the trailing add output. +void ggml_cuda_op_snake_fused(ggml_backend_cuda_context & ctx, + const ggml_tensor * x, + const ggml_tensor * a, + const ggml_tensor * inv_b, + ggml_tensor * dst) { + launch_snake(ctx, x, a, inv_b, dst); +} diff --git a/ggml/src/ggml-cuda/snake.cuh b/ggml/src/ggml-cuda/snake.cuh new file mode 100644 index 0000000..7f6f1cb --- /dev/null +++ b/ggml/src/ggml-cuda/snake.cuh @@ -0,0 +1,8 @@ +#include "common.cuh" + +// Fusion entry point. Caller supplies x/a/inv_b explicitly. +void ggml_cuda_op_snake_fused(ggml_backend_cuda_context & ctx, + const ggml_tensor * x, + const ggml_tensor * a, + const ggml_tensor * inv_b, + ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/softcap.cu b/ggml/src/ggml-cuda/softcap.cu index 40dfe45..9f0fa10 100644 --- a/ggml/src/ggml-cuda/softcap.cu +++ b/ggml/src/ggml-cuda/softcap.cu @@ -1,18 +1,21 @@ #include "softcap.cuh" static __global__ void softcap_f32(const float * x, float * dst, const float scale, const float softcap, const int k) { + ggml_cuda_pdl_lc(); const int i = blockDim.x*blockIdx.x + threadIdx.x; if (i >= k) { return; } + ggml_cuda_pdl_sync(); dst[i] = tanhf(scale * x[i]) * softcap; } static void softcap_f32_cuda(const float * x, float * dst, const float scale, const float softcap, const int k, cudaStream_t stream) { const int num_blocks = (k + CUDA_SOFTCAP_BLOCK_SIZE - 1) / CUDA_SOFTCAP_BLOCK_SIZE; - softcap_f32<<>>(x, dst, scale, softcap, k); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, CUDA_SOFTCAP_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(softcap_f32, launch_params, x, dst, scale, softcap, k); } // fused GGML_OP_SCALE + GGML_UNARY_OP_TANH + GGML_OP_SCALE diff --git a/ggml/src/ggml-cuda/softmax.cu b/ggml/src/ggml-cuda/softmax.cu index 285c0e9..f320c6f 100644 --- a/ggml/src/ggml-cuda/softmax.cu +++ b/ggml/src/ggml-cuda/softmax.cu @@ -116,6 +116,11 @@ static __global__ void soft_max_f32( vals[col] = val; } + if (block_size > WARP_SIZE) { + // sync is needed as we reuse buf_iw across block_reduce invocations, see #26385 + // for block_size <= WARP_SIZE, block_reduce does not access buf_iw + __syncthreads(); + } // find the sum of exps in the block tmp = block_reduce(tmp, buf_iw); @@ -142,6 +147,8 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ float * __restrict__ dst, float * __restrict__ tmp_maxs, float * __restrict__ tmp_sums, + float * shared_vals_max, + float * shared_vals_sum, const soft_max_params p) { namespace cg = cooperative_groups; @@ -154,7 +161,6 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ float local_vals[n_elem_per_thread] = { -INFINITY, -INFINITY, -INFINITY, -INFINITY }; float local_max = -INFINITY; const int step_size = gridDim.x * blockDim.x; - __shared__ float shared_vals[32]; // Compute thread-local max for (int col = col_start; col < p.ncols;) { @@ -171,7 +177,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ } // Compute CTA-level max - local_max = block_reduce(local_max, shared_vals); + local_max = block_reduce(local_max, shared_vals_max); // Store CTA-level max to GMEM if (tid == 0) { @@ -186,7 +192,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ } else { local_max = -INFINITY; } - local_max = block_reduce(local_max, shared_vals); + local_max = block_reduce(local_max, shared_vals_max); // Compute softmax dividends, accumulate divisor float tmp_expf = 0.0f; @@ -209,7 +215,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ } // Reduce divisor within CTA - tmp_expf = block_reduce(tmp_expf, shared_vals); + tmp_expf = block_reduce(tmp_expf, shared_vals_sum); // Store CTA-level sum to GMEM if (tid == 0) { @@ -223,7 +229,7 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __ } else { tmp_expf = 0.0f; } - tmp_expf = block_reduce(tmp_expf, shared_vals); + tmp_expf = block_reduce(tmp_expf, shared_vals_sum); // Divide dividend by global sum + store data for (int col = col_start; col < p.ncols;) { @@ -310,9 +316,11 @@ __launch_bounds__(8*WARP_SIZE, 1) static __global__ void soft_max_f32_paralleliz // https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#grid-synchronization // https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/device-callable-apis.html#class-cluster-group { + __shared__ float shared_vals[2][32]; + for (int rowx = 0; rowx < p.ne01 * p.ne02 * p.ne03; rowx++) { soft_max_f32_parallelize_cols_single_row(x + int64_t(rowx) * p.ncols, dst + int64_t(rowx) * p.ncols, tmp_maxs, - tmp_sums, p); + tmp_sums, shared_vals[0], shared_vals[1], p); } } diff --git a/ggml/src/ggml-cuda/solve_tri.cu b/ggml/src/ggml-cuda/solve_tri.cu index 07ca33f..d967834 100644 --- a/ggml/src/ggml-cuda/solve_tri.cu +++ b/ggml/src/ggml-cuda/solve_tri.cu @@ -65,15 +65,13 @@ static void solve_tri_f32_cublas(ggml_backend_cuda_context & ctx, get_batch_pointers<<<(total_batches + 255) / 256, 256, 0, stream>>>(A, X, A_ptrs_dev, X_ptrs_dev, ne02, total_batches, s02, s03, s2, s3); - CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream)); - // Yes, this is necessary, without this we get RMSE errors - CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_DEFAULT_MATH)); - CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(id), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_DEFAULT_MATH)); + CUBLAS_CHECK(cublasStrsmBatched(ctx.cublas_handle(), CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT, k, n, &alpha, A_ptrs_dev, n, X_ptrs_dev, k, total_batches)); // revert to standard mode from common.cuh - CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(id), CUBLAS_TF32_TENSOR_OP_MATH)); + CUBLAS_CHECK(cublasSetMathMode(ctx.cublas_handle(), CUBLAS_TF32_TENSOR_OP_MATH)); GGML_UNUSED_VARS(s12, s13); } diff --git a/ggml/src/ggml-cuda/ssm-conv.cu b/ggml/src/ggml-cuda/ssm-conv.cu index b77cdc1..f41438b 100644 --- a/ggml/src/ggml-cuda/ssm-conv.cu +++ b/ggml/src/ggml-cuda/ssm-conv.cu @@ -1,11 +1,18 @@ +#include "common.cuh" #include "ssm-conv.cuh" #include "unary.cuh" template -static __global__ void ssm_conv_f32(const float * __restrict__ src0, const float * __restrict__ src1, +static __global__ void ssm_conv_f32(const float * src0_ptr, const float * src1_ptr, + const float * bias_ptr, const int src0_nb0, const int src0_nb1, const int src0_nb2, const int src1_nb1, - float * __restrict__ dst, const int dst_nb0, const int dst_nb1, const int dst_nb2, + float * dst_ptr, const int dst_nb0, const int dst_nb1, const int dst_nb2, const int64_t n_t) { + ggml_cuda_pdl_lc(); + const float * GGML_CUDA_RESTRICT src0 = src0_ptr; + const float * GGML_CUDA_RESTRICT src1 = src1_ptr; + const float * GGML_CUDA_RESTRICT bias = bias_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; GGML_UNUSED(src0_nb0); const int tid = threadIdx.x; const int bidx = blockIdx.x; @@ -22,11 +29,14 @@ static __global__ void ssm_conv_f32(const float * __restrict__ src0, const float float x[d_conv] = { 0.0f }; float w[d_conv] = { 0.0f }; + ggml_cuda_pdl_sync(); #pragma unroll for (size_t j = 0; j < d_conv; j++) { w[j] = w_block[tid * stride_w + j]; } + float b = bias != nullptr ? bias[bidy * split_d_inner + tid] : 0.0f; + for (int64_t i = 0; i < n_t; i++) { float sumf = 0.0f; @@ -42,12 +52,14 @@ static __global__ void ssm_conv_f32(const float * __restrict__ src0, const float for (size_t j = 0; j < d_conv; j++) { sumf += x[(i + j) % d_conv] * w[j]; } + sumf += b; y_block[i * stride_y + tid] = apply_silu ? ggml_cuda_op_silu_single(sumf) : sumf; } } template static __global__ void ssm_conv_long_token_f32(const float * __restrict__ src0, const float * __restrict__ src1, + const float * __restrict__ bias, const int src0_nb0, const int src0_nb1, const int src0_nb2, const int src1_nb1, float * __restrict__ dst, const int dst_nb0, const int dst_nb1, const int dst_nb2, const int64_t n_t) { @@ -97,6 +109,8 @@ static __global__ void ssm_conv_long_token_f32(const float * __restrict__ src0, w[j] = w_block[tid * stride_w + j]; } + float b = bias != nullptr ? bias[bidy * split_d_inner + tid] : 0.0f; + // Compute from shared memory for (int64_t i = 0; i < local_n_t; i++) { float sumf = 0.0f; @@ -104,12 +118,13 @@ static __global__ void ssm_conv_long_token_f32(const float * __restrict__ src0, for (size_t j = 0; j < d_conv; j++) { sumf += smem[tid * n_cols + i + j] * w[j]; } + sumf += b; y_block[i * stride_y + tid] = apply_silu ? ggml_cuda_op_silu_single(sumf) : sumf; } } template -static void ssm_conv_f32_cuda(const float * src0, const float * src1, const int src0_nb0, const int src0_nb1, +static void ssm_conv_f32_cuda(const float * src0, const float * src1, const float * bias, const int src0_nb0, const int src0_nb1, const int src0_nb2, const int src1_nb1, float * dst, const int dst_nb0, const int dst_nb1, const int dst_nb2, const int64_t nc, const int64_t nr, const int64_t n_t, const int64_t n_s, cudaStream_t stream) { @@ -120,31 +135,41 @@ static void ssm_conv_f32_cuda(const float * src0, const float * src1, const int constexpr int kNC = decltype(NC)::value; if (n_t <= 32) { const dim3 blocks(n_s, (nr + threads - 1) / threads, 1); - ssm_conv_f32<<>>(src0, src1, src0_nb0, src0_nb1, src0_nb2, src1_nb1, - dst, dst_nb0, dst_nb1, dst_nb2, n_t); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, 0, stream); + ggml_cuda_kernel_launch(ssm_conv_f32, launch_params, src0, src1, bias, src0_nb0, src0_nb1, + src0_nb2, src1_nb1, dst, dst_nb0, dst_nb1, dst_nb2, n_t); } else { const int64_t split_n_t = 32; dim3 blocks(n_s, (nr + threads - 1) / threads, (n_t + split_n_t - 1) / split_n_t); const size_t smem_size = threads * (kNC - 1 + split_n_t) * sizeof(float); ssm_conv_long_token_f32<<>>( - src0, src1, src0_nb0, src0_nb1, src0_nb2, src1_nb1, dst, dst_nb0, dst_nb1, dst_nb2, n_t); + src0, src1, bias, src0_nb0, src0_nb1, src0_nb2, src1_nb1, dst, dst_nb0, dst_nb1, dst_nb2, n_t); } }; switch (nc) { - case 3: launch_kernel(std::integral_constant{}); break; - case 4: launch_kernel(std::integral_constant{}); break; - case 5: launch_kernel(std::integral_constant{}); break; - case 9: launch_kernel(std::integral_constant{}); break; - default: GGML_ABORT("Only support kernel sizes 3, 4, 5, 9 right now."); + case 3: launch_kernel(std::integral_constant{}); break; + case 4: launch_kernel(std::integral_constant{}); break; + case 5: launch_kernel(std::integral_constant{}); break; + case 7: launch_kernel(std::integral_constant{}); break; + case 9: launch_kernel(std::integral_constant{}); break; + case 15: launch_kernel(std::integral_constant{}); break; + default: GGML_ABORT("Only support kernel sizes 3, 4, 5, 7, 9, 15 right now."); } } -void ggml_cuda_op_ssm_conv(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * silu_dst) { +void ggml_cuda_op_ssm_conv(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * bias_add_node, ggml_tensor * silu_dst) { const struct ggml_tensor * src0 = dst->src[0]; // conv_x const struct ggml_tensor * src1 = dst->src[1]; // conv1d.weight + const bool fuse_bias = bias_add_node != nullptr; const bool fuse_silu = silu_dst != nullptr; + // bias always comes with silu. + GGML_ASSERT(!fuse_bias || fuse_silu); + + // The bias (when fused) is the non-conv operand of the ADD node. + const struct ggml_tensor * bias = fuse_bias ? (bias_add_node->src[0] == dst ? bias_add_node->src[1] : bias_add_node->src[0]) : nullptr; + // When fusing, write to silu_dst (the node downstream references). const struct ggml_tensor * out = fuse_silu ? silu_dst : dst; @@ -160,16 +185,23 @@ void ggml_cuda_op_ssm_conv(ggml_backend_cuda_context & ctx, ggml_tensor * dst, g const float * src0_d = (const float *) src0->data; const float * src1_d = (const float *) src1->data; + const float * bias_d = fuse_bias ? (const float *) bias->data : nullptr; float * dst_d = (float *) out->data; cudaStream_t stream = ctx.stream(); GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(out->type == GGML_TYPE_F32); + if (fuse_bias) { + GGML_ASSERT(bias->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(bias)); + GGML_ASSERT(ggml_nelements(bias) == nr); + } + if (fuse_silu) { - ssm_conv_f32_cuda(src0_d, src1_d, src0->nb[0], src0->nb[1], src0->nb[2], src1->nb[1], dst_d, out->nb[0], out->nb[1], + ssm_conv_f32_cuda(src0_d, src1_d, bias_d, src0->nb[0], src0->nb[1], src0->nb[2], src1->nb[1], dst_d, out->nb[0], out->nb[1], out->nb[2], nc, nr, n_t, n_s, stream); } else { - ssm_conv_f32_cuda(src0_d, src1_d, src0->nb[0], src0->nb[1], src0->nb[2], src1->nb[1], dst_d, out->nb[0], out->nb[1], + ssm_conv_f32_cuda(src0_d, src1_d, bias_d, src0->nb[0], src0->nb[1], src0->nb[2], src1->nb[1], dst_d, out->nb[0], out->nb[1], out->nb[2], nc, nr, n_t, n_s, stream); } } diff --git a/ggml/src/ggml-cuda/ssm-conv.cuh b/ggml/src/ggml-cuda/ssm-conv.cuh index f96a1cd..8514ca8 100644 --- a/ggml/src/ggml-cuda/ssm-conv.cuh +++ b/ggml/src/ggml-cuda/ssm-conv.cuh @@ -1,3 +1,3 @@ #include "common.cuh" -void ggml_cuda_op_ssm_conv(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * silu_dst = nullptr); +void ggml_cuda_op_ssm_conv(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * bias_add_node = nullptr, ggml_tensor * silu_dst = nullptr); diff --git a/ggml/src/ggml-cuda/ssm-scan.cu b/ggml/src/ggml-cuda/ssm-scan.cu index c1d4e2b..dcc199f 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cu +++ b/ggml/src/ggml-cuda/ssm-scan.cu @@ -1,6 +1,6 @@ -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070 +#if !defined(GGML_USE_HIP) && (defined(GGML_USE_MUSA) || CUDART_VERSION >= 11070) #define USE_CUB -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070 +#endif // !defined(GGML_USE_HIP) && (defined(GGML_USE_MUSA) || CUDART_VERSION >= 11070) #ifdef USE_CUB #include @@ -9,6 +9,21 @@ using namespace cub; #include "ssm-scan.cuh" + +// Minimum number of tokens to use SSD (State Space Duality) matmul path instead of scan path. +// For n_tok <= this threshold, the scan kernel is used (lower overhead for short sequences). +#define SSM_SSD_MIN_TOKENS 128 + +// prepare_dt kernel dimensions: one block per (head, seq), each block handles DT_MAX_ITEMS items. +#define SSM_SSD_DT_BLOCK 256 +#define SSM_SSD_DT_MAX_ITEMS 32 + +// Maximum tokens the SSD path supports, derived from the prepare_dt kernel block capacity. +#define SSM_SSD_MAX_TOKENS (SSM_SSD_DT_BLOCK * SSM_SSD_DT_MAX_ITEMS) + +// Chunk size for chunked SSD. Caps matmul cost at O(chunk^2) per chunk. +#define SSM_SSD_CHUNK_SIZE 256 + // We would like to keep pragma unroll for cases where L_template is not 0, // so we suppress the clang transformation warning. #ifdef __clang__ @@ -17,15 +32,24 @@ using namespace cub; #endif // __clang__ template __global__ void __launch_bounds__(splitD, 1) - ssm_scan_f32(const float *__restrict__ src0, const float *__restrict__ src1, const float *__restrict__ src2, - const float *__restrict__ src3, const float *__restrict__ src4, const float *__restrict__ src5, - const int32_t * __restrict__ src6, float * __restrict__ dst, + ssm_scan_f32(const float * src0_ptr, const float * src1_ptr, const float * src2_ptr, + const float * src3_ptr, const float * src4_ptr, const float * src5_ptr, + const int32_t * src6_ptr, float * dst_ptr, const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3, const int src2_nb1, const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3, const int64_t s_off, const int64_t d_inner, const int64_t L_param) { + const float * GGML_CUDA_RESTRICT src0 = src0_ptr; + const float * GGML_CUDA_RESTRICT src1 = src1_ptr; + const float * GGML_CUDA_RESTRICT src2 = src2_ptr; + const float * GGML_CUDA_RESTRICT src3 = src3_ptr; + const float * GGML_CUDA_RESTRICT src4 = src4_ptr; + const float * GGML_CUDA_RESTRICT src5 = src5_ptr; + const int32_t * GGML_CUDA_RESTRICT src6 = src6_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; const size_t L = L_template == 0 ? L_param : L_template; + ggml_cuda_pdl_sync(); const float *s0_block = (const float *)((const char *)src0 + src6[blockIdx.x] * src0_nb3 + blockIdx.y * splitD * src0_nb2); const float *x_block = (const float *)((const char *)src1 + (blockIdx.x * src1_nb3) + blockIdx.y * splitD * sizeof(float)); const float *dt_block = (const float *)((const char *)src2 + (blockIdx.x * src2_nb2) + blockIdx.y * splitD * sizeof(float)); @@ -58,6 +82,7 @@ __global__ void __launch_bounds__(splitD, 1) __shared__ CubTempStorage cub_temp_storage; BlockLoad(cub_temp_storage.load_temp).Load(A_block, regA); + __syncthreads(); BlockLoad(cub_temp_storage.load_temp).Load(s0_block, regs0); #else const int stride_s0 = src0_nb2 / sizeof(float); @@ -96,6 +121,7 @@ __global__ void __launch_bounds__(splitD, 1) regs0[n] = state; } y_block[i * stride_y + threadIdx.x] = sumf; + __syncthreads(); } #ifdef USE_CUB @@ -117,18 +143,33 @@ __global__ void __launch_bounds__(splitD, 1) template __global__ void __launch_bounds__(d_state, 1) ssm_scan_f32_group( - const float * __restrict__ src0, const float * __restrict__ src1, const float * __restrict__ src2, - const float * __restrict__ src3, const float * __restrict__ src4, const float * __restrict__ src5, - const int32_t * __restrict__ src6, float * __restrict__ dst, + const float * src0_ptr, const float * src1_ptr, const float * src2_ptr, + const float * src3_ptr, const float * src4_ptr, const float * src5_ptr, + const int32_t * src6_ptr, float * dst_ptr, const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3, const int src2_nb1, const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3, - const int64_t s_off, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok) { + char * s_base, const int64_t s_slot_bytes, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok, const int64_t K) { + const float * GGML_CUDA_RESTRICT src0 = src0_ptr; + const float * GGML_CUDA_RESTRICT src1 = src1_ptr; + const float * GGML_CUDA_RESTRICT src2 = src2_ptr; + const float * GGML_CUDA_RESTRICT src3 = src3_ptr; + const float * GGML_CUDA_RESTRICT src4 = src4_ptr; + const float * GGML_CUDA_RESTRICT src5 = src5_ptr; + const int32_t * GGML_CUDA_RESTRICT src6 = src6_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; const int warp = threadIdx.x / WARP_SIZE; const int lane = threadIdx.x % WARP_SIZE; const int warp_idx = blockIdx.x * c_factor + warp; + ggml_cuda_pdl_sync(); + + // the last block can have unused warps when n_head*d_head is not a multiple of c_factor + if (warp_idx >= n_head * d_head) { + return; + } + const int head_idx = warp_idx / d_head; const int head_off = (warp_idx % d_head) * sizeof(float); const int seq_idx = blockIdx.y; @@ -143,7 +184,7 @@ __global__ void __launch_bounds__(d_state, 1) const float * B_warp = (const float *) ((const char *) src4 + (seq_idx * src4_nb3) + (group_off)); const float * C_warp = (const float *) ((const char *) src5 + (seq_idx * src5_nb3) + (group_off)); float * y_warp = dst + (seq_idx * n_tok * n_head * d_head) + warp_idx; - float * s_warp = (float *) ((char *) dst + s_off + seq_idx * src0_nb3 + head_idx * src0_nb2 + head_off * d_state); + float * s_warp = (float *) (s_base + seq_idx * src0_nb3 + head_idx * src0_nb2 + head_off * d_state); // strides across n_seq_tokens const int stride_x = src1_nb2 / sizeof(float); @@ -182,6 +223,16 @@ __global__ void __launch_bounds__(d_state, 1) if (lane == 0) { y_warp[i * stride_y] = state_sum; } + + // Slot 0 is the final state written below; slots 1..K-1 are rollback snapshots. + const int64_t slot = n_tok - 1 - i; + if (K > 1 && slot > 0 && slot < K) { + float * s_snapshot_warp = (float *) ((char *) s_warp + slot * s_slot_bytes); +#pragma unroll + for (int j = 0; j < c_factor; j++) { + s_snapshot_warp[WARP_SIZE * j + lane] = state[j]; + } + } } // write back the state @@ -197,92 +248,109 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3, const int64_t s_off, const int64_t d_state, const int64_t head_dim, const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq, - cudaStream_t stream) { + const int64_t K, const ggml_cuda_ssm_scan_fused_cache * cache, cudaStream_t stream) { + // when fused, the states go straight into the recurrent cache and the dst tail is left alone + char * const s_base = cache ? (char *) cache->data : (char *) dst + s_off; + const int64_t s_slot_bytes = cache ? cache->slot_stride * (int64_t) sizeof(float) : n_seq * (int64_t) src0_nb3; + // NOTE: if you change conditions here, be sure to update the corresponding supports_op condition! if (src3_nb1 == sizeof(float)) { // Mamba-2 - if (d_state == 128) { + if (d_state == 96) { + constexpr int threads = 96; + constexpr int num_warps = threads/WARP_SIZE; + + const dim3 blocks((n_head * head_dim + (num_warps - 1)) / num_warps, n_seq, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, 0, stream); + ggml_cuda_kernel_launch(ssm_scan_f32_group<96/WARP_SIZE, 96>, launch_params, + src0, src1, src2, src3, src4, src5, src6, dst, + src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, + src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_base, s_slot_bytes, n_head, head_dim, n_group, n_tok, K); + } else if (d_state == 128) { constexpr int threads = 128; constexpr int num_warps = threads/WARP_SIZE; const dim3 blocks((n_head * head_dim + (num_warps - 1)) / num_warps, n_seq, 1); - ssm_scan_f32_group<128/WARP_SIZE, 128><<>>( + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, 0, stream); + ggml_cuda_kernel_launch(ssm_scan_f32_group<128/WARP_SIZE, 128>, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, - src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok); + src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_base, s_slot_bytes, n_head, head_dim, n_group, n_tok, K); } else if (d_state == 256) { // Falcon-H1 constexpr int threads = 256; constexpr int num_warps = threads/WARP_SIZE; const dim3 blocks((n_head * head_dim + (num_warps - 1)) / num_warps, n_seq, 1); - ssm_scan_f32_group<256/WARP_SIZE, 256><<>>( + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, 0, stream); + ggml_cuda_kernel_launch(ssm_scan_f32_group<256/WARP_SIZE, 256>, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, - src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok); + src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_base, s_slot_bytes, n_head, head_dim, n_group, n_tok, K); } else { - GGML_ABORT("doesn't support d_state!=(128 or 256)."); + GGML_ABORT("doesn't support d_state!=(96, 128 or 256)."); } } else { // Mamba-1 + GGML_ASSERT(K == 1); constexpr int threads = 128; GGML_ASSERT(n_head % threads == 0); GGML_ASSERT(head_dim == 1); GGML_ASSERT(n_group == 1); const dim3 blocks(n_seq, (n_head + threads - 1) / threads, 1); - const int smem_size = (threads * (d_state + 1) * 2) * sizeof(float); if (d_state == 16) { + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks, threads, 0, stream); switch (n_tok) { case 1: - ssm_scan_f32<<>>( + ggml_cuda_kernel_launch(ssm_scan_f32, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok); break; case 2: - ssm_scan_f32<<>>( + ggml_cuda_kernel_launch(ssm_scan_f32, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok); break; case 3: - ssm_scan_f32<<>>( + ggml_cuda_kernel_launch(ssm_scan_f32, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok); break; case 4: - ssm_scan_f32<<>>( + ggml_cuda_kernel_launch(ssm_scan_f32, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok); break; case 5: - ssm_scan_f32<<>>( + ggml_cuda_kernel_launch(ssm_scan_f32, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok); break; case 6: - ssm_scan_f32<<>>( + ggml_cuda_kernel_launch(ssm_scan_f32, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok); break; case 7: - ssm_scan_f32<<>>( + ggml_cuda_kernel_launch(ssm_scan_f32, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok); break; case 8: - ssm_scan_f32<<>>( + ggml_cuda_kernel_launch(ssm_scan_f32, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok); break; default: - ssm_scan_f32<<>>( + ggml_cuda_kernel_launch(ssm_scan_f32, launch_params, src0, src1, src2, src3, src4, src5, src6, dst, src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1, src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, n_tok); @@ -294,7 +362,431 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa } } -void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { +#if !defined(GGML_USE_HIP) +// ============================================================================ +// SSD (State Space Duality) kernels for Mamba-2 prefill (n_tok > SSM_SSD_MIN_TOKENS) +// +// Instead of a sequential scan, SSD reformulates the output as: +// Y = (L (.) (C @ B^T)) @ (X * dt) + decay * C @ s_init +// where L is a causal decay mask derived from A and dt. +// +// This converts the O(T*N) sequential scan into parallel matmuls. +// ============================================================================ +// Softplus(dt) and inclusive prefix sum per head using CUB BlockScan. +// Grid: (n_head, n_seqs) +template +__global__ void ssm_ssd_prepare_dt_kernel( + const float * __restrict__ dt_raw, + float * __restrict__ dt_sp_out, + float * __restrict__ cs_out, + const int n_head, const int n_tok, + const int dt_stride_tok, // elements between tokens in dt + const int dt_stride_seq) { // elements between sequences in dt + + const int h = blockIdx.x; + const int s = blockIdx.y; + + const float * dt_seq = dt_raw + s * dt_stride_seq; + + float * dt_sp_seq = dt_sp_out + s * n_tok * n_head; + float * cs_seq = cs_out + s * n_tok * n_head; + + const int items_per_thread = (n_tok + BLOCK_SIZE - 1) / BLOCK_SIZE; + + // Phase 1: softplus with interleaved distribution (t = i*BLOCK_SIZE + threadIdx.x). + // Each warp reads BLOCK_SIZE consecutive tokens, giving coalesced dt_raw loads + // (stride n_head between threads vs. items_per_thread*n_head in blocked layout). + float local_vals[MAX_ITEMS]; + for (int i = 0; i < items_per_thread; i++) { + const int t = i * BLOCK_SIZE + threadIdx.x; + if (t < n_tok) { + float val = dt_seq[h + t * dt_stride_tok]; + float sp = (val <= 20.0f) ? log1pf(expf(val)) : val; + local_vals[i] = sp; + dt_sp_seq[t * n_head + h] = sp; + } else { + local_vals[i] = 0.0f; + } + } + + // Phase 2+3: per-step inclusive scan to build cs[] in token order. + // With interleaved distribution the per-thread total scan would not give token-order + // prefix sums, so we scan one BLOCK_SIZE slab at a time and carry a running total. +#ifdef USE_CUB + using BlockScan = cub::BlockScan; + __shared__ typename BlockScan::TempStorage scan_temp; + __shared__ float step_total; + + float running = 0.0f; + for (int i = 0; i < items_per_thread; i++) { + float inclusive; + BlockScan(scan_temp).InclusiveSum(local_vals[i], inclusive); + const int t = i * BLOCK_SIZE + threadIdx.x; + if (t < n_tok) { + cs_seq[t * n_head + h] = running + inclusive; + } + if (threadIdx.x == BLOCK_SIZE - 1) { + step_total = inclusive; + } + __syncthreads(); + running += step_total; + } +#else + // Fallback: sequential prefix scan in shared memory, one slab at a time. + __shared__ float sdata[BLOCK_SIZE]; + float running = 0.0f; + for (int i = 0; i < items_per_thread; i++) { + const int t = i * BLOCK_SIZE + threadIdx.x; + sdata[threadIdx.x] = local_vals[i]; + __syncthreads(); + if (threadIdx.x == 0) { + for (int j = 1; j < BLOCK_SIZE; j++) { + sdata[j] += sdata[j - 1]; + } + } + __syncthreads(); + if (t < n_tok) { + cs_seq[t * n_head + h] = running + sdata[threadIdx.x]; + } + running += sdata[BLOCK_SIZE - 1]; + __syncthreads(); + } +#endif +} + +// Prepare SSD matmul inputs for one chunk: X_dt, B_weighted, C_scaled. +// T_matmul controls precision for X_dt, B_weighted (float or half). +// C_scaled is always float (pairs with float s_cur in step 3c). +// Computation is always FP32; only the final store converts to T_matmul. +// Also materializes the causal M matrix = exp(A*(cs_out - cs_in)) * CB (fused with prep to save a launch). +// Grid: (ceil(max(C*head_dim, d_state*C, chunk_len^2) / BLOCK), n_head, n_seqs) +template +__global__ void ssm_ssd_pre_matmul_kernel( + const float * __restrict__ cs, // {n_tok, n_head} cumulative dt sums + const float * __restrict__ dt_sp, // {n_tok, n_head} softplus(dt) + const float * __restrict__ A, // {1, n_head} + const float * __restrict__ x, // {head_dim, n_head, n_tok, n_seqs} + const float * __restrict__ B, // {d_state, n_group, n_tok, n_seqs} + const float * __restrict__ C_src, // {d_state, n_group, n_tok, n_seqs} + T_matmul * __restrict__ X_dt, // {head_dim, C, n_head} x * dt, d-fastest + T_matmul * __restrict__ B_weighted, // {d_state, C, n_head} B * decay_from_end + float * __restrict__ C_scaled, // {d_state, C, n_head} C * decay_to_pos (always float) + const float * __restrict__ CB, // {chunk_len, chunk_len, n_group, n_seqs} + half * __restrict__ M_out, // {chunk_len, chunk_len, n_head, n_seqs} + const int chunk_len, const int head_dim, const int n_head, const int n_group, + const int d_state, const int A_stride, + const int x_stride_tok, const int x_stride_seq, + const int B_stride_tok, const int B_stride_seq, + const int C_stride_tok, const int C_stride_seq, + const int chunk_offset, + const int n_tok_total) { + + const int h = blockIdx.y; + const int s = blockIdx.z; + const int g = h / (n_head / n_group); + + const float A_h = A[h * A_stride]; + const int idx = blockIdx.x * BLOCK_SIZE + threadIdx.x; + + const int cs_seq_off = s * n_tok_total * n_head; + const float cs_base = (chunk_offset > 0) ? cs[cs_seq_off + (chunk_offset - 1) * n_head + h] : 0.0f; + const float cs_last = cs[cs_seq_off + (chunk_offset + chunk_len - 1) * n_head + h] - cs_base; + + // Prepare X_dt = x * dt, stored d-fastest for coalesced reads and writes. + const int n_xdt = chunk_len * head_dim; + if (idx < n_xdt) { + const int d = idx % head_dim; + const int t = idx / head_dim; + + const float x_val = x[s * x_stride_seq + (chunk_offset + t) * x_stride_tok + d + h * head_dim]; + const float dt_val = dt_sp[cs_seq_off + (chunk_offset + t) * n_head + h]; + + X_dt[d + t * head_dim + h * n_xdt + s * n_xdt * n_head] = (T_matmul)(x_val * dt_val); + } + + // Prepare B_weighted and C_scaled together: both share the same index space (d_state * chunk_len) + // and the same cs_t load, so merging halves the cs[] global memory traffic. + const int n_bw = d_state * chunk_len; + if (idx < n_bw) { + const int n = idx % d_state; + const int t = idx / d_state; + + const float cs_t = cs[cs_seq_off + (chunk_offset + t) * n_head + h] - cs_base; + + const float B_val = B[s * B_stride_seq + (chunk_offset + t) * B_stride_tok + g * d_state + n]; + B_weighted[n + t * d_state + h * n_bw + s * n_bw * n_head] = (T_matmul)(B_val * __expf(A_h * (cs_last - cs_t))); + + const float C_val = C_src[s * C_stride_seq + (chunk_offset + t) * C_stride_tok + g * d_state + n]; + C_scaled[n + t * d_state + h * n_bw + s * n_bw * n_head] = C_val * __expf(A_h * cs_t); + } + + // Materialize M = exp(A*(cs_out - cs_in)) * CB with causal mask. + const int n_M = chunk_len * chunk_len; + if (idx < n_M) { + const int t_out = idx % chunk_len; + const int t_in = idx / chunk_len; + + half val; + if (t_in <= t_out) { + const float cs_out = cs[cs_seq_off + (chunk_offset + t_out) * n_head + h] - cs_base; + const float cs_in = cs[cs_seq_off + (chunk_offset + t_in) * n_head + h] - cs_base; + const float decay = __expf(A_h * (cs_out - cs_in)); + const float * CB_g = CB + (int64_t)s * chunk_len * chunk_len * n_group + + (int64_t)g * chunk_len * chunk_len; + const float cb_val = CB_g[t_out + t_in * chunk_len]; + val = __float2half(decay * cb_val); + } else { + val = __float2half(0.0f); + } + + M_out[(int64_t)s * n_M * n_head + (int64_t)h * n_M + t_in * chunk_len + t_out] = val; + } +} + +// Scale running state in-place: s_cur *= decay_total(chunk). +// Called BEFORE cuBLAS state update (beta=1) to fuse inter-chunk decay. +// Eliminates the s_old buffer and D2D memcpy vs the old approach of: +// memcpy(s_old, s_cur) -> cuBLAS(beta=0) -> s_cur += decay * s_old +// Grid: (ceil(d_state * head_dim / BLOCK), n_head, n_seqs) +template +__global__ void ssm_ssd_scale_state_kernel( + float * __restrict__ s_cur, // {d_state, head_dim, n_head, n_seqs} + const float * __restrict__ cs, // {n_tok, n_head} cumulative dt sums + const float * __restrict__ A, // {1, n_head} + const int d_state, const int head_dim, const int n_head, + const int chunk_offset, const int chunk_len, + const int n_tok_total, const int A_stride) { + + const int h = blockIdx.y; + const int s = blockIdx.z; + const int idx = blockIdx.x * BLOCK_SIZE + threadIdx.x; + const int state_per_head = d_state * head_dim; + if (idx >= state_per_head) return; + + const float A_h = A[h * A_stride]; + const int cs_seq_off = s * n_tok_total * n_head; + const float cs_base = (chunk_offset > 0) ? cs[cs_seq_off + (chunk_offset - 1) * n_head + h] : 0.0f; + const float cs_last = cs[cs_seq_off + (chunk_offset + chunk_len - 1) * n_head + h] - cs_base; + const float decay_total = __expf(A_h * cs_last); + + const int off = s * state_per_head * n_head + h * state_per_head + idx; + s_cur[off] *= decay_total; +} + +// Copy initial state from src0[ids[s]] into s_cur for each sequence. +// src0 and s_cur can alias when the state is written straight into the cache. +// Grid: (ceil(d_state * head_dim * n_head / BLOCK), n_seqs) +template +__global__ void ssm_ssd_init_state_kernel( + const float * src0, // {d_state, head_dim, n_head, n_rs} + const int32_t * __restrict__ ids, // {n_seqs} + float * s_cur, // {d_state, head_dim, n_head, n_seqs} + const int state_size, // d_state * head_dim * n_head + const int64_t s0_stride_seq) { // elements between state rows + const int s = blockIdx.y; + const int idx = blockIdx.x * BLOCK_SIZE + threadIdx.x; + if (idx >= state_size) return; + + const float * s_src = src0 + (int64_t)ids[s] * s0_stride_seq; + s_cur[s * state_size + idx] = s_src[idx]; +} + +// SSD (State Space Duality) dispatch for Mamba-2 prefill. +// Chunked matmuls: CB, materialize M + cuBLAS Y, S@C, B@X_dt. +// All strides are in elements (floats), not bytes. +static void ssm_scan_ssd_f32_cuda( + ggml_backend_cuda_context & ctx, + const float * src0_d, const float * src1_d, const float * src2_d, const float * src3_d, + const float * src4_d, const float * src5_d, const int32_t * src6_d, float * dst_d, + const int64_t s0_stride_seq, // state (src0) stride between seqs + const int x_stride_tok, const int x_stride_seq, // x (src1) strides + const int dt_stride_tok, const int dt_stride_seq, // dt (src2) strides + const int A_stride, // A (src3) stride between heads + const int B_stride_tok, const int B_stride_seq, // B (src4) strides + const int C_stride_tok, const int C_stride_seq, // C (src5) strides + float * s_cur, // state: dst state tail, or the cache when fused + const int64_t d_state, const int64_t head_dim, + const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq) { + + cudaStream_t stream = ctx.stream(); + const int64_t d_inner = head_dim * n_head; + + const int64_t chunk_size = SSM_SSD_CHUNK_SIZE; + const int64_t n_chunks = (n_tok + chunk_size - 1) / chunk_size; + + const int64_t state_per_head = d_state * head_dim; + + using matmul_t = half; + static constexpr cudaDataType_t matmul_dtype = CUDA_R_16F; + + ggml_cuda_pool_alloc dt_sp_buf(ctx.pool(), n_tok * n_head * n_seq); + ggml_cuda_pool_alloc cs_buf(ctx.pool(), n_tok * n_head * n_seq); + ggml_cuda_pool_alloc CB_buf(ctx.pool(), chunk_size * chunk_size * n_group * n_seq); + ggml_cuda_pool_alloc X_dt_buf(ctx.pool(), chunk_size * head_dim * n_head * n_seq); + ggml_cuda_pool_alloc B_w_buf(ctx.pool(), d_state * chunk_size * n_head * n_seq); + ggml_cuda_pool_alloc C_s_buf(ctx.pool(), d_state * chunk_size * n_head * n_seq); + float * dt_sp = dt_sp_buf.get(); + float * cs = cs_buf.get(); + float * CB = CB_buf.get(); + matmul_t * X_dt = X_dt_buf.get(); + matmul_t * B_weighted = B_w_buf.get(); + float * C_scaled = C_s_buf.get(); + + // Step 1: softplus(dt) and parallel prefix sum over full sequence + { + dim3 grid(n_head, n_seq); + ssm_ssd_prepare_dt_kernel<<>>( + src2_d, dt_sp, cs, n_head, n_tok, dt_stride_tok, dt_stride_seq); + CUDA_CHECK(cudaGetLastError()); + } + + // Step 2: initialize running state from src0[ids[s]] + { + constexpr int BLOCK = 256; + const int64_t state_size = d_state * head_dim * n_head; + dim3 grid((state_size + BLOCK - 1) / BLOCK, n_seq); + ssm_ssd_init_state_kernel<<>>( + src0_d, src6_d, s_cur, state_size, s0_stride_seq); + CUDA_CHECK(cudaGetLastError()); + } + + // Step 3: chunked SSD loop + // Per chunk: pre_matmul (incl. M) + 4 cuBLAS (CB, Y, S@C, state update) + scale_state + cublasHandle_t handle = ctx.cublas_handle(); + const float alpha_one = 1.0f; + const float beta_zero = 0.0f; + const float beta_one = 1.0f; + const int lda_C_src = C_stride_tok; // leading dim for C in CB = C^T @ B + const int ldb_B_src = B_stride_tok; // leading dim for B in CB = C^T @ B + + // Scratch buffer for causal M matrix, reused across chunks (max size at chunk_size) + const int64_t n_M_max = chunk_size * chunk_size; + ggml_cuda_pool_alloc M_buf(ctx.pool(), n_M_max * n_head * n_seq); + half * M_mat = M_buf.get(); + + for (int64_t k = 0; k < n_chunks; k++) { + const int64_t chunk_offset = k * chunk_size; + const int64_t chunk_len = (chunk_offset + chunk_size <= n_tok) ? chunk_size : (n_tok - chunk_offset); + + // 3a: CB = C^T @ B per group + for (int64_t s = 0; s < n_seq; s++) { + const float * C_s = src5_d + s * C_stride_seq + chunk_offset * C_stride_tok; + const float * B_s = src4_d + s * B_stride_seq + chunk_offset * B_stride_tok; + float * CB_s = CB + s * chunk_len * chunk_len * n_group; + + if (n_group == 1) { + CUBLAS_CHECK(cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_N, + chunk_len, chunk_len, d_state, + &alpha_one, C_s, lda_C_src, B_s, ldb_B_src, + &beta_zero, CB_s, (int)chunk_len)); + } else { + CUBLAS_CHECK(cublasGemmStridedBatchedEx(handle, CUBLAS_OP_T, CUBLAS_OP_N, + chunk_len, chunk_len, d_state, + &alpha_one, + C_s, CUDA_R_32F, lda_C_src, d_state, + B_s, CUDA_R_32F, ldb_B_src, d_state, + &beta_zero, + CB_s, CUDA_R_32F, (int)chunk_len, (long long)(chunk_len * chunk_len), + n_group, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } + } + + // 3b: prepare X_dt, B_weighted, C_scaled + materialize causal M matrix + const int64_t n_M = chunk_len * chunk_len; + { + constexpr int BLOCK = 256; + const int64_t n_xdt = chunk_len * head_dim; + const int64_t n_bw = d_state * chunk_len; + int64_t max_work = n_xdt; + if (n_bw > max_work) max_work = n_bw; + if (n_M > max_work) max_work = n_M; + dim3 grid((max_work + BLOCK - 1) / BLOCK, n_head, n_seq); + ssm_ssd_pre_matmul_kernel<<>>( + cs, dt_sp, src3_d, src1_d, src4_d, src5_d, + X_dt, B_weighted, C_scaled, + CB, M_mat, + chunk_len, head_dim, n_head, n_group, d_state, A_stride, + x_stride_tok, x_stride_seq, B_stride_tok, B_stride_seq, C_stride_tok, C_stride_seq, + chunk_offset, n_tok); + CUDA_CHECK(cudaGetLastError()); + } + + // 3c: dst = S_cur^T @ C_scaled (state contribution) + { + const int64_t stride_S = state_per_head; + const int64_t stride_Cs = d_state * chunk_len; + + for (int64_t s = 0; s < n_seq; s++) { + float * dst_chunk = dst_d + s * d_inner * n_tok + chunk_offset * d_inner; + + CUBLAS_CHECK(cublasGemmStridedBatchedEx(handle, CUBLAS_OP_T, CUBLAS_OP_N, + head_dim, chunk_len, d_state, + &alpha_one, + s_cur + s * stride_S * n_head, CUDA_R_32F, d_state, stride_S, + C_scaled + s * stride_Cs * n_head, CUDA_R_32F, d_state, stride_Cs, + &beta_zero, + dst_chunk, CUDA_R_32F, d_inner, head_dim, + n_head, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } + } + + // 3d: dst += X_dt @ M^T (intra-chunk contribution, adds to 3c result) + // M is stored as M[t_out, t_in] (lower-triangular), transpose needed for Y = X @ M^T. + { + const int64_t stride_M = n_M; + const int64_t stride_X_h = (int64_t)chunk_len * head_dim; + + for (int64_t s = 0; s < n_seq; s++) { + float * dst_chunk = dst_d + s * d_inner * n_tok + chunk_offset * d_inner; + CUBLAS_CHECK(cublasGemmStridedBatchedEx(handle, CUBLAS_OP_N, CUBLAS_OP_T, + head_dim, chunk_len, chunk_len, + &alpha_one, + X_dt + s * stride_X_h * n_head, matmul_dtype, head_dim, stride_X_h, + M_mat + s * stride_M * n_head, matmul_dtype, chunk_len, stride_M, + &beta_one, + dst_chunk, CUDA_R_32F, d_inner, head_dim, + n_head, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } + } + + // 3e: s_cur = B_weighted @ X_dt^T + decay_total * s_cur_old (state update) + { + // Scale s_cur in-place by per-head decay_total BEFORE cuBLAS overwrites it + constexpr int BLOCK = 256; + dim3 grid((state_per_head + BLOCK - 1) / BLOCK, n_head, n_seq); + ssm_ssd_scale_state_kernel<<>>( + s_cur, cs, src3_d, + d_state, head_dim, n_head, + chunk_offset, chunk_len, n_tok, A_stride); + CUDA_CHECK(cudaGetLastError()); + + // cuBLAS with beta=1: s_cur = B_weighted @ X_dt^T + 1.0 * s_cur (already scaled) + const int64_t stride_Bw = d_state * chunk_len; + const int64_t stride_X = chunk_len * head_dim; + const int64_t stride_S = state_per_head; + + for (int64_t s = 0; s < n_seq; s++) { + // X_dt is d-fastest {hd, C}, read as OP_T to get {C, hd} + CUBLAS_CHECK(cublasGemmStridedBatchedEx(handle, CUBLAS_OP_N, CUBLAS_OP_T, + d_state, head_dim, chunk_len, + &alpha_one, + B_weighted + s * stride_Bw * n_head, matmul_dtype, d_state, stride_Bw, + X_dt + s * stride_X * n_head, matmul_dtype, head_dim, stride_X, + &beta_one, + s_cur + s * stride_S * n_head, CUDA_R_32F, d_state, stride_S, + n_head, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } + } + } +} +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +static void ggml_cuda_op_ssm_scan_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + const ggml_cuda_ssm_scan_fused_cache * cache) { const struct ggml_tensor * src0 = dst->src[0]; // s const struct ggml_tensor * src1 = dst->src[1]; // x const struct ggml_tensor * src2 = dst->src[2]; // dt @@ -309,10 +801,12 @@ void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int64_t ng = src4->ne[1]; // n_group const int64_t n_t = src1->ne[2]; // number of tokens per sequence const int64_t n_s = src1->ne[3]; // number of sequences in the batch + const int32_t K_param = ggml_get_op_params_i32(dst, 0); + const int64_t K = K_param > 0 ? K_param : 1; const int64_t s_off = ggml_nelements(src1) * sizeof(float); - GGML_ASSERT(ggml_nelements(src1) + nc*nr*nh*n_s == ggml_nelements(dst)); + GGML_ASSERT(ggml_nelements(src1) + K*nc*nr*nh*n_s == ggml_nelements(dst)); GGML_ASSERT(src0->nb[0] == sizeof(float)); GGML_ASSERT(src1->nb[0] == sizeof(float)); GGML_ASSERT(src2->nb[0] == sizeof(float)); @@ -320,6 +814,7 @@ void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src4->nb[0] == sizeof(float)); GGML_ASSERT(src5->nb[0] == sizeof(float)); GGML_ASSERT(src6->nb[0] == sizeof(int32_t)); + GGML_ASSERT(src3->ne[0] == 1 || K == 1); const float * src0_d = (const float *) src0->data; const float * src1_d = (const float *) src1->data; @@ -335,8 +830,61 @@ void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src6->type == GGML_TYPE_I32); GGML_ASSERT(dst->type == GGML_TYPE_F32); + // Byte strides are narrowed to int for both scan and SSD paths. + GGML_ASSERT(src0->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src0->nb[3] <= (size_t)INT_MAX); + GGML_ASSERT(src1->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src1->nb[3] <= (size_t)INT_MAX); + GGML_ASSERT(src2->nb[1] <= (size_t)INT_MAX); + GGML_ASSERT(src2->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src3->nb[1] <= (size_t)INT_MAX); + GGML_ASSERT(src4->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src4->nb[3] <= (size_t)INT_MAX); + GGML_ASSERT(src5->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src5->nb[3] <= (size_t)INT_MAX); + +#if !defined(GGML_USE_HIP) + // Mamba-2 with scalar A per head: use SSD matmul path for long sequences. + // Requires NVIDIA Turing+ otherwise fallback to scan. + const bool is_mamba2 = (src3->nb[1] == sizeof(float)); + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const bool use_ssd = is_mamba2 && n_t > SSM_SSD_MIN_TOKENS + && K == 1 + && n_t <= SSM_SSD_MAX_TOKENS + && GGML_CUDA_CC_IS_NVIDIA(cc) + && cc >= GGML_CUDA_CC_TURING + && nr % 8 == 0; // cuBLAS requires 8-element (16-byte) alignment + + if (use_ssd) { + // ssm_ssd_init_state_kernel uses flat linear indexing within each sequence, + // so src0 must be fully contiguous across all inner dimensions. + // The scan path handles non-contiguous nb[2] via src0_nb2 but does not handle nb[1]. + GGML_ASSERT(src0->nb[1] == nc * sizeof(float)); + GGML_ASSERT(src0->nb[2] == nc * nr * sizeof(float)); + + ssm_scan_ssd_f32_cuda(ctx, + src0_d, src1_d, src2_d, src3_d, src4_d, src5_d, src6_d, dst_d, + (int64_t)(src0->nb[3] / sizeof(float)), + (int)(src1->nb[2] / sizeof(float)), (int)(src1->nb[3] / sizeof(float)), + (int)(src2->nb[1] / sizeof(float)), (int)(src2->nb[2] / sizeof(float)), + (int)(src3->nb[1] / sizeof(float)), + (int)(src4->nb[2] / sizeof(float)), (int)(src4->nb[3] / sizeof(float)), + (int)(src5->nb[2] / sizeof(float)), (int)(src5->nb[3] / sizeof(float)), + cache ? cache->data : (float *) ((char *) dst_d + s_off), nc, nr, nh, ng, n_t, n_s); + return; + } +#endif ssm_scan_f32_cuda(src0_d, src1_d, src2_d, src3_d, src4_d, src5_d, src6_d, dst_d, src0->nb[2], src0->nb[3], src1->nb[2], src1->nb[3], src2->nb[1], src2->nb[2], src3->nb[1], src4->nb[2], src4->nb[3], src5->nb[2], src5->nb[3], - s_off, nc, nr, nh, ng, n_t, n_s, stream); + s_off, nc, nr, nh, ng, n_t, n_s, K, cache, stream); +} + +void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + ggml_cuda_op_ssm_scan_impl(ctx, dst, nullptr); +} + +void ggml_cuda_op_ssm_scan_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + ggml_cuda_ssm_scan_fused_cache cache) { + ggml_cuda_op_ssm_scan_impl(ctx, dst, &cache); } diff --git a/ggml/src/ggml-cuda/ssm-scan.cuh b/ggml/src/ggml-cuda/ssm-scan.cuh index ee078f5..407a4c7 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cuh +++ b/ggml/src/ggml-cuda/ssm-scan.cuh @@ -1,3 +1,13 @@ #include "common.cuh" +// fused-kernel recurrent-state output; strides in elements (per-seq stride is always the state row size, set in-kernel) +struct ggml_cuda_ssm_scan_fused_cache { + float * data; // rollback slot 0 + int64_t slot_stride; // between rollback slots +}; + void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// same op, but writes the state snapshot(s) into the cache instead of dst (see ggml_cuda_try_ssm_scan_cache_fusion) +void ggml_cuda_op_ssm_scan_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst, + ggml_cuda_ssm_scan_fused_cache cache); diff --git a/ggml/src/ggml-cuda/sumrows.cu b/ggml/src/ggml-cuda/sumrows.cu index 4025771..aa8342b 100644 --- a/ggml/src/ggml-cuda/sumrows.cu +++ b/ggml/src/ggml-cuda/sumrows.cu @@ -7,10 +7,12 @@ void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int const dim3 block_nums(nrows, 1, 1); if ((nrows / nsm) < 2) { const dim3 block_dims(512, 1, 1); - reduce_rows_f32<<>>(x, dst, ncols); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + ggml_cuda_kernel_launch(reduce_rows_f32, launch_params, x, dst, ncols); } else { const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1); - reduce_rows_f32<<>>(x, dst, ncols); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + ggml_cuda_kernel_launch(reduce_rows_f32, launch_params, x, dst, ncols); } } @@ -22,22 +24,30 @@ void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous_rows(src0)); const int64_t ncols = src0->ne[0]; const int64_t nrows = ggml_nrows(src0); + if (ggml_is_contiguous(src0)) { + sum_rows_f32_cuda(src0_d, dst_d, ncols, nrows, stream); + return; + } + const dim3 block_nums(nrows, 1, 1); const int id = ggml_cuda_get_device(); const int nsm = ggml_cuda_info().devices[id].nsm; + dim3 block_dims; if ((nrows / nsm) < 2) { // Increase num threads to 512 for small nrows to better hide the latency - const dim3 block_dims(512, 1, 1); - reduce_rows_f32<<>>(src0_d, dst_d, ncols); + block_dims = dim3(512, 1, 1); } else { // Enough active SMs to hide latency, use smaller blocks to allow better scheduling - const dim3 block_dims(ncols < 1024 ? 32 : 128, 1, 1); - reduce_rows_f32<<>>(src0_d, dst_d, ncols); + block_dims = dim3(ncols < 1024 ? 32 : 128, 1, 1); } + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + const char * src0_d_bytes = (const char *) src0->data; + ggml_cuda_kernel_launch(reduce_rows_f32_strided, launch_params, src0_d_bytes, dst_d, ncols, + src0->ne[1], src0->ne[2], src0->nb[1], src0->nb[2], src0->nb[3]); } diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu index fb26abe..b2661b9 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu @@ -2,4 +2,5 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(192, 128, 1, 16); DECL_FATTN_MMA_F16_CASE(576, 512, 1, 16); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu index 1f554d8..8fc3b17 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu @@ -2,4 +2,5 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(320, 256, 1, 32); DECL_FATTN_MMA_F16_CASE(576, 512, 1, 32); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu index 22d3831..6ae77be 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu @@ -7,5 +7,6 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 1, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 1, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 1, 8); DECL_FATTN_MMA_F16_CASE(128, 128, 1, 8); +DECL_FATTN_MMA_F16_CASE(192, 128, 1, 8); DECL_FATTN_MMA_F16_CASE(256, 256, 1, 8); DECL_FATTN_MMA_F16_CASE(512, 512, 1, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu index 2e1883a..0c1a37f 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 16, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 16, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 16, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 16, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 16, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu index f011a20..fd41e71 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu @@ -2,4 +2,5 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(192, 128, 2, 16); DECL_FATTN_MMA_F16_CASE(576, 512, 2, 16); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu index 264751d..abd2b21 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu @@ -2,4 +2,5 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(320, 256, 2, 32); DECL_FATTN_MMA_F16_CASE(576, 512, 2, 32); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu index 84b674c..9f4bef1 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu @@ -7,5 +7,6 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 2, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 2, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 2, 8); DECL_FATTN_MMA_F16_CASE(128, 128, 2, 8); +DECL_FATTN_MMA_F16_CASE(192, 128, 2, 8); DECL_FATTN_MMA_F16_CASE(256, 256, 2, 8); DECL_FATTN_MMA_F16_CASE(512, 512, 2, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu index 407b6cf..d5a4d16 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 32, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 32, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 32, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 32, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 32, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu index f5fd0e2..cc41fa5 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu @@ -2,4 +2,5 @@ #include "../fattn-mma-f16.cuh" +DECL_FATTN_MMA_F16_CASE(192, 128, 4, 16); DECL_FATTN_MMA_F16_CASE(576, 512, 4, 16); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu index 5e46685..d8e7588 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 4, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 4, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 4, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 4, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu index 5906398..859bea5 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu @@ -7,5 +7,6 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 4, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 4, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 4, 8); DECL_FATTN_MMA_F16_CASE(128, 128, 4, 8); +DECL_FATTN_MMA_F16_CASE(192, 128, 4, 8); DECL_FATTN_MMA_F16_CASE(256, 256, 4, 8); DECL_FATTN_MMA_F16_CASE(512, 512, 4, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu index a6e6f09..e48e28c 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu @@ -8,3 +8,4 @@ DECL_FATTN_MMA_F16_CASE(96, 96, 8, 2); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 2); DECL_FATTN_MMA_F16_CASE(128, 128, 8, 2); DECL_FATTN_MMA_F16_CASE(256, 256, 8, 2); +DECL_FATTN_MMA_F16_CASE(512, 512, 8, 2); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu index 4bc60d6..c975ce6 100644 --- a/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +++ b/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu @@ -7,5 +7,6 @@ DECL_FATTN_MMA_F16_CASE(80, 80, 8, 8); DECL_FATTN_MMA_F16_CASE(96, 96, 8, 8); DECL_FATTN_MMA_F16_CASE(112, 112, 8, 8); DECL_FATTN_MMA_F16_CASE(128, 128, 8, 8); +DECL_FATTN_MMA_F16_CASE(192, 128, 8, 8); DECL_FATTN_MMA_F16_CASE(256, 256, 8, 8); DECL_FATTN_MMA_F16_CASE(512, 512, 8, 8); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu b/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu new file mode 100644 index 0000000..b571cca --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu @@ -0,0 +1,5 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-tile.cuh" + +DECL_FATTN_TILE_CASE(192, 128); diff --git a/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu b/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu new file mode 100644 index 0000000..c91f508 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu @@ -0,0 +1,5 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../fattn-tile.cuh" + +DECL_FATTN_TILE_CASE(320, 256); diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index 841059c..7be409b 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -3,7 +3,10 @@ from glob import glob import os -HEAD_SIZES_KQ = [40, 64, 72, 80, 96, 112, 128, 256, 512, 576] +HEAD_SIZES_KQ = [40, 64, 72, 80, 96, 112, 128, 192, 256, 320, 512, 576] + +# DKQ -> DV override for asymmetric head dims. +HEAD_SIZES_V_OVERRIDE = {576: 512, 320: 256, 192: 128} TYPES_KV = ["GGML_TYPE_F16", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_BF16"] @@ -33,6 +36,7 @@ TYPES_MMQ = [ "GGML_TYPE_Q1_0", + "GGML_TYPE_Q2_0", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_Q2_K", "GGML_TYPE_Q3_K", "GGML_TYPE_Q4_K", "GGML_TYPE_Q5_K", "GGML_TYPE_Q6_K", "GGML_TYPE_IQ2_XXS", "GGML_TYPE_IQ2_XS", "GGML_TYPE_IQ2_S", "GGML_TYPE_IQ3_XXS", "GGML_TYPE_IQ3_S", @@ -46,6 +50,12 @@ DECL_MMQ_CASE({type}); """ +TYPES_MMQ_W4A4 = ["GGML_TYPE_MXFP4", "GGML_TYPE_NVFP4"] + +SOURCE_MMQ_W4A4 = """ +DECL_MMQ_CASE_W4A4({type}); +""" + SOURCE_MMF = """// This file has been autogenerated by generate_cu_files.py, do not edit manually. #include "../mmf.cuh" @@ -62,7 +72,7 @@ def get_short_name(long_quant_name): os.remove(filename) for head_size_kq in HEAD_SIZES_KQ: - head_size_v = head_size_kq if head_size_kq != 576 else 512 + head_size_v = HEAD_SIZES_V_OVERRIDE.get(head_size_kq, head_size_kq) with open(f"fattn-tile-instance-dkq{head_size_kq}-dv{head_size_v}.cu", "w") as f: f.write(SOURCE_FATTN_TILE.format(head_size_kq=head_size_kq, head_size_v=head_size_v)) @@ -84,18 +94,25 @@ def get_short_name(long_quant_name): continue if head_size_kq == 72: continue - if head_size_kq == 512 and ncols2 not in (4, 8): + # Skip compilation of unused ncols2 values for niche head sizes: + if head_size_kq == 192 and ncols2 not in (8, 16): # MiMo-V2.5 + continue + if head_size_kq == 320 and ncols2 != 32: # Mistral Small 4 + continue + if head_size_kq == 512 and ncols2 not in (2, 4, 8): # Gemma 4 (+ MTP) continue - if head_size_kq != 576 and ncols2 in (16, 32): + if head_size_kq == 576 and ncols2 not in (4, 16, 32): # Deepseek, GLM 4.7 Flash continue - if head_size_kq == 576 and ncols2 not in (4, 16, 32): + if head_size_kq not in (192, 320, 576) and ncols2 in (16, 32): continue - head_size_v = head_size_kq if head_size_kq != 576 else 512 + head_size_v = HEAD_SIZES_V_OVERRIDE.get(head_size_kq, head_size_kq) f.write(SOURCE_FATTN_MMA_CASE.format(ncols1=ncols1, ncols2=ncols2, head_size_kq=head_size_kq, head_size_v=head_size_v)) for type in TYPES_MMQ: with open(f"mmq-instance-{get_short_name(type)}.cu", "w") as f: f.write(SOURCE_MMQ.format(type=type)) + if type in TYPES_MMQ_W4A4: + f.write(SOURCE_MMQ_W4A4.format(type=type)) for type in range(1, 17): with open(f"mmf-instance-ncols_{type}.cu", "w") as f: diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu index c14624c..0c1c7f2 100644 --- a/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu @@ -3,3 +3,5 @@ #include "../mmq.cuh" DECL_MMQ_CASE(GGML_TYPE_MXFP4); + +DECL_MMQ_CASE_W4A4(GGML_TYPE_MXFP4); diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu index 2cb140d..bed8c8e 100644 --- a/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu @@ -3,3 +3,5 @@ #include "../mmq.cuh" DECL_MMQ_CASE(GGML_TYPE_NVFP4); + +DECL_MMQ_CASE_W4A4(GGML_TYPE_NVFP4); diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu new file mode 100644 index 0000000..750180e --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu @@ -0,0 +1,5 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../mmq.cuh" + +DECL_MMQ_CASE(GGML_TYPE_Q2_0); diff --git a/ggml/src/ggml-cuda/top-k-mask.cu b/ggml/src/ggml-cuda/top-k-mask.cu new file mode 100644 index 0000000..9ff9952 --- /dev/null +++ b/ggml/src/ggml-cuda/top-k-mask.cu @@ -0,0 +1,141 @@ +#include "top-k-mask.cuh" +#include "convert.cuh" + +#define CUDA_TOP_K_MASK_BLOCK_SIZE 512 + +// one block per row: fill the row with -inf, then copy the mask cells of the live picks and the extra cells +template +static __global__ void top_k_mask_kernel( + const T * mask, const int32_t * top_k, const float * score, const int32_t * blocks, const int32_t * extra, T * dst, + const int64_t n_kv, const int64_t ne1, const int64_t ne2, + const int64_t m1, const int64_t m2, const int64_t m3, + const int64_t n_top, const int64_t n_pick, const int64_t n_blk, const int64_t n_extra, + const int64_t sm1, const int64_t sm2, const int64_t sm3, + const int64_t sd1, const int64_t sd2, const int64_t sd3, + const int64_t sk0, const int64_t sk1, const int64_t sk2, const int64_t sk3, + const int64_t ss0, const int64_t ss1, const int64_t ss2, const int64_t ss3, + const int64_t sb0, const int64_t sb1, + const int64_t se0, const int64_t se1, const int64_t se2, const int64_t se3, + const bool vec) { + ggml_cuda_pdl_lc(); + + const int64_t ir = blockIdx.x; + const int64_t i1 = ir % ne1; + const int64_t i2 = (ir / ne1) % ne2; + const int64_t i3 = ir / (ne1*ne2); + + const T * mask_row = mask + (i1 % m1)*sm1 + (i2 % m2)*sm2 + (i3 % m3)*sm3; + T * dst_row = dst + i1*sd1 + i2*sd2 + i3*sd3; + const int32_t * top_k_row = top_k + i1*sk1 + i2*sk2 + i3*sk3; + + ggml_cuda_pdl_sync(); + + const T ninf = ggml_cuda_cast(-INFINITY); + if (vec) { + constexpr int nv = sizeof(int4)/sizeof(T); + T fill[nv]; +#pragma unroll + for (int j = 0; j < nv; ++j) { + fill[j] = ninf; + } + const int4 fill4 = *(const int4 *) fill; + for (int64_t i = threadIdx.x; i < n_kv/nv; i += blockDim.x) { + ((int4 *) dst_row)[i] = fill4; + } + } else { + for (int64_t i = threadIdx.x; i < n_kv; i += blockDim.x) { + dst_row[i] = ninf; + } + } + + __syncthreads(); + + // one thread per pick or extra cell; cells kept twice get the same value + for (int64_t i = threadIdx.x; i < n_top + n_extra; i += blockDim.x) { + if (i >= n_top) { + const int32_t c = extra[(i - n_top)*se0 + i1*se1 + i2*se2 + i3*se3]; + if (c >= 0 && c < n_kv) { + dst_row[c] = mask_row[c]; + } + continue; + } + + const int32_t p = top_k_row[i*sk0]; + if (p < 0 || p >= n_pick) { + continue; + } + if (score && !(score[p*ss0 + i1*ss1 + i2*ss2 + i3*ss3] > -INFINITY)) { + continue; + } + if (!blocks) { + if (p < n_kv) { + dst_row[p] = mask_row[p]; + } + continue; + } + for (int64_t k = 0; k < n_blk; ++k) { + const int32_t c = blocks[k*sb0 + p*sb1]; + if (c >= 0 && c < n_kv) { + dst_row[c] = mask_row[c]; + } + } + } +} + +template +static void top_k_mask_cuda(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * mask = dst->src[0]; + const ggml_tensor * top_k = dst->src[1]; + const ggml_tensor * score = dst->src[2]; + const ggml_tensor * blocks = dst->src[3]; + const ggml_tensor * extra = dst->src[4]; + + const int64_t n_kv = mask->ne[0]; + + GGML_ASSERT(dst->nb[0] == sizeof(T) && mask->nb[0] == sizeof(T)); + GGML_ASSERT(dst->nb[1] % sizeof(T) == 0 && mask->nb[1] % sizeof(T) == 0); + + // 16-byte stores need every dst row aligned + const bool vec = (n_kv*sizeof(T)) % sizeof(int4) == 0 && + dst->nb[1] % sizeof(int4) == 0 && dst->nb[2] % sizeof(int4) == 0 && dst->nb[3] % sizeof(int4) == 0 && + (uintptr_t) dst->data % sizeof(int4) == 0; + + const int64_t n_pick = blocks ? blocks->ne[1] : score ? score->ne[0] : n_kv; + + const size_t si = sizeof(int32_t); + const size_t sf = sizeof(float); + + const dim3 block_dims(CUDA_TOP_K_MASK_BLOCK_SIZE, 1, 1); + const dim3 grid_dims(ggml_nrows(dst), 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, ctx.stream()); + + ggml_cuda_kernel_launch(top_k_mask_kernel, launch_params, + (const T *) mask->data, (const int32_t *) top_k->data, + score ? (const float *) score->data : nullptr, + blocks ? (const int32_t *) blocks->data : nullptr, + extra ? (const int32_t *) extra->data : nullptr, + (T *) dst->data, + n_kv, dst->ne[1], dst->ne[2], + mask->ne[1], mask->ne[2], mask->ne[3], + top_k->ne[0], n_pick, blocks ? blocks->ne[0] : 1, extra ? extra->ne[0] : 0, + mask->nb[1]/sizeof(T), mask->nb[2]/sizeof(T), mask->nb[3]/sizeof(T), + dst->nb[1]/sizeof(T), dst->nb[2]/sizeof(T), dst->nb[3]/sizeof(T), + top_k->nb[0]/si, top_k->nb[1]/si, top_k->nb[2]/si, top_k->nb[3]/si, + score ? score->nb[0]/sf : 0, score ? score->nb[1]/sf : 0, score ? score->nb[2]/sf : 0, score ? score->nb[3]/sf : 0, + blocks ? blocks->nb[0]/si : 0, blocks ? blocks->nb[1]/si : 0, + extra ? extra->nb[0]/si : 0, extra ? extra->nb[1]/si : 0, extra ? extra->nb[2]/si : 0, extra ? extra->nb[3]/si : 0, + vec); +} + +void ggml_cuda_op_top_k_mask(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + switch (dst->type) { + case GGML_TYPE_F16: + top_k_mask_cuda(ctx, dst); + break; + case GGML_TYPE_F32: + top_k_mask_cuda(ctx, dst); + break; + default: + GGML_ABORT("unsupported type"); + } +} diff --git a/ggml/src/ggml-cuda/top-k-mask.cuh b/ggml/src/ggml-cuda/top-k-mask.cuh new file mode 100644 index 0000000..930d666 --- /dev/null +++ b/ggml/src/ggml-cuda/top-k-mask.cuh @@ -0,0 +1,3 @@ +#include "common.cuh" + +void ggml_cuda_op_top_k_mask(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/top-k.cu b/ggml/src/ggml-cuda/top-k.cu index 59ce36f..b0f6099 100644 --- a/ggml/src/ggml-cuda/top-k.cu +++ b/ggml/src/ggml-cuda/top-k.cu @@ -1,14 +1,51 @@ #include "argsort.cuh" #include "top-k.cuh" +// Adjusted implementation thresholds from #28547, can be overridden at build time +#ifndef GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC +# if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) +// not measured on HIP/MUSA, keep the old split +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC 1024 +# else +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC 512 +# endif +#endif // GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC + +#ifndef GGML_CUDA_TOP_K_NCOLS_THRESHOLD_ARGSORT +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_ARGSORT 4096 +#endif // GGML_CUDA_TOP_K_NCOLS_THRESHOLD_ARGSORT + +// bitonic up to this width while nrows fits in one wave of SMs, 0 disables +#ifndef GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS +# if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS 0 +# else +# define GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS 1024 +# endif +#endif // GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS + #ifdef GGML_CUDA_USE_CUB # include -# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2) +// DeviceTopK has a race condition before CCCL 3.4.3. +// https://github.com/NVIDIA/cccl/pull/10627 +# if (CCCL_MAJOR_VERSION > 3 || \ + (CCCL_MAJOR_VERSION == 3 && CCCL_MINOR_VERSION > 4) || \ + (CCCL_MAJOR_VERSION == 3 && CCCL_MINOR_VERSION == 4 && CCCL_PATCH_VERSION >= 3)) # define CUB_TOP_K_AVAILABLE +# include using namespace cub; -# endif // CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2 +# endif // CCCL >= 3.4.3 #endif // GGML_CUDA_USE_CUB +// max rows for the per-row DeviceTopK / CUB argsort path before switching to radix / bitonic +#ifndef GGML_CUDA_TOP_K_NROWS_THRESHOLD +# ifdef CUB_TOP_K_AVAILABLE +# define GGML_CUDA_TOP_K_NROWS_THRESHOLD 2 +# else +# define GGML_CUDA_TOP_K_NROWS_THRESHOLD 1 +# endif +#endif // GGML_CUDA_TOP_K_NROWS_THRESHOLD + #ifdef CUB_TOP_K_AVAILABLE static void top_k_cub(ggml_cuda_pool & pool, @@ -35,7 +72,7 @@ static void top_k_cub(ggml_cuda_pool & pool, ncols, k, env)); } -#elif defined(GGML_CUDA_USE_CUB) // CUB_TOP_K_AVAILABLE +#endif // CUB_TOP_K_AVAILABLE static int next_power_of_2(int x) { int n = 1; @@ -45,7 +82,201 @@ static int next_power_of_2(int x) { return n; } -#endif // CUB_TOP_K_AVAILABLE +static __device__ __forceinline__ uint32_t top_k_float_to_ordered(float value) { + const uint32_t bits = __float_as_uint(value); + const uint32_t mask = (uint32_t) (-(int32_t) (bits >> 31)) | 0x80000000U; + return bits ^ mask; +} + +struct top_k_radix_state { + uint32_t prefix; + uint32_t prefix_mask; + int rank; + int greater_count; + int equal_count; +}; + +static __global__ void top_k_radix_init(top_k_radix_state * states, int nrows, int k) { + const int row = blockIdx.x * blockDim.x + threadIdx.x; + if (row < nrows) { + states[row] = {0, 0, k, 0, 0}; + } +} + +template +static __global__ void top_k_radix_histogram( + const float * __restrict__ src, + const top_k_radix_state * __restrict__ states, + int * __restrict__ block_histograms, + int ncols, + int blocks_per_row, + int shift) { + constexpr int NBINS = 1 << RADIX_BITS; + + const int row = blockIdx.x / blocks_per_row; + const int row_block = blockIdx.x % blocks_per_row; + const int tid = threadIdx.x; + const float * row_src = src + (size_t) row * ncols; + __shared__ int histogram[NBINS]; + + histogram[tid] = 0; + __syncthreads(); + + const top_k_radix_state state = states[row]; + for (int64_t col = row_block * BLOCK_SIZE + tid; + col < ncols; + col += blocks_per_row * BLOCK_SIZE) { + const uint32_t key = top_k_float_to_ordered(row_src[col]); + if ((key & state.prefix_mask) == state.prefix) { + atomicAdd(&histogram[(key >> shift) & (NBINS - 1)], 1); + } + } + __syncthreads(); + + const size_t histogram_offset = + ((size_t) row * blocks_per_row + row_block) * NBINS; + block_histograms[histogram_offset + tid] = histogram[tid]; +} + +template +static __global__ void top_k_radix_select( + const int * __restrict__ block_histograms, + top_k_radix_state * __restrict__ states, + int blocks_per_row, + int shift) { + constexpr int NBINS = 1 << RADIX_BITS; + + const int row = blockIdx.x; + const int tid = threadIdx.x; + __shared__ int histogram[NBINS]; + + int count = 0; + for (int row_block = 0; row_block < blocks_per_row; ++row_block) { + const size_t offset = ((size_t) row * blocks_per_row + row_block) * NBINS; + count += block_histograms[offset + tid]; + } + histogram[tid] = count; + __syncthreads(); + + if (tid == 0) { + top_k_radix_state state = states[row]; + int bin = NBINS - 1; + while (bin > 0 && histogram[bin] < state.rank) { + state.rank -= histogram[bin--]; + } + state.prefix |= (uint32_t) bin << shift; + state.prefix_mask |= (uint32_t) (NBINS - 1) << shift; + states[row] = state; + } +} + +static __global__ void top_k_radix_reset_counters(top_k_radix_state * states, int nrows) { + const int row = blockIdx.x * blockDim.x + threadIdx.x; + if (row < nrows) { + states[row].greater_count = 0; + states[row].equal_count = 0; + } +} + +template +static __global__ void top_k_radix_gather( + const float * __restrict__ src, + int * __restrict__ dst, + top_k_radix_state * __restrict__ states, + int ncols, + int k, + int blocks_per_row) { + const int row = blockIdx.x / blocks_per_row; + const int row_block = blockIdx.x % blocks_per_row; + const int tid = threadIdx.x; + const float * row_src = src + (size_t) row * ncols; + int * row_dst = dst + (size_t) row * k; + top_k_radix_state * state = &states[row]; + + for (int64_t col = row_block * BLOCK_SIZE + tid; + col < ncols; + col += blocks_per_row * BLOCK_SIZE) { + const uint32_t key = top_k_float_to_ordered(row_src[col]); + if (key > state->prefix) { + const int pos = atomicAdd(&state->greater_count, 1); + row_dst[pos] = col; + } else if (key == state->prefix) { + const int pos = atomicAdd(&state->equal_count, 1); + if (pos < state->rank) { + row_dst[k - state->rank + pos] = col; + } + } + } +} + +static void top_k_radix_cuda( + ggml_cuda_pool & pool, + const float * src, int * dst, int ncols, int64_t nrows, int k, cudaStream_t stream) { + constexpr int BLOCK_SIZE = 256; + constexpr int RADIX_BITS = 8; + constexpr int NBINS = 1 << RADIX_BITS; + const int blocks_per_row = (int) std::min(((int64_t) ncols + 1023) / 1024, 64); + + // chunk the rows to bound the histogram memory to 64 MB + const int64_t chunk_nrows = ggml_cuda_chunk_nrows((size_t) blocks_per_row * NBINS * sizeof(int), nrows); + + ggml_cuda_pool_alloc states_alloc(pool, chunk_nrows); + ggml_cuda_pool_alloc histograms_alloc(pool, (size_t) chunk_nrows * blocks_per_row * NBINS); + top_k_radix_state * states = states_alloc.get(); + int * histograms = histograms_alloc.get(); + + for (int64_t i = 0; i < nrows; i += chunk_nrows) { + const int iter_nrows = std::min(chunk_nrows, nrows - i); + + top_k_radix_init<<<(iter_nrows + BLOCK_SIZE - 1) / BLOCK_SIZE, BLOCK_SIZE, 0, stream>>>(states, iter_nrows, k); + + const dim3 row_grid(blocks_per_row * iter_nrows); + for (int shift = 32 - RADIX_BITS; shift >= 0; shift -= RADIX_BITS) { + top_k_radix_histogram + <<>>( + src, states, histograms, ncols, blocks_per_row, shift); + top_k_radix_select + <<>>(histograms, states, blocks_per_row, shift); + } + + top_k_radix_reset_counters + <<<(iter_nrows + BLOCK_SIZE - 1) / BLOCK_SIZE, BLOCK_SIZE, 0, stream>>>(states, iter_nrows); + top_k_radix_gather + <<>>( + src, dst, states, ncols, k, blocks_per_row); + + src += (size_t) ncols * iter_nrows; + dst += (size_t) k * iter_nrows; + } +} + +static void top_k_argsort_cuda( + ggml_cuda_pool & pool, + const float * src, int * dst, int ncols, int64_t nrows, int k, bool use_cub, cudaStream_t stream) { + const int64_t chunk_nrows = ggml_cuda_chunk_nrows((size_t) ncols * sizeof(int), nrows); + + ggml_cuda_pool_alloc tmp_alloc(pool, (size_t) ncols * chunk_nrows); + int * tmp = tmp_alloc.get(); + + for (int64_t i = 0; i < nrows; i += chunk_nrows) { + const int iter_nrows = std::min(chunk_nrows, nrows - i); + + if (use_cub) { +#ifdef GGML_CUDA_USE_CUB + argsort_f32_i32_cuda_cub(pool, src, tmp, ncols, iter_nrows, GGML_SORT_ORDER_DESC, stream); +#else + GGML_ABORT("CUB is not available"); +#endif // GGML_CUDA_USE_CUB + } else { + argsort_f32_i32_cuda_bitonic(src, tmp, ncols, iter_nrows, GGML_SORT_ORDER_DESC, stream); + } + CUDA_CHECK(cudaMemcpy2DAsync(dst, k * sizeof(int), tmp, ncols * sizeof(int), k * sizeof(int), iter_nrows, + cudaMemcpyDeviceToDevice, stream)); + + src += (size_t) ncols * iter_nrows; + dst += (size_t) k * iter_nrows; + } +} void ggml_cuda_op_top_k(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; @@ -62,34 +293,45 @@ void ggml_cuda_op_top_k(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int64_t nrows = ggml_nrows(src0); const int64_t k = dst->ne[0]; ggml_cuda_pool & pool = ctx.pool(); + + const int device = ggml_cuda_get_device(); + #ifdef CUB_TOP_K_AVAILABLE - // TODO: Switch to `DeviceSegmentedTopK` for multi-row TopK once implemented - // https://github.com/NVIDIA/cccl/issues/6391 - // TODO: investigate if there exists a point where parallelized argsort is faster than sequential top-k - for (int i = 0; i < nrows; i++) { - top_k_cub(pool, src0_d + i * ncols, dst_d + i * k, ncols, k, stream); + // a single row always uses DeviceTopK if available + const bool bitonic_short = nrows > 1 && ncols <= GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC; +#else + const bool bitonic_short = ncols <= GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC; +#endif // CUB_TOP_K_AVAILABLE + const bool bitonic_few_rows = nrows > GGML_CUDA_TOP_K_NROWS_THRESHOLD && + ncols <= GGML_CUDA_TOP_K_NCOLS_THRESHOLD_BITONIC_FEW_ROWS && + nrows <= ggml_cuda_info().devices[device].nsm; + + if (bitonic_short || bitonic_few_rows) { + // the padded row must fit in shared memory + const int ncols_pad = next_power_of_2(ncols); + if (ncols_pad * sizeof(int) <= ggml_cuda_info().devices[device].smpb) { + top_k_argsort_cuda(pool, src0_d, dst_d, ncols, nrows, k, false, stream); + return; + } } -#elif defined(GGML_CUDA_USE_CUB) // CUB_TOP_K_AVAILABLE - // Fall back to argsort + copy - const int ncols_pad = next_power_of_2(ncols); - const size_t shared_mem = ncols_pad * sizeof(int); - const size_t max_shared_mem = ggml_cuda_info().devices[ggml_cuda_get_device()].smpb; - ggml_cuda_pool_alloc temp_dst_alloc(pool, ncols * nrows); - int * tmp_dst = temp_dst_alloc.get(); + if (nrows > GGML_CUDA_TOP_K_NROWS_THRESHOLD) { + top_k_radix_cuda(pool, src0_d, dst_d, ncols, nrows, k, stream); + return; + } - if (shared_mem > max_shared_mem || ncols > 1024) { - argsort_f32_i32_cuda_cub(pool, src0_d, tmp_dst, ncols, nrows, GGML_SORT_ORDER_DESC, stream); +#ifdef CUB_TOP_K_AVAILABLE + // TODO: Assess perf of `DeviceBatchedTopK` for multi-row TopK & CCCL >= 3.5.0, re-running perf sweep of https://github.com/ggml-org/llama.cpp/pull/28713 + for (int64_t i = 0; i < nrows; i++) { + top_k_cub(pool, src0_d + i * ncols, dst_d + i * k, ncols, k, stream); + } +#elif defined(GGML_CUDA_USE_CUB) // CUB_TOP_K_AVAILABLE + if (ncols <= GGML_CUDA_TOP_K_NCOLS_THRESHOLD_ARGSORT) { + top_k_argsort_cuda(pool, src0_d, dst_d, ncols, nrows, k, true, stream); } else { - argsort_f32_i32_cuda_bitonic(src0_d, tmp_dst, ncols, nrows, GGML_SORT_ORDER_DESC, stream); + top_k_radix_cuda(pool, src0_d, dst_d, ncols, nrows, k, stream); } - CUDA_CHECK(cudaMemcpy2DAsync(dst_d, k * sizeof(int), tmp_dst, ncols * sizeof(int), k * sizeof(int), nrows, - cudaMemcpyDeviceToDevice, stream)); #else // GGML_CUDA_USE_CUB - ggml_cuda_pool_alloc temp_dst_alloc(pool, ncols * nrows); - int * tmp_dst = temp_dst_alloc.get(); - argsort_f32_i32_cuda_bitonic(src0_d, tmp_dst, ncols, nrows, GGML_SORT_ORDER_DESC, stream); - CUDA_CHECK(cudaMemcpy2DAsync(dst_d, k * sizeof(int), tmp_dst, ncols * sizeof(int), k * sizeof(int), nrows, - cudaMemcpyDeviceToDevice, stream)); -#endif + top_k_radix_cuda(pool, src0_d, dst_d, ncols, nrows, k, stream); +#endif // CUB_TOP_K_AVAILABLE } diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu index 3020e5c..158b736 100644 --- a/ggml/src/ggml-cuda/topk-moe.cu +++ b/ggml/src/ggml-cuda/topk-moe.cu @@ -8,8 +8,11 @@ // Kernel config struct - passed by value to CUDA kernel struct topk_moe_config { bool use_sigmoid; + bool use_sqrt_softplus; bool with_norm; bool delayed_softmax; + int n_groups; + int n_group_used; }; // Warp-local softmax used for both the pre-top-k logits and the post-top-k delayed path. @@ -67,6 +70,63 @@ __device__ void sigmoid_warp_inplace(float (&vals)[experts_per_thread], const in } } +template +__device__ void sqrt_softplus_warp_inplace(float (&vals)[experts_per_thread], const int limit, const int lane) { +#pragma unroll + for (int i = 0; i < experts_per_thread; i++) { + const int idx = lane + i * WARP_SIZE; + const bool active = !use_limit || (idx < limit); + vals[i] = active ? sqrtf(vals[i] > 20.0f ? vals[i] : log1pf(expf(vals[i]))) : -INFINITY; + } +} + +#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 +// Set -inf for experts outside the top n_group_used groups. Group score is the sum of its top-2 experts (DeepSeek V3). +template +__device__ void group_mask_warp_inplace(float (&vals)[experts_per_thread], const int n_groups, const int n_group_used, const int lane) { + const int n_per_group = n_experts / n_groups; + + float score = -INFINITY; // lane g holds the score of group g + for (int g = 0; g < n_groups; g++) { + float top1 = -INFINITY; + float top2 = -INFINITY; +#pragma unroll + for (int i = 0; i < experts_per_thread; i++) { + if ((i * WARP_SIZE + lane) / n_per_group == g) { + top2 = fmaxf(top2, fminf(top1, vals[i])); + top1 = fmaxf(top1, vals[i]); + } + } +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { + const float o1 = __shfl_xor_sync(0xFFFFFFFF, top1, offset, WARP_SIZE); + const float o2 = __shfl_xor_sync(0xFFFFFFFF, top2, offset, WARP_SIZE); + top2 = fmaxf(fminf(top1, o1), fmaxf(top2, o2)); + top1 = fmaxf(top1, o1); + } + if (lane == g) { + score = top1 + top2; + } + } + + // rank of each group, lower index wins ties + int rank = 0; + for (int g = 0; g < n_groups; g++) { + const float other = __shfl_sync(0xFFFFFFFF, score, g, WARP_SIZE); + rank += other > score || (other == score && g < lane); + } + const auto keep = __ballot_sync(0xFFFFFFFFULL, lane < n_groups && rank < n_group_used); + +#pragma unroll + for (int i = 0; i < experts_per_thread; i++) { + const int g = (i * WARP_SIZE + lane) / n_per_group; + if (g >= n_groups || !((keep >> g) & 1)) { + vals[i] = -INFINITY; + } + } +} +#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 + /* This kernel does the following: 1. optionally softmax over the logits per token [n_experts, n_tokens] @@ -77,16 +137,22 @@ __device__ void sigmoid_warp_inplace(float (&vals)[experts_per_thread], const in It is intended as fusion of softmax->top-k->get_rows pipeline for MoE models */ template -__launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * logits, - float * weights, - int32_t * ids, - float * bias, - const int n_rows, - const int n_expert_used, - const float clamp_val, - const float scale_val, - const topk_moe_config config) { +__launch_bounds__(TOPK_MOE_ROWS_PER_BLOCK * WARP_SIZE, 1) +__global__ void topk_moe_cuda(const float * logits, + float * weights, + int32_t * ids, + float * bias, + const int n_rows, + const int n_expert_used, + const float clamp_val, + const float scale_val, + const topk_moe_config config) { +#if defined(GGML_USE_MUSA) + // MUSA: every warp of a partially filled block must reach the barrier below. + const int row = MIN(blockIdx.x * blockDim.y + threadIdx.y, n_rows - 1); +#else const int row = blockIdx.x * blockDim.y + threadIdx.y; +#endif // defined(GGML_USE_MUSA) if (row >= n_rows) { return; } @@ -105,15 +171,21 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * wt[i] = -INFINITY; } + ggml_cuda_pdl_sync(); #pragma unroll for (int i = 0; i < n_experts; i += WARP_SIZE) { const int expert = i + threadIdx.x; wt[i / WARP_SIZE] = (n_experts % WARP_SIZE == 0 || expert < n_experts) ? logits[expert] : -INFINITY; } + // Weights and IDs can alias logits, so wait until every row in the block reads its logits. + __syncthreads(); + if (!config.delayed_softmax) { if (config.use_sigmoid) { sigmoid_warp_inplace(wt, n_experts, threadIdx.x); + } else if (config.use_sqrt_softplus) { + sqrt_softplus_warp_inplace(wt, n_experts, threadIdx.x); } else { softmax_warp_inplace(wt, n_experts, threadIdx.x); } @@ -133,7 +205,7 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * // selection_wt is only needed when bias is present (selection uses wt + bias) // when no bias, we use wt directly for both selection and weight values - float selection_wt[has_bias ? experts_per_thread : 1]; + [[maybe_unused]] float selection_wt[has_bias ? experts_per_thread : 1]; if constexpr (has_bias) { #pragma unroll @@ -148,6 +220,17 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * } } +#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 + // without bias, masking wt is safe: the matcher ensures the kept groups hold all used experts + if (config.n_groups > 0) { + if constexpr (has_bias) { + group_mask_warp_inplace(selection_wt, config.n_groups, config.n_group_used, threadIdx.x); + } else { + group_mask_warp_inplace(wt, config.n_groups, config.n_group_used, threadIdx.x); + } + } +#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 + //at this point, each thread holds either a portion of the softmax distribution //or the raw logits. We do the argmax reduce over n_expert_used, each time marking //the expert weight as -inf to exclude from the next iteration @@ -161,6 +244,7 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * output_weights[i] = 0.f; } + ggml_cuda_pdl_lc(); for (int k = 0; k < n_expert_used; k++) { float max_val = wt[0]; int max_expert = threadIdx.x; @@ -267,59 +351,67 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx, const topk_moe_config config) { GGML_ASSERT(!(config.with_norm && config.delayed_softmax) && "delayed softmax is not supported with weight normalization"); - const int rows_per_block = 4; + const int rows_per_block = TOPK_MOE_ROWS_PER_BLOCK; dim3 grid_dims((n_rows + rows_per_block - 1) / rows_per_block, 1, 1); dim3 block_dims(WARP_SIZE, rows_per_block, 1); cudaStream_t stream = ctx.stream(); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); + +#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000 + GGML_ASSERT(config.n_groups == 0 && "expert groups fusion is not supported on this HIP version"); +#endif switch (n_expert) { case 1: - topk_moe_cuda<1, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<1, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 2: - topk_moe_cuda<2, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<2, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 4: - topk_moe_cuda<4, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<4, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 8: - topk_moe_cuda<8, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<8, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 16: - topk_moe_cuda<16, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<16, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 32: - topk_moe_cuda<32, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<32, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 64: - topk_moe_cuda<64, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<64, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 128: - topk_moe_cuda<128, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<128, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 256: - topk_moe_cuda<256, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<256, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); + break; + case 288: // StepFun 3.7 + ggml_cuda_kernel_launch(topk_moe_cuda<288, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 512: - topk_moe_cuda<512, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 576: - topk_moe_cuda<576, has_bias><<>>(logits, weights, ids, bias, n_rows, n_expert_used, - clamp_val, scale_val, config); + ggml_cuda_kernel_launch(topk_moe_cuda<576, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; default: GGML_ASSERT(false && "fatal error"); - break; } } @@ -357,9 +449,12 @@ void ggml_cuda_op_topk_moe(ggml_backend_cuda_context & ctx, } topk_moe_config config; - config.use_sigmoid = args.sigmoid; - config.with_norm = with_norm; - config.delayed_softmax = args.delayed_softmax; + config.use_sigmoid = args.sigmoid; + config.use_sqrt_softplus = args.sqrt_softplus; + config.with_norm = with_norm; + config.delayed_softmax = args.delayed_softmax; + config.n_groups = args.n_groups; + config.n_group_used = args.n_group_used; if (bias) { launch_topk_moe_cuda(ctx, logits_d, weights_d, ids_d, bias_d, n_rows, n_experts, n_expert_used, clamp_val, @@ -374,8 +469,10 @@ bool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op, const ggml_tensor * weights, const ggml_tensor * logits, const ggml_tensor * ids) { + // must match an instantiation of launch_topk_moe_cuda: a power of 2 up to 512, + // or one of the non-power-of-2 expert counts of supported models const int n_expert = ids->nb[1] / ids->nb[0]; - if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 576) { + if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 288 && n_expert != 576) { return false; } @@ -406,7 +503,7 @@ bool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op, } else if (gating_op->op == GGML_OP_UNARY) { ggml_unary_op op = ggml_get_unary_op(gating_op); - if (op != GGML_UNARY_OP_SIGMOID) { + if (op != GGML_UNARY_OP_SIGMOID && op != GGML_UNARY_OP_SOFTPLUS) { return false; } } diff --git a/ggml/src/ggml-cuda/topk-moe.cuh b/ggml/src/ggml-cuda/topk-moe.cuh index 243dc2f..3637500 100644 --- a/ggml/src/ggml-cuda/topk-moe.cuh +++ b/ggml/src/ggml-cuda/topk-moe.cuh @@ -3,13 +3,19 @@ #include +// Rows that one CUDA block handles. +#define TOPK_MOE_ROWS_PER_BLOCK 8 + struct ggml_cuda_topk_moe_args { bool sigmoid{}; + bool sqrt_softplus{}; bool softmax{}; bool delayed_softmax{}; bool prob_bias{}; bool norm{}; bool scale{}; + int n_groups{}; // > 0: pick experts only from the top n_group_used expert groups + int n_group_used{}; }; void ggml_cuda_op_topk_moe(ggml_backend_cuda_context & ctx, diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu index 4ad30fa..b3b6aa4 100644 --- a/ggml/src/ggml-cuda/unary.cu +++ b/ggml/src/ggml-cuda/unary.cu @@ -65,6 +65,11 @@ static __device__ __forceinline__ float op_sqr(float x) { return x * x; } +static __device__ __forceinline__ float op_relu_sqr(float x) { + const float r = fmaxf(x, 0.0f); + return r * r; +} + static __device__ __forceinline__ float op_sqrt(float x) { return sqrtf(x); } @@ -86,7 +91,7 @@ static __device__ __forceinline__ float op_expm1(float x) { } static __device__ __forceinline__ float op_softplus(float x) { - return (x > 20.0f) ? x : logf(1.0f + expf(x)); + return (x > 20.0f) ? x : log1pf(expf(x)); } static __device__ __forceinline__ float op_elu(float x) { @@ -102,7 +107,7 @@ static __device__ __forceinline__ float op_ceil(float x) { } static __device__ __forceinline__ float op_round(float x) { - return round(x); + return roundf(x); } static __device__ __forceinline__ float op_trunc(float x) { @@ -111,19 +116,50 @@ static __device__ __forceinline__ float op_trunc(float x) { template static __global__ void unary_op_kernel(const T * x, T * dst, const int k) { + ggml_cuda_pdl_lc(); const int i = blockDim.x*blockIdx.x + threadIdx.x; if (i >= k) { return; } - dst[i] = (T)op((float)x[i]); + ggml_cuda_pdl_sync(); + dst[i] = ggml_cuda_cast(op(ggml_cuda_cast(x[i]))); } template static void unary_cuda(const T * x, T * dst, const int k, cudaStream_t stream) { const int num_blocks = (k + CUDA_NEG_BLOCK_SIZE - 1) / CUDA_NEG_BLOCK_SIZE; - unary_op_kernel<<>>(x, dst, k); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_NEG_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(unary_op_kernel, launch_params, x, dst, k); +} + +template +static __global__ void unary_op_kernel_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03) { + ggml_cuda_pdl_lc(); + const int i = blockDim.x*blockIdx.x + threadIdx.x; + + if (i >= k) { + return; + } + + int64_t rem = i; + const int64_t i0 = rem % ne00; rem /= ne00; + const int64_t i1 = rem % ne01; rem /= ne01; + const int64_t i2 = rem % ne02; + const int64_t i3 = rem / ne02; + const size_t src_byte_offset = i0 * nb00 + i1 * nb01 + i2 * nb02 + i3 * nb03; + const T * src_ptr = (const T *)((const char *)x + src_byte_offset); + + ggml_cuda_pdl_sync(); + dst[i] = ggml_cuda_cast(op(ggml_cuda_cast(*src_ptr))); +} + +template +static void unary_cuda_strided(const T * x, T * dst, const int k,const int64_t ne00,const int64_t ne01,const int64_t ne02,const size_t nb00,const size_t nb01,const size_t nb02,const size_t nb03, cudaStream_t stream) { + const int num_blocks = (k + CUDA_NEG_BLOCK_SIZE - 1) / CUDA_NEG_BLOCK_SIZE; + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_NEG_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(unary_op_kernel_strided, launch_params, x, dst, k, ne00,ne01,ne02,nb00,nb01,nb02,nb03); } template @@ -131,18 +167,34 @@ void ggml_cuda_op_unary(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const void * src0_d = src0->data; void * dst_d = dst->data; - cudaStream_t stream = ctx.stream(); - GGML_ASSERT(ggml_is_contiguous(src0)); + cudaStream_t stream = ctx.stream(); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); GGML_ASSERT(src0->type == dst->type); - if (src0->type == GGML_TYPE_F16) { - unary_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream); + if (ggml_is_contiguous(src0)) { + if (src0->type == GGML_TYPE_F16) { + unary_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), stream); + } else if (src0->type == GGML_TYPE_BF16) { + unary_cuda((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), stream); + } else { + unary_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream); + } } else { - unary_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), stream); + if (src0->type == GGML_TYPE_F16) { + unary_cuda_strided((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), + src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream); + } else if (src0->type == GGML_TYPE_BF16) { + unary_cuda_strided((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), + src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream); + } else { + unary_cuda_strided((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), + src0->ne[0], src0->ne[1], src0->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], stream); + } } } @@ -253,6 +305,7 @@ void ggml_cuda_op_softplus(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { template static __global__ void unary_gated_op_kernel(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1) { + ggml_cuda_pdl_lc(); const int64_t i = int64_t(blockDim.x)*blockIdx.x + threadIdx.x; if (i >= k) { @@ -263,13 +316,15 @@ static __global__ void unary_gated_op_kernel(const T * x, const T * g, T * dst, const int64_t j0 = (i / n) * o0 + (i % n); const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); - dst[i] = (T)(op((float)x[j0]) * (float)g[j1]); + ggml_cuda_pdl_sync(); + dst[i] = ggml_cuda_cast(op(ggml_cuda_cast(x[j0])) * ggml_cuda_cast(g[j1])); } template static void unary_gated_cuda(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, cudaStream_t stream) { const int64_t num_blocks = (k + CUDA_GLU_BLOCK_SIZE - 1) / CUDA_GLU_BLOCK_SIZE; - unary_gated_op_kernel<<>>(x, g, dst, k, n, o0, o1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_GLU_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(unary_gated_op_kernel, launch_params, x, g, dst, k, n, o0, o1); } template @@ -288,8 +343,7 @@ void ggml_cuda_op_unary_gated(ggml_backend_cuda_context & ctx, ggml_tensor * dst GGML_ASSERT(src0->nb[0] == ggml_element_size(src0)); GGML_ASSERT(ggml_is_contiguous(dst)); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); GGML_ASSERT(src0->type == dst->type); GGML_ASSERT(dst->ne[0] == nc); GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0)); @@ -313,6 +367,16 @@ void ggml_cuda_op_unary_gated(ggml_backend_cuda_context & ctx, ggml_tensor * dst } unary_gated_cuda(src0_p, src1_p, (half *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(half), src1_o / sizeof(half), stream); + } else if (src0->type == GGML_TYPE_BF16) { + nv_bfloat16 * src0_p = (nv_bfloat16 *) src0_d; + nv_bfloat16 * src1_p = (nv_bfloat16 *) src1_d; + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + unary_gated_cuda(src0_p, src1_p, (nv_bfloat16 *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(nv_bfloat16), src1_o / sizeof(nv_bfloat16), stream); } else { float * src0_p = (float *) src0_d; float * src1_p = (float *) src1_d; @@ -416,6 +480,81 @@ void ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst) swiglu_oai_cuda(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream); } +// swiglu_clamp + +template +static __global__ void swiglu_clamp_kernel(const T * gate, const T * up, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, float limit) { + const int64_t i = int64_t(blockDim.x)*blockIdx.x + threadIdx.x; + + if (i >= k) { + return; + } + + const int64_t j0 = (i / n) * o0 + (i % n); + const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); + + dst[i] = (T) ggml_cuda_op_swiglu_clamp_single((float) gate[j0], (float) up[j1], limit); +} + +template +static void swiglu_clamp_cuda(const T * gate, const T * up, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, const float limit, cudaStream_t stream) { + const int64_t num_blocks = (k + CUDA_GLU_BLOCK_SIZE - 1) / CUDA_GLU_BLOCK_SIZE; + swiglu_clamp_kernel<<>>(gate, up, dst, k, n, o0, o1, limit); +} + +void ggml_cuda_op_swiglu_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + void * src0_d = src0->data; + void * src1_d = src1 ? src1->data : src0->data; + const int64_t src0_o = src0->nb[1]; + const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + void * dst_d = dst->data; + const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2; + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(src0->nb[0] == ggml_element_size(src0)); + GGML_ASSERT(ggml_is_contiguous(dst)); + + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0)); + + if (src1) { + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src1->nb[0] == ggml_element_size(src1)); + GGML_ASSERT(src1->ne[0] == nc); + GGML_ASSERT(src0->type == src1->type); + } + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + const float limit = ggml_get_op_params_f32(dst, 3); + + if (src0->type == GGML_TYPE_F16) { + half * src0_p = (half *) src0_d; + half * src1_p = (half *) src1_d; + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + swiglu_clamp_cuda(src0_p, src1_p, (half *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(half), src1_o / sizeof(half), limit, stream); + } else { + float * src0_p = (float *) src0_d; + float * src1_p = (float *) src1_d; + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + swiglu_clamp_cuda(src0_p, src1_p, (float *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), limit, stream); + } +} + /* CUDA kernel + launcher for xIELU */ template @@ -451,8 +590,8 @@ void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); - GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16); + GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16); GGML_ASSERT(src0->type == dst->type); const float alpha_n = ggml_get_op_params_f32(dst, 1); @@ -462,6 +601,8 @@ void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { if (src0->type == GGML_TYPE_F16) { xielu_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream); + } else if (src0->type == GGML_TYPE_BF16) { + xielu_cuda((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream); } else { xielu_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream); } @@ -578,7 +719,7 @@ static void ggml_cuda_op_unary_mul_impl(ggml_backend_cuda_context & ctx, ggml_te GGML_ASSERT(other_src->nb[0] == ggml_element_size(other_src)); GGML_ASSERT(ggml_are_same_shape(unary_src, other_src)); - GGML_ASSERT(unary_src->type == GGML_TYPE_F32 || unary_src->type == GGML_TYPE_F16); + GGML_ASSERT(unary_src->type == GGML_TYPE_F32 || unary_src->type == GGML_TYPE_F16 || unary_src->type == GGML_TYPE_BF16); GGML_ASSERT(unary_src->type == other_src->type); GGML_ASSERT(unary_src->type == mul_node->type); @@ -593,6 +734,10 @@ static void ggml_cuda_op_unary_mul_impl(ggml_backend_cuda_context & ctx, ggml_te unary_gated_cuda((const half *) unary_src->data, (const half *) other_src->data, (half *) mul_node->data, k, nc, unary_stride / sizeof(half), other_stride / sizeof(half), stream); + } else if (unary_src->type == GGML_TYPE_BF16) { + unary_gated_cuda((const nv_bfloat16 *) unary_src->data, (const nv_bfloat16 *) other_src->data, + (nv_bfloat16 *) mul_node->data, k, nc, + unary_stride / sizeof(nv_bfloat16), other_stride / sizeof(nv_bfloat16), stream); } else { unary_gated_cuda((const float *) unary_src->data, (const float *) other_src->data, (float *) mul_node->data, k, nc, @@ -615,3 +760,21 @@ void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary GGML_ABORT("Unsupported unary op for fused unary+mul"); } } + +/* fused relu + sqr */ + +void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_node, ggml_tensor * sqr_node) { + const ggml_tensor * src = relu_node->src[0]; + cudaStream_t stream = ctx.stream(); + + GGML_ASSERT(ggml_is_contiguous(src)); + GGML_ASSERT(src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16); + GGML_ASSERT(src->type == sqr_node->type); + + const int k = ggml_nelements(src); + if (src->type == GGML_TYPE_F16) { + unary_cuda((const half *)src->data, (half *)sqr_node->data, k, stream); + } else { + unary_cuda((const float *)src->data, (float *)sqr_node->data, k, stream); + } +} diff --git a/ggml/src/ggml-cuda/unary.cuh b/ggml/src/ggml-cuda/unary.cuh index f1dd218..c6ffdb9 100644 --- a/ggml/src/ggml-cuda/unary.cuh +++ b/ggml/src/ggml-cuda/unary.cuh @@ -83,6 +83,8 @@ void ggml_cuda_op_swiglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +void ggml_cuda_op_swiglu_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + void ggml_cuda_op_geglu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_geglu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst); @@ -91,6 +93,8 @@ void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node); +void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_node, ggml_tensor * sqr_node); + __device__ __forceinline__ float ggml_cuda_op_silu_single(float x) { return x / (1.0f + expf(-x)); } @@ -110,3 +114,25 @@ __device__ __forceinline__ float ggml_cuda_op_swiglu_oai_single(float x, float g out_glu = out_glu * (1.0f + g); return out_glu; } + +__device__ __forceinline__ float ggml_cuda_op_swiglu_clamp_single(float gate, float up, float limit) { + gate = fminf(gate, limit); + up = fmaxf(fminf(up, limit), -limit); + + return ggml_cuda_op_silu_single(gate) * up; +} + +__device__ __forceinline__ float ggml_cuda_op_glu_single(ggml_glu_op op, float gate, float up, float limit) { + switch (op) { + case GGML_GLU_OP_SWIGLU: + return up * ggml_cuda_op_silu_single(gate); + case GGML_GLU_OP_GEGLU: + return up * ggml_cuda_op_gelu_single(gate); + case GGML_GLU_OP_SWIGLU_OAI: + return ggml_cuda_op_swiglu_oai_single(gate, up); + case GGML_GLU_OP_SWIGLU_CLAMP: + return ggml_cuda_op_swiglu_clamp_single(gate, up, limit); + default: + return up * gate; + } +} diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index d1741cc..d5ff93b 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -40,18 +40,18 @@ static __device__ __forceinline__ int2 get_int_from_table_16(const int & q4, con const uint32_t q_odd = (q4 >> 4); // Perform lookups in the lower half of the table (indices 0-7). - uint32_t v_even_low = __builtin_amdgcn_perm(values[1], values[0], q_even & 0x07070707); - uint32_t v_odd_low = __builtin_amdgcn_perm(values[1], values[0], q_odd & 0x07070707); + uint32_t v_even_low = ggml_hip_perm(values[1], values[0], q_even & 0x07070707); + uint32_t v_odd_low = ggml_hip_perm(values[1], values[0], q_odd & 0x07070707); // Perform lookups in the upper half of the table (indices 8-15). - uint32_t v_even_high = __builtin_amdgcn_perm(values[3], values[2], q_even & 0x07070707); - uint32_t v_odd_high = __builtin_amdgcn_perm(values[3], values[2], q_odd & 0x07070707); + uint32_t v_even_high = ggml_hip_perm(values[3], values[2], q_even & 0x07070707); + uint32_t v_odd_high = ggml_hip_perm(values[3], values[2], q_odd & 0x07070707); // Select between the low and high results based on the MSB of each index nibble. uint32_t mask_even = 0x03020100 | ((q_even & 0x08080808) >> 1); - uint32_t res_x = __builtin_amdgcn_perm(v_even_high, v_even_low, mask_even); + uint32_t res_x = ggml_hip_perm(v_even_high, v_even_low, mask_even); uint32_t mask_odd = 0x03020100 | ((q_odd & 0x08080808) >> 1); - uint32_t res_y = __builtin_amdgcn_perm(v_odd_high, v_odd_low, mask_odd); + uint32_t res_y = ggml_hip_perm(v_odd_high, v_odd_low, mask_odd); return make_int2(res_x, res_y); #elif !defined(GGML_USE_MUSA) @@ -109,6 +109,9 @@ static __device__ __forceinline__ uint32_t unpack_ksigns(const uint8_t v) { #define VDR_Q1_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism #define VDR_Q1_0_Q8_1_MMQ 4 // Q1_0 has 128 bits (4 ints) per block +#define VDR_Q2_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism +#define VDR_Q2_0_Q8_1_MMQ 2 // Q2_0 group 64: 128 bits (4 ints) per block, 2 32-element chunks + #define VDR_Q4_0_Q8_1_MMVQ 2 #define VDR_Q4_0_Q8_1_MMQ 4 @@ -357,12 +360,12 @@ static __device__ __forceinline__ float vec_dot_nvfp4_q8_1( return sum; } -#define VDR_Q2_K_Q8_1_MMVQ 1 +#define VDR_Q2_K_Q8_1_MMVQ 2 #define VDR_Q2_K_Q8_1_MMQ 4 // contiguous v/x values static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq( - const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales, + const int * __restrict__ v, const int * __restrict__ u, const int * __restrict__ scales, const half2 & dm2, const float * __restrict__ d8) { float sumf_d = 0.0f; @@ -370,17 +373,20 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq( #pragma unroll for (int i = 0; i < QR2_K; ++i) { - const int sc = scales[2*i]; + const int sc = scales[i/2] >> (16*(i%2)); - const int vi = (v >> (2*i)) & 0x03030303; + int sumi_d = 0; + int sumi_m = 0; +#pragma unroll + for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) { + const int vi = (v[j] >> (2*i)) & 0x03030303; - sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product + sumi_d = ggml_cuda_dp4a(vi, u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_d); // SIMD dot product + sumi_m = ggml_cuda_dp4a(0x01010101, u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_m); // sum of q8_1 values + } - // fill int with 4x m - int m = sc >> 4; - m |= m << 8; - m |= m << 16; - sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values + sumf_d += d8[i] * (sumi_d * (sc & 0xF)); + sumf_m += d8[i] * (sumi_m * ((sc >> 4) & 0xF)); // multiply constant q2_K part with sum of q8_1 values } const float2 dm2f = __half22float2(dm2); @@ -440,37 +446,35 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq( return sumf + d8*sumf_d8; } -#define VDR_Q3_K_Q8_1_MMVQ 1 +#define VDR_Q3_K_Q8_1_MMVQ 2 #define VDR_Q3_K_Q8_1_MMQ 2 // contiguous v/x values static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq( - const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales, - const int & scale_offset, const float & d3, const float * __restrict__ d8) { + const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int * __restrict__ scales, + const float & d3, const float * __restrict__ d8) { float sumf = 0.0f; #pragma unroll for (int i = 0; i < QR3_K; ++i) { - const int isc = scale_offset + 2*i; - - const int isc_low = isc % (QK_K/32); - const int sc_shift_low = 4 * (isc / (QK_K/32)); - const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF; + // 4 low bits from scales[0..7], 2 high bits from scales[8..11] + const int sc_low = (scales[i/2] >> (16*(i%2))) & 0x0F; + const int sc_high = (scales[2] >> (16*(i%2) + 2*(i/2))) & 0x03; + const int sc = (sc_low | (sc_high << 4)) - 32; - const int isc_high = isc % (QK_K/64); - const int sc_shift_high = 2 * (isc / (QK_K/64)); - const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4; - - const int sc = (sc_low | sc_high) - 32; - - const int vil = (vl >> (2*i)) & 0x03030303; - - const int vih = ((vh >> i) << 2) & 0x04040404; + int sumi = 0; +#pragma unroll + for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) { + const int vil = (vl[j] >> (2*i)) & 0x03030303; + const int vih = ((vh[j] >> i) << 2) & 0x04040404; - const int vi = __vsubss4(vil, vih); + // (vil | vih) - 4 for each byte, the - 4 is done by the second dp4a + sumi = ggml_cuda_dp4a(vil | vih, u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product + sumi = ggml_cuda_dp4a(0xFCFCFCFC, u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi); + } - sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product + sumf += d8[i] * (sumi * sc); } return d3 * sumf; @@ -617,12 +621,12 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq( return dm4f.x*sumf_d - dm4f.y*sumf_m; } -#define VDR_Q6_K_Q8_1_MMVQ 1 +#define VDR_Q6_K_Q8_1_MMVQ 2 #define VDR_Q6_K_Q8_1_MMQ 8 // contiguous v/x values static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq( - const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales, + const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int8_t * __restrict__ scales, const float & d, const float * __restrict__ d8) { float sumf = 0.0f; @@ -631,13 +635,18 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq( for (int i = 0; i < QR6_K; ++i) { const int sc = scales[4*i]; - const int vil = (vl >> (4*i)) & 0x0F0F0F0F; - - const int vih = ((vh >> (4*i)) << 4) & 0x30303030; + int sumi = 0; +#pragma unroll + for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) { + const int vil = (vl[j] >> (4*i)) & 0x0F0F0F0F; + const int vih = ((vh[j] >> (4*i)) << 4) & 0x30303030; - const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32 + // (vil | vih) - 32 for each byte, the - 32 is done by the second dp4a + sumi = ggml_cuda_dp4a(vil | vih, u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product + sumi = ggml_cuda_dp4a(0xE0E0E0E0, u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi); + } - sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product + sumf += d8[i] * (sumi * sc); } return d*sumf; @@ -681,40 +690,111 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1( // Q8_1: 32 elements per block with individual scales // iqs selects which of the 4 chunks of 32 elements to process (0-3) - const float d1 = bq1_0->d; + const float d1 = bq1_0->d; + const int16_t * qs = (const int16_t *) bq1_0->qs + iqs * 2; // Process only the chunk specified by iqs const block_q8_1 * bq8_1_chunk = bq8_1 + iqs; - // Load 32 bits (4 bytes) for this chunk from Q1_0 - const int offset = iqs * 4; - const int v = bq1_0->qs[offset + 0] | (bq1_0->qs[offset + 1] << 8) | - (bq1_0->qs[offset + 2] << 16) | (bq1_0->qs[offset + 3] << 24); - - // Unpack 32 bits into 32 signed values (-1 or +1) - int vi_bytes[8]; + int sumi = 0; #pragma unroll - for (int j = 0; j < 8; ++j) { - const int shift = j * 4; - const int bits4 = (v >> shift) & 0x0F; - const int b0 = (bits4 & 0x01) ? 1 : -1; - const int b1 = (bits4 & 0x02) ? 1 : -1; - const int b2 = (bits4 & 0x04) ? 1 : -1; - const int b3 = (bits4 & 0x08) ? 1 : -1; - vi_bytes[j] = (b0 & 0xFF) | ((b1 & 0xFF) << 8) | ((b2 & 0xFF) << 16) | ((b3 & 0xFF) << 24); + for (int j = 0; j < 2; ++j) { + const int q = qs[j]; + + const int u0 = get_int_b4(bq8_1_chunk->qs, j*4+0); + const int u1 = get_int_b4(bq8_1_chunk->qs, j*4+1); + const int u2 = get_int_b4(bq8_1_chunk->qs, j*4+2); + const int u3 = get_int_b4(bq8_1_chunk->qs, j*4+3); + +#if defined(GGML_USE_HIP) + const uint32_t q_bits = (uint16_t) q; + + // Expand each group of four Q1_0 bits into byte selectors for v_perm_b32. + const uint32_t q0_bits = q_bits >> 0; + const uint32_t q1_bits = q_bits >> 4; + const uint32_t q2_bits = q_bits >> 8; + const uint32_t q3_bits = q_bits >> 12; + + const uint32_t q0_indices = (q0_bits & 0x01) | ((q0_bits & 0x02) << 7) | ((q0_bits & 0x04) << 14) | ((q0_bits & 0x08) << 21); + const uint32_t q1_indices = (q1_bits & 0x01) | ((q1_bits & 0x02) << 7) | ((q1_bits & 0x04) << 14) | ((q1_bits & 0x08) << 21); + const uint32_t q2_indices = (q2_bits & 0x01) | ((q2_bits & 0x02) << 7) | ((q2_bits & 0x04) << 14) | ((q2_bits & 0x08) << 21); + const uint32_t q3_indices = (q3_bits & 0x01) | ((q3_bits & 0x02) << 7) | ((q3_bits & 0x04) << 14) | ((q3_bits & 0x08) << 21); + + const int v0 = ggml_hip_perm(0x000001FF, 0x000001FF, q0_indices); + const int v1 = ggml_hip_perm(0x000001FF, 0x000001FF, q1_indices); + const int v2 = ggml_hip_perm(0x000001FF, 0x000001FF, q2_indices); + const int v3 = ggml_hip_perm(0x000001FF, 0x000001FF, q3_indices); +#else + // unpack crumbs into nibble indices + const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13] + const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15] + // unpack nibbles into byte values + const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >> 0); + const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >> 0); + const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16); + const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16); + // unshuffle values + const int v0 = __byte_perm(s0, s1, 0x5410); + const int v1 = __byte_perm(s0, s1, 0x7632); + const int v2 = __byte_perm(s2, s3, 0x5410); + const int v3 = __byte_perm(s2, s3, 0x7632); +#endif // defined(GGML_USE_HIP) + + sumi = ggml_cuda_dp4a(v0, u0, sumi); + sumi = ggml_cuda_dp4a(v1, u1, sumi); + sumi = ggml_cuda_dp4a(v2, u2, sumi); + sumi = ggml_cuda_dp4a(v3, u3, sumi); } - // Compute dot product for this 32-element chunk + // Apply Q1_0's single scale and this chunk's Q8_1 scale + const float d8 = __low2float(bq8_1_chunk->ds); + return d1 * d8 * sumi; +} + +static __device__ __forceinline__ float vec_dot_q2_0_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq + kbx; + + // Q2_0 (group 64): 64 elements with ONE scale, 2 bits per element (4 elements per byte) + // Q8_1: 32 elements per block with individual scales + // iqs selects which of the 2 chunks of 32 elements to process (0-1) + + const float d2 = bq2_0->d; + const int16_t * qs = (const int16_t *) bq2_0->qs + iqs * 4; + + // Process only the chunk specified by iqs + const block_q8_1 * bq8_1_chunk = bq8_1 + iqs; + int sumi = 0; #pragma unroll - for (int j = 0; j < 8; ++j) { - const int u = get_int_b4(bq8_1_chunk->qs, j); - sumi = ggml_cuda_dp4a(vi_bytes[j], u, sumi); + for (int j = 0; j < 4; ++j) { + const int q = qs[j]; + const int u = get_int_b4(bq8_1_chunk->qs, j*2+0); + const int v = get_int_b4(bq8_1_chunk->qs, j*2+1); + +#if defined(GGML_USE_HIP) + const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18); + const uint32_t qy_bits = q >> 8; + const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18); + const int qx = ggml_hip_perm(0x020100FF, 0x020100FF, qx_indices); + const int qy = ggml_hip_perm(0x020100FF, 0x020100FF, qy_indices); +#else + // unpack even and odd crumbs into byte values + const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); + const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2); + // unshuffle values + const int qx = __byte_perm(qe, qo, 0x5140); + const int qy = __byte_perm(qe, qo, 0x7362); +#endif // defined(GGML_USE_HIP) + + sumi = ggml_cuda_dp4a(u, qx, sumi); + sumi = ggml_cuda_dp4a(v, qy, sumi); } - // Apply Q1_0's single scale and this chunk's Q8_1 scale + // Apply Q2_0's single scale and this chunk's Q8_1 scale const float d8 = __low2float(bq8_1_chunk->ds); - return d1 * d8 * sumi; + return d2 * d8 * sumi; } static __device__ __forceinline__ float vec_dot_q4_0_q8_1( @@ -816,18 +896,31 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1( const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx; + // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift) const int bq8_offset = QR2_K * (iqs / QI8_1); - const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + const int iqs8 = iqs % QI8_1; - const uint8_t * scales = bq2_K->scales + scale_offset; + int v[VDR_Q2_K_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) { + v[j] = get_int_b4(bq2_K->qs, iqs + j); + } + + // scale of shift i is byte 2*i + (iqs8 >= QI8_1/2) of the 8 bytes for this half + const int sc_shift = 8 * (iqs8 / (QI8_1/2)); + int scales[2]; + scales[0] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 0) >> sc_shift; + scales[1] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 1) >> sc_shift; - const int v = get_int_b4(bq2_K->qs, iqs); - int u[QR2_K]; + int u[VDR_Q2_K_Q8_1_MMVQ*QR2_K]; float d8[QR2_K]; #pragma unroll - for (int i = 0; i < QR2_K; ++ i) { - u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); + for (int i = 0; i < QR2_K; ++i) { +#pragma unroll + for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) { + u[VDR_Q2_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j); + } d8[i] = __low2float(bq8_1[bq8_offset + i].ds); } @@ -839,26 +932,42 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1( const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx; + // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift) const int bq8_offset = QR3_K * (iqs / (QI3_K/2)); - const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + const int iqs8 = iqs % QI8_1; const float d = bq3_K->d; - const int vl = get_int_b2(bq3_K->qs, iqs); + int vl[VDR_Q3_K_Q8_1_MMVQ]; + int vh[VDR_Q3_K_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) { + vl[j] = get_int_b2(bq3_K->qs, iqs + j); + vh[j] = get_int_b2(bq3_K->hmask, iqs8 + j) >> bq8_offset; + } - // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted - const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset; + // sub-block 8*(iqs/8) + 2*i + (iqs8 >= QI8_1/2) for shift i: + // 4 low bits in nibble iqs/8 of bytes, 2 high bits at bit 4*(iqs/8) + 2*(i/2) of bytes 8..11 + const int sc_shift = 8 * (iqs8 / (QI8_1/2)) + 4 * (iqs / (QI3_K/2)); + int scales[3]; +#pragma unroll + for (int l = 0; l < 3; ++l) { + scales[l] = get_int_b2(bq3_K->scales, l) >> sc_shift; + } - int u[QR3_K]; + int u[VDR_Q3_K_Q8_1_MMVQ*QR3_K]; float d8[QR3_K]; #pragma unroll for (int i = 0; i < QR3_K; ++i) { - u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); +#pragma unroll + for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) { + u[VDR_Q3_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j); + } d8[i] = __low2float(bq8_1[bq8_offset + i].ds); } - return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8); + return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, scales, d, d8); } static __device__ __forceinline__ float vec_dot_q4_K_q8_1( @@ -882,16 +991,20 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1( v[0] = q4[0]; v[1] = q4[4]; + // branchless so nvcc can hoist this out of the ncols_dst loop const uint16_t * scales = (const uint16_t *)bq4_K->scales; + const int j = bq8_offset/2; + const int jm = j & 1; + + const uint32_t s0 = scales[jm + 0]; + const uint32_t s2 = scales[jm + 2]; + const uint32_t s4 = scales[jm + 4]; + + const uint32_t hi = (uint32_t) -(int32_t) (j >= 2); + uint16_t aux[2]; - const int j = bq8_offset/2; - if (j < 2) { - aux[0] = scales[j+0] & 0x3f3f; - aux[1] = scales[j+2] & 0x3f3f; - } else { - aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2); - aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2); - } + aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi)); + aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi)); const uint8_t * sc = (const uint8_t *)aux; const uint8_t * m = sc + 2; @@ -927,16 +1040,21 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1( vh[0] = qh[0] >> bq8_offset; vh[1] = qh[4] >> bq8_offset; + // same as q4_K const uint16_t * scales = (const uint16_t *)bq5_K->scales; + const int j = bq8_offset/2; + const int jm = j & 1; + + const uint32_t s0 = scales[jm + 0]; + const uint32_t s2 = scales[jm + 2]; + const uint32_t s4 = scales[jm + 4]; + + const uint32_t hi = (uint32_t) -(int32_t) (j >= 2); + uint16_t aux[2]; - const int j = bq8_offset/2; - if (j < 2) { - aux[0] = scales[j+0] & 0x3f3f; - aux[1] = scales[j+2] & 0x3f3f; - } else { - aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2); - aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2); - } + aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi)); + aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi)); + const uint8_t * sc = (const uint8_t *)aux; const uint8_t * m = sc + 2; @@ -962,17 +1080,25 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1( const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8); const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4)); - const int vl = get_int_b2(bq6_K->ql, iqs); - const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift; + int vl[VDR_Q6_K_Q8_1_MMVQ]; + int vh[VDR_Q6_K_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) { + vl[j] = get_int_b2(bq6_K->ql, iqs + j); + vh[j] = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4) + j) >> vh_shift; + } const int8_t * scales = bq6_K->scales + scale_offset; - int u[QR6_K]; + int u[VDR_Q6_K_Q8_1_MMVQ*QR6_K]; float d8[QR6_K]; #pragma unroll for (int i = 0; i < QR6_K; ++i) { - u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1); +#pragma unroll + for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) { + u[VDR_Q6_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1 + j); + } d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds); } diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index 898fec3..8e8f5a4 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -6,10 +6,6 @@ #include #include -#if defined(GGML_HIP_ROCWMMA_FATTN) -#include -#endif // defined(GGML_HIP_ROCWMMA_FATTN) - #ifdef GGML_USE_NCCL #include #endif // GGML_USE_NCCL @@ -33,7 +29,6 @@ #define CU_MEM_LOCATION_TYPE_DEVICE hipMemLocationTypeDevice #define CU_MEM_ACCESS_FLAGS_PROT_READWRITE hipMemAccessFlagsProtReadWrite #define CU_CHECK(fn) {hipError_t err = fn; if(err != hipSuccess) { GGML_ABORT("HipVMM Failure: %s\n", hipGetErrorString(err)); }} -#define NCCL_CHECK(fn) {ncclResult_t err = fn; if(err != ncclSuccess) { GGML_ABORT("RCCL Failure RCCL returned: %i\n", err); }} #define __shfl_sync(mask, var, laneMask, width) __shfl(var, laneMask, width) #define __shfl_up_sync(mask, var, laneMask, width) __shfl_up(var, laneMask, width) #define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width) @@ -49,6 +44,8 @@ #define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS #define cublasSetStream hipblasSetStream #define cublasSgemm hipblasSgemm +#define cublasSgemmBatched hipblasSgemmBatched +#define cublasSgemmStridedBatched hipblasSgemmStridedBatched #define cublasStatus_t hipblasStatus_t #define cublasOperation_t hipblasOperation_t #define cudaDevAttrCooperativeLaunch hipDeviceAttributeCooperativeLaunch @@ -56,9 +53,11 @@ #define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess #define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess #define cudaDeviceGetAttribute hipDeviceGetAttribute +#define cudaDeviceGetPCIBusId hipDeviceGetPCIBusId #define cudaDeviceProp hipDeviceProp_t #define cudaDeviceSynchronize hipDeviceSynchronize #define cudaError_t hipError_t +#define cudaErrorMemoryAllocation hipErrorOutOfMemory #define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled #define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled #define cudaEventCreateWithFlags hipEventCreateWithFlags @@ -74,6 +73,10 @@ #define cudaGetDeviceProperties hipGetDeviceProperties #define cudaGetErrorString hipGetErrorString #define cudaGetLastError hipGetLastError +#define cudaHostAlloc hipHostMalloc +#define cudaHostAllocPortable hipHostMallocPortable +#define cudaHostAllocMapped hipHostMallocMapped +#define cudaHostGetDevicePointer hipHostGetDevicePointer #define cudaHostRegister hipHostRegister #define cudaHostRegisterPortable hipHostRegisterPortable #define cudaHostRegisterReadOnly hipHostRegisterReadOnly @@ -177,9 +180,9 @@ #define __CUDA_ARCH__ 1300 -#if defined(__gfx900__) || defined(__gfx906__) +#if defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__) #define GCN5 -#endif // defined(__gfx900__) || defined(__gfx906__) +#endif // defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__) #if defined(__gfx803__) #define GCN4 @@ -217,9 +220,9 @@ #define RDNA3 #endif // defined(__GFX11__) -#if defined(__gfx1150__) || defined(__gfx1151__) +#if defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__) #define RDNA3_5 -#endif // defined(__gfx1150__) || defined(__gfx1151__) +#endif // defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__) #if defined(RDNA3) && !defined(RDNA3_5) #define RDNA3_0 @@ -245,11 +248,11 @@ typedef __hip_bfloat16 nv_bfloat16; typedef __hip_bfloat162 nv_bfloat162; -#if HIP_VERSION >= 60200000 +#if HIP_VERSION >= 60300000 #include typedef __hip_fp8_e4m3 __nv_fp8_e4m3; #define FP8_AVAILABLE -#endif // HIP_VERSION >= 60200000 +#endif // HIP_VERSION >= 60300000 typedef int8_t int8x4_t __attribute__((ext_vector_type(4))); typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4))); @@ -274,7 +277,15 @@ static __device__ __forceinline__ int __vsubss4(const int a, const int b) { } static __device__ __forceinline__ int __vsub4(const int a, const int b) { - return __vsubss4(a, b); + // do some small modifications to a and b to make the subtraction not underflow + const unsigned int a_large = a | 0x80808080; + const unsigned int b_small = b & 0x7f7f7f7f; + const unsigned int result_low_7bits = a_large - b_small; + + // if two ops share the same high bit, we should flip the high bit of the result + const unsigned int should_flip_high_1bit = (a ^ ~b) & 0x80808080; + + return result_low_7bits ^ should_flip_high_1bit; } static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) { @@ -290,13 +301,32 @@ static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigne } static __device__ __forceinline__ unsigned int __vcmpne4(unsigned int a, unsigned int b) { - const uint8x4_t& va = reinterpret_cast(a); - const uint8x4_t& vb = reinterpret_cast(b); - unsigned int c; - uint8x4_t& vc = reinterpret_cast(c); + const unsigned int x = a ^ b; + + // any non-equal bit in a byte will set the high bit of that byte here + // the addition will not overflow in the byte as op1 and op2 are both less than 0x80 + const unsigned int ne_low_7bits = ((x & 0x7f7f7f7f) + 0x7f7f7f7f) & 0x80808080; + const unsigned int ne_high_1bit = x & 0x80808080; + const unsigned int ne_any_bit = ne_low_7bits | ne_high_1bit; + + return (ne_any_bit >> 7) * 0xff; +} + +// v_perm_b32 with selector bytes in [0, 7]: 0-3 pick bytes of lo, 4-7 pick bytes of hi. +static __device__ __forceinline__ uint32_t ggml_hip_perm(const uint32_t hi, const uint32_t lo, const uint32_t sel) { +#if __has_builtin(__builtin_amdgcn_is_invocable) + if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_perm)) { + return __builtin_amdgcn_perm(hi, lo, sel); + } + const uint64_t src = ((uint64_t) hi << 32) | lo; + uint32_t res = 0; #pragma unroll for (int i = 0; i < 4; ++i) { - vc[i] = va[i] == vb[i] ? 0x00 : 0xff; + const uint32_t byte_sel = (sel >> (8 * i)) & 0x07; + res |= (uint32_t) ((src >> (8 * byte_sel)) & 0xFF) << (8 * i); } - return c; + return res; +#else + return __builtin_amdgcn_perm(hi, lo, sel); +#endif // __has_builtin(__builtin_amdgcn_is_invocable) } diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 1abb8ac..22126de 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -5,6 +5,11 @@ #include #include #include + +#ifdef __MUSA_ARCH__ +#define __CUDA_ARCH__ 1300 // GGML_CUDA_CC_RUBIN +#endif // __MUSA_ARCH__ + #define CUBLAS_COMPUTE_16F CUDA_R_16F #define CUBLAS_COMPUTE_32F CUDA_R_32F #define CUBLAS_COMPUTE_32F_FAST_16F MUBLAS_COMPUTE_32F_FAST_16F @@ -32,6 +37,8 @@ #define cublasSetMathMode mublasSetMathMode #define cublasSetStream mublasSetStream #define cublasSgemm mublasSgemm +#define cublasSgemmBatched mublasSgemmBatched +#define cublasSgemmStridedBatched mublasSgemmStridedBatched #define cublasStatus_t mublasStatus_t #define cublasOperation_t mublasOperation_t #define cublasGetStatusString mublasGetStatusString @@ -39,9 +46,12 @@ #define cudaDeviceCanAccessPeer musaDeviceCanAccessPeer #define cudaDeviceDisablePeerAccess musaDeviceDisablePeerAccess #define cudaDeviceEnablePeerAccess musaDeviceEnablePeerAccess +#define cudaDeviceGetPCIBusId musaDeviceGetPCIBusId #define cudaDeviceProp musaDeviceProp #define cudaDeviceSynchronize musaDeviceSynchronize +#define cudaDeviceGetAttribute musaDeviceGetAttribute #define cudaError_t musaError_t +#define cudaErrorMemoryAllocation musaErrorMemoryAllocation #define cudaErrorPeerAccessAlreadyEnabled musaErrorPeerAccessAlreadyEnabled #define cudaErrorPeerAccessNotEnabled musaErrorPeerAccessNotEnabled #define cudaEventCreateWithFlags musaEventCreateWithFlags @@ -110,6 +120,7 @@ #define cuMemRelease muMemRelease #define cuMemSetAccess muMemSetAccess #define cuMemUnmap muMemUnmap +#define cudaDevAttrCooperativeLaunch musaDevAttrCooperativeLaunch #define cudaFuncAttributeMaxDynamicSharedMemorySize musaFuncAttributeMaxDynamicSharedMemorySize #define cudaFuncSetAttribute musaFuncSetAttribute #define cudaMemcpy3DPeerParms musaMemcpy3DPeerParms @@ -141,6 +152,9 @@ #define cudaStreamCaptureModeRelaxed musaStreamCaptureModeRelaxed #define cudaStreamBeginCapture musaStreamBeginCapture #define cudaStreamEndCapture musaStreamEndCapture +#define cudaStreamCaptureStatus musaStreamCaptureStatus +#define cudaStreamCaptureStatusNone musaStreamCaptureStatusNone +#define cudaStreamIsCapturing musaStreamIsCapturing #define cudaOccupancyMaxActiveBlocksPerMultiprocessor musaOccupancyMaxActiveBlocksPerMultiprocessor typedef __mt_bfloat16 nv_bfloat16; diff --git a/ggml/src/ggml-cuda/wkv.cu b/ggml/src/ggml-cuda/wkv.cu index d2fced7..0bf9977 100644 --- a/ggml/src/ggml-cuda/wkv.cu +++ b/ggml/src/ggml-cuda/wkv.cu @@ -79,9 +79,7 @@ static __global__ void rwkv_wkv7_f32(const int B, const int T, const int C, cons float state[head_size]; __shared__ float _r[head_size], _w[head_size], _k[head_size], _a[head_size], _b[head_size]; -#ifndef GGML_USE_MUSA #pragma unroll -#endif for (int i = 0; i < head_size; i++) { state[i] = s[batch_i * state_size + head_i * head_size * head_size + tid * head_size + i]; } @@ -141,6 +139,57 @@ static __global__ void rwkv_wkv7_f32(const int B, const int T, const int C, cons } } +template +static __global__ void __launch_bounds__(WARP_SIZE * rows_per_block, 2) +rwkv_wkv7_f32_t1_warp_row(const int T, const int C, const int H, const float * r, const float * w, const float * k, const float * v, const float * a, const float * b, const float * s, float * dst) { + constexpr int head_size = CUDA_WKV_BLOCK_SIZE; + constexpr int half_head = head_size / 2; + + const int lane = threadIdx.x; + const int row = blockIdx.y * rows_per_block + threadIdx.y; + const int bid = blockIdx.x; + + const int batch_i = bid / H; + const int head_i = bid % H; + const int state_size = C * head_size; + const int head_off = head_i * head_size; + const int t = batch_i * C + head_off + row; + + __shared__ float _r[head_size], _w[head_size], _k[head_size], _a[head_size], _b[head_size]; + + if (threadIdx.y == 0) { + _r[lane] = r[batch_i * C + head_off + lane]; + _w[lane] = w[batch_i * C + head_off + lane]; + _k[lane] = k[batch_i * C + head_off + lane]; + _a[lane] = a[batch_i * C + head_off + lane]; + _b[lane] = b[batch_i * C + head_off + lane]; + + _r[lane + half_head] = r[batch_i * C + head_off + lane + half_head]; + _w[lane + half_head] = w[batch_i * C + head_off + lane + half_head]; + _k[lane + half_head] = k[batch_i * C + head_off + lane + half_head]; + _a[lane + half_head] = a[batch_i * C + head_off + lane + half_head]; + _b[lane + half_head] = b[batch_i * C + head_off + lane + half_head]; + } + __syncthreads(); + + const int64_t state_base = batch_i * state_size + head_i * head_size * head_size + row * head_size; + const float s0 = s[state_base + lane]; + const float s1 = s[state_base + lane + half_head]; + const float sa = warp_reduce_sum(_a[lane] * s0 + _a[lane + half_head] * s1); + + const float vt = v[t]; + const float st0 = s0 * _w[lane] + _k[lane] * vt + sa * _b[lane]; + const float st1 = s1 * _w[lane + half_head] + _k[lane + half_head] * vt + sa * _b[lane + half_head]; + const float y = warp_reduce_sum(st0 * _r[lane] + st1 * _r[lane + half_head]); + + dst[T * C + state_base + lane] = st0; + dst[T * C + state_base + lane + half_head] = st1; + + if (lane == 0) { + dst[t] = y; + } +} + void ggml_cuda_op_rwkv_wkv6(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const float * k_d = (const float *)dst->src[0]->data; const float * v_d = (const float *)dst->src[1]->data; @@ -191,7 +240,10 @@ void ggml_cuda_op_rwkv_wkv7(ggml_backend_cuda_context & ctx, ggml_tensor * dst) GGML_ASSERT(C % H == 0); GGML_ASSERT(C / H == CUDA_WKV_BLOCK_SIZE || C / H == CUDA_WKV_BLOCK_SIZE * 2); - if (C / H == CUDA_WKV_BLOCK_SIZE) { + if (T / B == 1 && C / H == CUDA_WKV_BLOCK_SIZE) { + constexpr int rows_per_block = 4; + rwkv_wkv7_f32_t1_warp_row<<>>(T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d); + } else if (C / H == CUDA_WKV_BLOCK_SIZE) { rwkv_wkv7_f32<<>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d); } else { rwkv_wkv7_f32<<>>(B, T, C, H, r_d, w_d, k_d, v_d, a_d, b_d, s_d, dst_d); diff --git a/ggml/src/ggml-et/CMakeLists.txt b/ggml/src/ggml-et/CMakeLists.txt new file mode 100644 index 0000000..ee0ee37 --- /dev/null +++ b/ggml/src/ggml-et/CMakeLists.txt @@ -0,0 +1,246 @@ + +message(STATUS "Using ET backend") + +# Configure ET platform path +if (DEFINED ENV{ET_PLATFORM}) + set(ET_PLATFORM_PATH $ENV{ET_PLATFORM}) +else() + set(ET_PLATFORM_PATH "/opt/et") +endif() + +# Use sysemu for ET backend if compiled with `-DGGML_ET_SYSEMU=ON` +if (GGML_ET_SYSEMU) + message(STATUS "Using ET backend with sysemu instead of hardware") +else() + message(STATUS "Using ET backend with hardware device") +endif() + +# Add ET platform CMake modules and config files to search paths +list(APPEND CMAKE_PREFIX_PATH ${ET_PLATFORM_PATH}/lib/cmake) +list(APPEND CMAKE_MODULE_PATH ${ET_PLATFORM_PATH}/lib/cmake) +include(aifoundry-utils/ProjectFunctions) + +message(STATUS "Using ET Platform at ${ET_PLATFORM_PATH}") + +find_package(runtime REQUIRED) + +# Kernel list +set(KERNELS + el_map_f32 + flash_attn_ext_f32 + glu_f32 + scale_f32 + mul_mat_f32 + mul_mat_f32_matrix_engine + mul_mat_id_f32 + mul_mat_id_Q4_0 + mul_mat_id_Q8_0 + mul_mat_Q8_0 + mul_mat_Q4_0 + mul_mat_Q4_0_matrix_engine + mul_mat_f16 + mul_mat_f16_matrix_engine + rope_f32 + unary_f32 + sqr_f32 + clamp_f32 + sum_rows_f32 + mean_f32 + cumsum_f32 + norm_f32 + l2_norm_f32 + group_norm_f32 + rms_norm_f32 + rms_norm_mul_f32 + softmax_f32 + im2col + get_rows_f32 + concat_f32 + repeat_f32 + rwkv_wkv6_f32 + rwkv_wkv7_f32 + gated_delta_net_f32 + cont_f32 + cont_f16 + cpy_f32_f16 + flash_attn_ext_f16_me + set_rows_f32 + set_f32 + fill_f32 + pad_f32 + diag_f32 + tri_f32 + solve_tri_f32 + ssm_conv_f32 + ssm_scan_f32 + conv_2d_f32_me + memops + uberkernel +) + +# Kernels that we support dispatch form Uberkernel +set(UBERKERNEL_SUPPORTED_KERNELS + el_map_f32 + # unary_f32 + # cpy_f32_f16 + # cont_f32 + # get_rows_f32 + concat_f32 + cont_f16 + cumsum_f32 + diag_f32 + fill_f32 + flash_attn_ext_f16_me + flash_attn_ext_f32 + gated_delta_net_f32 + glu_f32 + group_norm_f32 + im2col + l2_norm_f32 + mul_mat_f16 + mul_mat_f16_matrix_engine + mul_mat_f32 + mul_mat_f32_matrix_engine + mul_mat_id_f32 + mul_mat_Q4_0 + mul_mat_Q8_0 + norm_f32 + pad_f32 + repeat_f32 + rms_norm_f32 + rms_norm_mul_f32 + rope_f32 + rwkv_wkv6_f32 + rwkv_wkv7_f32 + scale_f32 + set_f32 + set_rows_f32 + softmax_f32 + solve_tri_f32 + sqr_f32 + # ssm_conv_f32 + ssm_scan_f32 + sum_rows_f32 + tri_f32 +) + +set(UBERKERNEL_MAP_HPP ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.h) +set(UBERKERNEL_MAP_CPP ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.cpp) + +set(UBERKERNEL_KERNELS_SORTED ${UBERKERNEL_SUPPORTED_KERNELS}) +list(SORT UBERKERNEL_KERNELS_SORTED) + +set(UBERKERNEL_ENUM_ENTRIES "") +set(UBERKERNEL_MAP_ENTRIES "") +set(_uk_idx 1) +foreach(KERNEL ${UBERKERNEL_KERNELS_SORTED}) + string(TOUPPER ${KERNEL} _uk_upper) + string(APPEND UBERKERNEL_ENUM_ENTRIES + " GGML_ET_UBERKERNEL_KERNEL_${_uk_upper} = ${_uk_idx},\n") + string(APPEND UBERKERNEL_MAP_ENTRIES + " {\"${KERNEL}\", GGML_ET_UBERKERNEL_KERNEL_${_uk_upper}},\n") + math(EXPR _uk_idx "${_uk_idx} + 1") +endforeach() + +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-et-uberkernel-kernel-map.h.in + ${UBERKERNEL_MAP_HPP} + @ONLY) +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-et-uberkernel-kernel-map.cpp.in + ${UBERKERNEL_MAP_CPP} + @ONLY) + +add_custom_target(et-uberkernel-map + DEPENDS ${UBERKERNEL_MAP_HPP} ${UBERKERNEL_MAP_CPP} +) + +# Build ET kernels (cross-compiled in subdirectory scope) +add_subdirectory(et-kernels) + +# Embed kernels into C++ source +set(EMBED_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed_one_kernel.cmake) +set(EMBED_HPP ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-kernels-embed.hpp) +set(EMBED_CPP ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-kernels-embed.cpp) +set(EMBED_DIR ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/embed) +file(MAKE_DIRECTORY ${EMBED_DIR}) + +set(EMBED_KERNEL_SOURCES) +set(EMBED_EXTERNS "") +set(EMBED_MAP_ENTRIES "") + +foreach(KERNEL ${KERNELS}) + set(ELF_PATH ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/${KERNEL}.elf) + set(OUT_CPP ${EMBED_DIR}/${KERNEL}.cpp) + + add_custom_command( + OUTPUT ${OUT_CPP} + COMMAND ${CMAKE_COMMAND} + -DELF_FILE=${ELF_PATH} + -DOUT_FILE=${OUT_CPP} + -DVAR_NAME=${KERNEL} + -P ${EMBED_SCRIPT} + DEPENDS ${KERNEL}.elf ${EMBED_SCRIPT} + COMMENT "Embedding ${KERNEL}.elf" + VERBATIM + ) + list(APPEND EMBED_KERNEL_SOURCES ${OUT_CPP}) + + string(APPEND EMBED_EXTERNS + "extern unsigned char ${KERNEL}_data[];\n" + "extern const uint64_t ${KERNEL}_len;\n") + string(APPEND EMBED_MAP_ENTRIES + " {\"${KERNEL}\", {${KERNEL}_data, ${KERNEL}_len}},\n") +endforeach() + +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-et-kernels-embed.hpp.in + ${EMBED_HPP} + @ONLY) +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-et-kernels-embed.cpp.in + ${EMBED_CPP} + @ONLY) + +add_custom_target(et-kernels-embed ALL + DEPENDS ${EMBED_KERNEL_SOURCES} ${EMBED_HPP} ${EMBED_CPP} et-uberkernel-map +) + +ggml_add_backend_library(ggml-et + ggml-et.cpp + ggml-et-kernels.cpp + ggml-et-memops.cpp + ggml-et-ops.cpp + ggml-et-cpu-compare.cpp + ) + +# Mark generated files as such +set_source_files_properties( + ${EMBED_CPP} + ${EMBED_HPP} + ${EMBED_KERNEL_SOURCES} + ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.cpp + ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.h + PROPERTIES GENERATED TRUE +) + +# Add embedded kernel sources +target_sources(ggml-et PRIVATE + ${EMBED_CPP} + ${EMBED_HPP} + ${EMBED_KERNEL_SOURCES} + ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.cpp + ${CMAKE_CURRENT_BINARY_DIR}/et-kernels/ggml-et-uberkernel-kernel-map.h +) + +# Include directory for embedded headers +target_include_directories(ggml-et PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/et-kernels) + +target_link_libraries(ggml-et PRIVATE runtime::etrt_static deviceLayer::deviceLayer) +target_compile_definitions(ggml-et PRIVATE GGML_ET_UBERKERNEL_HOST_LOOKUP) +if (GGML_ET_SYSEMU) + target_compile_definitions(ggml-et PRIVATE GGML_ET_SYSEMU=1) +endif() + +# Ensure kernels are built and embedded before the backend library +add_dependencies(ggml-et et-kernels-embed et-uberkernel-map) diff --git a/ggml/src/ggml-et/cmake/embed_one_kernel.cmake b/ggml/src/ggml-et/cmake/embed_one_kernel.cmake new file mode 100644 index 0000000..cc01ecb --- /dev/null +++ b/ggml/src/ggml-et/cmake/embed_one_kernel.cmake @@ -0,0 +1,15 @@ +# Inputs (via -D): +# ELF_FILE - path to source .elf +# OUT_FILE - path to output .cpp +# VAR_NAME - C symbol base name (kernel name) + +file(READ "${ELF_FILE}" HEX HEX) +string(LENGTH "${HEX}" HEX_LEN) +math(EXPR SIZE "${HEX_LEN} / 2") +string(REGEX REPLACE "(..)" "0x\\1," BYTES "${HEX}") + +file(WRITE "${OUT_FILE}" +"// Auto-generated by embed_one_kernel.cmake. Do not edit.\n" +"#include \n" +"unsigned char ${VAR_NAME}_data[${SIZE}] = { ${BYTES} };\n" +"extern const uint64_t ${VAR_NAME}_len = ${SIZE};\n") diff --git a/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in b/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in new file mode 100644 index 0000000..95f6e40 --- /dev/null +++ b/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in @@ -0,0 +1,6 @@ +// Auto-generated kernel embeddings. Do not edit. +#include "ggml-et-kernels-embed.hpp" + +const std::unordered_map> ggml_et_embedded_kernels = { +@EMBED_MAP_ENTRIES@ +}; diff --git a/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in b/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in new file mode 100644 index 0000000..dd2c6ab --- /dev/null +++ b/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in @@ -0,0 +1,12 @@ +// Auto-generated kernel embeddings. Do not edit. +#pragma once + +#include +#include +#include +#include + +@EMBED_EXTERNS@ + +// Kernel name -> (data, length) lookup map +extern const std::unordered_map> ggml_et_embedded_kernels; diff --git a/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in b/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in new file mode 100644 index 0000000..ccee5d4 --- /dev/null +++ b/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in @@ -0,0 +1,18 @@ +// Auto-generated uberkernel kernel-id mapping. Do not edit. +#include "ggml-et-uberkernel-kernel-map.h" + +#ifdef GGML_ET_UBERKERNEL_HOST_LOOKUP +#include +#include + +uint16_t ggml_et_uberkernel_kernel_id_from_name(const char * kernel_name) { + if (kernel_name == nullptr) { + return GGML_ET_UBERKERNEL_KERNEL_INVALID; + } + static const std::unordered_map kernel_id_map = { +@UBERKERNEL_MAP_ENTRIES@ + }; + auto it = kernel_id_map.find(std::string(kernel_name)); + return it == kernel_id_map.end() ? GGML_ET_UBERKERNEL_KERNEL_INVALID : it->second; +} +#endif diff --git a/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in b/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in new file mode 100644 index 0000000..cebfb8a --- /dev/null +++ b/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in @@ -0,0 +1,13 @@ +// Auto-generated uberkernel kernel-id mapping. Do not edit. +#pragma once + +#include + +enum ggml_et_uberkernel_kernel_id { + GGML_ET_UBERKERNEL_KERNEL_INVALID = 0, +@UBERKERNEL_ENUM_ENTRIES@ +}; + +#ifdef GGML_ET_UBERKERNEL_HOST_LOOKUP +uint16_t ggml_et_uberkernel_kernel_id_from_name(const char * kernel_name); +#endif diff --git a/ggml/src/ggml-et/et-kernels/CMakeLists.txt b/ggml/src/ggml-et/et-kernels/CMakeLists.txt new file mode 100644 index 0000000..4b6baab --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/CMakeLists.txt @@ -0,0 +1,137 @@ +# ggml-et: Device kernels (cross-compiled within the main build) +# +# The RISC-V toolchain is set up in-scope so these targets use the +# cross-compiler while the rest of the build uses the host compiler. +# This keeps kernels in compile_commands.json for full IDE support. + +# --- RISC-V toolchain setup (scoped to this directory) --- +set(TOOLCHAIN_DIR ${ET_PLATFORM_PATH}) +include(${ET_PLATFORM_PATH}/lib/cmake/riscv64-ec-toolchain.cmake) +set(CMAKE_ADDR2LINE "${TOOLCHAIN_DIR}/bin/riscv64-unknown-elf-addr2line") +set(CMAKE_LINKER_TYPE LLD) + +# Ensure kernels are built in this directory even if a global output directory is set +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +message(STATUS "ET kernels using RISC-V toolchain at: ${TOOLCHAIN_DIR}") + +# DeviceUtils provides the add_riscv_executable macro +list(APPEND CMAKE_MODULE_PATH "${ET_PLATFORM_PATH}/lib/cmake/cmake-modules") +list(APPEND CMAKE_PREFIX_PATH "${ET_PLATFORM_PATH}/lib/cmake") +include(DeviceUtils) + +find_package(et-common-libs REQUIRED) +find_package(esperantoTrace REQUIRED) + +# --- Kernel configuration --- +if(NOT DEFINED ADDRESS) + set(ADDRESS "0x8005801000") + message(STATUS "ADDRESS not specified, using default: ${ADDRESS}") +endif() + +set(LINKER_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/src/linker.ld) +set(CHECK_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/scripts/check_unimplemented_instructions.sh) + +# Track address changes to trigger relinking +set(ADDRESS_FILE ${CMAKE_CURRENT_BINARY_DIR}/et_address.txt) +file(CONFIGURE OUTPUT ${ADDRESS_FILE} CONTENT "${ADDRESS}" @ONLY) + +# KERNELS defined in upper CMakeLists.txt +foreach(KERNEL ${KERNELS}) + add_riscv_executable(${KERNEL}) + target_sources(${KERNEL}.elf PRIVATE + src/${KERNEL}.c + src/crt.S + ) + target_include_directories(${KERNEL}.elf PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/.. + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/ggml/include + ${CMAKE_SOURCE_DIR}/ggml/src + ) + target_link_libraries(${KERNEL}.elf PRIVATE et-common-libs::cm-umode) + # C-only flags — must not apply to .S files + target_compile_options(${KERNEL}.elf PRIVATE + $<$:-fno-zero-initialized-in-bss> + $<$:-ffreestanding> + $<$:-std=gnu99> + $<$:-ffat-lto-objects> + $<$:-mcmodel=medany> + $<$:-mabi=lp64f> + $<$:-march=rv64imf> + $<$:-ffunction-sections> + $<$:-fdata-sections> + $<$:-O3> + $<$:-g0> + $<$:-nostdlib> + $<$:-ffreestanding> + ) + target_link_options(${KERNEL}.elf PRIVATE + -Wl,--defsym=BASE_ADDRESS=${ADDRESS} + -Wl,--entry=_start + ) + # Append to LINK_DEPENDS (macro already sets it for the linker script) + set_property(TARGET ${KERNEL}.elf APPEND PROPERTY + LINK_DEPENDS "${ADDRESS_FILE}" + ) + + # Post-build: strip and check (fails build if check script fails) + add_custom_command(TARGET ${KERNEL}.elf POST_BUILD + COMMAND ${CMAKE_STRIP} --strip-debug $ + COMMAND ${CHECK_SCRIPT} + ${CMAKE_OBJDUMP} ${CMAKE_ADDR2LINE} $ + DEPENDS ${CHECK_SCRIPT} + VERBATIM + ) +endforeach() + +add_dependencies(uberkernel.elf et-uberkernel-map) + +# Each supported kernel is compiled in its own translation unit with +# -Dentry_point=_entry +# so symbols and macros don't leak between kernels. The dispatcher +# (uberkernel.c) calls the renamed entries via extern declarations. +# +# HACK: we need to supresse _me kernels from setting up SCP themselves +set(_UBER_ME_KERNELS mul_mat_f16_matrix_engine mul_mat_f32_matrix_engine flash_attn_ext_f16_me) + +foreach(UK_KERNEL ${UBERKERNEL_SUPPORTED_KERNELS}) + set(_obj uber_${UK_KERNEL}) + add_library(${_obj} OBJECT src/${UK_KERNEL}.c) + target_compile_definitions(${_obj} PRIVATE "entry_point=${UK_KERNEL}_entry" ET_UBERKERNEL) + target_include_directories(${_obj} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/.. + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/ggml/include + ${CMAKE_SOURCE_DIR}/ggml/src + ) + target_link_libraries(${_obj} PRIVATE et-common-libs::cm-umode) + target_compile_options(${_obj} PRIVATE + $<$:-fno-zero-initialized-in-bss> + $<$:-ffreestanding> + $<$:-std=gnu99> + $<$:-ffat-lto-objects> + $<$:-mcmodel=medany> + $<$:-mabi=lp64f> + $<$:-march=rv64imf> + $<$:-ffunction-sections> + $<$:-fdata-sections> + $<$:-O3> + $<$:-g0> + $<$:-nostdlib> + ) + # ME kernels: suppress setup_cache_scp() (called once by the dispatcher) + if(UK_KERNEL IN_LIST _UBER_ME_KERNELS) + target_compile_definitions(${_obj} PRIVATE UBERKERNEL_SUPPRESS_SCP_SETUP) + endif() + target_sources(uberkernel.elf PRIVATE $) +endforeach() + +# Print summary +message(STATUS "GGML ET Kernels configured:") +foreach(KERNEL ${KERNELS}) + message(STATUS " - ${KERNEL}") +endforeach() +message(STATUS "Base address: ${ADDRESS}") diff --git a/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh b/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh new file mode 100755 index 0000000..83f7992 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +OBJDUMP=$1 +ADDR2LINE=$2 +TARGET_DEBUG=$3 +TARGET_ASM=${TARGET_DEBUG}.S +BAD_INST_FILE=${TARGET_DEBUG}-BAD-INST.log + +# grep expression to find unimplemented instructions +UNIMPLEMENTED_EXPR="fdiv.s\\|fsqrt.s\\|fcvt.l.s\\|fcvt.lu.s\\|fcvt.s.l\\|fcvt.s.lu\\|fdiv.pi\\|fdivu.pi\\|fremu.pi\\|frem.pi\\|fdiv.ps\\|fsqrt.ps\\|frsq.ps\\|fsin.ps" + +# dump assembly into .S file +${OBJDUMP} -lwdSC ${TARGET_DEBUG} > ${TARGET_ASM} + +# check with grep for unimplemented instructions +# Note: The exit status is 0 if selected lines are found, and 1 if not found. +grep ${UNIMPLEMENTED_EXPR} ${TARGET_ASM} > /dev/null +ret=$? + +if [ ${ret} -eq 0 ] +then + # unimplemented instructions are found + echo -e "BUILD ERROR: Executable file ${TARGET_DEBUG} contains unimplemented instructions. Please review the lines of code listed in ${BAD_INST_FILE}" + echo -e "\t For further details, please read paragraph 3.4 of the ETSoC-1 Programmer's Reference Manual (PRM)" + + # addr2line + grep ${UNIMPLEMENTED_EXPR} ${TARGET_ASM} | cut -d: -f 1 | ${ADDR2LINE} -i -e ${TARGET_DEBUG} > ${BAD_INST_FILE} + grep ${UNIMPLEMENTED_EXPR} ${TARGET_ASM} >> ${BAD_INST_FILE} + echo "------------------------------------------------------------" + cat ${BAD_INST_FILE} + echo "------------------------------------------------------------" + exit 1 + +else + rm -f ${BAD_INST_FILE} +fi diff --git a/ggml/src/ggml-et/et-kernels/src/RunBackend.sh b/ggml/src/ggml-et/et-kernels/src/RunBackend.sh new file mode 100644 index 0000000..b302e2a --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/RunBackend.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +LOG="llama_bench_$(date +%Y%m%d_%H%M%S).log" + +{ + echo "===== START =====" + date + hostname + uname -a + echo "Command:" + echo "./build/bin/llama-bench -m ../../models/Llama-3.2-1B-Instruct-Q8_0.gguf -fa 0 -p 32,64,128,256,512 -n 32,64,128,256,512" + echo "=================" + + ./build/bin/llama-bench \ + -m ../../models/Llama-3.2-1B-Instruct-Q8_0.gguf \ + -fa 0 \ + -p 32,64,128,256,512 \ + -n 32,64,128,256,512 + + echo "===== END =====" + date +} 2>&1 | tee "$LOG" diff --git a/ggml/src/ggml-et/et-kernels/src/block_ops.h b/ggml/src/ggml-et/et-kernels/src/block_ops.h new file mode 100644 index 0000000..78ffbde --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/block_ops.h @@ -0,0 +1,997 @@ +//****************************************************************************** +// ET Vectorized Block Operations Library +// Provides optimized block-level operations using ET hardware vector instructions +//****************************************************************************** + +#ifndef BLOCK_OPS_H +# define BLOCK_OPS_H + +# include "math_fp.h" +# include "quants.h" + +# include + +//****************************************************************************** +// Block Dot Product Operations +//****************************************************************************** +inline void __attribute__((always_inline)) excl_mode(uint64_t val) { + __asm__ __volatile__("csrw 0x7d3, %[csr_enc]\n" : : [csr_enc] "r"(val) : "x31"); +} + +static inline float compute_block_dot_product_q4_0(const block_q4_0 * a_block, const float * b_col_start) { + // Set mask register to enable all 8 vector elements + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); // Save current mask + __asm__ volatile("mov.m.x m0, x0, 0xFF"); // Enable all 8 elements + + // Use f10 as accumulator, init to 0 + __asm__ volatile("fbci.ps f10, 0" ::: "f10"); + + static const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + __asm__ volatile("flw.ps f31, %[gather]\n" : : [gather] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); + + // Process 32 elements in 2 chunks of 16 elements (8 bytes) each + for (int chunk = 0; chunk < 2; chunk++) { + int offset_a = chunk * 8; + int offset_b_low = chunk * 8; // Activations for lower nibbles + int offset_b_high = chunk * 8 + 16; // Activations for upper nibbles (16 elements later) + + __asm__ volatile( + "fgb.ps f11, f31(%[a_ptr])\n" // Gather 8 bytes (16 packed q4_0 weights) + + // 1. Extract & Multiply Lower Nibbles + "fandi.pi f12, f11, 15\n" // Mask lower 4 bits (x & 0xF) + "faddi.pi f12, f12, -8\n" // GGML offset to signed: (x & 0xF) - 8 + "fcvt.ps.pw f12, f12, rne\n" // Convert INT32 to FP32 + "flw.ps f13, 0(%[b_low])\n" // Load 8 B values (floats) + "fmadd.ps f10, f12, f13, f10, rne\n" // acc += A_low * B_low + + // 2. Extract & Multiply Upper Nibbles + "fsrli.pi f14, f11, 4\n" // Shift upper 4 bits down + "fandi.pi f14, f14, 15\n" // Mask new lower 4 bits + "faddi.pi f14, f14, -8\n" // GGML offset to signed + "fcvt.ps.pw f14, f14, rne\n" // Convert INT32 to FP32 + "flw.ps f15, 0(%[b_high])\n" // Load next 8 B values (floats) + "fmadd.ps f10, f14, f15, f10, rne\n" // acc += A_high * B_high + : + : [a_ptr] "r"(&a_block->qs[offset_a]), [b_low] "r"(&b_col_start[offset_b_low]), + [b_high] "r"(&b_col_start[offset_b_high]) + // Note: f10 is explicitly NOT listed in the clobbers here to ensure the compiler + // preserves the running sum across C loop iterations safely. + : "f11", "f12", "f13", "f14", "f15"); + } + + // Horizontal sum: reduce f10 into a single scalar + float final_sum; + __asm__ __volatile__( + // Pairwise sum within each 128-bit half + "fswizz.ps f1, f10, 0xB1 \n\t" // Swaps: e0<->e1 and e2<->e3 + "fadd.ps f2, f10, f1, rne \n\t" + // Complete the sum for each 128-bit half + "fswizz.ps f3, f2, 0x4E \n\t" // Swaps: e0,e1 <-> e2,e3 + "fadd.ps f4, f2, f3, rne \n\t" + // Sum across the two 128b halfs + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(final_sum)::"t0", "f1", "f2", "f3", "f4", "f5", "f10"); + + // Restore original mask + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + + const float scale = fp16_to_fp32(a_block->d); + return final_sum * scale; +} + +// Compute dot product between dequantized q8_0 block and f32 column vector +// Vectorized: processes 8 elements at a time using ET vector instructions +// Block size: 32 int8 values (QK8_0) +static inline float compute_block_dot_product_q8_0(const block_q8_0 * a_block, const float * b_col_start) { + // Set mask register to enable all 8 vector elements + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); // Save current mask + __asm__ volatile("mov.m.x m0, x0, 0xFF"); // Enable all 8 elements + __asm__ volatile("fbci.pi f10, 0" ::: "f10"); // Use f10 as accumulator, init to 0 + + static const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + + __asm__ volatile("flw.ps f31, %[gather]\n" : : [gather] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); + + // Process 32 elements in 4 chunks of 8 elements each + for (int chunk = 0; chunk < 4; chunk++) { + int offset = chunk << 3; // chunk * 8 + + __asm__ volatile( + "flw.ps f12, %[b_vec]\n" // Load 8 B values (floats) + "fgb.ps f11, f31(%[a_ptr])\n" // Gather 8 int8 bytes from A using pattern + "fcvt.ps.pw f11, f11\n" // Convert int8 vector to float vector + "fmadd.ps f10, f11, f12, f10\n" // acc += a_vec * b_vec (8-wide) + : + : [a_ptr] "r"(&a_block->qs[offset]), [b_vec] "m"(*(const float (*)[8]) & b_col_start[offset]), + [scale] "m"(a_block->d) + : "f10", "f11", "f12"); + } + + // Horizontal sum: reduce f10 into a single scalar + float final_sum; + __asm__ __volatile__( + // Pairwise sum within each 128-bit half + "fswizz.ps f1, f10, 0xB1 \n\t" // Swaps: e0<->e1 and e2<->e3 + "fadd.ps f2, f10, f1, rne \n\t" + // Complete the sum for each 128-bit half + "fswizz.ps f3, f2, 0x4E \n\t" // Swaps: e0,e1 <-> e2,e3 + "fadd.ps f4, f2, f3, rne \n\t" + // Sum across the two 128b halfs + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(final_sum)::"t0", "f10", "f2", "f3", "f4", "f5"); + + // Restore original mask + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + + const float scale = fp16_to_fp32(a_block->d); + return final_sum * scale; +} + +//****************************************************************************** +// Split-phase Q8_0 dot product API +// +// q8_dot_begin(st) — save mask, set mask 0xFF +// q8_dot_reset() — zero vector accumulator f20 +// q8_dot_tile(q, b, n) — accumulate n Q8_0 blocks into f20 +// q8_dot_reduce() — horizontal sum of f20, return scalar float +// q8_dot_teardown(st) — restore original mask +// +// Register contract: +// f20 — row accumulator (persistent across tiles, reset per row) +// f31 — gather pattern (reloaded per q8_dot_tile call) +// f10-f12 — scratch within tile +// f15 — scale broadcast within tile +// f1-f5, t0 — scratch within reduce +//****************************************************************************** + +static inline void __attribute__((always_inline)) q8_dot_reset(void) { + __asm__ volatile("fbci.pi f20, 0" ::: "f20"); +} + +// Accumulate n_blocks Q8_0 blocks into f20. +// Uses fg32b.ps (fast gather with scalar pattern) for aligned chunks, +// falls back to fgb.ps for chunks crossing a 32-byte boundary. +static inline void __attribute__((always_inline)) q8_dot_tile(const block_q8_0 * q_row, + const float * b_col, + int64_t n_blocks) { + const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + const uint64_t gather_0_to_7 = 0x398a418820ULL; + + __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); + + for (int64_t kb = 0; kb < n_blocks; kb++) { + const block_q8_0 * blk = q_row + kb; + const float * b_ptr = b_col + (kb << 5); + const uintptr_t qs_addr = (uintptr_t) blk->qs; + const uintptr_t qs_aligned = qs_addr & ~(uintptr_t) 31; + const uintptr_t qs_low = qs_addr & 31; + const int fast_chunks = (int) ((32 - qs_low) >> 3); + + if (fast_chunks >= 3) { + __asm__ volatile( + "fbci.pi f10, 0\n" + "flw.ps f12, %[bv0]\n" + "fg32b.ps f11, %[gi](%[ap0])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv1]\n" + "fg32b.ps f11, %[gi](%[ap1])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv2]\n" + "fg32b.ps f11, %[gi](%[ap2])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv3]\n" + "fgb.ps f11, f31(%[ap3])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + : + : [gi] "r"(gather_0_to_7), [ap0] "r"(qs_addr), [ap1] "r"(qs_aligned | ((qs_addr + 8) & 31)), + [ap2] "r"(qs_aligned | ((qs_addr + 16) & 31)), [ap3] "r"(&blk->qs[24]), + [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), + [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12"); + } else if (fast_chunks == 2) { + __asm__ volatile( + "fbci.pi f10, 0\n" + "flw.ps f12, %[bv0]\n" + "fg32b.ps f11, %[gi](%[ap0])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv1]\n" + "fg32b.ps f11, %[gi](%[ap1])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv2]\n" + "fgb.ps f11, f31(%[ap2])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv3]\n" + "fgb.ps f11, f31(%[ap3])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + : + : [gi] "r"(gather_0_to_7), [ap0] "r"(qs_addr), [ap1] "r"(qs_aligned | ((qs_addr + 8) & 31)), + [ap2] "r"(&blk->qs[16]), [ap3] "r"(&blk->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), + [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), + [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12"); + } else if (fast_chunks == 1) { + __asm__ volatile( + "fbci.pi f10, 0\n" + "flw.ps f12, %[bv0]\n" + "fg32b.ps f11, %[gi](%[ap0])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv1]\n" + "fgb.ps f11, f31(%[ap1])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv2]\n" + "fgb.ps f11, f31(%[ap2])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv3]\n" + "fgb.ps f11, f31(%[ap3])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + : + : [gi] "r"(gather_0_to_7), [ap0] "r"(qs_addr), [ap1] "r"(&blk->qs[8]), [ap2] "r"(&blk->qs[16]), + [ap3] "r"(&blk->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), + [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), + [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12"); + } else { + __asm__ volatile( + "fbci.pi f10, 0\n" + "flw.ps f12, %[bv0]\n" + "fgb.ps f11, f31(%[ap0])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv1]\n" + "fgb.ps f11, f31(%[ap1])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv2]\n" + "fgb.ps f11, f31(%[ap2])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + "flw.ps f12, %[bv3]\n" + "fgb.ps f11, f31(%[ap3])\n" + "fcvt.ps.pw f11, f11\n" + "fmadd.ps f10, f11, f12, f10\n" + : + : [ap0] "r"(&blk->qs[0]), [ap1] "r"(&blk->qs[8]), [ap2] "r"(&blk->qs[16]), [ap3] "r"(&blk->qs[24]), + [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), + [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12"); + } + + // f20 += f10 * broadcast(scale) — hardware fp16→fp32 via FCVT.PS.F16 + uint32_t scale_raw = (uint32_t) blk->d; + __asm__ volatile( + "fbcx.ps f15, %[sb]\n" + "fcvt.ps.f16 f15, f15\n" + "fmadd.ps f20, f10, f15, f20\n" + : + : [sb] "r"(scale_raw) + : "f15", "f20"); + } +} + +// Horizontal sum of 8-element vector accumulator f20. +static inline float __attribute__((always_inline)) q8_dot_reduce(void) { + float result; + __asm__ __volatile__( + "fswizz.ps f1, f20, 0xB1 \n\t" + "fadd.ps f2, f20, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(result)::"t0", "f1", "f2", "f3", "f4", "f5"); + return result; +} + +// Full-row dot product (convenience wrapper) +static inline float compute_row_dot_q8_0(const block_q8_0 * q_row, const float * b_col, int64_t K_blocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + q8_dot_reset(); + q8_dot_tile(q_row, b_col, K_blocks); + float result = q8_dot_reduce(); + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return result; +} + +//****************************************************************************** +// Hoisted Q8_0 dot API +// +// q8_dot_begin/end save/restore the vector mask once around a long sequence of +// dot products, so the per-row mask shuffles are hoisted out of the inner +// loops. q8_dot_compute does a full-row dot (no mask handling). The _x2 +// variant computes two rows together while reusing each loaded B chunk — +// only safe when both row pointers share the same 32-byte alignment phase +// (i.e. the Q8 row stride is a multiple of 32). +//****************************************************************************** + +typedef struct { + unsigned long saved_mask; +} q8_dot_state; + +static inline void q8_dot_begin(q8_dot_state * state) { + __asm__ volatile("mova.x.m %0" : "=r"(state->saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); +} + +static inline void q8_dot_end(const q8_dot_state * state) { + __asm__ volatile("mova.m.x %0" ::"r"(state->saved_mask)); +} + +// Equivalent to q8_dot_reset+tile+reduce, without touching the mask register. +// Caller is responsible for q8_dot_begin/end around the surrounding loop. +static inline float q8_dot_compute(const block_q8_0 * q_row, const float * b_col, int64_t K_blocks) { + q8_dot_reset(); + q8_dot_tile(q_row, b_col, K_blocks); + return q8_dot_reduce(); +} + +// Compute two row dots together while reusing the same loaded B chunks. +// +// Safe when every row starts at the same 32-byte offset, i.e. the Q8 row stride +// is a multiple of 32. In that case the gather/alignment pattern is the same +// for both rows at a given `kb`, so one set of B vector loads feeds both row +// accumulators. +static inline void q8_dot_compute_x2_aligned(const block_q8_0 * q_row0, + const block_q8_0 * q_row1, + const float * b_col, + int64_t K_blocks, + float * out0, + float * out1) { + const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + const uint64_t gather_0_to_7 = 0x398a418820ULL; + __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); + __asm__ volatile( + "fbci.pi f20, 0\n" + "fbci.pi f21, 0\n" :: + : "f20", "f21"); + + for (int64_t kb = 0; kb < K_blocks; kb++) { + const block_q8_0 * blk0 = q_row0 + kb; + const block_q8_0 * blk1 = q_row1 + kb; + const float * b_ptr = b_col + (kb << 5); + + const uintptr_t qs_addr0 = (uintptr_t) blk0->qs; + const uintptr_t qs_addr1 = (uintptr_t) blk1->qs; + const uintptr_t qs_aligned0 = qs_addr0 & ~(uintptr_t) 31; + const uintptr_t qs_aligned1 = qs_addr1 & ~(uintptr_t) 31; + const int fast_chunks = (int) ((32 - (qs_addr0 & 31)) >> 3); + + if (fast_chunks >= 3) { + __asm__ volatile( + "fbci.pi f10, 0\n" + "fbci.pi f11, 0\n" + + "flw.ps f12, %[bv0]\n" + "fg32b.ps f16, %[gi](%[r0ap0])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f12, f10\n" + "fg32b.ps f17, %[gi](%[r1ap0])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f12, f11\n" + + "flw.ps f13, %[bv1]\n" + "fg32b.ps f16, %[gi](%[r0ap1])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f13, f10\n" + "fg32b.ps f17, %[gi](%[r1ap1])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f13, f11\n" + + "flw.ps f14, %[bv2]\n" + "fg32b.ps f16, %[gi](%[r0ap2])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f14, f10\n" + "fg32b.ps f17, %[gi](%[r1ap2])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f14, f11\n" + + "flw.ps f15, %[bv3]\n" + "fgb.ps f16, f31(%[r0ap3])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f15, f10\n" + "fgb.ps f17, f31(%[r1ap3])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f15, f11\n" + : + : [gi] "r"(gather_0_to_7), [r0ap0] "r"(qs_addr0), [r0ap1] "r"(qs_aligned0 | ((qs_addr0 + 8) & 31)), + [r0ap2] "r"(qs_aligned0 | ((qs_addr0 + 16) & 31)), [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(qs_addr1), + [r1ap1] "r"(qs_aligned1 | ((qs_addr1 + 8) & 31)), [r1ap2] "r"(qs_aligned1 | ((qs_addr1 + 16) & 31)), + [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), + [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), + [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); + } else if (fast_chunks == 2) { + __asm__ volatile( + "fbci.pi f10, 0\n" + "fbci.pi f11, 0\n" + + "flw.ps f12, %[bv0]\n" + "fg32b.ps f16, %[gi](%[r0ap0])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f12, f10\n" + "fg32b.ps f17, %[gi](%[r1ap0])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f12, f11\n" + + "flw.ps f13, %[bv1]\n" + "fg32b.ps f16, %[gi](%[r0ap1])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f13, f10\n" + "fg32b.ps f17, %[gi](%[r1ap1])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f13, f11\n" + + "flw.ps f14, %[bv2]\n" + "fgb.ps f16, f31(%[r0ap2])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f14, f10\n" + "fgb.ps f17, f31(%[r1ap2])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f14, f11\n" + + "flw.ps f15, %[bv3]\n" + "fgb.ps f16, f31(%[r0ap3])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f15, f10\n" + "fgb.ps f17, f31(%[r1ap3])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f15, f11\n" + : + : [gi] "r"(gather_0_to_7), [r0ap0] "r"(qs_addr0), [r0ap1] "r"(qs_aligned0 | ((qs_addr0 + 8) & 31)), + [r0ap2] "r"(&blk0->qs[16]), [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(qs_addr1), + [r1ap1] "r"(qs_aligned1 | ((qs_addr1 + 8) & 31)), [r1ap2] "r"(&blk1->qs[16]), + [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), + [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), + [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); + } else if (fast_chunks == 1) { + __asm__ volatile( + "fbci.pi f10, 0\n" + "fbci.pi f11, 0\n" + + "flw.ps f12, %[bv0]\n" + "fg32b.ps f16, %[gi](%[r0ap0])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f12, f10\n" + "fg32b.ps f17, %[gi](%[r1ap0])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f12, f11\n" + + "flw.ps f13, %[bv1]\n" + "fgb.ps f16, f31(%[r0ap1])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f13, f10\n" + "fgb.ps f17, f31(%[r1ap1])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f13, f11\n" + + "flw.ps f14, %[bv2]\n" + "fgb.ps f16, f31(%[r0ap2])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f14, f10\n" + "fgb.ps f17, f31(%[r1ap2])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f14, f11\n" + + "flw.ps f15, %[bv3]\n" + "fgb.ps f16, f31(%[r0ap3])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f15, f10\n" + "fgb.ps f17, f31(%[r1ap3])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f15, f11\n" + : + : [gi] "r"(gather_0_to_7), [r0ap0] "r"(qs_addr0), [r0ap1] "r"(&blk0->qs[8]), [r0ap2] "r"(&blk0->qs[16]), + [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(qs_addr1), [r1ap1] "r"(&blk1->qs[8]), + [r1ap2] "r"(&blk1->qs[16]), [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), + [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), + [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); + } else { + __asm__ volatile( + "fbci.pi f10, 0\n" + "fbci.pi f11, 0\n" + + "flw.ps f12, %[bv0]\n" + "fgb.ps f16, f31(%[r0ap0])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f12, f10\n" + "fgb.ps f17, f31(%[r1ap0])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f12, f11\n" + + "flw.ps f13, %[bv1]\n" + "fgb.ps f16, f31(%[r0ap1])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f13, f10\n" + "fgb.ps f17, f31(%[r1ap1])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f13, f11\n" + + "flw.ps f14, %[bv2]\n" + "fgb.ps f16, f31(%[r0ap2])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f14, f10\n" + "fgb.ps f17, f31(%[r1ap2])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f14, f11\n" + + "flw.ps f15, %[bv3]\n" + "fgb.ps f16, f31(%[r0ap3])\n" + "fcvt.ps.pw f16, f16\n" + "fmadd.ps f10, f16, f15, f10\n" + "fgb.ps f17, f31(%[r1ap3])\n" + "fcvt.ps.pw f17, f17\n" + "fmadd.ps f11, f17, f15, f11\n" + : + : [r0ap0] "r"(&blk0->qs[0]), [r0ap1] "r"(&blk0->qs[8]), [r0ap2] "r"(&blk0->qs[16]), + [r0ap3] "r"(&blk0->qs[24]), [r1ap0] "r"(&blk1->qs[0]), [r1ap1] "r"(&blk1->qs[8]), + [r1ap2] "r"(&blk1->qs[16]), [r1ap3] "r"(&blk1->qs[24]), [bv0] "m"(*(const float (*)[8]) & b_ptr[0]), + [bv1] "m"(*(const float (*)[8]) & b_ptr[8]), [bv2] "m"(*(const float (*)[8]) & b_ptr[16]), + [bv3] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); + } + + const uint32_t scale_raw0 = (uint32_t) blk0->d; + const uint32_t scale_raw1 = (uint32_t) blk1->d; + __asm__ volatile( + "fbcx.ps f24, %[s0]\n" + "fcvt.ps.f16 f24, f24\n" + "fmadd.ps f20, f10, f24, f20\n" + "fbcx.ps f25, %[s1]\n" + "fcvt.ps.f16 f25, f25\n" + "fmadd.ps f21, f11, f25, f21\n" + : + : [s0] "r"(scale_raw0), [s1] "r"(scale_raw1) + : "f20", "f21", "f24", "f25"); + } + + float result0; + float result1; + __asm__ __volatile__( + "fswizz.ps f1, f20, 0xB1 \n\t" + "fadd.ps f2, f20, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(result0)::"t0", "f1", "f2", "f3", "f4", "f5"); + __asm__ __volatile__( + "fswizz.ps f1, f21, 0xB1 \n\t" + "fadd.ps f2, f21, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(result1)::"t0", "f1", "f2", "f3", "f4", "f5"); + + *out0 = result0; + *out1 = result1; +} + +// Compute dot product between f16 block and f32 column vector (NAIVE VERSION) +// Scalar implementation for debugging - no vectorization +// Block size: 32 f16 values (64 bytes = 1 cache line) +static inline float compute_block_dot_product_f16_naive(const uint16_t * a_block, const float * b_col_start) { + float acc_vec[8] __attribute__((aligned(32))) = { 0.0f }; + // Byte offsets for 16-bit (half-word) elements + static const int32_t gather_pattern[8] = { 0, 2, 4, 6, 8, 10, 12, 14 }; + unsigned long temp_mask; + + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + // Load the pattern once into f31 for the duration of all 4 chunks + __asm__ volatile("flw.ps f31, %[gather]\n" : : [gather] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); + + for (int chunk = 0; chunk < 4; chunk++) { + // Correct pointers: + // a_block elements are 2 bytes, b_col elements are 4 bytes + const uint16_t * a_ptr = &a_block[chunk << 3]; // chunk * 8 + const float * b_ptr = &b_col_start[chunk << 3]; // chunk * 8 + + __asm__ volatile( + "flw.ps f10, %[acc]\n" + "fgh.ps f11, f31(%[a_p])\n" // Uses {0,2,4,6,8,10,12,14} byte offsets + "fcvt.ps.f16 f11, f11\n" + "flw.ps f12, (%[b_p])\n" // Standard vector load (32-bit floats) + "fmadd.ps f10, f11, f12, f10\n" + "fsw.ps f10, %[result]\n" + + : [result] "=m"(*(float (*)[8]) acc_vec) + : [acc] "m"(*(const float (*)[8]) acc_vec), [a_p] "r"(a_ptr), [b_p] "r"(b_ptr) + : "f10", "f11", "f12"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + + return acc_vec[0] + acc_vec[1] + acc_vec[2] + acc_vec[3] + acc_vec[4] + acc_vec[5] + acc_vec[6] + acc_vec[7]; +} + +// Compute dot product between f16 block and f32 column vector +// SCALAR implementation for partial blocks +// Block size: up to 32 f16 values (can handle partial blocks for misaligned K) +static inline float compute_block_dot_product_f16_partial(const uint16_t * a_block, + const float * b_col_start, + int elements) { + // This matches compute_block_dot_product_f16_naive behavior + float sum = 0.0f; + + for (int i = 0; i < elements; i++) { + float a_val = fp16_to_fp32(a_block[i]); + float b_val = b_col_start[i]; + sum += a_val * b_val; + } + + return sum; +} + +// Compute dot product between f16 block and f16 column vector +// Scalar implementation for generic non-matrix-engine fallback paths. +static inline float compute_block_dot_product_f16_f16_partial(const uint16_t * a_block, + const uint16_t * b_col_start, + int elements) { + float sum = 0.0f; + + for (int i = 0; i < elements; i++) { + sum += fp16_to_fp32(a_block[i]) * fp16_to_fp32(b_col_start[i]); + } + + return sum; +} + +// Compute dot product between f16 block and f32 column vector +// Vectorized: processes 8 elements at a time using ET vector instructions +// Block size: 32 f16 values (64 bytes = 1 cache line) +static inline float compute_block_dot_product_f16(const uint16_t * a_block, const float * b_col_start) { + return compute_block_dot_product_f16_partial(a_block, b_col_start, QK_F16); +} + +// Compute dot product between f32 block and f32 column vector +// Vectorized: processes 8 elements at a time using ET vector instructions +// Block size: up to 16 f32 values (can handle partial blocks for misaligned K) +static inline float compute_block_dot_product_f32_partial(const float * a_block, + const float * b_col_start, + int elements) { + float acc_vec[8] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; // Accumulator vector + + // Calculate how many full 8-element chunks we can process + int vec_end = (elements / 8) * 8; + + if (vec_end > 0) { + // Set mask register to enable all 8 vector elements + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); // Save current mask + __asm__ volatile("mov.m.x m0, x0, 0xFF"); // Enable all 8 elements + + // Process full 8-element chunks + for (int i = 0; i < vec_end; i += 8) { + // Vectorized f32 multiply-accumulate + __asm__ volatile( + "flw.ps f10, %[acc]\n" // Load current accumulator (8 floats) + "flw.ps f11, %[a_vec]\n" // Load 8 A values (f32) + "flw.ps f12, %[b_vec]\n" // Load 8 B values (f32) + "fmadd.ps f10, f11, f12, f10\n" // acc += a_vec * b_vec (8-wide) + "fsw.ps f10, %[result]\n" // Store back to accumulator + + : [result] "=m"(*(float (*)[8]) acc_vec) + : [acc] "m"(*(const float (*)[8]) acc_vec), [a_vec] "m"(*(const float (*)[8])(a_block + i)), + [b_vec] "m"(*(const float (*)[8])(b_col_start + i)) + : "f10", "f11", "f12"); + } + + // Restore original mask + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + } + + // Horizontal sum: reduce 8 accumulator elements to single scalar + float final_sum = 0.0f; + for (int i = 0; i < 8; i++) { + final_sum += acc_vec[i]; + } + + // Handle remaining elements (< 8) with scalar operations + for (int i = vec_end; i < elements; i++) { + final_sum += a_block[i] * b_col_start[i]; + } + + return final_sum; +} + +// Compute dot product between f32 block and f16 column vector +// Scalar implementation for generic non-matrix-engine fallback paths. +static inline float compute_block_dot_product_f32_f16_partial(const float * a_block, + const uint16_t * b_col_start, + int elements) { + float sum = 0.0f; + + for (int i = 0; i < elements; i++) { + sum += a_block[i] * fp16_to_fp32(b_col_start[i]); + } + + return sum; +} + +// Compute dot product between f32 block and f32 column vector +// Vectorized: processes 8 elements at a time using ET vector instructions +// Block size: 16 f32 values (64 bytes = 1 cache line) +static inline float compute_block_dot_product_f32(const float * a_block, const float * b_col_start) { + return compute_block_dot_product_f32_partial(a_block, b_col_start, QK_F32); + + // float acc_vec[8]; + // unsigned long old_mask; + // __asm__ volatile( + // // Save current mask + // "mova.x.m %[old_mask]\n" + // // Enable all 8 lanes + // "mov.m.x m0, x0, 0xFF\n" + + // "flw.ps f11, %[a]\n" + // "flw.ps f12, %[b]\n" + // "fmadd.ps f10, f11, f12, f10\n" + // "fsw.ps f10, %[out]\n" + // "mova.m.x %[old_mask]\n" + + // : [out] "=m" (*(float(*)[8])acc_vec), + // [old_mask] "=r"(old_mask) + // : [a] "m" (*(const float(*)[8])a_block), + // [b] "m" (*(const float(*)[8])b_col_start) + // : "f10", "f11", "f12" + // ); + + // // Horizontal reduction + // return acc_vec[0] + acc_vec[1] + acc_vec[2] + acc_vec[3] + + // acc_vec[4] + acc_vec[5] + acc_vec[6] + acc_vec[7]; +} + +#endif // BLOCK_OPS_H + +static inline void __attribute__((always_inline)) q4_dot_reset(void) { + __asm__ volatile("fbci.pi f20, 0" ::: "f20"); +} + +static inline void __attribute__((always_inline)) q4_dot_tile(const block_q4_0 * q_row, + const float * b_col, + int64_t n_blocks) { + const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); + + for (int64_t kb = 0; kb < n_blocks; kb++) { + const block_q4_0 * blk = q_row + kb; + const float * b_ptr = b_col + (kb << 5); + + __asm__ volatile( + "fbci.pi f10, 0\n" + + "fgb.ps f11, f31(%[a_ptr0])\n" + "fandi.pi f12, f11, 15\n" + "faddi.pi f12, f12, -8\n" + "fcvt.ps.pw f12, f12, rne\n" + "flw.ps f13, %[b_low0]\n" + "fmadd.ps f10, f12, f13, f10, rne\n" + + "fsrli.pi f14, f11, 4\n" + "fandi.pi f14, f14, 15\n" + "faddi.pi f14, f14, -8\n" + "fcvt.ps.pw f14, f14, rne\n" + "flw.ps f15, %[b_high0]\n" + "fmadd.ps f10, f14, f15, f10, rne\n" + + "fgb.ps f11, f31(%[a_ptr1])\n" + "fandi.pi f12, f11, 15\n" + "faddi.pi f12, f12, -8\n" + "fcvt.ps.pw f12, f12, rne\n" + "flw.ps f13, %[b_low1]\n" + "fmadd.ps f10, f12, f13, f10, rne\n" + + "fsrli.pi f14, f11, 4\n" + "fandi.pi f14, f14, 15\n" + "faddi.pi f14, f14, -8\n" + "fcvt.ps.pw f14, f14, rne\n" + "flw.ps f15, %[b_high1]\n" + "fmadd.ps f10, f14, f15, f10, rne\n" + : + : [a_ptr0] "r"(&blk->qs[0]), [b_low0] "m"(*(const float (*)[8]) & b_ptr[0]), + [b_high0] "m"(*(const float (*)[8]) & b_ptr[16]), [a_ptr1] "r"(&blk->qs[8]), + [b_low1] "m"(*(const float (*)[8]) & b_ptr[8]), [b_high1] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12", "f13", "f14", "f15"); + + uint32_t scale_raw = (uint32_t) blk->d; + __asm__ volatile( + "fbcx.ps f15, %[sb]\n" + "fcvt.ps.f16 f15, f15\n" + "fmadd.ps f20, f10, f15, f20\n" + : + : [sb] "r"(scale_raw) + : "f15", "f20"); + } +} + +static inline float __attribute__((always_inline)) q4_dot_reduce(void) { + float result; + __asm__ __volatile__( + "fswizz.ps f1, f20, 0xB1 \n\t" + "fadd.ps f2, f20, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(result)::"t0", "f1", "f2", "f3", "f4", "f5"); + return result; +} + +static inline float compute_row_dot_q4_0(const block_q4_0 * q_row, const float * b_col, int64_t K_blocks) { + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + q4_dot_reset(); + q4_dot_tile(q_row, b_col, K_blocks); + float result = q4_dot_reduce(); + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return result; +} + +typedef struct { + unsigned long saved_mask; +} q4_dot_state; + +static inline void q4_dot_begin(q4_dot_state * state) { + __asm__ volatile("mova.x.m %0" : "=r"(state->saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); +} + +static inline void q4_dot_end(const q4_dot_state * state) { + __asm__ volatile("mova.m.x %0" ::"r"(state->saved_mask)); +} + +static inline float q4_dot_compute(const block_q4_0 * q_row, const float * b_col, int64_t K_blocks) { + q4_dot_reset(); + q4_dot_tile(q_row, b_col, K_blocks); + return q4_dot_reduce(); +} + +static inline void q4_dot_compute_x2_aligned(const block_q4_0 * q_row0, + const block_q4_0 * q_row1, + const float * b_col, + int64_t K_blocks, + float * out0, + float * out1) { + const int32_t gather_pattern[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + __asm__ volatile("flw.ps f31, %[g]\n" : : [g] "m"(*(const int32_t (*)[8]) gather_pattern) : "f31"); + __asm__ volatile( + "fbci.pi f20, 0\n" + "fbci.pi f21, 0\n" :: + : "f20", "f21"); + + for (int64_t kb = 0; kb < K_blocks; kb++) { + const block_q4_0 * blk0 = q_row0 + kb; + const block_q4_0 * blk1 = q_row1 + kb; + const float * b_ptr = b_col + (kb << 5); + + __asm__ volatile( + "fbci.pi f10, 0\n" + "fbci.pi f16, 0\n" + + "flw.ps f13, %[b_low0]\n" + "flw.ps f15, %[b_high0]\n" + + "fgb.ps f11, f31(%[a_ptr0_0])\n" + "fgb.ps f17, f31(%[a_ptr1_0])\n" + + "fandi.pi f12, f11, 15\n" + "faddi.pi f12, f12, -8\n" + "fcvt.ps.pw f12, f12, rne\n" + "fmadd.ps f10, f12, f13, f10, rne\n" + + "fandi.pi f18, f17, 15\n" + "faddi.pi f18, f18, -8\n" + "fcvt.ps.pw f18, f18, rne\n" + "fmadd.ps f16, f18, f13, f16, rne\n" + + "fsrli.pi f14, f11, 4\n" + "fandi.pi f14, f14, 15\n" + "faddi.pi f14, f14, -8\n" + "fcvt.ps.pw f14, f14, rne\n" + "fmadd.ps f10, f14, f15, f10, rne\n" + + "fsrli.pi f19, f17, 4\n" + "fandi.pi f19, f19, 15\n" + "faddi.pi f19, f19, -8\n" + "fcvt.ps.pw f19, f19, rne\n" + "fmadd.ps f16, f19, f15, f16, rne\n" + + "flw.ps f13, %[b_low1]\n" + "flw.ps f15, %[b_high1]\n" + + "fgb.ps f11, f31(%[a_ptr0_1])\n" + "fgb.ps f17, f31(%[a_ptr1_1])\n" + + "fandi.pi f12, f11, 15\n" + "faddi.pi f12, f12, -8\n" + "fcvt.ps.pw f12, f12, rne\n" + "fmadd.ps f10, f12, f13, f10, rne\n" + + "fandi.pi f18, f17, 15\n" + "faddi.pi f18, f18, -8\n" + "fcvt.ps.pw f18, f18, rne\n" + "fmadd.ps f16, f18, f13, f16, rne\n" + + "fsrli.pi f14, f11, 4\n" + "fandi.pi f14, f14, 15\n" + "faddi.pi f14, f14, -8\n" + "fcvt.ps.pw f14, f14, rne\n" + "fmadd.ps f10, f14, f15, f10, rne\n" + + "fsrli.pi f19, f17, 4\n" + "fandi.pi f19, f19, 15\n" + "faddi.pi f19, f19, -8\n" + "fcvt.ps.pw f19, f19, rne\n" + "fmadd.ps f16, f19, f15, f16, rne\n" + : + : [a_ptr0_0] "r"(&blk0->qs[0]), [a_ptr0_1] "r"(&blk0->qs[8]), [a_ptr1_0] "r"(&blk1->qs[0]), + [a_ptr1_1] "r"(&blk1->qs[8]), [b_low0] "m"(*(const float (*)[8]) & b_ptr[0]), + [b_high0] "m"(*(const float (*)[8]) & b_ptr[16]), [b_low1] "m"(*(const float (*)[8]) & b_ptr[8]), + [b_high1] "m"(*(const float (*)[8]) & b_ptr[24]) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f19"); + + const uint32_t scale_raw0 = (uint32_t) blk0->d; + const uint32_t scale_raw1 = (uint32_t) blk1->d; + __asm__ volatile( + "fbcx.ps f24, %[s0]\n" + "fcvt.ps.f16 f24, f24\n" + "fmadd.ps f20, f10, f24, f20\n" + "fbcx.ps f25, %[s1]\n" + "fcvt.ps.f16 f25, f25\n" + "fmadd.ps f21, f16, f25, f21\n" + : + : [s0] "r"(scale_raw0), [s1] "r"(scale_raw1) + : "f20", "f21", "f24", "f25"); + } + + float result0, result1; + __asm__ __volatile__( + "fswizz.ps f1, f20, 0xB1 \n\t" + "fadd.ps f2, f20, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(result0)::"t0", "f1", "f2", "f3", "f4", "f5"); + __asm__ __volatile__( + "fswizz.ps f1, f21, 0xB1 \n\t" + "fadd.ps f2, f21, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(result1)::"t0", "f1", "f2", "f3", "f4", "f5"); + + *out0 = result0; + *out1 = result1; +} diff --git a/ggml/src/ggml-et/et-kernels/src/clamp_f32.c b/ggml/src/ggml-et/et-kernels/src/clamp_f32.c new file mode 100644 index 0000000..cf091b4 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/clamp_f32.c @@ -0,0 +1,120 @@ +//****************************************************************************** +// CLAMP F32 Kernel +// Element-wise: dst[i] = min(max(src0[i], min_val), max_val) +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_clamp_params { + struct ggml_tensor src0; // F32 input (contiguous) + struct ggml_tensor dst; // F32 output (contiguous; may alias src0.data) + float min_val; + float max_val; +}; + +// Vectorized fmax/fmin clamp with scalar tail. n may be any non-negative int. +static inline void clamp_block_f32(float * dst, const float * src, float min_val, float max_val, int32_t n) { + int32_t i = 0; + const int32_t vec_end = (n / 8) * 8; + + if (vec_end > 0) { + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + for (; i < vec_end; i += 8) { + __asm__ volatile( + "flw.ps f10, %[s]\n" + "fbc.ps f11, %[mn]\n" + "fbc.ps f12, %[mx]\n" + "fmax.ps f13, f10, f11\n" + "fmin.ps f13, f13, f12\n" + "fsw.ps f13, %[d]\n" + : [d] "=m"(*(float (*)[8]) & dst[i]) + : [s] "m"(*(const float (*)[8]) & src[i]), [mn] "m"(min_val), [mx] "m"(max_val) + : "f10", "f11", "f12", "f13"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + } + + for (; i < n; i++) { + float v = src[i]; + if (v < min_val) { + v = min_val; + } + if (v > max_val) { + v = max_val; + } + dst[i] = v; + } +} + +int entry_point(struct ggml_et_clamp_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + const int64_t total_elements = src0->ne[0] * src0->ne[1] * src0->ne[2] * src0->ne[3]; + if (total_elements <= 0) { + return 0; + } + + const float min_val = params->min_val; + const float max_val = params->max_val; + + // Distribute by cache lines (16 F32 elements). Each thread owns disjoint + // cache lines, so a partial trailing line is written by exactly one + // thread — safe under non-coherent caches. + const int64_t elems_per_cl = 16; + const int64_t total_cl = (total_elements + elems_per_cl - 1) / elems_per_cl; + + const int64_t cl_per_thread = (total_cl + num_threads - 1) / num_threads; + const int64_t cl_start = (int64_t) thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cl) { + cl_end = total_cl; + } + if (cl_start >= total_cl) { + return 0; + } + + const int64_t es = cl_start * elems_per_cl; + int64_t ee = cl_end * elems_per_cl; + if (ee > total_elements) { + ee = total_elements; + } + + clamp_block_f32(dst_data + es, src0_data + es, min_val, max_val, (int32_t) (ee - es)); + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/concat_f32.c b/ggml/src/ggml-et/et-kernels/src/concat_f32.c new file mode 100644 index 0000000..dbdf4ae --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/concat_f32.c @@ -0,0 +1,175 @@ +//****************************************************************************** +// Concat F32 Kernel +// Concatenates two F32 tensors along a specified dimension. +// All copies are aligned to cacheline boundaries (64 bytes = 16 floats). +// +// For dim >= 1, entire rows are copied from src0 or src1 into dst. +// For dim == 0, use: +// - a fast vector path when both source row segments are cacheline-aligned +// - a scalar stride-aware path otherwise +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include +#include + +struct ggml_et_concat_params { + struct ggml_tensor src0; // F32 input tensor 0 + struct ggml_tensor src1; // F32 input tensor 1 + struct ggml_tensor dst; // F32 output tensor + int32_t dim; // Concatenation dimension +}; + +// Copy n floats from src to dst using 8-wide vector loads/stores. +// n must be a multiple of 16 (cacheline-aligned). +static inline void copy_row_aligned(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f11, %[src_vec]\n" + "fsw.ps f11, %[dst_vec]\n" + : [dst_vec] "=m"(*(float (*)[8]) & dst[i]) + : [src_vec] "m"(*(const float (*)[8]) & src[i]) + : "f11"); + } +} + +int entry_point(struct ggml_et_concat_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * dst = ¶ms->dst; + int32_t dim = params->dim; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * src1_data = (float *) src1->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !src1_data || !dst_data) { + return -1; + } + + const int64_t ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const int64_t ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; + const int64_t ne0 = dst->ne[0], ne1 = dst->ne[1], ne2 = dst->ne[2], ne3 = dst->ne[3]; + + // src strides in bytes + const size_t nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const size_t nb10 = src1->nb[0], nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + // dst strides in bytes + const size_t dnb1 = dst->nb[1], dnb2 = dst->nb[2], dnb3 = dst->nb[3]; + + // Total rows across all higher dimensions + const int64_t total_rows = ne1 * ne2 * ne3; + + // Generic slow path for dim==0 when either source segment is not suitable for + // aligned vector copies. Threading is done by cacheline-aligned row groups, + // so writers do not share destination cache lines. + if (dim == 0 && (ne00 % 16 != 0 || ne10 % 16 != 0 || nb00 != sizeof(float) || nb10 != sizeof(float))) { + const int64_t rows_per_group = et_rows_per_cacheline_group(ne0, sizeof(float)); + const int64_t total_groups = (total_rows + rows_per_group - 1) / rows_per_group; + + for (int64_t grp = thread_id; grp < total_groups; grp += num_threads) { + const int64_t row_start = grp * rows_per_group; + int64_t row_end = row_start + rows_per_group; + if (row_end > total_rows) { + row_end = total_rows; + } + + for (int64_t row = row_start; row < row_end; row++) { + int64_t i1 = row % ne1; + int64_t i2 = (row / ne1) % ne2; + int64_t i3 = row / (ne1 * ne2); + + float * dst_row = (float *) ((char *) dst_data + i1 * dnb1 + i2 * dnb2 + i3 * dnb3); + + const char * s0_base = (const char *) src0_data + i1 * nb01 + i2 * nb02 + i3 * nb03; + for (int64_t i0 = 0; i0 < ne00; i0++) { + dst_row[i0] = *(const float *) (s0_base + i0 * nb00); + } + + const char * s1_base = (const char *) src1_data + i1 * nb11 + i2 * nb12 + i3 * nb13; + for (int64_t i0 = 0; i0 < ne10; i0++) { + dst_row[ne00 + i0] = *(const float *) (s1_base + i0 * nb10); + } + } + } + return 0; + } + + // Standard path: ne0 % 16 == 0, aligned rows + for (int64_t row = thread_id; row < total_rows; row += num_threads) { + // Decompose linear row index into (i1, i2, i3) + int64_t i1 = row % ne1; + int64_t i2 = (row / ne1) % ne2; + int64_t i3 = row / (ne1 * ne2); + + float * dst_row = (float *) ((char *) dst_data + i1 * dnb1 + i2 * dnb2 + i3 * dnb3); + + if (dim == 0) { + // Concat along innermost dimension: [src0_row | src1_row] + // Both ne00 and ne10 are multiples of 16 (cacheline-aligned) + const float * s0_row = (const float *) ((const char *) src0_data + i1 * nb01 + i2 * nb02 + i3 * nb03); + const float * s1_row = (const float *) ((const char *) src1_data + i1 * nb11 + i2 * nb12 + i3 * nb13); + + copy_row_aligned(dst_row, s0_row, (int32_t) ne00); + copy_row_aligned(dst_row + ne00, s1_row, (int32_t) ne10); + + } else if (dim == 1) { + // Concat along dim 1: first ne01 rows from src0, rest from src1 + if (i1 < ne01) { + const float * s0_row = (const float *) ((const char *) src0_data + i1 * nb01 + i2 * nb02 + i3 * nb03); + copy_row_aligned(dst_row, s0_row, (int32_t) ne0); + } else { + const float * s1_row = + (const float *) ((const char *) src1_data + (i1 - ne01) * nb11 + i2 * nb12 + i3 * nb13); + copy_row_aligned(dst_row, s1_row, (int32_t) ne0); + } + + } else if (dim == 2) { + // Concat along dim 2: first ne02 slices from src0, rest from src1 + if (i2 < ne02) { + const float * s0_row = (const float *) ((const char *) src0_data + i1 * nb01 + i2 * nb02 + i3 * nb03); + copy_row_aligned(dst_row, s0_row, (int32_t) ne0); + } else { + const float * s1_row = + (const float *) ((const char *) src1_data + i1 * nb11 + (i2 - ne02) * nb12 + i3 * nb13); + copy_row_aligned(dst_row, s1_row, (int32_t) ne0); + } + + } else { + // dim == 3: first ne03 batches from src0, rest from src1 + if (i3 < ne03) { + const float * s0_row = (const float *) ((const char *) src0_data + i1 * nb01 + i2 * nb02 + i3 * nb03); + copy_row_aligned(dst_row, s0_row, (int32_t) ne0); + } else { + const float * s1_row = + (const float *) ((const char *) src1_data + i1 * nb11 + i2 * nb12 + (i3 - ne03) * nb13); + copy_row_aligned(dst_row, s1_row, (int32_t) ne0); + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/cont_f16.c b/ggml/src/ggml-et/et-kernels/src/cont_f16.c new file mode 100644 index 0000000..3ef08da --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/cont_f16.c @@ -0,0 +1,107 @@ +//****************************************************************************** +// Bare Metal CONT F16 Kernel +// Converts non-contiguous F16 tensors to contiguous memory layout +// +// Note: F16 is represented as uint16_t (IEEE 754 binary16 format) +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include +#include +#include + +struct ggml_et_cont_params { + struct ggml_tensor src0; // F16 input tensor (non-contiguous) + struct ggml_tensor dst; // F16 output tensor (contiguous) +}; + +int entry_point(struct ggml_et_cont_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = 2048; //get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + struct ggml_tensor * src0 = ¶ms->src0; // Non-contiguous input + struct ggml_tensor * dst = ¶ms->dst; // Contiguous output + + if (src0->type != GGML_TYPE_F16 || dst->type != GGML_TYPE_F16) { + return -1; // Unsupported type combination + } + + uint16_t * src0_data = (uint16_t *) src0->data; + uint16_t * dst_data = (uint16_t *) dst->data; + + if (!src0_data || !dst_data) { + return -1; // Null data pointer + } + + const int64_t src_elements = src0->ne[0] * src0->ne[1] * src0->ne[2] * src0->ne[3]; + const int64_t dst_elements = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; + if (src_elements != dst_elements) { + return -1; // Element count mismatch + } + + // Source tensor dimensions and strides + const int64_t ne00 = src0->ne[0]; + const int64_t ne01 = src0->ne[1]; + const int64_t ne02 = src0->ne[2]; + const int64_t ne03 = src0->ne[3]; + + const int64_t nb00 = src0->nb[0]; + const int64_t nb01 = src0->nb[1]; + const int64_t nb02 = src0->nb[2]; + const int64_t nb03 = src0->nb[3]; + + // Parallelize by rows (dimension 1) + const int64_t total_rows = ne01; + const int64_t rows_per_thread = (total_rows + num_threads - 1) / num_threads; + const int64_t start_row = thread_id * rows_per_thread; + const int64_t end_row = (start_row + rows_per_thread < total_rows) ? (start_row + rows_per_thread) : total_rows; + + if (start_row >= total_rows) { + return 0; + } + + // Iterate over source tensor dimensions + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + // Calculate base linear index for this (i03, i02) slice in destination + const int64_t dst_linear_base = i03 * ne02 * ne01 * ne00 + i02 * ne01 * ne00; + + // Process this thread's assigned rows + for (int64_t i01 = start_row; i01 < end_row; i01++) { + // Linear index for start of this row in destination + const int64_t dst_linear_row_base = dst_linear_base + i01 * ne00; + + // Inner loop over dimension 0 + for (int64_t i00 = 0; i00 < ne00; i00++) { + // Source offset using non-contiguous strides + const int64_t src_offset_bytes = i00 * nb00 + i01 * nb01 + i02 * nb02 + i03 * nb03; + const uint16_t * src_ptr = (const uint16_t *) ((const char *) src0_data + src_offset_bytes); + + // Destination linear index (contiguous layout) + const int64_t dst_linear_idx = dst_linear_row_base + i00; + + // Use atomic store for thread safety + atomic_store_f16((volatile uint16_t *) &dst_data[dst_linear_idx], *src_ptr); + } + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/cont_f32.c b/ggml/src/ggml-et/et-kernels/src/cont_f32.c new file mode 100644 index 0000000..88c8480 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/cont_f32.c @@ -0,0 +1,248 @@ +//****************************************************************************** +// Bare Metal CONT F32 Kernel +// Converts non-contiguous tensors to contiguous memory layout +// +// Fast path: src contiguous: flat vectorized copy by cache lines +// Aligned path: nb00==4 and ne00 % 16 == 0: distribute rows, no coherency issue +// Unaligned: nb00==4 and ne00 not aligned: distribute by cache lines, +// reverse-compute src coords, handle partial rows at boundaries +// Fallback: nb00 != 4: scalar per-element +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include +#include + +struct ggml_et_cont_params { + struct ggml_tensor src0; // F32 input tensor (non-contiguous) + struct ggml_tensor dst; // F32 output tensor (contiguous) +}; + +// Vectorized copy with scalar tail +static inline void vec_copy_f32(float * dst, const float * src, int32_t n) { + int32_t i = 0; + const int32_t vec_end = (n / 8) * 8; + for (; i < vec_end; i += 8) { + __asm__ volatile( + "flw.ps f10, %[s]\n" + "fsw.ps f10, %[d]\n" + : [d] "=m"(*(float (*)[8]) & dst[i]) + : [s] "m"(*(const float (*)[8]) & src[i]) + : "f10"); + } + for (; i < n; i++) { + dst[i] = src[i]; + } +} + +// Scalar copy +static inline void scalar_copy_f32(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i++) { + dst[i] = src[i]; + } +} + +// static inline size_t tensor_bytes(const struct ggml_tensor *t) { +// return (size_t)t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +// } + +int entry_point(struct ggml_et_cont_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + // evict_region_past_l2(src0_data, tensor_bytes(src0)); + + if (!src0_data || !dst_data) { + return -1; + } + + const int64_t ne00 = src0->ne[0]; + const int64_t ne01 = src0->ne[1]; + const int64_t ne02 = src0->ne[2]; + const int64_t ne03 = src0->ne[3]; + + const int64_t nb00 = src0->nb[0]; + const int64_t nb01 = src0->nb[1]; + const int64_t nb02 = src0->nb[2]; + const int64_t nb03 = src0->nb[3]; + + const int64_t total_elements = ne00 * ne01 * ne02 * ne03; + + if (total_elements == 0) { + return 0; + } + + const bool src_contiguous = ggml_tensor_is_contiguous(src0, 4); + + //========================================================================== + // Fast path: src is contiguous: flat vectorized copy by cache lines + //========================================================================== + if (src_contiguous) { + const int64_t elems_per_cl = 16; + const int64_t total_cl = (total_elements + elems_per_cl - 1) / elems_per_cl; + + const int64_t cl_per_thread = (total_cl + num_threads - 1) / num_threads; + const int64_t cl_start = thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cl) { + cl_end = total_cl; + } + if (cl_start >= total_cl) { + return 0; + } + + const int64_t es = cl_start * elems_per_cl; + int64_t ee = cl_end * elems_per_cl; + if (ee > total_elements) { + ee = total_elements; + } + + vec_copy_f32(dst_data + es, src0_data + es, (int32_t) (ee - es)); + return 0; + } + + //========================================================================== + // Non-contiguous paths: require nb00==4 (dim 0 contiguous in src) + //========================================================================== + if (nb00 != 4) { + // Fully non-contiguous scalar fallback — distribute by cache lines + const int64_t elems_per_cl = 16; + const int64_t total_cl = (total_elements + elems_per_cl - 1) / elems_per_cl; + + const int64_t cl_per_thread = (total_cl + num_threads - 1) / num_threads; + const int64_t cl_start = thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cl) { + cl_end = total_cl; + } + if (cl_start >= total_cl) { + return 0; + } + + const int64_t es = cl_start * elems_per_cl; + int64_t ee = cl_end * elems_per_cl; + if (ee > total_elements) { + ee = total_elements; + } + + for (int64_t idx = es; idx < ee; idx++) { + const int64_t i00 = idx % ne00; + const int64_t rem1 = idx / ne00; + const int64_t i01 = rem1 % ne01; + const int64_t rem2 = rem1 / ne01; + const int64_t i02 = rem2 % ne02; + const int64_t i03 = rem2 / ne02; + + const float * sp = + (const float *) ((const char *) src0_data + i00 * nb00 + i01 * nb01 + i02 * nb02 + i03 * nb03); + dst_data[idx] = *sp; + } + return 0; + } + + // nb00 == 4 from here: dim 0 is contiguous in src + + //========================================================================== + // Aligned path: ne00 % 16 == 0: rows are cache-line aligned, distribute rows + //========================================================================== + if (ne00 % 16 == 0) { + const int64_t total_rows = ne01 * ne02 * ne03; + const int64_t rows_per_thread = (total_rows + num_threads - 1) / num_threads; + const int64_t start_row = thread_id * rows_per_thread; + const int64_t end_row = (start_row + rows_per_thread < total_rows) ? (start_row + rows_per_thread) : total_rows; + + if (start_row >= total_rows) { + return 0; + } + + for (int64_t ir = start_row; ir < end_row; ir++) { + const int64_t i03 = ir / (ne02 * ne01); + const int64_t i02 = (ir - i03 * ne02 * ne01) / ne01; + const int64_t i01 = ir - i03 * ne02 * ne01 - i02 * ne01; + + const float * src_row = (const float *) ((const char *) src0_data + i01 * nb01 + i02 * nb02 + i03 * nb03); + float * dst_row = dst_data + ir * ne00; + + vec_copy_f32(dst_row, src_row, (int32_t) ne00); + } + return 0; + } + + //========================================================================== + // Unaligned path: ne00 % 16 != 0, nb00 == 4 + // Distribute cache-line-aligned chunks of dst, handle partial rows at edges + //========================================================================== + { + const int64_t elems_per_cl = 16; + const int64_t total_cl = (total_elements + elems_per_cl - 1) / elems_per_cl; + + const int64_t cl_per_thread = (total_cl + num_threads - 1) / num_threads; + const int64_t cl_start = thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cl) { + cl_end = total_cl; + } + if (cl_start >= total_cl) { + return 0; + } + + const int64_t es = cl_start * elems_per_cl; + int64_t ee = cl_end * elems_per_cl; + if (ee > total_elements) { + ee = total_elements; + } + + int64_t pos = es; + + // Compute starting row coordinates + int64_t row_idx = pos / ne00; + int64_t col = pos % ne00; + + while (pos < ee) { + // Decompose row_idx -> (i01, i02, i03) + const int64_t i03 = row_idx / (ne02 * ne01); + const int64_t i02 = (row_idx - i03 * ne02 * ne01) / ne01; + const int64_t i01 = row_idx - i03 * ne02 * ne01 - i02 * ne01; + + const float * src_row = (const float *) ((const char *) src0_data + i01 * nb01 + i02 * nb02 + i03 * nb03); + + // How many elements left in this row and in our chunk + int64_t row_remaining = ne00 - col; + int64_t chunk_remaining = ee - pos; + int32_t n = (int32_t) (row_remaining < chunk_remaining ? row_remaining : chunk_remaining); + + vec_copy_f32(dst_data + pos, src_row + col, n); + + pos += n; + col = 0; // subsequent rows start at column 0 + row_idx++; + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c b/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c new file mode 100644 index 0000000..7405379 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c @@ -0,0 +1,807 @@ +//****************************************************************************** +// 2D F32 convolution on the ET-SoC-1 matrix engine (GGML CONV_2D layout). +// +// LAYOUT (matches GGML's standard CONV_2D, cwhn=false; wireable directly): +// src1 input : ne = [W, H, Cin, N=1] memory: input [n][cin][h][w] +// src0 filter: ne = [Kw, Kh, Cin, Cout] memory: filter[oc][ic][kh][kw] +// dst output: ne = [W, H, Cout, N=1] memory: output[n][oc][h][w] +// +// CONSTRAINTS (enforced at supports_op): +// F32 throughout, N == 1, Cin % 16 == 0, Cout % 16 == 0, positive +// stride/pad, dilation == 1. Tile/L2SCP limits are checked here. +// +// MEMORY MODEL: +// Each active shire uses its own 2 MB local L2 SCP: +// filter slice | pin buffer 0 | pin buffer 1? | output staging? | scratch +// +// The filter slice contains only the output-channel tiles (`mt`) consumed +// by this shire's tile assignment. That keeps hart-0's inner-loop +// tensor_loads local to the shire and avoids packing unused filter slabs. +// +// THREADING (multi-minion, multi-shire): +// PHASE 1 (per-shire filter pack): hart-1's pack this shire's filter +// slice into local L2 SCP. Work is slab-striped across the 32 minions. +// +// PHASE 2 (per-shire compute): hart-1's pack the input pin chunks while +// hart-0's run the matrix engine. Pin double-buffering hides the next +// chunk pack behind the current chunk's FMA pipeline when Cin does not +// fit in one local buffer. +// +// PERFORMANCE STRATEGIES: +// 1. Local filter slice: pack only the `mt` values this shire consumes; +// inner-loop tensor_loads stay shire-local. +// 2. Pin Cin streaming + chunk double-buffer: pack one +// chunk while computing the prior one. +// 3. TenC save/restore: f0..f31 IS the TenC accumulator; +// spill/refill via L2 SCP scratch lets each hart hold multiple +// partial accumulators across chunks. +// 4. OW%16 staging: for partial-tile output, write to a +// padded L2 SCP region then have one hart scalar-emit to DRAM. +// +// WHY THE FILTER PACK EXISTS: +// GGML's OIHW filter has stride Kh*Kw*4 between consecutive Cin elements +// (e.g. 36 bytes for 3x3) — usually NOT a multiple of 64, so plain +// tensor_load cannot gather it directly. The per-slab pack into a +// Cin-innermost form gives every per-tap slab a flat 64-byte row stride +// and enables tensor_load. +// +// Picking M=Cout, N=W means TenC's natural row stride matches NCHW +// output's per-channel stride (H*W*4) — the output store is a clean +// tensor_store with no transpose. The price is that conv_size/conv_ctrl +// no longer help with W boundaries (mask gates M, not N), so we handle +// boundaries up-front by zero-padding the input in L2SCP. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" + +#include +#include +#include + +#define TILE 16 /* matrix engine native tile in M, K, N */ +/* L1 SCP layout: A double-buffered, B single-buffered. Per the SDK doc + `dst_start` is a 6-bit field (max 63) but empirical testing shows the + physical L1 SCP per minion is 48 lines — writes to lines >= 48 corrupt. + So we get 3 × 16-line buffers max: A_0, A_1, B. Pick A as the + double-buffered operand (filter-slab loads, the longer of the two). */ +#define LSCP_A_0 0 /* A buffer 0 at L1 SCP lines 0..15 */ +#define LSCP_A_1 16 /* A buffer 1 at L1 SCP lines 16..31 */ +#define LSCP_B 32 /* B (single buffer) at lines 32..47 */ +#define N_MIN_PER_SHIRE 32 /* ET-SoC-1 geometry: 32 minions/shire */ +#define N_SHIRES 32 /* default active shire count */ +#define MAX_TILES_PER_HART 2 /* per-hart TenC slots (save/restore) */ +#define MAX_DBL_BUFS 2 /* chunk pack buffers (double-buffered) */ + +/* Per-shire L2 SCP local budget. Per-shire SCP is 2 MB; we cap at + 1984 KB to leave 64 KB headroom for per-hart TenC scratch (32 minions × + 2 slots × 1 KB), which lives at the tail of the SCP outside the pin + sizing budget. Bigger budget here means bigger feasible chunk_KT, + which means fewer chunks (each chunk costs 2 SHIRE barriers + ~30 + TenC save/restore events per hart). */ +#define LOCAL_BUDGET (1984 * 1024) + +/* Cap on the per-shire filter region in local L2 SCP. The shire packs the + mt values it can consume under the current tile assignment, rather than + the whole Cout dimension. Reads in the inner loop are then fully + shire-local — no NoC fanout. */ +#define LOCAL_FILTER_CAP (1024 * 1024) /* 1 MB / shire ceiling */ + +#define SLAB_BYTES ((uint64_t) TILE * TILE * sizeof(float)) /* 1024 */ +#define SLAB_LINES ((SLAB_BYTES + 63) / 64) /* 16 */ + +/* Upper bound on the number of distinct mt values a single shire may pack. + This keeps the mt list stack-resident. Shapes that need more should fall + back until the filter-slice bookkeeping is made dynamic. */ +#define MAX_MY_MT (N_MIN_PER_SHIRE * MAX_TILES_PER_HART) + +typedef struct { + int mt; + int mt_idx; + int oh; + int ow_base; +} conv_tile_t; + +static inline int ceil_div_i32(int x, int y) { + return (x + y - 1) / y; +} + +static inline int round_up_tile_i32(int x) { + return (x + TILE - 1) & ~(TILE - 1); +} + +static inline int min_i32(int a, int b) { + return a < b ? a : b; +} + +static inline uint64_t min_u64(uint64_t a, uint64_t b) { + return a < b ? a : b; +} + +/* ===== Vector helpers for hart-1 pack ============================ + Both assume dst (and src for copy) are 32-byte aligned; n is in floats. + The 8-element tail is handled scalar. f30/f31 are scratch — clobbered + per-call via the asm clobber list. */ +static inline void vec_zero_aligned(float * dst, int n) { + int i = 0; + const int n8 = n & ~7; + for (; i < n8; i += 8) { + __asm__ volatile( + "fsub.ps f31, f31, f31\n" + "fsw.ps f31, %[d]\n" + : [d] "=m"(*(float (*)[8]) & dst[i]) + : + : "f31"); + } + for (; i < n; ++i) { + dst[i] = 0.0f; + } +} + +static inline void vec_copy_aligned(float * dst, const float * src, int n) { + int i = 0; + const int n8 = n & ~7; + for (; i < n8; i += 8) { + __asm__ volatile( + "flw.ps f30, %[s]\n" + "fsw.ps f30, %[d]\n" + : [d] "=m"(*(float (*)[8]) & dst[i]) + : [s] "m"(*(const float (*)[8]) & src[i]) + : "f30"); + } + for (; i < n; ++i) { + dst[i] = src[i]; + } +} + +/* ===== TenC save/restore ========================================= + The TenC accumulator IS the f0..f31 vector register file: row N occupies + f(2N) and f(2N+1) (two 8-fp32 vector regs per row). We save by + tensor_store-ing TILE rows × 64 bytes, and restore via 32 flw.ps after + forcing L1D to refetch from the L2SCP backing (tensor_store bypasses L1D + so the backing is always current). See feedback_tenc_save_restore.md. */ +static inline void tenc_restore_from_scratch(uint64_t scr) { + FENCE; + evict_to_l2((const void *) scr, TILE, 64); + WAIT_CACHEOPS; + __asm__ volatile( + "flw.ps f0, 0(%0)\n" + "flw.ps f1, 32(%0)\n" + "flw.ps f2, 64(%0)\n" + "flw.ps f3, 96(%0)\n" + "flw.ps f4, 128(%0)\n" + "flw.ps f5, 160(%0)\n" + "flw.ps f6, 192(%0)\n" + "flw.ps f7, 224(%0)\n" + "flw.ps f8, 256(%0)\n" + "flw.ps f9, 288(%0)\n" + "flw.ps f10, 320(%0)\n" + "flw.ps f11, 352(%0)\n" + "flw.ps f12, 384(%0)\n" + "flw.ps f13, 416(%0)\n" + "flw.ps f14, 448(%0)\n" + "flw.ps f15, 480(%0)\n" + "flw.ps f16, 512(%0)\n" + "flw.ps f17, 544(%0)\n" + "flw.ps f18, 576(%0)\n" + "flw.ps f19, 608(%0)\n" + "flw.ps f20, 640(%0)\n" + "flw.ps f21, 672(%0)\n" + "flw.ps f22, 704(%0)\n" + "flw.ps f23, 736(%0)\n" + "flw.ps f24, 768(%0)\n" + "flw.ps f25, 800(%0)\n" + "flw.ps f26, 832(%0)\n" + "flw.ps f27, 864(%0)\n" + "flw.ps f28, 896(%0)\n" + "flw.ps f29, 928(%0)\n" + "flw.ps f30, 960(%0)\n" + "flw.ps f31, 992(%0)\n" + : + : "r"(scr) + : "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "f16", + "f17", "f18", "f19", "f20", "f21", "f22", "f23", "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", + "memory"); +} + +/* ===== Pin pack context ========================================== + Loop-invariant state hart-1 needs to pack one Cin chunk's worth of + pin (Kw shifted, padded copies of input rows) into local L2 SCP. The + filter is not touched in this struct; it is packed into the per-shire + local slice before the per-chunk loop begins. */ +typedef struct { + const float * in_base; /* DRAM input base [Cin][H][W] */ + int Kw; + int chunk_KT; /* number of K_TILES (=16-wide) per chunk */ + int H, W, Hp, Wp_a; + int pad_h, pad_w, s0; + int minion; /* this hart's minion id (0..31) */ + uint64_t pin_copy_floats; /* per-_s pin plane size in floats */ + uint64_t l2_pad_in_buf[MAX_DBL_BUFS]; + uint64_t pin_chunk_bytes; /* one chunk pin buffer's total size */ +} pin_ctx_t; + +static inline int find_mt_idx(const int * my_mt, int n_my_mt, int mt) { + for (int j = 0; j < n_my_mt; ++j) { + if (my_mt[j] == mt) { + return j; + } + } + return 0; +} + +static inline conv_tile_t decode_tile(int t, int M_TILES, int w_tiles, const int * my_mt, int n_my_mt) { + conv_tile_t tile; + tile.mt = t % M_TILES; + t /= M_TILES; + const int wt = t % w_tiles; + t /= w_tiles; + tile.oh = t; + tile.ow_base = wt * TILE; + tile.mt_idx = find_mt_idx(my_mt, n_my_mt, tile.mt); + return tile; +} + +static inline uint64_t +filter_slab_addr(uint64_t l2_filter, int Kw, int K_TILES, int n_my_mt, int mt_idx, int kh, int kw, int kt_global) { + return l2_filter + (uint64_t) ((((kh * Kw + kw) * n_my_mt + mt_idx) * K_TILES + kt_global)) * SLAB_BYTES; +} + +static inline uint64_t pin_tile_addr(uint64_t l2_pad_in, + uint64_t pin_copy_bytes, + int ktc, + int kw, + int Hp, + int Wp_a, + int oh, + int ow_base, + int s1, + int kh) { + const int ir_pad = oh * s1 + kh; + return l2_pad_in + (uint64_t) kw * pin_copy_bytes + + (((uint64_t) (ktc * TILE) * Hp + ir_pad) * Wp_a + ow_base) * sizeof(float); +} + +static inline char * output_tile_addr(char * out_base, + const conv_tile_t * tile, + uint64_t out_chan_stride, + uint64_t out_row_stride) { + return out_base + (size_t) (tile->mt * TILE) * out_chan_stride + (size_t) tile->oh * out_row_stride + + (size_t) tile->ow_base * sizeof(float); +} + +static inline void flush_range_to_l2(const void * addr, uint64_t n_bytes) { + const uint64_t total_lines = (n_bytes + 63) / 64; + const char * fl_addr = (const char *) addr; + for (uint64_t done = 0; done < total_lines;) { + const uint64_t batch = min_u64(total_lines - done, 16); + flush_to_l2((const void *) (fl_addr + done * 64), batch, 64); + done += batch; + } +} + +static inline void evict_range_past_l2(const void * addr, uint64_t n_bytes) { + const uint64_t total_lines = (n_bytes + 63) / 64; + const char * fl_addr = (const char *) addr; + for (uint64_t done = 0; done < total_lines;) { + const uint64_t batch = min_u64(total_lines - done, 16); + evict_past_l2((const void *) (fl_addr + done * 64), batch, 64); + done += batch; + } +} + +/* One matrix-engine tile for one Cin chunk. This is the main optimization + surface: A is double-buffered, B is single-buffered due to L1 SCP space. */ +static inline void compute_tile_chunk(uint64_t l2_filter, + uint64_t l2_pad_in, + uint64_t pin_copy_bytes, + int Kh, + int Kw, + int K_TILES, + int chunk_KT, + int kt_base, + int n_my_mt, + int Hp, + int Wp_a, + int s1, + uint64_t a_row_stride, + uint64_t b_row_stride, + const conv_tile_t * tile, + bool first_fma_clears_tenc) { + const int n_iters = Kh * Kw * chunk_KT; + const uint64_t A_BUFS[2] = { LSCP_A_0, LSCP_A_1 }; + + const uint64_t a_addr0 = filter_slab_addr(l2_filter, Kw, K_TILES, n_my_mt, tile->mt_idx, 0, 0, kt_base); + tensor_load(false, false, A_BUFS[0], 0, 0, a_addr0, 0, (uint64_t) (TILE - 1), a_row_stride, 0); + + for (int iter = 0; iter < n_iters; ++iter) { + const int ktc = iter % chunk_KT; + const int rem = iter / chunk_KT; + const int kw = rem % Kw; + const int kh = rem / Kw; + + const uint64_t b_addr = + pin_tile_addr(l2_pad_in, pin_copy_bytes, ktc, kw, Hp, Wp_a, tile->oh, tile->ow_base, s1, kh); + tensor_load(false, false, LSCP_B, 0, 0, b_addr, 0, (uint64_t) (TILE - 1), b_row_stride, 1); + + tensor_wait(TENSOR_LOAD_WAIT_0); + tensor_wait(TENSOR_LOAD_WAIT_1); + + if (iter + 1 < n_iters) { + const int ktc_n = (iter + 1) % chunk_KT; + const int rem_n = (iter + 1) / chunk_KT; + const int kw_n = rem_n % Kw; + const int kh_n = rem_n / Kw; + const uint64_t a_addr_n = + filter_slab_addr(l2_filter, Kw, K_TILES, n_my_mt, tile->mt_idx, kh_n, kw_n, kt_base + ktc_n); + tensor_load(false, false, A_BUFS[(iter + 1) & 1], 0, 0, a_addr_n, 0, (uint64_t) (TILE - 1), a_row_stride, + 0); + } + + tensor_fma(false, 3, (uint64_t) (TILE - 1), (uint64_t) (TILE - 1), 0, false, false, false, false, LSCP_B, + A_BUFS[iter & 1], 0, first_fma_clears_tenc && (iter == 0)); + tensor_wait(TENSOR_FMA_WAIT); + } +} + +/* Pack only the slabs this shire's tiles actually consume, into local + L2 SCP. Slab layout in the filter buffer is [Kh][Kw][n_my_mt][K_TILES] + of TILE×TILE slabs (Cin-innermost form). Distributed across the 32 + hart-1's of this shire by `slab % 32 == minion`. + + This deliberately favors local inner-loop reads over global filter fanout. + Depending on tile shape, two shires may pack the same mt value; keep that + tradeoff visible when experimenting with shared-filter layouts. */ +static void pack_filter_local_mt(const float * flt_base, + int Kh, + int Kw, + int Cin, + int K_TILES, + const int * my_mt, + int n_my_mt, + int minion, + uint64_t l2_filter_base) { + const int n_slabs = Kh * Kw * n_my_mt * K_TILES; + const size_t kstep = (size_t) Kh * Kw; /* Cin stride in floats */ + + for (int slab = minion; slab < n_slabs; slab += N_MIN_PER_SHIRE) { + int t = slab; + const int kt = t % K_TILES; + t /= K_TILES; + const int mt_idx = t % n_my_mt; + t /= n_my_mt; + const int kw = t % Kw; + t /= Kw; + const int kh = t; + const int mt = my_mt[mt_idx]; + + const uint64_t slab_offset = (uint64_t) slab * SLAB_BYTES; + float * cell = (float *) (l2_filter_base + slab_offset); + + for (int oc_in = 0; oc_in < TILE; ++oc_in) { + const int oc = mt * TILE + oc_in; + const float * src = flt_base + (((size_t) oc * Cin + (size_t) kt * TILE) * Kh + kh) * Kw + kw; + float * row = cell + (size_t) oc_in * TILE; + float scratch[TILE] __attribute__((aligned(32))); + for (int ic_in = 0; ic_in < TILE; ++ic_in) { + scratch[ic_in] = src[(size_t) ic_in * kstep]; + } + vec_copy_aligned(row, scratch, TILE); + } + } + + /* Flush this hart's dirty L1D lines for the slabs it wrote. */ + FENCE; + for (int slab = minion; slab < n_slabs; slab += N_MIN_PER_SHIRE) { + const uint64_t slab_offset = (uint64_t) slab * SLAB_BYTES; + flush_to_l2((const void *) (l2_filter_base + slab_offset), SLAB_LINES, 64); + } + WAIT_CACHEOPS; +} + +/* Pack one Cin chunk of the input pin (Kw shifted padded copies) into the + buf_idx side of local L2SCP. Work distributed across the 32 hart-1's in + the shire by `plane % 32 == minion`. The final flush_to_l2 forces L1D + write-back so hart-0's tensor_load sees the freshly written bytes. */ +static void pack_pin_chunk(const pin_ctx_t * ctx, int chunk_id, int buf_idx) { + const int kt_base = chunk_id * ctx->chunk_KT; + const int Kw = ctx->Kw; + const int chunk_KT = ctx->chunk_KT; + const int H = ctx->H, W = ctx->W, Hp = ctx->Hp, Wp_a = ctx->Wp_a; + const int pad_h = ctx->pad_h, pad_w = ctx->pad_w, s0 = ctx->s0; + const int minion = ctx->minion; + + /* Pin pack: Kw shifted, padded copies of input rows. Bounds [vlo, vhi) + hoisted outside the row loop so the inner loop is three regions + (zero-prefix | bulk-copy | zero-suffix) with no per-element predicate. */ + float * pin0 = (float *) ctx->l2_pad_in_buf[buf_idx]; + const int chunk_Cin = chunk_KT * TILE; + const int n_pin_planes = Kw * chunk_Cin; + for (int p = minion; p < n_pin_planes; p += N_MIN_PER_SHIRE) { + const int s = p / chunk_Cin; + const int icc = p % chunk_Cin; + const int ic = kt_base * TILE + icc; + float * pin_s = pin0 + (size_t) s * ctx->pin_copy_floats; + + const int offset = s - pad_w; + int vlo = 0; + while (vlo < Wp_a && (s0 * vlo + offset) < 0) { + vlo++; + } + int vhi = Wp_a; + while (vhi > vlo && (s0 * (vhi - 1) + offset) >= W) { + vhi--; + } + const bool aligned = (s0 == 1) && ((vlo & 7) == 0) && (((vlo + offset) & 7) == 0); + + for (int r = 0; r < Hp; ++r) { + float * row = pin_s + ((size_t) icc * Hp + r) * Wp_a; + const int real_h = r - pad_h; + if (real_h < 0 || real_h >= H) { + vec_zero_aligned(row, Wp_a); + continue; + } + const float * src_row = ctx->in_base + ((size_t) ic * H + real_h) * W; + + for (int cc = 0; cc < vlo; ++cc) { + row[cc] = 0.0f; + } + + if (aligned) { + vec_copy_aligned(row + vlo, src_row + vlo + offset, vhi - vlo); + } else if (s0 == 1) { + const float * csrc = src_row + vlo + offset; + const int n = vhi - vlo; + for (int cc = 0; cc < n; ++cc) { + row[vlo + cc] = csrc[cc]; + } + } else { + for (int cc = vlo; cc < vhi; ++cc) { + row[cc] = src_row[s0 * cc + offset]; + } + } + + for (int cc = vhi; cc < Wp_a; ++cc) { + row[cc] = 0.0f; + } + } + } + + /* Flush this buffer's L1D-dirty lines down to L2SCP backing. */ + FENCE; + flush_range_to_l2((const void *) ctx->l2_pad_in_buf[buf_idx], ctx->pin_chunk_bytes); + WAIT_CACHEOPS; +} + +int entry_point(struct ggml_et_binary_params * params, void * env) { + (void) env; + + const int shire = get_shire_id(); + const int hart_id = get_hart_id(); + const int minion = (hart_id >> 1) & 0x1F; + const int hart1 = hart_id & 1; + + const struct ggml_tensor * flt = ¶ms->src0; /* [Kw,Kh,Cin,Cout] */ + const struct ggml_tensor * in = ¶ms->src1; /* [W, H, Cin,N=1 ] */ + struct ggml_tensor * out = ¶ms->dst; /* [W, H, Cout,N=1] */ + + const int Kw = (int) flt->ne[0]; + const int Kh = (int) flt->ne[1]; + const int Cin = (int) flt->ne[2]; + const int Cout = (int) flt->ne[3]; + + const int W = (int) in->ne[0]; + const int H = (int) in->ne[1]; + const int OW = (int) out->ne[0]; + const int OH = (int) out->ne[1]; + + /* op_params layout (set by ggml_conv_2d): + [0]=s0 [1]=s1 [2]=p0 [3]=p1 [4]=d0 [5]=d1 */ + const int s0 = out->op_params[0]; + const int s1 = out->op_params[1]; + const int pad_w = out->op_params[2]; + const int pad_h = out->op_params[3]; + + if (Cin <= 0 || Cout <= 0) { + return -1; + } + if (Cin % TILE != 0 || Cout % TILE != 0) { + return -1; + } + if (W <= 0 || H <= 0) { + return -1; + } + if (s0 <= 0 || s1 <= 0) { + return -1; + } + if (in->ne[2] != Cin || in->ne[3] != 1) { + return -1; + } + if (out->ne[2] != Cout || out->ne[3] != 1) { + return -1; + } + if (!flt->data || !in->data || !out->data) { + return -1; + } + + const int K_TILES = Cin / TILE; + const int M_TILES = Cout / TILE; + + const int Hp = H + 2 * pad_h; + const int Wp_a = round_up_tile_i32(OW); + const int OW_pad = Wp_a; + const bool need_stage = (OW % TILE != 0); + + /* ===================== Tile assignment & active-shire selection ===== + Computed up front because the per-shire mt set (and thus filter + region size) depends on n_active_shires. */ + const int w_tiles = ceil_div_i32(OW, TILE); + const int total_tiles = OH * w_tiles * M_TILES; + const int n_active_shires = need_stage ? 1 : min_i32(total_tiles, N_SHIRES); + + /* Inactive shires exit immediately. No global barrier — pack and + barriers are now per-shire, so unused shires don't need to vote. */ + if (shire >= n_active_shires) { + return 0; + } + + /* ===================== Determine this shire's mt set ================ + Standard tile assignment: tile t is owned by + shire = t % n_active_shires + minion = (t / n_active_shires) % N_MIN_PER_SHIRE + slot = t / (n_active_shires * N_MIN_PER_SHIRE) + So the set of mt's this shire actually consumes is the set of + (t % M_TILES) for all t this shire owns. Enumerate all shire-owned + tiles, not just the first MAX_TILES_PER_HART slots; the one-chunk + path can process more tiles serially. */ + int my_mt[MAX_MY_MT]; + int n_my_mt = 0; + for (int t = shire; t < total_tiles; t += n_active_shires) { + const int mt = t % M_TILES; + bool found = false; + for (int j = 0; j < n_my_mt; ++j) { + if (my_mt[j] == mt) { + found = true; + break; + } + } + if (!found) { + if (n_my_mt >= MAX_MY_MT) { + return -1; + } + my_mt[n_my_mt++] = mt; + } + } + if (n_my_mt == 0) { + return 0; /* no tiles for this shire */ + } + + const uint64_t filter_local_bytes = (uint64_t) Kh * Kw * n_my_mt * K_TILES * SLAB_BYTES; + if (filter_local_bytes > LOCAL_FILTER_CAP) { + return -1; + } + + /* ===================== L2 SCP local layout ========================= + filter (this shire's mt slice) | pin_buf[0] | pin_buf[1]? + | output_stage? | scratch (streaming) */ + const uint64_t l2_base = (uint64_t) et_shire_l2scp_local(0); + const uint64_t l2_filter = l2_base; + + /* Sizing for pin: budget = LOCAL_BUDGET - filter - output_stage. */ + const int64_t output_stage_bytes_full = need_stage ? (int64_t) Cout * OH * OW_pad * (int64_t) sizeof(float) : 0; + const int64_t budget_for_chunks = (int64_t) LOCAL_BUDGET - (int64_t) filter_local_bytes - output_stage_bytes_full; + if (budget_for_chunks <= 0) { + return -1; + } + const int64_t per_KT_pin_bytes = (int64_t) Kw * TILE * Hp * Wp_a * (int64_t) sizeof(float); + + int chunk_KT; + int n_buffers; + if ((int64_t) K_TILES * per_KT_pin_bytes <= budget_for_chunks) { + chunk_KT = K_TILES; + n_buffers = 1; + } else { + chunk_KT = K_TILES; + while (chunk_KT > 1 && 2 * (int64_t) chunk_KT * per_KT_pin_bytes > budget_for_chunks) { + chunk_KT--; + } + while (chunk_KT > 1 && K_TILES % chunk_KT != 0) { + chunk_KT--; + } + n_buffers = (chunk_KT < K_TILES) ? 2 : 1; + if (chunk_KT < 1) { + return -1; + } + } + const int n_chunks = K_TILES / chunk_KT; + + /* Streaming keeps partial sums in MAX_TILES_PER_HART scratch slots per + hart. The one-chunk path does not need scratch and can stream a longer + tile list serially, but multi-chunk shapes must fit this fixed slot + count until scratch scheduling is made more general. */ + const int shire_tile_capacity = shire + MAX_TILES_PER_HART * n_active_shires * N_MIN_PER_SHIRE; + if (n_chunks > 1 && shire_tile_capacity < total_tiles) { + return -1; + } + + const uint64_t pin_copy_floats = (uint64_t) chunk_KT * TILE * Hp * Wp_a; + const uint64_t pin_copy_bytes = pin_copy_floats * sizeof(float); + const uint64_t pin_chunk_bytes = (uint64_t) Kw * pin_copy_bytes; + + const uint64_t l2_pin_base = l2_filter + filter_local_bytes; + const uint64_t l2_pin_buf[MAX_DBL_BUFS] = { + l2_pin_base, + l2_pin_base + pin_chunk_bytes, + }; + + const uint64_t l2_output_stage = need_stage ? l2_pin_base + (uint64_t) n_buffers * pin_chunk_bytes : 0; + + const uint64_t scratch_per_hart = (uint64_t) MAX_TILES_PER_HART * (uint64_t) TILE * TILE * sizeof(float); + const uint64_t l2_scratch_base = need_stage ? l2_output_stage + (uint64_t) output_stage_bytes_full : + l2_pin_base + (uint64_t) n_buffers * pin_chunk_bytes; + + /* ===================== PHASE 1: Filter pack (per-shire mt slice) ==== + Hart-1's pack only this shire's mt slabs into local L2 SCP. The + SHIRE barrier below ensures the filter is in L2 SCP backing before + hart-0's first tensor_load. */ + if (hart1) { + pack_filter_local_mt((const float *) flt->data, Kh, Kw, Cin, K_TILES, my_mt, n_my_mt, minion, l2_filter); + } + + /* ===================== Hart 1: pin packer (per chunk) ============== + Double-buffered prefetch: pack chunk 0 synchronously, then per chunk c + signal "buf c ready", pack chunk c+1 into the alternate buffer + (overlaps hart-0's compute on c), signal "buf c done". */ + if (hart1) { + const pin_ctx_t ctx = { + .in_base = (const float *) in->data, + .Kw = Kw, + .chunk_KT = chunk_KT, + .H = H, + .W = W, + .Hp = Hp, + .Wp_a = Wp_a, + .pad_h = pad_h, + .pad_w = pad_w, + .s0 = s0, + .minion = minion, + .pin_copy_floats = pin_copy_floats, + .l2_pad_in_buf = { l2_pin_buf[0], l2_pin_buf[1] }, + .pin_chunk_bytes = pin_chunk_bytes, + }; + + pack_pin_chunk(&ctx, 0, 0); /* prologue */ + + for (int c = 0; c < n_chunks; ++c) { + et_barrier(ET_BARRIER_SHIRE); /* signal "buf c ready" */ + if (n_buffers > 1 && c + 1 < n_chunks) { + pack_pin_chunk(&ctx, c + 1, (c + 1) & 1); + } + et_barrier(ET_BARRIER_SHIRE); /* wait "buf c done" */ + } + + if (need_stage) { + et_barrier(ET_BARRIER_SHIRE); + } + return 0; + } + + /* ===================== Hart 0: matrix engine ====================== + Two execution modes: + - n_chunks == 1: full Cin in one shot. Each hart processes a list + of tiles serially; TenC resets between tiles via first_pass=true. + - n_chunks > 1: streaming. Each hart owns up to MAX_TILES_PER_HART + tiles. For each chunk c, restore TenC from scratch[k] (skip on + c==0), accumulate this chunk's FMAs, then either save TenC back + to scratch[k] (c < last) or tensor_store directly (c == last). */ + setup_cache_scp(); + CLEAR_TENSOR_ERROR; + + char * const out_base = need_stage ? (char *) l2_output_stage : (char *) out->data; + const int compute_OW = need_stage ? OW_pad : OW; + const uint64_t out_chan_stride = (uint64_t) OH * (uint64_t) compute_OW * sizeof(float); + const uint64_t out_row_stride = (uint64_t) compute_OW * sizeof(float); + + const uint64_t a_row_stride = (uint64_t) TILE * sizeof(float); /* 64 */ + const uint64_t b_row_stride = (uint64_t) Hp * (uint64_t) Wp_a * sizeof(float); + + /* Tile assignment: shire-strided so small workloads spread across + shires before stacking minions in one shire. */ + const int t_start = shire + minion * n_active_shires; + const int t_stride = n_active_shires * N_MIN_PER_SHIRE; + + if (n_chunks == 1) { + et_barrier(ET_BARRIER_SHIRE); /* wait for the (only) pin chunk */ + + const uint64_t l2_pad_in = l2_pin_buf[0]; + for (int t = t_start; t < total_tiles; t += t_stride) { + const conv_tile_t tile = decode_tile(t, M_TILES, w_tiles, my_mt, n_my_mt); + compute_tile_chunk(l2_filter, l2_pad_in, pin_copy_bytes, Kh, Kw, K_TILES, chunk_KT, 0, n_my_mt, Hp, Wp_a, + s1, a_row_stride, b_row_stride, &tile, /*first_fma_clears_tenc=*/true); + + char * dst_addr = output_tile_addr(out_base, &tile, out_chan_stride, out_row_stride); + tensor_store(0, 0, 3, (uint64_t) (TILE - 1), (uint64_t) dst_addr, 0, out_chan_stride); + tensor_wait(TENSOR_STORE_WAIT); + } + + et_barrier(ET_BARRIER_SHIRE); /* matches hart-1's second barrier */ + + } else { + /* Streaming path: each hart owns up to MAX_TILES_PER_HART tiles. */ + int my_tiles[MAX_TILES_PER_HART]; + int n_my_tiles = 0; + for (int slot = 0; slot < MAX_TILES_PER_HART; ++slot) { + const int t = t_start + slot * t_stride; + if (t < total_tiles) { + my_tiles[n_my_tiles++] = t; + } + } + + conv_tile_t tiles[MAX_TILES_PER_HART]; + for (int k = 0; k < n_my_tiles; ++k) { + tiles[k] = decode_tile(my_tiles[k], M_TILES, w_tiles, my_mt, n_my_mt); + } + + const uint64_t my_scratch_base = l2_scratch_base + (uint64_t) minion * scratch_per_hart; + + for (int c = 0; c < n_chunks; ++c) { + et_barrier(ET_BARRIER_SHIRE); /* pin chunk c packed */ + + const int buf = c & 1; + const uint64_t l2_pad_in = l2_pin_buf[buf]; + const int kt_base = c * chunk_KT; + + for (int k = 0; k < n_my_tiles; ++k) { + const conv_tile_t * tile = &tiles[k]; + const uint64_t scr = my_scratch_base + (uint64_t) k * (TILE * TILE * sizeof(float)); + + const bool first_pass_chunk = (c == 0); + if (!first_pass_chunk) { + tenc_restore_from_scratch(scr); + } + + compute_tile_chunk(l2_filter, l2_pad_in, pin_copy_bytes, Kh, Kw, K_TILES, chunk_KT, kt_base, n_my_mt, + Hp, Wp_a, s1, a_row_stride, b_row_stride, tile, first_pass_chunk); + + if (c == n_chunks - 1) { + char * dst_addr = output_tile_addr(out_base, tile, out_chan_stride, out_row_stride); + tensor_store(0, 0, 3, (uint64_t) (TILE - 1), (uint64_t) dst_addr, 0, out_chan_stride); + } else { + tensor_store(0, 0, 3, (uint64_t) (TILE - 1), (uint64_t) scr, 0, 64); + } + tensor_wait(TENSOR_STORE_WAIT); + } + + et_barrier(ET_BARRIER_SHIRE); /* hart-0 done with chunk c */ + } + } + + FENCE; + + /* ----------------------- DRAM emit phase --------------------------- + Only relevant when we staged into L2SCP because OW % 16 != 0. */ + if (need_stage) { + et_barrier(ET_BARRIER_SHIRE); + + if (minion == 0) { + const float * stage = (const float *) l2_output_stage; + float * dram = (float *) out->data; + for (int oc = 0; oc < Cout; ++oc) { + for (int oh2 = 0; oh2 < OH; ++oh2) { + const float * src = stage + ((size_t) oc * OH + oh2) * OW_pad; + float * dst = dram + ((size_t) oc * OH + oh2) * OW; + for (int ow2 = 0; ow2 < OW; ++ow2) { + dst[ow2] = src[ow2]; + } + } + } + FENCE; + const uint64_t total_bytes = (uint64_t) Cout * OH * OW * sizeof(float); + evict_range_past_l2((const void *) dram, total_bytes); + WAIT_CACHEOPS; + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c b/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c new file mode 100644 index 0000000..8bde57d --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c @@ -0,0 +1,110 @@ +//****************************************************************************** +// CPY F32 -> F16 Kernel +// Copies F32 source tensor to F16 destination tensor (contiguous output). +// Source may have arbitrary strides; destination must be contiguous. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include +#include + +struct ggml_et_cont_params { + struct ggml_tensor src0; + struct ggml_tensor dst; +}; + +int entry_point(struct ggml_et_cont_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env || !params) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F16) { + return -1; + } + + const char * src_data = (const char *) src0->data; + uint16_t * dst_data = (uint16_t *) dst->data; + + if (!src_data || !dst_data) { + return -1; + } + + const int64_t ne00 = src0->ne[0]; + const int64_t ne01 = src0->ne[1]; + const int64_t ne02 = src0->ne[2]; + const int64_t ne03 = src0->ne[3]; + + const int64_t nb00 = src0->nb[0]; + const int64_t nb01 = src0->nb[1]; + const int64_t nb02 = src0->nb[2]; + const int64_t nb03 = src0->nb[3]; + + const int64_t total_elements = ne00 * ne01 * ne02 * ne03; + + if (total_elements == 0) { + return 0; + } + + // Check if src is contiguous F32 + const bool src_contiguous = + (nb00 == 4 && nb01 == ne00 * 4 && nb02 == ne00 * ne01 * 4 && nb03 == ne00 * ne01 * ne02 * 4); + + // Distribute by cache lines (16 F16 elements = 32 bytes = half cache line) + // Use 32 elements per chunk to keep output cache-line aligned + const int64_t elems_per_cl = 32; + const int64_t total_cl = (total_elements + elems_per_cl - 1) / elems_per_cl; + + const int64_t cl_per_thread = (total_cl + num_threads - 1) / num_threads; + const int64_t cl_start = thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cl) { + cl_end = total_cl; + } + if (cl_start >= total_cl) { + return 0; + } + + const int64_t es = cl_start * elems_per_cl; + int64_t ee = cl_end * elems_per_cl; + if (ee > total_elements) { + ee = total_elements; + } + + if (src_contiguous) { + // Fast path: src is contiguous F32 + const float * src_f32 = (const float *) src_data; + for (int64_t i = es; i < ee; ++i) { + dst_data[i] = fp32_to_fp16(src_f32[i]); + } + } else { + // General path: stride-aware read + for (int64_t idx = es; idx < ee; ++idx) { + const int64_t i00 = idx % ne00; + const int64_t rem1 = idx / ne00; + const int64_t i01 = rem1 % ne01; + const int64_t rem2 = rem1 / ne01; + const int64_t i02 = rem2 % ne02; + const int64_t i03 = rem2 / ne02; + + const float val = *(const float *) (src_data + i00 * nb00 + i01 * nb01 + i02 * nb02 + i03 * nb03); + dst_data[idx] = fp32_to_fp16(val); + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/crt.S b/ggml/src/ggml-et/et-kernels/src/crt.S new file mode 100644 index 0000000..5f80272 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/crt.S @@ -0,0 +1,15 @@ +.section .text.init, "ax", @progbits +.global _start +_start: + # initialize global pointer +.option push +.option norelax + la gp, __global_pointer$ +.option pop + # Firmware sets stack pointer before launch + # bss not allowed, no init + call entry_point + li a2, 0 /* KERNEL_RETURN_SUCCESS (0) */ + mv a1, a0 + li a0, 8 /* SYSCALL_RETURN_FROM_KERNEL (8) */ + ecall diff --git a/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c b/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c new file mode 100644 index 0000000..008f78b --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c @@ -0,0 +1,96 @@ +//****************************************************************************** +// CUMSUM F32 Kernel +// Computes an inclusive prefix sum along dim 0 for each row in higher dims. +// First-pass implementation: scalar and row-contiguous input/output only. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_cumsum_params { + struct ggml_tensor src0; + struct ggml_tensor dst; +}; + +int entry_point(struct ggml_et_cumsum_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + const int64_t ne0 = src0->ne[0]; + const int64_t ne1 = src0->ne[1]; + const int64_t ne2 = src0->ne[2]; + const int64_t ne3 = src0->ne[3]; + + const size_t snb0 = src0->nb[0]; + const size_t snb1 = src0->nb[1]; + const size_t snb2 = src0->nb[2]; + const size_t snb3 = src0->nb[3]; + + const size_t dnb0 = dst->nb[0]; + const size_t dnb1 = dst->nb[1]; + const size_t dnb2 = dst->nb[2]; + const size_t dnb3 = dst->nb[3]; + + if (snb0 != sizeof(float) || dnb0 != sizeof(float)) { + return -1; + } + + const int64_t total_rows = ne1 * ne2 * ne3; + const int64_t rows_per_group = et_rows_per_cacheline_group(ne0, sizeof(float)); + const int64_t total_groups = (total_rows + rows_per_group - 1) / rows_per_group; + + for (int64_t grp = thread_id; grp < total_groups; grp += num_threads) { + const int64_t row_start = grp * rows_per_group; + int64_t row_end = row_start + rows_per_group; + if (row_end > total_rows) { + row_end = total_rows; + } + + for (int64_t row = row_start; row < row_end; ++row) { + int64_t i1 = row % ne1; + int64_t i2 = (row / ne1) % ne2; + int64_t i3 = row / (ne1 * ne2); + + const float * src_row = (const float *) ((const char *) src0_data + i1 * snb1 + i2 * snb2 + i3 * snb3); + float * dst_row = (float *) ((char *) dst_data + i1 * dnb1 + i2 * dnb2 + i3 * dnb3); + + float acc = 0.0f; + for (int64_t i0 = 0; i0 < ne0; ++i0) { + acc += src_row[i0]; + dst_row[i0] = acc; + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/diag_f32.c b/ggml/src/ggml-et/et-kernels/src/diag_f32.c new file mode 100644 index 0000000..50fd3a8 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/diag_f32.c @@ -0,0 +1,90 @@ +//****************************************************************************** +// Diag F32 Kernel +// Creates a diagonal matrix from a 1D vector. +// dst[i][j] = (i == j) ? src0[i] : 0.0f +// +// src0: [N, 1, ne2, ne3] (1D vector per batch) +// dst: [N, N, ne2, ne3] (diagonal matrix per batch) +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_diag_params { + struct ggml_tensor src0; // F32 input vector + struct ggml_tensor dst; // F32 output diagonal matrix +}; + +int entry_point(struct ggml_et_diag_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + const int64_t ne0 = dst->ne[0]; // N (row width = column count) + const int64_t ne1 = dst->ne[1]; // N (number of rows) + const int64_t ne2 = dst->ne[2]; + const int64_t ne3 = dst->ne[3]; + + const size_t nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + const size_t nb02 = src0->nb[2], nb03 = src0->nb[3]; + + // Total rows across all batches — parallelize over these + const int64_t total_rows = ne1 * ne2 * ne3; + + // Prepare zero vector for SIMD zeroing + float zero = 0.0f; + __asm__ volatile("fbc.ps f10, %[z]\n" : : [z] "m"(zero) : "f10"); + + for (int64_t row = thread_id; row < total_rows; row += num_threads) { + int64_t i1 = row % ne1; + int64_t i2 = (row / ne1) % ne2; + int64_t i3 = row / (ne1 * ne2); + + float * dst_row = (float *) ((char *) dst_data + i1 * nb1 + i2 * nb2 + i3 * nb3); + + // Zero the entire row with SIMD + int64_t i0 = 0; + const int64_t vec_end = (ne0 / 8) * 8; + for (; i0 < vec_end; i0 += 8) { + __asm__ volatile("fsw.ps f10, %[d]\n" : [d] "=m"(*(float (*)[8]) & dst_row[i0])::"f10"); + } + for (; i0 < ne0; i0++) { + dst_row[i0] = 0.0f; + } + + // Place the diagonal element: dst[i1][i1] = src0[i1] + const float * src_ptr = (const float *) ((const char *) src0_data + i2 * nb02 + i3 * nb03); + dst_row[i1] = src_ptr[i1]; + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/el_map_f32.c b/ggml/src/ggml-et/et-kernels/src/el_map_f32.c new file mode 100644 index 0000000..c40472f --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/el_map_f32.c @@ -0,0 +1,377 @@ +// Element-wise operations: dst[i] = src0[i] op src1[i] +#include "ggml_tensor.h" +#include "platform.h" + +#include + +// Generic m0-gated element-wise block operation. +// The OP parameter selects the instruction: "fmul.ps", "fadd.ps", "fsub.ps". +#define DEFINE_BLOCK_OP(name, op_insn) \ + static inline void name(float * dst_block, const float * src0_block, const float * src1_block, int elements) { \ + const int32_t vec_end = (elements / 8) * 8; \ + const int32_t tail = elements - vec_end; \ + \ + unsigned long temp_mask; \ + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); \ + __asm__ volatile("mov.m.x m0, x0, 0xFF"); \ + \ + for (int32_t i = 0; i < vec_end; i += 8) { \ + __asm__ volatile( \ + "flw.ps f10, %[s0]\n" \ + "flw.ps f11, %[s1]\n" op_insn \ + " f12, f10, f11\n" \ + "fsw.ps f12, %[d]\n" \ + : [d] "=m"(*(float (*)[8]) & dst_block[i]) \ + : [s0] "m"(*(const float (*)[8]) & src0_block[i]), [s1] "m"(*(const float (*)[8]) & src1_block[i]) \ + : "f10", "f11", "f12"); \ + } \ + /* Deal with tail chunks */ \ + if (tail > 0) { \ + const unsigned long tail_m0 = (1ul << tail) - 1; \ + __asm__ volatile( \ + "mov.m.x m0, %[tm], 0\n" \ + "flw.ps f10, 0(%[s0])\n" \ + "flw.ps f11, 0(%[s1])\n" op_insn \ + " f12, f10, f11\n" \ + "fsw.ps f12, 0(%[d])\n" \ + : \ + : [s0] "r"(&src0_block[vec_end]), [s1] "r"(&src1_block[vec_end]), [d] "r"(&dst_block[vec_end]), \ + [tm] "r"(tail_m0) \ + : "f10", "f11", "f12", "memory"); \ + } \ + \ + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); \ + } + +DEFINE_BLOCK_OP(block_mul_cache_aligned, "fmul.ps") +DEFINE_BLOCK_OP(block_add_cache_aligned, "fadd.ps") +DEFINE_BLOCK_OP(block_sub_cache_aligned, "fsub.ps") + +// Broadcast variants: src1 is a single scalar, broadcast to all 8 lanes. +#define DEFINE_BLOCK_OP_BROADCAST(name, op_insn) \ + static inline void name(float * dst_block, const float * src0_block, float scalar, int elements) { \ + const int32_t vec_end = (elements / 8) * 8; \ + const int32_t tail = elements - vec_end; \ + \ + unsigned long temp_mask; \ + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); \ + __asm__ volatile("mov.m.x m0, x0, 0xFF"); \ + \ + for (int32_t i = 0; i < vec_end; i += 8) { \ + __asm__ volatile( \ + "flw.ps f10, %[s0]\n" \ + "fbc.ps f11, %[s]\n" op_insn \ + " f12, f10, f11\n" \ + "fsw.ps f12, %[d]\n" \ + : [d] "=m"(*(float (*)[8]) & dst_block[i]) \ + : [s0] "m"(*(const float (*)[8]) & src0_block[i]), [s] "m"(scalar) \ + : "f10", "f11", "f12"); \ + } \ + \ + if (tail > 0) { \ + const unsigned long tail_m0 = (1ul << tail) - 1; \ + __asm__ volatile( \ + "mov.m.x m0, %[tm], 0\n" \ + "flw.ps f10, 0(%[s0])\n" \ + "fbc.ps f11, 0(%[ps])\n" op_insn \ + " f12, f10, f11\n" \ + "fsw.ps f12, 0(%[d])\n" \ + : \ + : [s0] "r"(&src0_block[vec_end]), [ps] "r"(&scalar), [d] "r"(&dst_block[vec_end]), [tm] "r"(tail_m0) \ + : "f10", "f11", "f12", "memory"); \ + } \ + \ + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); \ + } + +DEFINE_BLOCK_OP_BROADCAST(block_mul_broadcast, "fmul.ps") +DEFINE_BLOCK_OP_BROADCAST(block_add_broadcast, "fadd.ps") +DEFINE_BLOCK_OP_BROADCAST(block_sub_broadcast, "fsub.ps") + +static inline float scalar_el_map(float src0, float src1, enum ggml_op operation) { + switch (operation) { + case GGML_OP_MUL: + return src0 * src1; + case GGML_OP_ADD: + return src0 + src1; + case GGML_OP_SUB: + return src0 - src1; + default: + return 0.0f; + } +} + +int entry_point(struct ggml_et_binary_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; // Unsupported type combination + } + + float * src0_data = (float *) src0->data; + float * src1_data = (float *) src1->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !src1_data || !dst_data) { + return -1; // Null data pointer + } + +#ifdef ET_UBERKERNEL + // Consumer-side input eviction. Required because ET caches are + // incoherent across minions: if a previous kernel in this UK batch + // left stale lines for these addresses in this hart's L1, drop them + // so we read fresh from L3/DRAM (where the producer flushed its + // results). Standalone launches don't need this -- the host-side + // runtime boundary between kernel launches handles it. + const size_t src0_bytes = (size_t) src0->ne[0] * src0->ne[1] * src0->ne[2] * src0->ne[3] * src0->nb[0]; + const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * src1->nb[0]; + evict_region_past_l2(src0_data, src0_bytes); + evict_region_past_l2(src1_data, src1_bytes); + WAIT_CACHEOPS; + FENCE; + et_barrier(ET_BARRIER_GLOBAL); +#endif + + enum ggml_op operation = dst->op; + + if (operation != GGML_OP_MUL && operation != GGML_OP_ADD && operation != GGML_OP_SUB) { + return -1; // Unsupported operation + } + + const int64_t ne0 = dst->ne[0], ne1 = dst->ne[1], ne2 = dst->ne[2], ne3 = dst->ne[3]; + const int64_t ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const int64_t ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; + + const size_t nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + const size_t nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const size_t nb10 = src1->nb[0], nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + + const bool cache_aligned = (dst->ne[0] % 16 == 0); + + // Fast path: no broadcasting, contiguous + const bool no_broadcast = (ne10 == ne0 && ne11 == ne1 && ne12 == ne2 && ne13 == ne3); + const bool all_contiguous = + (nb0 == 4 && nb00 == 4 && nb10 == 4 && nb1 == ne0 * 4 && nb01 == ne0 * 4 && nb11 == ne0 * 4); + + if (no_broadcast && all_contiguous) { + const int64_t total_elements = ne0 * ne1 * ne2 * ne3; + const int64_t elements_per_cacheline = 16; // 64 bytes / 4 bytes + const int64_t total_cachelines = (total_elements + elements_per_cacheline - 1) / elements_per_cacheline; + + const int64_t cl_per_thread = (total_cachelines + num_threads - 1) / num_threads; + const int64_t cl_start = thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cachelines) { + cl_end = total_cachelines; + } + + if (cl_start >= total_cachelines) { + return 0; + } + + const int64_t elem_start = cl_start * elements_per_cacheline; + int64_t elem_end = cl_end * elements_per_cacheline; + if (elem_end > total_elements) { + elem_end = total_elements; + } + const int32_t count = (int32_t) (elem_end - elem_start); + + switch (operation) { + case GGML_OP_MUL: + block_mul_cache_aligned(dst_data + elem_start, src0_data + elem_start, src1_data + elem_start, count); + break; + case GGML_OP_ADD: + block_add_cache_aligned(dst_data + elem_start, src0_data + elem_start, src1_data + elem_start, count); + break; + case GGML_OP_SUB: + block_sub_cache_aligned(dst_data + elem_start, src0_data + elem_start, src1_data + elem_start, count); + break; + default: + return 1; + } +#ifdef ET_UBERKERNEL + // Producer-side flush: ET caches are incoherent across minions, so + // a consumer kernel running on a different minion can't see our + // dirty L1 lines via its own evict_region_past_l2. Push our writes + // all the way to DRAM so the next batched kernel reads fresh. + // Standalone launches don't need this -- the host runtime boundary + // between kernel launches handles cache writeback. + FENCE; + evict_region_past_l2(dst_data + elem_start, (size_t) count * sizeof(float)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; + } + + // Slow path: broadcasting or non-contiguous + const int64_t total_rows = ne1 * ne2 * ne3; + + int64_t start_row; + int64_t end_row; + + if (cache_aligned) { + const int64_t rows_per_thread = (total_rows + num_threads - 1) / num_threads; + start_row = thread_id * rows_per_thread; + end_row = (start_row + rows_per_thread < total_rows) ? (start_row + rows_per_thread) : total_rows; + } else { + const int64_t rows_per_group = et_rows_per_cacheline_group(ne0, sizeof(float)); + const int64_t total_groups = (total_rows + rows_per_group - 1) / rows_per_group; + + if (thread_id >= total_groups) { + return 0; + } + + const int64_t group_start = thread_id; + for (int64_t grp = group_start; grp < total_groups; grp += num_threads) { + const int64_t group_row_start = grp * rows_per_group; + int64_t group_row_end = group_row_start + rows_per_group; + if (group_row_end > total_rows) { + group_row_end = total_rows; + } + +#ifdef ET_UBERKERNEL + // First row written by this group (used for producer-side evict). + const int64_t first_i03 = group_row_start / (ne2 * ne1); + const int64_t first_i02 = (group_row_start - first_i03 * ne2 * ne1) / ne1; + const int64_t first_i01 = (group_row_start - first_i03 * ne2 * ne1 - first_i02 * ne1); + char * group_dst_base = (char *) dst_data + first_i03 * nb3 + first_i02 * nb2 + first_i01 * nb1; +#endif + + for (int64_t ir = group_row_start; ir < group_row_end; ir++) { + const int64_t i03 = ir / (ne2 * ne1); + const int64_t i02 = (ir - i03 * ne2 * ne1) / ne1; + const int64_t i01 = (ir - i03 * ne2 * ne1 - i02 * ne1); + + const int64_t i13 = i03 % ne13; + const int64_t i12 = i02 % ne12; + const int64_t i11 = i01 % ne11; + + float * dst_ptr = (float *) ((char *) dst_data + i03 * nb3 + i02 * nb2 + i01 * nb1); + const float * src0_ptr = + (const float *) ((const char *) src0_data + i03 * nb03 + i02 * nb02 + i01 * nb01); + const float * src1_ptr = + (const float *) ((const char *) src1_data + i13 * nb13 + i12 * nb12 + i11 * nb11); + + if (ne10 == 1) { + const float scalar = src1_ptr[0]; + for (int64_t i0 = 0; i0 < ne0; ++i0) { + dst_ptr[i0] = scalar_el_map(src0_ptr[i0], scalar, operation); + } + } else { + for (int64_t i0 = 0; i0 < ne0; ++i0) { + dst_ptr[i0] = scalar_el_map(src0_ptr[i0], src1_ptr[i0 % ne10], operation); + } + } + } + +#ifdef ET_UBERKERNEL + // Producer-side flush for this group's rows. Group rows are + // contiguous because nb1 = ne0*4 in the cacheline-group layout. + // Only needed inside a UK batch; see comment in fast path. + const int64_t nrows = group_row_end - group_row_start; + if (nrows > 0) { + FENCE; + evict_region_past_l2(group_dst_base, (size_t) nrows * nb1); + WAIT_CACHEOPS; + FENCE; + } +#endif + } + + return 0; + } + + if (start_row >= total_rows) { + return 0; + } + + for (int64_t ir = start_row; ir < end_row; ir++) { + // Convert flat row index to 3D coordinates + const int64_t i03 = ir / (ne2 * ne1); + const int64_t i02 = (ir - i03 * ne2 * ne1) / ne1; + const int64_t i01 = (ir - i03 * ne2 * ne1 - i02 * ne1); + + // Handle broadcasting: src1 coordinates with modulo + const int64_t i13 = i03 % ne13; + const int64_t i12 = i02 % ne12; + const int64_t i11 = i01 % ne11; + + // Calculate base pointers for this row using stride-based addressing + float * dst_ptr = (float *) ((char *) dst_data + i03 * nb3 + i02 * nb2 + i01 * nb1); + const float * src0_ptr = (const float *) ((const char *) src0_data + i03 * nb03 + i02 * nb02 + i01 * nb01); + const float * src1_ptr = (const float *) ((const char *) src1_data + i13 * nb13 + i12 * nb12 + i11 * nb11); + + if (ne10 == 1) { + // Broadcast scalar: src1 has ne[0]=1, broadcast across entire row + float scalar = src1_ptr[0]; + switch (operation) { + case GGML_OP_MUL: + block_mul_broadcast(dst_ptr, src0_ptr, scalar, (int) ne0); + break; + case GGML_OP_ADD: + block_add_broadcast(dst_ptr, src0_ptr, scalar, (int) ne0); + break; + case GGML_OP_SUB: + block_sub_broadcast(dst_ptr, src0_ptr, scalar, (int) ne0); + break; + default: + return 1; + } + } else { + // Broadcasting in dimension 0: src1 repeats across src0 + const int64_t nr0 = ne0 / ne10; + + for (int64_t r = 0; r < nr0; r++) { + const float * src0_block = src0_ptr + r * ne10; + float * dst_block = dst_ptr + r * ne10; + + switch (operation) { + case GGML_OP_MUL: + block_mul_cache_aligned(dst_block, src0_block, src1_ptr, (int) ne10); + break; + case GGML_OP_ADD: + block_add_cache_aligned(dst_block, src0_block, src1_ptr, (int) ne10); + break; + case GGML_OP_SUB: + block_sub_cache_aligned(dst_block, src0_block, src1_ptr, (int) ne10); + break; + default: + return 1; + } + } + } + } + +#ifdef ET_UBERKERNEL + // Producer-side flush for the cache-aligned slow path. Rows + // [start_row, end_row) are contiguous in dst because nb1 = ne0 * 4. + // Only needed inside a UK batch; see comment in fast path. + if (end_row > start_row) { + FENCE; + evict_region_past_l2((char *) dst_data + start_row * nb1, (size_t) (end_row - start_row) * nb1); + WAIT_CACHEOPS; + FENCE; + } +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/fill_f32.c b/ggml/src/ggml-et/et-kernels/src/fill_f32.c new file mode 100644 index 0000000..1847c8d --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/fill_f32.c @@ -0,0 +1,87 @@ +//****************************************************************************** +// Fill F32 Kernel +// Fills entire tensor with a constant scalar value. +// dst[i] = c for all elements +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_fill_params { + struct ggml_tensor dst; // F32 output tensor (contiguous) + float c; // Constant value to fill +}; + +int entry_point(struct ggml_et_fill_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * dst = ¶ms->dst; + + if (dst->type != GGML_TYPE_F32) { + return -1; + } + + float * dst_data = (float *) dst->data; + if (!dst_data) { + return -1; + } + + const int64_t total_elements = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; + + if (total_elements == 0) { + return 0; + } + + // Distribute by cache lines (16 floats = 64 bytes) + const int64_t elems_per_cl = 16; + const int64_t total_cl = (total_elements + elems_per_cl - 1) / elems_per_cl; + const int64_t cl_per_thread = (total_cl + num_threads - 1) / num_threads; + const int64_t cl_start = thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cl) { + cl_end = total_cl; + } + if (cl_start >= total_cl) { + return 0; + } + + const int64_t es = cl_start * elems_per_cl; + int64_t ee = cl_end * elems_per_cl; + if (ee > total_elements) { + ee = total_elements; + } + + // Broadcast constant to all SIMD lanes + float c = params->c; + __asm__ volatile("fbc.ps f10, %[v]\n" : : [v] "m"(c) : "f10"); + + // Vector fill (8-wide) + int64_t i = es; + const int64_t vec_end = es + ((ee - es) / 8) * 8; + for (; i < vec_end; i += 8) { + __asm__ volatile("fsw.ps f10, %[d]\n" : [d] "=m"(*(float (*)[8]) & dst_data[i])::"f10"); + } + // Scalar tail + for (; i < ee; i++) { + dst_data[i] = c; + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c b/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c new file mode 100644 index 0000000..c905b36 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c @@ -0,0 +1,1000 @@ +//****************************************************************************** +// Flash Attention with TensorFMA16A32 for QK^T +// +// Uses the matrix engine for the QK^T dot products (F16×F16→F32), +// scalar code for online softmax and V accumulation. +// +// Hart 0: tensor engine (Q load, K load from SCP, FMA, softmax, V accum) +// Hart 1: pack K into double-buffered L2 SCP panels, flush for tensor_load +// +// Requirements: +// - Q: F32 (converted to F16 internally) +// - K, V: F16 +// - dk must be a multiple of 32 (TensorFMA16A32 K-tile) +// - dv ≤ 512 (accumulator in shire-local L2 SCP) +// +// Parallelization: each minion independently processes one (qpos, head, batch) +// row, round-robin across all minion hart-0s. Hart 1 assists with K packing. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" +#include "tensor.h" + +#include +#include +#include + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +// QK^T tiles: 16 KV positions at a time, K in chunks of 32 F16 +#define TILE_KV 16 +#define TILE_K 32 + +// L1 scratchpad layout: A (Q) in lines 0-15, B (K interleaved) in lines 16-31 +#define A_L1_START 0 +#define B_L1_START 16 + +// Max head dimensions +#define FA_DV_MAX 512 // max value head dim (dv) +#define FA_DK_MAX 512 // max key head dim (dk) - some models use hsk > hsv + +typedef uint16_t et_fp16_t; + +#define ET_NEG_INF_F (-3.402823466e+38f) + +// L2 SCP layout per minion: +// [0..2047] accumulator (FA_DV_MAX * sizeof(float)) +// [2048..4095] kpanel buffer 0 (32 × 32 × 2 = 2048 bytes) +// [4096..6143] kpanel buffer 1 (2048 bytes) +// [6144..6207] stats line - (M_p at +0, S_p at +4), own cache line +// Double-buffering ensures hart 0 finishes buf[N%2] before hart 1 +// overwrites it at chunk N+2. +// +// The stats line reserves a cache-line-aligned slot for split-KV softmax +// partials (M_p, S_p). With k_splits=1 the slot is currently unused; step 2 +// will populate it and use peer minions' slots during the reduction. +#define SCP_ACC_OFF 0 +#define SCP_ACC_STRIDE (FA_DV_MAX * sizeof(float)) // 2048 +#define SCP_KPANEL_SIZE (32 * 32 * sizeof(et_fp16_t)) // 2048 +#define SCP_KP0_OFF SCP_ACC_STRIDE // 2048 +#define SCP_KP1_OFF (SCP_KP0_OFF + SCP_KPANEL_SIZE) // 4096 +#define SCP_STATS_OFF (SCP_KP1_OFF + SCP_KPANEL_SIZE) // 6144 +#define SCP_STATS_SIZE 64 // own cache line +#define SCP_PER_MINION (SCP_STATS_OFF + SCP_STATS_SIZE) // 6208 + +struct ggml_et_flash_attn_ext_params { + struct ggml_tensor src0; // Q (F32) + struct ggml_tensor src1; // K (F16) + struct ggml_tensor src2; // V (F16) + struct ggml_tensor mask; // mask (F16 or F32), zeroed when absent + struct ggml_tensor dst; // Output (F32) + float scale; + int32_t has_mask; +}; + +static inline float get_mask_val(const struct ggml_tensor * mask, int64_t iq1, int64_t ik1, int64_t iq2, int64_t iq3) { + const char * base = (const char *) mask->data + iq1 * mask->nb[1] + (iq2 % mask->ne[2]) * mask->nb[2] + + (iq3 % mask->ne[3]) * mask->nb[3]; + + if (mask->type == GGML_TYPE_F32) { + return *(const float *) (base + ik1 * mask->nb[0]); + } + return fp16_to_fp32(*(const uint16_t *) (base + ik1 * mask->nb[0])); +} + +static inline const char * get_mask_row_base(const struct ggml_tensor * mask, int64_t iq1, int64_t iq2, int64_t iq3) { + return (const char *) mask->data + iq1 * mask->nb[1] + (iq2 % mask->ne[2]) * mask->nb[2] + + (iq3 % mask->ne[3]) * mask->nb[3]; +} + +static inline float get_mask_val_from_base(const struct ggml_tensor * mask, const char * base, int64_t ik1) { + if (mask->type == GGML_TYPE_F32) { + return *(const float *) (base + ik1 * mask->nb[0]); + } + return fp16_to_fp32(*(const uint16_t *) (base + ik1 * mask->nb[0])); +} + +// Pack K rows for TensorLoadTranspose16 (even/odd deinterleave) +static inline void __attribute__((always_inline)) pack_k_for_transpose16(et_fp16_t * out, + const char * k_base, + int64_t kv_start, + int64_t dk_start, + int64_t kv_count, + int64_t nb1_k) { + unsigned long old_mask; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + : [ms] "=&r"(old_mask) + : + :); + + for (int j = 0; j < (int) kv_count; ++j) { + const et_fp16_t * k_row = (const et_fp16_t *) (k_base + (kv_start + j) * nb1_k) + dk_start; + et_fp16_t * even_row = out + (j * 2) * 32; + et_fp16_t * odd_row = out + (j * 2 + 1) * 32; + __asm__ volatile( + "flw.ps f2, 0(%[src0]) \n\t" // load row[0..15] + "flw.ps f3, 0(%[src1]) \n\t" // load row[16..31] + "fpackreph.pi f4, f2 \n\t" // even_lo from src0 + "fpackreph.pi f6, f3 \n\t" // even_lo from src1 (interleaved) + "fsrli.pi f5, f2, 16 \n\t" // shift src0 for odd + "fsrli.pi f7, f3, 16 \n\t" // shift src1 for odd (interleaved) + "fpackreph.pi f5, f5 \n\t" // odd from src0 + "fpackreph.pi f7, f7 \n\t" // odd from src1 + "mov.m.x m0, x0, 0x0F \n\t" + "fcmovm.ps f4, f4, f6 \n\t" // merge even halves + "fcmovm.ps f5, f5, f7 \n\t" // merge odd halves + "mov.m.x m0, x0, 0xFF \n\t" + "fsw.ps f4, 0(%[even]) \n\t" + "fsw.ps f5, 0(%[odd]) \n\t" + : + : [src0] "r"(k_row), [src1] "r"(k_row + 16), [even] "r"(even_row), [odd] "r"(odd_row) + : "f2", "f3", "f4", "f5", "f6", "f7", "memory"); + } + + __asm__ volatile("mova.m.x %[ms] \n\t" : : [ms] "r"(old_mask)); + + for (int j = (int) kv_count; j < TILE_KV; ++j) { + et_fp16_t * even_row = out + (j * 2) * 32; + et_fp16_t * odd_row = out + (j * 2 + 1) * 32; + for (int l = 0; l < TILE_K / 2; ++l) { + even_row[l] = 0; + odd_row[l] = 0; + } + } +} + +// Build interleaved B panel for TensorFMA16A32 (weights @ V). +static inline void __attribute__((always_inline)) pack_v_interleaved(et_fp16_t * out, + const char * v_head, + int64_t kv_base, + int64_t dv_start, + int64_t kv_count, + int64_t nb1_v) { + for (int k = 0; k < TILE_KV; ++k) { + const int l = k >> 1; + const int r = k & 1; + et_fp16_t * const dst = out + l * 32 + r; + if (k < (int) kv_count) { + const et_fp16_t * v_row = (const et_fp16_t *) (v_head + (kv_base + k) * nb1_v) + dv_start; + for (int n = 0; n < 16; ++n) { + dst[n * 2] = v_row[n]; + } + } else { + for (int n = 0; n < 16; ++n) { + dst[n * 2] = 0; + } + } + } +} + +// Prefetch KV rows for one chunk into L2. +static inline void __attribute__((always_inline)) prefetch_kv_to_l2(const char * head, + int64_t kv_start, + int64_t d_start, + int64_t kv_count, + int64_t nb1) { + const void * base = (const void *) (head + kv_start * nb1 + d_start * 2); + l2_prefetch(base, (uint64_t) kv_count, (uint64_t) nb1); +} + +static inline void __attribute__((always_inline)) convert_q_row_f32_to_f16(et_fp16_t * dst, + const float * src, + int64_t n) { + static const int32_t __attribute__((aligned(32))) offsets[8] = { 0, 2, 4, 6, 8, 10, 12, 14 }; + + unsigned long old_mask; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "flw.ps f1, 0(%[offs]) \n\t" + : [ms] "=&r"(old_mask) + : [offs] "r"(offsets) + : "f1"); + + for (int64_t d = 0; d < n; d += 8) { + __asm__ volatile( + "flw.ps f2, 0(%[src]) \n\t" + "fcvt.f16.ps f3, f2 \n\t" + "fsch.ps f3, f1(%[dst]) \n\t" + : + : [src] "r"(src + d), [dst] "r"(dst + d) + : "f2", "f3", "memory"); + } + + __asm__ volatile("mova.m.x %[ms] \n\t" : : [ms] "r"(old_mask)); +} + +static inline void __attribute__((always_inline)) zero_acc_vec(float * acc, int64_t dv) { + const float zero = 0.0f; + unsigned long old_mask; + __asm__ volatile("mova.x.m %0" : "=r"(old_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + __asm__ volatile("fbc.ps f2, 0(%[z])" ::[z] "r"(&zero) : "f2"); + + for (int64_t d = 0; d < dv; d += 8) { + __asm__ volatile("fsw.ps f2, 0(%[a]) \n\t" ::[a] "r"(acc + d) : "f2", "memory"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(old_mask)); +} + +static inline void __attribute__((always_inline)) scale_acc_vec(float * acc, int64_t dv, float scale) { + unsigned long old_mask; + __asm__ volatile("mova.x.m %0" : "=r"(old_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + for (int64_t d = 0; d < dv; d += 8) { + __asm__ volatile( + "fbc.ps f2, 0(%[s]) \n\t" + "flw.ps f3, 0(%[a]) \n\t" + "fmul.ps f3, f3, f2 \n\t" + "fsw.ps f3, 0(%[a]) \n\t" + : + : [s] "r"(&scale), [a] "r"(acc + d) + : "f2", "f3", "memory"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(old_mask)); +} + +static inline void __attribute__((always_inline)) normalize_store_vec(float * out, + float * acc, + int64_t dv, + float inv, + int use_fast_store) { + unsigned long old_mask; + __asm__ volatile("mova.x.m %0" : "=r"(old_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + for (int64_t d = 0; d < dv; d += 8) { + __asm__ volatile( + "fbc.ps f2, 0(%[inv]) \n\t" + "flw.ps f3, 0(%[a]) \n\t" + "fmul.ps f3, f3, f2 \n\t" + "fsw.ps f3, 0(%[a]) \n\t" + : + : [inv] "r"(&inv), [a] "r"(acc + d) + : "f2", "f3", "memory"); + if (use_fast_store) { + __asm__ volatile( + "flw.ps f4, 0(%[a]) \n\t" + "fsw.ps f4, 0(%[o]) \n\t" + : + : [a] "r"(acc + d), [o] "r"(out + d) + : "f4", "memory"); + } else { + atomic_store_f32((volatile float *) &out[d + 0], acc[d + 0]); + atomic_store_f32((volatile float *) &out[d + 1], acc[d + 1]); + atomic_store_f32((volatile float *) &out[d + 2], acc[d + 2]); + atomic_store_f32((volatile float *) &out[d + 3], acc[d + 3]); + atomic_store_f32((volatile float *) &out[d + 4], acc[d + 4]); + atomic_store_f32((volatile float *) &out[d + 5], acc[d + 5]); + atomic_store_f32((volatile float *) &out[d + 6], acc[d + 6]); + atomic_store_f32((volatile float *) &out[d + 7], acc[d + 7]); + } + } + + __asm__ volatile("mova.m.x %0" ::"r"(old_mask)); +} + +static inline size_t tensor_bytes_fa(const struct ggml_tensor * t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} + +// Evict a byte range from L1D to L2 SCP, splitting into batches of ≤16 +// cache lines (the hw limit for evict_to_l2). Use before a barrier when +// another minion in the shire needs to read the region, or after a barrier +// on the reader side to drop stale L1D copies before reading peer data. +static inline void __attribute__((always_inline)) evict_range_to_l2(const void * addr, int64_t bytes) { + if (bytes <= 0) { + return; + } + int64_t lines = (bytes + 63) / 64; + const char * p = (const char *) addr; + while (lines > 0) { + int64_t batch = lines > 16 ? 16 : lines; + evict_to_l2((const void *) p, (uint64_t) batch, 64); + p += batch * 64; + lines -= batch; + } +} + +// Split-KV online merge inner loop: +// +// for d in [0, dv) step 8: +// acc[d..d+8] = alpha_own * acc[d..d+8] + alpha_peer * peer_acc[d..d+8] +// +// Runs on the reducer (k_split == 0) after all tensor_fma ops for the row are +// complete, so f0..f31 are dead at entry. We still bracket the loop in inline +// asm with explicit f2/f3/f4/f5 clobbers to lock register usage down — per the +// MM register lifetime rule, never let the compiler mingle FP ops into code +// that sits anywhere near a tensor engine output window. +static inline void __attribute__((always_inline)) merge_rescale_add_asm(float * acc, + const float * peer_acc, + int64_t dv, + float alpha_own, + float alpha_peer) { + unsigned long old_mask; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "fbc.ps f4, 0(%[ao]) \n\t" // broadcast alpha_own + "fbc.ps f5, 0(%[ap]) \n\t" // broadcast alpha_peer + : [ms] "=&r"(old_mask) + : [ao] "r"(&alpha_own), [ap] "r"(&alpha_peer) + : "f4", "f5"); + + for (int64_t d = 0; d < dv; d += 8) { + __asm__ volatile( + "flw.ps f2, 0(%[a]) \n\t" // own + "flw.ps f3, 0(%[p]) \n\t" // peer + "fmul.ps f2, f2, f4 \n\t" // own *= alpha_own + "fmul.ps f3, f3, f5 \n\t" // peer *= alpha_peer + "fadd.ps f2, f2, f3 \n\t" + "fsw.ps f2, 0(%[a]) \n\t" + : + : [a] "r"(acc + d), [p] "r"(peer_acc + d) + : "f2", "f3", "memory"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(old_mask)); +} + +int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) { + return 0; + } + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + struct ggml_tensor * q = ¶ms->src0; + struct ggml_tensor * k = ¶ms->src1; + struct ggml_tensor * v = ¶ms->src2; + struct ggml_tensor * dst = ¶ms->dst; + const int32_t has_mask = params->has_mask; + struct ggml_tensor * mask = has_mask ? ¶ms->mask : (struct ggml_tensor *) 0; + + const char * q_data = (const char *) q->data; + const char * k_data = (const char *) k->data; + const char * v_data = (const char *) v->data; + char * dst_data = (char *) dst->data; + + // et_barrier(ET_BARRIER_GLOBAL); + evict_region_past_l2(q->data, tensor_bytes_fa(q)); + evict_region_past_l2(k->data, tensor_bytes_fa(k)); + evict_region_past_l2(v->data, tensor_bytes_fa(v)); + if (mask) { + evict_region_past_l2(mask->data, tensor_bytes_fa(mask)); + } + et_barrier(ET_BARRIER_GLOBAL); + + const int64_t dk = q->ne[0]; + const int64_t nq = q->ne[1]; + const int64_t nhq = q->ne[2]; + const int64_t no = q->ne[3]; + const int64_t nk = k->ne[1]; + const int64_t nhk = k->ne[2]; + const int64_t dv = v->ne[0]; + + if (dv > FA_DV_MAX || dk > FA_DK_MAX) { + return -1; + } + if (k->nb[0] != 2 || v->nb[0] != 2) { + return -1; + } + if ((dk % 8) != 0 || (dv % 16) != 0) { + return -1; + } + + const int64_t gqa_ratio = nhq / nhk; + const int64_t total_rows = nq * nhq * no; + const float scale = params->scale; + const int use_fast_store = (dv % 16 == 0); + + // Split-KV team layout (mirrors mul_mat_f16_matrix_engine.c) + // + // When total_rows is small compared to the total minion count (typical + // for decode: nq=1, nhq small), we group k_splits minions within the + // same shire into a team that cooperates on one row by splitting the + // KV dimension. Each team member computes a partial (M_p, S_p, acc_p) + // over its KV slab; the k_split==0 member merges the partials with the + // softmax combine rule. + // + // k_splits is a power of two, capped at MINIONS_PER_SHIRE (so a team + // never spans shires — L2 SCP is shire-local) and at nk_tiles (so each + // team member gets at least one KV tile). + const int64_t nk_tiles = (nk + TILE_KV - 1) / TILE_KV; + const int64_t total_minions = 2 * NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE; + int64_t k_splits = 1; + if (total_rows < total_minions) { + int64_t target = total_minions / total_rows; + int64_t ks = 1; + while (ks * 2 <= target && ks * 2 <= MINIONS_PER_SHIRE && ks * 2 <= nk_tiles) { + ks *= 2; + } + k_splits = ks; + } + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; + const int64_t k_split = (int64_t) local_minion % k_splits; + const int64_t local_tile_idx = (int64_t) local_minion / k_splits; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + + // KV slab for this k_split. With k_splits=1 this is the full range. + const int64_t tiles_per_split_rounded = (nk_tiles + k_splits - 1) / k_splits; + const int64_t tile_start = k_split * tiles_per_split_rounded; + int64_t tile_end = tile_start + tiles_per_split_rounded; + if (tile_end > nk_tiles) { + tile_end = nk_tiles; + } + const int64_t kv_start = tile_start * TILE_KV; + int64_t kv_end = tile_end * TILE_KV; + if (kv_end > nk) { + kv_end = nk; + } + + // L2 SCP pointers for this minion + uint64_t scp_base = local_minion * SCP_PER_MINION; + et_fp16_t * scp_kp[2] = { + (et_fp16_t *) et_shire_l2scp_local(scp_base + SCP_KP0_OFF), + (et_fp16_t *) et_shire_l2scp_local(scp_base + SCP_KP1_OFF), + }; + + // Hart 1 does K-panel packing + // + // When k_splits > 1, hart 1 must also participate in the two shire + // barriers that bracket the merge phase (one before and one after, so + // the reducer can read peer partials safely and the writers know when + // their acc/stats slab is free to reuse). Hart 1 has no useful work + // between those barriers. + // + // All teams in a shire must iterate the same number of times so the + // per-iter shire barriers stay balanced. Teams whose assigned row is + // past total_rows still call the barriers but skip the packing work. + et_barrier(ET_BARRIER_SHIRE); + // et_barrier(ET_BARRIER_GLOBAL); + if (is_hart1) { + uint32_t chunk_id = 0; + const int64_t row_base = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + + int64_t max_iters; + if (k_splits > 1) { + max_iters = (total_rows + tiles_stride - 1) / tiles_stride; + } else { + max_iters = (row_base >= total_rows) ? 0 : ((total_rows - row_base - 1) / tiles_stride + 1); + } + + for (int64_t iter = 0; iter < max_iters; iter++) { + const int64_t row = row_base + iter * tiles_stride; + const int has_work = (row < total_rows); + + if (has_work) { + const int64_t iq3 = row / (nhq * nq); + const int64_t rem = row % (nhq * nq); + const int64_t iq2 = rem / nq; + const int64_t ik2 = iq2 / gqa_ratio; + + const char * k_head = k_data + ik2 * k->nb[2] + iq3 * k->nb[3]; + + for (int64_t kv_base = kv_start; kv_base < kv_end; kv_base += TILE_KV) { + const int64_t kv_count = (kv_base + TILE_KV <= nk) ? TILE_KV : (nk - kv_base); + + for (int64_t dk_chunk = 0; dk_chunk < dk; dk_chunk += TILE_K) { + int buf = chunk_id & 1; + + // Back-pressure: before overwriting buf[buf] on chunk N + // (which will displace chunk N-2), wait for hart 0 to + // post that it's done with chunk N-2. Gates both + // directions of double-buffering. + // + // NOTE: we use et_sem_* (FCC 0 only) rather than + // et_barrier(ET_BARRIER_MINION) here because the + // minion barrier for minion 0 shares FLB 0 with + // ET_BARRIER_SHIRE. Mixing them deadlocks. See + // feedback_flb_collision. + if (chunk_id >= 2) { + et_sem_wait(ET_BARRIER_MINION); + } + + // Prefetch K data for this chunk + prefetch_kv_to_l2(k_head, kv_base, dk_chunk, kv_count, k->nb[1]); + + pack_k_for_transpose16(scp_kp[buf], k_head, kv_base, dk_chunk, kv_count, k->nb[1]); + + FENCE; + flush_to_l2(scp_kp[buf], 16, 64); + flush_to_l2((et_fp16_t *) ((char *) scp_kp[buf] + 1024), 16, 64); + WAIT_CACHEOPS; + + // Signal: this buf is ready for hart 0 to consume. + et_sem_post(ET_BARRIER_MINION); + + chunk_id++; + } + } + } + + // Shire barriers for split-KV merge (hart 1 is a passive arrival). + if (k_splits > 1) { + et_barrier(ET_BARRIER_SHIRE); // A: team has written its partial + et_barrier(ET_BARRIER_SHIRE); // B: reducer has finished merge + } + } + + // Self-drain phantom FCC 0 credits left by the wait-skip on the + // first 2 chunks. Hart 1 issued chunk_id posts but only + // (chunk_id - 2) waits (when chunk_id >= 2), so hart 1's FCC 0 + // carries +min(chunk_id,2) credits from hart 0's matching posts + // that hart 1 never consumed. + uint32_t drain = (chunk_id < 2) ? chunk_id : 2; + for (uint32_t d = 0; d < drain; d++) { + et_sem_wait(ET_BARRIER_MINION); + } + + // FENCE; + // et_barrier(ET_BARRIER_GLOBAL); + return 0; + } + + // Hart 0: tensor engine compute +#ifndef UBERKERNEL_SUPPRESS_SCP_SETUP + setup_cache_scp(); +#endif + CLEAR_TENSOR_ERROR; + + // Q converted to F16 (one row at a time) + et_fp16_t q_f16[FA_DK_MAX] __attribute__((aligned(64))); + + // Score buffer for QK^T output (16 scores per KV tile) + float scores[TILE_KV] __attribute__((aligned(64))); + + // Small buffers for V accumulation + et_fp16_t w_f16_buf[32] __attribute__((aligned(64))); // 64 bytes + et_fp16_t vpanel_buf[8 * 32] __attribute__((aligned(64))); // 512 bytes + + float * acc = (float *) et_shire_l2scp_local(scp_base + SCP_ACC_OFF); + + uint32_t chunk_id = 0; + + // Iter-based outer loop (matches hart 1). When k_splits > 1 all teams + // in a shire iterate the same number of times so the per-row shire + // barriers stay balanced; iterations with row >= total_rows skip the + // compute but still participate in the barriers. + const int64_t hart0_row_base = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; + int64_t hart0_max_iters; + if (k_splits > 1) { + hart0_max_iters = (total_rows + tiles_stride - 1) / tiles_stride; + } else { + hart0_max_iters = (hart0_row_base >= total_rows) ? 0 : ((total_rows - hart0_row_base - 1) / tiles_stride + 1); + } + + for (int64_t iter = 0; iter < hart0_max_iters; iter++) { + const int64_t row = hart0_row_base + iter * tiles_stride; + if (row >= total_rows) { + // No-work iteration: only participate in barriers (k_splits > 1). + if (k_splits > 1) { + et_barrier(ET_BARRIER_SHIRE); // A + et_barrier(ET_BARRIER_SHIRE); // B + } + continue; + } + + const int64_t iq3 = row / (nhq * nq); + const int64_t rem = row % (nhq * nq); + const int64_t iq2 = rem / nq; + const int64_t iq1 = rem % nq; + const int64_t ik2 = iq2 / gqa_ratio; + + // Read Q row (F32) and convert to F16 + const float * pq = (const float *) (q_data + iq1 * q->nb[1] + iq2 * q->nb[2] + iq3 * q->nb[3]); + convert_q_row_f32_to_f16(q_f16, pq, dk); + + // V base for this head + batch (K packing handled by hart 1) + const char * v_head = v_data + ik2 * v->nb[2] + iq3 * v->nb[3]; + + // Output pointer + float * out = (float *) (dst_data + iq2 * dst->nb[1] + iq1 * dst->nb[2] + iq3 * dst->nb[3]); + + zero_acc_vec(acc, dv); + float M = ET_NEG_INF_F; + float S = 0.0f; + const char * mask_base = has_mask ? get_mask_row_base(mask, iq1, iq2, iq3) : (const char *) 0; + + // Flush Q_f16 to L2 so tensor_load can see it + FENCE; + flush_to_l2(q_f16, (dk * 2 + 63) / 64, 64); + WAIT_CACHEOPS; + + for (int64_t kv_base = kv_start; kv_base < kv_end; kv_base += TILE_KV) { + const int64_t kv_count = (kv_base + TILE_KV <= nk) ? TILE_KV : (nk - kv_base); + + // Set tensor_mask for partial tiles + if (kv_count < TILE_KV) { + uint64_t tmask = (1ULL << kv_count) - 1; + __asm__ __volatile__("csrw 0x805, %0" : : "r"(tmask)); + } + + // ============================================================ + // QK^T via TensorFMA16A32 + // ============================================================ + + // Pipelined QK^T: + // - Q for the whole row is preloaded once into A_L1[0..n-1]. + // Each FMA picks its chunk via scp_loc_a = chunk_idx. + // - K is double-buffered in L1: K_BUFS[0]=lines 16..31, + // K_BUFS[1]=lines 32..47. + // - In iteration i (1..N-1), the K[i] load runs concurrently + // with the FMA on chunk i-1: they touch disjoint L1 regions + // (FMA reads K_BUFS[(i-1)&1], load writes K_BUFS[i&1]; FMA + // reads A_L1[i-1], load doesn't touch A_L1). + // + // L1 footprint: max dk=512 → Q uses 16 lines (0..15), K uses 32 + // lines (16..47). Within ET-SoC-1 L1 SCP (≥128 lines per minion). + const int64_t n_dk_chunks = dk / TILE_K; + const uint64_t K_BUFS[2] = { + (uint64_t) B_L1_START, // 16..31 + (uint64_t) (B_L1_START + 16), // 32..47 + }; + + // Preload entire Q row into A_L1[0..n_dk_chunks-1] (one tensor_load, + // one wait, regardless of dk). + tensor_load(false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, (uint64_t) q_f16, 0, + (uint64_t) (n_dk_chunks - 1), 64, 0); + + // Prologue: wait hart 1's K[0], issue K[0] load, wait both loads. + { + int buf = chunk_id & 1; + et_sem_wait(ET_BARRIER_MINION); + tensor_load(false, false, K_BUFS[0], TENSOR_LOAD_TRANSPOSE16, 0, (uint64_t) scp_kp[buf], 0, 15, 64, 1); + tensor_wait(TENSOR_LOAD_WAIT_0); // Q row complete + tensor_wait(TENSOR_LOAD_WAIT_1); // K[0] complete + et_sem_post(ET_BARRIER_MINION); + chunk_id++; + } + + // Main loop: in iter i, issue K[i] load and FMA chunk i-1 in + // parallel. The matrix engine is busy on FMA[i-1] while the + // load unit fetches K[i] from L2 SCP. + // + // Order of waits matters: wait K[i] load first, then sem_post + // immediately (frees scp_kp[buf] for hart 1 to refill chunk i+2), + // then wait FMA. Putting sem_post after FMA wait would stall + // hart 1 by a full FMA latency — defeating the producer pipeline. + for (int64_t i = 1; i < n_dk_chunks; i++) { + int buf = chunk_id & 1; + int k_slot_prev = (int) ((i - 1) & 1); + int k_slot = (int) (i & 1); + + et_sem_wait(ET_BARRIER_MINION); + tensor_load(false, false, K_BUFS[k_slot], TENSOR_LOAD_TRANSPOSE16, 0, (uint64_t) scp_kp[buf], 0, 15, 64, + 1); + + tensor_fma((kv_count < TILE_KV), 3, 0, 15, 0, false, false, false, false, K_BUFS[k_slot_prev], + (uint64_t) (i - 1), TENSOR_FMA_OP_FP16, (i == 1)); + + tensor_wait(TENSOR_LOAD_WAIT_1); // K[i] in L1 + et_sem_post(ET_BARRIER_MINION); // release scp_kp[buf] EARLY + tensor_wait(TENSOR_FMA_WAIT); // then wait FMA[i-1] + chunk_id++; + } + + // Epilogue: FMA on the last chunk (no overlapping load). + { + int k_slot_last = (int) ((n_dk_chunks - 1) & 1); + tensor_fma((kv_count < TILE_KV), 3, 0, 15, 0, false, false, false, false, K_BUFS[k_slot_last], + (uint64_t) (n_dk_chunks - 1), TENSOR_FMA_OP_FP16, (n_dk_chunks == 1)); + tensor_wait(TENSOR_FMA_WAIT); + } + + // Prefetch V rows for this tile. + // Only useful for the partial-tile path below + if (kv_count < TILE_KV) { + for (int64_t d = 0; d < dv; d += 32) { + prefetch_kv_to_l2(v_head, kv_base, d, kv_count, v->nb[1]); + } + } + + // Extract QK^T scores from vector register file + __asm__ volatile("" ::: "f0", "f1"); + { + unsigned long _ms; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "fbc.ps f2, 0(%[p_scale]) \n\t" + "fmul.ps f0, f0, f2 \n\t" + "fmul.ps f1, f1, f2 \n\t" + "fsw.ps f0, 0(%[dst]) \n\t" + "fsw.ps f1, 32(%[dst]) \n\t" + "mova.m.x %[ms] \n\t" + : [ms] "=&r"(_ms) + : [dst] "r"(scores), [p_scale] "r"(&scale) + : "f0", "f1", "f2", "memory"); + } + + // ============================================================ + // Two-phase softmax + V accumulation + // ============================================================ + + float weights[TILE_KV] __attribute__((aligned(64))); + { + // A1: apply mask to scores, pad unused slots + for (int64_t j = 0; j < kv_count; ++j) { + if (has_mask) { + float mv = get_mask_val_from_base(mask, mask_base, kv_base + j); + if (mv == ET_NEG_INF_F || mv != mv) { + scores[j] = ET_NEG_INF_F; + } else { + scores[j] += mv; + } + } + } + for (int64_t j = kv_count; j < TILE_KV; ++j) { + scores[j] = ET_NEG_INF_F; + } + + // A1b: SIMD horizontal max across all 16 scores + float tile_max; + { + unsigned long _ms; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "flw.ps f2, 0(%[sc]) \n\t" + "flw.ps f3, 32(%[sc]) \n\t" + "fmax.ps f2, f2, f3 \n\t" + "fswizz.ps f3, f2, 0xB1 \n\t" + "fmax.ps f2, f2, f3 \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fmax.ps f2, f2, f3 \n\t" + "fmvz.x.ps t0, f2, 4 \n\t" + "fbcx.ps f3, t0 \n\t" + "fmax.ps %[tm], f2, f3 \n\t" + "mova.m.x %[ms] \n\t" + : [ms] "=&r"(_ms), [tm] "=f"(tile_max) + : [sc] "r"(scores) + : "f2", "f3", "t0", "memory"); + } + + if (tile_max > ET_NEG_INF_F) { + // A2: rescale accumulator if this tile has a new global max + if (tile_max > M) { + float rescale = et_exp2f((M - tile_max) * 1.4426950408889634f); + scale_acc_vec(acc, dv, rescale); + S *= rescale; + M = tile_max; + } + + // A3: SIMD exp2 + horizontal sum + // Interleaved: f2/f3 chains alternate to hide ALU latency. + // fexp.ps has multi-cycle latency — the two independent + // exp2 calls naturally pipeline. + { + const float log2e = 1.4426950408889634f; + float S_tile; + unsigned long _ms; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "flw.ps f2, 0(%[sc]) \n\t" + "fbc.ps f4, 0(%[pM]) \n\t" + "flw.ps f3, 32(%[sc]) \n\t" + "fbc.ps f5, 0(%[pL]) \n\t" + "fsub.ps f2, f2, f4 \n\t" + "fsub.ps f3, f3, f4 \n\t" + "fmul.ps f2, f2, f5 \n\t" + "fmul.ps f3, f3, f5 \n\t" + "fexp.ps f2, f2 \n\t" + "fexp.ps f3, f3 \n\t" + "fsw.ps f2, 0(%[wt]) \n\t" + "fsw.ps f3, 32(%[wt]) \n\t" + "fadd.ps f2, f2, f3, rne \n\t" + "fswizz.ps f3, f2, 0xB1 \n\t" + "fadd.ps f2, f2, f3, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f2, f2, f3, rne \n\t" + "fmvz.x.ps t0, f2, 4 \n\t" + "fbcx.ps f3, t0 \n\t" + "fadd.ps %[st], f2, f3, rne \n\t" + "mova.m.x %[ms] \n\t" + : [ms] "=&r"(_ms), [st] "=f"(S_tile) + : [pM] "r"(&M), [pL] "r"(&log2e), [sc] "r"(scores), [wt] "r"(weights) + : "f2", "f3", "f4", "f5", "t0", "memory"); + S += S_tile; + } + + // Phase B: weights @ V via TensorFMA16A32 + { + // B1: convert weights F32 → F16 + convert_q_row_f32_to_f16(w_f16_buf, weights, TILE_KV); + + FENCE; + flush_to_l2(w_f16_buf, 1, 64); + WAIT_CACHEOPS; + + // Issue weights load (wait_id=0) and the first V chunk + // load (wait_id=1) concurrently. Weights comes from + // L2 SCP (just flushed); V[0] comes from DRAM via + // INTERLEAVE16 — running them in parallel hides the + // shorter load behind the longer one. For partial + // tiles, V is software-packed below — we only kick + // off the early V load on the full-tile fast path. + tensor_load(false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, (uint64_t) w_f16_buf, 0, 0, 64, 0); + + const int v_full_tile = (kv_count == TILE_KV); + const uintptr_t v_base = (uintptr_t) v_head + kv_base * v->nb[1]; + const uint64_t nb1_v = (uint64_t) v->nb[1]; + uint64_t b_cur = 8; + + if (v_full_tile) { + tensor_load(false, false, b_cur, TENSOR_LOAD_INTERLEAVE16, 0, (uint64_t) v_base, 0, 7, + nb1_v, 1); + } + + tensor_wait(TENSOR_LOAD_WAIT_0); // weights in A_L1 + if (v_full_tile) { + tensor_wait(TENSOR_LOAD_WAIT_1); // V[0] in b_cur + } + + // B2: process dv in chunks of 16 + if (v_full_tile) { + for (int64_t dv_off = 0; dv_off < dv; dv_off += 16) { + const uint64_t b_nxt = b_cur ^ 24; + + if (dv_off + 16 < dv) { + tensor_load(false, false, b_nxt, TENSOR_LOAD_INTERLEAVE16, 0, + (uint64_t) (v_base + (dv_off + 16) * 2), 0, 7, nb1_v, 1); + } + + tensor_fma(false, 3, 0, 7, 0, false, false, false, false, b_cur, A_L1_START, + TENSOR_FMA_OP_FP16, true); + tensor_wait(TENSOR_FMA_WAIT); + + __asm__ volatile("" ::: "f0", "f1"); + { + unsigned long _ms; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "flw.ps f2, 0(%[pa]) \n\t" + "flw.ps f3, 32(%[pa]) \n\t" + "fadd.ps f0, f0, f2 \n\t" + "fadd.ps f1, f1, f3 \n\t" + "fsw.ps f0, 0(%[pa]) \n\t" + "fsw.ps f1, 32(%[pa]) \n\t" + "mova.m.x %[ms] \n\t" + : [ms] "=&r"(_ms) + : [pa] "r"(acc + dv_off) + : "f0", "f1", "f2", "f3", "memory"); + } + + if (dv_off + 16 < dv) { + tensor_wait(TENSOR_LOAD_WAIT_1); + b_cur = b_nxt; + } + } + } else { + // Partial tile: software pack, no pipeline + for (int64_t dv_off = 0; dv_off < dv; dv_off += 16) { + pack_v_interleaved(vpanel_buf, v_head, kv_base, dv_off, kv_count, v->nb[1]); + FENCE; + flush_to_l2(vpanel_buf, 8, 64); + WAIT_CACHEOPS; + tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, (uint64_t) vpanel_buf, 0, 7, + 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + + tensor_fma(false, 3, 0, 7, 0, false, false, false, false, B_L1_START, A_L1_START, + TENSOR_FMA_OP_FP16, true); + tensor_wait(TENSOR_FMA_WAIT); + + __asm__ volatile("" ::: "f0", "f1"); + { + unsigned long _ms; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "flw.ps f2, 0(%[pa]) \n\t" + "flw.ps f3, 32(%[pa]) \n\t" + "fadd.ps f0, f0, f2 \n\t" + "fadd.ps f1, f1, f3 \n\t" + "fsw.ps f0, 0(%[pa]) \n\t" + "fsw.ps f1, 32(%[pa]) \n\t" + "mova.m.x %[ms] \n\t" + : [ms] "=&r"(_ms) + : [pa] "r"(acc + dv_off) + : "f0", "f1", "f2", "f3", "memory"); + } + } + } + } + } + } + } + + // Finalize row + // + // k_splits == 1: this minion computed the full row. Normalize in + // place and store to DRAM. + // + // k_splits > 1: this minion computed a KV slab. Publish the + // partial (M, S, acc) to L2 SCP, sync with the + // team, and let the k_split==0 member do the + // softmax combine and the final store. All tensor + // engine ops are complete before this block, so + // f0..f31 are free to use. + if (k_splits > 1) { + // Publish our partial. + volatile float * my_stats = (volatile float *) et_shire_l2scp_local(scp_base + SCP_STATS_OFF); + my_stats[0] = M; + my_stats[1] = S; + FENCE; + evict_range_to_l2(acc, (int64_t) dv * (int64_t) sizeof(float)); + evict_to_l2((const void *) my_stats, 1, 64); + WAIT_CACHEOPS; + + // A: team members have all written their partials. + et_barrier(ET_BARRIER_SHIRE); + + if (k_split == 0) { + // Online softmax merge: fold peers 1..k_splits-1 into our + // own (M_running, S_running, acc). For each peer p: + // M_new = max(M_running, M_p) + // α_own = exp2((M_running - M_new) * log2e) + // α_p = exp2((M_p - M_new) * log2e) + // acc[d] = α_own * acc[d] + α_p * peer_acc[d] + // S_running = α_own * S_running + α_p * S_p + float M_running = M; + float S_running = S; + const float log2e = 1.4426950408889634f; + + for (int64_t p = 1; p < k_splits; p++) { + uint64_t peer_scp = (local_tile_idx * k_splits + p) * SCP_PER_MINION; + volatile float * peer_stats = (volatile float *) et_shire_l2scp_local(peer_scp + SCP_STATS_OFF); + float * peer_acc = (float *) et_shire_l2scp_local(peer_scp + SCP_ACC_OFF); + + // Drop stale L1D copies before reading peer's data. + evict_to_l2((const void *) peer_stats, 1, 64); + evict_range_to_l2(peer_acc, (int64_t) dv * (int64_t) sizeof(float)); + WAIT_CACHEOPS; + + const float M_p = peer_stats[0]; + const float S_p = peer_stats[1]; + + const float M_new = (M_p > M_running) ? M_p : M_running; + const float alpha_own = (M_running == ET_NEG_INF_F) ? 0.0f : et_exp2f((M_running - M_new) * log2e); + const float alpha_p = (M_p == ET_NEG_INF_F) ? 0.0f : et_exp2f((M_p - M_new) * log2e); + + merge_rescale_add_asm(acc, peer_acc, dv, alpha_own, alpha_p); + + S_running = alpha_own * S_running + alpha_p * S_p; + M_running = M_new; + } + + const float S_inv = (S_running == 0.0f) ? 0.0f : et_fdiv(1.0f, S_running); + normalize_store_vec(out, acc, dv, S_inv, use_fast_store); + } + + // B: reducer is done, team may reuse its acc/stats slabs. + et_barrier(ET_BARRIER_SHIRE); + } else { + // k_splits == 1 fast path — this minion owns the full row. + const float S_inv = S == 0.0f ? 0.0f : et_fdiv(1.0f, S); + normalize_store_vec(out, acc, dv, S_inv, use_fast_store); + } + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c b/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c new file mode 100644 index 0000000..93b65b2 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c @@ -0,0 +1,217 @@ +//****************************************************************************** +// F32 Flash Attention for ET backend +// +// Supports: +// - arbitrary dk/dv (up to 128) +// - GQA (n_head_q can differ from n_head_kv) +// - mask (F16 or F32, causal pattern) +// - F16 or F32 K and V (with non-contiguous strides from KV cache permute) +// +// Limitations: +// - Q and dst must be F32 +// - no sinks, ALiBi, logit softcap +// +// Parallelization strategy: +// - flatten [query position, head, outer batch] into independent rows +// - assign rows round-robin across ET threads +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include +#include + +struct ggml_et_flash_attn_ext_params { + struct ggml_tensor src0; // Q tensor (F32) + struct ggml_tensor src1; // K tensor (F16 or F32) + struct ggml_tensor src2; // V tensor (F16 or F32) + struct ggml_tensor mask; // mask tensor (F16 or F32), zeroed when absent + struct ggml_tensor dst; // Output tensor (F32) + float scale; // Scale factor applied to QK + int32_t has_mask; // nonzero if mask is present +}; + +// Maximum head dimension supported (128 covers all common LLMs). +#define FA_DV_MAX 128 + +// Read element d from a row, handling F16 or F32 type. +// row_base points to the start of the row (byte address). +// nb0 is the stride per element (2 for F16, 4 for F32). +static inline float read_kv_f32(const char * row_base, int64_t d, int64_t nb0, int type) { + if (type == GGML_TYPE_F32) { + return *(const float *) (row_base + d * nb0); + } + // F16 + return fp16_to_fp32(*(const uint16_t *) (row_base + d * nb0)); +} + +// Dot product of F32 query vector with a K row (F16 or F32). +static inline float dot_qk(const float * q, const char * k_row, int64_t dk, int64_t k_nb0, int k_type) { + float acc = 0.0f; + if (k_type == GGML_TYPE_F32) { + const float * kf = (const float *) k_row; + for (int64_t i = 0; i < dk; ++i) { + acc += q[i] * kf[i]; + } + } else { + // F16 stride-aware read + for (int64_t i = 0; i < dk; ++i) { + acc += q[i] * fp16_to_fp32(*(const uint16_t *) (k_row + i * k_nb0)); + } + } + return acc; +} + +static inline float get_mask_val(const struct ggml_tensor * mask, int64_t iq1, int64_t ik1, int64_t iq2, int64_t iq3) { + // mask layout: [nk, nq, ne2, ne3] -> broadcast via modulo + const char * base = (const char *) mask->data + iq1 * mask->nb[1] + (iq2 % mask->ne[2]) * mask->nb[2] + + (iq3 % mask->ne[3]) * mask->nb[3]; + + if (mask->type == GGML_TYPE_F32) { + return *(const float *) (base + ik1 * mask->nb[0]); + } + // F16 + return fp16_to_fp32(*(const uint16_t *) (base + ik1 * mask->nb[0])); +} + +int entry_point(struct ggml_et_flash_attn_ext_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env || !params) { + return -1; + } + + const int thread_id = get_relative_thread_id(kernel_env->shire_mask); + const int num_threads = get_num_threads(kernel_env->shire_mask); + if (thread_id < 0 || num_threads <= 0) { + return 0; + } + + struct ggml_tensor * q = ¶ms->src0; + struct ggml_tensor * k = ¶ms->src1; + struct ggml_tensor * v = ¶ms->src2; + struct ggml_tensor * dst = ¶ms->dst; + const int32_t has_mask = params->has_mask; + struct ggml_tensor * mask = has_mask ? ¶ms->mask : (struct ggml_tensor *) 0; + + const char * q_data = (const char *) q->data; + const char * k_data = (const char *) k->data; + const char * v_data = (const char *) v->data; + char * dst_data = (char *) dst->data; + + const int k_type = k->type; + const int v_type = v->type; + const int64_t k_nb0 = k->nb[0]; + const int64_t v_nb0 = v->nb[0]; + + const int64_t dk = q->ne[0]; // head dim for keys/queries + const int64_t nq = q->ne[1]; // number of query positions + const int64_t nhq = q->ne[2]; // number of query heads + const int64_t no = q->ne[3]; // outer batch + + const int64_t nk = k->ne[1]; // number of key/value positions + const int64_t nhk = k->ne[2]; // number of kv heads + const int64_t dv = v->ne[0]; // head dim for values + + if (dv > FA_DV_MAX) { + return -1; + } + + // GQA: query heads per kv head + const int64_t gqa_ratio = nhq / nhk; + + const int64_t total_rows = nq * nhq * no; + const float scale = params->scale; + + // When dv is a multiple of 16 (64 bytes = cache line), output rows are + // cache-line aligned and we can use fast normal stores. Otherwise we must + // use atomic stores to avoid cache-line sharing corruption. + const int use_fast_store = (dv % 16 == 0); + + for (int64_t row = thread_id; row < total_rows; row += num_threads) { + const int64_t iq3 = row / (nhq * nq); + const int64_t rem = row % (nhq * nq); + const int64_t iq2 = rem / nq; // query head index + const int64_t iq1 = rem % nq; // query position + + // Map query head -> kv head for GQA + const int64_t ik2 = iq2 / gqa_ratio; + + // Q is always F32 + const float * pq = (const float *) (q_data + iq1 * q->nb[1] + iq2 * q->nb[2] + iq3 * q->nb[3]); + + // dst layout: [dv, nhq, nq, no] + float * out = (float *) (dst_data + iq2 * dst->nb[1] + iq1 * dst->nb[2] + iq3 * dst->nb[3]); + + // Base byte offsets for K and V head+batch slice + const int64_t kv_base = ik2 * k->nb[2] + iq3 * k->nb[3]; + const int64_t vv_base = ik2 * v->nb[2] + iq3 * v->nb[3]; + + float acc[FA_DV_MAX]; + for (int64_t d = 0; d < dv; ++d) { + acc[d] = 0.0f; + } + + float M = -3.402823466e+38f; + float S = 0.0f; + + for (int64_t ik1 = 0; ik1 < nk; ++ik1) { + // If mask is present, check for -inf (skip masked positions) + float mask_val = 0.0f; + if (has_mask) { + mask_val = get_mask_val(mask, iq1, ik1, iq2, iq3); + // llama.cpp uses -inf for masked positions + if (mask_val == -3.402823466e+38f || mask_val != mask_val) { + continue; + } + } + + const char * pk = k_data + ik1 * k->nb[1] + kv_base; + const char * pv = v_data + ik1 * v->nb[1] + vv_base; + + float s = dot_qk(pq, pk, dk, k_nb0, k_type) * scale + mask_val; + const float Mold = M; + + float ms = 1.0f; + float vs = 1.0f; + if (s > M) { + M = s; + ms = et_expf(Mold - M); + for (int64_t d = 0; d < dv; ++d) { + acc[d] *= ms; + } + } else { + vs = et_expf(s - M); + } + + // Accumulate weighted V + if (v_type == GGML_TYPE_F32) { + const float * pvf = (const float *) pv; + for (int64_t d = 0; d < dv; ++d) { + acc[d] += pvf[d] * vs; + } + } else { + for (int64_t d = 0; d < dv; ++d) { + acc[d] += fp16_to_fp32(*(const uint16_t *) (pv + d * v_nb0)) * vs; + } + } + + S = S * ms + vs; + } + + const float S_inv = S == 0.0f ? 0.0f : et_fdiv(1.0f, S); + if (use_fast_store) { + for (int64_t d = 0; d < dv; ++d) { + out[d] = acc[d] * S_inv; + } + } else { + for (int64_t d = 0; d < dv; ++d) { + atomic_store_f32((volatile float *) &out[d], acc[d] * S_inv); + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c b/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c new file mode 100644 index 0000000..c09c774 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c @@ -0,0 +1,346 @@ +//****************************************************************************** +// Gated Delta Net F32 Kernel +// +// Implements the gated delta rule recurrence: +// For each head h, timestep t: +// 1. Gate decay: S *= exp(g) (scalar or per-element KDA) +// 2. Delta update: delta[j] = (v[j] - dot(S_row_j, k)) * beta +// 3. Outer product: S_row_j += k * delta[j] +// 4. Attention: attn[j] = dot(S_row_j, q) * scale +// +// State is stored transposed: s_out[j*S_v + i] = S[i][j] +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include + +struct ggml_et_gated_delta_net_params { + struct ggml_tensor q; // [S_v, H_q, n_tokens, n_seqs_q] + struct ggml_tensor k; // [S_v, H_k, n_tokens, n_seqs_k] + struct ggml_tensor v; // [S_v, H, n_tokens, n_seqs] + struct ggml_tensor g; // [1 or S_v, H, n_tokens, n_seqs] + struct ggml_tensor beta; // [1, H, n_tokens, n_seqs] + struct ggml_tensor state_in; // [S_v*S_v*H, K, n_seqs] + struct ggml_tensor dst; // [S_v*H, n_tokens*n_seqs + S_v*n_seqs*K] + int32_t S_v; // head dimension + int32_t H; // number of value heads + int32_t H_q; // number of Q heads + int32_t H_k; // number of K heads + int32_t n_tokens; // total tokens + int32_t n_seqs; // number of sequences + int32_t n_seqs_q; // Q sequence count + int32_t n_seqs_k; // K sequence count + int32_t kda; // 1 if per-element gate, 0 if scalar + int32_t K; // snapshot slot count + float scale; // 1/sqrt(S_v) +}; + +static inline float hsum_f10(void) { + float result; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(result)::"t0", "f1", "f2", "f3", "f4", "f5"); + return result; +} + +int entry_point(struct ggml_et_gated_delta_net_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + const struct ggml_tensor * q_tsr = ¶ms->q; + const struct ggml_tensor * k_tsr = ¶ms->k; + const struct ggml_tensor * v_tsr = ¶ms->v; + const struct ggml_tensor * g_tsr = ¶ms->g; + const struct ggml_tensor * beta_tsr = ¶ms->beta; + const struct ggml_tensor * state_tsr = ¶ms->state_in; + const struct ggml_tensor * dst_tsr = ¶ms->dst; + + const float * q = (const float *) q_tsr->data; + const float * k = (const float *) k_tsr->data; + const float * v = (const float *) v_tsr->data; + const float * g = (const float *) g_tsr->data; + const float * beta = (const float *) beta_tsr->data; + const float * state_in = (const float *) state_tsr->data; + float * dst_data = (float *) dst_tsr->data; + + const int32_t S_v = params->S_v; + const int32_t H = params->H; + const int32_t H_q = params->H_q; + const int32_t H_k = params->H_k; + const int32_t n_tokens = params->n_tokens; + const int32_t n_seqs = params->n_seqs; + const int32_t n_seqs_q = params->n_seqs_q; + const int32_t n_seqs_k = params->n_seqs_k; + const int32_t kda = params->kda; + const int32_t K = params->K; + const float scale = params->scale; + + if (!q || !k || !v || !g || !beta || !state_in || !dst_data) { + return -1; + } + + // Preserve the original contract for every tensor except q, k, and v, which may be + // row-contiguous with strided higher dimensions. + if (q_tsr->nb[0] != sizeof(float) || k_tsr->nb[0] != sizeof(float) || v_tsr->nb[0] != sizeof(float) || + g_tsr->nb[0] != sizeof(float) || beta_tsr->nb[0] != sizeof(float) || state_tsr->nb[0] != sizeof(float) || + dst_tsr->nb[0] != sizeof(float)) { + return -1; + } + + const int32_t attn_elems = S_v * H * n_tokens * n_seqs; + float * attn_out_base = dst_data; + float * state_out_base = dst_data + attn_elems; + + const int32_t state_plane_floats = S_v * S_v * H * n_seqs; + + const int32_t G0 = kda ? S_v : 1; + + const size_t q_nb1 = q_tsr->nb[1]; + const size_t q_nb2 = q_tsr->nb[2]; + const size_t q_nb3 = q_tsr->nb[3]; + const size_t k_nb1 = k_tsr->nb[1]; + const size_t k_nb2 = k_tsr->nb[2]; + const size_t k_nb3 = k_tsr->nb[3]; + const size_t v_nb1 = v_tsr->nb[1]; + const size_t v_nb2 = v_tsr->nb[2]; + const size_t v_nb3 = v_tsr->nb[3]; + const int32_t g_stride_h = G0; + const int32_t g_stride_t = G0 * H; + const int32_t g_stride_s = G0 * H * n_tokens; + const int32_t b_stride_t = H; + const int32_t b_stride_s = H * n_tokens; + + float exp_g_buf[128]; + + // FP and SIMD share the same register file. Scalar FP needs the default + // mask; 8-wide .ps blocks need m0=255. Save once, toggle at boundaries. + unsigned long default_mask; + __asm__ volatile("mova.x.m %[ms]\n" : [ms] "=r"(default_mask)); + + // Parallelize over (j_block, head, seq). J_BLK must satisfy two separate + // cache-line alignment constraints at once: + // (a) State: J_BLK consecutive rows of s_out (each S_v floats) span an + // integer number of cache lines. For S_v * sizeof(float) >= 64 this + // is trivially any J_BLK >= 1. + // (b) Attention output: each j writes exactly one float into + // attn_ptr[j], which is densely packed. If J_BLK * sizeof(float) is + // less than a cache line, distinct threads will share a line and + // race on scalar stores — ET's L1 isn't coherent so we lose writes. + // + // (b) dominates: J_BLK must be at least ET_CACHE_LINE_SIZE_BYTES / 4 so + // that each thread owns a whole cache line of attn_ptr. That's 16 on + // ET-SoC-1, and it's also a whole number of state rows for every + // S_v >= 1, so (a) is automatically satisfied. + const int32_t J_BLK = ET_CACHE_LINE_SIZE_BYTES / (int32_t) sizeof(float); + const int32_t n_j_blocks = (S_v + J_BLK - 1) / J_BLK; + const int32_t total_work = n_j_blocks * H * n_seqs; + + for (int32_t ir = thread_id; ir < total_work; ir += num_threads) { + const int32_t jb = ir % n_j_blocks; + const int32_t head = (ir / n_j_blocks) % H; + const int32_t seq = ir / (n_j_blocks * H); + + const int32_t j_start = jb * J_BLK; + const int32_t j_end = (j_start + J_BLK < S_v) ? j_start + J_BLK : S_v; + + const int32_t h_q = head % H_q; + const int32_t h_k = head % H_k; + const int32_t seq_q = (n_seqs_q == n_seqs) ? seq : (seq * n_seqs_q / n_seqs); + const int32_t seq_k = (n_seqs_k == n_seqs) ? seq : (seq * n_seqs_k / n_seqs); + + const int32_t head_state_off = (seq * H + head) * S_v * S_v; + // Live RMW buffer = first snapshot plane (slot 0). + float * s_out = state_out_base + head_state_off; + // Input state: seq `seq`, head `head`. + const float * s_in = state_in + head_state_off; + + // Skip the explicit s_in -> s_out copy. At t=0 pass A/B read through + // src_state = s_in; pass B writes the first new row to s_out. From + // t=1 onward src_state flips to s_out (read-modify-write in place). + const float * src_state = s_in; + + const int32_t attn_stride_t = S_v * H; + float * attn_ptr = attn_out_base + (seq * n_tokens * H + head) * S_v; + + const float zero = 0.0f; + + for (int32_t t = 0; t < n_tokens; t++) { + const float * q_t = (const float *) ((const char *) q + seq_q * q_nb3 + t * q_nb2 + h_q * q_nb1); + const float * k_t = (const float *) ((const char *) k + seq_k * k_nb3 + t * k_nb2 + h_k * k_nb1); + const float * v_t = (const float *) ((const char *) v + seq * v_nb3 + t * v_nb2 + head * v_nb1); + const float * g_t = g + seq * g_stride_s + t * g_stride_t + head * g_stride_h; + const float beta_val = beta[seq * b_stride_s + t * b_stride_t + head]; + + // Precompute per-element gate for the kda path; scalar decay + // otherwise. Decay is fused into per-j pass A/B below, not + // applied to state in a separate pre-pass. + float decay = 0.0f; // only used when !kda + if (kda) { + const float log2e = 1.4426950408889634f; + __asm__ volatile("mov.m.x m0, x0, 255\n" :::); + __asm__ volatile("fbc.ps f20, %[l2e]\n" : : [l2e] "m"(log2e) : "f20"); + for (int32_t i = 0; i < S_v; i += 8) { + __asm__ volatile( + "flw.ps f10, %[g_vec]\n" + "fmul.ps f10, f10, f20, rne\n" + "fexp.ps f10, f10\n" + "fsw.ps f10, %[out]\n" + : [out] "=m"(*(float (*)[8]) & exp_g_buf[i]) + : [g_vec] "m"(*(const float (*)[8]) & g_t[i]) + : "f10"); + } + __asm__ volatile("mova.m.x %[ms]\n" : : [ms] "r"(default_mask)); + } else { + decay = et_expf(g_t[0]); + } + + for (int32_t j = j_start; j < j_end; j++) { + const float * src_row = src_state + j * S_v; + float * dst_row = s_out + j * S_v; + + __asm__ volatile("mov.m.x m0, x0, 255\n" :::); + if (kda) { + __asm__ volatile("fbc.ps f10, %[z]\n" : : [z] "m"(zero) : "f10"); + for (int32_t i = 0; i < S_v; i += 8) { + __asm__ volatile( + "flw.ps f11, %[s_vec]\n" + "flw.ps f12, %[g_vec]\n" + "flw.ps f13, %[k_vec]\n" + "fmul.ps f11, f11, f12\n" // row_dec = row * g + "fmadd.ps f10, f11, f13, f10\n" // acc += row_dec * k + : + : [s_vec] "m"(*(const float (*)[8]) & src_row[i]), + [g_vec] "m"(*(const float (*)[8]) & exp_g_buf[i]), + [k_vec] "m"(*(const float (*)[8]) & k_t[i]) + : "f10", "f11", "f12", "f13"); + } + } else { + __asm__ volatile( + "fbc.ps f10, %[z]\n" + "fbc.ps f22, %[d]\n" + : + : [z] "m"(zero), [d] "m"(decay) + : "f10", "f22"); + for (int32_t i = 0; i < S_v; i += 8) { + __asm__ volatile( + "flw.ps f11, %[s_vec]\n" + "flw.ps f13, %[k_vec]\n" + "fmul.ps f11, f11, f22\n" // row_dec = row * decay + "fmadd.ps f10, f11, f13, f10\n" // acc += row_dec * k + : + : [s_vec] "m"(*(const float (*)[8]) & src_row[i]), [k_vec] "m"(*(const float (*)[8]) & + k_t[i]) + : "f10", "f11", "f13"); + } + } + + float dot_sk = hsum_f10(); + __asm__ volatile("mova.m.x %[ms]\n" : : [ms] "r"(default_mask)); + + float delta_j = (v_t[j] - dot_sk) * beta_val; + + // -------- Pass B: decay + outer product + attn -------- + __asm__ volatile("mov.m.x m0, x0, 255\n" :::); + if (kda) { + __asm__ volatile( + "fbc.ps f10, %[z]\n" + "fbc.ps f21, %[dj]\n" + : + : [z] "m"(zero), [dj] "m"(delta_j) + : "f10", "f21"); + for (int32_t i = 0; i < S_v; i += 8) { + __asm__ volatile( + "flw.ps f11, %[s_vec]\n" + "flw.ps f12, %[g_vec]\n" + "flw.ps f13, %[k_vec]\n" + "flw.ps f14, %[q_vec]\n" + "fmul.ps f11, f11, f12\n" // row_dec = row * g + "fmadd.ps f11, f13, f21, f11\n" // row_new = row_dec + k*delta_j + "fsw.ps f11, %[s_out]\n" + "fmadd.ps f10, f11, f14, f10\n" // attn_acc += row_new * q + : [s_out] "=m"(*(float (*)[8]) & dst_row[i]) + : [s_vec] "m"(*(const float (*)[8]) & src_row[i]), + [g_vec] "m"(*(const float (*)[8]) & exp_g_buf[i]), + [k_vec] "m"(*(const float (*)[8]) & k_t[i]), [q_vec] "m"(*(const float (*)[8]) & q_t[i]) + : "f10", "f11", "f12", "f13", "f14"); + } + } else { + __asm__ volatile( + "fbc.ps f10, %[z]\n" + "fbc.ps f21, %[dj]\n" + "fbc.ps f22, %[d]\n" + : + : [z] "m"(zero), [dj] "m"(delta_j), [d] "m"(decay) + : "f10", "f21", "f22"); + for (int32_t i = 0; i < S_v; i += 8) { + __asm__ volatile( + "flw.ps f11, %[s_vec]\n" + "flw.ps f13, %[k_vec]\n" + "flw.ps f14, %[q_vec]\n" + "fmul.ps f11, f11, f22\n" // row_dec = row * decay + "fmadd.ps f11, f13, f21, f11\n" // row_new = row_dec + k*delta_j + "fsw.ps f11, %[s_out]\n" + "fmadd.ps f10, f11, f14, f10\n" // attn_acc += row_new * q + : [s_out] "=m"(*(float (*)[8]) & dst_row[i]) + : [s_vec] "m"(*(const float (*)[8]) & src_row[i]), + [k_vec] "m"(*(const float (*)[8]) & k_t[i]), [q_vec] "m"(*(const float (*)[8]) & q_t[i]) + : "f10", "f11", "f13", "f14"); + } + } + + float attn_val = hsum_f10(); + __asm__ volatile("mova.m.x %[ms]\n" : : [ms] "r"(default_mask)); + + attn_ptr[j] = attn_val * scale; + } + + // n-way merge snapshot: live state lives in slot 0 (== s_out). + // Copies state to target snapshot slots [1, K-1] in reverse chronological order. + // target_slot == 0 is the live buffer itself => no copy. + // target_slot >= K (when n_tokens > K) => older slots are discarded. + if (K > 1) { + const int32_t target_slot = (n_tokens - 1) - t; + if (target_slot > 0 && target_slot < K) { + float * snap = state_out_base + target_slot * state_plane_floats + head_state_off; + for (int32_t j = j_start; j < j_end; j++) { + const float * src = s_out + j * S_v; + float * dst = snap + j * S_v; + for (int32_t i = 0; i < S_v; i++) { + dst[i] = src[i]; + } + } + } + } + + // After t=0, state lives in s_out; flip src_state so subsequent + // timesteps read-modify-write in place. + src_state = s_out; + attn_ptr += attn_stride_t; + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c b/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c new file mode 100644 index 0000000..701f1db --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c @@ -0,0 +1,612 @@ +//****************************************************************************** +// Bare Metal GET_ROWS F32 Kernel +// Extracts specific rows from a source tensor based on row indices +// +// Algorithm: +// 1. Read row indices from src1 (int32 tensor) +// 2. For each index, extract the corresponding row from src0 +// 3. Copy the row data to the output tensor dst +// 4. Handle different input types: F32, Q8_0, Q4_0, and Q4_K (quantized) +// +// Operation: dst[i] = src0[indices[i]] for i = 0..num_indices +// +// Features supported: +// - F32 input data (direct copy) +// - Q4_0 quantized input data (dequantized to F32) +// - Q8_0 quantized input data (dequantized to F32) +// - Q4_K quantized input data (dequantized to F32) +// - Int32 row indices +// - Multi-dimensional tensor support +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" +#include "quants.h" + +#include +#include +#include + +#define CACHE_LINE_SIZE_BYTES 64 + +struct ggml_et_get_rows_params { + struct ggml_tensor src0; // Data tensor (F32, Q4_0, Q8_0, or Q4_K) + struct ggml_tensor src1; // Row indices tensor (I32) + struct ggml_tensor dst; // Output tensor (F32) +}; + +#define CACHE_LINE_SIZE_BYTES 64 +#define CACHE_ELEMENTS(elem_size) (CACHE_LINE_SIZE_BYTES / (elem_size)) + +// Copy a row of F32 data from source to destination +static void copy_f32_row(float * dst, const float * src, int64_t num_elements) { + // Simple memcpy for F32 data - no conversion needed + for (int64_t i = 0; i < num_elements; i++) { + dst[i] = src[i]; + } +} + +static void copy_f16_row(float * dst, const uint16_t * src, int64_t num_elements) { + for (int64_t i = 0; i < num_elements; i++) { + dst[i] = fp16_to_fp32(src[i]); + } +} + +// Copy a row of F32 data from source to destination, aligned to cache line boundaries +// using FP32 load/store instructions. They don't perform data conversion so is fine. +// Requirement: n_bytes is a multiple of CACHE_LINE_SIZE (64 bytes) +static void copy_row_cache_align(float * dst, const float * src, int64_t n_bytes) { + int num_f32_elem = n_bytes / sizeof(float); + + // Unrolled to do an entire cache line at a time + __asm__ volatile( + "1: \n\t" + // --- Process 64 Bytes (1 Cache Line) --- + // Load 256 bits (32 bytes) into f0 and the other into f1 + "flq2 f0, 0(%[src]) \n\t" + "flq2 f1, 32(%[src]) \n\t" + + // Store 256 bits (32 bytes) from f0 and f1 + "fsq2 f0, 0(%[dst]) \n\t" + "fsq2 f1, 32(%[dst]) \n\t" + + // Increment Pointers by 64 bytes + "addi %[src], %[src], 64 \n\t" + "addi %[dst], %[dst], 64 \n\t" + + // Decrement count by 16 elements + "addi %[n], %[n], -16 \n\t" + + // Loop if at least 16 elements remain + "bge %[n], %[stride_count], 1b \n\t" + + : [dst] "+r"(dst), [src] "+r"(src), [n] "+r"(num_f32_elem) + : [stride_count] "r"(16L) + : "f0", "f1", "memory"); +} + +// Copied from GGML: copy a row of Q4_0 data to F32 destination (with dequantization) +static void copy_q4_0_row(float * dst, const block_q4_0 * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK4_0 - 1) / QK4_0; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK4_0) : QK4_0; + + float temp_buffer[QK4_0]; + dequantize_q4_0_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK4_0 + i] = temp_buffer[i]; + } + } +} + +// Copy a row of Q8_0 data to F32 destination (with dequantization) +static void copy_q8_0_row(float * dst, const block_q8_0 * src_blocks, int64_t num_elements) { + // Number of Q8_0 blocks needed for this row + const int64_t num_blocks = (num_elements + QK8_0 - 1) / QK8_0; // Round up to handle partial blocks + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = + (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK8_0) : QK8_0; // Handle last partial block + + // Dequantize the block + float temp_buffer[QK8_0]; + dequantize_q8_0_block(&src_blocks[block_idx], temp_buffer); + + // Copy dequantized values to destination + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK8_0 + i] = temp_buffer[i]; + } + } +} + +// Copy a row of Q4_K data to F32 destination (with dequantization) +static void copy_q4_K_row(float * dst, const block_q4_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK_K) : QK_K; + + float temp_buffer[QK_K]; + dequantize_q4_K_block(&src_blocks[block_idx], temp_buffer); + + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK_K + i] = temp_buffer[i]; + } + } +} + +static void dequantize_q8_0_block_cache_aligned(const block_q8_0 * block, float * dst) { + const int8_t * qs_ptr = block->qs; + + uint64_t temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); // Save current mask + __asm__ volatile("mov.m.x m0, x0, 0xFF"); // Enable all 8 elements + + const int32_t __attribute__((aligned(32))) vec_indices[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + float scale = fp16_to_fp32(block->d); + __asm__ volatile( + "fbcx.ps f0, %0 \n\t" // Broadcast integer scale to all lanes + "flq2 f1, 0(%1) \n\t" // Load gether indicies + ::"r"(scale), + "r"(vec_indices) + : "f0", "f1"); + + for (int i = 0; i < 4; i++) { + __asm__ volatile( + "fgb.ps f2, f1(%0) \n\t" // Loads 8 bytes from (qs_ptr + indices) and sign-extends to 32-bit int. + "fcvt.ps.pw f2, f2, rne \n\t" // Convert Int32 to Float32 + "fmul.ps f2, f2, f0 \n\t" // f2 = f2 * f0 (scale) + "fsq2 f2, 0(%1) \n\t" // Store 256 bits (8 floats) to dst. + + ::"r"(qs_ptr), + "r"(dst) + : "f2", "memory"); + + // Advance pointers in C + qs_ptr += 8; + dst += 8; + } + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); +} + +// Copy a row of Q4_0 data to F32 destination (with dequantization), cache-aligned +static void copy_q4_0_row_cache_aligned(float * dst, const block_q4_0 * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK4_0 - 1) / QK4_0; + + // Scatter byte offsets: even lanes -> dst[j], odd lanes -> dst[j + QK4_0/2] + // For 4 consecutive packed bytes producing [low0, high0, low1, high1, low2, high2, low3, high3]: + // low_i -> byte offset i*4 (positions 0,1,2,3 in first half) + // high_i -> byte offset (16+i)*4 (positions 16,17,18,19 in second half) + const int32_t __attribute__((aligned(32))) scatter_offsets[8] = { 0 * 4, 16 * 4, 1 * 4, 17 * 4, + 2 * 4, 18 * 4, 3 * 4, 19 * 4 }; + + // Gather indices: each byte loaded twice for low/high nibble extraction + const int32_t __attribute__((aligned(32))) gather_indices[8] = { 0, 0, 1, 1, 2, 2, 3, 3 }; + + uint64_t temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); // Save current mask + __asm__ volatile("mov.m.x m0, x0, 0xFF"); // Enable all 8 elements + + // Load constant vectors once — shared across all blocks and iterations + __asm__ volatile( + "flq2 f4, 0(%0) \n\t" // f4 = scatter offsets + "flq2 f1, 0(%1) \n\t" // f1 = gather indices {0,0,1,1,2,2,3,3} + ::"r"(scatter_offsets), + "r"(gather_indices) + : "f1", "f4"); + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const block_q4_0 * block = &src_blocks[block_idx]; + const uint8_t * qs = block->qs; + float * block_dst = dst + block_idx * QK4_0; + + float scale = fp16_to_fp32(block->d); + float bias = -8.0f * scale; + + // Per-block: broadcast scale and bias + __asm__ volatile( + "fbcx.ps f0, %0 \n\t" // f0 = broadcast(scale) + "fbcx.ps f3, %1 \n\t" // f3 = broadcast(-8 * scale) + ::"r"(scale), + "r"(bias) + : "f0", "f3"); + + // 4 iterations x 4 packed bytes = 16 bytes = full block -> 32 floats + for (int i = 0; i < 4; i++) { + __asm__ volatile( + "fgb.ps f2, f1(%0) \n\t" // Gather: [b0,b0,b1,b1,b2,b2,b3,b3] + "mov.m.x m0, x0, 0xAA \n\t" // Odd lanes only (fills gather latency) + "fsrli.pi f2, f2, 4 \n\t" // Odd lanes: byte >> 4 (high nibble) + "mov.m.x m0, x0, 0xFF \n\t" // Restore full mask + "fslli.pi f2, f2, 28 \n\t" // Isolate low 4 bits: shift left 28 + "fsrli.pi f2, f2, 28 \n\t" // then right 28 -> nibble in [3:0] + "fcvt.ps.pw f2, f2, rne \n\t" // Int32 -> Float32 + "fmul.ps f2, f2, f0 \n\t" // * scale + "fadd.ps f2, f2, f3 \n\t" // + bias -> (nibble - 8) * scale + "fscw.ps f2, f4(%1) \n\t" // Scatter to GGML positions + + ::"r"(qs), + "r"(block_dst) + : "f2", "memory"); + + qs += 4; // 4 packed bytes consumed + block_dst += 4; // Advance base by 4 float positions + } + } + + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); // Restore mask +} + +// Copy a row of Q8_0 data to F32 destination (with dequantization) +static void copy_q8_0_row_cache_aligned(float * dst, const block_q8_0 * src_blocks, int64_t num_elements) { + // Number of Q8_0 blocks needed for this row + const int64_t num_blocks = (num_elements + QK8_0 - 1) / QK8_0; // Round up to handle partial blocks + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const int64_t elements_in_block = + (block_idx == num_blocks - 1) ? (num_elements - block_idx * QK8_0) : QK8_0; // Handle last partial block + + // Dequantize the block + float temp_buffer[QK8_0]; + dequantize_q8_0_block_cache_aligned(&src_blocks[block_idx], temp_buffer); + + // Copy dequantized values to destination + for (int64_t i = 0; i < elements_in_block; i++) { + dst[block_idx * QK8_0 + i] = temp_buffer[i]; + } + } +} + +// Vectorized dequantization of a Q4_K super-block (256 elements) to F32 +// Processes 8 groups of 32 elements, using ET SIMD for the inner loops. +// Output is sequential (no scatter needed unlike Q4_0). +static void copy_q4_K_row_cache_aligned(float * dst, const block_q4_K * src_blocks, int64_t num_elements) { + const int64_t num_blocks = (num_elements + QK_K - 1) / QK_K; + + // Gather indices for sequential byte access: {0,1,2,3,4,5,6,7} + const int32_t __attribute__((aligned(32))) gather_indices[8] = { 0, 1, 2, 3, 4, 5, 6, 7 }; + + uint64_t temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); // Save current mask + __asm__ volatile("mov.m.x m0, x0, 0xFF"); // Enable all 8 elements + + // Load gather indices once — shared across all blocks + __asm__ volatile("flq2 f1, 0(%0) \n\t" // f1 = gather indices {0,1,2,3,4,5,6,7} + ::"r"(gather_indices) + : "f1"); + + for (int64_t block_idx = 0; block_idx < num_blocks; block_idx++) { + const block_q4_K * block = &src_blocks[block_idx]; + const uint8_t * qs = block->qs; + float * block_dst = dst + block_idx * QK_K; + + const float d = fp16_to_fp32(block->d); + const float min = fp16_to_fp32(block->dmin); + + int is = 0; + for (int j = 0; j < QK_K; j += 64) { + // Extract per-group scales and mins (scalar — only 8 pairs per super-block) + uint8_t sc, m; + get_scale_min_k4(is + 0, block->scales, &sc, &m); + const float d1 = d * sc; + const float neg_m1 = -(min * m); + get_scale_min_k4(is + 1, block->scales, &sc, &m); + const float d2 = d * sc; + const float neg_m2 = -(min * m); + + // Low nibbles: 32 elements using d1, neg_m1 + __asm__ volatile( + "fbcx.ps f0, %0 \n\t" // f0 = broadcast(d1) + "fbcx.ps f3, %1 \n\t" // f3 = broadcast(-m1) + ::"r"(d1), + "r"(neg_m1) + : "f0", "f3"); + + const uint8_t * qs_lo = qs; + float * dst_lo = block_dst + j; + for (int k = 0; k < 4; k++) { + __asm__ volatile( + "fgb.ps f2, f1(%0) \n\t" // Gather 8 bytes, sign-extend to int32 + "fandi.pi f2, f2, 0xF \n\t" // Mask low nibble (imm10=15) + "fcvt.ps.pw f2, f2, rne \n\t" // Int32 -> Float32 + "fmadd.ps f2, f2, f0, f3\n\t" // d1 * nibble + (-m1) + "fsq2 f2, 0(%1) \n\t" // Store 8 floats + ::"r"(qs_lo), + "r"(dst_lo) + : "f2", "memory"); + qs_lo += 8; + dst_lo += 8; + } + + // High nibbles: 32 elements using d2, neg_m2 + __asm__ volatile( + "fbcx.ps f0, %0 \n\t" // f0 = broadcast(d2) + "fbcx.ps f3, %1 \n\t" // f3 = broadcast(-m2) + ::"r"(d2), + "r"(neg_m2) + : "f0", "f3"); + + const uint8_t * qs_hi = qs; + float * dst_hi = block_dst + j + 32; + for (int k = 0; k < 4; k++) { + __asm__ volatile( + "fgb.ps f2, f1(%0) \n\t" // Gather 8 bytes, sign-extend to int32 + "fsrli.pi f2, f2, 4 \n\t" // Shift right 4: high nibble + "fandi.pi f2, f2, 0xF \n\t" // Mask to 4 bits (clean any sign-ext artifacts) + "fcvt.ps.pw f2, f2, rne \n\t" // Int32 -> Float32 + "fmadd.ps f2, f2, f0, f3\n\t" // d2 * nibble + (-m2) + "fsq2 f2, 0(%1) \n\t" // Store 8 floats + ::"r"(qs_hi), + "r"(dst_hi) + : "f2", "memory"); + qs_hi += 8; + dst_hi += 8; + } + + qs += 32; // Advance to next 32 packed bytes + is += 2; + } + } + + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); // Restore mask +} + +// Determine the number of F32 elements per work unit for a given source type. +// For F32: 1 cacheline (16 elements) +// For quantized types: 1 quant block +static int64_t get_elements_per_work_unit(int type) { + const int64_t elements_per_cacheline = CACHE_LINE_SIZE_BYTES / sizeof(float); // 16 + switch (type) { + case GGML_TYPE_Q8_0: + return QK8_0; // 32 elements = 2 cachelines + case GGML_TYPE_Q4_0: + return QK4_0; // 32 elements = 2 cachelines + case GGML_TYPE_Q4_K: + return QK_K; // 256 elements = 16 cachelines + default: + return elements_per_cacheline; // 16 elements = 1 cacheline + } +} + +static int get_row_f32_mc_cacheline_aligned(struct ggml_et_get_rows_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + struct ggml_tensor * src0 = ¶ms->src0; // Data tensor + struct ggml_tensor * src1 = ¶ms->src1; // Row indices tensor (I32) + struct ggml_tensor * dst = ¶ms->dst; // Output tensor (F32) + + const int64_t ne00 = src0->ne[0]; // Source columns (row width) + const int64_t ne01 = src0->ne[1]; // Source rows (total available rows) + const int64_t ne02 = src0->ne[2]; // Source batch dimension + const int64_t ne03 = src0->ne[3]; // Source outer batch dimension + + const int64_t ne10 = src1->ne[0]; // Number of indices in dimension 0 + const int64_t ne11 = src1->ne[1]; // Number of indices in dimension 1 + const int64_t ne12 = src1->ne[2]; // Batch dimension for indices + const int64_t ne13 = src1->ne[3]; // Outer batch dimension for indices + + const int64_t total_rows_to_extract = ne10 * ne11 * ne12 * ne13; + + // Determine work unit size based on source type + const int64_t elements_per_wu = get_elements_per_work_unit(src0->type); + const int64_t wus_per_row = ne00 / elements_per_wu; + const int64_t total_wus = total_rows_to_extract * wus_per_row; + + // Distribute work units across threads (contiguous ranges) + const int64_t wus_per_thread = (total_wus + num_threads - 1) / num_threads; + const int64_t wu_start = thread_id * wus_per_thread; + int64_t wu_end = wu_start + wus_per_thread; + if (wu_end > total_wus) { + wu_end = total_wus; + } + + void * src0_data = src0->data; + int32_t * src1_data = (int32_t *) src1->data; + float * dst_data = (float *) dst->data; + + int64_t wu = wu_start; + while (wu < wu_end) { + // Determine which row this work unit belongs to and offset within row + const int64_t row_idx = wu / wus_per_row; + const int64_t wu_in_row = wu % wus_per_row; + + // How many work units to process in this row (batch contiguous WUs in same row) + int64_t wus_remaining_in_row = wus_per_row - wu_in_row; + int64_t wus_to_process = wu_end - wu; + if (wus_remaining_in_row < wus_to_process) { + wus_to_process = wus_remaining_in_row; + } + + // Calculate multi-dimensional index for this row + const int64_t i = row_idx; + const int64_t i13_idx = i / (ne12 * ne11 * ne10); + const int64_t i12_idx = (i - i13_idx * ne12 * ne11 * ne10) / (ne11 * ne10); + const int64_t i11_idx = (i - i13_idx * ne12 * ne11 * ne10 - i12_idx * ne11 * ne10) / ne10; + const int64_t i10_idx = i - i13_idx * ne12 * ne11 * ne10 - i12_idx * ne11 * ne10 - i11_idx * ne10; + + // Get the row index from src1 + const int64_t index_offset = i13_idx * ne12 * ne11 * ne10 + i12_idx * ne11 * ne10 + i11_idx * ne10 + i10_idx; + const int32_t row_index = src1_data[index_offset]; + + if (row_index < 0 || row_index >= ne01) { + return -1; // Index out of bounds + } + + const int64_t batch_offset = + i11_idx * ne01 * ne00 + i12_idx * ne02 * ne01 * ne00 + i13_idx * ne03 * ne02 * ne01 * ne00; + + const int64_t elem_offset_in_row = wu_in_row * elements_per_wu; + const int64_t num_elements = wus_to_process * elements_per_wu; + + float * dst_row = dst_data + row_idx * ne00 + elem_offset_in_row; + + if (src0->type == GGML_TYPE_F32) { + // F32 source: direct copy of cacheline-aligned chunk + const float * src_row = (const float *) src0_data + row_index * ne00 + batch_offset + elem_offset_in_row; + copy_row_cache_align(dst_row, src_row, num_elements * sizeof(float)); + } else if (src0->type == GGML_TYPE_F16) { + // F16 source: scalar conversion over a destination-aligned write chunk. + const uint16_t * src_row = + (const uint16_t *) src0_data + row_index * ne00 + batch_offset + elem_offset_in_row; + copy_f16_row(dst_row, src_row, num_elements); + } else if (src0->type == GGML_TYPE_Q8_0) { + // Q8_0 source: dequantize work-unit-aligned blocks + const int64_t blocks_per_row = (ne00 + QK8_0 - 1) / QK8_0; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const int64_t block_start = elem_offset_in_row / QK8_0; + const block_q8_0 * src_blocks = (const block_q8_0 *) src0_data + src_block_offset + block_start; + copy_q8_0_row_cache_aligned(dst_row, src_blocks, num_elements); + } else if (src0->type == GGML_TYPE_Q4_0) { + // Q4_0 source: dequantize work-unit-aligned blocks + const int64_t blocks_per_row = (ne00 + QK4_0 - 1) / QK4_0; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const int64_t block_start = elem_offset_in_row / QK4_0; + const block_q4_0 * src_blocks = (const block_q4_0 *) src0_data + src_block_offset + block_start; + copy_q4_0_row_cache_aligned(dst_row, src_blocks, num_elements); + } else if (src0->type == GGML_TYPE_Q4_K) { + // Q4_K source: dequantize work-unit-aligned blocks + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const int64_t block_start = elem_offset_in_row / QK_K; + const block_q4_K * src_blocks = (const block_q4_K *) src0_data + src_block_offset + block_start; + copy_q4_K_row_cache_aligned(dst_row, src_blocks, num_elements); + } + + wu += wus_to_process; + } + + return 0; +} + +static inline size_t tensor_bytes(const struct ggml_tensor * t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} + +int entry_point(struct ggml_et_get_rows_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + if (!kernel_env) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; // Data tensor (F32, Q4_0, Q8_0, or Q4_K) + struct ggml_tensor * src1 = ¶ms->src1; // Row indices tensor (I32) + struct ggml_tensor * dst = ¶ms->dst; // Output tensor (F32) + + // Fast path - we know how to deal with them multi-core + if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_Q8_0 || + src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K) && + src1->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32 && dst->ne[0] % CACHE_ELEMENTS(sizeof(float)) == 0) { + return get_row_f32_mc_cacheline_aligned(params, env); + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + if (thread_id < 0) { + return 0; + } + + if (thread_id != 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + if (dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) { + return -1; // Invalid output or index type + } + + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_Q8_0 && + src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K) { + return -1; // Unsupported input type + } + + void * src0_data = src0->data; + int32_t * src1_data = (int32_t *) src1->data; + float * dst_data = (float *) dst->data; +#ifdef ET_UBERKERNEL + evict_region_past_l2(src0_data, tensor_bytes(src0)); + evict_region_past_l2(src1_data, tensor_bytes(src1)); + evict_region_past_l2(dst_data, tensor_bytes(dst)); +#endif + + if (!src0_data || !src1_data || !dst_data) { + return -1; // Null data pointer + } + + const int64_t ne00 = src0->ne[0]; // Source columns (row width) + const int64_t ne01 = src0->ne[1]; // Source rows (total available rows) + const int64_t ne02 = src0->ne[2]; // Source batch dimension + const int64_t ne03 = src0->ne[3]; // Source outer batch dimension + + const int64_t ne10 = src1->ne[0]; // Number of indices in dimension 0 + const int64_t ne11 = src1->ne[1]; // Number of indices in dimension 1 + const int64_t ne12 = src1->ne[2]; // Batch dimension for indices + const int64_t ne13 = src1->ne[3]; // Outer batch dimension for indices + + const int64_t total_rows_to_extract = ne10 * ne11 * ne12 * ne13; +#ifdef ET_UBERKERNEL + et_barrier(ET_BARRIER_GLOBAL); +#endif + // Naive single-threaded implementation - process all rows sequentially + // XXX: Do we really need a single-threaded implementation? + for (int64_t i = 0; i < total_rows_to_extract; i++) { + // Calculate multi-dimensional index for the current output position + const int64_t i13_idx = i / (ne12 * ne11 * ne10); + const int64_t i12_idx = (i - i13_idx * ne12 * ne11 * ne10) / (ne11 * ne10); + const int64_t i11_idx = (i - i13_idx * ne12 * ne11 * ne10 - i12_idx * ne11 * ne10) / ne10; + const int64_t i10_idx = i - i13_idx * ne12 * ne11 * ne10 - i12_idx * ne11 * ne10 - i11_idx * ne10; + + // Get the row index from src1 + const int64_t index_offset = i13_idx * ne12 * ne11 * ne10 + i12_idx * ne11 * ne10 + i11_idx * ne10 + i10_idx; + const int32_t row_index = src1_data[index_offset]; + + if (row_index < 0 || row_index >= ne01) { + return -1; // Index out of bounds + } + + const int64_t batch_offset = + i11_idx * ne01 * ne00 + i12_idx * ne02 * ne01 * ne00 + i13_idx * ne03 * ne02 * ne01 * ne00; + + const int64_t dst_offset = i; + + if (src0->type == GGML_TYPE_F32) { + // F32 source: direct copy + const float * src_row = (const float *) src0_data + row_index * ne00 + batch_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_f32_row(dst_row, src_row, ne00); + } else if (src0->type == GGML_TYPE_F16) { + // F16 source: scalar conversion + const uint16_t * src_row = (const uint16_t *) src0_data + row_index * ne00 + batch_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_f16_row(dst_row, src_row, ne00); + } else if (src0->type == GGML_TYPE_Q8_0) { + // Q8_0 source: dequantize while copying + const int64_t blocks_per_row = (ne00 + QK8_0 - 1) / QK8_0; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q8_0 * src_blocks = (const block_q8_0 *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q8_0_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q4_0) { + // Q4_0 source: dequantize while copying + const int64_t blocks_per_row = (ne00 + QK4_0 - 1) / QK4_0; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q4_0 * src_blocks = (const block_q4_0 *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q4_0_row(dst_row, src_blocks, ne00); + } else if (src0->type == GGML_TYPE_Q4_K) { + // Q4_K source: dequantize while copying + const int64_t blocks_per_row = (ne00 + QK_K - 1) / QK_K; + const int64_t src_block_offset = (row_index * blocks_per_row) + (batch_offset / ne00) * blocks_per_row; + const block_q4_K * src_blocks = (const block_q4_K *) src0_data + src_block_offset; + float * dst_row = dst_data + dst_offset * ne00; + copy_q4_K_row(dst_row, src_blocks, ne00); + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h b/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h new file mode 100644 index 0000000..8585d56 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h @@ -0,0 +1,44 @@ +// ET kernel entry-point parameter structs and tensor helpers. + +#ifndef GGML_TENSOR_H +#define GGML_TENSOR_H + +#include +#include + +#include "ggml.h" + +struct ggml_et_binary_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor dst; +}; + +// bias.data == NULL -> unfused MUL_MAT; otherwise dst = mat_mul(...) + bias. +struct ggml_et_mm_q8_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor dst; + struct ggml_tensor bias; +}; + +struct ggml_et_mul_mat_id_params { + struct ggml_tensor src0; // [K, M, n_expert] + struct ggml_tensor src1; // [K, n_expert_used, batch] + struct ggml_tensor src2; // [n_expert_used, batch] (I32 expert indices) + struct ggml_tensor dst; // [M, n_expert_used, batch, 1] +}; + +// ne[i] == 1 axes are skipped: their stride is unobservable. +static inline int ggml_tensor_is_contiguous(const struct ggml_tensor * t, int type_size) { + int64_t expected = type_size; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (t->ne[i] > 1 && (int64_t) t->nb[i] != expected) { + return 0; + } + expected *= t->ne[i]; + } + return 1; +} + +#endif // GGML_TENSOR_H diff --git a/ggml/src/ggml-et/et-kernels/src/glu_f32.c b/ggml/src/ggml-et/et-kernels/src/glu_f32.c new file mode 100644 index 0000000..d376d6f --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/glu_f32.c @@ -0,0 +1,606 @@ +//****************************************************************************** +// GLU F32 Kernel (SwiGLU specifically) +// Gated Linear Unit: y[i] = silu(x[i]) * g[i] where silu(x) = x * sigmoid(x) +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include + +// GLU kernel parameters structure (from ET backend ops) +struct ggml_et_glu_params { + struct ggml_tensor src0; // F32 input tensor A (or combined tensor if src1 is null) + struct ggml_tensor src1; // F32 input tensor B (null for single tensor mode) + struct ggml_tensor dst; // F32 output tensor (n/2 columns) + int32_t glu_op_type; // GLU operation type (REGLU=0, GEGLU=1, SWIGLU=2, etc.) + int32_t swapped; // Whether gate and value are swapped + float alpha; // SWIGLU_OAI: sigmoid scaling factor + float limit; // GLU clamp limit +}; + +// SiLU activation function: silu(x) = x * sigmoid(x) = x / (1 + exp(-x)) +static inline float silu_f32(float x) { + // For numerical stability, use the mathematically equivalent form: + // silu(x) = x / (1 + exp(-x)) = x * sigmoid(x) + // For large negative x, exp(-x) -> inf, so silu(x) -> 0 + // For large positive x, exp(-x) -> 0, so silu(x) -> x + + if (x > 20.0f) { + // For x > 20, exp(-x) is negligible, silu(x) ~ x + return x; + } else if (x < -20.0f) { + // For x < -20, silu(x) ~ 0 + return 0.0f; + } else { + // Use standard formula: silu(x) = x / (1 + exp(-x)) + // Optimized using ET hardware division + float exp_neg_x = et_expf(-x); + float denominator = 1.0f + exp_neg_x; + return et_fdiv(x, denominator); + } +} + +// Vectorized GeGLU block processing (8 elements = 1 cache line, 64B aligned) +// gelu(x) = 0.5*x*(1 + tanh(z)) = x * (1 - 1/(exp(2z)+1)) +// where z = sqrt(2/pi) * x * (1 + 0.044715*x^2) +// Reformulated to avoid inf*0 NaN: uses x * sigmoid(2z) identity +static inline void block_geglu(float * dst_block, const float * x_block, const float * g_block, int elements) { + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + float one_const = 1.0f; + float coef_a_const = 0.044715f; + float sqrt2pi_const = 0.79788456080286535587989211986876f; // sqrt(2/pi) + float two_log2e_const = 2.8853900817779268f; // 2 * log2(e) + + for (int32_t i = 0; i < elements; i += 8) { + __asm__ volatile( + // Load inputs + "flw.ps f10, %[x_vec]\n" // f10 = x + "flw.ps f11, %[g_vec]\n" // f11 = g + + // Broadcast constants + "fbc.ps f20, %[one_ptr]\n" // f20 = 1.0 + "fbc.ps f22, %[coef_ptr]\n" // f22 = 0.044715 + "fbc.ps f23, %[sqrt2pi_ptr]\n" // f23 = sqrt(2/pi) + "fbc.ps f24, %[two_log2e_ptr]\n" // f24 = 2*log2(e) + + // inner = 1 + 0.044715 * x^2 + "fmul.ps f12, f10, f10\n" // f12 = x^2 + "fmadd.ps f13, f22, f12, f20\n" // f13 = 1 + 0.044715*x^2 + + // z = sqrt(2/pi) * x * inner + "fmul.ps f14, f23, f10\n" // f14 = sqrt(2/pi) * x + "fmul.ps f14, f14, f13\n" // f14 = z + + // exp(2z) via fexp.ps: feed z * 2*log2(e) since fexp computes 2^input + "fmul.ps f15, f14, f24\n" // f15 = 2z * log2(e) + "fexp.ps f15, f15\n" // f15 = exp(2z) + + // gelu(x) = x * (1 - 1/(exp(2z)+1)) [NaN-safe: no inf*0] + // exp(2z)->inf: rcp(inf)=0, 1-0=1, gelu=x + // exp(2z)->0: rcp(1)=1, 1-1=0, gelu=0 + "fadd.ps f16, f15, f20\n" // f16 = exp(2z) + 1 + "frcp.ps f16, f16\n" // f16 = 1/(exp(2z) + 1) + "fsub.ps f16, f20, f16\n" // f16 = 1 - 1/(exp(2z)+1) + "fmul.ps f16, f10, f16\n" // f16 = gelu(x) + + // Final result + "fmul.ps f18, f16, f11\n" // f18 = gelu(x) * g + + "fsw.ps f18, %[dst_out]\n" + + : [dst_out] "=m"(*(float (*)[8]) & dst_block[i]) + : [x_vec] "m"(*(const float (*)[8]) & x_block[i]), [g_vec] "m"(*(const float (*)[8]) & g_block[i]), + [one_ptr] "m"(one_const), [coef_ptr] "m"(coef_a_const), [sqrt2pi_ptr] "m"(sqrt2pi_const), + [two_log2e_ptr] "m"(two_log2e_const) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f18", "f20", "f22", "f23", "f24"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); +} + +// Vectorized SwiGLU block processing (16 elements = 1 cache line) +static inline void block_swiglu(float * dst_block, const float * x_block, const float * g_block, int elements) { + // Process 8 elements at a time using vector instructions + int32_t vec_end = (elements / 8) * 8; + + // Set mask register to enable all 8 vector elements + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); // Save current mask + __asm__ volatile("mov.m.x m0, x0, 0xFF"); // Enable all 8 elements + + // Constants for broadcasting + float zero_const = 0.0f; + float one_const = 1.0f; + float log2e_const = 1.4426950408889634f; // log2(e) + + for (int32_t i = 0; i < vec_end; i += 8) { + // Vectorized SwiGLU: dst = silu(x) * g = (x / (1 + exp(-x))) * g + // Using ET hardware: exp, reciprocal, multiply operations + __asm__ volatile( + // Load input vectors + "flw.ps f10, %[x_vec]\n" // f10 = x[0..7] + "flw.ps f11, %[g_vec]\n" // f11 = g[0..7] + + // Broadcast constants to vector registers + "fbc.ps f20, %[zero_ptr]\n" // f20 = broadcast(0.0f) to all 8 elements + "fbc.ps f21, %[one_ptr]\n" // f21 = broadcast(1.0f) to all 8 elements + + // Compute -x (negate x by subtracting from zero) + "fsub.ps f12, f20, f10\n" // f12 = 0 - x = -x + + // Convert to base-2 exponent: -x * log2(e) = -x * 1.44269504 + // Load log2(e) constant + "fbc.ps f22, %[log2e_ptr]\n" // f22 = broadcast(1.44269504f) + "fmul.ps f13, f12, f22\n" // f13 = -x * log2(e) + + // Compute 2^(-x * log2(e)) = exp(-x) + "fexp.ps f14, f13\n" // f14 = 2^(-x * log2(e)) = exp(-x) + + // Compute 1 + exp(-x) + "fadd.ps f15, f14, f21\n" // f15 = exp(-x) + 1 + + // Compute 1 / (1 + exp(-x)) using reciprocal + "frcp.ps f16, f15\n" // f16 = 1 / (1 + exp(-x)) + + // Compute silu(x) = x * (1 / (1 + exp(-x))) + "fmul.ps f17, f10, f16\n" // f17 = x * (1 / (1 + exp(-x))) = silu(x) + + // Compute final result: silu(x) * g + "fmul.ps f18, f17, f11\n" // f18 = silu(x) * g + + // Store result + "fsw.ps f18, %[dst_out]\n" // Store 8 results to destination + + : [dst_out] "=m"(*(float (*)[8]) & dst_block[i]) + : [x_vec] "m"(*(const float (*)[8]) & x_block[i]), [g_vec] "m"(*(const float (*)[8]) & g_block[i]), + [zero_ptr] "m"(zero_const), // Memory reference to 0.0f for broadcasting + [one_ptr] "m"(one_const), // Memory reference to 1.0f for broadcasting + [log2e_ptr] "m"(log2e_const) // Memory reference to log2(e) for broadcasting + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f20", "f21", "f22"); + } + + // Restore original mask + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + + // Handle remaining elements (< 8) with scalar operations + for (int32_t i = vec_end; i < elements; i++) { + dst_block[i] = silu_f32(x_block[i]) * g_block[i]; + } +} + +// Vectorized ReGLU block: dst = max(0, x) * g +static inline void block_reglu(float * dst_block, const float * x_block, const float * g_block, int elements) { + int32_t vec_end = (elements / 8) * 8; + + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + float zero_const = 0.0f; + + for (int32_t i = 0; i < vec_end; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x_vec]\n" // f10 = x + "flw.ps f11, %[g_vec]\n" // f11 = g + "fbc.ps f20, %[zero_ptr]\n" // f20 = 0.0 + "fmax.ps f12, f10, f20\n" // f12 = max(x, 0) + "fmul.ps f13, f12, f11\n" // f13 = relu(x) * g + "fsw.ps f13, %[dst_out]\n" + : [dst_out] "=m"(*(float (*)[8]) & dst_block[i]) + : [x_vec] "m"(*(const float (*)[8]) & x_block[i]), [g_vec] "m"(*(const float (*)[8]) & g_block[i]), + [zero_ptr] "m"(zero_const) + : "f10", "f11", "f12", "f13", "f20"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + + for (int32_t i = vec_end; i < elements; i++) { + float xv = x_block[i]; + dst_block[i] = (xv > 0.0f) ? xv * g_block[i] : 0.0f; + } +} + +// Vectorized GeGLU-Quick block: dst = x * sigmoid(1.702 * x) * g +// Using gelu_quick(x) = x / (1 + exp(-1.702*x)) +static inline void block_geglu_quick(float * dst_block, const float * x_block, const float * g_block, int elements) { + int32_t vec_end = (elements / 8) * 8; + + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + float zero_const = 0.0f; + float one_const = 1.0f; + // -1.702 * log2(e), so that fexp.ps(x * neg_k_log2e) = exp(-1.702*x) + float neg_k_log2e_const = -1.702f * 1.4426950408889634f; + + for (int32_t i = 0; i < vec_end; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x_vec]\n" // f10 = x + "flw.ps f11, %[g_vec]\n" // f11 = g + "fbc.ps f20, %[zero_ptr]\n" // f20 = 0 + "fbc.ps f21, %[one_ptr]\n" // f21 = 1 + "fbc.ps f22, %[k_ptr]\n" // f22 = -1.702*log2(e) + "fmul.ps f13, f10, f22\n" // f13 = -1.702*x*log2(e) + "fexp.ps f14, f13\n" // f14 = exp(-1.702*x) + "fadd.ps f15, f14, f21\n" // f15 = 1 + exp(-1.702*x) + "frcp.ps f16, f15\n" // f16 = sigmoid(1.702*x) + "fmul.ps f17, f10, f16\n" // f17 = gelu_quick(x) + "fmul.ps f18, f17, f11\n" // f18 = gelu_quick(x) * g + "fsw.ps f18, %[dst_out]\n" + : [dst_out] "=m"(*(float (*)[8]) & dst_block[i]) + : [x_vec] "m"(*(const float (*)[8]) & x_block[i]), [g_vec] "m"(*(const float (*)[8]) & g_block[i]), + [zero_ptr] "m"(zero_const), [one_ptr] "m"(one_const), [k_ptr] "m"(neg_k_log2e_const) + : "f10", "f11", "f13", "f14", "f15", "f16", "f17", "f18", "f20", "f21", "f22"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + + for (int32_t i = vec_end; i < elements; i++) { + float xv = x_block[i]; + // Reuse silu reciprocal path: sigmoid(1.702*x) = 1/(1+exp(-1.702*x)) + float e = et_expf(-1.702f * xv); + dst_block[i] = et_fdiv(xv, 1.0f + e) * g_block[i]; + } +} + +// Vectorized SwiGLU-OAI block (OpenAI gpt-oss variant): +// x_c = min(x, limit) +// y_c = clamp(g, -limit, limit) +// out = (x_c / (1 + exp(-alpha * x_c))) * (y_c + 1) +static inline void block_swiglu_oai(float * dst_block, + const float * x_block, + const float * g_block, + int elements, + float alpha, + float limit) { + int32_t vec_end = (elements / 8) * 8; + + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + float zero_const = 0.0f; + float one_const = 1.0f; + float limit_pos = limit; + float limit_neg = -limit; + // -alpha * log2(e): feed (x * neg_alpha_log2e) into fexp.ps to get exp(-alpha*x) + float neg_alpha_l2e = -alpha * 1.4426950408889634f; + + for (int32_t i = 0; i < vec_end; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x_vec]\n" // f10 = x raw + "flw.ps f11, %[g_vec]\n" // f11 = g raw + + "fbc.ps f20, %[zero_ptr]\n" // f20 = 0 + "fbc.ps f21, %[one_ptr]\n" // f21 = 1 + "fbc.ps f23, %[lim_pos]\n" // f23 = +limit + "fbc.ps f24, %[lim_neg]\n" // f24 = -limit + "fbc.ps f25, %[k_ptr]\n" // f25 = -alpha*log2(e) + + // x_c = min(x, +limit) (no lower bound on x per OAI spec) + "fmin.ps f12, f10, f23\n" // f12 = x_c + + // y_c = clamp(g, -limit, +limit) = min(max(g, -limit), +limit) + "fmax.ps f13, f11, f24\n" // f13 = max(g, -limit) + "fmin.ps f13, f13, f23\n" // f13 = y_c + + // sigmoid(alpha * x_c) = 1 / (1 + exp(-alpha * x_c)) + "fmul.ps f14, f12, f25\n" // f14 = -alpha*x_c*log2(e) + "fexp.ps f15, f14\n" // f15 = exp(-alpha*x_c) + "fadd.ps f15, f15, f21\n" // f15 = 1 + exp(-alpha*x_c) + "frcp.ps f16, f15\n" // f16 = sigmoid(alpha*x_c) + + // out_glu = x_c * sigmoid(alpha*x_c) + "fmul.ps f17, f12, f16\n" // f17 = swiglu_oai gate output + + // dst = out_glu * (y_c + 1) + "fadd.ps f18, f13, f21\n" // f18 = y_c + 1 + "fmul.ps f19, f17, f18\n" // f19 = final + "fsw.ps f19, %[dst_out]\n" + + : [dst_out] "=m"(*(float (*)[8]) & dst_block[i]) + : [x_vec] "m"(*(const float (*)[8]) & x_block[i]), [g_vec] "m"(*(const float (*)[8]) & g_block[i]), + [zero_ptr] "m"(zero_const), [one_ptr] "m"(one_const), [lim_pos] "m"(limit_pos), [lim_neg] "m"(limit_neg), + [k_ptr] "m"(neg_alpha_l2e) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f19", "f20", "f21", "f23", "f24", "f25"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + + // Scalar tail (mirrors CPU reference exactly) + for (int32_t i = vec_end; i < elements; i++) { + float xv = x_block[i]; + float yv = g_block[i]; + if (xv > limit) { + xv = limit; + } + if (yv > limit) { + yv = limit; + } + if (yv < -limit) { + yv = -limit; + } + float e = et_expf(-alpha * xv); + float out_glu = et_fdiv(xv, 1.0f + e); + dst_block[i] = out_glu * (yv + 1.0f); + } +} + +static inline void block_swiglu_clamp(float * dst_block, + const float * gate_block, + const float * up_block, + int elements, + float limit) { + int32_t vec_end = (elements / 8) * 8; + + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + float one_const = 1.0f; + float limit_pos = limit; + float limit_neg = -limit; + float neg_log2e = -1.4426950408889634f; + + for (int32_t i = 0; i < vec_end; i += 8) { + __asm__ volatile( + "flw.ps f10, %[gate_vec]\n" + "flw.ps f11, %[up_vec]\n" + "fbc.ps f21, %[one_ptr]\n" + "fbc.ps f23, %[lim_pos]\n" + "fbc.ps f24, %[lim_neg]\n" + "fbc.ps f25, %[k_ptr]\n" + "fmin.ps f12, f10, f23\n" + "fmax.ps f13, f11, f24\n" + "fmin.ps f13, f13, f23\n" + "fmul.ps f14, f12, f25\n" + "fexp.ps f15, f14\n" + "fadd.ps f15, f15, f21\n" + "frcp.ps f16, f15\n" + "fmul.ps f17, f12, f16\n" + "fmul.ps f18, f17, f13\n" + "fsw.ps f18, %[dst_out]\n" + : [dst_out] "=m"(*(float (*)[8]) & dst_block[i]) + : [gate_vec] "m"(*(const float (*)[8]) & gate_block[i]), [up_vec] "m"(*(const float (*)[8]) & up_block[i]), + [one_ptr] "m"(one_const), [lim_pos] "m"(limit_pos), [lim_neg] "m"(limit_neg), [k_ptr] "m"(neg_log2e) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f21", "f23", "f24", "f25"); + } + + __asm__ volatile("mova.m.x %0" :: "r"(temp_mask)); + + for (int32_t i = vec_end; i < elements; i++) { + float gate = gate_block[i] > limit ? limit : gate_block[i]; + float up = up_block[i]; + up = up > limit ? limit : up; + up = up < -limit ? -limit : up; + dst_block[i] = silu_f32(gate) * up; + } +} + +// Scalar erf approximation (Abramowitz & Stegun 7.1.26, max error ~1.5e-7) +static inline float erf_approx(float x) { + const float a1 = 0.254829592f; + const float a2 = -0.284496736f; + const float a3 = 1.421413741f; + const float a4 = -1.453152027f; + const float a5 = 1.061405429f; + const float p = 0.3275911f; + + float sign = (x < 0.0f) ? -1.0f : 1.0f; + float ax = (x < 0.0f) ? -x : x; + float t = et_fdiv(1.0f, 1.0f + p * ax); + float t2 = t * t; + float t3 = t2 * t; + float t4 = t3 * t; + float t5 = t4 * t; + float poly = a1 * t + a2 * t2 + a3 * t3 + a4 * t4 + a5 * t5; + float y = 1.0f - poly * et_expf(-ax * ax); + return sign * y; +} + +// GeGLU-Erf block: dst = 0.5 * x * (1 + erf(x / sqrt(2))) * g +// Scalar implementation — variant is rarely used so we keep complexity low. +static inline void block_geglu_erf(float * dst_block, const float * x_block, const float * g_block, int elements) { + const float sqrt_2_inv = 0.70710678118654752440f; + for (int32_t i = 0; i < elements; i++) { + float xv = x_block[i]; + dst_block[i] = 0.5f * xv * (1.0f + erf_approx(xv * sqrt_2_inv)) * g_block[i]; + } +} + +// Main entry point for GLU kernel +int entry_point(struct ggml_et_glu_params * params, void * env) { + // Cast env to proper type + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + // Validate environment pointer + if (!kernel_env) { + return -1; + } + + // Get thread info using shire mask from environment + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + // Basic safety check on params + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + // Supported variants: SwiGLU, SwiGLU-OAI, GeGLU, GeGLU-Erf, GeGLU-Quick, ReGLU + switch (params->glu_op_type) { + case GGML_GLU_OP_SWIGLU: + case GGML_GLU_OP_SWIGLU_OAI: + case GGML_GLU_OP_SWIGLU_CLAMP: + case GGML_GLU_OP_GEGLU: + case GGML_GLU_OP_GEGLU_ERF: + case GGML_GLU_OP_GEGLU_QUICK: + case GGML_GLU_OP_REGLU: + break; + default: + return -1; // Unsupported GLU operation + } + + // Extract tensor references + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = params->src1.data ? ¶ms->src1 : 0; + struct ggml_tensor * dst = ¶ms->dst; + int32_t swapped = params->swapped; + + // Validate tensor types (F32 only) + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; // Unsupported type combination + } + + if (src1 && src1->type != GGML_TYPE_F32) { + return -1; // Unsupported src1 type + } + + // Get data pointers + float * src0_data = (float *) src0->data; + float * src1_data = src1 ? (float *) src1->data : src0_data; + float * dst_data = (float *) dst->data; + + // Validate data pointers + if (!src0_data || !dst_data) { + return -1; // Null data pointer + } + + // Get tensor dimensions + const int64_t nc = dst->ne[0]; // Output columns (input columns / 2) + const int64_t nr = dst->ne[1] * dst->ne[2] * dst->ne[3]; // Total rows + + // Get strides + const size_t src0_stride = src0->nb[1]; // Stride between rows in src0 + const size_t src1_stride = src1 ? src1->nb[1] : src0->nb[1]; // Stride between rows in src1 + const size_t dst_stride = dst->nb[1]; // Stride between rows in dst + + // Validate dimensions for split SwiGLU + if (src1) { + // Split tensor mode: src0 and src1 should have same shape as dst + if (src0->ne[0] != nc || src1->ne[0] != nc) { + return -1; // Dimension mismatch in split mode + } + } else { + // Single tensor mode: src0 should have 2*nc columns + if (src0->ne[0] != 2 * nc) { + return -1; // Dimension mismatch in single tensor mode + } + } + + // Calculate total elements for cache line distribution + const int64_t elements_per_cacheline = 16; // 64 bytes / 4 bytes per float + const int64_t total_elements = nr * nc; + const int64_t total_cachelines = (total_elements + elements_per_cacheline - 1) / elements_per_cacheline; + + // Distribute cache lines across threads + int64_t cachelines_per_thread = (total_cachelines + num_threads - 1) / num_threads; + int64_t start_cacheline = thread_id * cachelines_per_thread; + int64_t end_cacheline = start_cacheline + cachelines_per_thread; + + // Clamp end_cacheline to actual number of cache lines + if (end_cacheline > total_cachelines) { + end_cacheline = total_cachelines; + } + + // Thread should return if no work to do + if (start_cacheline >= total_cachelines) { + return 0; + } + + // Process cache lines assigned to this thread + for (int64_t cl = start_cacheline; cl < end_cacheline; cl++) { + // Map cache line back to element coordinates + int64_t global_element_start = cl * elements_per_cacheline; + int64_t row = global_element_start / nc; + int64_t col = global_element_start % nc; + + // Skip if we're past the end of data + if (global_element_start >= total_elements) { + break; + } + + // Calculate how many elements to process in this cache line + int64_t elements_remaining = total_elements - global_element_start; + int elements_this_block = + (int) ((elements_remaining < elements_per_cacheline) ? elements_remaining : elements_per_cacheline); + + // Process elements that span across rows + int64_t elements_processed = 0; + while (elements_processed < elements_this_block && row < nr) { + // Calculate elements to process in current row + int64_t elements_in_row = nc - col; + int64_t elements_to_process = elements_this_block - elements_processed; + if (elements_to_process > elements_in_row) { + elements_to_process = elements_in_row; + } + + // Get pointers for current row and column range + float * dst_ptr = (float *) ((char *) dst_data + row * dst_stride) + col; + + float * x_ptr; + float * g_ptr; + + if (src1) { + // Split tensor mode + x_ptr = (float *) ((char *) src0_data + row * src0_stride) + col; + g_ptr = (float *) ((char *) src1_data + row * src1_stride) + col; + } else { + // Single tensor mode - src0 contains both x and g + float * src0_row = (float *) ((char *) src0_data + row * src0_stride); + if (swapped) { + g_ptr = src0_row + col; // First half is gate + x_ptr = src0_row + nc + col; // Second half is value + } else { + x_ptr = src0_row + col; // First half is value + g_ptr = src0_row + nc + col; // Second half is gate + } + } + + // Process this segment + switch (params->glu_op_type) { + case GGML_GLU_OP_GEGLU: + block_geglu(dst_ptr, x_ptr, g_ptr, (int) elements_to_process); + break; + case GGML_GLU_OP_SWIGLU: + block_swiglu(dst_ptr, x_ptr, g_ptr, (int) elements_to_process); + break; + case GGML_GLU_OP_REGLU: + block_reglu(dst_ptr, x_ptr, g_ptr, (int) elements_to_process); + break; + case GGML_GLU_OP_GEGLU_QUICK: + block_geglu_quick(dst_ptr, x_ptr, g_ptr, (int) elements_to_process); + break; + case GGML_GLU_OP_GEGLU_ERF: + block_geglu_erf(dst_ptr, x_ptr, g_ptr, (int) elements_to_process); + break; + case GGML_GLU_OP_SWIGLU_OAI: + block_swiglu_oai(dst_ptr, x_ptr, g_ptr, (int) elements_to_process, params->alpha, params->limit); + break; + case GGML_GLU_OP_SWIGLU_CLAMP: + block_swiglu_clamp(dst_ptr, x_ptr, g_ptr, (int) elements_to_process, params->limit); + break; + default: + return -1; + } + + // Update counters + elements_processed += elements_to_process; + col += elements_to_process; + + // Move to next row if current row is complete + if (col >= nc) { + row++; + col = 0; + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c b/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c new file mode 100644 index 0000000..600e7c9 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c @@ -0,0 +1,171 @@ +//****************************************************************************** +// GROUP_NORM F32 Kernel +// Baseline scalar implementation: +// normalize over (ne0 * ne1 * channels_in_group) for each (group, batch). +// +// Parallelization: +// - Work is partitioned across (group, batch) pairs. +// - For non-cache-aligned ne0, writes are emitted in row-groups so each thread's +// destination write footprint still spans an integer number of cache lines. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include + +struct ggml_et_group_norm_params { + struct ggml_tensor src0; + struct ggml_tensor dst; + int32_t n_groups; + float eps; +}; + +int entry_point(struct ggml_et_group_norm_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + const float * src0_data = (const float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + const int32_t n_groups = params->n_groups; + const float eps = params->eps; + + if (n_groups <= 0 || eps < 0.0f) { + return -1; + } + + const int64_t ne0 = dst->ne[0]; + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t ne3 = dst->ne[3]; + + if (src0->ne[0] != ne0 || src0->ne[1] != ne1 || src0->ne[2] != ne2 || src0->ne[3] != ne3) { + return -1; + } + + const int64_t nb1 = dst->nb[1]; + const int64_t nb2 = dst->nb[2]; + const int64_t nb3 = dst->nb[3]; + const int64_t nb01 = src0->nb[1]; + const int64_t nb02 = src0->nb[2]; + const int64_t nb03 = src0->nb[3]; + + const int64_t channels_per_group = (ne2 + n_groups - 1) / n_groups; + if (channels_per_group <= 0) { + return -1; + } + + const int64_t active_groups = (ne2 + channels_per_group - 1) / channels_per_group; + const int64_t total_work = active_groups * ne3; + const int64_t rows_per_write_group = et_rows_per_cacheline_group(ne0, sizeof(float)); + + for (int64_t work = thread_id; work < total_work; work += num_threads) { + const int64_t i3 = work / active_groups; + const int64_t group_idx = work % active_groups; + + const int64_t channel_start = group_idx * channels_per_group; + int64_t channel_end = channel_start + channels_per_group; + if (channel_end > ne2) { + channel_end = ne2; + } + + const int64_t channel_count = channel_end - channel_start; + if (channel_count <= 0) { + continue; + } + + float sum = 0.0f; + float denom = 0.0f; + for (int64_t i2 = channel_start; i2 < channel_end; ++i2) { + for (int64_t i1 = 0; i1 < ne1; ++i1) { + const float * src_row = (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + for (int64_t i0 = 0; i0 < ne0; ++i0) { + sum += src_row[i0]; + denom += 1.0f; + } + } + } + + const float mean = et_fdiv(sum, denom); + + float var_sum = 0.0f; + for (int64_t i2 = channel_start; i2 < channel_end; ++i2) { + for (int64_t i1 = 0; i1 < ne1; ++i1) { + const float * src_row = (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + for (int64_t i0 = 0; i0 < ne0; ++i0) { + const float centered = src_row[i0] - mean; + var_sum += centered * centered; + } + } + } + + const float variance = et_fdiv(var_sum, denom); + const float scale = et_fdiv(1.0f, et_sqrtf(variance + eps)); + + if (ne0 % 16 == 0) { + for (int64_t i2 = channel_start; i2 < channel_end; ++i2) { + for (int64_t i1 = 0; i1 < ne1; ++i1) { + const float * src_row = + (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_row = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + for (int64_t i0 = 0; i0 < ne0; ++i0) { + dst_row[i0] = (src_row[i0] - mean) * scale; + } + } + } + } else { + const int64_t total_rows_in_group = channel_count * ne1; + const int64_t total_write_groups = (total_rows_in_group + rows_per_write_group - 1) / rows_per_write_group; + + for (int64_t write_group = 0; write_group < total_write_groups; ++write_group) { + const int64_t row_start = write_group * rows_per_write_group; + int64_t row_end = row_start + rows_per_write_group; + if (row_end > total_rows_in_group) { + row_end = total_rows_in_group; + } + + for (int64_t row = row_start; row < row_end; ++row) { + const int64_t local_i2 = row / ne1; + const int64_t i1 = row % ne1; + const int64_t i2 = channel_start + local_i2; + + const float * src_row = + (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_row = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + for (int64_t i0 = 0; i0 < ne0; ++i0) { + dst_row[i0] = (src_row[i0] - mean) * scale; + } + } + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/im2col.c b/ggml/src/ggml-et/et-kernels/src/im2col.c new file mode 100644 index 0000000..252e66f --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/im2col.c @@ -0,0 +1,130 @@ +//****************************************************************************** +// IM2COL Kernel +// Baseline scalar implementation for: +// src1: [N, IC, IH, IW] -> dst: [N, OH, OW, IC*KH*KW] (2D) +// src1: [N, IC, IW] -> dst: [N, 1, OW, IC* KW] (1D) +// +// Work is distributed by row-groups so threads own cache-line-aligned chunks of +// destination rows even when ne0 is not cache aligned. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include + +static inline void im2col_store_elem(void * dst_base, enum ggml_type dst_type, int64_t idx, float value) { + if (dst_type == GGML_TYPE_F32) { + ((float *) dst_base)[idx] = value; + } else { + ((uint16_t *) dst_base)[idx] = fp32_to_fp16(value); + } +} + +static inline float im2col_load_src_elem(const void * src_base, enum ggml_type src_type, int64_t idx) { + if (src_type == GGML_TYPE_F32) { + return ((const float *) src_base)[idx]; + } + + return fp16_to_fp32(((const uint16_t *) src_base)[idx]); +} + +int entry_point(struct ggml_et_binary_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env || params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * dst = ¶ms->dst; + + if (!src1->data || !dst->data) { + return -1; + } + + if (!((dst->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) || + (dst->type == GGML_TYPE_F16 && (src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32)))) { + return -1; + } + + const int32_t s0 = ((const int32_t *) dst->op_params)[0]; + const int32_t s1 = ((const int32_t *) dst->op_params)[1]; + const int32_t p0 = ((const int32_t *) dst->op_params)[2]; + const int32_t p1 = ((const int32_t *) dst->op_params)[3]; + const int32_t d0 = ((const int32_t *) dst->op_params)[4]; + const int32_t d1 = ((const int32_t *) dst->op_params)[5]; + const int32_t is_2d = ((const int32_t *) dst->op_params)[6]; + + const int64_t N = is_2d ? src1->ne[3] : src1->ne[2]; + const int64_t IC = is_2d ? src1->ne[2] : src1->ne[1]; + const int64_t IH = is_2d ? src1->ne[1] : 1; + const int64_t IW = src1->ne[0]; + + const int64_t KH = is_2d ? src0->ne[1] : 1; + const int64_t KW = src0->ne[0]; + + const int64_t OH = is_2d ? dst->ne[2] : 1; + const int64_t OW = dst->ne[1]; + const int64_t row_elems = dst->ne[0]; + const int64_t total_rows = OW * OH * N; + + const size_t src_batch_stride = is_2d ? src1->nb[3] : src1->nb[2]; + const size_t src_channel_stride = is_2d ? src1->nb[2] : src1->nb[1]; + + const size_t dst_row_stride = dst->nb[1]; + const size_t dst_plane_stride = is_2d ? dst->nb[2] : 0; + const size_t dst_batch_stride = is_2d ? dst->nb[3] : dst->nb[2]; + + const int64_t dst_elem_size = (dst->type == GGML_TYPE_F32) ? (int64_t) sizeof(float) : (int64_t) sizeof(uint16_t); + const int64_t rows_per_group = et_rows_per_cacheline_group(row_elems, dst_elem_size); + const int64_t total_groups = (total_rows + rows_per_group - 1) / rows_per_group; + + for (int64_t grp = thread_id; grp < total_groups; grp += num_threads) { + const int64_t row_start = grp * rows_per_group; + int64_t row_end = row_start + rows_per_group; + if (row_end > total_rows) { + row_end = total_rows; + } + + for (int64_t row = row_start; row < row_end; ++row) { + const int64_t in = row / (OH * OW); + const int64_t rem = row % (OH * OW); + const int64_t ioh = rem / OW; + const int64_t iow = rem % OW; + + void * dst_row = (char *) dst->data + in * dst_batch_stride + ioh * dst_plane_stride + iow * dst_row_stride; + + for (int64_t iic = 0; iic < IC; ++iic) { + const void * src_channel = (const char *) src1->data + in * src_batch_stride + iic * src_channel_stride; + + for (int64_t ikh = 0; ikh < KH; ++ikh) { + for (int64_t ikw = 0; ikw < KW; ++ikw) { + const int64_t iiw = iow * s0 + ikw * d0 - p0; + const int64_t iih = ioh * s1 + ikh * d1 - p1; + const int64_t dst_idx = iic * (KH * KW) + ikh * KW + ikw; + + if (iiw < 0 || iiw >= IW || iih < 0 || iih >= IH) { + im2col_store_elem(dst_row, dst->type, dst_idx, 0.0f); + } else { + const int64_t src_idx = iih * IW + iiw; + const float value = im2col_load_src_elem(src_channel, src1->type, src_idx); + im2col_store_elem(dst_row, dst->type, dst_idx, value); + } + } + } + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c b/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c new file mode 100644 index 0000000..8b67117 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c @@ -0,0 +1,237 @@ +//****************************************************************************** +// L2 Norm F32 Kernel (L2 Normalization) +// y[i] = x[i] / max(sqrt(sum(x^2)), eps) +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include +#include +#include + +// L2 Norm kernel parameters structure +struct ggml_et_l2_norm_params { + struct ggml_tensor src0; // F32 input tensor + struct ggml_tensor dst; // F32 output tensor + float eps; // Epsilon parameter for numerical stability +}; + +int entry_point(struct ggml_et_l2_norm_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + float eps = params->eps; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; // Unsupported type combination + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; // Null data pointer + } + + if (eps < 0.0f) { + return -1; // Invalid epsilon + } + + const int64_t ne0 = dst->ne[0]; + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t ne3 = dst->ne[3]; + + const size_t nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + const size_t nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + + (void) nb0; + (void) nb00; + + if (src0->ne[0] != ne0 || src0->ne[1] != ne1 || src0->ne[2] != ne2 || src0->ne[3] != ne3) { + return -1; // Shape mismatch + } + + const int32_t total_rows = (int32_t) (ne1 * ne2 * ne3); + const int shire_threads = SOC_MINIONS_PER_SHIRE * NUM_HARTS_PER_MINION; + + if (total_rows >= shire_threads) { + // Row-parallel: each thread processes whole rows + for (int64_t i3 = 0; i3 < ne3; i3++) { + for (int64_t i2 = 0; i2 < ne2; i2++) { + for (int64_t i1 = thread_id; i1 < ne1; i1 += num_threads) { + const float * src_ptr = + (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_ptr = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + + float zero = 0.0f; + __asm__ volatile("fbc.ps f10, %[z]\n" : : [z] "m"(zero) : "f10"); + + for (int32_t i0 = 0; i0 < (int32_t) ne0; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fmadd.ps f10, f11, f11, f10\n" + : + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11"); + } + + float sum_sq; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(sum_sq)::"t0", "f1", "f2", "f3", "f4", "f5"); + + float l2_norm = et_powf(sum_sq, 0.5f); + if (l2_norm < eps) { + l2_norm = eps; + } + const float scale = et_fdiv(1.0f, l2_norm); + + for (int32_t i0 = 0; i0 < (int32_t) ne0; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fbc.ps f12, %[scale_ptr]\n" + "fmul.ps f13, f11, f12\n" + "fsw.ps f13, %[result]\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]), [scale_ptr] "m"(scale) + : "f11", "f12", "f13"); + } + } + } + } + } else { + // Intra-row: threads within each shire cooperate via L2 SCP + int shire_tid = thread_id % shire_threads; + int threads_per_row = shire_threads / total_rows; + int my_row = shire_tid / threads_per_row; + int local_tid = shire_tid % threads_per_row; + int group_base = my_row * threads_per_row; + + if (my_row >= total_rows) { + FENCE; + et_barrier(ET_BARRIER_SHIRE); + return 0; + } + + int64_t i1 = my_row % ne1; + int64_t i2 = (my_row / ne1) % ne2; + int64_t i3 = my_row / (ne1 * ne2); + + const float * src_ptr = (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_ptr = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + + const int32_t elems_per_cl = 16; + int32_t total_cls = ((int32_t) ne0 + elems_per_cl - 1) / elems_per_cl; + int32_t cls_per_thread = (total_cls + threads_per_row - 1) / threads_per_row; + int32_t my_start = local_tid * cls_per_thread * elems_per_cl; + int32_t my_end = my_start + cls_per_thread * elems_per_cl; + if (my_end > (int32_t) ne0) { + my_end = (int32_t) ne0; + } + if (my_start >= (int32_t) ne0) { + my_start = 0; + my_end = 0; + } + + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + // Phase 1: partial sum of squares + __asm__ volatile("fbci.pi f10, 0" ::: "f10"); + for (int32_t i0 = my_start; i0 < my_end; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fmadd.ps f10, f11, f11, f10\n" + : + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11"); + } + + float partial_sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(partial_sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + + // Phase 2: L2SCP exchange + volatile float * my_slot = (volatile float *) et_shire_l2scp_local((uint64_t) shire_tid * 64); + *my_slot = partial_sum; + FENCE; + evict_to_l2((const void *) my_slot, 1, 64); + WAIT_CACHEOPS; + + et_barrier(ET_BARRIER_SHIRE); + + // Phase 3: all threads reduce + apply scale to own chunk + int workers = threads_per_row < total_cls ? threads_per_row : total_cls; + + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + evict_to_l2((const void *) slot, 1, 64); + } + WAIT_CACHEOPS; + + float total_sum_sq = 0.0f; + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + total_sum_sq += *slot; + } + + float l2_norm = et_powf(total_sum_sq, 0.5f); + if (l2_norm < eps) { + l2_norm = eps; + } + const float scale = et_fdiv(1.0f, l2_norm); + + if (my_start < my_end) { + uint32_t scale_bits; + __asm__ volatile("fmv.x.s %0, %1" : "=r"(scale_bits) : "f"(scale)); + __asm__ volatile("fbcx.ps f13, %[sb]\n" : : [sb] "r"(scale_bits) : "f13"); + + for (int32_t i0 = my_start; i0 < my_end; i0 += 8) { + __asm__ volatile( + "flw.ps f12, %[x_vec]\n" + "fmul.ps f14, f12, f13\n" + "fsw.ps f14, %[result]\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f12", "f14"); + } + } + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/linker.ld b/ggml/src/ggml-et/et-kernels/src/linker.ld new file mode 100644 index 0000000..b7d3485 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/linker.ld @@ -0,0 +1,85 @@ +/*------------------------------------------------------------------------- + * Independent Linker Script for GGML Bare Metal Kernels + *------------------------------------------------------------------------- + */ + +/* Define maximum number of harts (threads) - simplified for bare metal */ +MAX_HARTS = 2112; + +SECTIONS +{ + /* Start at the base address passed by -Wl,--defsym=BASE_ADDRESS=... */ + . = BASE_ADDRESS; + + /* Export entry point symbol for runtime compatibility */ + KERNEL_UMODE_ENTRY = BASE_ADDRESS; + + /* Initialization section - must come first */ + .text.init : + { + *(.text.init) + } + + /* Align to cache line boundary */ + . = ALIGN(64); + + /* Main text section for code */ + .text : { + *(.text) + *(.text.*) + } + . = ALIGN(64); + + /* Data section with global pointer setup */ + .data : + { + _data_start = .; + . = ALIGN(64); + + /* Small data section and global pointer */ + *(.sdata .sdata.*) + PROVIDE( __global_pointer$ = . + 0x800 ); + + /* Regular data */ + *(.data .data.*) + . = ALIGN(64); + _data_end = .; + } + . = ALIGN(64); + + /* BSS section for uninitialized data */ + .bss(NOLOAD) : + { + _bss_start = .; + *(.sbss*); + *(.bss*); + . = ALIGN(64); + _bss_end = .; + } + + /* Thread Local Storage (TLS) sections */ + . = ALIGN(64); + .tdata : + { + *(.tdata*) + . = ALIGN(64); + } + __tdata_start = ADDR(.tdata); + + .tbss : { + __tbss_start = .; + *(.tbss*) + } + . = . + SIZEOF(.tbss); + . = ALIGN(64); + __tbss_end = .; + + /* TLS allocation area for all harts */ + .tls-alloc ALIGN(64) (NOLOAD) : { + __tls_alloc_start = .; + . = . + (ABSOLUTE(__tbss_end) - ABSOLUTE(__tdata_start)) * MAX_HARTS; + } + + /* End of kernel image */ + _end = .; +} diff --git a/ggml/src/ggml-et/et-kernels/src/math_fp.h b/ggml/src/ggml-et/et-kernels/src/math_fp.h new file mode 100644 index 0000000..552ee8d --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/math_fp.h @@ -0,0 +1,299 @@ +//****************************************************************************** +// ET Floating Point Math Library +// Provides ET hardware-specific math functions, FP16 conversion, and trig functions +// for bare metal kernels +//****************************************************************************** + +#ifndef MATH_FP_H +#define MATH_FP_H + +#include + +//****************************************************************************** +// ET Hardware Math Functions +//****************************************************************************** + +// ET hardware division function (uses FRCP.PS instruction) +static inline float et_fdiv(float a, float b) { + float d; + unsigned long temp; + + __asm__ volatile( + "mova.x.m %[temp] \n\t" + "mov.m.x m0, x0, 1 \n\t" + "frcp.ps %[d], %[b] \n\t" + "fmul.s %[d], %[d], %[a] \n\t" + "mova.m.x %[temp] \n\t" + : [temp] "=&r"(temp), [d] "=&f"(d) + : [a] "f"(a), [b] "f"(b)); + + return d; +} + +// Power function using ET hardware vector instructions +// Implements pow(base, exp) = exp(exp * ln(base)) using FLOG.PS and FEXP.PS +static inline float et_powf(float base, float exp) { + // Handle special cases + if (base <= 0.0f) { + if (base == 0.0f) { + if (exp > 0.0f) { + return 0.0f; + } + + // For exp <= 0, return +infinity (IEEE 754: sign=0, exp=0xFF, mantissa=0) + union { + float f; + uint32_t i; + } inf = { .i = 0x7F800000 }; + + return inf.f; + } + + // For negative base, return NaN (IEEE 754: exp=0xFF, mantissa!=0) + union { + float f; + uint32_t i; + } nan = { .i = 0x7FC00000 }; + + return nan.f; + } + if (base == 1.0f) { + return 1.0f; + } + if (exp == 0.0f) { + return 1.0f; + } + if (exp == 1.0f) { + return base; + } + + // Use ET hardware instructions following DNN library pattern: + // pow(base, exp) = exp(exp * ln(base)) + float result; + unsigned long temp; + + __asm__ volatile( + "mova.x.m %[temp] \n\t" // Save current mask state + "mov.m.x m0, x0, 1 \n\t" // Set mask register m0 to enable element 0 + "flog.ps %[result], %[base] \n\t" // result = ln(base) + "fmul.s %[result], %[result], %[exp]\n\t" // result = ln(base) * exp + "fexp.ps %[result], %[result] \n\t" // result = exp(ln(base) * exp) = base^exp + "mova.m.x %[temp] \n\t" // Restore mask state + : [temp] "=&r"(temp), [result] "=&f"(result) + : [base] "f"(base), [exp] "f"(exp)); + + return result; +} + +// Natural logarithm. +static inline float et_logf(float x) { + // Handle special cases + if (x < 0.0f) { + // Return NaN for negative input + union { + float f; + uint32_t i; + } nan = { .i = 0x7FC00000 }; + + return nan.f; + } + if (x == 0.0f) { + // Return -infinity for log(0) + union { + float f; + uint32_t i; + } inf = { .i = 0xFF800000 }; + + return inf.f; + } + if (x == 1.0f) { + return 0.0f; + } + + float log2_result; + unsigned long temp; + + __asm__ volatile( + "mova.x.m %[temp] \n\t" // Save current mask state + "mov.m.x m0, x0, 1 \n\t" // Set mask register m0 to enable element 0 + "flog.ps %[result], %[x] \n\t" // result = log2(x) + "mova.m.x %[temp] \n\t" // Restore mask state + : [temp] "=&r"(temp), [result] "=&f"(log2_result) + : [x] "f"(x)); + + // Convert log2 to natural log: ln(x) = log2(x) * ln(2) + const float ln2 = 0.69314718055994530942f; + return log2_result * ln2; +} + +// Square root function implemented as et_powf(x, 0.5) +static inline float et_sqrtf(float x) { + // Handle special cases + if (x < 0.0f) { + // Return NaN for negative input (IEEE 754: exp=0xFF, mantissa!=0) + union { + float f; + uint32_t i; + } nan = { .i = 0x7FC00000 }; + + return nan.f; + } + if (x == 0.0f) { + return 0.0f; + } + + return et_powf(x, 0.5f); +} + +// Base-2 exponential: returns 2^x using the ET hardware FEXP.PS instruction. +// No base conversion, no special-case clamping — this is the raw hardware op +// with just the mask save/restore wrapper. Caller is responsible for ensuring +// x is in a range that produces a useful result (roughly [-126, 128] for fp32). +static inline float __attribute__((always_inline)) et_exp2f(float x) { + unsigned long old_mask; + float out; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 1 \n\t" + "fexp.ps %[out], %[x] \n\t" + "mova.m.x %[ms] \n\t" + : [ms] "=&r"(old_mask), [out] "=&f"(out) + : [x] "f"(x)); + return out; +} + +// Exponential function using ET hardware FEXP.PS instruction +// Note: FEXP.PS computes 2^x, so we need to convert: exp(x) = 2^(x * log2(e)) +static inline float et_expf(float x) { + // Handle special cases + if (x > 88.0f) { + // For x > 88, exp(x) would overflow, return +infinity + union { + float f; + uint32_t i; + } inf = { .i = 0x7F800000 }; + + return inf.f; + } + if (x < -87.0f) { + // For x < -87, exp(x) is essentially 0 + return 0.0f; + } + + // Convert to base-2 exponent: x * log2(e) + const float log2e = 1.4426950408889634f; // log2(e) + float x_log2e = x * log2e; + + // Use ET hardware instruction: fexp.ps computes 2^x + float result; + unsigned long temp; + + __asm__ volatile( + "mova.x.m %[temp] \n\t" // Save current mask state + "mov.m.x m0, x0, 1 \n\t" // Set mask register m0 to enable element 0 + "fexp.ps %[result], %[x_log2e] \n\t" // result = 2^(x * log2(e)) = exp(x) + "mova.m.x %[temp] \n\t" // Restore mask state + : [temp] "=&r"(temp), [result] "=&f"(result) + : [x_log2e] "f"(x_log2e)); + + return result; +} + +//****************************************************************************** +// Trigonometric Functions +//****************************************************************************** + +// FSIN.PS + +// Sine function using Taylor series +static inline float et_sinf(float x) { + const float pi = 3.14159265358979323846f; + const float two_pi = 6.28318530717958647693f; + const float pi_over_2 = 1.57079632679489661923f; + + if (x > pi || x < -pi) { + float cycles = x * et_fdiv(1.0f, two_pi); + int n = (int) cycles; + if (x < 0.0f) { + n--; // Floor for negative + } + x = x - (float) n * two_pi; + } + + // sin(x) = sin(π - x) for x in [π/2, π] + // sin(x) = -sin(-π - x) for x in [-π, -π/2] + int negate = 0; + if (x > pi_over_2) { + x = pi - x; + } else if (x < -pi_over_2) { + x = -pi - x; + negate = 1; + } + + // sin(x) ≈ x - x^3/3! + x^5/5! - x^7/7! + x^9/9! - x^11/11! + const float x2 = x * x; + const float x3 = x2 * x; + const float x5 = x3 * x2; + const float x7 = x5 * x2; + const float x9 = x7 * x2; + const float x11 = x9 * x2; + + float result = x - x3 * et_fdiv(1.0f, 6.0f) // x^3/3! + + x5 * et_fdiv(1.0f, 120.0f) // x^5/5! + - x7 * et_fdiv(1.0f, 5040.0f) // x^7/7! + + x9 * et_fdiv(1.0f, 362880.0f) // x^9/9! + - x11 * et_fdiv(1.0f, 39916800.0f); // x^11/11! + + return negate ? -result : result; +} + +// Cosine function using identity cos(x) = sin(x + π/2) +static inline float et_cosf(float x) { + const float pi_over_2 = 1.57079632679489661923f; + return et_sinf(x + pi_over_2); +} + +//****************************************************************************** +// FP16 <-> FP32 Conversion Functions +//****************************************************************************** + +// Convert FP16 (IEEE 754 half precision) to FP32 (single precision) +// Uses ET hardware FCVT.PS.F16 instruction for accurate conversion +static inline float fp16_to_fp32(uint16_t h) { + float result; + unsigned long temp; + uint32_t raw = (uint32_t) h; + + __asm__ volatile( + "mova.x.m %[temp] \n\t" // Save current mask state + "mov.m.x m0, x0, 1 \n\t" // Set mask register m0 to enable element 0 + "fbcx.ps %[result], %[raw] \n\t" // Broadcast raw FP16 bits into vector register + "fcvt.ps.f16 %[result], %[result] \n\t" // Convert FP16 to FP32 + "mova.m.x %[temp] \n\t" // Restore mask state + : [temp] "=&r"(temp), [result] "=&f"(result) + : [raw] "r"(raw)); + + return result; +} + +// Convert FP32 (single precision) to FP16 (IEEE 754 half precision) +// Uses ET hardware FCVT.F16.PS instruction for accurate conversion +static inline uint16_t fp32_to_fp16(float f) { + float result_f; + unsigned long temp; + + __asm__ volatile( + "mova.x.m %[temp] \n\t" // Save current mask state + "mov.m.x m0, x0, 1 \n\t" // Set mask register m0 to enable element 0 + "fcvt.f16.ps %[result], %[f] \n\t" // Convert FP32 to FP16 (result in lower 16 bits) + "mova.m.x %[temp] \n\t" // Restore mask state + : [temp] "=&r"(temp), [result] "=&f"(result_f) + : [f] "f"(f)); + + // Extract lower 16 bits containing the FP16 value + // The instruction zero-extends to 32 bits, so upper 16 bits are 0 + uint32_t result_bits = *(uint32_t *) &result_f; + return (uint16_t) result_bits; +} + +#endif // MATH_FP_H diff --git a/ggml/src/ggml-et/et-kernels/src/mean_f32.c b/ggml/src/ggml-et/et-kernels/src/mean_f32.c new file mode 100644 index 0000000..cbb0064 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mean_f32.c @@ -0,0 +1,220 @@ +//****************************************************************************** +// MEAN F32 Kernel +// Row-wise mean reduction: dst[0, i1, i2, i3] = mean(src0[0..ne00-1, i1, i2, i3]) +// +// Modes: +// - total_rows >= shire_threads: row-parallel, each thread handles whole rows. +// - total_rows < shire_threads: intra-row reduction within a shire. Threads +// within a shire cooperate via shire-local L2 SCP slots. All shires +// duplicate the work because L2 SCP is per-shire (no cross-shire coherency). +// +// ne00 may be any positive size and rows may have any 4-byte alignment. We +// take the 8-wide vector path only when the row pointer is 32B-aligned and +// fall back to scalar for the leftover tail (or for the entire row when the +// row start is not 32B-aligned). +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include + +struct ggml_et_mean_params { + struct ggml_tensor src0; // F32 input [ne00, ne01, ne02, ne03] + struct ggml_tensor dst; // F32 output [1, ne01, ne02, ne03] +}; + +// Sum a contiguous F32 slice [base+i_lo, base+i_hi). Uses the 8-wide vector +// path only when `base + i_lo` is 32B-aligned; the tail (and the whole slice +// when misaligned) is summed with scalar fadd.s. +static inline float partial_sum_slice(const float * base, int32_t i_lo, int32_t i_hi) { + if (i_lo >= i_hi) { + return 0.0f; + } + + const float * p = base + i_lo; + int32_t n = i_hi - i_lo; + float acc = 0.0f; + int32_t i = 0; + + if (n >= 8 && (((uintptr_t) p) & 31) == 0) { + float zero = 0.0f; + __asm__ volatile("fbc.ps f10, %[z]\n" : : [z] "m"(zero) : "f10"); + + for (; i + 8 <= n; i += 8) { + __asm__ volatile( + "flw.ps f11, %[x]\n" + "fadd.ps f10, f10, f11\n" + : + : [x] "m"(*(const float (*)[8]) & p[i]) + : "f10", "f11"); + } + + float vec_sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(vec_sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + acc = vec_sum; + } + + for (; i < n; i++) { + acc += p[i]; + } + return acc; +} + +int entry_point(struct ggml_et_mean_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + const int64_t ne00 = src0->ne[0]; + const int64_t ne01 = src0->ne[1]; + const int64_t ne02 = src0->ne[2]; + const int64_t ne03 = src0->ne[3]; + + const size_t nb01 = src0->nb[1]; + const size_t nb02 = src0->nb[2]; + const size_t nb03 = src0->nb[3]; + + const size_t nb1 = dst->nb[1]; + const size_t nb2 = dst->nb[2]; + const size_t nb3 = dst->nb[3]; + + if (ne00 <= 0) { + return 0; + } + + const int32_t total_rows = (int32_t) (ne01 * ne02 * ne03); + const int shire_threads = SOC_MINIONS_PER_SHIRE * NUM_HARTS_PER_MINION; + const float inv_ne00 = et_fdiv(1.0f, (float) (int32_t) ne00); + + // Row-parallel: each thread owns whole rows. + if (total_rows >= shire_threads) { + for (int64_t ir = thread_id; ir < total_rows; ir += num_threads) { + const int64_t i03 = ir / (ne02 * ne01); + const int64_t i02 = (ir - i03 * ne02 * ne01) / ne01; + const int64_t i01 = ir - i03 * ne02 * ne01 - i02 * ne01; + + const float * src_row = (const float *) ((const char *) src0_data + i01 * nb01 + i02 * nb02 + i03 * nb03); + float * dst_ptr = (float *) ((char *) dst_data + i01 * nb1 + i02 * nb2 + i03 * nb3); + + float row_sum = partial_sum_slice(src_row, 0, (int32_t) ne00); + atomic_store_f32(dst_ptr, row_sum * inv_ne00); + } + // Shire co-work + } else { + int shire_tid = thread_id % shire_threads; + int threads_per_row = shire_threads / total_rows; + int my_row = shire_tid / threads_per_row; + int local_tid = shire_tid % threads_per_row; + int group_base = my_row * threads_per_row; + + if (my_row >= total_rows) { + FENCE; + et_barrier(ET_BARRIER_SHIRE); + return 0; + } + + int64_t i1 = my_row % ne01; + int64_t i2 = (my_row / ne01) % ne02; + int64_t i3 = my_row / (ne01 * ne02); + + const float * src_ptr = (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_ptr = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + + // Chunk size in elements, rounded up to a multiple of 8 so that every + // thread's slice start stays 32B-aligned relative to src_ptr (which + // matters for the vector path inside partial_sum_slice). + int32_t chunk = ((int32_t) ne00 + threads_per_row - 1) / threads_per_row; + chunk = (chunk + 7) & ~7; + if (chunk < 8) { + chunk = 8; + } + + int32_t my_start = local_tid * chunk; + int32_t my_end = my_start + chunk; + if (my_end > (int32_t) ne00) { + my_end = (int32_t) ne00; + } + if (my_start > (int32_t) ne00) { + my_start = my_end = (int32_t) ne00; + } + + int workers = ((int32_t) ne00 + chunk - 1) / chunk; + if (workers > threads_per_row) { + workers = threads_per_row; + } + + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + float partial_sum = partial_sum_slice(src_ptr, my_start, my_end); + + // Publish partial to shire-local L2 SCP slot (64B per slot, one per + // hart). evict_to_l2 is required on the WRITER because scalar stores + // land in L1D first; readers must also evict before reading. + volatile float * my_slot = (volatile float *) et_shire_l2scp_local((uint64_t) shire_tid * 64); + *my_slot = partial_sum; + FENCE; + evict_to_l2((const void *) my_slot, 1, 64); + WAIT_CACHEOPS; + + et_barrier(ET_BARRIER_SHIRE); + + if (local_tid == 0) { + // Reader-side evictions for every contributing peer slot. + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + evict_to_l2((const void *) slot, 1, 64); + } + WAIT_CACHEOPS; + + float total_sum = 0.0f; + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + total_sum += *slot; + } + atomic_store_f32(dst_ptr, total_sum * inv_ne00); + } + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/memops.c b/ggml/src/ggml-et/et-kernels/src/memops.c new file mode 100644 index 0000000..b2163a4 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/memops.c @@ -0,0 +1,181 @@ +//****************************************************************************** +// Memory Operations Kernel — tensor_store based memset +// +// Uses the tensor engine's store path (bypasses L1+L2 caches) to achieve hiher +// performance. Unrolled vector writes can write at ~25GB/s and tensor writes +// can so ~71 GB/s. Only even harts (hart 0 per minion) participate, as due to +// hardware design (only thye have matrix engine access and co-op stores seems +// slower) +//****************************************************************************** + +#include "platform.h" +#include "tensor.h" + +#include +#include + +// Operation identifiers for memops kernel +enum ggml_et_memop_type { + GGML_ET_MEMOP_MEMSET = 0, +}; + +// Memset operation parameters (must match host-side struct in ggml-et-memops.cpp) +struct memset_params { + uint32_t op_type; + uint32_t value; + void * dst_ptr; + size_t size; +}; + +// Fill all 32 f-regs with a replicated byte pattern +static inline void __attribute__((always_inline)) fill_fregs(uint32_t fill32) { + register uint64_t val __asm__("a2") = fill32; + __asm__ __volatile__( + "fbcx.ps f0, %[v]\n\t" + "fbcx.ps f1, %[v]\n\t" + "fbcx.ps f2, %[v]\n\t" + "fbcx.ps f3, %[v]\n\t" + "fbcx.ps f4, %[v]\n\t" + "fbcx.ps f5, %[v]\n\t" + "fbcx.ps f6, %[v]\n\t" + "fbcx.ps f7, %[v]\n\t" + "fbcx.ps f8, %[v]\n\t" + "fbcx.ps f9, %[v]\n\t" + "fbcx.ps f10, %[v]\n\t" + "fbcx.ps f11, %[v]\n\t" + "fbcx.ps f12, %[v]\n\t" + "fbcx.ps f13, %[v]\n\t" + "fbcx.ps f14, %[v]\n\t" + "fbcx.ps f15, %[v]\n\t" + "fbcx.ps f16, %[v]\n\t" + "fbcx.ps f17, %[v]\n\t" + "fbcx.ps f18, %[v]\n\t" + "fbcx.ps f19, %[v]\n\t" + "fbcx.ps f20, %[v]\n\t" + "fbcx.ps f21, %[v]\n\t" + "fbcx.ps f22, %[v]\n\t" + "fbcx.ps f23, %[v]\n\t" + "fbcx.ps f24, %[v]\n\t" + "fbcx.ps f25, %[v]\n\t" + "fbcx.ps f26, %[v]\n\t" + "fbcx.ps f27, %[v]\n\t" + "fbcx.ps f28, %[v]\n\t" + "fbcx.ps f29, %[v]\n\t" + "fbcx.ps f30, %[v]\n\t" + "fbcx.ps f31, %[v]\n\t" ::[v] "r"(val) + : "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "f16", + "f17", "f18", "f19", "f20", "f21", "f22", "f23", "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"); +} + +// Fill a partial region [start, end) using tensor_store for 16-byte-aligned +// chunks and byte stores for any remainder < 16 bytes. +// Assumes f-regs are already loaded with the fill pattern. +static void memset_tail(uint8_t * start, uint8_t * end, uint8_t val) { + uint8_t * cur = start; + + // Full 64-byte rows via tensor_store (up to 16 at a time = 1KB) + while (cur + 64 <= end) { + size_t rows = (end - cur) / 64; + if (rows > 16) { + rows = 16; + } + tensor_store(0, 0, 3, rows - 1, (uintptr_t) cur, 0, 64); + cur += rows * 64; + } + + // Remaining 16-byte aligned chunk (16, 32, or 48 bytes) + if (cur + 16 <= end) { + size_t cols = (end - cur) / 16; + tensor_store(0, 0, cols - 1, 0, (uintptr_t) cur, 0, 64); + cur += cols * 16; + } + + tensor_wait(TENSOR_STORE_WAIT); + + // Final < 16 bytes with byte stores + while (cur < end) { + *(volatile uint8_t *) cur = val; + cur++; + } +} + +#define ALIGN_UP(ptr, align) ((uint8_t *) (((uintptr_t) (ptr) + (align) - 1) & ~((uintptr_t) (align) - 1))) + +int entry_point(struct memset_params * params, kernel_environment_t * env) { + uint64_t hart_id = get_hart_id(); + + // Only even harts have tensor engine access + if (hart_id & 1) { + return 0; + } + + if (!params || ((uintptr_t) params & 0x7) != 0) { + return -1; + } + + if (params->op_type != GGML_ET_MEMOP_MEMSET) { + return -1; + } + + uint8_t * dst = (uint8_t *) params->dst_ptr; + size_t size = params->size; + + if (!dst || size == 0) { + return -1; + } + + // Dynamic hart count from shire_mask + int num_even_harts = manual_popcountll(env->shire_mask) * SOC_MINIONS_PER_SHIRE; + + // global_id: shire * 32 + minion (for even harts) + uint64_t global_id = ((hart_id >> 6) << 5) + ((hart_id >> 1) & 0x1F); + + uint8_t val = params->value & 0xFF; + uint32_t fill32 = val | ((uint32_t) val << 8) | ((uint32_t) val << 16) | ((uint32_t) val << 24); + + uint8_t * end = dst + size; + + setup_cache_scp(); + CLEAR_TENSOR_ERROR; + fill_fregs(fill32); + + // Align to 16 bytes (tensor_store minimum alignment) + uint8_t * base = ALIGN_UP(dst, 16); + if (base > end) { + base = end; + } + + // Hart 0 handles head bytes before alignment + if (global_id == 0) { + volatile uint8_t * p = dst; + while (p < (volatile uint8_t *) base) { + *p++ = val; + } + } + + // Bulk: 1KB blocks distributed across all harts (base is already 16-byte aligned) + size_t aligned_size = end - base; + size_t total_blocks = aligned_size / 1024; + + if (total_blocks > 0) { + size_t blocks_per_hart = total_blocks / num_even_harts; + size_t extra = total_blocks % num_even_harts; + size_t my_start = blocks_per_hart * global_id + (global_id < extra ? global_id : extra); + size_t my_count = blocks_per_hart + (global_id < extra ? 1 : 0); + + uint8_t * addr = base + my_start * 1024; + for (size_t b = 0; b < my_count; b++) { + tensor_store(0, 0, 3, 15, (uintptr_t) addr, 0, 64); + addr += 1024; + } + tensor_wait(TENSOR_STORE_WAIT); + } + + // Hart 0 handles the tail after the last full 1KB block + if (global_id == 0) { + memset_tail(base + total_blocks * 1024, end, val); + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c new file mode 100644 index 0000000..d128a99 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c @@ -0,0 +1,358 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include "block_ops.h" +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" +#include "quants.h" + +#include + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 256 /* K >= 8192 elements */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 64 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 256 /* K-tile size in Q4_0 blocks (8192 elems, 32KB B data) */ +#define KSPLIT_GROUP_ROWS 4 +#define SIMPLE_X2_ROWS 2 + +int entry_point(struct ggml_et_binary_params * params, void * env) { + uint64_t hart_id = get_hart_id(); + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Q4_0 block size is 32 + const int64_t K_blocks = K / 32; + const int use_simple_x2 = ((nb01 & 31) == 0); + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (≤4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts × 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (≤2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) && (rows_per_minion <= KSPLIT_MAX_ROWS) && + (rows_per_minion <= 4 || k_half <= TILE_KB)) || + use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit && (K_blocks >= KSPLIT_MIN_K_BLOCKS) && (rows_per_minion > 4) && + (rows_per_minion <= KSPLIT_MAX_ROWS); + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float * l2scp_slot = (volatile float *) et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q4_0 * q_row = (const block_q4_0 *) (src0_ptr2 + m * nb01); + + float partial = compute_row_dot_q4_0(q_row + k_start, b_col_base + k_start * 32, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void *) l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float * dst_entry = (float *) (dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float *) dst_entry, partial + other); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + * This keeps the K-split bandwidth benefit while cutting semaphore + * traffic by 4x relative to the old per-row exchange. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float * l2scp_slot = (volatile float *) et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + + for (int64_t m_base = minion_id; m_base < M; m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) { + tile_len = TILE_KB; + } + if (tile_len <= 0) { + break; + } + const float * b_tile = b_col_base + (k_start + kb) * 32; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += compute_row_dot_q4_0((const block_q4_0 *) (src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += compute_row_dot_q4_0((const block_q4_0 *) (src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += compute_row_dot_q4_0((const block_q4_0 *) (src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += compute_row_dot_q4_0((const block_q4_0 *) (src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void *) l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float * c_base = (float *) (dst_ptr2 + n * nbd1); + if (m0 < M) { + atomic_store_f32((volatile float *) (c_base + m0), s0 + p0); + } + if (m1 < M) { + atomic_store_f32((volatile float *) (c_base + m1), s1 + p1); + } + if (m2 < M) { + atomic_store_f32((volatile float *) (c_base + m2), s2 + p2); + } + if (m3 < M) { + atomic_store_f32((volatile float *) (c_base + m3), s3 + p3); + } + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + * Uses scalar float variables (not an array) to accumulate across + * tiles — avoids the flw/fadd.s/fsw stack ops that corrupt vector + * register state on ET-SoC-1's MMX-style shared FP file. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) { + tile_len = TILE_KB; + } + const float * b_tile = b_col_base + kb * 32; + + s0 += compute_row_dot_q4_0((const block_q4_0 *) (src0_ptr2 + m0 * nb01) + kb, b_tile, + tile_len); + if (m1 < M) { + s1 += compute_row_dot_q4_0((const block_q4_0 *) (src0_ptr2 + m1 * nb01) + kb, b_tile, + tile_len); + } + if (m2 < M) { + s2 += compute_row_dot_q4_0((const block_q4_0 *) (src0_ptr2 + m2 * nb01) + kb, b_tile, + tile_len); + } + if (m3 < M) { + s3 += compute_row_dot_q4_0((const block_q4_0 *) (src0_ptr2 + m3 * nb01) + kb, b_tile, + tile_len); + } + } + + float * dst_base = (float *) (dst_ptr2 + n * nbd1); + atomic_store_f32((volatile float *) (dst_base + m0), s0); + if (m1 < M) { + atomic_store_f32((volatile float *) (dst_base + m1), s1); + } + if (m2 < M) { + atomic_store_f32((volatile float *) (dst_base + m2), s2); + } + if (m3 < M) { + atomic_store_f32((volatile float *) (dst_base + m3), s3); + } + } + } + } + } + } else { + /* + * Simple path for small K. + * + * When `nb01` is 32-byte aligned, every row has the same block-alignment + * pattern. That lets us compute two rows together and reuse each loaded + * B chunk across both rows instead of reloading it in a second dot call. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + q4_dot_state q4_state; + q4_dot_begin(&q4_state); + + if (use_simple_x2) { + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * SIMPLE_X2_ROWS) { + const int64_t m1 = m0 + STRIDE_M; + const block_q4_0 * q_row0 = (const block_q4_0 *) (src0_ptr2 + m0 * nb01); + + if (m1 < M) { + const block_q4_0 * q_row1 = (const block_q4_0 *) (src0_ptr2 + m1 * nb01); + float s0, s1; + q4_dot_compute_x2_aligned(q_row0, q_row1, b_col_base, K_blocks, &s0, &s1); + + float * dst0 = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); + float * dst1 = (float *) (dst_ptr2 + n * nbd1 + m1 * sizeof(float)); + atomic_store_f32((volatile float *) dst0, s0); + atomic_store_f32((volatile float *) dst1, s1); + } else { + float sum = q4_dot_compute(q_row0, b_col_base, K_blocks); + float * dst = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); + atomic_store_f32((volatile float *) dst, sum); + } + } + } else { + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q4_0 * q_row = (const block_q4_0 *) (src0_ptr2 + m * nb01); + + float sum = q4_dot_compute(q_row, b_col_base, K_blocks); + + float * dst_entry = (float *) (dst_ptr2 + n * nbd1 + m * sizeof(float)); + atomic_store_f32((volatile float *) dst_entry, sum); + } + } + + q4_dot_end(&q4_state); + } + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c new file mode 100644 index 0000000..28a1030 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c @@ -0,0 +1,368 @@ +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" +#include "quants.h" +#include "tensor.h" + +#include +#include + +// Q4_0 x F32 -> F32 MUL_MAT on the tensor (matrix) engine, TensorFMA32. +// Hart 1: dequantize Q4_0 weights to FP32 into double-buffered L2 SCP. +// Hart 0: tensor engine compute (FMA, reduce, store). + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define BLOCK_K QK4_0 // 32 elements per Q4_0 block +#define FMA_K 16 // tensor FMA k-width for FP32 (a_num_cols = FMA_K-1) + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // L1 SCP lines 0..15 for A (activations) +#define B_L1_START 16 // L1 SCP lines 16..31 for B (dequantized weights) + +// L2 SCP layout per minion (double-buffered dequant panel + sync counters). +// panel = BLOCK_K k-lines x TILE_M m (FP32) = 32 * 64 = 2048 bytes, in TenB +// [k][m] order: panel[k*TILE_M + m]. +#define SCP_PANEL_SIZE (BLOCK_K * TILE_M * (uint64_t) sizeof(float)) // 2048 +#define SCP_READY_OFF (2 * SCP_PANEL_SIZE) // 4096 +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 4160 +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 4224 + +// Signal a counter value to the other hart via L2 SCP. +static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { + *flag = value; + FENCE; + evict_to_l2((const void *) flag, 1, 64); + WAIT_CACHEOPS; +} + +// Wait for a counter in L2 SCP to reach the expected value. +static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { + while (1) { + evict_to_l2((const void *) flag, 1, 64); + WAIT_CACHEOPS; + if (*flag >= expected) { + return; + } + } +} + +// Dequantize one 32-element Q4_0 block of TILE_M weight rows into the FP32 +// panel, written directly in TenB [k][m] order: panel[k*TILE_M + m]. +// Low nibble of byte i -> k = i +// High nibble of byte i -> k = i + 16 +// value = d * (nibble - 8) +// +// Vectorized: for each weight row m we gather 8 packed bytes at a time, expand +// the low/high nibbles to FP32 (nibble-8), scale by the block's fp16 d, and +// fscw.ps-scatter the 8 values down 8 panel lines (stride 64B) at column m. +// 4 groups of 8 cover the 32 k-values (low 0..15, high 16..31). +static inline void __attribute__((always_inline)) dequant_q4_0_panel(float * panel, + const char * src0_batch, + int64_t mb, + int64_t kb_block, + int64_t nb1_0) { + static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { + 0, 64, 128, 192, 256, 320, 384, 448 // byte offsets: 8 lines apart + }; + static const int32_t __attribute__((aligned(32))) gather_idx[8] = { + 0, 1, 2, 3, 4, 5, 6, 7 // 8 consecutive bytes + }; + + unsigned long old_mask; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" // all 8 lanes active + "flw.ps f1, (%[sidx]) \n\t" // f1 = scatter offsets + "flw.ps f2, (%[gidx]) \n\t" // f2 = gather offsets + : [ms] "=&r"(old_mask) + : [sidx] "r"(scatter_idx), [gidx] "r"(gather_idx) + : "f1", "f2"); + + char * pbase = (char *) panel; + for (int j = 0; j < TILE_M; ++j) { + const block_q4_0 * blk = (const block_q4_0 *) (src0_batch + (mb + j) * nb1_0) + kb_block; + uint32_t scale_raw = (uint32_t) blk->d; + const uint8_t * qs = blk->qs; + char * col = pbase + j * 4; // column m=j of the panel + + __asm__ volatile( + "fbcx.ps f3, %[sb] \n\t" // broadcast fp16 scale bits + "fcvt.ps.f16 f3, f3 \n\t" // -> d in all 8 lanes (fp32) + + "fgb.ps f4, f2(%[qs0]) \n\t" // gather qs[0..7] + "fandi.pi f5, f4, 15 \n\t" // low nibble + "faddi.pi f5, f5, -8 \n\t" + "fcvt.ps.pw f5, f5, rne \n\t" + "fmul.ps f5, f5, f3 \n\t" + "fscw.ps f5, f1(%[c0]) \n\t" // k=0..7 -> lines 0..7 + "fsrli.pi f6, f4, 4 \n\t" // high nibble + "fandi.pi f6, f6, 15 \n\t" + "faddi.pi f6, f6, -8 \n\t" + "fcvt.ps.pw f6, f6, rne \n\t" + "fmul.ps f6, f6, f3 \n\t" + "fscw.ps f6, f1(%[c16]) \n\t" // k=16..23 -> lines 16..23 + + "fgb.ps f4, f2(%[qs8]) \n\t" // gather qs[8..15] + "fandi.pi f5, f4, 15 \n\t" + "faddi.pi f5, f5, -8 \n\t" + "fcvt.ps.pw f5, f5, rne \n\t" + "fmul.ps f5, f5, f3 \n\t" + "fscw.ps f5, f1(%[c8]) \n\t" // k=8..15 -> lines 8..15 + "fsrli.pi f6, f4, 4 \n\t" + "fandi.pi f6, f6, 15 \n\t" + "faddi.pi f6, f6, -8 \n\t" + "fcvt.ps.pw f6, f6, rne \n\t" + "fmul.ps f6, f6, f3 \n\t" + "fscw.ps f6, f1(%[c24]) \n\t" // k=24..31 -> lines 24..31 + : + : [sb] "r"(scale_raw), [qs0] "r"(qs), [qs8] "r"(qs + 8), [c0] "r"(col), [c8] "r"(col + 8 * 64), + [c16] "r"(col + 16 * 64), [c24] "r"(col + 24 * 64) + : "f3", "f4", "f5", "f6", "memory"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(old_mask)); +} + +int entry_point(struct ggml_et_binary_params * params, void * env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) { + return 0; + } + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + if ((M % TILE_M) != 0) { + return 0; + } + if ((K % BLOCK_K) != 0) { + return 0; + } + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char * src0_base = (const char *) params->src0.data; + const char * src1_base = (const char *) params->src1.data; + char * dst_base = (char *) params->dst.data; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t k_steps = K / BLOCK_K; // number of Q4_0 blocks + + // Force a single K-split. + const int64_t k_splits = 1; + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; + const int64_t k_split = local_minion % k_splits; + const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + + const int64_t k_steps_per_split = k_steps / k_splits; + const int64_t kb_start = k_split * k_steps_per_split; // first block + const int64_t kb_end = kb_start + k_steps_per_split; // one past last + + // L2 SCP pointers for this minion's double-buffered panels + sync. + uint64_t scp_base = local_minion * SCP_PER_MINION; + float * scp_panel[2] = { + (float *) et_shire_l2scp_local(scp_base), + (float *) et_shire_l2scp_local(scp_base + SCP_PANEL_SIZE), + }; + volatile uint32_t * ready_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t * consumed_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // ================================================================ + // Hart 1: Q4_0 weight dequant producer + // ================================================================ + if (is_hart1) { + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + + uint32_t chunk_id = 0; + + for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = kb_start; kb < kb_end; ++kb) { + int buf = chunk_id & 1; + + // Back-pressure: wait for hart 0 to finish with this buffer. + if (chunk_id >= 2) { + scp_wait(consumed_ctr, chunk_id - 1); + } + + dequant_q4_0_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_panel[buf], BLOCK_K, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + + FENCE; + return 0; + } + + // ================================================================ + // Hart 0: tensor engine compute + // ================================================================ + uint64_t my_minion_id = get_minion_id(); + const uint64_t group_base_global = my_minion_id - k_split; + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + + evict_to_l2((const void *) ready_ctr, 1, 64); + WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); + WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + + for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + + // Partial-N tiles run TensorFMA32 with a_num_rows = n_cur-1. + // Errata Type D workaround for n_cur == 4 (AROWS==3): pad A to AROWS==4. + const int64_t arows_fma = (n_cur == 4) ? 4 : (n_cur - 1); + + if (n_cur == 4) { + // Zero the padded 5th A row (line A_L1_START+4) once; the per-pass A + // load only writes lines A_L1_START..+3, so this persists. + static const float __attribute__((aligned(64))) zero_line[16] = { 0 }; + tensor_load(false, false, A_L1_START + 4, TENSOR_LOAD_PLAIN, 0, (uint64_t) zero_line, 0, + 0, // 1 line + 64, 0); + tensor_wait(TENSOR_LOAD_WAIT_0); + } + + int first = 1; // first_pass=1 only for the very first FMA of the tile + + for (int64_t kb = kb_start; kb < kb_end; ++kb) { + int buf = chunk_id & 1; + + // Wait for hart 1 to finish dequantizing this block. + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + // Two FMA passes over the 32-wide block (16 K-cols each). + for (int half = 0; half < 2; ++half) { + const int64_t k_elem = kb * BLOCK_K + half * FMA_K; + + // Load A (activations) for this 16-K sub-tile, PLAIN. + tensor_load(false, false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t) (src1_batch + nb * nb1_1 + k_elem * (int64_t) sizeof(float)), 0, n_cur - 1, + (uint64_t) nb1_1, 0); + + // Load B (dequantized weights) half from L2 SCP panel, PLAIN. + tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t) (scp_panel[buf] + (int64_t) half * FMA_K * TILE_M), 0, FMA_K - 1, 64, 1); + + tensor_wait(TENSOR_LOAD_WAIT_0); + tensor_wait(TENSOR_LOAD_WAIT_1); + + tensor_fma(false, + 3, // b_num_col: (16/4)-1 + arows_fma, // a_num_rows (n_cur-1, or 4 for the n_cur==4 errata pad) + FMA_K - 1, // a_num_cols + 0, false, false, false, false, B_L1_START, A_L1_START, TENSOR_FMA_OP_FP32, first); + + tensor_wait(TENSOR_FMA_WAIT); + first = 0; + } + + // Signal that this buffer is free for hart 1 to reuse. + scp_signal(consumed_ctr, chunk_id); + } + + // K-split ring reduce. + if (k_splits > 1) { + const uint64_t num_regs = (uint64_t) n_cur * 2; + + if (k_split > 0) { + tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); + tensor_wait(TENSOR_REDUCE_WAIT); + } + + if (k_split < k_splits - 1) { + tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); + tensor_wait(TENSOR_REDUCE_WAIT); + } + } + + // Store FP32 result tile (only the last k-split owns the final sum). + if (k_split == k_splits - 1) { + tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), 0, + (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c new file mode 100644 index 0000000..ad21a3e --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c @@ -0,0 +1,413 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include "block_ops.h" +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" +#include "quants.h" + +#include + +#define STRIDE_M 2048 /* 32 shires x 32 minions x 2 harts */ +#define STRIDE_M_KSPLIT 1024 /* 32 shires x 32 minions (both harts share rows) */ +#define KSPLIT_MIN_K_BLOCKS 256 /* K >= 8192 elements */ +#define KSPLIT_SMALL_ROWS_K_BLOCKS 64 /* K >= 2048 elements for very small M */ +#define KSPLIT_MAX_ROWS 8 /* max rows per minion for K-split */ +#define TILE_KB 256 /* K-tile size in Q8_0 blocks (8192 elems, 32KB B data) */ +#define KSPLIT_GROUP_ROWS 4 +#define SIMPLE_X2_ROWS 2 + +static inline size_t tensor_bytes(const struct ggml_tensor * t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} + +int entry_point(struct ggml_et_mm_q8_params * params, void * env) { + uint64_t hart_id = get_hart_id(); + + // Matrix dimensions + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + const int64_t ne02 = params->src0.ne[2]; + const int64_t ne03 = params->src0.ne[3]; + const int64_t ne12 = params->src1.ne[2]; + const int64_t ne13 = params->src1.ne[3]; + + // Strides (in bytes) + const size_t nb01 = params->src0.nb[1]; + const size_t nb02 = params->src0.nb[2]; + const size_t nb03 = params->src0.nb[3]; + + const size_t nb11 = params->src1.nb[1]; + const size_t nb12 = params->src1.nb[2]; + const size_t nb13 = params->src1.nb[3]; + + const size_t nbd1 = params->dst.nb[1]; + const size_t nbd2 = params->dst.nb[2]; + const size_t nbd3 = params->dst.nb[3]; + + // Optional residual bias + const char * bias_base = (const char *) params->bias.data; + const size_t nbb1 = params->bias.nb[1]; + const size_t nbb2 = params->bias.nb[2]; + const size_t nbb3 = params->bias.nb[3]; + + // Q8_0 block size is 32 + const int64_t K_blocks = K / 32; + const int use_simple_x2 = ((nb01 & 31) == 0); + + // Broadcasting ratios + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + // K-split decision + const int64_t minion_id = hart_id >> 1; /* 0..1023 global */ + const int64_t local_minion = (hart_id >> 1) & 0x1F; /* 0..31 within shire */ + const int is_hart1 = hart_id & 1; + const int64_t rows_per_minion = (M + STRIDE_M_KSPLIT - 1) / STRIDE_M_KSPLIT; + const int64_t k_half = K_blocks / 2; + const int use_ksplit_small_rows = (rows_per_minion <= 2) && (K_blocks >= KSPLIT_SMALL_ROWS_K_BLOCKS); + /* + * K-split when K is large enough to benefit, and either: + * - few rows (≤4): always safe, proven working + * - more rows (5-8): only if each hart's half fits in one tile, + * otherwise L1 thrashing from 2 harts × 8 rows kills performance + * + * Also allow K-split earlier for the low-M regime (≤2 rows/minion). In + * that case the simple row-striped path leaves half the machine idle, so + * using both harts on each row pays off even for moderate K. + */ + const int use_ksplit = ((K_blocks >= KSPLIT_MIN_K_BLOCKS) && (rows_per_minion <= KSPLIT_MAX_ROWS) && + (rows_per_minion <= 4 || k_half <= TILE_KB)) || + use_ksplit_small_rows; + const int use_ksplit_group = !use_ksplit && (K_blocks >= KSPLIT_MIN_K_BLOCKS) && (rows_per_minion > 4) && + (rows_per_minion <= KSPLIT_MAX_ROWS); + + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); + if (params->bias.data) { + evict_region_past_l2(params->bias.data, tensor_bytes(¶ms->bias)); + } + + if (use_ksplit) { + /* Each hart processes half the K dimension */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + + /* One cache-line-aligned L2SCP slot per minion for exchange */ + volatile float * l2scp_slot = (volatile float *) et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; + + for (int64_t m = minion_id; m < M; m += STRIDE_M_KSPLIT) { + const block_q8_0 * q_row = (const block_q8_0 *) (src0_ptr2 + m * nb01); + + float partial = compute_row_dot_q8_0(q_row + k_start, b_col_base + k_start * 32, k_len); + + if (is_hart1) { + *l2scp_slot = partial; + FENCE; + flush_to_l2((const void *) l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + float other = *l2scp_slot; + et_sem_post(ET_BARRIER_MINION); + + float * dst_entry = (float *) (dst_ptr2 + n * nbd1 + m * sizeof(float)); + float sum = partial + other; + if (bias_n) { + sum += bias_n[m]; + } + atomic_store_f32((volatile float *) dst_entry, sum); + } + } + } + } + } + } else if (use_ksplit_group) { + /* + * Grouped K-split for the 5-8 rows/minion regime. + * + * Both harts process the same 4-row group, each on half of K, and + * exchange 4 partial sums once per group instead of once per row. + * This keeps the K-split bandwidth benefit while cutting semaphore + * traffic by 4x relative to the old per-row exchange. + */ + const int64_t k_start = is_hart1 ? k_half : 0; + const int64_t k_len = is_hart1 ? (K_blocks - k_half) : k_half; + volatile float * l2scp_slot = (volatile float *) et_shire_l2scp_local(local_minion * 64); + + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; + + for (int64_t m_base = minion_id; m_base < M; m_base += STRIDE_M_KSPLIT * KSPLIT_GROUP_ROWS) { + const int64_t m0 = m_base; + const int64_t m1 = m0 + STRIDE_M_KSPLIT; + const int64_t m2 = m1 + STRIDE_M_KSPLIT; + const int64_t m3 = m2 + STRIDE_M_KSPLIT; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = k_len - kb; + if (tile_len > TILE_KB) { + tile_len = TILE_KB; + } + if (tile_len <= 0) { + break; + } + const float * b_tile = b_col_base + (k_start + kb) * 32; + const int64_t row_kb = k_start + kb; + + if (m0 < M) { + s0 += compute_row_dot_q8_0((const block_q8_0 *) (src0_ptr2 + m0 * nb01) + row_kb, + b_tile, tile_len); + } + if (m1 < M) { + s1 += compute_row_dot_q8_0((const block_q8_0 *) (src0_ptr2 + m1 * nb01) + row_kb, + b_tile, tile_len); + } + if (m2 < M) { + s2 += compute_row_dot_q8_0((const block_q8_0 *) (src0_ptr2 + m2 * nb01) + row_kb, + b_tile, tile_len); + } + if (m3 < M) { + s3 += compute_row_dot_q8_0((const block_q8_0 *) (src0_ptr2 + m3 * nb01) + row_kb, + b_tile, tile_len); + } + } + + if (is_hart1) { + l2scp_slot[0] = s0; + l2scp_slot[1] = s1; + l2scp_slot[2] = s2; + l2scp_slot[3] = s3; + FENCE; + flush_to_l2((const void *) l2scp_slot, 1, 64); + WAIT_CACHEOPS; + et_sem_post(ET_BARRIER_MINION); + et_sem_wait(ET_BARRIER_MINION); + } else { + et_sem_wait(ET_BARRIER_MINION); + const float p0 = l2scp_slot[0]; + const float p1 = l2scp_slot[1]; + const float p2 = l2scp_slot[2]; + const float p3 = l2scp_slot[3]; + et_sem_post(ET_BARRIER_MINION); + + float * c_base = (float *) (dst_ptr2 + n * nbd1); + const float b0 = bias_n ? bias_n[m0] : 0.0f; + const float b1 = (bias_n && m1 < M) ? bias_n[m1] : 0.0f; + const float b2 = (bias_n && m2 < M) ? bias_n[m2] : 0.0f; + const float b3 = (bias_n && m3 < M) ? bias_n[m3] : 0.0f; + if (m0 < M) { + atomic_store_f32((volatile float *) (c_base + m0), s0 + p0 + b0); + } + if (m1 < M) { + atomic_store_f32((volatile float *) (c_base + m1), s1 + p1 + b1); + } + if (m2 < M) { + atomic_store_f32((volatile float *) (c_base + m2), s2 + p2 + b2); + } + if (m3 < M) { + atomic_store_f32((volatile float *) (c_base + m3), s3 + p3 + b3); + } + } + } + } + } + } + } else if (K_blocks > TILE_KB) { + /* + * Tile-outer with scalar row groups: process up to 4 rows per + * hart sharing each B tile before advancing to the next tile. + * Uses scalar float variables (not an array) to accumulate across + * tiles — avoids the flw/fadd.s/fsw stack ops that corrupt vector + * register state on ET-SoC-1's MMX-style shared FP file. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; + + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * 4) { + const int64_t m1 = m0 + STRIDE_M; + const int64_t m2 = m0 + STRIDE_M * 2; + const int64_t m3 = m0 + STRIDE_M * 3; + + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + + for (int64_t kb = 0; kb < K_blocks; kb += TILE_KB) { + int64_t tile_len = K_blocks - kb; + if (tile_len > TILE_KB) { + tile_len = TILE_KB; + } + const float * b_tile = b_col_base + kb * 32; + + s0 += compute_row_dot_q8_0((const block_q8_0 *) (src0_ptr2 + m0 * nb01) + kb, b_tile, + tile_len); + if (m1 < M) { + s1 += compute_row_dot_q8_0((const block_q8_0 *) (src0_ptr2 + m1 * nb01) + kb, b_tile, + tile_len); + } + if (m2 < M) { + s2 += compute_row_dot_q8_0((const block_q8_0 *) (src0_ptr2 + m2 * nb01) + kb, b_tile, + tile_len); + } + if (m3 < M) { + s3 += compute_row_dot_q8_0((const block_q8_0 *) (src0_ptr2 + m3 * nb01) + kb, b_tile, + tile_len); + } + } + + float * dst_base = (float *) (dst_ptr2 + n * nbd1); + const float b0 = bias_n ? bias_n[m0] : 0.0f; + const float b1 = (bias_n && m1 < M) ? bias_n[m1] : 0.0f; + const float b2 = (bias_n && m2 < M) ? bias_n[m2] : 0.0f; + const float b3 = (bias_n && m3 < M) ? bias_n[m3] : 0.0f; + atomic_store_f32((volatile float *) (dst_base + m0), s0 + b0); + if (m1 < M) { + atomic_store_f32((volatile float *) (dst_base + m1), s1 + b1); + } + if (m2 < M) { + atomic_store_f32((volatile float *) (dst_base + m2), s2 + b2); + } + if (m3 < M) { + atomic_store_f32((volatile float *) (dst_base + m3), s3 + b3); + } + } + } + } + } + } else { + /* + * Simple path for small K. + * + * When `nb01` is 32-byte aligned, every row has the same block-alignment + * pattern. That lets us compute two rows together and reuse each loaded + * B chunk across both rows instead of reloading it in a second dot call. + */ + for (int64_t i3 = 0; i3 < ne13; i3++) { + const int64_t i03 = i3 / r3; + const char * src0_ptr3 = (const char *) params->src0.data + i03 * nb03; + const char * src1_ptr3 = (const char *) params->src1.data + i3 * nb13; + char * dst_ptr3 = (char *) params->dst.data + i3 * nbd3; + const char * bias_ptr3 = bias_base ? bias_base + i3 * nbb3 : (const char *) 0; + + for (int64_t i2 = 0; i2 < ne12; i2++) { + const int64_t i02 = i2 / r2; + const char * src0_ptr2 = src0_ptr3 + i02 * nb02; + const char * src1_ptr2 = src1_ptr3 + i2 * nb12; + char * dst_ptr2 = dst_ptr3 + i2 * nbd2; + const char * bias_ptr2 = bias_ptr3 ? bias_ptr3 + i2 * nbb2 : (const char *) 0; + + for (int64_t n = 0; n < N; n++) { + const float * b_col_base = (const float *) (src1_ptr2 + n * nb11); + const float * bias_n = bias_ptr2 ? (const float *) (bias_ptr2 + n * nbb1) : (const float *) 0; + q8_dot_state q8_state; + q8_dot_begin(&q8_state); + + if (use_simple_x2) { + for (int64_t m0 = hart_id; m0 < M; m0 += STRIDE_M * SIMPLE_X2_ROWS) { + const int64_t m1 = m0 + STRIDE_M; + const block_q8_0 * q_row0 = (const block_q8_0 *) (src0_ptr2 + m0 * nb01); + + if (m1 < M) { + const block_q8_0 * q_row1 = (const block_q8_0 *) (src0_ptr2 + m1 * nb01); + float s0, s1; + q8_dot_compute_x2_aligned(q_row0, q_row1, b_col_base, K_blocks, &s0, &s1); + + float * dst0 = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); + float * dst1 = (float *) (dst_ptr2 + n * nbd1 + m1 * sizeof(float)); + if (bias_n) { + s0 += bias_n[m0]; + s1 += bias_n[m1]; + } + atomic_store_f32((volatile float *) dst0, s0); + atomic_store_f32((volatile float *) dst1, s1); + } else { + float sum = q8_dot_compute(q_row0, b_col_base, K_blocks); + float * dst = (float *) (dst_ptr2 + n * nbd1 + m0 * sizeof(float)); + if (bias_n) { + sum += bias_n[m0]; + } + atomic_store_f32((volatile float *) dst, sum); + } + } + } else { + for (int64_t m = hart_id; m < M; m += STRIDE_M) { + const block_q8_0 * q_row = (const block_q8_0 *) (src0_ptr2 + m * nb01); + + float sum = q8_dot_compute(q_row, b_col_base, K_blocks); + + float * dst_entry = (float *) (dst_ptr2 + n * nbd1 + m * sizeof(float)); + if (bias_n) { + sum += bias_n[m]; + } + atomic_store_f32((volatile float *) dst_entry, sum); + } + } + + q8_dot_end(&q8_state); + } + } + } + } + +#ifdef ET_UBERKERNEL + FENCE; + evict_region_past_l2(params->dst.data, tensor_bytes(¶ms->dst)); + WAIT_CACHEOPS; + FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c new file mode 100644 index 0000000..3f1fcd5 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c @@ -0,0 +1,142 @@ +//****************************************************************************** +// MUL_MAT Kernel +// Matrix multiplication: C[M,N] = A[M,K] * B[K,N] +//****************************************************************************** + +#include "block_ops.h" +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" +#include "quants.h" + +#include + +int entry_point(struct ggml_et_binary_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env || params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + // Thread coordination + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0 || (thread_id & 1)) { + return 0; // Skip odd threads to avoid resource contention + } + + int effective_thread_id = thread_id / 2; + int effective_num_threads = (num_threads + 1) / 2; + + // Extract tensor references + struct ggml_tensor * src0 = ¶ms->src0; // Weight matrix A (F16) + struct ggml_tensor * src1 = ¶ms->src1; // Activation matrix B (F16/F32) + struct ggml_tensor * dst = ¶ms->dst; // Output matrix C (F32) + + // Generic non-matrix-engine path: F16 x (F16/F32) -> F32 + if (src0->type != GGML_TYPE_F16 || (src1->type != GGML_TYPE_F16 && src1->type != GGML_TYPE_F32) || + dst->type != GGML_TYPE_F32) { + return -1; + } + + const uint16_t * src0_data = (const uint16_t *) src0->data; + float * dst_data = (float *) dst->data; + + // Dimensions and Strides + const int64_t K = src0->ne[0]; + const int64_t M = src0->ne[1]; + const int64_t N = src1->ne[1]; + + const int64_t ne02 = src0->ne[2], ne03 = src0->ne[3]; + const int64_t ne12 = src1->ne[2], ne13 = src1->ne[3]; + const int64_t ne2 = dst->ne[2], ne3 = dst->ne[3]; + + const size_t nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const size_t nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + const size_t nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + // F16 specific block size (Usually QK_F16) + const int block_size = QK_F16; + const int64_t K_blocks = K / block_size; + const int64_t K_remainder = K % block_size; + + // Threading distribution + const uint64_t total_elements = M * N * ne2 * ne3; + const uint64_t per_thread = 16; + const uint64_t threads_stride = per_thread * effective_num_threads; + + if (effective_thread_id * per_thread >= total_elements) { + return 0; + } + + // Broadcasting support + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + for (uint64_t base_idx = effective_thread_id * per_thread; base_idx < total_elements; base_idx += threads_stride) { + for (uint64_t j = 0; j < per_thread; j++) { + const uint64_t idx = base_idx + j; + if (idx >= total_elements) { + break; + } + + // Index decoding + const int64_t i3 = idx / (M * N * ne2); + const int64_t rem3 = idx % (M * N * ne2); + const int64_t i2 = rem3 / (M * N); + const int64_t rem2 = rem3 % (M * N); + const int64_t n = rem2 / M; + const int64_t m = rem2 % M; + + const int64_t i03 = i3 / r3, i02 = i2 / r2; + const int64_t i13 = (ne13 > 1) ? i3 : 0, i12 = (ne12 > 1) ? i2 : 0; + + float sum = 0.0f; + const uint16_t * f16_row = + (const uint16_t *) ((const char *) src0_data + m * nb01 + i02 * nb02 + i03 * nb03); + + if (src1->type == GGML_TYPE_F32) { + const float * src1_data = (const float *) src1->data; + + for (int64_t kb = 0; kb < K_blocks; kb++) { + const float * b_col_ptr = + (const float *) ((const char *) src1_data + (kb * block_size) * sizeof(float) + n * nb11 + + i12 * nb12 + i13 * nb13); + sum += compute_block_dot_product_f16_naive(&f16_row[kb * block_size], b_col_ptr); + } + + if (K_remainder > 0) { + const int64_t offset = K_blocks * block_size; + const float * b_col_ptr = (const float *) ((const char *) src1_data + offset * sizeof(float) + + n * nb11 + i12 * nb12 + i13 * nb13); + sum += compute_block_dot_product_f16_partial(&f16_row[offset], b_col_ptr, K_remainder); + } + } else { + const uint16_t * src1_data = (const uint16_t *) src1->data; + + for (int64_t kb = 0; kb < K_blocks; kb++) { + const uint16_t * b_col_ptr = + (const uint16_t *) ((const char *) src1_data + (kb * block_size) * sizeof(uint16_t) + n * nb11 + + i12 * nb12 + i13 * nb13); + sum += compute_block_dot_product_f16_f16_partial(&f16_row[kb * block_size], b_col_ptr, block_size); + } + + if (K_remainder > 0) { + const int64_t offset = K_blocks * block_size; + const uint16_t * b_col_ptr = + (const uint16_t *) ((const char *) src1_data + offset * sizeof(uint16_t) + n * nb11 + + i12 * nb12 + i13 * nb13); + sum += compute_block_dot_product_f16_f16_partial(&f16_row[offset], b_col_ptr, K_remainder); + } + } + + // Atomic store for output + volatile float * c_element = + (volatile float *) ((char *) dst_data + m * dst->nb[0] + n * nb1 + i2 * nb2 + i3 * nb3); + atomic_store_f32(c_element, sum); + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c new file mode 100644 index 0000000..2aab87a --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c @@ -0,0 +1,329 @@ +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" + +#include +#include + +// FP16 x FP16 -> FP32 MUL_MAT with hart 1 B-panel packing +// +// Hart 0: tensor engine (load A, load B from SCP, FMA, reduce, store) +// Hart 1: pack B into double-buffered L2 SCP panels, flush for tensor_load +// +// Sync: monotonic counters in L2 SCP with evict-based coherency. +// Double-buffered bpanel allows pack/FMA overlap. +// +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 + +#define TILE_M 16 +#define TILE_N 16 +#define TILE_K 32 + +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +#define A_L1_START 0 // SCP lines 0..15 for A +#define B_L1_START 16 // SCP lines 16..31 for B + +typedef uint16_t et_fp16_t; + +// L2 SCP layout per minion (double-buffered bpanel + sync counters) +// [0..1023] bpanel buffer 0 (16 lines x 64 bytes) +// [1024..2047] bpanel buffer 1 +// [2048..2111] ready counter (hart1 -> hart0, own cache line) +// [2112..2175] consumed counter (hart0 -> hart1, own cache line) +#define SCP_BPANEL_SIZE (16 * 32 * sizeof(et_fp16_t)) // 1024 bytes +#define SCP_READY_OFF (2 * SCP_BPANEL_SIZE) // 2048 +#define SCP_CONSUMED_OFF (SCP_READY_OFF + 64) // 2112 +#define SCP_PER_MINION (SCP_CONSUMED_OFF + 64) // 2176 + +// Signal a counter value to the other hart via L2 SCP. +static inline void __attribute__((always_inline)) scp_signal(volatile uint32_t * flag, uint32_t value) { + *flag = value; + FENCE; + evict_to_l2((const void *) flag, 1, 64); + WAIT_CACHEOPS; +} + +// Wait for a counter in L2 SCP to reach the expected value. +static inline void __attribute__((always_inline)) scp_wait(volatile uint32_t * flag, uint32_t expected) { + while (1) { + evict_to_l2((const void *) flag, 1, 64); + WAIT_CACHEOPS; + if (*flag >= expected) { + return; + } + } +} + +/** + * Build the interleaved B panel that TensorFMA16A32 expects (vectorized). + * + * Output: 16 lines x 32 fp16 = 1024 bytes, 64-byte aligned. + * out[l][j*2+0] = src0[mb + j][kb + 2*l] + * out[l][j*2+1] = src0[mb + j][kb + 2*l + 1] + * + * Uses fsch.ps scatter store: load 8 pairs per row, scatter to 8 output lines. + */ +static inline void __attribute__((always_inline)) pack_b_interleaved(et_fp16_t * out, + const char * src0_batch, + int64_t mb, + int64_t kb, + int64_t nb1_0) { + static const int32_t __attribute__((aligned(32))) scatter_idx[8] = { 0, 64, 128, 192, 256, 320, 384, 448 }; + + unsigned long old_mask; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "flw.ps f1, 0(%[idx]) \n\t" + : [ms] "=&r"(old_mask) + : [idx] "r"(scatter_idx) + : "f1"); + + for (int j = 0; j < TILE_M; ++j) { + const et_fp16_t * row = (const et_fp16_t *) (src0_batch + (mb + j) * nb1_0) + kb; + char * dst = (char *) out + j * 4; + + __asm__ volatile( + "flw.ps f2, 0(%[src]) \n\t" + "flw.ps f3, 32(%[src]) \n\t" + "fscw.ps f2, f1(%[d0]) \n\t" + "fscw.ps f3, f1(%[d1]) \n\t" + : + : [src] "r"(row), [d0] "r"(dst), [d1] "r"(dst + 512) + : "f2", "f3", "memory"); + } + + __asm__ volatile("mova.m.x %[ms] \n\t" : : [ms] "r"(old_mask)); +} + +int entry_point(struct ggml_et_binary_params * params, void * env) { + (void) env; + + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) { + return 0; + } + + const int is_hart1 = hart_id & 1; + uint64_t local_minion = (hart_id >> 1) & 0x1F; + + // Dimensions (both harts need these for tile assignment) + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char * src0_base = (const char *) params->src0.data; + const char * src1_base = (const char *) params->src1.data; + char * dst_base = (char *) params->dst.data; + + if ((M % TILE_M) != 0) { + return 0; + } + if ((K % TILE_K) != 0) { + return 0; + } + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t total_harts = NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE; + const int64_t k_steps = K / TILE_K; + + int64_t k_splits = 1; + if (base_tiles < total_harts) { + k_splits = (total_harts + base_tiles - 1) / base_tiles; + int64_t ks = 1; + while (ks * 2 <= k_splits && ks * 2 <= 32 && k_steps % (ks * 2) == 0) { + ks *= 2; + } + k_splits = ks; + } + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; + const int64_t k_split = local_minion % k_splits; + const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + + const int64_t k_steps_per_split = k_steps / k_splits; + const int64_t k_start = k_split * k_steps_per_split * TILE_K; + const int64_t k_end = k_start + k_steps_per_split * TILE_K; + + // L2 SCP pointers for this minion's double-buffered panels + sync + uint64_t scp_base = local_minion * SCP_PER_MINION; + et_fp16_t * scp_bp[2] = { + (et_fp16_t *) et_shire_l2scp_local(scp_base), + (et_fp16_t *) et_shire_l2scp_local(scp_base + SCP_BPANEL_SIZE), + }; + volatile uint32_t * ready_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_READY_OFF); + volatile uint32_t * consumed_ctr = (volatile uint32_t *) et_shire_l2scp_local(scp_base + SCP_CONSUMED_OFF); + + // ================================================================ + // Hart 1: B-panel packer + // ================================================================ + if (is_hart1) { + // Initialize sync counters + scp_signal(ready_ctr, 0); + scp_signal(consumed_ctr, 0); + + uint32_t chunk_id = 0; + + for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const int64_t mb = mb_idx * TILE_M; + + for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { + int buf = chunk_id & 1; + + // Back-pressure: wait for hart 0 to finish with this buffer + if (chunk_id >= 2) { + scp_wait(consumed_ctr, chunk_id - 1); + } + + pack_b_interleaved(scp_bp[buf], src0_batch, mb, kb, nb1_0); + + FENCE; + flush_to_l2(scp_bp[buf], 16, 64); + WAIT_CACHEOPS; + + chunk_id++; + scp_signal(ready_ctr, chunk_id); + } + } + + FENCE; + return 0; + } + + // ================================================================ + // Hart 0: tensor engine compute + // ================================================================ + uint64_t my_minion_id = get_minion_id(); + const uint64_t group_base_global = my_minion_id - k_split; + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + + // Evict any stale L1D copies of sync counters + evict_to_l2((const void *) ready_ctr, 1, 64); + WAIT_CACHEOPS; + evict_to_l2((const void *) consumed_ctr, 1, 64); + WAIT_CACHEOPS; + + uint32_t chunk_id = 0; + + for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + + const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + + // Set tensor_mask for partial N tiles + if (n_cur < TILE_N) { + uint64_t mask = (1ULL << n_cur) - 1; + __asm__ __volatile__("csrw 0x805, %0" : : "r"(mask)); + } + + for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { + int buf = chunk_id & 1; + + // Start loading A from DRAM (overlaps with waiting for hart 1) + tensor_load((n_cur < TILE_N), false, A_L1_START, TENSOR_LOAD_PLAIN, 0, + (uint64_t) (src1_batch + nb * nb1_1 + kb * (int64_t) sizeof(et_fp16_t)), 0, n_cur - 1, + (uint64_t) nb1_1, 0); + + // Wait for hart 1 to finish packing this chunk + chunk_id++; + scp_wait(ready_ctr, chunk_id); + + // Load B from L2 SCP (hart 1 already flushed it) + tensor_load(false, false, B_L1_START, TENSOR_LOAD_PLAIN, 0, (uint64_t) scp_bp[buf], 0, 15, 64, 1); + + tensor_wait(TENSOR_LOAD_WAIT_0); + tensor_wait(TENSOR_LOAD_WAIT_1); + + // TensorFMA16A32 + tensor_fma((n_cur < TILE_N), 3, n_cur - 1, 15, 0, false, false, false, false, B_L1_START, A_L1_START, + TENSOR_FMA_OP_FP16, (kb == k_start)); + + tensor_wait(TENSOR_FMA_WAIT); + + // Signal that this buffer is free for hart 1 to reuse + scp_signal(consumed_ctr, chunk_id); + } + + // K-split ring reduce + if (k_splits > 1) { + const uint64_t num_regs = (uint64_t) n_cur * 2; + + if (k_split > 0) { + tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); + tensor_wait(TENSOR_REDUCE_WAIT); + } + + if (k_split < k_splits - 1) { + tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); + tensor_wait(TENSOR_REDUCE_WAIT); + } + } + + // Store FP32 result tile + if (k_split == k_splits - 1) { + tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * (int64_t) sizeof(float)), 0, + (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c new file mode 100644 index 0000000..107bc50 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c @@ -0,0 +1,137 @@ +#include "block_ops.h" +#include "ggml_tensor.h" +#include "platform.h" +#include "quants.h" + +#include +#include +#include + +int entry_point(struct ggml_et_binary_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env || params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + // Thread coordination + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0 || (thread_id & 1)) { + return 0; // Skip odd threads to avoid resource contention + } + + int effective_thread_id = thread_id / 2; + int effective_num_threads = (num_threads + 1) / 2; + + // Extract tensor references + struct ggml_tensor * src0 = ¶ms->src0; // Weight matrix A (F32) + struct ggml_tensor * src1 = ¶ms->src1; // Activation matrix B (F16/F32) + struct ggml_tensor * dst = ¶ms->dst; // Output matrix C (F32) + + // Generic non-matrix-engine path: F32 x (F16/F32) -> F32 + if (src0->type != GGML_TYPE_F32 || (src1->type != GGML_TYPE_F16 && src1->type != GGML_TYPE_F32) || + dst->type != GGML_TYPE_F32) { + return -1; + } + + const float * src0_data = (const float *) src0->data; + float * dst_data = (float *) dst->data; + + // Dimensions and Strides + const int64_t K = src0->ne[0]; + const int64_t M = src0->ne[1]; + const int64_t N = src1->ne[1]; + + const int64_t ne02 = src0->ne[2], ne03 = src0->ne[3]; + const int64_t ne12 = src1->ne[2], ne13 = src1->ne[3]; + const int64_t ne2 = dst->ne[2], ne3 = dst->ne[3]; + + const size_t nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const size_t nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + const size_t nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + // F32 specific block size and counts + const int block_size = QK_F32; + const int64_t K_blocks = K / block_size; + const int64_t K_remainder = K % block_size; + + // Threading distribution + const uint64_t total_elements = M * N * ne2 * ne3; + const uint64_t per_thread = 16; + const uint64_t threads_stride = per_thread * effective_num_threads; + + if (effective_thread_id * per_thread >= total_elements) { + return 0; + } + + // Broadcasting support + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + for (uint64_t base_idx = effective_thread_id * per_thread; base_idx < total_elements; base_idx += threads_stride) { + for (uint64_t j = 0; j < per_thread; j++) { + const uint64_t idx = base_idx + j; + if (idx >= total_elements) { + break; + } + + // Index decoding + const int64_t i3 = idx / (M * N * ne2); + const int64_t rem3 = idx % (M * N * ne2); + const int64_t i2 = rem3 / (M * N); + const int64_t rem2 = rem3 % (M * N); + const int64_t n = rem2 / M; + const int64_t m = rem2 % M; + + const int64_t i03 = i3 / r3, i02 = i2 / r2; + const int64_t i13 = (ne13 > 1) ? i3 : 0, i12 = (ne12 > 1) ? i2 : 0; + + float sum = 0.0f; + const float * f32_row = (const float *) ((const char *) src0_data + m * nb01 + i02 * nb02 + i03 * nb03); + + if (src1->type == GGML_TYPE_F32) { + const float * src1_data = (const float *) src1->data; + + for (int64_t kb = 0; kb < K_blocks; kb++) { + const float * b_col_ptr = + (const float *) ((const char *) src1_data + (kb * block_size) * sizeof(float) + n * nb11 + + i12 * nb12 + i13 * nb13); + sum += compute_block_dot_product_f32(&f32_row[kb * block_size], b_col_ptr); + } + + if (K_remainder > 0) { + const int64_t offset = K_blocks * block_size; + const float * b_col_ptr = (const float *) ((const char *) src1_data + offset * sizeof(float) + + n * nb11 + i12 * nb12 + i13 * nb13); + sum += compute_block_dot_product_f32_partial(&f32_row[offset], b_col_ptr, K_remainder); + } + } else { + const uint16_t * src1_data = (const uint16_t *) src1->data; + + for (int64_t kb = 0; kb < K_blocks; kb++) { + const uint16_t * b_col_ptr = + (const uint16_t *) ((const char *) src1_data + (kb * block_size) * sizeof(uint16_t) + n * nb11 + + i12 * nb12 + i13 * nb13); + sum += compute_block_dot_product_f32_f16_partial(&f32_row[kb * block_size], b_col_ptr, block_size); + } + + if (K_remainder > 0) { + const int64_t offset = K_blocks * block_size; + const uint16_t * b_col_ptr = + (const uint16_t *) ((const char *) src1_data + offset * sizeof(uint16_t) + n * nb11 + + i12 * nb12 + i13 * nb13); + sum += compute_block_dot_product_f32_f16_partial(&f32_row[offset], b_col_ptr, K_remainder); + } + } + + // Atomic store for output + volatile float * c_element = + (volatile float *) ((char *) dst_data + m * dst->nb[0] + n * nb1 + i2 * nb2 + i3 * nb3); + atomic_store_f32(c_element, sum); + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c new file mode 100644 index 0000000..b2b61d5 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c @@ -0,0 +1,155 @@ +#include "ggml_tensor.h" +#include "platform.h" +#include "tensor.h" + +#include +#include + +/* + * F32 Matrix Multiply for ET-SoC-1 — TensorFMA32. + * + * K-parallel + interleaved tiles + ring reduce. + * No batched-K yet (needs investigation on hang). + * This is the last known working version. + */ + +#define NUM_COMPUTE_SHIRES 32 +#define MINIONS_PER_SHIRE 32 +#define TILE_K 16 +#define TILE_M 16 + +/* ── Tuning knobs ───────────────────────────────────────────────────── */ +#define TILE_N 16 +#define CACHEOP_MAX 0 +#define REP_RATE 0 + +/* ─────────────────────────────────────────────────────────────────── */ + +int entry_point(struct ggml_et_binary_params * params, void * env) { + uint64_t hart_id = get_hart_id(); + uint64_t shire_id = get_shire_id(); + + if (shire_id >= NUM_COMPUTE_SHIRES) { + return 0; + } + if (hart_id & 1) { + return 0; + } + + uint64_t local_minion = (hart_id >> 1) & 0x1F; + uint64_t my_minion_id = get_minion_id(); + + const int64_t K = params->src0.ne[0]; + const int64_t M = params->src0.ne[1]; + const int64_t N = params->src1.ne[1]; + + const int64_t ne2_0 = params->src0.ne[2], ne3_0 = params->src0.ne[3]; + const int64_t ne2_1 = params->src1.ne[2], ne3_1 = params->src1.ne[3]; + + const int64_t nb1_0 = params->src0.nb[1]; + const int64_t nb2_0 = params->src0.nb[2], nb3_0 = params->src0.nb[3]; + const int64_t nb1_1 = params->src1.nb[1]; + const int64_t nb2_1 = params->src1.nb[2], nb3_1 = params->src1.nb[3]; + const int64_t nb1_d = params->dst.nb[1]; + const int64_t nb2_d = params->dst.nb[2], nb3_d = params->dst.nb[3]; + + const char * src0_base = (const char *) params->src0.data; + const char * src1_base = (const char *) params->src1.data; + char * dst_base = (char *) params->dst.data; + + setup_cache_scp(); +#if CACHEOP_MAX > 0 || REP_RATE > 0 + ucache_control(1, REP_RATE, CACHEOP_MAX); +#endif + CLEAR_TENSOR_ERROR; + + const int64_t m_tiles = M / TILE_M; + const int64_t n_tiles = (N + TILE_N - 1) / TILE_N; + const int64_t batch_count = ne2_1 * ne3_1; + const int64_t base_tiles = m_tiles * n_tiles * batch_count; + + const int64_t r2 = ne2_1 / ne2_0; + const int64_t r3 = ne3_1 / ne3_0; + + const int64_t total_harts = NUM_COMPUTE_SHIRES * MINIONS_PER_SHIRE; + const int64_t k_steps = K / TILE_K; + int64_t k_splits = 1; + if (base_tiles < total_harts) { + k_splits = (total_harts + base_tiles - 1) / base_tiles; + int64_t ks = 1; + while (ks * 2 <= k_splits && ks * 2 <= 32 && k_steps % (ks * 2) == 0) { + ks *= 2; + } + k_splits = ks; + } + + const int64_t tiles_per_shire = MINIONS_PER_SHIRE / k_splits; + const int64_t k_split = local_minion % k_splits; + const int64_t local_tile_idx = local_minion / k_splits; + const int64_t tiles_stride = (int64_t) NUM_COMPUTE_SHIRES * tiles_per_shire; + + const int64_t k_steps_per_split = k_steps / k_splits; + const int64_t k_start = k_split * k_steps_per_split * TILE_K; + const int64_t k_end = k_start + k_steps_per_split * TILE_K; + + const uint64_t group_base_global = my_minion_id - k_split; + + for (int64_t tile = (int64_t) shire_id + local_tile_idx * NUM_COMPUTE_SHIRES; tile < base_tiles; + tile += tiles_stride) { + const int64_t tiles_per_batch = m_tiles * n_tiles; + const int64_t batch_idx = tile / tiles_per_batch; + const int64_t tile_in_batch = tile % tiles_per_batch; + const int64_t nb_idx = tile_in_batch / m_tiles; + const int64_t mb_idx = tile_in_batch % m_tiles; + + const int64_t i3 = batch_idx / ne2_1; + const int64_t i2 = batch_idx % ne2_1; + const int64_t i2_0 = i2 / r2; + const int64_t i3_0 = i3 / r3; + + const char * src0_batch = src0_base + i3_0 * nb3_0 + i2_0 * nb2_0; + const char * src1_batch = src1_base + i3 * nb3_1 + i2 * nb2_1; + char * dst_batch = dst_base + i3 * nb3_d + i2 * nb2_d; + + const int64_t mb = mb_idx * TILE_M; + const int64_t nb = nb_idx * TILE_N; + const int64_t n_cur = (nb + TILE_N <= N) ? TILE_N : (N - nb); + + for (int64_t kb = k_start; kb < k_end; kb += TILE_K) { + tensor_load(false, false, 0, 0, 0, (uint64_t) (src1_batch + nb * nb1_1 + kb * sizeof(float)), 0, n_cur - 1, + (uint64_t) nb1_1, 0); + + tensor_load(false, false, TILE_K, 7, 0, (uint64_t) (src0_batch + mb * nb1_0 + kb * sizeof(float)), 0, + TILE_K - 1, (uint64_t) nb1_0, 1); + + tensor_wait(TENSOR_LOAD_WAIT_0); + tensor_wait(TENSOR_LOAD_WAIT_1); + + tensor_fma(false, 3, n_cur - 1, TILE_K - 1, 0, false, false, false, false, TILE_K, 0, 0, (kb == k_start)); + + tensor_wait(TENSOR_FMA_WAIT); + } + + if (k_splits > 1) { + const uint64_t num_regs = (uint64_t) n_cur * 2; + + if (k_split > 0) { + tensor_reduce_recv(0, TENSOR_REDUCE_OP_FADD, num_regs, group_base_global + k_split - 1); + tensor_wait(TENSOR_REDUCE_WAIT); + } + if (k_split < k_splits - 1) { + tensor_reduce_send(0, num_regs, group_base_global + k_split + 1); + tensor_wait(TENSOR_REDUCE_WAIT); + } + } + + if (k_split == k_splits - 1) { + tensor_store(0, 0, 3, n_cur - 1, (uint64_t) (dst_batch + nb * nb1_d + mb * sizeof(float)), 0, + (uint64_t) nb1_d); + tensor_wait(TENSOR_STORE_WAIT); + } + } + + FENCE; + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c new file mode 100644 index 0000000..3685c25 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c @@ -0,0 +1,169 @@ +//****************************************************************************** +// MUL_MAT_ID kernel specialized for Q4_0 weights (Mixture of Experts). +// +// C[m, s, b] = Sum(k=0..K-1) A[k, m, ids[s,b]] * B[k, s % ne11, b] +// A: Q4_0 [K, M, n_expert] weights +// B: F32 [K, n_cols, batch] activations +// ids: I32 [n_expert_used, batch] +// C: F32 [M, n_expert_used, batch] +// +// Strategy: All harts active. Flat m-major output partition allows amortized +// expert lookups and 2-row x2 dot products. +//****************************************************************************** + +#include "block_ops.h" +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" +#include "quants.h" + +#include + +int entry_point(struct ggml_et_mul_mat_id_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + if (!kernel_env || !params) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + if (thread_id < 0) { + return 0; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * src2 = ¶ms->src2; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_Q4_0 || src1->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32 || + dst->type != GGML_TYPE_F32) { + return -1; + } + + const void * src0_data = src0->data; + const float * src1_data = (const float *) src1->data; + const int32_t * src2_data = (const int32_t *) src2->data; + float * dst_data = (float *) dst->data; + if (!src0_data || !src1_data || !src2_data || !dst_data) { + return -1; + } + + const int64_t K = src0->ne[0]; + const int64_t M = src0->ne[1]; + const int64_t n_expert = src0->ne[2]; + const int64_t n_expert_used = src2->ne[0]; + const int64_t batch = src2->ne[1]; + const int64_t ne11 = src1->ne[1]; + + if (K % QK4_0 != 0) { + return -1; + } + + const size_t nb01 = src0->nb[1]; // Q4_0 row stride + const size_t nb02 = src0->nb[2]; // expert stride + const size_t nb11 = src1->nb[1]; // activation column stride + const size_t nb12 = src1->nb[2]; // activation batch stride + const size_t nb20 = src2->nb[0]; + const size_t nb21 = src2->nb[1]; + const size_t nbd0 = dst->nb[0]; + const size_t nbd1 = dst->nb[1]; + const size_t nbd2 = dst->nb[2]; + + if (src0->nb[0] != sizeof(block_q4_0) || src1->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t) || + nbd0 != sizeof(float)) { + return -1; + } + + const int64_t K_blocks = K / QK4_0; + const int use_x2 = ((nb01 & 31) == 0); + + const uint64_t total_outputs = (uint64_t) M * (uint64_t) n_expert_used * (uint64_t) batch; + if (total_outputs == 0) { + return 0; + } + + // Even partition: hart h owns outputs [h*chunk, (h+1)*chunk). + const uint64_t chunk = (total_outputs + (uint64_t) num_threads - 1) / (uint64_t) num_threads; + const uint64_t my_start = (uint64_t) thread_id * chunk; + if (my_start >= total_outputs) { + return 0; + } + uint64_t my_end = my_start + chunk; + if (my_end > total_outputs) { + my_end = total_outputs; + } + + // Save mask register once; full lanes for vector dot. + q4_dot_state q4_state; + q4_dot_begin(&q4_state); + + const uint64_t per_batch = (uint64_t) M * (uint64_t) n_expert_used; + + uint64_t idx = my_start; + while (idx < my_end) { + // Decode (m, slot, batch) from the m-major linear index. + const int64_t batch_idx = (int64_t) (idx / per_batch); + const uint64_t rem = idx - (uint64_t) batch_idx * per_batch; + const int64_t slot_idx = (int64_t) (rem / (uint64_t) M); + const int64_t m0 = (int64_t) (rem - (uint64_t) slot_idx * (uint64_t) M); + + // How many outputs left in this (slot, batch) run AND in my range. + const uint64_t run_end_global = + (uint64_t) batch_idx * per_batch + (uint64_t) slot_idx * (uint64_t) M + (uint64_t) M; + const uint64_t end_in_my = (run_end_global < my_end) ? run_end_global : my_end; + int64_t run_len = (int64_t) (end_in_my - idx); + + // Resolve expert + B column + dst slot for this run. + const int32_t expert_id = + *(const int32_t *) ((const char *) src2_data + slot_idx * (int64_t) nb20 + batch_idx * (int64_t) nb21); + + char * dst_slot = (char *) dst_data + slot_idx * (int64_t) nbd1 + batch_idx * (int64_t) nbd2; + + if (expert_id < 0 || expert_id >= n_expert) { + // Invalid expert id — zero out this run's outputs. + int64_t m = m0; + for (int64_t i = 0; i < run_len; i++, m++) { + atomic_store_f32((volatile float *) (dst_slot + m * (int64_t) nbd0), 0.0f); + } + idx += (uint64_t) run_len; + continue; + } + + const int64_t col_idx = slot_idx % ne11; + const float * b_col_base = + (const float *) ((const char *) src1_data + col_idx * (int64_t) nb11 + batch_idx * (int64_t) nb12); + const char * expert_base = (const char *) src0_data + expert_id * (int64_t) nb02; + + int64_t m = m0; + int64_t left = run_len; + + // Paired-row dots: halves B bandwidth for runs >= 2. + if (use_x2) { + while (left >= 2) { + const block_q4_0 * row0 = (const block_q4_0 *) (expert_base + m * (int64_t) nb01); + const block_q4_0 * row1 = (const block_q4_0 *) (expert_base + (m + 1) * (int64_t) nb01); + float s0, s1; + q4_dot_compute_x2_aligned(row0, row1, b_col_base, K_blocks, &s0, &s1); + atomic_store_f32((volatile float *) (dst_slot + m * (int64_t) nbd0), s0); + atomic_store_f32((volatile float *) (dst_slot + (m + 1) * (int64_t) nbd0), s1); + m += 2; + left -= 2; + } + } + + // Tail / non-aligned fallback: single-row dots. + while (left > 0) { + const block_q4_0 * row = (const block_q4_0 *) (expert_base + m * (int64_t) nb01); + float s = q4_dot_compute(row, b_col_base, K_blocks); + atomic_store_f32((volatile float *) (dst_slot + m * (int64_t) nbd0), s); + m++; + left--; + } + + idx += (uint64_t) run_len; + } + + q4_dot_end(&q4_state); + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c new file mode 100644 index 0000000..d077a00 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c @@ -0,0 +1,160 @@ +//****************************************************************************** +// MUL_MAT_ID kernel specialized for Q8_0 weights (Mixture of Experts). +// +// C[m, s, b] = Sum(k=0..K-1) A[k, m, ids[s,b]] * B[k, s % ne11, b] +// A: Q8_0 [K, M, n_expert] weights +// B: F32 [K, n_cols, batch] activations +// ids: I32 [n_expert_used, batch] +// C: F32 [M, n_expert_used, batch] +// +// Strategy mirrors mul_mat_id_Q4_0.c. +//****************************************************************************** + +#include "block_ops.h" +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" +#include "quants.h" + +#include + +int entry_point(struct ggml_et_mul_mat_id_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + if (!kernel_env || !params) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + if (thread_id < 0) { + return 0; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * src2 = ¶ms->src2; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_Q8_0 || src1->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32 || + dst->type != GGML_TYPE_F32) { + return -1; + } + + const void * src0_data = src0->data; + const float * src1_data = (const float *) src1->data; + const int32_t * src2_data = (const int32_t *) src2->data; + float * dst_data = (float *) dst->data; + if (!src0_data || !src1_data || !src2_data || !dst_data) { + return -1; + } + + const int64_t K = src0->ne[0]; + const int64_t M = src0->ne[1]; + const int64_t n_expert = src0->ne[2]; + const int64_t n_expert_used = src2->ne[0]; + const int64_t batch = src2->ne[1]; + const int64_t ne11 = src1->ne[1]; + + if (K % QK8_0 != 0) { + return -1; + } + + const size_t nb01 = src0->nb[1]; + const size_t nb02 = src0->nb[2]; + const size_t nb11 = src1->nb[1]; + const size_t nb12 = src1->nb[2]; + const size_t nb20 = src2->nb[0]; + const size_t nb21 = src2->nb[1]; + const size_t nbd0 = dst->nb[0]; + const size_t nbd1 = dst->nb[1]; + const size_t nbd2 = dst->nb[2]; + + if (src0->nb[0] != sizeof(block_q8_0) || src1->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t) || + nbd0 != sizeof(float)) { + return -1; + } + + const int64_t K_blocks = K / QK8_0; + const int use_x2 = ((nb01 & 31) == 0); + + const uint64_t total_outputs = (uint64_t) M * (uint64_t) n_expert_used * (uint64_t) batch; + if (total_outputs == 0) { + return 0; + } + + const uint64_t chunk = (total_outputs + (uint64_t) num_threads - 1) / (uint64_t) num_threads; + const uint64_t my_start = (uint64_t) thread_id * chunk; + if (my_start >= total_outputs) { + return 0; + } + uint64_t my_end = my_start + chunk; + if (my_end > total_outputs) { + my_end = total_outputs; + } + + q8_dot_state q8_state; + q8_dot_begin(&q8_state); + + const uint64_t per_batch = (uint64_t) M * (uint64_t) n_expert_used; + + uint64_t idx = my_start; + while (idx < my_end) { + const int64_t batch_idx = (int64_t) (idx / per_batch); + const uint64_t rem = idx - (uint64_t) batch_idx * per_batch; + const int64_t slot_idx = (int64_t) (rem / (uint64_t) M); + const int64_t m0 = (int64_t) (rem - (uint64_t) slot_idx * (uint64_t) M); + + const uint64_t run_end_global = + (uint64_t) batch_idx * per_batch + (uint64_t) slot_idx * (uint64_t) M + (uint64_t) M; + const uint64_t end_in_my = (run_end_global < my_end) ? run_end_global : my_end; + int64_t run_len = (int64_t) (end_in_my - idx); + + const int32_t expert_id = + *(const int32_t *) ((const char *) src2_data + slot_idx * (int64_t) nb20 + batch_idx * (int64_t) nb21); + + char * dst_slot = (char *) dst_data + slot_idx * (int64_t) nbd1 + batch_idx * (int64_t) nbd2; + + if (expert_id < 0 || expert_id >= n_expert) { + int64_t m = m0; + for (int64_t i = 0; i < run_len; i++, m++) { + atomic_store_f32((volatile float *) (dst_slot + m * (int64_t) nbd0), 0.0f); + } + idx += (uint64_t) run_len; + continue; + } + + const int64_t col_idx = slot_idx % ne11; + const float * b_col_base = + (const float *) ((const char *) src1_data + col_idx * (int64_t) nb11 + batch_idx * (int64_t) nb12); + const char * expert_base = (const char *) src0_data + expert_id * (int64_t) nb02; + + int64_t m = m0; + int64_t left = run_len; + + if (use_x2) { + while (left >= 2) { + const block_q8_0 * row0 = (const block_q8_0 *) (expert_base + m * (int64_t) nb01); + const block_q8_0 * row1 = (const block_q8_0 *) (expert_base + (m + 1) * (int64_t) nb01); + float s0, s1; + q8_dot_compute_x2_aligned(row0, row1, b_col_base, K_blocks, &s0, &s1); + atomic_store_f32((volatile float *) (dst_slot + m * (int64_t) nbd0), s0); + atomic_store_f32((volatile float *) (dst_slot + (m + 1) * (int64_t) nbd0), s1); + m += 2; + left -= 2; + } + } + + while (left > 0) { + const block_q8_0 * row = (const block_q8_0 *) (expert_base + m * (int64_t) nb01); + float s = q8_dot_compute(row, b_col_base, K_blocks); + atomic_store_f32((volatile float *) (dst_slot + m * (int64_t) nbd0), s); + m++; + left--; + } + + idx += (uint64_t) run_len; + } + + q8_dot_end(&q8_state); + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c new file mode 100644 index 0000000..900aa0c --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c @@ -0,0 +1,288 @@ +//****************************************************************************** +// Bare Metal MUL_MAT_ID Kernel (Mixture of Experts) +// +// ALGORITHM: +// MUL_MAT_ID performs batched matrix multiplication with expert routing. +// Each output element selects which expert matrix to use based on an index tensor. +// +// INPUTS: +// src0 (as): Expert weight matrices [K, M, n_expert] +// - Stack of n_expert matrices, each of size [K, M] +// src1 (b): Activation vectors [K, n_cols, batch] +// - n_cols can be 1 (broadcast) or n_expert_used (per-expert inputs) +// src2 (ids): Expert selection indices [n_expert_used, batch] (int32) +// - For each (slot, batch), specifies which expert from src0 to use +// +// OUTPUT: +// dst: Result [M, n_expert_used, batch, 1] +// +// COMPUTATION: +// For each output position (m, slot, batch): +// expert_id = ids[slot, batch] // Which expert to use (0..n_expert-1) +// col_idx = slot % src1.ne[1] // Which column in src1 (handles broadcasting) +// dst[m, slot, batch] = dot_product( +// src0[0:K, m, expert_id], // Row m from selected expert matrix +// src1[0:K, col_idx, batch] // Column from activations (may broadcast) +// ) +// +// BROADCASTING: +// - When src1.ne[1] == 1: All expert slots use the same activation column +// - When src1.ne[1] == n_expert_used: Each slot has its own activation column +// - General case: col_idx = slot % src1.ne[1] (modulo handles both cases) +// +// MATH NOTATION: +// C[m, s, b] = Sum(k=0 to K-1) A[k, m, ids[s,b]] x B[k, s % ne11, b] +// where: +// m: [0, M) - output feature index +// s: [0, n_expert_used) - expert slot index +// b: [0, batch) - batch index +// k: [0, K) - hidden dimension +// ne11 = src1->ne[1] - number of columns in src1 +//****************************************************************************** + +#include "block_ops.h" +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" +#include "quants.h" + +#include + +// Main entry point for MUL_MAT_ID kernel (Mixture of Experts) +int entry_point(struct ggml_et_mul_mat_id_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + // Get thread coordination info + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return -1; + } + + // Use even threads only to avoid resource contention + // Each minion has 2 threads sharing instruction/data cache, NOC to RAM, and FPU + // Odd threads return immediately to avoid fighting for shared resources + if (thread_id & 1) { + return 0; // Odd thread - skip work + } + + // Adjust thread count and ID for even-only threading + int effective_thread_id = thread_id / 2; + int effective_num_threads = (num_threads + 1) / 2; // Ceiling division + + // Validate params + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + // Extract tensor references + struct ggml_tensor * src0 = ¶ms->src0; // Expert weight matrices [K, M, n_expert] + struct ggml_tensor * src1 = ¶ms->src1; // Activations [K, n_expert_used, batch] + struct ggml_tensor * src2 = ¶ms->src2; // Expert indices [n_expert_used, batch] (I32) + struct ggml_tensor * dst = ¶ms->dst; // Output [M, n_expert_used, batch, 1] + + // Validate tensor types + if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) { + return -1; + } + + // Get data pointers + const void * src0_data = src0->data; // Expert matrices (Q8_0/F16/F32) + const float * src1_data = (const float *) src1->data; // Activations (F32) + const int32_t * src2_data = (const int32_t *) src2->data; // Expert IDs (I32) + float * dst_data = (float *) dst->data; // Output (F32) + + if (!src0_data || !src1_data || !src2_data || !dst_data) { + return -1; + } + + // Determine block size based on src0 type + int block_size; + switch (src0->type) { + case GGML_TYPE_Q8_0: + block_size = QK8_0; + break; + case GGML_TYPE_Q4_0: + block_size = QK4_0; + break; + case GGML_TYPE_F16: + block_size = QK_F16; + break; + case GGML_TYPE_F32: + block_size = QK_F32; + break; + default: + return -1; + } + + // Get dimensions + // src0: [K, M, n_expert] - expert weight matrices + // src1: [K, n_expert_used, batch] - activations + // src2: [n_expert_used, batch] - expert indices + // dst: [M, n_expert_used, batch, 1] - output + const int64_t K = src0->ne[0]; // Hidden dimension + const int64_t M = src0->ne[1]; // Output features + const int64_t n_expert = src0->ne[2]; // Number of experts + const int64_t n_expert_used = src2->ne[0]; // Experts used per token + const int64_t batch = src2->ne[1]; // Batch size + + // Strides (in bytes) + const size_t nb01 = src0->nb[1]; // src0 row stride + const size_t nb02 = src0->nb[2]; // src0 expert stride + const size_t nb11 = src1->nb[1]; // src1 column stride + const size_t nb12 = src1->nb[2]; // src1 batch stride + const size_t nb20 = src2->nb[0]; // src2 element stride + const size_t nb21 = src2->nb[1]; // src2 batch stride + const size_t nb1 = dst->nb[1]; // dst column stride + const size_t nb2 = dst->nb[2]; // dst batch stride + + // Verify K dimension alignment for quantization + // Q8_0 requires strict alignment (quantized data must be block-aligned) + // F32 and F16 can handle partial blocks with scalar remainders + if ((src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_Q4_0) && K % block_size != 0) { + return -1; // Q8_0 requires K to be multiple of block_size + } + + // Verify first dimension is contiguous + size_t expected_element_size_src0; + if (src0->type == GGML_TYPE_Q8_0) { + expected_element_size_src0 = sizeof(block_q8_0); + } else if (src0->type == GGML_TYPE_Q4_0) { + expected_element_size_src0 = sizeof(block_q4_0); + } else if (src0->type == GGML_TYPE_F16) { + expected_element_size_src0 = sizeof(uint16_t); + } else if (src0->type == GGML_TYPE_F32) { + expected_element_size_src0 = sizeof(float); + } else { + return -1; + } + + if (src0->nb[0] != expected_element_size_src0 || src1->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t) || + dst->nb[0] != sizeof(float)) { + return -1; + } + + const int64_t K_blocks = K / block_size; + + // Threading: distribute output elements across threads + // Total output elements = M * n_expert_used * batch + const uint64_t total_elements = M * n_expert_used * batch; + + const uint64_t per_thread = 16; + const uint64_t threads_stride = per_thread * effective_num_threads; + + if (effective_thread_id * per_thread >= total_elements) { + return 0; + } + + // Process elements assigned to this thread + for (uint64_t base_idx = effective_thread_id * per_thread; base_idx < total_elements; base_idx += threads_stride) { + for (uint64_t j = 0; j < per_thread; j++) { + const uint64_t idx = base_idx + j; + + if (idx >= total_elements) { + break; + } + + // Decode linear index to (m, n_idx, batch_idx) + // Layout: m + M * (n_idx + n_expert_used * batch_idx) + const int64_t batch_idx = idx / (M * n_expert_used); + const int64_t rem = idx % (M * n_expert_used); + const int64_t n_idx = rem / M; + const int64_t m = rem % M; + + // Get expert ID from src2[n_idx, batch_idx] + const int32_t expert_id = *(const int32_t *) ((const char *) src2_data + n_idx * nb20 + batch_idx * nb21); + + // Validate expert ID + if (expert_id < 0 || expert_id >= n_expert) { + // Invalid expert ID - write zero and continue + volatile float * dst_element = + (volatile float *) ((char *) dst_data + m * dst->nb[0] + n_idx * nb1 + batch_idx * nb2); + atomic_store_f32(dst_element, 0.0f); + continue; + } + + // Compute dot product: expert_matrix[m, :] x activations[:, col_idx, batch_idx] + // Use modulo to handle broadcasting: when src1 has fewer columns than expert slots, + // multiple slots share the same activation column (col_idx = n_idx % src1->ne[1]) + const int64_t col_idx = n_idx % src1->ne[1]; + float sum = 0.0f; + + // Type switch hoisted outside block loop: one branch per element, not per block + const char * expert_row_base = (const char *) src0_data + m * nb01 + expert_id * nb02; + + switch (src0->type) { + case GGML_TYPE_Q8_0: + { + const block_q8_0 * q8_row = (const block_q8_0 *) expert_row_base; + const float * b_col_base = + (const float *) ((const char *) src1_data + col_idx * nb11 + batch_idx * nb12); + sum += compute_row_dot_q8_0(q8_row, b_col_base, K_blocks); + break; + } + case GGML_TYPE_Q4_0: + { + const block_q4_0 * q4_row = (const block_q4_0 *) expert_row_base; + const float * b_col_base = + (const float *) ((const char *) src1_data + col_idx * nb11 + batch_idx * nb12); + sum += compute_row_dot_q4_0(q4_row, b_col_base, K_blocks); + break; + } + case GGML_TYPE_F16: + { + const uint16_t * f16_row = (const uint16_t *) expert_row_base; + const int64_t K_remainder = K % block_size; + for (int64_t kb = 0; kb < K_blocks; kb++) { + const float * b_col_ptr = + (const float *) ((const char *) src1_data + (kb * block_size) * sizeof(float) + + col_idx * nb11 + batch_idx * nb12); + sum += compute_block_dot_product_f16_naive(&f16_row[kb * block_size], b_col_ptr); + } + if (K_remainder > 0) { + const int64_t offset = K_blocks * block_size; + const float * b_col_ptr = + (const float *) ((const char *) src1_data + offset * sizeof(float) + col_idx * nb11 + + batch_idx * nb12); + sum += compute_block_dot_product_f16_partial(&f16_row[offset], b_col_ptr, K_remainder); + } + break; + } + case GGML_TYPE_F32: + { + const float * f32_row = (const float *) expert_row_base; + const int64_t K_remainder = K % block_size; + for (int64_t kb = 0; kb < K_blocks; kb++) { + const float * b_col_ptr = + (const float *) ((const char *) src1_data + (kb * block_size) * sizeof(float) + + col_idx * nb11 + batch_idx * nb12); + sum += compute_block_dot_product_f32(&f32_row[kb * block_size], b_col_ptr); + } + if (K_remainder > 0) { + const int64_t offset = K_blocks * block_size; + const float * b_col_ptr = + (const float *) ((const char *) src1_data + offset * sizeof(float) + col_idx * nb11 + + batch_idx * nb12); + sum += compute_block_dot_product_f32_partial(&f32_row[offset], b_col_ptr, K_remainder); + } + break; + } + default: + return -1; + } + + // Store result using atomic store to avoid cache coherency issues + // when multiple threads write to the same cache line (64 bytes = 16 floats) + volatile float * dst_element = + (volatile float *) ((char *) dst_data + m * dst->nb[0] + n_idx * nb1 + batch_idx * nb2); + atomic_store_f32(dst_element, sum); + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/norm_f32.c b/ggml/src/ggml-et/et-kernels/src/norm_f32.c new file mode 100644 index 0000000..f172b6d --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/norm_f32.c @@ -0,0 +1,328 @@ +//****************************************************************************** +// Norm F32 Kernel (Layer Normalization) +// y[i] = (x[i] - mean) / sqrt(variance + eps) +// where mean = sum(x) / N, variance = sum((x - mean)^2) / N +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include +#include +#include + +// Norm kernel parameters structure +struct ggml_et_norm_params { + struct ggml_tensor src0; // F32 input tensor + struct ggml_tensor dst; // F32 output tensor + float eps; // Epsilon parameter for numerical stability +}; + +int entry_point(struct ggml_et_norm_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + float eps = params->eps; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; // Unsupported type combination + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; // Null data pointer + } + + + if (eps < 0.0f) { + return -1; // Invalid epsilon + } + + const int64_t ne0 = dst->ne[0]; + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t ne3 = dst->ne[3]; + + const size_t nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + const size_t nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + + if (src0->ne[0] != ne0 || src0->ne[1] != ne1 || src0->ne[2] != ne2 || src0->ne[3] != ne3) { + return -1; // Shape mismatch + } + + const int32_t total_rows = (int32_t) (ne1 * ne2 * ne3); + const int shire_threads = SOC_MINIONS_PER_SHIRE * NUM_HARTS_PER_MINION; + + if (total_rows >= shire_threads) { + // Row-parallel: each thread processes whole rows + for (int64_t i3 = 0; i3 < ne3; i3++) { + for (int64_t i2 = 0; i2 < ne2; i2++) { + for (int64_t i1 = thread_id; i1 < ne1; i1 += num_threads) { + const float * src_ptr = + (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_ptr = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + + // Step 1: sum for mean + float zero = 0.0f; + __asm__ volatile("fbc.ps f10, %[z]\n" : : [z] "m"(zero) : "f10"); + + for (int32_t i0 = 0; i0 < (int32_t) ne0; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fadd.ps f10, f10, f11\n" + : + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11"); + } + + float sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + + const float mean = et_fdiv(sum, (float) (int32_t) ne0); + + // Step 2: compute (x - mean) → dst, accumulate variance + __asm__ volatile("fbc.ps f10, %[z]\n" : : [z] "m"(zero) : "f10"); + + for (int32_t i0 = 0; i0 < (int32_t) ne0; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fbc.ps f12, %[mean_ptr]\n" + "fsub.ps f13, f11, f12\n" + "fsw.ps f13, %[result]\n" + "fmadd.ps f10, f13, f13, f10\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]), [mean_ptr] "m"(mean) + : "f10", "f11", "f12", "f13"); + } + + float var_sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(var_sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + + const float variance = et_fdiv(var_sum, (float) (int32_t) ne0); + const float scale = et_powf(variance + eps, -0.5f); + + if (!(scale > 0.0f)) { + return -1; + } + + // Step 3: apply scale to centered values in dst + for (int32_t i0 = 0; i0 < (int32_t) ne0; i0 += 8) { + __asm__ volatile( + "flw.ps f12, %[y_vec]\n" + "fbc.ps f13, %[scale_ptr]\n" + "fmul.ps f14, f12, f13\n" + "fsw.ps f14, %[result]\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [y_vec] "m"(*(const float (*)[8]) & dst_ptr[i0]), [scale_ptr] "m"(scale) + : "f12", "f13", "f14"); + } + } + } + } + } else { + // Intra-row: threads within each shire cooperate via L2 SCP. + // Two reductions needed: sum (for mean), then variance sum. + int shire_tid = thread_id % shire_threads; + int threads_per_row = shire_threads / total_rows; + int my_row = shire_tid / threads_per_row; + int local_tid = shire_tid % threads_per_row; + int group_base = my_row * threads_per_row; + + if (my_row >= total_rows) { + FENCE; + et_barrier(ET_BARRIER_SHIRE); + // Second barrier for variance exchange + FENCE; + et_barrier(ET_BARRIER_SHIRE); + return 0; + } + + int64_t i1 = my_row % ne1; + int64_t i2 = (my_row / ne1) % ne2; + int64_t i3 = my_row / (ne1 * ne2); + + const float * src_ptr = (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_ptr = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + + const int32_t elems_per_cl = 16; + int32_t total_cls = ((int32_t) ne0 + elems_per_cl - 1) / elems_per_cl; + int32_t cls_per_thread = (total_cls + threads_per_row - 1) / threads_per_row; + int32_t my_start = local_tid * cls_per_thread * elems_per_cl; + int32_t my_end = my_start + cls_per_thread * elems_per_cl; + if (my_end > (int32_t) ne0) { + my_end = (int32_t) ne0; + } + if (my_start >= (int32_t) ne0) { + my_start = 0; + my_end = 0; + } + + int workers = threads_per_row < total_cls ? threads_per_row : total_cls; + + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + // ---- Reduction 1: partial sum for mean ---- + __asm__ volatile("fbci.pi f10, 0" ::: "f10"); + for (int32_t i0 = my_start; i0 < my_end; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fadd.ps f10, f10, f11\n" + : + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11"); + } + + float partial_sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(partial_sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + + // L2SCP exchange for sum + volatile float * my_slot = (volatile float *) et_shire_l2scp_local((uint64_t) shire_tid * 64); + *my_slot = partial_sum; + FENCE; + evict_to_l2((const void *) my_slot, 1, 64); + WAIT_CACHEOPS; + + et_barrier(ET_BARRIER_SHIRE); + + // All threads read sum, compute mean + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + evict_to_l2((const void *) slot, 1, 64); + } + WAIT_CACHEOPS; + + float total_sum = 0.0f; + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + total_sum += *slot; + } + + const float mean = et_fdiv(total_sum, (float) (int32_t) ne0); + + // ---- Reduction 2: compute (x - mean) → dst chunk, partial variance ---- + __asm__ volatile("fbci.pi f10, 0" ::: "f10"); + + if (my_start < my_end) { + uint32_t mean_bits; + __asm__ volatile("fmv.x.s %0, %1" : "=r"(mean_bits) : "f"(mean)); + __asm__ volatile("fbcx.ps f15, %[mb]\n" : : [mb] "r"(mean_bits) : "f15"); + + for (int32_t i0 = my_start; i0 < my_end; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fsub.ps f13, f11, f15\n" + "fsw.ps f13, %[result]\n" + "fmadd.ps f10, f13, f13, f10\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11", "f13"); + } + } + + float partial_var; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(partial_var)::"t0", "f1", "f2", "f3", "f4", "f5"); + + // L2SCP exchange for variance (reuse same slots) + *my_slot = partial_var; + FENCE; + evict_to_l2((const void *) my_slot, 1, 64); + WAIT_CACHEOPS; + + et_barrier(ET_BARRIER_SHIRE); + + // All threads read variance, compute scale, apply to own chunk + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + evict_to_l2((const void *) slot, 1, 64); + } + WAIT_CACHEOPS; + + float total_var = 0.0f; + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + total_var += *slot; + } + + const float variance = et_fdiv(total_var, (float) (int32_t) ne0); + const float scale = et_powf(variance + eps, -0.5f); + + if (!(scale > 0.0f)) { + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return -1; + } + + // Apply scale to centered values (already in dst from reduction 2) + if (my_start < my_end) { + uint32_t scale_bits; + __asm__ volatile("fmv.x.s %0, %1" : "=r"(scale_bits) : "f"(scale)); + __asm__ volatile("fbcx.ps f13, %[sb]\n" : : [sb] "r"(scale_bits) : "f13"); + + for (int32_t i0 = my_start; i0 < my_end; i0 += 8) { + __asm__ volatile( + "flw.ps f12, %[y_vec]\n" + "fmul.ps f14, f12, f13\n" + "fsw.ps f14, %[result]\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [y_vec] "m"(*(const float (*)[8]) & dst_ptr[i0]) + : "f12", "f14"); + } + } + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/pad_f32.c b/ggml/src/ggml-et/et-kernels/src/pad_f32.c new file mode 100644 index 0000000..085336f --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/pad_f32.c @@ -0,0 +1,165 @@ +//****************************************************************************** +// Bare Metal PAD F32 Kernel +// Zero-pads an F32 tensor along dimensions 1-3. +// +// Constraints: +// - No dim0 padding (lp[0]==0, rp[0]==0) +// - dst contiguous +// - src nb[0] == 4 (dim0 contiguous for vectorized reads) +// - Zero-pad only (no circular mode) +// +// Two paths: +// Aligned (ne0 % 16 == 0): rows distributed across harts, vectorized. +// Small (16 % ne0 == 0): cache-line distributed, scalar per-element. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_pad_params { + struct ggml_tensor src0; + struct ggml_tensor dst; + int32_t lp[4]; + int32_t rp[4]; +}; + +// Vectorized copy with scalar tail +static inline void vec_copy_f32(float * dst, const float * src, int32_t n) { + int32_t i = 0; + const int32_t vec_end = (n / 8) * 8; + for (; i < vec_end; i += 8) { + __asm__ volatile( + "flw.ps f10, %[s]\n" + "fsw.ps f10, %[d]\n" + : [d] "=m"(*(float (*)[8]) & dst[i]) + : [s] "m"(*(const float (*)[8]) & src[i]) + : "f10"); + } + for (; i < n; i++) { + dst[i] = src[i]; + } +} + +int entry_point(struct ggml_et_pad_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + const float * src0_data = (const float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + // Dst dimensions + const int64_t ne0 = dst->ne[0]; + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t ne3 = dst->ne[3]; + + // Src strides (byte offsets) + const int64_t nb1_src = src0->nb[1]; + const int64_t nb2_src = src0->nb[2]; + const int64_t nb3_src = src0->nb[3]; + + // Padding values + const int32_t lp1 = params->lp[1]; + const int32_t rp1 = params->rp[1]; + const int32_t lp2 = params->lp[2]; + const int32_t rp2 = params->rp[2]; + const int32_t lp3 = params->lp[3]; + const int32_t rp3 = params->rp[3]; + + const int64_t total_rows = ne1 * ne2 * ne3; + const int64_t total_elements = ne0 * total_rows; + + if (total_elements == 0) { + return 0; + } + + // Broadcast 0.0f to SIMD register for vectorized zero-fill + float zero = 0.0f; + __asm__ volatile("fbc.ps f12, %[v]\n" : : [v] "m"(zero) : "f12"); + + // Aligned: ne0 % 16 == 0 -> row-based distribution, vectorized + if (ne0 % 16 == 0) { + for (int64_t row = thread_id; row < total_rows; row += num_threads) { + const int64_t i3 = row / (ne1 * ne2); + const int64_t i2 = (row / ne1) % ne2; + const int64_t i1 = row % ne1; + + float * dst_row = dst_data + row * ne0; + + if (i1 >= lp1 && i1 < ne1 - rp1 && i2 >= lp2 && i2 < ne2 - rp2 && i3 >= lp3 && i3 < ne3 - rp3) { + const float * src_row = (const float *) ((const char *) src0_data + (i1 - lp1) * nb1_src + + (i2 - lp2) * nb2_src + (i3 - lp3) * nb3_src); + vec_copy_f32(dst_row, src_row, (int32_t) ne0); + } else { + int64_t i = 0; + const int64_t vec_end = (ne0 / 8) * 8; + for (; i < vec_end; i += 8) { + __asm__ volatile("fsw.ps f12, %[d]\n" : [d] "=m"(*(float (*)[8]) & dst_row[i])::"f12"); + } + } + } + return 0; + } + + // Small-ne0 path: 16 % ne0 == 0 -> cache-line distributed, scalar + const int64_t elems_per_cl = 16; + const int64_t total_cl = (total_elements + elems_per_cl - 1) / elems_per_cl; + + const int64_t ne1_data_end = ne1 - rp1; + const int64_t ne2_data_end = ne2 - rp2; + const int64_t ne3_data_end = ne3 - rp3; + + for (int64_t cl = thread_id; cl < total_cl; cl += num_threads) { + const int64_t elem_start = cl * elems_per_cl; + int64_t elem_end = elem_start + elems_per_cl; + if (elem_end > total_elements) { + elem_end = total_elements; + } + + for (int64_t idx = elem_start; idx < elem_end; idx++) { + const int64_t i0 = idx % ne0; + const int64_t rem = idx / ne0; + const int64_t i1 = rem % ne1; + const int64_t rem2 = rem / ne1; + const int64_t i2 = rem2 % ne2; + const int64_t i3 = rem2 / ne2; + + if (i1 >= lp1 && i1 < ne1_data_end && i2 >= lp2 && i2 < ne2_data_end && i3 >= lp3 && i3 < ne3_data_end) { + const float * sp = (const float *) ((const char *) src0_data + i0 * 4 + (i1 - lp1) * nb1_src + + (i2 - lp2) * nb2_src + (i3 - lp3) * nb3_src); + dst_data[idx] = *sp; + } else { + dst_data[idx] = 0.0f; + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h new file mode 100644 index 0000000..cbec4c9 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -0,0 +1,545 @@ +//****************************************************************************** +// ET Platform Hardware Abstraction Layer +// Provides thread coordination, kernel infrastructure, and platform primitives +// for bare metal ET kernels +//****************************************************************************** + +#ifndef PLATFORM_H +#define PLATFORM_H + +#include "etsoc/common/utils.h" +#include "etsoc/isa/barriers.h" +#include "etsoc/isa/cacheops-umode.h" +#include "etsoc/isa/hart.h" + +#include + +#define SOC_MINIONS_PER_SHIRE 32 +#define NUM_HARTS_PER_MINION 2 +#define ET_CACHE_LINE_SIZE_BYTES 64 + +// Environment structure definition +typedef struct { + uint32_t version; // Version of the ABI (offset 0) + uint32_t padding1; // Padding to align shire_mask to offset 8 + uint64_t shire_mask; // Bitmask of active compute shires (offset 8) + uint32_t frequency; // Frequency of Minion cores in MHz (offset 16) + uint32_t padding2; // Padding to maintain alignment +} __attribute__((packed, aligned(64))) kernel_environment_t; + +// Manual implementation of count trailing zeros for bare metal environment +// NOTE: This simple loop-based implementation is used for portability. +// Production implementations (like libgcc's __ctzdi2) use optimized bit manipulation +// algorithms with lookup tables and parallel bit operations for O(log n) performance. +static inline int manual_ctzll(uint64_t x) { + if (x == 0) return 64; + int count = 0; + while ((x & 1) == 0) { + x >>= 1; + count++; + } + return count; +} + +// Manual implementation of population count for bare metal environment +// NOTE: This simple loop-based implementation is used for portability. +// Production implementations (like libgcc's __popcountdi2) use optimized bit-parallel +// algorithms with magic constants and bit manipulation tricks for O(1) performance. +static inline int manual_popcountll(uint64_t x) { + int count = 0; + while (x) { + count += x & 1; + x >>= 1; + } + return count; +} + +// Binary GCD (Stein's algorithm) — avoids expensive 64-bit division/remainder. +// Uses only shifts, subtraction, and comparison (all single-cycle on ET cores). +static inline int64_t et_gcd_i64(int64_t a, int64_t b) { + while (b) { + const int64_t t = b; + b = a % b; + a = t; + } + return a; +} + +// Return the number of consecutive rows of width row_elems needed so the +// combined write footprint spans an integer number of cache lines. +static inline int64_t et_rows_per_cacheline_group(int64_t row_elems, int64_t elem_size_bytes) { + if (row_elems <= 0 || elem_size_bytes <= 0) { + return 1; + } + + const int64_t row_bytes = row_elems * elem_size_bytes; + const int64_t gcd = et_gcd_i64(ET_CACHE_LINE_SIZE_BYTES, row_bytes); + return ET_CACHE_LINE_SIZE_BYTES / gcd; +} + +// Calculate relative thread ID from absolute hart ID using shire mask +// Returns -1 if this hart is not active (not in shire mask) +static inline int get_relative_thread_id(uint64_t shire_mask) { + int hart_id = (int) get_hart_id(); + + // Find starting hart offset from lowest active shire + int starting_hart = manual_ctzll(shire_mask) * SOC_MINIONS_PER_SHIRE * NUM_HARTS_PER_MINION; + + // Return -1 if not an active thread + if (hart_id < starting_hart) { + return -1; + } + + // Calculate relative thread ID + int thread_id = hart_id - starting_hart; + return thread_id; +} + +// Calculate total number of threads from shire mask +static inline int get_num_threads(uint64_t shire_mask) { + // Count active shires using popcount, multiply by minions per shire and harts per minion + return manual_popcountll(shire_mask) * SOC_MINIONS_PER_SHIRE * NUM_HARTS_PER_MINION; +} + +//****************************************************************************** +// Synchronization Primitives +//****************************************************************************** + +#define NOP __asm__ __volatile__("nop\n"); +#define FENCE __asm__ __volatile__("fence\n" ::: "memory"); +#define WFI __asm__ __volatile__("wfi\n"); + +//****************************************************************************** +// Atomic Operations +//****************************************************************************** + +// Global AMO primitives — ET custom 'g' suffix instructions that go through +// the NoC coherence fabric for chip-wide atomicity. + +// Atomic swap (word), returns previous value. +static inline uint32_t __attribute__((always_inline)) et_global_swap_w(volatile void * addr, uint32_t val) { + uint32_t ret; + __asm__ __volatile__("amoswapg.w %0, %1, (%2)" : "=r"(ret) : "r"(val), "r"(addr) : "memory"); + return ret; +} + +// Atomic add (word), returns previous value. +static inline uint32_t __attribute__((always_inline)) et_global_add_w(volatile void * addr, uint32_t val) { + uint32_t ret; + __asm__ __volatile__("amoaddg.w %0, %1, (%2)" : "=r"(ret) : "r"(val), "r"(addr) : "memory"); + return ret; +} + +// Atomic store (halfword, global). Address must be 16-bit aligned. +static inline void __attribute__((always_inline)) et_global_store_hw(volatile void * addr, uint16_t val) { + __asm__ __volatile__("shg %0, (%1)" : : "r"(val), "r"(addr) : "memory"); +} + +// Convenience wrappers — float types, fire-and-forget (old value discarded). +static inline void atomic_store_f32(volatile float * addr, float value) { + et_global_swap_w(addr, *(uint32_t *) &value); +} + +static inline void atomic_add_f32(volatile float * addr, float value) { + et_global_add_w(addr, *(uint32_t *) &value); +} + +static inline void atomic_store_f16(volatile uint16_t * addr, uint16_t value) { + et_global_store_hw(addr, value); +} + +//****************************************************************************** +// Barrier Primitives +// +// Hardware resources used (per shire): +// - 32 FLBs: 8-bit atomic counters, non-blocking (CSR 0x820) +// - 2 FCCs per hart: credit counters, hardware-stall on consume (CSR 0x821) +// +// Convention: +// MINION barriers: FLB = local_minion_id (0-31), FCC 0 +// SHIRE barriers: FLB 0, FCC 1 +// +// MINION and SHIRE barriers MUST NOT be concurrent. All minion barriers +// must complete before a shire barrier, and vice versa. FLB 0 is shared +// between minion 0's barrier and the shire barrier — safe only because +// the FLB counter auto-resets on match. +// +// FCC 0 is safe for all 32 concurrent minion barriers because each +// barrier's fcc_send targets only its own minion (per-hart private +// counters, scoped by CREDINC mask). FCC 1 is reserved for shire-wide +// broadcast. +//****************************************************************************** + +#define ET_DEFAULT_SHIRE_MASK 0xFFFFFFFFULL + +typedef enum { + ET_BARRIER_MINION, // sync both harts within each minion (FLB=minion_id, FCC 0) + ET_BARRIER_SHIRE, // sync all harts across the shire (FLB=0, FCC 1) + ET_BARRIER_GLOBAL, // sync all harts across all active shires (FLB+global AMO+FCC) +} et_barrier_scope_t; + +//****************************************************************************** +// Global Barrier (cross-shire) +// +// Synchronizes all harts across multiple shires on the chip. +// Algorithm: +// 1. FLB within each shire to elect one representative hart +// 2. Elected hart does a global atomic increment on a shared counter +// 3. The last shire to arrive resets the counter and sends FCC credits +// to all active shires to release them +// 4. All harts wait on FCC to complete the barrier +// +// Uses FLB 0, FCC 1 (same as ET_BARRIER_SHIRE, these must not overlap). +// The counter lives in a cache-line-aligned global to avoid coherency problems +//****************************************************************************** + +// Barrier counter cache-line aligned to avoid coherency problems +// Must be zero-initialized (BSS). +static uint32_t __attribute__((aligned(64))) et_global_barrier_count[64 / sizeof(uint32_t)] = { 0 }; + +// Cross-shire barrier: all harts in num_active_shires shires synchronize. +// Returns 1 if this hart was the globally-last to arrive, 0 otherwise. +// +// num_active_shires - number of shires participating +// (typically popcount(shire_mask) from kernel_environment_t) +static inline uint64_t __attribute__((always_inline)) et_barrier_global(uint64_t num_active_shires) { + uint64_t last_global = 0; + + // FLB within this shire. Elect one hart per shire. + // Master shire has only 16 minions (32 harts), others have 32 (64 harts). + uint64_t shire_id = get_shire_id(); + uint32_t harts_in_shire = (shire_id == SHIRE_MASTER) ? (SOC_MINIONS_PER_SHIRE / 2) * NUM_HARTS_PER_MINION : + SOC_MINIONS_PER_SHIRE * NUM_HARTS_PER_MINION; + uint64_t last_in_shire = flbarrier(0, harts_in_shire - 1); + + if (last_in_shire) { + // Global atomic increment. Count arriving shires + uint32_t prev = et_global_add_w(et_global_barrier_count, 1); + + if (prev == num_active_shires - 1) { + // Last shire. reset counter and fan out FCC to all shires + last_global = 1; + et_global_swap_w(et_global_barrier_count, 0); + + for (uint64_t sid = 0; sid < 33; sid++) { + // Send FCC 1 credit to all harts (both threads) in each shire + fcc_send(sid, THREAD_0, FCC_1, 0xFFFFFFFF); + fcc_send(sid, THREAD_1, FCC_1, 0xFFFFFFFF); + } + } + } + + // All harts wait for the FCC credit from the last shire + fcc_consume(FCC_1); + return last_global; +} + +// Barrier with scope-derived parameters. +// Returns 1 if this hart was the last to arrive, 0 otherwise. +// +// ET_BARRIER_GLOBAL uses ET_DEFAULT_SHIRE_MASK (32 shires). For a different +// shire count, use et_barrier_global(n) directly. +static inline uint64_t __attribute__((always_inline)) et_barrier(et_barrier_scope_t scope) { + if (scope == ET_BARRIER_MINION) { + uint32_t local_minion = (get_hart_id() >> 1) & 0x1F; + uint32_t mask = 1u << local_minion; + return shire_barrier(local_minion, 0, 2, mask, mask); + } else if (scope == ET_BARRIER_SHIRE) { + uint64_t shire_id = get_shire_id(); + uint32_t thread_count = (shire_id == SHIRE_MASTER) ? 32 : 64; + uint32_t mask = (shire_id == SHIRE_MASTER) ? 0xFFFF0000U : 0xFFFFFFFFU; + return shire_barrier(0, 1, thread_count, mask, mask); + } else { /* ET_BARRIER_GLOBAL */ + return et_barrier_global(manual_popcountll(ET_DEFAULT_SHIRE_MASK)); + } +} + +// Raw barrier — caller manages FLB/FCC allocation. +// Use when et_barrier() doesn't fit (custom thread counts, subgroups, +// only even harts active, etc). +// +// flb - which FLB counter (0-31) +// fcc - which FCC counter (0 or 1) +// thread_count - number of harts that will call this barrier +// mask_t0 - CREDINC bitmask: which minions' hart 0 gets a credit +// mask_t1 - CREDINC bitmask: which minions' hart 1 gets a credit +static inline uint64_t __attribute__((always_inline)) et_barrier_raw(uint32_t flb, + uint32_t fcc, + uint32_t thread_count, + uint32_t mask_t0, + uint32_t mask_t1) { + return shire_barrier(flb, fcc, thread_count, mask_t0, mask_t1); +} + +// One-way semaphore between harts (non-blocking post, blocking wait). +// +// et_sem_post(): increment the partner hart's semaphore. Non-blocking. +// the caller continues immediately. Multiple posts accumulate. +// +// et_sem_wait(): block until the semaphore is non-zero, then decrement it. +// +// Backed by hardware FCC (Flow Control Credit) counters. Uses FCC 0 for +// ET_BARRIER_MINION scope. Counters are per-hart private, so both harts +// can post/wait on the same scope independently. +// +// Must not be mixed with et_barrier() of the same scope in the +// same kernel (shared FCC channel). +static inline void __attribute__((always_inline)) et_sem_post(et_barrier_scope_t scope) { + if (scope == ET_BARRIER_MINION) { + uint64_t hart_id = get_hart_id(); + uint32_t local_minion = (hart_id >> 1) & 0x1F; + uint32_t mask = 1u << local_minion; + uint64_t shire_id = get_shire_id(); + + if (hart_id & 1) { + // Hart 1 → hart 0 + fcc_send(shire_id, THREAD_0, FCC_0, mask); + } else { + // Hart 0 → hart 1 + fcc_send(shire_id, THREAD_1, FCC_0, mask); + } + } +} + +// Block until a post from et_sem_post() is available, then consume it. +static inline void __attribute__((always_inline)) et_sem_wait(et_barrier_scope_t scope) { + if (scope == ET_BARRIER_MINION) { + fcc_consume(FCC_0); + } +} + +//****************************************************************************** +// Tensor Engine Wait & Error Macros +// +// These write to CSR 0x830 (tensor_wait) to stall the hart until the specified +// tensor unit completes its current operation. The immediate encodes which +// unit to wait on. +//****************************************************************************** + +#define WAIT_TENSOR_LOAD_0 __asm__ __volatile__("csrwi 0x830, 0\n" : :); +#define WAIT_TENSOR_LOAD_1 __asm__ __volatile__("csrwi 0x830, 1\n" : :); +#define WAIT_TENSOR_LOAD_L2_0 __asm__ __volatile__("csrwi 0x830, 2\n" : :); +#define WAIT_TENSOR_LOAD_L2_1 __asm__ __volatile__("csrwi 0x830, 3\n" : :); +#define WAIT_PREFETCH_0 __asm__ __volatile__("csrwi 0x830, 4\n" : :); +#define WAIT_PREFETCH_1 __asm__ __volatile__("csrwi 0x830, 5\n" : :); +#define WAIT_CACHEOPS __asm__ __volatile__("csrwi 0x830, 6\n" : :); +#define WAIT_TENSOR_FMA __asm__ __volatile__("csrwi 0x830, 7\n" : :); +#define WAIT_TENSOR_STORE __asm__ __volatile__("csrwi 0x830, 8\n" : :); +#define WAIT_TENSOR_REDUCE __asm__ __volatile__("csrwi 0x830, 9\n" : :); +#define WAIT_TENSOR_QUANT __asm__ __volatile__("csrwi 0x830, 10\n" : :); +#define STALL __asm__ __volatile__("csrw stall, x0\n" : :); + +// Write 0 to CSR 0x808 (tensor_error) to clear any latched tensor error bits. +// Must be issued before the first tensor operation in a kernel to avoid stale +// errors from a previous invocation causing spurious faults. +#define CLEAR_TENSOR_ERROR __asm__ __volatile__("csrwi 0x808, 0" : :); + +//****************************************************************************** +// L1 Data Cache / Scratchpad (SCP) Configuration +// +// The ET-SoC-1 L1 data cache can be split so that half its ways operate as a +// software-managed scratchpad (SCP). Tensor load/store/FMA instructions +// require SCP mode to be active. +// +// CSR 0x810 — ucache_control: +// +// Bit(s) Field Description +// ────── ──────────── ────────────────────────────────────────────────── +// [0] D1Split 1 = L1 is split (half cache, half SCP). +// Read-only from U-mode; set by M-mode firmware +// before kernel launch. Writing ScpEnable while +// D1Split=0 is silently ignored. +// [1] ScpEnable 1 = scratchpad is active and zeroed. +// [4:2] RepRate Cache-op replay rate (0 = no delay between ops). +// [10:6] CacheOpMax Max outstanding cache ops (0 = unlimited). +// +// Typical kernel prologue for tensor operations: +// setup_cache_scp(); // enables SCP, waits for zeroing +// CLEAR_TENSOR_ERROR; // clear stale error bits +//****************************************************************************** + +// Write the ucache_control CSR (0x810). +// +// scp_en — 1 to enable SCP mode (requires D1Split already set) +// cacheop_rate — cache-op replay rate (0–7; 0 = no delay) +// cacheop_max — max outstanding cache ops (0–31; 0 = unlimited) +static inline void __attribute__((always_inline)) ucache_control(uint64_t scp_en, + uint64_t cacheop_rate, + uint64_t cacheop_max) { + uint64_t csr_enc = ((cacheop_max & 0x1F) << 6) | ((cacheop_rate & 0x7) << 2) | ((scp_en & 0x1) << 1); + + __asm__ __volatile__("csrw 0x810, %[csr_enc]\n" : : [csr_enc] "r"(csr_enc) : "x31"); +} + +// Enable L1 scratchpad mode and wait for the transition to complete. +// After this call the SCP lines are zeroed and ready for tensor operations. +// +// Prerequisites: +// - D1Split must already be 1 (set by M-mode firmware at boot). +// - Only even harts (hart 0 per minion) should call this, as only they +// can issue tensor instructions. +static inline void setup_cache_scp(void) { + FENCE; // drain pending stores before reconfiguring cache + ucache_control(1, 0, 0); // ScpEnable=1 + WAIT_CACHEOPS; // wait for SCP mode transition + zeroing +} + +//****************************************************************************** +// L2 Scratchpad (L2 SCP) Address Computation +// +// Each shire has 4 MB of SRAM that can be split across L2 cache, L3 cache, +// and scratchpad. The scratchpad region occupies 0x00_8000_0000~0x00_FFFF_FFFF +// and is accessible via regular load/store from any minion core. +// +// Two addressing formats (differentiated by address bit 30): +// +// Format 0 (bit[30]=0): Direct shire addressing +// [29:23] = shire ID (0–33, or 0x7F for local shire) +// [22:0] = byte offset within shire's scratchpad +// +// Format 1 (bit[30]=1): Striped (round-robin) addressing +// [29:28] = shire ID[6:5] +// [27:11] = offset[22:6] (cache-line-aligned upper bits) +// [10:6] = shire ID[4:0] +// [5:0] = offset[5:0] (byte within cache line) +// Consecutive 64-byte cache lines cycle through different shires, +// distributing bandwidth across the mesh. +// +// Shire ID 0x7F always targets the local shire (instead of figureing out which +// shire you are on). +//****************************************************************************** + +#define L2SCP_BASE 0x0080000000ULL +#define L2SCP_SHIRE_LOCAL 0x7FULL + +// Format 0: direct address into a specific shire's L2 SCP. +// shire: 0–33 for explicit shire, L2SCP_SHIRE_LOCAL (0x7F) for local +// offset: byte offset within the shire's scratchpad +static inline void * __attribute__((always_inline)) et_shire_l2scp(uint64_t shire, uint64_t offset) { + return (void *) (L2SCP_BASE | ((shire & 0x7F) << 23) | (offset & 0x7FFFFF)); +} + +// Format 0: local shire shorthand — no cross-shire traffic. +static inline void * __attribute__((always_inline)) et_shire_l2scp_local(uint64_t offset) { + return (void *) (L2SCP_BASE | (L2SCP_SHIRE_LOCAL << 23) | (offset & 0x7FFFFF)); +} + +// Format 1: flat offset into a hardware-striped global address space. +// Consecutive 64-byte cache lines automatically land on different shires, +// distributing bandwidth across the mesh. No shire parameter — the +// hardware derives the target shire from the address bits. +static inline void * __attribute__((always_inline)) et_global_l2scp(uint64_t offset) { + return (void *) (L2SCP_BASE | (1ULL << 30) | (offset & 0x3FFFFFFF)); +} + +//****************************************************************************** +// Cache Operatons +//****************************************************************************** + +// Prefetch nlines cache lines into L2 starting at addr, with stride bytes +// between each line. Uses PrefetchVA (CSR 0x81F) with dest=L2 (bits 59:58=01). +// +// The hardware fetches nlines consecutive cache-line-sized (64B) blocks from +// DRAM/L3 into L2, starting at addr and advancing by stride bytes per line. +// This is asynchronous — use WAIT_PREFETCH_0 or WAIT_PREFETCH_1 if the hart +// must stall until the prefetch completes. +// +// NOTE: nlines is encoded in a 4-bit field (max 16). Passing nlines > 16 +// silently truncates. DO NOT pass nlines > 16. +static inline void __attribute__((always_inline)) l2_prefetch(const void * addr, uint64_t nlines, uint64_t stride) { + uint64_t csr_val = (0x1ULL << 58) | ((uint64_t) addr & 0xFFFFFFFFFFC0ULL) | ((nlines - 1) & 0xF); + + __asm__ __volatile__( + "mv x31, %[stride]\n" + "csrw 0x81f, %[val]\n" + : + : [stride] "r"(stride & 0xFFFFFFFFFFC0ULL), [val] "r"(csr_val) + : "x31", "memory"); +} + +// Flush nlines cache lines at stride apart starting at addr from L1 to L2. +// Uses FlushVA (CSR 0x8BF). Caller must FENCE before (to drain stores to L1) +// and WAIT_CACHEOPS after (to ensure flush completes before tensor loads). +// +// NOTE: nlines is encoded in a 4-bit field (max 16). Passing nlines > 16 +// silently truncates. DO NOT pass nlines > 16. +static inline void __attribute__((always_inline)) flush_to_l2(const void * addr, uint64_t nlines, uint64_t stride) { + // dest=01 (L2) in bits 59:58, VA in bits 47:6, numlines-1 in bits 3:0 + uint64_t csr_val = (0x1ULL << 58) | ((uint64_t) addr & 0xFFFFFFFFFFC0ULL) | ((nlines - 1) & 0xF); + uint64_t x31_val = stride & 0xFFFFFFFFFFC0ULL; + + __asm__ __volatile__( + "mv x31, %[x31]\n" + "csrw 0x8BF, %[val]\n" + : + : [x31] "r"(x31_val), [val] "r"(csr_val) + : "x31", "memory"); +} + +// Evict nlines cache lines at stride apart starting at addr from L1 to L2. +// Uses EvictVA (CSR 0x89F). Unlike flush_to_l2, this guarantees the line is +// NOT present in L1 after the operation - subsequent loads will miss and go +// to L2/SCP. Caller must FENCE before and WAIT_CACHEOPS after. +// +// NOTE: nlines is encoded in a 4-bit field (max 16). DO NOT pass nlines > 16. +static inline void __attribute__((always_inline)) evict_to_l2(const void * addr, uint64_t nlines, uint64_t stride) { + // dest=01 (L2) in bits 59:58, VA in bits 47:6, numlines-1 in bits 3:0 + uint64_t csr_val = (0x1ULL << 58) | ((uint64_t) addr & 0xFFFFFFFFFFC0ULL) | ((nlines - 1) & 0xF); + uint64_t x31_val = stride & 0xFFFFFFFFFFC0ULL; + + __asm__ __volatile__( + "mv x31, %[x31]\n" + "csrw 0x89F, %[val]\n" + : + : [x31] "r"(x31_val), [val] "r"(csr_val) + : "x31", "memory"); +} + +// Evict nlines cache lines at stride apart starting at addr from BOTH L1 +// and L2. Uses EvictVA (CSR 0x89F) with dest=10 (L3/DRAM). Guarantees the +// line is NOT present in L1 or L2 after the operation — subsequent loads +// will fetch from L3 or DRAM. Needed because both L1 and L2 are incoherent +// on ET-SoC-1 (L2 is per-shire). +// Caller must FENCE before and WAIT_CACHEOPS after. +// +// NOTE: nlines is encoded in a 4-bit field (max 16). DO NOT pass nlines > 16. +static inline void __attribute__((always_inline)) evict_past_l2(const void * addr, uint64_t nlines, uint64_t stride) { + // dest=10 in bits 59:58, VA in bits 47:6, numlines-1 in bits 3:0 + uint64_t csr_val = (0x2ULL << 58) | ((uint64_t) addr & 0xFFFFFFFFFFC0ULL) | ((nlines - 1) & 0xF); + uint64_t x31_val = stride & 0xFFFFFFFFFFC0ULL; + + __asm__ __volatile__( + "mv x31, %[x31]\n" + "csrw 0x89F, %[val]\n" + : + : [x31] "r"(x31_val), [val] "r"(csr_val) + : "x31", "memory"); +} + +// Evict a contiguous region from both L1 and L2 so subsequent loads fetch +// from L3/DRAM. Both L1 and L2 are incoherent on ET-SoC-1 (L2 is per-shire), +// so every op must evict its inputs before reading if a prior op in the same +// uberkernel batch may have written to them via fsw.ps or tensor_store. +// +// Handles regions larger than the 16-line hardware limit by issuing multiple +// evict_past_l2 calls. +static void evict_region_past_l2(const void * addr, size_t bytes) { + if (!addr || bytes == 0) { + return; + } + + const uint64_t CL = 64; + uint64_t base = (uint64_t) addr & ~(CL - 1); + uint64_t end = ((uint64_t) addr + bytes + CL - 1) & ~(CL - 1); + uint64_t nlines = (end - base) / CL; + // FENCE; + for (uint64_t off = 0; off < nlines; off += 16) { + uint64_t batch = nlines - off; + if (batch > 16) { + batch = 16; + } + evict_past_l2((const void *) (base + off * CL), batch, CL); + } +} + +#endif // PLATFORM_H diff --git a/ggml/src/ggml-et/et-kernels/src/quants.h b/ggml/src/ggml-et/et-kernels/src/quants.h new file mode 100644 index 0000000..692ca00 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/quants.h @@ -0,0 +1,72 @@ +// Scalar dequantization helpers and ET-side block-size aliases. + +#ifndef QUANTS_H +#define QUANTS_H + +#include "math_fp.h" + +#include + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" + +// 64-byte (one cache line) F16 / F32 block sizes. +#define QK_F16 32 +#define QK_F32 16 + +static inline void dequantize_q8_0_block(const block_q8_0 * block, float * dst) { + const float scale = fp16_to_fp32(block->d); + + for (int i = 0; i < QK8_0; i++) { + dst[i] = scale * (float) block->qs[i]; + } +} + +// Low nibbles -> dst[0..15], high nibbles -> dst[16..31]. +static inline void dequantize_q4_0_block(const block_q4_0 * block, float * dst) { + const float scale = fp16_to_fp32(block->d); + + for (int i = 0; i < QK4_0 / 2; i++) { + const uint8_t byte = block->qs[i]; + dst[i] = scale * (float) ((int) (byte & 0xF) - 8); + dst[i + QK4_0 / 2] = scale * (float) ((int) (byte >> 4) - 8); + } +} + +// Unpack the 6-bit scale/min pair for Q4_K group j (groups 4-7 split their high bits). +static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j + 4] & 63; + } else { + *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + *m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4); + } +} + +static inline void dequantize_q4_K_block(const block_q4_K * block, float * dst) { + const uint8_t * q = block->qs; + const float d = fp16_to_fp32(block->d); + const float min = fp16_to_fp32(block->dmin); + + int is = 0; + uint8_t sc, m; + for (int j = 0; j < QK_K; j += 64) { + get_scale_min_k4(is + 0, block->scales, &sc, &m); + const float d1 = d * sc; + const float m1 = min * m; + get_scale_min_k4(is + 1, block->scales, &sc, &m); + const float d2 = d * sc; + const float m2 = min * m; + for (int l = 0; l < 32; ++l) { + *dst++ = d1 * (q[l] & 0xF) - m1; + } + for (int l = 0; l < 32; ++l) { + *dst++ = d2 * (q[l] >> 4) - m2; + } + q += 32; + is += 2; + } +} + +#endif // QUANTS_H diff --git a/ggml/src/ggml-et/et-kernels/src/repeat_f32.c b/ggml/src/ggml-et/et-kernels/src/repeat_f32.c new file mode 100644 index 0000000..4c9b071 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/repeat_f32.c @@ -0,0 +1,118 @@ +//****************************************************************************** +// Repeat F32 Kernel +// Tiles src0 into dst: dst.ne[i] = src0.ne[i] * nr[i] for each dimension. +// All copies are cacheline-aligned (ne00 % 16 == 0). +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include +#include + +struct ggml_et_repeat_params { + struct ggml_tensor src0; // F32 input tensor (tile) + struct ggml_tensor dst; // F32 output tensor (tiled result) +}; + +// Copy n floats from src to dst using 8-wide vector loads/stores. +// n must be a multiple of 16 (cacheline-aligned). +static inline void copy_row_aligned(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f11, %[src_vec]\n" + "fsw.ps f11, %[dst_vec]\n" + : [dst_vec] "=m"(*(float (*)[8]) & dst[i]) + : [src_vec] "m"(*(const float (*)[8]) & src[i]) + : "f11"); + } +} + +// Broadcast a single scalar to n floats using fbc.ps (broadcast to all lanes). +// n must be a multiple of 16 (cacheline-aligned). +static inline void broadcast_scalar_aligned(float * dst, float val, int32_t n) { + __asm__ volatile("fbc.ps f11, %[v]\n" : : [v] "m"(val) : "f11"); + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile("fsw.ps f11, %[dst_vec]\n" : [dst_vec] "=m"(*(float (*)[8]) & dst[i])::"f11"); + } +} + +int entry_point(struct ggml_et_repeat_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + const int64_t ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const int64_t ne0 = dst->ne[0], ne1 = dst->ne[1], ne2 = dst->ne[2], ne3 = dst->ne[3]; + + // src0 strides in bytes + const size_t nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + // dst strides in bytes + const size_t dnb0 = dst->nb[0], dnb1 = dst->nb[1], dnb2 = dst->nb[2], dnb3 = dst->nb[3]; + + // Repeat counts per dimension + const int32_t nr0 = (int32_t) (ne0 / ne00); + const int32_t nr1 = (int32_t) (ne1 / ne01); + const int32_t nr2 = (int32_t) (ne2 / ne02); + const int32_t nr3 = (int32_t) (ne3 / ne03); + + // Total output rows across all dimensions (excluding dim 0 tiling) + const int64_t total_rows = ne1 * ne2 * ne3; + + for (int64_t row = thread_id; row < total_rows; row += num_threads) { + // Decompose linear row index into dst (i1, i2, i3) + int64_t i1 = row % ne1; + int64_t i2 = (row / ne1) % ne2; + int64_t i3 = row / (ne1 * ne2); + + // Map dst indices back to src0 indices (modular wrap) + int64_t k1 = i1 % ne01; + int64_t k2 = i2 % ne02; + int64_t k3 = i3 % ne03; + + const float * src_row = (const float *) ((const char *) src0_data + k1 * nb01 + k2 * nb02 + k3 * nb03); + float * dst_row = (float *) ((char *) dst_data + i1 * dnb1 + i2 * dnb2 + i3 * dnb3); + + if (ne00 == 1) { + // Scalar broadcast: splat single value across entire dst row + broadcast_scalar_aligned(dst_row, *src_row, (int32_t) ne0); + } else if (nr0 == 1) { + // No tiling along dim 0 - single cacheline-aligned row copy + copy_row_aligned(dst_row, src_row, (int32_t) ne00); + } else { + // Tile ne00-sized chunks across dim 0 + for (int32_t i0 = 0; i0 < nr0; i0++) { + copy_row_aligned(dst_row + i0 * ne00, src_row, (int32_t) ne00); + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c b/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c new file mode 100644 index 0000000..d203759 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c @@ -0,0 +1,270 @@ +//****************************************************************************** +// RMS Norm F32 Kernel +// Root Mean Square normalization: y[i] = x[i] / sqrt(mean(x^2) + eps) +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include +#include +#include + +// RMS norm kernel parameters structure +struct ggml_et_rms_norm_params { + struct ggml_tensor src0; // F32 input tensor + struct ggml_tensor dst; // F32 output tensor + float eps; // Epsilon parameter for numerical stability +}; + +int entry_point(struct ggml_et_rms_norm_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + float eps = params->eps; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; // Unsupported type combination + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; // Null data pointer + } + + + if (eps < 0.0f) { + return -1; // Invalid epsilon + } + + const int64_t ne0 = dst->ne[0]; // Inner dimension (row size) + const int64_t ne1 = dst->ne[1]; // Dimension 1 + const int64_t ne2 = dst->ne[2]; // Dimension 2 + const int64_t ne3 = dst->ne[3]; // Dimension 3 + + // Get dst strides (in bytes) + const size_t nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + // Get src0 strides (in bytes) + const size_t nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + + // Verify that src0 and dst have same shape (required for RMS norm) + if (src0->ne[0] != ne0 || src0->ne[1] != ne1 || src0->ne[2] != ne2 || src0->ne[3] != ne3) { + return -1; // Shape mismatch + } + + // RMS norm processes rows independently + // Parallelize across rows using simple striding + // TODO: ensure lines don't cross cache lines + // Precompute reciprocal of row length (constant across all rows) + const float inv_ne0 = et_fdiv(1.0f, (float) (int32_t) ne0); + const int32_t total_rows = (int32_t) (ne1 * ne2 * ne3); + + // Intra-row cooperation only works within a single shire (barrier + L2SCP + // are shire-local). Use per-shire thread count for the threshold. + const int shire_threads = SOC_MINIONS_PER_SHIRE * NUM_HARTS_PER_MINION; // 64 + + if (total_rows >= shire_threads) { + // Row-parallel: each thread processes whole rows + for (int64_t i3 = 0; i3 < ne3; i3++) { + for (int64_t i2 = 0; i2 < ne2; i2++) { + for (int64_t i1 = thread_id; i1 < ne1; i1 += num_threads) { + const float * src_ptr = + (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_ptr = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + + // Set mask to enable all 8 vector lanes + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + // Step 1: Compute sum of squares using 8-wide vectors + __asm__ volatile("fbci.pi f10, 0" ::: "f10"); + + for (int32_t i0 = 0; i0 < (int32_t) ne0; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fmadd.ps f10, f11, f11, f10\n" + : + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11"); + } + + // Horizontal reduce + float sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + + // Step 2: scale = rsqrt(mean + eps) + const float scale = et_powf(sum * inv_ne0 + eps, -0.5f); + + if (!(scale > 0.0f)) { + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return -1; + } + + // Step 3: Apply scaling: broadcast scale once, reuse across loop + uint32_t scale_bits; + __asm__ volatile("fmv.x.s %0, %1" : "=r"(scale_bits) : "f"(scale)); + __asm__ volatile("fbcx.ps f13, %[sb]\n" : : [sb] "r"(scale_bits) : "f13"); + + for (int32_t i0 = 0; i0 < (int32_t) ne0; i0 += 8) { + __asm__ volatile( + "flw.ps f12, %[x_vec]\n" + "fmul.ps f14, f12, f13\n" + "fsw.ps f14, %[result]\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f12", "f14"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + } + } + } + } else { + // Intra-row: threads within each shire cooperate on rows via L2 SCP. + // L2 SCP + barrier are shire-local, so use shire-local thread index. + int shire_tid = thread_id % shire_threads; // 0..63 within this shire + int threads_per_row = shire_threads / total_rows; + int my_row = shire_tid / threads_per_row; + int local_tid = shire_tid % threads_per_row; + int group_base = my_row * threads_per_row; // shire-local group base + + // Excess threads within this shire, barrier and leave + if (my_row >= total_rows) { + FENCE; + et_barrier(ET_BARRIER_SHIRE); + return 0; + } + + // Unflatten row index + int64_t i1 = my_row % ne1; + int64_t i2 = (my_row / ne1) % ne2; + int64_t i3 = my_row / (ne1 * ne2); + + const float * src_ptr = (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_ptr = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + + // Chunk boundaries aligned to 16 floats (64-byte cache line) + const int32_t elems_per_cl = 16; + int32_t total_cls = ((int32_t) ne0 + elems_per_cl - 1) / elems_per_cl; + int32_t cls_per_thread = (total_cls + threads_per_row - 1) / threads_per_row; + int32_t my_start = local_tid * cls_per_thread * elems_per_cl; + int32_t my_end = my_start + cls_per_thread * elems_per_cl; + if (my_end > (int32_t) ne0) { + my_end = (int32_t) ne0; + } + if (my_start >= (int32_t) ne0) { + my_start = 0; + my_end = 0; + } + + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + // Phase 1: each thread computes partial sum of squares on its chunk + __asm__ volatile("fbci.pi f10, 0" ::: "f10"); + for (int32_t i0 = my_start; i0 < my_end; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fmadd.ps f10, f11, f11, f10\n" + : + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11"); + } + + // Horizontal reduce to scalar + float partial_sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(partial_sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + + // Phase 2: write partial sum to L2 SCP, evict from L1D + volatile float * my_slot = (volatile float *) et_shire_l2scp_local((uint64_t) shire_tid * 64); + *my_slot = partial_sum; + FENCE; + evict_to_l2((const void *) my_slot, 1, 64); + WAIT_CACHEOPS; + + et_barrier(ET_BARRIER_SHIRE); + + // Phase 3: ALL threads read partial sums, compute scale, apply to own chunk. + // Each thread independently reduces to avoid a second barrier. + int workers = threads_per_row < total_cls ? threads_per_row : total_cls; + + // Evict stale L1D entries for worker slots + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + evict_to_l2((const void *) slot, 1, 64); + } + WAIT_CACHEOPS; + + // Every thread reduces the same partial sums -> same scale + float total_sum = 0.0f; + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + total_sum += *slot; + } + + const float scale = et_powf(total_sum * inv_ne0 + eps, -0.5f); + if (!(scale > 0.0f)) { + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return -1; + } + + // Each thread applies scale to its own chunk only + if (my_start < my_end) { + uint32_t scale_bits; + __asm__ volatile("fmv.x.s %0, %1" : "=r"(scale_bits) : "f"(scale)); + __asm__ volatile("fbcx.ps f13, %[sb]\n" : : [sb] "r"(scale_bits) : "f13"); + + for (int32_t i0 = my_start; i0 < my_end; i0 += 8) { + __asm__ volatile( + "flw.ps f12, %[x_vec]\n" + "fmul.ps f14, f12, f13\n" + "fsw.ps f14, %[result]\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f12", "f14"); + } + } + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c b/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c new file mode 100644 index 0000000..87e5772 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c @@ -0,0 +1,290 @@ + +// Fused RMS Norm + MUL F32 Kernel + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include +#include +#include + +// Fused RMS norm + MUL kernel parameters structure +struct ggml_et_rms_norm_mul_params { + struct ggml_tensor src0; // F32 input tensor (to be normalized) + struct ggml_tensor src1; // F32 weights tensor (element-wise multiply) + struct ggml_tensor dst; // F32 output tensor + float eps; // Epsilon for numerical stability +}; + +static inline size_t tensor_bytes(const struct ggml_tensor * t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} + +int entry_point(struct ggml_et_rms_norm_mul_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * dst = ¶ms->dst; + + float eps = params->eps; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; // Unsupported type combination + } + + float * src0_data = (float *) src0->data; + float * src1_data = (float *) src1->data; + float * dst_data = (float *) dst->data; + // #ifdef ET_UBERKERNEL + // evict_region_past_l2(src0_data, tensor_bytes(src0)); + // evict_region_past_l2(src1_data, tensor_bytes(src1)); + // // WAIT_CACHEOPS; + // FENCE; + // // et_barrier(ET_BARRIER_GLOBAL); + // #endif + if (!src0_data || !src1_data || !dst_data) { + return -1; // Null data pointer + } + + if (eps < 0.0f) { + return -1; // Invalid epsilon + } + + const int64_t ne0 = dst->ne[0]; // Inner dimension (row size) + const int64_t ne1 = dst->ne[1]; // Dimension 1 + const int64_t ne2 = dst->ne[2]; // Dimension 2 + const int64_t ne3 = dst->ne[3]; // Dimension 3 + + // Get dst strides (in bytes) + const size_t nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + // Get src0 strides (in bytes) + const size_t nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + + // Get src1 (weights) strides (in bytes), supports broadcasting in dims 1,2,3 + const size_t nb10 = src1->nb[0], nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + + // Verify that src0 and dst have same shape (required for RMS norm) + if (src0->ne[0] != ne0 || src0->ne[1] != ne1 || src0->ne[2] != ne2 || src0->ne[3] != ne3) { + return -1; // Shape mismatch + } + // et_barrier(ET_BARRIER_GLOBAL); + + const float inv_ne0 = et_fdiv(1.0f, (float) (int32_t) ne0); + const int32_t total_rows = (int32_t) (ne1 * ne2 * ne3); + const int shire_threads = SOC_MINIONS_PER_SHIRE * NUM_HARTS_PER_MINION; + + if (total_rows >= shire_threads) { + // Row-parallel: each thread processes whole rows + for (int64_t i3 = 0; i3 < ne3; i3++) { + for (int64_t i2 = 0; i2 < ne2; i2++) { + for (int64_t i1 = thread_id; i1 < ne1; i1 += num_threads) { + const float * src_ptr = + (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_ptr = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + + const float * wgt_ptr = (const float *) ((const char *) src1_data + (i3 % src1->ne[3]) * nb13 + + (i2 % src1->ne[2]) * nb12 + (i1 % src1->ne[1]) * nb11); + + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + // Sum of squares + __asm__ volatile("fbci.pi f10, 0" ::: "f10"); + for (int32_t i0 = 0; i0 < (int32_t) ne0; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fmadd.ps f10, f11, f11, f10\n" + : + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11"); + } + + float sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + + const float scale = et_powf(sum * inv_ne0 + eps, -0.5f); + if (!(scale > 0.0f)) { + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return -1; + } + + uint32_t scale_bits; + __asm__ volatile("fmv.x.s %0, %1" : "=r"(scale_bits) : "f"(scale)); + __asm__ volatile("fbcx.ps f13, %[sb]\n" : : [sb] "r"(scale_bits) : "f13"); + + for (int32_t i0 = 0; i0 < (int32_t) ne0; i0 += 8) { + __asm__ volatile( + "flw.ps f12, %[x_vec]\n" + "flw.ps f15, %[w_vec]\n" + "fmul.ps f14, f12, f13\n" + "fmul.ps f14, f14, f15\n" + "fsw.ps f14, %[result]\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]), [w_vec] "m"(*(const float (*)[8]) & + wgt_ptr[i0]) + : "f12", "f14", "f15"); + } + // #ifdef ET_UBERKERNEL + // FENCE; + // evict_region_past_l2(dst_ptr, (size_t)ne0 * sizeof(float)); + // WAIT_CACHEOPS; + // FENCE; + // #endif + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + } + } + } + } else { + // Intra-row: threads within each shire cooperate on rows via L2 SCP. + // L2 SCP + barrier are shire-local, so use shire-local thread index. + int shire_tid = thread_id % shire_threads; + int threads_per_row = shire_threads / total_rows; + int my_row = shire_tid / threads_per_row; + int local_tid = shire_tid % threads_per_row; + int group_base = my_row * threads_per_row; + + // Excess threads within this shire + if (my_row >= total_rows) { + __asm__ __volatile__("fence\n" ::: "memory"); + et_barrier(ET_BARRIER_SHIRE); + return 0; + } + + // Unflatten row index + int64_t i1 = my_row % ne1; + int64_t i2 = (my_row / ne1) % ne2; + int64_t i3 = my_row / (ne1 * ne2); + + const float * src_ptr = (const float *) ((const char *) src0_data + i3 * nb03 + i2 * nb02 + i1 * nb01); + float * dst_ptr = (float *) ((char *) dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + + const float * wgt_ptr = (const float *) ((const char *) src1_data + (i3 % src1->ne[3]) * nb13 + + (i2 % src1->ne[2]) * nb12 + (i1 % src1->ne[1]) * nb11); + + // Chunk boundaries aligned to 16 floats (64-byte cache line) + const int32_t elems_per_cl = 16; + int32_t total_cls = ((int32_t) ne0 + elems_per_cl - 1) / elems_per_cl; + int32_t cls_per_thread = (total_cls + threads_per_row - 1) / threads_per_row; + int32_t my_start = local_tid * cls_per_thread * elems_per_cl; + int32_t my_end = my_start + cls_per_thread * elems_per_cl; + if (my_end > (int32_t) ne0) { + my_end = (int32_t) ne0; + } + if (my_start >= (int32_t) ne0) { + my_start = 0; + my_end = 0; + } + + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + // Phase 1: partial sum of squares on own chunk + __asm__ volatile("fbci.pi f10, 0" ::: "f10"); + for (int32_t i0 = my_start; i0 < my_end; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fmadd.ps f10, f11, f11, f10\n" + : + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11"); + } + + float partial_sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(partial_sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + + // Phase 2: write partial sum to L2 SCP, evict from L1D + volatile float * my_slot = (volatile float *) et_shire_l2scp_local((uint64_t) shire_tid * 64); + *my_slot = partial_sum; + __asm__ __volatile__("fence\n" ::: "memory"); + evict_to_l2((const void *) my_slot, 1, 64); + WAIT_CACHEOPS; + + et_barrier(ET_BARRIER_SHIRE); + + // Phase 3: all threads read partial sums, compute scale, apply to own chunk + int workers = threads_per_row < total_cls ? threads_per_row : total_cls; + + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + evict_to_l2((const void *) slot, 1, 64); + } + WAIT_CACHEOPS; + + float total_sum = 0.0f; + for (int t = 0; t < workers; t++) { + volatile float * slot = (volatile float *) et_shire_l2scp_local((uint64_t) (group_base + t) * 64); + total_sum += *slot; + } + + const float scale = et_powf(total_sum * inv_ne0 + eps, -0.5f); + if (!(scale > 0.0f)) { + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + return -1; + } + + // Apply scale * weights to own chunk + if (my_start < my_end) { + uint32_t scale_bits; + __asm__ volatile("fmv.x.s %0, %1" : "=r"(scale_bits) : "f"(scale)); + __asm__ volatile("fbcx.ps f13, %[sb]\n" : : [sb] "r"(scale_bits) : "f13"); + + for (int32_t i0 = my_start; i0 < my_end; i0 += 8) { + __asm__ volatile( + "flw.ps f12, %[x_vec]\n" + "flw.ps f15, %[w_vec]\n" + "fmul.ps f14, f12, f13\n" + "fmul.ps f14, f14, f15\n" + "fsw.ps f14, %[result]\n" + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]), [w_vec] "m"(*(const float (*)[8]) & wgt_ptr[i0]) + : "f12", "f14", "f15"); + } + // #ifdef ET_UBERKERNEL + // FENCE; + // evict_region_past_l2(dst_ptr + my_start, (size_t)(my_end - my_start) * sizeof(float)); + // WAIT_CACHEOPS; + // FENCE; + // #endif + } + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/rope_f32.c b/ggml/src/ggml-et/et-kernels/src/rope_f32.c new file mode 100644 index 0000000..227d6d1 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/rope_f32.c @@ -0,0 +1,656 @@ +//****************************************************************************** +// ROPE (Rotary Position Encoding) Kernel +// Experiment 1: +// - Keep old scheduling and rotate logic +// - ONLY SIMD-ize sin/cos approximation inside compute_rope_cache() +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include +#include + +// ROPE constants (matching GGML definitions) +#define GGML_ROPE_TYPE_NEOX 2 +#define GGML_ROPE_TYPE_MROPE 8 +#define GGML_ROPE_TYPE_IMROPE 40 +#define MAX_ROPE_HALF_DIMS 256 // supports up to n_dims=512 + +#define ROPE_VEC_WIDTH 8 + +#define ROPE_PI 3.14159265358979323846f +#define ROPE_TWO_PI 6.28318530717958647693f +#define ROPE_PI_OVER_2 1.57079632679489661923f +#define ROPE_INV_TWO_PI 0.15915494309189533577f + +// ROPE operation parameters structure (matches ggml-et-ops.h) +typedef struct { + int32_t n_past; + int32_t n_dims; // Number of dimensions to apply ROPE to (must be even) + int32_t mode; // ROPE mode (0=normal, 2=neox) + int32_t n_ctx; + int32_t n_ctx_orig; + float freq_base; // Base frequency (usually 10000.0f) + float freq_scale; // Frequency scaling factor + float ext_factor; // Extension factor for YaRN + float attn_factor; // Attention factor for YaRN + float beta_fast; // Fast beta for YaRN + float beta_slow; // Slow beta for YaRN + int32_t sections[4]; // Sections for multi-modal ROPE +} rope_params_t; + +// ROPE kernel parameters structure (matches ggml_et_rope_params) +struct ggml_et_rope_params { + struct ggml_tensor src0; // F32 input tensor + struct ggml_tensor src1; // I32 position tensor + struct ggml_tensor src2; // F32 frequency factors (optional) + struct ggml_tensor dst; // F32 output tensor + rope_params_t rope_params; +}; + +//------------------------------------------------------------------------------ +// Existing scalar helpers +//------------------------------------------------------------------------------ + +// floor/ceil with ±inf and NaN passthrough. +static inline float rope_floorf(float x) { + union { + float f; + uint32_t u; + } v = { .f = x }; + + const uint32_t expo = (v.u >> 23) & 0xFF; + if (expo == 0xFF) { + return x; // inf or NaN + } + if (expo >= 23 + 127) { + return x; // already integer-valued + } + int i = (int) x; + return (x < 0.0f && (float) i != x) ? (float) (i - 1) : (float) i; +} + +static inline float rope_ceilf(float x) { + union { + float f; + uint32_t u; + } v = { .f = x }; + + const uint32_t expo = (v.u >> 23) & 0xFF; + if (expo == 0xFF) { + return x; // inf or NaN + } + if (expo >= 23 + 127) { + return x; // already integer-valued + } + int i = (int) x; + return (x > 0.0f && (float) i != x) ? (float) (i + 1) : (float) i; +} + +static inline float rope_yarn_ramp(const float low, const float high, const int i0) { + float denom = high - low; + if (denom < 0.001f) { + denom = 0.001f; + } + + const float y = et_fdiv((float) (i0 / 2) - low, denom); + const float clamped = y < 0.0f ? 0.0f : (y > 1.0f ? 1.0f : y); + return 1.0f - clamped; +} + +// Matches CPU reference (ggml_rope_yarn_corr_dim). +static inline float rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float beta, float freq_base) { + return (float) n_dims * + et_fdiv(et_logf(et_fdiv((float) n_ctx_orig, beta * ROPE_TWO_PI)), 2.0f * et_logf(freq_base)); +} + +static inline void rope_yarn_corr_dims(int n_dims, + int n_ctx_orig, + float freq_base, + float beta_fast, + float beta_slow, + float dims[2]) { + // Match CPU: floor on start, ceil on end, then clamp to [0, n_dims-1]. + float start = rope_floorf(rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base)); + float end = rope_ceilf(rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base)); + + dims[0] = start > 0.0f ? start : 0.0f; + dims[1] = end < (float) (n_dims - 1) ? end : (float) (n_dims - 1); +} + +//------------------------------------------------------------------------------ +// SIMD sin/cos approximation +//------------------------------------------------------------------------------ + +static const float rope_ps_one[ROPE_VEC_WIDTH] + __attribute__((aligned(32))) = { 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f }; +static const float rope_ps_c3[ROPE_VEC_WIDTH] + __attribute__((aligned(32))) = { 1.0f / 6.0f, 1.0f / 6.0f, 1.0f / 6.0f, 1.0f / 6.0f, + 1.0f / 6.0f, 1.0f / 6.0f, 1.0f / 6.0f, 1.0f / 6.0f }; +static const float rope_ps_c5[ROPE_VEC_WIDTH] + __attribute__((aligned(32))) = { 1.0f / 120.0f, 1.0f / 120.0f, 1.0f / 120.0f, 1.0f / 120.0f, + 1.0f / 120.0f, 1.0f / 120.0f, 1.0f / 120.0f, 1.0f / 120.0f }; +static const float rope_ps_c7[ROPE_VEC_WIDTH] + __attribute__((aligned(32))) = { 1.0f / 5040.0f, 1.0f / 5040.0f, 1.0f / 5040.0f, 1.0f / 5040.0f, + 1.0f / 5040.0f, 1.0f / 5040.0f, 1.0f / 5040.0f, 1.0f / 5040.0f }; +static const float rope_ps_c9[ROPE_VEC_WIDTH] + __attribute__((aligned(32))) = { 1.0f / 362880.0f, 1.0f / 362880.0f, 1.0f / 362880.0f, 1.0f / 362880.0f, + 1.0f / 362880.0f, 1.0f / 362880.0f, 1.0f / 362880.0f, 1.0f / 362880.0f }; +static const float rope_ps_c11[ROPE_VEC_WIDTH] + __attribute__((aligned(32))) = { 1.0f / 39916800.0f, 1.0f / 39916800.0f, 1.0f / 39916800.0f, 1.0f / 39916800.0f, + 1.0f / 39916800.0f, 1.0f / 39916800.0f, 1.0f / 39916800.0f, 1.0f / 39916800.0f }; + +static inline uint64_t rope_ps_enter_fullmask(void) { + uint64_t old_mask; + __asm__ volatile( + "mova.x.m %0 \n\t" + "li t0, -1 \n\t" + "mova.m.x t0 \n\t" + : "=r"(old_mask) + : + : "t0", "memory"); + return old_mask; +} + +static inline void rope_ps_leave_fullmask(uint64_t old_mask) { + __asm__ volatile("mova.m.x %0 \n\t" : : "r"(old_mask) : "memory"); +} + +static inline void rope_poly_sin_block8(float * out, const float * x) { + __asm__ volatile( + "flw.ps f0, %[x] \n\t" + "fmul.ps f1, f0, f0 \n\t" + + "flw.ps f2, %[c11] \n\t" + "flw.ps f3, %[c9] \n\t" + "fnmsub.ps f2, f1, f2, f3 \n\t" + + "flw.ps f3, %[c7] \n\t" + "fnmsub.ps f2, f1, f2, f3 \n\t" + + "flw.ps f3, %[c5] \n\t" + "fnmsub.ps f2, f1, f2, f3 \n\t" + + "flw.ps f3, %[c3] \n\t" + "fnmsub.ps f2, f1, f2, f3 \n\t" + + "flw.ps f3, %[one] \n\t" + "fnmsub.ps f2, f1, f2, f3 \n\t" + + "fmul.ps f4, f0, f2 \n\t" + "fsw.ps f4, %[out] \n\t" + : [out] "=m"(*(float (*)[ROPE_VEC_WIDTH]) out) + : [x] "m"(*(const float (*)[ROPE_VEC_WIDTH]) x), [one] "m"(*(const float (*)[ROPE_VEC_WIDTH]) rope_ps_one), + [c3] "m"(*(const float (*)[ROPE_VEC_WIDTH]) rope_ps_c3), + [c5] "m"(*(const float (*)[ROPE_VEC_WIDTH]) rope_ps_c5), + [c7] "m"(*(const float (*)[ROPE_VEC_WIDTH]) rope_ps_c7), + [c9] "m"(*(const float (*)[ROPE_VEC_WIDTH]) rope_ps_c9), + [c11] "m"(*(const float (*)[ROPE_VEC_WIDTH]) rope_ps_c11) + : "f0", "f1", "f2", "f3", "f4", "memory"); +} + +static inline void rope_sincos_block8(float * sin8, float * cos8, const float * theta8) { + float sin_fold[ROPE_VEC_WIDTH] __attribute__((aligned(32))); + float cos_fold[ROPE_VEC_WIDTH] __attribute__((aligned(32))); + float sin_sign[ROPE_VEC_WIDTH] __attribute__((aligned(32))); + float cos_sign[ROPE_VEC_WIDTH] __attribute__((aligned(32))); + + for (int i = 0; i < ROPE_VEC_WIDTH; ++i) { + float x = theta8[i]; + + if (x > ROPE_PI || x < -ROPE_PI) { + float cycles = x * ROPE_INV_TWO_PI; + int n = (int) cycles; + if (x < 0.0f) { + n--; + } + x = x - (float) n * ROPE_TWO_PI; + } + + { + float y = x; + float s = 1.0f; + if (y > ROPE_PI_OVER_2) { + y = ROPE_PI - y; + } else if (y < -ROPE_PI_OVER_2) { + y = -ROPE_PI - y; + s = -1.0f; + } + sin_fold[i] = y; + sin_sign[i] = s; + } + + { + float y = x + ROPE_PI_OVER_2; + if (y > ROPE_PI || y < -ROPE_PI) { + float cycles = y * ROPE_INV_TWO_PI; + int n = (int) cycles; + if (y < 0.0f) { + n--; + } + y = y - (float) n * ROPE_TWO_PI; + } + + float s = 1.0f; + if (y > ROPE_PI_OVER_2) { + y = ROPE_PI - y; + } else if (y < -ROPE_PI_OVER_2) { + y = -ROPE_PI - y; + s = -1.0f; + } + cos_fold[i] = y; + cos_sign[i] = s; + } + } + + { + const uint64_t saved_mask = rope_ps_enter_fullmask(); + + rope_poly_sin_block8(sin8, sin_fold); + rope_poly_sin_block8(cos8, cos_fold); + + __asm__ volatile( + "flw.ps f0, %[sinv] \n\t" + "flw.ps f1, %[sinsgn] \n\t" + "fmul.ps f2, f0, f1 \n\t" + "fsw.ps f2, %[sout] \n\t" + + "flw.ps f3, %[cosv] \n\t" + "flw.ps f4, %[cossgn] \n\t" + "fmul.ps f5, f3, f4 \n\t" + "fsw.ps f5, %[cout] \n\t" + : [sout] "=m"(*(float (*)[ROPE_VEC_WIDTH]) sin8), [cout] "=m"(*(float (*)[ROPE_VEC_WIDTH]) cos8) + : [sinv] "m"(*(const float (*)[ROPE_VEC_WIDTH]) sin8), + [sinsgn] "m"(*(const float (*)[ROPE_VEC_WIDTH]) sin_sign), + [cosv] "m"(*(const float (*)[ROPE_VEC_WIDTH]) cos8), + [cossgn] "m"(*(const float (*)[ROPE_VEC_WIDTH]) cos_sign) + : "f0", "f1", "f2", "f3", "f4", "f5", "memory"); + + rope_ps_leave_fullmask(saved_mask); + } +} + +//------------------------------------------------------------------------------ +// Cache build +//------------------------------------------------------------------------------ + +// scalar fallback for tail / tiny sizes +static inline void rope_yarn_scalar(float theta_extrap, + float freq_scale, + const float corr_dims[2], + int64_t i0, + float ext_factor, + float mscale, + float * cos_theta, + float * sin_theta) { + float theta_interp = freq_scale * theta_extrap; + float theta = theta_interp; + + if (ext_factor != 0.0f) { + float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], (int) i0) * ext_factor; + theta = theta_interp * (1.0f - ramp_mix) + theta_extrap * ramp_mix; + mscale *= 1.0f + 0.1f * et_logf(et_fdiv(1.0f, freq_scale)); + } + + *cos_theta = et_cosf(theta) * mscale; + *sin_theta = et_sinf(theta) * mscale; +} + +// Populate cos/sin cache for a given position using running theta product +// Experiment 1: +// - theta construction and YaRN mixing stay scalar +// - actual sin/cos approximation is done in vec8 blocks +static inline void compute_rope_cache(float * cos_cache, + float * sin_cache, + int32_t n_dims, + float theta_scale, + int32_t pos, + const float * freq_factors, + float freq_scale, + const float corr_dims[2], + float ext_factor, + float attn_factor) { + const int32_t half_dims = n_dims / 2; + float theta = 1.0f; + + int32_t dim_idx = 0; + + for (; dim_idx + ROPE_VEC_WIDTH <= half_dims; dim_idx += ROPE_VEC_WIDTH) { + float theta_block[ROPE_VEC_WIDTH] __attribute__((aligned(32))); + float theta_local = theta; + float mscale = attn_factor; + + if (ext_factor != 0.0f) { + mscale *= 1.0f + 0.1f * et_logf(et_fdiv(1.0f, freq_scale)); + } + + for (int i = 0; i < ROPE_VEC_WIDTH; ++i) { + const int32_t pair_idx = dim_idx + i; + const float ff = freq_factors ? freq_factors[pair_idx] : 1.0f; + const float theta_base = (float) pos * theta_local; + const float theta_extrap = et_fdiv(theta_base, ff); + + float theta_interp = freq_scale * theta_extrap; + float theta_mix = theta_interp; + + if (ext_factor != 0.0f) { + float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], pair_idx * 2) * ext_factor; + theta_mix = theta_interp * (1.0f - ramp_mix) + theta_extrap * ramp_mix; + } + + theta_block[i] = theta_mix; + theta_local *= theta_scale; + } + + rope_sincos_block8(&sin_cache[dim_idx], &cos_cache[dim_idx], theta_block); + + for (int i = 0; i < ROPE_VEC_WIDTH; ++i) { + sin_cache[dim_idx + i] *= mscale; + cos_cache[dim_idx + i] *= mscale; + } + + theta = theta_local; + } + + // tail fallback + for (; dim_idx < half_dims; ++dim_idx) { + const float ff = freq_factors ? freq_factors[dim_idx] : 1.0f; + const float theta_base = (float) pos * theta; + + rope_yarn_scalar(et_fdiv(theta_base, ff), freq_scale, corr_dims, dim_idx * 2, ext_factor, attn_factor, + &cos_cache[dim_idx], &sin_cache[dim_idx]); + + theta *= theta_scale; + } +} + +//------------------------------------------------------------------------------ +// IMROPE cache build (interleaved multi-modal RoPE for Qwen3VL) +//------------------------------------------------------------------------------ + +// Builds cos/sin cache with 4 interleaved position channels. +// Each dimension pair selects from {theta_t, theta_h, theta_w, theta_e} +// using a mod-3 sector pattern, matching the CPU reference exactly. +static inline void compute_imrope_cache(float * cos_cache, + float * sin_cache, + int32_t n_dims, + float theta_scale, + int32_t pos_t, + int32_t pos_h, + int32_t pos_w, + int32_t pos_e, + const int32_t sections[4], + const float * freq_factors, + float freq_scale, + const float corr_dims[2], + float ext_factor, + float attn_factor) { + const int32_t half_dims = n_dims / 2; + const int32_t sect_dims = sections[0] + sections[1] + sections[2] + sections[3]; + + float theta_t = (float) pos_t; + float theta_h = (float) pos_h; + float theta_w = (float) pos_w; + float theta_e = (float) pos_e; + + int32_t dim_idx = 0; + + for (; dim_idx + ROPE_VEC_WIDTH <= half_dims; dim_idx += ROPE_VEC_WIDTH) { + float theta_block[ROPE_VEC_WIDTH] __attribute__((aligned(32))); + float mscale = attn_factor; + + if (ext_factor != 0.0f) { + mscale *= 1.0f + 0.1f * et_logf(et_fdiv(1.0f, freq_scale)); + } + + for (int i = 0; i < ROPE_VEC_WIDTH; ++i) { + const int32_t pair_idx = dim_idx + i; + const int32_t sector = pair_idx % sect_dims; + const float ff = freq_factors ? freq_factors[pair_idx] : 1.0f; + + // Interleaved sector assignment (mod-3 pattern) + float theta; + if (sector % 3 == 1 && sector < 3 * sections[1]) { + theta = theta_h; + } else if (sector % 3 == 2 && sector < 3 * sections[2]) { + theta = theta_w; + } else if (sector % 3 == 0 && sector < 3 * sections[0]) { + theta = theta_t; + } else { + theta = theta_e; + } + + const float theta_extrap = et_fdiv(theta, ff); + float theta_interp = freq_scale * theta_extrap; + float theta_mix = theta_interp; + + if (ext_factor != 0.0f) { + float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], pair_idx * 2) * ext_factor; + theta_mix = theta_interp * (1.0f - ramp_mix) + theta_extrap * ramp_mix; + } + + theta_block[i] = theta_mix; + + // All 4 thetas advance every iteration + theta_t *= theta_scale; + theta_h *= theta_scale; + theta_w *= theta_scale; + theta_e *= theta_scale; + } + + rope_sincos_block8(&sin_cache[dim_idx], &cos_cache[dim_idx], theta_block); + + for (int i = 0; i < ROPE_VEC_WIDTH; ++i) { + sin_cache[dim_idx + i] *= mscale; + cos_cache[dim_idx + i] *= mscale; + } + } + + // Scalar tail + for (; dim_idx < half_dims; ++dim_idx) { + const int32_t sector = dim_idx % sect_dims; + const float ff = freq_factors ? freq_factors[dim_idx] : 1.0f; + + float theta; + if (sector % 3 == 1 && sector < 3 * sections[1]) { + theta = theta_h; + } else if (sector % 3 == 2 && sector < 3 * sections[2]) { + theta = theta_w; + } else if (sector % 3 == 0 && sector < 3 * sections[0]) { + theta = theta_t; + } else { + theta = theta_e; + } + + rope_yarn_scalar(et_fdiv(theta, ff), freq_scale, corr_dims, dim_idx * 2, ext_factor, attn_factor, + &cos_cache[dim_idx], &sin_cache[dim_idx]); + + theta_t *= theta_scale; + theta_h *= theta_scale; + theta_w *= theta_scale; + theta_e *= theta_scale; + } +} + +//------------------------------------------------------------------------------ +// Entry point +//------------------------------------------------------------------------------ + +int entry_point(struct ggml_et_rope_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return -1; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * src2 = ¶ms->src2; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + const float * src0_data = (const float *) src0->data; + const int32_t * src1_data = (const int32_t *) src1->data; + const float * freq_factors = (src2 && src2->data) ? (const float *) src2->data : NULL; + float * dst_data = (float *) dst->data; + + if (!src0_data || !src1_data || !dst_data) { + return -1; + } +#ifdef ET_UBERKERNEL + const size_t src0_bytes = (size_t) src0->ne[0] * src0->ne[1] * src0->ne[2] * src0->ne[3] * src0->nb[0]; + const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * src1->nb[0]; + evict_region_past_l2(src0_data, src0_bytes); + evict_region_past_l2(src1_data, src1_bytes); + WAIT_CACHEOPS; + FENCE; + et_barrier(ET_BARRIER_GLOBAL); +#endif + const int64_t head_dim = src0->ne[0]; + const int64_t heads = src0->ne[1]; + const int64_t seq_len = src0->ne[2]; + const int64_t batch = src0->ne[3]; + + const rope_params_t * rope_params = ¶ms->rope_params; + const int32_t n_dims = rope_params->n_dims; + const float freq_base = rope_params->freq_base; + const float freq_scale = rope_params->freq_scale; + const int32_t mode = rope_params->mode; + + if (n_dims <= 0 || n_dims > head_dim || (n_dims & 1) != 0) { + return -1; + } + + if (n_dims / 2 > MAX_ROPE_HALF_DIMS) { + return -1; + } + + float cos_cache[MAX_ROPE_HALF_DIMS]; + float sin_cache[MAX_ROPE_HALF_DIMS]; + + float corr_dims[2]; + rope_yarn_corr_dims(n_dims, rope_params->n_ctx_orig, freq_base, rope_params->beta_fast, rope_params->beta_slow, + corr_dims); + et_barrier(ET_BARRIER_GLOBAL); + + // Distribute by individual heads: total = batch * seq_len * heads. + const int64_t total_heads = batch * seq_len * heads; + const int64_t start_wu = (total_heads * thread_id) / num_threads; + const int64_t end_wu = (total_heads * (thread_id + 1)) / num_threads; + + if (start_wu >= end_wu) { + return 0; + } + + const float theta_scale = et_powf(freq_base, et_fdiv(-2.0f, (float) n_dims)); + const int32_t half_dims = n_dims / 2; + const int is_neox = (mode & GGML_ROPE_TYPE_NEOX) != 0; + const int is_imrope = (mode == GGML_ROPE_TYPE_IMROPE); + const int use_neox_rotation = is_neox || is_imrope; + + // For IMROPE position cache invalidation: track all 4 channels + int32_t last_pos = -1; + int32_t last_pos_h = -1; + int32_t last_pos_w = -1; + int32_t last_pos_e = -1; + + for (int64_t wu = start_wu; wu < end_wu; ++wu) { + const int64_t h = wu % heads; + const int64_t s = (wu / heads) % seq_len; + const int64_t b = wu / (heads * seq_len); + + if (is_imrope) { + // IMROPE: src1 layout is [p_t(0..S-1), p_h(0..S-1), p_w(0..S-1), p_e(0..S-1)] + const int32_t pt = src1_data[s] + rope_params->n_past; + const int32_t ph = src1_data[s + seq_len] + rope_params->n_past; + const int32_t pw = src1_data[s + seq_len * 2] + rope_params->n_past; + const int32_t pe = src1_data[s + seq_len * 3] + rope_params->n_past; + + if (pt != last_pos || ph != last_pos_h || pw != last_pos_w || pe != last_pos_e) { + compute_imrope_cache(cos_cache, sin_cache, n_dims, theta_scale, pt, ph, pw, pe, rope_params->sections, + freq_factors, freq_scale, corr_dims, rope_params->ext_factor, + rope_params->attn_factor); + last_pos = pt; + last_pos_h = ph; + last_pos_w = pw; + last_pos_e = pe; + } + } else { + const int32_t pos = src1_data[s] + rope_params->n_past; + + if (pos != last_pos) { + compute_rope_cache(cos_cache, sin_cache, n_dims, theta_scale, pos, freq_factors, freq_scale, corr_dims, + rope_params->ext_factor, rope_params->attn_factor); + last_pos = pos; + } + } + + const float * head_src = + (const float *) ((const char *) src0_data + b * src0->nb[3] + s * src0->nb[2] + h * src0->nb[1]); + + float * head_dst = (float *) ((char *) dst_data + b * dst->nb[3] + s * dst->nb[2] + h * dst->nb[1]); + + // Copy dimensions beyond n_dims unchanged + for (int64_t d = n_dims; d < head_dim; ++d) { + head_dst[d] = head_src[d]; + } + + if (use_neox_rotation) { + // NEOX/IMROPE: pairs at (i, i+half_dims) + uint64_t temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + for (int32_t dim_idx = 0; dim_idx < half_dims; dim_idx += 8) { + __asm__ volatile( + "flw.ps f0, %[x0_src] \n\t" + "flw.ps f1, %[x1_src] \n\t" + "flw.ps f2, %[sin_cache] \n\t" + "flw.ps f3, %[cos_cache] \n\t" + "fmul.ps f4, f0, f3 \n\t" + "fmul.ps f5, f0, f2 \n\t" + "fnmsub.ps f4, f1, f2, f4 \n\t" + "fmadd.ps f5, f1, f3, f5 \n\t" + "fsw.ps f4, %[x0_dst] \n\t" + "fsw.ps f5, %[x1_dst] \n\t" + : [x0_dst] "=m"(*(float (*)[8]) & head_dst[dim_idx]), [x1_dst] "=m"(*(float (*)[8]) & + head_dst[dim_idx + half_dims]) + : [x0_src] "m"(*(const float (*)[8]) & head_src[dim_idx]), + [x1_src] "m"(*(const float (*)[8]) & head_src[dim_idx + half_dims]), + [sin_cache] "m"(*(const float (*)[8]) & sin_cache[dim_idx]), + [cos_cache] "m"(*(const float (*)[8]) & cos_cache[dim_idx]) + : "f0", "f1", "f2", "f3", "f4", "f5", "memory"); + } + + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + } else { + // Standard: adjacent pairs (2i, 2i+1) + for (int32_t pair_idx = 0; pair_idx < half_dims; ++pair_idx) { + const int32_t dim_in_head = pair_idx * 2; + const float x0 = head_src[dim_in_head]; + const float x1 = head_src[dim_in_head + 1]; + + head_dst[dim_in_head] = x0 * cos_cache[pair_idx] - x1 * sin_cache[pair_idx]; + head_dst[dim_in_head + 1] = x0 * sin_cache[pair_idx] + x1 * cos_cache[pair_idx]; + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c b/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c new file mode 100644 index 0000000..4c00b1a --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c @@ -0,0 +1,184 @@ +//****************************************************************************** +// RWKV WKV6 F32 Kernel +// +// Implements the RWKV-6 linear attention recurrence: +// dst = r @ (time_faaaa * (k @ v) + state) +// state = time_decay * state + (k @ v) +// +// For each head h, timestep t, row i: +// kv[j] = v[j] * k[i] +// temp[j] = kv[j] * tf[i] + state[i][j] +// dst[j] += temp[j] * r[i] (accumulated across all i) +// state[i][j] = state[i][j] * td[i] + kv[j] +// +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_rwkv_wkv6_params { + float * k; // src[0]: [S, H, T] key + float * v; // src[1]: [S, H, T] value + float * r; // src[2]: [S, H, T] receptance + float * tf; // src[3]: [S, H] time_faaaa (per-head, not per-token) + float * td; // src[4]: [S, H, T] time_decay + float * state_in; // src[5]: [S*S*H, n_seqs] initial state + float * dst; // [C, T + S*n_seqs] output + state_out + int32_t C; // total channels (S * H) + int32_t H; // number of heads + int32_t S; // head size + int32_t T; // number of tokens + int32_t n_seqs; // number of sequences +}; + +int entry_point(struct ggml_et_rwkv_wkv6_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + const float * k = params->k; + const float * v = params->v; + const float * r = params->r; + const float * tf = params->tf; + const float * td = params->td; + const float * state_in = params->state_in; + float * dst_data = params->dst; + + const int32_t C = params->C; + const int32_t H = params->H; + const int32_t S = params->S; + const int32_t T = params->T; + const int32_t n_seqs = params->n_seqs; + + if (!k || !v || !r || !tf || !td || !state_in || !dst_data) { + return -1; + } + + const int32_t tps = T / n_seqs; // tokens per sequence + float * state_out = dst_data + C * T; + float zero = 0.0f; + + // Tile j by one cache line so each hart's dst/state writes never share + // a 64-B line with another hart's writes (the chip is non-coherent). + // Tiling on j (not i) is required for WKV6 because dst[j] is accumulated + // across i — splitting i across harts would race on dst writes. + // For S=64 this gives 4 tiles per head; for S<16 or odd S we fall back + // to one-hart-per-head (= the original parallelism). + const int32_t j_tile = (S % 16 == 0) ? 16 : S; + const int32_t tiles_per_head = S / j_tile; + const int32_t total_units = H * tiles_per_head; + + // Parallelize across (head, j-tile) pairs. The t loop stays inside this + // unit loop so the same hart owns the same column slice of state across + // all timesteps — required for the recurrence to read back its own + // writes without going through L2. + for (int32_t u = thread_id; u < total_units; u += num_threads) { + const int32_t h = u / tiles_per_head; + const int32_t tile = u % tiles_per_head; + const int32_t j_start = tile * j_tile; + const int32_t j_end = j_start + j_tile; + + const int32_t h_off = h * S; // offset within C for this head + const int32_t s2d = h * S * S; // offset within state for this head + + for (int32_t t = 0; t < T; t++) { + const int32_t seq = t / tps; + const int32_t t_in_seq = t % tps; + const int32_t seq_state = seq * S * C; + + const float * s_prev; + float * s_cur = state_out + seq_state + s2d; + + if (t_in_seq == 0) { + s_prev = state_in + seq_state + s2d; + } else { + s_prev = s_cur; + } + + const int32_t th = t * C + h_off; + + // Pointers for this timestep/head + const float * k_ptr = k + th; + const float * v_ptr = v + th; + const float * r_ptr = r + th; + const float * tf_ptr = tf + h_off; // tf is per-head, no t offset + const float * td_ptr = td + th; + + // Zero this hart's slice of dst: dst[th + j_start..th + j_end-1] + // WKV6 accumulates dst[j] across all i, so must start from zero + float * dst_row = dst_data + th; + for (int32_t j = j_start; j < j_end; j += 8) { + __asm__ volatile( + "fbc.ps f10, %[z]\n" + "fsw.ps f10, %[dst_vec]\n" + : [dst_vec] "=m"(*(float (*)[8]) & dst_row[j]) + : [z] "m"(zero) + : "f10"); + } + + for (int32_t i = 0; i < S; i++) { + const float * sp_row = s_prev + i * S; // state_prev row i + float * sc_row = s_cur + i * S; // state_cur row i + + float k_val = k_ptr[i]; + float r_val = r_ptr[i]; + float tf_val = tf_ptr[i]; + float td_val = td_ptr[i]; + + // Broadcast k[i], r[i], tf[i], td[i] to vector registers + __asm__ volatile( + "fbc.ps f20, %[kv]\n" // f20 = k[i] broadcast + "fbc.ps f21, %[rv]\n" // f21 = r[i] broadcast + "fbc.ps f22, %[tfv]\n" // f22 = tf[i] broadcast + "fbc.ps f23, %[tdv]\n" // f23 = td[i] broadcast + : + : [kv] "m"(k_val), [rv] "m"(r_val), [tfv] "m"(tf_val), [tdv] "m"(td_val) + : "f20", "f21", "f22", "f23"); + + for (int32_t j = j_start; j < j_end; j += 8) { + __asm__ volatile( + // Load v[j], state_prev[i][j], dst[j] + "flw.ps f10, %[v_vec]\n" // v[j..j+7] + "flw.ps f11, %[s_vec]\n" // state_prev[i][j..j+7] + "flw.ps f12, %[d_vec]\n" // dst[j..j+7] (accumulated) + + // kv = v * k_broadcast + "fmul.ps f13, f10, f20\n" // kv = v * k + + // temp = kv * tf_broadcast + state_prev + "fmadd.ps f14, f13, f22, f11\n" // temp = kv * tf + state + + // dst[j] += temp * r_broadcast + "fmadd.ps f12, f14, f21, f12\n" // dst += temp * r + "fsw.ps f12, %[d_out]\n" // store updated dst + + // state_cur[i][j] = state_prev * td_broadcast + kv + "fmadd.ps f11, f11, f23, f13\n" // state = state * td + kv + "fsw.ps f11, %[s_out]\n" // store new state + + : [d_out] "=m"(*(float (*)[8]) & dst_row[j]), [s_out] "=m"(*(float (*)[8]) & sc_row[j]) + : [v_vec] "m"(*(const float (*)[8]) & v_ptr[j]), [s_vec] "m"(*(const float (*)[8]) & sp_row[j]), + [d_vec] "m"(*(const float (*)[8]) & dst_row[j]) + : "f10", "f11", "f12", "f13", "f14"); + } + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c b/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c new file mode 100644 index 0000000..08e4ba2 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c @@ -0,0 +1,272 @@ +//****************************************************************************** +// RWKV WKV7 F32 Kernel +// +// Implements the RWKV-7 linear attention recurrence: +// For each head h, timestep t, row i: +// sa = dot(a, state[i]) +// state[i] = state[i] * w + v[i]*k + sa * b +// output[i]= dot(state[i], r) +// +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_rwkv_wkv7_params { + float * r; // [S, H, T] receptance + float * w; // [S, H, T] decay + float * k; // [S, H, T] key + float * v; // [S, H, T] value + float * a; // [S, H, T] bonus gate + float * b; // [S, H, T] bonus key + float * state_in; // [S*S*H, n_seqs] initial state + float * dst; // [C, T + S*n_seqs] output + state_out + int32_t C; // total channels (S * H) + int32_t H; // number of heads + int32_t S; // head size + int32_t T; // number of tokens + int32_t n_seqs; // number of sequences +}; + +// Horizontal sum of 8-wide vector register f10 -> scalar float +static inline float hsum_f10(void) { + float result; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(result)::"t0", "f1", "f2", "f3", "f4", "f5"); + return result; +} + +int entry_point(struct ggml_et_rwkv_wkv7_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + const float * r = params->r; + const float * w = params->w; + const float * k = params->k; + const float * v = params->v; + const float * a = params->a; + const float * b = params->b; + const float * state_in = params->state_in; + float * dst_data = params->dst; + + const int32_t C = params->C; + const int32_t H = params->H; + const int32_t S = params->S; + const int32_t T = params->T; + const int32_t n_seqs = params->n_seqs; + + if (!r || !w || !k || !v || !a || !b || !state_in || !dst_data) { + return -1; + } + + const int32_t tps = T / n_seqs; // tokens per sequence + float * state_out = dst_data + C * T; + + // Fix #2: hoist w[0..S-1] across the i loop. In the inner j-loop of pass + // 2, w/k/b/r are loop-invariant w.r.t. i but were being reloaded for every + // i value (16 times redundantly after Fix #1). Pinning all four arrays + // would need 32 vector regs (won't fit), so we hoist just w — it's used + // in the critical fmadd chain and lives cleanly in f24-f31, which the + // existing kernel never touches. Saves ~20% of pass-2 load issues. + // + // GCC local register variables: declared as `float` but the underlying + // f-reg holds the wide vector loaded by flw.ps. GCC reserves f24-f31 for + // these variables for the whole function and never generates code that + // touches them on its own, so the upper 7 lanes survive between asm + // blocks. Only used when S == 64 (the RWKV-7 case); other head sizes + // fall through to the original unhoisted path. + register float w_h0 __asm__("f24"); + register float w_h1 __asm__("f25"); + register float w_h2 __asm__("f26"); + register float w_h3 __asm__("f27"); + register float w_h4 __asm__("f28"); + register float w_h5 __asm__("f29"); + register float w_h6 __asm__("f30"); + register float w_h7 __asm__("f31"); + const int wkv7_fast = (S == 64); + + // Tile i by one cache line so each hart's output writes never share a + // 64-B line with another hart's writes (the chip is non-coherent). + // For S=64 this gives 4 tiles per head; for S<16 or odd S we fall back + // to one-hart-per-head (= the original parallelism). + const int32_t i_tile = (S % 16 == 0) ? 16 : S; + const int32_t tiles_per_head = S / i_tile; + const int32_t total_units = H * tiles_per_head; + + // Parallelize across (head, i-tile) pairs. The t loop stays inside this + // unit loop so the same hart owns the same state rows across all + // timesteps — required for the recurrence to read back its own writes + // without going through L2. + for (int32_t u = thread_id; u < total_units; u += num_threads) { + const int32_t h = u / tiles_per_head; + const int32_t tile = u % tiles_per_head; + const int32_t i_start = tile * i_tile; + const int32_t i_end = i_start + i_tile; + + const int32_t h_off = h * S; // offset within C for this head + const int32_t s2d = h * S * S; // offset within state for this head + + for (int32_t t = 0; t < T; t++) { + const int32_t seq = t / tps; + const int32_t t_in_seq = t % tps; + const int32_t seq_state = seq * S * C; // state offset for this sequence + + const float * s_prev; + float * s_cur = state_out + seq_state + s2d; + + if (t_in_seq == 0) { + s_prev = state_in + seq_state + s2d; + } else { + s_prev = s_cur; + } + + // Pointers for this timestep/head + const int32_t th = t * C + h_off; + const float * r_ptr = r + th; + const float * w_ptr = w + th; + const float * k_ptr = k + th; + const float * v_ptr = v + th; + const float * a_ptr = a + th; + const float * b_ptr = b + th; + + // Hoist w[0..63] into f24-f31 once per (h, t). These values are + // invariant across the i loop below, so the inner j-unroll can + // reference them by register name and skip the per-i reload. + if (wkv7_fast) { + __asm__ volatile( + "flw.ps f24, 0(%[wp])\n" + "flw.ps f25, 32(%[wp])\n" + "flw.ps f26, 64(%[wp])\n" + "flw.ps f27, 96(%[wp])\n" + "flw.ps f28, 128(%[wp])\n" + "flw.ps f29, 160(%[wp])\n" + "flw.ps f30, 192(%[wp])\n" + "flw.ps f31, 224(%[wp])\n" + : "=f"(w_h0), "=f"(w_h1), "=f"(w_h2), "=f"(w_h3), "=f"(w_h4), "=f"(w_h5), "=f"(w_h6), "=f"(w_h7) + : [wp] "r"(w_ptr)); + } + + for (int32_t i = i_start; i < i_end; i++) { + const float * sp_row = s_prev + i * S; // state_prev row i + float * sc_row = s_cur + i * S; // state_cur row i + + // ---------------------------------------------------------- + // Step 1: sa = dot(a, state_prev[i]) + // Accumulate in f10 + // ---------------------------------------------------------- + float zero = 0.0f; + __asm__ volatile("fbc.ps f10, %[z]\n" : : [z] "m"(zero) : "f10"); + + for (int32_t j = 0; j < S; j += 8) { + __asm__ volatile( + "flw.ps f11, %[a_vec]\n" + "flw.ps f12, %[s_vec]\n" + "fmadd.ps f10, f11, f12, f10\n" + : + : [a_vec] "m"(*(const float (*)[8]) & a_ptr[j]), [s_vec] "m"(*(const float (*)[8]) & sp_row[j]) + : "f10", "f11", "f12"); + } + + float sa = hsum_f10(); + + // ---------------------------------------------------------- + // Step 2: state update + result accumulation + // kv = v[i] * k[j] + // state[j] = state[j] * w[j] + kv + sa * b[j] + // result += state[j] * r[j] + // ---------------------------------------------------------- + float v_val = v_ptr[i]; + + // Broadcast v_val and sa, zero result accumulator (f10) + __asm__ volatile( + "fbc.ps f20, %[vv]\n" + "fbc.ps f21, %[sv]\n" + "fbc.ps f10, %[z]\n" + : + : [vv] "m"(v_val), [sv] "m"(sa), [z] "m"(zero) + : "f10", "f20", "f21"); + + if (wkv7_fast) { +// Fast path: 8 chunks unrolled, w hoisted to f24-f31. +// Saves one flw per chunk vs the original loop. +#define WKV7_PASS2_CHUNK(j_off, w_var) \ + __asm__ volatile( \ + "flw.ps f11, %[s_vec]\n" \ + "flw.ps f13, %[k_vec]\n" \ + "flw.ps f14, %[b_vec]\n" \ + "flw.ps f15, %[r_vec]\n" \ + "fmul.ps f16, f20, f13\n" \ + "fmadd.ps f11, f11, %[w_h], f16\n" \ + "fmadd.ps f11, f21, f14, f11\n" \ + "fsw.ps f11, %[sc_vec]\n" \ + "fmadd.ps f10, f11, f15, f10\n" \ + : [sc_vec] "=m"(*(float (*)[8]) & sc_row[j_off]) \ + : [s_vec] "m"(*(const float (*)[8]) & sp_row[j_off]), [k_vec] "m"(*(const float (*)[8]) & k_ptr[j_off]), \ + [b_vec] "m"(*(const float (*)[8]) & b_ptr[j_off]), [r_vec] "m"(*(const float (*)[8]) & r_ptr[j_off]), \ + [w_h] "f"(w_var) \ + : "f10", "f11", "f13", "f14", "f15", "f16") + + WKV7_PASS2_CHUNK(0, w_h0); + WKV7_PASS2_CHUNK(8, w_h1); + WKV7_PASS2_CHUNK(16, w_h2); + WKV7_PASS2_CHUNK(24, w_h3); + WKV7_PASS2_CHUNK(32, w_h4); + WKV7_PASS2_CHUNK(40, w_h5); + WKV7_PASS2_CHUNK(48, w_h6); + WKV7_PASS2_CHUNK(56, w_h7); + +#undef WKV7_PASS2_CHUNK + } else { + for (int32_t j = 0; j < S; j += 8) { + __asm__ volatile( + "flw.ps f11, %[s_vec]\n" // state_prev[j..j+7] + "flw.ps f12, %[w_vec]\n" // w[j..j+7] + "flw.ps f13, %[k_vec]\n" // k[j..j+7] + "flw.ps f14, %[b_vec]\n" // b[j..j+7] + "flw.ps f15, %[r_vec]\n" // r[j..j+7] + "fmul.ps f16, f20, f13\n" // kv = v_broadcast * k + "fmadd.ps f11, f11, f12, f16\n" // state*w + kv + "fmadd.ps f11, f21, f14, f11\n" // + sa*b + "fsw.ps f11, %[sc_vec]\n" // store new state + "fmadd.ps f10, f11, f15, f10\n" // result += new_state * r + + : [sc_vec] "=m"(*(float (*)[8]) & sc_row[j]) + : [s_vec] "m"(*(const float (*)[8]) & sp_row[j]), + [w_vec] "m"(*(const float (*)[8]) & w_ptr[j]), + [k_vec] "m"(*(const float (*)[8]) & k_ptr[j]), + [b_vec] "m"(*(const float (*)[8]) & b_ptr[j]), + [r_vec] "m"(*(const float (*)[8]) & r_ptr[j]) + : "f10", "f11", "f12", "f13", "f14", "f15", "f16"); + } + } + + dst_data[th + i] = hsum_f10(); + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/scale_f32.c b/ggml/src/ggml-et/et-kernels/src/scale_f32.c new file mode 100644 index 0000000..ad0c649 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/scale_f32.c @@ -0,0 +1,94 @@ +//****************************************************************************** +// Scale F32 Kernel +// dst[i] = src0[i] * scale + bias +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_scale_params { + struct ggml_tensor src0; // F32 input tensor + struct ggml_tensor dst; // F32 output tensor + float scale; // Scale factor + float bias; // Bias (additive offset) +}; + +int entry_point(struct ggml_et_scale_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + + float scale = params->scale; + float bias = params->bias; + + // Total elements across all dimensions + const int64_t total_elements = src0->ne[0] * src0->ne[1] * src0->ne[2] * src0->ne[3]; + + // Cache line = 64 bytes = 16 floats, but vector width = 8 floats + // Parallelize at cache line granularity (16 floats) + const int64_t elements_per_cacheline = 16; + const int64_t total_cachelines = (total_elements + elements_per_cacheline - 1) / elements_per_cacheline; + + int64_t cachelines_per_thread = (total_cachelines + num_threads - 1) / num_threads; + int64_t start_cacheline = thread_id * cachelines_per_thread; + int64_t end_cacheline = start_cacheline + cachelines_per_thread; + + if (end_cacheline > total_cachelines) { + end_cacheline = total_cachelines; + } + + if (start_cacheline >= total_cachelines) { + return 0; + } + + int64_t start_elem = start_cacheline * elements_per_cacheline; + int64_t end_elem = end_cacheline * elements_per_cacheline; + if (end_elem > total_elements) { + end_elem = total_elements; + } + + unsigned long temp_mask; + __asm__ volatile("mova.x.m %0" : "=r"(temp_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + __asm__ volatile("fbc.ps f20, %[scale_ptr]\n" : : [scale_ptr] "m"(scale) : "f20"); + __asm__ volatile("fbc.ps f21, %[bias_ptr]\n" : : [bias_ptr] "m"(bias) : "f21"); + + for (int64_t i = start_elem; i < end_elem; i += 8) { + __asm__ volatile( + "flw.ps f10, %[src]\n" + "fmadd.ps f10, f10, f20, f21\n" // dst = src*scale + bias + "fsw.ps f10, %[dst_out]\n" + : [dst_out] "=m"(*(float (*)[8]) & dst_data[i]) + : [src] "m"(*(const float (*)[8]) & src0_data[i]) + : "f10", "f20", "f21"); + } + __asm__ volatile("mova.m.x %0" ::"r"(temp_mask)); + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/set_f32.c b/ggml/src/ggml-et/et-kernels/src/set_f32.c new file mode 100644 index 0000000..aea2b61 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/set_f32.c @@ -0,0 +1,101 @@ +//****************************************************************************** +// SET F32 Kernel +// Minimal ET implementation for inplace F32 SET into a contiguous destination +// using a contiguous F32 source view and explicit destination view strides. +// +// Supported shape family: +// - dst/base is contiguous F32 +// - src1 is contiguous F32 +// - src1.ne[0] is cacheline-aligned (multiple of 16 floats) +// - destination view strides/offset are cacheline-aligned +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_set_params { + struct ggml_tensor src1; + struct ggml_tensor dst; + int32_t nb1; + int32_t nb2; + int32_t nb3; + int32_t offset; +}; + +static inline void copy_row_aligned(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f11, %[src_vec]\n" + "fsw.ps f11, %[dst_vec]\n" + : [dst_vec] "=m"(*(float (*)[8]) & dst[i]) + : [src_vec] "m"(*(const float (*)[8]) & src[i]) + : "f11"); + } +} + +int entry_point(struct ggml_et_set_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * dst = ¶ms->dst; + + if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + const float * src1_data = (const float *) src1->data; + float * dst_data = (float *) dst->data; + if (!src1_data || !dst_data) { + return -1; + } + + const int64_t ne10 = src1->ne[0]; + const int64_t ne11 = src1->ne[1]; + const int64_t ne12 = src1->ne[2]; + const int64_t ne13 = src1->ne[3]; + + if (src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || ne10 % 16 != 0) { + return -1; + } + + const int64_t nb11 = src1->nb[1]; + const int64_t nb12 = src1->nb[2]; + const int64_t nb13 = src1->nb[3]; + + const int64_t dnb1 = params->nb1; + const int64_t dnb2 = params->nb2; + const int64_t dnb3 = params->nb3; + const int64_t offset = params->offset; + + const int64_t total_rows = ne11 * ne12 * ne13; + + for (int64_t row = thread_id; row < total_rows; row += num_threads) { + const int64_t i1 = row % ne11; + const int64_t i2 = (row / ne11) % ne12; + const int64_t i3 = row / (ne11 * ne12); + + const float * src_row = (const float *) ((const char *) src1_data + i1 * nb11 + i2 * nb12 + i3 * nb13); + float * dst_row = (float *) ((char *) dst_data + offset + i1 * dnb1 + i2 * dnb2 + i3 * dnb3); + + copy_row_aligned(dst_row, src_row, (int32_t) ne10); + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c b/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c new file mode 100644 index 0000000..16e1758 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c @@ -0,0 +1,394 @@ +//****************************************************************************** +// Bare Metal SET_ROWS F32 Kernel +// Writes source data rows to specific indices in destination tensor +// +// Algorithm: +// 1. Read row indices from src1 (int64 tensor) +// 2. For each source row, write it to destination at the specified index +// 3. Handle type conversion: F32 source -> F32/F16 destination +// 4. Support multi-dimensional tensor operations +// +// Operation: dst[indices[i]] = src[i] for i = 0..num_source_rows +// This is the inverse of GET_ROWS operation +// +// As ET is not a cache coherent processor yet SET_ROWS often are setting +// small mount of large rows (KV cache). There's several strategies to +// optimize this operation, including cacheline-based parallelization. +// +// - distribute work at cacheline granularity +// - if previous does not work, find the LCM of cacheline size +// +// Features supported: +// - F32 source data (always F32 input) +// - F32 and F16 destination data (with transcoding) +// - Int64 row indices (vs Int32 in GET_ROWS) +// - Multi-dimensional tensor support +// - Sequential source reads, scattered destination writes +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include +#include +#include + +#define CACHE_LINE_SIZE_BYTES 64 +#define CACHE_LINE_F32_ELEMS 16 // 64 / 4 +#define CACHE_LINE_F16_ELEMS 32 // 64 / 2 + +static int64_t gcd64(int64_t a, int64_t b) { + while (b) { + int64_t t = b; + b = a % b; + a = t; + } + return a; +} + +struct ggml_et_set_rows_params { + struct ggml_tensor src0; // F32 source data tensor + struct ggml_tensor src1; // I64 row indices tensor + struct ggml_tensor dst; // F32/F16 destination tensor +}; + +// Copy exactly one cache line (64 bytes = 16 F32 elements) using wide loads/stores +static void copy_cache_aligned_f32(float * dst, const float * src) { + __asm__ volatile( + "flq2 f0, 0(%[src]) \n\t" // Load 32 bytes + "flq2 f1, 32(%[src]) \n\t" // Load next 32 bytes + "fsq2 f0, 0(%[dst]) \n\t" // Store 32 bytes + "fsq2 f1, 32(%[dst]) \n\t" // Store next 32 bytes + : + : [src] "r"(src), [dst] "r"(dst) + : "f0", "f1", "memory"); +} + +// Convert and copy one dst cache line worth of F32->F16 (32 elements src -> 64 bytes dst) +static void copy_cache_aligned_f16(uint16_t * dst, const float * src) { + unsigned long mask_temp; + + // Build offset vector for consecutive 16-bit stores: [0, 2, 4, 6, 8, 10, 12, 14] + float offset_vec_storage[8]; + uint32_t * offsets = (uint32_t *) offset_vec_storage; + for (int j = 0; j < 8; j++) { + offsets[j] = j * 2; + } + + __asm__ volatile( + "mova.x.m %[mask_temp] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "flw.ps f1, 0(%[offsets]) \n\t" + : [mask_temp] "=&r"(mask_temp) + : [offsets] "r"(offset_vec_storage) + : "f1"); + + // 4 iterations of 8 elements = 32 F16 elements = 64 bytes = 1 cache line + for (int i = 0; i < 32; i += 8) { + __asm__ volatile( + "flw.ps f2, 0(%[src_ptr]) \n\t" + "fcvt.f16.ps f3, f2 \n\t" + "fsch.ps f3, f1(%[dst_ptr]) \n\t" + : + : [src_ptr] "r"(src + i), [dst_ptr] "r"(dst + i) + : "f2", "f3", "memory"); + } + + __asm__ volatile("mova.m.x %[mask_temp] \n\t" : : [mask_temp] "r"(mask_temp)); +} + +static inline size_t tensor_bytes(const struct ggml_tensor * t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} + +int entry_point(struct ggml_et_set_rows_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + struct ggml_tensor * src0 = ¶ms->src0; // Source data tensor (F32) + struct ggml_tensor * src1 = ¶ms->src1; // Row indices tensor (I64) + struct ggml_tensor * dst = ¶ms->dst; // Destination tensor (F32/F16) + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I64) { + return -1; // Invalid source types + } + + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) { + return -1; // Unsupported destination type + } + + float * src0_data = (float *) src0->data; + int64_t * src1_data = (int64_t *) src1->data; + void * dst_data = dst->data; + + if (!src0_data || !src1_data || !dst_data) { + return -1; // Null data pointer + } + + const int64_t ne00 = src0->ne[0]; // Source columns (row width) + const int64_t ne01 = src0->ne[1]; // Source rows (number of rows to write) + const int64_t ne02 = src0->ne[2]; // Source batch dimension + const int64_t ne03 = src0->ne[3]; // Source outer batch dimension + + const int64_t nb01 = src0->nb[1]; + const int64_t nb02 = src0->nb[2]; + const int64_t nb03 = src0->nb[3]; + + const int64_t ne10 = src1->ne[0]; // Number of indices in dimension 0 + const int64_t ne11 = src1->ne[1]; // Number of indices in dimension 1 + const int64_t ne12 = src1->ne[2]; // Batch dimension for indices + + const int64_t nb10 = src1->nb[0]; + const int64_t nb11 = src1->nb[1]; + const int64_t nb12 = src1->nb[2]; + + const int64_t ne_dst1 = dst->ne[1]; // Number of rows in destination (for bounds checking) + + const int64_t nb1 = dst->nb[1]; + const int64_t nb2 = dst->nb[2]; + const int64_t nb3 = dst->nb[3]; + + // Validate that number of indices matches number of source rows + if (ne10 != ne01) { + return -1; // Number of indices must match number of source rows + } +#ifdef ET_UBERKERNEL + evict_region_past_l2(params->src0.data, tensor_bytes(¶ms->src0)); + evict_region_past_l2(params->src1.data, tensor_bytes(¶ms->src1)); + FENCE; + et_barrier(ET_BARRIER_GLOBAL); +#endif + const int64_t total_rows = ne01 * ne02 * ne03; + + // Determine cache-line element count based on destination type + const int64_t dst_cl_elems = (dst->type == GGML_TYPE_F16) ? CACHE_LINE_F16_ELEMS : CACHE_LINE_F32_ELEMS; + + // Check if rows are cache-line aligned in the destination + const bool row_cache_aligned = (ne00 >= dst_cl_elems) && (ne00 % dst_cl_elems == 0); + + if (row_cache_aligned) { + // Cache-aligned path: distribute dst cache lines across threads + // Each thread owns complete cache lines -> no coherence conflicts + const int64_t cls_per_row = ne00 / dst_cl_elems; + const int64_t total_cls = total_rows * cls_per_row; + const int64_t cls_per_thread = (total_cls + num_threads - 1) / num_threads; + const int64_t my_start = thread_id * cls_per_thread; + int64_t my_end = my_start + cls_per_thread; + if (my_end > total_cls) { + my_end = total_cls; + } + if (my_start >= total_cls) { + return 0; + } + + for (int64_t cl = my_start; cl < my_end; cl++) { + // Map flat cache-line index -> (row, offset within row) + const int64_t row_flat = cl / cls_per_row; + const int64_t cl_in_row = cl % cls_per_row; + + // Decompose flat row -> (i03, i02, i01) + const int64_t i01 = row_flat % ne01; + const int64_t tmp = row_flat / ne01; + const int64_t i02 = tmp % ne02; + const int64_t i03 = tmp / ne02; + + // Look up destination row index + const int64_t i12 = i03 % ne12; + const int64_t i11 = i02 % ne11; + const int64_t i10 = i01; + const int64_t index_byte_offset = i10 * nb10 + i11 * nb11 + i12 * nb12; + const int64_t dst_row_index = *(int64_t *) ((char *) src1_data + index_byte_offset); + + if (dst_row_index < 0 || dst_row_index >= ne_dst1) { + return -1; + } + + // Source pointer: row base + cache-line offset (always F32 source) + const int64_t elem_offset = cl_in_row * dst_cl_elems; + const float * src_ptr = + (const float *) ((char *) src0_data + i01 * nb01 + i02 * nb02 + i03 * nb03) + elem_offset; + + // Destination pointer: scattered row base + cache-line offset + char * dst_row_base = (char *) dst_data + dst_row_index * nb1 + i02 * nb2 + i03 * nb3; + + if (dst->type == GGML_TYPE_F32) { + float * dst_ptr = (float *) dst_row_base + elem_offset; + copy_cache_aligned_f32(dst_ptr, src_ptr); + } else { + uint16_t * dst_ptr = (uint16_t *) dst_row_base + elem_offset; + copy_cache_aligned_f16(dst_ptr, src_ptr); + } + } + } else if (nb1 % CACHE_LINE_SIZE_BYTES == 0) { + // LCM-aligned path: destination row stride is cache-line-aligned, so + // scattered rows never share a cache line even though ne00 doesn't + // fill complete cache lines. Group rows via lcm(ne00, dst_cl_elems) + // and distribute cache lines across threads — each thread exclusively + // owns its cache lines, so normal stores are safe (no atomics needed). + const int64_t g = gcd64(ne00, dst_cl_elems); + const int64_t rows_per_group = dst_cl_elems / g; // lcm / ne00 + const int64_t cls_per_group = ne00 / g; // lcm / dst_cl_elems + + const int64_t total_groups = (total_rows + rows_per_group - 1) / rows_per_group; + const int64_t total_cls = total_groups * cls_per_group; + const int64_t cls_per_thread = (total_cls + num_threads - 1) / num_threads; + const int64_t my_start = thread_id * cls_per_thread; + int64_t my_end = my_start + cls_per_thread; + if (my_end > total_cls) { + my_end = total_cls; + } + if (my_start >= total_cls) { + return 0; + } + +#ifdef BUILD_FOR_UBERKERNEL + et_barrier(ET_BARRIER_GLOBAL); + // evict_region_past_l2(src0_data, tensor_bytes(src0)); + // evict_region_past_l2(src1_data, tensor_bytes(src1)); + // // et_barrier(ET_BARRIER_GLOBAL); + // FENCE; +#endif + + + for (int64_t cl = my_start; cl < my_end; cl++) { + const int64_t group_idx = cl / cls_per_group; + const int64_t cl_in_group = cl % cls_per_group; + + // Element range [elem_start, elem_end) within the flattened group + const int64_t elem_start = cl_in_group * dst_cl_elems; + const int64_t elem_end = elem_start + dst_cl_elems; + + // Which row(s) inside this group does the cache line touch? + const int64_t r_first = elem_start / ne00; + const int64_t r_last = (elem_end - 1) / ne00; + + for (int64_t r = r_first; r <= r_last; r++) { + const int64_t row_flat = group_idx * rows_per_group + r; + if (row_flat >= total_rows) { + break; + } + + // Column range within this row + int64_t col_begin = (r == r_first) ? (elem_start - r * ne00) : 0; + int64_t col_end = (r == r_last) ? (elem_end - r * ne00) : ne00; + if (col_end > ne00) { + col_end = ne00; + } + + // Decompose flat row -> (i03, i02, i01) + const int64_t i01 = row_flat % ne01; + const int64_t tmp = row_flat / ne01; + const int64_t i02 = tmp % ne02; + const int64_t i03 = tmp / ne02; + + // Look up destination row index + const int64_t i12 = i03 % ne12; + const int64_t i11 = i02 % ne11; + const int64_t i10 = i01; + const int64_t index_byte_offset = i10 * nb10 + i11 * nb11 + i12 * nb12; + const int64_t dst_row_index = *(int64_t *) ((char *) src1_data + index_byte_offset); + + if (dst_row_index < 0 || dst_row_index >= ne_dst1) { + return -1; + } + + const float * src_row = (const float *) ((char *) src0_data + i01 * nb01 + i02 * nb02 + i03 * nb03); + char * dst_row_base = (char *) dst_data + dst_row_index * nb1 + i02 * nb2 + i03 * nb3; + + // nb1 is cache-line-aligned, so dst_row_base is too. + // Use aligned copy when the column range fills a complete + // cache line at a cache-line-aligned offset within the row. + const bool full_cl = (col_begin % dst_cl_elems == 0) && (col_end - col_begin == dst_cl_elems); + + if (dst->type == GGML_TYPE_F32) { + float * dp = (float *) dst_row_base; + if (full_cl) { + copy_cache_aligned_f32(dp + col_begin, src_row + col_begin); + } else { + for (int64_t i = col_begin; i < col_end; i++) { + dp[i] = src_row[i]; + } + } + } else { + uint16_t * dp = (uint16_t *) dst_row_base; + if (full_cl) { + copy_cache_aligned_f16(dp + col_begin, src_row + col_begin); + } else { + for (int64_t i = col_begin; i < col_end; i++) { + dp[i] = fp32_to_fp16(src_row[i]); + } + } + } + } + } + +#ifdef BUILD_FOR_UBERKERNEL + et_barrier(ET_BARRIER_GLOBAL); + // evict_region_past_l2(src0_data, tensor_bytes(src0)); + // evict_region_past_l2(src1_data, tensor_bytes(src1)); + // // et_barrier(ET_BARRIER_GLOBAL); + // FENCE; +#endif + + + } else { + // Fallback: nb1 not cache-line-aligned, so scattered destination rows + // may share a cache line. Use atomic global stores to bypass L1D. + for (int64_t row_flat = thread_id; row_flat < total_rows; row_flat += num_threads) { + const int64_t i01 = row_flat % ne01; + const int64_t tmp = row_flat / ne01; + const int64_t i02 = tmp % ne02; + const int64_t i03 = tmp / ne02; + + // Look up destination row index + const int64_t i12 = i03 % ne12; + const int64_t i11 = i02 % ne11; + const int64_t i10 = i01; + const int64_t index_byte_offset = i10 * nb10 + i11 * nb11 + i12 * nb12; + const int64_t dst_row_index = *(int64_t *) ((char *) src1_data + index_byte_offset); + + if (dst_row_index < 0 || dst_row_index >= ne_dst1) { + return -1; + } + + const float * src_row = (const float *) ((char *) src0_data + i01 * nb01 + i02 * nb02 + i03 * nb03); + char * dst_row_base = (char *) dst_data + dst_row_index * nb1 + i02 * nb2 + i03 * nb3; + + if (dst->type == GGML_TYPE_F32) { + volatile float * dst_row = (volatile float *) dst_row_base; + for (int64_t i = 0; i < ne00; i++) { + atomic_store_f32(dst_row + i, src_row[i]); + } + } else { + volatile uint16_t * dst_row = (volatile uint16_t *) dst_row_base; + for (int64_t i = 0; i < ne00; i++) { + atomic_store_f16(dst_row + i, fp32_to_fp16(src_row[i])); + } + } + } + } + +#ifdef BUILD_FOR_UBERKERNEL + et_barrier(ET_BARRIER_GLOBAL); + // evict_region_past_l2(src0_data, tensor_bytes(src0)); + // evict_region_past_l2(src1_data, tensor_bytes(src1)); + // // et_barrier(ET_BARRIER_GLOBAL); + // FENCE; +#endif + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/softmax_f32.c b/ggml/src/ggml-et/et-kernels/src/softmax_f32.c new file mode 100644 index 0000000..5b322db --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/softmax_f32.c @@ -0,0 +1,698 @@ +//****************************************************************************** +// Bare Metal Softmax F32 Kernel +// Softmax function: y[i] = exp(x[i] - max) / sum(exp(x[j] - max)) +// +// Algorithm: +// 1. Apply scaling: x' = x * scale +// 2. Add mask/bias if present: x' = x' + mask * slope (ALiBi support) +// 3. Find max value for numerical stability: max = max(x') +// 4. Compute exponentials: exp_vals[i] = exp(x'[i] - max) +// 5. Compute sum: sum = sum(exp_vals) +// 6. Normalize: y[i] = exp_vals[i] / sum +// +// Features supported: +// - Temperature scaling via scale parameter +// - Attention masking (transformer masks) +// - ALiBi (Attention with Linear Biases) positional encoding +// - Numerical stability (subtract max before exp) +// - ggml broadcasting rules for mask tensors +// +// Mask Broadcasting Rules (ggml-specific, not standard numpy): +// - Dimension 0: mask.ne[0] == input.ne[0] (exact match required) +// - Dimension 1: mask.ne[1] >= input.ne[1] (allows larger pre-allocated masks) +// - Dimension 2: input.ne[2] % mask.ne[2] == 0 (modulo broadcasting) +// - Dimension 3: input.ne[3] % mask.ne[3] == 0 (modulo broadcasting) +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include +#include +#include +#include + +// Softmax kernel parameters structure (from ggml-et-ops.h) +struct ggml_et_softmax_params { + struct ggml_tensor src0; // F32 input tensor + struct ggml_tensor src1; // F32 mask tensor (optional, may be zeroed if not used) + struct ggml_tensor src2; // F32 sinks tensor (optional, may be zeroed if not used) + struct ggml_tensor dst; // F32 output tensor + float scale; // Scale factor (temperature scaling) + float max_bias; // Max bias for ALiBi (0.0f if not used) +}; + +#define LOG2E_F 1.4426950408889634f + +typedef struct { + float max_val; + float sum_val; + uint32_t valid_mask; +} softmax_params_t; + +static inline bool softmax_lane_is_valid(float x) { + return (x == x) && (x != -INFINITY) && (x != INFINITY); +} + +static inline softmax_params_t softmax_params_empty(void) { + softmax_params_t p; + p.max_val = -INFINITY; + p.sum_val = 0.0f; + p.valid_mask = 0; + return p; +} + +// chunk_transform_ps_8_branchless_mask +// +// Vector transform for 8 logits: +// +// x = src * scale + (mask ? mask * slope : 0) +// +// Implemented branchlessly so masked and unmasked paths share the same +// instruction stream. Used by pass1 and pass2 vector loops. +static inline void chunk_transform_ps_8_branchless_mask(float * tmp8, + const float * src, + const float * mask, + float scale, + float slope) { + unsigned long ms; + const float zero = 0.0f; + const unsigned long mask_load_m0 = (mask != NULL) ? 0xFFul : 0x00ul; + const float * mp = (mask != NULL) ? mask : &zero; + + __asm__ volatile( + "mova.x.m %[ms] \n\t" + + "mov.m.x m0, x0, 0xFF \n\t" + "fbc.ps f10, 0(%[p_scale]) \n\t" + "fbc.ps f11, 0(%[p_slope]) \n\t" + "fbc.ps f1, 0(%[p_zero]) \n\t" + + "mov.m.x m0, %[maskm0], 0 \n\t" // load mask if needed + "flw.ps f1, 0(%[mp]) \n\t" + + "mov.m.x m0, x0, 0xFF \n\t" + + "flw.ps f0, 0(%[sp]) \n\t" + "fmul.ps f0, f0, f10 \n\t" + "fmul.ps f1, f1, f11 \n\t" + "fadd.ps f0, f0, f1, rne \n\t" + "fsw.ps f0, 0(%[tp]) \n\t" + + "mova.m.x %[ms] \n\t" + : [ms] "=&r"(ms) + : [tp] "r"(tmp8), [sp] "r"(src), [mp] "r"(mp), [p_zero] "r"(&zero), [p_scale] "r"(&scale), + [p_slope] "r"(&slope), [maskm0] "r"(mask_load_m0) + : "f0", "f1", "f10", "f11", "memory"); +} + +// chunk_transform_ps_8_tail +// +// Same as chunk_transform_ps_8_branchless_mask but gates loads, compute, +// and stores with a caller-supplied m0 mask so that only `count` elements +// (1-7) are touched. Used for the last sub-8 chunk of a non-aligned row. +static inline void chunk_transform_ps_8_tail(float * tmp8, + const float * src, + const float * mask, + float scale, + float slope, + unsigned long tail_m0) { + unsigned long ms; + const float zero = 0.0f; + const unsigned long mask_load_m0 = (mask != NULL) ? tail_m0 : 0x00ul; + const float * mp = (mask != NULL) ? mask : &zero; + + __asm__ volatile( + "mova.x.m %[ms] \n\t" + + // Broadcast constants with all lanes enabled + "mov.m.x m0, x0, 0xFF \n\t" + "fbc.ps f10, 0(%[p_scale]) \n\t" + "fbc.ps f11, 0(%[p_slope]) \n\t" + "fbc.ps f1, 0(%[p_zero]) \n\t" + + // Load mask data gated by tail mask + "mov.m.x m0, %[maskm0], 0 \n\t" + "flw.ps f1, 0(%[mp]) \n\t" + + // Load source, compute, and store gated by tail mask + "mov.m.x m0, %[tailm0], 0 \n\t" + + "flw.ps f0, 0(%[sp]) \n\t" + "fmul.ps f0, f0, f10 \n\t" + "fmul.ps f1, f1, f11 \n\t" + "fadd.ps f0, f0, f1, rne \n\t" + "fsw.ps f0, 0(%[tp]) \n\t" + + "mova.m.x %[ms] \n\t" + : [ms] "=&r"(ms) + : [tp] "r"(tmp8), [sp] "r"(src), [mp] "r"(mp), [p_zero] "r"(&zero), [p_scale] "r"(&scale), + [p_slope] "r"(&slope), [maskm0] "r"(mask_load_m0), [tailm0] "r"(tail_m0) + : "f0", "f1", "f10", "f11", "memory"); +} + +// softmax_pass1_range +// +// Computes the numerically-stable softmax scan over a sub-range of a row. +// +// This implements the 1st pass of online softmax +// +// max' = max(max, x) +// sum' = sum * exp(old_max - max') + exp(x - max') +// +// and returns a partial result containing: +// +// - max_val : maximum logit observed in this range +// - sum_val : exp-normalized sum relative to max_val +// +// These partial results can be merged with softmax_params_merge() to obtain +// the result for the full row. +static inline softmax_params_t softmax_pass1_range(const float * src, + const float * mask, + int begin, + int end, + float scale, + float slope) { + __attribute__((aligned(32))) float lane_max[8]; + __attribute__((aligned(32))) float lane_sum[8]; + __attribute__((aligned(32))) float tmp[8]; + + uint8_t valid_mask = 0; + + const float one_f = 1.0f; + const float zero_f = 0.0f; + const float neg_inf = -INFINITY; + const float log2e = LOG2E_F; + + unsigned long ms; + + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "fbc.ps f20, 0(%[p_ninf]) \n\t" + "fbc.ps f21, 0(%[p_zero]) \n\t" + "fbc.ps f22, 0(%[p_one]) \n\t" + "fbc.ps f23, 0(%[p_log2e]) \n\t" + : [ms] "=&r"(ms) + : [p_ninf] "r"(&neg_inf), [p_zero] "r"(&zero_f), [p_one] "r"(&one_f), [p_log2e] "r"(&log2e) + : "f20", "f21", "f22", "f23"); + + const int aligned_end = begin + ((end - begin) & ~7); + + // Process full 8-element chunks + int i = begin; + for (; i < aligned_end; i += 8) { + chunk_transform_ps_8_branchless_mask(tmp, src + i, mask ? (mask + i) : NULL, scale, slope); + + uint8_t cur_mask = 0; + for (int j = 0; j < 8; ++j) { + if (softmax_lane_is_valid(tmp[j])) { + cur_mask |= (uint8_t) (1u << j); + } + } + + const uint8_t init_mask = (uint8_t) (cur_mask & ~valid_mask); + const uint8_t upd_mask = (uint8_t) (cur_mask & valid_mask); + + if (init_mask || upd_mask) { + __asm__ volatile( + "flw.ps f0, 0(%[p_tmp]) \n\t" + + "mov.m.x m0, %[initm], 0 \n\t" + "fcmovm.ps f20, f0, f20 \n\t" + "fcmovm.ps f21, f22, f21 \n\t" + + "mov.m.x m0, %[updm], 0 \n\t" + "fmax.ps f1, f20, f0 \n\t" + + "fsub.ps f2, f20, f1, rne \n\t" + "fmul.ps f2, f2, f23 \n\t" + "fexp.ps f2, f2 \n\t" + + "fsub.ps f3, f0, f1, rne \n\t" + "fmul.ps f3, f3, f23 \n\t" + "fexp.ps f3, f3 \n\t" + + "fmul.ps f21, f21, f2 \n\t" + "fadd.ps f21, f21, f3, rne \n\t" + "fcmovm.ps f20, f1, f20 \n\t" + + "mov.m.x m0, x0, 0xFF \n\t" + : + : [p_tmp] "r"(tmp), [initm] "r"((unsigned long) init_mask), [updm] "r"((unsigned long) upd_mask) + : "f0", "f1", "f2", "f3", "memory"); + + valid_mask |= cur_mask; + } + } + + // Tail chunk: m0-gated load/compute/store for remaining 1-7 elements + if (i < end) { + const unsigned long tail_m0 = (1ul << (end - i)) - 1; + + // Fill tmp with NaN so invalid lanes fail softmax_lane_is_valid + for (int j = 0; j < 8; j++) { + tmp[j] = __builtin_nanf(""); + } + + chunk_transform_ps_8_tail(tmp, src + i, mask ? (mask + i) : NULL, scale, slope, tail_m0); + + uint8_t cur_mask = 0; + for (int j = 0; j < 8; ++j) { + if (softmax_lane_is_valid(tmp[j])) { + cur_mask |= (uint8_t) (1u << j); + } + } + + const uint8_t init_mask = (uint8_t) (cur_mask & ~valid_mask); + const uint8_t upd_mask = (uint8_t) (cur_mask & valid_mask); + + if (init_mask || upd_mask) { + __asm__ volatile( + "flw.ps f0, 0(%[p_tmp]) \n\t" + + "mov.m.x m0, %[initm], 0 \n\t" + "fcmovm.ps f20, f0, f20 \n\t" + "fcmovm.ps f21, f22, f21 \n\t" + + "mov.m.x m0, %[updm], 0 \n\t" + "fmax.ps f1, f20, f0 \n\t" + + "fsub.ps f2, f20, f1, rne \n\t" + "fmul.ps f2, f2, f23 \n\t" + "fexp.ps f2, f2 \n\t" + + "fsub.ps f3, f0, f1, rne \n\t" + "fmul.ps f3, f3, f23 \n\t" + "fexp.ps f3, f3 \n\t" + + "fmul.ps f21, f21, f2 \n\t" + "fadd.ps f21, f21, f3, rne \n\t" + "fcmovm.ps f20, f1, f20 \n\t" + + "mov.m.x m0, x0, 0xFF \n\t" + : + : [p_tmp] "r"(tmp), [initm] "r"((unsigned long) init_mask), [updm] "r"((unsigned long) upd_mask) + : "f0", "f1", "f2", "f3", "memory"); + + valid_mask |= cur_mask; + } + } + + __asm__ volatile( + "mov.m.x m0, x0, 0xFF \n\t" + "fsw.ps f20, 0(%[p_lmax]) \n\t" + "fsw.ps f21, 0(%[p_lsum]) \n\t" + "mova.m.x %[ms] \n\t" + : + : [p_lmax] "r"(lane_max), [p_lsum] "r"(lane_sum), [ms] "r"(ms) + : "memory"); + + softmax_params_t out = softmax_params_empty(); + out.valid_mask = valid_mask; + + for (int k = 0; k < 8; ++k) { + if (valid_mask & (1u << k)) { + if (out.valid_mask == (1u << k) || out.max_val == -INFINITY || lane_max[k] > out.max_val) { + out.max_val = lane_max[k]; + } + } + } + + if (out.max_val != -INFINITY) { + // Compute lane correction factors via fexp.ps to stay consistent + // with the fexp.ps used inside the online softmax loop above. + // corr[k] = exp2((lane_max[k] - out.max_val) * LOG2E) = exp(lane_max[k] - out.max_val) + const float neg_max_l2 = -out.max_val * LOG2E_F; + __attribute__((aligned(32))) float corr[8]; + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "fbc.ps f0, 0(%[p_nml2]) \n\t" + "fbc.ps f2, 0(%[p_l2e]) \n\t" + "flw.ps f1, 0(%[p_lmax]) \n\t" + "fmadd.ps f0, f1, f2, f0 \n\t" + "fexp.ps f0, f0 \n\t" + "fsw.ps f0, 0(%[p_corr]) \n\t" + "mova.m.x %[ms] \n\t" + : + : [p_nml2] "r"(&neg_max_l2), [p_l2e] "r"(&log2e), [p_lmax] "r"(lane_max), [p_corr] "r"(corr), [ms] "r"(ms) + : "f0", "f1", "f2", "memory"); + for (int k = 0; k < 8; ++k) { + if (valid_mask & (1u << k)) { + out.sum_val += lane_sum[k] * corr[k]; + } + } + } + + return out; +} + +// Pass 2 (normalize) over [begin, end). +// +// Computes: dst[i] = exp(x[i]*scale + mask[i]*slope - max) / sum +// +// Uses fexp.ps for the numerator; the denominator (params.sum_val) must +// already be fully computed by the caller (pass1 + any sink merge). +static inline void softmax_pass2_range(float * dst, + const float * src, + const float * mask, + int begin, + int end, + float scale, + float slope, + softmax_params_t params) { + const float s2 = scale * LOG2E_F; + const float sl2 = slope * LOG2E_F; + const float neg_ml2 = -params.max_val * LOG2E_F; + const float inv_sum = et_fdiv(1.0f, params.sum_val); + + unsigned long ms; + + __asm__ volatile( + "mova.x.m %[ms] \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + "fbc.ps f10, 0(%[p_s2]) \n\t" + "fbc.ps f12, 0(%[p_nml2]) \n\t" + "fbc.ps f13, 0(%[p_inv]) \n\t" + : [ms] "=&r"(ms) + : [p_s2] "r"(&s2), [p_nml2] "r"(&neg_ml2), [p_inv] "r"(&inv_sum) + : "f10", "f12", "f13"); + + const int aligned_end = begin + ((end - begin) & ~7); + + if (mask != NULL) { + __asm__ volatile("fbc.ps f11, 0(%[p_sl2]) \n\t" : : [p_sl2] "r"(&sl2) : "f11"); + + for (int c = begin; c < aligned_end; c += 8) { + __asm__ volatile( + "flw.ps f0, 0(%[sp]) \n\t" + "flw.ps f1, 0(%[mp]) \n\t" + "fmadd.ps f0, f0, f10, f12 \n\t" + "fmadd.ps f0, f1, f11, f0 \n\t" + "fexp.ps f0, f0 \n\t" + "fmul.ps f0, f0, f13 \n\t" + "fsw.ps f0, 0(%[dp]) \n\t" + : + : [sp] "r"(src + c), [mp] "r"(mask + c), [dp] "r"(dst + c) + : "f0", "f1", "memory"); + } + + // Tail chunk with m0 gating + if (aligned_end < end) { + const unsigned long tail_m0 = (1ul << (end - aligned_end)) - 1; + __asm__ volatile( + "mov.m.x m0, %[tm], 0 \n\t" + "flw.ps f0, 0(%[sp]) \n\t" + "flw.ps f1, 0(%[mp]) \n\t" + "fmadd.ps f0, f0, f10, f12 \n\t" + "fmadd.ps f0, f1, f11, f0 \n\t" + "fexp.ps f0, f0 \n\t" + "fmul.ps f0, f0, f13 \n\t" + "fsw.ps f0, 0(%[dp]) \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + : + : [sp] "r"(src + aligned_end), [mp] "r"(mask + aligned_end), [dp] "r"(dst + aligned_end), + [tm] "r"(tail_m0) + : "f0", "f1", "memory"); + } + } else { + for (int c = begin; c < aligned_end; c += 8) { + __asm__ volatile( + "flw.ps f0, 0(%[sp]) \n\t" + "fmadd.ps f0, f0, f10, f12 \n\t" + "fexp.ps f0, f0 \n\t" + "fmul.ps f0, f0, f13 \n\t" + "fsw.ps f0, 0(%[dp]) \n\t" + : + : [sp] "r"(src + c), [dp] "r"(dst + c) + : "f0", "memory"); + } + + // Tail chunk with m0 gating + if (aligned_end < end) { + const unsigned long tail_m0 = (1ul << (end - aligned_end)) - 1; + __asm__ volatile( + "mov.m.x m0, %[tm], 0 \n\t" + "flw.ps f0, 0(%[sp]) \n\t" + "fmadd.ps f0, f0, f10, f12 \n\t" + "fexp.ps f0, f0 \n\t" + "fmul.ps f0, f0, f13 \n\t" + "fsw.ps f0, 0(%[dp]) \n\t" + "mov.m.x m0, x0, 0xFF \n\t" + : + : [sp] "r"(src + aligned_end), [dp] "r"(dst + aligned_end), [tm] "r"(tail_m0) + : "f0", "memory"); + } + } + + __asm__ volatile("mova.m.x %[ms] \n\t" ::[ms] "r"(ms)); +} + +// Single-core row path. +// pass1_range and pass2_range handle non-8-aligned cols internally via +// m0-gated tail chunks, so this function just passes cols directly. +static inline void compute_softmax_row(float * dst, + const float * src, + const float * mask, + int cols, + float scale, + float slope, + float sink_value, + bool use_sinks) { + softmax_params_t params = softmax_pass1_range(src, mask, 0, cols, scale, slope); + + if (use_sinks) { + // For sinks, use fully scalar et_expf to match the reference CPU + // backend's expf precision. Sink tests use small arrays (ne<=32) + // so the scalar path has negligible performance impact. + float max_val = params.max_val; + if (sink_value > max_val) { + max_val = sink_value; + } + + // Compute sum = Σ exp(x'[i] - max) + exp(sink - max) (scalar) + float sum = 0.0f; + for (int i = 0; i < cols; ++i) { + float x = src[i] * scale; + if (mask != NULL) { + x += mask[i] * slope; + } + sum += et_expf(x - max_val); + } + sum += et_expf(sink_value - max_val); + + // Normalize: dst[i] = exp(x'[i] - max) / sum (scalar) + float inv_sum = et_fdiv(1.0f, sum); + for (int i = 0; i < cols; ++i) { + float x = src[i] * scale; + if (mask != NULL) { + x += mask[i] * slope; + } + dst[i] = et_expf(x - max_val) * inv_sum; + } + } else { + if (!params.valid_mask) { + return; + } + softmax_pass2_range(dst, src, mask, 0, cols, scale, slope, params); + } +} + +// Main entry point for Softmax kernel +int entry_point(struct ggml_et_softmax_params * params, void * env) { + // Cast env to proper type + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + // Validate environment pointer + if (!kernel_env) { + return -1; + } + + // Get thread info using shire mask from environment + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + // Return early if this hart is not active + if (thread_id < 0) { + return 0; + } + + // Basic safety check on params + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + // Extract tensor references + struct ggml_tensor * src0 = ¶ms->src0; // Input tensor + struct ggml_tensor * src1 = ¶ms->src1; // Mask tensor (optional) + struct ggml_tensor * src2 = ¶ms->src2; // Sinks tensor (optional) + struct ggml_tensor * dst = ¶ms->dst; // Output tensor + float scale = params->scale; // Scale factor + float max_bias = params->max_bias; // ALiBi max bias + + // Validate tensor types (F32 only) + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; // Unsupported type combination + } + + // Check if mask is used and validate type + bool use_mask = (src1->data != NULL && (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16)); + + bool use_sinks = (src2->data != NULL && src2->type == GGML_TYPE_F32); + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + float * mask_data = use_mask ? (float *) src1->data : NULL; + float * sinks_data = use_sinks ? (float *) src2->data : NULL; + + if (!src0_data || !dst_data) { + return -1; // Null data pointer + } + + + const int64_t ne00 = src0->ne[0]; // Sequence length (columns) + const int64_t ne01 = src0->ne[1]; // Number of rows + const int64_t ne02 = src0->ne[2]; // Batch/head dimension + const int64_t ne03 = src0->ne[3]; // Outer batch dimension + + // Fast path: softmax of a single element is always 1.0 + // (exp(x) / exp(x) == 1 for any x, regardless of scale/mask/bias) + // Skip all ALiBi, mask, and sink setup. + // + // Each output element is 4 bytes. A cache line is 64 bytes = 16 floats. + // L1 is not coherent across harts, so each thread must own whole cache + // lines to avoid cross-hart conflicts. + if (ne00 == 1) { + const int64_t total_elems = ne01 * ne02 * ne03; + const int64_t elems_per_cl = ET_CACHE_LINE_SIZE_BYTES / (int64_t) sizeof(float); // 16 + const int64_t total_cls = (total_elems + elems_per_cl - 1) / elems_per_cl; + + for (int64_t cl = thread_id; cl < total_cls; cl += num_threads) { + const int64_t start = cl * elems_per_cl; + int64_t end = start + elems_per_cl; + if (end > total_elems) { + end = total_elems; + } + for (int64_t idx = start; idx < end; idx++) { + dst_data[idx] = 1.0f; + } + } + return 0; + } + + const int64_t ne10 = use_mask ? src1->ne[0] : 0; // Mask sequence length + const int64_t ne11 = use_mask ? src1->ne[1] : 0; // Mask rows + const int64_t ne12 = use_mask ? src1->ne[2] : 0; // Mask batch/head dimension + const int64_t ne13 = use_mask ? src1->ne[3] : 0; // Mask outer batch dimension + + if (use_mask) { + // - Dimension 0: mask must equal input exactly + // - Dimension 1: mask must be >= input (allows larger pre-allocated masks) + // - Dimension 2: input must be divisible by mask (modulo broadcasting) + // - Dimension 3: input must be divisible by mask (modulo broadcasting) + if (ne10 != ne00 || // Dimension 0: exact match required + ne11 < ne01 || // Dimension 1: mask >= input + (ne12 > 0 && ne02 % ne12 != 0) || // Dimension 2: input % mask == 0 + (ne13 > 0 && ne03 % ne13 != 0)) { // Dimension 3: input % mask == 0 + return -1; // Incompatible dimensions for ggml softmax broadcasting + } + } + + // ALiBi slope calculation - compute per attention head + const uint32_t n_head = (uint32_t) ne02; + uint32_t n_head_log2 = 0; + float m0 = 1.0f; + float m1 = 1.0f; + + if (max_bias > 0.0f) { + // This is equivalent to: 1 << floor(log2(n_head)) + n_head_log2 = 1; + while (n_head_log2 < n_head) { + n_head_log2 <<= 1; + } + if (n_head_log2 > n_head) { + n_head_log2 >>= 1; + } + + // Compute base slopes for ALiBi + // m0 = 2^(-max_bias / n_head_log2) + // m1 = 2^(-max_bias / (2 * n_head_log2)) + float inv_n_head_log2 = et_fdiv(1.0f, (float) n_head_log2); + m0 = et_expf(-max_bias * 0.69314718f * inv_n_head_log2); // 0.69314718 = ln(2) + m1 = et_expf(-max_bias * 0.69314718f * inv_n_head_log2 * 0.5f); + } + + // Process tensor row by row in parallel across flattened rows. + // Flattened row index spans [i03, i02, i01] with row length ne00. + // + // When ne00 * sizeof(float) is not a multiple of the cache line size, + // adjacent rows share cache lines. Assign contiguous write groups to + // each thread so every thread's write footprint covers whole cache + // lines, preventing cross-hart L1 coherency issues. When rows ARE + // cache-line aligned, rows_per_wg == 1 and this degenerates to the + // original stride-by-num_threads distribution. + const int64_t rows_per_i03 = ne02 * ne01; + const int64_t total_rows = ne03 * rows_per_i03; + const int64_t rows_per_wg = et_rows_per_cacheline_group(ne00, sizeof(float)); + const int64_t total_wgs = (total_rows + rows_per_wg - 1) / rows_per_wg; + + for (int64_t wg = thread_id; wg < total_wgs; wg += num_threads) { + const int64_t row_start = wg * rows_per_wg; + int64_t row_end = row_start + rows_per_wg; + if (row_end > total_rows) { + row_end = total_rows; + } + + for (int64_t row = row_start; row < row_end; row++) { + const int64_t i03 = row / rows_per_i03; + const int64_t rem = row % rows_per_i03; + const int64_t i02 = rem / ne01; + const int64_t i01 = rem % ne01; + + // Calculate ALiBi slope for this attention head + float slope = 1.0f; + if (max_bias > 0.0f) { + const uint32_t h = (uint32_t) i02; // head index + if (h < n_head_log2) { + slope = m0; + for (uint32_t i = 0; i < h; i++) { + slope *= m0; + } + } else { + const uint32_t exp = 2 * (h - n_head_log2) + 1; + slope = m1; + for (uint32_t i = 1; i < exp; i++) { + slope *= m1; + } + } + } + + float sink_value = 0.0f; + if (use_sinks && sinks_data) { + sink_value = sinks_data[i02]; + } + + const int64_t src_offset = i03 * ne02 * ne01 * ne00 + i02 * ne01 * ne00 + i01 * ne00; + + const float * src_row = src0_data + src_offset; + float * dst_row = dst_data + src_offset; + const float * mask_row = NULL; + + if (use_mask && mask_data) { + const int64_t mask_i03 = (ne13 > 0) ? i03 % ne13 : 0; + const int64_t mask_i02 = (ne12 > 0) ? i02 % ne12 : 0; + const int64_t mask_i01 = i01; + + const int64_t mask_offset = mask_i03 * ne12 * ne11 * ne10 + mask_i02 * ne11 * ne10 + mask_i01 * ne10; + + mask_row = mask_data + mask_offset; + } + + compute_softmax_row(dst_row, src_row, mask_row, (int) ne00, scale, slope, sink_value, use_sinks); + } + } + + return 0; // Success +} diff --git a/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c b/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c new file mode 100644 index 0000000..b65e299 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c @@ -0,0 +1,109 @@ +//****************************************************************************** +// Solve Triangular F32 Kernel +// Forward substitution: solve AX = B where A is lower-triangular. +// +// src0 (A): [n, n, B1, B2] lower-triangular matrix +// src1 (B): [k, n, B1, B2] right-hand side +// dst (X): [k, n, B1, B2] solution +// +// For each column j (parallelized across threads): +// For i = 0..n-1: +// X[i,j] = (B[i,j] - dot(A[i,0..i-1], X[0..i-1,j])) / A[i,i] +// +// Lower-triangular, left-side, non-unit variant implemented. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include + +struct ggml_et_solve_tri_params { + struct ggml_tensor src0; // A: lower-triangular [n, n, B1, B2] + struct ggml_tensor src1; // B: RHS [k, n, B1, B2] + struct ggml_tensor dst; // X: solution [k, n, B1, B2] +}; + +int entry_point(struct ggml_et_solve_tri_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; // A + struct ggml_tensor * src1 = ¶ms->src1; // B + struct ggml_tensor * dst = ¶ms->dst; // X + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + const float * A_data = (const float *) src0->data; + const float * B_data = (const float *) src1->data; + float * X_data = (float *) dst->data; + + if (!A_data || !B_data || !X_data) { + return -1; + } + + const int64_t n = src0->ne[1]; // A is n×n + const int64_t k = src1->ne[0]; // number of RHS columns + const int64_t ne2 = src0->ne[2]; + const int64_t ne3 = src0->ne[3]; + + // Strides in bytes + const size_t nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const size_t nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + const size_t nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + // k % 16 == 0 guaranteed by supports_op. Rows are cache-line aligned, + // so column groups of 16 map to exclusive cache lines. + // TODO: Vectorize the thing + const int64_t cols_per_cl = 16; + const int64_t num_col_groups = k / cols_per_cl; + const int64_t total_work = num_col_groups * ne2 * ne3; + + for (int64_t work = thread_id; work < total_work; work += num_threads) { + const int64_t cg = work % num_col_groups; + const int64_t i2 = (work / num_col_groups) % ne2; + const int64_t i3 = work / (num_col_groups * ne2); + + const int64_t j_start = cg * cols_per_cl; + const int64_t j_end = j_start + cols_per_cl; + + const float * A_batch = (const float *) ((const char *) A_data + i2 * nb02 + i3 * nb03); + const float * B_batch = (const float *) ((const char *) B_data + i2 * nb12 + i3 * nb13); + float * X_batch = (float *) ((char *) X_data + i2 * nb2 + i3 * nb3); + + for (int64_t j = j_start; j < j_end; j++) { + for (int64_t i = 0; i < n; i++) { + const float * A_row = (const float *) ((const char *) A_batch + i * nb01); + float * X_row = (float *) ((char *) X_batch + i * nb1); + const float * B_row = (const float *) ((const char *) B_batch + i * nb11); + + float sum = 0.0f; + for (int64_t t = 0; t < i; t++) { + const float * X_t = (const float *) ((const char *) X_batch + t * nb1); + sum += A_row[t] * X_t[j]; + } + + X_row[j] = et_fdiv(B_row[j] - sum, A_row[i]); + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/sqr_f32.c b/ggml/src/ggml-et/et-kernels/src/sqr_f32.c new file mode 100644 index 0000000..c184ab7 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/sqr_f32.c @@ -0,0 +1,88 @@ +//****************************************************************************** +// SQR F32 Kernel +// Element-wise square: y[i] = x[i] * x[i] +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +// SQR kernel parameters structure (unary op: src0 -> dst) +struct ggml_et_sqr_params { + struct ggml_tensor src0; // F32 input tensor + struct ggml_tensor dst; // F32 output tensor +}; + +int entry_point(struct ggml_et_sqr_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; // Invalid pointer + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; // Unsupported type combination + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; // Null data pointer + } + + // Both src and dst are contiguous F32: flatten and distribute by cache lines + const int64_t total_elements = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; + const int64_t elements_per_cacheline = 16; // 64 bytes / 4 bytes per float + const int64_t total_cachelines = (total_elements + elements_per_cacheline - 1) / elements_per_cacheline; + + const int64_t cl_per_thread = (total_cachelines + num_threads - 1) / num_threads; + const int64_t cl_start = thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cachelines) { + cl_end = total_cachelines; + } + + if (cl_start >= total_cachelines) { + return 0; + } + + const int64_t elem_start = cl_start * elements_per_cacheline; + int64_t elem_end = cl_end * elements_per_cacheline; + if (elem_end > total_elements) { + elem_end = total_elements; + } + + const float * src_ptr = src0_data + elem_start; + float * dst_ptr = dst_data + elem_start; + const int32_t count = (int32_t) (elem_end - elem_start); + + // Process 8 elements at a time: dst[i] = src[i] * src[i] + for (int32_t i0 = 0; i0 < count; i0 += 8) { + __asm__ volatile( + "flw.ps f10, %[x_vec]\n" // Load 8 input values + "fmul.ps f11, f10, f10\n" // x * x (8-wide) + "fsw.ps f11, %[result]\n" // Store 8 results + + : [result] "=m"(*(float (*)[8]) & dst_ptr[i0]) + : [x_vec] "m"(*(const float (*)[8]) & src_ptr[i0]) + : "f10", "f11"); + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c b/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c new file mode 100644 index 0000000..d65ef87 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c @@ -0,0 +1,129 @@ +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_ssm_conv_params { + struct ggml_tensor src0; // conv_x: [d_conv - 1 + n_t, d_inner, n_seqs] + struct ggml_tensor src1; // conv1d.weight: [d_conv, d_inner] + struct ggml_tensor dst; // output: [d_inner, n_t, n_seqs] +}; + +int entry_point(struct ggml_et_ssm_conv_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + const float * src0_data = (const float *) src0->data; + const float * src1_data = (const float *) src1->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !src1_data || !dst_data) { + return -1; + } + + const int64_t nc = src1->ne[0]; + const int64_t ncs = src0->ne[0]; + const int64_t nr = src0->ne[1]; + const int64_t n_t = dst->ne[1]; + const int64_t n_s = dst->ne[2]; + + if (dst->ne[0] != nr || src1->ne[1] != nr || ncs != nc - 1 + n_t || src0->nb[0] != sizeof(float) || + src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || src0->nb[1] != (size_t) ncs * sizeof(float) || + src1->nb[1] != (size_t) nc * sizeof(float)) { + return -1; + } + + // Parallelize over d_inner in cache-line-aligned chunks (16 floats = 64B) + const int64_t chunk = 16; + const int64_t n_chunks = (nr + chunk - 1) / chunk; + + // Save and set vector mask to all 8 lanes + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + for (int64_t i3 = 0; i3 < n_s; ++i3) { + for (int64_t i2 = 0; i2 < n_t; ++i2) { + const float * s = (const float *) ((const char *) src0_data + i2 * src0->nb[0] + i3 * src0->nb[2]); + float * x = (float *) ((char *) dst_data + i2 * dst->nb[1] + i3 * dst->nb[2]); + + for (int64_t ci = thread_id; ci < n_chunks; ci += num_threads) { + const int64_t i1_start = ci * chunk; + const int64_t i1_end = i1_start + chunk < nr ? i1_start + chunk : nr; + + // Process 8 channels at a time with SIMD + int64_t i1 = i1_start; + for (; i1 + 8 <= i1_end; i1 += 8) { + // Gather 8 channels' data into contiguous buffers for each tap + float tmp_s[8], tmp_c[8]; + float acc[8] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; + + for (int64_t i0 = 0; i0 < nc; ++i0) { + // TODO: Some way to get rid of this gather + for (int j = 0; j < 8; ++j) { + tmp_s[j] = s[(i1 + j) * ncs + i0]; + tmp_c[j] = src1_data[(i1 + j) * nc + i0]; + } + + __asm__ volatile( + "flw.ps f10, %[acc]\n" + "flw.ps f11, %[sv]\n" + "flw.ps f12, %[cv]\n" + "fmadd.ps f10, f11, f12, f10\n" + "fsw.ps f10, %[out]\n" + : [out] "=m"(*(float (*)[8]) acc) + : [acc] "m"(*(const float (*)[8]) acc), [sv] "m"(*(const float (*)[8]) tmp_s), + [cv] "m"(*(const float (*)[8]) tmp_c) + : "f10", "f11", "f12"); + } + + // Store 8 results — dst is contiguous along d_inner + __asm__ volatile( + "flw.ps f10, %[acc]\n" + "fsw.ps f10, %[dst]\n" + : [dst] "=m"(*(float (*)[8])(x + i1)) + : [acc] "m"(*(const float (*)[8]) acc) + : "f10"); + } + + // Scalar tail for remaining channels + for (; i1 < i1_end; ++i1) { + const float * c = src1_data + i1 * nc; + const float * s_row = s + i1 * ncs; + float sumf = 0.0f; + for (int64_t i0 = 0; i0 < nc; ++i0) { + sumf += s_row[i0] * c[i0]; + } + x[i1] = sumf; + } + } + } + } + + // Restore mask + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c b/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c new file mode 100644 index 0000000..82ac430 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c @@ -0,0 +1,282 @@ +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include + +struct ggml_et_ssm_scan_params { + struct ggml_tensor src0; // s: [d_state, head_dim, n_head, n_seqs] + struct ggml_tensor src1; // x: [head_dim, n_head, n_seq_tokens, n_seqs] + struct ggml_tensor src2; // dt: [n_head, n_seq_tokens, n_seqs] + struct ggml_tensor src3; // A: [d_state, n_head] or [1, n_head] + struct ggml_tensor src4; // B: [d_state, n_group, n_seq_tokens, n_seqs] + struct ggml_tensor src5; // C: [d_state, n_group, n_seq_tokens, n_seqs] + struct ggml_tensor src6; // ids: [n_seqs] i32 + struct ggml_tensor dst; // packed [y, states] + int32_t K; +}; + +static inline float softplus_f32(float x) { + return x <= 20.0f ? et_logf(1.0f + et_expf(x)) : x; +} + +int entry_point(struct ggml_et_ssm_scan_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + const int thread_id = get_relative_thread_id(kernel_env->shire_mask); + const int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * src1 = ¶ms->src1; + struct ggml_tensor * src2 = ¶ms->src2; + struct ggml_tensor * src3 = ¶ms->src3; + struct ggml_tensor * src4 = ¶ms->src4; + struct ggml_tensor * src5 = ¶ms->src5; + struct ggml_tensor * src6 = ¶ms->src6; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_F32 || + src3->type != GGML_TYPE_F32 || src4->type != GGML_TYPE_F32 || src5->type != GGML_TYPE_F32 || + src6->type != GGML_TYPE_I32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + const float * s_data = (const float *) src0->data; + const float * x_data = (const float *) src1->data; + const float * dt_data = (const float *) src2->data; + const float * A_data = (const float *) src3->data; + const float * B_data = (const float *) src4->data; + const float * C_data = (const float *) src5->data; + const int32_t * ids = (const int32_t *) src6->data; + float * dst_data = (float *) dst->data; + + if (!s_data || !x_data || !dt_data || !A_data || !B_data || !C_data || !ids || !dst_data) { + return -1; + } + + const int64_t d_state = src0->ne[0]; + const int64_t head_dim = src0->ne[1]; + const int64_t n_head = src1->ne[1]; + const int64_t n_group = src4->ne[1]; + const int64_t n_seq_tokens = src1->ne[2]; + const int64_t n_seqs = src1->ne[3]; + const int64_t y_elems = src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3]; + const int64_t K = params->K; + + if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || src2->nb[0] != sizeof(float) || + src3->nb[0] != sizeof(float) || src4->nb[0] != sizeof(float) || src5->nb[0] != sizeof(float) || + src6->nb[0] != sizeof(int32_t) || dst->nb[0] != sizeof(float)) { + return -1; + } + + if (K < 1 || n_group <= 0 || n_head % n_group != 0) { + return -1; + } + + // Cache-line bundling on the dst output (1 dst float per (head, dim, token)). + // - When head_dim < 16: bundle 16/head_dim heads per work-unit (1 line of dst). + // - When head_dim >= 16: each head's dim slice spans head_dim/16 lines, so we + // can split dims into chunks of 16 across threads without false sharing. + const int64_t dst_lanes_per_cl = 16; + const int64_t heads_per_cacheline = head_dim >= dst_lanes_per_cl ? 1 : (dst_lanes_per_cl / head_dim); + const int64_t heads_per_block = heads_per_cacheline > 0 ? heads_per_cacheline : 1; + const int64_t blocks_per_seq = (n_head + heads_per_block - 1) / heads_per_block; + const int64_t dim_chunk_lanes = head_dim >= dst_lanes_per_cl ? dst_lanes_per_cl : head_dim; + const int64_t dim_chunks_per_head = (head_dim + dim_chunk_lanes - 1) / dim_chunk_lanes; + + // A "unit" = (seq, head_block, dim_chunk). This expands the parallelism by a + // factor of dim_chunks_per_head over the prior block-only scheme; for Mamba-2 + // shapes (head_dim=64) that's a 4x bump in active threads. + const int64_t units_per_seq = blocks_per_seq * dim_chunks_per_head; + const int64_t total_units = n_seqs * units_per_seq; + const int64_t units_per_thread = (total_units + num_threads - 1) / num_threads; + const int64_t unit_begin = (int64_t) thread_id * units_per_thread; + int64_t unit_end = unit_begin + units_per_thread; + + if (unit_begin >= total_units) { + return 0; + } + + if (unit_end > total_units) { + unit_end = total_units; + } + + const int A_broadcast = (src3->ne[0] == 1); + const int64_t d_state_vec = (d_state / 8) * 8; // largest multiple of 8 <= d_state + const float log2e_const = 1.4426950408889634f; + + for (int64_t unit = unit_begin; unit < unit_end; ++unit) { + const int64_t seq_idx = unit / units_per_seq; + const int64_t unit_in_seq = unit % units_per_seq; + const int64_t block_in_seq = unit_in_seq / dim_chunks_per_head; + const int64_t dim_chunk_idx = unit_in_seq % dim_chunks_per_head; + const int64_t head_begin = block_in_seq * heads_per_block; + int64_t head_end = head_begin + heads_per_block; + + if (head_end > n_head) { + head_end = n_head; + } + + const int64_t dim_begin = dim_chunk_idx * dim_chunk_lanes; + int64_t dim_end = dim_begin + dim_chunk_lanes; + if (dim_end > head_dim) { + dim_end = head_dim; + } + + const int32_t state_seq = ids[seq_idx]; + + for (int64_t head_idx = head_begin; head_idx < head_end; ++head_idx) { + const int64_t group_idx = head_idx / (n_head / n_group); + + // A pointer for this head: contiguous over state_idx when not broadcast + const float * A_row = (const float *) ((const char *) A_data + (size_t) head_idx * src3->nb[1]); + + for (int64_t dim_idx = dim_begin; dim_idx < dim_end; ++dim_idx) { + const float * state_src = + (const float *) ((const char *) s_data + (size_t) dim_idx * src0->nb[1] + + (size_t) head_idx * src0->nb[2] + (size_t) state_seq * src0->nb[3]); + + float * state_dst = + (float *) ((char *) dst_data + (size_t) y_elems * sizeof(float) + (size_t) dim_idx * src0->nb[1] + + (size_t) head_idx * src0->nb[2] + (size_t) seq_idx * src0->nb[3]); + + for (int64_t token_idx = 0; token_idx < n_seq_tokens; ++token_idx) { + const float * x_ptr = + (const float *) ((const char *) x_data + (size_t) dim_idx * src1->nb[0] + + (size_t) head_idx * src1->nb[1] + (size_t) token_idx * src1->nb[2] + + (size_t) seq_idx * src1->nb[3]); + + const float * dt_ptr = + (const float *) ((const char *) dt_data + (size_t) head_idx * src2->nb[0] + + (size_t) token_idx * src2->nb[1] + (size_t) seq_idx * src2->nb[2]); + + const float * B_row = + (const float *) ((const char *) B_data + (size_t) group_idx * src4->nb[1] + + (size_t) token_idx * src4->nb[2] + (size_t) seq_idx * src4->nb[3]); + + const float * C_row = + (const float *) ((const char *) C_data + (size_t) group_idx * src5->nb[1] + + (size_t) token_idx * src5->nb[2] + (size_t) seq_idx * src5->nb[3]); + + const float dt_softplus = softplus_f32(*dt_ptr); + const float x_dt = (*x_ptr) * dt_softplus; + const float dt_log2e = dt_softplus * log2e_const; + + // Source of "previous state" for this token: input state on token 0, + // last token's state thereafter (we wrote it into state_dst). + const float * prev_row = (token_idx == 0) ? state_src : state_dst; + + float sumf = 0.0f; + int64_t state_idx = 0; + + if (d_state_vec > 0) { + // Save mask, enable all 8 vector lanes for the state loop. + unsigned long saved_mask; + __asm__ volatile("mova.x.m %0" : "=r"(saved_mask)); + __asm__ volatile("mov.m.x m0, x0, 0xFF"); + + // Per-token broadcasts: + // f20 = x_dt (B*x_dt) + // f21 = dt_log2e (for fexp.ps when A is per-state) + // f22 = dA (only when A is broadcast scalar) + // f23 = sum-of-products accumulator (zeroed) + __asm__ volatile( + "fbc.ps f20, %[xdt]\n\t" + "fbc.ps f21, %[dtl]\n\t" + "fbci.pi f23, 0\n\t" + : + : [xdt] "m"(x_dt), [dtl] "m"(dt_log2e) + : "f20", "f21", "f23"); + + if (A_broadcast) { + // dA is a per-head scalar — compute once and splat. + const float dA_scalar = et_expf(dt_softplus * (*A_row)); + __asm__ volatile("fbc.ps f22, %[da]\n\t" : : [da] "m"(dA_scalar) : "f22"); + } + + for (; state_idx < d_state_vec; state_idx += 8) { + if (!A_broadcast) { + // f22 = exp(dt_softplus * A[state..state+7]) + // = 2^((dt_softplus * A) * log2e) via fexp.ps + __asm__ volatile( + "flw.ps f24, %[av]\n\t" + "fmul.ps f24, f24, f21\n\t" // A * dt_log2e + "fexp.ps f22, f24\n\t" // dA = 2^(...) + : + : [av] "m"(*(const float (*)[8]) & A_row[state_idx]) + : "f22", "f24"); + } + + // state = prev * dA + B * x_dt + // sumf += state * C + // Reads prev before writing state_dst — safe even when + // prev_row == state_dst (write-after-read, same index). + __asm__ volatile( + "flw.ps f25, %[prev]\n\t" + "flw.ps f26, %[bv]\n\t" + "flw.ps f27, %[cv]\n\t" + "fmul.ps f26, f26, f20\n\t" // B * x_dt + "fmadd.ps f25, f25, f22, f26\n\t" // state = prev*dA + B*x_dt + "fsw.ps f25, %[sd]\n\t" + "fmadd.ps f23, f25, f27, f23\n\t" // sum += state*C + : [sd] "=m"(*(float (*)[8]) & state_dst[state_idx]) + : [prev] "m"(*(const float (*)[8]) & prev_row[state_idx]), + [bv] "m"(*(const float (*)[8]) & B_row[state_idx]), + [cv] "m"(*(const float (*)[8]) & C_row[state_idx]) + : "f25", "f26", "f27"); + } + + // Horizontal reduce f23 (8 lanes) -> scalar sumf. + __asm__ volatile( + "fswizz.ps f1, f23, 0xB1\n\t" + "fadd.ps f2, f23, f1, rne\n\t" + "fswizz.ps f3, f2, 0x4E\n\t" + "fadd.ps f4, f2, f3, rne\n\t" + "fmvz.x.ps t0, f4, 4\n\t" + "fbcx.ps f5, t0\n\t" + "fadd.ps %[vout], f4, f5, rne\n\t" + : [vout] "=f"(sumf)::"t0", "f1", "f2", "f3", "f4", "f5"); + + __asm__ volatile("mova.m.x %0" ::"r"(saved_mask)); + } + + // Scalar tail (d_state not a multiple of 8). + for (; state_idx < d_state; ++state_idx) { + const float prev_state = prev_row[state_idx]; + const float A_val = A_broadcast ? *A_row : A_row[state_idx]; + const float dA = et_expf(dt_softplus * A_val); + const float st = prev_state * dA + B_row[state_idx] * x_dt; + state_dst[state_idx] = st; + sumf += st * C_row[state_idx]; + } + + const int64_t slot = n_seq_tokens - 1 - token_idx; + if (slot > 0 && slot < K) { + float * state_snapshot = + (float *) ((char *) state_dst + (size_t) slot * n_seqs * src0->nb[3]); + for (int64_t i = 0; i < d_state; ++i) { + state_snapshot[i] = state_dst[i]; + } + } + + dst_data[seq_idx * (n_seq_tokens * n_head * head_dim) + token_idx * (n_head * head_dim) + + head_idx * head_dim + dim_idx] = sumf; + } + } + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c b/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c new file mode 100644 index 0000000..968707f --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c @@ -0,0 +1,103 @@ +//****************************************************************************** +// SUM_ROWS F32 Kernel +// Row-wise sum reduction: dst[0, i1, i2, i3] = sum(src0[0..ne00-1, i1, i2, i3]) +// Vectorized 8-wide accumulation with horizontal reduction. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +struct ggml_et_sum_rows_params { + struct ggml_tensor src0; // F32 input tensor [ne00, ne01, ne02, ne03] + struct ggml_tensor dst; // F32 output tensor [1, ne01, ne02, ne03] +}; + +int entry_point(struct ggml_et_sum_rows_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + + const int64_t ne00 = src0->ne[0]; // Row length (to be summed) + const int64_t ne01 = src0->ne[1]; + const int64_t ne02 = src0->ne[2]; + const int64_t ne03 = src0->ne[3]; + + const size_t nb01 = src0->nb[1]; + const size_t nb02 = src0->nb[2]; + const size_t nb03 = src0->nb[3]; + + const size_t nb1 = dst->nb[1]; + const size_t nb2 = dst->nb[2]; + const size_t nb3 = dst->nb[3]; + + // Flatten rows across dimensions 1,2,3 and distribute across threads + const int64_t total_rows = ne01 * ne02 * ne03; + + for (int64_t ir = thread_id; ir < total_rows; ir += num_threads) { + const int64_t i03 = ir / (ne02 * ne01); + const int64_t i02 = (ir - i03 * ne02 * ne01) / ne01; + const int64_t i01 = ir - i03 * ne02 * ne01 - i02 * ne01; + + const float * src_row = (const float *) ((const char *) src0_data + i01 * nb01 + i02 * nb02 + i03 * nb03); + float * dst_ptr = (float *) ((char *) dst_data + i01 * nb1 + i02 * nb2 + i03 * nb3); + + // Vectorized 8-wide sum accumulation + float zero = 0.0f; + __asm__ volatile("fbc.ps f10, %[z]\n" : : [z] "m"(zero) : "f10"); + + for (int32_t i0 = 0; i0 < (int32_t) ne00; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[x_vec]\n" + "fadd.ps f10, f10, f11\n" + : + : [x_vec] "m"(*(const float (*)[8]) & src_row[i0]) + : "f10", "f11"); + } + + // Horizontal sum of 8 accumulated values in f10 + float row_sum; + __asm__ __volatile__( + "fswizz.ps f1, f10, 0xB1 \n\t" + "fadd.ps f2, f10, f1, rne \n\t" + "fswizz.ps f3, f2, 0x4E \n\t" + "fadd.ps f4, f2, f3, rne \n\t" + "fmvz.x.ps t0, f4, 4 \n\t" + "fbcx.ps f5, t0 \n\t" + "fadd.ps %[vout], f4, f5, rne \n\t" + : [vout] "=f"(row_sum)::"t0", "f1", "f2", "f3", "f4", "f5"); + + atomic_store_f32(dst_ptr, row_sum); + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/tensor.h b/ggml/src/ggml-et/et-kernels/src/tensor.h new file mode 100644 index 0000000..043a2a3 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/tensor.h @@ -0,0 +1,897 @@ +#ifndef __TENSORS_H +#define __TENSORS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__cplusplus) && (__cplusplus >= 201103L) +# include +# if (__cplusplus < 202002L) +# include +# endif +#else +# include +# include +#endif + +/*! \def QUANT_LAST_TRANS + \brief Tensor Quant instruction: Do not perform any more transformations. +*/ +#define QUANT_LAST_TRANS 0 + +/*! \def QUANT_INT32_TO_FP32 + \brief Tensor Quant instruction: Convert all elements of A from 32-bit signed integer values to single-precision + floating-point values. +*/ +#define QUANT_INT32_TO_FP32 1 + +/*! \def QUANT_FP32_TO_INT32 + \brief Tensor Quant instruction: Convert all elements of A from single-precision floating-point values to 32- + bit signed integer values. +*/ +#define QUANT_FP32_TO_INT32 2 + +/*! \def QUANT_RELU + \brief Tensor Quant instruction: Convert all negative INT32 values in A to 0 +*/ +#define QUANT_RELU 3 + +/*! \def QUANT_INT32_ADD_ROW + \brief Tensor Quant instruction: Read the low-order COLS+1 32-bit signed integer values from an L1 + scratchpad line, and add this vector to every row of the 32-bit signed integer + matrix A. +*/ +#define QUANT_INT32_ADD_ROW 4 + +/*! \def QUANT_INT32_ADD_COL + \brief Tensor Quant instruction: Read the low-order ROWS+1 32-bit signed integer values from an L1 + scratchpad line, and add this vector to every column of the 32-bit signed + integer matrix A. +*/ +#define QUANT_INT32_ADD_COL 5 + +/*! \def QUANT_FP32_MUL_ROW + \brief Tensor Quant instruction: Read the low-order COLS+1 single-precision floating-point values from an + L1 scratchpad line, and multiply the single-precision elements of each row + of matrix A element-wise by this vector. +*/ +#define QUANT_FP32_MUL_ROW 6 + +/*! \def QUANT_FP32_MUL_COL + \brief Tensor Quant instruction: Read the low-order ROWS+1 single-precision floating-point values from an + L1 scratchpad line, and multiply the single-precision elements of each col- + umn of matrix A element-wise by this vector. +*/ +#define QUANT_FP32_MUL_COL 7 + +/*! \def QUANT_SATINT8 + \brief Tensor Quant instruction: Clamp all 32-bit signed integer values in A to the range [-128, 127]. + The values are written in bits 7:0 of each element, with bits 31:8 set to zero. +*/ +#define QUANT_SATINT8 8 + +/*! \def QUANT_SATUINT8 + \brief Tensor Quant instruction: Clamp all 32-bit signed integer values in A to the range [0, 255]. The values + are written in bits 7:0 of each element, with bits 31:8 set to zero. +*/ +#define QUANT_SATUINT8 9 + +/*! \def QUANT_PACK_128B + \brief Tensor Quant instruction: Copy the low-order byte of the n-th 32-bit value in each row of A to the n-th + byte of the row. +*/ +#define QUANT_PACK_128B 10 + +/*! \def TENSOR_REDUCE_OP_FADD + \brief Tensor Reduce instruction: The result is the addition of the incoming single-precision floating-point data + and the single-precision floating-point values in the vector register file. +*/ +#define TENSOR_REDUCE_OP_FADD 0 + +// #define TENSOR_REDUCE_OP_FSUB 1 -- Not supported + +/*! \def TENSOR_REDUCE_OP_FMAX + \brief Tensor Reduce instruction: The result is the maximum of the incoming single-precision floating-point data +and the single-precision floating-point values in the vector register file. +*/ +#define TENSOR_REDUCE_OP_FMAX 2 + +/*! \def TENSOR_REDUCE_OP_FMIN + \brief Tensor Reduce instruction: The result is the minimum of the incoming single-precision floating-point data +and the single-precision floating-point values in the vector register file.. +*/ +#define TENSOR_REDUCE_OP_FMIN 3 + +/*! \def TENSOR_REDUCE_OP_IADD + \brief Tensor Reduce instruction: The result is the addition of the incoming 32-bit integer data and the 32-bit inte- +ger values in the vector register file. +*/ +#define TENSOR_REDUCE_OP_IADD 4 + +// #define TENSOR_REDUCE_OP_ISUB 5 -- Not supported + +/*! \def TENSOR_REDUCE_OP_IMAX + \brief Tensor Reduce instruction: The result is the maximum of the incoming 32-bit signed integer data and the +32-bit signed integer values in the vector register file. +*/ +#define TENSOR_REDUCE_OP_IMAX 6 + +/*! \def TENSOR_REDUCE_OP_IMIN + \brief Tensor Reduce instruction: The result is the minimum of the incoming 32-bit signed integer data and the +32-bit signed integer values in the vector register file. +*/ +#define TENSOR_REDUCE_OP_IMIN 7 + +/*! \def TENSOR_REDUCE_OP_FGET + \brief Tensor Reduce instruction get function to be performed +*/ +#define TENSOR_REDUCE_OP_FGET 8 + +/*! \def TENSOR_LOAD_WAIT_0 + \brief Tensor load to L1 Scratchpad with ID = 0 is complete. +*/ +#define TENSOR_LOAD_WAIT_0 0 + +/*! \def TENSOR_LOAD_WAIT_1 + \brief Tensor load to L1 Scratchpad with ID = 1 is complete. +*/ +#define TENSOR_LOAD_WAIT_1 1 + +/*! \def TENSOR_FMA_WAIT + \brief All previous tensor matrix multiplication instructions are complete. +*/ +#define TENSOR_FMA_WAIT 7 + +/*! \def TENSOR_STORE_WAIT + \brief All previous tensor store instructions are complete. +*/ +#define TENSOR_STORE_WAIT 8 + +/*! \def TENSOR_REDUCE_WAIT + \brief All previous tensor reduction instructions are complete +*/ +#define TENSOR_REDUCE_WAIT 9 + +/*! \def TENSOR_QUANT_WAIT + \brief TensorQuant is complete +*/ +#define TENSOR_QUANT_WAIT 10 + +// TensorFMA opcode values (tensor_fma CSR 0x801, bits 3:1) +#define TENSOR_FMA_OP_FP32 0 // TensorFMA32: FP32 x FP32 -> FP32 +#define TENSOR_FMA_OP_FP16 1 // TensorFMA16A32: FP16 x FP16 -> FP32 +// opcode 2 is reserved +#define TENSOR_FMA_OP_INT8 3 // TensorIMA8A32: INT8 x INT8 -> INT32 + +// TensorLoad transformation values (tensor_load CSR 0x83F, bits 61:59) +#define TENSOR_LOAD_PLAIN 0 // TensorLoad: 64B rows +#define TENSOR_LOAD_INTERLEAVE8 1 // TensorLoadInterleave8: for TensorIMA8A32 B +#define TENSOR_LOAD_INTERLEAVE16 2 // TensorLoadInterleave16: for TensorFMA16A32 B +// transformations 3-4 are reserved +#define TENSOR_LOAD_TRANSPOSE8 5 // TensorLoadTranspose8: 8-bit transpose +#define TENSOR_LOAD_TRANSPOSE16 6 // TensorLoadTranspose16: 16-bit transpose +#define TENSOR_LOAD_TRANSPOSE32 7 // TensorLoadTranspose32: 32-bit transpose + +/*! \def TENSOR_ERROR_LOAD_TRANSFORM + \brief Define for tensor load transform error. +*/ +#define TENSOR_ERROR_LOAD_TRANSFORM 1 + +/*! \def TENSOR_ERROR_FCC_OVERFLOW + \brief Define for tensor fcc overflow error. +*/ +#define TENSOR_ERROR_FCC_OVERFLOW 3 + +/*! \def TENSOR_ERROR_SCP_DISABLED + \brief Define for tensor scp disabled error. +*/ +#define TENSOR_ERROR_SCP_DISABLED 4 + +/*! \def TENSOR_ERROR_LOCKSW + \brief Define for tensor locksw error. +*/ +#define TENSOR_ERROR_LOCKSW 5 + +/*! \def TENSOR_ERROR_TL1_FMA + \brief Define for L1 FMA error. +*/ +#define TENSOR_ERROR_TL1_FMA 6 + +/*! \def TENSOR_ERROR_MEM_FAULT + \brief Define for Memory fault error. +*/ +#define TENSOR_ERROR_MEM_FAULT 7 + +/*! \def TENSOR_ERROR_STORE_COOP + \brief Define for store coop error. +*/ +#define TENSOR_ERROR_STORE_COOP 8 + +/*! \def TENSOR_ERROR_REDUCE + \brief Define for tensor reduce error. +*/ +#define TENSOR_ERROR_REDUCE 9 + +/*! \struct et_tensor_load_l2scp_conf + \brief Tensor load from scp instruction configuration structure. +*/ +typedef struct et_tensor_load_l2scp_conf { + bool use_tmask; + uint64_t dst_start; + uint64_t addr; + uint64_t num_lines; + uint64_t stride; + uint64_t id; +} et_tensor_load_l2scp_conf_t; + +/*! \enum reduce_transform_t + \brief enum transform mode for tensor reduce. +*/ +typedef enum { + FADD = 0x0ULL, + FSUB = 0x1ULL, + FMAX = 0x2ULL, + FMIN = 0x3ULL, + IADD = 0x4ULL, + ISUB = 0x5ULL, + IMAX = 0x6ULL, + IMIN = 0x7ULL, + FGET = 0x8ULL +} reduce_transform_t; + +/*! \struct et_tensor_load_conf + \brief Tensor load instruction configuration structure. +*/ +typedef struct et_tensor_load_conf { + bool use_tmask; + bool use_coop; + bool use_tenb; + uint64_t dst_start; + uint64_t transformation; + uint64_t rd_l2scp; + uint64_t addr; + uint64_t offset; + uint64_t num_lines; + uint64_t stride; + uint64_t id; +} et_tensor_load_conf_t; + +/*! \fn inline void tensor_wait(long id) + \brief Tensor wait instruction, Tensor Wait can be used to stall execution until + a previously issued tensor instruction completes. + \param id tensor ID + \return none + \tensorops Implementation of tensor_wait api +*/ +inline __attribute__((always_inline)) void tensor_wait(long id) { + __asm__ __volatile__(" csrw 0x830, %[id]\n" : : [id] "r"(id) : "memory"); +} + +/*! \fn inline void tensor_load (tensor_load *conf) + \brief Tensor load instruction, it loads data from memory (bypass-ing the L1 cache) + into the L1 scratchpad. Input parameter defines the configuration to tensor load. + \param use_tmask the tensor_mask register is used for this operation + \param use_coop the operation is a cooperative tensor load. + \param dst_start L1 Scratchpad starting cache line + \param transformation These bits, along with bit 52, decodes the type of tensor operation. + \param use_tenb This bit, along with transformation, decodes the type of tensor operation. + \param addr tensor load address + \param offset tensor load address offset + \param num_lines tensor load number of cache lines + \param stride tensor load stride value + \param id tensor load id + \return none + \tensorops Implementation of tensor_load api + +*/ +// 1. Load Matrix A segment (1 row x 16 cols) into SCP ID 0 +// dst_start 0 refers to the first line of L1 Scratchpad +// tensor_load(false, false, 0, 0, 0, +// (uint64_t)(src0_data + m * K + kb), 0, 1, 0, 0); + +inline void __attribute__((always_inline)) tensor_load(bool use_tmask, + bool use_coop, + uint64_t dst_start, + uint64_t transformation, + uint64_t use_tenb, + uint64_t addr, + uint64_t offset, + uint64_t num_lines, + uint64_t stride, + uint64_t id) { + // Address alignment depends on transformation type: + // Interleave8, Transpose8 (1,5): 16B aligned, addr bits 47:4 + // Interleave16, Transpose16 (2,6): 32B aligned, addr bits 47:5 + // Load, Transpose32, LoadB (0,7): 64B aligned, addr bits 47:6 + uint64_t addr_mask = (transformation == 1 || transformation == 5) ? 0xFFFFFFFFFFF0ULL : + (transformation == 2 || transformation == 6) ? 0xFFFFFFFFFFE0ULL : + 0xFFFFFFFFFFC0ULL; + uint64_t csr_enc = (((uint64_t) use_tmask & 1) << 63) | (((uint64_t) use_coop & 1) << 62) | + ((transformation & 0x7) << 59) | ((dst_start & 0x3F) << 53) | ((use_tenb & 0x1) << 52) | + ((addr & addr_mask)) | ((offset & 0x3) << 4) | ((num_lines & 0xF)); + + uint64_t x31_enc = (stride & 0xFFFFFFFFFFC0ULL) | (id & 0x1); + + __asm__ __volatile__( + "mv x31, %[x31v]\n" + "csrw 0x83f, %[csrv]\n" + : + : [x31v] "r"(x31_enc), [csrv] "r"(csr_enc) + : "x31", "memory"); +} + +/*! \fn inline void et_tensor_load (et_tensor_load_conf_t *conf) + \brief Tensor load instruction, it loads data from memory (bypass-ing the L1 cache) + into the L1 scratchpad. Input parameter defines the configuration to tensor load. + \param conf tensor load configuration + \return none + \tensorops Implementation of et_tensor_load api +*/ +inline void __attribute__((always_inline)) et_tensor_load(et_tensor_load_conf_t * conf) { + tensor_load(conf->use_tmask, conf->use_coop, conf->dst_start, conf->transformation, (uint64_t) conf->use_tenb, + conf->addr, conf->offset, conf->num_lines, conf->stride, conf->id); +} + +/*! \fn inline void tensor_load_setup_b(bool use_coop, uint64_t addr, uint64_t num_lines, uint64_t stride, uint64_t id) + \brief Tensor load instruction setup + \param use_coop the operation is a cooperative tensor load. + \param addr tensor load address + \param num_lines tensor load number of cache lines + \param stride tensor load stride value + \param id tensor load id + \return none + \tensorops Implementation of tensor_load_setup_b api +*/ +inline void __attribute__((always_inline)) tensor_load_setup_b(bool use_coop, + uint64_t addr, + uint64_t num_lines, + uint64_t stride, + uint64_t id) { + uint64_t csr_enc = + (((uint64_t) use_coop & 1) << 62) | (0x1ULL << 52) | ((addr & 0xFFFFFFFFFFC0ULL)) | ((num_lines & 0xF)); + uint64_t x31_enc = (stride & 0xFFFFFFFFFFC0ULL) | (id & 0x1); + + __asm__ __volatile__( + "mv x31, %[x31v]\n" + "csrw 0x83f, %[csrv]\n" + : + : [x31v] "r"(x31_enc), [csrv] "r"(csr_enc) + : "x31", "memory"); +} + +/*! \fn inline void et_tensor_load_l2scp (et_tensor_load_l2scp_conf_t *conf) + \brief Tensor load l2scp loads data from memory (bypassing the L1 and L2 caches) into the L2 scratchpad. + \param conf tensor load configuration + \return none + \tensorops Implementation of et_tensor_load_l2scp api +*/ +inline void __attribute__((always_inline)) et_tensor_load_l2scp(et_tensor_load_l2scp_conf_t * conf) { + uint64_t csr_enc = + (((((uint64_t) conf->use_tmask) & 1) << 63) | ((conf->dst_start & 0x1FFFCUL) << (48 - 2)) | + ((conf->dst_start & 0x3UL) << 4) | ((conf->addr & 0xFFFFFFFFFFC0UL)) | ((conf->num_lines & 0x0FUL))); + uint64_t x31_enc = (conf->stride & 0xFFFFFFFFFFC0ULL) | (conf->id & 0x1); + + __asm__ __volatile__( + "mv x31, %[x31v]\n" + "csrw 0x85f, %[csrv]\n" + : + : [x31v] "r"(x31_enc), [csrv] "r"(csr_enc) + : "x31", "memory"); +} + +/*! \fn inline void tensor_store_scp(uint64_t entry_stride, + uint64_t start_scp_entry, + uint64_t Arows, + uint64_t addr, + uint64_t stride) + \brief Tensor Store writes a series of 64-byte blocks of data from the L1 scratchpad into memory. + A matrix X can have up to 16 rows, and each row can be up to 64B in size (the number of columns depends on the type of elements of X). + \param entry_stride Register stride + \param start_scp_entry Start register + \param Arows A matrix row size + \param addr Virtual Address + \param stride This value is the distance in bytes between consecutive tensor rows in memory + \return none + \tensorops Implementation of tensor_store_scp api +*/ +inline void __attribute__((always_inline)) tensor_store_scp(uint64_t entry_stride, + uint64_t start_scp_entry, + uint64_t Arows, + uint64_t addr, + uint64_t stride) { + uint64_t csr_enc = ((entry_stride & 0x3) << 62) | ((start_scp_entry & 0x3F) << 56) | ((addr & 0xFFFFFFFFFFC0ULL)) | + ((Arows & 0xF) << 51) | (((uint64_t) 1) << 48); + uint64_t x31_enc = (stride & 0xFFFFFFFFFFC0UL); + + __asm__ __volatile__( + "mv x31, %[x31v]\n" + "csrw 0x87f, %[csrv]\n" + : + : [x31v] "r"(x31_enc), [csrv] "r"(csr_enc) + : "x31", "memory"); +} + +/*! \fn inline void tensor_store(uint64_t reg_stride, + uint64_t start_reg, + uint64_t cols, + uint64_t Arows, + uint64_t addr, + uint64_t coop_store, + uint64_t stride) + \brief The Tensor store instruction reads a tensor from the vector register files and writes it to memory, + bypassing the L1 data cache and the L2 cache. For the purposes of this instruction the tensor has ROWS+1 rows, + and each row is 16*SIZE+16 bytes in size. + \param reg_stride Register stride + \param start_reg start register address + \param cols matrix row size. + \param Arows matrix row size + \param addr Virtual Address + \param coop_store Number of minions to cooperate with + \param stride This value is the distance in bytes between consecutive tensor rows in memory + \return none + \tensorops Implementation of tensor_store api +*/ +inline void __attribute__((always_inline)) tensor_store(uint64_t reg_stride, + uint64_t start_reg, + uint64_t cols, + uint64_t Arows, + uint64_t addr, + uint64_t coop_store, + uint64_t stride) { + uint64_t warl = 0; + uint64_t csr_enc = ((reg_stride & 0x3) << 62) | ((start_reg & 0x1F) << 57) | ((cols & 0x3) << 55) | + ((addr & 0xFFFFFFFFFFF0)) | ((Arows & 0xF) << 51) | ((coop_store & 0x3) << 49) | ((warl & 0xF)); + + uint64_t x31_enc = (stride & 0xFFFFFFFFFF0UL); + + __asm__ __volatile__( + "mv x31, %[x31v]\n" + "csrw 0x87f, %[csrv]\n" + : + : [x31v] "r"(x31_enc), [csrv] "r"(csr_enc) + : "x31", "memory"); +} + +/*! \fn inline void tensor_fma(bool use_tmask, + uint64_t b_num_col, + uint64_t a_num_rows, + uint64_t a_num_cols, + uint64_t offset, + bool tenc_loc, + bool tenb_unsigned, + bool tena_unsigned, + bool tenb_loc, + uint64_t scp_loc_b, + uint64_t scp_loc_a, + uint64_t opcode, + bool first_pass) + \brief The Tensor FMA instruction multiplies two matrices A and B, optionally adds the resulting matrix + to a third matrix C, and writes the result back onto matrix C + \param use_tmask Use tensor_mask CSR to skip operations in an A row granularity. + \param b_num_col B matrix number of columns + \param a_num_rows A matrix number of rows + \param a_num_cols A matrix number of columns + \param offset A matrix starting column for the operation. + \param tenc_loc Location of matrix C (0 = L1 scratchpad, 1 = memory). + \param tenb_unsigned TenB is signed (0) or unsigned (1). + \param tena_unsigned TenA is signed (0) or unsigned (1). + \param tenb_loc Location of matrix B (0 = L1 scratchpad, 1 = memory). + \param scp_loc_b Starting L1 scratchpad cache line where matrix B is stored, ignored when xs[20] = 1. + \param scp_loc_a Starting L1 scratchpad cache line where matrix A is stored, ignored when xs[20] = 1. + \param opcode 0 = TensorFMA32 (F32xF32->F32), 1 = TensorFMA16A32 (F16xF16->F32), 3 = TensorIMA8A32 (I8xF8->I32). + Other opcodes are invalid. + \param first_pass if set to 0 then the initial value of TenC is added to the result + \return none + \tensorops Implementation of tensor_fma api +*/ +inline void __attribute__((always_inline)) tensor_fma(bool use_tmask, + uint64_t b_num_col, + uint64_t a_num_rows, + uint64_t a_num_cols, + uint64_t offset, + bool tenc_loc, + bool tenb_unsigned, + bool tena_unsigned, + bool tenb_loc, + uint64_t scp_loc_b, + uint64_t scp_loc_a, + uint64_t opcode, + bool first_pass) { + uint64_t csr_enc = (((uint64_t) use_tmask & 1) << 63) | ((b_num_col & 0x3) << 55) | ((a_num_rows & 0xF) << 51) | + ((a_num_cols & 0xF) << 47) | ((offset & 0xF) << 43) | (((uint64_t) tenc_loc & 1) << 23) | + (((uint64_t) tena_unsigned & 1) << 22) | (((uint64_t) tenb_unsigned & 1) << 21) | + (((uint64_t) tenb_loc & 1) << 20) | ((scp_loc_b & 0xFF) << 12) | ((scp_loc_a & 0xFF) << 4) | + ((opcode & 0x7) << 1) | ((uint64_t) first_pass & 1); + + __asm__ __volatile__("csrw 0x801, %[csr_enc]\n" : : [csr_enc] "r"(csr_enc) :); +} + +/*! \fn inline uint32_t tensor_reduce_uint32(uint32_t value, uint64_t operation, uint64_t partnerID, uint64_t action) + \brief Tensor reduce allows a group of harts to communicate values held in floating-point registers to collectively calculate a reduction + function. + \param value Register stride + \param operation Function to be performed. + \param partnerID Receiver minionID. + \param action action value + \return uint32_t value after reduction + \tensorops Implementation of tensor_reduce_uint32 api +*/ +inline uint32_t __attribute__((always_inline)) tensor_reduce_uint32(uint32_t value, + uint64_t operation, + uint64_t partnerID, + uint64_t action) { + uint64_t warl = 0; + uint32_t out; + uint64_t csr_enc = ((warl & 0x2) << 62) | ((0ULL & 0x1F) << 57) | ((warl & 0x1FFFFFFF) << 28) | + ((operation & 0xF) << 24) | ((1ULL & 0xFF) << 16) | ((partnerID & 0x1FFF) << 3) | + ((warl & 0x1) << 2) | ((action & 0x3)); + + __asm__ __volatile__( + "fmv.s.x f0, %[value]\n" + "csrw 0x800, %[csr_enc]\n" + "fmv.x.s %[out], f0\n" + : [out] "=r"(out) + : [csr_enc] "r"(csr_enc), [value] "r"(value) + : "f0"); + + return out; +} + +/*! \fn inline float tensor_reduce_float(float freg, uint64_t operation, uint64_t num_reg, uint64_t partnerID, uint64_t action) { + \brief TensorReduce allows a group of harts to communicate values held in floating-point registers to collectively calculate a reduction + function. + \param freg Freg register stride + \param operation Function to be performed. + \param num_reg number of registers to use + \param partnerID Receiver minionID. + \param action action value + \return float value after reduction + \tensorops Implementation of tensor_reduce_float api +*/ +inline float __attribute__((always_inline)) tensor_reduce_float(float freg, + uint64_t operation, + uint64_t num_reg, + uint64_t partnerID, + uint64_t action) { + uint64_t warl = 0; + float out; + uint64_t csr_enc = ((warl & 0x2) << 62) | ((0ULL & 0x1F) << 57) | ((warl & 0x1FFFFFFF) << 28) | + ((operation & 0xF) << 24) | ((num_reg & 0xFF) << 16) | ((partnerID & 0x1FFF) << 3) | + ((warl & 0x1) << 2) | ((action & 0x3)); + + __asm__ __volatile__( + "fmv.s f0, %[freg]\n" + "csrw 0x800, %[csr_enc]\n" + "fmv.s %[out], f0\n" + : [out] "=f"(out) + : [csr_enc] "r"(csr_enc), [freg] "f"(freg) + : "f0"); + + return out; +} + +//#define tensor_reduce_float1(fval, operation, partnerID, action) do { +// uint64_t warl = 0; +// float out; +// uint64_t csr_enc = ((warl & 0x2 ) << 62) | +// ((0 & 0x1F ) << 57) | +// ((warl & 0x1FFFFFFF ) << 28) | +// ((operation & 0xF ) << 24) | +// ((1 & 0xFF ) << 16) | +// ((partnerID & 0x1FFF ) << 3 ) | +// ((warl & 0x1 ) << 2 ) | +// ((action & 0x3 ) ); +// +// register float asm("f0") fval; +// __asm__ volatile ( +// "csrw 0x800, %[csr_enc]" +// : "+r" (ftmp) +// : [csr_enc] "r" (csr_enc) +// ); +//} while (0) +// +// +//inline float __attribute__((always_inline)) tensor_reduce_float(uint64_t fstart, uint64_t operation, uint64_t num_reg, uint64_t partnerID, uint64_t action) { +// uint64_t warl = 0; +// float out; +// uint64_t csr_enc = ((warl & 0x2 ) << 62) | +// ((fstart & 0x1F ) << 57) | +// ((warl & 0x1FFFFFFF ) << 28) | +// ((operation & 0xF ) << 24) | +// ((num_reg & 0xFF ) << 16) | +// ((partnerID & 0x1FFF ) << 3 ) | +// ((warl & 0x1 ) << 2 ) | +// ((action & 0x3 ) ); +// +// __asm__ volatile ( +// "csrw 0x800, %[csr_enc]\n" +// : /*empty*/ +// : [csr_enc] "r" (csr_enc), +// : /*"f0", "f1", "f2", "f3", "f4", +// "f5", "f6", "f7", "f8", "f9", +// "f10", "f11", "f12", "f13", "f14", +// "f15", "f16", "f17", "f18", "f19", +// "f20", "f21", "f22", "f23", "f24", +// "f25", "f26", "f27", "f28", "f29", +// "f30", "f31"*/ +// ); +// +// return out; +//} + +/*! \fn inline void tensor_reduce(uint64_t start_reg, uint64_t operation, uint64_t num_reg, uint64_t partnerID, uint64_t action) + \brief The TensorReduce instruction allows up to 216 harts to collectively calculate a reduction function. + \param start_reg starting register + \param operation Function to be performed. + \param num_reg number of registers + \param partnerID Receiver minionID. + \param action action value + \return uint32_t value after reduction + \tensorops Implementation of tensor_reduce api +*/ + +inline void __attribute__((always_inline)) tensor_reduce(uint64_t start_reg, + uint64_t operation, + uint64_t num_reg, + uint64_t partnerID, + uint64_t action) { + uint64_t warl = 0; + + uint64_t csr_enc = ((warl & 0x2) << 62) | ((start_reg & 0x1F) << 57) | ((warl & 0x1FFFFFFF) << 28) | + ((operation & 0xF) << 24) | ((num_reg & 0xFF) << 16) | ((partnerID & 0x1FFF) << 3) | + ((warl & 0x1) << 2) | ((action & 0x3)); + + __asm__ __volatile__("csrw 0x800, %[csr_enc]\n" : : [csr_enc] "r"(csr_enc) :); +} + +/*! \fn inline void tensor_reduce_send(uint64_t start_reg, uint64_t num_reg, uint64_t partnerID) + \brief This function applies reduce instruction to function and then sends to partner minion. + \param start_reg starting register + \param num_reg number of registers + \param partnerID Receiver minionID. + \return none + \tensorops Implementation of tensor_reduce_send api +*/ +inline void __attribute__((always_inline)) tensor_reduce_send(uint64_t start_reg, + uint64_t num_reg, + uint64_t partnerID) { + uint64_t warl = 0; + tensor_reduce(start_reg, warl, num_reg, partnerID, 0); +} + +/*! \fn inline void tensor_reduce_recv(uint64_t start_reg, uint64_t operation, uint64_t num_reg, uint64_t partnerID) + \brief This function recieves reduce function from partner minion. + \param start_reg starting register + \param operation operation to be performed + \param num_reg number of registers + \param partnerID Receiver minionID. + \return none + \tensorops Implementation of tensor_reduce_recv api +*/ +inline void __attribute__((always_inline)) tensor_reduce_recv(uint64_t start_reg, + uint64_t operation, + uint64_t num_reg, + uint64_t partnerID) { + tensor_reduce(start_reg, operation, num_reg, partnerID, 1); +} + +/*! \fn inline void tensor_reduce_auto(uint64_t start_reg, uint64_t operation, uint64_t num_reg, uint64_t tree_depth) + \brief The Tensor reduce instruction allows up to 216 harts to collectively calculate a reduction function. + \param start_reg starting register + \param operation operation to be performed + \param num_reg number of registers + \param tree_depth tree depth + \return none + \tensorops Implementation of tensor_reduce_auto api +*/ +inline void __attribute__((always_inline)) tensor_reduce_auto(uint64_t start_reg, + uint64_t operation, + uint64_t num_reg, + uint64_t tree_depth) { + tensor_reduce(start_reg, operation, num_reg, (0ULL << 4) | (tree_depth & 0xF), 3); +} + +/*! \fn inline void tensor_broadcast(uint64_t start_reg, uint64_t operation, uint64_t num_reg, uint64_t tree_depth) { + \brief The Tensor broadcast instruction allows up to 216 harts to receive values held in the vector registers + of one of the harts in the group. The broadcast operation is performed in a binary-tree fashion, where the source + data is originally in the root node and the final result ends up in the leaf nodes. + \param start_reg Starting floating-point register + \param operation operation to be performed + \param num_reg Number of floating-point registers + \param tree_depth tree depth + \return none + \tensorops Implementation of tensor_broadcast api +*/ +inline void __attribute__((always_inline)) tensor_broadcast(uint64_t start_reg, + uint64_t operation, + uint64_t num_reg, + uint64_t tree_depth) { + tensor_reduce(start_reg, operation, num_reg, (0ULL << 4) | (tree_depth & 0xF), 2); +} + +/*! \fn inline void tensor_reduce_autopair(uint64_t start_reg, uint64_t operation, uint64_t num_reg, uint64_t start_lvl, uint64_t end_lvl, uint64_t action) { + \brief This function is wrapper of Tensor Reduce (auto-pair variant) instruction. + \param start_reg Starting floating-point register + \param operation Function to be performed + \param num_reg Number of floating-point registers + \param start_lvl starting level value + \param end_lvl ending level value + \param action action value + \return none + \tensorops Implementation of tensor_reduce_autopair api + +*/ +inline void __attribute__((always_inline)) tensor_reduce_autopair(uint64_t start_reg, + uint64_t operation, + uint64_t num_reg, + uint64_t start_lvl, + uint64_t end_lvl, + uint64_t action) { + uint64_t partnerID; + // PRM-10 defines the partnerID field for Tensor Reduce (auto-pair variant) as following: + // [15:11] WARL(0) + // [10: 7] End level for autopair + // [ 6: 3] Start level for autopair + uint64_t warl = 0; + partnerID = ((warl & 0xF) << 11) | ((end_lvl & 0xF) << 7) | ((start_lvl & 0xF) << 3); + // Operations encoding: + // 0000=fadd, 0001=fsub, 0010=fmax, 0011=fmin, 0100=iadd, 0101=isub, 0110=imax, 0111=imin, 1000=fget + // + // Action encoding: + // 00=send, 01=receive, 10=auto-pair broadcast derive from hartid,11=auto-pair reduce derive from hartid + tensor_reduce(start_reg, operation, num_reg, (partnerID >> 3), action); +} + +/*! \fn inline void tensor_quant(uint64_t start_reg, uint64_t col, uint64_t row, uint64_t scp_loc, uint64_t transf9, uint64_t transf8, uint64_t transf7, uint64_t transf6, uint64_t transf5, uint64_t transf4, uint64_t transf3, uint64_t transf2, uint64_t transf1, uint64_t transf0 ) + \brief Tensor quantization (TensorQuant) instructions are encoded as writes to the tensor_quant CSR. The TensorQuant + instruction performs a sequence of up to 10 transformations to a matrix A + \param start_reg Starting register + \param col A matrix number of columns. + \param row A matrix number of rows. + \param scp_loc L1 scratchpad cache line where the first vector is stored. + \param transf9 Transformation 9. + \param transf8 Transformation 8. + \param transf7 Transformation 7. + \param transf6 Transformation 6. + \param transf5 Transformation 5. + \param transf4 Transformation 4. + \param transf3 Transformation 3. + \param transf2 Transformation 2. + \param transf1 Transformation 1. + \param transf0 Transformation 0. + \return none + \tensorops Implementation of tensor_quant api +*/ +inline void __attribute__((always_inline)) tensor_quant(uint64_t start_reg, + uint64_t col, + uint64_t row, + uint64_t scp_loc, + uint64_t transf9, + uint64_t transf8, + uint64_t transf7, + uint64_t transf6, + uint64_t transf5, + uint64_t transf4, + uint64_t transf3, + uint64_t transf2, + uint64_t transf1, + uint64_t transf0) { + uint64_t csr_enc = ((start_reg & 0x1F) << 57) | ((col & 0x3) << 55) | ((row & 0xF) << 51) | + ((scp_loc & 0x3F) << 45) | ((transf9 & 0xF) << 36) | ((transf8 & 0xF) << 32) | + ((transf7 & 0xF) << 28) | ((transf6 & 0xF) << 24) | ((transf5 & 0xF) << 20) | + ((transf4 & 0xF) << 16) | ((transf3 & 0xF) << 12) | ((transf2 & 0xF) << 8) | + ((transf1 & 0xF) << 4) | ((transf0 & 0xF) << 0); + + __asm__ __volatile__("csrw 0x806, %[csr_enc]\n" : : [csr_enc] "r"(csr_enc) :); +} + +/*! \fn inline void tensor_mask(uint64_t zeros, uint64_t mask_bits) + \brief The TensorLoad, TensorFMA, and CacheOp instructions can operate under the + control of the tensor_mask CSR. The tensor_mask CSR contains one bit for each + of the destination lines that TensorLoad can potentially write into the scratchpad + \param zeros all zeros + \param mask_bits tensor bit mask + \return none + \tensorops Implementation of tensor_mask api +*/ +inline void __attribute__((always_inline)) tensor_mask(uint64_t zeros, uint64_t mask_bits) { + uint64_t csr_enc = ((zeros & 0x000000000000) << 16) | (mask_bits & 0xFFFF); + + __asm__ __volatile__("csrw 0x805, %[csr_enc]\n" : : [csr_enc] "r"(csr_enc) :); +} + +/*! \fn inline void tensor_coop(uint64_t val) + \brief The tensor_coop instruction specifies which harts participate in cooperative tensor load operations. Only the first hart of each + selected Minion core participates in the cooperative operations, since the second hart cannot issue tensor load operations. + \param val value contains encoded coop id, minion and neigh mask + \return none + \tensorops Implementation of tensor_coop api +*/ +inline void __attribute__((always_inline)) tensor_coop(uint64_t val) { + __asm__ __volatile__("csrw 0x804, %[val]\n" : : [val] "r"(val) :); +} + +/*! \fn inline void convolution_ctrl(uint64_t row_start, uint64_t col_start) + \brief This function modifies the convolution control register. + This register encodes the location of a tensor inside a larger two-dimensional array. + \param row_start signed integer value specifying the row inside the array where the first row of the tensor resides + \param col_start signed integer value specifying the column inside the array where the first column of the tensor resides + \return none + \tensorops Implementation of convolution_ctrl api +*/ +inline void __attribute__((always_inline)) convolution_ctrl(uint64_t row_start, uint64_t col_start) { + uint64_t csr_enc = ((row_start & 0xFFFF) << 32) | (col_start & 0xFFFF); + + __asm__ __volatile__("csrw 0x803, %[csr_enc]\n" : : [csr_enc] "r"(csr_enc) :); +} + +/*! \fn inline void convolution_size(uint64_t srow, uint64_t nrow, uint64_t scol, uint64_t ncol) + \brief This function modifies the convolution size register. + This register specifies the layout of a two-dimensional array used for convolutions. + \param srow integer value specifying the row inside the array where the first row of the tensor resides + \param nrow integer values specifying the number of rows of the array + \param scol integer value specifying the distance, in number of columns, between consecutive column accesses to the array during + convolution operations + \param ncol integer values specifying the number of columns of the array + \return none + \tensorops Implementation of convolution_size api +*/ +inline void __attribute__((always_inline)) convolution_size(uint64_t srow, + uint64_t nrow, + uint64_t scol, + uint64_t ncol) { + uint64_t csr_enc = ((srow & 0xFF) << 56) | ((nrow & 0xFFFF) << 32) | ((scol & 0xFF) << 24) | ((ncol & 0xFFFF)); + + __asm__ __volatile__("csrw 0x802, %[csr_enc]\n" : : [csr_enc] "r"(csr_enc) :); +} + +/*! \fn inline unsigned get_tensor_error() + \brief This function returns tensor error register value. + The tensor_error register accrues errors that occur during the execution of tensor instructions and cache management operations. When the tensor coprocessor or the cache management coprocessor generates an exception, the exception is recorded in + the tensor_error register and execution does not trap. The tensor_error register is never cleared by the implementation. It is the + responsibility of the software to clear tensor_error + \return Tensor error value + \tensorops Implementation of get_tensor_error api +*/ +inline unsigned long __attribute__((always_inline)) get_tensor_error() { + unsigned long error; + + __asm__ __volatile__("csrr %0, 0x808" : "=r"(error)); + + return error; +} + +/*! \fn inline uint64_t get_tensor_mask() + \brief This function returns tensor mask register value. + \return Tensor mask value + \tensorops Implementation of get_tensor_mask api +*/ +inline uint64_t __attribute__((always_inline)) get_tensor_mask() { + uint64_t val; + + __asm__ __volatile__("csrr %0, 0x805" : "=r"(val)); + + return val; +} + +#define mask_set(msk, val) \ + do { \ + __asm__ volatile("mov.m.x m" #msk ", zero, %0" ::"n"(val)); \ + } while (0) + +#define flw_ps(fd, ptr) \ + do { \ + __asm__ volatile("flw.ps f" #fd ", (%0)" ::"r"(ptr)); \ + } while (0) + +#define fsw_ps(fd, ptr) \ + do { \ + __asm__ volatile("fsw.ps f" #fd ", (%0)" ::"r"(ptr) : "memory"); \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif // ! __TENSORS_H diff --git a/ggml/src/ggml-et/et-kernels/src/tri_f32.c b/ggml/src/ggml-et/et-kernels/src/tri_f32.c new file mode 100644 index 0000000..e33e4a3 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/tri_f32.c @@ -0,0 +1,244 @@ +//****************************************************************************** +// Tri F32 Kernel +// Triangular masking: zero out elements outside the triangular region. +// +// tri_type (matches ggml_tri_type enum): +// 0 = UPPER_DIAG: keep where i0 >= i1 +// 1 = UPPER: keep where i0 > i1 +// 2 = LOWER_DIAG: keep where i0 <= i1 +// 3 = LOWER: keep where i0 < i1 +// +// Distribution: cache-line aligned chunks of the flat contiguous dst. +// Each element is individually classified as keep or zero based on its +// (i0, i1) coordinates. This avoids cache-line sharing between threads +// when ne0 is not a multiple of 16. +//****************************************************************************** + +#include "ggml_tensor.h" +#include "platform.h" + +#include + +#define TRI_TYPE_UPPER_DIAG 0 +#define TRI_TYPE_UPPER 1 +#define TRI_TYPE_LOWER_DIAG 2 +#define TRI_TYPE_LOWER 3 + +struct ggml_et_tri_params { + struct ggml_tensor src0; + struct ggml_tensor dst; + int32_t tri_type; +}; + +static inline int keep_element(int32_t tri_type, int64_t i0, int64_t i1) { + switch (tri_type) { + case TRI_TYPE_LOWER: + return i0 < i1; + case TRI_TYPE_LOWER_DIAG: + return i0 <= i1; + case TRI_TYPE_UPPER: + return i0 > i1; + case TRI_TYPE_UPPER_DIAG: + return i0 >= i1; + default: + return 0; + } +} + +int entry_point(struct ggml_et_tri_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + int32_t tri_type = params->tri_type; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + const int64_t ne0 = dst->ne[0]; + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t ne3 = dst->ne[3]; + + const size_t nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const size_t nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + const int64_t total_rows = ne1 * ne2 * ne3; + + //========================================================================== + // Fast path: ne0 % 16 == 0 — rows are cache-line aligned, distribute rows + //========================================================================== + if (ne0 % 16 == 0) { + float zero = 0.0f; + __asm__ volatile("fbc.ps f10, %[z]\n" : : [z] "m"(zero) : "f10"); + + for (int64_t row = thread_id; row < total_rows; row += num_threads) { + const int64_t i1 = row % ne1; + const int64_t i2 = (row / ne1) % ne2; + const int64_t i3 = row / (ne1 * ne2); + + const float * src_row = (const float *) ((const char *) src0_data + i1 * nb01 + i2 * nb02 + i3 * nb03); + float * dst_row = (float *) ((char *) dst_data + i1 * nb1 + i2 * nb2 + i3 * nb3); + + int64_t keep_start, keep_end; + switch (tri_type) { + case TRI_TYPE_LOWER: + keep_start = 0; + keep_end = i1; + break; + case TRI_TYPE_LOWER_DIAG: + keep_start = 0; + keep_end = i1 + 1; + break; + case TRI_TYPE_UPPER: + keep_start = i1 + 1; + keep_end = ne0; + break; + case TRI_TYPE_UPPER_DIAG: + keep_start = i1; + keep_end = ne0; + break; + default: + return -1; + } + if (keep_end > ne0) { + keep_end = ne0; + } + + // Zero prefix [0, keep_start) — SIMD for aligned blocks, scalar tail + int64_t i0 = 0; + for (; i0 + 8 <= keep_start; i0 += 8) { + __asm__ volatile("fsw.ps f10, %[d]\n" : [d] "=m"(*(float (*)[8]) & dst_row[i0])::"f10"); + } + for (; i0 < keep_start; i0++) { + dst_row[i0] = 0.0f; + } + + // Copy kept region [keep_start, keep_end) — SIMD + scalar tail + for (; i0 + 8 <= keep_end; i0 += 8) { + __asm__ volatile( + "flw.ps f11, %[s]\n" + "fsw.ps f11, %[d]\n" + : [d] "=m"(*(float (*)[8]) & dst_row[i0]) + : [s] "m"(*(const float (*)[8]) & src_row[i0]) + : "f11"); + } + for (; i0 < keep_end; i0++) { + dst_row[i0] = src_row[i0]; + } + + // Zero suffix [keep_end, ne0) — SIMD + scalar tail + for (; i0 + 8 <= ne0; i0 += 8) { + __asm__ volatile("fsw.ps f10, %[d]\n" : [d] "=m"(*(float (*)[8]) & dst_row[i0])::"f10"); + } + for (; i0 < ne0; i0++) { + dst_row[i0] = 0.0f; + } + } + return 0; + } + + //========================================================================== + // Unaligned fallback: distribute by cache lines, scalar per element + //========================================================================== + { + const int64_t total_elements = ne0 * ne1 * ne2 * ne3; + const int64_t elems_per_cl = 16; + const int64_t total_cl = (total_elements + elems_per_cl - 1) / elems_per_cl; + + const int64_t cl_per_thread = (total_cl + num_threads - 1) / num_threads; + const int64_t cl_start = thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cl) { + cl_end = total_cl; + } + if (cl_start >= total_cl) { + return 0; + } + + const int64_t es = cl_start * elems_per_cl; + int64_t ee = cl_end * elems_per_cl; + if (ee > total_elements) { + ee = total_elements; + } + + int64_t row_idx = es / ne0; + int64_t col = es % ne0; + + int64_t pos = es; + while (pos < ee) { + const int64_t i1 = row_idx % ne1; + const int64_t i2 = (row_idx / ne1) % ne2; + const int64_t i3 = row_idx / (ne1 * ne2); + + const float * src_row = (const float *) ((const char *) src0_data + i1 * nb01 + i2 * nb02 + i3 * nb03); + + int64_t row_remaining = ne0 - col; + int64_t chunk_remaining = ee - pos; + int64_t n = row_remaining < chunk_remaining ? row_remaining : chunk_remaining; + + int64_t keep_start, keep_end; + switch (tri_type) { + case TRI_TYPE_LOWER: + keep_start = 0; + keep_end = i1; + break; + case TRI_TYPE_LOWER_DIAG: + keep_start = 0; + keep_end = i1 + 1; + break; + case TRI_TYPE_UPPER: + keep_start = i1 + 1; + keep_end = ne0; + break; + case TRI_TYPE_UPPER_DIAG: + keep_start = i1; + keep_end = ne0; + break; + default: + return -1; + } + if (keep_end > ne0) { + keep_end = ne0; + } + + int64_t end_col = col + n; + for (int64_t i0 = col; i0 < end_col; i0++) { + if (i0 >= keep_start && i0 < keep_end) { + dst_data[pos + (i0 - col)] = src_row[i0]; + } else { + dst_data[pos + (i0 - col)] = 0.0f; + } + } + + pos += n; + col = 0; + row_idx++; + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/uberkernel.c b/ggml/src/ggml-et/et-kernels/src/uberkernel.c new file mode 100644 index 0000000..40d1cf9 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/uberkernel.c @@ -0,0 +1,497 @@ +#include "ggml-et-uberkernel-common.h" +#include "ggml-et-uberkernel-kernel-map.h" +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include + +struct ggml_et_glu_params; +struct ggml_et_unary_params; +struct ggml_et_rope_params; +struct ggml_et_rms_norm_params; +struct ggml_et_rms_norm_mul_params; +struct ggml_et_softmax_params; +struct ggml_et_set_rows_params; +struct ggml_et_get_rows_params; +struct ggml_et_cont_params; +struct ggml_et_concat_params; +struct ggml_et_cumsum_params; +struct ggml_et_diag_params; +struct ggml_et_fill_params; +struct ggml_et_flash_attn_ext_params; +struct ggml_et_gated_delta_net_params; +struct ggml_et_group_norm_params; +struct ggml_et_im2col_params; +struct ggml_et_l2_norm_params; +struct ggml_et_mul_mat_id_params; +struct ggml_et_norm_params; +struct ggml_et_pad_params; +struct ggml_et_repeat_params; +struct ggml_et_rwkv_wkv6_params; +struct ggml_et_rwkv_wkv7_params; +struct ggml_et_scale_params; +struct ggml_et_set_params; +struct ggml_et_solve_tri_params; +struct ggml_et_sqr_params; +struct ggml_et_ssm_conv_params; +struct ggml_et_ssm_scan_params; +struct ggml_et_sum_rows_params; +struct ggml_et_tri_params; + +extern int el_map_f32_entry(struct ggml_et_binary_params *, void *); +extern int glu_f32_entry(struct ggml_et_glu_params *, void *); +extern int unary_f32_entry(struct ggml_et_unary_params *, void *); +extern int rope_f32_entry(struct ggml_et_rope_params *, void *); +extern int rms_norm_f32_entry(struct ggml_et_rms_norm_params *, void *); +extern int rms_norm_mul_f32_entry(struct ggml_et_rms_norm_mul_params *, void *); +extern int softmax_f32_entry(struct ggml_et_softmax_params *, void *); +extern int set_rows_f32_entry(struct ggml_et_set_rows_params *, void *); +extern int get_rows_f32_entry(struct ggml_et_get_rows_params *, void *); +extern int cont_f32_entry(struct ggml_et_cont_params *, void *); +extern int cont_f16_entry(struct ggml_et_cont_params *, void *); +extern int cpy_f32_f16_entry(struct ggml_et_cont_params *, void *); +extern int concat_f32_entry(struct ggml_et_concat_params *, void *); +extern int cumsum_f32_entry(struct ggml_et_cumsum_params *, void *); +extern int diag_f32_entry(struct ggml_et_diag_params *, void *); +extern int fill_f32_entry(struct ggml_et_fill_params *, void *); +extern int flash_attn_ext_f32_entry(struct ggml_et_flash_attn_ext_params *, void *); +extern int flash_attn_ext_f16_me_entry(struct ggml_et_flash_attn_ext_params *, void *); +extern int gated_delta_net_f32_entry(struct ggml_et_gated_delta_net_params *, void *); +extern int group_norm_f32_entry(struct ggml_et_group_norm_params *, void *); +extern int im2col_entry(struct ggml_et_im2col_params *, void *); +extern int l2_norm_f32_entry(struct ggml_et_l2_norm_params *, void *); +extern int mul_mat_id_f32_entry(struct ggml_et_mul_mat_id_params *, void *); +extern int norm_f32_entry(struct ggml_et_norm_params *, void *); +extern int pad_f32_entry(struct ggml_et_pad_params *, void *); +extern int repeat_f32_entry(struct ggml_et_repeat_params *, void *); +extern int rwkv_wkv6_f32_entry(struct ggml_et_rwkv_wkv6_params *, void *); +extern int rwkv_wkv7_f32_entry(struct ggml_et_rwkv_wkv7_params *, void *); +extern int scale_f32_entry(struct ggml_et_scale_params *, void *); +extern int set_f32_entry(struct ggml_et_set_params *, void *); +extern int solve_tri_f32_entry(struct ggml_et_solve_tri_params *, void *); +extern int sqr_f32_entry(struct ggml_et_sqr_params *, void *); +extern int ssm_conv_f32_entry(struct ggml_et_ssm_conv_params *, void *); +extern int ssm_scan_f32_entry(struct ggml_et_ssm_scan_params *, void *); +extern int sum_rows_f32_entry(struct ggml_et_sum_rows_params *, void *); +extern int tri_f32_entry(struct ggml_et_tri_params *, void *); +extern int mul_mat_f16_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_f16_matrix_engine_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_f32_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_f32_matrix_engine_entry(struct ggml_et_binary_params *, void *); +extern int mul_mat_Q8_0_entry(struct ggml_et_mm_q8_params *, void *); +extern int mul_mat_Q4_0_entry(struct ggml_et_binary_params *, void *); + +static inline size_t tensor_bytes(const struct ggml_tensor * t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} + +struct uber_glu_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor dst; + // trailing scalars omitted — not needed for eviction +}; + +struct uber_unary_params { + struct ggml_tensor src0; + struct ggml_tensor dst; +}; + +struct uber_rope_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor src2; + struct ggml_tensor dst; +}; + +struct uber_rms_norm_params { + struct ggml_tensor src0; + struct ggml_tensor dst; +}; + +struct uber_rms_norm_mul_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor dst; +}; + +struct uber_softmax_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor src2; + struct ggml_tensor dst; +}; + +struct uber_set_rows_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor dst; +}; + +struct uber_get_rows_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor dst; +}; + +struct uber_cont_params { + struct ggml_tensor src0; + struct ggml_tensor dst; +}; + +// src0 + src1 + dst (no trailing scalars needed for eviction) +struct uber_concat_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor dst; +}; + +struct uber_ssm_conv_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor dst; +}; + +struct uber_solve_tri_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor dst; +}; + +struct uber_mul_mat_id_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor src2; + struct ggml_tensor dst; +}; + +// flash_attn_ext: Q=src0, K=src1, V=src2, mask=src3, dst (mask optional) +struct uber_flash_attn_ext_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor src2; + struct ggml_tensor mask; + struct ggml_tensor dst; +}; + +// ssm_scan: 7 source tensors + dst +struct uber_ssm_scan_params { + struct ggml_tensor src0; + struct ggml_tensor src1; + struct ggml_tensor src2; + struct ggml_tensor src3; + struct ggml_tensor src4; + struct ggml_tensor src5; + struct ggml_tensor src6; + struct ggml_tensor dst; +}; + +// gated_delta_net: q,k,v,g,beta,state_in,dst +struct uber_gated_delta_net_params { + struct ggml_tensor q; + struct ggml_tensor k; + struct ggml_tensor v; + struct ggml_tensor g; + struct ggml_tensor beta; + struct ggml_tensor state_in; + struct ggml_tensor dst; +}; + +static void copy_f32_to_f16_row(uint16_t * dst, const float * src, int64_t num_elements) { + for (int64_t i = 0; i < num_elements; i++) { + dst[i] = fp32_to_fp16(src[i]); + } +} + +static void copy_f32_row(float * dst, const float * src, int64_t num_elements) { + for (int64_t i = 0; i < num_elements; i++) { + dst[i] = src[i]; + } +} + +static void evict_region_past_l2_local(const void * addr, size_t bytes) { + if (!addr || bytes == 0) { + return; + } + + const uint64_t CL = 64; + uint64_t base = (uint64_t) addr & ~(CL - 1); + uint64_t end = ((uint64_t) addr + bytes + CL - 1) & ~(CL - 1); + uint64_t nlines = (end - base) / CL; + cache_ops_priv_evict_sw(0, /*to_L2*/ 3, 0, 0, CL); +} + +int entry_point(struct ggml_et_uberkernel_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env || !params) { + return -1; + } + + struct ggml_et_uberkernel_inst * insts = (struct ggml_et_uberkernel_inst *) (uintptr_t) params->insts; + uint8_t * params_blob = (uint8_t *) (uintptr_t) params->params_blob; + + if (!insts || !params_blob || params->inst_stride < sizeof(struct ggml_et_uberkernel_inst)) { + return -1; + } + + for (uint32_t i = 0; i < params->num_insts; ++i) { + struct ggml_et_uberkernel_inst * inst = + (struct ggml_et_uberkernel_inst *) ((uint8_t *) insts + (i * params->inst_stride)); + void * inst_params = params_blob + inst->params_offset; + int rc = -1; + + et_barrier_global(32ULL); + + switch (inst->kernel_id) { + case GGML_ET_UBERKERNEL_KERNEL_EL_MAP_F32: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = el_map_f32_entry(p, env); + break; + } + // case GGML_ET_UBERKERNEL_KERNEL_UNARY_F32: { + // // struct uber_unary_params *p = (struct uber_unary_params *) inst_params; + // // et_barrier(ET_BARRIER_GLOBAL); + // rc = unary_f32_entry((struct ggml_et_unary_params *) inst_params, env); + // break; + // } + // case GGML_ET_UBERKERNEL_KERNEL_CPY_F32_F16: { + // struct uber_unary_params *p = (struct uber_unary_params *) inst_params; + // // evict_region_past_l2(p->src0.data, tensor_bytes(&p->src0)); + // rc = cpy_f32_f16_entry((struct ggml_et_cont_params *) inst_params, env); + // break; + // } + // case GGML_ET_UBERKERNEL_KERNEL_GET_ROWS_F32: { + // struct uber_get_rows_params *p = (struct uber_get_rows_params *) inst_params; + // rc = get_rows_f32_entry((struct ggml_et_get_rows_params *) inst_params, env); + // break; + // } + // case GGML_ET_UBERKERNEL_KERNEL_CONT_F32: { + // struct uber_cont_params *p = (struct uber_cont_params *) inst_params; + // // evict_region_past_l2_local(p->src0.data, tensor_bytes(&p->src0)); + // // evict_region_past_l2(p->dst.data, tensor_bytes(&p->dst)); + // rc = cont_f32_entry((struct ggml_et_cont_params *) inst_params, env); + // break; + // } + case GGML_ET_UBERKERNEL_KERNEL_GLU_F32: + { + rc = glu_f32_entry((struct ggml_et_glu_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_ROPE_F32: + { + rc = rope_f32_entry((struct ggml_et_rope_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_RMS_NORM_F32: + { + // struct ggml_et_rms_norm_params *p = (struct ggml_et_rms_norm_params *) inst_params; + // evict_region_past_l2(p->src0.data, tensor_bytes(&p->src0)); + rc = rms_norm_f32_entry((struct ggml_et_rms_norm_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_RMS_NORM_MUL_F32: + { + struct uber_rms_norm_mul_params * p = (struct uber_rms_norm_mul_params *) inst_params; + evict_region_past_l2(p->src0.data, tensor_bytes(&p->src0)); + evict_region_past_l2(p->src1.data, tensor_bytes(&p->src1)); + rc = rms_norm_mul_f32_entry((struct ggml_et_rms_norm_mul_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_SOFTMAX_F32: + { + rc = softmax_f32_entry((struct ggml_et_softmax_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_SET_ROWS_F32: + { + rc = set_rows_f32_entry((struct ggml_et_set_rows_params *) inst_params, env); + break; + } + + // Single-source ops (src0 → dst) + case GGML_ET_UBERKERNEL_KERNEL_SQR_F32: + { + rc = sqr_f32_entry((struct ggml_et_sqr_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_SCALE_F32: + { + rc = scale_f32_entry((struct ggml_et_scale_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_SUM_ROWS_F32: + { + rc = sum_rows_f32_entry((struct ggml_et_sum_rows_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_CUMSUM_F32: + { + rc = cumsum_f32_entry((struct ggml_et_cumsum_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_NORM_F32: + { + rc = norm_f32_entry((struct ggml_et_norm_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_L2_NORM_F32: + { + rc = l2_norm_f32_entry((struct ggml_et_l2_norm_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_GROUP_NORM_F32: + { + rc = group_norm_f32_entry((struct ggml_et_group_norm_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_REPEAT_F32: + { + rc = repeat_f32_entry((struct ggml_et_repeat_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_DIAG_F32: + { + rc = diag_f32_entry((struct ggml_et_diag_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_TRI_F32: + { + rc = tri_f32_entry((struct ggml_et_tri_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_PAD_F32: + { + rc = pad_f32_entry((struct ggml_et_pad_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_CONT_F16: + { + rc = cont_f16_entry((struct ggml_et_cont_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_FILL_F32: + { + rc = fill_f32_entry((struct ggml_et_fill_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_SET_F32: + { + rc = set_f32_entry((struct ggml_et_set_params *) inst_params, env); + break; + } + + // Two-source ops + case GGML_ET_UBERKERNEL_KERNEL_CONCAT_F32: + { + rc = concat_f32_entry((struct ggml_et_concat_params *) inst_params, env); + break; + } + // case GGML_ET_UBERKERNEL_KERNEL_SSM_CONV_F32: { + // rc = ssm_conv_f32_entry((struct ggml_et_ssm_conv_params *) inst_params, env); + // break; + // } + case GGML_ET_UBERKERNEL_KERNEL_SOLVE_TRI_F32: + { + rc = solve_tri_f32_entry((struct ggml_et_solve_tri_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_IM2COL: + { + rc = im2col_entry((struct ggml_et_im2col_params *) inst_params, env); + break; + } + + // Three-source ops + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_ID_F32: + { + rc = mul_mat_id_f32_entry((struct ggml_et_mul_mat_id_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_FLASH_ATTN_EXT_F32: + { + rc = flash_attn_ext_f32_entry((struct ggml_et_flash_attn_ext_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_FLASH_ATTN_EXT_F16_ME: + { + rc = flash_attn_ext_f16_me_entry((struct ggml_et_flash_attn_ext_params *) inst_params, env); + break; + } + + case GGML_ET_UBERKERNEL_KERNEL_GATED_DELTA_NET_F32: + { + rc = gated_delta_net_f32_entry((struct ggml_et_gated_delta_net_params *) inst_params, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_SSM_SCAN_F32: + { + rc = ssm_scan_f32_entry((struct ggml_et_ssm_scan_params *) inst_params, env); + break; + } + // rwkv: raw float* params, no ggml_tensor fields to evict via + case GGML_ET_UBERKERNEL_KERNEL_RWKV_WKV6_F32: + { + rc = rwkv_wkv6_f32_entry((struct ggml_et_rwkv_wkv6_params *) inst_params, env); + break; + } + + case GGML_ET_UBERKERNEL_KERNEL_RWKV_WKV7_F32: + { + rc = rwkv_wkv7_f32_entry((struct ggml_et_rwkv_wkv7_params *) inst_params, env); + break; + } + + // MUL_MAT: evict src1 (activations); src0=weights is + // read-only so never stale from a prior uberkernel op + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_F16: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_f16_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_F16_MATRIX_ENGINE: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_f16_matrix_engine_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_F32: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_f32_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_F32_MATRIX_ENGINE: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_f32_matrix_engine_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q8_0: + { + struct ggml_et_mm_q8_params * p = (struct ggml_et_mm_q8_params *) inst_params; + // evict_region_past_l2(p->src0.data, tensor_bytes(&p->src0)); + rc = mul_mat_Q8_0_entry(p, env); + break; + } + case GGML_ET_UBERKERNEL_KERNEL_MUL_MAT_Q4_0: + { + struct ggml_et_binary_params * p = (struct ggml_et_binary_params *) inst_params; + rc = mul_mat_Q4_0_entry(p, env); + break; + } + + default: + return -1; + } + + if (rc != 0) { + return rc; + } + } + + return 0; +} diff --git a/ggml/src/ggml-et/et-kernels/src/unary_f32.c b/ggml/src/ggml-et/et-kernels/src/unary_f32.c new file mode 100644 index 0000000..42282c0 --- /dev/null +++ b/ggml/src/ggml-et/et-kernels/src/unary_f32.c @@ -0,0 +1,705 @@ +//****************************************************************************** +// Unary F32 Kernel +// Element-wise unary operations: dst[i] = f(src0[i]) +// All ops vectorized using 8-wide ET SIMD (fexp.ps, frcp.ps, flog.ps, etc.) +// +// Supports: ABS, SGN, NEG, STEP, TANH, ELU, RELU, SIGMOID, GELU, GELU_QUICK, +// SILU, HARDSWISH, HARDSIGMOID, EXP, EXPM1, SOFTPLUS, GELU_ERF +//****************************************************************************** + +#include "ggml_tensor.h" +#include "math_fp.h" +#include "platform.h" + +#include + +// Unary kernel parameters structure +struct ggml_et_unary_params { + struct ggml_tensor src0; // F32 input tensor + struct ggml_tensor dst; // F32 output tensor + int32_t unary_op; // ggml_unary_op enum value +}; + +//****************************************************************************** +// Vectorized 8-wide block operations +// All process exactly 8 floats per call using ET vector instructions. +// ne0 is guaranteed % 16 == 0, so the inner loop always calls with i0 += 8. +//****************************************************************************** + +// NEG: dst = -x (zero - x) +static inline void vec_neg(float * dst, const float * src, int32_t n) { + float zero = 0.0f; + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "fbc.ps f10, %[z]\n" + "flw.ps f11, %[x]\n" + "fsub.ps f12, f10, f11\n" + "fsw.ps f12, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [z] "m"(zero) + : "f10", "f11", "f12"); + } +} + +// ABS: dst = |x| (negate negative values: abs = x * sgn, or max(x, -x)) +// Uses: negate then fmax.ps +static inline void vec_abs(float * dst, const float * src, int32_t n) { + float zero = 0.0f; + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "fbc.ps f10, %[z]\n" + "flw.ps f11, %[x]\n" + "fsub.ps f12, f10, f11\n" // f12 = -x + "fmax.ps f13, f11, f12\n" // f13 = max(x, -x) = |x| + "fsw.ps f13, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [z] "m"(zero) + : "f10", "f11", "f12", "f13"); + } +} + +// RELU: dst = max(0, x) +static inline void vec_relu(float * dst, const float * src, int32_t n) { + float zero = 0.0f; + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "fbc.ps f10, %[z]\n" + "flw.ps f11, %[x]\n" + "fmax.ps f12, f10, f11\n" // max(0, x) + "fsw.ps f12, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [z] "m"(zero) + : "f10", "f11", "f12"); + } +} + +// STEP: dst = x > 0 ? 1 : 0 (clamp to [0,1] via max then min-ish, or use sign bit) +// Trick: relu(x) then frcp gives inf for 0 and finite for >0, but simpler: +// step(x) = min(1, relu(x) * huge) ... too fragile. Scalar is fine for step/sgn. +static inline void vec_step(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i++) { + dst[i] = (src[i] > 0.0f) ? 1.0f : 0.0f; + } +} + +// SGN: dst = sign(x) = x>0 ? 1 : (x<0 ? -1 : 0) +static inline void vec_sgn(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i++) { + dst[i] = (src[i] > 0.0f) ? 1.0f : ((src[i] < 0.0f) ? -1.0f : 0.0f); + } +} + +// EXP: dst = exp(x) +// fexp.ps computes 2^x, so feed x * log2(e) +static inline void vec_exp(float * dst, const float * src, int32_t n) { + float log2e = 1.4426950408889634f; + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f11, %[l2e]\n" + "fmul.ps f12, f10, f11\n" // x * log2(e) + "fexp.ps f13, f12\n" // 2^(x*log2e) = exp(x) + "fsw.ps f13, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [l2e] "m"(log2e) + : "f10", "f11", "f12", "f13"); + } +} + +// EXPM1: dst = exp(x) - 1 +static inline void vec_expm1(float * dst, const float * src, int32_t n) { + float log2e = 1.4426950408889634f; + float one = 1.0f; + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f11, %[l2e]\n" + "fbc.ps f14, %[one]\n" + "fmul.ps f12, f10, f11\n" // x * log2(e) + "fexp.ps f13, f12\n" // exp(x) + "fsub.ps f13, f13, f14\n" // exp(x) - 1 + "fsw.ps f13, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [l2e] "m"(log2e), [one] "m"(one) + : "f10", "f11", "f12", "f13", "f14"); + } +} + +// SIGMOID: dst = 1 / (1 + exp(-x)) +// Same pattern as SwiGLU: exp(-x) via fexp.ps, then frcp.ps +static inline void vec_sigmoid(float * dst, const float * src, int32_t n) { + float zero = 0.0f; + float one = 1.0f; + float log2e = 1.4426950408889634f; + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f20, %[z]\n" + "fbc.ps f21, %[one]\n" + "fbc.ps f22, %[l2e]\n" + "fsub.ps f12, f20, f10\n" // -x + "fmul.ps f13, f12, f22\n" // -x * log2(e) + "fexp.ps f14, f13\n" // exp(-x) + "fadd.ps f15, f14, f21\n" // 1 + exp(-x) + "frcp.ps f16, f15\n" // 1 / (1 + exp(-x)) + "fsw.ps f16, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [z] "m"(zero), [one] "m"(one), [l2e] "m"(log2e) + : "f10", "f12", "f13", "f14", "f15", "f16", "f20", "f21", "f22"); + } +} + +// TANH: dst = (exp(2x) - 1) / (exp(2x) + 1) +// Rewrite as: 1 - 2/(exp(2x) + 1) to use frcp.ps +// Or equivalently: 2*sigmoid(2x) - 1 +static inline void vec_tanh(float * dst, const float * src, int32_t n) { + float one = 1.0f; + float two = 2.0f; + float two_log2e = 2.8853900817779268f; // 2 * log2(e) + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f20, %[one]\n" + "fbc.ps f21, %[two]\n" + "fbc.ps f22, %[tl2e]\n" + // exp(2x) via fexp.ps: feed 2x * log2(e) + "fmul.ps f12, f10, f22\n" // 2x * log2(e) + "fexp.ps f13, f12\n" // exp(2x) + "fadd.ps f14, f13, f20\n" // exp(2x) + 1 + "frcp.ps f15, f14\n" // 1 / (exp(2x) + 1) + "fmul.ps f16, f21, f15\n" // 2 / (exp(2x) + 1) + "fsub.ps f17, f20, f16\n" // 1 - 2/(exp(2x)+1) = tanh(x) + "fsw.ps f17, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [one] "m"(one), [two] "m"(two), [tl2e] "m"(two_log2e) + : "f10", "f12", "f13", "f14", "f15", "f16", "f17", "f20", "f21", "f22"); + } +} + +// SILU: dst = x / (1 + exp(-x)) = x * sigmoid(x) +// Copied from SwiGLU pattern but without the gate multiply +static inline void vec_silu(float * dst, const float * src, int32_t n) { + float zero = 0.0f; + float one = 1.0f; + float log2e = 1.4426950408889634f; + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f20, %[z]\n" + "fbc.ps f21, %[one]\n" + "fbc.ps f22, %[l2e]\n" + "fsub.ps f12, f20, f10\n" // -x + "fmul.ps f13, f12, f22\n" // -x * log2(e) + "fexp.ps f14, f13\n" // exp(-x) + "fadd.ps f15, f14, f21\n" // 1 + exp(-x) + "frcp.ps f16, f15\n" // 1 / (1 + exp(-x)) + "fmul.ps f17, f10, f16\n" // x * sigmoid(x) + "fsw.ps f17, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [z] "m"(zero), [one] "m"(one), [l2e] "m"(log2e) + : "f10", "f12", "f13", "f14", "f15", "f16", "f17", "f20", "f21", "f22"); + } +} + +// ELU: dst = x > 0 ? x : exp(x) - 1 +// Vector: compute exp(x)-1 for all lanes, then fmax(x, exp(x)-1) +// Works because for x>0: x > exp(x)-1 is not always true... +// Actually for x>0, exp(x)-1 > x (since exp(x) > x+1 for x>0). +// So fmax won't work. Use: compute both, blend via comparison. +// Simpler: exp(x)-1 for all, then for x>0 overwrite with x. +// Without per-lane masking, do scalar for ELU. +static inline void vec_elu(float * dst, const float * src, int32_t n) { + float log2e = 1.4426950408889634f; + float one = 1.0f; + // Compute exp(x)-1 vectorized, then fixup positive elements + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f11, %[l2e]\n" + "fbc.ps f14, %[one]\n" + "fmul.ps f12, f10, f11\n" // x * log2(e) + "fexp.ps f13, f12\n" // exp(x) + "fsub.ps f13, f13, f14\n" // exp(x) - 1 + "fsw.ps f13, %[r]\n" // store exp(x)-1 + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [l2e] "m"(log2e), [one] "m"(one) + : "f10", "f11", "f12", "f13", "f14"); + // Fixup: for x > 0, dst = x + for (int32_t j = 0; j < 8 && (i + j) < n; j++) { + if (src[i + j] > 0.0f) { + dst[i + j] = src[i + j]; + } + } + } +} + +// GELU: 0.5*x*(1 + tanh(sqrt(2/pi) * x * (1 + 0.044715*x^2))) +// Reformulated as: x * (1 - 1/(exp(2z)+1)) where z = sqrt(2/pi)*x*(1+0.044715*x^2) +// NaN-safe: avoids inf*0. Copied from GeGLU block pattern. +static inline void vec_gelu(float * dst, const float * src, int32_t n) { + float one = 1.0f; + float half = 0.5f; + float coef_a = 0.044715f; + float sqrt2pi = 0.79788456080286535587989211986876f; + float two_log2e = 2.8853900817779268f; // 2 * log2(e) + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f20, %[one]\n" + "fbc.ps f21, %[half]\n" + "fbc.ps f22, %[coef]\n" + "fbc.ps f23, %[s2pi]\n" + "fbc.ps f24, %[tl2e]\n" + // inner = 1 + 0.044715 * x^2 + "fmul.ps f12, f10, f10\n" // x^2 + "fmadd.ps f13, f22, f12, f20\n" // 1 + 0.044715*x^2 + // z = sqrt(2/pi) * x * inner + "fmul.ps f14, f23, f10\n" // sqrt(2/pi) * x + "fmul.ps f14, f14, f13\n" // z + // exp(2z) via fexp.ps + "fmul.ps f15, f14, f24\n" // 2z * log2(e) + "fexp.ps f15, f15\n" // exp(2z) + // gelu(x) = 0.5 * x * (1 + tanh(z)) + // = 0.5 * x * (1 + 1 - 2/(exp(2z)+1)) + // = x * (1 - 1/(exp(2z)+1)) ... wait, that's tanh-based + // Actually: 0.5*x*(1 + tanh) = 0.5*x*(1 + 1 - 2/(e2z+1)) = x*(1 - 1/(e2z+1)) + // Hmm: tanh = (e2z-1)/(e2z+1) = 1 - 2/(e2z+1) + // So 0.5*(1+tanh) = 0.5*(2 - 2/(e2z+1)) = 1 - 1/(e2z+1) + // gelu = x * (1 - 1/(e2z+1)) -- matches GeGLU pattern exactly + "fadd.ps f16, f15, f20\n" // exp(2z) + 1 + "frcp.ps f16, f16\n" // 1/(exp(2z) + 1) + "fsub.ps f16, f20, f16\n" // 1 - 1/(exp(2z)+1) = sigmoid(2z) + "fmul.ps f17, f10, f16\n" // x * sigmoid(2z) = gelu(x) + "fsw.ps f17, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [one] "m"(one), [half] "m"(half), [coef] "m"(coef_a), + [s2pi] "m"(sqrt2pi), [tl2e] "m"(two_log2e) + : "f10", "f12", "f13", "f14", "f15", "f16", "f17", "f20", "f21", "f22", "f23", "f24"); + } +} + +// GELU_QUICK: x * sigmoid(1.702 * x) = x / (1 + exp(-1.702*x)) +static inline void vec_gelu_quick(float * dst, const float * src, int32_t n) { + float one = 1.0f; + // -1.702 * log2(e) precomputed + float neg_coef_log2e = -1.702f * 1.4426950408889634f; // ~ -2.4542 + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f20, %[one]\n" + "fbc.ps f21, %[ncl2e]\n" + // exp(-1.702*x): feed -1.702*x*log2(e) = x * (-1.702*log2(e)) + "fmul.ps f12, f10, f21\n" // x * (-1.702*log2(e)) + "fexp.ps f13, f12\n" // exp(-1.702*x) + "fadd.ps f14, f13, f20\n" // 1 + exp(-1.702*x) + "frcp.ps f15, f14\n" // sigmoid(1.702*x) + "fmul.ps f16, f10, f15\n" // x * sigmoid(1.702*x) + "fsw.ps f16, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [one] "m"(one), [ncl2e] "m"(neg_coef_log2e) + : "f10", "f12", "f13", "f14", "f15", "f16", "f20", "f21"); + } +} + +// GELU_ERF: 0.5 * x * (1 + erf(x / sqrt(2))) +// erf approximation (Abramowitz & Stegun) is hard to vectorize cleanly, keep scalar +// but use et_expf for the exp(-z^2) part +static inline void vec_gelu_erf(float * dst, const float * src, int32_t n) { + const float SQRT_2_INV = 0.70710678118654752440084436210484f; + for (int32_t i = 0; i < n; i++) { + float x = src[i]; + float z = x * SQRT_2_INV; + float az = z < 0.0f ? -z : z; + + float t = et_fdiv(1.0f, 1.0f + 0.3275911f * az); + float t2 = t * t; + float t3 = t2 * t; + float t4 = t3 * t; + float t5 = t4 * t; + + float poly = 0.254829592f * t - 0.284496736f * t2 + 1.421413741f * t3 - 1.453152027f * t4 + 1.061405429f * t5; + + float erf_pos = 1.0f - poly * et_expf(-(az * az)); + float erf_val = (z < 0.0f) ? -erf_pos : erf_pos; + dst[i] = 0.5f * x * (1.0f + erf_val); + } +} + +// HARDSIGMOID: min(1, max(0, (x + 3) / 6)) +// Vector: compute (x+3)/6 via frcp, then clamp with fmax(0) and fmin(1) +static inline void vec_hardsigmoid(float * dst, const float * src, int32_t n) { + float zero = 0.0f; + float one = 1.0f; + float three = 3.0f; + float inv6 = 0.16666666666666666f; // 1/6 + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f20, %[z]\n" + "fbc.ps f21, %[one]\n" + "fbc.ps f22, %[thr]\n" + "fbc.ps f23, %[inv]\n" + "fadd.ps f12, f10, f22\n" // x + 3 + "fmul.ps f13, f12, f23\n" // (x + 3) / 6 + "fmax.ps f14, f13, f20\n" // max(0, ...) + "fmin.ps f15, f14, f21\n" // min(1, ...) + "fsw.ps f15, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [z] "m"(zero), [one] "m"(one), [thr] "m"(three), [inv] "m"(inv6) + : "f10", "f12", "f13", "f14", "f15", "f20", "f21", "f22", "f23"); + } +} + +// HARDSWISH: x * hardsigmoid(x) = x * min(1, max(0, (x+3)/6)) +static inline void vec_hardswish(float * dst, const float * src, int32_t n) { + float zero = 0.0f; + float one = 1.0f; + float three = 3.0f; + float inv6 = 0.16666666666666666f; + for (int32_t i = 0; i < n; i += 8) { + __asm__ volatile( + "flw.ps f10, %[x]\n" + "fbc.ps f20, %[z]\n" + "fbc.ps f21, %[one]\n" + "fbc.ps f22, %[thr]\n" + "fbc.ps f23, %[inv]\n" + "fadd.ps f12, f10, f22\n" // x + 3 + "fmul.ps f13, f12, f23\n" // (x + 3) / 6 + "fmax.ps f14, f13, f20\n" // max(0, ...) + "fmin.ps f15, f14, f21\n" // min(1, ...) + "fmul.ps f16, f10, f15\n" // x * hardsigmoid(x) + "fsw.ps f16, %[r]\n" + : [r] "=m"(*(float (*)[8]) & dst[i]) + : [x] "m"(*(const float (*)[8]) & src[i]), [z] "m"(zero), [one] "m"(one), [thr] "m"(three), [inv] "m"(inv6) + : "f10", "f12", "f13", "f14", "f15", "f16", "f20", "f21", "f22", "f23"); + } +} + +// FLOOR: largest integer <= x +static inline void vec_floor(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i++) { + float x = src[i]; + float t = (float) (int32_t) x; + dst[i] = (t > x) ? t - 1.0f : t; + } +} + +// CEIL: smallest integer >= x +static inline void vec_ceil(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i++) { + float x = src[i]; + float t = (float) (int32_t) x; + dst[i] = (t < x) ? t + 1.0f : t; + } +} + +// TRUNC: round towards zero +static inline void vec_trunc(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i++) { + dst[i] = (float) (int32_t) src[i]; + } +} + +// ROUND: round to nearest, ties to even (banker's rounding) +static inline void vec_round(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i++) { + float x = src[i]; + float t = (float) (int32_t) x; + float diff = x - t; + if (diff > 0.5f || (diff == 0.5f && ((int32_t) t & 1))) { + t += 1.0f; + } else if (diff < -0.5f || (diff == -0.5f && ((int32_t) t & 1))) { + t -= 1.0f; + } + dst[i] = t; + } +} + +// SOFTPLUS: log(1 + exp(x)) +// For large x (>20), softplus(x) ~ x. For moderate x, use fexp + flog. +// Scalar fallback since flog.ps computes log2, need conversion, and overflow guard +static inline void vec_softplus(float * dst, const float * src, int32_t n) { + for (int32_t i = 0; i < n; i++) { + float x = src[i]; + dst[i] = (x > 20.0f) ? x : et_logf(1.0f + et_expf(x)); + } +} + +static inline size_t tensor_bytes(const struct ggml_tensor * t) { + return (size_t) t->ne[0] * t->ne[1] * t->ne[2] * t->ne[3] * t->nb[0]; +} + +//****************************************************************************** +// Main entry point +//****************************************************************************** + +int entry_point(struct ggml_et_unary_params * params, void * env) { + kernel_environment_t * kernel_env = (kernel_environment_t *) env; + + if (!kernel_env) { + return -1; + } + + int thread_id = get_relative_thread_id(kernel_env->shire_mask); + int num_threads = get_num_threads(kernel_env->shire_mask); + + if (thread_id < 0) { + return 0; + } + + if (params == 0 || ((uint64_t) params & 0x7) != 0) { + return -1; + } + + struct ggml_tensor * src0 = ¶ms->src0; + struct ggml_tensor * dst = ¶ms->dst; + + // evict_region_past_l2(¶ms->unary_op, sizeof(int32_t)); + // WAIT_CACHEOPS; + // FENCE; + + int32_t unary_op = params->unary_op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return -1; + } + + float * src0_data = (float *) src0->data; + float * dst_data = (float *) dst->data; + + if (!src0_data || !dst_data) { + return -1; + } + + // evict_region_past_l2(src0_data, tensor_bytes(src0)); + // evict_region_past_l2(dst_data, tensor_bytes(dst)); + // WAIT_CACHEOPS; + // FENCE; + // et_barrier(ET_BARRIER_GLOBAL); + + // Tensor layout: src and dst are F32 with at least dim-0 contiguity + // - nb[0] == sizeof(float) (rows are dense; SIMD loads stay legal) + // - nb[1], nb[2], nb[3] may all be arbitrary strides for 4D views + // + // We walk rows independently and decompose row index r into (i1,i2,i3), + // computing per-row byte offsets via nb[1..3] of each tensor. + const int64_t nc = dst->ne[0]; // row width (logical) + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t nr = ne1 * ne2 * dst->ne[3]; // total rows + const int64_t total_elements = nr * nc; + const size_t s_nb1 = src0->nb[1], s_nb2 = src0->nb[2], s_nb3 = src0->nb[3]; + const size_t d_nb1 = dst->nb[1], d_nb2 = dst->nb[2], d_nb3 = dst->nb[3]; + + // evict_region_past_l2(src0_data, tensor_bytes(src0)); + // evict_region_past_l2(dst_data, tensor_bytes(dst)); + // FENCE; + // WAIT_CACHEOPS; + // et_barrier(ET_BARRIER_GLOBAL); + const int64_t elements_per_cacheline = 16; // 64 bytes / 4 bytes per float + const int64_t total_cachelines = (total_elements + elements_per_cacheline - 1) / elements_per_cacheline; + + const int64_t cl_per_thread = (total_cachelines + num_threads - 1) / num_threads; + const int64_t cl_start = thread_id * cl_per_thread; + int64_t cl_end = cl_start + cl_per_thread; + if (cl_end > total_cachelines) { + cl_end = total_cachelines; + } + + if (cl_start >= total_cachelines) { + return 0; + } + + const int64_t elem_start = cl_start * elements_per_cacheline; + int64_t elem_end = cl_end * elements_per_cacheline; + if (elem_end > total_elements) { + elem_end = total_elements; + } + + // Fast path: tensor is fully contiguous (no view), walk it as a flat array. + // This preserves perf for the common case and avoids the per-row dispatch loop. + const size_t row_bytes = (size_t) nc * sizeof(float); + // evict_region_past_l2((src0_data + elem_start), row_bytes); + // // evict_region_past_l2((dst_data + elem_start), row_bytes); + // FENCE; + // WAIT_CACHEOPS; + // et_barrier(ET_BARRIER_GLOBAL); + + const int is_flat = s_nb1 == row_bytes && s_nb2 == s_nb1 * (size_t) ne1 && s_nb3 == s_nb2 * (size_t) ne2 && + d_nb1 == row_bytes && d_nb2 == d_nb1 * (size_t) ne1 && d_nb3 == d_nb2 * (size_t) ne2; + + if (is_flat) { + float * src_ptr = src0_data + elem_start; + // evict_region_past_l2(src_ptr, 1024); + float * dst_ptr = dst_data + elem_start; + const int32_t count = (int32_t) (elem_end - elem_start); + switch (unary_op) { + case GGML_UNARY_OP_NEG: + vec_neg(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_ABS: + vec_abs(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_SGN: + vec_sgn(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_STEP: + vec_step(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_RELU: + vec_relu(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_EXP: + vec_exp(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_EXPM1: + vec_expm1(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_SIGMOID: + vec_sigmoid(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_TANH: + vec_tanh(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_SILU: + vec_silu(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_ELU: + vec_elu(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_GELU: + vec_gelu(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_GELU_QUICK: + vec_gelu_quick(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_GELU_ERF: + vec_gelu_erf(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_HARDSWISH: + vec_hardswish(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_HARDSIGMOID: + vec_hardsigmoid(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_SOFTPLUS: + vec_softplus(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_FLOOR: + vec_floor(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_CEIL: + vec_ceil(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_ROUND: + vec_round(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_TRUNC: + vec_trunc(dst_ptr, src_ptr, count); + break; + default: + return -1; + } + return 0; + } + + // Slow path: arbitrary 4D-strided view. Walk the assigned element range + // row-by-row, clipping each segment to a row boundary so we never cross + // nb[1]. For each row index r, decompose into (i1,i2,i3) and add the + // corresponding nb[*] byte offsets to the base pointers. + int64_t e = elem_start; + while (e < elem_end) { + int64_t row = e / nc; + int64_t col = e % nc; + int64_t take = nc - col; + if (take > elem_end - e) { + take = elem_end - e; + } + + // Decompose row into (i3,i2,i1) using row-major linearization + const int64_t i1 = row % ne1; + const int64_t r2 = row / ne1; + const int64_t i2 = r2 % ne2; + const int64_t i3 = r2 / ne2; + + float * src_ptr = (float *) ((char *) src0_data + i3 * s_nb3 + i2 * s_nb2 + i1 * s_nb1) + col; + float * dst_ptr = (float *) ((char *) dst_data + i3 * d_nb3 + i2 * d_nb2 + i1 * d_nb1) + col; + const int32_t count = (int32_t) take; + + // evict_region_past_l2(src_ptr, 1024); + // FENCE; + // et_barrier(ET_BARRIER_GLOBAL); + + switch (unary_op) { + case GGML_UNARY_OP_NEG: + vec_neg(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_ABS: + vec_abs(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_SGN: + vec_sgn(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_STEP: + vec_step(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_RELU: + vec_relu(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_EXP: + vec_exp(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_EXPM1: + vec_expm1(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_SIGMOID: + vec_sigmoid(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_TANH: + vec_tanh(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_SILU: + vec_silu(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_ELU: + vec_elu(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_GELU: + vec_gelu(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_GELU_QUICK: + vec_gelu_quick(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_GELU_ERF: + vec_gelu_erf(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_HARDSWISH: + vec_hardswish(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_HARDSIGMOID: + vec_hardsigmoid(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_SOFTPLUS: + vec_softplus(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_FLOOR: + vec_floor(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_CEIL: + vec_ceil(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_ROUND: + vec_round(dst_ptr, src_ptr, count); + break; + case GGML_UNARY_OP_TRUNC: + vec_trunc(dst_ptr, src_ptr, count); + break; + default: + return -1; + } + + e += take; + } + + return 0; +} diff --git a/ggml/src/ggml-et/ggml-et-common.h b/ggml/src/ggml-et/ggml-et-common.h new file mode 100644 index 0000000..a4132ee --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-common.h @@ -0,0 +1,86 @@ +#pragma once + +#include "ggml-backend-impl.h" +#include "ggml-et-uberkernel-common.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +std::shared_ptr ggml_et_runtime(); + +struct ggml_backend_et_buffer_type_context { + int devidx; + std::string name; +}; + +struct ggml_backend_et_buffer_context { + int devidx; + void * data; // Device memory pointer + size_t size; + rt::DeviceId rtid; +}; + +struct ggml_backend_et_context { + int devidx; +}; + +struct ggml_backend_et_device_context; + +// One slot in the uberkernel ring. The host vectors back the H2D copy and +// must outlive the upload; the device buffers feed the kernel that consumes +// them. pending_event lets us know when both have drained so the slot can +// be recycled. +struct ggml_backend_et_uberkernel_slot { + std::vector insts; + std::vector params_blob; + + std::byte * device_insts = nullptr; + std::byte * device_params = nullptr; + size_t device_insts_capacity = 0; + size_t device_params_capacity = 0; + + rt::EventId pending_event{}; + bool has_pending = false; +}; + +struct ggml_backend_et_uberkernel_context { + bool failed = false; + uint64_t shire_mask = 0; + + // Ring of slots. We accumulate into slots[current_slot]; on segment + // commit we fire the H2D + launch and rotate to the next slot, + // waiting on its previous launch only if it hasn't drained yet. + static constexpr size_t SLOT_COUNT = 4; + ggml_backend_et_uberkernel_slot slots[SLOT_COUNT]; + size_t current_slot = 0; +}; + +struct ggml_backend_et_device_context { + int devidx; + rt::DeviceId rtid; + std::string name; + std::string desc; + size_t total_mem; + ggml_backend_buffer_type_t buftype; + + // Kernel management - default stream for ordered execution on this device + rt::StreamId default_stream; + std::unordered_map loaded_kernels; + + // trace buffer - for printing support + std::byte * trace_buffer; + + bool uberkernel_enabled = false; + ggml_backend_et_uberkernel_context uberkernel; +}; + +struct ggml_backend_et_reg_ctx { + std::vector devices; +}; diff --git a/ggml/src/ggml-et/ggml-et-cpu-compare.cpp b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp new file mode 100644 index 0000000..5771679 --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-cpu-compare.cpp @@ -0,0 +1,502 @@ +#include "ggml-et-cpu-compare.h" + +#include "ggml-cpu/ggml-cpu-impl.h" +#include "ggml-cpu/ops.h" + +#include +#include +#include +#include + +bool ggml_et_cpu_compare_init_pre(ggml_et_cpu_compare_ctx * ctx, const ggml_tensor * node, ggml_op op) { + if (!ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for CPU compare init\n"); + return false; + } + + // Clear context + memset(ctx, 0, sizeof(*ctx)); + + // Calculate actual buffer sizes - use backend buffer size for accurate copy + auto get_tensor_buffer_size = [](const ggml_tensor * tensor) -> size_t { + if (!tensor) { + return 0; + } + + if (tensor->buffer) { + // Get actual backend buffer size + size_t buffer_size = ggml_backend_buffer_get_size(tensor->buffer); + + // Use the full buffer size to avoid any truncation issues + return buffer_size; + } else { + // Fallback to logical size if no buffer + return ggml_nbytes(tensor); + } + }; + + ctx->src0_size = get_tensor_buffer_size(node->src[0]); + ctx->src1_size = get_tensor_buffer_size(node->src[1]); + ctx->src2_size = get_tensor_buffer_size(node->src[2]); + ctx->dst_size = get_tensor_buffer_size(node); + + // Allocate CPU buffers for all tensors + if (ctx->src0_size > 0) { + ctx->cpu_src0_data = malloc(ctx->src0_size); + if (!ctx->cpu_src0_data) { + GGML_LOG_ERROR("ET: Failed to allocate CPU src0 buffer\n"); + goto cleanup; + } + } + + if (ctx->src1_size > 0) { + ctx->cpu_src1_data = malloc(ctx->src1_size); + if (!ctx->cpu_src1_data) { + GGML_LOG_ERROR("ET: Failed to allocate CPU src1 buffer\n"); + goto cleanup; + } + } + + if (ctx->src2_size > 0) { + ctx->cpu_src2_data = malloc(ctx->src2_size); + if (!ctx->cpu_src2_data) { + GGML_LOG_ERROR("ET: Failed to allocate CPU src2 buffer\n"); + goto cleanup; + } + } + + ctx->cpu_dst_data = malloc(ctx->dst_size); + if (!ctx->cpu_dst_data) { + GGML_LOG_ERROR("ET: Failed to allocate CPU dst buffer\n"); + goto cleanup; + } + + ctx->et_dst_data = malloc(ctx->dst_size); + if (!ctx->et_dst_data) { + GGML_LOG_ERROR("ET: Failed to allocate ET dst buffer\n"); + goto cleanup; + } + + // Copy data from ET device buffers to CPU host buffers + if (ctx->src0_size > 0) { + // Copy logical tensor size - ggml_backend_tensor_get handles stride layout internally + size_t logical_size = ggml_nbytes(node->src[0]); + ggml_backend_tensor_get(node->src[0], ctx->cpu_src0_data, 0, logical_size); + } + if (ctx->src1_size > 0) { + size_t logical_size = ggml_nbytes(node->src[1]); + ggml_backend_tensor_get(node->src[1], ctx->cpu_src1_data, 0, logical_size); + } + if (ctx->src2_size > 0) { + size_t logical_size = ggml_nbytes(node->src[2]); + ggml_backend_tensor_get(node->src[2], ctx->cpu_src2_data, 0, logical_size); + } + + // Copy destination data from device (for operations like SET_ROWS that modify existing data) + // Most ops create new tensors so this is unused, but SET_ROWS requires existing dst data + { + size_t logical_size = ggml_nbytes(node); + ggml_backend_tensor_get(node, ctx->cpu_dst_data, 0, logical_size); + } + + // Create CPU backend for reference computation + GGML_LOG_DEBUG("ET: Creating CPU backend for reference computation\n"); + ctx->cpu_backend = ggml_backend_cpu_init(); + if (!ctx->cpu_backend) { + GGML_LOG_ERROR("ET: Failed to create CPU backend\n"); + goto cleanup; + } + + // Create GGML context for CPU tensors + GGML_LOG_DEBUG("ET: Creating GGML context for CPU computation\n"); + ggml_init_params ctx_params; + ctx_params.mem_size = ggml_tensor_overhead() * 4 + ggml_graph_overhead(); // up to 4 tensors + graph + ctx_params.mem_buffer = nullptr; + ctx_params.no_alloc = true; // We'll manage data ourselves + ctx->ggml_ctx = ggml_init(ctx_params); + if (!ctx->ggml_ctx) { + GGML_LOG_ERROR("ET: Failed to create GGML context\n"); + goto cleanup; + } + + // Create CPU tensors with proper context + if (node->src[0]) { + ctx->cpu_src0 = ggml_new_tensor(ctx->ggml_ctx, node->src[0]->type, GGML_MAX_DIMS, node->src[0]->ne); + if (!ctx->cpu_src0) { + GGML_LOG_ERROR("ET: Failed to create CPU src0 tensor\n"); + goto cleanup; + } + ctx->cpu_src0->data = ctx->cpu_src0_data; + // Copy stride array (nb) for correct memory layout + memcpy(ctx->cpu_src0->nb, node->src[0]->nb, sizeof(node->src[0]->nb)); + // Copy op_params if present + memcpy(ctx->cpu_src0->op_params, node->src[0]->op_params, sizeof(node->src[0]->op_params)); + } + + if (node->src[1]) { + ctx->cpu_src1 = ggml_new_tensor(ctx->ggml_ctx, node->src[1]->type, GGML_MAX_DIMS, node->src[1]->ne); + if (!ctx->cpu_src1) { + GGML_LOG_ERROR("ET: Failed to create CPU src1 tensor\n"); + goto cleanup; + } + ctx->cpu_src1->data = ctx->cpu_src1_data; + // Copy stride array (nb) for correct memory layout + memcpy(ctx->cpu_src1->nb, node->src[1]->nb, sizeof(node->src[1]->nb)); + // Copy op_params if present + memcpy(ctx->cpu_src1->op_params, node->src[1]->op_params, sizeof(node->src[1]->op_params)); + } + + if (node->src[2]) { + ctx->cpu_src2 = ggml_new_tensor(ctx->ggml_ctx, node->src[2]->type, GGML_MAX_DIMS, node->src[2]->ne); + if (!ctx->cpu_src2) { + GGML_LOG_ERROR("ET: Failed to create CPU src2 tensor\n"); + goto cleanup; + } + ctx->cpu_src2->data = ctx->cpu_src2_data; + // Copy stride array (nb) for correct memory layout + memcpy(ctx->cpu_src2->nb, node->src[2]->nb, sizeof(node->src[2]->nb)); + // Copy op_params if present + memcpy(ctx->cpu_src2->op_params, node->src[2]->op_params, sizeof(node->src[2]->op_params)); + } + + return true; + +cleanup: + ggml_et_cpu_compare_free(ctx); + return false; +} + +bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ctx, + const ggml_tensor * node, + const ggml_et_cpu_compare_config * config) { + if (!ctx || !ctx->cpu_backend || !ctx->ggml_ctx || !node || !config) { + GGML_LOG_ERROR("ET: Invalid parameters for CPU compute and check\n"); + return false; + } + + // Create operation-specific CPU destination tensor based on the node's operation + ggml_op op = node->op; + switch (op) { + case GGML_OP_MUL: + ctx->cpu_dst = ggml_mul(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1); + break; + case GGML_OP_ADD: + ctx->cpu_dst = ggml_add(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1); + break; + case GGML_OP_MUL_MAT: + ctx->cpu_dst = ggml_mul_mat(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1); + break; + case GGML_OP_MUL_MAT_ID: + // MUL_MAT_ID: Mixture of Experts matrix multiplication + // src0 (as): expert weight matrices [K, M, n_expert] + // src1 (b): activations [K, n_expert_used, batch] + // src2 (ids): expert selection indices [n_expert_used, batch] + ctx->cpu_dst = ggml_mul_mat_id(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, ctx->cpu_src2); + break; + case GGML_OP_ROPE: + { + const int32_t * op_params = (const int32_t *) node->op_params; + const int32_t n_dims = op_params[1]; + const int32_t mode = op_params[2]; + const int32_t n_ctx_orig = op_params[4]; + const float freq_base = *((const float *) (op_params + 5)); + const float freq_scale = *((const float *) (op_params + 6)); + const float ext_factor = *((const float *) (op_params + 7)); + const float attn_factor = *((const float *) (op_params + 8)); + const float beta_fast = *((const float *) (op_params + 9)); + const float beta_slow = *((const float *) (op_params + 10)); + + if (mode & GGML_ROPE_TYPE_MROPE) { + int sections[GGML_MROPE_SECTIONS]; + memcpy(sections, op_params + 11, sizeof(sections)); + ctx->cpu_dst = ggml_rope_multi(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, ctx->cpu_src2, + n_dims, sections, mode, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + } else { + ctx->cpu_dst = ggml_rope_ext(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, ctx->cpu_src2, + n_dims, mode, n_ctx_orig, freq_base, freq_scale, ext_factor, + attn_factor, beta_fast, beta_slow); + } + } + break; + case GGML_OP_RMS_NORM: + // Extract epsilon parameter from op_params (stored as float) + { + float eps; + memcpy(&eps, node->op_params, sizeof(float)); + ctx->cpu_dst = ggml_rms_norm(ctx->ggml_ctx, ctx->cpu_src0, eps); + } + break; + case GGML_OP_SQR: + ctx->cpu_dst = ggml_sqr(ctx->ggml_ctx, ctx->cpu_src0); + break; + case GGML_OP_UNARY: + { + ggml_unary_op uop = (ggml_unary_op) ggml_get_op_params_i32(node, 0); + ctx->cpu_dst = ggml_unary(ctx->ggml_ctx, ctx->cpu_src0, uop); + } + break; + case GGML_OP_SUM_ROWS: + ctx->cpu_dst = ggml_sum_rows(ctx->ggml_ctx, ctx->cpu_src0); + break; + case GGML_OP_MEAN: + ctx->cpu_dst = ggml_mean(ctx->ggml_ctx, ctx->cpu_src0); + break; + case GGML_OP_CLAMP: + { + float clamp_min, clamp_max; + memcpy(&clamp_min, (const float *) node->op_params + 0, sizeof(float)); + memcpy(&clamp_max, (const float *) node->op_params + 1, sizeof(float)); + ctx->cpu_dst = ggml_clamp(ctx->ggml_ctx, ctx->cpu_src0, clamp_min, clamp_max); + } + break; + case GGML_OP_GLU: + // Extract GLU parameters from op_params (split mode only) + { + int32_t glu_op_type = ggml_get_op_params_i32(node, 0); // GLU variant + ggml_glu_op glu_op = (ggml_glu_op) glu_op_type; + + // Only support split tensor mode + if (!ctx->cpu_src1) { + GGML_LOG_ERROR("ET: GLU CPU comparison requires split tensor mode\n"); + return false; + } + if (glu_op == GGML_GLU_OP_SWIGLU_CLAMP) { + const float limit = ggml_get_op_params_f32(node, 3); + ctx->cpu_dst = ggml_swiglu_clamp(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, limit); + } else { + ctx->cpu_dst = ggml_glu_split(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, glu_op); + } + } + break; + case GGML_OP_SOFT_MAX: + { + // Extract scale and max_bias from op_params + float scale = 1.0f; + float max_bias = 0.0f; + memcpy(&scale, (const float *) node->op_params + 0, sizeof(float)); + memcpy(&max_bias, (const float *) node->op_params + 1, sizeof(float)); + + if (ctx->cpu_src1 || scale != 1.0f || max_bias != 0.0f) { + // Use extended softmax when mask or non-default parameters are present + ctx->cpu_dst = ggml_soft_max_ext(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1, scale, max_bias); + } else { + // Use simple softmax when no mask and default parameters + ctx->cpu_dst = ggml_soft_max(ctx->ggml_ctx, ctx->cpu_src0); + } + + // Add sinks if present + if (ctx->cpu_src2) { + ggml_soft_max_add_sinks(ctx->cpu_dst, ctx->cpu_src2); + } + } + break; + case GGML_OP_GET_ROWS: + ctx->cpu_dst = ggml_get_rows(ctx->ggml_ctx, ctx->cpu_src0, ctx->cpu_src1); + break; + case GGML_OP_CONT: + ctx->cpu_dst = ggml_cont(ctx->ggml_ctx, ctx->cpu_src0); + break; + case GGML_OP_SET_ROWS: + { + // SET_ROWS operation scatters src0 rows to dst[src1] positions + // Create destination tensor (this is the "view" that SET_ROWS returns) + ggml_tensor * cpu_dst_base = ggml_new_tensor(ctx->ggml_ctx, node->type, GGML_MAX_DIMS, node->ne); + if (!cpu_dst_base) { + GGML_LOG_ERROR("ET: Failed to create CPU destination base tensor for SET_ROWS\n"); + return false; + } + cpu_dst_base->data = ctx->cpu_dst_data; + memcpy(cpu_dst_base->nb, node->nb, sizeof(node->nb)); + + // Note: cpu_dst_data already contains the pre-existing destination data from device + // SET_ROWS will update specific rows, leaving others unchanged + + // Perform SET_ROWS operation: returns a view that scatters src0 rows to dst[src1] positions + ctx->cpu_dst = ggml_set_rows(ctx->ggml_ctx, cpu_dst_base, ctx->cpu_src0, ctx->cpu_src1); + } + break; + default: + GGML_LOG_ERROR("ET: Unsupported operation %s for CPU comparison\n", ggml_op_name(op)); + return false; + } + + if (!ctx->cpu_dst) { + GGML_LOG_ERROR("ET: Failed to create CPU destination tensor for operation %s\n", ggml_op_name(op)); + return false; + } + + ctx->cpu_dst->data = ctx->cpu_dst_data; + // Copy stride array (nb) for correct memory layout - except for CONT which should keep contiguous strides + if (op != GGML_OP_CONT) { + memcpy(ctx->cpu_dst->nb, node->nb, sizeof(node->nb)); + } + // For CONT operations, keep the contiguous strides created by ggml_cont() + + // Create minimal computation graph + ctx->cpu_graph = ggml_new_graph_custom(ctx->ggml_ctx, 1, false); + if (!ctx->cpu_graph) { + GGML_LOG_ERROR("ET: Failed to create CPU computation graph\n"); + return false; + } + ctx->cpu_graph->nodes[0] = ctx->cpu_dst; + ctx->cpu_graph->n_nodes = 1; + + // Log input data for debugging if enabled + if (config && config->log_differences) { + if (ctx->cpu_src0_data && ctx->src0_size >= 4) { + GGML_LOG_DEBUG("ET: CPU src0 first few bytes: %02x %02x %02x %02x\n", ((uint8_t *) ctx->cpu_src0_data)[0], + ((uint8_t *) ctx->cpu_src0_data)[1], ((uint8_t *) ctx->cpu_src0_data)[2], + ((uint8_t *) ctx->cpu_src0_data)[3]); + } + if (ctx->cpu_src1_data && ctx->src1_size >= 16) { + GGML_LOG_DEBUG("ET: CPU src1 first few floats: %.6f %.6f %.6f %.6f\n", ((float *) ctx->cpu_src1_data)[0], + ((float *) ctx->cpu_src1_data)[1], ((float *) ctx->cpu_src1_data)[2], + ((float *) ctx->cpu_src1_data)[3]); + } + } + + // Compute using CPU backend + ggml_status cpu_result = ggml_backend_graph_compute(ctx->cpu_backend, ctx->cpu_graph); + + if (cpu_result != GGML_STATUS_SUCCESS) { + GGML_LOG_ERROR("ET: CPU reference computation failed with status %d\n", cpu_result); + return false; + } + + // Log output data for debugging if enabled + if (config && config->log_differences && ctx->dst_size >= 16) { + GGML_LOG_DEBUG("ET: CPU dst first few floats after computation: %.6f %.6f %.6f %.6f\n", + ((float *) ctx->cpu_dst_data)[0], ((float *) ctx->cpu_dst_data)[1], + ((float *) ctx->cpu_dst_data)[2], ((float *) ctx->cpu_dst_data)[3]); + } + + // Now copy ET device destination to host for comparison + size_t dst_logical_size = ggml_nbytes(node); + ggml_backend_tensor_get(node, ctx->et_dst_data, 0, dst_logical_size); + + if (config->log_differences) { + size_t num_elements = ggml_nelements(node); + size_t max_log = std::min(num_elements, config->max_log_elements); + + // Check if this is an elementwise operation that can show src inputs + bool is_elementwise = (op == GGML_OP_MUL || op == GGML_OP_ADD || op == GGML_OP_GLU); + float * cpu_src0_float = is_elementwise ? (float *) ctx->cpu_src0_data : nullptr; + float * cpu_src1_float = is_elementwise ? (float *) ctx->cpu_src1_data : nullptr; + + // Helper to get float value from tensor data (handles f16 and f32) + auto get_float = [](const void * data, size_t idx, ggml_type type) -> float { + if (type == GGML_TYPE_F16) { + const ggml_fp16_t * fp16_data = (const ggml_fp16_t *) data; + return ggml_fp16_to_fp32(fp16_data[idx]); + } + + const float * float_data = (const float *) data; + return float_data[idx]; + }; + + // Compare all elements but log only the first max_log_elements + bool matches = true; + size_t total_mismatches = 0; + + // First pass: check all elements for mismatches + for (size_t i = 0; i < num_elements; i++) { + float cpu_val = get_float(ctx->cpu_dst_data, i, node->type); + float et_val = get_float(ctx->et_dst_data, i, node->type); + float diff = fabsf(cpu_val - et_val); + float rel_diff = diff / (fabsf(cpu_val) + 1e-8f); + + if (rel_diff > config->tolerance) { + matches = false; + total_mismatches++; + } + } + + // Second pass: log detailed info for first max_log elements only + for (size_t i = 0; i < max_log; i++) { + float cpu_val = get_float(ctx->cpu_dst_data, i, node->type); + float et_val = get_float(ctx->et_dst_data, i, node->type); + float diff = fabsf(cpu_val - et_val); + + if (is_elementwise && cpu_src0_float && cpu_src1_float) { + GGML_LOG_DEBUG("ET: [%zu] src0=%.6f, src1=%.6f -> CPU=%.6f, ET=%.6f, diff=%.6f\n", i, cpu_src0_float[i], + cpu_src1_float[i], cpu_val, et_val, diff); + } else if (is_elementwise && cpu_src0_float) { + GGML_LOG_DEBUG("ET: [%zu] src0=%.6f -> CPU=%.6f, ET=%.6f, diff=%.6f\n", i, cpu_src0_float[i], cpu_val, + et_val, diff); + } else { + GGML_LOG_DEBUG("ET: [%zu] CPU=%.6f, ET=%.6f, diff=%.6f\n", i, cpu_val, et_val, diff); + } + } + + // Check some elements from the middle and end for full coverage + if (num_elements > max_log) { + size_t mid = num_elements / 2; + size_t end = num_elements - 1; + float cpu_mid = get_float(ctx->cpu_dst_data, mid, node->type); + float et_mid = get_float(ctx->et_dst_data, mid, node->type); + float cpu_end = get_float(ctx->cpu_dst_data, end, node->type); + float et_end = get_float(ctx->et_dst_data, end, node->type); + + GGML_LOG_DEBUG("ET: Middle element [%zu]: CPU=%.6f, ET=%.6f\n", mid, cpu_mid, et_mid); + GGML_LOG_DEBUG("ET: Last element [%zu]: CPU=%.6f, ET=%.6f\n", end, cpu_end, et_end); + } + + GGML_LOG_DEBUG("ET: Results %s (%zu/%zu elements match within tolerance %.6f)\n", matches ? "MATCH" : "DIFFER", + num_elements - total_mismatches, num_elements, config->tolerance); + } + + // Copy CPU result to device if flag is set + if (config->use_cpu_result) { + GGML_LOG_DEBUG("ET: Overwriting ET device result with CPU result for correct inference\n"); + size_t dst_logical_size = ggml_nbytes(node); + ggml_backend_tensor_set(const_cast(node), ctx->cpu_dst_data, 0, dst_logical_size); + GGML_LOG_DEBUG("ET: CPU result copied to ET device buffer\n"); + } + + return true; +} + +void ggml_et_cpu_compare_free(ggml_et_cpu_compare_ctx * ctx) { + if (!ctx) { + return; + } + + if (ctx->cpu_src0_data) { + free(ctx->cpu_src0_data); + ctx->cpu_src0_data = nullptr; + } + if (ctx->cpu_src1_data) { + free(ctx->cpu_src1_data); + ctx->cpu_src1_data = nullptr; + } + if (ctx->cpu_src2_data) { + free(ctx->cpu_src2_data); + ctx->cpu_src2_data = nullptr; + } + if (ctx->cpu_dst_data) { + free(ctx->cpu_dst_data); + ctx->cpu_dst_data = nullptr; + } + if (ctx->et_dst_data) { + free(ctx->et_dst_data); + ctx->et_dst_data = nullptr; + } + + if (ctx->ggml_ctx) { + ggml_free(ctx->ggml_ctx); + ctx->ggml_ctx = nullptr; + } + + if (ctx->cpu_backend) { + ggml_backend_free(ctx->cpu_backend); + ctx->cpu_backend = nullptr; + } + + // Clear pointers + ctx->cpu_src0 = nullptr; + ctx->cpu_src1 = nullptr; + ctx->cpu_src2 = nullptr; + ctx->cpu_dst = nullptr; + ctx->cpu_graph = nullptr; +} diff --git a/ggml/src/ggml-et/ggml-et-cpu-compare.h b/ggml/src/ggml-et/ggml-et-cpu-compare.h new file mode 100644 index 0000000..da839fa --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-cpu-compare.h @@ -0,0 +1,54 @@ +#pragma once + +#include "ggml-cpu.h" +#include "ggml-et-common.h" +#include "ggml-impl.h" + +// Configuration for CPU comparison +struct ggml_et_cpu_compare_config { + bool enabled; // Whether to enable CPU comparison + bool use_cpu_result; // Whether to replace ET result with CPU result + bool log_differences; // Whether to log detailed element differences + float tolerance; // Relative tolerance for comparison (default: 1e-5f) + size_t max_log_elements; // Maximum number of elements to log (default: 10) +}; + +// Default configuration +static const ggml_et_cpu_compare_config ggml_et_cpu_compare_default_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 10 +}; + +// CPU comparison context for a single operation +struct ggml_et_cpu_compare_ctx { + ggml_backend_t cpu_backend; + ggml_context * ggml_ctx; + ggml_tensor * cpu_src0; + ggml_tensor * cpu_src1; + ggml_tensor * cpu_src2; + ggml_tensor * cpu_dst; + ggml_cgraph * cpu_graph; + void * cpu_src0_data; + void * cpu_src1_data; + void * cpu_src2_data; + void * cpu_dst_data; + void * et_dst_data; + size_t src0_size; + size_t src1_size; + size_t src2_size; + size_t dst_size; +}; + +// Phase 1: Initialize CPU comparison context and copy source buffers (call before ET kernel) +bool ggml_et_cpu_compare_init_pre(ggml_et_cpu_compare_ctx * ctx, const ggml_tensor * node, ggml_op op); + +// Phase 2: Execute CPU computation and compare with ET result (call after ET kernel) +bool ggml_et_cpu_compare_compute_and_check(ggml_et_cpu_compare_ctx * ctx, + const ggml_tensor * node, + const ggml_et_cpu_compare_config * config); + +// Free CPU comparison context resources +void ggml_et_cpu_compare_free(ggml_et_cpu_compare_ctx * ctx); diff --git a/ggml/src/ggml-et/ggml-et-kernels.cpp b/ggml/src/ggml-et/ggml-et-kernels.cpp new file mode 100644 index 0000000..3e11928 --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-kernels.cpp @@ -0,0 +1,508 @@ +#include "ggml-et-kernels.h" + +#include "ggml-et-kernels-embed.hpp" +#include "ggml-et-uberkernel-kernel-map.h" +#include "ggml-impl.h" + +#include +#include +#include + +#define ET_TRACE_DECODER_IMPL +#include +#include + +static constexpr size_t GGML_ET_UBERKERNEL_PARAM_ALIGN = 64; + +static size_t ggml_et_align_up(size_t value, size_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +static size_t ggml_et_next_capacity(size_t current_capacity, size_t required_capacity) { + if (current_capacity == 0) { + return required_capacity; + } + + size_t next_capacity = current_capacity; + while (next_capacity < required_capacity) { + next_capacity *= 2; + } + + return next_capacity; +} + +static ggml_backend_et_uberkernel_slot & ggml_et_uberkernel_current_slot(ggml_backend_et_uberkernel_context * uk_ctx) { + return uk_ctx->slots[uk_ctx->current_slot]; +} + +// Wait for any in-flight launch that previously used this slot to finish, +// so the host vectors and device buffers are safe to mutate / free. +static void ggml_et_uberkernel_slot_wait(ggml_backend_et_uberkernel_slot & slot, + const std::shared_ptr & runtime) { + if (!slot.has_pending || !runtime) { + return; + } + runtime->waitForEvent(slot.pending_event); + slot.has_pending = false; +} + +static void ggml_et_uberkernel_reset_segment(ggml_backend_et_uberkernel_context * uk_ctx) { + if (!uk_ctx) { + return; + } + + uk_ctx->shire_mask = 0; + auto & slot = ggml_et_uberkernel_current_slot(uk_ctx); + // Drain any prior launch on this slot before clearing its host buffers. + // begin_graph and abort_graph both come through here; in either case we + // must not yank the source memory out from under an in-flight DMA. + ggml_et_uberkernel_slot_wait(slot, ggml_et_runtime()); + slot.insts.clear(); + slot.params_blob.clear(); +} + +static bool ggml_et_uberkernel_ensure_slot_capacity(ggml_backend_et_uberkernel_slot & slot, + ggml_backend_et_device_context * dev_ctx, + size_t insts_size, + size_t params_size) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!dev_ctx || !runtime) { + return false; + } + + try { + if (slot.device_insts == nullptr || insts_size > slot.device_insts_capacity) { + const size_t new_capacity = ggml_et_next_capacity(slot.device_insts_capacity, insts_size); + if (slot.device_insts) { + runtime->freeDevice(dev_ctx->rtid, slot.device_insts); + } + slot.device_insts = runtime->mallocDevice(dev_ctx->rtid, new_capacity); + slot.device_insts_capacity = slot.device_insts ? new_capacity : 0; + } + + if (slot.device_params == nullptr || params_size > slot.device_params_capacity) { + const size_t new_capacity = ggml_et_next_capacity(slot.device_params_capacity, params_size); + if (slot.device_params) { + runtime->freeDevice(dev_ctx->rtid, slot.device_params); + } + slot.device_params = runtime->mallocDevice(dev_ctx->rtid, new_capacity); + slot.device_params_capacity = slot.device_params ? new_capacity : 0; + } + } catch (const std::exception & e) { + GGML_LOG_ERROR("ET: Failed to resize uberkernel buffers: %s\n", e.what()); + return false; + } + + return slot.device_insts != nullptr && slot.device_params != nullptr; +} + +// Get embedded kernel data by name +static std::vector ggml_et_get_embedded_kernel(const std::string & kernel_name) { + auto it = ggml_et_embedded_kernels.find(kernel_name); + if (it == ggml_et_embedded_kernels.end()) { + GGML_LOG_ERROR("ET: Unknown embedded kernel: %s\n", kernel_name.c_str()); + return {}; + } + + const unsigned char * data = it->second.first; + uint64_t size = it->second.second; + + std::vector buffer(size); + std::memcpy(buffer.data(), data, size); + + return buffer; +} + +// Read kernel from file (for development/override) +static std::vector ggml_et_read_kernel_file(const std::string & kernel_path) { + std::ifstream file(kernel_path, std::ios::binary | std::ios::ate); + if (!file) { + return {}; + } + + auto size = file.tellg(); + file.seekg(0, std::ios::beg); + + std::vector buffer(size); + file.read(reinterpret_cast(buffer.data()), size); + + return buffer; +} + +// Load kernel from file or embedded data +bool ggml_et_load_kernel(ggml_backend_et_device_context * dev_ctx, const std::string & kernel_name) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + GGML_LOG_ERROR("ET: Runtime not available for kernel loading\n"); + return false; + } + + // Check if kernel already loaded + if (dev_ctx->loaded_kernels.find(kernel_name) != dev_ctx->loaded_kernels.end()) { + GGML_LOG_DEBUG("ET: Kernel %s already loaded on device %d\n", kernel_name.c_str(), dev_ctx->devidx); + return true; + } + + std::vector kernel_data; + const char * kernels_path = getenv("GGML_ET_KERNELS_PATH"); + + // If GGML_ET_KERNELS_PATH is set, try to load from file first + if (kernels_path) { + std::string kernel_file = std::string(kernels_path) + "/" + kernel_name + ".elf"; + kernel_data = ggml_et_read_kernel_file(kernel_file); + + if (!kernel_data.empty()) { + GGML_LOG_INFO("ET: Loading kernel %s from file: %s\n", kernel_name.c_str(), kernel_file.c_str()); + } else { + GGML_LOG_INFO("ET: Kernel file not found: %s, falling back to embedded\n", kernel_file.c_str()); + } + } + + // If no file data, use embedded kernel + if (kernel_data.empty()) { + kernel_data = ggml_et_get_embedded_kernel(kernel_name); + if (kernel_data.empty()) { + GGML_LOG_ERROR("ET: Failed to get kernel data for %s\n", kernel_name.c_str()); + return false; + } + } + + try { + // Load kernel code using device's default stream + auto load_result = runtime->loadCode(dev_ctx->default_stream, kernel_data.data(), kernel_data.size()); + runtime->waitForEvent(load_result.event_); + + // Store kernel handle + dev_ctx->loaded_kernels[kernel_name] = load_result.kernel_; + return true; + + } catch (const std::exception & e) { + GGML_LOG_ERROR("ET: Failed to load kernel %s: %s\n", kernel_name.c_str(), e.what()); + return false; + } +} + +static bool ggml_et_launch_kernel_internal(ggml_backend_et_device_context * dev_ctx, + const std::string & kernel_name, + void * params, + size_t params_size, + uint64_t shire_mask, + bool enable_print, + bool sync_error_check, + rt::EventId * out_event = nullptr) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + GGML_LOG_ERROR("ET: Runtime not available for kernel launch\n"); + return false; + } + + // Lazy loading: check if kernel is loaded, load if needed + auto kernel_it = dev_ctx->loaded_kernels.find(kernel_name); + if (kernel_it == dev_ctx->loaded_kernels.end()) { + // Kernel not loaded - load it + if (!ggml_et_load_kernel(dev_ctx, kernel_name)) { + GGML_LOG_ERROR("ET: Failed to lazy-load kernel %s\n", kernel_name.c_str()); + return false; + } + + // Update iterator after successful load + kernel_it = dev_ctx->loaded_kernels.find(kernel_name); + if (kernel_it == dev_ctx->loaded_kernels.end()) { + GGML_LOG_ERROR("ET: Kernel %s not found after loading\n", kernel_name.c_str()); + return false; + } + } + + rt::KernelId kernel_id = kernel_it->second; + + try { + // Setup kernel launch options + rt::KernelLaunchOptions k_opts; + k_opts.setShireMask(shire_mask); // Default: all shires (0xFFFFFFFF) + k_opts.setBarrier(true); // Wait for completion + k_opts.setFlushL3(false); // No L3 flush needed + if (enable_print) { + k_opts.setUserTracing(reinterpret_cast(dev_ctx->trace_buffer), + static_cast(ET_TRACE_BUFFER_SIZE), + 0, // threshold + shire_mask, // shire mask + 0xFFFFFFFFFFFFFFFFULL, // threadMask - all threads + 0xFFFFFFFFU, // eventMask - all events + 0xFFFFFFFFU // filterMask - all levels + ); + } + + if (sync_error_check) { + runtime->waitForStream(dev_ctx->default_stream); + auto errors = runtime->retrieveStreamErrors(dev_ctx->default_stream); + if (!errors.empty()) { + GGML_LOG_ERROR("ET: Errors detected before kernel \"%s\" launch\n", kernel_name.c_str()); + for (const auto & error : errors) { + GGML_LOG_ERROR("ET: Error code: %d\n", (int) error.errorCode_); + } + abort(); + } + } + + rt::EventId launch_event = runtime->kernelLaunch(dev_ctx->default_stream, kernel_id, + reinterpret_cast(params), params_size, k_opts); + if (out_event) { + *out_event = launch_event; + } + + if (enable_print) { + std::vector host_trace_buf(ET_TRACE_BUFFER_SIZE); + runtime->memcpyDeviceToHost(dev_ctx->default_stream, dev_ctx->trace_buffer, host_trace_buf.data(), + ET_TRACE_BUFFER_SIZE); + runtime->waitForStream(dev_ctx->default_stream); + const auto * trace_header = reinterpret_cast(host_trace_buf.data()); + const trace_entry_header_t * entry = nullptr; + while ((entry = Trace_Decode(trace_header, entry))) { + if (entry->type != TRACE_TYPE_STRING) { + continue; + } + const auto * str_entry = reinterpret_cast(entry); + printf("[hart %d] %s", entry->hart_id, str_entry->string); + } + } + + if (sync_error_check) { + // Already triggered. No need to retrigger + if (!enable_print) { + runtime->waitForStream(dev_ctx->default_stream); + } + auto errors = runtime->retrieveStreamErrors(dev_ctx->default_stream); + if (!errors.empty()) { + GGML_LOG_ERROR("ET: Errors detected during kernel \"%s\" execution\n", kernel_name.c_str()); + for (const auto & error : errors) { + GGML_LOG_ERROR("ET: Error code: %d\n", (int) error.errorCode_); + } + abort(); + } + } + + return true; + } catch (const std::exception & e) { + GGML_LOG_ERROR("ET: Failed to launch kernel %s: %s\n", kernel_name.c_str(), e.what()); + return false; + } +} + +void ggml_et_uberkernel_begin_graph(ggml_backend_et_uberkernel_context * uk_ctx) { + if (!uk_ctx) { + return; + } + + uk_ctx->failed = false; + ggml_et_uberkernel_reset_segment(uk_ctx); +} + +static bool ggml_et_launch_uberkernel_segment(ggml_backend_et_device_context * dev_ctx, + ggml_backend_et_uberkernel_context * uk_ctx) { + if (!uk_ctx || !dev_ctx) { + return false; + } + + auto & slot = ggml_et_uberkernel_current_slot(uk_ctx); + if (slot.insts.empty()) { + return true; + } + + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + GGML_LOG_ERROR("ET: Runtime not available for uberkernel commit\n"); + uk_ctx->failed = true; + return false; + } + + const size_t insts_size = slot.insts.size() * sizeof(ggml_et_uberkernel_inst); + const size_t params_size = slot.params_blob.size(); + const uint64_t shire_mask = uk_ctx->shire_mask; + bool ok = false; + + try { + if (!ggml_et_uberkernel_ensure_slot_capacity(slot, dev_ctx, insts_size, params_size)) { + GGML_LOG_ERROR("ET: Failed to allocate uberkernel device buffers\n"); + uk_ctx->failed = true; + // Drop this segment but keep the slot drained so we don't leak + // host vectors into the next graph. + slot.insts.clear(); + slot.params_blob.clear(); + uk_ctx->shire_mask = 0; + return false; + } + + // Fire-and-forget H2D + launch on default_stream. In-stream FIFO + // ordering guarantees the kernel sees fully-uploaded buffers; the + // host source bytes (slot.insts / slot.params_blob) stay alive + // because we won't touch this slot again until pending_event fires. + runtime->memcpyHostToDevice(dev_ctx->default_stream, reinterpret_cast(slot.insts.data()), + slot.device_insts, insts_size, true); + runtime->memcpyHostToDevice(dev_ctx->default_stream, slot.params_blob.data(), slot.device_params, params_size, + true); + + ggml_et_uberkernel_params params = { + static_cast(slot.insts.size()), + static_cast(sizeof(ggml_et_uberkernel_inst)), + reinterpret_cast(slot.device_insts), + reinterpret_cast(slot.device_params), + }; + + rt::EventId launch_event{}; + ok = ggml_et_launch_kernel_internal(dev_ctx, "uberkernel", ¶ms, sizeof(params), shire_mask, false, false, + &launch_event); + if (ok) { + // The kernelLaunch above is the last thing on default_stream + // that touches this slot's device buffers. Recording its event + // lets the next reuse of this slot wait on that one event + // instead of the whole stream. + slot.pending_event = launch_event; + slot.has_pending = true; + } + } catch (const std::exception & e) { + GGML_LOG_ERROR("ET: Failed to commit uberkernel segment: %s\n", e.what()); + } + uk_ctx->failed = !ok; + + if (ok) { + uk_ctx->current_slot = (uk_ctx->current_slot + 1) % ggml_backend_et_uberkernel_context::SLOT_COUNT; + auto & next = ggml_et_uberkernel_current_slot(uk_ctx); + ggml_et_uberkernel_slot_wait(next, runtime); + next.insts.clear(); + next.params_blob.clear(); + } else { + slot.insts.clear(); + slot.params_blob.clear(); + } + uk_ctx->shire_mask = 0; + return ok; +} + +void ggml_et_uberkernel_abort_graph(ggml_backend_et_uberkernel_context * uk_ctx) { + if (!uk_ctx) { + return; + } + + uk_ctx->failed = false; + ggml_et_uberkernel_reset_segment(uk_ctx); +} + +bool ggml_et_uberkernel_failed(const ggml_backend_et_uberkernel_context * uk_ctx) { + return uk_ctx && uk_ctx->failed; +} + +static bool ggml_et_launch_uberkernel(ggml_backend_et_device_context * dev_ctx, + const std::string & kernel_name, + void * params, + size_t params_size, + uint64_t shire_mask, + bool enable_print, + bool sync_error_check) { + if (!dev_ctx) { + return false; + } + + ggml_backend_et_uberkernel_context * uk_ctx = &dev_ctx->uberkernel; + const uint16_t uberkernel_id = ggml_et_uberkernel_kernel_id_from_name(kernel_name.c_str()); + if (uberkernel_id == GGML_ET_UBERKERNEL_KERNEL_INVALID) { + if (!ggml_et_launch_uberkernel_segment(dev_ctx, uk_ctx)) { + return false; + } + return ggml_et_launch_kernel_internal(dev_ctx, kernel_name, params, params_size, shire_mask, enable_print, + sync_error_check); + } + + auto & slot = ggml_et_uberkernel_current_slot(uk_ctx); + const size_t params_offset = ggml_et_align_up(slot.params_blob.size(), GGML_ET_UBERKERNEL_PARAM_ALIGN); + if (params_offset > slot.params_blob.size()) { + slot.params_blob.resize(params_offset); + } + + const std::byte * params_bytes = reinterpret_cast(params); + slot.params_blob.insert(slot.params_blob.end(), params_bytes, params_bytes + params_size); + + ggml_et_uberkernel_inst inst = { + uberkernel_id, + 0, + static_cast(params_offset), + static_cast(params_size), + }; + slot.insts.push_back(inst); + + if (slot.insts.size() == 1) { + uk_ctx->shire_mask = shire_mask; + } + + return true; +} + +bool ggml_et_uberkernel_end_graph(ggml_backend_et_device_context * dev_ctx) { + if (!dev_ctx || !dev_ctx->uberkernel_enabled) { + return true; + } + + return ggml_et_launch_uberkernel_segment(dev_ctx, &dev_ctx->uberkernel); +} + +bool ggml_et_launch_kernel(ggml_backend_et_device_context * dev_ctx, + const std::string & kernel_name, + void * params, + size_t params_size, + uint64_t shire_mask, + bool enable_print, + bool sync_error_check) { + if (!dev_ctx) { + return false; + } + + if (!dev_ctx->uberkernel_enabled) { + return ggml_et_launch_kernel_internal(dev_ctx, kernel_name, params, params_size, shire_mask, enable_print, + sync_error_check); + } + + return ggml_et_launch_uberkernel(dev_ctx, kernel_name, params, params_size, shire_mask, enable_print, + sync_error_check); +} + +void ggml_et_unload_kernel(ggml_backend_et_device_context * dev_ctx, const std::string & kernel_name) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + return; + } + + auto kernel_it = dev_ctx->loaded_kernels.find(kernel_name); + if (kernel_it != dev_ctx->loaded_kernels.end()) { + try { + runtime->unloadCode(kernel_it->second); + dev_ctx->loaded_kernels.erase(kernel_it); + } catch (const std::exception & e) { + GGML_LOG_ERROR("ET: Failed to unload kernel %s: %s\n", kernel_name.c_str(), e.what()); + } + } +} + +void ggml_et_unload_all_kernels(ggml_backend_et_device_context * dev_ctx) { + if (!dev_ctx) { + return; + } + + // Make a copy of kernel names since ggml_et_unload_kernel modifies the map + std::vector kernel_names; + kernel_names.reserve(dev_ctx->loaded_kernels.size()); + for (const auto & kernel_pair : dev_ctx->loaded_kernels) { + kernel_names.push_back(kernel_pair.first); + } + + for (const auto & kernel_name : kernel_names) { + ggml_et_unload_kernel(dev_ctx, kernel_name); + } +} + +std::vector> ggml_et_get_loaded_kernels(ggml_backend_et_device_context * dev_ctx) { + std::vector> loaded_kernels; + loaded_kernels.reserve(dev_ctx->loaded_kernels.size()); + for (const auto & kernel_pair : dev_ctx->loaded_kernels) { + loaded_kernels.push_back(kernel_pair); + } + return loaded_kernels; +} diff --git a/ggml/src/ggml-et/ggml-et-kernels.h b/ggml/src/ggml-et/ggml-et-kernels.h new file mode 100644 index 0000000..76819f5 --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-kernels.h @@ -0,0 +1,48 @@ +#pragma once + +#include "ggml-et-common.h" + +#include +#include +#include + +#define ET_TRACE_BUFFER_SIZE (1024 * 1024 * 8UL) + +// Load kernel from file or embedded data and store handle in device context +// Returns true on success, false on failure +// +// Loading strategy: +// - If GGML_ET_KERNELS_PATH env var is set: tries to load from ${GGML_ET_KERNELS_PATH}/${kernel_name}.elf +// - If file not found or env var not set: falls back to embedded kernel data +// - Returns false if kernel cannot be loaded from either source +// +// Kernel is loaded using the device's default stream +bool ggml_et_load_kernel(ggml_backend_et_device_context * dev_ctx, const std::string & kernel_name); + +// Launch kernel with parameters on device's default stream +// Performs lazy loading: automatically loads kernel if not already loaded +// Kernel path: ${GGML_ET_KERNELS_PATH}/${kernel_name}.elf (default: /opt/et/ggml/kernels/) +// Returns true on success, false on failure +// Execution is synchronous - waits for completion +bool ggml_et_launch_kernel(ggml_backend_et_device_context * dev_ctx, + const std::string & kernel_name, + void * params, + size_t params_size, + uint64_t shire_mask = 0xFFFFFFFF, + bool enable_print = false, + bool sync_error_check = false); + +void ggml_et_uberkernel_begin_graph(ggml_backend_et_uberkernel_context * uk_ctx); +bool ggml_et_uberkernel_end_graph(ggml_backend_et_device_context * dev_ctx); +void ggml_et_uberkernel_abort_graph(ggml_backend_et_uberkernel_context * uk_ctx); +bool ggml_et_uberkernel_failed(const ggml_backend_et_uberkernel_context * uk_ctx); + +// Unload kernel from device and free resources +// Safe to call even if kernel not loaded +void ggml_et_unload_kernel(ggml_backend_et_device_context * dev_ctx, const std::string & kernel_name); + +// Unload all kernels from device context +// Called during device cleanup +void ggml_et_unload_all_kernels(ggml_backend_et_device_context * dev_ctx); + +std::vector> ggml_et_get_loaded_kernels(ggml_backend_et_device_context * dev_ctx); diff --git a/ggml/src/ggml-et/ggml-et-memops.cpp b/ggml/src/ggml-et/ggml-et-memops.cpp new file mode 100644 index 0000000..13242ab --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-memops.cpp @@ -0,0 +1,36 @@ +#include "ggml-et-memops.h" + +#include "ggml-et-kernels.h" +#include "ggml-impl.h" + +// Kernel parameter structure for memset operation +struct memset_params { + uint32_t op_type; // GGML_ET_MEMOP_MEMSET + uint32_t value; // Value to set (extended to uint32_t for alignment) + void * dst_ptr; // Destination device pointer + size_t size; // Number of bytes to set +}; + +bool ggml_et_memset(ggml_backend_et_device_context * dev_ctx, void * dst_ptr, uint8_t value, size_t size) { + if (!dev_ctx || !dst_ptr || size == 0) { + GGML_LOG_ERROR("ET: Invalid memset parameters\n"); + return false; + } + + // Prepare kernel parameters + memset_params params; + params.op_type = GGML_ET_MEMOP_MEMSET; + params.value = value; + params.dst_ptr = dst_ptr; + params.size = size; + + // Launch memops kernel (will lazy-load if not already loaded) + bool success = ggml_et_launch_kernel(dev_ctx, "memops", ¶ms, sizeof(params)); + + if (!success) { + GGML_LOG_ERROR("ET: memset kernel launch failed\n"); + return false; + } + + return true; +} diff --git a/ggml/src/ggml-et/ggml-et-memops.h b/ggml/src/ggml-et/ggml-et-memops.h new file mode 100644 index 0000000..37a4fd9 --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-memops.h @@ -0,0 +1,18 @@ +#pragma once + +#include "ggml-et-common.h" + +#include +#include + +// Memory operations using device kernel (memops.elf) +// Single kernel handles multiple operations via operation identifier + +// Operation identifiers for memops kernel +enum ggml_et_memop_type : uint32_t { + GGML_ET_MEMOP_MEMSET = 0, +}; + +// Memset operation: fill device memory with a value +// Returns true on success, false on failure +bool ggml_et_memset(ggml_backend_et_device_context * dev_ctx, void * dst_ptr, uint8_t value, size_t size); diff --git a/ggml/src/ggml-et/ggml-et-ops.cpp b/ggml/src/ggml-et/ggml-et-ops.cpp new file mode 100644 index 0000000..8765138 --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-ops.cpp @@ -0,0 +1,2584 @@ +#include "ggml-et-ops.h" + +#include "ggml-et-cpu-compare.h" +#include "ggml-et-kernels.h" +#include "ggml-impl.h" + +#include + +#include + +// CPU comparison configuration - can be enabled for debugging +static ggml_et_cpu_compare_config rope_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, // Replace ET result with CPU result + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config rms_norm_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config norm_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config l2_norm_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config group_norm_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config im2col_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config unary_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-4f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config sum_rows_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config clamp_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config mean_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config sqr_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config elmap_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config glu_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config mul_mat_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 0.01, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config mul_mat_id_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 0.01, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config softmax_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-5f, + /* .max_log_elements = */ 1024 +}; + +static ggml_et_cpu_compare_config get_rows_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 2048 +}; + +static ggml_et_cpu_compare_config pad_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config cont_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config concat_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config cumsum_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config repeat_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config ssm_conv_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config rwkv_wkv6_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-4f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config rwkv_wkv7_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-4f, + /* .max_log_elements = */ 4096 +}; + +static ggml_et_cpu_compare_config set_rows_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-6f, + /* .max_log_elements = */ 2048 +}; + +bool ggml_et_op_rms_norm_mul(ggml_backend_et_device_context * dev_ctx, + const ggml_tensor * rms_norm_node, + const ggml_tensor * mul_node) { + ET_PERF_START(); + + if (!dev_ctx || !rms_norm_node || !mul_node) { + GGML_LOG_ERROR("ET: Invalid parameters for fused RMS_NORM_MUL operation\n"); + return false; + } + + if (!rms_norm_node->src[0]) { + GGML_LOG_ERROR("ET: Fused RMS_NORM_MUL missing required input\n"); + return false; + } + + // Extract weights: the MUL operand that isn't the rms_norm output + const ggml_tensor * weights = (mul_node->src[0] == rms_norm_node) ? mul_node->src[1] : mul_node->src[0]; + + if (!weights) { + GGML_LOG_ERROR("ET: Fused RMS_NORM_MUL missing weights tensor\n"); + return false; + } + + float eps; + memcpy(&eps, rms_norm_node->op_params, sizeof(float)); + + ggml_et_rms_norm_mul_params params; + params.src0 = *rms_norm_node->src[0]; // input to normalize + params.src1 = *weights; // normalization weights + params.dst = *mul_node; // final output + params.eps = eps; + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "rms_norm_mul_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + ET_PERF_END_EXT("RMS_NORM_MUL", "rms_norm_mul_f32", mul_node, "eps=%.6f", (double) eps); + return kernel_result; +} + +bool ggml_et_op_scale(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for SCALE operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: SCALE operation missing required input\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: SCALE operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + float scale, bias; + memcpy(&scale, (const float *) node->op_params + 0, sizeof(float)); + memcpy(&bias, (const float *) node->op_params + 1, sizeof(float)); + + ggml_et_scale_params params; + params.src0 = *node->src[0]; + params.dst = *node; + params.scale = scale; + params.bias = bias; + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "scale_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + ET_PERF_END_EXT("SCALE", "scale_f32", node, "scale=%.6f|bias=%.6f", (double) scale, (double) bias); + return kernel_result; +} + +bool ggml_et_op_sqr(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for SQR operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: SQR operation missing required input\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: SQR operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + ggml_et_sqr_params params; + params.src0 = *node->src[0]; // F32 input tensor + params.dst = *node; // F32 output tensor + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (sqr_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SQR)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SQR operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "sqr_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &sqr_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for SQR operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("SQR", "sqr_f32", node); + return kernel_result; +} + +bool ggml_et_op_sum_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for SUM_ROWS operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: SUM_ROWS operation missing required input\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: SUM_ROWS operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + ggml_et_sum_rows_params params; + params.src0 = *node->src[0]; + params.dst = *node; + + // Phase 1: Initialize CPU comparison context + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (sum_rows_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SUM_ROWS)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SUM_ROWS operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "sum_rows_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &sum_rows_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for SUM_ROWS operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("SUM_ROWS", "sum_rows_f32", node); + return kernel_result; +} + +bool ggml_et_op_mean(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for MEAN operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: MEAN operation missing required input\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: MEAN operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + ggml_et_mean_params params; + params.src0 = *node->src[0]; + params.dst = *node; + + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (mean_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_MEAN)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for MEAN operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "mean_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &mean_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for MEAN operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("MEAN", "mean_f32", node); + return kernel_result; +} + +bool ggml_et_op_clamp(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for CLAMP operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: CLAMP operation missing required input\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: CLAMP operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + ggml_et_clamp_params params; + params.src0 = *node->src[0]; + params.dst = *node; + // op_params layout per ggml.c::ggml_clamp: { min, max } as floats + memcpy(¶ms.min_val, (const float *) node->op_params + 0, sizeof(float)); + memcpy(¶ms.max_val, (const float *) node->op_params + 1, sizeof(float)); + + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (clamp_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CLAMP)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CLAMP operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "clamp_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &clamp_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for CLAMP operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("CLAMP", "clamp_f32", node); + return kernel_result; +} + +bool ggml_et_op_unary(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for UNARY operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: UNARY operation missing required input\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: UNARY operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + const ggml_unary_op uop = ggml_get_unary_op(node); + const char * op_name = ggml_unary_op_name(uop); + + ggml_et_unary_params params; + params.src0 = *node->src[0]; // F32 input tensor + params.dst = *node; // F32 output tensor + params.unary_op = (int32_t) uop; + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (unary_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_UNARY)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for UNARY/%s operation\n", op_name); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "unary_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &unary_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for UNARY/%s operation\n", op_name); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("UNARY", "unary_f32", node, "op=%s", op_name); + return kernel_result; +} + +bool ggml_et_op_mul(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + // Delegate to generic element map operation + return ggml_et_op_elmap(dev_ctx, node); +} + +bool ggml_et_op_add(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + // Delegate to generic element map operation + return ggml_et_op_elmap(dev_ctx, node); +} + +bool ggml_et_op_sub(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + // Delegate to generic element map operation + return ggml_et_op_elmap(dev_ctx, node); +} + +bool ggml_et_op_elmap(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for element map operation\n"); + return false; + } + + if (!node->src[0] || !node->src[1]) { + GGML_LOG_ERROR("ET: Element map operation missing required inputs\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: Element map operation with unsupported types: dst=%s src0=%s src1=%s\n", + ggml_type_name(node->type), ggml_type_name(node->src[0]->type), + ggml_type_name(node->src[1]->type)); + return false; + } + + const char * op_name = ggml_op_name(node->op); + + ggml_et_elmap_params params; + params.src0 = *node->src[0]; + params.src1 = *node->src[1]; + params.dst = *node; // F32 output tensor (op type stored in dst.op) + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (elmap_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, node->op)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for %s operation\n", op_name); + } + } + + // fprintf(stderr, "ET: el_map s0 [%ld, %ld, %ld, %ld] s1 [%ld, %ld, %ld, %ld]\n", + // node->src[0]->ne[0], node->src[0]->ne[1], node->src[0]->ne[2], node->src[0]->ne[3], + // node->src[1]->ne[0], node->src[1]->ne[1], node->src[1]->ne[2], node->src[1]->ne[3]); + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "el_map_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &elmap_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for %s operation\n", op_name); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END(op_name, "el_map_f32", node); + return kernel_result; +} + +bool ggml_et_op_glu(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + // Validate inputs + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for GLU operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: GLU operation missing required input\n"); + return false; + } + + const bool is_split_mode = node->src[1] != nullptr; + + // Only support F32 (as validated by supports_op) + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || + (is_split_mode && node->src[1]->type != GGML_TYPE_F32)) { + return false; + } + + // Extract GLU operation parameters from op_params + int32_t glu_op_type = ggml_get_op_params_i32(node, 0); // GLU variant (REGLU, GEGLU, SWIGLU, etc.) + int32_t swapped = ggml_get_op_params_i32(node, 1); // Whether gate/value are swapped + + // Supported variants + switch (glu_op_type) { + case GGML_GLU_OP_REGLU: + case GGML_GLU_OP_GEGLU: + case GGML_GLU_OP_SWIGLU: + case GGML_GLU_OP_SWIGLU_OAI: + case GGML_GLU_OP_SWIGLU_CLAMP: + case GGML_GLU_OP_GEGLU_ERF: + case GGML_GLU_OP_GEGLU_QUICK: + break; + default: + GGML_LOG_ERROR("ET: GLU operation with unsupported variant: %s\n", + ggml_glu_op_name((ggml_glu_op) glu_op_type)); + return false; + } + + // Get GLU operation name for logging + const char * glu_op_name = ggml_glu_op_name((ggml_glu_op) glu_op_type); + + // Pack parameters. Single-tensor mode is encoded by zeroing src1. + ggml_et_glu_params params = {}; + params.src0 = *node->src[0]; + if (is_split_mode) { + params.src1 = *node->src[1]; + } + params.dst = *node; + params.glu_op_type = glu_op_type; + params.swapped = swapped; + params.alpha = 0.0f; + params.limit = 0.0f; + if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI) { + params.alpha = ggml_get_op_params_f32(node, 2); + } + if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI || glu_op_type == GGML_GLU_OP_SWIGLU_CLAMP) { + params.limit = ggml_get_op_params_f32(node, 3); + } + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (glu_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GLU)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for %s operation\n", glu_op_name); + } + } + + // Launch ET kernel + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "glu_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &glu_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for %s operation\n", glu_op_name); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("GLU", "glu_f32", node); + return kernel_result; +} + +bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx, + const ggml_tensor * node, + const ggml_tensor * add_node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for MUL_MAT operation\n"); + return false; + } + + if (!node->src[0] || !node->src[1]) { + GGML_LOG_ERROR("ET: MUL_MAT operation missing required inputs\n"); + return false; + } + + // Fused MM+ADD: when add_node is non-NULL the caller has already validated + // (Q8_0 weights, F32 acts, exact-shape ADD with stride parity to dst) via + // ggml_et_can_fuse({MUL_MAT, ADD}). The kernel writes dst = mm + bias and + // the ADD's output replaces MM's as the actual dst. + const ggml_tensor * fused_dst = add_node ? add_node : node; + const ggml_tensor * bias_tensor = nullptr; + if (add_node) { + bias_tensor = (add_node->src[0] == node) ? add_node->src[1] : add_node->src[0]; + } + + const char * kernel_name; + const char * src0_type_name; + + if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_0 && node->src[1]->type == GGML_TYPE_F32 && + node->src[1]->ne[1] >= 53 && // N >= 53 + node->src[0]->ne[1] % 16 == 0 && // M % TILE_M + node->src[0]->ne[0] % 32 == 0) { // K % BLOCK_K (Q4_0 block) + + // Matrix engine for N >= 53; partial N (via n_cur-1) and errata padding are handled in-kernel. + kernel_name = "mul_mat_Q4_0_matrix_engine"; + src0_type_name = "Q4_0"; + + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_0 && + node->src[1]->type == GGML_TYPE_F32) { + kernel_name = "mul_mat_Q4_0"; // N < 53, or M % 16 != 0 or K % 32 != 0 + src0_type_name = "Q4_0"; + + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q8_0 && + node->src[1]->type == GGML_TYPE_F32) { + kernel_name = "mul_mat_Q8_0"; + src0_type_name = "Q8_0"; + + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 && + node->src[1]->type == GGML_TYPE_F16 && node->ne[0] % 16 == 0 && node->src[0]->ne[0] % 16 == 0 && + node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[0] != 1) { + kernel_name = "mul_mat_f16_matrix_engine"; + src0_type_name = "F16"; + + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 && + (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) { + kernel_name = "mul_mat_f16"; + src0_type_name = "F16"; + + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && + node->src[1]->type == GGML_TYPE_F32 && node->ne[0] % 16 == 0 && node->src[0]->ne[0] % 16 == 0 && + node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[0] != 1) { // GEMV is faster with the generic path + + kernel_name = "mul_mat_f32_matrix_engine"; + src0_type_name = "F32"; + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && + (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) { + kernel_name = "mul_mat_f32"; + src0_type_name = "F32"; + } else { + GGML_LOG_ERROR("ET: MUL_MAT operation with unsupported types: dst=%s src0=%s src1=%s\n", + ggml_type_name(node->type), ggml_type_name(node->src[0]->type), + ggml_type_name(node->src[1]->type)); + return false; + } + + ggml_et_binary_params params; + params.src0 = *node->src[0]; // weight matrix + params.src1 = *node->src[1]; // activation matrix + params.dst = *fused_dst; // output (= add_node when fused, else node) + + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (mul_mat_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, fused_dst, GGML_OP_MUL_MAT)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for MUL_MAT operation\n"); + } + } + + bool kernel_result; + if (node->src[0]->type == GGML_TYPE_Q8_0) { + // Q8_0 kernel always takes the extended struct. bias.data is non-NULL + // only on the fused path; otherwise the kernel skips the add entirely. + ggml_et_mm_q8_params q8_params = {}; + q8_params.src0 = params.src0; + q8_params.src1 = params.src1; + q8_params.dst = params.dst; + if (bias_tensor) { + q8_params.bias = *bias_tensor; + } + kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &q8_params, sizeof(q8_params), 0xFFFFFFFF); + } else { + // Non-Q8 MM kernels don't yet support fused-add; the graph fuse check + // already rejects non-Q8 pairs, so add_node is always nullptr here. + kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + } + + // printf("Tensor error:"); + // if (params.src0.data != NULL) + // { + // printf("Ptr OK\n"); + // printf("node->data ptr = %p\n", node->data); + // // if (once < 100){ + // // // uint64_t * host_data = (uint64_t *) node->data; + // // // printf("Tensor error: %lu\n", host_data[0]); + + // // // printf("Tensor error:"); + // // once++; + // // } + // } + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, fused_dst, &mul_mat_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for MUL_MAT operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + { + // Calculate actual FLOPs including batch/sequence dimensions + // dst shape: [M, N, ne2, ne3] where M=ne[1], N=ne[0] + int64_t m = node->ne[1]; + int64_t n = node->ne[0]; + int64_t k = node->src[0]->ne[0]; + int64_t ne2 = node->ne[2]; + int64_t ne3 = node->ne[3]; + + // Total FLOPs = (batch_size) * M * N * (2*K - 1) + // Each MxN matrix-matrix multiply does M*N*(2*K-1) FLOPs + // Broadcasting is handled by repeating computation, so count actual operations + int64_t batch_size = ne2 * ne3; + int64_t total_flops = batch_size * m * n * (2 * k - 1); + + char kernel_variant[64]; + snprintf(kernel_variant, sizeof(kernel_variant), "%s_%sx%s", kernel_name, src0_type_name, + ggml_type_name(node->src[1]->type)); + ET_PERF_END_EXT("MUL_MAT", kernel_variant, node, "flops=%" PRId64, total_flops); + } + return kernel_result; +} + +bool ggml_et_op_mul_mat_id(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for MUL_MAT_ID operation\n"); + return false; + } + + if (!node->src[0] || !node->src[1] || !node->src[2]) { + GGML_LOG_ERROR("ET: MUL_MAT_ID operation missing required inputs\n"); + return false; + } + + const char * kernel_name; + const char * src0_type_name; + + // Support Q8_0/Q4_0/F16/F32 x F32 -> F32 matrix multiplication with expert selection + if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q8_0 && node->src[1]->type == GGML_TYPE_F32 && + node->src[2]->type == GGML_TYPE_I32) { + kernel_name = "mul_mat_id_Q8_0"; + src0_type_name = "Q8_0"; + + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_0 && + node->src[1]->type == GGML_TYPE_F32 && node->src[2]->type == GGML_TYPE_I32) { + kernel_name = "mul_mat_id_Q4_0"; + src0_type_name = "Q4_0"; + + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 && + node->src[1]->type == GGML_TYPE_F32 && node->src[2]->type == GGML_TYPE_I32) { + kernel_name = "mul_mat_id_f32"; + src0_type_name = "F16"; + + } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && + node->src[1]->type == GGML_TYPE_F32 && node->src[2]->type == GGML_TYPE_I32) { + kernel_name = "mul_mat_id_f32"; + src0_type_name = "F32"; + + } else { + GGML_LOG_ERROR("ET: MUL_MAT_ID operation with unsupported types: dst=%s src0=%s src1=%s src2=%s\n", + ggml_type_name(node->type), ggml_type_name(node->src[0]->type), + ggml_type_name(node->src[1]->type), ggml_type_name(node->src[2]->type)); + return false; + } + + // Pack parameters - copy full tensor structures + ggml_et_mul_mat_id_params params; + params.src0 = *node->src[0]; // Expert weight matrices (Q8_0/F16/F32) + params.src1 = *node->src[1]; // Activation matrix (F32) + params.src2 = *node->src[2]; // Expert indices (I32) + params.dst = *node; // Output matrix (F32) + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (mul_mat_id_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_MUL_MAT_ID)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for MUL_MAT_ID operation\n"); + } + } + + // Launch ET kernel + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &mul_mat_id_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for MUL_MAT_ID operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + // Calculate FLOPs (approximate - similar to MUL_MAT but with expert routing overhead) + // Each expert computation is similar to a MUL_MAT, but we only compute for selected experts + int64_t K = node->src[0]->ne[0]; + int64_t M = node->src[0]->ne[1]; + int64_t n_expert_used = node->src[2]->ne[0]; + int64_t batch = node->src[2]->ne[1]; + + int64_t total_flops = batch * n_expert_used * M * (2 * K - 1); + + char kernel_variant[64]; + snprintf(kernel_variant, sizeof(kernel_variant), "%s_%sx%s", kernel_name, src0_type_name, + ggml_type_name(node->src[1]->type)); + ET_PERF_END_EXT("MUL_MAT_ID", kernel_variant, node, "flops=%" PRId64 "|n_expert=%lld|n_expert_used=%lld", + total_flops, (long long) node->src[0]->ne[2], (long long) n_expert_used); + + return kernel_result; +} + +bool ggml_et_op_rope(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for ROPE operation\n"); + return false; + } + + if (!node->src[0] || !node->src[1]) { + GGML_LOG_ERROR("ET: ROPE operation missing required inputs\n"); + return false; + } + + const char * kernel_name; + + if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I32) { + kernel_name = "rope_f32"; + } else { + return false; + } + + // Pack parameters - copy full tensor structures and op_params + ggml_et_rope_params params; + params.src0 = *node->src[0]; // F32 input tensor + params.src1 = *node->src[1]; // I32 position tensor + if (node->src[2]) { + params.src2 = *node->src[2]; // F32 frequency factors (optional) + } else { + memset(¶ms.src2, 0, sizeof(params.src2)); // Zero if not provided + } + params.dst = *node; // F32 output tensor + + params.rope_params.n_past = ((const int32_t *) node->op_params)[0]; + params.rope_params.n_dims = ((const int32_t *) node->op_params)[1]; + params.rope_params.mode = ((const int32_t *) node->op_params)[2]; + params.rope_params.n_ctx = ((const int32_t *) node->op_params)[3]; + params.rope_params.n_ctx_orig = ((const int32_t *) node->op_params)[4]; + memcpy(¶ms.rope_params.freq_base, (const int32_t *) node->op_params + 5, sizeof(float)); + memcpy(¶ms.rope_params.freq_scale, (const int32_t *) node->op_params + 6, sizeof(float)); + memcpy(¶ms.rope_params.ext_factor, (const int32_t *) node->op_params + 7, sizeof(float)); + memcpy(¶ms.rope_params.attn_factor, (const int32_t *) node->op_params + 8, sizeof(float)); + memcpy(¶ms.rope_params.beta_fast, (const int32_t *) node->op_params + 9, sizeof(float)); + memcpy(¶ms.rope_params.beta_slow, (const int32_t *) node->op_params + 10, sizeof(float)); + if (params.rope_params.mode & GGML_ROPE_TYPE_MROPE) { + memcpy(params.rope_params.sections, (const int32_t *) node->op_params + 11, sizeof(int32_t) * 4); + } else { + memset(params.rope_params.sections, 0, sizeof(params.rope_params.sections)); + } + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (rope_cpu_compare_config.enabled) { + GGML_LOG_DEBUG("ET: Initializing CPU comparison for ROPE operation\n"); + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_ROPE)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for ROPE operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rope_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for ROPE operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("ROPE", kernel_name, node, "mode=0x%x|n_dims=%d|freq_base=%.2f|freq_scale=%.2f", + params.rope_params.mode, params.rope_params.n_dims, (double) params.rope_params.freq_base, + (double) params.rope_params.freq_scale); + return kernel_result; +} + +bool ggml_et_op_rms_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for RMS_NORM operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: RMS_NORM operation missing required input\n"); + return false; + } + + const char * kernel_name; + + if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) { + kernel_name = "rms_norm_f32"; + + } else { + GGML_LOG_ERROR("ET: RMS_NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + float eps; + memcpy(&eps, node->op_params, sizeof(float)); + + ggml_et_rms_norm_params params; + params.src0 = *node->src[0]; // F32 input tensor + params.dst = *node; // F32 output tensor + params.eps = eps; // Epsilon parameter for numerical stability + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (rms_norm_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_RMS_NORM)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for RMS_NORM operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rms_norm_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for RMS_NORM operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("RMS_NORM", kernel_name, node, "eps=%.6f", (double) eps); + return kernel_result; +} + +bool ggml_et_op_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for NORM operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: NORM operation missing required input\n"); + return false; + } + + const char * kernel_name; + + if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) { + kernel_name = "norm_f32"; + + } else { + GGML_LOG_ERROR("ET: NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + float eps; + memcpy(&eps, node->op_params, sizeof(float)); + + ggml_et_norm_params params; + params.src0 = *node->src[0]; // F32 input tensor + params.dst = *node; // F32 output tensor + params.eps = eps; // Epsilon parameter for numerical stability + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (norm_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_NORM)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for NORM operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &norm_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for NORM operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("NORM", kernel_name, node, "eps=%.6f", (double) eps); + return kernel_result; +} + +bool ggml_et_op_l2_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for L2_NORM operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: L2_NORM operation missing required input\n"); + return false; + } + + const char * kernel_name; + + if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) { + kernel_name = "l2_norm_f32"; + + } else { + GGML_LOG_ERROR("ET: L2_NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + float eps; + memcpy(&eps, node->op_params, sizeof(float)); + + ggml_et_l2_norm_params params; + params.src0 = *node->src[0]; // F32 input tensor + params.dst = *node; // F32 output tensor + params.eps = eps; // Epsilon parameter for numerical stability + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (l2_norm_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_L2_NORM)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for L2_NORM operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &l2_norm_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for L2_NORM operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("L2_NORM", kernel_name, node, "eps=%.6f", (double) eps); + return kernel_result; +} + +bool ggml_et_op_group_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for GROUP_NORM operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: GROUP_NORM operation missing required input\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: GROUP_NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + const int32_t n_groups = ggml_get_op_params_i32(node, 0); + float eps; + memcpy(&eps, (const float *) node->op_params + 1, sizeof(float)); + + ggml_et_group_norm_params params; + params.src0 = *node->src[0]; + params.dst = *node; + params.n_groups = n_groups; + params.eps = eps; + + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (group_norm_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GROUP_NORM)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for GROUP_NORM operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "group_norm_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &group_norm_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for GROUP_NORM operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("GROUP_NORM", "group_norm_f32", node, "eps=%.6f|n_groups=%d", (double) eps, n_groups); + return kernel_result; +} + +bool ggml_et_op_im2col(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for IM2COL operation\n"); + return false; + } + + if (!node->src[0] || !node->src[1]) { + GGML_LOG_ERROR("ET: IM2COL operation missing required inputs\n"); + return false; + } + + const bool supported_types = + (node->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32) || + (node->type == GGML_TYPE_F16 && (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)); + + if (!supported_types) { + GGML_LOG_ERROR("ET: IM2COL operation with unsupported types: dst=%s src1=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[1]->type)); + return false; + } + + ggml_et_im2col_params params; + params.src0 = *node->src[0]; + params.src1 = *node->src[1]; + params.dst = *node; + + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (im2col_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_IM2COL)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for IM2COL operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "im2col", ¶ms, sizeof(params), 0xFFFFFFFF); + + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &im2col_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for IM2COL operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("IM2COL", "im2col", node); + return kernel_result; +} + +bool ggml_et_op_conv_2d(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + return false; + } + if (!node->src[0] || !node->src[1]) { + return false; + } + if (!node->data || !node->src[0]->data || !node->src[1]->data) { + return false; + } + + // Kernel constraints (mirror supports_op; recheck here as a guard). + const ggml_tensor * flt = node->src[0]; // [Kw, Kh, Cin, Cout] + const ggml_tensor * in = node->src[1]; // [W, H, Cin, N] + if (node->type != GGML_TYPE_F32 || flt->type != GGML_TYPE_F32 || in->type != GGML_TYPE_F32) { + return false; + } + + const int32_t s0 = ggml_get_op_params_i32(node, 0); + const int32_t s1 = ggml_get_op_params_i32(node, 1); + const int32_t p0 = ggml_get_op_params_i32(node, 2); + const int32_t p1 = ggml_get_op_params_i32(node, 3); + const int32_t d0 = ggml_get_op_params_i32(node, 4); + const int32_t d1 = ggml_get_op_params_i32(node, 5); + + if (s0 < 1 || s1 < 1) { + return false; + } + if (d0 != 1 || d1 != 1) { + return false; + } + if (flt->ne[2] % 16 != 0 || flt->ne[3] % 16 != 0) { + return false; + } + if (in->ne[3] != 1) { + return false; + } + if (node->ne[0] <= 0) { + return false; // OW > 0 (any width OK; staging path handles non-16) + } + (void) p0; + (void) p1; + + ggml_et_binary_params params; + params.src0 = *node->src[0]; + params.src1 = *node->src[1]; + params.dst = *node; + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "conv_2d_f32_me", ¶ms, sizeof(params), 0xFFFFFFFFu); + + ET_PERF_END("CONV_2D", "conv_2d_f32_me", node); + return kernel_result; +} + +bool ggml_et_op_softmax(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for SOFTMAX operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: SOFTMAX operation missing required input\n"); + return false; + } + + const char * kernel_name; + + if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) { + kernel_name = "softmax_f32"; + + } else { + GGML_LOG_ERROR("ET: SOFTMAX operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + // Validate contiguity requirements + if (!ggml_is_contiguous(node)) { + GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous destination tensor\n"); + return false; + } + + if (!ggml_is_contiguous(node->src[0])) { + GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous source tensor\n"); + return false; + } + + // Check optional mask tensor + if (node->src[1]) { + if (node->src[1]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: SOFTMAX operation with unsupported mask type: %s (F32 required)\n", + ggml_type_name(node->src[1]->type)); + return false; + } + if (!ggml_is_contiguous(node->src[1])) { + GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous mask tensor\n"); + return false; + } + } + + // Check optional sinks tensor + if (node->src[2]) { + if (node->src[2]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: SOFTMAX operation with unsupported sinks type: %s (F32 required)\n", + ggml_type_name(node->src[2]->type)); + return false; + } + if (!ggml_is_contiguous(node->src[2])) { + GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous sinks tensor\n"); + return false; + } + } + + // Extract scale and max_bias from op_params + float scale = 1.0f; + float max_bias = 0.0f; + if (node->op_params) { + memcpy(&scale, (const float *) node->op_params + 0, sizeof(float)); + memcpy(&max_bias, (const float *) node->op_params + 1, sizeof(float)); + } + + ggml_et_softmax_params params; + params.src0 = *node->src[0]; // F32 input tensor + if (node->src[1]) { + params.src1 = *node->src[1]; // F32 mask tensor + } else { + memset(¶ms.src1, 0, sizeof(params.src1)); // Zero if no mask + } + if (node->src[2]) { + params.src2 = *node->src[2]; // F32 sinks tensor + } else { + memset(¶ms.src2, 0, sizeof(params.src2)); // Zero if no sinks + } + params.dst = *node; // F32 output tensor + params.scale = scale; // Scale factor + params.max_bias = max_bias; // ALiBi bias + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (softmax_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SOFT_MAX)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SOFTMAX operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &softmax_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for SOFTMAX operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("SOFTMAX", kernel_name, node, "scale=%.6f|max_bias=%.6f|has_mask=%s", (double) scale, + (double) max_bias, node->src[1] ? "yes" : "no"); + return kernel_result; +} + +bool ggml_et_op_flash_attn_ext(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for FLASH_ATTN_EXT operation\n"); + return false; + } + + if (!node->src[0] || !node->src[1] || !node->src[2]) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT operation missing required inputs\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT requires F32 Q and dst, got dst=%s q=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + // K and V can be F16 or F32 + if ((node->src[1]->type != GGML_TYPE_F32 && node->src[1]->type != GGML_TYPE_F16) || + (node->src[2]->type != GGML_TYPE_F32 && node->src[2]->type != GGML_TYPE_F16)) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT K/V must be F16 or F32, got k=%s v=%s\n", ggml_type_name(node->src[1]->type), + ggml_type_name(node->src[2]->type)); + return false; + } + + if (node->src[4] != nullptr) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel does not support sinks\n"); + return false; + } + + // Mask is optional; if present must be F16 or F32 + if (node->src[3] != nullptr && node->src[3]->type != GGML_TYPE_F32 && node->src[3]->type != GGML_TYPE_F16) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT mask must be F16 or F32, got %s\n", ggml_type_name(node->src[3]->type)); + return false; + } + + // Q and dst must be row-contiguous F32 + if (!ggml_is_contiguous_rows(node) || !ggml_is_contiguous_rows(node->src[0])) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT requires row-contiguous Q and dst\n"); + return false; + } + + if (node->nb[0] != sizeof(float) || node->src[0]->nb[0] != sizeof(float)) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT requires contiguous F32 rows for Q and dst\n"); + return false; + } + + // K/V must have element-sized stride in dim 0 + const size_t k_elem = node->src[1]->type == GGML_TYPE_F16 ? 2 : 4; + const size_t v_elem = node->src[2]->type == GGML_TYPE_F16 ? 2 : 4; + if (node->src[1]->nb[0] != k_elem || node->src[2]->nb[0] != v_elem) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT K/V must have element-sized stride in dim 0\n"); + return false; + } + + float scale = 1.0f; + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&scale, (const float *) node->op_params + 0, sizeof(scale)); + memcpy(&max_bias, (const float *) node->op_params + 1, sizeof(max_bias)); + memcpy(&logit_softcap, (const float *) node->op_params + 2, sizeof(logit_softcap)); + + if (max_bias != 0.0f || logit_softcap != 0.0f) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel does not support max_bias or logit_softcap\n"); + return false; + } + + const ggml_prec prec = ggml_flash_attn_ext_get_prec(node); + if (prec != GGML_PREC_F32 && prec != GGML_PREC_DEFAULT) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel only supports F32 precision\n"); + return false; + } + + // dk must match between Q and K; dv must match between V and dst + if (node->src[0]->ne[0] != node->src[1]->ne[0]) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dk mismatch: Q=%lld K=%lld\n", (long long) node->src[0]->ne[0], + (long long) node->src[1]->ne[0]); + return false; + } + + if (node->src[2]->ne[0] != node->ne[0]) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dv mismatch: V=%lld dst=%lld\n", (long long) node->src[2]->ne[0], + (long long) node->ne[0]); + return false; + } + + if (node->src[2]->ne[0] > 512) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dv=%lld exceeds maximum 512\n", (long long) node->src[2]->ne[0]); + return false; + } + + if (node->src[0]->ne[0] > 512) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dk=%lld exceeds maximum 512\n", (long long) node->src[0]->ne[0]); + return false; + } + + // GQA: n_head_q must be a multiple of n_head_kv + const int64_t nhq = node->src[0]->ne[2]; + const int64_t nhk = node->src[1]->ne[2]; + if (nhq % nhk != 0) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT n_head_q (%lld) not divisible by n_head_kv (%lld)\n", (long long) nhq, + (long long) nhk); + return false; + } + + // K and V must have matching sequence length, heads, and batch dims + if (node->src[1]->ne[1] != node->src[2]->ne[1] || node->src[1]->ne[2] != node->src[2]->ne[2] || + node->src[1]->ne[3] != node->src[2]->ne[3]) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT K/V shape mismatch\n"); + return false; + } + + // dst layout checks: [dv, nhq, nq, no] + if (node->src[0]->ne[1] != node->ne[2] || node->src[0]->ne[2] != node->ne[1] || + node->src[0]->ne[3] != node->ne[3]) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dst shape mismatch\n"); + return false; + } + + // Batch dims: Q batch must match K batch + if (node->src[0]->ne[3] != node->src[1]->ne[3]) { + GGML_LOG_ERROR("ET: FLASH_ATTN_EXT batch dimension mismatch\n"); + return false; + } + + ggml_et_flash_attn_ext_params params; + memset(¶ms, 0, sizeof(params)); + params.src0 = *node->src[0]; + params.src1 = *node->src[1]; + params.src2 = *node->src[2]; + if (node->src[3] != nullptr) { + params.mask = *node->src[3]; + params.has_mask = 1; + } + params.dst = *node; + params.scale = scale; + + // Use matrix engine kernel when K/V are F16 and dk is a multiple of 32 + const char * kernel_name; + if (node->src[1]->type == GGML_TYPE_F16 && node->src[2]->type == GGML_TYPE_F16 && (node->src[0]->ne[0] % 32) == 0) { + kernel_name = "flash_attn_ext_f16_me"; + } else { + kernel_name = "flash_attn_ext_f32"; + } + + const bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + ET_PERF_END_EXT("FLASH_ATTN_EXT", kernel_name, node, "scale=%.6f", (double) scale); + return kernel_result; +} + +bool ggml_et_op_get_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for GET_ROWS operation\n"); + return false; + } + + if (!node->src[0] || !node->src[1]) { + GGML_LOG_ERROR("ET: GET_ROWS operation missing required inputs\n"); + return false; + } + + const char * kernel_name; + + if (node->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I32 && + (node->src[0]->type == GGML_TYPE_F32 || node->src[0]->type == GGML_TYPE_F16 || + node->src[0]->type == GGML_TYPE_Q4_0 || node->src[0]->type == GGML_TYPE_Q8_0 || + node->src[0]->type == GGML_TYPE_Q4_K)) { + kernel_name = "get_rows_f32"; + + } else { + GGML_LOG_ERROR("ET: GET_ROWS operation with unsupported types: dst=%s src0=%s src1=%s\n", + ggml_type_name(node->type), ggml_type_name(node->src[0]->type), + ggml_type_name(node->src[1]->type)); + return false; + } + + // Validate contiguity requirements + if (!ggml_is_contiguous(node)) { + GGML_LOG_ERROR("ET: GET_ROWS operation requires contiguous destination tensor\n"); + return false; + } + + if (!ggml_is_contiguous(node->src[0])) { + GGML_LOG_ERROR("ET: GET_ROWS operation requires contiguous data tensor\n"); + return false; + } + + if (!ggml_is_contiguous(node->src[1])) { + GGML_LOG_ERROR("ET: GET_ROWS operation requires contiguous indices tensor\n"); + return false; + } + + // Validate dimension constraints from ggml implementation + if (node->src[0]->ne[2] != node->src[1]->ne[1] || node->src[1]->ne[3] != 1) { + GGML_LOG_ERROR( + "ET: GET_ROWS operation dimension constraint failed: src0.ne[2]=%lld != src1.ne[1]=%lld or src1.ne[3]=%lld " + "!= 1\n", + (long long) node->src[0]->ne[2], (long long) node->src[1]->ne[1], (long long) node->src[1]->ne[3]); + return false; + } + + ggml_et_get_rows_params params; + params.src0 = *node->src[0]; // Data tensor (F32 or Q8_0) + params.src1 = *node->src[1]; // Indices tensor (I32) + params.dst = *node; // Output tensor (F32) + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (get_rows_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GET_ROWS)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for GET_ROWS operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &get_rows_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for GET_ROWS operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("GET_ROWS", kernel_name, node); + return kernel_result; +} + +bool ggml_et_op_cont(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + // Validate source tensor exists + if (!node->src[0]) { + GGML_LOG_ERROR("ET: CONT operation missing source tensor\n"); + return false; + } + + // Validate types match (input and output must be same type) + if (node->type != node->src[0]->type) { + GGML_LOG_ERROR("ET: CONT operation type mismatch: src=%s dst=%s\n", ggml_type_name(node->src[0]->type), + ggml_type_name(node->type)); + return false; + } + + // Validate supported types + if (node->type != GGML_TYPE_F32 && node->type != GGML_TYPE_F16) { + GGML_LOG_ERROR("ET: CONT operation unsupported type: %s (only F32 and F16 supported)\n", + ggml_type_name(node->type)); + return false; + } + + // Validate contiguity - output must be contiguous, input can be non-contiguous + if (!ggml_is_contiguous(node)) { + GGML_LOG_ERROR("ET: CONT operation requires contiguous output tensor\n"); + return false; + } + + // Select kernel based on type + const char * kernel_name; + if (node->type == GGML_TYPE_F32) { + kernel_name = "cont_f32"; + } else if (node->type == GGML_TYPE_F16) { + kernel_name = "cont_f16"; + } else { + GGML_LOG_ERROR("ET: CONT operation with unsupported type: %s\n", ggml_type_name(node->type)); + return false; + } + + ggml_et_cont_params params; + params.src0 = *node->src[0]; // Input tensor (potentially non-contiguous) + params.dst = *node; // Output tensor (contiguous) + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (cont_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CONT)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CONT operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &cont_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for CONT operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("CONT", kernel_name, node); + return kernel_result; +} + +bool ggml_et_op_cumsum(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node || !node->src[0]) { + GGML_LOG_ERROR("ET: Invalid parameters for CUMSUM operation\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: CUMSUM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + const char * kernel_name = "cumsum_f32"; + + ggml_et_cumsum_params params; + params.src0 = *node->src[0]; + params.dst = *node; + + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (cumsum_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CUMSUM)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CUMSUM operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &cumsum_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for CUMSUM operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("CUMSUM", kernel_name, node); + return kernel_result; +} + +bool ggml_et_op_cpy(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + // CPY copies data from src[0] into the layout of dst (which matches src[1]) + // For same-type with contiguous dst, this is identical to CONT + if (!node->src[0]) { + GGML_LOG_ERROR("ET: CPY operation missing source tensor\n"); + return false; + } + + // Scalar / zero-element special path: if any dimension is 0, nothing to copy + const int64_t nelements = node->ne[0] * node->ne[1] * node->ne[2] * node->ne[3]; + if (nelements == 0) { + GGML_LOG_DEBUG("ET: CPY no-op (zero elements): ne=[%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 "]\n", + node->ne[0], node->ne[1], node->ne[2], node->ne[3]); + ET_PERF_END("CPY", "noop", node); + return true; + } + + // Only F32 and F16 supported for dst + if (node->type != GGML_TYPE_F32 && node->type != GGML_TYPE_F16) { + GGML_LOG_ERROR("ET: CPY unsupported dst type: %s\n", ggml_type_name(node->type)); + return false; + } + + // Select kernel based on src/dst type combination + const char * kernel_name; + if (node->src[0]->type == GGML_TYPE_F32 && node->type == GGML_TYPE_F32) { + kernel_name = "cont_f32"; + } else if (node->src[0]->type == GGML_TYPE_F16 && node->type == GGML_TYPE_F16) { + kernel_name = "cont_f16"; + } else if (node->src[0]->type == GGML_TYPE_F32 && node->type == GGML_TYPE_F16) { + kernel_name = "cpy_f32_f16"; + } else { + GGML_LOG_ERROR("ET: CPY unsupported type combination: src=%s dst=%s\n", ggml_type_name(node->src[0]->type), + ggml_type_name(node->type)); + return false; + } + + ggml_et_cont_params params; + params.src0 = *node->src[0]; + params.dst = *node; + + // CPU comparison for debugging + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (cont_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CPY)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CPY operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &cont_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for CPY operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("CPY", kernel_name, node); + return kernel_result; +} + +bool ggml_et_op_concat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for CONCAT operation\n"); + return false; + } + + if (!node->src[0] || !node->src[1]) { + GGML_LOG_ERROR("ET: CONCAT operation missing required inputs\n"); + return false; + } + + const char * kernel_name; + + if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32) { + kernel_name = "concat_f32"; + + } else { + GGML_LOG_ERROR("ET: CONCAT operation with unsupported types: dst=%s src0=%s src1=%s\n", + ggml_type_name(node->type), ggml_type_name(node->src[0]->type), + ggml_type_name(node->src[1]->type)); + return false; + } + + int32_t dim; + memcpy(&dim, node->op_params, sizeof(int32_t)); + + ggml_et_concat_params params; + params.src0 = *node->src[0]; + params.src1 = *node->src[1]; + params.dst = *node; + params.dim = dim; + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (concat_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CONCAT)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CONCAT operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &concat_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for CONCAT operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("CONCAT", kernel_name, node, "dim=%d", dim); + return kernel_result; +} + +bool ggml_et_op_repeat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for REPEAT operation\n"); + return false; + } + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: REPEAT operation missing required input\n"); + return false; + } + + const char * kernel_name; + + if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) { + // No-op REPEAT (every repeat factor is 1): the output is just a copy + // of the input. Route to cont_f32, whose contiguous fast path handles + // arbitrary sizes (including those rejected by repeat_f32's gate, + // e.g. ne[0]=1). + if (ggml_are_same_shape(node->src[0], node)) { + kernel_name = "cont_f32"; + } else { + kernel_name = "repeat_f32"; + } + + } else { + GGML_LOG_ERROR("ET: REPEAT operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type)); + return false; + } + + // ggml_et_cont_params and ggml_et_repeat_params have identical layouts + // (just src0 + dst), so the same payload works for either kernel. + ggml_et_repeat_params params; + params.src0 = *node->src[0]; + params.dst = *node; + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (repeat_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_REPEAT)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for REPEAT operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &repeat_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for REPEAT operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("REPEAT", kernel_name, node); + return kernel_result; +} + +bool ggml_et_op_ssm_conv(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node || !node->src[0] || !node->src[1]) { + GGML_LOG_ERROR("ET: Invalid parameters for SSM_CONV operation\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: SSM_CONV operation with unsupported types: dst=%s src0=%s src1=%s\n", + ggml_type_name(node->type), ggml_type_name(node->src[0]->type), + ggml_type_name(node->src[1]->type)); + return false; + } + + const char * kernel_name = "ssm_conv_f32"; + + ggml_et_ssm_conv_params params; + params.src0 = *node->src[0]; + params.src1 = *node->src[1]; + params.dst = *node; + + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (ssm_conv_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SSM_CONV)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SSM_CONV operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &ssm_conv_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for SSM_CONV operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("SSM_CONV", kernel_name, node); + return kernel_result; +} + +bool ggml_et_op_ssm_scan(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for SSM_SCAN operation\n"); + return false; + } + + for (int i = 0; i < 7; ++i) { + if (!node->src[i]) { + GGML_LOG_ERROR("ET: SSM_SCAN missing required input %d\n", i); + return false; + } + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32 || + node->src[2]->type != GGML_TYPE_F32 || node->src[3]->type != GGML_TYPE_F32 || + node->src[4]->type != GGML_TYPE_F32 || node->src[5]->type != GGML_TYPE_F32 || + node->src[6]->type != GGML_TYPE_I32) { + GGML_LOG_ERROR("ET: SSM_SCAN operation with unsupported types\n"); + return false; + } + + ggml_et_ssm_scan_params params; + params.src0 = *node->src[0]; + params.src1 = *node->src[1]; + params.src2 = *node->src[2]; + params.src3 = *node->src[3]; + params.src4 = *node->src[4]; + params.src5 = *node->src[5]; + params.src6 = *node->src[6]; + params.dst = *node; + params.K = ggml_get_op_params_i32(node, 0); + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "ssm_scan_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + ET_PERF_END("SSM_SCAN", "ssm_scan_f32", node); + return kernel_result; +} + +bool ggml_et_op_rwkv_wkv6(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for RWKV_WKV6 operation\n"); + return false; + } + + // Validate all 6 source tensors exist + for (int i = 0; i <= 5; i++) { + if (!node->src[i]) { + GGML_LOG_ERROR("ET: RWKV_WKV6 operation missing src[%d]\n", i); + return false; + } + } + + if (node->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: RWKV_WKV6 only supports F32, got %s\n", ggml_type_name(node->type)); + return false; + } + + const char * kernel_name = "rwkv_wkv6_f32"; + + const int64_t S = node->src[0]->ne[0]; // head_size + const int64_t H = node->src[0]->ne[1]; // num heads + const int64_t T = node->src[1]->ne[2]; // num tokens + const int64_t n_seqs = node->src[5]->ne[1]; // num sequences + const int64_t C = S * H; + + ggml_et_rwkv_wkv6_params params; + params.k = (float *) node->src[0]->data; + params.v = (float *) node->src[1]->data; + params.r = (float *) node->src[2]->data; + params.tf = (float *) node->src[3]->data; + params.td = (float *) node->src[4]->data; + params.state_in = (float *) node->src[5]->data; + params.dst = (float *) node->data; + params.C = (int32_t) C; + params.H = (int32_t) H; + params.S = (int32_t) S; + params.T = (int32_t) T; + params.n_seqs = (int32_t) n_seqs; + + // Phase 1: Initialize CPU comparison context + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (rwkv_wkv6_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_RWKV_WKV6)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for RWKV_WKV6 operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rwkv_wkv6_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for RWKV_WKV6 operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("RWKV_WKV6", kernel_name, node, "S=%d H=%d T=%d n_seqs=%d", (int) S, (int) H, (int) T, + (int) n_seqs); + return kernel_result; +} + +bool ggml_et_op_rwkv_wkv7(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for RWKV_WKV7 operation\n"); + return false; + } + + // Validate all 7 source tensors exist + for (int i = 0; i <= 6; i++) { + if (!node->src[i]) { + GGML_LOG_ERROR("ET: RWKV_WKV7 operation missing src[%d]\n", i); + return false; + } + } + + if (node->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: RWKV_WKV7 only supports F32, got %s\n", ggml_type_name(node->type)); + return false; + } + + const char * kernel_name = "rwkv_wkv7_f32"; + + const int64_t S = node->src[2]->ne[0]; // head_size + const int64_t H = node->src[2]->ne[1]; // num heads + const int64_t T = node->src[1]->ne[2]; // num tokens + const int64_t n_seqs = node->src[6]->ne[1]; // num sequences + const int64_t C = S * H; + + ggml_et_rwkv_wkv7_params params; + params.r = (float *) node->src[0]->data; + params.w = (float *) node->src[1]->data; + params.k = (float *) node->src[2]->data; + params.v = (float *) node->src[3]->data; + params.a = (float *) node->src[4]->data; + params.b = (float *) node->src[5]->data; + params.state_in = (float *) node->src[6]->data; + params.dst = (float *) node->data; + params.C = (int32_t) C; + params.H = (int32_t) H; + params.S = (int32_t) S; + params.T = (int32_t) T; + params.n_seqs = (int32_t) n_seqs; + + // Phase 1: Initialize CPU comparison context + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (rwkv_wkv7_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_RWKV_WKV7)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for RWKV_WKV7 operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rwkv_wkv7_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for RWKV_WKV7 operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("RWKV_WKV7", kernel_name, node, "S=%d H=%d T=%d n_seqs=%d", (int) S, (int) H, (int) T, + (int) n_seqs); + return kernel_result; +} + +static ggml_et_cpu_compare_config gated_delta_net_cpu_compare_config = { + /* .enabled = */ false, + /* .use_cpu_result = */ false, + /* .log_differences = */ true, + /* .tolerance = */ 1e-4f, + /* .max_log_elements = */ 4096 +}; + +bool ggml_et_op_gated_delta_net(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for GATED_DELTA_NET operation\n"); + return false; + } + + // Validate all 6 source tensors exist + for (int i = 0; i <= 5; i++) { + if (!node->src[i]) { + GGML_LOG_ERROR("ET: GATED_DELTA_NET operation missing src[%d]\n", i); + return false; + } + } + + if (node->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: GATED_DELTA_NET only supports F32, got %s\n", ggml_type_name(node->type)); + return false; + } + + const char * kernel_name = "gated_delta_net_f32"; + + const ggml_tensor * src_q = node->src[0]; + const ggml_tensor * src_k = node->src[1]; + const ggml_tensor * src_v = node->src[2]; + const ggml_tensor * src_g = node->src[3]; + const ggml_tensor * src_beta = node->src[4]; + const ggml_tensor * src_state = node->src[5]; + + const int64_t S_v = src_v->ne[0]; + const int64_t H = src_v->ne[1]; + const int64_t n_tokens = src_v->ne[2]; + const int64_t n_seqs = src_v->ne[3]; + const int64_t H_q = src_q->ne[1]; + const int64_t H_k = src_k->ne[1]; + const int64_t n_seqs_q = src_q->ne[3]; + const int64_t n_seqs_k = src_k->ne[3]; + + ggml_et_gated_delta_net_params params; + params.q = *src_q; + params.k = *src_k; + params.v = *src_v; + params.g = *src_g; + params.beta = *src_beta; + params.state_in = *src_state; + params.dst = *node; + params.S_v = (int32_t) S_v; + params.H = (int32_t) H; + params.H_q = (int32_t) H_q; + params.H_k = (int32_t) H_k; + params.n_tokens = (int32_t) n_tokens; + params.n_seqs = (int32_t) n_seqs; + params.n_seqs_q = (int32_t) n_seqs_q; + params.n_seqs_k = (int32_t) n_seqs_k; + params.kda = (src_g->ne[0] == S_v) ? 1 : 0; + params.K = ggml_get_op_params_i32(node, 0); + params.scale = 1.0f / sqrtf((float) S_v); + + // CPU comparison for debugging + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (gated_delta_net_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GATED_DELTA_NET)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for GATED_DELTA_NET operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &gated_delta_net_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for GATED_DELTA_NET operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END_EXT("GATED_DELTA_NET", kernel_name, node, "S_v=%d H=%d n_tokens=%d n_seqs=%d kda=%d", (int) S_v, + (int) H, (int) n_tokens, (int) n_seqs, params.kda); + return kernel_result; +} + +bool ggml_et_op_set_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!dev_ctx || !node) { + GGML_LOG_ERROR("ET: Invalid parameters for SET_ROWS operation\n"); + return false; + } + + if (!node->src[0] || !node->src[1] || !node->src[2]) { + GGML_LOG_ERROR( + "ET: SET_ROWS operation missing required inputs (needs src[0]=base, src[1]=indices, src[2]=data)\n"); + return false; + } + + const char * kernel_name; + + // Support F32 data with I64 indices -> F32/F16 output (scatter operation) + if (node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I64 && + (node->type == GGML_TYPE_F32 || node->type == GGML_TYPE_F16)) { + if (node->type == GGML_TYPE_F32 || node->type == GGML_TYPE_F16) { + kernel_name = "set_rows_f32"; + } else { + GGML_LOG_ERROR("ET: SET_ROWS unsupported output type: %s\n", ggml_type_name(node->type)); + return false; + } + + } else { + GGML_LOG_ERROR("ET: SET_ROWS operation with unsupported types: dst=%s src0=%s src1=%s\n", + ggml_type_name(node->type), ggml_type_name(node->src[0]->type), + ggml_type_name(node->src[1]->type)); + return false; + } + + // Validate contiguity requirements + if (!ggml_is_contiguous_rows(node)) { + GGML_LOG_ERROR("ET: SET_ROWS operation requires contiguous-rows destination tensor\n"); + return false; + } + + if (!ggml_is_contiguous_rows(node->src[0])) { + GGML_LOG_ERROR("ET: SET_ROWS operation requires contiguous-rows source tensor\n"); + return false; + } + + if (!ggml_is_contiguous(node->src[1])) { + GGML_LOG_ERROR("ET: SET_ROWS operation requires contiguous indices tensor\n"); + return false; + } + + // Validate dimension constraints from ggml implementation + if (!(node->ne[0] == node->src[0]->ne[0] && // same number of columns + node->ne[2] == node->src[0]->ne[2] && // same batch size + node->ne[3] == node->src[0]->ne[3] && // same outer dimension + node->src[0]->ne[1] == node->src[1]->ne[0] && // src rows = index count + node->src[0]->ne[2] % node->src[1]->ne[1] == 0 && // batch constraint + node->src[0]->ne[3] % node->src[1]->ne[2] == 0 && // outer constraint + node->src[1]->ne[3] == 1)) { // indices constraint + GGML_LOG_ERROR("ET: SET_ROWS operation dimension constraint failed\n"); + return false; + } + + ggml_et_set_rows_params params; + params.src0 = *node->src[0]; // F32 source data tensor + params.src1 = *node->src[1]; // I64 indices tensor + params.dst = *node; // F32/F16 destination tensor + + // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel) + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (set_rows_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SET_ROWS)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SET_ROWS operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF); + + // Phase 2: Execute CPU computation and compare with ET result (after ET kernel) + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &set_rows_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for SET_ROWS operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("SET_ROWS", kernel_name, node); + return kernel_result; +} + +bool ggml_et_op_fill(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + ggml_et_fill_params params; + params.dst = *node; + memcpy(¶ms.c, node->op_params, sizeof(float)); + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "fill_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + ET_PERF_END("FILL", "fill_f32", node); + return kernel_result; +} + +bool ggml_et_op_diag(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: DIAG operation missing source tensor\n"); + return false; + } + + ggml_et_diag_params params; + params.src0 = *node->src[0]; + params.dst = *node; + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "diag_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + ET_PERF_END("DIAG", "diag_f32", node); + return kernel_result; +} + +bool ggml_et_op_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: TRI operation missing source tensor\n"); + return false; + } + + ggml_et_tri_params params; + params.src0 = *node->src[0]; + params.dst = *node; + memcpy(¶ms.tri_type, node->op_params, sizeof(int32_t)); + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "tri_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + ET_PERF_END("TRI", "tri_f32", node); + return kernel_result; +} + +bool ggml_et_op_solve_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!node->src[0] || !node->src[1]) { + GGML_LOG_ERROR("ET: SOLVE_TRI operation missing source tensor(s)\n"); + return false; + } + + ggml_et_solve_tri_params params; + params.src0 = *node->src[0]; // A (lower-triangular) + params.src1 = *node->src[1]; // B (RHS) + params.dst = *node; // X (solution) + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "solve_tri_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + ET_PERF_END("SOLVE_TRI", "solve_tri_f32", node); + return kernel_result; +} + +bool ggml_et_op_set(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!node->src[0] || !node->src[1]) { + GGML_LOG_ERROR("ET: SET operation missing source tensor(s)\n"); + return false; + } + + const bool inplace = (bool) ((const int32_t *) node->op_params)[4]; + const size_t offset = ((const int32_t *) node->op_params)[3]; + const size_t nb1 = ((const int32_t *) node->op_params)[0]; + const size_t nb2 = ((const int32_t *) node->op_params)[1]; + const size_t nb3 = ((const int32_t *) node->op_params)[2]; + + if (!inplace) { + GGML_LOG_ERROR("ET: SET only supports inplace (inplace=%d)\n", inplace); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: SET only supports F32 (dst=%s src0=%s src1=%s)\n", ggml_type_name(node->type), + ggml_type_name(node->src[0]->type), ggml_type_name(node->src[1]->type)); + return false; + } + + if (!ggml_are_same_shape(node, node->src[0])) { + GGML_LOG_ERROR("ET: SET requires same-shape src0 and dst\n"); + return false; + } + + if (!ggml_is_contiguous(node) || !ggml_is_contiguous(node->src[0]) || !ggml_is_contiguous(node->src[1])) { + GGML_LOG_ERROR("ET: SET requires contiguous dst, src0, and src1\n"); + return false; + } + + ggml_et_set_params params; + params.src1 = *node->src[1]; + params.dst = *node; + params.nb1 = (int32_t) nb1; + params.nb2 = (int32_t) nb2; + params.nb3 = (int32_t) nb3; + params.offset = (int32_t) offset; + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "set_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + ET_PERF_END("SET", "set_f32", node); + return kernel_result; +} + +bool ggml_et_op_pad(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) { + ET_PERF_START(); + + if (!node->src[0]) { + GGML_LOG_ERROR("ET: PAD operation missing source tensor\n"); + return false; + } + + if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) { + GGML_LOG_ERROR("ET: PAD only supports F32 (src=%s dst=%s)\n", ggml_type_name(node->src[0]->type), + ggml_type_name(node->type)); + return false; + } + + if (!ggml_is_contiguous(node)) { + GGML_LOG_ERROR("ET: PAD requires contiguous output tensor\n"); + return false; + } + + if (node->src[0]->nb[0] != sizeof(float)) { + GGML_LOG_ERROR("ET: PAD requires element-contiguous src dim0 (nb[0]=%zu)\n", (size_t) node->src[0]->nb[0]); + return false; + } + + // Extract padding parameters from op_params + const int32_t * op_params = (const int32_t *) node->op_params; + + ggml_et_pad_params params; + params.src0 = *node->src[0]; + params.dst = *node; + params.lp[0] = op_params[0]; + params.rp[0] = op_params[1]; + params.lp[1] = op_params[2]; + params.rp[1] = op_params[3]; + params.lp[2] = op_params[4]; + params.rp[2] = op_params[5]; + params.lp[3] = op_params[6]; + params.rp[3] = op_params[7]; + + // v1: no dim0 padding + if (params.lp[0] != 0 || params.rp[0] != 0) { + GGML_LOG_ERROR("ET: PAD dim0 padding not supported (lp0=%d rp0=%d)\n", params.lp[0], params.rp[0]); + return false; + } + + ggml_et_cpu_compare_ctx cpu_cmp_ctx; + bool cpu_comparison_active = false; + if (pad_cpu_compare_config.enabled) { + if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_PAD)) { + cpu_comparison_active = true; + } else { + GGML_LOG_WARN("ET: Failed to initialize CPU comparison for PAD operation\n"); + } + } + + bool kernel_result = ggml_et_launch_kernel(dev_ctx, "pad_f32", ¶ms, sizeof(params), 0xFFFFFFFF); + + if (cpu_comparison_active) { + if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &pad_cpu_compare_config)) { + GGML_LOG_WARN("ET: CPU comparison failed for PAD operation\n"); + } + ggml_et_cpu_compare_free(&cpu_cmp_ctx); + } + + ET_PERF_END("PAD", "pad_f32", node); + return kernel_result; +} diff --git a/ggml/src/ggml-et/ggml-et-ops.h b/ggml/src/ggml-et/ggml-et-ops.h new file mode 100644 index 0000000..032f7a2 --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-ops.h @@ -0,0 +1,393 @@ +#pragma once + +#include "ggml-et-common.h" +#include "ggml.h" + +#include + +// Performance logging macros for ET ops +// Logs in machine-parseable pipe-delimited format: ET_PERF|field=value|... +#ifdef ET_PERF_RECORD +# define ET_PERF_START() int64_t _et_perf_start = ggml_time_us() + +# define ET_PERF_END(op_name, kernel_name, node) \ + do { \ + int64_t _et_perf_end = ggml_time_us(); \ + int64_t _et_perf_duration = _et_perf_end - _et_perf_start; \ + GGML_LOG_DEBUG("ET_PERF|op=%s|kernel=%s|duration_us=%" PRId64 "|tensor=%s|shape=[%" PRId64 ",%" PRId64 \ + ",%" PRId64 ",%" PRId64 "]|start_us=%" PRId64 "|end_us=%" PRId64 "\n", \ + op_name, kernel_name, _et_perf_duration, (node)->name, (node)->ne[0], (node)->ne[1], \ + (node)->ne[2], (node)->ne[3], _et_perf_start, _et_perf_end); \ + } while (0) + +# define ET_PERF_END_EXT(op_name, kernel_name, node, fmt, ...) \ + do { \ + int64_t _et_perf_end = ggml_time_us(); \ + int64_t _et_perf_duration = _et_perf_end - _et_perf_start; \ + GGML_LOG_DEBUG("ET_PERF|op=%s|kernel=%s|duration_us=%" PRId64 "|tensor=%s|shape=[%" PRId64 ",%" PRId64 \ + ",%" PRId64 ",%" PRId64 "]|start_us=%" PRId64 "|end_us=%" PRId64 "|" fmt "\n", \ + op_name, kernel_name, _et_perf_duration, (node)->name, (node)->ne[0], (node)->ne[1], \ + (node)->ne[2], (node)->ne[3], _et_perf_start, _et_perf_end, ##__VA_ARGS__); \ + } while (0) +#else + +# define ET_PERF_START() \ + do { \ + } while (0) +# define ET_PERF_END_EXT(op_name, kernel_name, node, fmt, ...) \ + do { \ + (void) (node); \ + } while (0) +# define ET_PERF_END(op_name, kernel_name, node) \ + do { \ + (void) (node); \ + } while (0) + +#endif // ET_PERF_RECORD + +struct ggml_et_binary_params { + ggml_tensor src0; + ggml_tensor src1; + ggml_tensor dst; +}; + +// Q8_0 mul_mat with optional residual bias. +// bias.data == NULL means "no bias" - kernel skips the add. +// When non-NULL, bias must have the same shape and strides as dst. +struct ggml_et_mm_q8_params { + ggml_tensor src0; + ggml_tensor src1; + ggml_tensor dst; + ggml_tensor bias; +}; + +struct ggml_et_im2col_params { + ggml_tensor src0; + ggml_tensor src1; + ggml_tensor dst; +}; + +// Element map parameters for embarrassingly parallel binary operations (MUL, ADD, etc.) +// Operation type is determined by dst->op (GGML_OP_MUL, GGML_OP_ADD, etc.) +struct ggml_et_elmap_params { + ggml_tensor src0; + ggml_tensor src1; + ggml_tensor dst; +}; + +struct ggml_et_rope_settings { + int32_t n_past; + int32_t n_dims; // Number of dimensions to apply ROPE to (must be even) + int32_t mode; // ROPE mode, GGML_ROPE_TYPE_* + int32_t n_ctx; + int32_t n_ctx_orig; + float freq_base; // Base frequency (usually 10000.0f) + float freq_scale; // Frequency scaling factor + float ext_factor; // Extension factor for YaRN + float attn_factor; // Attention factor for YaRN + float beta_fast; // Fast beta for YaRN + float beta_slow; // Slow beta for YaRN + int32_t sections[4]; // Sections for multi-modal ROPE +}; + +struct ggml_et_rope_params { + ggml_tensor src0; + ggml_tensor src1; + ggml_tensor src2; + ggml_tensor dst; + ggml_et_rope_settings rope_params; +}; + +struct ggml_et_rms_norm_params { + ggml_tensor src0; // F32 input tensor + ggml_tensor dst; // F32 output tensor + float eps; // Epsilon parameter for numerical stability +}; + +struct ggml_et_norm_params { + ggml_tensor src0; // F32 input tensor + ggml_tensor dst; // F32 output tensor + float eps; // Epsilon parameter for numerical stability +}; + +struct ggml_et_l2_norm_params { + ggml_tensor src0; // F32 input tensor + ggml_tensor dst; // F32 output tensor + float eps; // Epsilon parameter for numerical stability +}; + +struct ggml_et_group_norm_params { + ggml_tensor src0; // F32 input tensor + ggml_tensor dst; // F32 output tensor + int32_t n_groups; // Number of channel groups + float eps; // Epsilon parameter for numerical stability +}; + +struct ggml_et_glu_params { + ggml_tensor src0; // F32 input tensor A (or combined tensor if src1 is null) + ggml_tensor src1; // F32 input tensor B (null for single tensor mode) + ggml_tensor dst; // F32 output tensor (n/2 columns) + int32_t glu_op_type; // GLU operation type (REGLU=0, GEGLU=1, SWIGLU=2, etc.) + int32_t swapped; // Whether gate and value are swapped + float alpha; // SWIGLU_OAI: sigmoid scaling factor (unused for other variants) + float limit; // SWIGLU_OAI: clamp limit (unused for other variants) +}; + +struct ggml_et_softmax_params { + ggml_tensor src0; // F32 input tensor + ggml_tensor src1; // F32 mask tensor (optional, may be zeroed if not used) + ggml_tensor src2; // F32 sinks tensor (optional, may be zeroed if not used) + ggml_tensor dst; // F32 output tensor + float scale; // Scale factor + float max_bias; // Max bias for ALiBi (0.0f if not used) +}; + +struct ggml_et_flash_attn_ext_params { + ggml_tensor src0; // Q tensor (F32) + ggml_tensor src1; // K tensor (F32) + ggml_tensor src2; // V tensor (F32) + ggml_tensor mask; // mask tensor (F16 or F32), zeroed when absent + ggml_tensor dst; // Output tensor (F32) + float scale; // Scale factor applied to QK + int32_t has_mask; // nonzero if mask is present +}; + +struct ggml_et_get_rows_params { + ggml_tensor src0; // Data tensor (F32 or Q8_0) + ggml_tensor src1; // Row indices tensor (I32) + ggml_tensor dst; // Output tensor (F32) +}; + +struct ggml_et_cont_params { + ggml_tensor src0; // F32 input tensor (non-contiguous) + ggml_tensor dst; // F32 output tensor (contiguous) +}; + +struct ggml_et_concat_params { + ggml_tensor src0; // F32 input tensor 0 + ggml_tensor src1; // F32 input tensor 1 + ggml_tensor dst; // F32 output tensor + int32_t dim; // Concatenation dimension +}; + +struct ggml_et_repeat_params { + ggml_tensor src0; // F32 input tensor (tile) + ggml_tensor dst; // F32 output tensor (tiled result) +}; + +struct ggml_et_fill_params { + ggml_tensor dst; // F32 output tensor (contiguous) + float c; // Constant value to fill +}; + +struct ggml_et_tri_params { + ggml_tensor src0; // F32 input tensor + ggml_tensor dst; // F32 output tensor + int32_t tri_type; // ggml_tri_type enum value +}; + +struct ggml_et_solve_tri_params { + ggml_tensor src0; // A: lower-triangular [n, n, B1, B2] + ggml_tensor src1; // B: RHS [k, n, B1, B2] + ggml_tensor dst; // X: solution [k, n, B1, B2] +}; + +struct ggml_et_pad_params { + ggml_tensor src0; // F32 input (may be non-contiguous, nb[0] must == 4) + ggml_tensor dst; // F32 output (contiguous, ne[0] % 16 == 0) + int32_t lp[4]; // left padding per dimension + int32_t rp[4]; // right padding per dimension +}; + +struct ggml_et_diag_params { + ggml_tensor src0; // F32 input vector + ggml_tensor dst; // F32 output diagonal matrix +}; + +struct ggml_et_ssm_conv_params { + ggml_tensor src0; // conv_x: [d_conv - 1 + n_t, d_inner, n_seqs] + ggml_tensor src1; // conv1d.weight: [d_conv, d_inner] + ggml_tensor dst; // output: [d_inner, n_t, n_seqs] +}; + +struct ggml_et_ssm_scan_params { + ggml_tensor src0; // s: [d_state, head_dim, n_head, n_seqs] + ggml_tensor src1; // x: [head_dim, n_head, n_seq_tokens, n_seqs] + ggml_tensor src2; // dt: [n_head, n_seq_tokens, n_seqs] + ggml_tensor src3; // A: [d_state, n_head] or [1, n_head] + ggml_tensor src4; // B: [d_state, n_group, n_seq_tokens, n_seqs] + ggml_tensor src5; // C: [d_state, n_group, n_seq_tokens, n_seqs] + ggml_tensor src6; // ids: [n_seqs] i32 + ggml_tensor dst; // [y, states] packed output from ggml_ssm_scan() + int32_t K; +}; + +struct ggml_et_rwkv_wkv6_params { + float * k; // src[0]: [S, H, T] key + float * v; // src[1]: [S, H, T] value + float * r; // src[2]: [S, H, T] receptance + float * tf; // src[3]: [S, H] time_faaaa (per-head) + float * td; // src[4]: [S, H, T] time_decay + float * state_in; // src[5]: [S*S*H, n_seqs] initial state + float * dst; // [C, T + S*n_seqs] output + state_out + int32_t C; // total channels (S * H) + int32_t H; // number of heads + int32_t S; // head size + int32_t T; // number of tokens + int32_t n_seqs; // number of sequences +}; + +struct ggml_et_rwkv_wkv7_params { + float * r; // [S, H, T] receptance + float * w; // [S, H, T] decay + float * k; // [S, H, T] key + float * v; // [S, H, T] value + float * a; // [S, H, T] bonus gate + float * b; // [S, H, T] bonus key + float * state_in; // [S*S*H, n_seqs] initial state + float * dst; // [C, T + S*n_seqs] output + state_out + int32_t C; // total channels (S * H) + int32_t H; // number of heads + int32_t S; // head size + int32_t T; // number of tokens + int32_t n_seqs; // number of sequences +}; + +struct ggml_et_gated_delta_net_params { + ggml_tensor q; // [S_v, H_q, n_tokens, n_seqs_q] + ggml_tensor k; // [S_v, H_k, n_tokens, n_seqs_k] + ggml_tensor v; // [S_v, H, n_tokens, n_seqs] + ggml_tensor g; // [1 or S_v, H, n_tokens, n_seqs] + ggml_tensor beta; // [1, H, n_tokens, n_seqs] + ggml_tensor state_in; // [S_v*S_v*H, K, n_seqs] + ggml_tensor dst; // [S_v*H, n_tokens*n_seqs + S_v*n_seqs*K] + int32_t S_v; // head dimension (value size) + int32_t H; // number of value heads + int32_t H_q; // number of Q heads + int32_t H_k; // number of K heads + int32_t n_tokens; // total tokens + int32_t n_seqs; // number of sequences (from V) + int32_t n_seqs_q; // Q sequence count + int32_t n_seqs_k; // K sequence count + int32_t kda; // 1 if per-element gate (g_ne0 == S_v), 0 if scalar + int32_t K; // snapshot slot count + float scale; // 1/sqrt(S_v) +}; + +struct ggml_et_set_rows_params { + ggml_tensor src0; // F32 source data tensor + ggml_tensor src1; // I64 row indices tensor + ggml_tensor dst; // F32/F16 destination tensor +}; + +struct ggml_et_set_params { + ggml_tensor src1; // F32 source view to write into dst + ggml_tensor dst; // F32 destination/base tensor + int32_t nb1; // destination view stride for dim 1 + int32_t nb2; // destination view stride for dim 2 + int32_t nb3; // destination view stride for dim 3 + int32_t offset; // byte offset into destination +}; + +struct ggml_et_rms_norm_mul_params { + ggml_tensor src0; // F32 input tensor (to be normalized) + ggml_tensor src1; // F32 weights tensor (element-wise multiply) + ggml_tensor dst; // F32 output tensor + float eps; // Epsilon for numerical stability +}; + +struct ggml_et_mul_mat_id_params { + ggml_tensor src0; // Expert weight matrices (Q8_0/F16/F32) [K, M, n_expert] + ggml_tensor src1; // Activations (F32) [K, n_expert_used, batch] + ggml_tensor src2; // Expert indices (I32) [n_expert_used, batch] + ggml_tensor dst; // Output (F32) [M, n_expert_used, batch, 1] +}; + +struct ggml_et_sqr_params { + ggml_tensor src0; // F32 input tensor + ggml_tensor dst; // F32 output tensor +}; + +struct ggml_et_unary_params { + ggml_tensor src0; // F32 input tensor + ggml_tensor dst; // F32 output tensor + int32_t unary_op; // ggml_unary_op enum value +}; + +struct ggml_et_sum_rows_params { + ggml_tensor src0; // F32 input tensor [ne00, ne01, ne02, ne03] + ggml_tensor dst; // F32 output tensor [1, ne01, ne02, ne03] +}; + +struct ggml_et_mean_params { + ggml_tensor src0; // F32 input tensor [ne00, ne01, ne02, ne03] + ggml_tensor dst; // F32 output tensor [1, ne01, ne02, ne03] +}; + +struct ggml_et_clamp_params { + ggml_tensor src0; // F32 input tensor (contiguous) + ggml_tensor dst; // F32 output tensor (contiguous; may alias src0) + float min_val; + float max_val; +}; + +struct ggml_et_cumsum_params { + ggml_tensor src0; // F32 input tensor [ne00, ne01, ne02, ne03] + ggml_tensor dst; // F32 output tensor [ne00, ne01, ne02, ne03] +}; + +struct ggml_et_scale_params { + ggml_tensor src0; // F32 input tensor + ggml_tensor dst; // F32 output tensor + float scale; // Scale factor + float bias; // Bias (additive offset) +}; + +bool ggml_et_op_cumsum(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_sqr(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_unary(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_sum_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_mean(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_clamp(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_scale(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_mul(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_add(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_sub(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +// add_node is optional: when non-NULL and the pair (node, add_node) was +// validated by ggml_et_can_fuse({MUL_MAT, ADD}), the Q8_0 path writes +// dst = mm(...) + add_node's "other" operand (the bias) in one launch. +bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx, + const ggml_tensor * node, + const ggml_tensor * add_node = nullptr); +bool ggml_et_op_mul_mat_id(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_rope(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_rms_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_l2_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_group_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_glu(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_softmax(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_im2col(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_conv_2d(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_flash_attn_ext(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_get_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_set_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_cont(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_concat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_repeat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_rwkv_wkv6(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_rwkv_wkv7(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_cpy(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_gated_delta_net(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_elmap(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_fill(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_diag(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_solve_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_pad(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_set(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_ssm_conv(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_ssm_scan(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node); +bool ggml_et_op_rms_norm_mul(ggml_backend_et_device_context * dev_ctx, + const ggml_tensor * rms_norm_node, + const ggml_tensor * mul_node); diff --git a/ggml/src/ggml-et/ggml-et-uberkernel-common.h b/ggml/src/ggml-et/ggml-et-uberkernel-common.h new file mode 100644 index 0000000..6044473 --- /dev/null +++ b/ggml/src/ggml-et/ggml-et-uberkernel-common.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +struct ggml_et_uberkernel_inst { + uint16_t kernel_id; + uint16_t flags; + uint32_t params_offset; + uint32_t params_size; +}; + +struct ggml_et_uberkernel_params { + uint32_t num_insts; + uint32_t inst_stride; + uint64_t insts; + uint64_t params_blob; +}; diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp new file mode 100644 index 0000000..54a6d16 --- /dev/null +++ b/ggml/src/ggml-et/ggml-et.cpp @@ -0,0 +1,1881 @@ +#include "ggml-et.h" + +#include "ggml-backend-impl.h" +#include "ggml-backend.h" +#include "ggml-et-common.h" +#include "ggml-et-kernels.h" +#include "ggml-et-memops.h" +#include "ggml-et-ops.h" +#include "ggml-impl.h" +#include "ggml.h" + +#include + +#include +#include +#include +#include +#include +#include + +#if __has_include() +# include +namespace fs = std::filesystem; +#elif __has_include() +# include +namespace fs = std::experimental::filesystem; +#else +# error "cannot include the filesystem library" +#endif + +/* + * ggml_et_dump_tensor_metadata + * @brief prints the metadata of a single tensorf + */ +static void ggml_et_dump_tensor_metadata(const ggml_tensor * ggtensor, size_t indent_level, const char * title) { + std::string spaces(indent_level, ' '); + fprintf(stderr, + "%s%s: %s\n" + "%s type: %s\n" + "%s ne: %lld %lld %lld %lld\n" + "%s nb: %zu %zu %zu %zu\n" + "%s op: %s\n" + "%s data: %p\n" + "%s src0: %p\n", + spaces.c_str(), title, ggtensor->name, spaces.c_str(), ggml_type_name(ggtensor->type), spaces.c_str(), (long long) ggtensor->ne[0], + (long long) ggtensor->ne[1], (long long) ggtensor->ne[2], (long long) ggtensor->ne[3], spaces.c_str(), + ggtensor->nb[0], ggtensor->nb[1], ggtensor->nb[2], ggtensor->nb[3], spaces.c_str(), ggml_op_name(ggtensor->op), + spaces.c_str(), ggtensor->data, spaces.c_str(), (void *) ggtensor->src[0]); +} + +/* + * ggml_et_dump_operator_metadata + * @brief prints the metadata of a single tensor (or operator) including it's input and views + */ +static void ggml_et_dump_operator_metadata(const ggml_tensor * ggtensor) { + GGML_ASSERT(ggtensor != NULL); + ggml_et_dump_tensor_metadata(ggtensor, 0, "GGML tensor"); + for (int i = 0; i < GGML_MAX_SRC && ggtensor->src[i]; i++) { + char arr[16]; + int n = snprintf(arr, sizeof(arr), "src[%i]->name", i); + GGML_ASSERT((unsigned) n < sizeof(arr) && "printed too much data to stack buffer"); + ggml_et_dump_tensor_metadata(ggtensor->src[i], 2, arr); + } + if (ggtensor->view_src) { + ggml_et_dump_tensor_metadata(ggtensor, 2, "view_src"); + } +} + +static struct ggml_et_driver { + std::shared_ptr device_layer; + std::shared_ptr runtime; + std::unique_ptr profile_stream; + std::unique_ptr kernel_id_stream; + std::vector> kernel_map; + bool profiling_enabled = false; +} _drv; + +// Check at runtime environment variables for paths likely holding ET toolchain with sysemu elf files +static std::string ggml_et_get_default_et_path() { + // List of environment variables to check in order of preference + const char * const env_vars[] = { "ET_TOOLCHAIN", "TOOLCHAIN_ROOT" }; + + for (const char * var : env_vars) { + if (const char * et_path = std::getenv(var)) { + if (et_path && *et_path != '\0') { + return fs::path(et_path).string(); + } + } + } + + // Otherwise assume default + return fs::path("/opt/et").string(); +} + +// config when using sysemu instead of PCIe hardware device +// adapted from `ainekko/et-platform/esperanto-tools-libs/tools/src/bench.cpp` +static inline auto ggml_et_get_default_sysemu_options() { + constexpr uint64_t kSysEmuMaxCycles = std::numeric_limits::max(); + constexpr uint64_t kSysEmuMinionShiresMask = 0x1FFFFFFFFu; + const std::string et_path = ggml_et_get_default_et_path() + "/"; + + emu::SysEmuOptions sysEmuOptions; + + // Construct all paths + sysEmuOptions.bootromTrampolineToBL2ElfPath = + et_path + "lib/esperanto-fw/BootromTrampolineToBL2/BootromTrampolineToBL2.elf"; + sysEmuOptions.spBL2ElfPath = + et_path + "lib/esperanto-fw/ServiceProcessorBL2/fast-boot/ServiceProcessorBL2_fast-boot.elf"; + sysEmuOptions.machineMinionElfPath = et_path + "lib/esperanto-fw/MachineMinion/MachineMinion.elf"; + sysEmuOptions.masterMinionElfPath = et_path + "lib/esperanto-fw/MasterMinion/MasterMinion.elf"; + sysEmuOptions.workerMinionElfPath = et_path + "lib/esperanto-fw/WorkerMinion/WorkerMinion.elf"; + sysEmuOptions.executablePath = et_path + "bin/sys_emu"; + + // Check that each path has a valid existing non-zero file otherwise emulator just silently hangs + const std::vector required_files = { + sysEmuOptions.bootromTrampolineToBL2ElfPath, sysEmuOptions.spBL2ElfPath, + sysEmuOptions.machineMinionElfPath, sysEmuOptions.masterMinionElfPath, + sysEmuOptions.workerMinionElfPath, sysEmuOptions.executablePath, + }; + + for (const auto & file : required_files) { + if (!fs::exists(file) || fs::file_size(file) == 0) { + // Check that each path has a valid existing non-zero file otherwise emulator just silently hangs + GGML_LOG_ERROR("ET: Unable to find required sysemu file: %s", file.c_str()); + GGML_LOG_ERROR("ET: Confirm et-platform is correctly installed at configured path."); + abort(); + } + } + + sysEmuOptions.runDir = (fs::current_path().string() + "/"); + sysEmuOptions.maxCycles = kSysEmuMaxCycles; + sysEmuOptions.minionShiresMask = kSysEmuMinionShiresMask; + sysEmuOptions.puUart0Path = sysEmuOptions.runDir + "pu_uart0_tx.log"; + sysEmuOptions.puUart1Path = sysEmuOptions.runDir + "pu_uart1_tx.log"; + sysEmuOptions.spUart0Path = sysEmuOptions.runDir + "spio_uart0_tx.log"; + sysEmuOptions.spUart1Path = sysEmuOptions.runDir + "spio_uart1_tx.log"; + sysEmuOptions.startGdb = false; + sysEmuOptions.memcheck = false; + + return sysEmuOptions; +} + +// Forward declaration +static void ggml_et_driver_cleanup(); + +static bool ggml_et_driver_init() { + if (_drv.runtime != nullptr) { + assert(_drv.device_layer != nullptr); + } else { + try { +#if defined GGML_ET_SYSEMU && GGML_ET_SYSEMU + // For emulator device using sysEmuOptions provided by function above enabled compiling with `-DGGML_ET_SYSEMU=ON` + _drv.device_layer = dev::IDeviceLayer::createSysEmuDeviceLayer(ggml_et_get_default_sysemu_options()); +#else + // For physical PCIe device + _drv.device_layer = dev::IDeviceLayer::createPcieDeviceLayer(); +#endif // GGML_ET_SYSEMU + + _drv.runtime = rt::IRuntime::create(_drv.device_layer); + + // Initialize profiler if requested via environment variable + const char * profile_path = getenv("GGML_ET_PROFILE"); + if (profile_path) { + std::string output_path = std::string(profile_path) + "/et_runtime_trace.json"; + std::string kernel_id_path = std::string(profile_path) + "/kernel_id.json"; + + _drv.profile_stream = std::make_unique(output_path); + _drv.kernel_id_stream = std::make_unique(kernel_id_path); + if (!_drv.profile_stream->is_open()) { + GGML_LOG_ERROR("ET: Failed to open profiling output file: %s", output_path.c_str()); + abort(); + } + if (!_drv.kernel_id_stream->is_open()) { + GGML_LOG_ERROR("ET: Failed to open profiling kernel map: %s", kernel_id_path.c_str()); + abort(); + } + + auto * profiler = _drv.runtime->getProfiler(); + profiler->start(*_drv.profile_stream, rt::IProfiler::OutputType::Json); + _drv.profiling_enabled = true; + GGML_LOG_INFO("ET: Runtime profiler started (JSON format)"); + + // Register cleanup at program exit + std::atexit(ggml_et_driver_cleanup); + } + } catch (const std::exception & e) { + GGML_LOG_ERROR("ggml_et: %s", e.what()); + if (_drv.device_layer != nullptr) { + _drv.device_layer.reset(); + } + if (_drv.runtime != nullptr) { + _drv.runtime.reset(); + } + return false; + } + } + return true; +} + +static std::shared_ptr ggml_et_devicelayer() { + return _drv.device_layer; +} + +std::shared_ptr ggml_et_runtime() { + return _drv.runtime; +} + +static void ggml_et_driver_cleanup() { + if (_drv.profiling_enabled && _drv.runtime) { + GGML_LOG_INFO("ET: Stopping runtime profiler"); + auto * profiler = _drv.runtime->getProfiler(); + profiler->stop(); + _drv.profiling_enabled = false; + + if (_drv.profile_stream) { + _drv.profile_stream->close(); + _drv.profile_stream.reset(); + } + + // Save kernel map + if (_drv.kernel_id_stream && !_drv.kernel_map.empty()) { + auto & os = *_drv.kernel_id_stream; + // XXX: Manual JSON construction. Not pretty but removes dependency + os << "{\n"; + for (size_t i = 0; i < _drv.kernel_map.size(); i++) { + os << " \"" << _drv.kernel_map[i].first << "\": " << (int) _drv.kernel_map[i].second; + if (i + 1 < _drv.kernel_map.size()) { + os << ","; + } + os << "\n"; + } + os << "}\n"; + _drv.kernel_id_stream->close(); + _drv.kernel_id_stream.reset(); + } + } +} + +static ggml_backend_dev_t ggml_backend_et_reg_get_device(ggml_backend_reg_t reg, size_t devidx); + +static void ggml_backend_et_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_et_buffer_context * ctx = (ggml_backend_et_buffer_context *) buffer->context; + if (ctx->data != nullptr) { + std::shared_ptr runtime = ggml_et_runtime(); + if (runtime) { + runtime->freeDevice(ctx->rtid, static_cast(ctx->data)); + } + } + delete ctx; +} + +static void * ggml_backend_et_buffer_get_base(ggml_backend_buffer_t buffer) { + ggml_backend_et_buffer_context * ctx = (ggml_backend_et_buffer_context *) buffer->context; + return ctx->data; +} + +static ggml_status ggml_backend_et_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + // View tensors share buffer with their view_src, no additional initialization needed + if (tensor->view_src != NULL) { + return GGML_STATUS_SUCCESS; + } + + const size_t original_size = ggml_nbytes(tensor); + const size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor); + + // Clear padding bytes to avoid NaN values + // XXX: Martin - do we need this? + if (padded_size > original_size) { + const size_t padding_size = padded_size - original_size; + + // Get device context to access memops kernel + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buffer->buft->device->context; + if (!dev_ctx) { + GGML_LOG_ERROR("ET: Failed to get device context for padding clear"); + return GGML_STATUS_FAILED; + } + + // Use device-side memset kernel for efficient padding clear + std::byte * padding_ptr = static_cast(tensor->data) + original_size; + if (!ggml_et_memset(dev_ctx, padding_ptr, 0, padding_size)) { + GGML_LOG_ERROR("ET: Failed to clear padding using memset kernel for tensor %s", tensor->name); + return GGML_STATUS_FAILED; + } + } + + return GGML_STATUS_SUCCESS; +} + +static void ggml_backend_et_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + return; + } + + // Create short-lived stream for this transfer + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buffer->buft->device->context; + rt::StreamId stream = dev_ctx->default_stream; + + std::byte * dst_ptr = static_cast(tensor->data) + offset; + const std::byte * src_ptr = static_cast(data); + + rt::EventId event = runtime->memcpyHostToDevice(stream, src_ptr, dst_ptr, size, true /*barrier*/); + + runtime->waitForEvent(event); +} + +static void ggml_backend_et_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + return; + } + + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buffer->buft->device->context; + rt::StreamId stream = dev_ctx->default_stream; + + const std::byte * src_ptr = static_cast(tensor->data) + offset; + std::byte * dst_ptr = static_cast(data); + + rt::EventId event = runtime->memcpyDeviceToHost(stream, src_ptr, dst_ptr, size, true /*barrier*/); + + runtime->waitForEvent(event); +} + +static bool ggml_backend_et_buffer_cpy_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * src, + ggml_tensor * dst) { + GGML_UNUSED(buffer); + GGML_UNUSED(src); + GGML_UNUSED(dst); + return false; +} + +static void ggml_backend_et_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_et_buffer_context * ctx = (ggml_backend_et_buffer_context *) buffer->context; + + if (ctx->size == 0 || ctx->data == nullptr) { + return; + } + + // Get device context to access memops kernel + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buffer->buft->device->context; + if (!dev_ctx) { + GGML_LOG_ERROR("ET: Failed to get device context for buffer clear"); + return; + } + + // Use device-side memset kernel for efficient clearing + if (!ggml_et_memset(dev_ctx, ctx->data, value, ctx->size)) { + GGML_LOG_ERROR("ET: buffer_clear failed using memset kernel"); + return; + } + + GGML_LOG_DEBUG("ET: Buffer cleared successfully using memops kernel"); +} + +static const struct ggml_backend_buffer_i ggml_backend_et_buffer_i = { + /* .free_buffer = */ ggml_backend_et_buffer_free_buffer, + /* .get_base = */ ggml_backend_et_buffer_get_base, + /* .init_tensor = */ ggml_backend_et_buffer_init_tensor, + /* .memset_tensor = */ NULL, + /* .set_tensor = */ ggml_backend_et_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_et_buffer_get_tensor, + /* .set_tensor_2d = */ NULL, + /* .get_tensor_2d = */ NULL, + /* .cpy_tensor = */ ggml_backend_et_buffer_cpy_tensor, + /* .clear = */ ggml_backend_et_buffer_clear, + /* .reset = */ NULL, +}; + +static const char * ggml_backend_et_buffer_type_get_name(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + return GGML_ET_NAME; +} + +static ggml_backend_buffer_t ggml_backend_et_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + ggml_backend_et_buffer_type_context * btctx = (ggml_backend_et_buffer_type_context *) buft->context; + + ggml_backend_et_buffer_context * ctx = new ggml_backend_et_buffer_context; + ctx->devidx = btctx->devidx; + ctx->size = size; + + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + delete ctx; + return nullptr; + } + + std::vector rtids = runtime->getDevices(); + if (static_cast(btctx->devidx) >= rtids.size()) { + delete ctx; + return nullptr; + } + ctx->rtid = rtids[btctx->devidx]; + + ctx->data = runtime->mallocDevice(ctx->rtid, size); + if (ctx->data == nullptr) { + delete ctx; + return nullptr; + } + + return ggml_backend_buffer_init(buft, ggml_backend_et_buffer_i, ctx, size); +} + +static size_t ggml_backend_et_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime || !buft->device) { + return GGML_MEM_ALIGN; + } + + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buft->device->context; + rt::DeviceProperties prop = runtime->getDeviceProperties(dev_ctx->rtid); + return prop.cacheLineSize_; +} + +static size_t ggml_backend_et_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + if (buft->device) { + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buft->device->context; + return dev_ctx->total_mem; + } + return SIZE_MAX; +} + +static size_t ggml_backend_et_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + GGML_UNUSED(buft); + return ggml_nbytes_pad(tensor); +} + +static bool ggml_backend_et_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + return false; +} + +static const struct ggml_backend_buffer_type_i ggml_backend_et_buffer_type_i = { + /* .get_name = */ ggml_backend_et_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_et_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_et_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_et_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_et_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_et_buffer_type_is_host, +}; + +static const char * ggml_backend_et_get_name(ggml_backend_t backend) { + GGML_UNUSED(backend); + return GGML_ET_NAME; +} + +static void ggml_backend_et_free(ggml_backend_t backend) { + ggml_backend_et_context * et_ctx = (ggml_backend_et_context *) backend->context; + std::shared_ptr runtime = ggml_et_runtime(); + + // Clean up kernels on this device before freeing backend + ggml_backend_dev_t dev = ggml_backend_et_reg_get_device(ggml_backend_et_reg(), et_ctx->devidx); + if (dev && dev->context) { + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context; + + if (_drv.profiling_enabled) { + auto kernels = ggml_et_get_loaded_kernels(dev_ctx); + _drv.kernel_map.insert(_drv.kernel_map.end(), kernels.begin(), kernels.end()); + } + + ggml_et_unload_all_kernels(dev_ctx); + + if (runtime) { + if (dev_ctx->trace_buffer) { + runtime->freeDevice(dev_ctx->rtid, dev_ctx->trace_buffer); + dev_ctx->trace_buffer = nullptr; + } + // Drain any in-flight uberkernel launches before freeing the + // device buffers they read from. + runtime->waitForStream(dev_ctx->default_stream); + for (auto & slot : dev_ctx->uberkernel.slots) { + if (slot.device_insts) { + runtime->freeDevice(dev_ctx->rtid, slot.device_insts); + slot.device_insts = nullptr; + } + if (slot.device_params) { + runtime->freeDevice(dev_ctx->rtid, slot.device_params); + slot.device_params = nullptr; + } + slot.has_pending = false; + } + } + } + + delete et_ctx; + delete backend; +} + +static ggml_backend_buffer_type_t ggml_backend_et_get_default_buffer_type(ggml_backend_t backend) { + ggml_backend_et_context * et_ctx = (ggml_backend_et_context *) backend->context; + + return ggml_backend_et_buffer_type(et_ctx->devidx); +} + +static void ggml_backend_et_set_tensor_async(ggml_backend_t backend, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + return; + } + + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) backend->device->context; + rt::StreamId stream = dev_ctx->default_stream; + + std::byte * dst_ptr = static_cast(tensor->data) + offset; + const std::byte * src_ptr = static_cast(data); + + runtime->memcpyHostToDevice(stream, src_ptr, dst_ptr, size, true /*barrier*/); +} + +static void ggml_backend_et_get_tensor_async(ggml_backend_t backend, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + return; + } + + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) backend->device->context; + rt::StreamId stream = dev_ctx->default_stream; + + const std::byte * src_ptr = static_cast(tensor->data) + offset; + std::byte * dst_ptr = static_cast(data); + + runtime->memcpyDeviceToHost(stream, src_ptr, dst_ptr, size, true /*barrier*/); +} + +static bool ggml_backend_et_cpy_tensor_async(ggml_backend_t backend_src, + ggml_backend_t backend_dst, + const ggml_tensor * src, + ggml_tensor * dst) { + GGML_UNUSED(backend_src); + GGML_UNUSED(backend_dst); + GGML_UNUSED(src); + GGML_UNUSED(dst); + return false; +} + +static void ggml_backend_et_synchronize(ggml_backend_t backend) { + std::shared_ptr runtime = ggml_et_runtime(); + if (!runtime) { + return; + } + + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) backend->device->context; + runtime->waitForStream(dev_ctx->default_stream); + + auto errors = runtime->retrieveStreamErrors(dev_ctx->default_stream); + if (errors.empty()) { + return; + } + for (const auto & err : errors) { + GGML_LOG_ERROR("ET: stream error detected at synchronization point. Code: %d,Type: %d\n", (int) err.errorCode_, + (int) err.errorContext_.value()[0].type_); + } + abort(); +} + +static bool ggml_et_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { + if (!ggml_can_fuse(cgraph, node_idx, ops)) { + return false; + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_MUL_MAT && ops.begin()[1] == GGML_OP_ADD) { + const ggml_tensor * mm = cgraph->nodes[node_idx]; + const ggml_tensor * add = cgraph->nodes[node_idx + 1]; + + // Only Q8_0 weights x F32 activations -> F32 (the kernel that has + // the bias path). Other MM variants must wait for their own kernel + // bias support. + if (mm->type != GGML_TYPE_F32 || mm->src[0]->type != GGML_TYPE_Q8_0 || mm->src[1]->type != GGML_TYPE_F32) { + return false; + } + + // ADD must be F32 and one of its operands must be the MM output. + if (add->type != GGML_TYPE_F32) { + return false; + } + if (add->src[0] != mm && add->src[1] != mm) { + return false; + } + + const ggml_tensor * bias = (add->src[0] == mm) ? add->src[1] : add->src[0]; + + if (bias->type != GGML_TYPE_F32) { + return false; + } + + // No broadcasting: bias shape must equal MM output shape. + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (bias->ne[i] != mm->ne[i]) { + return false; + } + } + + // Bias and dst must be contiguous and have identical strides - the + // kernel uses dst-style offset arithmetic against bias's nb[]. + if (!ggml_is_contiguous(bias) || !ggml_is_contiguous(mm)) { + return false; + } + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if ((int64_t) bias->nb[i] != (int64_t) add->nb[i]) { + return false; + } + } + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { + const ggml_tensor * rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + + // ET only supports F32 + if (rms_norm->src[0]->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { + return false; + } + + // Identify the weights tensor (the MUL operand that isn't rms_norm output) + const ggml_tensor * weights = (mul->src[0] == rms_norm) ? mul->src[1] : mul->src[0]; + + if (weights->type != GGML_TYPE_F32) { + return false; + } + + // Both inputs must be contiguous (ET hardware requirement) + if (!ggml_is_contiguous(rms_norm->src[0]) || !ggml_is_contiguous_rows(weights)) { + return false; + } + + // ET requires cache-aligned rows (ne[0] % 16 == 0) + if (rms_norm->src[0]->ne[0] % 16 != 0 || weights->ne[0] % 16 != 0) { + return false; + } + + // Fused kernel doesn't handle dim-0 broadcasting + if (weights->ne[0] != rms_norm->src[0]->ne[0]) { + return false; + } + } + + return true; +} + +static ggml_status ggml_backend_et_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) backend->device->context; + ggml_et_uberkernel_begin_graph(&dev_ctx->uberkernel); + + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + + if (node->op == GGML_OP_NONE || node->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE || + node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) { + continue; + } + + // --- Fusion checks (before regular dispatch) --- + if (ggml_et_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { + ggml_et_op_rms_norm_mul(dev_ctx, node, cgraph->nodes[i + 1]); + i++; // skip the MUL node + continue; + } + if (ggml_et_can_fuse(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_ADD })) { + ggml_et_op_mul_mat(dev_ctx, node, cgraph->nodes[i + 1]); + i++; // skip the ADD node + continue; + } + + switch (node->op) { + case GGML_OP_SQR: + ggml_et_op_sqr(dev_ctx, node); + break; + + case GGML_OP_UNARY: + ggml_et_op_unary(dev_ctx, node); + break; + + case GGML_OP_SUM_ROWS: + ggml_et_op_sum_rows(dev_ctx, node); + break; + + case GGML_OP_MEAN: + ggml_et_op_mean(dev_ctx, node); + break; + + case GGML_OP_CLAMP: + ggml_et_op_clamp(dev_ctx, node); + break; + + case GGML_OP_MUL: + ggml_et_op_mul(dev_ctx, node); + break; + + case GGML_OP_ADD: + ggml_et_op_add(dev_ctx, node); + break; + + case GGML_OP_SUB: + ggml_et_op_sub(dev_ctx, node); + break; + + case GGML_OP_CUMSUM: + ggml_et_op_cumsum(dev_ctx, node); + break; + + case GGML_OP_MUL_MAT: + ggml_et_op_mul_mat(dev_ctx, node); + break; + + case GGML_OP_MUL_MAT_ID: + ggml_et_op_mul_mat_id(dev_ctx, node); + break; + + case GGML_OP_ROPE: + ggml_et_op_rope(dev_ctx, node); + break; + + case GGML_OP_RMS_NORM: + ggml_et_op_rms_norm(dev_ctx, node); + break; + + case GGML_OP_NORM: + ggml_et_op_norm(dev_ctx, node); + break; + + case GGML_OP_L2_NORM: + ggml_et_op_l2_norm(dev_ctx, node); + break; + + case GGML_OP_GROUP_NORM: + ggml_et_op_group_norm(dev_ctx, node); + break; + + case GGML_OP_SCALE: + ggml_et_op_scale(dev_ctx, node); + break; + + case GGML_OP_GLU: + ggml_et_op_glu(dev_ctx, node); + break; + + case GGML_OP_SOFT_MAX: + ggml_et_op_softmax(dev_ctx, node); + break; + + case GGML_OP_IM2COL: + ggml_et_op_im2col(dev_ctx, node); + break; + + case GGML_OP_CONV_2D: + ggml_et_op_conv_2d(dev_ctx, node); + break; + + case GGML_OP_FLASH_ATTN_EXT: + ggml_et_op_flash_attn_ext(dev_ctx, node); + break; + + case GGML_OP_GET_ROWS: + ggml_et_op_get_rows(dev_ctx, node); + break; + + case GGML_OP_CONT: + ggml_et_op_cont(dev_ctx, node); + break; + + case GGML_OP_CPY: + ggml_et_op_cpy(dev_ctx, node); + break; + + case GGML_OP_CONCAT: + ggml_et_op_concat(dev_ctx, node); + break; + + case GGML_OP_REPEAT: + ggml_et_op_repeat(dev_ctx, node); + break; + + case GGML_OP_SSM_CONV: + ggml_et_op_ssm_conv(dev_ctx, node); + break; + + case GGML_OP_SSM_SCAN: + ggml_et_op_ssm_scan(dev_ctx, node); + break; + + case GGML_OP_PAD: + ggml_et_op_pad(dev_ctx, node); + break; + + case GGML_OP_SET_ROWS: + ggml_et_op_set_rows(dev_ctx, node); + break; + + case GGML_OP_FILL: + ggml_et_op_fill(dev_ctx, node); + break; + + case GGML_OP_DIAG: + ggml_et_op_diag(dev_ctx, node); + break; + + case GGML_OP_TRI: + ggml_et_op_tri(dev_ctx, node); + break; + + case GGML_OP_SOLVE_TRI: + ggml_et_op_solve_tri(dev_ctx, node); + break; + + case GGML_OP_SET: + ggml_et_op_set(dev_ctx, node); + break; + + case GGML_OP_RWKV_WKV6: + ggml_et_op_rwkv_wkv6(dev_ctx, node); + break; + + case GGML_OP_RWKV_WKV7: + ggml_et_op_rwkv_wkv7(dev_ctx, node); + break; + + case GGML_OP_GATED_DELTA_NET: + ggml_et_op_gated_delta_net(dev_ctx, node); + break; + + default: + ggml_et_uberkernel_abort_graph(&dev_ctx->uberkernel); + GGML_LOG_ERROR("ET: Unsupported operation in graph: %s", ggml_op_name(node->op)); + return GGML_STATUS_FAILED; + } + + if (ggml_et_uberkernel_failed(&dev_ctx->uberkernel)) { + ggml_et_uberkernel_abort_graph(&dev_ctx->uberkernel); + return GGML_STATUS_FAILED; + } + } + + if (!ggml_et_uberkernel_end_graph(dev_ctx)) { + ggml_et_uberkernel_abort_graph(&dev_ctx->uberkernel); + return GGML_STATUS_FAILED; + } + + return GGML_STATUS_SUCCESS; +} + +// Check that elements within each row are contiguous (nb[0] == type_size). +// Higher-dim strides can be arbitrary - kernels navigate them via byte offsets. +static bool et_ggml_is_row_contiguous(const ggml_tensor * t) { + return t->nb[0] == ggml_type_size(t->type); +} + +static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + GGML_UNUSED(dev); + + bool supported = false; + switch (op->op) { + case GGML_OP_CUMSUM: + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + op->src[0]->nb[0] == sizeof(float) && ggml_is_contiguous(op); + break; + case GGML_OP_SQR: + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + op->ne[0] % 16 == 0 && ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]); + break; + case GGML_OP_SUM_ROWS: + // dst has ne[0]=1, src0 row length must be cache-aligned + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + op->src[0]->ne[0] % 16 == 0 && ggml_is_contiguous(op->src[0]); + break; + case GGML_OP_MEAN: + // Kernel handles arbitrary ne00 (per-row alignment guard with + // scalar tail), so no row-length divisibility constraint here. + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op->src[0]); + break; + case GGML_OP_CLAMP: + // Element-wise; kernel distributes by cache lines and handles a + // scalar tail, so any contiguous F32 size is fine - including the + // 1x1x1x1 scalar case. + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]); + break; + case GGML_OP_UNARY: + // Only require dim-0 contiguity (nb[0] == sizeof(float)). Higher + // dims may be arbitrarily strided views; the kernel walks per-row + // using all four nb[] values. See unary_f32.c entry_point. + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_nelements(op) % 16 == 0 && op->nb[0] == sizeof(float) && op->src[0]->nb[0] == sizeof(float)) { + switch (ggml_get_unary_op(op)) { + case GGML_UNARY_OP_ABS: + case GGML_UNARY_OP_SGN: + case GGML_UNARY_OP_NEG: + case GGML_UNARY_OP_STEP: + case GGML_UNARY_OP_TANH: + case GGML_UNARY_OP_ELU: + case GGML_UNARY_OP_RELU: + case GGML_UNARY_OP_SIGMOID: + case GGML_UNARY_OP_GELU: + case GGML_UNARY_OP_GELU_QUICK: + case GGML_UNARY_OP_SILU: + case GGML_UNARY_OP_HARDSWISH: + case GGML_UNARY_OP_HARDSIGMOID: + case GGML_UNARY_OP_EXP: + case GGML_UNARY_OP_EXPM1: + case GGML_UNARY_OP_SOFTPLUS: + case GGML_UNARY_OP_GELU_ERF: + case GGML_UNARY_OP_FLOOR: + case GGML_UNARY_OP_CEIL: + case GGML_UNARY_OP_ROUND: + case GGML_UNARY_OP_TRUNC: + supported = true; + break; + default: + break; + } + } + break; + case GGML_OP_MUL: + case GGML_OP_ADD: + case GGML_OP_SUB: + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && + op->src[1]->type == GGML_TYPE_F32 && op->nb[0] == sizeof(float) && + op->src[0]->nb[0] == sizeof(float) && + (op->src[1]->nb[0] == sizeof(float) || op->src[1]->ne[0] == 1) && + op->nb[1] == op->ne[0] * sizeof(float); + break; + case GGML_OP_MUL_MAT: + // Support Q8_0 x F32 -> F32, F16 x F32 -> F32, F16 x F16 -> F32, and F32 x F32 -> F32 matrix multiplication + // Stride requirements: first dimension must be contiguous for all tensors + if (op->type == GGML_TYPE_F32 && + ((op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32) || + (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16)) && + op->ne[0] % 16 == 0 && // dst row length for tensor-store path + op->src[0]->ne[1] % 16 == 0 && // m + op->src[0]->ne[0] % 16 == 0 && // k + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) { + // Special path for the FP32 TensorFMA kernel + // Limitation - generic kernels can tolerate non-cache-aligned dst rows + // because they publish each output element atomically. The matrix + // engine path still uses tiled tensor stores, so keep dst rows aligned. + // The m edge is difficult to do because of the 4 conseqtive load hardware limitation + // And the k edge is impossible because that is encoded as `stride & 0xFFFFFFFFFFC0ULL` which becomes 0 for stride 16 (4x FP32) :( + // FIXME: Right now this overwrites the mul_mat_f32 kernel - whatever. Fix later. Demo code + supported = true; + } else if (op->type == GGML_TYPE_F32 && op->src[0] && + (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) && op->src[1] && + (op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32)) { + // Check first dimension contiguity requirements + bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type)); + bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type)); + bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float)); + + // Check destination stride ordering (only for dimensions with ne > 1) + bool dst_properly_ordered = true; + for (int d = 0; d < 3; d++) { + if (op->ne[d] > 1 && op->ne[d + 1] > 1 && op->nb[d] > op->nb[d + 1]) { + dst_properly_ordered = false; + } + } + + supported = src0_first_dim_contiguous && src1_first_dim_contiguous && dst_first_dim_contiguous && + dst_properly_ordered; + } else if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_Q8_0 && op->src[1] && + op->src[1]->type == GGML_TYPE_F32) { + // Keep the existing quantized path constraints separate from the + // relaxed non-quant generic fallback. + bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type)); + bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type)); + bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float)); + + bool dst_properly_ordered = true; + for (int d = 0; d < 3; d++) { + if (op->ne[d] > 1 && op->ne[d + 1] > 1 && op->nb[d] > op->nb[d + 1]) { + dst_properly_ordered = false; + } + } + + supported = src0_first_dim_contiguous && src1_first_dim_contiguous && dst_first_dim_contiguous && + dst_properly_ordered; + + } else if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_Q4_0 && op->src[1] && + op->src[1]->type == GGML_TYPE_F32) { + // Keep the existing quantized path constraints separate from the + // relaxed non-quant generic fallback. + bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type)); + bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type)); + bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float)); + + bool dst_properly_ordered = true; + for (int d = 0; d < 3; d++) { + if (op->ne[d] > 1 && op->ne[d + 1] > 1 && op->nb[d] > op->nb[d + 1]) { + dst_properly_ordered = false; + } + } + + supported = src0_first_dim_contiguous && src1_first_dim_contiguous && dst_first_dim_contiguous && + dst_properly_ordered; + } else { + supported = false; + } + break; + case GGML_OP_MUL_MAT_ID: + // Support MUL_MAT_ID for Mixture of Experts: (Q8_0/Q4_0/F16/F32) x F32 -> F32 with I32 expert indices + // src0 (as): [K, M, n_expert] - expert weight matrices (can be quantized) + // src1 (b): [K, n_expert_used, batch] - activations (F32) + // src2 (ids): [n_expert_used, batch] - expert selection indices (I32) + // dst: [M, n_expert_used, batch, 1] - output (F32) + if (op->type == GGML_TYPE_F32 && op->src[0] && + (op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q4_0 || + op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) && + op->src[1] && op->src[1]->type == GGML_TYPE_F32 && op->src[2] && op->src[2]->type == GGML_TYPE_I32) { + // Check first dimension contiguity requirements (matching CPU backend) + bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type)); + bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type)); + bool src2_first_dim_contiguous = (op->src[2]->nb[0] == ggml_type_size(op->src[2]->type)); + bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float)); + + // Check destination stride ordering (only for dimensions with ne > 1) + bool dst_properly_ordered = true; + for (int d = 0; d < 3; d++) { + if (op->ne[d] > 1 && op->ne[d + 1] > 1 && op->nb[d] > op->nb[d + 1]) { + dst_properly_ordered = false; + } + } + + // Validate tensor dimension constraints from GGML definition + bool dims_valid = (op->src[0]->ne[3] == 1) && // as is 3d (one matrix per expert) + (op->src[1]->ne[3] == 1) && // b is 3d + (op->src[2]->ne[2] == 1 && op->src[2]->ne[3] == 1) && // ids is 2d + (op->src[2]->ne[1] == op->src[1]->ne[2]) && // must have expert list per b row + (op->src[0]->ne[0] == op->src[1]->ne[0]) && // K dimension must match + (op->src[2]->ne[0] % op->src[1]->ne[1] == 0); // can broadcast + + supported = src0_first_dim_contiguous && src1_first_dim_contiguous && src2_first_dim_contiguous && + dst_first_dim_contiguous && dst_properly_ordered && dims_valid; + } else { + supported = false; + } + break; + case GGML_OP_ROPE: + // Support F32 x I32 -> F32 RoPE for the modes implemented by rope_f32. + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && + op->src[1]->type == GGML_TYPE_I32 && ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0])) { + const int mode = ggml_get_op_params_i32(op, 2); + const int ndims = ggml_get_op_params_i32(op, 1); + const bool is_normal = mode == GGML_ROPE_TYPE_NORMAL; + const bool is_neox = mode == GGML_ROPE_TYPE_NEOX; + const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; + const bool zero_view_offset = op->src[0]->view_src == nullptr || op->src[0]->view_offs == 0; + const bool has_sections = ggml_get_op_params_i32(op, 11) > 0 || ggml_get_op_params_i32(op, 12) > 0 || + ggml_get_op_params_i32(op, 13) > 0; + // FIXME: support ggml_rope_set_offset + const bool zero_rot_offset = ggml_get_op_params_i32(op, 15) == 0; + + supported = + zero_view_offset && zero_rot_offset && ndims <= 512 && + (is_normal || (is_neox && ndims % 16 == 0) || (is_imrope && ndims % 16 == 0 && has_sections)); + } else { + supported = false; + } + break; + case GGML_OP_RMS_NORM: + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + op->ne[0] % 16 == 0 && ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0]); + break; + case GGML_OP_NORM: + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + op->ne[0] % 16 == 0 && ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0]); + break; + case GGML_OP_L2_NORM: + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + op->ne[0] % 16 == 0 && ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0]); + break; + case GGML_OP_GROUP_NORM: + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0]) && + ggml_get_op_params_i32(op, 0) > 0; + break; + case GGML_OP_IM2COL: + supported = op->src[0] && op->src[1] && + ((op->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32) || + (op->type == GGML_TYPE_F16 && + (op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32))) && + ggml_is_contiguous(op) && ggml_is_contiguous(op->src[1]) && + op->nb[0] == ggml_type_size(op->type) && op->src[1]->nb[0] == ggml_type_size(op->src[1]->type); + break; + case GGML_OP_CONV_2D: + { + // First-cut conv_2d_f32_me kernel constraints. Anything outside + // this falls back to CPU (it's a strict subset on purpose). + if (!op->src[0] || !op->src[1]) { + supported = false; + break; + } + if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || + op->src[1]->type != GGML_TYPE_F32) { + supported = false; + break; + } + if (!ggml_is_contiguous(op) || !ggml_is_contiguous(op->src[0]) || !ggml_is_contiguous(op->src[1])) { + supported = false; + break; + } + + const ggml_tensor * flt = op->src[0]; // [Kw, Kh, Cin, Cout] + const ggml_tensor * in = op->src[1]; // [W, H, Cin, N] + const int32_t s0 = ggml_get_op_params_i32(op, 0); + const int32_t s1 = ggml_get_op_params_i32(op, 1); + const int32_t p0 = ggml_get_op_params_i32(op, 2); + const int32_t p1 = ggml_get_op_params_i32(op, 3); + const int32_t d0 = ggml_get_op_params_i32(op, 4); + const int32_t d1 = ggml_get_op_params_i32(op, 5); + + const int64_t Kw = flt->ne[0]; + const int64_t Kh = flt->ne[1]; + const int64_t Cin = flt->ne[2]; + const int64_t Cout = flt->ne[3]; + const int64_t H = in->ne[1]; + (void) in->ne[0]; + + if (s0 < 1 || s1 < 1 || !(d0 == 1 && d1 == 1) || Cin % 16 != 0 || Cout % 16 != 0 || in->ne[3] != 1) { + supported = false; + break; + } + const int64_t OW = op->ne[0]; + const int64_t OH = op->ne[1]; + if (OW <= 0 || OH <= 0) { + supported = false; + break; + } + (void) p0; + (void) p1; + + // Mirror the kernel's sizing: + // if K_TILES * per_KT_bytes <= budget: 1 buffer, n_chunks=1 + // else: 2 buffers (double-buffer), shrink chunk_KT until + // 2*chunk_KT*per_KT_bytes <= budget. + const int64_t Hp = H + 2 * p1; + const int64_t OW_pad = (OW + 15) & ~15; + const int64_t Wp_a = OW_pad; + const bool need_stage = (OW % 16 != 0); + const int64_t stage_bytes = need_stage ? (Cout * OH * OW_pad * 4) : 0; + const int64_t L2SCP_BUDGET = 1500 * 1024; + // Per-hart partial-TenC scratch (mirrors kernel MAX_TILES_PER_HART=2): + // 32 minions x 2 tiles x 1024 bytes = 64 KB per shire. + const int64_t scratch_bytes = 32 * 2 * 16 * 16 * 4; + const int64_t budget = L2SCP_BUDGET - stage_bytes - scratch_bytes; + const int64_t per_KT_bytes = Kh * Kw * Cout * 16 * 4 + Kw * 16 * Hp * Wp_a * 4; + const int64_t K_TILES = Cin / 16; + + int64_t chunk_KT_calc; + int64_t n_chunks_calc; + if (K_TILES * per_KT_bytes <= budget) { + chunk_KT_calc = K_TILES; + n_chunks_calc = 1; + } else { + chunk_KT_calc = K_TILES; + while (chunk_KT_calc > 1 && 2 * chunk_KT_calc * per_KT_bytes > budget) { + chunk_KT_calc--; + } + while (chunk_KT_calc > 1 && K_TILES % chunk_KT_calc != 0) { + chunk_KT_calc--; + } + if (chunk_KT_calc < 1) { + supported = false; + break; + } + n_chunks_calc = K_TILES / chunk_KT_calc; + } + + if (n_chunks_calc > 1) { + const int64_t M_TILES = Cout / 16; + const int64_t w_tiles = (OW + 15) / 16; + const int64_t total_tiles = OH * w_tiles * M_TILES; + // MAX_TILES_PER_HART = 2 (mirrors kernel constant). + const int64_t max_workers = (need_stage ? 32 : 1024) * 2; + if (total_tiles > max_workers) { + supported = false; + break; + } + } + + supported = true; + break; + } + case GGML_OP_SCALE: + // F32 contiguous, total elements must be cache line aligned (16 floats) + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && (ggml_nelements(op) % 16 == 0); + break; + case GGML_OP_GLU: + // Note: we only require row-wise contiguity (ggml_is_contiguous_1) so that + // strided views over a packed up_proj tensor (the common split-GLU layout) + // are accepted. The kernel walks rows via nb[1] strides, so the inner + // dimension just needs to be densely packed. + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_nelements(op) % 16 == 0 && ggml_is_contiguous_1(op) && ggml_is_contiguous_1(op->src[0])) { + // Check GLU variant - support SWIGLU, SWIGLU_OAI, GEGLU, GEGLU_ERF, GEGLU_QUICK, REGLU + ggml_glu_op glu_type = ggml_get_glu_op(op); + const bool supported_variant = glu_type == GGML_GLU_OP_SWIGLU || glu_type == GGML_GLU_OP_SWIGLU_OAI || + glu_type == GGML_GLU_OP_SWIGLU_CLAMP || glu_type == GGML_GLU_OP_GEGLU || + glu_type == GGML_GLU_OP_GEGLU_ERF || + glu_type == GGML_GLU_OP_GEGLU_QUICK || glu_type == GGML_GLU_OP_REGLU; + + if (op->src[1]) { + supported = supported_variant && op->src[1]->type == GGML_TYPE_F32 && + ggml_is_contiguous_1(op->src[1]) && op->src[0]->ne[0] == op->ne[0] && + op->src[1]->ne[0] == op->ne[0]; + } else { + supported = supported_variant && op->src[0]->ne[0] == 2 * op->ne[0]; + } + } else { + supported = false; + } + break; + case GGML_OP_SOFT_MAX: + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && op->src[0]->ne[0] > 1) { + // Check optional mask tensor (F32 only) + if (op->src[1]) { + supported = op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[1]); + if (!supported) { + break; + } + } + // Check optional sinks tensor (F32 only) + if (op->src[2]) { + supported = op->src[2]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[2]); + } else { + supported = true; + } + } else { + supported = false; + } + break; + case GGML_OP_SSM_SCAN: + supported = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op) && op->src[0] && + op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && op->src[1] && + op->src[1]->type == GGML_TYPE_F32 && op->src[2] && op->src[2]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op->src[2]) && op->src[3] && op->src[3]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op->src[3]) && op->src[4] && op->src[4]->type == GGML_TYPE_F32 && + op->src[5] && op->src[5]->type == GGML_TYPE_F32 && op->src[6] && + op->src[6]->type == GGML_TYPE_I32 && ggml_is_contiguous(op->src[6]) && + op->src[1]->nb[0] == sizeof(float) && op->src[4]->nb[0] == sizeof(float) && + op->src[5]->nb[0] == sizeof(float) && + op->src[1]->nb[1] == (size_t) op->src[1]->ne[0] * sizeof(float) && + op->src[4]->nb[1] == (size_t) op->src[4]->ne[0] * sizeof(float) && + op->src[5]->nb[1] == (size_t) op->src[5]->ne[0] * sizeof(float) && + op->src[0]->ne[0] == op->src[4]->ne[0] && op->src[0]->ne[1] == op->src[1]->ne[0] && + op->src[0]->ne[2] == op->src[1]->ne[1] && op->src[1]->ne[2] == op->src[2]->ne[1] && + op->src[1]->ne[3] == op->src[2]->ne[2] && op->src[4]->ne[2] == op->src[1]->ne[2] && + op->src[4]->ne[3] == op->src[1]->ne[3] && ggml_are_same_shape(op->src[4], op->src[5]) && + op->src[6]->ne[0] == op->src[1]->ne[3] && op->src[3]->ne[1] == op->src[1]->ne[1] && + (op->src[3]->ne[0] == 1 || op->src[3]->ne[0] == op->src[0]->ne[0]) && + (op->src[1]->ne[1] % op->src[4]->ne[1] == 0); + break; + case GGML_OP_FLASH_ATTN_EXT: + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && + (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && op->src[2] && + (op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr && + ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0])) { + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(max_bias)); + memcpy(&logit_softcap, (const float *) op->op_params + 2, sizeof(logit_softcap)); + + const ggml_prec prec = ggml_flash_attn_ext_get_prec(op); + + // Mask must be F16 or F32 if present + bool mask_ok = (op->src[3] == nullptr) || (op->src[3]->type == GGML_TYPE_F32) || + (op->src[3]->type == GGML_TYPE_F16); + + // GQA: n_head_q must be a multiple of n_head_kv + const int64_t nhq = op->src[0]->ne[2]; + const int64_t nhk = op->src[1]->ne[2]; + + // K/V row stride must match element size + const size_t k_elem = op->src[1]->type == GGML_TYPE_F16 ? 2 : 4; + const size_t v_elem = op->src[2]->type == GGML_TYPE_F16 ? 2 : 4; + + // Only support matrix engine path (F16 K/V, dk%32==0); + // mask scalar F32 fallback to get baseline perf readings + const bool me_eligible = op->src[1]->type == GGML_TYPE_F16 && op->src[2]->type == GGML_TYPE_F16 && + (op->src[0]->ne[0] % 32) == 0; + + supported = me_eligible && mask_ok && (prec == GGML_PREC_F32 || prec == GGML_PREC_DEFAULT) && + max_bias == 0.0f && logit_softcap == 0.0f && op->src[0]->nb[0] == sizeof(float) && + op->src[1]->nb[0] == k_elem && op->src[2]->nb[0] == v_elem && op->nb[0] == sizeof(float) && + op->src[0]->ne[0] == op->src[1]->ne[0] && // dk matches + op->src[2]->ne[0] == op->ne[0] && // dv matches + op->src[2]->ne[0] <= 512 && // dv limit + op->src[0]->ne[0] <= 512 && // dk limit + nhq % nhk == 0 && // GQA ratio is integer + op->src[0]->ne[1] == op->ne[2] && op->src[0]->ne[2] == op->ne[1] && + op->src[0]->ne[3] == op->ne[3] && op->src[1]->ne[1] == op->src[2]->ne[1] && + op->src[1]->ne[2] == op->src[2]->ne[2] && op->src[1]->ne[3] == op->src[2]->ne[3] && + op->src[0]->ne[3] == op->src[1]->ne[3]; + } else { + supported = false; + } + break; + case GGML_OP_GET_ROWS: + // Support F32/F16/Q4_0/Q8_0/Q4_K data with I32 indices -> F32 output + if (op->type == GGML_TYPE_F32 && op->src[0] && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || + op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || + op->src[0]->type == GGML_TYPE_Q4_K) && + op->src[1] && op->src[1]->type == GGML_TYPE_I32 && ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) { + // Validate dimension constraints from ggml implementation + supported = (op->src[0]->ne[2] == op->src[1]->ne[1]) && (op->src[1]->ne[3] == 1); + } else { + supported = false; + } + break; + case GGML_OP_CONT: + // Support F32->F32 and F16->F16 CONT operations (rearrange non-contiguous to contiguous) + if ((op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && op->src[0] && + op->src[0]->type == op->type && ggml_is_contiguous(op)) { + // Defensive check: ensure dst and src0 are not aliased (separate buffers) + // While GGML design currently guarantees this, check for future robustness + if (op->data && op->src[0]->data && op->data == op->src[0]->data) { + GGML_LOG_WARN("ET: CONT operation detected aliased tensors (dst == src0), unsupported"); + supported = false; + } else { + supported = true; + } + } else { + supported = false; + } + break; + case GGML_OP_CPY: + // CPY copies src[0] data into dst layout (same as CONT for same-type) + // Special path: zero-element tensors (scalars) are accepted as no-ops + if (op->src[0]) { + const int64_t nelements = op->ne[0] * op->ne[1] * op->ne[2] * op->ne[3]; + if (nelements == 0) { + // Zero-element / scalar no-op case - always supported + supported = true; + } else if ((op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && op->src[0]->type == op->type && + ggml_is_contiguous(op)) { + // Same-type with contiguous dst - reuse CONT kernel + if (op->data && op->src[0]->data && op->data == op->src[0]->data) { + GGML_LOG_WARN("ET: CPY operation detected aliased tensors, unsupported"); + supported = false; + } else { + supported = true; + } + } else if (op->type == GGML_TYPE_F16 && op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op)) { + // F32 -> F16 conversion copy + supported = true; + } else { + supported = false; + } + } else { + supported = false; + } + break; + case GGML_OP_CONCAT: + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && + op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op)) { + const int32_t dim = ((const int32_t *) op->op_params)[0]; + if (dim == 0 && op->src[0]->ne[0] % 16 == 0 && op->src[1]->ne[0] % 16 == 0 && + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) { + // Fast dim==0 path: both source row segments are cacheline-aligned + // and contiguous, so the kernel can use vector row copies. + supported = true; + } else if (dim == 0 && ((op->src[0]->nb[0] % sizeof(float) == 0) || op->src[0]->ne[0] == 1) && + ((op->src[1]->nb[0] % sizeof(float) == 0) || op->src[1]->ne[0] == 1)) { + // Slow dim==0 path: scalar, stride-aware copies for non-contiguous + // or non-aligned source row segments. Destination remains contiguous. + supported = true; + } else if (op->ne[0] % 16 == 0 && op->src[0]->ne[0] % 16 == 0 && op->src[1]->ne[0] % 16 == 0 && + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) { + // Dim >= 1 path: full aligned row copies from one source or the other. + supported = true; + } + } + break; + case GGML_OP_SSM_CONV: + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && + op->src[1]->type == GGML_TYPE_F32 && op->src[0]->nb[0] == sizeof(float) && + op->src[1]->nb[0] == sizeof(float) && op->src[0]->nb[1] == op->src[0]->ne[0] * sizeof(float) && + op->src[1]->nb[1] == op->src[1]->ne[0] * sizeof(float) && ggml_is_contiguous(op) && + op->src[1]->ne[1] == op->src[0]->ne[1] && op->ne[0] == op->src[0]->ne[1] && + op->ne[1] == op->src[0]->ne[0] - op->src[1]->ne[0] + 1 && op->ne[2] == op->src[0]->ne[2]; + break; + case GGML_OP_PAD: + // F32 zero-pad only, no dim0 padding, dst contiguous + // ne[0] must be CL-aligned (% 16 == 0) or evenly divide a CL (16 % ne[0] == 0) + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op) && (op->ne[0] % 16 == 0 || 16 % op->ne[0] == 0) && + op->src[0]->nb[0] == sizeof(float)) { + const int32_t lp0 = ((const int32_t *) op->op_params)[0]; + const int32_t rp0 = ((const int32_t *) op->op_params)[1]; + const bool circular = (bool) ((const int32_t *) op->op_params)[8]; + if (lp0 == 0 && rp0 == 0 && !circular) { + supported = true; + } else { + supported = false; + } + } else { + supported = false; + } + break; + case GGML_OP_REPEAT: + // Two acceptable shapes: + // 1. No-op REPEAT (src and dst have identical shape): dispatched + // to cont_f32, which handles arbitrary contiguous sizes. + // 2. Real REPEAT via repeat_f32 kernel: dst ne[0] cacheline-aligned, + // src0 ne[0] cacheline-aligned or 1, dst.ne[i] % src0.ne[i] == 0. + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && ggml_are_same_shape(op->src[0], op)) { + supported = true; + } else if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + (op->src[0]->ne[0] == 1 || op->src[0]->ne[0] % 16 == 0) && op->ne[0] % 16 == 0 && + ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && op->ne[0] % op->src[0]->ne[0] == 0 && + op->ne[1] % op->src[0]->ne[1] == 0 && op->ne[2] % op->src[0]->ne[2] == 0 && + op->ne[3] % op->src[0]->ne[3] == 0) { + supported = true; + } else { + supported = false; + } + break; + case GGML_OP_FILL: + // F32 contiguous, ne[0] cacheline-aligned for SIMD fill + supported = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op) && op->ne[0] % 16 == 0; + break; + case GGML_OP_DIAG: + // F32 contiguous dst, src0 is 1D vector [N,1,...], dst is [N,N,...] + // ne[0] must be cacheline-aligned for SIMD zeroing + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + op->ne[0] % 16 == 0 && op->ne[0] == op->ne[1] && op->src[0]->ne[0] == op->ne[0] && + op->src[0]->ne[1] == 1 && ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]); + break; + case GGML_OP_TRI: + // F32 contiguous, same shape in/out + // Kernel handles arbitrary ne[0] with aligned fast path + scalar fallback + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]); + break; + case GGML_OP_SOLVE_TRI: + // F32 contiguous, A square, shapes compatible + // Only lower-triangular left-side non-unit variant + // Require k % 16 == 0 for cache-line-safe column parallelism + supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && + op->src[1]->type == GGML_TYPE_F32 && op->src[0]->ne[0] == op->src[0]->ne[1] && + op->src[0]->ne[1] == op->src[1]->ne[1] && op->src[1]->ne[0] % 16 == 0 && + ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); + break; + case GGML_OP_SET: + // Minimal useful support: inplace F32 SET of a contiguous src1 view into + // a contiguous dst/base tensor using explicit destination view strides. + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && + op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op->src[1]) && ggml_are_same_shape(op, op->src[0]) && op->src[1]->ne[0] % 16 == 0) { + const bool inplace = (bool) ((const int32_t *) op->op_params)[4]; + const size_t nb1 = ((const int32_t *) op->op_params)[0]; + const size_t nb2 = ((const int32_t *) op->op_params)[1]; + const size_t nb3 = ((const int32_t *) op->op_params)[2]; + const size_t offset = ((const int32_t *) op->op_params)[3]; + const size_t nb0 = ggml_element_size(op); + const size_t im0 = op->src[1]->ne[0] == 0 ? 0 : op->src[1]->ne[0] - 1; + const size_t im1 = op->src[1]->ne[1] == 0 ? 0 : op->src[1]->ne[1] - 1; + const size_t im2 = op->src[1]->ne[2] == 0 ? 0 : op->src[1]->ne[2] - 1; + const size_t im3 = op->src[1]->ne[3] == 0 ? 0 : op->src[1]->ne[3] - 1; + + const bool view_bounds_ok = offset + im0 * nb0 + im1 * nb1 + im2 * nb2 + im3 * nb3 <= ggml_nbytes(op); + + const bool cacheline_aligned = + (nb1 % 64 == 0) && (nb2 % 64 == 0) && (nb3 % 64 == 0) && (offset % 64 == 0); + + supported = inplace && view_bounds_ok && cacheline_aligned; + } + break; + case GGML_OP_RWKV_WKV6: + // F32 contiguous, head_size must be multiple of 8 for vectorization + // 6 sources: k, v, r, tf, td, state + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && + op->src[1]->type == GGML_TYPE_F32 && op->src[2] && op->src[2]->type == GGML_TYPE_F32 && op->src[3] && + op->src[3]->type == GGML_TYPE_F32 && op->src[4] && op->src[4]->type == GGML_TYPE_F32 && op->src[5] && + op->src[5]->type == GGML_TYPE_F32 && op->src[0]->ne[0] % 8 == 0 && // head_size multiple of 8 + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && ggml_is_contiguous(op->src[2]) && + ggml_is_contiguous(op->src[3]) && ggml_is_contiguous(op->src[4]) && ggml_is_contiguous(op->src[5])) { + supported = true; + } else { + supported = false; + } + break; + case GGML_OP_RWKV_WKV7: + // F32 contiguous, head_size must be multiple of 8 for vectorization + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && + op->src[1]->type == GGML_TYPE_F32 && op->src[2] && op->src[2]->type == GGML_TYPE_F32 && op->src[3] && + op->src[3]->type == GGML_TYPE_F32 && op->src[4] && op->src[4]->type == GGML_TYPE_F32 && op->src[5] && + op->src[5]->type == GGML_TYPE_F32 && op->src[6] && op->src[6]->type == GGML_TYPE_F32 && + op->src[2]->ne[0] % 8 == 0 && // head_size multiple of 8 + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && ggml_is_contiguous(op->src[2]) && + ggml_is_contiguous(op->src[3]) && ggml_is_contiguous(op->src[4]) && ggml_is_contiguous(op->src[5]) && + ggml_is_contiguous(op->src[6])) { + supported = true; + } else { + supported = false; + } + break; + case GGML_OP_GATED_DELTA_NET: + // F32, S_v must be multiple of 8 for vectorization + // q, k, v may be row-contiguous with strided higher dimensions. + // g, beta, state stay contiguous. + if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && // q + op->src[1] && op->src[1]->type == GGML_TYPE_F32 && // k + op->src[2] && op->src[2]->type == GGML_TYPE_F32 && // v + op->src[3] && op->src[3]->type == GGML_TYPE_F32 && // g + op->src[4] && op->src[4]->type == GGML_TYPE_F32 && // beta + op->src[5] && op->src[5]->type == GGML_TYPE_F32 && // state + op->src[2]->ne[0] % 8 == 0 && // S_v multiple of 8 + (op->src[3]->ne[0] == 1 || op->src[3]->ne[0] == op->src[2]->ne[0]) && // g is scalar or per-element + op->src[4]->ne[0] == 1 && // beta is scalar per position + et_ggml_is_row_contiguous(op->src[0]) && et_ggml_is_row_contiguous(op->src[1]) && + et_ggml_is_row_contiguous(op->src[2]) && ggml_is_contiguous(op->src[3]) && + ggml_is_contiguous(op->src[4]) && ggml_is_contiguous(op->src[5])) { + supported = true; + } else { + supported = false; + } + break; + case GGML_OP_VIEW: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + case GGML_OP_RESHAPE: + // Metadata-only no-ops, accept any type + supported = true; + break; + case GGML_OP_SET_ROWS: + // Support F32 data with I64 indices -> F16/F32 output (scatter operation) + if (op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && op->src[1]->type == GGML_TYPE_I64 && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && ggml_is_contiguous_rows(op) && + ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous(op->src[1])) { + // Validate dimension constraints from ggml implementation + supported = (op->ne[0] == op->src[0]->ne[0]) && // same number of columns + (op->ne[2] == op->src[0]->ne[2]) && // same batch size + (op->ne[3] == op->src[0]->ne[3]) && // same outer dimension + (op->src[0]->ne[1] == op->src[1]->ne[0]) && // src rows = index count + (op->src[0]->ne[2] % op->src[1]->ne[1] == 0) && // batch constraint + (op->src[0]->ne[3] % op->src[1]->ne[2] == 0) && // outer constraint + (op->src[1]->ne[3] == 1); // indices tensor constraint + } else { + supported = false; + } + break; + case GGML_OP_NONE: + // Always support NONE operations - they represent leaf nodes (parameters, inputs, constants) + // No computation needed, just memory management + supported = true; + break; + default: + supported = false; + break; + } + // if(!supported) { + // ggml_et_dump_operator_metadata(op); + // } + return supported; +} + +static bool ggml_backend_et_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { + GGML_UNUSED(dev); + return buft->iface.get_name == ggml_backend_et_buffer_type_get_name; +} + +static bool ggml_backend_et_device_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + // GET_ROWS (embedding lookup) uses a large weight (tok_embd) that lives on CPU (dev_input). + // The scheduler has no mechanism to cache cross-backend weight copies - it re-copies split + // inputs every graph_compute call. For GET_ROWS this means copying the entire embedding table + // (e.g. 266MB for Llama 3.1 1B) from host to device on every token, just to look up a few rows. + // Keep GET_ROWS on CPU and let the scheduler copy only the small result to the device. + // The other backends either only offload if the tensor lives on device or is large enough to + // justify the copy cost. + if (op->op == GGML_OP_GET_ROWS) { + return false; + } + return true; + + GGML_UNUSED(dev); +} + +static const struct ggml_backend_i ggml_backend_et_i = { + /* .get_name = */ ggml_backend_et_get_name, + /* .free = */ ggml_backend_et_free, + /* .set_tensor_async = */ ggml_backend_et_set_tensor_async, + /* .get_tensor_async = */ ggml_backend_et_get_tensor_async, + /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, + /* .cpy_tensor_async = */ NULL, + /* .synchronize = */ ggml_backend_et_synchronize, + /* .graph_plan_create = */ NULL, + /* .graph_plan_free = */ NULL, + /* .graph_plan_update = */ NULL, + /* .graph_plan_compute = */ NULL, + /* .graph_compute = */ ggml_backend_et_graph_compute, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .graph_optimize = */ NULL, +}; + +static const char * ggml_backend_et_device_get_name(ggml_backend_dev_t dev) { + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context; + return dev_ctx->name.c_str(); +} + +static const char * ggml_backend_et_device_get_description(ggml_backend_dev_t dev) { + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context; + return dev_ctx->desc.c_str(); +} + +static void ggml_backend_et_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context; + // Currently getFreeMemory is not available on a runtime without server. + // For now, report total memory as free. + *free = dev_ctx->total_mem; + *total = dev_ctx->total_mem; +} + +static enum ggml_backend_dev_type ggml_backend_et_device_get_type(ggml_backend_dev_t dev) { + GGML_UNUSED(dev); + return GGML_BACKEND_DEVICE_TYPE_GPU; +} + +static void ggml_backend_et_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { + GGML_UNUSED(dev); + props->name = ggml_backend_et_device_get_name(dev); + props->description = ggml_backend_et_device_get_description(dev); + props->type = ggml_backend_et_device_get_type(dev); + ggml_backend_et_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->device_id = NULL; // No PCI device ID available + props->caps = { + /* .async = */ true, + /* .host_buffer = */ false, + /* .buffer_from_host_ptr = */ false, + /* .events = */ false, + /* .mmap_support = */ true, + }; +} + +static ggml_backend_t ggml_backend_et_device_init_backend(ggml_backend_dev_t dev, const char * params) { + GGML_UNUSED(params); + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context; + return ggml_backend_et_init(dev_ctx->devidx); +} + +static ggml_backend_buffer_type_t ggml_backend_et_device_get_buffer_type(ggml_backend_dev_t dev) { + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context; + return dev_ctx->buftype; +} + +static ggml_backend_buffer_type_t ggml_backend_et_device_get_host_buffer_type(ggml_backend_dev_t dev) { + GGML_UNUSED(dev); + return ggml_backend_cpu_buffer_type(); +} + +static const struct ggml_backend_device_i ggml_backend_et_device_i = { + /* .get_name = */ ggml_backend_et_device_get_name, + /* .get_description = */ ggml_backend_et_device_get_description, + /* .get_memory = */ ggml_backend_et_device_get_memory, + /* .get_type = */ ggml_backend_et_device_get_type, + /* .get_props = */ ggml_backend_et_device_get_props, + /* .init_backend = */ ggml_backend_et_device_init_backend, + /* .get_buffer_type = */ ggml_backend_et_device_get_buffer_type, + /* .get_host_buffer_type = */ ggml_backend_et_device_get_host_buffer_type, + /* .buffer_from_host_ptr = */ NULL, + /* .supports_op = */ ggml_backend_et_device_supports_op, + /* .supports_buft = */ ggml_backend_et_device_supports_buft, + /* .offload_op = */ ggml_backend_et_device_offload_op, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_synchronize = */ NULL, +}; + +/* + Backend Registry. +*/ + +static const char * ggml_backend_et_reg_get_name(ggml_backend_reg_t reg) { + GGML_UNUSED(reg); + return GGML_ET_NAME; +} + +static size_t ggml_backend_et_reg_get_device_count(ggml_backend_reg_t reg) { + ggml_backend_et_reg_ctx * ctx = (ggml_backend_et_reg_ctx *) reg->context; + return ctx->devices.size(); +} + +static ggml_backend_dev_t ggml_backend_et_reg_get_device(ggml_backend_reg_t reg, size_t devidx) { + ggml_backend_et_reg_ctx * ctx = (ggml_backend_et_reg_ctx *) reg->context; + if (devidx >= ctx->devices.size()) { + return nullptr; + } + return ctx->devices[devidx]; +} + +static void * ggml_backend_et_get_proc_address(ggml_backend_reg_t reg, const char * name) { + GGML_UNUSED(reg); + GGML_UNUSED(name); + return nullptr; +} + +static const struct ggml_backend_reg_i ggml_backend_et_reg_i = { + /* .get_name = */ ggml_backend_et_reg_get_name, + /* .get_device_count = */ ggml_backend_et_reg_get_device_count, + /* .get_device = */ ggml_backend_et_reg_get_device, + /* .get_proc_address = */ ggml_backend_et_get_proc_address, +}; + +ggml_backend_reg_t ggml_backend_et_reg(void) { + static ggml_backend_reg_t _reg = []() -> ggml_backend_reg_t { + ggml_backend_et_reg_ctx * ctx = new ggml_backend_et_reg_ctx; + + if (!ggml_et_driver_init()) { + return nullptr; + } + + ggml_backend_reg_t r = new ggml_backend_reg{ + /* .api_version = */ GGML_BACKEND_API_VERSION, + /* .iface = */ ggml_backend_et_reg_i, + /* .context = */ nullptr, // Set later + }; + + std::vector rtids = ggml_et_runtime()->getDevices(); + + for (int i = 0; i < ggml_et_devicelayer()->getDevicesCount(); i++) { + ggml_backend_dev_t dev = new ggml_backend_device{ + /* .iface = */ ggml_backend_et_device_i, + /* .reg = */ r, + /* .context = */ nullptr // Set later + }; + + rt::DeviceId rtid = rtids[i]; + rt::DeviceProperties prop = ggml_et_runtime()->getDeviceProperties(rtid); + + // Create device context. + ggml_backend_et_device_context * dev_ctx = new ggml_backend_et_device_context; + dev_ctx->devidx = i; + dev_ctx->rtid = rtid; + dev_ctx->name = GGML_ET_NAME + std::to_string(i); + dev_ctx->desc = "ET device " + std::to_string(i); + dev_ctx->total_mem = static_cast(prop.memorySize_); + { + const char * env = getenv("GGML_ET_UBERKERNEL"); + dev_ctx->uberkernel_enabled = env && env[0] != '\0' && strcmp(env, "0") != 0; + } + // Add buffer type for device to device context. + ggml_backend_et_buffer_type_context * bufty_ctx = new ggml_backend_et_buffer_type_context; + bufty_ctx->devidx = i; + bufty_ctx->name = GGML_ET_NAME + std::to_string(i); + dev_ctx->buftype = new ggml_backend_buffer_type{ /* .iface = */ ggml_backend_et_buffer_type_i, + /* .device = */ dev, + /* .context = */ bufty_ctx }; + + // Create default stream for ordered execution on this device + dev_ctx->default_stream = ggml_et_runtime()->createStream(rtid); + + dev_ctx->trace_buffer = ggml_et_runtime()->mallocDevice(rtid, ET_TRACE_BUFFER_SIZE); + // Pre-size each slot's host buffers and device-side scratch so the + // first few graph_compute calls don't pay a malloc/grow penalty. + for (auto & slot : dev_ctx->uberkernel.slots) { + slot.insts.reserve(256); + slot.params_blob.reserve(1 << 20); + slot.device_insts_capacity = 256 * sizeof(ggml_et_uberkernel_inst); + slot.device_params_capacity = 1 << 20; + slot.device_insts = ggml_et_runtime()->mallocDevice(rtid, slot.device_insts_capacity); + slot.device_params = ggml_et_runtime()->mallocDevice(rtid, slot.device_params_capacity); + if (slot.device_insts == nullptr) { + slot.device_insts_capacity = 0; + } + if (slot.device_params == nullptr) { + slot.device_params_capacity = 0; + } + } + + dev->context = dev_ctx; + + ctx->devices.push_back(dev); + } + + r->context = ctx; + return r; + }(); + + return _reg; +} + +ggml_guid_t ggml_backend_et_guid(void) { + static ggml_guid guid = { 0x4b, 0xe0, 0x72, 0x88, 0xc0, 0xf6, 0x29, 0xb4, + 0x79, 0x9f, 0x70, 0x68, 0x71, 0x0f, 0x6d, 0xc8 }; + return &guid; +} + +ggml_backend_t ggml_backend_et_init(size_t devidx) { + if (!ggml_et_driver_init()) { + return nullptr; + } + + if (devidx >= (size_t) ggml_backend_et_get_device_count()) { + return nullptr; + } + + ggml_backend_et_context * ctx = new ggml_backend_et_context; + ctx->devidx = (int) devidx; + + ggml_backend_t backend = new ggml_backend{ + /* .guid = */ ggml_backend_et_guid(), + /* .iface = */ ggml_backend_et_i, + /* .device = */ ggml_backend_et_reg_get_device(ggml_backend_et_reg(), devidx), + /* .context = */ ctx, + }; + + return backend; +} + +bool ggml_backend_is_et(ggml_backend_t backend) { + return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_et_guid()); +} + +int ggml_backend_et_get_device_count(void) { + return ggml_backend_et_reg_get_device_count(ggml_backend_et_reg()); +} + +void ggml_backend_et_get_device_description(int devidx, char * description, size_t description_size) { + if (devidx < 0 || devidx >= ggml_backend_et_get_device_count()) { + snprintf(description, description_size, "ET Device %d (invalid)", devidx); + return; + } + + ggml_backend_dev_t dev = ggml_backend_et_reg_get_device(ggml_backend_et_reg(), devidx); + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context; + snprintf(description, description_size, "%s", dev_ctx->desc.c_str()); +} + +void ggml_backend_et_get_device_memory(int devidx, size_t * free, size_t * total) { + if (devidx < 0 || devidx >= ggml_backend_et_get_device_count()) { + *free = 0; + *total = 0; + return; + } + + ggml_backend_dev_t dev = ggml_backend_et_reg_get_device(ggml_backend_et_reg(), devidx); + ggml_backend_et_device_get_memory(dev, free, total); +} + +ggml_backend_buffer_type_t ggml_backend_et_buffer_type(size_t dev_num) { + if (dev_num >= (size_t) ggml_backend_et_get_device_count()) { + return nullptr; + } + + ggml_backend_dev_t dev = ggml_backend_et_reg_get_device(ggml_backend_et_reg(), dev_num); + ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context; + return dev_ctx->buftype; +} + +ggml_backend_buffer_type_t ggml_backend_et_host_buffer_type(void) { + static ggml_backend_buffer_type host_buffer_type = { + /* .iface = */ ggml_backend_et_buffer_type_i, + /* .device = */ nullptr, + /* .context = */ nullptr, + }; + return &host_buffer_type; +} + +GGML_BACKEND_DL_IMPL(ggml_backend_et_reg) diff --git a/ggml/src/ggml-feats.h b/ggml/src/ggml-feats.h new file mode 100644 index 0000000..c45e4ad --- /dev/null +++ b/ggml/src/ggml-feats.h @@ -0,0 +1,177 @@ +#pragma once + +#if defined(__aarch64__) || defined(_M_ARM64) + +#if defined(__linux__) +#include +#include + +#if !defined(HWCAP2_SVE2) +#define HWCAP2_SVE2 (1ULL << 1) +#endif + +#if !defined(HWCAP_FPHP) +#define HWCAP_FPHP (1 << 9) +#endif + +#if !defined(HWCAP_ASIMDHP) +#define HWCAP_ASIMDHP (1 << 10) +#endif + +#if !defined(HWCAP2_I8MM) +#define HWCAP2_I8MM (1ULL << 13) +#endif + +#if !defined(HWCAP2_SVEI8MM) +#define HWCAP2_SVEI8MM (1ULL << 9) +#endif + +#if !defined(HWCAP_ASIMDDP) +#define HWCAP_ASIMDDP (1 << 20) +#endif + +#if !defined(HWCAP_SVE) +#define HWCAP_SVE (1 << 22) +#endif + +#if !defined(HWCAP2_SME) +#define HWCAP2_SME (1ULL << 23) +#endif + +#if !defined(HWCAP2_SME2) +#define HWCAP2_SME2 (1ULL << 37) +#endif + +#if !defined(PR_SVE_GET_VL) +#define PR_SVE_GET_VL 51 +#endif + +#if !defined(PR_SVE_VL_LEN_MASK) +#define PR_SVE_VL_LEN_MASK 0xffff +#endif + +#elif defined(__APPLE__) +#include +#elif defined(_WIN32) +#include + +#if !defined(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43 +#endif + +#if !defined(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46 +#endif + +#if !defined(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47 +#endif + +#if !defined(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE 57 +#endif + +#if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66 +#endif + +#if !defined(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE 67 +#endif + +#if !defined(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SME_INSTRUCTIONS_AVAILABLE 70 +#endif + +#if !defined(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) +#define PF_ARM_SME2_INSTRUCTIONS_AVAILABLE 71 +#endif + +#endif + +typedef struct ggml_feats_arch64_runtime { + bool has_dotprod; + bool has_fp16; + bool has_sve; + bool has_sve2; + bool has_i8mm; + bool has_svei8mm; + bool has_sme; + bool has_sme2; + int sve_cnt; +} ggml_feats_arch64_runtime_t; + +static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) { + ggml_feats_arch64_runtime_t runtime_feat = {}; + +#if defined(__linux__) + const unsigned long hwcap = getauxval(AT_HWCAP); + const unsigned long hwcap2 = getauxval(AT_HWCAP2); + + runtime_feat.has_dotprod = !!(hwcap & HWCAP_ASIMDDP); + runtime_feat.has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);; + runtime_feat.has_sve = !!(hwcap & HWCAP_SVE); + runtime_feat.has_sve2 = !!(hwcap2 & HWCAP2_SVE2); + runtime_feat.has_i8mm = !!(hwcap2 & HWCAP2_I8MM); + runtime_feat.has_svei8mm = !!(hwcap2 & HWCAP2_SVEI8MM); + runtime_feat.has_sme = !!(hwcap2 & HWCAP2_SME); + runtime_feat.has_sme2 = !!(hwcap2 & HWCAP2_SME2); + + if (runtime_feat.has_sve) { + const int vl = prctl(PR_SVE_GET_VL); + if (vl >= 0) { + runtime_feat.sve_cnt = vl & PR_SVE_VL_LEN_MASK; + } + } +#elif defined(__APPLE__) + int oldp = 0; + size_t size = sizeof(oldp); + + if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_dotprod = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_FP16", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_fp16 = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_SVE", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_sve = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_SVE2", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_sve2 = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_i8mm = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_sme = static_cast(oldp); + } + + if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, nullptr, 0) == 0) { + runtime_feat.has_sme2 = static_cast(oldp); + } + + // Apple does not support userspace non-streaming SVE; keep SVE vector length unknown. + runtime_feat.sve_cnt = 0; +#elif defined (_WIN32) + runtime_feat.has_dotprod = IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_fp16 = IsProcessorFeaturePresent(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_sve = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_sve2 = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_i8mm = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_svei8mm = IsProcessorFeaturePresent(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_sme = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0; + runtime_feat.has_sme2 = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0; + + // Windows exposes SVE feature presence, but not the runtime SVE vector length here. + runtime_feat.sve_cnt = 0; +#endif + + return runtime_feat; +} + +#endif // defined(__aarch64__) || defined(_M_ARM64) diff --git a/ggml/src/ggml-hexagon/CMakeLists.txt b/ggml/src/ggml-hexagon/CMakeLists.txt index f3a5835..9444f73 100644 --- a/ggml/src/ggml-hexagon/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/CMakeLists.txt @@ -21,10 +21,11 @@ message(STATUS "hexagon: using ${HEXAGON_SDK_ROOT} and ${HEXAGON_TOOLS_ROOT} for include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake) include(ExternalProject) +find_package(Python3 REQUIRED COMPONENTS Interpreter) -option(GGML_HEXAGON_HTP_DEBUG "ggml-hexagon: enable HTP debug output" OFF) +option(GGML_HEXAGON_HTP_DEBUG "ggml-hexagon: enable HTP debug output" OFF) +set(GGML_HEXAGON_HTP_BUILD_TYPE "Release" CACHE STRING "ggml-hexagon: HTP skel build type (Release, RelWithDebInfo, Debug)") set(GGML_HEXAGON_HTP_CERT "$ENV{HEXAGON_HTP_CERT}" CACHE PATH "ggml-hexagon: enable HTP library signing using certificate") -set(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING "ggml-hexagon: quantize group size (32, 64, or 128)") add_library(htp_iface OBJECT ${CMAKE_CURRENT_BINARY_DIR}/htp_iface_stub.c) @@ -50,6 +51,15 @@ endif() set(TARGET_NAME ggml-hexagon) ggml_add_backend_library(${TARGET_NAME} ggml-hexagon.cpp + htp-common.h + htp-kparams.cpp + htp-kparams.h + htp-opnode.cpp + htp-opnode.h + htp-fusion.cpp + htp-fusion.h + htp-repack.cpp + htp-repack.h htp-drv.cpp htp-drv.h libdl.h @@ -60,30 +70,31 @@ target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ht # Build HTP skels set(HTP_SKELS) +set(HTP_PROJECTS) function(build_htp_skel V) ExternalProject_Add(htp-${V} SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/htp BUILD_ALWAYS ON - BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so CMAKE_ARGS - -DCMAKE_BUILD_TYPE=Release + -DCMAKE_BUILD_TYPE=${GGML_HEXAGON_HTP_BUILD_TYPE} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/htp/cmake-toolchain.cmake -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR} -DHEXAGON_SDK_ROOT=${HEXAGON_SDK_ROOT} -DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_ROOT} -DHEXAGON_HTP_DEBUG=${GGML_HEXAGON_HTP_DEBUG} - -DGGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE=${GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE} -DDSP_VERSION=${V} - -DPREBUILT_LIB_DIR="toolv19_${V}") + -DPython3_EXECUTABLE=${Python3_EXECUTABLE} + -DPREBUILT_LIB_DIR="toolv21_${V}") list(APPEND HTP_SKELS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so) + list(APPEND HTP_PROJECTS htp-${V}) set(HTP_SKELS ${HTP_SKELS} PARENT_SCOPE) + set(HTP_PROJECTS ${HTP_PROJECTS} PARENT_SCOPE) endfunction() -build_htp_skel(v68) -build_htp_skel(v69) build_htp_skel(v73) build_htp_skel(v75) build_htp_skel(v79) build_htp_skel(v81) +build_htp_skel(v85) # Install Hexagon skels required at runtime install(FILES ${HTP_SKELS} TYPE LIB) @@ -104,7 +115,7 @@ if (CMAKE_SYSTEM_NAME MATCHES Windows AND GGML_HEXAGON_HTP_CERT) set(LIBGGML_HTP_CAT ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp.cat) add_custom_target(libggml-htp-cat BYPRODUCTS ${LIBGGML_HTP_CAT} - DEPENDS libggml-htp.inf ${HTP_SKELS} + DEPENDS libggml-htp.inf ${HTP_PROJECTS} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/libggml-htp.inf ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${INF2CAT} /driver:${CMAKE_CURRENT_BINARY_DIR} /os:10_25H2_ARM64 COMMAND ${SIGNTOOL} sign /fd sha256 /f ${GGML_HEXAGON_HTP_CERT} ${LIBGGML_HTP_CAT} diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 3d68b80..cf2e37a 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -1,3 +1,5 @@ +#define _USE_MATH_DEFINES + #include #include #include @@ -6,17 +8,30 @@ #include #include +#include #include #include #include #include #include #include +#include +#include #include #include #include +#include +#include +#include +#include +#include #ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include # include #else # include @@ -24,7 +39,9 @@ #endif #pragma clang diagnostic ignored "-Wnested-anon-types" +#pragma clang diagnostic ignored "-Wlanguage-extension-token" #pragma clang diagnostic ignored "-Wgnu-anonymous-struct" +#pragma clang diagnostic ignored "-Wmicrosoft-enum-value" #include #include @@ -32,42 +49,110 @@ #define GGML_COMMON_IMPL_CPP #include "ggml-backend-impl.h" +#include "ggml-alloc.h" #include "ggml-common.h" #include "ggml-hexagon.h" #include "ggml-impl.h" #include "ggml-quants.h" -#include "op-desc.h" -#include "htp-ops.h" +#include "htp-opnode.h" +#include "htp/htp-ops.h" #include "htp_iface.h" #include "htp-drv.h" +#include "htp-repack.h" +#include "htp-common.h" +#include "htp-kparams.h" +#include "htp-fusion.h" -static size_t opt_ndev = 1; -static size_t opt_nhvx = 0; // use all -static int opt_arch = 0; // autodetect -static int opt_etm = 0; -static int opt_verbose = 0; -static int opt_profile = 0; -static int opt_hostbuf = 1; // hostbuf ON by default -static int opt_use_hmx = 1; // when set, enable HMX; when 0, use HVX only - -// Enable all stages by default -static int opt_opmask = HTP_OPMASK_QUEUE | HTP_OPMASK_COMPUTE; -static int opt_opsync = 0; // synchronous ops -static int opt_opbatch = 1024; // max number of ops in a batch -static int opt_opqueue = 16; // max number of pending batches -static std::regex* opt_opfilter = NULL; // regex of ops to not claim +using intvec = std::vector; +using uintvec = std::vector; +using u32vec = std::vector; + +#define GGML_HEXAGON_MAX_SESSIONS 16 + +struct ggml_hexagon_device_config { + int physical_idx = 0; + int virtual_idx = 0; + int domain_id = 0; + std::string domain_name; + std::string name; -#define HEX_VERBOSE(...) \ - if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__) + std::vector mdev_group; +}; + +static ggml_hexagon_device_config opt_device_configs[GGML_HEXAGON_MAX_SESSIONS]; + +int opt_arch = 0; // autodetect +size_t opt_ndev = 1; +size_t opt_nhvx = 0; // use all +int opt_nhmx = 1; // when set, enable HMX; when 0, use HVX only +size_t opt_vtcm_size = 0; // VTCM size override in bytes; 0 = auto-detect +size_t opt_vmem = HTP_OP_MAX_VMEM_DEFAULT; // max available va space for buffer mappings +int opt_etm = 0; +int opt_verbose = 0; +int opt_profile = 0; // profiling mode (0-disabled, 1-basic, 2-pmu) +bool opt_hostbuf = false; +bool opt_dma64 = false; +int opt_power_mode = HTP_POWER_MODE_BURST; // DCVS/HMX power profile requested at session start + +static const std::pair htp_power_mode_names[] = { + { "low_power_saver", HTP_POWER_MODE_LOW_POWER_SAVER }, + { "power_saver", HTP_POWER_MODE_POWER_SAVER }, + { "high_power_saver", HTP_POWER_MODE_HIGH_POWER_SAVER }, + { "low_balanced", HTP_POWER_MODE_LOW_BALANCED }, + { "balanced", HTP_POWER_MODE_BALANCED }, + { "high_performance", HTP_POWER_MODE_HIGH_PERFORMANCE }, + { "sustained_high_performance", HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE }, + { "burst", HTP_POWER_MODE_BURST }, + { "default", HTP_POWER_MODE_BURST }, // flip this to change the default mode +}; -static inline uint64_t hex_is_aligned(void * addr, uint32_t align) { - return ((size_t) addr & (align - 1)) == 0; +static int htp_power_mode_from_str(const char * s) { + for (auto & e : htp_power_mode_names) { +#if defined(_MSC_VER) + if (_stricmp(s, e.first) == 0) { +#else + if (strcasecmp(s, e.first) == 0) { +#endif + return e.second; + } + } + return -1; } -static inline size_t hex_round_up(size_t n, size_t m) { - return m * ((n + m - 1) / m); +static const char * htp_power_mode_to_str(int mode) { + for (auto & e : htp_power_mode_names) { + if (e.second == mode) { + return e.first; + } + } + return "unknown"; } +size_t opt_mbuf_dyn = 512ul * 1024 * 1024; // max dynamic (compute) buffer size +size_t opt_mbuf_static = 1ul * 1024 * 1024 * 1024; // max static (weight/KV) buffer size +size_t opt_mbuf_total = 0; // total buffer space limit (0 = unconstrained) + +int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) +int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) +int opt_fa_split = 1; // 1 = head-split + team-split when n_kv_heads < N_cores (default), 0 = off +int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported) +int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE +int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction + +// Default PMU events, if profiling with PMU (mode=2) is enabled +// See https://docs.qualcomm.com/doc/80-N2040-60/topic/pmu-events.html +// https://docs.qualcomm.com/doc/80-N2040-61/topic/hvx-pmu-events.html +static u32vec opt_pmu_evt { 0x3, 0x111, 0x100, 0x105, 0x240, 0x256, 0x7D, 0x8C }; + +int opt_opbatch = 2048; // max number of ops in a batch +int opt_opqueue = 8; // max number of pending batches +int opt_optrace = 0; // trace buffer size per thread (0 means default) +int opt_oppoll = 0; // polling for batch completions +int opt_opfusion = 1; // enable/disable op fusion +int opt_rt_repack = 1; // runtime repacking: 0 = disabled, 1 = quantized only (default), 2 = always + +static std::regex* opt_opfilter = NULL; // regex of ops to not claim + static const char * status_to_str(uint32_t status) { switch (status) { case HTP_STATUS_OK: @@ -87,36 +172,135 @@ static const char * status_to_str(uint32_t status) { // ** debug helpers -static void ggml_hexagon_dump_op_exec(const std::string &sess_name, const ggml_tensor * op, const uint32_t req_flags) { +static void ggml_hexagon_dump_op_exec(const char * sess_name, const htp_opnode & node, const uint32_t req_flags) { if (!opt_verbose) return; - op_desc desc(op); - GGML_LOG_DEBUG("ggml-hex: %s execute-op %s: %s : %s : %s : %s : %s : flags 0x%x\n", sess_name.c_str(), - ggml_op_desc(op), desc.names, desc.dims, desc.types, desc.strides, desc.buffs, req_flags); + GGML_LOG_DEBUG("ggml-hex: %s execute-op %s|flags 0x%x\n", + sess_name, htp_format_op(node, HTP_OPFMT_EXEC).c_str(), req_flags); } -static void ggml_hexagon_dump_op_supp(const std::string &sess_name, const struct ggml_tensor * op, bool supp) { +static void ggml_hexagon_dump_op_supp(const char * sess_name, const struct ggml_tensor * op, bool supp) { if (!opt_verbose) return; - op_desc desc(op); - GGML_LOG_DEBUG("ggml-hex: %s supports-op %s : %s : %s : %s : %s : %s : %s\n", sess_name.c_str(), - ggml_op_desc(op), desc.names, desc.dims, desc.types, desc.strides, desc.buffs, supp ? "yes" : "no"); + GGML_LOG_DEBUG("ggml-hex: %s supports-op %s|%s\n", + sess_name, htp_format_op(op, HTP_OPFMT_SUPP).c_str(), supp ? "yes" : "no"); +} + +static const char * htp_event_name(uint16_t id) { + switch (id) { + case HTP_TRACE_EVT_DMA: return "DMA"; + case HTP_TRACE_EVT_HVX_COMP: return "HVX_COMP"; + case HTP_TRACE_EVT_HVX_A_QUANT: return "HVX_A_QUANT"; + case HTP_TRACE_EVT_HVX_A_PREP: return "HVX_A_PREP"; + case HTP_TRACE_EVT_HVX_W_DEQUANT: return "HVX_W_DEQUANT"; + case HTP_TRACE_EVT_HVX_W_PREP: return "HVX_W_PREP"; + case HTP_TRACE_EVT_HVX_O_PROC: return "HVX_O_PROC"; + case HTP_TRACE_EVT_HVX_FA_QK: return "HVX_QK_FA"; + case HTP_TRACE_EVT_HVX_FA_SFM: return "HVX_SFM_FA"; + case HTP_TRACE_EVT_HVX_FA_Q_PREP: return "HVX_Q_PREP"; + case HTP_TRACE_EVT_HVX_FA_K_PREP: return "HVX_K_PREP"; + case HTP_TRACE_EVT_HVX_FA_V_PREP: return "HVX_V_PREP"; + case HTP_TRACE_EVT_HVX_GDN_PREP: return "HVX_GDN_PREP"; + case HTP_TRACE_EVT_HVX_GDN_SOLVE: return "HVX_GDN_SOLVE"; + case HTP_TRACE_EVT_HVX_GDN_V_PREP: return "HVX_GDN_V_PREP"; + case HTP_TRACE_EVT_HVX_GDN_D_PREP: return "HVX_GDN_D_PREP"; + case HTP_TRACE_EVT_HVX_GDN_OUT: return "HVX_GDN_OUT"; + case HTP_TRACE_EVT_HVX_GDN_STATE: return "HVX_GDN_STATE"; + case HTP_TRACE_EVT_HVX_GDN_REM: return "HVX_GDN_REM"; + case HTP_TRACE_EVT_HMX_COMP: return "HMX_COMP"; + case HTP_TRACE_EVT_L2FLUSH: return "L2FLUSH"; + case HTP_TRACE_EVT_INIT: return "INIT"; + case HTP_TRACE_EVT_BUFF: return "BUFF"; + case HTP_TRACE_EVT_FENCE: return "FENCE"; + default: return "UNKNOWN"; + } +} + +static void ggml_hexagon_dump_op_prof(const char * sess_name, const htp_opnode & node, const htp_prof_desc & pd) { + if (!opt_profile) return; + + uint32_t op_usec = pd.usecs; + uint32_t op_cycles = pd.cycles_stop - pd.cycles_start; + const uint32_t * pmu = pd.pmu; + + char pmu_str[256] = ""; + if (opt_profile == 2) { + static_assert(HTP_PROF_PMU_NCNT == 8, "current implementation assumes 8 PMU counters"); + snprintf(pmu_str, sizeof(pmu_str), " pmu [%u,%u,%u,%u,%u,%u,%u,%u]", + pmu[0], pmu[1], pmu[2], pmu[3], pmu[4], pmu[5], pmu[6], pmu[7]); + } + + float mhz = op_usec > 0 ? (float) op_cycles / op_usec : 0.0f; + GGML_LOG_DEBUG("ggml-hex: %s profile-op %s|usec %u cycles %u start %u mhz %.1f%s\n", + sess_name, htp_format_op(node, HTP_OPFMT_PROF).c_str(), op_usec, op_cycles, pd.cycles_start, mhz, pmu_str); } -static void ggml_hexagon_dump_op_prof(const std::string &sess_name, const ggml_tensor * op, - uint32_t op_usec, uint32_t op_cycles, uint32_t op_pkts, uint64_t call_usec) { - if (!opt_profile) return; +static void ggml_hexagon_dump_batch_prof(const char * sess_name, const htp_opbatch_rsp & rsp) { + uint64_t batch_cycles = rsp.cycles_stop - rsp.cycles_start; + float batch_mhz = rsp.usecs > 0 ? (float) batch_cycles / rsp.usecs : 0.0f; + + char evt_str[256] = "----"; + if (opt_profile == 3) { + snprintf(evt_str, sizeof(evt_str), "evt-cnt %u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u", + rsp.n_traces[0], rsp.n_traces[1], rsp.n_traces[2], rsp.n_traces[3], + rsp.n_traces[4], rsp.n_traces[5], rsp.n_traces[6], rsp.n_traces[7], + rsp.n_traces[8], rsp.n_traces[9], rsp.n_traces[10]); + } + + GGML_LOG_DEBUG("ggml-hex: %s profile-op OPBATCH|----|n-ops %u|%s|----|----|usec %u cycles %llu start %llu mhz %.1f\n", + sess_name, rsp.n_ops, evt_str, rsp.usecs, (unsigned long long) batch_cycles, (unsigned long long) rsp.cycles_start, batch_mhz); +} + +static void ggml_hexagon_dump_trace_events(const char * sess_name, const htp_opbatch_rsp & rsp, + const htp_trace_desc * trace_events, uint32_t n_traces) { + if (opt_profile == 3 && trace_events) { + uint32_t valid_cnt[HTP_MAX_NTHREADS + 1] = {0}; + for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) { + uint32_t count = rsp.n_traces[t]; + valid_cnt[t] = count > n_traces ? n_traces : count; + } - op_desc desc(op); - GGML_LOG_DEBUG("ggml-hex: %s profile-op %s: %s : %s : %s : %s : %s : op-usec %u op-cycles %u op-pkts %u (%f) call-usec %llu\n", sess_name.c_str(), - ggml_op_desc(op), desc.names, desc.dims, desc.types, desc.strides, desc.buffs, - op_usec, op_cycles, op_pkts, (float) op_cycles / op_pkts, (unsigned long long) call_usec); + for (uint32_t t = 0; t <= HTP_MAX_NTHREADS; t++) { + for (uint32_t idx = 0; idx < valid_cnt[t]; idx++) { + const auto & e = trace_events[t * n_traces + idx]; + bool is_stop = (e.info & 0x8000) != 0; + uint16_t info = e.info & 0x7FFF; + GGML_LOG_DEBUG("ggml-hex: %s trace-evt %s: thread %u info %u %s %u\n", + sess_name, htp_event_name(e.id), t, info, is_stop ? "stop" : "start", e.cycles); + } + } + } } // ** backend sessions struct ggml_hexagon_opbatch; -struct ggml_hexagon_opshm; +struct ggml_hexagon_opqueue; +struct ggml_hexagon_shared_buffer; +struct ggml_hexagon_fence_buffer; +struct ggml_hexagon_session; +struct ggml_backend_hexagon_device_context; + +struct ggml_hexagon_mdev_group { + uint32_t idx = 0; + uint32_t count = 1; + std::vector> sessions; +}; + +struct ggml_backend_hexagon_comm_context { + std::vector backends; + size_t n_backends = 0; + volatile uint32_t * fence_slots[GGML_HEXAGON_MAX_SESSIONS] = {}; + ggml_tensor fence_tensors[GGML_HEXAGON_MAX_SESSIONS] = {}; +}; + +struct ggml_hexagon_event { + ggml_hexagon_session * sess = nullptr; + ggml_hexagon_session * fence_sess = nullptr; + volatile uint32_t * fence_slot = nullptr; + ggml_tensor fence_tensor = {}; + uint32_t seq = 0; +}; struct ggml_hexagon_session { std::string name; @@ -125,1799 +309,1770 @@ struct ggml_hexagon_session { uint32_t session_id; uint32_t domain_id; uint64_t queue_id; - int dev_id; + int phys_idx; + int virt_idx; bool valid_session; bool valid_handle; bool valid_queue; bool valid_iface; - std::atomic op_pending; - ggml_hexagon_opbatch *op_batch; - ggml_hexagon_opshm *op_shm; + ggml_hexagon_opbatch* op_batch; + ggml_hexagon_opqueue* op_queue; + + std::unordered_map> cloned_buffers; + std::unordered_set virt_peers; + std::unordered_set phys_peers; - ggml_backend_buffer_type buffer_type = {}; - ggml_backend_buffer_type repack_buffer_type = {}; + uint32_t n_threads = 0; + uint32_t n_hvx = 0; + uint32_t n_hmx = 0; + uint64_t vtcm_size = 0; + size_t max_vmem = 0; + uint32_t fence_seq = 0; - ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false); + std::atomic batch_req_seq{0}; + std::atomic batch_rsp_seq{0}; + std::atomic last_error{HTP_STATUS_OK}; + + uint64_t cached_uid = 0; + std::vector cached_nodes; + + ggml_hexagon_mdev_group mdev; + ggml_backend_dev_t dev = nullptr; + ggml_backend_hexagon_device_context * dev_ctx = nullptr; + ggml_hexagon_fence_buffer * fence_buf = nullptr; + + ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev = nullptr, uint32_t mdev_idx = 0, uint32_t mdev_count = 0) noexcept(false); ~ggml_hexagon_session() noexcept(true); const char* c_name() const { return name.c_str(); } - void allocate(int dev_id) noexcept(false); + void allocate(const ggml_hexagon_device_config & config) noexcept(false); void release() noexcept(true); - void enqueue_op(htp_op_code opcode, const ggml_tensor *op); - void flush(bool all = true); + uint8_t * alloc_fence(uint32_t n_slots = 1); + void free_fence(void * ptr, uint32_t n_slots = 1); + ggml_tensor make_fence_tensor(const volatile void * slot, uint32_t n_slots = 1) const; + uint32_t next_fence_seq() { + if (++fence_seq == 0) fence_seq = 1; + return fence_seq; + } + + uint8_t * mdev_fence_slot = nullptr; + std::unordered_map cpy_fence_slots; + + void enqueue_mdev_group(); + void enqueue_op(const htp_opnode & node); + void enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor = nullptr, uint32_t fence_seq = 0); + void enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq = 0, bool wait = true); + void enqueue_allreduce(const ggml_tensor * dst, const std::vector & src_tensors, + const std::vector & sync_tensors, uint32_t rank, uint32_t n_ranks, + uint32_t fence_seq_entry = 0, uint32_t fence_seq_exit = 0); + + void start_batch(); + void flush_sync(bool all = true); + void flush_async(); + void flush_batch(size_t min_ops = 1); + void flush_peers(); + void flush_pending(bool all = true); + + ggml_hexagon_shared_buffer * mmap_tensor(const ggml_tensor * t); + bool clone_buffer(const ggml_hexagon_shared_buffer*); + void release_buffer(const ggml_hexagon_shared_buffer*); + void unclone_buffer(const ggml_hexagon_shared_buffer*); - void flush_pending(bool all = false); - void flush_batch(); + void add_peer(ggml_hexagon_session * peer) { + if (this->phys_idx == peer->phys_idx) { + virt_peers.insert(peer); + } else { + phys_peers.insert(peer); + } + } }; +const char * htp_session_name(const struct ggml_hexagon_session * sess) { return sess->c_name(); } +size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess) { return sess->vtcm_size; } +uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess) { return sess->n_threads; } +uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess) { return sess->n_hmx; } +uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess) { return sess->mdev.count; } + // ** backend buffers +struct ggml_backend_hexagon_device_context { + int dev_id; + ggml_hexagon_device_config config; + ggml_backend_dev_t dev = nullptr; + + ggml_backend_buffer_type buffer_type = {}; + ggml_backend_buffer_type host_buffer_type = {}; + ggml_backend_buffer_type fence_buffer_type = {}; + + std::unique_ptr sess; + + ggml_backend_hexagon_device_context(int dev_id, const ggml_hexagon_device_config & config, ggml_backend_dev_t dev); + ~ggml_backend_hexagon_device_context(); + + const char * c_name() const { return config.name.c_str(); } + + ggml_hexagon_session * session() { + if (!sess) { + sess = std::make_unique(config, dev); + } + return sess.get(); + } +}; + struct ggml_backend_hexagon_buffer_type_context { - ggml_backend_hexagon_buffer_type_context(const std::string & name, ggml_hexagon_session * sess) { - this->sess = sess; - this->name = name; + ggml_backend_hexagon_buffer_type_context(const std::string & name, ggml_backend_hexagon_device_context * dev_ctx) { + this->dev_ctx = dev_ctx; + this->name = name; + } + + ggml_backend_hexagon_device_context * dev_ctx; + std::string name; +}; + +struct ggml_hexagon_rpcmem_block { + uint8_t * base = nullptr; + int fd = -1; + size_t size = 0; + + std::unordered_set mapped_clones; + + ggml_hexagon_rpcmem_block(size_t size) { + base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size); + if (!base) { + throw std::runtime_error("ggml-hex: rpcmem_alloc failed"); + } + fd = rpcmem_to_fd(base); + if (fd < 0) { + rpcmem_free(base); + throw std::runtime_error("ggml-hex: rpcmem_to_fd failed"); + } + this->size = size; } - ggml_hexagon_session * sess; - std::string name; + ~ggml_hexagon_rpcmem_block() { + if (base) { + rpcmem_free(base); + } + } }; struct ggml_hexagon_shared_buffer { - ggml_hexagon_session * sess; - uint8_t * base; - size_t size; - int fd; - bool mapped; - bool pinned; - - void mmap(bool pinned = false) { - int err = fastrpc_mmap(sess->domain_id, this->fd, (void *) this->base, 0, this->size, FASTRPC_MAP_FD_DELAYED); + ggml_hexagon_session * sess; + std::shared_ptr mem; + std::vector tensor_extra; + bool mapped; + bool pinned; + bool extended; + + const char * c_name() const { return sess->c_name(); } + uint8_t * base() const { return mem ? mem->base : nullptr; } + size_t size() const { return mem ? mem->size : 0; } + int fd() const { return mem ? mem->fd : -1; } + + void mmap(bool extended = false) { + if (!this->mem) return; + if (this->mapped) return; + + GGML_ASSERT(!this->pinned || !extended); + + this->extended = extended; + fastrpc_map_flags flags = this->pinned ? FASTRPC_MAP_FD : (extended ? FASTRPC_MAP_FD_DELAYED_EXTENDED : FASTRPC_MAP_FD_DELAYED); + + int err = fastrpc_mmap(sess->domain_id, fd(), (void *) base(), 0, size(), flags); if (err != 0) { GGML_LOG_ERROR("ggml-hex: %s buffer mapping failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(), - sess->domain_id, this->size, this->fd, (unsigned) err); + sess->domain_id, size(), fd(), (unsigned) err); throw std::runtime_error("ggml-hex: fastrpc_mmap failed (see log for details)"); } - if (pinned) { - err = htp_iface_mmap(sess->handle, this->fd, this->size, pinned); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: %s buffer pinning failed : domain_id %d size %zu fd %d error 0x%08x\n", sess->c_name(), - sess->domain_id, this->size, this->fd, (unsigned) err); - throw std::runtime_error("ggml-hex: htp_iface_mmap failed (see log for details)"); - } - } + HEX_VERBOSE("ggml-hex: %s mapped buffer: base %p size %zu fd %d pinned %u extended %u\n", + sess->c_name(), (void *) base(), size(), fd(), pinned, extended); this->mapped = true; - this->pinned = pinned; - HEX_VERBOSE("ggml-hex: %s mapped buffer: base %p size %zu fd %d pinned %u\n", - sess->c_name(), (void *) this->base, this->size, this->fd, pinned); } void unmap() { if (!this->mapped) return; - htp_iface_munmap(sess->handle, this->fd); - fastrpc_munmap(sess->domain_id, this->fd, (void *) this->base, this->size); + if (!this->pinned && mem) { + // HTP might still hold a reference, tell it drop it + htp_iface_munmap(sess->handle, fd()); + } + + if (mem) { + fastrpc_munmap(sess->domain_id, fd(), (void *) base(), size()); + } HEX_VERBOSE("ggml-hex: %s unmapped buffer: base %p size %zu fd %d\n", sess->c_name(), - (void *) this->base, size, this->fd); + (void *) base(), size(), fd()); this->mapped = false; - this->fd = -1; } - void alloc(size_t size, bool pinned = false) { - if (this->base) return; - - this->base = (uint8_t *) rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, size); - if (!this->base) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer : size %zu\n", sess->c_name(), size); - throw std::runtime_error("ggml-hex: rpcmem_alloc failed (see log for details)"); - } + void alloc(size_t size) { + if (this->mem) return; - this->fd = rpcmem_to_fd(this->base); - if (this->fd < 0) { - GGML_LOG_ERROR("ggml-hex: %s failed to get FD for buffer %p\n", sess->c_name(), (void *) this->base); - throw std::runtime_error("ggml-hex: rpcmem_to_fd failed (see log for details)"); - } - this->size = size; + this->mem = std::make_shared(size); HEX_VERBOSE("ggml-hex: %s allocated buffer: base %p size %zu fd %d pinned %d\n", sess->c_name(), - (void *) this->base, this->size, this->fd, (int) pinned); - - mmap(pinned); + (void *) base(), this->size(), fd(), (int) pinned); + if (this->pinned) { + mmap(); + } } void free() { - if (!this->base) return; - unmap(); - rpcmem_free(this->base); - - HEX_VERBOSE("ggml-hex: %s freed buffer: base %p size %zu fd %d\n", sess->c_name(), - (void *) this->base, size, this->fd); - - this->base = NULL; + // The memory is freed when the shared_ptr refcount drops to 0. + HEX_VERBOSE("ggml-hex: %s release ref on buffer: base %p size %zu fd %d\n", sess->c_name(), + (void *) base(), size(), fd()); + this->mem = nullptr; } ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, size_t size, bool pinned = false) { - size += 4 * 1024; // extra page for padding + this->sess = sess; + this->mapped = false; + this->pinned = pinned; + this->extended = false; + + // Size adjustment inside the buffer class: 4K aligned data size + 4K guard page + size_t guard_offset = (size + 4095) & ~4095; + size_t total_size = guard_offset + 4096; + if (!pinned && opt_dma64) { + constexpr size_t extended_align = 2 * 1024 * 1024; + total_size = (total_size + extended_align - 1) & ~(extended_align - 1); + } + + alloc(total_size); + } + // Clone constructor for cross-session mapping + ggml_hexagon_shared_buffer(ggml_hexagon_session * sess, const ggml_hexagon_shared_buffer & other) { this->sess = sess; - this->size = 0; - this->base = nullptr; - this->fd = -1; + this->mem = other.mem; this->mapped = false; - - alloc(size, pinned); + this->pinned = other.pinned; + this->extended = other.extended; } ~ggml_hexagon_shared_buffer() { free(); + for (auto * extra : tensor_extra) { + delete extra; + } + } +}; + +struct ggml_hexagon_fence_buffer : public ggml_hexagon_shared_buffer { + uint32_t slot_count = 0; + uint32_t slot_head = 0; + std::vector free_slots; + ggml_backend_buffer backend_buffer{}; + + ggml_hexagon_fence_buffer(ggml_hexagon_session * sess, ggml_backend_buffer_type_t buft, size_t size) + : ggml_hexagon_shared_buffer(sess, size, false /* pinned */), + slot_count(size / HTP_FENCE_SLOT_SIZE), + slot_head(0) { + backend_buffer.buft = buft; + backend_buffer.context = static_cast(this); + backend_buffer.size = size; + mmap(false); + } + + uint8_t * alloc_slot(uint32_t n_slots = 1) { + uint8_t * ptr = nullptr; + if (n_slots == 1 && !free_slots.empty()) { + uint32_t slot = free_slots.back(); + free_slots.pop_back(); + ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE; + } else if (slot_head + n_slots <= slot_count) { + uint32_t slot = slot_head; + slot_head += n_slots; + ptr = base() + (size_t) slot * HTP_FENCE_SLOT_SIZE; + } + if (ptr) { + memset(ptr, 0, (size_t) n_slots * HTP_FENCE_SLOT_SIZE); + } + return ptr; + } + + void free_slot(void * ptr, uint32_t n_slots = 1) { + if (!ptr) return; + uint32_t slot = ((uint8_t *) ptr - base()) / HTP_FENCE_SLOT_SIZE; + for (uint32_t i = 0; i < n_slots; i++) { + free_slots.push_back(slot + i); + } } }; -static ggml_hexagon_session * ggml_backend_hexagon_buffer_get_sess(ggml_backend_buffer_t buffer) { - return static_cast(buffer->buft->context)->sess; +inline uint8_t * ggml_hexagon_session::alloc_fence(uint32_t n_slots) { + uint8_t * ptr = fence_buf->alloc_slot(n_slots); + GGML_ASSERT(ptr); + return ptr; +} + +inline void ggml_hexagon_session::free_fence(void * ptr, uint32_t n_slots) { + if (fence_buf) { + fence_buf->free_slot(ptr, n_slots); + } +} + +inline ggml_tensor ggml_hexagon_session::make_fence_tensor(const volatile void * slot, uint32_t n_slots) const { + static htp_tensor_extra fence_extra { {}, 0, HTP_TENSOR_FENCE }; + + ggml_tensor t {}; + t.buffer = &this->fence_buf->backend_buffer; + t.extra = &fence_extra; + t.data = const_cast(slot); + t.type = GGML_TYPE_I8; + t.ne[0] = HTP_FENCE_SLOT_SIZE; + t.ne[1] = (int64_t) n_slots; + t.ne[2] = 1; + t.ne[3] = 1; + t.nb[0] = 1; + t.nb[1] = HTP_FENCE_SLOT_SIZE; + t.nb[2] = t.nb[1] * t.ne[1]; + t.nb[3] = t.nb[2]; + t.op = GGML_OP_NONE; + return t; } static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) { auto sbuf = static_cast(buffer->context); + sbuf->sess->unclone_buffer(sbuf); delete sbuf; } static void * ggml_backend_hexagon_buffer_get_base(ggml_backend_buffer_t buffer) { auto sbuf = static_cast(buffer->context); - return sbuf->base; + return sbuf->base(); } static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { auto sbuf = static_cast(buffer->context); auto sess = sbuf->sess; - HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu usage %d\n", sess->c_name(), - tensor->name, (void *) sbuf->base, tensor->data, ggml_nbytes(tensor), (int) buffer->usage); - - if (tensor->view_src != NULL && tensor->view_offs == 0) { - return GGML_STATUS_SUCCESS; // nothing to do for the view - } - - return GGML_STATUS_SUCCESS; -} + HEX_VERBOSE("ggml-hex: %s init-tensor %s : base %p data %p nbytes %zu\n", sess->c_name(), + tensor->name, (void *) sbuf->base(), tensor->data, ggml_nbytes(tensor)); -// ======== Q4x4x2 ==================== -struct x2_q4 { - int v[2]; -}; + auto extra = new htp_tensor_extra(); + sbuf->tensor_extra.push_back(extra); -static x2_q4 unpack_q4(uint8_t v) { - x2_q4 x = { (int) (v & 0x0f) - 8, (int) (v >> 4) - 8 }; - return x; -} + tensor->extra = extra; -static void dump_block_q4_0(const block_q4_0 * b, int i) { - HEX_VERBOSE("ggml-hex: repack q4_0 %d: %d %d %d %d ... %d %d %d %d : %.6f\n", i, unpack_q4(b->qs[0]).v[0], - unpack_q4(b->qs[1]).v[0], unpack_q4(b->qs[2]).v[0], unpack_q4(b->qs[3]).v[0], unpack_q4(b->qs[12]).v[1], - unpack_q4(b->qs[13]).v[1], unpack_q4(b->qs[14]).v[1], unpack_q4(b->qs[15]).v[1], - GGML_FP16_TO_FP32(b->d)); + return GGML_STATUS_SUCCESS; } -static void dump_packed_block_q4x4x2(const uint8_t * v, unsigned int i, size_t k) { - static const int qk = QK_Q4_0x4x2; - const int dblk_size = 8 * 2; // 8x __fp16 - const int qblk_size = qk / 2; // int4 - const int qrow_size = k / 2; // int4 (not padded) +static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size) { + auto extra = (htp_tensor_extra *) tensor->extra; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; - const uint8_t * v_q = v + 0; // quants first - const uint8_t * v_d = v + qrow_size; // then scales + if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + extra->flags |= HTP_TENSOR_WEIGHT; + if (htp_is_tiled_weight_tensor(tensor)) { + extra->flags |= HTP_TENSOR_TILED; + } + } - const uint8_t * q = v_q + i * qblk_size; - const ggml_half * d = (const ggml_half *) (v_d + i * dblk_size); + HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); - HEX_VERBOSE("ggml-hex: repack q4x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\n", i, - unpack_q4(q[0]).v[0], unpack_q4(q[1]).v[0], unpack_q4(q[2]).v[0], unpack_q4(q[3]).v[0], - unpack_q4(q[60]).v[0], unpack_q4(q[61]).v[0], unpack_q4(q[62]).v[0], unpack_q4(q[63]).v[0], - unpack_q4(q[124]).v[0], unpack_q4(q[125]).v[0], unpack_q4(q[126]).v[0], unpack_q4(q[127]).v[0], - GGML_FP16_TO_FP32(d[0]), GGML_FP16_TO_FP32(d[1]), GGML_FP16_TO_FP32(d[2]), GGML_FP16_TO_FP32(d[3])); + if ((extra->flags & HTP_TENSOR_TILED) == 0) { + memcpy((char *) tensor->data + offset, data, size); + return; + } - HEX_VERBOSE("ggml-hex: repack q4x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\n", - i + 1, unpack_q4(q[0]).v[1], unpack_q4(q[1]).v[1], unpack_q4(q[2]).v[1], unpack_q4(q[3]).v[1], - unpack_q4(q[60]).v[1], unpack_q4(q[61]).v[1], unpack_q4(q[62]).v[1], unpack_q4(q[63]).v[1], - unpack_q4(q[124]).v[1], unpack_q4(q[125]).v[1], unpack_q4(q[126]).v[1], unpack_q4(q[127]).v[1], - GGML_FP16_TO_FP32(d[4]), GGML_FP16_TO_FP32(d[5]), GGML_FP16_TO_FP32(d[6]), GGML_FP16_TO_FP32(d[7])); -} + if (offset == 0 && size == ggml_nbytes(tensor) && extra->shadow_buf.empty()) { + htp_repack_tensor_tiled(tensor, data, size); + return; + } -static void unpack_q4_0_quants(uint8_t * qs, const block_q4_0 * x, unsigned int bi) { - static const int qk = QK4_0; + if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { + extra->shadow_buf.resize(ggml_nbytes(tensor)); + } + memcpy(extra->shadow_buf.data() + offset, data, size); + extra->shadow_size += size; - for (unsigned int i = 0; i < qk / 2; ++i) { - const int x0 = (x->qs[i] & 0x0F); - const int x1 = (x->qs[i] >> 4); - qs[bi * qk + i + 0] = x0; - qs[bi * qk + i + qk / 2] = x1; + if (extra->shadow_size >= ggml_nbytes(tensor)) { + htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); + extra->shadow_buf.clear(); + extra->shadow_buf.shrink_to_fit(); + extra->shadow_size = 0; } } -static void pack_q4_0_quants(block_q4_0 * x, const uint8_t * qs, unsigned int bi) { - static const int qk = QK4_0; +static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size) { + auto extra = (htp_tensor_extra *) tensor->extra; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + + HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu usage %d flags 0x%x\n", + sess->c_name(), tensor->name, data, offset, size, (int) buffer->usage, extra->flags); - for (unsigned int i = 0; i < qk / 2; ++i) { - const uint8_t x0 = qs[bi * qk + i + 0]; - const uint8_t x1 = qs[bi * qk + i + qk / 2]; - x->qs[i] = x0 | (x1 << 4); + if ((extra->flags & HTP_TENSOR_TILED) == 0) { + memcpy(data, (const char *) tensor->data + offset, size); + return; } + + htp_unpack_tensor_tiled(data, tensor, offset, size); } -static void repack_row_q4x4x2(uint8_t * y, const block_q4_0 * x, int64_t k) { - static const int qk = QK_Q4_0x4x2; - const int nb = (k + qk - 1) / qk; // number of blocks (padded) - const int nloe = k % qk; // leftovers +static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t buffer, + const struct ggml_tensor * src, + struct ggml_tensor * dst) { + // we might optimize this later, for now take the slow path (ie get/set_tensor) + return false; - const int dblk_size = 8 * 2; // 8x __fp16 - const int qblk_size = qk / 2; // int4 - const int qrow_size = k / 2; // int4 (not padded to blocks) + GGML_UNUSED(buffer); + GGML_UNUSED(src); + GGML_UNUSED(dst); +} - uint8_t * y_q = y + 0; // quants first - uint8_t * y_d = y + qrow_size; // then scales +static void ggml_backend_hexagon_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto extra = (htp_tensor_extra *) tensor->extra; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_block_q4_0(&x[i * 8 + 0], 0); - dump_block_q4_0(&x[i * 8 + 1], 1); - dump_block_q4_0(&x[i * 8 + 2], 2); - dump_block_q4_0(&x[i * 8 + 3], 3); - dump_block_q4_0(&x[i * 8 + 4], 4); - dump_block_q4_0(&x[i * 8 + 5], 5); - dump_block_q4_0(&x[i * 8 + 6], 6); - dump_block_q4_0(&x[i * 8 + 7], 7); + if (ggml_backend_buffer_get_usage(buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + extra->flags |= HTP_TENSOR_WEIGHT; + if (htp_is_tiled_weight_tensor(tensor)) { + extra->flags |= HTP_TENSOR_TILED; } } - // Repack the quants - for (int i = 0; i < nb; i++) { - uint8_t qs[QK_Q4_0x4x2]; // unpacked quants - unpack_q4_0_quants(qs, &x[i * 8 + 0], 0); - unpack_q4_0_quants(qs, &x[i * 8 + 1], 1); - unpack_q4_0_quants(qs, &x[i * 8 + 2], 2); - unpack_q4_0_quants(qs, &x[i * 8 + 3], 3); - unpack_q4_0_quants(qs, &x[i * 8 + 4], 4); - unpack_q4_0_quants(qs, &x[i * 8 + 5], 5); - unpack_q4_0_quants(qs, &x[i * 8 + 6], 6); - unpack_q4_0_quants(qs, &x[i * 8 + 7], 7); - - bool partial = (nloe && i == nb-1); + HEX_VERBOSE("ggml-hex: %s set-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d flags 0x%x\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage, extra->flags); - uint8_t * q = y_q + (i * qblk_size); - for (int j = 0; j < qk / 2; j++) { - q[j] = partial ? (qs[j*2+1] << 4) | qs[j*2+0] : (qs[j+128] << 4) | qs[j+000]; + if ((extra->flags & HTP_TENSOR_TILED) == 0) { + for (size_t i = 0; i < n_copies; i++) { + memcpy((uint8_t *) tensor->data + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); } + return; } - // Repack the scales - // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2) - // the last block is truncated and overridden by the scales. - for (int i = 0; i < nb; i++) { - // Repack the scales - ggml_half * d = (ggml_half *) (y_d + i * dblk_size); - d[0] = x[i * 8 + 0].d; - d[1] = x[i * 8 + 1].d; - d[2] = x[i * 8 + 2].d; - d[3] = x[i * 8 + 3].d; - d[4] = x[i * 8 + 4].d; - d[5] = x[i * 8 + 5].d; - d[6] = x[i * 8 + 6].d; - d[7] = x[i * 8 + 7].d; + if (extra->shadow_buf.size() < ggml_nbytes(tensor)) { + extra->shadow_buf.resize(ggml_nbytes(tensor)); } + for (size_t i = 0; i < n_copies; i++) { + memcpy(extra->shadow_buf.data() + offset + i * stride_tensor, (const uint8_t *) data + i * stride_data, size); + } + extra->shadow_size += n_copies * size; - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_packed_block_q4x4x2(y, i, k); - } + if (extra->shadow_size >= ggml_nbytes(tensor)) { + htp_repack_tensor_tiled(tensor, extra->shadow_buf.data(), extra->shadow_buf.size()); + extra->shadow_buf.clear(); + extra->shadow_buf.shrink_to_fit(); + extra->shadow_size = 0; } } -static void unpack_row_q4x4x2(block_q4_0 * x, const uint8_t * y, int64_t k) { - static const int qk = QK_Q4_0x4x2; - const int nb = (k + qk - 1) / qk; // number of blocks (padded) - const int nloe = k % qk; // leftovers +static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto extra = (htp_tensor_extra *) tensor->extra; + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; - const int dblk_size = 8 * 2; // 8x __fp16 - const int qblk_size = qk / 2; // int4 - const int qrow_size = k / 2; // int4 (not padded to blocks) + HEX_VERBOSE("ggml-hex: %s get-tensor-2d %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, (int) buffer->usage); - const uint8_t * y_q = y + 0; // quants first - const uint8_t * y_d = y + qrow_size; // then scales - - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_packed_block_q4x4x2(y, i, k); + if ((extra->flags & HTP_TENSOR_TILED) == 0) { + for (size_t i = 0; i < n_copies; i++) { + memcpy((uint8_t *)data + i * stride_data, (const uint8_t *)tensor->data + offset + i * stride_tensor, size); } + return; } - // Unpack the quants - for (int i = 0; i < nb; i++) { - uint8_t qs[QK_Q4_0x4x2]; // unpacked quants + size_t temp_size = n_copies > 0 ? (n_copies - 1) * stride_tensor + size : 0; + size_t slice_size = tensor->ne[1] * ggml_row_size(tensor->type, tensor->ne[0]); + size_t slice_offset = offset % slice_size; + size_t row_size_bytes = ggml_row_size(tensor->type, tensor->ne[0]); - bool partial = (nloe && i == nb-1); + GGML_ASSERT((slice_offset % row_size_bytes) == 0 && "offset must be aligned to row boundary"); + GGML_ASSERT((temp_size % row_size_bytes) == 0 && "temp_size must be a multiple of row size"); + GGML_ASSERT((offset + temp_size) <= ggml_nbytes(tensor)); - const uint8_t * q = y_q + (i * qblk_size); - for (int j = 0; j < qk / 2; j++) { - if (partial) { - qs[j*2+0] = q[j] & 0xf; - qs[j*2+1] = q[j] >> 4; - } else { - qs[j+000] = q[j] & 0xf; - qs[j+128] = q[j] >> 4; - } - } + std::vector temp_buf(temp_size); + htp_unpack_tensor_tiled(temp_buf.data(), tensor, offset, temp_size); - pack_q4_0_quants(&x[i * 8 + 0], qs, 0); - pack_q4_0_quants(&x[i * 8 + 1], qs, 1); - pack_q4_0_quants(&x[i * 8 + 2], qs, 2); - pack_q4_0_quants(&x[i * 8 + 3], qs, 3); - pack_q4_0_quants(&x[i * 8 + 4], qs, 4); - pack_q4_0_quants(&x[i * 8 + 5], qs, 5); - pack_q4_0_quants(&x[i * 8 + 6], qs, 6); - pack_q4_0_quants(&x[i * 8 + 7], qs, 7); - } - - // Repack the scales - // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2) - // the last block is truncated and overridden by the scales. - for (int i = 0; i < nb; i++) { - // Unpack the scales - const ggml_half * d = (const ggml_half *) (y_d + i * dblk_size); - x[i * 8 + 0].d = d[0]; - x[i * 8 + 1].d = d[1]; - x[i * 8 + 2].d = d[2]; - x[i * 8 + 3].d = d[3]; - x[i * 8 + 4].d = d[4]; - x[i * 8 + 5].d = d[5]; - x[i * 8 + 6].d = d[6]; - x[i * 8 + 7].d = d[7]; - } - - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_block_q4_0(&x[i * 8 + 0], 0); - dump_block_q4_0(&x[i * 8 + 1], 1); - dump_block_q4_0(&x[i * 8 + 2], 2); - dump_block_q4_0(&x[i * 8 + 3], 3); - dump_block_q4_0(&x[i * 8 + 4], 4); - dump_block_q4_0(&x[i * 8 + 5], 5); - dump_block_q4_0(&x[i * 8 + 6], 6); - dump_block_q4_0(&x[i * 8 + 7], 7); - } + for (size_t i = 0; i < n_copies; i++) { + memcpy((uint8_t *) data + i * stride_data, temp_buf.data() + i * stride_tensor, size); } } -static void init_row_q4x4x2(block_q4_0 * x, int64_t k) { - static const int qk = QK_Q4_0x4x2; - const int nb = (k + qk - 1) / qk; // number of blocks (padded) +static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; + auto sess = sbuf->sess; + HEX_VERBOSE("ggml-hex: %s clear-buff base %p size %zu\n", sess->c_name(), (void *) sbuf->base(), sbuf->size()); + memset(sbuf->base(), value, sbuf->size()); +} - // Init the quants such that they unpack into zeros - uint8_t qs[QK_Q4_0x4x2]; // unpacked quants - memset(qs, 8, sizeof(qs)); +static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = { + /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, + /* .get_base = */ ggml_backend_hexagon_buffer_get_base, + /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor, + /* .memset_tensor = */ NULL, + /* .set_tensor = */ ggml_backend_hexagon_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_hexagon_buffer_get_tensor, + /* .set_tensor_2d = */ ggml_backend_hexagon_buffer_set_tensor_2d, + /* .get_tensor_2d = */ ggml_backend_hexagon_buffer_get_tensor_2d, + /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, + /* .clear = */ ggml_backend_hexagon_buffer_clear, + /* .reset = */ NULL, +}; - for (int i = 0; i < nb; i++) { - pack_q4_0_quants(&x[i * 8 + 0], qs, 0); - pack_q4_0_quants(&x[i * 8 + 1], qs, 1); - pack_q4_0_quants(&x[i * 8 + 2], qs, 2); - pack_q4_0_quants(&x[i * 8 + 3], qs, 3); - pack_q4_0_quants(&x[i * 8 + 4], qs, 4); - pack_q4_0_quants(&x[i * 8 + 5], qs, 5); - pack_q4_0_quants(&x[i * 8 + 6], qs, 6); - pack_q4_0_quants(&x[i * 8 + 7], qs, 7); - } +// ** backend buffer type - // Init the scales - // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2) - // the last block is truncated and overridden by the scales. - for (int i = 0; i < nb; i++) { - // Unpack the scales - x[i * 8 + 0].d = 0; - x[i * 8 + 1].d = 0; - x[i * 8 + 2].d = 0; - x[i * 8 + 3].d = 0; - x[i * 8 + 4].d = 0; - x[i * 8 + 5].d = 0; - x[i * 8 + 6].d = 0; - x[i * 8 + 7].d = 0; - } +static void ggml_backend_hexagon_host_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size) { + memcpy((char *) tensor->data + offset, data, size); + GGML_UNUSED(buffer); } -// repack q4_0 data into q4x4x2 tensor -static void repack_q4_0_q4x4x2(ggml_tensor * t, const void * data, size_t size) { - int64_t nrows = ggml_nrows(t); - - size_t row_size = ggml_row_size(t->type, t->ne[0]); - size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_Q4_0x4x2)); // extra elements for the pad - size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any) +static void ggml_backend_hexagon_host_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, + size_t offset, + size_t size) { + memcpy(data, (const char *) tensor->data + offset, size); + GGML_UNUSED(buffer); +} - // Ensure we don't try to read more data than is available in the source buffer 'data' - // or write more than the tensor can hold. - const size_t total_tensor_size = (size_t)nrows * row_size; - const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size; +static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = { + /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, + /* .get_base = */ ggml_backend_hexagon_buffer_get_base, + /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor, + /* .memset_tensor = */ NULL, + /* .set_tensor = */ ggml_backend_hexagon_host_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_hexagon_host_buffer_get_tensor, + /* .set_tensor_2d = */ NULL, + /* .get_tensor_2d = */ NULL, + /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, + /* .clear = */ ggml_backend_hexagon_buffer_clear, + /* .reset = */ NULL, +}; - // Calculate how many full rows and how many remaining bytes we need to process. - const int64_t n_full_rows = n_bytes_to_copy / row_size; - const size_t n_rem_bytes = n_bytes_to_copy % row_size; +// ** backend buffer type - void * buf_pd = ggml_aligned_malloc(row_size_pd); - GGML_ASSERT(buf_pd != NULL); +static const char * ggml_backend_hexagon_buffer_type_name(ggml_backend_buffer_type_t buffer_type) { + return static_cast(buffer_type->context)->name.c_str(); +} - void * buf_rp = ggml_aligned_malloc(row_size_rp); - GGML_ASSERT(buf_rp != NULL); +static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( + ggml_backend_buffer_type_t buffer_type, size_t size) { + auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; + auto sess = dev_ctx->session(); + if (sess && sess->max_vmem && size > sess->max_vmem) { + GGML_LOG_ERROR("ggml-hex: %s buffer size %zu exceeds max_vmem %zu\n", + dev_ctx->c_name(), size, sess->max_vmem); + return nullptr; + } + try { + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); + return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); + } catch (const std::exception & exc) { + GGML_LOG_ERROR("ggml-hex: %s failed to allocate device buffer context: %s\n", dev_ctx->c_name(), exc.what()); + return nullptr; + } +} - HEX_VERBOSE("ggml-hex: repack-q4_0-q4x4x2 %s : data %p size %zu dims %ldx%ld row-size %zu\n", t->name, data, size, - t->ne[0], nrows, row_size); +static ggml_backend_buffer_t ggml_backend_hexagon_host_buffer_type_alloc_buffer( + ggml_backend_buffer_type_t buffer_type, size_t size) { + auto dev_ctx = static_cast(buffer_type->context)->dev_ctx; + auto sess = dev_ctx->session(); + if (sess && sess->max_vmem && size > sess->max_vmem) { + GGML_LOG_ERROR("ggml-hex: %s host buffer size %zu exceeds max_vmem %zu\n", + dev_ctx->c_name(), size, sess->max_vmem); + return nullptr; + } + try { + ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size, false); + return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_host_buffer_interface, sbuf, size); + } catch (const std::exception & exc) { + GGML_LOG_ERROR("ggml-hex: %s failed to allocate host buffer context: %s\n", dev_ctx->c_name(), exc.what()); + return nullptr; + } +} - init_row_q4x4x2((block_q4_0 *) buf_pd, t->ne[0]); // init padded buffer to make sure the tail is all zeros +static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + return 128; // HVX alignment + GGML_UNUSED(buft); +} - // 1. Process all the full rows - for (int64_t i = 0; i < n_full_rows; i++) { - const uint8_t * src = (const uint8_t *) data + (i * row_size); - uint8_t * dst = (uint8_t *) t->data + (i * row_size); +static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) { + if (!ggml_backend_buft_is_host(buft) && htp_is_tiled_weight_tensor(t)) { + int64_t ne0 = hex_round_up(t->ne[0], 32); + int64_t ne1 = hex_round_up(t->ne[1], 32); + int64_t ne2 = t->ne[2]; + int64_t ne3 = t->ne[3]; + size_t tiled_size = htp_tiled_row_size(t->type, ne0) * ne1 * ne2 * ne3; + return std::max(tiled_size, ggml_nbytes(t)); + } + return ggml_nbytes(t); +} - memcpy(buf_pd, src, row_size); - repack_row_q4x4x2((uint8_t *) buf_rp, (const block_q4_0 *) buf_pd, t->ne[0]); - memcpy(dst, buf_rp, row_size); +static size_t parse_size(const char * str, size_t default_unit = 1024 * 1024) { + if (!str || str[0] == '\0') { + return 0; + } + char * end = NULL; + double val = strtod(str, &end); + if (val < 0) { + return 0; } + if (end && *end) { + while (*end == ' ') end++; + if (*end == 'k' || *end == 'K') { + return (size_t) (val * 1024); + } else if (*end == 'm' || *end == 'M') { + return (size_t) (val * 1024 * 1024); + } else if (*end == 'g' || *end == 'G') { + return (size_t) (val * 1024 * 1024 * 1024); + } + } + return (size_t) (val * default_unit); +} + +static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + return opt_mbuf_dyn; + GGML_UNUSED(buft); +} - // 2. Process the final, potentially partial, row - if (n_rem_bytes > 0) { - const int64_t i = n_full_rows; - const uint8_t * src = (const uint8_t *) data + (i * row_size); - uint8_t * dst = (uint8_t *) t->data + (i * row_size); +static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + return false; + GGML_UNUSED(buft); +} - // re-init the row because we are potentially copying a partial row - init_row_q4x4x2((block_q4_0 *) buf_pd, t->ne[0]); +static bool ggml_backend_hexagon_host_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + return true; + GGML_UNUSED(buft); +} - // Copy only the remaining bytes from the source. - memcpy(buf_pd, src, n_rem_bytes); +struct ggml_backend_hexagon_alloc_buffer_n_plan_item { + size_t size; + int first; + int last; +}; - // Repack the entire buffer - repack_row_q4x4x2((uint8_t *) buf_rp, (const block_q4_0 *) buf_pd, t->ne[0]); +using ggml_backend_hexagon_alloc_buffer_n_plan_t = std::vector; - // Write only the corresponding remaining bytes to the destination tensor. - memcpy(dst, buf_rp, n_rem_bytes); +static const char * ggml_hexagon_kv_layer_suffix(const struct ggml_tensor * t) { + if (strncmp(t->name, "cache_", 6) != 0) { + return NULL; } - - ggml_aligned_free(buf_pd, row_size_pd); - ggml_aligned_free(buf_rp, row_size_rp); + const char * p = strstr(t->name, "_l"); + if (!p || !isdigit((unsigned char)p[2])) { + return NULL; + } + return p; } -// repack q4x4x2 tensor into q4_0 data -static void repack_q4x4x2_q4_0(void * data, const ggml_tensor * t, size_t size) { - int64_t nrows = ggml_nrows(t); - - size_t row_size = ggml_row_size(t->type, t->ne[0]); - size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_Q4_0x4x2)); // extra elements for the pad - size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any) +struct ggml_backend_hexagon_alloc_unit { + size_t size; + int first; + int last; +}; - // Ensure we don't try to copy more data than the tensor actually contains. - const size_t total_tensor_size = (size_t)nrows * row_size; - const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size; +static ggml_backend_hexagon_alloc_buffer_n_plan_t ggml_backend_hexagon_alloc_buffer_n_plan( + ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + ggml_backend_hexagon_alloc_buffer_n_plan_t plan; - // Calculate how many full rows and how many remaining bytes we need to process. - const int64_t n_full_rows = n_bytes_to_copy / row_size; - const size_t n_rem_bytes = n_bytes_to_copy % row_size; + const size_t alignment = ggml_backend_buft_get_alignment(buft); + const size_t max_size = opt_mbuf_static > 0 ? opt_mbuf_static : SIZE_MAX; - void * buf_pd = ggml_aligned_malloc(row_size_pd); - GGML_ASSERT(buf_pd != NULL); + std::vector units; - void * buf_rp = ggml_aligned_malloc(row_size_rp); - GGML_ASSERT(buf_rp != NULL); + int i = 0; + while (i < n_tensors) { + struct ggml_tensor * t = tensors[i]; + size_t unit_size = 0; + int unit_first = i; + int unit_last = i + 1; - HEX_VERBOSE("ggml-hex: repack-q4x4x2-q4_0 %s : data %p size %zu dims %ldx%ld row-size %zu\n", t->name, data, size, - t->ne[0], nrows, row_size); + if (t->data == NULL && t->view_src == NULL) { + unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment); + } - memset(buf_pd, 0, row_size_pd); // clear-out padded buffer to make sure the tail is all zeros + const char * layer_suffix = ggml_hexagon_kv_layer_suffix(t); - // 1. Process all the full rows - for (int64_t i = 0; i < n_full_rows; i++) { - const uint8_t * src = (const uint8_t *) t->data + (i * row_size); - uint8_t * dst = (uint8_t *) data + (i * row_size); + while (unit_last < n_tensors) { + struct ggml_tensor * next = tensors[unit_last]; - memcpy(buf_pd, src, row_size); - unpack_row_q4x4x2((block_q4_0 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]); - memcpy(dst, buf_rp, row_size); - } + if (next->view_src != NULL) { + unit_last++; + continue; + } - // 2. Process the final, potentially partial, row - if (n_rem_bytes > 0) { - const int64_t i = n_full_rows; - const uint8_t * src = (const uint8_t *) t->data + (i * row_size); - uint8_t * dst = (uint8_t *) data + (i * row_size); + if (layer_suffix != NULL) { + const char * next_suffix = ggml_hexagon_kv_layer_suffix(next); + if (next_suffix != NULL && strcmp(layer_suffix, next_suffix) == 0) { + if (next->data == NULL) { + unit_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, next), alignment); + } + unit_last++; + continue; + } + } - // We still need to read and unpack the entire source row because quantization is block-based. - memcpy(buf_pd, src, row_size); - unpack_row_q4x4x2((block_q4_0 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]); + break; + } - // But we only copy the remaining number of bytes to the destination. - memcpy(dst, buf_rp, n_rem_bytes); + units.push_back({ unit_size, unit_first, unit_last }); + i = unit_last; } - ggml_aligned_free(buf_pd, row_size_pd); - ggml_aligned_free(buf_rp, row_size_rp); -} - -// ======== Q8x4x2 ==================== -static void dump_block_q8_0(const block_q8_0 * b, int i) { - HEX_VERBOSE("ggml-hex: repack q8_0 %d: %d %d %d %d ... %d %d %d %d : %.6f\n", i, b->qs[0], b->qs[1], b->qs[2], - b->qs[3], b->qs[28], b->qs[29], b->qs[30], b->qs[31], GGML_FP16_TO_FP32(b->d)); -} + size_t cur_buf_size = 0; + int cur_buf_first = 0; -static void dump_packed_block_q8x4x2(const uint8_t * v, unsigned int i, size_t k) { - static const int qk = QK_Q8_0x4x2; - const int dblk_size = 8 * 2; // 8x __fp16 - const int qblk_size = qk; // int8 - const int qrow_size = k; // int8 (not padded) + for (const auto & unit : units) { + if (unit.size == 0) { + continue; + } - const uint8_t * v_q = v + 0; // quants first - const uint8_t * v_d = v + qrow_size; // then scales + if (cur_buf_size > 0 && (cur_buf_size + unit.size) > max_size) { + plan.push_back({ cur_buf_size, cur_buf_first, unit.first }); + cur_buf_size = 0; + cur_buf_first = unit.first; + } - const uint8_t * q = v_q + i * qblk_size; - const ggml_half * d = (const ggml_half *) (v_d + i * dblk_size); + cur_buf_size += unit.size; + } - HEX_VERBOSE("ggml-hex: repack q8x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\n", i, - q[0], q[1], q[2], q[3], q[60], q[61], q[62], q[63], q[124], q[125], q[126], q[127], - GGML_FP16_TO_FP32(d[0]), GGML_FP16_TO_FP32(d[1]), GGML_FP16_TO_FP32(d[2]), GGML_FP16_TO_FP32(d[3])); + if (cur_buf_size > 0) { + plan.push_back({ cur_buf_size, cur_buf_first, n_tensors }); + } - HEX_VERBOSE("ggml-hex: repack q8x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\n", - i + 1, q[128], q[129], q[130], q[131], q[192], q[193], q[194], q[195], q[252], q[253], q[254], q[255], - GGML_FP16_TO_FP32(d[4]), GGML_FP16_TO_FP32(d[5]), GGML_FP16_TO_FP32(d[6]), GGML_FP16_TO_FP32(d[7])); + return plan; } -static void unpack_q8_0_quants(uint8_t * qs, const block_q8_0 * x, unsigned int bi) { - static const int qk = QK8_0; +static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer_n( + ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + const ggml_backend_hexagon_alloc_buffer_n_plan_t plan = ggml_backend_hexagon_alloc_buffer_n_plan(buft, tensors, n_tensors); - for (unsigned int i = 0; i < qk; ++i) { - qs[bi * qk + i] = x->qs[i]; - } -} + std::vector buffers; + buffers.reserve(plan.size()); -static void pack_q8_0_quants(block_q8_0 * x, const uint8_t * qs, unsigned int bi) { - static const int qk = QK8_0; + for (const auto & item : plan) { + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, item.size); + if (buffer == NULL) { + GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), item.size); + for (ggml_backend_buffer_t b : buffers) { + ggml_backend_buffer_free(b); + } + return NULL; + } + + struct ggml_tallocr tallocr = ggml_tallocr_new(buffer); + + struct ggml_tensor * t_failed = NULL; + for (int j = item.first; j < item.last; j++) { + struct ggml_tensor * t = tensors[j]; + if (t->data == NULL) { + if (t->view_src == NULL) { + if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) { + t_failed = t; + break; + } + } else if (t->buffer == NULL) { + if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) { + t_failed = t; + break; + } + } + } else { + if (t->view_src != NULL && t->buffer == NULL) { + if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) { + t_failed = t; + break; + } + } + } + } + if (t_failed != NULL) { + GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name); + for (ggml_backend_buffer_t b : buffers) { + ggml_backend_buffer_free(b); + } + ggml_backend_buffer_free(buffer); + return NULL; + } - for (unsigned int i = 0; i < qk; ++i) { - x->qs[i] = qs[bi * qk + i]; + buffers.push_back(buffer); } -} - -static void repack_row_q8x4x2(uint8_t * y, const block_q8_0 * x, int64_t k) { - static const int qk = QK_Q8_0x4x2; - const int nb = (k + qk - 1) / qk; // number of blocks (padded) - const int dblk_size = 8 * 2; // 8x __fp16 - const int qblk_size = qk; // int8 - const int qrow_size = k; // int8 (not padded to blocks) - - uint8_t * y_q = y + 0; // quants first - uint8_t * y_d = y + qrow_size; // then scales + if (buffers.empty()) { + return NULL; + } - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_block_q8_0(&x[i * 8 + 0], 0); - dump_block_q8_0(&x[i * 8 + 1], 1); - dump_block_q8_0(&x[i * 8 + 2], 2); - dump_block_q8_0(&x[i * 8 + 3], 3); - dump_block_q8_0(&x[i * 8 + 4], 4); - dump_block_q8_0(&x[i * 8 + 5], 5); - dump_block_q8_0(&x[i * 8 + 6], 6); - dump_block_q8_0(&x[i * 8 + 7], 7); - } + if (buffers.size() == 1) { + return buffers[0]; } - // Repack the quants - for (int i = 0; i < nb; i++) { - uint8_t qs[QK_Q8_0x4x2]; // unpacked quants + return ggml_backend_multi_buffer_alloc_buffer(buffers.data(), buffers.size()); +} - unpack_q8_0_quants(qs, &x[i * 8 + 0], 0); - unpack_q8_0_quants(qs, &x[i * 8 + 1], 1); - unpack_q8_0_quants(qs, &x[i * 8 + 2], 2); - unpack_q8_0_quants(qs, &x[i * 8 + 3], 3); - unpack_q8_0_quants(qs, &x[i * 8 + 4], 4); - unpack_q8_0_quants(qs, &x[i * 8 + 5], 5); - unpack_q8_0_quants(qs, &x[i * 8 + 6], 6); - unpack_q8_0_quants(qs, &x[i * 8 + 7], 7); +static size_t ggml_backend_hexagon_buffer_type_get_alloc_size_n( + ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) { + const ggml_backend_hexagon_alloc_buffer_n_plan_t plan = ggml_backend_hexagon_alloc_buffer_n_plan(buft, tensors, n_tensors); - uint8_t * q = y_q + (i * qblk_size); - for (int j = 0; j < qk; j++) { - q[j] = qs[j]; - } + size_t total = 0; + for (const auto & item : plan) { + total += item.size; } + return total; +} - // Repack the scales - // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2) - // the last block is truncated and overridden by the scales. - for (int i = 0; i < nb; i++) { - // Repack the scales - ggml_half * d = (ggml_half *) (y_d + i * dblk_size); - d[0] = x[i * 8 + 0].d; - d[1] = x[i * 8 + 1].d; - d[2] = x[i * 8 + 2].d; - d[3] = x[i * 8 + 3].d; - d[4] = x[i * 8 + 4].d; - d[5] = x[i * 8 + 5].d; - d[6] = x[i * 8 + 6].d; - d[7] = x[i * 8 + 7].d; - } +static ggml_backend_buffer_type_i ggml_backend_hexagon_buffer_type_interface = { + /* .get_name = */ ggml_backend_hexagon_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_hexagon_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ ggml_backend_hexagon_buffer_type_alloc_buffer_n, + /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ ggml_backend_hexagon_buffer_type_get_alloc_size_n, + /* .is_host = */ ggml_backend_hexagon_buffer_type_is_host, +}; - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_packed_block_q8x4x2(y, i, k); - } - } -} +static ggml_backend_buffer_type_i ggml_backend_hexagon_host_buffer_type_interface = { + /* .get_name = */ ggml_backend_hexagon_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_hexagon_host_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ ggml_backend_hexagon_buffer_type_alloc_buffer_n, + /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ ggml_backend_hexagon_buffer_type_get_alloc_size_n, + /* .is_host = */ ggml_backend_hexagon_host_buffer_type_is_host, +}; -static void unpack_row_q8x4x2(block_q8_0 * x, const uint8_t * y, int64_t k) { - static const int qk = QK_Q8_0x4x2; - const int nb = (k + qk - 1) / qk; // number of blocks (padded) +ggml_backend_hexagon_device_context::ggml_backend_hexagon_device_context(int dev_id, const ggml_hexagon_device_config & config, ggml_backend_dev_t dev) + : dev_id(dev_id), config(config), dev(dev) { + buffer_type.device = dev; + buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; + buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name, this); - const int dblk_size = 8 * 2; // 8x __fp16 - const int qblk_size = qk; // int8 - const int qrow_size = k; // int8 (not padded to blocks) + host_buffer_type.device = dev; + host_buffer_type.iface = ggml_backend_hexagon_host_buffer_type_interface; + host_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-HOST", this); - const uint8_t * y_q = y + 0; // quants first - const uint8_t * y_d = y + qrow_size; // then scales + fence_buffer_type.device = dev; + fence_buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; + fence_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(config.name + "-FENCE", this); +} - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_packed_block_q8x4x2(y, i, k); - } - } +ggml_backend_hexagon_device_context::~ggml_backend_hexagon_device_context() { + delete static_cast(buffer_type.context); + delete static_cast(host_buffer_type.context); + delete static_cast(fence_buffer_type.context); +} - // Unpack the quants - for (int i = 0; i < nb; i++) { - uint8_t qs[QK_Q4_0x4x2]; // unpacked quants +static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) { + return b->buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment; +} - const uint8_t * q = y_q + (i * qblk_size); - for (int j = 0; j < qk; j++) { - qs[j] = q[j]; - } +struct ggml_hexagon_opbatch { + ggml_hexagon_session* sess; - pack_q8_0_quants(&x[i * 8 + 0], qs, 0); - pack_q8_0_quants(&x[i * 8 + 1], qs, 1); - pack_q8_0_quants(&x[i * 8 + 2], qs, 2); - pack_q8_0_quants(&x[i * 8 + 3], qs, 3); - pack_q8_0_quants(&x[i * 8 + 4], qs, 4); - pack_q8_0_quants(&x[i * 8 + 5], qs, 5); - pack_q8_0_quants(&x[i * 8 + 6], qs, 6); - pack_q8_0_quants(&x[i * 8 + 7], qs, 7); - } + std::vector ops; // htp_opnode of ops - // Repack the scales - // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q4_0x4x2) - // the last block is truncated and overridden by the scales. - for (int i = 0; i < nb; i++) { - // Unpack the scales - const ggml_half * d = (const ggml_half *) (y_d + i * dblk_size); - x[i * 8 + 0].d = d[0]; - x[i * 8 + 1].d = d[1]; - x[i * 8 + 2].d = d[2]; - x[i * 8 + 3].d = d[3]; - x[i * 8 + 4].d = d[4]; - x[i * 8 + 5].d = d[5]; - x[i * 8 + 6].d = d[6]; - x[i * 8 + 7].d = d[7]; - } + std::vector h_bufs; // htp buffer descriptors + std::vector h_tens; // htp tensor descriptors + std::vector h_ops; // htp op descriptors - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_block_q8_0(&x[i * 8 + 0], 0); - dump_block_q8_0(&x[i * 8 + 1], 1); - dump_block_q8_0(&x[i * 8 + 2], 2); - dump_block_q8_0(&x[i * 8 + 3], 3); - dump_block_q8_0(&x[i * 8 + 4], 4); - dump_block_q8_0(&x[i * 8 + 5], 5); - dump_block_q8_0(&x[i * 8 + 6], 6); - dump_block_q8_0(&x[i * 8 + 7], 7); - } - } -} + std::unordered_map t_map; // tensor ptr to index + std::unordered_multimap d_map; // tensor data to index -static void init_row_q8x4x2(block_q8_0 * x, int64_t k) { - static const int qk = QK_Q8_0x4x2; - const int nb = (k + qk - 1) / qk; // number of blocks (padded) + unsigned int n_bufs; // num buffers in the batch + unsigned int n_tens; // num tensors ... + unsigned int n_ops; // num ops ... + size_t b_vmem; // sum of non-extended buffer sizes - // Init the quants such that they unpack into zeros - uint8_t qs[QK_Q8_0x4x2]; // unpacked quants - memset(qs, 0, sizeof(qs)); + unsigned int n_bufs_max; + unsigned int n_tens_max; + unsigned int n_ops_max; + size_t b_vmem_max; - for (int i = 0; i < nb; i++) { - pack_q8_0_quants(&x[i * 8 + 0], qs, 0); - pack_q8_0_quants(&x[i * 8 + 1], qs, 1); - pack_q8_0_quants(&x[i * 8 + 2], qs, 2); - pack_q8_0_quants(&x[i * 8 + 3], qs, 3); - pack_q8_0_quants(&x[i * 8 + 4], qs, 4); - pack_q8_0_quants(&x[i * 8 + 5], qs, 5); - pack_q8_0_quants(&x[i * 8 + 6], qs, 6); - pack_q8_0_quants(&x[i * 8 + 7], qs, 7); - } + void reset() { + n_bufs = 0; + n_tens = 0; + n_ops = 0; + b_vmem = 0; - // Init the scales - // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_Q8_0x4x2) - // the last block is truncated and overridden by the scales. - for (int i = 0; i < nb; i++) { - // Unpack the scales - x[i * 8 + 0].d = 0; - x[i * 8 + 1].d = 0; - x[i * 8 + 2].d = 0; - x[i * 8 + 3].d = 0; - x[i * 8 + 4].d = 0; - x[i * 8 + 5].d = 0; - x[i * 8 + 6].d = 0; - x[i * 8 + 7].d = 0; + t_map.clear(); + d_map.clear(); + ops.resize(n_ops_max); } -} -// repack q8_0 data into q8x4x2 tensor -static void repack_q8_0_q8x4x2(ggml_tensor * t, const void * data, size_t size) { - int64_t nrows = ggml_nrows(t); + ggml_hexagon_opbatch(ggml_hexagon_session *sess, size_t batch_size, size_t max_vmem) { + this->sess = sess; - size_t row_size = ggml_row_size(t->type, t->ne[0]); - size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_Q8_0x4x2)); // extra elements for the pad - size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any) + n_bufs_max = HTP_OP_MAX_BUFS; + n_ops_max = batch_size; + n_tens_max = std::min(n_ops_max * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS); - // Ensure we don't try to read more data than is available in the source buffer 'data' - // or write more than the tensor can hold. - const size_t total_tensor_size = (size_t)nrows * row_size; - const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size; + b_vmem_max = max_vmem; - // Calculate how many full rows and how many remaining bytes we need to process. - const int64_t n_full_rows = n_bytes_to_copy / row_size; - const size_t n_rem_bytes = n_bytes_to_copy % row_size; + ops.resize(n_ops_max); - void * buf_pd = ggml_aligned_malloc(row_size_pd); - GGML_ASSERT(buf_pd != NULL); + h_bufs.resize(n_bufs_max); + h_tens.resize(n_tens_max); + h_ops.resize(n_ops_max); - void * buf_rp = ggml_aligned_malloc(row_size_rp); - GGML_ASSERT(buf_rp != NULL); + t_map.reserve(n_tens_max); + d_map.reserve(n_tens_max); - HEX_VERBOSE("ggml-hex: repack-q8_0-q8x4x2 %s : data %p size %zu dims %ldx%ld row-size %zu\n", t->name, data, size, - t->ne[0], nrows, row_size); + GGML_LOG_INFO("ggml-hex: %s op batching: n-bufs %u n-tensors %u n-ops %u vmem %zu\n", + sess->c_name(), n_bufs_max, n_tens_max, n_ops_max, b_vmem_max); - init_row_q8x4x2((block_q8_0 *) buf_pd, t->ne[0]); // init padded buffer to make sure the tail is all zeros + reset(); + } - // 1. Process all the full rows - for (int64_t i = 0; i < n_full_rows; i++) { - const uint8_t * src = (const uint8_t *) data + (i * row_size); - uint8_t * dst = (uint8_t *) t->data + (i * row_size); + bool empty() const { return n_ops == 0; } + bool full() const { return n_ops >= n_ops_max; } - memcpy(buf_pd, src, row_size); - repack_row_q8x4x2((uint8_t *) buf_rp, (const block_q8_0 *) buf_pd, t->ne[0]); - memcpy(dst, buf_rp, row_size); + int find_buffer(int fd) const { + for (unsigned int i = 0; i < n_bufs; i++) { + if ((int) h_bufs[i].fd == fd) { + return (int) i; + } + } + return -1; } - // 2. Process the final, potentially partial, row - if (n_rem_bytes > 0) { - const int64_t i = n_full_rows; - const uint8_t * src = (const uint8_t *) data + (i * row_size); - uint8_t * dst = (uint8_t *) t->data + (i * row_size); + // add buffer and return its index + int add_buffer(ggml_hexagon_shared_buffer * sbuf) { + // Lookup by fd + int bi = find_buffer(sbuf->fd()); + if (bi >= 0) { return bi; } + + // Add new buffer to the batch + GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS); + bi = n_bufs++; - // re-init the row because we are potentially copying a partial row - init_row_q8x4x2((block_q8_0 *) buf_pd, t->ne[0]); + htp_buf_desc &b = h_bufs[bi]; + b.base = (uint64_t) sbuf->base(); + b.fd = sbuf->fd(); + b.size = sbuf->size(); + b.flags = sbuf->extended ? HTP_BUF_EXTENDED : 0; - // Copy only the remaining bytes from the source. - memcpy(buf_pd, src, n_rem_bytes); + if (!sbuf->extended) { + b_vmem += b.size; + } - // Repack the entire buffer - repack_row_q8x4x2((uint8_t *) buf_rp, (const block_q8_0 *) buf_pd, t->ne[0]); + HEX_VERBOSE("ggml-hex: %s add-buffer #%u : fd %d base %p size %zu : vmem %zu\n", sess->c_name(), bi, b.fd, (void*) sbuf->base(), (size_t) b.size, b_vmem); - // Write only the corresponding remaining bytes to the destination tensor. - memcpy(dst, buf_rp, n_rem_bytes); + return bi; } - ggml_aligned_free(buf_pd, row_size_pd); - ggml_aligned_free(buf_rp, row_size_rp); -} + static void init_tensor_shape(htp_tensor & h, const ggml_tensor * t) { + auto extra = (htp_tensor_extra *) t->extra; -// repack q8x4x2 tensor into q8_0 data -static void repack_q8x4x2_q8_0(void * data, const ggml_tensor * t, size_t size) { - int64_t nrows = ggml_nrows(t); + h.type = t->type; + h.ne[0] = t->ne[0]; + h.ne[1] = t->ne[1]; + h.ne[2] = t->ne[2]; + h.ne[3] = t->ne[3]; + + if (extra->flags & HTP_TENSOR_TILED) { + if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_BF16) { + h.type = GGML_TYPE_F16; + h.nb[0] = sizeof(ggml_fp16_t); + } else { + h.nb[0] = t->nb[0]; + } + const int64_t ne0_padded = hex_round_up(t->ne[0], 32); + const int64_t ne1_padded = hex_round_up(t->ne[1], 32); + h.nb[1] = htp_tiled_row_size(t->type, ne0_padded); + h.nb[2] = h.nb[1] * ne1_padded; + h.nb[3] = h.nb[2] * h.ne[2]; + h.size = h.nb[3] * h.ne[3]; + } else { + h.nb[0] = t->nb[0]; + h.nb[1] = t->nb[1]; + h.nb[2] = t->nb[2]; + h.nb[3] = t->nb[3]; + h.size = ggml_nbytes(t); + } - size_t row_size = ggml_row_size(t->type, t->ne[0]); - size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_Q8_0x4x2)); // extra elements for the pad - size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any) + h.flags = extra->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_TILED | HTP_TENSOR_FENCE); + } - // Ensure we don't try to copy more data than the tensor actually contains. - const size_t total_tensor_size = (size_t)nrows * row_size; - const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size; + static bool same_shape(const htp_tensor & a, const htp_tensor & b) { + return (a.type == b.type) && + (a.ne[0] == b.ne[0]) && (a.ne[1] == b.ne[1]) && (a.ne[2] == b.ne[2]) && (a.ne[3] == b.ne[3]) && + (a.nb[0] == b.nb[0]) && (a.nb[1] == b.nb[1]) && (a.nb[2] == b.nb[2]) && (a.nb[3] == b.nb[3]); + } - // Calculate how many full rows and how many remaining bytes we need to process. - const int64_t n_full_rows = n_bytes_to_copy / row_size; - const size_t n_rem_bytes = n_bytes_to_copy % row_size; + // add tensor and return its index + int add_tensor(const ggml_tensor * t) { + // Lookup by tensor ptr + auto it = t_map.find(t); + if (it != t_map.end()) { return it->second; } - void * buf_pd = ggml_aligned_malloc(row_size_pd); - GGML_ASSERT(buf_pd != NULL); + htp_tensor shape {}; + init_tensor_shape(shape, t); - void * buf_rp = ggml_aligned_malloc(row_size_rp); - GGML_ASSERT(buf_rp != NULL); + // Lookup by tensor data and shape + auto range = d_map.equal_range(t->data); + for (auto dit = range.first; dit != range.second; ++dit) { + if (same_shape(h_tens[dit->second], shape)) { + t_map.insert({t, dit->second}); + return dit->second; + } + } - HEX_VERBOSE("ggml-hex: repack-q8x4x2-q8_0 %s : data %p size %zu dims %ldx%ld row-size %zu\n", t->name, data, size, - t->ne[0], nrows, row_size); + auto sbuf = static_cast(t->buffer->context); - memset(buf_pd, 0, row_size_pd); // clear-out padded buffer to make sure the tail is all zeros + // Add new tensor to the batch + int ti = n_tens++; + GGML_ASSERT(n_tens <= n_tens_max); - // 1. Process all the full rows - for (int64_t i = 0; i < n_full_rows; i++) { - const uint8_t * src = (const uint8_t *) t->data + (i * row_size); - uint8_t * dst = (uint8_t *) data + (i * row_size); + t_map.insert({t, ti}); + d_map.insert({t->data, ti}); - memcpy(buf_pd, src, row_size); - unpack_row_q8x4x2((block_q8_0 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]); - memcpy(dst, buf_rp, row_size); - } + uint64_t t_offset = (uint8_t *) t->data - sbuf->base(); - // 2. Process the final, potentially partial, row - if (n_rem_bytes > 0) { - const int64_t i = n_full_rows; - const uint8_t * src = (const uint8_t *) t->data + (i * row_size); - uint8_t * dst = (uint8_t *) data + (i * row_size); + htp_tensor & h = h_tens[ti]; + h = shape; + h.bi = add_buffer(sbuf); + h.ti = ti; + h.data = t_offset; - // We still need to read and unpack the entire source row because quantization is block-based. - memcpy(buf_pd, src, row_size); - unpack_row_q8x4x2((block_q8_0 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]); + HEX_VERBOSE("ggml-hex: %s add-tensor #%u %s : bi %d data %p offset %zu size %zu flags 0x%x : %zu:%zu:%zu:%zu\n", sess->c_name(), + ti, t->name, h.bi, (void*) t->data, (size_t) t_offset, (size_t) h.size, h.flags, + (size_t) h.ne[0], (size_t) h.ne[1], (size_t) h.ne[2], (size_t) h.ne[3]); - // But we only copy the remaining number of bytes to the destination. - memcpy(dst, buf_rp, n_rem_bytes); + return ti; } - ggml_aligned_free(buf_pd, row_size_pd); - ggml_aligned_free(buf_rp, row_size_rp); -} + bool fit_tensors(const htp_opnode & node) const { + // check how much extras we will need + size_t extra_bufs = 0; + size_t extra_vmem = 0; + size_t extra_tens = 0; -// ======== MXFP4x4x2 ==================== -struct x2_mxfp4 { - int v[2]; -}; + const bool check_tens = (n_tens + node.inputs.size() + node.outputs.size()) > n_tens_max; -static x2_mxfp4 unpack_mxfp4(uint8_t v) { - x2_mxfp4 x; - x.v[0] = kvalues_mxfp4[(v & 0x0f)]; - x.v[1] = kvalues_mxfp4[(v >> 4)]; - return x; -} + int seen_bufs[HTP_OP_MAX_BUFS]; + int n_seen_bufs = 0; -static void dump_block_mxfp4(const block_mxfp4 * b, int i) { - HEX_VERBOSE("ggml-hex: repack mxfp4 %d: %d %d %d %d ... %d %d %d %d : %.6f\n", i, unpack_mxfp4(b->qs[0]).v[0], - unpack_mxfp4(b->qs[1]).v[0], unpack_mxfp4(b->qs[2]).v[0], unpack_mxfp4(b->qs[3]).v[0], - unpack_mxfp4(b->qs[12]).v[1], unpack_mxfp4(b->qs[13]).v[1], unpack_mxfp4(b->qs[14]).v[1], - unpack_mxfp4(b->qs[15]).v[1], GGML_E8M0_TO_FP32_HALF(b->e)); -} + auto fit_tensor = [&](const ggml_tensor * t) { + if (!t) return; + if (check_tens) { + if (t_map.count(t)) return; + extra_tens++; + } + + auto sbuf = static_cast(t->buffer->context); + int fd = sbuf->fd(); + if (find_buffer(fd) < 0) { + for (int i = 0; i < n_seen_bufs; i++) { + if (seen_bufs[i] == fd) return; + } + if (n_seen_bufs < HTP_OP_MAX_BUFS) { + seen_bufs[n_seen_bufs++] = fd; + } + if (!sbuf->extended) { + extra_vmem += sbuf->size(); + } + extra_bufs += 1; + } + }; -static void dump_packed_block_mxfp4x4x2(const uint8_t * v, unsigned int i, size_t k) { - static const int qk = QK_MXFP4x4x2; - const int eblk_size = 8 * 1; // 8x E8M0 - const int qblk_size = qk / 2; // int4 - const int qrow_size = k / 2; // int4 (not padded) + for (const auto * src : node.get_inputs()) { + fit_tensor(src); + } + for (const auto * output : node.get_outputs()) { + fit_tensor(output); + } - const uint8_t * v_q = v + 0; // quants first - const uint8_t * v_e = v + qrow_size; // then scales + if ((extra_bufs + n_bufs) > n_bufs_max) return false; + if (check_tens && (extra_tens + n_tens) > n_tens_max) return false; + if ((extra_vmem + b_vmem) > b_vmem_max) return false; - const uint8_t * q = v_q + i * qblk_size; - const uint8_t * e = (const uint8_t *) (v_e + i * eblk_size); + return true; + } - HEX_VERBOSE("ggml-hex: repack mxfp4x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\n", i, - unpack_mxfp4(q[0]).v[0], unpack_mxfp4(q[1]).v[0], unpack_mxfp4(q[2]).v[0], unpack_mxfp4(q[3]).v[0], - unpack_mxfp4(q[60]).v[0], unpack_mxfp4(q[61]).v[0], unpack_mxfp4(q[62]).v[0], unpack_mxfp4(q[63]).v[0], - unpack_mxfp4(q[124]).v[0], unpack_mxfp4(q[125]).v[0], unpack_mxfp4(q[126]).v[0], - unpack_mxfp4(q[127]).v[0], GGML_E8M0_TO_FP32_HALF(e[0]), GGML_E8M0_TO_FP32_HALF(e[1]), - GGML_E8M0_TO_FP32_HALF(e[2]), GGML_E8M0_TO_FP32_HALF(e[3])); + void sync_op_desc(unsigned int n) { + const htp_opnode & node = ops[n]; + htp_op_desc & o = h_ops[n]; - HEX_VERBOSE("ggml-hex: repack mxfp4x4x2-%d: %d %d %d %d ... %d %d %d %d ... %d %d %d %d : %.6f %.6f %.6f %.6f\n", - i + 1, unpack_mxfp4(q[0]).v[1], unpack_mxfp4(q[1]).v[1], unpack_mxfp4(q[2]).v[1], - unpack_mxfp4(q[3]).v[1], unpack_mxfp4(q[60]).v[1], unpack_mxfp4(q[61]).v[1], unpack_mxfp4(q[62]).v[1], - unpack_mxfp4(q[63]).v[1], unpack_mxfp4(q[124]).v[1], unpack_mxfp4(q[125]).v[1], - unpack_mxfp4(q[126]).v[1], unpack_mxfp4(q[127]).v[1], GGML_E8M0_TO_FP32_HALF(e[4]), - GGML_E8M0_TO_FP32_HALF(e[5]), GGML_E8M0_TO_FP32_HALF(e[6]), GGML_E8M0_TO_FP32_HALF(e[7])); -} + memcpy(o.params, node.params, sizeof(o.params)); + memcpy(o.kernel_params, node.kernel_params, sizeof(o.kernel_params)); + o.opcode = node.opcode; -static void unpack_mxfp4_quants(uint8_t * qs, const block_mxfp4 * x, unsigned int bi) { - static const int qk = QK_MXFP4; + const auto & inputs = node.get_inputs(); + for (unsigned int i = 0; i < HTP_OP_MAX_INPUTS; i++) { + o.src[i] = (i < inputs.size() && inputs[i]) ? add_tensor(inputs[i]) : 0xffff; + } - for (unsigned int i = 0; i < qk / 2; ++i) { - const uint8_t x0 = (x->qs[i] & 0x0F); - const uint8_t x1 = (x->qs[i] >> 4); - qs[bi * qk + i + 0] = x0; - qs[bi * qk + i + qk / 2] = x1; + const auto & outputs = node.get_outputs(); + for (unsigned int i = 0; i < HTP_OP_MAX_OUTPUTS; i++) { + o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff; + } } -} -static void pack_mxfp4_quants(block_mxfp4 * x, const uint8_t * qs, unsigned int bi) { - static const int qk = QK4_0; + // caller must ensure the op fits + void add_op(const htp_opnode & node) { + unsigned int n = n_ops++; + GGML_ASSERT(n_ops <= n_ops_max); + + ops[n] = node; + h_ops[n].flags = 0; - for (unsigned int i = 0; i < qk / 2; ++i) { - const uint8_t x0 = qs[bi * qk + i + 0]; - const uint8_t x1 = qs[bi * qk + i + qk / 2]; - x->qs[i] = x0 | (x1 << 4); + ggml_hexagon_dump_op_exec(sess->c_name(), ops[n], h_ops[n].flags); + sync_op_desc(n); } -} -static void repack_row_mxfp4x4x2(uint8_t * y, const block_mxfp4 * x, int64_t k) { - static const int qk = QK_MXFP4x4x2; - const int nb = (k + qk - 1) / qk; // number of blocks (padded) - const int nloe = k % qk; // leftovers + void sort_buffers() { + if (n_bufs <= 1) return; - const int eblk_size = 8 * 1; // 8x E8M0 - const int qblk_size = qk / 2; // int4 - const int qrow_size = k / 2; // int4 (not padded to blocks) + std::vector order(n_bufs); + for (unsigned int i = 0; i < n_bufs; i++) { order[i] = (int) i; } - uint8_t * y_q = y + 0; // quants first - uint8_t * y_e = y + qrow_size; // then scales + std::stable_sort(order.begin(), order.end(), [&](int a, int b) { + return h_bufs[a].size > h_bufs[b].size; + }); - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_block_mxfp4(&x[i * 8 + 0], 0); - dump_block_mxfp4(&x[i * 8 + 1], 1); - dump_block_mxfp4(&x[i * 8 + 2], 2); - dump_block_mxfp4(&x[i * 8 + 3], 3); - dump_block_mxfp4(&x[i * 8 + 4], 4); - dump_block_mxfp4(&x[i * 8 + 5], 5); - dump_block_mxfp4(&x[i * 8 + 6], 6); - dump_block_mxfp4(&x[i * 8 + 7], 7); + bool already_sorted = true; + for (unsigned int i = 0; i < n_bufs; i++) { + if (order[i] != (int) i) { + already_sorted = false; + break; + } } - } + if (already_sorted) return; - // Repack the quants - for (int i = 0; i < nb; i++) { - uint8_t qs[QK_MXFP4x4x2]; // unpacked quants - - unpack_mxfp4_quants(qs, &x[i * 8 + 0], 0); - unpack_mxfp4_quants(qs, &x[i * 8 + 1], 1); - unpack_mxfp4_quants(qs, &x[i * 8 + 2], 2); - unpack_mxfp4_quants(qs, &x[i * 8 + 3], 3); - unpack_mxfp4_quants(qs, &x[i * 8 + 4], 4); - unpack_mxfp4_quants(qs, &x[i * 8 + 5], 5); - unpack_mxfp4_quants(qs, &x[i * 8 + 6], 6); - unpack_mxfp4_quants(qs, &x[i * 8 + 7], 7); + std::vector remap(n_bufs); + std::vector sorted_bufs(n_bufs); + for (unsigned int new_bi = 0; new_bi < n_bufs; new_bi++) { + int old_bi = order[new_bi]; + remap[old_bi] = (uint16_t) new_bi; + sorted_bufs[new_bi] = h_bufs[old_bi]; + } - bool partial = (nloe && i == nb-1); + for (unsigned int i = 0; i < n_bufs; i++) { + h_bufs[i] = sorted_bufs[i]; + } - uint8_t * q = y_q + (i * qblk_size); - for (int j = 0; j < qk / 2; j++) { - q[j] = partial ? (qs[j*2+1] << 4) | qs[j*2+0] : (qs[j+128] << 4) | qs[j+000]; + for (unsigned int i = 0; i < n_tens; i++) { + h_tens[i].bi = remap[h_tens[i].bi]; } } - // Repack the scales - // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_MXFP4x4x2) - // the last block is truncated and overridden by the scales. - for (int i = 0; i < nb; i++) { - // Repack the scales - uint8_t * e = (uint8_t *) (y_e + i * eblk_size); - e[0] = x[i * 8 + 0].e; - e[1] = x[i * 8 + 1].e; - e[2] = x[i * 8 + 2].e; - e[3] = x[i * 8 + 3].e; - e[4] = x[i * 8 + 4].e; - e[5] = x[i * 8 + 5].e; - e[6] = x[i * 8 + 6].e; - e[7] = x[i * 8 + 7].e; + void update_mdev_group(uint32_t mdev_idx) { + if (n_ops > 0 && h_ops[0].opcode == HTP_OP_MDEV_GROUP) { + h_ops[0].params[0] = (int32_t) mdev_idx; + } } - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_packed_block_mxfp4x4x2(y, i, k); + bool try_fuse(const htp_opnode & node) { + if (n_ops == 0) { + return false; + } + + htp_opnode & last_node = ops[n_ops - 1]; + if (!htp_fusion_try_fuse(sess, last_node, node)) { + return false; } + + sync_op_desc(n_ops - 1); + HEX_VERBOSE("ggml-hex: %s fused %s (N=%zu, #%u)\n", + sess->c_name(), last_node.name().c_str(), last_node.fused.size() + 1, n_ops - 1); + return true; } -} +}; -static void unpack_row_mxfp4x4x2(block_mxfp4 * x, const uint8_t * y, int64_t k) { - static const int qk = QK_MXFP4x4x2; - const int nb = (k + qk - 1) / qk; // number of blocks (padded) - const int nloe = k % qk; // leftovers +struct ggml_hexagon_registry { + ggml_hexagon_registry(ggml_backend_reg_t reg); + ~ggml_hexagon_registry(); - const int eblk_size = 8 * 1; // 8x E8M0 - const int qblk_size = qk / 2; // int4 - const int qrow_size = k / 2; // int4 (not padded to blocks) + ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS]; +}; - const uint8_t * y_q = y + 0; // quants first - const uint8_t * y_e = y + qrow_size; // then scales +struct ggml_hexagon_opqueue { + // Shared buffer for storing batches + ggml_hexagon_shared_buffer *shm_buf; + size_t shm_blk_size; + size_t depth; - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_packed_block_mxfp4x4x2(y, i, k); - } - } + using opvec = std::vector; - // Unpack the quants - for (int i = 0; i < nb; i++) { - uint8_t qs[QK_MXFP4x4x2]; // unpacked quants + std::vector op_cache; // per batch op cache - bool partial = (nloe && i == nb-1); + ggml_hexagon_opqueue(ggml_hexagon_session *sess, size_t batch_size, size_t depth) : depth(depth) { + size_t n_bufs = HTP_OP_MAX_BUFS; + size_t n_ops = batch_size; + size_t n_tensors = std::min(n_ops * 4 + HTP_OP_MAX_INPUTS + HTP_OP_MAX_OUTPUTS, HTP_OP_MAX_TENSORS); - const uint8_t * q = y_q + (i * qblk_size); - for (int j = 0; j < qk / 2; j++) { - if (partial) { - qs[j*2+0] = q[j] & 0xf; - qs[j*2+1] = q[j] >> 4; - } else { - qs[j+000] = q[j] & 0xf; - qs[j+128] = q[j] >> 4; - } + size_t tr_size = 0; + if (opt_profile == 3) { + tr_size = (HTP_MAX_NTHREADS + 1) * opt_optrace * sizeof(htp_trace_desc); } - pack_mxfp4_quants(&x[i * 8 + 0], qs, 0); - pack_mxfp4_quants(&x[i * 8 + 1], qs, 1); - pack_mxfp4_quants(&x[i * 8 + 2], qs, 2); - pack_mxfp4_quants(&x[i * 8 + 3], qs, 3); - pack_mxfp4_quants(&x[i * 8 + 4], qs, 4); - pack_mxfp4_quants(&x[i * 8 + 5], qs, 5); - pack_mxfp4_quants(&x[i * 8 + 6], qs, 6); - pack_mxfp4_quants(&x[i * 8 + 7], qs, 7); - } - - // Repack the scales - // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_MXFP4_0x4x2) - // the last block is truncated and overridden by the scales. - for (int i = 0; i < nb; i++) { - // Unpack the scales - const uint8_t * e = (const uint8_t *) (y_e + i * eblk_size); - x[i * 8 + 0].e = e[0]; - x[i * 8 + 1].e = e[1]; - x[i * 8 + 2].e = e[2]; - x[i * 8 + 3].e = e[3]; - x[i * 8 + 4].e = e[4]; - x[i * 8 + 5].e = e[5]; - x[i * 8 + 6].e = e[6]; - x[i * 8 + 7].e = e[7]; - } - - if (opt_verbose > 2) { - for (int i = 0; i < nb; i++) { - dump_block_mxfp4(&x[i * 8 + 0], 0); - dump_block_mxfp4(&x[i * 8 + 1], 1); - dump_block_mxfp4(&x[i * 8 + 2], 2); - dump_block_mxfp4(&x[i * 8 + 3], 3); - dump_block_mxfp4(&x[i * 8 + 4], 4); - dump_block_mxfp4(&x[i * 8 + 5], 5); - dump_block_mxfp4(&x[i * 8 + 6], 6); - dump_block_mxfp4(&x[i * 8 + 7], 7); - } - } -} + shm_blk_size = sizeof(htp_buf_desc) * n_bufs + + sizeof(htp_tensor) * n_tensors + + sizeof(htp_op_desc) * n_ops + + sizeof(htp_prof_desc) * n_ops + + tr_size; -static void init_row_mxfp4x4x2(block_mxfp4 * x, int64_t k) { - static const int qk = QK_MXFP4x4x2; - const int nb = (k + qk - 1) / qk; // number of blocks (padded) + shm_buf = new ggml_hexagon_shared_buffer(sess, shm_blk_size * depth, true /* pinned */); - // Init the quants such that they unpack into zeros - uint8_t qs[QK_MXFP4x4x2]; // unpacked quants - memset(qs, 0, sizeof(qs)); + op_cache.resize(depth); - for (int i = 0; i < nb; i++) { - pack_mxfp4_quants(&x[i * 8 + 0], qs, 0); - pack_mxfp4_quants(&x[i * 8 + 1], qs, 1); - pack_mxfp4_quants(&x[i * 8 + 2], qs, 2); - pack_mxfp4_quants(&x[i * 8 + 3], qs, 3); - pack_mxfp4_quants(&x[i * 8 + 4], qs, 4); - pack_mxfp4_quants(&x[i * 8 + 5], qs, 5); - pack_mxfp4_quants(&x[i * 8 + 6], qs, 6); - pack_mxfp4_quants(&x[i * 8 + 7], qs, 7); + if (opt_verbose) { + GGML_LOG_INFO("ggml-hex: %s allocated opqueue : batch-size %zu depth %zu shm-size %zu shm-block-size %zu\n", + sess->c_name(), batch_size, depth, shm_buf->size(), shm_blk_size); + } } - // Init the scales - // Note: Do not combine with the loop above. For tensor sizes not multiple of 256 (QK_MXFP4x4x2) - // the last block is truncated and overridden by the scales. - for (int i = 0; i < nb; i++) { - // Unpack the scales - x[i * 8 + 0].e = 0; - x[i * 8 + 1].e = 0; - x[i * 8 + 2].e = 0; - x[i * 8 + 3].e = 0; - x[i * 8 + 4].e = 0; - x[i * 8 + 5].e = 0; - x[i * 8 + 6].e = 0; - x[i * 8 + 7].e = 0; + ~ggml_hexagon_opqueue() { + delete shm_buf; } -} -// repack mxfp4 data into mxfp4x4x2 tensor -static void repack_mxfp4_mxfp4x4x2(ggml_tensor * t, const void * data, size_t size) { - int64_t nrows = ggml_nrows(t); + size_t shm_size() const { return shm_buf ? shm_buf->size() : 0; } - size_t row_size = ggml_row_size(t->type, t->ne[0]); - size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_MXFP4x4x2)); // extra elements for the pad - size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any) + // push new batch + bool push(htp_opbatch_req& req, dspqueue_buffer& dbuf, const ggml_hexagon_opbatch* op_batch, uint64_t seq) { + static_assert(sizeof(htp_opbatch_req) % 8 == 0, "sizeof(htp_opbatch_req) must be multiple of 8"); + static_assert(sizeof(htp_opbatch_rsp) % 8 == 0, "sizeof(htp_opbatch_rsp) must be multiple of 8"); + static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8"); + static_assert(sizeof(htp_tensor) % 8 == 0, "sizeof(htp_tensor) must be multiple of 8"); + static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8"); + static_assert(sizeof(htp_prof_desc) % 8 == 0, "sizeof(htp_prof_desc) must be multiple of 8"); - // Ensure we don't try to read more data than is available in the source buffer 'data' - // or write more than the tensor can hold. - const size_t total_tensor_size = (size_t)nrows * row_size; - const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size; + if (seq - shm_buf->sess->batch_rsp_seq > depth) { return false; } - // Calculate how many full rows and how many remaining bytes we need to process. - const int64_t n_full_rows = n_bytes_to_copy / row_size; - const size_t n_rem_bytes = n_bytes_to_copy % row_size; + const uint32_t slot = (uint32_t) ((seq - 1) % depth); - void * buf_pd = ggml_aligned_malloc(row_size_pd); - GGML_ASSERT(buf_pd != NULL); + req.seq = seq; + req.n_bufs = op_batch->n_bufs; + req.n_tensors = op_batch->n_tens; + req.n_ops = op_batch->n_ops; - void * buf_rp = ggml_aligned_malloc(row_size_rp); - GGML_ASSERT(buf_rp != NULL); + if (opt_profile) { + op_cache[slot].assign(op_batch->ops.begin(), op_batch->ops.begin() + req.n_ops); + } - HEX_VERBOSE("ggml-hex: repack-mxfp4-mxfp4x4x2 %s : data %p size %zu dims %ldx%ld row-size %zu\n", t->name, data, - size, t->ne[0], nrows, row_size); + const size_t b_size = sizeof(htp_buf_desc) * req.n_bufs; + const size_t t_size = sizeof(htp_tensor) * req.n_tensors; + const size_t o_size = sizeof(htp_op_desc) * req.n_ops; + const size_t p_size = sizeof(htp_prof_desc) * req.n_ops; - init_row_mxfp4x4x2((block_mxfp4 *) buf_pd, t->ne[0]); // init padded buffer to make sure the tail is all zeros + size_t tr_size = 0; + if (opt_profile == 3) { + req.n_traces = opt_optrace; + tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(htp_trace_desc); + } else { + req.n_traces = 0; + } - // 1. Process all the full rows - for (int64_t i = 0; i < n_full_rows; i++) { - const uint8_t * src = (const uint8_t *) data + (i * row_size); - uint8_t * dst = (uint8_t *) t->data + (i * row_size); + dbuf.ptr = shm_buf->base() + ((size_t) slot * shm_blk_size); + dbuf.fd = shm_buf->fd(); + dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; + dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) shm_buf->base(); + dbuf.size = b_size + t_size + o_size + p_size + tr_size; - memcpy(buf_pd, src, row_size); - repack_row_mxfp4x4x2((uint8_t *) buf_rp, (const block_mxfp4 *) buf_pd, t->ne[0]); - memcpy(dst, buf_rp, row_size); - } + GGML_ASSERT(dbuf.size <= shm_blk_size); - // 2. Process the final, potentially partial, row - if (n_rem_bytes > 0) { - const int64_t i = n_full_rows; - const uint8_t * src = (const uint8_t *) data + (i * row_size); - uint8_t * dst = (uint8_t *) t->data + (i * row_size); + uint8_t * m_ptr = (uint8_t*) dbuf.ptr; + uint8_t * b_ptr = m_ptr; m_ptr += b_size; + uint8_t * t_ptr = m_ptr; m_ptr += t_size; + uint8_t * o_ptr = m_ptr; - // re-init the row because we are potentially copying a partial row - init_row_mxfp4x4x2((block_mxfp4 *) buf_pd, t->ne[0]); + memcpy(b_ptr, (void *) op_batch->h_bufs.data(), b_size); + memcpy(t_ptr, (void *) op_batch->h_tens.data(), t_size); + memcpy(o_ptr, (void *) op_batch->h_ops.data(), o_size); - // Copy only the remaining bytes from the source. - memcpy(buf_pd, src, n_rem_bytes); + HEX_VERBOSE("ggml-hex: %s opqueue-push batch #%llu : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu m-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) req.seq, req.n_bufs, req.n_tensors, req.n_ops, op_batch->b_vmem, + b_size, t_size, o_size, (size_t) dbuf.size); - // Repack the entire buffer (partial data + zero padding). - repack_row_mxfp4x4x2((uint8_t *) buf_rp, (const block_mxfp4 *) buf_pd, t->ne[0]); + if (opt_verbose > 1) { + htp_buf_desc *b = (htp_buf_desc*) b_ptr; + for (unsigned int i=0; i < req.n_bufs; i++) { + GGML_LOG_DEBUG("ggml-hex: %s htp-buf #%u : fd %d base %p size %zu\n", shm_buf->sess->c_name(), i, + b[i].fd, (void *) b[i].base, (size_t) b[i].size); + } + htp_tensor *t = (htp_tensor*) t_ptr; + for (unsigned int i=0; i < req.n_tensors; i++) { + GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %llu size %u : %zu:%zu:%zu:%zu\n", + shm_buf->sess->c_name(), i, t[i].bi, (unsigned long long) t[i].data, t[i].size, + (size_t) t[i].ne[0], (size_t) t[i].ne[1], (size_t) t[i].ne[2], (size_t) t[i].ne[3]); + } + } - // Write only the corresponding remaining bytes to the destination tensor. - memcpy(dst, buf_rp, n_rem_bytes); + return true; } - ggml_aligned_free(buf_pd, row_size_pd); - ggml_aligned_free(buf_rp, row_size_rp); -} + void pop(htp_opbatch_rsp rsp, dspqueue_buffer dbuf) { + const uint32_t slot = (uint32_t) ((rsp.seq - 1) % depth); -// repack mxfp4x4x2 tensor into mxfp4 data -static void repack_mxfp4x4x2_mxfp4(void * data, const ggml_tensor * t, size_t size) { - int64_t nrows = ggml_nrows(t); + const size_t b_size = sizeof(htp_buf_desc) * rsp.n_bufs; + const size_t t_size = sizeof(htp_tensor) * rsp.n_tensors; + const size_t o_size = sizeof(htp_op_desc) * rsp.n_ops; + const size_t p_size = sizeof(htp_prof_desc) * rsp.n_ops; - size_t row_size = ggml_row_size(t->type, t->ne[0]); - size_t row_size_pd = ggml_row_size(t->type, hex_round_up(t->ne[0], QK_MXFP4x4x2)); // extra elements for the pad - size_t row_size_rp = row_size * 2; // extra space for tmp pad (if any) + size_t tr_size = 0; + uint32_t n_traces = 0; + if (opt_profile == 3) { + n_traces = opt_optrace; + tr_size = (HTP_MAX_NTHREADS + 1) * n_traces * sizeof(htp_trace_desc); + } - // Ensure we don't try to copy more data than the tensor actually contains. - const size_t total_tensor_size = (size_t)nrows * row_size; - const size_t n_bytes_to_copy = size < total_tensor_size ? size : total_tensor_size; + const size_t m_size = b_size + t_size + o_size + p_size + tr_size; + GGML_ASSERT(m_size <= shm_blk_size); - // Calculate how many full rows and how many remaining bytes we need to process. - const int64_t n_full_rows = n_bytes_to_copy / row_size; - const size_t n_rem_bytes = n_bytes_to_copy % row_size; + HEX_VERBOSE("ggml-hex: %s opqueue-pop batch #%llu : n-bufs %u n-tensors %u n-ops %u : m-size %zu b-size %zu t-size %zu o-size %zu\n", + shm_buf->sess->c_name(), (unsigned long long) rsp.seq, rsp.n_bufs, rsp.n_tensors, rsp.n_ops, + (size_t) dbuf.size, b_size, t_size, o_size); - void * buf_pd = ggml_aligned_malloc(row_size_pd); - GGML_ASSERT(buf_pd != NULL); + uint8_t * m_ptr = (uint8_t*) dbuf.ptr; + uint8_t * p_ptr = m_ptr + (b_size + t_size + o_size); - void * buf_rp = ggml_aligned_malloc(row_size_rp); - GGML_ASSERT(buf_rp != NULL); + if (rsp.n_ops > 0 && opt_profile) { + auto & ops = op_cache[slot]; + GGML_ASSERT(rsp.n_ops <= ops.size()); - HEX_VERBOSE("ggml-hex: repack-mxfp4x4x2-mxfp4 %s : data %p size %zu dims %ldx%ld row-size %zu\n", t->name, data, - size, t->ne[0], nrows, row_size); + const htp_prof_desc * pd = (const htp_prof_desc *) p_ptr; + const htp_trace_desc * trace_events = nullptr; + if (opt_profile == 3) { + trace_events = (const htp_trace_desc *) (p_ptr + p_size); + } - memset(buf_pd, 0, row_size_pd); // clear-out padded buffer to make sure the tail is all zeros + ggml_hexagon_dump_batch_prof(shm_buf->sess->c_name(), rsp); - // 1. Process all the full rows - for (int64_t i = 0; i < n_full_rows; i++) { - const uint8_t * src = (const uint8_t *) t->data + (i * row_size); - uint8_t * dst = (uint8_t *) data + (i * row_size); + for (uint32_t i = 0; i < rsp.n_ops; i++) { + ggml_hexagon_dump_op_prof(shm_buf->sess->c_name(), ops[i], pd[i]); + } - memcpy(buf_pd, src, row_size); - unpack_row_mxfp4x4x2((block_mxfp4 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]); - memcpy(dst, buf_rp, row_size); + ggml_hexagon_dump_trace_events(shm_buf->sess->c_name(), rsp, trace_events, n_traces); + } } +}; - // 2. Process the final, potentially partial, row - if (n_rem_bytes > 0) { - const int64_t i = n_full_rows; - const uint8_t * src = (const uint8_t *) t->data + (i * row_size); - uint8_t * dst = (uint8_t *) data + (i * row_size); +void ggml_hexagon_session::flush_peers() { + auto vpeers = std::move(virt_peers); + virt_peers.clear(); + for (auto * peer : vpeers) { + peer->flush_sync(); + } - // We still need to read and unpack the entire source row because the format is block-based. - memcpy(buf_pd, src, row_size); - unpack_row_mxfp4x4x2((block_mxfp4 *) buf_rp, (const uint8_t *) buf_pd, t->ne[0]); + auto ppeers = std::move(phys_peers); + phys_peers.clear(); + for (auto * peer : ppeers) { + peer->flush_async(); + } - // But we only copy the remaining number of bytes to the destination to respect the size limit. - memcpy(dst, buf_rp, n_rem_bytes); + for (auto & sub : this->mdev.sessions) { + sub->flush_peers(); } +} - ggml_aligned_free(buf_pd, row_size_pd); - ggml_aligned_free(buf_rp, row_size_rp); +void ggml_hexagon_session::flush_async() { + flush_peers(); + flush_batch(); } -static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, - ggml_tensor * tensor, - const void * data, - size_t offset, - size_t size) { - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; +void ggml_hexagon_session::flush_pending(bool all) { + for (auto & sub : this->mdev.sessions) { + sub->flush_pending(all); + if (sub->last_error > HTP_STATUS_OK) { + this->last_error = sub->last_error.load(); + } + } + + while (this->batch_rsp_seq < this->batch_req_seq) { + struct htp_opbatch_rsp rsp; + uint32_t rsp_size; + uint32_t flags; - HEX_VERBOSE("ggml-hex: %s set-tensor %s : data %p offset %zu size %zu\n", sess->c_name(), tensor->name, data, offset, size); + struct dspqueue_buffer dbuf; + uint32_t n_dbufs; - switch (tensor->type) { - case GGML_TYPE_Q4_0: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_q4_0_q4x4x2(tensor, data, size); - break; + // Read response packet from queue + const uint32_t timeo = opt_oppoll ? 0 : DSPQUEUE_TIMEOUT; - case GGML_TYPE_Q8_0: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_q8_0_q8x4x2(tensor, data, size); - break; + int err = dspqueue_read(this->queue, &flags, 1, &n_dbufs, &dbuf, sizeof(rsp), &rsp_size, (uint8_t *) &rsp, timeo); + if (err == AEE_EEXPIRED || err == AEE_EWOULDBLOCK) { + continue; + } - case GGML_TYPE_IQ4_NL: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - // IQ4_NL has identical block layout to Q4_0 (ggml_half d + uint8_t qs[16]) - repack_q4_0_q4x4x2(tensor, data, size); - break; + if (err != 0) { + GGML_ABORT("ggml-hex: dspqueue_read failed: 0x%08x\n", (unsigned) err); + } - case GGML_TYPE_MXFP4: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_mxfp4_mxfp4x4x2(tensor, data, size); - break; + // Basic sanity checks + if (rsp_size != sizeof(rsp) || n_dbufs != 1) { + GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs); + } - default: - memcpy((char *) tensor->data + offset, data, size); - break; + if (rsp.status > HTP_STATUS_OK) { + GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp %s\n", this->c_name(), status_to_str(rsp.status)); + this->last_error = rsp.status; + for (auto & sub : this->mdev.sessions) { + sub->last_error = rsp.status; + } + } + + op_queue->pop(rsp, dbuf); + + GGML_ASSERT(rsp.seq == this->batch_rsp_seq + 1); + this->batch_rsp_seq = rsp.seq; + + if (!all) break; } } -static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, - const ggml_tensor * tensor, - void * data, - size_t offset, - size_t size) { - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; +void ggml_hexagon_session::flush_sync(bool all) { + flush_async(); + flush_pending(all); +} - HEX_VERBOSE("ggml-hex: %s get-tensor %s : data %p offset %zu size %zu\n", sess->c_name(), tensor->name, data, offset, size); +void ggml_hexagon_session::start_batch() { + if (this->mdev.count > 1) { + enqueue_mdev_group(); + } +} - switch (tensor->type) { - case GGML_TYPE_Q4_0: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_q4x4x2_q4_0(data, tensor, size); - break; +void ggml_hexagon_session::flush_batch(size_t min_ops) { + if (op_batch->n_ops < min_ops) { return; } - case GGML_TYPE_Q8_0: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_q8x4x2_q8_0(data, tensor, size); - break; + op_batch->sort_buffers(); - case GGML_TYPE_IQ4_NL: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_q4x4x2_q4_0(data, tensor, size); - break; + htp_opbatch_req req {}; + dspqueue_buffer dbuf{}; - case GGML_TYPE_MXFP4: - GGML_ASSERT(offset == 0); - GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); - repack_mxfp4x4x2_mxfp4(data, tensor, size); - break; + const uint64_t seq = ++this->batch_req_seq; - default: - memcpy(data, (const char *) tensor->data + offset, size); - break; + op_batch->update_mdev_group(this->mdev.idx); + + if (!op_queue->push(req, dbuf, op_batch, seq)) { + flush_pending(false); + op_queue->push(req, dbuf, op_batch, seq); } -} -static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t buffer, - const struct ggml_tensor * src, - struct ggml_tensor * dst) { - GGML_UNUSED(buffer); - GGML_UNUSED(src); - GGML_UNUSED(dst); - // we might optimize this later, for now take the slow path (ie get/set_tensor) - return false; -} + for (auto & sub : this->mdev.sessions) { + htp_opbatch_req sub_req {}; + dspqueue_buffer sub_dbuf{}; -static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - auto sbuf = (ggml_hexagon_shared_buffer *) buffer->context; - auto sess = sbuf->sess; - HEX_VERBOSE("ggml-hex: %s clear-buff base %p size %zu\n", sess->c_name(), (void *) sbuf->base, sbuf->size); - memset(sbuf->base, value, sbuf->size); -} + sub->batch_req_seq = seq; + op_batch->update_mdev_group(sub->mdev.idx); -static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = { - /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, - /* .get_base = */ ggml_backend_hexagon_buffer_get_base, - /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor, - /* .memset_tensor = */ NULL, - /* .set_tensor = */ ggml_backend_hexagon_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_hexagon_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, - /* .get_tensor_2d = */ NULL, - /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, - /* .clear = */ ggml_backend_hexagon_buffer_clear, - /* .reset = */ NULL, -}; - -// ** backend buffer type - -static const char * ggml_backend_hexagon_buffer_type_name(ggml_backend_buffer_type_t buffer_type) { - return static_cast(buffer_type->context)->name.c_str(); -} + if (!sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq)) { + sub->flush_pending(false); + sub->op_queue->push(sub_req, sub_dbuf, op_batch, seq); + } -static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer( - ggml_backend_buffer_type_t buffer_type, size_t size) { - auto sess = static_cast(buffer_type->context)->sess; - try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size); - return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); - } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer context (host): %s\n", sess->c_name(), exc.what()); - return nullptr; - } -} + HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", sub->c_name(), sub_dbuf.ptr, sub_dbuf.size); -static ggml_backend_buffer_t ggml_backend_hexagon_repack_buffer_type_alloc_buffer( - ggml_backend_buffer_type_t buffer_type, size_t size) { - auto sess = static_cast(buffer_type->context)->sess; - try { - ggml_hexagon_shared_buffer * sbuf = new ggml_hexagon_shared_buffer(sess, size); - return ggml_backend_buffer_init(buffer_type, ggml_backend_hexagon_buffer_interface, sbuf, size); - } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: %s failed to allocate buffer context (repack): %s\n", sess->c_name(), exc.what()); - return nullptr; + int err = dspqueue_write(sub->queue, 0, 1, &sub_dbuf, sizeof(sub_req), (const uint8_t*) &sub_req, DSPQUEUE_TIMEOUT); + if (err != 0) { + GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", sub->c_name(), (unsigned) err); + } } -} -static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) { - return 128; // HVX alignment - GGML_UNUSED(buffer_type); -} - -static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) { - return ggml_nbytes(t); -} - -static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) { - return 1UL * 1024 * 1024 * 1024; // 1GB per buffer - GGML_UNUSED(buffer_type); -} + HEX_VERBOSE("ggml-hex: %s queue-opbatch: %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size); -static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - return opt_hostbuf; - GGML_UNUSED(buft); -} + int err = dspqueue_write(this->queue, 0, 1, &dbuf, sizeof(req), (const uint8_t*) &req, DSPQUEUE_TIMEOUT); + if (err != 0) { + GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err); + } -static bool ggml_backend_hexagon_repack_buffer_type_is_host(ggml_backend_buffer_type_t buft) { - return false; - GGML_UNUSED(buft); + op_batch->reset(); } -static ggml_backend_buffer_type_i ggml_backend_hexagon_buffer_type_interface = { - /* .get_name = */ ggml_backend_hexagon_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_hexagon_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size, - /* .is_host = */ ggml_backend_hexagon_buffer_type_is_host, -}; - -static ggml_backend_buffer_type_i ggml_backend_hexagon_repack_buffer_type_interface = { - /* .get_name = */ ggml_backend_hexagon_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_hexagon_repack_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_hexagon_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_hexagon_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_hexagon_buffer_type_get_alloc_size, - /* .is_host = */ ggml_backend_hexagon_repack_buffer_type_is_host, -}; - -// Backend session implementation - -struct ggml_hexagon_opshm { - ggml_hexagon_shared_buffer *sbuf; - - std::vector block_mask; - size_t block_size; - - uint8_t * base() const { return this->sbuf->base; } - int fd() const { return this->sbuf->fd; } - size_t n_blocks() const { return this->block_mask.size(); } - - ggml_hexagon_opshm(ggml_hexagon_session *sess, size_t max_batch, size_t max_pending) { - size_t n_bufs = HTP_OP_MAX_BUFS; - size_t n_ops = max_batch; - size_t n_tensors = n_ops + n_ops * HTP_OP_MAX_INPUTS; - - block_mask.resize(max_pending, true); - - block_size = sizeof(htp_buf_desc) * n_bufs + - sizeof(htp_tensor) * n_tensors + - sizeof(htp_op_desc) * n_ops; - - sbuf = new ggml_hexagon_shared_buffer(sess, block_size * block_mask.size(), true /* pinned */); - - if (opt_verbose) { - GGML_LOG_INFO("ggml-hex: %s allocated shared buf %zu : block-size %zu max-batch %zu max-pending %zu\n", - sess->c_name(), (size_t) sbuf->size, block_size, max_batch, max_pending); +void ggml_hexagon_session::enqueue_op(const htp_opnode & node) { + auto clone_tensor_buffer = [this](const ggml_tensor * t) { + auto sbuf = this->mmap_tensor(t); + if (!sbuf) return; + if (sbuf->sess != this) { + this->clone_buffer(sbuf); } - } + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(sbuf); + } + }; - ~ggml_hexagon_opshm() { - delete sbuf; + for (auto t : node.get_inputs()) { + clone_tensor_buffer(t); } - - uint8_t * allocate() { - auto it = std::find(block_mask.begin(), block_mask.end(), true); - if (it == block_mask.end()) - return nullptr; - - unsigned int i = std::distance(block_mask.begin(), it); - uint8_t* addr = sbuf->base + (i * block_size); - block_mask[i] = false; - - HEX_VERBOSE("ggml-hex: %s allocated op shm #%u %p\n", sbuf->sess->c_name(), i, (void*) addr); - return addr; + for (auto t : node.get_outputs()) { + clone_tensor_buffer(t); } - void release(uint8_t * addr) { - int i = (addr - sbuf->base) / block_size; - block_mask[i] = true; - HEX_VERBOSE("ggml-hex: %s released op shm #%u %p\n", sbuf->sess->c_name(), i, (void*) addr); + const bool fits_tensors = op_batch->fit_tensors(node); + if (opt_opfusion && fits_tensors && op_batch->try_fuse(node)) { + return; } -}; - -struct ggml_hexagon_opbatch { - const char* name; - - std::vector buffers; - std::vector tensors; - std::vector ops; - - std::unordered_map b_map; // buffer fd to index - std::unordered_map t_map; // tensor ptr to index - std::unordered_multimap d_map; // tensor data to index - - unsigned int n_bufs; // num buffers in the batch - unsigned int n_tens; // num tensors ... - unsigned int n_ops; // num ops ... - size_t b_vmem; // sum of all buffer sizes - - unsigned int n_bufs_max; - unsigned int n_tens_max; - unsigned int n_ops_max; - size_t b_vmem_max; - - void reset() { - n_bufs = 0; - n_tens = 0; - n_ops = 0; - b_vmem = 0; - b_map.clear(); - t_map.clear(); - d_map.clear(); + if (op_batch->full() || !fits_tensors) { + flush_async(); } - ggml_hexagon_opbatch(ggml_hexagon_session *sess, size_t max_batch) { - name = sess->c_name(); - - n_bufs_max = HTP_OP_MAX_BUFS; - n_ops_max = max_batch; - n_tens_max = n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS; - - b_vmem_max = HTP_OP_MAX_VMEM; - - buffers.resize(n_bufs_max); - tensors.resize(n_tens_max); - ops.resize(n_ops_max); - - b_map.reserve(n_bufs_max); - t_map.reserve(n_tens_max); - d_map.reserve(n_tens_max); - - reset(); + if (op_batch->empty()) { + start_batch(); + if (!op_batch->fit_tensors(node)) { + GGML_ABORT("ggml-hex: %s op does not fit into empty batch (vmem/tensor/buffer limit exceeded)\n", c_name()); + } } - bool empty() const { return n_ops == 0; } + op_batch->add_op(node); +} - // add buffer and return its index - int add_buffer(ggml_hexagon_shared_buffer * sbuf) { - // Lookup by fd - auto it = b_map.find(sbuf->fd); - if (it != b_map.end()) { return it->second; } +void ggml_hexagon_session::enqueue_mdev_group() { + htp_opnode group_node(HTP_OP_MDEV_GROUP); - // Add new buffer to the batch - int bi = n_bufs++; - GGML_ASSERT(n_bufs < HTP_OP_MAX_BUFS); + ggml_tensor dummy_t = make_fence_tensor(this->mdev_fence_slot, this->mdev.count); + dummy_t.op_params[0] = (int32_t) this->mdev.idx; - b_map.insert({sbuf->fd, bi}); + ggml_tensor * node = group_node.add_dummy(dummy_t); + node->src[0] = node; + group_node.init(node); + group_node.outputs.clear(); - htp_buf_desc &b = buffers[bi]; - b.base = (uint64_t) sbuf->base; - b.fd = sbuf->fd; - b.size = sbuf->size; + for (auto & sub : this->mdev.sessions) { + sub->clone_buffer(this->fence_buf); + } - b_vmem += b.size; + op_batch->add_op(group_node); +} - HEX_VERBOSE("ggml-hex: add-buffer #%u : fd %d base %p size %zu : vmem %zu\n", bi, b.fd, (void*) sbuf->base, (size_t) b.size, b_vmem); +void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * dst, const ggml_tensor * sync_tensor, uint32_t fence_seq) { + const bool with_fence = sync_tensor != nullptr; + htp_opnode cpy_node(with_fence ? HTP_OP_CPY_FENCE : HTP_OP_CPY); - return bi; + ggml_tensor* node = cpy_node.add_dummy(*dst); + node->op = GGML_OP_CPY; + node->src[0] = const_cast(src); + node->src[1] = with_fence ? cpy_node.add_dummy(*sync_tensor) : nullptr; + if (with_fence) { + node->op_params[0] = (int32_t) fence_seq; } - bool same_shape(const htp_tensor * h, const ggml_tensor * t) const { - return (h->ne[0] == t->ne[0]) && (h->ne[1] == t->ne[1]) && (h->ne[2] == t->ne[2]) && (h->ne[3] == t->ne[3]) && - (h->nb[0] == t->nb[0]) && (h->nb[1] == t->nb[1]) && (h->nb[2] == t->nb[2]) && (h->nb[3] == t->nb[3]); + cpy_node.init(node); + const bool ok = htp_kparams_solve(this, node, cpy_node.kernel_params); + if (ok && !with_fence && cpy_node.kparams()->total_elems == 0) { + return; } + this->enqueue_op(cpy_node); +} - // add tensor and return its index - int add_tensor(const ggml_tensor * t) { - auto sbuf = static_cast(t->buffer->context); - - // First lookup by tensor data - auto range = d_map.equal_range(t->data); - for (auto it = range.first; it != range.second; ++it) { - htp_tensor * h = &tensors[it->second]; - if (same_shape(h, t)) { return it->second; } - } - - // Lookup by tensor ptr - auto it = t_map.find(t); - if (it != t_map.end()) { return it->second; } - - // Add new tensor to the batch - int ti = n_tens++; - GGML_ASSERT(n_tens <= n_tens_max); +void ggml_hexagon_session::enqueue_fence(const ggml_tensor * sync_tensor, uint32_t fence_seq, bool wait) { + htp_opnode sync_node(HTP_OP_FENCE); - t_map.insert({t, ti}); - d_map.insert({t->data, ti}); + ggml_tensor* node = sync_node.add_dummy(*sync_tensor); + node->op = GGML_OP_NONE; + node->src[0] = node; + node->op_params[0] = (int32_t) fence_seq; + node->op_params[1] = wait ? 0 : 1; - uint64_t t_offset = (uint8_t *) t->data - sbuf->base; - size_t t_size = ggml_nbytes(t); + sync_node.init(node); + this->enqueue_op(sync_node); +} - htp_tensor &h = tensors[ti]; - h.bi = add_buffer(sbuf); - h.data = t_offset; - h.size = t_size; - h.type = t->type; - h.ne[0] = t->ne[0]; h.ne[1] = t->ne[1]; h.ne[2] = t->ne[2]; h.ne[3] = t->ne[3]; - h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3]; +void ggml_hexagon_session::enqueue_allreduce( + const ggml_tensor * dst, + const std::vector & src_tensors, + const std::vector & sync_tensors, + uint32_t rank, + uint32_t n_ranks, + uint32_t fence_seq_entry, + uint32_t fence_seq_exit +) { + htp_opnode ar_node(HTP_OP_ALLREDUCE); - h.flags = 0; - if (ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { - h.flags |= HTP_TENSOR_COMPUTE; - } + ggml_tensor* node = ar_node.add_dummy(*dst); + node->op = GGML_OP_NONE; + node->op_params[0] = (int32_t) fence_seq_entry; + node->op_params[1] = (int32_t) fence_seq_exit; - HEX_VERBOSE("ggml-hex: add-tensor #%u %s : bi %d data %p offset %zu size %zu flags 0x%x : %zu:%zu:%zu:%zu\n", - ti, t->name, h.bi, (void*) t->data, (size_t) t_offset, t_size, h.flags, - (size_t) t->ne[0], (size_t) t->ne[1], (size_t) t->ne[2], (size_t) t->ne[3]); + ar_node.init(node); - return ti; + ar_node.inputs.clear(); + for (size_t i = 0; i < src_tensors.size(); i++) { + ar_node.inputs.push_back(src_tensors[i]); } - - bool fit_op(const struct ggml_tensor *t) const { - if (n_ops >= n_ops_max ) return false; - - // check how much extras we will need - size_t extra_bufs = 0; - size_t extra_vmem = 0; - size_t extra_tens = 0; - - auto fit_tensor = [&](const ggml_tensor *t) { - if (!t_map.count(t)) { - extra_tens++; - - auto sbuf = static_cast(t->buffer->context); - if (!b_map.count(sbuf->fd)) { - extra_vmem += sbuf->size; - extra_bufs += 1; - } - } - }; - - for (unsigned int i=0; i < HTP_OP_MAX_INPUTS && t->src[i]; i++) { - fit_tensor(t->src[i]); - } - fit_tensor(t); - - if ((extra_bufs + n_bufs) > n_bufs_max) return false; - if ((extra_tens + n_tens) > n_tens_max) return false; - if ((extra_vmem + b_vmem) > b_vmem_max) return false; - - return true; + for (size_t i = 0; i < sync_tensors.size(); i++) { + ar_node.inputs.push_back(ar_node.add_dummy(*sync_tensors[i])); } - // assumes that fit_op() was called first and returned true - void add_op(htp_op_code opcode, const struct ggml_tensor * t) { - // Add new op - htp_op_desc &o = ops[n_ops++]; - GGML_ASSERT(n_ops <= n_ops_max); + ar_node.outputs.clear(); + for (size_t i = 0; i < src_tensors.size(); i++) { + ar_node.outputs.push_back(src_tensors[i]); + } - memcpy(&o.params, &t->op_params, sizeof(t->op_params)); - o.opcode = opcode; - o.flags = 0; + htp_kparams_solve_allreduce( + this, dst, rank, n_ranks, false, false, /*is_shard_ok=*/ false, + ar_node.kparams()); - if (!(opt_opmask & HTP_OPMASK_COMPUTE)) { - o.flags |= HTP_OPFLAGS_SKIP_COMPUTE; - } + this->enqueue_op(ar_node); +} - ggml_hexagon_dump_op_exec(name, t, o.flags); +ggml_hexagon_shared_buffer * ggml_hexagon_session::mmap_tensor(const ggml_tensor * t) { + if (!t) return nullptr; - for (unsigned int i=0; i < HTP_OP_MAX_INPUTS; i++) { - o.src[i] = t->src[i] ? add_tensor(t->src[i]) : 0xffff; - } - o.dst = add_tensor(t); + auto sbuf = static_cast(t->buffer->context); + if (!sbuf->mapped) { + const bool is_weight = ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + const bool extended = opt_dma64 && is_weight; + sbuf->mmap(extended); } + return sbuf; +} - size_t flush(uint8_t * mem_addr, size_t mem_size) { - static_assert(sizeof(htp_buf_desc) % 8 == 0, "sizeof(htp_buf_desc) must be multiple of 8"); - static_assert(sizeof(htp_tensor) % 8 == 0, "sizeof(htp_tensor) must be multiple of 8"); - static_assert(sizeof(htp_op_desc) % 8 == 0, "sizeof(htp_op_desc) must be multiple of 8"); - - const size_t b_size = sizeof(htp_buf_desc) * n_bufs; - const size_t t_size = sizeof(htp_tensor) * n_tens; - const size_t o_size = sizeof(htp_op_desc) * n_ops; - - const size_t m_size = b_size + t_size + o_size; - GGML_ASSERT(m_size <= mem_size); +bool ggml_hexagon_session::clone_buffer(const ggml_hexagon_shared_buffer * sbuf) +{ + GGML_ASSERT(sbuf && sbuf->mem); + if (sbuf->sess == this) return true; - uint8_t * b_ptr = (uint8_t *) mem_addr; - uint8_t * t_ptr = (uint8_t *) b_ptr + b_size; - uint8_t * o_ptr = (uint8_t *) t_ptr + t_size; + auto mem = sbuf->mem; + int fd = mem->fd; - memcpy(b_ptr, (void *) buffers.data(), b_size); - memcpy(t_ptr, (void *) tensors.data(), t_size); - memcpy(o_ptr, (void *) ops.data(), o_size); + GGML_ASSERT(fd >= 0); - HEX_VERBOSE("ggml-hex: %s flush-opbatch : n-bufs %u n-tensors %u n-ops %u vmem %zu : b-size %zu t-size %zu o-size %zu\n", - name, n_bufs, n_tens, n_ops, b_vmem, b_size, t_size, o_size); + if (this->cloned_buffers.find(fd) != this->cloned_buffers.end()) return true; - if (opt_verbose > 1) { - htp_buf_desc *b = (htp_buf_desc*) b_ptr; - for (unsigned int i=0; i < n_bufs; i++) { - GGML_LOG_DEBUG("ggml-hex: %s htp-buf #%u : fd %d base %p size %zu\n", name, i, - b[i].fd, (void *) b[i].base, (size_t) b[i].size); - } - htp_tensor *t = (htp_tensor*) t_ptr; - for (unsigned int i=0; i < n_tens; i++) { - GGML_LOG_DEBUG("ggml-hex: %s htp-tensor #%u : bi %u offset %u size %u : %zu:%zu:%zu:%zu\n", - name, i, t[i].bi, t[i].data, t[i].size, - (size_t) t[i].ne[0], (size_t) t[i].ne[1], (size_t) t[i].ne[2], (size_t) t[i].ne[3]); - } - } + GGML_ASSERT(sbuf->mapped); - reset(); + HEX_VERBOSE("ggml-hex: %s clone-buffer: %s base %p size %zu fd %d\n", this->name.c_str(), + sbuf->c_name(), sbuf->base(), sbuf->size(), fd); - return m_size; + auto clone = std::make_unique(this, *sbuf); + try { + clone->mmap(sbuf->extended); + } catch (const std::exception & exc) { + GGML_LOG_ERROR("ggml-hex: %s lazy mapping of buffer context failed: %s\n", this->c_name(), exc.what()); + return false; } -}; - -// Flush HTP response queue i.e wait for all outstanding requests to complete -void ggml_hexagon_session::flush_pending(bool all) { - while (this->op_pending) { - struct htp_opbatch_rsp rsp; - uint32_t rsp_size; - uint32_t flags; - - struct dspqueue_buffer dbuf; - uint32_t n_dbufs; - // Read response packet from queue - int err = dspqueue_read(this->queue, &flags, 1, &n_dbufs, &dbuf, sizeof(rsp), &rsp_size, (uint8_t *) &rsp, DSPQUEUE_TIMEOUT); - if (err == AEE_EEXPIRED) { - continue; - } + this->cloned_buffers[fd] = std::move(clone); + mem->mapped_clones.insert(this); + return true; +} - if (err != 0) { - GGML_ABORT("ggml-hex: dspqueue_read failed: 0x%08x\n", (unsigned) err); - } +void ggml_hexagon_session::release_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); - // Basic sanity checks - if (rsp_size != sizeof(rsp) || n_dbufs != 1) { - GGML_ABORT("ggml-hex: %s dspcall : bad response : size %u dspbufs %u\n", this->c_name(), rsp_size, n_dbufs); - } + auto mem = sbuf->mem; + int fd = mem->fd; - op_shm->release((uint8_t*) dbuf.ptr); + GGML_ASSERT(fd >= 0); - if (rsp.status != HTP_STATUS_OK) { - GGML_LOG_ERROR("ggml-hex: %s dspcall : dsp-rsp: %s\n", this->c_name(), status_to_str(rsp.status)); - // TODO: handle errors - } + auto it = this->cloned_buffers.find(fd); + if (it != this->cloned_buffers.end()) { + auto clone = std::move(it->second); + this->cloned_buffers.erase(it); + } + mem->mapped_clones.erase(this); +} - // FIXME: profile will be per opreq - // this->prof_usecs = rsp.prof_usecs; - // this->prof_cycles = rsp.prof_cycles; - // this->prof_pkts = rsp.prof_pkts; +void ggml_hexagon_session::unclone_buffer(const ggml_hexagon_shared_buffer * sbuf) { + GGML_ASSERT(sbuf && sbuf->mem); - this->op_pending--; // atomic dec + auto mem = sbuf->mem; + std::vector sessions(mem->mapped_clones.begin(), mem->mapped_clones.end()); - if (!all) break; + for (auto * sess : sessions) { + sess->release_buffer(sbuf); } } -void ggml_hexagon_session::flush_batch() { - if (op_batch->empty()) { return; } +static size_t ggml_hexagon_measure_max_vmem(ggml_hexagon_session *sess) { + // Allocate a bunch pinned buffers till failure. + // This is kind of expensive but handy for figuring out exactly how much we can mmap on a specific device. + // Typically we're going to allocate all/most of these buffers anyway for the model weights. - htp_opbatch_req req; - req.n_bufs = op_batch->n_bufs; - req.n_tensors = op_batch->n_tens; - req.n_ops = op_batch->n_ops; + std::vector sbufs; - dspqueue_buffer dbuf; - dbuf.fd = op_shm->fd(); - dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; - dbuf.ptr = op_shm->allocate(); - if (!dbuf.ptr) { - flush_pending(false); - dbuf.ptr = op_shm->allocate(); - } + const size_t MiB = 1024 * 1024; + const size_t GiB = MiB * 1024; - dbuf.offset = (uint8_t*) dbuf.ptr - (uint8_t*) op_shm->base(); - dbuf.size = op_batch->flush((uint8_t*) dbuf.ptr, op_shm->block_size); + size_t vmem = 0; + size_t step = 256u * MiB; - // Bump pending flag (cleared in the session::flush once we get the response) - this->op_pending++; // atomic inc + try { + sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB; + sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB; + sbufs.push_back(new ggml_hexagon_shared_buffer(sess, GiB, true)); vmem += GiB; - HEX_VERBOSE("ggml-hex: %s: queue-opbatch : %p size %u\n", this->c_name(), dbuf.ptr, dbuf.size); + while (1) { + sbufs.push_back(new ggml_hexagon_shared_buffer(sess, step, true)); + vmem += step; + } + } catch (...) { } - int err = dspqueue_write(this->queue, 0, 1, &dbuf, sizeof(req), (const uint8_t*) &req, DSPQUEUE_TIMEOUT); - if (err != 0) { - GGML_ABORT("ggml-hex: %s dspqueue_write failed: 0x%08x\n", this->c_name(), (unsigned) err); - } -} + for (auto b : sbufs) { delete b; } -void ggml_hexagon_session::enqueue_op(htp_op_code opcode, const ggml_tensor *op) { - if (!op_batch->fit_op(op)) { - flush_batch(); - } - op_batch->add_op(opcode, op); + return vmem - step; // backoff to account for overhead from internal mappings } -// Flush HTP response queue i.e wait for all outstanding requests to complete -void ggml_hexagon_session::flush(bool all) { - flush_batch(); - flush_pending(all); -} +void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) noexcept(false) { + int phys_idx = config.physical_idx; + int virt_idx = config.virtual_idx; -void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { this->valid_session = false; this->valid_handle = false; this->valid_queue = false; this->valid_iface = false; - this->domain_id = 3; // Default for CDSP, updated after the session is created - this->session_id = 0; // Default for CDSP, updated after the session is created - this->dev_id = dev_id; - this->name = std::string("HTP") + std::to_string(dev_id); - - this->op_pending = 0; + this->name = config.name; + this->phys_idx = phys_idx; + this->virt_idx = virt_idx; + this->domain_id = config.domain_id; + this->session_id = 0; + this->batch_req_seq = 0; + this->batch_rsp_seq = 0; + this->last_error = HTP_STATUS_OK; - GGML_LOG_INFO("ggml-hex: allocating new session: %s\n", this->name.c_str()); + GGML_LOG_DEBUG("ggml-hex: %s allocating new session : domain %u phys-idx %u virt-idx %u\n", this->name.c_str(), this->domain_id, phys_idx, virt_idx); - domain * my_domain = get_domain(this->domain_id); - if (my_domain == NULL) { - GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP\n"); - throw std::runtime_error("ggml-hex: failed to get CDSP domain (see log for details)"); + if (config.domain_id < 0 || config.domain_name.empty()) { + GGML_LOG_ERROR("ggml-hex: %s: invalid physical CDSP core %d\n", config.name.c_str(), config.physical_idx); + throw std::runtime_error("ggml-hex: invalid physical CDSP core"); } - // Create new session - if (dev_id != 0) { - struct remote_rpc_reserve_new_session n; - n.domain_name_len = strlen(CDSP_DOMAIN_NAME); - n.domain_name = const_cast(CDSP_DOMAIN_NAME); + const std::string & dom_name = config.domain_name; + + // Create new session if virtual_idx > 0 + if (virt_idx > 0) { + struct remote_rpc_reserve_new_session n {}; + n.domain_name_len = dom_name.size(); + n.domain_name = const_cast(dom_name.c_str()); n.session_name = const_cast(this->name.c_str()); n.session_name_len = this->name.size(); + n.session_id = virt_idx; int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n)); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: failed to reserve new session %d : error 0x%x\n", dev_id, err); + GGML_LOG_ERROR("ggml-hex: %s failed to reserve new session (physical %d, virtual %d) : error 0x%x\n", + this->c_name(), phys_idx, virt_idx, err); throw std::runtime_error("ggml-hex: remote_session_control(new-sess) failed (see log for details)"); } @@ -1925,9 +2080,32 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { this->session_id = n.session_id; this->domain_id = n.effective_domain_id; this->valid_session = true; + } else { + struct remote_rpc_effective_domain_id eff {}; + eff.domain_name = const_cast(dom_name.c_str()); + eff.domain_name_len = dom_name.size(); + eff.session_id = 0; + + int err = remote_session_control(FASTRPC_GET_EFFECTIVE_DOMAIN_ID, (void *) &eff, sizeof(eff)); + if (err == AEE_SUCCESS) { + this->domain_id = eff.effective_domain_id; + } else { + GGML_LOG_DEBUG("ggml-hex: %s FASTRPC_GET_EFFECTIVE_DOMAIN_ID returned 0x%x, using domain_id %d\n", + this->name.c_str(), err, this->domain_id); + } } - // Get session URI + // Enable unsigned modules + { + struct remote_rpc_control_unsigned_module u; + u.domain = this->domain_id; + u.enable = 1; + int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); + if (err != AEE_SUCCESS) { + GGML_LOG_ERROR("ggml-hex: %s failed to enable unsigned PD : error 0x%x\n", this->c_name(), err); + throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); + } + } char session_uri[256]; { @@ -1936,8 +2114,8 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { struct remote_rpc_get_uri u = {}; u.session_id = this->session_id; - u.domain_name = const_cast(CDSP_DOMAIN_NAME); - u.domain_name_len = strlen(CDSP_DOMAIN_NAME); + u.domain_name = const_cast(dom_name.c_str()); + u.domain_name_len = dom_name.size(); u.module_uri = const_cast(htp_uri); u.module_uri_len = strlen(htp_uri); u.uri = session_uri; @@ -1945,38 +2123,54 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u)); if (err != AEE_SUCCESS) { - // fallback to single session uris - int htp_URI_domain_len = strlen(htp_uri) + MAX_DOMAIN_NAMELEN; - - snprintf(session_uri, htp_URI_domain_len, "%s%s", htp_uri, my_domain->uri); - - GGML_LOG_WARN("ggml-hex: failed to get URI for session %d : error 0x%x. Falling back to single session URI: %s\n", dev_id, err, session_uri); - } - } + snprintf(session_uri, sizeof(session_uri), "%s&_dom=%s&_session=%u", + htp_uri, dom_name.c_str(), this->session_id); - // Enable Unsigned PD - { - struct remote_rpc_control_unsigned_module u; - u.domain = this->domain_id; - u.enable = 1; - int err = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *) &u, sizeof(u)); - if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: failed to enable unsigned PD for session %d : error 0x%x\n", dev_id, err); - throw std::runtime_error("ggml-hex: remote_session_control(unsign) failed (see log for details)"); + GGML_LOG_WARN("ggml-hex: %s failed to get URI (physical %d, virtual %d) : error 0x%x. Falling back to single session URI: %s\n", + this->c_name(), phys_idx, virt_idx, err, session_uri); } } // Open session int err = htp_iface_open(session_uri, &this->handle); if (err != AEE_SUCCESS) { - GGML_LOG_ERROR("ggml-hex: failed to open session %d : error 0x%x\n", dev_id, err); + GGML_LOG_ERROR("ggml-hex: %s failed to open session : uri %s error 0x%x\n", this->c_name(), session_uri, err); throw std::runtime_error("ggml-hex: failed to open session (see log for details)"); } this->valid_handle = true; - GGML_LOG_INFO("ggml-hex: new session: %s : session-id %d domain-id %d uri %s handle 0x%lx\n", this->name.c_str(), - this->session_id, this->domain_id, session_uri, (unsigned long) this->handle); + // Query HW info and resolve session options + { + unsigned int hw_n_threads = 0; + unsigned int hw_n_hvx = 0; + unsigned int hw_n_hmx = 0; + unsigned long long hw_vtcm_size = 0; + int hw_err = htp_iface_hwinfo(this->handle, &hw_n_threads, &hw_n_hvx, &hw_n_hmx, &hw_vtcm_size); + if (hw_err == 0) { + const uint32_t max_n_threads = (std::min)((uint32_t) HTP_MAX_NTHREADS, (uint32_t) hw_n_threads); + this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) max_n_threads) : max_n_threads; + this->n_hvx = this->n_threads; + this->n_hmx = (opt_nhmx != 0) ? (uint32_t)hw_n_hmx : 0; + this->vtcm_size = (uint64_t)hw_vtcm_size; + if (opt_vtcm_size > 0) { + if (opt_vtcm_size > hw_vtcm_size) { + GGML_LOG_WARN("ggml-hex: %s VTCM override %zu exceeds HW size %llu\n", this->c_name(), opt_vtcm_size, hw_vtcm_size); + } + this->vtcm_size = opt_vtcm_size; + } + GGML_LOG_INFO("ggml-hex: %s hwinfo: threads %u, hvx %u, hmx %u, vtcm %llu MB\n", + this->c_name(), this->n_threads, this->n_hvx, this->n_hmx, + (unsigned long long)(this->vtcm_size / (1024 * 1024))); + } else { + GGML_LOG_WARN("ggml-hex: %s failed to query hwinfo (0x%x), using defaults\n", this->c_name(), hw_err); + const uint32_t default_n_threads = (std::min)(8u, (uint32_t) HTP_MAX_NTHREADS); + this->n_threads = opt_nhvx > 0 ? (uint32_t) (std::min)(opt_nhvx, (size_t) HTP_MAX_NTHREADS) : default_n_threads; + this->n_hvx = this->n_threads; + this->n_hmx = (opt_nhmx != 0) ? 1 : 0; + this->vtcm_size = opt_vtcm_size > 0 ? opt_vtcm_size : 8 * 1024 * 1024; + } + } // Enable FastRPC QoS mode { @@ -1989,6 +2183,9 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { } } + GGML_LOG_INFO("ggml-hex: %s new session : session-id %d domain-id %d uri %s handle 0x%lx\n", this->c_name(), + this->session_id, this->domain_id, session_uri, (unsigned long) this->handle); + const size_t req_q_size = (sizeof(htp_opbatch_req) * opt_opqueue * 2) + 1024; const size_t rsp_q_size = (sizeof(htp_opbatch_rsp) * opt_opqueue * 2) + 1024; @@ -2016,50 +2213,96 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) { } if (opt_etm) { - err = htp_iface_enable_etm(this->handle); + err = htp_iface_etm(this->handle, 1); if (err != 0) { GGML_LOG_ERROR("ggml-hex: failed to enable ETM tracing: 0x%08x\n", (unsigned) err); } } - // Start the DSP-side service. We need to pass the queue ID to the - // DSP in a FastRPC call; the DSP side will import the queue and start - // listening for packets in a callback. - err = htp_iface_start(this->handle, dev_id, this->queue_id, opt_nhvx, opt_use_hmx); + // Allocate buffers and state for op batching + this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue); + + this->fence_buf = new ggml_hexagon_fence_buffer(this, &dev_ctx->fence_buffer_type, 64 * 1024); + if (this->mdev.count > 1) { + this->mdev_fence_slot = this->alloc_fence(this->mdev.count); + } + + if (!opt_vmem) { + opt_vmem = ggml_hexagon_measure_max_vmem(this); + GGML_LOG_INFO("ggml-hex: %s measured max vmem %zu\n", this->c_name(), opt_vmem); + } + const size_t shm_size = this->op_queue->shm_size(); + this->max_vmem = (opt_vmem > shm_size) ? (opt_vmem - shm_size) : opt_vmem; + + this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch, this->max_vmem); + + // Start dspqueue/opbatch processing + err = htp_iface_start(this->handle, this->session_id, this->queue_id, this->n_threads, opt_nhmx, opt_vtcm_size, this->max_vmem, + (uint32_t) opt_power_mode); if (err != 0) { - GGML_LOG_ERROR("ggml-hex: failed to start session: 0x%08x\n", (unsigned) err); + GGML_LOG_ERROR("ggml-hex: %s failed to start session: 0x%08x\n", this->c_name(), (unsigned) err); throw std::runtime_error("ggml-hex: iface start failed (see log for details)"); } this->valid_iface = true; + GGML_LOG_INFO("ggml-hex: %s power mode: %s\n", this->c_name(), htp_power_mode_to_str(opt_power_mode)); - // Allocate buffers and state for op batching - this->op_batch = new ggml_hexagon_opbatch(this, opt_opbatch); - this->op_shm = new ggml_hexagon_opshm(this, opt_opbatch, opt_opqueue); + if (opt_profile) { + htp_iface_pmu_conf pmu_conf{}; + std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events); + + err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err); + } + } } void ggml_hexagon_session::release() noexcept(true) { GGML_LOG_INFO("ggml-hex: releasing session: %s\n", this->name.c_str()); - int err; + this->mdev.sessions.clear(); - delete this->op_batch; - delete this->op_shm; + int err; - // Stop the DSP-side service and close the queue if (this->valid_iface) { + // Stop dspqueue/opbatch processing err = htp_iface_stop(this->handle); if (err != 0) { GGML_ABORT("ggml-hex: htp_iface_stop failed: 0x%08x\n", (unsigned) err); } } + delete this->op_batch; + delete this->op_queue; + for (auto & it : this->cpy_fence_slots) { + free_fence((void *) it.second, 1); + } + this->cpy_fence_slots.clear(); + + if (this->fence_buf) { + unclone_buffer(this->fence_buf); + delete this->fence_buf; + this->fence_buf = nullptr; + } + while (!this->cloned_buffers.empty()) { + release_buffer(this->cloned_buffers.begin()->second.get()); + } + if (opt_etm) { - err = htp_iface_disable_etm(this->handle); + err = htp_iface_etm(this->handle, 0); if (err != 0) { GGML_LOG_ERROR("ggml-hex: warn : failed to disable ETM tracing: 0x%08x\n", (unsigned) err); } } + if (opt_profile) { + htp_iface_pmu_conf pmu_conf{}; + err = htp_iface_profiler(this->handle, 0, &pmu_conf); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: warn : failed to disable profiling: 0x%08x\n", (unsigned) err); + } + } + if (this->valid_queue) { err = dspqueue_close(queue); if (err != 0) { @@ -2072,21 +2315,24 @@ void ggml_hexagon_session::release() noexcept(true) { } } -ggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) noexcept(false) { - buffer_type.device = dev; - repack_buffer_type.device = dev; - - op_batch = nullptr; - op_shm = nullptr; +ggml_hexagon_session::ggml_hexagon_session(const ggml_hexagon_device_config & config, ggml_backend_dev_t dev, uint32_t mdev_idx, uint32_t mdev_count) noexcept(false) { + this->dev = dev; + this->dev_ctx = static_cast(dev->context); + this->mdev.idx = mdev_idx; + this->mdev.count = mdev_count > 0 ? mdev_count : (uint32_t) (1 + config.mdev_group.size()); + op_batch = nullptr; + op_queue = nullptr; + fence_buf = nullptr; + fence_seq = ((uintptr_t)this) & 0xFFFF; try { - allocate(dev_id); - - buffer_type.iface = ggml_backend_hexagon_buffer_type_interface; - buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name, this); - - repack_buffer_type.iface = ggml_backend_hexagon_repack_buffer_type_interface; - repack_buffer_type.context = new ggml_backend_hexagon_buffer_type_context(this->name + "-REPACK", this); + allocate(config); + if (this->mdev.idx == 0 && !config.mdev_group.empty()) { + for (size_t i = 0; i < config.mdev_group.size(); i++) { + this->mdev.sessions.push_back(std::make_unique( + config.mdev_group[i], this->dev, (uint32_t) (i + 1), this->mdev.count)); + } + } } catch (const std::exception & exc) { release(); throw; @@ -2095,787 +2341,566 @@ ggml_hexagon_session::ggml_hexagon_session(int dev_id, ggml_backend_dev_t dev) n ggml_hexagon_session::~ggml_hexagon_session() noexcept(true) { release(); - - delete static_cast(buffer_type.context); - delete static_cast(repack_buffer_type.context); } // ** backend interface -static bool ggml_backend_buffer_is_hexagon(const struct ggml_backend_buffer * b) { - return b->buft->iface.get_alignment == ggml_backend_hexagon_buffer_type_get_alignment; +static const char * ggml_backend_hexagon_name(ggml_backend_t backend) { + auto sess = static_cast(backend->context); + return sess->c_name(); } -static inline bool ggml_backend_buffer_is_hexagon_repack(const struct ggml_backend_buffer * b) { - if (!opt_hostbuf) { - return ggml_backend_buffer_is_hexagon(b); - } - return b->buft->iface.alloc_buffer == ggml_backend_hexagon_repack_buffer_type_alloc_buffer; +static void ggml_backend_hexagon_free(ggml_backend_t backend) { + // we just need to delete the backend here + // the sessions are allocated & freed as part of the registry + delete backend; } -static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * src3 = op->src[3]; - const struct ggml_tensor * src4 = op->src[4]; - const struct ggml_tensor * dst = op; - - // Check for F16 support only as requested - if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) || src1->type != GGML_TYPE_F16 || src2->type != GGML_TYPE_F16) { - return false; - } - - if (src3 && src3->type != GGML_TYPE_F16) { // mask - return false; - } - - if (src4 && src4->type != GGML_TYPE_F32) { // sinks - return false; - } +static inline bool op_is_compute(ggml_tensor *node) +{ + return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE); +} - // For now we support F32 or F16 output as htp backend often converts output on the fly if needed, - // but the op implementation writes to F16 or F32. - // Let's assume dst can be F32 or F16. - if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) { - return false; - } +static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { + auto sess = static_cast(backend->context); - if (dst->ne[2] != 1 || dst->ne[3] != 1) { - // FA during prompt still needs work - return false; + if (sess->last_error > HTP_STATUS_OK) { + return GGML_STATUS_FAILED; } - return true; -} + HEX_VERBOSE("ggml-hex: %s graph-compute n_nodes %d\n", sess->c_name(), graph->n_nodes); -static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) { - const struct ggml_tensor * src0 = dst->src[0]; - const struct ggml_tensor * src1 = dst->src[1]; + const std::vector * nodes_ptr = nullptr; + std::vector computed_nodes; - if (dst->type != GGML_TYPE_F32) { - return false; - } + bool cache_hit = (graph->uid != 0 && sess->cached_uid == graph->uid); + if (cache_hit) { + nodes_ptr = &sess->cached_nodes; + } else { + htp_fusion_tag_graph(graph); - if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { - return false; - } + computed_nodes.reserve(graph->n_nodes); - switch (src0->type) { - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q8_0: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - if (src0->ne[0] % 32) { - return false; + for (int i = 0; i < graph->n_nodes; ++i) { + ggml_tensor * n = graph->nodes[i]; + if (!op_is_compute(n)) { + continue; } - if (ggml_nrows(src0) > 16 * 1024) { - return false; // typically the lm-head which would be too large for VTCM + const htp_op_code opcode = htp_op_remap(n); + if (opcode == HTP_OP_INVALID) { + GGML_ABORT("\nggml-hex: graph-compute %s is not supported\n", ggml_op_desc(n)); } - if (ggml_nrows(src1) > 1024 || src1->ne[2] != 1 || src1->ne[3] != 1) { - return false; // no huge batches or broadcasting (for now) + htp_opnode node(opcode, n); + if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || node.opcode == HTP_OP_GET_ROWS) { + ggml_tensor * src0 = node.node->src[0]; + if (htp_needs_repack(src0)) { + HEX_VERBOSE("ggml-hex: %s rt-repack %s|size %zu\n", sess->c_name(), htp_format_tensor(src0, HTP_OPFMT_TENSOR).c_str(), ggml_nbytes(src0)); + std::vector raw(ggml_nbytes(src0)); + memcpy(raw.data(), src0->data, raw.size()); + htp_repack_tensor_tiled(src0, raw.data(), raw.size()); + auto * extra = (htp_tensor_extra *) src0->extra; + extra->flags |= HTP_TENSOR_TILED; + } } - // src0 (weights) must be repacked - if (src0->buffer && !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) { - return false; + GGML_ASSERT(htp_kparams_solve(sess, node)); + if (node.opcode == HTP_OP_CPY && node.kparams()->total_elems == 0) { + continue; } - break; - case GGML_TYPE_F16: - if (src0->nb[1] < src0->nb[0]) { - GGML_LOG_DEBUG("ggml_hexagon_supported_mul_mat: permuted F16 src0 not supported\n"); - return false; - } - if (ggml_nrows(src1) > 1024) { - return false; // no huge batches (for now) - } - break; + computed_nodes.push_back(std::move(node)); + } - default: - return false; + if (graph->uid != 0) { + sess->cached_uid = graph->uid; + sess->cached_nodes = std::move(computed_nodes); + nodes_ptr = &sess->cached_nodes; + } else { + nodes_ptr = &computed_nodes; + } } - return true; -} - -static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; - - if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) { - return false; + for (const auto & node : *nodes_ptr) { + sess->enqueue_op(node); } - switch (src0->type) { - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q8_0: - case GGML_TYPE_IQ4_NL: - case GGML_TYPE_MXFP4: - if ((src0->ne[0] % 32)) { - return false; - } - - // src0 (weights) must be repacked - if (src0->buffer && !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) { - return false; - } - break; - - default: - return false; + if (sess->last_error > HTP_STATUS_OK) { + return GGML_STATUS_FAILED; } - return true; + return GGML_STATUS_SUCCESS; } -static bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * dst = op; +static void ggml_backend_hexagon_synchronize(ggml_backend_t backend) { + auto sess = static_cast(backend->context); - if (src0->type == GGML_TYPE_F32) { - if (src1->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; - } - } - else if (src0->type == GGML_TYPE_F16) { - if (src1->type != GGML_TYPE_F16) { - return false; - } - if (dst->type != GGML_TYPE_F16) { - return false; - } - } - else { - return false; - } + HEX_VERBOSE("ggml-hex: %s synchronize\n", sess->c_name()); - if (!ggml_are_same_shape(src0, dst)) { - return false; - } - if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) { - return false; + // Wait until all pending ops complete + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s synchronize failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); } - - return true; } -static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * dst = op; - - if (src0->type != GGML_TYPE_F32) { - return false; - } - if (src1->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; - } - if (!ggml_are_same_shape(src0, dst)) { - return false; - } - - // REVISIT: add support for non-contigiuos tensors - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { - return false; - } +enum ggml_hexagon_mem_range_type { + HEXAGON_MEM_RANGE_TYPE_SRC, + HEXAGON_MEM_RANGE_TYPE_DST, +}; - return true; -} +struct ggml_hexagon_mem_range { + uint64_t pb; + uint64_t p0; + uint64_t p1; + ggml_hexagon_mem_range_type pt; +}; -static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; +struct ggml_hexagon_mem_ranges { + std::vector ranges; - if (src0->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; - } - if (!ggml_are_same_shape(src0, dst)) { - return false; + void reset() { + ranges.clear(); } - // TODO: add support for non-contigiuos tensors - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; + void add(const ggml_hexagon_mem_range & mr) { + ranges.push_back(mr); } - return true; -} - -static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - if (src0->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; + bool check(const ggml_hexagon_mem_range & mr) const { + for (const auto & cmp : ranges) { + if (mr.pb != cmp.pb) { + continue; + } + if (mr.pt == HEXAGON_MEM_RANGE_TYPE_SRC && cmp.pt == HEXAGON_MEM_RANGE_TYPE_SRC) { + continue; + } + if (mr.p0 < cmp.p1 && mr.p1 > cmp.p0) { + return false; + } + } + return true; } +}; - // TODO: add support for non-contigiuos tensors - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; +static ggml_hexagon_mem_range ggml_hexagon_mem_range_from_tensor(const ggml_tensor * tensor, ggml_hexagon_mem_range_type pt) { + const ggml_tensor * base = tensor->view_src ? tensor->view_src : tensor; + ggml_hexagon_mem_range mr; + if (tensor->buffer) { + mr = { + /*.pb =*/ (uint64_t) tensor->buffer, + /*.p0 =*/ (uint64_t) tensor->data, + /*.p1 =*/ (uint64_t) tensor->data + ggml_backend_buft_get_alloc_size(tensor->buffer->buft, tensor), + /*.pt =*/ pt, + }; + } else { + mr = { + /*.pb =*/ (uint64_t) base, + /*.p0 =*/ 0, + /*.p1 =*/ 1024, + /*.pt =*/ pt, + }; } - - return true; + return mr; } -static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess, - const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * dst = op; +struct node_info { + ggml_tensor * node; + htp_opvec fused; - if (src0->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; + const ggml_tensor * dst() const { return fused.empty() ? node : fused.back(); } + + bool is_empty() const { + return ggml_op_is_empty(node->op); } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; + void add_fused(ggml_tensor * t) { + fused.push_back(t); } +}; - if (src1) { - if (src1->type != GGML_TYPE_F32) { - return false; - } - if (!ggml_are_same_shape(src0, src1)) { - return false; +static void ggml_hexagon_mem_ranges_add_node(ggml_hexagon_mem_ranges & mrs, const node_info & node) { + if (node.is_empty()) return; + + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (node.node->src[i]) { + mrs.add(ggml_hexagon_mem_range_from_tensor(node.node->src[i], HEXAGON_MEM_RANGE_TYPE_SRC)); } - if (!ggml_is_contiguous(src1)) { - return false; + } + for (const auto * fused : node.fused) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (fused->src[i]) { + mrs.add(ggml_hexagon_mem_range_from_tensor(fused->src[i], HEXAGON_MEM_RANGE_TYPE_SRC)); + } } } - - return true; + mrs.add(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST)); } -static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; - - if (src2) { - return false; // FIXME: add support for sinks - } - - if (src0->type != GGML_TYPE_F32) { - return false; - } - if (dst->type != GGML_TYPE_F32) { - return false; - } +static bool ggml_hexagon_mem_ranges_check_node(const ggml_hexagon_mem_ranges & mrs, const node_info & node) { + if (node.is_empty()) return true; - if (src1) { - if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { - return false; - } - if (src0->ne[0] != src1->ne[0]) { - return false; - } - if (src1->ne[1] < src0->ne[1]) { - return false; - } - if (src0->ne[2] % src1->ne[2] != 0) { - return false; - } - if (src0->ne[3] % src1->ne[3] != 0) { - return false; + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (node.node->src[i]) { + if (!mrs.check(ggml_hexagon_mem_range_from_tensor(node.node->src[i], HEXAGON_MEM_RANGE_TYPE_SRC))) { + return false; + } } } - - if (src1) { - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { - return false; - } - } else { - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; + for (const auto * fused : node.fused) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (fused->src[i]) { + if (!mrs.check(ggml_hexagon_mem_range_from_tensor(fused->src[i], HEXAGON_MEM_RANGE_TYPE_SRC))) { + return false; + } + } } } + return mrs.check(ggml_hexagon_mem_range_from_tensor(node.dst(), HEXAGON_MEM_RANGE_TYPE_DST)); +} - // Reject non-HVX-aligned sizes when ne[0] > HVX_F32_LANES - // The HVX softmax implementation has issues with tail handling for larger non-aligned sizes - // Small sizes (ne[0] <= 32) work correctly with tail-only processing - const int64_t ne0 = src0->ne[0]; - if (ne0 > 32 && (ne0 & (32 - 1)) != 0) { - return false; - } +static std::vector ggml_hexagon_graph_optimize_reorder(const std::vector & nodes) { + const int n = nodes.size(); - // HVX vector size constraints for softmax - #define SOFTMAX_MAX_ROW_SIZE 131072 // 128K elements max for numerical precision + std::vector res; + res.reserve(n); - // Reject very large row sizes to avoid numerical precision issues - // Softmax accumulation over many elements can lead to precision loss - if (ne0 > SOFTMAX_MAX_ROW_SIZE) { - return false; - } + std::vector used(n, false); - return true; -} + ggml_hexagon_mem_ranges mrs; -static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; // values - const struct ggml_tensor * src1 = op->src[1]; // indices - const struct ggml_tensor * dst = op; + // The main goal here is to stack the MUL_MAT ops with the same src1 input. + // This allows us to reuse dynamically quantized src1 in VTCM. - if (src0->type != GGML_TYPE_F32) { - return false; - } + for (int i0 = 0; i0 < n; i0++) { + if (used[i0]) { + continue; + } - if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) { - return false; - } + const auto & node0 = nodes[i0]; - if (dst->type != GGML_TYPE_F16) { - return false; - } + if (!htp_fusion_can_stack(node0.node)) { + res.push_back(i0); + used[i0] = true; + continue; + } - return true; -} + // that many nodes forward to search for stackable nodes that can reuse VTCM + constexpr int N_FORWARD = 64; -static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; // values - const struct ggml_tensor * src1 = op->src[1]; // indices - const struct ggml_tensor * dst = op; + htp_opvec stack; + stack.push_back(i0); - if (src0->type != GGML_TYPE_F32) { - return false; - } + mrs.reset(); - if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) { - return false; - } + for (int i1 = i0 + 1; i1 < i0 + N_FORWARD && i1 < n; i1++) { + if (used[i1]) { + continue; + } - if (dst->type != GGML_TYPE_F32) { - return false; + const auto & node1 = nodes[i1]; + + if (htp_fusion_can_stack(node0.node, node1.node) && ggml_hexagon_mem_ranges_check_node(mrs, node1)) { + stack.push_back(i1); + } else { + ggml_hexagon_mem_ranges_add_node(mrs, node1); + } + } + + for (int idx : stack) { + res.push_back(idx); + used[idx] = true; + } } - return true; + return res; } -static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; // values - const struct ggml_tensor * dst = op; // indices +static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgraph * gf, ggml_backend_graph_optimize_params * params) { + GGML_UNUSED(params); - if (src0->type != GGML_TYPE_F32) { - return false; - } + const int n = gf->n_nodes; - if (dst->type != GGML_TYPE_I32) { - return false; - } - - if (src0->ne[0] > (16*1024)) { - // reject tensors with huge rows for now - return false; - } - - return true; -} + std::vector nodes; + nodes.reserve(gf->n_nodes); -static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const int32_t * op_params = &op->op_params[0]; + // Pack nodes for reordering + for (int i = 0; i < n; i++) { + node_info node = { gf->nodes[i], {} }; - int mode = op_params[2]; + const int f = htp_fusion_max(gf, i); + for (int k = 1; k < f; k++) { + node.add_fused(gf->nodes[++i]); + } - if ((mode & GGML_ROPE_TYPE_MROPE) || (mode & GGML_ROPE_TYPE_VISION)) { - return false; - } - if (mode & 1) { - return false; + nodes.push_back(std::move(node)); } - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; + const auto order = ggml_hexagon_graph_optimize_reorder(nodes); - if (src0->type != GGML_TYPE_F32) { - return false; // FIXME: add support for GGML_TYPE_F16 for src0 - } - if (dst->type != GGML_TYPE_F32) { - return false; - } - if (src1->type != GGML_TYPE_I32) { - return false; - } - if (src2) { - if (src2->type != GGML_TYPE_F32) { - return false; - } - int n_dims = op_params[1]; - if (src2->ne[0] < (n_dims / 2)) { - return false; - } - } + // unfuse + { + int j = 0; + for (const auto i : order) { + const auto & node = nodes[i]; - if (src2) { - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(src2) || - !ggml_is_contiguous(dst)) { - return false; - } - } else { - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { - return false; + gf->nodes[j++] = node.node; + + for (auto * fused : node.fused) { + gf->nodes[j++] = fused; + } } } - return true; + GGML_UNUSED(backend); } -static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * dst = op; +static uint64_t ggml_hexagon_session_key(const ggml_hexagon_session * sess) { + return ((uint64_t) (uint32_t) sess->phys_idx << 32) | (uint32_t) sess->virt_idx; +} - // Only support FP32 for now - if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; - } +static bool ggml_hexagon_cpy_tensor_async_phys(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { + auto sess_src = static_cast(backend_src->context); + auto sess_dst = static_cast(backend_dst->context); - // Check IO tensor shapes and dims - if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) { - return false; // src0 should be effectively 3D - } + sess_src->mmap_tensor(src); + auto sbuf_dst = sess_dst->mmap_tensor(dst); - const int d_conv = src1->ne[0]; - const int d_inner = src0->ne[1]; - const int n_t = dst->ne[1]; - const int n_s = dst->ne[2]; + if (!sess_src->clone_buffer(sbuf_dst)) { return false; } - if (src0->ne[0] != d_conv - 1 + n_t || src0->ne[1] != d_inner || src0->ne[2] != n_s) { - return false; - } - if (src1->ne[0] != d_conv || src1->ne[1] != d_inner) { - return false; - } - if (dst->ne[0] != d_inner || dst->ne[1] != n_t || dst->ne[2] != n_s) { - return false; + const uint64_t src_key = ggml_hexagon_session_key(sess_src); + auto & fence_slot = sess_dst->cpy_fence_slots[src_key]; + if (!fence_slot) { + fence_slot = (volatile uint32_t *) sess_dst->alloc_fence(1); } - // TODO: add support for non-contiguous tensors - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { - return false; - } + if (!sess_src->clone_buffer(sess_dst->fence_buf)) { return false; } - return true; -} + uint32_t fence_seq = sess_dst->next_fence_seq(); -static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; + HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu : seq 0x%x\n", + sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src), fence_seq); - if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - return false; - } + ggml_tensor fence_tensor = sess_dst->make_fence_tensor(fence_slot); - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { - return false; - } + sess_src->enqueue_cpy(src, dst, &fence_tensor, fence_seq); + sess_dst->enqueue_fence(&fence_tensor, fence_seq, /* wait = */ true); + + sess_dst->add_peer(sess_src); - GGML_UNUSED(sess); return true; } -static const char * ggml_backend_hexagon_name(ggml_backend_t backend) { - auto sess = static_cast(backend->context); - return sess->c_name(); -} +static bool ggml_hexagon_cpy_tensor_async_virt(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { + auto sess_src = static_cast(backend_src->context); + auto sess_dst = static_cast(backend_dst->context); -static void ggml_backend_hexagon_free(ggml_backend_t backend) { - // we just need to delete the backend here - // the sessions are allocated & freed as part of the registry - delete backend; -} + auto sbuf_src = sess_src->mmap_tensor(src); + sess_dst->mmap_tensor(dst); -static htp_op_code op_remap_to_htp(const ggml_tensor * t) { - switch (t->op) { - case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT; - case GGML_OP_MUL_MAT: return HTP_OP_MUL_MAT; - case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID; - case GGML_OP_MUL: return HTP_OP_MUL; - case GGML_OP_ADD: return HTP_OP_ADD; - case GGML_OP_ADD_ID: return HTP_OP_ADD_ID; - case GGML_OP_SUB: return HTP_OP_SUB; - case GGML_OP_DIV: return HTP_OP_DIV; - case GGML_OP_CPY: return HTP_OP_CPY; - case GGML_OP_CONT: return HTP_OP_CPY; - case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS; - case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS; - case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS; - case GGML_OP_ARGSORT: return HTP_OP_ARGSORT; - case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM; - case GGML_OP_SCALE: return HTP_OP_SCALE; - case GGML_OP_SQR: return HTP_OP_SQR; - case GGML_OP_SQRT: return HTP_OP_SQRT; - case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX; - case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV; - case GGML_OP_ROPE: return HTP_OP_ROPE; - case GGML_OP_REPEAT: return HTP_OP_REPEAT; - case GGML_OP_CUMSUM: return HTP_OP_CUMSUM; - - case GGML_OP_UNARY: - switch (ggml_get_unary_op(t)) { - case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU; - case GGML_UNARY_OP_GELU: return HTP_OP_UNARY_GELU; - case GGML_UNARY_OP_SIGMOID: return HTP_OP_UNARY_SIGMOID; - case GGML_UNARY_OP_NEG: return HTP_OP_UNARY_NEG; - case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP; - case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS; - default: - break; - } - break; + if (!sess_dst->clone_buffer(sbuf_src)) { return false; } - case GGML_OP_GLU: - switch (ggml_get_glu_op(t)) { - case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU; - case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI; - case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU; - default: break; - } - break; + HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", + sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); - default: - GGML_ABORT("\nggml-hex: graph-compute %s is not supported\n", ggml_op_desc(t)); - } - return HTP_OP_INVALID; -} + sess_dst->enqueue_cpy(src, dst); + sess_dst->add_peer(sess_src); -static inline bool op_is_compute(ggml_tensor *node) -{ - return !ggml_op_is_empty(node->op) && !ggml_is_empty(node) && (node->flags & GGML_TENSOR_FLAG_COMPUTE); + return true; } -static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, ggml_cgraph * graph) { - auto sess = static_cast(backend->context); - - HEX_VERBOSE("ggml-hex: %s graph-compute n_nodes %d\n", sess->c_name(), graph->n_nodes); +static bool ggml_backend_hexagon_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) { + if (!ggml_backend_is_hexagon(backend_src) || !ggml_backend_is_hexagon(backend_dst)) { + return false; + } - for (int i = 0; i < graph->n_nodes; ++i) { - ggml_tensor * n = graph->nodes[i]; - if (op_is_compute(n)) { - sess->enqueue_op(op_remap_to_htp(n), n); - } + // FIXME: ggml-meta needs to call init_tensor on auxiliary tensors + if (!dst->extra) { + ggml_backend_buffer_init_tensor(dst->buffer, dst); } - // Wait until all pending ops complete - sess->flush(); + auto * dst_extra = static_cast(dst->extra); + const auto * src_extra = static_cast(src->extra); + dst_extra->flags = src_extra->flags & ~HTP_TENSOR_FUSEABLE; - return GGML_STATUS_SUCCESS; -} + auto sess_src = static_cast(backend_src->context); + auto sess_dst = static_cast(backend_dst->context); -static void ggml_backend_hexagon_synchronize(ggml_backend_t backend) { - auto sess = static_cast(backend->context); + if (sess_src == sess_dst) { + HEX_VERBOSE("ggml-hex: %s cpy-tensor-async %s -> %s size %zu\n", sess_dst->name.c_str(), src->name, dst->name, ggml_nbytes(src)); + sess_src->enqueue_cpy(src, dst); + return true; + } - HEX_VERBOSE("ggml-hex: %s synchronize\n", sess->c_name()); + if (sess_src->phys_idx != sess_dst->phys_idx) + return ggml_hexagon_cpy_tensor_async_phys(backend_src, backend_dst, src, dst); - // Wait until all pending ops complete - sess->flush(); + return ggml_hexagon_cpy_tensor_async_virt(backend_src, backend_dst, src, dst); } -struct node_info { - ggml_tensor * node; +static ggml_backend_event_t ggml_backend_hexagon_device_event_new(ggml_backend_dev_t dev) { + auto dev_ctx = static_cast(dev->context); + auto sess = dev_ctx->session(); - std::vector fused; + ggml_hexagon_event * hex_event = new ggml_hexagon_event(); + hex_event->fence_sess = sess; + hex_event->sess = sess; + hex_event->fence_slot = (volatile uint32_t *) sess->alloc_fence(1); + hex_event->fence_tensor = sess->make_fence_tensor(hex_event->fence_slot); - ggml_op op() const { - return node->op; - } + HEX_VERBOSE("ggml-hex: %s event-new : event %p fence %p\n", ggml_backend_dev_name(dev), (void *)hex_event, (void *)hex_event->fence_slot); - const ggml_tensor * dst() const { - return fused.empty() ? node : fused.back(); - } + return new ggml_backend_event { + /* .device = */ dev, + /* .context = */ hex_event, + }; +} - const ggml_tensor * src0() const { - return node->src[0]; +static void ggml_hexagon_event_synchronize(ggml_backend_dev_t dev, ggml_hexagon_event * hex_event) { + if (hex_event->seq == 0) { + return; } - const ggml_tensor * src1() const { - return node->src[1]; - } + HEX_VERBOSE("ggml-hex: %s event-synchronize : event %p seq 0x%x fence %p\n", + ggml_backend_dev_name(dev), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); - bool is_empty() const { - return ggml_op_is_empty(node->op); - } + auto * fence = reinterpret_cast *>(hex_event->fence_slot); - void add_fused(ggml_tensor * t) { - fused.push_back(t); + if ((int32_t)(fence[0].load(std::memory_order_relaxed) - hex_event->seq) < 0) { + hex_event->sess->flush_async(); } - bool stackable() const { - switch (this->op()) { - case GGML_OP_MUL_MAT: - case GGML_OP_MUL_MAT_ID: - return ggml_is_quantized(this->src0()->type); - default: - return false; - } - } - - bool same_input(const node_info& n) const { - return n.src1() == this->src1(); - } -}; - -static std::vector ggml_hexagon_graph_optimize_reorder(const std::vector & nodes) { - const int n = nodes.size(); - - std::vector res; - res.reserve(n); - - std::vector used(n, false); - - // The main goal here is to stack the MUL_MAT ops with the same src1 input. - // This allows use to reuse dynamically quantized src1 in VTCM. - - // TODO: the current version might do incorrect reordering in cases where quantized src0 - // input is an output of another Op. - - for (int i0 = 0; i0 < n; i0++) { - if (used[i0]) { - continue; - } - - res.push_back(i0); - - const auto & node0 = nodes[i0]; - - if (!node0.stackable()) { - continue; - } - - // that many nodes forward to search for stackable nodes that can reuse VTCM - constexpr int N_FORWARD = 16; - - for (int i1 = i0 + 1; i1 < i0 + N_FORWARD && i1 < n; i1++) { - if (used[i1]) { - continue; - } - - const auto & node1 = nodes[i1]; - - if (node1.stackable() && node1.same_input(node0)) { - res.push_back(i1); - used[i1] = true; + while (true) { + if ((int32_t)(fence[0].load(std::memory_order_acquire) - hex_event->seq) >= 0) { + uint32_t status = fence[1].load(std::memory_order_acquire); + if (status > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s event-synchronize failed : dsp-error %s\n", + hex_event->sess->c_name(), status_to_str(status)); } + break; } + std::this_thread::yield(); } +} - return res; +static void ggml_backend_hexagon_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { + auto * hex_event = static_cast(event->context); + ggml_hexagon_event_synchronize(dev, hex_event); + HEX_VERBOSE("ggml-hex: %s event-free : event %p\n", ggml_backend_dev_name(dev), (void *)hex_event); + hex_event->fence_sess->free_fence((void *) hex_event->fence_slot, 1); + delete hex_event; + delete event; } -static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgraph * gf) { - const int n = gf->n_nodes; +static void ggml_backend_hexagon_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) { + auto * hex_event = static_cast(event->context); + ggml_hexagon_event_synchronize(dev, hex_event); +} - constexpr int MAX_FUSE = 16; +static void ggml_backend_hexagon_event_record(ggml_backend_t backend, ggml_backend_event_t event) { + auto sess = static_cast(backend->context); + auto hex_event = static_cast(event->context); - enum ggml_op ops[MAX_FUSE]; + hex_event->sess = sess; + hex_event->seq = sess->next_fence_seq(); - std::vector nodes; - nodes.reserve(gf->n_nodes); + sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ false); - // fuse nodes: - // we don't want to make reorders that break fusing, so we first pack all fusable tensors - // and perform the reorder over the fused nodes. after the reorder is done, we unfuse - for (int i = 0; i < n; i++) { - node_info node = { - /*.node =*/gf->nodes[i], - /*.fused =*/{}, - }; + HEX_VERBOSE("ggml-hex: %s event-record : event %p seq 0x%x fence %p\n", + sess->c_name(), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); +} - // fuse only ops that start with these operations - // can be expanded when needed - if (node.op() == GGML_OP_ADD || - node.op() == GGML_OP_NORM || - node.op() == GGML_OP_RMS_NORM) { - ops[0] = node.op(); - - int f = i + 1; - while (f < n && f < i + MAX_FUSE) { - // conservatively allow fusing only these ops - // can be expanded when needed - if (gf->nodes[f]->op != GGML_OP_ADD && - gf->nodes[f]->op != GGML_OP_MUL && - gf->nodes[f]->op != GGML_OP_NORM && - gf->nodes[f]->op != GGML_OP_RMS_NORM) { - break; - } - ops[f - i] = gf->nodes[f]->op; - f++; - } +static void ggml_backend_hexagon_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { + auto sess = static_cast(backend->context); + auto hex_event = static_cast(event->context); - f -= i; - for (; f > 1; f--) { - if (ggml_can_fuse(gf, i, ops, f)) { - break; - } - } + if (hex_event->seq == 0) { + return; + } - // add the fused tensors into the node info so we can unfuse them later - for (int k = 1; k < f; k++) { - ++i; + HEX_VERBOSE("ggml-hex: %s event-wait : event %p seq 0x%x fence %p\n", + sess->c_name(), (void *)hex_event, hex_event->seq, (void *)hex_event->fence_slot); - // the .dst() becomes the last fused tensor - node.add_fused(gf->nodes[i]); - } + // same physical NPU runs sequentially in FIFO order + if (sess->phys_idx == hex_event->sess->phys_idx) { + if (sess != hex_event->sess) { + sess->add_peer(hex_event->sess); } - - nodes.push_back(std::move(node)); + return; } - const auto order = ggml_hexagon_graph_optimize_reorder(nodes); - - // unfuse - { - int j = 0; - for (const auto i : order) { - const auto & node = nodes[i]; + sess->clone_buffer(hex_event->fence_sess->fence_buf); + sess->add_peer(hex_event->sess); + sess->enqueue_fence(&hex_event->fence_tensor, hex_event->seq, /* wait = */ true); +} - gf->nodes[j++] = node.node; +static void ggml_backend_hexagon_set_tensor_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + auto sess = static_cast(backend->context); + HEX_VERBOSE("ggml-hex: %s set-tensor-async %s : data %p offset %zu size %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, tensor->buffer ? (int) tensor->buffer->usage : -1); + ggml_backend_tensor_set(tensor, data, offset, size); +} - for (auto * fused : node.fused) { - gf->nodes[j++] = fused; - } - } +static void ggml_backend_hexagon_get_tensor_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) { + auto sess = static_cast(backend->context); + HEX_VERBOSE("ggml-hex: %s get-tensor-async %s : data %p offset %zu size %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, tensor->buffer ? (int) tensor->buffer->usage : -1); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s get-tensor-async failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); + } + ggml_backend_tensor_get(tensor, data, offset, size); +} + +static void ggml_backend_hexagon_set_tensor_2d_async(ggml_backend_t backend, + struct ggml_tensor * tensor, + const void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto sess = static_cast(backend->context); + HEX_VERBOSE("ggml-hex: %s set-tensor-2d-async %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, tensor->buffer ? (int) tensor->buffer->usage : -1); + ggml_backend_tensor_set_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); +} + +static void ggml_backend_hexagon_get_tensor_2d_async(ggml_backend_t backend, + const struct ggml_tensor * tensor, + void * data, + size_t offset, + size_t size, + size_t n_copies, + size_t stride_tensor, + size_t stride_data) { + auto sess = static_cast(backend->context); + HEX_VERBOSE("ggml-hex: %s get-tensor-2d-async %s : data %p offset %zu size %zu n_copies %zu stride_tensor %zu stride_data %zu usage %d\n", + sess->c_name(), tensor->name, data, offset, size, n_copies, stride_tensor, stride_data, tensor->buffer ? (int) tensor->buffer->usage : -1); + sess->flush_sync(); + if (sess->last_error > HTP_STATUS_OK) { + GGML_ABORT("ggml-hex: %s get-tensor-2d-async failed : dsp-error %s\n", sess->c_name(), status_to_str(sess->last_error)); } + ggml_backend_tensor_get_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); } static struct ggml_backend_i hexagon_backend_i = { /* .get_name = */ ggml_backend_hexagon_name, /* .free = */ ggml_backend_hexagon_free, - /* .set_tensor_async = */ NULL, - /* .get_tensor_async = */ NULL, - /* .get_tensor_2d_async = */ NULL, - /* .set_tensor_2d_async = */ NULL, - /* .cpy_tensor_async = */ NULL, + /* .set_tensor_async = */ ggml_backend_hexagon_set_tensor_async, + /* .get_tensor_async = */ ggml_backend_hexagon_get_tensor_async, + /* .set_tensor_2d_async = */ ggml_backend_hexagon_set_tensor_2d_async, + /* .get_tensor_2d_async = */ ggml_backend_hexagon_get_tensor_2d_async, + /* .cpy_tensor_async = */ ggml_backend_hexagon_cpy_tensor_async, /* .synchronize = */ ggml_backend_hexagon_synchronize, /* .graph_plan_create = */ NULL, /* .graph_plan_free = */ NULL, /* .graph_plan_update = */ NULL, /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_hexagon_graph_compute, - /* .event_record = */ NULL, - /* .event_wait = */ NULL, + /* .event_record = */ ggml_backend_hexagon_event_record, + /* .event_wait = */ ggml_backend_hexagon_event_wait, /* .graph_optimize = */ ggml_backend_hexagon_graph_optimize, }; @@ -2892,7 +2917,8 @@ bool ggml_backend_is_hexagon(ggml_backend_t backend) { // device interface static ggml_backend_t ggml_backend_hexagon_device_init(ggml_backend_dev_t dev, const char * params) { - auto sess = static_cast(dev->context); + auto dev_ctx = static_cast(dev->context); + auto sess = dev_ctx->session(); return new ggml_backend{ /* .guid = */ ggml_backend_hexagon_guid(), @@ -2905,8 +2931,8 @@ static ggml_backend_t ggml_backend_hexagon_device_init(ggml_backend_dev_t dev, c } static const char * ggml_backend_hexagon_device_get_name(ggml_backend_dev_t dev) { - auto sess = static_cast(dev->context); - return sess->c_name(); + auto dev_ctx = static_cast(dev->context); + return dev_ctx->c_name(); GGML_UNUSED(dev); } @@ -2917,8 +2943,8 @@ static const char * ggml_backend_hexagon_device_get_description(ggml_backend_dev } static void ggml_backend_hexagon_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { - *free = 0; - *total = *free; + *free = opt_mbuf_total; + *total = opt_mbuf_total; GGML_UNUSED(dev); } @@ -2936,113 +2962,38 @@ static void ggml_backend_hexagon_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_hexagon_device_get_memory(dev, &props->memory_free, &props->memory_total); props->caps = { /* .async = */ true, - /* .host_buffer = */ (bool) opt_hostbuf, + /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, - /* .events = */ false, + /* .events = */ true, + /* .mmap_support = */ false, }; } static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_buffer_type(ggml_backend_dev_t dev) { - auto sess = static_cast(dev->context); - return &sess->buffer_type; -} - -static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_repack_buffer_type(ggml_backend_dev_t dev) { - auto sess = static_cast(dev->context); - return &sess->repack_buffer_type; + auto dev_ctx = static_cast(dev->context); + return &dev_ctx->buffer_type; } -static bool ggml_hexagon_supported_buffer(ggml_hexagon_session *sess, const struct ggml_tensor * t) { - if (t && t->buffer) { - if (ggml_backend_buffer_is_hexagon(t->buffer) == false) return false; // not our buffer - if (ggml_backend_hexagon_buffer_get_sess(t->buffer) != sess) return false; // wrong session - } - return true; -} - -static bool ggml_hexagon_supported_buffers(ggml_hexagon_session *sess, const struct ggml_tensor * t) { - // all srcs & dsts must be mapped to the same session - if (!ggml_hexagon_supported_buffer(sess, t)) { - return false; - } - - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (!ggml_hexagon_supported_buffer(sess, t->src[i])) { - return false; - } +static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_host_buffer_type(ggml_backend_dev_t dev) { + if (!opt_hostbuf) { + return NULL; } - - return true; -} - -static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - // for now we can do f32 -> f16 and f16 -> f32 (without reshaping) - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) return false; - if ( dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) return false; - - const bool sametype = (src0->type == dst->type); - const bool transposed = ggml_is_transposed(src0) || ggml_is_transposed(dst); - const bool sameshape = !transposed && ggml_are_same_shape(src0, dst); - - // can handle any shape and any same-type (pretty slow if reshaping is required) - if (sametype) return true; - - // cannot handle re-shaping and type conversion at the same time - if (!sameshape) return false; - - return true; -} - -static bool ggml_hexagon_supported_cont(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - GGML_UNUSED(sess); - const struct ggml_tensor * src0 = op->src[0]; - - // CONT is same-type only, supports f32 and f16 - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) return false; - - return true; -} - -static bool ggml_hexagon_supported_repeat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - GGML_UNUSED(sess); - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * dst = op; - - // Support f32 and f16 - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) return false; - - // src and dst must be the same type - if (src0->type != dst->type) return false; - - // dst dims must be multiples of src dims - if (dst->ne[0] % src0->ne[0] != 0) return false; - if (dst->ne[1] % src0->ne[1] != 0) return false; - if (dst->ne[2] % src0->ne[2] != 0) return false; - if (dst->ne[3] % src0->ne[3] != 0) return false; - - // require contiguous tensors (no transposition) - if (ggml_is_transposed(src0) || ggml_is_transposed(dst)) return false; - - return true; + auto dev_ctx = static_cast(dev->context); + return &dev_ctx->host_buffer_type; } +// IMPORTANT: supports_op runs before buffer allocation and before tensor data is populated. +// Never check buffer usage, buffer properties, or repacking state (is_tiled) here. +// Support must depend only on static tensor properties (shape, type, strides). static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { - auto sess = static_cast(dev->context); + auto dev_ctx = static_cast(dev->context); + auto sess = dev_ctx->session(); // reject ops that match the filter if (opt_opfilter && std::regex_match(ggml_op_desc(op), *opt_opfilter)) { return false; } - // all srcs & dsts must be mapped to the same session - if (!ggml_hexagon_supported_buffers(sess, op)) { - ggml_hexagon_dump_op_supp(sess->name, op, false); - return false; - } - bool supp = false; switch (op->op) { case GGML_OP_NONE: @@ -3053,146 +3004,24 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons supp = true; break; - case GGML_OP_MUL: - case GGML_OP_ADD: - case GGML_OP_SUB: - case GGML_OP_DIV: - supp = ggml_hexagon_supported_binary(sess, op); - break; - - case GGML_OP_MUL_MAT: - supp = ggml_hexagon_supported_mul_mat(sess, op); - break; - - case GGML_OP_MUL_MAT_ID: - supp = ggml_hexagon_supported_mul_mat_id(sess, op); - break; - - case GGML_OP_ADD_ID: - supp = ggml_hexagon_supported_add_id(sess, op); - break; - - case GGML_OP_RMS_NORM: - case GGML_OP_SCALE: - supp = ggml_hexagon_supported_unary(sess, op); - break; - - case GGML_OP_SQR: - case GGML_OP_SQRT: - supp = ggml_hexagon_supported_unary(sess, op); - break; - - case GGML_OP_SUM_ROWS: - supp = ggml_hexagon_supported_sum_rows(sess, op); - break; - - case GGML_OP_SOFT_MAX: - supp = ggml_hexagon_supported_softmax(sess, op); - break; - - case GGML_OP_UNARY: - switch (ggml_get_unary_op(op)) { - case GGML_UNARY_OP_NEG: - case GGML_UNARY_OP_EXP: - case GGML_UNARY_OP_SIGMOID: - case GGML_UNARY_OP_SOFTPLUS: - supp = ggml_hexagon_supported_unary(sess, op); - break; - case GGML_UNARY_OP_SILU: - case GGML_UNARY_OP_GELU: - supp = ggml_hexagon_supported_activations(sess, op); - break; - default: - break; - } - break; - - case GGML_OP_GLU: - switch (ggml_get_glu_op(op)) { - case GGML_GLU_OP_SWIGLU: - case GGML_GLU_OP_SWIGLU_OAI: - case GGML_GLU_OP_GEGLU: - supp = ggml_hexagon_supported_activations(sess, op); - break; - default: - break; - } - break; - - case GGML_OP_ROPE: - supp = ggml_hexagon_supported_rope(sess, op); - break; - - case GGML_OP_FLASH_ATTN_EXT: - supp = ggml_hexagon_supported_flash_attn_ext(sess, op); - break; - - case GGML_OP_SET_ROWS: - supp = ggml_hexagon_supported_set_rows(sess, op); - break; - - case GGML_OP_GET_ROWS: - supp = ggml_hexagon_supported_get_rows(sess, op); - break; - - case GGML_OP_CPY: - supp = ggml_hexagon_supported_cpy(sess, op); - break; - - case GGML_OP_CONT: - supp = ggml_hexagon_supported_cont(sess, op); - break; - - case GGML_OP_REPEAT: - supp = ggml_hexagon_supported_repeat(sess, op); - break; - - case GGML_OP_ARGSORT: - supp = ggml_hexagon_supported_argsort(sess, op); - break; - - case GGML_OP_SSM_CONV: - supp = ggml_hexagon_supported_ssm_conv(sess, op); - break; - - case GGML_OP_CUMSUM: - supp = ggml_hexagon_supported_cumsum(sess, op); - break; - default: + supp = htp_kparams_check(sess, op); break; } - ggml_hexagon_dump_op_supp(sess->name, op, supp); + ggml_hexagon_dump_op_supp(sess->c_name(), op, supp); return supp; } static bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - if (buft->iface.get_alignment != ggml_backend_hexagon_buffer_type_get_alignment) { - return false; - } - - auto s0 = static_cast(dev->context); - auto s1 = static_cast(buft->context)->sess; + auto dev_ctx = static_cast(dev->context); - // Need session/domain-id for buffers to be compatible - bool supp = (s0->session_id == s1->session_id); - - HEX_VERBOSE("ggml-hex: %s device-supports-buft %s (%d)\n", s0->name.c_str(), s1->name.c_str(), (int) supp); + bool supp = (buft == &dev_ctx->host_buffer_type) || (buft == &dev_ctx->buffer_type); + HEX_VERBOSE("ggml-hex: %s device-supports-buft %s %s\n", dev_ctx->c_name(), ggml_backend_buft_name(buft), supp ? "yes" : "no"); return supp; } -static ggml_backend_buffer_type_t * ggml_backend_hexagon_device_get_extra_buffers_type(ggml_backend_dev_t dev) { - auto s0 = static_cast(dev->context); - HEX_VERBOSE("ggml-hex: device-get-extra-buft : %s \n", s0->name.c_str()); - - static ggml_backend_buffer_type_t bufts[2]; - bufts[0] = ggml_backend_hexagon_device_get_repack_buffer_type(dev); - bufts[1] = NULL; - return bufts; -} - static const struct ggml_backend_device_i ggml_backend_hexagon_device_i = { /* .get_name = */ ggml_backend_hexagon_device_get_name, /* .get_description = */ ggml_backend_hexagon_device_get_description, @@ -3201,67 +3030,52 @@ static const struct ggml_backend_device_i ggml_backend_hexagon_device_i = { /* .get_props = */ ggml_backend_hexagon_device_get_props, /* .init_backend = */ ggml_backend_hexagon_device_init, /* .get_buffer_type = */ ggml_backend_hexagon_device_get_buffer_type, - /* .get_host_buffer_type = */ NULL, // ggml_backend_hexagon_device_get_host_buffer_type, + /* .get_host_buffer_type = */ ggml_backend_hexagon_device_get_host_buffer_type, /* .buffer_from_host_ptr = */ NULL, // ggml_backend_hexagon_device_buffer_from_ptr, /* .supports_op = */ ggml_backend_hexagon_device_supports_op, /* .supports_buft = */ ggml_backend_hexagon_device_supports_buft, /* .offload_op = */ NULL, // ggml_backend_hexagon_device_offload_op, - /* .event_new = */ NULL, - /* .event_free = */ NULL, - /* .event_synchronize = */ NULL, + /* .event_new = */ ggml_backend_hexagon_device_event_new, + /* .event_free = */ ggml_backend_hexagon_device_event_free, + /* .event_synchronize = */ ggml_backend_hexagon_device_event_synchronize, }; //** backend registry -#define GGML_HEXAGON_MAX_SESSIONS 16 - -struct ggml_hexagon_registry { - ggml_hexagon_registry(ggml_backend_reg_t reg); - ~ggml_hexagon_registry(); - - ggml_backend_device devices[GGML_HEXAGON_MAX_SESSIONS]; -}; - ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) { GGML_LOG_INFO("ggml-hex: Hexagon backend (experimental) : allocating new registry : ndev %zu\n", opt_ndev); - if (!opt_arch) { - int err = get_hex_arch_ver(CDSP_DOMAIN_ID, &opt_arch); - if (err != 0) { - GGML_LOG_ERROR("ggml-hex: failed to query HTP version (err %d) defaulting to v73\n", err); - opt_arch = 73; - } - } - -#if defined(__ANDROID__) - if (opt_arch < 75) { - opt_ndev = 1; - GGML_LOG_WARN("ggml-hex: forcing ndev to 1 for SoCs archs lower than v75.\n"); - } -#endif - - GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d\n", opt_arch); + GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d, DMA64 %s\n", opt_arch, opt_dma64 ? "enabled" : "disabled"); - // Create devices / sessions + // Create devices for (size_t i = 0; i < opt_ndev; i++) { - devices[i].iface = ggml_backend_hexagon_device_i; - devices[i].reg = reg; - try { - devices[i].context = new ggml_hexagon_session(i, &devices[i]); - } catch (const std::exception & exc) { - GGML_LOG_ERROR("ggml-hex: failed to create device/session %zu\n", i); - devices[i].context = nullptr; + const auto & cfg = opt_device_configs[i]; + if (cfg.mdev_group.empty()) { + GGML_LOG_INFO("ggml-hex: device %zu: %s (phys=%d, virt=%d, domain=%s:%d)\n", + i, cfg.name.c_str(), cfg.physical_idx, cfg.virtual_idx, cfg.domain_name.c_str(), cfg.domain_id); + } else { + std::string peers_str; + for (const auto & p : cfg.mdev_group) { + if (!peers_str.empty()) peers_str += ", "; + peers_str += p.name + " (phys=" + std::to_string(p.physical_idx) + ")"; + } + GGML_LOG_INFO("ggml-hex: device %zu: %s (phys=%d, virt=%d, domain=%s:%d) [mdev peers: %s]\n", + i, cfg.name.c_str(), cfg.physical_idx, cfg.virtual_idx, cfg.domain_name.c_str(), cfg.domain_id, peers_str.c_str()); } + devices[i].iface = ggml_backend_hexagon_device_i; + devices[i].reg = reg; + devices[i].context = new ggml_backend_hexagon_device_context(i, opt_device_configs[i], &devices[i]); } + } ggml_hexagon_registry::~ggml_hexagon_registry() { GGML_LOG_INFO("ggml-hex: releasing registry\n"); - // Release devices / sessions + // Release devices for (size_t i = 0; i < opt_ndev; i++) { - auto sess = static_cast(devices[i].context); - delete sess; + auto dev_ctx = static_cast(devices[i].context); + delete dev_ctx; } } @@ -3285,65 +3099,606 @@ static ggml_backend_dev_t ggml_backend_hexagon_reg_get_device(ggml_backend_reg_t return &hreg->devices[index]; } -static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) { - if (strcmp(name, "ggml_backend_dev_get_extra_bufts") == 0 && opt_hostbuf) { - ggml_backend_dev_get_extra_bufts_t fct = ggml_backend_hexagon_device_get_extra_buffers_type; - return (void *) fct; +// ** communication context for tensor-split allreduce + +static void * ggml_backend_hexagon_comm_init(ggml_backend_t * backends, size_t n_backends) { + if (n_backends < 2 || n_backends > 4) { + return nullptr; + } + + for (size_t i = 0; i < n_backends; ++i) { + if (!ggml_backend_is_hexagon(backends[i])) { + return nullptr; + } + } + + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(backends[i]->context); + for (size_t j = i + 1; j < n_backends; j++) { + auto sess_j = static_cast(backends[j]->context); + if (sess_i->phys_idx == sess_j->phys_idx) { + return nullptr; + } + } + } + + auto * ctx = new ggml_backend_hexagon_comm_context(); + ctx->backends.assign(backends, backends + n_backends); + ctx->n_backends = n_backends; + + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(backends[i]->context); + ctx->fence_slots[i] = (volatile uint32_t *) sess_i->alloc_fence(1); + ctx->fence_tensors[i] = sess_i->make_fence_tensor(ctx->fence_slots[i]); + } + + return ctx; +} + +static void ggml_backend_hexagon_comm_free(void * comm_ctx_v) { + if (!comm_ctx_v) return; + auto * ctx = static_cast(comm_ctx_v); + for (size_t i = 0; i < ctx->n_backends; i++) { + auto sess_i = static_cast(ctx->backends[i]->context); + sess_i->free_fence((void *) ctx->fence_slots[i], 1); + } + delete ctx; +} + +static bool ggml_backend_hexagon_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { + if (opt_ar_select == 0 || !comm_ctx_v) return false; + auto * comm_ctx = static_cast(comm_ctx_v); + const size_t n_backends = comm_ctx->n_backends; + + if (n_backends < 2 || n_backends > 4) return false; + + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(comm_ctx->backends[i]->context); + for (size_t j = i + 1; j < n_backends; j++) { + auto sess_j = static_cast(comm_ctx->backends[j]->context); + if (sess_i->phys_idx == sess_j->phys_idx) { + return false; + } + } + } + + for (size_t i = 0; i < n_backends; i++) { + if (!tensors[i] || !tensors[i]->buffer || !ggml_backend_buffer_is_hexagon(tensors[i]->buffer)) { + return false; + } + if (tensors[i]->type != tensors[0]->type) { + return false; + } + if (!ggml_is_contiguous(tensors[i])) { + return false; + } + if (ggml_nelements(tensors[i]) != ggml_nelements(tensors[0])) { + return false; + } } + if (tensors[0]->type != GGML_TYPE_F16 && tensors[0]->type != GGML_TYPE_F32) { + return false; + } + + for (size_t r = 0; r < n_backends; r++) { + auto sess = static_cast(comm_ctx->backends[r]->context); + if (!htp_kparams_check(sess, tensors[r], (uint32_t) r, (uint32_t) n_backends, false, false, false)) { + return false; + } + } + + uint32_t max_seq = static_cast(comm_ctx->backends[0]->context)->fence_seq; + for (size_t i = 1; i < n_backends; i++) { + auto sess_i = static_cast(comm_ctx->backends[i]->context); + if ((int32_t)(sess_i->fence_seq - max_seq) > 0) { + max_seq = sess_i->fence_seq; + } + } + if (++max_seq == 0) max_seq = 1; + uint32_t fence_seq_entry = max_seq; + if (++max_seq == 0) max_seq = 1; + uint32_t fence_seq_exit = max_seq; + + for (size_t i = 0; i < n_backends; i++) { + auto sess_i = static_cast(comm_ctx->backends[i]->context); + sess_i->fence_seq = max_seq; + } + + std::vector data_tensors(n_backends); + std::vector sync_tensors(n_backends); + for (size_t i = 0; i < n_backends; i++) { + data_tensors[i] = tensors[i]; + sync_tensors[i] = &comm_ctx->fence_tensors[i]; + } + + for (size_t r = 0; r < n_backends; r++) { + auto sess = static_cast(comm_ctx->backends[r]->context); + sess->enqueue_allreduce(tensors[r], data_tensors, sync_tensors, (uint32_t) r, (uint32_t) n_backends, fence_seq_entry, fence_seq_exit); + for (size_t j = 0; j < n_backends; j++) { + if (r != j) { + sess->add_peer(static_cast(comm_ctx->backends[j]->context)); + } + } + } + + return true; +} + +static ggml_backend_buffer_type_t ggml_backend_hexagon_split_buffer_type(int main_device, const float * tensor_split) { + GGML_UNUSED(tensor_split); + auto reg = ggml_backend_hexagon_reg(); + auto dev = ggml_backend_reg_dev_get(reg, main_device); + if (!dev) { + dev = ggml_backend_reg_dev_get(reg, 0); + } + if (!dev) return nullptr; + auto dev_ctx = static_cast(dev->context); + return &dev_ctx->buffer_type; +} + +static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, const char * name) { + GGML_UNUSED(reg); + if (strcmp(name, "ggml_backend_split_buffer_type") == 0) { + return (void *) ggml_backend_hexagon_split_buffer_type; + } + if (strcmp(name, "ggml_backend_comm_init") == 0) { + return (void *) ggml_backend_hexagon_comm_init; + } + if (strcmp(name, "ggml_backend_comm_free") == 0) { + return (void *) ggml_backend_hexagon_comm_free; + } + if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { + return (void *) ggml_backend_hexagon_comm_allreduce_tensor; + } return NULL; } +template std::vector str_to_vec(const char* str) { + std::stringstream ss(str); + std::vector v; + std::string t; + + while (std::getline(ss, t, ',')) { + v.push_back(std::stoul(t, nullptr, 0)); + } + + return v; +} + +template std::string vec_to_str(std::vector v) { + std::stringstream ss; + ss << std::setbase(BASE) << std::showbase; + for (auto i : v) { ss << i << ','; } + auto str = ss.str(); str.pop_back(); // drop last comma + return str; +} + +static void ggml_hexagon_resolve_device_domain(ggml_hexagon_device_config & cfg, bool discovery_supported, const std::unordered_map & cdsp_map) { + if (discovery_supported) { + auto it = cdsp_map.find(cfg.physical_idx); + if (it != cdsp_map.end()) { + cfg.domain_id = it->second.id; + cfg.domain_name = it->second.name; + } else { + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not found on device (%zu CDSP core(s) available)\n", + cfg.physical_idx, cdsp_map.size()); + cfg.domain_id = -1; + cfg.domain_name = ""; + } + } else { + switch (cfg.physical_idx) { + case 0: + cfg.domain_id = 3; + cfg.domain_name = CDSP_DOMAIN_NAME; + break; + case 1: + cfg.domain_id = 4; + cfg.domain_name = "cdsp1"; + break; + default: + GGML_LOG_ERROR("ggml-hex: physical CDSP core %d not supported without dynamic discovery\n", + cfg.physical_idx); + cfg.domain_id = -1; + cfg.domain_name = ""; + break; + } + } + for (auto & sub_cfg : cfg.mdev_group) { + ggml_hexagon_resolve_device_domain(sub_cfg, discovery_supported, cdsp_map); + } +} + +// Enumerate NPU (aka CDSP) domains via FASTRPC_GET_DOMAINS if supported, +// and populate domain_id and domain_name for all configured devices. +static void ggml_hexagon_discover_devices() { + std::unordered_map cdsp_map; + bool discovery_supported = false; + + system_req_payload domain_info = {}; + domain_info.id = FASTRPC_GET_DOMAINS; + domain_info.sys.domains = nullptr; + domain_info.sys.max_domains = 0; + domain_info.sys.flags = DOMAINS_LIST_FLAGS_SET_TYPE(0, FASTRPC_NSP); + + int err = remote_system_request(&domain_info); + if (err == AEE_SUCCESS && domain_info.sys.num_domains > 0) { + std::vector domains(domain_info.sys.num_domains); + domain_info.sys.domains = domains.data(); + domain_info.sys.max_domains = (int) domains.size(); + + err = remote_system_request(&domain_info); + if (err == AEE_SUCCESS) { + discovery_supported = true; + const int n_domains = std::min(domain_info.sys.num_domains, (int) domains.size()); + for (int i = 0; i < n_domains; i++) { + GGML_LOG_INFO("ggml-hex: FASTRPC_GET_DOMAINS[%d]: type %d id %d name '%s' status %d instance-id %d\n", + i, (int) domains[i].type, domains[i].id, domains[i].name, domains[i].status, domains[i].instance_id); + if (domains[i].type != FASTRPC_NSP) { + GGML_LOG_DEBUG("ggml-hex: skipping non-CDSP domain (type=%d)\n", (int) domains[i].type); + continue; + } + if (!domains[i].status) { + GGML_LOG_WARN("ggml-hex: skipping CDSP domain id=%d (status=down)\n", domains[i].id); + continue; + } + cdsp_map[domains[i].instance_id] = domains[i]; + GGML_LOG_INFO("ggml-hex: using CDSP domain: instance-id %d id %d name '%s'\n", + domains[i].instance_id, domains[i].id, domains[i].name); + } + } else { + GGML_LOG_WARN("ggml-hex: FASTRPC_GET_DOMAINS fetch failed (0x%x), using static CDSP domains\n", (unsigned) err); + } + } else if (err != AEE_SUCCESS) { + GGML_LOG_DEBUG("ggml-hex: FASTRPC_GET_DOMAINS query failed (0x%x), using static CDSP domains\n", (unsigned) err); + } + + // Populate domain IDs and names for all configured devices + for (size_t i = 0; i < opt_ndev; i++) { + ggml_hexagon_resolve_device_domain(opt_device_configs[i], discovery_supported, cdsp_map); + } +} + static void ggml_hexagon_init(ggml_backend_reg * reg) { // Basic sanity checks to make sure definitions match static_assert((unsigned int) HTP_TYPE_Q4_0 == (unsigned int) GGML_TYPE_Q4_0, "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q4_1 == (unsigned int) GGML_TYPE_Q4_1, + "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_Q8_0 == (unsigned int) GGML_TYPE_Q8_0, "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_MXFP4 == (unsigned int) GGML_TYPE_MXFP4, "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_IQ4_NL == (unsigned int) GGML_TYPE_IQ4_NL, "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q4_K == (unsigned int) GGML_TYPE_Q4_K, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q5_K == (unsigned int) GGML_TYPE_Q5_K, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q6_K == (unsigned int) GGML_TYPE_Q6_K, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q3_K == (unsigned int) GGML_TYPE_Q3_K, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q2_K == (unsigned int) GGML_TYPE_Q2_K, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q1_0 == (unsigned int) GGML_TYPE_Q1_0, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q2_0 == (unsigned int) GGML_TYPE_Q2_0, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_BF16 == (unsigned int) GGML_TYPE_BF16, + "please update hexagon_type to match ggml_type"); - const char * str_verbose = getenv("GGML_HEXAGON_VERBOSE"); - const char * str_hostbuf = getenv("GGML_HEXAGON_HOSTBUF"); - const char * str_opmask = getenv("GGML_HEXAGON_OPMASK"); - const char * str_opsync = getenv("GGML_HEXAGON_OPSYNC"); - const char * str_opbatch = getenv("GGML_HEXAGON_OPBATCH"); - const char * str_opqueue = getenv("GGML_HEXAGON_OPQUEUE"); - const char * str_opfilter= getenv("GGML_HEXAGON_OPFILTER"); - const char * str_profile = getenv("GGML_HEXAGON_PROFILE"); - const char * str_etm = getenv("GGML_HEXAGON_ETM"); - const char * str_nhvx = getenv("GGML_HEXAGON_NHVX"); - const char * str_use_hmx = getenv("GGML_HEXAGON_USE_HMX"); - const char * str_ndev = getenv("GGML_HEXAGON_NDEV"); - const char * str_arch = getenv("GGML_HEXAGON_ARCH"); + const char * str_verbose = getenv("GGML_HEXAGON_VERBOSE"); + const char * str_opbatch = getenv("GGML_HEXAGON_OPBATCH"); + const char * str_opqueue = getenv("GGML_HEXAGON_OPQUEUE"); + const char * str_oppoll = getenv("GGML_HEXAGON_OPPOLL"); + const char * str_opfusion = getenv("GGML_HEXAGON_OPFUSION"); + const char * str_rt_repack = getenv("GGML_HEXAGON_RT_REPACK"); + const char * str_opfilter = getenv("GGML_HEXAGON_OPFILTER"); + const char * str_profile = getenv("GGML_HEXAGON_PROFILE"); + const char * str_etm = getenv("GGML_HEXAGON_ETM"); + const char * str_nhvx = getenv("GGML_HEXAGON_NHVX"); + const char * str_nhmx = getenv("GGML_HEXAGON_NHMX"); + const char * str_vtcm_size = getenv("GGML_HEXAGON_VTCM_SIZE"); + const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT"); + const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT"); + const char * str_fa_split = getenv("GGML_HEXAGON_FA_SPLIT"); + const char * str_gdn_select = getenv("GGML_HEXAGON_GDN_SELECT"); + const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT"); + const char * str_ar_scatter = getenv("GGML_HEXAGON_AR_SCATTER"); + const char * str_ndev = getenv("GGML_HEXAGON_NDEV"); + const char * str_arch = getenv("GGML_HEXAGON_ARCH"); + const char * str_vmem = getenv("GGML_HEXAGON_VMEM"); + const char * str_mbuf = getenv("GGML_HEXAGON_MBUF"); + const char * str_optrace = getenv("GGML_HEXAGON_OPTRACE"); + const char * str_hostbuf = getenv("GGML_HEXAGON_HOSTBUF"); + const char * str_dma64 = getenv("GGML_HEXAGON_DMA64"); + const char * str_power_mode = getenv("GGML_HEXAGON_POWER_MODE"); + + // Init Arch first since it affects other defaults + if (!str_arch) { + int err = htpdrv_get_arch(CDSP_DOMAIN_ID, &opt_arch); + if (err != 0) { + GGML_LOG_ERROR("ggml-hex: failed to query HTP version (err %d) defaulting to v73\n", err); + opt_arch = 73; + } else { + if (opt_arch < 73) { + GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is under supported range, capping at v73\n", opt_arch); + opt_arch = 73; + } else if (opt_arch > 85) { + GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is over supported range, capping at v85\n", opt_arch); + opt_arch = 85; + } + } + } else { + if (str_arch[0] == 'v' || str_arch[0] == 'V') { + str_arch++; + } + opt_arch = strtoul(str_arch, NULL, 0); + } + + size_t MiB = 1024 * 1024; + + // Update vmem default + opt_vmem = opt_arch >= 75 ? HTP_OP_MAX_VMEM_DEFAULT : 3000 * MiB; + opt_dma64 = opt_arch > 79 && (!str_dma64 || atoi(str_dma64) != 0); auto RE_ICASE = std::regex_constants::icase; - opt_opfilter = str_opfilter ? new std::regex(str_opfilter, RE_ICASE) : NULL; - opt_verbose = str_verbose ? atoi(str_verbose) : 0; - opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) : opt_hostbuf; - opt_opmask = str_opmask ? strtoul(str_opmask, NULL, 0) : opt_opmask; - opt_opsync = str_opsync ? atoi(str_opsync) : opt_opsync; - opt_opbatch = str_opbatch ? strtoul(str_opbatch, NULL, 0) : opt_opbatch; - opt_opqueue = str_opqueue ? strtoul(str_opqueue, NULL, 0) : opt_opqueue; - opt_profile = str_profile ? atoi(str_profile) : 0; - opt_etm = str_etm ? atoi(str_etm) : 0; - opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx; - opt_use_hmx = str_use_hmx ? atoi(str_use_hmx) : opt_use_hmx; - opt_ndev = str_ndev ? strtoul(str_ndev, NULL, 0) : opt_ndev; - opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) : opt_hostbuf; - - if (opt_ndev > GGML_HEXAGON_MAX_SESSIONS) { - opt_ndev = GGML_HEXAGON_MAX_SESSIONS; - } - - if (str_arch) { - if (str_arch[0] == 'v') { - str_arch++; + opt_opfilter = str_opfilter ? new std::regex(str_opfilter, RE_ICASE) : NULL; + opt_verbose = str_verbose ? atoi(str_verbose) : 0; + opt_opbatch = str_opbatch ? strtoul(str_opbatch, NULL, 0) : opt_opbatch; + opt_opqueue = str_opqueue ? strtoul(str_opqueue, NULL, 0) : opt_opqueue; + opt_optrace = str_optrace ? strtoul(str_optrace, NULL, 0) : (opt_opbatch * HTP_TRACE_MAX_EVENTS + 131072); + opt_oppoll = str_oppoll ? strtoul(str_oppoll, NULL, 0) : opt_oppoll; + opt_opfusion = str_opfusion ? atoi(str_opfusion) : opt_opfusion; + opt_rt_repack = str_rt_repack ? atoi(str_rt_repack) : opt_rt_repack; + opt_profile = str_profile ? atoi(str_profile) : 0; + opt_etm = str_etm ? atoi(str_etm) : 0; + if (str_power_mode) { + int m = htp_power_mode_from_str(str_power_mode); + if (m < 0) { + GGML_LOG_WARN("ggml-hex: unknown GGML_HEXAGON_POWER_MODE '%s', falling back to 'burst'\n", str_power_mode); + } else { + opt_power_mode = m; + } + } + opt_nhvx = str_nhvx ? strtoul(str_nhvx, NULL, 0) : opt_nhvx; + opt_nhmx = str_nhmx ? atoi(str_nhmx) : opt_nhmx; + opt_vtcm_size = str_vtcm_size ? parse_size(str_vtcm_size) : opt_vtcm_size; + opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select; + opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select; + opt_fa_split = str_fa_split ? atoi(str_fa_split) : opt_fa_split; + opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select; + opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select; + opt_ar_scatter = str_ar_scatter ? atoi(str_ar_scatter) : opt_ar_scatter; + + if (str_mbuf) { + const char * p = str_mbuf; + for (int idx = 0; idx < 3 && p && *p; idx++) { + while (*p == ' ') p++; + const char * comma = strchr(p, ','); + size_t len = comma ? (size_t)(comma - p) : strlen(p); + while (len > 0 && p[len - 1] == ' ') len--; + if (len > 0) { + std::string token(p, len); + if (idx == 0) opt_mbuf_dyn = parse_size(token.c_str()); + if (idx == 1) opt_mbuf_static = parse_size(token.c_str()); + if (idx == 2) opt_mbuf_total = parse_size(token.c_str()); + } + if (!comma) break; + p = comma + 1; } - opt_arch = strtoul(str_arch, NULL, 0); + } + opt_vmem = str_vmem ? parse_size(str_vmem) : opt_vmem; + opt_hostbuf = str_hostbuf ? atoi(str_hostbuf) != 0 : opt_hostbuf; + + // Parse device configuration + const char * str_devices = getenv("GGML_HEXAGON_DEVICES"); + if (!str_devices && str_ndev && str_ndev[0] != '\0') { + GGML_LOG_WARN("DEPRECATED: GGML_HEXAGON_NDEV is deprecated. use GGML_HEXAGON_DEVICES instead\n"); + str_devices = str_ndev; + } + + if (str_devices && str_devices[0] != '\0') { + bool is_single_number = true; + for (int i = 0; str_devices[i] != '\0'; i++) { + if (!isdigit((unsigned char)str_devices[i])) { + is_single_number = false; + break; + } + } + if (is_single_number) { + int n = atoi(str_devices); + if (n < 1) n = 1; + if (n > GGML_HEXAGON_MAX_SESSIONS) n = GGML_HEXAGON_MAX_SESSIONS; + opt_ndev = n; + for (size_t i = 0; i < opt_ndev; i++) { + opt_device_configs[i].physical_idx = 0; + opt_device_configs[i].virtual_idx = (int)i; + opt_device_configs[i].name = "HTP" + std::to_string(i); + opt_device_configs[i].mdev_group.clear(); + } + } else { + std::string s_devices(str_devices); + std::vector items; + std::string curr_item; + int bracket_depth = 0; + for (char ch : s_devices) { + if (ch == '[') { + bracket_depth++; + curr_item += ch; + } else if (ch == ']') { + if (bracket_depth > 0) bracket_depth--; + curr_item += ch; + } else if (ch == ',' && bracket_depth == 0) { + size_t s = curr_item.find_first_not_of(" \t\r\n"); + size_t e = curr_item.find_last_not_of(" \t\r\n"); + if (s != std::string::npos) { + items.push_back(curr_item.substr(s, e - s + 1)); + } + curr_item.clear(); + } else { + curr_item += ch; + } + } + size_t s = curr_item.find_first_not_of(" \t\r\n"); + size_t e = curr_item.find_last_not_of(" \t\r\n"); + if (s != std::string::npos) { + items.push_back(curr_item.substr(s, e - s + 1)); + } + + opt_ndev = 0; + for (const auto & item : items) { + size_t b_open = item.find('['); + size_t b_close = item.rfind(']'); + + if (b_open != std::string::npos && b_close != std::string::npos && b_close > b_open) { + // Grouped / composite syntax: Name[phys_spec:virt] or Name[phys_spec] + std::string dev_name = item.substr(0, b_open); + std::string content = item.substr(b_open + 1, b_close - b_open - 1); + + int virt = 0; + std::string phys_spec = content; + size_t colon_pos = content.find(':'); + if (colon_pos != std::string::npos) { + phys_spec = content.substr(0, colon_pos); + try { + virt = std::stoi(content.substr(colon_pos + 1)); + } catch (...) { + virt = 0; + } + } else { + size_t dev_colon = dev_name.find(':'); + if (dev_colon != std::string::npos) { + try { + virt = std::stoi(dev_name.substr(dev_colon + 1)); + } catch (...) { + virt = 0; + } + } + } + + // Parse physical indices from phys_spec (e.g. 0-1, 0,1, 0-3, etc.) + std::vector phys_list; + std::stringstream pss(phys_spec); + std::string p_part; + while (std::getline(pss, p_part, ',')) { + size_t ps = p_part.find_first_not_of(" \t\r\n"); + size_t pe = p_part.find_last_not_of(" \t\r\n"); + if (ps == std::string::npos) continue; + p_part = p_part.substr(ps, pe - ps + 1); + + size_t dash_pos = p_part.find('-'); + if (dash_pos != std::string::npos) { + try { + int p_start = std::stoi(p_part.substr(0, dash_pos)); + int p_end = std::stoi(p_part.substr(dash_pos + 1)); + for (int p = p_start; p <= p_end; p++) { + if (std::find(phys_list.begin(), phys_list.end(), p) == phys_list.end()) { + phys_list.push_back(p); + } + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse physical range in '%s'\n", p_part.c_str()); + } + } else { + try { + int p = std::stoi(p_part); + if (std::find(phys_list.begin(), phys_list.end(), p) == phys_list.end()) { + phys_list.push_back(p); + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse physical index in '%s'\n", p_part.c_str()); + } + } + } + + if (phys_list.empty()) { + phys_list.push_back(0); + } + + if (opt_ndev < GGML_HEXAGON_MAX_SESSIONS) { + auto & cfg = opt_device_configs[opt_ndev]; + cfg.name = dev_name; + cfg.physical_idx = phys_list[0]; + cfg.virtual_idx = virt; + cfg.mdev_group.clear(); + + for (size_t k = 1; k < phys_list.size(); k++) { + ggml_hexagon_device_config sub_cfg; + sub_cfg.physical_idx = phys_list[k]; + sub_cfg.virtual_idx = virt; + sub_cfg.name = "HTP" + std::to_string(phys_list[k]) + ":" + std::to_string(virt); + cfg.mdev_group.push_back(sub_cfg); + } + opt_ndev++; + } else { + GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str()); + } + } else if (item.rfind("HTP", 0) == 0) { + std::string rest = item.substr(3); + size_t colon_pos = rest.find(':'); + int phys = 0; + int virt = 0; + try { + if (colon_pos == std::string::npos) { + phys = std::stoi(rest); + virt = 0; + } else { + phys = std::stoi(rest.substr(0, colon_pos)); + virt = std::stoi(rest.substr(colon_pos + 1)); + } + } catch (...) { + GGML_LOG_WARN("ggml-hex: failed to parse device index in '%s'\n", item.c_str()); + continue; + } + + if (opt_ndev < GGML_HEXAGON_MAX_SESSIONS) { + opt_device_configs[opt_ndev].physical_idx = phys; + opt_device_configs[opt_ndev].virtual_idx = virt; + opt_device_configs[opt_ndev].name = colon_pos == std::string::npos + ? "HTP" + std::to_string(phys) + : "HTP" + std::to_string(phys) + ":" + std::to_string(virt); + opt_device_configs[opt_ndev].mdev_group.clear(); + opt_ndev++; + } else { + GGML_LOG_WARN("ggml-hex: max sessions limit reached (%d), ignoring device %s\n", GGML_HEXAGON_MAX_SESSIONS, item.c_str()); + } + } else { + GGML_LOG_WARN("ggml-hex: invalid device name format '%s', must start with HTP\n", item.c_str()); + } + } + } + } else { + opt_ndev = 1; + opt_device_configs[0].physical_idx = 0; + opt_device_configs[0].virtual_idx = 0; + opt_device_configs[0].name = "HTP0"; + opt_device_configs[0].mdev_group.clear(); + } + +#if defined(__ANDROID__) + if (opt_arch < 75) { + opt_ndev = 1; + GGML_LOG_WARN("ggml-hex: forcing ndev to 1 for SoCs archs lower than v75.\n"); + } +#endif + + // Resolve domain info for all configured devices + ggml_hexagon_discover_devices(); + + if (str_profile) { + opt_pmu_evt = [&]() -> std::vector { + auto v = str_to_vec(str_profile); + switch (v.size()) { + case 1: opt_profile = v[0]; return opt_pmu_evt; // mode with default pmu events + case 8: opt_profile = 2; return v; // mode with custom pmu events + default: opt_profile = 0; return {}; // garbage input + }}(); + if (opt_profile == 1) opt_pmu_evt = {}; + GGML_LOG_INFO("ggml-hex: Profiling mode %u : pmu-evt [ %s ]\n", opt_profile, + vec_to_str(opt_pmu_evt).c_str()); } reg->context = new ggml_hexagon_registry(reg); diff --git a/ggml/src/ggml-hexagon/htp-common.h b/ggml/src/ggml-hexagon/htp-common.h new file mode 100644 index 0000000..e51340d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-common.h @@ -0,0 +1,106 @@ +#ifndef HTP_COMMON_H +#define HTP_COMMON_H + +#include "htp-opnode.h" +#include "htp-repack.h" + +struct htp_tensor_extra { + std::vector shadow_buf; + size_t shadow_size { 0 }; + uint32_t flags { 0 }; +}; + +extern int opt_arch; +extern size_t opt_ndev; +extern size_t opt_nhvx; +extern int opt_nhmx; +extern size_t opt_vmem; +extern int opt_etm; +extern int opt_verbose; +extern int opt_profile; +extern bool opt_hostbuf; +extern bool opt_dma64; +extern int opt_power_mode; + +extern size_t opt_mbuf_dyn; +extern size_t opt_mbuf_static; +extern size_t opt_mbuf_total; + +extern int opt_mm_select; +extern int opt_fa_select; +extern int opt_fa_split; +extern int opt_gdn_select; +extern int opt_ar_select; +extern int opt_ar_scatter; + +extern int opt_opbatch; +extern int opt_opqueue; +extern int opt_optrace; +extern int opt_oppoll; +extern int opt_opfusion; +extern int opt_rt_repack; + +#ifndef HEX_VERBOSE +#define HEX_VERBOSE(...) \ + if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__) +#endif + +static inline bool htp_tensor_is_fuseable(const struct ggml_tensor * t) { + if (!t || !t->extra) return false; + const auto * extra = (const struct htp_tensor_extra *) t->extra; + return (extra->flags & HTP_TENSOR_FUSEABLE) != 0; +} + +static inline bool htp_can_rt_repack(const struct ggml_tensor * t) { + if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) { + return false; + } + if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) { + return false; + } + return opt_rt_repack > 1 || ggml_is_quantized(t->type); +} + +static inline bool htp_tensor_is_tiled(const struct ggml_tensor * t) { + const auto * extra = (const struct htp_tensor_extra *) t->extra; + if (extra && (extra->flags & HTP_TENSOR_TILED)) { + return true; + } + return htp_can_rt_repack(t); +} + +static inline bool htp_needs_repack(const struct ggml_tensor * t) { + const auto * extra = (const struct htp_tensor_extra *) t->extra; + if (extra && (extra->flags & HTP_TENSOR_TILED)) { + return false; + } + return htp_can_rt_repack(t); +} + +static inline bool htp_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) { + const uintptr_t a0 = (uintptr_t) a->data; + const uintptr_t b0 = (uintptr_t) b->data; + const uintptr_t a1 = a0 + ggml_nbytes(a); + const uintptr_t b1 = b0 + ggml_nbytes(b); + + return a0 < b1 && b0 < a1; +} + +static inline bool htp_can_row_partition(const struct ggml_tensor * t) { + if (t->ne[1] > 1 && (t->nb[1] & 127) != 0) return false; + if (t->ne[2] > 1 && (t->nb[2] & 127) != 0) return false; + if (t->ne[3] > 1 && (t->nb[3] & 127) != 0) return false; + return true; +} + +struct ggml_hexagon_session; + +const char * htp_session_name(const struct ggml_hexagon_session * sess); +size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess); +uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess); +uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess); +uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess); + +htp_op_code htp_op_remap(const struct ggml_tensor * t); + +#endif // HTP_COMMON_H diff --git a/ggml/src/ggml-hexagon/htp-drv.cpp b/ggml/src/ggml-hexagon/htp-drv.cpp index 4c376b5..437e367 100644 --- a/ggml/src/ggml-hexagon/htp-drv.cpp +++ b/ggml/src/ggml-hexagon/htp-drv.cpp @@ -1,13 +1,8 @@ -// sample drv interface - -#pragma clang diagnostic ignored "-Wgnu-anonymous-struct" -#pragma clang diagnostic ignored "-Wmissing-prototypes" -#pragma clang diagnostic ignored "-Wsign-compare" - #include #include #include #include + #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN # ifndef NOMINMAX @@ -16,9 +11,17 @@ # include # include #else -# include -# include +# include +# include #endif + +#pragma clang diagnostic ignored "-Wgnu-anonymous-struct" +#pragma clang diagnostic ignored "-Wmissing-prototypes" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wlanguage-extension-token" +#pragma clang diagnostic ignored "-Wmicrosoft-enum-value" +#pragma clang diagnostic ignored "-Wnested-anon-types" + #include "ggml-impl.h" #include "htp-drv.h" #include "libdl.h" @@ -56,7 +59,11 @@ typedef AEEResult (*dspqueue_read_pfn_t)(dspqueue_t queue, uint32_t *flags, uint32_t max_message_length, uint32_t *message_length, uint8_t *message, uint32_t timeout_us); - +typedef AEEResult (*dspqueue_read_noblock_pfn_t)(dspqueue_t queue, uint32_t *flags, + uint32_t max_buffers, uint32_t *num_buffers, + struct dspqueue_buffer *buffers, + uint32_t max_message_length, + uint32_t *message_length, uint8_t *message); typedef int (*fastrpc_mmap_pfn_t)(int domain, int fd, void *addr, int offset, size_t length, enum fastrpc_map_flags flags); typedef int (*fastrpc_munmap_pfn_t)(int domain, int fd, void *addr, size_t length); @@ -66,6 +73,7 @@ typedef int (*remote_handle64_close_pfn_t)(remote_handle h); typedef int (*remote_handle_control_pfn_t)(uint32_t req, void* data, uint32_t datalen); typedef int (*remote_handle64_control_pfn_t)(remote_handle64 h, uint32_t req, void* data, uint32_t datalen); typedef int (*remote_session_control_pfn_t)(uint32_t req, void *data, uint32_t datalen); +typedef int (*remote_system_request_pfn_t)(system_req_payload * req); // // Driver API pfns @@ -79,11 +87,12 @@ rpcmem_to_fd_pfn_t rpcmem_to_fd_pfn = nullptr; fastrpc_mmap_pfn_t fastrpc_mmap_pfn = nullptr; fastrpc_munmap_pfn_t fastrpc_munmap_pfn = nullptr; -dspqueue_create_pfn_t dspqueue_create_pfn = nullptr; -dspqueue_close_pfn_t dspqueue_close_pfn = nullptr; -dspqueue_export_pfn_t dspqueue_export_pfn = nullptr; -dspqueue_write_pfn_t dspqueue_write_pfn = nullptr; -dspqueue_read_pfn_t dspqueue_read_pfn = nullptr; +dspqueue_create_pfn_t dspqueue_create_pfn = nullptr; +dspqueue_close_pfn_t dspqueue_close_pfn = nullptr; +dspqueue_export_pfn_t dspqueue_export_pfn = nullptr; +dspqueue_write_pfn_t dspqueue_write_pfn = nullptr; +dspqueue_read_pfn_t dspqueue_read_pfn = nullptr; +dspqueue_read_noblock_pfn_t dspqueue_read_noblock_pfn = nullptr; remote_handle64_open_pfn_t remote_handle64_open_pfn = nullptr; remote_handle64_invoke_pfn_t remote_handle64_invoke_pfn = nullptr; @@ -91,6 +100,7 @@ remote_handle64_close_pfn_t remote_handle64_close_pfn = nullptr; remote_handle_control_pfn_t remote_handle_control_pfn = nullptr; remote_handle64_control_pfn_t remote_handle64_control_pfn = nullptr; remote_session_control_pfn_t remote_session_control_pfn = nullptr; +remote_system_request_pfn_t remote_system_request_pfn = nullptr; // // Driver API @@ -164,6 +174,12 @@ AEEResult dspqueue_read(dspqueue_t queue, uint32_t * message_length, uint8_t * message, uint32_t timeout_us) { +#ifdef _WIN32 + if (timeout_us == 0) { + return dspqueue_read_noblock_pfn(queue, flags, max_buffers, num_buffers, buffers, max_message_length, + message_length, message); + } +#endif return dspqueue_read_pfn(queue, flags, max_buffers, num_buffers, buffers, max_message_length, message_length, message, timeout_us); } @@ -192,6 +208,13 @@ HTPDRV_API int remote_session_control(uint32_t req, void * data, uint32_t datale return remote_session_control_pfn(req, data, datalen); } +HTPDRV_API int remote_system_request(system_req_payload * req) { + if (!remote_system_request_pfn) { + return AEE_EUNSUPPORTEDAPI; + } + return remote_system_request_pfn(req); +} + #ifdef _WIN32 static std::string wstr_to_str(std::wstring_view wstr) { @@ -346,12 +369,14 @@ int htpdrv_init() { dlsym(handle.get(), dspqueue_export_pfn_t, dspqueue_export_pfn, dspqueue_export, false); dlsym(handle.get(), dspqueue_write_pfn_t, dspqueue_write_pfn, dspqueue_write, false); dlsym(handle.get(), dspqueue_read_pfn_t, dspqueue_read_pfn, dspqueue_read, false); + dlsym(handle.get(), dspqueue_read_noblock_pfn_t, dspqueue_read_noblock_pfn, dspqueue_read_noblock, false); dlsym(handle.get(), remote_handle64_open_pfn_t, remote_handle64_open_pfn, remote_handle64_open, false); dlsym(handle.get(), remote_handle64_invoke_pfn_t, remote_handle64_invoke_pfn, remote_handle64_invoke, false); dlsym(handle.get(), remote_handle_control_pfn_t, remote_handle_control_pfn, remote_handle_control, false); dlsym(handle.get(), remote_handle64_control_pfn_t, remote_handle64_control_pfn, remote_handle64_control, false); dlsym(handle.get(), remote_session_control_pfn_t, remote_session_control_pfn, remote_session_control, false); dlsym(handle.get(), remote_handle64_close_pfn_t, remote_handle64_close_pfn, remote_handle64_close, false); + dlsym(handle.get(), remote_system_request_pfn_t, remote_system_request_pfn, remote_system_request, true); lib_cdsp_rpc_handle = std::move(handle); initialized = true; @@ -359,7 +384,7 @@ int htpdrv_init() { return AEE_SUCCESS; } -domain * get_domain(int domain_id) { +domain * htpdrv_get_domain(int domain_id) { int i = 0; int size = sizeof(supported_domains) / sizeof(domain); @@ -372,7 +397,7 @@ domain * get_domain(int domain_id) { return NULL; } -int get_hex_arch_ver(int domain, int * arch) { +int htpdrv_get_arch(int domain, int * arch) { if (!remote_handle_control_pfn) { GGML_LOG_ERROR("ggml-hex: remote_handle_control is not supported on this device\n"); return AEE_EUNSUPPORTEDAPI; @@ -394,25 +419,7 @@ int get_hex_arch_ver(int domain, int * arch) { return err; } - switch (arch_ver.capability & 0xff) { - case 0x68: - *arch = 68; - return 0; - case 0x69: - *arch = 69; - return 0; - case 0x73: - *arch = 73; - return 0; - case 0x75: - *arch = 75; - return 0; - case 0x79: - *arch = 79; - return 0; - case 0x81: - *arch = 81; - return 0; - } - return -1; + uint32_t val = arch_ver.capability & 0xff; + *arch = (int) ((val >> 4) * 10 + (val & 0x0f)); + return 0; } diff --git a/ggml/src/ggml-hexagon/htp-drv.h b/ggml/src/ggml-hexagon/htp-drv.h index 6eba7ba..8232780 100644 --- a/ggml/src/ggml-hexagon/htp-drv.h +++ b/ggml/src/ggml-hexagon/htp-drv.h @@ -96,17 +96,17 @@ extern "C" { HTPDRV_API int htpdrv_init(void); /** - * get_domain API: get domain struct from domain value. + * htpdrv_get_domain API: get domain struct from domain value. * * @param[in] domain value of a domain * @return Returns domain struct of the domain if it is supported or else * returns NULL. * */ -HTPDRV_API domain * get_domain(int domain_id); +HTPDRV_API domain * htpdrv_get_domain(int domain_id); /** - * get_hex_arch_ver API: query the Hexagon processor architecture version information + * htpdrv_get_arch API: query the Hexagon processor architecture version information * * @param[in] domain_id value of a domain * @param[out] Arch version (73, 75, ...) @@ -114,7 +114,9 @@ HTPDRV_API domain * get_domain(int domain_id); * non-zero if error, return value points to the error. * */ -HTPDRV_API int get_hex_arch_ver(int domain, int * arch); +HTPDRV_API int htpdrv_get_arch(int domain, int * arch); + +HTPDRV_API int remote_system_request(system_req_payload * req); #ifdef __cplusplus } diff --git a/ggml/src/ggml-hexagon/htp-fusion.cpp b/ggml/src/ggml-hexagon/htp-fusion.cpp new file mode 100644 index 0000000..ba2dd64 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-fusion.cpp @@ -0,0 +1,649 @@ +#include "htp-fusion.h" +#include "htp-kparams.h" +#include "ggml-impl.h" + +#include +#include + +static inline bool htp_same_tensor_data(const ggml_tensor * a, const ggml_tensor * b) { + return a && b && (a == b || a->data == b->data); +} + +static inline bool htp_same_shape_and_contiguity(const ggml_tensor * a, const ggml_tensor * b) { + return ggml_are_same_shape(a, b) && (ggml_is_contiguous(a) == ggml_is_contiguous(b)); +} + +static inline bool htp_inplace_overlap_unsafe(const ggml_tensor * dst, const ggml_tensor * in) { + return htp_tensors_overlap(dst, in) && (dst->data != in->data || !ggml_are_same_stride(dst, in)); +} + +static inline bool htp_inputs_overlap_unsafe(const htp_opnode & prev, const ggml_tensor * dst, const ggml_tensor * extra = nullptr) { + if (extra && htp_inplace_overlap_unsafe(dst, extra)) return true; + for (const auto * in : prev.inputs) { + if (htp_inplace_overlap_unsafe(dst, in)) return true; + } + return false; +} + +static inline bool htp_match_consumer( + const htp_opnode & prev, + const htp_opnode & cur, + bool commutative, + const ggml_tensor ** other_src = nullptr +) { + if (prev.outputs.size() != 1) { + return false; + } + const ggml_tensor * prev_out = prev.dst(); + if (!htp_tensor_is_fuseable(prev_out)) { + return false; + } + if (htp_same_tensor_data(cur.src0(), prev_out)) { + if (other_src) *other_src = cur.src1(); + return true; + } + if (commutative && htp_same_tensor_data(cur.src1(), prev_out)) { + if (other_src) *other_src = cur.src0(); + return true; + } + return false; +} + +static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) { + if (kparams->n_hmx) { + return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; + } + return htp_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW); +} + +static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) { + if (kparams->n_hmx) { + return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D; + } + return htp_tensor_is_tiled(src0) && + (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK); +} + +// --- Fusion rules --- + +static bool fuse_allreduce_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (opt_ar_select != 2 || prev.opcode != HTP_OP_ALLREDUCE) return false; + + const auto * ar_kparams = prev.kparams(); + const uint32_t rank = (uint32_t) ar_kparams->rank; + const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks; + + const ggml_tensor * ar_local = prev.inputs[rank]; + if (!htp_tensor_is_fuseable(ar_local)) return false; + + const ggml_tensor * add_dst = cur.dst(); + const ggml_tensor * res_tensor = nullptr; + if (htp_same_tensor_data(cur.src0(), ar_local)) { + res_tensor = cur.src1(); + } else if (htp_same_tensor_data(cur.src1(), ar_local)) { + res_tensor = cur.src0(); + } else { + return false; + } + + if (ar_local->type != res_tensor->type) return false; + + const bool is_same_shape = ggml_are_same_shape(ar_local, res_tensor); + const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] && + res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1); + if (!is_same_shape && !is_row_bcast) return false; + + if (is_same_shape && !ggml_are_same_stride(ar_local, res_tensor)) { + return false; + } + if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) return false; + + const bool is_shard_ok = n_ranks <= HTP_OP_MAX_OUTPUTS && add_dst->data == ar_local->data; + + struct htp_allreduce_kernel_params kparams; + if (!htp_kparams_solve_allreduce(sess, add_dst, rank, n_ranks, true, is_row_bcast, is_shard_ok, &kparams)) { + HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", htp_session_name(sess)); + return false; + } + + const bool scatter_ok = kparams.n_dsts > 1; + kparams.mode = scatter_ok ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL; + + for (uint32_t r = 0; r < n_ranks; r++) { + const ggml_tensor * ar_src = prev.inputs[r]; + if (!htp_tensors_overlap(add_dst, ar_src)) continue; + if (scatter_ok && r == rank) continue; + HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", htp_session_name(sess), r); + return false; + } + + prev.opcode = HTP_OP_ALLREDUCE_ADD; + prev.inputs.push_back(res_tensor); + if (kparams.mode != HTP_ALLREDUCE_SHARDED_FANOUT) { + prev.outputs = { add_dst }; + } + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_rms_norm_mul(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_RMS_NORM) return false; + + const ggml_tensor * weight = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &weight)) return false; + + const ggml_tensor * src0 = prev.src0(); + const ggml_tensor * dst = cur.dst(); + if (src0->type != GGML_TYPE_F32 || src0->ne[0] != weight->ne[0] || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false; + if (htp_inputs_overlap_unsafe(prev, dst, weight)) return false; + + const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1); + const bool is_same_shape = ggml_are_same_shape(src0, weight); + if (!is_row_bcast && !is_same_shape) return false; + + struct htp_unary_kernel_params kparams; + if (!htp_kparams_solve_rms_norm_mul(sess, src0, weight, dst, &kparams)) { + HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n", + htp_session_name(sess), kparams.vtcm_size, htp_session_vtcm_size(sess)); + return false; + } + + prev.opcode = HTP_OP_RMS_NORM_MUL; + prev.inputs = { src0, weight }; + prev.outputs = { dst }; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_rms_norm_scale(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + GGML_UNUSED(sess); + if (prev.opcode != HTP_OP_RMS_NORM && prev.opcode != HTP_OP_RMS_NORM_MUL) return false; + if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false; + + const ggml_tensor * src0 = prev.src0(); + const ggml_tensor * dst = cur.dst(); + if (src0->type != GGML_TYPE_F32 || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false; + if (htp_inputs_overlap_unsafe(prev, dst)) return false; + + float scale = 0.0f; + float bias = 0.0f; + memcpy(&scale, (const float *) cur.params + 0, sizeof(float)); + memcpy(&bias, (const float *) cur.params + 1, sizeof(float)); + if (bias != 0.0f || scale == 0.0f) return false; + + float prev_scale = 0.0f; + memcpy(&prev_scale, &prev.params[1], sizeof(float)); + if (prev_scale != 0.0f) { + scale *= prev_scale; + } + memcpy(&prev.params[1], &scale, sizeof(float)); + + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + return true; +} + +static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_MUL_MAT) return false; + + const ggml_tensor * src2 = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &src2)) return false; + if (src2->type != GGML_TYPE_F32) return false; + + const ggml_tensor * src0 = prev.node->src[0]; + const ggml_tensor * src1 = prev.node->src[1]; + const ggml_tensor * dst = cur.node; + if (!ggml_are_same_shape(prev.node, dst)) return false; + + const bool prev_collapsed = htp_matmul_can_collapse(src0, src1, prev.node); + const bool cur_collapsed = htp_matmul_add_can_collapse(src0, src1, src2, dst); + if (prev_collapsed != cur_collapsed) { + return false; + } + + const auto * orig_kparams = prev.kparams(); + struct htp_mm_kernel_params kparams; + if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams)) { + return false; + } + + const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; + if (kparams.n_hmx == 0 && src1_nrows != 1) return false; + + if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) { + if (kparams.m_chunk < orig_kparams->m_chunk || + kparams.n_chunk < orig_kparams->n_chunk || + kparams.n_act_threads < orig_kparams->n_act_threads) { + HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n", + htp_session_name(sess), orig_kparams->m_chunk, kparams.m_chunk, + orig_kparams->n_chunk, kparams.n_chunk, + orig_kparams->n_act_threads, kparams.n_act_threads); + return false; + } + } + + prev.opcode = HTP_OP_MUL_MAT_ADD; + prev.inputs = { src0, src1, src2 }; + prev.outputs = { dst }; + htp_opnode_collapse(sess, prev); + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_MUL_MAT && prev.opcode != HTP_OP_MUL_MAT_NX) return false; + + const uint32_t curr_n = (uint32_t) prev.outputs.size(); + if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) return false; + + const ggml_tensor * w0 = prev.inputs[0]; + const ggml_tensor * x_orig = prev.node->src[1]; + const ggml_tensor * w_in = cur.node->src[0]; + const ggml_tensor * x_in = cur.node->src[1]; + const ggml_tensor * d_in = cur.node; + + auto * orig_kparams = prev.kparams(); + const auto * cur_kparams = cur.kparams(); + + if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] || + w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 || + htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) || + orig_kparams->n_hmx != cur_kparams->n_hmx) { + return false; + } + + const bool collapse = htp_matmul_can_collapse(w0, x_orig, prev.node) && htp_matmul_can_collapse(w_in, x_in, d_in); + if ((x_orig->ne[2] > 1 || x_orig->ne[3] > 1) && !collapse) return false; + + if (prev.opcode == HTP_OP_MUL_MAT_NX) { + orig_kparams->n_weights = curr_n + 1; + } else { + struct htp_mm_kernel_params kparams; + const ggml_tensor x_collapsed = collapse ? htp_tensor_collapse_rows(x_orig) : *x_orig; + if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false; + if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) return false; + + prev.set_kparams(kparams); + prev.opcode = HTP_OP_MUL_MAT_NX; + } + + prev.inputs.insert(prev.inputs.begin() + curr_n, w_in); + prev.outputs.push_back(collapse ? prev.add_dummy(htp_tensor_collapse_rows(d_in)) : d_in); + prev.fused.push_back(cur.node); + return true; +} + +static bool fuse_mul_mat_id_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_MUL_MAT_ID && prev.opcode != HTP_OP_MUL_MAT_ID_NX) return false; + + const uint32_t curr_n = (uint32_t) prev.outputs.size(); + if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 2 >= HTP_OP_MAX_INPUTS) return false; + + const ggml_tensor * w0 = prev.inputs[0]; + const ggml_tensor * x = prev.inputs[curr_n]; + const ggml_tensor * ids = prev.inputs[curr_n + 1]; + const ggml_tensor * w_in = cur.src0(); + const ggml_tensor * x_in = cur.src1(); + const ggml_tensor * ids_in = cur.node->src[2]; + const ggml_tensor * d_in = cur.dst(); + + auto * orig_kparams = prev.kparams(); + const auto * cur_kparams = cur.kparams(); + + if (x_in != x || ids_in != ids || w_in->type != w0->type || + w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] || + htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) || + orig_kparams->n_hmx != cur_kparams->n_hmx) { + return false; + } + + struct htp_mm_kernel_params kparams; + if (d_in->nb[1] == prev.node->nb[1]) { + kparams = *orig_kparams; + kparams.n_weights = curr_n + 1; + } else if (!htp_kparams_solve_mmidnx(sess, w0, x, d_in, curr_n + 1, &kparams)) { + return false; + } + if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) return false; + + prev.opcode = HTP_OP_MUL_MAT_ID_NX; + prev.inputs.insert(prev.inputs.begin() + curr_n, w_in); + prev.outputs.push_back(d_in); + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_gdn_cpy(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + GGML_UNUSED(sess); + if (prev.opcode != HTP_OP_GATED_DELTA_NET || prev.outputs.size() != 1) return false; + + const ggml_tensor * gdn_out = prev.dst(); + const ggml_tensor * cpy_node = cur.node; + const ggml_tensor * cpy_src = cur.src0(); + const ggml_tensor * cpy_dst = cur.dst(); + + if (gdn_out->type != GGML_TYPE_F32 || cpy_src->type != GGML_TYPE_F32 || cpy_dst->type != GGML_TYPE_F32) return false; + if ((gdn_out->flags & GGML_TENSOR_FLAG_OUTPUT) || (cpy_node->flags & GGML_TENSOR_FLAG_OUTPUT)) return false; + + const ggml_tensor * v = prev.node->src[2]; + if (!v) return false; + + const int64_t S_v = v->ne[0]; + const int64_t H = v->ne[1]; + const int64_t n_tokens = v->ne[2]; + const int64_t n_seqs = v->ne[3]; + const int64_t K = ggml_get_op_params_i32(prev.node, 0); + const size_t tail_off = (size_t) S_v * H * n_tokens * n_seqs * sizeof(float); + const int64_t D = S_v * S_v * H; + const int64_t n_written = std::min(n_tokens, K); + + if (cpy_src->op != GGML_OP_VIEW || !htp_same_tensor_data(cpy_src->view_src, gdn_out) || + cpy_src->view_offs != tail_off || !ggml_is_contiguous(cpy_src)) { + return false; + } + + if (cpy_dst->ne[0] != D || cpy_dst->ne[1] != n_seqs || cpy_dst->ne[2] != n_written || !ggml_is_contiguous(cpy_dst)) { + return false; + } + + prev.outputs.push_back(cpy_dst); + prev.fused.push_back(cur.node); + return true; +} + +static bool fuse_ssm_conv_silu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + GGML_UNUSED(sess); + if (prev.opcode != HTP_OP_SSM_CONV) return false; + + auto * kparams = prev.kparams(); + if (kparams->silu) return false; + if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false; + + const ggml_tensor * conv_dst = prev.dst(); + const ggml_tensor * dst = cur.dst(); + if (!ggml_are_same_layout(conv_dst, dst)) return false; + if (htp_tensors_overlap(dst, prev.src0()) || htp_tensors_overlap(dst, prev.src1())) return false; + + kparams->silu = 1; + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + return true; +} + +static bool fuse_repeat_get_rows(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_REPEAT) return false; + if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false; + + const ggml_tensor * repeat_out = prev.dst(); + const ggml_tensor * repeat_src0 = prev.src0(); + const ggml_tensor * gr_src1 = cur.src1(); + const ggml_tensor * gr_dst = cur.dst(); + + if (repeat_src0->ne[0] != repeat_out->ne[0] || repeat_src0->ne[1] != repeat_out->ne[1]) return false; + if (repeat_src0->type != repeat_out->type) return false; + + struct htp_get_rows_kernel_params kparams; + if (!htp_kparams_solve_repeat_get_rows(sess, repeat_src0, gr_src1, gr_dst, &kparams) || kparams.vtcm_size == 0) return false; + + prev.opcode = HTP_OP_GET_ROWS; + prev.inputs = { repeat_src0, gr_src1 }; + prev.outputs = { gr_dst }; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_unary_mul_glu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + htp_op_code glu_op = HTP_OP_INVALID; + switch (prev.opcode) { + case HTP_OP_UNARY_SILU: glu_op = HTP_OP_GLU_SWIGLU; break; + case HTP_OP_UNARY_GELU: glu_op = HTP_OP_GLU_GEGLU; break; + case HTP_OP_UNARY_GELU_QUICK: glu_op = HTP_OP_GLU_GEGLU_QUICK; break; + case HTP_OP_UNARY_GELU_ERF: glu_op = HTP_OP_GLU_GEGLU_ERF; break; + case HTP_OP_UNARY_RELU: glu_op = HTP_OP_GLU_REGLU; break; + case HTP_OP_UNARY_SIGMOID: glu_op = HTP_OP_GLU_SIGMOID; break; + default: return false; + } + + const ggml_tensor * up = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &up)) return false; + + const ggml_tensor * gate = prev.src0(); + const ggml_tensor * dst = cur.dst(); + if (htp_inputs_overlap_unsafe(prev, dst, up)) return false; + + prev.opcode = glu_op; + prev.inputs = { gate, up }; + prev.outputs = { dst }; + if (!htp_kparams_solve(sess, prev)) return false; + + prev.fused.push_back(cur.node); + memset(prev.params, 0, sizeof(prev.params)); + return true; +} + +static bool fuse_mul_mul_scalar(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != HTP_OP_MUL || prev.inputs.size() != 2) return false; + + const auto * k0 = prev.kparams(); + const auto * k1 = cur.kparams(); + if (k0->kernel_type != HTP_BINARY_KERNEL_SCALAR || k1->kernel_type != HTP_BINARY_KERNEL_SCALAR) return false; + if (k0->src1_strided || k1->src1_strided) return false; + + const ggml_tensor * src2 = nullptr; + if (!htp_match_consumer(prev, cur, /*commutative=*/ false, &src2)) return false; + + const ggml_tensor * src1 = prev.src1(); + const ggml_tensor * dst = cur.dst(); + if (!ggml_are_same_shape(src1, src2) || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false; + if (htp_inputs_overlap_unsafe(prev, dst, src2)) return false; + + const size_t vtcm_size = htp_session_vtcm_size(sess); + struct htp_binary_kernel_params kparams = *k0; + kparams.src2_size = kparams.src1_size; + kparams.rows_per_buffer = 0; + if (kparams.src1_size > 65536 || kparams.src1_size + kparams.src2_size > vtcm_size) return false; + + struct htp_binary_vtcm_layout L; + htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size); + if (L.rows_per_buffer == 0 || L.total_bytes > vtcm_size) return false; + + kparams.rows_per_buffer = L.rows_per_buffer; + kparams.vtcm_size = L.total_bytes; + + prev.inputs.push_back(src2); + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +static bool fuse_binary_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) { + if (prev.opcode != cur.opcode) return false; + + const uint32_t curr_n = (uint32_t) prev.inputs.size(); + if (curr_n < 2 || curr_n >= HTP_OP_MAX_INPUTS) return false; + + const auto * k0 = prev.kparams(); + const auto * k1 = cur.kparams(); + const bool k0_ok = (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) || + (k0->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k0->is_scalar); + const bool k1_ok = (k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) || + (k1->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k1->is_scalar); + if (!k0_ok || !k1_ok) return false; + + const bool is_comm = (cur.opcode == HTP_OP_ADD || cur.opcode == HTP_OP_MUL); + const ggml_tensor * new_src = nullptr; + if (!htp_match_consumer(prev, cur, is_comm, &new_src)) return false; + + const ggml_tensor * src0 = prev.inputs[0]; + const ggml_tensor * dst = cur.dst(); + if (!ggml_are_same_shape(src0, prev.inputs[1]) || + !ggml_are_same_shape(src0, new_src) || + !htp_same_shape_and_contiguity(prev.dst(), dst)) { + return false; + } + if (htp_inputs_overlap_unsafe(prev, dst, new_src)) return false; + + const size_t vtcm_size = htp_session_vtcm_size(sess); + struct htp_binary_kernel_params kparams = *k0; + struct htp_binary_vtcm_layout L; + memset(&L, 0, sizeof(L)); + + const uint32_t n_srcs = curr_n + 1; + if (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE || k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) { + if (k0->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE) { + kparams = *k1; + } + kparams.n_srcs = (uint8_t) n_srcs; + kparams.rows_per_buffer = 0; + + auto check_split = [&](const ggml_tensor * t) { + if (src0->ne[2] > 1 && t->nb[2] != t->ne[1] * t->nb[1]) kparams.split_at_ne01 = 1; + if (src0->ne[3] > 1 && t->nb[3] != t->ne[2] * t->nb[2]) kparams.split_at_ne02 = 1; + }; + for (const auto * in : prev.inputs) check_split(in); + check_split(new_src); + check_split(dst); + + htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size); + } + + if (L.rows_per_buffer == 0 || L.total_bytes == 0 || L.total_bytes > vtcm_size) { + bool all_contig = ggml_is_contiguous(new_src) && ggml_is_contiguous(dst); + for (const auto * in : prev.inputs) { + all_contig = all_contig && ggml_is_contiguous(in); + } + if (!all_contig) return false; + + const uint32_t total_elems = (uint32_t) ggml_nelements(src0); + const uint32_t elem_size = (uint32_t) ggml_type_size(src0->type); + const uint32_t max_threads = htp_session_n_threads(sess); + if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_threads, n_srcs, false, vtcm_size)) { + return false; + } + } + + kparams.rows_per_buffer = L.rows_per_buffer; + kparams.vtcm_size = (uint32_t) L.total_bytes; + + prev.inputs.push_back(new_src); + prev.outputs[0] = dst; + prev.fused.push_back(cur.node); + prev.set_kparams(kparams); + return true; +} + +bool htp_fusion_can_stack(const struct ggml_tensor * a) { + if (!a || !a->src[0] || !a->src[1]) return false; + switch (a->op) { + case GGML_OP_MUL_MAT: + if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX)) return false; + if (htp_is_hadamard(a)) return false; + if (a->src[0]->ne[2] != 1 || a->src[0]->ne[3] != 1) return false; + return ggml_is_quantized(a->src[0]->type); + case GGML_OP_MUL_MAT_ID: + if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX)) return false; + return a->src[2] != nullptr && ggml_is_quantized(a->src[0]->type); + default: + return false; + } +} + +bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b) { + if (!htp_fusion_can_stack(b) || a->op != b->op || a->src[1] != b->src[1]) { + return false; + } + if (a->op == GGML_OP_MUL_MAT_ID && a->src[2] != b->src[2]) { + return false; + } + const ggml_tensor * w0 = a->src[0]; + const ggml_tensor * w1 = b->src[0]; + return w0->type == w1->type && w0->ne[0] == w1->ne[0] && w0->ne[2] == w1->ne[2]; +} + +int htp_fusion_max(const struct ggml_cgraph * graph, int idx) { + const int n = graph->n_nodes; + const ggml_op op0 = graph->nodes[idx]->op; + if (op0 != GGML_OP_ADD && op0 != GGML_OP_NORM && op0 != GGML_OP_RMS_NORM) { + return 1; + } + + enum ggml_op ops[HTP_OP_MAX_FUSED]; + ops[0] = op0; + + int f = idx + 1; + while (f < n && f < idx + HTP_OP_MAX_FUSED) { + const ggml_op op = graph->nodes[f]->op; + if (op != GGML_OP_ADD && op != GGML_OP_MUL && op != GGML_OP_SCALE && + op != GGML_OP_NORM && op != GGML_OP_RMS_NORM) { + break; + } + ops[f - idx] = op; + f++; + } + + f -= idx; + for (; f > 1; f--) { + if (ggml_can_fuse(graph, idx, ops, f)) { + break; + } + } + return f; +} + +void htp_fusion_tag_graph(struct ggml_cgraph * graph) { + for (int i = 0; i < graph->n_nodes; i++) { + auto * extra = (htp_tensor_extra *) graph->nodes[i]->extra; + if (!extra) continue; + + extra->flags &= ~HTP_TENSOR_FUSEABLE; + + const ggml_op op = graph->nodes[i]->op; + if ((op == GGML_OP_RMS_NORM || op == GGML_OP_SCALE || op == GGML_OP_REPEAT || + op == GGML_OP_UNARY || op == GGML_OP_ADD || op == GGML_OP_SUB || + op == GGML_OP_MUL || op == GGML_OP_DIV || op == GGML_OP_MUL_MAT_ID || + op == GGML_OP_SSM_CONV || + (op == GGML_OP_MUL_MAT && !htp_is_hadamard(graph->nodes[i]))) && + ggml_node_has_n_uses(graph, i, 1)) { + extra->flags |= HTP_TENSOR_FUSEABLE; + } + } +} + +bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur) { + if (!opt_opfusion) return false; + + switch (cur.opcode) { + case HTP_OP_ADD: + return (htp_fusion_enabled(HTP_FUSE_ALLREDUCE_ADD) && fuse_allreduce_add(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_MUL_MAT_ADD) && fuse_mul_mat_add(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur)); + case HTP_OP_MUL: + return (htp_fusion_enabled(HTP_FUSE_RMS_NORM_MUL) && fuse_rms_norm_mul(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_UNARY_MUL_GLU) && fuse_unary_mul_glu(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_MUL_MUL_SCALAR) && fuse_mul_mul_scalar(sess, prev, cur)) || + (htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur)); + case HTP_OP_SUB: + case HTP_OP_DIV: + return htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur); + case HTP_OP_SCALE: + return htp_fusion_enabled(HTP_FUSE_RMS_NORM_SCALE) && fuse_rms_norm_scale(sess, prev, cur); + case HTP_OP_UNARY_SILU: + return htp_fusion_enabled(HTP_FUSE_SSM_CONV_SILU) && fuse_ssm_conv_silu(sess, prev, cur); + case HTP_OP_GET_ROWS: + return htp_fusion_enabled(HTP_FUSE_REPEAT_GET_ROWS) && fuse_repeat_get_rows(sess, prev, cur); + case HTP_OP_CPY: + return htp_fusion_enabled(HTP_FUSE_GDN_CPY) && fuse_gdn_cpy(sess, prev, cur); + case HTP_OP_MUL_MAT: + return htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX) && fuse_mul_mat_nx(sess, prev, cur); + case HTP_OP_MUL_MAT_ID: + return htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX) && fuse_mul_mat_id_nx(sess, prev, cur); + default: + return false; + } +} diff --git a/ggml/src/ggml-hexagon/htp-fusion.h b/ggml/src/ggml-hexagon/htp-fusion.h new file mode 100644 index 0000000..e794c1d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-fusion.h @@ -0,0 +1,33 @@ +#ifndef HTP_FUSION_H +#define HTP_FUSION_H + +#include "htp-common.h" + +enum htp_fusion_flags { + HTP_FUSE_ALLREDUCE_ADD = (1 << 1), // 2 + HTP_FUSE_RMS_NORM_MUL = (1 << 2), // 4 + HTP_FUSE_MUL_MAT_ADD = (1 << 3), // 8 + HTP_FUSE_MUL_MAT_NX = (1 << 4), // 16 + HTP_FUSE_MUL_MAT_ID_NX = (1 << 5), // 32 + HTP_FUSE_GDN_CPY = (1 << 6), // 64 + HTP_FUSE_REPEAT_GET_ROWS = (1 << 7), // 128 + HTP_FUSE_UNARY_MUL_GLU = (1 << 8), // 256 + HTP_FUSE_MUL_MUL_SCALAR = (1 << 9), // 512 + HTP_FUSE_BINARY_NX = (1 << 10), // 1024 + HTP_FUSE_RMS_NORM_SCALE = (1 << 11), // 2048 + HTP_FUSE_SSM_CONV_SILU = (1 << 12), // 4096 +}; + +static inline bool htp_fusion_enabled(int flag) { + if (opt_opfusion <= 0) return false; + if (opt_opfusion == 1) return true; // 1 enables all + return (opt_opfusion & flag) != 0; +} + +bool htp_fusion_can_stack(const struct ggml_tensor * a); +bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b); +int htp_fusion_max(const struct ggml_cgraph * graph, int idx); +void htp_fusion_tag_graph(struct ggml_cgraph * graph); +bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur); + +#endif // HTP_FUSION_H diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp new file mode 100644 index 0000000..b8d1aee --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -0,0 +1,3514 @@ +#define _USE_MATH_DEFINES + +#include "htp-kparams.h" +#include "htp-opnode.h" +#include "ggml-impl.h" + +#include +#include +#include + +#ifndef M_LOG2E +# define M_LOG2E 1.44269504088896340736 +#endif + +htp_op_code htp_op_remap(const struct ggml_tensor * t) { + switch (t->op) { + case GGML_OP_FLASH_ATTN_EXT: return HTP_OP_FLASH_ATTN_EXT; + case GGML_OP_MUL_MAT: return htp_is_hadamard(t) ? HTP_OP_HADAMARD : HTP_OP_MUL_MAT; + case GGML_OP_MUL_MAT_ID: return HTP_OP_MUL_MAT_ID; + case GGML_OP_MUL: return HTP_OP_MUL; + case GGML_OP_ADD: return HTP_OP_ADD; + case GGML_OP_ADD_ID: return HTP_OP_ADD_ID; + case GGML_OP_SUB: return HTP_OP_SUB; + case GGML_OP_DIV: return HTP_OP_DIV; + case GGML_OP_DUP: + case GGML_OP_CPY: + case GGML_OP_CONT: return HTP_OP_CPY; + case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS; + case GGML_OP_SET_ROWS: return HTP_OP_SET_ROWS; + case GGML_OP_SUM: return HTP_OP_SUM; + case GGML_OP_SUM_ROWS: return HTP_OP_SUM_ROWS; + case GGML_OP_POOL_2D: return HTP_OP_POOL_2D; + case GGML_OP_POOL_1D: return HTP_OP_POOL_1D; + case GGML_OP_ARGSORT: return HTP_OP_ARGSORT; + case GGML_OP_TOP_K: return HTP_OP_TOP_K; + case GGML_OP_ARGMAX: return HTP_OP_ARGMAX; + case GGML_OP_NORM: return HTP_OP_NORM; + case GGML_OP_L2_NORM: return HTP_OP_L2_NORM; + case GGML_OP_RMS_NORM: return HTP_OP_RMS_NORM; + case GGML_OP_CONCAT: return HTP_OP_CONCAT; + case GGML_OP_SCALE: return HTP_OP_SCALE; + case GGML_OP_CLAMP: return HTP_OP_CLAMP; + case GGML_OP_LEAKY_RELU: return HTP_OP_LEAKY_RELU; + case GGML_OP_SQR: return HTP_OP_SQR; + case GGML_OP_SQRT: return HTP_OP_SQRT; + case GGML_OP_LOG: return HTP_OP_UNARY_LOG; + case GGML_OP_SOFT_MAX: return HTP_OP_SOFTMAX; + case GGML_OP_SSM_CONV: return HTP_OP_SSM_CONV; + case GGML_OP_GATED_DELTA_NET: return HTP_OP_GATED_DELTA_NET; + case GGML_OP_ROPE: return HTP_OP_ROPE; + case GGML_OP_REPEAT: return HTP_OP_REPEAT; + case GGML_OP_CUMSUM: return HTP_OP_CUMSUM; + case GGML_OP_FILL: return HTP_OP_FILL; + case GGML_OP_DIAG: return HTP_OP_DIAG; + case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI; + case GGML_OP_TRI: return HTP_OP_TRI; + case GGML_OP_PAD: return HTP_OP_PAD; + case GGML_OP_IM2COL: return HTP_OP_IM2COL; + case GGML_OP_IM2COL_3D: return HTP_OP_IM2COL_3D; + case GGML_OP_ROLL: return HTP_OP_ROLL; + + case GGML_OP_UNARY: + switch (ggml_get_unary_op(t)) { + case GGML_UNARY_OP_SILU: return HTP_OP_UNARY_SILU; + case GGML_UNARY_OP_GELU: return HTP_OP_UNARY_GELU; + case GGML_UNARY_OP_GELU_QUICK: return HTP_OP_UNARY_GELU_QUICK; + case GGML_UNARY_OP_GELU_ERF: return HTP_OP_UNARY_GELU_ERF; + case GGML_UNARY_OP_SIGMOID: return HTP_OP_UNARY_SIGMOID; + case GGML_UNARY_OP_NEG: return HTP_OP_UNARY_NEG; + case GGML_UNARY_OP_EXP: return HTP_OP_UNARY_EXP; + case GGML_UNARY_OP_SOFTPLUS: return HTP_OP_UNARY_SOFTPLUS; + case GGML_UNARY_OP_TANH: return HTP_OP_UNARY_TANH; + case GGML_UNARY_OP_ABS: return HTP_OP_UNARY_ABS; + case GGML_UNARY_OP_RELU: return HTP_OP_UNARY_RELU; + case GGML_UNARY_OP_STEP: return HTP_OP_UNARY_STEP; + case GGML_UNARY_OP_CEIL: return HTP_OP_UNARY_CEIL; + case GGML_UNARY_OP_SGN: return HTP_OP_UNARY_SGN; + default: break; + } + break; + + case GGML_OP_GLU: + switch (ggml_get_glu_op(t)) { + case GGML_GLU_OP_REGLU: return HTP_OP_GLU_REGLU; + case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU; + case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI; + case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP; + case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU; + case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK; + case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF; + default: break; + } + break; + + default: + break; + } + return HTP_OP_INVALID; +} + +bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride) { + size_t s = 0, next = 0; + for (int i = 1; i < GGML_MAX_DIMS; i++) { + if (ne[i] == 1) continue; + if (s == 0) { s = nb[i]; next = s * ne[i]; continue; } + if (nb[i] != next) return false; + next *= ne[i]; + } + *stride = s ? s : nb[1]; + return true; +} + +bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst) { + size_t s1, sd; + return (dst->op == GGML_OP_MUL_MAT || dst->op == GGML_OP_ADD) && + src0->ne[2] == 1 && src0->ne[3] == 1 && src1->ne[2] * src1->ne[3] > 1 && + src1->nb[0] == ggml_type_size(src1->type) && htp_rows_stride(src1->ne, src1->nb, &s1) && + dst->nb[0] == ggml_type_size(dst->type) && htp_rows_stride(dst->ne, dst->nb, &sd); +} + +ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t) { + size_t stride = 0; + htp_rows_stride(t->ne, t->nb, &stride); + ggml_tensor c = *t; + c.ne[1] = t->ne[1] * t->ne[2] * t->ne[3]; + c.ne[2] = 1; + c.ne[3] = 1; + c.nb[1] = stride; + c.nb[2] = c.nb[1] * c.ne[1]; + c.nb[3] = c.nb[2]; + return c; +} + +bool htp_kparams_solve_allreduce( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * dst, + uint32_t rank, + uint32_t n_ranks, + bool has_add, + bool is_row_bcast, + bool is_shard_ok, + struct htp_allreduce_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->rank = (int32_t) rank; + kparams->n_ranks = (int32_t) n_ranks; + kparams->is_row_bcast = (has_add && is_row_bcast) ? 1 : 0; + + const uint32_t max_threads = htp_session_n_threads(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const uint32_t nelem = (uint32_t) ggml_nelements(dst); + const uint32_t elem_size = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); + const bool is_contiguous = ggml_is_contiguous(dst); + + const uint32_t ne0 = (uint32_t) dst->ne[0]; + const uint32_t ne1 = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + kparams->ne0 = (int32_t) ne0; + kparams->ne1 = (int32_t) ne1; + + const bool use_1d = is_contiguous && !(has_add && is_row_bcast && ne1 > 1); + + if (has_add) { + if (opt_ar_scatter && is_shard_ok && use_1d && !is_row_bcast) { + const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128); + const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem); + const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem); + kparams->n_dsts = (int32_t) n_ranks; + kparams->rank_elem_start = (int32_t) rank_elem_start; + kparams->rank_nelem = (int32_t) (rank_elem_end - rank_elem_start); + } else if (use_1d) { + kparams->n_dsts = 1; + kparams->rank_elem_start = 0; + kparams->rank_nelem = (int32_t) nelem; + } else { + kparams->n_dsts = 1; + kparams->rank_elem_start = 0; + kparams->rank_nelem = (int32_t) ne1; + } + } else { + kparams->n_dsts = (int32_t) n_ranks; + if (use_1d) { + const uint32_t rank_chunk_elems = hex_round_up((nelem + n_ranks - 1) / n_ranks, 128); + const uint32_t rank_elem_start = (std::min)(rank * rank_chunk_elems, nelem); + const uint32_t rank_elem_end = (std::min)(rank_elem_start + rank_chunk_elems, nelem); + const uint32_t rank_nelem = rank_elem_end - rank_elem_start; + kparams->rank_elem_start = (int32_t) rank_elem_start; + kparams->rank_nelem = (int32_t) rank_nelem; + } else { + const uint32_t rank_chunk_rows = (ne1 + n_ranks - 1) / n_ranks; + const uint32_t rank_r0 = (std::min)(rank * rank_chunk_rows, ne1); + const uint32_t rank_r1 = (std::min)(rank_r0 + rank_chunk_rows, ne1); + const uint32_t rank_nrows = rank_r1 - rank_r0; + kparams->rank_elem_start = (int32_t) rank_r0; + kparams->rank_nelem = (int32_t) rank_nrows; + } + } + + kparams->mode = (kparams->n_dsts > 1) ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL; + + if (use_1d) { + const uint32_t rank_nelem = (uint32_t) kparams->rank_nelem; + const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nelem / 128)); + kparams->n_threads = n_threads; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); + + uint32_t block_elems = 65536; + if (block_elems > rank_nelem / n_threads && rank_nelem / n_threads > 128) { + block_elems = hex_round_up(rank_nelem / (n_threads * 2), 128); + } + block_elems = (std::max)(128u, block_elems); + + kparams->block_elems = block_elems; + kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + + while ((size_t) kparams->vtcm_size > vtcm_size && block_elems > 128) { + const size_t max_bytes_per_buf = vtcm_size / (n_vtcm_buffers * 2); + block_elems = (uint32_t) hex_align_down((size_t) (max_bytes_per_buf / elem_size), 128); + if (block_elems < 128) break; + kparams->block_elems = block_elems; + kparams->vtcm_size_per_thread = 2 * block_elems * elem_size; + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + } + + if (vtcm_size < (size_t) kparams->vtcm_size || block_elems < 128) { + HEX_VERBOSE("ggml-hex: %s allreduce 1D solver failed to fit VTCM (%d > %zu)\n", + htp_session_name(sess), kparams->vtcm_size, vtcm_size); + return false; + } + + kparams->elems_per_thread = hex_round_up((rank_nelem + n_threads - 1) / n_threads, block_elems); + kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_1D; + return true; + } else { + const uint32_t rank_nrows = (uint32_t) kparams->rank_nelem; + const uint32_t n_threads = (std::min)(max_threads, (std::max)(1u, rank_nrows)); + kparams->n_threads = n_threads; + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(n_ranks, n_threads, has_add, is_row_bcast); + + const uint32_t row_bytes = ne0 * elem_size; + const uint32_t row_size_aligned = (uint32_t) hex_align_up(row_bytes, 128); + kparams->row_size_aligned = row_size_aligned; + + const uint32_t nrows_per_thread = (rank_nrows + n_threads - 1) / n_threads; + uint32_t block_rows = (std::min)(128u, nrows_per_thread); + block_rows = (std::max)(1u, block_rows); + kparams->block_elems = block_rows; + + kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + + while ((size_t) kparams->vtcm_size > vtcm_size && block_rows > 1) { + const size_t max_rows_per_buf = vtcm_size / (n_vtcm_buffers * 2 * row_size_aligned); + block_rows = (std::max)(1u, (uint32_t) max_rows_per_buf); + kparams->block_elems = block_rows; + kparams->vtcm_size_per_thread = 2 * (block_rows * row_size_aligned); + kparams->vtcm_size = n_vtcm_buffers * kparams->vtcm_size_per_thread; + if (max_rows_per_buf == 0) break; + } + + if (vtcm_size < (size_t) kparams->vtcm_size || block_rows < 1) { + HEX_VERBOSE("ggml-hex: %s allreduce 2D solver failed to fit VTCM (%d > %zu)\n", + htp_session_name(sess), kparams->vtcm_size, vtcm_size); + return false; + } + + kparams->elems_per_thread = nrows_per_thread; + kparams->kernel_type = HTP_ALLREDUCE_KERNEL_DMA_2D; + return true; + } +} + +static bool htp_flash_attn_is_hmx_eligible( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * v, + const struct ggml_tensor * sinks +) { + if (htp_session_n_hmx(sess) == 0 || opt_fa_select < 2) { + return false; + } + + if ((k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) || + (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) { + return false; + } + + const uint32_t DK = q->ne[0]; + const uint32_t DV = v->ne[0]; + + if (DK % 8 != 0 || DV % 8 != 0) { + return false; + } + + GGML_UNUSED(sinks); + + if (opt_fa_select > 2) { + return true; + } + + const uint32_t M = q->ne[1]; + if (M > 1) { + return true; + } + + const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_head = q->ne[2]; + const uint32_t n_kv_heads = k->ne[2]; + const uint32_t G = n_kv_heads > 0 ? n_head / n_kv_heads : 1; + const uint32_t S = k->ne[1]; + const uint32_t total_threads = n_threads * htp_session_mdev_count(sess); + + if (DK > 256 || (DK == 256 && (DV > 256 || n_kv_heads < total_threads))) { + return true; + } + + const bool is_tile_aligned = (G > 0 && (32 % G == 0)); + if (!is_tile_aligned) { + if (n_threads >= 6) { + return false; + } + return S >= 1024; + } + + uint32_t s_cross = 512; + if (DK == 256) { + s_cross = 4096; + } else if (DK <= 64) { + s_cross = (n_threads >= 8) ? 2048 : ((n_threads >= 6) ? 768 : 512); + } else { + s_cross = (n_threads >= 8) ? 1024 : 512; + } + + return S >= s_cross; +} + +static bool htp_kparams_solve_flash_attn( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_fa_kernel_params * kparams +) { + if (opt_fa_select < 1) { + return false; + } + + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * q = op->src[0]; + const struct ggml_tensor * k = op->src[1]; + const struct ggml_tensor * v = op->src[2]; + const struct ggml_tensor * mask = op->src[3]; + const struct ggml_tensor * sinks = op->src[4]; + const struct ggml_tensor * dst = op; + + if ((q->type != GGML_TYPE_F16 && q->type != GGML_TYPE_F32) || + (k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0) || + (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_Q8_0)) { + return false; + } + if (mask && mask->type != GGML_TYPE_F16) { + return false; + } + if (sinks && sinks->type != GGML_TYPE_F32) { + return false; + } + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) { + return false; + } + + const uint32_t n_threads = htp_session_n_threads(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const uint32_t neq0 = q->ne[0]; + const uint32_t neq1 = q->ne[1]; + const uint32_t neq2 = q->ne[2]; + const uint32_t nek1 = k->ne[1]; + const uint32_t nev0 = v->ne[0]; + + const uint32_t DK = neq0; + const uint32_t DV = nev0; + + const uint32_t n_kv_heads = k->ne[2]; + const uint32_t G = neq2 / n_kv_heads; + + float scale = 1.0f; + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&scale, &op->op_params[0], sizeof(float)); + memcpy(&max_bias, &op->op_params[1], sizeof(float)); + memcpy(&logit_softcap, &op->op_params[2], sizeof(float)); + + if (logit_softcap != 0.0f) { + scale /= logit_softcap; + } + + kparams->scale = scale; + kparams->max_bias = max_bias; + kparams->logit_softcap = logit_softcap; + kparams->head_split = (opt_fa_split != 0) ? 1 : 0; + kparams->flags = 0; + kparams->G = G; + + // team-split only when a team has >1 member (n_kv_heads < N_cores) + uint32_t fa_teams = 1, fa_team_size = 1; + const uint32_t mdev_count = htp_session_mdev_count(sess); + if (opt_fa_split && mdev_count > 1) { + htp_fa_mdev_teams(n_kv_heads, mdev_count, kparams->head_split != 0, &fa_teams, &fa_team_size); + if (fa_team_size > 1) { + kparams->flags |= HTP_FA_FLAG_TEAM_SPLIT; + } + } + + const uint32_t n_head = q->ne[2]; + kparams->n_head_log2 = 1u << (uint32_t) std::floor(std::log2(n_head)); + kparams->m0 = std::pow(2.0f, -(max_bias) / kparams->n_head_log2); + kparams->m1 = std::pow(2.0f, -(max_bias / 2.0f) / kparams->n_head_log2); + + if (htp_flash_attn_is_hmx_eligible(sess, q, k, v, sinks)) { + const uint32_t DK_pad = hex_round_up(DK, 64); + const uint32_t DV_pad = hex_round_up(DV, 64); + size_t Br = 0, Bc = 0; + int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK_pad, DV_pad, neq1, nek1, vtcm_size, n_threads, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head, fa_team_size); + if (ret == 0) { + kparams->kernel_type = HTP_FA_KERNEL_HMX; + if (logit_softcap == 0.0f) { + kparams->scale = scale * (float) M_LOG2E; + kparams->logit_softcap = 0.0f; + } else { + kparams->scale = scale; + kparams->logit_softcap = logit_softcap * (float) M_LOG2E; + } + kparams->Br = Br; + kparams->Bc = Bc; + kparams->n_kv_blocks = (nek1 + Bc - 1) / Bc; + kparams->n_threads = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1; + kparams->u.hmx.g_br = hex_align_up(G * Br, 32); + kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && n_threads >= 2) ? 1 : 0; + kparams->vtcm_size = hmx_fa_compute_vtcm_usage(G, DK_pad, DV_pad, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, (q->type == GGML_TYPE_F32), sinks != nullptr, n_head); + kparams->u.hmx.mask_broadcast = (mask != nullptr && mask->ne[2] == 1) ? 1 : 0; + kparams->u.hmx.div_G = init_fastdiv_values(G); + if (mask) { + kparams->src3_div2 = init_fastdiv_values(mask->ne[2]); + kparams->src3_div3 = init_fastdiv_values(mask->ne[3]); + } + return true; + } + } + + kparams->kernel_type = HTP_FA_KERNEL_HVX; + kparams->Br = 1; + kparams->Bc = 64; + kparams->n_kv_blocks = (k->ne[1] + 64 - 1) / 64; + kparams->n_threads = n_threads; + + const size_t size_q_row_padded = hex_round_up(q->ne[0] * ((q->type == GGML_TYPE_F32) ? 4 : 2), 128); + const size_t size_k_row_padded = hex_round_up(k->ne[0] * 2, 128); + const size_t size_v_row_padded = hex_round_up(v->ne[0] * 2, 128); + + kparams->vtcm_size = hvx_fa_compute_vtcm_usage(DK, DV, (q->type == GGML_TYPE_F32), mask != nullptr, sinks != nullptr, n_head, n_threads); + if ((size_t) kparams->vtcm_size > vtcm_size) { + HEX_VERBOSE("ggml-hex: skip flash_attn_ext because VTCM needed (%d) > budget (%zu)\n", + kparams->vtcm_size, vtcm_size); + return false; + } + + kparams->u.hvx.size_q_row_padded = size_q_row_padded; + kparams->u.hvx.size_k_row_padded = size_k_row_padded; + kparams->u.hvx.size_v_row_padded = size_v_row_padded; + kparams->u.hvx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]); + kparams->u.hvx.src0_div1 = init_fastdiv_values(q->ne[1]); + kparams->broadcast_rk2 = init_fastdiv_values(q->ne[2] / k->ne[2]); + kparams->broadcast_rk3 = init_fastdiv_values(q->ne[3] / k->ne[3]); + kparams->broadcast_rv2 = init_fastdiv_values(q->ne[2] / v->ne[2]); + kparams->broadcast_rv3 = init_fastdiv_values(q->ne[3] / v->ne[3]); + if (mask) { + kparams->src3_div2 = init_fastdiv_values(mask->ne[2]); + kparams->src3_div3 = init_fastdiv_values(mask->ne[3]); + } + + return true; +} + +static bool htp_kparams_solve_gated_delta_net( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_gdn_kernel_params * kparams +) { + if (opt_gdn_select < 1) { + return false; + } + + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * q = op->src[0]; + const struct ggml_tensor * k = op->src[1]; + const struct ggml_tensor * v = op->src[2]; + const struct ggml_tensor * g = op->src[3]; + const struct ggml_tensor * beta = op->src[4]; + const struct ggml_tensor * state = op->src[5]; + const struct ggml_tensor * dst = op; + + if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 || + g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 || + dst->type != GGML_TYPE_F32) { + return false; + } + + if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) || + !ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) || + !ggml_is_contiguous(dst)) { + return false; + } + + const int64_t S_v = v->ne[0]; + const int64_t H = v->ne[1]; + const int64_t n_tokens = v->ne[2]; + const int64_t n_seqs = v->ne[3]; + const int64_t K = ggml_get_op_params_i32(op, 0); + + if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) { + return false; + } + if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 || + q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 || + (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) { + return false; + } + if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) { + return false; + } + if (state->ne[0] != S_v || state->ne[1] != S_v || state->ne[2] != H || state->ne[3] != n_seqs) { + return false; + } + if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K || + dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + + const uint32_t rq3 = (uint32_t) (n_seqs / q->ne[3]); + const uint32_t rk3 = (uint32_t) (n_seqs / k->ne[3]); + const uint32_t total_rows = (uint32_t) (H * n_seqs); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), total_rows); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const bool can_use_hmx = (opt_gdn_select >= 2) && + (htp_session_n_hmx(sess) > 0) && + (S_v % 64 == 0) && + (n_tokens >= HTP_GDN_MIN_TOKENS) && + (g->ne[0] == 1) && + (K == 1); + + struct htp_gdn_hmx_vtcm_layout hmx_layout; + struct htp_gdn_vtcm_layout hvx_layout; + uint32_t n_heads_batch = 1; + + if (can_use_hmx && htp_gdn_hmx_solve_layout(&hmx_layout, (uint32_t) S_v, HTP_GDN_CHUNK_SIZE, total_rows, vtcm_size, n_threads, true, &n_heads_batch)) { + kparams->kernel_type = HTP_GDN_KERNEL_HMX_CHUNKED; + kparams->pipeline = hmx_layout.pipeline ? 1 : 0; + kparams->chunk_size = HTP_GDN_CHUNK_SIZE; + kparams->n_chunks = ((uint32_t) n_tokens + HTP_GDN_CHUNK_SIZE - 1) / HTP_GDN_CHUNK_SIZE; + kparams->n_heads_batch = (uint16_t) n_heads_batch; + kparams->vtcm_size = (uint32_t) hmx_layout.total_bytes; + } else { + htp_gdn_vtcm_layout_build(&hvx_layout, (uint32_t) S_v, n_threads); + if (hvx_layout.total_bytes > vtcm_size) { + return false; + } + kparams->kernel_type = HTP_GDN_KERNEL_HVX_RECURRENT; + kparams->pipeline = 0; + kparams->n_heads_batch = 1; + kparams->vtcm_size = (uint32_t) hvx_layout.total_bytes; + } + + kparams->n_threads = n_threads; + kparams->S_v = (uint32_t) S_v; + kparams->H = (uint32_t) H; + kparams->n_tokens = (uint32_t) n_tokens; + kparams->n_seqs = (uint32_t) n_seqs; + kparams->K = (uint32_t) K; + kparams->total_rows = total_rows; + kparams->kda = (g->ne[0] == S_v) ? 1 : 0; + kparams->state_seq_stride = (uint32_t) (state->nb[3] / sizeof(float)); + kparams->state_size_per_snap = (uint32_t) (S_v * S_v * H * n_seqs); + kparams->scale = 1.0f / sqrtf((float) S_v); + + if (H > 0) kparams->div_H = init_fastdiv_values((uint32_t) H); + if (q->ne[1] > 0) kparams->div_q1 = init_fastdiv_values((uint32_t) q->ne[1]); + if (k->ne[1] > 0) kparams->div_k1 = init_fastdiv_values((uint32_t) k->ne[1]); + if (rq3 > 0) kparams->div_rq3 = init_fastdiv_values(rq3); + if (rk3 > 0) kparams->div_rk3 = init_fastdiv_values(rk3); + return true; +} + +static bool htp_matmul_is_hmx_eligible( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + bool is_matmul_id, + bool is_batched +) { + GGML_UNUSED(dst); + + if (src1->type != GGML_TYPE_F32 && (src1->type != GGML_TYPE_F16 || is_matmul_id)) { + return false; + } + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int wtype = src0->type; + + if (ne01 % HTP_TENSOR_TILE_N_ROWS != 0 && is_matmul_id) { + return false; + } + + if (!htp_is_hmx_weight_type((ggml_type) wtype)) { + return false; + } + + if (ne00 % HTP_TENSOR_TILE_N_COLS != 0) { + return false; + } + + const bool is_tiled = htp_tensor_is_tiled(src0); + if (ggml_is_quantized((ggml_type) wtype) && !is_tiled) { + return false; + } + if (!is_matmul_id && is_batched && is_tiled) { + return false; + } + + if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) { + return false; + } + + const int m = is_matmul_id ? ne12 : ne11; + if (m <= HTP_MM_HMX_MIN_NROWS) { + return false; + } + + return true; +} + +static bool htp_kparams_solve_hmx_mm( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + const struct ggml_tensor * dst, + int wtype, + int ne00_padded, + int ne01_padded, + int ne02, + int ne11, + int ne12, + int ne11_padded, + bool is_matmul_id, + bool is_batched, + size_t src2_size, + size_t vtcm_budget, + struct htp_mm_kernel_params * kparams +) { + const bool is_tiled = htp_tensor_is_tiled(src0); + const int aligned_tile_size = is_tiled ? htp_tensor_get_tile_padded_size(wtype) : 0; + const int n_threads = (int) htp_session_n_threads(sess); + const uint32_t mdev_count = htp_session_mdev_count(sess); + const int ne10 = act->ne[0]; + + int m_for_solver = ne11; + int m_for_solver_padded = ne11_padded; + if (!is_matmul_id && mdev_count > 1 && ((uint32_t) ne11 >= mdev_count)) { + const bool dst_row_split = dst ? htp_can_row_partition(dst) : true; + const bool act_row_split = htp_can_row_partition(act); + if (dst_row_split && act_row_split) { + m_for_solver = (ne11 + (int) mdev_count - 1) / (int) mdev_count; + m_for_solver_padded = hex_round_up(std::max(m_for_solver, 32), 32); + } + } + const bool pipeline = is_matmul_id ? false : htp_mm_hmx_pipeline(m_for_solver); + const bool is_batched_val = is_matmul_id ? false : is_batched; + const int group_size = (ne02 > 0 ? ne12 / ne02 : 1); + + size_t m_chunk = 0; + size_t n_chunk = 0; + size_t vtcm_size = 0; + bool use_grouped = false; + int act_threads_selected = 0; + + if (is_batched_val && wtype == GGML_TYPE_F16 && !is_tiled && group_size > 1) { + if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, m_for_solver, group_size, n_threads, pipeline, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { + use_grouped = true; + } + } + + if (!use_grouped) { + int m_id_rows = 0; + if (dst && is_matmul_id) { + const int n_experts = ne02 > 0 ? ne02 : 1; + const size_t total_expert_rows = (size_t) dst->ne[1] * dst->ne[2]; + int m_per_expert = (int) ((total_expert_rows + n_experts - 1) / n_experts); + if (mdev_count > 1 && htp_can_row_partition(dst)) { + m_per_expert = (m_per_expert + (int) mdev_count - 1) / (int) mdev_count; + } + m_id_rows = hex_round_up(std::max(m_per_expert, 32), 32); + } + const uint32_t cost_m = is_matmul_id ? (uint32_t) m_id_rows : (uint32_t) m_for_solver; + if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, m_for_solver_padded, cost_m, n_threads, pipeline, is_matmul_id, aligned_tile_size, src2_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { + return false; + } + } + + kparams->n_hmx = 1; + kparams->pipeline = pipeline ? 1 : 0; + kparams->m_chunk = m_chunk; + kparams->n_chunk = n_chunk; + kparams->n_threads = n_threads; + kparams->n_act_threads = act_threads_selected; + kparams->tile_size = is_tiled ? htp_tensor_get_tile_size(wtype) : 0; + kparams->aligned_tile_size = aligned_tile_size; + kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + kparams->vtcm_size = vtcm_size; + kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected); + kparams->div_ne00_padded = init_fastdiv_values(ne00_padded); + kparams->vtcm_bias_size = (int32_t) src2_size; + kparams->kernel_type = (is_batched && !is_matmul_id) ? HTP_MM_KERNEL_HMX_BATCHED : HTP_MM_KERNEL_HMX_2D; + return true; +} + +static void htp_kparams_solve_hvx_mm( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + int wtype, + bool is_tiled, + int ne02, + int ne03, + int ne10, + int ne11, + int ne12, + int ne13, + bool is_matmul_id, + const size_t src2_row_size, + size_t vtcm_budget, + struct htp_mm_kernel_params * kparams +) { + if (opt_mm_select < 1) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; + } + + const uint32_t n_threads = htp_session_n_threads(sess); + kparams->n_hmx = 0; + kparams->n_threads = n_threads; + + GGML_UNUSED(ne02); + GGML_UNUSED(ne03); + + const int src1_nrows = ne11 * ne12 * ne13; + + if (is_tiled) { + kparams->tile_size = htp_tensor_get_tile_size(wtype); + kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + + const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0); + + if (is_matmul_id) { + kparams->kernel_type = (src1_nrows < (int) n_threads) + ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; + kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + + struct htp_mm_hvx_vtcm_layout L; + uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + uint32_t best_n_prefetch = 2; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads, + 0, src0->nb[1], kparams->act_row_size, 0, d, true, false + ); + if (L.total_bytes <= vtcm_budget) { + best_n_prefetch = d; + break; + } + } + if (L.total_bytes > vtcm_budget) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; + } + kparams->n_prefetch = best_n_prefetch; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_bias_size = 0; + return; + } else { + if (k_align && opt_mm_select >= 1) { + kparams->act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + kparams->kernel_type = (src1_nrows < (int) n_threads) + ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; + + struct htp_mm_hvx_vtcm_layout L; + uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + uint32_t best_n_prefetch = 2; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + htp_mm_hvx_vtcm_layout_build( + &L, kparams->kernel_type, wtype, ne10, src1_nrows, n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, d, false, false + ); + if (L.total_bytes <= vtcm_budget) { + best_n_prefetch = d; + break; + } + } + + uint32_t m_chunk = 0; + if (htp_mm_hvx_solve_vtcm_params( + kparams->kernel_type, wtype, ne10, src1_nrows, n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, best_n_prefetch, vtcm_budget, + &L, &m_chunk)) { + kparams->n_prefetch = best_n_prefetch; + kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_bias_size = L.bias_bytes; + return; + } + } + + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; + } + } else if (is_matmul_id) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return; + } else if (wtype == GGML_TYPE_F16 || wtype == GGML_TYPE_F32 || wtype == GGML_TYPE_BF16) { + int32_t ktype = HTP_MM_KERNEL_HVX_F16_F16_VTCM; + int32_t act_elem = 2; + if (wtype == GGML_TYPE_F32) { + ktype = HTP_MM_KERNEL_HVX_F32_F32_VTCM; + act_elem = 4; + } else if (wtype == GGML_TYPE_BF16) { + ktype = HTP_MM_KERNEL_HVX_BF16_F32_VTCM; + act_elem = 4; + } + struct htp_mm_hvx_vtcm_layout L; + uint32_t m_chunk = 0; + if (htp_mm_hvx_solve_vtcm_params( + ktype, wtype, ne10, src1_nrows, n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], src2_row_size, 16, vtcm_budget, + &L, &m_chunk)) { + kparams->kernel_type = ktype; + kparams->m_chunk = (m_chunk < (uint32_t) src1_nrows) ? m_chunk : 0; + kparams->act_row_size = hex_round_up(ne10 * act_elem, 128); + kparams->vtcm_size = L.total_bytes; + kparams->vtcm_bias_size = L.bias_bytes; + kparams->n_prefetch = 16; + return; + } + } + + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; +} + +static bool htp_kparams_solve_matmul_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + const size_t src2_row_size, + const size_t src2_size, + struct htp_mm_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; + + const bool is_tiled = htp_tensor_is_tiled(src0); + const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; + const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00; + const int ne11_padded = hex_round_up(ne11, 32); + const int ne01_tiled = hex_round_up(ne01, 32); + + const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID); + const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1); + const size_t vtcm_budget = htp_session_vtcm_size(sess); + + const bool hmx_enabled = (htp_session_n_hmx(sess) > 0) && (opt_mm_select >= 2); + if (hmx_enabled && htp_matmul_is_hmx_eligible(src0, src1, dst, is_matmul_id, is_batched)) { + if (htp_kparams_solve_hmx_mm(sess, src0, src1, dst, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, src2_size, vtcm_budget, kparams)) { + goto finalize; + } + } + + htp_kparams_solve_hvx_mm(sess, src0, src1, dst, wtype, is_tiled, ne02, ne03, ne10, ne11, ne12, ne13, is_matmul_id, src2_row_size, vtcm_budget, kparams); + +finalize: + kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11); + kparams->div_ne1 = init_fastdiv_values(ne11); + kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1); + kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1); + kparams->div_ne12 = init_fastdiv_values(ne12); + + if (kparams->kernel_type == HTP_MM_KERNEL_UNSUPPORTED || (size_t) kparams->vtcm_size > vtcm_budget) { + HEX_VERBOSE("ggml-hex: %s supported %s VTCM size needed (%d) > budget (%zu)\n", + htp_session_name(sess), is_matmul_id ? "MUL_MAT_ID" : "MUL_MAT", kparams->vtcm_size, vtcm_budget); + return false; + } + return true; +} + +static bool htp_matmul_check_types( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst +) { + const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID); + + if (dst->type != GGML_TYPE_F32) { + return false; + } + + if (is_matmul_id) { + const struct ggml_tensor * src2 = dst->src[2]; + if (src1->type != GGML_TYPE_F32 || !src2 || src2->type != GGML_TYPE_I32) { + return false; + } + if (ggml_get_op_params_i32(dst, 3) == GGML_PREC_F32) { + return false; + } + switch (src0->type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q2_K: + if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) { + return false; + } + if (src0->ne[0] % ggml_blck_size(src0->type) != 0) { + return false; + } + return true; + + case GGML_TYPE_F16: + case GGML_TYPE_F32: + case GGML_TYPE_BF16: + if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0)) { + return false; + } + return src0->ne[0] % 32 == 0; + + default: + return false; + } + } + + if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { + return false; + } + if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3] || + src1->ne[2] % src0->ne[2] != 0 || src1->ne[3] % src0->ne[3] != 0) { + return false; + } + + switch (src0->type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q2_K: + if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) { + return false; + } + return src0->ne[0] % ggml_blck_size(src0->type) == 0; + + case GGML_TYPE_F32: + if (src1->type != GGML_TYPE_F32) { + return false; + } + return src0->nb[1] >= src0->nb[0]; + + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + return src0->nb[1] >= src0->nb[0]; + + default: + return false; + } +} + +static bool htp_kparams_solve_matmul( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_mm_kernel_params * kparams +) { + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (!htp_matmul_check_types(src0, src1, dst)) { + return false; + } + if (htp_matmul_can_collapse(src0, src1, dst)) { + const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1); + const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst); + return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, 0, 0, kparams); + } + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, kparams); +} + +bool htp_matmul_add_can_collapse( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * src2, + const struct ggml_tensor * dst +) { + if (!htp_matmul_can_collapse(src0, src1, dst)) { + return false; + } + if (!src2) { + return true; + } + const int64_t src2_nrows = src2->ne[1] * src2->ne[2] * src2->ne[3]; + if (src2_nrows == 1) { + return true; + } + size_t s2; + return src2->nb[0] == ggml_type_size(src2->type) && + ggml_are_same_shape(src2, dst) && + htp_rows_stride(src2->ne, src2->nb, &s2); +} + +bool htp_kparams_solve_matmul_add( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * src2, + const struct ggml_tensor * dst, + struct htp_mm_kernel_params * kparams +) { + if (htp_matmul_add_can_collapse(src0, src1, src2, dst)) { + const ggml_tensor src1_collapsed = htp_tensor_collapse_rows(src1); + const ggml_tensor dst_collapsed = htp_tensor_collapse_rows(dst); + const ggml_tensor src2_collapsed = (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) + ? htp_tensor_collapse_rows(src2) + : (src2 ? *src2 : ggml_tensor{}); + const struct ggml_tensor * p_src2 = src2 ? &src2_collapsed : nullptr; + const size_t src2_size = p_src2 ? hex_round_up(ggml_nbytes(p_src2), 128) : 0; + return htp_kparams_solve_matmul_impl(sess, src0, &src1_collapsed, &dst_collapsed, p_src2 ? p_src2->nb[1] : 0, src2_size, kparams); + } + const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0; + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, kparams); +} + +bool htp_kparams_solve_mmnx( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + int32_t n_weights, + struct htp_mm_kernel_params * kparams +) { + const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_hmx = htp_session_n_hmx(sess); + const size_t vtcm_budget = htp_session_vtcm_size(sess); + + memset(kparams, 0, sizeof(*kparams)); + kparams->n_threads = n_threads; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const int ne10 = act->ne[0]; + const int ne11 = act->ne[1]; + const int ne12 = act->ne[2]; + const int ne13 = act->ne[3]; + + const bool is_tiled = htp_tensor_is_tiled(src0); + const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; + const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00; + const int ne11_padded = hex_round_up(ne11, 32); + const int ne01_tiled = hex_round_up(ne01, 32); + const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1); + + const bool hmx_enabled = (n_hmx > 0) && (opt_mm_select >= 2); + if (hmx_enabled && htp_matmul_is_hmx_eligible(src0, act, nullptr, false, is_batched)) { + if (htp_kparams_solve_hmx_mm(sess, src0, act, nullptr, wtype, ne00_padded, ne01_tiled, ne02, ne11, ne12, ne11_padded, false, is_batched, 0, vtcm_budget, kparams)) { + kparams->n_weights = n_weights; + goto finalize; + } + } + + if (!is_tiled || act->type != GGML_TYPE_F32 || ne10 % 32 != 0) { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return false; + } + + { + const int act_nrows = ne11 * ne12 * ne13; + const size_t act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + const size_t src0_row_size = src0->nb[1]; + + const uint32_t max_prefetch = (act_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + uint32_t best_n_prefetch = 2; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads, + 0, src0_row_size, act_row_size, 0, d, false, true + ); + if (L.total_bytes <= vtcm_budget) { + best_n_prefetch = d; + break; + } + } + + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, HTP_MM_KERNEL_HVX_QUANT_ROW, wtype, ne10, act_nrows, n_threads, + 0, src0_row_size, act_row_size, 0, best_n_prefetch, false, true + ); + + if (opt_mm_select >= 1 && L.total_bytes <= vtcm_budget) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; + kparams->tile_size = htp_tensor_get_tile_size(wtype); + kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + kparams->act_row_size = act_row_size; + kparams->vtcm_bias_size = 0; + kparams->vtcm_size = L.total_bytes; + kparams->n_prefetch = best_n_prefetch; + kparams->n_weights = n_weights; + } else { + kparams->kernel_type = HTP_MM_KERNEL_UNSUPPORTED; + return false; + } + } + +finalize: + kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11); + kparams->div_ne1 = init_fastdiv_values(ne11); + kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1); + kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1); + kparams->div_ne12 = init_fastdiv_values(ne12); + return (size_t) kparams->vtcm_size <= vtcm_budget; +} + +bool htp_kparams_solve_mmidnx( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + const struct ggml_tensor * dst, + int32_t n_weights, + struct htp_mm_kernel_params * kparams +) { + const bool ok = htp_kparams_solve_matmul_impl(sess, src0, act, dst, 0, 0, kparams); + kparams->n_weights = n_weights; + return ok; +} + +static bool htp_kparams_solve_binary( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op_tensor, + struct htp_binary_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const uint32_t op = htp_op_remap(op_tensor); + const struct ggml_tensor * src0 = op_tensor->src[0]; + const struct ggml_tensor * src1 = op_tensor->src[1]; + const struct ggml_tensor * src2 = (op == HTP_OP_ADD_ID) ? op_tensor->src[2] : nullptr; + const struct ggml_tensor * dst = op_tensor; + + const bool is_add_id = (op == HTP_OP_ADD_ID); + if (is_add_id) { + if (!src2) { + return false; + } + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src2->type != GGML_TYPE_I32) { + return false; + } + if (!ggml_are_same_shape(src0, dst)) { + return false; + } + if (src1->ne[0] != src0->ne[0] || src1->ne[2] != 1 || src1->ne[3] != 1) { + return false; + } + if (src2->ne[0] != src0->ne[1] || src2->ne[1] != src0->ne[2]) { + return false; + } + if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || src2->nb[0] != sizeof(int32_t)) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { + return false; + } + if (src2->nb[1] < src2->ne[0] * sizeof(int32_t)) { + return false; + } + } else { + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || + src1->type != src0->type || dst->type != src0->type) { + return false; + } + if (ggml_is_permuted(src0) || ggml_is_permuted(dst) || !ggml_are_same_shape(src0, dst)) { + return false; + } + if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) { + return false; + } + } + + const uint32_t max_threads_sess = htp_session_n_threads(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const size_t elem_size = ggml_type_size(src0->type); + const size_t src0_row_size = src0->ne[0] * elem_size; + const size_t src1_row_size = src1->ne[0] * elem_size; + const size_t dst_row_size = dst->ne[0] * elem_size; + + const size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128); + const size_t src1_row_size_aligned = hex_round_up(src1_row_size, 128); + const size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128); + + if (!htp_binary_strides_safe(src0, src1, dst, elem_size)) { + return false; + } + + const bool is_scalar = !is_add_id && src1->ne[0] == 1; + const bool is_row_bcast = !is_add_id && !is_scalar && + src1->ne[0] == src0->ne[0] && + (src0->ne[1] > 1 || src0->ne[2] > 1 || src0->ne[3] > 1) && + src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1; + const bool is_same_shape = !is_add_id && !is_scalar && + src1->ne[0] == src0->ne[0] && + src1->ne[1] == src0->ne[1] && + (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) && + (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1); + const bool is_complex = !is_add_id && !is_scalar && !is_same_shape && !is_row_bcast && (src1->ne[0] == src0->ne[0]); + const bool is_contig = ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst); + const bool is_scalar_broadcast = !is_add_id && (ggml_nelements(src1) == 1); + + if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) { + const uint32_t total_elems = (uint32_t) ggml_nelements(src0); + const uint32_t max_chunk_elems = 32768 / elem_size; + const uint32_t min_chunk_elems = 256; + const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads_sess) - 1) / (2 * max_threads_sess), 32); + const uint32_t chunk_size = (std::min)(max_chunk_elems, (std::max)(target_chunk_elems, min_chunk_elems)); + const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); + const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size; + const uint32_t n_threads = (total_chunks >= 2) ? (std::min)(max_threads_sess, total_chunks) : 1; + + kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; + kparams->n_threads = n_threads; + kparams->n_srcs = 2; + kparams->rows_per_buffer = 1; + kparams->src0_row_size_aligned = src0_row_size_aligned; + kparams->src1_row_size_aligned = is_scalar_broadcast ? 0 : src1_row_size_aligned; + kparams->dst_row_size_aligned = dst_row_size_aligned; + kparams->src1_size = 0; + kparams->chunk_size = chunk_size; + kparams->chunk_bytes = chunk_bytes; + kparams->is_scalar = is_scalar_broadcast ? 1 : 0; + + struct htp_binary_vtcm_layout L; + htp_binary_vtcm_layout_build(&L, kparams, vtcm_size); + if (L.total_bytes == 0 || L.total_bytes > vtcm_size) { + return false; + } + + kparams->vtcm_size = L.total_bytes; + return true; + } + + enum htp_binary_kernel_type kernel_type; + size_t src1_size = 0; + size_t src2_size = 0; + + if (is_add_id) { + kernel_type = HTP_BINARY_KERNEL_ADD_ID; + src1_size = hex_round_up(src1->ne[1] * src1_row_size_aligned, 128); + src2_size = hex_round_up(ggml_nelements(src2) * sizeof(int32_t), 128); + if (src1_size + src2_size > vtcm_size) { + return false; + } + } else if (is_row_bcast) { + kernel_type = HTP_BINARY_KERNEL_ROW_BCAST; + src1_size = src1_row_size_aligned; + } else if (is_scalar) { + src1_size = hex_round_up(ggml_nelements(src1) * elem_size, 128); + if (src1_size > vtcm_size) { + return false; + } + kernel_type = HTP_BINARY_KERNEL_SCALAR; + kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0; + } else if (is_same_shape) { + kernel_type = HTP_BINARY_KERNEL_SAME_SHAPE; + } else if (is_complex) { + src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128); + if (src1_size > vtcm_size) { + return false; + } + kernel_type = HTP_BINARY_KERNEL_COMPLEX; + kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0; + } else { + src1_size = hex_round_up(ggml_nelements(src1) * elem_size + 128, 128); + if (src1_size > vtcm_size) { + return false; + } + kernel_type = HTP_BINARY_KERNEL_REPEAT; + kparams->src1_strided = !ggml_is_contiguous(src1) ? 1 : 0; + } + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); + + kparams->kernel_type = kernel_type; + kparams->n_threads = (std::min)(max_threads_sess, max_threads); + kparams->n_srcs = 2; + kparams->src0_row_size_aligned = src0_row_size_aligned; + kparams->src1_row_size_aligned = src1_row_size_aligned; + kparams->dst_row_size_aligned = dst_row_size_aligned; + kparams->src1_size = src1_size; + kparams->src2_size = src2_size; + + const bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]); + const bool src1_contig_dim1 = (src1->nb[2] == src1->ne[1] * src1->nb[1]); + const bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]); + + const bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]); + const bool src1_contig_dim2 = (src1->nb[3] == src1->ne[2] * src1->nb[2]); + const bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]); + + bool split_at_ne01 = false; + if (src0->ne[2] > 1) { + if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) { + split_at_ne01 = (src1->ne[2] != src0->ne[2]) || !src0_contig_dim1 || !src1_contig_dim1 || !dst_contig_dim1; + } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { + split_at_ne01 = !src0_contig_dim1 || !dst_contig_dim1; + } else { + split_at_ne01 = true; + } + } + kparams->split_at_ne01 = split_at_ne01 ? 1 : 0; + + bool split_at_ne02 = false; + if (src0->ne[3] > 1) { + if (kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) { + split_at_ne02 = (src1->ne[3] != src0->ne[3]) || !src0_contig_dim2 || !src1_contig_dim2 || !dst_contig_dim2; + } else if (kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { + split_at_ne02 = !src0_contig_dim2 || !dst_contig_dim2; + } else { + split_at_ne02 = true; + } + } + kparams->split_at_ne02 = split_at_ne02 ? 1 : 0; + + kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); + kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]); + kparams->div_ne012 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[2])); + kparams->div_ne11 = init_fastdiv_values((uint32_t) src1->ne[1]); + kparams->div_ne12 = init_fastdiv_values((uint32_t) src1->ne[2]); + kparams->div_ne13 = init_fastdiv_values((uint32_t) src1->ne[3]); + + struct htp_binary_vtcm_layout L; + htp_binary_vtcm_layout_build(&L, kparams, vtcm_size); + if (L.rows_per_buffer == 0 || L.total_bytes > vtcm_size) { + if (!is_add_id && is_contig && (ggml_are_same_shape(src0, src1) || is_scalar_broadcast)) { + const uint32_t total_elems = (uint32_t) ggml_nelements(src0); + return htp_binary_solve_chunked(&L, kparams, total_elems, (uint32_t) elem_size, + max_threads_sess, 2, is_scalar_broadcast, vtcm_size); + } + return false; + } + + kparams->rows_per_buffer = L.rows_per_buffer; + kparams->vtcm_size = L.total_bytes; + + return true; +} + +// collapse contiguous dims of elementwise ops so that rows fill at least one HVX vector +static void htp_collapse_dims(struct ggml_tensor ** tensors, size_t n_tensors, size_t elem_size, uint32_t n_threads) { + if (n_tensors == 0 || !tensors[0]) { + return; + } + for (size_t t = 0; t < n_tensors; t++) { + if (!tensors[t] || tensors[t]->nb[0] != elem_size) { + return; + } + if (t > 0 && !ggml_are_same_shape(tensors[0], tensors[t])) { + return; + } + } + + struct ggml_tensor * t0 = tensors[0]; + + while (t0->ne[1] > 1 || t0->ne[2] > 1 || t0->ne[3] > 1) { + const size_t row_size = (size_t) t0->ne[0] * elem_size; + bool rows_contig = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->nb[1] != (size_t) tensors[t]->ne[0] * elem_size) { + rows_contig = false; + break; + } + } + bool all_ne1_one = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->ne[1] != 1) { + all_ne1_one = false; + break; + } + } + const bool can_merge = all_ne1_one || + (rows_contig && (t0->ne[0] == 1 || (row_size < 128 && row_size * t0->ne[1] <= 2048))); + if (!can_merge) { + break; + } + for (size_t t = 0; t < n_tensors; t++) { + tensors[t]->ne[0] *= tensors[t]->ne[1]; + tensors[t]->ne[1] = tensors[t]->ne[2]; + tensors[t]->ne[2] = tensors[t]->ne[3]; + tensors[t]->ne[3] = 1; + tensors[t]->nb[1] = tensors[t]->nb[2]; + tensors[t]->nb[2] = tensors[t]->nb[3]; + tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2]; + } + } + + // ne1 too big to merge in full: merge it in chunks of k rows (k divides ne1) + const size_t row_size = (size_t) t0->ne[0] * elem_size; + bool rows_contig = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->nb[1] != row_size) { + rows_contig = false; + break; + } + } + if (t0->ne[1] > 1 && row_size < 128 && rows_contig) { + int64_t k = 2048 / row_size; + while (k > 1 && t0->ne[1] % k != 0) { + k--; + } + if (k > 1) { + for (size_t t = 0; t < n_tensors; t++) { + tensors[t]->ne[0] *= k; + tensors[t]->ne[1] /= k; + tensors[t]->nb[1] *= k; + tensors[t]->nb[2] = tensors[t]->nb[1] * tensors[t]->ne[1]; + tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2]; + } + } + } + + // nrows too small to saturate threads: split ne0 into k rows (k divides ne0) + const size_t row_size_curr = (size_t) t0->ne[0] * elem_size; + const uint32_t nrows = (uint32_t) (t0->ne[1] * t0->ne[2] * t0->ne[3]); + rows_contig = true; + for (size_t t = 0; t < n_tensors; t++) { + if (tensors[t]->ne[1] != 1 && tensors[t]->nb[1] != row_size_curr) { + rows_contig = false; + break; + } + } + if (nrows < n_threads && rows_contig && row_size_curr >= 4096) { + const size_t vec_elems = 128 / elem_size; + int64_t k = std::max(row_size_curr / 16384, (n_threads * 4 + nrows - 1) / nrows); + k = std::min(k, row_size_curr / 2048); + while (k > 1 && t0->ne[0] % (k * vec_elems) != 0) { + k--; + } + if (k > 1) { + for (size_t t = 0; t < n_tensors; t++) { + tensors[t]->ne[0] /= k; + tensors[t]->ne[1] *= k; + tensors[t]->nb[1] = tensors[t]->ne[0] * elem_size; + tensors[t]->nb[2] = tensors[t]->nb[1] * tensors[t]->ne[1]; + tensors[t]->nb[3] = tensors[t]->nb[2] * tensors[t]->ne[2]; + } + } + } +} + +static bool htp_kparams_solve_unary_impl( + const struct ggml_hexagon_session * sess, + uint32_t op, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (!htp_op_is_unary((htp_op_code) op)) { + return false; + } + + if (op == HTP_OP_HADAMARD) { + const int64_t n = src0->ne[0]; + if (n < 32 || (n & (n - 1)) != 0) { + return false; + } + if (!dst->src[0] || dst->src[0]->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || + (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16)) { + return false; + } + } else { + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || dst->type != src0->type) { + return false; + } + if (op == HTP_OP_TRI && (src0->type != GGML_TYPE_F32 || !ggml_is_contiguous(src0))) { + return false; + } + if (src0->type == GGML_TYPE_F16) { + switch (op) { + case HTP_OP_NORM: + case HTP_OP_RMS_NORM: + case HTP_OP_L2_NORM: + case HTP_OP_SCALE: + case HTP_OP_CLAMP: + case HTP_OP_SQR: + case HTP_OP_SQRT: + case HTP_OP_UNARY_LOG: + case HTP_OP_UNARY_ABS: + case HTP_OP_UNARY_STEP: + case HTP_OP_UNARY_CEIL: + case HTP_OP_UNARY_SGN: + break; + default: + return false; + } + } + } + + if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous(dst) || !ggml_are_same_shape(src0, dst)) { + return false; + } + + const size_t elem_size = ggml_type_size(src0->type); + + const size_t dst_data_row_size = dst->ne[0] * ggml_type_size(dst->type); + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); + const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + kparams->n_threads = n_threads; + + const bool broadcast_weight = (op == HTP_OP_RMS_NORM_MUL) && (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); + kparams->broadcast_weight = broadcast_weight; + + const bool src0_contig = ggml_is_contiguous_1(src0); + const bool src1_contig = (op == HTP_OP_RMS_NORM_MUL && !broadcast_weight) ? ggml_is_contiguous_1(src1) : true; + const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + const uint32_t min_block_rows = (uint32_t) (std::max(1, 2048 / dst_data_row_size)); + uint32_t max_block = (src0_contig && src1_contig) ? src0_nrows : (uint32_t) src0->ne[1]; + if (src0_nrows_per_thread >= 2 * min_block_rows) { + max_block = (std::min)(max_block, (std::max)(min_block_rows, src0_nrows_per_thread / 4)); + } + + kparams->src0_contig = src0_contig ? 1 : 0; + kparams->src1_contig = src1_contig ? 1 : 0; + + struct htp_unary_vtcm_layout L; + uint32_t col_tile = 0; + uint32_t vtcm_row_per_thread = 0; + + htp_unary_solve_layout(&L, op, src0->ne[0], dst->ne[0], + op == HTP_OP_RMS_NORM_MUL ? src1->ne[0] : 0, + max_block, broadcast_weight, n_threads, vtcm_size, elem_size, + &col_tile, &vtcm_row_per_thread); + + kparams->col_tile = col_tile; + kparams->vtcm_size = L.total_bytes; + kparams->block = col_tile ? 0 : vtcm_row_per_thread; + + const uint32_t tiles_per_row = col_tile > 0 ? (src0->ne[0] + col_tile - 1) / col_tile : 1; + kparams->tiles_per_row = tiles_per_row; + kparams->div_ne01 = init_fastdiv_values(src0->ne[1]); + kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); + kparams->div_ne012 = init_fastdiv_values(src0->ne[1] * src0->ne[2]); + kparams->div_tpr = init_fastdiv_values(tiles_per_row); + + return (col_tile > 0 || vtcm_row_per_thread > 0) && L.total_bytes <= vtcm_size; +} + +static bool htp_kparams_solve_unary( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_unary_kernel_params * kparams +) { + const htp_op_code opcode = htp_op_remap(op); + const struct ggml_tensor * src0 = (opcode == HTP_OP_HADAMARD) ? op->src[1] : op->src[0]; + const struct ggml_tensor * src1 = (opcode == HTP_OP_HADAMARD) ? nullptr : op->src[1]; + return htp_kparams_solve_unary_impl(sess, opcode, src0, src1, op, kparams); +} + +bool htp_kparams_solve_rms_norm_mul( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +) { + return htp_kparams_solve_unary_impl(sess, HTP_OP_RMS_NORM_MUL, src0, src1, dst, kparams); +} + +bool htp_kparams_solve_repeat_get_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_get_rows_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16 && src0->type != GGML_TYPE_I32 && + src0->type != GGML_TYPE_Q1_0 && src0->type != GGML_TYPE_Q2_0 && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && + src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K) { + return false; + } + + if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(src1) || !ggml_is_contiguous_rows(dst)) { + return false; + } + + if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) { + return false; + } + + if (src0->type == dst->type) { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) { + return false; + } + } else if (src0->type == GGML_TYPE_I32) { + return false; + } else if (dst->type != GGML_TYPE_F32) { + return false; + } + + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne02 = src0->ne[2]; + const uint32_t ne03 = src0->ne[3]; + + const uint32_t ne10 = src1->ne[0]; + const uint32_t ne11 = src1->ne[1]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t nr = ne10 * ne11 * ne12; + + const bool tiled = htp_tensor_is_tiled(src0); + const int wtype = (tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; + + if (tiled) { + kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED; + } else if (src0->type == dst->type) { + kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE; + } else { + kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT; + } + + const uint32_t total_tasks = nr; + const uint32_t min_tasks_per_thread = (uint32_t) (std::max(1, (128 + dst->nb[1] - 1) / dst->nb[1])); + const uint32_t max_threads = (std::max)(1u, total_tasks / min_tasks_per_thread); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + kparams->n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + + struct htp_get_rows_vtcm_layout vtcm_layout = {}; + while (kparams->n_threads > 0) { + htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, kparams->n_threads); + if (vtcm_layout.total_bytes <= vtcm_size) { + break; + } + --kparams->n_threads; + } + + if (kparams->n_threads == 0 && total_tasks > 0) { + htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, wtype, ne00, total_tasks, src1->type, 1); + } + + kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes; + kparams->total_tasks = total_tasks; + + kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0}; + kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0}; + kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0}; + kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0}; + + if (total_tasks == 0) { + return true; + } + return kparams->n_threads > 0 && (size_t) kparams->vtcm_size <= vtcm_size; +} + +static bool htp_kparams_solve_get_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_get_rows_kernel_params * kparams +) { + return htp_kparams_solve_repeat_get_rows(sess, op->src[0], op->src[1], op, kparams); +} + +static bool htp_kparams_solve_set_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_set_rows_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op->src[2] ? op->src[2] : op; + + if (dst->type == GGML_TYPE_Q8_0 && src0->ne[0] < 32) { + return false; + } + if (src0->type != GGML_TYPE_F32) { + return false; + } + if (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64) { + return false; + } + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_Q8_0) { + return false; + } + + const uint32_t nr = src0->ne[1]; + + kparams->n_threads = (std::min)(htp_session_n_threads(sess), nr); + kparams->total_tasks = nr; + + struct htp_set_rows_vtcm_layout vtcm_layout; + htp_set_rows_vtcm_layout_build(&vtcm_layout, dst->type, src0->ne[0], kparams->n_threads); + kparams->vtcm_size = vtcm_layout.total_bytes; + + return vtcm_layout.total_bytes <= htp_session_vtcm_size(sess); +} + +static bool htp_kparams_solve_softmax( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_softmax_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * src2 = op->src[2]; + const struct ggml_tensor * dst = op; + + if (src2) { + return false; + } + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src0->ne[2] > 512) { + return false; + } + if (src1) { + if ((src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) || + src0->ne[0] != src1->ne[0] || src1->ne[1] < src0->ne[1] || + src0->ne[2] % src1->ne[2] != 0 || src0->ne[3] % src1->ne[3] != 0 || + !ggml_is_contiguous(src1)) { + return false; + } + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + return false; + } + + const int64_t ne0 = src0->ne[0]; + if ((ne0 > 32 && (ne0 & (32 - 1)) != 0) || ne0 > 131072) { + return false; + } + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_data_row_size = op->ne[0] * ggml_type_size(op->type); + const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_data_row_size - 1) / dst_data_row_size)); + const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + + float scale = 1.0f; + float max_bias = 0.0f; + memcpy(&scale, &op->op_params[0], sizeof(float)); + memcpy(&max_bias, &op->op_params[1], sizeof(float)); + + kparams->scale = scale; + kparams->max_bias = max_bias; + + const uint32_t n_head = src0->ne[2]; + const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); + kparams->n_head = n_head; + kparams->n_head_log2 = n_head_log2; + + if (max_bias > 0.0f && n_head_log2 > 0) { + kparams->m0 = powf(2.0f, -(max_bias) / n_head_log2); + kparams->m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + } else { + kparams->m0 = 1.0f; + kparams->m1 = 1.0f; + } + + kparams->use_src1 = (src1 != nullptr) ? 1 : 0; + kparams->use_f16 = (src1 != nullptr && src1->type == GGML_TYPE_F16) ? 1 : 0; + + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne10 = src1 ? src1->ne[0] : 1; + + struct htp_softmax_vtcm_layout layout; + htp_softmax_vtcm_layout_build(&layout, ne00, ne10, kparams->use_src1 != 0, kparams->use_f16 != 0, n_threads); + if (layout.total_bytes > htp_session_vtcm_size(sess)) { + return false; + } + + kparams->n_threads = n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + + if (!kparams->use_src1) { + kparams->kernel_type = HTP_SOFTMAX_KERNEL_NOMASK; + } else if (kparams->use_f16) { + kparams->kernel_type = HTP_SOFTMAX_KERNEL_MASK_F16; + } else { + kparams->kernel_type = HTP_SOFTMAX_KERNEL_MASK_F32; + } + + if (src0->ne[1] > 0) kparams->div_ne01 = init_fastdiv_values(src0->ne[1]); + if (src0->ne[2] > 0) kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); + const uint32_t ne12 = src1 ? src1->ne[2] : 1; + const uint32_t ne13 = src1 ? src1->ne[3] : 1; + if (ne12 > 0) kparams->div_ne12 = init_fastdiv_values(ne12); + if (ne13 > 0) kparams->div_ne13 = init_fastdiv_values(ne13); + + return true; +} + +static bool htp_kparams_solve_rope( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_rope_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * src2 = op->src[2]; + const struct ggml_tensor * dst = op; + + if (!ggml_are_same_shape(src0, dst)) { + return false; + } + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) { + return false; + } + if (src0->ne[0] <= 0) { + return false; + } + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + if (src0_nrows == 0) { + return false; + } + + const int32_t * op_params = &op->op_params[0]; + const int n_dims = op_params[1]; + const int mode = op_params[2]; + const int n_offs = op_params[15]; + + if (n_dims < 0 || n_dims % 2 != 0) { + return false; + } + if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) { + return false; + } + + float freq_base; + memcpy(&freq_base, op_params + 5, sizeof(float)); + if (freq_base < 0.0f) { + return false; + } + + if (mode != GGML_ROPE_TYPE_NORMAL && + mode != GGML_ROPE_TYPE_NEOX && + mode != GGML_ROPE_TYPE_MROPE && + mode != GGML_ROPE_TYPE_VISION && + mode != GGML_ROPE_TYPE_IMROPE) { + return false; + } + + const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0; + + if (mode == GGML_ROPE_TYPE_VISION && (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0)) { + return false; + } + if (is_mrope) { + const int32_t * sections = op_params + 11; + if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) { + return false; + } + } + + const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2]; + if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) { + return false; + } + + if (src2) { + if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2) || src2->ne[0] < (n_dims / 2)) { + return false; + } + } + + if (src0->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float) || + src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) { + return false; + } + + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), src0_nrows); + const uint32_t n_freq_factors = src2 ? (uint32_t) src2->ne[0] : 0; + + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads, n_freq_factors); + if (layout.total_bytes > htp_session_vtcm_size(sess)) { + return false; + } + + kparams->n_threads = n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); + kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); + + return true; +} + +static bool htp_kparams_solve_sum_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_sum_rows_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32) { + return false; + } + if (op->op == GGML_OP_ARGMAX) { + if (dst->type != GGML_TYPE_I32) { + return false; + } + } else if (dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + return false; + } + + const uint32_t ne00 = src0->ne[0]; + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t max_threads_sess = htp_session_n_threads(sess); + + if (op->op == GGML_OP_SUM) { + const uint32_t total_elems = (uint32_t) ggml_nelements(src0); + const uint32_t n_threads = (total_elems >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1; + const uint32_t raw_chunk = (total_elems + n_threads - 1) / n_threads; + + kparams->n_threads = n_threads; + kparams->elems_per_thread = hex_round_up(raw_chunk, 32); + } else if (op->op == GGML_OP_ARGMAX && src0_nrows <= 1) { + const uint32_t n_threads = (ne00 >= 1024) ? (std::min)(max_threads_sess, (uint32_t) HTP_MAX_NTHREADS) : 1; + const uint32_t raw_chunk = (ne00 + n_threads - 1) / n_threads; + + kparams->n_threads = n_threads; + kparams->elems_per_thread = hex_round_up(raw_chunk, 32); + } else { + const uint32_t min_rpt = (src0->nb[1] < 1024) ? 32 : 1; + const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rpt); + + kparams->n_threads = (std::min)(max_threads_sess, max_threads); + } + + return true; +} + +bool htp_kparams_solve_unary_mul_glu( + const struct ggml_hexagon_session * sess, + uint32_t op, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_glu_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (!htp_op_is_glu((htp_op_code) op)) { + return false; + } + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || dst->type != src0->type) { + return false; + } + if ((op == HTP_OP_GLU_GEGLU_QUICK || op == HTP_OP_GLU_GEGLU_ERF) && src0->type != GGML_TYPE_F32) { + return false; + } + + const size_t elem_size = ggml_type_size(src0->type); + const auto is_valid_src = [elem_size](const struct ggml_tensor * t) { + return ggml_is_contiguous_1(t) && t->nb[0] == elem_size && + (t->ne[1] <= 1 || t->nb[1] >= (size_t) t->ne[0] * elem_size); + }; + + if (!is_valid_src(src0) || !ggml_is_contiguous(dst)) { + return false; + } + if (src1 && (src1->type != src0->type || !ggml_are_same_shape(src0, src1) || !is_valid_src(src1))) { + return false; + } + + const size_t dst_row_size = dst->ne[0] * elem_size; + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_threads = (std::max)(1u, src0_nrows / min_rows_per_thread); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + const size_t row_size_aligned = hex_round_up(dst_row_size, 128); + uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * 3 * row_size_aligned)); + if (vtcm_row_per_thread < 2) { + return false; + } + + const uint32_t src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + if (src0_nrows_per_thread >= 2 && vtcm_row_per_thread > src0_nrows_per_thread) { + uint32_t target_rpt = (src0_nrows_per_thread >= 4) ? (src0_nrows_per_thread / 2) & ~1u : 2; + vtcm_row_per_thread = (std::max)(2u, (std::min)(vtcm_row_per_thread, target_rpt)); + } + + struct htp_glu_vtcm_layout L; + htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, vtcm_row_per_thread); + + kparams->n_threads = n_threads; + kparams->vtcm_row_per_thread = L.vtcm_row_per_thread; + kparams->vtcm_size = (uint32_t) L.total_bytes; + + return true; +} + +static bool htp_kparams_solve_glu( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_glu_kernel_params * kparams +) { + return htp_kparams_solve_unary_mul_glu(sess, htp_op_remap(op), op->src[0], op->src[1], op, kparams); +} + +static bool htp_kparams_solve_repeat( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_repeat_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) || src0->type != dst->type) { + return false; + } + if (!ggml_can_repeat(src0, dst) || ggml_is_transposed(src0) || ggml_is_transposed(dst)) { + return false; + } + + const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; + const size_t dst_row_size = dst->ne[0] * ggml_type_size(dst->type); + const uint32_t min_rows_per_thread = (uint32_t) (std::max(1, (128 + dst_row_size - 1) / dst_row_size)); + const uint32_t max_threads = (std::max)(1u, total_dst_rows / min_rows_per_thread); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + + kparams->n_threads = n_threads; + kparams->nr0 = dst->ne[0] / src0->ne[0]; + kparams->nr1 = dst->ne[1] / src0->ne[1]; + kparams->nr2 = dst->ne[2] / src0->ne[2]; + kparams->nr3 = dst->ne[3] / src0->ne[3]; + + kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); + kparams->div_ne12 = init_fastdiv_values(dst->ne[1] * dst->ne[2]); + kparams->div_ne01 = init_fastdiv_values(src0->ne[1]); + kparams->div_ne02 = init_fastdiv_values(src0->ne[2]); + kparams->div_ne03 = init_fastdiv_values(src0->ne[3]); + + return true; +} + +static bool htp_kparams_solve_ssm_conv( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_ssm_conv_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) { + return false; + } + + const int64_t d_conv_64 = src1->ne[0]; + const int64_t d_inner_64 = src0->ne[1]; + const int64_t n_t_64 = dst->ne[1]; + const int64_t n_s_64 = dst->ne[2]; + + if (d_conv_64 <= 0 || d_conv_64 > 32 || d_inner_64 <= 0) { + return false; + } + if (src0->ne[0] != d_conv_64 - 1 + n_t_64 || src0->ne[1] != d_inner_64 || src0->ne[2] != n_s_64) { + return false; + } + if (src1->ne[0] != d_conv_64 || src1->ne[1] != d_inner_64) { + return false; + } + if (dst->ne[0] != d_inner_64 || dst->ne[1] != n_t_64 || dst->ne[2] != n_s_64) { + return false; + } + if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (src0->nb[1] != src0->ne[0] * sizeof(float) || src1->nb[1] != src1->ne[0] * sizeof(float)) { + return false; + } + + const uint32_t d_conv = (uint32_t) d_conv_64; + const uint32_t d_inner = (uint32_t) d_inner_64; + const uint32_t n_t = (uint32_t) n_t_64; + const uint32_t n_s = (uint32_t) n_s_64; + const uint32_t ncs = (uint32_t) src0->ne[0]; + + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), (d_inner + 31) / 32); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + kparams->n_threads = n_threads; + kparams->d_conv = d_conv; + kparams->d_inner = d_inner; + kparams->n_t = n_t; + kparams->n_s = n_s; + + const uint32_t raw_rpt = (d_inner + n_threads - 1) / n_threads; + const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, 32); + + if (n_t == 1) { + kparams->d_inner_tile = d_inner_per_thread; + } else { + const uint32_t src1_raw_bytes = hex_round_up(d_inner_per_thread * d_conv * sizeof(float), 128) + 128; + const uint32_t src1_T_bytes = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 128); + const uint32_t vtcm_src1_per_thread = src1_raw_bytes + src1_T_bytes; + + const size_t vtcm_budget = (vtcm_size > 0 ? vtcm_size / n_threads : (1024 * 1024)); + const uint32_t src0_block_T = hex_round_up(ncs * 32 * sizeof(float), 128); + const size_t fixed_bytes = vtcm_src1_per_thread + src0_block_T; + const size_t avail_for_tiles = vtcm_budget > fixed_bytes ? vtcm_budget - fixed_bytes : (128 * 1024); + + uint32_t target_max_tile = hex_round_up((d_inner_per_thread + 3) / 4, 32); + target_max_tile = (std::max)(target_max_tile, 32u); + target_max_tile = (std::min)(target_max_tile, 128u); + + uint32_t d_inner_tile = (uint32_t) (avail_for_tiles / (2 * (ncs + n_t) * sizeof(float))); + d_inner_tile = (d_inner_tile / 32) * 32; + if (d_inner_tile == 0) { + d_inner_tile = 32; + } + if (d_inner_tile > target_max_tile) { + d_inner_tile = target_max_tile; + } + if (d_inner_tile > d_inner_per_thread) { + d_inner_tile = d_inner_per_thread; + } + kparams->d_inner_tile = d_inner_tile; + } + + struct htp_ssm_conv_vtcm_layout L; + htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, d_conv, n_t, kparams->d_inner_tile, n_threads); + + kparams->vtcm_size = (uint32_t) L.total_bytes; + kparams->div_d_inner_tile = init_fastdiv_values(kparams->d_inner_tile); + + return L.total_bytes <= vtcm_size; +} + +static bool htp_kparams_solve_sort( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_sort_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + const bool is_top_k = (op->op == GGML_OP_TOP_K); + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_I32) { + return false; + } + + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t ne00 = src0->ne[0]; + const uint32_t k = dst->ne[0]; + const uint32_t n_threads_max = htp_session_n_threads(sess) > 0 ? htp_session_n_threads(sess) : 4; + const size_t vtcm_budget = htp_session_vtcm_size(sess) > 0 ? htp_session_vtcm_size(sess) : (8 * 1024 * 1024); + + int32_t order = GGML_SORT_ORDER_DESC; + if (!is_top_k) { + order = ((const int32_t *) op->op_params)[0]; + } + + struct htp_sort_vtcm_layout layout; + if (!htp_sort_solve_layout(&layout, ne00, total_rows, k, n_threads_max, vtcm_budget, is_top_k)) { + return false; + } + + kparams->n_threads = layout.n_threads; + kparams->order = (uint32_t) order; + kparams->is_top_k = is_top_k ? 1 : 0; + kparams->chunk_elems = layout.chunk_elems; + kparams->n_chunks = layout.n_chunks; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->merge_elems = layout.merge_elems; + kparams->n_slots = layout.n_slots; + kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); + kparams->div_ne01_ne02 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[2])); + + return true; +} + +static void htp_pool_interior_range( + uint32_t src_x, uint32_t dst_x, uint32_t kernel_x, uint32_t stride_x, int32_t pad_x, + uint32_t * ox_lo, uint32_t * ox_hi +) { + const uint32_t lo_raw = ((uint32_t) pad_x + stride_x - 1) / stride_x; + *ox_lo = (lo_raw < dst_x) ? lo_raw : dst_x; + + const int32_t numer_hi = (int32_t) src_x - (int32_t) kernel_x + pad_x; + if (numer_hi < 0) { + *ox_hi = 0; + } else { + const uint32_t hi_raw = (uint32_t) numer_hi / stride_x + 1; + *ox_hi = (hi_raw < dst_x) ? hi_raw : dst_x; + } + if (*ox_hi < *ox_lo) { + *ox_hi = *ox_lo; + } +} + +static bool htp_kparams_solve_pool_2d( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_pool_2d_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + const bool is_pool_1d = (op->op == GGML_OP_POOL_1D); + + const int32_t * params = dst->op_params; + if (params[0] != GGML_OP_POOL_AVG && params[0] != GGML_OP_POOL_MAX) { + return false; + } + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + return false; + } + if (ggml_nbytes(src0) > UINT32_MAX - 256 || ggml_nbytes(dst) > UINT32_MAX - 256) { + return false; + } + + const int32_t kernel_x = params[1]; + const int32_t kernel_y = is_pool_1d ? 1 : params[2]; + const int32_t stride_x = params[is_pool_1d ? 2 : 3]; + const int32_t stride_y = is_pool_1d ? 1 : params[4]; + const int32_t pad_x = params[is_pool_1d ? 3 : 5]; + const int32_t pad_y = is_pool_1d ? 0 : params[6]; + + if (kernel_x <= 0 || kernel_y <= 0 || stride_x <= 0 || stride_y <= 0 || + pad_x < 0 || pad_y < 0) { + return false; + } + + const uint32_t src_x = (uint32_t) src0->ne[0]; + const uint32_t src_y = is_pool_1d ? 1 : (uint32_t) src0->ne[1]; + const uint32_t dst_x = (uint32_t) dst->ne[0]; + const uint32_t dst_y = is_pool_1d ? 1 : (uint32_t) dst->ne[1]; + const uint32_t dst_plane_bytes = dst_x * dst_y * sizeof(float); + const uint32_t planes = (uint32_t) (is_pool_1d + ? (uint64_t) src0->ne[1] * (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3] + : (uint64_t) src0->ne[2] * (uint64_t) src0->ne[3]); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), planes > 0 ? planes : 1); + const uint32_t planes_per_chunk = (dst_plane_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_plane_bytes, HEX_L2_LINE_SIZE)) : 1; + + struct htp_pool_vtcm_layout layout; + if (!htp_pool_solve_layout(&layout, src_x, src_y, dst_x, dst_y, n_threads, htp_session_vtcm_size(sess))) { + return false; + } + + kparams->src_x = src_x; + kparams->src_y = src_y; + kparams->dst_x = dst_x; + kparams->dst_y = dst_y; + kparams->kernel_x = (uint32_t) kernel_x; + kparams->kernel_y = (uint32_t) kernel_y; + kparams->stride_x = (uint32_t) stride_x; + kparams->stride_y = (uint32_t) stride_y; + kparams->pad_x = pad_x; + kparams->pad_y = pad_y; + kparams->n_threads = n_threads; + kparams->planes = planes; + kparams->planes_per_chunk = planes_per_chunk; + kparams->pool_op = (uint32_t) params[0]; + + const bool exact_tiling = (kparams->pad_x == 0 && kparams->pad_y == 0 && + kparams->stride_x == kparams->kernel_x && kparams->stride_y == kparams->kernel_y); + const bool fast_ok = exact_tiling && (kparams->kernel_x == 1 || kparams->kernel_x == 2); + const bool narrow_ok = fast_ok && ((uint64_t) kparams->dst_x * kparams->kernel_x <= 32); + const bool global_ok = exact_tiling && + kparams->kernel_x == kparams->src_x && kparams->kernel_y == kparams->src_y && + kparams->dst_x == 1 && kparams->dst_y == 1; + const bool block_ok = exact_tiling && + kparams->kernel_y == 1 && kparams->stride_y == 1 && + kparams->kernel_x >= 32; + + if (global_ok) { + kparams->kernel_type = HTP_POOL_KERNEL_GLOBAL; + } else if (block_ok) { + kparams->kernel_type = HTP_POOL_KERNEL_BLOCK; + } else if (narrow_ok) { + kparams->kernel_type = HTP_POOL_KERNEL_HVX_NARROW; + } else if (fast_ok) { + kparams->kernel_type = HTP_POOL_KERNEL_HVX; + } else { + kparams->kernel_type = HTP_POOL_KERNEL_GENERAL; + } + + kparams->avg_divide_count = (is_pool_1d && kparams->pool_op == GGML_OP_POOL_AVG) ? 1 : 0; + htp_pool_interior_range(kparams->src_x, kparams->dst_x, kparams->kernel_x, + kparams->stride_x, kparams->pad_x, + &kparams->ox_lo, &kparams->ox_hi); + kparams->inv_kernel_area = 1.0f / (float) (kparams->kernel_x * kparams->kernel_y); + + return true; +} + +static bool htp_kparams_solve_concat( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_concat_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->kernel_type = HTP_CONCAT_KERNEL_UNSUPPORTED; + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (!src0 || !src1 || !dst) { + return false; + } + + int dim = ((const int32_t *) op->op_params)[0]; + if (dim < 0 || dim >= GGML_MAX_DIMS) { + return false; + } + kparams->dim = dim; + + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) { + return false; + } + if (src0->type != dst->type || src1->type != dst->type) { + return false; + } + + const uint32_t type_size = ggml_type_size(dst->type); + const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t mdev_count = htp_session_mdev_count(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + for (int d = 0; d < GGML_MAX_DIMS; d++) { + const int64_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d]; + if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) { + return false; + } + } + + const uint32_t max_gather_ne0 = 128 / type_size; + const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + const bool b_dense = (src1->nb[0] == type_size); + const bool gather_strides_ok = + src0->nb[1] >= (size_t) src0->ne[0] * type_size && + (b_dense ? src1->nb[1] >= (size_t) src1->ne[0] * type_size + : src1->nb[0] >= (size_t) src1->ne[1] * type_size); + + if (dim == 0 && + (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16) && + dst->ne[0] <= max_gather_ne0 && + dst->nb[0] == type_size && dst->nb[1] == (size_t) dst->ne[0] * type_size && + src0->nb[0] == type_size && + (src1->nb[0] == type_size || src1->nb[1] == type_size) && + gather_strides_ok && + htp_gather_rows_tensor_outer_contiguous(dst) && + htp_gather_rows_tensor_outer_contiguous(src0) && + htp_gather_rows_tensor_outer_contiguous(src1)) { + + if (htp_gather_rows_solve_layout(&kparams->u.gather, + (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], + (uint32_t) src1->ne[0], (uint32_t) src1->nb[1], b_dense, + type_size, total_rows, + n_threads, mdev_count, vtcm_size)) { + kparams->kernel_type = HTP_CONCAT_KERNEL_GATHER_ROWS; + kparams->n_threads = (uint8_t) n_threads; + kparams->vtcm_size = kparams->u.gather.slice * n_threads; + return true; + } + } + + const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size); + if (dma_strides_ok) { + kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR; + kparams->n_threads = 1; + return true; + } + + const bool is_src1_transposed = (src1->nb[0] > src1->nb[1]); + const bool is_src0_transposed = (src0->nb[0] > src0->nb[1]); + const bool transposed_rows_ok = (src0->nb[0] == type_size && src1->nb[1] == type_size && dst->nb[0] == type_size); + + if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok && + (dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) { + + const uint32_t nth = n_threads > 0 ? n_threads : 8; + struct htp_concat_transposed_vtcm_layout layout; + htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, nth); + + if (vtcm_size > 0 && layout.total_bytes > vtcm_size) { + return false; + } + + kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED; + kparams->n_threads = nth; + kparams->vtcm_size = layout.total_bytes; + kparams->u.transposed.div_ne2 = init_fastdiv_values((uint32_t) dst->ne[2]); + return true; + } + + return false; +} + +static bool htp_kparams_solve_cpy( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_copy_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->kernel_type = HTP_COPY_KERNEL_UNSUPPORTED; + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (!src0 || !dst) { + return false; + } + + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_I32) { + return false; + } + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) { + return false; + } + + const int64_t nelem_src = ggml_nelements(src0); + const int64_t nelem_dst = ggml_nelements(dst); + if (nelem_src != nelem_dst || nelem_src < 0) { + return false; + } + + const uint32_t src_type_size = ggml_type_size(src0->type); + const uint32_t dst_type_size = ggml_type_size(dst->type); + + kparams->src0_type_size = (uint8_t) src_type_size; + kparams->dst_type_size = (uint8_t) dst_type_size; + kparams->total_elems = (uint32_t) nelem_src; + + if (nelem_src == 0) { + kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG; + return true; + } + + if (nelem_src == 1) { + if (src0->type == dst->type) { + kparams->kernel_type = HTP_COPY_KERNEL_SCALAR; + return true; + } + if ((src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) || + (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) || + (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) || + (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32)) { + kparams->kernel_type = HTP_COPY_KERNEL_SCALAR; + return true; + } + return false; + } + + const bool sametype = (src0->type == dst->type); + const bool same_extents = ggml_are_same_shape(src0, dst); + + const bool transposed = (src0->nb[0] > src0->nb[1]) || (dst->nb[0] > dst->nb[1]) || + (src0->nb[0] != src_type_size) || (dst->nb[0] != dst_type_size) || + (src0->nb[1] < (size_t) src0->ne[0] * src_type_size) || (dst->nb[1] < (size_t) dst->ne[0] * dst_type_size); + const bool sameshape = same_extents && !transposed; + + const bool src_is_contiguous = ggml_is_contiguous(src0); + const bool dst_is_contiguous = ggml_is_contiguous(dst); + + const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t mdev_count = htp_session_mdev_count(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + if (sametype) { + if (src_is_contiguous && dst_is_contiguous) { + kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG; + return true; + } + + const uint32_t max_gather_ne0 = 128 / src_type_size; + const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); + if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) && + dst_is_contiguous && !src_is_contiguous && + src0->nb[0] == src_type_size && src0->ne[0] <= max_gather_ne0 && + src0->nb[1] >= (size_t) src0->ne[0] * src_type_size && + total_rows > 1 && + htp_gather_rows_tensor_outer_contiguous(src0)) { + + if (htp_gather_rows_solve_layout(&kparams->u.gather, + (uint32_t) src0->ne[0], (uint32_t) src0->nb[1], + 0, 0, false, + src_type_size, total_rows, + n_threads, mdev_count, vtcm_size)) { + kparams->kernel_type = HTP_COPY_KERNEL_GATHER_ROWS; + kparams->n_threads = (uint8_t) n_threads; + kparams->vtcm_size = kparams->u.gather.slice * n_threads; + return true; + } + } + + if (sameshape) { + kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE; + kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); + return true; + } + + kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE; + kparams->n_threads = n_threads > 0 ? (uint8_t) n_threads : 4; + kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]); + kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0])); + kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0])); + kparams->u.reshape.div_ne00 = init_fastdiv_values((uint32_t) src0->ne[0]); + kparams->u.reshape.div_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[0])); + kparams->u.reshape.div_ne02_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1] * src0->ne[0])); + return true; + } + + if (!sameshape) { + return false; + } + + const bool valid_conversion = (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) || + (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) || + (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) || + (src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32); + if (!valid_conversion) { + return false; + } + + const uint32_t nth = n_threads > 0 ? n_threads : 4; + struct htp_copy_convert_vtcm_layout layout; + htp_copy_convert_solve_layout(&layout, (uint32_t) src0->ne[0], (uint32_t) src0->ne[1], src_type_size, dst_type_size, nth); + + if (vtcm_size > 0 && layout.total_bytes > vtcm_size) { + return false; + } + + kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT; + kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); + kparams->n_threads = (uint8_t) nth; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->u.convert.blk_rows = layout.blk_rows; + kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); + kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1])); + + return true; +} + +static bool im2col_3d_calc_output_size( + uint32_t input, uint32_t kernel, int32_t stride, int32_t padding, int32_t dilation, uint32_t * output +) { + if (stride <= 0 || padding < 0 || dilation <= 0 || kernel == 0) { + return false; + } + + const int64_t numerator = (int64_t) input + 2LL * padding - (int64_t) dilation * (kernel - 1) - 1; + if (numerator < 0) { + return false; + } + + const uint64_t size = (uint64_t) (numerator / stride) + 1; + if (size == 0 || size > UINT32_MAX) { + return false; + } + + *output = (uint32_t) size; + return true; +} + +static bool htp_kparams_solve_im2col( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_im2col_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED; + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (!src0 || !src1 || !dst) { + return false; + } + if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) { + return false; + } + if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) { + return false; + } + + const int32_t * op_params = (const int32_t *) op->op_params; + const int32_t s0 = op_params[0]; + const int32_t s1 = op_params[1]; + const int32_t p0 = op_params[2]; + const int32_t p1 = op_params[3]; + const int32_t d0 = op_params[4]; + const int32_t d1 = op_params[5]; + const bool is_2D = (op_params[6] == 1); + + if (s0 <= 0 || d0 <= 0 || p0 < 0) { + return false; + } + if (is_2D && (s1 <= 0 || d1 <= 0 || p1 < 0)) { + return false; + } + + const uint32_t N = is_2D ? (uint32_t) src1->ne[3] : (uint32_t) src1->ne[2]; + const uint32_t IC = is_2D ? (uint32_t) src1->ne[2] : (uint32_t) src1->ne[1]; + const uint32_t IH = is_2D ? (uint32_t) src1->ne[1] : 1; + const uint32_t IW = (uint32_t) src1->ne[0]; + const uint32_t KH = is_2D ? (uint32_t) src0->ne[1] : 1; + const uint32_t KW = (uint32_t) src0->ne[0]; + const uint32_t OH = is_2D ? (uint32_t) dst->ne[2] : 1; + const uint32_t OW = (uint32_t) dst->ne[1]; + + if (KW == 0 || KH == 0 || IW == 0 || IH == 0 || IC == 0 || OW == 0 || OH == 0 || N == 0) { + return false; + } + + const uint64_t patch_stride_64 = (uint64_t) IC * KH * KW; + const uint64_t total_rows_64 = (uint64_t) N * OH; + if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX) { + return false; + } + const uint32_t patch_stride = (uint32_t) patch_stride_64; + const uint32_t total_rows = (uint32_t) total_rows_64; + + if (dst->ne[0] != patch_stride || dst->ne[1] != OW || (is_2D && dst->ne[2] != OH)) { + return false; + } + + const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); + + bool is_patchembed = ggml_is_contiguous(src1) && (s0 == (int32_t) KW && p0 == 0 && d0 == 1); + if (is_2D) { + is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1); + } + + const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8; + const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads; + const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0; + const uint32_t row_size = (uint32_t) dst->nb[2]; + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + + if (is_patchembed) { + const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * IW * sizeof(float); + const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem; + if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) { + const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256); + const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads); + + if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { + kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA; + kparams->n_threads = (uint8_t) n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->src_row_bytes = src_row_bytes; + kparams->dst_row_bytes = dst_row_bytes; + kparams->owb = OW; + kparams->wb = IW; + kparams->nbpr = 1; + kparams->N = N; + kparams->IC = IC; + kparams->ID = 1; + kparams->IH = IH; + kparams->IW = IW; + kparams->KD = 1; + kparams->KH = KH; + kparams->KW = KW; + kparams->OD = 1; + kparams->OH = OH; + kparams->OW = OW; + kparams->patch_stride = patch_stride; + kparams->total_rows = total_rows; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_OH = init_fastdiv_values(OH); + kparams->div_OD = init_fastdiv_values(1); + return true; + } + } + } + + for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) { + const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1; + if (Wb > UINT32_MAX) { + if (owb == 1) break; + continue; + } + + const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * Wb * sizeof(float); + const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem; + if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) { + if (owb == 1) break; + continue; + } + + const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256); + const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads); + + if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { + kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA; + kparams->n_threads = (uint8_t) n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->src_row_bytes = src_row_bytes; + kparams->dst_row_bytes = dst_row_bytes; + kparams->owb = owb; + kparams->wb = (uint32_t) Wb; + kparams->nbpr = (OW + owb - 1) / owb; + kparams->N = N; + kparams->IC = IC; + kparams->ID = 1; + kparams->IH = IH; + kparams->IW = IW; + kparams->KD = 1; + kparams->KH = KH; + kparams->KW = KW; + kparams->OD = 1; + kparams->OH = OH; + kparams->OW = OW; + kparams->patch_stride = patch_stride; + kparams->total_rows = total_rows; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_OH = init_fastdiv_values(OH); + kparams->div_OD = init_fastdiv_values(1); + return true; + } + if (owb == 1) break; + } + + return false; +} + +static bool htp_kparams_solve_im2col_3d( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_im2col_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED; + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (!src0 || !src1 || !dst) { + return false; + } + if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) { + return false; + } + if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) { + return false; + } + + const int32_t * params = (const int32_t *) op->op_params; + const int32_t s0 = params[0], s1 = params[1], s2 = params[2]; + const int32_t p0 = params[3], p1 = params[4], p2 = params[5]; + const int32_t d0 = params[6], d1 = params[7], d2 = params[8]; + const int32_t ic = params[9]; + + if (s0 <= 0 || s1 <= 0 || s2 <= 0 || + p0 < 0 || p1 < 0 || p2 < 0 || + d0 <= 0 || d1 <= 0 || d2 <= 0 || ic <= 0) { + return false; + } + if (src1->ne[3] % ic != 0) { + return false; + } + + const uint32_t IC = (uint32_t) ic; + const uint32_t N = (uint32_t) (src1->ne[3] / IC); + if (N == 0) { + return false; + } + + uint32_t OW, OH, OD; + if (!im2col_3d_calc_output_size(src1->ne[0], src0->ne[0], s0, p0, d0, &OW) || + !im2col_3d_calc_output_size(src1->ne[1], src0->ne[1], s1, p1, d1, &OH) || + !im2col_3d_calc_output_size(src1->ne[2], src0->ne[2], s2, p2, d2, &OD)) { + return false; + } + + const uint32_t KD = (uint32_t) src0->ne[2]; + const uint32_t KH = (uint32_t) src0->ne[1]; + const uint32_t KW = (uint32_t) src0->ne[0]; + const uint32_t ID = (uint32_t) src1->ne[2]; + const uint32_t IH = (uint32_t) src1->ne[1]; + const uint32_t IW = (uint32_t) src1->ne[0]; + + const uint64_t patch_stride_64 = (uint64_t) IC * KD * KH * KW; + const uint64_t total_rows_64 = (uint64_t) N * OD * OH; + const uint64_t dst_ne3_64 = (uint64_t) OD * N; + + if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX || dst_ne3_64 > UINT32_MAX) { + return false; + } + const uint32_t patch_stride = (uint32_t) patch_stride_64; + const uint32_t total_rows = (uint32_t) total_rows_64; + const uint32_t dst_ne3 = (uint32_t) dst_ne3_64; + + if (dst->ne[0] != patch_stride || dst->ne[1] != OW || dst->ne[2] != OH || dst->ne[3] != dst_ne3) { + return false; + } + + const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float); + + const bool is_patchembed = ggml_is_contiguous(src1) && + (s0 == (int32_t) KW && s1 == (int32_t) KH && s2 == (int32_t) KD && + p0 == 0 && p1 == 0 && p2 == 0 && + d0 == 1 && d1 == 1 && d2 == 1); + + const uint32_t max_threads = (sess && htp_session_n_threads(sess) > 0) ? htp_session_n_threads(sess) : 8; + const uint32_t n_threads = (total_rows < max_threads) ? total_rows : max_threads; + const uint32_t vtcm_size = (sess && htp_session_vtcm_size(sess) > 0) ? (uint32_t) htp_session_vtcm_size(sess) : 0; + const uint32_t row_size = (uint32_t) dst->nb[2]; + const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1; + + if (is_patchembed) { + const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * IW * sizeof(float); + const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem; + if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) { + const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256); + const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads); + + if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { + kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA; + kparams->n_threads = (uint8_t) n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->src_row_bytes = src_row_bytes; + kparams->dst_row_bytes = dst_row_bytes; + kparams->owb = OW; + kparams->wb = IW; + kparams->nbpr = 1; + kparams->N = N; + kparams->IC = IC; + kparams->ID = ID; + kparams->IH = IH; + kparams->IW = IW; + kparams->KD = KD; + kparams->KH = KH; + kparams->KW = KW; + kparams->OD = OD; + kparams->OH = OH; + kparams->OW = OW; + kparams->patch_stride = patch_stride; + kparams->total_rows = total_rows; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_OH = init_fastdiv_values(OH); + kparams->div_OD = init_fastdiv_values(OD); + return true; + } + } + } + + for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) { + const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1; + if (Wb > UINT32_MAX) { + if (owb == 1) break; + continue; + } + + const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * Wb * sizeof(float); + const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem; + if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) { + if (owb == 1) break; + continue; + } + + const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256); + const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256); + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads); + + if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) { + kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA; + kparams->n_threads = (uint8_t) n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->src_row_bytes = src_row_bytes; + kparams->dst_row_bytes = dst_row_bytes; + kparams->owb = owb; + kparams->wb = (uint32_t) Wb; + kparams->nbpr = (OW + owb - 1) / owb; + kparams->N = N; + kparams->IC = IC; + kparams->ID = ID; + kparams->IH = IH; + kparams->IW = IW; + kparams->KD = KD; + kparams->KH = KH; + kparams->KW = KW; + kparams->OD = OD; + kparams->OH = OH; + kparams->OW = OW; + kparams->patch_stride = patch_stride; + kparams->total_rows = total_rows; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_OH = init_fastdiv_values(OH); + kparams->div_OD = init_fastdiv_values(OD); + return true; + } + if (owb == 1) break; + } + + return false; +} + +static bool htp_kparams_solve_pad( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_pad_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + const int32_t lp0 = ((const int32_t *) op->op_params)[0]; + const int32_t rp0 = ((const int32_t *) op->op_params)[1]; + const bool circular = (((const int32_t *) op->op_params)[8] != 0); + + if (circular && (lp0 > src0->ne[0] || rp0 > src0->ne[0])) { + return false; + } + + const uint32_t type_size = (uint32_t) ggml_type_size(src0->type); + const uint32_t total_dst_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]); + const uint32_t max_threads = (std::max)(1u, total_dst_rows); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + + struct htp_pad_vtcm_layout layout; + htp_pad_vtcm_layout_build(&layout, (uint32_t) src0->ne[0] * type_size, (uint32_t) dst->ne[0] * type_size, n_threads); + + const bool use_dma = (src0->nb[0] == type_size) && (src0->ne[0] >= 512) && + (htp_session_vtcm_size(sess) >= layout.total_bytes); + + kparams->n_threads = n_threads; + kparams->total_rows = total_dst_rows; + kparams->use_dma = use_dma ? 1 : 0; + kparams->elem_size = type_size; + kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0; + + if (circular && use_dma) { + kparams->kernel_type = HTP_PAD_KERNEL_HVX_CIRCULAR_DMA; + } else if (circular) { + kparams->kernel_type = HTP_PAD_KERNEL_HVX_CIRCULAR; + } else if (use_dma) { + kparams->kernel_type = HTP_PAD_KERNEL_HVX_DMA; + } else { + kparams->kernel_type = HTP_PAD_KERNEL_HVX; + } + + kparams->div_ne1 = init_fastdiv_values((uint32_t) dst->ne[1]); + kparams->div_ne1_ne2 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[2])); + kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); + kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]); + kparams->div_ne03 = init_fastdiv_values((uint32_t) src0->ne[3]); + + return true; +} + +static bool htp_kparams_solve_cumsum( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_cumsum_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + return false; + } + + const uint32_t total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); + const uint32_t max_threads = (std::max)(1u, total_rows); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + + struct htp_cumsum_vtcm_layout layout; + htp_cumsum_vtcm_layout_build(&layout, (uint32_t) src0->nb[1], (uint32_t) dst->nb[1], n_threads); + + const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes); + + kparams->n_threads = n_threads; + kparams->total_rows = total_rows; + kparams->use_dma = use_dma ? 1 : 0; + kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0; + + return true; +} + +static bool htp_kparams_solve_fill( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_fill_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * dst = op; + + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) { + return false; + } + if (!ggml_is_contiguous_rows(dst) || ggml_is_permuted(dst)) { + return false; + } + + const uint32_t ne1 = (uint32_t) dst->ne[1]; + const uint32_t ne2 = (uint32_t) dst->ne[2]; + const uint32_t ne3 = (uint32_t) dst->ne[3]; + const uint32_t nb1 = (uint32_t) dst->nb[1]; + + const uint32_t total_rows = ne1 * ne2 * ne3; + const uint32_t max_threads = (std::max)(1u, total_rows); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t elem_size = (uint32_t) ggml_type_size(dst->type); + const uint32_t rows_per_chunk = (nb1 > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(nb1, HEX_L2_LINE_SIZE)) : 1; + + kparams->n_threads = n_threads; + kparams->total_rows = total_rows; + kparams->elem_size = elem_size; + kparams->kernel_type = ggml_is_contiguous(dst) ? HTP_FILL_KERNEL_CONTIGUOUS : HTP_FILL_KERNEL_STRIDED; + kparams->rows_per_chunk = rows_per_chunk; + kparams->div_ne1 = init_fastdiv_values(ne1); + kparams->div_ne1_ne2 = init_fastdiv_values(ne1 * ne2); + + return true; +} + +static bool htp_kparams_solve_roll( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_roll_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_are_same_shape(src0, dst)) { + return false; + } + if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous(dst)) { + return false; + } + + const uint32_t ne0 = (uint32_t) dst->ne[0]; + const uint32_t ne1 = (uint32_t) dst->ne[1]; + const uint32_t ne2 = (uint32_t) dst->ne[2]; + const uint32_t ne3 = (uint32_t) dst->ne[3]; + + const int32_t s0 = ((const int32_t *) op->op_params)[0]; + uint32_t i0_src0 = 0; + if (-s0 < 0) { + i0_src0 = (uint32_t) (-s0 + (int32_t) ne0); + } else if ((uint32_t) (-s0) >= ne0) { + i0_src0 = (uint32_t) (-s0) - ne0; + } else { + i0_src0 = (uint32_t) (-s0); + } + + const uint32_t total_rows = ne1 * ne2 * ne3; + const uint32_t max_threads = (std::max)(1u, total_rows); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + + kparams->n_threads = n_threads; + kparams->total_rows = total_rows; + kparams->i0_src0 = i0_src0; + kparams->n0 = ne0 - i0_src0; + + if (htp_session_mdev_count(sess) <= 1) { + if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) { + kparams->kernel_type = HTP_ROLL_KERNEL_DMA_CONTIGUOUS; + } else { + kparams->kernel_type = HTP_ROLL_KERNEL_DMA_STRIDED; + } + } else { + kparams->kernel_type = HTP_ROLL_KERNEL_HVX; + } + + kparams->div_ne1 = init_fastdiv_values(ne1); + kparams->div_ne2_ne1 = init_fastdiv_values(ne2 * ne1); + + return true; +} + +static bool htp_kparams_solve_diag( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_diag_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (src0->ne[1] != 1 || dst->ne[0] != dst->ne[1] || dst->ne[0] != src0->ne[0]) { + return false; + } + if (!ggml_is_contiguous_rows(src0) || !ggml_is_contiguous_rows(dst) || + ggml_is_permuted(src0) || ggml_is_permuted(dst)) { + return false; + } + + const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]); + const uint32_t max_threads = (std::max)(1u, total_batches); + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), max_threads); + const uint32_t dst_batch_size = (uint32_t) (dst->ne[1] * dst->nb[1]); + + bool can_split = (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !ggml_is_permuted(dst); + uint32_t batches_per_chunk = 1; + if (can_split) { + if (dst->ne[2] > 1 && (dst->nb[2] & (HEX_L2_LINE_SIZE - 1)) == 0 && + (dst->ne[3] <= 1 || (dst->nb[3] & (HEX_L2_LINE_SIZE - 1)) == 0)) { + batches_per_chunk = 1; + } else if (dst->nb[2] == dst_batch_size && + (dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) { + batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1; + } else { + can_split = false; + } + } + + const uint32_t src_batch_size = (uint32_t) (src0->ne[0] * sizeof(float)); + const uint32_t dst_row_size = (uint32_t) (dst->ne[0] * sizeof(float)); + + struct htp_diag_vtcm_layout layout; + htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads); + + const bool use_dma = (htp_session_vtcm_size(sess) >= layout.total_bytes); + + kparams->n_threads = n_threads; + kparams->total_batches = total_batches; + kparams->use_dma = use_dma ? 1 : 0; + kparams->can_split = can_split ? 1 : 0; + kparams->batches_per_chunk = batches_per_chunk; + kparams->vtcm_size = use_dma ? (uint32_t) layout.total_bytes : 0; + kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]); + + return true; +} + +static bool htp_kparams_solve_solve_tri( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_solve_tri_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (src0->ne[0] != src0->ne[1] || src0->ne[1] != src1->ne[1] || + src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3] || + !ggml_are_same_shape(src1, dst)) { + return false; + } + + const uint32_t max_threads_sess = htp_session_n_threads(sess); + const uint32_t k = (uint32_t) src1->ne[0]; + const uint32_t col_block = 32; + const uint32_t k_chunks = (k + col_block - 1) / col_block; + const uint32_t total_batches = (uint32_t) (src0->ne[2] * src0->ne[3]); + const bool batched = total_batches >= max_threads_sess; + const uint32_t total_jobs = batched ? total_batches : (total_batches * k_chunks); + const uint32_t max_threads = (std::max)(1u, total_jobs); + const uint32_t n_threads = (std::min)(max_threads_sess, max_threads); + const uint32_t batch_size = (uint32_t) dst->nb[2]; + const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1; + + kparams->n_threads = n_threads; + kparams->total_batches = total_batches; + kparams->total_jobs = total_jobs; + kparams->k_chunks = k_chunks; + kparams->col_block = col_block; + kparams->batched = batched ? 1 : 0; + kparams->batches_per_chunk = batches_per_chunk; + kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]); + kparams->div_k_chunks = init_fastdiv_values(k_chunks > 0 ? k_chunks : 1); + + return true; +} + +template +static inline bool htp_kparams_invoke( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + void * kparams +) { + using KParams = decltype(htp_kparams_type(SolveFn)); + KParams tmp; + return SolveFn(sess, op, kparams ? (KParams *) kparams : &tmp); +} + +void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & node) { + const uint32_t n_threads = htp_session_n_threads(sess); + + if (htp_op_is_unary(node.opcode) && htp_unary_is_elementwise(node.opcode)) { + if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) { + return; + } + if (!ggml_are_same_shape(node.inputs[0], node.outputs[0])) { + return; + } + const size_t elem_size = ggml_type_size(node.inputs[0]->type); + struct ggml_tensor src0_c = *node.inputs[0]; + struct ggml_tensor dst_c = *node.outputs[0]; + struct ggml_tensor * tensors[2] = { &src0_c, &dst_c }; + htp_collapse_dims(tensors, 2, elem_size, n_threads); + if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { + node.inputs[0] = node.add_dummy(src0_c); + node.outputs[0] = node.add_dummy(dst_c); + } + return; + } + + if (htp_op_is_glu(node.opcode)) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + if (!ggml_are_same_shape(node.inputs[0], node.outputs[0]) || + !ggml_are_same_shape(node.inputs[1], node.outputs[0])) { + return; + } + const size_t elem_size = ggml_type_size(node.inputs[0]->type); + struct ggml_tensor src0_c = *node.inputs[0]; + struct ggml_tensor src1_c = *node.inputs[1]; + struct ggml_tensor dst_c = *node.outputs[0]; + struct ggml_tensor * tensors[3] = { &src0_c, &src1_c, &dst_c }; + htp_collapse_dims(tensors, 3, elem_size, n_threads); + if (src0_c.ne[0] != node.inputs[0]->ne[0] || src0_c.ne[1] != node.inputs[0]->ne[1]) { + node.inputs[0] = node.add_dummy(src0_c); + node.inputs[1] = node.add_dummy(src1_c); + node.outputs[0] = node.add_dummy(dst_c); + } + return; + } + + if (node.opcode == HTP_OP_MUL_MAT) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * src0 = node.inputs[0]; + const struct ggml_tensor * src1 = node.inputs[1]; + const struct ggml_tensor * dst = node.outputs[0]; + if (htp_matmul_can_collapse(src0, src1, dst)) { + node.inputs[1] = node.add_dummy(htp_tensor_collapse_rows(src1)); + node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst)); + } + return; + } + + if (node.opcode == HTP_OP_MUL_MAT_ADD) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * src0 = node.inputs[0]; + const struct ggml_tensor * src1 = node.inputs[1]; + const struct ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const struct ggml_tensor * dst = node.outputs[0]; + if (htp_matmul_add_can_collapse(src0, src1, src2, dst)) { + node.inputs[1] = node.add_dummy(htp_tensor_collapse_rows(src1)); + node.outputs[0] = node.add_dummy(htp_tensor_collapse_rows(dst)); + if (src2 && (src2->ne[1] * src2->ne[2] * src2->ne[3] > 1)) { + node.inputs[2] = node.add_dummy(htp_tensor_collapse_rows(src2)); + } + } + return; + } +} + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + htp_opnode & node +) { + htp_opnode_collapse(sess, node); + + const ggml_tensor * src0 = node.inputs.empty() ? nullptr : node.inputs[0]; + const ggml_tensor * src1 = node.inputs.size() > 1 ? node.inputs[1] : nullptr; + const ggml_tensor * dst = node.dst(); + + if (htp_op_is_unary(node.opcode)) { + return htp_kparams_solve_unary_impl(sess, node.opcode, src0, src1, dst, node.kparams()); + } + if (htp_op_is_glu(node.opcode)) { + return htp_kparams_solve_unary_mul_glu(sess, node.opcode, src0, src1, dst, node.kparams()); + } + if (node.opcode == HTP_OP_MUL_MAT) { + if (!htp_matmul_check_types(src0, src1, dst)) { + return false; + } + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, 0, 0, node.kparams()); + } + if (node.opcode == HTP_OP_MUL_MAT_ADD) { + const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0; + return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, node.kparams()); + } + + return htp_kparams_solve(sess, node.node, node.kernel_params); +} + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + void * kparams +) { + const htp_op_code opcode = htp_op_remap(op); + if (htp_op_is_unary(opcode) || htp_op_is_glu(opcode)) { + htp_opnode tmp(opcode, op); + if (!htp_kparams_solve(sess, tmp)) { + return false; + } + if (kparams) { + memcpy(kparams, tmp.kernel_params, sizeof(tmp.kernel_params)); + } + return true; + } + + switch (opcode) { + case HTP_OP_MUL_MAT: + case HTP_OP_MUL_MAT_ID: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_MUL: + case HTP_OP_ADD: + case HTP_OP_ADD_ID: + case HTP_OP_SUB: + case HTP_OP_DIV: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_FLASH_ATTN_EXT: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_GET_ROWS: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_SET_ROWS: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_ROPE: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_SSM_CONV: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_SOFTMAX: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_GATED_DELTA_NET: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_ARGSORT: + case HTP_OP_TOP_K: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_POOL_2D: + case HTP_OP_POOL_1D: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_CONCAT: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_CPY: + case HTP_OP_CPY_FENCE: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_IM2COL: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_IM2COL_3D: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_SUM_ROWS: + case HTP_OP_SUM: + case HTP_OP_ARGMAX: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_REPEAT: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_PAD: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_CUMSUM: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_FILL: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_ROLL: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_DIAG: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_SOLVE_TRI: + return htp_kparams_invoke(sess, op, kparams); + + default: + if (htp_op_is_unary(opcode)) { + return htp_kparams_invoke(sess, op, kparams); + } + if (htp_op_is_glu(opcode)) { + return htp_kparams_invoke(sess, op, kparams); + } + return false; + } +} diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h new file mode 100644 index 0000000..0bde658 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-kparams.h @@ -0,0 +1,136 @@ +#ifndef HTP_KPARAMS_H +#define HTP_KPARAMS_H + +#include "htp-common.h" + +#include "htp/allreduce-ops.h" +#include "htp/argsort-ops.h" +#include "htp/binary-ops.h" +#include "htp/concat-ops.h" +#include "htp/cpy-ops.h" +#include "htp/cumsum-ops.h" +#include "htp/diag-ops.h" +#include "htp/fill-ops.h" +#include "htp/flash-attn-ops.h" +#include "htp/gated-delta-net-ops.h" +#include "htp/get-rows-ops.h" +#include "htp/glu-ops.h" +#include "htp/im2col-ops.h" +#include "htp/matmul-ops.h" +#include "htp/pad-ops.h" +#include "htp/pool-ops.h" +#include "htp/repeat-ops.h" +#include "htp/roll-ops.h" +#include "htp/rope-ops.h" +#include "htp/set-rows-ops.h" +#include "htp/softmax-ops.h" +#include "htp/solve-tri-ops.h" +#include "htp/ssm-conv.h" +#include "htp/sum-rows-ops.h" +#include "htp/unary-ops.h" + +#include +#include + +bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride); +bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst); +bool htp_matmul_add_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * src2, const struct ggml_tensor * dst); +ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t); + +template +auto htp_kparams_type(bool (*)(Args...)) -> std::remove_pointer_t(), ...))>>; + +template +static inline bool htp_kparams_check(Args &&... args) { + decltype(htp_kparams_type(SolveFn)) kparams; + return SolveFn(std::forward(args)..., &kparams); +} + +struct htp_opnode; + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + struct htp_opnode & node +); + +void htp_opnode_collapse( + const struct ggml_hexagon_session * sess, + struct htp_opnode & node +); + +bool htp_kparams_solve( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + void * kparams = nullptr +); + +static inline bool htp_kparams_check( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op +) { + return htp_kparams_solve(sess, op, nullptr); +} + +bool htp_kparams_solve_allreduce( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * dst, + uint32_t rank, + uint32_t n_ranks, + bool has_add, + bool is_row_bcast, + bool is_shard_ok, + struct htp_allreduce_kernel_params * kparams +); + +bool htp_kparams_solve_rms_norm_mul( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +); + +bool htp_kparams_solve_matmul_add( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * src2, + const struct ggml_tensor * dst, + struct htp_mm_kernel_params * kparams +); + +bool htp_kparams_solve_mmnx( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + int32_t n_weights, + struct htp_mm_kernel_params * kparams +); + +bool htp_kparams_solve_mmidnx( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * act, + const struct ggml_tensor * dst, + int32_t n_weights, + struct htp_mm_kernel_params * kparams +); + +bool htp_kparams_solve_repeat_get_rows( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_get_rows_kernel_params * kparams +); + +bool htp_kparams_solve_unary_mul_glu( + const struct ggml_hexagon_session * sess, + uint32_t op, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + struct htp_glu_kernel_params * kparams +); + +#endif // HTP_KPARAMS_H diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp new file mode 100644 index 0000000..3509ffe --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-opnode.cpp @@ -0,0 +1,260 @@ +#include "htp-opnode.h" + +#include "htp/allreduce-ops.h" +#include "htp/argsort-ops.h" +#include "htp/binary-ops.h" +#include "htp/cumsum-ops.h" +#include "htp/diag-ops.h" +#include "htp/fill-ops.h" +#include "htp/flash-attn-ops.h" +#include "htp/gated-delta-net-ops.h" +#include "htp/get-rows-ops.h" +#include "htp/glu-ops.h" +#include "htp/matmul-ops.h" +#include "htp/pad-ops.h" +#include "htp/repeat-ops.h" +#include "htp/roll-ops.h" +#include "htp/softmax-ops.h" +#include "htp/solve-tri-ops.h" +#include "htp/ssm-conv.h" +#include "htp/sum-rows-ops.h" +#include "htp/unary-ops.h" + +#include + +std::string htp_opnode::name() const { + switch (opcode) { + case HTP_OP_MDEV_GROUP: return "MDEV_GROUP"; + case HTP_OP_CPY_FENCE: return "CPY+FENCE"; + case HTP_OP_FENCE: return (node && node->op_params[1] == 0) ? "FENCE_WAIT" : "FENCE_SIGNAL"; + case HTP_OP_ALLREDUCE: return "ALLREDUCE"; + case HTP_OP_ALLREDUCE_ADD: return "ALLREDUCE+ADD"; + case HTP_OP_HADAMARD: return "MUL_MAT_HADAMARD"; + case HTP_OP_MUL_MAT_NX: return "MUL_MAT_NX"; + case HTP_OP_MUL_MAT_ID_NX: return "MUL_MAT_ID_NX"; + case HTP_OP_ADD: + case HTP_OP_SUB: + case HTP_OP_MUL: + case HTP_OP_DIV: + if (!fused.empty()) { + const auto * kp = kparams(); + if (kp->kernel_type != HTP_BINARY_KERNEL_SCALAR) { + switch (opcode) { + case HTP_OP_ADD: return "ADD_NX"; + case HTP_OP_SUB: return "SUB_NX"; + case HTP_OP_MUL: return "MUL_NX"; + case HTP_OP_DIV: return "DIV_NX"; + default: break; + } + } + } + break; + default: + break; + } + + if (!node) { + return ""; + } + if (htp_is_hadamard(node)) { + return "MUL_MAT_HADAMARD"; + } + + std::string res = ggml_op_desc(node); + for (const auto * t : fused) { + res += "+"; + res += ggml_op_desc(t); + } + return res; +} + +static const char * format_tensor_name(const struct ggml_tensor * t) { + return t ? t->name : "NONE"; +} + +static std::string format_tensor_dims(const struct ggml_tensor * t) { + if (!t) { + return "NONE"; + } + char buf[64]; + if (t->ne[2] == 1 && t->ne[3] == 1) { + snprintf(buf, sizeof(buf), "%d:%d", (int) t->ne[0], (int) t->ne[1]); + } else { + snprintf(buf, sizeof(buf), "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]); + } + return buf; +} + +static const char * format_tensor_type(const struct ggml_tensor * t) { + return t ? ggml_type_name(t->type) : "NONE"; +} + +static std::string format_tensor_strides(const struct ggml_tensor * t) { + if (!t) { + return "NONE"; + } + char buf[64]; + const char * c = ggml_is_contiguous(t) ? "" : "!"; + if (t->ne[2] == 1 && t->ne[3] == 1) { + snprintf(buf, sizeof(buf), "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c); + } else { + snprintf(buf, sizeof(buf), "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c); + } + return buf; +} + +static const char * format_tensor_buff(const struct ggml_tensor * t) { + if (t && t->buffer) { + return ggml_backend_buffer_name(t->buffer); + } + return "NONE"; +} + +template +static std::string format_op_tensors(const htp_opnode & node, Fn fn) { + std::string res; + const auto & inputs = node.get_inputs(); + const auto & outputs = node.get_outputs(); + + if (!inputs.empty()) { + for (size_t i = 0; i < inputs.size(); i++) { + if (i > 0) res += " x "; + res += fn(inputs[i]); + } + res += " -> "; + } + + if (outputs.empty()) { + res += fn(node.dst()); + } else { + for (size_t i = 0; i < outputs.size(); i++) { + if (i > 0) res += " x "; + res += fn(outputs[i]); + } + } + return res; +} + +static std::string format_kernel_params(const htp_opnode & node) { + char str[128]; + const size_t max_size = sizeof(str); + + if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID || + node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ID_NX || + node.opcode == HTP_OP_MUL_MAT_ADD) { + const auto * kparams = node.kparams(); + const char * path = "unknown"; + int32_t type = kparams->kernel_type; + if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_BATCHED) { + path = (kparams->tile_size > 0) ? "hmx-tiled" : "hmx-flat"; + } else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM || + type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM) { + path = "hvx-flat"; + } else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { + path = "hvx-tiled"; + } + snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) { + const auto * kparams = node.kparams(); + const char * path = "unknown"; + int32_t type = kparams->kernel_type; + if (type == HTP_FA_KERNEL_HMX) { + path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq"; + } else if (type == HTP_FA_KERNEL_HVX) { + path = "hvx"; + } + snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size); + } else if (htp_op_is_unary(node.opcode)) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) { + snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]); + } else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) { + snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]); + } else if (node.opcode == HTP_OP_ALLREDUCE && node.node) { + snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]); + } else if (node.opcode == HTP_OP_SSM_CONV) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_SOFTMAX) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_type, (int) kparams->n_threads, (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_GATED_DELTA_NET) { + const auto * kparams = node.kparams(); + const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent"; + snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar", + (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || + node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d", + node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort", + (int) kparams->n_threads, (int) kparams->n_chunks, + (int) kparams->chunk_elems, (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_GET_ROWS) { + const auto * kparams = node.kparams(); + const char * ktype_str = "unknown"; + switch (kparams->kernel_type) { + case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break; + case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break; + case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break; + } + snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size); + } else if (htp_op_is_glu(node.opcode)) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "nth %d vtcm %u", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_PAD) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "k%d nth %d vtcm %u", (int) kparams->kernel_type, (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_CUMSUM) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_FILL) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "%s nth %d", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (int) kparams->n_threads); + } else if (node.opcode == HTP_OP_ROLL) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "k%d nth %d", (int) kparams->kernel_type, (int) kparams->n_threads); + } else if (node.opcode == HTP_OP_DIAG) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_SOLVE_TRI) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "%s nth %d", kparams->batched ? "batched" : "chunked", (int) kparams->n_threads); + } else { + snprintf(str, max_size, "----"); + } + return str; +} + +static void append_section(std::string & res, const std::string & sec) { + if (!res.empty()) { + res += "|"; + } + res += sec; +} + +std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags) { + std::string res; + if (flags & HTP_OPFMT_NAMES) append_section(res, format_tensor_name(t)); + if (flags & HTP_OPFMT_DIMS) append_section(res, format_tensor_dims(t)); + if (flags & HTP_OPFMT_TYPES) append_section(res, format_tensor_type(t)); + if (flags & HTP_OPFMT_STRIDES) append_section(res, format_tensor_strides(t)); + if (flags & HTP_OPFMT_BUFFS) append_section(res, format_tensor_buff(t)); + return res; +} + +std::string htp_format_op(const htp_opnode & node, uint32_t flags) { + std::string res; + if (flags & HTP_OPFMT_OP) append_section(res, node.name()); + if (flags & HTP_OPFMT_NAMES) append_section(res, format_op_tensors(node, format_tensor_name)); + if (flags & HTP_OPFMT_DIMS) append_section(res, format_op_tensors(node, format_tensor_dims)); + if (flags & HTP_OPFMT_TYPES) append_section(res, format_op_tensors(node, format_tensor_type)); + if (flags & HTP_OPFMT_STRIDES) append_section(res, format_op_tensors(node, format_tensor_strides)); + if (flags & HTP_OPFMT_BUFFS) append_section(res, format_op_tensors(node, format_tensor_buff)); + if (flags & HTP_OPFMT_KPARAMS) append_section(res, format_kernel_params(node)); + return res; +} diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h new file mode 100644 index 0000000..a30c5b6 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -0,0 +1,172 @@ +#ifndef HTP_OPNODE_H +#define HTP_OPNODE_H + +#include "ggml-backend-impl.h" +#include "ggml-impl.h" +#include "htp/htp-ops.h" + +#include +#include +#include +#include +#include + +#define HTP_OP_MAX_FUSED 16 + +// local std::inplace_vector replacement +template +struct htp_opvec { + T data[MAX_N] {}; + size_t n = 0; + + htp_opvec() = default; + htp_opvec(std::initializer_list init) { + GGML_ASSERT(init.size() <= MAX_N); + for (const auto & v : init) { + data[n++] = v; + } + } + + size_t size() const { return n; } + bool empty() const { return n == 0; } + void clear() { n = 0; } + T * begin() { return data; } + T * end() { return data + n; } + const T * begin() const { return data; } + const T * end() const { return data + n; } + T & back() { return data[n - 1]; } + const T & back() const { return data[n - 1]; } + T & operator[](size_t i) { return data[i]; } + const T & operator[](size_t i) const { return data[i]; } + + void push_back(T val) { + GGML_ASSERT(n < MAX_N); + data[n++] = val; + } + + void insert(T * pos, T val) { + GGML_ASSERT(n < MAX_N); + for (T * p = data + n; p > pos; --p) { + *p = *(p - 1); + } + *pos = val; + n++; + } +}; + +static inline bool htp_is_hadamard(const struct ggml_tensor * t) { + return t && t->op == GGML_OP_MUL_MAT && ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD; +} + +struct htp_opnode { + const ggml_tensor * node { nullptr }; + htp_op_code opcode { HTP_OP_INVALID }; + int32_t params[HTP_OP_MAX_PARAMS] {0}; + int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0}; + + std::vector> dummy; + + htp_opvec fused; + htp_opvec inputs; + htp_opvec outputs; + + int n_active_src(const ggml_tensor * t) const { + if (!t) return 0; + for (int i = GGML_MAX_SRC - 1; i >= 0; i--) { + if (t->src[i]) { + return i + 1; + } + } + return 0; + } + + void init(const ggml_tensor * node) { + static_assert(sizeof(this->params) == sizeof(this->node->op_params), "params size mismatch"); + + this->node = node; + if (!node) return; + + memcpy(this->params, this->node->op_params, sizeof(this->params)); + + // Build inputs (preserving optional nullptrs) + if (htp_is_hadamard(this->node)) { + this->inputs.push_back(this->node->src[1]); + } else { + int n_inputs = n_active_src(this->node); + for (int i = 0; i < n_inputs; i++) { + this->inputs.push_back(this->node->src[i]); + } + } + + // Build outputs + this->outputs.push_back(this->dst()); + } + + htp_opnode(htp_op_code opcode = HTP_OP_INVALID, const ggml_tensor * node = nullptr) : opcode(opcode) { + init(node); + } + + htp_opnode(const ggml_tensor * node) : opcode(HTP_OP_INVALID) { + init(node); + } + + const ggml_tensor * src0() const { return node->src[0]; } + const ggml_tensor * src1() const { return node->src[1]; } + const ggml_tensor * dst() const { return outputs.empty() ? node : outputs.back(); } + + ggml_tensor * add_dummy(const ggml_tensor & t) { + dummy.push_back(std::make_shared(t)); + return dummy.back().get(); + } + + const htp_opvec & get_inputs() const { + return inputs; + } + + const htp_opvec & get_outputs() const { + return outputs; + } + + template + T * kparams() { + static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS"); + static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t"); + return reinterpret_cast(kernel_params); + } + + template + const T * kparams() const { + static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS"); + static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t"); + return reinterpret_cast(kernel_params); + } + + template + void set_kparams(const T & kp) { + static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS"); + static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t"); + memcpy(kernel_params, &kp, sizeof(T)); + } + + std::string name() const; +}; + +enum htp_opformat_flags { + HTP_OPFMT_OP = (1 << 0), + HTP_OPFMT_NAMES = (1 << 1), + HTP_OPFMT_DIMS = (1 << 2), + HTP_OPFMT_TYPES = (1 << 3), + HTP_OPFMT_STRIDES = (1 << 4), + HTP_OPFMT_BUFFS = (1 << 5), + HTP_OPFMT_KPARAMS = (1 << 6), + + HTP_OPFMT_EXEC = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS | HTP_OPFMT_KPARAMS, + HTP_OPFMT_SUPP = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS, + HTP_OPFMT_PROF = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_KPARAMS, + HTP_OPFMT_TENSOR = HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES, +}; + +std::string htp_format_op(const htp_opnode & node, uint32_t flags = HTP_OPFMT_EXEC); +std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags = HTP_OPFMT_TENSOR); + +#endif // HTP_OPNODE_H diff --git a/ggml/src/ggml-hexagon/htp-repack.cpp b/ggml/src/ggml-hexagon/htp-repack.cpp new file mode 100644 index 0000000..d2e3df2 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-repack.cpp @@ -0,0 +1,1636 @@ +#include "htp-repack.h" + +#define GGML_COMMON_IMPL_CPP +#include "ggml-common.h" +#include "ggml-impl.h" +#include "ggml-quants.h" + +#include +#include +#include +#include +#include + +#if defined(__ARM_NEON) || defined(__aarch64__) +# include +#endif + +// --- 4-bit block-32 nibble helpers (shared by Q4_0, IQ4_NL, Q4_1, MXFP4) --- + +static inline void unpack_nibbles_32(uint8_t * dst, const uint8_t * qs) { + for (unsigned int i = 0; i < 16; ++i) { + dst[i + 0] = qs[i] & 0x0F; + dst[i + 16] = qs[i] >> 4; + } +} + +static inline void pack_nibbles_32(uint8_t * qs, const uint8_t * src) { + for (unsigned int i = 0; i < 16; ++i) { + qs[i] = src[i + 0] | (src[i + 16] << 4); + } +} + +static inline void transpose_nibbles_to_tile(uint8_t * tile_dst, const uint8_t tile_quants[32][32]) { + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; + } + } +} + +static inline void transpose_tile_to_nibbles(uint8_t tile_quants[32][32], const uint8_t * tile_src) { + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + const uint8_t val = tile_src[cp * 32 + row]; + tile_quants[row][2 * cp + 0] = val & 0x0F; + tile_quants[row][2 * cp + 1] = val >> 4; + } + } +} + +#if defined(__ARM_NEON) || defined(__aarch64__) +static void repack_neon_transpose8x8(const uint8x8_t rows[8], uint8x8_t columns[8]) { + const uint8x8x2_t t0 = vtrn_u8(rows[0], rows[1]); + const uint8x8x2_t t1 = vtrn_u8(rows[2], rows[3]); + const uint8x8x2_t t2 = vtrn_u8(rows[4], rows[5]); + const uint8x8x2_t t3 = vtrn_u8(rows[6], rows[7]); + const uint16x4x2_t t4 = vtrn_u16(vreinterpret_u16_u8(t0.val[0]), vreinterpret_u16_u8(t1.val[0])); + const uint16x4x2_t t5 = vtrn_u16(vreinterpret_u16_u8(t0.val[1]), vreinterpret_u16_u8(t1.val[1])); + const uint16x4x2_t t6 = vtrn_u16(vreinterpret_u16_u8(t2.val[0]), vreinterpret_u16_u8(t3.val[0])); + const uint16x4x2_t t7 = vtrn_u16(vreinterpret_u16_u8(t2.val[1]), vreinterpret_u16_u8(t3.val[1])); + const uint32x2x2_t t8 = vtrn_u32(vreinterpret_u32_u16(t4.val[0]), vreinterpret_u32_u16(t6.val[0])); + const uint32x2x2_t t9 = vtrn_u32(vreinterpret_u32_u16(t4.val[1]), vreinterpret_u32_u16(t6.val[1])); + const uint32x2x2_t ta = vtrn_u32(vreinterpret_u32_u16(t5.val[0]), vreinterpret_u32_u16(t7.val[0])); + const uint32x2x2_t tb = vtrn_u32(vreinterpret_u32_u16(t5.val[1]), vreinterpret_u32_u16(t7.val[1])); + columns[0] = vreinterpret_u8_u32(t8.val[0]); + columns[1] = vreinterpret_u8_u32(ta.val[0]); + columns[2] = vreinterpret_u8_u32(t9.val[0]); + columns[3] = vreinterpret_u8_u32(tb.val[0]); + columns[4] = vreinterpret_u8_u32(t8.val[1]); + columns[5] = vreinterpret_u8_u32(ta.val[1]); + columns[6] = vreinterpret_u8_u32(t9.val[1]); + columns[7] = vreinterpret_u8_u32(tb.val[1]); +} + +static uint8x8_t repack_neon_pack_4bit_pairs(uint8x8_t first, uint8x8_t second, bool high) { + const uint8x8x2_t separated = vuzp_u8(first, second); + const uint8x8_t mask = vdup_n_u8(0x0F); + uint8x8_t even = separated.val[0]; + uint8x8_t odd = separated.val[1]; + + if (high) { + even = vshr_n_u8(even, 4); + odd = vshr_n_u8(odd, 4); + } + + even = vand_u8(even, mask); + odd = vand_u8(odd, mask); + return vorr_u8(vshl_n_u8(odd, 4), even); +} + +static void repack_neon_4bit_rows(const uint8_t * qs[8], uint8_t * tile_dst, int row0) { + uint8x8_t low_rows[8]; + uint8x8_t high_rows[8]; + uint8x8_t columns[8]; + + for (int rr = 0; rr < 8; ++rr) { + const uint8x16_t packed = vld1q_u8(qs[rr]); + const uint8x8_t first = vget_low_u8(packed); + const uint8x8_t second = vget_high_u8(packed); + low_rows[rr] = repack_neon_pack_4bit_pairs(first, second, false); + high_rows[rr] = repack_neon_pack_4bit_pairs(first, second, true); + } + + repack_neon_transpose8x8(low_rows, columns); + for (int cp = 0; cp < 8; ++cp) { + vst1_u8(tile_dst + cp * 32 + row0, columns[cp]); + } + + repack_neon_transpose8x8(high_rows, columns); + for (int cp = 0; cp < 8; ++cp) { + vst1_u8(tile_dst + (cp + 8) * 32 + row0, columns[cp]); + } +} + +static void repack_neon_k4_rows(const uint8_t * qs[8], uint8_t * tile_dst, int row0, bool high) { + uint8x8_t first_rows[8]; + uint8x8_t second_rows[8]; + uint8x8_t columns[8]; + + for (int rr = 0; rr < 8; ++rr) { + first_rows[rr] = repack_neon_pack_4bit_pairs(vld1_u8(qs[rr]), vld1_u8(qs[rr] + 8), high); + second_rows[rr] = repack_neon_pack_4bit_pairs(vld1_u8(qs[rr] + 16), vld1_u8(qs[rr] + 24), high); + } + + repack_neon_transpose8x8(first_rows, columns); + for (int cp = 0; cp < 8; ++cp) { + vst1_u8(tile_dst + cp * 32 + row0, columns[cp]); + } + + repack_neon_transpose8x8(second_rows, columns); + for (int cp = 0; cp < 8; ++cp) { + vst1_u8(tile_dst + (cp + 8) * 32 + row0, columns[cp]); + } +} + +static void repack_neon_q8_rows(const uint8_t * qs[8], uint8_t * tile_dst, int row0) { + uint8x8_t rows[8]; + uint8x8_t columns[8]; + + for (int cp = 0; cp < 16; cp += 4) { + for (int rr = 0; rr < 8; ++rr) { + rows[rr] = vld1_u8(qs[rr] + 2 * cp); + } + repack_neon_transpose8x8(rows, columns); + + for (int pair = 0; pair < 4; ++pair) { + const uint8x8x2_t interleaved = vzip_u8(columns[2 * pair], columns[2 * pair + 1]); + uint8_t * dst = tile_dst + (cp + pair) * 64 + 2 * row0; + vst1_u8(dst, interleaved.val[0]); + vst1_u8(dst + 8, interleaved.val[1]); + } + } +} + +static inline void repack_store_u32(uint8_t * dst, uint32_t v) { + memcpy(dst, &v, sizeof(v)); +} + +static inline uint32x4_t repack_neon_transpose_2bit_4x4(uint32x4_t w) { + uint32x4_t t = vandq_u32(veorq_u32(w, vshrq_n_u32(w, 6)), vdupq_n_u32(0x00CC00CCu)); + w = veorq_u32(w, veorq_u32(t, vshlq_n_u32(t, 6))); + t = vandq_u32(veorq_u32(w, vshrq_n_u32(w, 12)), vdupq_n_u32(0x0000F0F0u)); + return veorq_u32(w, veorq_u32(t, vshlq_n_u32(t, 12))); +} + +static inline void repack_neon_q2_0_block(const uint8_t * qs, uint8_t * tile, int row) { + const uint32x4_t w = repack_neon_transpose_2bit_4x4(vreinterpretq_u32_u8(vld1q_u8(qs))); + repack_store_u32(tile + 0 * 128 + 4 * row, vgetq_lane_u32(w, 0)); + repack_store_u32(tile + 1 * 128 + 4 * row, vgetq_lane_u32(w, 1)); + repack_store_u32(tile + 2 * 128 + 4 * row, vgetq_lane_u32(w, 2)); + repack_store_u32(tile + 3 * 128 + 4 * row, vgetq_lane_u32(w, 3)); +} + +static inline void repack_neon_q1_0_block(const uint8_t * qs, uint8_t * tile, int row) { + uint32x4_t w = vreinterpretq_u32_u8(vld1q_u8(qs)); + uint32x4_t t = vandq_u32(veorq_u32(w, vshrq_n_u32(w, 1)), vdupq_n_u32(0x22222222u)); + w = veorq_u32(w, veorq_u32(t, vshlq_n_u32(t, 1))); + t = vandq_u32(veorq_u32(w, vshrq_n_u32(w, 3)), vdupq_n_u32(0x0A0A0A0Au)); + w = veorq_u32(w, veorq_u32(t, vshlq_n_u32(t, 3))); + w = repack_neon_transpose_2bit_4x4(w); + repack_store_u32(tile + 0 * 128 + 4 * row, vgetq_lane_u32(w, 0)); + repack_store_u32(tile + 1 * 128 + 4 * row, vgetq_lane_u32(w, 1)); + repack_store_u32(tile + 2 * 128 + 4 * row, vgetq_lane_u32(w, 2)); + repack_store_u32(tile + 3 * 128 + 4 * row, vgetq_lane_u32(w, 3)); +} + +static inline void repack_neon_pack_2bit_plane(uint8x16_t v0, uint8x16_t v1, uint8_t * dst, int row) { + const uint32x4x2_t z = vuzpq_u32(vreinterpretq_u32_u8(v0), vreinterpretq_u32_u8(v1)); + const uint32x4_t p01 = vorrq_u32(z.val[0], vshlq_n_u32(z.val[1], 2)); + const uint32x2x2_t u = vuzp_u32(vget_low_u32(p01), vget_high_u32(p01)); + const uint32x2_t out = vorr_u32(u.val[0], vshl_n_u32(u.val[1], 4)); + repack_store_u32(dst + 0 * 128 + row * 4, vget_lane_u32(out, 0)); + repack_store_u32(dst + 1 * 128 + row * 4, vget_lane_u32(out, 1)); +} + +static inline void repack_neon_q2_3_k_low2(const uint8_t * qs, int kt_local, uint8_t * lo_pl, int row) { + const uint8_t * qs_sub = qs + (kt_local / 4) * 32; + const int8x16_t shift = vdupq_n_s8(-2 * (kt_local % 4)); + const uint8x16_t mask = vdupq_n_u8(0x03); + const uint8x16_t v0 = vandq_u8(vshlq_u8(vld1q_u8(qs_sub + 0), shift), mask); + const uint8x16_t v1 = vandq_u8(vshlq_u8(vld1q_u8(qs_sub + 16), shift), mask); + repack_neon_pack_2bit_plane(v0, v1, lo_pl, row); +} + +static inline void repack_neon_q3_k_hmask(const uint8_t * hmask, int kt_local, uint8_t * neg_pl, int row) { + static const uint32_t mask_words[4] = { 0x01010101u, 0x02020202u, 0x04040404u, 0x08080808u }; + const uint8x16_t m0 = vreinterpretq_u8_u32(vld1q_u32(mask_words)); + const uint8x16_t m1 = vshlq_n_u8(m0, 4); + const uint8x16_t hbit = vdupq_n_u8((uint8_t) (1u << kt_local)); + const uint8x16_t b0 = vbicq_u8(m0, vtstq_u8(vld1q_u8(hmask + 0), hbit)); + const uint8x16_t b1 = vbicq_u8(m1, vtstq_u8(vld1q_u8(hmask + 16), hbit)); + const uint32x4_t b01 = vreinterpretq_u32_u8(vorrq_u8(b0, b1)); + const uint32x2_t f0 = vorr_u32(vget_low_u32(b01), vget_high_u32(b01)); + const uint32x2_t f1 = vpadd_u32(f0, f0); + repack_store_u32(neg_pl + row * 4, vget_lane_u32(f1, 0)); +} + +static inline void repack_neon_q6_k(const block_q6_K * b, int kt_local, uint8_t * lo_pl, uint8_t * hi_pl, int row) { + const int c = kt_local / 4; + const int g_sb = kt_local % 4; + const uint8_t * ql = b->ql + c * 64 + (g_sb & 1) * 32; + const uint8_t * qh = b->qh + c * 32; + const int8x16_t lshift = vdupq_n_s8((g_sb >= 2) ? -4 : 0); + const int8x16_t hshift = vdupq_n_s8(-2 * g_sb); + const uint8x16_t mask4 = vdupq_n_u8(0x0F); + const uint8x16_t mask2 = vdupq_n_u8(0x03); + + const uint8x16_t lo0 = vandq_u8(vshlq_u8(vld1q_u8(ql + 0), lshift), mask4); + const uint8x16_t lo1 = vandq_u8(vshlq_u8(vld1q_u8(ql + 16), lshift), mask4); + const uint32x4x2_t lz = vuzpq_u32(vreinterpretq_u32_u8(lo0), vreinterpretq_u32_u8(lo1)); + const uint32x4_t lout = vorrq_u32(lz.val[0], vshlq_n_u32(lz.val[1], 4)); + repack_store_u32(lo_pl + 0 * 128 + row * 4, vgetq_lane_u32(lout, 0)); + repack_store_u32(lo_pl + 1 * 128 + row * 4, vgetq_lane_u32(lout, 1)); + repack_store_u32(lo_pl + 2 * 128 + row * 4, vgetq_lane_u32(lout, 2)); + repack_store_u32(lo_pl + 3 * 128 + row * 4, vgetq_lane_u32(lout, 3)); + + const uint8x16_t hi0 = vandq_u8(vshlq_u8(vld1q_u8(qh + 0), hshift), mask2); + const uint8x16_t hi1 = vandq_u8(vshlq_u8(vld1q_u8(qh + 16), hshift), mask2); + repack_neon_pack_2bit_plane(hi0, hi1, hi_pl, row); +} + +static inline void repack_neon_q5_k_qh(const uint8_t * qh, uint8_t hbit, uint8_t * plane, int row) { + static const uint8_t mask_bytes[16] = { 1, 2, 1, 2, 1, 2, 1, 2, 4, 8, 4, 8, 4, 8, 4, 8 }; + const uint8x16_t m0 = vld1q_u8(mask_bytes); + const uint8x16_t m1 = vshlq_n_u8(m0, 4); + const uint8x16_t vbit = vdupq_n_u8(hbit); + const uint8x16_t b0 = vandq_u8(vtstq_u8(vld1q_u8(qh + 0), vbit), m0); + const uint8x16_t b1 = vandq_u8(vtstq_u8(vld1q_u8(qh + 16), vbit), m1); + const uint8x16_t b01 = vorrq_u8(b0, b1); + const uint8x8_t fold = vorr_u8(vget_low_u8(b01), vget_high_u8(b01)); + const uint8x8_t out = vpadd_u8(fold, fold); + plane[0 * 32 + row] = vget_lane_u8(out, 0); + plane[1 * 32 + row] = vget_lane_u8(out, 1); + plane[2 * 32 + row] = vget_lane_u8(out, 2); + plane[3 * 32 + row] = vget_lane_u8(out, 3); +} + +static inline uint16x8_t repack_neon_load_f16x8(const uint16_t * src) { + return vld1q_u16(src); +} + +#if defined(__aarch64__) || defined(__ARM_FP16_FORMAT_IEEE) +static inline uint16x8_t repack_neon_load_f16x8(const float * src) { + const float16x4_t lo = vcvt_f16_f32(vld1q_f32(src)); + const float16x4_t hi = vcvt_f16_f32(vld1q_f32(src + 4)); + return vreinterpretq_u16_f16(vcombine_f16(lo, hi)); +} + +static inline uint16x8_t repack_neon_load_f16x8(const ggml_bf16_t * src) { + const uint16x8_t bf = vld1q_u16((const uint16_t *) src); + const float32x4_t f_lo = vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)); + const float32x4_t f_hi = vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)); + const float16x4_t lo = vcvt_f16_f32(f_lo); + const float16x4_t hi = vcvt_f16_f32(f_hi); + return vreinterpretq_u16_f16(vcombine_f16(lo, hi)); +} +#else +static inline uint16x8_t repack_neon_load_f16x8(const float * src) { + uint16_t tmp[8]; + for (int i = 0; i < 8; ++i) { + tmp[i] = GGML_FP32_TO_FP16(src[i]); + } + return vld1q_u16(tmp); +} + +static inline uint16x8_t repack_neon_load_f16x8(const ggml_bf16_t * src) { + uint16_t tmp[8]; + for (int i = 0; i < 8; ++i) { + tmp[i] = GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(src[i])); + } + return vld1q_u16(tmp); +} +#endif + +template +static void repack_neon_f16_tile(uint16_t * tile_dst, const SrcT * src_slice, int64_t row_stride, int ct, int kt) { + for (int row0 = 0; row0 < 32; row0 += 4) { + const SrcT * r0 = src_slice + (int64_t) (ct * 32 + row0 + 0) * row_stride + kt * 32; + const SrcT * r1 = src_slice + (int64_t) (ct * 32 + row0 + 1) * row_stride + kt * 32; + const SrcT * r2 = src_slice + (int64_t) (ct * 32 + row0 + 2) * row_stride + kt * 32; + const SrcT * r3 = src_slice + (int64_t) (ct * 32 + row0 + 3) * row_stride + kt * 32; + for (int cp = 0; cp < 16; cp += 4) { + const uint32x4_t v0 = vreinterpretq_u32_u16(repack_neon_load_f16x8(r0 + 2 * cp)); + const uint32x4_t v1 = vreinterpretq_u32_u16(repack_neon_load_f16x8(r1 + 2 * cp)); + const uint32x4_t v2 = vreinterpretq_u32_u16(repack_neon_load_f16x8(r2 + 2 * cp)); + const uint32x4_t v3 = vreinterpretq_u32_u16(repack_neon_load_f16x8(r3 + 2 * cp)); + + const uint32x2x2_t t0 = vtrn_u32(vget_low_u32(v0), vget_low_u32(v1)); + const uint32x2x2_t t1 = vtrn_u32(vget_high_u32(v0), vget_high_u32(v1)); + const uint32x2x2_t t2 = vtrn_u32(vget_low_u32(v2), vget_low_u32(v3)); + const uint32x2x2_t t3 = vtrn_u32(vget_high_u32(v2), vget_high_u32(v3)); + + vst1q_u16(tile_dst + (cp + 0) * 64 + row0 * 2, vreinterpretq_u16_u32(vcombine_u32(t0.val[0], t2.val[0]))); + vst1q_u16(tile_dst + (cp + 1) * 64 + row0 * 2, vreinterpretq_u16_u32(vcombine_u32(t0.val[1], t2.val[1]))); + vst1q_u16(tile_dst + (cp + 2) * 64 + row0 * 2, vreinterpretq_u16_u32(vcombine_u32(t1.val[0], t3.val[0]))); + vst1q_u16(tile_dst + (cp + 3) * 64 + row0 * 2, vreinterpretq_u16_u32(vcombine_u32(t1.val[1], t3.val[1]))); + } + } +} +#endif + +// --- Generic 32x32 tile grid walkers for block-32 and float types --- + +template +static void repack_parallel_for_ct(size_t slice_bytes, int n_col_tiles, Fn && fn) { + int n_threads = 1; + if (slice_bytes > 128 * 1024) { + for (int nt : { 4, 3, 2 }) { + if (n_col_tiles % nt == 0) { + n_threads = nt; + break; + } + } + } + + if (n_threads <= 1) { + fn(0, n_col_tiles); + return; + } + + const int chunk = n_col_tiles / n_threads; + std::thread workers[3]; + for (int i = 0; i < n_threads - 1; ++i) { + workers[i] = std::thread([&fn, i, chunk]() { + fn(i * chunk, (i + 1) * chunk); + }); + } + fn((n_threads - 1) * chunk, n_col_tiles); + for (int i = 0; i < n_threads - 1; ++i) { + workers[i].join(); + } +} + +template +static void repack_to_tiled_grid_32(ggml_tensor * t, const void * data, size_t offset, size_t size, + size_t tile_size, PackTileFn && pack_tile) { + const SrcElemT * src_matrix = (const SrcElemT *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = hex_round_up(ne0, 32) / 32; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + + const size_t slice_size = (size_t) ne1 * ggml_row_size(t->type, ne0); + const int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } + + const int64_t row_stride = (ne0 / (32 / ELEMS_PER_KTILE)); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const SrcElemT * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * row_stride); + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; + + repack_parallel_for_ct(matrix_size, n_col_tiles, [&](int ct0, int ct1) { + for (int ct = ct0; ct < ct1; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + pack_tile(tile_dst, src_slice, ne0, ne1, row_stride, ct, kt); + } + } + }); + } +} + +template +static void unpack_from_tiled_grid_32(void * data, const ggml_tensor * t, size_t offset, size_t size, + size_t tile_size, UnpackTileFn && unpack_tile) { + uint8_t * dst_bytes = (uint8_t *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = hex_round_up(ne0, 32) / 32; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + + const size_t row_size_bytes = ggml_row_size(t->type, ne0); + const size_t slice_size = (size_t) ne1 * row_size_bytes; + const int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } + + const int64_t row_stride = (ne0 / (32 / ELEMS_PER_KTILE)); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + const size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; + const size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; + + const int64_t start_row = slice_offset_start / row_size_bytes; + const int64_t end_row = (std::min)((int64_t) ((slice_offset_end + row_size_bytes - 1) / row_size_bytes), ne1); + + const int start_ct = (int) (start_row / 32); + const int end_ct = (std::min)((int) ((end_row + 31) / 32), n_col_tiles); + + DstElemT * dst_slice = (DstElemT *) (dst_bytes + (cur_start_byte - offset)); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = start_ct; ct < end_ct; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; + unpack_tile(dst_slice, tile_src, ne0, start_row, end_row, row_stride, ct, kt); + } + } + } +} + +// --- Block-32 Quantized Types (Q4_0, IQ4_NL, Q4_1, Q8_0, MXFP4) --- + +static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_0, + [](uint8_t * tile_dst, const block_q4_0 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + scale_dst[row] = src_slice[(ct * 32 + row) * row_stride + kt].d; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_q4_0 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + scale_dst[row] = b.d; + } else { + memset(tile_quants[row], 8, 32); + scale_dst[row] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q4_0, + [](block_q4_0 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q4_0 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + b.d = scale_src[row]; + } + } + }); +} + +static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_1, + [](uint8_t * tile_dst, const block_q4_1 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + const block_q4_1 & src_block = src_slice[(ct * 32 + row) * row_stride + kt]; + scale_dst[2 * row + 0] = src_block.d; + scale_dst[2 * row + 1] = src_block.m; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_q4_1 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + scale_dst[2 * row + 0] = b.d; + scale_dst[2 * row + 1] = b.m; + } else { + memset(tile_quants[row], 0, 32); + scale_dst[2 * row + 0] = 0; + scale_dst[2 * row + 1] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q4_1, + [](block_q4_1 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q4_1 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + b.d = scale_src[2 * row + 0]; + b.m = scale_src[2 * row + 1]; + } + } + }); +} + +static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_MXFP4, + [](uint8_t * tile_dst, const block_mxfp4 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + uint8_t * scale_dst = tile_dst + 512; +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_4bit_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + scale_dst[row] = src_slice[(ct * 32 + row) * row_stride + kt].e; + } + return; + } +#endif + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + const block_mxfp4 & b = src_slice[r * row_stride + kt]; + unpack_nibbles_32(tile_quants[row], b.qs); + scale_dst[row] = b.e; + } else { + memset(tile_quants[row], 0, 32); + scale_dst[row] = 0; + } + } + transpose_nibbles_to_tile(tile_dst, tile_quants); + }); +} + +static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32(data, t, offset, size, HTP_TENSOR_TILE_SIZE_MXFP4, + [](block_mxfp4 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + uint8_t tile_quants[32][32]; + transpose_tile_to_nibbles(tile_quants, tile_src); + const uint8_t * scale_src = tile_src + 512; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_mxfp4 & b = dst_slice[(r - start_row) * row_stride + kt]; + pack_nibbles_32(b.qs, tile_quants[row]); + b.e = scale_src[row]; + } + } + }); +} + +static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q8_0, + [](uint8_t * tile_dst, const block_q8_0 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + ggml_half * scale_dst = (ggml_half *) (tile_dst + 1024); +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt < ne0 / 32) { + for (int row0 = 0; row0 < 32; row0 += 8) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = (const uint8_t *) src_slice[(ct * 32 + row0 + rr) * row_stride + kt].qs; + } + repack_neon_q8_rows(qs, tile_dst, row0); + } + + for (int row = 0; row < 32; ++row) { + scale_dst[row] = src_slice[(ct * 32 + row) * row_stride + kt].d; + } + return; + } +#endif + for (int cp = 0; cp < 16; cp++) { + const int col0 = cp * 2; + const int col1 = col0 + 1; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + const block_q8_0 * b = (r < ne1 && kt < ne0 / 32) ? &src_slice[r * row_stride + kt] : nullptr; + tile_dst[cp * 64 + 2 * row + 0] = b ? b->qs[col0] : 0; + tile_dst[cp * 64 + 2 * row + 1] = b ? b->qs[col1] : 0; + } + } + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * row_stride + kt].d : 0; + } + }); +} + +static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_from_tiled_grid_32(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q8_0, + [](block_q8_0 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + for (int cp = 0; cp < 16; cp++) { + const int col0 = cp * 2; + const int col1 = col0 + 1; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + block_q8_0 & b = dst_slice[(r - start_row) * row_stride + kt]; + b.qs[col0] = tile_src[cp * 64 + 2 * row + 0]; + b.qs[col1] = tile_src[cp * 64 + 2 * row + 1]; + } + } + } + const ggml_half * scale_src = (const ggml_half *) (tile_src + 1024); + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row && kt < ne0 / 32) { + dst_slice[(r - start_row) * row_stride + kt].d = scale_src[row]; + } + } + }); +} + +// --- Float Types (F16, F32, BF16 -> f16_tiled HMX layout) --- + +static void repack_q2_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + GGML_ASSERT(offset == 0); + GGML_ASSERT(t->ne[0] % 64 == 0); + GGML_ASSERT(size == ggml_nbytes(t)); + + const block_q2_0 * src = (const block_q2_0 *) data; + const int64_t blocks_per_row = t->ne[0] / 64; + const int64_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32; + const int64_t n_weight_tiles = blocks_per_row; + const size_t matrix_size = (size_t) n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0; + + for (int64_t slice = 0; slice < t->ne[2] * t->ne[3]; ++slice) { + const block_q2_0 * src_slice = src + slice * t->ne[1] * blocks_per_row; + uint8_t * dst_slice = (uint8_t *) t->data + slice * matrix_size; + const size_t ct_bytes = (size_t) n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0; + + repack_parallel_for_ct(matrix_size, (int) n_col_tiles, [&](int ct0, int ct1) { + memset(dst_slice + (size_t) ct0 * ct_bytes, 0, (size_t) (ct1 - ct0) * ct_bytes); + const int64_t r0 = (int64_t) ct0 * 32; + const int64_t r1 = (std::min)((int64_t) ct1 * 32, t->ne[1]); + for (int64_t row_index = r0; row_index < r1; ++row_index) { + const int64_t ct = row_index / 32; + const int row = (int) (row_index % 32); + for (int64_t kt = 0; kt < n_weight_tiles; ++kt) { + const block_q2_0 * block = src_slice + row_index * blocks_per_row + kt; + uint8_t * tile = dst_slice + (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0; +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q2_0_block(block->qs, tile, row); +#else + for (int group = 0; group < 16; ++group) { + for (int byte = 0; byte < 4; ++byte) { + const int k = 4 * group + byte; + const uint8_t code = (block->qs[k / 4] >> (2 * (k % 4))) & 3; + tile[(group >> 2) * 128 + 4 * row + byte] |= code << (2 * (group & 3)); + } + } +#endif + memcpy(tile + 512 + 2 * row, &block->d, sizeof(block->d)); + } + } + }); + } +} + +static void repack_tiled_q2_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + GGML_ASSERT(t->ne[0] % 64 == 0); + GGML_ASSERT(offset + size <= ggml_nbytes(t)); + + const size_t first_block = offset / sizeof(block_q2_0); + const size_t end_block = (offset + size + sizeof(block_q2_0) - 1) / sizeof(block_q2_0); + const size_t blocks_per_row = t->ne[0] / 64; + const size_t blocks_per_slice = t->ne[1] * blocks_per_row; + const size_t n_weight_tiles = blocks_per_row; + const size_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32; + const size_t matrix_size = n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0; + + for (size_t block_index = first_block; block_index < end_block; ++block_index) { + const size_t slice = block_index / blocks_per_slice; + const size_t in_slice = block_index % blocks_per_slice; + const size_t row_index = in_slice / blocks_per_row; + const size_t kt = in_slice % blocks_per_row; + const size_t ct = row_index / 32; + const size_t row = row_index % 32; + const uint8_t * tile = (const uint8_t *) t->data + slice * matrix_size + + (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0; + + block_q2_0 block = {}; + memcpy(&block.d, tile + 512 + 2 * row, sizeof(block.d)); + for (int group = 0; group < 16; ++group) { + for (int byte = 0; byte < 4; ++byte) { + const uint8_t packed = tile[(group >> 2) * 128 + 4 * row + byte]; + const uint8_t code = (packed >> (2 * (group & 3))) & 3; + block.qs[group] |= code << (2 * byte); + } + } + + const size_t block_begin = block_index * sizeof(block_q2_0); + const size_t copy_begin = (std::max)(block_begin, offset); + const size_t copy_end = (std::min)(block_begin + sizeof(block_q2_0), offset + size); + memcpy((uint8_t *) data + copy_begin - offset, (const uint8_t *) &block + copy_begin - block_begin, copy_end - copy_begin); + } +} + +template +static void repack_float_to_f16_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size, ToFp16Fn && to_fp16) { + repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_F16, + [&](uint8_t * tile_bytes, const SrcT * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) { + uint16_t * tile_dst = (uint16_t *) tile_bytes; +#if defined(__ARM_NEON) || defined(__aarch64__) + if (ct * 32 + 32 <= ne1 && kt * 32 + 32 <= ne0) { + repack_neon_f16_tile(tile_dst, src_slice, row_stride, ct, kt); + return; + } +#endif + for (int cp = 0; cp < 16; cp++) { + const int64_t c0 = kt * 32 + cp * 2; + const int64_t c1 = c0 + 1; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + const SrcT * r_ptr = (r < ne1) ? &src_slice[r * row_stride] : nullptr; + tile_dst[cp * 64 + row * 2 + 0] = (r_ptr && c0 < ne0) ? to_fp16(r_ptr[c0]) : 0; + tile_dst[cp * 64 + row * 2 + 1] = (r_ptr && c1 < ne0) ? to_fp16(r_ptr[c1]) : 0; + } + } + }); +} + +template +static void unpack_f16_tiled_to_float(void * data, const ggml_tensor * t, size_t offset, size_t size, FromFp16Fn && from_fp16) { + unpack_from_tiled_grid_32(data, t, offset, size, HTP_TENSOR_TILE_SIZE_F16, + [&](DstT * dst_slice, const uint8_t * tile_bytes, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) { + const uint16_t * tile_src = (const uint16_t *) tile_bytes; + for (int cp = 0; cp < 16; cp++) { + const int64_t c0 = kt * 32 + cp * 2; + const int64_t c1 = c0 + 1; + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= start_row && r < end_row) { + DstT * r_ptr = dst_slice + (r - start_row) * row_stride; + if (c0 < ne0) r_ptr[c0] = from_fp16(tile_src[cp * 64 + row * 2 + 0]); + if (c1 < ne0) r_ptr[c1] = from_fp16(tile_src[cp * 64 + row * 2 + 1]); + } + } + } + }); +} + +// --- Generic K-Quant (super-block 256) grid walkers --- + +template +static void repack_kquant_to_tiled_grid(ggml_tensor * t, const void * data, size_t offset, size_t size, + size_t tile_size, PackKtFn && pack_kt) { + GGML_ASSERT(offset == 0); + GGML_UNUSED(size); + + const BlockT * src_matrix = (const BlockT *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + GGML_ASSERT(ne0 % QK_K == 0); + + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = hex_round_up(ne0, 32) / 32; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const int64_t sb_per_row = ne0 / QK_K; + + for (int64_t slice_idx = 0; slice_idx < ne2 * ne3; slice_idx++) { + const BlockT * src_slice = src_matrix + slice_idx * (ne1 * sb_per_row); + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; + const size_t ct_bytes = (size_t) n_k_tiles * tile_size; + + repack_parallel_for_ct(matrix_size, n_col_tiles, [&](int ct0, int ct1) { + // Padding rows and OR-ed bit planes require zeroed tiles + memset(matrix_dst + (size_t) ct0 * ct_bytes, 0, (size_t) (ct1 - ct0) * ct_bytes); + + const int64_t r0 = (int64_t) ct0 * 32; + const int64_t r1 = (std::min)((int64_t) ct1 * 32, ne1); + for (int64_t r = r0; r < r1; r++) { + const int ct = (int) (r / 32); + const int row = (int) (r % 32); + const BlockT * src_row = src_slice + r * sb_per_row; + + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + pack_kt(tile, &src_row[kt / 8], row, kt % 8); + } + } + }); + } +} + +template +static void unpack_kquant_from_tiled_grid(void * data, const ggml_tensor * t, size_t offset, size_t size, + size_t tile_size, UnpackSbFn && unpack_sb) { + uint8_t * dst_bytes = (uint8_t *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + GGML_ASSERT(ne0 % QK_K == 0); + + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = hex_round_up(ne0, 32) / 32; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const int64_t sb_per_row = ne0 / QK_K; + + const size_t row_size_bytes = ggml_row_size(t->type, ne0); + const size_t slice_size = (size_t) ne1 * row_size_bytes; + const int64_t start_slice = offset / slice_size; + int64_t end_slice = (offset + size + slice_size - 1) / slice_size; + if (end_slice > ne2 * ne3) { + end_slice = ne2 * ne3; + } + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + const size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size; + const size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size; + + const int64_t start_row = slice_offset_start / row_size_bytes; + const int64_t end_row = (std::min)((int64_t) ((slice_offset_end + row_size_bytes - 1) / row_size_bytes), ne1); + + BlockT * dst_slice = (BlockT *) (dst_bytes + (cur_start_byte - offset)); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int64_t r = start_row; r < end_row; r++) { + const int ct = (int) (r / 32); + const int row = (int) (r % 32); + BlockT * dst_row = dst_slice + (r - start_row) * sb_per_row; + + for (int64_t sb = 0; sb < sb_per_row; sb++) { + BlockT * b = &dst_row[sb]; + memset(b, 0, sizeof(BlockT)); + const uint8_t * sb_tiles = matrix_src + ((size_t) ct * n_k_tiles + sb * 8) * tile_size; + unpack_sb(b, sb_tiles, tile_size, row); + } + } + } +} + +// --- K-Quant Helpers & Per-Type Implementations (Q2_K, Q3_K, Q4_K, Q5_K, Q6_K) --- + +#if !defined(__ARM_NEON) && !defined(__aarch64__) +// unsigned 6-bit value (0..63) of element e of a Q6_K block, same bit layout as dequantize_row_q6_K +static inline uint8_t q6_K_get_quant(const block_q6_K * b, int e) { + const int c = e / 128; + const int w = e % 128; + const int g = w / 32; + const int l = w % 32; + const uint8_t * ql = b->ql + c * 64; + const uint8_t * qh = b->qh + c * 32; + uint8_t lo, hi; + switch (g) { + case 0: lo = ql[l] & 0xF; hi = (qh[l] >> 0) & 3; break; + case 1: lo = ql[l + 32] & 0xF; hi = (qh[l] >> 2) & 3; break; + case 2: lo = ql[l] >> 4; hi = (qh[l] >> 4) & 3; break; + default: lo = ql[l + 32] >> 4; hi = (qh[l] >> 6) & 3; break; + } + return (uint8_t) (lo | (hi << 4)); +} +#endif + +static void repack_q6_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_kquant_to_tiled_grid(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q6_K, + [](uint8_t * tile, const block_q6_K * b, int row, int kt_local) { + const float d = GGML_FP16_TO_FP32(b->d); + uint8_t * lo_pl = tile; + uint8_t * hi_pl = tile + 512; + ggml_half * sc_pl = (ggml_half *) (tile + 768); + +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q6_k(b, kt_local, lo_pl, hi_pl, row); +#else + for (int lk = 0; lk < 32; lk++) { + const uint8_t q6 = q6_K_get_quant(b, kt_local * 32 + lk); + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + lo_pl[(g >> 1) * 128 + pos] |= (uint8_t) ((q6 & 0xF) << ((g & 1) * 4)); + hi_pl[(g >> 2) * 128 + pos] |= (uint8_t) ((q6 >> 4) << ((g & 3) * 2)); + } +#endif + for (int sub = 0; sub < 2; sub++) { + sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) b->scales[kt_local * 2 + sub]); + } + }); +} + +static void repack_tiled_q6_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q6_K, + [](block_q6_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + float sub_scales[16]; + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile = sb_tiles + kt_local * tile_size; + const uint8_t * lo_pl = tile; + const uint8_t * hi_pl = tile + 512; + const ggml_half * sc_pl = (const ggml_half *) (tile + 768); + + const int c = kt_local / 4; + const int g = kt_local % 4; + uint8_t * ql = b->ql + c * 64; + uint8_t * qh = b->qh + c * 32; + + for (int lk = 0; lk < 32; lk++) { + const int g_tile = lk >> 2; + const int pos = row * 4 + (lk & 3); + const uint8_t lo = (lo_pl[(g_tile >> 1) * 128 + pos] >> ((g_tile & 1) * 4)) & 0xF; + const uint8_t hi = (hi_pl[(g_tile >> 2) * 128 + pos] >> ((g_tile & 3) * 2)) & 3; + + switch (g) { + case 0: ql[lk] |= lo; qh[lk] |= (hi << 0); break; + case 1: ql[lk + 32] |= lo; qh[lk] |= (hi << 2); break; + case 2: ql[lk] |= (lo << 4); qh[lk] |= (hi << 4); break; + default: ql[lk + 32] |= (lo << 4); qh[lk] |= (hi << 6); break; + } + } + + for (int sub = 0; sub < 2; sub++) { + sub_scales[kt_local * 2 + sub] = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]); + } + } + + float max_abs_scale = 0.0f; + for (int s = 0; s < 16; s++) { + max_abs_scale = (std::max)(max_abs_scale, fabsf(sub_scales[s])); + } + + if (max_abs_scale == 0.0f) { + b->d = GGML_FP32_TO_FP16(0.0f); + } else { + b->d = GGML_FP32_TO_FP16(max_abs_scale / 127.0f); + const float d_actual = GGML_FP16_TO_FP32(b->d); + const float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f; + for (int s = 0; s < 16; s++) { + const int sc = (int) roundf(sub_scales[s] * inv_d); + b->scales[s] = (int8_t) (std::max)(-128, (std::min)(127, sc)); + } + } + }); +} + +#if !defined(__ARM_NEON) && !defined(__aarch64__) +// low 2 bits (0..3) of element e of a Q2_K or Q3_K block, same bit layout as dequantize_row_q2_K / q3_K +static inline uint8_t q2_3_K_get_low2(const uint8_t * qs, int e) { + const int c = e / 128; + const int j = (e % 128) / 32; + const int l = e % 32; + return (qs[c * 32 + l] >> (2 * j)) & 3; +} + +// hmask bit of element e of a Q3_K block, same bit layout as dequantize_row_q3_K +static inline bool q3_K_get_hbit(const block_q3_K * b, int e) { + const int c = e / 128; + const int j = (e % 128) / 32; + const int l = e % 32; + return (b->hmask[l] >> (c * 4 + j)) & 1; +} +#endif + +// signed 6-bit scale j (-32..31) of a Q3_K block, same packing as quantize_row_q3_K_ref +static inline int q3_K_get_scale(const uint8_t * scales, int j) { + const int lo = (j < 8) ? (scales[j] & 0xF) : (scales[j - 8] >> 4); + const int hi = (scales[8 + j % 4] >> (2 * (j / 4))) & 3; + return (lo | (hi << 4)) - 32; +} + +// read-back: find fp16 d and l[j] in [lmin, lmax] with fp16(d * l[j]) == prod[j] for all j, false if none +static bool hexagon_recover_k_scales(const ggml_half * prod, int n, int lmin, int lmax, ggml_half * d_out, int * l_out) { + int jmax = 0; + for (int j = 1; j < n; j++) { + if (fabsf(GGML_FP16_TO_FP32(prod[j])) > fabsf(GGML_FP16_TO_FP32(prod[jmax]))) { + jmax = j; + } + } + const float pmax = GGML_FP16_TO_FP32(prod[jmax]); + if (pmax == 0.0f) { + *d_out = GGML_FP32_TO_FP16(0.0f); + for (int j = 0; j < n; j++) { + l_out[j] = 0; + } + return true; + } + + // the quantizers put the largest scale at or near the range end, so try large |l| first + const int lext = (std::max)(-lmin, lmax); + for (int a = lext; a >= 1; a--) { + for (int sign : { 1, -1 }) { + const int lj = sign * a; + if (lj < lmin || lj > lmax) { + continue; + } + const ggml_half d0 = GGML_FP32_TO_FP16(pmax / (float) lj); + for (int ulp : { 0, -1, 1 }) { + ggml_half d = d0; + uint16_t bits; + memcpy(&bits, &d, sizeof(bits)); + bits = (uint16_t) (bits + ulp); + memcpy(&d, &bits, sizeof(bits)); + + const float df = GGML_FP16_TO_FP32(d); + if (!std::isfinite(df) || df == 0.0f) { + continue; + } + bool ok = true; + for (int j = 0; j < n && ok; j++) { + const int l = (int) roundf(GGML_FP16_TO_FP32(prod[j]) / df); + const ggml_half p = GGML_FP32_TO_FP16(df * (float) l); + ok = l >= lmin && l <= lmax && memcmp(&p, &prod[j], sizeof(p)) == 0; + l_out[j] = l; + } + if (ok) { + *d_out = d; + return true; + } + } + } + } + return false; +} + +static void repack_q3_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_kquant_to_tiled_grid(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q3_K, + [](uint8_t * tile, const block_q3_K * b, int row, int kt_local) { + const float d = GGML_FP16_TO_FP32(b->d); + uint8_t * lo_pl = tile; + uint8_t * neg_pl = tile + 256; + ggml_half * sc_pl = (ggml_half *) (tile + 384); + +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q2_3_k_low2(b->qs, kt_local, lo_pl, row); + repack_neon_q3_k_hmask(b->hmask, kt_local, neg_pl, row); +#else + for (int lk = 0; lk < 32; lk++) { + const int e = kt_local * 32 + lk; + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, e) << ((g & 3) * 2)); + if (!q3_K_get_hbit(b, e)) { + neg_pl[pos] |= (uint8_t) (1 << g); + } + } +#endif + for (int sub = 0; sub < 2; sub++) { + sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) q3_K_get_scale(b->scales, kt_local * 2 + sub)); + } + }); +} + +static void repack_tiled_q3_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q3_K, + [](block_q3_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + ggml_half sub_scales[16]; + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile = sb_tiles + kt_local * tile_size; + const uint8_t * lo_pl = tile; + const uint8_t * neg_pl = tile + 256; + const ggml_half * sc_pl = (const ggml_half *) (tile + 384); + + for (int lk = 0; lk < 32; lk++) { + const int e = kt_local * 32 + lk; + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + const uint8_t lo = (lo_pl[(g >> 2) * 128 + pos] >> ((g & 3) * 2)) & 3; + + const int c = e / 128; + const int j = (e % 128) / 32; + const int l = e % 32; + b->qs[c * 32 + l] |= (uint8_t) (lo << (2 * j)); + if (!((neg_pl[pos] >> g) & 1)) { + b->hmask[l] |= (uint8_t) (1 << (c * 4 + j)); + } + } + + for (int sub = 0; sub < 2; sub++) { + sub_scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row]; + } + } + + int ls[16]; + if (!hexagon_recover_k_scales(sub_scales, 16, -32, 31, &b->d, ls)) { + float max_scale = 0.0f; + for (int s = 0; s < 16; s++) { + if (fabsf(GGML_FP16_TO_FP32(sub_scales[s])) > fabsf(max_scale)) { + max_scale = GGML_FP16_TO_FP32(sub_scales[s]); + } + } + b->d = GGML_FP32_TO_FP16(-max_scale / 32.0f); + const float d_actual = GGML_FP16_TO_FP32(b->d); + const float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f; + for (int s = 0; s < 16; s++) { + ls[s] = (std::max)(-32, (std::min)(31, (int) roundf(GGML_FP16_TO_FP32(sub_scales[s]) * inv_d))); + } + } + + for (int s = 0; s < 16; s++) { + const int l = ls[s] + 32; + if (s < 8) { + b->scales[s] = l & 0xF; + } else { + b->scales[s - 8] |= (uint8_t) ((l & 0xF) << 4); + } + b->scales[s % 4 + 8] |= (uint8_t) ((l >> 4) << (2 * (s / 4))); + } + }); +} + +static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + repack_kquant_to_tiled_grid(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q2_K, + [](uint8_t * tile, const block_q2_K * b, int row, int kt_local) { + uint8_t * lo_pl = tile; + uint8_t * sc_pl = tile + 256; + ggml_half * dd_pl = (ggml_half *) (tile + 320); + +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q2_3_k_low2(b->qs, kt_local, lo_pl, row); +#else + for (int lk = 0; lk < 32; lk++) { + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, kt_local * 32 + lk) << ((g & 3) * 2)); + } +#endif + for (int sub = 0; sub < 2; sub++) { + sc_pl[sub * 32 + row] = b->scales[kt_local * 2 + sub]; + } + if (row / 8 == kt_local % 4) { + dd_pl[row % 8] = kt_local < 4 ? b->d : (ggml_half) (b->dmin ^ 0x8000); + } + }); +} + +static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q2_K, + [](block_q2_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile = sb_tiles + kt_local * tile_size; + const uint8_t * lo_pl = tile; + const uint8_t * sc_pl = tile + 256; + const ggml_half * dd_pl = (const ggml_half *) (tile + 320); + + for (int lk = 0; lk < 32; lk++) { + const int e = kt_local * 32 + lk; + const int g = lk >> 2; + const int pos = row * 4 + (lk & 3); + const uint8_t lo = (lo_pl[(g >> 2) * 128 + pos] >> ((g & 3) * 2)) & 3; + b->qs[(e / 128) * 32 + e % 32] |= (uint8_t) (lo << (2 * ((e % 128) / 32))); + } + + for (int sub = 0; sub < 2; sub++) { + b->scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row]; + } + if (row / 8 == kt_local % 4) { + if (kt_local < 4) { + b->d = dd_pl[row % 8]; + } else { + b->dmin = (ggml_half) (dd_pl[row % 8] ^ 0x8000); + } + } + } + }); +} + +static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j + 4] & 63; + } else { + *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); + } +} + +static inline void pack_k4_scales_mins(uint8_t * scales, ggml_half * d, ggml_half * dmin, + const float * sub_scales, const float * sub_mins) { + float max_scale = 0.0f; + float max_min = 0.0f; + for (int j = 0; j < 8; j++) { + if (sub_scales[j] > max_scale) max_scale = sub_scales[j]; + if (sub_mins[j] > max_min) max_min = sub_mins[j]; + } + + float inv_scale = 0.0f; + if (max_scale > 0.0f) { + *d = GGML_FP32_TO_FP16(max_scale / 63.0f); + const float d_actual = GGML_FP16_TO_FP32(*d); + inv_scale = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f; + } else { + *d = GGML_FP32_TO_FP16(0.0f); + } + + float inv_min = 0.0f; + if (max_min > 0.0f) { + *dmin = GGML_FP32_TO_FP16(max_min / 63.0f); + const float dmin_actual = GGML_FP16_TO_FP32(*dmin); + inv_min = (dmin_actual > 0.0f) ? (1.0f / dmin_actual) : 0.0f; + } else { + *dmin = GGML_FP32_TO_FP16(0.0f); + } + + for (int j = 0; j < 8; j++) { + uint8_t ls = (std::min)((uint8_t) 63, (uint8_t) roundf(inv_scale * sub_scales[j])); + uint8_t lm = (std::min)((uint8_t) 63, (uint8_t) roundf(inv_min * sub_mins[j])); + if (j < 4) { + scales[j] = ls; + scales[j + 4] = lm; + } else { + scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); + scales[j - 4] |= ((ls >> 4) << 6); + scales[j - 0] |= ((lm >> 4) << 6); + } + } +} + +template +static inline void repack_k4_quants(uint8_t * tile_dst, const BlockT * b, int row, int kt_local, + int64_t sb_per_row, bool use_neon) { +#if defined(__ARM_NEON) || defined(__aarch64__) + if (use_neon) { + if ((row & 7) == 0) { + const uint8_t * qs[8]; + for (int rr = 0; rr < 8; ++rr) { + qs[rr] = b[rr * sb_per_row].qs + (kt_local / 2) * 32; + } + repack_neon_k4_rows(qs, tile_dst, row, (kt_local & 1) != 0); + } + return; + } +#else + GGML_UNUSED(sb_per_row); + GGML_UNUSED(use_neon); +#endif + const uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; + const int shift = (kt_local & 1) ? 4 : 0; + + for (int cp = 0; cp < 16; cp++) { + const uint8_t q0 = (qs_sub[2 * cp + 0] >> shift) & 0x0F; + const uint8_t q1 = (qs_sub[2 * cp + 1] >> shift) & 0x0F; + tile_dst[cp * 32 + row] = (uint8_t) ((q1 << 4) | q0); + } +} + +static void repack_q4_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + const int64_t sb_per_row = t->ne[0] / QK_K; + const bool use_neon = (t->ne[1] % 8 == 0); + repack_kquant_to_tiled_grid(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_K, + [sb_per_row, use_neon](uint8_t * tile_dst, const block_q4_K * b, int row, int kt_local) { + const float d = GGML_FP16_TO_FP32(b->d); + const float dmin = GGML_FP16_TO_FP32(b->dmin); + + uint8_t sc, m; + get_scale_min_k4(kt_local, b->scales, &sc, &m); + + repack_k4_quants(tile_dst, b, row, kt_local, sb_per_row, use_neon); + + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(d * (float) sc); + scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(-dmin * (float) m); + }); +} + +static void repack_tiled_q4_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q4_K, + [](block_q4_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + float sub_scales[8]; + float sub_mins[8]; + + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile_src = sb_tiles + kt_local * tile_size; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + + uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; + const int shift = (kt_local & 1) ? 4 : 0; + + for (int cp = 0; cp < 16; cp++) { + const uint8_t val = tile_src[cp * 32 + row]; + qs_sub[2 * cp + 0] |= (uint8_t) ((val & 0x0F) << shift); + qs_sub[2 * cp + 1] |= (uint8_t) ((val >> 4) << shift); + } + + const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]); + const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]); + sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f; + sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f; + } + + pack_k4_scales_mins(b->scales, &b->d, &b->dmin, sub_scales, sub_mins); + }); +} + +static void repack_q5_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + const int64_t sb_per_row = t->ne[0] / QK_K; + const bool use_neon = (t->ne[1] % 8 == 0); + repack_kquant_to_tiled_grid(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q5_K, + [sb_per_row, use_neon](uint8_t * tile_dst, const block_q5_K * b, int row, int kt_local) { + const float d = GGML_FP16_TO_FP32(b->d); + const float dmin = GGML_FP16_TO_FP32(b->dmin); + uint8_t * plane = tile_dst + 640; + + uint8_t sc, m; + get_scale_min_k4(kt_local, b->scales, &sc, &m); + + repack_k4_quants(tile_dst, b, row, kt_local, sb_per_row, use_neon); + + const uint8_t hbit = (uint8_t) (1 << kt_local); +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q5_k_qh(b->qh, hbit, plane, row); +#else + for (int cp = 0; cp < 16; cp++) { + const int i = cp / 4; + const int lane = (cp % 4) * 32 + row; + if (b->qh[2 * cp + 0] & hbit) { + plane[lane] |= (uint8_t) (1 << (2 * i)); + } + if (b->qh[2 * cp + 1] & hbit) { + plane[lane] |= (uint8_t) (1 << (2 * i + 1)); + } + } +#endif + + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(d * (float) sc); + scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(-dmin * (float) m); + }); +} + +static void repack_tiled_q5_K(void * data, const ggml_tensor * t, size_t offset, size_t size) { + unpack_kquant_from_tiled_grid(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q5_K, + [](block_q5_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) { + float sub_scales[8]; + float sub_mins[8]; + + for (int kt_local = 0; kt_local < 8; kt_local++) { + const uint8_t * tile_src = sb_tiles + kt_local * tile_size; + const uint8_t * plane = tile_src + 640; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + + uint8_t * qs_sub = b->qs + (kt_local / 2) * 32; + const int shift = (kt_local & 1) ? 4 : 0; + const uint8_t hbit = (uint8_t) (1 << kt_local); + + for (int cp = 0; cp < 16; cp++) { + const uint8_t val = tile_src[cp * 32 + row]; + qs_sub[2 * cp + 0] |= (uint8_t) ((val & 0x0F) << shift); + qs_sub[2 * cp + 1] |= (uint8_t) ((val >> 4) << shift); + + const int i = cp / 4; + const int lane = (cp % 4) * 32 + row; + if (plane[lane] & (1 << (2 * i))) { + b->qh[2 * cp + 0] |= hbit; + } + if (plane[lane] & (1 << (2 * i + 1))) { + b->qh[2 * cp + 1] |= hbit; + } + } + + const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]); + const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]); + sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f; + sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f; + } + + pack_k4_scales_mins(b->scales, &b->d, &b->dmin, sub_scales, sub_mins); + }); +} + +static inline void q1_0_tiled_bit_location(int k, int row, int & byte_offset, int & bit) { + const int quarter = k / 32; + const int kq = k % 32; + + byte_offset = quarter * 128 + 4 * row + (kq % 4); + bit = kq / 4; +} + +static void repack_q1_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + const block_q1_0 * src_matrix = (const block_q1_0 *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = ne0 / QK1_0; + const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + const int64_t start_slice = offset / slice_size; + const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3); + + GGML_ASSERT(ne0 % QK1_0 == 0); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const block_q1_0 * src_slice = src_matrix + (slice_idx - start_slice) * ne1 * n_k_tiles; + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; + + repack_parallel_for_ct(matrix_size, n_col_tiles, [&](int ct0, int ct1) { + for (int ct = ct0; ct < ct1; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + memset(tile_dst, 0, tile_size); + + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= ne1) { + continue; + } + + const block_q1_0 & block = src_slice[r * n_k_tiles + kt]; + scale_dst[row] = block.d; +#if defined(__ARM_NEON) || defined(__aarch64__) + repack_neon_q1_0_block(block.qs, tile_dst, row); +#else + for (int k = 0; k < QK1_0; k++) { + int byte_offset; + int bit; + q1_0_tiled_bit_location(k, row, byte_offset, bit); + if (block.qs[k / 8] & (1u << (k % 8))) { + tile_dst[byte_offset] |= (uint8_t) (1u << bit); + } + } +#endif + } + } + } + }); + } +} + +static void repack_tiled_q1_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + block_q1_0 * dst_matrix = (block_q1_0 *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = ne0 / QK1_0; + const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const size_t row_size_bytes = ggml_row_size(t->type, ne0); + const size_t slice_size = ne1 * row_size_bytes; + const int64_t start_slice = offset / slice_size; + const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3); + + GGML_ASSERT(ne0 % QK1_0 == 0); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + const int64_t start_row = (cur_start_byte - (size_t) slice_idx * slice_size) / row_size_bytes; + const int64_t end_row = (std::min)((int64_t) ((cur_end_byte - (size_t) slice_idx * slice_size + row_size_bytes - 1) / row_size_bytes), ne1); + block_q1_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q1_0); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = start_row / 32; ct < (end_row + 31) / 32; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < start_row || r >= end_row) { + continue; + } + + block_q1_0 & block = dst_slice[(r - start_row) * n_k_tiles + kt]; + memset(block.qs, 0, sizeof(block.qs)); + block.d = scale_src[row]; + for (int k = 0; k < QK1_0; k++) { + int byte_offset; + int bit; + q1_0_tiled_bit_location(k, row, byte_offset, bit); + if (tile_src[byte_offset] & (1u << bit)) { + block.qs[k / 8] |= (uint8_t) (1u << (k % 8)); + } + } + } + } + } + } +} + +// --- Top-level Repack / Unpack Dispatchers --- + +void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) { + switch (tensor->type) { + case GGML_TYPE_Q1_0: + repack_q1_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q4_0: + case GGML_TYPE_IQ4_NL: + repack_q4_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q4_1: + repack_q4_1_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q8_0: + repack_q8_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_MXFP4: + repack_mxfp4_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q2_0: + repack_q2_0_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q2_K: + repack_q2_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q3_K: + repack_q3_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q4_K: + repack_q4_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q5_K: + repack_q5_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_Q6_K: + repack_q6_K_tiled(tensor, data, 0, size); + break; + case GGML_TYPE_F16: + repack_float_to_f16_tiled(tensor, data, 0, size, + [](uint16_t v) { return v; }); + break; + case GGML_TYPE_F32: + repack_float_to_f16_tiled(tensor, data, 0, size, + [](float v) { return (uint16_t) GGML_FP32_TO_FP16(v); }); + break; + case GGML_TYPE_BF16: + repack_float_to_f16_tiled(tensor, data, 0, size, + [](ggml_bf16_t v) { return (uint16_t) GGML_FP32_TO_FP16(GGML_BF16_TO_FP32(v)); }); + break; + default: + break; + } +} + +void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t offset, size_t size) { + GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); + + switch (tensor->type) { + case GGML_TYPE_Q1_0: + repack_tiled_q1_0(data, tensor, offset, size); + break; + case GGML_TYPE_Q4_0: + case GGML_TYPE_IQ4_NL: + repack_tiled_q4_0(data, tensor, offset, size); + break; + case GGML_TYPE_Q4_1: + repack_tiled_q4_1(data, tensor, offset, size); + break; + case GGML_TYPE_Q8_0: + repack_tiled_q8_0(data, tensor, offset, size); + break; + case GGML_TYPE_MXFP4: + repack_tiled_mxfp4(data, tensor, offset, size); + break; + case GGML_TYPE_Q2_0: + repack_tiled_q2_0(data, tensor, offset, size); + break; + case GGML_TYPE_Q2_K: + repack_tiled_q2_K(data, tensor, offset, size); + break; + case GGML_TYPE_Q3_K: + repack_tiled_q3_K(data, tensor, offset, size); + break; + case GGML_TYPE_Q4_K: + repack_tiled_q4_K(data, tensor, offset, size); + break; + case GGML_TYPE_Q5_K: + repack_tiled_q5_K(data, tensor, offset, size); + break; + case GGML_TYPE_Q6_K: + repack_tiled_q6_K(data, tensor, offset, size); + break; + case GGML_TYPE_F16: + unpack_f16_tiled_to_float(data, tensor, offset, size, + [](uint16_t v) { return v; }); + break; + case GGML_TYPE_F32: + unpack_f16_tiled_to_float(data, tensor, offset, size, + [](uint16_t v) { return ggml_fp16_to_fp32((ggml_fp16_t) v); }); + break; + case GGML_TYPE_BF16: + unpack_f16_tiled_to_float(data, tensor, offset, size, + [](uint16_t v) { return ggml_fp32_to_bf16(ggml_fp16_to_fp32((ggml_fp16_t) v)); }); + break; + default: + memcpy(data, (const char *) tensor->data + offset, size); + break; + } +} diff --git a/ggml/src/ggml-hexagon/htp-repack.h b/ggml/src/ggml-hexagon/htp-repack.h new file mode 100644 index 0000000..f656e9b --- /dev/null +++ b/ggml/src/ggml-hexagon/htp-repack.h @@ -0,0 +1,69 @@ +#ifndef HTP_REPACK_H +#define HTP_REPACK_H + +#include +#include + +#include "ggml.h" +#include "htp/htp-tensor.h" + +static inline bool htp_is_tiled_weight_type(enum ggml_type type) { + return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || + type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL || + type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q1_0 || + type == GGML_TYPE_Q6_K || + type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || + type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K || + type == GGML_TYPE_Q2_0 || + type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || + type == GGML_TYPE_BF16; +} + +static inline bool htp_is_tiled_weight_tensor(const struct ggml_tensor * t) { + if (!ggml_is_contiguous(t) || ggml_is_view(t)) { + return false; + } + if (!htp_is_tiled_weight_type(t->type)) { + return false; + } + if (t->ne[0] < 32) { + return false; + } + if (!ggml_is_quantized(t->type)) { + return (t->ne[0] % 32 == 0) && (t->ne[1] % 32 == 0) && (t->ne[2] == 1) && (t->ne[3] == 1); + } + return t->ne[3] == 1; +} + +// Size of one tiled row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins, +// so they are larger than the ggml blocks. For the other tiled types (Q2_K included) the tile has the same size. +static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) { + if (type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || type == GGML_TYPE_BF16) { + return (size_t) ne0 * sizeof(ggml_fp16_t); + } + if (type == GGML_TYPE_Q6_K) { + return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q6_K / 32); + } + if (type == GGML_TYPE_Q2_0) { + return (size_t) (ne0 / 64) * 18; + } + if (type == GGML_TYPE_Q4_K) { + return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q4_K / 32); + } + if (type == GGML_TYPE_Q5_K) { + return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q5_K / 32); + } + if (type == GGML_TYPE_Q3_K) { + return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q3_K / 32); + } + return ggml_row_size(type, ne0); +} + +static inline bool htp_is_hmx_weight_type(enum ggml_type type) { + return htp_is_tiled_weight_type(type); +} + +void htp_repack_tensor_tiled(struct ggml_tensor * t, const void * data, size_t size); +void htp_unpack_tensor_tiled(void * data, const struct ggml_tensor * t, size_t offset, size_t size); + +#endif // HTP_REPACK_H diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index 9ca7594..2edd653 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -17,51 +17,53 @@ set(HTP_LIB ggml-htp-${DSP_VERSION}) add_library(${HTP_LIB} SHARED main.c htp_iface_skel.c - worker-pool.c - hex-dma.c + work-queue.c + dma-queue.c + hmx-queue.c + htp-dcache.c matmul-ops.c + flash-attn-ops.c + gated-delta-net-ops.c binary-ops.c unary-ops.c sum-rows-ops.c softmax-ops.c - act-ops.c + glu-ops.c rope-ops.c - flash-attn-ops.c set-rows-ops.c get-rows-ops.c cpy-ops.c + gather-rows.c repeat-ops.c - argsort-ops.c ssm-conv.c cumsum-ops.c + fill-ops.c + concat-ops.c + diag-ops.c + solve-tri-ops.c + pad-ops.c + argsort-ops.c + im2col-ops.c + roll-ops.c + pool-ops.c + allreduce-ops.c ) target_compile_definitions(${HTP_LIB} PRIVATE $,HTP_DEBUG=1,NDEBUG=1> - $,FARF_HIGH=1,> - FP32_QUANTIZE_GROUP_SIZE=${GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE}) - -# HMX acceleration: available on v73+ architectures -set(HTP_HMX_VERSIONS v73 v75 v79 v81) -list(FIND HTP_HMX_VERSIONS ${DSP_VERSION} _hmx_idx) + $,FARF_HIGH=1,>) -if (_hmx_idx GREATER_EQUAL 0) - target_sources(${HTP_LIB} PRIVATE - hmx-queue.c - hmx-matmul-ops.c - ) +build_idl(htp_iface.idl ${HTP_LIB}) - # -mhmx enables HMX instruction set (needed by files that include hmx-utils.h) - set_source_files_properties( - hmx-matmul-ops.c - PROPERTIES COMPILE_OPTIONS "-mhmx" - ) +set_target_properties(${HTP_LIB} PROPERTIES EXPORT_COMPILE_COMMANDS ON) - target_compile_definitions(${HTP_LIB} PRIVATE HTP_HAS_HMX=1) +if (NOT Python3_EXECUTABLE) + find_package(Python3 REQUIRED COMPONENTS Interpreter) endif() -build_idl(htp_iface.idl ${HTP_LIB}) - -set_target_properties(${HTP_LIB} PROPERTIES EXPORT_COMPILE_COMMANDS ON) +add_custom_command(TARGET ${HTP_LIB} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E echo "Adding SWIV section $" + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/htp-swiv.py -i $ -o $ + VERBATIM) install(TARGETS ${HTP_LIB}) diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c deleted file mode 100644 index 6416d2d..0000000 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ /dev/null @@ -1,782 +0,0 @@ -#pragma clang diagnostic ignored "-Wunused-variable" -#pragma clang diagnostic ignored "-Wunused-function" -#pragma clang diagnostic ignored "-Wunused-but-set-variable" - -#include -#include - -#include -#include - -#include "hex-dma.h" -#include "hvx-utils.h" - -#define GGML_COMMON_DECL_C -#include "ggml-common.h" -#include "htp-ctx.h" -#include "htp-ops.h" -#include "htp-ops.h" - -#define htp_act_preamble \ - const struct htp_tensor * src0 = actx->octx->src[0]; \ - const struct htp_tensor * src1 = actx->octx->src[1]; \ - const struct htp_tensor * dst = actx->octx->dst; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t ne10 = src1 ? src1->ne[0] : 0; \ - const uint32_t ne11 = src1 ? src1->ne[1] : 0; \ - const uint32_t ne12 = src1 ? src1->ne[2] : 0; \ - const uint32_t ne13 = src1 ? src1->ne[3] : 0; \ - \ - const uint32_t nb10 = src1 ? src1->nb[0] : 0; \ - const uint32_t nb11 = src1 ? src1->nb[1] : 0; \ - const uint32_t nb12 = src1 ? src1->nb[2] : 0; \ - const uint32_t nb13 = src1 ? src1->nb[3] : 0; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; - -struct htp_act_context { - struct htp_ops_context * octx; - - // Precomputed values - const uint8_t * data_src0; - const uint8_t * data_src1; - uint8_t * data_dst; - - size_t src0_row_size; - size_t src1_row_size; - size_t dst_row_size; - - size_t src0_row_size_aligned; - size_t src1_row_size_aligned; - size_t dst_row_size_aligned; - - size_t src0_spad_half_size; - size_t src1_spad_half_size; - size_t dst_spad_half_size; - - uint32_t block; - uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; - int nc; -}; - -static void glu_swiglu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; - - size_t src0_row_size = actx->src0_row_size; - size_t src1_row_size = actx->src1_row_size; - size_t dst_row_size = actx->dst_row_size; - - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } - - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint8_t * restrict data_src0 = actx->data_src0; - const uint8_t * restrict data_src1 = actx->data_src1; - uint8_t * restrict data_dst = actx->data_dst; - - const int nc = actx->nc; - - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread); - uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t src1_spad_half_size = actx->src1_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - const int BLOCK = actx->block; - if (BLOCK == 0) { - FARF(ERROR, - "swiglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; - - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)), - src1_row_size_aligned, src1_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; - float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float)); - const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float)); - float * dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float)); - - //swiglu(x) = x1 * sigmoid(x0) - hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, nc); - hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, - (const uint8_t *) src1_spad_ptr, nc); - } - - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size, - dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)), - src1_row_size_aligned, src1_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "swiglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); -} - -static void glu_swiglu_oai_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; - - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - size_t src0_row_size = actx->src0_row_size; - size_t src1_row_size = actx->src1_row_size; - size_t dst_row_size = actx->dst_row_size; - - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } - - const uint8_t * restrict data_src0 = actx->data_src0; - const uint8_t * restrict data_src1 = actx->data_src1; - uint8_t * restrict data_dst = actx->data_dst; - - const int nc = actx->nc; - - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread); - uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t src1_spad_half_size = actx->src1_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - const int BLOCK = actx->block; - if (BLOCK == 0) { - FARF(ERROR, - "swiglu-oai-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least " - "%zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; - } - const float alpha = ((const float *) (actx->octx->op_params))[2]; - const float limit = ((const float *) (actx->octx->op_params))[3]; - - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; - - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm( - dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - dma_queue_push_ddr_to_vtcm( - dma_queue, - dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)), - src1_row_size_aligned, src1_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; - float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float)); - const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float)); - float * dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float)); - - // x (src0_spad_data) = std::min(src0_p[k], limit); - hvx_min_scalar_f32((uint8_t *) src0_spad_ptr, (const uint8_t *) src0_spad_ptr, limit, nc); - // y1 (src1_spad_data) = std::clamp(src1_p[k], -limit, limit); - hvx_clamp_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, -limit, limit, nc); - // y (src1_spad_data) = y1 + 1.f - hvx_add_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, 1.0, nc); - // x1 (dst_spad_data) = alpha * (x) - hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, alpha, nc); - // x2 (dst_spad_data) = sigmoid(x1) = 1/(1+exp(-x1)) - hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc); - // out = x * sigmoid(alpha * x) * (y + 1.f) - hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, - (const uint8_t *) src1_spad_ptr, nc); - } - - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size, - dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)), - src1_row_size_aligned, src1_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "swiglu-oai-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, src0->ne[0], - src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); -} - - -static void unary_gelu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; - - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const size_t src0_row_size = actx->src0_row_size; - const size_t dst_row_size = actx->dst_row_size; - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } - - const uint8_t * data_src0 = actx->data_src0; - uint8_t * data_dst = actx->data_dst; - - // nc/ne0 matches. - const int ne0_val = actx->nc; // == dst->ne[0] - - uint8_t * src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - // In gelu = x*sigmoid(x*1.702) - const int BLOCK = actx->block; - - if (BLOCK == 0) { - FARF(ERROR, "gelu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; - - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float* dst_spad = (float *) dma_queue_pop(dma_queue).src; - float* src0_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float)); - float* dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float)); - - // gelu = x * sigmoid(1.702 * x) // current implementation - hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (float) 1.702, ne0_val); - hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val); - hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val); - } - - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), - dst_row_size, dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "gelu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, ne00, ne01, ne02, - ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); -} - - -static void unary_silu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; - - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const size_t src0_row_size = actx->src0_row_size; - const size_t dst_row_size = actx->dst_row_size; - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } - - const uint8_t * data_src0 = actx->data_src0; - uint8_t * data_dst = actx->data_dst; - - const int ne0_val = actx->nc; // == dst->ne[0] - - uint8_t * src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - const int BLOCK = actx->block; - - if (BLOCK == 0) { - FARF(ERROR, "silu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; - - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float* dst_spad = (float *) dma_queue_pop(dma_queue).src; - float* src0_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float)); - float* dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float)); - - // silu = x * sigmoid(x) - hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, ne0_val); - hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val); - } - - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), - dst_row_size, dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "silu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, ne00, ne01, ne02, - ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); -} - -static const float GELU_COEF_A = 0.044715f; -static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; - -static void glu_geglu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; - - size_t src0_row_size = actx->src0_row_size; - size_t src1_row_size = actx->src1_row_size; - size_t dst_row_size = actx->dst_row_size; - - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } - - const uint8_t * restrict data_src0 = actx->data_src0; - const uint8_t * restrict data_src1 = actx->data_src1; - uint8_t * restrict data_dst = actx->data_dst; - - const int nc = actx->nc; - - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread); - uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t src1_spad_half_size = actx->src1_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - const int BLOCK = actx->block; - if (BLOCK == 0) { - FARF(ERROR, - "geglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; - - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)), - src1_row_size_aligned, src1_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; - float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const uint8_t * src0_spad_ptr = (const uint8_t *)(src0_spad + ib * (src0_row_size_aligned / sizeof(float))); - const uint8_t * src1_spad_ptr = (const uint8_t *)(src1_spad + ib * (src1_row_size_aligned / sizeof(float))); - uint8_t * dst_spad_ptr = (uint8_t *)(dst_spad + ib * (dst_row_size_aligned / sizeof(float))); - - // geglu tanh implementation - // geglu(x, g) = gelu(x) * g - // gelu(x) = 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x))) - hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, src0_spad_ptr, nc); // res = x*x - hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, GELU_COEF_A, nc); // res = res * GELU_COEF_A - hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 1.0f, nc); // res = res + 1.0f - hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc); // res = res * x - hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, SQRT_2_OVER_PI, nc); // res = result * SQRT_2_OVER_PI - hvx_tanh_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc); // res = tanh(res) - hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, 1.0f, nc); // res = res + 1.0f - hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc); // res = res * x - hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 0.5f, nc); // res = res + 0.5f - hvx_mul_f32_aaa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, src1_spad_ptr, nc); // res = res * g - } - - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size, - dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)), - src1_row_size_aligned, src1_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "geglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); -} - -static int execute_op_activations_f32(struct htp_ops_context * octx) { - const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * src1 = octx->src[1]; - const struct htp_tensor * dst = octx->dst; - - if (((src0->ne[0] * SIZEOF_FP32) != src0->nb[1]) || ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1])) { - FARF(ERROR, "Non-contiguous tensors are not supported at this time \n"); - return HTP_STATUS_NO_SUPPORT; - } - - worker_callback_t act_op_func; - const char * op_type = NULL; - - switch (octx->op) { - case HTP_OP_UNARY_SILU: - act_op_func = (worker_callback_t)unary_silu_f32_per_thread; - op_type = "silu-f32"; - break; - - case HTP_OP_GLU_SWIGLU: - act_op_func = (worker_callback_t)glu_swiglu_f32_per_thread; - op_type = "swiglu-f32"; - break; - - case HTP_OP_GLU_SWIGLU_OAI: - act_op_func = (worker_callback_t)glu_swiglu_oai_f32_per_thread; - op_type = "swiglu-oai-f32"; - break; - case HTP_OP_UNARY_GELU: - act_op_func = (worker_callback_t)unary_gelu_f32_per_thread; - op_type = "gelu-f32"; - break; - - case HTP_OP_GLU_GEGLU: - act_op_func = (worker_callback_t)glu_geglu_f32_per_thread; - op_type = "geglu-f32"; - break; - default: - FARF(ERROR, "Unsupported activations Op %u\n", octx->op); - return HTP_STATUS_NO_SUPPORT; - } - - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - - size_t src0_row_size = src0->nb[1]; - size_t src1_row_size = src1 ? src1->nb[1] : src0->nb[1]; - size_t dst_row_size = dst->nb[1]; - - const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); - - // VTCM scratchpads for all tensors - // N rows per thread, padded to HVX vector size - size_t spad_size_per_row = (src0_row_size_aligned + src1_row_size_aligned) + dst_row_size_aligned; - size_t vtcm_row_per_thread = (octx->ctx->vtcm_size)/ (n_threads* spad_size_per_row); - - // Make sure the reserved vtcm size is sufficient - if (vtcm_row_per_thread == 0) { - FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, - spad_size_per_row * n_threads); - return HTP_STATUS_VTCM_TOO_SMALL; - } - - octx->src0_spad.size_per_thread = src0_row_size_aligned * vtcm_row_per_thread; - octx->src1_spad.size_per_thread = src1_row_size_aligned * vtcm_row_per_thread; - octx->dst_spad.size_per_thread = dst_row_size_aligned * vtcm_row_per_thread; - - octx->dst_spad.size = n_threads* octx->dst_spad.size_per_thread; - octx->src0_spad.size = n_threads* octx->src0_spad.size_per_thread; - octx->src1_spad.size = n_threads* octx->src1_spad.size_per_thread; - - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; - octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; - - octx->src0_spad.src = NULL; - octx->src1_spad.src = NULL; - octx->dst_spad.src = NULL; - - if (src1) { - FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", - op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], octx->src0_spad.size, octx->src1_spad.size, - octx->dst_spad.size); - } else { - FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", op_type, - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size); - } - - if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - return HTP_STATUS_OK; - } - - // Prepare context - struct htp_act_context actx; - actx.octx = octx; - - actx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - - actx.src0_row_size = src0_row_size; - actx.src1_row_size = src1_row_size; - actx.dst_row_size = dst_row_size; - - actx.src0_row_size_aligned = src0_row_size_aligned; - actx.src1_row_size_aligned = src1_row_size_aligned; - actx.dst_row_size_aligned = dst_row_size_aligned; - - actx.src0_spad_half_size = octx->src0_spad.size_per_thread / 2; - actx.src1_spad_half_size = octx->src1_spad.size_per_thread / 2; - actx.dst_spad_half_size = octx->dst_spad.size_per_thread / 2; - - actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned; - actx.src0_nrows = src0_nrows; - - actx.nc = dst->ne[0]; - - // Pointers and GLU logic - const uint8_t * data_src0 = (const uint8_t *) src0->data; - const uint8_t * data_src1 = src1 ? (const uint8_t *) src1->data : NULL; - - if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || octx->op == HTP_OP_GLU_SWIGLU_OAI || octx->op == HTP_OP_GLU_GEGLU)) { - const int32_t swapped = octx->op_params[1]; - data_src1 = data_src0; - actx.src1_row_size = actx.src0_row_size; - - size_t nc_in_bytes = actx.nc * SIZEOF_FP32; - if (swapped) { - data_src0 += nc_in_bytes; - } else { - data_src1 += nc_in_bytes; - } - } - - actx.data_src0 = data_src0; - actx.data_src1 = data_src1; - actx.data_dst = (uint8_t *) dst->data; - - worker_pool_run_func(octx->ctx->worker_pool, act_op_func, &actx, n_threads); - return HTP_STATUS_OK; -} - -int op_activations(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - - switch (octx->src[0]->type) { - case HTP_TYPE_F32: - err = execute_op_activations_f32(octx); - break; - - default: - err = HTP_STATUS_NO_SUPPORT; - break; - } - - return err; -} diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.c b/ggml/src/ggml-hexagon/htp/allreduce-ops.c new file mode 100644 index 0000000..7fcc4e8 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.c @@ -0,0 +1,475 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include +#include +#include +#include + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "hvx-utils.h" +#include "htp-tensor.h" +#include "dma-queue.h" +#include "hex-profile.h" +#include "allreduce-ops.h" +#include "htp-fence.h" + +struct htp_allreduce_context { + struct htp_ops_context * octx; + uint32_t n_ranks; + uint32_t n_dsts; + uint32_t nelem; + uint32_t ne0; + uint32_t ne1; + uint32_t row_size_aligned; + uint32_t rank_elem_start; + uint32_t rank_nelem; + uint32_t elems_per_thread; + uint32_t block_elems; + uint32_t vtcm_size_per_thread; + bool is_row_bcast; + uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS]; + uint8_t * dst_spad_base; + uint8_t * res_spad_base; +}; + +#define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \ +static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ + struct htp_ops_context * octx = actx->octx; \ + \ + const uint32_t n_ranks = actx->n_ranks; \ + const uint32_t n_dsts = actx->n_dsts; \ + const uint32_t block_elems = actx->block_elems; \ + \ + const uint32_t dr = actx->elems_per_thread; \ + const uint32_t ir0 = actx->rank_elem_start + dr * ith; \ + const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \ + if (ir0 >= ir1) return; \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + \ + const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \ + uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \ + uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_thread_offset) : NULL; \ + \ + const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + uint32_t ir_prefetch = ir0; \ + int spad_idx = 0; \ + \ + for (int k = 0; k < 2 && ir_prefetch < ir1; k++) { \ + uint32_t cur_elems = MIN(block_elems, ir1 - ir_prefetch); \ + size_t cur_bytes = cur_elems * sizeof(TYPE); \ + uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + dma_addr_t d_ddr = octx->dsts[d]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \ + } \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \ + const dma_addr_t s_ddr = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ + } \ + if (HAS_ADD) { \ + uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \ + const dma_addr_t r_ddr = octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(r_spad, r_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \ + } \ + ir_prefetch += cur_elems; \ + spad_idx ^= 1; \ + } \ + \ + int comp_spad_idx = 0; \ + for (uint32_t ir = ir0; ir < ir1; ) { \ + uint32_t cur_elems = MIN(block_elems, ir1 - ir); \ + size_t cur_bytes = cur_elems * sizeof(TYPE); \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + dma_queue_pop(dma_q); \ + } \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + dma_queue_pop(dma_q); \ + } \ + if (HAS_ADD) { \ + dma_queue_pop(dma_q); \ + } \ + uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \ + uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * r_spad = HAS_ADD ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ + HVX_ADD_FN(d_spad, s0_spad, s1_spad, cur_elems); \ + for (uint32_t s = 2; s < n_ranks; s++) { \ + uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + HVX_ADD_FN(d_spad, d_spad, ss_spad, cur_elems); \ + } \ + if (HAS_ADD) { \ + HVX_ADD_FN(d_spad, d_spad, r_spad, cur_elems); \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + dma_addr_t d_ddr = octx->dsts[d]->data + ir * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 1); \ + } \ + if (ir_prefetch < ir1) { \ + uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \ + size_t next_bytes = next_elems * sizeof(TYPE); \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + const dma_addr_t s_next = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(s_spad, s_next), next_bytes, next_bytes, next_bytes, 1); \ + } \ + if (HAS_ADD) { \ + uint8_t * r_spad_next = res_spad_base + comp_spad_idx * spad_half; \ + const dma_addr_t r_next = octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \ + dma_queue_push(dma_q, dma_make_data(r_spad_next, r_next), next_bytes, next_bytes, next_bytes, 1); \ + } \ + ir_prefetch += next_elems; \ + } \ + comp_spad_idx ^= 1; \ + ir += cur_elems; \ + } \ + dma_queue_flush(dma_q); \ +} + +DEFINE_ALLREDUCE_THREAD_DMA_1D(f16, __fp16, hvx_add_f16_aaa, 0) +DEFINE_ALLREDUCE_THREAD_DMA_1D(f32, float, hvx_add_f32_aaa, 0) +DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f16, __fp16, hvx_add_f16_aaa, 1) +DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f32, float, hvx_add_f32_aaa, 1) + +#define DEFINE_ALLREDUCE_THREAD_DMA_2D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD, IS_ROW_BCAST) \ +static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \ + struct htp_ops_context * octx = actx->octx; \ + \ + const uint32_t n_ranks = actx->n_ranks; \ + const uint32_t n_dsts = actx->n_dsts; \ + const uint32_t ne0 = actx->ne0; \ + const uint32_t block_rows = actx->block_elems; \ + const uint32_t row_size_aligned = actx->row_size_aligned; \ + const uint32_t row_bytes = ne0 * sizeof(TYPE); \ + \ + const uint32_t dr = actx->elems_per_thread; \ + const uint32_t r0 = actx->rank_elem_start + dr * ith; \ + const uint32_t r1 = MIN(r0 + dr, actx->rank_elem_start + actx->rank_nelem); \ + if (r0 >= r1) return; \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + \ + const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \ + uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \ + uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_thread_offset)) : NULL; \ + \ + const size_t spad_half = actx->vtcm_size_per_thread / 2; \ + uint32_t r_prefetch = r0; \ + int spad_idx = 0; \ + \ + for (int k = 0; k < 2 && r_prefetch < r1; k++) { \ + uint32_t cur_rows = MIN(block_rows, r1 - r_prefetch); \ + uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + dma_addr_t d_ddr = octx->dsts[d]->data + r_prefetch * octx->dsts[d]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \ + } \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \ + const dma_addr_t s_ddr = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \ + } \ + if (HAS_ADD && !IS_ROW_BCAST) { \ + uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \ + const dma_addr_t r_ddr = octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(r_spad, r_ddr), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, cur_rows); \ + } \ + r_prefetch += cur_rows; \ + spad_idx ^= 1; \ + } \ + \ + int comp_spad_idx = 0; \ + for (uint32_t r = r0; r < r1; ) { \ + uint32_t cur_rows = MIN(block_rows, r1 - r); \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + dma_queue_pop(dma_q); \ + } \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + dma_queue_pop(dma_q); \ + } \ + if (HAS_ADD && !IS_ROW_BCAST) { \ + dma_queue_pop(dma_q); \ + } \ + uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \ + uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ + for (uint32_t row = 0; row < cur_rows; row++) { \ + uint8_t * d_row = d_spad + row * row_size_aligned; \ + const uint8_t * s0_row = s0_spad + row * row_size_aligned; \ + const uint8_t * s1_row = s1_spad + row * row_size_aligned; \ + HVX_ADD_FN(d_row, s0_row, s1_row, ne0); \ + for (uint32_t s = 2; s < n_ranks; s++) { \ + const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half + row * row_size_aligned; \ + HVX_ADD_FN(d_row, d_row, ss_row, ne0); \ + } \ + if (HAS_ADD) { \ + const uint8_t * res_row = IS_ROW_BCAST ? res_spad_base : (r_spad + row * row_size_aligned); \ + HVX_ADD_FN(d_row, d_row, res_row, ne0); \ + } \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \ + for (uint32_t d = 0; d < n_dsts; d++) { \ + dma_addr_t d_ddr = octx->dsts[d]->data + r * octx->dsts[d]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, cur_rows); \ + } \ + if (r_prefetch < r1) { \ + uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \ + for (uint32_t s = 0; s < n_ranks; s++) { \ + uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \ + const dma_addr_t s_next = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(s_spad, s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \ + } \ + if (HAS_ADD && !IS_ROW_BCAST) { \ + uint8_t * r_spad_next = res_spad_base + comp_spad_idx * spad_half; \ + const dma_addr_t r_next = octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \ + dma_queue_push(dma_q, dma_make_data(r_spad_next, r_next), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, next_rows); \ + } \ + r_prefetch += next_rows; \ + } \ + comp_spad_idx ^= 1; \ + r += cur_rows; \ + } \ + dma_queue_flush(dma_q); \ +} + +DEFINE_ALLREDUCE_THREAD_DMA_2D(f16, __fp16, hvx_add_f16_aaa, 0, 0) +DEFINE_ALLREDUCE_THREAD_DMA_2D(f32, float, hvx_add_f32_aaa, 0, 0) +DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f16, __fp16, hvx_add_f16_aaa, 1, 0) +DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f32, float, hvx_add_f32_aaa, 1, 0) +DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f16, __fp16, hvx_add_f16_aaa, 1, 1) +DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f32, float, hvx_add_f32_aaa, 1, 1) + +static int validate_allreduce( + struct htp_ops_context * octx, + const struct htp_allreduce_kernel_params * kparams, + uint32_t n_ranks +) { + if (kparams->n_threads == 0 || (uint32_t) kparams->n_threads > octx->ctx->n_threads) { + return HTP_STATUS_INVAL_PARAMS; + } + octx->n_threads = (uint32_t) kparams->n_threads; + + if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) { + return HTP_STATUS_INVAL_PARAMS; + } + + const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); + const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count( + n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0); + const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread; + if (vtcm_size != (size_t) kparams->vtcm_size) { + return HTP_STATUS_INVAL_PARAMS; + } + if (vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_F32) { + return HTP_STATUS_NO_SUPPORT; + } + + return HTP_STATUS_OK; +} + +int op_allreduce(struct htp_ops_context * octx) { + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + + const struct htp_allreduce_kernel_params * kparams = (const struct htp_allreduce_kernel_params *) octx->kernel_params; + const struct htp_tensor * dst = octx->dst; + + const uint32_t rank = (uint32_t) kparams->rank; + const uint32_t n_ranks = (uint32_t) kparams->n_ranks; + + if (n_ranks < 2 || n_ranks > HTP_ALLREDUCE_MAX_RANKS || rank >= n_ranks) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0]; + const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1]; + + const struct htp_tensor * my_sync = octx->src[n_ranks + rank]; + atomic_uint * my_fence = (atomic_uint *) (uintptr_t) my_sync->data; + + const int status = validate_allreduce(octx, kparams, n_ranks); + if (status != HTP_STATUS_OK) { + if (status == HTP_STATUS_NO_SUPPORT) { + FARF(ERROR, "ggml-hex: allreduce unsupported type %d : rank %u\n", dst->type, rank); + } + htp_fence_write(my_fence, fence_seq_exit, status); + return status; + } + + const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD); + const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; + const int32_t mode = kparams->mode; + + // 1. Entry Barrier: Synchronize all ranks before reading + struct htp_thread_trace * tr0 = &octx->ctx->trace[0]; + htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); + + htp_fence_write(my_fence, fence_seq_entry, octx->status); + + for (uint32_t j = 0; j < n_ranks; j++) { + if (j == rank) continue; + const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; + atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data; + uint64_t spins = 0; + while (1) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - fence_seq_entry) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: allreduce entry peer %u failed with status %u\n", j, peer_status); + htp_fence_write(my_fence, fence_seq_exit, peer_status); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); + return peer_status; + } + break; + } + if (++spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n", + rank, j, peer_fence, fence_seq_entry, peer_seq); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); + return HTP_STATUS_INTERNAL_ERR; + } + hex_pause(); + } + } + asm volatile ("syncht" : : : "memory"); + + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry); + + // 2. Multi-threaded Reduction across assigned rank chunk + if (nelem > 0) { + const uint32_t n_threads = (uint32_t) kparams->n_threads; + const uint32_t block_elems = (uint32_t) kparams->block_elems; + const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread; + const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread; + + struct htp_allreduce_context actx; + actx.octx = octx; + actx.n_ranks = n_ranks; + actx.n_dsts = (uint32_t) kparams->n_dsts ? (uint32_t) kparams->n_dsts : n_ranks; + actx.nelem = nelem; + actx.ne0 = (uint32_t) kparams->ne0; + actx.ne1 = (uint32_t) kparams->ne1; + actx.row_size_aligned = (uint32_t) kparams->row_size_aligned; + actx.rank_elem_start = (uint32_t) kparams->rank_elem_start; + actx.rank_nelem = (uint32_t) kparams->rank_nelem; + actx.elems_per_thread = elems_per_thread; + actx.block_elems = block_elems; + actx.vtcm_size_per_thread = vtcm_size_per_thread; + actx.is_row_bcast = (kparams->is_row_bcast != 0); + + work_queue_func_t reduce_fun = NULL; + switch (kparams->kernel_type) { + case HTP_ALLREDUCE_KERNEL_DMA_1D: + if (has_add) { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_add_f16 : allreduce_thread_dma_1d_add_f32; + } else { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_f16 : allreduce_thread_dma_1d_f32; + } + break; + case HTP_ALLREDUCE_KERNEL_DMA_2D: + if (has_add) { + if (kparams->is_row_bcast) { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_bcast_f16 : allreduce_thread_dma_2d_add_bcast_f32; + } else { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_f16 : allreduce_thread_dma_2d_add_f32; + } + } else { + reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_f16 : allreduce_thread_dma_2d_f32; + } + break; + default: + FARF(ERROR, "ggml-hex: allreduce unsupported kernel %d : rank %u\n", kparams->kernel_type, rank); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_NO_SUPPORT); + return HTP_STATUS_NO_SUPPORT; + } + + uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base; + for (uint32_t s = 0; s < n_ranks; s++) { + actx.src_spad_base[s] = vtcm_ptr; + vtcm_ptr += n_threads * vtcm_size_per_thread; + } + actx.dst_spad_base = vtcm_ptr; + vtcm_ptr += n_threads * vtcm_size_per_thread; + if (has_add) { + actx.res_spad_base = vtcm_ptr; + vtcm_ptr += (actx.is_row_bcast ? 1 : n_threads) * vtcm_size_per_thread; + } + + if (has_add && actx.is_row_bcast) { + const dma_addr_t r_ddr = octx->src[2 * n_ranks]->data; + const uint32_t row_bytes = actx.ne0 * (dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float)); + dma_queue * dma_q = octx->ctx->dma[0]; + dma_queue_push(dma_q, dma_make_data(actx.res_spad_base, r_ddr), actx.row_size_aligned, 0, row_bytes, 1); + dma_queue_pop(dma_q); + } + + work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads); + } + + // 4. Exit Barrier: Synchronize all ranks after writing + htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); + + // drain fan-out DMA writes before exit barrier + if (mode == HTP_ALLREDUCE_SHARDED_FANOUT) { + asm volatile ("syncht" : : : "memory"); + } + + htp_fence_write(my_fence, fence_seq_exit, octx->status); + + for (uint32_t j = 0; j < n_ranks; j++) { + if (j == rank) continue; + const struct htp_tensor * peer_sync = octx->src[n_ranks + j]; + atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data; + uint64_t spins = 0; + while (1) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - fence_seq_exit) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: allreduce exit peer %u failed with status %u\n", j, peer_status); + htp_fence_write(my_fence, fence_seq_exit, peer_status); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); + return peer_status; + } + break; + } + if (++spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n", + rank, j, peer_fence, fence_seq_exit, peer_seq); + htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR); + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); + return HTP_STATUS_INTERNAL_ERR; + } + hex_pause(); + } + } + asm volatile ("syncht" : : : "memory"); + + htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit); + + return octx->status; +} diff --git a/ggml/src/ggml-hexagon/htp/allreduce-ops.h b/ggml/src/ggml-hexagon/htp/allreduce-ops.h new file mode 100644 index 0000000..1264c1c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/allreduce-ops.h @@ -0,0 +1,63 @@ +#ifndef ALLREDUCE_OPS_H +#define ALLREDUCE_OPS_H + +#include +#include +#include + +#define HTP_ALLREDUCE_MAX_RANKS 4 + +#ifdef __cplusplus +extern "C" { +#endif + +enum htp_allreduce_kernel_type { + HTP_ALLREDUCE_KERNEL_UNSUPPORTED = 0, + HTP_ALLREDUCE_KERNEL_DMA_1D, + HTP_ALLREDUCE_KERNEL_DMA_2D, +}; + +enum htp_allreduce_mode { + HTP_ALLREDUCE_FULL = 0, // all ranks reduce full tensor + HTP_ALLREDUCE_SHARDED_FANOUT = 1, // each rank reduces 1/N shard and fans out to all buffers +}; + +static inline size_t htp_allreduce_vtcm_buffer_count( + uint32_t n_ranks, + uint32_t n_threads, + bool has_add, + bool is_row_bcast +) { + return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0); +} + +struct htp_allreduce_kernel_params { + uint8_t kernel_type; + int32_t rank; + int32_t n_ranks; + int32_t n_threads; + int32_t block_elems; // 1D: block_elems, 2D: block_rows + int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread + int32_t vtcm_size_per_thread; + int32_t vtcm_size; + int32_t ne0; + int32_t ne1; + int32_t row_size_aligned; + int32_t rank_elem_start; + int32_t rank_nelem; + int32_t n_dsts; + int32_t is_row_bcast; + int32_t mode; // enum htp_allreduce_mode: FULL or SHARDED_FANOUT +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob"); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* ALLREDUCE_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c index bdd0623..7aa7ca8 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.c +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c @@ -9,285 +9,1029 @@ #include "ggml.h" #include "hvx-utils.h" -#include "hex-dma.h" +#include "hvx-copy.h" +#include "hvx-reduce.h" +#include "dma-queue.h" +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" +#include "argsort-ops.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) #endif -struct htp_argsort_context { - struct htp_ops_context * octx; - uint32_t nrows_per_thread; -}; -static inline bool all_greater_f32(HVX_Vector x, HVX_Vector y) -{ - const HVX_Vector one = Q6_V_vsplat_R(1); - const HVX_Vector zero = Q6_V_vzero(); +static void quicksort_values_indices_asc(float * values, int32_t * indices, int left, int right) { + while (left < right) { + float pivot = values[left + (right - left) / 2]; + int i = left; + int j = right; - HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(x, y); - HVX_Vector matches = Q6_V_vmux_QVV(pred, one, zero); - HVX_Vector sum = hvx_vec_reduce_sum_i32(matches); - return hvx_vec_get_i32(sum) == 32; -} + while (i <= j) { + while (values[i] < pivot) i++; + while (values[j] > pivot) j--; -// Sorts values and mirrors swaps to indices. -static void quicksort_values_indices_asc(float * values, int32_t * indices, int left, int right) { - if (left >= right) return; + if (i <= j) { + float tmp_val = values[i]; + values[i] = values[j]; + values[j] = tmp_val; + + int32_t tmp_idx = indices[i]; + indices[i] = indices[j]; + indices[j] = tmp_idx; + i++; + j--; + } + } - int pivot_idx = (left + right) / 2; - float pivot = values[pivot_idx]; - int i = left; - int j = right; + // Tail-recursion elimination to bound stack depth + if (j - left < right - i) { + if (left < j) quicksort_values_indices_asc(values, indices, left, j); + left = i; + } else { + if (i < right) quicksort_values_indices_asc(values, indices, i, right); + right = j; + } + } +} + +static void quicksort_values_indices_desc(float * values, int32_t * indices, int left, int right) { + while (left < right) { + float pivot = values[left + (right - left) / 2]; + int i = left; + int j = right; - HVX_Vector pivot_vec = hvx_vec_splat_f32(pivot); - while (i <= j) { - // Vectorized scan for i while (i <= j) { - // Check if we have at least one full vector - if (i + 32 <= j) { - HVX_Vector vals_vec = *(HVX_UVector *)(values + i); - if (all_greater_f32(pivot_vec, vals_vec)) { - // If all elements are < pivot, we can skip this whole block - i += 32; - continue; - } - } + while (values[i] > pivot) i++; + while (values[j] < pivot) j--; + + if (i <= j) { + float tmp_val = values[i]; + values[i] = values[j]; + values[j] = tmp_val; - // Scalar fallback / cleanup - if (values[i] < pivot) { + int32_t tmp_idx = indices[i]; + indices[i] = indices[j]; + indices[j] = tmp_idx; i++; - } else { - break; + j--; } } - // Vectorized scan for j - while (i <= j) { - if (j - 32 >= i) { - // Load 32 elements ending at j. - // Since we want `values[j] > pivot`, let's load from j-31 to j. - HVX_Vector vals_vec = *(HVX_UVector *)(values + j - 31); - if (all_greater_f32(vals_vec, pivot_vec)) { - j -= 32; - continue; + // Tail-recursion elimination to bound stack depth + if (j - left < right - i) { + if (left < j) quicksort_values_indices_desc(values, indices, left, j); + left = i; + } else { + if (i < right) quicksort_values_indices_desc(values, indices, i, right); + right = j; + } + } +} + +static inline uint32_t top_k_max_value_index(const float * values, uint32_t n, float * value) { + int32_t max_idx = 0; + hvx_argmax_f32(values, n, 0, value, &max_idx); + return (uint32_t) max_idx; +} + +static const int32_t argsort_ramp_lut[32] __attribute__((aligned(VLEN))) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 +}; + +static inline void init_indices_ramp(int32_t * indices_buf, uint32_t count, uint32_t base) { + const HVX_Vector ind_init_vec = Q6_Vw_vadd_VwVw(*(const HVX_Vector *) argsort_ramp_lut, Q6_V_vsplat_R(base)); + const HVX_Vector ind_diff_vec = Q6_V_vsplat_R(32); + HVX_Vector * indices_buf_vec = (HVX_Vector *) indices_buf; + uint32_t num_vecs = count / 32; + HVX_Vector curr_ind_vec = ind_init_vec; + for (uint32_t j = 0; j < num_vecs; j++) { + indices_buf_vec[j] = curr_ind_vec; + curr_ind_vec = Q6_Vw_vadd_VwVw(curr_ind_vec, ind_diff_vec); + } + uint32_t rem = count % 32; + if (rem > 0) { + hvx_vec_store_a((void *) &indices_buf_vec[num_vecs], rem * sizeof(int32_t), curr_ind_vec); + } +} + +__attribute__((always_inline)) +static inline void vec_cas(HVX_Vector * X_val, HVX_Vector * X_idx, HVX_Vector * Y_val, HVX_Vector * Y_idx, bool asc) { + HVX_VectorPred pred = asc ? Q6_Q_vcmp_gt_VsfVsf(*X_val, *Y_val) + : Q6_Q_vcmp_gt_VsfVsf(*Y_val, *X_val); + HVX_Vector next_X_val = Q6_V_vmux_QVV(pred, *Y_val, *X_val); + HVX_Vector next_Y_val = Q6_V_vmux_QVV(pred, *X_val, *Y_val); + HVX_Vector next_X_idx = Q6_V_vmux_QVV(pred, *Y_idx, *X_idx); + HVX_Vector Y_tmp_idx = Q6_V_vmux_QVV(pred, *X_idx, *Y_idx); + *X_val = next_X_val; + *Y_val = next_Y_val; + *X_idx = next_X_idx; + *Y_idx = Y_tmp_idx; +} + +__attribute__((always_inline)) +static inline void bitonic_cas_32(HVX_Vector * V, HVX_Vector * I, int d, HVX_VectorPred dir_mask, HVX_Vector idx_vec, HVX_Vector zero_vec) { + HVX_VectorPred mask_left; + HVX_Vector V_rot_left, V_rot_right; + HVX_Vector I_rot_left, I_rot_right; + + if (d == 1) { + mask_left = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(idx_vec, Q6_V_vsplat_R(1)), zero_vec); + V_rot_left = Q6_V_vror_VR(*V, 4); + V_rot_right = Q6_V_vror_VR(*V, 124); + I_rot_left = Q6_V_vror_VR(*I, 4); + I_rot_right = Q6_V_vror_VR(*I, 124); + } else if (d == 2) { + mask_left = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(idx_vec, Q6_V_vsplat_R(2)), zero_vec); + V_rot_left = Q6_V_vror_VR(*V, 8); + V_rot_right = Q6_V_vror_VR(*V, 120); + I_rot_left = Q6_V_vror_VR(*I, 8); + I_rot_right = Q6_V_vror_VR(*I, 120); + } else if (d == 4) { + mask_left = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(idx_vec, Q6_V_vsplat_R(4)), zero_vec); + V_rot_left = Q6_V_vror_VR(*V, 16); + V_rot_right = Q6_V_vror_VR(*V, 112); + I_rot_left = Q6_V_vror_VR(*I, 16); + I_rot_right = Q6_V_vror_VR(*I, 112); + } else if (d == 8) { + mask_left = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(idx_vec, Q6_V_vsplat_R(8)), zero_vec); + V_rot_left = Q6_V_vror_VR(*V, 32); + V_rot_right = Q6_V_vror_VR(*V, 96); + I_rot_left = Q6_V_vror_VR(*I, 32); + I_rot_right = Q6_V_vror_VR(*I, 96); + } else { + mask_left = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(idx_vec, Q6_V_vsplat_R(16)), zero_vec); + V_rot_left = Q6_V_vror_VR(*V, 64); + V_rot_right = Q6_V_vror_VR(*V, 64); + I_rot_left = Q6_V_vror_VR(*I, 64); + I_rot_right = Q6_V_vror_VR(*I, 64); + } + + HVX_Vector V_paired = Q6_V_vmux_QVV(mask_left, V_rot_left, V_rot_right); + HVX_Vector I_paired = Q6_V_vmux_QVV(mask_left, I_rot_left, I_rot_right); + + HVX_VectorPred V_gt_Vpaired = Q6_Q_vcmp_gt_VsfVsf(*V, V_paired); + HVX_VectorPred Vpaired_gt_V = Q6_Q_vcmp_gt_VsfVsf(V_paired, *V); + HVX_VectorPred mask_right = Q6_Q_not_Q(mask_left); + HVX_VectorPred Q_asc = Q6_Q_or_QQ( + Q6_Q_and_QQ(mask_left, V_gt_Vpaired), + Q6_Q_and_QQ(Vpaired_gt_V, mask_right) + ); + HVX_VectorPred Q_swap = Q6_Q_or_QQ( + Q6_Q_and_QQ(dir_mask, Q_asc), + Q6_Q_and_QQ(Q6_Q_not_Q(dir_mask), Q6_Q_not_Q(Q_asc)) + ); + + *V = Q6_V_vmux_QVV(Q_swap, V_paired, *V); + *I = Q6_V_vmux_QVV(Q_swap, I_paired, *I); +} + +__attribute__((always_inline)) +static inline void bitonic_sort_generic_hvx(uint8_t * values, uint8_t * indices, int K, bool asc_order) { + HVX_Vector V[32]; + HVX_Vector I[32]; + + HVX_Vector zero_vec = Q6_V_vzero(); + HVX_Vector idx_vec = *(const HVX_Vector *) argsort_ramp_lut; + + for (int v = 0; v < K; v++) { + V[v] = *(HVX_Vector *)(values + v * 128); + I[v] = Q6_Vw_vadd_VwVw(idx_vec, Q6_V_vsplat_R(v * 32)); + } + + HVX_VectorPred pred_all_1s = Q6_Q_vcmp_eq_VwVw(zero_vec, zero_vec); + HVX_VectorPred pred_all_0s = Q6_Q_not_Q(pred_all_1s); + + int M = 5; + while ((1 << (M - 5)) < K) M++; + + for (int s = 1; s <= M; s++) { + for (int stage_d = s - 1; stage_d >= 0; stage_d--) { + int d = 1 << stage_d; + if (d >= 32) { + int v_dist = d / 32; + for (int v1 = 0; v1 < K; v1++) { + if ((v1 & v_dist) == 0) { + int v2 = v1 + v_dist; + bool asc = (s < M) ? ((((v1 * 32) >> s) % 2) == 0) : asc_order; + vec_cas(&V[v1], &I[v1], &V[v2], &I[v2], asc); + } + } + } else { + if (s < 5) { + HVX_VectorPred dir_mask = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(idx_vec, Q6_V_vsplat_R(1 << s)), zero_vec); + for (int v = 0; v < K; v++) { + bitonic_cas_32(&V[v], &I[v], d, dir_mask, idx_vec, zero_vec); + } + } else { + for (int v = 0; v < K; v++) { + bool asc = (s < M) ? ((((v * 32) >> s) % 2) == 0) : asc_order; + HVX_VectorPred dir_mask = asc ? pred_all_1s : pred_all_0s; + bitonic_cas_32(&V[v], &I[v], d, dir_mask, idx_vec, zero_vec); + } } } + } + } - if (values[j] > pivot) { - j--; + for (int v = 0; v < K; v++) { + *(HVX_Vector *)(values + v * 128) = V[v]; + *(HVX_Vector *)(indices + v * 128) = I[v]; + } +} + +static void bitonic_sort_vtcm_desc(uint8_t * values, uint8_t * indices, uint32_t n_vec, bool init_indices) { + HVX_Vector zero_vec = Q6_V_vzero(); + HVX_Vector idx_vec = *(const HVX_Vector *) argsort_ramp_lut; + + HVX_VectorPred pred_all_1s = Q6_Q_vcmp_eq_VwVw(zero_vec, zero_vec); + HVX_VectorPred pred_all_0s = Q6_Q_not_Q(pred_all_1s); + + if (init_indices) { + for (uint32_t v = 0; v < n_vec; v++) { + HVX_Vector idx = Q6_Vw_vadd_VwVw(idx_vec, Q6_V_vsplat_R(v * 32)); + *(HVX_Vector *)(indices + v * 128) = idx; + } + } + + int M = 5; + while ((1u << (M - 5)) < n_vec) M++; + + for (int s = 1; s <= M; s++) { + for (int stage_d = s - 1; stage_d >= 0; stage_d--) { + int d = 1 << stage_d; + if (d >= 32) { + uint32_t v_dist = d / 32; + for (uint32_t v1 = 0; v1 < n_vec; v1++) { + if ((v1 & v_dist) == 0) { + uint32_t v2 = v1 + v_dist; + bool asc = (s < M) ? ((((v1 * 32) >> s) % 2) == 0) : false; + + HVX_Vector Vv1 = *(HVX_Vector *)(values + v1 * 128); + HVX_Vector Iv1 = *(HVX_Vector *)(indices + v1 * 128); + HVX_Vector Vv2 = *(HVX_Vector *)(values + v2 * 128); + HVX_Vector Iv2 = *(HVX_Vector *)(indices + v2 * 128); + + vec_cas(&Vv1, &Iv1, &Vv2, &Iv2, asc); + + *(HVX_Vector *)(values + v1 * 128) = Vv1; + *(HVX_Vector *)(indices + v1 * 128) = Iv1; + *(HVX_Vector *)(values + v2 * 128) = Vv2; + *(HVX_Vector *)(indices + v2 * 128) = Iv2; + } + } } else { - break; + if (s < 5) { + HVX_VectorPred dir_mask = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(idx_vec, Q6_V_vsplat_R(1 << s)), zero_vec); + for (uint32_t v = 0; v < n_vec; v++) { + HVX_Vector Vv = *(HVX_Vector *)(values + v * 128); + HVX_Vector Iv = *(HVX_Vector *)(indices + v * 128); + + bitonic_cas_32(&Vv, &Iv, d, dir_mask, idx_vec, zero_vec); + + *(HVX_Vector *)(values + v * 128) = Vv; + *(HVX_Vector *)(indices + v * 128) = Iv; + } + } else { + for (uint32_t v = 0; v < n_vec; v++) { + bool asc = (s < M) ? ((((v * 32) >> s) % 2) == 0) : false; + HVX_VectorPred dir_mask = asc ? pred_all_1s : pred_all_0s; + + HVX_Vector Vv = *(HVX_Vector *)(values + v * 128); + HVX_Vector Iv = *(HVX_Vector *)(indices + v * 128); + + bitonic_cas_32(&Vv, &Iv, d, dir_mask, idx_vec, zero_vec); + + *(HVX_Vector *)(values + v * 128) = Vv; + *(HVX_Vector *)(indices + v * 128) = Iv; + } + } } } + } +} + +static void top_k_select_tiled(dma_queue * dma_q, struct htp_thread_trace * tr, + dma_addr_t src_addr, uint32_t n, uint32_t k, + float * values_buf, int32_t * indices_buf, + float * out_values, int32_t * out_indices, + uint32_t chunk_base) { + const uint32_t tile_elems = 1024; + uint32_t n_tiles = (n + tile_elems - 1) / tile_elems; + uint32_t candidate_count = n_tiles * k; + uint32_t merge_n_vec = hmx_ceil_div(candidate_count, 32); + uint32_t merge_n_vec_pow2 = 1; + while (merge_n_vec_pow2 < merge_n_vec) merge_n_vec_pow2 <<= 1; + uint32_t merge_elems = merge_n_vec_pow2 * 32; + float * candidate_values = values_buf + tile_elems; + int32_t * candidate_indices = indices_buf + tile_elems; + uint32_t candidate_pos = 0; + + for (uint32_t offset = 0; offset < n; offset += tile_elems) { + uint32_t tile_count = MIN(tile_elems, n - offset); + dma_sync_read(dma_q, values_buf, src_addr + (size_t) offset * sizeof(float), tile_count * sizeof(float)); - if (i <= j) { - float tmp_val = values[i]; - values[i] = values[j]; - values[j] = tmp_val; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) offset); + if (tile_count < tile_elems) { + hvx_splat_f32_u((uint8_t *) (values_buf + tile_count), -INFINITY, tile_elems - tile_count); + } + + bitonic_sort_vtcm_desc((uint8_t *) values_buf, (uint8_t *) indices_buf, tile_elems / 32, true); + uint32_t tile_k = MIN(k, tile_count); + for (uint32_t j = 0; j < tile_k; j++) { + candidate_values[candidate_pos] = values_buf[j]; + candidate_indices[candidate_pos] = indices_buf[j] + (int32_t) offset; + candidate_pos++; + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) offset); + } - int32_t tmp_idx = indices[i]; - indices[i] = indices[j]; - indices[j] = tmp_idx; - i++; - j--; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); + if (merge_elems > candidate_pos) { + hvx_splat_f32_u((uint8_t *) (candidate_values + candidate_pos), -INFINITY, merge_elems - candidate_pos); + for (uint32_t j = candidate_pos; j < merge_elems; j++) { + candidate_indices[j] = 0; } } - if (left < j) quicksort_values_indices_asc(values, indices, left, j); - if (i < right) quicksort_values_indices_asc(values, indices, i, right); + bitonic_sort_vtcm_desc((uint8_t *) candidate_values, (uint8_t *) candidate_indices, merge_n_vec_pow2, false); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); + + for (uint32_t j = 0; j < k; j++) { + out_values[j] = candidate_values[j]; + out_indices[j] = candidate_indices[j] + (int32_t) chunk_base; + } } -static void quicksort_values_indices_desc(float * values, int32_t * indices, int left, int right) { - if (left >= right) return; +__attribute__((always_inline)) +static inline void sort32_f32_hvx(uint8_t * values, uint8_t * indices, enum ggml_sort_order order) { + bitonic_sort_generic_hvx(values, indices, 1, order == GGML_SORT_ORDER_ASC); +} - int pivot_idx = (left + right) / 2; - float pivot = values[pivot_idx]; - int i = left; - int j = right; +__attribute__((always_inline)) +static inline void sort64_f32_hvx(uint8_t * values, uint8_t * indices, enum ggml_sort_order order) { + bitonic_sort_generic_hvx(values, indices, 2, order == GGML_SORT_ORDER_ASC); +} - HVX_Vector pivot_vec = hvx_vec_splat_f32(pivot); +__attribute__((always_inline)) +static inline void sort128_f32_hvx(uint8_t * values, uint8_t * indices, enum ggml_sort_order order) { + bitonic_sort_generic_hvx(values, indices, 4, order == GGML_SORT_ORDER_ASC); +} - while (i <= j) { - // Vectorized scan for i (values[i] > pivot) - while (i <= j) { - if (i + 32 <= j) { - HVX_Vector vals_vec = *(HVX_UVector *)(values + i); - if (all_greater_f32(vals_vec, pivot_vec)) { - i += 32; - continue; - } - } +__attribute__((always_inline)) +static inline void sort256_f32_hvx(uint8_t * values, uint8_t * indices, enum ggml_sort_order order) { + bitonic_sort_generic_hvx(values, indices, 8, order == GGML_SORT_ORDER_ASC); +} - if (values[i] > pivot) { - i++; +__attribute__((always_inline)) +static inline void sort512_f32_hvx(uint8_t * values, uint8_t * indices, enum ggml_sort_order order) { + bitonic_sort_generic_hvx(values, indices, 16, order == GGML_SORT_ORDER_ASC); +} + +__attribute__((always_inline)) +static inline void sort1024_f32_hvx(uint8_t * values, uint8_t * indices, enum ggml_sort_order order) { + bitonic_sort_generic_hvx(values, indices, 32, order == GGML_SORT_ORDER_ASC); +} + +static void merge_runs( + const float * restrict in_val0, const int32_t * restrict in_idx0, uint32_t n0, + const float * restrict in_val1, const int32_t * restrict in_idx1, uint32_t n1, + float * restrict out_val, int32_t * restrict out_idx, + bool asc +) { + uint32_t i0 = 0; + uint32_t i1 = 0; + uint32_t out = 0; + + if (asc) { + while (i0 < n0 && i1 < n1) { + if (in_val0[i0] <= in_val1[i1]) { + if (out_val) out_val[out] = in_val0[i0]; + out_idx[out] = in_idx0[i0]; + i0++; } else { - break; + if (out_val) out_val[out] = in_val1[i1]; + out_idx[out] = in_idx1[i1]; + i1++; } + out++; } - - // Vectorized scan for j (values[j] < pivot) - while (i <= j) { - if (j - 32 >= i) { - HVX_Vector vals_vec = *(HVX_UVector *)(values + j - 31); - if (all_greater_f32(pivot_vec, vals_vec)) { - j -= 32; - continue; - } + } else { + while (i0 < n0 && i1 < n1) { + if (in_val0[i0] >= in_val1[i1]) { + if (out_val) out_val[out] = in_val0[i0]; + out_idx[out] = in_idx0[i0]; + i0++; + } else { + if (out_val) out_val[out] = in_val1[i1]; + out_idx[out] = in_idx1[i1]; + i1++; } + out++; + } + } - if (values[j] < pivot) { - j--; - } else { - break; + while (i0 < n0) { + if (out_val) out_val[out] = in_val0[i0]; + out_idx[out] = in_idx0[i0]; + i0++; + out++; + } + + while (i1 < n1) { + if (out_val) out_val[out] = in_val1[i1]; + out_idx[out] = in_idx1[i1]; + i1++; + out++; + } +} + +struct htp_sort_chunk_ctx { + struct htp_ops_context * octx; + const struct htp_sort_kernel_params * kparams; + uint8_t * phase1_base; + size_t phase1_slot_size; + float * merge_values; + int32_t * merge_indices; + uint32_t real_count[HTP_MAX_NTHREADS]; +}; + +static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { + struct htp_sort_chunk_ctx * cctx = (struct htp_sort_chunk_ctx *) data; + struct htp_ops_context * octx = cctx->octx; + const struct htp_sort_kernel_params * kparams = cctx->kparams; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; + uint32_t ne00 = src0->ne[0]; + uint32_t chunk_base = i * chunk_elems; + + uint32_t real_count = (chunk_base < ne00) ? MIN(chunk_elems, ne00 - chunk_base) : 0; + cctx->real_count[i] = real_count; + + uint8_t * spad = cctx->phase1_base + cctx->phase1_slot_size * i; + float * values_buf = (float *) spad; + int32_t * indices_buf = (int32_t *) (spad + chunk_elems * sizeof(float)); + + float * merge_values = cctx->merge_values; + int32_t * merge_indices = cctx->merge_indices; + + struct htp_thread_trace * tr = &octx->ctx->trace[i]; + const dma_addr_t src_addr = src0->data + (size_t) chunk_base * sizeof(float); + + if (kparams->is_top_k) { + uint32_t k = dst->ne[0]; + uint32_t local_k = MIN(k, chunk_elems); + + if (real_count == 0) { + for (uint32_t j = 0; j < local_k; j++) { + merge_values[i * local_k + j] = -INFINITY; + merge_indices[i * local_k + j] = 0; } + return; + } + + if (local_k == 1 && ne00 >= 128*1024) { + dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + float max_value; + uint32_t max_index = top_k_max_value_index(values_buf, real_count, &max_value); + merge_values[i] = max_value; + merge_indices[i] = (int32_t) (max_index + chunk_base); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + return; + } + + if (local_k > 1 && local_k <= 64 && chunk_elems > 1024) { + top_k_select_tiled(octx->ctx->dma[i], tr, src_addr, real_count, local_k, + values_buf, indices_buf, + &merge_values[i * local_k], &merge_indices[i * local_k], + chunk_base); + return; + } + + dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + if (chunk_elems > real_count) { + hvx_splat_f32_u((uint8_t *)(values_buf + real_count), -INFINITY, chunk_elems - real_count); + } + + bitonic_sort_vtcm_desc((uint8_t *) values_buf, (uint8_t *) indices_buf, chunk_elems / 32, true); + + for (uint32_t j = 0; j < local_k; j++) { + merge_values[i * local_k + j] = values_buf[j]; + merge_indices[i * local_k + j] = indices_buf[j] + (int32_t) chunk_base; } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + } else { + if (real_count > 0) { + dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); - if (i <= j) { - float tmp_val = values[i]; - values[i] = values[j]; - values[j] = tmp_val; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + init_indices_ramp(indices_buf, real_count, chunk_base); - int32_t tmp_idx = indices[i]; - indices[i] = indices[j]; - indices[j] = tmp_idx; - i++; - j--; + if (kparams->order == GGML_SORT_ORDER_ASC) { + quicksort_values_indices_asc(values_buf, indices_buf, 0, real_count - 1); + } else { + quicksort_values_indices_desc(values_buf, indices_buf, 0, real_count - 1); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); } } +} - if (left < j) quicksort_values_indices_desc(values, indices, left, j); - if (i < right) quicksort_values_indices_desc(values, indices, i, right); +struct htp_argsort_merge_l1_ctx { + struct htp_sort_chunk_ctx * cctx; + bool asc; +}; + +static void htp_argsort_merge_l1_job(unsigned int n, unsigned int i, void * data) { + struct htp_argsort_merge_l1_ctx * mctx = (struct htp_argsort_merge_l1_ctx *) data; + struct htp_sort_chunk_ctx * cctx = mctx->cctx; + const struct htp_sort_kernel_params * kparams = cctx->kparams; + + uint32_t c0 = i * 2; + uint32_t c1 = c0 + 1; + + uint32_t count0 = cctx->real_count[c0]; + uint32_t count1 = cctx->real_count[c1]; + + size_t slot_size = cctx->phase1_slot_size; + uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; + + const float * val0 = (const float *) (cctx->phase1_base + slot_size * c0); + const int32_t * idx0 = (const int32_t *) (cctx->phase1_base + slot_size * c0 + chunk_elems * sizeof(float)); + const float * val1 = (const float *) (cctx->phase1_base + slot_size * c1); + const int32_t * idx1 = (const int32_t *) (cctx->phase1_base + slot_size * c1 + chunk_elems * sizeof(float)); + + uint32_t out_offset = (2 * i) * chunk_elems; + float * out_val = cctx->merge_values + out_offset; + int32_t * out_idx = cctx->merge_indices + out_offset; + + struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); } -// LUT for ramp initialization of argsort output (first 32 members) -int32_t argosrt_ramp_lut[32] __attribute__((aligned(VLEN))) = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 +struct htp_argsort_merge_l2_ctx { + struct htp_sort_chunk_ctx * cctx; + bool asc; }; -static void htp_argsort_f32(unsigned int n, unsigned int i, void * data) { - struct htp_argsort_context * actx = (struct htp_argsort_context *)data; - struct htp_ops_context * octx = actx->octx; +static void htp_argsort_merge_l2_job(unsigned int n, unsigned int i, void * data) { + struct htp_argsort_merge_l2_ctx * mctx = (struct htp_argsort_merge_l2_ctx *) data; + struct htp_sort_chunk_ctx * cctx = mctx->cctx; + const struct htp_sort_kernel_params * kparams = cctx->kparams; - // Unpack context - const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * dst = octx->dst; + size_t slot_size = cctx->phase1_slot_size; + uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; + + uint32_t off0 = (4 * i) * chunk_elems; + uint32_t off1 = (4 * i + 2) * chunk_elems; + + const float * val0 = cctx->merge_values + off0; + const int32_t * idx0 = cctx->merge_indices + off0; + const float * val1 = cctx->merge_values + off1; + const int32_t * idx1 = cctx->merge_indices + off1; + + uint32_t count0 = cctx->real_count[4*i + 0] + cctx->real_count[4*i + 1]; + uint32_t count1 = cctx->real_count[4*i + 2] + cctx->real_count[4*i + 3]; + + uint8_t * slot_base = cctx->phase1_base + (4 * i) * slot_size; + float * out_val = (float *) slot_base; + int32_t * out_idx = (int32_t *) (slot_base + 4 * chunk_elems * sizeof(float)); + + struct htp_thread_trace * tr = &cctx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + merge_runs(val0, idx0, count0, val1, idx1, count1, out_val, out_idx, mctx->asc); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); +} + +struct htp_sort_multi_row_ctx { + struct htp_ops_context * octx; + const struct htp_sort_kernel_params * kparams; + uint8_t * phase1_base; + size_t phase1_slot_size; + uint32_t row_start; + uint32_t row_end; + uint32_t nrows_per_thread; +}; + +struct htp_tensor_row_coord { + uint32_t i01; + uint32_t i02; + uint32_t i03; +}; + +static inline struct htp_tensor_row_coord init_row_coord( + uint32_t r, + uint32_t ne01, + uint32_t ne02, + const struct fastdiv_values * div_ne01, + const struct fastdiv_values * div_ne01_ne02 +) { + uint32_t ne01_ne02 = ne01 * ne02; + uint32_t i03 = (ne01_ne02 > 0) ? fastdiv(r, div_ne01_ne02) : 0; + uint32_t rem = (ne01_ne02 > 0) ? (r - i03 * ne01_ne02) : 0; + uint32_t i02 = (ne01 > 0) ? fastdiv(rem, div_ne01) : 0; + uint32_t i01 = (ne01 > 0) ? (rem - i02 * ne01) : 0; + return (struct htp_tensor_row_coord){ i01, i02, i03 }; +} + +static inline void step_row_coord(struct htp_tensor_row_coord * c, uint32_t ne01, uint32_t ne02) { + c->i01++; + if (c->i01 == ne01) { + c->i01 = 0; + c->i02++; + if (c->i02 == ne02) { + c->i02 = 0; + c->i03++; + } + } +} - // Scratchpad memory - uint8_t * spad = octx->src0_spad.data + octx->src0_spad.size_per_thread * i; +static inline size_t coord_offset(struct htp_tensor_row_coord c, size_t nb1, size_t nb2, size_t nb3) { + return (size_t) c.i01 * nb1 + (size_t) c.i02 * nb2 + (size_t) c.i03 * nb3; +} + +static inline void compute_row_sort( + float * cur_values, + int32_t * cur_indices, + uint32_t ne00, + uint32_t chunk_elems, + bool is_top_k, + enum ggml_sort_order order +) { + if (is_top_k) { + if (ne00 == 32 || ne00 == 64 || ne00 == 128 || ne00 == 256 || ne00 == 512 || ne00 == 1024) { + switch (ne00) { + case 32: sort32_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; + case 64: sort64_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; + case 128: sort128_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; + case 256: sort256_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; + case 512: sort512_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; + case 1024: sort1024_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; + } + } else { + if (chunk_elems > ne00) { + hvx_splat_f32_u((uint8_t *)(cur_values + ne00), -INFINITY, chunk_elems - ne00); + } + bitonic_sort_vtcm_desc((uint8_t *) cur_values, (uint8_t *) cur_indices, chunk_elems / 32, true); + } + } else { + if (ne00 == 32 || ne00 == 64 || ne00 == 128 || ne00 == 256 || ne00 == 512 || ne00 == 1024) { + switch (ne00) { + case 32: sort32_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, order); break; + case 64: sort64_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, order); break; + case 128: sort128_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, order); break; + case 256: sort256_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, order); break; + case 512: sort512_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, order); break; + case 1024: sort1024_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, order); break; + } + } else { + init_indices_ramp(cur_indices, ne00, 0); + if (order == GGML_SORT_ORDER_ASC) { + quicksort_values_indices_asc(cur_values, cur_indices, 0, ne00 - 1); + } else { + quicksort_values_indices_desc(cur_values, cur_indices, 0, ne00 - 1); + } + } + } +} + +static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data) { + struct htp_sort_multi_row_ctx * mctx = (struct htp_sort_multi_row_ctx *) data; + struct htp_ops_context * octx = mctx->octx; + const struct htp_sort_kernel_params * kparams = mctx->kparams; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; - // Dimensions uint32_t ne00 = src0->ne[0]; uint32_t ne01 = src0->ne[1]; uint32_t ne02 = src0->ne[2]; - uint32_t ne03 = src0->ne[3]; uint32_t nb01 = src0->nb[1]; - //uint32_t nb02 = src0->nb[2]; - //uint32_t nb03 = src0->nb[3]; + uint32_t nb02 = src0->nb[2]; + uint32_t nb03 = src0->nb[3]; uint32_t nb1 = dst->nb[1]; - //uint32_t nb2 = dst->nb[2]; - //uint32_t nb3 = dst->nb[3]; - - // Sort order - enum ggml_sort_order order = (enum ggml_sort_order) octx->op_params[0]; - - // Rows to process - uint32_t total_rows = ne01 * ne02 * ne03; - uint32_t rows_per_thread = actx->nrows_per_thread; - uint32_t start_row = rows_per_thread * i; - uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); - - // Scratchpad layout: - // We need space for one row of float data (values) and one row of int32 indices. - // values: ne00 * sizeof(float) - // indices: ne00 * sizeof(int32_t) - // Padded to 128 bytes. - - size_t values_size = hex_round_up(ne00 * sizeof(float), 128); - size_t num_vec_ind_values = hmx_ceil_div(ne00, VLEN/(sizeof(int32_t))); - float * values_buf = (float *) spad; - int32_t * indices_buf = (int32_t *) (spad + values_size); - HVX_Vector * indices_buf_vec = (HVX_Vector *) (spad + values_size); - const HVX_Vector ind_init_vec = *(HVX_Vector *)argosrt_ramp_lut; - const HVX_Vector ind_diff_vec = Q6_V_vsplat_R(32); + uint32_t nb2 = dst->nb[2]; + uint32_t nb3 = dst->nb[3]; + + uint32_t row_start = mctx->row_start; + uint32_t row_end = mctx->row_end; + uint32_t r_start = row_start + ith * mctx->nrows_per_thread; + uint32_t r_end = MIN(r_start + mctx->nrows_per_thread, row_end); + + if (r_start >= r_end) return; + + size_t slot_size = mctx->phase1_slot_size; + size_t spad_offset = ith * slot_size * (size_t) kparams->n_slots; + uint8_t * thread_spad = mctx->phase1_base + spad_offset; + + uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; + float * values_buf = (float *) thread_spad; + int32_t * indices_buf = (int32_t *) (thread_spad + chunk_elems * sizeof(float)); - for (uint32_t r = start_row; r < end_row; r++) { - uint32_t src_offset = r * nb01; - uint32_t dst_offset = r * nb1; + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - uint8_t * src_ptr = (uint8_t *) src0->data + src_offset; - uint8_t * dst_ptr = (uint8_t *) dst->data + dst_offset; + bool is_top_k = kparams->is_top_k != 0; + uint32_t k = dst->ne[0]; + enum ggml_sort_order order = (enum ggml_sort_order) kparams->order; - hex_l2fetch(src_ptr, ne00 * sizeof(float), ne00 * sizeof(float), 1); - hvx_copy_f32_au((uint8_t*)values_buf, src_ptr, ne00); + const uint32_t src_bytes = ne00 * sizeof(float); + const uint32_t dst_bytes = (is_top_k ? k : ne00) * sizeof(int32_t); - // Initialize indices - Start with values 0..31, add 32 for additional vec iterations - HVX_Vector curr_ind_vec = ind_init_vec; - for (uint32_t j_vec = 0; j_vec < num_vec_ind_values; j_vec++) { - indices_buf_vec[j_vec] = curr_ind_vec; - curr_ind_vec = Q6_Vw_vadd_VwVw(curr_ind_vec, ind_diff_vec); + if (kparams->n_slots == 2) { + struct htp_tensor_row_coord prime_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); + for (uint32_t r = r_start, spad_idx = 0; r < r_end && spad_idx < 2; r++, spad_idx++) { + uint8_t * cur_spad = thread_spad + spad_idx * slot_size; + float * cur_values = (float *) cur_spad; + int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float)); + + dma_addr_t dst_addr = dst->data + coord_offset(prime_coord, nb1, nb2, nb3); + dma_addr_t src_addr = src0->data + coord_offset(prime_coord, nb01, nb02, nb03); + + // Dummy dst writeback to establish queue ordering + dma_queue_push(dma_q, dma_make_data(dst_addr, cur_indices), + dst_bytes, dst_bytes, dst_bytes, 0); + + // Prefetch input row + dma_queue_push(dma_q, dma_make_data(cur_values, src_addr), + src_bytes, src_bytes, src_bytes, 1); + + step_row_coord(&prime_coord, ne01, ne02); } - // Sort values and mirror swaps to indices - if (order == GGML_SORT_ORDER_ASC) { - quicksort_values_indices_asc(values_buf, indices_buf, 0, ne00 - 1); - } else { - quicksort_values_indices_desc(values_buf, indices_buf, 0, ne00 - 1); + struct htp_tensor_row_coord dst_coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); + struct htp_tensor_row_coord next_src_coord = init_row_coord(r_start + 2, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); + + for (uint32_t r = r_start; r < r_end; r++) { + uint32_t cur_slot = (r - r_start) & 1; + uint8_t * cur_spad = thread_spad + cur_slot * slot_size; + float * cur_values = (float *) cur_spad; + int32_t * cur_indices = (int32_t *) (cur_spad + chunk_elems * sizeof(float)); + + // Wait for previous writeback of this slot to complete + dma_queue_pop(dma_q); + + // Wait for input row DMA read into this slot to complete + dma_queue_pop(dma_q); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + compute_row_sort(cur_values, cur_indices, ne00, chunk_elems, is_top_k, order); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + + // Push writeback of current slot + dma_addr_t dst_addr = dst->data + coord_offset(dst_coord, nb1, nb2, nb3); + dma_queue_push(dma_q, dma_make_data(dst_addr, cur_indices), + dst_bytes, dst_bytes, dst_bytes, 1); + step_row_coord(&dst_coord, ne01, ne02); + + // Prefetch next row into this slot + const uint32_t next_row = r + 2; + if (next_row < r_end) { + dma_addr_t next_src_addr = src0->data + coord_offset(next_src_coord, nb01, nb02, nb03); + dma_queue_push(dma_q, dma_make_data(cur_values, next_src_addr), + src_bytes, src_bytes, src_bytes, 1); + step_row_coord(&next_src_coord, ne01, ne02); + } } - // Copy indices back to DDR - hvx_copy_f32_ua(dst_ptr, (const uint8_t *) indices_buf, ne00); + dma_queue_flush(dma_q); + } else { + struct htp_tensor_row_coord coord = init_row_coord(r_start, ne01, ne02, &kparams->div_ne01, &kparams->div_ne01_ne02); + for (uint32_t r = r_start; r < r_end; r++) { + dma_addr_t src_addr = src0->data + coord_offset(coord, nb01, nb02, nb03); + dma_addr_t dst_addr = dst->data + coord_offset(coord, nb1, nb2, nb3); + + dma_sync_read(dma_q, values_buf, src_addr, src_bytes); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + compute_row_sort(values_buf, indices_buf, ne00, chunk_elems, is_top_k, order); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + + dma_sync_write(dma_q, dst_addr, indices_buf, dst_bytes); + step_row_coord(&coord, ne01, ne02); + } } } -int op_argsort(struct htp_ops_context * octx) { - // Check supported types +static int op_sort_common(struct htp_ops_context * octx, bool is_top_k) { if (octx->src[0]->type != HTP_TYPE_F32) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t total_rows = octx->src[0]->ne[1] * octx->src[0]->ne[2] * octx->src[0]->ne[3]; - const uint32_t n_threads = MIN(total_rows, octx->n_threads); + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } - // Allocate scratchpad - // We need 1 row of float + 1 row of int32 per thread. - uint32_t ne00 = octx->src[0]->ne[0]; - size_t values_size = hex_round_up(ne00 * sizeof(float), 128); - size_t indices_size = hex_round_up(ne00 * sizeof(int32_t), 128); - size_t spad_per_thread = values_size + indices_size; + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_row_size = dst->ne[0] * sizeof(int32_t); - // Make sure we round up to 256 for alignment requirements - spad_per_thread = hex_round_up(spad_per_thread, 256); + uint32_t row_start = 0; + uint32_t row_end = total_rows; + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(int32_t), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + row_end = range.start + range.count; + } - size_t total_spad_size = spad_per_thread * n_threads; + const uint32_t nrows = row_end - row_start; + if (nrows == 0) { + return HTP_STATUS_OK; + } - if (octx->ctx->vtcm_size < total_spad_size) { - FARF(ERROR, "argsort: VTCM size too small. Needed %zu, have %zu", total_spad_size, octx->ctx->vtcm_size); + uint32_t ne00 = src0->ne[0]; + uint32_t k = dst->ne[0]; + + const struct htp_sort_kernel_params * kparams = (const struct htp_sort_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + struct htp_sort_vtcm_layout layout; + htp_sort_vtcm_layout_build(&layout, + total_rows, + (uint32_t) kparams->chunk_elems, + (uint32_t) kparams->n_chunks, + (uint32_t) kparams->merge_elems, + (uint32_t) kparams->n_threads, + (uint32_t) kparams->n_slots); + + if (octx->ctx->vtcm_size < layout.total_bytes) { + FARF(ERROR, "sort: VTCM size too small. Needed %zu, have %zu", layout.total_bytes, octx->ctx->vtcm_size); return HTP_STATUS_VTCM_TOO_SMALL; } - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->src0_spad.size = total_spad_size; - octx->src0_spad.size_per_thread = spad_per_thread; + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + uint8_t * phase1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_phase1); + float * merge_values = VTCM_LAYOUT_PTR(float, vtcm_base, layout.off_merge_values); + int32_t * merge_indices = VTCM_LAYOUT_PTR(int32_t, vtcm_base, layout.off_merge_indices); + + if (total_rows == 1 && kparams->n_chunks > 1) { + struct htp_sort_chunk_ctx cctx; + cctx.octx = octx; + cctx.kparams = kparams; + cctx.phase1_base = phase1_base; + cctx.phase1_slot_size = layout.phase1_slot_size; + cctx.merge_values = merge_values; + cctx.merge_indices = merge_indices; + memset(cctx.real_count, 0, sizeof(cctx.real_count)); + + work_queue_run(octx->ctx->work_queue, htp_sort_chunk_job, &cctx, (uint32_t) kparams->n_chunks); - FARF(HIGH, "argsort: %ux%ux%ux%u -> %ux%ux%ux%u (0x%x, 0x%x)", - octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], - octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], - octx->src[0]->data, octx->dst->data); + if (is_top_k) { + uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; + uint32_t local_k = MIN(k, chunk_elems); - struct htp_argsort_context actx; - actx.octx = octx; - actx.nrows_per_thread = (total_rows + n_threads - 1) / n_threads; + if (k == 1 && ne00 >= 128*1024) { + float * cand_vals = merge_values; + int32_t * cand_idxs = merge_indices; + float best_val = cand_vals[0]; + int32_t best_idx = cand_idxs[0]; + for (int32_t c = 1; c < kparams->n_chunks; c++) { + if (cand_vals[c] > best_val) { + best_val = cand_vals[c]; + best_idx = cand_idxs[c]; + } + } + int32_t * dst_ptr = (int32_t *) dst->data; + dst_ptr[0] = best_idx; + return HTP_STATUS_OK; + } + + uint32_t total_candidates = (uint32_t) kparams->n_chunks * local_k; + float * cand_vals = merge_values; + int32_t * cand_idxs = merge_indices; + uint32_t merge_elems = (uint32_t) kparams->merge_elems; + + if (merge_elems > total_candidates) { + hvx_splat_f32_u((uint8_t *) (cand_vals + total_candidates), -INFINITY, merge_elems - total_candidates); + for (uint32_t j = total_candidates; j < merge_elems; j++) { + cand_idxs[j] = 0; + } + } + + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); + bitonic_sort_vtcm_desc((uint8_t *) cand_vals, (uint8_t *) cand_idxs, merge_elems / 32, false); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); + dma_sync_write(octx->ctx->dma[0], dst->data, cand_idxs, k * sizeof(int32_t)); + } else { + bool asc = kparams->order == GGML_SORT_ORDER_ASC; + if (kparams->n_chunks == 8) { + struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc }; + work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 4); + + struct htp_argsort_merge_l2_ctx l2ctx = { &cctx, asc }; + work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l2_job, &l2ctx, 2); + + size_t slot_size = layout.phase1_slot_size; + uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; + + const float * val0 = (const float *) phase1_base; + const int32_t * idx0 = (const int32_t *) (phase1_base + 4 * chunk_elems * sizeof(float)); + const float * val1 = (const float *) (phase1_base + 4 * slot_size); + const int32_t * idx1 = (const int32_t *) (phase1_base + 4 * slot_size + 4 * chunk_elems * sizeof(float)); + + uint32_t n01 = cctx.real_count[0] + cctx.real_count[1] + cctx.real_count[2] + cctx.real_count[3]; + uint32_t n23 = cctx.real_count[4] + cctx.real_count[5] + cctx.real_count[6] + cctx.real_count[7]; + + int32_t * final_idx = merge_indices; + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); + merge_runs(val0, idx0, n01, val1, idx1, n23, NULL, final_idx, asc); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); + dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); + } else if (kparams->n_chunks == 4) { + struct htp_argsort_merge_l1_ctx l1ctx = { &cctx, asc }; + work_queue_run(octx->ctx->work_queue, htp_argsort_merge_l1_job, &l1ctx, 2); + + uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; + uint32_t n01 = cctx.real_count[0] + cctx.real_count[1]; + uint32_t n23 = cctx.real_count[2] + cctx.real_count[3]; + + const float * val01 = merge_values; + const int32_t * idx01 = merge_indices; + const float * val23 = merge_values + 2 * chunk_elems; + const int32_t * idx23 = merge_indices + 2 * chunk_elems; + + int32_t * final_idx = (int32_t *) phase1_base; + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); + merge_runs(val01, idx01, n01, val23, idx23, n23, NULL, final_idx, asc); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); + dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); + } else if (kparams->n_chunks == 2) { + size_t slot_size = layout.phase1_slot_size; + uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; - // Run jobs - worker_pool_run_func(octx->ctx->worker_pool, htp_argsort_f32, &actx, n_threads); + const float * val0 = (const float *) (phase1_base); + const int32_t * idx0 = (const int32_t *) (phase1_base + chunk_elems * sizeof(float)); + const float * val1 = (const float *) (phase1_base + slot_size); + const int32_t * idx1 = (const int32_t *) (phase1_base + slot_size + chunk_elems * sizeof(float)); + + int32_t * final_idx = merge_indices; + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); + merge_runs(val0, idx0, cctx.real_count[0], val1, idx1, cctx.real_count[1], NULL, final_idx, asc); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); + dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); + } else { + uint32_t chunk_elems = (uint32_t) kparams->chunk_elems; + int32_t * final_idx = (int32_t *) (phase1_base + chunk_elems * sizeof(float)); + dma_sync_write(octx->ctx->dma[0], dst->data, final_idx, ne00 * sizeof(int32_t)); + } + } + return HTP_STATUS_OK; + } + + struct htp_sort_multi_row_ctx mctx; + mctx.octx = octx; + mctx.kparams = kparams; + mctx.phase1_base = phase1_base; + mctx.phase1_slot_size = layout.phase1_slot_size; + mctx.row_start = row_start; + mctx.row_end = row_end; + + uint32_t dst_row_bytes = (uint32_t) dst_row_size; + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(int32_t), dst_row_bytes, &rows_per_chunk); + if (rows_per_chunk == 0) { + rows_per_chunk = 1; + } + uint32_t dr = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + if (rows_per_chunk > 1) { + dr = hex_round_up(dr, rows_per_chunk); + } + mctx.nrows_per_thread = dr; + + work_queue_run(octx->ctx->work_queue, htp_sort_multi_row_job, &mctx, octx->n_threads); return HTP_STATUS_OK; } + +int op_argsort(struct htp_ops_context * octx) { + return op_sort_common(octx, false); +} + +int op_top_k(struct htp_ops_context * octx) { + return op_sort_common(octx, true); +} diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.h b/ggml/src/ggml-hexagon/htp/argsort-ops.h new file mode 100644 index 0000000..f74189f --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.h @@ -0,0 +1,163 @@ +#ifndef HTP_ARGSORT_OPS_H +#define HTP_ARGSORT_OPS_H + +#include +#include +#include +#include + +#include "hex-fastdiv.h" +#include "htp-vtcm.h" + +struct htp_sort_kernel_params { + uint32_t n_threads; + uint32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1) + uint32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT + uint32_t chunk_elems; + uint32_t n_chunks; + uint32_t vtcm_size; + uint32_t merge_elems; + uint32_t n_slots; + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne01_ne02; +}; + +struct htp_sort_vtcm_layout { + size_t total_bytes; + size_t phase1_slot_size; + size_t off_phase1; + size_t off_merge_values; + size_t off_merge_indices; + uint32_t chunk_elems; + uint32_t n_chunks; + uint32_t merge_elems; + uint32_t n_threads; + uint32_t n_slots; +}; + +static inline void htp_sort_vtcm_layout_build( + struct htp_sort_vtcm_layout * L, + uint32_t total_rows, + uint32_t chunk_elems, + uint32_t n_chunks, + uint32_t merge_elems, + uint32_t n_threads, + uint32_t n_slots) { + + memset(L, 0, sizeof(*L)); + L->chunk_elems = chunk_elems; + L->n_chunks = n_chunks; + L->merge_elems = merge_elems; + L->n_threads = n_threads; + L->n_slots = n_slots; + + const size_t values_size = ((chunk_elems * sizeof(float)) + 127) & ~127; + const size_t indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127; + L->phase1_slot_size = ((values_size + indices_size) + 255) & ~255; + + size_t off = 0; + if (total_rows > 1) { + VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_threads); + } else { + VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_chunks * n_slots); + const size_t merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127; + const size_t merge_indices_size = ((merge_elems * sizeof(int32_t)) + 127) & ~127; + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_values, merge_values_size, n_chunks > 1); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_indices, merge_indices_size, n_chunks > 1); + } + L->total_bytes = off; +} + +static inline bool htp_sort_solve_layout( + struct htp_sort_vtcm_layout * layout, + uint32_t ne00, + uint32_t total_rows, + uint32_t k, + uint32_t n_threads_max, + size_t vtcm_budget, + bool is_top_k) { + + memset(layout, 0, sizeof(*layout)); + + if (total_rows > 1) { + uint32_t n_threads = total_rows < n_threads_max ? total_rows : n_threads_max; + + uint32_t n_vec = (ne00 + 31) / 32; + uint32_t n_vec_pow2 = 1; + while (n_vec_pow2 < n_vec) n_vec_pow2 <<= 1; + uint32_t ne00_padded = n_vec_pow2 * 32; + + size_t values_size = ((ne00_padded * sizeof(float)) + 127) & ~127; + size_t indices_size = ((ne00_padded * sizeof(int32_t)) + 127) & ~127; + size_t spad_per_slot = ((values_size + indices_size) + 255) & ~255; + + uint32_t n_slots = 2; + if (spad_per_slot * 2 > vtcm_budget) { + n_slots = 1; + } + + size_t spad_per_thread = spad_per_slot * n_slots; + + while (n_threads > 1 && (spad_per_thread * n_threads) > vtcm_budget) { + n_threads--; + } + + htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_threads, n_slots); + return layout->total_bytes <= vtcm_budget; + } + + uint32_t n_vec = (ne00 + 31) / 32; + uint32_t n_vec_pow2 = 1; + while (n_vec_pow2 < n_vec) n_vec_pow2 <<= 1; + + uint32_t n_chunks = 1; + if (ne00 > 1024) { + while (n_chunks * 2 <= n_threads_max && n_chunks * 2 <= n_vec_pow2) { + n_chunks *= 2; + } + } + + uint32_t chunk_n_vec = n_vec_pow2 / n_chunks; + uint32_t chunk_elems = chunk_n_vec * 32; + uint32_t merge_elems = 0; + + if (n_chunks > 1) { + if (is_top_k) { + uint32_t local_k = k < chunk_elems ? k : chunk_elems; + uint32_t total_candidates = n_chunks * local_k; + uint32_t merge_n_vec = (total_candidates + 31) / 32; + uint32_t merge_n_vec_pow2 = 1; + while (merge_n_vec_pow2 < merge_n_vec) merge_n_vec_pow2 <<= 1; + merge_elems = merge_n_vec_pow2 * 32; + } else { + merge_elems = n_vec_pow2 * 32; + } + } + + htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, n_chunks, 1); + + if (layout->total_bytes > vtcm_budget && n_chunks > 1) { + n_chunks = 1; + chunk_elems = n_vec_pow2 * 32; + merge_elems = 0; + htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, 1, 1); + } + + if (layout->total_bytes > vtcm_budget) { + return false; + } + + if (n_chunks == 1 && layout->phase1_slot_size * 2 <= vtcm_budget) { + htp_sort_vtcm_layout_build(layout, 1, chunk_elems, 1, 0, 1, 2); + } + + return true; +} + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_sort_kernel_params) <= 128, "htp_sort_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_sort_kernel_params) <= 128, "htp_sort_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_ARGSORT_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.c b/ggml/src/ggml-hexagon/htp/binary-ops.c index 52013ad..48f461e 100644 --- a/ggml/src/ggml-hexagon/htp/binary-ops.c +++ b/ggml/src/ggml-hexagon/htp/binary-ops.c @@ -8,14 +8,17 @@ #include #include -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" +#include "binary-ops.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #ifndef MIN #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -24,6 +27,8 @@ // Context for binary operations struct htp_binary_context { struct htp_ops_context * octx; + struct htp_binary_vtcm_layout vtcm_layout; + uint8_t * vtcm_base; struct fastdiv_values src0_dim1_div; // ne01 struct fastdiv_values src0_dim2_div; // ne02 @@ -35,39 +40,45 @@ struct htp_binary_context { uint32_t block_max; uint32_t nrows_per_thread; + uint32_t total_rows; + uint32_t row_start; + uint32_t n_srcs; size_t src0_row_size_aligned; size_t src1_row_size_aligned; size_t dst_row_size_aligned; + size_t row_size_bytes; bool split_at_ne01; bool split_at_ne02; + + void * compute; }; #define htp_binary_preamble \ const struct htp_tensor * src0 = octx->src[0]; \ const struct htp_tensor * src1 = octx->src[1]; \ const struct htp_tensor * dst = octx->dst; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne10 = src1->ne[0]; \ - const uint32_t ne11 = src1->ne[1]; \ - const uint32_t ne12 = src1->ne[2]; \ - const uint32_t ne13 = src1->ne[3]; \ - \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb11 = src1->nb[1]; \ - const uint32_t nb12 = src1->nb[2]; \ - const uint32_t nb13 = src1->nb[3]; \ - \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t ne10 = src1->ne[0]; \ + const uint32_t ne11 = src1->ne[1]; \ + const uint32_t ne12 = src1->ne[2]; \ + const uint32_t ne13 = src1->ne[3]; \ + \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb11 = src1->nb[1]; \ + const uint32_t nb12 = src1->nb[2]; \ + const uint32_t nb13 = src1->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_t ir, uint32_t end_row, uint32_t ne01, uint32_t ne02) { @@ -91,115 +102,192 @@ static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_ return MIN(bctx->block_max, block_limit); } -// Macro for scalar op switch -#define COMPUTE_SCALAR_OP(DST, SRC, VAL, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_SUB: hvx_sub_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_MUL: hvx_mul_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \ - case HTP_OP_DIV: hvx_mul_scalar_f32_aa(DST, SRC, 1.0f / (*(float *)VAL), N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_SUB: hvx_sub_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_MUL: hvx_mul_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - case HTP_OP_DIV: hvx_div_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \ - default: break; \ - } \ - } +// Out-of-line compute micro-kernels + +typedef void (*compute_scalar_t)( + uint8_t * dst, const uint8_t * src0, const void * s1_table, + uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, + size_t dst_stride, size_t src0_stride, uint32_t ne00); + +#define DEFINE_COMPUTE_SCALAR(NAME, TYPE, HVX_STMT) \ +static void compute_scalar_##NAME( \ + uint8_t * dst, const uint8_t * src0, const void * s1_table, \ + uint32_t cur_i11, uint32_t ne11, uint32_t n_rows, \ + size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ + const TYPE * table = (const TYPE *) s1_table; \ + for (uint32_t r = 0; r < n_rows; r++) { \ + uint8_t * r_dst = dst + r * dst_stride; \ + const uint8_t * r_src0 = src0 + r * src0_stride; \ + TYPE val = table[cur_i11]; \ + HVX_STMT; \ + if (ne11 > 1 && ++cur_i11 == ne11) { \ + cur_i11 = 0; \ + } \ + } \ +} -// Macro for vector op switch (All Aligned) -#define COMPUTE_VECTOR_OP_AAA(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f32_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f32_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f32_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f32_aaa(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f16_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f16_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f16_aaa(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f16_aaa(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } +DEFINE_COMPUTE_SCALAR(add_f32, float, hvx_add_scalar_f32_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(add_f16, _Float16, hvx_add_scalar_f16_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(sub_f32, float, hvx_sub_scalar_f32_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(sub_f16, _Float16, hvx_sub_scalar_f16_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(mul_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(mul_f16, _Float16, hvx_mul_scalar_f16_aa(r_dst, r_src0, val, ne00)) +DEFINE_COMPUTE_SCALAR(div_f32, float, hvx_mul_scalar_f32_aa(r_dst, r_src0, 1.0f / (val), ne00)) +DEFINE_COMPUTE_SCALAR(div_f16, _Float16, hvx_div_scalar_f16_aa(r_dst, r_src0, val, ne00)) + +typedef void (*compute_same_shape_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, size_t src1_stride, uint32_t ne00); + +#define DEFINE_COMPUTE_SAME_SHAPE(NAME, HVX_FN) \ +static void compute_same_shape_##NAME( \ + uint8_t * dst, const uint8_t * src0, const uint8_t * src1, \ + uint32_t n_rows, size_t dst_stride, size_t src0_stride, size_t src1_stride, uint32_t ne00) { \ + for (uint32_t r = 0; r < n_rows; r++) { \ + HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, src1 + r * src1_stride, ne00); \ + } \ +} -// Macro for vector op switch (Dst Aligned, Src0 Aligned, Src1 Unaligned) -#define COMPUTE_VECTOR_OP_AAU(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f32_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f32_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f32_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f32_aau(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f16_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f16_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f16_aau(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f16_aau(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } +DEFINE_COMPUTE_SAME_SHAPE(add_f32, hvx_add_f32_aaa) +DEFINE_COMPUTE_SAME_SHAPE(add_f16, hvx_add_f16_aaa) +DEFINE_COMPUTE_SAME_SHAPE(sub_f32, hvx_sub_f32_aaa) +DEFINE_COMPUTE_SAME_SHAPE(sub_f16, hvx_sub_f16_aaa) +DEFINE_COMPUTE_SAME_SHAPE(mul_f32, hvx_mul_f32_aaa) +DEFINE_COMPUTE_SAME_SHAPE(mul_f16, hvx_mul_f16_aaa) +DEFINE_COMPUTE_SAME_SHAPE(div_f32, hvx_div_f32_aaa) +DEFINE_COMPUTE_SAME_SHAPE(div_f16, hvx_div_f16_aaa) + +typedef void (*compute_row_bcast_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); + +#define DEFINE_COMPUTE_ROW_BCAST(NAME, HVX_FN) \ +static void compute_row_bcast_##NAME( \ + uint8_t * dst, const uint8_t * src0, const uint8_t * src1, \ + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ + for (uint32_t r = 0; r < n_rows; r++) { \ + HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, src1, ne00); \ + } \ +} + +DEFINE_COMPUTE_ROW_BCAST(add_f32, hvx_add_f32_aaa) +DEFINE_COMPUTE_ROW_BCAST(add_f16, hvx_add_f16_aaa) +DEFINE_COMPUTE_ROW_BCAST(sub_f32, hvx_sub_f32_aaa) +DEFINE_COMPUTE_ROW_BCAST(sub_f16, hvx_sub_f16_aaa) +DEFINE_COMPUTE_ROW_BCAST(mul_f32, hvx_mul_f32_aaa) +DEFINE_COMPUTE_ROW_BCAST(mul_f16, hvx_mul_f16_aaa) +DEFINE_COMPUTE_ROW_BCAST(div_f32, hvx_div_f32_aaa) +DEFINE_COMPUTE_ROW_BCAST(div_f16, hvx_div_f16_aaa) + +typedef void (*compute_complex_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); + +#define DEFINE_COMPUTE_COMPLEX(NAME, HVX_FN) \ +static void compute_complex_##NAME( \ + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \ + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \ + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { \ + for (uint32_t r = 0; r < n_rows; r++) { \ + uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \ + const uint8_t * r_src1 = src1_plane + i11 * src1_stride; \ + HVX_FN(dst + r * dst_stride, src0 + r * src0_stride, r_src1, ne00); \ + } \ +} -// Macro for vector op switch (All Unaligned - generic loop used in element repeat) -#define COMPUTE_VECTOR_OP_UUU(DST, SRC0, SRC1, TYPE, N) \ - if(TYPE == HTP_TYPE_F32) { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f32_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f32_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f32_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f32_uuu(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ - } \ - else { \ - switch (octx->op) { \ - case HTP_OP_ADD: hvx_add_f16_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_SUB: hvx_sub_f16_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_MUL: hvx_mul_f16_uuu(DST, SRC0, SRC1, N); break; \ - case HTP_OP_DIV: hvx_div_f16_uuu(DST, SRC0, SRC1, N); break; \ - default: break; \ - } \ +DEFINE_COMPUTE_COMPLEX(add_f32, hvx_add_f32_aau) +DEFINE_COMPUTE_COMPLEX(add_f16, hvx_add_f16_aau) +DEFINE_COMPUTE_COMPLEX(sub_f32, hvx_sub_f32_aau) +DEFINE_COMPUTE_COMPLEX(sub_f16, hvx_sub_f16_aau) +DEFINE_COMPUTE_COMPLEX(mul_f32, hvx_mul_f32_aau) +DEFINE_COMPUTE_COMPLEX(mul_f16, hvx_mul_f16_aau) +DEFINE_COMPUTE_COMPLEX(div_f32, hvx_div_f32_aau) +DEFINE_COMPUTE_COMPLEX(div_f16, hvx_div_f16_aau) + +typedef void (*compute_repeat_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10); + +#define DEFINE_COMPUTE_REPEAT(NAME, TYPE, HVX_FN) \ +static void compute_repeat_##NAME( \ + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_plane, \ + uint32_t i01, uint32_t ne11, const struct fastdiv_values * div11, uint32_t src1_stride, \ + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00, uint32_t ne10) { \ + for (uint32_t r = 0; r < n_rows; r++) { \ + uint32_t i11 = fastmodulo(i01 + r, ne11, div11); \ + const uint8_t * r_src1_row = src1_plane + i11 * src1_stride; \ + uint8_t * r_dst = dst + r * dst_stride; \ + const uint8_t * r_src0 = src0 + r * src0_stride; \ + for (uint32_t c = 0; c < ne00; c += ne10) { \ + uint32_t len = MIN(ne10, ne00 - c); \ + HVX_FN(r_dst + c * sizeof(TYPE), r_src0 + c * sizeof(TYPE), r_src1_row, len); \ + } \ + } \ +} + +DEFINE_COMPUTE_REPEAT(add_f32, float, hvx_add_f32_uuu) +DEFINE_COMPUTE_REPEAT(add_f16, _Float16, hvx_add_f16_uuu) +DEFINE_COMPUTE_REPEAT(sub_f32, float, hvx_sub_f32_uuu) +DEFINE_COMPUTE_REPEAT(sub_f16, _Float16, hvx_sub_f16_uuu) +DEFINE_COMPUTE_REPEAT(mul_f32, float, hvx_mul_f32_uuu) +DEFINE_COMPUTE_REPEAT(mul_f16, _Float16, hvx_mul_f16_uuu) +DEFINE_COMPUTE_REPEAT(div_f32, float, hvx_div_f32_uuu) +DEFINE_COMPUTE_REPEAT(div_f16, _Float16, hvx_div_f16_uuu) + +typedef void (*compute_add_id_t)( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids, + uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00); + +static void compute_add_id_f32( + uint8_t * dst, const uint8_t * src0, const uint8_t * src1_data, const int32_t * src2_ids, + uint32_t i01, uint32_t i02, uint32_t ne01, uint32_t src1_stride, + uint32_t n_rows, size_t dst_stride, size_t src0_stride, uint32_t ne00) { + for (uint32_t r = 0; r < n_rows; r++) { + uint32_t r_i01 = i01 + r; + const int32_t idx = src2_ids[i02 * ne01 + r_i01]; + if (idx < 0) { + memcpy(dst + r * dst_stride, src0 + r * src0_stride, ne00 * sizeof(float)); + continue; + } + const uint8_t * r_src1 = src1_data + idx * src1_stride; + const uint8_t * r_src0 = src0 + r * src0_stride; + uint8_t * r_dst = dst + r * dst_stride; + hvx_add_f32_aaa(r_dst, r_src0, r_src1, ne00); } +} -// 1. Scalar src1 (ne10 == 1) -static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { +// 1. Scalar src1 in VTCM via DMA (ne10 == 1) +static void binary_thread_scalar(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const size_t plane_stride = (size_t) ne12 * ne11 * elem_size; + const size_t row_stride = (size_t) ne11 * elem_size; + + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; + const void * s1_table = VTCM_LAYOUT_PTR(const void, bctx->vtcm_base, layout->off_src1); - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; - // Preamble for (int k = 0; k < 2 && ir_prefetch < end_row; k++) { uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); uint32_t i03, i02, i01, rem; @@ -208,24 +296,27 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { i02 = fastdiv(rem, &bctx->src0_dim1_div); i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, + current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } - // Main loop + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_scalar_t compute = (compute_scalar_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i03, i02, i01, rem; i03 = fastdiv(ir, &bctx->src0_dim12_div); @@ -233,65 +324,64 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) { i02 = fastdiv(rem, &bctx->src0_dim1_div); i01 = rem - i02 * ne01; - // src1 indices (broadcast/repeat) uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); uint32_t i11 = fastmodulo(i01, ne11, &bctx->src1_dim1_div); + const uint8_t * scalar_table = (const uint8_t *) s1_table + + i13 * plane_stride + i12 * row_stride; - uint8_t * src1_ptr = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; - uint32_t s1_stride = (ne11 == 1) ? 0 : nb11; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, scalar_table, i11, ne11, current_block_size, + bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - for (uint32_t r = 0; r < current_block_size; r++) { - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; - COMPUTE_SCALAR_OP(r_dst, r_src0, src1_ptr, src0_type, ne00); - src1_ptr += s1_stride; - } - - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, + current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03, p02, p01, prem; - p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - prem = ir_prefetch - p03 * (ne02 * ne01); - p02 = fastdiv(prem, &bctx->src0_dim1_div); - p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - ir_prefetch += next_block_size; + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, + next_block_size); + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 2. Vector Same Shape (ne1x == ne0x) or Simple Broadcast -static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_same_shape(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * src1_spad_base = octx->src1_spad.data + (ith * octx->src1_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * src1_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t src1_spad_half = octx->src1_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + size_t src0_spad_half = layout->src0_spad_half_size; + size_t src1_spad_half = layout->src1_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; + const uint32_t n_srcs = bctx->n_srcs; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -303,98 +393,110 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi i02 = fastdiv(rem, &bctx->src0_dim1_div); i01 = rem - i02 * ne01; - uint32_t i13 = (ne13 == 1) ? 0 : i03; - uint32_t i12 = (ne12 == 1) ? 0 : i02; - uint32_t i11 = (ne11 == 1) ? 0 : i01; - - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * src1_curr = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * s1_spad = src1_spad_base + spad_idx * src1_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); - dma_queue_push(q, dma_make_ptr(s1_spad, src1_curr), bctx->src1_row_size_aligned, nb11, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + for (uint32_t s = 1; s < n_srcs; s++) { + const struct htp_tensor * src_s = octx->src[s]; + uint32_t is3 = (src_s->ne[3] == 1) ? 0 : i03; + uint32_t is2 = (src_s->ne[2] == 1) ? 0 : i02; + uint32_t is1 = (src_s->ne[1] == 1) ? 0 : i01; + dma_addr_t s_curr = src_s->data + is3 * src_s->nb[3] + is2 * src_s->nb[2] + is1 * src_s->nb[1]; + uint8_t * s_spad = s1_spad + (s - 1) * 2 * src1_spad_half; + dma_queue_push(dma_q, dma_make_data(s_spad, s_curr), bctx->src1_row_size_aligned, src_s->nb[1], row_size_bytes, current_block_size); + } ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_same_shape_t compute = (compute_same_shape_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; - uint8_t * s1_spad = (uint8_t *) dma_queue_pop(q).dst; - - for (uint32_t r = 0; r < current_block_size; r++) { - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_src1 = s1_spad + r * bctx->src1_row_size_aligned; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; - COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00); + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + uint8_t * s1_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, s1_spad, current_block_size, + bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, bctx->src1_row_size_aligned, ne00); + for (uint32_t s = 2; s < n_srcs; s++) { + uint8_t * ss_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + compute(d_spad, d_spad, ss_spad, current_block_size, + bctx->dst_row_size_aligned, bctx->dst_row_size_aligned, bctx->src1_row_size_aligned, ne00); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint32_t i03, i02, i01, rem; i03 = fastdiv(ir, &bctx->src0_dim12_div); rem = ir - i03 * (ne02 * ne01); i02 = fastdiv(rem, &bctx->src0_dim1_div); i01 = rem - i02 * ne01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03, p02, p01, prem; - p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - prem = ir_prefetch - p03 * (ne02 * ne01); - p02 = fastdiv(prem, &bctx->src0_dim1_div); - p01 = prem - p02 * ne01; - - uint32_t p13 = (ne13 == 1) ? 0 : p03; - uint32_t p12 = (ne12 == 1) ? 0 : p02; - uint32_t p11 = (ne11 == 1) ? 0 : p01; - - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - uint8_t * s1_next = (uint8_t *)src1->data + p13 * nb13 + p12 * nb12 + p11 * nb11; - - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - dma_queue_push(q, dma_make_ptr(s1_spad, s1_next), bctx->src1_row_size_aligned, nb11, row_size_bytes, next_block_size); + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + + for (uint32_t s = 1; s < n_srcs; s++) { + const struct htp_tensor * src_s = octx->src[s]; + uint32_t ps3 = (src_s->ne[3] == 1) ? 0 : p03; + uint32_t ps2 = (src_s->ne[2] == 1) ? 0 : p02; + uint32_t ps1 = (src_s->ne[1] == 1) ? 0 : p01; + dma_addr_t s_next = src_s->data + ps3 * src_s->nb[3] + ps2 * src_s->nb[2] + ps1 * src_s->nb[1]; + uint8_t * s_spad = s1_spad + (s - 1) * 2 * src1_spad_half; + dma_queue_push(dma_q, dma_make_data(s_spad, s_next), bctx->src1_row_size_aligned, src_s->nb[1], row_size_bytes, next_block_size); + } - ir_prefetch += next_block_size; + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 3. Row Broadcast (ne11 == 1, ne12 == 1, single row src1) -static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_row_broadcast(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * src1_spad_base = octx->src1_spad.data + (ith * octx->src1_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; - void * s1_ptr = (void *) src1_spad_base; + void * s1_ptr = VTCM_LAYOUT_PTR(void, bctx->vtcm_base, layout->off_src1); for (int k = 0; k < 2 && ir_prefetch < end_row; k++) { uint32_t current_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); @@ -403,73 +505,82 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith, uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_row_bcast_t compute = (compute_row_bcast_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; - - for (uint32_t r = 0; r < current_block_size; r++) { - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_src1 = (uint8_t *)s1_ptr; // Constant - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; - COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00); - } + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, (const uint8_t *)s1_ptr, current_block_size, + bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - uint32_t prem = ir_prefetch - p03 * (ne02 * ne01); - uint32_t p02 = fastdiv(prem, &bctx->src0_dim1_div); - uint32_t p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - ir_prefetch += next_block_size; + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 4. Vector Complex (ne10 == ne00, complex broadcast) -static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_vector_complex(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16); - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; + const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); - dma_queue * q = octx->ctx->dma[ith]; + const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const size_t vtcm_row_stride = (size_t) ne10 * elem_size; + const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride; + const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride; + + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -480,82 +591,87 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void * uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_complex_t compute = (compute_complex_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - for (uint32_t r = 0; r < current_block_size; r++) { - uint32_t r_i01 = i01 + r; - uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); - uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); - uint32_t i11 = fastmodulo(r_i01, ne11, &bctx->src1_dim1_div); - - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_src1 = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; + uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); + uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); + const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride; - // Read src1 from DDR (unaligned) - COMPUTE_VECTOR_OP_AAU(r_dst, r_src0, r_src1, src0_type, ne00); - } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride, + current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - uint32_t prem = ir_prefetch - p03 * (ne02 * ne01); - uint32_t p02 = fastdiv(prem, &bctx->src0_dim1_div); - uint32_t p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - ir_prefetch += next_block_size; + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 5. Element Repeat (ne10 != ne00) -static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_element_repeat(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; htp_binary_preamble; - const uint32_t src0_type = octx->src[0]->type; - const uint32_t elem_size_bytes = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); - const uint32_t row_size_bytes = ne00 * elem_size_bytes;; - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; + const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); - FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + const uint32_t elem_size = (src0->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const size_t vtcm_row_stride = (size_t) ne10 * elem_size; + const size_t vtcm_plane_stride = (size_t) ne11 * vtcm_row_stride; + const size_t vtcm_vol_stride = (size_t) ne12 * vtcm_plane_stride; - dma_queue * q = octx->ctx->dma[ith]; + FARF(HIGH, "binary-repeat: %d/%d (%u:%u) row-size %u (%u)", + ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned); + + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -566,100 +682,97 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void * uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + compute_repeat_t compute = (compute_repeat_t) bctx->compute; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - for (uint32_t r = 0; r < current_block_size; r++) { - uint32_t r_i01 = i01 + r; - uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); - uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); - uint32_t i11 = fastmodulo(r_i01, ne11, &bctx->src1_dim1_div); - - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_src1_row = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; - - // Repeat src1 row - for (uint32_t c = 0; c < ne00; c += ne10) { - uint32_t len = MIN(ne10, ne00 - c); - // Use UUU for speed and simplicity - COMPUTE_VECTOR_OP_UUU(r_dst + c * elem_size_bytes, r_src0 + c * elem_size_bytes, r_src1_row, src0_type, len); - } - } + uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div); + uint32_t i12 = fastmodulo(i02, ne12, &bctx->src1_dim2_div); + const uint8_t * src1_plane = vtcm_src1 + i13 * vtcm_vol_stride + i12 * vtcm_plane_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute(d_spad, s0_spad, src1_plane, i01, ne11, &bctx->src1_dim1_div, vtcm_row_stride, + current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00, ne10); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { - uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - uint32_t prem = ir_prefetch - p03 * (ne02 * ne01); - uint32_t p02 = fastdiv(prem, &bctx->src0_dim1_div); - uint32_t p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); - ir_prefetch += next_block_size; + uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); + ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); } // 6. ADD_ID (src1 gathered via src2 indices) -static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { +static void binary_thread_add_id_f32(unsigned int nth, unsigned int ith, void * data) { struct htp_binary_context * bctx = (struct htp_binary_context *) data; struct htp_ops_context * octx = bctx->octx; const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; - const struct htp_tensor * src2 = octx->src[2]; const struct htp_tensor * dst = octx->dst; const uint32_t ne00 = src0->ne[0]; const uint32_t ne01 = src0->ne[1]; const uint32_t ne02 = src0->ne[2]; const uint32_t ne03 = src0->ne[3]; - const uint32_t ne11 = src1->ne[1]; // for bounds check const uint32_t nb01 = src0->nb[1]; const uint32_t nb02 = src0->nb[2]; const uint32_t nb03 = src0->nb[3]; - const uint32_t nb11 = src1->nb[1]; // src1 row stride + const uint32_t src1_stride = bctx->src1_row_size_aligned; const uint32_t nb1 = dst->nb[1]; const uint32_t nb2 = dst->nb[2]; const uint32_t nb3 = dst->nb[3]; - const uint32_t total_rows = ne01 * ne02 * ne03; - const uint32_t start_row = bctx->nrows_per_thread * ith; - const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows); + const uint32_t row_size_bytes = bctx->row_size_bytes; + const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows); if (start_row >= end_row) return; - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - size_t src0_spad_half = octx->src0_spad.size_per_thread / 2; - size_t dst_spad_half = octx->dst_spad.size_per_thread / 2; + const struct htp_binary_vtcm_layout * layout = &bctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, bctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + const uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(const uint8_t, bctx->vtcm_base, layout->off_src1); + const int32_t * vtcm_src2 = VTCM_LAYOUT_PTR(const int32_t, bctx->vtcm_base, layout->off_src2); + size_t src0_spad_half = layout->src0_spad_half_size; + size_t dst_spad_half = layout->dst_spad_half_size; - dma_queue * q = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; uint32_t ir_prefetch = start_row; int spad_idx = 0; @@ -670,198 +783,604 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) { uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - uint8_t * src0_curr = (uint8_t *)src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_addr_t src0_curr = src0->data + i03 * nb03 + i02 * nb02 + i01 * nb01; + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; uint8_t * s0_spad = src0_spad_base + spad_idx * src0_spad_half; uint8_t * d_spad = dst_spad_base + spad_idx * dst_spad_half; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, ne00 * sizeof(float), 0); - dma_queue_push(q, dma_make_ptr(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, ne00 * sizeof(float), current_block_size); + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, src0_curr), bctx->src0_row_size_aligned, nb01, row_size_bytes, current_block_size); ir_prefetch += current_block_size; spad_idx ^= 1; } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = start_row; ir < end_row; ) { uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02); - uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src; - uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst; + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div); uint32_t rem = ir - i03 * (ne02 * ne01); uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div); uint32_t i01 = rem - i02 * ne01; - for (uint32_t r = 0; r < current_block_size; r++) { - uint32_t r_i01 = i01 + r; // linear within block since we split at ne01 - - const int32_t idx = *(int32_t *)((char *)src2->data + r_i01 * src2->nb[0] + i02 * src2->nb[1]); - - uint8_t * r_src1 = (uint8_t *)src1->data + idx * nb11; - uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned; - uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned; - - hvx_add_f32_aau(r_dst, r_src0, r_src1, ne00); - } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + compute_add_id_t compute = (compute_add_id_t) bctx->compute; + compute(d_spad, s0_spad, vtcm_src1, vtcm_src2, + i01, i02, ne01, src1_stride, + current_block_size, bctx->dst_row_size_aligned, bctx->src0_row_size_aligned, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; - dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, ne00 * sizeof(float), current_block_size); + dma_addr_t dst_curr = dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size); if (ir_prefetch < end_row) { uint32_t next_block_size = calc_block_size(bctx, ir_prefetch, end_row, ne01, ne02); - uint32_t p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); - uint32_t prem = ir_prefetch - p03 * (ne02 * ne01); - uint32_t p02 = fastdiv(prem, &bctx->src0_dim1_div); - uint32_t p01 = prem - p02 * ne01; - uint8_t * s0_next = (uint8_t *)src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; - dma_queue_push(q, dma_make_ptr(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, ne00 * sizeof(float), next_block_size); + uint32_t p03, p02, p01, prem; + p03 = fastdiv(ir_prefetch, &bctx->src0_dim12_div); + prem = ir_prefetch - p03 * (ne02 * ne01); + p02 = fastdiv(prem, &bctx->src0_dim1_div); + p01 = prem - p02 * ne01; + dma_addr_t s0_next = src0->data + p03 * nb03 + p02 * nb02 + p01 * nb01; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), bctx->src0_row_size_aligned, nb01, row_size_bytes, next_block_size); ir_prefetch += next_block_size; } ir += current_block_size; } - dma_queue_flush(q); + + dma_queue_flush(dma_q); +} + +static inline void hvx_div_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float val, const uint32_t num_elems) { + hvx_mul_scalar_f32(dst, src, 1.0f / val, num_elems); +} + +static inline void hvx_div_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, const uint32_t num_elems) { + hvx_div_scalar_f16_aa(dst, src, val, num_elems); +} + +typedef void (*compute_binary_chunked_t)( + uint8_t * restrict dst, + const uint8_t * restrict src0, + const uint8_t * restrict src1, + const uint32_t num_elems +); + +typedef void (*compute_binary_scalar_chunked_f32_t)( + uint8_t * restrict dst, + const uint8_t * restrict src0, + const float val, + const uint32_t num_elems +); + +typedef void (*compute_binary_scalar_chunked_f16_t)( + uint8_t * restrict dst, + const uint8_t * restrict src0, + const _Float16 val, + const uint32_t num_elems +); + +struct binary_chunked_context { + struct htp_ops_context * octx; + struct htp_binary_vtcm_layout vtcm_layout; + uint8_t * vtcm_base; + uint32_t elem_start; + uint32_t nelem; + uint32_t chunk_size; + uint32_t chunks_per_thread; + uint32_t total_chunks; + uint32_t n_srcs; + bool is_scalar; + float scalar_f32; + _Float16 scalar_f16; + compute_binary_chunked_t compute; + compute_binary_scalar_chunked_f32_t compute_scalar_f32; + compute_binary_scalar_chunked_f16_t compute_scalar_f16; +}; + +static void binary_thread_chunked(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct binary_chunked_context * ctx = (struct binary_chunked_context *) data; + struct htp_ops_context * octx = ctx->octx; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t start_chunk = ctx->chunks_per_thread * ith; + const uint32_t end_chunk = MIN(start_chunk + ctx->chunks_per_thread, ctx->total_chunks); + if (start_chunk >= end_chunk) { + return; + } + + const uint32_t src0_type = src0->type; + const size_t elem_size = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); + const uint32_t chunk_size = ctx->chunk_size; + const size_t chunk_bytes = ctx->vtcm_layout.src0_spad_half_size; + + FARF(HIGH, "binary-chunked: %d/%d (%u:%u) chunks %u elems %u", + ith, nth, start_chunk, end_chunk, ctx->total_chunks, ctx->nelem); + + const struct htp_binary_vtcm_layout * layout = &ctx->vtcm_layout; + uint8_t * src0_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src0) + (ith * layout->src0_bytes_per_thread); + uint8_t * src1_spad_base = ctx->is_scalar ? NULL : (VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_src1) + (ith * layout->src1_bytes_per_thread)); + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, ctx->vtcm_base, layout->off_dst) + (ith * layout->dst_bytes_per_thread); + + dma_queue * dma_q = octx->ctx->dma[ith]; + const uint32_t n_srcs = ctx->n_srcs; + uint32_t prefetch_chunk = start_chunk; + int spad_idx = 0; + + for (int k = 0; k < 2 && prefetch_chunk < end_chunk; k++) { + const uint32_t c_start = ctx->elem_start + prefetch_chunk * chunk_size; + const uint32_t c_end = MIN(c_start + chunk_size, ctx->elem_start + ctx->nelem); + const uint32_t cur_elems = c_end - c_start; + const uint32_t cur_bytes = cur_elems * elem_size; + + dma_addr_t s0_curr = src0->data + (size_t) c_start * elem_size; + dma_addr_t d_curr = dst->data + (size_t) c_start * elem_size; + + uint8_t * s0_spad = src0_spad_base + spad_idx * chunk_bytes; + uint8_t * d_spad = dst_spad_base + spad_idx * chunk_bytes; + + dma_queue_push(dma_q, dma_make_data(d_curr, d_spad), chunk_bytes, chunk_bytes, cur_bytes, 0); + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_curr), chunk_bytes, chunk_bytes, cur_bytes, 1); + if (!ctx->is_scalar) { + uint8_t * s1_spad = src1_spad_base + spad_idx * chunk_bytes; + for (uint32_t s = 1; s < n_srcs; s++) { + dma_addr_t s_curr = octx->src[s]->data + (size_t) c_start * elem_size; + uint8_t * s_spad = s1_spad + (s - 1) * 2 * chunk_bytes; + dma_queue_push(dma_q, dma_make_data(s_spad, s_curr), chunk_bytes, chunk_bytes, cur_bytes, 1); + } + } + + prefetch_chunk++; + spad_idx ^= 1; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + for (uint32_t c = start_chunk; c < end_chunk; c++) { + const uint32_t c_start = ctx->elem_start + c * chunk_size; + const uint32_t c_end = MIN(c_start + chunk_size, ctx->elem_start + ctx->nelem); + const uint32_t cur_elems = c_end - c_start; + const uint32_t cur_bytes = cur_elems * elem_size; + + uint8_t * d_spad = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * s0_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + uint8_t * s1_spad = ctx->is_scalar ? NULL : (uint8_t *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) c); + if (ctx->is_scalar) { + if (src0_type == HTP_TYPE_F32) { + ctx->compute_scalar_f32(d_spad, s0_spad, ctx->scalar_f32, cur_elems); + } else { + ctx->compute_scalar_f16(d_spad, s0_spad, ctx->scalar_f16, cur_elems); + } + } else { + ctx->compute(d_spad, s0_spad, s1_spad, cur_elems); + for (uint32_t s = 2; s < n_srcs; s++) { + uint8_t * s_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + ctx->compute(d_spad, d_spad, s_spad, cur_elems); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) c); + + dma_addr_t dst_curr = dst->data + (size_t) c_start * elem_size; + dma_queue_push(dma_q, dma_make_data(dst_curr, d_spad), chunk_bytes, chunk_bytes, cur_bytes, 1); + + if (prefetch_chunk < end_chunk) { + const uint32_t pc_start = ctx->elem_start + prefetch_chunk * chunk_size; + const uint32_t pc_end = MIN(pc_start + chunk_size, ctx->elem_start + ctx->nelem); + const uint32_t p_elems = pc_end - pc_start; + const uint32_t p_bytes = p_elems * elem_size; + + dma_addr_t s0_next = src0->data + (size_t) pc_start * elem_size; + dma_queue_push(dma_q, dma_make_data(s0_spad, s0_next), chunk_bytes, chunk_bytes, p_bytes, 1); + if (!ctx->is_scalar) { + for (uint32_t s = 1; s < n_srcs; s++) { + dma_addr_t s_next = octx->src[s]->data + (size_t) pc_start * elem_size; + uint8_t * s_spad = s1_spad + (s - 1) * 2 * chunk_bytes; + dma_queue_push(dma_q, dma_make_data(s_spad, s_next), chunk_bytes, chunk_bytes, p_bytes, 1); + } + } + + prefetch_chunk++; + } + } + + dma_queue_flush(dma_q); } static int execute_op_binary(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; + const struct htp_binary_kernel_params * kparams = (const struct htp_binary_kernel_params *) octx->kernel_params; + const uint32_t n_srcs = kparams->n_srcs > 2 ? kparams->n_srcs : 2; const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - // Use packed row sizes for VTCM allocation + // Use packed row sizes for VTCM allocation and alignment const uint32_t src0_type = octx->src[0]->type; const size_t elem_size = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16); const size_t src0_row_size = src0->ne[0] * elem_size; const size_t src1_row_size = src1->ne[0] * elem_size; const size_t dst_row_size = dst->ne[0] * elem_size; - size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN); - size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); + if (kparams->kernel_type == HTP_BINARY_KERNEL_CHUNKED) { + const bool is_scalar = kparams->is_scalar; + + compute_binary_chunked_t compute = NULL; + compute_binary_scalar_chunked_f32_t compute_scalar_f32 = NULL; + compute_binary_scalar_chunked_f16_t compute_scalar_f16 = NULL; + + if (is_scalar) { + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_scalar_f32 = hvx_add_scalar_f32; break; + case HTP_OP_SUB: compute_scalar_f32 = hvx_sub_scalar_f32; break; + case HTP_OP_MUL: compute_scalar_f32 = hvx_mul_scalar_f32; break; + case HTP_OP_DIV: compute_scalar_f32 = hvx_div_scalar_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_scalar_f16 = hvx_add_scalar_f16; break; + case HTP_OP_SUB: compute_scalar_f16 = hvx_sub_scalar_f16; break; + case HTP_OP_MUL: compute_scalar_f16 = hvx_mul_scalar_f16; break; + case HTP_OP_DIV: compute_scalar_f16 = hvx_div_scalar_f16; break; + default: break; + } + } + if (!compute_scalar_f32 && !compute_scalar_f16) { + return HTP_STATUS_NO_SUPPORT; + } + } else { + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute = hvx_add_f32; break; + case HTP_OP_SUB: compute = hvx_sub_f32; break; + case HTP_OP_MUL: compute = hvx_mul_f32; break; + case HTP_OP_DIV: compute = hvx_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute = hvx_add_f16; break; + case HTP_OP_SUB: compute = hvx_sub_f16; break; + case HTP_OP_MUL: compute = hvx_mul_f16; break; + case HTP_OP_DIV: compute = hvx_div_f16; break; + default: break; + } + } + if (!compute) { + return HTP_STATUS_NO_SUPPORT; + } + } - bool is_add_id = (octx->op == HTP_OP_ADD_ID); - bool is_scalar = !is_add_id && (src1->ne[0] == 1); + const uint32_t total_elems = (uint32_t) (src0->ne[0] * src0->ne[1] * src0->ne[2] * src0->ne[3]); + if (total_elems == 0) { + return HTP_STATUS_OK; + } - bool is_transposed = (src0->nb[1] < src0_row_size || src1->nb[1] < src1_row_size || dst->nb[1] < dst_row_size); + uint32_t elem_start = 0; + uint32_t nelem = total_elems; + const uint32_t elems_per_line = (src0_type == HTP_TYPE_F32) ? 32 : 64; + + if (octx->ctx->mdev.count > 1) { + bool can_split = htp_tensor_mdev_data_aligned(dst) && + htp_tensor_mdev_data_aligned(src0) && + (is_scalar || htp_tensor_mdev_data_aligned(src1)) && + htp_tensor_is_contiguous(dst, elem_size) && + htp_tensor_is_contiguous(src0, elem_size) && + (is_scalar || htp_tensor_is_contiguous(src1, elem_size)); + for (uint32_t s = 2; can_split && s < n_srcs; s++) { + can_split = htp_tensor_mdev_data_aligned(octx->src[s]) && + htp_tensor_is_contiguous(octx->src[s], elem_size); + } + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_elems, can_split ? elems_per_line : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelem = range.count; + } - bool is_same_shape = !is_add_id && !is_scalar && !is_transposed && - (src1->ne[0] == src0->ne[0] && src0->ne[0] % VLEN == 0) && - (src1->ne[1] == src0->ne[1] || src1->ne[1] == 1) && - (src1->ne[2] == src0->ne[2] || src1->ne[2] == 1) && - (src1->ne[3] == src0->ne[3] || src1->ne[3] == 1); + if (nelem == 0) { + return HTP_STATUS_OK; + } - bool is_row_bcast = is_same_shape && (src1->ne[1] == 1 && src1->ne[2] == 1 && src1->ne[3] == 1); - bool is_complex = !is_add_id && !is_scalar && !is_same_shape && (src1->ne[0] == src0->ne[0]); - bool is_repeat = !is_add_id && !is_scalar && !is_same_shape && (src1->ne[0] != src0->ne[0]); + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } - size_t spad_row_total; - if (is_same_shape) { - spad_row_total = 2 * (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned); - } else { - spad_row_total = 2 * (src0_row_size_aligned + dst_row_size_aligned); - } + struct htp_binary_vtcm_layout vtcm_layout; + htp_binary_vtcm_layout_build(&vtcm_layout, kparams, octx->ctx->vtcm_size); + if (vtcm_layout.total_bytes == 0 || vtcm_layout.total_bytes > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } - size_t rows_per_buffer = octx->ctx->vtcm_size / (n_threads * spad_row_total); + const uint32_t chunk_size = kparams->chunk_size; + const uint32_t total_chunks = (nelem + chunk_size - 1) / chunk_size; + const uint32_t n_threads = (total_chunks >= 2) ? MIN(octx->n_threads, total_chunks) : 1; + const uint32_t chunks_per_thread = (total_chunks + n_threads - 1) / n_threads; + + float scalar_f32 = 0.0f; + _Float16 scalar_f16 = 0; + if (is_scalar) { + uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, vtcm_layout.off_src1); + dma_queue * dma_q = octx->ctx->dma[0]; + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), 128, 0, elem_size, 1); + dma_queue_pop(dma_q); + + if (src0_type == HTP_TYPE_F32) { + scalar_f32 = ((const float *) vtcm_src1)[0]; + } else { + scalar_f16 = ((const _Float16 *) vtcm_src1)[0]; + } + } - // Adjust for static src1 in row_bcast case - if (is_row_bcast) { - size_t needed_static = src1_row_size_aligned; - if (octx->ctx->vtcm_size < needed_static) return HTP_STATUS_VTCM_TOO_SMALL; - size_t avail = octx->ctx->vtcm_size - needed_static; - rows_per_buffer = avail / (n_threads * spad_row_total); + struct binary_chunked_context cctx = { + .octx = octx, + .vtcm_layout = vtcm_layout, + .vtcm_base = (uint8_t *) octx->ctx->vtcm_base, + .elem_start = elem_start, + .nelem = nelem, + .chunk_size = chunk_size, + .chunks_per_thread = chunks_per_thread, + .total_chunks = total_chunks, + .n_srcs = n_srcs, + .is_scalar = is_scalar, + .scalar_f32 = scalar_f32, + .scalar_f16 = scalar_f16, + .compute = compute, + .compute_scalar_f32 = compute_scalar_f32, + .compute_scalar_f16 = compute_scalar_f16, + }; + + work_queue_run(octx->ctx->work_queue, binary_thread_chunked, &cctx, n_threads); + return HTP_STATUS_OK; } - if (rows_per_buffer < 1) { - FARF(ERROR, "binary: VTCM too small\n"); - return HTP_STATUS_VTCM_TOO_SMALL; - } + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; - octx->src0_spad.size_per_thread = rows_per_buffer * 2 * src0_row_size_aligned; - octx->dst_spad.size_per_thread = rows_per_buffer * 2 * dst_row_size_aligned; + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } - if (is_add_id || is_scalar || is_complex || is_repeat || is_row_bcast) { - octx->src1_spad.size_per_thread = 0; - } else { - octx->src1_spad.size_per_thread = rows_per_buffer * 2 * src1_row_size_aligned; + if (nrows == 0) { + return HTP_STATUS_OK; } - octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread; - octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread; - if (is_row_bcast) { - octx->src1_spad.size = src1_row_size_aligned; - } else { - octx->src1_spad.size = n_threads * octx->src1_spad.size_per_thread; + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; } - if (octx->ctx->vtcm_size < (octx->src0_spad.size + octx->src1_spad.size + octx->dst_spad.size)) { + const uint32_t n_threads = octx->n_threads; + const size_t src0_row_size_aligned = kparams->src0_row_size_aligned; + const size_t src1_row_size_aligned = kparams->src1_row_size_aligned; + const size_t dst_row_size_aligned = kparams->dst_row_size_aligned; + + struct htp_binary_context bctx; + bctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + htp_binary_vtcm_layout_build(&bctx.vtcm_layout, kparams, octx->ctx->vtcm_size); + + if (bctx.vtcm_layout.rows_per_buffer == 0 || bctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL; - octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->dst_spad.src = NULL; + worker_callback_t worker_func = NULL; + void * compute_func = NULL; + + switch (kparams->kernel_type) { + case HTP_BINARY_KERNEL_SAME_SHAPE: + worker_func = binary_thread_vector_same_shape; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_same_shape_add_f32; break; + case HTP_OP_SUB: compute_func = compute_same_shape_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_same_shape_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_same_shape_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_same_shape_add_f16; break; + case HTP_OP_SUB: compute_func = compute_same_shape_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_same_shape_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_same_shape_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_ROW_BCAST: + worker_func = binary_thread_vector_row_broadcast; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_row_bcast_add_f32; break; + case HTP_OP_SUB: compute_func = compute_row_bcast_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_row_bcast_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_row_bcast_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_row_bcast_add_f16; break; + case HTP_OP_SUB: compute_func = compute_row_bcast_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_row_bcast_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_row_bcast_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_SCALAR: + worker_func = binary_thread_scalar; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_scalar_add_f32; break; + case HTP_OP_SUB: compute_func = compute_scalar_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_scalar_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_scalar_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_scalar_add_f16; break; + case HTP_OP_SUB: compute_func = compute_scalar_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_scalar_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_scalar_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_COMPLEX: + worker_func = binary_thread_vector_complex; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_complex_add_f32; break; + case HTP_OP_SUB: compute_func = compute_complex_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_complex_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_complex_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_complex_add_f16; break; + case HTP_OP_SUB: compute_func = compute_complex_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_complex_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_complex_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_REPEAT: + worker_func = binary_thread_element_repeat; + if (src0_type == HTP_TYPE_F32) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_repeat_add_f32; break; + case HTP_OP_SUB: compute_func = compute_repeat_sub_f32; break; + case HTP_OP_MUL: compute_func = compute_repeat_mul_f32; break; + case HTP_OP_DIV: compute_func = compute_repeat_div_f32; break; + default: break; + } + } else if (src0_type == HTP_TYPE_F16) { + switch (octx->op) { + case HTP_OP_ADD: compute_func = compute_repeat_add_f16; break; + case HTP_OP_SUB: compute_func = compute_repeat_sub_f16; break; + case HTP_OP_MUL: compute_func = compute_repeat_mul_f16; break; + case HTP_OP_DIV: compute_func = compute_repeat_div_f16; break; + default: break; + } + } + break; + case HTP_BINARY_KERNEL_ADD_ID: + if (octx->op == HTP_OP_ADD_ID && src0_type == HTP_TYPE_F32) { + worker_func = binary_thread_add_id_f32; + compute_func = (void *) compute_add_id_f32; + } + break; + default: break; + } - if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - return HTP_STATUS_OK; + if (!worker_func || !compute_func) { + return HTP_STATUS_NO_SUPPORT; } - dma_queue * q = octx->ctx->dma[0]; - if (is_row_bcast) { - dma_queue_push(q, dma_make_ptr(octx->src1_spad.data, (const void *) src1->data), src1_row_size_aligned, 0, src1->ne[0] * elem_size, 1); + dma_queue * dma_q = octx->ctx->dma[0]; + uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src1); + if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST) { + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1->ne[0] * elem_size, 1); + } else if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR || + kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT || + kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX) { + if (!kparams->src1_strided) { + const size_t src1_bytes = (size_t) src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3] * elem_size; + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), bctx.vtcm_layout.src1_size, 0, src1_bytes, 1); + if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR && octx->src[2] != NULL) { + uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2); + dma_queue_push(dma_q, dma_make_data(vtcm_src2, octx->src[2]->data), bctx.vtcm_layout.src2_size, 0, src1_bytes, 1); + } + } else { + const uint32_t ne11 = src1->ne[1]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t ne13 = src1->ne[3]; + const size_t row_bytes = src1->ne[0] * elem_size; + for (uint32_t i13 = 0; i13 < ne13; i13++) { + for (uint32_t i12 = 0; i12 < ne12; i12++) { + dma_addr_t src_addr = src1->data + i13 * src1->nb[3] + i12 * src1->nb[2]; + uint8_t * dst_ptr = vtcm_src1 + (i13 * ne12 * ne11 + i12 * ne11) * row_bytes; + while (!dma_queue_push(dma_q, dma_make_data(dst_ptr, src_addr), + row_bytes, src1->nb[1], row_bytes, ne11)) { + dma_queue_pop(dma_q); + } + } + } + } + } else if (kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) { + const struct htp_tensor * src2 = octx->src[2]; + uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2); + dma_queue_push(dma_q, dma_make_data(vtcm_src1, src1->data), + kparams->src1_row_size_aligned, src1->nb[1], + src1->ne[0] * elem_size, src1->ne[1]); + const size_t src2_row_bytes = src2->ne[0] * sizeof(int32_t); + dma_queue_push(dma_q, dma_make_data(vtcm_src2, src2->data), + src2_row_bytes, src2->nb[1], + src2_row_bytes, src2->ne[1]); } - struct htp_binary_context bctx; bctx.octx = octx; - bctx.nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - bctx.block_max = rows_per_buffer; + bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + bctx.total_rows = nrows; + bctx.row_start = row_start; + bctx.n_srcs = n_srcs; + bctx.block_max = bctx.vtcm_layout.rows_per_buffer; bctx.src0_row_size_aligned = src0_row_size_aligned; bctx.src1_row_size_aligned = src1_row_size_aligned; bctx.dst_row_size_aligned = dst_row_size_aligned; - - bctx.src0_dim1_div = init_fastdiv_values(src0->ne[1]); - bctx.src0_dim2_div = init_fastdiv_values(src0->ne[2]); - bctx.src0_dim12_div = init_fastdiv_values(src0->ne[1] * src0->ne[2]); - - bctx.src1_dim1_div = init_fastdiv_values(src1->ne[1]); - bctx.src1_dim2_div = init_fastdiv_values(src1->ne[2]); - bctx.src1_dim3_div = init_fastdiv_values(src1->ne[3]); - - bool src0_contig_dim1 = (src0->nb[2] == src0->ne[1] * src0->nb[1]); - bool dst_contig_dim1 = (dst->nb[2] == src0->ne[1] * dst->nb[1]); - - bool src0_contig_dim2 = (src0->nb[3] == src0->ne[2] * src0->nb[2]); - bool dst_contig_dim2 = (dst->nb[3] == src0->ne[2] * dst->nb[2]); - - bctx.split_at_ne01 = (src0->ne[2] > 1) && ((src1->ne[1] > 1) || (src1->ne[2] > 1) || !src0_contig_dim1 || !dst_contig_dim1); - bctx.split_at_ne02 = (src0->ne[3] > 1) && ((src1->ne[2] > 1) || (src1->ne[3] > 1) || !src0_contig_dim2 || !dst_contig_dim2); - - worker_callback_t worker_func; - if (is_add_id) worker_func = binary_job_add_id; - else if (is_scalar) worker_func = binary_job_scalar; - else if (is_row_bcast) worker_func = binary_job_vector_row_broadcast; - else if (is_same_shape) worker_func = binary_job_vector_same_shape; - else if (is_complex) worker_func = binary_job_vector_complex; - else worker_func = binary_job_element_repeat; - - if (is_row_bcast) { - dma_queue_pop(q); + bctx.row_size_bytes = src0_row_size; + + bctx.src0_dim1_div = kparams->div_ne01; + bctx.src0_dim2_div = kparams->div_ne02; + bctx.src0_dim12_div = kparams->div_ne012; + + bctx.src1_dim1_div = kparams->div_ne11; + bctx.src1_dim2_div = kparams->div_ne12; + bctx.src1_dim3_div = kparams->div_ne13; + + bctx.split_at_ne01 = kparams->split_at_ne01; + bctx.split_at_ne02 = kparams->split_at_ne02; + bctx.compute = compute_func; + + if (kparams->kernel_type == HTP_BINARY_KERNEL_ROW_BCAST || + kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR || + kparams->kernel_type == HTP_BINARY_KERNEL_REPEAT || + kparams->kernel_type == HTP_BINARY_KERNEL_COMPLEX || + kparams->kernel_type == HTP_BINARY_KERNEL_ADD_ID) { + dma_queue_flush(dma_q); + if (kparams->kernel_type == HTP_BINARY_KERNEL_SCALAR && octx->src[2] != NULL && octx->op == HTP_OP_MUL) { + uint8_t * vtcm_src2 = VTCM_LAYOUT_PTR(uint8_t, bctx.vtcm_base, bctx.vtcm_layout.off_src2); + const uint32_t n_scalars = src1->ne[0] * src1->ne[1] * src1->ne[2] * src1->ne[3]; + if (src0_type == HTP_TYPE_F32) { + hvx_mul_f32_aaa(vtcm_src1, vtcm_src1, vtcm_src2, n_scalars); + } else if (src0_type == HTP_TYPE_F16) { + hvx_mul_f16_aaa(vtcm_src1, vtcm_src1, vtcm_src2, n_scalars); + } + } } - worker_pool_run_func(octx->ctx->worker_pool, worker_func, &bctx, n_threads); + work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads); return HTP_STATUS_OK; } int op_binary(struct htp_ops_context * octx) { - - // Does not support permutations of src1 - const struct htp_tensor * src1 = octx->src[1]; - if (src1->nb[1] < src1->nb[0]) { - return HTP_STATUS_NO_SUPPORT; - } - const uint32_t src0_type = octx->src[0]->type; if ((src0_type == HTP_TYPE_F32) || (src0_type == HTP_TYPE_F16)) { return execute_op_binary(octx); @@ -869,4 +1388,3 @@ int op_binary(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - diff --git a/ggml/src/ggml-hexagon/htp/binary-ops.h b/ggml/src/ggml-hexagon/htp/binary-ops.h new file mode 100644 index 0000000..8c5e683 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/binary-ops.h @@ -0,0 +1,247 @@ +#ifndef HTP_BINARY_OPS_H +#define HTP_BINARY_OPS_H + +#include +#include +#include + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-ops.h" +#include "htp-vtcm.h" + +enum htp_binary_kernel_type { + HTP_BINARY_KERNEL_SAME_SHAPE = 0, + HTP_BINARY_KERNEL_ROW_BCAST, + HTP_BINARY_KERNEL_SCALAR, + HTP_BINARY_KERNEL_ADD_ID, + HTP_BINARY_KERNEL_COMPLEX, + HTP_BINARY_KERNEL_REPEAT, + HTP_BINARY_KERNEL_CHUNKED, +}; + +struct htp_binary_kernel_params { + uint8_t kernel_type; + uint8_t split_at_ne01; + uint8_t split_at_ne02; + uint8_t src1_strided; + + uint8_t n_srcs; + uint8_t is_scalar; + uint8_t pad[2]; + + uint32_t n_threads; + uint32_t rows_per_buffer; + + uint32_t src0_row_size_aligned; + uint32_t src1_row_size_aligned; + uint32_t dst_row_size_aligned; + + uint32_t src1_size; + uint32_t src2_size; + uint32_t vtcm_size; + + uint32_t chunk_size; + uint32_t chunk_bytes; + + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne012; + + struct fastdiv_values div_ne11; + struct fastdiv_values div_ne12; + struct fastdiv_values div_ne13; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob"); + +struct ggml_tensor; +static inline bool htp_binary_strides_safe( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + size_t elem_size +) { + if ((src0->ne[0] != 1 && src0->nb[0] != elem_size) || + (src1->ne[0] != 1 && src1->nb[0] != elem_size) || + (dst->ne[0] != 1 && dst->nb[0] != elem_size)) { + return false; + } + if ((src0->ne[1] > 1 && src0->nb[1] < src0->ne[0] * elem_size) || + (src1->ne[1] > 1 && src1->nb[1] < src1->ne[0] * elem_size) || + (dst->ne[1] > 1 && dst->nb[1] < dst->ne[0] * elem_size)) { + return false; + } + return true; +} +#else +_Static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_binary_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_src1; + size_t off_src2; + size_t off_dst; + + size_t src0_bytes_per_thread; + size_t src1_bytes_per_thread; + size_t dst_bytes_per_thread; + + size_t src0_spad_half_size; + size_t src1_spad_half_size; + size_t dst_spad_half_size; + + size_t src1_size; + size_t src2_size; + uint32_t rows_per_buffer; +}; + +static inline void htp_binary_vtcm_layout_build( + struct htp_binary_vtcm_layout * L, + const struct htp_binary_kernel_params * kparams, + size_t vtcm_size +) { + memset(L, 0, sizeof(*L)); + + const uint32_t n_threads = kparams->n_threads; + if (n_threads == 0) { + return; + } + + const uint32_t n_srcs = kparams->n_srcs > 2 ? kparams->n_srcs : 2; + + if (kparams->kernel_type == HTP_BINARY_KERNEL_CHUNKED) { + const size_t chunk_bytes = kparams->chunk_bytes; + if (chunk_bytes == 0) { + return; + } + + L->src0_bytes_per_thread = 2 * chunk_bytes; + L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes); + L->dst_bytes_per_thread = 2 * chunk_bytes; + + L->src0_spad_half_size = chunk_bytes; + L->src1_spad_half_size = kparams->is_scalar ? 0 : chunk_bytes; + L->dst_spad_half_size = chunk_bytes; + + L->rows_per_buffer = 1; + L->src1_size = 0; + + const size_t src0_total = n_threads * L->src0_bytes_per_thread; + const size_t src1_total = kparams->is_scalar ? 128 : (n_threads * L->src1_bytes_per_thread); + const size_t dst_total = n_threads * L->dst_bytes_per_thread; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); + VTCM_LAYOUT_ALLOC(off, off_src1, src1_total); + VTCM_LAYOUT_ALLOC(off, off_dst, dst_total); + + if (off > vtcm_size) { + return; + } + + L->total_bytes = off; + return; + } + + const size_t spad_row_total = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) + ? 2 * (kparams->src0_row_size_aligned + (n_srcs - 1) * kparams->src1_row_size_aligned + kparams->dst_row_size_aligned) + : 2 * (kparams->src0_row_size_aligned + kparams->dst_row_size_aligned); + + const size_t static_vtcm_size = kparams->src1_size + kparams->src2_size; + if (spad_row_total == 0 || vtcm_size < static_vtcm_size) { + return; + } + + const size_t rows_per_buffer = (kparams->rows_per_buffer > 0) + ? kparams->rows_per_buffer + : (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total); + if (rows_per_buffer == 0) { + return; + } + + L->rows_per_buffer = (uint32_t) rows_per_buffer; + L->src1_size = kparams->src1_size; + L->src2_size = kparams->src2_size; + + L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned; + L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned; + L->src1_bytes_per_thread = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) + ? rows_per_buffer * 2 * (n_srcs - 1) * kparams->src1_row_size_aligned + : 0; + + L->src0_spad_half_size = L->src0_bytes_per_thread / 2; + L->src1_spad_half_size = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) + ? rows_per_buffer * kparams->src1_row_size_aligned + : 0; + L->dst_spad_half_size = L->dst_bytes_per_thread / 2; + + const size_t src0_total = n_threads * L->src0_bytes_per_thread; + const size_t src1_total = (kparams->src1_size > 0) + ? kparams->src1_size + : n_threads * L->src1_bytes_per_thread; + const size_t src2_total = kparams->src2_size; + const size_t dst_total = n_threads * L->dst_bytes_per_thread; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, src0_total); + VTCM_LAYOUT_ALLOC(off, off_src1, src1_total); + VTCM_LAYOUT_ALLOC(off, off_src2, src2_total); + VTCM_LAYOUT_ALLOC(off, off_dst, dst_total); + + L->total_bytes = off; +} + +static inline bool htp_binary_solve_chunked( + struct htp_binary_vtcm_layout * L, + struct htp_binary_kernel_params * kparams, + uint32_t total_elems, + uint32_t elem_size, + uint32_t max_threads, + uint32_t n_srcs, + bool is_scalar, + size_t vtcm_size +) { + const size_t n_bufs = is_scalar ? 2 : (size_t) n_srcs + 1; + const size_t scalar_bytes = is_scalar ? 128 : 0; + const size_t budget_per_buf = (max_threads > 0 && vtcm_size > scalar_bytes) + ? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_threads * n_bufs), 128) + : 0; + const uint32_t max_chunk_bytes = (budget_per_buf > 0 && budget_per_buf < 32768) ? (uint32_t) budget_per_buf : 32768u; + const uint32_t max_chunk_elems = max_chunk_bytes / elem_size; + if (max_chunk_elems == 0) { + return false; + } + + const uint32_t min_chunk_elems = 256 < max_chunk_elems ? 256 : max_chunk_elems; + const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads) - 1) / (2 * max_threads), 32); + const uint32_t chunk_size = (target_chunk_elems < min_chunk_elems) ? min_chunk_elems + : (target_chunk_elems > max_chunk_elems) ? max_chunk_elems + : target_chunk_elems; + const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128); + const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size; + const uint32_t n_threads = (total_chunks >= 2) ? (max_threads < total_chunks ? max_threads : total_chunks) : 1; + + kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED; + kparams->n_threads = n_threads; + kparams->n_srcs = (uint8_t) n_srcs; + kparams->rows_per_buffer = 1; + kparams->src1_row_size_aligned = is_scalar ? 0 : kparams->src0_row_size_aligned; + kparams->src1_size = 0; + kparams->chunk_size = chunk_size; + kparams->chunk_bytes = chunk_bytes; + kparams->is_scalar = is_scalar ? 1 : 0; + + htp_binary_vtcm_layout_build(L, kparams, vtcm_size); + if (L->total_bytes == 0 || L->total_bytes > vtcm_size) { + return false; + } + + kparams->vtcm_size = (uint32_t) L->total_bytes; + return true; +} + +#endif diff --git a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake index 7fa236e..3eff2a3 100644 --- a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +++ b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake @@ -3,7 +3,7 @@ if (HEXAGON_TOOLCHAIN_INCLUDED) endif() set(HEXAGON_TOOLCHAIN_INCLUDED true) -#Cross Compiling for Hexagon +# Cross Compiling for Hexagon set(HEXAGON TRUE) set(CMAKE_SYSTEM_NAME QURT) set(CMAKE_SYSTEM_PROCESSOR Hexagon) @@ -14,7 +14,6 @@ set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) set(CUSTOM_RUNELF_PATH "") -#To fix backward compatibility with EAI addon. if (NOT HEXAGON_SDK_ROOT) set(HEXAGON_SDK_ROOT $ENV{HEXAGON_SDK_ROOT}) endif() @@ -31,7 +30,6 @@ endif() file(TO_CMAKE_PATH "${HEXAGON_TOOLS_ROOT}" HEXAGON_TOOLS_ROOT) file(TO_CMAKE_PATH "${HEXAGON_SDK_ROOT}" HEXAGON_SDK_ROOT) -#Get the Binary extension of the Hexagon Toolchain if(CMAKE_HOST_SYSTEM_NAME STREQUAL Windows) set(HEXAGON_TOOLCHAIN_SUFFIX .exe) endif() @@ -48,12 +46,12 @@ set(CMAKE_TRY_COMPILE_PLATFORM_VARIABLES HEXAGON_TOOLS_ROOT ) -#QURT Related includes and linker flags +# QURT Related includes and linker flags set(V_ARCH ${HEXAGON_ARCH}) set(_QURT_INSTALL_DIR "${HEXAGON_SDK_ROOT}/rtos/qurt/ADSP${V_ARCH}MP${V_ARCH_EXTN}") set(_QURT_INSTALL_DIR "${HEXAGON_SDK_ROOT}/rtos/qurt/compute${V_ARCH}${V_ARCH_EXTN}") -if( ${TREE} MATCHES PAKMAN ) +if (${TREE} MATCHES PAKMAN) set(_QURT_INSTALL_DIR "${QURT_IMAGE_DIR}/compute${V_ARCH}${V_ARCH_EXTN}") endif() message(DEBUG "_QURT_INSTALL_DIR:${_QURT_INSTALL_DIR}") @@ -83,11 +81,9 @@ set(QURT_START_LINK_LIBS ) STRING(REPLACE ";" " " QURT_START_LINK_LIBS "${QURT_START_LINK_LIBS}") -set(QURT_END_LINK_LIBS - ${TARGET_DIR}/fini.o - ) +set(QURT_END_LINK_LIBS ${TARGET_DIR}/fini.o) -#Non QURT related includes and linker flags +# Non QURT related includes and linker flags set(TARGET_DIR_NOOS "${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib/${HEXAGON_ARCH}") @@ -99,8 +95,10 @@ if (NOT NO_WRAP_MEM_API) set(WRAP_MEMALIGN -Wl,--wrap=memalign) endif() +set(ARCH_FLAGS "-mcpu=${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH} -mhmx") + set(PIC_SHARED_LD_FLAGS - -mcpu=${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH} + ${ARCH_FLAGS} -G0 -fpic -Wl,-Bsymbolic @@ -120,13 +118,13 @@ STRING(REPLACE ";" " " PIC_SHARED_LD_FLAGS "${PIC_SHARED_LD_FLAGS}") set(HEXAGON_PIC_SHARED_LINK_OPTIONS "${PIC_SHARED_LD_FLAGS}") -#System include paths +# System include paths include_directories(SYSTEM ${HEXAGON_SDK_ROOT}/incs) include_directories(SYSTEM ${HEXAGON_SDK_ROOT}/incs/stddef) include_directories(SYSTEM ${HEXAGON_SDK_ROOT}/ipc/fastrpc/incs) -#LLVM toolchain setup -#Compiler paths, options and architecture +# LLVM toolchain setup +# Compiler paths, options and architecture set(CMAKE_C_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang${HEXAGON_TOOLCHAIN_SUFFIX}) set(CMAKE_CXX_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang++${HEXAGON_TOOLCHAIN_SUFFIX}) set(CMAKE_AR ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-ar${HEXAGON_TOOLCHAIN_SUFFIX}) @@ -137,16 +135,16 @@ set(CMAKE_PREFIX_PATH ${HEXAGON_TOOLCHAIN}/Tools/target/hexagon) set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "-Wl,-soname,") set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG "-Wl,-soname,") -#Compiler Options -set(COMMON_FLAGS "-mcpu=hexagon${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH} -fvectorize -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}") +# Compiler Options +set(COMMON_FLAGS "${ARCH_FLAGS} -fvectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}") set(CMAKE_CXX_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g") -set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O3 -g") -set(CMAKE_CXX_FLAGS_RELEASE "${COMMON_FLAGS} -O3") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O2 -g") +set(CMAKE_CXX_FLAGS_RELEASE "${COMMON_FLAGS} -O2") set(CMAKE_C_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g") -set(CMAKE_C_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O3 -g") -set(CMAKE_C_FLAGS_RELEASE "${COMMON_FLAGS} -O3") +set(CMAKE_C_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O2 -g") +set(CMAKE_C_FLAGS_RELEASE "${COMMON_FLAGS} -O2") set(CMAKE_ASM_FLAGS_DEBUG "${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_DEBUG}") set(CMAKE_ASM_FLAGS_RELEASE "${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE}") diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c new file mode 100644 index 0000000..2036988 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -0,0 +1,349 @@ +#include "concat-ops.h" +#include "dma-queue.h" +#include "hex-common.h" +#include "dma-copy.h" +#include "hex-fastdiv.h" +#include "hex-profile.h" +#include "hexagon_protos.h" +#include "hexagon_types.h" +#include "htp-ctx.h" +#include "htp-fence.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "hvx-utils.h" +#include "hvx_hexagon_protos.h" +#include "gather-rows.h" + +#include + +struct htp_concat_context { + struct htp_ops_context * octx; + uint8_t * spad0_base; + uint8_t * spad1_base; + uint32_t spad0_size_per_thread; + uint32_t spad1_size_per_thread; + uint32_t row_start; + uint32_t nrows; + uint32_t nrows_per_thread; + uint32_t nplanes; + struct fastdiv_values div_ne2; +}; + +static inline dma_addr_t concat_plane_addr(const struct htp_tensor * t, uint32_t p, const struct fastdiv_values * div_ne2, uint32_t ne2) { + const uint32_t i3 = fastdiv(p, div_ne2); + const uint32_t i2 = p - i3 * ne2; + return t->data + i2 * t->nb[2] + i3 * t->nb[3]; +} + +static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * data) { + struct htp_concat_context * cctx = (struct htp_concat_context *) data; + struct htp_ops_context * octx = cctx->octx; + + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t src0_ne0 = src0->ne[0]; + const uint32_t src1_ne0 = src1->ne[0]; + + const uint32_t row_end = cctx->row_start + cctx->nrows; + const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; + const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; + if (start_i >= end_i) return; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread; + uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread; + + const uint32_t block_i = 32; + const uint32_t spad1_stride = block_i * sizeof(float); + + const HVX_Vector offsets = hvx_vec_gather_offsets_w(spad1_stride); + const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 32); + const uint32_t src0_row_bytes = src0_ne0 * sizeof(float); + const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN); + const uint32_t src0_pre = src0_row_padded - src0_row_bytes; + const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(float); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const struct fastdiv_values * div_ne2 = &cctx->div_ne2; + const uint32_t ne2 = dst->ne[2]; + + uint32_t p = 0; + uint32_t i = start_i; + + const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(float), src1_ne0); + + const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i)); + + dma_queue_pop(dma_q); // src1 + + while (p < cctx->nplanes) { + const uint32_t current_block_i = MIN(end_i - i, block_i); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + for (uint32_t j = 0; j < src1_ne0; j += 32) { + const uint8_t * src_ptr = spad1_base + j * spad1_stride; + uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(float); + hvx_transpose_32x32_w_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 32)); + } + hvx_gather_sync(spad0_base + src0_row_padded); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + + uint32_t np = p; + uint32_t ni = i + block_i; + if (ni >= end_i) { + ni = start_i; + np++; + } + const bool has_next = np < cctx->nplanes; + const uint32_t next_block_i = MIN(end_i - ni, block_i); + + // spad1 is free after the gather sync, prefetch next src1 ahead of the dst write + if (has_next) { + const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(float), src1_ne0); + } + + dma_queue_pop(dma_q); // src0 + + const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1]; + dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(float), current_block_i); + + if (has_next) { + dma_queue_pop(dma_q); // next src1 + } + dma_queue_pop(dma_q); // dst + + // spad0 is free after the dst write + if (has_next) { + const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i); + } + + p = np; + i = ni; + } + dma_queue_flush(dma_q); +} + +static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * data) { + struct htp_concat_context * cctx = (struct htp_concat_context *) data; + struct htp_ops_context * octx = cctx->octx; + + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t src0_ne0 = src0->ne[0]; + const uint32_t src1_ne0 = src1->ne[0]; + + const uint32_t row_end = cctx->row_start + cctx->nrows; + const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread; + const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end; + if (start_i >= end_i) return; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread; + uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread; + + const uint32_t block_i = 64; + const uint32_t spad1_stride = block_i * sizeof(__fp16); + + const HVX_Vector offsets = hvx_vec_gather_offsets_h(spad1_stride); + const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 64); + const uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16); + const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN); + const uint32_t src0_pre = src0_row_padded - src0_row_bytes; + const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(__fp16); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const struct fastdiv_values * div_ne2 = &cctx->div_ne2; + const uint32_t ne2 = dst->ne[2]; + + uint32_t p = 0; + uint32_t i = start_i; + + const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(__fp16), src1_ne0); + + const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i)); + + dma_queue_pop(dma_q); // src1 + + while (p < cctx->nplanes) { + const uint32_t current_block_i = MIN(end_i - i, block_i); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + for (uint32_t j = 0; j < src1_ne0; j += 64) { + const uint8_t * src_ptr = spad1_base + j * spad1_stride; + uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(__fp16); + hvx_transpose_64x64_h_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 64)); + } + hvx_gather_sync(spad0_base + src0_row_padded); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + + uint32_t np = p; + uint32_t ni = i + block_i; + if (ni >= end_i) { + ni = start_i; + np++; + } + const bool has_next = np < cctx->nplanes; + const uint32_t next_block_i = MIN(end_i - ni, block_i); + + // spad1 is free after the gather sync, prefetch next src1 ahead of the dst write + if (has_next) { + const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(__fp16), src1_ne0); + } + + dma_queue_pop(dma_q); // src0 + + const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1]; + dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(__fp16), current_block_i); + + if (has_next) { + dma_queue_pop(dma_q); // next src1 + } + dma_queue_pop(dma_q); // dst + + // spad0 is free after the dst write + if (has_next) { + const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1]; + dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i); + } + + p = np; + i = ni; + } + dma_queue_flush(dma_q); +} + +static int concat_regular(struct htp_ops_context * octx, int dim, uint32_t type_size) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + struct htp_tensor view0 = *dst; + struct htp_tensor view1 = *dst; + for (int d = 0; d < HTP_OP_MAX_DIMS; d++) { + view0.ne[d] = src0->ne[d]; + view1.ne[d] = src1->ne[d]; + } + view1.data += src0->ne[dim] * dst->nb[dim]; + + const uint32_t total_rows_0 = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t total_rows_1 = src1->ne[1] * src1->ne[2] * src1->ne[3]; + + uint32_t rstart0 = 0, nrows0 = total_rows_0; + uint32_t rstart1 = 0, nrows1 = total_rows_1; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range0 = htp_tensor_mdev_partition( + total_rows_0, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + rstart0 = range0.start; + nrows0 = range0.count; + + const struct htp_tensor_mdev_range range1 = htp_tensor_mdev_partition( + total_rows_1, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + rstart1 = range1.start; + nrows1 = range1.count; + } + + dma_queue * q = octx->ctx->dma[0]; + + dma_cpy_sametype_sameshape_range(q, &view0, src0, type_size, rstart0, nrows0); + dma_cpy_sametype_sameshape_range(q, &view1, src1, type_size, rstart1, nrows1); + dma_queue_flush(q); + + return HTP_STATUS_OK; +} + +static int concat_transposed(struct htp_ops_context * octx, const struct htp_concat_kernel_params * kparams, uint32_t type_size) { + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const struct htp_tensor * dst = octx->dst; + + const uint32_t total_rows = dst->ne[1]; + uint32_t row_start = 0; + uint32_t nrows = total_rows; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0 || dst->ne[2] == 0 || dst->ne[3] == 0) { + return HTP_STATUS_OK; + } + + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + const uint32_t n_threads = octx->n_threads; + + struct htp_concat_transposed_vtcm_layout layout; + htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads); + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + + struct htp_concat_context cctx; + cctx.octx = octx; + cctx.spad0_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad0); + cctx.spad1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad1); + cctx.spad0_size_per_thread = layout.src0_spad_size_per_thread; + cctx.spad1_size_per_thread = layout.src1_spad_size_per_thread; + cctx.row_start = row_start; + cctx.nrows = nrows; + cctx.nplanes = dst->ne[2] * dst->ne[3]; + cctx.div_ne2 = kparams->u.transposed.div_ne2; + cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + + work_queue_func_t worker_func = (type_size == 4) ? concat_2d_f32_transposed : concat_2d_f16_transposed; + work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads); + return HTP_STATUS_OK; +} + +int op_concat(struct htp_ops_context * octx) { + const struct htp_concat_kernel_params * kparams = (const struct htp_concat_kernel_params *) octx->kernel_params; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + int status = HTP_STATUS_OK; + const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2; + + switch (kparams->kernel_type) { + case HTP_CONCAT_KERNEL_REGULAR: + status = concat_regular(octx, kparams->dim, type_size); + break; + + case HTP_CONCAT_KERNEL_TRANSPOSED: + status = concat_transposed(octx, kparams, type_size); + break; + + case HTP_CONCAT_KERNEL_GATHER_ROWS: + status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads); + break; + + default: + status = HTP_STATUS_NO_SUPPORT; + break; + } + + htp_ops_context_set_status(octx, status); + + return octx->status; +} diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.h new file mode 100644 index 0000000..a5d9328 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/concat-ops.h @@ -0,0 +1,70 @@ +#ifndef HTP_CONCAT_OPS_H +#define HTP_CONCAT_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include "gather-rows.h" +#include + +enum htp_concat_kernel_type { + HTP_CONCAT_KERNEL_UNSUPPORTED = 0, + HTP_CONCAT_KERNEL_REGULAR = 1, + HTP_CONCAT_KERNEL_TRANSPOSED = 2, + HTP_CONCAT_KERNEL_GATHER_ROWS = 3, +}; + +struct htp_concat_transposed_params { + struct fastdiv_values div_ne2; +}; + +struct htp_concat_kernel_params { + uint8_t kernel_type; + uint8_t dim; + uint8_t n_threads; + uint8_t pad; + + uint32_t vtcm_size; + + union { + struct htp_concat_transposed_params transposed; + struct htp_gather_rows_params gather; + } u; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_concat_transposed_vtcm_layout { + uint32_t src0_spad_size_per_thread; + uint32_t src1_spad_size_per_thread; + size_t off_spad0; + size_t off_spad1; + uint32_t total_bytes; +}; + +static inline void htp_concat_transposed_vtcm_layout_build( + struct htp_concat_transposed_vtcm_layout * L, + uint32_t src0_ne0, + uint32_t src1_ne0, + uint32_t type_size, + uint32_t n_threads) { + + uint32_t block_i = (type_size == 4) ? 32 : 64; + uint32_t spad1_stride = block_i * type_size; + uint32_t src1_ne0_padded = hex_round_up(src1_ne0, block_i); + uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + src1_ne0_padded * type_size; + + L->src0_spad_size_per_thread = block_i * spad0_row_bytes; + L->src1_spad_size_per_thread = src1_ne0_padded * spad1_stride; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_threads * L->src0_spad_size_per_thread); + VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_threads * L->src1_spad_size_per_thread); + L->total_bytes = (uint32_t) off; +} + +#endif // HTP_CONCAT_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index e5b9d35..fce45c2 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -4,249 +4,485 @@ #include #include +#include #include #include #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "cpy-ops.h" +#include "dma-copy.h" #include "htp-ctx.h" +#include "htp-fence.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-utils.h" +#include "gather-rows.h" struct htp_copy_context { - struct htp_ops_context * octx; - - uint32_t src0_type_size; - uint32_t src0_block_size; - - uint32_t dst_type_size; - uint32_t dst_block_size; + struct htp_ops_context * octx; + const struct htp_copy_kernel_params * kparams; - uint32_t src0_blocks_per_row; - uint32_t dst_blocks_per_row; + uint32_t row_start; + uint32_t nrows; + uint32_t src0_nrows_per_thread; - uint32_t src0_nrows_per_thread; + uint32_t elem_start; + uint32_t nelem; + uint32_t elem_per_thread; - void (*copy)(struct htp_copy_context * ct, struct htp_ops_context * octx, int nth, int ith); + struct htp_copy_convert_vtcm_layout layout; + uint8_t * vtcm_src0; + uint8_t * vtcm_dst; }; #define cpy_preamble \ const struct htp_tensor *src0 = octx->src[0]; \ const struct htp_tensor *dst = octx->dst; \ \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ - \ - const uint32_t nr = ne01; - -static void cpy_thread_sametype_sameshape(struct htp_copy_context * ct, struct htp_ops_context * octx, const int nth, const int ith) { - cpy_preamble; - - // parallelize by src0 rows - const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - - // copy by rows - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - #pragma unroll(2) - for (uint32_t i01 = ir0; i01 < ir1; i01++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - hex_l2fetch(src0_ptr, ne00 * ct->src0_type_size, nb01, 2); - hvx_copy_uu(dst_ptr, src0_ptr, ne00, ct->src0_type_size); - } - } - } + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; + +#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \ +static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_copy_context * ct = (struct htp_copy_context *) data; \ + struct htp_ops_context * octx = ct->octx; \ + cpy_preamble; \ + const uint32_t th_nelem = ct->elem_per_thread; \ + const uint32_t th_start = ct->elem_start + ith * th_nelem; \ + const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \ + if (th_start >= th_end) return; \ + \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + \ + const uint32_t ne01_ne00 = ne01 * ne00; \ + const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \ + const uint32_t ne1_ne0 = ne1 * ne0; \ + const uint32_t ne2_ne1_ne0 = ne2 * ne1_ne0; \ + \ + const struct htp_copy_reshape_params * rsh = &ct->kparams->u.reshape; \ + uint32_t e = th_start; \ + uint32_t i13 = fastdiv(e, &rsh->div_ne2_ne1_ne0); \ + uint32_t rem = e - i13 * ne2_ne1_ne0; \ + uint32_t i12 = fastdiv(rem, &rsh->div_ne1_ne0); \ + uint32_t rem2 = rem - i12 * ne1_ne0; \ + uint32_t i11 = fastdiv(rem2, &rsh->div_ne0); \ + uint32_t i10 = rem2 - i11 * ne0; \ + \ + uint32_t i03 = fastdiv(e, &rsh->div_ne02_ne01_ne00); \ + uint32_t rem_s = e - i03 * ne02_ne01_ne00; \ + uint32_t i02 = fastdiv(rem_s, &rsh->div_ne01_ne00); \ + uint32_t rem2_s = rem_s - i02 * ne01_ne00; \ + uint32_t i01 = fastdiv(rem2_s, &rsh->div_ne00); \ + uint32_t i00 = rem2_s - i01 * ne00; \ + \ + dma_addr_t dst_addr = dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \ + dma_addr_t src0_addr = src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \ + \ + const bool rows_contig = (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \ + \ + while (e < th_end) { \ + const uint32_t run = MIN(MIN(ne00 - i00, ne0 - i10), th_end - e); \ + if (rows_contig) { \ + dma_cpy_sametype_reshape_contig(dma_q, dst_addr, src0_addr, run * ELEM_SIZE); \ + } else { \ + dma_cpy_push_2d_chunked(dma_q, dst_addr, src0_addr, nb0, nb00, ELEM_SIZE, run); \ + } \ + e += run; \ + \ + dst_addr += run * nb0; \ + i10 += run; \ + if (i10 == ne0) { \ + i10 = 0; \ + if (++i11 == ne1) { \ + i11 = 0; \ + if (++i12 == ne2) { \ + i12 = 0; \ + i13++; \ + } \ + } \ + dst_addr = dst->data + i11*nb1 + i12*nb2 + i13*nb3; \ + } \ + \ + src0_addr += run * nb00; \ + i00 += run; \ + if (i00 == ne00) { \ + i00 = 0; \ + if (++i01 == ne01) { \ + i01 = 0; \ + if (++i02 == ne02) { \ + i02 = 0; \ + i03++; \ + } \ + } \ + src0_addr = src0->data + i01*nb01 + i02*nb02 + i03*nb03; \ + } \ + } \ + dma_queue_flush(dma_q); \ } -static void cpy_thread_sametype_reshape(struct htp_copy_context * ct, struct htp_ops_context * octx, int nth, int ith) { - cpy_preamble; - - // parallelize by src0 rows - const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - - // dst counters - int64_t k10 = 0; - int64_t i11 = 0; - int64_t i12 = 0; - int64_t i13 = 0; - - // number of blocks in a row - const int64_t nk00 = ct->src0_blocks_per_row; - const int64_t nk0 = ct->dst_blocks_per_row; - - for (int64_t i03 = 0; i03 < ne03; i03++) { - for (int64_t i02 = 0; i02 < ne02; i02++) { - k10 += nk00 * ir0; - while (k10 >= nk0) { - k10 -= nk0; - if (++i11 == ne1) { - i11 = 0; - if (++i12 == ne2) { - i12 = 0; - if (++i13 == ne3) { - i13 = 0; - } - } - } - } - for (int64_t i01 = ir0; i01 < ir1; i01++) { - for (int64_t k00 = 0; k00 < nk00; k00++) { - const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); - char * dst_ptr = ((char *) dst->data + k10*nb0 + i11*nb1 + i12*nb2 + i13*nb3); - memcpy(dst_ptr, src0_ptr, ct->dst_type_size); - - if (++k10 == nk0) { - k10 = 0; - if (++i11 == ne1) { - i11 = 0; - if (++i12 == ne2) { - i12 = 0; - if (++i13 == ne3) { - i13 = 0; - } - } - } - } - } - } - k10 += nk00 * (ne01 - ir1); - while (k10 >= nk0) { - k10 -= nk0; - if (++i11 == ne1) { - i11 = 0; - if (++i12 == ne2) { - i12 = 0; - if (++i13 == ne3) { - i13 = 0; - } - } - } - } +DEFINE_CPY_RESHAPE(f32, float, 4) +DEFINE_CPY_RESHAPE(f16, __fp16, 2) +DEFINE_CPY_RESHAPE(i32, int32_t, 4) + +struct cpy_row_cursor { + uint32_t r; + uint32_t i01; + uint32_t i02; + uint32_t i03; +}; + +static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uint32_t ne01, uint32_t ne02) { + c->r += n; + c->i01 += n; + if (c->i01 == ne01) { + c->i01 = 0; + if (++c->i02 == ne02) { + c->i02 = 0; + c->i03++; } } } -static void cpy_thread_f16_f32_sameshape(struct htp_copy_context * ct, struct htp_ops_context * octx, const int nth, const int ith) { - cpy_preamble; - - // parallelize by src0 rows - const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - - // copy by rows - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - #pragma unroll(2) - for (uint32_t i01 = ir0; i01 < ir1; i01++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2); - hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); - } - } - } +#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \ +static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_copy_context * ct = (struct htp_copy_context *) data; \ + struct htp_ops_context * octx = ct->octx; \ + cpy_preamble; \ + \ + const uint32_t dr = ct->src0_nrows_per_thread; \ + const uint32_t ir0 = ct->row_start + dr * ith; \ + const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \ + if (ir0 >= ir1) return; \ + \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + \ + const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \ + const struct htp_copy_convert_vtcm_layout * L = &ct->layout; \ + const uint32_t src0_buf_size = L->src0_buf_size; \ + const uint32_t dst_buf_size = L->dst_buf_size; \ + const uint32_t src0_row_stride = L->src0_row_stride; \ + const uint32_t dst_row_stride = L->dst_row_stride; \ + const uint32_t blk_rows = L->blk_rows; \ + uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_thread; \ + uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_thread; \ + const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \ + const uint32_t dst_row_size = ne00 * ct->kparams->dst_type_size; \ + \ + const uint32_t ne02_ne01 = ne02 * ne01; \ + uint32_t i03 = fastdiv(ir0, &cvt->div_ne02_ne01); \ + uint32_t rem = ir0 - i03 * ne02_ne01; \ + uint32_t i02 = fastdiv(rem, &cvt->div_ne01); \ + uint32_t i01 = rem - i02 * ne01; \ + \ + struct cpy_row_cursor f = { ir0, i01, i02, i03 }; \ + struct cpy_row_cursor c = f; \ + \ + for (uint32_t b = 0; b < 2 && f.r < ir1; ++b) { \ + uint8_t * src_spad = vtcm_src0_base + b * src0_buf_size; \ + uint8_t * dst_spad = vtcm_dst_base + b * dst_buf_size; \ + const uint32_t nr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \ + dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \ + dma_queue_push(dma_q, dma_make_data(dst->data, dst_spad), \ + dst_row_size, dst_buf_size, dst_row_size, 0); \ + dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \ + src0_row_stride, nb01, src0_row_size, nr); \ + cpy_cursor_advance(&f, nr, ne01, ne02); \ + } \ + \ + for (uint32_t blk = 0; c.r < ir1; ++blk) { \ + const uint32_t nr = MIN(blk_rows, MIN(ne01 - c.i01, ir1 - c.r)); \ + uint8_t * dst_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; \ + uint8_t * src_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \ + for (uint32_t r = 0; r < nr; ++r) { \ + CONV_FUNC(dst_spad + r * dst_row_stride, src_spad + r * src0_row_stride, ne00); \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \ + \ + dma_addr_t c_dst_addr = dst->data + c.i01*nb1 + c.i02*nb2 + c.i03*nb3; \ + dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \ + nb1, dst_row_stride, dst_row_size, nr); \ + cpy_cursor_advance(&c, nr, ne01, ne02); \ + \ + if (f.r < ir1) { \ + const uint32_t fnr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \ + dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \ + dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \ + src0_row_stride, nb01, src0_row_size, fnr); \ + cpy_cursor_advance(&f, fnr, ne01, ne02); \ + } \ + } \ + dma_queue_flush(dma_q); \ } -static void cpy_thread_f32_f16_sameshape(struct htp_copy_context * ct, struct htp_ops_context * octx, const int nth, const int ith) { - cpy_preamble; - - // parallelize by src0 rows - const uint32_t dr = ct->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; - - // copy by rows - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - #pragma unroll(2) - for (uint32_t i01 = ir0; i01 < ir1; i01++) { - uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; - uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; - hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2); - hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00); - } - } +DEFINE_CPY_CONVERT_SAMESHAPE(f16_f32, hvx_copy_f16_f32_aa) +DEFINE_CPY_CONVERT_SAMESHAPE(f32_f16, hvx_copy_f32_f16_aa) +DEFINE_CPY_CONVERT_SAMESHAPE(i32_f32, hvx_copy_i32_f32_aa) +DEFINE_CPY_CONVERT_SAMESHAPE(f32_i32, hvx_copy_f32_i32_aa) + +static int cpy_scalar(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + if (src0->type == dst->type) { + dma_cpy_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, kparams->src0_type_size); + dma_queue_flush(octx->ctx->dma[0]); + return HTP_STATUS_OK; } + + dma_queue * dma_q = octx->ctx->dma[0]; + uint8_t * s_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, 0); + uint8_t * d_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, VLEN); + dma_addr_t s_vtcm = (dma_addr_t)(uintptr_t) s_ptr; + dma_addr_t d_vtcm = (dma_addr_t)(uintptr_t) d_ptr; + const uint32_t s_size = kparams->src0_type_size; + const uint32_t d_size = kparams->dst_type_size; + + dma_queue_push(dma_q, dma_make_data(s_vtcm, src0->data), s_size, s_size, s_size, 1); + dma_queue_pop(dma_q); + + const HVX_Vector v_src = hvx_vmem(s_ptr); + HVX_Vector v_dst; + + if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_I32) { + v_dst = Q6_Vw_equals_Vsf(v_src); + } else if (src0->type == HTP_TYPE_I32 && dst->type == HTP_TYPE_F32) { + v_dst = Q6_Vsf_equals_Vw(v_src); + } else if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_F16) { + v_dst = hvx_vec_f32_to_f16(v_src, v_src); + } else if (src0->type == HTP_TYPE_F16 && dst->type == HTP_TYPE_F32) { + v_dst = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src)); + } else { + return HTP_STATUS_NO_SUPPORT; + } + + hvx_vmem(d_ptr) = v_dst; + + dma_queue_push(dma_q, dma_make_data(dst->data, d_vtcm), d_size, d_size, d_size, 1); + dma_queue_flush(dma_q); + return HTP_STATUS_OK; } -static void cpy_work_func(unsigned int n, unsigned int i, void *data) { - struct htp_copy_context *ct = (struct htp_copy_context *) data; - ct->copy(ct, ct->octx, n, i); +static int cpy_1d_contig(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + uint32_t elem_start = 0; + uint32_t nelem = kparams->total_elems; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->total_elems, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelem = range.count; + } + + if (nelem > 0) { + dma_queue * q = octx->ctx->dma[0]; + const uint32_t type_size = kparams->src0_type_size; + dma_addr_t dst_addr = dst->data + elem_start * type_size; + dma_addr_t src0_addr = src0->data + elem_start * type_size; + dma_cpy_sametype_reshape_contig(q, dst_addr, src0_addr, nelem * type_size); + dma_queue_flush(q); + } + + return HTP_STATUS_OK; } -int op_cpy(struct htp_ops_context * octx) { - cpy_preamble; +static int cpy_sameshape_sametype(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; - const uint32_t n_threads = MIN(nr, octx->n_threads); + uint32_t row_start = 0; + uint32_t nrows = kparams->total_rows; - struct htp_copy_context ct; - ct.octx = octx; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } - switch (src0->type) { - case HTP_TYPE_F32: ct.src0_type_size = 4; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break; - case HTP_TYPE_F16: ct.src0_type_size = 2; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break; - default: - return HTP_STATUS_NO_SUPPORT; + if (nrows > 0) { + dma_queue * q = octx->ctx->dma[0]; + dma_cpy_sametype_sameshape_range(q, dst, src0, kparams->src0_type_size, row_start, nrows); + dma_queue_flush(q); } - switch (dst->type) { - case HTP_TYPE_F32: ct.dst_type_size = 4; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break; - case HTP_TYPE_F16: ct.dst_type_size = 2; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break; - default: - return HTP_STATUS_NO_SUPPORT; + return HTP_STATUS_OK; +} + +static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + uint32_t row_start = 0; + uint32_t nrows = kparams->total_rows; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { return HTP_STATUS_OK; } - const bool sametype = (src0->type == dst->type); - const bool transposed = (nb00 > nb01) || (nb0 > nb1); - const bool sameshape = !transposed && (ne00 == ne0 && ne01 == ne1 && ne02 == ne2 && ne03 == ne3); + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } - ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads; + const uint32_t n_threads = octx->n_threads; + const struct htp_copy_convert_params * cvt = &kparams->u.convert; - if (sametype && sameshape) { - ct.copy = cpy_thread_sametype_sameshape; - } else if (sameshape) { - /**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) - ct.copy = cpy_thread_f16_f32_sameshape; - else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) - ct.copy = cpy_thread_f32_f16_sameshape; - else - return HTP_STATUS_NO_SUPPORT; - } else if (sametype) { - ct.copy = cpy_thread_sametype_reshape; + struct htp_copy_context ct; + htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_threads); + + ct.octx = octx; + ct.kparams = kparams; + ct.row_start = row_start; + ct.nrows = nrows; + ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + ct.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_src0); + ct.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_dst); + + work_queue_func_t copy_fun = NULL; + if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) { + copy_fun = cpy_thread_f16_f32_sameshape; + } else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) { + copy_fun = cpy_thread_f32_f16_sameshape; + } else if (dst->type == HTP_TYPE_I32 && src0->type == HTP_TYPE_F32) { + copy_fun = cpy_thread_i32_f32_sameshape; + } else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_I32) { + copy_fun = cpy_thread_f32_i32_sameshape; } else { return HTP_STATUS_NO_SUPPORT; } - worker_pool_run_func(octx->ctx->worker_pool, cpy_work_func, &ct, n_threads); + work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); + return HTP_STATUS_OK; +} + +static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + uint32_t elem_start = 0; + uint32_t nelem = kparams->total_elems; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->total_elems, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + elem_start = range.start; + nelem = range.count; + } + + if (nelem == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + struct htp_copy_context ct; + ct.octx = octx; + ct.kparams = kparams; + ct.elem_start = elem_start; + ct.nelem = nelem; + ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div); + + work_queue_func_t copy_fun = NULL; + switch (src0->type) { + case HTP_TYPE_F32: copy_fun = cpy_thread_f32_reshape; break; + case HTP_TYPE_F16: copy_fun = cpy_thread_f16_reshape; break; + case HTP_TYPE_I32: copy_fun = cpy_thread_i32_reshape; break; + default: return HTP_STATUS_NO_SUPPORT; + } + + work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads); return HTP_STATUS_OK; } + +int op_cpy(struct htp_ops_context * octx) { + const struct htp_copy_kernel_params * kparams = (const struct htp_copy_kernel_params *) octx->kernel_params; + int status = HTP_STATUS_OK; + + switch (kparams->kernel_type) { + case HTP_COPY_KERNEL_SCALAR: + status = cpy_scalar(octx, kparams); + break; + case HTP_COPY_KERNEL_1D_CONTIG: + status = cpy_1d_contig(octx, kparams); + break; + case HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE: + status = cpy_sameshape_sametype(octx, kparams); + break; + case HTP_COPY_KERNEL_SAMESHAPE_CONVERT: + status = cpy_sameshape_convert(octx, kparams); + break; + case HTP_COPY_KERNEL_RESHAPE: + status = cpy_reshape(octx, kparams); + break; + case HTP_COPY_KERNEL_GATHER_ROWS: + status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads); + break; + default: + status = HTP_STATUS_NO_SUPPORT; + break; + } + + htp_ops_context_set_status(octx, status); + + if (octx->op == HTP_OP_CPY_FENCE) { + if (octx->ctx->mdev.count > 1) { + htp_mdev_group_barrier(octx); + } + + if (octx->ctx->mdev.idx == 0) { + const struct htp_tensor * sync = octx->src[1]; + const uint32_t seq = (uint32_t) octx->op_params[0]; + atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; + htp_fence_write(sync_fence, seq, octx->status); + + FARF(HIGH, "ggml-hex: sync-release : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status); + } + } + + return octx->status; +} diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h new file mode 100644 index 0000000..c3b97c0 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h @@ -0,0 +1,125 @@ +#ifndef HTP_CPY_OPS_H +#define HTP_CPY_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include "gather-rows.h" +#include + +enum htp_copy_kernel_type { + HTP_COPY_KERNEL_UNSUPPORTED = 0, + HTP_COPY_KERNEL_1D_CONTIG = 1, + HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE = 2, + HTP_COPY_KERNEL_SAMESHAPE_CONVERT = 3, + HTP_COPY_KERNEL_RESHAPE = 4, + HTP_COPY_KERNEL_SCALAR = 5, + HTP_COPY_KERNEL_GATHER_ROWS = 6, +}; + +struct htp_copy_convert_params { + uint32_t blk_rows; // rows moved per DMA descriptor + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02_ne01; +}; + +struct htp_copy_reshape_params { + struct fastdiv_values div_ne0; + struct fastdiv_values div_ne1_ne0; + struct fastdiv_values div_ne2_ne1_ne0; + struct fastdiv_values div_ne00; + struct fastdiv_values div_ne01_ne00; + struct fastdiv_values div_ne02_ne01_ne00; +}; + +struct htp_copy_kernel_params { + uint8_t kernel_type; + uint8_t src0_type_size; + uint8_t dst_type_size; + uint8_t n_threads; + + uint32_t total_elems; + uint32_t total_rows; + uint32_t vtcm_size; + + union { + struct htp_copy_convert_params convert; + struct htp_copy_reshape_params reshape; + struct htp_gather_rows_params gather; + } u; +}; + +// Size of one VTCM staging buffer: big enough that a DMA descriptor moves many short +// rows at once. The copy kernels keep four per thread (two in flight each way), about +// 1 MB of VTCM with 8 threads. +#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024) + +struct htp_copy_convert_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + uint32_t src0_row_stride; + uint32_t dst_row_stride; + uint32_t blk_rows; + uint32_t src0_buf_size; + uint32_t dst_buf_size; + uint32_t spad0_size_per_thread; + uint32_t spad1_size_per_thread; +}; + +static inline void htp_copy_convert_vtcm_layout_build( + struct htp_copy_convert_vtcm_layout * L, + uint32_t ne00, + uint32_t src_type_size, + uint32_t dst_type_size, + uint32_t blk_rows, + uint32_t n_threads) { + + const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256); + const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256); + + L->src0_row_stride = src0_row_stride; + L->dst_row_stride = dst_row_stride; + L->blk_rows = blk_rows; + L->src0_buf_size = blk_rows * src0_row_stride; + L->dst_buf_size = blk_rows * dst_row_stride; + L->spad0_size_per_thread = 2 * L->src0_buf_size; + L->spad1_size_per_thread = 2 * L->dst_buf_size; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_threads * L->spad0_size_per_thread); + VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_threads * L->spad1_size_per_thread); + L->total_bytes = off; +} + +// ne01 bounds a block: rows of one block share the dim-1 stride. +static inline void htp_copy_convert_solve_layout( + struct htp_copy_convert_vtcm_layout * layout, + uint32_t ne00, + uint32_t ne01, + uint32_t src_type_size, + uint32_t dst_type_size, + uint32_t n_threads) { + + const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256); + const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256); + + const uint32_t max_row = src0_row_stride > dst_row_stride ? src0_row_stride : dst_row_stride; + uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row; + if (blk_rows > ne01) { + blk_rows = ne01; + } + if (blk_rows == 0) { + blk_rows = 1; + } + + htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads); +} + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_CPY_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.c b/ggml/src/ggml-hexagon/htp/cumsum-ops.c index 2ced197..36fc812 100644 --- a/ggml/src/ggml-hexagon/htp/cumsum-ops.c +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.c @@ -7,34 +7,39 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" #include "hvx-types.h" #include "hvx-utils.h" -#include "hex-dma.h" +#include "dma-queue.h" +#include "cumsum-ops.h" #define htp_cumsum_tensors_preamble \ const struct htp_tensor * restrict src0 = octx->src[0]; \ const struct htp_tensor * restrict dst = octx->dst; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; struct htp_cumsum_context { @@ -45,13 +50,14 @@ struct htp_cumsum_context { size_t dst_row_size_aligned; uint32_t rows_per_thread; uint32_t total_rows; + uint32_t row_start; }; #define htp_cumsum_preamble \ struct htp_cumsum_context * cctx = (struct htp_cumsum_context *) data; \ struct htp_ops_context * octx = cctx->octx; \ htp_cumsum_tensors_preamble; \ - dma_queue * dma_queue = octx->ctx->dma[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; // --------------------------------------------------------------------------- // HVX prefix scan helpers @@ -85,8 +91,8 @@ static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) { } static inline void hvx_cumsum_row_f32(const float * restrict src, float * restrict dst, uint32_t n) { - const uint32_t nvec = n / VLEN_FP32; - const uint32_t nloe = n % VLEN_FP32; + const uint32_t nvec = n >> 5; + const uint32_t nloe = n & (VLEN_FP32 - 1); HVX_Vector carry = Q6_V_vsplat_R(0); @@ -115,11 +121,8 @@ static inline void hvx_cumsum_row_f32(const float * restrict src, float * restri static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t ir0 = cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); if (ir0 >= ir1) { return; @@ -130,49 +133,51 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat const size_t src_row_size_aligned = cctx->src_row_size_aligned; const size_t dst_row_size_aligned = cctx->dst_row_size_aligned; - const uint8_t * src_data = (const uint8_t *) src0->data; - uint8_t * dst_data = (uint8_t *) dst->data; + const dma_addr_t src_data = src0->data; + const dma_addr_t dst_data = dst->data; uint8_t * src_spad = octx->src0_spad.data + (ith * src_row_size_aligned * 2); uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned * 2); for (uint32_t ir = ir0, spad_idx = 0; ir < ir1 && spad_idx < 2; ir++, spad_idx++) { // Dummy dst writeback to establish queue ordering - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(dst_data, dst_spad + (spad_idx * dst_row_size_aligned)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src_spad + (spad_idx * src_row_size_aligned), - src_data + (ir * src_row_size)), - src_row_size_aligned, src_row_size, 1); + dma_queue_push(dma_q, + dma_make_data(dst_data, dst_spad + (spad_idx * dst_row_size_aligned)), + dst_row_size, dst_row_size_aligned, dst_row_size, 0); + + dma_queue_push(dma_q, + dma_make_data(src_spad + (spad_idx * src_row_size_aligned), + src_data + (ir * src_row_size)), + src_row_size_aligned, src_row_size, src_row_size, 1); } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + for (uint32_t ir = ir0; ir < ir1; ir++) { - float * dst_spad_row = (float *) dma_queue_pop(dma_queue).src; - float * src_spad_row = (float *) dma_queue_pop(dma_queue).dst; + float * dst_spad_row = (float *) dma_queue_pop(dma_q).src; + float * src_spad_row = (float *) dma_queue_pop(dma_q).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); hvx_cumsum_row_f32(src_spad_row, dst_spad_row, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(dst_data + (ir * dst_row_size), (uint8_t *) dst_spad_row), - dst_row_size, dst_row_size_aligned, 1); + dma_queue_push(dma_q, + dma_make_data(dst_data + (ir * dst_row_size), dst_spad_row), + dst_row_size, dst_row_size_aligned, dst_row_size, 1); const uint32_t next_row = ir + 2; if (next_row < ir1) { - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr((uint8_t *) src_spad_row, src_data + (next_row * src_row_size)), - src_row_size_aligned, src_row_size, 1); + dma_queue_push(dma_q, + dma_make_data(src_spad_row, src_data + (next_row * src_row_size)), + src_row_size_aligned, src_row_size, src_row_size, 1); } } - dma_queue_flush(dma_queue); - t2 = HAP_perf_get_qtimer_count(); + dma_queue_flush(dma_q); - FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } // --------------------------------------------------------------------------- @@ -182,14 +187,14 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { htp_cumsum_preamble; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - const uint8_t * src_data = (const uint8_t *) src0->data; uint8_t * dst_data = (uint8_t *) dst->data; - const uint32_t ir0 = cctx->rows_per_thread * ith; - const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows); + const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); for (uint32_t ir = ir0; ir < ir1; ir++) { const float * restrict src_row = (const float *) (src_data + ir * cctx->src_row_size); @@ -197,56 +202,77 @@ static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) { hvx_cumsum_row_f32(src_row, dst_row, ne00); } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); - FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, - dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); } int op_cumsum_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; + const struct htp_cumsum_kernel_params * kparams = + (const struct htp_cumsum_kernel_params *) octx->kernel_params; + + const uint32_t total_rows = kparams->total_rows; + const size_t src_row_size = src0->nb[1]; + const size_t dst_row_size = dst->nb[1]; + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + if (nrows == 0) { return HTP_STATUS_OK; } - const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_rows); - - const size_t src_row_size = src0->nb[1]; - const size_t dst_row_size = dst->nb[1]; - const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); - - // 2 ping-pong buffers per thread for src and dst - const size_t spad_per_thread = 2 * (src_row_size_aligned + dst_row_size_aligned); + if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } - octx->src0_spad.size_per_thread = src_row_size_aligned * 2; - octx->dst_spad.size_per_thread = dst_row_size_aligned * 2; + const uint32_t n_threads = octx->n_threads; - octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread; - octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread; + if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { + return HTP_STATUS_NO_SUPPORT; + } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL; + struct htp_cumsum_vtcm_layout layout = { 0 }; + if (kparams->use_dma) { + htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads); + + octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread; + octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread; + octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread; + octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread; + octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); + octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); + octx->src0_spad.src = NULL; + octx->dst_spad.src = NULL; + } struct htp_cumsum_context cctx = { .octx = octx, .src_row_size = src_row_size, .dst_row_size = dst_row_size, - .src_row_size_aligned = src_row_size_aligned, - .dst_row_size_aligned = dst_row_size_aligned, - .rows_per_thread = (total_rows + n_threads - 1) / n_threads, - .total_rows = total_rows, + .src_row_size_aligned = layout.src_row_size_aligned, + .dst_row_size_aligned = layout.dst_row_size_aligned, + .rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_rows = nrows, + .row_start = row_start, }; - if (octx->ctx->vtcm_size < spad_per_thread * n_threads) { - worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32, &cctx, n_threads); + if (kparams->use_dma) { + work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads); } else { - worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32_dma, &cctx, n_threads); + work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads); } return HTP_STATUS_OK; @@ -255,16 +281,10 @@ int op_cumsum_f32(struct htp_ops_context * octx) { int op_cumsum(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; - int err = HTP_STATUS_OK; - switch (dst->type) { case HTP_TYPE_F32: - err = op_cumsum_f32(octx); - break; + return op_cumsum_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/ggml/src/ggml-hexagon/htp/cumsum-ops.h b/ggml/src/ggml-hexagon/htp/cumsum-ops.h new file mode 100644 index 0000000..6e1ac31 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/cumsum-ops.h @@ -0,0 +1,49 @@ +#ifndef HTP_CUMSUM_OPS_H +#define HTP_CUMSUM_OPS_H + +#include "hex-common.h" +#include "htp-vtcm.h" +#include + +struct htp_cumsum_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + size_t src_row_size_aligned; + size_t dst_row_size_aligned; + size_t src0_bytes_per_thread; + size_t dst_bytes_per_thread; +}; + +static inline void htp_cumsum_vtcm_layout_build( + struct htp_cumsum_vtcm_layout * L, + size_t src_row_size, + size_t dst_row_size, + uint32_t n_threads +) { + L->src_row_size_aligned = hex_round_up(src_row_size, 128); + L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); + L->src0_bytes_per_thread = 2 * L->src_row_size_aligned; + L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + + L->total_bytes = off; +} + +struct htp_cumsum_kernel_params { + uint32_t n_threads; + uint32_t total_rows; + uint32_t use_dma; + uint32_t vtcm_size; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_CUMSUM_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.c b/ggml/src/ggml-hexagon/htp/diag-ops.c new file mode 100644 index 0000000..7f03e85 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/diag-ops.c @@ -0,0 +1,243 @@ +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "hvx-types.h" +#include "hex-utils.h" +#include "hvx-copy.h" +#include "dma-queue.h" +#include "diag-ops.h" + +#define htp_diag_tensors_preamble \ + const struct htp_tensor * restrict src0 = octx->src[0]; \ + const struct htp_tensor * restrict dst = octx->dst; \ + \ + const uint32_t ne02 = src0->ne[2]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; + +struct htp_diag_context { + struct htp_ops_context * octx; + const struct htp_diag_kernel_params * kparams; + size_t src_batch_size; + size_t dst_row_size; + size_t src_batch_size_aligned; + size_t dst_row_size_aligned; + uint32_t batches_per_thread; + uint32_t total_batches; + uint32_t batch_start; +}; + +#define htp_diag_preamble \ + struct htp_diag_context * dctx = (struct htp_diag_context *) data; \ + struct htp_ops_context * octx = dctx->octx; \ + htp_diag_tensors_preamble; + +static inline void hvx_diag_row_f32(const float * restrict src, float * restrict dst, + uint32_t row_idx, uint32_t n) { + hvx_splat_f32_a((uint8_t *) dst, 0.0f, n); + dst[row_idx] = src[row_idx]; +} + +// --------------------------------------------------------------------------- +// Per thread worker: DMA src fetch, compute in VTCM, DMA dst writeback +// --------------------------------------------------------------------------- + +static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) { + htp_diag_preamble; + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + + if (ib0 >= ib1) { + return; + } + + const size_t src_batch_size = dctx->src_batch_size; + const size_t dst_row_size = dctx->dst_row_size; + const size_t src_batch_size_aligned = dctx->src_batch_size_aligned; + const size_t dst_row_size_aligned = dctx->dst_row_size_aligned; + + const dma_addr_t src_data = src0->data; + const dma_addr_t dst_data = dst->data; + + // 1 src buffer + 1 dst row buffer per thread in VTCM + uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned); + uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + for (uint32_t ib = ib0; ib < ib1; ib++) { + const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02); + const uint32_t i2 = ib - i3 * ne02; + + const dma_addr_t src_batch = src_data + i3 * nb03 + i2 * nb02; + + // Fetch source vector into VTCM + dma_queue_push(dma_q, + dma_make_data(src_spad, src_batch), + src_batch_size_aligned, src_batch_size, src_batch_size, 1); + dma_queue_flush(dma_q); + + const float * src_spad_f32 = (const float *) src_spad; + float * dst_spad_f32 = (float *) dst_spad; + + for (uint32_t i1 = 0; i1 < ne1; i1++) { + // Compute row in VTCM + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1)); + hvx_diag_row_f32(src_spad_f32, dst_spad_f32, i1, ne0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1)); + + // Write completed row back to DDR + const dma_addr_t dst_row = dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1; + dma_queue_push(dma_q, + dma_make_data(dst_row, dst_spad), + dst_row_size, dst_row_size_aligned, dst_row_size, 1); + dma_queue_flush(dma_q); + } + } + + FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", + ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1, + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); +} + +// --------------------------------------------------------------------------- +// Per thread worker: Direct HVX (no DMA) +// --------------------------------------------------------------------------- + +static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) { + htp_diag_preamble; + + const uint8_t * src_data = (const uint8_t *) src0->data; + uint8_t * dst_data = (uint8_t *) dst->data; + + const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith; + const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); + + for (uint32_t ib = ib0; ib < ib1; ib++) { + const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02); + const uint32_t i2 = ib - i3 * ne02; + + const float * restrict src_batch = (const float *)(src_data + i3 * nb03 + i2 * nb02); + + for (uint32_t i1 = 0; i1 < ne1; i1++) { + float * restrict dst_row = (float *)(dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1); + hvx_diag_row_f32(src_batch, dst_row, i1, ne0); + } + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0); + + FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n", + ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1, + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); +} + +int op_diag_f32(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + const struct htp_diag_kernel_params * kparams = + (const struct htp_diag_kernel_params *) octx->kernel_params; + + const uint32_t total_batches = kparams->total_batches; + + uint32_t batch_start = 0; + uint32_t nbatches = total_batches; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = (kparams->can_split != 0) && htp_tensor_mdev_data_aligned(dst); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? kparams->batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + batch_start = range.start; + nbatches = range.count; + } + + if (nbatches == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nbatches))) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + + if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { + return HTP_STATUS_NO_SUPPORT; + } + + const size_t src_batch_size = src0->ne[0] * sizeof(float); + const size_t dst_row_size = dst->ne[0] * sizeof(float); + + struct htp_diag_vtcm_layout layout = { 0 }; + if (kparams->use_dma) { + htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads); + + octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread; + octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread; + octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread; + octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread; + octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); + octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); + octx->src0_spad.src = NULL; + octx->dst_spad.src = NULL; + } + + struct htp_diag_context dctx = { + .octx = octx, + .kparams = kparams, + .src_batch_size = src_batch_size, + .dst_row_size = dst_row_size, + .src_batch_size_aligned = layout.src_batch_size_aligned, + .dst_row_size_aligned = layout.dst_row_size_aligned, + .batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div), + .total_batches = nbatches, + .batch_start = batch_start, + }; + + if (kparams->use_dma) { + work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads); + } else { + work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads); + } + + return HTP_STATUS_OK; +} + +int op_diag(struct htp_ops_context * octx) { + const struct htp_tensor * dst = octx->dst; + + int err = HTP_STATUS_OK; + + switch (dst->type) { + case HTP_TYPE_F32: + err = op_diag_f32(octx); + break; + default: + err = HTP_STATUS_NO_SUPPORT; + break; + } + + return err; +} diff --git a/ggml/src/ggml-hexagon/htp/diag-ops.h b/ggml/src/ggml-hexagon/htp/diag-ops.h new file mode 100644 index 0000000..983bc02 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/diag-ops.h @@ -0,0 +1,54 @@ +#ifndef HTP_DIAG_OPS_H +#define HTP_DIAG_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include + +struct htp_diag_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + size_t src_batch_size_aligned; + size_t dst_row_size_aligned; + size_t src0_bytes_per_thread; + size_t dst_bytes_per_thread; +}; + +static inline void htp_diag_vtcm_layout_build( + struct htp_diag_vtcm_layout * L, + size_t src_batch_size, + size_t dst_row_size, + uint32_t n_threads +) { + L->src_batch_size_aligned = hex_round_up(src_batch_size, 128); + L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); + L->src0_bytes_per_thread = L->src_batch_size_aligned; + L->dst_bytes_per_thread = L->dst_row_size_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + + L->total_bytes = off; +} + +struct htp_diag_kernel_params { + uint32_t n_threads; + uint32_t total_batches; + uint32_t use_dma; + uint32_t can_split; + uint32_t batches_per_chunk; + uint32_t vtcm_size; + + struct fastdiv_values div_ne02; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_DIAG_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/dma-copy.h b/ggml/src/ggml-hexagon/htp/dma-copy.h new file mode 100644 index 0000000..b2c91a1 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dma-copy.h @@ -0,0 +1,169 @@ +#ifndef HTP_DMA_COPY_H +#define HTP_DMA_COPY_H + +// DDR<->DDR DMA copies of same-type, same-shape tensors with arbitrary strides. +// Used by CPY for the copy itself and by CONCAT, which is two such copies into +// two views of its destination. Every helper only pushes descriptors; the +// caller flushes the queue when it needs the data. + +#include "dma-queue.h" +#include "hex-common.h" +#include "htp-tensor.h" + +#include +#include + +// Contiguous byte run, as 1d transfers of at most DMA_SAFE_CHUNK_SIZE each. +static inline void dma_cpy_sametype_reshape_contig(dma_queue * dma_q, + dma_addr_t dst, + dma_addr_t src0, + uint32_t total_bytes) { + if (total_bytes == 0) { + return; + } + + const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE; + while (total_bytes > 0) { + const uint32_t chunk = MIN(total_bytes, max_chunk); + if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1)) { + dma_queue_flush(dma_q); + dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1); + } + dst += chunk; + src0 += chunk; + total_bytes -= chunk; + } +} + +// One 2d transfer, split at the 16-bit nrows field. +static inline void dma_cpy_push_2d_chunked(dma_queue * dma_q, + dma_addr_t dst, + dma_addr_t src, + size_t dst_stride, + size_t src_stride, + size_t row_size, + uint32_t nrows) { + if (row_size == 0 || nrows == 0) { + return; + } + + while (nrows > 0) { + const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS); + if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) { + dma_queue_flush(dma_q); + dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows); + } + dst += cur_rows * dst_stride; + src += cur_rows * src_stride; + nrows -= cur_rows; + } +} + +// Copy a range of rows [row_start, row_start + nrows) from src0 into dst: +// same type, same ne[], any nb[] above dim 0, dim 0 dense on both sides (nb[0] == elem_size). +static inline void dma_cpy_sametype_sameshape_range(dma_queue * dma_q, + const struct htp_tensor * dst, + const struct htp_tensor * src0, + uint32_t elem_size, + uint32_t row_start, + uint32_t nrows) { + if (nrows == 0) { + return; + } + + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne01 = src0->ne[1]; + const uint32_t ne02 = src0->ne[2]; + const uint32_t ne03 = src0->ne[3]; + + if (ne00 == 0 || ne01 == 0 || ne02 == 0 || ne03 == 0) { + return; + } + + const uint32_t nb01 = src0->nb[1]; + const uint32_t nb02 = src0->nb[2]; + const uint32_t nb03 = src0->nb[3]; + + const uint32_t nb1 = dst->nb[1]; + const uint32_t nb2 = dst->nb[2]; + const uint32_t nb3 = dst->nb[3]; + + const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size); + + if (contiguous) { + dma_cpy_sametype_reshape_contig(dma_q, + dst->data + (dma_addr_t) row_start * ne00 * elem_size, + src0->data + (dma_addr_t) row_start * ne00 * elem_size, + nrows * ne00 * elem_size); + return; + } + + // The single-descriptor path flattens (i01,i02,i03) into one row index, so every + // row must sit at a constant stride: nb01 on the source, nb1 on the destination. + // Walk the outer dims and require each to continue that progression. A dim of + // extent 1 spans no rows, so it is skipped -- but its own stride must NOT then be + // used to justify the next dim's stride, which is what comparing nb03 against + // ne02*nb02 did: ggml leaves the stride of an extent-1 dim meaningless, so a view + // could pass the check while its rows were nowhere near that stride. + uint32_t exp_src = ne01 * nb01; + uint32_t exp_dst = ne01 * nb1; + bool contiguous_outer = true; + if (ne02 != 1) { + contiguous_outer = contiguous_outer && (nb02 == exp_src) && (nb2 == exp_dst); + } + exp_src *= ne02; + exp_dst *= ne02; + if (ne03 != 1) { + contiguous_outer = contiguous_outer && (nb03 == exp_src) && (nb3 == exp_dst); + } + + if (contiguous_outer) { + dma_cpy_push_2d_chunked(dma_q, + dst->data + (dma_addr_t) row_start * nb1, + src0->data + (dma_addr_t) row_start * nb01, + nb1, nb01, ne00 * elem_size, nrows); + return; + } + + const uint32_t ne02_ne01 = ne02 * ne01; + uint32_t i03 = row_start / ne02_ne01; + uint32_t rem = row_start - i03 * ne02_ne01; + uint32_t i02 = rem / ne01; + uint32_t i01 = rem - i02 * ne01; + + dma_addr_t cur_dst = dst->data + (dma_addr_t) i01 * nb1 + (dma_addr_t) i02 * nb2 + (dma_addr_t) i03 * nb3; + dma_addr_t cur_src0 = src0->data + (dma_addr_t) i01 * nb01 + (dma_addr_t) i02 * nb02 + (dma_addr_t) i03 * nb03; + + uint32_t r = row_start; + const uint32_t row_end = row_start + nrows; + while (r < row_end) { + uint32_t cur_rows = MIN(row_end - r, ne01 - i01); + dma_cpy_push_2d_chunked(dma_q, cur_dst, cur_src0, nb1, nb01, ne00 * elem_size, cur_rows); + r += cur_rows; + i01 += cur_rows; + if (i01 == ne01) { + i01 = 0; + if (++i02 == ne02) { + i02 = 0; + i03++; + } + cur_dst = dst->data + (dma_addr_t) i02 * nb2 + (dma_addr_t) i03 * nb3; + cur_src0 = src0->data + (dma_addr_t) i02 * nb02 + (dma_addr_t) i03 * nb03; + } else { + cur_dst += cur_rows * nb1; + cur_src0 += cur_rows * nb01; + } + } +} + +// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on +// both sides (nb[0] == elem_size). +static inline void dma_cpy_sametype_sameshape(dma_queue * dma_q, + const struct htp_tensor * dst, + const struct htp_tensor * src0, + uint32_t elem_size) { + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + dma_cpy_sametype_sameshape_range(dma_q, dst, src0, elem_size, 0, total_rows); +} + +#endif /* HTP_DMA_COPY_H */ diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.c b/ggml/src/ggml-hexagon/htp/dma-queue.c new file mode 100644 index 0000000..ef61d2d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dma-queue.c @@ -0,0 +1,212 @@ +#include "dma-queue.h" + +#include +#include +#include + +#pragma clang diagnostic ignored "-Wunused-function" + +static inline uint32_t pow2_ceil(uint32_t x) { + if (x <= 1) { + return 1; + } + int p = 2; + x--; + while (x >>= 1) { + p <<= 1; + } + return p; +} + +static inline uintptr_t align_up(uintptr_t addr, size_t align) { + return (addr + align - 1) & ~(align - 1); +} + +static inline size_t dma_ring_sizeof(size_t capacity) { + capacity = pow2_ceil(capacity); + + size_t size_r = sizeof(dma_ring); + size_t offset_desc = align_up(size_r, HEX_L2_LINE_SIZE); + size_t size_desc = capacity * sizeof(dma_descriptor_2d); + size_t offset_data = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); + size_t size_data = capacity * sizeof(dma_data); + + return offset_data + size_data; +} + +static inline dma_ring * dma_ring_init(void * ptr, size_t capacity, struct htp_thread_trace * trace) { + capacity = pow2_ceil(capacity); + + size_t size_r = sizeof(dma_ring); + size_t offset_desc = align_up(size_r, HEX_L2_LINE_SIZE); + size_t size_desc = capacity * sizeof(dma_descriptor_2d); + size_t offset_data = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE); + + dma_ring * r = (dma_ring *) ptr; + r->trace = trace; + r->capacity = capacity; + r->idx_mask = capacity - 1; + r->push_idx = 0; + r->pop_idx = 0; + r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc); + r->data = (dma_data *) ((uintptr_t) ptr + offset_data); + r->tail = &r->desc[capacity - 1]; + + return r; +} + +size_t dma_queue_sizeof(size_t capacity) { + size_t size_q = sizeof(dma_queue); + size_t offset_r0 = align_up(size_q, HEX_L2_LINE_SIZE); + size_t size_r0 = dma_ring_sizeof(capacity); + size_t offset_r1 = align_up(offset_r0 + size_r0, HEX_L2_LINE_SIZE); + size_t size_r1 = dma_ring_sizeof(DMA_FALLBACK_CAPACITY); + + return offset_r1 + size_r1; +} + +size_t dma_queue_alignof(void) { + return HEX_L2_LINE_SIZE; +} + +dma_queue_t dma_queue_init(void * ptr, size_t capacity, struct htp_thread_trace * trace) { + size_t total_size = dma_queue_sizeof(capacity); + memset(ptr, 0, total_size); + + dma_queue * q = (dma_queue *) ptr; + + size_t size_q = sizeof(dma_queue); + size_t offset_r0 = align_up(size_q, HEX_L2_LINE_SIZE); + size_t size_r0 = dma_ring_sizeof(capacity); + size_t offset_r1 = align_up(offset_r0 + size_r0, HEX_L2_LINE_SIZE); + + q->ring0 = dma_ring_init((void *) ((uintptr_t) ptr + offset_r0), capacity, trace); + q->ring1 = dma_ring_init((void *) ((uintptr_t) ptr + offset_r1), DMA_FALLBACK_CAPACITY, trace); + q->alias = false; + + FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", (unsigned) capacity, total_size); + + return q; +} + +void dma_queue_free(dma_queue_t q) { + (void) q; +} + +size_t dma_queue_alias_sizeof(void) { + return sizeof(dma_queue); +} + +dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q) { + dma_queue * q = (dma_queue *) ptr; + memset(q, 0, sizeof(dma_queue)); + + q->ring0 = main_q->ring0; + q->ring1 = main_q->ring1; + q->alias = true; + + return q; +} + +void dma_queue_alias_free(dma_queue_t q) { + (void) q; +} + +bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + dma_ring * r0 = q->ring0; + dma_ring * r1 = q->ring1; + + if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) { + return false; + } + + r1->tail = r0->tail; + + size_t rem_rows = nrows; + dma_addr_t cur_dst = ddata.dst; + dma_addr_t cur_src = ddata.src; + + while (rem_rows > 0) { + const uint32_t cur_rows = MIN(rem_rows, DMA_MAX_NROWS); + dma_data cur_data = dma_make_data(cur_dst, cur_src); + if (!dma_ring_push_single_2d(r1, cur_data, dst_stride, src_stride, row_size, cur_rows)) { + dma_ring_flush(r1); + dma_ring_push_single_2d(r1, cur_data, dst_stride, src_stride, row_size, cur_rows); + } + cur_dst += cur_rows * dst_stride; + cur_src += cur_rows * src_stride; + rem_rows -= cur_rows; + } + + dma_ring_flush(r1); + r0->tail = r1->tail; + + return dma_ring_push_single_2d(r0, ddata, 0, 0, 0, /*nrows=*/ 0); +} + +bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total) { + dma_ring * r0 = q->ring0; + dma_ring * r1 = q->ring1; + + if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) { + return false; + } + + r1->tail = r0->tail; + + size_t rem_bytes = total; + dma_addr_t cur_dst = ddata.dst; + dma_addr_t cur_src = ddata.src; + + while (rem_bytes > 0) { + const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE); + dma_data cur_data = dma_make_data(cur_dst, cur_src); + if (!dma_ring_push_single_contig(r1, cur_data, cur_bytes)) { + dma_ring_flush(r1); + dma_ring_push_single_contig(r1, cur_data, cur_bytes); + } + cur_dst += cur_bytes; + cur_src += cur_bytes; + rem_bytes -= cur_bytes; + } + + dma_ring_flush(r1); + r0->tail = r1->tail; + + return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0); +} + +#if __HVX_ARCH__ < 75 + +bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + dma_ring * r0 = q->ring0; + dma_ring * r1 = q->ring1; + + if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) { + return false; + } + + r1->tail = r0->tail; + + size_t rem_rows = nrows; + dma_addr_t cur_dst = ddata.dst; + dma_addr_t cur_src = ddata.src; + + while (rem_rows > 0) { + dma_data cur_data = dma_make_data(cur_dst, cur_src); + if (!dma_ring_push_single_1d(r1, cur_data, row_size)) { + dma_ring_flush(r1); + dma_ring_push_single_1d(r1, cur_data, row_size); + } + cur_dst += dst_stride; + cur_src += src_stride; + rem_rows -= 1; + } + + dma_ring_flush(r1); + r0->tail = r1->tail; + + return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0); +} + +#endif diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h new file mode 100644 index 0000000..05f3730 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -0,0 +1,525 @@ +#ifndef HTP_DMA_H +#define HTP_DMA_H + +#include +#include +#include +#include +#include +#include +#include "hex-utils.h" + +#include "hex-profile.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date +typedef struct dma_descriptor_1d_s { + void * next; + uint32_t size:24; + uint32_t desc_size:2; + uint32_t dst_comp:1; + uint32_t src_comp:1; + uint32_t dst_bypass:1; + uint32_t src_bypass:1; + uint32_t order:1; + uint32_t done:1; + uint32_t src; + uint32_t dst; +} dma_descriptor_1d; + +#if __HVX_ARCH__ < 75 + +typedef struct dma_descriptor_2d_s { + void * next; + uint32_t reserved0:24; + uint32_t desc_size:2; + uint32_t dst_comp:1; + uint32_t src_comp:1; + uint32_t dst_bypass:1; + uint32_t src_bypass:1; + uint32_t order:1; + uint32_t done:1; + uint32_t src; + uint32_t dst; + uint32_t desc_type:8; + uint32_t reserved1:24; + uint32_t row_size:16; + uint32_t nrows:16; + uint32_t src_stride:16; + uint32_t dst_stride:16; + uint32_t src_offset:16; + uint32_t dst_offset:16; +} dma_descriptor_2d; + +#else + +typedef struct dma_descriptor_2d_s { + void * next; + uint32_t dst_stride:24; + uint32_t desc_size:2; + uint32_t dst_comp:1; + uint32_t src_comp:1; + uint32_t dst_bypass:1; + uint32_t src_bypass:1; + uint32_t order:1; + uint32_t done:1; + uint32_t src; + uint32_t dst; + uint32_t desc_type:8; +#if __HVX_ARCH__ > 79 + uint32_t src_upper:8; + uint32_t dst_upper:8; + uint32_t allocation:2; + uint32_t reserved0:2; + uint32_t transform:4; +#else + uint32_t reserved0:24; +#endif + uint32_t row_size:24; + uint32_t nrows_lo:8; + uint32_t nrows_hi:8; + uint32_t src_stride:24; + uint32_t offset:24; + uint32_t reserved1:8; +} dma_descriptor_2d; + +#endif + +#if __HVX_ARCH__ > 79 +typedef uint64_t dma_addr_t; +#else +typedef uint32_t dma_addr_t; +#endif + +typedef struct { + dma_addr_t dst; + dma_addr_t src; +} dma_data; + +// Hardware descriptor field limits +#define DMA_MAX_NROWS 0xFFFFu // 16-bit HW descriptor limit (65535) +#define DMA_MAX_SIZE_16B 0xFFFFu // 16-bit HW descriptor limit for row_size (65535) +#define DMA_MAX_STRIDE_16B 0xFFFFu // 16-bit HW descriptor limit for strides (65535) +#define DMA_MAX_SIZE_24B 0x00FFFFFFu // 24-bit HW descriptor limit for row_size / 1D size (16MB - 1) +#define DMA_MAX_STRIDE_24B 0x00FFFFFFu // 24-bit HW descriptor limit for strides (16MB - 1) +#define DMA_SAFE_CHUNK_SIZE 0x00F00000u // ~15MB safe contiguous chunk size + +#if __HVX_ARCH__ < 75 +#define DMA_MAX_2D_ROW_SIZE DMA_MAX_SIZE_16B +#define DMA_MAX_2D_STRIDE DMA_MAX_STRIDE_16B +#else +#define DMA_MAX_2D_ROW_SIZE DMA_MAX_SIZE_24B +#define DMA_MAX_2D_STRIDE DMA_MAX_STRIDE_24B +#endif + +#define DMA_FALLBACK_CAPACITY 16u // descriptors in secondary fallback ring + +typedef struct dma_ring_s dma_ring; +struct dma_ring_s { + dma_descriptor_2d * desc; // descriptor pointers + dma_descriptor_2d * tail; // tail pointer + dma_data * data; // dst/src data + uint32_t push_idx; + uint32_t pop_idx; + uint32_t capacity; + uint32_t idx_mask; + struct htp_thread_trace * trace; +}; + +typedef struct dma_queue_s dma_queue; +typedef dma_queue * dma_queue_t; + +struct dma_queue_s { + dma_ring * ring0; // Main descriptor ring state + dma_ring * ring1; // Secondary fallback descriptor ring state + bool alias; // When set, dma_queue_delete will not free the ring +}; + +size_t dma_queue_sizeof(size_t capacity); +size_t dma_queue_alignof(void); +dma_queue_t dma_queue_init(void * ptr, size_t capacity, struct htp_thread_trace * trace); +void dma_queue_free(dma_queue_t q); + +size_t dma_queue_alias_sizeof(void); +dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q); +void dma_queue_alias_free(dma_queue_t q); + +bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows); +bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total); +#if __HVX_ARCH__ < 75 +bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows); +#endif + +// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead +// but those do not seem to always compiler properly. +static inline void dmstart(void * next) { + asm volatile(" release(%0):at" : : "r"(next)); + asm volatile(" dmstart(%0)" : : "r"(next)); +} + +static inline void dmlink(void * cur, void * next) { + asm volatile(" release(%0):at" : : "r"(next)); + asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next)); +} + +static inline unsigned int dmpoll(void) { + unsigned int ret = 0; + asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory"); + return ret; +} + +static inline unsigned int dmwait(void) { + unsigned int ret = 0; + asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory"); + return ret; +} + +static inline dma_data dma_make_data_impl(dma_addr_t dst, dma_addr_t src) +{ + dma_data d = { dst, src }; + return d; +} + +#define dma_make_data(dst, src) dma_make_data_impl((dma_addr_t) (dst), (dma_addr_t) (src)) + +static inline bool dma_ring_push_single_1d(dma_ring * r, dma_data ddata, size_t size) { +#if __HVX_ARCH__ > 79 + assert(!((ddata.src | ddata.dst) >> 32) || size == 0); +#endif + + if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { + return false; + } + + dma_descriptor_1d * desc = (dma_descriptor_1d *) &r->desc[r->push_idx]; + desc->src = (uint32_t) ddata.src; + desc->dst = (uint32_t) ddata.dst; + desc->size = size; + + r->data[r->push_idx] = ddata; + + htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx); + + if (size) { + desc->next = NULL; + desc->desc_size = 0; // 1D mode + desc->src_bypass = 1; + desc->dst_bypass = 1; + desc->order = 0; + desc->done = 0; + + dmlink(r->tail, desc); + r->tail = (dma_descriptor_2d *) desc; + } else { + desc->desc_size = 0; + desc->done = 1; + } + + r->push_idx = (r->push_idx + 1) & r->idx_mask; + return true; +} + +static inline bool dma_ring_push_single_2d(dma_ring * r, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { +#if __HVX_ARCH__ > 79 + assert(!((ddata.src | ddata.dst) >> 40) || nrows == 0); + const uint32_t src_hi = (uint32_t) (ddata.src >> 32); + const uint32_t dst_hi = (uint32_t) (ddata.dst >> 32); + const bool is_ext = (src_hi | dst_hi) != 0; +#endif + + if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { + return false; + } + + dma_descriptor_2d * desc = &r->desc[r->push_idx]; + + desc->next = NULL; + desc->reserved1 = 0; + desc->desc_size = 1; // 2d mode + desc->src_bypass = 1; + desc->dst_bypass = 1; + desc->src_comp = 0; + desc->dst_comp = 0; + desc->order = 0; + desc->done = 0; + desc->src_stride = src_stride; + desc->dst_stride = dst_stride; + desc->src = (uint32_t) ddata.src; + desc->dst = (uint32_t) ddata.dst; + desc->row_size = row_size; + +#if __HVX_ARCH__ < 75 + desc->reserved0 = 0; + desc->desc_type = 0; // 2d (16-bit) mode + desc->nrows = nrows; + desc->src_offset = 0; + desc->dst_offset = 0; +#else +#if __HVX_ARCH__ > 79 + desc->src_upper = src_hi; + desc->dst_upper = dst_hi; + desc->allocation = 0; + desc->reserved0 = 0; + desc->transform = 0; + desc->desc_type = is_ext ? 10 : 9; // 2d 40-bit or 24-bit mode +#else + desc->reserved0 = 0; + desc->desc_type = 9; // 2d (24-bit) mode +#endif + desc->nrows_lo = (nrows & 0xff); + desc->nrows_hi = (nrows >> 8); + desc->offset = 0; +#endif + + r->data[r->push_idx] = ddata; + + htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx); + + if (nrows) { + dmlink(r->tail, desc); + r->tail = desc; + } else { + desc->done = 1; + } + + r->push_idx = (r->push_idx + 1) & r->idx_mask; + return true; +} + +#if __HVX_ARCH__ < 75 +static inline bool dma_ring_push_single_contig(dma_ring * r, dma_data ddata, size_t size) { + return dma_ring_push_single_1d(r, ddata, size); +} +#else +static inline bool dma_ring_push_single_contig(dma_ring * r, dma_data ddata, size_t size) { + return dma_ring_push_single_2d(r, ddata, size, size, size, 1); +} +#endif + +static inline dma_data dma_ring_pop(dma_ring * r) { + dma_data ddata = { 0 }; + + if (r->push_idx == r->pop_idx) { + return ddata; + } + + ddata = r->data[r->pop_idx]; + + volatile dma_descriptor_2d * desc = &r->desc[r->pop_idx]; + + // Wait for desc to complete + if (!desc->done) { + // FARF(ALWAYS, "dma-poll: idx %u dst %p src %p", r->pop_idx, ddata.dst, ddata.src); + while (!desc->done) { + dmpoll(); + } + } + + htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx); + + r->pop_idx = (r->pop_idx + 1) & r->idx_mask; + return ddata; +} + +static inline dma_data dma_ring_pop_nowait(dma_ring * r) { + dma_data ddata = { 0 }; + + if (r->push_idx == r->pop_idx) { + return ddata; + } + + ddata = r->data[r->pop_idx]; + + htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx); + + r->pop_idx = (r->pop_idx + 1) & r->idx_mask; + return ddata; +} + +static inline bool dma_ring_empty(dma_ring * r) { + return r->push_idx == r->pop_idx; +} + +static inline void dma_ring_flush(dma_ring * r) { + while (!dma_ring_empty(r)) { + dma_ring_pop(r); + } +} + +static inline uint32_t dma_ring_depth(dma_ring * r) { + return (r->push_idx - r->pop_idx) & r->idx_mask; +} + +static inline uint32_t dma_ring_capacity(dma_ring * r) { + return r->capacity; +} + +static inline bool dma_queue_push_single_1d(dma_queue * q, dma_data ddata, size_t size) { + return dma_ring_push_single_1d(q->ring0, ddata, size); +} + +static inline bool dma_queue_push_single_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + return dma_ring_push_single_2d(q->ring0, ddata, dst_stride, src_stride, row_size, nrows); +} + +static inline dma_data dma_queue_pop(dma_queue * q) { + return dma_ring_pop(q->ring0); +} + +static inline dma_data dma_queue_pop_nowait(dma_queue * q) { + return dma_ring_pop_nowait(q->ring0); +} + +static inline bool dma_queue_empty(dma_queue * q) { + return dma_ring_empty(q->ring0); +} + +static inline void dma_queue_flush(dma_queue * q) { + dma_ring_flush(q->ring0); +} + +static inline uint32_t dma_queue_depth(dma_queue * q) { + return dma_ring_depth(q->ring0); +} + +static inline uint32_t dma_queue_capacity(dma_queue * q) { + return dma_ring_capacity(q->ring0); +} + +static inline bool dma_queue_push(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + if (__builtin_expect(nrows == 0, 0)) { + return dma_ring_push_single_1d(q->ring0, ddata, 0); + } + + // 1. Hot path: Contiguous or single-row (80-90% of calls) + if (nrows == 1 || !((row_size ^ src_stride) | (row_size ^ dst_stride))) { + const size_t total = row_size * nrows; + if (__builtin_expect(total <= DMA_MAX_SIZE_24B, 1)) { + return dma_ring_push_single_contig(q->ring0, ddata, total); + } + return dma_queue_push_fallback_contig(q, ddata, total); + } + + // 2. Hot path: Standard strided 2D (10-20% of calls) + if (__builtin_expect(nrows <= DMA_MAX_NROWS && + (row_size | src_stride | dst_stride) <= DMA_MAX_2D_ROW_SIZE, 1)) { + return dma_ring_push_single_2d(q->ring0, ddata, dst_stride, src_stride, row_size, nrows); + } + + // 3. Cold path: Descriptor chunking fallbacks (< 0.1%) +#if __HVX_ARCH__ < 75 + if (row_size <= DMA_MAX_SIZE_16B && (src_stride | dst_stride) <= DMA_MAX_STRIDE_16B) { + return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows); + } + return dma_queue_push_fallback_1d(q, ddata, dst_stride, src_stride, row_size, nrows); +#else + return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows); +#endif +} + +static inline void dma_sync_read(dma_queue * dma_q, void * dst, dma_addr_t src, size_t bytes) { + const uint32_t b = (uint32_t) bytes; + if (b > 0) { + dma_queue_push(dma_q, dma_make_data(dst, src), b, b, b, 1); + dma_queue_pop(dma_q); + } +} + +static inline void dma_sync_write(dma_queue * dma_q, dma_addr_t dst, const void * src, size_t bytes) { + const uint32_t b = (uint32_t) bytes; + if (b > 0) { + dma_queue_push(dma_q, dma_make_data(dst, src), b, b, b, 1); + dma_queue_pop(dma_q); + } +} + +#define DMA_CACHE_MAX_SIZE 128U + +// Fully assoc LRU cache +typedef struct __attribute__((aligned(128))) { + uint8_t *base; + uint32_t line_size; + uint32_t capacity; + dma_addr_t src[DMA_CACHE_MAX_SIZE]; + uint16_t age[DMA_CACHE_MAX_SIZE]; +} dma_cache_fa; + +static inline void dma_cache_fa_init(dma_cache_fa *c, uint8_t *base, uint32_t line_size, uint32_t capacity) +{ + c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity; + c->base = base; + c->line_size = line_size; + + for (unsigned i=0; i < c->capacity; i++) { + c->src[i] = 0; + c->age[i] = 0; + } +} + +static inline bool dma_cache_fa_push(dma_queue *q, dma_cache_fa *c, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) +{ + uint32_t o_idx = 0; + uint16_t o_age = 0; + uint8_t * dst = 0; + + for (unsigned i=0; i < c->capacity; i++) { + if (c->src[i] == src_addr) { + c->age[i] = 0; + dst = c->base + (i * c->line_size); nrows = 0; // dummy dma + } else { + c->age[i]++; + if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; } + } + } + if (!dst) { + c->age[o_idx] = 0; + c->src[o_idx] = src_addr; + dst = c->base + o_idx * c->line_size; // normal nrows dma + return dma_queue_push(q, dma_make_data(dst, src_addr), dst_stride, src_stride, row_size, nrows); + } + + return dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0); +} + +// Direct mapped cache +typedef struct __attribute__((aligned(128))) { + uint8_t *base; + uint32_t line_size; + uint32_t capacity; + uint32_t idx_mask; + dma_addr_t src[DMA_CACHE_MAX_SIZE]; +} dma_cache_dm; + +static inline void dma_cache_dm_init(dma_cache_dm *c, uint8_t *base, uint32_t line_size, uint32_t capacity) +{ + c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity; + c->idx_mask = c->capacity - 1; + c->base = base; + c->line_size = line_size; + + for (unsigned i=0; i < c->capacity; i++) { + c->src[i] = 0; + } +} + +static inline bool dma_cache_dm_push(dma_queue *q, dma_cache_dm *c, uint32_t slot, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) +{ + const uint32_t i = slot & c->idx_mask; + uint8_t * dst = c->base + (i * c->line_size); + + if (c->src[i] == src_addr) { + return dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0); // dummy dma + } + + c->src[i] = src_addr; + return dma_queue_push(q, dma_make_data(dst, src_addr), dst_stride, src_stride, row_size, nrows); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* HTP_DMA_H */ diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.c b/ggml/src/ggml-hexagon/htp/fill-ops.c new file mode 100644 index 0000000..2cab31b --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/fill-ops.c @@ -0,0 +1,162 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include + +#include "hex-common.h" +#include "hex-profile.h" + +#include "hvx-copy.h" +#include "hvx-utils.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "fill-ops.h" + +// ggml op_params layout for FILL: +// op_params[0] (as float) - the scalar fill value + +#define fill_preamble \ + const struct htp_tensor * dst = octx->dst; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; + +struct htp_fill_context { + struct htp_ops_context * octx; + const struct htp_fill_kernel_params * kparams; + uint32_t nrows_per_thread; + uint32_t total_rows; // ne1 * ne2 * ne3 + uint32_t row_start; + HVX_Vector splat_vec; + uint32_t elem_size; +}; + +static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * data) { + const struct htp_fill_context * fctx = (const struct htp_fill_context *) data; + struct htp_ops_context * octx = fctx->octx; + fill_preamble; + + // Parallelise over the flat row index spanning ne1*ne2*ne3 + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); + + if (ir0 >= ir1) { + return; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); + + const uint32_t elem_start = ir0 * ne0; + const uint32_t elem_end = ir1 * ne0; + uint8_t * dst_ptr = (uint8_t *) dst->data + elem_start * fctx->elem_size; + hvx_splat_u(dst_ptr, fctx->splat_vec, elem_end - elem_start, fctx->elem_size); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1); + FARF(HIGH, "fill-contig %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); +} + +static void fill_thread_strided(unsigned int nth, unsigned int ith, void * data) { + const struct htp_fill_context * fctx = (const struct htp_fill_context *) data; + struct htp_ops_context * octx = fctx->octx; + const struct htp_fill_kernel_params * kparams = fctx->kparams; + fill_preamble; + + // Parallelise over the flat row index spanning ne1*ne2*ne3 + const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith; + const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows); + + if (ir0 >= ir1) { + return; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0); + + for (uint32_t ir = ir0; ir < ir1; ++ir) { + const uint32_t i3 = fastdiv(ir, &kparams->div_ne1_ne2); + const uint32_t rem = ir - i3 * (ne1 * ne2); + const uint32_t i2 = fastdiv(rem, &kparams->div_ne1); + const uint32_t i1 = rem - i2 * ne1; + uint8_t * dst_ptr = (uint8_t *) dst->data + i1*nb1 + i2*nb2 + i3*nb3; + hvx_splat_u(dst_ptr, fctx->splat_vec, ne0, fctx->elem_size); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1); + FARF(HIGH, "fill-strided %u/%u: rows %u:%u\n", ith, nth, ir0, ir1); +} + +int op_fill(struct htp_ops_context * octx) { + const struct htp_tensor * dst = octx->dst; + const struct htp_fill_kernel_params * kparams = + (const struct htp_fill_kernel_params *) octx->kernel_params; + + if (htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + uint32_t row_start = 0; + uint32_t nrows = kparams->total_rows; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + + FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u ktype=%u\n", + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, kparams->kernel_type); + + float val_f32 = 0.f; + memcpy(&val_f32, &octx->op_params[0], sizeof(float)); + + struct htp_fill_context fctx = { + .octx = octx, + .kparams = kparams, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_rows = nrows, + .row_start = row_start, + .elem_size = kparams->elem_size, + }; + + switch (dst->type) { + case HTP_TYPE_F32: + fctx.splat_vec = hvx_vec_splat_f32(val_f32); + break; + case HTP_TYPE_F16: + fctx.splat_vec = hvx_vec_splat_f16((_Float16) val_f32); + break; + default: + return HTP_STATUS_NO_SUPPORT; + } + + work_queue_func_t worker = (kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS) + ? fill_thread_contiguous + : fill_thread_strided; + + work_queue_run(octx->ctx->work_queue, worker, &fctx, n_threads); + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/fill-ops.h b/ggml/src/ggml-hexagon/htp/fill-ops.h new file mode 100644 index 0000000..d62f997 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/fill-ops.h @@ -0,0 +1,29 @@ +#ifndef HTP_FILL_OPS_H +#define HTP_FILL_OPS_H + +#include "hex-fastdiv.h" +#include + +enum htp_fill_kernel_type { + HTP_FILL_KERNEL_CONTIGUOUS = 0, + HTP_FILL_KERNEL_STRIDED = 1, +}; + +struct htp_fill_kernel_params { + uint8_t kernel_type; + uint32_t n_threads; + uint32_t total_rows; + uint32_t elem_size; + uint32_t rows_per_chunk; + + struct fastdiv_values div_ne1; + struct fastdiv_values div_ne1_ne2; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_FILL_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c index d296a32..ee25817 100644 --- a/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.c @@ -3,215 +3,60 @@ #pragma clang diagnostic ignored "-Wunused-but-set-variable" #include +#include #include -#include #include +#include +#include +#include +#include #include -#include "hex-dma.h" +#include "dma-queue.h" +#include "hex-fastdiv.h" +#include "hex-profile.h" +#include "hmx-queue.h" +#include "hmx-utils.h" #include "hvx-utils.h" #include "hvx-dump.h" +#include "hvx-copy.h" +#include "hvx-reduce.h" +#include "hvx-flash-attn.h" +#include "htp-vtcm.h" +#include "work-queue.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-quant.h" + +#include "flash-attn-ops.h" +#include "hvx-fa-kernels.h" +#include "hmx-fa-kernels.h" // Must be multiple of 32 #define FLASH_ATTN_BLOCK_SIZE (32 * 2) -// This is a bit of a hack because the compiler is strugling to properly inline -// the default hvx_vec_f32_to_f16 with output into the local array. -static void __attribute__((noinline)) hvx_vec_f32_to_f16_a(void *ptr, HVX_Vector v0, HVX_Vector v1) -{ - *(HVX_Vector *) ptr = hvx_vec_f32_to_f16(v0, v1); -} - -// Dot product of two F16 vectors, accumulating to float -static inline void hvx_dot_f16_f16_aa(float * restrict r, const void * restrict x, const void * restrict y, unsigned int n, float s) { - const HVX_Vector * restrict vx = (const HVX_Vector * restrict) x; // fp16 - const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16 - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_VectorPair rsum_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - - uint32_t i = 0; - - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, vx[i], vy[i]); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]); - HVX_Vector x_hf = Q6_V_vand_QV(bmask, vx[i]); - - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); - } - - HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p))); - rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(hvx_vec_splat_f32(s), hvx_vec_reduce_sum_f32(rsum))); - hvx_vec_store_u(r, 4, rsum); -} - -static inline HVX_Vector hvx_dot_f16_f16_aa_rx4(const void * restrict y, - const uint8_t * restrict x, - const size_t stride_x, - const size_t nvec, - const size_t nloe) { - const HVX_Vector * restrict vx0 = (const HVX_Vector * restrict) x; // fp16 - const HVX_Vector * restrict vx1 = (const HVX_Vector * restrict) (x + stride_x); // fp16 - const HVX_Vector * restrict vx2 = (const HVX_Vector * restrict) (x + stride_x * 2); // fp16 - const HVX_Vector * restrict vx3 = (const HVX_Vector * restrict) (x + stride_x * 3); // fp16 - const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16 - - HVX_VectorPair rsum0_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - HVX_VectorPair rsum1_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - HVX_VectorPair rsum2_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - HVX_VectorPair rsum3_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); - - uint32_t i = 0; - - for (i = 0; i < nvec; i++) { - HVX_Vector y_hf = vy[i]; - HVX_Vector x0_hf = vx0[i]; - HVX_Vector x1_hf = vx1[i]; - HVX_Vector x2_hf = vx2[i]; - HVX_Vector x3_hf = vx3[i]; - - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); - rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf); - rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf); - } - - if (nloe) { - // Load x (fp16) and zero-out unused elements - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]); - HVX_Vector x0_hf = Q6_V_vand_QV(bmask, vx0[i]); - HVX_Vector x1_hf = Q6_V_vand_QV(bmask, vx1[i]); - HVX_Vector x2_hf = Q6_V_vand_QV(bmask, vx2[i]); - HVX_Vector x3_hf = Q6_V_vand_QV(bmask, vx3[i]); - - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); - rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf); - rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf); - } - - HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p))); - HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p))); - HVX_Vector rsum2 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum2_p), Q6_V_hi_W(rsum2_p))); - HVX_Vector rsum3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum3_p), Q6_V_hi_W(rsum3_p))); - - HVX_Vector_x4 rsum0123 = { .v = { rsum0, rsum1, rsum2, rsum3 } }; - return hvx_vec_reduce_sum_f32x4(rsum0123); -} - -static inline HVX_Vector hvx_dot_f16_f16_aa_rx32(const void * restrict y, - const uint8_t * restrict x, - const size_t stride_x, - const size_t n, - float s) { - - const size_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - const size_t nloe = n % VLEN_FP16; // leftover elements - - HVX_Vector sums; // initialize at j = 0 - const size_t stride_x_4 = stride_x * 4; - for (uint32_t j = 0; j < VLEN_FP32; j += 4) { - HVX_Vector sums_x4 = hvx_dot_f16_f16_aa_rx4(y, x, stride_x, nvec, nloe); - HVX_VectorPred pred = Q6_Q_vsetq_R(j * SIZEOF_FP32); - sums = Q6_V_vmux_QVV(pred, sums, sums_x4); - x += stride_x_4; - } - - sums = Q6_Vqf32_vmpy_VsfVsf(hvx_vec_splat_f32(s), sums); - return Q6_Vsf_equals_Vqf32(sums); -} - -// MAD: y (F32) += x (F16) * s (F16) -static inline void hvx_mad_f32_f16_aa(float * restrict y, const void * restrict x, const __fp16 * restrict s, int n) { - const HVX_Vector * restrict vx0 = (const HVX_Vector *) x; - - HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; - HVX_Vector * restrict vy = (HVX_Vector *) y; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_Vector S0 = hvx_vec_splat_f16(*s); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; ++i) { - vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); - } - - if (nloe) { - HVX_VectorPair xy_p = vy_p[i]; - xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); - - HVX_Vector xy = Q6_V_lo_W(xy_p); - i = 2 * i; // index for vy - - if (nloe >= VLEN_FP32) { - vy[i] = xy; - nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); - } - - if (nloe) { - hvx_vec_store_a(&vy[i], nloe * 4, xy); - } - } -} - -// MAD: y (F32) += x0 (F16) * s0 (F16) + x1 (F16) * s1 (F16) -static inline void hvx_mad_f32_f16_aa_rx2(float * restrict y, const void * restrict x0, const void * restrict x1, - const __fp16 * restrict s0, const __fp16 * restrict s1, int n) { - const HVX_Vector * restrict vx0 = (const HVX_Vector *) x0; - const HVX_Vector * restrict vx1 = (const HVX_Vector *) x1; - - HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; - HVX_Vector * restrict vy = (HVX_Vector *) y; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_Vector S0 = hvx_vec_splat_f16(*s0); - HVX_Vector S1 = hvx_vec_splat_f16(*s1); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; ++i) { - vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); - vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx1[i]), S1); - } - - if (nloe) { - HVX_VectorPair xy_p = vy_p[i]; - xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); - xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx1[i]), S1); - - HVX_Vector xy = Q6_V_lo_W(xy_p); - i = 2 * i; // index for vy +// Large scratch lives in ctx->ddr_spad to keep stack usage low. +// Ops can run on the dspqueue callback thread, which has a small stack. +struct hvx_fa_ddr_spad { + __fp16 slopes[512]; + uint16_t active_blocks[FA_MAX_KV_BLOCKS]; + uint16_t thread_active_blocks[HTP_MAX_NTHREADS][FA_MAX_KV_BLOCKS] __attribute__((aligned(128))); +}; - if (nloe >= VLEN_FP32) { - vy[i] = xy; - nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); - } +struct hmx_fa_ddr_spad { + uint16_t active_blocks[FA_MAX_KV_BLOCKS]; + __fp16 slopes[] __attribute__((aligned(128))); // [g_br] +}; - if (nloe) { - hvx_vec_store_a(&vy[i], nloe * 4, xy); - } +static inline void * fa_get_ddr_spad(const struct htp_ops_context * octx, size_t size) { + if (!octx->ctx->ddr_spad_base || octx->ctx->ddr_spad_size < size) { + return NULL; } + return octx->ctx->ddr_spad_base; } struct htp_fa_context { @@ -230,13 +75,16 @@ struct htp_fa_context { float scale; float max_bias; - float logit_softcap; + bool has_softcap; + __fp16 logit_softcap; uint32_t n_head_log2; float m0; float m1; uint32_t n_blocks; + uint32_t n_active_blocks; + bool shared_active_blocks; size_t size_q_row_padded; size_t size_k_row_padded; @@ -248,31 +96,168 @@ struct htp_fa_context { uint32_t qrows; uint32_t qrows_per_thread; + uint32_t qrow_start; bool is_q_fp32; - uint64_t t_start; + size_t size_q_block; + size_t size_vkq_acc; + + uint8_t * spad_q; + uint8_t * spad_k; + uint8_t * spad_v; + uint8_t * spad_m; + uint8_t * spad_a; + float * spad_sinks; + + struct hvx_fa_ddr_spad * ddr; + dma_cache_dm * m_cache; // [n_threads] + + const struct htp_tensor * k; + const struct htp_tensor * v; +}; + +struct hmx_fa_context { + const struct htp_ops_context * octx; + const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent + const struct htp_tensor * k; + const struct htp_tensor * v; + bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2 + uint32_t n_threads; + + // Op parameters + __fp16 scale; + float max_bias; + bool has_softcap; + __fp16 logit_softcap; + uint32_t n_head_log2; + float m0, m1; + + // Dimensions + uint32_t DK, DV; + uint32_t DK_pad, DV_pad; // head_dim rounded up to 64 for HMX tiling + uint32_t n_kv; // kv_len + uint32_t n_kv_heads; // number of KV heads + uint32_t n_heads; // number of Q heads + uint32_t G; // GQA factor = n_heads / n_kv_heads + struct fastdiv_values div_G; + struct fastdiv_values src3_div2; + struct fastdiv_values src3_div3; + uint32_t n_kv_blocks; + uint32_t neq1; // Q token count + + // Types + bool is_q_fp32; + bool is_dst_fp32; + + // Dynamic block sizes + uint32_t Br; // Q tokens per block (before GQA expansion) + uint32_t Bc; + uint32_t g_br; // hex_align_up(G * Br, 32) - actual tile row dim + + // VTCM buffers (allocated by vtcm_seq_alloc) + __fp16 * vtcm_q_dma; // Q DMA fetch buffer + __fp16 * vtcm_q_tiles; // Q tile format [g_br, D] + __fp16 * vtcm_o_tiles[2]; // O ping-pong [g_br, D] + __fp16 * vtcm_k_fp16[2]; // K DMA double-buffer [Bc, D] + __fp16 * vtcm_v_fp16[2]; // V DMA double-buffer [Bc, D] + __fp16 * vtcm_k_tiles[2]; // K tiles (transposed, double-buffered) + __fp16 * vtcm_v_tiles[2]; // V tiles (column-major, double-buffered) + __fp16 * vtcm_s_tiles[2]; // S = QK^T [g_br, Bc] (double-buffered) + __fp16 * vtcm_p_tiles[2]; // P = softmax(S) [g_br, Bc] + __fp16 * vtcm_d_tiles[2]; // Diagonal rescale, g_br/32 packed diagonal tiles (double-buffered) + __fp16 * vtcm_d_inv_l; // Diagonal rescale (1/l), same packed layout + HVX_Vector * vtcm_m_vec; // Row max [g_br] + HVX_Vector * vtcm_l_vec; // Row sum [g_br] + HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br] + HVX_Vector * vtcm_p_rowsum; // Softmax intermediate [g_br] + HVX_Vector * vtcm_row_bufs; // Per-thread softmax row scratch [n_threads][2][Bc/64] + uint8_t * vtcm_hmx_scales_id; // HMX output scales (identity) + uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale) + __fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block + __fp16 * vtcm_slopes; // ALiBi slopes [g_br] + float * vtcm_sinks; // Attention sinks + __fp16 * ddr_slopes; // ALiBi slopes staging buffer in DDR [g_br] + size_t row_buf_stride; // HVX vectors per row buffer (Bc/64) + size_t mask_buf_row_stride; // elements (__fp16) per row in mask buffer + size_t q_tile_bytes; + size_t k_tile_bytes; + size_t o_tile_bytes; + size_t col_vec_bytes; + size_t d_tile_bytes; + bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA) + dma_cache_fa * m_cache; }; -static inline void hvx_scale_vec_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, HVX_Vector vs) { - assert((size_t) dst % 128 == 0); - assert((size_t) src % 128 == 0); +// Scan mask rows [q_start, q_start + n_rows_q) and build compact list of KV blocks with any entry != -inf. +static uint32_t fa_scan_active_blocks( + dma_queue * dma_q, + const struct htp_tensor * mask, + uint32_t q_start, + uint32_t n_rows_q, + uint32_t im2, + uint32_t im3, + uint32_t nek1, + uint32_t Bc, + uint32_t n_kv_blocks, + void * vtcm_scratch, + size_t scratch_bytes, + uint16_t * active_blocks) { + const size_t row_bytes = (size_t) nek1 * sizeof(__fp16); + const size_t row_stride = hex_round_up(row_bytes, 128); + + if (!mask || n_kv_blocks <= 1 || n_kv_blocks > FA_MAX_KV_BLOCKS || + (size_t) n_rows_q * row_stride > scratch_bytes) { + #pragma clang loop vectorize(disable) + for (uint32_t b = 0; b < n_kv_blocks; ++b) { + active_blocks[b] = (uint16_t) b; + } + return n_kv_blocks; + } + + const dma_addr_t m_src = mask->data + q_start * mask->nb[1] + im2 * mask->nb[2] + im3 * mask->nb[3]; + dma_queue_push(dma_q, dma_make_data(vtcm_scratch, m_src), row_stride, mask->nb[1], row_bytes, n_rows_q); + const uint8_t * m_buf = (const uint8_t *) dma_queue_pop(dma_q).dst; - const HVX_Vector * restrict vsrc = (const HVX_Vector * restrict) src; - HVX_Vector * restrict vdst = (HVX_Vector * restrict) dst; + const HVX_Vector v_ninf = Q6_Vh_vsplat_R(0xFC00); + uint32_t n_active = 0; - const uint32_t nvec = n / VLEN_FP32; - const uint32_t nloe = n % VLEN_FP32; + for (uint32_t b = 0; b < n_kv_blocks; ++b) { + const uint32_t col0 = b * Bc; + const uint32_t cols = MIN(Bc, nek1 - col0); - uint32_t i = 0; - #pragma unroll(4) - for (; i < nvec; ++i) { - vdst[i] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs)); + for (uint32_t r = 0; r < n_rows_q; ++r) { + const __fp16 * row_ptr = (const __fp16 *) (m_buf + r * row_stride) + col0; + HVX_Vector v_any = Q6_V_vzero(); + uint32_t c = 0; + for (; c + 64 <= cols; c += 64) { + const HVX_Vector v_m = *(const HVX_Vector *) (row_ptr + c); + v_any = Q6_V_vor_VV(v_any, Q6_V_vxor_VV(v_m, v_ninf)); + } + if (c < cols) { + const uint32_t rem = cols - c; + const HVX_VectorPred q_rem = Q6_Q_vsetq2_R(rem * sizeof(__fp16)); + const HVX_Vector v_m = *(const HVX_Vector *) (row_ptr + c); + const HVX_Vector v_diff = Q6_V_vand_QV(q_rem, Q6_V_vxor_VV(v_m, v_ninf)); + v_any = Q6_V_vor_VV(v_any, v_diff); + } + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 64)); + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 32)); + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 16)); + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 8)); + v_any = Q6_V_vor_VV(v_any, Q6_V_vror_VR(v_any, 4)); + if (Q6_R_vextract_VR(v_any, 0) != 0) { + active_blocks[n_active++] = (uint16_t) b; + break; + } + } } - if (nloe) { - HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs); - hvx_vec_store_a(&vdst[i], nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v)); + + if (n_active == 0 && n_kv_blocks > 0) { + active_blocks[0] = 0; + n_active = 1; } + return n_active; } static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) { @@ -321,35 +306,49 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t nb3 = dst->nb[3]; // total rows in q - const uint32_t nr = factx->qrows; - const uint32_t dr = factx->qrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, nr); + const uint32_t dr = factx->qrows_per_thread; + const uint32_t ir0 = factx->qrow_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, factx->qrow_start + factx->qrows); if (ir0 >= ir1) return; - dma_queue * dma = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + dma_queue * dma_q = octx->ctx->dma[ith]; const uint32_t DK = nek0; const uint32_t DV = nev0; const size_t size_q_row = DK * ((q->type == HTP_TYPE_F32) ? 4 : 2); - const size_t size_k_row = DK * sizeof(__fp16); - const size_t size_v_row = DV * sizeof(__fp16); + const size_t size_k_row = htp_tensor_get_row_size(k->type, DK); + const size_t size_v_row = htp_tensor_get_row_size(v->type, DV); // Scratchpad buffers for Q, K, V, Mask, and VKQ32 accumulator - uint8_t * spad_q = octx->src0_spad.data + octx->src0_spad.size_per_thread * ith; - uint8_t * spad_k = octx->src1_spad.data + octx->src1_spad.size_per_thread * ith; - uint8_t * spad_v = octx->src2_spad.data + octx->src2_spad.size_per_thread * ith; - uint8_t * spad_m = octx->src3_spad.data + octx->src3_spad.size_per_thread * ith; - uint8_t * spad_a = octx->dst_spad.data + octx->dst_spad.size_per_thread * ith; - - const HVX_Vector logit_cap = hvx_vec_splat_f32(factx->logit_softcap); + uint8_t * spad_q = factx->spad_q + factx->size_q_block * ith; + uint8_t * spad_k = factx->spad_k + factx->size_k_block * 2 * ith; + uint8_t * spad_v = factx->spad_v + factx->size_v_block * 2 * ith; + uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith; + uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith; + + dma_cache_dm * m_cache = &factx->m_cache[ith]; + dma_cache_dm_init(m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE); + + uint16_t * thread_active_blocks = factx->ddr->thread_active_blocks[ith]; + const uint16_t * active_blocks = factx->shared_active_blocks ? factx->ddr->active_blocks : thread_active_blocks; + uint32_t n_active_blocks = factx->n_active_blocks; + dma_addr_t prev_mp_base = 0; + + if (!factx->shared_active_blocks && !mask) { + #pragma clang loop vectorize(disable) + for (uint32_t b = 0; b < factx->n_blocks; ++b) { + thread_active_blocks[b] = (uint16_t) b; + } + } - dma_cache m_cache; - dma_cache_init(&m_cache, spad_m, factx->size_m_block, DMA_CACHE_MAX_SIZE); + const size_t size_vkq_acc_single = hex_round_up(DV * sizeof(float), 128); - for (uint32_t ir = ir0; ir < ir1; ++ir) { + uint32_t ir = ir0; + while (ir < ir1) { const uint32_t iq3 = fastdiv(ir, &factx->src0_div21); const uint32_t iq2 = fastdiv(ir - iq3*neq2*neq1, &factx->src0_div1); const uint32_t iq1 = (ir - iq3*neq2*neq1 - iq2 * neq1); @@ -360,253 +359,2302 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * const uint32_t iv3 = fastdiv(iq3, &factx->broadcast_rv3); const uint32_t iv2 = fastdiv(iq2, &factx->broadcast_rv2); - // Fetch Q row - const uint8_t * q_row_ptr = (const uint8_t *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3); - dma_queue_push(dma, dma_make_ptr(spad_q, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1); + uint32_t G_local = 1; + if (neq1 == 1 && (mask == NULL || mask->ne[2] == 1)) { + while (ir + G_local < ir1 && G_local < FA_HVX_G_MAX) { + const uint32_t next_ir = ir + G_local; + const uint32_t next_iq3 = fastdiv(next_ir, &factx->src0_div21); + const uint32_t next_iq2 = fastdiv(next_ir - next_iq3*neq2*neq1, &factx->src0_div1); + const uint32_t next_iq1 = (next_ir - next_iq3*neq2*neq1 - next_iq2 * neq1); + + const uint32_t next_ik3 = fastdiv(next_iq3, &factx->broadcast_rk3); + const uint32_t next_ik2 = fastdiv(next_iq2, &factx->broadcast_rk2); + + const uint32_t next_iv3 = fastdiv(next_iq3, &factx->broadcast_rv3); + const uint32_t next_iv2 = fastdiv(next_iq2, &factx->broadcast_rv2); - // FARF(HIGH, "fa %u: prefetch Q: ir %u iq1 %u iq2 %u iq3 %u q_row_ptr %p size %u : usec %u", ith, ir, iq1, iq2, iq3, q_row_ptr, size_q_row, - // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start)); + if (next_ik2 != ik2 || next_ik3 != ik3 || next_iv2 != iv2 || next_iv3 != iv3 || next_iq1 != iq1 || next_iq3 != iq3) { + break; + } + G_local++; + } + } - const __fp16 * mp_base = NULL; + dma_addr_t mp_base = 0; if (mask) { const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2); const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3); - mp_base = (const __fp16 *) ((const uint8_t *) mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3]); + mp_base = mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3]; + if (!factx->shared_active_blocks && mp_base != prev_mp_base) { + n_active_blocks = fa_scan_active_blocks( + dma_q, mask, iq1, 1, im2, im3, nek1, FLASH_ATTN_BLOCK_SIZE, + factx->n_blocks, spad_k, factx->size_k_block * 2, thread_active_blocks); + prev_mp_base = mp_base; + } + } + + uint32_t heads[FA_HVX_G_MAX]; + HVX_Vector slope_vecs[FA_HVX_G_MAX] __attribute__((aligned(128))); + HVX_Vector S_vec[FA_HVX_G_MAX] __attribute__((aligned(128))); + HVX_Vector M_vec[FA_HVX_G_MAX] __attribute__((aligned(128))); + uint8_t * q_ptrs[FA_HVX_G_MAX]; + float * vkq_ptrs[FA_HVX_G_MAX]; + + for (uint32_t g = 0; g < G_local; ++g) { + const uint32_t r = ir + g; + const uint32_t r_iq3 = fastdiv(r, &factx->src0_div21); + const uint32_t r_iq2 = fastdiv(r - r_iq3*neq2*neq1, &factx->src0_div1); + const uint32_t r_iq1 = (r - r_iq3*neq2*neq1 - r_iq2 * neq1); + + heads[g] = r_iq2; + const __fp16 slope = factx->ddr->slopes[r_iq2]; + slope_vecs[g] = hvx_vec_splat_f16(slope); + + S_vec[g] = hvx_vec_splat_f32(0.0f); + M_vec[g] = hvx_vec_splat_f32(HTP_FA_M_INITIAL_VAL); + + uint8_t * q_dst = spad_q + g * factx->size_q_row_padded; + q_ptrs[g] = q_dst; + + float * vkq_dst = (float *)(spad_a + g * size_vkq_acc_single); + vkq_ptrs[g] = vkq_dst; + hvx_splat_f32_a((uint8_t *) vkq_dst, 0, DV); + + // Fetch Q row g + const dma_addr_t q_row_ptr = q->data + r_iq1*nbq1 + r_iq2*nbq2 + r_iq3*nbq3; + dma_queue_push(dma_q, dma_make_data(q_dst, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1); } - // Prefetch first two blocks - for (uint32_t ib = 0; ib < MIN(factx->n_blocks, 2); ++ib) { + // Prefetch first two active blocks + for (uint32_t step = 0; step < MIN(n_active_blocks, 2); ++step) { + const uint32_t ib = active_blocks[step]; const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE; const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); // K - const uint8_t * k_src = (const uint8_t *) k->data + (ic_start*nbk1 + ik2*nbk2 + ik3*nbk3); - uint8_t * k_dst = spad_k + (ib % 2) * factx->size_k_block; - dma_queue_push(dma, dma_make_ptr(k_dst, k_src), factx->size_k_row_padded, nbk1, size_k_row, current_block_size); + const dma_addr_t k_src = k->data + ic_start*nbk1 + ik2*nbk2 + ik3*nbk3; + uint8_t * k_dst = spad_k + (step % 2) * factx->size_k_block; + dma_queue_push(dma_q, dma_make_data(k_dst, k_src), factx->size_k_row_padded, nbk1, size_k_row, current_block_size); // V - const uint8_t * v_src = (const uint8_t *) v->data + (ic_start*nbv1 + iv2*nbv2 + iv3*nbv3); - uint8_t * v_dst = spad_v + (ib % 2) * factx->size_v_block; - dma_queue_push(dma, dma_make_ptr(v_dst, v_src), factx->size_v_row_padded, nbv1, size_v_row, current_block_size); + const dma_addr_t v_src = v->data + ic_start*nbv1 + iv2*nbv2 + iv3*nbv3; + uint8_t * v_dst = spad_v + (step % 2) * factx->size_v_block; + dma_queue_push(dma_q, dma_make_data(v_dst, v_src), factx->size_v_row_padded, nbv1, size_v_row, current_block_size); // Mask if (mask) { - const uint8_t * m_src = (const uint8_t *) (mp_base + ic_start); - // Mask is 1D contiguous for this row - dma_cache_push(dma, &m_cache, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1); + const dma_addr_t m_src = mp_base + ic_start * sizeof(__fp16); + dma_cache_dm_push(dma_q, m_cache, ib, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1); } - - // FARF(HIGH, "fa %u: prefetch KVM: ir %u ib %u iq1 %u iq2 %u iq3 %u : size_k_row %u size_v_row %u bs %u: usec %u", - // ith, ir, ib, iq1, iq2, iq3, - // size_k_row, size_v_row, current_block_size, - // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start)); } - const uint32_t h = iq2; // head index - const float slope = (factx->max_bias > 0.0f) ? (h < factx->n_head_log2 ? powf(factx->m0, h + 1) : powf(factx->m1, 2*(h - factx->n_head_log2) + 1)) : 1.0f; - - HVX_Vector S_vec = hvx_vec_splat_f32(0.0f); - HVX_Vector M_vec = hvx_vec_splat_f32(-INFINITY); - - // Clear accumulator - hvx_splat_f32_a(spad_a, 0, DV); - float * VKQ32 = (float *) (spad_a + 0); - - uint8_t * q_ptr_vtcm = dma_queue_pop(dma).dst; - if (factx->is_q_fp32) { - hvx_copy_f16_f32_aa(q_ptr_vtcm, q_ptr_vtcm, DK); // inplace convert f32 to f16 + // Pop all Q rows + for (uint32_t g = 0; g < G_local; ++g) { + uint8_t * q_ptr_vtcm = (void *) dma_queue_pop(dma_q).dst; + if (factx->is_q_fp32) { + hvx_copy_f16_f32_aa(q_ptr_vtcm, q_ptr_vtcm, DK); + } } - const HVX_Vector slope_vec = hvx_vec_splat_f16(slope); - for (uint32_t ib = 0; ib < factx->n_blocks; ++ib) { + const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff); + const bool has_softcap = factx->has_softcap; + const HVX_Vector v_cap = has_softcap ? hvx_vec_splat_f16(factx->logit_softcap) : Q6_V_vzero(); + const HVX_Vector vinf = Q6_Vh_vsplat_R(0xFC00); + const HVX_Vector vmin = Q6_Vh_vsplat_R(0xFBFF); + const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F); + const uint32_t stride_v2 = factx->size_v_row_padded * 2; + + for (uint32_t step = 0; step < n_active_blocks; ++step) { + const uint32_t ib = active_blocks[step]; const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE; const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start); // Wait for DMA - uint8_t * k_base = dma_queue_pop(dma).dst; // K - uint8_t * v_base = dma_queue_pop(dma).dst; // V - __fp16 * m_base = mask ? dma_queue_pop(dma).dst : NULL; // M - - // FARF(HIGH, "fa %u: process: ir %u ib %u : iq1 %u iq2 %u iq3 %u q_ptr_vtcm %p : usec %u", - // ith, ir, ib, iq1, iq2, iq3, q_ptr_vtcm, - // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start)); - - // Inner loop processing the block from VTCM - uint32_t ic = 0; - - // Process in sub-blocks of 32 (VLEN_FP32) - HVX_Vector sb_scores[FLASH_ATTN_BLOCK_SIZE / VLEN_FP32]; - HVX_Vector v_max = hvx_vec_splat_f32(-INFINITY); - for (uint32_t iv = 0; ic + VLEN_FP32 <= current_block_size; ic += VLEN_FP32, ++iv) { - // 1. Compute scores - HVX_Vector scores = hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base + ic * factx->size_k_row_padded, factx->size_k_row_padded, DK, factx->scale); - - // 2. Softcap - if (factx->logit_softcap != 0.0f) { - scores = hvx_vec_tanh_f32(scores); - scores = Q6_Vqf32_vmpy_VsfVsf(scores, logit_cap); - scores = Q6_Vsf_equals_Vqf32(scores); + uint8_t * k_base = (void *) dma_queue_pop(dma_q).dst; // K + uint8_t * v_base = (void *) dma_queue_pop(dma_q).dst; // V + __fp16 * m_base = mask ? (__fp16 *) dma_queue_pop(dma_q).dst : NULL; // M + + if (factx->k->type == HTP_TYPE_Q8_0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir); + for (uint32_t r = 0; r < current_block_size; ++r) { + __fp16 * row_k = (__fp16 *)(k_base + r * factx->size_k_row_padded); + hvx_dequantize_row_flat_q8_0_f16(row_k, row_k, DK); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir); + } + if (factx->v->type == HTP_TYPE_Q8_0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir); + for (uint32_t r = 0; r < current_block_size; ++r) { + __fp16 * row_v = (__fp16 *)(v_base + r * factx->size_v_row_padded); + hvx_dequantize_row_flat_q8_0_f16(row_v, row_v, DV); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir); + } + + for (uint32_t g = 0; g < G_local; ++g) { + const uint32_t head_ir = ir + g; + uint8_t * q_ptr_vtcm = q_ptrs[g]; + float * VKQ32 = vkq_ptrs[g]; + const HVX_Vector slope_vec = slope_vecs[g]; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_QK, head_ir); + + HVX_Vector scores_f16 = Q6_V_vzero(); + if (current_block_size > 0) { + HVX_Vector scores0 = hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base, factx->size_k_row_padded, DK, factx->scale); + HVX_Vector scores1 = (current_block_size > 32) ? hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base + 32 * factx->size_k_row_padded, factx->size_k_row_padded, DK, factx->scale) : Q6_V_vzero(); + scores_f16 = hvx_vec_f32_to_f16(scores0, scores1); + } + + if (has_softcap) { + scores_f16 = hvx_vec_tanh_f16(scores_f16); + scores_f16 = hvx_vec_mul_f16_f16(scores_f16, v_cap); + } + + HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(current_block_size * sizeof(__fp16)); - // 3. Mask if (mask) { - const __fp16 * mp = m_base + ic; - HVX_Vector m_vals_f16 = *(const HVX_UVector *) mp; - HVX_VectorPair m_vals_f32_pair = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(m_vals_f16), slope_vec); - HVX_Vector add_val = Q6_V_lo_W(m_vals_f32_pair); - scores = Q6_Vqf32_vadd_Vqf32Vsf(add_val, scores); - scores = Q6_Vsf_equals_Vqf32(scores); + HVX_Vector m_vals_f16 = *(const HVX_UVector *) m_base; + HVX_VectorPred is_inf = Q6_Q_vcmp_eq_VhVh(m_vals_f16, vinf); + m_vals_f16 = Q6_V_vmux_QVV(is_inf, vmin, m_vals_f16); + + HVX_Vector m_scaled = hvx_vec_mul_f16_f16(m_vals_f16, slope_vec); + scores_f16 = Q6_V_vmux_QVV(q_tail_keep, hvx_vec_add_f16_f16(scores_f16, m_scaled), v_neg_inf); + } else { + scores_f16 = Q6_V_vmux_QVV(q_tail_keep, scores_f16, v_neg_inf); } - sb_scores[iv] = scores; - v_max = hvx_vec_reduce_max2_f32(scores, v_max); // All lanes have block max - } + HVX_Vector v_max_f16 = hvx_vec_reduce_max_f16(scores_f16); + HVX_Vector v_max = Q6_V_lo_W(hvx_vec_f16_to_f32(v_max_f16)); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_QK, head_ir); - { - // 4. Online Softmax Update - HVX_Vector M_new_vec = Q6_Vsf_vmax_VsfVsf(v_max, M_vec); - HVX_Vector diff_vec = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(M_vec, M_new_vec)); - HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec); - M_vec = M_new_vec; - - hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); - - HVX_Vector p_sum_vec = hvx_vec_splat_f32(0.0f); - for (uint32_t ic2 = 0, iv = 0; ic2 + VLEN_FP32 <= current_block_size; ic2 += VLEN_FP32, ++iv) { - HVX_Vector scores = sb_scores[iv]; - HVX_Vector scores_shifted = Q6_Vqf32_vsub_VsfVsf(scores, M_vec); - HVX_Vector P = hvx_vec_exp_f32(Q6_Vsf_equals_Vqf32(scores_shifted)); - - p_sum_vec = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(p_sum_vec, P)); - - // 5. Accumulate V - __fp16 __attribute__((aligned(VLEN))) p_arr[VLEN_FP16]; - hvx_vec_f32_to_f16_a(p_arr, P, hvx_vec_splat_f32(0)); - - for (uint32_t j = 0; j < VLEN_FP32; j += 2) { - const uint32_t cur_ic = ic2 + j; - const uint8_t * v_ptr = v_base + cur_ic * factx->size_v_row_padded; - hvx_mad_f32_f16_aa_rx2(VKQ32, v_ptr, v_ptr + factx->size_v_row_padded, (p_arr + j), (p_arr + j + 1), DV); - } + // prefetch K for step + 2 after last head finished QK + if (g + 1 == G_local && step + 2 < n_active_blocks) { + const uint32_t next_ib = active_blocks[step + 2]; + const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE; + const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start); + + const dma_addr_t k_src = k->data + next_ic_start*nbk1 + ik2*nbk2 + ik3*nbk3; + dma_queue_push(dma_q, dma_make_data(k_base, k_src), factx->size_k_row_padded, nbk1, size_k_row, next_block_size); } - p_sum_vec = hvx_vec_reduce_sum_f32(p_sum_vec); - S_vec = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(S_vec, ms_vec)), p_sum_vec)); - } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, head_ir); + { + HVX_Vector M_new_vec = Q6_Vsf_vmax_VsfVsf(v_max, M_vec[g]); + HVX_Vector diff_vec = HVX_OP_SUB_F32(M_vec[g], M_new_vec); + + HVX_Vector diff_f16 = hvx_vec_f32_to_f16(diff_vec, diff_vec); + HVX_Vector diff_base2 = hvx_vec_mul_f16_f16(diff_f16, v_log2e); + HVX_Vector ms_f16 = hvx_vec_exp2_f16(diff_base2); + HVX_Vector ms_vec = Q6_V_lo_W(hvx_vec_f16_to_f32(ms_f16)); + + M_vec[g] = M_new_vec; + + HVX_Vector v_m_vec_f16 = hvx_vec_f32_to_f16(M_vec[g], M_vec[g]); + HVX_Vector v_s_minus_m = Q6_Vqf16_vsub_VhfVhf(scores_f16, v_m_vec_f16); + HVX_Vector v_s_minus_m_base2 = hvx_vec_mul_f16_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m), v_log2e); + + HVX_Vector P = hvx_vec_exp2_f16(v_s_minus_m_base2); + P = Q6_V_vmux_QVV(q_tail_keep, P, Q6_V_vzero()); + + HVX_VectorPair P_pair = hvx_vec_f16_to_f32(P); + HVX_Vector P0 = Q6_V_lo_W(P_pair); + HVX_Vector P1 = Q6_V_hi_W(P_pair); + HVX_Vector p_sum_vec = hvx_vec_reduce_sum_f32(HVX_OP_ADD_F32(P0, P1)); + + S_vec[g] = HVX_OP_ADD_F32(HVX_OP_MUL_F32(S_vec[g], ms_vec), p_sum_vec); + + const uint8_t * v_ptr = v_base; + + if (DV == 64) { + HVX_VectorPair vkq0 = *((const HVX_VectorPair *) VKQ32); + vkq0 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq0), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq0), ms_vec) + ); + + for (uint32_t j = 0; j < current_block_size; j += 2) { + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + const HVX_Vector * vx0 = (const HVX_Vector *) v_ptr; + if (j + 1 == current_block_size) { + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + break; + } + + HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2)); + const HVX_Vector * vx1 = (const HVX_Vector *) (v_ptr + factx->size_v_row_padded); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx1[0]), S1); + v_ptr += stride_v2; + } + + *((HVX_VectorPair *) VKQ32) = vkq0; + } else if (DV == 128) { + HVX_VectorPair vkq0 = ((const HVX_VectorPair *) VKQ32)[0]; + HVX_VectorPair vkq1 = ((const HVX_VectorPair *) VKQ32)[1]; + vkq0 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq0), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq0), ms_vec) + ); + vkq1 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq1), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq1), ms_vec) + ); + + for (uint32_t j = 0; j < current_block_size; j += 2) { + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + const HVX_Vector * vx0 = (const HVX_Vector *) v_ptr; + if (j + 1 == current_block_size) { + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0); + break; + } + + HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2)); + const HVX_Vector * vx1 = (const HVX_Vector *) (v_ptr + factx->size_v_row_padded); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx1[0]), S1); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx1[1]), S1); + v_ptr += stride_v2; + } + + ((HVX_VectorPair *) VKQ32)[0] = vkq0; + ((HVX_VectorPair *) VKQ32)[1] = vkq1; + } else if (DV == 256) { + HVX_VectorPair vkq0 = ((const HVX_VectorPair *) VKQ32)[0]; + HVX_VectorPair vkq1 = ((const HVX_VectorPair *) VKQ32)[1]; + HVX_VectorPair vkq2 = ((const HVX_VectorPair *) VKQ32)[2]; + HVX_VectorPair vkq3 = ((const HVX_VectorPair *) VKQ32)[3]; + vkq0 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq0), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq0), ms_vec) + ); + vkq1 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq1), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq1), ms_vec) + ); + vkq2 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq2), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq2), ms_vec) + ); + vkq3 = Q6_W_vcombine_VV( + HVX_OP_MUL_F32(Q6_V_hi_W(vkq3), ms_vec), + HVX_OP_MUL_F32(Q6_V_lo_W(vkq3), ms_vec) + ); + + for (uint32_t j = 0; j < current_block_size; j += 2) { + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + const HVX_Vector * vx0 = (const HVX_Vector *) v_ptr; + if (j + 1 == current_block_size) { + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0); + vkq2 = hvx_vec_mpyacc_f32_f16(vkq2, Q6_Vh_vshuff_Vh(vx0[2]), S0); + vkq3 = hvx_vec_mpyacc_f32_f16(vkq3, Q6_Vh_vshuff_Vh(vx0[3]), S0); + break; + } + + HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2)); + const HVX_Vector * vx1 = (const HVX_Vector *) (v_ptr + factx->size_v_row_padded); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx0[0]), S0); + vkq0 = hvx_vec_mpyacc_f32_f16(vkq0, Q6_Vh_vshuff_Vh(vx1[0]), S1); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx0[1]), S0); + vkq1 = hvx_vec_mpyacc_f32_f16(vkq1, Q6_Vh_vshuff_Vh(vx1[1]), S1); + vkq2 = hvx_vec_mpyacc_f32_f16(vkq2, Q6_Vh_vshuff_Vh(vx0[2]), S0); + vkq2 = hvx_vec_mpyacc_f32_f16(vkq2, Q6_Vh_vshuff_Vh(vx1[2]), S1); + vkq3 = hvx_vec_mpyacc_f32_f16(vkq3, Q6_Vh_vshuff_Vh(vx0[3]), S0); + vkq3 = hvx_vec_mpyacc_f32_f16(vkq3, Q6_Vh_vshuff_Vh(vx1[3]), S1); + v_ptr += stride_v2; + } + + ((HVX_VectorPair *) VKQ32)[0] = vkq0; + ((HVX_VectorPair *) VKQ32)[1] = vkq1; + ((HVX_VectorPair *) VKQ32)[2] = vkq2; + ((HVX_VectorPair *) VKQ32)[3] = vkq3; + } else { + hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); + + for (uint32_t j = 0; j < current_block_size; j += 2) { + if (j + 1 == current_block_size) { + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + hvx_mad_f32_f16_aa_vec(VKQ32, v_ptr, S0, DV); + break; + } - if (ic < current_block_size) { - // Sync scalars for leftover/next block if needed - float M = hvx_vec_get_f32(M_vec); - float S = hvx_vec_get_f32(S_vec); - - // Leftover - for (; ic < current_block_size; ++ic) { - float s_val; - const uint8_t * k_ptr = k_base + ic * factx->size_k_row_padded; - hvx_dot_f16_f16_aa(&s_val, q_ptr_vtcm, k_ptr, DK, factx->scale); - if (factx->logit_softcap != 0.0f) { - s_val = factx->logit_softcap * tanhf(s_val); + HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2)); + HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2)); + + hvx_mad_f32_f16_aa_rx2_vec(VKQ32, v_ptr, v_ptr + factx->size_v_row_padded, S0, S1, DV); + v_ptr += stride_v2; + } } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, head_ir); + + // prefetch V and mask for step + 2 after last head finished V accumulation + if (g + 1 == G_local && step + 2 < n_active_blocks) { + const uint32_t next_ib = active_blocks[step + 2]; + const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE; + const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start); + + // V + const dma_addr_t v_src = v->data + next_ic_start*nbv1 + iv2*nbv2 + iv3*nbv3; + dma_queue_push(dma_q, dma_make_data(v_base, v_src), factx->size_v_row_padded, nbv1, size_v_row, next_block_size); + // Mask if (mask) { - const float m_val = m_base[ic]; - s_val += slope * m_val; + const dma_addr_t m_src = mp_base + next_ic_start * sizeof(__fp16); + dma_cache_dm_push(dma_q, m_cache, next_ib, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1); } + } + } // end for g + } // end for step - const float Mold = M; - __fp16 vs = 1.0f; + for (uint32_t g = 0; g < G_local; ++g) { + const uint32_t head_ir = ir + g; + const uint32_t h = heads[g]; + float * VKQ32 = vkq_ptrs[g]; - if (s_val > M) { - M = s_val; - HVX_Vector diff_vec = hvx_vec_splat_f32(Mold - M); - HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec); - hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, head_ir); + + float M = hvx_vec_get_f32(M_vec[g]); + float S = hvx_vec_get_f32(S_vec[g]); + + if (sinks) { + const float s = factx->spad_sinks[h]; + + float vs = 1.0f; + + if (s > M) { + HVX_Vector diff_vec = hvx_vec_splat_f32(M - s); + HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec); + hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); + + float ms = hvx_vec_get_f32(ms_vec); + S = S * ms + vs; + } else { + HVX_Vector diff_vec = hvx_vec_splat_f32(s - M); + vs = hvx_vec_get_f32(hvx_vec_exp_f32(diff_vec)); + S += vs; + } + } + + const float S_inv = S == 0.0f ? 0.0f : 1.0f/S; + hvx_scale_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, S_inv); + + const uint32_t r_iq3 = fastdiv(head_ir, &factx->src0_div21); + const uint32_t r_iq2 = fastdiv(head_ir - r_iq3*neq2*neq1, &factx->src0_div1); + const uint32_t r_iq1 = (head_ir - r_iq3*neq2*neq1 - r_iq2 * neq1); + + uint8_t * dst_ptr = (uint8_t *) dst->data + r_iq2 * dst->nb[1] + r_iq1 * dst->nb[2] + r_iq3 * dst->nb[3]; + + if (dst->type == HTP_TYPE_F32) { + hvx_copy_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV); + } else if (dst->type == HTP_TYPE_F16) { + hvx_copy_f16_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, head_ir); + } + + ir += G_local; + } +} + +// ============================================================================ +// HMX Phase args and thread logic +// ============================================================================ + +typedef struct { + struct hmx_fa_context * factx; + uint32_t kv_rows; + size_t src_stride; + void * curr_k; + uint32_t kv_start; + uint32_t rows_per_t; + size_t buf_idx; +} fa_k_int_args_t; + +static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) { + fa_k_int_args_t * args = (fa_k_int_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const uint32_t total_rows = args->kv_rows; + const uint32_t rows_per_t = args->rows_per_t; + const uint32_t start = i * rows_per_t; + const uint32_t end = (uint32_t) hex_smin(start + rows_per_t, total_rows); + + if (start >= total_rows) { + return; + } + + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); + if (factx->k->type == HTP_TYPE_Q8_0) { + for (uint32_t r = start; r < end; ++r) { + __fp16 * row_k = (__fp16 *)((char *)args->curr_k + r * args->src_stride * sizeof(__fp16)); + hvx_dequantize_row_flat_q8_0_f16(row_k, row_k, factx->DK); + } + } + hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK_pad, + args->src_stride, start, end); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start)); +} + +static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start, size_t buf_idx) { + work_queue_t wp = factx->octx->ctx->work_queue; + uint32_t n = 1; + if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { + n = factx->n_threads; + } + uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); + fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t, buf_idx }; + if (n > 1) { + work_queue_run(wp, fa_k_interleave_thread, &args, n); + } else { + fa_k_interleave_thread(1, 0, &args); + } +} + +typedef struct { + struct hmx_fa_context * factx; + uint32_t kv_rows; + size_t src_stride; + void * v_src; + void * v_tiles_dst; + size_t n_col_tiles; + uint32_t kv_start; + uint32_t rows_per_t; +} fa_v_int_args_t; + +static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) { + fa_v_int_args_t * args = (fa_v_int_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const uint32_t total_rows = args->kv_rows; + const uint32_t rows_per_t = args->rows_per_t; + const uint32_t start = i * rows_per_t; + const uint32_t end = (uint32_t) hex_smin(start + rows_per_t, total_rows); + + if (start >= total_rows) { + return; + } + + __fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst; + + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); + if (factx->v->type == HTP_TYPE_Q8_0) { + for (uint32_t r = start; r < end; ++r) { + __fp16 * row_v = (__fp16 *)((char *)args->v_src + r * args->src_stride * sizeof(__fp16)); + hvx_dequantize_row_flat_q8_0_f16(row_v, row_v, factx->DV); + } + } + hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV_pad, + args->src_stride, (uint32_t) args->n_col_tiles, start, end); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start)); +} + +static void fa_phase_v_interleave(struct hmx_fa_context * factx, + uint32_t kv_rows, + size_t src_stride, + void * v_src, + void * v_tiles_dst, + size_t n_col_tiles, + uint32_t kv_start) { + work_queue_t wp = factx->octx->ctx->work_queue; + uint32_t n = 1; + if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) { + n = factx->n_threads; + } + uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2); + fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t }; + if (n > 1) { + work_queue_run(wp, fa_v_interleave_thread, &args, n); + } else { + fa_v_interleave_thread(1, 0, &args); + } +} + +typedef struct { + struct hmx_fa_context * factx; + const struct htp_tensor * q; + uint32_t q_start; + uint32_t kv_head; + uint32_t ib3; + size_t n_rows_g; + size_t rows_per_t; + size_t n_rows_q; + bool q_transposed; + atomic_uint barrier; +} fa_q_load_args_t; + +static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) { + fa_q_load_args_t * args = (fa_q_load_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const size_t n_rows_g = args->n_rows_g; + const size_t G = factx->G; + const size_t DK = factx->DK; + + // Partition the padded Q rows (g_br) across threads. + // Keep start/end even so r and r+1 are always in the same thread's range. + const size_t rows_per_t = args->rows_per_t; + const size_t start = (size_t) i * rows_per_t; + const size_t end = hex_smin(start + rows_per_t, factx->g_br); + + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start)); + + // Parallel initialization of per-block state + { + const uint32_t g_br = factx->g_br; + const uint32_t DV = factx->DV; + + const size_t col_vec_bytes = factx->col_vec_bytes; + const size_t d_tile_bytes = factx->d_tile_bytes; + + // Initialize vtcm_l_vec & vtcm_m_vec + const size_t l_bytes_per_t = hex_align_up(col_vec_bytes / n, 128); + const size_t l_start = i * l_bytes_per_t; + const size_t l_end = hex_smin(l_start + l_bytes_per_t, col_vec_bytes); + + const size_t m_bytes_per_t = hex_align_up(col_vec_bytes / n, 128); + const size_t m_start = i * m_bytes_per_t; + const size_t m_end = hex_smin(m_start + m_bytes_per_t, col_vec_bytes); + + if (l_start < col_vec_bytes) { + hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start); + } + if (m_start < col_vec_bytes) { + hvx_splat_f32_a((char *) factx->vtcm_m_vec + m_start, HTP_FA_M_INITIAL_VAL, (m_end - m_start) / sizeof(float)); + } + + // Zero the whole rescale region: vtcm_d_tiles[0], the optional vtcm_d_tiles[1] + // and vtcm_d_inv_l are equal-sized and allocated back to back, so one run covers + // them all. The scatter only ever writes the diagonal, ignore the rest. + const size_t d_bytes_per_t = hex_align_up(d_tile_bytes / n, 128); + const size_t d_start = i * d_bytes_per_t; + const size_t d_end = hex_smin(d_start + d_bytes_per_t, d_tile_bytes); + if (d_start < d_tile_bytes) { + hvx_splat_u8_a((char *) factx->vtcm_d_tiles[0] + d_start, 0, d_end - d_start); + } + } + + if (start < factx->g_br) { + const struct htp_tensor * q = args->q; + const uint32_t q_start = args->q_start; + const uint32_t kv_head = args->kv_head; + const uint32_t ib3 = args->ib3; + + const bool use_q_dma = (factx->vtcm_q_dma != NULL); + + __fp16 * q_tiles = factx->vtcm_q_tiles; + const size_t DK_pad = factx->DK_pad; + if (use_q_dma) { + const size_t g_rows_end = hex_smin(end, n_rows_g); + const uint32_t d_limit = factx->is_q_fp32 ? DK / 32 : DK / 64; + + uint8_t * q_flat = (uint8_t *) factx->vtcm_q_dma; + if (DK_pad != DK) { + if (factx->is_q_fp32) { + hmx_fa_q_prep_fp32_pad(q_tiles, q_flat, start, end, g_rows_end, DK, DK_pad, G, args->n_rows_q, &factx->div_G, args->q_transposed); + } else { + hmx_fa_q_prep_fp16_pad(q_tiles, q_flat, start, end, g_rows_end, DK, DK_pad, G, args->n_rows_q, &factx->div_G, args->q_transposed); + } + } else if (factx->is_q_fp32) { + switch (d_limit) { + case 2: hmx_fa_q_prep_fp32_d2(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; + case 4: hmx_fa_q_prep_fp32_d4(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; + default: hmx_fa_q_prep_fp32( q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, d_limit, args->q_transposed); break; + } + } else { + switch (d_limit) { + case 1: hmx_fa_q_prep_fp16_d1(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; + case 2: hmx_fa_q_prep_fp16_d2(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break; + default: hmx_fa_q_prep_fp16( q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, d_limit, args->q_transposed); break; + } + } + } else { + // Fallback: direct-from-DDR/L2 path + hmx_fa_q_prep_fallback(q_tiles, q->data, q->nb[1], q->nb[2], q->nb[3], + q_start, kv_head, ib3, start, end, n_rows_g, G, DK, DK_pad, factx->is_q_fp32, &factx->div_G); + } + } + + // Synchronize threads before zeroing out vtcm_o_tiles[0] to prevent race condition + if (n > 1) { + atomic_fetch_sub(&args->barrier, 1); + while (atomic_load(&args->barrier) > 0) { + // spin wait + } + } + + // Zero out vtcm_o_tiles[0] as it was used as temp_q_vtcm + { + const uint32_t g_br = factx->g_br; + const uint32_t DV = factx->DV; + const size_t o_tile_bytes = factx->o_tile_bytes; + const size_t o_bytes_per_t = hex_align_up(o_tile_bytes / n, 128); + const size_t o_start = i * o_bytes_per_t; + const size_t o_end = hex_smin(o_start + o_bytes_per_t, o_tile_bytes); + if (o_start < o_tile_bytes) { + hvx_splat_u8_a((char *) factx->vtcm_o_tiles[0] + o_start, 0, o_end - o_start); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start)); +} + +static void fa_phase_q_load(struct hmx_fa_context * factx, + const struct htp_tensor * q, + uint32_t q_start, + uint32_t kv_head, + uint32_t ib3, + size_t n_rows_g) { + work_queue_t wp = factx->octx->ctx->work_queue; + uint32_t n = 1; + if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { + n = factx->n_threads; + } + size_t rows_per_t = hex_align_up(hmx_ceil_div(factx->g_br, n), 2); + const uint32_t n_rows_q = hex_smin(factx->Br, factx->neq1 - q_start); + fa_q_load_args_t args; + args.factx = factx; + args.q = q; + args.q_start = q_start; + args.kv_head = kv_head; + args.ib3 = ib3; + args.n_rows_g = n_rows_g; + args.rows_per_t = rows_per_t; + args.n_rows_q = n_rows_q; + args.q_transposed = q->nb[1] < q->nb[2]; + atomic_init(&args.barrier, n); + if (n > 1) { + work_queue_run(wp, fa_q_load_thread, &args, n); + } else { + fa_q_load_thread(1, 0, &args); + } +} + +typedef struct { + struct hmx_fa_context * factx; + const struct htp_tensor * dst; + const __fp16 * o_tile_src; + uint32_t q_start; + uint32_t kv_head; + uint32_t ib3; + size_t n_rows_g; + size_t rows_per_t; +} fa_o_store_args_t; + +static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) { + fa_o_store_args_t * args = (fa_o_store_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const size_t n_rows_g = args->n_rows_g; + const size_t G = factx->G; + const size_t DV = factx->DV; + + const size_t rows_per_t = args->rows_per_t; + const size_t start = (size_t) i * rows_per_t; + const size_t end = hex_smin(start + rows_per_t, n_rows_g); + + if (start >= n_rows_g) { + return; + } + + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); + + const struct htp_tensor * dst = args->dst; + const __fp16 * o_tile_src = args->o_tile_src; + const uint32_t q_start = args->q_start; + const uint32_t kv_head = args->kv_head; + const uint32_t ib3 = args->ib3; + + const size_t DV_pad = factx->DV_pad; + + size_t q_idx = fastdiv(start, &factx->div_G); + size_t h_idx = fastmodulo(start, G, &factx->div_G); + + for (size_t r = start; r < end; ++r) { + float * out = (float *) ((uint8_t *) dst->data + (kv_head * G + h_idx) * dst->nb[1] + + (q_start + q_idx) * dst->nb[2] + ib3 * dst->nb[3]); + + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV_pad; + + for (uint32_t d = 0; d < DV / 32; ++d) { + const HVX_Vector * in_tile = (const HVX_Vector *) (tile_row_base + d * HMX_FP16_TILE_N_ELMS); + HVX_VectorPair vp = hvx_vec_f16_to_f32_shuff(in_tile[r1 / 2]); + if (r1 % 2 == 0) { + *(HVX_UVector *) (out + d * 32) = Q6_V_lo_W(vp); + } else { + *(HVX_UVector *) (out + d * 32) = Q6_V_hi_W(vp); + } + } + // Ragged tail: DV not a multiple of 32 (e.g. 72 -> last 8 lanes). Partial vector-write + // for the remaining (DV % 32) floats. + const uint32_t d_tail = DV / 32; + const uint32_t rem = DV - d_tail * 32; + if (rem) { + const HVX_Vector * in_tile = (const HVX_Vector *) (tile_row_base + d_tail * HMX_FP16_TILE_N_ELMS); + HVX_VectorPair vp = hvx_vec_f16_to_f32_shuff(in_tile[r1 / 2]); + HVX_Vector vd = (r1 % 2 == 0) ? Q6_V_lo_W(vp) : Q6_V_hi_W(vp); + hvx_vec_store_u((void *) (out + d_tail * 32), rem * sizeof(float), vd); + } + + h_idx++; + if (h_idx == G) { + h_idx = 0; + q_idx++; + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); +} + +static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) { + fa_o_store_args_t * args = (fa_o_store_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const size_t n_rows_g = args->n_rows_g; + const size_t rows_per_t = args->rows_per_t; + const size_t G = factx->G; + const size_t DV = factx->DV; + const size_t start = (size_t) i * rows_per_t; + const size_t end = hex_smin(start + rows_per_t, n_rows_g); + + if (start >= n_rows_g) { + return; + } + + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); + + const struct htp_tensor * dst = args->dst; + const __fp16 * o_tile_src = args->o_tile_src; + const uint32_t q_start = args->q_start; + const uint32_t kv_head = args->kv_head; + const uint32_t ib3 = args->ib3; + + // O-tiles use the padded head dim (DV_pad); dst holds the real DV lanes. + const size_t DV_pad = factx->DV_pad; + + size_t q_idx = fastdiv(start, &factx->div_G); + size_t h_idx = fastmodulo(start, G, &factx->div_G); + + for (size_t r = start; r < end; ++r) { + __fp16 * out = (__fp16 *) ((uint8_t *) dst->data + (kv_head * G + h_idx) * dst->nb[1] + + (q_start + q_idx) * dst->nb[2] + ib3 * dst->nb[3]); + + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV_pad; + + for (uint32_t d = 0; d < DV / 64; ++d) { + const __fp16 * in_dtile = tile_row_base + d * HMX_FP16_TILE_N_ELMS * 2; + const HVX_Vector * pv_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2; + const HVX_Vector * pv_in1 = pv_in0 + 16; + HVX_VectorPair vp = Q6_W_vdeal_VVR(*pv_in1, *pv_in0, -2); + if (r1 % 2 == 0) { + *(HVX_UVector *) (out + d * 64) = Q6_V_lo_W(vp); + } else { + *(HVX_UVector *) (out + d * 64) = Q6_V_hi_W(vp); + } + } + // Ragged tail when DV is not a multiple of 64. + const uint32_t d_tail = DV / 64; + const uint32_t rem = DV - d_tail * 64; + if (rem) { + const __fp16 * in_dtile = tile_row_base + d_tail * HMX_FP16_TILE_N_ELMS * 2; + const HVX_Vector * pv_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2; + const HVX_Vector * pv_in1 = pv_in0 + 16; + HVX_VectorPair vp = Q6_W_vdeal_VVR(*pv_in1, *pv_in0, -2); + HVX_Vector vd = (r1 % 2 == 0) ? Q6_V_lo_W(vp) : Q6_V_hi_W(vp); + hvx_vec_store_u((void *) (out + d_tail * 64), rem * sizeof(__fp16), vd); + } + + h_idx++; + if (h_idx == G) { + h_idx = 0; + q_idx++; + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start)); +} + +static void fa_phase_o_store(struct hmx_fa_context * factx, + const struct htp_tensor * dst, + const __fp16 * o_tile_src, + uint32_t q_start, + uint32_t kv_head, + uint32_t ib3, + size_t n_rows_g) { + work_queue_t wp = factx->octx->ctx->work_queue; + uint32_t n = 1; + if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) { + n = factx->n_threads; + } + size_t rows_per_t = hmx_ceil_div(n_rows_g, n); + fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t }; + worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16; + if (n > 1) { + work_queue_run(wp, store_fn, &args, n); + } else { + store_fn(1, 0, &args); + } +} - float ms = hvx_vec_get_f32(ms_vec); - S = S * ms + vs; +typedef struct { + struct hmx_fa_context * factx; + size_t buf_idx; + size_t kv_rows; + size_t n_rows_g; + size_t n_col_tiles; + size_t n_tiles_per_bc; + size_t n_row_tiles; + size_t n_row_tiles_g_br; + uint32_t Bc; + uint32_t G; + uint32_t kv_head; + uint32_t kv_start; + uint32_t q_start; + uint32_t ib3; + bool has_alibi; // true when max_bias != 0 (need slope * mask + add) + __fp16 * slopes; + const struct htp_tensor * mask; + const __fp16 * mask_vtcm; // VTCM mask buffer base (NULL = DDR fallback) + size_t mask_vtcm_row_stride; // elements (__fp16) per row in VTCM mask buffer + struct fastdiv_values thread_div; +} fa_softmax_args_t; + +static inline void fa_softmax_impl( + unsigned int n, unsigned int i, void * data, + const bool has_mask, + const bool mask_broadcast, + const bool is_g1, + const bool has_alibi, + const bool has_softcap +) { + fa_softmax_args_t * args = (fa_softmax_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const size_t n_rows_g = args->n_rows_g; + const size_t kv_rows = args->kv_rows; + const size_t Bc = args->Bc; + const size_t G = args->G; + const size_t n_tiles_per_bc = args->n_tiles_per_bc; + const size_t n_row_vec_cnt = hmx_ceil_div(n_rows_g, 64); + const uint32_t im3 = has_mask ? fastmodulo(args->ib3, args->mask->ne[3], &factx->src3_div3) : 0; + + size_t vec_start = 0; + size_t vec_end = n_row_vec_cnt; + if (n > 1) { + const size_t vecs_per_t = fastdiv(n_row_vec_cnt + n - 1, &args->thread_div); + vec_start = i * vecs_per_t; + vec_end = hex_smin(vec_start + vecs_per_t, n_row_vec_cnt); + } + + if (vec_start >= n_row_vec_cnt) { + return; + } + + struct htp_thread_trace * tr = &factx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); + + // Per-thread row scratch: thread i uses bufs at offset i * 2 * stride + const size_t row_buf_stride = factx->row_buf_stride; + HVX_Vector * my_row_buf0 = factx->vtcm_row_bufs + i * 2 * row_buf_stride; + HVX_Vector * my_row_buf1 = my_row_buf0 + row_buf_stride; + + const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff); + + for (size_t r_vec_idx = vec_start; r_vec_idx < vec_end; ++r_vec_idx) { + HVX_Vector rowmax_acc_v = v_neg_inf; + HVX_Vector rowsum_acc_v = Q6_V_vzero(); + HVX_Vector m_prev_v0 = factx->vtcm_m_vec[r_vec_idx * 2 + 0]; + HVX_Vector m_prev_v1 = factx->vtcm_m_vec[r_vec_idx * 2 + 1]; + + HVX_Vector v_slopes = Q6_V_vzero(); + if (has_alibi) { + v_slopes = hvx_vmem(args->slopes + r_vec_idx * 64); + } + + for (uint32_t r_vec_off = 0; r_vec_off < 64; r_vec_off += 2) { + uint32_t r = r_vec_idx * 64 + r_vec_off; + if (r >= hex_align_up(n_rows_g, 2)) { + break; + } + + uint32_t r0 = r / HMX_FP16_TILE_N_ROWS; + uint32_t r1 = r % HMX_FP16_TILE_N_ROWS; + + const __fp16 * s_ld_base = factx->vtcm_s_tiles[args->buf_idx] + r0 * HMX_FP16_TILE_N_ROWS * Bc; + __fp16 * p_st_base = factx->vtcm_p_tiles[args->buf_idx] + r0 * HMX_FP16_TILE_N_ROWS * Bc; + + // Decode 2 rows from S tiles into per-thread row buffers + if (has_softcap) { + const HVX_Vector v_cap = hvx_vec_splat_f16(factx->logit_softcap); + for (size_t c = 0; c < kv_rows; c += 64) { + size_t ci = c / 64; + const __fp16 * in_dtile = s_ld_base + ci * HMX_FP16_TILE_N_ELMS * 2; + const HVX_Vector * pv_s_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2; + const HVX_Vector * pv_s_in1 = pv_s_in0 + 16; + + HVX_VectorPair vp_s_drow = Q6_W_vdeal_VVR(*pv_s_in1, *pv_s_in0, -2); + HVX_Vector v_s_row0 = Q6_V_lo_W(vp_s_drow); + HVX_Vector v_s_row1 = Q6_V_hi_W(vp_s_drow); + + HVX_Vector t0 = hvx_vec_tanh_f16(v_s_row0); + my_row_buf0[ci] = hvx_vec_mul_f16_f16(t0, v_cap); + + HVX_Vector t1 = hvx_vec_tanh_f16(v_s_row1); + my_row_buf1[ci] = hvx_vec_mul_f16_f16(t1, v_cap); + } + } else { + size_t c = 0; + for (; c + 64 < kv_rows; c += 128) { + size_t ci0 = c / 64; + size_t ci1 = ci0 + 1; + const __fp16 * in_dtile0 = s_ld_base + ci0 * HMX_FP16_TILE_N_ELMS * 2; + const __fp16 * in_dtile1 = s_ld_base + ci1 * HMX_FP16_TILE_N_ELMS * 2; + const HVX_Vector * pv_s_in0_0 = ((const HVX_Vector *) in_dtile0) + r1 / 2; + const HVX_Vector * pv_s_in1_0 = pv_s_in0_0 + 16; + const HVX_Vector * pv_s_in0_1 = ((const HVX_Vector *) in_dtile1) + r1 / 2; + const HVX_Vector * pv_s_in1_1 = pv_s_in0_1 + 16; + + HVX_VectorPair vp_s_drow0 = Q6_W_vdeal_VVR(*pv_s_in1_0, *pv_s_in0_0, -2); + my_row_buf0[ci0] = Q6_V_lo_W(vp_s_drow0); + my_row_buf1[ci0] = Q6_V_hi_W(vp_s_drow0); + + HVX_VectorPair vp_s_drow1 = Q6_W_vdeal_VVR(*pv_s_in1_1, *pv_s_in0_1, -2); + my_row_buf0[ci1] = Q6_V_lo_W(vp_s_drow1); + my_row_buf1[ci1] = Q6_V_hi_W(vp_s_drow1); + } + for (; c < kv_rows; c += 64) { + size_t ci = c / 64; + const __fp16 * in_dtile = s_ld_base + ci * HMX_FP16_TILE_N_ELMS * 2; + const HVX_Vector * pv_s_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2; + const HVX_Vector * pv_s_in1 = pv_s_in0 + 16; + + HVX_VectorPair vp_s_drow = Q6_W_vdeal_VVR(*pv_s_in1, *pv_s_in0, -2); + my_row_buf0[ci] = Q6_V_lo_W(vp_s_drow); + my_row_buf1[ci] = Q6_V_hi_W(vp_s_drow); + } + } + + // Apply mask & compute rowmax(S) + HVX_Vector v_slope0 = Q6_V_vzero(); + HVX_Vector v_slope1 = Q6_V_vzero(); + if (has_alibi) { + v_slope0 = hvx_vec_repl_f16(Q6_V_vror_VR(v_slopes, r_vec_off * 2)); + v_slope1 = (r + 1 < n_rows_g) ? hvx_vec_repl_f16(Q6_V_vror_VR(v_slopes, (r_vec_off + 1) * 2)) : Q6_V_vzero(); + } + + const HVX_Vector v_threshold = Q6_Vh_vsplat_R(0xcc00); // fp16 -16.0 + + HVX_Vector v_s_rowmax0 = v_neg_inf; + HVX_Vector v_s_rowmax1 = v_neg_inf; + if (has_mask) { + for (size_t c = 0; c < kv_rows; c += 64) { + size_t ci = c / 64; + const size_t ne = hex_smin(kv_rows - c, 64); + HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(ne * sizeof(__fp16)); + + HVX_Vector v_mask0, v_mask1; + + if (mask_broadcast) { + if (is_g1) { + const size_t qi0 = r + 0; + v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + qi0 * args->mask_vtcm_row_stride + c); + v_mask1 = v_neg_inf; + if (r + 1 < n_rows_g) { + const size_t qi1 = r + 1; + v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + qi1 * args->mask_vtcm_row_stride + c); + } + } else { + const size_t qi0 = fastdiv(r + 0, &factx->div_G); + v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + qi0 * args->mask_vtcm_row_stride + c); + v_mask1 = v_neg_inf; + if (r + 1 < n_rows_g) { + const size_t qi1 = fastdiv(r + 1, &factx->div_G); + if (qi1 == qi0) { + v_mask1 = v_mask0; + } else { + v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + qi1 * args->mask_vtcm_row_stride + c); + } + } + } + } else { + // Head-dependent mask: pre-interleaved per row r. + const size_t r0 = r + 0; + v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + r0 * args->mask_vtcm_row_stride + c); + v_mask1 = v_neg_inf; + if (r + 1 < n_rows_g) { + const size_t r1 = r + 1; + v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + r1 * args->mask_vtcm_row_stride + c); + } + } + + // Threshold: mask values below -16.0 are treated as -inf (causal mask). + HVX_VectorPred q_keep0 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask0, v_threshold), q_tail_keep); + HVX_VectorPred q_keep1 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask1, v_threshold), q_tail_keep); + + // Scale mask values by log2(e) for base-2 calculations + const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F); + HVX_Vector v_mask0_scaled = hvx_vec_mul_f16_f16(v_mask0, v_log2e); + HVX_Vector v_mask1_scaled = hvx_vec_mul_f16_f16(v_mask1, v_log2e); + + if (has_alibi) { + HVX_Vector v_sm0 = hvx_vec_mul_f16_f16(v_mask0_scaled, v_slope0); + HVX_Vector v_sm1 = hvx_vec_mul_f16_f16(v_mask1_scaled, v_slope1); + my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_sm0), v_neg_inf); + my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_sm1), v_neg_inf); } else { - HVX_Vector diff_vec = hvx_vec_splat_f32(s_val - M); - vs = hvx_vec_get_f32(hvx_vec_exp_f32(diff_vec)); - S += vs; + my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_mask0_scaled), v_neg_inf); + my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_mask1_scaled), v_neg_inf); } - const uint8_t * v_ptr = v_base + ic * factx->size_v_row_padded; + v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci]); + v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci]); + } + } else { + size_t c = 0; + for (; c + 64 < kv_rows; c += 128) { + size_t ci0 = c / 64; + size_t ci1 = ci0 + 1; + v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci0]); + v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci0]); + v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci1]); + v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci1]); + } + for (; c < kv_rows; c += 64) { + size_t ci = c / 64; + const size_t ne = hex_smin(kv_rows - c, 64); + HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(ne * sizeof(__fp16)); + if (ne < 64) { + my_row_buf0[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf0[ci], v_neg_inf); + my_row_buf1[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf1[ci], v_neg_inf); + } + v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci]); + v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci]); + } + } - hvx_mad_f32_f16_aa(VKQ32, v_ptr, &vs, DV); + v_s_rowmax0 = hvx_vec_reduce_max_f16(v_s_rowmax0); + v_s_rowmax1 = hvx_vec_reduce_max_f16(v_s_rowmax1); + + // Splat m_prev[r], m_prev[r+1] from the float per-row accumulators and convert to fp16 vectors + HVX_Vector v_m_prev0, v_m_prev1; + if (r_vec_off < 32) { + HVX_Vector v0 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v0, r_vec_off * 4)); + v_m_prev0 = hvx_vec_f32_to_f16(v0, v0); + if (r + 1 < n_rows_g) { + HVX_Vector v1 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v0, (r_vec_off + 1) * 4)); + v_m_prev1 = hvx_vec_f32_to_f16(v1, v1); + } else { + v_m_prev1 = Q6_V_vzero(); + } + } else { + HVX_Vector v0 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v1, (r_vec_off - 32) * 4)); + v_m_prev0 = hvx_vec_f32_to_f16(v0, v0); + if (r + 1 < n_rows_g) { + HVX_Vector v1 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v1, (r_vec_off + 1 - 32) * 4)); + v_m_prev1 = hvx_vec_f32_to_f16(v1, v1); + } else { + v_m_prev1 = Q6_V_vzero(); } + } + + HVX_Vector v_dup_m0 = Q6_Vhf_vmax_VhfVhf(v_m_prev0, v_s_rowmax0); + HVX_Vector v_dup_m1 = Q6_Vhf_vmax_VhfVhf(v_m_prev1, v_s_rowmax1); - M_vec = hvx_vec_splat_f32(M); - S_vec = hvx_vec_splat_f32(S); + // Insert row r, r+1 rowmax into rowmax_acc_v + { + HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2); + HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2); + HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2); + HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start); + HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid); + rowmax_acc_v = Q6_V_vmux_QVV(p_lane0, v_dup_m0, rowmax_acc_v); + rowmax_acc_v = Q6_V_vmux_QVV(p_lane1, v_dup_m1, rowmax_acc_v); } - // Issue DMA for next+1 block (if exists) - if (ib + 2 < factx->n_blocks) { - const uint32_t next_ib = ib + 2; - const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE; - const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start); + // Compute P = exp(S - m_new) + const HVX_Vector v_zero = Q6_V_vzero(); + HVX_Vector v_p_rowsum0 = v_zero; + HVX_Vector v_p_rowsum1 = v_zero; + + size_t c = 0; + for (; c + 64 < kv_rows; c += 128) { + size_t ci0 = c / 64; + size_t ci1 = ci0 + 1; + + HVX_Vector v_s_minus_m0_0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci0], v_dup_m0); + HVX_Vector v_s_minus_m1_0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci0], v_dup_m1); + HVX_Vector v_s_minus_m0_1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci1], v_dup_m0); + HVX_Vector v_s_minus_m1_1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci1], v_dup_m1); + + HVX_Vector v_p_row0_hf_0 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0_0)); + HVX_Vector v_p_row1_hf_0 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1_0)); + HVX_Vector v_p_row0_hf_1 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0_1)); + HVX_Vector v_p_row1_hf_1 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1_1)); + + __fp16 * out_dtile0 = p_st_base + ci0 * HMX_FP16_TILE_N_ELMS * 2; + __fp16 * out_dtile1 = p_st_base + ci1 * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_p_out0_0 = ((HVX_Vector *) out_dtile0) + r1 / 2; + HVX_Vector * pv_p_out1_0 = pv_p_out0_0 + 16; + HVX_Vector * pv_p_out0_1 = ((HVX_Vector *) out_dtile1) + r1 / 2; + HVX_Vector * pv_p_out1_1 = pv_p_out0_1 + 16; + + HVX_VectorPair vp_p_dual0 = Q6_W_vshuff_VVR(v_p_row1_hf_0, v_p_row0_hf_0, -2); + *pv_p_out0_0 = Q6_V_lo_W(vp_p_dual0); + *pv_p_out1_0 = Q6_V_hi_W(vp_p_dual0); + + HVX_VectorPair vp_p_dual1 = Q6_W_vshuff_VVR(v_p_row1_hf_1, v_p_row0_hf_1, -2); + *pv_p_out0_1 = Q6_V_lo_W(vp_p_dual1); + *pv_p_out1_1 = Q6_V_hi_W(vp_p_dual1); + + HVX_VectorPair vp_p0_0 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf_0); + HVX_VectorPair vp_p1_0 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf_0); + HVX_VectorPair vp_p0_1 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf_1); + HVX_VectorPair vp_p1_1 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf_1); + + v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0_0), Q6_V_hi_W(vp_p0_0))); + v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0_1), Q6_V_hi_W(vp_p0_1))); + v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1_0), Q6_V_hi_W(vp_p1_0))); + v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1_1), Q6_V_hi_W(vp_p1_1))); + } + for (size_t c_rem = c; c_rem < kv_rows; c_rem += 64) { + size_t ci = c_rem / 64; + HVX_Vector v_s_minus_m0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci], v_dup_m0); + HVX_Vector v_s_minus_m1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci], v_dup_m1); + + HVX_Vector v_p_row0_hf = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0)); + HVX_Vector v_p_row1_hf = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1)); + __fp16 * out_dtile = p_st_base + ci * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_p_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_p_out1 = pv_p_out0 + 16; + + HVX_VectorPair vp_p_dual = Q6_W_vshuff_VVR(v_p_row1_hf, v_p_row0_hf, -2); + *pv_p_out0 = Q6_V_lo_W(vp_p_dual); + *pv_p_out1 = Q6_V_hi_W(vp_p_dual); + + HVX_VectorPair vp_p0 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf); + HVX_VectorPair vp_p1 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf); + + v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0), Q6_V_hi_W(vp_p0))); + v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1), Q6_V_hi_W(vp_p1))); + } - // K - const uint8_t * k_src = (const uint8_t *) k->data + (next_ic_start*nbk1 + ik2*nbk2 + ik3*nbk3); - dma_queue_push(dma, dma_make_ptr(k_base, k_src), factx->size_k_row_padded, nbk1, size_k_row, next_block_size); + HVX_Vector rowsum0_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum0)); + HVX_Vector rowsum1_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum1)); + { + HVX_Vector rv0_v = hvx_vec_f32_to_f16(rowsum0_sf, rowsum0_sf); + HVX_Vector rv1_v = hvx_vec_f32_to_f16(rowsum1_sf, rowsum1_sf); + + HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2); + HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2); + HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2); + HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start); + HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid); + rowsum_acc_v = Q6_V_vmux_QVV(p_lane0, rv0_v, rowsum_acc_v); + rowsum_acc_v = Q6_V_vmux_QVV(p_lane1, rv1_v, rowsum_acc_v); + } + } - // V - const uint8_t * v_src = (const uint8_t *) v->data + (next_ic_start*nbv1 + iv2*nbv2 + iv3*nbv3); - dma_queue_push(dma, dma_make_ptr(v_base, v_src), factx->size_v_row_padded, nbv1, size_v_row, next_block_size); + // Inline fa_ml_update_and_build_d for this vector (lock-free and in parallel) + HVX_VectorPair rowmax_acc_pair = hvx_vec_f16_to_f32(rowmax_acc_v); + HVX_Vector v_rowmax_acc_f32_0 = Q6_V_lo_W(rowmax_acc_pair); + HVX_Vector v_rowmax_acc_f32_1 = Q6_V_hi_W(rowmax_acc_pair); - // Mask - if (mask) { - const uint8_t * m_src = (const uint8_t *) (mp_base + next_ic_start); - dma_cache_push(dma, &m_cache, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1); + HVX_Vector v_m_curr0 = Q6_Vsf_vmax_VsfVsf(m_prev_v0, v_rowmax_acc_f32_0); + HVX_Vector v_m_curr1 = Q6_Vsf_vmax_VsfVsf(m_prev_v1, v_rowmax_acc_f32_1); + + HVX_Vector v_m_diff0 = HVX_OP_SUB_F32(m_prev_v0, v_m_curr0); + HVX_Vector v_m_diff1 = HVX_OP_SUB_F32(m_prev_v1, v_m_curr1); + + HVX_Vector v_m_diff_f16 = hvx_vec_f32_to_f16(v_m_diff0, v_m_diff1); + HVX_Vector exp_m_diff_f16 = hvx_vec_exp2_f16(v_m_diff_f16); + + HVX_VectorPair exp_m_diff_pair = hvx_vec_f16_to_f32(exp_m_diff_f16); + HVX_Vector exp_m_diff0 = Q6_V_lo_W(exp_m_diff_pair); + HVX_Vector exp_m_diff1 = Q6_V_hi_W(exp_m_diff_pair); + + HVX_VectorPair rowsum_acc_pair = hvx_vec_f16_to_f32(rowsum_acc_v); + HVX_Vector v_rowsum_acc_f32_0 = Q6_V_lo_W(rowsum_acc_pair); + HVX_Vector v_rowsum_acc_f32_1 = Q6_V_hi_W(rowsum_acc_pair); + + HVX_Vector l_prev_v0 = factx->vtcm_l_vec[r_vec_idx * 2 + 0]; + HVX_Vector l_prev_v1 = factx->vtcm_l_vec[r_vec_idx * 2 + 1]; + HVX_Vector v_l_curr0 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v0, exp_m_diff0), v_rowsum_acc_f32_0); + HVX_Vector v_l_curr1 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v1, exp_m_diff1), v_rowsum_acc_f32_1); + + factx->vtcm_m_vec[r_vec_idx * 2 + 0] = v_m_curr0; + factx->vtcm_m_vec[r_vec_idx * 2 + 1] = v_m_curr1; + factx->vtcm_l_vec[r_vec_idx * 2 + 0] = v_l_curr0; + factx->vtcm_l_vec[r_vec_idx * 2 + 1] = v_l_curr1; + + // Build diagonal tile D = diag(exp(m_diff)) + const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets; + const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16)); + HVX_Vector v_exp_m_diff = exp_m_diff_f16; + + __fp16 * const d_tiles_out = factx->vtcm_d_tiles[args->buf_idx]; + + size_t t0 = r_vec_idx * 2; + if (t0 < args->n_row_tiles) { + const HVX_Vector v_content = v_exp_m_diff; + __fp16 * out_base = d_tiles_out + t0 * HMX_FP16_TILE_N_ELMS; + Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); + } + + size_t t1 = r_vec_idx * 2 + 1; + if (t1 < args->n_row_tiles) { + const HVX_Vector v_content = Q6_V_vror_VR(v_exp_m_diff, 64); + __fp16 * out_base = d_tiles_out + t1 * HMX_FP16_TILE_N_ELMS; + Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64)); +} + +static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(n, i, data, + /*has_mask=*/false, + /*mask_broadcast=*/false, + /*is_g1=*/false, + /*has_alibi=*/false, + /*has_softcap=*/false); +} + +static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(n, i, data, + /*has_mask=*/true, + /*mask_broadcast=*/true, + /*is_g1=*/true, + /*has_alibi=*/false, + /*has_softcap=*/false); +} + +static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, void * data) { + fa_softmax_impl(n, i, data, + /*has_mask=*/true, + /*mask_broadcast=*/true, + /*is_g1=*/false, + /*has_alibi=*/false, + /*has_softcap=*/false); +} + +static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) { + fa_softmax_args_t * args = (fa_softmax_args_t *) data; + struct hmx_fa_context * factx = args->factx; + + const bool has_mask = (args->mask != NULL); + const bool mask_broadcast = factx->mask_broadcast; + const bool is_g1 = (args->G == 1); + const bool has_alibi = args->has_alibi; + const bool has_softcap = factx->has_softcap; + + fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap); +} + +static inline HVX_Vector hvx_fa_load_sinks(const float * sinks_data, size_t idx, size_t count) { + const HVX_Vector * v_src = (const HVX_Vector *) sinks_data + (idx >> 5); + const size_t lane_off = idx & 31; + const size_t byte_off = lane_off * sizeof(float); + return (lane_off + count > 32) + ? Q6_V_valign_VVR(v_src[1], v_src[0], byte_off) + : Q6_V_vror_VR(v_src[0], byte_off); +} + +static __attribute__((noinline)) void fa_build_d_diag_inv_l(struct hmx_fa_context * factx, + size_t n_row_tiles, + size_t n_row_tiles_g_br, + uint32_t kv_head, + size_t n_rows_g) { + (void) n_row_tiles_g_br; + + // sinks: l += exp(sink - m * ln2) in f32, m and l are in the log2 domain + if (factx->sinks) { + const float * sinks_data = factx->vtcm_sinks; + const size_t G = factx->G; + const size_t head_base = (size_t) kv_head * G; + const HVX_Vector v_ln2 = Q6_V_vsplat_R(EXP_LOGN2); + const HVX_Vector v_head_sinks = (G <= 32) ? hvx_fa_load_sinks(sinks_data, head_base, G) : Q6_V_vzero(); + HVX_Vector v_sink = Q6_V_vzero(); + size_t g_prev = SIZE_MAX; + + for (size_t i = 0; i < n_row_tiles; ++i) { + // the sink pattern of a tile only depends on (i * 32) % G, rebuild it when that changes + const size_t r0 = i * 32; + const size_t g0 = fastmodulo(r0, G, &factx->div_G); + if (g0 != g_prev) { + const size_t rem_g = G - g0; + if (G <= 32) { + const HVX_Vector v_lo = Q6_V_vror_VR(v_head_sinks, g0 * sizeof(float)); + const HVX_Vector v_hi = Q6_V_vror_VR(v_head_sinks, 128 - rem_g * sizeof(float)); + v_sink = Q6_V_vmux_QVV(Q6_Q_vsetq2_R(rem_g * sizeof(float)), v_lo, v_hi); + for (size_t len = G; len < 32; len <<= 1) { + const HVX_VectorPred q_keep = Q6_Q_vsetq_R(len * sizeof(float)); + const HVX_Vector v_shifted = Q6_V_vror_VR(v_sink, 128 - len * sizeof(float)); + v_sink = Q6_V_vmux_QVV(q_keep, v_sink, v_shifted); + } + } else if (rem_g >= 32) { + v_sink = hvx_fa_load_sinks(sinks_data, head_base + g0, 32); + } else { + const HVX_Vector v_lo = hvx_fa_load_sinks(sinks_data, head_base + g0, rem_g); + const HVX_Vector v_hi = hvx_fa_load_sinks(sinks_data, head_base, 32 - rem_g); + v_sink = Q6_V_vmux_QVV(Q6_Q_vsetq_R(rem_g * sizeof(float)), v_lo, Q6_V_vror_VR(v_hi, 128 - rem_g * sizeof(float))); } + g_prev = g0; + } - // FARF(HIGH, "fa %u: prefetch KVM: ir %u ib %u : iq1 %u iq2 %u iq3 %u : size_k_row %u size_v_row %u bs %u: usec %u", - // ith, ir, next_ib, iq1, iq2, iq3, - // size_k_row, size_v_row, next_block_size, - // (unsigned)HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - factx->t_start)); + HVX_Vector v_d = HVX_OP_SUB_F32(v_sink, HVX_OP_MUL_F32(factx->vtcm_m_vec[i], v_ln2)); + if (r0 + 32 > n_rows_g) { + const HVX_VectorPred q_valid = Q6_Q_vsetq_R((n_rows_g - r0) * sizeof(float)); + v_d = Q6_V_vmux_QVV(q_valid, v_d, Q6_V_vzero()); } + factx->vtcm_l_vec[i] = HVX_OP_ADD_F32(factx->vtcm_l_vec[i], hvx_vec_exp_f32(v_d)); + } + } + + const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets; + const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16)); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + + HVX_Vector v_content = Q6_V_vzero(); + for (size_t i = 0; i < n_row_tiles; ++i) { + if ((i % 2) == 0) { + HVX_Vector inv_lo = HVX_OP_MUL_F32(one, hvx_vec_inverse_f32(factx->vtcm_l_vec[i])); + HVX_Vector inv_hi = (i + 1 < n_row_tiles) ? HVX_OP_MUL_F32(one, hvx_vec_inverse_f32(factx->vtcm_l_vec[i + 1])) : Q6_V_vzero(); + v_content = hvx_vec_f32_to_f16(inv_lo, inv_hi); + } else { + v_content = Q6_V_vror_VR(v_content, 64); + } + + __fp16 * out_base = factx->vtcm_d_inv_l + i * HMX_FP16_TILE_N_ELMS; + Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content); + } +} + +static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx, + fa_softmax_args_t * sargs, + size_t n_row_tiles, + size_t n_row_tiles_g_br) { + work_queue_t wp = factx->octx->ctx->work_queue; + const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64); + + worker_callback_t softmax_fn = fa_softmax_thread; + if (sargs->mask == NULL && !factx->has_softcap && !sargs->has_alibi) { + softmax_fn = fa_softmax_thread_nomask; + } else if (sargs->mask != NULL && factx->mask_broadcast && !factx->has_softcap && !sargs->has_alibi) { + if (sargs->G == 1) { + softmax_fn = fa_softmax_thread_mask_broadcast_g1; + } else { + softmax_fn = fa_softmax_thread_mask_broadcast_gn; } + } - // sinks - float M = hvx_vec_get_f32(M_vec); - float S = hvx_vec_get_f32(S_vec); + if (factx->n_threads > 1 && n_row_vec_cnt >= 2) { + uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt); + sargs->thread_div = init_fastdiv_values(n_use); + work_queue_run(wp, softmax_fn, sargs, n_use); + } else { + softmax_fn(1, 0, sargs); + } +} - if (sinks) { - const float s = ((float *)((char *) sinks->data))[h]; +// ============================================================================ +// HMX job structs and worker functions +// ============================================================================ + +typedef struct { + const __fp16 * q_tiles; + const __fp16 * k_tiles; + __fp16 * s_tiles; + size_t n_row_tiles; + size_t n_col_tiles; + size_t n_dot_tiles; // DK / 32 + size_t n_tiles_per_bc; + uint8_t * hmx_scales; +} hmx_fa_qk_job_t; + +static void hmx_fa_qk_dot_worker(void * data) { + hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data; + const size_t n_row_tiles = job->n_row_tiles; + const size_t n_col_tiles = job->n_col_tiles; + const size_t n_dot_tiles = job->n_dot_tiles; + const size_t n_tiles_per_bc = job->n_tiles_per_bc; + const __fp16 * restrict q_tiles = job->q_tiles; + const __fp16 * restrict k_tiles = job->k_tiles; + __fp16 * restrict s_tiles = job->s_tiles; + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(n_dot_tiles > 0); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales)); + const size_t dot_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS; + for (size_t r = 0; r < n_row_tiles; ++r) { + const __fp16 * row_tiles = q_tiles + r * dot_stride; + const __fp16 * col_tiles = k_tiles; + __fp16 * out_tile = s_tiles + r * n_tiles_per_bc * HMX_FP16_TILE_N_ELMS; + + for (size_t c = 0; c < n_col_tiles; ++c) { + hmx_fa_qk_dot_tile(row_tiles, col_tiles, out_tile, n_dot_tiles); + col_tiles += dot_stride; + out_tile += HMX_FP16_TILE_N_ELMS; + } + } +} - float vs = 1.0f; +typedef struct { + __fp16 * o_curr; + const __fp16 * o_prev; + const __fp16 * p_tiles; + const __fp16 * v_tiles; + const __fp16 * d_tiles; + uint8_t * hmx_scales; + size_t n_row_tiles; + size_t n_col_tiles; + size_t n_row_tiles_g_br; + size_t n_tiles_per_bc; + size_t DV; +} hmx_fa_o_update_job_t; + +static void hmx_fa_o_update_worker(void * data) { + hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data; + const size_t n_row_tiles = job->n_row_tiles; + const size_t n_col_tiles = job->n_col_tiles; + const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; + const size_t n_tiles_per_bc = job->n_tiles_per_bc; + const size_t DV_tiles = job->DV / 32; + const __fp16 * restrict d_tiles = job->d_tiles; + const __fp16 * restrict p_tiles = job->p_tiles; + const __fp16 * restrict v_tiles = job->v_tiles; + const __fp16 * restrict o_prev = job->o_prev; + __fp16 * restrict o_curr = job->o_curr; + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(DV_tiles > 0); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales)); + const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS; + const size_t v_stride = n_tiles_per_bc * HMX_FP16_TILE_N_ELMS; + for (size_t r = 0; r < n_row_tiles; ++r) { + const __fp16 * d_diag = d_tiles + r * HMX_FP16_TILE_N_ELMS; + const __fp16 * p_tile_in = p_tiles + (r * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS; + const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS; + const __fp16 * v_tile_in = v_tiles; + __fp16 * o_tile_out = o_curr + r * HMX_FP16_TILE_N_ELMS; + + for (size_t c = 0; c < DV_tiles; ++c) { + hmx_fa_o_update_tile(d_diag, o_rc, p_tile_in, v_tile_in, o_tile_out, n_col_tiles); + o_rc += o_stride; + v_tile_in += v_stride; + o_tile_out += o_stride; + } + } +} - if (s > M) { - HVX_Vector diff_vec = hvx_vec_splat_f32(M - s); - HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec); - hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec); +typedef struct { + __fp16 * o_curr; // output (row-major tile layout) + const __fp16 * o_prev; // input (column-major tile layout) + const __fp16 * d_tiles; // diag(1/l) tiles + uint8_t * hmx_scales; + size_t n_row_tiles; + size_t n_row_tiles_g_br; + size_t DV; +} hmx_fa_o_norm_job_t; + +static void hmx_fa_o_norm_worker(void * data) { + hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data; + const size_t n_row_tiles = job->n_row_tiles; + const size_t n_row_tiles_g_br = job->n_row_tiles_g_br; + const size_t DV_tiles = job->DV / 32; + const __fp16 * restrict d_tiles = job->d_tiles; + const __fp16 * restrict o_prev = job->o_prev; + __fp16 * restrict o_curr = job->o_curr; + __builtin_assume(n_row_tiles > 0); + __builtin_assume(DV_tiles > 0); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales)); + const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS; + for (size_t r = 0; r < n_row_tiles; ++r) { + const __fp16 * d_diag = d_tiles + r * HMX_FP16_TILE_N_ELMS; + const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS; + __fp16 * o_out = o_curr + r * DV_tiles * HMX_FP16_TILE_N_ELMS; + + for (size_t c = 0; c < DV_tiles; ++c) { + hmx_fa_o_norm_tile(d_diag, o_rc, o_out); + o_rc += o_stride; + o_out += HMX_FP16_TILE_N_ELMS; + } + } +} - float ms = hvx_vec_get_f32(ms_vec); - S = S * ms + vs; - } else { - HVX_Vector diff_vec = hvx_vec_splat_f32(s - M); - vs = hvx_vec_get_f32(hvx_vec_exp_f32(diff_vec)); - S += vs; +// Populate per-GQA-row ALiBi slopes for a given KV head. +static __attribute__((noinline)) void fa_compute_slopes( + const struct hmx_fa_context * factx, + uint32_t kv_head, + size_t n_rows_g) { + __fp16 * slopes = factx->vtcm_slopes; + if (factx->max_bias == 0.0f) { + hvx_splat_f16_a(slopes, 1.0f, n_rows_g); + return; + } + + const uint32_t G = factx->G; + const uint32_t n_head_log2 = factx->n_head_log2; + const float m0 = factx->m0; + const float m1 = factx->m1; + + __fp16 * temp_slopes = factx->ddr_slopes; + if (G <= 32) { + // Fast path: Compute G unique slope values in vector registers + HVX_Vector v_val = hvx_alibi_slopes(kv_head, G, n_head_log2, m0, m1); + + __fp16 temp_slopes_aligned[64] __attribute__((aligned(128))); + hvx_vmem(temp_slopes_aligned) = hvx_vec_f32_to_f16(v_val, Q6_V_vzero()); + + for (uint32_t i = 0; i < G; ++i) { + temp_slopes[i] = temp_slopes_aligned[i]; + } + } else { + // Fallback path: G > 32 (rare configurations) + for (uint32_t i = 0; i < G; ++i) { + temp_slopes[i] = (__fp16)alibi_slope(kv_head * G + i, n_head_log2, m0, m1); + } + } + + // Replicate the G values in DDR to avoid scalar writes to VTCM (which generates L2 misses) + for (size_t r = G; r < n_rows_g; ++r) { + temp_slopes[r] = temp_slopes[r - G]; + } + + // Copy to VTCM slopes using HVX block copy (both are aligned to 128 bytes) + hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)temp_slopes, n_rows_g); +} + +static void fa_push_mask_dma_gqa( + dma_queue * dma_q, + const struct htp_tensor * mask, + uint32_t q_start, + uint32_t im3, + uint32_t kv_start, + uint32_t kv_head, + uint32_t G, + uint32_t m_line_bytes, + uint32_t kv_rows, + uint32_t n_rows_q, + struct hmx_fa_context * factx +) { + for (uint32_t g = 0; g < G; ++g) { + const uint32_t h_idx = kv_head * G + g; + const uint32_t im2 = fastmodulo(h_idx, mask->ne[2], &factx->src3_div2); + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + + im2 * mask->nb[2] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); + uint8_t * ms_dst = (uint8_t *) factx->vtcm_mask_buf + g * m_line_bytes; + dma_queue_push(dma_q, dma_make_data(ms_dst, ms_src), G * m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); + } +} + +static void fa_pop_mask_dma_gqa(dma_queue * dma_q, uint32_t G) { + for (uint32_t g = 0; g < G; ++g) { + dma_queue_pop(dma_q); + } +} + +static inline void fa_prefetch_block(dma_queue * dma_q, const struct htp_tensor * k, const struct htp_tensor * v, const struct htp_tensor * mask, + uint32_t b, size_t Bc, size_t size_k_row_padded, size_t size_k_row, size_t size_v_row_padded, size_t size_v_row, + uint32_t ik2, uint32_t ik3, uint32_t iv2, uint32_t iv3, uint32_t q_start, uint32_t im3, uint32_t kv_head, uint32_t G, + size_t m_line_bytes, size_t n_rows_q, size_t nek1, size_t prefetch_buf, struct hmx_fa_context * factx) { + const uint32_t prefetch_start = b * Bc; + const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start); + const dma_addr_t k_prefetch_src = k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx->vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); + const dma_addr_t v_prefetch_src = v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx->vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); + + if (mask) { + if (__builtin_expect(factx->mask_broadcast, true)) { + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16); + dma_cache_fa_push(dma_q, factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q); + } else { + fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx); + } + } +} + +// ============================================================================ +// Core HMX flash attention algorithm (GQA-merged) +// ============================================================================ + +int hmx_flash_attn_ext(struct htp_ops_context * octx) { + struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0]; + struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS]; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * v = octx->src[2]; + const struct htp_tensor * mask = (octx->src[3] && octx->src[3]->data) ? octx->src[3] : NULL; + const struct htp_tensor * dst = octx->dst; + + struct htp_context * const ctx = octx->ctx; + + if (!ctx->hmx_enabled) { + return HTP_STATUS_NO_SUPPORT; + } + + // Dimensions + const uint32_t neq0 = q->ne[0]; // head_dim (DK) + const uint32_t neq1 = q->ne[1]; // n_tokens + const uint32_t neq2 = q->ne[2]; // n_heads + const uint32_t neq3 = q->ne[3]; // n_seqs + + const uint32_t nek0 = k->ne[0]; // head_dim + const uint32_t nek1 = k->ne[1]; // kv_len + + const uint32_t nev0 = v->ne[0]; // head_dim (DV) + + const uint32_t DK = neq0; + const uint32_t DV = nev0; + + // HMX tiles head_dim in units of 64. head_dim need not be 64- (or 32-) aligned: + // we can operate on DK/DV rounded up to 64 with tail lanes [D, D_pad) zero-filled. + const uint32_t DK_pad = hex_round_up(DK, 64); + const uint32_t DV_pad = hex_round_up(DV, 64); + if (DK == 0 || DV == 0) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params; + const uint32_t n_kv_heads = k->ne[2]; + + // ======== Build context ======== + struct hmx_fa_context factx; + memset(&factx, 0, sizeof(factx)); + factx.octx = octx; + factx.sinks = octx->src[4]; // NULL if this op has no attention sinks + factx.k = k; + factx.v = v; + factx.n_threads = kparams->n_threads; + factx.DK = DK; + factx.DV = DV; + factx.DK_pad = DK_pad; + factx.DV_pad = DV_pad; + factx.n_kv = nek1; + factx.n_kv_heads = n_kv_heads; + factx.n_heads = neq2; + factx.G = kparams->G; + factx.div_G = kparams->u.hmx.div_G; + factx.neq1 = neq1; + factx.Br = kparams->Br; + factx.Bc = kparams->Bc; + factx.g_br = kparams->u.hmx.g_br; + factx.n_kv_blocks = kparams->n_kv_blocks; + factx.is_q_fp32 = (q->type == HTP_TYPE_F32); + factx.is_dst_fp32 = (dst->type == HTP_TYPE_F32); + factx.pipeline = (kparams->u.hmx.pipeline != 0); + factx.mask_broadcast = (kparams->u.hmx.mask_broadcast != 0); + if (mask) { + factx.src3_div2 = kparams->src3_div2; + factx.src3_div3 = kparams->src3_div3; + } + + factx.has_softcap = (kparams->logit_softcap != 0.0f); + factx.scale = (__fp16) kparams->scale; + factx.max_bias = kparams->max_bias; + factx.logit_softcap = 0; + if (factx.has_softcap) { + factx.logit_softcap = (__fp16) kparams->logit_softcap; + } + + factx.n_head_log2 = kparams->n_head_log2; + factx.m0 = kparams->m0; + factx.m1 = kparams->m1; + + const uint32_t Br = factx.Br; + const uint32_t Bc = factx.Bc; + const uint32_t g_br = factx.g_br; + const bool pipeline = factx.pipeline; + const uint32_t n_threads = factx.n_threads; + const uint32_t G = factx.G; + + // Multi-device: prefer head-parallel partitioning (each core owns a disjoint head + // shard), falling back to Q-block (token) split when heads don't divide evenly. + const uint32_t n_q_blocks = (neq1 + Br - 1) / Br; + uint32_t q_start_min = 0; + uint32_t q_start_max = neq1; + uint32_t kv_head_min = 0; + uint32_t kv_head_max = n_kv_heads; + + if (octx->ctx->mdev.count > 1) { + const uint32_t mdev_count = octx->ctx->mdev.count; + const uint32_t mdev_idx = octx->ctx->mdev.idx; + const uint32_t dst_e_size = (dst->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(__fp16); + const bool can_split = htp_tensor_can_row_partition(dst, dst_e_size); + + if ((kparams->flags & HTP_FA_FLAG_TEAM_SPLIT) && can_split) { + // KV heads split across teams, Q-blocks split within a team + uint32_t n_teams, team_size; + htp_fa_mdev_teams(n_kv_heads, mdev_count, kparams->head_split != 0, &n_teams, &team_size); + + const uint32_t team = mdev_idx / team_size; + const uint32_t member = mdev_idx - team * team_size; + const uint32_t kv_per_team = n_kv_heads / n_teams; + kv_head_min = team * kv_per_team; + kv_head_max = kv_head_min + kv_per_team; + + const uint32_t base = n_q_blocks / team_size; + const uint32_t rem = n_q_blocks - base * team_size; + const uint32_t b0 = member * base + MIN(member, rem); + const uint32_t nblk = base + (member < rem ? 1 : 0); + + if (nblk == 0) { + return HTP_STATUS_OK; } + + q_start_min = b0 * Br; + q_start_max = MIN((b0 + nblk) * Br, neq1); + } else if (kparams->head_split && can_split && n_kv_heads >= mdev_count && n_kv_heads % mdev_count == 0) { + const uint32_t kv_per_core = n_kv_heads / mdev_count; + kv_head_min = mdev_idx * kv_per_core; + kv_head_max = kv_head_min + kv_per_core; + } else { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_q_blocks, can_split ? 1 : 0, mdev_idx, mdev_count, &octx->ctx->mdev.count_div); + const uint32_t block_start = range.start; + const uint32_t block_end = range.start + range.count; + + if (block_start >= block_end) { + return HTP_STATUS_OK; + } + + q_start_min = block_start * Br; + q_start_max = MIN(block_end * Br, neq1); } + } + + // ======== VTCM allocation (GQA-aware) ======== + // K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used + // throughout the KV loop below. The DMA copies only the real DK/DV columns; the + // staging rows are padded to hold DK_pad/DV_pad columns (tail zero-filled below) + // so the HMX interleave/tile logic can operate on 64-aligned head dims. + const size_t size_k_row = htp_tensor_get_row_size(k->type, DK); + const size_t size_v_row = htp_tensor_get_row_size(v->type, DV); + const size_t size_k_row_padded = hex_round_up(DK_pad * sizeof(__fp16), 128); + const size_t size_v_row_padded = hex_round_up(DV_pad * sizeof(__fp16), 128); + + // Build the VTCM layout once (shared with the host estimator) and place every + // scratch buffer at its computed offset. Padded head dims size the HMX tiles. + struct hmx_fa_vtcm_layout L; + hmx_fa_vtcm_layout_build(&L, G, DK_pad, DV_pad, Br, Bc, n_threads, pipeline, factx.is_q_fp32, factx.sinks != NULL, factx.n_heads); + + if (L.total_bytes > ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + uint8_t * const base = ctx->vtcm_base; + + factx.vtcm_q_dma = VTCM_LAYOUT_PTR(__fp16, base, L.off_q_dma); + factx.vtcm_q_tiles = VTCM_LAYOUT_PTR(__fp16, base, L.off_q_tiles); + factx.vtcm_o_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_o_tiles[0]); + factx.vtcm_o_tiles[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_o_tiles[1]); + factx.vtcm_k_fp16[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_fp16[0]); + factx.vtcm_k_fp16[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_fp16[1]); + factx.vtcm_v_fp16[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_fp16[0]); + factx.vtcm_v_fp16[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_fp16[1]); + factx.vtcm_k_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_tiles[0]); + factx.vtcm_k_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_k_tiles[1], pipeline); + factx.vtcm_v_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_tiles[0]); + factx.vtcm_v_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_v_tiles[1], pipeline); + factx.vtcm_s_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_s_tiles[0]); + factx.vtcm_s_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_s_tiles[1], pipeline); + factx.vtcm_p_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_p_tiles[0]); + factx.vtcm_p_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_p_tiles[1], pipeline); + factx.vtcm_d_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_tiles[0]); + factx.vtcm_d_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_d_tiles[1], pipeline); + factx.vtcm_d_inv_l = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_inv_l); + factx.vtcm_m_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_m_vec); + factx.vtcm_l_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_l_vec); + factx.vtcm_s_rowmax = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_s_rowmax); + factx.vtcm_p_rowsum = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_p_rowsum); + factx.vtcm_row_bufs = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_row_bufs); + factx.row_buf_stride = L.row_buf_stride; + factx.vtcm_hmx_scales_id = VTCM_LAYOUT_PTR(uint8_t, base, L.off_hmx_scales_id); + factx.vtcm_hmx_scales_qk = VTCM_LAYOUT_PTR(uint8_t, base, L.off_hmx_scales_qk); + factx.vtcm_mask_buf = VTCM_LAYOUT_PTR(__fp16, base, L.off_mask_buf); + factx.mask_buf_row_stride = L.mask_buf_row_stride; + factx.q_tile_bytes = L.q_tile_bytes; + factx.k_tile_bytes = L.k_tile_bytes; + factx.o_tile_bytes = L.o_tile_bytes; + factx.col_vec_bytes = L.col_vec_bytes; + factx.d_tile_bytes = L.d_tile_bytes; + factx.vtcm_slopes = VTCM_LAYOUT_PTR(__fp16, base, L.off_slopes); + factx.vtcm_sinks = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_sinks, factx.sinks != NULL); + + const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop + + struct hmx_fa_ddr_spad * ddr = (struct hmx_fa_ddr_spad *) fa_get_ddr_spad(octx, sizeof(struct hmx_fa_ddr_spad) + factx.g_br * sizeof(__fp16)); + if (!ddr) { + return HTP_STATUS_INTERNAL_ERR; + } + + factx.ddr_slopes = ddr->slopes; + factx.m_cache = &octx->dma_cache_fa[0]; + dma_cache_fa_init(factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE); + + // Head-dim padding: the K/V DMA staging buffers and the flat-Q buffer are laid out + // with padded row strides (size_{k,v,q}_row_padded, covering D_pad columns) but the + // DMA only writes the real D columns per row. Zero the whole staging buffers once up + // front so tail lanes [D, D_pad) stay zero for all KV blocks. No-op when already aligned. + if (DK_pad != DK || DV_pad != DV) { + const size_t k_buf_bytes = (size_t) factx.Bc * size_k_row_padded; + const size_t v_buf_bytes = (size_t) factx.Bc * size_v_row_padded; + hvx_splat_u8_a((char *) factx.vtcm_k_fp16[0], 0, k_buf_bytes); + hvx_splat_u8_a((char *) factx.vtcm_k_fp16[1], 0, k_buf_bytes); + hvx_splat_u8_a((char *) factx.vtcm_v_fp16[0], 0, v_buf_bytes); + hvx_splat_u8_a((char *) factx.vtcm_v_fp16[1], 0, v_buf_bytes); + // Flat-Q DMA scratch + if (factx.vtcm_q_dma) { + const size_t q_dma_bytes = hex_align_up(factx.g_br * DK * (factx.is_q_fp32 ? sizeof(float) : sizeof(__fp16)), 128); + hvx_splat_u8_a((char *) factx.vtcm_q_dma, 0, q_dma_bytes); + } + } + + // ======== Initialize HMX output scales ======== + hmx_init_column_scales(factx.vtcm_hmx_scales_id, Q6_V_vsplat_R(0x3c00)); // 1.0 + { + union { __fp16 f; uint16_t u; } sc = { .f = factx.scale }; + hmx_init_column_scales(factx.vtcm_hmx_scales_qk, Q6_V_vsplat_R(sc.u)); + } + + // ======== DMA setup ======== + dma_queue * const dma_q = ctx->dma[0]; + + if (factx.sinks) { + dma_queue_push(dma_q, dma_make_data(factx.vtcm_sinks, factx.sinks->data), L.sinks_bytes, 0, factx.sinks->size, 1); + dma_queue_pop(dma_q); + } - const float S_inv = S == 0.0f ? 0.0f : 1.0f/S; - hvx_scale_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, S_inv); + const size_t n_row_tiles_g_br = g_br / HMX_FP16_TILE_N_ROWS; + const size_t n_tiles_per_bc = Bc / HMX_FP16_TILE_N_COLS; + + const size_t qo_element_size = factx.is_q_fp32 ? sizeof(float) : sizeof(__fp16); + + const bool q_transposed = q->nb[1] < q->nb[2]; + const size_t q_src_stride = q_transposed ? q->nb[2] : q->nb[1]; + const size_t q_row_bytes_untransposed = factx.G * factx.DK * qo_element_size; + const size_t q_row_bytes_trans_factor = factx.DK * qo_element_size; + + // ======== Reusable job descriptors for pipeline ======== + hmx_fa_qk_job_t qk_job; + hmx_fa_o_update_job_t ou_job; + hmx_fa_o_norm_job_t on_job; + + // ======== Main loop ======== + for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) { + const uint32_t im3 = mask ? fastmodulo(ib3, mask->ne[3], &factx.src3_div3) : 0; + for (uint32_t q_start = q_start_min; q_start < q_start_max; q_start += Br) { + const uint32_t n_rows_q = hex_smin(Br, neq1 - q_start); + const size_t n_rows_g = n_rows_q * G; + const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS); + const size_t n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS; + + uint16_t * active_blocks = ddr->active_blocks; + uint32_t n_active_blocks = factx.n_kv_blocks; + if (mask && factx.mask_broadcast && factx.n_kv_blocks > 1) { + const size_t scratch_bytes = factx.k_tile_bytes * (factx.pipeline ? 2 : 1); + n_active_blocks = fa_scan_active_blocks( + dma_q, mask, q_start, n_rows_q, 0, im3, nek1, Bc, + factx.n_kv_blocks, factx.vtcm_k_tiles[0], scratch_bytes, active_blocks); + } else { + #pragma clang loop vectorize(disable) + for (uint32_t b = 0; b < factx.n_kv_blocks; ++b) { + active_blocks[b] = (uint16_t) b; + } + } + const uint32_t b0 = (n_active_blocks > 0) ? active_blocks[0] : 0; + const uint32_t b0_start = b0 * Bc; + const uint32_t b0_rows = hex_smin(Bc, nek1 - b0_start); + + for (uint32_t kv_head = kv_head_min; kv_head < kv_head_max; ++kv_head) { + const uint32_t ik2 = kv_head; + const uint32_t ik3 = fastdiv(ib3, &kparams->broadcast_rk3); + const uint32_t iv2 = kv_head; + const uint32_t iv3 = fastdiv(ib3, &kparams->broadcast_rv3); + + // 1. Push Q and KV DMAs for the first kv_head of this q_start block. + // Subsequent kv_head iterations are enqueued early at the end of the previous iteration. + if (kv_head == kv_head_min) { + const dma_addr_t q_ptr = q->data + q_start * q->nb[1] + + (kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3]; + const size_t q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed; + const size_t n_rows = q_transposed ? factx.G : n_rows_q; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows); + + if (n_active_blocks > 0) { + const dma_addr_t k_src = k->data + b0_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, b0_rows); + + const dma_addr_t v_src = v->data + b0_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, b0_rows); + + if (factx.pipeline && mask) { + if (__builtin_expect(factx.mask_broadcast, true)) { + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16); + dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q); + } else { + fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx); + } + } + } + } + + // 2. Pop Q DMA (blocks until Q is loaded) + dma_queue_pop(dma_q); + + // ---- Load Q block & Initialize per-block state ---- + fa_phase_q_load(&factx, q, q_start, kv_head, ib3, n_rows_g); - // Store result - // dst indices - const int i1 = iq1; - const int i2 = iq2; - const int i3 = iq3; + __fp16 * o_tile_prev = factx.vtcm_o_tiles[0]; + __fp16 * o_tile_curr = factx.vtcm_o_tiles[1]; - // dst is permuted - uint8_t * dst_ptr = (uint8_t *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1) * nb1; + // ---- KV block loop with DMA double-buffering ---- + size_t buf_idx = 0; - if (dst->type == HTP_TYPE_F32) { - hvx_copy_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV); - } else if (dst->type == HTP_TYPE_F16) { - hvx_copy_f16_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV); + htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start); + fa_compute_slopes(&factx, kv_head, n_rows_g); + htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start); + + const size_t k_src_stride = size_k_row_padded / sizeof(__fp16); + const size_t v_src_stride = size_v_row_padded / sizeof(__fp16); + + hmx_queue_t hmx_q = ctx->hmx_queue; + + if (factx.pipeline) { + // Double-buffered job structs because HMX queue runs asynchronously + hmx_fa_qk_job_t qk_job[2]; + hmx_fa_o_update_job_t ou_job[2]; + + // Prefetch second active block early if there are multiple active blocks + if (n_active_blocks > 1) { + fa_prefetch_block(dma_q, k, v, mask, active_blocks[1], Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, + ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, 1, &factx); + } + + // Prep and start QK-dot(0) + void * curr_k0 = (void *) dma_queue_pop(dma_q).dst; + fa_phase_k_interleave(&factx, b0_rows, k_src_stride, curr_k0, b0_start, 0); + + qk_job[0].q_tiles = factx.vtcm_q_tiles; + qk_job[0].k_tiles = factx.vtcm_k_tiles[0]; + qk_job[0].s_tiles = factx.vtcm_s_tiles[0]; + qk_job[0].n_row_tiles = n_row_tiles; + qk_job[0].n_col_tiles = hmx_ceil_div(b0_rows, HMX_FP16_TILE_N_COLS); + qk_job[0].n_dot_tiles = DK_pad / 32; + qk_job[0].n_tiles_per_bc = n_tiles_per_bc; + qk_job[0].hmx_scales = factx.vtcm_hmx_scales_qk; + hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[0])); + + for (uint32_t step = 0; step < n_active_blocks; ++step) { + const uint32_t kv_blk = active_blocks[step]; + const uint32_t kv_start = kv_blk * Bc; + const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); + const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); + + // ---- 1. Pop and run V-prep for current block ---- + void * curr_v = (void *) dma_queue_pop(dma_q).dst; + fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[buf_idx], n_tiles_per_bc, kv_start); + + // ---- 2. Pop and run mask-prep for current block ---- + __fp16 * current_mask_vtcm = NULL; + if (mask) { + if (__builtin_expect(factx.mask_broadcast, true)) { + current_mask_vtcm = (__fp16 *) dma_queue_pop(dma_q).dst; + } else { + fa_pop_mask_dma_gqa(dma_q, G); + current_mask_vtcm = factx.vtcm_mask_buf; + } + } + + // ---- 3. Start HMX O update for step - 1 (reads P[1 - buf_idx], V[1 - buf_idx], D) ---- + // O update relies on the previous block's P and V tiles. + // O update MUST be pushed before the next block's QK-dot: hmx_queue_pop() retires the + // oldest descriptor, so push order alone decides which pop waits for which job. + // If OU went in after QK(i+1), the pop below would retire QK(i+1) and leave + // OU(i-1) in flight into the next iteration, where V-prep overwrites V[prev_buf]. + if (step > 0) { + const uint32_t prev_blk = active_blocks[step - 1]; + const size_t prev_buf = 1 - buf_idx; + ou_job[prev_buf].o_curr = o_tile_curr; + ou_job[prev_buf].o_prev = o_tile_prev; + ou_job[prev_buf].p_tiles = factx.vtcm_p_tiles[prev_buf]; + ou_job[prev_buf].v_tiles = factx.vtcm_v_tiles[prev_buf]; + ou_job[prev_buf].d_tiles = factx.vtcm_d_tiles[prev_buf]; + ou_job[prev_buf].hmx_scales = factx.vtcm_hmx_scales_id; + ou_job[prev_buf].n_row_tiles = n_row_tiles; + ou_job[prev_buf].n_col_tiles = + hmx_ceil_div(hex_smin(Bc, nek1 - prev_blk * Bc), HMX_FP16_TILE_N_COLS); + ou_job[prev_buf].n_row_tiles_g_br = n_row_tiles_g_br; + ou_job[prev_buf].n_tiles_per_bc = n_tiles_per_bc; + ou_job[prev_buf].DV = DV_pad; + hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[prev_buf])); + } + + // ---- 4. Pop and run K-prep for next active block & push next QK-dot ---- + if (step + 1 < n_active_blocks) { + const uint32_t next_blk = active_blocks[step + 1]; + const uint32_t next_start = next_blk * Bc; + const uint32_t next_rows = hex_smin(Bc, nek1 - next_start); + const size_t next_buf = 1 - buf_idx; + + void * next_k = (void *) dma_queue_pop(dma_q).dst; + fa_phase_k_interleave(&factx, next_rows, k_src_stride, next_k, next_start, next_buf); + + qk_job[next_buf].q_tiles = factx.vtcm_q_tiles; + qk_job[next_buf].k_tiles = factx.vtcm_k_tiles[next_buf]; + qk_job[next_buf].s_tiles = factx.vtcm_s_tiles[next_buf]; + qk_job[next_buf].n_row_tiles = n_row_tiles; + qk_job[next_buf].n_col_tiles = hmx_ceil_div(next_rows, HMX_FP16_TILE_N_COLS); + qk_job[next_buf].n_dot_tiles = DK_pad / 32; + qk_job[next_buf].n_tiles_per_bc = n_tiles_per_bc; + qk_job[next_buf].hmx_scales = factx.vtcm_hmx_scales_qk; + hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[next_buf])); + } + + // ---- 5. Wait for current block's QK-dot to finish ---- + hmx_queue_pop(hmx_q); + + // ---- 6. Phase 2: softmax + build_D ---- + fa_softmax_args_t sargs; + memset(&sargs, 0, sizeof(sargs)); + sargs.factx = &factx; + sargs.buf_idx = buf_idx; + sargs.kv_rows = kv_rows; + sargs.n_rows_g = n_rows_g; + sargs.n_col_tiles = n_col_tiles; + sargs.n_tiles_per_bc = n_tiles_per_bc; + sargs.n_row_tiles = n_row_tiles; + sargs.n_row_tiles_g_br = n_row_tiles_g_br; + sargs.Bc = Bc; + sargs.G = G; + sargs.kv_head = kv_head; + sargs.kv_start = kv_start; + sargs.q_start = q_start; + sargs.ib3 = ib3; + sargs.has_alibi = (factx.max_bias != 0.0f); + sargs.mask = mask; + sargs.mask_vtcm = current_mask_vtcm; + sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride; + sargs.slopes = factx.vtcm_slopes; + + // Run Softmax on HVX (blocking call) + fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); + + // Wait for HMX O update for step - 1 to finish + if (step > 0) { + hmx_queue_pop(hmx_q); + hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); + } + + // Prefetch block step + 2 + if (step + 2 < n_active_blocks) { + fa_prefetch_block(dma_q, k, v, mask, active_blocks[step + 2], Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row, + ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, buf_idx, &factx); + } + + buf_idx = 1 - buf_idx; + } + + // Epilogue + if (n_active_blocks > 0) { + const uint32_t last_blk = active_blocks[n_active_blocks - 1]; + const size_t last_cols = hmx_ceil_div(hex_smin(Bc, nek1 - last_blk * Bc), HMX_FP16_TILE_N_COLS); + ou_job[0].o_curr = o_tile_curr; + ou_job[0].o_prev = o_tile_prev; + ou_job[0].p_tiles = factx.vtcm_p_tiles[1 - buf_idx]; + ou_job[0].v_tiles = factx.vtcm_v_tiles[1 - buf_idx]; + ou_job[0].d_tiles = factx.vtcm_d_tiles[1 - buf_idx]; + ou_job[0].hmx_scales = factx.vtcm_hmx_scales_id; + ou_job[0].n_row_tiles = n_row_tiles; + ou_job[0].n_col_tiles = last_cols; + ou_job[0].n_row_tiles_g_br = n_row_tiles_g_br; + ou_job[0].n_tiles_per_bc = n_tiles_per_bc; + ou_job[0].DV = DV_pad; + hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[0])); + + // Overlapped: run HVX build diag inv L while HMX is busy executing the update + htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); + fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br, kv_head, n_rows_g); + htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); + hmx_queue_pop(hmx_q); + + hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); + } + + } else { + // Fallback path + for (uint32_t step = 0; step < n_active_blocks; ++step) { + const uint32_t kv_blk = active_blocks[step]; + const uint32_t kv_start = kv_blk * Bc; + const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start); + const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS); + + if (mask) { + if (__builtin_expect(factx.mask_broadcast, true)) { + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16); + dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q); + } else { + fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx); + } + } + + if (step + 1 < n_active_blocks) { + const uint32_t prefetch_blk = active_blocks[step + 1]; + const uint32_t prefetch_start = prefetch_blk * Bc; + const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start); + const size_t prefetch_buf = 1 - buf_idx; + const dma_addr_t k_prefetch_src = k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows); + const dma_addr_t v_prefetch_src = v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows); + } + + // Wait for current K DMA and interleave + void * curr_k = (void *) dma_queue_pop(dma_q).dst; + fa_phase_k_interleave(&factx, kv_rows, k_src_stride, curr_k, kv_start, 0); + + { + qk_job.q_tiles = factx.vtcm_q_tiles; + qk_job.k_tiles = factx.vtcm_k_tiles[0]; + qk_job.s_tiles = factx.vtcm_s_tiles[0]; + qk_job.n_row_tiles = n_row_tiles; + qk_job.n_col_tiles = n_col_tiles; + qk_job.n_dot_tiles = (size_t) (DK_pad / 32); + qk_job.n_tiles_per_bc = n_tiles_per_bc; + qk_job.hmx_scales = factx.vtcm_hmx_scales_qk; + + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job)); + hmx_queue_pop(ctx->hmx_queue); + } + + // Wait for current V DMA and interleave + void * curr_v = (void *) dma_queue_pop(dma_q).dst; + fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[0], n_tiles_per_bc, kv_start); + + // ---- Phase 3: softmax + build_D ---- + __fp16 * current_mask_vtcm = NULL; + if (mask) { + if (__builtin_expect(factx.mask_broadcast, true)) { + current_mask_vtcm = (__fp16 *) dma_queue_pop(dma_q).dst; + } else { + fa_pop_mask_dma_gqa(dma_q, G); + current_mask_vtcm = factx.vtcm_mask_buf; + } + } + + fa_softmax_args_t sargs; + memset(&sargs, 0, sizeof(sargs)); + sargs.factx = &factx; + sargs.kv_rows = kv_rows; + sargs.n_rows_g = n_rows_g; + sargs.n_col_tiles = n_col_tiles; + sargs.n_tiles_per_bc = n_tiles_per_bc; + sargs.n_row_tiles = n_row_tiles; + sargs.n_row_tiles_g_br = n_row_tiles_g_br; + sargs.Bc = Bc; + sargs.G = G; + sargs.kv_head = kv_head; + sargs.kv_start = kv_start; + sargs.q_start = q_start; + sargs.ib3 = ib3; + sargs.has_alibi = (factx.max_bias != 0.0f); + sargs.mask = mask; + sargs.mask_vtcm = current_mask_vtcm; + sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride; + sargs.slopes = factx.vtcm_slopes; + fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br); + + { + ou_job.o_curr = o_tile_curr; + ou_job.o_prev = o_tile_prev; + ou_job.p_tiles = factx.vtcm_p_tiles[0]; + ou_job.v_tiles = factx.vtcm_v_tiles[0]; + ou_job.d_tiles = factx.vtcm_d_tiles[0]; + ou_job.hmx_scales = factx.vtcm_hmx_scales_id; + ou_job.n_row_tiles = n_row_tiles; + ou_job.n_col_tiles = n_col_tiles; + ou_job.n_row_tiles_g_br = n_row_tiles_g_br; + ou_job.n_tiles_per_bc = n_tiles_per_bc; + ou_job.DV = DV_pad; + + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job)); + if (step + 1 == n_active_blocks) { + // Overlapped: run HVX build diag inv L while HMX is busy executing the update + htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); + fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br, kv_head, n_rows_g); + htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start); + } + hmx_queue_pop(ctx->hmx_queue); + + hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev); + } + + buf_idx = 1 - buf_idx; + } + } + + // Enqueue DMAs for the next kv_head early so they overlap with O-PROC + const uint32_t next_kv_head = kv_head + 1; + const bool has_next = (next_kv_head < kv_head_max); + + if (has_next) { + const dma_addr_t next_q_ptr = q->data + q_start * q->nb[1] + (next_kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3]; + const size_t next_q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed; + const size_t next_n_rows = q_transposed ? factx.G : n_rows_q; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_q_dma, next_q_ptr), next_q_row_bytes, hex_smax(q_src_stride, next_q_row_bytes), next_q_row_bytes, next_n_rows); + + if (n_active_blocks > 0) { + const uint32_t next_ik2 = next_kv_head; + const uint32_t next_iv2 = next_kv_head; + + const dma_addr_t next_k_src = k->data + b0_start * k->nb[1] + next_ik2 * k->nb[2] + ik3 * k->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_k_fp16[0], next_k_src), size_k_row_padded, k->nb[1], size_k_row, b0_rows); + + const dma_addr_t next_v_src = v->data + b0_start * v->nb[1] + next_iv2 * v->nb[2] + iv3 * v->nb[3]; + dma_queue_push(dma_q, dma_make_data(factx.vtcm_v_fp16[0], next_v_src), size_v_row_padded, v->nb[1], size_v_row, b0_rows); + + if (factx.pipeline && mask) { + if (__builtin_expect(factx.mask_broadcast, true)) { + const dma_addr_t ms_src = mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + b0_start * sizeof(__fp16); + dma_cache_fa_push(dma_q, factx.m_cache, ms_src, m_line_bytes, mask->nb[1], b0_rows * sizeof(__fp16), n_rows_q); + } else { + fa_push_mask_dma_gqa(dma_q, mask, q_start, im3, b0_start, next_kv_head, G, m_line_bytes, b0_rows, n_rows_q, &factx); + } + } + } + } + + // ---- Final normalization ---- + { + on_job.o_curr = o_tile_curr; + on_job.o_prev = o_tile_prev; + on_job.d_tiles = factx.vtcm_d_inv_l; + on_job.hmx_scales = factx.vtcm_hmx_scales_id; + on_job.n_row_tiles = n_row_tiles; + on_job.n_row_tiles_g_br = n_row_tiles_g_br; + on_job.DV = DV_pad; + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_norm_worker, &on_job)); + hmx_queue_pop(ctx->hmx_queue); + } + + // ---- Store O block ---- + fa_phase_o_store(&factx, dst, o_tile_curr, q_start, kv_head, ib3, n_rows_g); + } } } + + return HTP_STATUS_OK; } int op_flash_attn_ext(struct htp_ops_context * octx) { @@ -617,99 +2665,164 @@ int op_flash_attn_ext(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; // Check support - if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) || k->type != HTP_TYPE_F16 || v->type != HTP_TYPE_F16) { + if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) || + (k->type != HTP_TYPE_F16 && k->type != HTP_TYPE_Q8_0) || + (v->type != HTP_TYPE_F16 && v->type != HTP_TYPE_Q8_0)) { return HTP_STATUS_NO_SUPPORT; } - struct htp_fa_context factx; - factx.octx = octx; + if (htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } - factx.t_start = HAP_perf_get_qtimer_count(); + const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params; - factx.src0_div21 = init_fastdiv_values(q->ne[2] * q->ne[1]); - factx.src0_div1 = init_fastdiv_values(q->ne[1]); + if (kparams->kernel_type == HTP_FA_KERNEL_UNSUPPORTED || kparams->n_kv_blocks > FA_MAX_KV_BLOCKS) { + return HTP_STATUS_NO_SUPPORT; + } - factx.broadcast_rk2 = init_fastdiv_values(q->ne[2]/k->ne[2]); - factx.broadcast_rk3 = init_fastdiv_values(q->ne[3]/k->ne[3]); - factx.broadcast_rv2 = init_fastdiv_values(q->ne[2]/v->ne[2]); - factx.broadcast_rv3 = init_fastdiv_values(q->ne[3]/v->ne[3]); + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } - if (mask) { - factx.src3_div2 = init_fastdiv_values(mask->ne[2]); - factx.src3_div3 = init_fastdiv_values(mask->ne[3]); + if (kparams->kernel_type == HTP_FA_KERNEL_HMX) { + return hmx_flash_attn_ext(octx); } - factx.is_q_fp32 = (q->type == HTP_TYPE_F32); - factx.size_q_row_padded = hex_round_up(q->ne[0] * (factx.is_q_fp32 ? 4 : 2), 128); - factx.size_k_row_padded = hex_round_up(k->ne[0] * sizeof(__fp16), 128); - factx.size_v_row_padded = hex_round_up(v->ne[0] * sizeof(__fp16), 128); + struct htp_fa_context factx; + factx.octx = octx; + factx.k = k; + factx.v = v; - size_t size_q_block = factx.size_q_row_padded * 1; // single row for now - factx.size_k_block = factx.size_k_row_padded * FLASH_ATTN_BLOCK_SIZE; - factx.size_v_block = factx.size_v_row_padded * FLASH_ATTN_BLOCK_SIZE; - factx.size_m_block = hex_round_up(FLASH_ATTN_BLOCK_SIZE * sizeof(__fp16), 128); + factx.src0_div21 = kparams->u.hvx.src0_div21; + factx.src0_div1 = kparams->u.hvx.src0_div1; - factx.n_blocks = (k->ne[1] + FLASH_ATTN_BLOCK_SIZE - 1) / FLASH_ATTN_BLOCK_SIZE; + factx.broadcast_rk2 = kparams->broadcast_rk2; + factx.broadcast_rk3 = kparams->broadcast_rk3; + factx.broadcast_rv2 = kparams->broadcast_rv2; + factx.broadcast_rv3 = kparams->broadcast_rv3; - float scale = 1.0f; - float max_bias = 0.0f; - float logit_softcap = 0.0f; + if (mask) { + factx.src3_div2 = kparams->src3_div2; + factx.src3_div3 = kparams->src3_div3; + } - memcpy(&scale, (float *) octx->op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) octx->op_params + 1, sizeof(float)); - memcpy(&logit_softcap, (float *) octx->op_params + 2, sizeof(float)); + factx.is_q_fp32 = (q->type == HTP_TYPE_F32); + factx.size_q_row_padded = kparams->u.hvx.size_q_row_padded; + factx.size_k_row_padded = kparams->u.hvx.size_k_row_padded; + factx.size_v_row_padded = kparams->u.hvx.size_v_row_padded; + + factx.n_blocks = kparams->n_kv_blocks; + + factx.scale = kparams->scale; + factx.max_bias = kparams->max_bias; + factx.has_softcap = (kparams->logit_softcap != 0.0f); + factx.logit_softcap = 0; + if (factx.has_softcap) { + factx.logit_softcap = (__fp16) kparams->logit_softcap; + } - if (logit_softcap != 0.0f) { - scale /= logit_softcap; + factx.n_head_log2 = kparams->n_head_log2; + factx.m0 = kparams->m0; + factx.m1 = kparams->m1; + + const uint32_t n_head = q->ne[2]; + if (n_head > 512) { + return HTP_STATUS_NO_SUPPORT; } - factx.scale = scale; - factx.max_bias = max_bias; - factx.logit_softcap = logit_softcap; + factx.ddr = (struct hvx_fa_ddr_spad *) fa_get_ddr_spad(octx, sizeof(struct hvx_fa_ddr_spad)); + if (!factx.ddr) { + return HTP_STATUS_INTERNAL_ERR; + } + factx.m_cache = octx->dma_cache_dm; - uint32_t n_head = q->ne[2]; - factx.n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); - factx.m0 = powf(2.0f, -(max_bias ) / factx.n_head_log2); - factx.m1 = powf(2.0f, -(max_bias / 2.0f) / factx.n_head_log2); + for (uint32_t h = 0; h < n_head; ++h) { + factx.ddr->slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f); + } // total rows in q - const uint32_t neq0 = q->ne[0]; const uint32_t neq1 = q->ne[1]; const uint32_t neq2 = q->ne[2]; const uint32_t neq3 = q->ne[3]; + const uint32_t total_qrows = neq1 * neq2 * neq3; + + uint32_t qrow_start = 0; + uint32_t qrows = total_qrows; + + if (octx->ctx->mdev.count > 1) { + const uint32_t mdev_count = octx->ctx->mdev.count; + const uint32_t mdev_idx = octx->ctx->mdev.idx; + const uint32_t n_kv_heads = k->ne[2]; + const uint32_t dst_e_size = (dst->type == HTP_TYPE_F32) ? sizeof(float) : sizeof(__fp16); + const bool can_split = htp_tensor_can_row_partition(dst, dst_e_size); + + // head range is contiguous in flat row space only when neq3 == 1 + if (kparams->head_split && can_split && neq3 == 1 && n_kv_heads >= mdev_count && n_kv_heads % mdev_count == 0) { + const uint32_t G = kparams->G; + const uint32_t kv_per_core = n_kv_heads / mdev_count; + const uint32_t heads_per_core = kv_per_core * G; + const uint32_t head_start = mdev_idx * heads_per_core; + qrow_start = head_start * neq1; + qrows = heads_per_core * neq1; + } else { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_qrows, can_split ? 1 : 0, mdev_idx, mdev_count, &octx->ctx->mdev.count_div); + qrow_start = range.start; + qrows = range.count; + } + } - factx.qrows = neq1*neq2*neq3; - factx.qrows_per_thread = (factx.qrows + octx->n_threads - 1) / octx->n_threads; - - size_t size_vkq_acc = hex_round_up(v->ne[0] * sizeof(float), 128); // VKQ32 + if (qrows == 0) { + return HTP_STATUS_OK; + } - octx->src0_spad.size_per_thread = size_q_block * 1; - octx->src1_spad.size_per_thread = factx.size_k_block * 2; - octx->src2_spad.size_per_thread = factx.size_v_block * 2; - octx->src3_spad.size_per_thread = mask ? factx.size_m_block * DMA_CACHE_MAX_SIZE : 0; - octx->dst_spad.size_per_thread = size_vkq_acc; + const uint32_t n_threads = octx->n_threads; - octx->src0_spad.size = octx->src0_spad.size_per_thread * octx->n_threads; - octx->src1_spad.size = octx->src1_spad.size_per_thread * octx->n_threads; - octx->src2_spad.size = octx->src2_spad.size_per_thread * octx->n_threads; - octx->src3_spad.size = octx->src3_spad.size_per_thread * octx->n_threads; - octx->dst_spad.size = octx->dst_spad.size_per_thread * octx->n_threads; + factx.qrows = qrows; + factx.qrow_start = qrow_start; + factx.qrows_per_thread = fastdiv(qrows + n_threads - 1, &octx->n_threads_div); - size_t total_spad = octx->src0_spad.size + octx->src1_spad.size + octx->src2_spad.size + octx->src3_spad.size + octx->dst_spad.size; + const bool has_mask = (mask != NULL); + const bool has_sinks = (octx->src[4] != NULL); + struct hvx_fa_vtcm_layout L; + hvx_fa_vtcm_layout_build(&L, k->ne[0], v->ne[0], factx.is_q_fp32, has_mask, has_sinks, n_head, n_threads); - if (octx->ctx->vtcm_size < total_spad) { + if (L.total_bytes > octx->ctx->vtcm_size) { return HTP_STATUS_VTCM_TOO_SMALL; } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL; - octx->src2_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->src2_spad.src = NULL; - octx->src3_spad.data = octx->src2_spad.data + octx->src2_spad.size; octx->src3_spad.src = NULL; - octx->dst_spad.data = octx->src3_spad.data + octx->src3_spad.size; octx->dst_spad.src = NULL; + factx.size_q_block = L.size_q_block; + factx.size_k_block = L.size_k_block; + factx.size_v_block = L.size_v_block; + factx.size_m_block = L.size_m_block; + factx.size_vkq_acc = L.size_vkq_acc; + + uint8_t * const base = octx->ctx->vtcm_base; + + factx.spad_q = VTCM_LAYOUT_PTR(uint8_t, base, L.off_q); + factx.spad_k = VTCM_LAYOUT_PTR(uint8_t, base, L.off_k); + factx.spad_v = VTCM_LAYOUT_PTR(uint8_t, base, L.off_v); + factx.spad_m = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_m, has_mask); + factx.spad_a = VTCM_LAYOUT_PTR(uint8_t, base, L.off_a); + factx.spad_sinks = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_sinks, has_sinks); + factx.n_active_blocks = factx.n_blocks; + factx.shared_active_blocks = false; + + if (has_mask && neq1 == 1 && neq3 == 1 && mask->ne[2] == 1 && factx.n_blocks > 1 && factx.n_blocks <= FA_MAX_KV_BLOCKS) { + factx.n_active_blocks = fa_scan_active_blocks( + octx->ctx->dma[0], mask, 0, 1, 0, 0, k->ne[1], FLASH_ATTN_BLOCK_SIZE, + factx.n_blocks, factx.spad_k, factx.size_k_block * 2, factx.ddr->active_blocks); + factx.shared_active_blocks = true; + } - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - worker_pool_run_func(octx->ctx->worker_pool, flash_attn_ext_f16_thread, &factx, octx->n_threads); + if (has_sinks) { + const struct htp_tensor * sinks = octx->src[4]; + dma_queue * dma_q = octx->ctx->dma[0]; + dma_queue_push(dma_q, dma_make_data(factx.spad_sinks, sinks->data), L.size_sinks, 0, sinks->size, 1); + dma_queue_pop(dma_q); } + work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, n_threads); + return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/flash-attn-ops.h b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h new file mode 100644 index 0000000..f2654f0 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/flash-attn-ops.h @@ -0,0 +1,410 @@ +#ifndef HTP_FLASH_ATTN_OPS_H +#define HTP_FLASH_ATTN_OPS_H + +#include +#include +#include + +#include "hex-fastdiv.h" +#include "hex-common.h" +#include "htp-vtcm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Tile constants (mirrored from hmx-utils.h for use on host side if needed) +#define HTP_FA_HMX_TILE_SIZE 2048 +#define HMX_FP16_TILE_SIZE 2048 +#define HMX_FP16_TILE_N_ROWS 32 +#define HMX_FP16_TILE_N_COLS 32 +#define HMX_FP16_TILE_N_ELMS 1024 + +#define HVX_FA_DMA_CACHE_SIZE 128 +#define HMX_FA_DMA_CACHE_SIZE 4 + + +#define HTP_FA_M_INITIAL_VAL -10000.0f + +enum htp_fa_kernel_type { + HTP_FA_KERNEL_UNSUPPORTED = 0, + HTP_FA_KERNEL_HVX, + HTP_FA_KERNEL_HMX +}; + +// Multicore: gcd(n_kv_heads, N) teams own the KV heads, team members share the Q-blocks +#define HTP_FA_FLAG_TEAM_SPLIT 0x01 + +static inline void htp_fa_mdev_teams(uint32_t n_kv_heads, uint32_t mdev_count, bool head_split, + uint32_t * n_teams, uint32_t * team_size) { + const uint32_t t = (head_split && n_kv_heads > 0 && mdev_count > 1) ? hex_gcd_u32(n_kv_heads, mdev_count) : 1; + *n_teams = t; + *team_size = mdev_count > 1 ? mdev_count / t : 1; +} + +struct htp_fa_kernel_params { + uint8_t kernel_type; // enum htp_fa_kernel_type + uint8_t head_split; // 1 = partition by KV heads in multicore, 0 = token partition + uint8_t flags; // HTP_FA_FLAG_* + uint8_t n_threads; // Number of threads to run + + // Common parameters + uint16_t Br; + uint16_t Bc; + uint16_t n_kv_blocks; // also HVX's n_blocks + uint16_t G; // GQA factor (n_heads / n_kv_heads) + + float scale; + float max_bias; + float logit_softcap; + uint32_t vtcm_size; + + float m0; + float m1; + uint32_t n_head_log2; + + struct fastdiv_values src3_div2; + struct fastdiv_values src3_div3; + + struct fastdiv_values broadcast_rk2; + struct fastdiv_values broadcast_rk3; + struct fastdiv_values broadcast_rv2; + struct fastdiv_values broadcast_rv3; + + union { + struct { + uint32_t g_br; + int32_t mask_broadcast; + int32_t pipeline; + struct fastdiv_values div_G; + } hmx; + struct { + uint32_t size_q_row_padded; + uint32_t size_k_row_padded; + uint32_t size_v_row_padded; + struct fastdiv_values src0_div21; + struct fastdiv_values src0_div1; + } hvx; + } u; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob"); +#endif + +// VTCM region layout for the HMX flash-attention kernel. +// +// Single source of truth for both the host (which needs the total size to pick a +// (Br, Bc) tiling that fits the VTCM budget) and the device (which needs the actual +// byte offsets to place each scratch buffer). Building the layout once and reading +// offsets/total from it makes host estimate and device allocation impossible to +// desync -- previously they were duplicated formulas in two files and drifted. +// +// All fields are byte offsets / byte sizes -- no HVX_Vector type is named here so the +// header stays host-includable. The device casts (base + off_*) to the proper type. +// An offset of 0 marks a region that is not allocated for this configuration (only +// off_v_tiles[1], which exists only when pipelining); the device sets such pointers NULL. +struct hmx_fa_vtcm_layout { + // Byte offsets from vtcm_base for each region. + size_t off_q_tiles; + size_t off_q_dma; + size_t off_o_tiles[2]; + size_t off_k_fp16[2]; + size_t off_v_fp16[2]; + size_t off_k_tiles[2]; + size_t off_v_tiles[2]; + size_t off_s_tiles[2]; + size_t off_p_tiles[2]; + size_t off_d_tiles[2]; + size_t off_d_inv_l; + size_t off_m_vec; + size_t off_l_vec; + size_t off_s_rowmax; + size_t off_p_rowsum; + size_t off_row_bufs; + size_t off_hmx_scales_id; + size_t off_hmx_scales_qk; + size_t off_mask_buf; + size_t off_slopes; + size_t off_sinks; + + // Region byte sizes reused by the device at runtime (not just for allocation). + size_t q_tile_bytes; + size_t o_tile_bytes; + size_t k_tile_bytes; + size_t s_tile_bytes; // S and P tiles (same size) + size_t d_tile_bytes; // d_tiles[0..1] + d_inv_l, allocated back to back + size_t m_line_bytes; // one mask row + size_t m_buf_slot_bytes; // one dma_cache slot = align_up(Br * m_line_bytes, 4096) + size_t col_vec_bytes; + size_t sinks_bytes; + + // Derived strides. + size_t row_buf_stride; // HVX vectors (128B) per row buffer + size_t mask_buf_row_stride; // __fp16 elements per row in the mask buffer + + bool pipeline; + size_t total_bytes; +}; + +#define FA_MAX_KV_BLOCKS 1024 + +// Build the VTCM layout. + +static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L, + size_t gqa_factor, size_t DK, size_t DV, + size_t Br, size_t Bc, size_t n_threads, + bool pipeline, bool is_q_fp32, + bool has_sinks, size_t n_heads) { + const size_t g_br = hex_align_up(gqa_factor * Br, HMX_FP16_TILE_N_ROWS); + const size_t q_tile_size = hex_align_up(g_br * DK * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + const size_t o_tile_size = hex_align_up(g_br * DV * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + const size_t k_tile_size = hex_align_up(Bc * DK * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + const size_t v_tile_size = hex_align_up(Bc * DV * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + const size_t s_tile_size = hex_align_up(g_br * Bc * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE); + + // The rescale matrices are diagonal: the HMX kernels only ever load the g_br/32 + // tiles that sit on the diagonal, so store just those, packed back to back with + // a stride of one tile. The old [g_br, g_br] square layout allocated g_br/32 + // times more than it used, which is also why a second D buffer was unaffordable. + const size_t d_tile_size = (g_br / HMX_FP16_TILE_N_ROWS) * HTP_FA_HMX_TILE_SIZE; + + const size_t q_dma_size = hex_align_up(g_br * DK * (is_q_fp32 ? sizeof(float) : sizeof(__fp16)), 128); + const size_t k_dma_size = hex_align_up(Bc * hex_round_up(DK * sizeof(__fp16), 128), 128); + const size_t v_dma_size = hex_align_up(Bc * hex_round_up(DV * sizeof(__fp16), 128), 128); + const size_t col_vec_size = hex_align_up(g_br * sizeof(float), 256); + const size_t row_vec_size = hex_align_up(Bc * sizeof(__fp16), 256); + const size_t m_line_size = hex_align_up(Bc * sizeof(__fp16), 128); + const size_t m_buf_slot = hex_align_up(Br * m_line_size, 256); + const size_t m_buf_size = m_buf_slot * HMX_FA_DMA_CACHE_SIZE; + const size_t slopes_size = hex_align_up(g_br * sizeof(__fp16), 128); + const size_t sinks_size = hex_round_up(n_heads * sizeof(float), 128); + + size_t off = 0; + + // Group A (Part 1 - HMX Tiled buffers) + VTCM_LAYOUT_ALLOC(off, off_q_tiles, q_tile_size); + VTCM_LAYOUT_ALLOC(off, off_o_tiles[0], o_tile_size); + VTCM_LAYOUT_ALLOC(off, off_o_tiles[1], o_tile_size); + VTCM_LAYOUT_ALLOC(off, off_d_tiles[0], d_tile_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_d_tiles[1], d_tile_size, pipeline); + VTCM_LAYOUT_ALLOC(off, off_d_inv_l, d_tile_size); + + // Group B & C share start offset (Group B tiles must be 2KB aligned) + size_t off_group_b_c = hex_align_up(off, HTP_FA_HMX_TILE_SIZE); + + // Group B: Compute-only buffers + size_t off_group_b = off_group_b_c; + VTCM_LAYOUT_ALLOC(off_group_b, off_k_tiles[0], k_tile_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_k_tiles[1], k_tile_size, pipeline); + VTCM_LAYOUT_ALLOC(off_group_b, off_v_tiles[0], v_tile_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_v_tiles[1], v_tile_size, pipeline); + VTCM_LAYOUT_ALLOC(off_group_b, off_s_tiles[0], s_tile_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_s_tiles[1], s_tile_size, pipeline); + VTCM_LAYOUT_ALLOC(off_group_b, off_p_tiles[0], s_tile_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_p_tiles[1], s_tile_size, pipeline); + VTCM_LAYOUT_ALLOC(off_group_b, off_s_rowmax, col_vec_size); + VTCM_LAYOUT_ALLOC(off_group_b, off_p_rowsum, col_vec_size); + VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_threads); + + const size_t group_b_size = off_group_b - off_group_b_c; + + // Group C: Q fetch DMA buffer + size_t off_group_c = off_group_b_c; + VTCM_LAYOUT_ALLOC(off_group_c, off_q_dma, q_dma_size); + + const size_t group_c_size = off_group_c - off_group_b_c; + + off = off_group_b_c + hex_smax(group_b_size, group_c_size); + + // Group A (Part 2 - remaining non-HMX buffers) + VTCM_LAYOUT_ALLOC(off, off_k_fp16[0], k_dma_size); + VTCM_LAYOUT_ALLOC(off, off_k_fp16[1], k_dma_size); + VTCM_LAYOUT_ALLOC(off, off_v_fp16[0], v_dma_size); + VTCM_LAYOUT_ALLOC(off, off_v_fp16[1], v_dma_size); + VTCM_LAYOUT_ALLOC(off, off_m_vec, col_vec_size); + VTCM_LAYOUT_ALLOC(off, off_l_vec, col_vec_size); + VTCM_LAYOUT_ALLOC(off, off_hmx_scales_id, 256); + VTCM_LAYOUT_ALLOC(off, off_hmx_scales_qk, 256); + VTCM_LAYOUT_ALLOC(off, off_mask_buf, m_buf_size); + VTCM_LAYOUT_ALLOC(off, off_slopes, slopes_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, sinks_size, has_sinks); + + L->q_tile_bytes = q_tile_size; + L->o_tile_bytes = o_tile_size; + L->k_tile_bytes = k_tile_size; + L->col_vec_bytes = col_vec_size; + L->s_tile_bytes = s_tile_size; + // Measured from the actual offsets rather than assumed to be N * d_tile_size, so + // that inserting a region between them (or adding padding to VTCM_LAYOUT_ALLOC) + // cannot silently leave the tail of the run unzeroed. + L->d_tile_bytes = (L->off_d_inv_l + d_tile_size) - L->off_d_tiles[0]; + L->m_line_bytes = m_line_size; + L->m_buf_slot_bytes = m_buf_slot; + L->row_buf_stride = row_vec_size / 128; + L->mask_buf_row_stride = m_line_size / sizeof(__fp16); + L->sinks_bytes = has_sinks ? sinks_size : 0; + L->pipeline = pipeline; + L->total_bytes = off; +} + +// Exact VTCM usage for a given (gqa_factor, DK, DV, Br, Bc) configuration. +static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) { + struct hmx_fa_vtcm_layout L; + hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_threads, pipeline, is_q_fp32, has_sinks, n_heads); + return L.total_bytes; +} + +#define FA_HVX_BLOCK_SIZE 64 +#define FA_HVX_G_MAX 8 + +struct hvx_fa_vtcm_layout { + size_t off_q; + size_t off_k; + size_t off_v; + size_t off_m; + size_t off_a; + size_t off_sinks; + + size_t size_q_block; + size_t size_k_block; + size_t size_v_block; + size_t size_m_block; + size_t size_vkq_acc; + size_t size_sinks; + + size_t total_bytes; +}; + +static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L, + size_t DK, size_t DV, + bool is_q_fp32, bool has_mask, + bool has_sinks, size_t n_heads, + size_t n_threads) { + const size_t size_q_row_padded = hex_round_up(DK * (is_q_fp32 ? 4 : 2), 128); + const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128); + const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128); + + const size_t size_q_block = size_q_row_padded * FA_HVX_G_MAX; + const size_t size_k_block = size_k_row_padded * FA_HVX_BLOCK_SIZE; + const size_t size_v_block = size_v_row_padded * FA_HVX_BLOCK_SIZE; + const size_t size_m_block = hex_round_up(FA_HVX_BLOCK_SIZE * sizeof(__fp16), 128); + const size_t size_vkq_acc = hex_round_up(DV * sizeof(float), 128) * FA_HVX_G_MAX; + const size_t size_sinks = hex_round_up(n_heads * sizeof(float), 128); + + size_t off = 0; + + VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_threads); + VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_threads); + VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_threads); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_threads, has_mask); + VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_threads); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, size_sinks, has_sinks); + + L->size_q_block = size_q_block; + L->size_k_block = size_k_block; + L->size_v_block = size_v_block; + L->size_m_block = size_m_block; + L->size_vkq_acc = size_vkq_acc; + L->size_sinks = has_sinks ? size_sinks : 0; + L->total_bytes = off; +} + +static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_threads) { + struct hvx_fa_vtcm_layout L; + hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_threads); + return L.total_bytes; +} + +#define FA_MIN_KV_BLOCKS 3 + +// Cost-based (Br, Bc) search for flash attention with pipeline constraint. +static inline int hmx_fa_find_chunk_size(size_t * Br_out, + size_t * Bc_out, + size_t gqa_factor, + size_t DK, + size_t DV, + size_t qo_len, + size_t kv_len, + size_t vtcm_budget, + size_t n_threads, + bool is_q_fp32, + bool has_sinks, + size_t n_heads, + size_t q_split) { + const size_t T = HMX_FP16_TILE_N_ROWS; // 32 + const size_t br_unit = hmx_ceil_div(T, gqa_factor); + const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2; // 64 + const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_threads >= 2); + + // Br_max: largest Br aligned to br_unit that does not exceed qo_len. + const size_t Br_max = qo_len >= br_unit ? hex_align_down(qo_len, br_unit) : br_unit; + + // Pipeline constraint: cap Bc so n_kv_blocks >= FA_MIN_KV_BLOCKS. + // Only relax when kv_len is too short to form enough blocks. + size_t Bc_limit = can_pipeline ? hex_align_down(kv_len / FA_MIN_KV_BLOCKS, bc_unit) : + (kv_len >= bc_unit ? hex_align_down(kv_len, bc_unit) : bc_unit); + // For small-Q SWA-sized KV caches (qo_len <= 16, kv_len <= 2048), cap Bc at 256 for block-skipping granularity + if (qo_len <= 16 && kv_len <= 2048 && Bc_limit > 256) { + Bc_limit = 256; + } + // Cost coefficients calibrated from profiling + const size_t c_q_fixed = 800; // per-Q-block: q_load + epilogue o_update + o_norm + o_store + const size_t c_iter_base = 200; // per-KV-iter base (HMX dot/update + DMA) + const size_t c_softmax = 600; // per 64-row vector chunk on HVX + + size_t best_cost = SIZE_MAX, best_mn = 0; + size_t best_Br = 0, best_Bc = 0; + + for (size_t Br = Br_max; Br >= br_unit; Br -= br_unit) { + // Try all Bc candidates from Bc_limit down to bc_unit + for (size_t Bc = Bc_limit; Bc >= bc_unit; Bc -= bc_unit) { + size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline, is_q_fp32, has_sinks, n_heads); + if (vtcm_needed <= vtcm_budget) { + // This Bc fits for this Br! + const size_t q_blocks = (qo_len + Br - 1) / Br; + const size_t kv_blocks = (kv_len + Bc - 1) / Bc; + const size_t actual_threads = (kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1; + const size_t n_rows_g = Br * gqa_factor; + const size_t n_row_vec_cnt = (n_rows_g + 63) / 64; + const size_t n_use = n_row_vec_cnt < actual_threads ? n_row_vec_cnt : actual_threads; + const size_t vecs_per_t = n_use > 0 ? (n_row_vec_cnt + n_use - 1) / n_use : 1; + + const size_t c_iter_actual = c_iter_base + c_softmax * vecs_per_t; + const size_t q_blocks_nsp = (q_blocks + q_split - 1) / (q_split ? q_split : 1); + const size_t cost = q_blocks_nsp * (c_q_fixed + kv_blocks * c_iter_actual); + const size_t mn = Br * Bc; + + if (cost < best_cost || (cost == best_cost && mn > best_mn)) { + best_cost = cost; + best_mn = mn; + best_Br = Br; + best_Bc = Bc; + } + // Since we iterate Bc from largest to smallest, this is the largest Bc that fits + // for this Br. We can break to the next Br. + break; + } + } + + if (Br == br_unit) { + break; + } + } + + if (best_Br == 0 || best_Bc == 0) { + return -1; + } + + *Br_out = best_Br; + *Bc_out = best_Bc; + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif /* HTP_FLASH_ATTN_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c new file mode 100644 index 0000000..5252ac3 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c @@ -0,0 +1,2357 @@ +#include +#include +#include +#include +#include + +#include "hvx-base.h" +#include "hvx-copy.h" +#include "hvx-reduce.h" +#include "hvx-exp.h" +#include "dma-queue.h" +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "hmx-utils.h" +#include "hmx-fa-kernels.h" +#include "hmx-queue.h" +#include "gated-delta-net-ops.h" + +#ifndef MIN +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +struct htp_gdn_context { + struct htp_ops_context * octx; + const struct htp_gdn_kernel_params * kparams; + struct htp_gdn_vtcm_layout layout; + uint8_t * vtcm_base; + uint32_t row_start; + uint32_t nrows; +}; + +static inline HVX_Vector gdn_mul_dot_f32(float * restrict dst, const HVX_Vector * restrict mul, const HVX_Vector * restrict dot, uint32_t n) { + HVX_Vector acc = Q6_V_vzero(); + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + for (uint32_t i = 0; i < nvec; ++i) { + HVX_Vector vd = hvx_vmemu(dst + i * epv); + HVX_Vector vm = mul[i]; + HVX_Vector vdot = dot[i]; + HVX_Vector out = hvx_vec_mul_f32_f32(vd, vm); + hvx_vmemu(dst + i * epv) = out; + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(out, vdot)); + } + + if (nloe) { + const uint32_t off = nvec * epv; + HVX_Vector vm = mul[nvec]; + HVX_Vector vdot = dot[nvec]; + HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out = hvx_vec_mul_f32_f32(hvx_vmemu(dst + off), vm); + hvx_vec_store_u(dst + off, nloe * sizeof(float), out); + acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out, vdot), zero)); + } + + return hvx_vec_reduce_sum_f32(acc); +} + +static inline HVX_Vector gdn_mul_scalar_dot_f32(float * restrict dst, HVX_Vector vmul, const HVX_Vector * restrict dot, uint32_t n) { + HVX_Vector acc = Q6_V_vzero(); + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + for (uint32_t i = 0; i < nvec; ++i) { + HVX_Vector vd = hvx_vmemu(dst + i * epv); + HVX_Vector vdot = dot[i]; + HVX_Vector out = hvx_vec_mul_f32_f32(vd, vmul); + hvx_vmemu(dst + i * epv) = out; + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(out, vdot)); + } + + if (nloe) { + const uint32_t off = nvec * epv; + HVX_Vector vdot = dot[nvec]; + HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out = hvx_vec_mul_f32_f32(hvx_vmemu(dst + off), vmul); + hvx_vec_store_u(dst + off, nloe * sizeof(float), out); + acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out, vdot), zero)); + } + + return hvx_vec_reduce_sum_f32(acc); +} + +static inline HVX_Vector gdn_add_scaled_dot_f32(float * restrict dst, const HVX_Vector * restrict src, + HVX_Vector vscale, const HVX_Vector * restrict dot, uint32_t n) { + HVX_Vector acc = Q6_V_vzero(); + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + for (uint32_t i = 0; i < nvec; ++i) { + HVX_Vector vd = hvx_vmemu(dst + i * epv); + HVX_Vector vs = src[i]; + HVX_Vector vdot = dot[i]; + HVX_Vector out = hvx_vec_add_f32_f32(vd, hvx_vec_mul_f32_f32(vs, vscale)); + hvx_vmemu(dst + i * epv) = out; + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(out, vdot)); + } + + if (nloe) { + const uint32_t off = nvec * epv; + HVX_Vector vs = src[nvec]; + HVX_Vector vdot = dot[nvec]; + HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out = hvx_vec_add_f32_f32(hvx_vmemu(dst + off), hvx_vec_mul_f32_f32(vs, vscale)); + hvx_vec_store_u(dst + off, nloe * sizeof(float), out); + acc = hvx_vec_add_f32_f32(acc, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out, vdot), zero)); + } + + return hvx_vec_reduce_sum_f32(acc); +} + +static inline HVX_Vector gdn_mul_dot4_f32(float * restrict dst0, float * restrict dst1, + float * restrict dst2, float * restrict dst3, + const HVX_Vector * restrict mul, const HVX_Vector * restrict dot, uint32_t n) { + HVX_Vector acc0 = Q6_V_vzero(); + HVX_Vector acc1 = Q6_V_vzero(); + HVX_Vector acc2 = Q6_V_vzero(); + HVX_Vector acc3 = Q6_V_vzero(); + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + for (uint32_t i = 0; i < nvec; ++i) { + HVX_Vector vm = mul[i]; + HVX_Vector vdot = dot[i]; + + HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vm); + HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vm); + HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + i * epv), vm); + HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + i * epv), vm); + + hvx_vmemu(dst0 + i * epv) = out0; + hvx_vmemu(dst1 + i * epv) = out1; + hvx_vmemu(dst2 + i * epv) = out2; + hvx_vmemu(dst3 + i * epv) = out3; + + acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot)); + acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot)); + acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot)); + acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot)); + } + + if (nloe) { + const uint32_t off = nvec * epv; + HVX_Vector vm = mul[nvec]; + HVX_Vector vdot = dot[nvec]; + HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + off), vm); + HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + off), vm); + HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + off), vm); + HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + off), vm); + + hvx_vec_store_u(dst0 + off, nloe * sizeof(float), out0); + hvx_vec_store_u(dst1 + off, nloe * sizeof(float), out1); + hvx_vec_store_u(dst2 + off, nloe * sizeof(float), out2); + hvx_vec_store_u(dst3 + off, nloe * sizeof(float), out3); + + acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero)); + acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero)); + acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero)); + acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero)); + } + + HVX_Vector_x4 acc = { .v = { acc0, acc1, acc2, acc3 } }; + return hvx_vec_reduce_sum_f32x4(acc); +} + +static inline HVX_Vector gdn_mul_scalar_dot4_f32(float * restrict dst0, float * restrict dst1, + float * restrict dst2, float * restrict dst3, + HVX_Vector vmul, const HVX_Vector * restrict dot, uint32_t n) { + HVX_Vector acc0 = Q6_V_vzero(); + HVX_Vector acc1 = Q6_V_vzero(); + HVX_Vector acc2 = Q6_V_vzero(); + HVX_Vector acc3 = Q6_V_vzero(); + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + for (uint32_t i = 0; i < nvec; ++i) { + HVX_Vector vdot = dot[i]; + + HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vmul); + HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vmul); + HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + i * epv), vmul); + HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + i * epv), vmul); + + hvx_vmemu(dst0 + i * epv) = out0; + hvx_vmemu(dst1 + i * epv) = out1; + hvx_vmemu(dst2 + i * epv) = out2; + hvx_vmemu(dst3 + i * epv) = out3; + + acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot)); + acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot)); + acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot)); + acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot)); + } + + if (nloe) { + const uint32_t off = nvec * epv; + HVX_Vector vdot = dot[nvec]; + HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + off), vmul); + HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + off), vmul); + HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + off), vmul); + HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + off), vmul); + + hvx_vec_store_u(dst0 + off, nloe * sizeof(float), out0); + hvx_vec_store_u(dst1 + off, nloe * sizeof(float), out1); + hvx_vec_store_u(dst2 + off, nloe * sizeof(float), out2); + hvx_vec_store_u(dst3 + off, nloe * sizeof(float), out3); + + acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero)); + acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero)); + acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero)); + acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero)); + } + + HVX_Vector_x4 acc = { .v = { acc0, acc1, acc2, acc3 } }; + return hvx_vec_reduce_sum_f32x4(acc); +} + +static inline HVX_Vector gdn_add_scaled_dot4_f32(float * restrict dst0, float * restrict dst1, + float * restrict dst2, float * restrict dst3, + const HVX_Vector * restrict src, const float * restrict scale, + const HVX_Vector * restrict dot, uint32_t n) { + HVX_Vector acc0 = Q6_V_vzero(); + HVX_Vector acc1 = Q6_V_vzero(); + HVX_Vector acc2 = Q6_V_vzero(); + HVX_Vector acc3 = Q6_V_vzero(); + const HVX_Vector scale0 = hvx_vec_splat_f32(scale[0]); + const HVX_Vector scale1 = hvx_vec_splat_f32(scale[1]); + const HVX_Vector scale2 = hvx_vec_splat_f32(scale[2]); + const HVX_Vector scale3 = hvx_vec_splat_f32(scale[3]); + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + for (uint32_t i = 0; i < nvec; ++i) { + HVX_Vector vs = src[i]; + HVX_Vector vdot = dot[i]; + + HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + i * epv), hvx_vec_mul_f32_f32(vs, scale0)); + HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + i * epv), hvx_vec_mul_f32_f32(vs, scale1)); + HVX_Vector out2 = hvx_vec_add_f32_f32(hvx_vmemu(dst2 + i * epv), hvx_vec_mul_f32_f32(vs, scale2)); + HVX_Vector out3 = hvx_vec_add_f32_f32(hvx_vmemu(dst3 + i * epv), hvx_vec_mul_f32_f32(vs, scale3)); + + hvx_vmemu(dst0 + i * epv) = out0; + hvx_vmemu(dst1 + i * epv) = out1; + hvx_vmemu(dst2 + i * epv) = out2; + hvx_vmemu(dst3 + i * epv) = out3; + + acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot)); + acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot)); + acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot)); + acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot)); + } + + if (nloe) { + const uint32_t off = nvec * epv; + HVX_Vector vs = src[nvec]; + HVX_Vector vdot = dot[nvec]; + HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + off), hvx_vec_mul_f32_f32(vs, scale0)); + HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + off), hvx_vec_mul_f32_f32(vs, scale1)); + HVX_Vector out2 = hvx_vec_add_f32_f32(hvx_vmemu(dst2 + off), hvx_vec_mul_f32_f32(vs, scale2)); + HVX_Vector out3 = hvx_vec_add_f32_f32(hvx_vmemu(dst3 + off), hvx_vec_mul_f32_f32(vs, scale3)); + + hvx_vec_store_u(dst0 + off, nloe * sizeof(float), out0); + hvx_vec_store_u(dst1 + off, nloe * sizeof(float), out1); + hvx_vec_store_u(dst2 + off, nloe * sizeof(float), out2); + hvx_vec_store_u(dst3 + off, nloe * sizeof(float), out3); + + acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero)); + acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero)); + acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero)); + acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero)); + } + + HVX_Vector_x4 acc = { .v = { acc0, acc1, acc2, acc3 } }; + return hvx_vec_reduce_sum_f32x4(acc); +} + +static inline HVX_Vector gdn_mul_dot8_f32(float * restrict dst0, float * restrict dst1, + float * restrict dst2, float * restrict dst3, float * restrict dst4, + float * restrict dst5, float * restrict dst6, float * restrict dst7, + const HVX_Vector * restrict mul, const HVX_Vector * restrict dot, uint32_t n) { + HVX_Vector acc0 = Q6_V_vzero(); + HVX_Vector acc1 = Q6_V_vzero(); + HVX_Vector acc2 = Q6_V_vzero(); + HVX_Vector acc3 = Q6_V_vzero(); + HVX_Vector acc4 = Q6_V_vzero(); + HVX_Vector acc5 = Q6_V_vzero(); + HVX_Vector acc6 = Q6_V_vzero(); + HVX_Vector acc7 = Q6_V_vzero(); + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + for (uint32_t i = 0; i < nvec; ++i) { + HVX_Vector vm = mul[i]; + HVX_Vector vdot = dot[i]; + + HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vm); + HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vm); + HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + i * epv), vm); + HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + i * epv), vm); + HVX_Vector out4 = hvx_vec_mul_f32_f32(hvx_vmemu(dst4 + i * epv), vm); + HVX_Vector out5 = hvx_vec_mul_f32_f32(hvx_vmemu(dst5 + i * epv), vm); + HVX_Vector out6 = hvx_vec_mul_f32_f32(hvx_vmemu(dst6 + i * epv), vm); + HVX_Vector out7 = hvx_vec_mul_f32_f32(hvx_vmemu(dst7 + i * epv), vm); + + hvx_vmemu(dst0 + i * epv) = out0; + hvx_vmemu(dst1 + i * epv) = out1; + hvx_vmemu(dst2 + i * epv) = out2; + hvx_vmemu(dst3 + i * epv) = out3; + hvx_vmemu(dst4 + i * epv) = out4; + hvx_vmemu(dst5 + i * epv) = out5; + hvx_vmemu(dst6 + i * epv) = out6; + hvx_vmemu(dst7 + i * epv) = out7; + + acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot)); + acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot)); + acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot)); + acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot)); + acc4 = hvx_vec_add_f32_f32(acc4, hvx_vec_mul_f32_f32(out4, vdot)); + acc5 = hvx_vec_add_f32_f32(acc5, hvx_vec_mul_f32_f32(out5, vdot)); + acc6 = hvx_vec_add_f32_f32(acc6, hvx_vec_mul_f32_f32(out6, vdot)); + acc7 = hvx_vec_add_f32_f32(acc7, hvx_vec_mul_f32_f32(out7, vdot)); + } + + if (nloe) { + const uint32_t off = nvec * epv; + HVX_Vector vm = mul[nvec]; + HVX_Vector vdot = dot[nvec]; + HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + off), vm); + HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + off), vm); + HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + off), vm); + HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + off), vm); + HVX_Vector out4 = hvx_vec_mul_f32_f32(hvx_vmemu(dst4 + off), vm); + HVX_Vector out5 = hvx_vec_mul_f32_f32(hvx_vmemu(dst5 + off), vm); + HVX_Vector out6 = hvx_vec_mul_f32_f32(hvx_vmemu(dst6 + off), vm); + HVX_Vector out7 = hvx_vec_mul_f32_f32(hvx_vmemu(dst7 + off), vm); + + hvx_vec_store_u(dst0 + off, nloe * sizeof(float), out0); + hvx_vec_store_u(dst1 + off, nloe * sizeof(float), out1); + hvx_vec_store_u(dst2 + off, nloe * sizeof(float), out2); + hvx_vec_store_u(dst3 + off, nloe * sizeof(float), out3); + hvx_vec_store_u(dst4 + off, nloe * sizeof(float), out4); + hvx_vec_store_u(dst5 + off, nloe * sizeof(float), out5); + hvx_vec_store_u(dst6 + off, nloe * sizeof(float), out6); + hvx_vec_store_u(dst7 + off, nloe * sizeof(float), out7); + + acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero)); + acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero)); + acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero)); + acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero)); + acc4 = hvx_vec_add_f32_f32(acc4, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out4, vdot), zero)); + acc5 = hvx_vec_add_f32_f32(acc5, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out5, vdot), zero)); + acc6 = hvx_vec_add_f32_f32(acc6, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out6, vdot), zero)); + acc7 = hvx_vec_add_f32_f32(acc7, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out7, vdot), zero)); + } + + HVX_Vector_x4 accA = { .v = { acc0, acc1, acc2, acc3 } }; + HVX_Vector_x4 accB = { .v = { acc4, acc5, acc6, acc7 } }; + HVX_Vector rA = hvx_vec_reduce_sum_f32x4(accA); + HVX_Vector rB = hvx_vec_reduce_sum_f32x4(accB); + HVX_VectorPred q16 = Q6_Q_vsetq2_R(16); + return Q6_V_vmux_QVV(q16, rA, Q6_V_vror_VR(rB, 128 - 16)); +} + +static inline HVX_Vector gdn_mul_scalar_dot8_f32(float * restrict dst0, float * restrict dst1, + float * restrict dst2, float * restrict dst3, float * restrict dst4, + float * restrict dst5, float * restrict dst6, float * restrict dst7, + HVX_Vector vmul, const HVX_Vector * restrict dot, uint32_t n) { + HVX_Vector acc0 = Q6_V_vzero(); + HVX_Vector acc1 = Q6_V_vzero(); + HVX_Vector acc2 = Q6_V_vzero(); + HVX_Vector acc3 = Q6_V_vzero(); + HVX_Vector acc4 = Q6_V_vzero(); + HVX_Vector acc5 = Q6_V_vzero(); + HVX_Vector acc6 = Q6_V_vzero(); + HVX_Vector acc7 = Q6_V_vzero(); + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + for (uint32_t i = 0; i < nvec; ++i) { + HVX_Vector vdot = dot[i]; + + HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + i * epv), vmul); + HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + i * epv), vmul); + HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + i * epv), vmul); + HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + i * epv), vmul); + HVX_Vector out4 = hvx_vec_mul_f32_f32(hvx_vmemu(dst4 + i * epv), vmul); + HVX_Vector out5 = hvx_vec_mul_f32_f32(hvx_vmemu(dst5 + i * epv), vmul); + HVX_Vector out6 = hvx_vec_mul_f32_f32(hvx_vmemu(dst6 + i * epv), vmul); + HVX_Vector out7 = hvx_vec_mul_f32_f32(hvx_vmemu(dst7 + i * epv), vmul); + + hvx_vmemu(dst0 + i * epv) = out0; + hvx_vmemu(dst1 + i * epv) = out1; + hvx_vmemu(dst2 + i * epv) = out2; + hvx_vmemu(dst3 + i * epv) = out3; + hvx_vmemu(dst4 + i * epv) = out4; + hvx_vmemu(dst5 + i * epv) = out5; + hvx_vmemu(dst6 + i * epv) = out6; + hvx_vmemu(dst7 + i * epv) = out7; + + acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot)); + acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot)); + acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot)); + acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot)); + acc4 = hvx_vec_add_f32_f32(acc4, hvx_vec_mul_f32_f32(out4, vdot)); + acc5 = hvx_vec_add_f32_f32(acc5, hvx_vec_mul_f32_f32(out5, vdot)); + acc6 = hvx_vec_add_f32_f32(acc6, hvx_vec_mul_f32_f32(out6, vdot)); + acc7 = hvx_vec_add_f32_f32(acc7, hvx_vec_mul_f32_f32(out7, vdot)); + } + + if (nloe) { + const uint32_t off = nvec * epv; + HVX_Vector vdot = dot[nvec]; + HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out0 = hvx_vec_mul_f32_f32(hvx_vmemu(dst0 + off), vmul); + HVX_Vector out1 = hvx_vec_mul_f32_f32(hvx_vmemu(dst1 + off), vmul); + HVX_Vector out2 = hvx_vec_mul_f32_f32(hvx_vmemu(dst2 + off), vmul); + HVX_Vector out3 = hvx_vec_mul_f32_f32(hvx_vmemu(dst3 + off), vmul); + HVX_Vector out4 = hvx_vec_mul_f32_f32(hvx_vmemu(dst4 + off), vmul); + HVX_Vector out5 = hvx_vec_mul_f32_f32(hvx_vmemu(dst5 + off), vmul); + HVX_Vector out6 = hvx_vec_mul_f32_f32(hvx_vmemu(dst6 + off), vmul); + HVX_Vector out7 = hvx_vec_mul_f32_f32(hvx_vmemu(dst7 + off), vmul); + + hvx_vec_store_u(dst0 + off, nloe * sizeof(float), out0); + hvx_vec_store_u(dst1 + off, nloe * sizeof(float), out1); + hvx_vec_store_u(dst2 + off, nloe * sizeof(float), out2); + hvx_vec_store_u(dst3 + off, nloe * sizeof(float), out3); + hvx_vec_store_u(dst4 + off, nloe * sizeof(float), out4); + hvx_vec_store_u(dst5 + off, nloe * sizeof(float), out5); + hvx_vec_store_u(dst6 + off, nloe * sizeof(float), out6); + hvx_vec_store_u(dst7 + off, nloe * sizeof(float), out7); + + acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero)); + acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero)); + acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero)); + acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero)); + acc4 = hvx_vec_add_f32_f32(acc4, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out4, vdot), zero)); + acc5 = hvx_vec_add_f32_f32(acc5, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out5, vdot), zero)); + acc6 = hvx_vec_add_f32_f32(acc6, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out6, vdot), zero)); + acc7 = hvx_vec_add_f32_f32(acc7, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out7, vdot), zero)); + } + + HVX_Vector_x4 accA = { .v = { acc0, acc1, acc2, acc3 } }; + HVX_Vector_x4 accB = { .v = { acc4, acc5, acc6, acc7 } }; + HVX_Vector rA = hvx_vec_reduce_sum_f32x4(accA); + HVX_Vector rB = hvx_vec_reduce_sum_f32x4(accB); + HVX_VectorPred q16 = Q6_Q_vsetq2_R(16); + return Q6_V_vmux_QVV(q16, rA, Q6_V_vror_VR(rB, 128 - 16)); +} + +static inline HVX_Vector gdn_add_scaled_dot8_f32(float * restrict dst0, float * restrict dst1, + float * restrict dst2, float * restrict dst3, float * restrict dst4, + float * restrict dst5, float * restrict dst6, float * restrict dst7, + const HVX_Vector * restrict src, const float * restrict scale, + const HVX_Vector * restrict dot, uint32_t n) { + HVX_Vector acc0 = Q6_V_vzero(); + HVX_Vector acc1 = Q6_V_vzero(); + HVX_Vector acc2 = Q6_V_vzero(); + HVX_Vector acc3 = Q6_V_vzero(); + HVX_Vector acc4 = Q6_V_vzero(); + HVX_Vector acc5 = Q6_V_vzero(); + HVX_Vector acc6 = Q6_V_vzero(); + HVX_Vector acc7 = Q6_V_vzero(); + const HVX_Vector scale0 = hvx_vec_splat_f32(scale[0]); + const HVX_Vector scale1 = hvx_vec_splat_f32(scale[1]); + const HVX_Vector scale2 = hvx_vec_splat_f32(scale[2]); + const HVX_Vector scale3 = hvx_vec_splat_f32(scale[3]); + const HVX_Vector scale4 = hvx_vec_splat_f32(scale[4]); + const HVX_Vector scale5 = hvx_vec_splat_f32(scale[5]); + const HVX_Vector scale6 = hvx_vec_splat_f32(scale[6]); + const HVX_Vector scale7 = hvx_vec_splat_f32(scale[7]); + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + for (uint32_t i = 0; i < nvec; ++i) { + HVX_Vector vs = src[i]; + HVX_Vector vdot = dot[i]; + + HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + i * epv), hvx_vec_mul_f32_f32(vs, scale0)); + HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + i * epv), hvx_vec_mul_f32_f32(vs, scale1)); + HVX_Vector out2 = hvx_vec_add_f32_f32(hvx_vmemu(dst2 + i * epv), hvx_vec_mul_f32_f32(vs, scale2)); + HVX_Vector out3 = hvx_vec_add_f32_f32(hvx_vmemu(dst3 + i * epv), hvx_vec_mul_f32_f32(vs, scale3)); + HVX_Vector out4 = hvx_vec_add_f32_f32(hvx_vmemu(dst4 + i * epv), hvx_vec_mul_f32_f32(vs, scale4)); + HVX_Vector out5 = hvx_vec_add_f32_f32(hvx_vmemu(dst5 + i * epv), hvx_vec_mul_f32_f32(vs, scale5)); + HVX_Vector out6 = hvx_vec_add_f32_f32(hvx_vmemu(dst6 + i * epv), hvx_vec_mul_f32_f32(vs, scale6)); + HVX_Vector out7 = hvx_vec_add_f32_f32(hvx_vmemu(dst7 + i * epv), hvx_vec_mul_f32_f32(vs, scale7)); + + hvx_vmemu(dst0 + i * epv) = out0; + hvx_vmemu(dst1 + i * epv) = out1; + hvx_vmemu(dst2 + i * epv) = out2; + hvx_vmemu(dst3 + i * epv) = out3; + hvx_vmemu(dst4 + i * epv) = out4; + hvx_vmemu(dst5 + i * epv) = out5; + hvx_vmemu(dst6 + i * epv) = out6; + hvx_vmemu(dst7 + i * epv) = out7; + + acc0 = hvx_vec_add_f32_f32(acc0, hvx_vec_mul_f32_f32(out0, vdot)); + acc1 = hvx_vec_add_f32_f32(acc1, hvx_vec_mul_f32_f32(out1, vdot)); + acc2 = hvx_vec_add_f32_f32(acc2, hvx_vec_mul_f32_f32(out2, vdot)); + acc3 = hvx_vec_add_f32_f32(acc3, hvx_vec_mul_f32_f32(out3, vdot)); + acc4 = hvx_vec_add_f32_f32(acc4, hvx_vec_mul_f32_f32(out4, vdot)); + acc5 = hvx_vec_add_f32_f32(acc5, hvx_vec_mul_f32_f32(out5, vdot)); + acc6 = hvx_vec_add_f32_f32(acc6, hvx_vec_mul_f32_f32(out6, vdot)); + acc7 = hvx_vec_add_f32_f32(acc7, hvx_vec_mul_f32_f32(out7, vdot)); + } + + if (nloe) { + const uint32_t off = nvec * epv; + HVX_Vector vs = src[nvec]; + HVX_Vector vdot = dot[nvec]; + HVX_VectorPred mask = Q6_Q_vsetq2_R(nloe * sizeof(float)); + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector out0 = hvx_vec_add_f32_f32(hvx_vmemu(dst0 + off), hvx_vec_mul_f32_f32(vs, scale0)); + HVX_Vector out1 = hvx_vec_add_f32_f32(hvx_vmemu(dst1 + off), hvx_vec_mul_f32_f32(vs, scale1)); + HVX_Vector out2 = hvx_vec_add_f32_f32(hvx_vmemu(dst2 + off), hvx_vec_mul_f32_f32(vs, scale2)); + HVX_Vector out3 = hvx_vec_add_f32_f32(hvx_vmemu(dst3 + off), hvx_vec_mul_f32_f32(vs, scale3)); + HVX_Vector out4 = hvx_vec_add_f32_f32(hvx_vmemu(dst4 + off), hvx_vec_mul_f32_f32(vs, scale4)); + HVX_Vector out5 = hvx_vec_add_f32_f32(hvx_vmemu(dst5 + off), hvx_vec_mul_f32_f32(vs, scale5)); + HVX_Vector out6 = hvx_vec_add_f32_f32(hvx_vmemu(dst6 + off), hvx_vec_mul_f32_f32(vs, scale6)); + HVX_Vector out7 = hvx_vec_add_f32_f32(hvx_vmemu(dst7 + off), hvx_vec_mul_f32_f32(vs, scale7)); + + hvx_vec_store_u(dst0 + off, nloe * sizeof(float), out0); + hvx_vec_store_u(dst1 + off, nloe * sizeof(float), out1); + hvx_vec_store_u(dst2 + off, nloe * sizeof(float), out2); + hvx_vec_store_u(dst3 + off, nloe * sizeof(float), out3); + hvx_vec_store_u(dst4 + off, nloe * sizeof(float), out4); + hvx_vec_store_u(dst5 + off, nloe * sizeof(float), out5); + hvx_vec_store_u(dst6 + off, nloe * sizeof(float), out6); + hvx_vec_store_u(dst7 + off, nloe * sizeof(float), out7); + + acc0 = hvx_vec_add_f32_f32(acc0, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out0, vdot), zero)); + acc1 = hvx_vec_add_f32_f32(acc1, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out1, vdot), zero)); + acc2 = hvx_vec_add_f32_f32(acc2, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out2, vdot), zero)); + acc3 = hvx_vec_add_f32_f32(acc3, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out3, vdot), zero)); + acc4 = hvx_vec_add_f32_f32(acc4, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out4, vdot), zero)); + acc5 = hvx_vec_add_f32_f32(acc5, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out5, vdot), zero)); + acc6 = hvx_vec_add_f32_f32(acc6, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out6, vdot), zero)); + acc7 = hvx_vec_add_f32_f32(acc7, Q6_V_vmux_QVV(mask, hvx_vec_mul_f32_f32(out7, vdot), zero)); + } + + HVX_Vector_x4 accA = { .v = { acc0, acc1, acc2, acc3 } }; + HVX_Vector_x4 accB = { .v = { acc4, acc5, acc6, acc7 } }; + HVX_Vector rA = hvx_vec_reduce_sum_f32x4(accA); + HVX_Vector rB = hvx_vec_reduce_sum_f32x4(accB); + HVX_VectorPred q16 = Q6_Q_vsetq2_R(16); + return Q6_V_vmux_QVV(q16, rA, Q6_V_vror_VR(rB, 128 - 16)); +} + +static inline void gdn_step_kda_f32( + float * restrict s_work, + float * restrict attn_out, + const float * restrict q_t, + const float * restrict k_t, + const float * restrict v_t, + const float * restrict g_t, + float beta_val, + float scale, + uint32_t S_v +) { + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = S_v / epv; + const uint32_t nloe = S_v % epv; + + HVX_Vector vq[4]; + HVX_Vector vk[4]; + HVX_Vector vg[4]; + + for (uint32_t i = 0; i < nvec; ++i) { + vq[i] = hvx_vmemu(q_t + i * epv); + vk[i] = hvx_vmemu(k_t + i * epv); + vg[i] = hvx_vec_exp_f32(hvx_vmemu(g_t + i * epv)); + } + if (nloe) { + vq[nvec] = hvx_vmemu(q_t + nvec * epv); + vk[nvec] = hvx_vmemu(k_t + nvec * epv); + vg[nvec] = hvx_vec_exp_f32(hvx_vmemu(g_t + nvec * epv)); + } + + const HVX_Vector vbeta = hvx_vec_splat_f32(beta_val); + const HVX_Vector vscale = hvx_vec_splat_f32(scale); + + float delta[8] __attribute__((aligned(128))); + + uint32_t j = 0; + for (; j + 8 <= S_v; j += 8) { + float * row0 = s_work + (uint64_t) (j + 0) * S_v; + float * row1 = s_work + (uint64_t) (j + 1) * S_v; + float * row2 = s_work + (uint64_t) (j + 2) * S_v; + float * row3 = s_work + (uint64_t) (j + 3) * S_v; + float * row4 = s_work + (uint64_t) (j + 4) * S_v; + float * row5 = s_work + (uint64_t) (j + 5) * S_v; + float * row6 = s_work + (uint64_t) (j + 6) * S_v; + float * row7 = s_work + (uint64_t) (j + 7) * S_v; + + HVX_Vector vsums = gdn_mul_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, + vg, vk, S_v); + + HVX_Vector vv_t = hvx_vmemu(v_t + j); + HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, vsums); + HVX_Vector vdelta = hvx_vec_mul_f32_f32(diff, vbeta); + hvx_vec_store_u(delta, 8 * sizeof(float), vdelta); + + HVX_Vector vattn = gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, + vk, delta, vq, S_v); + + HVX_Vector res_attn = hvx_vec_mul_f32_f32(vattn, vscale); + hvx_vec_store_u(attn_out + j, 8 * sizeof(float), res_attn); + } + for (; j + 4 <= S_v; j += 4) { + float * row0 = s_work + (uint64_t) (j + 0) * S_v; + float * row1 = s_work + (uint64_t) (j + 1) * S_v; + float * row2 = s_work + (uint64_t) (j + 2) * S_v; + float * row3 = s_work + (uint64_t) (j + 3) * S_v; + + HVX_Vector vsums = gdn_mul_dot4_f32(row0, row1, row2, row3, vg, vk, S_v); + + HVX_Vector vv_t = hvx_vmemu(v_t + j); + HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, vsums); + HVX_Vector vdelta = hvx_vec_mul_f32_f32(diff, vbeta); + hvx_vec_store_u(delta, 4 * sizeof(float), vdelta); + + HVX_Vector vattn = gdn_add_scaled_dot4_f32(row0, row1, row2, row3, vk, delta, vq, S_v); + + HVX_Vector res_attn = hvx_vec_mul_f32_f32(vattn, vscale); + hvx_vec_store_u(attn_out + j, 4 * sizeof(float), res_attn); + } + for (; j < S_v; ++j) { + float * row = s_work + (uint64_t) j * S_v; + HVX_Vector vsum = gdn_mul_dot_f32(row, vg, vk, S_v); + HVX_Vector vv_t = hvx_vec_splat_f32(v_t[j]); + HVX_Vector vdj = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv_t, vsum), vbeta); + HVX_Vector vres = gdn_add_scaled_dot_f32(row, vk, vdj, vq, S_v); + attn_out[j] = hvx_vec_get_f32(hvx_vec_mul_f32_f32(vres, vscale)); + } +} + +static inline void gdn_step_scalar_f32( + float * restrict s_work, + float * restrict attn_out, + const float * restrict q_t, + const float * restrict k_t, + const float * restrict v_t, + const float * restrict g_t, + float beta_val, + float scale, + uint32_t S_v +) { + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = S_v / epv; + const uint32_t nloe = S_v % epv; + + HVX_Vector vq[4]; + HVX_Vector vk[4]; + + for (uint32_t i = 0; i < nvec; ++i) { + vq[i] = hvx_vmemu(q_t + i * epv); + vk[i] = hvx_vmemu(k_t + i * epv); + } + if (nloe) { + vq[nvec] = hvx_vmemu(q_t + nvec * epv); + vk[nvec] = hvx_vmemu(k_t + nvec * epv); + } + + const HVX_Vector vgate = hvx_vec_exp_f32(hvx_vec_splat_f32(g_t[0])); + const HVX_Vector vbeta = hvx_vec_splat_f32(beta_val); + const HVX_Vector vscale = hvx_vec_splat_f32(scale); + + float delta[8] __attribute__((aligned(128))); + + uint32_t j = 0; + for (; j + 8 <= S_v; j += 8) { + float * row0 = s_work + (uint64_t) (j + 0) * S_v; + float * row1 = s_work + (uint64_t) (j + 1) * S_v; + float * row2 = s_work + (uint64_t) (j + 2) * S_v; + float * row3 = s_work + (uint64_t) (j + 3) * S_v; + float * row4 = s_work + (uint64_t) (j + 4) * S_v; + float * row5 = s_work + (uint64_t) (j + 5) * S_v; + float * row6 = s_work + (uint64_t) (j + 6) * S_v; + float * row7 = s_work + (uint64_t) (j + 7) * S_v; + + HVX_Vector vsums = gdn_mul_scalar_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, + vgate, vk, S_v); + + HVX_Vector vv_t = hvx_vmemu(v_t + j); + HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, vsums); + HVX_Vector vdelta = hvx_vec_mul_f32_f32(diff, vbeta); + hvx_vec_store_u(delta, 8 * sizeof(float), vdelta); + + HVX_Vector vattn = gdn_add_scaled_dot8_f32(row0, row1, row2, row3, row4, row5, row6, row7, + vk, delta, vq, S_v); + + HVX_Vector res_attn = hvx_vec_mul_f32_f32(vattn, vscale); + hvx_vec_store_u(attn_out + j, 8 * sizeof(float), res_attn); + } + for (; j + 4 <= S_v; j += 4) { + float * row0 = s_work + (uint64_t) (j + 0) * S_v; + float * row1 = s_work + (uint64_t) (j + 1) * S_v; + float * row2 = s_work + (uint64_t) (j + 2) * S_v; + float * row3 = s_work + (uint64_t) (j + 3) * S_v; + + HVX_Vector vsums = gdn_mul_scalar_dot4_f32(row0, row1, row2, row3, vgate, vk, S_v); + + HVX_Vector vv_t = hvx_vmemu(v_t + j); + HVX_Vector diff = hvx_vec_sub_f32_f32(vv_t, vsums); + HVX_Vector vdelta = hvx_vec_mul_f32_f32(diff, vbeta); + hvx_vec_store_u(delta, 4 * sizeof(float), vdelta); + + HVX_Vector vattn = gdn_add_scaled_dot4_f32(row0, row1, row2, row3, vk, delta, vq, S_v); + + HVX_Vector res_attn = hvx_vec_mul_f32_f32(vattn, vscale); + hvx_vec_store_u(attn_out + j, 4 * sizeof(float), res_attn); + } + for (; j < S_v; ++j) { + float * row = s_work + (uint64_t) j * S_v; + HVX_Vector vsum = gdn_mul_scalar_dot_f32(row, vgate, vk, S_v); + HVX_Vector vv_t = hvx_vec_splat_f32(v_t[j]); + HVX_Vector vdj = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv_t, vsum), vbeta); + HVX_Vector vres = gdn_add_scaled_dot_f32(row, vk, vdj, vq, S_v); + attn_out[j] = hvx_vec_get_f32(hvx_vec_mul_f32_f32(vres, vscale)); + } +} + +static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, void * data) { + struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; + struct htp_ops_context * octx = gctx->octx; + const struct htp_gdn_kernel_params * kparams = gctx->kparams; + + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * v = octx->src[2]; + const struct htp_tensor * g = octx->src[3]; + const struct htp_tensor * beta = octx->src[4]; + const struct htp_tensor * state = octx->src[5]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t S_v = kparams->S_v; + const uint32_t H = kparams->H; + const uint32_t n_tokens = kparams->n_tokens; + const uint32_t n_seqs = kparams->n_seqs; + const uint32_t K = kparams->K; + const uint32_t row_end = gctx->row_start + gctx->nrows; + + if (ith >= gctx->nrows) { + return; + } + + const struct htp_tensor * dst_cache = octx->dsts[1]; + const float scale = kparams->scale; + float * dst_base = (float *) (uintptr_t) dst->data; + float * state_out_base = dst_cache ? (float *) (uintptr_t) dst_cache->data : (dst_base + S_v * H * n_tokens * n_seqs); + + dma_queue * dma_q = octx->ctx->dma[ith]; + const struct htp_gdn_vtcm_layout * layout = &gctx->layout; + float * s_work[2]; + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); + + const struct fastdiv_values * fd_H = &kparams->div_H; + const struct fastdiv_values * fd_q1 = &kparams->div_q1; + const struct fastdiv_values * fd_k1 = &kparams->div_k1; + const struct fastdiv_values * fd_rq3 = &kparams->div_rq3; + const struct fastdiv_values * fd_rk3 = &kparams->div_rk3; + + const uint32_t state_seq_stride = kparams->state_seq_stride; + const uint64_t state_size_per_snap = (uint64_t) kparams->state_size_per_snap; + const dma_addr_t state_out_dma_base = dst_cache ? dst_cache->data : (dst->data + S_v * H * n_tokens * n_seqs * sizeof(float)); + + uint32_t ir_prefetch = gctx->row_start + ith; + int spad_idx = 0; + + // Prefetch preamble (up to 2 steps) + for (int step = 0; step < 2 && ir_prefetch < row_end; step++) { + const uint32_t piv1 = fastmodulo(ir_prefetch, H, fd_H); + const uint32_t piv3 = fastdiv(ir_prefetch, fd_H); + dma_addr_t ps_in = state->data + ((uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v) * sizeof(float); + dma_addr_t ps_out = state_out_dma_base + ((uint64_t) piv3 * H + piv1) * S_v * S_v * sizeof(float); + + // Push dummy write-back + dma_queue_push(dma_q, dma_make_data(ps_out, s_work[spad_idx]), + S_v * sizeof(float), S_v * sizeof(float), + S_v * sizeof(float), 0); + + // Push fetch + dma_queue_push(dma_q, dma_make_data(s_work[spad_idx], ps_in), + S_v * sizeof(float), S_v * sizeof(float), + S_v * sizeof(float), S_v); + + ir_prefetch += nth; + spad_idx ^= 1; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + int curr_spad_idx = 0; + for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + float * s_work_curr = s_work[curr_spad_idx]; + + const uint32_t iv1 = fastmodulo(ir, H, fd_H); + const uint32_t iv3 = fastdiv(ir, fd_H); + + const uint32_t iq1 = fastmodulo(iv1, q->ne[1], fd_q1); + const uint32_t ik1 = fastmodulo(iv1, k->ne[1], fd_k1); + const uint32_t iq3 = fastdiv(iv3, fd_rq3); + const uint32_t ik3 = fastdiv(iv3, fd_rk3); + + dma_addr_t s_out = state_out_dma_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v * sizeof(float); + float * attn_data = dst_base + ((uint64_t) iv3 * n_tokens * H + iv1) * S_v; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + for (uint32_t t = 0; t < n_tokens; ++t) { + const float * q_t = (const float *) ((const uint8_t *) (uintptr_t) q->data + + (uint64_t) iq3 * q->nb[3] + (uint64_t) t * q->nb[2] + (uint64_t) iq1 * q->nb[1]); + const float * k_t = (const float *) ((const uint8_t *) (uintptr_t) k->data + + (uint64_t) ik3 * k->nb[3] + (uint64_t) t * k->nb[2] + (uint64_t) ik1 * k->nb[1]); + const float * v_t = (const float *) ((const uint8_t *) (uintptr_t) v->data + + (uint64_t) iv3 * v->nb[3] + (uint64_t) t * v->nb[2] + (uint64_t) iv1 * v->nb[1]); + const float * g_t = (const float *) ((const uint8_t *) (uintptr_t) g->data + + (uint64_t) iv3 * g->nb[3] + (uint64_t) t * g->nb[2] + (uint64_t) iv1 * g->nb[1]); + const float beta_val = *(const float *) ((const uint8_t *) (uintptr_t) beta->data + + (uint64_t) iv3 * beta->nb[3] + (uint64_t) t * beta->nb[2] + (uint64_t) iv1 * beta->nb[1]); + + if (kparams->kda) { + gdn_step_kda_f32(s_work_curr, attn_data, q_t, k_t, v_t, g_t, beta_val, scale, S_v); + } else { + gdn_step_scalar_f32(s_work_curr, attn_data, q_t, k_t, v_t, g_t, beta_val, scale, S_v); + } + + if (K > 1) { + const int64_t target_slot = (int64_t) n_tokens - 1 - (int64_t) t; + if (target_slot > 0 && target_slot < (int64_t) K) { + float * curr_state_o = state_out_base + (uint64_t) target_slot * state_size_per_snap + ((uint64_t) iv3 * H + iv1) * S_v * S_v; + hvx_copy_f32_uu((uint8_t *) curr_state_o, (const uint8_t *) s_work_curr, S_v * S_v); + } + } + + attn_data += (uint64_t) S_v * H; + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + + // Push real write-back + dma_queue_push(dma_q, dma_make_data(s_out, s_work_curr), + S_v * sizeof(float), S_v * sizeof(float), + S_v * sizeof(float), S_v); + + // Prefetch next block (if any) + if (ir_prefetch < row_end) { + const uint32_t piv1 = fastmodulo(ir_prefetch, H, fd_H); + const uint32_t piv3 = fastdiv(ir_prefetch, fd_H); + dma_addr_t ps_in = state->data + ((uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v) * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(s_work[spad_idx], ps_in), + S_v * sizeof(float), S_v * sizeof(float), + S_v * sizeof(float), S_v); + + ir_prefetch += nth; + spad_idx ^= 1; + } + + curr_spad_idx ^= 1; + } + dma_queue_flush(dma_q); +} + +static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, void * data) { + struct htp_gdn_context * gctx = (struct htp_gdn_context *) data; + struct htp_ops_context * octx = gctx->octx; + const struct htp_gdn_kernel_params * kparams = gctx->kparams; + + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * v = octx->src[2]; + const struct htp_tensor * g = octx->src[3]; + const struct htp_tensor * beta = octx->src[4]; + const struct htp_tensor * state = octx->src[5]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t S_v = kparams->S_v; + const uint32_t H = kparams->H; + const uint32_t n_seqs = kparams->n_seqs; + const uint32_t row_end = gctx->row_start + gctx->nrows; + + if (ith >= gctx->nrows) { + return; + } + + const struct htp_tensor * dst_cache = octx->dsts[1]; + const float scale = kparams->scale; + float * dst_base = (float *) (uintptr_t) dst->data; + + dma_queue * dma_q = octx->ctx->dma[ith]; + const struct htp_gdn_vtcm_layout * layout = &gctx->layout; + float * s_work[2]; + s_work[0] = (float *) (gctx->vtcm_base + layout->bytes_per_thread * ith); + s_work[1] = s_work[0] + layout->state_aligned / sizeof(float); + + const struct fastdiv_values * fd_H = &kparams->div_H; + const struct fastdiv_values * fd_q1 = &kparams->div_q1; + const struct fastdiv_values * fd_k1 = &kparams->div_k1; + const struct fastdiv_values * fd_rq3 = &kparams->div_rq3; + const struct fastdiv_values * fd_rk3 = &kparams->div_rk3; + + const uint32_t state_seq_stride = kparams->state_seq_stride; + const dma_addr_t state_out_dma_base = dst_cache ? dst_cache->data : (dst->data + S_v * H * n_seqs * sizeof(float)); + + uint32_t ir_prefetch = gctx->row_start + ith; + int spad_idx = 0; + + // Prefetch preamble (up to 2 steps) + for (int step = 0; step < 2 && ir_prefetch < row_end; step++) { + const uint32_t piv1 = fastmodulo(ir_prefetch, H, fd_H); + const uint32_t piv3 = fastdiv(ir_prefetch, fd_H); + dma_addr_t ps_in = state->data + ((uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v) * sizeof(float); + dma_addr_t ps_out = state_out_dma_base + ((uint64_t) piv3 * H + piv1) * S_v * S_v * sizeof(float); + + // Push dummy write-back + dma_queue_push(dma_q, dma_make_data(ps_out, s_work[spad_idx]), + S_v * sizeof(float), S_v * sizeof(float), + S_v * sizeof(float), 0); + + // Push fetch + dma_queue_push(dma_q, dma_make_data(s_work[spad_idx], ps_in), + S_v * sizeof(float), S_v * sizeof(float), + S_v * sizeof(float), S_v); + + ir_prefetch += nth; + spad_idx ^= 1; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + int curr_spad_idx = 0; + for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) { + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + float * s_work_curr = s_work[curr_spad_idx]; + + const uint32_t iv1 = fastmodulo(ir, H, fd_H); + const uint32_t iv3 = fastdiv(ir, fd_H); + + const uint32_t iq1 = fastmodulo(iv1, q->ne[1], fd_q1); + const uint32_t ik1 = fastmodulo(iv1, k->ne[1], fd_k1); + const uint32_t iq3 = fastdiv(iv3, fd_rq3); + const uint32_t ik3 = fastdiv(iv3, fd_rk3); + + dma_addr_t s_out = state_out_dma_base + ((uint64_t) iv3 * H + iv1) * S_v * S_v * sizeof(float); + float * attn_data = dst_base + ((uint64_t) iv3 * H + iv1) * S_v; + + const float * q_t = (const float *) ((const uint8_t *) (uintptr_t) q->data + + (uint64_t) iq3 * q->nb[3] + (uint64_t) iq1 * q->nb[1]); + const float * k_t = (const float *) ((const uint8_t *) (uintptr_t) k->data + + (uint64_t) ik3 * k->nb[3] + (uint64_t) ik1 * k->nb[1]); + const float * v_t = (const float *) ((const uint8_t *) (uintptr_t) v->data + + (uint64_t) iv3 * v->nb[3] + (uint64_t) iv1 * v->nb[1]); + const float * g_t = (const float *) ((const uint8_t *) (uintptr_t) g->data + + (uint64_t) iv3 * g->nb[3] + (uint64_t) iv1 * g->nb[1]); + const float beta_val = *(const float *) ((const uint8_t *) (uintptr_t) beta->data + + (uint64_t) iv3 * beta->nb[3] + (uint64_t) iv1 * beta->nb[1]); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + if (kparams->kda) { + gdn_step_kda_f32(s_work_curr, attn_data, q_t, k_t, v_t, g_t, beta_val, scale, S_v); + } else { + gdn_step_scalar_f32(s_work_curr, attn_data, q_t, k_t, v_t, g_t, beta_val, scale, S_v); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + + // Push real write-back + dma_queue_push(dma_q, dma_make_data(s_out, s_work_curr), + S_v * sizeof(float), S_v * sizeof(float), + S_v * sizeof(float), S_v); + + // Prefetch next block (if any) + if (ir_prefetch < row_end) { + const uint32_t piv1 = fastmodulo(ir_prefetch, H, fd_H); + const uint32_t piv3 = fastdiv(ir_prefetch, fd_H); + dma_addr_t ps_in = state->data + ((uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v) * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(s_work[spad_idx], ps_in), + S_v * sizeof(float), S_v * sizeof(float), + S_v * sizeof(float), S_v); + + ir_prefetch += nth; + spad_idx ^= 1; + } + + curr_spad_idx ^= 1; + } + dma_queue_flush(dma_q); +} + +struct htp_gdn_hmx_gemm_task { + const __fp16 * row_tiles; + const __fp16 * col_tiles; + __fp16 * out_tiles; + uint32_t n_row_tiles; + uint32_t n_col_tiles; + uint32_t n_dot_tiles; + uint32_t dot_stride; + uint8_t * hmx_scales; +}; + +static void htp_gdn_hmx_gemm_worker(void * data) { + struct htp_gdn_hmx_gemm_task * task = (struct htp_gdn_hmx_gemm_task *) data; + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)task->hmx_scales)); + + const size_t dot_stride = task->dot_stride; + for (uint32_t r = 0; r < task->n_row_tiles; ++r) { + const __fp16 * r_tiles = task->row_tiles + r * dot_stride; + const __fp16 * c_tiles = task->col_tiles; + __fp16 * o_tile = task->out_tiles + r * task->n_col_tiles * HMX_FP16_TILE_N_ELMS; + + for (uint32_t c = 0; c < task->n_col_tiles; ++c) { + hmx_fa_qk_dot_tile(r_tiles, c_tiles, o_tile, task->n_dot_tiles); + c_tiles += dot_stride; + o_tile += HMX_FP16_TILE_N_ELMS; + } + } +} + +static inline void htp_gdn_push_hmx_gemm_task( + hmx_queue_t q, + struct htp_gdn_hmx_gemm_task * task, + const __fp16 * row_tiles, + const __fp16 * col_tiles, + __fp16 * out_tiles, + uint32_t n_row_tiles, + uint32_t n_col_tiles, + uint32_t n_dot_tiles, + uint8_t * scales +) { + task->row_tiles = row_tiles; + task->col_tiles = col_tiles; + task->out_tiles = out_tiles; + task->n_row_tiles = n_row_tiles; + task->n_col_tiles = n_col_tiles; + task->n_dot_tiles = n_dot_tiles; + task->dot_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS; + task->hmx_scales = scales; + + hmx_queue_push(q, hmx_queue_make_desc(htp_gdn_hmx_gemm_worker, task)); +} + +static inline void gdn_unpack_64x64_tiles_to_vectors( + HVX_Vector * restrict rows, + const __fp16 * restrict tiles +) { + const HVX_Vector * t00 = (const HVX_Vector *) (tiles + 0 * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t01 = (const HVX_Vector *) (tiles + 1 * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t10 = (const HVX_Vector *) (tiles + 2 * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t11 = (const HVX_Vector *) (tiles + 3 * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp0 = Q6_W_vdeal_VVR(t01[r], t00[r], -2); + rows[2 * r + 0] = Q6_V_lo_W(vp0); + rows[2 * r + 1] = Q6_V_hi_W(vp0); + + HVX_VectorPair vp1 = Q6_W_vdeal_VVR(t11[r], t10[r], -2); + rows[32 + 2 * r + 0] = Q6_V_lo_W(vp1); + rows[32 + 2 * r + 1] = Q6_V_hi_W(vp1); + } +} + +static inline void gdn_pack_64x64_vectors_to_tiles( + __fp16 * restrict tiles, + const HVX_Vector * restrict rows +) { + HVX_Vector * t00 = (HVX_Vector *) (tiles + 0 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * t01 = (HVX_Vector *) (tiles + 1 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * t10 = (HVX_Vector *) (tiles + 2 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * t11 = (HVX_Vector *) (tiles + 3 * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp0 = Q6_W_vshuff_VVR(rows[2 * r + 1], rows[2 * r + 0], -2); + t00[r] = Q6_V_lo_W(vp0); + t01[r] = Q6_V_hi_W(vp0); + + HVX_VectorPair vp1 = Q6_W_vshuff_VVR(rows[32 + 2 * r + 1], rows[32 + 2 * r + 0], -2); + t10[r] = Q6_V_lo_W(vp1); + t11[r] = Q6_V_hi_W(vp1); + } +} + +static inline void gdn_unpack_64xS_tiles_to_f32( + float * restrict dst_f32, + const __fp16 * restrict tiles, + uint32_t S_v +) { + const uint32_t n_col_tiles = S_v / 32; + for (uint32_t r0 = 0; r0 < 2; ++r0) { + for (uint32_t d = 0; d < S_v / 64; ++d) { + const HVX_Vector * t0 = (const HVX_Vector *) (tiles + (r0 * n_col_tiles + 2 * d + 0) * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t1 = (const HVX_Vector *) (tiles + (r0 * n_col_tiles + 2 * d + 1) * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp01 = Q6_W_vdeal_VVR(t1[r], t0[r], -2); + HVX_VectorPair p0 = hvx_vec_f16_to_f32(Q6_V_lo_W(vp01)); + HVX_VectorPair p1 = hvx_vec_f16_to_f32(Q6_V_hi_W(vp01)); + + float * out0 = dst_f32 + (r0 * 32 + 2 * r + 0) * S_v + d * 64; + float * out1 = dst_f32 + (r0 * 32 + 2 * r + 1) * S_v + d * 64; + + hvx_vmem(out0 + 0) = Q6_V_lo_W(p0); + hvx_vmem(out0 + 32) = Q6_V_hi_W(p0); + hvx_vmem(out1 + 0) = Q6_V_lo_W(p1); + hvx_vmem(out1 + 32) = Q6_V_hi_W(p1); + } + } + } +} + +static inline void gdn_unpack_64xS_tiles_to_f16( + __fp16 * restrict dst_f16, + const __fp16 * restrict tiles, + uint32_t S_v +) { + const uint32_t n_col_tiles = S_v / 32; + for (uint32_t r0 = 0; r0 < 2; ++r0) { + for (uint32_t d = 0; d < S_v / 64; ++d) { + const HVX_Vector * t0 = (const HVX_Vector *) (tiles + (r0 * n_col_tiles + 2 * d + 0) * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t1 = (const HVX_Vector *) (tiles + (r0 * n_col_tiles + 2 * d + 1) * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp01 = Q6_W_vdeal_VVR(t1[r], t0[r], -2); + __fp16 * out0 = dst_f16 + (r0 * 32 + 2 * r + 0) * S_v + d * 64; + __fp16 * out1 = dst_f16 + (r0 * 32 + 2 * r + 1) * S_v + d * 64; + + hvx_vmem(out0) = Q6_V_lo_W(vp01); + hvx_vmem(out1) = Q6_V_hi_W(vp01); + } + } + } +} + +static inline void gdn_unpack_SxS_tiles_to_f32( + float * restrict dst_f32, + const __fp16 * restrict tiles, + uint32_t S_v +) { + const uint32_t n_tiles = S_v / 32; + for (uint32_t r0 = 0; r0 < n_tiles; ++r0) { + for (uint32_t d = 0; d < S_v / 64; ++d) { + const HVX_Vector * t0 = (const HVX_Vector *) (tiles + (r0 * n_tiles + 2 * d + 0) * HMX_FP16_TILE_N_ELMS); + const HVX_Vector * t1 = (const HVX_Vector *) (tiles + (r0 * n_tiles + 2 * d + 1) * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp01 = Q6_W_vdeal_VVR(t1[r], t0[r], -2); + HVX_VectorPair p0 = hvx_vec_f16_to_f32(Q6_V_lo_W(vp01)); + HVX_VectorPair p1 = hvx_vec_f16_to_f32(Q6_V_hi_W(vp01)); + + float * out0 = dst_f32 + (r0 * 32 + 2 * r + 0) * S_v + d * 64; + float * out1 = dst_f32 + (r0 * 32 + 2 * r + 1) * S_v + d * 64; + + hvx_vmem(out0 + 0) = Q6_V_lo_W(p0); + hvx_vmem(out0 + 32) = Q6_V_hi_W(p0); + hvx_vmem(out1 + 0) = Q6_V_lo_W(p1); + hvx_vmem(out1 + 32) = Q6_V_hi_W(p1); + } + } + } +} + +static inline void gdn_f32_to_hmx_row_tiles_and_f16( + __fp16 * restrict dst_tiles, + __fp16 * restrict dst_prime_tiles, + __fp16 * restrict dst_f16, + const float * restrict src, + const __fp16 * restrict scale_per_row, + uint32_t n_rows, + uint32_t n_cols +) { + const uint32_t n_col_tiles = n_cols / 32; + const uint32_t * scale_pairs = (const uint32_t *) scale_per_row; + + for (uint32_t r = 0; r < n_rows; r += 2) { + uint32_t r0 = r / 32; + uint32_t r1 = (r % 32) / 2; + const float * p0 = src + (r + 0) * n_cols; + const float * p1 = src + (r + 1) * n_cols; + + HVX_Vector v_scale; + if (dst_prime_tiles) { + uint32_t scale_pair = scale_pairs ? scale_pairs[r / 2] : 0x3c003c00; + v_scale = Q6_V_vsplat_R(scale_pair); + } + + for (uint32_t c = 0; c < n_col_tiles; c += 2) { + HVX_Vector v0_0 = hvx_vmem(p0 + (c + 0) * 32); + HVX_Vector v1_0 = hvx_vmem(p1 + (c + 0) * 32); + HVX_Vector v0_1 = hvx_vmem(p0 + (c + 1) * 32); + HVX_Vector v1_1 = hvx_vmem(p1 + (c + 1) * 32); + + HVX_Vector vh0 = hvx_vec_f32_to_f16_shuff(v0_0, v1_0); + HVX_Vector vh1 = hvx_vec_f32_to_f16_shuff(v0_1, v1_1); + __fp16 * tile0 = dst_tiles + (r0 * n_col_tiles + c + 0) * HMX_FP16_TILE_N_ELMS; + __fp16 * tile1 = dst_tiles + (r0 * n_col_tiles + c + 1) * HMX_FP16_TILE_N_ELMS; + ((HVX_Vector *) tile0)[r1] = vh0; + ((HVX_Vector *) tile1)[r1] = vh1; + + if (dst_prime_tiles) { + HVX_Vector vh0_s = hvx_vec_mul_f16_f16(vh0, v_scale); + HVX_Vector vh1_s = hvx_vec_mul_f16_f16(vh1, v_scale); + __fp16 * tile0_s = dst_prime_tiles + (r0 * n_col_tiles + c + 0) * HMX_FP16_TILE_N_ELMS; + __fp16 * tile1_s = dst_prime_tiles + (r0 * n_col_tiles + c + 1) * HMX_FP16_TILE_N_ELMS; + ((HVX_Vector *) tile0_s)[r1] = vh0_s; + ((HVX_Vector *) tile1_s)[r1] = vh1_s; + } + + if (dst_f16) { + HVX_VectorPair vp01 = Q6_W_vdeal_VVR(vh1, vh0, -2); + hvx_vmem(dst_f16 + (r + 0) * n_cols + c * 32) = Q6_V_lo_W(vp01); + hvx_vmem(dst_f16 + (r + 1) * n_cols + c * 32) = Q6_V_hi_W(vp01); + } + } + } +} + +static inline void hvx_transpose_32x32_words(HVX_Vector * restrict m, HVX_Vector * restrict tmp) { + for (int i = 0; i < 16; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[2*i + 1], m[2*i], -4); + tmp[2*i + 0] = Q6_V_lo_W(p); + tmp[2*i + 1] = Q6_V_hi_W(p); + } + + for (int b = 0; b < 32; b += 4) { + HVX_VectorPair p0 = Q6_W_vshuff_VVR(tmp[b + 2], tmp[b + 0], -8); + HVX_VectorPair p1 = Q6_W_vshuff_VVR(tmp[b + 3], tmp[b + 1], -8); + m[b + 0] = Q6_V_lo_W(p0); m[b + 1] = Q6_V_hi_W(p0); + m[b + 2] = Q6_V_lo_W(p1); m[b + 3] = Q6_V_hi_W(p1); + } + + for (int b = 0; b < 32; b += 8) { + for (int i = 0; i < 4; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[b + i + 4], m[b + i], -16); + tmp[b + 2*i + 0] = Q6_V_lo_W(p); + tmp[b + 2*i + 1] = Q6_V_hi_W(p); + } + } + + for (int b = 0; b < 32; b += 16) { + for (int i = 0; i < 8; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(tmp[b + i + 8], tmp[b + i], -32); + m[b + 2*i + 0] = Q6_V_lo_W(p); + m[b + 2*i + 1] = Q6_V_hi_W(p); + } + } + + for (int i = 0; i < 16; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[i + 16], m[i], -64); + tmp[2 * i + 0] = Q6_V_lo_W(p); + tmp[2 * i + 1] = Q6_V_hi_W(p); + } + + for (int i = 0; i < 32; ++i) { + m[i] = tmp[i]; + } +} + +static inline void gdn_pack_d_t_row_tiles( + __fp16 * restrict dst_tiles, + const __fp16 * restrict src_d, + uint32_t S_v, + HVX_Vector * restrict m, + HVX_Vector * restrict tmp +) { + for (uint32_t col_half = 0; col_half < S_v / 64; ++col_half) { + uint32_t r0_base = col_half * 2; + for (uint32_t c0 = 0; c0 < 2; ++c0) { + for (uint32_t s_local = 0; s_local < 32; ++s_local) { + uint32_t s = c0 * 32 + s_local; + m[s_local] = hvx_vmem(src_d + s * S_v + col_half * 64); + } + + hvx_transpose_32x32_words(m, tmp); + + uint32_t tile0_idx = (r0_base + 0) * 2 + c0; + uint32_t tile1_idx = (r0_base + 1) * 2 + c0; + HVX_Vector * t0 = (HVX_Vector *)(dst_tiles + tile0_idx * HMX_FP16_TILE_N_ELMS); + HVX_Vector * t1 = (HVX_Vector *)(dst_tiles + tile1_idx * HMX_FP16_TILE_N_ELMS); + + for (uint32_t r = 0; r < 16; ++r) { + t0[r] = m[r]; + t1[r] = m[16 + r]; + } + } + } +} + +static __attribute__((noinline)) void gdn_build_inv_l_blocks( + __fp16 * restrict inv_row_tiles, + const HVX_Vector * restrict rows_kk, + const __fp16 * restrict decay_m, + const float * restrict beta, + __fp16 * restrict l10_tile, + __fp16 * restrict neg_a11_tile +) { + const HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0f); + const HVX_VectorPred q_mask64 = Q6_Q_vsetq2_R(64); + + uint16_t beta_u16[64] __attribute__((aligned(128))); + uint16_t l00[32][32] __attribute__((aligned(128))); + uint16_t l11[32][32] __attribute__((aligned(128))); + + HVX_Vector * restrict p_l00 = (HVX_Vector *) l00; + HVX_Vector * restrict p_l11 = (HVX_Vector *) l11; + HVX_Vector * restrict p_l10_tile = (HVX_Vector *) l10_tile; + + HVX_Vector * restrict tile00 = (HVX_Vector *) (inv_row_tiles + 0 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * restrict tile01 = (HVX_Vector *) (inv_row_tiles + 1 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * restrict tile11 = (HVX_Vector *) (inv_row_tiles + 3 * HMX_FP16_TILE_N_ELMS); + HVX_Vector * restrict p_neg_a11 = (HVX_Vector *) neg_a11_tile; + + hvx_vmem(beta_u16) = hvx_vec_f32_to_f16(hvx_vmem(beta + 0), hvx_vmem(beta + 32)); + + for (uint32_t r = 0; r < 16; ++r) { + tile01[r] = Q6_V_vzero(); + } + + for (uint32_t r = 0; r < 16; ++r) { + uint32_t t0 = 2 * r; + uint32_t t1 = t0 + 1; + + HVX_Vector v_d0 = hvx_vmem(decay_m + t0 * 64); + HVX_Vector v_d1 = hvx_vmem(decay_m + t1 * 64); + HVX_Vector v_b0 = Q6_Vh_vsplat_R(beta_u16[t0]); + HVX_Vector v_b1 = Q6_Vh_vsplat_R(beta_u16[t1]); + + HVX_Vector r0 = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(rows_kk[t0], v_d0), v_b0); + HVX_Vector r1 = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(rows_kk[t1], v_d1), v_b1); + + p_l00[r] = Q6_V_vmux_QVV(q_mask64, r0, Q6_V_vror_VR(r1, 64)); + } + + for (uint32_t r = 0; r < 16; ++r) { + uint32_t t0 = 32 + 2 * r; + uint32_t t1 = t0 + 1; + + HVX_Vector v_d0 = hvx_vmem(decay_m + t0 * 64); + HVX_Vector v_d1 = hvx_vmem(decay_m + t1 * 64); + HVX_Vector v_b0 = Q6_Vh_vsplat_R(beta_u16[t0]); + HVX_Vector v_b1 = Q6_Vh_vsplat_R(beta_u16[t1]); + + HVX_Vector r0 = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(rows_kk[t0], v_d0), v_b0); + HVX_Vector r1 = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(rows_kk[t1], v_d1), v_b1); + + HVX_VectorPair vp_l10 = Q6_W_vshuff_VVR(r1, r0, -2); + p_l10_tile[r] = Q6_V_lo_W(vp_l10); + p_l11[r] = Q6_V_vmux_QVV(q_mask64, Q6_V_vror_VR(r0, 64), r1); + } + + HVX_Vector a_rows[32]; + for (uint32_t t = 0; t < 32; ++t) { + HVX_Vector v_inv = Q6_V_vzero(); + for (uint32_t k = 0; k < t; ++k) { + HVX_Vector v_lk = Q6_Vh_vsplat_R(l00[t][k]); + v_inv = hvx_vec_sub_f16_f16(v_inv, hvx_vec_mul_f16_f16(v_lk, a_rows[k])); + } + HVX_VectorPred q_diag = (t == 0) ? Q6_Q_vsetq2_R(2) : Q6_Q_and_QQn(Q6_Q_vsetq2_R(2 * (t + 1)), Q6_Q_vsetq2_R(2 * t)); + a_rows[t] = Q6_V_vand_QV(q_mask64, Q6_V_vmux_QVV(q_diag, v_one_f16, v_inv)); + } + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp = Q6_W_vshuff_VVR(a_rows[2 * r + 1], a_rows[2 * r + 0], -2); + tile00[r] = Q6_V_lo_W(vp); + } + + for (uint32_t t = 0; t < 32; ++t) { + HVX_Vector v_inv = Q6_V_vzero(); + for (uint32_t k = 0; k < t; ++k) { + HVX_Vector v_lk = Q6_Vh_vsplat_R(l11[t][k]); + v_inv = hvx_vec_sub_f16_f16(v_inv, hvx_vec_mul_f16_f16(v_lk, a_rows[k])); + } + HVX_VectorPred q_diag = (t == 0) ? Q6_Q_vsetq2_R(2) : Q6_Q_and_QQn(Q6_Q_vsetq2_R(2 * (t + 1)), Q6_Q_vsetq2_R(2 * t)); + a_rows[t] = Q6_V_vand_QV(q_mask64, Q6_V_vmux_QVV(q_diag, v_one_f16, v_inv)); + } + + for (uint32_t r = 0; r < 16; ++r) { + HVX_VectorPair vp = Q6_W_vshuff_VVR(a_rows[2 * r + 1], a_rows[2 * r + 0], -2); + tile11[r] = Q6_V_lo_W(vp); + + HVX_Vector n0 = hvx_vec_sub_f16_f16(Q6_V_vzero(), a_rows[2 * r + 0]); + HVX_Vector n1 = hvx_vec_sub_f16_f16(Q6_V_vzero(), a_rows[2 * r + 1]); + HVX_VectorPair vp_neg = Q6_W_vshuff_VVR(n1, n0, -2); + p_neg_a11[r] = Q6_V_lo_W(vp_neg); + } +} + + +static inline void gdn_dma_push_chunk_inputs( + dma_queue * dma_q, + float * vtcm_q, + float * vtcm_k, + float * vtcm_v, + const struct htp_tensor * q, + const struct htp_tensor * k, + const struct htp_tensor * v, + uint32_t iq3, uint32_t iq1, + uint32_t ik3, uint32_t ik1, + uint32_t iv3, uint32_t iv1, + uint32_t t_chunk, + uint32_t chunk_size, + uint32_t S_v +) { + const dma_addr_t q_dma = q->data + (uint64_t) iq3 * q->nb[3] + (uint64_t) t_chunk * q->nb[2] + (uint64_t) iq1 * q->nb[1]; + const dma_addr_t k_dma = k->data + (uint64_t) ik3 * k->nb[3] + (uint64_t) t_chunk * k->nb[2] + (uint64_t) ik1 * k->nb[1]; + const dma_addr_t v_dma = v->data + (uint64_t) iv3 * v->nb[3] + (uint64_t) t_chunk * v->nb[2] + (uint64_t) iv1 * v->nb[1]; + + dma_queue_push(dma_q, dma_make_data(vtcm_q, q_dma), S_v * sizeof(float), q->nb[2], S_v * sizeof(float), chunk_size); + dma_queue_push(dma_q, dma_make_data(vtcm_k, k_dma), S_v * sizeof(float), k->nb[2], S_v * sizeof(float), chunk_size); + dma_queue_push(dma_q, dma_make_data(vtcm_v, v_dma), S_v * sizeof(float), v->nb[2], S_v * sizeof(float), chunk_size); +} + +static inline void gdn_dma_push_chunk_gb( + dma_queue * dma_q, + float * vtcm_g_raw, + float * vtcm_b_raw, + const struct htp_tensor * g, + const struct htp_tensor * beta, + uint32_t iv3, + uint32_t iv1, + uint32_t t_chunk, + uint32_t chunk_size, + uint32_t n_batch +) { + const dma_addr_t g_dma = g->data + (uint64_t) iv3 * g->nb[3] + (uint64_t) t_chunk * g->nb[2] + (uint64_t) iv1 * g->nb[1]; + const dma_addr_t beta_dma = beta->data + (uint64_t) iv3 * beta->nb[3] + (uint64_t) t_chunk * beta->nb[2] + (uint64_t) iv1 * beta->nb[1]; + const uint32_t row_bytes = n_batch * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(vtcm_g_raw, g_dma), row_bytes, g->nb[2], row_bytes, chunk_size); + dma_queue_push(dma_q, dma_make_data(vtcm_b_raw, beta_dma), row_bytes, beta->nb[2], row_bytes, chunk_size); +} + +static inline void gdn_pack_s_col_tiles( + __fp16 * restrict vtcm_s_col_tiles, + __fp16 * restrict vtcm_s_f16, + const float * restrict vtcm_s_state, + uint32_t S_v +) { + for (uint32_t j = 0; j < S_v; ++j) { + for (uint32_t i = 0; i < S_v; i += 64) { + HVX_Vector v0 = hvx_vmem(vtcm_s_state + j * S_v + i + 0); + HVX_Vector v1 = (i + 32 < S_v) ? hvx_vmem(vtcm_s_state + j * S_v + i + 32) : Q6_V_vzero(); + hvx_vmem(vtcm_s_f16 + j * S_v + i) = hvx_vec_f32_to_f16(v0, v1); + } + } + hmx_interleave_rows_to_tiles(vtcm_s_col_tiles, vtcm_s_f16, S_v, S_v, S_v, 0, S_v); +} + +struct htp_gdn_head_ptrs { + float * s_state; + __fp16 * s_f16; + __fp16 * s_col_tiles; + float * s_update_f32; + __fp16 * s_update_tiles; + + float * q_f32[2]; + float * k_f32[2]; + float * v_f32[2]; + float * g_f32[2]; + float * b_f32[2]; + float * o_f32[2]; + + float * v_inter_f32; + float * o_inter_f32; + float * o_intra_f32; + + __fp16 * k_f16; + __fp16 * v_prime_f16; + __fp16 * delta_f16; + __fp16 * d_f16; + + __fp16 * q_row_tiles; + __fp16 * q_prime_row_tiles; + __fp16 * k_row_tiles; + __fp16 * k_col_tiles; + __fp16 * k_prime_row_tiles; + __fp16 * k_col_tiles_64x128; + __fp16 * kk_tiles; + __fp16 * qk_tiles; + __fp16 * v_inter_tiles; + __fp16 * o_inter_tiles; + __fp16 * inv_row_tiles; + __fp16 * a_row_tiles; + __fp16 * v_prime_col_tiles; + __fp16 * delta_tiles; + __fp16 * delta_col_tiles; + __fp16 * o_intra_tiles; + __fp16 * d_row_tiles; + + __fp16 * gamma; + float * lambda_init; + __fp16 * lambda_init_f16; + __fp16 * decay_m; + __fp16 * decay_a; + + HVX_Vector * rows_kk; + HVX_Vector * rows_qk; + HVX_Vector * rows_inv; + HVX_Vector * rows_a; + + HVX_Vector * vtcm_m; + HVX_Vector * vtcm_tmp; + + uint32_t iv1; + uint32_t iv3; + uint32_t iq1; + uint32_t ik1; + uint32_t iq3; + uint32_t ik3; + dma_addr_t state_in_dma; + dma_addr_t state_out_dma; +}; + +static inline void gdn_init_head_ptrs( + struct htp_gdn_head_ptrs * head, + const struct htp_gdn_hmx_vtcm_layout * L, + uint8_t * vtcm_base, + uint32_t h, + uint32_t base_iv1, + uint32_t iv3, + const struct htp_tensor * q, + const struct htp_tensor * k, + const struct htp_tensor * v, + const struct htp_tensor * state, + const struct htp_tensor * dst, + const struct htp_tensor * dst_cache, + const struct htp_gdn_kernel_params * kparams, + uint32_t S_v, + uint32_t H, + uint32_t n_tokens, + uint32_t chunk_size +) { + const size_t dma_scalar_sz = hex_round_up(chunk_size * sizeof(float), 128); + const size_t decay_sz = 64 * 64 * sizeof(__fp16); + const size_t row_vecs_sz = 64 * 128; + + head->s_state = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_s_state + h * L->state_f32_bytes); + head->s_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_s_f16 + h * L->state_f16_bytes); + head->s_col_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_s_col_tiles + h * L->state_tiles_bytes); + head->s_update_f32 = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_s_update_f32 + h * L->state_f32_bytes); + head->s_update_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_s_update_tiles + h * L->state_tiles_bytes); + + head->q_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_q_f32[0] + h * L->dma_chunk_bytes); + head->q_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_q_f32[1] + h * L->dma_chunk_bytes) : head->q_f32[0]; + head->k_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_k_f32[0] + h * L->dma_chunk_bytes); + head->k_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_k_f32[1] + h * L->dma_chunk_bytes) : head->k_f32[0]; + head->v_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_v_f32[0] + h * L->dma_chunk_bytes); + head->v_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_v_f32[1] + h * L->dma_chunk_bytes) : head->v_f32[0]; + head->g_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_g_f32[0] + h * dma_scalar_sz); + head->g_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_g_f32[1] + h * dma_scalar_sz) : head->g_f32[0]; + head->b_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_b_f32[0] + h * dma_scalar_sz); + head->b_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_b_f32[1] + h * dma_scalar_sz) : head->b_f32[0]; + head->o_f32[0] = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_o_f32[0] + h * L->dma_chunk_bytes); + head->o_f32[1] = L->pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L->off_o_f32[1] + h * L->dma_chunk_bytes) : head->o_f32[0]; + + head->v_inter_f32 = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_v_inter_f32 + h * L->dma_chunk_bytes); + head->o_inter_f32 = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_o_inter_f32 + h * L->dma_chunk_bytes); + head->o_intra_f32 = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_o_intra_f32 + h * L->dma_chunk_bytes); + + head->k_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_f16 + h * L->act_f16_bytes); + head->v_prime_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_v_prime_f16 + h * L->act_f16_bytes); + head->delta_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_delta_f16 + h * L->act_f16_bytes); + head->d_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_d_f16 + h * L->act_f16_bytes); + + head->q_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_q_row_tiles + h * L->tile_64xSv_bytes); + head->q_prime_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_q_prime_row_tiles + h * L->tile_64xSv_bytes); + head->k_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_row_tiles + h * L->tile_64xSv_bytes); + head->k_col_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_col_tiles + h * L->tile_64xSv_bytes); + head->k_prime_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_prime_row_tiles + h * L->tile_64xSv_bytes); + head->k_col_tiles_64x128 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_k_col_tiles_64x128 + h * L->tile_64xSv_bytes); + head->kk_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_kk_tiles + h * L->tile_64x64_bytes); + head->qk_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_qk_tiles + h * L->tile_64x64_bytes); + head->v_inter_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_v_inter_tiles + h * L->tile_64xSv_bytes); + head->o_inter_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_o_inter_tiles + h * L->tile_64xSv_bytes); + head->inv_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_inv_row_tiles + h * L->tile_64x64_bytes); + head->a_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_a_row_tiles + h * L->tile_64x64_bytes); + head->v_prime_col_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_v_prime_col_tiles + h * L->tile_64xSv_bytes); + head->delta_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_delta_tiles + h * L->tile_64xSv_bytes); + head->delta_col_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_delta_col_tiles + h * L->tile_64xSv_bytes); + head->o_intra_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_o_intra_tiles + h * L->tile_64xSv_bytes); + head->d_row_tiles = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_d_row_tiles + h * L->tile_64xSv_bytes); + + head->gamma = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_gamma + h * dma_scalar_sz); + head->lambda_init_f16 = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_gamma + h * dma_scalar_sz + 128); + head->lambda_init = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_lambda_init + h * dma_scalar_sz); + head->decay_m = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_decay_m + h * decay_sz); + head->decay_a = VTCM_LAYOUT_PTR(__fp16, vtcm_base, L->off_decay_a + h * decay_sz); + + head->rows_kk = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_kk + h * row_vecs_sz); + head->rows_qk = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_qk + h * row_vecs_sz); + head->rows_inv = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_inv + h * row_vecs_sz); + head->rows_a = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_rows_a + h * row_vecs_sz); + + head->vtcm_m = VTCM_LAYOUT_PTR(HVX_Vector, vtcm_base, L->off_thread_scratch + h * (64 * 128)); + head->vtcm_tmp = head->vtcm_m + 32; + + head->iv1 = base_iv1 + h; + head->iv3 = iv3; + head->iq1 = fastmodulo(head->iv1, q->ne[1], &kparams->div_q1); + head->ik1 = fastmodulo(head->iv1, k->ne[1], &kparams->div_k1); + head->iq3 = fastdiv(head->iv3, &kparams->div_rq3); + head->ik3 = fastdiv(head->iv3, &kparams->div_rk3); + + head->state_in_dma = state->data + + ((uint64_t) head->iv3 * kparams->state_seq_stride + (uint64_t) head->iv1 * S_v * S_v) * sizeof(float); + + head->state_out_dma = dst_cache ? + (dst_cache->data + ((uint64_t) head->iv3 * H + head->iv1) * S_v * S_v * sizeof(float)) : + (dst->data + ((uint64_t) S_v * H * n_tokens * kparams->n_seqs + (uint64_t) (head->iv3 * H + head->iv1) * S_v * S_v) * sizeof(float)); +} + +struct htp_gdn_batch_context { + struct htp_gdn_head_ptrs * heads; + const float * vtcm_g_raw; + const float * vtcm_b_raw; + uint32_t curr_buf; + uint32_t c; + uint32_t n_batch; + uint32_t S_v; + float scale; + struct htp_ops_context * octx; + const struct htp_gdn_kernel_params * kparams; +}; + +static void gdn_hvx_init_state_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + gdn_pack_s_col_tiles(head->s_col_tiles, head->s_f16, head->s_state, bctx->S_v); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); +} + +static inline __attribute__((unused)) HVX_Vector hvx_clamp_neg20_0(HVX_Vector v, HVX_Vector v_zero, HVX_Vector v_neg20) { + HVX_VectorPred p_gt = Q6_Q_vcmp_gt_VsfVsf(v, v_zero); + v = Q6_V_vmux_QVV(p_gt, v_zero, v); + HVX_VectorPred p_lt = Q6_Q_vcmp_gt_VsfVsf(v_neg20, v); + return Q6_V_vmux_QVV(p_lt, v_neg20, v); +} + +static inline HVX_Vector hvx_prefix_scan_f32(HVX_Vector v, HVX_Vector carry_in) { + const HVX_Vector zero = Q6_V_vzero(); + + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 4)); + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 8)); + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 16)); + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 32)); + v = hvx_vec_add_f32_f32(v, Q6_V_vlalign_VVR(v, zero, 64)); + v = hvx_vec_add_f32_f32(v, carry_in); + + return v; +} + +static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) { + return hvx_vec_repl4(Q6_V_vror_VR(v, 124)); +} + +static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + const uint32_t S_v = bctx->S_v; + const uint32_t n_batch = bctx->n_batch; + + if (n_batch == 1) { + hvx_vmem(head->g_f32[curr_buf] + 0) = hvx_vmem(bctx->vtcm_g_raw + 0); + hvx_vmem(head->g_f32[curr_buf] + 32) = hvx_vmem(bctx->vtcm_g_raw + 32); + hvx_vmem(head->b_f32[curr_buf] + 0) = hvx_vmem(bctx->vtcm_b_raw + 0); + hvx_vmem(head->b_f32[curr_buf] + 32) = hvx_vmem(bctx->vtcm_b_raw + 32); + } else { + int32_t offsets[32] __attribute__((aligned(128))); + for (int k = 0; k < 32; ++k) { + offsets[k] = k * n_batch * sizeof(float); + } + HVX_Vector vv = *(const HVX_Vector *) offsets; + const size_t rt_g = (size_t) ((const uint8_t *) bctx->vtcm_g_raw + i * sizeof(float)); + const size_t rt_b = (size_t) ((const uint8_t *) bctx->vtcm_b_raw + i * sizeof(float)); + const size_t mu = 64 * n_batch * sizeof(float); + + Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 0), rt_g, mu, vv); + Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 32), rt_g + 32 * n_batch * sizeof(float), mu, vv); + Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 0), rt_b, mu, vv); + Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 32), rt_b + 32 * n_batch * sizeof(float), mu, vv); + } + + const uint32_t t_chunk = bctx->c * 64; + const uint32_t valid_tokens = hex_smin(64, bctx->kparams->n_tokens - t_chunk); + if (valid_tokens < 64) { + for (uint32_t t = valid_tokens; t < 64; ++t) { + head->g_f32[curr_buf][t] = 0.0f; + head->b_f32[curr_buf][t] = 0.0f; + } + const HVX_Vector vzero = Q6_V_vzero(); + for (uint32_t t = valid_tokens; t < 64; ++t) { + for (uint32_t j = 0; j < S_v; j += 32) { + hvx_vmem(head->q_f32[curr_buf] + t * S_v + j) = vzero; + hvx_vmem(head->k_f32[curr_buf] + t * S_v + j) = vzero; + hvx_vmem(head->v_f32[curr_buf] + t * S_v + j) = vzero; + } + } + } + + const HVX_Vector v_g0 = hvx_vmem(head->g_f32[curr_buf] + 0); + const HVX_Vector v_g1 = hvx_vmem(head->g_f32[curr_buf] + 32); + + HVX_Vector v_gamma0 = hvx_prefix_scan_f32(v_g0, Q6_V_vzero()); + HVX_Vector v_carry = hvx_splat_last_f32(v_gamma0); + HVX_Vector v_gamma1 = hvx_prefix_scan_f32(v_g1, v_carry); + + const HVX_Vector v_zero = Q6_V_vzero(); + const HVX_Vector v_neg20 = hvx_vec_splat_f32(-20.0f); + + hvx_vmem(head->gamma) = hvx_vec_f32_to_f16(v_gamma0, v_gamma1); + + HVX_Vector v_l0 = hvx_vec_exp_f32(hvx_clamp_neg20_0(v_gamma0, v_zero, v_neg20)); + HVX_Vector v_l1 = hvx_vec_exp_f32(hvx_clamp_neg20_0(v_gamma1, v_zero, v_neg20)); + + hvx_vmem(head->lambda_init + 0) = v_l0; + hvx_vmem(head->lambda_init + 32) = v_l1; + hvx_vmem(head->lambda_init_f16) = hvx_vec_f32_to_f16(v_l0, v_l1); + + gdn_f32_to_hmx_row_tiles_and_f16(head->k_row_tiles, head->k_prime_row_tiles, head->k_f16, + head->k_f32[curr_buf], head->lambda_init_f16, 64, S_v); + hmx_interleave_rows_to_tiles(head->k_col_tiles, head->k_f16, 64, S_v, S_v, 0, 64); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); +} + +static void gdn_hvx_phase1b_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + const uint32_t S_v = bctx->S_v; + + gdn_f32_to_hmx_row_tiles_and_f16(head->q_row_tiles, head->q_prime_row_tiles, NULL, + head->q_f32[curr_buf], head->lambda_init_f16, 64, S_v); + + hmx_interleave_cols_to_tiles(head->k_col_tiles_64x128, head->k_f16, 64, S_v, S_v, 2, 0, 64); + + const uint16_t * gamma_u16 = (const uint16_t *) head->gamma; + const HVX_Vector v_gamma = hvx_vmem(head->gamma); + + const HVX_Vector v_zero_f16 = Q6_V_vzero(); + const HVX_Vector v_neg20_f16 = hvx_vec_splat_f16(-20.0f); + const HVX_Vector v_log2e_f16 = hvx_vec_splat_f16(1.4426950408889634f); + const HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0f); + + hvx_vmem(head->decay_m + 0) = Q6_V_vzero(); + hvx_vmem(head->decay_a + 0) = Q6_V_vand_QV(Q6_Q_vsetq2_R(2), v_one_f16); + + for (uint32_t t = 1; t < 63; t += 2) { + uint32_t t0 = t; + uint32_t t1 = t + 1; + + HVX_Vector v_gamma_t0 = Q6_Vh_vsplat_R(gamma_u16[t0]); + HVX_Vector v_gamma_t1 = Q6_Vh_vsplat_R(gamma_u16[t1]); + + HVX_Vector diff0 = hvx_vec_sub_f16_f16(v_gamma_t0, v_gamma); + HVX_Vector diff1 = hvx_vec_sub_f16_f16(v_gamma_t1, v_gamma); + + HVX_VectorPred p_gt0 = Q6_Q_vcmp_gt_VhfVhf(diff0, v_zero_f16); + HVX_VectorPred p_gt1 = Q6_Q_vcmp_gt_VhfVhf(diff1, v_zero_f16); + + diff0 = Q6_V_vmux_QVV(p_gt0, v_zero_f16, diff0); + diff1 = Q6_V_vmux_QVV(p_gt1, v_zero_f16, diff1); + + diff0 = Q6_Vhf_vmax_VhfVhf(v_neg20_f16, diff0); + diff1 = Q6_Vhf_vmax_VhfVhf(v_neg20_f16, diff1); + + HVX_Vector diff_log2e0 = hvx_vec_mul_f16_f16(diff0, v_log2e_f16); + HVX_Vector diff_log2e1 = hvx_vec_mul_f16_f16(diff1, v_log2e_f16); + + HVX_Vector v_exp0 = hvx_vec_exp2_f16(diff_log2e0); + HVX_Vector v_exp1 = hvx_vec_exp2_f16(diff_log2e1); + + HVX_VectorPred mask_lt0 = Q6_Q_vsetq2_R(2 * t0); + HVX_VectorPred mask_lt1 = Q6_Q_vsetq2_R(2 * t1); + + HVX_Vector v_m0 = Q6_V_vand_QV(mask_lt0, v_exp0); + HVX_Vector v_m1 = Q6_V_vand_QV(mask_lt1, v_exp1); + + HVX_VectorPred mask_le0 = Q6_Q_vsetq2_R(2 * (t0 + 1)); + HVX_VectorPred mask_le1 = Q6_Q_vsetq2_R(2 * (t1 + 1)); + + HVX_VectorPred mask_diag0 = Q6_Q_and_QQn(mask_le0, mask_lt0); + HVX_VectorPred mask_diag1 = Q6_Q_and_QQn(mask_le1, mask_lt1); + + HVX_Vector v_a0 = Q6_V_vmux_QVV(mask_diag0, v_one_f16, v_m0); + HVX_Vector v_a1 = Q6_V_vmux_QVV(mask_diag1, v_one_f16, v_m1); + + hvx_vmem(head->decay_m + t0 * 64) = v_m0; + hvx_vmem(head->decay_a + t0 * 64) = v_a0; + hvx_vmem(head->decay_m + t1 * 64) = v_m1; + hvx_vmem(head->decay_a + t1 * 64) = v_a1; + } + + { + HVX_Vector v_gamma_t = Q6_Vh_vsplat_R(gamma_u16[63]); + HVX_Vector diff = hvx_vec_sub_f16_f16(v_gamma_t, v_gamma); + HVX_VectorPred p_gt = Q6_Q_vcmp_gt_VhfVhf(diff, v_zero_f16); + diff = Q6_V_vmux_QVV(p_gt, v_zero_f16, diff); + diff = Q6_Vhf_vmax_VhfVhf(v_neg20_f16, diff); + + HVX_Vector diff_log2e = hvx_vec_mul_f16_f16(diff, v_log2e_f16); + HVX_Vector v_exp = hvx_vec_exp2_f16(diff_log2e); + + HVX_VectorPred mask_lt_t = Q6_Q_vsetq2_R(2 * 63); + HVX_Vector v_m = Q6_V_vand_QV(mask_lt_t, v_exp); + + HVX_VectorPred mask_le_t = Q6_Q_vcmp_eq_VhVh(v_zero_f16, v_zero_f16); + HVX_VectorPred mask_diag = Q6_Q_and_QQn(mask_le_t, mask_lt_t); + HVX_Vector v_a = Q6_V_vmux_QVV(mask_diag, v_one_f16, v_m); + + hvx_vmem(head->decay_m + 63 * 64) = v_m; + hvx_vmem(head->decay_a + 63 * 64) = v_a; + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_PREP, info); +} + +static void gdn_hvx_phase2_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + + gdn_unpack_64x64_tiles_to_vectors(head->rows_kk, head->kk_tiles); + + gdn_build_inv_l_blocks( + head->inv_row_tiles, + head->rows_kk, + head->decay_m, + head->b_f32[curr_buf], + (__fp16 *) head->vtcm_m, + (__fp16 *) head->vtcm_m + HMX_FP16_TILE_N_ELMS + ); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_SOLVE, info); +} + +static void gdn_hvx_phase3_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + const uint32_t S_v = bctx->S_v; + + gdn_unpack_64xS_tiles_to_f32(head->v_inter_f32, head->v_inter_tiles, S_v); + + HVX_VectorAlias local_b[2]; + local_b[0].v = hvx_vmem(head->b_f32[curr_buf] + 0); + local_b[1].v = hvx_vmem(head->b_f32[curr_buf] + 32); + + for (uint32_t t = 0; t < 64; ++t) { + HVX_Vector vb = hvx_vec_splat_f32(local_b[t / 32].fp32[t % 32]); + for (uint32_t j = 0; j < S_v; j += 64) { + HVX_Vector vv0 = hvx_vmem(head->v_f32[curr_buf] + t * S_v + j + 0); + HVX_Vector vv1 = (j + 32 < S_v) ? hvx_vmem(head->v_f32[curr_buf] + t * S_v + j + 32) : Q6_V_vzero(); + HVX_Vector vi0 = hvx_vmem(head->v_inter_f32 + t * S_v + j + 0); + HVX_Vector vi1 = (j + 32 < S_v) ? hvx_vmem(head->v_inter_f32 + t * S_v + j + 32) : Q6_V_vzero(); + + HVX_Vector vp0 = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv0, vi0), vb); + HVX_Vector vp1 = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(vv1, vi1), vb); + + hvx_vmem(head->v_prime_f16 + t * S_v + j) = hvx_vec_f32_to_f16(vp0, vp1); + } + } + + hmx_interleave_cols_to_tiles(head->v_prime_col_tiles, head->v_prime_f16, 64, S_v, S_v, 2, 0, 64); + + gdn_unpack_64x64_tiles_to_vectors(head->rows_qk, head->qk_tiles); + for (uint32_t t = 0; t < 64; ++t) { + HVX_Vector v_decay_a = hvx_vmem(head->decay_a + t * 64); + head->rows_a[t] = hvx_vec_mul_f16_f16(head->rows_qk[t], v_decay_a); + } + gdn_pack_64x64_vectors_to_tiles(head->a_row_tiles, head->rows_a); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_V_PREP, info); +} + +static void gdn_hvx_phase4_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t S_v = bctx->S_v; + + gdn_unpack_64xS_tiles_to_f16(head->delta_f16, head->delta_tiles, S_v); + hmx_interleave_cols_to_tiles(head->delta_col_tiles, head->delta_f16, 64, S_v, S_v, 2, 0, 64); + + const uint16_t * decay_last = (const uint16_t *) (head->decay_a + 63 * 64); + const HVX_Vector vzero = Q6_V_vzero(); + + for (uint32_t s = 0; s < 64; ++s) { + HVX_Vector vs = Q6_Vh_vsplat_R(decay_last[s]); + HVX_VectorPred p_zero = Q6_Q_vcmp_eq_VhVh(vs, vzero); + for (uint32_t j = 0; j < S_v; j += 64) { + HVX_Vector vd = hvx_vmem(head->delta_f16 + s * S_v + j); + HVX_Vector prod = hvx_vec_mul_f16_f16(vd, vs); + hvx_vmem(head->d_f16 + s * S_v + j) = Q6_V_vmux_QVV(p_zero, vzero, prod); + } + } + + gdn_pack_d_t_row_tiles(head->d_row_tiles, head->d_f16, S_v, head->vtcm_m, head->vtcm_tmp); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_D_PREP, info); +} + +static void gdn_hvx_phase5_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t curr_buf = bctx->curr_buf; + const uint32_t S_v = bctx->S_v; + const float scale = bctx->scale; + + gdn_unpack_64xS_tiles_to_f32(head->o_inter_f32, head->o_inter_tiles, S_v); + gdn_unpack_64xS_tiles_to_f32(head->o_intra_f32, head->o_intra_tiles, S_v); + + HVX_Vector vscale = hvx_vec_splat_f32(scale); + for (uint32_t j = 0; j < 64 * S_v / 32; ++j) { + HVX_Vector vi = hvx_vmem(head->o_inter_f32 + j * 32); + HVX_Vector va = hvx_vmem(head->o_intra_f32 + j * 32); + hvx_vmem(head->o_f32[curr_buf] + j * 32) = hvx_vec_mul_f32_f32(hvx_vec_add_f32_f32(vi, va), vscale); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_OUT, info); +} + +static void gdn_hvx_phase6_worker(unsigned int n, unsigned int i, void * data) { + (void) n; + struct htp_gdn_batch_context * bctx = (struct htp_gdn_batch_context *) data; + struct htp_thread_trace * tr = &bctx->octx->ctx->trace[i]; + const uint16_t info = (uint16_t) bctx->c; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_GDN_STATE, info); + + struct htp_gdn_head_ptrs * head = &bctx->heads[i]; + const uint32_t S_v = bctx->S_v; + const uint32_t c = bctx->c; + const uint32_t n_chunks = bctx->kparams->n_chunks; + + gdn_unpack_SxS_tiles_to_f32(head->s_update_f32, head->s_update_tiles, S_v); + + HVX_VectorAlias last_lambda; + last_lambda.v = hvx_vmem(head->lambda_init + 32); + HVX_Vector v_l_final = hvx_vec_splat_f32(last_lambda.fp32[31]); + + for (uint32_t j = 0; j < S_v * S_v / 32; ++j) { + HVX_Vector vs_old = hvx_vmem(head->s_state + j * 32); + HVX_Vector vsu = hvx_vmem(head->s_update_f32 + j * 32); + hvx_vmem(head->s_state + j * 32) = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vs_old, v_l_final), vsu); + } + + if (c + 1 < n_chunks) { + gdn_pack_s_col_tiles(head->s_col_tiles, head->s_f16, head->s_state, S_v); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_GDN_STATE, info); +} + + +static int gated_delta_net_f32_hmx_chunked( + struct htp_ops_context * octx, + const struct htp_gdn_kernel_params * kparams, + uint32_t row_start, + uint32_t nrows +) { + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * v = octx->src[2]; + const struct htp_tensor * g = octx->src[3]; + const struct htp_tensor * beta = octx->src[4]; + const struct htp_tensor * state = octx->src[5]; + const struct htp_tensor * dst = octx->dst; + const struct htp_tensor * dst_cache = octx->dsts[1]; + + const uint32_t S_v = kparams->S_v; + const uint32_t H = kparams->H; + const uint32_t n_tokens = kparams->n_tokens; + const float scale = kparams->scale; + const uint32_t chunk_size = kparams->chunk_size; + const uint32_t n_chunks = kparams->n_chunks; + const uint32_t n_sv_tiles = S_v / 32; + + struct htp_gdn_hmx_vtcm_layout L; + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, kparams->n_heads_batch, kparams->n_threads, kparams->pipeline != 0); + + if (L.total_bytes > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + + float * vtcm_g_raw[2] = { + VTCM_LAYOUT_PTR(float, vtcm_base, L.off_g_raw[0]), + L.pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L.off_g_raw[1]) : VTCM_LAYOUT_PTR(float, vtcm_base, L.off_g_raw[0]) + }; + float * vtcm_b_raw[2] = { + VTCM_LAYOUT_PTR(float, vtcm_base, L.off_b_raw[0]), + L.pipeline ? VTCM_LAYOUT_PTR(float, vtcm_base, L.off_b_raw[1]) : VTCM_LAYOUT_PTR(float, vtcm_base, L.off_b_raw[0]) + }; + + uint8_t * vtcm_scales_1 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L.off_scales_1); + hmx_init_column_scales(vtcm_scales_1, Q6_V_vsplat_R(0x3c00)); + + hmx_queue_t hmx_q = octx->ctx->hmx_queue; + dma_queue * dma_q = octx->ctx->dma[0]; + work_queue_t wp = octx->ctx->work_queue; + + struct htp_gdn_head_ptrs heads[8]; + struct htp_gdn_hmx_gemm_task gemm_tasks[8][9]; + + uint32_t n_batch = 1; + for (uint32_t r = row_start; r < row_start + nrows; r += n_batch) { + const uint32_t head_in_seq = fastmodulo(r, H, &kparams->div_H); + const uint32_t iv3 = fastdiv(r, &kparams->div_H); + const uint32_t heads_left_in_seq = H - head_in_seq; + const uint32_t heads_left_in_range = (row_start + nrows) - r; + n_batch = hex_smin((uint32_t) kparams->n_heads_batch, hex_smin(heads_left_in_seq, heads_left_in_range)); + + for (uint32_t h = 0; h < n_batch; ++h) { + gdn_init_head_ptrs(&heads[h], &L, vtcm_base, h, head_in_seq, iv3, + q, k, v, state, dst, dst_cache, kparams, S_v, H, n_tokens, chunk_size); + } + + struct htp_gdn_batch_context bctx; + bctx.heads = heads; + bctx.vtcm_g_raw = NULL; + bctx.vtcm_b_raw = NULL; + bctx.curr_buf = 0; + bctx.c = 0; + bctx.n_batch = n_batch; + bctx.S_v = S_v; + bctx.scale = scale; + bctx.octx = octx; + bctx.kparams = kparams; + + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_push(dma_q, dma_make_data(heads[h].s_state, heads[h].state_in_dma), + S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); + } + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + } + + if (n_chunks > 0) { + work_queue_run(wp, gdn_hvx_init_state_worker, &bctx, n_batch); + + const uint32_t chunk0_tokens = hex_smin(chunk_size, n_tokens); + for (uint32_t h = 0; h < n_batch; ++h) { + gdn_dma_push_chunk_inputs(dma_q, heads[h].q_f32[0], heads[h].k_f32[0], heads[h].v_f32[0], + q, k, v, heads[h].iq3, heads[h].iq1, heads[h].ik3, heads[h].ik1, + heads[h].iv3, heads[h].iv1, 0, chunk0_tokens, S_v); + } + gdn_dma_push_chunk_gb(dma_q, vtcm_g_raw[0], vtcm_b_raw[0], g, beta, iv3, head_in_seq, 0, chunk0_tokens, n_batch); + } + + for (uint32_t c = 0; c < n_chunks; ++c) { + const uint32_t curr_buf = c & 1; + const uint32_t next_buf = (c + 1) & 1; + const uint32_t t_chunk = c * chunk_size; + + bctx.curr_buf = curr_buf; + bctx.c = c; + bctx.vtcm_g_raw = vtcm_g_raw[curr_buf]; + bctx.vtcm_b_raw = vtcm_b_raw[curr_buf]; + + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + } + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + if (c + 1 < n_chunks) { + const uint32_t next_t_chunk = (c + 1) * chunk_size; + const uint32_t next_tokens = hex_smin(chunk_size, n_tokens - next_t_chunk); + for (uint32_t h = 0; h < n_batch; ++h) { + gdn_dma_push_chunk_inputs(dma_q, heads[h].q_f32[next_buf], heads[h].k_f32[next_buf], heads[h].v_f32[next_buf], + q, k, v, heads[h].iq3, heads[h].iq1, heads[h].ik3, heads[h].ik1, + heads[h].iv3, heads[h].iv1, next_t_chunk, next_tokens, S_v); + } + gdn_dma_push_chunk_gb(dma_q, vtcm_g_raw[next_buf], vtcm_b_raw[next_buf], + g, beta, iv3, head_in_seq, next_t_chunk, next_tokens, n_batch); + } + + if (c > 0) { + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + } + } + + work_queue_run(wp, gdn_hvx_phase1a_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][0], heads[h].k_row_tiles, heads[h].k_col_tiles, heads[h].kk_tiles, 2, 2, n_sv_tiles, vtcm_scales_1); + } + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][2], heads[h].k_prime_row_tiles, heads[h].s_col_tiles, heads[h].v_inter_tiles, 2, n_sv_tiles, n_sv_tiles, vtcm_scales_1); + } + + work_queue_run(wp, gdn_hvx_phase1b_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][1], heads[h].q_row_tiles, heads[h].k_col_tiles, heads[h].qk_tiles, 2, 2, n_sv_tiles, vtcm_scales_1); + } + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][3], heads[h].q_prime_row_tiles, heads[h].s_col_tiles, heads[h].o_inter_tiles, 2, n_sv_tiles, n_sv_tiles, vtcm_scales_1); + } + + work_queue_run(wp, gdn_hvx_phase2_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task( + hmx_q, &gemm_tasks[h][7], + (__fp16 *) heads[h].vtcm_m, + heads[h].inv_row_tiles + 0 * HMX_FP16_TILE_N_ELMS, + (__fp16 *) heads[h].vtcm_tmp, + 1, 1, 1, vtcm_scales_1 + ); + } + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task( + hmx_q, &gemm_tasks[h][8], + (__fp16 *) heads[h].vtcm_m + HMX_FP16_TILE_N_ELMS, + (__fp16 *) heads[h].vtcm_tmp, + heads[h].inv_row_tiles + 2 * HMX_FP16_TILE_N_ELMS, + 1, 1, 1, vtcm_scales_1 + ); + } + + work_queue_run(wp, gdn_hvx_phase3_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][4], heads[h].inv_row_tiles, heads[h].v_prime_col_tiles, heads[h].delta_tiles, 2, n_sv_tiles, 2, vtcm_scales_1); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + work_queue_run(wp, gdn_hvx_phase4_worker, &bctx, n_batch); + + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][5], heads[h].a_row_tiles, heads[h].delta_col_tiles, heads[h].o_intra_tiles, 2, n_sv_tiles, 2, vtcm_scales_1); + } + for (uint32_t h = 0; h < n_batch; ++h) { + htp_gdn_push_hmx_gemm_task(hmx_q, &gemm_tasks[h][6], heads[h].d_row_tiles, heads[h].k_col_tiles_64x128, heads[h].s_update_tiles, n_sv_tiles, n_sv_tiles, 2, vtcm_scales_1); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + work_queue_run(wp, gdn_hvx_phase5_worker, &bctx, n_batch); + + const uint32_t valid_tokens = hex_smin(chunk_size, n_tokens - t_chunk); + for (uint32_t h = 0; h < n_batch; ++h) { + const dma_addr_t attn_chunk_dma = dst->data + + ((uint64_t) heads[h].iv3 * n_tokens * H + (uint64_t) t_chunk * H + heads[h].iv1) * S_v * sizeof(float); + dma_queue_push(dma_q, dma_make_data(attn_chunk_dma, heads[h].o_f32[curr_buf]), + dst->nb[1], S_v * sizeof(float), S_v * sizeof(float), valid_tokens); + } + + for (uint32_t h = 0; h < n_batch; ++h) { + hmx_queue_pop(hmx_q); + } + + work_queue_run(wp, gdn_hvx_phase6_worker, &bctx, n_batch); + } + + if (n_chunks > 0) { + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + } + } + + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_push(dma_q, dma_make_data(heads[h].state_out_dma, heads[h].s_state), + S_v * sizeof(float), S_v * sizeof(float), S_v * sizeof(float), S_v); + } + for (uint32_t h = 0; h < n_batch; ++h) { + dma_queue_pop(dma_q); + } + } + + dma_queue_flush(dma_q); + return HTP_STATUS_OK; +} + +int op_gated_delta_net(struct htp_ops_context * octx) { + const struct htp_tensor * dst = octx->dst; + + for (int i = 0; i < 6; i++) { + if (htp_tensor_is_extended(octx->src[i])) { + return HTP_STATUS_NO_SUPPORT; + } + } + if (htp_tensor_is_extended(octx->dst)) { + return HTP_STATUS_NO_SUPPORT; + } + if (octx->dsts[1]) { + const struct htp_tensor * dst_cache = octx->dsts[1]; + if (dst_cache->type != HTP_TYPE_F32 || htp_tensor_is_extended(dst_cache)) { + return HTP_STATUS_NO_SUPPORT; + } + } + + const struct htp_gdn_kernel_params * kparams = (const struct htp_gdn_kernel_params *) octx->kernel_params; + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t S_v = kparams->S_v; + const uint32_t n_tokens = kparams->n_tokens; + const uint32_t total_rows = kparams->total_rows; + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && + ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, + can_split ? 1 : 0, + octx->ctx->mdev.idx, + octx->ctx->mdev.count, + &octx->ctx->mdev.count_div + ); + row_start = range.start; + nrows = range.count; + } else if (octx->op_params[1] != 0) { + row_start = octx->op_params[1]; + nrows = octx->op_params[2]; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + if (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) { + return gated_delta_net_f32_hmx_chunked(octx, kparams, row_start, nrows); + } + + const uint32_t n_threads = (nrows < octx->n_threads) ? nrows : octx->n_threads; + + struct htp_gdn_context gctx; + gctx.octx = octx; + gctx.kparams = kparams; + gctx.row_start = row_start; + gctx.nrows = nrows; + + htp_gdn_vtcm_layout_build(&gctx.layout, S_v, n_threads); + gctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, gctx.layout.off_state); + + if (gctx.layout.total_bytes > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + FARF(HIGH, "gated-delta-net-f32: q(%ux%ux%ux%u) k(%ux%ux%ux%u) v(%ux%ux%ux%u) state(%ux%ux%ux%u) -> (%ux%ux%ux%u) : " + "vtcm-size %zu n_threads %u\n", + octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], + octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], + octx->src[2]->ne[0], octx->src[2]->ne[1], octx->src[2]->ne[2], octx->src[2]->ne[3], + octx->src[5]->ne[0], octx->src[5]->ne[1], octx->src[5]->ne[2], octx->src[5]->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + gctx.layout.total_bytes, n_threads); + + if (n_tokens == 1) { + work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_tg_thread, &gctx, n_threads); + } else { + work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_pp_thread, &gctx, n_threads); + } + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h new file mode 100644 index 0000000..0625c4d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h @@ -0,0 +1,297 @@ +#ifndef HTP_GATED_DELTA_NET_OPS_H +#define HTP_GATED_DELTA_NET_OPS_H + +#include +#include +#include + +#include "hex-fastdiv.h" +#include "hex-common.h" +#include "htp-vtcm.h" + +#define HTP_GDN_MAX_SV 128 +#define HTP_GDN_CHUNK_SIZE 64 +#define HTP_GDN_MIN_TOKENS 8 + +#ifndef HMX_FP16_TILE_SIZE +#define HMX_FP16_TILE_SIZE 2048 +#endif + +enum htp_gdn_kernel_type { + HTP_GDN_KERNEL_HVX_RECURRENT = 0, + HTP_GDN_KERNEL_HMX_CHUNKED = 1, +}; + +struct htp_gdn_kernel_params { + uint8_t kernel_type; + uint8_t pipeline; + uint16_t chunk_size; + uint16_t n_chunks; + uint16_t n_heads_batch; + + uint32_t n_threads; + uint32_t S_v; + uint32_t H; + uint32_t n_tokens; + uint32_t n_seqs; + uint32_t K; + + uint32_t total_rows; + uint32_t kda; + uint32_t vtcm_size; + uint32_t state_seq_stride; + uint32_t state_size_per_snap; + + float scale; + + struct fastdiv_values div_H; + struct fastdiv_values div_q1; + struct fastdiv_values div_k1; + struct fastdiv_values div_rq3; + struct fastdiv_values div_rk3; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_gdn_vtcm_layout { + size_t state_aligned; + size_t bytes_per_thread; + size_t off_state; + size_t total_bytes; +}; + +static inline void htp_gdn_vtcm_layout_build( + struct htp_gdn_vtcm_layout * L, + uint32_t S_v, + uint32_t n_threads +) { + size_t state_bytes = (size_t) S_v * S_v * sizeof(float); + L->state_aligned = hex_round_up(state_bytes, 128); + L->bytes_per_thread = 2 * L->state_aligned; + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_thread * n_threads); + L->total_bytes = off; +} + +struct htp_gdn_hmx_vtcm_layout { + size_t off_s_state; + size_t off_s_f16; + size_t off_s_col_tiles; + size_t off_s_update_f32; + size_t off_s_update_tiles; + + size_t off_q_f32[2]; + size_t off_k_f32[2]; + size_t off_v_f32[2]; + size_t off_g_f32[2]; + size_t off_b_f32[2]; + size_t off_g_raw[2]; + size_t off_b_raw[2]; + size_t off_o_f32[2]; + + size_t off_v_inter_f32; + size_t off_o_inter_f32; + size_t off_o_intra_f32; + size_t off_k_f16; + size_t off_v_prime_f16; + size_t off_delta_f16; + size_t off_d_f16; + + size_t off_q_row_tiles; + size_t off_q_prime_row_tiles; + size_t off_k_row_tiles; + size_t off_k_col_tiles; + size_t off_k_prime_row_tiles; + size_t off_k_col_tiles_64x128; + size_t off_kk_tiles; + size_t off_qk_tiles; + size_t off_v_inter_tiles; + size_t off_o_inter_tiles; + size_t off_inv_row_tiles; + size_t off_a_row_tiles; + size_t off_v_prime_col_tiles; + size_t off_delta_tiles; + size_t off_delta_col_tiles; + size_t off_o_intra_tiles; + size_t off_d_row_tiles; + + size_t off_gamma; + size_t off_lambda_init; + size_t off_decay_m; + size_t off_decay_a; + size_t off_rows_kk; + size_t off_rows_qk; + size_t off_rows_inv; + size_t off_rows_a; + + size_t off_thread_scratch; + size_t off_scales_1; + + size_t state_f32_bytes; + size_t state_f16_bytes; + size_t state_tiles_bytes; + size_t dma_chunk_bytes; + size_t act_f16_bytes; + size_t tile_64xSv_bytes; + size_t tile_64x64_bytes; + + uint32_t n_heads_batch; + uint32_t n_threads; + bool pipeline; + size_t total_bytes; +}; + +static inline void htp_gdn_hmx_vtcm_layout_build( + struct htp_gdn_hmx_vtcm_layout * L, + uint32_t S_v, + uint32_t chunk_size, + uint32_t n_heads_batch, + uint32_t n_threads, + bool pipeline +) { + memset(L, 0, sizeof(*L)); + L->n_heads_batch = n_heads_batch; + L->n_threads = n_threads; + L->pipeline = pipeline; + + const size_t bh = (size_t) n_heads_batch; + const size_t nth = (size_t) (n_threads > 0 ? n_threads : 1); + + const size_t state_f32_sz = hex_round_up(S_v * S_v * sizeof(float), 2048); + const size_t state_f16_sz = hex_round_up(S_v * S_v * sizeof(__fp16), 2048); + const size_t n_sv_tiles = S_v / 32; + const size_t state_tiles_sz = n_sv_tiles * n_sv_tiles * 2048; + + const size_t dma_chunk_sz = hex_round_up(chunk_size * S_v * sizeof(float), 2048); + const size_t dma_scalar_sz = hex_round_up(chunk_size * sizeof(float), 128); + + const size_t act_f16_sz = hex_round_up(chunk_size * S_v * sizeof(__fp16), 2048); + const size_t tile_64xSv_sz = 2 * n_sv_tiles * 2048; + const size_t tile_64x64_sz = 4 * 2048; + + const size_t decay_sz = 64 * 64 * sizeof(__fp16); + const size_t row_vecs_sz = 64 * 128; + + L->state_f32_bytes = state_f32_sz; + L->state_f16_bytes = state_f16_sz; + L->state_tiles_bytes = state_tiles_sz; + L->dma_chunk_bytes = dma_chunk_sz; + L->act_f16_bytes = act_f16_sz; + L->tile_64xSv_bytes = tile_64xSv_sz; + L->tile_64x64_bytes = tile_64x64_sz; + + size_t off = 0; + + VTCM_LAYOUT_ALLOC(off, off_s_state, bh * state_f32_sz); + VTCM_LAYOUT_ALLOC(off, off_s_f16, bh * state_f16_sz); + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_s_col_tiles, bh * state_tiles_sz); + VTCM_LAYOUT_ALLOC(off, off_s_update_f32, bh * state_f32_sz); + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_s_update_tiles, bh * state_tiles_sz); + + VTCM_LAYOUT_ALLOC(off, off_q_f32[0], bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_q_f32[1], bh * dma_chunk_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_k_f32[0], bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_k_f32[1], bh * dma_chunk_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_v_f32[0], bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_v_f32[1], bh * dma_chunk_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_g_f32[0], bh * dma_scalar_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_f32[1], bh * dma_scalar_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_b_f32[0], bh * dma_scalar_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_b_f32[1], bh * dma_scalar_sz, pipeline); + const size_t raw_gb_sz = hex_round_up(bh * chunk_size * sizeof(float), 128); + VTCM_LAYOUT_ALLOC(off, off_g_raw[0], raw_gb_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_raw[1], raw_gb_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_b_raw[0], raw_gb_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_b_raw[1], raw_gb_sz, pipeline); + VTCM_LAYOUT_ALLOC(off, off_o_f32[0], bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_o_f32[1], bh * dma_chunk_sz, pipeline); + + VTCM_LAYOUT_ALLOC(off, off_v_inter_f32, bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC(off, off_o_inter_f32, bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC(off, off_o_intra_f32, bh * dma_chunk_sz); + VTCM_LAYOUT_ALLOC(off, off_k_f16, bh * act_f16_sz); + VTCM_LAYOUT_ALLOC(off, off_v_prime_f16, bh * act_f16_sz); + VTCM_LAYOUT_ALLOC(off, off_delta_f16, bh * act_f16_sz); + VTCM_LAYOUT_ALLOC(off, off_d_f16, bh * act_f16_sz); + + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_q_row_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_q_prime_row_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_k_row_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_k_col_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_k_prime_row_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_k_col_tiles_64x128, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_kk_tiles, bh * tile_64x64_sz); + VTCM_LAYOUT_ALLOC(off, off_qk_tiles, bh * tile_64x64_sz); + VTCM_LAYOUT_ALLOC(off, off_v_inter_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_o_inter_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_inv_row_tiles, bh * tile_64x64_sz); + VTCM_LAYOUT_ALLOC(off, off_a_row_tiles, bh * tile_64x64_sz); + VTCM_LAYOUT_ALLOC(off, off_v_prime_col_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_delta_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_delta_col_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_o_intra_tiles, bh * tile_64xSv_sz); + VTCM_LAYOUT_ALLOC(off, off_d_row_tiles, bh * tile_64xSv_sz); + + VTCM_LAYOUT_ALLOC(off, off_gamma, bh * hex_round_up(chunk_size * sizeof(float), 128)); + VTCM_LAYOUT_ALLOC(off, off_lambda_init, bh * hex_round_up(chunk_size * sizeof(float), 128)); + VTCM_LAYOUT_ALLOC(off, off_decay_m, bh * decay_sz); + VTCM_LAYOUT_ALLOC(off, off_decay_a, bh * decay_sz); + VTCM_LAYOUT_ALLOC(off, off_rows_kk, bh * row_vecs_sz); + VTCM_LAYOUT_ALLOC(off, off_rows_qk, bh * row_vecs_sz); + VTCM_LAYOUT_ALLOC(off, off_rows_inv, bh * row_vecs_sz); + VTCM_LAYOUT_ALLOC(off, off_rows_a, bh * row_vecs_sz); + + const size_t thread_scratch_sz = 64 * 128; + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_thread_scratch, nth * thread_scratch_sz); + off = hex_align_up(off, HMX_FP16_TILE_SIZE); + VTCM_LAYOUT_ALLOC(off, off_scales_1, HMX_FP16_TILE_SIZE); + + L->total_bytes = off; +} + +static inline bool htp_gdn_hmx_solve_layout( + struct htp_gdn_hmx_vtcm_layout * layout_out, + uint32_t S_v, + uint32_t chunk_size, + uint32_t total_rows, + size_t vtcm_budget, + uint32_t n_threads, + bool pipeline, + uint32_t * n_heads_batch_out +) { + uint32_t max_batch = 8; + if (max_batch > total_rows) { + max_batch = total_rows; + } + if (max_batch > n_threads) { + max_batch = n_threads; + } + static const uint32_t candidates[] = { 8, 6, 4, 2, 1 }; + for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) { + uint32_t bh = candidates[i]; + if (bh > max_batch) { + continue; + } + struct htp_gdn_hmx_vtcm_layout L; + htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_threads, pipeline); + if (L.total_bytes <= vtcm_budget) { + *layout_out = L; + *n_heads_batch_out = bh; + return true; + } + } + if (pipeline) { + return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_threads, false, n_heads_batch_out); + } + return false; +} + +#endif // HTP_GATED_DELTA_NET_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.c b/ggml/src/ggml-hexagon/htp/gather-rows.c new file mode 100644 index 0000000..1e88445 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/gather-rows.c @@ -0,0 +1,403 @@ +#include "gather-rows.h" + +#include + +#include "hex-common.h" +#include "hex-profile.h" +#include "hvx-utils.h" +#include "dma-queue.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "work-queue.h" + +typedef void (*gather_rows_compute_fn_t)( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur); + +struct gather_rows_task { + struct htp_ops_context * octx; + const struct htp_tensor * a; + const struct htp_tensor * b; + dma_addr_t dst; + uint32_t r_start; + uint32_t rows; + uint32_t rows_per_thread; + const struct htp_gather_rows_params * kparams; + gather_rows_compute_fn_t compute; +}; + +static inline __attribute__((always_inline)) void gather_rows_dma_in( + dma_queue * q, const struct gather_rows_task * task, + uint8_t * region, uint32_t r, uint32_t nr, uint32_t elem_size) { + const struct htp_tensor * a = task->a; + const struct htp_tensor * b = task->b; + const struct htp_gather_rows_params * kparams = task->kparams; + + const size_t a_size = (nr - 1) * a->nb[1] + kparams->na * elem_size; + dma_queue_push(q, dma_make_data(region, a->data + r * a->nb[1]), a_size, a_size, a_size, 1); + if (b) { + if (kparams->b_dense) { + const size_t b_size = (nr - 1) * b->nb[1] + kparams->nb * elem_size; + dma_queue_push(q, dma_make_data(region + kparams->v1, b->data + r * b->nb[1]), b_size, b_size, b_size, 1); + } else { + const size_t b_size = (nr - 1) * b->nb[1] + elem_size; + for (uint32_t k = 0; k < kparams->nb; k++) { + dma_queue_push(q, dma_make_data(region + kparams->v1 + k * kparams->span1, + b->data + k * b->nb[0] + r * b->nb[1]), b_size, b_size, b_size, 1); + } + } + } +} + +static inline void gather_rows_dma_out(dma_queue * q, const struct gather_rows_task * task, + uint8_t * out, uint32_t r, uint32_t nr, uint32_t elem_size) { + const struct htp_gather_rows_params * kparams = task->kparams; + const size_t out_size = nr * kparams->ne * elem_size; + dma_queue_push(q, dma_make_data(task->dst + r * kparams->ne * elem_size, out), out_size, out_size, out_size, 1); +} + +static __attribute__((noinline)) void gather_rows_init_tables_f32( + HVX_Vector * tab, HVX_Vector * inc, + const struct htp_gather_rows_params * kparams, + const struct htp_tensor * a, const struct htp_tensor * b) { + + const int32_t s_a = kparams->rg * a->nb[1]; + const int32_t s_b = b ? kparams->rg * b->nb[1] : 0; + const uint32_t b_nb1 = b ? b->nb[1] : 0; + uint32_t row = 0; + uint32_t col = 0; + int32_t t_buf[32] __attribute__((aligned(128))); + int32_t s_buf[32] __attribute__((aligned(128))); + for (uint32_t k = 0; k < kparams->vg; k++) { + for (uint32_t lane = 0; lane < 32; lane++) { + if (col < kparams->na) { + t_buf[lane] = row * a->nb[1] + col * 4; + s_buf[lane] = s_a; + } else { + t_buf[lane] = kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1; + s_buf[lane] = s_b; + } + if (++col == kparams->ne) { + col = 0; + row++; + } + } + tab[k] = *(const HVX_Vector *) t_buf; + inc[k] = *(const HVX_Vector *) s_buf; + } +} + +static __attribute__((noinline)) void gather_rows_init_tables_f16( + HVX_Vector * tab, HVX_Vector * inc, + const struct htp_gather_rows_params * kparams, + const struct htp_tensor * a, const struct htp_tensor * b) { + + const int16_t s_a = (int16_t) (kparams->rg * a->nb[1]); + const int16_t s_b = (int16_t) (b ? kparams->rg * b->nb[1] : 0); + const uint32_t b_nb1 = b ? b->nb[1] : 0; + uint32_t row = 0; + uint32_t col = 0; + int16_t t_buf[64] __attribute__((aligned(128))); + int16_t s_buf[64] __attribute__((aligned(128))); + for (uint32_t k = 0; k < kparams->vg; k++) { + for (uint32_t lane = 0; lane < 64; lane++) { + if (col < kparams->na) { + t_buf[lane] = (int16_t) (row * a->nb[1] + col * 2); + s_buf[lane] = s_a; + } else { + t_buf[lane] = (int16_t) (kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1); + s_buf[lane] = s_b; + } + if (++col == kparams->ne) { + col = 0; + row++; + } + } + tab[k] = *(const HVX_Vector *) t_buf; + inc[k] = *(const HVX_Vector *) s_buf; + } +} + +static void gather_rows_compute_f32_vg1( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + (void) cur; + const uint32_t region_size = kparams->region_size; + HVX_Vector vcur = tab[0]; + const HVX_Vector vinc = inc[0]; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur); + vcur = Q6_Vw_vadd_VwVw(vcur, vinc); + } +} + +static void gather_rows_compute_f32_vgn( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + const uint32_t vg = kparams->vg; + const uint32_t region_size = kparams->region_size; + for (uint32_t k = 0; k < vg; k++) { + cur[k] = tab[k]; + } + uint32_t k = 0; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]); + cur[k] = Q6_Vw_vadd_VwVw(cur[k], inc[k]); + if (++k == vg) { + k = 0; + } + } +} + +static void gather_rows_compute_f16_vg1( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + (void) cur; + const uint32_t region_size = kparams->region_size; + HVX_Vector vcur = tab[0]; + const HVX_Vector vinc = inc[0]; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur); + vcur = Q6_Vh_vadd_VhVh(vcur, vinc); + } +} + +static void gather_rows_compute_f16_vgn( + uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec, + const struct htp_gather_rows_params * restrict kparams, + const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) { + const uint32_t vg = kparams->vg; + const uint32_t region_size = kparams->region_size; + for (uint32_t k = 0; k < vg; k++) { + cur[k] = tab[k]; + } + uint32_t k = 0; + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]); + cur[k] = Q6_Vh_vadd_VhVh(cur[k], inc[k]); + if (++k == vg) { + k = 0; + } + } +} + +static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct gather_rows_task * task = (const struct gather_rows_task *) data; + const struct htp_gather_rows_params * kparams = task->kparams; + const struct htp_tensor * a = task->a; + const struct htp_tensor * b = task->b; + + const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows); + const uint32_t r_beg = task->r_start + r_offset; + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows); + if (r_beg >= r_end) { + return; + } + + struct htp_thread_trace * tr = &task->octx->ctx->trace[ith]; + + struct htp_gather_rows_vtcm_layout layout; + htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 4); + + uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice); + HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab); + HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc); + HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur); + + uint8_t * region[2] = { + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0), + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1), + }; + uint8_t * out[2] = { + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0), + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1), + }; + + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + gather_rows_init_tables_f32(tab, inc, kparams, a, b); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + + dma_queue * q = task->octx->ctx->dma[ith]; + const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); + const uint32_t nr_max = kparams->nr_max; + const uint32_t nr0 = MIN(nr_max, r_end - r_beg); + + dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[0], r_beg, nr0, 4); + + const uint32_t r1 = r_beg + nr_max; + if (r1 < r_end) { + const uint32_t nr1 = MIN(nr_max, r_end - r1); + dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[1], r1, nr1, 4); + } + + uint32_t buf = 0; + for (uint32_t r = r_beg; r < r_end; r += nr_max) { + const uint32_t nr = MIN(nr_max, r_end - r); + const uint32_t next_buf = buf ^ 1; + + dma_queue_pop(q); // complete out + for (uint32_t i = 0; i < n_in; i++) { + dma_queue_pop(q); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + const uint32_t nvec = (nr * kparams->ne + 31) >> 5; + task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur); + hvx_gather_sync(out[buf]); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + + gather_rows_dma_out(q, task, out[buf], r, nr, 4); + + const uint32_t r_prefetch = r + 2 * nr_max; + if (r_prefetch < r_end) { + const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch); + gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 4); + } + + buf = next_buf; + } + dma_queue_flush(q); +} + +static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct gather_rows_task * task = (const struct gather_rows_task *) data; + const struct htp_gather_rows_params * kparams = task->kparams; + const struct htp_tensor * a = task->a; + const struct htp_tensor * b = task->b; + + const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows); + const uint32_t r_beg = task->r_start + r_offset; + const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows); + if (r_beg >= r_end) { + return; + } + + struct htp_thread_trace * tr = &task->octx->ctx->trace[ith]; + + struct htp_gather_rows_vtcm_layout layout; + htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 2); + + uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice); + HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab); + HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc); + HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur); + + uint8_t * region[2] = { + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0), + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1), + }; + uint8_t * out[2] = { + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0), + VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1), + }; + + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + gather_rows_init_tables_f16(tab, inc, kparams, a, b); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg); + + dma_queue * q = task->octx->ctx->dma[ith]; + const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0); + + const uint32_t nr_max = kparams->nr_max; + const uint32_t nr0 = MIN(nr_max, r_end - r_beg); + dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[0], r_beg, nr0, 2); + + const uint32_t r1 = r_beg + nr_max; + if (r1 < r_end) { + const uint32_t nr1 = MIN(nr_max, r_end - r1); + dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out + gather_rows_dma_in(q, task, region[1], r1, nr1, 2); + } + + uint32_t buf = 0; + for (uint32_t r = r_beg; r < r_end; r += nr_max) { + const uint32_t nr = MIN(nr_max, r_end - r); + const uint32_t next_buf = buf ^ 1; + + dma_queue_pop(q); // complete out + for (uint32_t i = 0; i < n_in; i++) { + dma_queue_pop(q); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + const uint32_t nvec = (nr * kparams->ne + 63) >> 6; + task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur); + hvx_gather_sync(out[buf]); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); + + gather_rows_dma_out(q, task, out[buf], r, nr, 2); + + const uint32_t r_prefetch = r + 2 * nr_max; + if (r_prefetch < r_end) { + const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch); + gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 2); + } + + buf = next_buf; + } + dma_queue_flush(q); +} + +int htp_gather_rows( + struct htp_ops_context * octx, + const struct htp_tensor * a, + const struct htp_tensor * b, + const struct htp_gather_rows_params * kparams, + uint32_t n_threads) { + + if (!htp_ops_context_set_n_threads(octx, n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size); + + const uint32_t total_rows = kparams->total_rows; + uint32_t r_start = 0; + uint32_t r_count = total_rows; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, kparams->rg, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + r_start = range.start; + r_count = range.count; + } + + if (r_count == 0) { + return HTP_STATUS_OK; + } + + const uint32_t elem_size = (a->type == HTP_TYPE_F16) ? 2 : 4; + + struct gather_rows_task task; + task.octx = octx; + task.a = a; + task.b = b; + task.dst = octx->dst->data; + task.r_start = r_start; + task.rows = r_count; + task.kparams = kparams; + task.rows_per_thread = kparams->rows_per_thread; + + if (elem_size == 2) { + task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn; + } else { + task.compute = (kparams->vg == 1) ? gather_rows_compute_f32_vg1 : gather_rows_compute_f32_vgn; + } + + work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32; + work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads); + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/gather-rows.h b/ggml/src/ggml-hexagon/htp/gather-rows.h new file mode 100644 index 0000000..c08ef4d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/gather-rows.h @@ -0,0 +1,189 @@ +#ifndef HTP_GATHER_ROWS_H +#define HTP_GATHER_ROWS_H + +#include "hex-common.h" +#include "htp-vtcm.h" +#include +#include + +#ifndef GGML_MAX_DIMS +#define GGML_MAX_DIMS 4 +#endif + +struct htp_gather_rows_params { + uint32_t na; + uint32_t nb; + uint32_t ne; + uint32_t rg; + uint32_t vg; + uint32_t nr_max; + uint32_t v1; + uint32_t span1; + uint32_t region_size; + uint32_t slice; + uint32_t total_rows; + uint32_t rows_per_thread; + uint8_t b_dense; + uint8_t pad[3]; +}; + +struct htp_gather_rows_vtcm_layout { + size_t off_tab; + size_t off_inc; + size_t off_cur; + size_t off_region0; + size_t off_out0; + size_t off_region1; + size_t off_out1; + size_t total_per_thread; +}; + +static inline void htp_gather_rows_vtcm_layout_build( + struct htp_gather_rows_vtcm_layout * L, + uint32_t vg, + uint32_t region_size, + uint32_t nr_max, + uint32_t ne, + uint32_t elem_size) { + + const size_t vec_bytes = (size_t) vg * 128; + const size_t region_aligned = hex_round_up(region_size, 128); + const size_t out_aligned = hex_round_up(region_aligned + (size_t) nr_max * ne * elem_size, 128) - region_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_tab, vec_bytes); + VTCM_LAYOUT_ALLOC(off, off_inc, vec_bytes); + VTCM_LAYOUT_ALLOC(off, off_cur, vec_bytes); + VTCM_LAYOUT_ALLOC(off, off_region0, region_aligned); + VTCM_LAYOUT_ALLOC(off, off_out0, out_aligned); + VTCM_LAYOUT_ALLOC(off, off_region1, region_aligned); + VTCM_LAYOUT_ALLOC(off, off_out1, out_aligned); + L->total_per_thread = off; +} + +static inline bool htp_gather_rows_solve_layout( + struct htp_gather_rows_params * kparams, + uint32_t na, uint32_t a_nb1, + uint32_t nb, uint32_t b_nb1, bool b_dense, + uint32_t elem_size, + uint32_t total_rows, + uint32_t n_threads, + uint32_t mdev_count, + uint32_t vtcm_size) { + + const uint32_t max_ne0 = 128 / elem_size; + const uint32_t ne = na + nb; + if (ne == 0 || ne > max_ne0) { + return false; + } + + if (elem_size != 4 && elem_size != 2) { + return false; + } + + const uint32_t v_lanes = 128 / elem_size; + const uint32_t g = hex_gcd_u32(ne, v_lanes); + const uint32_t rg = v_lanes / g; + const uint32_t vg = ne / g; + + const uint32_t slice = (vtcm_size / n_threads) & ~127u; + const uint32_t total_chunks = total_rows / rg; + const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count; + const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count) + ? (chunks_per_dev * rg + (total_rows % rg)) + : total_rows; + const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg); + + const uint32_t fixed = 3 * vg * 128; + const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size; + if (slice <= fixed + 2 * per_row * rg) { + return false; + } + + const uint32_t avail_per_buf = (slice - fixed) / 2; + uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread); + if (elem_size == 2) { + const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0); + if (per_row_region > 0) { + nr_max = MIN(nr_max, (32768u - 256u) / per_row_region); + } + } + nr_max = (nr_max / rg) * rg; + + uint32_t v1 = 0; + uint32_t span1 = 0; + uint32_t region_size = 0; + + while (nr_max > 0) { + v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128); + if (nb > 0) { + if (b_dense) { + span1 = elem_size; + region_size = v1 + (nr_max - 1) * b_nb1 + nb * elem_size; + } else { + span1 = (nr_max - 1) * b_nb1 + elem_size; + region_size = v1 + nb * span1; + } + } else { + span1 = 0; + region_size = v1; + } + + struct htp_gather_rows_vtcm_layout layout; + htp_gather_rows_vtcm_layout_build(&layout, vg, region_size, nr_max, ne, elem_size); + + if ((elem_size != 2 || region_size <= 32768) && layout.total_per_thread <= slice) { + break; + } + + nr_max -= rg; + } + + if (nr_max == 0) { + return false; + } + + kparams->na = na; + kparams->nb = nb; + kparams->ne = ne; + kparams->rg = rg; + kparams->vg = vg; + kparams->nr_max = nr_max; + kparams->v1 = v1; + kparams->span1 = span1; + kparams->region_size = region_size; + kparams->slice = slice; + kparams->total_rows = total_rows; + kparams->rows_per_thread = rows_per_thread; + kparams->b_dense = b_dense ? 1 : 0; + kparams->pad[0] = 0; + kparams->pad[1] = 0; + kparams->pad[2] = 0; + return true; +} + +#ifdef __cplusplus +struct ggml_tensor; +static inline bool htp_gather_rows_tensor_outer_contiguous(const struct ggml_tensor * t) { + size_t next_nb = ((const size_t *) t->nb)[1] * ((const int64_t *) t->ne)[1]; + for (int i = 2; i < GGML_MAX_DIMS; i++) { + if (((const int64_t *) t->ne)[i] != 1 && ((const size_t *) t->nb)[i] != next_nb) { + return false; + } + next_nb *= ((const int64_t *) t->ne)[i]; + } + return true; +} +#else +struct htp_ops_context; +struct htp_tensor; + +int htp_gather_rows( + struct htp_ops_context * octx, + const struct htp_tensor * a, + const struct htp_tensor * b, + const struct htp_gather_rows_params * kparams, + uint32_t n_threads); +#endif + +#endif // HTP_GATHER_ROWS_H diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index 5a1dc93..39acdfd 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -10,19 +10,42 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "dma-queue.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-utils.h" +#include "hvx-quant.h" +#include "get-rows-ops.h" +#include "work-queue.h" + +struct get_rows_context; + +typedef uint32_t (*get_rows_read_index_fn_t)(const void * src1, uint32_t idx, int idx_type); +typedef void (*get_rows_flat_compute_fn_t)(float * dst_spad, const void * src_spad, uint32_t cur_elems); +typedef void (*get_rows_tiled_compute_fn_t)(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row); struct get_rows_context { struct htp_ops_context * octx; - uint32_t src1_nrows_per_thread; - struct fastdiv_values get_rows_div_ne10; - struct fastdiv_values get_rows_div_ne10_ne11; + const struct htp_get_rows_kernel_params * kparams; + struct htp_get_rows_vtcm_layout vtcm_layout; + uint8_t * vtcm_base; + const void * src1_data; + get_rows_read_index_fn_t read_index; + get_rows_flat_compute_fn_t compute_flat; + get_rows_tiled_compute_fn_t compute_tiled; + uint32_t task_start; + uint32_t tasks; + uint32_t tasks_per_thread; + uint32_t tile_size; + uint32_t tile_stride; + uint32_t src0_row_size; + uint32_t dst_row_size; + int idx_type; }; -#define get_rows_preamble \ +#define get_rows_preamble \ const uint32_t ne00 = octx->src[0]->ne[0]; \ const uint32_t ne01 = octx->src[0]->ne[1]; \ const uint32_t ne02 = octx->src[0]->ne[2]; \ @@ -52,73 +75,436 @@ struct get_rows_context { \ const uint32_t nr = ne10 * ne11 * ne12; -static void get_rows_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) { - struct get_rows_context * grctx = (struct get_rows_context *)data; - struct htp_ops_context * octx = grctx->octx; - get_rows_preamble; +static uint32_t get_rows_read_index_ddr( + const void * src1, + uint32_t idx, + int idx_type +) { + const uint32_t off = idx * ((idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t)); + return *(const uint32_t *) ((const uint8_t *) src1 + off); +} - uint64_t qt = HAP_perf_get_qtimer_count(); +static __attribute__((noinline)) uint32_t get_rows_read_index_vtcm( + const void * src1, + uint32_t idx, + int idx_type +) { + const uint32_t off = idx * ((idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t)); + const HVX_Vector v = hvx_vmem((const uint8_t *) src1 + off); // off & ~127u in HW + // vextract is normally slow, but here it avoids an HVX->L2 store and L1D miss + return (uint32_t) Q6_R_vextract_VR(v, off); +} - // parallelize by src1 elements (which correspond to dst rows) - const uint32_t dr = grctx->src1_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr; +static void get_rows_thread_sametype(unsigned int nth, unsigned int ith, void * data) { + struct get_rows_context * grctx = (struct get_rows_context *) data; + struct htp_ops_context * octx = grctx->octx; + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; + get_rows_preamble; - const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); + const uint32_t dr = grctx->tasks_per_thread; + const uint32_t ir0 = grctx->task_start + dr * ith; + if (ir0 >= grctx->task_start + grctx->tasks) { + return; + } + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); + const uint32_t row_size_bytes = grctx->src0_row_size; + dma_queue * dma_q = octx->ctx->dma[ith]; for (uint32_t i = ir0; i < ir1; ++i) { - const uint32_t i12 = fastdiv(i, &grctx->get_rows_div_ne10_ne11); + const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); const uint32_t rem = i - i12 * ne11 * ne10; - const uint32_t i11 = fastdiv(rem, &grctx->get_rows_div_ne10); + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); const uint32_t i10 = rem - i11 * ne10; - const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12; + const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type); + assert(i01 < ne01); + + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); + const uint32_t i02 = i11 - q02 * ne02; + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); + const uint32_t i03 = i12 - q03 * ne03; - uint32_t i01 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr; + const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; + const dma_addr_t dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; - if (i01 >= ne01) { - // invalid index, skip for now to avoid crash - continue; + while (!dma_queue_push(dma_q, dma_make_data(dst_data, src0_data), nb1, nb01, row_size_bytes, 1)) { + dma_queue_pop(dma_q); } + } + dma_queue_flush(dma_q); +} + + + +struct get_rows_flat_task { + dma_addr_t src0_data; + dma_addr_t dst_data; +}; + +static inline struct get_rows_flat_task get_rows_flat_calc_task( + const struct htp_ops_context * octx, + const struct get_rows_context * grctx, + uint32_t i, + bool need_src0 +) { + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; + get_rows_preamble; + + const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); + const uint32_t rem = i - i12 * ne11 * ne10; + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); + const uint32_t i10 = rem - i11 * ne10; + + struct get_rows_flat_task task; + task.src0_data = 0; + task.dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; - const uintptr_t src0_ptr = octx->src[0]->data + i01*nb01 + i11*nb02 + i12*nb03; - const uintptr_t dst_ptr = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; - hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, ne00); + if (need_src0) { + const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type); + assert(i01 < ne01); + + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); + const uint32_t i02 = i11 - q02 * ne02; + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); + const uint32_t i03 = i12 - q03 * ne03; + + task.src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03; } - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "get-rows-f32-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt); + return task; } -int op_get_rows(struct htp_ops_context * octx) { +static void get_rows_thread_flat(unsigned int nth, unsigned int ith, void * data) { + struct get_rows_context * grctx = (struct get_rows_context *) data; + struct htp_ops_context * octx = grctx->octx; get_rows_preamble; - const uint32_t n_threads = MIN(nr, octx->n_threads); + const uint32_t dr = grctx->tasks_per_thread; + const uint32_t ir0 = grctx->task_start + dr * ith; + if (ir0 >= grctx->task_start + grctx->tasks) { + return; + } + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); - if (octx->src[0]->type != HTP_TYPE_F32) { - return HTP_STATUS_NO_SUPPORT; + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; + uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread; + uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread; + const uint32_t src0_bytes = grctx->src0_row_size; + const uint32_t dst_bytes = grctx->dst_row_size; + get_rows_flat_compute_fn_t compute = grctx->compute_flat; + + for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) { + const uint32_t i = ir0 + step; + struct get_rows_flat_task task = get_rows_flat_calc_task(octx, grctx, i, true); + + dma_queue_push(dma_q, + dma_make_data(octx->dst->data, + vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), + dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0); + dma_queue_push(dma_q, + dma_make_data(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size, task.src0_data), + vtcm_layout->src0_spad_half_size, src0_bytes, src0_bytes, 1); + } + + for (uint32_t step = 0; step < ir1 - ir0; ++step) { + const uint32_t i = ir0 + step; + void * dst_spad = (void *) dma_queue_pop(dma_q).src; + void * src_spad = (void *) dma_queue_pop(dma_q).dst; + + struct get_rows_flat_task task = get_rows_flat_calc_task(octx, grctx, i, false); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + compute((float *) dst_spad, src_spad, ne00); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + + dma_queue_push(dma_q, + dma_make_data(task.dst_data, dst_spad), + dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1); + + const uint32_t next_step = step + 2; + if (next_step < ir1 - ir0) { + const uint32_t pi = ir0 + next_step; + struct get_rows_flat_task next_task = get_rows_flat_calc_task(octx, grctx, pi, true); + dma_queue_push(dma_q, + dma_make_data(src_spad, next_task.src0_data), + vtcm_layout->src0_spad_half_size, src0_bytes, src0_bytes, 1); + } + } + + dma_queue_flush(dma_q); +} + + + +struct get_rows_tiled_task { + dma_addr_t tile_src_base; + dma_addr_t dst_data; + uint32_t row; +}; + +static inline struct get_rows_tiled_task get_rows_tiled_calc_task( + const struct htp_ops_context * octx, + const struct get_rows_context * grctx, + uint32_t i, + uint32_t n_k_tiles, + uint32_t tile_size +) { + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; + get_rows_preamble; + + const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); + const uint32_t rem = i - i12 * ne11 * ne10; + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); + const uint32_t i10 = rem - i11 * ne10; + const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type); + assert(i01 < ne01); + + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); + const uint32_t i02 = i11 - q02 * ne02; + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); + const uint32_t i03 = i12 - q03 * ne03; + const uint32_t column_tile = i01 / HTP_TENSOR_TILE_N_ROWS; + const uint32_t row = i01 % HTP_TENSOR_TILE_N_ROWS; + const dma_addr_t matrix = octx->src[0]->data + i02*nb02 + i03*nb03; + + struct get_rows_tiled_task task; + task.tile_src_base = matrix + (column_tile * n_k_tiles) * tile_size; + task.dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; + task.row = row; + return task; +} + +static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) { + struct get_rows_context * grctx = (struct get_rows_context *) data; + struct htp_ops_context * octx = grctx->octx; + get_rows_preamble; + + const uint32_t dr = grctx->tasks_per_thread; + const uint32_t ir0 = grctx->task_start + dr * ith; + if (ir0 >= grctx->task_start + grctx->tasks) { + return; + } + + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); + const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type); + const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; + uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread; + uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread; + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const uint32_t tile_size = grctx->tile_size; + const uint32_t tile_stride = grctx->tile_stride; + const uint32_t dst_bytes = grctx->dst_row_size; + get_rows_tiled_compute_fn_t compute_tiled = grctx->compute_tiled; + + for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) { + const uint32_t i = ir0 + step; + struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size); + + // Dummy writeback to prime the queue with dst descriptor + dma_queue_push(dma_q, + dma_make_data(task.dst_data, dst_spad_base + spad_idx * vtcm_layout->dst_spad_half_size), + dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0); + + // Prefetch row tiles + dma_queue_push(dma_q, + dma_make_data(src_spad_base + spad_idx * vtcm_layout->src0_spad_half_size, task.tile_src_base), + tile_stride, tile_size, tile_size, n_k_tiles); } - if (octx->dst->type != HTP_TYPE_F32) { + for (uint32_t step = 0; step < ir1 - ir0; ++step) { + const uint32_t i = ir0 + step; + float * dst_spad = (float *) dma_queue_pop(dma_q).src; + uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + + struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + compute_tiled(dst_spad, src_spad, n_k_tiles, tile_stride, task.row); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + + // Real writeback of dst_spad + dma_queue_push(dma_q, + dma_make_data(task.dst_data, dst_spad), + dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1); + + const uint32_t next_step = step + 2; + if (next_step < ir1 - ir0) { + const uint32_t ni = ir0 + next_step; + struct get_rows_tiled_task next_task = get_rows_tiled_calc_task(octx, grctx, ni, n_k_tiles, tile_size); + dma_queue_push(dma_q, + dma_make_data(src_spad, next_task.tile_src_base), + tile_stride, tile_size, tile_size, n_k_tiles); + } + } + + dma_queue_flush(dma_q); +} + +static __attribute__((noinline)) void get_rows_dma_src1( + struct htp_ops_context * octx, + uint8_t * vtcm_src1, + uint32_t elem_size +) { + const uint32_t ne10 = octx->src[1]->ne[0]; + const uint32_t ne11 = octx->src[1]->ne[1]; + const uint32_t ne12 = octx->src[1]->ne[2]; + const uint32_t ne13 = octx->src[1]->ne[3]; + const uint32_t nb11 = octx->src[1]->nb[1]; + const uint32_t nb12 = octx->src[1]->nb[2]; + const uint32_t nb13 = octx->src[1]->nb[3]; + + dma_queue * dma_q = octx->ctx->dma[0]; + uint8_t * cur_dst = vtcm_src1; + for (uint32_t i13 = 0; i13 < ne13; ++i13) { + for (uint32_t i12 = 0; i12 < ne12; ++i12) { + const dma_addr_t src_plane = octx->src[1]->data + i12 * nb12 + i13 * nb13; + while (!dma_queue_push(dma_q, dma_make_data(cur_dst, src_plane), + ne10 * elem_size, nb11, ne10 * elem_size, ne11)) { + dma_queue_pop(dma_q); + } + cur_dst += ne10 * ne11 * elem_size; + } + } + dma_queue_flush(dma_q); +} + +int op_get_rows(struct htp_ops_context * octx) { + const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params; + + const uint32_t ne00 = octx->src[0]->ne[0]; + const uint32_t src0_row_size = htp_tensor_get_row_size(octx->src[0]->type, ne00); + if (src0_row_size == 0) { return HTP_STATUS_NO_SUPPORT; } + if (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) { + if (octx->src[0]->type != octx->dst->type) { + return HTP_STATUS_NO_SUPPORT; + } + } else { + if (octx->dst->type != HTP_TYPE_F32) { + return HTP_STATUS_NO_SUPPORT; + } + } + if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) { return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + const struct htp_tensor * dst = octx->dst; + const uint32_t total_tasks = kparams->total_tasks; + const uint32_t dst_row_size = (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) ? src0_row_size : ne00 * sizeof(float); + + uint32_t task_start = 0; + uint32_t tasks = total_tasks; + + if (octx->ctx->mdev.count > 1) { + uint32_t tasks_per_chunk = 1; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst->nb[0], dst_row_size, &tasks_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + task_start = range.start; + tasks = range.count; + } + + if (tasks == 0) { return HTP_STATUS_OK; } + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + struct get_rows_context grctx; grctx.octx = octx; - grctx.get_rows_div_ne10 = init_fastdiv_values(octx->src[1]->ne[0]); - grctx.get_rows_div_ne10_ne11 = init_fastdiv_values(octx->src[1]->ne[0] * octx->src[1]->ne[1]); + grctx.kparams = kparams; + grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; + grctx.compute_flat = NULL; + grctx.compute_tiled = NULL; + grctx.task_start = task_start; + grctx.tasks = tasks; + grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); + grctx.tile_size = 0; + grctx.tile_stride = 0; + grctx.src0_row_size = src0_row_size; + grctx.dst_row_size = dst_row_size; + grctx.idx_type = octx->src[1]->type; + + htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_threads); + + if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) { + FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n", + octx->ctx->vtcm_size, grctx.vtcm_layout.total_bytes); + return HTP_STATUS_INVAL_PARAMS; + } + + work_queue_func_t q_func = NULL; + switch (kparams->kernel_type) { + case HTP_GET_ROWS_KERNEL_SAMETYPE: + q_func = get_rows_thread_sametype; + break; + case HTP_GET_ROWS_KERNEL_TILED: + switch (octx->src[0]->type) { + case HTP_TYPE_F16: grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break; + case HTP_TYPE_Q1_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q1_0_f32; break; + case HTP_TYPE_Q2_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q2_0_f32; break; + case HTP_TYPE_Q8_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q8_0_f32; break; + case HTP_TYPE_Q4_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_0_f32; break; + case HTP_TYPE_Q4_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_k_f32; break; + case HTP_TYPE_Q5_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q5_k_f32; break; + case HTP_TYPE_Q6_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q6_k_f32; break; + default: return HTP_STATUS_NO_SUPPORT; + } + grctx.tile_size = htp_tensor_get_tile_size(octx->src[0]->type); + grctx.tile_stride = htp_tensor_get_tile_padded_size(octx->src[0]->type); + q_func = get_rows_thread_tiled; + break; + case HTP_GET_ROWS_KERNEL_FLAT: + switch (octx->src[0]->type) { + case HTP_TYPE_F16: grctx.compute_flat = hvx_dequantize_row_flat_f16_f32; break; + case HTP_TYPE_BF16: grctx.compute_flat = hvx_dequantize_row_flat_bf16_f32; break; + case HTP_TYPE_Q1_0: grctx.compute_flat = hvx_dequantize_row_flat_q1_0_f32; break; + case HTP_TYPE_Q2_0: grctx.compute_flat = hvx_dequantize_row_flat_q2_0_f32; break; + case HTP_TYPE_Q8_0: grctx.compute_flat = hvx_dequantize_row_flat_q8_0_f32; break; + case HTP_TYPE_Q4_0: grctx.compute_flat = hvx_dequantize_row_flat_q4_0_f32; break; + case HTP_TYPE_Q4_K: grctx.compute_flat = hvx_dequantize_row_flat_q4_k_f32; break; + case HTP_TYPE_Q5_K: grctx.compute_flat = hvx_dequantize_row_flat_q5_k_f32; break; + case HTP_TYPE_Q6_K: grctx.compute_flat = hvx_dequantize_row_flat_q6_k_f32; break; + default: return HTP_STATUS_NO_SUPPORT; + } + q_func = get_rows_thread_flat; + break; + default: + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t elem_size = (grctx.idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t); + const size_t src1_bytes = (size_t) total_tasks * elem_size; + + if (!htp_tensor_is_extended(octx->src[1]) && src1_bytes <= 128 && + htp_tensor_is_contiguous(octx->src[1], elem_size)) { + grctx.src1_data = (const void *) (uintptr_t) octx->src[1]->data; + grctx.read_index = get_rows_read_index_ddr; + } else { + uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, grctx.vtcm_base, grctx.vtcm_layout.off_src1); + get_rows_dma_src1(octx, vtcm_src1, elem_size); + grctx.src1_data = vtcm_src1; + grctx.read_index = get_rows_read_index_vtcm; + } - grctx.src1_nrows_per_thread = (nr + n_threads - 1) / n_threads; + FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n", + octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], + octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], + octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], + grctx.vtcm_layout.src0_bytes_per_thread * n_threads, + grctx.vtcm_layout.dst_bytes_per_thread * n_threads, + kparams->kernel_type, n_threads); - worker_pool_run_func(octx->ctx->worker_pool, get_rows_thread_f32_f32, &grctx, n_threads); + work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h new file mode 100644 index 0000000..4634be2 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h @@ -0,0 +1,91 @@ +#ifndef HTP_GET_ROWS_OPS_H +#define HTP_GET_ROWS_OPS_H + +#include +#include + +#include "hex-fastdiv.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" + +enum htp_get_rows_kernel_type { + HTP_GET_ROWS_KERNEL_SAMETYPE = 0, + HTP_GET_ROWS_KERNEL_TILED, + HTP_GET_ROWS_KERNEL_FLAT, +}; + +struct htp_get_rows_kernel_params { + uint8_t kernel_type; + uint32_t n_threads; + uint32_t total_tasks; + uint32_t vtcm_size; + + // Fastdiv helpers + struct fastdiv_values div_ne10; + struct fastdiv_values div_ne10_ne11; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne03; +}; + +struct htp_get_rows_vtcm_layout { + size_t total_bytes; + size_t off_src1; + size_t src1_bytes; + size_t off_src0; + size_t off_dst; + + size_t src0_bytes_per_thread; + size_t dst_bytes_per_thread; + + size_t src0_spad_half_size; + size_t dst_spad_half_size; +}; + +static inline void htp_get_rows_vtcm_layout_build( + struct htp_get_rows_vtcm_layout * L, + int kernel_type, + int type, + uint32_t ne00, + uint32_t total_tasks, + int idx_type, + uint32_t n_threads) { + + memset(L, 0, sizeof(*L)); + + const size_t elem_size = (idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t); + const size_t src1_raw_bytes = total_tasks * elem_size; + const size_t src1_bytes = ((src1_raw_bytes + 127) & ~127) + 128; + + L->src1_bytes = src1_bytes; + const size_t src1_alloc_bytes = (src1_bytes + 255) & ~255; + + if (kernel_type != HTP_GET_ROWS_KERNEL_SAMETYPE) { + size_t src0_row_size = 0; + if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) { + const size_t tile_stride = htp_tensor_get_tile_padded_size(type); + const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(type); + src0_row_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride; + } else { + src0_row_size = htp_tensor_get_row_size(type, ne00); + } + + L->src0_spad_half_size = (src0_row_size + 255) & ~255; + L->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255; + L->src0_bytes_per_thread = 2 * L->src0_spad_half_size; + L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; + } + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src1, src1_alloc_bytes); + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + L->total_bytes = off; +} + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_GET_ROWS_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.c b/ggml/src/ggml-hexagon/htp/glu-ops.c new file mode 100644 index 0000000..b4edddf --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/glu-ops.c @@ -0,0 +1,1219 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include + +#include +#include + +#include "dma-queue.h" +#include "hvx-utils.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "hex-common.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "glu-ops.h" + +#define htp_glu_preamble \ + const struct htp_tensor * src0 = gctx->octx->src[0]; \ + const struct htp_tensor * src1 = gctx->octx->src[1]; \ + const struct htp_tensor * dst = gctx->octx->dst; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t ne10 = src1 ? src1->ne[0] : 0; \ + const uint32_t ne11 = src1 ? src1->ne[1] : 0; \ + const uint32_t ne12 = src1 ? src1->ne[2] : 0; \ + const uint32_t ne13 = src1 ? src1->ne[3] : 0; \ + \ + const uint32_t nb10 = src1 ? src1->nb[0] : 0; \ + const uint32_t nb11 = src1 ? src1->nb[1] : 0; \ + const uint32_t nb12 = src1 ? src1->nb[2] : 0; \ + const uint32_t nb13 = src1 ? src1->nb[3] : 0; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; + +struct htp_glu_context; + +typedef void (*glu_compute_f32_fn_t)(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx); + +typedef void (*glu_compute_f16_fn_t)(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx); + +struct htp_glu_context { + struct htp_ops_context * octx; + + void * compute; + const char * op_str; + + // Precomputed values + dma_addr_t data_src0; + dma_addr_t data_src1; + dma_addr_t data_dst; + + size_t src0_row_size; + size_t src1_row_size; + size_t dst_row_size; + + size_t src0_row_stride; + size_t src1_row_stride; + + size_t src0_row_size_aligned; + size_t src1_row_size_aligned; + size_t dst_row_size_aligned; + + size_t src0_spad_half_size; + size_t src1_spad_half_size; + size_t dst_spad_half_size; + + uint32_t block; + uint32_t src0_nrows; + uint32_t src0_nrows_per_thread; + uint32_t row_start; + int nc; + + uint8_t * vtcm_src0; + uint8_t * vtcm_src1; + uint8_t * vtcm_dst; + + size_t vtcm_src0_size_per_thread; + size_t vtcm_src1_size_per_thread; + size_t vtcm_dst_size_per_thread; +}; + +#define htp_glu_op_preamble \ + const size_t src0_row_size_aligned = gctx->src0_row_size_aligned; \ + const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \ + const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \ + const int nc = gctx->nc; + +// swiglu(x) = x1 * sigmoid(x0) +static void swiglu_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_sigmoid_f32_aa(dst_ptr, src0_ptr, nc); + hvx_mul_mul_f32_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); + } +} + +// out = x * sigmoid(alpha * x) * (clamp(y, -limit, limit) + 1.f) +static void swiglu_oai_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + const float alpha = ((const float *) (gctx->octx->op_params))[2]; + const float limit = ((const float *) (gctx->octx->op_params))[3]; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + // x (src0_ptr) = std::min(src0_p[k], limit); + hvx_min_scalar_f32((uint8_t *) src0_ptr, src0_ptr, limit, nc); + // y1 (src1_ptr) = std::clamp(src1_p[k], -limit, limit); + hvx_clamp_scalar_f32((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc); + // y (src1_ptr) = y1 + 1.f + hvx_add_scalar_f32((uint8_t *) src1_ptr, src1_ptr, 1.0, nc); + // x1 (dst_ptr) = alpha * x + hvx_mul_scalar_f32(dst_ptr, src0_ptr, alpha, nc); + // x2 (dst_ptr) = sigmoid(x1) = 1/(1+exp(-x1)) + hvx_sigmoid_f32_aa(dst_ptr, dst_ptr, nc); + // out = x * sigmoid(alpha * x) * (y + 1.f) + hvx_mul_mul_f32_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); + } +} + +static void swiglu_clamp_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + const float limit = ((const float *) (gctx->octx->op_params))[3]; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_min_scalar_f32((uint8_t *) src0_ptr, src0_ptr, limit, nc); + hvx_clamp_scalar_f32((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc); + hvx_sigmoid_f32_aa(dst_ptr, src0_ptr, nc); + hvx_mul_mul_f32_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); + } +} + +static const float GELU_COEF_A = 0.044715f; +static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; + +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) { + v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3)); + + HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + // Newton-Raphson with 2 iterations + HVX_Vector two_sf = hvx_vec_splat_f32(2.0f); + HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); + HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( + i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); + r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( + r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); + HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf); + + res = Q6_Vqf32_vmpy_VsfVsf(v3, res); + + return Q6_Vsf_equals_Vqf32(res); +} + +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v, + HVX_Vector one, + HVX_Vector max_exp, + HVX_Vector min_exp) { + const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v); + const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp); + + HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v); + out = Q6_V_vmux_QVV(pred_max, out, one); + return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); +} + +static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const float GELU_COEF_A_TIMES_SQRT = GELU_COEF_A * SQRT_2_OVER_PI; + + const HVX_Vector v_coef_a_times_sqrt = hvx_vec_splat_f32(GELU_COEF_A_TIMES_SQRT); + const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f32(SQRT_2_OVER_PI); + const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); + const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector coef = hvx_vec_mul_f32_f32(x2, v_coef_a_times_sqrt); + coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi); + HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef); + + // y2 = 2 * inner = inner + inner + HVX_Vector y2 = hvx_vec_add_f32_f32(inner, inner); + + // Fast sigmoid approximation (2 iterations) + HVX_Vector sig2y = hvx_vec_fast_sigmoid_f32_guard_2it(y2, v_one, v_max_exp, v_min_exp); + + HVX_Vector gelu_x = hvx_vec_mul_f32_f32(x, sig2y); + vdst[i] = hvx_vec_mul_f32_f32(gelu_x, g); + } + + if (nloe) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector coef = hvx_vec_mul_f32_f32(x2, v_coef_a_times_sqrt); + coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi); + HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef); + + HVX_Vector y2 = hvx_vec_add_f32_f32(inner, inner); + + HVX_Vector sig2y = hvx_vec_fast_sigmoid_f32_guard_2it(y2, v_one, v_max_exp, v_min_exp); + + HVX_Vector gelu_x = hvx_vec_mul_f32_f32(x, sig2y); + HVX_Vector res = hvx_vec_mul_f32_f32(gelu_x, g); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res); + } +} + +static inline void hvx_geglu_quick_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const HVX_Vector v_scale = hvx_vec_splat_f32(1.702f); + const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); + const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + HVX_Vector scaled_x = hvx_vec_mul_f32_f32(x, v_scale); + HVX_Vector sigmoid_x = hvx_vec_fast_sigmoid_f32_guard_2it(scaled_x, v_one, v_max_exp, v_min_exp); + vdst[i] = hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(x, sigmoid_x), g); + } + + if (nloe) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + HVX_Vector scaled_x = hvx_vec_mul_f32_f32(x, v_scale); + HVX_Vector sigmoid_x = hvx_vec_fast_sigmoid_f32_guard_2it(scaled_x, v_one, v_max_exp, v_min_exp); + HVX_Vector result = hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(x, sigmoid_x), g); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), result); + } +} + +static inline void hvx_geglu_erf_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + vdst[i] = hvx_vec_mul_f32_f32(hvx_vec_gelu_erf_f32(x), g); + } + + if (nloe) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + HVX_Vector result = hvx_vec_mul_f32_f32(hvx_vec_gelu_erf_f32(x), g); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), result); + } +} + +static inline void hvx_geglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(_Float16); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const HVX_Vector v_coef_a = hvx_vec_splat_f16((_Float16) GELU_COEF_A); + const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f16((_Float16) SQRT_2_OVER_PI); + const HVX_Vector v_one = hvx_vec_splat_f16((_Float16) 1.0f); + const HVX_Vector v_half = hvx_vec_splat_f16((_Float16) 0.5f); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + + // gelu(x) = 0.5*x*(1 + tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))) + HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x); + HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one); + HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi); + + HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one); + + HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half); + vdst[i] = hvx_vec_mul_f16_f16(gelu_x, g); + } + + if (nloe) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + + HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x); + HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one); + HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi); + + HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one); + + HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half); + HVX_Vector res = hvx_vec_mul_f16_f16(gelu_x, g); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(_Float16), res); + } +} + +// geglu(x, g) = gelu(x) * g +static void geglu_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_geglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +// geglu_quick(x, g) = x * sigmoid(1.702 * x) * g +static void geglu_quick_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_geglu_quick_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +// geglu_erf(x, g) = gelu_erf(x) * g +static void geglu_erf_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_geglu_erf_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +static inline void hvx_reglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const HVX_Vector v_zero = Q6_V_vzero(); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = Q6_Vsf_vmax_VsfVsf(vsrc0[i], v_zero); + vdst[i] = hvx_vec_mul_f32_f32(x, vsrc1[i]); + } + + if (nloe) { + HVX_Vector x = Q6_Vsf_vmax_VsfVsf(vsrc0[i], v_zero); + HVX_Vector res = hvx_vec_mul_f32_f32(x, vsrc1[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res); + } +} + +// reglu(x, g) = relu(x) * g +static void reglu_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_reglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +static __attribute__((noinline)) void hvx_sigglu_f32_vec(HVX_Vector * restrict vdst, + const HVX_Vector * restrict vsrc0, + const HVX_Vector * restrict vsrc1, + uint32_t nvec) { + const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); + const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); + + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(vsrc0[i], v_one, v_max_exp, v_min_exp); + vdst[i] = hvx_vec_mul_f32_f32(sig, vsrc1[i]); + } +} + +static __attribute__((noinline)) void hvx_sigglu_f32_tail(HVX_Vector * restrict vdst, + const HVX_Vector * restrict vsrc0, + const HVX_Vector * restrict vsrc1, + uint32_t nloe) { + const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); + const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); + + HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(vsrc0[0], v_one, v_max_exp, v_min_exp); + HVX_Vector res = hvx_vec_mul_f32_f32(sig, vsrc1[0]); + hvx_vec_store_a((void *) vdst, nloe * sizeof(float), res); +} + +static inline void hvx_sigglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + if (nvec) { + hvx_sigglu_f32_vec(vdst, vsrc0, vsrc1, nvec); + } + if (nloe) { + hvx_sigglu_f32_tail(vdst + nvec, vsrc0 + nvec, vsrc1 + nvec, nloe); + } +} + +// sigglu(x, g) = sigmoid(x) * g +static void sigglu_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_sigglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +// swiglu(x) = x1 * sigmoid(x0) +static void swiglu_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc); + hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); + } +} + +// out = x * sigmoid(alpha * x) * (clamp(y, -limit, limit) + 1.f) +static void swiglu_oai_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + const _Float16 alpha = (_Float16) ((const float *) (gctx->octx->op_params))[2]; + const _Float16 limit = (_Float16) ((const float *) (gctx->octx->op_params))[3]; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + // x (src0_ptr) = std::min(src0_p[k], limit); + hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc); + // y1 (src1_ptr) = std::clamp(src1_p[k], -limit, limit); + hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc); + // y (src1_ptr) = y1 + 1.f + hvx_add_scalar_f16((uint8_t *) src1_ptr, src1_ptr, (_Float16) 1.0f, nc); + // x1 (dst_ptr) = alpha * x + hvx_mul_scalar_f16(dst_ptr, src0_ptr, alpha, nc); + // x2 (dst_ptr) = sigmoid(x1) = 1/(1+exp(-x1)) + hvx_sigmoid_f16_aa(dst_ptr, dst_ptr, nc); + // out = x * sigmoid(alpha * x) * (y + 1.f) + hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); + } +} + +static void swiglu_clamp_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + const _Float16 limit = (_Float16) ((const float *) (gctx->octx->op_params))[3]; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc); + hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc); + hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc); + hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); + } +} + +// geglu(x, g) = gelu(x) * g, gelu via tanh approximation +static void geglu_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_geglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +static inline void hvx_reglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(_Float16); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const HVX_Vector v_zero = Q6_V_vzero(); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = Q6_Vhf_vmax_VhfVhf(vsrc0[i], v_zero); + vdst[i] = hvx_vec_mul_f16_f16(x, vsrc1[i]); + } + + if (nloe) { + HVX_Vector x = Q6_Vhf_vmax_VhfVhf(vsrc0[i], v_zero); + HVX_Vector res = hvx_vec_mul_f16_f16(x, vsrc1[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(_Float16), res); + } +} + +// reglu(x, g) = relu(x) * g +static void reglu_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_reglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +static __attribute__((noinline)) void hvx_sigglu_f16_vec(HVX_Vector * restrict vdst, + const HVX_Vector * restrict vsrc0, + const HVX_Vector * restrict vsrc1, + uint32_t nvec) { + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector sig = hvx_vec_fast_sigmoid_f16(vsrc0[i]); + vdst[i] = hvx_vec_mul_f16_f16(sig, vsrc1[i]); + } +} + +static __attribute__((noinline)) void hvx_sigglu_f16_tail(HVX_Vector * restrict vdst, + const HVX_Vector * restrict vsrc0, + const HVX_Vector * restrict vsrc1, + uint32_t nloe) { + HVX_Vector sig = hvx_vec_fast_sigmoid_f16(vsrc0[0]); + HVX_Vector res = hvx_vec_mul_f16_f16(sig, vsrc1[0]); + hvx_vec_store_a((void *) vdst, nloe * sizeof(_Float16), res); +} + +static inline void hvx_sigglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(_Float16); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + if (nvec) { + hvx_sigglu_f16_vec(vdst, vsrc0, vsrc1, nvec); + } + if (nloe) { + hvx_sigglu_f16_tail(vdst + nvec, vsrc0 + nvec, vsrc1 + nvec, nloe); + } +} + +// sigglu(x, g) = sigmoid(x) * g +static void sigglu_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_glu_context * gctx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_sigglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +#define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \ + static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_glu_context * gctx = (struct htp_glu_context *) data; \ + htp_glu_preamble; \ + \ + struct htp_thread_trace * tr = gctx->octx->ctx ? &gctx->octx->ctx->trace[ith] : NULL; \ + \ + size_t src0_row_size = gctx->src0_row_size; \ + size_t src1_row_size = gctx->src1_row_size; \ + size_t dst_row_size = gctx->dst_row_size; \ + \ + size_t src0_row_stride = gctx->src0_row_stride; \ + size_t src1_row_stride = gctx->src1_row_stride; \ + \ + const uint32_t src0_nrows = gctx->src0_nrows; \ + const uint32_t src0_nrows_per_thread = gctx->src0_nrows_per_thread; \ + \ + const uint32_t src0_start_row = gctx->row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, gctx->row_start + src0_nrows); \ + \ + /* no work for this thread */ \ + if (src0_start_row >= src0_end_row) { \ + return; \ + } \ + \ + const dma_addr_t data_src0 = gctx->data_src0; \ + const dma_addr_t data_src1 = gctx->data_src1; \ + const dma_addr_t data_dst = gctx->data_dst; \ + \ + const size_t src0_row_size_aligned = gctx->src0_row_size_aligned; \ + const size_t src1_row_size_aligned = gctx->src1_row_size_aligned; \ + const size_t dst_row_size_aligned = gctx->dst_row_size_aligned; \ + \ + uint8_t * restrict src0_spad_data = gctx->vtcm_src0 + (ith * gctx->vtcm_src0_size_per_thread); \ + uint8_t * restrict src1_spad_data = gctx->vtcm_src1 + (ith * gctx->vtcm_src1_size_per_thread); \ + uint8_t * restrict dst_spad_data = gctx->vtcm_dst + (ith * gctx->vtcm_dst_size_per_thread); \ + \ + size_t src0_spad_half_size = gctx->src0_spad_half_size; \ + size_t src1_spad_half_size = gctx->src1_spad_half_size; \ + size_t dst_spad_half_size = gctx->dst_spad_half_size; \ + \ + const int BLOCK = gctx->block; \ + \ + dma_queue * dma_q = gctx->octx->ctx->dma[ith]; \ + glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) gctx->compute; \ + \ + for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \ + dma_queue_push(dma_q, \ + dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \ + dst_row_size, dst_row_size_aligned, dst_row_size, 0); \ + \ + dma_queue_push(dma_q, \ + dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \ + \ + dma_queue_push(dma_q, \ + dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \ + } \ + \ + for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + TYPE * dst_spad = (TYPE *) dma_queue_pop(dma_q).src; \ + TYPE * src0_spad = (TYPE *) dma_queue_pop(dma_q).dst; \ + TYPE * src1_spad = (TYPE *) dma_queue_pop(dma_q).dst; \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + compute(src0_spad, src1_spad, dst_spad, block_size, gctx); \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + \ + dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), \ + dst_row_size, dst_row_size_aligned, dst_row_size, block_size); \ + \ + /* prefetch N+2 loop iteration if any */ \ + const uint32_t pref_block = (ir + BLOCK * 2); \ + if (pref_block < src0_end_row) { \ + const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \ + dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \ + dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \ + } \ + } \ + \ + dma_queue_flush(dma_q); \ + } + +DEFINE_GLU_PER_THREAD(float, f32) +DEFINE_GLU_PER_THREAD(_Float16, f16) + +static int execute_op_glu_f32(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + glu_compute_f32_fn_t compute_fn = NULL; + const char * op_type = NULL; + + switch (octx->op) { + case HTP_OP_GLU_SWIGLU: + compute_fn = swiglu_f32; + op_type = "swiglu-f32"; + break; + + case HTP_OP_GLU_SWIGLU_OAI: + compute_fn = swiglu_oai_f32; + op_type = "swiglu-oai-f32"; + break; + + case HTP_OP_GLU_SWIGLU_CLAMP: + compute_fn = swiglu_clamp_f32; + op_type = "swiglu-clamp-f32"; + break; + + case HTP_OP_GLU_GEGLU: + compute_fn = geglu_f32; + op_type = "geglu-f32"; + break; + + case HTP_OP_GLU_GEGLU_QUICK: + compute_fn = geglu_quick_f32; + op_type = "geglu-quick-f32"; + break; + + case HTP_OP_GLU_GEGLU_ERF: + compute_fn = geglu_erf_f32; + op_type = "geglu-erf-f32"; + break; + + case HTP_OP_GLU_REGLU: + compute_fn = reglu_f32; + op_type = "reglu-f32"; + break; + + case HTP_OP_GLU_SIGMOID: + compute_fn = sigglu_f32; + op_type = "sigglu-f32"; + break; + + default: + FARF(ERROR, "Unsupported GLU Op %u\n", octx->op); + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_row_size = dst->ne[0] * SIZEOF_FP32; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). + // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). + const size_t nc_bytes = dst_row_size; + const size_t src0_row_size = nc_bytes; + const size_t src1_row_size = nc_bytes; + const size_t src0_row_stride = src0->nb[1]; + const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; + + struct htp_glu_vtcm_layout L; + htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread); + + const size_t src0_row_size_aligned = L.row_size_aligned; + const size_t src1_row_size_aligned = L.row_size_aligned; + const size_t dst_row_size_aligned = L.row_size_aligned; + + // Make sure the reserved vtcm size is sufficient + if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { + FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (src1) { + FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", + op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, + L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + } else { + FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + } + + // Prepare context + struct htp_glu_context gctx; + gctx.octx = octx; + gctx.compute = (void *) compute_fn; + gctx.op_str = op_type; + + gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + + gctx.src0_row_size = src0_row_size; + gctx.src1_row_size = src1_row_size; + gctx.dst_row_size = dst_row_size; + + gctx.src0_row_size_aligned = src0_row_size_aligned; + gctx.src1_row_size_aligned = src1_row_size_aligned; + gctx.dst_row_size_aligned = dst_row_size_aligned; + + gctx.src0_row_stride = src0_row_stride; + gctx.src1_row_stride = src1_row_stride; + + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + gctx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + + gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread; + gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread; + gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread; + + gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1; + gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1; + gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1; + + gctx.block = L.block; + gctx.src0_nrows = nrows; + gctx.row_start = row_start; + + gctx.nc = dst->ne[0]; + + // Addresses and GLU logic + dma_addr_t data_src0 = src0->data; + dma_addr_t data_src1 = src1 ? src1->data : 0; + + if (!src1) { + const int32_t swapped = octx->op_params[1]; + data_src1 = data_src0; + gctx.src1_row_size = gctx.src0_row_size; + + size_t nc_in_bytes = gctx.nc * SIZEOF_FP32; + if (swapped) { + data_src0 += nc_in_bytes; + } else { + data_src1 += nc_in_bytes; + } + } + + gctx.data_src0 = data_src0; + gctx.data_src1 = data_src1; + gctx.data_dst = dst->data; + + work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f32_per_thread, &gctx, n_threads); + return HTP_STATUS_OK; +} + +static int execute_op_glu_f16(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + glu_compute_f16_fn_t compute_fn = NULL; + const char * op_type = NULL; + + switch (octx->op) { + case HTP_OP_GLU_SWIGLU: + compute_fn = swiglu_f16; + op_type = "swiglu-f16"; + break; + + case HTP_OP_GLU_SWIGLU_OAI: + compute_fn = swiglu_oai_f16; + op_type = "swiglu-oai-f16"; + break; + + case HTP_OP_GLU_SWIGLU_CLAMP: + compute_fn = swiglu_clamp_f16; + op_type = "swiglu-clamp-f16"; + break; + + case HTP_OP_GLU_GEGLU: + compute_fn = geglu_f16; + op_type = "geglu-f16"; + break; + + case HTP_OP_GLU_REGLU: + compute_fn = reglu_f16; + op_type = "reglu-f16"; + break; + + case HTP_OP_GLU_SIGMOID: + compute_fn = sigglu_f16; + op_type = "sigglu-f16"; + break; + + default: + FARF(ERROR, "Unsupported GLU Op %u\n", octx->op); + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_glu_kernel_params * kparams = (const struct htp_glu_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_row_size = dst->ne[0] * SIZEOF_FP16; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(_Float16), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). + // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). + const size_t nc_bytes = dst_row_size; + const size_t src0_row_size = nc_bytes; + const size_t src1_row_size = nc_bytes; + const size_t src0_row_stride = src0->nb[1]; + const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; + + struct htp_glu_vtcm_layout L; + htp_glu_vtcm_layout_build(&L, dst_row_size, n_threads, kparams->vtcm_row_per_thread); + + const size_t src0_row_size_aligned = L.row_size_aligned; + const size_t src1_row_size_aligned = L.row_size_aligned; + const size_t dst_row_size_aligned = L.row_size_aligned; + + // Make sure the reserved vtcm size is sufficient + if (kparams->vtcm_row_per_thread == 0 || octx->ctx->vtcm_size < kparams->vtcm_size) { + FARF(ERROR, "glu-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (src1) { + FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", + op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, + L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + } else { + FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + } + + // Prepare context + struct htp_glu_context gctx; + gctx.octx = octx; + gctx.compute = (void *) compute_fn; + gctx.op_str = op_type; + + gctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + + gctx.src0_row_size = src0_row_size; + gctx.src1_row_size = src1_row_size; + gctx.dst_row_size = dst_row_size; + + gctx.src0_row_size_aligned = src0_row_size_aligned; + gctx.src1_row_size_aligned = src1_row_size_aligned; + gctx.dst_row_size_aligned = dst_row_size_aligned; + + gctx.src0_row_stride = src0_row_stride; + gctx.src1_row_stride = src1_row_stride; + + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + gctx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + gctx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + gctx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + + gctx.vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes_per_thread; + gctx.vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes_per_thread; + gctx.vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes_per_thread; + + gctx.src0_spad_half_size = gctx.vtcm_src0_size_per_thread >> 1; + gctx.src1_spad_half_size = gctx.vtcm_src1_size_per_thread >> 1; + gctx.dst_spad_half_size = gctx.vtcm_dst_size_per_thread >> 1; + + gctx.block = L.block; + gctx.src0_nrows = nrows; + gctx.row_start = row_start; + + gctx.nc = dst->ne[0]; + + // Addresses and GLU logic + dma_addr_t data_src0 = src0->data; + dma_addr_t data_src1 = src1 ? src1->data : 0; + + if (!src1) { + const int32_t swapped = octx->op_params[1]; + data_src1 = data_src0; + gctx.src1_row_size = gctx.src0_row_size; + + size_t nc_in_bytes = gctx.nc * SIZEOF_FP16; + if (swapped) { + data_src0 += nc_in_bytes; + } else { + data_src1 += nc_in_bytes; + } + } + + gctx.data_src0 = data_src0; + gctx.data_src1 = data_src1; + gctx.data_dst = dst->data; + + work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &gctx, n_threads); + return HTP_STATUS_OK; +} + + +int op_glu(struct htp_ops_context * octx) { + switch (octx->src[0]->type) { + case HTP_TYPE_F32: + return execute_op_glu_f32(octx); + + case HTP_TYPE_F16: + return execute_op_glu_f16(octx); + + default: + return HTP_STATUS_NO_SUPPORT; + } +} diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.h b/ggml/src/ggml-hexagon/htp/glu-ops.h new file mode 100644 index 0000000..41b3857 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/glu-ops.h @@ -0,0 +1,76 @@ +#ifndef HTP_GLU_OPS_H +#define HTP_GLU_OPS_H + +#include "hex-common.h" +#include "htp-ops.h" +#include "htp-vtcm.h" +#include + +struct htp_glu_kernel_params { + uint32_t n_threads; + uint32_t vtcm_row_per_thread; + uint32_t vtcm_size; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob"); +#endif + +static inline bool htp_op_is_glu(uint32_t opcode) { + switch (opcode) { + case HTP_OP_GLU_SWIGLU: + case HTP_OP_GLU_SWIGLU_OAI: + case HTP_OP_GLU_SWIGLU_CLAMP: + case HTP_OP_GLU_GEGLU: + case HTP_OP_GLU_GEGLU_QUICK: + case HTP_OP_GLU_GEGLU_ERF: + case HTP_OP_GLU_REGLU: + case HTP_OP_GLU_SIGMOID: + return true; + default: + return false; + } +} + +struct htp_glu_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_src1; + size_t off_dst; + + size_t row_size_aligned; + size_t src0_bytes_per_thread; + size_t src1_bytes_per_thread; + size_t dst_bytes_per_thread; + + uint32_t vtcm_row_per_thread; + uint32_t block; +}; + +static inline void htp_glu_vtcm_layout_build( + struct htp_glu_vtcm_layout * L, + size_t dst_row_size, + uint32_t n_threads, + uint32_t vtcm_row_per_thread +) { + const size_t row_size_aligned = hex_round_up(dst_row_size, 128); + + L->row_size_aligned = row_size_aligned; + L->vtcm_row_per_thread = vtcm_row_per_thread; + L->block = vtcm_row_per_thread >> 1; + + L->src0_bytes_per_thread = row_size_aligned * vtcm_row_per_thread; + L->src1_bytes_per_thread = row_size_aligned * vtcm_row_per_thread; + L->dst_bytes_per_thread = row_size_aligned * vtcm_row_per_thread; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + + L->total_bytes = off; +} + +#endif // HTP_GLU_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/hex-bitmap.h b/ggml/src/ggml-hexagon/htp/hex-bitmap.h new file mode 100644 index 0000000..1408988 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hex-bitmap.h @@ -0,0 +1,24 @@ +#ifndef HEX_BITMAP_H +#define HEX_BITMAP_H + +#include +#include +#include + +static inline void bitmap_set(uint32_t * bitmap, uint32_t idx) { + bitmap[idx / 32] |= (1U << (idx % 32)); +} + +static inline void bitmap_clear(uint32_t * bitmap, uint32_t idx) { + bitmap[idx / 32] &= ~(1U << (idx % 32)); +} + +static inline bool bitmap_test(const uint32_t * bitmap, uint32_t idx) { + return (bitmap[idx / 32] & (1U << (idx % 32))) != 0; +} + +static inline void bitmap_reset(uint32_t * bitmap, size_t size_in_bits) { + memset(bitmap, 0, ((size_in_bits + 31) / 32) * sizeof(uint32_t)); +} + +#endif // HEX_BITMAP_H diff --git a/ggml/src/ggml-hexagon/htp/hex-common.h b/ggml/src/ggml-hexagon/htp/hex-common.h new file mode 100644 index 0000000..b943453 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hex-common.h @@ -0,0 +1,93 @@ +#ifndef HEX_COMMON_H +#define HEX_COMMON_H + +#include +#include +#include + +#ifndef SIZE_MAX +#define SIZE_MAX ((size_t)-1) +#endif + +#ifndef MAX +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif + +#ifndef MIN +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +#ifndef HEX_L2_LINE_SIZE +#define HEX_L2_LINE_SIZE 128 +#endif + +static inline uint32_t hex_ceil_pow2(uint32_t x) { + if (x <= 1) { return 1; } + int p = 2; + x--; + while (x >>= 1) { p <<= 1; } + return p; +} + +static inline size_t hmx_ceil_div(size_t num, size_t den) { + return (num + den - 1) / den; +} + +static inline int32_t hex_is_aligned(const void * addr, uint32_t align) { + return ((size_t) addr & (align - 1)) == 0; +} + +static inline size_t hex_align_up(size_t v, size_t align) { + return hmx_ceil_div(v, align) * align; +} + +static inline size_t hex_align_down(size_t v, size_t align) { + return (v / align) * align; +} + +static inline int32_t hex_is_one_chunk(void * addr, uint32_t n, uint32_t chunk_size) { + uint32_t left_off = (size_t) addr & (chunk_size - 1); + uint32_t right_off = left_off + n; + return right_off <= chunk_size; +} + +static inline uint32_t hex_round_up(uint32_t n, uint32_t m) { + return m * ((n + m - 1) / m); +} + +static inline size_t hex_smin(size_t a, size_t b) { + return a < b ? a : b; +} + +static inline size_t hex_smax(size_t a, size_t b) { + return a > b ? a : b; +} + +static inline void hex_swap_ptr(void ** p1, void ** p2) { + void * t = *p1; + *p1 = *p2; + *p2 = t; +} + +static inline bool hex_mul_overflow(size_t a, size_t b, size_t *out) { + if (a != 0 && b > SIZE_MAX / a) return true; + *out = a * b; + return false; +} + +static inline bool hex_add_overflow(size_t a, size_t b, size_t *out) { + if (a > SIZE_MAX - b) return true; + *out = a + b; + return false; +} + +static inline uint32_t hex_gcd_u32(uint32_t a, uint32_t b) { + while (b != 0) { + uint32_t t = b; + b = a % b; + a = t; + } + return a; +} + +#endif // HEX_COMMON_H diff --git a/ggml/src/ggml-hexagon/htp/hex-dma.c b/ggml/src/ggml-hexagon/htp/hex-dma.c deleted file mode 100644 index b66e2d2..0000000 --- a/ggml/src/ggml-hexagon/htp/hex-dma.c +++ /dev/null @@ -1,63 +0,0 @@ -#include "hex-dma.h" - -#include -#include -#include - -#pragma clang diagnostic ignored "-Wunused-function" - -static inline uint32_t pow2_ceil(uint32_t x) { - if (x <= 1) { - return 1; - } - int p = 2; - x--; - while (x >>= 1) { - p <<= 1; - } - return p; -} - -dma_queue * dma_queue_create(size_t capacity) { - dma_queue * q = (dma_queue *) memalign(32, sizeof(dma_queue)); - if (q == NULL) { - FARF(ERROR, "%s: failed to allocate DMA queue\n", __FUNCTION__); - return NULL; - } - - capacity = pow2_ceil(capacity); - - memset(q, 0, sizeof(dma_queue)); - q->capacity = capacity; - q->idx_mask = capacity - 1; - - q->desc = (dma_descriptor_2d *) memalign(64, capacity * sizeof(dma_descriptor_2d)); - memset(q->desc, 0, capacity * sizeof(dma_descriptor_2d)); - - q->dptr = (dma_ptr *) memalign(4, capacity * sizeof(dma_ptr)); - memset(q->dptr, 0, capacity * sizeof(dma_ptr)); - - q->tail = &q->desc[capacity - 1]; - - if (!q->desc && !q->dptr) { - FARF(ERROR, "%s: failed to allocate DMA queue items\n", __FUNCTION__); - return NULL; - } - - FARF(HIGH, "dma-queue: capacity %u\n", capacity); - - return q; -} - -void dma_queue_delete(dma_queue * q) { - if (!q) { - return; - } - free(q->desc); - free(q->dptr); - free(q); -} - -void dma_queue_flush(dma_queue * q) { - while (dma_queue_pop(q).dst != NULL) ; -} diff --git a/ggml/src/ggml-hexagon/htp/hex-dma.h b/ggml/src/ggml-hexagon/htp/hex-dma.h deleted file mode 100644 index 7685473..0000000 --- a/ggml/src/ggml-hexagon/htp/hex-dma.h +++ /dev/null @@ -1,372 +0,0 @@ -#ifndef HTP_DMA_H -#define HTP_DMA_H - -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date -typedef struct dma_descriptor_1d_s { - void * next; - uint32_t size:24; - uint32_t desc_size:2; - uint32_t dst_comp:1; - uint32_t src_comp:1; - uint32_t dst_bypass:1; - uint32_t src_bypass:1; - uint32_t order:1; - uint32_t done:1; - void * src; - void * dst; -} dma_descriptor_1d; - -#if __HVX_ARCH__ < 75 - -typedef struct dma_descriptor_2d_s { - void * next; - uint32_t reserved0:24; - uint32_t desc_size:2; - uint32_t dst_comp:1; - uint32_t src_comp:1; - uint32_t dst_bypass:1; - uint32_t src_bypass:1; - uint32_t order:1; - uint32_t done:1; - void * src; - void * dst; - uint32_t desc_type:8; - uint32_t reserved1:24; - uint32_t row_size:16; - uint32_t nrows:16; - uint32_t src_stride:16; - uint32_t dst_stride:16; - uint32_t src_offset:16; - uint32_t dst_offset:16; -} dma_descriptor_2d; - -#else - -typedef struct dma_descriptor_2d_s { - void * next; - uint32_t dst_stride:24; - uint32_t desc_size:2; - uint32_t dst_comp:1; - uint32_t src_comp:1; - uint32_t dst_bypass:1; - uint32_t src_bypass:1; - uint32_t order:1; - uint32_t done:1; - void * src; - void * dst; - uint32_t desc_type:8; - uint32_t reserved0:24; - uint32_t row_size:24; - uint32_t nrows_lo:8; - uint32_t nrows_hi:8; - uint32_t src_stride:24; - uint32_t offset:24; - uint32_t reserved1:8; -} dma_descriptor_2d; - -#endif - -typedef struct { - void *dst; - const void *src; -} dma_ptr; - -typedef struct { - dma_descriptor_2d * desc; // descriptor pointers - dma_descriptor_2d * tail; // tail pointer - dma_ptr * dptr; // dst/src pointers - uint32_t push_idx; - uint32_t pop_idx; - uint32_t capacity; - uint32_t idx_mask; -} dma_queue; - -dma_queue * dma_queue_create(size_t capacity); -void dma_queue_delete(dma_queue * q); -void dma_queue_flush(dma_queue * q); - -// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead -// but those do not seem to always compiler properly. -static inline void dmstart(void * next) { - asm volatile(" release(%0):at" : : "r"(next)); - asm volatile(" dmstart(%0)" : : "r"(next)); -} - -static inline void dmlink(void * cur, void * next) { - asm volatile(" release(%0):at" : : "r"(next)); - asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next)); -} - -static inline unsigned int dmpoll(void) { - unsigned int ret = 0; - asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory"); - return ret; -} - -static inline unsigned int dmwait(void) { - unsigned int ret = 0; - asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory"); - return ret; -} - -static inline dma_ptr dma_make_ptr(void *dst, const void *src) -{ - dma_ptr p = { dst, src }; - return p; -} - -#if __HVX_ARCH__ < 73 -static const uint32_t dma_src_l2_bypass_on = 1; -static const uint32_t dma_dst_l2_bypass_on = 0; -#else -static const uint32_t dma_src_l2_bypass_on = 1; -static const uint32_t dma_dst_l2_bypass_on = 1; -#endif - -static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) { - if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) { - FARF(HIGH, "dma-push: queue full\n"); - return false; - } - - dma_descriptor_1d * desc = (dma_descriptor_1d *) &q->desc[q->push_idx]; - desc->next = NULL; - desc->desc_size = 0; // 1D mode - desc->src_bypass = dma_src_l2_bypass_on; - desc->dst_bypass = dma_dst_l2_bypass_on; - desc->order = 0; - desc->done = 0; - desc->src = (void *) dptr.src; - desc->dst = (void *) dptr.dst; - desc->size = size; - - q->dptr[q->push_idx] = dptr; - - if (size) { - dmlink(q->tail, desc); - q->tail = (dma_descriptor_2d *) desc; - } else { - desc->done = 1; - } - - // FARF(ERROR, "dma-push: i %u row-size %u nrows %d dst %p src %p\n", q->push_idx, row_size, nrows, dptr.dst, dptr.src); - q->push_idx = (q->push_idx + 1) & q->idx_mask; - return true; -} - -static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) { - FARF(HIGH, "dma-push: queue full\n"); - return false; - } - - dma_descriptor_2d * desc = &q->desc[q->push_idx]; - - desc->next = NULL; - desc->reserved0 = 0; - desc->reserved1 = 0; - desc->desc_size = 1; // 2d mode - desc->src_bypass = dma_src_l2_bypass_on; - desc->dst_bypass = dma_dst_l2_bypass_on; - desc->src_comp = 0; - desc->dst_comp = 0; - desc->order = 0; - desc->done = 0; - desc->src_stride = src_stride; - desc->dst_stride = dst_stride; - desc->src = (void *) dptr.src; - desc->dst = (void *) dptr.dst; - desc->row_size = row_size; - -#if __HVX_ARCH__ < 75 - desc->desc_type = 0; // 2d (16-bit) mode - desc->nrows = nrows; - desc->src_offset = 0; - desc->dst_offset = 0; -#else - desc->desc_type = 9; // 2d (24-bit) mode - desc->nrows_lo = (nrows & 0xff); - desc->nrows_hi = (nrows >> 8); - desc->offset = 0; -#endif - - q->dptr[q->push_idx] = dptr; - - if (nrows) { - dmlink(q->tail, desc); - q->tail = desc; - } else { - desc->done = 1; - } - - // FARF(ERROR, "dma-push: i %u row-size %u nrows %d dst %p src %p\n", q->push_idx, row_size, nrows, dptr.dst, dptr.src); - q->push_idx = (q->push_idx + 1) & q->idx_mask; - return true; -} - -static inline dma_ptr dma_queue_pop(dma_queue * q) { - dma_ptr dptr = { NULL }; - - if (q->push_idx == q->pop_idx) { - return dptr; - } - - dma_descriptor_2d * desc = &q->desc[q->pop_idx]; - - // Wait for desc to complete - while (!desc->done) { - // FARF(ERROR, "dma-pop: waiting for DMA : %u\n", q->pop_idx); - dmpoll(); - } - - dptr = q->dptr[q->pop_idx]; - - // FARF(ERROR, "dma-pop: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src); - q->pop_idx = (q->pop_idx + 1) & q->idx_mask; - return dptr; -} - -static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) { - dma_ptr dptr = { NULL }; - - if (q->push_idx == q->pop_idx) { - return dptr; - } - - dptr = q->dptr[q->pop_idx]; - - // FARF(ERROR, "dma-pop-nowait: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src); - q->pop_idx = (q->pop_idx + 1) & q->idx_mask; - return dptr; -} - -static inline bool dma_queue_empty(dma_queue * q) { - return q->push_idx == q->pop_idx; -} - -static inline uint32_t dma_queue_depth(dma_queue * q) { - return (q->push_idx - q->pop_idx) & q->idx_mask; -} - -static inline uint32_t dma_queue_capacity(dma_queue * q) { - return q->capacity; -} - -#if __HVX_ARCH__ < 75 - -// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535. -// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions. - -#define DMA_MAX_FIELD_VAL 65535u - -static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - // Fast path: everything fits in 16 bits - if (nrows == 0 || __builtin_expect( - row_size <= DMA_MAX_FIELD_VAL && - nrows <= DMA_MAX_FIELD_VAL && - src_stride <= DMA_MAX_FIELD_VAL && - dst_stride <= DMA_MAX_FIELD_VAL, 1)) { - return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); - } - - // Contiguous block - // Use 1d DMA mode which supports sizes up to 24-bits (16MB) - if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) { - size_t total = row_size * nrows; - return dma_queue_push_single_1d(q, dptr, total); - } - - // Stride overflow — fall back to row-by-row. - { - const uint8_t *src = (const uint8_t *) dptr.src; - uint8_t *dst = (uint8_t *) dptr.dst; - for (size_t r = 0; r < nrows; ++r) { - dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride); - if (!dma_queue_push_single_1d(q, p, row_size)) - return false; - if (r + 1 < nrows) - dma_queue_pop(q); - } - return true; - } -} - -#else // HVX_ARCH >= 75 - -static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - // On v75 and up we always use 2d 24-bit mode - return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows); -} - -#endif - -static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { - return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows); -} - -static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) { - return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows); -} - -#define DMA_CACHE_MAX_SIZE 64U - -typedef struct { - uint8_t *base; - uint32_t line_size; - uint32_t capacity; - uint32_t src[DMA_CACHE_MAX_SIZE]; - uint16_t age[DMA_CACHE_MAX_SIZE]; -} dma_cache; - -static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity) -{ - c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity; - c->base = base; - c->line_size = line_size; - - for (unsigned i=0; i < c->capacity; i++) { - c->src[i] = 0; - c->age[i] = 0; - } -} - -static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows) -{ - uint32_t o_idx = 0; - uint16_t o_age = 0; - uint8_t * dst = 0; - - for (unsigned i=0; i < c->capacity; i++) { - if (c->src[i] == (uint32_t) src) { - c->age[i] = 0; - dst = c->base + (i * c->line_size); nrows = 0; // dummy dma - // FARF(ERROR, "dma-cache: found %p", src); - } else { - c->age[i]++; - if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; } - } - } - if (!dst) { - // FARF(ERROR, "dma-cache: replacing #%u : age %u %p -> %p", o_idx, c->age[o_idx], (void *) c->src[o_idx], src); - c->age[o_idx] = 0; - c->src[o_idx] = (uint32_t) src; - dst = c->base + o_idx * c->line_size; // normal nrows dma - } - - return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows); -} - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* HTP_DMA_H */ diff --git a/ggml/src/ggml-hexagon/htp/hex-profile.h b/ggml/src/ggml-hexagon/htp/hex-profile.h new file mode 100644 index 0000000..5c7a1a4 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hex-profile.h @@ -0,0 +1,66 @@ +#ifndef HEX_PROFILE_H +#define HEX_PROFILE_H + +#include +#include +#include + +#include "hex-utils.h" +#include "htp-ops.h" + +#define HTP_TRACE_EVT_START 0 +#define HTP_TRACE_EVT_STOP 1 + +#ifndef HEX_NUM_PMU_COUNTERS +#define HEX_NUM_PMU_COUNTERS 8 +#endif + +static inline void hex_get_pmu(uint32_t counters[]) { +#if __HVX_ARCH__ >= 79 + asm volatile("%0 = upmucnt0" : "=r"(counters[0])); + asm volatile("%0 = upmucnt1" : "=r"(counters[1])); + asm volatile("%0 = upmucnt2" : "=r"(counters[2])); + asm volatile("%0 = upmucnt3" : "=r"(counters[3])); + asm volatile("%0 = upmucnt4" : "=r"(counters[4])); + asm volatile("%0 = upmucnt5" : "=r"(counters[5])); + asm volatile("%0 = upmucnt6" : "=r"(counters[6])); + asm volatile("%0 = upmucnt7" : "=r"(counters[7])); +#else + counters[0] = qurt_pmu_get(QURT_PMUCNT0); + counters[1] = qurt_pmu_get(QURT_PMUCNT1); + counters[2] = qurt_pmu_get(QURT_PMUCNT2); + counters[3] = qurt_pmu_get(QURT_PMUCNT3); + counters[4] = qurt_pmu_get(QURT_PMUCNT4); + counters[5] = qurt_pmu_get(QURT_PMUCNT5); + counters[6] = qurt_pmu_get(QURT_PMUCNT6); + counters[7] = qurt_pmu_get(QURT_PMUCNT7); +#endif +} + +struct htp_thread_trace { + uint32_t count; + uint32_t max_events; + struct htp_trace_desc * events; +}; + +static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) { + if (!tr->max_events) { + return; + } + uint32_t i = tr->count++; + if (i < tr->max_events) { + tr->events[i].id = id; + tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0); + tr->events[i].cycles = (uint32_t) hex_get_cycles(); + } +} + +static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { + htp_trace_event(tr, id, info, HTP_TRACE_EVT_START); +} + +static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) { + htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP); +} + +#endif /* HEX_PROFILE_H */ diff --git a/ggml/src/ggml-hexagon/htp/hex-utils.h b/ggml/src/ggml-hexagon/htp/hex-utils.h index f6713c5..abb44da 100644 --- a/ggml/src/ggml-hexagon/htp/hex-utils.h +++ b/ggml/src/ggml-hexagon/htp/hex-utils.h @@ -4,20 +4,14 @@ #include #include #include +#include #include "hexagon_types.h" #include "hexagon_protos.h" #include "hex-fastdiv.h" #include "hex-dump.h" - -#ifndef MAX -#define MAX(a, b) ((a) > (b) ? (a) : (b)) -#endif - -#ifndef MIN -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#endif +#include "hex-common.h" static inline uint64_t hex_get_cycles() { uint64_t cycles = 0; @@ -31,68 +25,29 @@ static inline uint64_t hex_get_pktcnt() { return pktcnt; } -static inline uint32_t hex_ceil_pow2(uint32_t x) { - if (x <= 1) { return 1; } - int p = 2; - x--; - while (x >>= 1) { p <<= 1; } - return p; -} - -static inline size_t hmx_ceil_div(size_t num, size_t den) { - return (num + den - 1) / den; -} - -static inline int32_t hex_is_aligned(const void * addr, uint32_t align) { - return ((size_t) addr & (align - 1)) == 0; -} - -static inline size_t hex_align_up(size_t v, size_t align) { - return hmx_ceil_div(v, align) * align; -} - -static inline size_t hex_align_down(size_t v, size_t align) { - return (v / align) * align; -} - -static inline int32_t hex_is_one_chunk(void * addr, uint32_t n, uint32_t chunk_size) { - uint32_t left_off = (size_t) addr & (chunk_size - 1); - uint32_t right_off = left_off + n; - return right_off <= chunk_size; -} - -static inline uint32_t hex_round_up(uint32_t n, uint32_t m) { - return m * ((n + m - 1) / m); -} - -static inline size_t hex_smin(size_t a, size_t b) { - return a < b ? a : b; -} - -static inline size_t hex_smax(size_t a, size_t b) { - return a > b ? a : b; -} - static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride, uint32_t height) { const uint64_t control = Q6_P_combine_RR(stride, Q6_R_combine_RlRl(width, height)); Q6_l2fetch_AP((void *) p, control); } -#define HEX_L2_LINE_SIZE 64 -#define HEX_L2_FLUSH_SIZE (128 * 1024) +static inline void hex_l2fetch_block(const void * addr, size_t size) { + if (size == 0) return; + const uint32_t width = 16384; // 16KB rows + const uint32_t height = (size + width - 1) / width; + hex_l2fetch(addr, width, width, height); +} + +#define HEX_L2_LINE_SIZE 128 +#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread) +#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024) +#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024) static inline void hex_l2flush(void * addr, size_t size) { - if (size > HEX_L2_FLUSH_SIZE) { - qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); - } else { - const uint32_t s = (uint32_t) addr; - const uint32_t e = s + size; - for (uint32_t i = s; i < e; i += HEX_L2_LINE_SIZE * 4) { - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2); - Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3); - } + if (size == 0) return; + const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1); + const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1); + for (uint32_t i = s; i < e; i += HEX_L2_LINE_SIZE) { + Q6_dccleaninva_A((void *) i); } } diff --git a/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h b/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h new file mode 100644 index 0000000..d5fb48a --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h @@ -0,0 +1,705 @@ +#ifndef HMX_FA_KERNELS_H +#define HMX_FA_KERNELS_H + +#include +#include +#include +#include "hvx-utils.h" +#include "hmx-utils.h" +#include "hex-fastdiv.h" + +// HMX-specific parameters, offsets and inner kernels for Flash Attention + +// Scatter offsets for diagonal tile: entry[2i] = i*136, entry[2i+1] = i*136+6 +// 136 = 4 * 32 + 8 = byte offset to diagonal in a 32x32 fp16 interleaved tile +static const int16_t d_tile_scatter_offsets[64] __attribute__((aligned(128))) = { + 0 * 136, 0 * 136 + 6, + 1 * 136, 1 * 136 + 6, + 2 * 136, 2 * 136 + 6, + 3 * 136, 3 * 136 + 6, + 4 * 136, 4 * 136 + 6, + 5 * 136, 5 * 136 + 6, + 6 * 136, 6 * 136 + 6, + 7 * 136, 7 * 136 + 6, + 8 * 136, 8 * 136 + 6, + 9 * 136, 9 * 136 + 6, + 10 * 136, 10 * 136 + 6, + 11 * 136, 11 * 136 + 6, + 12 * 136, 12 * 136 + 6, + 13 * 136, 13 * 136 + 6, + 14 * 136, 14 * 136 + 6, + 15 * 136, 15 * 136 + 6, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, +}; +// Inner HMX tile computation kernels + +static inline void hmx_fa_qk_dot_tile( + const __fp16 * row_tiles, + const __fp16 * col_tiles, + __fp16 * out_tile, + size_t n_dot_tiles +) { + if (n_dot_tiles == 2) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + : + : "r"(2047), + "r"(row_tiles + 0 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 0 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 1 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 1 * HMX_FP16_TILE_N_ELMS) + ); + } else if (n_dot_tiles == 4) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + HMX_LOAD_MPY_F16("%5", "%6", "%0") + HMX_LOAD_MPY_F16("%7", "%8", "%0") + : + : "r"(2047), + "r"(row_tiles + 0 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 0 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 1 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 1 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 2 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 2 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 3 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 3 * HMX_FP16_TILE_N_ELMS) + ); + } else if (n_dot_tiles == 8) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + HMX_LOAD_MPY_F16("%5", "%6", "%0") + HMX_LOAD_MPY_F16("%7", "%8", "%0") + HMX_LOAD_MPY_F16("%9", "%10", "%0") + HMX_LOAD_MPY_F16("%11", "%12", "%0") + HMX_LOAD_MPY_F16("%13", "%14", "%0") + HMX_LOAD_MPY_F16("%15", "%16", "%0") + : + : "r"(2047), + "r"(row_tiles + 0 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 0 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 1 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 1 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 2 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 2 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 3 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 3 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 4 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 4 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 5 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 5 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 6 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 6 * HMX_FP16_TILE_N_ELMS), + "r"(row_tiles + 7 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 7 * HMX_FP16_TILE_N_ELMS) + ); + } else { + for (size_t k = 0; k < n_dot_tiles; ++k) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + : + : "r"(2047), "r"(row_tiles), "r"(col_tiles) + ); + row_tiles += HMX_FP16_TILE_N_ELMS; + col_tiles += HMX_FP16_TILE_N_ELMS; + } + } + asm volatile( + HMX_STORE_AFTER_F16("%0", "%1") + : + : "r"(out_tile), "r"(0) + : "memory" + ); +} + +static inline void hmx_fa_o_update_tile( + const __fp16 * d_diag, + const __fp16 * o_rc, + const __fp16 * p_tile_in, + const __fp16 * v_tile_in, + __fp16 * o_tile_out, + size_t n_col_tiles +) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + : + : "r"(2047), "r"(d_diag), "r"(o_rc) + ); + if (n_col_tiles == 2) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + : + : "r"(2047), + "r"(p_tile_in + 0 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 0 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 1 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 1 * HMX_FP16_TILE_N_ELMS) + ); + } else if (n_col_tiles == 4) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + HMX_LOAD_MPY_F16("%5", "%6", "%0") + HMX_LOAD_MPY_F16("%7", "%8", "%0") + : + : "r"(2047), + "r"(p_tile_in + 0 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 0 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 1 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 1 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 2 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 2 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 3 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 3 * HMX_FP16_TILE_N_ELMS) + ); + } else if (n_col_tiles == 8) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + HMX_LOAD_MPY_F16("%3", "%4", "%0") + HMX_LOAD_MPY_F16("%5", "%6", "%0") + HMX_LOAD_MPY_F16("%7", "%8", "%0") + HMX_LOAD_MPY_F16("%9", "%10", "%0") + HMX_LOAD_MPY_F16("%11", "%12", "%0") + HMX_LOAD_MPY_F16("%13", "%14", "%0") + HMX_LOAD_MPY_F16("%15", "%16", "%0") + : + : "r"(2047), + "r"(p_tile_in + 0 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 0 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 1 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 1 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 2 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 2 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 3 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 3 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 4 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 4 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 5 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 5 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 6 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 6 * HMX_FP16_TILE_N_ELMS), + "r"(p_tile_in + 7 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 7 * HMX_FP16_TILE_N_ELMS) + ); + } else { + for (size_t k = 0; k < n_col_tiles; ++k) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + : + : "r"(2047), "r"(p_tile_in), "r"(v_tile_in) + ); + p_tile_in += HMX_FP16_TILE_N_ELMS; + v_tile_in += HMX_FP16_TILE_N_ELMS; + } + } + asm volatile( + HMX_STORE_AFTER_F16("%0", "%1") + : + : "r"(o_tile_out), "r"(0) + : "memory" + ); +} + +static inline void hmx_fa_o_norm_tile( + const __fp16 * d_diag, + const __fp16 * o_rc, + __fp16 * o_out +) { + asm volatile( + HMX_LOAD_MPY_F16("%1", "%2", "%0") + : + : "r"(2047), "r"(d_diag), "r"(o_rc) + ); + asm volatile( + HMX_STORE_AFTER_F16("%0", "%1") + : + : "r"(o_out), "r"(0) + : "memory" + ); +} + +static inline void hmx_fa_q_prep_fp32_d2( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + ((HVX_Vector *) (out_base + 0 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + ((HVX_Vector *) (out_base + 1 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(float)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(float)) + : NULL; + + { + HVX_Vector v0 = pv_in0[0]; + HVX_Vector v1 = pv_in1 ? pv_in1[0] : Q6_V_vzero(); + HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); + ((HVX_Vector *) (out_base + 0 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = v_hf; + } + { + HVX_Vector v0 = pv_in0[1]; + HVX_Vector v1 = pv_in1 ? pv_in1[1] : Q6_V_vzero(); + HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); + ((HVX_Vector *) (out_base + 1 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = v_hf; + } + } +} + +static inline void hmx_fa_q_prep_fp32_d4( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < 4; ++d) { + ((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(float)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(float)) + : NULL; + + for (uint32_t d = 0; d < 4; ++d) { + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); + ((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = v_hf; + } + } +} + +static inline void hmx_fa_q_prep_fp32( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, uint32_t d_limit, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < d_limit; ++d) { + ((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(float)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(float)) + : NULL; + + for (uint32_t d = 0; d < d_limit; ++d) { + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1); + + HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS); + out_tile[r1 / 2] = v_hf; + } + } +} + +static inline void hmx_fa_q_prep_fp16_d1( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + __fp16 * out_dtile = out_base + 0 * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(__fp16)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(__fp16)) + : NULL; + + HVX_Vector v0 = pv_in0[0]; + HVX_Vector v1 = pv_in1 ? pv_in1[0] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + __fp16 * out_dtile = out_base + 0 * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } +} + +static inline void hmx_fa_q_prep_fp16_d2( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < 2; ++d) { + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(__fp16)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(__fp16)) + : NULL; + + { + HVX_Vector v0 = pv_in0[0]; + HVX_Vector v1 = pv_in1 ? pv_in1[0] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + __fp16 * out_dtile = out_base + 0 * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + { + HVX_Vector v0 = pv_in0[1]; + HVX_Vector v1 = pv_in1 ? pv_in1[1] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + __fp16 * out_dtile = out_base + 1 * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + } +} + +static inline void hmx_fa_q_prep_fp16( + __fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm, + size_t start, size_t end, size_t g_rows_end, + size_t DK, size_t G, size_t n_rows_q, + const struct fastdiv_values * div_G, uint32_t d_limit, bool q_transposed +) { + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < d_limit; ++d) { + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(__fp16)); + const HVX_Vector * pv_in1 = (r + 1 < g_rows_end) + ? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(__fp16)) + : NULL; + + for (uint32_t d = 0; d < d_limit; ++d) { + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + } +} + + +// Head-dim-padded Q-prep (f32). Used when DK is not a multiple of 64. +static inline void hmx_fa_q_prep_fp32_pad(__fp16 * vtcm_q_tiles, + const uint8_t * temp_q_vtcm, + size_t start, + size_t end, + size_t g_rows_end, + size_t dk_in, + size_t dk_out, + size_t G, + size_t n_rows_q, + const struct fastdiv_values * div_G, + bool q_transposed) { + const uint32_t n_out_tiles = (uint32_t) (dk_out / 32); + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * dk_out; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < n_out_tiles; ++d) { + ((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_UVector * pv_in0 = (const HVX_UVector *) (temp_q_vtcm + offset0 * dk_in * sizeof(float)); + const HVX_UVector * pv_in1 = (r + 1 < g_rows_end) ? (const HVX_UVector *) (temp_q_vtcm + offset1 * dk_in * sizeof(float)) : NULL; + + for (uint32_t d = 0; d < n_out_tiles; ++d) { + HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS); + const size_t base_lane = (size_t) d * 32; + const size_t real_lanes = (base_lane < dk_in) ? hex_smin(32, dk_in - base_lane) : 0; + + if (real_lanes == 0) { + out_tile[r1 / 2] = Q6_V_vzero(); + continue; + } + + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + if (real_lanes < 32) { + // Straddle tile: keep the first real_lanes floats, zero the padded tail so + // the packed f16 lanes beyond DK are zero. + const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(float))); + v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero()); + } + out_tile[r1 / 2] = hvx_vec_f32_to_f16_shuff(v0, v1); + } + } +} + +// Head-dim-padded Q-prep (f16). Used when DK is not a multiple of 64. +static inline void hmx_fa_q_prep_fp16_pad(__fp16 * vtcm_q_tiles, + const uint8_t * temp_q_vtcm, + size_t start, + size_t end, + size_t g_rows_end, + size_t dk_in, + size_t dk_out, + size_t G, + size_t n_rows_q, + const struct fastdiv_values * div_G, + bool q_transposed) { + const uint32_t n_out_pairs = (uint32_t) (dk_out / 64); + for (size_t r = start; r < end; r += 2) { + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * dk_out; + + if (r >= g_rows_end) { + for (uint32_t d = 0; d < n_out_pairs; ++d) { + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + } + continue; + } + + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0); + const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1); + + const HVX_UVector * pv_in0 = (const HVX_UVector *) (temp_q_vtcm + offset0 * dk_in * sizeof(__fp16)); + const HVX_UVector * pv_in1 = (r + 1 < g_rows_end) ? (const HVX_UVector *) (temp_q_vtcm + offset1 * dk_in * sizeof(__fp16)) : NULL; + + for (uint32_t d = 0; d < n_out_pairs; ++d) { + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + const size_t base_lane = (size_t) d * 64; + const size_t real_lanes = (base_lane < dk_in) ? hex_smin(64, dk_in - base_lane) : 0; + + if (real_lanes == 0) { + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + continue; + } + + HVX_Vector v0 = pv_in0[d]; + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + if (real_lanes < 64) { + const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(__fp16))); + v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero()); + } + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + } +} + +static inline void hmx_fa_q_prep_fallback( + __fp16 * vtcm_q_tiles, uintptr_t q_data, + size_t q_nb1, size_t q_nb2, size_t q_nb3, + uint32_t q_start, uint32_t kv_head, uint32_t ib3, + size_t start, size_t end, size_t n_rows_g, + size_t G, size_t dk_in, size_t dk_out, bool is_q_fp32, + const struct fastdiv_values * div_G +) { + for (size_t r = start; r < end; r += 2) { + const size_t q_idx0 = fastdiv(r + 0, div_G); + const size_t h_idx0 = fastmodulo(r + 0, G, div_G); + const size_t q_idx1 = fastdiv(r + 1, div_G); + const size_t h_idx1 = fastmodulo(r + 1, G, div_G); + + const uint8_t * q_ptr0 = (r + 0 < n_rows_g) ? ((const uint8_t *) q_data + (q_start + q_idx0) * q_nb1 + + (kv_head * G + h_idx0) * q_nb2 + ib3 * q_nb3) : + NULL; + const uint8_t * q_ptr1 = (r + 1 < n_rows_g) ? ((const uint8_t *) q_data + (q_start + q_idx1) * q_nb1 + + (kv_head * G + h_idx1) * q_nb2 + ib3 * q_nb3) : + NULL; + + size_t r0 = r / HMX_FP16_TILE_N_ROWS; + size_t r1 = r % HMX_FP16_TILE_N_ROWS; + __fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * dk_out; + + if (is_q_fp32) { + const HVX_UVector * pv_in0 = q_ptr0 ? (const HVX_UVector *) q_ptr0 : NULL; + const HVX_UVector * pv_in1 = q_ptr1 ? (const HVX_UVector *) q_ptr1 : NULL; + + for (uint32_t d = 0; d < dk_out / 32; ++d) { + HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS); + const size_t base_lane = (size_t) d * 32; + const size_t real_lanes = (base_lane < dk_in) ? hex_smin(32, dk_in - base_lane) : 0; + + if (real_lanes == 0) { + out_tile[r1 / 2] = Q6_V_vzero(); + continue; + } + HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero(); + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + if (real_lanes < 32) { + const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(float))); + v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero()); + } + out_tile[r1 / 2] = hvx_vec_f32_to_f16_shuff(v0, v1); + } + } else { + const HVX_UVector * pv_in0 = q_ptr0 ? (const HVX_UVector *) q_ptr0 : NULL; + const HVX_UVector * pv_in1 = q_ptr1 ? (const HVX_UVector *) q_ptr1 : NULL; + + for (uint32_t d = 0; d < dk_out / 64; ++d) { + __fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2; + HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2; + HVX_Vector * pv_out1 = pv_out0 + 16; + + const size_t base_lane = (size_t) d * 64; + const size_t real_lanes = (base_lane < dk_in) ? hex_smin(64, dk_in - base_lane) : 0; + + if (real_lanes == 0) { + *pv_out0 = Q6_V_vzero(); + *pv_out1 = Q6_V_vzero(); + continue; + } + HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero(); + HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero(); + if (real_lanes < 64) { + const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(__fp16))); + v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero()); + } + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + *pv_out0 = Q6_V_lo_W(vp); + *pv_out1 = Q6_V_hi_W(vp); + } + } + } +} + +#endif /* HMX_FA_KERNELS_H */ diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c deleted file mode 100644 index dbca822..0000000 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ /dev/null @@ -1,1810 +0,0 @@ -#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" -#pragma clang diagnostic ignored "-Wunused-function" -#pragma clang diagnostic ignored "-Wunused-variable" -#pragma clang diagnostic ignored "-Wunused-but-set-variable" - -#include -#include -#include -#include -#include - -#include -#include - -#define GGML_COMMON_DECL_C -#include "ggml-common.h" - -#include "hex-dma.h" -#include "worker-pool.h" - -#include "hvx-utils.h" -#include "hvx-dump.h" -#include "htp-ctx.h" -#include "htp-ops.h" - -#include "hmx-ops.h" -#include "hmx-utils.h" -#include "hmx-queue.h" -#include "hmx-profile.h" - -static const __fp16 q4_0_to_fp16_lut[64] __attribute__((aligned(VLEN))) = { - -8, 0, -7, 0, -6, 0, -5, 0, -4, 0, -3, 0, -2, 0, -1, 0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, -}; - -// MXFP4 dequantization LUT: maps 4-bit index to fp16 mantissa value -// kvalues: 0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6 -static const __fp16 mxfp4_to_fp16_lut[64] __attribute__((aligned(VLEN))) = { - 0, 0, 0.5, 0, 1, 0, 1.5, 0, 2, 0, 3, 0, 4, 0, 6, 0, 0, 0, -0.5, 0, -1, 0, -1.5, 0, -2, 0, -3, 0, -4, 0, -6, 0, -}; - -static const __fp16 iq4_nl_to_fp16_lut[64] __attribute__((aligned(VLEN))) = { - -127, 0, -104, 0, -83, 0, -65, 0, -49, 0, -35, 0, -22, 0, -10, 0, - 1, 0, 13, 0, 25, 0, 38, 0, 53, 0, 69, 0, 89, 0, 113, 0, -}; - -// vscatter offsets for fused dequant+transpose: write K-values directly to [K][N] tile. -// word[i] = i*128 maps K-row-pair i to byte offset i*128 in the tile. -// Column offset (n*4) is added at runtime. Only entries 0..15 are used (masked by predicate). -static const int32_t weight_transpose_scatter_offsets[32] __attribute__((aligned(VLEN))) = { - 0*128, 1*128, 2*128, 3*128, 4*128, 5*128, 6*128, 7*128, - 8*128, 9*128, 10*128, 11*128, 12*128, 13*128, 14*128, 15*128, - 16*128, 17*128, 18*128, 19*128, 20*128, 21*128, 22*128, 23*128, - 24*128, 25*128, 26*128, 27*128, 28*128, 29*128, 30*128, 31*128 -}; - -// Scales per x4x2 logical block: 8 × sizeof(__fp16) = 16 bytes -#define HMX_X4X2_SCALES_PER_BLK 8 -#define HMX_X4X2_DBLK_SIZE 16 // 8 * 2 bytes (fp16 scales for Q4_0/Q8_0/IQ4_NL) -#define HMX_X4X2_MXFP4_EBLK_SIZE 8 // 8 * 1 byte (E8M0 scales for MXFP4) - -static inline void swap_ptr(void **p1, void **p2) { - void *t = *p1; - *p1 = *p2; - *p2 = t; -} - -typedef struct { - uint8_t *dst; - const uint8_t *src; - dma_queue *dma; - size_t n_rows; - size_t src_stride; // DDR row stride (full row_stride) - size_t dst_stride; // VTCM sub-block row stride - size_t quant_off; // quant byte offset in each DDR row - size_t quant_width; // quant bytes to copy per row - size_t scale_off; // scale byte offset in each DDR row - size_t scale_width; // scale bytes to copy per row -} qweight_fetch_task_state_t; - -// Compute the byte stride of one row in x4x2 format. -// Numerically equals ggml_row_size(type, k) when k is 256-aligned, because -// x4x2 packing has the same density as block_q4_0 / block_q8_0. -// Layout per row: [quants: nb*128 (Q4) or nb*256 (Q8)][scales: nb*16 bytes] -// Total per row = nb * (128+16) = 144*nb (Q4) or nb * (256+16) = 272*nb (Q8). -// Callers must ensure k is a multiple of 256 (enforced by proc_hmx_matmul_req). -static inline size_t get_x4x2_row_stride(int weight_type, int k) { - int nb = (k + QK_Q4_0x4x2 - 1) / QK_Q4_0x4x2; - switch (weight_type) { - case HTP_TYPE_Q4_0: - case HTP_TYPE_IQ4_NL: - return (size_t) nb * (QK_Q4_0x4x2 / 2 + HMX_X4X2_DBLK_SIZE); // 144 * nb - case HTP_TYPE_Q8_0: - return (size_t) nb * (QK_Q8_0x4x2 + HMX_X4X2_DBLK_SIZE); // 272 * nb - case HTP_TYPE_MXFP4: - return (size_t) nb * (QK_MXFP4x4x2 / 2 + HMX_X4X2_MXFP4_EBLK_SIZE); // 136 * nb - default: - return 0; - } -} - -// --- Overflow-safe arithmetic for VTCM budget calculation --- - -static inline bool hmx_mul_overflow(size_t a, size_t b, size_t *out) { - if (a != 0 && b > SIZE_MAX / a) return true; - *out = a * b; - return false; -} - -static inline bool hmx_add_overflow(size_t a, size_t b, size_t *out) { - if (a > SIZE_MAX - b) return true; - *out = a + b; - return false; -} - -// Search for optimal (mc, nc) chunk sizes within VTCM budget. -// -// VTCM model: nc * per_n_cost + mc * per_m_cost + mc * nc * per_mn_cost + overhead -// -// Minimize ceil(m/mc) * m_block_cost + ceil(n/nc) * n_block_cost. -// All matmul paths repeat weight processing per M-block and activation loading -// per N-block, so discrete block counts drive total overhead. -// Tie-break: when cost is equal, prefer larger mc * nc. -// -// Caller-provided coefficients: -// m_block_cost: penalty per extra M-block (weight redundancy, scales with n). -// n_block_cost: penalty per extra N-block (activation redundancy, scales with m). -// -// Algorithm: nc sweeps from n_max down by 32, analytically solving for mc_max. -// Returns 0 on success, -1 if VTCM is insufficient. -static int hmx_compute_chunks(size_t vtcm_total, - size_t overhead, - size_t per_n_cost, - size_t per_m_cost, - size_t per_mn_cost, - int m, - int n, - size_t m_block_cost, - size_t n_block_cost, - size_t * m_chunk_out, - size_t * n_chunk_out, - size_t * total_out) { - if (m <= 0 || n <= 0) return -1; - if (vtcm_total <= overhead) return -1; - if (per_n_cost == 0 || per_m_cost == 0 || per_mn_cost == 0) return -1; - - const size_t usable = vtcm_total - overhead; - - size_t best_cost = SIZE_MAX; - size_t best_mn = 0; - size_t best_m = 0, best_n = 0; - - const size_t n_max = hex_align_down((size_t)n, HMX_FP16_TILE_N_COLS); - for (size_t nc = n_max; nc >= HMX_FP16_TILE_N_COLS; nc -= HMX_FP16_TILE_N_COLS) { - size_t n_fixed = 0, ncmn = 0, mc_denom = 0; - if (hmx_mul_overflow(nc, per_n_cost, &n_fixed)) continue; - if (n_fixed >= usable) goto next_nc; - - if (hmx_mul_overflow(nc, per_mn_cost, &ncmn)) goto next_nc; - if (hmx_add_overflow(per_m_cost, ncmn, &mc_denom) || mc_denom == 0) goto next_nc; - - { - size_t remain = usable - n_fixed; - size_t mc = remain / mc_denom; - mc = hex_align_down(mc, HMX_FP16_TILE_N_ROWS); - mc = hex_smin(mc, (size_t)m); - - if (mc == 0) { - goto next_nc; - } - - size_t mblocks = ((size_t) m + mc - 1) / mc; - size_t nblocks = ((size_t) n + nc - 1) / nc; - size_t cost = mblocks * m_block_cost + nblocks * n_block_cost; - size_t mn = mc * nc; - if (cost < best_cost || (cost == best_cost && mn > best_mn)) { - best_cost = cost; - best_mn = mn; - best_m = mc; - best_n = nc; - } - } - -next_nc: - if (nc == HMX_FP16_TILE_N_COLS) break; // avoid size_t underflow - } - - if (best_m == 0 || best_n == 0) return -1; - - // Compute exact total (with overflow checks) - size_t t0 = 0, t1 = 0, t2 = 0, mn = 0, total = 0; - if (hmx_mul_overflow(best_n, per_n_cost, &t0)) return -1; - if (hmx_mul_overflow(best_m, per_m_cost, &t1)) return -1; - if (hmx_mul_overflow(best_m, best_n, &mn)) return -1; - if (hmx_mul_overflow(mn, per_mn_cost, &t2)) return -1; - if (hmx_add_overflow(t0, t1, &total)) return -1; - if (hmx_add_overflow(total, t2, &total)) return -1; - if (hmx_add_overflow(total, overhead, &total)) return -1; - - *m_chunk_out = best_m; - *n_chunk_out = best_n; - *total_out = total; - return 0; -} - -// forward declaration – defined after transfer_activation_chunk_fp32_to_fp16 -void transfer_activation_chunk_threaded(struct htp_context *ctx, __fp16 *dst, const float *src, int n_rows, int k_block, int k_stride); - -// Scatter row-major FP16 weight (already in VTCM scratch) directly into transposed [K][N] tiles. -// vtcm_src: [n_cols][k] row-major fp16 in VTCM scratch buffer -// vtcm_dst: [n_col_tiles][n_k_tiles][HMX_FP16_TILE_N_ELMS] tile-major interleaved fp16 -static void interleave_fp16_weight_chunk_to_tiles(__fp16 *restrict vtcm_dst, - const __fp16 *restrict vtcm_src, - int n_cols, int k) { - assert(n_cols % HMX_FP16_TILE_N_COLS == 0); - assert(k % HMX_FP16_TILE_N_COLS == 0); - - const int n_k_tiles = k / HMX_FP16_TILE_N_COLS; - const HVX_Vector v_scat_base = hvx_vmem(weight_transpose_scatter_offsets); - const HVX_Vector v_scat_step = Q6_V_vsplat_R(4); - const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64); - - for (int r = 0; r < n_cols; r += 2) { - int ct = r / HMX_FP16_TILE_N_ROWS; // N-dimension tile index - int local_r = r % HMX_FP16_TILE_N_ROWS; // intra-tile row index - const bool next_row_valid = (r + 1) < n_cols; - - // Offset vectors for N-columns local_r and local_r+1, reused across K-tiles. - HVX_Vector v_off0 = Q6_Vw_vadd_VwVw(v_scat_base, Q6_V_vsplat_R(local_r * 4)); - HVX_Vector v_off1 = Q6_Vw_vadd_VwVw(v_off0, v_scat_step); - - for (int c = 0; c < k; c += HMX_FP16_TILE_N_COLS) { - int kt = c / HMX_FP16_TILE_N_COLS; - int tile_idx = ct * n_k_tiles + kt; - __fp16 *tile_base = vtcm_dst + tile_idx * HMX_FP16_TILE_N_ELMS; - - HVX_Vector v0 = hvx_vmemu(vtcm_src + r * k + c); - HVX_Vector v1 = next_row_valid ? hvx_vmemu(vtcm_src + (r + 1) * k + c) : Q6_V_vzero(); - - Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HMX_FP16_TILE_SIZE - 1, v_off0, v0); - Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HMX_FP16_TILE_SIZE - 1, v_off1, v1); - } - } -} - -// --- x4x2 format dequantizers --- - -// Dequantize one x4x2 Q4_0 group (32 elements from 32 packed bytes) -> 32 FP16 in first 64 bytes. -// In x4x2, sub-blocks 0..3 use lower nibbles, sub-blocks 4..7 use upper nibbles -// of the same 32 packed bytes. -static inline HVX_Vector dequantize_x4x2_q4_0_group_hvx( - const uint8_t *packed_32, bool upper_nibbles, - const __fp16 *scale, const HVX_Vector vlut_cvt) { - HVX_Vector vq = hvx_vmemu(packed_32); - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - HVX_Vector v_scales = hvx_vec_splat_f16(*scale); - // q4x4x2 stores two int4 values per byte. Keep only the selected nibble. - HVX_Vector v_quants = Q6_Vub_vlsr_VubR(vq, 4 * upper_nibbles); - v_quants = Q6_V_vand_VV(v_quants, mask_h4); - // Shuffle before LUT - v_quants = Q6_Vb_vshuff_Vb(v_quants); - // Use standard vlut16 (not _nomatch) to avoid stale-register NaN. - // _nomatch retains the previous destination-register value for colliding - // indices, but the C intrinsic doesn't model the implicit read so the - // compiler may allocate a register containing garbage/NaN. - HVX_VectorPair vp = Q6_Wh_vlut16_VbVhR(v_quants, vlut_cvt, 0); - HVX_Vector v_hf = Q6_V_lo_W(vp); - - return Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(v_hf, v_scales)); -} - -// Batch-dequantize 4 contiguous x4x2 Q4_0 groups (4x32 = 128 packed bytes) using -// full HVX vector width. One vmemu + one vlut16 replaces 4 separate calls. -// Output: out[0..3] each hold 32 FP16 values in the first 64 bytes. -static inline void dequantize_x4x2_q4_0_x4groups_hvx( - const uint8_t *packed_128, bool upper_nibbles, - const __fp16 *scales_4, const HVX_Vector vlut_cvt, - HVX_Vector out[4]) { - // Load all 128 packed bytes (4 contiguous 32-byte groups) - HVX_Vector vq = hvx_vmemu(packed_128); - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - HVX_Vector v_quants = Q6_Vub_vlsr_VubR(vq, 4 * upper_nibbles); - v_quants = Q6_V_vand_VV(v_quants, mask_h4); - - // Shuffle before LUT - v_quants = Q6_Vb_vshuff_Vb(v_quants); - - // Full-width vlut16: 128 byte lookups -> 128 fp16 results in a VectorPair - HVX_VectorPair vp = Q6_Wh_vlut16_VbVhR(v_quants, vlut_cvt, 0); - HVX_Vector v_lo = Q6_V_lo_W(vp); // [group0: 32 fp16 | group1: 32 fp16] - HVX_Vector v_hi = Q6_V_hi_W(vp); // [group2: 32 fp16 | group3: 32 fp16] - - // Build per-group scale vectors: first 64 bytes use scale_a, last 64 use scale_b - HVX_VectorPred q64 = Q6_Q_vsetq_R(64); - HVX_Vector v_sc01 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f16(scales_4[0]), hvx_vec_splat_f16(scales_4[1])); - HVX_Vector v_sc23 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f16(scales_4[2]), hvx_vec_splat_f16(scales_4[3])); - - v_lo = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(v_lo, v_sc01)); - v_hi = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(v_hi, v_sc23)); - - // Extract individual groups: scatter uses q_mask64 so only first 64 bytes matter - out[0] = v_lo; // group0 already in [0:63] - out[1] = v_hi; // group2 already in [0:63] -} - -// Dequantize one x4x2 Q8_0 group (32 int8 quants) -> 32 FP16 in first 64 bytes. -static inline HVX_Vector dequantize_x4x2_q8_0_group_hvx( - const int8_t *quants_32, const __fp16 *scale) { - HVX_Vector vq = hvx_vmemu(quants_32); - HVX_Vector v_scales = hvx_vec_splat_f16(*scale); - HVX_Vector v0 = Q6_V_lo_W(Q6_Wh_vunpack_Vb(vq)); - HVX_Vector v_hf = Q6_Vhf_equals_Vh(v0); - return Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(v_hf, v_scales)); -} - -// --- MXFP4 E8M0 scale conversion and dequantization --- -// -// HVX batch-convert 8 E8M0 bytes (one x4x2 block's scales) to __fp16[8] on stack. -// Scalar loads from the stack array execute on the scalar pipeline, in parallel -// with HVX vlut16/vmpy/vscatter — freeing HVX slots in the hot loop. -// Arithmetic: fp16_bits = clamp(e - 112, 0, 30) << 10 -// e=0..112 -> 0 (underflow), e=113..142 -> valid fp16, e>=143 -> clamped to 2^15. - -typedef struct { - __fp16 v[8] __attribute__((aligned(16))); -} mxfp4_scales_t; - -static inline mxfp4_scales_t mxfp4_convert_scales(const uint8_t * e8m0_8) { - mxfp4_scales_t s; - HVX_Vector v = hvx_vmemu(e8m0_8); - HVX_Vector vh = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(v)); - vh = Q6_Vh_vsub_VhVh(vh, Q6_Vh_vsplat_R(112)); - vh = Q6_Vh_vmax_VhVh(vh, Q6_V_vzero()); - vh = Q6_Vh_vmin_VhVh(vh, Q6_Vh_vsplat_R(30)); - vh = Q6_Vh_vasl_VhR(vh, 10); - hvx_vec_store_u(s.v, 16, vh); - return s; -} - -static inline HVX_Vector mxfp4_extract_splat(mxfp4_scales_t scales, int idx) { - return hvx_vec_splat_f16(scales.v[idx]); -} - -// Dequantize one x4x2 MXFP4 group (32 elements from 32 packed bytes) -> 32 FP16. -static inline HVX_Vector dequantize_x4x2_mxfp4_group_hvx(const uint8_t * packed_32, - bool upper_nibbles, - int sub_blk, - const HVX_Vector vlut_cvt, - mxfp4_scales_t scales) { - HVX_Vector vq = hvx_vmemu(packed_32); - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - HVX_Vector v_quants = upper_nibbles ? Q6_Vub_vlsr_VubR(vq, 4) : vq; - v_quants = Q6_V_vand_VV(v_quants, mask_h4); - - HVX_Vector v_sc = mxfp4_extract_splat(scales, sub_blk); - - v_quants = Q6_Vb_vshuff_Vb(v_quants); - HVX_VectorPair vp = Q6_Wh_vlut16_VbVhR(v_quants, vlut_cvt, 0); - HVX_Vector v_hf = Q6_V_lo_W(vp); - - return Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(v_hf, v_sc)); -} - -// Batch-dequantize 4 contiguous x4x2 MXFP4 groups (4x32 = 128 packed bytes). -static inline void dequantize_x4x2_mxfp4_x4groups_hvx(const uint8_t * packed_128, - bool upper_nibbles, - int sub_blk_base, - const HVX_Vector vlut_cvt, - mxfp4_scales_t scales, - HVX_Vector out[4]) { - HVX_Vector vq = hvx_vmemu(packed_128); - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - HVX_Vector v_quants = upper_nibbles ? Q6_Vub_vlsr_VubR(vq, 4) : vq; - v_quants = Q6_V_vand_VV(v_quants, mask_h4); - - v_quants = Q6_Vb_vshuff_Vb(v_quants); - - HVX_VectorPair vp = Q6_Wh_vlut16_VbVhR(v_quants, vlut_cvt, 0); - HVX_Vector v_lo = Q6_V_lo_W(vp); - HVX_Vector v_hi = Q6_V_hi_W(vp); - - HVX_VectorPred q64 = Q6_Q_vsetq_R(64); - HVX_Vector v_sc01 = Q6_V_vmux_QVV(q64, mxfp4_extract_splat(scales, sub_blk_base + 0), - mxfp4_extract_splat(scales, sub_blk_base + 1)); - HVX_Vector v_sc23 = Q6_V_vmux_QVV(q64, mxfp4_extract_splat(scales, sub_blk_base + 2), - mxfp4_extract_splat(scales, sub_blk_base + 3)); - - v_lo = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(v_lo, v_sc01)); - v_hi = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(v_hi, v_sc23)); - - out[0] = v_lo; - out[1] = Q6_V_vror_VR(v_lo, 64); - out[2] = v_hi; - out[3] = Q6_V_vror_VR(v_hi, 64); -} - -// Dequantize a tile range from x4x2 weight data (already in VTCM) to tile-major FP16. -// Input: vtcm_src has n_cols rows of x4x2 data, each row_stride bytes. -// Output: vtcm_dst in tile-major FP16 layout. -static void dequantize_x4x2_weight_to_fp16_tiles_task( - __fp16 *restrict vtcm_dst, - const uint8_t *restrict vtcm_src, - int n_cols, int k_block, - size_t row_stride, int weight_type, - int start_tile, int end_tile) { - - const int n_k_tiles = (unsigned)k_block / HMX_FP16_TILE_N_COLS; - const bool is_q4 = (weight_type == HTP_TYPE_Q4_0 || weight_type == HTP_TYPE_IQ4_NL); - const int qrow_size = is_q4 ? ((unsigned)k_block / 2) : k_block; - - const HVX_Vector vlut_cvt = (weight_type == HTP_TYPE_IQ4_NL) ? hvx_vmem(iq4_nl_to_fp16_lut) : - (weight_type == HTP_TYPE_MXFP4) ? hvx_vmem(mxfp4_to_fp16_lut) : - hvx_vmem(q4_0_to_fp16_lut); - - // vscatter setup: write dequantized K-values directly to transposed [K][N] tile positions. - // Each int32 element holds a K-row-pair (2 adjacent fp16 values). word[i] at offset i*128 - // maps to K-rows 2i and 2i+1. Column offset (n*4) added per row. - const HVX_Vector v_scat_base = hvx_vmem(weight_transpose_scatter_offsets); - const HVX_Vector v_scat_step = Q6_V_vsplat_R(4); // 4 bytes = 1 column step - const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64); // first 16 words (64 bytes) - - unsigned ct = (unsigned)start_tile / n_k_tiles; // column tile index - unsigned kt = (unsigned)start_tile % n_k_tiles; // K tile index - for (unsigned t = start_tile; t < end_tile; ) { - if (kt >= n_k_tiles) { kt = 0; ct++; } - - // --- Batch-4 fast path for Q4: process 4 contiguous K-tiles with one vlut16 per row --- - if (is_q4 && (kt % 4 == 0) && (t + 4 <= end_tile) && ((t + 3) / n_k_tiles == ct)) { - unsigned blk_idx = (kt * 32) / QK_Q4_0x4x2; - unsigned sub_blk_base = ((kt * 32) % QK_Q4_0x4x2) / 32; // 0 or 4 - bool upper = (sub_blk_base >= 4); - unsigned packed_off = blk_idx * (QK_Q4_0x4x2 / 2); // 128 contiguous packed bytes - unsigned scale_off = qrow_size + blk_idx * HMX_X4X2_DBLK_SIZE - + sub_blk_base * (int)sizeof(__fp16); // 4 consecutive scales - - __fp16 *tile_bases[4]; - for (unsigned g = 0; g < 4; g++) { tile_bases[g] = vtcm_dst + (t + g) * HMX_FP16_TILE_N_ELMS; } - - HVX_Vector v_off = v_scat_base; - - unsigned row_offset = ct * HMX_FP16_TILE_N_COLS * row_stride; - unsigned row1 = ct * HMX_FP16_TILE_N_COLS + 1; - - for (int r = 0; r < HMX_FP16_TILE_N_ROWS; r += 2, row1 += 2) { - HVX_Vector v0[2]; - const uint8_t *r0 = vtcm_src + row_offset; row_offset += row_stride; - dequantize_x4x2_q4_0_x4groups_hvx(r0 + packed_off, upper, (const __fp16 *)(r0 + scale_off), vlut_cvt, v0); - Q6_vscatter_RMVwV((size_t)tile_bases[0], 2 * HMX_FP16_TILE_SIZE - 1, v_off, v0[0]); - Q6_vscatter_RMVwV((size_t)tile_bases[2], 2 * HMX_FP16_TILE_SIZE - 1, v_off, v0[1]); - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - - - r0 = vtcm_src + row_offset; row_offset += row_stride; - dequantize_x4x2_q4_0_x4groups_hvx(r0 + packed_off, upper, (const __fp16 *)(r0 + scale_off), vlut_cvt, v0); - Q6_vscatter_RMVwV((size_t)tile_bases[0], 2 * HMX_FP16_TILE_SIZE - 1, v_off, v0[0]); - Q6_vscatter_RMVwV((size_t)tile_bases[2], 2 * HMX_FP16_TILE_SIZE - 1, v_off, v0[1]); - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - } - - for (int g = 0; g < 4; g++) { (void) *(volatile HVX_Vector *)(tile_bases[g]); } - t += 4; kt += 4; - continue; - } - - // --- Batch-4 fast path for MXFP4: same nibble layout but E8M0 scales --- - if (weight_type == HTP_TYPE_MXFP4 && (kt % 4 == 0) && (t + 4 <= end_tile) && ((t + 3) / n_k_tiles == ct)) { - int blk_idx = (kt * 32) / QK_MXFP4x4x2; - int sub_blk_base = ((kt * 32) % QK_MXFP4x4x2) / 32; // 0 or 4 - bool upper = (sub_blk_base >= 4); - int packed_off = blk_idx * (QK_MXFP4x4x2 / 2); // 128 contiguous packed bytes - int e8m0_blk_off = qrow_size + blk_idx * HMX_X4X2_MXFP4_EBLK_SIZE; // all 8 E8M0 scales - - __fp16 * tile_bases[4]; - for (int g = 0; g < 4; g++) { - tile_bases[g] = vtcm_dst + (t + g) * HMX_FP16_TILE_N_ELMS; - } - - HVX_Vector v_off = v_scat_base; - for (int r = 0; r < HMX_FP16_TILE_N_ROWS; r += 2) { - int row0 = ct * HMX_FP16_TILE_N_COLS + r; - int row1 = row0 + 1; - const uint8_t * r0 = vtcm_src + row0 * row_stride; - const uint8_t * r1 = vtcm_src + row1 * row_stride; - - // Batch-convert all 8 E8M0 scales once per row (stays in HVX register) - mxfp4_scales_t r0_e8 = mxfp4_convert_scales(r0 + e8m0_blk_off); - - HVX_Vector v0[4], v1[4]; - dequantize_x4x2_mxfp4_x4groups_hvx(r0 + packed_off, upper, sub_blk_base, vlut_cvt, r0_e8, v0); - if (row1 < n_cols) { - mxfp4_scales_t r1_e8 = mxfp4_convert_scales(r1 + e8m0_blk_off); - dequantize_x4x2_mxfp4_x4groups_hvx(r1 + packed_off, upper, sub_blk_base, vlut_cvt, r1_e8, v1); - } else { - v1[0] = v1[1] = v1[2] = v1[3] = Q6_V_vzero(); - } - - for (int g = 0; g < 4; g++) { - Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_bases[g], HMX_FP16_TILE_SIZE - 1, v_off, v0[g]); - } - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - for (int g = 0; g < 4; g++) { - Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_bases[g], HMX_FP16_TILE_SIZE - 1, v_off, v1[g]); - } - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - } - - for (int g = 0; g < 4; g++) { - (void) *(volatile HVX_Vector *) (tile_bases[g]); - } - - t += 4; - continue; - } - - // --- Single-tile fallback --- - __fp16 *tile_base = vtcm_dst + t * HMX_FP16_TILE_N_ELMS; - - if (is_q4) { - unsigned blk_idx = (kt * 32) / QK_Q4_0x4x2; - unsigned sub_blk = ((kt * 32) % QK_Q4_0x4x2) / 32; - bool upper = (sub_blk >= 4); - unsigned byte_off = blk_idx * (QK_Q4_0x4x2 / 2) + (upper ? (sub_blk - 4) : sub_blk) * 32; - unsigned scale_off = qrow_size + blk_idx * HMX_X4X2_DBLK_SIZE + sub_blk * (int)sizeof(__fp16); - - HVX_Vector v_off = v_scat_base; // reset to column 0 - unsigned row_offset = ct * HMX_FP16_TILE_N_COLS * row_stride; - unsigned row1 = ct * HMX_FP16_TILE_N_COLS + 1; - for (int r = 0; r < HMX_FP16_TILE_N_ROWS; r += 2, row1 += 2) { - const uint8_t *r0 = vtcm_src + row_offset; row_offset += row_stride; - const uint8_t *r1 = vtcm_src + row_offset; row_offset += row_stride; - - HVX_Vector v0 = dequantize_x4x2_q4_0_group_hvx( - r0 + byte_off, upper, (const __fp16 *)(r0 + scale_off), vlut_cvt); - HVX_Vector v1 = (row1 < n_cols) - ? dequantize_x4x2_q4_0_group_hvx( - r1 + byte_off, upper, (const __fp16 *)(r1 + scale_off), vlut_cvt) - : Q6_V_vzero(); - - Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HMX_FP16_TILE_SIZE - 1, v_off, v0); - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HMX_FP16_TILE_SIZE - 1, v_off, v1); - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - } - (void) *(volatile HVX_Vector *)(tile_base); - } else if (weight_type == HTP_TYPE_MXFP4) { - int blk_idx = (kt * 32) / QK_MXFP4x4x2; - int sub_blk = ((kt * 32) % QK_MXFP4x4x2) / 32; - bool upper = (sub_blk >= 4); - int byte_off = blk_idx * (QK_MXFP4x4x2 / 2) + (upper ? (sub_blk - 4) : sub_blk) * 32; - int e8m0_blk_off = qrow_size + blk_idx * HMX_X4X2_MXFP4_EBLK_SIZE; - - HVX_Vector v_off = v_scat_base; - for (int r = 0; r < HMX_FP16_TILE_N_ROWS; r += 2) { - int row0 = ct * HMX_FP16_TILE_N_COLS + r; - int row1 = row0 + 1; - - const uint8_t * r0 = vtcm_src + row0 * row_stride; - const uint8_t * r1 = vtcm_src + row1 * row_stride; - - // Batch-convert all 8 E8M0 scales once per row (stays in HVX register) - mxfp4_scales_t r0_e8 = mxfp4_convert_scales(r0 + e8m0_blk_off); - - HVX_Vector v0 = dequantize_x4x2_mxfp4_group_hvx(r0 + byte_off, upper, sub_blk, vlut_cvt, r0_e8); - HVX_Vector v1; - if (row1 < n_cols) { - mxfp4_scales_t r1_e8 = mxfp4_convert_scales(r1 + e8m0_blk_off); - v1 = dequantize_x4x2_mxfp4_group_hvx(r1 + byte_off, upper, sub_blk, vlut_cvt, r1_e8); - } else { - v1 = Q6_V_vzero(); - } - - Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, HMX_FP16_TILE_SIZE - 1, v_off, v0); - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, HMX_FP16_TILE_SIZE - 1, v_off, v1); - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - } - (void) *(volatile HVX_Vector *) (tile_base); - } else { - // Q8_0 - int blk_idx = (kt * 32) / QK_Q8_0x4x2; - int sub_blk = ((kt * 32) % QK_Q8_0x4x2) / 32; - int byte_off = blk_idx * QK_Q8_0x4x2 + sub_blk * 32; - int scale_off = qrow_size + blk_idx * HMX_X4X2_DBLK_SIZE + sub_blk * (int)sizeof(__fp16); - - HVX_Vector v_off = v_scat_base; // reset to column 0 - for (int r = 0; r < HMX_FP16_TILE_N_ROWS; r += 2) { - int row0 = ct * HMX_FP16_TILE_N_COLS + r; - int row1 = row0 + 1; - - const uint8_t *r0 = vtcm_src + row0 * row_stride; - const uint8_t *r1 = vtcm_src + row1 * row_stride; - - HVX_Vector v0 = dequantize_x4x2_q8_0_group_hvx( - (const int8_t *)(r0 + byte_off), (const __fp16 *)(r0 + scale_off)); - HVX_Vector v1 = (row1 < n_cols) - ? dequantize_x4x2_q8_0_group_hvx( - (const int8_t *)(r1 + byte_off), (const __fp16 *)(r1 + scale_off)) - : Q6_V_vzero(); - - Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HMX_FP16_TILE_SIZE - 1, v_off, v0); - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HMX_FP16_TILE_SIZE - 1, v_off, v1); - v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); - } - (void) *(volatile HVX_Vector *)(tile_base); - } - ++t; ++kt; - } - - // Drain HVX scatter write buffer: a vmem load on the same HW thread retires - // all pending scatter entries to VTCM. Without this, the main thread's HMX - // reads may see stale data because atomic_fetch_sub (release) only orders - // regular stores, not the HVX scatter buffer. - if (start_tile < end_tile) { - (void) *(volatile HVX_Vector *)(vtcm_dst + (end_tile - 1) * HMX_FP16_TILE_N_ELMS); - } -} - -typedef struct { - __fp16 *dst; - const uint8_t *src; - int n_cols; - int k_block; - size_t row_stride; - int weight_type; - int n_tot_tiles; - int n_tiles_per_task; - int n_tasks; -} x4x2_dequantize_state_t; - -static void dequantize_x4x2_worker_loop(unsigned int n, unsigned int i, void *data) { - x4x2_dequantize_state_t *state = (x4x2_dequantize_state_t *)data; - - for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { - int start = task_id * state->n_tiles_per_task; - int end = hex_smin(start + state->n_tiles_per_task, state->n_tot_tiles); - - dequantize_x4x2_weight_to_fp16_tiles_task( - state->dst, state->src, state->n_cols, state->k_block, - state->row_stride, state->weight_type, start, end); - } -} - -static void dequantize_x4x2_weight_chunk_to_fp16_tiles( - struct htp_context *ctx, __fp16 *vtcm_dst, - const void *vtcm_src, int n_cols, int k_block, - size_t row_stride, int weight_type) { - - assert(n_cols % HMX_FP16_TILE_N_COLS == 0); - assert(k_block % HMX_FP16_TILE_N_COLS == 0); - - size_t n_col_tiles = n_cols / HMX_FP16_TILE_N_COLS; - size_t n_k_tiles = k_block / HMX_FP16_TILE_N_COLS; - size_t n_tot_tiles = n_col_tiles * n_k_tiles; - - size_t n_tiles_per_task = hmx_ceil_div(n_tot_tiles, ctx->n_threads); - - x4x2_dequantize_state_t state; - state.n_tasks = (n_tot_tiles + n_tiles_per_task - 1) / n_tiles_per_task; - state.n_tot_tiles = n_tot_tiles; - state.n_tiles_per_task = n_tiles_per_task; - state.dst = vtcm_dst; - state.src = (const uint8_t *)vtcm_src; - state.n_cols = n_cols; - state.k_block = k_block; - state.row_stride = row_stride; - state.weight_type = weight_type; - - worker_pool_run_func(ctx->worker_pool, dequantize_x4x2_worker_loop, &state, ctx->n_threads); -} - -// --- End x4x2 dequantizers --- - -// requires external HMX lock -static void core_dot_chunk_fp16(__fp16 *restrict output, const __fp16 *restrict activation, const __fp16 *restrict weight, const __fp16 *restrict scales, - int n_row_tiles, int n_col_tiles, int n_dot_tiles) { - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(n_dot_tiles > 0); - - Q6_bias_mxmem2_A((void *)scales); - for (int r = 0; r < n_row_tiles; ++r) { - for (size_t c = 0; c < n_col_tiles; ++c) { - Q6_mxclracc_hf(); - - const __fp16 *row_tiles = activation + r * n_dot_tiles * HMX_FP16_TILE_N_ELMS; - const __fp16 *col_tiles = weight + c * n_dot_tiles * HMX_FP16_TILE_N_ELMS; - - for (int k = 0; k < n_dot_tiles; ++k) { - Q6_activation_hf_mxmem_RR((unsigned int)row_tiles, 2047); - Q6_weight_hf_mxmem_RR((unsigned int)col_tiles, 2047); - row_tiles += HMX_FP16_TILE_N_ELMS; - col_tiles += HMX_FP16_TILE_N_ELMS; - } - - __fp16 *out_tile = output + (r * n_col_tiles + c) * HMX_FP16_TILE_N_ELMS; - Q6_mxmem_AR_after_hf(out_tile, 0); - } - } -} - -// --- Async HMX matmul job (for pipeline overlap) --- - -typedef struct { - __fp16 * output; - const __fp16 * activation; - const __fp16 * weight; - const __fp16 * scales; - uint32_t n_row_tiles; - uint32_t n_col_tiles; - uint32_t n_dot_tiles; -} hmx_matmul_job_t; - -static void hmx_matmul_worker_fn(void * data) { - hmx_matmul_job_t * job = (hmx_matmul_job_t *) data; - FARF(HIGH, "hmx-mm-job: n_row_tiles %u n_col_tiles %u n_dot_tiles %u", job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); - core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales, job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); -} - -static inline void hmx_matmul_job_init(hmx_matmul_job_t * job, - __fp16 * output, - const __fp16 * activation, - const __fp16 * weight, - const __fp16 * scales, - int n_row_tiles, - int n_col_tiles, - int n_dot_tiles) { - job->output = output; - job->activation = activation; - job->weight = weight; - job->scales = scales; - job->n_row_tiles = n_row_tiles; - job->n_col_tiles = n_col_tiles; - job->n_dot_tiles = n_dot_tiles; -} - -// --- End async HMX matmul job --- - -static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 *restrict vtcm_src, int n_rows, int n_cols, int n) { - assert(n_cols % HMX_FP16_TILE_N_COLS == 0); - const size_t tile_row_stride = (n_cols / HMX_FP16_TILE_N_COLS) * HMX_FP16_TILE_N_ELMS; - - const HVX_Vector one = hvx_vec_splat_f16(1.0); - - for (size_t r = 0; r < n_rows; r += 2) { - const size_t r0 = r / HMX_FP16_TILE_N_ROWS; - const size_t r1 = (r % HMX_FP16_TILE_N_ROWS) / 2; // index of the row pair within the tile - const __fp16 *row_base = vtcm_src + r0 * tile_row_stride; - float *output_row_base = dst + r * n; // global memory row base for row r (and r+1) - - #pragma unroll(4) - for (size_t c = 0; c < n_cols; c += HMX_FP16_TILE_N_COLS) { - const size_t c0 = c / HMX_FP16_TILE_N_COLS; - const __fp16 *tile = row_base + c0 * HMX_FP16_TILE_N_ELMS; - HVX_Vector v = ((const HVX_Vector *) tile)[r1]; - HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one); - - volatile HVX_Vector *pv_out0 = (volatile HVX_Vector *) (output_row_base + c + 0); - volatile HVX_Vector *pv_out1 = (volatile HVX_Vector *) (output_row_base + c + n); // next row in global memory - - *pv_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); - if (r + 1 < n_rows) { - *pv_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); - } - } - } -} - -typedef struct { - const __fp16 *vtcm_src; - float *dst; - int n_tasks; - int n_tot_chunks; - int n_chunks_per_task; - int n_cols; - int n; // DDR row stride (total output columns) -} output_transfer_task_state_t; - -static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { - output_transfer_task_state_t *st = (output_transfer_task_state_t *) data; - - for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { - int chunk_idx = task_id * st->n_chunks_per_task; - size_t chunk_size = hex_smin(st->n_tot_chunks - chunk_idx, st->n_chunks_per_task); - - float *dst = st->dst + chunk_idx * st->n; - const __fp16 *vtcm_src = st->vtcm_src + chunk_idx * st->n_cols; - transfer_output_chunk_fp16_to_fp32(dst, vtcm_src, chunk_size, st->n_cols, st->n); - } -} - -static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, const __fp16 *vtcm_src, - int n_rows, int n_cols, int n) { - assert(n_cols % HMX_FP16_TILE_N_COLS == 0); - - size_t n_tot_chunks = n_rows; - size_t n_chunks_per_task = HMX_FP16_TILE_N_ROWS; // must be multiple of HMX_FP16_TILE_N_ROWS (32) - - output_transfer_task_state_t state; - state.n_tasks = (n_tot_chunks + n_chunks_per_task - 1) / n_chunks_per_task; - state.n_tot_chunks = n_tot_chunks; - state.n_chunks_per_task = n_chunks_per_task; - state.dst = dst; - state.vtcm_src = vtcm_src; - state.n_cols = n_cols; - state.n = n; - - worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_worker_fn, &state, ctx->n_threads); -} - -static inline int hmx_matmul_batch_r2(const hmx_matmul_w16a32_batched_params_t *params) { - return params->ne02 > 0 ? params->ne12 / params->ne02 : 1; -} - -static inline int hmx_matmul_batch_r3(const hmx_matmul_w16a32_batched_params_t *params) { - return params->ne03 > 0 ? params->ne13 / params->ne03 : 1; -} - -static inline const __fp16 *hmx_matmul_weight_batch_ptr(const hmx_matmul_w16a32_batched_params_t *params, - int dst_b2, int dst_b3) { - const int r2 = hmx_matmul_batch_r2(params); - const int r3 = hmx_matmul_batch_r3(params); - return (const __fp16 *) ((const uint8_t *) params->permuted_weight + - (size_t) (dst_b2 / r2) * params->src0_nb2 + - (size_t) (dst_b3 / r3) * params->src0_nb3); -} - -static inline const float *hmx_matmul_activation_batch_ptr(const hmx_matmul_w16a32_batched_params_t *params, - int dst_b2, int dst_b3) { - return (const float *) ((const uint8_t *) params->activation + - (size_t) dst_b2 * params->src1_nb2 + - (size_t) dst_b3 * params->src1_nb3); -} - -static inline float *hmx_matmul_dst_batch_ptr(const hmx_matmul_w16a32_batched_params_t *params, - int dst_b2, int dst_b3) { - return (float *) ((uint8_t *) params->dst + - (size_t) dst_b2 * params->dst_nb2 + - (size_t) dst_b3 * params->dst_nb3); -} - -static int hmx_mat_mul_permuted_w16a32_batched_legacy(struct htp_context *ctx, - const hmx_matmul_w16a32_batched_params_t *params) { - int ret = 0; - for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) { - for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) { - ret = hmx_mat_mul_permuted_w16a32(ctx, - hmx_matmul_dst_batch_ptr(params, b2, b3), - hmx_matmul_activation_batch_ptr(params, b2, b3), - hmx_matmul_weight_batch_ptr(params, b2, b3), - params->m, params->k, params->n, - params->act_stride, params->weight_stride); - } - } - return ret; -} - -int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmul_w16a32_batched_params_t *params) { - if (!ctx || !params || !params->dst || !params->activation || !params->permuted_weight) { return -1; } - if (!params->m || !params->k || !params->n) { return -1; } - if (params->act_stride < params->k || params->weight_stride < params->k || params->dst_stride < params->n) { return -1; } - if (params->ne02 <= 0 || params->ne03 <= 0 || params->ne12 <= 0 || params->ne13 <= 0) { return -1; } - if (params->ne12 % params->ne02 != 0 || params->ne13 % params->ne03 != 0) { return -1; } - if (params->k % 32 != 0 || params->n % 32 != 0) { return -1; } - - if (!hex_is_aligned(params->dst, VLEN) || - !hex_is_aligned(params->activation, VLEN) || - !hex_is_aligned(params->permuted_weight, VLEN)) { - return -1; - } - - const int group_size = hmx_matmul_batch_r2(params); - - if (group_size <= 1) { - FARF(MEDIUM, "%s: no dim2 GQA reuse (group=%d), using legacy batched loop", __func__, group_size); - return hmx_mat_mul_permuted_w16a32_batched_legacy(ctx, params); - } - - // Grouped path: reuse interleaved weight across all q_heads sharing a - // kv_head. Each q_head gets its own activation buffer in VTCM (so - // activation is loaded once per m_chunk and reused across all n_chunks), - // and each q_head is computed individually to avoid tile-major packing - // issues. m_chunk_n_rows is always a multiple of 32 (from - // hmx_compute_chunks), so per-head tile arrays don't overlap. - const size_t vtcm_budget = ctx->vtcm_size; - const size_t vec_dot_size = params->k * sizeof(__fp16); - - // When the activation has a large stride (e.g. permuted Q tensor with - // act_stride >> k), HVX vector loads from strided DDR thrash L2 cache. - // Allocate an F32 scratch buffer in VTCM and use 2D DMA to gather - // strided rows into a contiguous block before the F32->F16 conversion. - const bool use_dma_activation = (params->act_stride > params->k); - const size_t f32_scratch_per_m = use_dma_activation ? (size_t) params->k * sizeof(float) : 0; - - size_t m_chunk_n_rows = 0, n_chunk_n_cols = 0, vtcm_used = 0; - // FP16 weight: interleave and activation load have similar per-element cost. - if (hmx_compute_chunks(vtcm_budget, /*overhead=*/256, - /*per_n=*/3 * vec_dot_size, - /*per_m=*/group_size * vec_dot_size + f32_scratch_per_m, - /*per_mn=*/sizeof(__fp16), params->m, params->n, - /*m_block_cost=*/(size_t) params->n, - /*n_block_cost=*/(size_t) params->m, &m_chunk_n_rows, &n_chunk_n_cols, &vtcm_used) != 0) { - FARF(HIGH, "%s: grouped path does not fit VTCM, falling back to legacy batched loop", __func__); - return hmx_mat_mul_permuted_w16a32_batched_legacy(ctx, params); - } - - const size_t act_head_stride = m_chunk_n_rows * (size_t) params->k; // fp16 elements between heads - const size_t weight_area_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HMX_FP16_TILE_SIZE); - const size_t activation_area_size = hex_align_up(group_size * m_chunk_n_rows * vec_dot_size, HMX_FP16_TILE_SIZE); - const size_t output_area_size = hex_align_up(m_chunk_n_rows * n_chunk_n_cols * sizeof(__fp16), HMX_FP16_TILE_SIZE); - const size_t scratch_area_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HMX_FP16_TILE_SIZE); - const size_t f32_scratch_size = use_dma_activation - ? hex_align_up(m_chunk_n_rows * (size_t) params->k * sizeof(float), HMX_FP16_TILE_SIZE) : 0; - - uint8_t *vtcm_ptr = (uint8_t *) ctx->vtcm_base; - __fp16 *vtcm_weight = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size); - __fp16 *vtcm_activation = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, activation_area_size); - __fp16 *vtcm_output = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, output_area_size); - void *vtcm_scratch0 = vtcm_seq_alloc(&vtcm_ptr, scratch_area_size); - void *vtcm_scratch1 = vtcm_seq_alloc(&vtcm_ptr, scratch_area_size); - __fp16 *vtcm_scales = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, 256); - float *vtcm_f32_act = use_dma_activation ? (float *) vtcm_seq_alloc(&vtcm_ptr, f32_scratch_size) : NULL; - - if ((size_t) (vtcm_ptr - (uint8_t *) ctx->vtcm_base) > vtcm_budget) { - FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to legacy batched loop", __func__); - return hmx_mat_mul_permuted_w16a32_batched_legacy(ctx, params); - } - - hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 - - FARF(MEDIUM, "%s: grouped path m=%d k=%d n=%d group=%d streams=%d mc=%zu nc=%zu vtcm=%zu/%zu", - __func__, params->m, params->k, params->n, group_size, params->ne13, - m_chunk_n_rows, n_chunk_n_cols, - (size_t) (vtcm_ptr - (uint8_t *) ctx->vtcm_base), vtcm_budget); - - TIMER_DEFINE(activation_load); - TIMER_DEFINE(weight_load); - TIMER_DEFINE(hmx_core); - TIMER_DEFINE(output_store); - TIMER_DEFINE(total); - - TIMER_START(total); - - const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16); - const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16); - - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - - for (int b3 = 0; b3 < params->ne13; ++b3) { - for (int b2_base = 0; b2_base < params->ne12; b2_base += group_size) { - const __fp16 *weight_group = hmx_matmul_weight_batch_ptr(params, b2_base, b3); - - for (size_t mr = 0; mr < (size_t) params->m; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin((size_t) params->m - mr, m_chunk_n_rows); - const size_t n_row_tiles = hmx_ceil_div((int) n_rows, HMX_FP16_TILE_N_ROWS); - - // Pre-load activations for all heads in the group (once per m_chunk). - // When the source is strided (permuted Q), use 2D DMA to gather - // contiguous rows into a VTCM scratch buffer first, then HVX - // converts from the contiguous VTCM buffer. This avoids L2 cache - // thrashing from HVX loads at large strides. - TIMER_START(activation_load); - for (int g = 0; g < group_size; ++g) { - const float *activation_chunk = hmx_matmul_activation_batch_ptr(params, b2_base + g, b3) + mr * params->act_stride; - __fp16 *vtcm_act_g = vtcm_activation + (size_t) g * act_head_stride; - if (use_dma_activation) { - const size_t row_bytes = (size_t) params->k * sizeof(float); - const size_t stride_bytes = (size_t) params->act_stride * sizeof(float); - dma_queue_push(ctx->dma[0], - dma_make_ptr(vtcm_f32_act, activation_chunk), - row_bytes, stride_bytes, row_bytes, n_rows); - dma_queue_pop(ctx->dma[0]); - transfer_activation_chunk_threaded(ctx, vtcm_act_g, - vtcm_f32_act, (int) n_rows, - params->k, params->k); - } else { - transfer_activation_chunk_threaded(ctx, vtcm_act_g, - activation_chunk, (int) n_rows, - params->k, params->act_stride); - } - } - TIMER_STOP(activation_load); - - void *buf_curr = vtcm_scratch0; - void *buf_next = vtcm_scratch1; - - { - const size_t n_cols_first = hex_smin((size_t) params->n, n_chunk_n_cols); - dma_queue_push(ctx->dma[0], dma_make_ptr(buf_curr, weight_group), - fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_first); - } - - for (size_t nc = 0; nc < (size_t) params->n; nc += n_chunk_n_cols) { - const size_t n_cols = hex_smin((size_t) params->n - nc, n_chunk_n_cols); - const size_t n_col_tiles = hmx_ceil_div((int) n_cols, HMX_FP16_TILE_N_COLS); - - TIMER_START(weight_load); - { - dma_queue_pop(ctx->dma[0]); - - const size_t nc_next = nc + n_chunk_n_cols; - if (nc_next < (size_t) params->n) { - const size_t n_cols_next = hex_smin((size_t) params->n - nc_next, n_chunk_n_cols); - const __fp16 *next_weight_chunk = weight_group + nc_next * params->weight_stride; - - dma_queue_push(ctx->dma[0], dma_make_ptr(buf_next, next_weight_chunk), - fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_next); - } - - interleave_fp16_weight_chunk_to_tiles(vtcm_weight, (const __fp16 *) buf_curr, n_cols, params->k); - swap_ptr(&buf_curr, &buf_next); - } - TIMER_STOP(weight_load); - - // Reuse the interleaved weight for every q_head in this GQA group - for (int g = 0; g < group_size; ++g) { - TIMER_START(hmx_core); - { - const __fp16 * vtcm_act_g = vtcm_activation + (size_t) g * act_head_stride; - core_dot_chunk_fp16(vtcm_output, vtcm_act_g, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, - params->k / 32); - } - TIMER_STOP(hmx_core); - - TIMER_START(output_store); - { - float *output = hmx_matmul_dst_batch_ptr(params, b2_base + g, b3) + mr * params->dst_stride + nc; - transfer_output_chunk_threaded(ctx, output, vtcm_output, (int) n_rows, (int) n_cols, params->dst_stride); - } - TIMER_STOP(output_store); - } - } - } - } - } - - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); - - TIMER_STOP(total); - -#if defined(ENABLE_PROFILE_TIMERS) - FARF(HIGH, "%s: %lld us, m=%d k=%d n=%d group=%d", __func__, TIMER_US(total), - params->m, params->k, params->n, group_size); - FARF(HIGH, " activation_load: %lld us, weight_load: %lld us, hmx_core: %lld us, output_store: %lld us", - TIMER_US(activation_load), TIMER_US(weight_load), TIMER_US(hmx_core), TIMER_US(output_store)); -#endif - - return 0; -} - -int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, const float *restrict activation, - const __fp16 *restrict permuted_weight, int m, int k, int n, - int act_stride, int weight_stride) { - if (!dst || !activation || !permuted_weight || !m || !n || !k) { return -1; } - if (act_stride < k || weight_stride < k) { return -1; } - if (k % 32 != 0 || n % 32 != 0) { return -1; } - - if (!hex_is_aligned(dst, VLEN) || !hex_is_aligned(activation, VLEN) || !hex_is_aligned(permuted_weight, VLEN)) { - return -1; - } - - // --- Dynamic VTCM layout --- - const size_t vtcm_budget = ctx->vtcm_size; - const size_t vec_dot_size = k * sizeof(__fp16); - - // DMA-based activation gather for strided tensors (see batched path comment). - const bool use_dma_activation = (act_stride > k); - const size_t f32_scratch_per_m = use_dma_activation ? (size_t) k * sizeof(float) : 0; - - size_t m_chunk_n_rows = 0, n_chunk_n_cols = 0, vtcm_used = 0; - // FP16 weight: interleave and activation load have similar per-element cost. - if (hmx_compute_chunks(vtcm_budget, - /*overhead=*/256, - /*per_n=*/3 * vec_dot_size, // W + S0 + S1 - /*per_m=*/vec_dot_size + f32_scratch_per_m, // A + optional F32 scratch - /*per_mn=*/sizeof(__fp16), // O - m, n, - /*m_block_cost=*/(size_t) n, - /*n_block_cost=*/(size_t) m, &m_chunk_n_rows, &n_chunk_n_cols, &vtcm_used) != 0) { - FARF(HIGH, "%s: VTCM too small (m=%d k=%d n=%d budget=%zu)", __func__, m, k, n, vtcm_budget); - return -1; - } - - const size_t weight_area_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HMX_FP16_TILE_SIZE); - const size_t activation_area_size = hex_align_up(m_chunk_n_rows * vec_dot_size, HMX_FP16_TILE_SIZE); - const size_t output_area_size = hex_align_up(m_chunk_n_rows * n_chunk_n_cols * sizeof(__fp16), HMX_FP16_TILE_SIZE); - const size_t scratch_area_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HMX_FP16_TILE_SIZE); - const size_t f32_scratch_size = use_dma_activation - ? hex_align_up(m_chunk_n_rows * (size_t) k * sizeof(float), HMX_FP16_TILE_SIZE) : 0; - - // VTCM layout: weight | activation | output | scratch0 | scratch1 | scales | [f32_scratch] - uint8_t *vtcm_ptr = (uint8_t *) ctx->vtcm_base; - __fp16 *vtcm_weight = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size); - __fp16 *vtcm_activation = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, activation_area_size); - __fp16 *vtcm_output = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, output_area_size); - void *vtcm_scratch0 = vtcm_seq_alloc(&vtcm_ptr, scratch_area_size); - void *vtcm_scratch1 = vtcm_seq_alloc(&vtcm_ptr, scratch_area_size); - __fp16 *vtcm_scales = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, 256); - float *vtcm_f32_act = use_dma_activation ? (float *) vtcm_seq_alloc(&vtcm_ptr, f32_scratch_size) : NULL; - if ((size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base) > vtcm_budget) { - FARF(ERROR, "%s: vtcm overflow: used=%zu limit=%zu", __func__, - (size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base), vtcm_budget); - return -1; - } - - hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 - - FARF(MEDIUM, "%s: m=%d k=%d n=%d mc=%zu nc=%zu vtcm=%zu/%zu", - __func__, m, k, n, m_chunk_n_rows, n_chunk_n_cols, - (size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base), vtcm_budget); - - TIMER_DEFINE(activation_load); - TIMER_DEFINE(weight_load); - TIMER_DEFINE(hmx_core); - TIMER_DEFINE(output_store); - - TIMER_DEFINE(total); - TIMER_START(total); - - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - - for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { - // transfer activation matrix chunk into VTCM - const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); - const size_t n_row_tiles = hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS); - - TIMER_START(activation_load); - { - const float *activation_chunk = activation + mr * act_stride; - if (use_dma_activation) { - const size_t row_bytes = (size_t) k * sizeof(float); - const size_t stride_bytes = (size_t) act_stride * sizeof(float); - dma_queue_push(ctx->dma[0], - dma_make_ptr(vtcm_f32_act, activation_chunk), - row_bytes, stride_bytes, row_bytes, n_rows); - dma_queue_pop(ctx->dma[0]); - transfer_activation_chunk_threaded(ctx, vtcm_activation, - vtcm_f32_act, n_rows, k, k); - } else { - transfer_activation_chunk_threaded(ctx, vtcm_activation, - activation_chunk, n_rows, k, act_stride); - } - } - TIMER_STOP(activation_load); - - const size_t fp16_row_bytes = (size_t) k * sizeof(__fp16); - const size_t weight_row_bytes = (size_t) weight_stride * sizeof(__fp16); - - void *buf_curr = vtcm_scratch0; - void *buf_next = vtcm_scratch1; - - // issue async DMA for the first weight chunk - // NOTE: use 2D DMA (n_cols rows x fp16_row_bytes) to avoid 16-bit roiwidth overflow. - // The source rows can be strided (e.g. KV-cache K after ggml_permute). - { - const size_t n_cols_first = hex_smin(n, n_chunk_n_cols); - - dma_queue_push(ctx->dma[0], dma_make_ptr(buf_curr, permuted_weight), - fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_first); - } - - for (size_t nc = 0; nc < n; nc += n_chunk_n_cols) { - const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); - const size_t n_col_tiles = hmx_ceil_div(n_cols, HMX_FP16_TILE_N_COLS); - - TIMER_START(weight_load); - { - dma_queue_pop(ctx->dma[0]); // wait until current weight chunk is ready - - // issue async DMA for the next weight chunk (double buffering) - const size_t nc_next = nc + n_chunk_n_cols; - if (nc_next < n) { - const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); - const __fp16 *next_weight_chunk = permuted_weight + nc_next * weight_stride; - - dma_queue_push(ctx->dma[0], dma_make_ptr(buf_next, next_weight_chunk), - fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_next); - } - - // interleave row-major fp16 from scratch into tile-major in vtcm_weight - interleave_fp16_weight_chunk_to_tiles(vtcm_weight, (const __fp16 *)buf_curr, n_cols, k); - - swap_ptr(&buf_curr, &buf_next); - } - TIMER_STOP(weight_load); - - TIMER_START(hmx_core); - { - core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, k / 32); - } - TIMER_STOP(hmx_core); - - TIMER_START(output_store); - { - float *output = dst + (mr * n + nc); - transfer_output_chunk_threaded(ctx, output, vtcm_output, n_rows, n_cols, n); - } - TIMER_STOP(output_store); - } - - } - - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); - - TIMER_STOP(total); - -#if defined(ENABLE_PROFILE_TIMERS) - FARF(HIGH, "%s: %lld us, m=%d k=%d n=%d", __func__, TIMER_US(total), m, k, n); - FARF(HIGH, " activation_load: %lld us, weight_load: %lld us, hmx_core: %lld us, output_store: %lld us", - TIMER_US(activation_load), TIMER_US(weight_load), TIMER_US(hmx_core), TIMER_US(output_store)); - { - size_t weight_size = (size_t)k * n * sizeof(__fp16); - float bandwidth = 1e-3f * weight_size / (float)TIMER_US(weight_load); - FARF(HIGH, " weight load bandwidth: %.2f GB/s", bandwidth); - } -#endif - - return 0; -} - -int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict out, const float *restrict x, const uint8_t *restrict w, int m, - int k, int n, int w_type); - -#define FALLBACK_TO_STANDARD 1 - -int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict dst, const float *restrict activation, - const uint8_t *restrict permuted_weight, int m, int k, int n, - int weight_type) { - if (!dst || !activation || !permuted_weight || !m || !n || !k) { return -1; } - if (k % 32 != 0 || n % 32 != 0) { return -1; } - - if (!hex_is_aligned(dst, VLEN) || !hex_is_aligned(activation, VLEN) || !hex_is_aligned(permuted_weight, VLEN)) { - return -1; - } - - // for large m, k (e.g. prefill FFN Down), use out-stationary version - if (m >= 128 && k > n && n > 1024) { - int rc = mat_mul_qk_0_d16a32_out_stationary(ctx, dst, activation, permuted_weight, m, k, n, weight_type); - if (rc != FALLBACK_TO_STANDARD) { - return rc; // 0 success, -1 error - } - FARF(MEDIUM, "hmx_matmul_qk: out-stationary fallback to standard m=%d k=%d n=%d", m, k, n); - // fall through to standard path - } - - size_t row_stride = get_x4x2_row_stride(weight_type, k); - if (row_stride == 0) { - return -1; - } - - FARF(MEDIUM, "hmx_matmul_qk: STANDARD path m=%d k=%d n=%d type=%d", m, k, n, weight_type); - - // --- Dynamic VTCM layout --- - const size_t vtcm_budget = ctx->vtcm_size; - const size_t vec_dot_size = k * sizeof(__fp16); - const bool use_pipeline = (m >= 128) && (k <= n); - - // Select cost parameters based on execution path - size_t per_n_cost, per_mn_cost; - if (use_pipeline) { - per_n_cost = row_stride + 2 * vec_dot_size; // Q + S0 + S1 (dequant bufs) - per_mn_cost = 2 * sizeof(__fp16); // O x 2 (output double buffer) - } else { - per_n_cost = vec_dot_size + 2 * row_stride; // W + S0 + S1 (x4x2 DMA bufs) - per_mn_cost = sizeof(__fp16); // O x 1 - } - - size_t m_chunk_n_rows = 0, n_chunk_n_cols = 0, vtcm_used = 0; - // Quantized weight: dequant ~1.5x more expensive per element than activation load. - if (hmx_compute_chunks(vtcm_budget, /*overhead=*/256, per_n_cost, /*per_m=*/vec_dot_size, per_mn_cost, m, n, - /*m_block_cost=*/(size_t) n * 3, - /*n_block_cost=*/(size_t) m * 2, &m_chunk_n_rows, &n_chunk_n_cols, &vtcm_used) != 0) { - FARF(HIGH, "%s: VTCM too small (m=%d k=%d n=%d pipe=%d budget=%zu)", - __func__, m, k, n, use_pipeline, vtcm_budget); - return -1; - } - - // Compute precise buffer sizes per execution path - const size_t weight_area_size = hex_align_up( - n_chunk_n_cols * (use_pipeline ? row_stride : vec_dot_size), HMX_FP16_TILE_SIZE); - const size_t activation_area_size = hex_align_up(m_chunk_n_rows * vec_dot_size, HMX_FP16_TILE_SIZE); - const size_t output_area_size = hex_align_up( - m_chunk_n_rows * n_chunk_n_cols * sizeof(__fp16), HMX_FP16_TILE_SIZE); - - size_t scratch0_size, scratch1_size, scratch2_size; - if (use_pipeline) { - scratch0_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HMX_FP16_TILE_SIZE); // dequant buf 0 - scratch1_size = scratch0_size; // dequant buf 1 - scratch2_size = output_area_size; // output buf 1 - } else { - scratch0_size = hex_align_up(n_chunk_n_cols * row_stride, HMX_FP16_TILE_SIZE); // x4x2 DMA buf 0 - scratch1_size = scratch0_size; // x4x2 DMA buf 1 - scratch2_size = 0; // unused - } - - uint8_t *vtcm_ptr = (uint8_t *) ctx->vtcm_base; - __fp16 *vtcm_weight = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size); - __fp16 *vtcm_activation = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, activation_area_size); - __fp16 *vtcm_output = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, output_area_size); - void *vtcm_scratch0 = vtcm_seq_alloc(&vtcm_ptr, scratch0_size); - void *vtcm_scratch1 = vtcm_seq_alloc(&vtcm_ptr, scratch1_size); - void *vtcm_scratch2 = scratch2_size ? vtcm_seq_alloc(&vtcm_ptr, scratch2_size) : NULL; - __fp16 *vtcm_scales = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, 256); - if ((size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base) > vtcm_budget) { - FARF(ERROR, "%s: vtcm overflow: used=%zu limit=%zu", __func__, - (size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base), vtcm_budget); - return -1; - } - - hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 - - FARF(MEDIUM, "%s: m=%d k=%d n=%d wtype=%d pipe=%d mc=%zu nc=%zu vtcm=%zu/%zu", - __func__, m, k, n, weight_type, use_pipeline, - m_chunk_n_rows, n_chunk_n_cols, - (size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base), vtcm_budget); - - TIMER_DEFINE(activation_load); - TIMER_DEFINE(weight_load); - TIMER_DEFINE(hmx_core); - TIMER_DEFINE(output_store); - - TIMER_DEFINE(total); - TIMER_START(total); - - FARF(MEDIUM, "hmx_matmul_qk: %s mc=%zu nc=%zu vtcm=%zu/%zu", - use_pipeline ? "PIPELINE" : "SEQUENTIAL", m_chunk_n_rows, n_chunk_n_cols, - (size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base), vtcm_budget); - - if (!use_pipeline) { - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { - // transfer activation matrix chunk into VTCM - const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); - const size_t n_row_tiles = hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS); - - TIMER_START(activation_load); - { - const float *activation_chunk = activation + mr * k; - transfer_activation_chunk_threaded(ctx, vtcm_activation, activation_chunk, n_rows, k, k); - } - TIMER_STOP(activation_load); - - void *buf_curr = vtcm_scratch0; - void *buf_next = vtcm_scratch1; - - { - const size_t n_cols_first = hex_smin(n, n_chunk_n_cols); - dma_queue_push(ctx->dma[0], dma_make_ptr(buf_curr, permuted_weight), row_stride, row_stride, row_stride, n_cols_first); - } - - for (size_t nc = 0; nc < n; nc += n_chunk_n_cols) { - const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); - const size_t n_col_tiles = hmx_ceil_div(n_cols, HMX_FP16_TILE_N_COLS); - - TIMER_START(weight_load); - { - dma_queue_pop(ctx->dma[0]); // wait until current weight chunk become ready - - const size_t nc_next = nc + n_chunk_n_cols; - if (nc_next < n) { - const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); - - const uint8_t *next_weight_chunk = permuted_weight + nc_next * row_stride; - - dma_queue_push(ctx->dma[0], dma_make_ptr(buf_next, next_weight_chunk), row_stride, row_stride, row_stride, n_cols_next); - } - - // Dequant + vscatter writes directly to [K, N] transposed tiles. - // HMX computes C = A x B, where A=[M,K] activation, B=[K,N] weight. - dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight, buf_curr, n_cols, k, row_stride, weight_type); - - swap_ptr(&buf_curr, &buf_next); - } - TIMER_STOP(weight_load); - - TIMER_START(hmx_core); - { - core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, k / 32); - } - TIMER_STOP(hmx_core); - - TIMER_START(output_store); - { - float *output = dst + (mr * n + nc); - transfer_output_chunk_threaded(ctx, output, vtcm_output, n_rows, n_cols, n); - } - TIMER_STOP(output_store); - } - } - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); - } else { - // 4-stage pipeline: DMA load (A), dequantize (B), HMX matmul (C), store (D) - // HMX compute (C) runs on dedicated worker thread, overlapping with HVX stages (B, D). - - // A --> B: vtcm_qweight, 1 buffer - // B --> C: vtcm_weight0/vtcm_weight1, 2 buffers - // C --> D: vtcm_output0/vtcm_output1, 2 buffers - - // Async timeline (C overlaps B+D): - // main+HVX: [A0][Act][B0][A1][sub C0][B1‖C0][A2][wait,sub C1][D0+B2‖C1][wait,sub C2][D1‖C2][wait][D2] - // HMX queue: [████ C0 ████████][████ C1 ████████████][████ C2 ████████] - - int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); - hmx_matmul_job_t job_slots[2]; // persistent double-buffered job descriptors - - for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { - const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); - - void *vtcm_qweight = vtcm_weight; - void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 }; - void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; - - // prologue: A0 - const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); - { - // Use 2D DMA (n_cols rows x row_stride) to avoid 16-bit roiwidth overflow. - const uint8_t *qweight_chunk_A0 = permuted_weight; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_qweight, qweight_chunk_A0), row_stride, row_stride, row_stride, n_cols_A0); - } - - { - const float *activation_chunk = activation + mr * k; - transfer_activation_chunk_threaded(ctx, vtcm_activation, activation_chunk, n_rows, k, k); - } - - // prologue: B0, A1, submit C0 (async), B1 (overlaps C0) - { - // B0: wait for DMA, dequant weight chunk 0 - dma_queue_pop(ctx->dma[0]); - dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight_bufs[0], vtcm_qweight, n_cols_A0, k, row_stride, weight_type); - - // A1: issue DMA for weight chunk 1 - const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); - if (1 < n_chunk_cnt) { - const uint8_t *qweight_chunk_A1 = permuted_weight + n_chunk_n_cols * row_stride; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_qweight, qweight_chunk_A1), row_stride, row_stride, row_stride, n_cols_A1); - } - - // submit C0 (non-blocking — HMX worker executes in parallel) - hmx_matmul_job_init(&job_slots[0], (__fp16 *) vtcm_output_bufs[0], (__fp16 *) vtcm_activation, - (__fp16 *) vtcm_weight_bufs[0], vtcm_scales, - hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), - hmx_ceil_div(n_cols_A0, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS); - hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job_slots[0])); - - // B1: DMA pop + dequant (runs in parallel with C0 on HMX worker) - if (1 < n_chunk_cnt) { - dma_queue_pop(ctx->dma[0]); - dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight_bufs[1], vtcm_qweight, n_cols_A1, k, row_stride, weight_type); - } - } - - // main loop: wait C_i → submit C_{i+1} → D_i + B_{i+2} (parallel with C_{i+1}) - for (int i = 0; i < n_chunk_cnt; ++i) { - const size_t nc = i * n_chunk_n_cols; - const size_t nc_p1 = nc + 1 * n_chunk_n_cols; - const size_t nc_p2 = nc + 2 * n_chunk_n_cols; - - const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); - const size_t n_cols_p1 = hex_smin(n - nc_p1, n_chunk_n_cols); - const size_t n_cols_p2 = hex_smin(n - nc_p2, n_chunk_n_cols); - - // issue A_{i+2}: DMA push (non-blocking) - if (i + 2 < n_chunk_cnt) { - const uint8_t *qweight_chunk_p2 = permuted_weight + nc_p2 * row_stride; - dma_queue_push(ctx->dma[0], dma_make_ptr(vtcm_qweight, qweight_chunk_p2), row_stride, row_stride, row_stride, n_cols_p2); - } - - // wait C_i: block until prologue/previous C completes - hmx_queue_pop(ctx->hmx_queue); - - // submit C_{i+1} (non-blocking, overlaps with D_i + B_{i+2} below) - // job_slots[(i+1)%2] is safe: C_i just completed, freeing slot i%2's - // counterpart — and (i+1)%2 was last used by C_{i-1} which completed - // before C_i was submitted. - if (i + 1 < n_chunk_cnt) { - hmx_matmul_job_init(&job_slots[(i + 1) % 2], (__fp16 *) vtcm_output_bufs[(i + 1) % 2], - (__fp16 *) vtcm_activation, (__fp16 *) vtcm_weight_bufs[(i + 1) % 2], - vtcm_scales, hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), - hmx_ceil_div(n_cols_p1, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS); - hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job_slots[(i + 1) % 2])); - } - - // D_i: store output (multi-thread HVX, parallel with C_{i+1}) - float *output_chunk = dst + (mr * n + nc); - transfer_output_chunk_threaded(ctx, output_chunk, vtcm_output_bufs[i % 2], n_rows, n_cols, n); - - // B_{i+2}: DMA pop + dequant (multi-thread HVX, parallel with C_{i+1}) - if (i + 2 < n_chunk_cnt) { - dma_queue_pop(ctx->dma[0]); - dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight_bufs[(i + 2) % 2], vtcm_qweight, n_cols_p2, k, row_stride, weight_type); - } - } - } - - hmx_queue_suspend(ctx->hmx_queue); - } - - TIMER_STOP(total); - -#if defined(ENABLE_PROFILE_TIMERS) - FARF(HIGH, "%s: %lld us, m=%d k=%d n=%d pipeline=%d", __func__, TIMER_US(total), m, k, n, use_pipeline); - if (!use_pipeline) { - FARF(HIGH, " activation_load: %lld us, weight_load: %lld us, hmx_core: %lld us, output_store: %lld us", - TIMER_US(activation_load), TIMER_US(weight_load), TIMER_US(hmx_core), TIMER_US(output_store)); - size_t weight_size = (size_t)n * row_stride; - float bandwidth = 1e-3f * weight_size / (float)TIMER_US(weight_load); - FARF(HIGH, " weight load bandwidth: %.2f GB/s", bandwidth); - } -#endif - - return 0; -} - -// C += AB -void core_mma_chunk_fp16(__fp16 *restrict c, const __fp16 *restrict a, const __fp16 *restrict b, const __fp16 *restrict col_scales, const __fp16 *restrict eye_tile, - int n_row_tiles, int n_col_tiles, int n_dot_tiles, bool zero_init) { - __builtin_assume(n_row_tiles > 0); - __builtin_assume(n_col_tiles > 0); - __builtin_assume(n_dot_tiles > 0); - - Q6_bias_mxmem2_A((void *)col_scales); - - const size_t dot_tile_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS; - for (size_t i = 0; i < n_row_tiles; ++i) { - const __fp16 *row_base = a + i * dot_tile_stride; - __fp16 *res_base = c + i * n_col_tiles * HMX_FP16_TILE_N_ELMS; - for (size_t j = 0; j < n_col_tiles; ++j) { - Q6_mxclracc_hf(); - - const __fp16 *col_tiles = b + j * dot_tile_stride; - const __fp16 *row_tiles = row_base; - __fp16 *accum_tile = res_base + j * HMX_FP16_TILE_N_ELMS; - if (!zero_init) { - Q6_activation_hf_mxmem_RR((unsigned int)accum_tile, 2047); - Q6_weight_hf_mxmem_RR((unsigned int)eye_tile, 2047); - } - - for (int k = 0; k < n_dot_tiles; ++k) { - Q6_activation_hf_mxmem_RR((unsigned int)row_tiles, 2047); - Q6_weight_hf_mxmem_RR((unsigned int)col_tiles, 2047); - row_tiles += HMX_FP16_TILE_N_ELMS; - col_tiles += HMX_FP16_TILE_N_ELMS; - } - Q6_mxmem_AR_after_hf(accum_tile, 0); - } - } -} - -static void transfer_activation_chunk_fp32_to_fp16(__fp16 *restrict vtcm_dst, const float *restrict src, int n_rows, - int k_block, int k_stride) { - for (int r = 0; r < n_rows; r += 2) { - int r0 = r / HMX_FP16_TILE_N_ROWS; // tile row index - int r1 = r % HMX_FP16_TILE_N_ROWS; // intra-tile row idx - - const bool next_row_valid = (r + 1) < n_rows; - - const HVX_Vector *pv_in0 = (const HVX_Vector *) (src + (r + 0) * k_stride); - const HVX_Vector *pv_in1 = (const HVX_Vector *) (src + (r + 1) * k_stride); - for (int c = 0; c < k_block; c += 32) { - HVX_Vector v0 = *pv_in0++; - HVX_Vector v1 = next_row_valid ? *pv_in1++ : Q6_V_vzero(); - - HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); - - // compute output position - int c0 = c / HMX_FP16_TILE_N_COLS; // tile column index - int tile_idx = r0 * (k_block / HMX_FP16_TILE_N_COLS) + c0; - - HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HMX_FP16_TILE_N_ELMS); - tile[r1 / 2] = v_out; - } - } -} - -typedef struct { - __fp16 *dst; - const float *src; - int n_tasks; - int n_tot_chunks; - int n_chunks_per_task; - int k_block; - int k_stride; -} activation_transfer_task_state_t; - -static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { - activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data; - - for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { - // one chunk: one row - int chunk_idx = task_id * st->n_chunks_per_task; - size_t chunk_size = hex_smin(st->n_tot_chunks - chunk_idx, st->n_chunks_per_task); - - __fp16 *dst = st->dst + chunk_idx * st->k_block; - const float *src = st->src + chunk_idx * st->k_stride; - transfer_activation_chunk_fp32_to_fp16(dst, src, chunk_size, st->k_block, st->k_stride); - } -} - -void transfer_activation_chunk_threaded(struct htp_context *ctx, __fp16 *dst, const float *src, int n_rows, int k_block, int k_stride) { - assert(k_block % HMX_FP16_TILE_N_COLS == 0 && k_stride % HMX_FP16_TILE_N_COLS == 0); - assert(VLEN == 32 * sizeof(float)); - - size_t n_tot_chunks = n_rows; - size_t n_chunks_per_task = 32; // must be multiple of 32 to ensure correct destination address - - activation_transfer_task_state_t state; - state.n_tasks = (n_tot_chunks + n_chunks_per_task - 1) / n_chunks_per_task; - state.n_tot_chunks = n_tot_chunks; - state.n_chunks_per_task = n_chunks_per_task; - state.dst = dst; - state.src = src; - state.k_block = k_block; - state.k_stride = k_stride; - - worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_worker_fn, &state, ctx->n_threads); -} - -int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict out, const float *restrict x, const uint8_t *restrict w, - int m, int k, int n, int weight_type) { - // assume k % 32 == 0 && n % 32 == 0 - const size_t row_stride = get_x4x2_row_stride(weight_type, k); - if (row_stride == 0) { - return -1; - } - - const size_t vtcm_budget = ctx->vtcm_size; - - const size_t K_BLOCK_SIZE = 1024; - - // Fallback: if k doesn't need K-blocking, out-stationary has no advantage - const size_t k_iters_check = (k + K_BLOCK_SIZE - 1) / K_BLOCK_SIZE; - if (k_iters_check <= 1) { - FARF(MEDIUM, "%s: K_BLK=%zu >= k=%d, fallback to standard path", __func__, K_BLOCK_SIZE, k); - return FALLBACK_TO_STANDARD; - } - - // Dynamic M,N search via hmx_compute_chunks - const size_t sub_row_stride_alloc = get_x4x2_row_stride(weight_type, K_BLOCK_SIZE); - const size_t per_m = K_BLOCK_SIZE * sizeof(float) // scratch1: M×K×4 (act DMA staging F32) - + K_BLOCK_SIZE * sizeof(__fp16); // activation: M×K×2 (F16 tiles) - const size_t per_n = sub_row_stride_alloc // scratch0: N×sub_row(K) (packed quant) - + K_BLOCK_SIZE * sizeof(__fp16); // weight: N×K×2 (F16 tiles) - const size_t per_mn = sizeof(__fp16); // output: M×N×2 (out-stationary) - // Alignment margin: hex_align_up can add up to 2047 bytes per buffer; - // scratch1 (mc×6144) is naturally 2048-aligned, remaining 4 buffers need margin - const size_t align_margin = 4 * HMX_FP16_TILE_SIZE; - const size_t overhead = HMX_FP16_TILE_SIZE + 256 + align_margin; // eye_tile + scales + alignment - - size_t M_BLOCK_SIZE, N_BLOCK_SIZE, vtcm_used; - // Cost-based search: minimize ceil(m/mc)*m_block_cost + ceil(n/nc)*n_block_cost. - // From profiling: wt_dequant per element ≈ 1.5× activation load per element. - // m_block_cost = n*3: each extra M-block re-dequants all N×K weight (expensive). - // n_block_cost = m*2: each extra N-block re-loads all M×K activation (cheaper). - const size_t m_block_cost = (size_t) n * 3; - const size_t n_block_cost = (size_t) m * 2; - if (hmx_compute_chunks(vtcm_budget, overhead, per_n, per_m, per_mn, m, n, m_block_cost, n_block_cost, &M_BLOCK_SIZE, - &N_BLOCK_SIZE, &vtcm_used) != 0) { - FARF(HIGH, "%s: VTCM too small (m=%d k=%d n=%d budget=%zu)", __func__, m, k, n, vtcm_budget); - return -1; - } - - // Compute precise buffer sizes from searched M,N and fixed K - const size_t weight_size = hex_align_up(N_BLOCK_SIZE * K_BLOCK_SIZE * sizeof(__fp16), HMX_FP16_TILE_SIZE); - const size_t act_size = hex_align_up(M_BLOCK_SIZE * K_BLOCK_SIZE * sizeof(__fp16), HMX_FP16_TILE_SIZE); - const size_t out_size = hex_align_up(M_BLOCK_SIZE * N_BLOCK_SIZE * sizeof(__fp16), HMX_FP16_TILE_SIZE); - const size_t scratch0_sz = hex_align_up(N_BLOCK_SIZE * sub_row_stride_alloc, HMX_FP16_TILE_SIZE); - const size_t scratch1_sz = hex_align_up(M_BLOCK_SIZE * K_BLOCK_SIZE * sizeof(float), HMX_FP16_TILE_SIZE); - - const size_t total_vtcm = weight_size + act_size + out_size + scratch0_sz + scratch1_sz + HMX_FP16_TILE_SIZE + 256; - if (total_vtcm > vtcm_budget) { - FARF(HIGH, "%s: VTCM overflow after search: need %zu have %zu (M=%zu N=%zu K=%zu)", __func__, total_vtcm, - vtcm_budget, M_BLOCK_SIZE, N_BLOCK_SIZE, K_BLOCK_SIZE); - return -1; - } - - uint8_t *vtcm_ptr = (uint8_t *) ctx->vtcm_base; - __fp16 *vtcm_weight = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_size); - __fp16 *vtcm_activation = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, act_size); - __fp16 *vtcm_output = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, out_size); - uint8_t *vtcm_scratch0 = vtcm_seq_alloc(&vtcm_ptr, scratch0_sz); - uint8_t *vtcm_scratch1 = vtcm_seq_alloc(&vtcm_ptr, scratch1_sz); - __fp16 *vtcm_eye_tile = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, HMX_FP16_TILE_SIZE); - __fp16 *vtcm_scales = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, 256); - assert((size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base) <= vtcm_budget); - - FARF(HIGH, "hmx-mm: m=%d k=%d n=%d wtype=%d block M=%zu N=%zu K=%zu vtcm=%zu/%zu", __func__, m, k, n, weight_type, - M_BLOCK_SIZE, N_BLOCK_SIZE, K_BLOCK_SIZE, (size_t) (vtcm_ptr - (uint8_t *) ctx->vtcm_base), vtcm_budget); - - // initialize eye tile (32x32 identity matrix) - { - HVX_Vector v; - v = Q6_V_vzero(); - v = Q6_Vw_vinsert_VwR(v, 0x3c000000); - v = Q6_V_vror_VR(v, VLEN - 4); - v = Q6_Vw_vinsert_VwR(v, 0x00003c00); - for (int i = 0; i < 16; ++i) { - ((HVX_Vector *) vtcm_eye_tile)[i] = v; - v = Q6_V_vror_VR(v, VLEN - 8); - } - } - hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 - - TIMER_DEFINE(fetch); - TIMER_DEFINE(act_load); - TIMER_DEFINE(wt_dequant); - TIMER_DEFINE(core); - - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - - for (size_t mr = 0; mr < m; mr += M_BLOCK_SIZE) { - size_t m_blk_sz = hex_smin(m - mr, M_BLOCK_SIZE); - for (size_t nc = 0; nc < n; nc += N_BLOCK_SIZE) { - size_t n_blk_sz = hex_smin(n - nc, N_BLOCK_SIZE); - - const int n_row_tiles = hmx_ceil_div(m_blk_sz, HMX_FP16_TILE_N_ROWS); - const int n_col_tiles = hmx_ceil_div(n_blk_sz, HMX_FP16_TILE_N_COLS); - - for (size_t kk = 0; kk < k; kk += K_BLOCK_SIZE) { - const size_t k_blk_sz = hex_smin(k - kk, K_BLOCK_SIZE); - - TIMER_START(fetch); - // fetch activation block into VTCM - { - const float *activation_block = x + mr * k + kk; - - dma_queue_push(ctx->dma[0], - dma_make_ptr(vtcm_scratch1, activation_block), - k_blk_sz * sizeof(float), - k * sizeof(float), - k_blk_sz * sizeof(float), - m_blk_sz); - } - - // fetch weight block into VTCM (x4x2 sub-block: quants + scales) - const size_t sub_row_stride = get_x4x2_row_stride(weight_type, k_blk_sz); - { - qweight_fetch_task_state_t s; - - const int blk_start = kk / QK_Q4_0x4x2; - const int nb_sub = (k_blk_sz + QK_Q4_0x4x2 - 1) / QK_Q4_0x4x2; - const int full_qrow = (weight_type == HTP_TYPE_Q8_0) ? k : (k / 2); - const int scale_blk_size = - (weight_type == HTP_TYPE_MXFP4) ? HMX_X4X2_MXFP4_EBLK_SIZE : HMX_X4X2_DBLK_SIZE; - - s.dst = vtcm_scratch0; - s.src = w + nc * row_stride; - s.n_rows = n_blk_sz; - s.src_stride = row_stride; - s.dst_stride = sub_row_stride; - s.quant_off = - (weight_type == HTP_TYPE_Q8_0) ? (blk_start * QK_Q8_0x4x2) : (blk_start * (QK_Q4_0x4x2 / 2)); - s.quant_width = - (weight_type == HTP_TYPE_Q8_0) ? (nb_sub * QK_Q8_0x4x2) : (nb_sub * (QK_Q4_0x4x2 / 2)); - s.scale_off = full_qrow + blk_start * scale_blk_size; - s.scale_width = nb_sub * scale_blk_size; - - // 2D DMA: quants sub-range - dma_queue_push(ctx->dma[0], dma_make_ptr(s.dst, s.src + s.quant_off), - s.dst_stride, s.src_stride, s.quant_width, s.n_rows); - // 2D DMA: scales sub-range - dma_queue_push(ctx->dma[0], dma_make_ptr(s.dst + s.quant_width, s.src + s.scale_off), - s.dst_stride, s.src_stride, s.scale_width, s.n_rows); - } - TIMER_STOP(fetch); - - TIMER_START(act_load); - // load activation block - { - dma_queue_pop(ctx->dma[0]); // wait for act DNA - transfer_activation_chunk_threaded(ctx, vtcm_activation, (float *) vtcm_scratch1, m_blk_sz, k_blk_sz, k_blk_sz); - } - TIMER_STOP(act_load); - - TIMER_START(wt_dequant); - // dequantize weight block - { - dma_queue_pop(ctx->dma[0]); - dma_queue_pop(ctx->dma[0]); - // vtcm_scratch0 is used to store the qweight chunk - // worker_pool_run_func already returned, so fetch is done - dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight, vtcm_scratch0, - n_blk_sz, k_blk_sz, sub_row_stride, weight_type); - } - TIMER_STOP(wt_dequant); - - // core mma - TIMER_START(core); - { - core_mma_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, vtcm_scales, vtcm_eye_tile, n_row_tiles, - n_col_tiles, k_blk_sz / HMX_FP16_TILE_N_COLS, kk == 0); - } - TIMER_STOP(core); - } - - // store output block - { - float *output_block = out + (mr * n + nc); - transfer_output_chunk_threaded(ctx, output_block, vtcm_output, m_blk_sz, n_blk_sz, n); - } - } - } - - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); - -#if defined(ENABLE_PROFILE_TIMERS) - FARF(HIGH, "fetch: %lld us, act_load: %lld us, wt_dequant: %lld us, core: %lld us", - TIMER_US(fetch), TIMER_US(act_load), TIMER_US(wt_dequant), TIMER_US(core)); -#endif - return 0; -} diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h new file mode 100644 index 0000000..65f90c1 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -0,0 +1,1708 @@ +#include "hmx-utils.h" +#include "hmx-queue.h" + +// MXFP4 dequantization LUT: maps 4-bit index to fp16 mantissa value +// kvalues: 0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6 +static const __fp16 mxfp4_to_fp16_lut[64] __attribute__((aligned(VLEN))) = { + 0, 0, 0.5, 0, 1, 0, 1.5, 0, 2, 0, 3, 0, 4, 0, 6, 0, 0, 0, -0.5, 0, -1, 0, -1.5, 0, -2, 0, -3, 0, -4, 0, -6, 0, +}; + +static const __fp16 iq4_nl_to_fp16_lut[64] __attribute__((aligned(VLEN))) = { + -127, 0, -104, 0, -83, 0, -65, 0, -49, 0, -35, 0, -22, 0, -10, 0, + 1, 0, 13, 0, 25, 0, 38, 0, 53, 0, 69, 0, 89, 0, 113, 0, +}; + +// --- tiled format dequantizers --- + +typedef struct tiled_dequantize_state tiled_dequantize_state_t; +typedef void (*tiled_dequant_task_fn_t)(const tiled_dequantize_state_t * state, uint32_t start_tile, uint32_t end_tile); + +struct tiled_dequantize_state { + struct htp_context * ctx; + struct htp_thread_trace * traces; + __fp16 * dst; + const uint8_t * src; + tiled_dequant_task_fn_t task_fn; + + struct fastdiv_values n_k_tiles_div; + uint32_t n_k_tiles; + uint32_t n_weight_tiles; + uint32_t n_tot_tiles; + uint32_t n_tiles_per_task; + uint32_t tile_size; + uint32_t aligned_tile_size; + uint32_t n_tasks; + uint32_t n_cols; + uint32_t k_block; + size_t row_stride; + uint32_t weight_type; +}; + +static void dequantize_tiled_weight_to_fp16_task_q1_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + const HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + 4 * t * HTP_MM_HMX_TILE_N_ELMS; + + const HVX_Vector v_sc = hvx_vmem(tile_src + 512); + const HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + for (uint32_t q = 0; q < 4; q++) { + const HVX_Vector v_plane = hvx_vmem(tile_src + q * 128); + __fp16 * quarter_dst = dst_ptr + q * HTP_MM_HMX_TILE_N_ELMS; + + #pragma unroll + for (int g = 0; g < 8; g++) { + const HVX_Vector v_q = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one); + const HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); + const HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + + hvx_vmem(quarter_dst + (2 * g + 0) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); + hvx_vmem(quarter_dst + (2 * g + 1) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + } + } + } +} + +// Dequantize a single tile from tiled weight data (already in VTCM) to tile-major FP16. +static void dequantize_tiled_weight_to_fp16_task_q4_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector i8 = Q6_Vb_vsplat_R(8); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = hvx_vmem(tile_src + 512); + HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + // Load all 4 groups in parallel + HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128); + HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128); + HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128); + HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128); + + // Nibble extraction + HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4); + HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4); + HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4); + HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4); + HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4); + HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4); + HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4); + HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4); + + // Offsetting (-8) + v_lo0 = Q6_Vb_vsub_VbVb(v_lo0, i8); + v_hi0 = Q6_Vb_vsub_VbVb(v_hi0, i8); + v_lo1 = Q6_Vb_vsub_VbVb(v_lo1, i8); + v_hi1 = Q6_Vb_vsub_VbVb(v_hi1, i8); + v_lo2 = Q6_Vb_vsub_VbVb(v_lo2, i8); + v_hi2 = Q6_Vb_vsub_VbVb(v_hi2, i8); + v_lo3 = Q6_Vb_vsub_VbVb(v_lo3, i8); + v_hi3 = Q6_Vb_vsub_VbVb(v_hi3, i8); + + // Shuffling + HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1); + HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1); + HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1); + HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1); + + // Unpack to 16-bit + HVX_VectorPair vp_int16_lo0 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf0)); + HVX_VectorPair vp_int16_hi0 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf0)); + HVX_VectorPair vp_int16_lo1 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf1)); + HVX_VectorPair vp_int16_hi1 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf1)); + HVX_VectorPair vp_int16_lo2 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf2)); + HVX_VectorPair vp_int16_hi2 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf2)); + HVX_VectorPair vp_int16_lo3 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf3)); + HVX_VectorPair vp_int16_hi3 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf3)); + + // Convert and scale multiplication + HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo0)), v_scale_duplicated)); + HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo0)), v_scale_duplicated)); + HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi0)), v_scale_duplicated)); + HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi0)), v_scale_duplicated)); + + HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo1)), v_scale_duplicated)); + HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo1)), v_scale_duplicated)); + HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi1)), v_scale_duplicated)); + HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi1)), v_scale_duplicated)); + + HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo2)), v_scale_duplicated)); + HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo2)), v_scale_duplicated)); + HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi2)), v_scale_duplicated)); + HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi2)), v_scale_duplicated)); + + HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo3)), v_scale_duplicated)); + HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo3)), v_scale_duplicated)); + HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi3)), v_scale_duplicated)); + HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi3)), v_scale_duplicated)); + + hvx_vmem(dst_ptr + 0 * 64) = v_grp0_0; + hvx_vmem(dst_ptr + 1 * 64) = v_grp0_1; + hvx_vmem(dst_ptr + 2 * 64) = v_grp0_2; + hvx_vmem(dst_ptr + 3 * 64) = v_grp0_3; + + hvx_vmem(dst_ptr + 4 * 64) = v_grp1_0; + hvx_vmem(dst_ptr + 5 * 64) = v_grp1_1; + hvx_vmem(dst_ptr + 6 * 64) = v_grp1_2; + hvx_vmem(dst_ptr + 7 * 64) = v_grp1_3; + + hvx_vmem(dst_ptr + 8 * 64) = v_grp2_0; + hvx_vmem(dst_ptr + 9 * 64) = v_grp2_1; + hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2; + hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3; + + hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0; + hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1; + hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2; + hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3; + } +} + +static void dequantize_tiled_weight_to_fp16_task_q4_1( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector vscale_offset = hvx_vmem(tile_src + 512); + HVX_VectorPair dm_deal = Q6_W_vdeal_VVR(vscale_offset, vscale_offset, -2); + HVX_Vector vd = Q6_V_lo_W(dm_deal); + HVX_Vector vm = Q6_V_hi_W(dm_deal); + + HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(vd, vd, -2)); + HVX_Vector v_offset_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(vm, vm, -2)); + + // Load all 4 groups in parallel + HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128); + HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128); + HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128); + HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128); + + // Nibble extraction + HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4); + HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4); + HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4); + HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4); + HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4); + HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4); + HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4); + HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4); + + // Shuffling + HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1); + HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1); + HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1); + HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1); + + // Unpack to 16-bit + HVX_VectorPair vp_int16_lo0 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf0)); + HVX_VectorPair vp_int16_hi0 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf0)); + HVX_VectorPair vp_int16_lo1 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf1)); + HVX_VectorPair vp_int16_hi1 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf1)); + HVX_VectorPair vp_int16_lo2 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf2)); + HVX_VectorPair vp_int16_hi2 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf2)); + HVX_VectorPair vp_int16_lo3 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf3)); + HVX_VectorPair vp_int16_hi3 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf3)); + + // Convert, multiply, add offset + HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo0)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo0)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi0)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi0)), v_scale_duplicated), v_offset_duplicated)); + + HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo1)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo1)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi1)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi1)), v_scale_duplicated), v_offset_duplicated)); + + HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo2)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo2)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi2)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi2)), v_scale_duplicated), v_offset_duplicated)); + + HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo3)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo3)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi3)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi3)), v_scale_duplicated), v_offset_duplicated)); + + // Parallel Stores + hvx_vmem(dst_ptr + 0 * 64) = v_grp0_0; + hvx_vmem(dst_ptr + 1 * 64) = v_grp0_1; + hvx_vmem(dst_ptr + 2 * 64) = v_grp0_2; + hvx_vmem(dst_ptr + 3 * 64) = v_grp0_3; + + hvx_vmem(dst_ptr + 4 * 64) = v_grp1_0; + hvx_vmem(dst_ptr + 5 * 64) = v_grp1_1; + hvx_vmem(dst_ptr + 6 * 64) = v_grp1_2; + hvx_vmem(dst_ptr + 7 * 64) = v_grp1_3; + + hvx_vmem(dst_ptr + 8 * 64) = v_grp2_0; + hvx_vmem(dst_ptr + 9 * 64) = v_grp2_1; + hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2; + hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3; + + hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0; + hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1; + hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2; + hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3; + } +} + +static void dequantize_tiled_weight_to_fp16_task_iq4_nl( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector vlut_cvt = hvx_vmem(iq4_nl_to_fp16_lut); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = hvx_vmem(tile_src + 512); + HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + // Load all 4 groups in parallel + HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128); + HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128); + HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128); + HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128); + + // Nibble extraction + HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4); + HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4); + HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4); + HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4); + HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4); + HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4); + HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4); + HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4); + + // Shuffling + HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1); + HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1); + HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1); + HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1); + + // Shuffle for LUT lookup + HVX_Vector v_q_lo0 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf0)); + HVX_Vector v_q_hi0 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf0)); + HVX_Vector v_q_lo1 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf1)); + HVX_Vector v_q_hi1 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf1)); + HVX_Vector v_q_lo2 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf2)); + HVX_Vector v_q_hi2 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf2)); + HVX_Vector v_q_lo3 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf3)); + HVX_Vector v_q_hi3 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf3)); + + // LUT lookup + HVX_VectorPair vp_lo0 = Q6_Wh_vlut16_VbVhR(v_q_lo0, vlut_cvt, 0); + HVX_VectorPair vp_hi0 = Q6_Wh_vlut16_VbVhR(v_q_hi0, vlut_cvt, 0); + HVX_VectorPair vp_lo1 = Q6_Wh_vlut16_VbVhR(v_q_lo1, vlut_cvt, 0); + HVX_VectorPair vp_hi1 = Q6_Wh_vlut16_VbVhR(v_q_hi1, vlut_cvt, 0); + HVX_VectorPair vp_lo2 = Q6_Wh_vlut16_VbVhR(v_q_lo2, vlut_cvt, 0); + HVX_VectorPair vp_hi2 = Q6_Wh_vlut16_VbVhR(v_q_hi2, vlut_cvt, 0); + HVX_VectorPair vp_lo3 = Q6_Wh_vlut16_VbVhR(v_q_lo3, vlut_cvt, 0); + HVX_VectorPair vp_hi3 = Q6_Wh_vlut16_VbVhR(v_q_hi3, vlut_cvt, 0); + + // Convert and scale multiplication + HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo0), v_scale_duplicated)); + HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo0), v_scale_duplicated)); + HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi0), v_scale_duplicated)); + HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi0), v_scale_duplicated)); + + HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo1), v_scale_duplicated)); + HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo1), v_scale_duplicated)); + HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi1), v_scale_duplicated)); + HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi1), v_scale_duplicated)); + + HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo2), v_scale_duplicated)); + HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo2), v_scale_duplicated)); + HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi2), v_scale_duplicated)); + HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi2), v_scale_duplicated)); + + HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo3), v_scale_duplicated)); + HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo3), v_scale_duplicated)); + HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi3), v_scale_duplicated)); + HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi3), v_scale_duplicated)); + + hvx_vmem(dst_ptr + 0 * 64) = v_grp0_0; + hvx_vmem(dst_ptr + 1 * 64) = v_grp0_1; + hvx_vmem(dst_ptr + 2 * 64) = v_grp0_2; + hvx_vmem(dst_ptr + 3 * 64) = v_grp0_3; + + hvx_vmem(dst_ptr + 4 * 64) = v_grp1_0; + hvx_vmem(dst_ptr + 5 * 64) = v_grp1_1; + hvx_vmem(dst_ptr + 6 * 64) = v_grp1_2; + hvx_vmem(dst_ptr + 7 * 64) = v_grp1_3; + + hvx_vmem(dst_ptr + 8 * 64) = v_grp2_0; + hvx_vmem(dst_ptr + 9 * 64) = v_grp2_1; + hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2; + hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3; + + hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0; + hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1; + hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2; + hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3; + } +} + +static void dequantize_tiled_weight_to_fp16_task_mxfp4( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector vlut_cvt = hvx_vmem(mxfp4_to_fp16_lut); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v = hvx_vmem(tile_src + 512); + HVX_Vector vh = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(v)); + vh = Q6_Vh_vsub_VhVh(vh, Q6_Vh_vsplat_R(112)); + vh = Q6_Vh_vmax_VhVh(vh, Q6_V_vzero()); + vh = Q6_Vh_vmin_VhVh(vh, Q6_Vh_vsplat_R(30)); + vh = Q6_Vh_vasl_VhR(vh, 10); + + HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(vh, vh, -2)); + + // Load all 4 groups in parallel + HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128); + HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128); + HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128); + HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128); + + // Nibble extraction + HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4); + HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4); + HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4); + HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4); + HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4); + HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4); + HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4); + HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4); + + // Shuffling + HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1); + HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1); + HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1); + HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1); + + // Shuffle for LUT lookup + HVX_Vector v_q_lo0 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf0)); + HVX_Vector v_q_hi0 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf0)); + HVX_Vector v_q_lo1 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf1)); + HVX_Vector v_q_hi1 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf1)); + HVX_Vector v_q_lo2 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf2)); + HVX_Vector v_q_hi2 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf2)); + HVX_Vector v_q_lo3 = Q6_Vb_vshuff_Vb(Q6_V_lo_W(vp_shuf3)); + HVX_Vector v_q_hi3 = Q6_Vb_vshuff_Vb(Q6_V_hi_W(vp_shuf3)); + + // LUT lookup + HVX_VectorPair vp_lo0 = Q6_Wh_vlut16_VbVhR(v_q_lo0, vlut_cvt, 0); + HVX_VectorPair vp_hi0 = Q6_Wh_vlut16_VbVhR(v_q_hi0, vlut_cvt, 0); + HVX_VectorPair vp_lo1 = Q6_Wh_vlut16_VbVhR(v_q_lo1, vlut_cvt, 0); + HVX_VectorPair vp_hi1 = Q6_Wh_vlut16_VbVhR(v_q_hi1, vlut_cvt, 0); + HVX_VectorPair vp_lo2 = Q6_Wh_vlut16_VbVhR(v_q_lo2, vlut_cvt, 0); + HVX_VectorPair vp_hi2 = Q6_Wh_vlut16_VbVhR(v_q_hi2, vlut_cvt, 0); + HVX_VectorPair vp_lo3 = Q6_Wh_vlut16_VbVhR(v_q_lo3, vlut_cvt, 0); + HVX_VectorPair vp_hi3 = Q6_Wh_vlut16_VbVhR(v_q_hi3, vlut_cvt, 0); + + // Convert and scale multiplication + HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo0), v_scale_duplicated)); + HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo0), v_scale_duplicated)); + HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi0), v_scale_duplicated)); + HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi0), v_scale_duplicated)); + + HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo1), v_scale_duplicated)); + HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo1), v_scale_duplicated)); + HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi1), v_scale_duplicated)); + HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi1), v_scale_duplicated)); + + HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo2), v_scale_duplicated)); + HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo2), v_scale_duplicated)); + HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi2), v_scale_duplicated)); + HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi2), v_scale_duplicated)); + + HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_lo3), v_scale_duplicated)); + HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_lo3), v_scale_duplicated)); + HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_lo_W(vp_hi3), v_scale_duplicated)); + HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_V_hi_W(vp_hi3), v_scale_duplicated)); + + hvx_vmem(dst_ptr + 0 * 64) = v_grp0_0; + hvx_vmem(dst_ptr + 1 * 64) = v_grp0_1; + hvx_vmem(dst_ptr + 2 * 64) = v_grp0_2; + hvx_vmem(dst_ptr + 3 * 64) = v_grp0_3; + + hvx_vmem(dst_ptr + 4 * 64) = v_grp1_0; + hvx_vmem(dst_ptr + 5 * 64) = v_grp1_1; + hvx_vmem(dst_ptr + 6 * 64) = v_grp1_2; + hvx_vmem(dst_ptr + 7 * 64) = v_grp1_3; + + hvx_vmem(dst_ptr + 8 * 64) = v_grp2_0; + hvx_vmem(dst_ptr + 9 * 64) = v_grp2_1; + hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2; + hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3; + + hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0; + hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1; + hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2; + hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3; + } +} + +static void dequantize_tiled_weight_to_fp16_task_q8_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = hvx_vmem(tile_src + 1024); + HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + // Load groups 0-3 in parallel + HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128); + HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128); + HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128); + HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128); + + HVX_VectorPair vp_int16_0 = Q6_Wh_vunpack_Vb(vq0); + HVX_VectorPair vp_int16_1 = Q6_Wh_vunpack_Vb(vq1); + HVX_VectorPair vp_int16_2 = Q6_Wh_vunpack_Vb(vq2); + HVX_VectorPair vp_int16_3 = Q6_Wh_vunpack_Vb(vq3); + + // Load groups 4-7 in parallel + HVX_Vector vq4 = hvx_vmem(tile_src + 4 * 128); + HVX_Vector vq5 = hvx_vmem(tile_src + 5 * 128); + HVX_Vector vq6 = hvx_vmem(tile_src + 6 * 128); + HVX_Vector vq7 = hvx_vmem(tile_src + 7 * 128); + + HVX_VectorPair vp_int16_4 = Q6_Wh_vunpack_Vb(vq4); + HVX_VectorPair vp_int16_5 = Q6_Wh_vunpack_Vb(vq5); + HVX_VectorPair vp_int16_6 = Q6_Wh_vunpack_Vb(vq6); + HVX_VectorPair vp_int16_7 = Q6_Wh_vunpack_Vb(vq7); + + // Convert and scale multiply for groups 0-3 + HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_0)), v_scale_duplicated)); + HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_0)), v_scale_duplicated)); + HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_1)), v_scale_duplicated)); + HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_1)), v_scale_duplicated)); + HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_2)), v_scale_duplicated)); + HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_2)), v_scale_duplicated)); + HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_3)), v_scale_duplicated)); + HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_3)), v_scale_duplicated)); + + // Store groups 0-3 + hvx_vmem(dst_ptr + 0 * 64) = v_grp0_0; + hvx_vmem(dst_ptr + 1 * 64) = v_grp0_1; + hvx_vmem(dst_ptr + 2 * 64) = v_grp1_0; + hvx_vmem(dst_ptr + 3 * 64) = v_grp1_1; + hvx_vmem(dst_ptr + 4 * 64) = v_grp2_0; + hvx_vmem(dst_ptr + 5 * 64) = v_grp2_1; + hvx_vmem(dst_ptr + 6 * 64) = v_grp3_0; + hvx_vmem(dst_ptr + 7 * 64) = v_grp3_1; + + // Convert and scale multiply for groups 4-7 + HVX_Vector v_grp4_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_4)), v_scale_duplicated)); + HVX_Vector v_grp4_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_4)), v_scale_duplicated)); + HVX_Vector v_grp5_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_5)), v_scale_duplicated)); + HVX_Vector v_grp5_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_5)), v_scale_duplicated)); + HVX_Vector v_grp6_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_6)), v_scale_duplicated)); + HVX_Vector v_grp6_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_6)), v_scale_duplicated)); + HVX_Vector v_grp7_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_7)), v_scale_duplicated)); + HVX_Vector v_grp7_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_7)), v_scale_duplicated)); + + // Store groups 4-7 + hvx_vmem(dst_ptr + 8 * 64) = v_grp4_0; + hvx_vmem(dst_ptr + 9 * 64) = v_grp4_1; + hvx_vmem(dst_ptr + 10 * 64) = v_grp5_0; + hvx_vmem(dst_ptr + 11 * 64) = v_grp5_1; + hvx_vmem(dst_ptr + 12 * 64) = v_grp6_0; + hvx_vmem(dst_ptr + 13 * 64) = v_grp6_1; + hvx_vmem(dst_ptr + 14 * 64) = v_grp7_0; + hvx_vmem(dst_ptr + 15 * 64) = v_grp7_1; + } +} + +static void dequantize_tiled_weight_to_fp16_task_q5_k( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector vscale_offset = hvx_vmem(tile_src + 512); + HVX_VectorPair dm_deal = Q6_W_vdeal_VVR(vscale_offset, vscale_offset, -2); + HVX_Vector vd = Q6_V_lo_W(dm_deal); + HVX_Vector vm = Q6_V_hi_W(dm_deal); + + HVX_Vector v_scale_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(vd, vd, -2)); + HVX_Vector v_offset_duplicated = Q6_V_lo_W(Q6_W_vshuff_VVR(vm, vm, -2)); + + // Load all 4 groups in parallel + HVX_Vector vq0 = hvx_vmem(tile_src + 0 * 128); + HVX_Vector vq1 = hvx_vmem(tile_src + 1 * 128); + HVX_Vector vq2 = hvx_vmem(tile_src + 2 * 128); + HVX_Vector vq3 = hvx_vmem(tile_src + 3 * 128); + + // Nibble extraction + HVX_Vector v_lo0 = Q6_V_vand_VV(vq0, mask_h4); + HVX_Vector v_hi0 = Q6_Vub_vlsr_VubR(vq0, 4); + HVX_Vector v_lo1 = Q6_V_vand_VV(vq1, mask_h4); + HVX_Vector v_hi1 = Q6_Vub_vlsr_VubR(vq1, 4); + HVX_Vector v_lo2 = Q6_V_vand_VV(vq2, mask_h4); + HVX_Vector v_hi2 = Q6_Vub_vlsr_VubR(vq2, 4); + HVX_Vector v_lo3 = Q6_V_vand_VV(vq3, mask_h4); + HVX_Vector v_hi3 = Q6_Vub_vlsr_VubR(vq3, 4); + + // Q5_K: OR in the 5th bit from the plane + HVX_Vector v_plane = hvx_vmem(tile_src + 640); + v_lo0 = hvx_q5k_or_hibit(v_lo0, v_plane, 0); + v_hi0 = hvx_q5k_or_hibit(v_hi0, v_plane, 1); + v_lo1 = hvx_q5k_or_hibit(v_lo1, v_plane, 2); + v_hi1 = hvx_q5k_or_hibit(v_hi1, v_plane, 3); + v_lo2 = hvx_q5k_or_hibit(v_lo2, v_plane, 4); + v_hi2 = hvx_q5k_or_hibit(v_hi2, v_plane, 5); + v_lo3 = hvx_q5k_or_hibit(v_lo3, v_plane, 6); + v_hi3 = hvx_q5k_or_hibit(v_hi3, v_plane, 7); + + // Shuffling + HVX_VectorPair vp_shuf0 = Q6_W_vshuff_VVR(v_hi0, v_lo0, -1); + HVX_VectorPair vp_shuf1 = Q6_W_vshuff_VVR(v_hi1, v_lo1, -1); + HVX_VectorPair vp_shuf2 = Q6_W_vshuff_VVR(v_hi2, v_lo2, -1); + HVX_VectorPair vp_shuf3 = Q6_W_vshuff_VVR(v_hi3, v_lo3, -1); + + // Unpack to 16-bit + HVX_VectorPair vp_int16_lo0 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf0)); + HVX_VectorPair vp_int16_hi0 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf0)); + HVX_VectorPair vp_int16_lo1 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf1)); + HVX_VectorPair vp_int16_hi1 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf1)); + HVX_VectorPair vp_int16_lo2 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf2)); + HVX_VectorPair vp_int16_hi2 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf2)); + HVX_VectorPair vp_int16_lo3 = Q6_Wh_vunpack_Vb(Q6_V_lo_W(vp_shuf3)); + HVX_VectorPair vp_int16_hi3 = Q6_Wh_vunpack_Vb(Q6_V_hi_W(vp_shuf3)); + + // Convert, multiply, add offset + HVX_Vector v_grp0_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo0)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp0_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo0)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp0_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi0)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp0_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi0)), v_scale_duplicated), v_offset_duplicated)); + + HVX_Vector v_grp1_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo1)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp1_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo1)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp1_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi1)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp1_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi1)), v_scale_duplicated), v_offset_duplicated)); + + HVX_Vector v_grp2_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo2)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp2_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo2)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp2_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi2)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp2_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi2)), v_scale_duplicated), v_offset_duplicated)); + + HVX_Vector v_grp3_0 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_lo3)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp3_1 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_lo3)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp3_2 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_int16_hi3)), v_scale_duplicated), v_offset_duplicated)); + HVX_Vector v_grp3_3 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_int16_hi3)), v_scale_duplicated), v_offset_duplicated)); + + // Parallel Stores + hvx_vmem(dst_ptr + 0 * 64) = v_grp0_0; + hvx_vmem(dst_ptr + 1 * 64) = v_grp0_1; + hvx_vmem(dst_ptr + 2 * 64) = v_grp0_2; + hvx_vmem(dst_ptr + 3 * 64) = v_grp0_3; + + hvx_vmem(dst_ptr + 4 * 64) = v_grp1_0; + hvx_vmem(dst_ptr + 5 * 64) = v_grp1_1; + hvx_vmem(dst_ptr + 6 * 64) = v_grp1_2; + hvx_vmem(dst_ptr + 7 * 64) = v_grp1_3; + + hvx_vmem(dst_ptr + 8 * 64) = v_grp2_0; + hvx_vmem(dst_ptr + 9 * 64) = v_grp2_1; + hvx_vmem(dst_ptr + 10 * 64) = v_grp2_2; + hvx_vmem(dst_ptr + 11 * 64) = v_grp2_3; + + hvx_vmem(dst_ptr + 12 * 64) = v_grp3_0; + hvx_vmem(dst_ptr + 13 * 64) = v_grp3_1; + hvx_vmem(dst_ptr + 14 * 64) = v_grp3_2; + hvx_vmem(dst_ptr + 15 * 64) = v_grp3_3; + } +} + +// Q6_K stores 6-bit weights and one fp16 scale per 16 k, see HTP_TENSOR_TILE_SIZE_Q6_K. +// A k-group holds 4 k per row, the HMX tile holds 2, so each group is dealt into two tiles. +static void dequantize_tiled_weight_to_fp16_task_q6_k( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector i32 = Q6_Vb_vsplat_R(32); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size); + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = vptr[6]; + HVX_Vector v_sc_k16 = Q6_V_vror_VR(v_sc, 64); + HVX_Vector v_scale_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2)); + + #pragma unroll + for (int g = 0; g < 8; g += 4) { + const HVX_Vector v_scale = (g < 4) ? v_scale_k0 : v_scale_k16; + + HVX_Vector v_q0 = unpack_q6_k_group(vptr, g + 0, mask_0f, mask_03, i32); + HVX_Vector v_q1 = unpack_q6_k_group(vptr, g + 1, mask_0f, mask_03, i32); + HVX_Vector v_q2 = unpack_q6_k_group(vptr, g + 2, mask_0f, mask_03, i32); + HVX_Vector v_q3 = unpack_q6_k_group(vptr, g + 3, mask_0f, mask_03, i32); + + HVX_VectorPair vp16_0 = Q6_Wh_vunpack_Vb(v_q0); + HVX_VectorPair vp16_1 = Q6_Wh_vunpack_Vb(v_q1); + HVX_VectorPair vp16_2 = Q6_Wh_vunpack_Vb(v_q2); + HVX_VectorPair vp16_3 = Q6_Wh_vunpack_Vb(v_q3); + HVX_VectorPair vp_k0 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_0), Q6_V_lo_W(vp16_0), -4); + HVX_VectorPair vp_k1 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_1), Q6_V_lo_W(vp16_1), -4); + HVX_VectorPair vp_k2 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_2), Q6_V_lo_W(vp16_2), -4); + HVX_VectorPair vp_k3 = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16_3), Q6_V_lo_W(vp16_3), -4); + + HVX_Vector v_out00 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k0)), v_scale)); + HVX_Vector v_out01 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k0)), v_scale)); + HVX_Vector v_out10 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k1)), v_scale)); + HVX_Vector v_out11 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k1)), v_scale)); + HVX_Vector v_out20 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k2)), v_scale)); + HVX_Vector v_out21 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k2)), v_scale)); + HVX_Vector v_out30 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k3)), v_scale)); + HVX_Vector v_out31 = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k3)), v_scale)); + + hvx_vmem(dst_ptr + (2 * g + 0) * 64) = v_out00; + hvx_vmem(dst_ptr + (2 * g + 1) * 64) = v_out01; + hvx_vmem(dst_ptr + (2 * g + 2) * 64) = v_out10; + hvx_vmem(dst_ptr + (2 * g + 3) * 64) = v_out11; + hvx_vmem(dst_ptr + (2 * g + 4) * 64) = v_out20; + hvx_vmem(dst_ptr + (2 * g + 5) * 64) = v_out21; + hvx_vmem(dst_ptr + (2 * g + 6) * 64) = v_out30; + hvx_vmem(dst_ptr + (2 * g + 7) * 64) = v_out31; + } + } +} + +// Q3_K stores 3-bit weights and one fp16 scale per 16 k, see HTP_TENSOR_TILE_SIZE_Q3_K. +static void dequantize_tiled_weight_to_fp16_task_q3_k( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size); + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = vptr[3]; + HVX_Vector v_sc_k16 = Q6_V_vror_VR(v_sc, 64); + HVX_Vector v_scale_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2)); + + #pragma unroll + for (int g = 0; g < 8; g++) { + const HVX_Vector v_scale = (g < 4) ? v_scale_k0 : v_scale_k16; + + HVX_Vector v_q = unpack_q3_k_group(vptr, g, mask_03); + HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); + HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + + hvx_vmem(dst_ptr + (2 * g + 0) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); + hvx_vmem(dst_ptr + (2 * g + 1) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + } + } +} + +// Q2_K stores 2-bit weights, the ggml scale bytes and d / -dmin per super-block, see HTP_TENSOR_TILE_SIZE_Q2_K. +static void dequantize_tiled_weight_to_fp16_task_q2_k( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + HVX_Vector v_d = Q6_V_vzero(); + HVX_Vector v_dm = Q6_V_vzero(); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile = state->src + t * state->aligned_tile_size; + const HVX_Vector * vptr = (const HVX_Vector *) tile; + __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + + // rows hold all k tiles and n_k_tiles is a multiple of 8, so t % 8 is the tile within its super-block + if (t == start_tile || (t & 7) == 0) { + q2_k_load_dmin(tile - (t & 7) * state->aligned_tile_size, &v_d, &v_dm); + } + + HVX_Vector v_sc, v_m; + q2_k_unpack_scales(tile, v_d, v_dm, &v_sc, &v_m); + HVX_Vector v_sc_k16 = Q6_V_vror_VR(v_sc, 64); + HVX_Vector v_m_k16 = Q6_V_vror_VR(v_m, 64); + HVX_Vector v_scale_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2)); + HVX_Vector v_offset_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_m, v_m, -2)); + HVX_Vector v_offset_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_m_k16, v_m_k16, -2)); + + #pragma unroll + for (int g = 0; g < 8; g++) { + const HVX_Vector v_scale = (g < 4) ? v_scale_k0 : v_scale_k16; + const HVX_Vector v_offset = (g < 4) ? v_offset_k0 : v_offset_k16; + + HVX_Vector v_q = unpack_q3_k_low2(vptr, g, mask_03); + HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); + HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + + hvx_vmem(dst_ptr + (2 * g + 0) * 64) = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf( + Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale), v_offset)); + hvx_vmem(dst_ptr + (2 * g + 1) * 64) = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vadd_Vqf16Vhf( + Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale), v_offset)); + } + } +} + +// Q2_0 stores codes 0..3 (value code - 1) and one fp16 scale per row, see HTP_MM_WEIGHT_TILE_SIZE_Q2_0. +// One packed 64-k tile expands into two consecutive 32-k fp16 tiles. +static void dequantize_tiled_weight_to_fp16_task_q2_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector i1 = Q6_Vb_vsplat_R(1); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size); + __fp16 * dst_base = state->dst + 2 * t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = vptr[4]; + HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + for (int half = 0; half < 2; half++) { + __fp16 * dst_ptr = dst_base + half * HTP_MM_HMX_TILE_N_ELMS; + + #pragma unroll + for (int g = 0; g < 8; g++) { + HVX_Vector v_q = Q6_Vb_vsub_VbVb(unpack_q3_k_low2(vptr + 2 * half, g, mask_03), i1); + HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); + HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + + hvx_vmem(dst_ptr + (2 * g + 0) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); + hvx_vmem(dst_ptr + (2 * g + 1) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + } + } + } +} + +static __attribute__((noinline)) +void convert_f16_weight_to_fp16_tiles_task( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const uint32_t n_k_tiles = state->n_k_tiles; + const struct fastdiv_values n_k_tiles_div = state->n_k_tiles_div; + + const HVX_Vector v_scat_base = hvx_vmem(hmx_transpose_scatter_offsets); + const HVX_Vector v_scat_step = Q6_V_vsplat_R(4); + const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64); + + unsigned ct = fastdiv((unsigned)start_tile, &n_k_tiles_div); + unsigned kt = fastmodulo((unsigned)start_tile, n_k_tiles, &n_k_tiles_div); + + for (unsigned t = start_tile; t < (unsigned)end_tile; ) { + if (kt >= (unsigned)n_k_tiles) { kt = 0; ct++; } + + __fp16 *tile_base = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + { + uint32_t byte_off = kt * 32 * sizeof(__fp16); + + HVX_Vector v_off = v_scat_base; + for (uint32_t r = 0; r < HTP_MM_HMX_TILE_N_ROWS; r += 2) { + uint32_t row0 = ct * HTP_MM_HMX_TILE_N_COLS + r; + uint32_t row1 = row0 + 1; + + const uint8_t *r0 = state->src + row0 * state->row_stride; + const uint8_t *r1 = state->src + row1 * state->row_stride; + + HVX_Vector v0 = (row0 < state->n_cols) ? hvx_vmemu((const __fp16 *)(r0 + byte_off)) : Q6_V_vzero(); + HVX_Vector v1 = (row1 < state->n_cols) ? hvx_vmemu((const __fp16 *)(r1 + byte_off)) : Q6_V_vzero(); + + Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v0); + v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); + Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v1); + v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); + } + } + ++t; ++kt; + } +} + +static __attribute__((noinline)) +void quantize_f32_weight_to_fp16_tiles_task( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const uint32_t n_k_tiles = state->n_k_tiles; + const struct fastdiv_values n_k_tiles_div = state->n_k_tiles_div; + + const HVX_Vector v_scat_base = hvx_vmem(hmx_transpose_scatter_offsets); + const HVX_Vector v_scat_step = Q6_V_vsplat_R(4); + const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64); + + unsigned ct = fastdiv((unsigned)start_tile, &n_k_tiles_div); + unsigned kt = fastmodulo((unsigned)start_tile, n_k_tiles, &n_k_tiles_div); + + for (unsigned t = start_tile; t < (unsigned)end_tile; ) { + if (kt >= (unsigned)n_k_tiles) { kt = 0; ct++; } + + __fp16 *tile_base = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + { + uint32_t byte_off = kt * 32 * sizeof(float); + + HVX_Vector v_off = v_scat_base; + for (uint32_t r = 0; r < HTP_MM_HMX_TILE_N_ROWS; r += 2) { + uint32_t row0 = ct * HTP_MM_HMX_TILE_N_COLS + r; + uint32_t row1 = row0 + 1; + + const uint8_t *r0 = state->src + row0 * state->row_stride; + const uint8_t *r1 = state->src + row1 * state->row_stride; + + HVX_Vector v0_f32 = (row0 < state->n_cols) ? hvx_vmem((const float *)(r0 + byte_off)) : Q6_V_vzero(); + HVX_Vector v1_f32 = (row1 < state->n_cols) ? hvx_vmem((const float *)(r1 + byte_off)) : Q6_V_vzero(); + + HVX_Vector v_out = hvx_vec_f32_to_f16(v0_f32, v1_f32); + + Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out); + v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); + + HVX_Vector v_out_hi = Q6_V_vror_VR(v_out, 64); + Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out_hi); + v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); + } + } + ++t; ++kt; + } +} + +static __attribute__((noinline)) +void convert_bf16_weight_to_fp16_tiles_task( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const uint32_t n_k_tiles = state->n_k_tiles; + const struct fastdiv_values n_k_tiles_div = state->n_k_tiles_div; + + const HVX_Vector v_scat_base = hvx_vmem(hmx_transpose_scatter_offsets); + const HVX_Vector v_scat_step = Q6_V_vsplat_R(4); + const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64); + + unsigned ct = fastdiv((unsigned)start_tile, &n_k_tiles_div); + unsigned kt = fastmodulo((unsigned)start_tile, n_k_tiles, &n_k_tiles_div); + + for (unsigned t = start_tile; t < (unsigned)end_tile; ) { + if (kt >= (unsigned)n_k_tiles) { kt = 0; ct++; } + + __fp16 *tile_base = state->dst + t * HTP_MM_HMX_TILE_N_ELMS; + { + uint32_t byte_off = kt * 32 * sizeof(uint16_t); + + HVX_Vector v_off = v_scat_base; + for (uint32_t r = 0; r < HTP_MM_HMX_TILE_N_ROWS; r += 2) { + uint32_t row0 = ct * HTP_MM_HMX_TILE_N_COLS + r; + uint32_t row1 = row0 + 1; + + const uint8_t *r0 = state->src + row0 * state->row_stride; + const uint8_t *r1 = state->src + row1 * state->row_stride; + + HVX_Vector v0_raw = (row0 < state->n_cols) ? hvx_vmemu((const void *)(r0 + byte_off)) : Q6_V_vzero(); + HVX_Vector v1_raw = (row1 < state->n_cols) ? hvx_vmemu((const void *)(r1 + byte_off)) : Q6_V_vzero(); + + HVX_Vector v0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(Q6_Ww_vunpack_Vh(v0_raw)), 16); + HVX_Vector v1_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(Q6_Ww_vunpack_Vh(v1_raw)), 16); + + HVX_Vector v_out = hvx_vec_f32_to_f16(v0_f32, v1_f32); + + Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out); + v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); + + HVX_Vector v_out_hi = Q6_V_vror_VR(v_out, 64); + Q6_vscatter_QRMVwV(q_mask64, (size_t)tile_base, HTP_MM_HMX_TILE_SIZE - 1, v_off, v_out_hi); + v_off = Q6_Vw_vadd_VwVw(v_off, v_scat_step); + } + } + ++t; ++kt; + } +} + +// --- End tiled dequantizers --- + +// dot-chunk functions require external HMX lock + +static void core_dot_chunk_fp16_short(__fp16 *restrict output, const __fp16 *restrict activation, + const __fp16 *restrict weight, const __fp16 *restrict scales, + uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles) { + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(n_dot_tiles > 0); + __builtin_assume(n_dot_tiles <= 32); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)scales)); + + const size_t dot_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; + const uint32_t range = 2048u * n_dot_tiles - 1; + + for (uint32_t r = 0; r < n_row_tiles; ++r) { + const __fp16 *row_base = activation + r * dot_stride; + const __fp16 *col_base = weight; + __fp16 *out_tile = output + r * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS; + + for (size_t c = 0; c < n_col_tiles; ++c) { + asm volatile(HMX_CLRACC_F16()); + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_base), "r"(col_base)); + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(out_tile), "r"(0) : "memory"); + col_base += dot_stride; + out_tile += HTP_MM_HMX_TILE_N_ELMS; + } + } +} + +static void core_dot_chunk_fp16(__fp16 *restrict output, const __fp16 *restrict activation, + const __fp16 *restrict weight, const __fp16 *restrict scales, + uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles) { + if (n_dot_tiles <= 32) { + core_dot_chunk_fp16_short(output, activation, weight, scales, n_row_tiles, n_col_tiles, n_dot_tiles); + return; + } + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(n_dot_tiles > 32); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)scales)); + + const size_t dot_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; + + for (uint32_t r = 0; r < n_row_tiles; ++r) { + const __fp16 *row_base = activation + r * dot_stride; + const __fp16 *col_base = weight; + __fp16 *out_tile = output + r * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS; + + for (size_t c = 0; c < n_col_tiles; ++c) { + const __fp16 *row_tiles = row_base; + const __fp16 *col_tiles = col_base; + + asm volatile(HMX_CLRACC_F16()); + + const uint32_t n_loops = n_dot_tiles / 32; + const uint32_t rem = n_dot_tiles % 32; + + for (uint32_t l = 0; l < n_loops; ++l) { + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(65535), "r"(row_tiles), "r"(col_tiles)); + row_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS; + col_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS; + } + + if (rem > 0) { + const uint32_t range = 2048u * rem - 1; + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_tiles), "r"(col_tiles)); + } + + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(out_tile), "r"(0) : "memory"); + + col_base += dot_stride; + out_tile += HTP_MM_HMX_TILE_N_ELMS; + } + } +} + +static void core_mma_chunk_fp16_short(__fp16 *restrict c, const __fp16 *restrict a, const __fp16 *restrict b, + const __fp16 *restrict col_scales, const __fp16 *restrict eye_tile, + uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles, bool zero_init) { + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(n_dot_tiles > 0); + __builtin_assume(n_dot_tiles <= 32); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)col_scales)); + + const size_t dot_tile_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; + const uint32_t range = 2048u * n_dot_tiles - 1; + + for (size_t i = 0; i < n_row_tiles; ++i) { + const __fp16 *row_base = a + i * dot_tile_stride; + __fp16 *res_base = c + i * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS; + const __fp16 *col_base = b; + __fp16 *accum_tile = res_base; + + for (size_t j = 0; j < n_col_tiles; ++j) { + asm volatile(HMX_CLRACC_F16()); + + if (!zero_init) { + asm volatile(HMX_LOAD_MPY_F16("%1", "%2", "%0") : : "r"(2047), "r"(accum_tile), "r"(eye_tile)); + } + + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_base), "r"(col_base)); + + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(accum_tile), "r"(0) : "memory"); + + col_base += dot_tile_stride; + accum_tile += HTP_MM_HMX_TILE_N_ELMS; + } + } +} + +static void core_mma_chunk_fp16(__fp16 *restrict c, const __fp16 *restrict a, const __fp16 *restrict b, + const __fp16 *restrict col_scales, const __fp16 *restrict eye_tile, + uint32_t n_row_tiles, uint32_t n_col_tiles, uint32_t n_dot_tiles, bool zero_init) { + if (n_dot_tiles <= 32) { + core_mma_chunk_fp16_short(c, a, b, col_scales, eye_tile, n_row_tiles, n_col_tiles, n_dot_tiles, zero_init); + return; + } + __builtin_assume(n_row_tiles > 0); + __builtin_assume(n_col_tiles > 0); + __builtin_assume(n_dot_tiles > 32); + + asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)col_scales)); + + const size_t dot_tile_stride = n_dot_tiles * HTP_MM_HMX_TILE_N_ELMS; + + for (size_t i = 0; i < n_row_tiles; ++i) { + const __fp16 *row_base = a + i * dot_tile_stride; + __fp16 *res_base = c + i * n_col_tiles * HTP_MM_HMX_TILE_N_ELMS; + const __fp16 *col_base = b; + __fp16 *accum_tile = res_base; + + for (size_t j = 0; j < n_col_tiles; ++j) { + const __fp16 *col_tiles = col_base; + const __fp16 *row_tiles = row_base; + + asm volatile(HMX_CLRACC_F16()); + + if (!zero_init) { + asm volatile(HMX_LOAD_MPY_F16("%1", "%2", "%0") : : "r"(2047), "r"(accum_tile), "r"(eye_tile)); + } + + const uint32_t n_loops = n_dot_tiles / 32; + const uint32_t rem = n_dot_tiles % 32; + + for (uint32_t l = 0; l < n_loops; ++l) { + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(65535), "r"(row_tiles), "r"(col_tiles)); + row_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS; + col_tiles += 32 * HTP_MM_HMX_TILE_N_ELMS; + } + + if (rem > 0) { + const uint32_t range = 2048u * rem - 1; + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") : : "r"(range), "r"(row_tiles), "r"(col_tiles)); + } + + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") : : "r"(accum_tile), "r"(0) : "memory"); + + col_base += dot_tile_stride; + accum_tile += HTP_MM_HMX_TILE_N_ELMS; + } + } +} + +// output : fp16 -> f32p + +static void transfer_output_chunk_fp16_to_fp32_col_chunk( + float *restrict dst, + const float *restrict src2, + const __fp16 *restrict vtcm_src, + uint32_t start_row, + uint32_t n_rows, + uint32_t c_len, + uint32_t total_n_cols, + uint32_t dst_stride, + uint32_t src2_stride, + uint32_t dst_cols +) { + const size_t tile_row_stride = hmx_ceil_div(total_n_cols, HTP_MM_HMX_TILE_N_COLS) * HTP_MM_HMX_TILE_N_ELMS; + + const HVX_Vector one = hvx_vec_splat_f16(1.0); + + const size_t limit_c = hex_smin(c_len, dst_cols); + const size_t limit_c_aligned = (limit_c & ~31); + + for (size_t r = 0; r < n_rows; r += 2) { + const size_t r_idx0 = start_row + r + 0; + const size_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; + const size_t r1 = (r_idx0 % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile + const __fp16 *row_base = vtcm_src + r0 * tile_row_stride; + float *output_row_base = dst + r * dst_stride; // global memory row base for row r (and r+1) + const float *src2_row_base = src2 ? (src2 + r * src2_stride) : NULL; + + #pragma unroll(4) + for (size_t c = 0; c < limit_c_aligned; c += HTP_MM_HMX_TILE_N_COLS) { + const size_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + const __fp16 *tile = row_base + c0 * HTP_MM_HMX_TILE_N_ELMS; + HVX_Vector v = ((const HVX_Vector *) tile)[r1]; + HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one); + + HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0); + v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0); + } + hvx_vmemu(output_row_base + c + 0) = v_out0; + + if (r + 1 < n_rows) { + HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride); + v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1); + } + hvx_vmemu(output_row_base + c + dst_stride) = v_out1; + } + } + + if (limit_c_aligned < limit_c) { + size_t c = limit_c_aligned; + size_t valid_c = limit_c - c; + const size_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + const __fp16 *tile = row_base + c0 * HTP_MM_HMX_TILE_N_ELMS; + HVX_Vector v = ((const HVX_Vector *) tile)[r1]; + HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one); + + HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0); + v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0); + } + hvx_vec_store_u(output_row_base + c, valid_c * sizeof(float), v_out0); + + if (r + 1 < n_rows) { + HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + if (src2_row_base) { + HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride); + v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1); + } + hvx_vec_store_u(output_row_base + c + dst_stride, valid_c * sizeof(float), v_out1); + } + } + } +} + +static inline void transfer_output_chunk_fp16_to_fp32( + float *restrict dst, + const float *restrict src2, + const __fp16 *restrict vtcm_src, + uint32_t start_row, + uint32_t n_rows, + uint32_t n_cols, + uint32_t dst_stride, + uint32_t src2_stride, + uint32_t dst_cols +) { + transfer_output_chunk_fp16_to_fp32_col_chunk( + dst, src2, vtcm_src, start_row, n_rows, n_cols, n_cols, dst_stride, src2_stride, dst_cols + ); +} + +typedef struct { + const __fp16 *vtcm_src; + float *dst; + const float *src2; + uint32_t n_tasks; + uint32_t n_tot_chunks; + uint32_t n_chunks_per_task; + uint32_t n_cols; + uint32_t dst_stride; // DDR row stride + uint32_t src2_stride; // DDR row stride for residual + uint32_t dst_cols; // Actual output columns + struct htp_thread_trace * traces; +} output_transfer_task_state_t; + +// activations : fp32 -> fp16 + +static void transfer_activation_row_pair_fp32_to_fp16( + __fp16 *restrict vtcm_dst, + const float *restrict row0, + const float *restrict row1, + uint32_t r, + uint32_t k_block, + uint32_t k_valid, + bool row0_valid, + bool row1_valid) { + + uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + uint32_t c = 0; + for (; c + 32 <= k_valid; c += 32) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + if (c < k_block) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); + + uint32_t rem = k_valid - c; + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); + v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } +} + +// F16-input variant of transfer_activation_row_pair_fp32_to_fp16, for F16 activation +// (src1). Same shape as the F16 Q-prep in hmx-fa-kernels.h: one 128-byte load carries 64 f16 +// = two tile columns, and Q6_W_vshuff_VVR interleaves the two rows straight into the HMX tile +// layout, so no F32 round-trip is needed. Rows are only 64-byte aligned when k_block is an odd +// multiple of the tile width, hence the unaligned load type. +static void transfer_activation_row_pair_f16_to_f16(__fp16 * restrict vtcm_dst, + const __fp16 * restrict row0, + const __fp16 * restrict row1, + uint32_t r, + uint32_t k_block, + uint32_t k_valid, + bool row0_valid, + bool row1_valid) { + uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + const uint32_t n_tile_cols = k_block / HTP_MM_HMX_TILE_N_COLS; + __fp16 * restrict tile_row = vtcm_dst + (size_t) r0 * n_tile_cols * HTP_MM_HMX_TILE_N_ELMS; + + const HVX_UVector * pv0 = row0_valid ? (const HVX_UVector *) row0 : NULL; + const HVX_UVector * pv1 = row1_valid ? (const HVX_UVector *) row1 : NULL; + + uint32_t c = 0; + for (; c + 64 <= k_valid; c += 64) { + HVX_Vector v0 = pv0 ? pv0[c / 64] : Q6_V_vzero(); + HVX_Vector v1 = pv1 ? pv1[c / 64] : Q6_V_vzero(); + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + + HVX_Vector * tile0 = (HVX_Vector *) (tile_row + (size_t) c0 * HTP_MM_HMX_TILE_N_ELMS); + HVX_Vector * tile1 = (HVX_Vector *) (tile_row + (size_t) (c0 + 1) * HTP_MM_HMX_TILE_N_ELMS); + + tile0[r1 / 2] = Q6_V_lo_W(vp); + tile1[r1 / 2] = Q6_V_hi_W(vp); + } + // Tail: fewer than 64 valid columns left, plus the k_valid..k_block padding that HMX will + // still multiply, so it has to be written as zeros. + for (; c < k_block; c += 64) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + + if (c < k_valid) { + uint32_t rem = k_valid - c; // 1..63 valid f16 lanes + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem * sizeof(__fp16)); + if (pv0) { + v0 = Q6_V_vmux_QVV(mask, pv0[c / 64], Q6_V_vzero()); + } + if (pv1) { + v1 = Q6_V_vmux_QVV(mask, pv1[c / 64], Q6_V_vzero()); + } + } + + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + + HVX_Vector * tile0 = (HVX_Vector *) (tile_row + (size_t) c0 * HTP_MM_HMX_TILE_N_ELMS); + tile0[r1 / 2] = Q6_V_lo_W(vp); + + if (c0 + 1 < n_tile_cols) { + HVX_Vector * tile1 = (HVX_Vector *) (tile_row + (size_t) (c0 + 1) * HTP_MM_HMX_TILE_N_ELMS); + tile1[r1 / 2] = Q6_V_hi_W(vp); + } + } +} + +static void transfer_activation_row_pair_fp32_to_fp16_col_chunk( + __fp16 *restrict vtcm_dst, + const float *restrict row0, // offset by c_first + const float *restrict row1, // offset by c_first + uint32_t r, + uint32_t k_block, + uint32_t c_first, + uint32_t c_len, + uint32_t k_chunk_valid, + bool row0_valid, + bool row1_valid) { + + uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + uint32_t c = 0; + for (; c + 32 <= k_chunk_valid; c += 32) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + if (c < c_len) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c); + + uint32_t rem = (k_chunk_valid > c) ? (k_chunk_valid - c) : 0; + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); + v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } +} + +static void transfer_activation_chunk_fp32_to_fp16_gathered( + __fp16 *restrict vtcm_dst, + const float *restrict src, + uint32_t start_row, + uint32_t vtcm_start_row, + uint32_t n_rows, + uint32_t k_block, + const struct mmid_row_mapping *matrix_rows, + uint32_t cur_a, + uint32_t mapping_stride, + uint32_t ne11, + const struct fastdiv_values * ne11_div, + size_t nb11, + size_t nb12, + uint32_t cne1, + uint32_t k_valid) { + const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + const uint32_t n_rows_tiled = (n_rows / HTP_MM_HMX_TILE_N_ROWS) * HTP_MM_HMX_TILE_N_ROWS; + + uint32_t r = 0; + + #pragma unroll(2) + for (r = 0; r < n_rows_tiled; r += 2) { + uint32_t r_idx0 = start_row + r + 0; + uint32_t r_idx1 = start_row + r + 1; + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0]; + struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1]; + + uint32_t i11_0 = fastmodulo(mapping0.i1, ne11, ne11_div); + uint32_t i11_1 = fastmodulo(mapping1.i1, ne11, ne11_div); + + const float *row0_ptr = (const float *) ((const uint8_t *) src + i11_0 * nb11 + mapping0.i2 * nb12); + const float *row1_ptr = (const float *) ((const uint8_t *) src + i11_1 * nb11 + mapping1.i2 * nb12); + + uint32_t c = 0; + for (; c + 32 <= k_valid; c += 32) { + HVX_Vector v0 = *(const HVX_Vector *)(row0_ptr + c); + HVX_Vector v1 = *(const HVX_Vector *)(row1_ptr + c); + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + if (c < k_block) { + HVX_Vector v0 = *(const HVX_Vector *)(row0_ptr + c); + HVX_Vector v1 = *(const HVX_Vector *)(row1_ptr + c); + + uint32_t rem = k_valid - c; + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); + v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + } + + for (; r < n_rows_padded; r += 2) { + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + const bool row0_valid = (start_row + r + 0) < cne1; + const bool row1_valid = (start_row + r + 1) < cne1; + + const float *row0_ptr = NULL; + const float *row1_ptr = NULL; + + if (row0_valid) { + struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + (start_row + r + 0)]; + uint32_t i11_0 = fastmodulo(mapping0.i1, ne11, ne11_div); + row0_ptr = (const float *) ((const uint8_t *) src + i11_0 * nb11 + mapping0.i2 * nb12); + } + if (row1_valid) { + struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + (start_row + r + 1)]; + uint32_t i11_1 = fastmodulo(mapping1.i1, ne11, ne11_div); + row1_ptr = (const float *) ((const uint8_t *) src + i11_1 * nb11 + mapping1.i2 * nb12); + } + + uint32_t c = 0; + for (; c + 32 <= k_valid; c += 32) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0_ptr + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1_ptr + c); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + if (c < k_block) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0_ptr + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1_ptr + c); + + uint32_t rem = k_valid - c; + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); + v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + } +} + +static void transfer_activation_chunk_fp32_to_fp16_gathered_flat( + __fp16 *restrict vtcm_dst, + const float *restrict src, + uint32_t start_row, + uint32_t vtcm_start_row, + uint32_t n_rows, + uint32_t k_block, + const struct mmid_row_mapping *matrix_rows, + uint32_t cur_a, + uint32_t mapping_stride, + size_t nb12, + uint32_t cne1, + uint32_t k_valid) { + const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + const uint32_t n_rows_tiled = (n_rows / HTP_MM_HMX_TILE_N_ROWS) * HTP_MM_HMX_TILE_N_ROWS; + + uint32_t r = 0; + + #pragma unroll(2) + for (r = 0; r < n_rows_tiled; r += 2) { + uint32_t r_idx0 = start_row + r + 0; + uint32_t r_idx1 = start_row + r + 1; + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0]; + struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1]; + + const float *row0_ptr = (const float *) ((const uint8_t *) src + mapping0.i2 * nb12); + const float *row1_ptr = (const float *) ((const uint8_t *) src + mapping1.i2 * nb12); + + uint32_t c = 0; + for (; c + 32 <= k_valid; c += 32) { + HVX_Vector v0 = *(const HVX_Vector *)(row0_ptr + c); + HVX_Vector v1 = *(const HVX_Vector *)(row1_ptr + c); + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + if (c < k_block) { + HVX_Vector v0 = *(const HVX_Vector *)(row0_ptr + c); + HVX_Vector v1 = *(const HVX_Vector *)(row1_ptr + c); + + uint32_t rem = k_valid - c; + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); + v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + } + + for (; r < n_rows_padded; r += 2) { + uint32_t lr = vtcm_start_row + r; // vtcm-local row + uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index + uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx + + const bool row0_valid = (start_row + r + 0) < cne1; + const bool row1_valid = (start_row + r + 1) < cne1; + + const float *row0_ptr = NULL; + const float *row1_ptr = NULL; + + if (row0_valid) { + struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + (start_row + r + 0)]; + row0_ptr = (const float *) ((const uint8_t *) src + mapping0.i2 * nb12); + } + if (row1_valid) { + struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + (start_row + r + 1)]; + row1_ptr = (const float *) ((const uint8_t *) src + mapping1.i2 * nb12); + } + + uint32_t c = 0; + for (; c + 32 <= k_valid; c += 32) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0_ptr + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1_ptr + c); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + if (c < k_block) { + HVX_Vector v0 = Q6_V_vzero(); + HVX_Vector v1 = Q6_V_vzero(); + if (row0_valid) v0 = *(const HVX_Vector *)(row0_ptr + c); + if (row1_valid) v1 = *(const HVX_Vector *)(row1_ptr + c); + + uint32_t rem = k_valid - c; + HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0); + v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero()); + v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero()); + + HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1); + + uint32_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0; + + HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS); + tile[r1 / 2] = v_out; + } + } +} + +static void transfer_output_chunk_fp16_to_fp32_scattered( + float *restrict dst, + const __fp16 *restrict vtcm_src, + uint32_t start_row, + uint32_t vtcm_start_row, + uint32_t n_rows, + uint32_t n_cols, + const struct mmid_row_mapping *matrix_rows, + uint32_t cur_a, + uint32_t mapping_stride, + size_t dst_nb1, + size_t dst_nb2, + uint32_t cne1) { + assert(n_cols % HTP_MM_HMX_TILE_N_COLS == 0); + const size_t tile_row_stride = (n_cols / HTP_MM_HMX_TILE_N_COLS) * HTP_MM_HMX_TILE_N_ELMS; + + const HVX_Vector one = hvx_vec_splat_f16(1.0); + + for (size_t r = 0; r < n_rows; r += 2) { + uint32_t r_idx0 = start_row + r + 0; + uint32_t r_idx1 = start_row + r + 1; + uint32_t lr = vtcm_start_row + r; // vtcm-local row + const size_t r0 = (lr / HTP_MM_HMX_TILE_N_ROWS); + const size_t r1 = (lr % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile + const __fp16 *row_base = vtcm_src + r0 * tile_row_stride; + + if (r_idx0 >= cne1) break; + + struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0]; + float *output_row0 = (float *) ((uint8_t *) dst + mapping0.i1 * dst_nb1 + mapping0.i2 * dst_nb2); + + float *output_row1 = NULL; + if (r_idx1 < cne1) { + struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1]; + output_row1 = (float *) ((uint8_t *) dst + mapping1.i1 * dst_nb1 + mapping1.i2 * dst_nb2); + } + + #pragma unroll(4) + for (size_t c = 0; c < (size_t)n_cols; c += HTP_MM_HMX_TILE_N_COLS) { + const size_t c0 = c / HTP_MM_HMX_TILE_N_COLS; + const __fp16 *tile = row_base + c0 * HTP_MM_HMX_TILE_N_ELMS; + HVX_Vector v = ((const HVX_Vector *) tile)[r1]; + HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one); + + hvx_vmemu(output_row0 + c) = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); + if (output_row1) { + hvx_vmemu(output_row1 + c) = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp)); + } + } + } +} diff --git a/ggml/src/ggml-hexagon/htp/hmx-ops.h b/ggml/src/ggml-hexagon/htp/hmx-ops.h deleted file mode 100644 index fb95d36..0000000 --- a/ggml/src/ggml-hexagon/htp/hmx-ops.h +++ /dev/null @@ -1,68 +0,0 @@ -// HMX operation entry-point declarations. -// Ported from htp-ops-lib/include/dsp/ops.h (renamed, benchmark kernels removed). (https://github.com/haozixu/htp-ops-lib) - -#ifndef HMX_OPS_H -#define HMX_OPS_H - -#include -#include - -#include "htp-ops.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - float *dst; - const float *activation; - const __fp16 *permuted_weight; - int m; - int k; - int n; - int act_stride; - int weight_stride; - int dst_stride; - int ne02; - int ne03; - int ne12; - int ne13; - size_t src0_nb2; - size_t src0_nb3; - size_t src1_nb2; - size_t src1_nb3; - size_t dst_nb2; - size_t dst_nb3; -} hmx_matmul_w16a32_batched_params_t; - -// HMX matrix multiplication — tile-permuted FP16 weights, FP32 activation/output -// act_stride: activation row stride in elements (= k for contiguous, or -// nb[1]/sizeof(float) for permuted tensors like attention Q). -// weight_stride: weight row stride in elements (= k for compact weights, or -// nb[1]/sizeof(__fp16) for permuted KV-cache views used by QK). -int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, - float *restrict dst, - const float *activation, - const __fp16 *permuted_weight, - int m, int k, int n, - int act_stride, - int weight_stride); - -// Batched F16 wrapper over hmx_mat_mul_permuted_w16a32. -// Batch semantics match ggml_mul_mat(): src0 broadcasts to src1 in dims 2/3. -int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, - const hmx_matmul_w16a32_batched_params_t *params); - -// HMX matrix multiplication — tile-permuted quantised weights (Q4_0/Q8_0/IQ4_NL) -int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, - float *restrict dst, - const float *activation, - const uint8_t *permuted_weight, - int m, int k, int n, - int weight_type); - -#ifdef __cplusplus -} -#endif - -#endif // HMX_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/hmx-profile.h b/ggml/src/ggml-hexagon/htp/hmx-profile.h deleted file mode 100644 index 01eece7..0000000 --- a/ggml/src/ggml-hexagon/htp/hmx-profile.h +++ /dev/null @@ -1,34 +0,0 @@ -// Conditional fine-grained profiling macros for HMX operations. -// -// Define ENABLE_PROFILE_TIMERS (via compiler flag or before including this -// header) to instrument sub-operation latencies with HAP qtimer. When the -// macro is not defined the TIMER_* helpers expand to nothing so there is zero -// overhead. -// -// Usage: -// TIMER_DEFINE(my_phase); // declare accumulator variable -// TIMER_START(my_phase); // snapshot start time -// ... work ... -// TIMER_STOP(my_phase); // accumulate elapsed ticks -// FARF(ALWAYS, "my_phase: %lld us", TIMER_US(my_phase)); - -#ifndef HMX_PROFILE_H -#define HMX_PROFILE_H - -#include - -// #define ENABLE_PROFILE_TIMERS - -#if defined(ENABLE_PROFILE_TIMERS) -# define TIMER_DEFINE(name) int64_t name##_ticks = 0 -# define TIMER_START(name) int64_t name##_t0 = HAP_perf_get_qtimer_count() -# define TIMER_STOP(name) name##_ticks += HAP_perf_get_qtimer_count() - name##_t0 -# define TIMER_US(name) HAP_perf_qtimer_count_to_us(name##_ticks) -#else -# define TIMER_DEFINE(name) -# define TIMER_START(name) -# define TIMER_STOP(name) -# define TIMER_US(name) 0LL -#endif - -#endif // HMX_PROFILE_H diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.c b/ggml/src/ggml-hexagon/htp/hmx-queue.c index 5b1d83a..c369d3d 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.c +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.c @@ -6,6 +6,7 @@ #include #include +#include #include @@ -13,7 +14,7 @@ #define QURT_LOWEST_PRIO (254) -static inline void hmx_lock(struct hmx_queue *q) +static inline void hmx_lock(hmx_queue_t q) { if (!q->hmx_locked) { HAP_compute_res_hmx_lock(q->hap_rctx); @@ -21,7 +22,7 @@ static inline void hmx_lock(struct hmx_queue *q) } } -static inline void hmx_unlock(struct hmx_queue *q) +static inline void hmx_unlock(hmx_queue_t q) { if (q->hmx_locked) { HAP_compute_res_hmx_unlock(q->hap_rctx); @@ -29,7 +30,7 @@ static inline void hmx_unlock(struct hmx_queue *q) } } -static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) { +static inline void hmx_queue_process(hmx_queue_t q, bool* killed) { unsigned int ir = atomic_load(&q->idx_read); while (ir != atomic_load(&q->idx_write)) { @@ -37,14 +38,17 @@ static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) { if (!d->done) { FARF(HIGH, "hmx-queue-process: ir %u func %p data %p", ir, d->func, d->data); - enum hmx_queue_signal sig = (enum hmx_queue_signal) (unsigned int) d->func; + uintptr_t sig = (uintptr_t) d->func; switch (sig) { case HMX_QUEUE_NOOP: /* noop */; break; case HMX_QUEUE_KILL: *killed = true; break; case HMX_QUEUE_SUSPEND: hmx_unlock(q); break; + case HMX_QUEUE_WAKEUP: hmx_lock(q); break; default: hmx_lock(q); + htp_trace_event_start(q->trace, HTP_TRACE_EVT_HMX_COMP, ir); d->func(d->data); + htp_trace_event_stop(q->trace, HTP_TRACE_EVT_HMX_COMP, ir); break; } @@ -57,7 +61,7 @@ static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) { } static void hmx_queue_thread(void * arg) { - struct hmx_queue * q = (struct hmx_queue *) arg; + hmx_queue_t q = (hmx_queue_t) arg; FARF(HIGH, "hmx-queue-thread: started"); @@ -68,9 +72,14 @@ static void hmx_queue_thread(void * arg) { while (!killed) { unsigned int seqn = atomic_load(&q->seqn); if (seqn == prev_seqn) { + // drop HVX context while spinning + if (poll_cnt > 1 && poll_cnt == HMX_QUEUE_POLL_COUNT) { + qurt_hvx_unlock(); + } if (--poll_cnt) { hex_pause(); continue; } FARF(HIGH, "hmx-queue-thread: sleeping"); qurt_futex_wait(&q->seqn, prev_seqn); + poll_cnt = HMX_QUEUE_POLL_COUNT; continue; } prev_seqn = seqn; @@ -84,34 +93,41 @@ static void hmx_queue_thread(void * arg) { FARF(HIGH, "hmx-queue-thread: stopped"); } -struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) { +size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) { capacity = hex_ceil_pow2(capacity); + size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE); + size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE); + size_t size_stack = stack_size; + return size_q + size_desc + size_stack; +} + +size_t hmx_queue_alignof(void) { + return HEX_L2_LINE_SIZE; +} + +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) { + capacity = hex_ceil_pow2(capacity); + size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE); + size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE); + + uint8_t * block = (uint8_t *) ptr; + + hmx_queue_t q = (hmx_queue_t) block; block += size_q; + memset(q, 0, sizeof(struct hmx_queue_s)); - struct hmx_queue * q = (struct hmx_queue *) memalign(32, sizeof(struct hmx_queue)); - if (q == NULL) { - FARF(ERROR, "%s: failed to allocate DMA queue\n", __FUNCTION__); - return NULL; - } - memset(q, 0, sizeof(struct hmx_queue)); q->capacity = capacity; q->idx_mask = capacity - 1; q->hap_rctx = hap_rctx; + q->external_mem = true; - q->desc = (struct hmx_queue_desc *) memalign(64, capacity * sizeof(struct hmx_queue_desc)); - if (!q->desc) { - FARF(ERROR, "hmx-queue: failed to allocate HMX queue descriptors\n"); - return NULL; - } + q->desc = (struct hmx_queue_desc *) block; block += size_desc; memset(q->desc, 0, capacity * sizeof(struct hmx_queue_desc)); - const size_t stack_size = HMX_QUEUE_THREAD_STACK_SIZE; - q->stack = (unsigned char *) memalign(64, stack_size); - if (!q->stack) { - FARF(ERROR, "hmx-queue: thread stack allocation failed (%zu bytes)", stack_size); - return NULL; - } + q->stack = block; memset(q->stack, 0, stack_size); + q->trace = trace; + // Match caller thread priority (same pattern as worker-pool.c). int prio = qurt_thread_get_priority(qurt_thread_get_id()); if (prio < 1) { @@ -139,7 +155,7 @@ struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) { return q; } -void hmx_queue_delete(struct hmx_queue * q) { +void hmx_queue_free(hmx_queue_t q) { if (!q) { return; } @@ -151,8 +167,4 @@ void hmx_queue_delete(struct hmx_queue * q) { int status; qurt_thread_join(q->thread, &status); - - free(q->desc); - free(q->stack); - free(q); } diff --git a/ggml/src/ggml-hexagon/htp/hmx-queue.h b/ggml/src/ggml-hexagon/htp/hmx-queue.h index 0d48c28..c2b1859 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-queue.h +++ b/ggml/src/ggml-hexagon/htp/hmx-queue.h @@ -11,19 +11,24 @@ #include #include "hex-utils.h" +#include "hex-profile.h" #ifdef __cplusplus extern "C" { #endif -#define HMX_QUEUE_THREAD_STACK_SIZE (16 * 1024) -#define HMX_QUEUE_POLL_COUNT 2000 +#if __HVX_ARCH__ > 79 +#define HMX_QUEUE_POLL_COUNT 2000 +#else +#define HMX_QUEUE_POLL_COUNT 1 +#endif typedef void (*hmx_queue_func)(void *); // Dummy funcs used as signals enum hmx_queue_signal { HMX_QUEUE_NOOP = 0, // aka NULL + HMX_QUEUE_WAKEUP, HMX_QUEUE_SUSPEND, HMX_QUEUE_KILL }; @@ -34,7 +39,7 @@ struct hmx_queue_desc { atomic_uint done; }; -struct hmx_queue { +struct hmx_queue_s { struct hmx_queue_desc * desc; atomic_uint idx_write; // updated by producer (push) atomic_uint idx_read; // updated by consumer (process) @@ -47,19 +52,25 @@ struct hmx_queue { void * stack; uint32_t hap_rctx; bool hmx_locked; + struct htp_thread_trace * trace; + bool external_mem; // memory owned externally }; -struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx); -void hmx_queue_delete(struct hmx_queue * q); +typedef struct hmx_queue_s * hmx_queue_t; + +size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size); +size_t hmx_queue_alignof(void); +hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace); +void hmx_queue_free(hmx_queue_t q); static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) { struct hmx_queue_desc d = { func, data }; return d; } -static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d) { +static inline bool hmx_queue_push(hmx_queue_t q, struct hmx_queue_desc d) { unsigned int ir = atomic_load(&q->idx_read); - unsigned int iw = q->idx_write; + unsigned int iw = atomic_load(&q->idx_write); if (((iw + 1) & q->idx_mask) == ir) { FARF(HIGH, "hmx-queue-push: queue is full\n"); @@ -79,25 +90,25 @@ static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d) return true; } -static inline bool hmx_queue_signal(struct hmx_queue *q, enum hmx_queue_signal sig) { +static inline bool hmx_queue_signal(hmx_queue_t q, enum hmx_queue_signal sig) { return hmx_queue_push(q, hmx_queue_make_desc((hmx_queue_func) sig, NULL)); } -static inline bool hmx_queue_empty(struct hmx_queue * q) { - return q->idx_pop == q->idx_write; +static inline bool hmx_queue_empty(hmx_queue_t q) { + return q->idx_pop == atomic_load(&q->idx_write); } -static inline uint32_t hmx_queue_depth(struct hmx_queue * q) { - return (q->idx_read - q->idx_read) & q->idx_mask; +static inline uint32_t hmx_queue_depth(hmx_queue_t q) { + return (atomic_load(&q->idx_write) - atomic_load(&q->idx_read)) & q->idx_mask; } -static inline uint32_t hmx_queue_capacity(struct hmx_queue * q) { +static inline uint32_t hmx_queue_capacity(hmx_queue_t q) { return q->capacity; } -static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) { +static inline struct hmx_queue_desc hmx_queue_pop_one(hmx_queue_t q) { unsigned int ip = q->idx_pop; - unsigned int iw = q->idx_write; + unsigned int iw = atomic_load(&q->idx_write); struct hmx_queue_desc rd = { NULL, NULL }; if (ip == iw) { @@ -118,13 +129,28 @@ static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) { return rd; } -static inline void hmx_queue_flush(struct hmx_queue * q) { - while (hmx_queue_pop(q).func != NULL) ; +static inline struct hmx_queue_desc hmx_queue_pop(hmx_queue_t q) { + while (1) { + struct hmx_queue_desc d = hmx_queue_pop_one(q); + + uint32_t sig = (uint32_t) d.func; + if (sig && sig <= HMX_QUEUE_KILL) + continue; + + return d; + } +} + +static inline void hmx_queue_flush(hmx_queue_t q) { + while (hmx_queue_pop_one(q).func != NULL) ; +} + +static inline void hmx_queue_wakeup(hmx_queue_t q) { + hmx_queue_signal(q, HMX_QUEUE_WAKEUP); } -static inline void hmx_queue_suspend(struct hmx_queue *q) { +static inline void hmx_queue_suspend(hmx_queue_t q) { hmx_queue_signal(q, HMX_QUEUE_SUSPEND); - hmx_queue_flush(q); } #ifdef __cplusplus diff --git a/ggml/src/ggml-hexagon/htp/hmx-utils.h b/ggml/src/ggml-hexagon/htp/hmx-utils.h index af04619..1952aaa 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-utils.h +++ b/ggml/src/ggml-hexagon/htp/hmx-utils.h @@ -4,6 +4,9 @@ #ifndef HMX_UTILS_H #define HMX_UTILS_H +#include "hvx-base.h" + +#include #include #include @@ -12,21 +15,216 @@ #define HMX_FP16_TILE_N_ELMS 1024 #define HMX_FP16_TILE_SIZE 2048 -#define HMX_INLINE_ALWAYS inline __attribute__((unused, always_inline)) - // Initialise aligned 256-byte area with scale vector + zero padding. -static HMX_INLINE_ALWAYS void hmx_init_column_scales(void *out_scales, HVX_Vector v_scale) { - HVX_Vector *pv = (HVX_Vector *)out_scales; - *pv++ = v_scale; - *pv = Q6_V_vzero(); +static inline void hmx_init_column_scales(void *out_scales, HVX_Vector v_scale) { + volatile HVX_Vector *pv = (HVX_Vector *) out_scales; + pv[0] = v_scale; + pv[1] = Q6_V_vzero(); +} + +// --- Shared scatter offsets and interleave helper --- + +// vscatter offsets for fused dequant+transpose: write K-values directly to [K][N] tile. +// word[i] = i*128 maps K-row-pair i to byte offset i*128. +// Column offset (n*4) is added at runtime. Entries 0..15 cover one tile (region 2047); +// entries 16..31 cover the next adjacent tile (region 4095) - pick region size at the +// call site to scatter into one tile (masked) or two contiguous tiles (unmasked). +static const int32_t hmx_transpose_scatter_offsets[32] __attribute__((aligned(VLEN))) = { + 0 * 128, 1 * 128, 2 * 128, 3 * 128, 4 * 128, 5 * 128, 6 * 128, 7 * 128, 8 * 128, 9 * 128, 10 * 128, + 11 * 128, 12 * 128, 13 * 128, 14 * 128, 15 * 128, 16 * 128, 17 * 128, 18 * 128, 19 * 128, 20 * 128, 21 * 128, + 22 * 128, 23 * 128, 24 * 128, 25 * 128, 26 * 128, 27 * 128, 28 * 128, 29 * 128, 30 * 128, 31 * 128, +}; + +// Scatter row-major FP16 data (in VTCM scratch) into transposed [K][N] tiles. +// vtcm_src: [n_cols][src_stride] row-major fp16 (only first k elements per row are used) +// vtcm_dst: [n_col_tiles][n_k_tiles][HMX_FP16_TILE_N_ELMS] tile-major interleaved fp16 +// Processes rows [start_row, end_row) for multi-thread slicing. +// Full range: start_row=0, end_row=n_cols. +static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst, + const __fp16 * restrict vtcm_src, + uint32_t n_cols, + uint32_t k, + size_t src_stride, + uint32_t start_row, + uint32_t end_row) { + assert(k % HMX_FP16_TILE_N_COLS == 0); + + const uint32_t n_k_tiles = k / HMX_FP16_TILE_N_COLS; + const HVX_Vector v_scat_base = hvx_vmem(hmx_transpose_scatter_offsets); + const HVX_Vector v_scat_step = Q6_V_vsplat_R(4); + const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64); + // Each hvx_vmemu load brings 64 fp16 = 128 bytes covering 2 adjacent K-tiles. + // When n_k_tiles is even, scatter into 2 K-tiles per call (region 4095, no mask) + // using the upper half of hmx_transpose_scatter_offsets. Tail one K-tile (when + // n_k_tiles is odd) falls back to single-tile masked scatter. + const bool pair_scatter = (n_k_tiles & 1) == 0; + const size_t pair_region = (size_t) (2 * HMX_FP16_TILE_SIZE - 1); + const size_t single_region = (size_t) (HMX_FP16_TILE_SIZE - 1); + __builtin_assume(k > 0); + __builtin_assume(end_row > start_row); + + if (pair_scatter) { + // Step c by 64 fp16 (two K-tiles per scatter), advance dst by 2 tiles per iter. + const uint32_t c_step = 2 * HMX_FP16_TILE_N_COLS; + const size_t c_byte_step = (size_t) c_step * sizeof(__fp16); + const size_t dst_step = 2 * (size_t) HMX_FP16_TILE_N_ELMS; + const uint32_t n_c_iters = k / c_step; + + for (uint32_t r = start_row; r < end_row; r += 2) { + const uint32_t ct = r / HMX_FP16_TILE_N_ROWS; + const uint32_t local_r = r % HMX_FP16_TILE_N_ROWS; + const bool row0_valid = r < n_cols; + const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_cols; + const HVX_Vector v_off0 = Q6_Vw_vadd_VwVw(v_scat_base, Q6_V_vsplat_R(local_r * 4)); + const HVX_Vector v_off1 = Q6_Vw_vadd_VwVw(v_off0, v_scat_step); + + __fp16 * tile_base = vtcm_dst + (size_t) ct * n_k_tiles * HMX_FP16_TILE_N_ELMS; + const uint8_t * p0 = row0_valid ? (const uint8_t *) (vtcm_src + r * src_stride) : NULL; + const uint8_t * p1 = next_row_valid ? (const uint8_t *) (vtcm_src + (r + 1) * src_stride) : NULL; + + assert(!p0 || hex_is_aligned(p0, 128)); + assert(!p1 || hex_is_aligned(p1, 128)); + assert(c_byte_step % 128 == 0); + + if (p0 && p1) { + for (uint32_t i = 0; i < n_c_iters; ++i) { + HVX_Vector v0 = hvx_vmem(p0); p0 += c_byte_step; + HVX_Vector v1 = hvx_vmem(p1); p1 += c_byte_step; + Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off0, v0); + Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off1, v1); + tile_base += dst_step; + } + } else { + const HVX_Vector vzero = Q6_V_vzero(); + for (uint32_t i = 0; i < n_c_iters; ++i) { + HVX_Vector v0 = p0 ? hvx_vmem(p0) : vzero; + if (p0) p0 += c_byte_step; + HVX_Vector v1 = p1 ? hvx_vmem(p1) : vzero; + if (p1) p1 += c_byte_step; + Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off0, v0); + Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off1, v1); + tile_base += dst_step; + } + } + } + } else { + // Fallback: scatter one K-tile per call (region 2047, masked). + const uint32_t c_step = HMX_FP16_TILE_N_COLS; + const size_t c_byte_step = (size_t) c_step * sizeof(__fp16); + const size_t dst_step = (size_t) HMX_FP16_TILE_N_ELMS; + const uint32_t n_c_iters = k / c_step; + + for (uint32_t r = start_row; r < end_row; r += 2) { + const uint32_t ct = r / HMX_FP16_TILE_N_ROWS; + const uint32_t local_r = r % HMX_FP16_TILE_N_ROWS; + const bool row0_valid = r < n_cols; + const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_cols; + const HVX_Vector v_off0 = Q6_Vw_vadd_VwVw(v_scat_base, Q6_V_vsplat_R(local_r * 4)); + const HVX_Vector v_off1 = Q6_Vw_vadd_VwVw(v_off0, v_scat_step); + + __fp16 * tile_base = vtcm_dst + (size_t) ct * n_k_tiles * HMX_FP16_TILE_N_ELMS; + const uint8_t * p0 = row0_valid ? (const uint8_t *) (vtcm_src + r * src_stride) : NULL; + const uint8_t * p1 = next_row_valid ? (const uint8_t *) (vtcm_src + (r + 1) * src_stride) : NULL; + + if (p0 && p1) { + for (uint32_t i = 0; i < n_c_iters; ++i) { + HVX_Vector v0 = hvx_vmemu(p0); p0 += c_byte_step; + HVX_Vector v1 = hvx_vmemu(p1); p1 += c_byte_step; + Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off0, v0); + Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off1, v1); + tile_base += dst_step; + } + } else { + const HVX_Vector vzero = Q6_V_vzero(); + for (uint32_t i = 0; i < n_c_iters; ++i) { + HVX_Vector v0 = p0 ? hvx_vmemu(p0) : vzero; + if (p0) p0 += c_byte_step; + HVX_Vector v1 = p1 ? hvx_vmemu(p1) : vzero; + if (p1) p1 += c_byte_step; + Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off0, v0); + Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off1, v1); + tile_base += dst_step; + } + } + } + } } -// --- VTCM sequential allocator (from htp-ops-lib/include/dsp/vtcm_mgr.h) --- +// Interleave row-major FP16 data into column-major tile format. +// Input: [n_rows, head_dim] row-major. Output: tile[dim_tile][row_tile]. +// Processes rows [start_row, end_row) for multi-thread slicing. +// Full range: start_row=0, end_row=n_rows. +static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out, + const __fp16 * restrict src, + uint32_t n_rows, + uint32_t head_dim, + size_t src_stride, + uint32_t n_row_tiles, + uint32_t start_row, + uint32_t end_row) { + __builtin_assume(head_dim > 0); + const size_t tile_stride_elms = (size_t) n_row_tiles * HMX_FP16_TILE_N_ELMS; + + for (uint32_t r = start_row; r < end_row; r += 2) { + const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_rows; + + const HVX_Vector * pv_in0 = (const HVX_Vector *) (src + r * src_stride); + const HVX_Vector * pv_in1 = next_row_valid ? (const HVX_Vector *) (src + (r + 1) * src_stride) : NULL; + + // Row-pair invariants hoisted out of the c loop. + const uint32_t r0 = r / HMX_FP16_TILE_N_ROWS; + const uint32_t r1_half = (r % HMX_FP16_TILE_N_ROWS) / 2; -static inline uint8_t *vtcm_seq_alloc(uint8_t **vtcm_ptr, size_t size) { - uint8_t *p = *vtcm_ptr; - *vtcm_ptr += size; - return p; + // tb0 starts at tile (c0=0, r0); tb1 at the adjacent dim-tile (c0=1, r0). + // Each c step (+= 64) advances both by 2 dim-tiles worth of fp16. + __fp16 * tb0 = tiles_out + (size_t) r0 * HMX_FP16_TILE_N_ELMS; + __fp16 * tb1 = tb0 + tile_stride_elms; + const size_t tb_step = 2 * tile_stride_elms; + + if (pv_in1) { + for (uint32_t c = 0; c < head_dim; c += 64) { + HVX_Vector v0 = *pv_in0++; + HVX_Vector v1 = *pv_in1++; + HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2); + ((HVX_Vector *) tb0)[r1_half] = Q6_V_lo_W(vp); + ((HVX_Vector *) tb1)[r1_half] = Q6_V_hi_W(vp); + tb0 += tb_step; + tb1 += tb_step; + } + } else { + const HVX_Vector vzero = Q6_V_vzero(); + for (uint32_t c = 0; c < head_dim; c += 64) { + HVX_Vector v0 = *pv_in0++; + HVX_VectorPair vp = Q6_W_vshuff_VVR(vzero, v0, -2); + ((HVX_Vector *) tb0)[r1_half] = Q6_V_lo_W(vp); + ((HVX_Vector *) tb1)[r1_half] = Q6_V_hi_W(vp); + tb0 += tb_step; + tb1 += tb_step; + } + } + } } +// --- HMX inline asm macros for load-store packetization --- +#define HMX_LOAD_MPY_F16(act, wt, range) \ + "{\n" \ + " activation.hf = mxmem(" act ", " range ")\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ + "}\n" + +#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \ + "{\n" \ + " activation.hf = mxmem(" act ", " range "):deep\n" \ + " weight.hf = mxmem(" wt ", " range ")\n" \ + "}\n" + +#define HMX_STORE_AFTER_F16(out, scale_reg) \ + "mxmem(" out ", " scale_reg "):after.hf = acc\n" + +#define HMX_SET_BIAS(scales) \ + "bias = mxmem2(" scales ")\n" + +#define HMX_CLRACC_F16() \ + "mxclracc.hf\n" + #endif // HMX_UTILS_H diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index 8b5e47a..5d14a85 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -1,25 +1,34 @@ #ifndef HTP_CTX_H #define HTP_CTX_H -#include "hex-dma.h" +#include "dma-queue.h" #include "hmx-queue.h" +#include "htp-dcache.h" #include "htp-ops.h" -#include "worker-pool.h" +#include "hex-profile.h" +#include "work-queue.h" +#include "hex-fastdiv.h" #include #include #include +#include #include +#include +#include +#ifndef HTP_MAX_NTHREADS #define HTP_MAX_NTHREADS 10 -#define HTP_MAX_MMAPS 16 +#endif + +#define HTP_MAX_MMAPS 64 // Memory mapping struct htp_mmap { uint64_t size; uint64_t base; uint32_t fd; - uint32_t pinned; + uint32_t flags; }; // Scratchpad state @@ -31,6 +40,14 @@ struct htp_spad { uint32_t size_per_thread; // size per thread }; +struct htp_mdev_group { + uint16_t idx; + uint16_t count; + struct fastdiv_values count_div; + uint8_t * fence_base; + uint32_t fence_seq; +}; + struct htp_context; // Context while processing an Op @@ -40,33 +57,50 @@ struct htp_ops_context { enum htp_op_code op; // FIXME: rename to opcode int32_t op_params[HTP_OP_MAX_PARAMS]; + int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS]; const struct htp_tensor * src[HTP_OP_MAX_INPUTS]; - const struct htp_tensor * dst; + union { + const struct htp_tensor * dst; + const struct htp_tensor * dsts[HTP_OP_MAX_OUTPUTS]; + }; // TODO convert these to an array struct htp_spad src0_spad; struct htp_spad src1_spad; - struct htp_spad src2_spad; - struct htp_spad src3_spad; struct htp_spad dst_spad; - uint32_t n_threads; - uint32_t flags; + uint32_t flags; + uint32_t n_threads; + struct fastdiv_values n_threads_div; + int status; + + // Must stay last: not cleared by htp_ops_context_reset(), ops init the caches they use + union { + dma_cache_dm dma_cache_dm[HTP_MAX_NTHREADS]; + dma_cache_fa dma_cache_fa[HTP_MAX_NTHREADS]; + } __attribute__((aligned(128))); }; // Main context for htp DSP backend struct htp_context { - dspqueue_t queue; - dma_queue * dma[HTP_MAX_NTHREADS]; + dspqueue_t dsp_queue; + struct htp_mmap mmap[HTP_MAX_MMAPS]; - worker_pool_context_t worker_pool; + dma_queue_t dma[HTP_MAX_NTHREADS]; + struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1]; + work_queue_t work_queue; + hmx_queue_t hmx_queue; + uint32_t n_threads; + struct fastdiv_values n_threads_div; int thread_id; int thread_prio; - int hmx_enabled; + bool hmx_enabled; + bool etm; + uint32_t profiler; uint8_t * vtcm_base; size_t vtcm_size; @@ -74,19 +108,60 @@ struct htp_context { atomic_bool vtcm_valid; atomic_bool vtcm_needs_release; + uint64_t max_vmem; + struct htp_dirty_range dirty_ranges[HTP_MAX_DIRTY_RANGES]; + + // Persistent DDR scratchpad for MUL_MAT_ID mappings + void * ddr_spad_base; + size_t ddr_spad_size; + + struct htp_mdev_group mdev; struct htp_ops_context octx; -#ifdef HTP_HAS_HMX - struct hmx_queue * hmx_queue; // Async HMX queue for pipeline overlap -#endif + qurt_thread_t main_thread; + void * main_stack; + atomic_bool killed; + size_t footprint; }; +static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct htp_context * ctx) { + memset(octx, 0, offsetof(struct htp_ops_context, dma_cache_dm)); + octx->ctx = ctx; + octx->n_threads = ctx->n_threads; + octx->n_threads_div = ctx->n_threads_div; +} + +static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) { + if (n_threads == 0 || n_threads > octx->ctx->n_threads) { + return false; + } + + if (n_threads != octx->n_threads) { + octx->n_threads = n_threads; + octx->n_threads_div = n_threads == octx->ctx->n_threads + ? octx->ctx->n_threads_div + : init_fastdiv_values(n_threads); + } + + return true; +} + +static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) { + if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) { + octx->status = status; + } +} + int op_matmul(struct htp_ops_context * octx); int op_matmul_id(struct htp_ops_context * octx); +int op_matmul_nx(struct htp_ops_context * octx); +int op_matmul_id_nx(struct htp_ops_context * octx); int op_binary(struct htp_ops_context * octx); int op_unary(struct htp_ops_context * octx); +int op_sum(struct htp_ops_context * octx); int op_sum_rows(struct htp_ops_context * octx); -int op_activations(struct htp_ops_context * octx); +int op_argmax(struct htp_ops_context * octx); +int op_glu(struct htp_ops_context * octx); int op_softmax(struct htp_ops_context * octx); int op_add_id(struct htp_ops_context * octx); int op_rope(struct htp_ops_context * octx); @@ -96,7 +171,20 @@ int op_get_rows(struct htp_ops_context * octx); int op_cpy(struct htp_ops_context * octx); int op_repeat(struct htp_ops_context * octx); int op_argsort(struct htp_ops_context * octx); +int op_top_k(struct htp_ops_context * octx); int op_ssm_conv(struct htp_ops_context * octx); int op_cumsum(struct htp_ops_context * octx); +int op_fill(struct htp_ops_context * octx); +int op_concat(struct htp_ops_context * octx); +int op_diag(struct htp_ops_context * octx); +int op_solve_tri(struct htp_ops_context * octx); +int op_gated_delta_net(struct htp_ops_context * octx); +int op_pad(struct htp_ops_context * octx); +int op_im2col(struct htp_ops_context * octx); +int op_im2col_3d(struct htp_ops_context * octx); +int op_pool_2d(struct htp_ops_context * octx); +int op_pool_1d(struct htp_ops_context * octx); +int op_allreduce(struct htp_ops_context * octx); +int op_roll(struct htp_ops_context * octx); #endif /* HTP_CTX_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-dcache.c b/ggml/src/ggml-hexagon/htp/htp-dcache.c new file mode 100644 index 0000000..840b1ed --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-dcache.c @@ -0,0 +1,360 @@ +#include "htp-dcache.h" + +#include +#include +#include + +#include "hex-common.h" +#include "hex-utils.h" +#include "hex-fastdiv.h" +#include "hex-profile.h" +#include "htp-ctx.h" +#include "work-queue.h" + +struct l2flush_range { + uint32_t start; // line-aligned start address + uint32_t end; // line-aligned end address + uint32_t block_first; // global block index of this range's first block + uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial) +}; + +struct l2flush_multi_task { + struct htp_thread_trace * trace; + struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES]; + uint32_t n_ranges; + uint32_t total_blocks; + uint32_t blocks_per_thread; +}; + +static void flush_all_dcache(struct htp_context * ctx) { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + hex_l2fetch_block(ctx, ctx->footprint); + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0); + memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); +} + +static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) { + struct l2flush_multi_task * task = (struct l2flush_multi_task *) data; + (void) n; + + const uint32_t gb_first = i * task->blocks_per_thread; + uint32_t gb_last = gb_first + task->blocks_per_thread; + if (gb_last > task->total_blocks) { + gb_last = task->total_blocks; + } + if (gb_first >= gb_last) { + return; + } + + struct htp_thread_trace * tr = &task->trace[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); + + for (uint32_t r = 0; r < task->n_ranges; r++) { + const struct l2flush_range * rg = &task->ranges[r]; + const uint32_t rb_first = rg->block_first; + const uint32_t rb_last = rg->block_first + rg->n_blocks; + + const uint32_t lo = gb_first > rb_first ? gb_first : rb_first; + const uint32_t hi = gb_last < rb_last ? gb_last : rb_last; + if (lo >= hi) { + continue; + } + + const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE; + uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE; + if (e > rg->end) { + e = rg->end; + } + hex_l2flush((void *) (uintptr_t) s, e - s); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first); +} + +static void merge_dirty_ranges(struct htp_context * ctx) { + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) continue; + + for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES;) { + struct htp_dirty_range * s = &ctx->dirty_ranges[j]; + if (i == j || !s->start || r->end < s->start || s->end < r->start) { + j++; + continue; + } + + r->start = MIN(r->start, s->start); + r->end = MAX(r->end, s->end); + s->start = 0; + s->end = 0; + j = 0; + } + } +} + +void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { + const struct htp_tensor * pending[HTP_OP_MAX_OUTPUTS]; + uint32_t n_pending = 0; + + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) { + continue; + } + + uint32_t t_start = t->data; + uint32_t t_end = t_start + t->size; + + bool merged = false; + for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES; j++) { + struct htp_dirty_range * r = &ctx->dirty_ranges[j]; + if (!r->start) continue; + + if (r->start <= t_end && t_start <= r->end) { + uint32_t new_start = (t_start < r->start) ? t_start : r->start; + uint32_t new_end = (t_end > r->end) ? t_end : r->end; + r->start = new_start; + r->end = new_end; + merged = true; + } + } + + if (!merged) { + pending[n_pending++] = t; + } + } + + merge_dirty_ranges(ctx); + + if (n_pending == 0) { + return; + } + + uint32_t empty_indices[HTP_MAX_DIRTY_RANGES]; + uint32_t active_indices[HTP_MAX_DIRTY_RANGES]; + uint32_t n_active = 0; + uint32_t n_empty = 0; + for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES; j++) { + if (ctx->dirty_ranges[j].start) { + active_indices[n_active++] = j; + } else { + empty_indices[n_empty++] = j; + } + } + + if (n_pending <= n_empty) { + for (uint32_t i = 0; i < n_pending; i++) { + uint32_t idx = empty_indices[i]; + struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; + r->start = pending[i]->data; + r->end = pending[i]->data + pending[i]->size; + } + merge_dirty_ranges(ctx); + return; + } + + uint32_t n_evict = n_pending - n_empty; + uint32_t total_evict_size = 0; + for (uint32_t i = 0; i < n_evict; i++) { + uint32_t idx = active_indices[i]; + struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; + total_evict_size += r->end - r->start; + } + + if (total_evict_size > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); + for (uint32_t i = 0; i < n_pending; i++) { + struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + r->start = pending[i]->data; + r->end = pending[i]->data + pending[i]->size; + } + merge_dirty_ranges(ctx); + return; + } + + if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) { + struct l2flush_multi_task task; + task.trace = ctx->trace; + task.n_ranges = n_evict; + + uint32_t block_acc = 0; + for (uint32_t i = 0; i < n_evict; i++) { + uint32_t idx = active_indices[i]; + struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; + + struct l2flush_range * rg = &task.ranges[i]; + rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); + rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); + rg->block_first = block_acc; + rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + block_acc += rg->n_blocks; + } + + task.total_blocks = block_acc; + task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + + work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + } else { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); + for (uint32_t i = 0; i < n_evict; i++) { + uint32_t idx = active_indices[i]; + struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; + uint32_t size = r->end - r->start; + hex_l2flush((void *) (uintptr_t) r->start, size); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0); + } + + for (uint32_t i = 0; i < n_evict; i++) { + uint32_t idx = active_indices[i]; + struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; + r->start = pending[i]->data; + r->end = pending[i]->data + pending[i]->size; + } + + for (uint32_t i = 0; i < n_empty; i++) { + uint32_t idx = empty_indices[i]; + struct htp_dirty_range * r = &ctx->dirty_ranges[idx]; + r->start = pending[n_evict + i]->data; + r->end = pending[n_evict + i]->data + pending[n_evict + i]->size; + } + + merge_dirty_ranges(ctx); +} + +static void make_tensor_clean(struct htp_context * ctx, const struct htp_tensor * t) { + if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) { + return; + } + + uint32_t t_start = t->data; + uint32_t t_end = t_start + t->size; + + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) continue; + + if (r->start < t_end && t_start < r->end) { + if (t_start <= r->start && r->end <= t_end) { + r->start = 0; + r->end = 0; + } else if (t_start <= r->start) { + r->start = t_end; + } else if (r->end <= t_end) { + r->end = t_start; + } + } + } +} + +static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_tensor * t) { + if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) { + return false; + } + + uint32_t t_start = t->data; + uint32_t t_end = t_start + t->size; + + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) continue; + + if (r->start < t_end && t_start < r->end) { + return true; + } + } + return false; +} + +static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) { + if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) { + struct l2flush_multi_task task; + task.trace = ctx->trace; + task.n_ranges = n_ranges; + + uint32_t block_acc = 0; + for (uint32_t i = 0; i < n_ranges; i++) { + const struct htp_dirty_range * r = &ranges[i]; + struct l2flush_range * rg = &task.ranges[i]; + rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE); + rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE); + rg->block_first = block_acc; + rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE; + block_acc += rg->n_blocks; + } + + task.total_blocks = block_acc; + task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div); + + work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads); + } else { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0); + for (uint32_t i = 0; i < n_ranges; i++) { + const struct htp_dirty_range * r = &ranges[i]; + hex_l2flush((void *) (uintptr_t) r->start, r->end - r->start); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0); + } +} + +void htp_dcache_flush_dirty_ranges(struct htp_context * ctx) { + struct htp_dirty_range ranges[HTP_MAX_DIRTY_RANGES]; + uint32_t n_ranges = 0; + uint64_t total_dirty = 0; + + for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) { + const struct htp_dirty_range * r = &ctx->dirty_ranges[i]; + if (!r->start) { + continue; + } + ranges[n_ranges++] = *r; + total_dirty += r->end - r->start; + } + + if (total_dirty == 0) { + return; + } + + if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); + return; + } + + flush_dirty_ranges(ctx, ranges, n_ranges, total_dirty); + memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); +} + +void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) { + const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS]; + struct htp_dirty_range ranges[HTP_OP_MAX_INPUTS]; + uint32_t n_dirty = 0; + uint64_t total_dirty = 0; + + for (uint32_t i = 0; i < n; i++) { + const struct htp_tensor * t = tensors[i]; + if (is_tensor_dirty(ctx, t)) { + dirty_tensors[n_dirty++] = t; + ranges[n_dirty - 1].start = t->data; + ranges[n_dirty - 1].end = t->data + t->size; + total_dirty += t->size; + } + } + + if (total_dirty == 0) { + return; + } + + if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) { + flush_all_dcache(ctx); + return; + } + + flush_dirty_ranges(ctx, ranges, n_dirty, total_dirty); + for (uint32_t i = 0; i < n_dirty; i++) { + make_tensor_clean(ctx, dirty_tensors[i]); + } +} diff --git a/ggml/src/ggml-hexagon/htp/htp-dcache.h b/ggml/src/ggml-hexagon/htp/htp-dcache.h new file mode 100644 index 0000000..1996ab2 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-dcache.h @@ -0,0 +1,20 @@ +#ifndef HTP_DCACHE_H +#define HTP_DCACHE_H + +#include +#include "htp-ops.h" + +#define HTP_MAX_DIRTY_RANGES 64 + +struct htp_dirty_range { + uint32_t start; + uint32_t end; +}; + +struct htp_context; + +void htp_dcache_flush_dirty_ranges(struct htp_context * ctx); +void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); +void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n); + +#endif // HTP_DCACHE_H diff --git a/ggml/src/ggml-hexagon/htp/htp-fence.h b/ggml/src/ggml-hexagon/htp/htp-fence.h new file mode 100644 index 0000000..7450b5d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-fence.h @@ -0,0 +1,89 @@ +#ifndef HTP_FENCE_H +#define HTP_FENCE_H + +#include +#include + +#include + +#include "hex-utils.h" +#include "htp-ops.h" +#include "htp-ctx.h" + +static inline atomic_uint * htp_mdev_fence_slot(const void * fence_base, uint32_t idx) { + return (atomic_uint *) ((const uint8_t *) fence_base + (size_t) idx * HTP_FENCE_SLOT_SIZE); +} + +static inline void htp_fence_write(void * fence_ptr, uint32_t seq, uint32_t status) { + atomic_uint * fence = (atomic_uint *) fence_ptr; + atomic_store(&fence[1], status); + atomic_store(&fence[0], seq); + asm volatile ("syncht" : : : "memory"); + Q6_dccleaninva_A((void *) fence); +} + +static inline void htp_fence_read(const void * fence_ptr, uint32_t * seq, uint32_t * status) { + const atomic_uint * fence = (const atomic_uint *) fence_ptr; + Q6_dccleaninva_A((void *) fence); + asm volatile ("syncht" : : : "memory"); + *seq = atomic_load(&fence[0]); + *status = atomic_load(&fence[1]); +} + +static inline void htp_mdev_group_barrier(struct htp_ops_context * octx) { + struct htp_context * ctx = octx->ctx; + if (ctx->mdev.count <= 1) { + return; + } + + const uint32_t seq = ++ctx->mdev.fence_seq; + + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + + const uint32_t mdev_idx = ctx->mdev.idx; + const uint32_t mdev_count = ctx->mdev.count; + + uint8_t * fence_base = ctx->mdev.fence_base; + atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx); + htp_fence_write(my_fence, seq, octx->status); + + for (uint32_t d = 0; d < mdev_count; d++) { + if (d == mdev_idx) continue; + atomic_uint * peer_fence = htp_mdev_fence_slot(fence_base, d); + uint64_t spins = 0; + while (1) { + uint32_t peer_seq; + uint32_t peer_status; + htp_fence_read(peer_fence, &peer_seq, &peer_status); + if ((int32_t)(peer_seq - seq) >= 0) { + if (peer_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: mdev %u peer %u failed with status %u : seq 0x%08x\n", + mdev_idx, d, peer_status, seq); + htp_ops_context_set_status(octx, peer_status); + } + break; + } + if (++spins == 10000) { + FARF(ALWAYS, "ggml-hex: mdev %u waiting for mdev %u : seq 0x%08x (b %u op %u) my-fence %p peer-fence %p peer-seq 0x%08x (diff %d)\n", + mdev_idx, d, seq, seq >> 12, seq & 0xfff, my_fence, peer_fence, peer_seq, (int32_t)(peer_seq - seq)); + } + if (spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: mdev %u timeout waiting for mdev %u : seq 0x%08x (b %u op %u) peer-fence %p peer-seq 0x%08x\n", + mdev_idx, d, seq, seq >> 12, seq & 0xfff, peer_fence, peer_seq); + htp_ops_context_set_status(octx, HTP_STATUS_INTERNAL_ERR); + break; + } + hex_pause(); + } + } + asm volatile ("syncht" : : : "memory"); + + if (octx->status > HTP_STATUS_OK) { + htp_fence_write(my_fence, seq, octx->status); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); +} + +#endif // HTP_FENCE_H diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index 79b5ecd..f3b2d6b 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -20,33 +20,24 @@ enum htp_data_type { HTP_TYPE_F32 = 0, HTP_TYPE_F16 = 1, HTP_TYPE_Q4_0 = 2, + HTP_TYPE_Q4_1 = 3, HTP_TYPE_Q8_0 = 8, + HTP_TYPE_Q2_K = 10, + HTP_TYPE_Q3_K = 11, + HTP_TYPE_Q4_K = 12, + HTP_TYPE_Q5_K = 13, + HTP_TYPE_Q6_K = 14, HTP_TYPE_IQ4_NL = 20, HTP_TYPE_I32 = 26, HTP_TYPE_I64 = 27, + HTP_TYPE_BF16 = 30, HTP_TYPE_MXFP4 = 39, - - // types used internally for repack, dyn.quant, etc - HTP_TYPE_Q4_0x4x2 = 200, - HTP_TYPE_Q8_0x4x2, - HTP_TYPE_MXFP4x4x2, + HTP_TYPE_Q1_0 = 41, + HTP_TYPE_Q2_0 = 42, HTP_TYPE_INVALID }; -// Constats for internal types -#define QK_Q4_0x4x2 256 // 4x Q4_0 blocks packed with next 4x Q4_0 blocks (size in bytes 128) -#define QK_Q8_0x4x2 256 // 4x Q8_0 blocks concat with next 4x Q8_0 blocks -#define QK_MXFP4x4x2 256 // 4x MXFP4 blocks concat with next 4x MXFP4 blocks - - -// Mask to enable various stages of the Ops. -// Used for debugging and profiling. -enum htp_op_mask { - HTP_OPMASK_QUEUE = (1 << 0), // Enable Queueing (ie calls into the DSP) - HTP_OPMASK_COMPUTE = (1 << 1), // Enable Compute -}; - // Do not reorder first 4 (used as an index) enum htp_op_code { HTP_OP_MUL = 0, @@ -55,16 +46,26 @@ enum htp_op_code { HTP_OP_DIV = 3, HTP_OP_MUL_MAT, HTP_OP_MUL_MAT_ID, + HTP_OP_MUL_MAT_NX, + HTP_OP_MUL_MAT_ID_NX, + HTP_OP_MUL_MAT_ADD, HTP_OP_RMS_NORM, + HTP_OP_RMS_NORM_MUL, HTP_OP_UNARY_SILU, HTP_OP_UNARY_GELU, HTP_OP_UNARY_SIGMOID, HTP_OP_UNARY_EXP, HTP_OP_UNARY_NEG, HTP_OP_UNARY_SOFTPLUS, + HTP_OP_UNARY_TANH, + HTP_OP_UNARY_ABS, + HTP_OP_UNARY_LOG, + HTP_OP_UNARY_RELU, + HTP_OP_UNARY_STEP, HTP_OP_GLU_SWIGLU, HTP_OP_GLU_SWIGLU_OAI, HTP_OP_GLU_GEGLU, + HTP_OP_GLU_GEGLU_QUICK, HTP_OP_SOFTMAX, HTP_OP_ADD_ID, HTP_OP_ROPE, @@ -73,45 +74,85 @@ enum htp_op_code { HTP_OP_GET_ROWS, HTP_OP_SCALE, HTP_OP_CPY, + HTP_OP_CPY_FENCE, HTP_OP_ARGSORT, + HTP_OP_TOP_K, HTP_OP_SQR, HTP_OP_SQRT, + HTP_OP_SUM, HTP_OP_SUM_ROWS, HTP_OP_SSM_CONV, HTP_OP_REPEAT, HTP_OP_CUMSUM, + HTP_OP_FILL, + HTP_OP_DIAG, + HTP_OP_SOLVE_TRI, + HTP_OP_L2_NORM, + HTP_OP_GATED_DELTA_NET, + HTP_OP_TRI, + HTP_OP_PAD, + HTP_OP_NORM, + HTP_OP_CONCAT, + HTP_OP_CLAMP, + HTP_OP_LEAKY_RELU, + HTP_OP_IM2COL, + HTP_OP_FENCE, + HTP_OP_ALLREDUCE, + HTP_OP_ALLREDUCE_ADD, + HTP_OP_GLU_SWIGLU_CLAMP, + HTP_OP_MDEV_GROUP, + HTP_OP_ROLL, + HTP_OP_ARGMAX, + HTP_OP_UNARY_GELU_ERF, + HTP_OP_GLU_GEGLU_ERF, + HTP_OP_POOL_2D, + HTP_OP_POOL_1D, + HTP_OP_UNARY_GELU_QUICK, + HTP_OP_IM2COL_3D, + HTP_OP_HADAMARD, + HTP_OP_GLU_REGLU, + HTP_OP_GLU_SIGMOID, + HTP_OP_UNARY_CEIL, + HTP_OP_UNARY_SGN, HTP_OP_INVALID }; #define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS -#define HTP_OP_MAX_INPUTS 6 // aka GGML_MAX_SRCS +#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS +#define HTP_OP_MAX_OUTPUTS 4 #define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS +#define HTP_OP_MAX_KERN_PARAMS 32 -#define HTP_OP_MAX_BUFS 8 -#define HTP_OP_MAX_REQS 256 -#define HTP_OP_MAX_TENSORS (HTP_OP_MAX_REQS * HTP_OP_MAX_INPUTS + HTP_OP_MAX_REQS) +#define HTP_OP_MAX_BUFS 64 +#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16) -#if __HVX_ARCH__ < 75 -#define HTP_OP_MAX_VMEM (3167538380u) -#else -#define HTP_OP_MAX_VMEM (3221225472u) -#endif +#define HTP_FENCE_TIMEOUT (1000000000ULL) +#define HTP_FENCE_SLOT_SIZE 128 + +#define HTP_OP_MAX_VMEM_DEFAULT (3355443200u) #define HTP_MMAP_MAX_VMEM (2147483648u) enum htp_tensor_flags { - HTP_TENSOR_COMPUTE = (1U << 0), // Tensor buffer temporal compute data (not weights) - HTP_TENSOR_FLUSHED = (1U << 1) // Tensor buffer has been flushed (set by the NPU) + HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute) + HTP_TENSOR_TILED = (1U << 1), // Tensor is in tiled format + HTP_TENSOR_FENCE = (1U << 2), // Tensor is synchronization fence (explicitly managed) + HTP_TENSOR_FUSEABLE = (1U << 3), // Tensor can be fused (host-only) +}; + +enum htp_buf_flags { + HTP_BUF_EXTENDED = (1U << 0), }; // Tensor descriptor struct htp_tensor { - uint32_t data; // Buffer offset in the messages, and data pointer on the NPU + uint64_t data; // Buffer offset in the messages, and data pointer on the NPU uint32_t size; // Data size in bytes uint32_t flags; // Buffer / tensor flags - uint16_t type; // Data type + uint32_t type; // Data type uint16_t bi; // Buffer index + uint16_t ti; // Tensor index uint32_t ne[HTP_OP_MAX_DIMS]; // Number of elements uint32_t nb[HTP_OP_MAX_DIMS]; // Stride in bytes (see ggml.h ggml_tensor) }; @@ -120,12 +161,12 @@ struct htp_tensor { struct htp_buf_desc { uint64_t base; // base address uint64_t size; // total size - uint32_t flags; // buffer flags (unused) + uint32_t flags; // HTP_BUF_* uint32_t fd; // file descriptor }; enum htp_op_flags { - HTP_OPFLAGS_SKIP_COMPUTE = (1U << 0), // Skip actual computation (used for profiling) + HTP_OPFLAGS_STUB = (1U << 0), }; // Op descriptor @@ -133,29 +174,108 @@ struct htp_op_desc { uint32_t opcode; // GGML/HTP Op uint32_t flags; // Op flags int32_t params[HTP_OP_MAX_PARAMS]; // Params for the op, e.g. epsilon of RMS norm + int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS]; // generic blob for host-precomputed parameters uint16_t src[HTP_OP_MAX_INPUTS]; // Input tensors indices - uint16_t dst; // Output tensor index + uint16_t dst[HTP_OP_MAX_OUTPUTS]; // Output tensor indices + uint16_t pad[2]; // padding to align to 64 bits +}; + +#ifndef HTP_MAX_NTHREADS +#define HTP_MAX_NTHREADS 10 +#endif + +#define HTP_TRACE_MAX_EVENTS 256 + +enum htp_profiler_mode { + HTP_PROF_DISABLED = 0, + HTP_PROF_BASIC = 1, + HTP_PROF_PMU = 2, + HTP_PROF_TRACE = 3, +}; + +// DCVS/HMX power profile requested for the session. +enum htp_power_mode { + HTP_POWER_MODE_LOW_POWER_SAVER = 0, + HTP_POWER_MODE_POWER_SAVER = 1, + HTP_POWER_MODE_HIGH_POWER_SAVER = 2, + HTP_POWER_MODE_LOW_BALANCED = 3, + HTP_POWER_MODE_BALANCED = 4, + HTP_POWER_MODE_HIGH_PERFORMANCE = 5, + HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE = 6, + HTP_POWER_MODE_BURST = 7, + + HTP_POWER_MODE_COUNT +}; + +enum htp_trace_event_id { + HTP_TRACE_EVT_DMA = 0, + HTP_TRACE_EVT_L2FLUSH = 1, + HTP_TRACE_EVT_INIT = 2, + HTP_TRACE_EVT_BUFF = 3, + HTP_TRACE_EVT_FENCE = 4, + + HTP_TRACE_EVT_HVX_COMP = 20, + HTP_TRACE_EVT_HVX_A_QUANT = 21, + HTP_TRACE_EVT_HVX_A_PREP = 22, + HTP_TRACE_EVT_HVX_W_DEQUANT = 23, + HTP_TRACE_EVT_HVX_W_PREP = 24, + HTP_TRACE_EVT_HVX_O_PROC = 25, + HTP_TRACE_EVT_HVX_FA_QK = 26, + HTP_TRACE_EVT_HVX_FA_SFM = 27, + HTP_TRACE_EVT_HVX_FA_Q_PREP = 28, + HTP_TRACE_EVT_HVX_FA_K_PREP = 29, + HTP_TRACE_EVT_HVX_FA_V_PREP = 30, + + HTP_TRACE_EVT_HVX_GDN_PREP = 31, + HTP_TRACE_EVT_HVX_GDN_SOLVE = 32, + HTP_TRACE_EVT_HVX_GDN_V_PREP = 33, + HTP_TRACE_EVT_HVX_GDN_D_PREP = 34, + HTP_TRACE_EVT_HVX_GDN_OUT = 35, + HTP_TRACE_EVT_HVX_GDN_STATE = 36, + HTP_TRACE_EVT_HVX_GDN_REM = 37, + + HTP_TRACE_EVT_HMX_COMP = 40, +}; + +struct htp_trace_desc { + uint32_t cycles; // lower 32-bits of cycle counter + uint16_t id; // Event ID + uint16_t info; // bit 15: is_stop. bits 14-0: tile/chunk index or other metadata. +}; + +#define HTP_PROF_PMU_NCNT 8 - // the rest is filled in-place by the NPU - uint32_t prof_usecs; // Number of usec per request - uint32_t prof_cycles; // Number of cycles per request - uint32_t prof_pkts; // Number of instruction packets per request - uint32_t unused; +// Profile descriptor +struct htp_prof_desc { + uint32_t opcode; // GGML/HTP Op + uint32_t usecs; // Number of usec + uint32_t cycles_start; // Start cycle counter + uint32_t cycles_stop; // Stop cycle counter + uint32_t pmu[HTP_PROF_PMU_NCNT]; // PMU counters }; struct htp_opbatch_req { + uint64_t seq; // Sequence number uint32_t n_bufs; // Number of buffers uint32_t n_tensors; // Number of tensors uint32_t n_ops; // Number of ops - uint32_t flags; // unused + uint32_t n_traces; // Number of trace descriptors per thread // struct htp_buf_desc bufs[]; -- dspqueue buf 0 // struct htp_tensor tensors[]; -- dspqueue buf 0 // struct htp_op_desc ops[]; -- dspqueue buf 0 }; struct htp_opbatch_rsp { - uint32_t status; // HTP_STATUS_... - // struct htp_op_req ops[]; -- dspqueue buf 0 + uint64_t seq; // Sequence number + uint64_t cycles_start; // Start cycle counter + uint64_t cycles_stop; // Stop cycle counter + uint32_t status; // HTP_STATUS_... + uint32_t n_bufs; // Number of buffers + uint32_t n_tensors; // Number of tensors + uint32_t n_ops; // Number of op profile descriptors + uint32_t usecs; // Number of usec + uint32_t n_traces[HTP_MAX_NTHREADS + 1]; + // struct htp_prof_desc profs[]; -- dspqueue buf 0 }; #endif /* HTP_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-swiv.py b/ggml/src/ggml-hexagon/htp/htp-swiv.py new file mode 100644 index 0000000..15d7bef --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-swiv.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 + +import argparse +import logging +import os +import shutil +import struct +import sys +import tempfile +import zlib + +ELF_MAGIC = b'\x7fELF' +ELF_CLASS_INDEX = 4 +ELFCLASS32 = 1 +ELFCLASS64 = 2 +ELF_VERSION_INDEX = 6 +EV_CURRENT = 1 +MAX_PHDR_COUNT = 100 + +LOAD_TYPE = 0x1 + +SHT_NOBITS = 8 + +SWIV_MAGIC = b'SWIV' +SWIV_SEG_SIZE_MIN = 16 +# SHT_LOUSER plus ASCII "SWIV". +SWIV_SECTION_TYPE = 0xD3574956 + +logger = logging.getLogger("htp-swiv") + + +class Elf_Ehdr_common: + s = struct.Struct('16sHHI') + + def __init__(self, data): + unpacked_data = (Elf_Ehdr_common.s).unpack(data) + self.e_ident = unpacked_data[0] + self.e_type = unpacked_data[1] + self.e_machine = unpacked_data[2] + self.e_version = unpacked_data[3] + + +class Elf32_Ehdr: + s = struct.Struct('16sHHIIIIIHHHHHH') + + def __init__(self, data): + unpacked_data = (Elf32_Ehdr.s).unpack(data) + self.e_ident = unpacked_data[0] + self.e_type = unpacked_data[1] + self.e_machine = unpacked_data[2] + self.e_version = unpacked_data[3] + self.e_entry = unpacked_data[4] + self.e_phoff = unpacked_data[5] + self.e_shoff = unpacked_data[6] + self.e_flags = unpacked_data[7] + self.e_ehsize = unpacked_data[8] + self.e_phentsize = unpacked_data[9] + self.e_phnum = unpacked_data[10] + self.e_shentsize = unpacked_data[11] + self.e_shnum = unpacked_data[12] + self.e_shstrndx = unpacked_data[13] + + def get_packed_data(self): + values = [self.e_ident, + self.e_type, + self.e_machine, + self.e_version, + self.e_entry, + self.e_phoff, + self.e_shoff, + self.e_flags, + self.e_ehsize, + self.e_phentsize, + self.e_phnum, + self.e_shentsize, + self.e_shnum, + self.e_shstrndx + ] + + return (Elf32_Ehdr.s).pack(*values) + + +class Elf32_Phdr: + s = struct.Struct('I' * 8) + + def __init__(self, data): + unpacked_data = (Elf32_Phdr.s).unpack(data) + self.p_type = unpacked_data[0] + self.p_offset = unpacked_data[1] + self.p_vaddr = unpacked_data[2] + self.p_paddr = unpacked_data[3] + self.p_filesz = unpacked_data[4] + self.p_memsz = unpacked_data[5] + self.p_flags = unpacked_data[6] + self.p_align = unpacked_data[7] + + def get_packed_data(self): + values = [self.p_type, + self.p_offset, + self.p_vaddr, + self.p_paddr, + self.p_filesz, + self.p_memsz, + self.p_flags, + self.p_align + ] + + return (Elf32_Phdr.s).pack(*values) + + +class Elf32_Shdr: + s = struct.Struct('I' * 10) + + def __init__(self, data): + unpacked_data = (Elf32_Shdr.s).unpack(data) + self.sh_name = unpacked_data[0] + self.sh_type = unpacked_data[1] + self.sh_flags = unpacked_data[2] + self.sh_addr = unpacked_data[3] + self.sh_offset = unpacked_data[4] + self.sh_size = unpacked_data[5] + self.sh_link = unpacked_data[6] + self.sh_info = unpacked_data[7] + self.sh_addralign = unpacked_data[8] + self.sh_entsize = unpacked_data[9] + self.sh_name_str = "" + + def get_packed_data(self): + values = [self.sh_name, + self.sh_type, + self.sh_flags, + self.sh_addr, + self.sh_offset, + self.sh_size, + self.sh_link, + self.sh_info, + self.sh_addralign, + self.sh_entsize] + + return (Elf32_Shdr.s).pack(*values) + + +class Elf64_Ehdr: + s = struct.Struct('16sHHIQQQIHHHHHH') + + def __init__(self, data): + unpacked_data = (Elf64_Ehdr.s).unpack(data) + self.e_ident = unpacked_data[0] + self.e_type = unpacked_data[1] + self.e_machine = unpacked_data[2] + self.e_version = unpacked_data[3] + self.e_entry = unpacked_data[4] + self.e_phoff = unpacked_data[5] + self.e_shoff = unpacked_data[6] + self.e_flags = unpacked_data[7] + self.e_ehsize = unpacked_data[8] + self.e_phentsize = unpacked_data[9] + self.e_phnum = unpacked_data[10] + self.e_shentsize = unpacked_data[11] + self.e_shnum = unpacked_data[12] + self.e_shstrndx = unpacked_data[13] + + def get_packed_data(self): + values = [self.e_ident, + self.e_type, + self.e_machine, + self.e_version, + self.e_entry, + self.e_phoff, + self.e_shoff, + self.e_flags, + self.e_ehsize, + self.e_phentsize, + self.e_phnum, + self.e_shentsize, + self.e_shnum, + self.e_shstrndx + ] + + return (Elf64_Ehdr.s).pack(*values) + + +class Elf64_Phdr: + s = struct.Struct('IIQQQQQQ') + + def __init__(self, data): + unpacked_data = (Elf64_Phdr.s).unpack(data) + self.p_type = unpacked_data[0] + self.p_flags = unpacked_data[1] + self.p_offset = unpacked_data[2] + self.p_vaddr = unpacked_data[3] + self.p_paddr = unpacked_data[4] + self.p_filesz = unpacked_data[5] + self.p_memsz = unpacked_data[6] + self.p_align = unpacked_data[7] + + def get_packed_data(self): + values = [self.p_type, + self.p_flags, + self.p_offset, + self.p_vaddr, + self.p_paddr, + self.p_filesz, + self.p_memsz, + self.p_align + ] + + return (Elf64_Phdr.s).pack(*values) + + +class Elf64_Shdr: + s = struct.Struct('IIQQQQIIQQ') + + def __init__(self, data): + unpacked_data = (Elf64_Shdr.s).unpack(data) + self.sh_name = unpacked_data[0] + self.sh_type = unpacked_data[1] + self.sh_flags = unpacked_data[2] + self.sh_addr = unpacked_data[3] + self.sh_offset = unpacked_data[4] + self.sh_size = unpacked_data[5] + self.sh_link = unpacked_data[6] + self.sh_info = unpacked_data[7] + self.sh_addralign = unpacked_data[8] + self.sh_entsize = unpacked_data[9] + self.sh_name_str = "" + + def get_packed_data(self): + values = [self.sh_name, + self.sh_type, + self.sh_flags, + self.sh_addr, + self.sh_offset, + self.sh_size, + self.sh_link, + self.sh_info, + self.sh_addralign, + self.sh_entsize] + + return (Elf64_Shdr.s).pack(*values) + + +class SWIV_Seg: + s = struct.Struct('4sIQ') + + def __init__(self): + self.magic = SWIV_MAGIC + self.crc = 0 + self.padding = 0 + + def get_packed_data(self): + values = [self.magic, + self.crc, + self.padding + ] + + return (SWIV_Seg.s).pack(*values) + + +def is_elf64(elf_header): + return elf_header.e_ident[ELF_CLASS_INDEX] == ELFCLASS64 + + +def verify_elf_header(elf_header): + ident = elf_header.e_ident + return (ident.startswith(ELF_MAGIC) + and ident[ELF_CLASS_INDEX] in (ELFCLASS32, ELFCLASS64) + and ident[ELF_VERSION_INDEX] == EV_CURRENT) + + +def file_copy_offset(in_fp, in_off, out_fp, out_off, num_bytes): + in_fp.seek(in_off) + read_in = in_fp.read(num_bytes) + out_fp.seek(out_off) + out_fp.write(read_in) + + return num_bytes + + +def preprocess_elf_file(elf_file_name): + with open(elf_file_name, 'rb') as elf_fp: + elf_header = Elf_Ehdr_common(elf_fp.read(Elf_Ehdr_common.s.size)) + + if verify_elf_header(elf_header) is False: + raise RuntimeError("Input ELF verification failed : " + elf_file_name) + + elf_fp.seek(0) + + if is_elf64(elf_header): + elf_header = Elf64_Ehdr(elf_fp.read(Elf64_Ehdr.s.size)) + else: + elf_header = Elf32_Ehdr(elf_fp.read(Elf32_Ehdr.s.size)) + + phdr_table = [] + + phdr_size = elf_header.e_phentsize + + file_offset = elf_header.e_phoff + elf_fp.seek(file_offset) + + for _ in range(elf_header.e_phnum): + if is_elf64(elf_header): + phdr_table.append(Elf64_Phdr(elf_fp.read(phdr_size))) + else: + phdr_table.append(Elf32_Phdr(elf_fp.read(phdr_size))) + + return [elf_header, phdr_table] + + +def process_section_header(elf_file_name): + with open(elf_file_name, 'rb') as elf_fp: + elf_header = Elf_Ehdr_common(elf_fp.read(Elf_Ehdr_common.s.size)) + + if verify_elf_header(elf_header) is False: + raise RuntimeError("Input ELF verification failed : " + elf_file_name) + + elf_fp.seek(0) + + if is_elf64(elf_header): + elf_header = Elf64_Ehdr(elf_fp.read(Elf64_Ehdr.s.size)) + else: + elf_header = Elf32_Ehdr(elf_fp.read(Elf32_Ehdr.s.size)) + + shdr_table = [] + + shdr_size = elf_header.e_shentsize + elf_fp.seek(elf_header.e_shoff) + + for _ in range(elf_header.e_shnum): + if is_elf64(elf_header): + shdr_table.append(Elf64_Shdr(elf_fp.read(shdr_size))) + else: + shdr_table.append(Elf32_Shdr(elf_fp.read(shdr_size))) + + return shdr_table + + +def crc32_update_file(elf_fp, offset, size, crc): + elf_fp.seek(offset) + remaining = size + while remaining > 0: + chunk = elf_fp.read(min(1024 * 1024, remaining)) + if not chunk: + raise RuntimeError("Unexpected EOF while calculating CRC") + crc = zlib.crc32(chunk, crc) + remaining -= len(chunk) + return crc + + +def crc32_update_zeros(size, crc): + zero_chunk = b'\x00' * min(1024 * 1024, size) + remaining = size + while remaining > 0: + chunk = zero_chunk[:min(len(zero_chunk), remaining)] + crc = zlib.crc32(chunk, crc) + remaining -= len(chunk) + return crc + + +def crc32_calculate(elf_fp, phdr_table): + crc = 0 + + # SWIV validates loadable segment contents, including BSS as zeros. + for curr_phdr in phdr_table: + if curr_phdr.p_type != LOAD_TYPE: + continue + + crc = crc32_update_file(elf_fp, curr_phdr.p_offset, curr_phdr.p_filesz, crc) + + zero_size = curr_phdr.p_memsz - curr_phdr.p_filesz + if zero_size > 0: + crc = crc32_update_zeros(zero_size, crc) + + crc &= 0xffffffff + logger.info("calculated CRC: %#x", crc) + + return crc + + +def verify_swiv_seg(elf_in_fp, offset): + + elf_in_fp.seek(offset) + swiv_seg = SWIV_Seg() + unpacked_data = (SWIV_Seg.s).unpack(elf_in_fp.read(SWIV_SEG_SIZE_MIN)) + swiv_seg.magic = unpacked_data[0] + + return swiv_seg.magic == SWIV_MAGIC + + +def same_output_file(input_elf, output_elf): + try: + return os.path.exists(output_elf) and os.path.samefile(input_elf, output_elf) + except FileNotFoundError: + return False + + +def make_output_target(input_elf, output_elf): + if not same_output_file(input_elf, output_elf): + return output_elf, None + + output_dir = os.path.dirname(os.path.abspath(output_elf)) + output_base = os.path.basename(output_elf) + fd, temp_output = tempfile.mkstemp(prefix=output_base + ".", suffix=".tmp", dir=output_dir) + os.close(fd) + shutil.copymode(input_elf, temp_output) + return temp_output, temp_output + + +def swiv_generate(input_elf, output_elf): + output_target, temp_output = make_output_target(input_elf, output_elf) + + try: + with open(input_elf, "rb") as elf_in_fp, open(output_target, "wb+") as elf_out_fp: + [elf_header, phdr_table] = preprocess_elf_file(input_elf) + + num_phdrs = elf_header.e_phnum + num_shdrs = elf_header.e_shnum + phdr_size = elf_header.e_phentsize + + if num_phdrs == 0 or num_phdrs > MAX_PHDR_COUNT: + raise RuntimeError("Input ELF has invalid number of program headers.") + + if elf_header.e_shoff == 0 or num_shdrs == 0: + raise RuntimeError("Input ELF does not contain a section header table.") + + shdr_table_input = process_section_header(input_elf) + shdr_table_output = process_section_header(input_elf) + + last_phdr = phdr_table[num_phdrs - 1] + if last_phdr.p_memsz == SWIV_SEG_SIZE_MIN and verify_swiv_seg(elf_in_fp, last_phdr.p_offset): + raise RuntimeError("A SWIV segment is detected in the input ELF.") + + last_shdr = shdr_table_input[elf_header.e_shnum - 1] + if last_shdr.sh_size == SWIV_SEG_SIZE_MIN and verify_swiv_seg(elf_in_fp, last_shdr.sh_offset): + raise RuntimeError("A SWIV section is detected in the input ELF.") + + if is_elf64(elf_header): + swiv_shdr = Elf64_Shdr(b'\0' * Elf64_Shdr.s.size) + else: + swiv_shdr = Elf32_Shdr(b'\0' * Elf32_Shdr.s.size) + + swiv_shdr.sh_name = 0 + swiv_shdr.sh_type = SWIV_SECTION_TYPE + swiv_shdr.sh_flags = 0 + swiv_shdr.sh_addr = 0 + swiv_shdr.sh_size = SWIV_SEG_SIZE_MIN + swiv_shdr.sh_offset = elf_header.e_shoff + (elf_header.e_shnum + 1) * elf_header.e_shentsize + shdr_table_output.append(swiv_shdr) + elf_header.e_shnum += 1 + + if elf_header.e_phoff != 0: + for phdr in phdr_table: + file_copy_offset(elf_in_fp, phdr.p_offset, elf_out_fp, phdr.p_offset, phdr.p_filesz) + + elf_out_fp.seek(0) + elf_out_fp.write(elf_header.get_packed_data()) + + phdr_start = elf_header.e_phoff + for phdr in phdr_table: + elf_out_fp.seek(phdr_start) + elf_out_fp.write(phdr.get_packed_data()) + phdr_start += phdr_size + + for shdr in shdr_table_input[:num_shdrs]: + if shdr.sh_type == SHT_NOBITS or shdr.sh_size == 0: + continue + file_copy_offset(elf_in_fp, shdr.sh_offset, elf_out_fp, shdr.sh_offset, shdr.sh_size) + + shdr_start = elf_header.e_shoff + for shdr in shdr_table_output: + elf_out_fp.seek(shdr_start) + elf_out_fp.write(shdr.get_packed_data()) + shdr_start += elf_header.e_shentsize + + elf_out_fp.flush() + + swiv_seg = SWIV_Seg() + swiv_seg.crc = crc32_calculate(elf_out_fp, phdr_table) + elf_out_fp.seek(swiv_shdr.sh_offset) + elf_out_fp.write(swiv_seg.get_packed_data()) + + if temp_output is not None: + os.replace(temp_output, output_elf) + temp_output = None + + return 0 + finally: + if temp_output is not None and os.path.exists(temp_output): + os.unlink(temp_output) + + +def main(argv): + parser = argparse.ArgumentParser(description="Add a Software Image Validation (SWIV) section to an HTP ELF file.") + parser.add_argument("-i", "--input", dest="input_file", metavar="INPUT", required=True) + parser.add_argument("-o", "--output", dest="output_file", metavar="OUTPUT", required=True) + parser.add_argument("-v", "--verbose", action="store_true", help="show calculated CRC") + args = parser.parse_args(argv) + + logging.basicConfig( + level=logging.INFO if args.verbose else logging.WARNING, + format="%(message)s", + ) + + try: + return swiv_generate(args.input_file, args.output_file) + except RuntimeError as exc: + logger.error("ERROR: %s", exc) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h new file mode 100644 index 0000000..a5ea0d5 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h @@ -0,0 +1,199 @@ +#ifndef HTP_TENSOR_H +#define HTP_TENSOR_H + +#include +#include +#include "htp-ops.h" +#include "hex-bitmap.h" +#include "hex-common.h" +#include "hex-fastdiv.h" + +#define HTP_TENSOR_MDEV_LINE_SIZE 128 + +// Currently HMX requires 32x32 tile formats (do not change) +#define HTP_TENSOR_TILE_N_COLS 32 +#define HTP_TENSOR_TILE_N_ROWS 32 + +#define HTP_TENSOR_TILE_SIZE_F16 2048 +#define HTP_TENSOR_TILE_SIZE_Q4_0 576 +#define HTP_TENSOR_TILE_SIZE_Q4_1 640 +#define HTP_TENSOR_TILE_SIZE_Q8_0 1088 +#define HTP_TENSOR_TILE_SIZE_IQ4_NL 576 +#define HTP_TENSOR_TILE_SIZE_MXFP4 544 +#define HTP_TENSOR_TILE_SIZE_Q4_K 640 +#define HTP_TENSOR_TILE_SIZE_Q5_K 768 +#define HTP_TENSOR_TILE_SIZE_Q6_K 896 +#define HTP_TENSOR_TILE_SIZE_Q3_K 512 +#define HTP_TENSOR_TILE_SIZE_Q2_K 336 +#define HTP_TENSOR_TILE_K_Q1_0 128 +#define HTP_TENSOR_TILE_SIZE_Q1_0 576 +#define HTP_TENSOR_TILE_K_Q2_0 64 +#define HTP_TENSOR_TILE_SIZE_Q2_0 576 + +// Tiled weight tensor layouts (Q4_0, Q8_0, IQ4_NL, MXFP4, Q1_0, Q2_0, K-quants): +// See docs/backend/snapdragon/developer.md ("Tiled Weight Formats"). + +struct htp_tensor_mdev_range { + uint32_t start; + uint32_t count; +}; + +static inline bool htp_tensor_is_extended(const struct htp_tensor * t) { + return t && (t->data >> 32) != 0; +} + +static inline bool htp_tensor_is_contiguous(const struct htp_tensor * t, uint32_t type_size) { + uint32_t next_nb = type_size; + if (t->ne[0] != 1 && t->nb[0] != next_nb) { + return false; + } + next_nb *= t->ne[0]; + for (int i = 1; i < HTP_OP_MAX_DIMS; i++) { + if (t->ne[i] != 1 && t->nb[i] != next_nb) { + return false; + } + next_nb *= t->ne[i]; + } + return true; +} + +static inline bool htp_tensor_is_permuted(const struct htp_tensor * t) { + return t->nb[0] > t->nb[1] || t->nb[1] > t->nb[2] || t->nb[2] > t->nb[3]; +} + +static inline bool htp_tensor_mdev_data_aligned(const struct htp_tensor * t) { + return ((uintptr_t) t->data & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0; +} + +static inline bool htp_tensor_can_row_partition(const struct htp_tensor * t, uint32_t elem_size) { + if (!htp_tensor_mdev_data_aligned(t)) { + return false; + } + if (t->ne[0] != 1 && t->nb[0] != elem_size) { + return false; + } + if (htp_tensor_is_permuted(t)) { + return false; + } + if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + if (t->ne[2] > 1 && (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + if (t->ne[3] > 1 && (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false; + return true; +} + +static inline bool htp_tensor_mdev_rows_per_chunk(const struct htp_tensor * t, uint32_t elem_size, uint32_t row_size, uint32_t * rows_per_chunk) { + *rows_per_chunk = 0; + + if (!htp_tensor_mdev_data_aligned(t)) { + return false; + } + if (t->ne[0] != 1 && t->nb[0] != elem_size) { + return false; + } + if (htp_tensor_is_permuted(t)) { + return false; + } + if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && + (t->ne[2] <= 1 || (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0) && + (t->ne[3] <= 1 || (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) { + *rows_per_chunk = 1; + return true; + } + if (t->nb[1] == row_size && + (t->ne[2] <= 1 || t->nb[2] == t->nb[1] * t->ne[1]) && + (t->ne[3] <= 1 || t->nb[3] == t->nb[2] * t->ne[2])) { + *rows_per_chunk = (row_size > 0) ? (HTP_TENSOR_MDEV_LINE_SIZE / hex_gcd_u32(row_size, HTP_TENSOR_MDEV_LINE_SIZE)) : 1; + return true; + } + return false; +} + +static inline struct htp_tensor_mdev_range htp_tensor_mdev_partition(uint32_t total_units, uint32_t units_per_chunk, uint32_t mdev_idx, uint32_t mdev_count, const struct fastdiv_values * mdev_count_div) { + struct htp_tensor_mdev_range range = { 0, total_units }; + + if (mdev_count <= 1) { + return range; + } + + if (units_per_chunk == 0) { + range.start = (mdev_idx == 0) ? 0 : total_units; + range.count = (mdev_idx == 0) ? total_units : 0; + return range; + } + + const uint32_t total_chunks = total_units / units_per_chunk; + if (total_chunks < mdev_count) { + range.start = (mdev_idx == 0) ? 0 : total_units; + range.count = (mdev_idx == 0) ? total_units : 0; + return range; + } + + const uint32_t chunks_per_mdev = fastdiv(total_chunks + mdev_count - 1, mdev_count_div); + range.start = MIN(mdev_idx * chunks_per_mdev * units_per_chunk, total_units); + if (mdev_idx == mdev_count - 1) { + range.count = total_units - range.start; + } else { + range.count = MIN(chunks_per_mdev * units_per_chunk, total_units - range.start); + } + return range; +} + +static inline uint32_t htp_tensor_get_tile_size(int type) { + switch (type) { + case HTP_TYPE_Q4_0: return HTP_TENSOR_TILE_SIZE_Q4_0; + case HTP_TYPE_IQ4_NL: return HTP_TENSOR_TILE_SIZE_IQ4_NL; + case HTP_TYPE_Q4_1: return HTP_TENSOR_TILE_SIZE_Q4_1; + case HTP_TYPE_Q4_K: return HTP_TENSOR_TILE_SIZE_Q4_K; + case HTP_TYPE_Q8_0: return HTP_TENSOR_TILE_SIZE_Q8_0; + case HTP_TYPE_Q5_K: return HTP_TENSOR_TILE_SIZE_Q5_K; + case HTP_TYPE_Q6_K: return HTP_TENSOR_TILE_SIZE_Q6_K; + case HTP_TYPE_Q3_K: return HTP_TENSOR_TILE_SIZE_Q3_K; + case HTP_TYPE_Q2_K: return HTP_TENSOR_TILE_SIZE_Q2_K; + case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_SIZE_Q1_0; + case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_SIZE_Q2_0; + case HTP_TYPE_MXFP4: return HTP_TENSOR_TILE_SIZE_MXFP4; + case HTP_TYPE_F16: return HTP_TENSOR_TILE_SIZE_F16; + default: return 0; + } +} + +static inline uint32_t htp_tensor_get_tile_k(int type) { + switch (type) { + case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_K_Q1_0; + case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_K_Q2_0; + default: return 32; + } +} + +static inline uint32_t htp_tensor_get_tile_padded_size(int type) { + return (htp_tensor_get_tile_size(type) + 127u) & ~127u; +} + +static inline bool htp_tensor_is_tiled(const struct htp_tensor * t) { + return t && (t->flags & HTP_TENSOR_TILED) != 0; +} + +static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { + switch (type) { + case HTP_TYPE_F32: return ne00 * 4; + case HTP_TYPE_F16: + case HTP_TYPE_BF16: return ne00 * 2; + case HTP_TYPE_Q8_0: return (ne00 / 32) * 34; + case HTP_TYPE_Q4_0: + case HTP_TYPE_IQ4_NL: return (ne00 / 32) * 18; + case HTP_TYPE_Q4_1: return (ne00 / 32) * 20; + case HTP_TYPE_MXFP4: return (ne00 / 32) * 17; + case HTP_TYPE_Q2_K: return (ne00 / 256) * 84; + case HTP_TYPE_Q2_0: return (ne00 / 64) * 18; + case HTP_TYPE_Q3_K: return (ne00 / 256) * 110; + case HTP_TYPE_Q4_K: return (ne00 / 256) * 144; + case HTP_TYPE_Q5_K: return (ne00 / 256) * 176; + case HTP_TYPE_Q6_K: return (ne00 / 256) * 210; + case HTP_TYPE_Q1_0: return (ne00 / HTP_TENSOR_TILE_K_Q1_0) * 18; + case HTP_TYPE_I32: return ne00 * 4; + case HTP_TYPE_I64: return ne00 * 8; + default: return 0; + } +} + +#endif // HTP_TENSOR_H diff --git a/ggml/src/ggml-hexagon/htp/htp-vtcm.h b/ggml/src/ggml-hexagon/htp/htp-vtcm.h new file mode 100644 index 0000000..a1e9057 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/htp-vtcm.h @@ -0,0 +1,19 @@ +#ifndef HTP_VTCM_H +#define HTP_VTCM_H + +#include +#include + +static inline uint8_t *vtcm_seq_alloc(uint8_t **vtcm_ptr, size_t size) { + uint8_t *p = *vtcm_ptr; + *vtcm_ptr += size; + return p; +} + +#define VTCM_LAYOUT_ALLOC(off, field, sz) do { (L)->field = (off); (off) += (sz); } while (0) +#define VTCM_LAYOUT_ALLOC_OPTIONAL(off, field, sz, cond) do { if (cond) { VTCM_LAYOUT_ALLOC(off, field, sz); } else { (L)->field = 0; } } while (0) + +#define VTCM_LAYOUT_PTR(type, base, offset) ((type *)((uint8_t *)(base) + (offset))) +#define VTCM_LAYOUT_PTR_OPTIONAL(type, base, offset, cond) ((cond) ? VTCM_LAYOUT_PTR(type, base, offset) : NULL) + +#endif // HTP_VTCM_H diff --git a/ggml/src/ggml-hexagon/htp/htp_iface.idl b/ggml/src/ggml-hexagon/htp/htp_iface.idl index 3eb5d5a..6e3390b 100644 --- a/ggml/src/ggml-hexagon/htp/htp_iface.idl +++ b/ggml/src/ggml-hexagon/htp/htp_iface.idl @@ -6,13 +6,18 @@ #include "AEEStdDef.idl" #include "remote.idl" +struct htp_iface_pmu_conf { + uint32 events[8]; +}; + interface htp_iface : remote_handle64 { - AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 use_hmx); + AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 vtcm_size, in uint64 max_vmem, in uint32 power_mode); AEEResult stop(); - AEEResult mmap(in uint32 fd, in uint32 size, in uint32 pinned); + AEEResult mmap(in uint32 fd, in uint64 size); AEEResult munmap(in uint32 fd); - AEEResult enable_etm(); - AEEResult disable_etm(); + AEEResult profiler(in uint32 mode, in htp_iface_pmu_conf pmu); + AEEResult etm(in uint32 enable); + AEEResult hwinfo(rout uint32 n_threads, rout uint32 n_hvx, rout uint32 n_hmx, rout uint64 vtcm_size); }; #endif /* HTP_IDL */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index 82e3416..ec0de8e 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -7,6 +7,7 @@ #include #include "hvx-base.h" +#include "hvx-floor.h" #include "hex-utils.h" // @@ -16,25 +17,25 @@ #define UNUSED(x) (void)(x) #define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src0_type * restrict vsrc0 = (src0_type *) src0; \ - src1_type * restrict vsrc1 = (src1_type *) src1; \ - \ - const uint32_t epv = 128 / (elem_size); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \ - } \ - if (nloe) { \ - HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \ - vec_store((void *) &vdst[i], nloe * (elem_size), v); \ - } \ + do { \ + dst_type * vdst = (dst_type *) dst; \ + src0_type * vsrc0 = (src0_type *) src0; \ + src1_type * vsrc1 = (src1_type *) src1; \ + \ + const uint32_t epv = 128 / (elem_size); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \ + vec_store((void *) &vdst[i], nloe * (elem_size), v); \ + } \ } while(0) #if __HVX_ARCH__ < 79 @@ -56,43 +57,43 @@ #define HVX_OP_MUL_F16(a, b) hvx_vec_mul_f16_f16(a, b) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \ -static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ -static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src1 % 128 == 0); \ - hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ -static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ +#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \ +static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) dst % 128 == 0); \ + hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ +static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src0 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ + assert((uintptr_t) src1 % 128 == 0); \ + hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ +static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \ hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ +} \ DEFINE_HVX_BINARY_OP_VARIANTS(hvx_add_f32, HVX_OP_ADD_F32, float) DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f32, HVX_OP_SUB_F32, float) @@ -103,25 +104,25 @@ DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f16, HVX_OP_SUB_F16, _Float16) DEFINE_HVX_BINARY_OP_VARIANTS(hvx_mul_f16, HVX_OP_MUL_F16, _Float16) // Dispatcher logic -#define HVX_BINARY_DISPATCHER(OP_NAME) \ +#define HVX_BINARY_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128)) { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ - else OP_NAME##_aau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ - else OP_NAME##_auu(dst, src0, src1, num_elems); \ - } \ - } else { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ - else OP_NAME##_uau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ - else OP_NAME##_uuu(dst, src0, src1, num_elems); \ - } \ - } \ + if (hex_is_aligned((void *) dst, 128)) { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ + else OP_NAME##_aau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ + else OP_NAME##_auu(dst, src0, src1, num_elems); \ + } \ + } else { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ + else OP_NAME##_uau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ + else OP_NAME##_uuu(dst, src0, src1, num_elems); \ + } \ + } \ } HVX_BINARY_DISPATCHER(hvx_add_f32) @@ -164,46 +165,77 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re } } +static inline void hvx_mul_mul_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint8_t * restrict src2, const uint32_t num_elems) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + assert((unsigned long) src2 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc0 = (HVX_Vector *) src0; + HVX_Vector * restrict vsrc1 = (HVX_Vector *) src1; + HVX_Vector * restrict vsrc2 = (HVX_Vector *) src2; + + const uint32_t elem_size = sizeof(_Float16); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = num_elems / epv; + const uint32_t nloe = num_elems % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]); + vdst[i] = HVX_OP_MUL_F16(v1, vsrc2[i]); + } + + if (nloe) { + HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]); + HVX_Vector v2 = HVX_OP_MUL_F16(v1, vsrc2[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v2); + } +} + // Scalar Operations -#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const uint32_t epv = 128 / (elem_size); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - HVX_Vector v = vsrc[i]; \ - vdst[i] = scalar_op_macro(v); \ - } \ - if (nloe) { \ - HVX_Vector v = vsrc[i]; \ - v = scalar_op_macro(v); \ - vec_store((void *) &vdst[i], nloe * (elem_size), v); \ - } \ +#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / (elem_size); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + HVX_Vector v = vsrc[i]; \ + vdst[i] = scalar_op_macro(v); \ + } \ + if (nloe) { \ + HVX_Vector v = vsrc[i]; \ + v = scalar_op_macro(v); \ + vec_store((void *) &vdst[i], nloe * (elem_size), v); \ + } \ } while(0) -#define HVX_OP_ADD_SCALAR_F32(v) \ - ({ \ +#define HVX_OP_ADD_SCALAR_F32(v) \ + ({ \ const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VwVw(inf, v); \ - HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \ - Q6_V_vmux_QVV(pred_inf, inf, out); \ + HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \ + Q6_V_vmux_QVV(pred_inf, inf, out); \ }) #define HVX_OP_MUL_SCALAR_F32(v) HVX_OP_MUL_F32(v, val_vec) #define HVX_OP_SUB_SCALAR_F32(v) HVX_OP_SUB_F32(v, val_vec) -#define HVX_OP_ADD_SCALAR_F16(v) \ - ({ \ +#define HVX_OP_ADD_SCALAR_F16(v) \ + ({ \ const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VhVh(inf, v); \ - HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \ - Q6_V_vmux_QVV(pred_inf, inf, out); \ + HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \ + Q6_V_vmux_QVV(pred_inf, inf, out); \ }) #define HVX_OP_MUL_SCALAR_F16(v) HVX_OP_MUL_F16(v, val_vec) @@ -212,31 +244,31 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re // Scalar Variants // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \ +#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \ static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src % 128 == 0); \ - hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src % 128 == 0); \ + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) dst % 128 == 0); \ - hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) dst % 128 == 0); \ + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \ +} \ static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - assert((uintptr_t) src % 128 == 0); \ - hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + assert((uintptr_t) src % 128 == 0); \ + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \ - const HVX_Vector val_vec = SPLAT_MACRO(val); \ - const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ - hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ -} \ + const HVX_Vector val_vec = SPLAT_MACRO(val); \ + const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \ + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \ +} \ DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_add_scalar_f32, HVX_OP_ADD_SCALAR_F32, hvx_vec_splat_f32, float) DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f32, HVX_OP_SUB_SCALAR_F32, hvx_vec_splat_f32, float) @@ -247,17 +279,17 @@ DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f16, HVX_OP_SUB_SCALAR_F16, DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_mul_scalar_f16, HVX_OP_MUL_SCALAR_F16, hvx_vec_splat_f16, _Float16) // Dispatcher logic -#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \ +#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_aa(dst, src, val, num_elems); \ - } else if (hex_is_aligned((void *) dst, 128)) { \ - OP_NAME##_au(dst, src, val, num_elems); \ - } else if (hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_ua(dst, src, val, num_elems); \ - } else { \ - OP_NAME##_uu(dst, src, val, num_elems); \ - } \ + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_aa(dst, src, val, num_elems); \ + } else if (hex_is_aligned((void *) dst, 128)) { \ + OP_NAME##_au(dst, src, val, num_elems); \ + } else if (hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_ua(dst, src, val, num_elems); \ + } else { \ + OP_NAME##_uu(dst, src, val, num_elems); \ + } \ } HVX_BINARY_SCALAR_DISPATCHER(hvx_add_scalar_f32, float) @@ -308,14 +340,92 @@ static inline void hvx_min_scalar_f32(uint8_t * restrict dst, const uint8_t * re } } +// MAX Scalar variants + +#define HVX_OP_MAX_SCALAR(v) Q6_Vsf_vmax_VsfVsf(val_vec, v) + +static inline void hvx_max_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f32(val); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_MAX_SCALAR); +} + +static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float val, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_max_scalar_f32_aa(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_max_scalar_f32_au(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_max_scalar_f32_ua(dst, src, val, num_elems); + } else { + hvx_max_scalar_f32_uu(dst, src, val, num_elems); + } +} + +#define HVX_OP_MIN_SCALAR_F16(v) Q6_Vhf_vmin_VhfVhf(val_vec, v) + +static inline void hvx_min_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f16(val); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16); +} + +static inline void hvx_min_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f16(val); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16); +} + +static inline void hvx_min_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f16(val); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16); +} + +static inline void hvx_min_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f16(val); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16); +} + +static inline void hvx_min_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_min_scalar_f16_aa(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_min_scalar_f16_au(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_min_scalar_f16_ua(dst, src, val, num_elems); + } else { + hvx_min_scalar_f16_uu(dst, src, val, num_elems); + } +} + // CLAMP Scalar variants -#define HVX_OP_CLAMP_SCALAR(v) \ - ({ \ +#define HVX_OP_CLAMP_SCALAR(v) \ + ({ \ HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VsfVsf(v, max_vec); \ HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VsfVsf(min_vec, v); \ - HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ - Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ + HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ + Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ }) static inline void hvx_clamp_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) { @@ -358,11 +468,348 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t * } } +#define HVX_OP_CLAMP_SCALAR_F16(v) \ + ({ \ + HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VhfVhf(v, max_vec); \ + HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VhfVhf(min_vec, v); \ + HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \ + Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \ + }) + +static inline void hvx_clamp_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) { + const HVX_Vector min_vec = hvx_vec_splat_f16(min); + const HVX_Vector max_vec = hvx_vec_splat_f16(max); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16); +} + +static inline void hvx_clamp_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_clamp_scalar_f16_aa(dst, src, min, max, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_clamp_scalar_f16_au(dst, src, min, max, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_clamp_scalar_f16_ua(dst, src, min, max, num_elems); + } else { + hvx_clamp_scalar_f16_uu(dst, src, min, max, num_elems); + } +} + +#define HVX_OP_LEAKY_RELU_SCALAR(v) \ + ({ \ + HVX_VectorPred pred_neg = Q6_Q_vcmp_gt_VsfVsf(zero_vec, v); \ + HVX_Vector scaled = HVX_OP_MUL_F32(v, ns_vec); \ + Q6_V_vmux_QVV(pred_neg, scaled, v); \ + }) + +static inline void hvx_leaky_relu_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) { + const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f); + const HVX_Vector ns_vec = hvx_vec_splat_f32(ns); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_LEAKY_RELU_SCALAR); +} + +static inline void hvx_leaky_relu_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_leaky_relu_scalar_f32_aa(dst, src, ns, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_leaky_relu_scalar_f32_au(dst, src, ns, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_leaky_relu_scalar_f32_ua(dst, src, ns, num_elems); + } else { + hvx_leaky_relu_scalar_f32_uu(dst, src, ns, num_elems); + } +} + +// +// Abs +// + +static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(float); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_abs_f32(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_abs_f32(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +#define hvx_abs_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(_Float16); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_abs_f16(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = hvx_vec_abs_f16(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +static inline void hvx_abs_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_abs_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_abs_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + hvx_abs_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_abs_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) src % 128 == 0); + hvx_abs_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_abs_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_abs_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_abs_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { + if (hex_is_aligned((void *) dst, 128)) { + if (hex_is_aligned((void *) src, 128)) { + hvx_abs_f16_aa(dst, src, num_elems); + } else { + hvx_abs_f16_au(dst, src, num_elems); + } + } else { + if (hex_is_aligned((void *) src, 128)) { + hvx_abs_f16_ua(dst, src, num_elems); + } else { + hvx_abs_f16_uu(dst, src, num_elems); + } + } +} + +// +// Step +// + +static inline void hvx_step_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(float); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_step_f32(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_step_f32(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +static inline void hvx_sgn_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(float); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_sgn_f32(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_sgn_f32(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +static inline void hvx_sgn_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(_Float16); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + vdst[i] = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p))); + } + if (nloe) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p))); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +static inline void hvx_step_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(_Float16); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_step_f16(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_step_f16(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +// +// Ceil +// + +static inline void hvx_ceil_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(float); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_ceil_f32(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_ceil_f32(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +static inline void hvx_ceil_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(_Float16); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + vdst[i] = hvx_vec_f32_to_f16(hvx_vec_ceil_f32(Q6_V_lo_W(p)), hvx_vec_ceil_f32(Q6_V_hi_W(p))); + } + if (nloe) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_ceil_f32(Q6_V_lo_W(p)), hvx_vec_ceil_f32(Q6_V_hi_W(p))); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + // // Square // -#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \ +#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \ do { \ dst_type * restrict vdst = (dst_type *) dst; \ src_type * restrict vsrc = (src_type *) src; \ @@ -376,10 +823,10 @@ static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t * \ _Pragma("unroll(4)") \ for (; i < nvec; i++) { \ - vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ + vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ } \ if (nloe) { \ - HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ + HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \ vec_store((void *) &vdst[i], nloe * elem_size, v); \ } \ } while(0) @@ -420,6 +867,64 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict } } +#define hvx_sqr_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(_Float16); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +static inline void hvx_sqr_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_sqr_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_sqr_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + hvx_sqr_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_sqr_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) src % 128 == 0); + hvx_sqr_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_sqr_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_sqr_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { + if (hex_is_aligned((void *) dst, 128)) { + if (hex_is_aligned((void *) src, 128)) { + hvx_sqr_f16_aa(dst, src, num_elems); + } else { + hvx_sqr_f16_au(dst, src, num_elems); + } + } else { + if (hex_is_aligned((void *) src, 128)) { + hvx_sqr_f16_ua(dst, src, num_elems); + } else { + hvx_sqr_f16_uu(dst, src, num_elems); + } + } +} + #undef HVX_OP_ADD_F32 #undef HVX_OP_SUB_F32 #undef HVX_OP_MUL_F32 @@ -435,8 +940,13 @@ static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict #undef HVX_OP_MUL_SCALAR_F16 #undef hvx_scalar_loop_body #undef HVX_OP_MIN_SCALAR +#undef HVX_OP_MAX_SCALAR +#undef HVX_OP_MIN_SCALAR_F16 #undef HVX_OP_CLAMP_SCALAR +#undef HVX_OP_CLAMP_SCALAR_F16 +#undef HVX_OP_LEAKY_RELU_SCALAR #undef DEFINE_HVX_BINARY_OP_VARIANTS +#undef DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS #undef HVX_BINARY_DISPATCHER #undef UNUSED diff --git a/ggml/src/ggml-hexagon/htp/hvx-base.h b/ggml/src/ggml-hexagon/htp/hvx-base.h index ed6026e..e87a30d 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-base.h +++ b/ggml/src/ggml-hexagon/htp/hvx-base.h @@ -77,6 +77,12 @@ static inline int32_t hvx_vec_get_i32(HVX_Vector v) { return x; } +static inline _Float16 hvx_vec_get_f16(HVX_Vector v) { + _Float16 __attribute__((aligned(128))) x; + hvx_vec_store_a(&x, 2, v); + return x; +} + static inline HVX_Vector hvx_vec_abs_f16(HVX_Vector v) { // abs by clearing the fp16 sign bit HVX_Vector mask = Q6_Vh_vsplat_R(0x7fff); @@ -105,6 +111,31 @@ static inline HVX_Vector hvx_vec_neg_f32(HVX_Vector v) { #endif // __HVX_ARCH__ > 75 } +static inline HVX_Vector hvx_vec_step_f32(HVX_Vector v) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + HVX_VectorPred q = Q6_Q_vcmp_gt_VsfVsf(v, zero); + return Q6_V_vmux_QVV(q, one, zero); +} + +static inline HVX_Vector hvx_vec_sgn_f32(HVX_Vector v) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + const HVX_Vector sign_mask = Q6_V_vsplat_R(0x80000000); + const HVX_Vector mag = hvx_vec_abs_f32(v); + const HVX_Vector signed_one = Q6_V_vor_VV(one, Q6_V_vand_VV(v, sign_mask)); + // The ordered magnitude comparison maps NaN and both signed zeros to zero. + const HVX_VectorPred nonzero = Q6_Q_vcmp_gt_VsfVsf(mag, zero); + return Q6_V_vmux_QVV(nonzero, signed_one, zero); +} + +static inline HVX_Vector hvx_vec_step_f16(HVX_Vector v) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector one = hvx_vec_splat_f16((_Float16) 1.0f); + HVX_VectorPred q = Q6_Q_vcmp_gt_VhfVhf(v, zero); + return Q6_V_vmux_QVV(q, one, zero); +} + static inline HVX_VectorPred hvx_vec_is_nan_f16(HVX_Vector v) { const HVX_Vector vnan_exp = Q6_Vh_vsplat_R(0x7C00); const HVX_Vector vnan_frac = Q6_Vh_vsplat_R(0x7FFF); @@ -128,16 +159,7 @@ static inline HVX_Vector hvx_vec_f32_to_f16_shuff(HVX_Vector v0, HVX_Vector v1) } static inline HVX_Vector hvx_vec_f32_to_f16(HVX_Vector v0, HVX_Vector v1) { - HVX_Vector v = Q6_Vh_vdeal_Vh(hvx_vec_f32_to_f16_shuff(v0, v1)); - -#if __HVX_ARCH__ < 79 - // replace NaNs with -INF, older arches produce NaNs for (-INF + 0.0) - const HVX_Vector neg_inf = hvx_vec_splat_f16(-INFINITY); - HVX_VectorPred nan = hvx_vec_is_nan_f16(v); - v = Q6_V_vmux_QVV(nan, neg_inf, v); -#endif - - return v; + return Q6_Vh_vdeal_Vh(hvx_vec_f32_to_f16_shuff(v0, v1)); } #if __HVX_ARCH__ >= 79 @@ -164,26 +186,6 @@ static inline HVX_VectorPair hvx_vec_f16_to_f32(HVX_Vector v) { } #endif -/* Q6_Vsf_equals_Vw is only available on v73+.*/ -#if __HVX_ARCH__ < 73 -static inline HVX_Vector hvx_vec_i32_to_qf32(HVX_Vector const in) -{ - HVX_Vector const vzero = Q6_V_vzero(); - HVX_VectorPred is_zero = Q6_Q_vcmp_eq_VwVw(in, vzero); - HVX_Vector lshift = Q6_Vw_vnormamt_Vw(in); - HVX_Vector normalized = Q6_Vw_vasl_VwVw(in, lshift); - HVX_Vector vexp = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7f + 30), lshift); - HVX_Vector mant = Q6_V_vand_VV(Q6_V_vsplat_R(0xFFFFFF00), normalized); - HVX_Vector ret = Q6_V_vmux_QVV(is_zero, vzero, Q6_Vw_vadd_VwVw(mant, vexp)); - return ret; -} - -static inline HVX_Vector Q6_Vsf_equals_Vw(HVX_Vector const in) -{ - return Q6_Vsf_equals_Vqf32(hvx_vec_i32_to_qf32(in)); -} -#endif - static inline HVX_Vector hvx_vec_i16_from_hf_rnd_sat(HVX_Vector vin) { // This looks complicated. // Ideally should just be Q6_Vh_equals_Vhf(vin) @@ -256,6 +258,18 @@ static inline HVX_Vector hvx_vec_mul_f16_f16(HVX_Vector a, HVX_Vector b) return Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(a, b)); } +static inline HVX_Vector hvx_vec_add_f32_f32(HVX_Vector a, HVX_Vector b) { + return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)); +} + +static inline HVX_Vector hvx_vec_sub_f32_f32(HVX_Vector a, HVX_Vector b) { + return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a, b)); +} + +static inline HVX_Vector hvx_vec_mul_f32_f32(HVX_Vector a, HVX_Vector b) { + return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)); +} + #else static inline HVX_Vector hvx_vec_add_f16_f16(HVX_Vector a, HVX_Vector b) @@ -273,6 +287,164 @@ static inline HVX_Vector hvx_vec_mul_f16_f16(HVX_Vector a, HVX_Vector b) return Q6_Vhf_vmpy_VhfVhf(a, b); } +static inline HVX_Vector hvx_vec_add_f32_f32(HVX_Vector a, HVX_Vector b) { + return Q6_Vsf_vadd_VsfVsf(a, b); +} + +static inline HVX_Vector hvx_vec_sub_f32_f32(HVX_Vector a, HVX_Vector b) { + return Q6_Vsf_vsub_VsfVsf(a, b); +} + +static inline HVX_Vector hvx_vec_mul_f32_f32(HVX_Vector a, HVX_Vector b) { + return Q6_Vsf_vmpy_VsfVsf(a, b); +} + #endif // __HVX_ARCH__ < 79 +static inline HVX_Vector hvx_vec_load_act_tile(const uint8_t * y_q, uint32_t kt, HVX_Vector * v_act_all) { + if (kt % 4 == 0) { + *v_act_all = hvx_vmem(y_q + kt * 32); + return *v_act_all; + } else if (kt % 4 == 1) { + return Q6_V_vror_VR(*v_act_all, 32); + } else if (kt % 4 == 2) { + return Q6_V_vror_VR(*v_act_all, 64); + } else { + return Q6_V_vror_VR(*v_act_all, 96); + } +} + +// Full tile, in place. tmp: 32 vectors scratch. +// Use when result must stay in m. Slowest (spills). +static inline void hvx_transpose_32x32_w_shuff_inplace(HVX_Vector * restrict m, HVX_Vector * restrict tmp) { + for (int i = 0; i < 16; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[2*i + 1], m[2*i], -4); + tmp[2*i + 0] = Q6_V_lo_W(p); + tmp[2*i + 1] = Q6_V_hi_W(p); + } + + for (int b = 0; b < 32; b += 4) { + HVX_VectorPair p0 = Q6_W_vshuff_VVR(tmp[b + 2], tmp[b + 0], -8); + HVX_VectorPair p1 = Q6_W_vshuff_VVR(tmp[b + 3], tmp[b + 1], -8); + m[b + 0] = Q6_V_lo_W(p0); m[b + 1] = Q6_V_hi_W(p0); + m[b + 2] = Q6_V_lo_W(p1); m[b + 3] = Q6_V_hi_W(p1); + } + + for (int b = 0; b < 32; b += 8) { + for (int i = 0; i < 4; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[b + i + 4], m[b + i], -16); + tmp[b + 2*i + 0] = Q6_V_lo_W(p); + tmp[b + 2*i + 1] = Q6_V_hi_W(p); + } + } + + for (int b = 0; b < 32; b += 16) { + for (int i = 0; i < 8; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(tmp[b + i + 8], tmp[b + i], -32); + m[b + 2*i + 0] = Q6_V_lo_W(p); + m[b + 2*i + 1] = Q6_V_hi_W(p); + } + } + + for (int i = 0; i < 16; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[i + 16], m[i], -64); + tmp[2 * i + 0] = Q6_V_lo_W(p); + tmp[2 * i + 1] = Q6_V_hi_W(p); + } + + for (int i = 0; i < 32; ++i) { + m[i] = tmp[i]; + } +} + +// Rows [0, nrows) of the transpose of src[0, ncols) to dst. src is clobbered, must not alias dst. +// Use when not in VTCM or for full tiles. +static inline void hvx_transpose_32x32_w_shuff(uint8_t * restrict dst, uint32_t dst_stride, HVX_Vector * restrict src, + uint32_t nrows, uint32_t ncols) { + for (uint32_t g = 0; g < ncols; g += 8) { + HVX_Vector * s = src + g; + + HVX_VectorPair a0 = Q6_W_vshuff_VVR(s[1], s[0], -4); + HVX_VectorPair a1 = Q6_W_vshuff_VVR(s[3], s[2], -4); + HVX_VectorPair a2 = Q6_W_vshuff_VVR(s[5], s[4], -4); + HVX_VectorPair a3 = Q6_W_vshuff_VVR(s[7], s[6], -4); + + HVX_VectorPair b0 = Q6_W_vshuff_VVR(Q6_V_lo_W(a1), Q6_V_lo_W(a0), -8); + HVX_VectorPair b1 = Q6_W_vshuff_VVR(Q6_V_hi_W(a1), Q6_V_hi_W(a0), -8); + HVX_VectorPair b2 = Q6_W_vshuff_VVR(Q6_V_lo_W(a3), Q6_V_lo_W(a2), -8); + HVX_VectorPair b3 = Q6_W_vshuff_VVR(Q6_V_hi_W(a3), Q6_V_hi_W(a2), -8); + + HVX_VectorPair c0 = Q6_W_vshuff_VVR(Q6_V_lo_W(b2), Q6_V_lo_W(b0), -16); + HVX_VectorPair c1 = Q6_W_vshuff_VVR(Q6_V_hi_W(b2), Q6_V_hi_W(b0), -16); + HVX_VectorPair c2 = Q6_W_vshuff_VVR(Q6_V_lo_W(b3), Q6_V_lo_W(b1), -16); + HVX_VectorPair c3 = Q6_W_vshuff_VVR(Q6_V_hi_W(b3), Q6_V_hi_W(b1), -16); + + s[0] = Q6_V_lo_W(c0); s[1] = Q6_V_hi_W(c0); + s[2] = Q6_V_lo_W(c1); s[3] = Q6_V_hi_W(c1); + s[4] = Q6_V_lo_W(c2); s[5] = Q6_V_hi_W(c2); + s[6] = Q6_V_lo_W(c3); s[7] = Q6_V_hi_W(c3); + } + + for (uint32_t k = 0; 4 * k < nrows; k++) { + HVX_VectorPair d0 = Q6_W_vshuff_VVR(src[k + 8], src[k], -32); + HVX_VectorPair d1 = Q6_W_vshuff_VVR(src[k + 24], src[k + 16], -32); + + HVX_VectorPair e0 = Q6_W_vshuff_VVR(Q6_V_lo_W(d1), Q6_V_lo_W(d0), -64); + HVX_VectorPair e1 = Q6_W_vshuff_VVR(Q6_V_hi_W(d1), Q6_V_hi_W(d0), -64); + + uint8_t * d = dst + 4 * k * dst_stride; + *(HVX_Vector *) (d + 0 * dst_stride) = Q6_V_lo_W(e0); + *(HVX_Vector *) (d + 1 * dst_stride) = Q6_V_hi_W(e0); + *(HVX_Vector *) (d + 2 * dst_stride) = Q6_V_lo_W(e1); + *(HVX_Vector *) (d + 3 * dst_stride) = Q6_V_hi_W(e1); + } +} + +// Specialized store that will follow outstanding scatters or gathers to make sure they complete +static inline void hvx_gather_sync(void * vtcm_addr) { + asm volatile("vmem(%0+#0):scatter_release\n" + "v0 = vmem(%0+#0)\n" :: "r"(vtcm_addr) : "v0", "memory"); +} + +// Lane k = k * stride. Offsets for the _gather transposes. +static inline HVX_Vector hvx_vec_gather_offsets_w(uint32_t stride) { + int32_t offsets[32] __attribute__((aligned(128))); + for (int k = 0; k < 32; k++) { + offsets[k] = k * stride; + } + return *(HVX_Vector *) offsets; +} + +static inline HVX_Vector hvx_vec_gather_offsets_h(uint32_t stride) { + int16_t offsets[64] __attribute__((aligned(128))); + for (int k = 0; k < 64; k++) { + offsets[k] = k * stride; + } + return *(HVX_Vector *) offsets; +} + +// Rows [0, nrows) of the transpose of ncols src vectors (src_stride apart) to dst. +// src, dst in VTCM. Async: hvx_gather_sync() before use. +// Use for partial tiles (small ncols). Fewest packets. +static inline void hvx_transpose_32x32_w_gather(uint8_t * restrict dst, uint32_t dst_stride, const uint8_t * restrict src, + uint32_t src_stride, HVX_Vector offsets, uint32_t nrows, uint32_t ncols) { + const uint32_t mu = ncols * src_stride; + + #pragma unroll(4) + for (uint32_t r = 0; r < nrows; r++) { + Q6_vgather_ARMVw((HVX_Vector *) (dst + r * dst_stride), (size_t) (src + r * sizeof(int32_t)), mu, offsets); + } +} + +// 16-bit version of hvx_transpose_32x32_w_gather. +static inline void hvx_transpose_64x64_h_gather(uint8_t * restrict dst, uint32_t dst_stride, const uint8_t * restrict src, + uint32_t src_stride, HVX_Vector offsets, uint32_t nrows, uint32_t ncols) { + const uint32_t mu = ncols * src_stride; + + #pragma unroll(4) + for (uint32_t r = 0; r < nrows; r++) { + Q6_vgather_ARMVh((HVX_Vector *) (dst + r * dst_stride), (size_t) (src + r * sizeof(int16_t)), mu, offsets); + } +} + #endif /* HVX_BASE_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-copy.h b/ggml/src/ggml-hexagon/htp/hvx-copy.h index 851482e..08eb7cd 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-copy.h +++ b/ggml/src/ggml-hexagon/htp/hvx-copy.h @@ -7,7 +7,8 @@ #include "hvx-base.h" -#define hvx_splat_loop_body(dst_type, vec_store) \ +#define hvx_splat_pragma(x) _Pragma(#x) +#define hvx_splat_loop_body(dst_type, vec_store, unroll_cnt) \ do { \ dst_type * restrict vdst = (dst_type *) dst; \ \ @@ -16,7 +17,7 @@ \ uint32_t i = 0; \ \ - _Pragma("unroll(4)") \ + hvx_splat_pragma(unroll(unroll_cnt)) \ for (; i < nvec; i++) { \ vdst[i] = src; \ } \ @@ -25,31 +26,47 @@ } \ } while(0) -static inline void hvx_splat_a(uint8_t * restrict dst, HVX_Vector src, uint32_t n, uint32_t elem_size) { +static inline void hvx_splat_a(void * restrict dst, HVX_Vector src, uint32_t n, uint32_t elem_size) { assert((unsigned long) dst % 128 == 0); - hvx_splat_loop_body(HVX_Vector, hvx_vec_store_a); + hvx_splat_loop_body(HVX_Vector, hvx_vec_store_a, 4); } -static inline void hvx_splat_u(uint8_t * restrict dst, HVX_Vector src, uint32_t n, uint32_t elem_size) { - hvx_splat_loop_body(HVX_UVector, hvx_vec_store_u); +static inline void hvx_splat_u(void * restrict dst, HVX_Vector src, uint32_t n, uint32_t elem_size) { + hvx_splat_loop_body(HVX_UVector, hvx_vec_store_u, 4); } -static inline void hvx_splat_f32_a(uint8_t * restrict dst, float v, uint32_t n) { +static inline void hvx_splat_f32_a(void * restrict dst, float v, uint32_t n) { hvx_splat_a(dst, hvx_vec_splat_f32(v), n, sizeof(float)); } -static inline void hvx_splat_f32_u(uint8_t * restrict dst, float v, uint32_t n) { +static inline void hvx_splat_f32_u(void * restrict dst, float v, uint32_t n) { hvx_splat_u(dst, hvx_vec_splat_f32(v), n, sizeof(float)); } -static inline void hvx_splat_f16_a(uint8_t * restrict dst, _Float16 v, uint32_t n) { - hvx_splat_u(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16)); +static inline void hvx_splat_f16_a(void * restrict dst, _Float16 v, uint32_t n) { + hvx_splat_a(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16)); } -static inline void hvx_splat_f16_u(uint8_t * restrict dst, _Float16 v, uint32_t n) { +static inline void hvx_splat_f16_u(void * restrict dst, _Float16 v, uint32_t n) { hvx_splat_u(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16)); } +static inline void hvx_splat_u16_a(void * restrict dst, uint16_t v, uint32_t n) { + hvx_splat_a(dst, Q6_Vh_vsplat_R(v), n, sizeof(uint16_t)); +} + +static inline void hvx_splat_u16_u(void * restrict dst, uint16_t v, uint32_t n) { + hvx_splat_u(dst, Q6_Vh_vsplat_R(v), n, sizeof(uint16_t)); +} + +static inline void hvx_splat_u8_a(void * restrict dst, uint8_t v, uint32_t n) { + hvx_splat_a(dst, Q6_Vb_vsplat_R(v), n, 1); +} + +static inline void hvx_splat_u8_u(void * restrict dst, uint8_t v, uint32_t n) { + hvx_splat_u(dst, Q6_Vb_vsplat_R(v), n, 1); +} + #define hvx_copy_loop_body(dst_type, src_type, vec_store) \ do { \ dst_type * restrict vdst = (dst_type *) dst; \ @@ -89,42 +106,42 @@ static inline void hvx_copy_uu(uint8_t * restrict dst, const uint8_t * restrict hvx_copy_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); } -// copy n fp16 elements : source and destination are aligned to HVX Vector (128) +// copy n fp16 elements : destination and source are aligned to HVX Vector (128) static inline void hvx_copy_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_aa(dst, src, n, sizeof(__fp16)); } -// copy n fp16 elements : source is aligned, destination is potentially unaligned +// copy n fp16 elements : destination is aligned, source is unaligned static inline void hvx_copy_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_au(dst, src, n, sizeof(__fp16)); } -// copy n fp16 elements : source is aligned, destination is potentially unaligned +// copy n fp16 elements : destination is unaligned, source is aligned static inline void hvx_copy_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_ua(dst, src, n, sizeof(__fp16)); } -// copy n fp16 elements : source is aligned, destination is potentially unaligned +// copy n fp16 elements : destination and source are unaligned static inline void hvx_copy_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_uu(dst, src, n, sizeof(__fp16)); } -// copy n fp32 elements : source and destination are aligned to HVX Vector (128) +// copy n fp32 elements : destination and source are aligned to HVX Vector (128) static inline void hvx_copy_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_aa(dst, src, n, sizeof(float)); } -// copy n fp32 elements : source is aligned, destination is unaligned -static inline void hvx_copy_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { - hvx_copy_ua(dst, src, n, sizeof(float)); -} - -// copy n fp32 elements : source is unaligned, destination is aligned +// copy n fp32 elements : destination is aligned, source is unaligned static inline void hvx_copy_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_au(dst, src, n, sizeof(float)); } -// copy n fp32 elements : source is unaligned, destination unaligned +// copy n fp32 elements : destination is unaligned, source is aligned +static inline void hvx_copy_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_copy_ua(dst, src, n, sizeof(float)); +} + +// copy n fp32 elements : destination and source are unaligned static inline void hvx_copy_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_uu(dst, src, n, sizeof(float)); } @@ -153,26 +170,26 @@ static inline void hvx_copy_f32_uu(uint8_t * restrict dst, const uint8_t * restr } \ } while(0) -// copy/convert n fp32 elements into n fp16 elements : source is aligned, destination is aligned +// copy/convert n fp32 elements into n fp16 elements : destination and source are aligned static inline void hvx_copy_f16_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src % 128 == 0); hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); } -// copy/convert n fp32 elements into n fp16 elements : source is unaligned, destination is aligned +// copy/convert n fp32 elements into n fp16 elements : destination is aligned, source is unaligned static inline void hvx_copy_f16_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); } -// copy/convert n fp32 elements into n fp16 elements : source is aligned, destination is unaligned +// copy/convert n fp32 elements into n fp16 elements : destination is unaligned, source is aligned static inline void hvx_copy_f16_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) src % 128 == 0); hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); } -// copy/convert n fp32 elements into n fp16 elements : source is unaligned, destination is unaligned +// copy/convert n fp32 elements into n fp16 elements : destination and source are unaligned static inline void hvx_copy_f16_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); } @@ -218,28 +235,98 @@ static inline void hvx_copy_f16_f32_uu(uint8_t * restrict dst, const uint8_t * r } \ } while(0) -// copy/convert n fp16 elements into n fp32 elements : source is aligned, destination is aligned +// copy/convert n fp16 elements into n fp32 elements : destination and source are aligned static inline void hvx_copy_f32_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src % 128 == 0); hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); } -// copy/convert n fp16 elements into n fp32 elements : source is unaligned, destination is aligned +// copy/convert n fp16 elements into n fp32 elements : destination is aligned, source is unaligned static inline void hvx_copy_f32_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) dst % 128 == 0); hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); } -// copy/convert n fp16 elements into n fp32 elements : source is aligned, destination is unaligned +// copy/convert n fp16 elements into n fp32 elements : destination is unaligned, source is aligned static inline void hvx_copy_f32_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { assert((unsigned long) src % 128 == 0); hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); } -// copy/convert n fp16 elements into n fp32 elements : source is unaligned, destination is unaligned +// copy/convert n fp16 elements into n fp32 elements : destination and source are unaligned static inline void hvx_copy_f32_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); } +//// fp32 -> int32 + +#define hvx_copy_i32_f32_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(int32_t); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = Q6_Vw_equals_Vsf(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = Q6_Vw_equals_Vsf(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +// copy/convert n fp32 elements into n int32 elements : destination and source are aligned +static inline void hvx_copy_i32_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_copy_i32_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +// copy/convert n fp32 elements into n int32 elements : destination and source are unaligned +static inline void hvx_copy_i32_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_copy_i32_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +//// int32 -> fp32 + +#define hvx_copy_f32_i32_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t elem_size = sizeof(float); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = Q6_Vsf_equals_Vw(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector v = Q6_Vsf_equals_Vw(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * elem_size, v); \ + } \ + } while(0) + +// copy/convert n int32 elements into n fp32 elements : destination and source are aligned +static inline void hvx_copy_f32_i32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_copy_f32_i32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +// copy/convert n int32 elements into n fp32 elements : destination and source are unaligned +static inline void hvx_copy_f32_i32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_copy_f32_i32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + #endif // HVX_COPY_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-div.h b/ggml/src/ggml-hexagon/htp/hvx-div.h index 53ee304..bb7ab05 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-div.h +++ b/ggml/src/ggml-hexagon/htp/hvx-div.h @@ -219,64 +219,64 @@ static inline HVX_Vector hvx_vec_hybrid_div_f16(HVX_Vector vec1, HVX_Vector vec2 } while(0) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ +#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src0 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src0 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src0 % 128 == 0); \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src0 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src0 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - assert((uintptr_t) src1 % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src1 % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \ -} \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \ +} \ // Dispatcher logic -#define HVX_DIV_DISPATCHER(OP_NAME) \ +#define HVX_DIV_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128)) { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ - else OP_NAME##_aau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ - else OP_NAME##_auu(dst, src0, src1, num_elems); \ - } \ - } else { \ - if (hex_is_aligned((void *) src0, 128)) { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ - else OP_NAME##_uau(dst, src0, src1, num_elems); \ - } else { \ - if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ - else OP_NAME##_uuu(dst, src0, src1, num_elems); \ - } \ - } \ + if (hex_is_aligned((void *) dst, 128)) { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \ + else OP_NAME##_aau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \ + else OP_NAME##_auu(dst, src0, src1, num_elems); \ + } \ + } else { \ + if (hex_is_aligned((void *) src0, 128)) { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \ + else OP_NAME##_uau(dst, src0, src1, num_elems); \ + } else { \ + if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \ + else OP_NAME##_uuu(dst, src0, src1, num_elems); \ + } \ + } \ } DEFINE_HVX_DIV_OP_VARIANTS(hvx_div_f32, hvx_div_f32_loop_body) diff --git a/ggml/src/ggml-hexagon/htp/hvx-erf.h b/ggml/src/ggml-hexagon/htp/hvx-erf.h new file mode 100644 index 0000000..b17c657 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-erf.h @@ -0,0 +1,74 @@ +#ifndef HVX_ERF_H +#define HVX_ERF_H + +#include "hvx-base.h" +#include "hvx-exp.h" +#include "hvx-inverse.h" + +// Maximum error is about 1.5e-7 for the Abramowitz-Stegun approximation. +static __attribute__((noinline)) HVX_Vector hvx_vec_erf_f32(HVX_Vector x) { + const HVX_Vector zero = hvx_vec_splat_f32(0.0f); + const HVX_Vector ax = hvx_vec_abs_f32(x); + HVX_Vector t = hvx_vec_inverse_f32(hvx_vec_add_f32_f32( + hvx_vec_splat_f32(1.0f), hvx_vec_mul_f32_f32(hvx_vec_splat_f32(0.3275911f), ax))); + + HVX_Vector poly = hvx_vec_splat_f32(1.061405429f); + poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(-1.453152027f), hvx_vec_mul_f32_f32(poly, t)); + poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.421413741f), hvx_vec_mul_f32_f32(poly, t)); + poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(-0.284496736f), hvx_vec_mul_f32_f32(poly, t)); + poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(0.254829592f), hvx_vec_mul_f32_f32(poly, t)); + + const HVX_Vector exp_term = hvx_vec_exp_f32(hvx_vec_neg_f32(hvx_vec_mul_f32_f32(ax, ax))); + HVX_Vector result = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f), + hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(poly, t), exp_term)); + + const HVX_VectorPred neg = Q6_Q_vcmp_gt_VsfVsf(zero, x); + result = Q6_V_vmux_QVV(neg, hvx_vec_neg_f32(result), result); + return result; +} + +// GELU_ERF uses the erf definition. +static inline HVX_Vector hvx_vec_gelu_erf_f32(HVX_Vector x) { + const HVX_Vector scale = hvx_vec_splat_f32(0.7071067811865475f); + const HVX_Vector half = hvx_vec_splat_f32(0.5f); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + const HVX_Vector max_x = hvx_vec_splat_f32(10.0f); + const HVX_Vector min_x = hvx_vec_splat_f32(-10.0f); + const HVX_VectorPred neg_large = Q6_Q_vcmp_gt_VsfVsf(min_x, x); + const HVX_VectorPred pos_large = Q6_Q_vcmp_gt_VsfVsf(x, max_x); + HVX_Vector x_calc = Q6_V_vmux_QVV(neg_large, min_x, x); + x_calc = Q6_V_vmux_QVV(pos_large, max_x, x_calc); + const HVX_Vector erf = hvx_vec_erf_f32(hvx_vec_mul_f32_f32(x_calc, scale)); + + HVX_Vector result = hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(half, x_calc), hvx_vec_add_f32_f32(one, erf)); + + result = Q6_V_vmux_QVV(neg_large, hvx_vec_splat_f32(0.0f), result); + result = Q6_V_vmux_QVV(pos_large, x, result); + return result; +} + +static inline void hvx_gelu_erf_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(float); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_gelu_erf_f32(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_gelu_erf_f32(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +#endif /* HVX_ERF_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-exp.h b/ggml/src/ggml-hexagon/htp/hvx-exp.h index 84e4836..93ca8cf 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-exp.h +++ b/ggml/src/ggml-hexagon/htp/hvx-exp.h @@ -16,8 +16,9 @@ #define EXP_COEFF_0 (0x3F000000) // 0.5 = 1/(2!) #define EXP_LOGN2 (0x3F317218) // ln(2) = 0.6931471805 #define EXP_LOG2E (0x3FB8AA3B) // log2(e) = 1/ln(2) = 1.4426950408 +#define EXP_LOG2E_F 1.44269504f #define EXP_ONE (0x3f800000) // 1.0 -#define EXP_RANGE_R (0x42B16666) // 88.7 +#define EXP_RANGE_R (0x42B17218) // ln(FLT_MAX) approx = 88.7228 #define EXP_RANGE_L (0xC2B00000) // -88.0 (approx log(FLT_MIN)) static inline HVX_Vector hvx_vec_exp_f32(HVX_Vector in_vec) { @@ -163,7 +164,7 @@ static inline void hvx_exp_f32(uint8_t * restrict dst, const uint8_t * restrict HVX_Vector vec_out = Q6_V_vzero(); static const float kInf = INFINITY; - static const float kMaxExp = 88.7f; + static const float kMaxExp = 88.7228f; const HVX_Vector max_exp = hvx_vec_splat_f32(kMaxExp); const HVX_Vector inf = hvx_vec_splat_f32(kInf); @@ -172,7 +173,7 @@ static inline void hvx_exp_f32(uint8_t * restrict dst, const uint8_t * restrict HVX_Vector * p_vec_in1 = (HVX_Vector *) src; HVX_Vector * p_vec_out = (HVX_Vector *) dst; - #pragma unroll(4) + #pragma unroll(2) for (int i = 0; i < num_elems_whole; i += VLEN_FP32) { if (true == negate) { HVX_Vector neg_vec_in = hvx_vec_neg_f32(*p_vec_in1++); @@ -182,7 +183,7 @@ static inline void hvx_exp_f32(uint8_t * restrict dst, const uint8_t * restrict } } } else { - #pragma unroll(4) + #pragma unroll(2) for (int i = 0; i < num_elems_whole; i += VLEN_FP32) { HVX_Vector in = *(HVX_UVector *) (src + i * SIZEOF_FP32); @@ -213,4 +214,42 @@ static inline void hvx_exp_f32(uint8_t * restrict dst, const uint8_t * restrict } } +static inline HVX_Vector hvx_vec_exp2_f16(HVX_Vector x_v) { + const HVX_Vector zero_v = Q6_V_vzero(); + const HVX_Vector half_hf_v = Q6_Vh_vsplat_R(0x3800); // fp16 0.5 + + // Clamp input to prevent integer underflow in FP16-to-INT16 conversion + const HVX_Vector v_clamp_min = hvx_vec_splat_f16(-24.0f); + x_v = Q6_Vhf_vmax_VhfVhf(v_clamp_min, x_v); + + // k = round_toward_neg_inf(x); f = (float)k; frac = x - f + HVX_Vector x_minus_half = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vsub_VhfVhf(x_v, half_hf_v)); + HVX_Vector k_v = Q6_Vh_equals_Vhf(x_minus_half); // truncate to int16 + HVX_Vector f_v = Q6_Vhf_equals_Vh(k_v); // back to fp16 + + HVX_Vector x_qf16 = Q6_Vqf16_vsub_VhfVhf(x_v, f_v); // fractional part in qf16 + + // Horner: y = ((((E5*x + E4)*x + E3)*x + E2)*x + E1)*x + E0 + HVX_Vector y = Q6_Vqf16_vmpy_Vqf16Vqf16(Q6_Vh_vsplat_R(0x5082), x_qf16); // E5*x + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x157d)); // + E4 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x20ed)); // + E3 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x2b1b)); // + E2 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x33b0)); // + E1 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x398c)); // + E0 + y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); // y = y * x + y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x3c00)); // + 1.0 + + // Combine polynomial (mantissa) with integer part (exponent): result = y * 2^k + y = Q6_Vhf_equals_Vqf16(y); + HVX_Vector y_exp = Q6_Vuh_vlsr_VuhR(Q6_Vh_vasl_VhR(y, 1), 11); + y_exp = Q6_Vh_vadd_VhVh(k_v, y_exp); + HVX_VectorPred q_underflow = Q6_Q_vcmp_gt_VhVh(zero_v, y_exp); + y = Q6_Vh_vaslacc_VhVhR(y, k_v, 10); + return Q6_V_vmux_QVV(q_underflow, zero_v, y); +} + #endif /* HVX_EXP_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h b/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h new file mode 100644 index 0000000..5b18f16 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h @@ -0,0 +1,303 @@ +#ifndef HVX_FA_KERNELS_H +#define HVX_FA_KERNELS_H + +#include +#include +#include "hvx-utils.h" + +// Little inner kernels for HVX + +#if __HVX_ARCH__ < 79 +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)) +#define HVX_OP_SUB_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a, b)) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) +#else +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b) +#define HVX_OP_SUB_F32(a, b) Q6_Vsf_vsub_VsfVsf(a, b) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) +#endif + +// This is a bit of a hack because the compiler is struggling to properly inline +// the default hvx_vec_f32_to_f16 with output into the local array. +static __attribute__((unused)) __attribute__((noinline)) void hvx_vec_f32_to_f16_a(void *ptr, HVX_Vector v0, HVX_Vector v1) +{ + *(HVX_Vector *) ptr = hvx_vec_f32_to_f16(v0, v1); +} + +// Dot product of two F16 vectors, accumulating to float +static inline void hvx_dot_f16_f16_aa(float * restrict r, const void * restrict x, const void * restrict y, unsigned int n, float s) { + const HVX_Vector * restrict vx = (const HVX_Vector * restrict) x; // fp16 + const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16 + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_VectorPair rsum_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, vx[i], vy[i]); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]); + HVX_Vector x_hf = Q6_V_vand_QV(bmask, vx[i]); + + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); + } + + HVX_Vector rsum = HVX_OP_ADD_F32(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p)); + rsum = HVX_OP_MUL_F32(hvx_vec_splat_f32(s), hvx_vec_reduce_sum_f32(rsum)); + hvx_vec_store_u(r, 4, rsum); +} + +static inline HVX_Vector hvx_dot_f16_f16_aa_rx4(const void * restrict y, + const uint8_t * restrict x, + const size_t stride_x, + const size_t nvec, + const size_t nloe) { + const HVX_Vector * restrict vx0 = (const HVX_Vector * restrict) x; // fp16 + const HVX_Vector * restrict vx1 = (const HVX_Vector * restrict) (x + stride_x); // fp16 + const HVX_Vector * restrict vx2 = (const HVX_Vector * restrict) (x + stride_x * 2); // fp16 + const HVX_Vector * restrict vx3 = (const HVX_Vector * restrict) (x + stride_x * 3); // fp16 + const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16 + + HVX_VectorPair rsum0_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + HVX_VectorPair rsum1_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + HVX_VectorPair rsum2_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + HVX_VectorPair rsum3_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0)); + + uint32_t i = 0; + + for (i = 0; i < nvec; i++) { + HVX_Vector y_hf = vy[i]; + HVX_Vector x0_hf = vx0[i]; + HVX_Vector x1_hf = vx1[i]; + HVX_Vector x2_hf = vx2[i]; + HVX_Vector x3_hf = vx3[i]; + + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); + rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf); + rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf); + } + + if (nloe) { + // Load x (fp16) and zero-out unused elements + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]); + HVX_Vector x0_hf = Q6_V_vand_QV(bmask, vx0[i]); + HVX_Vector x1_hf = Q6_V_vand_QV(bmask, vx1[i]); + HVX_Vector x2_hf = Q6_V_vand_QV(bmask, vx2[i]); + HVX_Vector x3_hf = Q6_V_vand_QV(bmask, vx3[i]); + + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); + rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf); + rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf); + } + + HVX_Vector rsum0 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p)); + HVX_Vector rsum1 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p)); + HVX_Vector rsum2 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum2_p), Q6_V_hi_W(rsum2_p)); + HVX_Vector rsum3 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum3_p), Q6_V_hi_W(rsum3_p)); + + HVX_Vector_x4 rsum0123 = { .v = { rsum0, rsum1, rsum2, rsum3 } }; + return hvx_vec_reduce_sum_f32x4(rsum0123); +} + +static inline HVX_Vector hvx_dot_f16_f16_aa_rx32(const void * restrict y, + const uint8_t * restrict x, + const size_t stride_x, + const size_t n, + float s) { + + const size_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + const size_t nloe = n % VLEN_FP16; // leftover elements + + HVX_Vector sums = Q6_V_vzero(); + const size_t stride_x_4 = stride_x * 4; + for (uint32_t j = 0; j < VLEN_FP32; j += 4) { + HVX_Vector sums_x4 = hvx_dot_f16_f16_aa_rx4(y, x, stride_x, nvec, nloe); + HVX_VectorPred pred = Q6_Q_vsetq_R(j * SIZEOF_FP32); + sums = Q6_V_vmux_QVV(pred, sums, sums_x4); + x += stride_x_4; + } + + return HVX_OP_MUL_F32(hvx_vec_splat_f32(s), sums); +} + +// MAD: y (F32) += x (F16) * s (F16) +static inline void hvx_mad_f32_f16_aa(float * restrict y, const void * restrict x, const __fp16 * restrict s, uint32_t n) { + const HVX_Vector * restrict vx0 = (const HVX_Vector *) x; + + HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; + HVX_Vector * restrict vy = (HVX_Vector *) y; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_Vector S0 = hvx_vec_splat_f16(*s); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; ++i) { + vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); + } + + if (nloe) { + HVX_VectorPair xy_p = vy_p[i]; + xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); + + HVX_Vector xy = Q6_V_lo_W(xy_p); + i = 2 * i; // index for vy + + if (nloe >= VLEN_FP32) { + vy[i] = xy; + nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); + } + + if (nloe) { + hvx_vec_store_a(&vy[i], nloe * 4, xy); + } + } +} + +// MAD: y (F32) += x0 (F16) * s0 (F16) + x1 (F16) * s1 (F16) +static inline void hvx_mad_f32_f16_aa_rx2(float * restrict y, const void * restrict x0, const void * restrict x1, + const __fp16 * restrict s0, const __fp16 * restrict s1, uint32_t n) { + const HVX_Vector * restrict vx0 = (const HVX_Vector *) x0; + const HVX_Vector * restrict vx1 = (const HVX_Vector *) x1; + + HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; + HVX_Vector * restrict vy = (HVX_Vector *) y; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_Vector S0 = hvx_vec_splat_f16(*s0); + HVX_Vector S1 = hvx_vec_splat_f16(*s1); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; ++i) { + vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); + vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx1[i]), S1); + } + + if (nloe) { + HVX_VectorPair xy_p = vy_p[i]; + xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); + xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx1[i]), S1); + + HVX_Vector xy = Q6_V_lo_W(xy_p); + i = 2 * i; // index for vy + + if (nloe >= VLEN_FP32) { + vy[i] = xy; + nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); + } + + if (nloe) { + hvx_vec_store_a(&vy[i], nloe * 4, xy); + } + } +} +static inline void hvx_mad_f32_f16_aa_vec(float * restrict y, const void * restrict x, HVX_Vector S0, uint32_t n) { + const HVX_Vector * restrict vx0 = (const HVX_Vector *) x; + + HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; + HVX_Vector * restrict vy = (HVX_Vector *) y; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; ++i) { + vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); + } + + if (nloe) { + HVX_VectorPair xy_p = vy_p[i]; + xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); + + HVX_Vector xy = Q6_V_lo_W(xy_p); + i = 2 * i; // index for vy + + if (nloe >= VLEN_FP32) { + vy[i] = xy; + nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); + } + + if (nloe) { + hvx_vec_store_a(&vy[i], nloe * 4, xy); + } + } +} + +static inline void hvx_mad_f32_f16_aa_rx2_vec(float * restrict y, const void * restrict x0, const void * restrict x1, + HVX_Vector S0, HVX_Vector S1, uint32_t n) { + const HVX_Vector * restrict vx0 = (const HVX_Vector *) x0; + const HVX_Vector * restrict vx1 = (const HVX_Vector *) x1; + + HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y; + HVX_Vector * restrict vy = (HVX_Vector *) y; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; ++i) { + vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0); + vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx1[i]), S1); + } + + if (nloe) { + HVX_VectorPair xy_p = vy_p[i]; + xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0); + xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx1[i]), S1); + + HVX_Vector xy = Q6_V_lo_W(xy_p); + i = 2 * i; // index for vy + + if (nloe >= VLEN_FP32) { + vy[i] = xy; + nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p); + } + + if (nloe) { + hvx_vec_store_a(&vy[i], nloe * 4, xy); + } + } +} + +static inline void hvx_scale_vec_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t n, HVX_Vector vs) { + assert((size_t) dst % 128 == 0); + assert((size_t) src % 128 == 0); + + const HVX_Vector * restrict vsrc = (const HVX_Vector * restrict) src; + HVX_Vector * restrict vdst = (HVX_Vector * restrict) dst; + + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + + uint32_t i = 0; + #pragma unroll(4) + for (; i < nvec; ++i) { + vdst[i] = HVX_OP_MUL_F32(vsrc[i], vs); + } + if (nloe) { + hvx_vec_store_a(&vdst[i], nloe * sizeof(float), HVX_OP_MUL_F32(vsrc[i], vs)); + } +} + +#endif /* HVX_FA_KERNELS_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h b/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h new file mode 100644 index 0000000..f1f2e49 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h @@ -0,0 +1,47 @@ +#ifndef HVX_FLASH_ATTN_H +#define HVX_FLASH_ATTN_H + +#include +#include "hvx-utils.h" + +// Scalar helper to compute a single ALiBi slope. +static inline float alibi_slope(uint32_t h, uint32_t n_head_log2, float m0, float m1) { + return (h < n_head_log2) ? powf(m0, h + 1) : powf(m1, 2 * (h - n_head_log2) + 1); +} + +// Vectorized helper to compute 32 ALiBi slopes starting from (kv_head * G). +static inline HVX_Vector hvx_alibi_slopes( + uint32_t kv_head, + uint32_t G, + uint32_t n_head_log2, + float m0, + float m1 +) { + static const float ramp_32[32] __attribute__((aligned(128))) = { + 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, + 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, + 16.0f, 17.0f, 18.0f, 19.0f, 20.0f, 21.0f, 22.0f, 23.0f, + 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0f, 31.0f + }; + HVX_Vector v_ramp = hvx_vmem(ramp_32); + HVX_Vector v_h_base = hvx_vec_splat_f32((float)(kv_head * G)); + HVX_Vector v_h = hvx_vec_add_f32_f32(v_h_base, v_ramp); + + // Compute exponent_m0: h + 1 + HVX_Vector v_exp_m0 = hvx_vec_add_f32_f32(v_h, hvx_vec_splat_f32(1.0f)); + + // Compute exponent_m1: 2 * (h - n_head_log2) + 1 + HVX_Vector v_n_head_log2 = hvx_vec_splat_f32((float)n_head_log2); + HVX_Vector v_h_minus = hvx_vec_sub_f32_f32(v_h, v_n_head_log2); + HVX_Vector v_exp_m1 = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(hvx_vec_splat_f32(2.0f), v_h_minus), hvx_vec_splat_f32(1.0f)); + + // Compute powers + HVX_Vector v_pow_m0 = hvx_vec_pow_const_base_f32(m0, v_exp_m0); + HVX_Vector v_pow_m1 = hvx_vec_pow_const_base_f32(m1, v_exp_m1); + + // Select based on h < n_head_log2 + HVX_VectorPred p_cond = Q6_Q_vcmp_gt_VsfVsf(v_n_head_log2, v_h); // v_n_head_log2 > v_h <=> h < n_head_log2 + return Q6_V_vmux_QVV(p_cond, v_pow_m0, v_pow_m1); +} + +#endif /* HVX_FLASH_ATTN_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-floor.h b/ggml/src/ggml-hexagon/htp/hvx-floor.h index 6a1bfde..6757d61 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-floor.h +++ b/ggml/src/ggml-hexagon/htp/hvx-floor.h @@ -97,4 +97,12 @@ static inline HVX_Vector hvx_vec_floor_f32(HVX_Vector in_vec) { return vout; } +// ceil(x) = -floor(-x); keep +-0 as is since floor(-0) returns -1 +static inline HVX_Vector hvx_vec_ceil_f32(HVX_Vector in_vec) { + HVX_Vector vout = hvx_vec_neg_f32(hvx_vec_floor_f32(hvx_vec_neg_f32(in_vec))); + HVX_Vector vabs = Q6_V_vand_VV(in_vec, Q6_V_vsplat_R(0x7fffffff)); + HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(vabs, Q6_V_vzero()); + return Q6_V_vmux_QVV(q_zero, in_vec, vout); +} + #endif /* HVX_FLOOR_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-inverse.h b/ggml/src/ggml-hexagon/htp/hvx-inverse.h index f2054f4..256a884 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-inverse.h +++ b/ggml/src/ggml-hexagon/htp/hvx-inverse.h @@ -169,36 +169,36 @@ static inline HVX_Vector hvx_vec_inverse_f16_guard(HVX_Vector v_sf, HVX_Vector n } while(0) // Generic macro to define alignment permutations for an op -#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ +#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \ static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - assert((uintptr_t) src % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + assert((uintptr_t) src % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) dst % 128 == 0); \ - OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \ -} \ + assert((uintptr_t) dst % 128 == 0); \ + OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \ +} \ static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - assert((uintptr_t) src % 128 == 0); \ - OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \ -} \ + assert((uintptr_t) src % 128 == 0); \ + OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \ +} \ static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \ - OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \ -} \ + OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \ +} \ // Dispatcher logic -#define HVX_INV_DISPATCHER(OP_NAME) \ +#define HVX_INV_DISPATCHER(OP_NAME) \ static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { \ - if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_aa(dst, src, num_elems); \ - } else if (hex_is_aligned((void *) dst, 128)) { \ - OP_NAME##_au(dst, src, num_elems); \ - } else if (hex_is_aligned((void *) src, 128)) { \ - OP_NAME##_ua(dst, src, num_elems); \ - } else { \ - OP_NAME##_uu(dst, src, num_elems); \ - } \ + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_aa(dst, src, num_elems); \ + } else if (hex_is_aligned((void *) dst, 128)) { \ + OP_NAME##_au(dst, src, num_elems); \ + } else if (hex_is_aligned((void *) src, 128)) { \ + OP_NAME##_ua(dst, src, num_elems); \ + } else { \ + OP_NAME##_uu(dst, src, num_elems); \ + } \ } DEFINE_HVX_INV_OP_VARIANTS(hvx_inverse_f32, hvx_inverse_f32_loop_body) diff --git a/ggml/src/ggml-hexagon/htp/hvx-log.h b/ggml/src/ggml-hexagon/htp/hvx-log.h new file mode 100644 index 0000000..491041d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-log.h @@ -0,0 +1,118 @@ +#ifndef HVX_LOG_H +#define HVX_LOG_H + +#include "hvx-base.h" + +// Approximates ln(x) element-wise for float vectors. +// x must contain positive float elements. +// Uses Abramowitz & Stegun polynomial approximation 4.1.44 for ln(1+y) over [0, 1]. +static inline HVX_Vector hvx_vec_log_f32(HVX_Vector x) { + // x = m * 2^e, where m in [1, 2) + HVX_Vector biased_e = Q6_Vuw_vlsr_VuwR(x, 23); + HVX_Vector e_int = Q6_Vw_vsub_VwVw(biased_e, Q6_V_vsplat_R(127)); + HVX_Vector e_float = Q6_Vsf_equals_Vw(e_int); + + // Extract mantissa and set exponent to 127 (which represents float value in [1.0, 2.0)) + HVX_Vector mant_mask = Q6_V_vsplat_R(0x007FFFFF); + HVX_Vector exp_127 = Q6_V_vsplat_R(0x3F800000); + HVX_Vector m = Q6_V_vor_VV(Q6_V_vand_VV(x, mant_mask), exp_127); + + // y = m - 1.0f, y in [0, 1) + HVX_Vector y = hvx_vec_sub_f32_f32(m, hvx_vec_splat_f32(1.0f)); + + // Abramowitz & Stegun 4.1.44 polynomial approximation of ln(1+y) + HVX_Vector c; + HVX_Vector res; + + c = hvx_vec_splat_f32(-0.0064535442f); + res = hvx_vec_mul_f32_f32(y, c); + + c = hvx_vec_splat_f32(0.0360884937f); + res = hvx_vec_add_f32_f32(res, c); + res = hvx_vec_mul_f32_f32(y, res); + + c = hvx_vec_splat_f32(-0.0953293897f); + res = hvx_vec_add_f32_f32(res, c); + res = hvx_vec_mul_f32_f32(y, res); + + c = hvx_vec_splat_f32(0.1676540711f); + res = hvx_vec_add_f32_f32(res, c); + res = hvx_vec_mul_f32_f32(y, res); + + c = hvx_vec_splat_f32(-0.2407338084f); + res = hvx_vec_add_f32_f32(res, c); + res = hvx_vec_mul_f32_f32(y, res); + + c = hvx_vec_splat_f32(0.3317990258f); + res = hvx_vec_add_f32_f32(res, c); + res = hvx_vec_mul_f32_f32(y, res); + + c = hvx_vec_splat_f32(-0.4998741238f); + res = hvx_vec_add_f32_f32(res, c); + res = hvx_vec_mul_f32_f32(y, res); + + c = hvx_vec_splat_f32(0.9999964239f); + res = hvx_vec_add_f32_f32(res, c); + res = hvx_vec_mul_f32_f32(y, res); + + // ln(x) = e * ln(2) + ln(1+y) + HVX_Vector ln2 = hvx_vec_splat_f32(0.69314718056f); + HVX_Vector term_e = hvx_vec_mul_f32_f32(e_float, ln2); + + return hvx_vec_add_f32_f32(term_e, res); +} + +static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t elem_size = sizeof(float); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_log_f32(vsrc[i]); + } + if (nloe) { + HVX_Vector v = hvx_vec_log_f32(vsrc[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v); + } +} + +// Compute log(x) for f16 by promoting to f32, applying hvx_vec_log_f32, and narrowing back. +static inline void hvx_log_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t nvec = n / VLEN_FP16; + const uint32_t nloe = n % VLEN_FP16; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p)); + HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p)); + vdst[i] = hvx_vec_f32_to_f16(r0, r1); + } + if (nloe) { + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); + HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p)); + HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p)); + HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a((void *) &vdst[i], nloe * SIZEOF_FP16, v); + } +} + +#endif /* HVX_LOG_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h new file mode 100644 index 0000000..d748897 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h @@ -0,0 +1,575 @@ +#ifndef HVX_MM_KERNELS_FLOAT_H +#define HVX_MM_KERNELS_FLOAT_H + +#include "hvx-utils.h" +#include "htp-tensor.h" + +// Float activation copy/quantization kernels (DDR -> VTCM) + +static inline void quantize_f32_f32_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_stride, + size_t dst_stride +) { + (void) tmp_data; + const size_t src_row_size = ne0 * sizeof(float); + for (uint32_t i = 0; i < nrows; ++i) { + hex_l2fetch(src_data, src_row_size, src_stride, 2); + hvx_copy_f32_au(dst_data, src_data, ne0); + + dst_data += dst_stride; + src_data += src_stride; + } +} + +static inline void quantize_f32_f16_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_stride, + size_t dst_stride +) { + (void) tmp_data; + const size_t src_row_size = ne0 * sizeof(float); + for (uint32_t i = 0; i < nrows; ++i) { + hex_l2fetch(src_data, src_row_size, src_stride, 2); + hvx_copy_f16_f32_au(dst_data, src_data, ne0); + + dst_data += dst_stride; + src_data += src_stride; + } +} + +static inline void quantize_f16_f16_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_stride, + size_t dst_stride +) { + (void) tmp_data; + const size_t src_row_size = ne0 * sizeof(float); + for (uint32_t i = 0; i < nrows; ++i) { + hex_l2fetch(src_data, src_row_size, src_stride, 2); + hvx_copy_f16_au(dst_data, src_data, ne0); + + dst_data += dst_stride; + src_data += src_stride; + } +} + +// Float dot product kernels (HVX) + +#if __HVX_ARCH__ < 79 +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b)) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b)) +#else +#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b) +#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b) +#endif + +static inline void flat_vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors + uint32_t nloe = n % VLEN_FP32; // leftover elements + + HVX_Vector rsum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + HVX_Vector prod = HVX_OP_MUL_F32(x[i], y[i]); + rsum = HVX_OP_ADD_F32(rsum, prod); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector x_sf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector prod = HVX_OP_MUL_F32(x_sf, y_sf); + rsum = HVX_OP_ADD_F32(rsum, prod); + } + + *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void flat_vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / VLEN_FP32; + uint32_t nloe = n % VLEN_FP32; + + HVX_Vector rsum0 = Q6_V_vzero(); + HVX_Vector rsum1 = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector y_sf = y[i]; + HVX_Vector prod0 = HVX_OP_MUL_F32(x0[i], y_sf); + HVX_Vector prod1 = HVX_OP_MUL_F32(x1[i], y_sf); + rsum0 = HVX_OP_ADD_F32(rsum0, prod0); + rsum1 = HVX_OP_ADD_F32(rsum1, prod1); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector x0_sf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector x1_sf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector prod0 = HVX_OP_MUL_F32(x0_sf, y_sf); + HVX_Vector prod1 = HVX_OP_MUL_F32(x1_sf, y_sf); + rsum0 = HVX_OP_ADD_F32(rsum0, prod0); + rsum1 = HVX_OP_ADD_F32(rsum1, prod1); + } + + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void flat_vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / VLEN_FP32; + uint32_t nloe = n % VLEN_FP32; + + HVX_Vector r0_c0_sum = Q6_V_vzero(); + HVX_Vector r0_c1_sum = Q6_V_vzero(); + HVX_Vector r1_c0_sum = Q6_V_vzero(); + HVX_Vector r1_c1_sum = Q6_V_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector r0_sf = x0[i]; + HVX_Vector r1_sf = x1[i]; + HVX_Vector c0_sf = y0[i]; + HVX_Vector c1_sf = y1[i]; + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + + HVX_Vector r0_sf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector r1_sf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector c0_sf = Q6_V_vand_QV(bmask, y0[i]); + HVX_Vector c1_sf = Q6_V_vand_QV(bmask, y1[i]); + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf)); + } + + // Reduce and store results + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(s0, 8, r0_r1_c0_sum); + hvx_vec_store_u(s1, 8, r0_r1_c1_sum); +} + +static inline void flat_vec_dot_bf16_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / 64; + uint32_t rem = n % 64; + + HVX_Vector rsum0 = Q6_V_vzero(); + HVX_Vector rsum1 = Q6_V_vzero(); + + uint32_t i = 0; + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector v_bf16 = x[i]; + HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16); + HVX_Vector w0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16); + HVX_Vector w1_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p), 16); + + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y[2 * i + 0])); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w1_f32, y[2 * i + 1])); + } + + if (rem) { + HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2); + HVX_Vector v_bf16 = Q6_V_vand_QV(bmask, x[i]); + HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16); + HVX_Vector w0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16); + HVX_Vector w1_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p), 16); + + if (rem <= 32) { + HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4); + HVX_Vector y0 = Q6_V_vand_QV(a_mask, y[2 * i + 0]); + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y0)); + } else { + HVX_Vector y0 = y[2 * i + 0]; + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y0)); + + HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4); + HVX_Vector y1 = Q6_V_vand_QV(a_mask, y[2 * i + 1]); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w1_f32, y1)); + } + } + + HVX_Vector rsum = HVX_OP_ADD_F32(rsum0, rsum1); + *s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void flat_vec_dot_bf16_f32_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / 64; + uint32_t rem = n % 64; + + HVX_Vector rsum0 = Q6_V_vzero(); + HVX_Vector rsum1 = Q6_V_vzero(); + + uint32_t i = 0; + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(x0[i]); + HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(x1[i]); + HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16); + HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16); + HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16); + HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16); + + HVX_Vector y0 = y[2 * i + 0]; + HVX_Vector y1 = y[2 * i + 1]; + + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0)); + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w01_f32, y1)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w11_f32, y1)); + } + + if (rem) { + HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2); + HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x0[i])); + HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x1[i])); + HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16); + HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16); + HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16); + HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16); + + if (rem <= 32) { + HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4); + HVX_Vector y0 = Q6_V_vand_QV(a_mask, y[2 * i + 0]); + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0)); + } else { + HVX_Vector y0 = y[2 * i + 0]; + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0)); + + HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4); + HVX_Vector y1 = Q6_V_vand_QV(a_mask, y[2 * i + 1]); + rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w01_f32, y1)); + rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w11_f32, y1)); + } + } + + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void flat_vec_dot_bf16_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / 64; + uint32_t rem = n % 64; + + HVX_Vector r0_c0_sum = Q6_V_vzero(); + HVX_Vector r0_c1_sum = Q6_V_vzero(); + HVX_Vector r1_c0_sum = Q6_V_vzero(); + HVX_Vector r1_c1_sum = Q6_V_vzero(); + + uint32_t i = 0; + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(x0[i]); + HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(x1[i]); + HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16); + HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16); + HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16); + HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16); + + HVX_Vector c0_0 = y0[2 * i + 0]; + HVX_Vector c0_1 = y0[2 * i + 1]; + HVX_Vector c1_0 = y1[2 * i + 0]; + HVX_Vector c1_1 = y1[2 * i + 1]; + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0)); + + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w01_f32, c0_1)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w01_f32, c1_1)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w11_f32, c0_1)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w11_f32, c1_1)); + } + + if (rem) { + HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2); + HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x0[i])); + HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x1[i])); + HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16); + HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16); + HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16); + HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16); + + if (rem <= 32) { + HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4); + HVX_Vector c0_0 = Q6_V_vand_QV(a_mask, y0[2 * i + 0]); + HVX_Vector c1_0 = Q6_V_vand_QV(a_mask, y1[2 * i + 0]); + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0)); + } else { + HVX_Vector c0_0 = y0[2 * i + 0]; + HVX_Vector c1_0 = y1[2 * i + 0]; + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0)); + + HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4); + HVX_Vector c0_1 = Q6_V_vand_QV(a_mask, y0[2 * i + 1]); + HVX_Vector c1_1 = Q6_V_vand_QV(a_mask, y1[2 * i + 1]); + r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w01_f32, c0_1)); + r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w01_f32, c1_1)); + r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w11_f32, c0_1)); + r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w11_f32, c1_1)); + } + } + + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(s0, 8, r0_r1_c0_sum); + hvx_vec_store_u(s1, 8, r0_r1_c1_sum); +} + +#undef HVX_OP_ADD_F32 +#undef HVX_OP_MUL_F32 + +static inline void flat_vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) { + const HVX_Vector * restrict x = (const HVX_Vector *) vx; + const HVX_Vector * restrict y = (const HVX_Vector *) vy; + + uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors + uint32_t nloe = n % VLEN_FP16; // leftover elements + + HVX_VectorPair rsum_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(4) + for (i = 0; i < nvec; i++) { + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x[i], y[i]); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); + } + + HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p))); + hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum)); +} + +static inline void flat_vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + + uint32_t nvec = n / VLEN_FP16; + uint32_t nloe = n % VLEN_FP16; + + HVX_VectorPair rsum0_p = Q6_W_vzero(); + HVX_VectorPair rsum1_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector y_hf = y[i]; + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0[i], y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1[i], y_hf); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); + HVX_Vector x0_hf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector x1_hf = Q6_V_vand_QV(bmask, x1[i]); + rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); + rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); + } + + HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p))); + HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p))); + HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); + hvx_vec_store_u(s0, 8, rsum); +} + +static inline void flat_vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx0, const void * restrict vx1, + const void * restrict vy0, const void * restrict vy1) { + const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; + const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; + const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + + uint32_t nvec = n / VLEN_FP16; + uint32_t nloe = n % VLEN_FP16; + + // Row sums (sf) - 4 accumulators for 2x2 tile + HVX_VectorPair r0_c0_sum_p = Q6_W_vzero(); + HVX_VectorPair r0_c1_sum_p = Q6_W_vzero(); + HVX_VectorPair r1_c0_sum_p = Q6_W_vzero(); + HVX_VectorPair r1_c1_sum_p = Q6_W_vzero(); + + uint32_t i = 0; + + #pragma unroll(2) + for (i = 0; i < nvec; i++) { + HVX_Vector r0_hf = x0[i]; + HVX_Vector r1_hf = x1[i]; + HVX_Vector c0_hf = y0[i]; + HVX_Vector c1_hf = y1[i]; + + // Compute 4 dot products: r0xc0, r0xc1, r1xc0, r1xc1 + r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); + r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); + r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); + r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + } + + if (nloe) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + + HVX_Vector r0_hf = Q6_V_vand_QV(bmask, x0[i]); + HVX_Vector r1_hf = Q6_V_vand_QV(bmask, x1[i]); + HVX_Vector c0_hf = Q6_V_vand_QV(bmask, y0[i]); + HVX_Vector c1_hf = Q6_V_vand_QV(bmask, y1[i]); + + r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); + r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); + r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); + r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + } + + HVX_Vector r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c0_sum_p), Q6_V_hi_W(r0_c0_sum_p))); + HVX_Vector r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c1_sum_p), Q6_V_hi_W(r0_c1_sum_p))); + HVX_Vector r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c0_sum_p), Q6_V_hi_W(r1_c0_sum_p))); + HVX_Vector r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c1_sum_p), Q6_V_hi_W(r1_c1_sum_p))); + + // Reduce and store results + HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); + HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + + hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0 + hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1 +} + + + +static inline void hvx_tensor_add_f32_grid( + const struct htp_tensor * restrict dst, + const struct htp_tensor * restrict src2, + uint32_t start_row, + uint32_t end_row, + uint32_t start_col, + uint32_t end_col, + const struct fastdiv_values * div_ne11_12, + const struct fastdiv_values * div_ne11 +) { + if (start_row >= end_row || start_col >= end_col) return; + const uint32_t nb1 = dst->nb[1]; // row stride in bytes + + const uint32_t ne11 = dst->ne[1]; + const uint32_t ne12 = dst->ne[2]; + const uint32_t ne11_12 = ne11 * ne12; + + const bool is_broadcast1 = (src2->ne[1] == 1); + const bool is_broadcast2 = (src2->ne[2] == 1); + const bool is_broadcast3 = (src2->ne[3] == 1); + + for (uint32_t r = start_row; r < end_row; r++) { + float * dst_row = (float *) ((uint8_t *) dst->data + (size_t) r * nb1); + + uint32_t i13 = fastdiv(r, div_ne11_12); + uint32_t i12 = fastdiv(r - i13 * ne11_12, div_ne11); + uint32_t i11 = r - i13 * ne11_12 - i12 * ne11; + + uint32_t i23 = is_broadcast3 ? 0 : i13; + uint32_t i22 = is_broadcast2 ? 0 : i12; + uint32_t i21 = is_broadcast1 ? 0 : i11; + + const float * src2_row = (const float *) ((const uint8_t *) src2->data + + (size_t) i21 * src2->nb[1] + (size_t) i22 * src2->nb[2] + (size_t) i23 * src2->nb[3]); + + float * dst_ptr = &dst_row[start_col]; + const float * src2_ptr = &src2_row[start_col]; + int remaining = end_col - start_col; + while (remaining >= 32) { + HVX_Vector v_out = hvx_vmemu(dst_ptr); + HVX_Vector v_z = hvx_vmemu(src2_ptr); + hvx_vmemu(dst_ptr) = hvx_vec_add_f32_f32(v_out, v_z); + dst_ptr += 32; + src2_ptr += 32; + remaining -= 32; + } + if (remaining > 0) { + HVX_Vector v_out = hvx_vmemu(dst_ptr); + HVX_Vector v_z = hvx_vmemu(src2_ptr); + hvx_vec_store_u(dst_ptr, remaining * sizeof(float), hvx_vec_add_f32_f32(v_out, v_z)); + } + } +} + +#endif // HVX_MM_KERNELS_FLOAT_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h new file mode 100644 index 0000000..01c6716 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h @@ -0,0 +1,2101 @@ +// Dynamic quantizers that produce tiled activations + +// q8_1: vector 9 holds d * sum(q) of the 32 k, replicated. +// q8_1_s16 (Q2_K, sums16): one fp16 d (vector 8) for the 256 k super-block starting at x_sb, and vector 9 holds the +// integer sums of k 0..15 (low halfword) and k 16..31 (high halfword) in every word +__attribute__((always_inline)) +static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_t * restrict y_block, const bool sums16, + const float * restrict x_sb) { + assert((unsigned long) x % 128 == 0); + assert((unsigned long) y_block % 128 == 0); + + HVX_Vector * vx = (HVX_Vector *) x; + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector vmax0_sf, vmax1_sf, vmax2_sf, vmax3_sf; + if (sums16) { + const HVX_Vector * vsb = (const HVX_Vector *) x_sb; + HVX_Vector vm = hvx_vec_abs_f32(vsb[0]); + for (int i = 1; i < 8; i++) { + vm = Q6_Vsf_vmax_VsfVsf(vm, hvx_vec_abs_f32(vsb[i])); + } + vmax0_sf = vmax1_sf = vmax2_sf = vmax3_sf = hvx_vec_reduce_max_f32(vm); + } else { + vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); + vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); + vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); + vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); + } + + HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); + HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); + HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); + HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); + + HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); + HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); + HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); + HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); + vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); + + HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); + HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); + HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); + HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); + + HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); + HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); + HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); + HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); + vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); + + HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); + HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); + HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16); + + const HVX_Vector ones = Q6_Vb_vsplat_R(1); + HVX_Vector v_sums = Q6_Vw_vrmpy_VbVb(vx_i8, ones); + v_sums = Q6_Vw_vadd_VwVw(v_sums, Q6_V_vror_VR(v_sums, 4)); + v_sums = Q6_Vw_vadd_VwVw(v_sums, Q6_V_vror_VR(v_sums, 8)); + if (!sums16) { + v_sums = Q6_Vw_vadd_VwVw(v_sums, Q6_V_vror_VR(v_sums, 16)); + } + // word 8b: sum of block b (sums16: k 0..15, and word 8b + 4: k 16..31) + + const HVX_Vector v_inv127 = hvx_vec_splat_f32(1.0f / 127.0f); + HVX_Vector vd0_sf = hvx_vec_mul_f32_f32(vmax0_sf, v_inv127); + HVX_Vector vd1_sf = hvx_vec_mul_f32_f32(vmax1_sf, v_inv127); + HVX_Vector vd2_sf = hvx_vec_mul_f32_f32(vmax2_sf, v_inv127); + HVX_Vector vd3_sf = hvx_vec_mul_f32_f32(vmax3_sf, v_inv127); + + HVX_Vector v_sums_sf = Q6_Vsf_equals_Vw(v_sums); + HVX_Vector voff0_sf = hvx_vec_mul_f32_f32(vd0_sf, v_sums_sf); + HVX_Vector voff1_sf = hvx_vec_mul_f32_f32(vd1_sf, Q6_V_vror_VR(v_sums_sf, 32)); + HVX_Vector voff2_sf = hvx_vec_mul_f32_f32(vd2_sf, Q6_V_vror_VR(v_sums_sf, 64)); + HVX_Vector voff3_sf = hvx_vec_mul_f32_f32(vd3_sf, Q6_V_vror_VR(v_sums_sf, 96)); + + HVX_Vector voff01_hf = hvx_vec_f32_to_f16(voff0_sf, voff1_sf); + HVX_Vector voff23_hf = hvx_vec_f32_to_f16(voff2_sf, voff3_sf); + + int32_t __attribute__((aligned(128))) ws[32]; + if (sums16) { + // |sum of 16 int8| <= 2032 fits a halfword + *(HVX_Vector *) ws = v_sums; + } + + static const uint8_t __attribute__((aligned(128))) repl[128] = { + 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + }; + HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; + + #pragma unroll + for (int b = 0; b < 4; b++) { + HVX_Vector v_act = Q6_V_vror_VR(vx_i8, b * 32); + HVX_Vector vd_b = (b < 2) ? vd01_hf : vd23_hf; + HVX_Vector voff_b = (b < 2) ? voff01_hf : voff23_hf; + HVX_Vector r_scale = hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(vd_b, 64) : vd_b); + HVX_Vector r_offset = sums16 + ? Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF)) + : hvx_vec_repl_f16((b & 1) ? Q6_V_vror_VR(voff_b, 64) : voff_b); + + HVX_Vector r0 = Q6_V_vdelta_VV(v_act, v_repl_ctrl); + HVX_Vector r1 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 4), v_repl_ctrl); + HVX_Vector r2 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 8), v_repl_ctrl); + HVX_Vector r3 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 12), v_repl_ctrl); + HVX_Vector r4 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 16), v_repl_ctrl); + HVX_Vector r5 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 20), v_repl_ctrl); + HVX_Vector r6 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 24), v_repl_ctrl); + HVX_Vector r7 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 28), v_repl_ctrl); + + HVX_Vector * restrict dst = (HVX_Vector *) (y_block + b * 1280); + dst[0] = r0; + dst[1] = r1; + dst[2] = r2; + dst[3] = r3; + dst[4] = r4; + dst[5] = r5; + dst[6] = r6; + dst[7] = r7; + dst[8] = r_scale; + dst[9] = r_offset; + } +} + +static inline void quantize_block_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y_block) { + quantize_block_f32_q8_1_tiled_impl(x, y_block, false, NULL); +} + +static inline void quantize_block_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y_block, const float * restrict x_sb) { + quantize_block_f32_q8_1_tiled_impl(x, y_block, true, x_sb); +} + +static inline void quantize_block_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y_block) { + assert((unsigned long) x % 128 == 0); + assert((unsigned long) y_block % 128 == 0); + + HVX_Vector * vx = (HVX_Vector *) x; + HVX_Vector zero = Q6_V_vzero(); + + HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); + HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); + HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); + HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); + + HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); + HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); + HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); + HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); + + HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); + HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); + HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); + HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); + + HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); + HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); + + HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); + HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); + + HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); + HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); + HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); + HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); + + HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); + HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); + vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); + vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); + + HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); + HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); + HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16); + + HVX_VectorPair vp01 = Q6_W_vshuff_VVR(vd01_hf, vd01_hf, -64); + HVX_VectorPair vp23 = Q6_W_vshuff_VVR(vd23_hf, vd23_hf, -64); + + static const uint8_t __attribute__((aligned(128))) repl[128] = { + 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + }; + HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl; + + #pragma unroll + for (int b = 0; b < 4; b++) { + HVX_Vector v_act = Q6_V_vror_VR(vx_i8, b * 32); + HVX_Vector r_scale; + if (b == 0) { + r_scale = Q6_V_lo_W(vp01); + } else if (b == 1) { + r_scale = Q6_V_hi_W(vp01); + } else if (b == 2) { + r_scale = Q6_V_lo_W(vp23); + } else { + r_scale = Q6_V_hi_W(vp23); + } + + HVX_Vector r0 = Q6_V_vdelta_VV(v_act, v_repl_ctrl); + HVX_Vector r1 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 4), v_repl_ctrl); + HVX_Vector r2 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 8), v_repl_ctrl); + HVX_Vector r3 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 12), v_repl_ctrl); + HVX_Vector r4 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 16), v_repl_ctrl); + HVX_Vector r5 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 20), v_repl_ctrl); + HVX_Vector r6 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 24), v_repl_ctrl); + HVX_Vector r7 = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act, 28), v_repl_ctrl); + + HVX_Vector * restrict dst = (HVX_Vector *) (y_block + b * 1152); + dst[0] = r0; + dst[1] = r1; + dst[2] = r2; + dst[3] = r3; + dst[4] = r4; + dst[5] = r5; + dst[6] = r6; + dst[7] = r7; + dst[8] = r_scale; + } +} + +static void quantize_row_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) { + assert(k % 32 == 0); + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (k + qk - 1) / qk; + + for (uint32_t i = 0; i < nb; i++) { + uint8_t * restrict y_block = y + i * 4 * 1152; + quantize_block_f32_q8_0_tiled(x + i * qk, y_block); + } +} + +static void quantize_row_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) { + assert(k % 32 == 0); + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (k + qk - 1) / qk; + + for (uint32_t i = 0; i < nb; i++) { + uint8_t * restrict y_block = y + i * 4 * 1280; + quantize_block_f32_q8_1_tiled(x + i * qk, y_block); + } +} + +static void quantize_row_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y, uint32_t k) { + assert(k % 32 == 0); + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (k + qk - 1) / qk; + + for (uint32_t i = 0; i < nb; i++) { + uint8_t * restrict y_block = y + i * 4 * 1280; + quantize_block_f32_q8_1_s16_tiled(x + i * qk, y_block, x + (i & ~1u) * qk); // k % 256 == 0 for Q2_K + } +} + +static inline void quantize_half_f16_tiled_splat(HVX_Vector v_src, HVX_Vector * restrict dst) { + static const uint8_t __attribute__((aligned(128))) repl[128] = { + 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + }; + HVX_Vector v_repl_ctrl = *(const HVX_Vector *) repl; + + #pragma unroll(4) + for (int cp = 0; cp < 32; cp++) { + dst[cp] = Q6_V_vdelta_VV(cp == 0 ? v_src : Q6_V_vror_VR(v_src, cp * 4), v_repl_ctrl); + } +} + +static inline void quantize_block_f32_f16_tiled(const float * restrict x, uint8_t * restrict y_block) { + assert((unsigned long) x % 128 == 0); + assert((unsigned long) y_block % 128 == 0); + + const HVX_Vector * vx = (const HVX_Vector *) x; + HVX_Vector * restrict dst = (HVX_Vector *) y_block; + + HVX_Vector vx01_hf = hvx_vec_f32_to_f16(vx[0], vx[1]); + quantize_half_f16_tiled_splat(vx01_hf, dst); + + HVX_Vector vx23_hf = hvx_vec_f32_to_f16(vx[2], vx[3]); + quantize_half_f16_tiled_splat(vx23_hf, dst + 32); +} + +static inline void quantize_block_f16_f16_tiled(const __fp16 * restrict x, uint8_t * restrict y_block) { + assert((unsigned long) x % 128 == 0); + assert((unsigned long) y_block % 128 == 0); + + const HVX_Vector * vx = (const HVX_Vector *) x; + HVX_Vector * restrict dst = (HVX_Vector *) y_block; + + quantize_half_f16_tiled_splat(vx[0], dst); + quantize_half_f16_tiled_splat(vx[1], dst + 32); +} + +static void quantize_row_f32_f16_tiled(const float * restrict x, uint8_t * restrict y, uint32_t k) { + assert(k % 32 == 0); + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (k + qk - 1) / qk; + + for (uint32_t i = 0; i < nb; i++) { + uint8_t * restrict y_block = y + i * 4 * 2048; + quantize_block_f32_f16_tiled(x + i * qk, y_block); + } +} + +static void quantize_row_f16_f16_tiled(const __fp16 * restrict x, uint8_t * restrict y, uint32_t k) { + assert(k % 32 == 0); + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (k + qk - 1) / qk; + + for (uint32_t i = 0; i < nb; i++) { + uint8_t * restrict y_block = y + i * 4 * 2048; + quantize_block_f16_f16_tiled(x + i * qk, y_block); + } +} + +// Dot kernels & helpers that consume tiled activations + +static inline HVX_Vector hvx_vec_mul_f16_f16_to_f32_lower32(HVX_Vector v1, HVX_Vector v2) { +#if __HVX_ARCH__ >= 79 + HVX_VectorPair p = Q6_Wsf_vmpy_VhfVhf(v1, v2); + return Q6_V_lo_W(Q6_W_vshuff_VVR(Q6_V_hi_W(p), Q6_V_lo_W(p), -4)); +#else + HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(v1, v2); + HVX_Vector hi = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); + HVX_Vector lo = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); + return Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -4)); +#endif +} + +// both halves of hvx_vec_mul_f16_f16_to_f32_lower32: lo = products of lanes 0..31, hi = lanes 32..63 +static inline HVX_VectorPair hvx_vec_mul_f16_f16_to_f32_pair(HVX_Vector v1, HVX_Vector v2) { +#if __HVX_ARCH__ >= 79 + HVX_VectorPair p = Q6_Wsf_vmpy_VhfVhf(v1, v2); + return Q6_W_vshuff_VVR(Q6_V_hi_W(p), Q6_V_lo_W(p), -4); +#else + HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(v1, v2); + HVX_Vector hi = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); + HVX_Vector lo = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); + return Q6_W_vshuff_VVR(hi, lo, -4); +#endif +} + +static inline HVX_Vector unpack_and_interleave_4bit(HVX_Vector v_a, HVX_Vector v_b, HVX_Vector mask_h4) { + HVX_Vector v_W0 = Q6_V_vand_VV(v_a, mask_h4); + HVX_Vector v_W1 = Q6_Vub_vlsr_VubR(v_a, 4); + HVX_Vector v_W2 = Q6_V_vand_VV(v_b, mask_h4); + HVX_Vector v_W3 = Q6_Vub_vlsr_VubR(v_b, 4); + + HVX_VectorPair v01_pair = Q6_W_vshuff_VVR(v_W1, v_W0, -1); + HVX_VectorPair v23_pair = Q6_W_vshuff_VVR(v_W3, v_W2, -1); + HVX_VectorPair v0123_pair = Q6_W_vshuff_VVR(Q6_V_lo_W(v23_pair), Q6_V_lo_W(v01_pair), -2); + return Q6_V_lo_W(v0123_pair); +} + +static inline HVX_VectorPair unpack_and_interleave_4bit_x2(HVX_Vector v_src, HVX_Vector mask_h4) { + HVX_Vector v_lo = Q6_V_vand_VV(v_src, mask_h4); + HVX_Vector v_hi = Q6_Vub_vlsr_VubR(v_src, 4); + HVX_VectorPair v01_pair = Q6_W_vshuff_VVR(v_hi, v_lo, -1); + HVX_Vector v01_lo = Q6_V_lo_W(v01_pair); + HVX_Vector v01_hi = Q6_V_hi_W(v01_pair); + + HVX_Vector v23_lo = Q6_V_valign_VVR(v01_hi, v01_lo, 64); + HVX_Vector v_W0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v23_lo, v01_lo, -2)); + + HVX_Vector v67_lo = Q6_V_valign_VVR(v01_lo, v01_hi, 64); + HVX_Vector v_W1 = Q6_V_lo_W(Q6_W_vshuff_VVR(v67_lo, v01_hi, -2)); + + return Q6_W_vcombine_VV(v_W1, v_W0); +} + +static inline HVX_Vector unpack_q1_0_group( + HVX_Vector v_plane, + int g, + HVX_Vector v_neg_one, + HVX_Vector v_pos_one +) { + HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << g); + return Q6_V_vmux_QVV(q, v_pos_one, v_neg_one); +} + +static inline HVX_Vector accum_q1_0_32x1( + HVX_Vector v_plane, + const HVX_Vector * restrict v_act, + HVX_Vector v_neg_one, + HVX_Vector v_pos_one +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one); + HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one); + v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi); +} + +static inline HVX_Vector accum_4bit_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act, + HVX_Vector i8 +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_4bit_x2(vptr[i], mask_h4); + HVX_Vector v_W0 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(v_W_pair), i8); + HVX_Vector v_W1 = Q6_Vb_vsub_VbVb(Q6_V_hi_W(v_W_pair), i8); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W0, v_act[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W1, v_act[i * 2 + 1]); + } + + return Q6_Vw_vadd_VwVw(v_sum0, v_sum1); +} + +static inline HVX_Vector accum_4bit_32x1_lut( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act, + HVX_Vector mask_h4, + HVX_Vector lut +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_4bit_x2(vptr[i], mask_h4); + HVX_Vector v_W0 = Q6_Vb_vlut32_VbVbI(Q6_V_lo_W(v_W_pair), lut, 0); + HVX_Vector v_W1 = Q6_Vb_vlut32_VbVbI(Q6_V_hi_W(v_W_pair), lut, 0); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W0, v_act[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W1, v_act[i * 2 + 1]); + } + + return Q6_Vw_vadd_VwVw(v_sum0, v_sum1); +} + +static inline HVX_VectorPair accum_q1_0_32x2( + HVX_Vector v_plane, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector v_neg_one, + HVX_Vector v_pos_one +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one); + HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + return Q6_W_vcombine_VV( + Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi), + Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi)); +} + +static inline HVX_VectorPair accum_4bit_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector i8 +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_4bit_x2(vptr[i], mask_h4); + HVX_Vector v_W0 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(v_W_pair), i8); + HVX_Vector v_W1 = Q6_Vb_vsub_VbVb(Q6_V_hi_W(v_W_pair), i8); + + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W0, v_act0[i * 2 + 0]); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W1, v_act0[i * 2 + 1]); + + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W0, v_act1[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W1, v_act1[i * 2 + 1]); + } + + return Q6_W_vcombine_VV(v_sum1, v_sum0); +} + +static inline HVX_VectorPair accum_4bit_32x2_lut( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector mask_h4, + HVX_Vector lut +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_4bit_x2(vptr[i], mask_h4); + HVX_Vector v_W0 = Q6_Vb_vlut32_VbVbI(Q6_V_lo_W(v_W_pair), lut, 0); + HVX_Vector v_W1 = Q6_Vb_vlut32_VbVbI(Q6_V_hi_W(v_W_pair), lut, 0); + + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W0, v_act0[i * 2 + 0]); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W1, v_act0[i * 2 + 1]); + + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W0, v_act1[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W1, v_act1[i * 2 + 1]); + } + + return Q6_W_vcombine_VV(v_sum1, v_sum0); +} + +static inline HVX_Vector accum_q8_0_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act +) { + HVX_Vector v_sum = Q6_V_vzero(); + #pragma unroll + for (int g = 0; g < 8; g++) { + HVX_Vector v_rot = Q6_V_vror_VR(vptr[g], 64); + HVX_Vector v_W = Q6_V_lo_W(Q6_W_vshuff_VVR(v_rot, vptr[g], -2)); + v_sum = Q6_Vw_vrmpyacc_VwVbVb(v_sum, v_W, v_act[g]); + } + return v_sum; +} + +static inline HVX_VectorPair accum_q8_0_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1 +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + #pragma unroll + for (int g = 0; g < 8; g++) { + HVX_Vector v_rot = Q6_V_vror_VR(vptr[g], 64); + HVX_Vector v_W = Q6_V_lo_W(Q6_W_vshuff_VVR(v_rot, vptr[g], -2)); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W, v_act0[g]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W, v_act1[g]); + } + return Q6_W_vcombine_VV(v_sum1, v_sum0); +} + +// Q5_K: OR 0x10 into every lane of v whose flag j is set in the plane (see HTP_TENSOR_TILE_SIZE_Q5_K) +static inline HVX_Vector hvx_q5k_or_hibit(HVX_Vector v, HVX_Vector v_plane, int j) { + HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << j); + return Q6_V_vandor_VQR(v, q, 0x10101010); +} + +// 5-bit variant: the high bit comes from the plane, see hvx_q5k_or_hibit +static inline HVX_VectorPair unpack_and_interleave_5bit_x2(HVX_Vector v_src, HVX_Vector v_plane, int i, HVX_Vector mask_h4) { + HVX_Vector v_lo = hvx_q5k_or_hibit(Q6_V_vand_VV(v_src, mask_h4), v_plane, 2 * i); + HVX_Vector v_hi = hvx_q5k_or_hibit(Q6_Vub_vlsr_VubR(v_src, 4), v_plane, 2 * i + 1); + HVX_VectorPair v01_pair = Q6_W_vshuff_VVR(v_hi, v_lo, -1); + HVX_Vector v01_lo = Q6_V_lo_W(v01_pair); + HVX_Vector v01_hi = Q6_V_hi_W(v01_pair); + + HVX_Vector v23_lo = Q6_V_valign_VVR(v01_hi, v01_lo, 64); + HVX_Vector v_W0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v23_lo, v01_lo, -2)); + + HVX_Vector v67_lo = Q6_V_valign_VVR(v01_lo, v01_hi, 64); + HVX_Vector v_W1 = Q6_V_lo_W(Q6_W_vshuff_VVR(v67_lo, v01_hi, -2)); + + return Q6_W_vcombine_VV(v_W1, v_W0); +} + +static inline HVX_Vector accum_5bit_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act, + HVX_Vector i8 +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector v_plane = vptr[5]; + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_5bit_x2(vptr[i], v_plane, i, mask_h4); + HVX_Vector v_W0 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(v_W_pair), i8); + HVX_Vector v_W1 = Q6_Vb_vsub_VbVb(Q6_V_hi_W(v_W_pair), i8); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W0, v_act[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W1, v_act[i * 2 + 1]); + } + + return Q6_Vw_vadd_VwVw(v_sum0, v_sum1); +} + +static inline HVX_VectorPair accum_5bit_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector i8 +) { + HVX_Vector v_sum0 = Q6_V_vzero(); + HVX_Vector v_sum1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector v_plane = vptr[5]; + + #pragma unroll + for (int i = 0; i < 4; i++) { + HVX_VectorPair v_W_pair = unpack_and_interleave_5bit_x2(vptr[i], v_plane, i, mask_h4); + HVX_Vector v_W0 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(v_W_pair), i8); + HVX_Vector v_W1 = Q6_Vb_vsub_VbVb(Q6_V_hi_W(v_W_pair), i8); + + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W0, v_act0[i * 2 + 0]); + v_sum0 = Q6_Vw_vrmpyacc_VwVbVb(v_sum0, v_W1, v_act0[i * 2 + 1]); + + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W0, v_act1[i * 2 + 0]); + v_sum1 = Q6_Vw_vrmpyacc_VwVbVb(v_sum1, v_W1, v_act1[i * 2 + 1]); + } + + return Q6_W_vcombine_VV(v_sum1, v_sum0); +} + +// Q6_K weights are stored unsigned (0..63), see HTP_TENSOR_TILE_SIZE_Q6_K. Unpack k-group g of a tile to signed bytes (q - 32) +static inline HVX_Vector unpack_q6_k_group(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_0f, HVX_Vector mask_03, HVX_Vector i32) { + HVX_Vector v_lo = (g & 1) ? Q6_Vub_vlsr_VubR(vptr[g >> 1], 4) : Q6_V_vand_VV(vptr[g >> 1], mask_0f); + HVX_Vector v_hi = (g & 3) ? Q6_Vub_vlsr_VubR(vptr[4 + (g >> 2)], 2 * (g & 3)) : vptr[4 + (g >> 2)]; + HVX_Vector v_q = Q6_V_vor_VV(v_lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(v_hi, mask_03), 4)); + return Q6_Vb_vsub_VbVb(v_q, i32); +} + +// k 0..15 and k 16..31 of a Q6_K tile have different scales: lo half of the pair sums k 0..15, hi half sums k 16..31 +static inline HVX_VectorPair accum_q6_k_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act, + HVX_Vector i32 +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32); + HVX_Vector v_W_hi = unpack_q6_k_group(vptr, g + 4, mask_0f, mask_03, i32); + v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_W_vcombine_VV(v_sum_hi, v_sum_lo); +} + +static inline void accum_q6_k_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector i32, + HVX_VectorPair * v_sums0, + HVX_VectorPair * v_sums1 +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32); + HVX_Vector v_W_hi = unpack_q6_k_group(vptr, g + 4, mask_0f, mask_03, i32); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + *v_sums0 = Q6_W_vcombine_VV(v_sum0_hi, v_sum0_lo); + *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo); +} + +// scale the two half sums with the per-row tile scales (v_scale_w = vptr[6]) and the activation scale +static inline HVX_Vector scale_q6_k_32x1(HVX_VectorPair v_sums, HVX_Vector v_scale_w, HVX_Vector v_scale_a) { + HVX_VectorPair v_scale = hvx_vec_mul_f16_f16_to_f32_pair(v_scale_w, v_scale_a); + HVX_Vector v_lo = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(v_sums)), Q6_V_lo_W(v_scale)); + HVX_Vector v_hi = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(v_sums)), Q6_V_hi_W(v_scale)); + return hvx_vec_add_f32_f32(v_lo, v_hi); +} + +// Q3_K / Q2_K: low 2 bits of k-group g, see HTP_TENSOR_TILE_SIZE_Q3_K +static inline HVX_Vector unpack_q3_k_low2(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_03) { + HVX_Vector v = vptr[g >> 2]; + if ((g & 3) == 3) { + return Q6_Vub_vlsr_VubR(v, 6); + } + if (g & 3) { + v = Q6_Vub_vlsr_VubR(v, 2 * (g & 3)); + } + return Q6_V_vand_VV(v, mask_03); +} + +// Q3_K k-group g as signed bytes: low2 | 0xFC (= low2 - 4) where bit g of vector 2 is set +static inline HVX_Vector unpack_q3_k_group(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_03) { + HVX_VectorPred q_neg = Q6_Q_vand_VR(vptr[2], 0x01010101u << g); + return Q6_V_vandor_VQR(unpack_q3_k_low2(vptr, g, mask_03), q_neg, 0xFCFCFCFC); +} + +// same half split as accum_q6_k_32x1 +static inline HVX_VectorPair accum_q3_k_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_group(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_group(vptr, g + 4, mask_03); + v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_W_vcombine_VV(v_sum_hi, v_sum_lo); +} + +static inline void accum_q3_k_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_VectorPair * v_sums0, + HVX_VectorPair * v_sums1 +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_group(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_group(vptr, g + 4, mask_03); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + *v_sums0 = Q6_W_vcombine_VV(v_sum0_hi, v_sum0_lo); + *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo); +} + +// Q2_K: the Q3_K dot products without the -4 flags, scales and mins are applied in q2_k_int_accum +static inline HVX_VectorPair accum_q2_k_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03); + v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_W_vcombine_VV(v_sum_hi, v_sum_lo); +} + +static inline void accum_q2_k_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_VectorPair * v_sums0, + HVX_VectorPair * v_sums1 +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + *v_sums0 = Q6_W_vcombine_VV(v_sum0_hi, v_sum0_lo); + *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo); +} + +// Q2_0: unsigned codes 0..3; the -1 offset is applied via the Q8_1 block sum (v_act[9]) +static inline HVX_Vector accum_q2_0_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03); + v_sum_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi); +} + +static inline void accum_q2_0_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector * v_sum0, + HVX_Vector * v_sum1 +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + *v_sum0 = Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi); + *v_sum1 = Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi); +} + +// Q2_K: fp16 d (*v_d) and -dmin (*v_m) of the 32 rows of a super-block (8 tiles, VTCM stride 384), each in both vector halves +static inline void q2_k_load_dmin(const uint8_t * restrict sb, HVX_Vector * restrict v_d, HVX_Vector * restrict v_m) { + HVX_Vector v_dd = Q6_V_vzero(); // lanes 0..31: d, lanes 32..63: -dmin + + #pragma unroll + for (int j = 0; j < 8; j++) { + HVX_Vector v = hvx_vmem(sb + j * 384 + 256); // the slice is at bytes 64..79 + v_dd = Q6_V_vmux_QVV(Q6_Q_vsetq_R(16 * j), v_dd, Q6_V_vror_VR(v, (64 - 16 * j) & 127)); + } + + HVX_VectorPred q_lo = Q6_Q_vsetq_R(64); + HVX_Vector v_r = Q6_V_vror_VR(v_dd, 64); + *v_d = Q6_V_vmux_QVV(q_lo, v_dd, v_r); + *v_m = Q6_V_vmux_QVV(q_lo, v_r, v_dd); +} + +// Q2_K: fp16 D = d * scale and M = -dmin * min of a tile from its scale bytes, in the lanes of the Q6_K scales +static inline void q2_k_unpack_scales(const uint8_t * restrict tile, HVX_Vector v_d, HVX_Vector v_m, + HVX_Vector * restrict v_D, HVX_Vector * restrict v_M) { + HVX_Vector v_sc = hvx_vmem(tile + 256); + HVX_Vector v_l = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_V_vand_VV(v_sc, Q6_Vb_vsplat_R(0x0F)))); + HVX_Vector v_mn = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_Vub_vlsr_VubR(v_sc, 4))); + // qf32 product: rounds like GGML_FP32_TO_FP16(d * scale) (qf16 is off by 1 ulp for many values on v73) + *v_D = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_l), v_d)); + *v_M = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_mn), v_m)); +} + +// Q2_K scale and min nibbles of a tile as halfword pairs: word r = sub-block 1 << 16 | sub-block 0 of row r +static inline void q2_k_scale_pairs(const uint8_t * restrict tile, HVX_Vector * restrict v_sc2, HVX_Vector * restrict v_mn2) { + HVX_Vector v_h = Q6_Vh_vshuff_Vh(Q6_V_lo_W(Q6_Wuh_vunpack_Vub(hvx_vmem(tile + 256)))); + *v_sc2 = Q6_V_vand_VV(v_h, Q6_Vh_vsplat_R(0x000F)); + *v_mn2 = Q6_Vuh_vlsr_VuhR(v_h, 4); +} + +// Q2_K with q8_1_s16 activations, one tile: isum += scale * (q . a), msum += min * sum(a), per row, in int32. +// |sum of 16 q * a| <= 3 * 128 * 16 fits a halfword, so both sub-blocks go through one vdmpy (no saturation) +static inline void q2_k_int_accum(HVX_VectorPair v_sums, HVX_Vector v_sc2, HVX_Vector v_mn2, HVX_Vector v_bsum, + HVX_Vector * restrict v_isum, HVX_Vector * restrict v_msum) { + HVX_Vector v_s2 = Q6_Vh_vshuffe_VhVh(Q6_V_hi_W(v_sums), Q6_V_lo_W(v_sums)); + *v_isum = Q6_Vw_vadd_VwVw(*v_isum, Q6_Vw_vdmpy_VhVh_sat(v_s2, v_sc2)); + *v_msum = Q6_Vw_vadd_VwVw(*v_msum, Q6_Vw_vdmpy_VhVh_sat(v_mn2, v_bsum)); +} + +// Q2_K super-block result: d_a * (d * isum - dmin * msum), with v_d / v_m the fp16 d / -dmin of q2_k_load_dmin +static inline HVX_Vector q2_k_int_scale(HVX_Vector v_isum, HVX_Vector v_msum, HVX_Vector v_d, HVX_Vector v_m, HVX_Vector v_act8) { + HVX_Vector v_d32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_d)); + HVX_Vector v_m32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_m)); + HVX_Vector v_da32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_act8)); + HVX_Vector v_t = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(v_d32, Q6_Vsf_equals_Vw(v_isum)), + hvx_vec_mul_f32_f32(v_m32, Q6_Vsf_equals_Vw(v_msum))); + return hvx_vec_mul_f32_f32(v_t, v_da32); +} + +static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_qf = Q6_V_vzero(); + HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + + const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0)); + HVX_Vector v_tile_qf = Q6_V_vzero(); + + #pragma unroll + for (int quarter = 0; quarter < 4; quarter++) { + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (4 * kt + quarter) * HTP_MM_ACT_TILE_SIZE_Q8_0); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q1_0_32x1(vptr[quarter], v_act, v_neg_one, v_pos_one)); + HVX_Vector v_da_sf = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8])); + HVX_Vector v_q_qf = Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf); + v_tile_qf = (quarter == 0) ? v_q_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_q_qf); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_qf = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf)); + } + + HVX_Vector v_sum_float = Q6_Vsf_equals_Vqf32(v_sum_qf); + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_qf_c0 = Q6_V_vzero(); + HVX_Vector v_sum_qf_c1 = Q6_V_vzero(); + HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + + const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0)); + HVX_Vector v_tile_qf_c0 = Q6_V_vzero(); + HVX_Vector v_tile_qf_c1 = Q6_V_vzero(); + + for (int quarter = 0; quarter < 4; quarter++) { + const uint32_t act_tile = 4 * kt + quarter; + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0); + HVX_VectorPair v_sums = accum_q1_0_32x2(vptr[quarter], v_act0, v_act1, v_neg_one, v_pos_one); + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(Q6_V_lo_W(v_sums)); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(Q6_V_hi_W(v_sums)); + HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8])); + HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8])); + + v_tile_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c0, v_da_sf_c0)); + v_tile_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, Q6_Vqf32_vmpy_VsfVsf(v_sum_sf_c1, v_da_sf_c1)); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_qf_c0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c0, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf)); + v_sum_qf_c1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_sum_qf_c1, Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf)); + } + + HVX_Vector v_sum_float_c0 = Q6_Vsf_equals_Vqf32(v_sum_qf_c0); + HVX_Vector v_sum_float_c1 = Q6_Vsf_equals_Vqf32(v_sum_qf_c1); + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector i8 = Q6_Vb_vsplat_R(8); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152); + + HVX_Vector v_sum = accum_4bit_32x1(vptr, v_act, i8); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); + + HVX_Vector v_scale_w = vptr[4]; + HVX_Vector v_scale_a = v_act[8]; + HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a); + HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); + + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q4_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + HVX_Vector i8 = Q6_Vb_vsplat_R(8); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); + + HVX_VectorPair v_sums = accum_4bit_32x2(vptr, v_act0, v_act1, i8); + HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); + HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); + + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); + + HVX_Vector v_scale_w = vptr[4]; + HVX_Vector v_scale_a_c0 = v_act0[8]; + HVX_Vector v_scale_a_c1 = v_act1[8]; + + HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c0); + HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c1); + + HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); + HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_q4_1_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280); + + HVX_Vector v_sum = accum_4bit_32x1(vptr, v_act, Q6_V_vzero()); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); + + HVX_Vector v_scale_offset = vptr[4]; + HVX_VectorPair p_deal = Q6_W_vdeal_VVR(v_scale_offset, v_scale_offset, -2); + HVX_Vector v_scale = Q6_V_lo_W(p_deal); + HVX_Vector v_offset = Q6_V_hi_W(p_deal); + + HVX_Vector v_scale_a = v_act[8]; + HVX_Vector v_sum_a = v_act[9]; + + HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale, v_scale_a); + HVX_Vector v_offset_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset, v_sum_a); + + HVX_Vector v_scaled_dot = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); + HVX_Vector v_sum_scaled = hvx_vec_add_f32_f32(v_scaled_dot, v_offset_comb); + + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q4_1_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280); + + HVX_VectorPair v_sums = accum_4bit_32x2(vptr, v_act0, v_act1, Q6_V_vzero()); + HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); + HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); + + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); + + HVX_Vector v_scale_offset = vptr[4]; + HVX_VectorPair p_deal = Q6_W_vdeal_VVR(v_scale_offset, v_scale_offset, -2); + HVX_Vector v_scale = Q6_V_lo_W(p_deal); + HVX_Vector v_offset = Q6_V_hi_W(p_deal); + + HVX_Vector v_scale_a_c0 = v_act0[8]; + HVX_Vector v_sum_a_c0 = v_act0[9]; + HVX_Vector v_scale_a_c1 = v_act1[8]; + HVX_Vector v_sum_a_c1 = v_act1[9]; + + HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale, v_scale_a_c0); + HVX_Vector v_offset_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset, v_sum_a_c0); + HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale, v_scale_a_c1); + HVX_Vector v_offset_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset, v_sum_a_c1); + + HVX_Vector v_scaled_dot_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); + HVX_Vector v_sum_scaled_c0 = hvx_vec_add_f32_f32(v_scaled_dot_c0, v_offset_comb_c0); + + HVX_Vector v_scaled_dot_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); + HVX_Vector v_sum_scaled_c1 = hvx_vec_add_f32_f32(v_scaled_dot_c1, v_offset_comb_c1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_q8_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 1152); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152); + + HVX_Vector v_sum = accum_q8_0_32x1(vptr, v_act); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); + + HVX_Vector v_scale_w = vptr[8]; + HVX_Vector v_scale_a = v_act[8]; + HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a); + HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); + + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 1152); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); + + HVX_VectorPair v_sums = accum_q8_0_32x2(vptr, v_act0, v_act1); + HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); + HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); + + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); + + HVX_Vector v_scale_w = vptr[8]; + HVX_Vector v_scale_a_c0 = v_act0[8]; + HVX_Vector v_scale_a_c1 = v_act1[8]; + + HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c0); + HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c1); + + HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); + HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_q5_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 768); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280); + + HVX_Vector v_sum = accum_5bit_32x1(vptr, v_act, Q6_V_vzero()); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); + + HVX_Vector v_scale_offset = vptr[4]; + HVX_VectorPair p_deal = Q6_W_vdeal_VVR(v_scale_offset, v_scale_offset, -2); + HVX_Vector v_scale = Q6_V_lo_W(p_deal); + HVX_Vector v_offset = Q6_V_hi_W(p_deal); + + HVX_Vector v_scale_a = v_act[8]; + HVX_Vector v_sum_a = v_act[9]; + + HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale, v_scale_a); + HVX_Vector v_offset_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset, v_sum_a); + + HVX_Vector v_scaled_dot = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); + HVX_Vector v_sum_scaled = hvx_vec_add_f32_f32(v_scaled_dot, v_offset_comb); + + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q5_k_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 768); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280); + + HVX_VectorPair v_sums = accum_5bit_32x2(vptr, v_act0, v_act1, Q6_V_vzero()); + HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); + HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); + + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); + + HVX_Vector v_scale_offset = vptr[4]; + HVX_VectorPair p_deal = Q6_W_vdeal_VVR(v_scale_offset, v_scale_offset, -2); + HVX_Vector v_scale = Q6_V_lo_W(p_deal); + HVX_Vector v_offset = Q6_V_hi_W(p_deal); + + HVX_Vector v_scale_a_c0 = v_act0[8]; + HVX_Vector v_sum_a_c0 = v_act0[9]; + HVX_Vector v_scale_a_c1 = v_act1[8]; + HVX_Vector v_sum_a_c1 = v_act1[9]; + + HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale, v_scale_a_c0); + HVX_Vector v_offset_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset, v_sum_a_c0); + HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale, v_scale_a_c1); + HVX_Vector v_offset_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_offset, v_sum_a_c1); + + HVX_Vector v_scaled_dot_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); + HVX_Vector v_sum_scaled_c0 = hvx_vec_add_f32_f32(v_scaled_dot_c0, v_offset_comb_c0); + + HVX_Vector v_scaled_dot_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); + HVX_Vector v_sum_scaled_c1 = hvx_vec_add_f32_f32(v_scaled_dot_c1, v_offset_comb_c1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_q6_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector i32 = Q6_Vb_vsplat_R(32); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152); + + HVX_VectorPair v_sums = accum_q6_k_32x1(vptr, v_act, i32); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q6_k_32x1(v_sums, vptr[6], v_act[8])); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q6_k_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + HVX_Vector i32 = Q6_Vb_vsplat_R(32); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); + + HVX_VectorPair v_sums0, v_sums1; + accum_q6_k_32x2(vptr, v_act0, v_act1, i32, &v_sums0, &v_sums1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q6_k_32x1(v_sums0, vptr[6], v_act0[8])); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q6_k_32x1(v_sums1, vptr[6], v_act1[8])); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_q3_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152); + + HVX_VectorPair v_sums = accum_q3_k_32x1(vptr, v_act); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q6_k_32x1(v_sums, vptr[3], v_act[8])); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q3_k_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); + + HVX_VectorPair v_sums0, v_sums1; + accum_q3_k_32x2(vptr, v_act0, v_act1, &v_sums0, &v_sums1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q6_k_32x1(v_sums0, vptr[3], v_act0[8])); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q6_k_32x1(v_sums1, vptr[3], v_act1[8])); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_q2_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) { + HVX_Vector v_d, v_m; + q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m); + + HVX_Vector v_isum = Q6_V_vzero(); + HVX_Vector v_msum = Q6_V_vzero(); + for (uint32_t kt = kb; kt < kb + 8; kt++) { + const uint8_t * restrict tile = tile_ptr + kt * 384; + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280); + + HVX_Vector v_sc2, v_mn2; + q2_k_scale_pairs(tile, &v_sc2, &v_mn2); + q2_k_int_accum(accum_q2_k_32x1((const HVX_Vector *) tile, v_act), v_sc2, v_mn2, v_act[9], &v_isum, &v_msum); + } + const HVX_Vector * restrict v_act_sb = (const HVX_Vector *) (y_q + kb * 1280); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, q2_k_int_scale(v_isum, v_msum, v_d, v_m, v_act_sb[8])); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q2_k_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) { + HVX_Vector v_d, v_m; + q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m); + + HVX_Vector v_isum0 = Q6_V_vzero(); + HVX_Vector v_msum0 = Q6_V_vzero(); + HVX_Vector v_isum1 = Q6_V_vzero(); + HVX_Vector v_msum1 = Q6_V_vzero(); + for (uint32_t kt = kb; kt < kb + 8; kt++) { + const uint8_t * restrict tile = tile_ptr + kt * 384; + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280); + + HVX_Vector v_sc2, v_mn2; + q2_k_scale_pairs(tile, &v_sc2, &v_mn2); + + HVX_VectorPair v_sums0, v_sums1; + accum_q2_k_32x2((const HVX_Vector *) tile, v_act0, v_act1, &v_sums0, &v_sums1); + q2_k_int_accum(v_sums0, v_sc2, v_mn2, v_act0[9], &v_isum0, &v_msum0); + q2_k_int_accum(v_sums1, v_sc2, v_mn2, v_act1[9], &v_isum1, &v_msum1); + } + const HVX_Vector * restrict v_act0_sb = (const HVX_Vector *) (y0_q + kb * 1280); + const HVX_Vector * restrict v_act1_sb = (const HVX_Vector *) (y1_q + kb * 1280); + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, q2_k_int_scale(v_isum0, v_msum0, v_d, v_m, v_act0_sb[8])); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, q2_k_int_scale(v_isum1, v_msum1, v_d, v_m, v_act1_sb[8])); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +// One Q2_0 weight tile covers 64 k: act tiles 2 * kt and 2 * kt + 1, each 32 k wide (Q8_1) +static void tiled_vec_dot_q2_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0); + + uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0; + for (uint32_t kt = 0; kt < n_weight_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0); + HVX_Vector v_tile_qf = Q6_V_vzero(); + + #pragma unroll + for (int half = 0; half < 2; half++) { + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1); + + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q2_0_32x1(vptr + 2 * half, v_act)); + HVX_Vector v_da_sf = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8])); + HVX_Vector v_sa_qf = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act[9], v_one_f16)); + HVX_Vector v_h_qf = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf), v_sa_qf); + v_tile_qf = (half == 0) ? v_h_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_h_qf); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf)); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q2_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0); + + uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0; + for (uint32_t kt = 0; kt < n_weight_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0); + HVX_Vector v_tile_qf_c0 = Q6_V_vzero(); + HVX_Vector v_tile_qf_c1 = Q6_V_vzero(); + + #pragma unroll + for (int half = 0; half < 2; half++) { + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1); + + HVX_Vector v_sum_c0, v_sum_c1; + accum_q2_0_32x2(vptr + 2 * half, v_act0, v_act1, &v_sum_c0, &v_sum_c1); + + HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8])); + HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8])); + HVX_Vector v_sa_qf_c0 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act0[9], v_one_f16)); + HVX_Vector v_sa_qf_c1 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act1[9], v_one_f16)); + + HVX_Vector v_h_qf_c0 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c0), v_da_sf_c0), v_sa_qf_c0); + HVX_Vector v_h_qf_c1 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c1), v_da_sf_c1), v_sa_qf_c1); + + v_tile_qf_c0 = (half == 0) ? v_h_qf_c0 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, v_h_qf_c0); + v_tile_qf_c1 = (half == 0) ? v_h_qf_c1 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, v_h_qf_c1); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf)); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf)); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector lut = *(const HVX_Vector *) kvalues_iq4nl_lut; + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152); + + HVX_Vector v_sum = accum_4bit_32x1_lut(vptr, v_act, mask_h4, lut); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); + + HVX_Vector v_scale_w = vptr[4]; + HVX_Vector v_scale_a = v_act[8]; + HVX_Vector v_scale_comb = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a); + HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); + + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_iq4nl_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector lut = *(const HVX_Vector *) kvalues_iq4nl_lut; + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); + + HVX_VectorPair v_sums = accum_4bit_32x2_lut(vptr, v_act0, v_act1, mask_h4, lut); + HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); + HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); + + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); + + HVX_Vector v_scale_w = vptr[4]; + HVX_Vector v_scale_a_c0 = v_act0[8]; + HVX_Vector v_scale_a_c1 = v_act1[8]; + + HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c0); + HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a_c1); + + HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); + HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_mxfp4_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector lut = *(const HVX_Vector *) kvalues_mxfp4_lut; + HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; + HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152); + + HVX_Vector v_sum = accum_4bit_32x1_lut(vptr, v_act, mask_h4, lut); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(v_sum); + + HVX_Vector v_scale_w = hvx_vmem(tile_ptr + kt * 640 + 512); + HVX_Vector r0_d = Q6_V_vdelta_VV(v_scale_w, expand); + r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); + HVX_Vector v_scale_w_f32 = Q6_Vw_vasl_VwR(r0_d, 23); + + HVX_Vector v_scale_a_f16 = v_act[8]; + HVX_VectorPair p_scale_a_f32 = hvx_vec_f16_to_f32_shuff(v_scale_a_f16); + HVX_Vector v_scale_a = Q6_V_lo_W(p_scale_a_f32); + + HVX_Vector v_scale_comb = hvx_vec_mul_f32_f32(v_scale_w_f32, v_scale_a); + HVX_Vector v_sum_scaled = hvx_vec_mul_f32_f32(v_sum_sf, v_scale_comb); + + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, v_sum_scaled); + } + + v_sum_float = hvx_vec_mul_f32_f32(v_sum_float, hvx_vec_splat_f32(0.5f)); + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_mxfp4_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); + HVX_Vector lut = *(const HVX_Vector *) kvalues_mxfp4_lut; + HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; + HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); + + uint32_t n_k_tiles = n / 32; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 640); + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152); + + HVX_VectorPair v_sums = accum_4bit_32x2_lut(vptr, v_act0, v_act1, mask_h4, lut); + HVX_Vector v_sum_c0 = Q6_V_lo_W(v_sums); + HVX_Vector v_sum_c1 = Q6_V_hi_W(v_sums); + + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(v_sum_c0); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(v_sum_c1); + + HVX_Vector v_scale_w = hvx_vmem(tile_ptr + kt * 640 + 512); + HVX_Vector r0_d = Q6_V_vdelta_VV(v_scale_w, expand); + r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); + HVX_Vector v_scale_w_f32 = Q6_Vw_vasl_VwR(r0_d, 23); + + HVX_Vector v_scale_a_c0_f16 = v_act0[8]; + HVX_Vector v_scale_a_c1_f16 = v_act1[8]; + + HVX_VectorPair p_scale_a_c0_f32 = hvx_vec_f16_to_f32_shuff(v_scale_a_c0_f16); + HVX_VectorPair p_scale_a_c1_f32 = hvx_vec_f16_to_f32_shuff(v_scale_a_c1_f16); + + HVX_Vector v_scale_a_c0 = Q6_V_lo_W(p_scale_a_c0_f32); + HVX_Vector v_scale_a_c1 = Q6_V_lo_W(p_scale_a_c1_f32); + + HVX_Vector v_scale_comb_c0 = hvx_vec_mul_f32_f32(v_scale_w_f32, v_scale_a_c0); + HVX_Vector v_scale_comb_c1 = hvx_vec_mul_f32_f32(v_scale_w_f32, v_scale_a_c1); + + HVX_Vector v_sum_scaled_c0 = hvx_vec_mul_f32_f32(v_sum_sf_c0, v_scale_comb_c0); + HVX_Vector v_sum_scaled_c1 = hvx_vec_mul_f32_f32(v_sum_sf_c1, v_scale_comb_c1); + + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, v_sum_scaled_c0); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, v_sum_scaled_c1); + } + + v_sum_float_c0 = hvx_vec_mul_f32_f32(v_sum_float_c0, hvx_vec_splat_f32(0.5f)); + v_sum_float_c1 = hvx_vec_mul_f32_f32(v_sum_float_c1, hvx_vec_splat_f32(0.5f)); + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + +static void tiled_vec_dot_f16_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) vx; + const HVX_Vector * restrict v_act = (const HVX_Vector *) vy; + const uint32_t n_vecs = ((n + 31) / 32) * 16; + + HVX_VectorPair acc = Q6_W_vzero(); + + #pragma unroll(4) + for (uint32_t i = 0; i < n_vecs; i++) { + acc = hvx_vec_mpyacc_f32_f16(acc, vptr[i], v_act[i]); + } + + HVX_Vector v_sum = hvx_vec_add_f32_f32(Q6_V_lo_W(acc), Q6_V_hi_W(acc)); + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum); + } +} + +static void tiled_vec_dot_f16_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) vx; + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) vy0; + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) vy1; + const uint32_t n_vecs = ((n + 31) / 32) * 16; + + HVX_VectorPair acc0 = Q6_W_vzero(); + HVX_VectorPair acc1 = Q6_W_vzero(); + + #pragma unroll(4) + for (uint32_t i = 0; i < n_vecs; i++) { + HVX_Vector v_w = vptr[i]; + acc0 = hvx_vec_mpyacc_f32_f16(acc0, v_w, v_act0[i]); + acc1 = hvx_vec_mpyacc_f32_f16(acc1, v_w, v_act1[i]); + } + + HVX_Vector v_sum0 = hvx_vec_add_f32_f32(Q6_V_lo_W(acc0), Q6_V_hi_W(acc0)); + HVX_Vector v_sum1 = hvx_vec_add_f32_f32(Q6_V_lo_W(acc1), Q6_V_hi_W(acc1)); + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum1); + } +} + + +static inline void quantize_f32_q8_0_tiled_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_row_size, + size_t dst_row_size +) { + (void) tmp_data; + for (uint32_t i = 0; i < nrows; ++i) { + quantize_row_f32_q8_0_tiled((float *) src_data, dst_data, ne0); + dst_data += dst_row_size; + src_data += src_row_size; + } +} + +static inline void quantize_f32_q8_1_tiled_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_row_size, + size_t dst_row_size +) { + (void) tmp_data; + for (uint32_t i = 0; i < nrows; ++i) { + quantize_row_f32_q8_1_tiled((float *) src_data, dst_data, ne0); + dst_data += dst_row_size; + src_data += src_row_size; + } +} + +static inline void quantize_f32_q8_1_s16_tiled_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_row_size, + size_t dst_row_size +) { + (void) tmp_data; + for (uint32_t i = 0; i < nrows; ++i) { + quantize_row_f32_q8_1_s16_tiled((float *) src_data, dst_data, ne0); + dst_data += dst_row_size; + src_data += src_row_size; + } +} + +static inline void quantize_f32_f16_tiled_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_row_size, + size_t dst_row_size +) { + (void) tmp_data; + for (uint32_t i = 0; i < nrows; ++i) { + quantize_row_f32_f16_tiled((const float *) src_data, dst_data, ne0); + dst_data += dst_row_size; + src_data += src_row_size; + } +} + +static inline void quantize_f16_f16_tiled_kernel( + const uint8_t * restrict src_data, + uint8_t * restrict dst_data, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t nrows, + size_t src_row_size, + size_t dst_row_size +) { + (void) tmp_data; + for (uint32_t i = 0; i < nrows; ++i) { + quantize_row_f16_f16_tiled((const __fp16 *) src_data, dst_data, ne0); + dst_data += dst_row_size; + src_data += src_row_size; + } +} + +static inline void quantize_f32_q8_0_tiled_block_kernel( + const float * restrict src, + uint8_t * restrict dst, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t ib_first, + uint32_t ib_last, + size_t src_row_size, + size_t dst_row_size, + uint32_t r, + uint32_t c +) { + (void) tmp_data; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne0 + qk - 1) / qk; + + for (uint32_t ib = ib_first; ib < ib_last; ++ib) { + const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float)); + uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 1152; + + quantize_block_f32_q8_0_tiled((float *) src_ptr, dst_ptr); + + c++; + if (c == nb) { + c = 0; + r++; + } + } +} + +static inline void quantize_f32_q8_1_tiled_block_kernel( + const float * restrict src, + uint8_t * restrict dst, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t ib_first, + uint32_t ib_last, + size_t src_row_size, + size_t dst_row_size, + uint32_t r, + uint32_t c +) { + (void) tmp_data; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne0 + qk - 1) / qk; + + for (uint32_t ib = ib_first; ib < ib_last; ++ib) { + const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float)); + uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 1280; + + quantize_block_f32_q8_1_tiled((float *) src_ptr, dst_ptr); + + c++; + if (c == nb) { + c = 0; + r++; + } + } +} + +static inline void quantize_f32_q8_1_s16_tiled_block_kernel( + const float * restrict src, + uint8_t * restrict dst, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t ib_first, + uint32_t ib_last, + size_t src_row_size, + size_t dst_row_size, + uint32_t r, + uint32_t c +) { + (void) tmp_data; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne0 + qk - 1) / qk; + + for (uint32_t ib = ib_first; ib < ib_last; ++ib) { + const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float)); + uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 1280; + + quantize_block_f32_q8_1_s16_tiled((float *) src_ptr, dst_ptr, src_ptr - (c & 1) * qk); // super-block of 256 k + + c++; + if (c == nb) { + c = 0; + r++; + } + } +} + +static inline void quantize_f32_f16_tiled_block_kernel( + const float * restrict src, + uint8_t * restrict dst, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t ib_first, + uint32_t ib_last, + size_t src_row_size, + size_t dst_row_size, + uint32_t r, + uint32_t c +) { + (void) tmp_data; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne0 + qk - 1) / qk; + + for (uint32_t ib = ib_first; ib < ib_last; ++ib) { + const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float)); + uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 2048; + + quantize_block_f32_f16_tiled(src_ptr, dst_ptr); + + c++; + if (c == nb) { + c = 0; + r++; + } + } +} + +static inline void quantize_f16_f16_tiled_block_kernel( + const __fp16 * restrict src, + uint8_t * restrict dst, + uint8_t * restrict tmp_data, + uint32_t ne0, + uint32_t ib_first, + uint32_t ib_last, + size_t src_row_size, + size_t dst_row_size, + uint32_t r, + uint32_t c +) { + (void) tmp_data; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne0 + qk - 1) / qk; + + for (uint32_t ib = ib_first; ib < ib_last; ++ib) { + const __fp16 * restrict src_ptr = (const __fp16 *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(__fp16)); + uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 2048; + + quantize_block_f16_f16_tiled(src_ptr, dst_ptr); + + c++; + if (c == nb) { + c = 0; + r++; + } + } +} diff --git a/ggml/src/ggml-hexagon/htp/hvx-norm.h b/ggml/src/ggml-hexagon/htp/hvx-norm.h new file mode 100644 index 0000000..145c308 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-norm.h @@ -0,0 +1,462 @@ +#ifndef HVX_NORM_H +#define HVX_NORM_H + +#include +#include "hvx-base.h" +#include "hvx-reduce.h" +#include "hvx-inverse.h" +#include "hvx-sqrt.h" +#include "hvx-repl.h" + +static inline void hvx_fast_rms_norm_f32(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon, + float scale) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP32; // number of full vectors + const int nloe = num_elems % VLEN_FP32; // leftover elements + + // Compute sum of squares for full vectors + HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); + } + + // Handle tail elements using vectorized ops with masking + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); + } + + // Reduce HVX sum + sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v); + HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v); + + // Scale full vectors + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); + if (scale != 0.0f) { + scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale))); + } + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); + v_dst[i] = Q6_Vsf_equals_Vqf32(v2); + } + + // Handle tail elements using vectorized ops with masking + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); + HVX_Vector result = Q6_Vsf_equals_Vqf32(v2); + + // Store with masking to avoid overwriting memory beyond the tensor + hvx_vec_store_a(&v_dst[nvec], nloe * 4, result); + } +} + +static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src, + const uint8_t * restrict weight, + uint8_t * restrict dst, + const int num_elems, + float epsilon, + float scale) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + const HVX_Vector * restrict v_weight = (const HVX_Vector *) weight; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP32; // number of full vectors + const int nloe = num_elems % VLEN_FP32; // leftover elements + + // Compute sum of squares for full vectors + HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); + } + + // Handle tail elements using vectorized ops with masking + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); + } + + // Reduce HVX sum + sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v); + HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v); + + // Scale and multiply + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); + if (scale != 0.0f) { + scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale))); + } + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2); + HVX_Vector result = Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[i]); + v_dst[i] = Q6_Vsf_equals_Vqf32(result); + } + + // Handle tail elements using vectorized ops with masking + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2); + HVX_Vector result = Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[nvec]); + HVX_Vector res_v = Q6_Vsf_equals_Vqf32(result); + + // Store with masking to avoid overwriting memory beyond the tensor + hvx_vec_store_a(&v_dst[nvec], nloe * 4, res_v); + } +} + +static inline void hvx_fast_norm_f32(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP32; // number of full vectors + const int nloe = num_elems % VLEN_FP32; // leftover elements + + // Compute sum of squares and sum of values for full vectors + HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, v2); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(v1, Q6_V_vzero())); + } + + // Handle tail elements using vectorized ops with masking + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, v2); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(v1, Q6_V_vzero())); + } + + // Reduce HVX sums + sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v)); + sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v); + HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v); + HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v)); + HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v); + HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v); + + // scale = rsqrt(variance + epsilon), mean_x broadcast for subtraction + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v)); + HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v)); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, mean_x_b); + HVX_Vector v3 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v2), scale_v); + v_dst[i] = Q6_Vsf_equals_Vqf32(v3); + } + + // Handle tail elements using vectorized ops with masking + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, mean_x_b); + HVX_Vector v3 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v2), scale_v); + HVX_Vector result = Q6_Vsf_equals_Vqf32(v3); + + // Store with masking to avoid overwriting memory beyond the tensor + hvx_vec_store_a(&v_dst[nvec], nloe * 4, result); + } +} + +static inline void hvx_fast_l2_norm_f32(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + HVX_Vector sum_v = hvx_vec_splat_f32(0.0f); + + const int nvec = num_elems / VLEN_FP32; + const int nloe = num_elems % VLEN_FP32; + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq); + } + + // Include tail elements in the sum-of-squares using a predicate mask + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq); + } + + // Compute scale = 1/fmax(sqrt(sum), epsilon) entirely in HVX registers. + // hvx_vec_rsqrt_f32 + hvx_vec_inverse_f32 avoids scalar extraction. + HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf); // 1/sqrt(sum) + HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v); // sqrt(sum) + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v); // fmax(sqrt(sum), epsilon) + HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v); // 1/fmax(sqrt(sum), epsilon) + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + v_dst[i] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v)); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_Vector result = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v)); + hvx_vec_store_a(&v_dst[nvec], nloe * 4, result); + } +} + +// F16 norm kernels: reduce and scale in f32 (via promote/narrow), matching the +// precision-preserving pattern used by the flash-attn f16 kernels. + +static inline void hvx_fast_rms_norm_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP16; // number of full f16 vectors + const int nloe = num_elems % VLEN_FP16; // leftover elements + + HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v); + HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v); + + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + v_dst[i] = hvx_vec_f32_to_f16(r0, r1); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + HVX_Vector result = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result); + } +} + +static inline void hvx_fast_norm_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP16; + const int nloe = num_elems % VLEN_FP16; + + HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero())); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero())); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero())); + sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero())); + } + + sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v)); + sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v)); + + HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); + HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); + HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v); + HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v); + HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v)); + HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v); + HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v); + + HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v)); + HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v)); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b); + HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v)); + v_dst[i] = hvx_vec_f32_to_f16(r0, r1); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b); + HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v)); + HVX_Vector result = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result); + } +} + +static inline void hvx_fast_l2_norm_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float epsilon) { + + const HVX_Vector * restrict v_src = (HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const int nvec = num_elems / VLEN_FP16; + const int nloe = num_elems % VLEN_FP16; + + HVX_Vector sum_v = hvx_vec_splat_f32(0.0f); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector p0 = Q6_V_lo_W(p); + HVX_Vector p1 = Q6_V_hi_W(p); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0)); + sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1)); + } + + HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf); + HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v); + HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v); + HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v); + + #pragma unroll(4) + for (int i = 0; i < nvec; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + v_dst[i] = hvx_vec_f32_to_f16(r0, r1); + } + + if (nloe > 0) { + HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16); + HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); + HVX_VectorPair p = hvx_vec_f16_to_f32(v1); + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v)); + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v)); + HVX_Vector result = hvx_vec_f32_to_f16(r0, r1); + hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result); + } +} + +#endif // HVX_NORM_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-pow.h b/ggml/src/ggml-hexagon/htp/hvx-pow.h new file mode 100644 index 0000000..48fe0e8 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-pow.h @@ -0,0 +1,42 @@ +#ifndef HVX_POW_H +#define HVX_POW_H + +#include +#include "hvx-base.h" +#include "hvx-exp.h" +#include "hvx-log.h" + +// Approximates base^exponent element-wise for float vectors. +// base must be a positive constant. exponent is an HVX f32 vector. +// Uses base^x = exp(x * ln(base)). +static inline HVX_Vector hvx_vec_pow_const_base_f32(float base, HVX_Vector exponent) { + float ln_base = logf(base); + HVX_Vector ln_base_v = hvx_vec_splat_f32(ln_base); + HVX_Vector x = hvx_vec_mul_f32_f32(exponent, ln_base_v); + + static const float kInf = INFINITY; + static const float kMaxExp = 88.7228f; + + const HVX_Vector max_exp = hvx_vec_splat_f32(kMaxExp); + const HVX_Vector inf = hvx_vec_splat_f32(kInf); + + return hvx_vec_exp_f32_guard(x, max_exp, inf); +} + +// Approximates base^exponent element-wise for float vectors. +// base and exponent are HVX f32 vectors. base elements must be positive. +// Uses base^exponent = exp(exponent * ln(base)). +static inline HVX_Vector hvx_vec_pow_f32(HVX_Vector base, HVX_Vector exponent) { + HVX_Vector ln_base = hvx_vec_log_f32(base); + HVX_Vector x = hvx_vec_mul_f32_f32(exponent, ln_base); + + static const float kInf = INFINITY; + static const float kMaxExp = 88.7228f; + + const HVX_Vector max_exp = hvx_vec_splat_f32(kMaxExp); + const HVX_Vector inf = hvx_vec_splat_f32(kInf); + + return hvx_vec_exp_f32_guard(x, max_exp, inf); +} + +#endif /* HVX_POW_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-quant.h b/ggml/src/ggml-hexagon/htp/hvx-quant.h new file mode 100644 index 0000000..b0ca6ec --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-quant.h @@ -0,0 +1,726 @@ +#ifndef HVX_QUANT_H +#define HVX_QUANT_H + +#include +#include +#include + +#include "htp-tensor.h" +#include "hvx-arith.h" +#include "hvx-base.h" +#include "hvx-reduce.h" +#include "hvx-repl.h" +#include "hvx-utils.h" + +#ifndef GGML_COMMON_DECL_C +#define GGML_COMMON_DECL_C +#endif +#include "ggml-common.h" +#include "ggml-impl.h" + +static inline void hvx_quantize_row_q8_0_f32(void * restrict dst_ptr, const float * restrict src_ptr, int n) { + const int nb = n / QK8_0; + block_q8_0 * dst = (block_q8_0 *) dst_ptr; + HVX_Vector zero = Q6_V_vzero(); + + int i = 0; + for (; i + 3 < nb; i += 4) { + HVX_Vector * vx = (HVX_Vector *) (src_ptr + i * QK8_0); + + HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); + HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); + HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); + HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); + + HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); + HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); + HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); + HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); + + HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); + HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); + HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); + HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); + + HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); + HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); + + HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); + HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); + + HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 + HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); + HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); + + HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); + HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); + vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); + vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); + + HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); + HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); + HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16); + + hvx_vec_store_u(&dst[i + 0].d, 2, vd01_hf); + hvx_vec_store_u(dst[i + 0].qs, 32, vx_i8); + + hvx_vec_store_u(&dst[i + 1].d, 2, Q6_V_vror_VR(vd01_hf, 64)); + hvx_vec_store_u(dst[i + 1].qs, 32, Q6_V_vror_VR(vx_i8, 32)); + + hvx_vec_store_u(&dst[i + 2].d, 2, vd23_hf); + hvx_vec_store_u(dst[i + 2].qs, 32, Q6_V_vror_VR(vx_i8, 64)); + + hvx_vec_store_u(&dst[i + 3].d, 2, Q6_V_vror_VR(vd23_hf, 64)); + hvx_vec_store_u(dst[i + 3].qs, 32, Q6_V_vror_VR(vx_i8, 96)); + } + + for (; i < nb; i++) { + const float * block_src = src_ptr + i * QK8_0; + HVX_Vector vx = *(const HVX_UVector *) block_src; + HVX_Vector v_abs = hvx_vec_abs_f32(vx); + HVX_Vector v_max = hvx_vec_reduce_max_f32(v_abs); + float amax = hvx_vec_get_f32(v_max); + + const float d = amax / 127.0f; + const float id = d ? (1.0f / d) : 0.0f; + dst[i].d = GGML_FP32_TO_FP16(d); + + HVX_Vector vid = hvx_vec_splat_f32(id); + HVX_Vector v_scaled = hvx_vec_mul_f32_f32(vx, vid); + HVX_Vector v_scaled_qf = Q6_Vqf32_vsub_VsfVsf(v_scaled, zero); + HVX_Vector v_scaled_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(zero, v_scaled_qf))); + HVX_Vector v_i16 = hvx_vec_i16_from_hf_rnd_sat(v_scaled_hf); + HVX_Vector v_i8 = Q6_Vb_vpack_VhVh_sat(zero, v_i16); + + hvx_vec_store_u(dst[i].qs, 32, v_i8); + } +} + +static inline void hvx_dequantize_row_flat_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK8_0; + const block_q8_0 * src = (const block_q8_0 *) src_ptr; + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16); + HVX_Vector vd = Q6_V_lo_W(vp_f32); + + HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs; + + HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8); + HVX_Vector v_i16 = Q6_V_lo_W(p16); + HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16); + HVX_Vector v_i32 = Q6_V_lo_W(p32); + + HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32); + HVX_Vector res = hvx_vec_mul_f32_f32(v_f32, vd); + + float * block_dst = dst_ptr + i * QK8_0; + hvx_vmem(block_dst) = res; + } +} + +static inline void hvx_dequantize_row_flat_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK8_0; + const block_q8_0 * src = (const block_q8_0 *) src_ptr; + + for (int i = nb - 1; i >= 0; i--) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16); + HVX_Vector vd = Q6_V_lo_W(vp_f32); + + HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs; + + HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8); + HVX_Vector v_i16 = Q6_V_lo_W(p16); + HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16); + HVX_Vector v_i32 = Q6_V_lo_W(p32); + + HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32); + HVX_Vector res_f32 = hvx_vec_mul_f32_f32(v_f32, vd); + + HVX_Vector res_f16 = hvx_vec_f32_to_f16(res_f32, Q6_V_vzero()); + + __fp16 * block_dst = dst_ptr + i * QK8_0; + hvx_vec_store_u(block_dst, QK8_0 * sizeof(__fp16), res_f16); + } +} + +static inline void hvx_dequantize_row_flat_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + const _Float16 * src = (const _Float16 *) src_ptr; + + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * VLEN_FP32); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16); + HVX_Vector res = Q6_V_lo_W(vp_f32); + + hvx_vmem(dst_ptr + i * VLEN_FP32) = res; + } + + if (nloe) { + HVX_Vector v_f16 = *(const HVX_UVector *) (src + nvec * VLEN_FP32); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16); + HVX_Vector res = Q6_V_lo_W(vp_f32); + + hvx_vec_store_a(dst_ptr + nvec * VLEN_FP32, nloe * sizeof(float), res); + } +} + +static inline void hvx_dequantize_row_flat_bf16_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + const uint16_t * src = (const uint16_t *) src_ptr; + + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector v_bf16 = *(const HVX_UVector *) (src + i * VLEN_FP32); + HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16); + HVX_Vector res = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16); + + hvx_vmem(dst_ptr + i * VLEN_FP32) = res; + } + + if (nloe) { + HVX_Vector v_bf16 = *(const HVX_UVector *) (src + nvec * VLEN_FP32); + HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16); + HVX_Vector res = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16); + + hvx_vec_store_a(dst_ptr + nvec * VLEN_FP32, nloe * sizeof(float), res); + } +} + +static inline void hvx_dequantize_row_flat_q1_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + static const uint32_t __attribute__((aligned(128))) bit_mask[32] = { + 1u << 0, 1u << 1, 1u << 2, 1u << 3, 1u << 4, 1u << 5, 1u << 6, 1u << 7, + 1u << 8, 1u << 9, 1u << 10, 1u << 11, 1u << 12, 1u << 13, 1u << 14, 1u << 15, + 1u << 16, 1u << 17, 1u << 18, 1u << 19, 1u << 20, 1u << 21, 1u << 22, 1u << 23, + 1u << 24, 1u << 25, 1u << 26, 1u << 27, 1u << 28, 1u << 29, 1u << 30, 1u << 31, + }; + const int nb = n / QK1_0; + const block_q1_0 * src = (const block_q1_0 *) src_ptr; + const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask; + const HVX_Vector v_zero = Q6_V_vzero(); + const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000); + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16)); + HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit); + HVX_Vector v_qs = *(const HVX_UVector *) src[i].qs; + + float * block_dst = dst_ptr + i * QK1_0; + #pragma unroll + for (int q = 0; q < 4; q++) { + HVX_Vector v_bits = hvx_vec_repl_u32(q == 0 ? v_qs : Q6_V_vror_VR(v_qs, 4 * q)); + HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero); + hvx_vmem(block_dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd); + } + } +} + +static inline void hvx_dequantize_row_flat_q2_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK2_0; + const block_q2_0 * src = (const block_q2_0 *) src_ptr; + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector v_one = Q6_Vb_vsplat_R(1); + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16)); + + HVX_Vector vq = *(const HVX_UVector *) src[i].qs; + HVX_Vector v0 = Q6_V_vand_VV(vq, mask_03); + HVX_Vector v1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 2), mask_03); + HVX_Vector v2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 4), mask_03); + HVX_Vector v3 = Q6_Vub_vlsr_VubR(vq, 6); + + HVX_Vector v01 = Q6_V_lo_W(Q6_W_vshuff_VVR(v1, v0, -1)); + HVX_Vector v23 = Q6_V_lo_W(Q6_W_vshuff_VVR(v3, v2, -1)); + HVX_Vector v64 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(v23, v01, -2)), v_one); + + HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v64); + HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + + float * block_dst = dst_ptr + i * QK2_0; + hvx_vmem(block_dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), vd); + hvx_vmem(block_dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), vd); + } +} + +static inline void hvx_dequantize_row_flat_q4_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK4_0; + const block_q4_0 * src = (const block_q4_0 *) src_ptr; + + const HVX_VectorPred q16 = Q6_Q_vsetq_R(16); + const HVX_Vector v_eight = Q6_Vb_vsplat_R(8); + const HVX_Vector v_zero = Q6_V_vzero(); + + for (int i = 0; i < nb; i++) { + HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d); + HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16); + HVX_Vector vd = Q6_V_lo_W(vp_f32); + + HVX_Vector vq = *(const HVX_UVector *) src[i].qs; + HVX_Vector v_lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); + HVX_Vector v_hi = Q6_Vub_vlsr_VubR(vq, 4); + + HVX_Vector v_lo_s8 = Q6_Vb_vsub_VbVb(v_lo, v_eight); + HVX_Vector v_hi_s8 = Q6_Vb_vsub_VbVb(v_hi, v_eight); + + HVX_Vector v32 = Q6_V_vmux_QVV(q16, v_lo_s8, Q6_V_vlalign_VVR(v_hi_s8, v_zero, 16)); + + HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v32); + HVX_Vector v_i16 = Q6_V_lo_W(p16); + HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16); + HVX_Vector v_i32 = Q6_V_lo_W(p32); + + HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32); + HVX_Vector res = hvx_vec_mul_f32_f32(v_f32, vd); + + hvx_vmem(dst_ptr + i * QK4_0) = res; + } +} + +static inline void hvx_dequantize_row_flat_q4_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK_K; + const block_q4_K * src = (const block_q4_K *) src_ptr; + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(src[i].d); + const float min = GGML_FP16_TO_FP32(src[i].dmin); + const uint8_t * sc = src[i].scales; + + float * y = dst_ptr + i * QK_K; + + for (int step = 0; step < 4; ++step) { + uint8_t sc1, m1_u, sc2, m2_u; + if (step < 2) { + sc1 = sc[2 * step + 0] & 63; + m1_u = sc[2 * step + 4] & 63; + sc2 = sc[2 * step + 1] & 63; + m2_u = sc[2 * step + 5] & 63; + } else { + const int s = step - 2; + sc1 = (sc[8 + 2 * s + 0] & 0xF) | ((sc[2 * s + 0] >> 6) << 4); + m1_u = (sc[8 + 2 * s + 0] >> 4) | ((sc[4 + 2 * s + 0] >> 6) << 4); + sc2 = (sc[8 + 2 * s + 1] & 0xF) | ((sc[2 * s + 1] >> 6) << 4); + m2_u = (sc[8 + 2 * s + 1] >> 4) | ((sc[4 + 2 * s + 1] >> 6) << 4); + } + + HVX_Vector vq = *(const HVX_UVector *) (src[i].qs + step * 32); + HVX_Vector v_lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); + HVX_Vector v_hi = Q6_Vub_vlsr_VubR(vq, 4); + + HVX_VectorPair p16_lo = Q6_Wuh_vunpack_Vub(v_lo); + HVX_VectorPair p32_lo = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_lo)); + HVX_Vector v_i32_lo = Q6_V_lo_W(p32_lo); + HVX_Vector v_f32_lo = Q6_Vsf_equals_Vw(v_i32_lo); + + HVX_Vector vd1 = hvx_vec_splat_f32(d * sc1); + HVX_Vector vm1 = hvx_vec_splat_f32(min * m1_u); + HVX_Vector v_out1 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_lo, vd1), vm1); + hvx_vmem(y + 0) = v_out1; + + HVX_VectorPair p16_hi = Q6_Wuh_vunpack_Vub(v_hi); + HVX_VectorPair p32_hi = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_hi)); + HVX_Vector v_i32_hi = Q6_V_lo_W(p32_hi); + HVX_Vector v_f32_hi = Q6_Vsf_equals_Vw(v_i32_hi); + + HVX_Vector vd2 = hvx_vec_splat_f32(d * sc2); + HVX_Vector vm2 = hvx_vec_splat_f32(min * m2_u); + HVX_Vector v_out2 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_hi, vd2), vm2); + hvx_vmem(y + 32) = v_out2; + + y += 64; + } + } +} + +static inline void hvx_dequantize_row_flat_q5_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK_K; + const block_q5_K * src = (const block_q5_K *) src_ptr; + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(src[i].d); + const float min = GGML_FP16_TO_FP32(src[i].dmin); + const uint8_t * sc = src[i].scales; + + HVX_Vector v_qh = *(const HVX_UVector *) src[i].qh; + float * y = dst_ptr + i * QK_K; + + for (int step = 0; step < 4; ++step) { + uint8_t sc1, m1_u, sc2, m2_u; + if (step < 2) { + sc1 = sc[2 * step + 0] & 63; + m1_u = sc[2 * step + 4] & 63; + sc2 = sc[2 * step + 1] & 63; + m2_u = sc[2 * step + 5] & 63; + } else { + const int s = step - 2; + sc1 = (sc[8 + 2 * s + 0] & 0xF) | ((sc[2 * s + 0] >> 6) << 4); + m1_u = (sc[8 + 2 * s + 0] >> 4) | ((sc[4 + 2 * s + 0] >> 6) << 4); + sc2 = (sc[8 + 2 * s + 1] & 0xF) | ((sc[2 * s + 1] >> 6) << 4); + m2_u = (sc[8 + 2 * s + 1] >> 4) | ((sc[4 + 2 * s + 1] >> 6) << 4); + } + + HVX_Vector v_ql = *(const HVX_UVector *) (src[i].qs + step * 32); + HVX_Vector v_lo = Q6_V_vand_VV(v_ql, Q6_Vb_vsplat_R(0x0F)); + HVX_Vector v_hi = Q6_Vub_vlsr_VubR(v_ql, 4); + + HVX_Vector v_h1 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2 * step), Q6_Vb_vsplat_R(1)), 4); + HVX_Vector v_val1 = Q6_V_vor_VV(v_lo, v_h1); + + HVX_VectorPair p16_1 = Q6_Wuh_vunpack_Vub(v_val1); + HVX_VectorPair p32_1 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_1)); + HVX_Vector v_i32_1 = Q6_V_lo_W(p32_1); + HVX_Vector v_f32_1 = Q6_Vsf_equals_Vw(v_i32_1); + + HVX_Vector vd1 = hvx_vec_splat_f32(d * sc1); + HVX_Vector vm1 = hvx_vec_splat_f32(min * m1_u); + HVX_Vector v_out1 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_1, vd1), vm1); + hvx_vmem(y + 0) = v_out1; + + HVX_Vector v_h2 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2 * step + 1), Q6_Vb_vsplat_R(1)), 4); + HVX_Vector v_val2 = Q6_V_vor_VV(v_hi, v_h2); + + HVX_VectorPair p16_2 = Q6_Wuh_vunpack_Vub(v_val2); + HVX_VectorPair p32_2 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_2)); + HVX_Vector v_i32_2 = Q6_V_lo_W(p32_2); + HVX_Vector v_f32_2 = Q6_Vsf_equals_Vw(v_i32_2); + + HVX_Vector vd2 = hvx_vec_splat_f32(d * sc2); + HVX_Vector vm2 = hvx_vec_splat_f32(min * m2_u); + HVX_Vector v_out2 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_2, vd2), vm2); + hvx_vmem(y + 32) = v_out2; + + y += 64; + } + } +} + +static inline void hvx_dequantize_row_flat_q6_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) { + const int nb = n / QK_K; + const block_q6_K * src = (const block_q6_K *) src_ptr; + + const HVX_VectorPred q64 = Q6_Q_vsetq_R(64); + const HVX_Vector v32_b = Q6_Vb_vsplat_R(32); + const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(src[i].d); + + const uint8_t * restrict ql = src[i].ql; + const uint8_t * restrict qh = src[i].qh; + const int8_t * restrict sc = src[i].scales; + + float * y = dst_ptr + i * QK_K; + + for (int half = 0; half < 2; ++half) { + float s[8]; + for (int j = 0; j < 8; ++j) { + s[j] = d * sc[j]; + } + + HVX_Vector vscale1 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[0]), hvx_vec_splat_f32(s[1])); + HVX_Vector vscale2 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[2]), hvx_vec_splat_f32(s[3])); + HVX_Vector vscale3 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[4]), hvx_vec_splat_f32(s[5])); + HVX_Vector vscale4 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[6]), hvx_vec_splat_f32(s[7])); + + HVX_Vector v_ql0 = *(const HVX_UVector *) (ql); + HVX_Vector v_ql1 = *(const HVX_UVector *) (ql + 32); + HVX_Vector v_qh = *(const HVX_UVector *) (qh); + + HVX_Vector ql0_lo = Q6_V_vand_VV(v_ql0, mask_0f); + HVX_Vector qh_0 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(v_qh, mask_03), 4); + HVX_Vector v_q1 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql0_lo, qh_0), v32_b); + + HVX_Vector ql1_lo = Q6_V_vand_VV(v_ql1, mask_0f); + HVX_Vector qh_1 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2), mask_03), 4); + HVX_Vector v_q2 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql1_lo, qh_1), v32_b); + + HVX_Vector ql0_hi = Q6_Vub_vlsr_VubR(v_ql0, 4); + HVX_Vector qh_2 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 4), mask_03), 4); + HVX_Vector v_q3 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql0_hi, qh_2), v32_b); + + HVX_Vector ql1_hi = Q6_Vub_vlsr_VubR(v_ql1, 4); + HVX_Vector qh_3 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 6), mask_03), 4); + HVX_Vector v_q4 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql1_hi, qh_3), v32_b); + + HVX_VectorPair p16_1 = Q6_Wh_vunpack_Vb(v_q1); + HVX_VectorPair p32_1 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_1)); + HVX_Vector v_f32_1 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_1)); + hvx_vmem(y + 0) = hvx_vec_mul_f32_f32(v_f32_1, vscale1); + + HVX_VectorPair p16_2 = Q6_Wh_vunpack_Vb(v_q2); + HVX_VectorPair p32_2 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_2)); + HVX_Vector v_f32_2 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_2)); + hvx_vmem(y + 32) = hvx_vec_mul_f32_f32(v_f32_2, vscale2); + + HVX_VectorPair p16_3 = Q6_Wh_vunpack_Vb(v_q3); + HVX_VectorPair p32_3 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_3)); + HVX_Vector v_f32_3 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_3)); + hvx_vmem(y + 64) = hvx_vec_mul_f32_f32(v_f32_3, vscale3); + + HVX_VectorPair p16_4 = Q6_Wh_vunpack_Vb(v_q4); + HVX_VectorPair p32_4 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_4)); + HVX_Vector v_f32_4 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_4)); + hvx_vmem(y + 96) = hvx_vec_mul_f32_f32(v_f32_4, vscale4); + + y += 128; + ql += 64; + qh += 32; + sc += 8; + } + } +} + +// bytes row, row + 32, row + 64 and row + 96 of v, to bytes 0..3 +static inline HVX_Vector hvx_dequantize_gather4(HVX_Vector v, uint32_t row) { + v = Q6_V_vror_VR(v, row); + HVX_Vector packed = Q6_V_vmux_QVV(Q6_Q_vsetq_R(1), v, Q6_V_vror_VR(v, 31)); + packed = Q6_V_vmux_QVV(Q6_Q_vsetq_R(2), packed, Q6_V_vror_VR(v, 62)); + return Q6_V_vmux_QVV(Q6_Q_vsetq_R(3), packed, Q6_V_vror_VR(v, 93)); +} + +static inline void hvx_dequantize_row_tiled_q8_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first2 = Q6_Q_vsetq_R(2); + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector vq = Q6_V_vzero(); + for (int group = 7; group >= 0; --group) { + const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row); + const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62)); + vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4)); + } + + const HVX_Vector scales = hvx_vmem(tile + 1024); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + *(HVX_Vector *) dst = values; + } +} + +static inline void hvx_dequantize_row_tiled_q4_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector vq = Q6_V_vzero(); + for (int group = 3; group >= 0; --group) { + vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4)); + } + + const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); + const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4); + vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1)); + vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8)); + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + *(HVX_Vector *) dst = values; + } +} + +static inline void hvx_dequantize_row_tiled_q4_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector vq = Q6_V_vzero(); + for (int group = 3; group >= 0; --group) { + vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4)); + } + + const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); + const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4); + vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1)); + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2)); + const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf)); + + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + *(HVX_Vector *) dst = hvx_vec_add_f32_f32(values, offset); + } +} + +static inline void hvx_dequantize_row_tiled_q5_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + const HVX_Vector one = Q6_Vb_vsplat_R(1); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + const HVX_Vector p5 = hvx_dequantize_gather4(hvx_vmem(tile + 640), row); + HVX_Vector vq = Q6_V_vzero(); + HVX_Vector vh = Q6_V_vzero(); + for (int group = 3; group >= 0; --group) { + vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4)); + vh = Q6_V_vmux_QVV(first4, Q6_Vub_vlsr_VubR(p5, 2 * group), Q6_V_vror_VR(vh, VLEN - 4)); + } + + const HVX_Vector lo = Q6_V_vor_VV(Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)), Q6_Vw_vasl_VwR(Q6_V_vand_VV(vh, one), 4)); + const HVX_Vector hi = Q6_V_vor_VV(Q6_Vub_vlsr_VubR(vq, 4), Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vh, 1), one), 4)); + vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1)); + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2)); + const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf)); + + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + *(HVX_Vector *) dst = hvx_vec_add_f32_f32(values, offset); + } +} + +static inline void hvx_dequantize_row_tiled_q6_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + const HVX_VectorPred first16 = Q6_Q_vsetq_R(16 * sizeof(float)); + const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F); + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector vq = Q6_V_vzero(); + for (int group = 7; group >= 0; --group) { + const HVX_Vector lo_plane = Q6_V_vror_VR(hvx_vmem(tile + (group >> 1) * VLEN), 4 * row); + const HVX_Vector hi_plane = Q6_V_vror_VR(hvx_vmem(tile + 512 + (group >> 2) * VLEN), 4 * row); + const HVX_Vector lo = (group & 1) ? Q6_Vub_vlsr_VubR(lo_plane, 4) : Q6_V_vand_VV(lo_plane, mask_0f); + const HVX_Vector hi = Q6_Vub_vlsr_VubR(hi_plane, 2 * (group & 3)); + const HVX_Vector packed = Q6_V_vor_VV(lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(hi, mask_03), 4)); + vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4)); + } + + const HVX_Vector scales = hvx_vmem(tile + 768); + const HVX_Vector scale_lo_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale_hi_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 64 + 2 * row)); + const HVX_Vector scale_lo = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_lo_hf)); + const HVX_Vector scale_hi = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hi_hf)); + const HVX_Vector scale = Q6_V_vmux_QVV(first16, scale_lo, scale_hi); + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(32))); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + *(HVX_Vector *) dst = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + } +} + +static inline HVX_Vector hvx_dequantize_tiled_f16_gather4(const uint8_t * p, uint32_t r_byte) { + HVX_Vector w0 = Q6_V_vror_VR(hvx_vmem(p + 0 * VLEN), r_byte); + HVX_Vector w1 = Q6_V_vror_VR(hvx_vmem(p + 1 * VLEN), r_byte); + HVX_Vector w01 = Q6_V_lo_W(Q6_W_vshuff_VVR(w1, w0, 4)); + HVX_Vector w2 = Q6_V_vror_VR(hvx_vmem(p + 2 * VLEN), r_byte); + HVX_Vector w3 = Q6_V_vror_VR(hvx_vmem(p + 3 * VLEN), r_byte); + HVX_Vector w23 = Q6_V_lo_W(Q6_W_vshuff_VVR(w3, w2, 4)); + return Q6_V_lo_W(Q6_W_vshuff_VVR(w23, w01, 8)); +} + +static inline void hvx_dequantize_row_tiled_f16_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const uint32_t r_byte = 4 * row; + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS; + + HVX_Vector g0 = hvx_dequantize_tiled_f16_gather4(tile + 0 * VLEN, r_byte); + HVX_Vector g1 = hvx_dequantize_tiled_f16_gather4(tile + 4 * VLEN, r_byte); + HVX_Vector g01 = Q6_V_lo_W(Q6_W_vshuff_VVR(g1, g0, 16)); + + HVX_Vector g2 = hvx_dequantize_tiled_f16_gather4(tile + 8 * VLEN, r_byte); + HVX_Vector g3 = hvx_dequantize_tiled_f16_gather4(tile + 12 * VLEN, r_byte); + HVX_Vector g23 = Q6_V_lo_W(Q6_W_vshuff_VVR(g3, g2, 16)); + + HVX_Vector g = Q6_V_lo_W(Q6_W_vshuff_VVR(g23, g01, 32)); + *(HVX_Vector *) dst = Q6_V_lo_W(hvx_vec_f16_to_f32(g)); + } +} + +static inline void hvx_dequantize_row_tiled_q1_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + static const uint32_t __attribute__((aligned(128))) bit_mask[32] = { + 1u << 0, 1u << 8, 1u << 16, 1u << 24, + 1u << 1, 1u << 9, 1u << 17, 1u << 25, + 1u << 2, 1u << 10, 1u << 18, 1u << 26, + 1u << 3, 1u << 11, 1u << 19, 1u << 27, + 1u << 4, 1u << 12, 1u << 20, 1u << 28, + 1u << 5, 1u << 13, 1u << 21, 1u << 29, + 1u << 6, 1u << 14, 1u << 22, 1u << 30, + 1u << 7, 1u << 15, 1u << 23, 1u << 31, + }; + const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask; + const HVX_Vector v_zero = Q6_V_vzero(); + const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q1_0; + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit); + + #pragma unroll + for (int q = 0; q < 4; ++q) { + const HVX_Vector v_bits = hvx_vec_repl_u32(Q6_V_vror_VR(hvx_vmem(tile + q * VLEN), 4 * row)); + const HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero); + hvx_vmem(dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd); + } + } +} + +static inline void hvx_dequantize_row_tiled_q2_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) { + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector v_one = Q6_Vb_vsplat_R(1); + + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q2_0; + + const HVX_Vector v_packed = hvx_dequantize_tiled_f16_gather4(tile, 4 * row); + const HVX_Vector p0 = Q6_V_vand_VV(v_packed, mask_03); + const HVX_Vector p1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 2), mask_03); + const HVX_Vector p2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 4), mask_03); + const HVX_Vector p3 = Q6_Vub_vlsr_VubR(v_packed, 6); + + const HVX_Vector p01 = Q6_V_lo_W(Q6_W_vshuff_VVR(p1, p0, -4)); + const HVX_Vector p23 = Q6_V_lo_W(Q6_W_vshuff_VVR(p3, p2, -4)); + const HVX_Vector vq = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(p23, p01, -8)), v_one); + + const HVX_Vector scales = hvx_vmem(tile + 512); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + hvx_vmem(dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + hvx_vmem(dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), scale); + } +} + +#endif // HVX_QUANT_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-reduce.h b/ggml/src/ggml-hexagon/htp/hvx-reduce.h index 3c0073e..1b6a6f1 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-reduce.h +++ b/ggml/src/ggml-hexagon/htp/hvx-reduce.h @@ -286,6 +286,119 @@ static inline float hvx_sum_of_squares_f32(const uint8_t * restrict src, const i } } +// Signed 32-bit Integer Max variants + +static inline HVX_Vector hvx_vec_reduce_max_n_i32(HVX_Vector in, unsigned int n) { + unsigned int total = n * 4; // total vec nbytes + unsigned int width = 4; // int32 nbytes + + HVX_Vector max_val = in, max_t; + while (width < total) { + max_t = Q6_V_vror_VR(max_val, width); // rotate right + max_val = Q6_Vw_vmax_VwVw(max_t, max_val); // elementwise signed max + width = width << 1; + } + return max_val; +} + +static inline HVX_Vector hvx_vec_reduce_max_i32(HVX_Vector in) { + return hvx_vec_reduce_max_n_i32(in, 32); +} + +static inline int32_t hvx_reduce_max_i32_a(const uint8_t * restrict src, const int num_elems) { + HVX_Vector init_vec = Q6_V_vsplat_R(((const int32_t *) src)[0]); + HVX_Vector pad_vec = Q6_V_vsplat_R(0x80000000); + assert((uintptr_t) src % 128 == 0); + hvx_reduce_loop_body(HVX_Vector, init_vec, pad_vec, Q6_Vw_vmax_VwVw, hvx_vec_reduce_max_i32, hvx_vec_get_i32); +} + +static inline int32_t hvx_reduce_max_i32_u(const uint8_t * restrict src, const int num_elems) { + HVX_Vector init_vec = Q6_V_vsplat_R(((const int32_t *) src)[0]); + HVX_Vector pad_vec = Q6_V_vsplat_R(0x80000000); + hvx_reduce_loop_body(HVX_UVector, init_vec, pad_vec, Q6_Vw_vmax_VwVw, hvx_vec_reduce_max_i32, hvx_vec_get_i32); +} + +static inline int32_t hvx_reduce_max_i32(const uint8_t * restrict src, const int num_elems) { + if (hex_is_aligned((void *) src, 128)) { + return hvx_reduce_max_i32_a(src, num_elems); + } else { + return hvx_reduce_max_i32_u(src, num_elems); + } +} + +static inline void hvx_argmax_f32( + const float * restrict src, + uint32_t n, + uint32_t offset, + float * out_val, + int32_t * out_idx +) { + if (n == 0) { + *out_val = -INFINITY; + *out_idx = (int32_t) offset; + return; + } + + if (n < 32 || !hex_is_aligned((void *) src, 128)) { + float best_val = src[0]; + int32_t best_idx = (int32_t) offset; + for (uint32_t i = 1; i < n; i++) { + if (src[i] > best_val) { + best_val = src[i]; + best_idx = (int32_t) (offset + i); + } + } + *out_val = best_val; + *out_idx = best_idx; + return; + } + + static const int32_t c_lane_idx[32] __attribute__((aligned(128))) = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + }; + + const HVX_Vector v_init_idx = *(const HVX_Vector *) c_lane_idx; + const HVX_Vector v_step = Q6_V_vsplat_R(32); + HVX_Vector v_cur_idx = Q6_Vw_vadd_VwVw(v_init_idx, Q6_V_vsplat_R((int32_t) offset)); + HVX_Vector v_max_val = hvx_vec_splat_f32(-INFINITY); + HVX_Vector v_max_idx = v_cur_idx; + + const HVX_Vector * vsrc = (const HVX_Vector *) src; + const uint32_t nvec = n / 32; + + for (uint32_t vi = 0; vi < nvec; vi++) { + HVX_Vector v = vsrc[vi]; + HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(v, v_max_val); + v_max_val = Q6_V_vmux_QVV(pred, v, v_max_val); + v_max_idx = Q6_V_vmux_QVV(pred, v_cur_idx, v_max_idx); + v_cur_idx = Q6_Vw_vadd_VwVw(v_cur_idx, v_step); + } + + HVX_VectorAlias u_val, u_idx; + u_val.v = v_max_val; + u_idx.v = v_max_idx; + + float best_val = u_val.fp32[0]; + int32_t best_idx = (int32_t) u_idx.w[0]; + for (int i = 1; i < 32; i++) { + if (u_val.fp32[i] > best_val) { + best_val = u_val.fp32[i]; + best_idx = (int32_t) u_idx.w[i]; + } + } + + for (uint32_t i = nvec * 32; i < n; i++) { + if (src[i] > best_val) { + best_val = src[i]; + best_idx = (int32_t) (offset + i); + } + } + + *out_val = best_val; + *out_idx = best_idx; +} + #undef hvx_reduce_loop_body #undef HVX_REDUCE_MAX_OP #undef HVX_REDUCE_SUM_OP diff --git a/ggml/src/ggml-hexagon/htp/hvx-repl.h b/ggml/src/ggml-hexagon/htp/hvx-repl.h new file mode 100644 index 0000000..fdc7e6c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-repl.h @@ -0,0 +1,74 @@ +#ifndef HVX_REPL_H +#define HVX_REPL_H + +#include +#include +#include + +#include "hvx-base.h" + +static inline HVX_Vector hvx_vec_repl(HVX_Vector v, const uint8_t * ctrl) { + return Q6_V_vdelta_VV(v, hvx_vmem(ctrl)); +} + +static inline HVX_Vector hvx_vec_repl_u32(HVX_Vector v) { + // vdelta control to replicate first 4 bytes across all lanes + static const uint8_t __attribute__((aligned(128))) repl[128] = { + 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + }; + return hvx_vec_repl(v, repl); +} + +static inline HVX_Vector hvx_vec_repl_f32(HVX_Vector v) { + // vdelta control to replicate first 4 bytes across all lanes + static const uint8_t __attribute__((aligned(128))) repl[128] = { + 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04, + }; + return hvx_vec_repl(v, repl); +} + +static inline HVX_Vector hvx_vec_repl_f16(HVX_Vector v) { + // vdelta control to replicate first two bytes across all lanes + static const uint8_t __attribute__((aligned(128))) repl[128] = { + 0x00, 0x00, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x20, 0x20, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x40, 0x40, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x20, 0x20, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + }; + return hvx_vec_repl(v, repl); +} + +static inline HVX_Vector hvx_vec_repl_2x_f16(HVX_Vector v) { + // vdelta control to splat a pair of f16s: first half = f16[0], second half = f16[1] + static const uint8_t __attribute__((aligned(128))) repl[128] = { + 0x00, 0x00, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x20, 0x20, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, + 0x02, 0x02, 0x40, 0x40, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, + 0x02, 0x02, 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, + 0x02, 0x02, 0x20, 0x20, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, + 0x02, 0x02, 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, + }; + return hvx_vec_repl(v, repl); +} + +#endif // HVX_REPL_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-scale.h b/ggml/src/ggml-hexagon/htp/hvx-scale.h index c65c986..5d06503 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-scale.h +++ b/ggml/src/ggml-hexagon/htp/hvx-scale.h @@ -68,30 +68,30 @@ static inline void hvx_scale_f32(uint8_t * restrict dst, const uint8_t * restric } } -#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - HVX_Vector vs = hvx_vec_splat_f32(scale); \ - HVX_Vector vo = hvx_vec_splat_f32(offset); \ - \ - const uint32_t elem_size = sizeof(float); \ - const uint32_t epv = 128 / elem_size; \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; ++i) { \ +#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + HVX_Vector vs = hvx_vec_splat_f32(scale); \ + HVX_Vector vo = hvx_vec_splat_f32(offset); \ + \ + const uint32_t elem_size = sizeof(float); \ + const uint32_t epv = 128 / elem_size; \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; ++i) { \ HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \ - vdst[i] = Q6_Vsf_equals_Vqf32(v); \ - } \ - if (nloe) { \ + vdst[i] = Q6_Vsf_equals_Vqf32(v); \ + } \ + if (nloe) { \ HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \ - vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \ - } \ + vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \ + } \ } while(0) static inline void hvx_scale_offset_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { @@ -130,4 +130,70 @@ static inline void hvx_scale_offset_f32(uint8_t * restrict dst, const uint8_t * } } +// Scale+offset computed by promoting f16 -> f32, then narrowing the result back to f16. +#define hvx_scale_offset_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + HVX_Vector vs = hvx_vec_splat_f32(scale); \ + HVX_Vector vo = hvx_vec_splat_f32(offset); \ + \ + const uint32_t nvec = n / VLEN_FP16; \ + const uint32_t nloe = n % VLEN_FP16; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; ++i) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \ + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \ + vdst[i] = hvx_vec_f32_to_f16(r0, r1); \ + } \ + if (nloe) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \ + HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \ + HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \ + vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \ + } \ + } while(0) + +static inline void hvx_scale_offset_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + assert((size_t) dst % 128 == 0); + assert((size_t) src % 128 == 0); + hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_scale_offset_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + assert((size_t) dst % 128 == 0); + hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_scale_offset_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + assert((size_t) src % 128 == 0); + hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_scale_offset_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_scale_offset_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) { + if (((size_t) dst & 127) == 0) { + if (((size_t) src & 127) == 0) { + hvx_scale_offset_f16_aa(dst, src, n, scale, offset); + } else { + hvx_scale_offset_f16_au(dst, src, n, scale, offset); + } + } else { + if (((size_t) src & 127) == 0) { + hvx_scale_offset_f16_ua(dst, src, n, scale, offset); + } else { + hvx_scale_offset_f16_uu(dst, src, n, scale, offset); + } + } +} + #endif // HVX_SCALE_H diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h index 37f3e7b..d4d67ff 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h @@ -3,6 +3,7 @@ #include "hvx-base.h" #include "hvx-inverse.h" +#include "hvx-exp.h" #define FAST_SIGMOID_LOG2F (0x3fb8aa3b) // 1.442695022 #define FAST_SIGMOID_C1 (0x3d009076) // 0.03138777 @@ -67,50 +68,58 @@ static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) { return Q6_Vsf_equals_Vqf32(res); } -#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const HVX_Vector one = hvx_vec_splat_f32(1.f); \ - const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \ - const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \ - \ - const uint32_t epv = 128 / sizeof(float); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ - } \ - if (nloe) { \ +static inline HVX_Vector hvx_vec_silu_f32(HVX_Vector x) { + HVX_Vector one = hvx_vec_splat_f32(1.0f); + HVX_Vector max_exp = hvx_vec_splat_f32(87.f); + HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); + HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(x, one, max_exp, min_exp); + return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, sig)); +} + +#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const HVX_Vector one = hvx_vec_splat_f32(1.f); \ + const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \ + const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \ + \ + const uint32_t epv = 128 / sizeof(float); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ + } \ + if (nloe) { \ HVX_Vector tmp = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \ - vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ - } \ + vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ + } \ } while(0) -#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \ - do { \ - dst_type * restrict vdst = (dst_type *) dst; \ - src_type * restrict vsrc = (src_type *) src; \ - \ - const uint32_t epv = 128 / sizeof(float); \ - const uint32_t nvec = n / epv; \ - const uint32_t nloe = n % epv; \ - \ - uint32_t i = 0; \ - \ - _Pragma("unroll(4)") \ - for (; i < nvec; i++) { \ - vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \ - } \ - if (nloe) { \ - HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \ +#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / sizeof(float); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \ vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \ - } \ + } \ } while(0) static inline void hvx_sigmoid_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { @@ -139,4 +148,96 @@ static inline void hvx_tanh_f32_aa(uint8_t * restrict dst, const uint8_t * restr hvx_tanh_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); } +static inline HVX_Vector hvx_vec_fast_sigmoid_f16(HVX_Vector x_v) { + const HVX_Vector v_one = hvx_vec_splat_f16(1.0f); + const HVX_Vector v_neg_log2e = hvx_vec_splat_f16(-EXP_LOG2E_F); + const HVX_Vector em_mask = Q6_Vh_vsplat_R(0x7FFF); + + // Compute absolute value of x_v + HVX_Vector abs_x = Q6_V_vand_VV(x_v, em_mask); + + // Compute u = -abs_x * log2(e) <= 0. + HVX_Vector u = hvx_vec_mul_f16_f16(abs_x, v_neg_log2e); + + // Clamp input to prevent underflow in exp2 + const HVX_Vector v_clamp_min = hvx_vec_splat_f16(-24.0f); + u = Q6_Vhf_vmax_VhfVhf(v_clamp_min, u); + + HVX_Vector exp_val = hvx_vec_exp2_f16(u); + HVX_Vector denom = hvx_vec_add_f16_f16(v_one, exp_val); + HVX_Vector sig_abs = hvx_vec_inverse_f16(denom); + + // check if x_v < 0 (using integer comparison on absolute value) + HVX_VectorPred is_neg = Q6_Q_vcmp_gt_VhVh(abs_x, x_v); + + // If x_v < 0, return 1.0f - sig_abs + HVX_Vector sig_neg = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vsub_VhfVhf(v_one, sig_abs)); + return Q6_V_vmux_QVV(is_neg, sig_neg, sig_abs); +} + +static inline HVX_Vector hvx_vec_tanh_f16(HVX_Vector x) { + // tanh(x) = 2 * sigmoid(2x) - 1 + const HVX_Vector v_two = hvx_vec_splat_f16(2.0f); + + HVX_Vector x2 = hvx_vec_mul_f16_f16(x, v_two); + HVX_Vector sig2x = hvx_vec_fast_sigmoid_f16(x2); + + const HVX_Vector v_neg_one = hvx_vec_splat_f16(-1.0f); + return hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(sig2x, v_two), v_neg_one); +} + +#define hvx_sigmoid_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / sizeof(_Float16); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_fast_sigmoid_f16(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector tmp = hvx_vec_fast_sigmoid_f16(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \ + } \ + } while(0) + +#define hvx_tanh_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / sizeof(_Float16); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_tanh_f16(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector tmp = hvx_vec_tanh_f16(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \ + } \ + } while(0) + +static inline void hvx_sigmoid_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_sigmoid_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_tanh_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_tanh_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + #endif /* HVX_SIGMOID_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h new file mode 100644 index 0000000..8648af0 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h @@ -0,0 +1,78 @@ +#ifndef HVX_SIN_COS_H +#define HVX_SIN_COS_H + +#include "hvx-base.h" +#include "hvx-floor.h" + +// Range-reduce x to y in [-pi/2, pi/2] and the quadrant sign (-1)^n. +// Floor/truncate need IEEE bits, so convert qf32 back to sf before them. +static inline void hvx_vec_sincos_reduce_f32(HVX_Vector x, HVX_Vector * y, HVX_Vector * sign) { + HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); + HVX_Vector const_half = hvx_vec_splat_f32(0.5f); + HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); + HVX_Vector const_one = hvx_vec_splat_f32(1.0f); + HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f); + HVX_Vector const_one_i = Q6_V_vsplat_R(1); + + HVX_Vector x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, const_inv_pi)); + x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(x_over_pi, const_half)); + + HVX_Vector n_float = hvx_vec_floor_f32(x_over_pi); + HVX_Vector n_int = hvx_vec_truncate_f32(n_float); + + HVX_Vector n_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(n_float, const_pi)); + *y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(x, n_pi)); + + HVX_VectorPred is_odd = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(n_int, const_one_i), const_one_i); + *sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); +} + +static inline void hvx_vec_sincos_f32(HVX_Vector x, HVX_Vector * vcos, HVX_Vector * vsin) { + HVX_Vector y; + HVX_Vector sign; + hvx_vec_sincos_reduce_f32(x, &y, &sign); + + HVX_Vector z = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, y)); + + HVX_Vector c4 = hvx_vec_splat_f32(2.3557242013849433e-05f); + HVX_Vector c3 = hvx_vec_splat_f32(-0.0013871428263450528f); + HVX_Vector c2 = hvx_vec_splat_f32(0.041665895266688284f); + HVX_Vector c1 = hvx_vec_splat_f32(-0.4999999360426369f); + HVX_Vector c0 = hvx_vec_splat_f32(0.9999999999071725f); + + HVX_Vector cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, c4)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); + + HVX_Vector s4 = hvx_vec_splat_f32(2.642186986152672e-06f); + HVX_Vector s3 = hvx_vec_splat_f32(-0.00019825318964070864f); + HVX_Vector s2 = hvx_vec_splat_f32(0.00833326283319605f); + HVX_Vector s1 = hvx_vec_splat_f32(-0.16666666082087775f); + HVX_Vector s0 = hvx_vec_splat_f32(0.999999999915155f); + + HVX_Vector sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, s4)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, sin_y)); + + *vcos = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(cos_y, sign)); + *vsin = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(sin_y, sign)); +} + +static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) { + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(x, &vcos, &vsin); + return vcos; +} + +static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) { + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(x, &vcos, &vsin); + return vsin; +} + +#endif /* HVX_SIN_COS_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-sqrt.h b/ggml/src/ggml-hexagon/htp/hvx-sqrt.h index e31a100..abdded5 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sqrt.h @@ -123,4 +123,67 @@ static inline void hvx_sqrt_f32(uint8_t * restrict dst, const uint8_t * restrict } } +// Compute sqrt(x) for f16 by promoting to f32, applying hvx_vec_rsqrt_f32, and narrowing back. +#define hvx_sqrt_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t nvec = n / VLEN_FP16; \ + const uint32_t nloe = n % VLEN_FP16; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \ + HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \ + vdst[i] = hvx_vec_f32_to_f16(r0, r1); \ + } \ + if (nloe) { \ + HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \ + HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \ + HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \ + HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \ + vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \ + } \ + } while(0) + +static inline void hvx_sqrt_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_sqrt_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_sqrt_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + hvx_sqrt_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a); +} + +static inline void hvx_sqrt_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) src % 128 == 0); + hvx_sqrt_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u); +} + +static inline void hvx_sqrt_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + hvx_sqrt_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u); +} + +static inline void hvx_sqrt_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int num_elems) { + if ((unsigned long) dst % 128 == 0) { + if ((unsigned long) src % 128 == 0) { + hvx_sqrt_f16_aa(dst, src, num_elems); + } else { + hvx_sqrt_f16_au(dst, src, num_elems); + } + } else { + if ((unsigned long) src % 128 == 0) { + hvx_sqrt_f16_ua(dst, src, num_elems); + } else { + hvx_sqrt_f16_uu(dst, src, num_elems); + } + } +} + #endif /* HVX_SQRT_H */ diff --git a/ggml/src/ggml-hexagon/htp/hvx-utils.h b/ggml/src/ggml-hexagon/htp/hvx-utils.h index a518ad3..dbdd7e0 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-utils.h +++ b/ggml/src/ggml-hexagon/htp/hvx-utils.h @@ -5,14 +5,21 @@ #include "hvx-types.h" #include "hvx-copy.h" +#include "hvx-repl.h" #include "hvx-scale.h" #include "hvx-exp.h" +#include "hvx-erf.h" #include "hvx-inverse.h" #include "hvx-reduce.h" #include "hvx-sigmoid.h" #include "hvx-sqrt.h" #include "hvx-arith.h" #include "hvx-div.h" +#include "hvx-floor.h" +#include "hvx-sin-cos.h" #include "hvx-base.h" +#include "hvx-pow.h" +#include "hvx-log.h" +#include "hvx-norm.h" #endif /* HVX_UTILS_H */ diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c new file mode 100644 index 0000000..fc49c40 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c @@ -0,0 +1,932 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include +#include +#include + +#include "hex-common.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "hvx-utils.h" +#include "dma-queue.h" +#include "hex-profile.h" +#include "htp-vtcm.h" +#include "htp-tensor.h" +#include "im2col-ops.h" + +struct htp_im2col_context { + struct htp_ops_context * octx; + + uint32_t pe_row_base; // first output row index assigned to this dev (DMA path) + uint32_t pe_nrows; // number of output rows assigned to this dev (DMA path) + uint32_t pe_rows_per_thread; // output rows per worker + uint32_t pe_src_row_bytes; // one output row's source, rounded 256 + uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256 + + // Patch-embed DMA path VTCM ping-pong. + uint8_t * pe_vtcm_src; // base of the 2x src buffers region + uint8_t * pe_vtcm_dst; // base of the 2x dst buffers region + uint32_t pe_src_size_per_thread; // 2 * pe_src_row_bytes + uint32_t pe_dst_size_per_thread; // 2 * pe_dst_row_bytes + + uint32_t pe_owb; // output-col block size + uint32_t pe_wb; // staged source window width + uint32_t pe_nbpr; // blocks per row + + // Precomputed dimensions for worker threads + uint32_t N; + uint32_t IC; + uint32_t ID, IH, IW; + uint32_t KD, KH, KW; + uint32_t OD, OH, OW; + uint32_t patch_stride; + + // Per-thread initial coordinates (removes division from thread prologue) + uint32_t thread_in[HTP_MAX_NTHREADS]; + uint32_t thread_iod[HTP_MAX_NTHREADS]; + uint32_t thread_ioh[HTP_MAX_NTHREADS]; + + void * compute; + uint32_t dst_elem; +}; + +typedef void (*im2col_blocked_compute_fn_t)( + const struct htp_im2col_context * ictx, + void * restrict dstb, const float * restrict srcb, + float * restrict scratch, + uint32_t nb, uint32_t c0, uint32_t ioh); + +typedef void (*im2col_patchembed_compute_fn_t)( + const struct htp_im2col_context * ictx, + void * restrict dstb, const float * restrict srcb, uint32_t ioh); + +typedef void (*im2col_3d_blocked_compute_fn_t)( + const struct htp_im2col_context * ictx, + void * restrict dstb, const float * restrict srcb, + float * restrict scratch, + uint32_t nb, uint32_t c0, uint32_t iod, uint32_t ioh); + +typedef void (*im2col_3d_patchembed_compute_fn_t)( + const struct htp_im2col_context * ictx, + void * restrict dstb, const float * restrict srcb); + +struct im2col_cursor { + uint32_t r; + uint32_t in; + uint32_t iod; + uint32_t ioh; + uint32_t c0; +}; + +static inline void im2col_step_row(struct im2col_cursor * c, const struct htp_im2col_context * ictx) { + c->r++; + c->ioh++; + if (c->ioh >= ictx->OH) { + c->ioh = 0; + c->in++; + } +} + +static inline void im2col_step_block(struct im2col_cursor * c, const struct htp_im2col_context * ictx) { + c->c0 += ictx->pe_owb; + if (c->c0 >= ictx->OW) { + c->c0 = 0; + c->r++; + c->ioh++; + if (c->ioh >= ictx->OH) { + c->ioh = 0; + c->in++; + } + } +} + +static void im2col_blocked_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q, + float * srcb, const struct im2col_cursor * c) { + const uint32_t in = c->in, ioh = c->ioh, c0 = c->c0; + const struct htp_ops_context * octx = ictx->octx; + const struct htp_tensor * src1 = octx->src[1]; + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; + const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; + const int32_t d1 = octx->op_params[5]; + const bool is_2D = (octx->op_params[6] == 1); + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; + const uint32_t KH = ictx->KH; + const uint32_t Wb = ictx->pe_wb; + const dma_addr_t src_data = src1->data; + const int32_t win0 = (int32_t) c0 * s0 - p0; + const size_t src_channel_stride = is_2D ? src1->nb[2] : src1->nb[1]; + const size_t batch_offset = is_2D ? (size_t) in * src1->nb[3] : (size_t) in * src1->nb[2]; + + for (uint32_t ikh = 0; ikh < KH; ikh++) { + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; + if (iih < 0 || iih >= (int32_t) IH) { + dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0); + continue; + } + const int32_t lo = win0 < 0 ? -win0 : 0; + int32_t hi = (int32_t) IW - win0; + if (hi > (int32_t) Wb) { + hi = (int32_t) Wb; + } + if (hi <= lo) { + dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0); + continue; + } + const uint32_t cpw = (uint32_t) (hi - lo); + float * vdst = srcb + (size_t) ikh * Wb + lo; + const dma_addr_t vsrc = src_data + batch_offset + (size_t) iih * src1->nb[1] + (size_t) (win0 + lo) * src1->nb[0]; + dma_queue_push(dma_q, dma_make_data(vdst, vsrc), + (size_t) KH * Wb * sizeof(float), src_channel_stride, + cpw * sizeof(float), IC); + } +} + +#define IM2COL_BLOCKED_COMPUTE_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN) \ +static void FNAME( \ + const struct htp_im2col_context * ictx, \ + void * restrict dstb_raw, const float * restrict srcb, \ + float * restrict scratch, \ + uint32_t nb, uint32_t c0, uint32_t ioh) { \ + DST_CTYPE * restrict dstb = (DST_CTYPE *) dstb_raw; \ + const struct htp_ops_context * octx = ictx->octx; \ + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; \ + const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; \ + const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; \ + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; \ + const uint32_t KH = ictx->KH, KW = ictx->KW; \ + const uint32_t Wb = ictx->pe_wb; \ + const uint32_t patch_stride = ictx->patch_stride; \ + const int32_t win0 = (int32_t) c0 * s0 - p0; \ + \ + for (uint32_t j = 0; j < nb; j++) { \ + const uint32_t iow = c0 + j; \ + DST_CTYPE * dst_patch = dstb + (size_t) j * patch_stride; \ + const int32_t iiw0 = (int32_t) iow * s0 - p0; \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ + const int okh = (iih >= 0 && iih < (int32_t) IH); \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + DST_CTYPE * out_run = dst_patch + (size_t) (iic * KH + ikh) * KW; \ + if (!okh) { \ + SPLAT_FN(out_run, 0, KW); \ + continue; \ + } \ + const float * vrow = srcb + (size_t) (iic * KH + ikh) * Wb; \ + if (d0 == 1) { \ + const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \ + int32_t hi = (int32_t) IW - iiw0; \ + if (hi > (int32_t) KW) { \ + hi = (int32_t) KW; \ + } \ + if (hi <= lo) { \ + SPLAT_FN(out_run, 0, KW); \ + } else { \ + if (lo > 0) { \ + SPLAT_FN(out_run, 0, (uint32_t) lo); \ + } \ + COPY_FN((uint8_t *) (out_run + lo), \ + (const uint8_t *) (vrow + (iiw0 + lo - win0)), \ + (uint32_t) (hi - lo)); \ + if (hi < (int32_t) KW) { \ + SPLAT_FN(out_run + hi, 0, (uint32_t) (KW - hi)); \ + } \ + } \ + continue; \ + } \ + for (uint32_t ikw = 0; ikw < KW; ikw += 64) { \ + const uint32_t chunk = MIN(KW - ikw, 64); \ + for (uint32_t c = 0; c < chunk; c++) { \ + const int32_t iiw = iiw0 + (int32_t) (ikw + c) * d0; \ + scratch[c] = (iiw < 0 || iiw >= (int32_t) IW) ? 0.0f : vrow[iiw - win0]; \ + } \ + COPY_FN((uint8_t *) (out_run + ikw), (const uint8_t *) scratch, chunk); \ + } \ + } \ + } \ + } \ +} + +IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u) +IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u) + +static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; + struct htp_ops_context * octx = ictx->octx; + struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + const uint32_t OW = ictx->OW; + const uint32_t KH = ictx->KH; + const uint32_t owb = ictx->pe_owb; + const uint32_t nbpr = ictx->pe_nbpr; + const uint32_t patch_stride = ictx->patch_stride; + const uint32_t dst_elem = ictx->dst_elem; + im2col_blocked_compute_fn_t compute = (im2col_blocked_compute_fn_t) ictx->compute; + const dma_addr_t dst_data = octx->dst->data; + dma_queue * dma_q = octx->ctx->dma[ith]; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; + void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; + float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes); + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + if (row_start >= row_end) { + return; + } + const uint32_t nrows_local = row_end - row_start; + const uint32_t total_blocks = nrows_local * nbpr; + if (total_blocks == 0) { + return; + } + struct im2col_cursor cur = { + .r = row_start, + .in = ictx->thread_in[ith], + .iod = 0, + .ioh = ictx->thread_ioh[ith], + .c0 = 0, + }; + struct im2col_cursor stage = cur; + for (uint32_t i = 0; i < total_blocks && i < 2; i++) { + dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0); + im2col_blocked_dma_stage(ictx, dma_q, srcb2[i], &stage); + im2col_step_block(&stage, ictx); + } + for (uint32_t bi = 0; bi < total_blocks; bi++) { + const uint32_t buf = bi & 1u; + float * srcb = srcb2[buf]; + void * dstb = dstb2[buf]; + const uint32_t nb = MIN(owb, OW - cur.c0); + dma_queue_pop(dma_q); + for (uint32_t k = 0; k < KH; k++) { + dma_queue_pop(dma_q); + } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + compute(ictx, dstb, srcb, scratch, nb, cur.c0, cur.ioh); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + const dma_addr_t ddr = dst_data + (size_t) (cur.r * OW + cur.c0) * patch_stride * dst_elem; + dma_queue_push(dma_q, dma_make_data(ddr, dstb), + nb * patch_stride * dst_elem, nb * patch_stride * dst_elem, + nb * patch_stride * dst_elem, 1); + if (bi + 2 < total_blocks) { + im2col_blocked_dma_stage(ictx, dma_q, srcb, &stage); + im2col_step_block(&stage, ictx); + } + im2col_step_block(&cur, ictx); + } + dma_queue_flush(dma_q); +} + +static void im2col_patchembed_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q, + float * srcb, const struct im2col_cursor * c) { + const struct htp_ops_context * octx = ictx->octx; + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; + const uint32_t KH = ictx->KH; + const uint32_t in = c->in, ioh = c->ioh; + const size_t src_channel_stride = (size_t) IH * IW * sizeof(float); + const size_t vtcm_channel_stride = (size_t) KH * IW * sizeof(float); + const dma_addr_t src_data = octx->src[1]->data; + const dma_addr_t vsrc = src_data + (size_t) (((in * IC) * IH + ioh * KH) * IW) * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(srcb, vsrc), + vtcm_channel_stride, src_channel_stride, + vtcm_channel_stride, IC); +} + +static void im2col_patchembed_compute_f16( + const struct htp_im2col_context * ictx, + void * restrict dstb_raw, const float * restrict srcb, uint32_t ioh) { + uint16_t * restrict dstb = (uint16_t *) dstb_raw; + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; + const uint32_t KH = ictx->KH, KW = ictx->KW; + const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride; + + for (uint32_t iow = 0; iow < OW; iow++) { + uint16_t * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t ikh = 0; ikh < KH; ikh++) { + const int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; + for (uint32_t iic = 0; iic < IC; iic++) { + uint16_t * out_run = dst_patch + (size_t) (iic * KH + ikh) * KW; + if (iih < 0 || iih >= (int32_t) IH) { + hvx_splat_u16_u(out_run, 0, KW); + continue; + } + const float * src_run = srcb + (size_t) (iic * KH + ikh) * IW + (size_t) iow * KW; + hvx_copy_f16_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW); + } + } + } +} + +static void im2col_patchembed_compute_f32( + const struct htp_im2col_context * ictx, + void * restrict dstb_raw, const float * restrict srcb, uint32_t ioh) { + float * restrict dstb = (float *) dstb_raw; + const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; + const uint32_t KH = ictx->KH, KW = ictx->KW; + const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride; + + for (uint32_t iow = 0; iow < OW; iow++) { + float * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t ikh = 0; ikh < KH; ikh++) { + const int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh; + for (uint32_t iic = 0; iic < IC; iic++) { + float * out_run = dst_patch + (size_t) (iic * KH + ikh) * KW; + if (iih < 0 || iih >= (int32_t) IH) { + hvx_splat_f32_u(out_run, 0, KW); + continue; + } + const float * src_run = srcb + (size_t) (iic * KH + ikh) * IW + (size_t) iow * KW; + hvx_copy_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW); + } + } + } +} + +static void im2col_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; + struct htp_ops_context * octx = ictx->octx; + struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + const uint32_t OW = ictx->OW; + const uint32_t patch_stride = ictx->patch_stride; + const uint32_t dst_elem = ictx->dst_elem; + im2col_patchembed_compute_fn_t compute = (im2col_patchembed_compute_fn_t) ictx->compute; + const dma_addr_t dst_data = octx->dst->data; + dma_queue * dma_q = octx->ctx->dma[ith]; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; + void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + if (row_start >= row_end) { + return; + } + const uint32_t total_rows = row_end - row_start; + if (total_rows == 0) { + return; + } + struct im2col_cursor cur = { + .r = row_start, + .in = ictx->thread_in[ith], + .iod = 0, + .ioh = ictx->thread_ioh[ith], + .c0 = 0, + }; + struct im2col_cursor stage = cur; + for (uint32_t i = 0; i < total_rows && i < 2; i++) { + dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0); + im2col_patchembed_dma_stage(ictx, dma_q, srcb2[i], &stage); + im2col_step_row(&stage, ictx); + } + for (uint32_t ri = 0; ri < total_rows; ri++) { + const uint32_t buf = ri & 1u; + float * srcb = srcb2[buf]; + void * dstb = dstb2[buf]; + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + compute(ictx, dstb, srcb, cur.ioh); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + const dma_addr_t ddr_row = dst_data + (size_t) cur.r * OW * patch_stride * dst_elem; + dma_queue_push(dma_q, dma_make_data(ddr_row, dstb), + OW * patch_stride * dst_elem, OW * patch_stride * dst_elem, + OW * patch_stride * dst_elem, 1); + if (ri + 2 < total_rows) { + im2col_patchembed_dma_stage(ictx, dma_q, srcb, &stage); + im2col_step_row(&stage, ictx); + } + im2col_step_row(&cur, ictx); + } + dma_queue_flush(dma_q); +} + +static inline void im2col_3d_step_row(struct im2col_cursor * c, const struct htp_im2col_context * ictx) { + c->r++; + c->ioh++; + if (c->ioh >= ictx->OH) { + c->ioh = 0; + c->iod++; + if (c->iod >= ictx->OD) { + c->iod = 0; + c->in++; + } + } +} + +static inline void im2col_3d_step_block(struct im2col_cursor * c, const struct htp_im2col_context * ictx) { + c->c0 += ictx->pe_owb; + if (c->c0 >= ictx->OW) { + c->c0 = 0; + c->r++; + c->ioh++; + if (c->ioh >= ictx->OH) { + c->ioh = 0; + c->iod++; + if (c->iod >= ictx->OD) { + c->iod = 0; + c->in++; + } + } + } +} + +static void im2col_3d_blocked_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q, + float * srcb, const struct im2col_cursor * c) { + const uint32_t in = c->in, iod = c->iod, ioh = c->ioh, c0 = c->c0; + const struct htp_ops_context * octx = ictx->octx; + const struct htp_tensor * src1 = octx->src[1]; + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1], s2 = octx->op_params[2]; + const int32_t p0 = octx->op_params[3], p1 = octx->op_params[4], p2 = octx->op_params[5]; + const int32_t d1 = octx->op_params[7], d2 = octx->op_params[8]; + const uint32_t IC = ictx->IC; + const uint32_t ID = ictx->ID, IH = ictx->IH, IW = ictx->IW; + const uint32_t KD = ictx->KD, KH = ictx->KH; + const uint32_t Wb = ictx->pe_wb; + const int32_t win0 = (int32_t) c0 * s0 - p0; + const dma_addr_t src_data = src1->data; + const size_t src_channel_stride = src1->nb[3]; + const size_t vtcm_channel_stride = (size_t) KD * KH * Wb * sizeof(float); + const size_t batch_offset = (size_t) (in * IC) * src1->nb[3]; + + for (uint32_t ikd = 0; ikd < KD; ikd++) { + const int32_t iid = (int32_t) iod * s2 + (int32_t) ikd * d2 - p2; + const bool valid_d = (iid >= 0 && iid < (int32_t) ID); + for (uint32_t ikh = 0; ikh < KH; ikh++) { + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; + if (!valid_d || iih < 0 || iih >= (int32_t) IH) { + dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0); + continue; + } + const int32_t lo = win0 < 0 ? -win0 : 0; + int32_t hi = (int32_t) IW - win0; + if (hi > (int32_t) Wb) { + hi = (int32_t) Wb; + } + if (hi <= lo) { + dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0); + continue; + } + float * vdst = srcb + (size_t) (ikd * KH + ikh) * Wb + lo; + const dma_addr_t vsrc = src_data + batch_offset + + (size_t) iid * src1->nb[2] + + (size_t) iih * src1->nb[1] + + (size_t) (win0 + lo) * src1->nb[0]; + dma_queue_push(dma_q, dma_make_data(vdst, vsrc), + vtcm_channel_stride, src_channel_stride, + (size_t) (hi - lo) * sizeof(float), IC); + } + } +} + +#define IM2COL_3D_BLOCKED_COMPUTE_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN) \ +static void FNAME(const struct htp_im2col_context * ictx, \ + void * restrict dstb_raw, const float * restrict srcb, \ + float * restrict scratch, \ + uint32_t nb, uint32_t c0, uint32_t iod, uint32_t ioh) { \ + DST_CTYPE * restrict dstb = (DST_CTYPE *) dstb_raw; \ + const struct htp_ops_context * octx = ictx->octx; \ + const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1], s2 = octx->op_params[2]; \ + const int32_t p0 = octx->op_params[3], p1 = octx->op_params[4], p2 = octx->op_params[5]; \ + const int32_t d0 = octx->op_params[6], d1 = octx->op_params[7], d2 = octx->op_params[8]; \ + const uint32_t IC = ictx->IC; \ + const uint32_t ID = ictx->ID, IH = ictx->IH, IW = ictx->IW; \ + const uint32_t KD = ictx->KD, KH = ictx->KH, KW = ictx->KW; \ + const uint32_t patch_stride = ictx->patch_stride; \ + const uint32_t Wb = ictx->pe_wb; \ + const int32_t win0 = (int32_t) c0 * s0 - p0; \ + for (uint32_t j = 0; j < nb; j++) { \ + const uint32_t iow = c0 + j; \ + const int32_t iiw0 = (int32_t) iow * s0 - p0; \ + DST_CTYPE * dst_patch = dstb + (size_t) j * patch_stride; \ + for (uint32_t iic = 0; iic < IC; iic++) { \ + for (uint32_t ikd = 0; ikd < KD; ikd++) { \ + const int32_t iid = (int32_t) iod * s2 + (int32_t) ikd * d2 - p2; \ + for (uint32_t ikh = 0; ikh < KH; ikh++) { \ + const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \ + DST_CTYPE * out_run = dst_patch + \ + (size_t) (iic * KD * KH + ikd * KH + ikh) * KW; \ + if (iid < 0 || iid >= (int32_t) ID || iih < 0 || iih >= (int32_t) IH) { \ + SPLAT_FN(out_run, 0, KW); \ + continue; \ + } \ + const float * vrow = srcb + (size_t) (iic * KD * KH + ikd * KH + ikh) * Wb; \ + if (d0 == 1) { \ + const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \ + int32_t hi = (int32_t) IW - iiw0; \ + if (hi > (int32_t) KW) { \ + hi = (int32_t) KW; \ + } \ + if (hi <= lo) { \ + SPLAT_FN(out_run, 0, KW); \ + } else { \ + if (lo > 0) { \ + SPLAT_FN(out_run, 0, (uint32_t) lo); \ + } \ + COPY_FN((uint8_t *) (out_run + lo), \ + (const uint8_t *) (vrow + (iiw0 + lo - win0)), \ + (uint32_t) (hi - lo)); \ + if (hi < (int32_t) KW) { \ + SPLAT_FN(out_run + hi, 0, (uint32_t) (KW - hi)); \ + } \ + } \ + continue; \ + } \ + for (uint32_t ikw = 0; ikw < KW; ikw += 64) { \ + const uint32_t chunk = MIN(KW - ikw, 64); \ + for (uint32_t c = 0; c < chunk; c++) { \ + const int32_t iiw = iiw0 + (int32_t) (ikw + c) * d0; \ + scratch[c] = (iiw < 0 || iiw >= (int32_t) IW) ? 0.0f : vrow[iiw - win0]; \ + } \ + COPY_FN((uint8_t *) (out_run + ikw), (const uint8_t *) scratch, chunk); \ + } \ + } \ + } \ + } \ + } \ +} + +IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u) +IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u) + +static void im2col_3d_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; + struct htp_ops_context * octx = ictx->octx; + struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + const struct htp_tensor * restrict dst = octx->dst; + const uint32_t OW = ictx->OW; + const uint32_t KD = ictx->KD, KH = ictx->KH; + const uint32_t kd_kh = KD * KH; + const uint32_t patch_stride = ictx->patch_stride; + const uint32_t owb = ictx->pe_owb; + const uint32_t nbpr = ictx->pe_nbpr; + const uint32_t dst_elem = ictx->dst_elem; + const dma_addr_t dst_data = dst->data; + dma_queue * dma_q = octx->ctx->dma[ith]; + uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) }; + void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; + float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes); + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + if (row_start >= row_end) { + return; + } + const uint32_t nrows_local = row_end - row_start; + const uint32_t total_blocks = nrows_local * nbpr; + if (total_blocks == 0) { + return; + } + struct im2col_cursor cur = { + .r = row_start, + .in = ictx->thread_in[ith], + .iod = ictx->thread_iod[ith], + .ioh = ictx->thread_ioh[ith], + .c0 = 0, + }; + struct im2col_cursor stage = cur; + for (uint32_t i = 0; i < total_blocks && i < 2; i++) { + dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0); + im2col_3d_blocked_dma_stage(ictx, dma_q, srcb2[i], &stage); + im2col_3d_step_block(&stage, ictx); + } + + im2col_3d_blocked_compute_fn_t compute = (im2col_3d_blocked_compute_fn_t) ictx->compute; + for (uint32_t bi = 0; bi < total_blocks; bi++) { + const uint32_t buf = bi & 1u; + float * srcb = srcb2[buf]; + void * dstb = dstb2[buf]; + const uint32_t nb = MIN(owb, OW - cur.c0); + dma_queue_pop(dma_q); + for (uint32_t p = 0; p < kd_kh; p++) { + dma_queue_pop(dma_q); + } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + compute(ictx, dstb, srcb, scratch, nb, cur.c0, cur.iod, cur.ioh); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + const dma_addr_t ddr = dst_data + (size_t) (cur.r * OW + cur.c0) * patch_stride * dst_elem; + dma_queue_push(dma_q, dma_make_data(ddr, dstb), + nb * patch_stride * dst_elem, nb * patch_stride * dst_elem, + nb * patch_stride * dst_elem, 1); + if (bi + 2 < total_blocks) { + im2col_3d_blocked_dma_stage(ictx, dma_q, srcb, &stage); + im2col_3d_step_block(&stage, ictx); + } + im2col_3d_step_block(&cur, ictx); + } + dma_queue_flush(dma_q); +} +static void im2col_3d_patchembed_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q, + float * srcb, const struct im2col_cursor * c) { + const struct htp_ops_context * octx = ictx->octx; + const uint32_t IC = ictx->IC; + const uint32_t ID = ictx->ID, IH = ictx->IH, IW = ictx->IW; + const uint32_t KD = ictx->KD, KH = ictx->KH; + const dma_addr_t src_data = octx->src[1]->data; + const uint32_t in = c->in, iod = c->iod, ioh = c->ioh; + const size_t src_channel_stride = (size_t) ID * IH * IW * sizeof(float); + const size_t vtcm_channel_stride = (size_t) KD * KH * IW * sizeof(float); + const size_t row_size = (size_t) KH * IW * sizeof(float); + + for (uint32_t ikd = 0; ikd < KD; ikd++) { + const uint32_t iid = iod * KD + ikd; + float * vdst = srcb + (size_t) ikd * KH * IW; + const dma_addr_t vsrc = src_data + (size_t) (((in * IC) * ID + iid) * IH + ioh * KH) * IW * sizeof(float); + dma_queue_push(dma_q, dma_make_data(vdst, vsrc), + vtcm_channel_stride, src_channel_stride, row_size, IC); + } +} +static void im2col_3d_patchembed_compute_f16( + const struct htp_im2col_context * ictx, + void * restrict dstb_raw, const float * restrict srcb) { + uint16_t * restrict dstb = (uint16_t *) dstb_raw; + const uint32_t OW = ictx->OW; + const uint32_t IC = ictx->IC; + const uint32_t KD = ictx->KD; + const uint32_t KH = ictx->KH; + const uint32_t KW = ictx->KW; + const uint32_t IW = ictx->IW; + const uint32_t patch_stride = ictx->patch_stride; + + for (uint32_t iow = 0; iow < OW; iow++) { + uint16_t * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t iic = 0; iic < IC; iic++) { + for (uint32_t ikd = 0; ikd < KD; ikd++) { + for (uint32_t ikh = 0; ikh < KH; ikh++) { + uint16_t * out_run = dst_patch + (size_t) (iic * KD * KH + ikd * KH + ikh) * KW; + const float * src_run = srcb + (size_t) (iic * KD * KH + ikd * KH + ikh) * IW + (size_t) iow * KW; + hvx_copy_f16_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW); + } + } + } + } +} + +static void im2col_3d_patchembed_compute_f32( + const struct htp_im2col_context * ictx, + void * restrict dstb_raw, const float * restrict srcb) { + float * restrict dstb = (float *) dstb_raw; + const uint32_t OW = ictx->OW; + const uint32_t IC = ictx->IC; + const uint32_t KD = ictx->KD; + const uint32_t KH = ictx->KH; + const uint32_t KW = ictx->KW; + const uint32_t IW = ictx->IW; + const uint32_t patch_stride = ictx->patch_stride; + + for (uint32_t iow = 0; iow < OW; iow++) { + float * dst_patch = dstb + (size_t) iow * patch_stride; + for (uint32_t iic = 0; iic < IC; iic++) { + for (uint32_t ikd = 0; ikd < KD; ikd++) { + for (uint32_t ikh = 0; ikh < KH; ikh++) { + float * out_run = dst_patch + (size_t) (iic * KD * KH + ikd * KH + ikh) * KW; + const float * src_run = srcb + (size_t) (iic * KD * KH + ikd * KH + ikh) * IW + (size_t) iow * KW; + hvx_copy_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW); + } + } + } + } +} + +static void im2col_3d_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; + struct htp_ops_context * octx = ictx->octx; + struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; + const struct htp_tensor * restrict dst = octx->dst; + const uint32_t KD = ictx->KD; + const uint32_t OW = ictx->OW; + const uint32_t patch_stride = ictx->patch_stride; + const uint32_t dst_elem = ictx->dst_elem; + const dma_addr_t dst_data = dst->data; + dma_queue * dma_q = octx->ctx->dma[ith]; + const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; + const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith; + const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max); + if (row_start >= row_end) { + return; + } + + const uint32_t nrows_local = row_end - row_start; + uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread; + uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; + float * srcb2[2] = { (float *) src_base, (float *) (src_base + ictx->pe_src_row_bytes) }; + void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes }; + struct im2col_cursor cur = { + .r = row_start, + .in = ictx->thread_in[ith], + .iod = ictx->thread_iod[ith], + .ioh = ictx->thread_ioh[ith], + .c0 = 0, + }; + struct im2col_cursor stage = cur; + for (uint32_t i = 0; i < nrows_local && i < 2; i++) { + dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0); + im2col_3d_patchembed_dma_stage(ictx, dma_q, srcb2[i], &stage); + im2col_3d_step_row(&stage, ictx); + } + + im2col_3d_patchembed_compute_fn_t compute = (im2col_3d_patchembed_compute_fn_t) ictx->compute; + for (uint32_t idx = 0; idx < nrows_local; idx++) { + const uint32_t buf = idx & 1u; + float * srcb = srcb2[buf]; + void * dstb = dstb2[buf]; + dma_queue_pop(dma_q); + for (uint32_t p = 0; p < KD; p++) { + dma_queue_pop(dma_q); + } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + compute(ictx, dstb, srcb); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r); + const dma_addr_t ddr_row = dst_data + (size_t) cur.r * OW * patch_stride * dst_elem; + dma_queue_push(dma_q, dma_make_data(ddr_row, dstb), + OW * patch_stride * dst_elem, OW * patch_stride * dst_elem, + OW * patch_stride * dst_elem, 1); + if (idx + 2 < nrows_local) { + im2col_3d_patchembed_dma_stage(ictx, dma_q, srcb, &stage); + im2col_3d_step_row(&stage, ictx); + } + im2col_3d_step_row(&cur, ictx); + } + dma_queue_flush(dma_q); +} + +int op_im2col(struct htp_ops_context * octx) { + const struct htp_im2col_kernel_params * kparams = (const struct htp_im2col_kernel_params *) octx->kernel_params; + if (kparams->kernel_type == HTP_IM2COL_KERNEL_UNSUPPORTED) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_tensor * dst = octx->dst; + + uint32_t row_base = 0; + uint32_t nrows = kparams->total_rows; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = + htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_base = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads); + + struct htp_im2col_context ictx = { 0 }; + uint8_t * const base = octx->ctx->vtcm_base; + + const uint32_t pth = octx->n_threads; + ictx.octx = octx; + ictx.pe_row_base = row_base; + ictx.pe_nrows = nrows; + ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div); + ictx.pe_src_row_bytes = kparams->src_row_bytes; + ictx.pe_dst_row_bytes = kparams->dst_row_bytes; + ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src); + ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst); + ictx.pe_src_size_per_thread = layout.src_bytes_per_thread; + ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread; + ictx.pe_owb = kparams->owb; + ictx.pe_wb = kparams->wb; + ictx.pe_nbpr = kparams->nbpr; + ictx.N = kparams->N; + ictx.IC = kparams->IC; + ictx.ID = kparams->ID; + ictx.IH = kparams->IH; + ictx.IW = kparams->IW; + ictx.KD = kparams->KD; + ictx.KH = kparams->KH; + ictx.KW = kparams->KW; + ictx.OD = kparams->OD; + ictx.OH = kparams->OH; + ictx.OW = kparams->OW; + ictx.patch_stride = kparams->patch_stride; + + for (uint32_t ith = 0; ith < pth; ith++) { + const uint32_t r = row_base + ictx.pe_rows_per_thread * ith; + const uint32_t in = fastdiv(r, &kparams->div_OH); + ictx.thread_ioh[ith] = r - in * kparams->OH; + ictx.thread_in[ith] = in; + } + + ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float); + + if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) { + ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_patchembed_compute_f16 + : (void *) im2col_patchembed_compute_f32; + work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth); + return HTP_STATUS_OK; + } else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) { + ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16 + : (void *) im2col_blocked_compute_f32; + work_queue_run(octx->ctx->work_queue, im2col_blocked_dma_thread, &ictx, pth); + return HTP_STATUS_OK; + } + + return HTP_STATUS_NO_SUPPORT; +} + +int op_im2col_3d(struct htp_ops_context * octx) { + const struct htp_im2col_kernel_params * kparams = + (const struct htp_im2col_kernel_params *) octx->kernel_params; + if (kparams->kernel_type == HTP_IM2COL_KERNEL_UNSUPPORTED) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_tensor * dst = octx->dst; + + uint32_t row_base = 0; + uint32_t nrows = kparams->total_rows; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = + htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_base = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } + + struct htp_im2col_vtcm_layout layout; + htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads); + + struct htp_im2col_context ictx = { 0 }; + uint8_t * const base = octx->ctx->vtcm_base; + + const uint32_t pth = octx->n_threads; + ictx.octx = octx; + ictx.pe_row_base = row_base; + ictx.pe_nrows = nrows; + ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div); + ictx.pe_src_row_bytes = kparams->src_row_bytes; + ictx.pe_dst_row_bytes = kparams->dst_row_bytes; + ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src); + ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst); + ictx.pe_src_size_per_thread = layout.src_bytes_per_thread; + ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread; + ictx.pe_owb = kparams->owb; + ictx.pe_wb = kparams->wb; + ictx.pe_nbpr = kparams->nbpr; + ictx.N = kparams->N; + ictx.IC = kparams->IC; + ictx.ID = kparams->ID; + ictx.IH = kparams->IH; + ictx.IW = kparams->IW; + ictx.KD = kparams->KD; + ictx.KH = kparams->KH; + ictx.KW = kparams->KW; + ictx.OD = kparams->OD; + ictx.OH = kparams->OH; + ictx.OW = kparams->OW; + ictx.patch_stride = kparams->patch_stride; + + for (uint32_t ith = 0; ith < pth; ith++) { + const uint32_t rem = row_base + ictx.pe_rows_per_thread * ith; + const uint32_t q_oh = fastdiv(rem, &kparams->div_OH); + ictx.thread_ioh[ith] = rem - q_oh * kparams->OH; + const uint32_t in = fastdiv(q_oh, &kparams->div_OD); + ictx.thread_iod[ith] = q_oh - in * kparams->OD; + ictx.thread_in[ith] = in; + } + + ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float); + + if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) { + ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_patchembed_compute_f16 + : (void *) im2col_3d_patchembed_compute_f32; + work_queue_run(octx->ctx->work_queue, im2col_3d_patchembed_dma_thread, &ictx, pth); + return HTP_STATUS_OK; + } else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) { + ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_blocked_compute_f16 + : (void *) im2col_3d_blocked_compute_f32; + work_queue_run(octx->ctx->work_queue, im2col_3d_blocked_dma_thread, &ictx, pth); + return HTP_STATUS_OK; + } + + return HTP_STATUS_NO_SUPPORT; +} diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.h b/ggml/src/ggml-hexagon/htp/im2col-ops.h new file mode 100644 index 0000000..85aa3ca --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.h @@ -0,0 +1,69 @@ +#ifndef HTP_IM2COL_OPS_H +#define HTP_IM2COL_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include + +enum htp_im2col_kernel_type { + HTP_IM2COL_KERNEL_UNSUPPORTED = 0, + HTP_IM2COL_KERNEL_PATCHEMBED_DMA = 1, + HTP_IM2COL_KERNEL_BLOCKED_DMA = 2, +}; + +struct htp_im2col_vtcm_layout { + size_t total_bytes; + size_t off_src; + size_t off_dst; + uint32_t src_bytes_per_thread; + uint32_t dst_bytes_per_thread; +}; + +static inline void htp_im2col_vtcm_layout_build( + struct htp_im2col_vtcm_layout * L, + uint32_t src_row_bytes, + uint32_t dst_row_bytes, + uint32_t n_threads) { + + L->src_bytes_per_thread = 2 * src_row_bytes + 256; + L->dst_bytes_per_thread = 2 * dst_row_bytes; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_thread * n_threads); + L->total_bytes = off; +} + +struct htp_im2col_kernel_params { + uint8_t kernel_type; + uint8_t n_threads; + + uint32_t vtcm_size; + uint32_t src_row_bytes; + uint32_t dst_row_bytes; + + uint32_t owb; + uint32_t wb; + uint32_t nbpr; + + uint32_t N; + uint32_t IC; + uint32_t ID, IH, IW; + uint32_t KD, KH, KW; + uint32_t OD, OH, OW; + uint32_t patch_stride; + uint32_t total_rows; + uint32_t rows_per_chunk; + + struct fastdiv_values div_OH; + struct fastdiv_values div_OD; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_im2col_kernel_params) <= 128, "htp_im2col_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_im2col_kernel_params) <= 128, "htp_im2col_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_IM2COL_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 5091623..414c6f6 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -12,162 +12,231 @@ #include #include #include +#include #include #include #include #include #include +#include #include "hex-utils.h" -#include "hex-dma.h" +#include "dma-queue.h" #include "hmx-queue.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-bitmap.h" #include "htp-ctx.h" +#include "htp-dcache.h" #include "htp-ops.h" -#include "htp-ops.h" -#include "worker-pool.h" - -AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) { +#include "htp_iface.h" +#include "work-queue.h" +#include "hex-profile.h" +#include "allreduce-ops.h" +#include "htp-fence.h" + +#define HMX_QUEUE_CAPACITY 128 +#define HMX_QUEUE_STACK_SIZE 16384 +#define WORK_QUEUE_CAPACITY 16 +#define WORK_QUEUE_STACK_SIZE 16384 +#define MAIN_THREAD_STACK_SIZE 32768 + +_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS, + "work-queue thread cap must be >= HTP_MAX_NTHREADS"); + +struct htp_handle { struct htp_context * ctx; - int err = 0; - - ctx = calloc(1, sizeof(*ctx)); - if (ctx == NULL) { - return AEE_ENOMEMORY; - } - - // Use the context structure as the handle - *handle = (remote_handle64) ctx; - - // Enable FARF logs - HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0); - - // Set client class - { - HAP_power_request_t request; - memset(&request, 0, sizeof(HAP_power_request_t)); - request.type = HAP_power_set_apptype; - request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS; +}; - if ((err = HAP_power_set((void *) ctx, &request)) != 0) { - return err; +static inline uint64_t htp_mmap(uint32_t fd, uint64_t size, uint32_t flags) { +#if __HVX_ARCH__ > 79 + if (flags & HTP_BUF_EXTENDED) { + HAP_mem_req_payload_t payload; + memset(&payload, 0, sizeof(payload)); + payload.request_id = HAP_MEM_MAP; + payload.mmap.len = size; + payload.mmap.prot = HAP_MEM_CACHE_NON_SHARED | HAP_PROT_READ; + payload.mmap.flags = HAP_MEM_FLAGS_EXTENDED_MAP; + payload.mmap.fd = fd; + + if (HAP_mem_request(&payload) != 0) { + FARF(ERROR, "extended mmap failed : fd %u size %llu", fd, (unsigned long long) size); + return 0; } + + return payload.mmap.dsp_va; + } +#else + if (flags & HTP_BUF_EXTENDED) { + FARF(ERROR, "extended mmap is unsupported on v%d", __HVX_ARCH__); + return 0; } +#endif - { - HAP_power_request_t request; - memset(&request, 0, sizeof(request)); + if (size > UINT32_MAX) { + FARF(ERROR, "mmap failed : size %llu exceeds 32-bit limit", (unsigned long long) size); + return 0; + } - request.type = HAP_power_set_DCVS_v3; - request.dcvs_v3.set_dcvs_enable = TRUE; - request.dcvs_v3.dcvs_enable = TRUE; - request.dcvs_v3.dcvs_option = HAP_DCVS_V2_PERFORMANCE_MODE; - request.dcvs_v3.set_bus_params = TRUE; - request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.set_core_params = TRUE; - request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX; - request.dcvs_v3.set_sleep_disable = TRUE; - request.dcvs_v3.sleep_disable = TRUE; - if ((err = HAP_power_set((void *) ctx, &request)) != 0) { - return err; + void * va = (void *)-1; + for (int retry = 0; retry < 2; retry++) { +#if __HVX_ARCH__ > 73 + va = HAP_mmap2(NULL, (size_t) size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); +#else + if (size > HTP_MMAP_MAX_VMEM) { + FARF(ERROR, "mmap failed : size %llu exceeds 2GB limit for HAP_mmap", (unsigned long long) size); + abort(); } - - memset(&request, 0, sizeof(request)); - request.type = HAP_power_set_HVX; - request.hvx.power_up = TRUE; - if ((err = HAP_power_set((void *) ctx, &request)) != 0) { - return err; + va = HAP_mmap(NULL, (int) size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); +#endif + if (va != (void *)-1 && va != NULL) { + return (uint64_t) (uintptr_t) va; } - } - - { - // Power on HMX - HAP_power_request_t request; - memset(&request, 0, sizeof(HAP_power_request_t)); - request.type = HAP_power_set_HMX; - request.hmx.power_up = TRUE; - FARF(ALWAYS, "Powering HMX on\n"); - err = HAP_power_set((void *) &ctx, &request); - if (err != AEE_SUCCESS) { - FARF(ERROR, "Error powering on HMX."); - return err; + if (retry == 0) { + FARF(HIGH, "mmap failed first try (va %p fd %u size %llu), retrying...", va, fd, (unsigned long long) size); } } - - return AEE_SUCCESS; + return 0; } -AEEResult htp_iface_close(remote_handle64 handle) { - struct htp_context * ctx = (struct htp_context *) handle; - - if (!ctx) { - return AEE_EBADPARM; - } - - if (ctx->queue) { - FARF(ERROR, "Closing handle with queue still open"); - return AEE_EITEMBUSY; +static inline void htp_munmap(uint64_t va, uint64_t size, uint32_t flags) { +#if __HVX_ARCH__ > 79 + if (flags & HTP_BUF_EXTENDED) { + HAP_mem_req_payload_t payload; + memset(&payload, 0, sizeof(payload)); + payload.request_id = HAP_MEM_UNMAP; + payload.munmap.dsp_va = va; + payload.munmap.len = size; + HAP_mem_request(&payload); + return; } +#endif - // release the mmaps (if any) - for (uint32_t i=0; immap[i].size) { #if __HVX_ARCH__ > 73 - HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size); + HAP_munmap2((void *) (uintptr_t) va, (size_t) size); #else - HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size); + HAP_munmap((void *) (uintptr_t) va, (int) size); #endif - ctx->mmap[i].size = 0; - ctx->mmap[i].base = NULL; - ctx->mmap[i].fd = -1; - } +} + +AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) { + (void) uri; + struct htp_handle * h = calloc(1, sizeof(*h)); + if (h == NULL) { + return AEE_ENOMEMORY; } - free(ctx); + *handle = (remote_handle64) h; return AEE_SUCCESS; } -AEEResult htp_iface_enable_etm(remote_handle64 handle) { - int err = HAP_user_etm_enable(); +AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h) { + return AEE_EBADPARM; + } + + int err = enable ? HAP_user_etm_enable() : HAP_user_etm_disable(); if (err) { if (err == AEE_EVERSIONNOTSUPPORT) { - FARF(ERROR, "API HAP_user_etm_enable is not supported\n"); + FARF(ERROR, "API HAP_user_etm_enable/disable is not supported\n"); } else { - FARF(ERROR, "Error executing HAP_user_etm_enable with error code : 0x%x\n", err); + FARF(ERROR, "Error executing HAP_user_etm_enable/disable with error code : 0x%x\n", err); } } return err; } -AEEResult htp_iface_disable_etm(remote_handle64 handle) { - int err = HAP_user_etm_disable(); - if (err) { - if (err == AEE_EVERSIONNOTSUPPORT) { - FARF(ERROR, "API HAP_user_etm_disable is not supported\n"); - } else { - FARF(ERROR, "Error executing HAP_user_etm_disable with error code : 0x%x\n", err); +AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_iface_pmu_conf* pmu_conf) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { + return AEE_EBADPARM; + } + struct htp_context * ctx = h->ctx; + + if (mode == HTP_PROF_PMU) { + const uint32_t* events = pmu_conf->events; + + // Pack 4 event IDs (low 8 bits) into each 32-bit config register + uint32_t evtcfg = 0, evtcfg1 = 0, cfg = 0, i = 0; + for (; i < HEX_NUM_PMU_COUNTERS/2; i++) { + evtcfg |= ((events[i + 0] & 0xFF) << (i * 8)); + evtcfg1 |= ((events[i + 4] & 0xFF) << (i * 8)); + } + + // For events >255 pack high 2 bits of all 8 event IDs into cfg register + // 2 bits per counter: bits [1:0] for counter 0, [3:2] for counter 1, etc. + for (i = 0; i < HEX_NUM_PMU_COUNTERS; i++) { + cfg |= (((events[i] >> 8) & 3) << (i * 2)); } + + FARF(ALWAYS, "Configuring PMU registers: evtcfg = 0x%x, evtcfg1 = 0x%x, pmucfg = 0x%x", evtcfg, evtcfg1, cfg); + + // Configure PMU registers + qurt_pmu_set(QURT_PMUCFG, cfg); + qurt_pmu_set(QURT_PMUEVTCFG, evtcfg); + qurt_pmu_set(QURT_PMUEVTCFG1, evtcfg1); + qurt_pmu_enable(1); } - return err; + + ctx->profiler = mode; + + return AEE_SUCCESS; } -AEEResult htp_iface_mmap(remote_handle64 handle, int fd, uint32_t size, uint32_t pinned) { - struct htp_context * ctx = (struct htp_context *) handle; - if (!ctx) { +AEEResult htp_iface_close(remote_handle64 handle) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; + if (ctx) { + if (ctx->dsp_queue) { + FARF(ERROR, "Closing handle with queue still open"); + return AEE_EITEMBUSY; + } + + // release the mmaps (if any) + for (uint32_t i=0; immap[i].size) { + htp_munmap(ctx->mmap[i].base, ctx->mmap[i].size, ctx->mmap[i].flags); + ctx->mmap[i].size = 0; + ctx->mmap[i].base = 0; + ctx->mmap[i].fd = -1; + ctx->mmap[i].flags = 0; + } + } + + if (ctx->profiler) { + qurt_pmu_enable(1); + } + + if (ctx->etm) { + HAP_user_etm_disable(); + } + + // Free the unified block (ctx is the base address of the block) + free(ctx); + h->ctx = NULL; + } + + free(h); + return AEE_SUCCESS; +} + +AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint64_t size) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { + return AEE_EBADPARM; + } + struct htp_context * ctx = h->ctx; + // See if we already have this mapping for (uint32_t i=0; immap[i]; if (m->fd == fd) { - m->pinned = pinned; return AEE_SUCCESS; } } @@ -176,26 +245,17 @@ AEEResult htp_iface_mmap(remote_handle64 handle, int fd, uint32_t size, uint32_t for (uint32_t i=0; immap[i]; if (!m->size) { - FARF(HIGH, "mmap : fd %u size %u pinned %u", fd, size, pinned); -#if __HVX_ARCH__ > 73 - void *va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); -#else - if (size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB - FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size); - abort(); // can't do much else at this point - } - - void *va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); -#endif - if (va == (void*)-1) { - FARF(ERROR, "mmap failed : va %p fd %u size %u", va, fd, (uint32_t) size); + FARF(HIGH, "mmap : fd %u size %llu", fd, (unsigned long long) size); + uint64_t va = htp_mmap(fd, size, 0); + if (va == 0) { + FARF(ERROR, "mmap failed : fd %u size %llu", fd, (unsigned long long) size); return AEE_EFAILED; } - m->base = (uint64_t) va; + m->base = va; m->fd = fd; m->size = size; - m->pinned = pinned; + m->flags = 0; return AEE_SUCCESS; } @@ -204,25 +264,22 @@ AEEResult htp_iface_mmap(remote_handle64 handle, int fd, uint32_t size, uint32_t return AEE_ENOMEMORY; } -AEEResult htp_iface_munmap(remote_handle64 handle, int fd) { - struct htp_context * ctx = (struct htp_context *) handle; - if (!ctx) { +AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; for (uint32_t i=0; immap[i]; if (fd < 0 || m->fd == fd) { - FARF(HIGH, "unmmap : base %p fd %u size %u", (void*) m->base, m->fd, (uint32_t) m->size); -#if __HVX_ARCH__ > 73 - HAP_munmap2((void *) m->base, m->size); -#else - HAP_munmap((void *) m->base, m->size); -#endif + FARF(HIGH, "unmmap : base 0x%llx fd %u size %llu", (unsigned long long) m->base, m->fd, (unsigned long long) m->size); + htp_munmap(m->base, m->size, m->flags); m->size = 0; m->base = NULL; m->fd = -1; - m->pinned = 0; + m->flags = 0; } } @@ -230,15 +287,15 @@ AEEResult htp_iface_munmap(remote_handle64 handle, int fd) { } static void vtcm_acquire(struct htp_context * ctx) { - if (!ctx->vtcm_valid) { - int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 1000000u); + if (!atomic_load(&ctx->vtcm_valid)) { + int err = HAP_compute_res_acquire_cached(ctx->vtcm_rctx, 10000000u); if (err != 0) { FARF(ERROR, "ggml-hex: failed to acquire VTCM: 0x%08x", (unsigned)err); abort(); } - ctx->vtcm_needs_release = false; - ctx->vtcm_valid = true; + atomic_store(&ctx->vtcm_needs_release, false); + atomic_store(&ctx->vtcm_valid, true); // Drop the priority to make sure we get the release callback from other GGML-HTP and QNN-HTP sessions HAP_compute_res_update_priority(ctx->vtcm_rctx, ctx->thread_prio + 10); @@ -246,22 +303,33 @@ static void vtcm_acquire(struct htp_context * ctx) { } static void vtcm_release(struct htp_context * ctx) { - if (ctx->vtcm_valid) { - ctx->vtcm_valid = false; - ctx->vtcm_needs_release = false; + if (atomic_load(&ctx->vtcm_valid)) { + atomic_store(&ctx->vtcm_valid, false); + atomic_store(&ctx->vtcm_needs_release, false); HAP_compute_res_release_cached(ctx->vtcm_rctx); } } static int vtcm_release_callback(unsigned int rctx, void * state) { struct htp_context * ctx = (struct htp_context *) state; - ctx->vtcm_needs_release = true; + atomic_store(&ctx->vtcm_needs_release, true); return 0; } -static int vtcm_alloc(struct htp_context * ctx) { +static unsigned int vtcm_query_size(void) { unsigned int vtcm_size = 8 * 1024 * 1024; // 8MB default - HAP_compute_res_query_VTCM(0, &vtcm_size, NULL, NULL, NULL); + unsigned int vtcm_avail; + compute_res_vtcm_page_t total_layout, avail_layout; + // some images (e.g. FuSa cDSP) fail the query when the layout out-params are NULL + HAP_compute_res_query_VTCM(0, &vtcm_size, &total_layout, &vtcm_avail, &avail_layout); + return vtcm_size; +} + +static int vtcm_alloc(struct htp_context * ctx, uint64_t size_override) { + unsigned int vtcm_size = vtcm_query_size(); + if (size_override) { + vtcm_size = (unsigned int) size_override; + } compute_res_attr_t attr; HAP_compute_res_attr_init(&attr); @@ -288,8 +356,8 @@ static int vtcm_alloc(struct htp_context * ctx) { ctx->vtcm_base = (uint8_t *) vtcm_ptr; ctx->vtcm_size = vtcm_size; ctx->vtcm_rctx = rctx; - ctx->vtcm_valid = false; - ctx->vtcm_needs_release = false; + atomic_store(&ctx->vtcm_valid, false); + atomic_store(&ctx->vtcm_needs_release, false); return 0; } @@ -302,81 +370,340 @@ static void vtcm_free(struct htp_context * ctx) { } } +static void htp_main_thread(void * context); static void htp_packet_callback(dspqueue_t queue, int error, void * context); static void htp_error_callback(dspqueue_t queue, int error, void * context); -AEEResult htp_iface_start(remote_handle64 handle, uint32 sess_id, uint64 dsp_queue_id, uint32 n_hvx, uint32 use_hmx) { - struct htp_context * ctx = (struct htp_context *) handle; +// Non-BURST modes leave DCVS enabled and keep the HTP awake. +// BURST pins all corners to MAX and disables DCVS. +// Corner constants are from HAP_power.h (HAP_dcvs_voltage_corner_t / HAP_dcvs_exp_vcorner_t). +struct htp_power_profile { + boolean dcvs_enable; + uint32_t bus_min_corner; + uint32_t bus_max_corner; + uint32_t bus_target_corner; + uint32_t core_min_corner; + uint32_t core_max_corner; + uint32_t core_target_corner; + boolean sleep_disable; + uint32_t hmx_min_corner; // only used when __HVX_ARCH__ >= 75 + uint32_t hmx_max_corner; + uint32_t hmx_target_corner; + uint32_t hmx_perf_mode; +}; + +#define HTP_POWER_PROFILE_ENTRY(min_c, max_c, hmx_c) \ + { \ + .dcvs_enable = TRUE, \ + .bus_min_corner = (min_c), \ + .bus_max_corner = (max_c), \ + .bus_target_corner = (max_c), \ + .core_min_corner = (min_c), \ + .core_max_corner = (max_c), \ + .core_target_corner = (max_c), \ + .sleep_disable = TRUE, \ + .hmx_min_corner = (hmx_c), \ + .hmx_max_corner = (hmx_c), \ + .hmx_target_corner = (hmx_c), \ + .hmx_perf_mode = HAP_CLK_PERF_HIGH, \ + } + +static const struct htp_power_profile htp_power_profiles[HTP_POWER_MODE_COUNT] = { + [HTP_POWER_MODE_LOW_POWER_SAVER] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_SVS2, HAP_DCVS_VCORNER_SVS2, HAP_DCVS_EXP_VCORNER_MIN_SVS), + [HTP_POWER_MODE_POWER_SAVER] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_SVS, HAP_DCVS_VCORNER_SVS_PLUS, HAP_DCVS_EXP_VCORNER_SVS), + [HTP_POWER_MODE_HIGH_POWER_SAVER] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_SVS_PLUS, HAP_DCVS_VCORNER_NOM, HAP_DCVS_EXP_VCORNER_SVS_L2), + [HTP_POWER_MODE_LOW_BALANCED] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_NOM, HAP_DCVS_VCORNER_NOM_PLUS, HAP_DCVS_EXP_VCORNER_NOM), + [HTP_POWER_MODE_BALANCED] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_NOM_PLUS, HAP_DCVS_VCORNER_TURBO, HAP_DCVS_EXP_VCORNER_NOM_L1), + [HTP_POWER_MODE_HIGH_PERFORMANCE] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_TURBO, HAP_DCVS_VCORNER_TURBO_PLUS, HAP_DCVS_EXP_VCORNER_TUR_L2), + [HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE] = + HTP_POWER_PROFILE_ENTRY(HAP_DCVS_VCORNER_NOM_PLUS, HAP_DCVS_VCORNER_TURBO_PLUS, HAP_DCVS_EXP_VCORNER_TUR_L2), + [HTP_POWER_MODE_BURST] = { + .dcvs_enable = FALSE, + .bus_min_corner = HAP_DCVS_VCORNER_MAX, + .bus_max_corner = HAP_DCVS_VCORNER_MAX, + .bus_target_corner = HAP_DCVS_VCORNER_MAX, + .core_min_corner = HAP_DCVS_VCORNER_MAX, + .core_max_corner = HAP_DCVS_VCORNER_MAX, + .core_target_corner = HAP_DCVS_VCORNER_MAX, + .sleep_disable = TRUE, + .hmx_min_corner = HAP_DCVS_EXP_VCORNER_MAX, + .hmx_max_corner = HAP_DCVS_EXP_VCORNER_MAX, + .hmx_target_corner = HAP_DCVS_EXP_VCORNER_MAX, + .hmx_perf_mode = HAP_CLK_PERF_HIGH, + }, +}; - if (!ctx) { +#undef HTP_POWER_PROFILE_ENTRY + +AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t vtcm_size, uint64_t max_vmem, uint32_t power_mode) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h) { return AEE_EBADPARM; } - if (ctx->queue) { + if (power_mode >= HTP_POWER_MODE_COUNT) { + FARF(ERROR, "invalid power-mode %u, falling back to burst", (unsigned) power_mode); + power_mode = HTP_POWER_MODE_BURST; + } + const struct htp_power_profile * pwr = &htp_power_profiles[power_mode]; + FARF(ALWAYS, "power-mode %u : dcvs-enable %d bus[%u,%u,%u] core[%u,%u,%u] sleep-disable %d hmx[%u,%u,%u]", + (unsigned) power_mode, (int) pwr->dcvs_enable, + (unsigned) pwr->bus_min_corner, (unsigned) pwr->bus_max_corner, (unsigned) pwr->bus_target_corner, + (unsigned) pwr->core_min_corner, (unsigned) pwr->core_max_corner, (unsigned) pwr->core_target_corner, + (int) pwr->sleep_disable, + (unsigned) pwr->hmx_min_corner, (unsigned) pwr->hmx_max_corner, (unsigned) pwr->hmx_target_corner); + + if (h->ctx) { FARF(ERROR, "Queue already open"); return AEE_EITEMBUSY; } - // Import queue created on the CPU - int err = dspqueue_import(dsp_queue_id, // Queue ID from dspqueue_export - htp_packet_callback, // Packet callback - htp_error_callback, // Error callback; no errors expected on the DSP - (void *) ctx, // Callback context - &ctx->queue); + // Cache the original FastRPC thread priority, then calculate compute priority + int fastrpc_tid = qurt_thread_get_id(); + int fastrpc_prio = qurt_thread_get_priority(fastrpc_tid); + int main_prio = fastrpc_prio - 10; + if (main_prio < 1) main_prio = 1; + + dspqueue_t dsp_queue = NULL; + bool use_callbacks = false; + + // Import queue with NULL callbacks to avoid starting dspueue internal threads + int err = dspqueue_import(dsp_queue_id, NULL, NULL, (void *) h, &dsp_queue); + if (err == AEE_EBADPARM) { + // Fallback for devices that don't support NULL callbacks + FARF(HIGH, "dspqueue import with NULL callbacks failed, trying with callbacks"); + use_callbacks = true; + err = dspqueue_import(dsp_queue_id, htp_packet_callback, htp_error_callback, (void *) h, &dsp_queue); + } if (err) { FARF(ERROR, "Queue import failed with 0x%08x", (unsigned) err); return err; } - ctx->thread_id = qurt_thread_get_id(); - ctx->thread_prio = qurt_thread_get_priority(ctx->thread_id); + qurt_sysenv_max_hthreads_t hw_threads; + qurt_sysenv_get_max_hw_threads(&hw_threads); + uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + + if (n_hvx == 0) { + n_hvx = hw_nhvx; + } + if (n_hvx > hw_threads.max_hthreads) { + n_hvx = hw_threads.max_hthreads; + } + if (n_hvx > HTP_MAX_NTHREADS) { + n_hvx = HTP_MAX_NTHREADS; + } + + // layout segments of our contiguous block + + // 1. htp_context : sits at the base (block is 4K-aligned via memalign below) + size_t offset = sizeof(struct htp_context); + + // 2. main_stack + size_t offset_main_stack = 0; + size_t size_main_stack = 0; + if (!use_callbacks) { + offset_main_stack = hex_align_up(offset, 4096); + size_main_stack = MAIN_THREAD_STACK_SIZE; + offset = offset_main_stack + size_main_stack; + } + + // 3. work_queue + size_t wq_align = work_queue_alignof(); + size_t offset_wq = hex_align_up(offset, wq_align); + size_t size_wq = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + offset = offset_wq + size_wq; + + // 4. dma_queue + size_t dma_align = dma_queue_alignof(); + size_t offset_dma = hex_align_up(offset, dma_align); + size_t size_dma = 0; + for (uint32_t i = 0; i < n_hvx; i++) { + size_dma = hex_align_up(size_dma, dma_queue_alignof()); + size_dma += dma_queue_sizeof(256); + } + offset = offset_dma + size_dma; + + // 5. hmx_queue + size_t offset_hmx = 0; + size_t size_hmx = 0; + if (n_hmx) { + size_t hmx_align = hmx_queue_alignof(); + offset_hmx = hex_align_up(offset, hmx_align); + size_hmx = hmx_queue_sizeof(HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE); + offset = offset_hmx + size_hmx; + } + + size_t footprint = hex_align_up(offset, 128); - // allocate VTCM - err = vtcm_alloc(ctx); + void * block = memalign(4096, footprint); + if (!block) { + FARF(ERROR, "Unable to allocate unified block of size %zu\n", footprint); + dspqueue_close(dsp_queue); + return AEE_ENOMEMORY; + } + memset(block, 0, footprint); + + h->ctx = (struct htp_context *) block; + struct htp_context * ctx = h->ctx; + ctx->footprint = footprint; + + ctx->thread_id = fastrpc_tid; + ctx->thread_prio = main_prio; + ctx->max_vmem = max_vmem; + ctx->dsp_queue = dsp_queue; + + err = vtcm_alloc(ctx, vtcm_size); if (err != AEE_SUCCESS) { FARF(ERROR, "Unable to allocate VTCM"); + htp_iface_stop(handle); return AEE_ENOMEMORY; } -#ifdef HTP_HAS_HMX - ctx->hmx_enabled = use_hmx; - ctx->hmx_queue = NULL; - if (use_hmx) { - ctx->hmx_queue = hmx_queue_create(16, ctx->vtcm_rctx); - if (!ctx->hmx_queue) { - FARF(ERROR, "hmx-queue-create failed"); - ctx->hmx_enabled = false; + HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0); + + // Set client class + { + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_apptype; + request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS; + + if ((err = HAP_power_set((void *) ctx, &request)) != 0) { + htp_iface_stop(handle); + return err; } } - FARF(HIGH, "HMX %s (use_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", use_hmx); + + // DCVS setup + { + HAP_power_request_t request; + memset(&request, 0, sizeof(request)); + + request.type = HAP_power_set_DCVS_v3; + request.dcvs_v3.set_dcvs_enable = TRUE; + request.dcvs_v3.dcvs_enable = pwr->dcvs_enable; + request.dcvs_v3.set_bus_params = TRUE; + request.dcvs_v3.bus_params.min_corner = pwr->bus_min_corner; + request.dcvs_v3.bus_params.max_corner = pwr->bus_max_corner; + request.dcvs_v3.bus_params.target_corner = pwr->bus_target_corner; + request.dcvs_v3.set_core_params = TRUE; + request.dcvs_v3.core_params.min_corner = pwr->core_min_corner; + request.dcvs_v3.core_params.max_corner = pwr->core_max_corner; + request.dcvs_v3.core_params.target_corner = pwr->core_target_corner; + request.dcvs_v3.set_sleep_disable = TRUE; + request.dcvs_v3.sleep_disable = pwr->sleep_disable; + +#if (__HEXAGON_ARCH__ >= 79) + HAP_set_dcvs_v3_protected_bus_corners(&request, 1); #endif + if ((err = HAP_power_set((void *) ctx, &request)) != 0) { + htp_iface_stop(handle); + return err; + } - qurt_sysenv_max_hthreads_t hw_threads; - qurt_sysenv_get_max_hw_threads(&hw_threads); - uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + memset(&request, 0, sizeof(request)); + request.type = HAP_power_set_HVX; + request.hvx.power_up = TRUE; + if ((err = HAP_power_set((void *) ctx, &request)) != 0) { + htp_iface_stop(handle); + return err; + } + } - if (n_hvx == 0) { - n_hvx = hw_nhvx; +#if __HVX_ARCH__ >= 75 + { + // Power on HMX and set HMX clock + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_HMX_v2; + request.hmx_v2.set_power = TRUE; + request.hmx_v2.power_up = TRUE; + request.hmx_v2.set_clock = TRUE; + request.hmx_v2.target_corner = pwr->hmx_target_corner; + request.hmx_v2.min_corner = pwr->hmx_min_corner; + request.hmx_v2.max_corner = pwr->hmx_max_corner; + request.hmx_v2.perf_mode = pwr->hmx_perf_mode; + FARF(ALWAYS, "Setting HMX clock\n"); + err = HAP_power_set((void *) ctx, &request); + if (err != AEE_SUCCESS) { + FARF(ERROR, "ggml-hex: error setting HMX clock."); + htp_iface_stop(handle); + return err; + } } - if (n_hvx > hw_threads.max_hthreads) { - n_hvx = hw_threads.max_hthreads; +#else + { + // Power on HMX + HAP_power_request_t request; + memset(&request, 0, sizeof(HAP_power_request_t)); + request.type = HAP_power_set_HMX; + request.hmx.power_up = TRUE; + FARF(ALWAYS, "Powering HMX on\n"); + err = HAP_power_set((void *) ctx, &request); + if (err != AEE_SUCCESS) { + FARF(ERROR, "ggml-hex: error powering on HMX."); + htp_iface_stop(handle); + return err; + } } - if (n_hvx > HTP_MAX_NTHREADS) { - n_hvx = HTP_MAX_NTHREADS; +#endif + + ctx->hmx_enabled = n_hmx; + ctx->hmx_queue = NULL; + if (n_hmx) { + void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx); + ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]); } + FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx); ctx->n_threads = n_hvx; + ctx->n_threads_div = init_fastdiv_values(ctx->n_threads); + + // Initialize DMA queues + uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma); + size_t size_dma_q = dma_queue_sizeof(256); + for (int i = 0; i < ctx->n_threads; i++) { - // see discussion https://github.com/ggml-org/llama.cpp/pull/18151#discussion_r2632388541 - ctx->dma[i] = dma_queue_create(128); + dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof()); + ctx->dma[i] = dma_queue_init(dma_ptr_curr, 256, &ctx->trace[i]); + dma_ptr_curr += size_dma_q; } - // init worker pool - err = worker_pool_init(&ctx->worker_pool, n_hvx); - if (err != AEE_SUCCESS) { - FARF(ERROR, "Unable to create worker pool"); - return err; + ctx->ddr_spad_size = 512 * 1024; // 512 KB + ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size); + + void * wq_ptr = (void *) ((uintptr_t) block + offset_wq); + ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + + ctx->main_stack = NULL; + ctx->main_thread = 0; + atomic_store(&ctx->killed, false); + + if (!use_callbacks) { + // Start main compute thread + ctx->main_stack = (void *) ((uintptr_t) block + offset_main_stack); + + qurt_thread_attr_t attr; + qurt_thread_attr_init(&attr); + qurt_thread_attr_set_stack_addr(&attr, ctx->main_stack); + qurt_thread_attr_set_stack_size(&attr, size_main_stack); + qurt_thread_attr_set_priority(&attr, main_prio); + qurt_thread_attr_set_name(&attr, "htp-main"); + + int err_thread = qurt_thread_create(&ctx->main_thread, &attr, htp_main_thread, ctx); + if (err_thread) { + FARF(ERROR, "Unable to create htp main thread: %d", err_thread); + htp_iface_stop(handle); + return AEE_ENOMEMORY; + } } FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n", @@ -386,43 +713,76 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32 sess_id, uint64 dsp_que } AEEResult htp_iface_stop(remote_handle64 handle) { - struct htp_context * ctx = (struct htp_context *) handle; - if (!ctx) { + struct htp_handle * h = (struct htp_handle *) handle; + if (!h || !h->ctx) { return AEE_EBADPARM; } + struct htp_context * ctx = h->ctx; - if (!ctx->queue) { - FARF(ERROR, "Queue not open"); - return AEE_EBADSTATE; + if (ctx->main_thread) { + atomic_store(&ctx->killed, true); + int status; + (void) qurt_thread_join(ctx->main_thread, &status); + ctx->main_thread = 0; } - // Close queue. dspqueue_close() will also wait for callbacks to finish. - int err = dspqueue_close(ctx->queue); - ctx->queue = NULL; + int err = dspqueue_close(ctx->dsp_queue); ctx->dsp_queue = NULL; if (err != 0) { FARF(ERROR, "Queue close failed with 0x%08x", (unsigned) err); return err; } - if (ctx->worker_pool) { - // Release worker pool - worker_pool_release(&ctx->worker_pool); - } + work_queue_free(ctx->work_queue); for (int i = 0; i < ctx->n_threads; i++) { - dma_queue_delete(ctx->dma[i]); + dma_queue_free(ctx->dma[i]); } -#ifdef HTP_HAS_HMX if (ctx->hmx_queue) { - hmx_queue_delete(ctx->hmx_queue); + hmx_queue_free(ctx->hmx_queue); ctx->hmx_queue = NULL; } ctx->hmx_enabled = false; -#endif vtcm_free(ctx); + if (ctx->ddr_spad_base) { + free(ctx->ddr_spad_base); + ctx->ddr_spad_base = NULL; + ctx->ddr_spad_size = 0; + } + + free(ctx); + h->ctx = NULL; + + return AEE_SUCCESS; +} + +AEEResult htp_iface_hwinfo(remote_handle64 handle, uint32_t * n_threads, uint32_t * n_hvx, uint32_t * n_hmx, uint64_t * vtcm_size) { + (void)handle; + if (!n_threads || !n_hvx || !n_hmx || !vtcm_size) { + return AEE_EBADPARM; + } + + qurt_sysenv_max_hthreads_t hw_threads; + qurt_sysenv_get_max_hw_threads(&hw_threads); + uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + + uint32_t n_hvx_val = hw_nhvx; + if (n_hvx_val > hw_threads.max_hthreads) { + n_hvx_val = hw_threads.max_hthreads; + } + if (n_hvx_val > HTP_MAX_NTHREADS) { + n_hvx_val = HTP_MAX_NTHREADS; + } + + // for now we force n_threads == n_hvx + *n_threads = n_hvx_val; + *n_hvx = n_hvx_val; + *n_hmx = 1; + + *vtcm_size = vtcm_query_size(); + return AEE_SUCCESS; } @@ -433,30 +793,133 @@ static void htp_error_callback(dspqueue_t queue, int error, void * context) { struct profile_data { uint64_t usecs; - uint64_t cycles; - uint64_t pkts; + uint64_t cycles_start; + uint64_t cycles_stop; + uint32_t pmu_counters[HEX_NUM_PMU_COUNTERS]; }; -static inline void profile_start(struct profile_data * d) { - d->usecs = HAP_perf_get_qtimer_count(); - d->cycles = hex_get_cycles(); - d->pkts = hex_get_pktcnt(); +static inline void profile_start(uint32_t mode, struct profile_data * d) { + switch (mode) { + case HTP_PROF_PMU: + hex_get_pmu(d->pmu_counters); + // fallthrough + case HTP_PROF_BASIC: + case HTP_PROF_TRACE: + d->usecs = HAP_perf_get_qtimer_count(); + d->cycles_start = hex_get_cycles(); + break; + default: + break; + } } -static inline void profile_stop(struct profile_data * d) { - d->usecs = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - d->usecs); - d->cycles = hex_get_cycles() - d->cycles; - d->pkts = hex_get_pktcnt() - d->pkts; +static inline void profile_stop(uint32_t mode, struct profile_data * d) { + uint32_t pmu_counters[HEX_NUM_PMU_COUNTERS]; + switch (mode) { + case HTP_PROF_PMU: + hex_get_pmu(pmu_counters); + for (int i = 0; i < HEX_NUM_PMU_COUNTERS; i++) { + d->pmu_counters[i] = pmu_counters[i] - d->pmu_counters[i]; + } + // fallthrough + case HTP_PROF_BASIC: + case HTP_PROF_TRACE: + d->usecs = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - d->usecs); + d->cycles_stop = hex_get_cycles(); + break; + default: + break; + } +} + +static int op_fence(struct htp_ops_context * octx) { + struct htp_context *ctx = octx->ctx; + struct htp_thread_trace * tr = &ctx->trace[0]; + const uint32_t seq = (uint32_t) octx->op_params[0]; + const uint32_t mode = (uint32_t) octx->op_params[1]; + + htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + + const struct htp_tensor * sync = octx->src[0]; + atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data; + + if (mode == 1) { + htp_dcache_flush_dirty_ranges(ctx); + + htp_mdev_group_barrier(octx); + + if (ctx->mdev.idx == 0) { + htp_fence_write(sync_fence, seq, octx->status); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + FARF(HIGH, "ggml-hex: sync-signal : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status); + return octx->status; + } + + int status = HTP_STATUS_OK; + uint64_t spins = 0; + while (1) { + uint32_t sync_seq; + uint32_t sync_status; + htp_fence_read(sync_fence, &sync_seq, &sync_status); + if ((int32_t)(sync_seq - seq) >= 0) { + if (sync_status > HTP_STATUS_OK) { + FARF(ERROR, "ggml-hex: sync-wait peer failed with status %u : fence %p seq 0x%x\n", sync_status, sync_fence, seq); + status = sync_status; + } + break; + } + if (++spins > HTP_FENCE_TIMEOUT) { + FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq); + status = HTP_STATUS_INTERNAL_ERR; + break; + } + hex_pause(); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq); + + FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq); + return status; +} + +static int op_mdev_group(struct htp_ops_context * octx) { + struct htp_context * ctx = octx->ctx; + const struct htp_tensor * sync = octx->src[0]; + ctx->mdev.idx = (uint16_t) octx->op_params[0]; + ctx->mdev.count = (uint16_t) sync->ne[1]; + if (ctx->mdev.count > 1) { + ctx->mdev.count_div = init_fastdiv_values(ctx->mdev.count); + ctx->mdev.fence_base = (uint8_t *) sync->data; + } + return HTP_STATUS_OK; } static int execute_op(struct htp_ops_context * octx) { switch (octx->op) { + case HTP_OP_MDEV_GROUP: + return op_mdev_group(octx); + + case HTP_OP_FENCE: + return op_fence(octx); + + case HTP_OP_ALLREDUCE: + case HTP_OP_ALLREDUCE_ADD: + return op_allreduce(octx); + case HTP_OP_MUL_MAT: + case HTP_OP_MUL_MAT_ADD: return op_matmul(octx); case HTP_OP_MUL_MAT_ID: return op_matmul_id(octx); + case HTP_OP_MUL_MAT_ID_NX: + return op_matmul_id_nx(octx); + + case HTP_OP_MUL_MAT_NX: + return op_matmul_nx(octx); + case HTP_OP_MUL: case HTP_OP_ADD: case HTP_OP_SUB: @@ -464,22 +927,42 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_ADD_ID: return op_binary(octx); + case HTP_OP_NORM: case HTP_OP_RMS_NORM: + case HTP_OP_RMS_NORM_MUL: case HTP_OP_SCALE: + case HTP_OP_CLAMP: + case HTP_OP_LEAKY_RELU: case HTP_OP_SQR: case HTP_OP_SQRT: case HTP_OP_UNARY_SOFTPLUS: case HTP_OP_UNARY_SIGMOID: + case HTP_OP_UNARY_SILU: + case HTP_OP_UNARY_GELU: + case HTP_OP_UNARY_GELU_QUICK: + case HTP_OP_UNARY_GELU_ERF: case HTP_OP_UNARY_NEG: case HTP_OP_UNARY_EXP: + case HTP_OP_UNARY_TANH: + case HTP_OP_UNARY_ABS: + case HTP_OP_UNARY_LOG: + case HTP_OP_UNARY_RELU: + case HTP_OP_UNARY_STEP: + case HTP_OP_UNARY_CEIL: + case HTP_OP_UNARY_SGN: + case HTP_OP_L2_NORM: + case HTP_OP_HADAMARD: return op_unary(octx); - case HTP_OP_UNARY_SILU: - case HTP_OP_UNARY_GELU: case HTP_OP_GLU_SWIGLU: case HTP_OP_GLU_SWIGLU_OAI: + case HTP_OP_GLU_SWIGLU_CLAMP: case HTP_OP_GLU_GEGLU: - return op_activations(octx); + case HTP_OP_GLU_GEGLU_QUICK: + case HTP_OP_GLU_GEGLU_ERF: + case HTP_OP_GLU_REGLU: + case HTP_OP_GLU_SIGMOID: + return op_glu(octx); case HTP_OP_SOFTMAX: return op_softmax(octx); @@ -496,10 +979,14 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_GET_ROWS: return op_get_rows(octx); + case HTP_OP_SUM: + return op_sum(octx); + case HTP_OP_SUM_ROWS: return op_sum_rows(octx); case HTP_OP_CPY: + case HTP_OP_CPY_FENCE: return op_cpy(octx); case HTP_OP_REPEAT: @@ -508,20 +995,60 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_ARGSORT: return op_argsort(octx); + case HTP_OP_TOP_K: + return op_top_k(octx); + + case HTP_OP_ARGMAX: + return op_argmax(octx); + case HTP_OP_SSM_CONV: return op_ssm_conv(octx); case HTP_OP_CUMSUM: return op_cumsum(octx); + case HTP_OP_FILL: + return op_fill(octx); + + case HTP_OP_DIAG: + return op_diag(octx); + + case HTP_OP_SOLVE_TRI: + return op_solve_tri(octx); + + case HTP_OP_PAD: + return op_pad(octx); + + case HTP_OP_IM2COL: + return op_im2col(octx); + + case HTP_OP_IM2COL_3D: + return op_im2col_3d(octx); + + case HTP_OP_ROLL: + return op_roll(octx); + + case HTP_OP_POOL_2D: + return op_pool_2d(octx); + + case HTP_OP_POOL_1D: + return op_pool_1d(octx); + + case HTP_OP_CONCAT: + return op_concat(octx); + + case HTP_OP_GATED_DELTA_NET: + return op_gated_delta_net(octx); + + case HTP_OP_TRI: + return op_unary(octx); + case HTP_OP_INVALID: break; - - // No default to catch missing cases } FARF(ERROR, "Unknown Op %u", octx->op); - return -1; + return HTP_STATUS_NO_SUPPORT; } static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) { @@ -529,7 +1056,7 @@ static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct for (uint32_t i=0; immap + i; - if (m->size && m->fd == b->fd) { + if (m->size && m->fd == b->fd && m->flags == b->flags) { b->base = m->base; *m_reuse |= (1 << i); return true; @@ -540,160 +1067,337 @@ static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct } static inline void drop_mmap(struct htp_context *ctx, struct htp_mmap *m) { - if (m->size && !m->pinned) { - FARF(HIGH, "unmap : fd %u base %p size %u pinned %u", m->fd, (void*) m->base, (uint32_t) m->size, m->pinned); -#if __HVX_ARCH__ > 73 - HAP_munmap2((void *) m->base, m->size); -#else - HAP_munmap((void *) m->base, m->size); -#endif + if (m->size) { + FARF(ALWAYS, "unmap : fd %u base 0x%llx size %llu", m->fd, (unsigned long long) m->base, (unsigned long long) m->size); + htp_munmap(m->base, m->size, m->flags); m->size = 0; m->base = 0; m->fd = -1; + m->flags = 0; } } -static inline void mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) { - if (b->base) return; // already mapped +static inline bool mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) { + if (b->base) return true; // already mapped // find unused mapping for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { struct htp_mmap *m = &ctx->mmap[i]; if (!m->size) { -#if __HVX_ARCH__ > 73 - void *va = HAP_mmap2(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0); -#else - if (b->size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB - FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) b->size); - abort(); // can't do much else at this point - } - - void *va = HAP_mmap(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0); -#endif - if (va == (void*)-1) { - FARF(ERROR, "mmap failed : va %p fd %u size %u", va, b->fd, (uint32_t) b->size); - abort(); // can't do much else at this point + uint64_t va = htp_mmap(b->fd, b->size, b->flags); + if (va == 0) { + FARF(HIGH, "mmap failed (will attempt defrag) : fd %u size %llu", b->fd, (unsigned long long) b->size); + return false; } - m->base = b->base = (uint64_t) va; + m->base = b->base = va; m->fd = b->fd; m->size = b->size; - m->pinned = 0; + m->flags = b->flags; - FARF(HIGH, "mmap : fd %u base %p size %u pinned %u", m->fd, (void*) m->base, (uint32_t) m->size, m->pinned); - return; + FARF(ALWAYS, "mmap : fd %u base 0x%llx size %llu flags 0x%x", m->fd, (unsigned long long) m->base, (unsigned long long) m->size, m->flags); + return true; } } + + FARF(ERROR, "mmap failed : exceeded mapping capacity limit of %u", HTP_MAX_MMAPS); + return false; } static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t n_bufs) { uint32_t m_reuse = 0; // mmap reuse mask (index from ctx->mmap array) uint32_t b_reuse = 0; // buf reuse count - size_t m_vmem = 0; // mapped vmem - size_t e_vmem = 0; // extra vmem + uint64_t m_vmem = 0; // mapped vmem + uint64_t e_vmem = 0; // extra vmem // See what we can reuse for (uint32_t i=0; i < n_bufs; i++) { struct htp_buf_desc *b = bufs + i; - if (reuse_buf(ctx, &m_reuse, b)) { b_reuse++; } else { e_vmem += b->size; } - FARF(HIGH, "prep-buf #%u : pass0 fd %u base %p size %u flags 0x%x", i, b->fd, (void*) b->base, (uint32_t) b->size, b->flags); + if (reuse_buf(ctx, &m_reuse, b)) { + b_reuse++; + } else if (!(b->flags & HTP_BUF_EXTENDED)) { + e_vmem += b->size; + } + FARF(HIGH, "prep-buf #%u : pass0 fd %u base 0x%llx size %llu flags 0x%x", i, b->fd, (unsigned long long) b->base, (unsigned long long) b->size, b->flags); } if (b_reuse == n_bufs) return; // all bufs reuse existing mappings // See how much vmem we have mmaped right now - for (uint32_t i=0; immap[i].size; } + for (uint32_t i=0; immap[i].flags & HTP_BUF_EXTENDED)) { + m_vmem += ctx->mmap[i].size; + } + } - FARF(HIGH, "prep-bufs : pass1 mmap-vmem %zu extra-vmem %zu n-bufs %u b-reuse %u", m_vmem, e_vmem, n_bufs, b_reuse); + FARF(HIGH, "prep-bufs : pass1 mmap-vmem %zu extra-vmem %zu max-vmem %zu : n-bufs %u b-reuse %u", + (size_t) m_vmem, (size_t) e_vmem, (size_t) ctx->max_vmem, n_bufs, b_reuse); - if ((m_vmem + e_vmem) > HTP_OP_MAX_VMEM) { + if ((m_vmem + e_vmem) > ctx->max_vmem) { // Drop unused mappings for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { bool used = m_reuse & (1<mmap + i); } + if (!used && !(ctx->mmap[i].flags & HTP_BUF_EXTENDED)) { + drop_mmap(ctx, ctx->mmap + i); + } } } - // Create missing mappings + // Create missing mappings (pass 1) + bool mmap_ok = true; for (uint32_t i=0; i < n_bufs; i++) { struct htp_buf_desc *b = bufs + i; - mmap_buf(ctx, b); - FARF(HIGH, "prep-buf #%u : pass1 fd %u base %p size %u flags 0x%x", i, b->fd, (void*) b->base, (uint32_t) b->size, b->flags); + if (!mmap_buf(ctx, b)) { + mmap_ok = false; + break; + } + FARF(HIGH, "prep-buf #%u : pass1 fd %u base 0x%llx size %llu flags 0x%x", i, b->fd, (unsigned long long) b->base, (unsigned long long) b->size, b->flags); + } + + if (!mmap_ok) { + // Attempt defragmentation: drop 32-bit mappings and remap (pass 2) + FARF(HIGH, "prep-bufs : dropping 32-bit mappings to defragment address space"); + for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { + if (!(ctx->mmap[i].flags & HTP_BUF_EXTENDED)) { + drop_mmap(ctx, ctx->mmap + i); + } + } + + for (uint32_t i=0; i < n_bufs; i++) { + struct htp_buf_desc *b = bufs + i; + if (!(b->flags & HTP_BUF_EXTENDED)) { + b->base = 0; + } + if (!mmap_buf(ctx, b)) { + FARF(ERROR, "prep-bufs : mmap failed after defragmentation (fd %u size %llu)", b->fd, (unsigned long long) b->size); + abort(); + } + FARF(HIGH, "prep-buf #%u : pass2 fd %u base 0x%llx size %llu flags 0x%x", i, b->fd, (unsigned long long) b->base, (unsigned long long) b->size, b->flags); + } } } -static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t idx, struct htp_tensor *t) { - uint32_t offset = t->data; - uint32_t size = t->size; +static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t idx, struct htp_tensor *t) { + uint64_t offset = t->data; uint32_t bi = t->bi; - t->data = bufs[bi].base + offset; // update data to the actual pointer + t->data = bufs[bi].base + offset; // update data to the actual pointer - FARF(HIGH, "prep-tensor #%u: bi %u offset %u size %u data %p : %u:%u:%u:%u", idx, t->bi, offset, t->size, (void*) t->data, - t->ne[0], t->ne[1], t->ne[3], t->ne[3]); + FARF(HIGH, "prep-tensor #%u: bi %u offset %llu size %u data 0x%llx : %u:%u:%u:%u", idx, t->bi, (unsigned long long) offset, t->size, (unsigned long long) t->data, + t->ne[0], t->ne[1], t->ne[2], t->ne[3]); } static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t n_tens) { for (uint32_t i=0; i < n_tens; i++) { - prep_tensor(ctx, bufs, i, tens + i); + prep_tensor(ctx, bufs, tens, i, tens + i); } } -static void proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, uint32_t idx, struct htp_op_desc * op) { - memcpy(octx->op_params, op->params, sizeof(octx->op_params)); - octx->flags = op->flags; - octx->op = op->opcode; +static void mdev_group_init(struct htp_context * ctx, const struct htp_opbatch_req * req) { + memset(&ctx->mdev, 0, sizeof(ctx->mdev)); + ctx->mdev.fence_seq = (uint32_t)((req->seq & 0xfffff) << 12); +} + +static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs, uint32_t n_bufs, + struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) { + memcpy(octx->op_params, op->params, sizeof(octx->op_params)); + memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params)); + octx->flags = op->flags; + octx->op = op->opcode; + octx->n_threads = octx->ctx->n_threads; + octx->n_threads_div = octx->ctx->n_threads_div; FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags); // Prep input tensors for (uint32_t i=0; isrc[i] == 0xffff ? NULL : tens + op->src[i]; + uint16_t src_idx = op->src[i]; + if (src_idx == 0xffff) { + octx->src[i] = NULL; + continue; + } + + struct htp_tensor *src = tens + src_idx; + octx->src[i] = src; + + FARF(HIGH, "prep-src #%u: data 0x%llx size %u : %u:%u:%u:%u", op->src[i], (unsigned long long) src->data, src->size, + src->ne[0], src->ne[1], src->ne[2], src->ne[3]); + } + + htp_dcache_flush_dirty_tensors(octx->ctx, octx->src, HTP_OP_MAX_INPUTS); + + // Prep output tensors + for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) { + uint16_t dst_idx = op->dst[i]; + if (dst_idx == 0xffff) { + octx->dsts[i] = NULL; + continue; + } + struct htp_tensor *dst = tens + dst_idx; + octx->dsts[i] = dst; + + FARF(HIGH, "prep-dst[%u] #%u: data 0x%llx size %u : %u:%u:%u:%u", i, dst_idx, (unsigned long long) dst->data, dst->size, + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); + } + + htp_dcache_mark_dirty_tensors(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS); + + htp_mdev_group_barrier(octx); + + int status = execute_op(octx); + + htp_ops_context_set_status(octx, status); + + octx->src0_spad.src = NULL; + octx->src1_spad.src = NULL; + octx->dst_spad.src = NULL; - octx->src[i] = src; - if (!src) continue; + return octx->status; +} + +static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) { + dspqueue_t queue = ctx->dsp_queue; + int err; + + const uint32_t n_bufs = req->n_bufs; + const uint32_t n_tens = req->n_tensors; + const uint32_t n_ops = req->n_ops; + + const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs; + const uint32_t t_size = sizeof(struct htp_tensor) * n_tens; + const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops; + const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops; + const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc); + + if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) { + FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size); + return; + } - if (!(src->flags & HTP_TENSOR_FLUSHED) && (src->flags & HTP_TENSOR_COMPUTE)) { - // flush compute buffers on input - hex_l2flush((void *) src->data, src->size); + FARF(HIGH, "processing opbatch #%llu: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", (unsigned long long) req->seq, + n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size); + + // Setup descriptor pointers + uint8_t * m_ptr = dbuf->ptr; + struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size; + struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size; + struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; m_ptr += o_size; + struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr; + + struct profile_data batch_prof; + profile_start(HTP_PROF_BASIC, &batch_prof); + + memset(ctx->trace, 0, sizeof(ctx->trace)); + if (ctx->profiler == HTP_PROF_TRACE) { + struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size); + for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + ctx->trace[t].events = &trace_events[t * req->n_traces]; + ctx->trace[t].max_events = req->n_traces; } + } + + // Clean cache at the start of the batch + htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + hex_l2fetch_block(ctx, ctx->footprint); + memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges)); + htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + + htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_BUFF, 0); + prep_op_bufs(ctx, bufs, n_bufs); + htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_BUFF, 0); + + prep_tensors(ctx, bufs, tens, n_tens); + + struct htp_ops_context *octx = &ctx->octx; + htp_ops_context_reset(octx, ctx); + + mdev_group_init(ctx, req); - FARF(HIGH, "prep-src #%u: data %p size %u : %u:%u:%u:%u", op->src[i], (void*) src->data, src->size, - src->ne[0], src->ne[1], src->ne[3], src->ne[3]); + work_queue_wakeup(ctx->work_queue); + if (ctx->hmx_queue) { + hmx_queue_wakeup(ctx->hmx_queue); } - // Prep output tensor - struct htp_tensor *dst = tens + op->dst; + int op_status = HTP_STATUS_OK; + octx->status = HTP_STATUS_OK; + for (uint32_t i = 0; i < n_ops; i++) { + struct profile_data prof; + + profile_start(ctx->profiler, &prof); + + op_status = proc_op_req(octx, bufs, n_bufs, tens, i, &ops[i]); - octx->dst = dst; + profile_stop(ctx->profiler, &prof); - FARF(HIGH, "prep-dst #%u: data %p size %u : %u:%u:%u:%u", op->dst, (void*) dst->data, dst->size, - dst->ne[0], dst->ne[1], dst->ne[3], dst->ne[3]); + htp_ops_context_set_status(octx, op_status); + + if (ctx->profiler) { + pds[i].opcode = ops[i].opcode; + pds[i].usecs = prof.usecs; + pds[i].cycles_start = prof.cycles_start; + pds[i].cycles_stop = prof.cycles_stop; + for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) { + pds[i].pmu[j] = prof.pmu_counters[j]; + } + } + } - (void) execute_op(octx); + if (ctx->hmx_queue) { + hmx_queue_suspend(ctx->hmx_queue); + hmx_queue_flush(ctx->hmx_queue); + } + work_queue_suspend(ctx->work_queue); + + // Flush remaining dirty tensors at the end of the batch + htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE); + htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0); + + htp_mdev_group_barrier(octx); + + profile_stop(HTP_PROF_BASIC, &batch_prof); + + struct htp_opbatch_rsp rsp; + memset(&rsp, 0, sizeof(rsp)); + rsp.seq = req->seq; + rsp.status = octx->status; + rsp.n_bufs = n_bufs; + rsp.n_tensors = n_tens; + rsp.n_ops = n_ops; + rsp.usecs = batch_prof.usecs; + rsp.cycles_start = batch_prof.cycles_start; + rsp.cycles_stop = batch_prof.cycles_stop; + + if (ctx->profiler == HTP_PROF_TRACE) { + for (int t = 0; t <= HTP_MAX_NTHREADS; t++) { + rsp.n_traces[t] = ctx->trace[t].count; + } + } - // flush buffers on output - hex_l2flush((void *) dst->data, dst->size); - dst->flags |= HTP_TENSOR_FLUSHED; + struct dspqueue_buffer write_dbuf = *dbuf; + write_dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; - FARF(HIGH, "post-dst #%u: data %p size %u : %u:%u:%u:%u", op->dst, (void*) dst->data, dst->size, - dst->ne[0], dst->ne[1], dst->ne[3], dst->ne[3]); + err = dspqueue_write(queue, 0, 1, &write_dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE); + if (err != 0) { + FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err); + } } +#define DSPQUEUE_READ_TIMEOUT_USEC 5000 #define DSPQUEUE_POLL_TIMEOUT_USEC 100 #define DSPQUEUE_POLL_COUNT 100 -static void htp_packet_callback(dspqueue_t queue, int error, void * context) { - struct htp_context * ctx = (struct htp_context *) context; - +static void process_ops(struct htp_context * ctx) { + dspqueue_t queue = ctx->dsp_queue; int err; uint32_t poll_count = DSPQUEUE_POLL_COUNT; vtcm_acquire(ctx); - while (!ctx->vtcm_needs_release) { + while (!atomic_load(&ctx->vtcm_needs_release) && !atomic_load(&ctx->killed)) { struct htp_opbatch_req req; uint32_t r_size = sizeof(req); @@ -720,62 +1424,44 @@ static void htp_packet_callback(dspqueue_t queue, int error, void * context) { continue; } - const uint32_t n_bufs = req.n_bufs; - const uint32_t n_tens = req.n_tensors; - const uint32_t n_ops = req.n_ops; - - const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs; - const uint32_t t_size = sizeof(struct htp_tensor) * n_tens; - const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops; - - if (dbuf.size < b_size + t_size + o_size) { - FARF(ERROR, "invalid opbatch memory block size %u", dbuf.size); - break; - } - // Reset poll count for valid requests poll_count = DSPQUEUE_POLL_COUNT; - uint8_t * m_ptr = dbuf.ptr; - struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size; - struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size; - struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; - - FARF(HIGH, "processing opbatch: n-bufs %u n-tensors %u n-ops %u : m-size %u b-size %u t-size %u o-size %u", - n_bufs, n_tens, n_ops, dbuf.size, b_size, t_size, o_size); - - prep_op_bufs(ctx, bufs, n_bufs); - prep_tensors(ctx, bufs, tens, n_tens); - - struct htp_ops_context *octx = &ctx->octx; - memset(octx, 0, sizeof(*octx)); - octx->n_threads = ctx->n_threads; - octx->ctx = ctx; + process_opbatch(ctx, &req, &dbuf); + } - for (uint32_t i=0; i < n_ops; i++) { - struct profile_data prof; - profile_start(&prof); + vtcm_release(ctx); +} - proc_op_req(octx, tens, i, &ops[i]); +static void htp_packet_callback(dspqueue_t queue, int error, void * context) { + (void) queue; + (void) error; + struct htp_handle * h = (struct htp_handle *) context; + if (h && h->ctx) { + process_ops(h->ctx); + } +} - profile_stop(&prof); - ops[i].prof_usecs = prof.usecs; - ops[i].prof_cycles = prof.cycles; - ops[i].prof_pkts = prof.pkts; - } +static void htp_main_thread(void * context) { + struct htp_context * ctx = (struct htp_context *) context; - // dspqueue_write_early_wakeup_noblock(ctx->queue, 10, 0); + FARF(HIGH, "htp-main-thread: started"); - struct htp_opbatch_rsp rsp; - rsp.status = HTP_STATUS_OK; // FIXME + while (!atomic_load(&ctx->killed)) { + uint32_t flags = 0; + uint32_t num_buffers = 0; + uint32_t message_length = 0; - dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT; - err = dspqueue_write(queue, 0, 1, &dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE); - if (err != 0) { - FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err); + int err = dspqueue_peek(ctx->dsp_queue, &flags, &num_buffers, &message_length, 50000); + if (err == 0) { + process_ops(ctx); + } else if (err == AEE_EWOULDBLOCK || err == AEE_EEXPIRED) { + continue; + } else { + FARF(ERROR, "dspqueue_peek failed: 0x%08x", (unsigned) err); break; } } - vtcm_release(ctx); + FARF(HIGH, "htp-main-thread: stopped"); } diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index bac0669..b4b9430 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -5,51 +5,163 @@ #include #include +#include #include #include +#include -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #include "hvx-dump.h" +#include "hvx-arith.h" +#include "hvx-reduce.h" +#include "hvx-copy.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" -#include "hmx-ops.h" - -#define MM_SPAD_SRC0_NROWS 16 -#define MM_SPAD_SRC1_NROWS 16 -#define MM_SPAD_DST_NROWS 2 +#include "htp-tensor.h" +#include "matmul-ops.h" +#include "htp-vtcm.h" + +typedef struct { + float *dst; + dma_addr_t bias_addr; + size_t bias_bytes; + dma_addr_t act_dma_addr; + dma_addr_t weight; + dma_queue * weight_dma; + int m; + int k; + int n; + int act_stride; + uint32_t act_elem_size; // 4=F32 src1, 2=F16 src1 + int weight_stride; + int weight_type; + int dst_stride; + int dst_cols; + uint32_t bias_stride; + int tile_size; + int aligned_tile_size; + int ne02; + int ne03; + int ne12; + int ne13; + size_t src0_nb2; + size_t src0_nb3; + size_t act_nb2; + size_t act_nb3; + size_t bias_nb2; + size_t bias_nb3; + size_t dst_nb2; + size_t dst_nb3; + int r2; + int r3; + struct fastdiv_values div_r2; + struct fastdiv_values div_r3; +} hmx_mm_batched_params_t; + +static bool htp_matmul_has_extended_weight(const struct htp_ops_context * octx, uint32_t n_weights) { + for (uint32_t i = 0; i < n_weights; ++i) { + if (htp_tensor_is_extended(octx->src[i])) { + return true; + } + } + return false; +} -struct htp_matmul_context { +struct htp_mm_context { const char * type; struct htp_ops_context * octx; + const struct htp_tensor * act; - void (*vec_dot_1x1)(const int n, float * restrict s0, + void (*vec_dot_1x1)(const uint32_t n, float * restrict s0, const void * restrict vx0, const void * restrict vy0); - void (*vec_dot_2x1)(const int n, float * restrict s0, + void (*vec_dot_2x1)(const uint32_t n, float * restrict s0, const void * restrict vx0, const void * restrict vx1, const void * restrict vy0); - void (*vec_dot_2x2)(const int n, float * restrict s0, float * restrict s1, + void (*vec_dot_2x2)(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx0, const void * restrict vx1, const void * restrict vy0, const void * restrict vy1); + void (*vec_dot_32x1)(const uint32_t n, float * restrict s, + const void * restrict vx, + const void * restrict vy, uint32_t valid_rows, + const float * restrict sz); + + void (*vec_dot_32x2)(const uint32_t n, float * restrict s0, float * restrict s1, + const void * restrict vx, + const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, + const float * restrict sz0, const float * restrict sz1); + // Precomputed values uint32_t src0_nrows_per_thread; - uint32_t src1_nrows_per_thread; + uint32_t src0_row_start; + uint32_t src0_row_end; + uint32_t src0_row_size_padded; + uint32_t act_nrows; + uint32_t cur_m_start; + uint32_t cur_m_rows; struct fastdiv_values mm_div_ne12_ne1; struct fastdiv_values mm_div_ne1; struct fastdiv_values mm_div_r2; struct fastdiv_values mm_div_r3; + struct fastdiv_values mm_div_ne11; + + // Precomputed block-parallel quantization values + uint32_t quant_ib_first[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_ib_last[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_r[WORK_QUEUE_MAX_N_THREADS]; + uint32_t quant_c[WORK_QUEUE_MAX_N_THREADS]; + uint32_t n_quant_tasks; + uint32_t n_quant_rows_per_thread; + + // Fields for scattered mapping & HMX support in MUL_MAT_ID + const uint32_t * matrix_row_counts; + const struct mmid_row_mapping * matrix_rows; + uint32_t mapping_stride; + + // Dynamic VTCM pointers allocated sequentially + uint8_t * vtcm_src0; + uint8_t * vtcm_act; + uint8_t * vtcm_bias; + uint8_t * vtcm_dst; + uint8_t * vtcm_act_raw; + + // Cached strides + uint32_t vtcm_src0_stride; + uint32_t vtcm_act_stride; + uint32_t vtcm_act_raw_stride; + + // Cached thread offsets/sizes + uint32_t vtcm_src0_size_per_thread; + uint32_t vtcm_act_size_per_thread; + uint32_t vtcm_dst_size_per_thread; }; +static int htp_mm_init_context( + struct htp_ops_context * octx, + const struct htp_mm_kernel_params * kparams +) { + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->n_hmx) { + if (kparams->n_act_threads <= 0 || kparams->n_act_threads > (int32_t) octx->n_threads) { + return HTP_STATUS_INVAL_PARAMS; + } + } + + return HTP_STATUS_OK; +} + // vdelta control to expand first 32 e8m0 values into 32 uint32 elements static const uint8_t __attribute__((aligned(128))) expand_x32_e8m0[128] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x02, 0x00, 0x08, 0x08, 0x01, 0x02, 0x00, 0x04, 0x04, 0x00, 0x00, @@ -79,3115 +191,4404 @@ static const uint8_t __attribute__((aligned(VLEN))) kvalues_mxfp4_lut[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; -static inline HVX_Vector_x8 hvx_vec_load_iq4nlx4x8_full(const uint8_t * restrict ptr) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr; - - HVX_Vector v0_1 = vptr[0]; // first 256 elements (128 bytes) - HVX_Vector v2_3 = vptr[1]; // ... - HVX_Vector v4_5 = vptr[2]; // ... - HVX_Vector v6_7 = vptr[3]; // ... - - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - const HVX_Vector lut = *(const HVX_Vector *) kvalues_iq4nl_lut; - - HVX_Vector v0 = Q6_V_vand_VV(v0_1, mask_h4); // & 0x0F - HVX_Vector v1 = Q6_Vub_vlsr_VubR(v0_1, 4); // >> 4 - HVX_Vector v2 = Q6_V_vand_VV(v2_3, mask_h4); // & 0x0F - HVX_Vector v3 = Q6_Vub_vlsr_VubR(v2_3, 4); // >> 4 - HVX_Vector v4 = Q6_V_vand_VV(v4_5, mask_h4); // & 0x0F - HVX_Vector v5 = Q6_Vub_vlsr_VubR(v4_5, 4); // >> 4 - HVX_Vector v6 = Q6_V_vand_VV(v6_7, mask_h4); // & 0x0F - HVX_Vector v7 = Q6_Vub_vlsr_VubR(v6_7, 4); // >> 4 - - v0 = Q6_Vb_vlut32_VbVbI(v0, lut, 0); - v1 = Q6_Vb_vlut32_VbVbI(v1, lut, 0); - v2 = Q6_Vb_vlut32_VbVbI(v2, lut, 0); - v3 = Q6_Vb_vlut32_VbVbI(v3, lut, 0); - v4 = Q6_Vb_vlut32_VbVbI(v4, lut, 0); - v5 = Q6_Vb_vlut32_VbVbI(v5, lut, 0); - v6 = Q6_Vb_vlut32_VbVbI(v6, lut, 0); - v7 = Q6_Vb_vlut32_VbVbI(v7, lut, 0); - - HVX_Vector_x8 r = { v0, v1, v2, v3, v4, v5, v6, v7 }; - return r; -} - -static inline HVX_Vector_x8 hvx_vec_load_iq4nlx4x8_partial(const uint8_t * restrict ptr, uint32_t n) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr; - - const uint32_t qk = QK_Q4_0x4x2; // 256 - const uint32_t nb = n / qk; - const uint32_t nloe = n % qk; +#define htp_matmul_tensors_preamble \ + const struct htp_tensor * restrict src0 = octx->src[0]; \ + const struct htp_tensor * restrict src1 = octx->src[1]; \ + const struct htp_tensor * restrict src2 = octx->src[2]; \ + const struct htp_tensor * restrict dst = octx->dst; \ + const struct htp_tensor * restrict act = src1; \ + \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t ne10 = src1->ne[0]; \ + const uint32_t ne11 = src1->ne[1]; \ + const uint32_t ne12 = src1->ne[2]; \ + const uint32_t ne13 = src1->ne[3]; \ + \ + const uint32_t ne20 = src2 ? src2->ne[0] : 0; \ + const uint32_t ne21 = src2 ? src2->ne[1] : 0; \ + const uint32_t ne22 = src2 ? src2->ne[2] : 0; \ + const uint32_t ne23 = src2 ? src2->ne[3] : 0; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb10 = src1->nb[0]; \ + const uint32_t nb11 = src1->nb[1]; \ + const uint32_t nb12 = src1->nb[2]; \ + const uint32_t nb13 = src1->nb[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - const HVX_Vector lut = *(const HVX_Vector *) kvalues_iq4nl_lut; +#define htp_matmul_preamble \ + struct htp_mm_context * mmctx = data; \ + struct htp_ops_context * octx = mmctx->octx; \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; \ + htp_matmul_tensors_preamble; - HVX_Vector_x8 r; - uint32_t i = 0; - #pragma unroll(2) - for (i = 0; i < nb; i++) { - HVX_Vector v = vptr[i]; // 256 elements (128 bytes) - HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : first 128 elements - HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : second 128 elements - r.v[i * 2 + 0] = Q6_Vb_vlut32_VbVbI(v0, lut, 0); - r.v[i * 2 + 1] = Q6_Vb_vlut32_VbVbI(v1, lut, 0); - } - if (nloe) { - HVX_Vector v = vptr[i]; // 256 elements (128 bytes) - HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : even 128 elements - HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : odd 128 elements - HVX_VectorPair v0_1_p = Q6_W_vshuff_VVR(v1, v0, -1); // zip even:odd:... - r.v[i * 2 + 0] = Q6_Vb_vlut32_VbVbI(Q6_V_lo_W(v0_1_p), lut, 0); - r.v[i * 2 + 1] = Q6_Vb_vlut32_VbVbI(Q6_V_hi_W(v0_1_p), lut, 0); - } - return r; -} +// hvx kernels first: the HMX Q6_K dequantizer reuses unpack_q6_k_group from there +#include "hvx-mm-kernels-tiled.h" +#include "hvx-mm-kernels-float.h" +#include "hmx-mm-kernels-tiled.h" -// q4x4x2 and q8x4x2 are the flat q4/8_0 formats where all quants are stored first followed by all scales +static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; -static inline size_t q8x4x2_row_size(uint32_t ne) { - // ensures perfect alignment of quants and full row - const uint32_t qk = QK_Q8_0x4x2; - const uint32_t nb = (ne + qk - 1) / qk; - return hex_round_up(ne + nb * 8 * sizeof(__fp16), 128); -} + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; + const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); + const uint32_t cur_m_start = mmctx->cur_m_start; -static inline HVX_Vector_x8 hvx_vec_load_q4x4x8_full(const uint8_t * restrict ptr) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr; + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); - HVX_Vector v0_1 = vptr[0]; // first 256 elements (128 bytes) - HVX_Vector v2_3 = vptr[1]; // ... - HVX_Vector v4_5 = vptr[2]; // ... - HVX_Vector v6_7 = vptr[3]; // ... + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - const HVX_Vector i8 = Q6_Vb_vsplat_R(8); + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - HVX_Vector v0 = Q6_V_vand_VV(v0_1, mask_h4); // & 0x0F : first 128 elements - HVX_Vector v1 = Q6_Vub_vlsr_VubR(v0_1, 4); // >> 4 : second 128 elements - HVX_Vector v2 = Q6_V_vand_VV(v2_3, mask_h4); // & 0x0F ... - HVX_Vector v3 = Q6_Vub_vlsr_VubR(v2_3, 4); // >> 4 - HVX_Vector v4 = Q6_V_vand_VV(v4_5, mask_h4); // & 0x0F - HVX_Vector v5 = Q6_Vub_vlsr_VubR(v4_5, 4); // >> 4 - HVX_Vector v6 = Q6_V_vand_VV(v6_7, mask_h4); // & 0x0F - HVX_Vector v7 = Q6_Vub_vlsr_VubR(v6_7, 4); // >> 4 + const size_t dst_row_size = nb1; + const size_t act_row_size = nb11; + const size_t act_stride = mmctx->vtcm_act_stride; + const size_t src2_stride = src2 ? ((src2->ne[1] == 1) ? 0 : src2->nb[1]) : 0; - // Convert uint4 to int4 (i.e. x - 8) - v0 = Q6_Vb_vsub_VbVb(v0, i8); - v1 = Q6_Vb_vsub_VbVb(v1, i8); - v2 = Q6_Vb_vsub_VbVb(v2, i8); - v3 = Q6_Vb_vsub_VbVb(v3, i8); - v4 = Q6_Vb_vsub_VbVb(v4, i8); - v5 = Q6_Vb_vsub_VbVb(v5, i8); - v6 = Q6_Vb_vsub_VbVb(v6, i8); - v7 = Q6_Vb_vsub_VbVb(v7, i8); - - HVX_Vector_x8 r = { v0, v1, v2, v3, v4, v5, v6, v7 }; - return r; -} + uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; -static HVX_Vector_x8 hvx_vec_load_q4x4x8_partial(const uint8_t * restrict ptr, uint32_t n) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr; + const dma_addr_t src0_row = src0->data; - const uint32_t qk = QK_Q4_0x4x2; // 256 - const uint32_t nb = n / qk; - const uint32_t nloe = n % qk; + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - const HVX_Vector i8 = Q6_Vb_vsplat_R(8); + uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); + uint32_t tile_row_stride = n_k_tiles_w * tile_size; + uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; - HVX_Vector_x8 r; - uint32_t i = 0; + uint32_t ct_start = src0_start_row / 32; + uint32_t ct_end = (src0_end_row + 31) / 32; - #pragma unroll(2) - for (i=0; i < nb; i++) { - HVX_Vector v = vptr[i]; // 256 elements (128 bytes) - HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : first 128 elements - HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : second 128 elements - r.v[i*2+0] = Q6_Vb_vsub_VbVb(v0, i8); - r.v[i*2+1] = Q6_Vb_vsub_VbVb(v1, i8); + uint32_t push_ct = ct_start; + if (src0_start_row < src0_end_row) { + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } } - if (nloe) { - HVX_Vector v = vptr[i]; // 256 elements (128 bytes) - HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : even 128 elements - HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : odd 128 elements - HVX_VectorPair v0_1_p = Q6_W_vshuff_VVR(v1, v0, -1); // zip even:odd:... - r.v[i*2+0] = Q6_Vb_vsub_VbVb(Q6_V_lo_W(v0_1_p), i8); - r.v[i*2+1] = Q6_Vb_vsub_VbVb(Q6_V_hi_W(v0_1_p), i8); + if (src0_start_row >= src0_end_row) { + return; } - return r; -} - -static inline HVX_Vector_x8 hvx_vec_load_mxfp4x4x8_full(const uint8_t * restrict ptr) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr; + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - HVX_Vector v0_1 = vptr[0]; // first 256 elements (128 bytes) - HVX_Vector v2_3 = vptr[1]; // ... - HVX_Vector v4_5 = vptr[2]; // ... - HVX_Vector v6_7 = vptr[3]; // ... + int valid_rows = (int)ne0 - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - const HVX_Vector lut = *(const HVX_Vector *) kvalues_mxfp4_lut; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + uint32_t ir1 = 0; + for (; ir1 + 1 < act_nrows; ir1 += 2) { + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (ir1+0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (ir1+1) * act_stride); + float * restrict dst_row0 = (float *) (dst->data + ((cur_m_start + ir1+0) * dst_row_size)); + float * restrict dst_row1 = (float *) (dst->data + ((cur_m_start + ir1+1) * dst_row_size)); + + float * dst_ptr0 = &dst_row0[ct * 32]; + float * dst_ptr1 = &dst_row1[ct * 32]; + + const float * src2_ptr0 = NULL; + const float * src2_ptr1 = NULL; + if (src2) { + const float * restrict src2_row0 = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1+0) * src2_stride)); + const float * restrict src2_row1 = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1+1) * src2_stride)); + src2_ptr0 = &src2_row0[ct * 32]; + src2_ptr1 = &src2_row1[ct * 32]; + } + mmctx->vec_dot_32x2(ne10, dst_ptr0, dst_ptr1, w_tile, act_col0, act_col1, valid_rows, src2_ptr0, src2_ptr1); + } - HVX_Vector v0 = Q6_V_vand_VV(v0_1, mask_h4); // & 0x0F - HVX_Vector v1 = Q6_Vub_vlsr_VubR(v0_1, 4); // >> 4 - HVX_Vector v2 = Q6_V_vand_VV(v2_3, mask_h4); // & 0x0F - HVX_Vector v3 = Q6_Vub_vlsr_VubR(v2_3, 4); // >> 4 - HVX_Vector v4 = Q6_V_vand_VV(v4_5, mask_h4); // & 0x0F - HVX_Vector v5 = Q6_Vub_vlsr_VubR(v4_5, 4); // >> 4 - HVX_Vector v6 = Q6_V_vand_VV(v6_7, mask_h4); // & 0x0F - HVX_Vector v7 = Q6_Vub_vlsr_VubR(v6_7, 4); // >> 4 + for (; ir1 < act_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + ir1 * act_stride); + float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); + float * dst_ptr = &dst_row[ct * 32]; - v0 = Q6_Vb_vlut32_VbVbI(v0, lut, 0); - v1 = Q6_Vb_vlut32_VbVbI(v1, lut, 0); - v2 = Q6_Vb_vlut32_VbVbI(v2, lut, 0); - v3 = Q6_Vb_vlut32_VbVbI(v3, lut, 0); - v4 = Q6_Vb_vlut32_VbVbI(v4, lut, 0); - v5 = Q6_Vb_vlut32_VbVbI(v5, lut, 0); - v6 = Q6_Vb_vlut32_VbVbI(v6, lut, 0); - v7 = Q6_Vb_vlut32_VbVbI(v7, lut, 0); - - HVX_Vector_x8 r = { v0, v1, v2, v3, v4, v5, v6, v7 }; - return r; -} - -static inline HVX_Vector_x8 hvx_vec_load_mxfp4x4x8_partial(const uint8_t * restrict ptr, uint32_t n) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr; - - const uint32_t qk = QK_Q4_0x4x2; // 256 - const uint32_t nb = n / qk; - const uint32_t nloe = n % qk; + const float * src2_ptr = NULL; + if (src2) { + const float * restrict src2_row = (const float *) ((const uint8_t *) src2->data + ((cur_m_start + ir1) * src2_stride)); + src2_ptr = &src2_row[ct * 32]; + } + mmctx->vec_dot_32x1(ne10, dst_ptr, w_tile, act_col, valid_rows, src2_ptr); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); - const HVX_Vector mask_h4 = Q6_Vb_vsplat_R(0x0F); - const HVX_Vector lut = *(const HVX_Vector *) kvalues_mxfp4_lut; - - HVX_Vector_x8 r; - uint32_t i = 0; - - #pragma unroll(2) - for (i=0; i < nb; i++) { - HVX_Vector v = vptr[i]; // 256 elements (128 bytes) - HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : first 128 elements - HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : second 128 elements - r.v[i*2+0] = Q6_Vb_vlut32_VbVbI(v0, lut, 0); - r.v[i*2+1] = Q6_Vb_vlut32_VbVbI(v1, lut, 0); - } - - if (nloe) { - HVX_Vector v = vptr[i]; // 256 elements (128 bytes) - HVX_Vector v0 = Q6_V_vand_VV(v, mask_h4); // & 0x0F : even 128 elements - HVX_Vector v1 = Q6_Vub_vlsr_VubR(v, 4); // >> 4 : odd 128 elements - HVX_VectorPair v0_1_p = Q6_W_vshuff_VVR(v1, v0, -1); // zip even:odd:... - r.v[i*2+0] = Q6_Vb_vlut32_VbVbI(Q6_V_lo_W(v0_1_p), lut, 0); - r.v[i*2+1] = Q6_Vb_vlut32_VbVbI(Q6_V_hi_W(v0_1_p), lut, 0); + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } } - - return r; -} - -static inline HVX_Vector_x8 hvx_vec_load_q8x4x8_full(const uint8_t * restrict ptr) { - const HVX_Vector * restrict vptr = (const HVX_Vector *) ptr; - - HVX_Vector v0 = vptr[0]; // first 128 vals - HVX_Vector v1 = vptr[1]; // ... - HVX_Vector v2 = vptr[2]; // ... - HVX_Vector v3 = vptr[3]; // ... - HVX_Vector v4 = vptr[4]; // ... - HVX_Vector v5 = vptr[5]; // ... - HVX_Vector v6 = vptr[6]; // ... - HVX_Vector v7 = vptr[7]; // ... - - HVX_Vector_x8 r = { v0, v1, v2, v3, v4, v5, v6, v7 }; - return r; } -static inline HVX_Vector_x8 hvx_vec_load_q8x4x8_partial(const uint8_t * restrict ptr, uint32_t nloe) { - return hvx_vec_load_q8x4x8_full(ptr); -} - -// Reduce multiply 1024 x 1024 int8 elements (32x q4/8 blocks in 8x HVX vectors). -// Accumulate each block into a single int32 value. -// Return a single HVX vector with 32x int32 accumulators. -// This version is parameterized to support less than 1024 elements. -// if() checks are optimized out at compile time -- make sure to pass N as a constexpr. - -static inline HVX_Vector hvx_vec_rmpy_x8_n(HVX_Vector_x8 x, HVX_Vector_x8 y, unsigned int n) { - HVX_Vector r0 = Q6_V_vzero(); - HVX_Vector r1 = Q6_V_vzero(); - HVX_Vector r2 = Q6_V_vzero(); - HVX_Vector r3 = Q6_V_vzero(); - HVX_Vector r4 = Q6_V_vzero(); - HVX_Vector r5 = Q6_V_vzero(); - HVX_Vector r6 = Q6_V_vzero(); - HVX_Vector r7 = Q6_V_vzero(); - - HVX_VectorPair p3; - HVX_VectorPair p2; - HVX_VectorPair p1; - HVX_VectorPair p0; +static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; - if (n >= 128) { r0 = Q6_Vw_vrmpy_VbVb(x.v[0], y.v[0]); } - if (n >= 256) { r1 = Q6_Vw_vrmpy_VbVb(x.v[1], y.v[1]); } - if (n >= 384) { r2 = Q6_Vw_vrmpy_VbVb(x.v[2], y.v[2]); } - if (n >= 512) { r3 = Q6_Vw_vrmpy_VbVb(x.v[3], y.v[3]); } - if (n >= 640) { r4 = Q6_Vw_vrmpy_VbVb(x.v[4], y.v[4]); } - if (n >= 768) { r5 = Q6_Vw_vrmpy_VbVb(x.v[5], y.v[5]); } - if (n >= 896) { r6 = Q6_Vw_vrmpy_VbVb(x.v[6], y.v[6]); } - if (n >= 1024) { r7 = Q6_Vw_vrmpy_VbVb(x.v[7], y.v[7]); } + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - if (n >= 128) { p0 = Q6_W_vdeal_VVR(r1, r0, -4); } - if (n >= 384) { p1 = Q6_W_vdeal_VVR(r3, r2, -4); } - if (n >= 640) { p2 = Q6_W_vdeal_VVR(r5, r4, -4); } - if (n >= 896) { p3 = Q6_W_vdeal_VVR(r7, r6, -4); } + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); - if (n >= 128) { r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0)); } - if (n >= 384) { r1 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p1), Q6_V_hi_W(p1)); } - if (n >= 640) { r2 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p2), Q6_V_hi_W(p2)); } - if (n >= 896) { r3 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p3), Q6_V_hi_W(p3)); } + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - if (n >= 128) { p0 = Q6_W_vdeal_VVR(r1, r0, -4); } - if (n >= 640) { p1 = Q6_W_vdeal_VVR(r3, r2, -4); } + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - if (n >= 128) { r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0)); } - if (n >= 640) { r1 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p1), Q6_V_hi_W(p1)); } + const size_t dst_row_size = nb1; + const size_t act_row_size = nb11; + const size_t act_stride = mmctx->vtcm_act_stride; - if (n >= 128) { p0 = Q6_W_vdeal_VVR(r1, r0, -4); } - if (n >= 128) { r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0)); } + uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; + uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * act_data = mmctx->vtcm_act; - return r0; -} + float * tmp = (float *) vtcm_dst_ptr; -static inline HVX_Vector hvx_vec_rmpy_x8_full(HVX_Vector_x8 x, HVX_Vector_x8 y) { - HVX_Vector r0 = Q6_Vw_vrmpy_VbVb(x.v[0], y.v[0]); - HVX_Vector r1 = Q6_Vw_vrmpy_VbVb(x.v[1], y.v[1]); - HVX_Vector r2 = Q6_Vw_vrmpy_VbVb(x.v[2], y.v[2]); - HVX_Vector r3 = Q6_Vw_vrmpy_VbVb(x.v[3], y.v[3]); - HVX_Vector r4 = Q6_Vw_vrmpy_VbVb(x.v[4], y.v[4]); - HVX_Vector r5 = Q6_Vw_vrmpy_VbVb(x.v[5], y.v[5]); - HVX_Vector r6 = Q6_Vw_vrmpy_VbVb(x.v[6], y.v[6]); - HVX_Vector r7 = Q6_Vw_vrmpy_VbVb(x.v[7], y.v[7]); + const dma_addr_t src0_row = src0->data; - HVX_VectorPair p0 = Q6_W_vdeal_VVR(r1, r0, -4); - HVX_VectorPair p1 = Q6_W_vdeal_VVR(r3, r2, -4); - HVX_VectorPair p2 = Q6_W_vdeal_VVR(r5, r4, -4); - HVX_VectorPair p3 = Q6_W_vdeal_VVR(r7, r6, -4); + const uint8_t * restrict act_col = (const uint8_t *) act_data; + float * restrict dst_col = (float *) dst->data; - r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0)); - r1 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p1), Q6_V_hi_W(p1)); - r2 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p2), Q6_V_hi_W(p2)); - r3 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p3), Q6_V_hi_W(p3)); + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); + + uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); + uint32_t tile_row_stride = n_k_tiles_w * tile_size; + uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; + + uint32_t ct_start = src0_start_row / 32; + uint32_t ct_end = (src0_end_row + 31) / 32; + + uint32_t push_ct = ct_start; + if (src0_start_row < src0_end_row) { + if (src2) { + float * vtcm_bias_ptr = (float *) mmctx->vtcm_bias + src0_start_row; + const dma_addr_t src2_addr = src2->data + src0_start_row * sizeof(float); + int slice_size = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; + if (slice_size > 0) { + dma_queue_push(dma_q, dma_make_data(vtcm_bias_ptr, src2_addr), + slice_size * sizeof(float), slice_size * sizeof(float), slice_size * sizeof(float), 1); + dma_queue_pop_nowait(dma_q); + } + } + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } + } - p0 = Q6_W_vdeal_VVR(r1, r0, -4); - p1 = Q6_W_vdeal_VVR(r3, r2, -4); + if (src0_start_row >= src0_end_row) { + return; + } - r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0)); - r1 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p1), Q6_V_hi_W(p1)); + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - p0 = Q6_W_vdeal_VVR(r1, r0, -4); - r0 = Q6_Vw_vadd_VwVw(Q6_V_lo_W(p0), Q6_V_hi_W(p0)); + float * dst_ptr = &tmp[ct * 32 - src0_start_row]; + int valid_rows = (int)ne0 - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); - return r0; -} + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + mmctx->vec_dot_32x1(ne10, dst_ptr, w_tile, act_col, valid_rows, NULL); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); -static inline HVX_Vector hvx_vec_rmpy_x8_partial(HVX_Vector_x8 x, HVX_Vector_x8 y, unsigned int n) { - if (n >= 512) - return hvx_vec_rmpy_x8_full(x, y); + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } - return hvx_vec_rmpy_x8_partial(x, y, 512); + int copy_cnt = (int)MIN(src0_end_row, ne0) - (int)src0_start_row; + if (copy_cnt > 0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); + if (src2) { + hvx_add_f32_uuu((uint8_t *) &dst_col[src0_start_row], + (const uint8_t *) tmp, + (const uint8_t *) ((const float *) mmctx->vtcm_bias + src0_start_row), + copy_cnt); + } else { + hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct_end); + } } -static void vec_dot_q4x4x2_q8x4x2_1x1(const int n, float * restrict s0, const void * restrict vx0, const void * restrict vy0) { - assert(n % 32 == 0); // min sub-block size - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); - - const uint32_t qk = QK_Q4_0x4x2 * 4; - - const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t x_qblk_size = qk / 2; // int4 - const uint32_t x_qrow_size = n / 2; // int4 (not padded) +static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) { + struct htp_mm_context * mmctx = data; + struct htp_ops_context * octx = mmctx->octx; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_weights = kparams->n_weights; + + const struct htp_tensor * restrict act = octx->src[n_weights]; /* x */ + const uint32_t ne10 = act->ne[0]; + const uint32_t act_nrows = act->ne[1] * act->ne[2] * act->ne[3]; + const size_t act_stride = mmctx->vtcm_act_stride; + + uint8_t * restrict vtcm_weight_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + + for (uint32_t widx = 0; widx < n_weights; widx++) { + const struct htp_tensor * restrict src_w = octx->src[widx]; + const struct htp_tensor * restrict dst = octx->dsts[widx]; + if (!src_w || !dst) continue; + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); + const uint32_t ne00 = src_w->ne[0]; + const uint32_t ne01 = src_w->ne[1]; + const size_t dst_row_size = dst->nb[1]; + const dma_addr_t src_w_row = src_w->data; + + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src_w->type); + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; + + uint32_t src0_start_row = 0; + uint32_t src0_end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_start_row = range.start; + src0_end_row = range.start + range.count; + } - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) + const uint32_t nrows = src0_end_row - src0_start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); + src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0 + 0); // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0 + x_qrow_size); // then scales + const uint32_t start_row = src0_start_row + src0_nrows_per_thread * ith; + const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_end_row); + if (start_row >= end_row) continue; - const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first - const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales + uint32_t ct_start = start_row / 32; + uint32_t ct_end = (end_row + 31) / 32; - // Row sum (sf) - HVX_Vector r0_sum = Q6_V_vzero(); + uint32_t push_ct = ct_start; + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_weight_ptr + d * tile_row_transfer_size_aligned, + src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } - // Multiply and accumulate into int32. - // Compute combined scale (fp32). - // Apply scale to acc and accumulate into the row sum (qf32). + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; + int valid_rows = (int)dst->ne[0] - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); - const uint32_t nb = n / qk; // num full blocks - const uint32_t nloe = n % qk; // num leftover elemements + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + uint32_t ir1 = 0; + for (; ir1 + 1 < act_nrows; ir1 += 2) { + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (ir1+0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (ir1+1) * act_stride); - uint32_t i = 0; - for (; i < nb; i++) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size); - HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_full(r0_x_q + i * x_qblk_size); + float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); + float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); + float * dst_ptr0 = &dst_row0[ct * 32]; + float * dst_ptr1 = &dst_row1[ct * 32]; - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q)); + mmctx->vec_dot_32x2(ne10, dst_ptr0, dst_ptr1, w_tile, act_col0, act_col1, valid_rows, NULL, NULL); + } - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); + for (; ir1 < act_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + ir1 * act_stride); + float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); + float * dst_ptr = &dst_row[ct * 32]; + mmctx->vec_dot_32x1(ne10, dst_ptr, w_tile, act_col, valid_rows, NULL); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src_w_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } + } +} - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); +static void hvx_mm_transfer_act_dma( + struct htp_ops_context * octx, + const struct htp_mm_kernel_params * kparams, + const struct htp_tensor * act, + uint8_t * dst_base, + size_t dst_row_size, + uint32_t m_start, + uint32_t m_rows +) { + if (m_rows == 0) { + return; + } - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); + dma_queue * dma_q = octx->ctx->dma[0]; + const uint32_t ne0 = act->ne[0]; + const size_t elem_size = (act->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + const size_t row_bytes = ne0 * elem_size; + const size_t act_nb1 = act->nb[1]; + const dma_addr_t act_base = act->data; + + const bool is_contiguous = (act->nb[2] == act->ne[1] * act_nb1) && + (act->nb[3] == act->ne[2] * act->nb[2]); + + if (is_contiguous) { + const dma_addr_t src_addr = act_base + m_start * act_nb1; + dma_queue_push(dma_q, dma_make_data(dst_base, src_addr), + dst_row_size, act_nb1, row_bytes, m_rows); + dma_queue_pop(dma_q); + } else { + const uint32_t ne12_ne1 = act->ne[2] * act->ne[1]; + const bool use_fastdiv = kparams->div_ne12_ne1.mp != 0; + for (uint32_t ir = 0; ir < m_rows; ++ir) { + const uint32_t ir1 = m_start + ir; + uint32_t i11, i12, i13; + if (use_fastdiv) { + i13 = fastdiv(ir1, &kparams->div_ne12_ne1); + const uint32_t rem = ir1 - i13 * ne12_ne1; + i12 = fastdiv(rem, &kparams->div_ne1); + i11 = rem - i12 * act->ne[1]; + } else { + i13 = ne12_ne1 ? ir1 / ne12_ne1 : 0; + const uint32_t rem = ir1 - i13 * ne12_ne1; + i12 = act->ne[1] ? rem / act->ne[1] : 0; + i11 = rem - i12 * act->ne[1]; + } + const dma_addr_t row_src = act_base + (i11 * act->nb[1] + + i12 * act->nb[2] + + i13 * act->nb[3]); + uint8_t * row_dst = dst_base + ir * dst_row_size; + dma_queue_push(dma_q, dma_make_data(row_dst, row_src), + dst_row_size, act_nb1, row_bytes, 1); + dma_queue_pop(dma_q); + } } - // Process leftovers - if (nloe) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_partial(r0_x_q + i * x_qblk_size, nloe); + if (dst_row_size > row_bytes) { + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) m_start); + const uint32_t pad_elems = (dst_row_size - row_bytes) / elem_size; + if (elem_size == sizeof(float)) { + for (uint32_t ir = 0; ir < m_rows; ++ir) { + hvx_splat_f32_u(dst_base + ir * dst_row_size + row_bytes, 0.0f, pad_elems); + } + } else { + for (uint32_t ir = 0; ir < m_rows; ++ir) { + hvx_splat_f16_u(dst_base + ir * dst_row_size + row_bytes, (_Float16) 0.0f, pad_elems); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) m_start); + } +} - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe)); +#define QUANTIZE_IMPL(name, log_name, kernel_fn, dst_row_size_expr) \ +static void name(unsigned int nth, unsigned int ith, void * data) { \ + (void) nth; \ + struct htp_mm_context * mmctx = data; \ + struct htp_ops_context * octx = mmctx->octx; \ + const struct htp_tensor * src = mmctx->act; \ + const uint32_t ne0 = src->ne[0]; \ + const uint32_t nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; \ + const uint32_t nrows_per_thread = mmctx->n_quant_rows_per_thread; \ + \ + const uint32_t ir_first = nrows_per_thread * ith; \ + if (ir_first >= nrows) { \ + return; \ + } \ + \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ + \ + uint8_t * restrict dst = mmctx->vtcm_act; \ + const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); \ + const size_t raw_row_size = mmctx->vtcm_act_raw_stride; \ + const size_t dst_row_size = (dst_row_size_expr); \ + \ + const uint8_t * restrict src_data = (const uint8_t *) mmctx->vtcm_act_raw + (raw_row_size * ir_first); \ + uint8_t * restrict dst_data = (uint8_t *) dst + (dst_row_size * ir_first); \ + kernel_fn(src_data, dst_data, NULL, ne0, ir_last - ir_first, raw_row_size, dst_row_size); \ + \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \ +} - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); +QUANTIZE_IMPL(quantize_f32_q8_0_tiled, "quantize-f32-q8_0_tiled", quantize_f32_q8_0_tiled_kernel, htp_mm_q8_0_tiled_row_size(ne0)) +QUANTIZE_IMPL(quantize_f32_q8_1_tiled, "quantize-f32-q8_1_tiled", quantize_f32_q8_1_tiled_kernel, htp_mm_q8_1_tiled_row_size(ne0)) +QUANTIZE_IMPL(quantize_f32_q8_1_s16_tiled, "quantize-f32-q8_1_s16_tiled", quantize_f32_q8_1_s16_tiled_kernel, htp_mm_q8_1_tiled_row_size(ne0)) +QUANTIZE_IMPL(quantize_f32_f16_tiled, "quantize-f32-f16_tiled", quantize_f32_f16_tiled_kernel, htp_mm_f16_tiled_row_size(ne0)) +QUANTIZE_IMPL(quantize_f16_f16_tiled, "quantize-f16-f16_tiled", quantize_f16_f16_tiled_kernel, htp_mm_f16_tiled_row_size(ne0)) +QUANTIZE_IMPL(quantize_f32_f32, "quantize-f32-f32", quantize_f32_f32_kernel, mmctx->vtcm_act_stride) +QUANTIZE_IMPL(quantize_f32_f16, "quantize-f32-f16", quantize_f32_f16_kernel, mmctx->vtcm_act_stride) +QUANTIZE_IMPL(quantize_f16_f16, "quantize-f16-f16", quantize_f16_f16_kernel, mmctx->vtcm_act_stride) + +static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_mm_context * mmctx = data; + if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { + return; + } + struct htp_ops_context * octx = mmctx->octx; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); + + const struct htp_tensor * src = mmctx->act; + + quantize_f32_q8_0_tiled_block_kernel( + (const float *) mmctx->vtcm_act_raw, + mmctx->vtcm_act, + NULL, + src->ne[0], + mmctx->quant_ib_first[ith], + mmctx->quant_ib_last[ith], + mmctx->vtcm_act_raw_stride, + htp_mm_q8_0_tiled_row_size(src->ne[0]), + mmctx->quant_r[ith], + mmctx->quant_c[ith] + ); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); +} - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); +static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_mm_context * mmctx = data; + if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { + return; + } + struct htp_ops_context * octx = mmctx->octx; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); + + const struct htp_tensor * src = mmctx->act; + + quantize_f32_q8_1_tiled_block_kernel( + (const float *) mmctx->vtcm_act_raw, + mmctx->vtcm_act, + NULL, + src->ne[0], + mmctx->quant_ib_first[ith], + mmctx->quant_ib_last[ith], + mmctx->vtcm_act_raw_stride, + htp_mm_q8_1_tiled_row_size(src->ne[0]), + mmctx->quant_r[ith], + mmctx->quant_c[ith] + ); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); +} - // Zero out unused elements - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_dd = Q6_V_vand_QV(bmask, r0_dd); - r0_ia = Q6_V_vand_QV(bmask, r0_ia); +static void quantize_f32_q8_1_s16_tiled_block(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_mm_context * mmctx = data; + if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { + return; + } + struct htp_ops_context * octx = mmctx->octx; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); + + const struct htp_tensor * src = mmctx->act; + + quantize_f32_q8_1_s16_tiled_block_kernel( + (const float *) mmctx->vtcm_act_raw, + mmctx->vtcm_act, + NULL, + src->ne[0], + mmctx->quant_ib_first[ith], + mmctx->quant_ib_last[ith], + mmctx->vtcm_act_raw_stride, + htp_mm_q8_1_tiled_row_size(src->ne[0]), + mmctx->quant_r[ith], + mmctx->quant_c[ith] + ); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); +} - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); +static void quantize_f32_f16_tiled_block(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_mm_context * mmctx = data; + if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { + return; + } + struct htp_ops_context * octx = mmctx->octx; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); + + const struct htp_tensor * src = mmctx->act; + + quantize_f32_f16_tiled_block_kernel( + (const float *) mmctx->vtcm_act_raw, + mmctx->vtcm_act, + NULL, + src->ne[0], + mmctx->quant_ib_first[ith], + mmctx->quant_ib_last[ith], + mmctx->vtcm_act_raw_stride, + htp_mm_f16_tiled_row_size(src->ne[0]), + mmctx->quant_r[ith], + mmctx->quant_c[ith] + ); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); +} - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); +static void quantize_f16_f16_tiled_block(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + struct htp_mm_context * mmctx = data; + if (mmctx->quant_ib_first[ith] >= mmctx->quant_ib_last[ith]) { + return; } + struct htp_ops_context * octx = mmctx->octx; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); + + const struct htp_tensor * src = mmctx->act; + + quantize_f16_f16_tiled_block_kernel( + (const __fp16 *) mmctx->vtcm_act_raw, + mmctx->vtcm_act, + NULL, + src->ne[0], + mmctx->quant_ib_first[ith], + mmctx->quant_ib_last[ith], + mmctx->vtcm_act_raw_stride, + htp_mm_f16_tiled_row_size(src->ne[0]), + mmctx->quant_r[ith], + mmctx->quant_c[ith] + ); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]); +} - r0_sum = hvx_vec_reduce_sum_f32(r0_sum); +// q8_1 for weight types with offsets (q8_1_s16 for Q2_K), f16_tiled for F16, otherwise q8_0 +static inline worker_callback_t htp_mm_act_quant_row_func(int weight_type, int act_type) { + if (weight_type == HTP_TYPE_F16) { + return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled : quantize_f32_f16_tiled; + } + if (weight_type == HTP_TYPE_Q2_K) { + return quantize_f32_q8_1_s16_tiled; + } + return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled; +} - hvx_vec_store_u(s0, 4, r0_sum); +static inline worker_callback_t htp_mm_act_quant_block_func(int weight_type, int act_type) { + if (weight_type == HTP_TYPE_F16) { + return (act_type == HTP_TYPE_F16) ? quantize_f16_f16_tiled_block : quantize_f32_f16_tiled_block; + } + if (weight_type == HTP_TYPE_Q2_K) { + return quantize_f32_q8_1_s16_tiled_block; + } + return htp_mm_weight_has_offset(weight_type) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block; } -static void vec_dot_q4x4x2_q8x4x2_2x1(const int n, float * restrict s0, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0) { - assert(n % 32 == 0); // min sub-block size - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vx1 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); +static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; + + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; + const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); + const uint32_t cur_m_start = mmctx->cur_m_start; - const uint32_t qk = QK_Q4_0x4x2 * 4; + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); - const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t x_qblk_size = qk / 2; // int4 - const uint32_t x_qrow_size = n / 2; // int4 (not padded) + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales - const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first - const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales + const size_t dst_row_size = nb1; + const size_t act_stride = mmctx->vtcm_act_stride; - const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first - const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; - // Row sum (sf) - HVX_Vector r0_sum = Q6_V_vzero(); - HVX_Vector r1_sum = Q6_V_vzero(); + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); - // Multiply and accumulate into int32. - // Compute combined scale (fp32). - // Apply scale to acc and accumulate into the row sum (qf32). + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; + const uint32_t src0_slice_stride = ((ne01 + 31) / 32) * tile_row_stride; - const uint32_t nb = n / qk; // num full blocks - const uint32_t nloe = n % qk; // num leftover elemements + const uint32_t ct_start = src0_start_row / 32; + const uint32_t ct_end = (src0_end_row + 31) / 32; - uint32_t i = 0; - for (; i < nb; i++) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size); - HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_full(r0_x_q + i * x_qblk_size); - HVX_Vector_x8 r1_q = hvx_vec_load_q4x4x8_full(r1_x_q + i * x_qblk_size); + if (src0_start_row >= src0_end_row || cur_m_rows == 0) { + return; + } - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q)); - HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy_q)); + const uint32_t total_batches = ne12 * ne13; + const uint32_t b_start = fastdiv(cur_m_start, &kparams->div_ne1); + uint32_t b_end = fastdiv(cur_m_start + cur_m_rows + ne11 - 1, &kparams->div_ne1); + b_end = MIN(b_end, total_batches); + + uint32_t b_grp_start = b_start; + while (b_grp_start < b_end) { + const uint32_t b3 = fastdiv(b_grp_start, &kparams->div_ne12); + const uint32_t b2 = b_grp_start - b3 * ne12; + const uint32_t i02 = fastdiv(b2, &kparams->div_r2); + const uint32_t i03 = fastdiv(b3, &kparams->div_r3); + + uint32_t b_grp_end = b_grp_start + 1; + while (b_grp_end < b_end) { + const uint32_t cur_b3 = fastdiv(b_grp_end, &kparams->div_ne12); + const uint32_t cur_b2 = b_grp_end - cur_b3 * ne12; + const uint32_t cur_i02 = fastdiv(cur_b2, &kparams->div_r2); + const uint32_t cur_i03 = fastdiv(cur_b3, &kparams->div_r3); + if (cur_i02 != i02 || cur_i03 != i03) { + break; + } + b_grp_end++; + } - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); + const uint32_t slice_idx = i03 * ne02 + i02; + const dma_addr_t src0_slice = src0->data + (size_t) slice_idx * src0_slice_stride; - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); - HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d))); + uint32_t push_ct = ct_start; + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, + src0_slice + (size_t) push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd); + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; + + int valid_rows = (int)ne0 - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + for (uint32_t b = b_grp_start; b < b_grp_end; b++) { + const uint32_t b_m_start = b * ne11; + const uint32_t m_first = MAX(cur_m_start, b_m_start); + const uint32_t m_last = MIN(cur_m_start + cur_m_rows, b_m_start + ne11); + if (m_first >= m_last) continue; + + const uint32_t cur_b3 = fastdiv(b, &kparams->div_ne12); + const uint32_t cur_b2 = b - cur_b3 * ne12; + uint8_t * dst_batch_base = (uint8_t *) dst->data + (size_t) cur_b2 * nb2 + (size_t) cur_b3 * nb3; + + const uint32_t chunk_m_offset = m_first - cur_m_start; + const uint32_t dst_m_offset = m_first - b_m_start; + const uint32_t batch_nrows = m_last - m_first; + + uint32_t ir1 = 0; + for (; ir1 + 1 < batch_nrows; ir1 += 2) { + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (chunk_m_offset + ir1 + 0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (chunk_m_offset + ir1 + 1) * act_stride); + float * restrict dst_row0 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 0) * dst_row_size); + float * restrict dst_row1 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 1) * dst_row_size); + float * dst_ptr0 = &dst_row0[ct * 32]; + float * dst_ptr1 = &dst_row1[ct * 32]; + mmctx->vec_dot_32x2(ne10, dst_ptr0, dst_ptr1, w_tile, act_col0, act_col1, valid_rows, NULL, NULL); + } + for (; ir1 < batch_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (chunk_m_offset + ir1) * act_stride); + float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); + float * dst_ptr = &dst_row[ct * 32]; + mmctx->vec_dot_32x1(ne10, dst_ptr, w_tile, act_col, valid_rows, NULL); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum)); + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_slice + (size_t) push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } + b_grp_start = b_grp_end; } + if (src2) { + hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + cur_m_rows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); + } +} - // Process leftovers - if (nloe) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_partial(r0_x_q + i * x_qblk_size, nloe); - HVX_Vector_x8 r1_q = hvx_vec_load_q4x4x8_partial(r1_x_q + i * x_qblk_size, nloe); +static void hvx_mm_2d_flat(unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe)); - HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy_q, nloe)); + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + const uint32_t prefetch_mask = n_prefetch - 1; - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows + const uint32_t act_nrows = mmctx->cur_m_rows ? mmctx->cur_m_rows : mmctx->act_nrows; + const uint32_t cur_m_start = mmctx->cur_m_start; - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); - HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d))); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - // Zero out unused elements - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_dd = Q6_V_vand_QV(bmask, r0_dd); - r1_dd = Q6_V_vand_QV(bmask, r1_dd); - r0_ia = Q6_V_vand_QV(bmask, r0_ia); - r1_ia = Q6_V_vand_QV(bmask, r1_ia); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd); + const size_t dst_row_size = nb1; + const size_t src0_row_size = nb01; + const size_t act_row_size = nb11; - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum)); - } + const size_t src0_stride = mmctx->vtcm_src0_stride; + const size_t act_stride = mmctx->vtcm_act_stride; - HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(r0_sum, r1_sum); - hvx_vec_store_u(s0, 8, rsum); -} + // Per-thread VTCMs for all tensors + uint8_t * restrict vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; -static void vec_dot_q4x4x2_q8x4x2_2x2(const int n, float * restrict s0, float * restrict s1, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0, const void * restrict vy1) { - assert(n % 32 == 0); - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vx1 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); - assert((unsigned long) vy1 % 128 == 0); - - const uint32_t qk = QK_Q4_0x4x2 * 4; - - const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t x_qblk_size = qk / 2; // int4 - const uint32_t x_qrow_size = n / 2; // int4 (not padded) - - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) - - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales - const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first - const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales - - const uint8_t * restrict y0_q = ((const uint8_t *) vy0) + 0; // quants first - const uint8_t * restrict y0_d = ((const uint8_t *) vy0) + y_qrow_size; // then scales - const uint8_t * restrict y1_q = ((const uint8_t *) vy1) + 0; // quants first - const uint8_t * restrict y1_d = ((const uint8_t *) vy1) + y_qrow_size; // then scales - - // Row sums (sf) - 4 accumulators for 2×2 tile - HVX_Vector r0_c0_sum = Q6_V_vzero(); - HVX_Vector r0_c1_sum = Q6_V_vzero(); - HVX_Vector r1_c0_sum = Q6_V_vzero(); - HVX_Vector r1_c1_sum = Q6_V_vzero(); - - const uint32_t nb = n / qk; // num full blocks - const uint32_t nloe = n % qk; // num leftover elements - - uint32_t i = 0; - for (; i < nb; i++) { - // Load src1 columns (reused across both src0 rows) - HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_full(y0_q + i * y_qblk_size); - HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_full(y1_q + i * y_qblk_size); - - // Load src0 rows (reused across both src1 columns) - HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_full(r0_x_q + i * x_qblk_size); - HVX_Vector_x8 r1_q = hvx_vec_load_q4x4x8_full(r1_x_q + i * x_qblk_size); - - // Compute 4 dot products: r0×c0, r0×c1, r1×c0, r1×c1 - HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy0_q)); - HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy1_q)); - HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy0_q)); - HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy1_q)); - - // Load scales - HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size)); - HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); - - // Compute combined scales - HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d))); - HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d))); - HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d))); - HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d))); - - // Apply scales and accumulate - HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd); - HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd); - HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd); - HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd); - - r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum)); - r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum)); - r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum)); - r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum)); - } + const dma_addr_t src0_row = src0->data; - // Process leftovers - if (nloe) { - HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_partial(y0_q + i * y_qblk_size, nloe); - HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_partial(y1_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_q4x4x8_partial(r0_x_q + i * x_qblk_size, nloe); - HVX_Vector_x8 r1_q = hvx_vec_load_q4x4x8_partial(r1_x_q + i * x_qblk_size, nloe); - HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy0_q, nloe)); - HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy1_q, nloe)); - HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy0_q, nloe)); - HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy1_q, nloe)); + // Prefill vtcm with src0 rows + if (src0_start_row < src0_end_row) { + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const int is0 = (ir0 - src0_start_row); + if (is0 >= (int)n_prefetch) { + break; + } + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + } + } - HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size)); - HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); + if (src0_start_row >= src0_end_row) { + return; + } - HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d))); - HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d))); - HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d))); - HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d))); + // Process src0 rows + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; - // Zero out unused scales - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_c0_dd = Q6_V_vand_QV(bmask, r0_c0_dd); - r0_c1_dd = Q6_V_vand_QV(bmask, r0_c1_dd); - r1_c0_dd = Q6_V_vand_QV(bmask, r1_c0_dd); - r1_c1_dd = Q6_V_vand_QV(bmask, r1_c1_dd); - r0_c0_ia = Q6_V_vand_QV(bmask, r0_c0_ia); - r0_c1_ia = Q6_V_vand_QV(bmask, r0_c1_ia); - r1_c0_ia = Q6_V_vand_QV(bmask, r1_c0_ia); - r1_c1_ia = Q6_V_vand_QV(bmask, r1_c1_ia); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + // Process act columns in pairs (2x2 tiling) + uint32_t ir1 = 0; + for (; ir1 + 1 < act_nrows; ir1 += 2) { + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (ir1+0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (ir1+1) * act_stride); + float * restrict dst_row0 = (float *) (dst->data + ((cur_m_start + ir1+0) * dst_row_size)); + float * restrict dst_row1 = (float *) (dst->data + ((cur_m_start + ir1+1) * dst_row_size)); + mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, act_col0, act_col1); + } - HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd); - HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd); - HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd); - HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd); + // Handle remaining act rows (fallback to 2x1) + for (; ir1 < act_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + ir1 * act_stride); + float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); + mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, act_col); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum)); - r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum)); - r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum)); - r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum)); + // Prefetch next (n + vtcm_nrows) row + const int pr0 = (ir0 + n_prefetch); + const int is0 = (pr0 - src0_start_row) & prefetch_mask; + if (pr0 < src0_end_row_x2) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + } } - // Reduce and store results - HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); - HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + // Process the last row (if any) + if (src0_end_row != src0_end_row_x2) { + uint32_t ir0 = src0_end_row_x2; + const int is0 = (ir0 - src0_start_row) & prefetch_mask; + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 1); + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; - hvx_vec_store_u(s0, 8, r0_r1_c0_sum); // row0,col0 row1,col0 - hvx_vec_store_u(s1, 8, r0_r1_c1_sum); // row0,col1 row1,col1 + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + #pragma unroll(2) + for (uint32_t ir1 = 0; ir1 < act_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + ir1 * act_stride); + float * restrict dst_row = (float *) (dst->data + ((cur_m_start + ir1) * dst_row_size)); + mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, act_col); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + } + if (src2) { + hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + act_nrows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); + } } -static void vec_dot_q8x4x2_q8x4x2_1x1(const int n, float * restrict s0, const void * restrict vx0, const void * restrict vy0) { - assert(n % 32 == 0); // min sub-block size - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); +static void hvx_mv_2d_flat(unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; - const uint32_t qk = QK_Q4_0x4x2 * 4; + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; - const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t x_qblk_size = qk; // int8 - const uint32_t x_qrow_size = n; // int8 (not padded) + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0 + 0); // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0 + x_qrow_size); // then scales + const size_t dst_row_size = nb1; + const size_t src0_row_size = nb01; + const size_t act_row_size = nb11; - const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first - const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales + const size_t src0_stride = mmctx->vtcm_src0_stride; + const size_t act_stride = mmctx->vtcm_act_stride; - // Row sum (sf) - HVX_Vector r0_sum = Q6_V_vzero(); + // Per-thread VTCMs for all tensors + uint8_t * vtcm_dst_ptr = mmctx->vtcm_dst + mmctx->vtcm_dst_size_per_thread * ith; + uint8_t * vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * act_data = mmctx->vtcm_act; - // Multiply and accumulate into int32. - // Compute combined scale (fp32). - // Apply scale to acc and accumulate into the row sum (qf32). + float * tmp = (float *) vtcm_dst_ptr; - const uint32_t nb = n / qk; // num full blocks - int32_t nloe = n % qk; // num leftover elemements (must be signed) + const dma_addr_t src0_row = src0->data; + const uint8_t * restrict act_col = (const uint8_t *) act_data; + float * restrict dst_col = (float *) dst->data; - uint32_t i = 0; - for (; i < nb; i++) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size); - HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_full(r0_x_q + i * x_qblk_size); + const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q)); + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + const uint32_t prefetch_mask = n_prefetch - 1; + + // Prefill vtcm with 2x src0 rows + if (src0_start_row < src0_end_row) { + if (src2) { + float * vtcm_bias_ptr = (float *) mmctx->vtcm_bias + src0_start_row; + const dma_addr_t src2_addr = src2->data + src0_start_row * sizeof(float); + int slice_size = (int)src0_end_row - (int)src0_start_row; + if (slice_size > 0) { + dma_queue_push(dma_q, dma_make_data(vtcm_bias_ptr, src2_addr), + slice_size * sizeof(float), slice_size * sizeof(float), slice_size * sizeof(float), 1); + dma_queue_pop_nowait(dma_q); + } + } + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const uint32_t is0 = (ir0 - src0_start_row); + if (is0 >= n_prefetch) { + break; + } + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + } + } - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); + if (src0_start_row >= src0_end_row) { + return; + } - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); + // Process src0 rows + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + mmctx->vec_dot_2x1(ne00, &tmp[ir0 - src0_start_row], ss0, ss0 + src0_stride, act_col); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + + // Prefetch next (n + vtcm_nrows) row + const uint32_t pr0 = (ir0 + n_prefetch); + const uint32_t is0 = (pr0 - src0_start_row) & prefetch_mask; + if (pr0 < src0_end_row_x2) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + pr0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + } + } - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); + // Process the last row (if any) + if (src0_end_row != src0_end_row_x2) { + const uint32_t ir0 = src0_end_row_x2; + const uint32_t is0 = (ir0 - src0_start_row) & prefetch_mask; + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 1); + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + mmctx->vec_dot_1x1(ne00, &tmp[ir0 - src0_start_row], ss0, act_col); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); } - // Process leftovers - if (nloe) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_partial(r0_x_q + i * x_qblk_size, nloe); + int copy_cnt = src0_end_row - src0_start_row; + if (copy_cnt > 0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, src0_end_row); + if (src2) { + hvx_add_f32_uuu((uint8_t *) &dst_col[src0_start_row], + (const uint8_t *) tmp, + (const uint8_t *) ((const float *) mmctx->vtcm_bias + src0_start_row), + copy_cnt); + } else { + hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, copy_cnt); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, src0_end_row); + } +} - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe)); +static void hvx_mm_4d_flat(unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + const uint32_t prefetch_mask = n_prefetch - 1; - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; + const uint32_t cur_m_rows = mmctx->cur_m_rows ? mmctx->cur_m_rows : (ne11 * ne12 * ne13); + const uint32_t cur_m_start = mmctx->cur_m_start; - // Zero out unused elements - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_dd = Q6_V_vand_QV(bmask, r0_dd); - r0_ia = Q6_V_vand_QV(bmask, r0_ia); + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); + const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - } + const size_t dst_row_size = nb1; + const size_t src0_row_size = nb01; + const size_t src0_stride = mmctx->vtcm_src0_stride; + const size_t act_stride = mmctx->vtcm_act_stride; - r0_sum = hvx_vec_reduce_sum_f32(r0_sum); + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; - hvx_vec_store_u(s0, 4, r0_sum); -} -static void vec_dot_q8x4x2_q8x4x2_2x1(const int n, float * restrict s0, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0) { - assert(n % 32 == 0); // min sub-block size - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vx1 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); + if (src0_start_row >= src0_end_row || cur_m_rows == 0) { + return; + } - const uint32_t qk = QK_Q4_0x4x2 * 4; + const uint32_t total_batches = ne12 * ne13; + const uint32_t b_start = fastdiv(cur_m_start, &kparams->div_ne1); + uint32_t b_end = fastdiv(cur_m_start + cur_m_rows + ne11 - 1, &kparams->div_ne1); + b_end = MIN(b_end, total_batches); + + uint32_t b_grp_start = b_start; + while (b_grp_start < b_end) { + const uint32_t b3 = fastdiv(b_grp_start, &kparams->div_ne12); + const uint32_t b2 = b_grp_start - b3 * ne12; + const uint32_t i02 = fastdiv(b2, &kparams->div_r2); + const uint32_t i03 = fastdiv(b3, &kparams->div_r3); + + uint32_t b_grp_end = b_grp_start + 1; + while (b_grp_end < b_end) { + const uint32_t cur_b3 = fastdiv(b_grp_end, &kparams->div_ne12); + const uint32_t cur_b2 = b_grp_end - cur_b3 * ne12; + const uint32_t cur_i02 = fastdiv(cur_b2, &kparams->div_r2); + const uint32_t cur_i03 = fastdiv(cur_b3, &kparams->div_r3); + if (cur_i02 != i02 || cur_i03 != i03) { + break; + } + b_grp_end++; + } - const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t x_qblk_size = qk; // int8 - const uint32_t x_qrow_size = n; // int8 (not padded) + const dma_addr_t src0_row = src0->data + ((size_t) i02 * nb02 + (size_t) i03 * nb03); - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const int is0 = (ir0 - src0_start_row); + if (is0 >= (int)n_prefetch) { + break; + } + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + (size_t) ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + } - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales - const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first - const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales + for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + for (uint32_t b = b_grp_start; b < b_grp_end; b++) { + const uint32_t b_m_start = b * ne11; + const uint32_t m_first = MAX(cur_m_start, b_m_start); + const uint32_t m_last = MIN(cur_m_start + cur_m_rows, b_m_start + ne11); + if (m_first >= m_last) continue; + + const uint32_t cur_b3 = fastdiv(b, &kparams->div_ne12); + const uint32_t cur_b2 = b - cur_b3 * ne12; + uint8_t * dst_batch_base = (uint8_t *) dst->data + (size_t) cur_b2 * nb2 + (size_t) cur_b3 * nb3; + + const uint32_t chunk_m_offset = m_first - cur_m_start; + const uint32_t dst_m_offset = m_first - b_m_start; + const uint32_t batch_nrows = m_last - m_first; + + uint32_t ir1 = 0; + for (; ir1 + 1 < batch_nrows; ir1 += 2) { + const uint8_t * restrict act_col0 = (const uint8_t *) (act_data + (chunk_m_offset + ir1 + 0) * act_stride); + const uint8_t * restrict act_col1 = (const uint8_t *) (act_data + (chunk_m_offset + ir1 + 1) * act_stride); + float * restrict dst_row0 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 0) * dst_row_size); + float * restrict dst_row1 = (float *) (dst_batch_base + (dst_m_offset + ir1 + 1) * dst_row_size); + mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, act_col0, act_col1); + } + for (; ir1 < batch_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (chunk_m_offset + ir1) * act_stride); + float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); + mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, act_col); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); - const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first - const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales + const int pr0 = (ir0 + n_prefetch); + const int is0 = (pr0 - src0_start_row) & prefetch_mask; + if (pr0 < src0_end_row_x2) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + (size_t) pr0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 2); + } + } - // Row sum (qf32) - HVX_Vector r0_sum = Q6_V_vzero(); - HVX_Vector r1_sum = Q6_V_vzero(); + if (src0_end_row != src0_end_row_x2) { + uint32_t ir0 = src0_end_row_x2; + const int is0 = (ir0 - src0_start_row) & prefetch_mask; + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + is0 * src0_stride, src0_row + (size_t) ir0 * src0_row_size), + src0_stride, src0_row_size, src0_row_size, 1); + const uint8_t * ss0 = (void *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + for (uint32_t b = b_grp_start; b < b_grp_end; b++) { + const uint32_t b_m_start = b * ne11; + const uint32_t m_first = MAX(cur_m_start, b_m_start); + const uint32_t m_last = MIN(cur_m_start + cur_m_rows, b_m_start + ne11); + if (m_first >= m_last) continue; + + const uint32_t cur_b3 = fastdiv(b, &kparams->div_ne12); + const uint32_t cur_b2 = b - cur_b3 * ne12; + uint8_t * dst_batch_base = (uint8_t *) dst->data + (size_t) cur_b2 * nb2 + (size_t) cur_b3 * nb3; + + const uint32_t chunk_m_offset = m_first - cur_m_start; + const uint32_t dst_m_offset = m_first - b_m_start; + const uint32_t batch_nrows = m_last - m_first; + + for (uint32_t ir1 = 0; ir1 < batch_nrows; ++ir1) { + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (chunk_m_offset + ir1) * act_stride); + float * restrict dst_row = (float *) (dst_batch_base + (dst_m_offset + ir1) * dst_row_size); + mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, act_col); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir0); + } - // Multiply and accumulate into int32. - // Compute combined scale (fp32). - // Apply scale to acc and accumulate into the row sum (qf32). + b_grp_start = b_grp_end; + } - const uint32_t nb = n / qk; // num full blocks - int32_t nloe = n % qk; // num leftover elemements (must be signed) + if (src2) { + hvx_tensor_add_f32_grid(dst, src2, cur_m_start, cur_m_start + cur_m_rows, src0_start_row, src0_end_row, &kparams->div_ne12_ne1, &kparams->div_ne1); + } +} - uint32_t i = 0; - for (; i < nb; i++) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size); - HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_full(r0_x_q + i * x_qblk_size); - HVX_Vector_x8 r1_q = hvx_vec_load_q8x4x8_full(r1_x_q + i * x_qblk_size); +#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * mmctx->mapping_stride + (i1)] - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q)); - HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy_q)); +static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); + const struct htp_tensor * restrict ids = octx->src[2]; - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); - HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d))); + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert + const uint32_t act_nrows = ne11; + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd); - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum)); + if (src0_start_row >= src0_end_row) { + return; } - // Process leftovers - if (nloe) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_partial(r0_x_q + i * x_qblk_size, nloe); - HVX_Vector_x8 r1_q = hvx_vec_load_q8x4x8_partial(r1_x_q + i * x_qblk_size, nloe); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe)); - HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy_q, nloe)); + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); + const uint32_t n_ids = ids->ne[0]; // n_expert_used + const uint32_t n_as = ne02; // n_expert - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); - HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d))); + const uint32_t * matrix_row_counts = mmctx->matrix_row_counts; + const struct mmid_row_mapping * matrix_rows = mmctx->matrix_rows; - // Zero out unused elements - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_dd = Q6_V_vand_QV(bmask, r0_dd); - r1_dd = Q6_V_vand_QV(bmask, r1_dd); - r0_ia = Q6_V_vand_QV(bmask, r0_ia); - r1_ia = Q6_V_vand_QV(bmask, r1_ia); + const size_t dst_row_size = nb1; + const size_t act_row_size = htp_mm_q8_0_tiled_row_size(ne10); - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd); + const size_t act_stride = mmctx->vtcm_act_stride; - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum)); - } + // Per-thread VTCMs for all tensors + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; - HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(r0_sum, r1_sum); - hvx_vec_store_u(s0, 8, rsum); -} + for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { + const int32_t cne1 = matrix_row_counts[cur_a]; + if (cne1 == 0) { + continue; + } -static void vec_dot_q8x4x2_q8x4x2_2x2(const int n, float * restrict s0, float * restrict s1, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0, const void * restrict vy1) { - assert(n % 32 == 0); - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vx1 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); - assert((unsigned long) vy1 % 128 == 0); - - const uint32_t qk = QK_Q8_0x4x2 * 4; - - const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t x_qblk_size = qk; // int8 - const uint32_t x_qrow_size = n; // int8 (not padded) - - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) - - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales - const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first - const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales - - const uint8_t * restrict y0_q = ((const uint8_t *) vy0) + 0; // quants first - const uint8_t * restrict y0_d = ((const uint8_t *) vy0) + y_qrow_size; // then scales - const uint8_t * restrict y1_q = ((const uint8_t *) vy1) + 0; // quants first - const uint8_t * restrict y1_d = ((const uint8_t *) vy1) + y_qrow_size; // then scales - - // Row sums (sf) - 4 accumulators for 2×2 tile - HVX_Vector r0_c0_sum = Q6_V_vzero(); - HVX_Vector r0_c1_sum = Q6_V_vzero(); - HVX_Vector r1_c0_sum = Q6_V_vzero(); - HVX_Vector r1_c1_sum = Q6_V_vzero(); - - const uint32_t nb = n / qk; // num full blocks - const uint32_t nloe = n % qk; // num leftover elements - - uint32_t i = 0; - for (; i < nb; i++) { - // Load src1 columns (reused across both src0 rows) - HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_full(y0_q + i * y_qblk_size); - HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_full(y1_q + i * y_qblk_size); - - // Load src0 rows (reused across both src1 columns) - HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_full(r0_x_q + i * x_qblk_size); - HVX_Vector_x8 r1_q = hvx_vec_load_q8x4x8_full(r1_x_q + i * x_qblk_size); - - // Compute 4 dot products: r0×c0, r0×c1, r1×c0, r1×c1 - HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy0_q)); - HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy1_q)); - HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy0_q)); - HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy1_q)); - - // Load scales - HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size)); - HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); - - // Compute combined scales - HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d))); - HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d))); - HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d))); - HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d))); - - // Apply scales and accumulate - HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd); - HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd); - HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd); - HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd); - - r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum)); - r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum)); - r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum)); - r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum)); - } - - // Process leftovers - if (nloe) { - HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_partial(y0_q + i * y_qblk_size, nloe); - HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_partial(y1_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_q8x4x8_partial(r0_x_q + i * x_qblk_size, nloe); - HVX_Vector_x8 r1_q = hvx_vec_load_q8x4x8_partial(r1_x_q + i * x_qblk_size, nloe); + const dma_addr_t src0_row = src0->data + cur_a * nb02; - HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy0_q, nloe)); - HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy1_q, nloe)); - HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy0_q, nloe)); - HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy1_q, nloe)); + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; - HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size)); - HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); + const uint32_t ct_start = src0_start_row / 32; + const uint32_t ct_end = (src0_end_row + 31) / 32; - HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d))); - HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d))); - HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d))); - HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d))); + uint32_t push_ct = ct_start; + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } - // Zero out unused elements - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_c0_dd = Q6_V_vand_QV(bmask, r0_c0_dd); - r0_c1_dd = Q6_V_vand_QV(bmask, r0_c1_dd); - r1_c0_dd = Q6_V_vand_QV(bmask, r1_c0_dd); - r1_c1_dd = Q6_V_vand_QV(bmask, r1_c1_dd); - r0_c0_ia = Q6_V_vand_QV(bmask, r0_c0_ia); - r0_c1_ia = Q6_V_vand_QV(bmask, r0_c1_ia); - r1_c0_ia = Q6_V_vand_QV(bmask, r1_c0_ia); - r1_c1_ia = Q6_V_vand_QV(bmask, r1_c1_ia); + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd); - HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd); - HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd); - HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd); + int valid_rows = (int)ne0 - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); - r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum)); - r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum)); - r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum)); - r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum)); - } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + for (uint32_t cid = 0; cid < cne1; ++cid) { + struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, cid); + const int rm1 = row_mapping.i1; // expert idx + const int rm2 = row_mapping.i2; // token idx - // Reduce and store results - HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); - HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); + const uint32_t ir1 = fastmodulo(rm1, ne11, &mmctx->mm_div_ne11); // act row idx + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (ir1 + rm2 * ne11 + 0) * act_stride); + float * restrict dst_row = (float *) (dst->data + (rm1 * nb1 + rm2 * nb2 + 0)); - hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0 - hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1 -} + mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, act_col, valid_rows, NULL); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); -// ======== IQ4_NL x Q8_0 vec_dot kernels ======== -// Same structure as Q4_0 vec_dot but uses IQ4_NL LUT-based load (4-bit index -> int8 kvalue). -// Scale format is identical to Q4_0 (fp16 scales). + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } + } +} -static void vec_dot_iq4nlx4x2_q8x4x2_1x1(const int n, - float * restrict s0, - const void * restrict vx0, - const void * restrict vy0) { - assert(n % 32 == 0); - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); +static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { + htp_matmul_preamble; - const uint32_t qk = QK_Q4_0x4x2 * 4; + const struct htp_tensor * restrict ids = octx->src[2]; - const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t x_qblk_size = qk / 2; // int4 - const uint32_t x_qrow_size = n / 2; // int4 (not padded) + const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert + const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0 + 0); // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0 + x_qrow_size); // then scales + if (src0_start_row >= src0_end_row) { + return; + } - const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first - const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - HVX_Vector r0_sum = Q6_V_vzero(); + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - const uint32_t nb = n / qk; - const uint32_t nloe = n % qk; + assert(ne13 % ne03 == 0); - uint32_t i = 0; - for (; i < nb; i++) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size); - HVX_Vector_x8 r0_q = hvx_vec_load_iq4nlx4x8_full(r0_x_q + i * x_qblk_size); + const size_t dst_row_size = nb1; + const size_t act_row_size = htp_mm_q8_0_tiled_row_size(ne10); - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q)); + const uint32_t n_aids = src2->ne[0]; // num activated experts + const uint32_t n_ids = ne02; // num experts - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); + // Per-thread VTCMs for all tensors + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); + for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert + const int32_t eid = *(const int32_t *) ((const uint8_t *) src2->data + ie1 * src2->nb[0]); + if (eid < 0) { + continue; + } + assert(eid >= 0 && eid < (int32_t) n_ids); - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); + const dma_addr_t src0_row = src0->data + eid * nb02; + const uint8_t * restrict act_col = (const uint8_t *) act_data; + float * restrict dst_row = (float *) (dst->data + ie1 * nb1); - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - } + const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; - if (nloe) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_iq4nlx4x8_partial(r0_x_q + i * x_qblk_size, nloe); + const uint32_t ct_start = src0_start_row / 32; + const uint32_t ct_end = (src0_end_row + 31) / 32; - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe)); + uint32_t push_ct = ct_start; + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); + int valid_rows = (int)ne0 - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_dd = Q6_V_vand_QV(bmask, r0_dd); - r0_ia = Q6_V_vand_QV(bmask, r0_ia); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + mmctx->vec_dot_32x1(ne10, &dst_row[ct * 32], w_tile, act_col, valid_rows, NULL); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } } - - r0_sum = hvx_vec_reduce_sum_f32(r0_sum); - - hvx_vec_store_u(s0, 4, r0_sum); } -static void vec_dot_iq4nlx4x2_q8x4x2_2x1(const int n, - float * restrict s0, - const void * restrict vx0, - const void * restrict vx1, - const void * restrict vy0) { - assert(n % 32 == 0); - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vx1 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); - - const uint32_t qk = QK_Q4_0x4x2 * 4; - - const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t x_qblk_size = qk / 2; // int4 - const uint32_t x_qrow_size = n / 2; // int4 (not padded) - - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) - - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales - const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first - const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales - - const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first - const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales - - HVX_Vector r0_sum = Q6_V_vzero(); - HVX_Vector r1_sum = Q6_V_vzero(); - - const uint32_t nb = n / qk; - const uint32_t nloe = n % qk; - - uint32_t i = 0; - for (; i < nb; i++) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full(y_q + i * y_qblk_size); - HVX_Vector_x8 r0_q = hvx_vec_load_iq4nlx4x8_full(r0_x_q + i * x_qblk_size); - HVX_Vector_x8 r1_q = hvx_vec_load_iq4nlx4x8_full(r1_x_q + i * x_qblk_size); - - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q)); - HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy_q)); - - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); +static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) { + struct htp_mm_context * mmctx = (struct htp_mm_context *) data; + struct htp_ops_context * octx = mmctx->octx; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_weights = kparams->n_weights; + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict act = octx->src[n_weights]; + const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; + + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); + + const uint32_t n_aids = ids->ne[0]; + const uint32_t n_ids = src0->ne[2]; + + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; + + for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { + const int32_t eid = *(const int32_t *) ((const uint8_t *) ids->data + ie1 * ids->nb[0]); + if (eid < 0) continue; + assert(eid < (int32_t) n_ids); + + for (uint32_t p = 0; p < n_weights; ++p) { + const struct htp_tensor * restrict src_w = octx->src[p]; + const struct htp_tensor * restrict dst = octx->dsts[p]; + if (!src_w || !dst) continue; + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t ne01 = src_w->ne[1]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); - HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d))); + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); + src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); + if (src0_start_row >= src0_end_row) continue; - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum)); - } + const dma_addr_t src0_row = src_w->data + eid * src_w->nb[2]; + const uint8_t * restrict act_col = (const uint8_t *) act_data; + float * restrict dst_row = (float *) (dst->data + ie1 * dst->nb[1]); - if (nloe) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial(y_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_iq4nlx4x8_partial(r0_x_q + i * x_qblk_size, nloe); - HVX_Vector_x8 r1_q = hvx_vec_load_iq4nlx4x8_partial(r1_x_q + i * x_qblk_size, nloe); + const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); + const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type); + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe)); - HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy_q, nloe)); + const uint32_t ct_start = src0_start_row / 32; + const uint32_t ct_end = (src0_end_row + 31) / 32; - HVX_Vector vy_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); + uint32_t push_ct = ct_start; + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy_d))); - HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy_d))); + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_dd = Q6_V_vand_QV(bmask, r0_dd); - r1_dd = Q6_V_vand_QV(bmask, r1_dd); - r0_ia = Q6_V_vand_QV(bmask, r0_ia); - r1_ia = Q6_V_vand_QV(bmask, r1_ia); + int valid_rows = (int)dst->ne[0] - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + mmctx->vec_dot_32x1(act->ne[0], &dst_row[ct * 32], w_tile, act_col, valid_rows, NULL); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum)); + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } + } } - - HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(r0_sum, r1_sum); - hvx_vec_store_u(s0, 8, rsum); } -static void vec_dot_iq4nlx4x2_q8x4x2_2x2(const int n, - float * restrict s0, - float * restrict s1, - const void * restrict vx0, - const void * restrict vx1, - const void * restrict vy0, - const void * restrict vy1) { - assert(n % 32 == 0); - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vx1 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); - assert((unsigned long) vy1 % 128 == 0); - - const uint32_t qk = QK_Q4_0x4x2 * 4; - - const uint32_t x_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t x_qblk_size = qk / 2; // int4 - const uint32_t x_qrow_size = n / 2; // int4 (not padded) - - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) - - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; - const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; - const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; - - const uint8_t * restrict y0_q = ((const uint8_t *) vy0) + 0; - const uint8_t * restrict y0_d = ((const uint8_t *) vy0) + y_qrow_size; - const uint8_t * restrict y1_q = ((const uint8_t *) vy1) + 0; - const uint8_t * restrict y1_d = ((const uint8_t *) vy1) + y_qrow_size; - - HVX_Vector r0_c0_sum = Q6_V_vzero(); - HVX_Vector r0_c1_sum = Q6_V_vzero(); - HVX_Vector r1_c0_sum = Q6_V_vzero(); - HVX_Vector r1_c1_sum = Q6_V_vzero(); - - const uint32_t nb = n / qk; - const uint32_t nloe = n % qk; - - uint32_t i = 0; - for (; i < nb; i++) { - HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_full(y0_q + i * y_qblk_size); - HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_full(y1_q + i * y_qblk_size); - HVX_Vector_x8 r0_q = hvx_vec_load_iq4nlx4x8_full(r0_x_q + i * x_qblk_size); - HVX_Vector_x8 r1_q = hvx_vec_load_iq4nlx4x8_full(r1_x_q + i * x_qblk_size); - - HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy0_q)); - HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy1_q)); - HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy0_q)); - HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy1_q)); +static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) { + struct htp_mm_context * mmctx = (struct htp_mm_context *) data; + struct htp_ops_context * octx = mmctx->octx; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_weights = kparams->n_weights; + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict act = octx->src[n_weights]; + const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; - HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size)); - HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); - HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d))); - HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d))); - HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d))); - HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d))); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd); - HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd); - HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd); - HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd); - - r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum)); - r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum)); - r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum)); - r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum)); - } - - if (nloe) { - HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_partial(y0_q + i * y_qblk_size, nloe); - HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_partial(y1_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_iq4nlx4x8_partial(r0_x_q + i * x_qblk_size, nloe); - HVX_Vector_x8 r1_q = hvx_vec_load_iq4nlx4x8_partial(r1_x_q + i * x_qblk_size, nloe); - - HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy0_q, nloe)); - HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy1_q, nloe)); - HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy0_q, nloe)); - HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy1_q, nloe)); - - HVX_Vector vy0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y0_d + i * y_dblk_size)); - HVX_Vector vy1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (y1_d + i * y_dblk_size)); - HVX_Vector r0_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r0_x_d + i * x_dblk_size)); - HVX_Vector r1_d = Q6_Vh_vshuff_Vh(*(const HVX_UVector *) (r1_x_d + i * x_dblk_size)); - - HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy0_d))); - HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r0_d, vy1_d))); - HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy0_d))); - HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(r1_d, vy1_d))); + const uint32_t n_prefetch = kparams->n_prefetch; + assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_c0_dd = Q6_V_vand_QV(bmask, r0_c0_dd); - r0_c1_dd = Q6_V_vand_QV(bmask, r0_c1_dd); - r1_c0_dd = Q6_V_vand_QV(bmask, r1_c0_dd); - r1_c1_dd = Q6_V_vand_QV(bmask, r1_c1_dd); - r0_c0_ia = Q6_V_vand_QV(bmask, r0_c0_ia); - r0_c1_ia = Q6_V_vand_QV(bmask, r0_c1_ia); - r1_c0_ia = Q6_V_vand_QV(bmask, r1_c0_ia); - r1_c1_ia = Q6_V_vand_QV(bmask, r1_c1_ia); - - HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd); - HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd); - HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd); - HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd); - - r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum)); - r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum)); - r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum)); - r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum)); - } - - HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); - HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); - - hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); - hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); -} + const uint32_t n_as = src0->ne[2]; -static void vec_dot_mxfp4x4x2_q8x4x2_1x1(const int n, float * restrict s0, const void * restrict vx0, const void * restrict vy0) { - assert(n % 32 == 0); // min sub-block size - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); + const uint32_t * matrix_row_counts = mmctx->matrix_row_counts; + const struct mmid_row_mapping * matrix_rows = mmctx->matrix_rows; - const uint32_t qk = QK_MXFP4x4x2 * 4; + const size_t act_stride = mmctx->vtcm_act_stride; - const uint32_t x_dblk_size = 8 * 4 * 1; // 32x e8m0 - const uint32_t x_qblk_size = qk / 2; // fp4 - const uint32_t x_qrow_size = n / 2; // fp4 (not padded) + uint8_t * restrict vtcm_src0_ptr = mmctx->vtcm_src0 + mmctx->vtcm_src0_size_per_thread * ith; + uint8_t * restrict act_data = mmctx->vtcm_act; - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) - - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0 + 0); // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0 + x_qrow_size); // then scales + for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { + const int32_t cne1 = matrix_row_counts[cur_a]; + if (cne1 == 0) continue; + + for (uint32_t p = 0; p < n_weights; ++p) { + const struct htp_tensor * restrict src_w = octx->src[p]; + const struct htp_tensor * restrict dst = octx->dsts[p]; + if (!src_w || !dst) continue; + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t ne01 = src_w->ne[1]; + uint32_t start_row = 0; + uint32_t end_row = ne01; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + start_row = range.start; + end_row = range.start + range.count; + } - const uint8_t * restrict y_q = ((const uint8_t *) vy0 + 0); // quants first - const uint8_t * restrict y_d = ((const uint8_t *) vy0 + y_qrow_size); // then scales + const uint32_t nrows = end_row - start_row; + uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); + src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); - // Row sum (sf) - HVX_Vector r0_sum = Q6_V_vzero(); + const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row); + if (src0_start_row >= src0_end_row) continue; - // Multiply and accumulate into int32. - // Compute combined scale (fp32). - // Apply scale to acc and accumulate into the row sum (qf32). + const dma_addr_t src0_row = src_w->data + cur_a * src_w->nb[2]; - const uint32_t nb = n / qk; // num full blocks - int32_t nloe = n % qk; // num leftover elemements (must be signed) + const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); + const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type); + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; - uint32_t i = 0; - for (; i < nb; i++) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full( y_q + i * y_qblk_size); - HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_full(r0_x_q + i * x_qblk_size); + const uint32_t ct_start = src0_start_row / 32; + const uint32_t ct_end = (src0_end_row + 31) / 32; - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q)); + uint32_t push_ct = ct_start; + for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { + dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + } - HVX_Vector vy_d = *(const HVX_UVector *) (y_d + i * y_dblk_size); - HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size); + for (uint32_t ct = ct_start; ct < ct_end; ct++) { + const uint8_t * w_tile = (void *) dma_queue_pop(dma_q).dst; - // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving - HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16 - vy_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy_d), half)); - vy_d = Q6_Vsf_equals_Vqf32(vy_d); + int valid_rows = (int)dst->ne[0] - (int)(ct * 32); + valid_rows = MIN(32, MAX(0, valid_rows)); - // Convert rX_d scales from e8m0 to fp32 - // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ... - // Left shift with zero fill to create FP32 - // FIXME: might need to handle zero as a special case (see ggml-cpu code) - HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; - HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); - r0_d = Q6_V_vdelta_VV(r0_d, expand); - r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); - r0_d = Q6_Vw_vasl_VwR(r0_d, 23); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ct); + for (uint32_t cid = 0; cid < (uint32_t) cne1; ++cid) { + struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, cid); + const int rm1 = row_mapping.i1; + const int rm2 = row_mapping.i2; - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy_d)); + const uint32_t ir1 = fastmodulo(rm1, act->ne[1], &mmctx->mm_div_ne11); + const uint8_t * restrict act_col = (const uint8_t *) (act_data + (ir1 + rm2 * act->ne[1]) * act_stride); + float * restrict dst_row = (float *) (dst->data + (rm1 * dst->nb[1] + rm2 * dst->nb[2])); - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); + mmctx->vec_dot_32x1(act->ne[0], &dst_row[ct * 32], w_tile, act_col, valid_rows, NULL); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ct); - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); + if (push_ct < ct_end) { + dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); + push_ct++; + } + } + } } +} - // Process leftovers - if (nloe) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial( y_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_partial(r0_x_q + i * x_qblk_size, nloe); - - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy_q, nloe)); - - HVX_Vector vy_d = *(const HVX_UVector *) (y_d + i * y_dblk_size); - HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size); - - // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving - HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16 - vy_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy_d), half)); - vy_d = Q6_Vsf_equals_Vqf32(vy_d); - - // Convert rX_d scales from e8m0 to fp32 - // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ... - // Left shift with zero fill to create FP32 - // FIXME: might need to handle zero as a special case (see ggml-cpu code) - HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; - HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); - r0_d = Q6_V_vdelta_VV(r0_d, expand); - r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); - r0_d = Q6_Vw_vasl_VwR(r0_d, 23); - - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy_d)); - - // Zero-out unused scales - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_dd = Q6_V_vand_QV(bmask, r0_dd); - r0_ia = Q6_V_vand_QV(bmask, r0_ia); - - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); +static int hvx_mm_init_tiled_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type type) { + switch (type) { + case HTP_TYPE_Q4_0: + mmctx->type = "q4_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q4_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q4_0_32x2; + return 0; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: + mmctx->type = "q4_1_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q4_1_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q4_1_32x2; + return 0; + case HTP_TYPE_Q8_0: + mmctx->type = "q8_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q8_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q8_0_32x2; + return 0; + case HTP_TYPE_Q5_K: + mmctx->type = "q5_k_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q5_k_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q5_k_32x2; + return 0; + case HTP_TYPE_Q6_K: + mmctx->type = "q6_k_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q6_k_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q6_k_32x2; + return 0; + case HTP_TYPE_Q3_K: + mmctx->type = "q3_k_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q3_k_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q3_k_32x2; + return 0; + case HTP_TYPE_Q2_K: + mmctx->type = "q2_k_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q2_k_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q2_k_32x2; + return 0; + case HTP_TYPE_Q2_0: + mmctx->type = "q2_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q2_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q2_0_32x2; + return 0; + case HTP_TYPE_IQ4_NL: + mmctx->type = "iq4nl_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_iq4nl_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_iq4nl_32x2; + return 0; + case HTP_TYPE_MXFP4: + mmctx->type = "mxfp4_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_mxfp4_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_mxfp4_32x2; + return 0; + case HTP_TYPE_Q1_0: + mmctx->type = "q1_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q1_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q1_0_32x2; + return 0; + case HTP_TYPE_F16: + mmctx->type = "f16_tiled-f16"; + mmctx->vec_dot_32x1 = tiled_vec_dot_f16_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_f16_32x2; + return 0; + default: + return -1; } - - r0_sum = hvx_vec_reduce_sum_f32(r0_sum); - - hvx_vec_store_u(s0, 4, r0_sum); } -static void vec_dot_mxfp4x4x2_q8x4x2_2x1(const int n, float * restrict s0, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0) { - assert(n % 32 == 0); // min sub-block size - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vx1 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); - - const uint32_t qk = QK_MXFP4x4x2 * 4; - - const uint32_t x_dblk_size = 8 * 4 * 1; // 32x e8m0 - const uint32_t x_qblk_size = qk / 2; // fp4 - const uint32_t x_qrow_size = n / 2; // fp4 (not padded) - - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) - - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales - const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first - const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales - - const uint8_t * restrict y_q = ((const uint8_t *) vy0) + 0; // quants first - const uint8_t * restrict y_d = ((const uint8_t *) vy0) + y_qrow_size; // then scales - - // Row sum (sf) - HVX_Vector r0_sum = Q6_V_vzero(); - HVX_Vector r1_sum = Q6_V_vzero(); - - // Multiply and accumulate into int32. - // Compute combined scale (fp32). - // Apply scale to acc and accumulate into the row sum (f32). - - const uint32_t nb = n / qk; // num full blocks - int32_t nloe = n % qk; // num leftover elemements (must be signed) - - uint32_t i = 0; - for (; i < nb; i++) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_full( y_q + i * y_qblk_size); - HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_full(r0_x_q + i * x_qblk_size); - HVX_Vector_x8 r1_q = hvx_vec_load_mxfp4x4x8_full(r1_x_q + i * x_qblk_size); - - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q)); - HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy_q)); - - HVX_Vector vy_d = *(const HVX_UVector *) (y_d + i * y_dblk_size); - HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size); - HVX_Vector r1_d = *(const HVX_UVector *) (r1_x_d + i * x_dblk_size); - - // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving - HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16 - vy_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy_d), half)); - vy_d = Q6_Vsf_equals_Vqf32(vy_d); +static int hvx_mm_matmul(struct htp_ops_context * octx) { + htp_matmul_tensors_preamble; - // Convert rX_d scales from e8m0 to fp32 - // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ... - // Left shift with zero fill to create FP32 - // FIXME: might need to handle zero as a special case (see ggml-cpu code) - HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; - HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); - r0_d = Q6_V_vdelta_VV(r0_d, expand); - r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); - r0_d = Q6_Vw_vasl_VwR(r0_d, 23); - r1_d = Q6_V_vdelta_VV(r1_d, expand); - r1_d = Q6_V_vand_VV(r1_d, e8m0_mask); - r1_d = Q6_Vw_vasl_VwR(r1_d, 23); - - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy_d)); - HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy_d)); - - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd); - - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum)); - } - - // Process leftovers - if (nloe) { - HVX_Vector_x8 vy_q = hvx_vec_load_q8x4x8_partial( y_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_partial(r0_x_q + i * x_qblk_size, nloe); - HVX_Vector_x8 r1_q = hvx_vec_load_mxfp4x4x8_partial(r1_x_q + i * x_qblk_size, nloe); - - HVX_Vector r0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy_q)); - HVX_Vector r1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy_q)); - - HVX_Vector vy_d = *(const HVX_UVector *) (y_d + i * y_dblk_size); - HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size); - HVX_Vector r1_d = *(const HVX_UVector *) (r1_x_d + i * x_dblk_size); - - // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving - HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16 - vy_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy_d), half)); - vy_d = Q6_Vsf_equals_Vqf32(vy_d); - - // Convert rX_d scales from e8m0 to fp32 - // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ... - // Left shift with zero fill to create FP32 - // FIXME: might need to handle zero as a special case (see ggml-cpu code) - HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; - HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); - r0_d = Q6_V_vdelta_VV(r0_d, expand); - r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); - r0_d = Q6_Vw_vasl_VwR(r0_d, 23); - r1_d = Q6_V_vdelta_VV(r1_d, expand); - r1_d = Q6_V_vand_VV(r1_d, e8m0_mask); - r1_d = Q6_Vw_vasl_VwR(r1_d, 23); - - HVX_Vector r0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy_d)); - HVX_Vector r1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy_d)); - - // Zero-out unused values - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_dd = Q6_V_vand_QV(bmask, r0_dd); - r1_dd = Q6_V_vand_QV(bmask, r1_dd); - r0_ia = Q6_V_vand_QV(bmask, r0_ia); - r1_ia = Q6_V_vand_QV(bmask, r1_ia); - - HVX_Vector r0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_ia, r0_dd); - HVX_Vector r1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_ia, r1_dd); - - r0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_fa, r0_sum)); - r1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_fa, r1_sum)); - } - - HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(r0_sum, r1_sum); - hvx_vec_store_u(s0, 8, rsum); -} - -static void vec_dot_mxfp4x4x2_q8x4x2_2x2(const int n, float * restrict s0, float * restrict s1, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0, const void * restrict vy1) { - assert(n % 32 == 0); - assert((unsigned long) vx0 % 128 == 0); - assert((unsigned long) vx1 % 128 == 0); - assert((unsigned long) vy0 % 128 == 0); - assert((unsigned long) vy1 % 128 == 0); - - const uint32_t qk = QK_MXFP4x4x2 * 4; - - const uint32_t x_dblk_size = 8 * 4 * 1; // 32x e8m0 - const uint32_t x_qblk_size = qk / 2; // fp4 - const uint32_t x_qrow_size = n / 2; // fp4 (not padded) - - const uint32_t y_dblk_size = 8 * 4 * 2; // 32x __fp16 - const uint32_t y_qblk_size = qk; // int8 - const uint32_t y_qrow_size = n; // int8 (not padded) - - const uint8_t * restrict r0_x_q = ((const uint8_t *) vx0) + 0; // quants first - const uint8_t * restrict r0_x_d = ((const uint8_t *) vx0) + x_qrow_size; // then scales - const uint8_t * restrict r1_x_q = ((const uint8_t *) vx1) + 0; // quants first - const uint8_t * restrict r1_x_d = ((const uint8_t *) vx1) + x_qrow_size; // then scales - - const uint8_t * restrict y0_q = ((const uint8_t *) vy0) + 0; // quants first - const uint8_t * restrict y0_d = ((const uint8_t *) vy0) + y_qrow_size; // then scales - const uint8_t * restrict y1_q = ((const uint8_t *) vy1) + 0; // quants first - const uint8_t * restrict y1_d = ((const uint8_t *) vy1) + y_qrow_size; // then scales - - // Row sums (sf) - 4 accumulators for 2×2 tile - HVX_Vector r0_c0_sum = Q6_V_vzero(); - HVX_Vector r0_c1_sum = Q6_V_vzero(); - HVX_Vector r1_c0_sum = Q6_V_vzero(); - HVX_Vector r1_c1_sum = Q6_V_vzero(); - - const uint32_t nb = n / qk; // num full blocks - const uint32_t nloe = n % qk; // num leftover elements - - uint32_t i = 0; - for (; i < nb; i++) { - // Load src1 columns (reused across both src0 rows) - HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_full(y0_q + i * y_qblk_size); - HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_full(y1_q + i * y_qblk_size); - - // Load src0 rows (reused across both src1 columns) - HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_full(r0_x_q + i * x_qblk_size); - HVX_Vector_x8 r1_q = hvx_vec_load_mxfp4x4x8_full(r1_x_q + i * x_qblk_size); - - // Compute 4 dot products: r0×c0, r0×c1, r1×c0, r1×c1 - HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy0_q)); - HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r0_q, vy1_q)); - HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy0_q)); - HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_full(r1_q, vy1_q)); - - // Load scales - HVX_Vector vy0_d = *(const HVX_UVector *) (y0_d + i * y_dblk_size); - HVX_Vector vy1_d = *(const HVX_UVector *) (y1_d + i * y_dblk_size); - HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size); - HVX_Vector r1_d = *(const HVX_UVector *) (r1_x_d + i * x_dblk_size); - - // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving - HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16 - vy0_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy0_d), half)); - vy0_d = Q6_Vsf_equals_Vqf32(vy0_d); - vy1_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy1_d), half)); - vy1_d = Q6_Vsf_equals_Vqf32(vy1_d); - - // Convert rX_d scales from e8m0 to fp32 - // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ... - // Left shift with zero fill to create FP32 - // FIXME: might need to handle zero as a special case (see ggml-cpu code) - HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; - HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); - r0_d = Q6_V_vdelta_VV(r0_d, expand); - r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); - r0_d = Q6_Vw_vasl_VwR(r0_d, 23); - r1_d = Q6_V_vdelta_VV(r1_d, expand); - r1_d = Q6_V_vand_VV(r1_d, e8m0_mask); - r1_d = Q6_Vw_vasl_VwR(r1_d, 23); - - // Compute combined scales - HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy0_d)); - HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy1_d)); - HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy0_d)); - HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy1_d)); - - // Apply scales and accumulate - HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd); - HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd); - HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd); - HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd); - - r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum)); - r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum)); - r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum)); - r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum)); - } - - // Process leftovers - if (nloe) { - HVX_Vector_x8 vy0_q = hvx_vec_load_q8x4x8_partial( y0_q + i * y_qblk_size, nloe); - HVX_Vector_x8 vy1_q = hvx_vec_load_q8x4x8_partial( y1_q + i * y_qblk_size, nloe); - HVX_Vector_x8 r0_q = hvx_vec_load_mxfp4x4x8_partial(r0_x_q + i * x_qblk_size, nloe); - HVX_Vector_x8 r1_q = hvx_vec_load_mxfp4x4x8_partial(r1_x_q + i * x_qblk_size, nloe); - - HVX_Vector r0_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy0_q, nloe)); - HVX_Vector r0_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r0_q, vy1_q, nloe)); - HVX_Vector r1_c0_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy0_q, nloe)); - HVX_Vector r1_c1_ia = Q6_Vsf_equals_Vw(hvx_vec_rmpy_x8_partial(r1_q, vy1_q, nloe)); - - HVX_Vector vy0_d = *(const HVX_UVector *) (y0_d + i * y_dblk_size); - HVX_Vector vy1_d = *(const HVX_UVector *) (y1_d + i * y_dblk_size); - HVX_Vector r0_d = *(const HVX_UVector *) (r0_x_d + i * x_dblk_size); - HVX_Vector r1_d = *(const HVX_UVector *) (r1_x_d + i * x_dblk_size); - - // Convert vy_d from fp16 to fp32 while applying 0.5 scaling which is used for e8m0 halving - HVX_Vector half = Q6_Vh_vsplat_R(0x3800); // 0.5 in fp16 - vy0_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy0_d), half)); - vy0_d = Q6_Vsf_equals_Vqf32(vy0_d); - vy1_d = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vy1_d), half)); - vy1_d = Q6_Vsf_equals_Vqf32(vy1_d); - - // Convert rX_d scales from e8m0 to fp32 - // Expand and zero-pad 32x uint8 e8m0 values to uint32s : 0 0 0 0, 0 0 0 1, 0 0 0 2, ... - // Left shift with zero fill to create FP32 - // FIXME: might need to handle zero as a special case (see ggml-cpu code) - HVX_Vector expand = *(const HVX_Vector *) expand_x32_e8m0; - HVX_Vector e8m0_mask = Q6_V_vsplat_R(0x000000ff); - r0_d = Q6_V_vdelta_VV(r0_d, expand); - r0_d = Q6_V_vand_VV(r0_d, e8m0_mask); - r0_d = Q6_Vw_vasl_VwR(r0_d, 23); - r1_d = Q6_V_vdelta_VV(r1_d, expand); - r1_d = Q6_V_vand_VV(r1_d, e8m0_mask); - r1_d = Q6_Vw_vasl_VwR(r1_d, 23); - - HVX_Vector r0_c0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy0_d)); - HVX_Vector r0_c1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r0_d, vy1_d)); - HVX_Vector r1_c0_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy0_d)); - HVX_Vector r1_c1_dd = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r1_d, vy1_d)); - - // Zero out unused scales - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe / 8); - r0_c0_dd = Q6_V_vand_QV(bmask, r0_c0_dd); - r0_c1_dd = Q6_V_vand_QV(bmask, r0_c1_dd); - r1_c0_dd = Q6_V_vand_QV(bmask, r1_c0_dd); - r1_c1_dd = Q6_V_vand_QV(bmask, r1_c1_dd); - r0_c0_ia = Q6_V_vand_QV(bmask, r0_c0_ia); - r0_c1_ia = Q6_V_vand_QV(bmask, r0_c1_ia); - r1_c0_ia = Q6_V_vand_QV(bmask, r1_c0_ia); - r1_c1_ia = Q6_V_vand_QV(bmask, r1_c1_ia); - - HVX_Vector r0_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c0_ia, r0_c0_dd); - HVX_Vector r0_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r0_c1_ia, r0_c1_dd); - HVX_Vector r1_c0_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c0_ia, r1_c0_dd); - HVX_Vector r1_c1_fa = Q6_Vqf32_vmpy_VsfVsf(r1_c1_ia, r1_c1_dd); - - r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c0_fa, r0_c0_sum)); - r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r0_c1_fa, r0_c1_sum)); - r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c0_fa, r1_c0_sum)); - r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(r1_c1_fa, r1_c1_sum)); - } - - // Reduce and store results - HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); - HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); - - hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0 - hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1 -} - -static void vec_dot_f16_f16_aa_1x1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { - const HVX_Vector * restrict x = (const HVX_Vector *) vx; - const HVX_Vector * restrict y = (const HVX_Vector *) vy; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - HVX_VectorPair rsum_p = Q6_W_vzero(); - - uint32_t i = 0; - - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x[i], y[i]); - } - - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); - rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf); - } - - HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p))); - hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum)); -} + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); -static void vec_dot_f16_f16_aa_2x1(const int n, float * restrict s0, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y = (const HVX_Vector *) vy0; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; + mmctx->octx = octx; + mmctx->act = act; - uint32_t nvec = n / VLEN_FP16; - uint32_t nloe = n % VLEN_FP16; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - HVX_VectorPair rsum0_p = Q6_W_vzero(); - HVX_VectorPair rsum1_p = Q6_W_vzero(); + const uint32_t src0_nrows = ne01; + const uint32_t act_nrows = ne11 * ne12 * ne13; + mmctx->act_nrows = act_nrows; - uint32_t i = 0; + uint32_t src0_row_start = 0; + uint32_t src0_row_end = src0_nrows; - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector y_hf = y[i]; - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0[i], y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1[i], y_hf); + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_row_start = range.start; + src0_row_end = range.start + range.count; } - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); - HVX_Vector x0_hf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector x1_hf = Q6_V_vand_QV(bmask, x1[i]); - rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf); - rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf); + if (src0_row_start >= src0_row_end) { + return HTP_STATUS_OK; } - HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p))); - HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p))); - HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1); - hvx_vec_store_u(s0, 8, rsum); -} - -static void vec_dot_f16_f16_aa_2x2(const int n, float * restrict s0, float * restrict s1, - const void * restrict vx0, const void * restrict vx1, - const void * restrict vy0, const void * restrict vy1) { - const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0; - const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1; - const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0; - const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1; + const uint32_t nrows = src0_row_end - src0_row_start; + mmctx->src0_row_start = src0_row_start; + mmctx->src0_row_end = src0_row_end; - uint32_t nvec = n / VLEN_FP16; - uint32_t nloe = n % VLEN_FP16; + const bool is_tiled = htp_tensor_is_tiled(src0); - // Row sums (sf) - 4 accumulators for 2×2 tile - HVX_VectorPair r0_c0_sum_p = Q6_W_vzero(); - HVX_VectorPair r0_c1_sum_p = Q6_W_vzero(); - HVX_VectorPair r1_c0_sum_p = Q6_W_vzero(); - HVX_VectorPair r1_c1_sum_p = Q6_W_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - HVX_Vector r0_hf = x0[i]; - HVX_Vector r1_hf = x1[i]; - HVX_Vector c0_hf = y0[i]; - HVX_Vector c1_hf = y1[i]; - - // Compute 4 dot products: r0×c0, r0×c1, r1×c0, r1×c1 - r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); - r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); - r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); - r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + // Compute src0_nrows_per_thread + mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + if (is_tiled) { + mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); + } else { + mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even } - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); + const size_t src0_row_size = nb01; + const size_t dst_row_size = nb1; + size_t act_row_size = nb11; - HVX_Vector r0_hf = Q6_V_vand_QV(bmask, x0[i]); - HVX_Vector r1_hf = Q6_V_vand_QV(bmask, x1[i]); - HVX_Vector c0_hf = Q6_V_vand_QV(bmask, y0[i]); - HVX_Vector c1_hf = Q6_V_vand_QV(bmask, y1[i]); + const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf); - r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf); - r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf); - r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf); + worker_callback_t quant_task_func; + worker_callback_t matmul_job_func; + uint32_t n_quant_tasks = 1; + const bool is_batched = (ne12 > 1 || ne13 > 1 || ne02 > 1 || ne03 > 1); + if (is_batched) { + matmul_job_func = is_tiled ? hvx_mm_4d_tiled : hvx_mm_4d_flat; + } else if (act_nrows > 1) { + matmul_job_func = is_tiled ? hvx_mm_2d_tiled : hvx_mm_2d_flat; + } else { + matmul_job_func = is_tiled ? hvx_mv_2d_tiled : hvx_mv_2d_flat; } - HVX_Vector r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c0_sum_p), Q6_V_hi_W(r0_c0_sum_p))); - HVX_Vector r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c1_sum_p), Q6_V_hi_W(r0_c1_sum_p))); - HVX_Vector r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c0_sum_p), Q6_V_hi_W(r1_c0_sum_p))); - HVX_Vector r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c1_sum_p), Q6_V_hi_W(r1_c1_sum_p))); - - // Reduce and store results - HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum); - HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum); - - hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0 - hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1 -} + bool need_quant = true; -static void vec_dot_f16_f16_uu_1x1(const int n, float * restrict s, const void * restrict vx, const void * restrict vy) { - const HVX_UVector * restrict x = (const HVX_UVector *) vx; - const HVX_UVector * restrict y = (const HVX_UVector *) vy; + switch (kparams->kernel_type) { + case HTP_MM_KERNEL_HVX_F16_F16_VTCM: + quant_task_func = (act->type == HTP_TYPE_F32) ? quantize_f32_f16 : quantize_f16_f16; + need_quant = (act->type == HTP_TYPE_F32); + mmctx->type = (act->type == HTP_TYPE_F32) ? "f32-f16" : "f16-f16"; + mmctx->vec_dot_1x1 = flat_vec_dot_f16_f16_aa_1x1; + mmctx->vec_dot_2x1 = flat_vec_dot_f16_f16_aa_2x1; + mmctx->vec_dot_2x2 = flat_vec_dot_f16_f16_aa_2x2; + act_row_size = kparams->act_row_size; + break; - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements + case HTP_MM_KERNEL_HVX_F32_F32_VTCM: + quant_task_func = NULL; + need_quant = false; + mmctx->type = "f32-f32"; + mmctx->vec_dot_1x1 = flat_vec_dot_f32_f32_aa_1x1; + mmctx->vec_dot_2x1 = flat_vec_dot_f32_f32_aa_2x1; + mmctx->vec_dot_2x2 = flat_vec_dot_f32_f32_aa_2x2; + act_row_size = kparams->act_row_size; + break; - HVX_Vector rsum = Q6_V_vzero(); + case HTP_MM_KERNEL_HVX_BF16_F32_VTCM: + quant_task_func = NULL; + need_quant = false; + mmctx->type = "bf16-f32"; + mmctx->vec_dot_1x1 = flat_vec_dot_bf16_f32_aa_1x1; + mmctx->vec_dot_2x1 = flat_vec_dot_bf16_f32_aa_2x1; + mmctx->vec_dot_2x2 = flat_vec_dot_bf16_f32_aa_2x2; + act_row_size = kparams->act_row_size; + break; - uint32_t i = 0; + case HTP_MM_KERNEL_HVX_QUANT_BLOCK: + case HTP_MM_KERNEL_HVX_QUANT_ROW: + default: { + if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) != 0) { + return HTP_STATUS_NO_SUPPORT; + } - #pragma unroll(4) - for (i = 0; i < nvec; i++) { - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x[i], y[i]); - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne10 + qk - 1) / qk; + const uint32_t total_nb = act_nrows * nb; + + if (act_nrows < octx->n_threads && !is_batched) { + n_quant_tasks = MIN(total_nb, octx->n_threads); + quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); + for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { + uint32_t ib_first = (total_nb * ith) / n_quant_tasks; + uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; + mmctx->quant_ib_first[ith] = ib_first; + mmctx->quant_ib_last[ith] = ib_last; + mmctx->quant_r[ith] = ib_first / nb; + mmctx->quant_c[ith] = ib_first % nb; + } + } else { + n_quant_tasks = MIN(act_nrows, octx->n_threads); + quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); + } + act_row_size = kparams->act_row_size; + break; + } } - if (nloe) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]); - HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]); - - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); - hvx_vec_store_u(&s[0], 4, rsum); -} - -static void vec_dot_f16_f32_uu_1x1(const int n, float * restrict s, const void * restrict x, const void * restrict y) { - const HVX_UVector * restrict vx = (const HVX_UVector * restrict) x; - const HVX_UVector * restrict vy = (const HVX_UVector * restrict) y; - - uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors - uint32_t nloe = n % VLEN_FP16; // leftover elements - - const HVX_Vector zero = Q6_V_vzero(); + const uint32_t m_chunk = (kparams->m_chunk > 0 && (uint32_t) kparams->m_chunk < act_nrows) + ? (uint32_t) kparams->m_chunk : act_nrows; + const uint32_t m_layout_rows = m_chunk; - HVX_Vector rsum = Q6_V_vzero(); - - uint32_t i = 0; - - #pragma unroll(2) - for (i = 0; i < nvec; i++) { - // Load y (fp32) and convert into fp16 - HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements - HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements - HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); - - // Load x (fp16) - HVX_Vector x_hf = vx[i]; - - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, m_layout_rows, octx->n_threads, + dst_row_size, src0_row_size, act_row_size, src2 ? src2->nb[1] : 0, kparams->n_prefetch, false, false); - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); + if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || + kparams->kernel_type == HTP_MM_KERNEL_HVX_F32_F32_VTCM || + kparams->kernel_type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM || + kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || + kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { + mmctx->vtcm_act_size_per_thread = L.act_bytes; + } else { + mmctx->vtcm_act_size_per_thread = fastdiv(L.act_bytes, &octx->n_threads_div); } - if (nloe) { - // Load y (fp32) and convert into fp16 - HVX_Vector y0_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+0], zero); // 32 elements - HVX_Vector y1_qf = Q6_Vqf32_vsub_VsfVsf(vy[i*2+1], zero); // 32 elements - HVX_Vector y_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(y1_qf, y0_qf))); - - // Load x (fp16) - HVX_Vector x_hf = vx[i]; - - // Zero-out unused elements - // Note that we need to clear both x and y because they may contain NANs - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2); - x_hf = Q6_V_vand_QV(bmask, x_hf); - y_hf = Q6_V_vand_QV(bmask, y_hf); - - HVX_VectorPair xy_qf = Q6_Wqf32_vmpy_VhfVhf(x_hf, y_hf); - - rsum = Q6_Vqf32_vadd_Vqf32Vqf32(rsum, Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(xy_qf), Q6_V_hi_W(xy_qf))); - } - - // Convert into fp32 and reduce - rsum = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(rsum)); - hvx_vec_store_u(&s[0], 4, rsum); -} + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); -#define htp_matmul_tensors_preamble \ - const struct htp_tensor * restrict src0 = octx->src[0]; \ - const struct htp_tensor * restrict src1 = octx->src[1]; \ - const struct htp_tensor * restrict src2 = octx->src[2]; \ - const struct htp_tensor * restrict dst = octx->dst; \ - struct htp_spad * restrict src0_spad = &octx->src0_spad; \ - struct htp_spad * restrict src1_spad = &octx->src1_spad; \ - struct htp_spad * restrict dst_spad = &octx->dst_spad; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne10 = src1->ne[0]; \ - const uint32_t ne11 = src1->ne[1]; \ - const uint32_t ne12 = src1->ne[2]; \ - const uint32_t ne13 = src1->ne[3]; \ - \ - const uint32_t ne20 = src2->ne[0]; \ - const uint32_t ne21 = src2->ne[1]; \ - const uint32_t ne22 = src2->ne[2]; \ - const uint32_t ne23 = src2->ne[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb10 = src1->nb[0]; \ - const uint32_t nb11 = src1->nb[1]; \ - const uint32_t nb12 = src1->nb[2]; \ - const uint32_t nb13 = src1->nb[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; + const size_t vtcm_size = L.total_bytes; -#define htp_matmul_preamble \ - struct htp_matmul_context * mmctx = data; \ - struct htp_ops_context * octx = mmctx->octx; \ - htp_matmul_tensors_preamble; \ - dma_queue *dma_queue = octx->ctx->dma[ith]; \ - uint32_t src0_nrows_per_thread = mmctx->src0_nrows_per_thread; + FARF(HIGH, "matmul-%s : src0-vtcm-size %zu act-vtcm-size %zu dst-vtcm-size %zu (%zu)\n", mmctx->type, + L.src0_bytes, L.act_bytes, L.dst_bytes, vtcm_size); -// *** matmul with support for 4d tensors and full broadcasting + FARF(HIGH, "matmul-%s : %ux%ux%ux%u * %ux%ux%ux%u-> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\n", mmctx->type, src0->ne[0], + src0->ne[1], src0->ne[2], src0->ne[3], act->ne[0], act->ne[1], act->ne[2], act->ne[3], dst->ne[0], + dst->ne[1], dst->ne[2], dst->ne[3], src0->data, act->data, dst->data); -static void matmul_4d(unsigned int nth, unsigned int ith, void * data) { - htp_matmul_preamble; + if (octx->ctx->vtcm_size < vtcm_size) { + FARF(ERROR, "matmul-%s : current VTCM reservation %zu is too small, needed %zu\n", mmctx->type, + octx->ctx->vtcm_size, vtcm_size); + return HTP_STATUS_VTCM_TOO_SMALL; + } - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + mmctx->vtcm_act = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act); + mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + mmctx->vtcm_bias = VTCM_LAYOUT_PTR(uint8_t, base, L.off_bias); + mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + mmctx->vtcm_act_raw = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act_raw); + + mmctx->vtcm_src0_stride = src0_row_size_padded; + mmctx->vtcm_act_stride = act_row_size; + if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW) { + mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10); + } else if (kparams->kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM) { + mmctx->vtcm_act_raw_stride = hex_round_up(ne10 * sizeof(float), 128); + } else { + mmctx->vtcm_act_raw_stride = 0; + } - assert(ne12 % ne02 == 0); - assert(ne13 % ne03 == 0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + + if (kparams->m_chunk > 0 && (uint32_t) kparams->m_chunk < act_nrows) { + for (uint32_t m_start = 0; m_start < act_nrows; m_start += m_chunk) { + const uint32_t cur_m_rows = MIN(act_nrows - m_start, m_chunk); + mmctx->cur_m_start = m_start; + mmctx->cur_m_rows = cur_m_rows; + + if (need_quant) { + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, m_start, cur_m_rows); + + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne10 + qk - 1) / qk; + const uint32_t total_nb = cur_m_rows * nb; + uint32_t quant_tasks; + work_queue_func_t q_func; + if (cur_m_rows < octx->n_threads && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW)) { + quant_tasks = MIN(total_nb, octx->n_threads); + q_func = htp_mm_act_quant_block_func(src0->type, act->type); + for (uint32_t ith = 0; ith < quant_tasks; ++ith) { + uint32_t ib_first = (total_nb * ith) / quant_tasks; + uint32_t ib_last = (total_nb * (ith + 1)) / quant_tasks; + mmctx->quant_ib_first[ith] = ib_first; + mmctx->quant_ib_last[ith] = ib_last; + mmctx->quant_r[ith] = ib_first / nb; + mmctx->quant_c[ith] = ib_first % nb; + } + } else { + quant_tasks = MIN(cur_m_rows, octx->n_threads); + q_func = quant_task_func; + mmctx->n_quant_rows_per_thread = (cur_m_rows + quant_tasks - 1) / quant_tasks; + } + mmctx->n_quant_tasks = quant_tasks; + work_queue_run(octx->ctx->work_queue, q_func, mmctx, quant_tasks); + } else { + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, m_start, cur_m_rows); + } - // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) - const uint32_t nr0 = ne0; + work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads); + } + } else { + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = act_nrows; + + if (need_quant) { + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); + mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_tasks = n_quant_tasks; + work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); + } else { + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act, mmctx->vtcm_act_stride, 0, act_nrows); + } - // This is the size of the rest of the dimensions of the result - const uint32_t nr1 = ne1 * ne2 * ne3; + work_queue_run(octx->ctx->work_queue, matmul_job_func, mmctx, octx->n_threads); + } - // distribute the thread work across the inner or outer loop based on which one is larger - uint32_t nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows - uint32_t nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows + return HTP_STATUS_OK; +} - // The number of elements in each chunk - const uint32_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; - const uint32_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; +static void dequantize_tiled_worker_loop(unsigned int n, unsigned int i, void * data) { + tiled_dequantize_state_t * state = (tiled_dequantize_state_t *) data; + struct htp_thread_trace * tr = &state->traces[i]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); + for (unsigned int task_id = i; task_id < (unsigned int) state->n_tasks; task_id += n) { + int start = task_id * state->n_tiles_per_task; + int end = hex_smin(start + state->n_tiles_per_task, state->n_tot_tiles); + state->task_fn(state, start, end); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); +} - uint32_t current_chunk = ith; +static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { + output_transfer_task_state_t *st = (output_transfer_task_state_t *) data; - const uint32_t ith0 = current_chunk % nchunk0; - const uint32_t ith1 = current_chunk / nchunk0; + struct htp_thread_trace * tr = &st->traces[i]; - const uint32_t ir0_start = dr0 * ith0; - const uint32_t ir0_end = MIN(ir0_start + dr0, nr0); + int start_chunk_idx = i * st->n_chunks_per_task; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx); - const uint32_t ir1_start = dr1 * ith1; - const uint32_t ir1_end = MIN(ir1_start + dr1, nr1); + for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { + int chunk_idx = task_id * st->n_chunks_per_task; + size_t chunk_size = hex_smin(st->n_tot_chunks - chunk_idx, st->n_chunks_per_task); - // no work for this thread - if (ir0_start >= ir0_end || ir1_start >= ir1_end) { - return; + float *dst = st->dst + chunk_idx * st->dst_stride; + const float *src2 = st->src2 ? (st->src2 + chunk_idx * st->src2_stride) : NULL; + transfer_output_chunk_fp16_to_fp32(dst, src2, st->vtcm_src, chunk_idx, chunk_size, st->n_cols, st->dst_stride, st->src2_stride, st->dst_cols); } - // block-tiling attempt - const uint32_t blck_0 = 64; - const uint32_t blck_1 = 64; - - for (uint32_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { - for (uint32_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { - for (uint32_t ir1 = iir1; ir1 < MIN(iir1 + blck_1, ir1_end); ir1++) { - const uint32_t i13 = fastdiv(ir1, &mmctx->mm_div_ne12_ne1); - const uint32_t i12 = fastdiv(ir1 - i13 * ne12 * ne1, &mmctx->mm_div_ne1); - const uint32_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx); +} - // broadcast src0 into src1 - const uint32_t i03 = fastdiv(i13, &mmctx->mm_div_r3); - const uint32_t i02 = fastdiv(i12, &mmctx->mm_div_r2); +typedef struct { + struct htp_context * ctx; + struct htp_thread_trace * traces; + __fp16 * dst; + dma_addr_t act_dma_addr; + float * vtcm_f32_act; + uint32_t n_tasks; + uint32_t n_tot_chunks; + uint32_t n_chunks_per_task; + uint32_t k_block; + uint32_t k_stride; + uint32_t k_valid; + size_t vtcm_f32_act_bytes_per_thread; + uint32_t dma_step_rows; + uint32_t dma_step_rows_shift; + uint32_t act_elem_size; // 4=F32 src1, 2=F16 src1 +} activation_transfer_task_state_t; + +typedef struct { + struct htp_context * ctx; + struct htp_thread_trace * traces; + __fp16 * dst; + dma_addr_t act_dma_addr; + float * vtcm_f32_act; + uint32_t n_rows; + uint32_t k_block; + uint32_t k_stride; + uint32_t k_valid; + uint32_t n_col_chunks; + struct fastdiv_values n_threads_div; + size_t vtcm_f32_act_bytes; + uint32_t dma_step_rows; + uint32_t dma_step_rows_shift; +} activation_transfer_col_chunk_state_t; + +static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( + dma_queue *dma_q, + __fp16 *restrict vtcm_dst, + dma_addr_t act_dma_addr, + uint32_t n_rows, + uint32_t k_block, + uint32_t k_stride, + uint32_t k_chunk_valid, + uint32_t c_first, + uint32_t c_len, + float *thread_f32_act, + struct htp_thread_trace *tr, + uint32_t dma_step_rows, + uint32_t dma_step_rows_shift) { + + const uint32_t R = dma_step_rows; + const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + + const uint32_t n_steps = n_rows_padded >> dma_step_rows_shift; + + // Push step 0 + if (n_steps > 0 && n_rows > 0) { + uint32_t nrows_to_fetch = hex_smin(n_rows, R); + dma_queue_push(dma_q, dma_make_data(thread_f32_act, act_dma_addr + (size_t) c_first * sizeof(float)), + c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); + } + // Push step 1 + if (n_steps > 1) { + uint32_t next_r = R * 1; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + float *next_buf = thread_f32_act + 1 * R * c_len; + dma_queue_push(dma_q, dma_make_data(next_buf, act_dma_addr + ((size_t) next_r * k_stride + c_first) * sizeof(float)), + c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); + } + } + for (uint32_t s = 0; s < n_steps; ++s) { + uint32_t r = s << dma_step_rows_shift; + float *curr_buf = thread_f32_act; - const uint32_t i1 = i11; - const uint32_t i2 = i12; - const uint32_t i3 = i13; + if (r < n_rows) { + curr_buf = (float *) dma_queue_pop(dma_q).dst; + } - const uint8_t * restrict src0_base = (const uint8_t *) src0->data + (0 + i02 * nb02 + i03 * nb03); - const uint8_t * restrict src1_col = (const uint8_t *) src1->data + (i11 * nb11 + i12 * nb12 + i13 * nb13); - float * dst_col = (float *) ((uint8_t * restrict) dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + for (uint32_t p = 0; p < (R >> 1); ++p) { + uint32_t row_idx = r + (p << 1); + float *pair_buf = curr_buf + (p << 1) * c_len; + bool r0_valid = ((row_idx + 0) < n_rows); + bool r1_valid = ((row_idx + 1) < n_rows); - const uint32_t ir0_block_end = MIN(iir0 + blck_0, ir0_end); - for (uint32_t ir0 = iir0; ir0 < ir0_block_end; ir0++) { - const uint8_t * restrict src0_row = src0_base + ir0 * nb01; - mmctx->vec_dot_1x1(ne00, &dst_col[ir0], src0_row, src1_col); - } - } + transfer_activation_row_pair_fp32_to_fp16_col_chunk( + vtcm_dst, pair_buf, pair_buf + c_len, row_idx, k_block, c_first, c_len, k_chunk_valid, r0_valid, r1_valid + ); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + + // Push step s + 2 + uint32_t next_s = s + 2; + uint32_t next_r = next_s << dma_step_rows_shift; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + dma_queue_push(dma_q, dma_make_data(curr_buf, act_dma_addr + ((size_t) next_r * k_stride + c_first) * sizeof(float)), + c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch); } } - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "matmul-4d %d/%d: %ux%ux%ux%u (%u:%u %u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0_start, ir0_end, ir1_start, ir1_end, src1->ne[0], - src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); } -// src1 tensor is already in VTCM spad -static void matmul_2d(unsigned int nth, unsigned int ith, void * data) { - htp_matmul_preamble; - - const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows - const uint32_t src1_nrows = ne11 * ne12 * ne13; // src1 rows +static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { + activation_transfer_col_chunk_state_t *st = (activation_transfer_col_chunk_state_t *) data; + struct htp_thread_trace * tr = &st->traces[i]; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); + uint32_t n_blocks = st->k_block / 32; + uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + uint32_t c_first = b_first * 32; + uint32_t c_last = b_last * 32; + uint32_t c_len = c_last - c_first; - // no work for this thread - if (src0_start_row >= src0_end_row) { + if (c_len == 0) { return; } - const size_t dst_row_size = nb1; - const size_t src0_row_size = nb01; - const size_t src1_row_size = nb11; - - const size_t src0_stride = src0_spad->stride; - const size_t src1_stride = src1_spad->stride; + uint32_t k_chunk_valid = 0; + if (st->k_valid > c_first) { + k_chunk_valid = hex_smin(st->k_valid, c_last) - c_first; + } - // Per-thread VTCM scratchpads for all tensors - // Note that the entire src1 tensor is already in VTCM - // For other tensors we allocate N rows per thread, padded to HVX vector size - uint8_t * restrict spad_dst = dst_spad->data + dst_spad->size_per_thread * ith; - uint8_t * restrict spad_src0 = src0_spad->data + src0_spad->size_per_thread * ith; - uint8_t * restrict src1_data = src1_spad->data; + __fp16 *dst = st->dst; - volatile uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_threads_div), 128); + float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * thread_scratch_bytes); - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; + transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk( + st->ctx->dma[i], dst, st->act_dma_addr, st->n_rows, st->k_block, st->k_stride, k_chunk_valid, + c_first, c_len, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift + ); +} - // Prefill spad with src0 rows - #pragma unroll(4) - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const int is0 = (ir0 - src0_start_row); - if (is0 >= MM_SPAD_SRC0_NROWS) { - break; +static void transfer_activation_chunk_to_fp16_dma_pipelined( + dma_queue *dma_q, + __fp16 *restrict vtcm_dst, + dma_addr_t act_dma_addr, + uint32_t n_rows, + uint32_t k_block, + uint32_t k_stride, + uint32_t k_valid, + uint8_t *thread_act, + uint32_t act_elem_size, + struct htp_thread_trace *tr, + uint32_t dma_step_rows, + uint32_t dma_step_rows_shift) { + + const uint32_t R = dma_step_rows; + const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS); + + const uint32_t n_steps = n_rows_padded >> dma_step_rows_shift; + + const bool act_is_f16 = (act_elem_size == sizeof(__fp16)); + const size_t row_bytes = (size_t) k_block * act_elem_size; // staging row (DMA dst stride) + const size_t src_row_bytes = (size_t) k_stride * act_elem_size; // DDR row (DMA src stride) + const size_t width_bytes = (size_t) k_valid * act_elem_size; + + // Push step 0 + if (n_steps > 0 && n_rows > 0) { + uint32_t nrows_to_fetch = hex_smin(n_rows, R); + dma_queue_push(dma_q, dma_make_data(thread_act, act_dma_addr), + row_bytes, src_row_bytes, width_bytes, nrows_to_fetch); + } + // Push step 1 (if valid) + if (n_steps > 1) { + uint32_t next_r = R * 1; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + uint8_t *next_buf = thread_act + 1 * R * row_bytes; + dma_queue_push(dma_q, dma_make_data(next_buf, act_dma_addr + (size_t) next_r * src_row_bytes), + row_bytes, src_row_bytes, width_bytes, nrows_to_fetch); } - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, 2); } + for (uint32_t s = 0; s < n_steps; ++s) { + uint32_t r = s << dma_step_rows_shift; + uint8_t *curr_buf = thread_act; - // Process src0 rows - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - - // Process src1 columns in pairs (2×2 tiling) - uint32_t ir1 = 0; - for (; ir1 + 1 < src1_nrows; ir1 += 2) { - const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride); - const uint8_t * restrict src1_col1 = (const uint8_t *) (src1_data + (ir1+1) * src1_stride); - float * restrict dst_row0 = (float *) (dst->data + ((ir1+0) * dst_row_size)); - float * restrict dst_row1 = (float *) (dst->data + ((ir1+1) * dst_row_size)); - mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1); + if (r < n_rows) { + curr_buf = (uint8_t *) dma_queue_pop(dma_q).dst; } - // Handle remaining src1 rows (fallback to 2×1) - for (; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); - float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); - mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_stride, src1_col); - } + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + // Two copies of the pair loop so the type is resolved once per step and the + // row-pair kernels stay direct (inlinable) calls. + if (act_is_f16) { + for (uint32_t p = 0; p < (R >> 1); ++p) { + uint32_t row_idx = r + (p << 1); + const __fp16 *pair_buf = (const __fp16 *) (curr_buf + (p << 1) * row_bytes); + bool r0_valid = ((row_idx + 0) < n_rows); + bool r1_valid = ((row_idx + 1) < n_rows); + + transfer_activation_row_pair_f16_to_f16(vtcm_dst, pair_buf, pair_buf + k_block, row_idx, k_block, k_valid, r0_valid, r1_valid); + } + } else { + for (uint32_t p = 0; p < (R >> 1); ++p) { + uint32_t row_idx = r + (p << 1); + const float *pair_buf = (const float *) (curr_buf + (p << 1) * row_bytes); + bool r0_valid = ((row_idx + 0) < n_rows); + bool r1_valid = ((row_idx + 1) < n_rows); - // Prefetch next (n + spad_nrows) row - const int pr0 = (ir0 + MM_SPAD_SRC0_NROWS); - const int is0 = (pr0 - src0_start_row) % MM_SPAD_SRC0_NROWS; - if (pr0 < src0_end_row_x2) { - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + pr0 * src0_row_size), - src0_stride, src0_row_size, 2); + transfer_activation_row_pair_fp32_to_fp16(vtcm_dst, pair_buf, pair_buf + k_block, row_idx, k_block, k_valid, r0_valid, r1_valid); + } } - } - - // Process the last row (if any) - if (src0_end_row != src0_end_row_x2) { - uint32_t ir0 = src0_end_row_x2; - const int is0 = (ir0 - src0_start_row); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, 1); - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - - #pragma unroll(2) - for (uint32_t ir1 = 0; ir1 < src1_nrows; ++ir1) { - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride); - float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size)); - mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, r); + + // Push step s + 2 + uint32_t next_s = s + 2; + uint32_t next_r = next_s << dma_step_rows_shift; + if (next_r < n_rows) { + uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R); + dma_queue_push(dma_q, dma_make_data(curr_buf, act_dma_addr + (size_t) next_r * src_row_bytes), + row_bytes, src_row_bytes, width_bytes, nrows_to_fetch); } } - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "matmul-%s %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", mmctx->type, ith, nth, - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], src1->ne[1], - src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); } -// q8x4x2 src1 tensor is already in VTCM spad -static void matvec_2d(unsigned int nth, unsigned int ith, void * data) { - htp_matmul_preamble; +static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { + activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data; - const uint32_t src0_nrows = ne01; + struct htp_thread_trace * tr = &st->traces[i]; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); + for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) { + int chunk_idx = task_id * st->n_chunks_per_task; + size_t chunk_size = hex_smin(st->n_tot_chunks - chunk_idx, st->n_chunks_per_task); - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } + __fp16 *dst = st->dst + chunk_idx * st->k_block; + const dma_addr_t act_dma_addr = st->act_dma_addr + (size_t) chunk_idx * st->k_stride * st->act_elem_size; - const size_t dst_row_size = nb1; - const size_t src0_row_size = nb01; - const size_t src1_row_size = nb11; + uint8_t *thread_act = (uint8_t *) st->vtcm_f32_act + i * st->vtcm_f32_act_bytes_per_thread; + transfer_activation_chunk_to_fp16_dma_pipelined( + st->ctx->dma[i], dst, act_dma_addr, chunk_size, st->k_block, st->k_stride, st->k_valid, thread_act, st->act_elem_size, tr, st->dma_step_rows, st->dma_step_rows_shift + ); + } +} - const size_t src0_stride = src0_spad->stride; - const size_t src1_stride = src1_spad->stride; +typedef struct { + struct htp_thread_trace * traces; + const struct mmid_row_mapping * matrix_rows; + __fp16 * dst; + const float * src; + uint32_t n_tasks; + uint32_t n_tot_chunks; + uint32_t n_chunks_per_task; + uint32_t k_block; + uint32_t cur_a; + uint32_t mapping_stride; + uint32_t ne11; + struct fastdiv_values ne11_div; + size_t nb11; + size_t nb12; + uint32_t start_row; + uint32_t cne1; + uint32_t k_valid; +} activation_transfer_gathered_task_state_t; + +typedef struct { + struct htp_thread_trace * traces; + const struct mmid_row_mapping * matrix_rows; + const __fp16 * vtcm_src; + float * dst; + uint32_t n_tasks; + uint32_t n_tot_chunks; + uint32_t n_chunks_per_task; + uint32_t n_cols; + uint32_t cur_a; + uint32_t mapping_stride; + size_t dst_nb1; + size_t dst_nb2; + uint32_t start_row; + uint32_t cne1; +} output_transfer_scattered_task_state_t; + +static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) { + activation_transfer_gathered_task_state_t *st = data; + struct htp_thread_trace * tr = &st->traces[i]; + int chunk_idx = i; + int chunk_size = st->n_chunks_per_task; + int vtcm_start_row = chunk_idx * chunk_size; + int start_row = st->start_row + vtcm_start_row; + int n_rows = hex_smin(st->cne1 - start_row, chunk_size); + if (n_rows > 0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); + transfer_activation_chunk_fp32_to_fp16_gathered( + st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block, + st->matrix_rows, st->cur_a, st->mapping_stride, + st->ne11, &st->ne11_div, st->nb11, st->nb12, st->cne1, st->k_valid); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); + } +} - // Per-thread VTCM scratchpads for all tensors - // Note that the entire src1 tensor is already in VTCM - // For other tensors we allocate N rows per thread, padded to HVX vector size - uint8_t * spad_dst = dst_spad->data + dst_spad->size_per_thread * ith; - uint8_t * spad_src0 = src0_spad->data + src0_spad->size_per_thread * ith; - uint8_t * src1_data = src1_spad->data; +static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unsigned int i, void *data) { + activation_transfer_gathered_task_state_t *st = data; + struct htp_thread_trace * tr = &st->traces[i]; + int chunk_idx = i; + int chunk_size = st->n_chunks_per_task; + int vtcm_start_row = chunk_idx * chunk_size; + int start_row = st->start_row + vtcm_start_row; + int n_rows = hex_smin(st->cne1 - start_row, chunk_size); + if (n_rows > 0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); + transfer_activation_chunk_fp32_to_fp16_gathered_flat( + st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block, + st->matrix_rows, st->cur_a, st->mapping_stride, + st->nb12, st->cne1, st->k_valid); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx); + } +} - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); +static void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned int i, void *data) { + output_transfer_scattered_task_state_t *st = data; + struct htp_thread_trace * tr = &st->traces[i]; + int chunk_idx = i; + int chunk_size = st->n_chunks_per_task; + int vtcm_start_row = chunk_idx * chunk_size; + int start_row = st->start_row + vtcm_start_row; + int n_rows = hex_smin(st->cne1 - start_row, chunk_size); + if (n_rows > 0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx); + transfer_output_chunk_fp16_to_fp32_scattered( + st->dst, st->vtcm_src, start_row, vtcm_start_row, n_rows, st->n_cols, + st->matrix_rows, st->cur_a, st->mapping_stride, + st->dst_nb1, st->dst_nb2, st->cne1); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx); + } +} - float * tmp = (float *) spad_dst; +// --- HMX Dispatchers & Entry Points --- + +static inline tiled_dequant_task_fn_t hmx_get_dequant_worker_fn(int weight_type, bool is_tiled) { + switch (weight_type) { + case HTP_TYPE_Q4_0: return dequantize_tiled_weight_to_fp16_task_q4_0; + case HTP_TYPE_IQ4_NL: return dequantize_tiled_weight_to_fp16_task_iq4_nl; + case HTP_TYPE_Q4_1: + case HTP_TYPE_Q4_K: return dequantize_tiled_weight_to_fp16_task_q4_1; + case HTP_TYPE_MXFP4: return dequantize_tiled_weight_to_fp16_task_mxfp4; + case HTP_TYPE_Q8_0: return dequantize_tiled_weight_to_fp16_task_q8_0; + case HTP_TYPE_Q5_K: return dequantize_tiled_weight_to_fp16_task_q5_k; + case HTP_TYPE_Q6_K: return dequantize_tiled_weight_to_fp16_task_q6_k; + case HTP_TYPE_Q3_K: return dequantize_tiled_weight_to_fp16_task_q3_k; + case HTP_TYPE_Q2_K: return dequantize_tiled_weight_to_fp16_task_q2_k; + case HTP_TYPE_Q1_0: return dequantize_tiled_weight_to_fp16_task_q1_0; + case HTP_TYPE_Q2_0: return dequantize_tiled_weight_to_fp16_task_q2_0; + case HTP_TYPE_F16: return is_tiled ? NULL : convert_f16_weight_to_fp16_tiles_task; + case HTP_TYPE_BF16: return convert_bf16_weight_to_fp16_tiles_task; + case HTP_TYPE_F32: return quantize_f32_weight_to_fp16_tiles_task; + default: return NULL; + } +} - const uint8_t * restrict src0_row = (const uint8_t *) src0->data; - const uint8_t * restrict src1_col = (const uint8_t *) src1_data; - float * restrict dst_col = (float *) dst->data; +static void dequantize_tiled_weight_chunk_to_fp16_tiles( + struct htp_context *ctx, __fp16 *vtcm_dst, + const void *weight_src_ddr, + int n_cols, int k_block, + size_t row_stride, int weight_type, + int n_k_tiles, struct fastdiv_values n_k_tiles_div, + tiled_dequant_task_fn_t dequant_worker_fn, int n_threads) { - // Prefill spad with 2x src0 rows - #pragma unroll(2) - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint32_t is0 = (ir0 - src0_start_row); - if (is0 >= MM_SPAD_SRC0_NROWS) { - break; - } - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, 2); + if (!dequant_worker_fn) { + return; } - // Process src0 rows - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - mmctx->vec_dot_2x1(ne00, &tmp[ir0 - src0_start_row], ss0, ss0 + src0_stride, src1_col); - - // Prefetch next (n + spad_nrows) row - const uint32_t pr0 = (ir0 + MM_SPAD_SRC0_NROWS); - const uint32_t is0 = (pr0 - src0_start_row) % MM_SPAD_SRC0_NROWS; - if (pr0 < src0_end_row_x2) { - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + pr0 * src0_row_size), - src0_stride, src0_row_size, 2); - } + assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0); + + const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); + const uint32_t n_weight_tiles = k_block / htp_tensor_get_tile_k(weight_type); + const size_t n_tot_tiles = n_col_tiles * n_weight_tiles; + + size_t n_tiles_per_task = (n_threads == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_threads); + + tiled_dequantize_state_t state; + state.n_tasks = (n_tot_tiles + n_tiles_per_task - 1) / n_tiles_per_task; + state.n_tot_tiles = n_tot_tiles; + state.n_tiles_per_task = n_tiles_per_task; + state.dst = vtcm_dst; + state.src = (const uint8_t *)weight_src_ddr; + state.task_fn = dequant_worker_fn; + state.n_cols = n_cols; + state.k_block = k_block; + state.row_stride = row_stride; + state.weight_type = weight_type; + state.n_k_tiles = n_k_tiles; + state.n_weight_tiles = n_weight_tiles; + state.n_k_tiles_div = n_k_tiles_div; + state.traces = ctx->trace; + state.ctx = ctx; + + state.tile_size = htp_tensor_get_tile_size(weight_type); + state.aligned_tile_size = htp_tensor_get_tile_padded_size(weight_type); + + if (state.n_tasks == 1 || n_threads == 1) { + dequantize_tiled_worker_loop(1, 0, &state); + } else { + int n_tasks = hex_smin((int) state.n_tasks, n_threads); + worker_pool_run_func(ctx->worker_pool, dequantize_tiled_worker_loop, &state, n_tasks); } +} - // Process the last row (if any) - if (src0_end_row != src0_end_row_x2) { - const uint32_t ir0 = src0_end_row_x2; - const uint32_t is0 = (ir0 - src0_start_row); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_stride, src0_row + ir0 * src0_row_size), - src0_stride, src0_row_size, 1); - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - mmctx->vec_dot_1x1(ne00, &tmp[ir0 - src0_start_row], ss0, src1_col); +typedef struct { + struct htp_context * ctx; + struct htp_thread_trace * traces; + float * dst; + const __fp16 * vtcm_src; + const float * src2; + uint32_t n_rows; + uint32_t n_cols; + uint32_t dst_stride; + uint32_t src2_stride; + uint32_t dst_cols; + struct fastdiv_values n_threads_div; +} output_transfer_col_chunk_state_t; + +static void transfer_output_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) { + (void) n; + output_transfer_col_chunk_state_t *st = (output_transfer_col_chunk_state_t *) data; + struct htp_thread_trace * tr = &st->traces[i]; + + // Round up: the last block is partial when N is not 32-aligned. Its pad + // columns are dropped by the dst_cols clamp inside the store. + uint32_t n_blocks = hmx_ceil_div(st->n_cols, 32); + uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div); + uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div); + uint32_t c_first = b_first * 32; + uint32_t c_last = b_last * 32; + uint32_t c_len = c_last - c_first; + + if (c_len == 0) return; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, c_first); + + const __fp16 *vtcm_src = st->vtcm_src + b_first * HTP_MM_HMX_TILE_N_ELMS; + const float *src2 = st->src2 ? (st->src2 + c_first) : NULL; + float *dst = st->dst + c_first; + + int chunk_dst_cols = (int)st->dst_cols - (int)c_first; + if (chunk_dst_cols > 0) { + transfer_output_chunk_fp16_to_fp32_col_chunk( + dst, src2, vtcm_src, 0, st->n_rows, c_len, st->n_cols, + st->dst_stride, st->src2_stride, (uint32_t)chunk_dst_cols + ); } - hvx_copy_f32_ua((uint8_t *) &dst_col[src0_start_row], (uint8_t *) tmp, src0_end_row - src0_start_row); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, c_first); +} - t2 = HAP_perf_get_qtimer_count(); +static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, const float *src2, const __fp16 *vtcm_src, + int n_rows, int n_cols, int dst_stride, uint32_t src2_stride, int dst_cols, int n_threads) { + if (n_rows <= 0) return; + + uint32_t n_blocks = hmx_ceil_div((uint32_t) n_cols, 32); + if (n_threads > 1 && n_blocks >= (uint32_t)n_threads) { + struct fastdiv_values n_threads_div = (n_threads == (int)ctx->n_threads) ? ctx->n_threads_div : init_fastdiv_values(n_threads); + output_transfer_col_chunk_state_t col_state; + col_state.dst = dst; + col_state.src2 = src2; + col_state.vtcm_src = vtcm_src; + col_state.n_rows = (uint32_t)n_rows; + col_state.n_cols = (uint32_t)n_cols; + col_state.dst_stride = (uint32_t)dst_stride; + col_state.src2_stride = src2_stride; + col_state.dst_cols = (uint32_t)dst_cols; + col_state.n_threads_div = n_threads_div; + col_state.traces = ctx->trace; + col_state.ctx = ctx; + + worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_col_chunk_worker_fn, &col_state, n_threads); + return; + } - FARF(HIGH, "matvec-%s %u/%u: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", mmctx->type, ith, nth, - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], src1->ne[1], - src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + size_t n_tot_chunks = n_rows; + size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : hmx_ceil_div(n_rows, n_threads); + n_chunks_per_task = hex_align_up(n_chunks_per_task, 2); + + int actual_threads = hmx_ceil_div(n_rows, n_chunks_per_task); + + output_transfer_task_state_t state; + state.n_tasks = actual_threads; + state.n_tot_chunks = n_tot_chunks; + state.n_chunks_per_task = n_chunks_per_task; + state.dst = dst; + state.src2 = src2; + state.vtcm_src = vtcm_src; + state.n_cols = n_cols; + state.dst_stride = dst_stride; + state.src2_stride = src2_stride; + state.dst_cols = dst_cols; + state.traces = ctx->trace; + + if (actual_threads <= 1) { + transfer_output_chunk_worker_fn(1, 0, &state); + } else { + worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_worker_fn, &state, actual_threads); + } } -#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * ids->ne[0] * ids->ne[1] + (i1)] - -struct mmid_row_mapping { - uint32_t i1; - uint32_t i2; +struct activation_transfer_params { + struct htp_context * ctx; + __fp16 * dst; + dma_addr_t act_dma_addr; + int n_rows; + int k_block; + int k_stride; + int n_threads; + const struct fastdiv_values * act_threads_div; + const struct fastdiv_values * k_div; + int k_valid; + float * vtcm_f32_act; + size_t vtcm_f32_act_bytes; + uint32_t act_elem_size; // 4=F32 src1, 2=F16 src1 }; -// src1 tensor is already in VTCM spad -static void matmul_id(unsigned int nth, unsigned int ith, void * data) { - htp_matmul_preamble; +static void transfer_activation_chunk_threaded(const struct activation_transfer_params * params) { + struct htp_context * ctx = params->ctx; + __fp16 * dst = params->dst; + const dma_addr_t act_dma_addr = params->act_dma_addr; + int n_rows = params->n_rows; + int k_block = params->k_block; + int k_stride = params->k_stride; + int n_threads = params->n_threads; + const struct fastdiv_values * act_threads_div = params->act_threads_div; + const struct fastdiv_values * k_div = params->k_div; + int k_valid = params->k_valid; + float * vtcm_f32_act = params->vtcm_f32_act; + size_t vtcm_f32_act_bytes = params->vtcm_f32_act_bytes; + // element size of the activation rows (4 = F32, 2 = F16). + const uint32_t act_elem_size = params->act_elem_size ? params->act_elem_size : (uint32_t) sizeof(float); + + if (n_rows <= 0) { + return; + } - const struct htp_tensor * restrict ids = octx->src[2]; - struct htp_spad * restrict src2_spad = &octx->src2_spad; + const size_t n_tasks = (n_rows + 31) >> 5; + if (act_elem_size == sizeof(float) && n_threads > 1 && k_block > 32 && n_tasks < (size_t) n_threads) { + // Calculate step rows parameters for column-chunked dma pipelining + uint32_t dma_step_rows = 2; + uint32_t dma_step_rows_shift = 1; + if (vtcm_f32_act_bytes > 0) { + size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + size_t thread_scratch_elements = thread_scratch_bytes / sizeof(float); + size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); + if (dma_step_rows_max >= 4) { + dma_step_rows = 4; + dma_step_rows_shift = 2; + } + } - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + activation_transfer_col_chunk_state_t col_state; + col_state.dst = dst; + col_state.act_dma_addr = act_dma_addr; + col_state.n_rows = n_rows; + col_state.k_block = k_block; + col_state.k_stride = k_stride; + col_state.k_valid = k_valid; + col_state.n_col_chunks = n_threads; + col_state.n_threads_div = *act_threads_div; + col_state.vtcm_f32_act = vtcm_f32_act; + col_state.vtcm_f32_act_bytes = vtcm_f32_act_bytes; + col_state.traces = ctx->trace; + col_state.ctx = ctx; + col_state.dma_step_rows = dma_step_rows; + col_state.dma_step_rows_shift = dma_step_rows_shift; + + worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_threads); + return; + } - const uint32_t src0_nrows = ne01; // src0 rows per expert - const uint32_t src1_nrows = ne11; + assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0); + + size_t n_tot_chunks = n_rows; + size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address + + activation_transfer_task_state_t state; + state.n_tasks = (n_threads == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32); + state.n_tot_chunks = n_tot_chunks; + state.n_chunks_per_task = n_chunks_per_task; + state.dst = dst; + state.act_dma_addr = act_dma_addr; + state.k_block = k_block; + state.k_stride = k_stride; + state.k_valid = k_valid; + state.traces = ctx->trace; + state.ctx = ctx; + state.vtcm_f32_act = vtcm_f32_act; + state.act_elem_size = act_elem_size; + + state.vtcm_f32_act_bytes_per_thread = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128); + + uint32_t dma_step_rows = 2; + uint32_t dma_step_rows_shift = 1; + if (state.vtcm_f32_act_bytes_per_thread > 0) { + size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_thread / sizeof(float); + size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div); + if (dma_step_rows_max >= 4) { + dma_step_rows = 4; + dma_step_rows_shift = 2; + } + } + state.dma_step_rows = dma_step_rows; + state.dma_step_rows_shift = dma_step_rows_shift; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); + int active_threads = hex_smin(n_threads, (int)state.n_tasks); + if (state.n_tasks == 1 || n_threads == 1) { + transfer_activation_chunk_worker_fn(1, 0, &state); + } else { + worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_worker_fn, &state, active_threads); + } +} +// --- Async HMX matmul job (for pipeline overlap) --- + +typedef struct { + __fp16 * output; + const __fp16 * activation; + const __fp16 * weight; + const __fp16 * scales; + uint32_t n_row_tiles; + uint32_t n_col_tiles; + uint32_t n_dot_tiles; +} hmx_matmul_job_t; + +static void hmx_matmul_worker_fn(void * data) { + hmx_matmul_job_t * job = (hmx_matmul_job_t *) data; + FARF(HIGH, "hmx-mm-job: n_row_tiles %u n_col_tiles %u n_dot_tiles %u", job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); + core_dot_chunk_fp16(job->output, job->activation, job->weight, job->scales, job->n_row_tiles, job->n_col_tiles, job->n_dot_tiles); +} - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; +static inline void hmx_matmul_job_init(hmx_matmul_job_t * job, + __fp16 * output, + const __fp16 * activation, + const __fp16 * weight, + const __fp16 * scales, + uint32_t n_row_tiles, + uint32_t n_col_tiles, + uint32_t n_dot_tiles) { + job->output = output; + job->activation = activation; + job->weight = weight; + job->scales = scales; + job->n_row_tiles = n_row_tiles; + job->n_col_tiles = n_col_tiles; + job->n_dot_tiles = n_dot_tiles; +} + +static int hmx_mm_2d_f32(struct htp_context *ctx, + dma_queue *weight_dma, + float *restrict dst, + dma_addr_t src2_addr, + size_t src2_bytes, + dma_addr_t act_dma_addr, + dma_addr_t weight, + int m, int k, int n, + int act_stride, + uint32_t act_elem_size, // 4=F32 src1, 2=F16 src1 + int weight_stride, + int weight_type, + int k_valid, + int dst_stride, + uint32_t src2_stride, + int dst_cols, + int m_chunk, + int n_chunk, + int pipeline, + int n_threads, + int act_threads, + const struct fastdiv_values * act_threads_div, + const struct fastdiv_values * k_div, + int tile_size, + int aligned_tile_size, + int vtcm_size) { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + + const bool is_tiled = (tile_size > 0); + + if (k % 32 != 0) { + return -1; } + if (!hex_is_aligned(dst, sizeof(float))) { return -1; } - const uint32_t n_ids = ids->ne[0]; // n_expert_used - const uint32_t n_as = ne02; // n_expert + size_t row_stride = htp_tensor_get_row_size(weight_type, k); + if (row_stride == 0) { + return -1; + } - const size_t matrix_row_counts_size = n_as * sizeof(uint32_t); - const size_t matrix_row_map_size = n_as * ids->ne[0] * ids->ne[1] * sizeof(struct mmid_row_mapping); + tiled_dequant_task_fn_t dequant_worker_fn = hmx_get_dequant_worker_fn(weight_type, is_tiled); - const uint32_t * matrix_row_counts = (const uint32_t *) src2_spad->data + 0; - const struct mmid_row_mapping * matrix_rows = (const void *) src2_spad->data + matrix_row_counts_size; + const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type); + const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - const size_t dst_row_size = nb1; - const size_t src0_row_size = nb01; - const size_t src1_row_size = q8x4x2_row_size(ne10); + const size_t vec_dot_size = k * sizeof(__fp16); + const size_t vtcm_budget = ctx->vtcm_size; - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); + const uint32_t dma_dst_stride = is_tiled ? aligned_tile_size : row_stride; + const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; + const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride; - // Per-thread VTCM scratchpads for all tensors - // Note that the entire src1 tensor is already in VTCM - // For other tensors we allocate N rows per thread, padded to HVX vector size - uint8_t * restrict spad_dst = dst_spad->data + dst_spad->size_per_thread * ith; - uint8_t * restrict spad_src0 = src0_spad->data + src0_spad->size_per_thread * ith; - uint8_t * restrict src1_data = src1_spad->data; + size_t m_chunk_n_rows = m_chunk; + size_t n_chunk_n_cols = n_chunk; + size_t vtcm_used = vtcm_size; - for (uint32_t cur_a = 0; cur_a < n_as; ++cur_a) { - const int32_t cne1 = matrix_row_counts[cur_a]; + struct htp_mm_hmx_vtcm_layout L; + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, src2_bytes); - if (cne1 == 0) { - continue; - } + vtcm_used = L.total_bytes; + if (vtcm_used > vtcm_budget) { + FARF(ERROR, "hmx-mm-2d-precomputed: VTCM overflow: used %zu budget %zu, m %d k %d n %d mc %zu nc %zu", + vtcm_used, vtcm_budget, m, k, n, m_chunk_n_rows, n_chunk_n_cols); + return -1; + } - const uint8_t * src0_row = (const uint8_t *) src0->data + (0 + cur_a * nb02 + 0); + uint8_t * const base = (uint8_t *) ctx->vtcm_base; + __fp16 *vtcm_weight_raw[2] = { + VTCM_LAYOUT_PTR(__fp16, base, L.off_weight[0]), + VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_weight[1], pipeline) + }; + __fp16 *vtcm_f16_act = VTCM_LAYOUT_PTR(__fp16, base, L.off_act); + float *vtcm_f32_act = VTCM_LAYOUT_PTR(float, base, L.off_act_f32); + __fp16 *vtcm_output = VTCM_LAYOUT_PTR(__fp16, base, L.off_dst[0]); + void *vtcm_scratch0 = VTCM_LAYOUT_PTR(void, base, L.off_scratch[0]); + void *vtcm_scratch1 = VTCM_LAYOUT_PTR_OPTIONAL(void, base, L.off_scratch[1], pipeline); + void *vtcm_scratch2 = VTCM_LAYOUT_PTR_OPTIONAL(void, base, L.off_dst[1], pipeline); + __fp16 *vtcm_scales = VTCM_LAYOUT_PTR(__fp16, base, L.off_scales); + + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 + + const bool has_bias = (src2_bytes > 0 && src2_addr != 0); + float *vtcm_bias = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_bias, has_bias); + if (has_bias) { + dma_queue_push(weight_dma, dma_make_data(vtcm_bias, src2_addr), hex_align_up(src2_bytes, 128), 0, src2_bytes, 1); + dma_queue_pop(weight_dma); + } - // Prefill spad with src0 rows - #pragma unroll(4) - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const int is0 = (ir0 - src0_start_row); - if (is0 >= MM_SPAD_SRC0_NROWS) { - break; + FARF(HIGH, "hmx-mm-2d: m %d k %d n %d wtype %d mc %zu nc %zu vtcm %zu/%zu", + m, k, n, weight_type, m_chunk_n_rows, n_chunk_n_cols, vtcm_used, vtcm_budget); + + int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + + if (pipeline) { + // --- Asynchronous Pipelined Loop --- + hmx_matmul_job_t job_slots[2]; // persistent double-buffered job descriptors + void *vtcm_raw_bufs[4] = { + vtcm_weight_raw[0], + vtcm_weight_raw[1], + dequant_worker_fn ? vtcm_weight_raw[0] : vtcm_scratch0, + dequant_worker_fn ? vtcm_weight_raw[1] : vtcm_scratch1, + }; + void *vtcm_weight_bufs[4] = { + dequant_worker_fn ? vtcm_scratch0 : vtcm_weight_raw[0], + dequant_worker_fn ? vtcm_scratch1 : vtcm_weight_raw[1], + vtcm_scratch0, + vtcm_scratch1, + }; + void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; + + for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_f16_act, + .act_dma_addr = act_dma_addr + (size_t) mr * act_stride * act_elem_size, // byte offset (F16/F32) + .n_rows = (int) n_rows, + .k_block = k, + .k_stride = act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = k_valid, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = act_elem_size, + }; + transfer_activation_chunk_threaded(&act_params); + + // Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1) + const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); + const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0); + + if (1 < n_chunk_cnt) { + const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); + const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1); } - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + ir0 * src0_row_size), - src0_row_size_padded, src0_row_size, 2); - } - // Process src0 rows - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; + // Main loop: pop A_i -> dequantize A_i -> push A_{i+2} -> submit C_i -> wait C_{i-1} and store D_{i-1} + for (int i = 0; i < n_chunk_cnt; ++i) { + const size_t nc = i * n_chunk_n_cols; + const size_t nc_p2 = nc + 2 * n_chunk_n_cols; - for (uint32_t cid = 0; cid < cne1; ++cid) { - struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, cid); - const int rm1 = row_mapping.i1; // expert idx - const int rm2 = row_mapping.i2; // token idx + const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); + const size_t n_cols_p2 = hex_smin(n - nc_p2, n_chunk_n_cols); + + // 1. pop A_i + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; - const uint32_t ir1 = src1_nrows == 1 ? 0 : rm1; // src1 row idx - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (ir1 + rm2 * ne11 + 0) * src1_row_size); - float * dst_row = (float *) (dst->data + (rm1 * nb1 + rm2 * nb2 + 0)); + // 2. dequantize A_i + dequantize_tiled_weight_chunk_to_fp16_tiles( + ctx, vtcm_weight_bufs[i % 4], curr_raw, + n_cols, k, row_stride, weight_type, + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); - mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_row_size_padded, src1_col); + // 3. push A_{i+2} (if i+2 < n_chunk_cnt) + if (i + 2 < n_chunk_cnt) { + const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2); + } + + // 4. submit C_i + hmx_matmul_job_init(&job_slots[i % 2], (__fp16 *) vtcm_output_bufs[i % 2], + (__fp16 *) vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 4], + vtcm_scales, hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS), + hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS), k / HTP_MM_HMX_TILE_N_ROWS); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job_slots[i % 2])); + + // 5. wait C_{i-1} and store D_{i-1} (multi-thread HVX, parallel with C_i) + if (i > 0) { + hmx_queue_pop(ctx->hmx_queue); + const size_t nc_prev = (i - 1) * n_chunk_n_cols; + const size_t n_cols_prev = hex_smin(n - nc_prev, n_chunk_n_cols); + float *output_chunk = dst + (mr * dst_stride + nc_prev); + const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_prev) : NULL; + int chunk_dst_cols = dst_cols - (int)nc_prev; + if (chunk_dst_cols > 0) { + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, src2_stride, chunk_dst_cols, n_threads); + } + } } - // Prefetch next (n + spad_nrows) row - const int pr0 = (ir0 + MM_SPAD_SRC0_NROWS); - const int is0 = (pr0 - src0_start_row) % MM_SPAD_SRC0_NROWS; - if (pr0 < src0_end_row_x2) { - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + pr0 * src0_row_size), - src0_row_size_padded, src0_row_size, 2); + // Epilogue: wait C_{last} and store D_{last} + hmx_queue_pop(ctx->hmx_queue); + const size_t nc_last = (n_chunk_cnt - 1) * n_chunk_n_cols; + const size_t n_cols_last = hex_smin(n - nc_last, n_chunk_n_cols); + float *output_chunk = dst + (mr * dst_stride + nc_last); + const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc_last) : NULL; + int chunk_dst_cols = dst_cols - (int)nc_last; + if (chunk_dst_cols > 0) { + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, src2_stride, chunk_dst_cols, n_threads); } } + } else { + // --- Synchronous loop (m <= 32 or fallback) --- + hmx_matmul_job_t job; + void *vtcm_raw_bufs[2] = { vtcm_weight_raw[0], dequant_worker_fn ? vtcm_weight_raw[0] : vtcm_scratch0 }; + void *vtcm_weight_bufs[2] = { dequant_worker_fn ? vtcm_scratch0 : vtcm_weight_raw[0], vtcm_scratch0 }; + + for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_f16_act, + .act_dma_addr = act_dma_addr + (size_t) mr * act_stride * act_elem_size, // byte offset (F16/F32) + .n_rows = (int) n_rows, + .k_block = k, + .k_stride = act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = k_valid, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = act_elem_size, + }; + transfer_activation_chunk_threaded(&act_params); - // Process the last row (if any) - if (src0_end_row != src0_end_row_x2) { - uint32_t ir0 = src0_end_row_x2; - const uint32_t is0 = (ir0 - src0_start_row); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + ir0 * src0_row_size), - src0_row_size_padded, src0_row_size, 1); - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - - for (uint32_t cid = 0; cid < cne1; ++cid) { - struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, cid); - const int rm1 = row_mapping.i1; // expert idx - const int rm2 = row_mapping.i2; // token idx + // A0: Pre-fetch the first weight chunk (nc = 0) + if (n > 0) { + const size_t n_cols = hex_smin(n, n_chunk_n_cols); + const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); + } - const uint32_t ir1 = src1_nrows == 1 ? 0 : rm1; // src1 row idx - const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + (ir1 + rm2 * ne11 + 0) * src1_row_size); - float * dst_row = (float *) (dst->data + (rm1 * nb1 + rm2 * nb2 + 0)); + for (int i = 0; i < n_chunk_cnt; ++i) { + const size_t nc = (size_t) i * n_chunk_n_cols; + const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); + const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); + const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); + + // A: Wait for weight DMA + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; + + // B: Weight Dequantize (Threaded) + dequantize_tiled_weight_chunk_to_fp16_tiles( + ctx, vtcm_weight_bufs[i % 2], curr_raw, + n_cols, k, row_stride, weight_type, + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + + // Start weight DMA for the next chunk early + const size_t nc_next = nc + n_chunk_n_cols; + if (nc_next < n) { + const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); + const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); + } - mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col); + // C: HMX Compute (Queue-based) + hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 2], vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); + hmx_queue_pop(ctx->hmx_queue); + + // D: Output Store + float *output_chunk = dst + (mr * dst_stride + nc); + const float *bias_chunk = has_bias ? (vtcm_bias + mr * src2_stride + nc) : NULL; + int chunk_dst_cols = dst_cols - (int)nc; + if (chunk_dst_cols > 0) { + transfer_output_chunk_threaded(ctx, output_chunk, bias_chunk, vtcm_output, n_rows, n_cols, dst_stride, src2_stride, chunk_dst_cols, n_threads); + } } } } - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "matmul-id-%s %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u (%ux%ux%ux%u) -> %ux%ux%ux%u usec %u\n", mmctx->type, - ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], - src1->ne[1], src1->ne[2], src1->ne[3], ids->ne[0], ids->ne[1], ids->ne[2], ids->ne[3], dst->ne[0], dst->ne[1], - dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + return 0; } -// src1 tensor is already in VTCM spad -static void matvec_id(unsigned int nth, unsigned int ith, void * data) { - htp_matmul_preamble; +static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_kernel_params * kparams) { + struct htp_context * ctx = octx->ctx; + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - const struct htp_tensor * restrict ids = octx->src[2]; - struct htp_spad * restrict src2_spad = &octx->src2_spad; + const uint32_t n_weights = kparams->n_weights; + if (n_weights == 0 || n_weights > HTP_OP_MAX_OUTPUTS) { + return HTP_STATUS_INVAL_PARAMS; + } - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict act = octx->src[n_weights]; - const uint32_t src0_nrows = ne01; // src0 rows per expert + const int weight_type = (int) src0->type; + const int k = (int) act->ne[0]; + const int k_valid = (int) act->ne[0]; + const int m = (int) (act->ne[1] * act->ne[2] * act->ne[3]); + const uint32_t act_elem_size = (act->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + const int act_stride = (int) (act->nb[1] / act_elem_size); + const dma_addr_t act_dma_addr = act->data; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U); + if (k % 32 != 0) { return HTP_STATUS_NO_SUPPORT; } - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; + size_t row_stride = htp_tensor_get_row_size(weight_type, k); + if (row_stride == 0) { + return HTP_STATUS_NO_SUPPORT; } - assert(ne13 % ne03 == 0); + const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type); + const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - const size_t dst_row_size = nb1; - const size_t src0_row_size = nb01; - const size_t src1_row_size = q8x4x2_row_size(ne10); + const int tile_size = kparams->tile_size; + const int aligned_tile_size = kparams->aligned_tile_size; - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); + const bool is_tiled = (tile_size > 0); + tiled_dequant_task_fn_t dequant_worker_fn = hmx_get_dequant_worker_fn(weight_type, is_tiled); + const size_t vtcm_budget = ctx->vtcm_size; - const uint32_t n_aids = src2->ne[0]; // num activated experts - const uint32_t n_ids = ne02; // num experts + const int m_chunk_n_rows = kparams->m_chunk; + const int n_chunk_n_cols = kparams->n_chunk; + const int pipeline = kparams->pipeline; + const int n_threads = octx->n_threads; + const int act_threads = kparams->n_act_threads; + const struct fastdiv_values * act_threads_div = &kparams->div_n_act_threads; + const struct fastdiv_values * k_div = &kparams->div_ne00_padded; - // Per-thread VTCM scratchpads for all tensors - // Note that the entire src1 tensor is already in VTCM - // For other tensors we allocate N rows per thread, padded to HVX vector size - uint8_t * restrict spad_dst = dst_spad->data + dst_spad->size_per_thread * ith; - uint8_t * restrict spad_src0 = src0_spad->data + src0_spad->size_per_thread * ith; - uint8_t * restrict src1_data = src1_spad->data; + const uint32_t dma_dst_stride = is_tiled ? aligned_tile_size : row_stride; + const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride; - for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert - const uint32_t eid = *(const int32_t *) ((const uint8_t *) src2->data + ie1 * src2->nb[0]); - assert(eid < n_ids); + struct htp_mm_hmx_vtcm_layout L; + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, 0); - const uint8_t * restrict src0_row = (const uint8_t *) src0->data + eid * nb02; - const uint8_t * restrict src1_col = (const uint8_t *) src1_data; - float * restrict dst_row = (float *) (dst->data + ie1 * nb1); + if (L.total_bytes > vtcm_budget) { + FARF(ERROR, "hmx-mm-nx-2d: VTCM overflow: used %zu budget %zu, m %d k %d mc %d nc %d", + L.total_bytes, vtcm_budget, m, k, m_chunk_n_rows, n_chunk_n_cols); + return HTP_STATUS_VTCM_TOO_SMALL; + } - // Prefill spad with src0 rows - #pragma unroll(4) - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const int is0 = (ir0 - src0_start_row); - if (is0 >= MM_SPAD_SRC0_NROWS) { - break; + uint8_t * const base = (uint8_t *) ctx->vtcm_base; + __fp16 *vtcm_weight_raw[2] = { + VTCM_LAYOUT_PTR(__fp16, base, L.off_weight[0]), + VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_weight[1], pipeline) + }; + __fp16 *vtcm_f16_act = VTCM_LAYOUT_PTR(__fp16, base, L.off_act); + float *vtcm_f32_act = VTCM_LAYOUT_PTR(float, base, L.off_act_f32); + __fp16 *vtcm_output = VTCM_LAYOUT_PTR(__fp16, base, L.off_dst[0]); + void *vtcm_scratch0 = VTCM_LAYOUT_PTR(void, base, L.off_scratch[0]); + void *vtcm_scratch1 = VTCM_LAYOUT_PTR_OPTIONAL(void, base, L.off_scratch[1], pipeline); + void *vtcm_scratch2 = VTCM_LAYOUT_PTR_OPTIONAL(void, base, L.off_dst[1], pipeline); + __fp16 *vtcm_scales = VTCM_LAYOUT_PTR(__fp16, base, L.off_scales); + + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 + + int m_start = 0; + int m_rows = m; + if (octx->ctx->mdev.count > 1) { + bool can_split = htp_tensor_can_row_partition(act, act_elem_size); + if (can_split) { + for (uint32_t p = 0; p < n_weights; ++p) { + const struct htp_tensor * restrict dst = octx->dsts[p]; + if (!htp_tensor_can_row_partition(dst, sizeof(float))) { + can_split = false; + break; + } } - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + ir0 * src0_row_size), - src0_row_size_padded, src0_row_size, 2); } + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_rows = (int) range.count; + } + + if (m_rows == 0) { + return HTP_STATUS_OK; + } - // Process src0 rows - for (uint32_t ir0 = src0_start_row; ir0 < src0_end_row_x2; ir0 += 2) { - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - mmctx->vec_dot_2x1(ne00, &dst_row[ir0], ss0, ss0 + src0_row_size_padded, src1_col); + FARF(HIGH, "hmx-mm-nx-2d: n_weights %u m %d (%d..%d) k %d wtype %d mc %d nc %d vtcm %zu/%zu", + n_weights, m, m_start, m_start + m_rows, k, weight_type, m_chunk_n_rows, n_chunk_n_cols, L.total_bytes, vtcm_budget); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + + const size_t mr_end = (size_t)(m_start + m_rows); + + if (pipeline) { + hmx_matmul_job_t job_slots[2]; + void *vtcm_raw_bufs[4] = { + vtcm_weight_raw[0], + vtcm_weight_raw[1], + dequant_worker_fn ? vtcm_weight_raw[0] : vtcm_scratch0, + dequant_worker_fn ? vtcm_weight_raw[1] : vtcm_scratch1, + }; + void *vtcm_weight_bufs[4] = { + dequant_worker_fn ? vtcm_scratch0 : vtcm_weight_raw[0], + dequant_worker_fn ? vtcm_scratch1 : vtcm_weight_raw[1], + vtcm_scratch0, + vtcm_scratch1, + }; + void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 }; + + for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); + + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_f16_act, + .act_dma_addr = act_dma_addr + (size_t) mr * act_stride * act_elem_size, + .n_rows = (int) n_rows, + .k_block = k, + .k_stride = act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = k_valid, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = act_elem_size, + }; + transfer_activation_chunk_threaded(&act_params); + + for (uint32_t p = 0; p < n_weights; p++) { + const struct htp_tensor * restrict src_w = octx->src[p]; + const struct htp_tensor * restrict dst = octx->dsts[p]; + if (!src_w || !dst) continue; + + const dma_addr_t weight = src_w->data; + dma_queue * weight_dma = octx->ctx->dma[0]; + float * dst_ptr = (float *) dst->data; + const size_t n = src_w->ne[1]; + if (n == 0) continue; + const size_t weight_stride = src_w->nb[1]; + const size_t dst_stride = dst->nb[1] / sizeof(float); + const int dst_cols = (int) dst->ne[0]; + const int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); + + const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; + + const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); + const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0); + + if (1 < n_chunk_cnt) { + const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); + const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1); + } - // Prefetch next (n + spad_nrows) row - const int pr0 = (ir0 + MM_SPAD_SRC0_NROWS); - const int is0 = (pr0 - src0_start_row) % MM_SPAD_SRC0_NROWS; - if (pr0 < src0_end_row_x2) { - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + pr0 * src0_row_size), - src0_row_size_padded, src0_row_size, 2); + for (int i = 0; i < n_chunk_cnt; ++i) { + const size_t nc = i * n_chunk_n_cols; + const size_t nc_p2 = nc + 2 * n_chunk_n_cols; + + const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); + const size_t n_cols_p2 = hex_smin(n - nc_p2, n_chunk_n_cols); + + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; + + dequantize_tiled_weight_chunk_to_fp16_tiles( + ctx, vtcm_weight_bufs[i % 4], curr_raw, + n_cols, k, row_stride, weight_type, + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + + if (i + 2 < n_chunk_cnt) { + const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2); + } + + hmx_matmul_job_init(&job_slots[i % 2], (__fp16 *) vtcm_output_bufs[i % 2], + (__fp16 *) vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 4], + vtcm_scales, hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS), + hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS), k / HTP_MM_HMX_TILE_N_ROWS); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job_slots[i % 2])); + + if (i > 0) { + hmx_queue_pop(ctx->hmx_queue); + const size_t nc_prev = (i - 1) * n_chunk_n_cols; + const size_t n_cols_prev = hex_smin(n - nc_prev, n_chunk_n_cols); + float *output_chunk = dst_ptr + (mr * dst_stride + nc_prev); + int chunk_dst_cols = dst_cols - (int)nc_prev; + if (chunk_dst_cols > 0) { + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(i - 1) % 2], n_rows, n_cols_prev, dst_stride, 0, chunk_dst_cols, n_threads); + } + } + } + + hmx_queue_pop(ctx->hmx_queue); + const size_t nc_last = (n_chunk_cnt - 1) * n_chunk_n_cols; + const size_t n_cols_last = hex_smin(n - nc_last, n_chunk_n_cols); + float *output_chunk = dst_ptr + (mr * dst_stride + nc_last); + int chunk_dst_cols = dst_cols - (int)nc_last; + if (chunk_dst_cols > 0) { + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output_bufs[(n_chunk_cnt - 1) % 2], n_rows, n_cols_last, dst_stride, 0, chunk_dst_cols, n_threads); + } } } + } else { + hmx_matmul_job_t job; + void *vtcm_raw_bufs[2] = { vtcm_weight_raw[0], dequant_worker_fn ? vtcm_weight_raw[0] : vtcm_scratch0 }; + void *vtcm_weight_bufs[2] = { dequant_worker_fn ? vtcm_scratch0 : vtcm_weight_raw[0], vtcm_scratch0 }; + + for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows); + + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_f16_act, + .act_dma_addr = act_dma_addr + (size_t) mr * act_stride * act_elem_size, + .n_rows = (int) n_rows, + .k_block = k, + .k_stride = act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = k_valid, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = act_elem_size, + }; + transfer_activation_chunk_threaded(&act_params); + + for (uint32_t p = 0; p < n_weights; p++) { + const struct htp_tensor * restrict src_w = octx->src[p]; + const struct htp_tensor * restrict dst = octx->dsts[p]; + if (!src_w || !dst) continue; + + const dma_addr_t weight = src_w->data; + dma_queue * weight_dma = octx->ctx->dma[0]; + float * dst_ptr = (float *) dst->data; + const size_t n = src_w->ne[1]; + if (n == 0) continue; + const size_t weight_stride = src_w->nb[1]; + const size_t dst_stride = dst->nb[1] / sizeof(float); + const int dst_cols = (int) dst->ne[0]; + const int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); + + const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; + + if (n > 0) { + const size_t n_cols = hex_smin(n, n_chunk_n_cols); + const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); + } - // Process the last row (if any) - if (src0_end_row != src0_end_row_x2) { - uint32_t ir0 = src0_end_row_x2; - const uint32_t is0 = (ir0 - src0_start_row); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0 + is0 * src0_row_size_padded, src0_row + ir0 * src0_row_size), - src0_row_size_padded, src0_row_size, 1); - const uint8_t * ss0 = dma_queue_pop(dma_queue).dst; - mmctx->vec_dot_1x1(ne00, &dst_row[ir0], ss0, src1_col); + for (int i = 0; i < n_chunk_cnt; ++i) { + const size_t nc = (size_t) i * n_chunk_n_cols; + const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); + const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); + const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); + + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; + + dequantize_tiled_weight_chunk_to_fp16_tiles( + ctx, vtcm_weight_bufs[i % 2], curr_raw, + n_cols, k, row_stride, weight_type, + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); + + const size_t nc_next = nc + n_chunk_n_cols; + if (nc_next < n) { + const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); + const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); + } + + hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 2], vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); + hmx_queue_pop(ctx->hmx_queue); + + float *output_chunk = dst_ptr + (mr * dst_stride + nc); + int chunk_dst_cols = dst_cols - (int)nc; + if (chunk_dst_cols > 0) { + transfer_output_chunk_threaded(ctx, output_chunk, NULL, vtcm_output, n_rows, n_cols, dst_stride, 0, chunk_dst_cols, n_threads); + } + } + } } } - t2 = HAP_perf_get_qtimer_count(); + return HTP_STATUS_OK; +} - FARF(HIGH, "matvec-id-%s %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u (%ux%ux%ux%u) -> %ux%ux%ux%u usec %u\n", mmctx->type, - ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], - src1->ne[1], src1->ne[2], src1->ne[3], src2->ne[0], src2->ne[1], src2->ne[2], src2->ne[3], dst->ne[0], - dst->ne[1], dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); +static inline dma_addr_t hmx_mm_weight_batch_data(const hmx_mm_batched_params_t *params, + int dst_b2, int dst_b3) { + const size_t b2_idx = (params->r2 <= 1) ? (size_t) dst_b2 : (size_t) fastdiv((uint32_t) dst_b2, ¶ms->div_r2); + const size_t b3_idx = (params->r3 <= 1) ? (size_t) dst_b3 : (size_t) fastdiv((uint32_t) dst_b3, ¶ms->div_r3); + return params->weight + b2_idx * params->src0_nb2 + b3_idx * params->src0_nb3; } -// *** dynamic quant +static inline dma_addr_t hmx_mm_act_batch_addr(const hmx_mm_batched_params_t *params, + int dst_b2, int dst_b3) { + return params->act_dma_addr + dst_b2 * params->act_nb2 + + dst_b3 * params->act_nb3; +} -static inline void quantize_block_f32_q8x1(float * restrict x, uint8_t * restrict y_q, uint8_t * restrict y_d) { - assert((unsigned long) x % 128 == 0); - assert((unsigned long) y_q % 128 == 0); +static inline float *hmx_mm_dst_batch_ptr(const hmx_mm_batched_params_t *params, + int dst_b2, int dst_b3) { + return (float *) ((uint8_t *) params->dst + + (size_t) dst_b2 * params->dst_nb2 + + (size_t) dst_b3 * params->dst_nb3); +} - HVX_Vector * vx = (HVX_Vector *) x; - HVX_Vector zero = Q6_V_vzero(); +static int hmx_mm_batched_simple(struct htp_context *ctx, + const hmx_mm_batched_params_t *params, + int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size, + const struct fastdiv_values * act_threads_div, const struct fastdiv_values * k_div) { + int ret = 0; + for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) { + for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) { + dma_addr_t cur_bias_addr = params->bias_addr ? (params->bias_addr + + b2 * params->bias_nb2 + + b3 * params->bias_nb3) : 0; + ret = hmx_mm_2d_f32(ctx, params->weight_dma, hmx_mm_dst_batch_ptr(params, b2, b3), + cur_bias_addr, params->bias_bytes, + hmx_mm_act_batch_addr(params, b2, b3), + hmx_mm_weight_batch_data(params, b2, b3), + params->m, params->k, params->n, + params->act_stride, params->act_elem_size, params->weight_stride, + params->weight_type, params->k, params->dst_stride, params->bias_stride, params->dst_cols, + m_chunk, n_chunk, pipeline, n_threads, act_threads, + act_threads_div, k_div, params->tile_size, params->aligned_tile_size, vtcm_size); + } + } + return ret; +} - // Use reduce max fp32 to find max(abs(e)) first - HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0])); - HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1])); - HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2])); - HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3])); - // Load and convert into QF32 - HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); // 32 elements - HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); // 32 elements - HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); // 32 elements - HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); // 32 elements +static int hmx_mm_batched(struct htp_context *ctx, const hmx_mm_batched_params_t *params, + int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, + const struct fastdiv_values * act_threads_div, + const struct fastdiv_values * k_div, + int vtcm_size) { + const size_t row_stride = htp_tensor_get_row_size(params->weight_type, params->k); + if (row_stride == 0) { return -1; } + if (params->act_stride < params->k || (params->tile_size == 0 && (size_t) params->weight_stride < row_stride) || params->dst_stride < params->dst_cols) { return -1; } + if (params->ne02 <= 0 || params->ne03 <= 0 || params->ne12 <= 0 || params->ne13 <= 0) { return -1; } + if (params->ne12 % params->ne02 != 0 || params->ne13 % params->ne03 != 0) { return -1; } + // N (the weight row count) does not have to be 32-aligned: + if (params->k % 32 != 0) { return -1; } + if (!hex_is_aligned(params->dst, sizeof(float))) { return -1; } + + const int group_size = params->r2; + const size_t vtcm_budget = ctx->vtcm_size; + + // Check if the precomputed parameters are grouped or simple. + // If simple, or if group_size <= 1, we use simple fallback loop. + // Grouped path is only valid for flat F16 if group_size > 1 and it fits within VTCM budget. + bool run_grouped = (params->weight_type == HTP_TYPE_F16 && params->tile_size == 0 && group_size > 1 && (size_t)vtcm_size <= vtcm_budget); + if (!run_grouped) { + return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); + } - // Convert to QF32 - HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero); // replicated over all lanes - HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero); // replicated over all lanes - HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero); // replicated over all lanes - HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero); // replicated over all lanes + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - // Combine and convert to fp16 - HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf))); - HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf))); + size_t m_chunk_n_rows = m_chunk; + size_t n_chunk_n_cols = n_chunk; - // Convert into fp16 - HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); - HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); + struct htp_mm_hmx_vtcm_layout L; + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, HTP_TYPE_F16, params->k, m_chunk_n_rows, n_chunk_n_cols, group_size, false, act_threads, 0, params->bias_bytes); - HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); - HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); + if (L.total_bytes > vtcm_budget) { + FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + return hmx_mm_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div); + } - hvx_vec_store_u(y_d + 0, 2, vd01_hf); - HVX_Vector rotated_vd_hf = Q6_V_vror_VR(vd01_hf, 64); - hvx_vec_store_u(y_d + 2, 2, rotated_vd_hf); + uint8_t * const base = (uint8_t *) ctx->vtcm_base; + __fp16 *vtcm_weight = VTCM_LAYOUT_PTR(__fp16, base, L.off_weight[0]); + __fp16 *vtcm_f16_act = VTCM_LAYOUT_PTR(__fp16, base, L.off_act); + __fp16 *vtcm_output = VTCM_LAYOUT_PTR(__fp16, base, L.off_dst[0]); + void *vtcm_scratch0 = VTCM_LAYOUT_PTR(void, base, L.off_scratch[0]); + void *vtcm_scratch1 = VTCM_LAYOUT_PTR(void, base, L.off_scratch[1]); + __fp16 *vtcm_scales = VTCM_LAYOUT_PTR(__fp16, base, L.off_scales); + float *vtcm_f32_act = VTCM_LAYOUT_PTR(float, base, L.off_act_f32); + + const bool has_bias = (params->bias_bytes > 0 && params->bias_addr != 0); + float *vtcm_bias = VTCM_LAYOUT_PTR_OPTIONAL(float, base, L.off_bias, has_bias); + if (has_bias) { + dma_queue_push(params->weight_dma, dma_make_data(vtcm_bias, params->bias_addr), hex_align_up(params->bias_bytes, 128), 0, params->bias_bytes, 1); + dma_queue_pop(params->weight_dma); + } - hvx_vec_store_u(y_d + 4, 2, vd23_hf); - rotated_vd_hf = Q6_V_vror_VR(vd23_hf, 64); - hvx_vec_store_u(y_d + 6, 2, rotated_vd_hf); + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 + + FARF(HIGH, "%s: grouped path m=%d k=%d n=%d group=%d streams=%d mc=%zu nc=%zu vtcm=%zu/%zu", + __func__, params->m, params->k, params->n, group_size, params->ne13, + m_chunk_n_rows, n_chunk_n_cols, + L.total_bytes, vtcm_budget); + + const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16); + const size_t weight_row_bytes = (size_t) params->weight_stride; + + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); + + hmx_matmul_job_t job; + + for (int b3 = 0; b3 < params->ne13; ++b3) { + for (int b2_base = 0; b2_base < params->ne12; b2_base += group_size) { + const dma_addr_t weight_group = hmx_mm_weight_batch_data(params, b2_base, b3); + dma_queue * weight_dma = params->weight_dma; + + for (size_t mr = 0; mr < (size_t) params->m; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin((size_t) params->m - mr, m_chunk_n_rows); + const size_t n_row_tiles = hmx_ceil_div((int) n_rows, HTP_MM_HMX_TILE_N_ROWS); + + // Pre-load activations for all heads in the group (once per m_chunk). + // When the source is strided (permuted Q), use 2D DMA to gather + // contiguous rows into a VTCM scratch buffer first, then HVX + // converts from the contiguous VTCM buffer. This avoids L2 cache + // thrashing from HVX loads at large strides. + for (int g = 0; g < group_size; ++g) { + const dma_addr_t act_dma_addr = hmx_mm_act_batch_addr(params, b2_base + g, b3) + + (size_t) mr * params->act_stride * params->act_elem_size; + __fp16 *vtcm_act_g = vtcm_f16_act + (size_t) g * L.act_head_stride; + struct activation_transfer_params act_params = { + .ctx = ctx, + .dst = vtcm_act_g, + .act_dma_addr = act_dma_addr, + .n_rows = (int) n_rows, + .k_block = params->k, + .k_stride = params->act_stride, + .n_threads = act_threads, + .act_threads_div = act_threads_div, + .k_div = k_div, + .k_valid = params->k, + .vtcm_f32_act = vtcm_f32_act, + .vtcm_f32_act_bytes = L.act_f32_bytes, + .act_elem_size = params->act_elem_size, + }; + transfer_activation_chunk_threaded(&act_params); + } - // Divide input by the scale - HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); - HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); - vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); - vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); + // Prologue: Push A0 and A1 (if exists) + { + const size_t n_cols_first = hex_smin((size_t) params->n, n_chunk_n_cols); + dma_queue_push(weight_dma, dma_make_data(vtcm_scratch0, weight_group), + fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_first); + } + if (n_chunk_n_cols < (size_t) params->n) { + const size_t n_cols_second = hex_smin((size_t) params->n - n_chunk_n_cols, n_chunk_n_cols); + const dma_addr_t second_weight_chunk = weight_group + n_chunk_n_cols * weight_row_bytes; + dma_queue_push(weight_dma, dma_make_data(vtcm_scratch1, second_weight_chunk), + fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_second); + } - // Convert to int8 - HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); - HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); - HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16); + for (size_t nc = 0; nc < (size_t) params->n; nc += n_chunk_n_cols) { + const size_t n_cols = hex_smin((size_t) params->n - nc, n_chunk_n_cols); + const size_t n_col_tiles = hmx_ceil_div((int) n_cols, HTP_MM_HMX_TILE_N_COLS); + + { + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; + + const size_t n_cols_tiled = hex_align_up(n_cols, HTP_MM_HMX_TILE_N_COLS); + + hmx_interleave_rows_to_tiles(vtcm_weight, (const __fp16 *) curr_raw, (uint32_t) n_cols, params->k, params->k, 0, (uint32_t) n_cols_tiled); + + const size_t nc_next = nc + n_chunk_n_cols * 2; + if (nc_next < (size_t) params->n) { + const size_t n_cols_next = hex_smin((size_t) params->n - nc_next, n_chunk_n_cols); + const dma_addr_t next_weight_chunk = weight_group + nc_next * weight_row_bytes; + + dma_queue_push(weight_dma, dma_make_data(curr_raw, next_weight_chunk), + fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_next); + } + } + + // Reuse the interleaved weight for every q_head in this GQA group + for (int g = 0; g < group_size; ++g) { + { + const __fp16 * vtcm_act_g = vtcm_f16_act + (size_t) g * L.act_head_stride; + hmx_matmul_job_init(&job, vtcm_output, vtcm_act_g, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, params->k / 32); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); + hmx_queue_pop(ctx->hmx_queue); + } + + { + float *output = hmx_mm_dst_batch_ptr(params, b2_base + g, b3) + mr * params->dst_stride + nc; + const float *bias_chunk = has_bias ? (vtcm_bias + mr * params->bias_stride + nc) : NULL; + int chunk_dst_cols = params->dst_cols - (int)nc; + if (chunk_dst_cols > 0) { + transfer_output_chunk_threaded(ctx, output, bias_chunk, vtcm_output, (int) n_rows, (int) n_cols, + params->dst_stride, params->bias_stride, chunk_dst_cols, n_threads); + } + } + } + } + } + } + } - *(HVX_Vector *) y_q = vx_i8; + return 0; } -static inline void quantize_block_f32_q8x2(float * restrict x, uint8_t * restrict y_q, uint8_t * restrict y_d) { - assert((unsigned long) x % 128 == 0); - assert((unsigned long) y_q % 128 == 0); - - HVX_Vector * vx = (HVX_Vector *) x; - - // Load and convert into QF32 - HVX_Vector zero = Q6_V_vzero(); - HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); // 32 elements - HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); // 32 elements - HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); // 32 elements - HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); // 32 elements +static void transfer_activation_chunk_gathered_threaded( + struct htp_context *ctx, + __fp16 *dst, + const float *src, + int start_row, + int n_rows, + int k_block, + const struct mmid_row_mapping *matrix_rows, + int cur_a, + int mapping_stride, + int ne11, + size_t nb11, + size_t nb12, + int cne1, + int n_threads, + int k_valid) { + if (n_rows <= 0) return; + int chunks_per_thread = hmx_ceil_div(n_rows, n_threads); + chunks_per_thread = hex_align_up(chunks_per_thread, 2); + + int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread); + + activation_transfer_gathered_task_state_t state = { + .dst = dst, + .src = src, + .n_tasks = actual_threads, + .n_tot_chunks = n_rows, + .n_chunks_per_task = chunks_per_thread, + .k_block = k_block, + .matrix_rows = matrix_rows, + .cur_a = cur_a, + .mapping_stride = mapping_stride, + .ne11 = ne11, + .ne11_div = ne11 > 1 ? init_fastdiv_values(ne11) : (struct fastdiv_values){0, 0}, + .nb11 = nb11, + .nb12 = nb12, + .start_row = start_row, + .cne1 = cne1, + .k_valid = k_valid, + .traces = ctx->trace, + }; + + worker_callback_t worker_fn = ne11 == 1 ? transfer_activation_chunk_gathered_worker_flat_fn : + transfer_activation_chunk_gathered_worker_fn; + + if (actual_threads <= 1) { + worker_fn(1, 0, &state); + } else { + worker_pool_run_func(ctx->worker_pool, worker_fn, &state, actual_threads); + } +} - // Convert into fp16 - HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); - HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); +static void transfer_output_chunk_scattered_threaded( + struct htp_context *ctx, + float *dst, + const __fp16 *vtcm_src, + int start_row, + int n_rows, + int n_cols, + const struct mmid_row_mapping *matrix_rows, + int cur_a, + int mapping_stride, + size_t dst_nb1, + size_t dst_nb2, + int cne1, + int n_threads) { + if (n_rows <= 0) return; + int chunks_per_thread = hmx_ceil_div(n_rows, n_threads); + chunks_per_thread = hex_align_up(chunks_per_thread, 2); + + int actual_threads = hmx_ceil_div(n_rows, chunks_per_thread); + + output_transfer_scattered_task_state_t state = { + .vtcm_src = vtcm_src, + .dst = dst, + .n_tasks = actual_threads, + .n_tot_chunks = n_rows, + .n_chunks_per_task = chunks_per_thread, + .n_cols = n_cols, + .matrix_rows = matrix_rows, + .cur_a = cur_a, + .mapping_stride = mapping_stride, + .dst_nb1 = dst_nb1, + .dst_nb2 = dst_nb2, + .start_row = start_row, + .cne1 = cne1, + .traces = ctx->trace, + }; + + if (actual_threads <= 1) { + transfer_output_chunk_scattered_worker_fn(1, 0, &state); + } else { + worker_pool_run_func(ctx->worker_pool, transfer_output_chunk_scattered_worker_fn, &state, actual_threads); + } +} - // Compute max and scale - HVX_Vector vmax01_hf = hvx_vec_reduce_max_f16(hvx_vec_abs_f16(vx01_hf)); // replicated over all lanes - HVX_Vector vmax23_hf = hvx_vec_reduce_max_f16(hvx_vec_abs_f16(vx23_hf)); // replicated over all lanes +static int hmx_mm_id_2d_f32(struct htp_context *ctx, + dma_queue *weight_dma, + float *restrict dst, + const float *activation, + dma_addr_t weight, + int m, int k, int n, + int k_valid, + int ne11, + size_t act_nb1, size_t act_nb2, + size_t dst_nb1, size_t dst_nb2, + int weight_stride, + int weight_type, + int tile_size, + int aligned_tile_size, + const struct mmid_row_mapping *matrix_rows, + int cur_a, + int mapping_stride, + int m_start, + int m_end, + int n_threads) { + struct htp_thread_trace * tr = &ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + + const int cne1 = m; + const int m_core = m_end - m_start; + const int m_core_padded = hex_align_up(m_core > 0 ? m_core : 1, 32); + + if (k % 32 != 0 || n % 32 != 0) { return -1; } + if (!hex_is_aligned(dst, VLEN) || !hex_is_aligned(activation, VLEN)) { return -1; } + + size_t row_stride = htp_tensor_get_row_size(weight_type, k); + if (row_stride == 0) { + return -1; + } - HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16); - HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16); + const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(weight_type); + const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); - hvx_vec_store_u(y_d + 0, 4, vd01_hf); - hvx_vec_store_u(y_d + 4, 4, vd23_hf); + const size_t vec_dot_size = k * sizeof(__fp16); + const size_t vtcm_budget = ctx->vtcm_size; + size_t vtcm_used = 0; - // Divide input by the scale - HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf); - HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf); - vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf)); - vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf)); + const bool is_tiled = (tile_size > 0); + tiled_dequant_task_fn_t dequant_worker_fn = hmx_get_dequant_worker_fn(weight_type, is_tiled); - // Convert to int8 - HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); - HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); - HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16); + const uint32_t dma_dst_stride = is_tiled ? aligned_tile_size : row_stride; + const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; + const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride; - *(HVX_Vector *) y_q = vx_i8; -} + const size_t qweight_row_stride = is_tiled ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0; + const size_t weight_row_stride = is_tiled ? qweight_row_stride : row_stride; -static inline void quantize_block_f32_q8x4(float * restrict x, uint8_t * restrict y_q, uint8_t * restrict y_d) { - assert((unsigned long) x % 128 == 0); - assert((unsigned long) y_q % 128 == 0); + size_t size_per_n = 0, size_per_m = 0, size_per_mn = 0; + htp_mm_hmx_get_2d_chunk_costs(weight_type, k, /*pipeline=*/false, aligned_tile_size, + &size_per_n, &size_per_m, &size_per_mn); - HVX_Vector * vx = (HVX_Vector *) x; - - // Load and convert into QF32 - HVX_Vector zero = Q6_V_vzero(); - HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero); // 32 elements - HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero); // 32 elements - HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero); // 32 elements - HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero); // 32 elements + const size_t overhead = htp_mm_hmx_get_2d_overhead(/*pipeline=*/false, /*is_matmul_id=*/true); + size_t m_chunk_n_rows = 0, n_chunk_n_cols = 0; + if (htp_mm_hmx_compute_chunks(vtcm_budget, overhead, size_per_n, size_per_m, size_per_mn, + m_core_padded, n, + /*m_block_cost=*/(size_t) n * HTP_MM_HMX_COST_W_DEQUANT, + /*n_block_cost=*/(size_t) m_core_padded * HTP_MM_HMX_COST_A_CONVERT, &m_chunk_n_rows, &n_chunk_n_cols, &vtcm_used)) { + FARF(ERROR, "hmx-mm-id-2d: VTCM too small : m %d k %d n %d budget %zu", m_core_padded, k, n, vtcm_budget); + return -1; + } - // Convert into fp16 - HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf))); - HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf))); + const size_t weight_area_size = hex_align_up(n_chunk_n_cols * weight_row_stride, HTP_MM_HMX_TILE_SIZE); + const size_t act_area_size = hex_align_up(m_chunk_n_rows * vec_dot_size, HTP_MM_HMX_TILE_SIZE); + const size_t output_area_size = hex_align_up(m_chunk_n_rows * n_chunk_n_cols * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE); - // Compute max and scale - HVX_Vector vmax_hf = hvx_vec_reduce_max_f16(hvx_vec_abs_f16(vx01_hf)); - vmax_hf = hvx_vec_reduce_max2_f16(hvx_vec_abs_f16(vx23_hf), vmax_hf); // replicated over all lanes + size_t scratch0_size = hex_align_up(n_chunk_n_cols * vec_dot_size, HTP_MM_HMX_TILE_SIZE); - HVX_Vector vd_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0 - HVX_Vector vd_hf = Q6_Vhf_equals_Vqf16(vd_qf16); + uint8_t *vtcm_ptr = (uint8_t *) ctx->vtcm_base; + __fp16 *vtcm_weight = weight_area_size ? (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, weight_area_size) : NULL; + __fp16 *vtcm_f16_act = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, act_area_size); + __fp16 *vtcm_output = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, output_area_size); + void *vtcm_scratch0 = vtcm_seq_alloc(&vtcm_ptr, scratch0_size); + __fp16 *vtcm_scales = (__fp16 *) vtcm_seq_alloc(&vtcm_ptr, 256); - *(HVX_UVector *) y_d = vd_hf; + vtcm_used = vtcm_ptr - (uint8_t *) ctx->vtcm_base; + if (vtcm_used > vtcm_budget) { + FARF(ERROR, "hmx-mm-id-2d: VTCM overflow: used %zu budget %zu", vtcm_used, vtcm_budget); + return -1; + } - // Divide input by the scale - HVX_Vector vd_inv_hf = hvx_vec_inverse_f16(vd_hf); - vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd_inv_hf)); - vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd_inv_hf)); + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); - // Convert to int8 - HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf); - HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf); - HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - *(HVX_Vector *) y_q = vx_i8; -} + hmx_matmul_job_t job; + void *vtcm_raw_bufs[2] = { vtcm_weight, dequant_worker_fn ? vtcm_weight : vtcm_scratch0 }; + void *vtcm_weight_bufs[2] = { dequant_worker_fn ? vtcm_scratch0 : vtcm_weight, vtcm_scratch0 }; + const int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols); -// Overrides input x -static void quantize_row_f32_q8x4x2(float * restrict x, uint8_t * restrict y, uint32_t k) { - assert(k % 32 == 0); - const uint32_t qk = QK_Q8_0x4x2; - const uint32_t nb = (k + qk - 1) / qk; + for (size_t mr = (size_t) m_start; mr < (size_t) m_end; mr += m_chunk_n_rows) { + const size_t n_rows = hex_smin((size_t) m_end - mr, m_chunk_n_rows); + const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS); - const uint32_t qrow_size = k; // int8 + transfer_activation_chunk_gathered_threaded( + ctx, vtcm_f16_act, activation, (int) mr, (int) n_rows, k, + matrix_rows, cur_a, mapping_stride, ne11, act_nb1, act_nb2, cne1, n_threads, k_valid); - const uint32_t dblk_size = 8 * 2; // 8x __fp16 - const uint32_t qblk_size = QK_Q8_0x4x2; // int8 + // A0: Pre-fetch the first weight chunk (nc = 0) + if (n > 0) { + const size_t n_cols = hex_smin((size_t) n, n_chunk_n_cols); + const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), + dma_dst_stride, dma_src_stride, dma_width_bytes, height); + } - uint8_t * restrict y_q = (y + 0); // quants first - uint8_t * restrict y_d = (y + qrow_size); // then scales + for (int i = 0; i < n_chunk_cnt; ++i) { + const size_t nc = (size_t) i * n_chunk_n_cols; + const size_t n_cols = hex_smin((size_t) n - nc, n_chunk_n_cols); + const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); + + // A: Wait for weight DMA + void * curr_raw = (void *) dma_queue_pop(weight_dma).dst; + + // B: Weight Dequantize (Threaded) + dequantize_tiled_weight_chunk_to_fp16_tiles( + ctx, vtcm_weight_bufs[i % 2], curr_raw, + n_cols, k, row_stride, weight_type, + n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads + ); + + // Start weight DMA for the next chunk early + const size_t nc_next = nc + n_chunk_n_cols; + if (nc_next < (size_t) n) { + const size_t n_cols_next = hex_smin((size_t) n - nc_next, n_chunk_n_cols); + const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next; + dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), + dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); + } - // Temp scales override input since we're working off of the aligned temp buffer in VTCM - uint8_t * restrict t_d = (uint8_t *) x; + // C: HMX Compute (Queue-based) + hmx_matmul_job_init(&job, vtcm_output, vtcm_f16_act, (__fp16 *) vtcm_weight_bufs[i % 2], vtcm_scales, n_row_tiles, n_col_tiles, k / HTP_MM_HMX_TILE_N_ROWS); + hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_matmul_worker_fn, &job)); + hmx_queue_pop(ctx->hmx_queue); - for (uint32_t i = 0; i < nb; i++) { -#if FP32_QUANTIZE_GROUP_SIZE == 32 - quantize_block_f32_q8x1(x + (i*2 + 0) * qk/2, y_q + (i*2 + 0) * qblk_size/2, t_d + (i*2 + 0) * dblk_size/2); - quantize_block_f32_q8x1(x + (i*2 + 1) * qk/2, y_q + (i*2 + 1) * qblk_size/2, t_d + (i*2 + 1) * dblk_size/2); -#elif FP32_QUANTIZE_GROUP_SIZE == 64 - quantize_block_f32_q8x2(x + (i*2 + 0) * qk/2, y_q + (i*2 + 0) * qblk_size/2, t_d + (i*2 + 0) * dblk_size/2); - quantize_block_f32_q8x2(x + (i*2 + 1) * qk/2, y_q + (i*2 + 1) * qblk_size/2, t_d + (i*2 + 1) * dblk_size/2); -#elif FP32_QUANTIZE_GROUP_SIZE == 128 - quantize_block_f32_q8x4(x + (i*2 + 0) * qk/2, y_q + (i*2 + 0) * qblk_size/2, t_d + (i*2 + 0) * dblk_size/2); - quantize_block_f32_q8x4(x + (i*2 + 1) * qk/2, y_q + (i*2 + 1) * qblk_size/2, t_d + (i*2 + 1) * dblk_size/2); -#else -#error "FP32_QUANTIZE_GROUP_SIZE must be 32, 64, or 128" -#endif + // D: Output Store + transfer_output_chunk_scattered_threaded( + ctx, dst + nc, vtcm_output, (int) mr, (int) n_rows, (int) n_cols, + matrix_rows, cur_a, mapping_stride, dst_nb1, dst_nb2, cne1, n_threads); + } } - // now copy the scales into final location - hvx_copy_f16_ua(y_d, t_d, nb * 8); + return 0; } -static void quantize_f32_q8x4x2(unsigned int nth, unsigned int ith, void * data) { - struct htp_matmul_context * mmctx = data; - struct htp_ops_context * octx = mmctx->octx; - - const struct htp_tensor * src = octx->src[1]; - uint8_t * restrict dst = octx->src1_spad.data; - struct htp_spad * spad = &octx->src0_spad; - uint32_t nrows_per_thread = mmctx->src1_nrows_per_thread; +// --- Dispatchers and Public Entry Points --- - uint64_t t1 = HAP_perf_get_qtimer_count(); +static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_kernel_params * kparams) { + htp_matmul_tensors_preamble; - const uint32_t ne0 = src->ne[0]; - const uint32_t ne1 = src->ne[1]; - const uint32_t ne2 = src->ne[2]; - const uint32_t ne3 = src->ne[3]; + int k = (int) src0->ne[0]; + int n = (int) src0->ne[1]; + const int m_total = (int) act->ne[1]; + const uint32_t act_elem_size = (act->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + const int act_stride = (int) (act->nb[1] / act_elem_size); + + int m_start = 0; + int m_rows = m_total; + if (octx->ctx->mdev.count > 1) { + bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)) && + htp_tensor_can_row_partition(act, act_elem_size); + if (src2 && src2->ne[1] > 1 && !htp_tensor_can_row_partition(src2, sizeof(float))) { + can_split = false; + } + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_total, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_rows = (int) range.count; + } - const uint32_t nrows = ne1 * ne2 * ne3; // total n_rows + if (m_rows == 0) { + return HTP_STATUS_OK; + } - const uint32_t ir_first = nrows_per_thread * ith; // first row - const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); // last row + dma_addr_t src2_addr = 0; + size_t src2_bytes = 0; + uint32_t src2_stride = 0; + size_t src2_nb2 = 0; + size_t src2_nb3 = 0; + if (src2) { + src2_stride = (src2->ne[1] == 1) ? 0 : (uint32_t) (src2->nb[1] / sizeof(float)); + src2_addr = src2->data + m_start * src2_stride * sizeof(float); + src2_bytes = (size_t) kparams->vtcm_bias_size; + src2_nb2 = (src2->ne[2] == 1) ? 0 : src2->nb[2]; + src2_nb3 = (src2->ne[3] == 1) ? 0 : src2->nb[3]; + } - const size_t src_row_size = src->nb[1]; - const size_t dst_row_size = q8x4x2_row_size(ne0); + const int dst_stride = (int)(dst->nb[1] / sizeof(float)); + float * dst_ptr = (float *) dst->data + m_start * dst_stride; + // byte offset, act_stride is in activation elements (F16 or F32) + const dma_addr_t act_addr = act->data + (size_t) m_start * act_stride * act_elem_size; - uint8_t * restrict src_data = (uint8_t *) src->data + (src_row_size * ir_first); - uint8_t * restrict dst_data = (uint8_t *) dst + (dst_row_size * ir_first); - uint8_t * restrict tmp_data = (uint8_t *) spad->data + (spad->size_per_thread * ith); + int ret = -1; + const int n_threads = kparams->n_threads; + if (kparams->kernel_type == HTP_MM_KERNEL_HMX_BATCHED) { + hmx_mm_batched_params_t batch_params = { + .dst = dst_ptr, + .bias_addr = src2_addr, + .bias_bytes = src2_bytes, + .act_dma_addr = act_addr, + .weight = src0->data, + .weight_dma = octx->ctx->dma[0], + .m = m_rows, + .k = k, + .n = n, + .act_stride = act_stride, + .act_elem_size = act_elem_size, + .weight_stride = (int) src0->nb[1], + .weight_type = (int) src0->type, + .dst_stride = dst_stride, + .dst_cols = (int) dst->ne[0], + .bias_stride = src2_stride, + .tile_size = kparams->tile_size, + .aligned_tile_size = kparams->aligned_tile_size, + .ne02 = ne02, + .ne03 = ne03, + .ne12 = ne12, + .ne13 = ne13, + .src0_nb2 = src0->nb[2], + .src0_nb3 = src0->nb[3], + .act_nb2 = act->nb[2], + .act_nb3 = act->nb[3], + .dst_nb2 = dst->nb[2], + .dst_nb3 = dst->nb[3], + .bias_nb2 = src2_nb2, + .bias_nb3 = src2_nb3, + .r2 = (ne02 > 0) ? (ne12 / ne02) : 1, + .r3 = (ne03 > 0) ? (ne13 / ne03) : 1, + .div_r2 = kparams->div_r2, + .div_r3 = kparams->div_r3, + }; + ret = hmx_mm_batched(octx->ctx, &batch_params, + kparams->m_chunk, kparams->n_chunk, + kparams->pipeline, n_threads, + kparams->n_act_threads, + &kparams->div_n_act_threads, + &kparams->div_ne00_padded, + kparams->vtcm_size); + } else { + ret = hmx_mm_2d_f32( + octx->ctx, octx->ctx->dma[0], dst_ptr, src2_addr, src2_bytes, + act_addr, src0->data, + m_rows, k, n, act_stride, act_elem_size, (int) src0->nb[1], (int) src0->type, (int) act->ne[0], + dst_stride, src2_stride, (int)dst->ne[0], + kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads, + kparams->n_act_threads, + &kparams->div_n_act_threads, + &kparams->div_ne00_padded, + kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size + ); + } - const size_t src_row_size_padded = hex_round_up(src_row_size, QK_Q8_0x4x2 * sizeof(float)); - memset(tmp_data, 0, src_row_size_padded); // zero-out temp row data for padding + if (ret != 0) { + FARF(ERROR, "HMX matmul failed (ret=%d)\n", ret); + return HTP_STATUS_INTERNAL_ERR; + } + return HTP_STATUS_OK; +} - for (uint32_t i = ir_first; i < ir_last; ++i) { - hex_l2fetch(src_data, src_row_size, src_row_size, 2); - hvx_copy_f32_aa(tmp_data, src_data, ne0); +static int op_matmul_impl(struct htp_ops_context * octx) { + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - // FARF(HIGH, "quantize-q8x4-row: %u\n", i); - quantize_row_f32_q8x4x2((float *) tmp_data, dst_data, ne0); - dst_data += dst_row_size; - src_data += src_row_size; + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; } - uint64_t t2 = HAP_perf_get_qtimer_count(); + if (kparams->n_hmx) { + return hmx_mm_op_matmul(octx, kparams); + } - FARF(HIGH, "quantize-f32-q8x4: %u/%u : n-rows %u (%u:%u) row-size %u -> %u usec %u\n", ith, nth, nrows, ir_first, - ir_last, src_row_size, dst_row_size, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + return hvx_mm_matmul(octx); } -static void quantize_f32_f16(unsigned int nth, unsigned int ith, void * data) { - struct htp_matmul_context * mmctx = data; - struct htp_ops_context * octx = mmctx->octx; - - const struct htp_tensor * src = octx->src[1]; - uint8_t * restrict dst = octx->src1_spad.data; - uint32_t nrows_per_thread = mmctx->src1_nrows_per_thread; - uint32_t dst_stride = octx->src1_spad.stride; +int op_matmul(struct htp_ops_context * octx) { + return op_matmul_impl(octx); +} - uint64_t t1 = HAP_perf_get_qtimer_count(); +static int hmx_mm_op_matmul_id( + struct htp_ops_context * octx, + struct htp_mm_context * mmctx +) { + const uint32_t * matrix_row_counts = mmctx->matrix_row_counts; + const struct mmid_row_mapping * matrix_rows = mmctx->matrix_rows; + htp_matmul_tensors_preamble; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const int n_ids = octx->src[2]->ne[0]; + const int n_as = ne02; - const uint32_t ne0 = src->ne[0]; - const uint32_t ne1 = src->ne[1]; - const uint32_t ne2 = src->ne[2]; - const uint32_t ne3 = src->ne[3]; + const bool mdev_split = (octx->ctx->mdev.count > 1) && htp_tensor_can_row_partition(dst, sizeof(float)); + if (octx->ctx->mdev.count > 1 && !mdev_split && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + uint32_t n_active = 0; + if (mdev_split) { + for (uint32_t a = 0; a < (uint32_t) n_as; ++a) { + if (matrix_row_counts[a] > 0) n_active++; + } + } + const bool expert_split = mdev_split && (n_active >= octx->ctx->mdev.count); - const uint32_t nrows = ne1 * ne2 * ne3; // total n_rows + uint32_t target_dev = 0; + for (uint32_t cur_a = 0; cur_a < (uint32_t) n_as; ++cur_a) { + const int32_t cne1 = matrix_row_counts[cur_a]; + if (cne1 == 0) continue; + + const int m_padded = hex_align_up(cne1, 32); + int m_start = 0, m_end = m_padded; + if (expert_split) { + const bool my_expert = (target_dev == octx->ctx->mdev.idx); + if (++target_dev == octx->ctx->mdev.count) { + target_dev = 0; + } + if (!my_expert) continue; + } else if (mdev_split) { + const bool can_split = (uint32_t) cne1 >= octx->ctx->mdev.count; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_end = (int) (range.start + range.count); + } + if (m_start >= m_end) continue; + + int ret = hmx_mm_id_2d_f32(octx->ctx, octx->ctx->dma[0], (float*) dst->data, (float*) act->data, + src0->data + cur_a * nb02, + cne1, ne00, ne01, + ne10, + ne11, + nb11, nb12, + nb1, nb2, + (int) src0->nb[1], (int) src0->type, + kparams->tile_size, kparams->aligned_tile_size, + matrix_rows, cur_a, mmctx->mapping_stride, + m_start, m_end, (int) octx->n_threads); + if (ret != 0) { + FARF(ERROR, "HMX matmul failed for expert %u, error %d\n", cur_a, ret); + return HTP_STATUS_NO_SUPPORT; + } + } - const uint32_t ir_first = nrows_per_thread * ith; // first row - const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); // last row + return HTP_STATUS_OK; +} - const size_t src_row_size = ne0 * sizeof(float); - const size_t src_stride = src->nb[1]; +static int hvx_mm_matmul_id( + struct htp_ops_context * octx, + struct htp_mm_context * mmctx, + work_queue_func_t hvx_mmid_task_func +) { + htp_matmul_tensors_preamble; + const uint32_t src0_row_size_padded = mmctx->src0_row_size_padded; + const uint32_t act_nrows = mmctx->act_nrows; - uint8_t * restrict src_data = (uint8_t *) src->data + (src_stride * ir_first); - uint8_t * restrict dst_data = (uint8_t *) dst + (dst_stride * ir_first); + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - for (uint32_t i = ir_first; i < ir_last; ++i) { - hex_l2fetch(src_data, src_row_size, src_stride, 2); - hvx_copy_f16_f32_au(dst_data, src_data, ne0); + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const struct htp_tensor * restrict ids = octx->src[2]; + const size_t src0_row_size = nb01; - dst_data += dst_stride; - src_data += src_stride; + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (ne10 + qk - 1) / qk; + const uint32_t total_nb = act_nrows * nb; + + work_queue_func_t quant_task_func; + uint32_t n_quant_tasks = 1; + if (act_nrows < octx->n_threads) { + n_quant_tasks = MIN(total_nb, octx->n_threads); + quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); + for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { + uint32_t ib_first = (total_nb * ith) / n_quant_tasks; + uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; + mmctx->quant_ib_first[ith] = ib_first; + mmctx->quant_ib_last[ith] = ib_last; + mmctx->quant_r[ith] = ib_first / nb; + mmctx->quant_c[ith] = ib_first % nb; + } + } else { + n_quant_tasks = MIN(act_nrows, octx->n_threads); + quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } + size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, ne10); - uint64_t t2 = HAP_perf_get_qtimer_count(); + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, act_nrows, octx->n_threads, + 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false); - FARF(HIGH, "quantize-f32-f16: %u/%u : n-rows %u (%u:%u) row-size %u (%u) -> %u usec %u\n", ith, nth, nrows, ir_first, - ir_last, src_row_size, src_stride, dst_stride, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); -} + const size_t vtcm_size = L.total_bytes; -// TODO just a plain copy that should be done via the DMA during the Op setup -static void quantize_f16_f16(unsigned int nth, unsigned int ith, void * data) { - struct htp_matmul_context * mmctx = data; - struct htp_ops_context * octx = mmctx->octx; + FARF(HIGH, "matmul-id-%s : src0-spad-size %zu act-spad-size %zu bias-spad-size 0 dst-spad-size %zu (%zu)\n", mmctx->type, + L.src0_bytes, L.act_bytes, L.dst_bytes, vtcm_size); - const struct htp_tensor * src = octx->src[1]; - uint8_t * restrict dst = octx->src1_spad.data; - uint32_t nrows_per_thread = mmctx->src1_nrows_per_thread; - uint32_t dst_stride = octx->src1_spad.stride; + FARF(HIGH, "matmul-id-%s : %ux%ux%ux%u * %ux%ux%ux%u (%ux%ux%ux%u) -> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\n", mmctx->type, + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], act->ne[0], act->ne[1], act->ne[2], act->ne[3], + ids->ne[0], ids->ne[1], ids->ne[2], ids->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], src0->data, + act->data, dst->data); - uint64_t t1 = HAP_perf_get_qtimer_count(); + // Make sure the reserved vtcm size is sufficient + if (octx->ctx->vtcm_size < vtcm_size) { + FARF(ERROR, "matmul-id-%s : current VTCM reservation %zu is too small, needed %zu\n", mmctx->type, octx->ctx->vtcm_size, vtcm_size); + return HTP_STATUS_VTCM_TOO_SMALL; + } - const uint32_t ne0 = src->ne[0]; - const uint32_t ne1 = src->ne[1]; - const uint32_t ne2 = src->ne[2]; - const uint32_t ne3 = src->ne[3]; + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + mmctx->vtcm_act = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act); + mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + mmctx->vtcm_bias = NULL; + mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + mmctx->vtcm_act_raw = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act_raw); - const uint32_t nrows = ne1 * ne2 * ne3; // total n_rows + mmctx->vtcm_src0_stride = src0_row_size_padded; + mmctx->vtcm_act_stride = act_row_size; + mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(ne10); - const uint32_t ir_first = nrows_per_thread * ith; // first row - const uint32_t ir_last = MIN(ir_first + nrows_per_thread, nrows); // last row + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); + mmctx->vtcm_act_size_per_thread = L.act_bytes; + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); - const size_t src_row_size = ne0 * sizeof(float); - const size_t src_stride = src->nb[1]; + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = act_nrows; - uint8_t * restrict src_data = (uint8_t *) src->data + (src_stride * ir_first); - uint8_t * restrict dst_data = (uint8_t *) dst + (dst_stride * ir_first); + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - for (uint32_t i = ir_first; i < ir_last; ++i) { - hex_l2fetch(src_data, src_row_size, src_stride, 2); - hvx_copy_f16_au(dst_data, src_data, ne0); + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - dst_data += dst_stride; - src_data += src_stride; - } + mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_tasks = n_quant_tasks; + work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); - uint64_t t2 = HAP_perf_get_qtimer_count(); + work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads); - FARF(HIGH, "quantize-f16-f16: %u/%u : n-rows %u (%u:%u) row-size %u (%u) -> %u usec %u\n", ith, nth, nrows, ir_first, - ir_last, src_row_size, src_stride, dst_stride, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + return HTP_STATUS_OK; } +static int hmx_mm_op_matmul_id_nx( + struct htp_ops_context * octx, + struct htp_mm_context * mmctx +) { + const uint32_t * matrix_row_counts = mmctx->matrix_row_counts; + const struct mmid_row_mapping * matrix_rows = mmctx->matrix_rows; + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_weights = kparams->n_weights; + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict act = octx->src[n_weights]; + const int n_as = src0->ne[2]; + + bool mdev_split = (octx->ctx->mdev.count > 1); + for (uint32_t p = 0; p < n_weights && mdev_split; ++p) { + const struct htp_tensor * restrict dst = octx->dsts[p]; + mdev_split = htp_tensor_can_row_partition(dst, sizeof(float)); + } + if (octx->ctx->mdev.count > 1 && !mdev_split && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + uint32_t n_active = 0; + if (mdev_split) { + for (uint32_t a = 0; a < (uint32_t) n_as; ++a) { + if (matrix_row_counts[a] > 0) n_active++; + } + } + const bool expert_split = mdev_split && (n_active >= octx->ctx->mdev.count); -static inline bool htp_is_permuted(const struct htp_tensor * t) { - return t->nb[0] > t->nb[1] || t->nb[1] > t->nb[2] || t->nb[2] > t->nb[3]; -} - -static int htp_mminit_vec_dot(struct htp_matmul_context * mmctx, enum htp_data_type type) { - switch (type) { - case HTP_TYPE_Q4_0: - mmctx->type = "q4x4x2-f32"; - mmctx->vec_dot_1x1 = vec_dot_q4x4x2_q8x4x2_1x1; - mmctx->vec_dot_2x1 = vec_dot_q4x4x2_q8x4x2_2x1; - mmctx->vec_dot_2x2 = vec_dot_q4x4x2_q8x4x2_2x2; - return 0; - case HTP_TYPE_Q8_0: - mmctx->type = "q8x4x2-f32"; - mmctx->vec_dot_1x1 = vec_dot_q8x4x2_q8x4x2_1x1; - mmctx->vec_dot_2x1 = vec_dot_q8x4x2_q8x4x2_2x1; - mmctx->vec_dot_2x2 = vec_dot_q8x4x2_q8x4x2_2x2; - return 0; - case HTP_TYPE_IQ4_NL: - mmctx->type = "iq4nlx4x2-f32"; - mmctx->vec_dot_1x1 = vec_dot_iq4nlx4x2_q8x4x2_1x1; - mmctx->vec_dot_2x1 = vec_dot_iq4nlx4x2_q8x4x2_2x1; - mmctx->vec_dot_2x2 = vec_dot_iq4nlx4x2_q8x4x2_2x2; - return 0; - case HTP_TYPE_MXFP4: - mmctx->type = "mxfp4x4x2-f32"; - mmctx->vec_dot_1x1 = vec_dot_mxfp4x4x2_q8x4x2_1x1; - mmctx->vec_dot_2x1 = vec_dot_mxfp4x4x2_q8x4x2_2x1; - mmctx->vec_dot_2x2 = vec_dot_mxfp4x4x2_q8x4x2_2x2; - return 0; - default: - return -1; + uint32_t target_dev = 0; + for (uint32_t cur_a = 0; cur_a < (uint32_t) n_as; ++cur_a) { + const int32_t cne1 = matrix_row_counts[cur_a]; + if (cne1 == 0) continue; + + const int m_padded = hex_align_up(cne1, 32); + int m_start = 0, m_end = m_padded; + if (expert_split) { + const bool my_expert = (target_dev == octx->ctx->mdev.idx); + if (++target_dev == octx->ctx->mdev.count) { + target_dev = 0; + } + if (!my_expert) continue; + } else if (mdev_split) { + const bool can_split = (uint32_t) cne1 >= octx->ctx->mdev.count; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + m_start = (int) range.start; + m_end = (int) (range.start + range.count); + } + if (m_start >= m_end) continue; + + for (uint32_t p = 0; p < n_weights; ++p) { + const struct htp_tensor * restrict src_w = octx->src[p]; + const struct htp_tensor * restrict dst = octx->dsts[p]; + if (!src_w || !dst) continue; + + int ret = hmx_mm_id_2d_f32(octx->ctx, octx->ctx->dma[0], (float*) dst->data, (float*) act->data, + src_w->data + cur_a * src_w->nb[2], + cne1, src_w->ne[0], src_w->ne[1], + act->ne[0], + act->ne[1], + act->nb[1], act->nb[2], + dst->nb[1], dst->nb[2], + (int) src_w->nb[1], (int) src_w->type, + kparams->tile_size, kparams->aligned_tile_size, + matrix_rows, cur_a, mmctx->mapping_stride, + m_start, m_end, (int) octx->n_threads); + if (ret != 0) { + FARF(ERROR, "HMX matmul ID NX failed for expert %u weight %u, error %d\n", cur_a, p, ret); + return HTP_STATUS_NO_SUPPORT; + } + } } -} -static void htp_mminit_spad(struct htp_ops_context * octx, - size_t dst_row_size, - size_t src0_row_size_padded, - size_t src1_row_size, - uint32_t src1_nrows, - size_t src2_spad_size_per_thread) { - octx->dst_spad.size_per_thread = hex_round_up(MM_SPAD_DST_NROWS * dst_row_size, 256); - octx->src0_spad.size_per_thread = hex_round_up(MM_SPAD_SRC0_NROWS * src0_row_size_padded, 256); - octx->src1_spad.size_per_thread = hex_round_up(src1_row_size * src1_nrows, 256); + return HTP_STATUS_OK; +} - if (src2_spad_size_per_thread > 0) { - octx->src2_spad.size_per_thread = src2_spad_size_per_thread; - octx->src2_spad.size = octx->src2_spad.size_per_thread; +static int hvx_mm_matmul_id_nx( + struct htp_ops_context * octx, + struct htp_mm_context * mmctx, + work_queue_func_t hvx_mmid_task_func +) { + const uint32_t src0_row_size_padded = mmctx->src0_row_size_padded; + const uint32_t act_nrows = mmctx->act_nrows; + + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); + + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + const uint32_t n_weights = kparams->n_weights; + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict act = octx->src[n_weights]; + const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; + const size_t src0_row_size = src0->nb[1]; + + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (act->ne[0] + qk - 1) / qk; + const uint32_t total_nb = act_nrows * nb; + + work_queue_func_t quant_task_func; + uint32_t n_quant_tasks = 1; + if (act_nrows < octx->n_threads) { + n_quant_tasks = MIN(total_nb, octx->n_threads); + quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); + for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { + uint32_t ib_first = (total_nb * ith) / n_quant_tasks; + uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; + mmctx->quant_ib_first[ith] = ib_first; + mmctx->quant_ib_last[ith] = ib_last; + mmctx->quant_r[ith] = ib_first / nb; + mmctx->quant_c[ith] = ib_first % nb; + } + } else { + n_quant_tasks = MIN(act_nrows, octx->n_threads); + quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } + size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]); - // src0 spad is also used in dynamic quantizer to store padded src1 rows - size_t src1_row_size_padded = hex_round_up(src1_row_size, QK_Q8_0x4x2 * sizeof(float)); - if (octx->src0_spad.size_per_thread < src1_row_size_padded) { - octx->src0_spad.size_per_thread = src1_row_size_padded; - } + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads, + 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, true, false); - octx->src1_spad.size = octx->src1_spad.size_per_thread; - octx->src0_spad.size = octx->src0_spad.size_per_thread * octx->n_threads; - octx->dst_spad.size = octx->dst_spad.size_per_thread * octx->n_threads; -} + const size_t vtcm_size = L.total_bytes; -static int op_matmul_hvx(struct htp_ops_context * octx) { - htp_matmul_tensors_preamble; + if (octx->ctx->vtcm_size < vtcm_size) { + FARF(ERROR, "matmul-id-nx: current VTCM reservation %zu is too small, needed %zu\n", + octx->ctx->vtcm_size, vtcm_size); + return HTP_STATUS_VTCM_TOO_SMALL; + } - struct htp_matmul_context mmctx_struct = {0}; - struct htp_matmul_context * mmctx = &mmctx_struct; - mmctx->octx = octx; + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + mmctx->vtcm_act = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act); + mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + mmctx->vtcm_act_raw = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act_raw); - const uint32_t src0_nrows = ne01 * ne02 * ne03; - const uint32_t src1_nrows = ne11 * ne12 * ne13; + mmctx->vtcm_src0_stride = 0; + mmctx->vtcm_act_stride = act_row_size; + mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]); - // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = (src0_nrows + octx->n_threads - 1) / octx->n_threads; - mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); + mmctx->vtcm_act_size_per_thread = L.act_bytes; + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); - const size_t src0_row_size = nb01; - const size_t dst_row_size = nb1; - size_t src1_row_size = nb11; + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = act_nrows; - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - size_t src1_row_size_padded; + FARF(HIGH, "matmul-id-nx: src0 %d:%d:%d type %s nrows %u, act %d:%d:%d nrows %u, vtcm %zu/%zu, threads %d\n", + src0->ne[0], src0->ne[1], src0->ne[2], mmctx->type, src0->ne[1], + act->ne[0], act->ne[1], act->ne[2], act_nrows, + L.total_bytes, octx->ctx->vtcm_size, octx->n_threads); - worker_callback_t quant_job_func; - worker_callback_t matmul_job_func = src1_nrows > 1 ? matmul_2d : matvec_2d; + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - bool need_quant = true; + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - if (src0->type == HTP_TYPE_F16) { - // Try optimized f16-f16 path first (src1 in VTCM) - const size_t f16_src1_row_size = hex_round_up(ne10 * 2, 128); - const size_t f16_src1_spad_size = hex_round_up(f16_src1_row_size * src1_nrows, 256); - const size_t f16_src0_spad_size = hex_round_up(MM_SPAD_SRC0_NROWS * src0_row_size_padded, 256) * octx->n_threads; - const size_t f16_dst_spad_size = hex_round_up(MM_SPAD_DST_NROWS * dst_row_size, 256) * octx->n_threads; - - const size_t f16_total_size = f16_src1_spad_size + f16_src0_spad_size + f16_dst_spad_size; - - // Default matmul implementation does not support multi-batch src0 (N-vs-N broadcasting). - // It only supports 1-vs-N broadcasting (src0 is 2D) or standard 2D matmul. - const bool is_batched = (ne02 > 1) || (ne03 > 1); - const bool is_permuted = htp_is_permuted(octx->src[0]) || htp_is_permuted(octx->src[1]); - - if (!is_batched && !is_permuted && f16_total_size <= octx->ctx->vtcm_size) { - // Optimized path - quant_job_func = (src1->type == HTP_TYPE_F32) ? quantize_f32_f16 : quantize_f16_f16; - mmctx->type = "f16-f16"; - mmctx->vec_dot_1x1 = vec_dot_f16_f16_aa_1x1; - mmctx->vec_dot_2x1 = vec_dot_f16_f16_aa_2x1; - mmctx->vec_dot_2x2 = vec_dot_f16_f16_aa_2x2; - - src1_row_size = f16_src1_row_size; // row size post quantization - - octx->dst_spad.size_per_thread = hex_round_up(MM_SPAD_DST_NROWS * dst_row_size, 256); - octx->src0_spad.size_per_thread = hex_round_up(MM_SPAD_SRC0_NROWS * src0_row_size_padded, 256); - octx->src1_spad.size_per_thread = hex_round_up(src1_row_size * src1_nrows, 256); - - octx->src1_spad.size = octx->src1_spad.size_per_thread; - octx->src0_spad.size = octx->src0_spad.size_per_thread * octx->n_threads; - octx->dst_spad.size = octx->dst_spad.size_per_thread * octx->n_threads; - } else { - // Fallback to f16/f32 (DDR) if src1 doesn't fit in VTCM or broadcasting is required - quant_job_func = NULL; - if (src1->type == HTP_TYPE_F32) { - mmctx->type = "f16-f32"; - mmctx->vec_dot_1x1 = vec_dot_f16_f32_uu_1x1; - matmul_job_func = matmul_4d; - } else { - mmctx->type = "f16-f16"; - mmctx->vec_dot_1x1 = vec_dot_f16_f16_uu_1x1; - matmul_job_func = matmul_4d; - } + mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_tasks = n_quant_tasks; + work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); - src1_row_size = nb11; // original row size in DDR + work_queue_run(octx->ctx->work_queue, hvx_mmid_task_func, mmctx, octx->n_threads); - octx->dst_spad.size_per_thread = hex_round_up(MM_SPAD_DST_NROWS * dst_row_size, 256); - octx->src0_spad.size_per_thread = hex_round_up(MM_SPAD_SRC0_NROWS * src0_row_size, 256); - octx->src1_spad.size_per_thread = hex_round_up(MM_SPAD_SRC1_NROWS * src1_row_size, 256); + return HTP_STATUS_OK; +} - octx->src0_spad.size = octx->src0_spad.size_per_thread * octx->n_threads; - octx->src1_spad.size = octx->src1_spad.size_per_thread * octx->n_threads; - octx->dst_spad.size = octx->dst_spad.size_per_thread * octx->n_threads; +static __attribute__((noinline)) uint32_t scan_expert_ids_n( + const struct htp_tensor * ids, + const uint32_t n_ids, + uint32_t n_as, + uint32_t * counts, + struct mmid_row_mapping * matrix_rows, + uint32_t mapping_stride +) { + const size_t ids_nb1 = ids->nb[1]; + const uint8_t * ids_data = (const uint8_t *) ids->data; + uint32_t n_skipped = 0; + + for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { + const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1); + for (uint32_t id = 0; id < n_ids; ++id) { + const int32_t i02 = row_ptr[id]; + if (i02 < 0) { + n_skipped++; + continue; + } + assert(i02 >= 0 && i02 < (int32_t) n_as); - // Init fastdiv for matmul_4d (supports broadcasting) - mmctx->mm_div_ne12_ne1 = init_fastdiv_values(src1->ne[2] * dst->ne[1]); - mmctx->mm_div_ne1 = init_fastdiv_values(dst->ne[1]); - mmctx->mm_div_r2 = init_fastdiv_values(src1->ne[2] / src0->ne[2]); - mmctx->mm_div_r3 = init_fastdiv_values(src1->ne[3] / src0->ne[3]); + if (matrix_rows) { + matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 }; + } + counts[i02] += 1; + } + } + return n_skipped; +} - need_quant = false; +static __attribute__((noinline)) uint32_t scan_expert_ids( + const struct htp_tensor * ids, + uint32_t n_ids, + uint32_t n_as, + uint32_t * counts, + struct mmid_row_mapping * matrix_rows, + uint32_t mapping_stride +) { + const size_t ids_nb0 = ids->nb[0]; + + if (ids_nb0 == 4) { + switch (n_ids) { + case 8: return scan_expert_ids_n(ids, 8, n_as, counts, matrix_rows, mapping_stride); + case 4: return scan_expert_ids_n(ids, 4, n_as, counts, matrix_rows, mapping_stride); + case 2: return scan_expert_ids_n(ids, 2, n_as, counts, matrix_rows, mapping_stride); + default: return scan_expert_ids_n(ids, n_ids, n_as, counts, matrix_rows, mapping_stride); } } else { - if (htp_mminit_vec_dot(mmctx, src0->type) != 0) { - return HTP_STATUS_NO_SUPPORT; - } + // Strided fallback + const size_t ids_nb1 = ids->nb[1]; + const uint8_t * ids_data = (const uint8_t *) ids->data; + uint32_t n_skipped = 0; + for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { + const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1); + for (uint32_t id = 0; id < n_ids; ++id) { + const int32_t i02 = *(const int32_t *) ((const uint8_t *) row_ptr + id * ids_nb0); + if (i02 < 0) { + n_skipped++; + continue; + } + assert(i02 >= 0 && i02 < (int32_t) n_as); - quant_job_func = quantize_f32_q8x4x2; - src1_row_size = q8x4x2_row_size(ne10); - htp_mminit_spad(octx, dst_row_size, src0_row_size_padded, src1_row_size, src1_nrows, 0); + if (matrix_rows) { + matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 }; + } + counts[i02] += 1; + } + } + return n_skipped; } +} - // VTCM scratchpads for all tensors - size_t spad_size = octx->src1_spad.size + octx->src0_spad.size + octx->dst_spad.size; - - FARF(HIGH, "matmul-%s : src0-spad-size %u src1-spad-size %u dst-spad-size %u (%zu)\n", mmctx->type, - octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size, spad_size); - - FARF(HIGH, "matmul-%s : %ux%ux%ux%u * %ux%ux%ux%u-> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\n", mmctx->type, src0->ne[0], - src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], - dst->ne[1], dst->ne[2], dst->ne[3], src0->data, src1->data, dst->data); +static __attribute__((noinline)) uint32_t count_skipped_ids_single(const struct htp_tensor * ids, uint32_t n_ids) { + const uint8_t * ids_data = (const uint8_t *) ids->data; + const size_t ids_nb0 = ids->nb[0]; + uint32_t n_skipped = 0; + for (uint32_t id = 0; id < n_ids; ++id) { + const int32_t i02 = *(const int32_t *) (ids_data + id * ids_nb0); + if (i02 < 0) { + n_skipped++; + } + } + return n_skipped; +} - // Make sure the reserved vtcm size is sufficient - if (octx->ctx->vtcm_size < spad_size) { - FARF(ERROR, "matmul-%s : current VTCM reservation %zu is too small, needed %zu\n", mmctx->type, - octx->ctx->vtcm_size, spad_size); - return HTP_STATUS_VTCM_TOO_SMALL; +// Zero rows for skipped slots (ids < 0). Matmuls only write rows that an expert owns. +static void zero_skipped_dst_rows( + struct htp_ops_context * octx, + const struct htp_tensor * ids, + uint32_t n_ids, + const struct htp_tensor * dst +) { + const uint32_t elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(__fp16) : sizeof(float); + uint32_t col_start = 0; + uint32_t col_count = dst->ne[0]; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, elem_size); + const uint32_t chunk_elems = (dst->type == HTP_TYPE_F16) ? 64 : 32; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + dst->ne[0], can_split ? chunk_elems : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + col_start = range.start; + col_count = range.count; } - // Place src1 spad first. We use it for dyn.quant and may reuse between ops - octx->src1_spad.data = octx->ctx->vtcm_base; - octx->src0_spad.data = octx->src1_spad.data + octx->src1_spad.size; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; + if (col_count == 0) { + return; + } - octx->src1_spad.src = (src1 == octx->src1_spad.src) ? src1 : NULL; - octx->src0_spad.src = NULL; - octx->dst_spad.src = NULL; + const uint8_t * ids_data = (const uint8_t *) ids->data; + const size_t ids_nb0 = ids->nb[0]; + const size_t ids_nb1 = ids->nb[1]; + const size_t dst_nb1 = dst->nb[1]; + const size_t dst_nb2 = dst->nb[2]; + const uint32_t col_offset_bytes = col_start * elem_size; + + for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { + const uint8_t * ids_row = ids_data + iid1 * ids_nb1; + uint8_t * dst_row_group = (uint8_t *) dst->data + iid1 * dst_nb2 + col_offset_bytes; + for (uint32_t id = 0; id < n_ids; ++id) { + const int32_t i02 = *(const int32_t *) (ids_row + id * ids_nb0); + if (i02 < 0) { + hvx_splat_u(dst_row_group + id * dst_nb1, Q6_V_vzero(), col_count, elem_size); + } + } + } +} - octx->src0_spad.stride = src0_row_size_padded; - octx->src1_spad.stride = src1_row_size; +static int htp_mmid_prepare_mapping( + struct htp_ops_context * octx, + const struct htp_tensor * ids, + uint32_t n_ids, + uint32_t n_as, + uint32_t act_nrows, + uint8_t ** out_mapping_buf, + uint32_t ** out_matrix_row_counts, + struct mmid_row_mapping ** out_matrix_rows, + uint32_t * out_mapping_stride, + uint32_t * out_n_skipped +) { + uint8_t * mapping_buf = octx->ctx->ddr_spad_base; + uint32_t mapping_stride = 1; + uint32_t * matrix_row_counts = (uint32_t *) mapping_buf; + struct mmid_row_mapping * matrix_rows = NULL; + uint32_t n_skipped = 0; + + if (act_nrows > 1) { + const size_t matrix_row_counts_size = n_as * sizeof(uint32_t); + assert(octx->ctx->ddr_spad_size >= matrix_row_counts_size); + + hex_l2fetch_block((const void *) ids->data, ids->ne[1] * ids->nb[1]); + + hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t)); + n_skipped = scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, NULL, 0); + + uint32_t max_count = hvx_reduce_max_i32((const uint8_t *) matrix_row_counts, n_as); + mapping_stride = max_count > 0 ? max_count : 1; + + size_t matrix_row_map_size = n_as * mapping_stride * sizeof(struct mmid_row_mapping); + const size_t total_map_size = matrix_row_counts_size + matrix_row_map_size; + + if (total_map_size > octx->ctx->ddr_spad_size) { + mapping_buf = memalign(128, total_map_size); + if (!mapping_buf) { + return HTP_STATUS_INTERNAL_ERR; + } + } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) - return HTP_STATUS_OK; + matrix_row_counts = (uint32_t *) mapping_buf; + matrix_rows = (struct mmid_row_mapping *) (mapping_buf + matrix_row_counts_size); - if (need_quant && !octx->src1_spad.src) { - const uint32_t n_quant_jobs = MIN(src1_nrows, octx->n_threads); - mmctx->src1_nrows_per_thread = (src1_nrows + n_quant_jobs - 1) / n_quant_jobs; - worker_pool_run_func(octx->ctx->worker_pool, quant_job_func, mmctx, n_quant_jobs); - octx->src1_spad.src = src1; + hvx_splat_u(matrix_row_counts, Q6_V_vzero(), n_as, sizeof(uint32_t)); + scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride); + } else { + n_skipped = count_skipped_ids_single(ids, n_ids); } - const uint32_t n_matmul_jobs = octx->n_threads; - worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs); + *out_mapping_buf = mapping_buf; + *out_matrix_row_counts = matrix_row_counts; + *out_matrix_rows = matrix_rows; + *out_mapping_stride = mapping_stride; + *out_n_skipped = n_skipped; return HTP_STATUS_OK; } -int op_matmul(struct htp_ops_context * octx) { +int op_matmul_id(struct htp_ops_context * octx) { htp_matmul_tensors_preamble; -#ifndef HTP_HAS_HMX - return op_matmul_hvx(octx); -#else - if (!octx->ctx->hmx_enabled) { - return op_matmul_hvx(octx); - } + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; - // HMX weight tile requires N to be 32-aligned. - if (src0->ne[1] % 32 != 0) { - return op_matmul_hvx(octx); + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; } - // HMX supports F16, Q4_0, Q8_0, IQ4_NL, MXFP4 weights. - // Other types fall back to HVX. - uint32_t wtype = src0->type; - if (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_Q4_0 && wtype != HTP_TYPE_Q8_0 && wtype != HTP_TYPE_IQ4_NL && wtype != HTP_TYPE_MXFP4) { - return op_matmul_hvx(octx); - } + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - // Quantised HMX path requires K aligned to 256 (x4x2 super-block). - // F16 HMX path requires K aligned to 32 (tile width). - if (wtype != HTP_TYPE_F16 && src0->ne[0] % 256 != 0) { - return op_matmul_hvx(octx); - } + mmctx->octx = octx; + mmctx->act = act; - if (wtype == HTP_TYPE_F16 && src0->ne[0] % 32 != 0) { - return op_matmul_hvx(octx); + const struct htp_tensor * restrict ids = octx->src[2]; + if (htp_tensor_is_extended(ids) || htp_tensor_is_extended(act) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; } - const bool is_batched = (src0->ne[2] * src0->ne[3] > 1 || src1->ne[2] * src1->ne[3] > 1); + const size_t src0_row_size = nb01; + const size_t dst_row_size = nb1; + + const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); + + const uint32_t src0_nrows = ne01; // per expert + const uint32_t act_nrows = ne11 * ne12 * ne13; + + // row groups + const int n_ids = ids->ne[0]; // n_expert_used + const int n_as = ne02; // n_expert - // Quantised HMX kernels only handle flat 2D matmul (host already rejects - // batched quantised, but guard here too). F16 batched matmul is handled - // by the dedicated wrapper in hmx-matmul-ops.c. - if (is_batched && src0->type != HTP_TYPE_F16) { - return op_matmul_hvx(octx); + uint8_t * mapping_buf = NULL; + uint32_t * matrix_row_counts = NULL; + struct mmid_row_mapping * matrix_rows = NULL; + uint32_t mapping_stride = 1; + uint32_t n_skipped = 0; + + int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows, + &mapping_buf, &matrix_row_counts, &matrix_rows, + &mapping_stride, &n_skipped); + if (map_status != HTP_STATUS_OK) { + return map_status; } - // HMX assumes contiguous row-major layout. Fall back for permuted - // tensors where strides are non-monotonic (e.g. transposed KV cache). - if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) { - return op_matmul_hvx(octx); + if (n_skipped > 0) { + zero_skipped_dst_rows(octx, ids, n_ids, dst); } - // M alignment: when M > 32 but not 32-aligned, we split into - // HMX (first m_hmx = M & ~31 rows) + HVX (remaining m_tail rows). - // When M <= 32 and not 32-aligned, fall back entirely to HVX. - const int m_total = (int) src1->ne[1]; - const int m_tail = m_total % 32; - const int m_hmx = m_total - m_tail; + mmctx->matrix_row_counts = matrix_row_counts; + mmctx->matrix_rows = matrix_rows; + mmctx->mapping_stride = mapping_stride; + mmctx->mm_div_ne11 = kparams->div_ne1; + mmctx->src0_row_size_padded = src0_row_size_padded; + mmctx->act_nrows = act_nrows; + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = act_nrows; - if (m_hmx == 0) { - return op_matmul_hvx(octx); - } + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - // Always re-quantize src1 since HMX kernel overwrites vtcm/spad, - // so any previously cached quantized data is invalid. - octx->src1_spad.src = NULL; + int s; + if (kparams->n_hmx) { + s = hmx_mm_op_matmul_id(octx, mmctx); + } else { + uint32_t src0_row_start = 0; + uint32_t src0_row_end = src0_nrows; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + src0_row_start = range.start; + src0_row_end = range.start + range.count; + } + + if (src0_row_start >= src0_row_end) { + if (mapping_buf != octx->ctx->ddr_spad_base) { + free(mapping_buf); + } + return HTP_STATUS_OK; + } - int k = (int) src0->ne[0]; // inner dimension - int n = (int) src0->ne[1]; // weight columns + const uint32_t nrows = src0_row_end - src0_row_start; + mmctx->src0_row_start = src0_row_start; + mmctx->src0_row_end = src0_row_end; - // --- Phase 1: HMX on the first m_hmx (32-aligned) rows --- - int ret = -1; + mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div); + mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32); - // Row strides in elements. For compact tensors these equal k; for - // permuted attention views they can be larger, so pass the real stride. - const int act_stride = (int)(src1->nb[1] / sizeof(float)); - const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16)); - - if (src0->type == HTP_TYPE_F16) { - if (is_batched) { - hmx_matmul_w16a32_batched_params_t batch_params = { - .dst = (float *) dst->data, - .activation = (float *) src1->data, - .permuted_weight = (const __fp16 *) src0->data, - .m = m_hmx, - .k = k, - .n = n, - .act_stride = act_stride, - .weight_stride = wgt_stride, - .dst_stride = (int) (dst->nb[1] / sizeof(float)), - .ne02 = ne02, - .ne03 = ne03, - .ne12 = ne12, - .ne13 = ne13, - .src0_nb2 = src0->nb[2], - .src0_nb3 = src0->nb[3], - .src1_nb2 = src1->nb[2], - .src1_nb3 = src1->nb[3], - .dst_nb2 = dst->nb[2], - .dst_nb3 = dst->nb[3], - }; - ret = hmx_mat_mul_permuted_w16a32_batched(octx->ctx, &batch_params); + if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) == 0) { + s = hvx_mm_matmul_id(octx, mmctx, act_nrows > 1 ? hvx_mm_id_tiled : hvx_mv_id_tiled); } else { - ret = hmx_mat_mul_permuted_w16a32(octx->ctx, - (float*) dst->data, (float*) src1->data, (const __fp16 *) src0->data, - m_hmx, k, n, act_stride, wgt_stride); + s = HTP_STATUS_NO_SUPPORT; } - } else { - ret = hmx_mat_mul_permuted_qk_0_d16a32(octx->ctx, - (float*) dst->data, (float*) src1->data, (const uint8_t *) src0->data, - m_hmx, k, n, (int) src0->type); } - if (ret != 0) { - FARF(HIGH, "HMX matmul failed (ret=%d), falling back to HVX", ret); - return op_matmul(octx); + if (mapping_buf != octx->ctx->ddr_spad_base) { + free(mapping_buf); } - // --- Phase 2: HVX on the remaining m_tail rows --- - if (m_tail > 0) { - // copy of src1 and dst - struct htp_tensor src1_tail = *src1; - struct htp_tensor dst_tail = *dst; + return s; +} - src1_tail.ne[1] = m_tail; // only tail rows - dst_tail.ne[1] = m_tail; // only tail rows +int op_matmul_id_nx(struct htp_ops_context * octx) { + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; - // Offset activation and dst pointers past the HMX-processed rows. - // Use nb[1] (row stride in bytes) to compute the byte offset. - src1_tail.data += (uint32_t) m_hmx * src1->nb[1]; - dst_tail.data += (uint32_t) m_hmx * dst->nb[1]; + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; + } - octx->src[1] = &src1_tail; - octx->dst = &dst_tail; + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - FARF(HIGH, "hmx-matmul: HVX tail m_tail %d src1 %p dst %p", m_tail, (void *) src1_tail.data, (void *) dst_tail.data); - return op_matmul_hvx(octx); + mmctx->octx = octx; + const uint32_t n_weights = kparams->n_weights; + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict act = octx->src[n_weights]; + const struct htp_tensor * restrict ids = octx->src[n_weights + 1]; + if (htp_tensor_is_extended(ids) || htp_tensor_is_extended(act)) { + return HTP_STATUS_NO_SUPPORT; + } + for (uint32_t p = 0; p < n_weights; p++) { + if (htp_tensor_is_extended(octx->dsts[p])) { + return HTP_STATUS_NO_SUPPORT; + } } - return 0; -#endif // HTP_HAS_HMX -} + mmctx->act = act; -int op_matmul_id(struct htp_ops_context * octx) { - htp_matmul_tensors_preamble; + const size_t src0_row_size = src0->nb[1]; + const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); - struct htp_matmul_context mmctx_struct = {0}; - struct htp_matmul_context * mmctx = &mmctx_struct; - mmctx->octx = octx; + const uint32_t act_nrows = act->ne[1] * act->ne[2] * act->ne[3]; - const struct htp_tensor * restrict ids = octx->src[2]; + const int n_ids = ids->ne[0]; + const int n_as = src0->ne[2]; - const size_t src0_row_size = nb01; - const size_t dst_row_size = nb1; + uint8_t * mapping_buf = NULL; + uint32_t * matrix_row_counts = NULL; + struct mmid_row_mapping * matrix_rows = NULL; + uint32_t mapping_stride = 1; + uint32_t n_skipped = 0; - const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128); + int map_status = htp_mmid_prepare_mapping(octx, ids, n_ids, n_as, act_nrows, + &mapping_buf, &matrix_row_counts, &matrix_rows, + &mapping_stride, &n_skipped); + if (map_status != HTP_STATUS_OK) { + return map_status; + } - const uint32_t src0_nrows = ne01; // per expert - const uint32_t src1_nrows = ne11 * ne12 * ne13; + if (n_skipped > 0) { + for (uint32_t p = 0; p < n_weights; ++p) { + const struct htp_tensor * restrict d = octx->dsts[p]; + if (d) { + zero_skipped_dst_rows(octx, ids, n_ids, d); + } + } + } - worker_callback_t quant_job_func; - worker_callback_t matmul_id_job_func = src1_nrows > 1 ? matmul_id : matvec_id; + mmctx->matrix_row_counts = matrix_row_counts; + mmctx->matrix_rows = matrix_rows; + mmctx->mapping_stride = mapping_stride; + mmctx->mm_div_ne11 = kparams->div_ne1; + mmctx->src0_row_size_padded = src0_row_size_padded; + mmctx->act_nrows = act_nrows; + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = act_nrows; - // Compute src0_nrows_per_thread - mmctx->src0_nrows_per_thread = (src0_nrows + octx->n_threads - 1) / octx->n_threads; - mmctx->src0_nrows_per_thread += (mmctx->src0_nrows_per_thread & 1); // round up to even + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - size_t src1_row_size; - size_t src1_row_size_padded; + int s; + if (kparams->n_hmx) { + s = hmx_mm_op_matmul_id_nx(octx, mmctx); + } else { + if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) == 0) { + s = hvx_mm_matmul_id_nx(octx, mmctx, act_nrows > 1 ? hvx_mm_id_nx_tiled : hvx_mv_id_nx_tiled); + } else { + s = HTP_STATUS_NO_SUPPORT; + } + } - // row groups - const int n_ids = ids->ne[0]; // n_expert_used - const int n_as = ne02; // n_expert + if (mapping_buf != octx->ctx->ddr_spad_base) { + free(mapping_buf); + } - size_t matrix_row_counts_size = n_as * sizeof(uint32_t); - size_t matrix_row_map_size = n_as * ids->ne[0] * ids->ne[1] * sizeof(struct mmid_row_mapping); + return s; +} +static int op_matmul_nx_impl(struct htp_ops_context * octx) { + const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; - if (htp_mminit_vec_dot(mmctx, src0->type) != 0) { - return HTP_STATUS_NO_SUPPORT; + const int status = htp_mm_init_context(octx, kparams); + if (status != HTP_STATUS_OK) { + return status; } - quant_job_func = quantize_f32_q8x4x2; - src1_row_size = q8x4x2_row_size(ne10); + if (kparams->n_hmx) { + return hmx_mm_nx_2d_f32(octx, kparams); + } - const size_t src2_spad_size_per_thread = hex_round_up(matrix_row_counts_size + matrix_row_map_size, 256); - htp_mminit_spad(octx, dst_row_size, src0_row_size_padded, src1_row_size, src1_nrows, src2_spad_size_per_thread); + struct htp_thread_trace * tr = &octx->ctx->trace[0]; + htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0); - size_t spad_size = octx->src2_spad.size + octx->src1_spad.size + octx->src0_spad.size + octx->dst_spad.size; + const uint32_t n_weights = kparams->n_weights; - FARF(HIGH, "matmul-id-%s : src0-spad-size %u src1-spad-size %u src2-spad-size %u dst-spad-size %u (%zu)\n", mmctx->type, - octx->src0_spad.size, octx->src1_spad.size, octx->src2_spad.size, octx->dst_spad.size, spad_size); + const struct htp_tensor * restrict src0 = octx->src[0]; // first weight + const struct htp_tensor * restrict act = octx->src[n_weights]; // activation x - FARF(HIGH, "matmul-id-%s : %ux%ux%ux%u * %ux%ux%ux%u (%ux%ux%ux%u) -> %ux%ux%ux%u (0x%p, 0x%p, 0x%p)\n", mmctx->type, - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], - ids->ne[0], ids->ne[1], ids->ne[2], ids->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], src0->data, - src1->data, dst->data); + struct htp_mm_context mmctx_struct = {0}; + struct htp_mm_context * mmctx = &mmctx_struct; + mmctx->octx = octx; + mmctx->act = act; - // Make sure the reserved vtcm size is sufficient - if (octx->ctx->vtcm_size < spad_size) { - FARF(ERROR, "matmul-id-%s : current VTCM reservation %zu is too small, needed %zu\n", mmctx->type, octx->ctx->vtcm_size, spad_size); - return HTP_STATUS_VTCM_TOO_SMALL; + const uint32_t act_nrows = act->ne[1] * act->ne[2] * act->ne[3]; + mmctx->act_nrows = act_nrows; + mmctx->cur_m_start = 0; + mmctx->cur_m_rows = act_nrows; + + const size_t src0_row_size = src0->nb[1]; + + if (hvx_mm_init_tiled_vec_dot(mmctx, src0->type) != 0) { + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t qk = HTP_MM_ACT_BLOCK_K; + const uint32_t nb = (act->ne[0] + qk - 1) / qk; + const uint32_t total_nb = act_nrows * nb; + + worker_callback_t quant_task_func; + uint32_t n_quant_tasks = 1; + if (act_nrows < octx->n_threads) { + n_quant_tasks = MIN(total_nb, octx->n_threads); + quant_task_func = htp_mm_act_quant_block_func(src0->type, act->type); + for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) { + uint32_t ib_first = (total_nb * ith) / n_quant_tasks; + uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks; + mmctx->quant_ib_first[ith] = ib_first; + mmctx->quant_ib_last[ith] = ib_last; + mmctx->quant_r[ith] = ib_first / nb; + mmctx->quant_c[ith] = ib_first % nb; + } + } else { + n_quant_tasks = MIN(act_nrows, octx->n_threads); + quant_task_func = htp_mm_act_quant_row_func(src0->type, act->type); } - // Place src1 spad first. We use it for dyn.quant and may reuse in subseq ops. - octx->src1_spad.data = octx->ctx->vtcm_base; - octx->src0_spad.data = octx->src1_spad.data + octx->src1_spad.size; - octx->src2_spad.data = octx->src0_spad.data + octx->src0_spad.size; - octx->dst_spad.data = octx->src2_spad.data + octx->src2_spad.size; + const size_t act_row_size = htp_mm_tiled_act_row_size(src0->type, act->ne[0]); - octx->src1_spad.src = (src1 == octx->src1_spad.src) ? src1 : NULL; - octx->src0_spad.src = NULL; - octx->src2_spad.src = NULL; - octx->dst_spad.src = NULL; + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], act_nrows, octx->n_threads, + 0, src0_row_size, act_row_size, 0, kparams->n_prefetch, false, true); - octx->src0_spad.stride = src0_row_size_padded; - octx->src1_spad.stride = src1_row_size; + const size_t vtcm_size = L.total_bytes; - if (src1_nrows > 1) { - // initialize matrix_row_counts and map - uint32_t * matrix_row_counts = (uint32_t *) octx->src2_spad.data + 0; - struct mmid_row_mapping * matrix_rows = (void *) octx->src2_spad.data + matrix_row_counts_size; + if (octx->ctx->vtcm_size < vtcm_size) { + FARF(ERROR, "matmul-nx: current VTCM reservation %zu is too small, needed %zu\n", + octx->ctx->vtcm_size, vtcm_size); + return HTP_STATUS_VTCM_TOO_SMALL; + } - memset(matrix_row_counts, 0, n_as * sizeof(uint32_t)); + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + mmctx->vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + mmctx->vtcm_act = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act); + mmctx->vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + mmctx->vtcm_act_raw = VTCM_LAYOUT_PTR(uint8_t, base, L.off_act_raw); - // group rows by src0 matrix - for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { // token idx - for (uint32_t id = 0; id < n_ids; ++id) { // expert idx - const uint32_t i02 = *(const uint32_t *) ((const uint8_t *) ids->data + iid1 * ids->nb[1] + id * ids->nb[0]); + mmctx->vtcm_src0_stride = 0; + mmctx->vtcm_act_stride = act_row_size; + mmctx->vtcm_act_raw_stride = htp_mm_act_raw_row_size(act->ne[0]); - assert(i02 >= 0 && i02 < n_as); + mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div); + mmctx->vtcm_act_size_per_thread = L.act_bytes; + mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div); - MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) { id, iid1 }; - matrix_row_counts[i02] += 1; - } - } - } + htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0); - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) - return HTP_STATUS_OK; + hvx_mm_transfer_act_dma(octx, kparams, act, mmctx->vtcm_act_raw, mmctx->vtcm_act_raw_stride, 0, act_nrows); - if (octx->src1_spad.src != src1) { - const uint32_t n_quant_jobs = MIN(src1_nrows, octx->n_threads); - mmctx->src1_nrows_per_thread = (src1_nrows + n_quant_jobs - 1) / n_quant_jobs; - worker_pool_run_func(octx->ctx->worker_pool, quant_job_func, mmctx, n_quant_jobs); - octx->src1_spad.src = src1; - } + mmctx->n_quant_rows_per_thread = (act_nrows + n_quant_tasks - 1) / n_quant_tasks; + mmctx->n_quant_tasks = n_quant_tasks; + work_queue_run(octx->ctx->work_queue, quant_task_func, mmctx, n_quant_tasks); - const uint32_t n_matmul_jobs = octx->n_threads; - worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, n_matmul_jobs); + work_queue_run(octx->ctx->work_queue, hvx_mm_nx_2d_tiled, mmctx, octx->n_threads); return HTP_STATUS_OK; } + +int op_matmul_nx(struct htp_ops_context * octx) { + return op_matmul_nx_impl(octx); +} diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h new file mode 100644 index 0000000..4971ed9 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -0,0 +1,774 @@ +#ifndef HTP_MATMUL_OPS_H +#define HTP_MATMUL_OPS_H + +#include +#include +#include "htp-ops.h" +#include "htp-tensor.h" +#include "hex-fastdiv.h" +#include "hex-common.h" +#include "htp-vtcm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// --- HMX Tile Constraints --- +#define HTP_MM_HMX_TILE_N_COLS 32 +#define HTP_MM_HMX_TILE_N_ROWS 32 +#define HTP_MM_HMX_TILE_SIZE (32 * 32 * sizeof(__fp16)) // 2048 bytes +#define HTP_MM_HMX_TILE_N_ELMS 1024 +#define HTP_MM_HMX_MIN_NROWS 4 + +#define HTP_MM_WEIGHT_TILE_K_Q2_0 64 +#define HTP_MM_WEIGHT_TILE_SIZE_Q2_0 HTP_TENSOR_TILE_SIZE_Q2_0 +#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0 640 + +// --- Activation Tiled Block Sizes (including padding) --- +#define HTP_MM_ACT_BLOCK_K 128 +#define HTP_MM_ACT_TILE_SIZE_Q8_0 1152 +#define HTP_MM_ACT_TILE_SIZE_Q8_1 1280 +#define HTP_MM_ACT_TILE_SIZE_F16 2048 + +#define HTP_MM_MAX_PREFETCH 16 + +// --- Solver Cost Model Penalty Weights (HMX-specific) --- +#define HTP_MM_HMX_COST_W_DEQUANT 3 // cost penalty for quantized weight loading/dequantization +#define HTP_MM_HMX_COST_A_CONVERT 2 // cost penalty for activation loading/conversion + +// --- DMA Activation Transfer Configuration --- +#define HTP_MM_DMA_ACT_ROWS_PER_STEP 2 +#define HTP_MM_DMA_ACT_MULTIPLIER (2 * HTP_MM_DMA_ACT_ROWS_PER_STEP) + +enum htp_mm_kernel_type { + HTP_MM_KERNEL_UNSUPPORTED = 0, + + // HMX paths + HTP_MM_KERNEL_HMX_2D, + HTP_MM_KERNEL_HMX_BATCHED, + + // HVX floating-point paths + HTP_MM_KERNEL_HVX_F16_F16_VTCM, + HTP_MM_KERNEL_HVX_F32_F32_VTCM, + HTP_MM_KERNEL_HVX_BF16_F32_VTCM, + + // HVX quantized paths + HTP_MM_KERNEL_HVX_QUANT_ROW, // standard row-wise parallel quantization + HTP_MM_KERNEL_HVX_QUANT_BLOCK, // parallel block-wise quantization +}; + +// Op-specific struct for precomputed matmul params +struct htp_mm_kernel_params { + uint8_t kernel_type; // enum htp_mm_kernel_type + uint8_t pipeline; // 1 = pipelined execution, 0 = standard + uint8_t unused; // reserved + uint8_t n_hmx; // 1 = use HMX, 0 = use HVX + + uint8_t n_threads; // Number of threads to spawn + uint8_t n_act_threads; // Number of threads for activation preparation + uint8_t n_prefetch; // Prefetch lookahead buffers/rows in VTCM + uint8_t n_weights; // Number of weights for fused NX + + int32_t m_chunk; // Row chunk size (M chunk) + int32_t n_chunk; // Col chunk size (N chunk) + int32_t tile_size; // Weight tile size + int32_t aligned_tile_size; // Aligned weight tile size (padded to 128) + int32_t act_row_size; // Row size for activation scratchpad + int32_t vtcm_size; // Total required scratchpad size in VTCM + int32_t vtcm_bias_size; // bias scratchpad size in VTCM (fused only) + + // Precomputed division values + struct fastdiv_values div_ne12_ne1; + struct fastdiv_values div_ne1; + struct fastdiv_values div_r2; + struct fastdiv_values div_r3; + struct fastdiv_values div_ne12; + struct fastdiv_values div_n_act_threads; + struct fastdiv_values div_ne00_padded; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_mm_kernel_params) <= 128, "htp_matmul_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_mm_kernel_params) <= 128, "htp_matmul_kernel_params is too large for kernel_params blob"); +#endif + +struct mmid_row_mapping { + uint32_t i1; + uint32_t i2; +}; + +// Search for optimal (mc, nc) chunk sizes within VTCM budget. +static inline int htp_mm_hmx_compute_chunks(size_t vtcm_total, + size_t overhead, + size_t per_n_cost, + size_t per_m_cost, + size_t per_mn_cost, + size_t m, + size_t n, + size_t m_block_cost, + size_t n_block_cost, + size_t * m_chunk_out, + size_t * n_chunk_out, + size_t * total_out) { + if (m == 0 || n == 0) return -1; + if (vtcm_total <= overhead) return -1; + if (per_n_cost == 0 || per_m_cost == 0 || per_mn_cost == 0) return -1; + + const size_t usable = vtcm_total - overhead; + + size_t best_cost = SIZE_MAX; + size_t best_tail_waste = SIZE_MAX; + size_t best_mn = 0; + size_t best_m = 0, best_n = 0; + + const size_t max_nc_budget = (usable / per_n_cost); + const size_t n_max = hex_align_down(hex_smin((size_t)n, max_nc_budget), HTP_MM_HMX_TILE_N_COLS); + for (size_t nc = n_max; nc >= HTP_MM_HMX_TILE_N_COLS; nc -= HTP_MM_HMX_TILE_N_COLS) { + size_t n_fixed = 0, ncmn = 0, mc_denom = 0; + if (hex_mul_overflow(nc, per_n_cost, &n_fixed)) continue; + if (n_fixed >= usable) goto next_nc; + + if (hex_mul_overflow(nc, per_mn_cost, &ncmn)) goto next_nc; + if (hex_add_overflow(per_m_cost, ncmn, &mc_denom) || mc_denom == 0) goto next_nc; + + { + size_t remain = usable - n_fixed; + size_t mc = remain / mc_denom; + mc = hex_align_down(mc, HTP_MM_HMX_TILE_N_ROWS); + mc = hex_smin(mc, m); + + if (mc == 0) { + goto next_nc; + } + + size_t mblocks = ((size_t) m + mc - 1) / mc; + size_t nblocks = ((size_t) n + nc - 1) / nc; + size_t cost = mblocks * m_block_cost + nblocks * n_block_cost; + size_t rem = n % nc; + size_t tail_waste = (rem == 0) ? 0 : (nc - rem); + size_t mn = mc * nc; + if (cost < best_cost || + (cost == best_cost && tail_waste < best_tail_waste) || + (cost == best_cost && tail_waste == best_tail_waste && mn > best_mn)) { + best_cost = cost; + best_tail_waste = tail_waste; + best_mn = mn; + best_m = mc; + best_n = nc; + } + } + +next_nc: + if (nc == HTP_MM_HMX_TILE_N_COLS) break; // avoid size_t underflow + } + + if (best_m == 0 || best_n == 0) return -1; + + // Compute exact total (with overflow checks) + size_t t0 = 0, t1 = 0, t2 = 0, mn = 0, total = 0; + if (hex_mul_overflow(best_n, per_n_cost, &t0)) return -1; + if (hex_mul_overflow(best_m, per_m_cost, &t1)) return -1; + if (hex_mul_overflow(best_m, best_n, &mn)) return -1; + if (hex_mul_overflow(mn, per_mn_cost, &t2)) return -1; + if (hex_add_overflow(t0, t1, &total)) return -1; + if (hex_add_overflow(total, t2, &total)) return -1; + if (hex_add_overflow(total, overhead, &total)) return -1; + + *m_chunk_out = best_m; + *n_chunk_out = best_n; + *total_out = total; + return 0; +} + +// weight types whose tiles carry a per-block offset (x = d * q + m): the activations need block sums (q8_1) +// (Q2_K: per-16 k sums, q8_1_s16) +static inline bool htp_mm_weight_has_offset(int weight_type) { + return weight_type == HTP_TYPE_Q4_1 || weight_type == HTP_TYPE_Q4_K || weight_type == HTP_TYPE_Q5_K || + weight_type == HTP_TYPE_Q2_K || weight_type == HTP_TYPE_Q2_0; +} + +// --- Activation/Row Size Helpers --- +static inline size_t htp_mm_q8_0_tiled_row_size(uint32_t ne) { + const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K); + const uint32_t nb_32 = ne_padded / 32; + return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_0; +} + +static inline size_t htp_mm_q8_1_tiled_row_size(uint32_t ne) { + const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K); + const uint32_t nb_32 = ne_padded / 32; + return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_1; +} + +static inline size_t htp_mm_f16_tiled_row_size(uint32_t ne) { + const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K); + const uint32_t nb_32 = ne_padded / 32; + return nb_32 * HTP_MM_ACT_TILE_SIZE_F16; +} + +static inline size_t htp_mm_tiled_act_row_size(int weight_type, uint32_t ne) { + if (weight_type == HTP_TYPE_F16) { + return htp_mm_f16_tiled_row_size(ne); + } + return htp_mm_weight_has_offset(weight_type) ? htp_mm_q8_1_tiled_row_size(ne) : htp_mm_q8_0_tiled_row_size(ne); +} + +static inline size_t htp_mm_act_raw_row_size(uint32_t ne) { + return hex_round_up(ne, HTP_MM_ACT_BLOCK_K) * sizeof(float); +} + +static inline size_t htp_mm_round_up(size_t n, size_t m) { + return ((n + m - 1) / m) * m; +} + +static inline bool htp_mm_hmx_pipeline(uint32_t m) { + return m > 32; +} + +static inline void htp_mm_hmx_get_2d_chunk_costs( + int wtype, uint32_t k, bool pipeline, uint32_t aligned_tile_size, + size_t * size_per_n_out, size_t * size_per_m_out, size_t * size_per_mn_out +) { + const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16); + const size_t row_stride = htp_tensor_get_row_size(wtype, k); + const size_t vec_dot_size = k * sizeof(uint16_t); + const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype); + const size_t qweight_row_stride = is_quant ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0; + + *size_per_n_out = (pipeline ? 2 : 1) * (is_quant ? qweight_row_stride : row_stride) + + (pipeline ? 2 * vec_dot_size : vec_dot_size); + *size_per_m_out = vec_dot_size; + *size_per_mn_out = (pipeline ? 2 : 1) * sizeof(uint16_t); +} + +static inline void htp_mm_hmx_get_batched_chunk_costs( + uint32_t k, uint32_t group_size, + size_t * size_per_n_out, size_t * size_per_m_out, size_t * size_per_mn_out +) { + const size_t vec_dot_size = k * sizeof(uint16_t); + *size_per_n_out = 3 * vec_dot_size; + *size_per_m_out = group_size * vec_dot_size; + *size_per_mn_out = sizeof(uint16_t); +} + +static inline size_t htp_mm_hmx_get_2d_overhead(bool pipeline, bool is_matmul_id) { + size_t num_regions = pipeline ? 7 : (is_matmul_id ? 4 : 5); + return num_regions * HTP_MM_HMX_TILE_SIZE + 256; +} + +static inline size_t htp_mm_hmx_get_batched_overhead(void) { + return 5 * HTP_MM_HMX_TILE_SIZE + 256; +} + +struct htp_mm_hmx_vtcm_layout { + // Byte offsets from vtcm_base for each region + size_t off_weight[2]; // [1] is only used when pipelined + size_t off_act; + size_t off_act_f32; // fp32 activation conversion scratch + size_t off_dst[2]; // [1] is only used when pipelined + size_t off_scratch[2]; // dequantization scratch pads + size_t off_scales; // HMX scales (256 bytes) + size_t off_bias; // bias in VTCM + + // Cached sizes of regions for HMX kernel use + size_t weight_area_bytes; + size_t act_area_bytes; + size_t act_f32_bytes; + size_t output_area_bytes; + size_t scratch_bytes[2]; + size_t act_head_stride; + size_t bias_bytes; + + size_t total_bytes; +}; + +struct htp_mm_hvx_vtcm_layout { + // Byte offsets from vtcm_base for each region + size_t off_act; // vtcm_act (activation) + size_t off_src0; // vtcm_src0 (weight/Wk) + size_t off_bias; // vtcm_bias (bias / fused add only) + size_t off_dst; // vtcm_dst (output scratch) + size_t off_act_raw; // vtcm_act_raw (raw activation DMA staging) + + // Cached sizes + size_t src0_bytes; + size_t act_bytes; + size_t bias_bytes; + size_t dst_bytes; + size_t act_raw_bytes; + + size_t total_bytes; +}; + +static inline void htp_mm_hmx_vtcm_layout_build( + struct htp_mm_hmx_vtcm_layout * L, + int kernel_type, + int wtype, + uint32_t k, + size_t mc, + size_t nc, + uint32_t group_size, + bool pipeline, + uint32_t act_threads, + uint32_t aligned_tile_size, + size_t bias_size +) { + size_t off = 0; + + if (kernel_type == HTP_MM_KERNEL_HMX_BATCHED) { + const size_t vec_dot_size = k * sizeof(uint16_t); + const size_t act_head_stride = mc * k; + const size_t weight_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); + const size_t activation_area_size = hex_align_up(group_size * act_head_stride * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); + const size_t output_area_size = hex_align_up(group_size * mc * nc * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE); + const size_t scratch_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); + const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); + + // Group A: Permanent activation tiles and scales + size_t off_group_a = 0; + VTCM_LAYOUT_ALLOC(off_group_a, off_act, activation_area_size); + VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_bias, hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE), bias_size > 0); + + // Group B: Compute-only buffers (starts at off_group_a) + size_t off_group_b = off_group_a; + VTCM_LAYOUT_ALLOC(off_group_b, off_weight[0], weight_area_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_weight[1], weight_area_size, false); + VTCM_LAYOUT_ALLOC(off_group_b, off_dst[0], output_area_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_dst[1], output_area_size, false); + VTCM_LAYOUT_ALLOC(off_group_b, off_scratch[0], scratch_area_size); + VTCM_LAYOUT_ALLOC(off_group_b, off_scratch[1], scratch_area_size); + + const size_t group_b_size = off_group_b - off_group_a; + + // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) + const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128); + size_t off_group_c = off_group_a; + VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size); + + const size_t group_c_size = off_group_c - off_group_a; + + L->weight_area_bytes = weight_area_size; + L->act_area_bytes = activation_area_size; + L->act_f32_bytes = act_f32_size; + L->output_area_bytes = output_area_size; + L->scratch_bytes[0] = scratch_area_size; + L->scratch_bytes[1] = scratch_area_size; + L->act_head_stride = act_head_stride; + L->bias_bytes = bias_size; + + off = off_group_a + hex_smax(group_b_size, group_c_size); + } else { + // HTP_MM_KERNEL_HMX_2D + const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16); + const size_t row_stride = htp_tensor_get_row_size(wtype, k); + const size_t vec_dot_size = k * sizeof(uint16_t); + const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype); + + const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); + const size_t weight_area_size = is_quant + ? hex_align_up((nc / 32) * n_weight_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) + : hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE); + const size_t act_area_size = hex_align_up(mc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); + const size_t output_area_size = hex_align_up(mc * nc * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE); + + const size_t scratch0_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); + const size_t scratch1_size = pipeline ? scratch0_size : 0; + + // Group A: Scales and activation tiles (must not overlap with Group B or C) + size_t off_group_a = 0; + VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data + VTCM_LAYOUT_ALLOC(off_group_a, off_act, act_area_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_bias, hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE), bias_size > 0); + + // Group B: Compute-only buffers (starts at off_group_a) + size_t off_group_b = off_group_a; + VTCM_LAYOUT_ALLOC(off_group_b, off_weight[0], weight_area_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_weight[1], weight_area_size, pipeline); + VTCM_LAYOUT_ALLOC(off_group_b, off_dst[0], output_area_size); + VTCM_LAYOUT_ALLOC(off_group_b, off_scratch[0], scratch0_size); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_scratch[1], scratch0_size, pipeline); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_dst[1], output_area_size, pipeline); + + const size_t group_b_size = off_group_b - off_group_a; + + // Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a) + const size_t max_f32_size = act_threads * 64 * k * sizeof(float); + const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128); + size_t off_group_c = off_group_a; + VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size); + + const size_t group_c_size = off_group_c - off_group_a; + + L->weight_area_bytes = weight_area_size; + L->act_area_bytes = act_area_size; + L->act_f32_bytes = act_f32_size; + L->output_area_bytes = output_area_size; + L->scratch_bytes[0] = scratch0_size; + L->scratch_bytes[1] = scratch1_size; + L->act_head_stride = 0; + L->bias_bytes = bias_size; + + off = off_group_a + hex_smax(group_b_size, group_c_size); + } + + L->total_bytes = off; +} + +static inline void htp_mm_hvx_vtcm_layout_build( + struct htp_mm_hvx_vtcm_layout * L, + int kernel_type, + int wtype, + uint32_t ne10, // k + uint32_t act_nrows, // m_total + uint32_t n_threads, + size_t dst_row_size, + size_t src0_row_size, + size_t act_row_size, + size_t bias_row_size, + uint32_t n_prefetch, + bool is_matmul_id, + bool is_fused_nx +) { + (void)act_row_size; + size_t src0_sz = 0; + size_t act_sz = 0; + size_t bias_sz = bias_row_size > 0 ? htp_mm_round_up(bias_row_size, 128) : 0; + size_t dst_sz = 0; + size_t act_raw_sz = 0; + + if (is_fused_nx) { + uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); + uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + size_t weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); + + size_t tiled_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + size_t q_act_sz = hex_round_up(tiled_act_row_size * act_nrows, 128); + size_t raw_row_size = htp_mm_act_raw_row_size(ne10); + + src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer + act_sz = q_act_sz; // quantized activation buffer + bias_sz = 0; + dst_sz = 0; + act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); + } else if (is_matmul_id) { + const size_t act_row_size_tiled = htp_mm_tiled_act_row_size(wtype, ne10); + const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + const uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); + const uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * tile_row_size, 256); + + act_sz = htp_mm_round_up(act_row_size_tiled * act_nrows, 256); + + size_t raw_row_size = htp_mm_act_raw_row_size(ne10); + + src0_sz = src0_sz_per_thread * n_threads; + dst_sz = 0; + bias_sz = 0; + act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); + } else { + const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128); + const size_t dst_nrows = (act_nrows > 1) ? 0 : 1; + + switch (kernel_type) { + case HTP_MM_KERNEL_HVX_F16_F16_VTCM: { + size_t f16_act_row_size = htp_mm_round_up(ne10 * 2, 128); + act_sz = htp_mm_round_up(f16_act_row_size * act_nrows, 256); + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + act_raw_sz = hex_round_up(hex_round_up(ne10 * sizeof(float), 128) * act_nrows, 128); + break; + } + case HTP_MM_KERNEL_HVX_F32_F32_VTCM: + case HTP_MM_KERNEL_HVX_BF16_F32_VTCM: { + size_t f32_act_row_size = htp_mm_round_up(ne10 * 4, 128); + act_sz = htp_mm_round_up(f32_act_row_size * act_nrows, 256); + src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads; + dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0; + act_raw_sz = 0; + break; + } + case HTP_MM_KERNEL_HVX_QUANT_BLOCK: + case HTP_MM_KERNEL_HVX_QUANT_ROW: { + size_t q_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); + uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); + uint32_t n_weight_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); + uint32_t tile_row_size = n_weight_tiles * aligned_tile_size; + size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); + + src0_sz = tiled_vtcm_size * n_threads; + act_sz = htp_mm_round_up(q_act_row_size * act_nrows, 256); + + size_t dst_slice_per_thread = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0; + dst_sz = dst_slice_per_thread * n_threads; + size_t raw_row_size = htp_mm_act_raw_row_size(ne10); + act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128); + break; + } + default: + break; + } + } + + // Group A: Persistent buffers across chunk compute + size_t off_group_a = 0; + VTCM_LAYOUT_ALLOC(off_group_a, off_act, act_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_bias, bias_sz, bias_sz > 0); + + // Group B: Compute-only buffers (starts at off_group_a) + size_t off_group_b = off_group_a; + VTCM_LAYOUT_ALLOC(off_group_b, off_src0, src0_sz); + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_dst, dst_sz, dst_sz > 0); + const size_t group_b_size = off_group_b - off_group_a; + + // Group C: Raw activation staging buffer (overlaps Group B, starts at off_group_a) + size_t off_group_c = off_group_a; + VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_c, off_act_raw, act_raw_sz, act_raw_sz > 0); + const size_t group_c_size = off_group_c - off_group_a; + + L->src0_bytes = src0_sz; + L->act_bytes = act_sz; + L->bias_bytes = bias_sz; + L->dst_bytes = dst_sz; + L->act_raw_bytes = act_raw_sz; + L->total_bytes = off_group_a + hex_smax(group_b_size, group_c_size); +} + +static inline bool htp_mm_hvx_solve_vtcm_params( + int kernel_type, + int wtype, + uint32_t ne10, + uint32_t act_nrows, + uint32_t n_threads, + size_t dst_row_size, + size_t src0_row_size, + size_t act_row_size, + size_t bias_row_size, + uint32_t n_prefetch, + size_t vtcm_budget, + struct htp_mm_hvx_vtcm_layout * L_out, + uint32_t * m_chunk_out +) { + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build( + &L, kernel_type, wtype, ne10, act_nrows, n_threads, + dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false + ); + + if (L.total_bytes <= vtcm_budget) { + *L_out = L; + *m_chunk_out = act_nrows; + return true; + } + + const size_t fixed_bytes = L.src0_bytes + L.bias_bytes + L.dst_bytes; + if (vtcm_budget <= fixed_bytes) { + return false; + } + + const size_t avail_act = vtcm_budget - fixed_bytes; + size_t row_size = 0; + if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { + row_size = htp_mm_tiled_act_row_size(wtype, ne10); + } else if (kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM) { + row_size = hex_round_up(ne10 * 2, 128); + } else { + row_size = hex_round_up(ne10 * 4, 128); + } + if (row_size == 0) { + return false; + } + + size_t eff_row_size = row_size; + if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) { + eff_row_size += htp_mm_act_raw_row_size(ne10); + } + + uint32_t m_chunk = (uint32_t) (avail_act / eff_row_size); + if (m_chunk > 1) { + m_chunk &= ~1U; + } + if (m_chunk > act_nrows) { + m_chunk = act_nrows; + } + if (m_chunk < 1) { + return false; + } + + htp_mm_hvx_vtcm_layout_build( + &L, kernel_type, wtype, ne10, m_chunk, n_threads, + dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false + ); + + while (m_chunk > 2 && L.total_bytes > vtcm_budget) { + m_chunk -= 2; + htp_mm_hvx_vtcm_layout_build( + &L, kernel_type, wtype, ne10, m_chunk, n_threads, + dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false + ); + } + + if (L.total_bytes <= vtcm_budget) { + *L_out = L; + *m_chunk_out = m_chunk; + return true; + } + + return false; +} + +static inline size_t htp_mm_hmx_get_2d_vtcm_size( + int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size, size_t bias_size +) { + struct htp_mm_hmx_vtcm_layout L; + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_threads, aligned_tile_size, bias_size); + return L.total_bytes; +} + +static inline size_t htp_mm_hmx_get_batched_vtcm_size( + int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_threads, size_t bias_size) { + (void)pipeline; + struct htp_mm_hmx_vtcm_layout L; + htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_threads, 0, bias_size); + return L.total_bytes; +} + +static inline bool htp_mm_hmx_solve_batched_params( + int wtype, + uint32_t k, + uint32_t ne01_padded, + uint32_t ne11, + uint32_t group_size, + int n_threads, + bool pipeline, + size_t bias_size, + size_t vtcm_budget, + size_t * m_chunk_out, + size_t * n_chunk_out, + int * act_threads_out, + size_t * vtcm_size_out +) { + size_t best_mblocks = SIZE_MAX; + int best_act_threads = 0; + size_t best_m_chunk = 0; + size_t best_n_chunk = 0; + size_t best_vtcm_size = 0; + + int act_threads = n_threads; + while (act_threads >= 1) { + size_t group_overhead = htp_mm_hmx_get_batched_overhead() + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0); + size_t group_size_per_n, group_size_per_m, group_size_per_mn; + htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn); + + size_t m_chunk_candidate = 0; + size_t n_chunk_candidate = 0; + size_t vtcm_size_candidate = 0; + + if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded, + (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, + &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { + size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_threads, bias_size); + if (exact_size <= vtcm_budget) { + size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate; + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + best_mblocks = mblocks; + best_act_threads = act_threads; + best_m_chunk = m_chunk_candidate; + best_n_chunk = n_chunk_candidate; + best_vtcm_size = exact_size; + } + } + } + if (act_threads == 1) { + act_threads = 0; + } else { + act_threads /= 2; + } + } + + if (best_act_threads > 0) { + *m_chunk_out = best_m_chunk; + *n_chunk_out = best_n_chunk; + *vtcm_size_out = best_vtcm_size; + *act_threads_out = best_act_threads; + return true; + } + return false; +} + +static inline bool htp_mm_hmx_solve_2d_params( + int wtype, + uint32_t k, + uint32_t m_id_rows, + uint32_t ne01_padded, + uint32_t ne11_padded, + uint32_t m_for_cost, + int n_threads, + bool pipeline, + bool is_matmul_id, + uint32_t aligned_tile_size, + size_t bias_size, + size_t vtcm_budget, + size_t * m_chunk_out, + size_t * n_chunk_out, + int * act_threads_out, + size_t * vtcm_size_out +) { + size_t best_mblocks = SIZE_MAX; + int best_act_threads = 0; + size_t best_m_chunk = 0; + size_t best_n_chunk = 0; + size_t best_vtcm_size = 0; + + const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded; + + int act_threads = n_threads; + while (act_threads >= 1) { + size_t simple_2d_overhead = htp_mm_hmx_get_2d_overhead(pipeline, is_matmul_id) + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0); + size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn; + htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn); + + size_t m_chunk_candidate = 0; + size_t n_chunk_candidate = 0; + size_t vtcm_size_candidate = 0; + + if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded, + (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT, + &m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) { + size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size, bias_size); + if (exact_size <= vtcm_budget) { + size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate; + if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) { + best_mblocks = mblocks; + best_act_threads = act_threads; + best_m_chunk = m_chunk_candidate; + best_n_chunk = n_chunk_candidate; + best_vtcm_size = exact_size; + } + } + } + if (act_threads == 1) { + act_threads = 0; + } else { + act_threads /= 2; + } + } + + if (best_act_threads > 0) { + *m_chunk_out = best_m_chunk; + *n_chunk_out = best_n_chunk; + *vtcm_size_out = best_vtcm_size; + *act_threads_out = best_act_threads; + return true; + } + return false; +} + +#ifdef __cplusplus +} +#endif + +#endif // HTP_MATMUL_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.c b/ggml/src/ggml-hexagon/htp/pad-ops.c new file mode 100644 index 0000000..cda6df3 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/pad-ops.c @@ -0,0 +1,588 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include + +#include + +#include "dma-queue.h" +#include "hvx-utils.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "pad-ops.h" + +/* Circular wrap: maps any integer x into [0, n) */ +static inline uint32_t wrap_around(int32_t x, uint32_t n, const struct fastdiv_values * div_n) { + if (x >= 0) { + const uint32_t ux = (uint32_t) x; + return ux < n ? ux : fastmodulo(ux, n, div_n); + } + const uint32_t ux = (uint32_t) (-x); + const uint32_t rem = ux < n ? ux : fastmodulo(ux, n, div_n); + return rem ? (n - rem) : 0; +} + +/* Decompose a flat dst row index into (i1, i2, i3) */ +static inline void pad_decompose_row(uint32_t ir, uint32_t ne1, uint32_t ne2, + const struct htp_pad_kernel_params * kparams, + uint32_t *i1, uint32_t *i2, uint32_t *i3) { + *i3 = fastdiv(ir, &kparams->div_ne1_ne2); + const uint32_t rem = ir - (*i3) * (ne1 * ne2); + *i2 = fastdiv(rem, &kparams->div_ne1); + *i1 = rem - (*i2) * ne1; +} + +/* Return non-zero if row (i1,i2,i3) falls in the non-padded interior */ +static inline int pad_is_interior(uint32_t i1, uint32_t i2, uint32_t i3, + int32_t lp1, int32_t rp1, uint32_t ne1, + int32_t lp2, int32_t rp2, uint32_t ne2, + int32_t lp3, int32_t rp3, uint32_t ne3) { + return ((int32_t)i1 >= lp1 && (int32_t)i1 < (int32_t)ne1 - rp1) && + ((int32_t)i2 >= lp2 && (int32_t)i2 < (int32_t)ne2 - rp2) && + ((int32_t)i3 >= lp3 && (int32_t)i3 < (int32_t)ne3 - rp3); +} + +/* Compute the DDR src row pointer for a zero-pad interior row */ +static inline const uint8_t * pad_src_row_ptr(const struct htp_tensor * src, + uint32_t i1, uint32_t i2, uint32_t i3, + int32_t lp1, int32_t lp2, int32_t lp3) { + return (const uint8_t *) src->data + + (i1 - (uint32_t)lp1) * src->nb[1] + + (i2 - (uint32_t)lp2) * src->nb[2] + + (i3 - (uint32_t)lp3) * src->nb[3]; +} + +static inline dma_addr_t pad_src_row_data(const struct htp_tensor * src, + uint32_t i1, uint32_t i2, uint32_t i3, + int32_t lp1, int32_t lp2, int32_t lp3) { + return src->data + + (i1 - (uint32_t)lp1) * src->nb[1] + + (i2 - (uint32_t)lp2) * src->nb[2] + + (i3 - (uint32_t)lp3) * src->nb[3]; +} + +/* Compute the DDR src row pointer for a circular row (wrap-around indexing) */ +static inline const uint8_t * pad_circ_src_row_ptr(const struct htp_tensor * src, + const struct htp_pad_kernel_params * kparams, + uint32_t i1, uint32_t i2, uint32_t i3, + int32_t lp1, int32_t lp2, int32_t lp3) { + return (const uint8_t *) src->data + + wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1] + + wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2] + + wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3]; +} + +static inline dma_addr_t pad_circ_src_row_data(const struct htp_tensor * src, + const struct htp_pad_kernel_params * kparams, + uint32_t i1, uint32_t i2, uint32_t i3, + int32_t lp1, int32_t lp2, int32_t lp3) { + return src->data + + wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1] + + wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2] + + wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3]; +} + +struct htp_pad_context { + struct htp_ops_context * octx; + const struct htp_pad_kernel_params * kparams; + + int32_t lp0, rp0; + int32_t lp1, rp1; + int32_t lp2, rp2; + int32_t lp3, rp3; + + uint32_t nrows_per_thread; + uint32_t total_dst_rows; + uint32_t row_start; + + size_t type_size; + + // Row sizes for DMA kernel (populated when VTCM is available) + size_t src_row_size; + size_t src_row_size_aligned; + size_t dst_row_size; + size_t dst_row_size_aligned; +}; + +#define htp_pad_preamble \ + const struct htp_tensor * src = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + \ + const uint32_t ne00 = src->ne[0]; \ + const uint32_t nb00 = src->nb[0]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ + const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \ + const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \ + const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \ + const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \ + \ + const size_t type_size = pctx->type_size; \ + \ + const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \ + const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows); + + +#define htp_pad_dma_preamble \ + const size_t src_row_size = pctx->src_row_size; \ + const size_t src_row_size_aligned = pctx->src_row_size_aligned; \ + const size_t dst_row_size = pctx->dst_row_size; \ + const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \ + \ + uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \ + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \ + \ + dma_queue * dma_q = octx->ctx->dma[ith]; + +// --------------------------------------------------------------------------- +// HVX vectorized PAD kernel +// --------------------------------------------------------------------------- + +static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * data) { + const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; + struct htp_ops_context * octx = pctx->octx; + htp_pad_preamble; + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); + + for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { + uint32_t i1, i2, i3; + pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3); + + uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; + + const int interior = pad_is_interior(i1, i2, i3, + lp1, rp1, ne1, + lp2, rp2, ne2, + lp3, rp3, ne3); + + if (!interior) { + hvx_splat_f32_u(dst_ptr, 0.0f, ne0); + } else { + const uint8_t * src_ptr = pad_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3); + + if (lp0 > 0) { + hvx_splat_f32_u(dst_ptr, 0.0f, (uint32_t)lp0); + } + + uint8_t * dst_row_start = dst_ptr + (size_t)lp0 * type_size; + if (nb00 == type_size) { + hvx_copy_f32_uu(dst_row_start, src_ptr, ne00); + } else { + for (uint32_t i = 0; i < ne00; i++) { + memcpy(dst_row_start + i * type_size, + src_ptr + (size_t)i * nb00, + type_size); + } + } + + if (rp0 > 0) { + hvx_splat_f32_u(dst_ptr + ((size_t)lp0 + ne00) * type_size, 0.0f, (uint32_t)rp0); + } + } + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); + + FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", + ith, nth, + src->ne[0], src->ne[1], src->ne[2], src->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + row_start, row_end); +} + +// --------------------------------------------------------------------------- +// HVX + DMA PAD kernel - aligned, double-buffered +// --------------------------------------------------------------------------- + +static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) { + const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; + struct htp_ops_context * octx = pctx->octx; + htp_pad_preamble; + htp_pad_dma_preamble; + + // ----------------------------------------------------------------------- + // Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the + // double-buffer pipeline before the main loop begins. + // ----------------------------------------------------------------------- + for (uint32_t ir = row_start, spad_idx = 0; ir < row_end && spad_idx < 2; ir++, spad_idx++) { + uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned; + uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned; + + dma_queue_push(dma_q, + dma_make_data(dst->data, dst_spad_cur), + dst_row_size, dst_row_size_aligned, dst_row_size, 0); + + uint32_t i1, i2, i3; + pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3); + const int interior = pad_is_interior(i1, i2, i3, + lp1, rp1, ne1, + lp2, rp2, ne2, + lp3, rp3, ne3); + + const dma_addr_t src_data = interior + ? pad_src_row_data(src, i1, i2, i3, lp1, lp2, lp3) : src->data; + + // Interior row: real DMA (1 row) from DDR to VTCM. + // Border row: null DMA (nrows=0) + dma_queue_push(dma_q, + dma_make_data(src_spad_cur, src_data), + src_row_size_aligned, src_row_size, src_row_size, interior ? 1 : 0); + } + + // ----------------------------------------------------------------------- + // Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops, + // push dst DMA and prefetch src for the next+1 row. + // ----------------------------------------------------------------------- + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + for (uint32_t ir = row_start; ir < row_end; ir++) { + uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst; + + uint32_t i1, i2, i3; + pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3); + + const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; + + const int interior = pad_is_interior(i1, i2, i3, + lp1, rp1, ne1, + lp2, rp2, ne2, + lp3, rp3, ne3); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + if (!interior) { + hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0); + } else { + hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0); + + uint8_t * dst_interior = dst_spad_cur + (size_t)lp0 * type_size; + + if ((uintptr_t)dst_interior % VLEN == 0) { + hvx_copy_f32_aa(dst_interior, src_spad_cur, ne00); + } else { + hvx_copy_f32_ua(dst_interior, src_spad_cur, ne00); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + + dma_queue_push(dma_q, + dma_make_data(dst_data, dst_spad_cur), + dst_row_size, dst_row_size_aligned, dst_row_size, 1); + + const uint32_t next_row = ir + 2; + if (next_row < row_end) { + uint32_t ni1, ni2, ni3; + pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &ni1, &ni2, &ni3); + const int next_interior = pad_is_interior(ni1, ni2, ni3, + lp1, rp1, ne1, + lp2, rp2, ne2, + lp3, rp3, ne3); + const dma_addr_t next_src_data = next_interior + ? pad_src_row_data(src, ni1, ni2, ni3, lp1, lp2, lp3) : src->data; + + dma_queue_push(dma_q, + dma_make_data(src_spad_cur, next_src_data), + src_row_size_aligned, src_row_size, src_row_size, next_interior ? 1 : 0); + } + } + + dma_queue_flush(dma_q); + + FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", + ith, nth, + src->ne[0], src->ne[1], src->ne[2], src->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + row_start, row_end); +} + +// --------------------------------------------------------------------------- +// HVX circular PAD kernel +// --------------------------------------------------------------------------- + +static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, void * data) { + const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; + struct htp_ops_context * octx = pctx->octx; + htp_pad_preamble; + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); + + for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { + uint32_t i1, i2, i3; + pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3); + + uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; + const uint8_t * src_row = pad_circ_src_row_ptr(src, pctx->kparams, i1, i2, i3, lp1, lp2, lp3); + + if (nb00 == type_size) { + + if (lp0 > 0) { + if ((uint32_t)lp0 < 32) { + memcpy(dst_ptr, + src_row + (size_t)(ne00 - (uint32_t)lp0) * type_size, + (size_t)lp0 * type_size); + } else { + hvx_copy_f32_uu(dst_ptr, + src_row + (size_t)(ne00 - (uint32_t)lp0) * type_size, + (uint32_t)lp0); + } + } + hvx_copy_f32_uu(dst_ptr + (size_t)lp0 * type_size, src_row, ne00); + if (rp0 > 0) { + if ((uint32_t)rp0 < 32) { + memcpy(dst_ptr + ((size_t)lp0 + ne00) * type_size, + src_row, + (size_t)rp0 * type_size); + } else { + hvx_copy_f32_uu(dst_ptr + ((size_t)lp0 + ne00) * type_size, + src_row, + (uint32_t)rp0); + } + } + } else { + for (uint32_t i = 0; i < (uint32_t)lp0; i++) { + *(float *)(dst_ptr + i * type_size) = + *(const float *)(src_row + (size_t)(ne00 - (uint32_t)lp0 + i) * nb00); + } + for (uint32_t i = 0; i < ne00; i++) { + *(float *)(dst_ptr + ((size_t)lp0 + i) * type_size) = + *(const float *)(src_row + (size_t)i * nb00); + } + for (uint32_t i = 0; i < (uint32_t)rp0; i++) { + *(float *)(dst_ptr + ((size_t)lp0 + ne00 + i) * type_size) = + *(const float *)(src_row + (size_t)i * nb00); + } + } + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); + + FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", + ith, nth, + src->ne[0], src->ne[1], src->ne[2], src->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + row_start, row_end); +} + +// --------------------------------------------------------------------------- +// HVX + DMA circular PAD kernel - aligned, double-buffered +// --------------------------------------------------------------------------- + +static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) { + const struct htp_pad_context * pctx = (const struct htp_pad_context *) data; + struct htp_ops_context * octx = pctx->octx; + htp_pad_preamble; + htp_pad_dma_preamble; + + // ----------------------------------------------------------------------- + // Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the + // double-buffer pipeline. Every row is a real src DMA (no null DMAs). + // ----------------------------------------------------------------------- + for (uint32_t ir = row_start, spad_idx = 0; ir < row_end && spad_idx < 2; ir++, spad_idx++) { + uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned; + uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned; + + dma_queue_push(dma_q, + dma_make_data(dst->data, dst_spad_cur), + dst_row_size, dst_row_size_aligned, dst_row_size, 0); + + uint32_t pi1, pi2, pi3; + pad_decompose_row(ir, ne1, ne2, pctx->kparams, &pi1, &pi2, &pi3); + const dma_addr_t src_data = pad_circ_src_row_data(src, pctx->kparams, pi1, pi2, pi3, lp1, lp2, lp3); + dma_queue_push(dma_q, + dma_make_data(src_spad_cur, src_data), + src_row_size_aligned, src_row_size, src_row_size, 1); + } + + // ----------------------------------------------------------------------- + // Main loop: pop completed DMAs, assemble circular row in VTCM with + // aligned HVX ops, push dst DMA and prefetch src for the next+1 row. + // ----------------------------------------------------------------------- + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + for (uint32_t ir = row_start; ir < row_end; ir++) { + uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src; + uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst; + + uint32_t i1, i2, i3; + pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3); + const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + if (lp0 > 0) { + uint8_t * dst_left = dst_spad_cur; + const uint8_t * src_left = src_spad_cur + (size_t)(ne00 - (uint32_t)lp0) * type_size; + if ((uint32_t)lp0 < 32) { + memcpy(dst_left, src_left, (size_t)lp0 * type_size); + } else { + hvx_copy_f32_uu(dst_left, src_left, (uint32_t)lp0); + } + } + + { + uint8_t * dst_mid = dst_spad_cur + (size_t)lp0 * type_size; + if ((uintptr_t)dst_mid % VLEN == 0) { + hvx_copy_f32_aa(dst_mid, src_spad_cur, ne00); + } else { + hvx_copy_f32_ua(dst_mid, src_spad_cur, ne00); + } + } + + if (rp0 > 0) { + uint8_t * dst_right = dst_spad_cur + ((size_t)lp0 + ne00) * type_size; + if ((uint32_t)rp0 < 32) { + memcpy(dst_right, src_spad_cur, (size_t)rp0 * type_size); + } else { + if ((uintptr_t)dst_right % VLEN == 0) { + hvx_copy_f32_aa(dst_right, src_spad_cur, (uint32_t)rp0); + } else { + hvx_copy_f32_ua(dst_right, src_spad_cur, (uint32_t)rp0); + } + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); + + dma_queue_push(dma_q, + dma_make_data(dst_data, dst_spad_cur), + dst_row_size, dst_row_size_aligned, dst_row_size, 1); + + const uint32_t next_row = ir + 2; + if (next_row < row_end) { + uint32_t nri1, nri2, nri3; + pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &nri1, &nri2, &nri3); + const dma_addr_t next_src_data = pad_circ_src_row_data(src, pctx->kparams, nri1, nri2, nri3, lp1, lp2, lp3); + dma_queue_push(dma_q, + dma_make_data(src_spad_cur, next_src_data), + src_row_size_aligned, src_row_size, src_row_size, 1); + } + } + + dma_queue_flush(dma_q); + + FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", + ith, nth, + src->ne[0], src->ne[1], src->ne[2], src->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + row_start, row_end); +} + +int op_pad(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + const struct htp_pad_kernel_params * kparams = + (const struct htp_pad_kernel_params *) octx->kernel_params; + + const int32_t lp0 = octx->op_params[0]; + const int32_t rp0 = octx->op_params[1]; + const int32_t lp1 = octx->op_params[2]; + const int32_t rp1 = octx->op_params[3]; + const int32_t lp2 = octx->op_params[4]; + const int32_t rp2 = octx->op_params[5]; + const int32_t lp3 = octx->op_params[6]; + const int32_t rp3 = octx->op_params[7]; + const int32_t circular = octx->op_params[8]; + + const uint32_t total_dst_rows = kparams->total_rows; + const size_t type_size = kparams->elem_size; + const size_t src_row_size = src0->ne[0] * type_size; + const size_t dst_row_size = dst->ne[0] * type_size; + + uint32_t row_start = 0; + uint32_t nrows = total_dst_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + + if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { + return HTP_STATUS_NO_SUPPORT; + } + + struct htp_pad_vtcm_layout layout = { 0 }; + if (kparams->use_dma) { + htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads); + + octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread; + octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread; + octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread; + octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread; + octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0); + octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst); + octx->src0_spad.src = NULL; + octx->dst_spad.src = NULL; + } + + struct htp_pad_context pctx = { + .octx = octx, + .kparams = kparams, + .lp0 = lp0, .rp0 = rp0, + .lp1 = lp1, .rp1 = rp1, + .lp2 = lp2, .rp2 = rp2, + .lp3 = lp3, .rp3 = rp3, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_dst_rows = nrows, + .row_start = row_start, + .type_size = type_size, + .src_row_size = src_row_size, + .src_row_size_aligned = layout.src_row_size_aligned, + .dst_row_size = dst_row_size, + .dst_row_size_aligned = layout.dst_row_size_aligned, + }; + + FARF(HIGH, "pad-hvx%s%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) pads=(%d,%d,%d,%d,%d,%d,%d,%d)\n", + circular ? "-circ" : "", + kparams->use_dma ? "-dma" : "", + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3); + + switch (kparams->kernel_type) { + case HTP_PAD_KERNEL_HVX_CIRCULAR_DMA: + work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); + break; + case HTP_PAD_KERNEL_HVX_CIRCULAR: + work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); + break; + case HTP_PAD_KERNEL_HVX_DMA: + work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); + break; + default: + work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); + break; + } + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/pad-ops.h b/ggml/src/ggml-hexagon/htp/pad-ops.h new file mode 100644 index 0000000..5b85575 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/pad-ops.h @@ -0,0 +1,65 @@ +#ifndef HTP_PAD_OPS_H +#define HTP_PAD_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include + +enum htp_pad_kernel_type { + HTP_PAD_KERNEL_HVX = 0, + HTP_PAD_KERNEL_HVX_DMA = 1, + HTP_PAD_KERNEL_HVX_CIRCULAR = 2, + HTP_PAD_KERNEL_HVX_CIRCULAR_DMA = 3, +}; + +struct htp_pad_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + size_t src_row_size_aligned; + size_t dst_row_size_aligned; + size_t src0_bytes_per_thread; + size_t dst_bytes_per_thread; +}; + +static inline void htp_pad_vtcm_layout_build( + struct htp_pad_vtcm_layout * L, + size_t src_row_size, + size_t dst_row_size, + uint32_t n_threads +) { + L->src_row_size_aligned = hex_round_up(src_row_size, 128); + L->dst_row_size_aligned = hex_round_up(dst_row_size, 128); + L->src0_bytes_per_thread = 2 * L->src_row_size_aligned; + L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + + L->total_bytes = off; +} + +struct htp_pad_kernel_params { + uint8_t kernel_type; + uint32_t n_threads; + uint32_t total_rows; + uint32_t use_dma; + uint32_t elem_size; + uint32_t vtcm_size; + + struct fastdiv_values div_ne1; + struct fastdiv_values div_ne1_ne2; + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne03; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_PAD_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.c b/ggml/src/ggml-hexagon/htp/pool-ops.c new file mode 100644 index 0000000..b41c7be --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/pool-ops.c @@ -0,0 +1,390 @@ +#pragma clang diagnostic ignored "-Wunused-variable" + +#include +#include + +#include "hex-common.h" +#include "dma-queue.h" +#include "hex-profile.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-inverse.h" +#include "hvx-types.h" +#include "hvx-utils.h" +#include "pool-ops.h" + +#define HTP_POOL_MAX 0 +#define HTP_POOL_AVG 1 + +// Fast path: exact non-overlapping tiling (stride == kernel, no padding), kernel_x in {1,2}. +// Every window is guaranteed fully in-bounds, so this never needs boundary clamping. +static void pool_plane_hvx( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + const bool is_max = (p->pool_op == HTP_POOL_MAX); + const HVX_Vector scale = hvx_vec_splat_f32(p->inv_kernel_area); + const HVX_Vector seed = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + + for (uint32_t oy = 0; oy < p->dst_y; ++oy) { + const uint32_t sy = oy * p->kernel_y; + for (uint32_t ox = 0; ox < p->dst_x; ox += VLEN_FP32) { + const uint32_t rem = p->dst_x - ox; + const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN; + HVX_Vector acc = seed; + for (uint32_t ky = 0; ky < p->kernel_y; ++ky) { + const float * row = src + (sy + ky) * p->src_x; + if (p->kernel_x == 2) { + const HVX_Vector v0 = *(const HVX_UVector *) (row + ox * 2); + const HVX_Vector v1 = *(const HVX_UVector *) (row + ox * 2 + VLEN_FP32); + const HVX_VectorPair deinterleaved = Q6_W_vdeal_VVR(v1, v0, -4); + const HVX_Vector lo = Q6_V_lo_W(deinterleaved); + const HVX_Vector hi = Q6_V_hi_W(deinterleaved); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, Q6_Vsf_vmax_VsfVsf(lo, hi)) + : hvx_vec_add_f32_f32(acc, hvx_vec_add_f32_f32(lo, hi)); + } else if (p->kernel_x == 1) { + const HVX_Vector v = *(const HVX_UVector *) (row + ox); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) : hvx_vec_add_f32_f32(acc, v); + } + } + hvx_vec_store_u(dst + oy * p->dst_x + ox, nbytes, is_max ? acc : hvx_vec_mul_f32_f32(acc, scale)); + } + } +} + +// Narrow exact-tiling path. The input is staged in VTCM with one vector of +// guard space, so full-width loads are safe even when src_x is below 32. +static void pool_plane_hvx_narrow( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + const bool is_max = (p->pool_op == HTP_POOL_MAX); + const HVX_Vector scale = hvx_vec_splat_f32(p->inv_kernel_area); + const HVX_Vector seed = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + + for (uint32_t oy = 0; oy < p->dst_y; ++oy) { + const uint32_t sy = oy * p->kernel_y; + HVX_Vector acc = seed; + for (uint32_t ky = 0; ky < p->kernel_y; ++ky) { + const float * row = src + (sy + ky) * p->src_x; + const HVX_Vector v0 = *(const HVX_UVector *) row; + HVX_Vector v = v0; + if (p->kernel_x == 2) { + // The second vector is zero because a narrow row has fewer + // than 32 input elements. The low lanes still contain the + // complete even/odd pairs needed by the output. + const HVX_Vector zero = Q6_V_vsplat_R(0); + const HVX_VectorPair deinterleaved = Q6_W_vdeal_VVR(zero, v0, -4); + const HVX_Vector even = Q6_V_lo_W(deinterleaved); + const HVX_Vector odd = Q6_V_hi_W(deinterleaved); + v = is_max ? Q6_Vsf_vmax_VsfVsf(even, odd) + : hvx_vec_add_f32_f32(even, odd); + } + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) + : hvx_vec_add_f32_f32(acc, v); + } + hvx_vec_store_u(dst + oy * p->dst_x, p->dst_x * sizeof(float), + is_max ? acc : hvx_vec_mul_f32_f32(acc, scale)); + } +} + +static void pool_plane_global( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + const uint32_t n = p->src_x * p->src_y; + float val; + if (p->pool_op == HTP_POOL_MAX) { + val = hvx_reduce_max_f32((const uint8_t *) src, n); + } else { + val = hvx_reduce_sum_f32((const uint8_t *) src, n) * p->inv_kernel_area; + } + hvx_vec_store_u(dst, sizeof(float), hvx_vec_splat_f32(val)); +} + +static void pool_plane_block( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + for (uint32_t oy = 0; oy < p->dst_y; ++oy) { + const float * row = src + oy * p->src_x; + float * dst_row = dst + oy * p->dst_x; + for (uint32_t ox = 0; ox < p->dst_x; ++ox) { + const uint8_t * block = (const uint8_t *) (row + ox * p->kernel_x); + float val; + if (p->pool_op == HTP_POOL_MAX) { + val = hvx_reduce_max_f32(block, p->kernel_x); + } else { + val = hvx_reduce_sum_f32(block, p->kernel_x) * p->inv_kernel_area; + } + hvx_vec_store_u(dst_row + ox, sizeof(float), hvx_vec_splat_f32(val)); + } + } +} + +// General path: arbitrary kernel/stride/padding + +// Vertical padding is uniform across a whole output row (iy0 depends only on oy, not ox), +// so it collapses to one valid-ky range per row instead of a per-element check. +static inline void pool_row_bounds_y( + const struct htp_pool_2d_kernel_params * p, uint32_t oy, int32_t * iy0, uint32_t * ky_lo, uint32_t * ky_hi) { + *iy0 = (int32_t) (oy * p->stride_y) - p->pad_y; + const int32_t lo = -(*iy0); + const int32_t hi = (int32_t) p->src_y - *iy0; + *ky_lo = (uint32_t) MAX(0, lo); + *ky_hi = (uint32_t) MAX(0, MIN((int32_t) p->kernel_y, hi)); +} + +static inline void pool_pixel_boundary_vec( + const float * src, float * dst_row, const struct htp_pool_2d_kernel_params * p, + uint32_t ox, int32_t iy0, uint32_t ky_lo, uint32_t ky_hi, bool is_max) { + const int32_t ix0 = (int32_t) (ox * p->stride_x) - p->pad_x; + const int32_t kx_lo = MAX(0, -ix0); + const int32_t kx_hi = MIN((int32_t) p->kernel_x, (int32_t) p->src_x - ix0); + + if (kx_lo >= kx_hi || ky_lo >= ky_hi) { + HVX_Vector empty_val = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + hvx_vec_store_u(dst_row + ox, sizeof(float), empty_val); + return; + } + + const uint32_t valid_kx = (uint32_t) (kx_hi - kx_lo); + const HVX_Vector mask_identity = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + + HVX_Vector acc = mask_identity; + for (uint32_t ky = ky_lo; ky < ky_hi; ++ky) { + const float * row = src + (uint32_t) (iy0 + (int32_t) ky) * p->src_x; + for (uint32_t k = 0; k < valid_kx; k += VLEN_FP32) { + const uint32_t k_rem = valid_kx - k; + const uint32_t n = (k_rem < VLEN_FP32) ? k_rem : VLEN_FP32; + const HVX_VectorPred q = Q6_Q_vsetq_R(n * sizeof(float)); + const HVX_Vector raw = *(const HVX_UVector *) (row + ix0 + kx_lo + k); + const HVX_Vector v = Q6_V_vmux_QVV(q, raw, mask_identity); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) : hvx_vec_add_f32_f32(acc, v); + } + } + + HVX_Vector reduced = is_max ? hvx_vec_reduce_max_f32(acc) : hvx_vec_reduce_sum_f32(acc); + if (!is_max) { + HVX_Vector scale_vec; + if (p->avg_divide_count) { + const uint32_t count = (ky_hi - ky_lo) * valid_kx; + scale_vec = hvx_vec_inverse_f32(hvx_vec_splat_f32((float) count)); + } else { + scale_vec = hvx_vec_splat_f32(p->inv_kernel_area); + } + reduced = hvx_vec_mul_f32_f32(reduced, scale_vec); + } + hvx_vec_store_u(dst_row + ox, sizeof(float), reduced); +} + +static inline void pool_row_general_boundary_vec( + const float * src, float * dst_row, const struct htp_pool_2d_kernel_params * p, + uint32_t ox_start, uint32_t ox_end, int32_t iy0, uint32_t ky_lo, uint32_t ky_hi, bool is_max) { + for (uint32_t ox = ox_start; ox < ox_end; ++ox) { + pool_pixel_boundary_vec(src, dst_row, p, ox, iy0, ky_lo, ky_hi, is_max); + } +} + +// Vectorized interior loop. +static inline void pool_row_general_vec( + const float * src, float * dst_row, const struct htp_pool_2d_kernel_params * p, + uint32_t ox_start, uint32_t ox_end, int32_t iy0, uint32_t ky_lo, uint32_t ky_hi, bool is_max) { + if (ox_start >= ox_end) { + return; + } + + if (p->stride_x != 1 && p->stride_x != 2) { + pool_row_general_boundary_vec(src, dst_row, p, ox_start, ox_end, iy0, ky_lo, ky_hi, is_max); + return; + } + + const HVX_Vector scale = hvx_vec_splat_f32(p->inv_kernel_area); + const HVX_Vector seed = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0); + + for (uint32_t ox = ox_start; ox < ox_end; ox += VLEN_FP32) { + const uint32_t rem = ox_end - ox; + const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN; + const int32_t ix0 = (int32_t) (ox * p->stride_x) - p->pad_x; + HVX_Vector acc = seed; + + for (uint32_t ky = ky_lo; ky < ky_hi; ++ky) { + const float * row = src + (uint32_t) (iy0 + (int32_t) ky) * p->src_x; + if (p->stride_x == 1) { + for (uint32_t kx = 0; kx < p->kernel_x; ++kx) { + const HVX_Vector v = *(const HVX_UVector *) (row + ix0 + (int32_t) kx); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) : hvx_vec_add_f32_f32(acc, v); + } + } else { + for (uint32_t kx = 0; kx < p->kernel_x; kx += 2) { + const HVX_Vector v0 = *(const HVX_UVector *) (row + ix0 + (int32_t) kx); + const HVX_Vector v1 = *(const HVX_UVector *) (row + ix0 + (int32_t) kx + VLEN_FP32); + const HVX_VectorPair deinterleaved = Q6_W_vdeal_VVR(v1, v0, -4); + const HVX_Vector lo = Q6_V_lo_W(deinterleaved); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, lo) : hvx_vec_add_f32_f32(acc, lo); + if (kx + 1 < p->kernel_x) { + const HVX_Vector hi = Q6_V_hi_W(deinterleaved); + acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, hi) : hvx_vec_add_f32_f32(acc, hi); + } + } + } + } + hvx_vec_store_u(dst_row + ox, nbytes, is_max ? acc : hvx_vec_mul_f32_f32(acc, scale)); + } +} + +static void pool_plane_general( + const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) { + const bool is_max = (p->pool_op == HTP_POOL_MAX); + + for (uint32_t oy = 0; oy < p->dst_y; ++oy) { + int32_t iy0; + uint32_t ky_lo, ky_hi; + pool_row_bounds_y(p, oy, &iy0, &ky_lo, &ky_hi); + float * dst_row = dst + oy * p->dst_x; + + pool_row_general_boundary_vec(src, dst_row, p, 0, p->ox_lo, iy0, ky_lo, ky_hi, is_max); + pool_row_general_vec(src, dst_row, p, p->ox_lo, p->ox_hi, iy0, ky_lo, ky_hi, is_max); + pool_row_general_boundary_vec(src, dst_row, p, p->ox_hi, p->dst_x, iy0, ky_lo, ky_hi, is_max); + } +} + +typedef void (*pool_plane_fn_t)(const float * src, float * dst, const struct htp_pool_2d_kernel_params * p); + +struct pool_2d_context { + struct htp_ops_context * octx; + const struct htp_pool_2d_kernel_params * kparams; + struct htp_pool_vtcm_layout layout; + pool_plane_fn_t pool_plane; + uint32_t n_threads; + uint32_t plane_start; + uint32_t plane_count; + uint32_t planes_per_thread; +}; + +static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) { + struct pool_2d_context * ctx = (struct pool_2d_context *) data; + const struct htp_pool_2d_kernel_params * p = ctx->kparams; + const struct htp_pool_vtcm_layout * L = &ctx->layout; + const struct htp_tensor * src0 = ctx->octx->src[0]; + const struct htp_tensor * dst = ctx->octx->dst; + pool_plane_fn_t pool_plane = ctx->pool_plane; + const uint32_t planes_per_thread = ctx->planes_per_thread; + const uint32_t first = ctx->plane_start + ith * planes_per_thread; + const uint32_t last = MIN(first + planes_per_thread, ctx->plane_start + ctx->plane_count); + + if (first >= last) { + return; + } + + struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith]; + dma_queue * dma_queue = ctx->octx->ctx->dma[ith]; + + const uint32_t src_plane_bytes = L->src_plane_bytes; + const uint32_t dst_plane_bytes = L->dst_plane_bytes; + const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size; + const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size; + + uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base; + uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread; + uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread; + + float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) }; + float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) }; + + const uint32_t total = last - first; + + // Warm up the pipeline: push up to 2 initial (dummy dst, src) transfer pairs. + for (uint32_t i = 0; i < total && i < 2; ++i) { + dma_queue_push(dma_queue, + dma_make_data(dst->data, dstb2[i]), + dst_plane_bytes, dst_spad_half, + dst_plane_bytes, 0); + + const dma_addr_t src_addr = src0->data + (first + i) * src_plane_bytes; + dma_queue_push(dma_queue, + dma_make_data(srcb2[i], src_addr), + src_spad_half, src_plane_bytes, + src_plane_bytes, 1); + } + + for (uint32_t i = 0; i < total; ++i) { + const uint32_t plane = first + i; + const uint32_t buf = i & 1u; + float * srcb = srcb2[buf]; + float * dstb = dstb2[buf]; + + dma_queue_pop(dma_queue); // dst writeback from plane i - 2 (or dummy on iter 0, 1) + dma_queue_pop(dma_queue); // input for plane i + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) plane); + pool_plane(srcb, dstb, p); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) plane); + + const dma_addr_t dst_addr = dst->data + plane * dst_plane_bytes; + dma_queue_push(dma_queue, + dma_make_data(dst_addr, dstb), + dst_plane_bytes, dst_spad_half, + dst_plane_bytes, 1); + + if (i + 2 < total) { + const dma_addr_t next_src_addr = src0->data + (plane + 2) * src_plane_bytes; + dma_queue_push(dma_queue, + dma_make_data(srcb, next_src_addr), + src_spad_half, src_plane_bytes, + src_plane_bytes, 1); + } + } + + dma_queue_flush(dma_queue); + + FARF(HIGH, "pool2d-f32-dma %d/%d: %ux%ux%ux%u -> %ux%ux%ux%u (%u:%u)\n", + ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], first, last); + (void) nth; +} + +int op_pool_2d(struct htp_ops_context * octx) { + const struct htp_tensor * dst = octx->dst; + const struct htp_pool_2d_kernel_params * p = + (const struct htp_pool_2d_kernel_params *) octx->kernel_params; + + uint32_t plane_start = 0; + uint32_t plane_count = p->planes; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + plane_count, htp_tensor_mdev_data_aligned(dst) ? p->planes_per_chunk : 0, + octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + plane_start = range.start; + plane_count = range.count; + } + if (plane_count == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = MIN(p->n_threads, plane_count); + if (!htp_ops_context_set_n_threads(octx, n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t planes_per_thread = fastdiv(plane_count + n_threads - 1, &octx->n_threads_div); + + pool_plane_fn_t pool_plane; + switch (p->kernel_type) { + case HTP_POOL_KERNEL_GLOBAL: pool_plane = pool_plane_global; break; + case HTP_POOL_KERNEL_BLOCK: pool_plane = pool_plane_block; break; + case HTP_POOL_KERNEL_HVX_NARROW: pool_plane = pool_plane_hvx_narrow; break; + case HTP_POOL_KERNEL_HVX: pool_plane = pool_plane_hvx; break; + default: pool_plane = pool_plane_general; break; + } + + struct pool_2d_context ctx = { + .octx = octx, + .kparams = p, + .pool_plane = pool_plane, + .n_threads = n_threads, + .plane_start = plane_start, + .plane_count = plane_count, + .planes_per_thread = planes_per_thread, + }; + htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_threads); + work_queue_run(octx->ctx->work_queue, pool_2d_thread, &ctx, n_threads); + return HTP_STATUS_OK; +} + +int op_pool_1d(struct htp_ops_context * octx) { + return op_pool_2d(octx); +} diff --git a/ggml/src/ggml-hexagon/htp/pool-ops.h b/ggml/src/ggml-hexagon/htp/pool-ops.h new file mode 100644 index 0000000..178e5d0 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/pool-ops.h @@ -0,0 +1,95 @@ +#ifndef HTP_POOL_OPS_H +#define HTP_POOL_OPS_H + +#include +#include +#include +#include + +#include "hex-common.h" +#include "htp-vtcm.h" + +enum htp_pool_kernel_type { + HTP_POOL_KERNEL_GENERAL = 0, + HTP_POOL_KERNEL_HVX, + HTP_POOL_KERNEL_HVX_NARROW, + HTP_POOL_KERNEL_GLOBAL, + HTP_POOL_KERNEL_BLOCK, +}; + +struct htp_pool_2d_kernel_params { + uint8_t kernel_type; + uint32_t src_x; + uint32_t src_y; + uint32_t dst_x; + uint32_t dst_y; + uint32_t kernel_x; + uint32_t kernel_y; + uint32_t stride_x; + uint32_t stride_y; + int32_t pad_x; + int32_t pad_y; + uint32_t n_threads; + uint32_t planes; + uint32_t planes_per_chunk; + uint32_t pool_op; + uint32_t avg_divide_count; + uint32_t ox_lo; + uint32_t ox_hi; + float inv_kernel_area; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_pool_2d_kernel_params) <= 128, "htp_pool_2d_kernel_params is too large"); +#else +_Static_assert(sizeof(struct htp_pool_2d_kernel_params) <= 128, "htp_pool_2d_kernel_params is too large"); +#endif + +struct htp_pool_vtcm_layout { + size_t total_bytes; + size_t off_src; + size_t off_dst; + uint32_t src_plane_bytes; + uint32_t dst_plane_bytes; + size_t src_bytes_per_thread; + size_t dst_bytes_per_thread; + size_t src_spad_half_size; + size_t dst_spad_half_size; +}; + +static inline void htp_pool_vtcm_layout_build( + struct htp_pool_vtcm_layout * L, + uint32_t src_x, + uint32_t src_y, + uint32_t dst_x, + uint32_t dst_y, + uint32_t n_threads +) { + // Full-plane double buffering (256 bytes guard space for vector loads) + L->src_plane_bytes = src_x * src_y * sizeof(float); + L->dst_plane_bytes = dst_x * dst_y * sizeof(float); + L->src_spad_half_size = hex_round_up(L->src_plane_bytes + 256, 128); + L->dst_spad_half_size = hex_round_up(L->dst_plane_bytes, 128); + L->src_bytes_per_thread = 2 * L->src_spad_half_size; + L->dst_bytes_per_thread = 2 * L->dst_spad_half_size; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + L->total_bytes = off; +} + +static inline bool htp_pool_solve_layout( + struct htp_pool_vtcm_layout * layout, + uint32_t src_x, + uint32_t src_y, + uint32_t dst_x, + uint32_t dst_y, + uint32_t n_threads, + size_t vtcm_budget +) { + htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_threads); + return layout->total_bytes <= vtcm_budget; +} + +#endif // HTP_POOL_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.c b/ggml/src/ggml-hexagon/htp/repeat-ops.c index a6f2f0e..827ef24 100644 --- a/ggml/src/ggml-hexagon/htp/repeat-ops.c +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.c @@ -12,19 +12,19 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" +#include "repeat-ops.h" struct htp_repeat_context { - struct htp_ops_context * octx; - - uint32_t nr0; - uint32_t nr1; - uint32_t nr2; - uint32_t nr3; + struct htp_ops_context * octx; + const struct htp_repeat_kernel_params * kparams; uint32_t nrows_per_thread; uint32_t total_dst_rows; // ne1 * ne2 * ne3 + uint32_t row_start; size_t type_size; }; @@ -32,6 +32,7 @@ struct htp_repeat_context { static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * data) { const struct htp_repeat_context * rctx = (const struct htp_repeat_context *) data; struct htp_ops_context * octx = rctx->octx; + const struct htp_repeat_kernel_params * kparams = rctx->kparams; const struct htp_tensor * src = octx->src[0]; const struct htp_tensor * dst = octx->dst; @@ -55,29 +56,30 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat const uint32_t nb2 = dst->nb[2]; const uint32_t nb3 = dst->nb[3]; - const uint32_t nr0 = rctx->nr0; - const uint32_t nr1 = rctx->nr1; - const uint32_t nr2 = rctx->nr2; - const uint32_t nr3 = rctx->nr3; + const uint32_t nr0 = kparams->nr0; const size_t row_bytes = ne00 * rctx->type_size; - const uint32_t row_start = rctx->nrows_per_thread * ith; - const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->total_dst_rows); + const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows); + if (row_start >= row_end) { + return; + } - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) { // Decompose flat dst row index into (i1, i2, i3) - const uint32_t i1 = dst_row % ne1; - const uint32_t i2 = (dst_row / ne1) % ne2; - const uint32_t i3 = dst_row / (ne1 * ne2); + const uint32_t dst_row_div_ne1 = fastdiv(dst_row, &kparams->div_ne1); + const uint32_t i1 = dst_row - dst_row_div_ne1 * ne1; + const uint32_t i3 = fastdiv(dst_row, &kparams->div_ne12); + const uint32_t i2 = dst_row_div_ne1 - i3 * ne2; // Map to source indices (tiling) - const uint32_t k1 = i1 % ne01; - const uint32_t k2 = i2 % ne02; - const uint32_t k3 = i3 % ne03; + const uint32_t k1 = fastmodulo(i1, ne01, &kparams->div_ne01); + const uint32_t k2 = fastmodulo(i2, ne02, &kparams->div_ne02); + const uint32_t k3 = fastmodulo(i3, ne03, &kparams->div_ne03); const uint8_t * src_row = (const uint8_t *) src->data + k1 * nb01 + k2 * nb02 + k3 * nb03; uint8_t * dst_base = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3; @@ -89,27 +91,18 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat } } - t2 = HAP_perf_get_qtimer_count(); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); - FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n", + FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n", ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - row_start, row_end, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + row_start, row_end); } int op_repeat(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * dst = octx->dst; - // Validate that dst dims are multiples of src dims - if (dst->ne[0] % src0->ne[0] != 0 || - dst->ne[1] % src0->ne[1] != 0 || - dst->ne[2] % src0->ne[2] != 0 || - dst->ne[3] % src0->ne[3] != 0) { - FARF(ERROR, "repeat: dst dims must be multiples of src dims\n"); - return HTP_STATUS_INVAL_PARAMS; - } - size_t type_size; switch (src0->type) { case HTP_TYPE_F32: type_size = 4; break; @@ -119,30 +112,52 @@ int op_repeat(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows); + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_repeat_kernel_params * kparams = + (const struct htp_repeat_kernel_params *) octx->kernel_params; - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3]; + const size_t dst_row_size = dst->ne[0] * type_size; + + uint32_t row_start = 0; + uint32_t nrows = total_dst_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { return HTP_STATUS_OK; } + const uint32_t n_threads = octx->n_threads; + struct htp_repeat_context rctx = { .octx = octx, - .nr0 = dst->ne[0] / src0->ne[0], - .nr1 = dst->ne[1] / src0->ne[1], - .nr2 = dst->ne[2] / src0->ne[2], - .nr3 = dst->ne[3] / src0->ne[3], - .nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads, - .total_dst_rows = total_dst_rows, + .kparams = kparams, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .total_dst_rows = nrows, + .row_start = row_start, .type_size = type_size, }; FARF(HIGH, "repeat: (%ux%ux%ux%u) -> (%ux%ux%ux%u) nr=(%u,%u,%u,%u)\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - rctx.nr0, rctx.nr1, rctx.nr2, rctx.nr3); + kparams->nr0, kparams->nr1, kparams->nr2, kparams->nr3); - worker_pool_run_func(octx->ctx->worker_pool, repeat_job_per_thread, &rctx, n_threads); + work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/repeat-ops.h b/ggml/src/ggml-hexagon/htp/repeat-ops.h new file mode 100644 index 0000000..47759e0 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/repeat-ops.h @@ -0,0 +1,27 @@ +#ifndef HTP_REPEAT_OPS_H +#define HTP_REPEAT_OPS_H + +#include "hex-fastdiv.h" +#include + +struct htp_repeat_kernel_params { + uint32_t n_threads; + uint32_t nr0; + uint32_t nr1; + uint32_t nr2; + uint32_t nr3; + + struct fastdiv_values div_ne1; + struct fastdiv_values div_ne12; + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne03; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_REPEAT_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.c b/ggml/src/ggml-hexagon/htp/roll-ops.c new file mode 100644 index 0000000..77ded0f --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/roll-ops.c @@ -0,0 +1,304 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include + +#include + +#include "dma-queue.h" +#include "hvx-utils.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "hex-common.h" +#include "hex-profile.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "roll-ops.h" + +struct htp_roll_context { + struct htp_ops_context * octx; + const struct htp_roll_kernel_params * kparams; + + uint32_t row_start; + uint32_t nrows; + uint32_t nrows_per_thread; +}; + +static inline uint32_t htp_roll_wrap(int32_t i, uint32_t ne) { + if (i < 0) { + return (uint32_t) (i + (int32_t) ne); + } + if ((uint32_t) i >= ne) { + return (uint32_t) i - ne; + } + return (uint32_t) i; +} + +#define htp_roll_preamble \ + const struct htp_tensor * src0 = octx->src[0]; \ + const struct htp_tensor * dst = octx->dst; \ + const struct htp_roll_kernel_params * kparams = \ + (const struct htp_roll_kernel_params *) octx->kernel_params; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ + \ + const int32_t s0 = octx->op_params[0]; \ + const int32_t s1 = octx->op_params[1]; \ + const int32_t s2 = octx->op_params[2]; \ + const int32_t s3 = octx->op_params[3]; \ + \ + const uint32_t i0_src0 = kparams->i0_src0; \ + const uint32_t n0 = kparams->n0; + +#define htp_roll_dma_preamble dma_queue * q = octx->ctx->dma[0]; + +static inline void roll_dma_push(dma_queue * q, + dma_addr_t dst, + dma_addr_t src, + uint32_t dst_stride, + uint32_t src_stride, + uint32_t bytes, + uint32_t nrows) { + if (bytes == 0 || nrows == 0) { + return; + } + + if (!dma_queue_push(q, dma_make_data(dst, src), dst_stride, src_stride, bytes, nrows)) { + dma_queue_flush(q); + dma_queue_push(q, dma_make_data(dst, src), + dst_stride, src_stride, bytes, nrows); + } +} + +static inline void roll_dma_push_rows(dma_queue * q, + const struct htp_tensor * dst, + const struct htp_tensor * src0, + uint32_t dst_row, + uint32_t src_row, + uint32_t nrows, + uint32_t row_size, + uint32_t i0_src0, + uint32_t n0) { + const dma_addr_t dst_base = dst->data + (size_t) dst_row * row_size; + const dma_addr_t src_base = src0->data + (size_t) src_row * row_size; + + roll_dma_push(q, dst_base, src_base + (size_t) i0_src0 * sizeof(float), + row_size, row_size, n0 * sizeof(float), nrows); + roll_dma_push(q, dst_base + (size_t) n0 * sizeof(float), src_base, + row_size, row_size, i0_src0 * sizeof(float), nrows); +} + +// Same row-wrap split as roll_dma_push_rows, but addressed with explicit byte strides so it +// also works for a src0 that is row-contiguous only (e.g. a permuted view) rather than fully packed. +static inline void roll_dma_push_range(dma_queue * q, + dma_addr_t dst_row, + dma_addr_t src_row, + uint32_t dst_stride, + uint32_t src_stride, + uint32_t nrows, + uint32_t i0_src0, + uint32_t n0) { + roll_dma_push(q, dst_row, src_row + (size_t) i0_src0 * sizeof(float), + dst_stride, src_stride, n0 * sizeof(float), nrows); + roll_dma_push(q, dst_row + (size_t) n0 * sizeof(float), src_row, + dst_stride, src_stride, i0_src0 * sizeof(float), nrows); +} + +static int roll_dma_f32_contiguous(struct htp_ops_context * octx) { + htp_roll_preamble; + htp_roll_dma_preamble; + + const uint32_t row_size = ne0 * sizeof(float); + + if (s1 == 0 && s2 == 0 && s3 == 0) { + roll_dma_push_rows(q, dst, src0, 0, 0, ne1 * ne2 * ne3, row_size, i0_src0, n0); + dma_queue_flush(q); + return HTP_STATUS_OK; + } + + if (s1 == 0) { + const uint32_t i2_src0 = htp_roll_wrap(-s2, ne2); + for (uint32_t i3 = 0; i3 < ne3; i3++) { + const uint32_t i03 = htp_roll_wrap((int32_t) i3 - s3, ne3); + const uint32_t dst_row0 = i3 * ne2 * ne1; + const uint32_t src_row0 = (i03 * ne2 + i2_src0) * ne1; + const uint32_t n2_first = ne2 - i2_src0; + + roll_dma_push_rows(q, dst, src0, dst_row0, src_row0, n2_first * ne1, + row_size, i0_src0, n0); + roll_dma_push_rows(q, dst, src0, dst_row0 + n2_first * ne1, i03 * ne2 * ne1, + i2_src0 * ne1, row_size, i0_src0, n0); + } + + dma_queue_flush(q); + return HTP_STATUS_OK; + } + + const uint32_t i1_src0 = htp_roll_wrap(-s1, ne1); + const uint32_t n1_first = ne1 - i1_src0; + + for (uint32_t i3 = 0; i3 < ne3; i3++) { + const uint32_t i03 = htp_roll_wrap((int32_t) i3 - s3, ne3); + for (uint32_t i2 = 0; i2 < ne2; i2++) { + const uint32_t i02 = htp_roll_wrap((int32_t) i2 - s2, ne2); + const uint32_t dst_row0 = (i3 * ne2 + i2) * ne1; + const uint32_t src_row0 = (i03 * ne2 + i02) * ne1; + + roll_dma_push_rows(q, dst, src0, dst_row0, src_row0 + i1_src0, + n1_first, row_size, i0_src0, n0); + roll_dma_push_rows(q, dst, src0, dst_row0 + n1_first, src_row0, + i1_src0, row_size, i0_src0, n0); + } + } + + dma_queue_flush(q); + return HTP_STATUS_OK; +} + +// DMA path for a row-contiguous but otherwise arbitrarily strided src0 (e.g. a permuted view). +// Same row-wrap split as above, one DMA push per (i2,i3), addressed via the real nb01/nb02/nb03 +// instead of assuming a packed layout. +static int roll_dma_f32_strided(struct htp_ops_context * octx) { + htp_roll_preamble; + htp_roll_dma_preamble; + + const uint32_t i1_src0 = htp_roll_wrap(-s1, ne1); + const uint32_t n1_first = ne1 - i1_src0; + + for (uint32_t i3 = 0; i3 < ne3; i3++) { + const uint32_t i03 = htp_roll_wrap((int32_t) i3 - s3, ne3); + for (uint32_t i2 = 0; i2 < ne2; i2++) { + const uint32_t i02 = htp_roll_wrap((int32_t) i2 - s2, ne2); + + const dma_addr_t dst_row0 = dst->data + (size_t) i2 * nb2 + (size_t) i3 * nb3; + const dma_addr_t src_row0 = src0->data + (size_t) i02 * nb02 + (size_t) i03 * nb03; + + roll_dma_push_range(q, dst_row0, src_row0 + (size_t) i1_src0 * nb01, + nb1, nb01, n1_first, i0_src0, n0); + roll_dma_push_range(q, dst_row0 + (size_t) n1_first * nb1, src_row0, + nb1, nb01, i1_src0, i0_src0, n0); + } + } + + dma_queue_flush(q); + return HTP_STATUS_OK; +} + +static void roll_thread_f32(unsigned int nth, unsigned int ith, void * data) { + struct htp_roll_context * rctx = (struct htp_roll_context *) data; + struct htp_ops_context * octx = rctx->octx; + + htp_roll_preamble; + + const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith; + const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->nrows); + if (row_start >= row_end) { + return; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start); + + for (uint32_t row = row_start; row < row_end; row++) { + const uint32_t i3 = fastdiv(row, &kparams->div_ne2_ne1); + const uint32_t rem = row - i3 * ne2 * ne1; + const uint32_t i2 = fastdiv(rem, &kparams->div_ne1); + const uint32_t i1 = rem - i2 * ne1; + + const uint32_t i01 = htp_roll_wrap((int32_t) i1 - s1, ne1); + const uint32_t i02 = htp_roll_wrap((int32_t) i2 - s2, ne2); + const uint32_t i03 = htp_roll_wrap((int32_t) i3 - s3, ne3); + + const uint8_t * src_row = (const uint8_t *) (uintptr_t) src0->data + i01*nb01 + i02*nb02 + i03*nb03; + uint8_t * dst_row = (uint8_t *) (uintptr_t) dst->data + i1*nb1 + i2*nb2 + i3*nb3; + + hex_l2fetch(src_row + i0_src0 * sizeof(float), n0 * sizeof(float), ne0 * sizeof(float), 1); + hvx_copy_uu(dst_row, src_row + i0_src0 * sizeof(float), n0, sizeof(float)); + + if (i0_src0 != 0) { + hex_l2fetch(src_row, i0_src0 * sizeof(float), ne0 * sizeof(float), 1); + hvx_copy_uu(dst_row + n0 * sizeof(float), src_row, i0_src0, sizeof(float)); + } + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start); + + FARF(HIGH, "roll %d/%d: (%ux%ux%ux%u) rows %u:%u shift=(%d,%d,%d,%d)\n", + ith, nth, ne0, ne1, ne2, ne3, + row_start, row_end, s0, s1, s2, s3); +} + +int execute_op_roll_f32(struct htp_ops_context * octx) { + htp_roll_preamble; + + if (kparams->kernel_type == HTP_ROLL_KERNEL_DMA_CONTIGUOUS) { + return roll_dma_f32_contiguous(octx); + } + if (kparams->kernel_type == HTP_ROLL_KERNEL_DMA_STRIDED) { + return roll_dma_f32_strided(octx); + } + + const uint32_t total_rows = kparams->total_rows; + const size_t dst_row_size = ne0 * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + struct htp_roll_context rctx = { + .octx = octx, + .kparams = kparams, + .row_start = row_start, + .nrows = nrows, + .nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + }; + + work_queue_run(octx->ctx->work_queue, roll_thread_f32, &rctx, n_threads); + + return HTP_STATUS_OK; +} + +int op_roll(struct htp_ops_context * octx) { + switch (octx->src[0]->type) { + case HTP_TYPE_F32: + return execute_op_roll_f32(octx); + + default: + return HTP_STATUS_NO_SUPPORT; + } +} diff --git a/ggml/src/ggml-hexagon/htp/roll-ops.h b/ggml/src/ggml-hexagon/htp/roll-ops.h new file mode 100644 index 0000000..2646aaa --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/roll-ops.h @@ -0,0 +1,30 @@ +#ifndef HTP_ROLL_OPS_H +#define HTP_ROLL_OPS_H + +#include "hex-fastdiv.h" +#include + +enum htp_roll_kernel_type { + HTP_ROLL_KERNEL_DMA_CONTIGUOUS = 1, + HTP_ROLL_KERNEL_DMA_STRIDED = 2, + HTP_ROLL_KERNEL_HVX = 3, +}; + +struct htp_roll_kernel_params { + uint8_t kernel_type; + uint32_t n_threads; + uint32_t total_rows; + uint32_t i0_src0; + uint32_t n0; + + struct fastdiv_values div_ne1; + struct fastdiv_values div_ne2_ne1; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_roll_kernel_params) <= 128, "htp_roll_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_roll_kernel_params) <= 128, "htp_roll_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_ROLL_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 1d8b079..8c55ac3 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -7,8 +7,9 @@ #include #include +#include -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #include "hex-fastdiv.h" @@ -16,14 +17,15 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" +#include "rope-ops.h" -// Redefined the types GGML_ROPE_TYPE_NORMAL & GGML_ROPE_TYPE_NEOX as we can't include ggml.h +// Redefined the rope type constants as we can't include ggml.h #define HTP_ROPE_TYPE_NORMAL 0 #define HTP_ROPE_TYPE_NEOX 2 - -#define HTP_ROPE_SPAD_NROWS 16 -#define HTP_ROPE_SPAD_BLOCK (HTP_ROPE_SPAD_NROWS/2) +#define HTP_ROPE_TYPE_MROPE 8 +#define HTP_ROPE_TYPE_VISION 24 +#define HTP_ROPE_TYPE_IMROPE 40 #define htp_rope_preamble \ const uint32_t ne00 = src0->ne[0]; \ @@ -48,6 +50,7 @@ struct htp_rope_context { int32_t n_dims; + int32_t n_offs; int32_t mode; int32_t n_ctx_orig; int32_t sections[4]; @@ -59,21 +62,31 @@ struct htp_rope_context { float beta_fast; float beta_slow; float theta_scale; + float theta_scale_32; + float theta_powers[32]; float corr_dims[2]; uint32_t src0_nrows_per_thread; - size_t spad_stride; struct htp_ops_context * octx; + uint8_t * vtcm_base; + size_t spad_per_thread; + size_t theta_cache_offset; + size_t src0_row_size; + size_t src0_row_stride; size_t dst_row_size; + size_t dst_row_stride; size_t src0_row_size_aligned; - size_t dst_row_size_aligned; - size_t theta_cache_offset; uint32_t src0_nrows; + uint32_t row_start; + uint32_t nrows; + + struct fastdiv_values div_ne2_ne1; + struct fastdiv_values div_ne1; - uint64_t t_start; + const float * freq_factors; }; static float rope_yarn_ramp(const float low, const float high, const int i0) { @@ -82,40 +95,285 @@ static float rope_yarn_ramp(const float low, const float high, const int i0) { return (1 - MIN(1, MAX(0, y))); } -static void rope_cache_init(const float theta_base, +// Compute one (cos, sin) pair into cache[i0], cache[i0+1] applying YaRN scaling. +static inline void rope_yarn_one(float theta, float freq_scale, float * corr_dims, + uint32_t i0, float ext_factor, float mscale, + float * cache) { + float theta_extrap = theta; + + // Get n-d rotational scaling corrected for extrapolation + float theta_interp = freq_scale * theta_extrap; + float theta_final = theta_interp; + float mscale_final = mscale; + + if (ext_factor != 0.0f) { + float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor; + theta_final = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix; + + // Get n-d magnitude scaling corrected for interpolation + mscale_final *= 1.0f + 0.1f * logf(1.0f / freq_scale); + } + + const uint32_t b = i0 / 64; + const uint32_t k = (i0 % 64) / 2; + cache[b * 64 + k] = cosf(theta_final) * mscale_final; + cache[b * 64 + 32 + k] = sinf(theta_final) * mscale_final; +} + +// 32 thetas -> 32 deinterleaved pairs [cos[32] | sin[32]] at cache[i0]. +static inline void rope_cache_hvx_32(float * cache, uint32_t i0, + HVX_Vector v_theta, + const float * freq_factors, + HVX_Vector v_freq_scale, + HVX_Vector v_mscale) { + if (freq_factors) { + HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2); + v_theta = hvx_vec_mul_f32_f32(v_theta, hvx_vec_inverse_f32(v_ff)); + } + + HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale); + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(v_theta_final, &vcos, &vsin); + vcos = hvx_vec_mul_f32_f32(vcos, v_mscale); + vsin = hvx_vec_mul_f32_f32(vsin, v_mscale); + + if (((uintptr_t) (cache + i0)) % 128 == 0) { + hvx_vmem(cache + i0 + 0) = vcos; + hvx_vmem(cache + i0 + 32) = vsin; + } else { + hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), vcos); + hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), vsin); + } +} + +static __attribute__((noinline)) void rope_cache_init(const float theta_base, const float freq_scale, const float * freq_factors, float * corr_dims, - const uint32_t ne0, + const uint32_t n_cache, const float ext_factor, const float mscale, float * cache, - const float theta_scale) { + const float theta_scale, + const float * theta_powers, + const float theta_scale_32) { // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py - float theta = theta_base; + if (ext_factor == 0.0f) { + // Fast path: fully vectorized + // We process 32 pairs (64 elements) per iteration. + const uint32_t n_blocks = n_cache / 64; + + HVX_Vector v_theta_powers = hvx_vmemu(theta_powers); + HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); + HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); + + float theta_block = theta_base; + + for (uint32_t b = 0; b < n_blocks; b++) { + uint32_t i0 = b * 64; + HVX_Vector v_theta_base = hvx_vec_splat_f32(theta_block); + HVX_Vector v_theta = hvx_vec_mul_f32_f32(v_theta_base, v_theta_powers); + rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale); + theta_block *= theta_scale_32; + } - for (uint32_t i0 = 0; i0 < ne0; i0 += 2) { - const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + // Leftovers + float theta = theta_block; + for (uint32_t i0 = n_blocks * 64; i0 < n_cache; i0 += 2) { + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); + theta *= theta_scale; + } + } else { + float theta = theta_base; + for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) { + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); + theta *= theta_scale; + } + } +} + +static inline float mrope_pick_theta(float theta_t, float theta_h, float theta_w, float theta_e, + int sector, const int32_t sections[4], int sec_w, int sec_e, + bool is_imrope) { + if (is_imrope) { + if (sector % 3 == 0 && sector < 3 * sections[0]) { return theta_t; } + else if (sector % 3 == 1 && sector < 3 * sections[1]) { return theta_h; } + else if (sector % 3 == 2 && sector < 3 * sections[2]) { return theta_w; } + else { return theta_e; } + } + if (sector < sections[0]) { return theta_t; } + else if (sector < sec_w) { return theta_h; } + else if (sector < sec_e) { return theta_w; } + else { return theta_e; } +} + +// lane j is 1 when (j % 3) == rem +static const float __attribute__((aligned(128))) mrope_mod3_eq0[32] = { + 1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0 +}; +static const float __attribute__((aligned(128))) mrope_mod3_eq1[32] = { + 0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1 +}; +static const float __attribute__((aligned(128))) mrope_mod3_eq2[32] = { + 0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0 +}; + +static const float __attribute__((aligned(128))) mrope_k_ramp[32] = { + 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, + 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +}; + +static inline HVX_VectorPred mrope_mask_eq1(const float * m) { + return Q6_Q_vcmp_gt_VsfVsf(hvx_vmemu(m), Q6_V_vzero()); +} + +// IMROPE without wrap: theta[k] = pos[k % 3] * scale^k +static inline HVX_Vector mrope_thetas_imrope_mod3(float pos_t, float pos_h, float pos_w, + uint32_t k0, HVX_Vector v_powers, float scale_block) { + const int r = (int) (k0 % 3); + const float * mt = (r == 0) ? mrope_mod3_eq0 : (r == 1) ? mrope_mod3_eq2 : mrope_mod3_eq1; + const float * mh = (r == 0) ? mrope_mod3_eq1 : (r == 1) ? mrope_mod3_eq0 : mrope_mod3_eq2; + + HVX_Vector v = hvx_vec_splat_f32(pos_w); + v = Q6_V_vmux_QVV(mrope_mask_eq1(mh), hvx_vec_splat_f32(pos_h), v); + v = Q6_V_vmux_QVV(mrope_mask_eq1(mt), hvx_vec_splat_f32(pos_t), v); + v = hvx_vec_mul_f32_f32(v, v_powers); + return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block)); +} + +// Contiguous MROPE without wrap: theta[k] = pos[section(k)] * scale^k +static inline HVX_Vector mrope_thetas_contig(float pos_t, float pos_h, float pos_w, float pos_e, + uint32_t k0, int s0, int sec_w, int sec_e, + HVX_Vector v_powers, float scale_block) { + HVX_Vector v_k = hvx_vec_add_f32_f32(hvx_vec_splat_f32((float) k0), hvx_vmemu(mrope_k_ramp)); + HVX_VectorPred lt_s0 = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) s0), v_k); + HVX_VectorPred lt_sw = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_w), v_k); + HVX_VectorPred lt_se = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_e), v_k); + + HVX_Vector v = hvx_vec_splat_f32(pos_e); + v = Q6_V_vmux_QVV(lt_se, hvx_vec_splat_f32(pos_w), v); + v = Q6_V_vmux_QVV(lt_sw, hvx_vec_splat_f32(pos_h), v); + v = Q6_V_vmux_QVV(lt_s0, hvx_vec_splat_f32(pos_t), v); + v = hvx_vec_mul_f32_f32(v, v_powers); + return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block)); +} + +// pos_t/h/w/e: the four position ids for this sequence step (t=time, h=height, w=width, e=extra). +// sections[4]: number of head dims assigned to each position component. +static __attribute__((noinline)) void mrope_cache_init(const float pos_t, + const float pos_h, + const float pos_w, + const float pos_e, + const int32_t sections[4], + const bool is_imrope, + const bool indep_sects, + const float freq_scale, + const float * freq_factors, + float * corr_dims, + const uint32_t n_cache, + const float ext_factor, + const float mscale, + float * cache, + const float theta_scale, + const float * theta_powers, + const float theta_scale_32) { + const int sect_dims = sections[0] + sections[1] + sections[2] + sections[3]; + const int sec_w = sections[0] + sections[1]; + const int sec_e = sec_w + sections[2]; + const uint32_t n_pairs = n_cache / 2; + + const bool no_wrap = (sect_dims > 0) && (n_pairs <= (uint32_t) sect_dims); + const bool imrope_mod3 = is_imrope && !indep_sects && no_wrap + && sections[0] > 0 && sections[1] > 0 && sections[2] > 0 + && n_pairs <= (uint32_t) (3 * sections[0]) + && n_pairs <= (uint32_t) (3 * sections[1]) + && n_pairs <= (uint32_t) (3 * sections[2]); + const bool contig = !is_imrope && !indep_sects && no_wrap; + + if (ext_factor == 0.0f && (imrope_mod3 || contig)) { + HVX_Vector v_powers = hvx_vmemu(theta_powers); + HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); + HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); + float scale_block = 1.0f; + const uint32_t n_blocks = n_cache / 64; + + for (uint32_t b = 0; b < n_blocks; b++) { + const uint32_t i0 = b * 64; + const uint32_t k0 = b * 32; + HVX_Vector v_theta = imrope_mod3 + ? mrope_thetas_imrope_mod3(pos_t, pos_h, pos_w, k0, v_powers, scale_block) + : mrope_thetas_contig(pos_t, pos_h, pos_w, pos_e, k0, sections[0], sec_w, sec_e, + v_powers, scale_block); + rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale); + scale_block *= theta_scale_32; + } + + float theta_k = scale_block; + for (uint32_t k = n_blocks * 32; k < n_pairs; k++) { + const uint32_t i0 = 2 * k; + const float pos = mrope_pick_theta(pos_t, pos_h, pos_w, pos_e, + (int) k, sections, sec_w, sec_e, is_imrope); + const float ff = freq_factors ? freq_factors[k] : 1.0f; + rope_yarn_one(pos * theta_k / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); + theta_k *= theta_scale; + } + return; + } + + float theta_t = pos_t; + float theta_h = pos_h; + float theta_w = pos_w; + float theta_e = pos_e; - float theta_extrap = theta / ff; + const bool use_hvx = (ext_factor == 0.0f); + float __attribute__((aligned(128))) thetas[32]; + uint32_t n_thetas = 0; + uint32_t block_i0 = 0; - // Get n-d rotational scaling corrected for extrapolation - float theta_interp = freq_scale * theta_extrap; - float theta_final = theta_interp; - float mscale_final = mscale; + HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); + HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); - if (ext_factor != 0.0f) { - float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor; - theta_final = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix; + for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) { + const int sector = (i0 / 2) % sect_dims; + + if (indep_sects) { + // Reset theta when crossing into a new section. + if (sector == 0) { theta_t = pos_t; } + else if (sector == sections[0]) { theta_h = pos_h; } + else if (sector == sec_w) { theta_w = pos_w; } + else if (sector == sec_e) { theta_e = pos_e; } + } - // Get n-d magnitude scaling corrected for interpolation - mscale_final *= 1.0f + 0.1f * logf(1.0f / freq_scale); + const float theta = mrope_pick_theta(theta_t, theta_h, theta_w, theta_e, + sector, sections, sec_w, sec_e, is_imrope); + + if (use_hvx) { + if (n_thetas == 0) { + block_i0 = i0; + } + thetas[n_thetas++] = theta; + if (n_thetas == 32) { + rope_cache_hvx_32(cache, block_i0, hvx_vmemu(thetas), freq_factors, v_freq_scale, v_mscale); + n_thetas = 0; + } + } else { + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); } - cache[i0 + 0] = cosf(theta_final) * mscale_final; - cache[i0 + 1] = sinf(theta_final) * mscale_final; + theta_t *= theta_scale; + theta_h *= theta_scale; + theta_w *= theta_scale; + theta_e *= theta_scale; + } - theta *= theta_scale; + for (uint32_t k = 0; k < n_thetas; k++) { + const uint32_t i0 = block_i0 + 2 * k; + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(thetas[k] / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); } } @@ -133,119 +391,144 @@ static void rope_corr_dims(int n_dims, dims[1] = MIN(n_dims - 1, end); } +static inline void hvx_rope_neox_mul(HVX_Vector v0, HVX_Vector v1, HVX_Vector vcos, HVX_Vector vsin, + HVX_Vector * o0, HVX_Vector * o1) { + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, vsin); + *o0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s)); + *o1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c)); +} + +// theta_cache full 32-pair blocks are deinterleaved [cos | sin]. static inline void hvx_rope_neox_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) { - const HVX_Vector * restrict vsrc = (const HVX_Vector *) src0; - const HVX_Vector * restrict vtheta = (const HVX_Vector *) theta_cache; - HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const uint32_t he = ne / 2; + const uint32_t nvec = he / 32; + const uint32_t nloe = he % 32; + + if (nloe == 0) { + const HVX_Vector * vs = (const HVX_Vector *) src0; + const HVX_Vector * vt = (const HVX_Vector *) theta_cache; + HVX_Vector * vd = (HVX_Vector *) dst; + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector o0, o1; + hvx_rope_neox_mul(vs[i], vs[nvec + i], vt[i * 2 + 0], vt[i * 2 + 1], &o0, &o1); + vd[i] = o0; + vd[nvec + i] = o1; + } + return; + } - uint32_t nvec = (ne / (VLEN_FP32 * 2) * 2); // 2 vecs per loop, step of 2 + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector o0, o1; + hvx_rope_neox_mul(((const HVX_Vector *) src0)[i], + hvx_vmemu(src0 + he + i * 32), + ((const HVX_Vector *) theta_cache)[i * 2 + 0], + ((const HVX_Vector *) theta_cache)[i * 2 + 1], + &o0, &o1); + ((HVX_Vector *) dst)[i] = o0; + hvx_vmemu(dst + he + i * 32) = o1; + } + + HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32); + HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); + HVX_Vector o0, o1; + hvx_rope_neox_mul(v0, v1, vcos, vsin, &o0, &o1); + hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), o0); + hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), o1); +} - uint32_t he = ne / 2; // half_dims offset in elements - uint32_t hv = he / VLEN_FP32; // half_dims offset in vectors +static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) { + const uint32_t nvec = ne / 64; + const uint32_t nloe = ne % 64; - #pragma unroll(2) - for (uint32_t i = 0; i < nvec; i += 2) { - HVX_Vector v0 = vsrc[i/2+0]; - HVX_Vector v1 = vsrc[i/2+hv]; + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector v0 = ((const HVX_Vector *) src0)[i * 2 + 0]; + HVX_Vector v1 = ((const HVX_Vector *) src0)[i * 2 + 1]; - HVX_Vector v2 = vtheta[i+0]; - HVX_Vector v3 = vtheta[i+1]; + HVX_Vector vcos = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; + HVX_Vector vsin = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); // vcos_sin[0] = cos_theta, vcos_sin[1] = sin_theta + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); - vdst[i/2+0] = Q6_Vsf_equals_Vqf32(v4); - vdst[i/2+hv] = Q6_Vsf_equals_Vqf32(v5); - } + HVX_VectorPair vstore = Q6_W_vshuff_VVR(Q6_Vsf_equals_Vqf32(v5), Q6_Vsf_equals_Vqf32(v4), -4); - for (uint32_t i = nvec * VLEN_FP32; i < ne; i += 2) { - const float cos_theta = theta_cache[i+0]; - const float sin_theta = theta_cache[i+1]; - float x0 = src0[i/2]; - float x1 = src0[i/2 + he]; - dst[i/2] = x0 * cos_theta - x1 * sin_theta; - dst[i/2 + he] = x0 * sin_theta + x1 * cos_theta; + ((HVX_Vector *) dst)[i * 2 + 0] = Q6_V_lo_W(vstore); + ((HVX_Vector *) dst)[i * 2 + 1] = Q6_V_hi_W(vstore); } -} -static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) { - const HVX_Vector * restrict vsrc = (const HVX_Vector *) src0; - const HVX_Vector * restrict vtheta = (const HVX_Vector *) theta_cache; - HVX_Vector * restrict vdst = (HVX_Vector *) dst; + if (nloe > 0) { + if (nloe <= 32) { + HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); - uint32_t nvec = (ne / (VLEN_FP32 * 2)) * 2; // 2 vecs per loop, step of two + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4); - #pragma unroll(2) - for (uint32_t i = 0; i < nvec; i+=2) { - HVX_Vector v0 = vsrc[i+0]; - HVX_Vector v1 = vsrc[i+1]; + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); - HVX_Vector v2 = vtheta[i+0]; - HVX_Vector v3 = vtheta[i+1]; + HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); + HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); // vx0_x1[0] = x0, vx0_x1[1] = x1 - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); // vcos_sin[0] = cos_theta, vcos_sin[1] = sin_theta + HVX_VectorPair vstore = Q6_W_vshuff_VVR(Q6_Vsf_equals_Vqf32(v5), Q6_Vsf_equals_Vqf32(v4), -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + hvx_vec_store_u(dst + nvec * 64, nloe * sizeof(float), Q6_V_lo_W(vstore)); + } else { + HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64); + HVX_Vector v1 = hvx_vmemu(src0 + nvec * 64 + 32); - HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); - HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); - HVX_VectorPair vstore = Q6_W_vshuff_VVR(Q6_Vsf_equals_Vqf32(v5), Q6_Vsf_equals_Vqf32(v4), -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - vdst[i+0] = Q6_V_lo_W(vstore); - vdst[i+1] = Q6_V_hi_W(vstore); - } + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); + + HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); + HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); + + HVX_VectorPair vstore = Q6_W_vshuff_VVR(Q6_Vsf_equals_Vqf32(v5), Q6_Vsf_equals_Vqf32(v4), -4); - for (uint32_t i = nvec * VLEN_FP32; i < ne; i += 2) { - const float cos_theta = theta_cache[i+0]; - const float sin_theta = theta_cache[i+1]; - float x0 = src0[i+0]; - float x1 = src0[i+1]; - dst[i+0] = x0 * cos_theta - x1 * sin_theta; - dst[i+1] = x0 * sin_theta + x1 * cos_theta; + ((HVX_Vector *) dst)[nvec * 2 + 0] = Q6_V_lo_W(vstore); + hvx_vec_store_u(dst + nvec * 64 + 32, (nloe - 32) * sizeof(float), Q6_V_hi_W(vstore)); + } } } -static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { +static void inline rope_basic_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, + uint32_t nr, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - - hvx_rope_f32_aa(d, s, rctx->n_dims, theta_cache); - - // fill the remain channels with data from src tensor - if (rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims); - } + hvx_rope_f32_aa(s + n_offs, s + n_offs, rctx->n_dims, theta_cache); } } -static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { +static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, + uint32_t nr, uint32_t ne, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - - hvx_rope_neox_f32_aa(d, s, rctx->n_dims, theta_cache); - - // fill the remain channels with data from src tensor - if (rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + rctx->n_dims), (uint8_t *)(s + rctx->n_dims), ne0 - rctx->n_dims); - } + hvx_rope_neox_f32_aa(s + n_offs, s + n_offs, ne, theta_cache); } } @@ -260,125 +543,153 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_rope_preamble; - const uint32_t src0_nrows = rctx->src0_nrows; + const uint32_t src0_nrows = rctx->nrows; const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, rctx->row_start + src0_nrows); // no work for this thread if (src0_start_row >= src0_end_row) { return; } - uint64_t tt = HAP_perf_get_qtimer_count(); - const int32_t mode = rctx->mode; - const bool is_neox = mode & HTP_ROPE_TYPE_NEOX; + // MROPE, IMROPE and VISION use NEOX-style pairing for the rotation + const bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE); + const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); // VTCM setup - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); + uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread); float * theta_cache = (float *) (src0_spad_base); src0_spad_base = src0_spad_base + rctx->theta_cache_offset; - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); - dma_queue * dma_queue = octx->ctx->dma[ith]; - const int32_t * pos = (const int32_t *) src1->data; - const float * freq_factors = src2 ? (const float *) src2->data : NULL; + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + const int32_t * pos = (const int32_t *) (uintptr_t) src1->data; + const float * freq_factors = rctx->freq_factors; + + const uint32_t i3_start = fastdiv(src0_start_row, &rctx->div_ne2_ne1); + const uint32_t rem = fastmodulo(src0_start_row, ne2 * ne1, &rctx->div_ne2_ne1); + const uint32_t i2_start = fastdiv(rem, &rctx->div_ne1); + const uint32_t i1_start = fastmodulo(rem, ne1, &rctx->div_ne1); - uint32_t ir = 0; + uint32_t ir = src0_start_row; uint32_t prev_i2 = (uint32_t) -1; + uint32_t cur_slot = 0; - for (uint32_t i3 = 0; i3 < ne3; i3++) { // batch - for (uint32_t i2 = 0; i2 < ne2; i2++) { // seq-len - for (uint32_t i1 = 0; i1 < ne1; ) { // attn-heads - if (ir < src0_start_row) { ir++; i1++; continue; } + for (uint32_t i3 = i3_start; i3 < ne3; i3++) { // batch + const uint32_t i2_init = (i3 == i3_start) ? i2_start : 0; + for (uint32_t i2 = i2_init; i2 < ne2; i2++) { // seq-len + const uint32_t i1_init = (i3 == i3_start && i2 == i2_start) ? i1_start : 0; + for (uint32_t i1 = i1_init; i1 < ne1; ) { // attn-heads if (ir >= src0_end_row) goto done; // Rows in this block const uint32_t nrows = MIN(src0_end_row - ir, ne1 - i1); // Depth before prefetch - uint32_t dma_depth = dma_queue_depth(dma_queue); - - // FARF(HIGH, "rope-block %u: ir %u n-rows %u dma-depth %u : usec %u", ith, ir, nrows, dma_depth, - // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); - - // Prefetch loop - for (uint32_t pnr = 0, pr = 0; pr < nrows && pr < HTP_ROPE_SPAD_NROWS; pr += pnr) { - pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK); + const uint32_t dma_depth = dma_queue_depth(dma_q); - uint32_t pi1 = i1 + pr; - uint32_t pir = ir + pr; + // Prefetch up to 2 blocks + const uint32_t p_nrows = MIN(nrows, 2 * HTP_ROPE_SPAD_BLOCK); + for (uint32_t pr = 0; pr < p_nrows; pr += HTP_ROPE_SPAD_BLOCK) { + const uint32_t pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK); + const uint32_t slot = (cur_slot + pr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + uint8_t * spad_slot = rope_spad_slot(src0_spad_base, slot, rctx->src0_row_size_aligned); + const dma_addr_t src0_data = src0->data + i3 * nb03 + i2 * nb02 + (i1 + pr) * nb01; - // Dummy DMA transaction for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr((void *) dst->data, dst_spad_base + pr * rctx->dst_row_size_aligned), 0, 0, 0); + // Dummy DMA transaction for sequencing (interleaving wr, rd, wr, rd, ...) + dma_queue_push(dma_q, dma_make_data(dst->data, spad_slot), 0, 0, 0, 0); - const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned; - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr), - rctx->src0_row_size_aligned, rctx->src0_row_size, pnr); - - // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); + dma_queue_push(dma_q, dma_make_data(spad_slot, src0_data), + rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); } // Update theta cache if (i2 != prev_i2) { prev_i2 = i2; - const int32_t p = pos[i2]; - rope_cache_init(p, rctx->freq_scale, freq_factors, rctx->corr_dims, ne0, rctx->ext_factor, rctx->attn_factor, theta_cache, rctx->theta_scale); - - // FARF(HIGH, "rope-theta %u: ir %u i1 %u i2 %u i3 %u cache %p : usec %u", ith, ir, i1, i2, i3, theta_cache, - // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); + // VISION rotates the full row; other modes only rotate n_dims. + const uint32_t n_cache = is_vision ? ne0 : (uint32_t) rctx->n_dims; + const bool is_mrope = (rctx->mode & HTP_ROPE_TYPE_MROPE) != 0; + if (is_mrope) { + // src1 holds four position arrays stacked along ne0: + // pos[i2], pos[i2+ne2], pos[i2+ne2*2], pos[i2+ne2*3] + const bool is_imrope = (rctx->mode == HTP_ROPE_TYPE_IMROPE); + mrope_cache_init( + (float) pos[i2], + (float) pos[i2 + ne2], + (float) pos[i2 + ne2 * 2], + (float) pos[i2 + ne2 * 3], + rctx->sections, is_imrope, is_vision, + rctx->freq_scale, freq_factors, rctx->corr_dims, + n_cache, rctx->ext_factor, rctx->attn_factor, + theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); + } else { + rope_cache_init(pos[i2], rctx->freq_scale, freq_factors, rctx->corr_dims, + n_cache, rctx->ext_factor, rctx->attn_factor, + theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); } // Skip output DMA transactions from prev block (if any) - // No need to wait for those here since we're explicitly waiting for the latest prefecthes below. - for (uint32_t d=0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); } + for (uint32_t d = 0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_q); } // Compute loop - for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr, ir += cnr, i1 += cnr) { - // Number of rows to compute - cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t ne = is_vision ? ne0 : rctx->n_dims; + const uint32_t base_i1 = i1; + const uint32_t base_ir = ir; - uint8_t * dst_spad = (uint8_t *) dma_queue_pop(dma_queue).src; - uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_queue).dst; + for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr) { + cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t slot = (cur_slot + cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + const uint32_t cur_ir = base_ir + cr; + const uint32_t cur_i1 = base_i1 + cr; - // FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr, - // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); + dma_queue_pop(dma_q); + uint8_t * cur_spad = (uint8_t *) dma_queue_pop(dma_q).dst; - if (is_neox) { - rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir); + if (is_neox || is_vision) { + rope_neox_f32_inplace(rctx, cur_spad, cnr, ne, theta_cache); } else { - rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + rope_basic_f32_inplace(rctx, cur_spad, cnr, theta_cache); } - - uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1; - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(dst_addr, dst_spad), rctx->dst_row_size, rctx->dst_row_size_aligned, cnr); - - // Prefetch more rows (if any) - if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) { - uint32_t pnr = MIN(nrows - (cr + HTP_ROPE_SPAD_NROWS), HTP_ROPE_SPAD_BLOCK); - uint32_t pi1 = i1 + HTP_ROPE_SPAD_NROWS; - uint32_t pir = ir + HTP_ROPE_SPAD_NROWS; - - const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src_spad, src_addr), - rctx->src0_row_size_aligned, rctx->src0_row_size, pnr); - - // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir); + + const dma_addr_t dst_data = dst->data + i3 * nb3 + i2 * nb2 + cur_i1 * nb1; + dma_queue_push(dma_q, dma_make_data(dst_data, cur_spad), + rctx->dst_row_stride, rctx->src0_row_size_aligned, rctx->dst_row_size, cnr); + + // Prefetch 2 blocks ahead into the slot just freed + if ((cr + 2 * HTP_ROPE_SPAD_BLOCK) < nrows) { + const uint32_t p_cr = cr + 2 * HTP_ROPE_SPAD_BLOCK; + const uint32_t pnr = MIN(nrows - p_cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t p_slot = (cur_slot + p_cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + uint8_t * p_spad = rope_spad_slot(src0_spad_base, p_slot, rctx->src0_row_size_aligned); + const dma_addr_t p_src0_data = src0->data + i3 * nb03 + i2 * nb02 + (base_i1 + p_cr) * nb01; + + dma_queue_push(dma_q, dma_make_data(p_spad, p_src0_data), + rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); } } + + const uint32_t n_chunks = (nrows + HTP_ROPE_SPAD_BLOCK - 1) / HTP_ROPE_SPAD_BLOCK; + cur_slot = (cur_slot + n_chunks) % HTP_ROPE_SPAD_NSLOTS; + + ir += nrows; + i1 += nrows; } } } done: - dma_queue_flush(dma_queue); - tt = HAP_perf_get_qtimer_count() - tt; + dma_queue_flush(dma_q); - FARF(HIGH, "rope-f32: %d/%d: (%u:%u) usec %u\n", ith, nth, src0_start_row, src0_end_row, (unsigned) HAP_perf_qtimer_count_to_us(tt)); + FARF(HIGH, "rope-f32: %d/%d: (%u:%u)\n", ith, nth, src0_start_row, src0_end_row); } static int execute_op_rope_f32(struct htp_ops_context * octx) { @@ -389,62 +700,69 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const struct htp_tensor * src2 = octx->src[2]; const struct htp_tensor * dst = octx->dst; - const char * op_type = "rope-f32"; + const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params; + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + assert(octx->ctx->vtcm_size >= kparams->vtcm_size); - switch (octx->op) { - case HTP_OP_ROPE: - break; + if (htp_tensor_is_extended(src1)) { + return HTP_STATUS_NO_SUPPORT; + } - default: - FARF(ERROR, "Unsupported Op %u\n", octx->op); - return HTP_STATUS_NO_SUPPORT; + const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = total_rows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; } - const uint32_t ne0 = dst->ne[0]; - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - - const size_t src0_row_size = src0->nb[1]; - const size_t dst_row_size = dst->nb[1]; - - // Aligned row sizes for VTCM - const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); - const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 128); - - // Calculate spad sizes per thread - size_t src0_spad_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; - size_t dst_spad_per_thread = HTP_ROPE_SPAD_NROWS * dst_row_size_aligned; - size_t spad_per_thread = src0_spad_per_thread + dst_spad_per_thread; - - // Check if we fit in VTCM - size_t total_vtcm_needed = spad_per_thread * n_threads; - if (octx->ctx->vtcm_size < total_vtcm_needed) { - FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, total_vtcm_needed); - return HTP_STATUS_VTCM_TOO_SMALL; + if (nrows == 0) { + return HTP_STATUS_OK; } - octx->src0_spad.size_per_thread = src0_spad_per_thread; - octx->dst_spad.size_per_thread = dst_spad_per_thread; - octx->src0_spad.size = n_threads * src0_spad_per_thread; - octx->dst_spad.size = n_threads * dst_spad_per_thread; - octx->src1_spad.size = 0; + const uint32_t n_threads = octx->n_threads; + + const uint32_t ne0 = dst->ne[0]; + const size_t src0_row_size = src0->ne[0] * sizeof(float); + const size_t src0_row_stride = src0->nb[1]; + const size_t dst_row_size = dst->ne[0] * sizeof(float); + const size_t dst_row_stride = dst->nb[1]; - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = NULL; octx->src1_spad.src = NULL; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL; + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], kparams->n_threads, src2 ? src2->ne[0] : 0); struct htp_rope_context rctx; memset(&rctx, 0, sizeof(struct htp_rope_context)); - rctx.t_start = HAP_perf_get_qtimer_count(); - - rctx.octx = octx; + rctx.octx = octx; + rctx.vtcm_base = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_spad); + rctx.spad_per_thread = layout.bytes_per_thread; + rctx.theta_cache_offset = layout.theta_cache_size_aligned; + + if (src2) { + dma_queue * dma_q = octx->ctx->dma[0]; + const size_t ff_size = src2->ne[0] * sizeof(float); + float * vtcm_freq_factors = VTCM_LAYOUT_PTR(float, octx->ctx->vtcm_base, layout.off_freq_factors); + dma_queue_push(dma_q, dma_make_data(vtcm_freq_factors, src2->data), + layout.freq_factors_size_aligned, 0, ff_size, 1); + dma_queue_pop(dma_q); + rctx.freq_factors = vtcm_freq_factors; + } const int32_t * op_params = &octx->op_params[0]; rctx.n_dims = ((const int32_t *) op_params)[1]; rctx.mode = ((const int32_t *) op_params)[2]; rctx.n_ctx_orig = ((const int32_t *) op_params)[4]; + rctx.n_offs = ((const int32_t *) op_params)[15]; memcpy(&rctx.freq_base, (int32_t *) op_params + 5, sizeof(float)); memcpy(&rctx.freq_scale, (int32_t *) op_params + 6, sizeof(float)); @@ -455,40 +773,41 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { memcpy(&rctx.sections, (int32_t *) op_params + 11, sizeof(int) * 4); rctx.theta_scale = powf(rctx.freq_base, -2.0f / rctx.n_dims); + rctx.theta_powers[0] = 1.0f; + for (int j = 1; j < 32; j++) { + rctx.theta_powers[j] = rctx.theta_powers[j - 1] * rctx.theta_scale; + } + rctx.theta_scale_32 = rctx.theta_powers[31] * rctx.theta_scale; rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims); - rctx.src0_row_size = src0_row_size; - rctx.dst_row_size = dst_row_size; - rctx.src0_row_size_aligned = src0_row_size_aligned; - rctx.dst_row_size_aligned = dst_row_size_aligned; - rctx.theta_cache_offset = theta_cache_size_aligned; + rctx.src0_row_size = src0_row_size; + rctx.src0_row_stride = src0_row_stride; + rctx.dst_row_size = dst_row_size; + rctx.dst_row_stride = dst_row_stride; + rctx.src0_row_size_aligned = layout.src0_row_size_aligned; - rctx.src0_nrows = src0_nrows; - rctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + rctx.src0_nrows = nrows; + rctx.nrows = nrows; + rctx.row_start = row_start; + rctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + rctx.div_ne2_ne1 = kparams->div_ne2_ne1; + rctx.div_ne1 = kparams->div_ne1; FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - worker_pool_run_func(octx->ctx->worker_pool, rope_job_f32, &rctx, n_threads); - } + work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads); return err; } int op_rope(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - switch (octx->src[0]->type) { case HTP_TYPE_F32: - err = execute_op_rope_f32(octx); - break; + return execute_op_rope_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h new file mode 100644 index 0000000..0c28ef8 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/rope-ops.h @@ -0,0 +1,62 @@ +#ifndef HTP_ROPE_OPS_H +#define HTP_ROPE_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" + +#define HTP_ROPE_SPAD_BLOCK 8 +#define HTP_ROPE_SPAD_NSLOTS 4 +#define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS) + +struct htp_rope_kernel_params { + uint32_t n_threads; + uint32_t vtcm_size; + + struct fastdiv_values div_ne2_ne1; + struct fastdiv_values div_ne1; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_rope_vtcm_layout { + size_t total_bytes; + size_t off_spad; + size_t off_freq_factors; + size_t bytes_per_thread; + size_t theta_cache_size_aligned; + size_t src0_row_size_aligned; + size_t freq_factors_size_aligned; +}; + +static inline void htp_rope_vtcm_layout_build( + struct htp_rope_vtcm_layout * L, + uint32_t ne00, + uint32_t n_threads, + uint32_t n_freq_factors +) { + const size_t src0_row_size = ne00 * sizeof(float); + const size_t src0_row_size_aligned = hex_round_up((uint32_t) src0_row_size, 128); + const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256); + const size_t freq_factors_size_aligned = hex_round_up(n_freq_factors * sizeof(float), 256); + + L->src0_row_size_aligned = src0_row_size_aligned; + L->theta_cache_size_aligned = theta_cache_size_aligned; + L->freq_factors_size_aligned = freq_factors_size_aligned; + L->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_spad, L->bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_freq_factors, freq_factors_size_aligned, n_freq_factors > 0); + L->total_bytes = off; +} + +static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) { + return base + (slot * HTP_ROPE_SPAD_BLOCK) * row_size_aligned; +} + +#endif // HTP_ROPE_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.c b/ggml/src/ggml-hexagon/htp/set-rows-ops.c index 0def7b4..191cf0c 100644 --- a/ggml/src/ggml-hexagon/htp/set-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.c @@ -8,14 +8,21 @@ #include #include -#include "hex-dma.h" +#include "dma-queue.h" +#include "work-queue.h" #include "hvx-utils.h" +#include "hex-utils.h" +#include "hvx-copy.h" +#include "hvx-quant.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" + +#include "hex-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp/set-rows-ops.h" #define set_rows_preamble \ const uint32_t ne00 = octx->src[0]->ne[0]; \ @@ -47,138 +54,201 @@ \ const uint32_t nr = ne01; -struct htp_set_rows_context { +struct set_rows_context { struct htp_ops_context * octx; - struct fastdiv_values div_ne12; - struct fastdiv_values div_ne11; - uint32_t src0_nrows_per_thread; + const struct htp_set_rows_kernel_params * kparams; + struct htp_set_rows_vtcm_layout vtcm_layout; + uint8_t * vtcm_base; + uint32_t task_start; + uint32_t tasks; + uint32_t tasks_per_thread; }; -static void set_rows_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) { - struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data; - struct htp_ops_context * octx = srctx->octx; - - set_rows_preamble; - - uint64_t qt = HAP_perf_get_qtimer_count(); - - // parallelize by rows of src0 - const uint32_t dr = srctx->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr; - - const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); - - for (uint32_t i03 = 0; i03 < ne03; ++i03) { - for (uint32_t i02 = 0; i02 < ne02; ++i02) { - for (uint32_t i = ir0; i < ir1; ++i) { - const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12); - const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11); - const uint32_t i10 = i; - - const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12; - - uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr; - if (i1 >= ne1) { - // ignore invalid indices - continue; - } - - const uintptr_t src0_ptr = octx->src[0]->data + i*nb01 + i02*nb02 + i03*nb03; - const uintptr_t dst_ptr = octx->dst->data + i1*nb1 + i02*nb2 + i03*nb3; - - // copy row - hvx_copy_f32_uu((uint8_t *)dst_ptr, (const uint8_t *)src0_ptr, ne00); - } - } - } - - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "set-rows-f32-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt); +#define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \ +static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned int ith, void *data) { \ + struct set_rows_context * srctx = (struct set_rows_context *)data; \ + struct htp_ops_context * octx = srctx->octx; \ + const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \ + set_rows_preamble; \ + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \ + const uint32_t dr = srctx->tasks_per_thread; \ + const uint32_t ir0 = srctx->task_start + dr * ith; \ + if (ir0 >= srctx->task_start + srctx->tasks) { \ + return; \ + } \ + const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \ + dma_queue * dma_q = octx->ctx->dma[ith]; \ + const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \ + uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_src0) + \ + ith * vtcm_layout->src0_bytes_per_thread; \ + uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, srctx->vtcm_base, vtcm_layout->off_dst) + \ + ith * vtcm_layout->dst_bytes_per_thread; \ + const uint32_t src0_row_size = ne00 * sizeof(float); \ + const uint32_t dst_row_size = htp_tensor_get_row_size(octx->dst->type, ne00); \ + const uint32_t nrows_per_thread = ir1 - ir0; \ + const uint32_t total_steps = ne03 * ne02 * nrows_per_thread; \ + uint32_t pi_step = 0; \ + uint32_t pi02 = 0; \ + uint32_t pi03 = 0; \ + for (uint32_t step = 0, spad_idx = 0; step < total_steps && spad_idx < 2; ++step, spad_idx++) { \ + uint32_t i = ir0 + pi_step; \ + const dma_addr_t src0_data = octx->src[0]->data + i*nb01 + pi02*nb02 + pi03*nb03; \ + dma_queue_push(dma_q, \ + dma_make_data(octx->dst->data, \ + vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size), \ + dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \ + dma_queue_push(dma_q, \ + dma_make_data(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size, \ + src0_data), \ + vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ + pi_step++; \ + if (pi_step == nrows_per_thread) { \ + pi_step = 0; \ + pi02++; \ + if (pi02 == ne02) { \ + pi02 = 0; \ + pi03++; \ + } \ + } \ + } \ + uint32_t ci_step = 0; \ + uint32_t ci02 = 0; \ + uint32_t ci03 = 0; \ + uint32_t ci11_base = 0; \ + uint32_t ci12_base = 0; \ + for (uint32_t step = 0; step < total_steps; ++step) { \ + void * dst_spad = (void *) dma_queue_pop(dma_q).src; \ + void * src_spad = (void *) dma_queue_pop(dma_q).dst; \ + uint32_t i = ir0 + ci_step; \ + const uintptr_t src1_addr = octx->src[1]->data + i*nb10 + ci11_base*nb11 + ci12_base*nb12; \ + const IDX_TYPE i1 = *(const IDX_TYPE *)src1_addr; \ + const bool valid_i1 = ((uint64_t)i1 < (uint64_t)ne1); \ + const uint32_t target_i1 = (uint32_t)i1; \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, step); \ + if (valid_i1) { \ + COMPUTE_EXPR; \ + } \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, step); \ + if (valid_i1) { \ + const dma_addr_t dst_data = octx->dst->data + target_i1*nb1 + ci02*nb2 + ci03*nb3; \ + dma_queue_push(dma_q, \ + dma_make_data(dst_data, dst_spad), \ + dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 1); \ + } else { \ + dma_queue_push(dma_q, \ + dma_make_data(octx->dst->data, dst_spad), \ + dst_row_size, vtcm_layout->dst_spad_half_size, dst_row_size, 0); \ + } \ + const uint32_t next_step = step + 2; \ + if (next_step < total_steps) { \ + uint32_t ni = ir0 + pi_step; \ + const dma_addr_t psrc0_data = octx->src[0]->data + ni*nb01 + pi02*nb02 + pi03*nb03; \ + dma_queue_push(dma_q, \ + dma_make_data(src_spad, psrc0_data), \ + vtcm_layout->src0_spad_half_size, src0_row_size, src0_row_size, 1); \ + pi_step++; \ + if (pi_step == nrows_per_thread) { \ + pi_step = 0; \ + pi02++; \ + if (pi02 == ne02) { \ + pi02 = 0; \ + pi03++; \ + } \ + } \ + } \ + ci_step++; \ + if (ci_step == nrows_per_thread) { \ + ci_step = 0; \ + ci02++; \ + ci11_base++; \ + if (ci11_base == ne11) { \ + ci11_base = 0; \ + } \ + if (ci02 == ne02) { \ + ci02 = 0; \ + ci03++; \ + ci12_base++; \ + if (ci12_base == ne12) { \ + ci12_base = 0; \ + } \ + } \ + } \ + } \ + dma_queue_flush(dma_q); \ } -static void set_rows_thread_f16_f32(unsigned int nth, unsigned int ith, void *data) { - struct htp_set_rows_context * srctx = (struct htp_set_rows_context *)data; - struct htp_ops_context * octx = srctx->octx; +SET_ROWS_THREAD_DMA_FN(f32, int32_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); }) +SET_ROWS_THREAD_DMA_FN(f32, int64_t, { hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); }) - set_rows_preamble; +SET_ROWS_THREAD_DMA_FN(f16, int32_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); }) +SET_ROWS_THREAD_DMA_FN(f16, int64_t, { hvx_copy_f16_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, ne00); }) - uint64_t qt = HAP_perf_get_qtimer_count(); +SET_ROWS_THREAD_DMA_FN(q8_0, int32_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); }) +SET_ROWS_THREAD_DMA_FN(q8_0, int64_t, { hvx_quantize_row_q8_0_f32(dst_spad, (const float *)src_spad, ne00); }) - // parallelize by rows of src0 - const uint32_t dr = srctx->src0_nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = (ir0 + dr < nr) ? (ir0 + dr) : nr; +int op_set_rows(struct htp_ops_context * octx) { + const struct htp_set_rows_kernel_params * kparams = (const struct htp_set_rows_kernel_params *)octx->kernel_params; + set_rows_preamble; - const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); + if (htp_tensor_is_extended(octx->src[1])) { + return HTP_STATUS_NO_SUPPORT; + } - for (uint32_t i03 = 0; i03 < ne03; ++i03) { - for (uint32_t i02 = 0; i02 < ne02; ++i02) { - for (uint32_t i = ir0; i < ir1; ++i) { - const uint32_t i12 = fastmodulo(i03, ne12, &srctx->div_ne12); - const uint32_t i11 = fastmodulo(i02, ne11, &srctx->div_ne11); - const uint32_t i10 = i; + const struct htp_tensor * dst = octx->dst; + const uint32_t total_tasks = kparams->total_tasks; - const uintptr_t src1_addr = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12; + uint32_t task_start = 0; + uint32_t tasks = total_tasks; - uint32_t i1 = is_i32 ? *(int32_t *)src1_addr : *(int64_t *)src1_addr; - if (i1 >= ne1) { - // ignore invalid indices - continue; - } + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && !htp_tensor_is_permuted(dst); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + task_start = range.start; + tasks = range.count; + } - const uint8_t* src0_ptr = (const uint8_t *) octx->src[0]->data + i*nb01 + i02*nb02 + i03*nb03; - uint8_t* dst_ptr = (uint8_t *) octx->dst->data + i1*nb1 + i02*nb2 + i03*nb3; + if (tasks == 0) { + return HTP_STATUS_OK; + } - hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00); - } - } + if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; } - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "set-rows-f16-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, ir0, ir1, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, (unsigned) qt); -} + const uint32_t n_threads = octx->n_threads; -int op_set_rows(struct htp_ops_context * octx) { - set_rows_preamble; + // l2fetch the src1 (indices) tensor in the main thread + hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]); - const uint32_t n_threads = MIN(nr, octx->n_threads); + struct set_rows_context srctx; + srctx.octx = octx; + srctx.kparams = kparams; + srctx.task_start = task_start; + srctx.tasks = tasks; + srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); - if (octx->src[0]->type != HTP_TYPE_F32) { - return HTP_STATUS_NO_SUPPORT; - } + htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads); + srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; - if (octx->dst->type != HTP_TYPE_F32 && octx->dst->type != HTP_TYPE_F16) { - return HTP_STATUS_NO_SUPPORT; - } + work_queue_func_t q_func = NULL; + const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); - if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) { - return HTP_STATUS_NO_SUPPORT; + switch (octx->dst->type) { + case HTP_TYPE_F32: q_func = is_i32 ? set_rows_thread_dma_f32_int32_t : set_rows_thread_dma_f32_int64_t; break; + case HTP_TYPE_F16: q_func = is_i32 ? set_rows_thread_dma_f16_int32_t : set_rows_thread_dma_f16_int64_t; break; + case HTP_TYPE_Q8_0: q_func = is_i32 ? set_rows_thread_dma_q8_0_int32_t : set_rows_thread_dma_q8_0_int64_t; break; + default: return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; - } + FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n", + octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], + octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], + octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], + srctx.vtcm_layout.src0_bytes_per_thread * n_threads, + srctx.vtcm_layout.dst_bytes_per_thread * n_threads, + n_threads); - struct htp_set_rows_context srctx; - srctx.octx = octx; - srctx.div_ne12 = init_fastdiv_values(ne12); - srctx.div_ne11 = init_fastdiv_values(ne11); - - srctx.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads; - - switch(octx->dst->type) { - case HTP_TYPE_F32: - worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f32_f32, &srctx, n_threads); - break; - case HTP_TYPE_F16: - worker_pool_run_func(octx->ctx->worker_pool, set_rows_thread_f16_f32, &srctx, n_threads); - break; - default: - return HTP_STATUS_NO_SUPPORT; - } + work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/set-rows-ops.h b/ggml/src/ggml-hexagon/htp/set-rows-ops.h new file mode 100644 index 0000000..b9cb6cc --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/set-rows-ops.h @@ -0,0 +1,56 @@ +#ifndef HTP_SET_ROWS_OPS_H +#define HTP_SET_ROWS_OPS_H + +#include "hex-fastdiv.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" + +struct htp_set_rows_kernel_params { + uint32_t n_threads; + uint32_t total_tasks; + uint32_t vtcm_size; +}; + +struct htp_set_rows_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + + size_t src0_bytes_per_thread; + size_t dst_bytes_per_thread; + + size_t src0_spad_half_size; + size_t dst_spad_half_size; +}; + +static inline void htp_set_rows_vtcm_layout_build( + struct htp_set_rows_vtcm_layout * L, + int dst_type, + uint32_t ne00, + uint32_t n_threads) { + + size_t src0_row_size = ne00 * sizeof(float); + size_t dst_row_size = htp_tensor_get_row_size(dst_type, ne00); + + size_t src0_row_size_aligned = (src0_row_size + 255) & ~255; + size_t dst_row_size_aligned = (dst_row_size + 255) & ~255; + + L->src0_spad_half_size = src0_row_size_aligned; + L->dst_spad_half_size = dst_row_size_aligned; + + L->src0_bytes_per_thread = src0_row_size_aligned * 2; + L->dst_bytes_per_thread = dst_row_size_aligned * 2; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + L->total_bytes = off; +} + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_set_rows_kernel_params) <= 128, "htp_set_rows_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_SET_ROWS_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.c b/ggml/src/ggml-hexagon/htp/softmax-ops.c index d78bcc0..d65c3fa 100644 --- a/ggml/src/ggml-hexagon/htp/softmax-ops.c +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.c @@ -8,52 +8,49 @@ #include #include -#include "hex-dma.h" +#include "dma-queue.h" +#include "work-queue.h" #include "hvx-utils.h" #include "hex-fastdiv.h" +#include "hex-common.h" +#include "hex-profile.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" - -#define htp_softmax_preamble3 \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t ne10 = src1 ? src1->ne[0] : 1; \ - const uint32_t ne11 = src1 ? src1->ne[1] : 1; \ - const uint32_t ne12 = src1 ? src1->ne[2] : 1; \ - const uint32_t ne13 = src1 ? src1->ne[3] : 1; \ - \ - const uint32_t nb10 = src1 ? src1->nb[0] : 1; \ - const uint32_t nb11 = src1 ? src1->nb[1] : 1; \ - const uint32_t nb12 = src1 ? src1->nb[2] : 1; \ - const uint32_t nb13 = src1 ? src1->nb[3] : 1; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "htp/softmax-ops.h" +#include "hvx-flash-attn.h" struct htp_softmax_context { struct htp_ops_context * octx; + const struct htp_softmax_kernel_params * kparams; + + void * compute; + + dma_addr_t data_src0; + dma_addr_t data_src1; + dma_addr_t data_dst; + + uint8_t * vtcm_src0; + uint8_t * vtcm_src1; + uint8_t * vtcm_dst; + + uint32_t vtcm_src0_size_per_thread; + uint32_t vtcm_src1_size_per_thread; + uint32_t vtcm_dst_size_per_thread; + + uint32_t src0_spad_half_size; + uint32_t src1_spad_half_size; + uint32_t dst_spad_half_size; + + uint32_t src0_row_size_aligned; + uint32_t src1_row_size_aligned; + uint32_t dst_row_size_aligned; bool use_f16; - bool use_src1; uint32_t n_head; uint32_t n_head_log2; @@ -63,66 +60,78 @@ struct htp_softmax_context { float m0; float m1; - struct fastdiv_values fastdiv_ne01; - struct fastdiv_values fastdiv_ne02; - struct fastdiv_values fastdiv_ne12; // For mask broadcasting - struct fastdiv_values fastdiv_ne13; // For mask broadcasting + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne12; + struct fastdiv_values div_ne13; uint32_t src0_nrows_per_thread; + uint32_t row_start; + uint32_t nrows; + + float slopes[512] __attribute__((aligned(128))); }; -static void apply_mask(float * restrict wp0, - const float * restrict mp_f32, - const __fp16 * restrict mp_f16, - uint32_t ne00, - float slope, - bool use_f16) { - if (!mp_f32) { - return; - } - if (use_f16) { - for (uint32_t i = 0; i < ne00; ++i) { - wp0[i] += slope * (float) mp_f16[i]; - } - } else { - for (uint32_t i = 0; i < ne00; ++i) { - wp0[i] += slope * mp_f32[i]; - } - } -} +typedef void (*softmax_compute_fn_t)( + void * restrict dst, + const void * restrict src0, + const void * restrict mask, + uint32_t ne00, + float scale, + float slope +); -static void init_softmax_ctx(struct htp_softmax_context * smctx, struct htp_ops_context * octx) { - const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * src1 = octx->src[1]; +static void hvx_fast_softmax_prep_f16(const uint8_t * restrict src, + uint8_t * restrict dst, + const int num_elems, + float scale, + const uint8_t * restrict mask, + float slope) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const HVX_Vector * restrict v_mask = (const HVX_Vector *) mask; - memset(smctx, 0, sizeof(struct htp_softmax_context)); + HVX_Vector scale_vec = hvx_vec_splat_f32(scale); + HVX_Vector slope_vec = hvx_vec_splat_f32(slope); - memcpy(&smctx->scale, (float *) octx->op_params, sizeof(float)); - memcpy(&smctx->max_bias, (float *) octx->op_params + 1, sizeof(float)); + const int nvec_64 = num_elems / VLEN_FP16; + const int nloe_64 = num_elems % VLEN_FP16; - smctx->n_head = src0->ne[2]; - smctx->n_head_log2 = 1u << (uint32_t) floor(log2(smctx->n_head)); + #pragma unroll(2) + for (int i = 0; i < nvec_64; i++) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_mask[i]); + HVX_Vector m0 = Q6_V_lo_W(p); + HVX_Vector m1 = Q6_V_hi_W(p); - smctx->m0 = powf(2.0f, -(smctx->max_bias) / smctx->n_head_log2); - smctx->m1 = powf(2.0f, -(smctx->max_bias / 2.0f) / smctx->n_head_log2); + HVX_Vector s0 = v_src[2 * i]; + HVX_Vector s1 = v_src[2 * i + 1]; - smctx->use_src1 = (src1 != 0); - smctx->use_f16 = (src1 != 0) && (src1->type == HTP_TYPE_F16); + HVX_Vector v0 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(s0, scale_vec), Q6_Vqf32_vmpy_VsfVsf(m0, slope_vec)); + HVX_Vector v1 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(s1, scale_vec), Q6_Vqf32_vmpy_VsfVsf(m1, slope_vec)); - smctx->octx = octx; + v_dst[2 * i] = Q6_Vsf_equals_Vqf32(v0); + v_dst[2 * i + 1] = Q6_Vsf_equals_Vqf32(v1); + } - // Initialize fastdiv values - const uint32_t ne01 = src0->ne[1]; - const uint32_t ne02 = src0->ne[2]; + if (nloe_64 > 0) { + HVX_VectorPair p = hvx_vec_f16_to_f32(v_mask[nvec_64]); + HVX_Vector m0 = Q6_V_lo_W(p); - if (ne01 > 0) smctx->fastdiv_ne01 = init_fastdiv_values(ne01); - if (ne02 > 0) smctx->fastdiv_ne02 = init_fastdiv_values(ne02); + HVX_Vector s0 = v_src[2 * nvec_64]; + HVX_Vector v0 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(s0, scale_vec), Q6_Vqf32_vmpy_VsfVsf(m0, slope_vec)); - const uint32_t ne12 = src1 ? src1->ne[2] : 1; - const uint32_t ne13 = src1 ? src1->ne[3] : 1; + if (nloe_64 <= VLEN_FP32) { + hvx_vec_store_a(&v_dst[2 * nvec_64], nloe_64 * sizeof(float), Q6_Vsf_equals_Vqf32(v0)); + } else { + v_dst[2 * nvec_64] = Q6_Vsf_equals_Vqf32(v0); + + HVX_Vector m1 = Q6_V_hi_W(p); + HVX_Vector s1 = v_src[2 * nvec_64 + 1]; + HVX_Vector v1 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(s1, scale_vec), Q6_Vqf32_vmpy_VsfVsf(m1, slope_vec)); - if (ne12 > 0) smctx->fastdiv_ne12 = init_fastdiv_values(ne12); - if (ne13 > 0) smctx->fastdiv_ne13 = init_fastdiv_values(ne13); + hvx_vec_store_a(&v_dst[2 * nvec_64 + 1], (nloe_64 - VLEN_FP32) * sizeof(float), Q6_Vsf_equals_Vqf32(v1)); + } + } } static void hvx_fast_softmax_prep_f32(const uint8_t * restrict src, @@ -131,57 +140,68 @@ static void hvx_fast_softmax_prep_f32(const uint8_t * restrict src, float scale, const uint8_t * restrict mask, float slope) { - const uint8_t * restrict src_curr = src; - uint8_t * restrict dst_curr = dst; - const uint8_t * restrict mask_curr = mask; + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const HVX_Vector * restrict v_mask = (const HVX_Vector *) mask; HVX_Vector scale_vec = hvx_vec_splat_f32(scale); HVX_Vector slope_vec = hvx_vec_splat_f32(slope); - int step_of_1 = num_elems >> 5; + const int nvec = num_elems / VLEN_FP32; + const int nloe = num_elems % VLEN_FP32; #pragma unroll(4) - for (int i = 0; i < step_of_1; i++) { - HVX_Vector v1 = *(HVX_Vector *) src_curr; - - HVX_Vector v3 = *(HVX_Vector *) mask_curr; + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_src[i]; + HVX_Vector v3 = v_mask[i]; HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec); - HVX_Vector v4 = Q6_Vqf32_vmpy_VsfVsf(v3, slope_vec); - HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, v4); - *(HVX_Vector *) dst_curr = Q6_Vsf_equals_Vqf32(v5); + v_dst[i] = Q6_Vsf_equals_Vqf32(v5); + } + + if (nloe > 0) { + HVX_Vector v1 = v_src[nvec]; + HVX_Vector v3 = v_mask[nvec]; - src_curr += VLEN; - dst_curr += VLEN; - mask_curr += VLEN; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec); + HVX_Vector v4 = Q6_Vqf32_vmpy_VsfVsf(v3, slope_vec); + HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, v4); + + hvx_vec_store_a(&v_dst[nvec], nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v5)); } } -static void hvx_fast_softmax_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint8_t * restrict pad, const int num_elems) { - const HVX_Vector * restrict v_src = (HVX_Vector *) src; - HVX_Vector * restrict v_pad = (HVX_Vector *) pad; +static void hvx_fast_softmax_f32(const uint8_t * restrict src, uint8_t * restrict dst, const int num_elems) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; HVX_Vector * restrict v_dst = (HVX_Vector *) dst; - HVX_Vector sum_vec = Q6_V_vsplat_R(0x00000000); - HVX_Vector max_vec = hvx_vec_splat_f32(((const float *) src)[0]); - HVX_Vector zero_v = Q6_V_vzero(); - HVX_Vector one_v = hvx_vec_splat_f32(1.0); + const int nvec = num_elems / VLEN_FP32; + const int nloe = num_elems % VLEN_FP32; - int step_of_1 = num_elems >> 5; + HVX_Vector max_vec = hvx_vec_splat_f32(((const float *) src)[0]); - #pragma unroll(4) - for (int i = 0; i < step_of_1; i++) { + #pragma unroll(2) + for (int i = 0; i < nvec; i++) { HVX_Vector v1 = v_src[i]; max_vec = Q6_Vsf_vmax_VsfVsf(max_vec, v1); } - max_vec = hvx_vec_reduce_max_f32(max_vec); // replicated over all lanes + if (nloe > 0) { + HVX_VectorPred q_mask = Q6_Q_vsetq_R(nloe * sizeof(float)); + HVX_Vector neg_inf = hvx_vec_splat_f32(-INFINITY); + HVX_Vector v_tail = Q6_V_vmux_QVV(q_mask, v_src[nvec], neg_inf); + max_vec = Q6_Vsf_vmax_VsfVsf(max_vec, v_tail); + } - #pragma unroll(4) - for (int i = 0; i < step_of_1; i++) { + max_vec = hvx_vec_reduce_max_f32(max_vec); + + HVX_Vector sum_vec = Q6_V_vsplat_R(0x00000000); + + #pragma unroll(2) + for (int i = 0; i < nvec; i++) { HVX_Vector v1 = v_src[i]; HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, max_vec); @@ -189,205 +209,396 @@ static void hvx_fast_softmax_f32(const uint8_t * restrict src, uint8_t * restric sum_vec = Q6_Vqf32_vadd_VsfVsf(Q6_Vsf_equals_Vqf32(sum_vec), v3); - v_pad[i] = v3; + v_dst[i] = v3; + } + + if (nloe > 0) { + HVX_VectorPred q_mask = Q6_Q_vsetq_R(nloe * sizeof(float)); + HVX_Vector v1 = v_src[nvec]; + HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, max_vec); + HVX_Vector v3 = hvx_vec_exp_f32(Q6_Vsf_equals_Vqf32(v2)); + HVX_Vector v3_pad = Q6_V_vmux_QVV(q_mask, v3, Q6_V_vzero()); + + sum_vec = Q6_Vqf32_vadd_VsfVsf(Q6_Vsf_equals_Vqf32(sum_vec), v3_pad); + v_dst[nvec] = v3_pad; } - sum_vec = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_vec)); // replicated over all lanes + sum_vec = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_vec)); - HVX_VectorPred pos_sum = Q6_Q_vcmp_gt_VwVw(sum_vec, zero_v); + HVX_VectorPred pos_sum = Q6_Q_vcmp_gt_VwVw(sum_vec, Q6_V_vzero()); HVX_Vector v4 = hvx_vec_inverse_f32(sum_vec); - HVX_Vector scale_vec = Q6_V_vmux_QVV(pos_sum, v4, one_v); + HVX_Vector scale_vec = Q6_V_vmux_QVV(pos_sum, v4, hvx_vec_splat_f32(1.0f)); - #pragma unroll(4) - for (int i = 0; i < step_of_1; i++) { - HVX_Vector v1 = v_pad[i]; + #pragma unroll(2) + for (int i = 0; i < nvec; i++) { + HVX_Vector v1 = v_dst[i]; HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec); v_dst[i] = Q6_Vsf_equals_Vqf32(v2); } + + if (nloe > 0) { + HVX_Vector v1 = v_dst[nvec]; + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_vec); + hvx_vec_store_a(&v_dst[nvec], nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v2)); + } } -static float hvx_softmax_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint8_t * restrict spad, const int num_elems, const float max) { - hvx_sub_scalar_f32(spad, src, max, num_elems); +static void compute_fast_softmax_f32_nomask( + void * restrict dst, + const void * restrict src0, + const void * restrict mask, + uint32_t ne00, + float scale, + float slope +) { + (void) mask; + (void) slope; + hvx_scale_f32((uint8_t *) dst, (const uint8_t *) src0, ne00, scale); + hvx_fast_softmax_f32((const uint8_t *) dst, (uint8_t *) dst, ne00); +} - hvx_exp_f32(dst, spad, num_elems, false); - return hvx_reduce_sum_f32(dst, num_elems); +static void compute_fast_softmax_f32_mask_f32( + void * restrict dst, + const void * restrict src0, + const void * restrict mask, + uint32_t ne00, + float scale, + float slope +) { + hvx_fast_softmax_prep_f32((const uint8_t *) src0, (uint8_t *) dst, ne00, scale, (const uint8_t *) mask, slope); + hvx_fast_softmax_f32((const uint8_t *) dst, (uint8_t *) dst, ne00); } -static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) { - struct htp_softmax_context * smctx = (struct htp_softmax_context *) data; - struct htp_ops_context * octx = smctx->octx; +static void compute_fast_softmax_f32_mask_f16( + void * restrict dst, + const void * restrict src0, + const void * restrict mask, + uint32_t ne00, + float scale, + float slope +) { + hvx_fast_softmax_prep_f16((const uint8_t *) src0, (uint8_t *) dst, ne00, scale, (const uint8_t *) mask, slope); + hvx_fast_softmax_f32((const uint8_t *) dst, (uint8_t *) dst, ne00); +} + +static const softmax_compute_fn_t softmax_kernels[HTP_SOFTMAX_KERNEL_COUNT] = { + [HTP_SOFTMAX_KERNEL_NOMASK] = compute_fast_softmax_f32_nomask, + [HTP_SOFTMAX_KERNEL_MASK_F32] = compute_fast_softmax_f32_mask_f32, + [HTP_SOFTMAX_KERNEL_MASK_F16] = compute_fast_softmax_f32_mask_f16, +}; +static void softmax_thread_dma(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; + struct htp_ops_context * octx = smctx->octx; const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - htp_softmax_preamble3; - - const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows + const uint32_t src0_nrows = smctx->nrows; const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); - // no work for this thread if (src0_start_row >= src0_end_row) { return; } - uint64_t qt = HAP_perf_get_qtimer_count(); + const dma_addr_t data_src0 = smctx->data_src0; + const dma_addr_t data_dst = smctx->data_dst; - int is_aligned = 1; - int opt_path = 0; + const size_t src0_row_size = src0->ne[0] * sizeof(float); + const size_t dst_row_size = src0->ne[0] * sizeof(float); - if (!hex_is_aligned((void *) src0->data, VLEN) || !hex_is_aligned((void *) dst->data, VLEN)) { - is_aligned = 0; - FARF(HIGH, "softmax-f32: unaligned addresses in elementwise op, possibly slower execution\n"); - } + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); - // Only use the fast path when aligned AND row size is multiple of VLEN (128 bytes) - // The fast path (hvx_fast_softmax_f32) doesn't handle tail elements - // The non-opt path uses hvx_softmax_f32 which properly handles all sizes via its helper functions - if ((1 == is_aligned) && !(nb01 & (VLEN - 1))) { - opt_path = 1; - } + const size_t src0_vtcm_half = smctx->src0_spad_half_size; + const size_t dst_vtcm_half = smctx->dst_spad_half_size; - uint8_t * src0_spad_data = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * src1_spad_data = octx->src1_spad.data + (ith * octx->src1_spad.size_per_thread); - uint8_t * dst_spad_data = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); + dma_queue * dma_q = octx->ctx->dma[ith]; - float * wp0 = (float *) src0_spad_data; - float * wp1 = (float *) src1_spad_data; - float * wp2 = (float *) dst_spad_data; + for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { + dma_addr_t cur_dst = data_dst + r * dst_row_size; + dma_addr_t cur_src0 = data_src0 + r * src0_row_size; + void * d_spad = dst_vtcm_base + idx * dst_vtcm_half; + void * s_spad = src0_vtcm_base + idx * src0_vtcm_half; - uint32_t prev_i2 = (uint32_t)-1; - float slope = 1.0f; + dma_queue_push(dma_q, dma_make_data(cur_dst, d_spad), + dst_row_size, smctx->dst_row_size_aligned, dst_row_size, 0); + dma_queue_push(dma_q, dma_make_data(s_spad, cur_src0), + smctx->src0_row_size_aligned, src0_row_size, src0_row_size, 1); + } + + softmax_compute_fn_t compute = (softmax_compute_fn_t) smctx->compute; + const uint32_t ne00 = src0->ne[0]; for (uint32_t r = src0_start_row; r < src0_end_row; ++r) { - uint32_t i1 = fastmodulo(r, ne01, &smctx->fastdiv_ne01); - uint32_t r_div_ne01 = fastdiv(r, &smctx->fastdiv_ne01); - uint32_t i2 = fastmodulo(r_div_ne01, ne02, &smctx->fastdiv_ne02); - uint32_t i3 = fastdiv(r_div_ne01, &smctx->fastdiv_ne02); - - // Map to original logic indices - // i01 = i1 - // i02 = i2 - // i03 = i3 - - const uint32_t i11 = i1; - // const uint32_t i12 = i2 % ne12; - // const uint32_t i13 = i3 % ne13; - - uint32_t i12, i13; - if (ne12 == ne02) { - i12 = i2; - } else { - i12 = fastmodulo(i2, ne12, &smctx->fastdiv_ne12); + void * d_spad = (void *) dma_queue_pop(dma_q).src; + void * s_spad = (void *) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); + compute(d_spad, s_spad, NULL, ne00, smctx->scale, 1.0f); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, r); + + dma_addr_t cur_dst = data_dst + r * dst_row_size; + dma_queue_push(dma_q, dma_make_data(cur_dst, d_spad), + dst_row_size, smctx->dst_row_size_aligned, dst_row_size, 1); + + const uint32_t next_r = r + 2; + if (next_r < src0_end_row) { + dma_addr_t next_src0 = data_src0 + next_r * src0_row_size; + dma_queue_push(dma_q, dma_make_data(s_spad, next_src0), + smctx->src0_row_size_aligned, src0_row_size, src0_row_size, 1); } + } - if (ne13 == ne03) { - i13 = i3; - } else { - i13 = fastmodulo(i3, ne13, &smctx->fastdiv_ne13); - } + dma_queue_flush(dma_q); +} - // ALiBi - if (i2 != prev_i2) { - const uint32_t h = i2; // head - slope = (smctx->max_bias > 0.0f) ? h < smctx->n_head_log2 ? powf(smctx->m0, h + 1) : powf(smctx->m1, 2 * (h - smctx->n_head_log2) + 1) : 1.0f; - prev_i2 = i2; - } +static void softmax_thread_mask_dma(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_softmax_context * smctx = (const struct htp_softmax_context *) data; + struct htp_ops_context * octx = smctx->octx; + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - float * sp = (float *) ((char *) src0->data + i1 * nb01 + i2 * nb02 + i3 * nb03); - float * dp = (float *) ((char *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3); + const uint32_t src0_nrows = smctx->nrows; + const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread; - // broadcast the mask across rows - __fp16 * mp_f16 = (smctx->use_src1) ? (__fp16 *) ((char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13) : NULL; - float * mp_f32 = (smctx->use_src1) ? (float *) ((char *) src1->data + i11 * nb11 + i12 * nb12 + i13 * nb13) : NULL; + const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows); - if ((1 == opt_path) && (mp_f32) && !(smctx->use_f16)) { - hvx_fast_softmax_prep_f32((const uint8_t *) sp, (uint8_t *) wp0, ne00, smctx->scale, (const uint8_t *) mp_f32, slope); - hvx_fast_softmax_f32((const uint8_t *) wp0, (uint8_t *) dp, (uint8_t *) wp1, ne00); - } else if (1 == opt_path) { - hvx_scale_f32((uint8_t *) wp0, (const uint8_t *) sp, ne00, smctx->scale); - apply_mask(wp0, mp_f32, mp_f16, ne00, slope, smctx->use_f16); - hvx_fast_softmax_f32((const uint8_t *) wp0, (uint8_t *) dp, (uint8_t *) wp1, ne00); - } else { - // Non-optimized path: uses HVX helper functions that properly handle all tensor sizes - // including non-multiples of 32 (the HVX vector lane count for f32) - hvx_scale_f32((uint8_t *) wp0, (const uint8_t *) sp, ne00, smctx->scale); - apply_mask(wp0, mp_f32, mp_f16, ne00, slope, smctx->use_f16); - float max = hvx_reduce_max_f32((const uint8_t *) wp0, ne00); - float sum = hvx_softmax_f32((const uint8_t *) wp0, (uint8_t *) wp2, (uint8_t *) wp1, ne00, max); - sum = sum > 0.0 ? (1.0 / sum) : 1; - hvx_scale_f32((uint8_t *) dp, (const uint8_t *) wp2, ne00, sum); + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src0 = smctx->data_src0; + const dma_addr_t data_src1 = smctx->data_src1; + const dma_addr_t data_dst = smctx->data_dst; + + const size_t src0_row_size = src0->ne[0] * sizeof(float); + const size_t dst_row_size = src0->ne[0] * sizeof(float); + const size_t mask_row_size = smctx->use_f16 ? (src1->ne[0] * sizeof(__fp16)) : (src1->ne[0] * sizeof(float)); + + uint8_t * src0_vtcm_base = smctx->vtcm_src0 + (ith * smctx->vtcm_src0_size_per_thread); + uint8_t * src1_vtcm_base = smctx->vtcm_src1 + (ith * smctx->vtcm_src1_size_per_thread); + uint8_t * dst_vtcm_base = smctx->vtcm_dst + (ith * smctx->vtcm_dst_size_per_thread); + + const size_t src0_vtcm_half = smctx->src0_spad_half_size; + const size_t src1_vtcm_half = smctx->src1_spad_half_size; + const size_t dst_vtcm_half = smctx->dst_spad_half_size; + + const uint32_t nb11 = src1->nb[1]; + const uint32_t nb12 = src1->nb[2]; + const uint32_t nb13 = src1->nb[3]; + + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne01 = src0->ne[1]; + const uint32_t ne02 = src0->ne[2]; + const uint32_t ne03 = src0->ne[3]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t ne13 = src1->ne[3]; + + const struct fastdiv_values * div_ne01 = &smctx->div_ne01; + const struct fastdiv_values * div_ne02 = &smctx->div_ne02; + const struct fastdiv_values * div_ne12 = &smctx->div_ne12; + const struct fastdiv_values * div_ne13 = &smctx->div_ne13; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + for (uint32_t r = src0_start_row, idx = 0; r < src0_end_row && idx < 2; r++, idx++) { + dma_addr_t cur_dst = data_dst + r * dst_row_size; + dma_addr_t cur_src0 = data_src0 + r * src0_row_size; + + uint32_t i1 = fastmodulo(r, ne01, div_ne01); + uint32_t r_div_ne01 = fastdiv(r, div_ne01); + uint32_t i2 = fastmodulo(r_div_ne01, ne02, div_ne02); + uint32_t i3 = fastdiv(r_div_ne01, div_ne02); + uint32_t i12 = (ne12 == ne02) ? i2 : fastmodulo(i2, ne12, div_ne12); + uint32_t i13 = (ne13 == ne03) ? i3 : fastmodulo(i3, ne13, div_ne13); + dma_addr_t cur_src1 = data_src1 + i1 * nb11 + i12 * nb12 + i13 * nb13; + + void * d_spad = dst_vtcm_base + idx * dst_vtcm_half; + void * s_spad = src0_vtcm_base + idx * src0_vtcm_half; + void * m_spad = src1_vtcm_base + idx * src1_vtcm_half; + + dma_queue_push(dma_q, dma_make_data(cur_dst, d_spad), + dst_row_size, smctx->dst_row_size_aligned, dst_row_size, 0); + dma_queue_push(dma_q, dma_make_data(s_spad, cur_src0), + smctx->src0_row_size_aligned, src0_row_size, src0_row_size, 1); + dma_queue_push(dma_q, dma_make_data(m_spad, cur_src1), + smctx->src1_row_size_aligned, mask_row_size, mask_row_size, 1); + } + + softmax_compute_fn_t compute = (softmax_compute_fn_t) smctx->compute; + const bool has_bias = smctx->max_bias > 0.0f; + uint32_t prev_i2 = (uint32_t)-1; + float slope = 1.0f; + + for (uint32_t r = src0_start_row; r < src0_end_row; ++r) { + void * d_spad = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * s_spad = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + void * m_spad = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + if (has_bias) { + uint32_t r_div_ne01 = fastdiv(r, div_ne01); + uint32_t i2 = fastmodulo(r_div_ne01, ne02, div_ne02); + if (i2 != prev_i2) { + slope = smctx->slopes[i2]; + prev_i2 = i2; + } + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); + compute(d_spad, s_spad, m_spad, ne00, smctx->scale, slope); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, r); + + dma_addr_t cur_dst = data_dst + r * dst_row_size; + dma_queue_push(dma_q, dma_make_data(cur_dst, d_spad), + dst_row_size, smctx->dst_row_size_aligned, dst_row_size, 1); + + const uint32_t next_r = r + 2; + if (next_r < src0_end_row) { + dma_addr_t next_src0 = data_src0 + next_r * src0_row_size; + + uint32_t ni1 = fastmodulo(next_r, ne01, div_ne01); + uint32_t nr_div_ne01 = fastdiv(next_r, div_ne01); + uint32_t ni2 = fastmodulo(nr_div_ne01, ne02, div_ne02); + uint32_t ni3 = fastdiv(nr_div_ne01, div_ne02); + uint32_t ni12 = (ne12 == ne02) ? ni2 : fastmodulo(ni2, ne12, div_ne12); + uint32_t ni13 = (ne13 == ne03) ? ni3 : fastmodulo(ni3, ne13, div_ne13); + dma_addr_t next_src1 = data_src1 + ni1 * nb11 + ni12 * nb12 + ni13 * nb13; + + dma_queue_push(dma_q, dma_make_data(s_spad, next_src0), + smctx->src0_row_size_aligned, src0_row_size, src0_row_size, 1); + dma_queue_push(dma_q, dma_make_data(m_spad, next_src1), + smctx->src1_row_size_aligned, mask_row_size, mask_row_size, 1); } } - qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt); - FARF(HIGH, "softmax-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u : opt %u f16 %u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, - ne0, ne1, ne2, ne3, opt_path, smctx->use_f16, (unsigned) qt); + dma_queue_flush(dma_q); } static int execute_op_softmax_f32(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; - const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - struct htp_softmax_context smctx; const char * op_type = "softmax-f32"; - init_softmax_ctx(&smctx, octx); + const struct htp_softmax_kernel_params * kparams = + (const struct htp_softmax_kernel_params *) octx->kernel_params; - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - - smctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; - - const size_t src0_row_size = src0->nb[1]; - const size_t src1_row_size = src0_row_size; - const size_t dst_row_size = dst->nb[1]; - - // VTCM scratchpads for all tensors - // 4 rows per thread, padded to HVX vector size - octx->src0_spad.size_per_thread = hex_round_up(4 * src0_row_size, 128); - octx->src1_spad.size_per_thread = hex_round_up(4 * src1_row_size, 128); - octx->dst_spad.size_per_thread = hex_round_up(4 * dst_row_size, 128); - - octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads; - octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads; - octx->dst_spad.size = octx->dst_spad.size_per_thread * n_threads; - - size_t spad_size = octx->src0_spad.size + octx->src1_spad.size + octx->dst_spad.size; - - if (src1) { - FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", - op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], octx->src0_spad.size, octx->src1_spad.size, - octx->dst_spad.size); - } else { - FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", op_type, - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size); + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; } - // Make sure the reserved vtcm size is sufficient - if (octx->ctx->vtcm_size < spad_size) { - FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, spad_size); + if (kparams->kernel_type >= HTP_SOFTMAX_KERNEL_COUNT) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (octx->ctx->vtcm_size < (size_t) kparams->vtcm_size) { + FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %u\n", + op_type, octx->ctx->vtcm_size, kparams->vtcm_size); return HTP_STATUS_VTCM_TOO_SMALL; } - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL; - octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->dst_spad.src = NULL; + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t elem_size = sizeof(float); + const size_t dst_row_size = dst->nb[1]; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + if (nrows < octx->n_threads) { + htp_ops_context_set_n_threads(octx, nrows ? nrows : 1); + } + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) return err; + const uint32_t n_threads = octx->n_threads; + uint8_t * const vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + + struct htp_softmax_vtcm_layout layout; + htp_softmax_vtcm_layout_build(&layout, + src0->ne[0], + kparams->use_src1 ? octx->src[1]->ne[0] : 0, + kparams->use_src1 != 0, + kparams->use_f16 != 0, + kparams->n_threads); + + struct htp_softmax_context smctx = { + .octx = octx, + .kparams = kparams, + .compute = (void *) softmax_kernels[kparams->kernel_type], + + .data_src0 = src0->data, + .data_src1 = kparams->use_src1 ? octx->src[1]->data : 0, + .data_dst = dst->data, + + .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_src0), + .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_dst), + .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, vtcm_base, layout.off_src1, kparams->use_src1), + + .vtcm_src0_size_per_thread = (uint32_t) layout.src0_bytes_per_thread, + .vtcm_src1_size_per_thread = (uint32_t) layout.src1_bytes_per_thread, + .vtcm_dst_size_per_thread = (uint32_t) layout.dst_bytes_per_thread, + + .src0_spad_half_size = (uint32_t) layout.src0_spad_half_size, + .src1_spad_half_size = (uint32_t) layout.src1_spad_half_size, + .dst_spad_half_size = (uint32_t) layout.dst_spad_half_size, + + .src0_row_size_aligned = (uint32_t) layout.src0_spad_half_size, + .src1_row_size_aligned = (uint32_t) layout.src1_spad_half_size, + .dst_row_size_aligned = (uint32_t) layout.dst_spad_half_size, + + .use_f16 = kparams->use_f16 != 0, + + .n_head = kparams->n_head, + .n_head_log2 = kparams->n_head_log2, + + .scale = kparams->scale, + .max_bias = kparams->max_bias, + .m0 = kparams->m0, + .m1 = kparams->m1, + + .div_ne01 = kparams->div_ne01, + .div_ne02 = kparams->div_ne02, + .div_ne12 = kparams->div_ne12, + .div_ne13 = kparams->div_ne13, + + .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .row_start = row_start, + .nrows = nrows, + }; + + if (kparams->max_bias > 0.0f && kparams->use_src1) { + if (kparams->n_head > 512) { + return HTP_STATUS_INVAL_PARAMS; + } + for (uint32_t h = 0; h < kparams->n_head; h += 32) { + HVX_Vector v_slopes = hvx_alibi_slopes(h, 1, kparams->n_head_log2, kparams->m0, kparams->m1); + hvx_vmem(&smctx.slopes[h]) = v_slopes; + } + } - worker_pool_run_func(octx->ctx->worker_pool, softmax_job_f32, &smctx, n_threads); + work_queue_func_t task_func = kparams->use_src1 ? softmax_thread_mask_dma : softmax_thread_dma; + work_queue_run(octx->ctx->work_queue, task_func, &smctx, n_threads); - return err; + return HTP_STATUS_OK; } int op_softmax(struct htp_ops_context * octx) { diff --git a/ggml/src/ggml-hexagon/htp/softmax-ops.h b/ggml/src/ggml-hexagon/htp/softmax-ops.h new file mode 100644 index 0000000..ef65673 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/softmax-ops.h @@ -0,0 +1,94 @@ +#ifndef HTP_SOFTMAX_OPS_H +#define HTP_SOFTMAX_OPS_H + +#include +#include +#include +#include +#include "hex-fastdiv.h" +#include "hex-common.h" +#include "htp-vtcm.h" + +enum htp_softmax_kernel_type { + HTP_SOFTMAX_KERNEL_NOMASK = 0, + HTP_SOFTMAX_KERNEL_MASK_F32, + HTP_SOFTMAX_KERNEL_MASK_F16, + HTP_SOFTMAX_KERNEL_COUNT, +}; + +struct htp_softmax_kernel_params { + uint8_t kernel_type; + uint32_t n_threads; + uint32_t vtcm_size; + + uint32_t n_head; + uint32_t n_head_log2; + uint32_t use_src1; + uint32_t use_f16; + + float scale; + float max_bias; + float m0; + float m1; + + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne12; + struct fastdiv_values div_ne13; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_softmax_kernel_params) <= 128, "htp_softmax_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_softmax_kernel_params) <= 128, "htp_softmax_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_softmax_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_dst; + size_t off_src1; + + size_t src0_bytes_per_thread; + size_t dst_bytes_per_thread; + size_t src1_bytes_per_thread; + + size_t src0_spad_half_size; + size_t dst_spad_half_size; + size_t src1_spad_half_size; +}; + +static inline void htp_softmax_vtcm_layout_build( + struct htp_softmax_vtcm_layout * L, + uint32_t ne00, + uint32_t ne10, + bool use_src1, + bool use_f16, + uint32_t n_threads +) { + size_t src0_row_size = ne00 * sizeof(float); + size_t dst_row_size = ne00 * sizeof(float); + size_t src1_row_size = use_src1 ? (ne10 * (use_f16 ? 2 : 4)) : 0; + + size_t src0_row_size_aligned = hex_round_up(src0_row_size, 128); + size_t dst_row_size_aligned = hex_round_up(dst_row_size, 128); + size_t src1_row_size_aligned = use_src1 ? hex_round_up(src1_row_size, 128) : 0; + + L->src0_spad_half_size = src0_row_size_aligned; + L->dst_spad_half_size = dst_row_size_aligned; + L->src1_spad_half_size = src1_row_size_aligned; + + // Double buffering: 2 half-buffers per thread + L->src0_bytes_per_thread = src0_row_size_aligned * 2; + L->dst_bytes_per_thread = dst_row_size_aligned * 2; + L->src1_bytes_per_thread = src1_row_size_aligned * 2; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes_per_thread * n_threads, use_src1); + + L->total_bytes = off; +} + +#endif // HTP_SOFTMAX_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.c b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c new file mode 100644 index 0000000..628ca65 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.c @@ -0,0 +1,290 @@ +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include + +#include "hex-common.h" +#include "hex-profile.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-types.h" +#include "hvx-utils.h" +#include "solve-tri-ops.h" + +struct htp_solve_tri_context { + struct htp_ops_context * octx; + const struct htp_solve_tri_kernel_params * kparams; + uint32_t jobs_per_thread; + uint32_t total_jobs; + uint32_t job_start; + uint32_t k_chunks; + uint32_t col_block; +}; + +static inline void solve_tri_row_scalar(const float * A_row, + const float * B_row, + float * X, + uint32_t row, + uint32_t k, + uint32_t col0, + uint32_t coln, + float inv_diag) { + for (uint32_t col = col0; col < col0 + coln; ++col) { + float sum = 0.0f; + for (uint32_t t = 0; t < row; ++t) { + sum += A_row[t] * X[t * k + col]; + } + X[row * k + col] = (B_row[col] - sum) * inv_diag; + } +} + +static inline HVX_Vector hvx_load_partial_f32(const float * src, uint32_t n) { + HVX_Vector v = *((const HVX_UVector *) src); + HVX_VectorPred mask = Q6_Q_vsetq2_R(n * sizeof(float)); + return Q6_V_vmux_QVV(mask, v, Q6_V_vzero()); +} + +static inline void solve_tri_row_hvx(const float * A_row, + const float * B_row, + float * X, + uint32_t row, + uint32_t k, + uint32_t col0, + uint32_t coln, + float inv_diag) { + const bool full = (coln == VLEN_FP32); + + HVX_Vector sum_v = Q6_V_vzero(); + for (uint32_t t = 0; t < row; ++t) { + const float a = A_row[t]; + const float * x_row_col = X + t * k + col0; + + HVX_Vector x_v = full ? *((const HVX_UVector *) x_row_col) : hvx_load_partial_f32(x_row_col, coln); + HVX_Vector a_v = hvx_vec_splat_f32(a); + sum_v = hvx_vec_add_f32_f32(sum_v, hvx_vec_mul_f32_f32(x_v, a_v)); + } + + const float * b_row_col = B_row + col0; + float * x_out_col = X + row * k + col0; + + HVX_Vector b_v = full ? *((const HVX_UVector *) b_row_col) : hvx_load_partial_f32(b_row_col, coln); + HVX_Vector inv_diag_v = hvx_vec_splat_f32(inv_diag); + + HVX_Vector out_v = hvx_vec_mul_f32_f32(hvx_vec_sub_f32_f32(b_v, sum_v), inv_diag_v); + hvx_vec_store_u((void *) x_out_col, coln * sizeof(float), out_v); +} + +// Batch-level thread: each job is one full batch. +static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void * data) { + struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data; + struct htp_ops_context * octx = sctx->octx; + const struct htp_solve_tri_kernel_params * kparams = sctx->kparams; + + const struct htp_tensor * src0 = octx->src[0]; // A + const struct htp_tensor * src1 = octx->src[1]; // B + const struct htp_tensor * dst = octx->dst; // X + + const uint32_t n = src0->ne[0]; + const uint32_t k = src1->ne[0]; + + const uint32_t ne02 = src0->ne[2]; + + const uint32_t col_block = VLEN_FP32; + const uint32_t k_full = k & ~(col_block - 1); + + const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith; + const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch); + + for (uint32_t batch = start_batch; batch < end_batch; ++batch) { + const uint32_t i03 = fastdiv(batch, &kparams->div_ne02); + const uint32_t i02 = batch - i03 * ne02; + + const float * A_batch = + (const float *) ((const uint8_t *) (uintptr_t) src0->data + i02 * src0->nb[2] + i03 * src0->nb[3]); + const float * B_batch = + (const float *) ((const uint8_t *) (uintptr_t) src1->data + i02 * src1->nb[2] + i03 * src1->nb[3]); + float * X_batch = (float *) ((uint8_t *) (uintptr_t) dst->data + i02 * dst->nb[2] + i03 * dst->nb[3]); + + for (uint32_t row = 0; row < n; ++row) { + const float diag = A_batch[row * n + row]; + const float inv_diag = 1.0f / diag; + const float * A_row = A_batch + row * n; + const float * B_row = B_batch + row * k; + + uint32_t col0 = 0; + for (; col0 < k_full; col0 += col_block) { + solve_tri_row_hvx(A_row, B_row, X_batch, row, k, col0, col_block, inv_diag); + } + + if (col0 < k) { + const uint32_t coln = k - col0; + if (coln >= 8) { + solve_tri_row_hvx(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); + } else { + solve_tri_row_scalar(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); + } + } + } + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_batch); + + FARF(HIGH, "solve-tri-batch %d/%d: A=(%ux%u) B=(%ux%u) batch %u:%u\n", + ith, nth, n, n, k, n, start_batch, end_batch); +} + +// Chunk-level thread: each job is one (batch, col_chunk) pair. +static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void * data) { + struct htp_solve_tri_context * sctx = (struct htp_solve_tri_context *) data; + struct htp_ops_context * octx = sctx->octx; + const struct htp_solve_tri_kernel_params * kparams = sctx->kparams; + + const struct htp_tensor * src0 = octx->src[0]; // A + const struct htp_tensor * src1 = octx->src[1]; // B + const struct htp_tensor * dst = octx->dst; // X + + const uint32_t n = src0->ne[0]; + const uint32_t k = src1->ne[0]; + + const uint32_t ne02 = src0->ne[2]; + + const uint32_t start_job = sctx->job_start + sctx->jobs_per_thread * ith; + const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job); + + for (uint32_t job = start_job; job < end_job; ++job) { + const uint32_t batch = fastdiv(job, &kparams->div_k_chunks); + const uint32_t chunk = job - batch * sctx->k_chunks; + + const uint32_t i03 = fastdiv(batch, &kparams->div_ne02); + const uint32_t i02 = batch - i03 * ne02; + + const float * A_batch = + (const float *) ((const uint8_t *) (uintptr_t) src0->data + i02 * src0->nb[2] + i03 * src0->nb[3]); + const float * B_batch = + (const float *) ((const uint8_t *) (uintptr_t) src1->data + i02 * src1->nb[2] + i03 * src1->nb[3]); + float * X_batch = (float *) ((uint8_t *) (uintptr_t) dst->data + i02 * dst->nb[2] + i03 * dst->nb[3]); + + const uint32_t col0 = chunk * sctx->col_block; + const uint32_t coln = MIN(sctx->col_block, k - col0); + + for (uint32_t row = 0; row < n; ++row) { + const float diag = A_batch[row * n + row]; + const float inv_diag = 1.0f / diag; + + const float * A_row = A_batch + row * n; + const float * B_row = B_batch + row * k; + + if (coln >= 8) { + solve_tri_row_hvx(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); + } else { + solve_tri_row_scalar(A_row, B_row, X_batch, row, k, col0, coln, inv_diag); + } + } + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_job); + + FARF(HIGH, "solve-tri-chunk %d/%d: A=(%ux%u) B=(%ux%u) jobs %u:%u\n", + ith, nth, n, n, k, n, start_job, end_job); +} + +int op_solve_tri(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; // A + const struct htp_tensor * src1 = octx->src[1]; // B + const struct htp_tensor * dst = octx->dst; // X + const struct htp_solve_tri_kernel_params * kparams = + (const struct htp_solve_tri_kernel_params *) octx->kernel_params; + + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t col_block = kparams->col_block; + const uint32_t k_chunks = kparams->k_chunks; + const uint32_t total_batches = kparams->total_batches; + const bool batched = (kparams->batched != 0); + + FARF(HIGH, "solve-tri: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : batched %d\n", + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], batched); + + if (batched) { + uint32_t job_start = 0; + uint32_t njobs = total_batches; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? kparams->batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + job_start = range.start; + njobs = range.count; + } + + if (njobs == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + + struct htp_solve_tri_context sctx = { + .octx = octx, + .kparams = kparams, + .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .total_jobs = njobs, + .job_start = job_start, + .k_chunks = k_chunks, + .col_block = col_block, + }; + + work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads); + } else { + const uint32_t total_jobs = kparams->total_jobs; + + uint32_t job_start = 0; + uint32_t njobs = total_jobs; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_jobs, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + job_start = range.start; + njobs = range.count; + } + + if (njobs == 0) { + return HTP_STATUS_OK; + } + + if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, njobs))) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t n_threads = octx->n_threads; + + struct htp_solve_tri_context sctx = { + .octx = octx, + .kparams = kparams, + .jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div), + .total_jobs = njobs, + .job_start = job_start, + .k_chunks = k_chunks, + .col_block = col_block, + }; + + work_queue_run(octx->ctx->work_queue, solve_tri_chunk_thread_f32, &sctx, n_threads); + } + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/solve-tri-ops.h b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h new file mode 100644 index 0000000..3274158 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/solve-tri-ops.h @@ -0,0 +1,26 @@ +#ifndef HTP_SOLVE_TRI_OPS_H +#define HTP_SOLVE_TRI_OPS_H + +#include "hex-fastdiv.h" +#include + +struct htp_solve_tri_kernel_params { + uint32_t n_threads; + uint32_t total_batches; + uint32_t total_jobs; + uint32_t k_chunks; + uint32_t col_block; + uint32_t batched; + uint32_t batches_per_chunk; + + struct fastdiv_values div_ne02; + struct fastdiv_values div_k_chunks; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_solve_tri_kernel_params) <= 128, "htp_solve_tri_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_solve_tri_kernel_params) <= 128, "htp_solve_tri_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_SOLVE_TRI_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c index a28fd03..dae339e 100644 --- a/ggml/src/ggml-hexagon/htp/ssm-conv.c +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -15,326 +14,634 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" -#include "hex-dma.h" -#include "htp-ops.h" +#include "dma-queue.h" +#include "hex-profile.h" #include "htp-ops.h" +#include "htp-tensor.h" #include "hvx-utils.h" - -#define htp_ssm_conv_tensors_preamble \ - const struct htp_tensor * restrict src0 = octx->src[0]; \ - const struct htp_tensor * restrict src1 = octx->src[1]; \ - const struct htp_tensor * restrict dst = octx->dst; \ - struct htp_spad * restrict src0_spad = &octx->src0_spad; \ - struct htp_spad * restrict src1_spad = &octx->src1_spad; \ - struct htp_spad * restrict dst_spad = &octx->dst_spad; \ - \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t ne10 = src1->ne[0]; \ - const uint32_t ne11 = src1->ne[1]; \ - const uint32_t ne12 = src1->ne[2]; \ - const uint32_t ne13 = src1->ne[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t nb10 = src1->nb[0]; \ - const uint32_t nb11 = src1->nb[1]; \ - const uint32_t nb12 = src1->nb[2]; \ - const uint32_t nb13 = src1->nb[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; +#include "ssm-conv.h" struct htp_ssm_conv_context { - struct htp_ops_context * octx; - uint32_t nrows_per_thread; - uint64_t t_start; + struct htp_ops_context * octx; + const struct htp_ssm_conv_kernel_params * kparams; + uint32_t nrows_per_thread; + uint32_t d_inner_tile; + uint32_t row_start; + uint32_t nrows; }; -#define htp_ssm_conv_preamble \ - struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; \ - struct htp_ops_context * octx = scctx->octx; \ - htp_ssm_conv_tensors_preamble; \ - dma_queue * dma_queue = octx->ctx->dma[ith]; - -// Scalar FP32 SSM_CONV implementation -static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) { - htp_ssm_conv_preamble; - - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const uint32_t d_conv = src1->ne[0]; - const uint32_t d_inner = src0->ne[1]; - const uint32_t n_t = dst->ne[1]; - const uint32_t n_s = dst->ne[2]; - - const uint32_t src0_stride_inner = src0->nb[1] / sizeof(float); // stride for inner dimension - const uint32_t src0_stride_seq = src0->nb[2] / sizeof(float); // stride for sequence dimension - const uint32_t src1_stride_inner = src1->nb[1] / sizeof(float); // stride for inner dimension - const uint32_t dst_stride_token = dst->nb[1] / sizeof(float); // stride for token dimension - const uint32_t dst_stride_seq = dst->nb[2] / sizeof(float); // stride for sequence dimension - - const float * src0_data = (const float *) src0->data; - const float * src1_data = (const float *) src1->data; - float * dst_data = (float *) dst->data; - - // Calculate row range for this thread - const uint32_t d_inner_per_thread = scctx->nrows_per_thread; - const uint32_t d_inner_start = d_inner_per_thread * ith; - const uint32_t d_inner_end = MIN(d_inner_start + d_inner_per_thread, d_inner); - - // No work for this thread - if (d_inner_start >= d_inner_end) { - return; +// In-register 32x32 fp32 transpose using std 5-stage HVX vshuff butterfly. +static inline void hvx_transpose_32x32_f32(HVX_Vector m[32]) { + HVX_Vector tmp[32]; + + // Stage 0 (R = -4): pair (2i, 2i+1) for i = 0..15. m -> tmp. + for (int i = 0; i < 16; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[2*i + 1], m[2*i], -4); + tmp[2*i + 0] = Q6_V_lo_W(p); + tmp[2*i + 1] = Q6_V_hi_W(p); } - for (uint32_t i3 = 0; i3 < n_s; ++i3) { - for (uint32_t i2 = 0; i2 < n_t; ++i2) { - for (uint32_t i1 = d_inner_start; i1 < d_inner_end; ++i1) { - float sumf = 0.0f; + // Stage 1 (R = -8): per block of 4, pair (b+0, b+2) and (b+1, b+3). tmp -> m. + for (int b = 0; b < 32; b += 4) { + HVX_VectorPair p0 = Q6_W_vshuff_VVR(tmp[b + 2], tmp[b + 0], -8); + HVX_VectorPair p1 = Q6_W_vshuff_VVR(tmp[b + 3], tmp[b + 1], -8); + m[b + 0] = Q6_V_lo_W(p0); m[b + 1] = Q6_V_hi_W(p0); + m[b + 2] = Q6_V_lo_W(p1); m[b + 3] = Q6_V_hi_W(p1); + } + + // Stage 2 (R = -16): per block of 8, pair (b+i, b+i+4) for i = 0..3. m -> tmp. + for (int b = 0; b < 32; b += 8) { + for (int i = 0; i < 4; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[b + i + 4], m[b + i], -16); + tmp[b + 2*i + 0] = Q6_V_lo_W(p); + tmp[b + 2*i + 1] = Q6_V_hi_W(p); + } + } - for (uint32_t i0 = 0; i0 < d_conv; ++i0) { - const uint32_t src0_idx = (i2 + i0) + i1 * src0_stride_inner + i3 * src0_stride_seq; - const uint32_t src1_idx = i0 + i1 * src1_stride_inner; + // Stage 3 (R = -32): per block of 16, pair (b+i, b+i+8) for i = 0..7. tmp -> m. + for (int b = 0; b < 32; b += 16) { + for (int i = 0; i < 8; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(tmp[b + i + 8], tmp[b + i], -32); + m[b + 2*i + 0] = Q6_V_lo_W(p); + m[b + 2*i + 1] = Q6_V_hi_W(p); + } + } - sumf += src0_data[src0_idx] * src1_data[src1_idx]; - } + // Stage 4 (R = -64): pair (i, i+16) for i = 0..15. m -> tmp -> m. + for (int i = 0; i < 16; ++i) { + HVX_VectorPair p = Q6_W_vshuff_VVR(m[i + 16], m[i], -64); + tmp[2 * i + 0] = Q6_V_lo_W(p); + tmp[2 * i + 1] = Q6_V_hi_W(p); + } - const uint32_t dst_idx = i1 + i2 * dst_stride_token + i3 * dst_stride_seq; - dst_data[dst_idx] = sumf; - } + for (int i = 0; i < 32; ++i) { + m[i] = tmp[i]; + } +} + +// HVX deinterleave for d_conv == 4: channel-major raw VTCM -> tap-major T VTCM +static inline void hvx_ssm_conv_unpack_to_T_4(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride) { + for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { + HVX_Vector v0 = *(const HVX_Vector *)(raw + (cb + 0) * 4); + HVX_Vector v1 = *(const HVX_Vector *)(raw + (cb + 8) * 4); + HVX_Vector v2 = *(const HVX_Vector *)(raw + (cb + 16) * 4); + HVX_Vector v3 = *(const HVX_Vector *)(raw + (cb + 24) * 4); + + HVX_VectorPair p01 = Q6_W_vdeal_VVR(v1, v0, -4); + HVX_VectorPair p23 = Q6_W_vdeal_VVR(v3, v2, -4); + + HVX_VectorPair p_w02 = Q6_W_vdeal_VVR(Q6_V_lo_W(p23), Q6_V_lo_W(p01), -4); + HVX_VectorPair p_w13 = Q6_W_vdeal_VVR(Q6_V_hi_W(p23), Q6_V_hi_W(p01), -4); + + *(HVX_Vector *)(T + 0 * d_inner_stride + cb) = Q6_V_lo_W(p_w02); + *(HVX_Vector *)(T + 1 * d_inner_stride + cb) = Q6_V_lo_W(p_w13); + *(HVX_Vector *)(T + 2 * d_inner_stride + cb) = Q6_V_hi_W(p_w02); + *(HVX_Vector *)(T + 3 * d_inner_stride + cb) = Q6_V_hi_W(p_w13); + } +} + +// HVX transpose for general d_conv <= 32: channel-major raw VTCM -> tap-major T VTCM +static inline void hvx_ssm_conv_unpack_to_T_gen(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { + uint32_t __attribute__((aligned(VLEN))) mask_buf[VLEN_FP32] = { 0 }; + for (uint32_t j = 0; j < d_conv; ++j) { + mask_buf[j] = 0xFFFFFFFF; + } + const HVX_Vector mask = *(const HVX_Vector *) mask_buf; + + for (uint32_t cb = 0; cb < d_inner_per_thread; cb += VLEN_FP32) { + const uint32_t cb_n = MIN(VLEN_FP32, d_inner_per_thread - cb); + HVX_Vector sub[32]; + for (uint32_t r = 0; r < cb_n; ++r) { + const float * ch_ptr = raw + (cb + r) * d_conv; + sub[r] = Q6_V_vand_VV(*(const HVX_UVector *) ch_ptr, mask); + } + for (uint32_t r = cb_n; r < 32; ++r) { + sub[r] = hvx_vec_splat_f32(0.0f); + } + + hvx_transpose_32x32_f32(sub); + + for (uint32_t j = 0; j < d_conv; ++j) { + *(HVX_Vector *)(T + j * d_inner_stride + cb) = sub[j]; } } +} - t2 = HAP_perf_get_qtimer_count(); +static inline void hvx_ssm_conv_unpack_to_T(const float * raw, float * T, uint32_t d_inner_per_thread, uint32_t d_inner_stride, uint32_t d_conv) { + if (d_conv == 4 && (d_inner_per_thread % VLEN_FP32 == 0)) { + hvx_ssm_conv_unpack_to_T_4(raw, T, d_inner_per_thread, d_inner_stride); + } else { + hvx_ssm_conv_unpack_to_T_gen(raw, T, d_inner_per_thread, d_inner_stride, d_conv); + } +} - FARF(HIGH, "ssm-conv-f32 %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", - ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], d_inner_start, d_inner_end, - src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], - dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); +static __attribute__((noinline)) void hvx_ssm_conv_silu_vtcm(float * restrict buf, uint32_t n_vecs) { + HVX_Vector * restrict vbuf = (HVX_Vector *) buf; + for (uint32_t i = 0; i < n_vecs; ++i) { + vbuf[i] = hvx_vec_silu_f32(vbuf[i]); + } } -// HVX FP32 SSM_CONV implementation - vectorizes across d_inner dimension -static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) { - htp_ssm_conv_preamble; +// Decode dot product specialization for d_conv == 4: multiply in the raw channel-major layout, +// then deinterleave the products so each vector holds one tap of 32 channels, and sum. +// Keeps both operands in DMA layout - no transpose, no scratch. +static inline void hvx_ssm_conv_decode_4(const float * x, const float * w, float * out, uint32_t n_ch) { + for (uint32_t cb = 0; cb < n_ch; cb += VLEN_FP32) { + const float * xp = x + cb * 4; + const float * wp = w + cb * 4; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + HVX_Vector p0 = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *)(xp + 0), *(const HVX_Vector *)(wp + 0)); + HVX_Vector p1 = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *)(xp + 32), *(const HVX_Vector *)(wp + 32)); + HVX_Vector p2 = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *)(xp + 64), *(const HVX_Vector *)(wp + 64)); + HVX_Vector p3 = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *)(xp + 96), *(const HVX_Vector *)(wp + 96)); - const int nc = src1->ne[0]; // d_conv - const int ncs = src0->ne[0]; // d_conv - 1 + n_t + HVX_VectorPair p01 = Q6_W_vdeal_VVR(p1, p0, -4); + HVX_VectorPair p23 = Q6_W_vdeal_VVR(p3, p2, -4); - const uint32_t d_conv = src1->ne[0]; - const uint32_t d_inner = src0->ne[1]; - const uint32_t n_t = dst->ne[1]; - const uint32_t n_s = dst->ne[2]; + HVX_VectorPair q02 = Q6_W_vdeal_VVR(Q6_V_lo_W(p23), Q6_V_lo_W(p01), -4); + HVX_VectorPair q13 = Q6_W_vdeal_VVR(Q6_V_hi_W(p23), Q6_V_hi_W(p01), -4); - const float * src0_data = (const float *) src0->data; - const float * src1_data = (const float *) src1->data; - float * dst_data = (float *) dst->data; + HVX_Vector a = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(q02), Q6_V_lo_W(q13)); + HVX_Vector b = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_hi_W(q02), Q6_V_hi_W(q13)); - // Calculate row range for this thread - const int dr = scctx->nrows_per_thread; - const uint32_t ir0 = dr * ith; - const uint32_t ir1 = MIN(ir0 + dr, d_inner); - const uint32_t ir = ir1 - ir0; + *(HVX_Vector *)(out + cb) = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(a, b)); + } +} - if (ir0 >= ir1) { - return; // No work for this thread +// Transpose src0 for prefill: one 32-channel block {32, ncs} (VTCM) -> T {ncs, 32} (VTCM). +// One VTCM gather per output row: lane r picks channel cb+r, the region bound drops +// the lanes past the channel tail. +static inline void hvx_ssm_conv_transpose_block(const float * raw_block, + float * T, + uint32_t ncs, + uint32_t cb_n, + HVX_Vector vv) { + const size_t base = (size_t) raw_block; + const uint32_t mu = cb_n * ncs * sizeof(float) - 1; + + for (uint32_t t = 0; t < ncs; ++t) { + Q6_vgather_ARMVw((HVX_Vector *) (T + (size_t) t * VLEN_FP32), base + t * sizeof(float), mu, vv); } +} + +// Single-token decode worker (n_t == 1) +static void ssm_conv_thread_f32_decode(unsigned int nth, unsigned int ith, void * data) { + struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; + struct htp_ops_context * octx = scctx->octx; + const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; + + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict src1 = octx->src[1]; + const struct htp_tensor * restrict dst = octx->dst; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t d_conv = kparams->d_conv; + const uint32_t d_inner = kparams->d_inner; + const uint32_t n_s = kparams->n_s; + const bool apply_silu = kparams->silu != 0; - // src0 and src1 gather offsets - uint32_t __attribute__((aligned(VLEN))) src0_offsets[VLEN_FP32] = { 0 }; - uint32_t __attribute__((aligned(VLEN))) src1_offsets[VLEN_FP32] = { 0 }; + const uint32_t dr = scctx->nrows_per_thread; + const uint32_t ir0 = scctx->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); - for (uint32_t i = 0; i < VLEN_FP32; ++i) { - src0_offsets[i] = i * (ncs) * sizeof(float); - src1_offsets[i] = i * (d_conv) * sizeof(float); + if (ir0 >= ir1) { + return; } - const uint32_t src0_gather_len = VLEN * ncs; - const uint32_t src1_gather_len = VLEN * d_conv; + const uint32_t d_inner_per_thread = ir1 - ir0; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); + const uint32_t d_inner_tile = scctx->d_inner_tile; - // gather scratchpads - HVX_Vector * src0_vec = (HVX_Vector *) (octx->ctx->vtcm_base + ith * VLEN*2 + 0); - HVX_Vector * src1_vec = (HVX_Vector *) (octx->ctx->vtcm_base + ith * VLEN*2 + VLEN); + const size_t src0_stride_inner_bytes = src0->nb[1]; + const size_t src0_stride_seq_bytes = src0->nb[2]; + const size_t dst_stride_seq_bytes = dst->nb[2]; - float * data_src0 = (float *) ((char *) src0->data + ir0 * src0->nb[1]); - float * data_src1 = (float *) ((char *) src1->data + ir0 * src1->nb[1]); + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; - uint8_t * spad_src0 = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; - uint8_t * spad_src1 = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; + const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_raw_size = hex_round_up(weight_bytes, 128); - // copy src1 workload to VTCM - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src1, data_src1), nb11, nb11, ir); + float * src1_raw = (float *) src1_spad_base; + float * src1_T = (float *) (src1_spad_base + weight_raw_size); - // FARF(HIGH, "ssm-conv-src1-fetch %d: ir0 %u size %u\n", ith, ir0, nb11 * ir); + const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * d_conv * sizeof(float), 128); + const size_t dst_tile_bytes = hex_round_up(d_inner_tile * sizeof(float), 128); - for (uint32_t i3 = 0; i3 < n_s; ++i3) { - float * src0_data_ptr = (float *) ((char *) data_src0 + i3 * (src0->nb[2])); + float * src0_tile_raw[2] = { (float *) src0_spad_base, (float *) (src0_spad_base + src0_tile_raw_bytes) }; + float * src0_T = (float *) (src0_spad_base + 2 * src0_tile_raw_bytes); - // copy src0 workload to VTCM - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(spad_src0, src0_data_ptr), nb01, nb01, ir); + float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) }; - // FARF(HIGH, "ssm-conv-src0-fetch %d: ir0 %u i3 %u size %u\n", ith, ir0, i3, nb01 * ir); + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; - dma_queue_flush(dma_queue); + // raw_dot keeps both operands in the DMA layout, so src1 needs no prep pass + const bool raw_dot = (d_conv == 4) && (d_inner_per_thread % VLEN_FP32 == 0); - for (uint32_t i2 = 0; i2 < n_t; ++i2) { - float * dst_ptr = (float *) ((char *) dst->data + ir0 * (dst->nb[0]) + i2 * (dst->nb[1]) + i3 * (dst->nb[2])); + const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile); + const uint32_t n_chunks = n_s * n_tiles; + const size_t row_bytes = d_conv * sizeof(float); - const uint32_t nvec = ir / VLEN_FP32; - const uint32_t nloe = ir % VLEN_FP32; - uint32_t i1 = 0; + uint32_t s_fetch = 0; + uint32_t tile_off_fetch = 0; - for (uint32_t vi1 = 0; vi1 < nvec; vi1++) { - HVX_Vector acc_vec = Q6_V_vsplat_R(0); + #define SSM_CONV_DECODE_PUSH_FETCH(c) \ + do { \ + const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off_fetch); \ + const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \ + (ir0 + tile_off_fetch) * src0_stride_inner_bytes; \ + dma_queue_push(dma_q, \ + dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \ + row_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \ + tile_off_fetch += d_inner_tile; \ + if (tile_off_fetch >= d_inner_per_thread) { \ + tile_off_fetch = 0; \ + s_fetch++; \ + } \ + } while (0) - for (uint32_t i0 = 0; i0 < d_conv; ++i0) { - uint32_t src0_base = (uint32_t) spad_src0 + (i0 + i1 * ncs) * sizeof(float) + i2 * (src0->nb[0]); - uint32_t src1_base = (uint32_t) spad_src1 + (i0 + i1 * nc) * sizeof(float); - Q6_vgather_ARMVw(src0_vec, src0_base, src0_gather_len, (*(const HVX_Vector *) src0_offsets)); - Q6_vgather_ARMVw(src1_vec, src1_base, src1_gather_len, (*(const HVX_Vector *) src1_offsets)); + // Queue weights and initial input tiles together so DDR reads overlap + const dma_addr_t src1_ddr = src1->data + ir0 * d_conv * sizeof(float); + dma_queue_push(dma_q, dma_make_data((uint8_t *) src1_raw, src1_ddr), weight_bytes, weight_bytes, weight_bytes, 1); - HVX_Vector prod = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *) src0_vec, *(const HVX_Vector *) src1_vec); - acc_vec = Q6_Vqf32_vadd_Vqf32Vqf32(acc_vec, prod); - } + SSM_CONV_DECODE_PUSH_FETCH(0); + if (n_chunks > 1) { + SSM_CONV_DECODE_PUSH_FETCH(1); + } - *(HVX_UVector *) (dst_ptr + i1) = Q6_Vsf_equals_Vqf32(acc_vec); - i1 += VLEN_FP32; - } + dma_queue_pop(dma_q); // weights - if (nloe) { - HVX_Vector acc_vec = Q6_V_vsplat_R(0); + if (!raw_dot) { + // Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); + } - for (uint32_t i0 = 0; i0 < d_conv; ++i0) { - uint32_t src0_base = (uint32_t) spad_src0 + (i0 + i1 * ncs) * sizeof(float) + i2 * (src0->nb[0]); - uint32_t src1_base = (uint32_t) spad_src1 + (i0 + i1 * nc) * sizeof(float); - Q6_vgather_ARMVw(src0_vec, src0_base, src0_gather_len, (*(const HVX_Vector *) src0_offsets)); - Q6_vgather_ARMVw(src1_vec, src1_base, src1_gather_len, (*(const HVX_Vector *) src1_offsets)); + uint32_t i3 = 0; + uint32_t tile_off = 0; - HVX_Vector prod = Q6_Vqf32_vmpy_VsfVsf(*(const HVX_Vector *) src0_vec, *(const HVX_Vector *) src1_vec); - acc_vec = Q6_Vqf32_vadd_Vqf32Vqf32(acc_vec, prod); - } + for (uint32_t c = 0; c < n_chunks; ++c) { + const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off); - hvx_vec_store_u(dst_ptr + i1, (ir - i1) * 4, Q6_Vsf_equals_Vqf32(acc_vec)); + if (c >= 2) { + dma_queue_pop(dma_q); // writeback of chunk c-2, frees dst_tile[c & 1] + } + dma_queue_pop(dma_q); // fetch chunk c + + float * restrict out = dst_tile[c & 1]; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i3); + if (raw_dot) { + const float * xp = src0_tile_raw[c & 1]; + const float * wp = src1_raw + tile_off * 4; + hvx_ssm_conv_decode_4(xp, wp, out, tile_n); + } else { + const uint32_t tile_stride = hex_round_up(tile_n, VLEN_FP32); + hvx_ssm_conv_unpack_to_T(src0_tile_raw[c & 1], src0_T, tile_n, tile_stride, d_conv); + + for (uint32_t cb = 0; cb < tile_n; cb += VLEN_FP32) { + HVX_Vector acc = hvx_vec_splat_f32(0.0f); + for (uint32_t j = 0; j < d_conv; ++j) { + HVX_Vector x = *(const HVX_Vector *)(src0_T + j * tile_stride + cb); + HVX_Vector w = *(const HVX_Vector *)(src1_T + j * d_inner_stride + tile_off + cb); + acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w)); + } + *(HVX_Vector *)(out + cb) = Q6_Vsf_equals_Vqf32(acc); } } + if (apply_silu) { + hvx_ssm_conv_silu_vtcm(out, hex_round_up(tile_n, VLEN_FP32) / VLEN_FP32); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i3); + + const dma_addr_t dst_ddr = dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float); + const size_t tile_out_bytes = (size_t) tile_n * sizeof(float); + dma_queue_push(dma_q, dma_make_data(dst_ddr, (uint8_t *) out), + tile_out_bytes, tile_out_bytes, tile_out_bytes, 1); + + if (c + 2 < n_chunks) { + SSM_CONV_DECODE_PUSH_FETCH(c + 2); + } + + tile_off += d_inner_tile; + if (tile_off >= d_inner_per_thread) { + tile_off = 0; + i3++; + } + } + + for (uint32_t k = MIN(n_chunks, 2); k > 0; --k) { + dma_queue_pop(dma_q); // drain the last writebacks } - t2 = HAP_perf_get_qtimer_count(); + #undef SSM_CONV_DECODE_PUSH_FETCH - FARF(HIGH, "ssm-conv-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", + FARF(HIGH, "ssm-conv-f32-decode %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], - dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + dst->ne[2], dst->ne[3]); } -int op_ssm_conv_f32(struct htp_ops_context * octx) { - htp_ssm_conv_tensors_preamble; +// Multi-token prefill worker (n_t > 1) +static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void * data) { + struct htp_ssm_conv_context * scctx = (struct htp_ssm_conv_context *) data; + struct htp_ops_context * octx = scctx->octx; + const struct htp_ssm_conv_kernel_params * kparams = scctx->kparams; + + const struct htp_tensor * restrict src0 = octx->src[0]; + const struct htp_tensor * restrict src1 = octx->src[1]; + const struct htp_tensor * restrict dst = octx->dst; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t d_conv = kparams->d_conv; + const uint32_t d_inner = kparams->d_inner; + const uint32_t n_t = kparams->n_t; + const uint32_t n_s = kparams->n_s; + const uint32_t ncs = src0->ne[0]; + const bool apply_silu = kparams->silu != 0; + + const uint32_t dr = scctx->nrows_per_thread; + const uint32_t ir0 = scctx->row_start + dr * ith; + const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows); + + if (ir0 >= ir1) { + return; + } - if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) { - FARF(ERROR, "ssm_conv: only (F32 x F32 -> F32) OPs supported"); - return HTP_STATUS_NO_SUPPORT; + const uint32_t d_inner_per_thread = ir1 - ir0; + const uint32_t d_inner_stride = hex_round_up(d_inner_per_thread, VLEN_FP32); + const uint32_t d_inner_tile = scctx->d_inner_tile; + + const size_t src0_stride_inner_bytes = src0->nb[1]; + const size_t src0_stride_seq_bytes = src0->nb[2]; + const size_t dst_stride_token_bytes = dst->nb[1]; + const size_t dst_stride_seq_bytes = dst->nb[2]; + + uint8_t * src1_spad_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread; + uint8_t * src0_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; + uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; + + const size_t weight_bytes = (size_t) d_inner_per_thread * d_conv * sizeof(float); + const size_t weight_raw_size = hex_round_up(weight_bytes, 128); + + float * src1_raw = (float *) src1_spad_base; + float * src1_T = (float *) (src1_spad_base + weight_raw_size); + + // src0 spad holds two raw tiles (fetch of tile n+1 overlaps compute of tile n) plus + // the transposed block. dst spad holds two tiles so a writeback can stay in flight. + const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * ncs * sizeof(float), 128); + const size_t dst_tile_bytes = hex_round_up(d_inner_tile * n_t * sizeof(float), 128); + + float * src0_tile_raw[2] = { (float *) src0_spad_base, (float *) (src0_spad_base + src0_tile_raw_bytes) }; + float * src0_T = (float *) (src0_spad_base + 2 * src0_tile_raw_bytes); + + float * dst_tile[2] = { (float *) dst_spad_base, (float *) (dst_spad_base + dst_tile_bytes) }; + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const uint32_t n_tiles = fastdiv(d_inner_per_thread + d_inner_tile - 1, &kparams->div_d_inner_tile); + const uint32_t n_chunks = n_s * n_tiles; + const size_t row_bytes = ncs * sizeof(float); + + uint32_t s_fetch = 0; + uint32_t tile_off_fetch = 0; + + // Chunk c fetches into src0_tile_raw[c & 1] and writes back from dst_tile[c & 1]. + // Two fetches run ahead, so the queue order is F0 F1 W0 F2 W1 ... and pops follow it. + #define SSM_CONV_PUSH_FETCH(c) \ + do { \ + const uint32_t cur_tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off_fetch); \ + const dma_addr_t fetch_ddr = src0->data + s_fetch * src0_stride_seq_bytes + \ + (ir0 + tile_off_fetch) * src0_stride_inner_bytes; \ + dma_queue_push(dma_q, \ + dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \ + row_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \ + tile_off_fetch += d_inner_tile; \ + if (tile_off_fetch >= d_inner_per_thread) { \ + tile_off_fetch = 0; \ + s_fetch++; \ + } \ + } while (0) + + // Queue weights and initial input tiles together so DDR reads overlap + const dma_addr_t src1_ddr = src1->data + ir0 * d_conv * sizeof(float); + dma_queue_push(dma_q, dma_make_data((uint8_t *) src1_raw, src1_ddr), weight_bytes, weight_bytes, weight_bytes, 1); + + SSM_CONV_PUSH_FETCH(0); + if (n_chunks > 1) { + SSM_CONV_PUSH_FETCH(1); } - struct htp_ssm_conv_context scctx = { 0 }; - scctx.octx = octx; + dma_queue_pop(dma_q); // weights - const uint32_t d_conv = src1->ne[0]; - const uint32_t d_inner = src0->ne[1]; - const uint32_t n_t = dst->ne[1]; // tokens per sequence - const uint32_t n_s = dst->ne[2]; // number of sequences in the batch + // Unpack/transpose src1_raw into src1_T {d_conv, d_inner_stride} + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); + hvx_ssm_conv_unpack_to_T(src1_raw, src1_T, d_inner_per_thread, d_inner_stride, d_conv); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) ir0); - const uint32_t n_threads = MIN(octx->n_threads, d_inner); + const uint32_t C_TILE = VLEN_FP32; - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - uint32_t use_hvx = 0; - if (d_inner >= VLEN_FP32 && d_inner % VLEN_FP32 == 0) { - int is_aligned = hex_is_aligned((void *) src0->data, VLEN) && - hex_is_aligned((void *) src1->data, VLEN) && - hex_is_aligned((void *) dst->data, VLEN); + // gather offsets: lane r reads channel r of the raw tile block + uint32_t __attribute__((aligned(VLEN))) gather_off[VLEN_FP32]; + for (uint32_t r = 0; r < VLEN_FP32; ++r) { + gather_off[r] = r * ncs * sizeof(float); + } + const HVX_Vector vv = *(const HVX_Vector *) gather_off; - if (is_aligned) { - use_hvx = 1; - } + uint32_t i3 = 0; + uint32_t tile_off = 0; + + for (uint32_t c = 0; c < n_chunks; ++c) { + const uint32_t tile_n = MIN(d_inner_tile, d_inner_per_thread - tile_off); + + const float * restrict raw = src0_tile_raw[c & 1]; + float * restrict out = dst_tile[c & 1]; + + if (c >= 2) { + dma_queue_pop(dma_q); // writeback of chunk c-2, frees dst_tile[c & 1] } + dma_queue_pop(dma_q); // fetch chunk c + + // Channel block outer, token inner: the taps and the sliding window stay in + // registers, so a new output row costs one src0_T load. + const uint32_t dst_tile_stride = hex_round_up(tile_n, C_TILE); + + for (uint32_t cb = 0; cb < tile_n; cb += C_TILE) { + const uint32_t cb_n = MIN(C_TILE, tile_n - cb); - if (use_hvx) { - scctx.nrows_per_thread = (d_inner + n_threads - 1) / n_threads; // d_inner chunks per thread - scctx.nrows_per_thread += (scctx.nrows_per_thread & 1); // round up to even + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) tile_off); + hvx_ssm_conv_transpose_block(raw + (size_t) cb * ncs, src0_T, ncs, cb_n, vv); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) tile_off); - octx->src0_spad.size_per_thread = hex_round_up(scctx.nrows_per_thread * nb01, 256); - octx->src1_spad.size_per_thread = hex_round_up(scctx.nrows_per_thread * nb11, 256); - octx->dst_spad.size_per_thread = hex_round_up(scctx.nrows_per_thread * sizeof(float), 256); + const float * restrict wp = src1_T + tile_off + cb; + float * restrict op = out + cb; - octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads; - octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads; - octx->dst_spad.size = octx->dst_spad.size_per_thread * n_threads; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); + if (d_conv == 4) { + const HVX_Vector w0 = *(const HVX_Vector *) (wp); + const HVX_Vector w1 = *(const HVX_Vector *) (wp + d_inner_stride); + const HVX_Vector w2 = *(const HVX_Vector *) (wp + 2 * d_inner_stride); + const HVX_Vector w3 = *(const HVX_Vector *) (wp + 3 * d_inner_stride); - // Compute gather scratchpad size for src0 and src1 - const size_t gather_spad_size = n_threads * VLEN * 2; + HVX_Vector x0 = *(const HVX_Vector *) (src0_T); + HVX_Vector x1 = *(const HVX_Vector *) (src0_T + C_TILE); + HVX_Vector x2 = *(const HVX_Vector *) (src0_T + 2 * C_TILE); - octx->src0_spad.data = octx->ctx->vtcm_base + gather_spad_size; octx->src0_spad.src = NULL; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL; - octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->dst_spad.src = NULL; + for (uint32_t t = 0; t < n_t; ++t) { + const HVX_Vector x3 = *(const HVX_Vector *) (src0_T + (t + 3) * C_TILE); - FARF(HIGH, "ssm_conv-f32: gather-spad:%zu spad-per-thread:(%u:%u:%u) spad-sizes:(%u:%u:%u) spad-data:(%p:%p:%p)\n", - gather_spad_size, octx->src0_spad.size_per_thread, octx->src1_spad.size_per_thread, - octx->dst_spad.size_per_thread, octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size, - octx->src0_spad.data, octx->src1_spad.data, octx->dst_spad.data); + HVX_Vector a = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(x0, w0), + Q6_Vqf32_vmpy_VsfVsf(x1, w1)); + HVX_Vector b = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(x2, w2), + Q6_Vqf32_vmpy_VsfVsf(x3, w3)); - const size_t total_spad_size = - gather_spad_size + octx->src0_spad.size + octx->src1_spad.size + octx->dst_spad.size; + *(HVX_Vector *) (op + t * dst_tile_stride) = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(a, b)); - if (total_spad_size > octx->ctx->vtcm_size) { - FARF(HIGH, "ssm_conv-f32: HVX scratchpad size %zu exceeds VTCM size %zu", total_spad_size, - octx->ctx->vtcm_size); - use_hvx = 0; + x0 = x1; + x1 = x2; + x2 = x3; + } + } else { + for (uint32_t t = 0; t < n_t; ++t) { + HVX_Vector acc = hvx_vec_splat_f32(0.0f); + for (uint32_t j = 0; j < d_conv; ++j) { + HVX_Vector x = *(const HVX_Vector *) (src0_T + (t + j) * C_TILE); + HVX_Vector w = *(const HVX_Vector *) (wp + j * d_inner_stride); + acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w)); + } + *(HVX_Vector *) (op + t * dst_tile_stride) = Q6_Vsf_equals_Vqf32(acc); + } } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); + } + + if (apply_silu) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); + hvx_ssm_conv_silu_vtcm(out, n_t * (dst_tile_stride / C_TILE)); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) tile_off); } - FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_hvx %d\n", src0->ne[0], - src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], - dst->ne[1], dst->ne[2], dst->ne[3], use_hvx); + // Writeback dst_tile VTCM -> DDR via 2D DMA + dma_queue_push(dma_q, dma_make_data(dst->data + i3 * dst_stride_seq_bytes + (ir0 + tile_off) * sizeof(float), (uint8_t *) out), + dst_stride_token_bytes, dst_tile_stride * sizeof(float), tile_n * sizeof(float), n_t); - if (use_hvx) { - worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32_hvx, &scctx, n_threads); - } else { - worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32, &scctx, n_threads); + if (c + 2 < n_chunks) { + SSM_CONV_PUSH_FETCH(c + 2); + } + + tile_off += d_inner_tile; + if (tile_off >= d_inner_per_thread) { + tile_off = 0; + i3++; } } + for (uint32_t k = MIN(n_chunks, 2); k > 0; --k) { + dma_queue_pop(dma_q); // drain the last writebacks + } + + #undef SSM_CONV_PUSH_FETCH + + FARF(HIGH, "ssm-conv-f32-prefill %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n", + ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, + src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1], + dst->ne[2], dst->ne[3]); +} + +int op_ssm_conv_f32(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + const struct htp_ssm_conv_kernel_params * kparams = (const struct htp_ssm_conv_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + uint32_t row_start = 0; + uint32_t nrows = kparams->d_inner; + + if (octx->ctx->mdev.count > 1) { + const uint32_t elems_per_chunk = VLEN_FP32; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + kparams->d_inner, + htp_tensor_mdev_data_aligned(dst) ? elems_per_chunk : 0, + octx->ctx->mdev.idx, + octx->ctx->mdev.count, + &octx->ctx->mdev.count_div + ); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + const uint32_t n_threads = octx->n_threads; + const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + const uint32_t d_inner_per_thread = hex_round_up(raw_rpt, VLEN_FP32); + + struct htp_ssm_conv_vtcm_layout L; + htp_ssm_conv_vtcm_layout_build(&L, d_inner_per_thread, kparams->d_conv, kparams->n_t, kparams->d_inner_tile, n_threads); + + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + + octx->src0_spad.size_per_thread = L.src0_bytes_per_thread; + octx->src1_spad.size_per_thread = L.src1_bytes_per_thread; + octx->dst_spad.size_per_thread = L.dst_bytes_per_thread; + + octx->src0_spad.size = L.src0_bytes; + octx->src1_spad.size = L.src1_bytes; + octx->dst_spad.size = L.dst_bytes; + + octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + octx->src1_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + octx->src0_spad.src = NULL; + octx->src1_spad.src = NULL; + octx->dst_spad.src = NULL; + + struct htp_ssm_conv_context scctx = { + .octx = octx, + .kparams = kparams, + .nrows_per_thread = d_inner_per_thread, + .d_inner_tile = kparams->d_inner_tile, + .row_start = row_start, + .nrows = nrows, + }; + + FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : mode %s\n", + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], + src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + kparams->n_t == 1 ? "decode" : "prefill"); + + if (kparams->n_t == 1) { + work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_decode, &scctx, n_threads); + } else { + work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_prefill, &scctx, n_threads); + } + return HTP_STATUS_OK; } int op_ssm_conv(struct htp_ops_context * octx) { const struct htp_tensor * dst = octx->dst; - int err = HTP_STATUS_OK; - switch (dst->type) { case HTP_TYPE_F32: - err = op_ssm_conv_f32(octx); - break; + return op_ssm_conv_f32(octx); default: - err = HTP_STATUS_NO_SUPPORT; - break; + return HTP_STATUS_NO_SUPPORT; } - - return err; } diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.h b/ggml/src/ggml-hexagon/htp/ssm-conv.h new file mode 100644 index 0000000..43b2575 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/ssm-conv.h @@ -0,0 +1,81 @@ +#ifndef HTP_SSM_CONV_H +#define HTP_SSM_CONV_H + +#include + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-ops.h" +#include "htp-vtcm.h" + +struct htp_ssm_conv_kernel_params { + uint32_t n_threads; + uint32_t d_conv; + uint32_t d_inner; + uint32_t n_t; + uint32_t n_s; + uint32_t d_inner_tile; + + uint32_t vtcm_size; + uint32_t silu; + + struct fastdiv_values div_d_inner_tile; +}; + +struct htp_ssm_conv_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_src1; + size_t off_dst; + size_t src0_bytes_per_thread; + size_t src1_bytes_per_thread; + size_t dst_bytes_per_thread; + size_t src0_bytes; + size_t src1_bytes; + size_t dst_bytes; +}; + +static inline void htp_ssm_conv_vtcm_layout_build( + struct htp_ssm_conv_vtcm_layout * L, + uint32_t d_inner_per_thread, + uint32_t d_conv, + uint32_t n_t, + uint32_t d_inner_tile, + uint32_t n_threads +) { + const size_t src1_raw_bytes = hex_round_up((size_t) d_inner_per_thread * d_conv * sizeof(float), 128) + 128; + const size_t src1_T_bytes = hex_round_up((size_t) d_conv * d_inner_per_thread * sizeof(float), 128); + + L->src1_bytes_per_thread = src1_raw_bytes + src1_T_bytes; + + if (n_t == 1) { + const size_t src0_tile_raw_bytes = hex_round_up((size_t) d_inner_tile * d_conv * sizeof(float), 128); + const size_t src0_T_bytes = hex_round_up((size_t) d_conv * d_inner_tile * sizeof(float), 128); + L->src0_bytes_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes; + L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128); + } else { + const uint32_t ncs = d_conv - 1 + n_t; + const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128); + const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * ncs * sizeof(float), 128); + L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T; + L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128); + } + + L->src0_bytes = L->src0_bytes_per_thread * n_threads; + L->src1_bytes = L->src1_bytes_per_thread * n_threads; + L->dst_bytes = L->dst_bytes_per_thread * n_threads; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes); + VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes); + L->total_bytes = off; +} + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_ssm_conv_kernel_params) <= 128, "htp_ssm_conv_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_ssm_conv_kernel_params) <= 128, "htp_ssm_conv_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_SSM_CONV_H diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c index 874c41a..3d931ed 100644 --- a/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.c @@ -8,40 +8,44 @@ #include #include -#include "hex-dma.h" +#include "dma-queue.h" #include "hvx-utils.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" +#include "sum-rows-ops.h" #define sum_rows_preamble \ const struct htp_tensor *src0 = octx->src[0]; \ const struct htp_tensor *dst = octx->dst; \ \ - const uint32_t ne00 = src0->ne[0]; \ - const uint32_t ne01 = src0->ne[1]; \ - const uint32_t ne02 = src0->ne[2]; \ - const uint32_t ne03 = src0->ne[3]; \ - \ - const uint32_t nb00 = src0->nb[0]; \ - const uint32_t nb01 = src0->nb[1]; \ - const uint32_t nb02 = src0->nb[2]; \ - const uint32_t nb03 = src0->nb[3]; \ - \ - const uint32_t ne0 = dst->ne[0]; \ - const uint32_t ne1 = dst->ne[1]; \ - const uint32_t ne2 = dst->ne[2]; \ - const uint32_t ne3 = dst->ne[3]; \ - \ - const uint32_t nb0 = dst->nb[0]; \ - const uint32_t nb1 = dst->nb[1]; \ - const uint32_t nb2 = dst->nb[2]; \ - const uint32_t nb3 = dst->nb[3]; \ + const uint32_t ne00 = src0->ne[0]; \ + const uint32_t ne01 = src0->ne[1]; \ + const uint32_t ne02 = src0->ne[2]; \ + const uint32_t ne03 = src0->ne[3]; \ + \ + const uint32_t nb00 = src0->nb[0]; \ + const uint32_t nb01 = src0->nb[1]; \ + const uint32_t nb02 = src0->nb[2]; \ + const uint32_t nb03 = src0->nb[3]; \ + \ + const uint32_t ne0 = dst->ne[0]; \ + const uint32_t ne1 = dst->ne[1]; \ + const uint32_t ne2 = dst->ne[2]; \ + const uint32_t ne3 = dst->ne[3]; \ + \ + const uint32_t nb0 = dst->nb[0]; \ + const uint32_t nb1 = dst->nb[1]; \ + const uint32_t nb2 = dst->nb[2]; \ + const uint32_t nb3 = dst->nb[3]; \ struct sum_rows_context { + struct htp_ops_context * octx; const uint8_t * src_data; uint8_t * dst_data; uint32_t ne00; @@ -49,7 +53,6 @@ struct sum_rows_context { size_t dst_stride; uint32_t rows_per_thread; uint32_t total_rows; - bool opt_path; }; static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) { @@ -68,7 +71,6 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) const size_t src_stride = smctx->src_stride; const size_t dst_stride = smctx->dst_stride; const uint32_t ne00 = smctx->ne00; - const bool opt_path = smctx->opt_path; const float * restrict src_th = (const float *) (smctx->src_data + (start_row * src_stride)); float * restrict dst_th = (float *) (smctx->dst_data + (start_row * dst_stride)); @@ -76,19 +78,21 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data) // Calculate actual number of rows for this thread const uint32_t n_rows = end_row - start_row; + struct htp_thread_trace * tr = &smctx->octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); + for (uint32_t ir = 0; ir < n_rows; ir++) { - const float * restrict src_local = src_th + (ir * (src_stride / sizeof(float))); + const float * restrict src_local = (const float *) ((const uint8_t *) src_th + ir * src_stride); + float * restrict dst_local = (float *) ((uint8_t *) dst_th + ir * dst_stride); if (ir + 1 < n_rows) { - hex_l2fetch(src_local + (src_stride / sizeof(float)), src_stride, src_stride, 1); + hex_l2fetch((const uint8_t *) src_local + src_stride, src_stride, src_stride, 1); } - if (opt_path) { - dst_th[ir] = hvx_reduce_sum_f32_a((const uint8_t *) src_local, ne00); - } else { - dst_th[ir] = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00); - } + *dst_local = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00); } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row); } int op_sum_rows(struct htp_ops_context * octx) { @@ -98,31 +102,317 @@ int op_sum_rows(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) { - return HTP_STATUS_OK; + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_sum_rows_kernel_params * kparams = + (const struct htp_sum_rows_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; } - const uint32_t src0_nrows = ne01 * ne02 * ne03; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - const uint32_t rows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + const uint32_t src0_nrows = ne01 * ne02 * ne03; + const size_t dst_data_row_size = dst->ne[0] * sizeof(float); + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } - bool opt_path = false; - if ((0 == hex_is_aligned((void *) src0->data, VLEN)) && !(nb01 & (VLEN - 1))) { - opt_path = true; + if (nrows == 0) { + return HTP_STATUS_OK; } + const uint32_t n_threads = octx->n_threads; + const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + struct sum_rows_context smctx = { - .src_data = (const uint8_t *) src0->data, - .dst_data = (uint8_t *) dst->data, + .octx = octx, + .src_data = (const uint8_t *) src0->data + row_start * nb01, + .dst_data = (uint8_t *) dst->data + row_start * nb1, .ne00 = ne00, .src_stride = nb01, .dst_stride = nb1, .rows_per_thread = rows_per_thread, - .total_rows = src0_nrows, - .opt_path = opt_path, + .total_rows = nrows, + }; + + work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads); + + return HTP_STATUS_OK; +} + +struct sum_context { + struct htp_ops_context * octx; + const float * src_data; + float partial_sums[HTP_MAX_NTHREADS]; + uint32_t total_elems; + uint32_t elems_per_thread; +}; + +static void sum_thread_f32(unsigned int nth, unsigned int ith, void * data) { + struct sum_context * sctx = (struct sum_context *) data; + const uint32_t start = sctx->elems_per_thread * ith; + const uint32_t end = MIN(start + sctx->elems_per_thread, sctx->total_elems); + + if (start >= end) { + sctx->partial_sums[ith] = 0.0f; + return; + } + + const uint32_t n = end - start; + const float * src = sctx->src_data + start; + + struct htp_thread_trace * tr = &sctx->octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); + + hex_l2fetch_block((const void *) src, n * sizeof(float)); + + sctx->partial_sums[ith] = hvx_reduce_sum_f32((const uint8_t *) src, n); + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); +} + +int op_sum(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + if (src0->type != HTP_TYPE_F32) { + return HTP_STATUS_NO_SUPPORT; + } + + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_sum_rows_kernel_params * kparams = + (const struct htp_sum_rows_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + + const uint32_t total_elems = (uint32_t) (src0->ne[0] * src0->ne[1] * src0->ne[2] * src0->ne[3]); + if (total_elems == 0) { + ((float *) dst->data)[0] = 0.0f; + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const uint32_t elems_per_thread = kparams->elems_per_thread; + + struct sum_context sctx = { + .octx = octx, + .src_data = (const float *) src0->data, + .total_elems = total_elems, + .elems_per_thread = elems_per_thread, }; - worker_pool_run_func(octx->ctx->worker_pool, sum_rows_thread_f32, &smctx, n_threads); + work_queue_run(octx->ctx->work_queue, sum_thread_f32, &sctx, n_threads); + + float sum = 0.0f; + for (uint32_t i = 0; i < n_threads; i++) { + sum += sctx.partial_sums[i]; + } + ((float *) dst->data)[0] = sum; + + return HTP_STATUS_OK; +} + +static inline void argmax_slice_f32( + const float * restrict src, + uint32_t n, + uint32_t offset, + float * out_val, + int32_t * out_idx +) { + hvx_argmax_f32(src, n, offset, out_val, out_idx); +} + +struct argmax_context { + struct htp_ops_context * octx; + const float * src_data; + int32_t * dst_data; + uint32_t ne00; + uint32_t src_stride; + uint32_t dst_stride; + uint32_t row_start; + uint32_t nrows; + uint32_t rows_per_thread; + uint32_t elems_per_thread; + + float partial_max[HTP_MAX_NTHREADS]; + int32_t partial_idx[HTP_MAX_NTHREADS]; +}; + +static void argmax_thread_single_row(unsigned int nth, unsigned int ith, void * data) { + struct argmax_context * actx = (struct argmax_context *) data; + const uint32_t start = actx->elems_per_thread * ith; + const uint32_t end = MIN(start + actx->elems_per_thread, actx->ne00); + + if (start >= end) { + actx->partial_max[ith] = -INFINITY; + actx->partial_idx[ith] = 0; + return; + } + + const uint32_t n = end - start; + const float * src = actx->src_data + start; + + struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); + + hex_l2fetch_block((const void *) src, n * sizeof(float)); + + float max_val; + int32_t max_idx; + argmax_slice_f32(src, n, start, &max_val, &max_idx); + + actx->partial_max[ith] = max_val; + actx->partial_idx[ith] = max_idx; + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start); +} + +static void argmax_thread_multi_row(unsigned int nth, unsigned int ith, void * data) { + struct argmax_context * actx = (struct argmax_context *) data; + const uint32_t r0 = actx->row_start + actx->rows_per_thread * ith; + const uint32_t r1 = MIN(r0 + actx->rows_per_thread, actx->row_start + actx->nrows); + + if (r0 >= r1) { + return; + } + + struct htp_thread_trace * tr = &actx->octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r0); + + for (uint32_t r = r0; r < r1; r++) { + const float * src_row = (const float *) ((const uint8_t *) actx->src_data + r * actx->src_stride); + int32_t * dst_val = (int32_t *) ((uint8_t *) actx->dst_data + r * actx->dst_stride); + + hex_l2fetch_block((const void *) src_row, actx->ne00 * sizeof(float)); + + float max_val; + int32_t max_idx; + argmax_slice_f32(src_row, actx->ne00, 0, &max_val, &max_idx); + + *dst_val = max_idx; + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r0); +} + +int op_argmax(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + if (src0->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_I32) { + return HTP_STATUS_NO_SUPPORT; + } + + if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) { + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_sum_rows_kernel_params * kparams = + (const struct htp_sum_rows_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const uint32_t ne00 = src0->ne[0]; + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + + if (ne00 == 0 || src0_nrows == 0) { + return HTP_STATUS_OK; + } + + if (src0_nrows == 1) { + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) { + return HTP_STATUS_OK; + } + + if (ne00 == 1) { + ((int32_t *) dst->data)[0] = 0; + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const uint32_t elems_per_thread = kparams->elems_per_thread; + + struct argmax_context actx = { + .octx = octx, + .src_data = (const float *) src0->data, + .dst_data = (int32_t *) dst->data, + .ne00 = ne00, + .src_stride = src0->nb[1] > 0 ? (uint32_t) src0->nb[1] : (uint32_t) (ne00 * sizeof(float)), + .dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t), + .row_start = 0, + .nrows = 1, + .rows_per_thread = 1, + .elems_per_thread = elems_per_thread, + }; + + work_queue_run(octx->ctx->work_queue, argmax_thread_single_row, &actx, n_threads); + + float best_val = actx.partial_max[0]; + int32_t best_idx = actx.partial_idx[0]; + for (uint32_t i = 1; i < n_threads; i++) { + if (actx.partial_max[i] > best_val) { + best_val = actx.partial_max[i]; + best_idx = actx.partial_idx[i]; + } + } + ((int32_t *) dst->data)[0] = best_idx; + return HTP_STATUS_OK; + } + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && + htp_tensor_is_contiguous(dst, sizeof(int32_t)); + const uint32_t elems_per_chunk = can_split ? 32 : 0; + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + src0_nrows, elems_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + + struct argmax_context actx = { + .octx = octx, + .src_data = (const float *) src0->data, + .dst_data = (int32_t *) dst->data, + .ne00 = ne00, + .src_stride = src0->nb[1] > 0 ? (uint32_t) src0->nb[1] : (uint32_t) (ne00 * sizeof(float)), + .dst_stride = dst->nb[0] > 0 ? (uint32_t) dst->nb[0] : (uint32_t) sizeof(int32_t), + .row_start = row_start, + .nrows = nrows, + .rows_per_thread = rows_per_thread, + .elems_per_thread = 0, + }; + work_queue_run(octx->ctx->work_queue, argmax_thread_multi_row, &actx, n_threads); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/sum-rows-ops.h b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h new file mode 100644 index 0000000..cc9ddd4 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/sum-rows-ops.h @@ -0,0 +1,17 @@ +#ifndef HTP_SUM_ROWS_OPS_H +#define HTP_SUM_ROWS_OPS_H + +#include + +struct htp_sum_rows_kernel_params { + uint32_t n_threads; + uint32_t elems_per_thread; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_sum_rows_kernel_params) <= 128, "htp_sum_rows_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_sum_rows_kernel_params) <= 128, "htp_sum_rows_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_SUM_ROWS_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index 03eccfd..3b63ac6 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -8,39 +8,125 @@ #include #include -#include "hex-dma.h" +#include "dma-queue.h" +#include "hex-fastdiv.h" #include "hvx-exp.h" #include "hvx-sigmoid.h" #include "hvx-utils.h" +#include "unary-ops.h" #define GGML_COMMON_DECL_C #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "hex-profile.h" + +struct htp_unary_context; + +typedef void (*unary_compute_fn_t)(const void * restrict src, + void * restrict dst, + uint32_t num_rows, + const struct htp_unary_context * uctx); + +typedef void (*unary_rms_norm_mul_compute_fn_t)(const void * restrict src, + const void * restrict weight, + void * restrict dst, + uint32_t num_rows, + const struct htp_unary_context * uctx); + +typedef void (*unary_tri_compute_fn_t)(const void * restrict src, + void * restrict dst, + uint32_t num_rows, + uint32_t ir, + const struct htp_unary_context * uctx); + +typedef void (*unary_tile_compute_fn_t)(void * restrict dst, + const void * restrict src, + uint32_t tw, + const struct htp_unary_context * uctx); + +typedef void (*unary_tiled_tri_compute_fn_t)(const void * restrict src, + void * restrict dst, + uint32_t tile_elems, + uint32_t col_start, + uint32_t i01, + uint32_t ne0, + int32_t ttype); struct htp_unary_context { struct htp_ops_context * octx; + const struct htp_unary_kernel_params * kparams; + + void * compute; - // Precomputed values - const uint8_t * data_src0; - uint8_t * data_dst; + dma_addr_t data_src0; + dma_addr_t data_src1; // weight/scale tensor for RMS_NORM_MUL + dma_addr_t data_dst; - size_t src0_row_size; - size_t dst_row_size; + size_t src0_data_row_size; // actual data bytes per row + size_t src1_data_row_size; + size_t dst_data_row_size; // actual data bytes per row size_t src0_row_size_aligned; + size_t src1_row_size_aligned; size_t dst_row_size_aligned; - size_t src0_spad_half_size; - size_t dst_spad_half_size; + size_t src0_vtcm_half_size; + size_t src1_vtcm_half_size; + size_t dst_vtcm_half_size; uint32_t block; uint32_t src0_nrows; uint32_t src0_nrows_per_thread; + uint32_t row_start; uint32_t nc; + uint32_t col_tile; // tiled mode + bool broadcast_weight; + + uint8_t * vtcm_src0; + uint8_t * vtcm_src1; + uint8_t * vtcm_dst; + + size_t vtcm_src0_size_per_thread; + size_t vtcm_src1_size_per_thread; + size_t vtcm_dst_size_per_thread; }; +// Convert flat row index to DDR byte offset using the tensor's actual strides. +// ir = i1 + ne1*(i2 + ne2*i3) => offset = i1*nb1 + i2*nb2 + i3*nb3 +static inline size_t unary_row_offset(uint32_t ir, + uint32_t ne1, uint32_t ne2, + const struct fastdiv_values * div_ne1, + const struct fastdiv_values * div_ne2, + const struct fastdiv_values * div_ne12, + size_t nb1, size_t nb2, size_t nb3) { + const uint32_t i1 = fastmodulo(ir, ne1, div_ne1); + const uint32_t ir_div_ne1 = fastdiv(ir, div_ne1); + const uint32_t i2 = fastmodulo(ir_div_ne1, ne2, div_ne2); + const uint32_t i3 = fastdiv(ir, div_ne12); + return i1 * nb1 + i2 * nb2 + i3 * nb3; +} + +// Safe DMA block size from row `ir`: clamp to the dim-1 slice boundary +// when non-contiguous so the nb1 stride stays valid for all rows. +static inline uint32_t unary_block_size(uint32_t ir, + uint32_t end_row, + uint32_t block, + bool contig, + uint32_t ne1, + const struct fastdiv_values * div_ne1) { + uint32_t limit = MIN(block, end_row - ir); + + if (!contig) { + const uint32_t slice_end = (fastdiv(ir, div_ne1) + 1) * ne1; + limit = MIN(limit, slice_end - ir); + } + + return limit; +} + #define htp_unary_preamble \ const uint32_t ne00 = src->ne[0]; \ const uint32_t ne01 = src->ne[1]; \ @@ -62,443 +148,1846 @@ struct htp_unary_context { const uint32_t nb2 = dst->nb[2]; \ const uint32_t nb3 = dst->nb[3]; -static void hvx_fast_rms_norm_f32(const uint8_t * restrict src, - uint8_t * restrict dst, - uint8_t * restrict pad, - const int num_elems, - float epsilon) { - (void)pad; +#define htp_unary_op_preamble \ + int32_t * op_params = uctx->octx->op_params; \ + const uint32_t ne0 = uctx->nc; \ + const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; \ + const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; - const HVX_Vector * restrict v_src = (HVX_Vector *) src; - HVX_Vector * restrict v_dst = (HVX_Vector *) dst; +static void scale_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float scale = 0.f; + float bias = 0.f; + memcpy(&scale, &op_params[0], sizeof(float)); + memcpy(&bias, &op_params[1], sizeof(float)); - const int nvec = num_elems / VLEN_FP32; // number of full vectors - const int nloe = num_elems % VLEN_FP32; // leftover elements + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - // Compute sum of squares for full vectors - HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000); - HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon); + hvx_scale_offset_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0, scale, bias); + } +} - #pragma unroll(4) - for (int i = 0; i < nvec; i++) { - HVX_Vector v1 = v_src[i]; - HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); - sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); +static void clamp_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float min = 0.f; + float max = 0.f; + memcpy(&min, &op_params[0], sizeof(float)); + memcpy(&max, &op_params[1], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_clamp_scalar_f32(dst_local, src_local, min, max, ne0); } +} - // Handle tail elements using vectorized ops with masking - if (nloe > 0) { - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); - HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1); - sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2); +static void leaky_relu_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float negative_slope = 0.f; + memcpy(&negative_slope, &op_params[0], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_leaky_relu_scalar_f32(dst_local, src_local, negative_slope, ne0); + } +} + +static void rms_norm_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + float scale = 0.f; + memcpy(&epsilon, &op_params[0], sizeof(float)); + memcpy(&scale, &op_params[1], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon, scale); } +} + +static void rms_norm_mul_f32(const void * restrict src, + const void * restrict weight, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + float scale = 0.f; + memcpy(&epsilon, &op_params[0], sizeof(float)); + memcpy(&scale, &op_params[1], sizeof(float)); - // Reduce HVX sum - sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v)); + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + const uint8_t * restrict w_local = (const uint8_t *)weight + (uctx->broadcast_weight ? 0 : ir * uctx->src1_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems); - HVX_Vector denom_v = hvx_vec_inverse_f32(t_v); - HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v); - HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v); + hvx_fast_rms_norm_mul_f32(src_local, w_local, dst_local, ne0, epsilon, scale); + } +} - // Scale full vectors - HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v)); +static void norm_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); - #pragma unroll(4) - for (int i = 0; i < nvec; i++) { - HVX_Vector v1 = v_src[i]; - HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); - v_dst[i] = Q6_Vsf_equals_Vqf32(v2); + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon); } +} - // Handle tail elements using vectorized ops with masking - if (nloe > 0) { +static void sqr_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sqr_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void sqrt_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; - HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4); - HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]); - HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v); - HVX_Vector result = Q6_Vsf_equals_Vqf32(v2); + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - // Store with masking to avoid overwriting memory beyond the tensor - hvx_vec_store_a(&v_dst[nvec], nloe * 4, result); + hvx_sqrt_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); } } -static void scale_f32(const float * restrict src, - float * restrict dst, - uint8_t * restrict spad, +static void scale_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, - const uint32_t row_elems, - const size_t row_size, - int32_t * op_params) { + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; float scale = 0.f; float bias = 0.f; memcpy(&scale, &op_params[0], sizeof(float)); memcpy(&bias, &op_params[1], sizeof(float)); for (uint32_t ir = 0; ir < num_rows; ir++) { - const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size); - uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size); + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_scale_offset_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0, scale, bias); + } +} + +static void clamp_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float min = 0.f; + float max = 0.f; + memcpy(&min, &op_params[0], sizeof(float)); + memcpy(&max, &op_params[1], sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_scale_offset_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, row_elems, scale, bias); + hvx_clamp_scalar_f16(dst_local, src_local, (_Float16) min, (_Float16) max, ne0); } } -static void rms_norm_f32(const float * restrict src, - float * restrict dst, - uint8_t * restrict spad, +static void rms_norm_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, - const uint32_t row_elems, - const size_t row_size, - int32_t * op_params) { + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_rms_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon); + } +} + +static void norm_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; float epsilon = 0.f; memcpy(&epsilon, op_params, sizeof(float)); for (uint32_t ir = 0; ir < num_rows; ir++) { - const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size); - uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size); + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_fast_rms_norm_f32((const uint8_t *) src_local, (uint8_t *) dst_local, spad, row_elems, epsilon); + hvx_fast_norm_f16((const uint8_t *) src_local, (uint8_t *) dst_local, ne0, epsilon); } } -static void sqr_f32(const float * restrict src, - float * restrict dst, - uint8_t * restrict spad, +static void sqr_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, - const uint32_t row_elems, - const size_t row_size, - int32_t * op_params) { + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; for (uint32_t ir = 0; ir < num_rows; ir++) { - const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size); - uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size); + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_sqr_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, row_elems); + hvx_sqr_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); } } -static void sqrt_f32(const float * restrict src, - float * restrict dst, - uint8_t * restrict spad, +static void sqrt_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, - const uint32_t row_elems, - const size_t row_size, - int32_t * op_params) { + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sqrt_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void abs_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; for (uint32_t ir = 0; ir < num_rows; ir++) { - const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size); - uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size); + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_sqrt_f32_aa((uint8_t *) dst_local, (const uint8_t *) src_local, row_elems); + hvx_abs_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); } } -static void neg_f32(const float * restrict src, - float * restrict dst, - uint8_t * restrict spad, +static void log_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, - const uint32_t row_elems, - const size_t row_size, - int32_t * op_params) { + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_log_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void step_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_step_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); + } +} + +static void ceil_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; for (uint32_t ir = 0; ir < num_rows; ir++) { - const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size); - uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size); + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_scale_f32_aa(dst_local, src_local, row_elems, -1.0f); + hvx_ceil_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); } } -static void exp_f32(const float * restrict src, - float * restrict dst, - uint8_t * restrict spad, +static void sgn_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, - const uint32_t row_elems, - const size_t row_size, - int32_t * op_params) { + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; for (uint32_t ir = 0; ir < num_rows; ir++) { - const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size); - uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size); + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_exp_f32(dst_local, src_local, row_elems, false); + hvx_sgn_f16_aa((uint8_t *) dst_local, (const uint8_t *) src_local, ne0); } } -static void sigmoid_f32(const float * restrict src, - float * restrict dst, - uint8_t * restrict spad, +static void l2_norm_f16(const void * restrict src, + void * restrict dst, const uint32_t num_rows, - const uint32_t row_elems, - const size_t row_size, - int32_t * op_params) { + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); for (uint32_t ir = 0; ir < num_rows; ir++) { - const uint8_t * restrict src_local = (const uint8_t *)src + (ir * row_size); - uint8_t * restrict dst_local = (uint8_t *)dst + (ir * row_size); + const uint8_t * restrict src_f = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_f = (uint8_t *)dst + (ir * dst_row_size_aligned); - hvx_sigmoid_f32_aa(dst_local, src_local, row_elems); + hvx_fast_l2_norm_f16((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon); } } -static void softplus_f32(const float * restrict src, - float * restrict dst, - uint8_t * restrict spad, - const uint32_t num_rows, - const uint32_t row_elems, - const size_t row_size, - int32_t * op_params) { - // softplus(x) = log(1 + exp(x)) - // Match CPU reference: ggml_compute_softplus_f32() in ggml-impl.h - for (uint32_t ir = 0; ir < num_rows; ir++) { - const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * row_size)); - float * restrict dst_f = (float *)((uint8_t *)dst + (ir * row_size)); - - for (uint32_t i = 0; i < row_elems; i++) { - float x = src_f[i]; - // For x > 20: softplus(x) ≈ x (avoids exp overflow) - dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x)); - } +static void neg_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_scale_f32_aa(dst_local, src_local, ne0, -1.0f); } } -static void unary_job_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; - struct htp_ops_context * octx = uctx->octx; - const struct htp_tensor * src = octx->src[0]; - const struct htp_tensor * dst = octx->dst; +static void exp_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; - htp_unary_preamble; + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - int htp_op = octx->op; - int32_t * op_params = octx->op_params; - uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + hvx_exp_f32(dst_local, src_local, ne0, false); + } +} - const size_t src0_row_size = uctx->src0_row_size; - const size_t dst_row_size = uctx->dst_row_size; +static void sigmoid_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; - const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; - const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - const uint32_t src0_nrows = uctx->src0_nrows; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + hvx_sigmoid_f32_aa(dst_local, src_local, ne0); + } +} - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; +// silu(x) = x * sigmoid(x) +static void silu_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sigmoid_f32_aa(dst_local, src_local, ne0); + hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0); + } +} + +// GELU uses the tanh approximation. +static __attribute__((noinline)) HVX_Vector hvx_vec_gelu_f32(HVX_Vector x) { + const HVX_Vector half = hvx_vec_splat_f32(0.5f); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + + HVX_Vector inner = hvx_vec_mul_f32_f32(x, x); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.044715f)); + inner = hvx_vec_add_f32_f32(inner, one); + inner = hvx_vec_mul_f32_f32(inner, x); + inner = hvx_vec_mul_f32_f32(inner, hvx_vec_splat_f32(0.7978845608028654f)); + + return hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(half, x), + hvx_vec_add_f32_f32(one, hvx_vec_tanh_f32(inner))); +} + +static inline void hvx_gelu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; + + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + + uint32_t i = 0; + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_gelu_f32(vsrc[i]); + } + if (nloe) { + hvx_vec_store_a(&vdst[i], nloe * sizeof(float), hvx_vec_gelu_f32(vsrc[i])); } +} + +// GELU_QUICK uses x * sigmoid(1.702 * x). +static __attribute__((noinline)) HVX_Vector hvx_vec_gelu_quick_f32(HVX_Vector x) { + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + const HVX_Vector max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector min_exp = hvx_vec_splat_f32(-87.0f); + const HVX_Vector scaled = hvx_vec_mul_f32_f32(x, hvx_vec_splat_f32(1.702f)); + const HVX_Vector sigmoid = hvx_vec_fast_sigmoid_f32_guard(scaled, one, max_exp, min_exp); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + return hvx_vec_mul_f32_f32(x, sigmoid); +} - const uint8_t * restrict data_src = uctx->data_src0; - uint8_t * restrict data_dst = uctx->data_dst; +static inline void hvx_gelu_quick_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); - uint8_t * src0_spad_data = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); - uint8_t * dst_spad_data = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc = (HVX_Vector *) src; - size_t src0_spad_half_size = uctx->src0_spad_half_size; - size_t dst_spad_half_size = uctx->dst_spad_half_size; + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; - const int BLOCK = uctx->block; - if (BLOCK == 0) { - FARF(ERROR, "unary-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", - octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; + uint32_t i = 0; + _Pragma("unroll(4)") + for (; i < nvec; i++) { + vdst[i] = hvx_vec_gelu_quick_f32(vsrc[i]); } + if (nloe) { + hvx_vec_store_a(&vdst[i], nloe * sizeof(float), hvx_vec_gelu_quick_f32(vsrc[i])); + } +} - dma_queue * dma_queue = octx->ctx->dma[ith]; +static void gelu_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_gelu_f32_aa(dst_local, src_local, ne0); + } +} - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); +static void gelu_quick_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); + hvx_gelu_quick_f32_aa(dst_local, src_local, ne0); } +} - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); +static void gelu_erf_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *) src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *) dst + (ir * dst_row_size_aligned); - // Process block in VTCM - switch (htp_op) { - case HTP_OP_RMS_NORM: - rms_norm_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params); - break; - case HTP_OP_SCALE: - scale_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params); - break; - case HTP_OP_SQR: - sqr_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params); - break; - case HTP_OP_SQRT: - sqrt_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params); - break; - case HTP_OP_UNARY_NEG: - neg_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params); - break; - case HTP_OP_UNARY_EXP: - exp_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params); - break; - case HTP_OP_UNARY_SIGMOID: - sigmoid_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params); - break; - case HTP_OP_UNARY_SOFTPLUS: - softplus_f32(src0_spad, dst_spad, NULL, block_size, ne0, src0_row_size_aligned, op_params); - break; - default: - break; + hvx_gelu_erf_f32_aa(dst_local, src_local, ne0); + } +} + +static void tri_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const uint32_t ir, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + const int32_t ttype = op_params[0]; + const HVX_Vector zero = hvx_vec_splat_f32(0.0f); + const uint32_t nvec = ne0 / VLEN_FP32; + const uint32_t nloe = ne0 % VLEN_FP32; + + const uint32_t ne01 = uctx->octx->src[0]->ne[1]; + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + + for (uint32_t b = 0; b < num_rows; b++) { + const uint32_t abs_row = ir + b; + const uint32_t i01 = fastmodulo(abs_row, ne01, div_ne01); + + const HVX_Vector * restrict v_src = (const HVX_Vector *) ((const uint8_t *) src + b * src0_row_size_aligned); + HVX_Vector * restrict v_dst = (HVX_Vector *) ((uint8_t *) dst + b * dst_row_size_aligned); + + uint32_t boundary; + int keep_left; + switch (ttype) { + case 0: boundary = i01; keep_left = 0; break; // keep col >= row + case 1: boundary = i01 + 1; keep_left = 0; break; // keep col > row + case 2: boundary = i01 + 1; keep_left = 1; break; // keep col <= row + case 3: boundary = i01; keep_left = 1; break; // keep col < row + default: boundary = 0; keep_left = 0; break; + } + if (boundary > ne0) boundary = ne0; + + // Full HVX vectors - each starts at a 128-byte aligned offset + for (uint32_t i = 0; i < nvec; i++) { + const uint32_t vec_start = i * VLEN_FP32; + const uint32_t vec_end = vec_start + VLEN_FP32; + if (keep_left) { + if (vec_end <= boundary) { + v_dst[i] = v_src[i]; + } else if (vec_start >= boundary) { + v_dst[i] = zero; + } else { + HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - vec_start) * sizeof(float)); + v_dst[i] = Q6_V_vmux_QVV(mask, v_src[i], zero); + } + } else { + if (vec_end <= boundary) { + v_dst[i] = zero; + } else if (vec_start >= boundary) { + v_dst[i] = v_src[i]; + } else { + HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - vec_start) * sizeof(float)); + v_dst[i] = Q6_V_vmux_QVV(mask, zero, v_src[i]); + } + } } - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), - dst_row_size, dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad, data_src + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); + // Tail elements (row_elems not a multiple of VLEN_FP32) + if (nloe > 0) { + const uint32_t abs_start = nvec * VLEN_FP32; + const uint32_t abs_end = abs_start + nloe; + HVX_Vector tail_val; + if (keep_left) { + if (abs_end <= boundary) { + tail_val = v_src[nvec]; + } else if (abs_start >= boundary) { + tail_val = zero; + } else { + HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float)); + tail_val = Q6_V_vmux_QVV(mask, v_src[nvec], zero); + } + } else { + if (abs_end <= boundary) { + tail_val = zero; + } else if (abs_start >= boundary) { + tail_val = v_src[nvec]; + } else { + HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float)); + tail_val = Q6_V_vmux_QVV(mask, zero, v_src[nvec]); + } + } + hvx_vec_store_a(&v_dst[nvec], nloe * sizeof(float), tail_val); } } +} + +// softplus(x) = log(1 + exp(x)) +// Match CPU reference: return x for x > 20 to avoid overflow. +static inline void hvx_softplus_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const HVX_Vector twenty = hvx_vec_splat_f32(20.0f); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); - dma_queue_flush(dma_queue); + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; - t2 = HAP_perf_get_qtimer_count(); + #pragma unroll(4) + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector x = v_src[i]; + HVX_VectorPred pred_gt20 = Q6_Q_vcmp_gt_VsfVsf(x, twenty); + HVX_Vector expx = hvx_vec_exp_f32(x); + HVX_Vector logv = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx)); + v_dst[i] = Q6_V_vmux_QVV(pred_gt20, x, logv); + } - FARF(HIGH, "unary-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, src->ne[0], - src->ne[1], src->ne[2], src->ne[3], src0_start_row, src0_end_row, dst->ne[0], dst->ne[1], dst->ne[2], - dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + if (nloe > 0) { + HVX_Vector x = v_src[nvec]; + HVX_VectorPred pred_gt20 = Q6_Q_vcmp_gt_VsfVsf(x, twenty); + HVX_Vector expx = hvx_vec_exp_f32(x); + HVX_Vector logv = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx)); + HVX_Vector sp = Q6_V_vmux_QVV(pred_gt20, x, logv); + hvx_vec_store_a((void *) &v_dst[nvec], nloe * sizeof(float), sp); + } } -static int execute_op_unary_f32(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; +static void softplus_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; - const struct htp_tensor * src0 = octx->src[0]; - const struct htp_tensor * dst = octx->dst; + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - const char * op_type = NULL; + hvx_softplus_f32_aa(dst_local, src_local, ne0); + } +} - switch (octx->op) { - case HTP_OP_RMS_NORM: - op_type = "rmsnorm-f32"; - break; - case HTP_OP_SCALE: - op_type = "scale-f32"; - break; - case HTP_OP_SQR: - op_type = "sqr-f32"; - break; - case HTP_OP_SQRT: - op_type = "sqrt-f32"; - break; - case HTP_OP_UNARY_NEG: - op_type = "neg-f32"; - break; - case HTP_OP_UNARY_EXP: - op_type = "exp-f32"; - break; - case HTP_OP_UNARY_SIGMOID: - op_type = "sigmoid-f32"; - break; - case HTP_OP_UNARY_SOFTPLUS: - op_type = "softplus-f32"; - break; +static void l2_norm_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + float epsilon = 0.f; + memcpy(&epsilon, op_params, sizeof(float)); - default: - FARF(ERROR, "Unsupported unary Op %u\n", octx->op); - return HTP_STATUS_NO_SUPPORT; + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_f = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_f = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_fast_l2_norm_f32((const uint8_t *)src_f, (uint8_t *)dst_f, ne0, epsilon); } +} - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); +static inline HVX_VectorPair hvx_fwht_stage_pair(HVX_Vector v0, HVX_Vector v1, int32_t b) { + HVX_VectorPair p = Q6_W_vshuff_VVR(v1, v0, b); + HVX_Vector s = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p)); + HVX_Vector d = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_lo_W(p), Q6_V_hi_W(p)); + return Q6_W_vshuff_VVR(d, s, b); +} - const size_t src0_row_size = src0->nb[1]; - const size_t dst_row_size = dst->nb[1]; +static inline HVX_VectorPair hvx_fwht32_pair(HVX_Vector v0_sf, HVX_Vector v1_sf, HVX_Vector v_scale) { + HVX_Vector v0 = Q6_Vqf32_vmpy_VsfVsf(v0_sf, v_scale); + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(v1_sf, v_scale); + HVX_VectorPair p; + p = hvx_fwht_stage_pair(v0, v1, 4); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 8); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 16); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 32); + p = hvx_fwht_stage_pair(Q6_V_lo_W(p), Q6_V_hi_W(p), 64); + return p; +} - const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); +static inline void hvx_fwht_store32(HVX_Vector * restrict v_dst, HVX_VectorPair p) { + v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); +} - // VTCM scratchpads for all tensors - // N rows per thread, padded to HVX vector size - // Double buffering requires 2x size per buffer +static inline void hvx_fwht_store64(HVX_Vector * restrict v_dst, HVX_VectorPair p) { + HVX_Vector u = Q6_V_lo_W(p); + HVX_Vector v = Q6_V_hi_W(p); + v_dst[0] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(u, v)); + v_dst[1] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(u, v)); +} - size_t spad_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned); - size_t vtcm_row_per_thread = (octx->ctx->vtcm_size)/ (n_threads * spad_size_per_row); +static inline void hvx_fwht_tail_f32(HVX_Vector * restrict v_dst, uint32_t nvec) { + uint32_t len_vec = 2; + for (; len_vec * 2 < nvec; len_vec <<= 2) { + const uint32_t step = len_vec * 4; + for (uint32_t i = 0; i < nvec; i += step) { + for (uint32_t j = 0; j < len_vec; j++) { + HVX_Vector a0 = v_dst[i + j]; + HVX_Vector a1 = v_dst[i + j + len_vec]; + HVX_Vector a2 = v_dst[i + j + 2 * len_vec]; + HVX_Vector a3 = v_dst[i + j + 3 * len_vec]; + + HVX_Vector t0 = Q6_Vqf32_vadd_VsfVsf(a0, a1); + HVX_Vector t1 = Q6_Vqf32_vsub_VsfVsf(a0, a1); + HVX_Vector t2 = Q6_Vqf32_vadd_VsfVsf(a2, a3); + HVX_Vector t3 = Q6_Vqf32_vsub_VsfVsf(a2, a3); + + v_dst[i + j] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t0, t2)); + v_dst[i + j + len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(t1, t3)); + v_dst[i + j + 2 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t0, t2)); + v_dst[i + j + 3 * len_vec] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(t1, t3)); + } + } + } - // Make sure the reserved vtcm size is sufficient - if (vtcm_row_per_thread == 0) { - FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, - spad_size_per_row * n_threads); - return HTP_STATUS_VTCM_TOO_SMALL; + if (len_vec < nvec) { + for (uint32_t i = 0; i < nvec; i += 2 * len_vec) { + for (uint32_t j = 0; j < len_vec; j++) { + HVX_Vector u = v_dst[i + j]; + HVX_Vector v = v_dst[i + len_vec + j]; + v_dst[i + j] = hvx_vec_add_f32_f32(u, v); + v_dst[i + len_vec + j] = hvx_vec_sub_f32_f32(u, v); + } + } } +} - octx->src0_spad.size_per_thread = src0_row_size_aligned * vtcm_row_per_thread * 2; - octx->dst_spad.size_per_thread = dst_row_size_aligned * vtcm_row_per_thread * 2; +static inline void hvx_fwht_row_f32(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const uint32_t nvec = n / VLEN_FP32; - octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread; - octx->dst_spad.size = n_threads * octx->dst_spad.size_per_thread; + if (nvec == 1) { + HVX_VectorPair p = hvx_fwht32_pair(v_src[0], v_src[0], v_scale); + hvx_fwht_store32(v_dst, p); + return; + } - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; + for (uint32_t i = 0; i < nvec; i += 2) { + HVX_VectorPair p = hvx_fwht32_pair(v_src[i], v_src[i + 1], v_scale); + hvx_fwht_store64(&v_dst[i], p); + } - FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", op_type, - src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size); + hvx_fwht_tail_f32(v_dst, nvec); +} - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - struct htp_unary_context uctx = { - .octx = octx, - .src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads, - .src0_nrows = src0_nrows, +static inline void hvx_fwht_row_f16(const uint8_t * restrict src, uint8_t * restrict dst, uint32_t n, HVX_Vector v_scale) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const uint32_t nvec = n / VLEN_FP32; - .data_src0 = (const uint8_t *)src0->data, - .data_dst = (uint8_t *)dst->data, + if (nvec == 1) { + HVX_Vector v0 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src[0])); + HVX_VectorPair p = hvx_fwht32_pair(v0, v0, v_scale); + hvx_fwht_store32(v_dst, p); + return; + } - .src0_row_size = src0_row_size, - .dst_row_size = dst_row_size, + const uint32_t nvec_f16 = n / VLEN_FP16; + for (uint32_t i = 0; i < nvec_f16; i++) { + HVX_VectorPair f32p = hvx_vec_f16_to_f32(v_src[i]); + HVX_VectorPair p = hvx_fwht32_pair(Q6_V_lo_W(f32p), Q6_V_hi_W(f32p), v_scale); + hvx_fwht_store64(&v_dst[2 * i], p); + } - .src0_row_size_aligned = src0_row_size_aligned, - .dst_row_size_aligned = dst_row_size_aligned, + hvx_fwht_tail_f32(v_dst, nvec); +} - .src0_spad_half_size = octx->src0_spad.size_per_thread / 2, - .dst_spad_half_size = octx->dst_spad.size_per_thread / 2, +static void hadamard_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0)); - .block = (octx->src0_spad.size_per_thread / 2) / src0_row_size_aligned, - .nc = src0->ne[0], - }; + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned); - worker_pool_run_func(octx->ctx->worker_pool, unary_job_f32_per_thread, &uctx, n_threads); + hvx_fwht_row_f32(src_row, dst_row, ne0, v_scale); } +} - return err; +static void hadamard_f16(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + const HVX_Vector v_scale = hvx_vec_splat_f32(1.0f / sqrtf((float) ne0)); + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_row = (const uint8_t *) src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_row = (uint8_t *) dst + (ir * dst_row_size_aligned); + + hvx_fwht_row_f16(src_row, dst_row, ne0, v_scale); + } } -int op_unary(struct htp_ops_context * octx) { - int err = HTP_STATUS_OK; +static void tanh_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; - switch (octx->src[0]->type) { - case HTP_TYPE_F32: - err = execute_op_unary_f32(octx); - break; + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - default: - err = HTP_STATUS_NO_SUPPORT; - break; + hvx_tanh_f32_aa(dst_local, src_local, ne0); } +} - return err; +static void abs_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_abs_f32_aa(dst_local, src_local, ne0); + } +} + +static void relu_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_max_scalar_f32(dst_local, src_local, 0.0f, ne0); + } +} + +static void step_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_step_f32_aa(dst_local, src_local, ne0); + } +} + +static void ceil_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_ceil_f32_aa(dst_local, src_local, ne0); + } +} + +static void sgn_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sgn_f32_aa(dst_local, src_local, ne0); + } +} + +static void log_f32(const void * restrict src, + void * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_log_f32_aa(dst_local, src_local, ne0); + } +} + +#// Pointwise unary ops on one column tile in VTCM. +static void tile_scale_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + float scale = 0.f; + float bias = 0.f; + memcpy(&scale, &uctx->octx->op_params[0], sizeof(float)); + memcpy(&bias, &uctx->octx->op_params[1], sizeof(float)); + hvx_scale_offset_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw, scale, bias); +} + +static void tile_clamp_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + float min = 0.f; + float max = 0.f; + memcpy(&min, &uctx->octx->op_params[0], sizeof(float)); + memcpy(&max, &uctx->octx->op_params[1], sizeof(float)); + hvx_clamp_scalar_f32((uint8_t *) dst, (const uint8_t *) src, min, max, tw); +} + +static void tile_leaky_relu_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + float negative_slope = 0.f; + memcpy(&negative_slope, &uctx->octx->op_params[0], sizeof(float)); + hvx_leaky_relu_scalar_f32((uint8_t *) dst, (const uint8_t *) src, negative_slope, tw); +} + +static void tile_sqr_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sqr_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_sqrt_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sqrt_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_neg_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_scale_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw, -1.0f); +} + +static void tile_exp_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_exp_f32((uint8_t *) dst, (const uint8_t *) src, tw, false); +} + +static void tile_sigmoid_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sigmoid_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_silu_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sigmoid_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); + hvx_mul_f32_aaa((uint8_t *) dst, (const uint8_t *) src, (uint8_t *) dst, tw); +} + +static void tile_gelu_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_gelu_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_gelu_quick_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_gelu_quick_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_gelu_erf_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_gelu_erf_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_softplus_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_softplus_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_tanh_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_tanh_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_abs_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_abs_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_log_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_log_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_relu_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_max_scalar_f32((uint8_t *) dst, (const uint8_t *) src, 0.0f, tw); +} + +static void tile_step_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_step_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_ceil_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_ceil_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tile_sgn_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { + (void) uctx; + hvx_sgn_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); +} + +static void tri_apply_tile_f32(const void * restrict src, void * restrict dst, + uint32_t tile_elems, uint32_t col_start, uint32_t i01, + uint32_t ne0, int32_t ttype) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + const HVX_Vector zero = hvx_vec_splat_f32(0.0f); + + uint32_t boundary; + int keep_left; + switch (ttype) { + case 0: boundary = i01; keep_left = 0; break; + case 1: boundary = i01 + 1; keep_left = 0; break; + case 2: boundary = i01 + 1; keep_left = 1; break; + case 3: boundary = i01; keep_left = 1; break; + default: boundary = 0; keep_left = 0; break; + } + if (boundary > ne0) boundary = ne0; + + const uint32_t nvec = tile_elems / VLEN_FP32; + const uint32_t nloe = tile_elems % VLEN_FP32; + + for (uint32_t i = 0; i < nvec; i++) { + const uint32_t abs_start = col_start + i * VLEN_FP32; + const uint32_t abs_end = abs_start + VLEN_FP32; + if (keep_left) { + if (abs_end <= boundary) { + v_dst[i] = v_src[i]; + } else if (abs_start >= boundary) { + v_dst[i] = zero; + } else { + HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float)); + v_dst[i] = Q6_V_vmux_QVV(mask, v_src[i], zero); + } + } else { + if (abs_end <= boundary) { + v_dst[i] = zero; + } else if (abs_start >= boundary) { + v_dst[i] = v_src[i]; + } else { + HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float)); + v_dst[i] = Q6_V_vmux_QVV(mask, zero, v_src[i]); + } + } + } + + if (nloe > 0) { + const uint32_t abs_start = col_start + nvec * VLEN_FP32; + const uint32_t abs_end = abs_start + nloe; + HVX_Vector tail_val; + if (keep_left) { + if (abs_end <= boundary) { + tail_val = v_src[nvec]; + } else if (abs_start >= boundary) { + tail_val = zero; + } else { + HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float)); + tail_val = Q6_V_vmux_QVV(mask, v_src[nvec], zero); + } + } else { + if (abs_end <= boundary) { + tail_val = zero; + } else if (abs_start >= boundary) { + tail_val = v_src[nvec]; + } else { + HVX_VectorPred mask = Q6_Q_vsetq_R((boundary - abs_start) * sizeof(float)); + tail_val = Q6_V_vmux_QVV(mask, zero, v_src[nvec]); + } + } + hvx_vec_store_a(&v_dst[nvec], nloe * sizeof(float), tail_val); + } +} + +// 1. Standard row-block unary task (F32 and F16). +static void unary_thread_row_block(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const size_t src0_data_row_size = uctx->src0_data_row_size; + const size_t dst_data_row_size = uctx->dst_data_row_size; + const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; + const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; + const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; + + const bool src0_contig = uctx->kparams->src0_contig; + const uint32_t BLOCK = uctx->block; + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); + + dma_queue_push(dma_q, + dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), + nb1, dst_row_size_aligned, dst_data_row_size, 0); + + const size_t src0_off = src0_contig ? ((size_t) ir * nb01) : + unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); + + ir += block_size; + } + + unary_compute_fn_t compute = (unary_compute_fn_t) uctx->compute; + + for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, src0_contig, ne01, div_ne01); + + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); + compute(src0_vtcm, dst_vtcm, block_size, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); + + const size_t dst_off = (size_t) ir * nb1; + dma_queue_push(dma_q, + dma_make_data(data_dst + dst_off, dst_vtcm), + nb1, dst_row_size_aligned, dst_data_row_size, block_size); + + const uint32_t next_ir = ir + block_size; + if (next_ir < src0_end_row) { + const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, src0_contig, + ne01, div_ne01); + const uint32_t pref_ir = next_ir + next_block_size; + if (pref_ir < src0_end_row) { + const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, src0_contig, + ne01, div_ne01); + const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) : + unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm, data_src + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + } + } + ir += block_size; + } + + dma_queue_flush(dma_q); +} + +// 2. RMS_NORM_MUL row-block task with weight buffer. +static void unary_thread_rms_norm_mul_f32(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const size_t src0_data_row_size = uctx->src0_data_row_size; + const size_t dst_data_row_size = uctx->dst_data_row_size; + const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; + const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_src1 = uctx->data_src1; + const dma_addr_t data_dst = uctx->data_dst; + + const struct htp_tensor * src1 = octx->src[1]; + const uint32_t nb11 = src1->nb[1]; + const uint32_t nb12 = src1->nb[2]; + const uint32_t nb13 = src1->nb[3]; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * src1_vtcm_data = uctx->vtcm_src1 ? (uctx->vtcm_src1 + (ith * uctx->vtcm_src1_size_per_thread)) : NULL; + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; + const size_t src1_vtcm_half_size = uctx->src1_vtcm_half_size; + const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; + + const bool src0_contig = uctx->kparams->src0_contig; + const bool src1_contig = uctx->kparams->src1_contig; + const bool block_contig = src0_contig && src1_contig; + const uint32_t BLOCK = uctx->block; + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + if (uctx->broadcast_weight) { + dma_queue_push(dma_q, dma_make_data(src1_vtcm_data, data_src1), + uctx->src1_row_size_aligned, 0, uctx->src1_data_row_size, 1); + dma_queue_flush(dma_q); + } + + for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_contig, ne01, div_ne01); + + dma_queue_push(dma_q, + dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), + nb1, dst_row_size_aligned, dst_data_row_size, 0); + + const size_t src0_off = src0_contig ? ((size_t) ir * nb01) : + unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); + + if (!uctx->broadcast_weight) { + const size_t src1_off = src1_contig ? ((size_t) ir * nb11) : + unary_row_offset(ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); + dma_queue_push(dma_q, + dma_make_data(src1_vtcm_data + (vtcm_idx * src1_vtcm_half_size), data_src1 + src1_off), + uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, block_size); + } + + ir += block_size; + } + + unary_rms_norm_mul_compute_fn_t compute = (unary_rms_norm_mul_compute_fn_t) uctx->compute; + + for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { + const uint32_t block_size = unary_block_size(ir, src0_end_row, BLOCK, block_contig, ne01, div_ne01); + + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + void * src1_vtcm = NULL; + if (!uctx->broadcast_weight) { + src1_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); + const void * w = uctx->broadcast_weight ? (const void *) src1_vtcm_data : src1_vtcm; + compute(src0_vtcm, w, dst_vtcm, block_size, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); + + const size_t dst_off = (size_t) ir * nb1; + dma_queue_push(dma_q, + dma_make_data(data_dst + dst_off, dst_vtcm), + nb1, dst_row_size_aligned, dst_data_row_size, block_size); + + const uint32_t next_ir = ir + block_size; + if (next_ir < src0_end_row) { + const uint32_t next_block_size = unary_block_size(next_ir, src0_end_row, BLOCK, block_contig, + ne01, div_ne01); + const uint32_t pref_ir = next_ir + next_block_size; + if (pref_ir < src0_end_row) { + const uint32_t pref_block_size = unary_block_size(pref_ir, src0_end_row, BLOCK, block_contig, + ne01, div_ne01); + const size_t src0_pref_off = src0_contig ? ((size_t) pref_ir * nb01) : + unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03); + dma_queue_push(dma_q, + dma_make_data(src0_vtcm, data_src + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + + if (!uctx->broadcast_weight) { + const size_t src1_pref_off = src1_contig ? ((size_t) pref_ir * nb11) : + unary_row_offset(pref_ir, ne01, ne02, div_ne01, div_ne02, div_ne012, nb11, nb12, nb13); + dma_queue_push(dma_q, + dma_make_data(src1_vtcm, data_src1 + src1_pref_off), + uctx->src1_row_size_aligned, nb11, uctx->src1_data_row_size, pref_block_size); + } + } + } + ir += block_size; + } + + dma_queue_flush(dma_q); +} + +// 3. TRI row-block task with row index ir. +static void unary_thread_tri_f32(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const size_t src0_data_row_size = uctx->src0_data_row_size; + const size_t dst_data_row_size = uctx->dst_data_row_size; + const size_t src0_row_size_aligned = uctx->src0_row_size_aligned; + const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_vtcm_half_size = uctx->src0_vtcm_half_size; + const size_t dst_vtcm_half_size = uctx->dst_vtcm_half_size; + const uint32_t BLOCK = uctx->block; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + for (uint32_t ir = src0_start_row, vtcm_idx = 0; ir < src0_end_row && vtcm_idx < 2; vtcm_idx++) { + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); + + dma_queue_push(dma_q, + dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_vtcm_half_size)), + nb1, dst_row_size_aligned, dst_data_row_size, 0); + + const size_t src0_off = (size_t) ir * nb01; + dma_queue_push(dma_q, + dma_make_data(src0_vtcm_data + (vtcm_idx * src0_vtcm_half_size), data_src + src0_off), + src0_row_size_aligned, nb01, src0_data_row_size, block_size); + + ir += block_size; + } + + unary_tri_compute_fn_t compute = (unary_tri_compute_fn_t) uctx->compute; + + for (uint32_t ir = src0_start_row; ir < src0_end_row; ) { + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); + + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src0_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); + compute(src0_vtcm, dst_vtcm, block_size, ir, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); + + const size_t dst_off = (size_t) ir * nb1; + dma_queue_push(dma_q, + dma_make_data(data_dst + dst_off, dst_vtcm), + nb1, dst_row_size_aligned, dst_data_row_size, block_size); + + const uint32_t next_ir = ir + block_size; + if (next_ir < src0_end_row) { + const uint32_t next_block_size = MIN(BLOCK, src0_end_row - next_ir); + const uint32_t pref_ir = next_ir + next_block_size; + if (pref_ir < src0_end_row) { + const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_ir); + const size_t src0_pref_off = (size_t) pref_ir * nb01; + dma_queue_push(dma_q, + dma_make_data(src0_vtcm, data_src + src0_pref_off), + src0_row_size_aligned, nb01, src0_data_row_size, pref_block_size); + } + } + ir += block_size; + } + + dma_queue_flush(dma_q); +} + +// 4. Pointwise tiled unary task. +static void unary_thread_tiled(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const uint32_t col_tile = uctx->col_tile; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_half = uctx->src0_vtcm_half_size; + const size_t dst_half = uctx->dst_vtcm_half_size; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; + const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; + const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; + + const uint32_t tiles_per_row = uctx->kparams->tiles_per_row; + const bool src0_contig = uctx->kparams->src0_contig; + + const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; + + for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { + const uint32_t row = src0_start_row + fastdiv(t, div_tpr); + const uint32_t col = fastmodulo(t, tiles_per_row, div_tpr) * col_tile; + const uint32_t tw = MIN(col_tile, ne0 - col); + const size_t tb = (size_t) tw * sizeof(float); + const size_t soff = (src0_contig ? ((size_t) row * nb01) : + unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) + + (size_t) col * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); + dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1); + } + + unary_tile_compute_fn_t compute = (unary_tile_compute_fn_t) uctx->compute; + + uint32_t row = src0_start_row; + uint32_t col = 0; + uint32_t tile_in_row = 0; + + uint32_t prow = src0_start_row + fastdiv(2, div_tpr); + uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; + uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); + + for (uint32_t t = 0; t < total_tiles; t++) { + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + const uint32_t tw = MIN(col_tile, ne0 - col); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); + compute(dst_vtcm, src_vtcm, tw, uctx); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); + + const size_t doff = (size_t) row * nb1 + (size_t) col * sizeof(float); + const size_t tb = (size_t) tw * sizeof(float); + dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1); + + const uint32_t pt = t + 2; + if (pt < total_tiles) { + const uint32_t ptw = MIN(col_tile, ne0 - pcol); + const size_t ptb = (size_t) ptw * sizeof(float); + const size_t psoff = (src0_contig ? ((size_t) prow * nb01) : + unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, + nb03)) + + (size_t) pcol * sizeof(float); + dma_queue_push(dma_q, dma_make_data(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); + } + + tile_in_row++; + col += col_tile; + if (tile_in_row == tiles_per_row) { + tile_in_row = 0; + col = 0; + row++; + } + + ptile_in_row++; + pcol += col_tile; + if (ptile_in_row == tiles_per_row) { + ptile_in_row = 0; + pcol = 0; + prow++; + } + } + + dma_queue_flush(dma_q); +} + +// 5. TRI tiled task. +static void unary_thread_tiled_tri_f32(unsigned int nth, unsigned int ith, void * data) { + (void) nth; + const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; + struct htp_ops_context * octx = uctx->octx; + const struct htp_tensor * src = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + htp_unary_preamble; + + const uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; + const int32_t * op_params = octx->op_params; + const uint32_t col_tile = uctx->col_tile; + + const uint32_t src0_nrows = uctx->src0_nrows; + const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); + + if (src0_start_row >= src0_end_row) { + return; + } + + const dma_addr_t data_src = uctx->data_src0; + const dma_addr_t data_dst = uctx->data_dst; + + uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); + uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); + + const size_t src0_half = uctx->src0_vtcm_half_size; + const size_t dst_half = uctx->dst_vtcm_half_size; + + dma_queue * dma_q = octx->ctx->dma[ith]; + + const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; + const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; + + const uint32_t tiles_per_row = uctx->kparams->tiles_per_row; + const int32_t tri_ttype = op_params[0]; + + const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; + + for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { + const uint32_t row = src0_start_row + fastdiv(t, div_tpr); + const uint32_t col = fastmodulo(t, tiles_per_row, div_tpr) * col_tile; + const uint32_t tw = MIN(col_tile, ne0 - col); + const size_t tb = (size_t) tw * sizeof(float); + const size_t soff = (size_t) row * nb01 + (size_t) col * sizeof(float); + + dma_queue_push(dma_q, dma_make_data(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); + dma_queue_push(dma_q, dma_make_data(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1); + } + + unary_tiled_tri_compute_fn_t compute = (unary_tiled_tri_compute_fn_t) uctx->compute; + + uint32_t row = src0_start_row; + uint32_t col = 0; + uint32_t tile_in_row = 0; + uint32_t i01 = fastmodulo(row, ne01, div_ne01); + + uint32_t prow = src0_start_row + fastdiv(2, div_tpr); + uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; + uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); + + for (uint32_t t = 0; t < total_tiles; t++) { + void * dst_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).src; + void * src_vtcm = (void *) (uintptr_t) dma_queue_pop(dma_q).dst; + + const uint32_t tw = MIN(col_tile, ne0 - col); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); + compute(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); + + const size_t doff = (size_t) row * nb1 + (size_t) col * sizeof(float); + const size_t tb = (size_t) tw * sizeof(float); + dma_queue_push(dma_q, dma_make_data(data_dst + doff, dst_vtcm), tb, tb, tb, 1); + + const uint32_t pt = t + 2; + if (pt < total_tiles) { + const uint32_t ptw = MIN(col_tile, ne0 - pcol); + const size_t ptb = (size_t) ptw * sizeof(float); + const size_t psoff = (size_t) prow * nb01 + (size_t) pcol * sizeof(float); + dma_queue_push(dma_q, dma_make_data(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); + } + + tile_in_row++; + col += col_tile; + if (tile_in_row == tiles_per_row) { + tile_in_row = 0; + col = 0; + row++; + i01++; + if (i01 == ne01) { + i01 = 0; + } + } + + ptile_in_row++; + pcol += col_tile; + if (ptile_in_row == tiles_per_row) { + ptile_in_row = 0; + pcol = 0; + prow++; + } + } + + dma_queue_flush(dma_q); +} + +static int execute_op_unary(struct htp_ops_context * octx) { + int err = HTP_STATUS_OK; + + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * dst = octx->dst; + + const bool is_f16 = (src0->type == HTP_TYPE_F16); + + const char * op_type = NULL; + + switch (octx->op) { + case HTP_OP_NORM: op_type = is_f16 ? "norm-f16" : "norm-f32"; break; + case HTP_OP_RMS_NORM: op_type = is_f16 ? "rmsnorm-f16" : "rmsnorm-f32"; break; + case HTP_OP_RMS_NORM_MUL: op_type = "rmsnorm-mul-f32"; break; + case HTP_OP_SCALE: op_type = is_f16 ? "scale-f16" : "scale-f32"; break; + case HTP_OP_CLAMP: op_type = is_f16 ? "clamp-f16" : "clamp-f32"; break; + case HTP_OP_LEAKY_RELU: op_type = "leaky-relu-f32"; break; + case HTP_OP_SQR: op_type = is_f16 ? "sqr-f16" : "sqr-f32"; break; + case HTP_OP_SQRT: op_type = is_f16 ? "sqrt-f16" : "sqrt-f32"; break; + case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break; + case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break; + case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break; + case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break; + case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break; + case HTP_OP_UNARY_GELU_QUICK: op_type = "gelu-quick-f32"; break; + case HTP_OP_UNARY_GELU_ERF: op_type = "gelu-erf-f32"; break; + case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break; + case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break; + case HTP_OP_UNARY_ABS: op_type = is_f16 ? "abs-f16" : "abs-f32"; break; + case HTP_OP_UNARY_LOG: op_type = is_f16 ? "log-f16" : "log-f32"; break; + case HTP_OP_UNARY_RELU: op_type = "relu-f32"; break; + case HTP_OP_UNARY_STEP: op_type = is_f16 ? "step-f16" : "step-f32"; break; + case HTP_OP_UNARY_CEIL: op_type = is_f16 ? "ceil-f16" : "ceil-f32"; break; + case HTP_OP_UNARY_SGN: op_type = is_f16 ? "sgn-f16" : "sgn-f32"; break; + case HTP_OP_L2_NORM: op_type = is_f16 ? "l2norm-f16" : "l2norm-f32"; break; + case HTP_OP_TRI: op_type = "tri-f32"; break; + case HTP_OP_HADAMARD: op_type = is_f16 ? "hadamard-f16" : "hadamard-f32"; break; + default: + FARF(ERROR, "Unsupported unary Op %u\n", octx->op); + return HTP_STATUS_NO_SUPPORT; + } + + const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params; + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const size_t src0_elem_size = is_f16 ? sizeof(_Float16) : sizeof(float); + const size_t dst_elem_size = (dst->type == HTP_TYPE_F16) ? sizeof(_Float16) : sizeof(float); + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t src0_data_row_size = src0->ne[0] * src0_elem_size; + const size_t dst_data_row_size = dst->ne[0] * dst_elem_size; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst_elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + // Always 0 for F16 - htp_unary_solve_layout() keeps F16 on the row-block path, + // since only F32 has unary_task_f32_tiled_* kernels. + const uint32_t col_tile = kparams->col_tile; + + size_t src1_data_row_size = 0; + bool broadcast_weight = kparams->broadcast_weight; + const struct htp_tensor * src1 = NULL; + + // RMS_NORM_MUL fusion is F32-only (its weight tensor is always F32; see + // try_fuse_node()'s type guard), so this never triggers when is_f16 is true. + if (octx->op == HTP_OP_RMS_NORM_MUL) { + src1 = octx->src[1]; + src1_data_row_size = src1->ne[0] * sizeof(float); + } + + if ((!col_tile && kparams->block == 0) || octx->ctx->vtcm_size < (size_t) kparams->vtcm_size) { + FARF(ERROR, "unary-%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, (size_t) kparams->vtcm_size); + return HTP_STATUS_VTCM_TOO_SMALL; + } + + octx->src0_spad.src = NULL; + octx->src1_spad.src = NULL; + octx->dst_spad.src = NULL; + + struct htp_unary_vtcm_layout L; + htp_unary_vtcm_layout_build(&L, octx->op, src0_data_row_size, dst_data_row_size, src1_data_row_size, + src0_elem_size, col_tile, kparams->block, broadcast_weight, kparams->n_threads); + + FARF(HIGH, "%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + L.src0_bytes * n_threads, L.src1_bytes * n_threads, L.dst_bytes * n_threads); + + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + struct htp_unary_context uctx = { + .octx = octx, + .kparams = kparams, + .src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div), + .src0_nrows = nrows, + .row_start = row_start, + + .data_src0 = src0->data, + .data_src1 = (octx->op == HTP_OP_RMS_NORM_MUL) ? src1->data : 0, + .data_dst = dst->data, + + .src0_data_row_size = src0_data_row_size, + .src1_data_row_size = src1_data_row_size, + .dst_data_row_size = dst_data_row_size, + + .src0_row_size_aligned = L.src0_row_size_aligned, + .src1_row_size_aligned = L.src1_row_size_aligned, + .dst_row_size_aligned = L.dst_row_size_aligned, + + .src0_vtcm_half_size = L.src0_bytes >> 1, + .src1_vtcm_half_size = (octx->op == HTP_OP_RMS_NORM_MUL) ? (L.src1_bytes >> (broadcast_weight ? 0 : 1)) : 0, + .dst_vtcm_half_size = L.dst_bytes >> 1, + + .block = kparams->block, + .nc = src0->ne[0], + .col_tile = col_tile, + .broadcast_weight = broadcast_weight, + + .vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0), + .vtcm_src1 = VTCM_LAYOUT_PTR_OPTIONAL(uint8_t, base, L.off_src1, L.src1_bytes > 0), + .vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst), + + .vtcm_src0_size_per_thread = L.src0_bytes, + .vtcm_src1_size_per_thread = L.src1_bytes, + .vtcm_dst_size_per_thread = L.dst_bytes, + }; + + FARF(HIGH, "%s: %s mode (col_tile %u)\n", op_type, col_tile ? "tiled" : "row-block", col_tile); + + worker_callback_t task_func = NULL; + void * compute_func = NULL; + + if (col_tile) { + task_func = unary_thread_tiled; + switch (octx->op) { + case HTP_OP_SCALE: compute_func = (void *) tile_scale_f32; break; + case HTP_OP_CLAMP: compute_func = (void *) tile_clamp_f32; break; + case HTP_OP_LEAKY_RELU: compute_func = (void *) tile_leaky_relu_f32; break; + case HTP_OP_SQR: compute_func = (void *) tile_sqr_f32; break; + case HTP_OP_SQRT: compute_func = (void *) tile_sqrt_f32; break; + case HTP_OP_UNARY_NEG: compute_func = (void *) tile_neg_f32; break; + case HTP_OP_UNARY_EXP: compute_func = (void *) tile_exp_f32; break; + case HTP_OP_UNARY_SIGMOID: compute_func = (void *) tile_sigmoid_f32; break; + case HTP_OP_UNARY_SILU: compute_func = (void *) tile_silu_f32; break; + case HTP_OP_UNARY_GELU: compute_func = (void *) tile_gelu_f32; break; + case HTP_OP_UNARY_GELU_QUICK: compute_func = (void *) tile_gelu_quick_f32; break; + case HTP_OP_UNARY_GELU_ERF: compute_func = (void *) tile_gelu_erf_f32; break; + case HTP_OP_UNARY_SOFTPLUS: compute_func = (void *) tile_softplus_f32; break; + case HTP_OP_UNARY_TANH: compute_func = (void *) tile_tanh_f32; break; + case HTP_OP_UNARY_ABS: compute_func = (void *) tile_abs_f32; break; + case HTP_OP_UNARY_LOG: compute_func = (void *) tile_log_f32; break; + case HTP_OP_UNARY_RELU: compute_func = (void *) tile_relu_f32; break; + case HTP_OP_UNARY_STEP: compute_func = (void *) tile_step_f32; break; + case HTP_OP_UNARY_CEIL: compute_func = (void *) tile_ceil_f32; break; + case HTP_OP_UNARY_SGN: compute_func = (void *) tile_sgn_f32; break; + case HTP_OP_TRI: + task_func = unary_thread_tiled_tri_f32; + compute_func = (void *) tri_apply_tile_f32; + break; + default: break; + } + } else if (is_f16) { + task_func = unary_thread_row_block; + switch (octx->op) { + case HTP_OP_NORM: compute_func = (void *) norm_f16; break; + case HTP_OP_RMS_NORM: compute_func = (void *) rms_norm_f16; break; + case HTP_OP_SCALE: compute_func = (void *) scale_f16; break; + case HTP_OP_CLAMP: compute_func = (void *) clamp_f16; break; + case HTP_OP_SQR: compute_func = (void *) sqr_f16; break; + case HTP_OP_SQRT: compute_func = (void *) sqrt_f16; break; + case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f16; break; + case HTP_OP_UNARY_ABS: compute_func = (void *) abs_f16; break; + case HTP_OP_UNARY_LOG: compute_func = (void *) log_f16; break; + case HTP_OP_UNARY_STEP: compute_func = (void *) step_f16; break; + case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f16; break; + case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f16; break; + case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f16; break; + default: break; + } + } else { + task_func = unary_thread_row_block; + switch (octx->op) { + case HTP_OP_NORM: compute_func = (void *) norm_f32; break; + case HTP_OP_RMS_NORM: compute_func = (void *) rms_norm_f32; break; + case HTP_OP_RMS_NORM_MUL: + task_func = unary_thread_rms_norm_mul_f32; + compute_func = (void *) rms_norm_mul_f32; + break; + case HTP_OP_SCALE: compute_func = (void *) scale_f32; break; + case HTP_OP_CLAMP: compute_func = (void *) clamp_f32; break; + case HTP_OP_LEAKY_RELU: compute_func = (void *) leaky_relu_f32; break; + case HTP_OP_SQR: compute_func = (void *) sqr_f32; break; + case HTP_OP_SQRT: compute_func = (void *) sqrt_f32; break; + case HTP_OP_UNARY_NEG: compute_func = (void *) neg_f32; break; + case HTP_OP_UNARY_EXP: compute_func = (void *) exp_f32; break; + case HTP_OP_UNARY_SIGMOID: compute_func = (void *) sigmoid_f32; break; + case HTP_OP_UNARY_SILU: compute_func = (void *) silu_f32; break; + case HTP_OP_UNARY_GELU: compute_func = (void *) gelu_f32; break; + case HTP_OP_UNARY_GELU_QUICK: compute_func = (void *) gelu_quick_f32; break; + case HTP_OP_UNARY_GELU_ERF: compute_func = (void *) gelu_erf_f32; break; + case HTP_OP_UNARY_SOFTPLUS: compute_func = (void *) softplus_f32; break; + case HTP_OP_UNARY_TANH: compute_func = (void *) tanh_f32; break; + case HTP_OP_UNARY_ABS: compute_func = (void *) abs_f32; break; + case HTP_OP_UNARY_LOG: compute_func = (void *) log_f32; break; + case HTP_OP_UNARY_RELU: compute_func = (void *) relu_f32; break; + case HTP_OP_UNARY_STEP: compute_func = (void *) step_f32; break; + case HTP_OP_UNARY_CEIL: compute_func = (void *) ceil_f32; break; + case HTP_OP_UNARY_SGN: compute_func = (void *) sgn_f32; break; + case HTP_OP_L2_NORM: compute_func = (void *) l2_norm_f32; break; + case HTP_OP_HADAMARD: compute_func = (void *) hadamard_f32; break; + case HTP_OP_TRI: + task_func = unary_thread_tri_f32; + compute_func = (void *) tri_f32; + break; + default: break; + } + } + + if (!task_func || !compute_func) { + FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op); + return HTP_STATUS_NO_SUPPORT; + } + + uctx.compute = compute_func; + work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads); + + return err; +} + +int op_unary(struct htp_ops_context * octx) { + switch (octx->src[0]->type) { + case HTP_TYPE_F32: + case HTP_TYPE_F16: + return execute_op_unary(octx); + + default: + return HTP_STATUS_NO_SUPPORT; + } } diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h new file mode 100644 index 0000000..713e40d --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -0,0 +1,215 @@ +#ifndef HTP_UNARY_OPS_H +#define HTP_UNARY_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-ops.h" +#include "htp-vtcm.h" + +// Op-specific struct for precomputed unary params +struct htp_unary_kernel_params { + uint32_t n_threads; + uint32_t col_tile; + uint32_t block; + uint32_t broadcast_weight; + + uint32_t src0_contig; + uint32_t src1_contig; + + uint32_t vtcm_size; + uint32_t tiles_per_row; + + // Fastdiv helpers + struct fastdiv_values div_ne01; + struct fastdiv_values div_ne02; + struct fastdiv_values div_ne012; + struct fastdiv_values div_tpr; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_unary_kernel_params) <= 128, "htp_unary_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_unary_kernel_params) <= 128, "htp_unary_kernel_params is too large for kernel_params blob"); +#endif + +static inline bool htp_op_is_unary(uint32_t opcode) { + switch (opcode) { + case HTP_OP_CLAMP: + case HTP_OP_LEAKY_RELU: + case HTP_OP_NORM: + case HTP_OP_RMS_NORM: + case HTP_OP_RMS_NORM_MUL: + case HTP_OP_SCALE: + case HTP_OP_SQR: + case HTP_OP_SQRT: + case HTP_OP_UNARY_NEG: + case HTP_OP_UNARY_EXP: + case HTP_OP_UNARY_SIGMOID: + case HTP_OP_UNARY_SILU: + case HTP_OP_UNARY_GELU: + case HTP_OP_UNARY_GELU_QUICK: + case HTP_OP_UNARY_GELU_ERF: + case HTP_OP_UNARY_SOFTPLUS: + case HTP_OP_UNARY_TANH: + case HTP_OP_UNARY_ABS: + case HTP_OP_UNARY_LOG: + case HTP_OP_UNARY_RELU: + case HTP_OP_UNARY_STEP: + case HTP_OP_UNARY_CEIL: + case HTP_OP_UNARY_SGN: + case HTP_OP_L2_NORM: + case HTP_OP_TRI: + case HTP_OP_HADAMARD: + return true; + default: + return false; + } +} + +static inline bool htp_unary_is_elementwise(uint32_t opcode) { + if (!htp_op_is_unary(opcode)) { + return false; + } + switch (opcode) { + case HTP_OP_NORM: + case HTP_OP_RMS_NORM: + case HTP_OP_RMS_NORM_MUL: + case HTP_OP_L2_NORM: + case HTP_OP_HADAMARD: + case HTP_OP_TRI: + return false; + default: + return true; + } +} + +struct htp_unary_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_src1; + size_t off_dst; + + size_t src0_row_size_aligned; + size_t src1_row_size_aligned; + size_t dst_row_size_aligned; + + size_t src0_bytes; + size_t src1_bytes; + size_t dst_bytes; +}; + +static inline void htp_unary_vtcm_layout_build( + struct htp_unary_vtcm_layout * L, + uint32_t op, + size_t src0_data_row_size, + size_t dst_data_row_size, + size_t src1_data_row_size, + size_t elem_size, + uint32_t col_tile, + uint32_t block, + bool broadcast_weight, + uint32_t n_threads +) { + L->src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); + L->dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); + L->src1_row_size_aligned = (op == HTP_OP_RMS_NORM_MUL) ? hex_round_up(src1_data_row_size, 128) : 0; + + if (col_tile > 0) { + const size_t col_tile_bytes = col_tile * elem_size; + L->src0_bytes = col_tile_bytes * 2; + L->dst_bytes = col_tile_bytes * 2; + L->src1_bytes = 0; + } else { + L->src0_bytes = L->src0_row_size_aligned * block * 2; + L->dst_bytes = L->dst_row_size_aligned * block * 2; + if (op == HTP_OP_RMS_NORM_MUL) { + L->src1_bytes = broadcast_weight ? L->src1_row_size_aligned : (L->src1_row_size_aligned * block * 2); + } else { + L->src1_bytes = 0; + } + } + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes * n_threads); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_src1, L->src1_bytes * n_threads, L->src1_bytes > 0); + VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes * n_threads); + L->total_bytes = off; +} + +static inline void htp_unary_solve_layout( + struct htp_unary_vtcm_layout * L, + uint32_t op, + uint32_t ne00, + uint32_t ne10, + uint32_t ne11, + uint32_t max_block, + bool broadcast_weight, + uint32_t n_threads, + size_t vtcm_size, + size_t elem_size, + uint32_t * out_col_tile, + uint32_t * out_vtcm_row_per_thread +) { + const size_t dst_elem_size = (op == HTP_OP_HADAMARD) ? sizeof(float) : elem_size; + const size_t src0_data_row_size = ne00 * elem_size; + const size_t dst_data_row_size = ne10 * dst_elem_size; + const size_t src1_data_row_size = (op == HTP_OP_RMS_NORM_MUL) ? (ne11 * sizeof(float)) : 0; + + const size_t src0_row_size_aligned = hex_round_up(src0_data_row_size, 128); + const size_t dst_row_size_aligned = hex_round_up(dst_data_row_size, 128); + const size_t src1_row_size_aligned = (op == HTP_OP_RMS_NORM_MUL) ? hex_round_up(src1_data_row_size, 128) : 0; + + size_t vtcm_size_per_row = 0; + size_t vtcm_row_per_thread = 0; + + if (op == HTP_OP_RMS_NORM_MUL) { + if (broadcast_weight) { + size_t available_vtcm = vtcm_size; + size_t src1_vtcm_total = n_threads * src1_row_size_aligned; + if (available_vtcm > src1_vtcm_total) { + available_vtcm -= src1_vtcm_total; + } else { + available_vtcm = 0; + } + vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned); + vtcm_row_per_thread = available_vtcm / (n_threads * vtcm_size_per_row); + } else { + vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned + src1_row_size_aligned); + vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row); + } + } else { + vtcm_size_per_row = 2 * (src0_row_size_aligned + dst_row_size_aligned); + vtcm_row_per_thread = vtcm_size / (n_threads * vtcm_size_per_row); + } + + if (max_block > 0 && vtcm_row_per_thread > max_block) { + vtcm_row_per_thread = max_block; + } + + const bool is_reduction = (op == HTP_OP_NORM || op == HTP_OP_RMS_NORM || + op == HTP_OP_RMS_NORM_MUL || op == HTP_OP_L2_NORM || + op == HTP_OP_HADAMARD); + // The tiled fallback path below only has F32 task functions (unary_task_f32_tiled_*); + // F16 has no tiled kernels, so it must stay on the row-block path like reduction ops. + // NOTE: if F16 ends up with vtcm_row_per_thread == 0 here (row too large for the VTCM + // budget), execute_op_unary() will see BLOCK == 0 and skip computation for that op + // (logged via FARF(ERROR, ...)) since there is no F16 tiled fallback. This is a known + // limitation; supporting it would require adding F16 tiled kernels. + const bool is_f16 = (elem_size == sizeof(_Float16)); + uint32_t col_tile = 0; + + if (vtcm_row_per_thread == 0 && !is_reduction && !is_f16) { + const size_t per_thread_budget = vtcm_size / n_threads; + const size_t col_tile_bytes = hex_align_down(per_thread_budget / 4, 128); + col_tile = (uint32_t) (col_tile_bytes / elem_size); + } + + const uint32_t block = col_tile ? 0 : (uint32_t) vtcm_row_per_thread; + htp_unary_vtcm_layout_build(L, op, src0_data_row_size, dst_data_row_size, src1_data_row_size, + elem_size, col_tile, block, broadcast_weight, n_threads); + + *out_col_tile = col_tile; + *out_vtcm_row_per_thread = (uint32_t) vtcm_row_per_thread; +} + +#endif /* HTP_UNARY_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/work-queue.c b/ggml/src/ggml-hexagon/htp/work-queue.c new file mode 100644 index 0000000..6c74445 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/work-queue.c @@ -0,0 +1,243 @@ +#include "work-queue.h" +#include "hex-utils.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "HAP_farf.h" + +#define LOWEST_USABLE_QURT_PRIO (254) + +// internal structure kept in thread-local storage per instance of work queue +typedef struct { + work_queue_t queue; + unsigned int id; +} worker_context_t; + +struct work_queue_task_s { + work_queue_func_t func; + void * data; + unsigned int n_threads; + atomic_uint barrier; +}; + +// internal structure kept in thread-local storage per instance of work queue +struct work_queue_s { + atomic_uint seqn; // seqno used to detect new jobs + atomic_uint idx_read; // Updated by producer (pop/reclaim) + atomic_uint idx_write; // Updated by producer (push) + uint32_t idx_mask; + uint32_t capacity; + + qurt_thread_t thread[WORK_QUEUE_MAX_N_THREADS]; // thread ID's of the workers + worker_context_t context[WORK_QUEUE_MAX_N_THREADS]; // worker contexts + void * stack[WORK_QUEUE_MAX_N_THREADS]; // thread stack pointers + unsigned int n_threads; // total threads (workers + main) + unsigned int n_workers; // number of active threads (just workers) + + atomic_uint active; // workers are polling/active + atomic_bool killed; // threads need to exit + bool external_mem; // memory owned externally + + struct work_queue_task_s queue[] __attribute__((aligned(HEX_L2_LINE_SIZE))); +}; + +static void work_queue_thread(void * context) { + worker_context_t * me = (worker_context_t *) context; + work_queue_t q = me->queue; + + FARF(HIGH, "work-queue: thread %u started", me->id); + + unsigned int prev_seqn = 0; + + while (!atomic_load_explicit(&q->killed, memory_order_relaxed)) { + unsigned int seqn = atomic_load_explicit(&q->seqn, memory_order_acquire); + if (seqn == prev_seqn) { + if (atomic_load_explicit(&q->active, memory_order_relaxed)) { + hex_pause(); + } else { + qurt_futex_wait(&q->active, 0); + } + continue; + } + + prev_seqn = seqn; + + // Process all active tasks in the queue + unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed); + unsigned int iw = atomic_load_explicit(&q->idx_write, memory_order_relaxed); + + while (ir != iw) { + struct work_queue_task_s * task = &q->queue[ir]; + + unsigned int n = task->n_threads; + unsigned int i = me->id; + if (i < n) { + task->func(n, i, task->data); + + atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); + } else { + while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) { + hex_pause(); + } + } + + ir = (ir + 1) & q->idx_mask; + } + } + + FARF(HIGH, "work-queue: thread %u stopped", me->id); +} + +bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { + if (n > q->n_threads) { + FARF(ERROR, "work-queue: invalid number of jobs %u for n-threads %u", n, q->n_threads); + return false; + } + + unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed); + unsigned int iw = atomic_load_explicit(&q->idx_write, memory_order_relaxed); + + if (((iw + 1) & q->idx_mask) == ir) { + FARF(ERROR, "work-queue-push: queue is full\n"); + return false; + } + + struct work_queue_task_s * task = &q->queue[iw]; + task->func = func; + task->data = data; + task->n_threads = n; + atomic_store_explicit(&task->barrier, n, memory_order_relaxed); + + atomic_store_explicit(&q->idx_write, (iw + 1) & q->idx_mask, memory_order_relaxed); + + // publish job to workers (already awake and polling) + atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); + + // main thread runs job #0 + func(n, 0, data); + + atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release); + + while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) { + hex_pause(); + } + + atomic_thread_fence(memory_order_acquire); + + atomic_store_explicit(&q->idx_read, (ir + 1) & q->idx_mask, memory_order_relaxed); + + return true; +} + +size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size) { + capacity = hex_ceil_pow2(capacity); + uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0; + size_t size_stacks = stack_size * n_workers; + size_t size_q = hex_align_up(sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s), HEX_L2_LINE_SIZE); + return size_stacks + size_q; +} + +size_t work_queue_alignof(void) { + return 4096; +} + +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size) { + capacity = hex_ceil_pow2(capacity); + uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0; + unsigned char * mem_blob = (unsigned char *) ptr; + + work_queue_t q = (work_queue_t) (mem_blob + stack_size * n_workers); + memset(q, 0, sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s)); + + q->n_threads = n_threads; + q->n_workers = n_workers; + q->external_mem = true; + q->capacity = capacity; + + for (unsigned int i = 0; i < n_workers; i++) { + q->stack[i] = mem_blob; mem_blob += stack_size; + q->thread[i] = 0; + q->context[i].id = i + 1; + q->context[i].queue = q; + } + + atomic_init(&q->idx_read, 0); + atomic_init(&q->idx_write, 0); + atomic_init(&q->seqn, 0); + atomic_init(&q->active, 0); + q->idx_mask = capacity - 1; + q->killed = 0; + for (int i = 0; i < (int) capacity; i++) { + atomic_init(&q->queue[i].barrier, 0); + q->queue[i].func = NULL; + q->queue[i].data = NULL; + q->queue[i].n_threads = 0; + } + + // launch the workers + qurt_thread_attr_t attr; + qurt_thread_attr_init(&attr); + + for (unsigned int i = 0; i < n_workers; i++) { + qurt_thread_attr_set_stack_addr(&attr, q->stack[i]); + qurt_thread_attr_set_stack_size(&attr, stack_size); + + char thread_name[32]; + snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i); + qurt_thread_attr_set_name(&attr, thread_name); + + // set up priority - by default, match the creating thread's prio + int prio = qurt_thread_get_priority(qurt_thread_get_id()); + if (prio < 1) { + prio = 1; + } + if (prio > LOWEST_USABLE_QURT_PRIO) { + prio = LOWEST_USABLE_QURT_PRIO; + } + + qurt_thread_attr_set_priority(&attr, prio); + + int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i]); + if (err) { + FARF(ERROR, "Could not launch worker threads!"); + work_queue_free(q); + return NULL; + } + } + + return q; +} + +void work_queue_free(work_queue_t q) { + if (!q) { return; } + + atomic_store_explicit(&q->killed, 1, memory_order_relaxed); + atomic_store_explicit(&q->active, 1, memory_order_release); + atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release); + qurt_futex_wake(&q->active, q->n_workers); + + for (unsigned int i = 0; i < q->n_workers; i++) { + if (q->thread[i]) { + int status; + (void) qurt_thread_join(q->thread[i], &status); + } + } +} + +void work_queue_wakeup(work_queue_t q) { + if (!atomic_load_explicit(&q->active, memory_order_relaxed)) { + atomic_store_explicit(&q->active, 1, memory_order_release); + qurt_futex_wake(&q->active, q->n_workers); + } +} + +void work_queue_suspend(work_queue_t q) { + atomic_store_explicit(&q->active, 0, memory_order_release); +} diff --git a/ggml/src/ggml-hexagon/htp/work-queue.h b/ggml/src/ggml-hexagon/htp/work-queue.h new file mode 100644 index 0000000..09ca4b1 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/work-queue.h @@ -0,0 +1,38 @@ +#ifndef HTP_WORK_QUEUE_H +#define HTP_WORK_QUEUE_H + +#include +#include +#include + +typedef void (*work_queue_func_t)(unsigned int n, unsigned int i, void *); + +struct work_queue_s; +typedef struct work_queue_s * work_queue_t; + +#define WORK_QUEUE_MAX_N_THREADS 10 + +size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size); +size_t work_queue_alignof(void); +work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size); +void work_queue_free(work_queue_t q); + +void work_queue_wakeup(work_queue_t q); +void work_queue_suspend(work_queue_t q); + +bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n); + +static inline bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) { + if (n <= 1) { + func(n, 0, data); + return true; + } + return work_queue_run_async(q, func, data, n); +} + +// Legacy compatibility +typedef work_queue_func_t worker_callback_t; +#define worker_pool_run_func work_queue_run +#define worker_pool work_queue + +#endif // #ifndef HTP_WORK_QUEUE_H diff --git a/ggml/src/ggml-hexagon/htp/worker-pool.c b/ggml/src/ggml-hexagon/htp/worker-pool.c deleted file mode 100644 index 172e289..0000000 --- a/ggml/src/ggml-hexagon/htp/worker-pool.c +++ /dev/null @@ -1,293 +0,0 @@ -#include "worker-pool.h" - -#include -#include -#include -#include -#include -#include - -#include "HAP_farf.h" - -#define WORKER_THREAD_STACK_SZ (2 * 16384) -#define LOWEST_USABLE_QURT_PRIO (254) - -struct worker_pool_s; - -// internal structure kept in thread-local storage per instance of worker pool -typedef struct { - struct worker_pool_s * pool; - unsigned int id; -} worker_context_t; - -// internal structure kept in thread-local storage per instance of worker pool -typedef struct worker_pool_s { - worker_pool_job_t job[MAX_NUM_WORKERS]; // list of job descriptors - qurt_thread_t thread[MAX_NUM_WORKERS]; // thread ID's of the workers - worker_context_t context[MAX_NUM_WORKERS]; // worker contexts - void * stack[MAX_NUM_WORKERS]; // thread stack pointers - unsigned int n_threads; // number of workers in this pool - - atomic_uint seqn; // seqno used to detect new jobs - atomic_uint next_job; // next job index - atomic_uint n_pending; // number of pending jobs - atomic_uint n_jobs; // number of current jobs - atomic_bool killed; // threads need to exit -} worker_pool_t; - -static void worker_pool_main(void * context) { - worker_context_t * me = (worker_context_t *) context; - worker_pool_t * pool = me->pool; - - FARF(HIGH, "worker-pool: thread %u started", me->id); - - unsigned int prev_seqn = 0; - while (!atomic_load(&pool->killed)) { - unsigned int seqn = atomic_load(&pool->seqn); - if (seqn == prev_seqn) { - // Nothing to do - qurt_futex_wait(&pool->seqn, prev_seqn); - continue; - } - - // New job - prev_seqn = seqn; - - unsigned int n = atomic_load(&pool->n_jobs); - unsigned int i = atomic_fetch_add(&pool->next_job, 1); - if (i >= n) { - // Spurious wakeup - continue; - } - - pool->job[i].func(n, i, pool->job[i].data); - - atomic_fetch_sub(&pool->n_pending, 1); - } - - FARF(HIGH, "worker-pool: thread %u stopped", me->id); -} - -AEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context, uint32_t n_threads, uint32_t stack_size) { - int err = 0; - - if (NULL == context) { - FARF(ERROR, "NULL context passed to worker_pool_init()."); - return AEE_EBADPARM; - } - - // Allocations - int size = (stack_size * n_threads) + (sizeof(worker_pool_t)); - - unsigned char * mem_blob = (unsigned char *) malloc(size); - if (!mem_blob) { - FARF(ERROR, "Could not allocate memory for worker pool!!"); - return AEE_ENOMEMORY; - } - - worker_pool_t * me = (worker_pool_t *) (mem_blob + stack_size * n_threads); - - // name for the first worker, useful in debugging threads - char name[19]; - snprintf(name, 12, "0x%8x:", (int) me); - strcat(name, "worker0"); - me->n_threads = n_threads; - - // initializations - for (unsigned int i = 0; i < me->n_threads; i++) { - me->stack[i] = NULL; - me->thread[i] = 0; - - me->context[i].id = i; - me->context[i].pool = me; - } - - // initialize job queue - me->n_pending = 0; - me->n_jobs = 0; - me->next_job = 0; - me->seqn = 0; - me->killed = 0; - - // launch the workers - qurt_thread_attr_t attr; - qurt_thread_attr_init(&attr); - - for (unsigned int i = 0; i < me->n_threads; i++) { - // set up stack - me->stack[i] = mem_blob; - mem_blob += stack_size; - qurt_thread_attr_set_stack_addr(&attr, me->stack[i]); - qurt_thread_attr_set_stack_size(&attr, stack_size); - - // set up name - qurt_thread_attr_set_name(&attr, name); - name[17] = (name[17] + 1); - // name threads context:worker0, context:worker1, .. (recycle at 9, but num threads should be less than that anyway) - if (name[17] > '9') { - name[17] = '0'; - } - - // set up priority - by default, match the creating thread's prio - int prio = qurt_thread_get_priority(qurt_thread_get_id()); - - if (prio < 1) { - prio = 1; - } - if (prio > LOWEST_USABLE_QURT_PRIO) { - prio = LOWEST_USABLE_QURT_PRIO; - } - - qurt_thread_attr_set_priority(&attr, prio); - - // launch - err = qurt_thread_create(&me->thread[i], &attr, worker_pool_main, (void *) &me->context[i]); - if (err) { - FARF(ERROR, "Could not launch worker threads!"); - worker_pool_release((worker_pool_context_t *) &me); - return AEE_EQURTTHREADCREATE; - } - } - *context = (worker_pool_context_t *) me; - return AEE_SUCCESS; -} - -AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads) { - return worker_pool_init_with_stack_size(context, n_threads, WORKER_THREAD_STACK_SZ); -} - -// clean up worker pool -void worker_pool_release(worker_pool_context_t * context) { - worker_pool_t * me = (worker_pool_t *) *context; - - // if no worker pool exists, return error. - if (NULL == me) { - return; - } - - atomic_store(&me->killed, 1); - atomic_fetch_add(&me->seqn, 1); - qurt_futex_wake(&me->seqn, me->n_threads); - - // de-initializations - for (unsigned int i = 0; i < me->n_threads; i++) { - if (me->thread[i]) { - int status; - (void) qurt_thread_join(me->thread[i], &status); - } - } - - // free allocated memory (were allocated as a single buffer starting at stack[0]) - if (me->stack[0]) { - free(me->stack[0]); - } - - *context = NULL; -} - -// run jobs -AEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n) { - worker_pool_t * me = (worker_pool_t *) context; - if (NULL == me) { - FARF(ERROR, "worker-pool: invalid context"); - return AEE_EBADPARM; - } - - if (n > me->n_threads) { - FARF(ERROR, "worker-pool: invalid number of jobs %u for n-threads %u", n, me->n_threads); - return AEE_EBADPARM; - } - - memcpy(me->job, job, sizeof(worker_pool_job_t) * n); - - if (n > 1) { - atomic_store(&me->next_job, 1); - atomic_store(&me->n_jobs, n); - atomic_store(&me->n_pending, n - 1); - - // wake up workers - atomic_fetch_add(&me->seqn, 1); - qurt_futex_wake(&me->seqn, n - 1); - } - - // main thread runs job #0 - me->job[0].func(n, 0, me->job[0].data); - - if (n > 1) { - while (atomic_load(&me->n_pending)) - ; - } - - return 0; -} - -// run func -AEEResult worker_pool_run_func(worker_pool_context_t context, worker_callback_t func, void * data, unsigned int n) { - worker_pool_job_t job[n]; - - for (unsigned int i = 0; i < n; i++) { - job[i].func = func; - job[i].data = data; - } - - return worker_pool_run_jobs(context, job, n); -} - -AEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio) { - worker_pool_t * me = (worker_pool_t *) context; - - // if no worker pool exists, return error. - if (!me) { - return AEE_ENOMORE; - } - - int result = AEE_SUCCESS; - if (prio < 1) { - prio = 1; - } - if (prio > LOWEST_USABLE_QURT_PRIO) { - prio = LOWEST_USABLE_QURT_PRIO; - } - - for (unsigned int i = 0; i < me->n_threads; i++) { - int res = qurt_thread_set_priority(me->thread[i], (unsigned short) prio); - if (0 != res) { - result = AEE_EBADPARM; - FARF(ERROR, "QURT failed to set priority of thread %d, ERROR = %d", me->thread[i], res); - } - } - - return result; -} - -AEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids) { - worker_pool_t * me = (worker_pool_t *) context; - if (!me) { - FARF(ERROR, "worker-pool: invalid context"); - return AEE_EBADPARM; - ; - } - - for (int i = 0; i < me->n_threads; i++) { - tids[i] = me->thread[i]; - } - - return AEE_SUCCESS; -} - -AEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio) { - worker_pool_t * me = (worker_pool_t *) context; - if (!me) { - FARF(ERROR, "worker-pool: invalid context"); - return AEE_EBADPARM; - } - - int priority = qurt_thread_get_priority(me->thread[0]); - if (priority > 0) { - *prio = priority; - return 0; - } else { - *prio = 0; - return AEE_EBADSTATE; - } -} diff --git a/ggml/src/ggml-hexagon/htp/worker-pool.h b/ggml/src/ggml-hexagon/htp/worker-pool.h deleted file mode 100644 index 6f8c905..0000000 --- a/ggml/src/ggml-hexagon/htp/worker-pool.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifndef HTP_WORKER_POOL_H -#define HTP_WORKER_POOL_H - -// MACRO enables function to be visible in shared-library case. -#define WORKERPOOL_API __attribute__((visibility("default"))) - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/// signature of callbacks to be invoked by worker threads -typedef void (*worker_callback_t)(unsigned int n, unsigned int i, void *); - -/// Typedef of worker_pool context -typedef void * worker_pool_context_t; - -/// descriptor for requested callback -typedef struct { - worker_callback_t func; - void * data; -} worker_pool_job_t; - -/// Maximum supported number of worker threads. -#define MAX_NUM_WORKERS 10 - -// Initialize worker pool. -WORKERPOOL_API AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads); - -// Initialize worker pool with custom stack size -WORKERPOOL_API AEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context, - uint32_t n_threads, - uint32_t stack_size); - -// Kill worker threads and release worker pool resources -WORKERPOOL_API void worker_pool_release(worker_pool_context_t * context); - -// Run jobs with the worker pool. -WORKERPOOL_API AEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n); - -WORKERPOOL_API AEEResult worker_pool_run_func(worker_pool_context_t context, - worker_callback_t func, - void * data, - unsigned int n); - -WORKERPOOL_API AEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio); -WORKERPOOL_API AEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio); -WORKERPOOL_API AEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids); - -#ifdef __cplusplus -} -#endif - -#endif // #ifndef HTP_WORKER_POOL_H diff --git a/ggml/src/ggml-hexagon/libggml-htp.inf b/ggml/src/ggml-hexagon/libggml-htp.inf index 656d2d9..ff6ac27 100644 --- a/ggml/src/ggml-hexagon/libggml-htp.inf +++ b/ggml/src/ggml-hexagon/libggml-htp.inf @@ -8,17 +8,17 @@ CatalogFile = libggml-htp.cat PnpLockDown = 1 [DestinationDirs] -Drivers_Dir = 6 +Drivers_Dir = 13 [SourceDisksNames] 1 = %DiskId% [SourceDisksFiles] -libggml-htp-v68.so = 1 -libggml-htp-v69.so = 1 libggml-htp-v73.so = 1 libggml-htp-v75.so = 1 +libggml-htp-v79.so = 1 libggml-htp-v81.so = 1 +libggml-htp-v85.so = 1 [ControlFlags] ExcludeFromSelect = * @@ -27,11 +27,11 @@ ExcludeFromSelect = * CopyFiles=Drivers_Dir [Drivers_Dir] -libggml-htp-v68.so,,,0x10 ;COPYFLG_NO_OVERWRITE -libggml-htp-v69.so,,,0x10 ;COPYFLG_NO_OVERWRITE libggml-htp-v73.so,,,0x10 ;COPYFLG_NO_OVERWRITE libggml-htp-v75.so,,,0x10 ;COPYFLG_NO_OVERWRITE +libggml-htp-v79.so,,,0x10 ;COPYFLG_NO_OVERWRITE libggml-htp-v81.so,,,0x10 ;COPYFLG_NO_OVERWRITE +libggml-htp-v85.so,,,0x10 ;COPYFLG_NO_OVERWRITE [Strings] GGML = 'GGML' diff --git a/ggml/src/ggml-hexagon/op-desc.h b/ggml/src/ggml-hexagon/op-desc.h deleted file mode 100644 index a1e8ddd..0000000 --- a/ggml/src/ggml-hexagon/op-desc.h +++ /dev/null @@ -1,153 +0,0 @@ -#ifndef OP_DESC_H -#define OP_DESC_H - -#define GGML_COMMON_IMPL_CPP -#include "ggml-backend-impl.h" -#include "ggml-common.h" - -#include -#include - -struct op_desc { - char strides[64 * GGML_MAX_SRC]; - char dims[64 * GGML_MAX_SRC]; - char types[16 * GGML_MAX_SRC]; - char buffs[64 * GGML_MAX_SRC]; - char names[64 * GGML_MAX_SRC]; - - int format_tensor_dims(char * str, const struct ggml_tensor * t) { - if (t->ne[2] == 1 && t->ne[3] == 1) { - return sprintf(str, "%d:%d", (int) t->ne[0], (int) t->ne[1]); - } else { - return sprintf(str, "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]); - } - } - - void format_op_dims(char * str, const struct ggml_tensor * t) { - char * p = str; - - // append src0 and src1 (if any) - if (t->src[0]) { - p += format_tensor_dims(p, t->src[0]); - - for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) { - p += sprintf(p, " x "); - p += format_tensor_dims(p, t->src[i]); - } - - p += sprintf(p, " -> "); - } - - // format self dims separately for better visual alignment - char self[64]; - format_tensor_dims(self, t); - - p += sprintf(p, "%s", self); - } - - int format_tensor_strides(char * str, const struct ggml_tensor * t) { - const char * c = ggml_is_contiguous(t) ? "" : "!"; - - if (t->ne[2] == 1 && t->ne[3] == 1) { - return sprintf(str, "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c); - } else { - return sprintf(str, "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c); - } - } - - void format_op_strides(char * str, const struct ggml_tensor * t) { - char * p = str; - - // append src0 and src1 (if any) - if (t->src[0]) { - p += format_tensor_strides(p, t->src[0]); - - for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) { - p += sprintf(p, " x "); - p += format_tensor_strides(p, t->src[i]); - } - - p += sprintf(p, " -> "); - } - - // format self dims separately for better visual alignment - char self[64]; - format_tensor_strides(self, t); - - p += sprintf(p, "%s", self); - } - - void format_op_types(char * str, const struct ggml_tensor * t) { - char * p = str; - - // append src0 and src1 (if any) - if (t->src[0]) { - p += sprintf(p, "%s", ggml_type_name(t->src[0]->type)); - - for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) { - p += sprintf(p, " x "); - p += sprintf(p, "%s", ggml_type_name(t->src[i]->type)); - } - - p += sprintf(p, " -> "); - } - - p += sprintf(p, "%s", ggml_type_name(t->type)); - } - - const char * tensor_buff_name(const struct ggml_tensor * t) { - if (t->buffer) { - return ggml_backend_buffer_name(t->buffer); - } - return "NONE"; - } - - void format_op_buffs(char * str, const struct ggml_tensor * t) { - char * p = str; - - // append src0 and src1 (if any) - if (t->src[0]) { - p += sprintf(p, "%s", tensor_buff_name(t->src[0])); - - for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) { - p += sprintf(p, " x "); - p += sprintf(p, "%s", tensor_buff_name(t->src[i])); - } - - p += sprintf(p, " -> "); - } - - p += sprintf(p, "%s", tensor_buff_name(t)); - } - - void format_op_names(char * str, const struct ggml_tensor * t) { - char * p = str; - - // append src0 and src1 (if any) - if (t->src[0]) { - p += sprintf(p, "%s", t->src[0]->name); - - for (int i = 1; i < GGML_MAX_SRC && t->src[i]; i++) { - p += sprintf(p, " x "); - p += sprintf(p, "%s", t->src[i]->name); - } - - p += sprintf(p, " -> "); - } - - p += sprintf(p, "%s", t->name); - } - - void format(const ggml_tensor * op) { - format_op_dims(dims, op); - format_op_strides(strides, op); - format_op_types(types, op); - format_op_buffs(buffs, op); - format_op_names(names, op); - } - - op_desc() {} - op_desc(const ggml_tensor * op) { format(op); } -}; - -#endif // OP_DESC_H diff --git a/ggml/src/ggml-hip/CMakeLists.txt b/ggml/src/ggml-hip/CMakeLists.txt index a7d4e0e..75d4d60 100644 --- a/ggml/src/ggml-hip/CMakeLists.txt +++ b/ggml/src/ggml-hip/CMakeLists.txt @@ -12,7 +12,7 @@ list(APPEND CMAKE_PREFIX_PATH ${ROCM_PATH}) list(APPEND CMAKE_PREFIX_PATH "${ROCM_PATH}/lib64/cmake") if (NOT DEFINED CMAKE_HIP_FLAGS_DEBUG) - set(CMAKE_HIP_FLAGS_DEBUG "-g -O2") + set(CMAKE_HIP_FLAGS_DEBUG "-g -Xarch_device -O2 -Xarch_host -O0") endif() # CMake on Windows doesn't support the HIP language yet @@ -70,17 +70,8 @@ list(APPEND GGML_SOURCES_ROCM ${SRCS}) file(GLOB SRCS "../ggml-cuda/template-instances/mmf*.cu") list(APPEND GGML_SOURCES_ROCM ${SRCS}) -if (GGML_CUDA_FA_ALL_QUANTS) - file(GLOB SRCS "../ggml-cuda/template-instances/fattn-vec*.cu") - list(APPEND GGML_SOURCES_ROCM ${SRCS}) - add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS) -else() - list(APPEND GGML_SOURCES_ROCM - ../ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu) -endif() +ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR}/../ggml-cuda SRCS) +list(APPEND GGML_SOURCES_ROCM ${SRCS}) ggml_add_backend_library(ggml-hip ${GGML_HEADERS_ROCM} @@ -114,10 +105,6 @@ if (GGML_HIP_NO_VMM) add_compile_definitions(GGML_HIP_NO_VMM) endif() -if (GGML_HIP_ROCWMMA_FATTN) - add_compile_definitions(GGML_HIP_ROCWMMA_FATTN) -endif() - if (NOT GGML_HIP_MMQ_MFMA) add_compile_definitions(GGML_HIP_NO_MMQ_MFMA) endif() diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index 62b76ab..ae26e0c 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -160,6 +160,18 @@ static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t return ((const float *)(tensor->op_params))[i]; } +// [TAG_GGML_PREC] +// - GGML_OP_MUL_MAT +// 0 - acc +// 1 - hint +// 2 - src0 precision +// 3 - src1 precision +// +// - GGML_OP_MUL_MAT_ID +// 0 - acc +// 1 - hint +// 2 - src0 precision +// 3 - src1 precision static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) { assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t)); ((int32_t *)(tensor->op_params))[i] = value; diff --git a/ggml/src/ggml-metal/CMakeLists.txt b/ggml/src/ggml-metal/CMakeLists.txt index 42054d8..91691b2 100644 --- a/ggml/src/ggml-metal/CMakeLists.txt +++ b/ggml/src/ggml-metal/CMakeLists.txt @@ -10,7 +10,9 @@ ggml_add_backend_library(ggml-metal ggml-metal-device.cpp ggml-metal-common.cpp ggml-metal-context.m + ggml-metal-fusion.cpp ggml-metal-ops.cpp + ggml-metal-tuning.cpp ) target_link_libraries(ggml-metal PRIVATE @@ -24,65 +26,177 @@ if (GGML_METAL_NDEBUG) endif() set(METALLIB_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/../ggml-common.h") +set(METALLIB_KERNELS_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/kernels/common.h") +set(METALLIB_KERNELS_DEQUANTIZE "${CMAKE_CURRENT_SOURCE_DIR}/kernels/dequantize.h") +set(METALLIB_KERNELS_QUANTIZE "${CMAKE_CURRENT_SOURCE_DIR}/kernels/quantize.h") +set(METALLIB_KERNELS_FA_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/kernels/fa_common.metal") +set(METALLIB_KERNELS_FA_VEC_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/kernels/fa_vec_common.metal") +set(METALLIB_KERNELS_FA_SHARED + ${METALLIB_KERNELS_FA_COMMON} + ${METALLIB_KERNELS_FA_VEC_COMMON} +) + +set(METALLIB_KERNEL_SOURCES + kernels/fa_aux.metal + kernels/fa_f16.metal + kernels/fa_f32.metal + kernels/fa_q4_0.metal + kernels/fa_q4_1.metal + kernels/fa_q5_0.metal + kernels/fa_q5_1.metal + kernels/fa_q8_0.metal + kernels/fa_tensor.metal + kernels/fa_vec_f16.metal + kernels/fa_vec_f32.metal + kernels/fa_vec_q4_0.metal + kernels/fa_vec_q4_1.metal + kernels/fa_vec_q5_0.metal + kernels/fa_vec_q5_1.metal + kernels/fa_vec_q8_0.metal + kernels/mul_mv.metal + kernels/mul_mv_mma.metal + kernels/mul_mm.metal + kernels/quantize.metal + kernels/softmax.metal + kernels/norm.metal + kernels/unary.metal + kernels/binbcast.metal + kernels/reduce.metal + kernels/tri.metal + kernels/ssm.metal + kernels/wkv.metal + kernels/gated_delta_net.metal + kernels/solve_tri.metal + kernels/rope.metal + kernels/conv.metal + kernels/upscale.metal + kernels/argsort.metal + kernels/pool.metal + kernels/misc.metal +) + if (GGML_METAL_EMBED_LIBRARY) enable_language(ASM) add_compile_definitions(GGML_METAL_EMBED_LIBRARY) - set(METALLIB_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal") - set(METALLIB_IMPL "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal-impl.h") + set(METALLIB_IMPL "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal-impl.h") file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/autogenerated") - # merge ggml-common.h and ggml-metal.metal into a single file - set(METALLIB_EMBED_ASM "${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed.s") - set(METALLIB_SOURCE_EMBED "${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed.metal") - set(METALLIB_SOURCE_EMBED_TMP "${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed.metal.tmp") + set(METALLIB_EMBED_ASM_FILES "") + foreach(src ${METALLIB_KERNEL_SOURCES}) + get_filename_component(kind ${src} NAME_WE) + # symbol names must be valid C identifiers ('-' is not allowed) + string(REPLACE "-" "_" kind_sym ${kind}) - add_custom_command( - OUTPUT "${METALLIB_EMBED_ASM}" - COMMAND echo "Embedding Metal library" - COMMAND sed -e "/__embed_ggml-common.h__/r ${METALLIB_COMMON}" -e "/__embed_ggml-common.h__/d" < "${METALLIB_SOURCE}" > "${METALLIB_SOURCE_EMBED_TMP}" - COMMAND sed -e "/\#include \"ggml-metal-impl.h\"/r ${METALLIB_IMPL}" -e "/\#include \"ggml-metal-impl.h\"/d" < "${METALLIB_SOURCE_EMBED_TMP}" > "${METALLIB_SOURCE_EMBED}" - COMMAND echo ".section __DATA,__ggml_metallib" > "${METALLIB_EMBED_ASM}" - COMMAND echo ".globl _ggml_metallib_start" >> "${METALLIB_EMBED_ASM}" - COMMAND echo "_ggml_metallib_start:" >> "${METALLIB_EMBED_ASM}" - COMMAND echo .incbin "\"${METALLIB_SOURCE_EMBED}\"" >> "${METALLIB_EMBED_ASM}" - COMMAND echo ".globl _ggml_metallib_end" >> "${METALLIB_EMBED_ASM}" - COMMAND echo "_ggml_metallib_end:" >> "${METALLIB_EMBED_ASM}" - DEPENDS ../ggml-common.h ggml-metal.metal ggml-metal-impl.h - COMMENT "Generate assembly for embedded Metal library" - VERBATIM - ) + set(SRC "${CMAKE_CURRENT_SOURCE_DIR}/kernels/${kind}.metal") + set(EMBED "${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed-${kind}.metal") + set(ASM "${CMAKE_CURRENT_BINARY_DIR}/autogenerated/ggml-metal-embed-${kind}.s") + + # only prepend headers that this source actually includes + set(HEADERS_FOR_SRC ${METALLIB_KERNELS_COMMON}) + file(STRINGS ${SRC} _has_dequantize REGEX "#include \"dequantize\\.h\"") + file(STRINGS ${SRC} _has_quantize REGEX "#include \"quantize\\.h\"") + if(_has_dequantize) + list(APPEND HEADERS_FOR_SRC ${METALLIB_KERNELS_DEQUANTIZE}) + endif() + if(_has_quantize) + list(APPEND HEADERS_FOR_SRC ${METALLIB_KERNELS_QUANTIZE}) + endif() + file(STRINGS ${SRC} _has_fa_common REGEX "#include \"fa_common\\.metal\"") + file(STRINGS ${SRC} _has_fa_vec_common REGEX "#include \"fa_vec_common\\.metal\"") + if(_has_fa_common) + list(APPEND HEADERS_FOR_SRC ${METALLIB_KERNELS_FA_COMMON}) + endif() + if(_has_fa_vec_common) + list(APPEND HEADERS_FOR_SRC ${METALLIB_KERNELS_FA_VEC_COMMON}) + endif() - target_sources(ggml-metal PRIVATE "${METALLIB_EMBED_ASM}") + add_custom_command( + OUTPUT "${ASM}" + # Step 1: concatenate shared headers + this kernel source + COMMAND cat ${HEADERS_FOR_SRC} ${SRC} > "${EMBED}.tmp1" + # Step 2: remove internal #include and #pragma once + COMMAND sed -e "/\#include \"common.h\"/d" -e "/\#include \"dequantize.h\"/d" -e "/\#include \"quantize.h\"/d" -e "/\#include \"fa_common.metal\"/d" -e "/\#include \"fa_vec_common.metal\"/d" -e "/\#pragma once/d" < "${EMBED}.tmp1" > "${EMBED}.tmp2" + # Step 3: inline ggml-common.h (replacing __embed_ggml-common.h__ sentinel) + COMMAND sed -e "/__embed_ggml-common.h__/r ${METALLIB_COMMON}" -e "/__embed_ggml-common.h__/d" < "${EMBED}.tmp2" > "${EMBED}.tmp3" + # Step 4: inline ggml-metal-impl.h + COMMAND sed -e "/\#include \"ggml-metal-impl.h\"/r ${METALLIB_IMPL}" -e "/\#include \"ggml-metal-impl.h\"/d" < "${EMBED}.tmp3" > "${EMBED}" + # Step 5: emit an asm chunk with kind-specific start/end symbols + # note: '-' is illegal in C symbols, so we use kind_sym; the macOS + # section name is limited to 16 chars so we keep it shared + # across kinds (__ggml_metallib) and only vary the global symbols. + COMMAND echo ".section __DATA,__ggml_metallib" > "${ASM}" + COMMAND echo ".globl _ggml_metallib_${kind_sym}_start" >> "${ASM}" + COMMAND echo "_ggml_metallib_${kind_sym}_start:" >> "${ASM}" + COMMAND echo .incbin "\"${EMBED}\"" >> "${ASM}" + COMMAND echo ".globl _ggml_metallib_${kind_sym}_end" >> "${ASM}" + COMMAND echo "_ggml_metallib_${kind_sym}_end:" >> "${ASM}" + DEPENDS ${HEADERS_FOR_SRC} ../ggml-common.h ggml-metal-impl.h + kernels/${kind}.metal + COMMENT "Generate embedded Metal library for ${kind}" + VERBATIM + ) + + list(APPEND METALLIB_EMBED_ASM_FILES "${ASM}") + endforeach() + + target_sources(ggml-metal PRIVATE ${METALLIB_EMBED_ASM_FILES}) else() - # copy metal files to bin directory + # copy header files to bin directory configure_file(../ggml-common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COPYONLY) - configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY) configure_file(ggml-metal-impl.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h COPYONLY) + file(MAKE_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels") + configure_file(kernels/common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/common.h COPYONLY) + configure_file(kernels/dequantize.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/dequantize.h COPYONLY) + configure_file(kernels/quantize.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/quantize.h COPYONLY) + foreach(hdr ${METALLIB_KERNELS_FA_SHARED}) + get_filename_component(hdr_name ${hdr} NAME) + configure_file(${hdr} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/${hdr_name} COPYONLY) + endforeach() + + foreach(src ${METALLIB_KERNEL_SOURCES}) + configure_file(${src} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${src} COPYONLY) + endforeach() + + # CMAKE_OSX_SYSROOT is an SDK name or path - xcrun accepts both + set(METAL_SDK ${CMAKE_OSX_SYSROOT}) + if (NOT METAL_SDK) + set(METAL_SDK macosx) + endif() + + if (CMAKE_OSX_SYSROOT MATCHES "[Ss]imulator") + set(METAL_TARGET_SIM "-simulator") + else() + set(METAL_TARGET_SIM "") + endif() + if (GGML_METAL_SHADER_DEBUG) - # custom command to do the following: - # xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air - # xcrun -sdk macosx metallib ggml-metal.air -o default.metallib - # - # note: this is the only way I found to disable fast-math in Metal. it's ugly, but at least it works - # disabling fast math is needed in order to pass tests/test-backend-ops + # note: disabling fast math is needed in order to pass tests/test-backend-ops # note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1 # note: unfortunately, we have to call it default.metallib instead of ggml.metallib # ref: https://github.com/ggml-org/whisper.cpp/issues/1720 # note: adding -g causes segmentation fault during compile - #set(XC_FLAGS -fno-fast-math -fno-inline -g) set(XC_FLAGS -fno-fast-math -fno-inline) else() set(XC_FLAGS -O3) endif() - # Append macOS metal versioning flags + execute_process(COMMAND xcrun -sdk ${METAL_SDK} --show-sdk-version OUTPUT_VARIABLE METAL_SDK_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) + if (METAL_SDK_VERSION VERSION_GREATER_EQUAL 26.0) + set(GGML_METAL_HAS_TENSOR_LIB ON) + else() + message(STATUS "Metal SDK ${METAL_SDK_VERSION} does not support the tensor API, skipping ggml-tensor.metallib") + endif() + if (GGML_METAL_MACOSX_VERSION_MIN) message(STATUS "Adding -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN} flag to metal compilation") list (APPEND XC_FLAGS -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN}) + elseif (NOT GGML_METAL_TARGET_OS STREQUAL "macos" AND CMAKE_OSX_DEPLOYMENT_TARGET) + message(STATUS "Adding -mtargetos=${GGML_METAL_TARGET_OS}${CMAKE_OSX_DEPLOYMENT_TARGET}${METAL_TARGET_SIM} flag to metal compilation") + list (APPEND XC_FLAGS -mtargetos=${GGML_METAL_TARGET_OS}${CMAKE_OSX_DEPLOYMENT_TARGET}${METAL_TARGET_SIM}) endif() if (GGML_METAL_STD) @@ -90,35 +204,80 @@ else() list (APPEND XC_FLAGS -std=${GGML_METAL_STD}) endif() + # Compile each kernel source to .air, then link into default.metallib + set(AIR_FILES "") + foreach(src ${METALLIB_KERNEL_SOURCES}) + get_filename_component(name ${src} NAME_WE) + set(AIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${name}.air") + list(APPEND AIR_FILES ${AIR}) + add_custom_command( + OUTPUT ${AIR} + COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS} -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${src} -o ${AIR} + DEPENDS ${src} ${METALLIB_KERNELS_FA_SHARED} kernels/common.h kernels/dequantize.h kernels/quantize.h ${METALLIB_COMMON} ggml-metal-impl.h + COMMENT "Compiling ${src}" + VERBATIM + ) + endforeach() + + set(METALLIB_FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib) + + # the tensor API kernels go in a separate metallib, loaded only where supported + if (GGML_METAL_HAS_TENSOR_LIB) + set(AIR_MM_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/mul_mm_tensor.air") + # the tensor API needs OS 26+ + set(XC_FLAGS_TENSOR ${XC_FLAGS} -mtargetos=${GGML_METAL_TARGET_OS}26.0${METAL_TARGET_SIM}) + add_custom_command( + OUTPUT ${AIR_MM_TENSOR} + COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/mul_mm.metal -o ${AIR_MM_TENSOR} + DEPENDS kernels/mul_mm.metal kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h + COMMENT "Compiling kernels/mul_mm.metal (tensor API)" + VERBATIM + ) + + set(AIR_FA_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/fa_tensor_tensor.air") + add_custom_command( + OUTPUT ${AIR_FA_TENSOR} + COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/fa_tensor.metal -o ${AIR_FA_TENSOR} + DEPENDS kernels/fa_tensor.metal kernels/common.h ${METALLIB_COMMON} ggml-metal-impl.h + COMMENT "Compiling kernels/fa_tensor.metal (tensor API)" + VERBATIM + ) + + add_custom_command( + OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib + COMMAND xcrun -sdk ${METAL_SDK} metallib ${AIR_MM_TENSOR} ${AIR_FA_TENSOR} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib + DEPENDS ${AIR_MM_TENSOR} ${AIR_FA_TENSOR} + COMMENT "Linking tensor API Metal kernels into ggml-tensor.metallib" + ) + + list(APPEND METALLIB_FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-tensor.metallib) + endif() + add_custom_command( OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib - COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o - | - xcrun -sdk macosx metallib - -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib + COMMAND xcrun -sdk ${METAL_SDK} metallib ${AIR_FILES} -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h - COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal - DEPENDS ggml-metal.metal ${METALLIB_COMMON} - COMMENT "Compiling Metal kernels" - ) + COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal-impl.h + COMMAND rm -rf ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels + DEPENDS ${AIR_FILES} ${AIR_MM_TENSOR} ${AIR_FA_TENSOR} + COMMENT "Linking Metal kernels into default.metallib" + ) - # FIXME: only add to the ggml-metal target? add_custom_target( ggml-metal-lib ALL - DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib - ) + DEPENDS ${METALLIB_FILES} + ) endif() # GGML_METAL_EMBED_LIBRARY if (NOT GGML_METAL_EMBED_LIBRARY) install( - FILES src/ggml-metal/ggml-metal.metal - PERMISSIONS - OWNER_READ - OWNER_WRITE - GROUP_READ - WORLD_READ - DESTINATION ${CMAKE_INSTALL_BINDIR}) - - install( - FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib - DESTINATION ${CMAKE_INSTALL_BINDIR} - ) + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/kernels/ + DESTINATION ${CMAKE_INSTALL_BINDIR}/kernels + FILES_MATCHING PATTERN "*.metal" PATTERN "*.h" + ) + + install( + FILES ${METALLIB_FILES} + DESTINATION ${CMAKE_INSTALL_BINDIR} + ) endif() diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cpp index 2eb9820..70fd551 100644 --- a/ggml/src/ggml-metal/ggml-metal-common.cpp +++ b/ggml/src/ggml-metal/ggml-metal-common.cpp @@ -1,10 +1,162 @@ #include "ggml-metal-common.h" +#include "ggml-metal-fusion.h" +#include "ggml.h" #include "ggml-impl.h" #include "ggml-backend-impl.h" #include +// must stay in sync with the kernel_fwht__ templates in misc.metal. Widths up to +// 512 run on the simdgroup kernel and need no threadgroup memory. The wider ones allocate +// float[N] per threadgroup, so they are only available where that fits. +static bool ggml_metal_fwht_supported_size(int64_t n, size_t max_tg_mem) { + if (n == 64 || n == 128 || n == 256 || n == 512) { + return true; + } + + if (n == 1024 || n == 2048 || n == 4096 || n == 8192) { + return (size_t) n * sizeof(float) <= max_tg_mem; + } + + return false; +} + +// the FWHT kernels handle a Hadamard-hinted MUL_MAT only under these conditions. supports_op +// and the dispatch must ask the same question: an F16 src1 that is admitted but then falls +// through reaches the generic path, which has no F32 src0 by F16 src1 kernel. +bool ggml_metal_op_mul_mat_use_fwht(const struct ggml_tensor * op, size_t max_tg_mem) { + return ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD && op->type == GGML_TYPE_F32 && + (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && ggml_is_contiguous(op->src[1]) && + ggml_is_contiguous(op) && ggml_are_same_shape(op->src[1], op) && + ggml_metal_fwht_supported_size(op->src[1]->ne[0], max_tg_mem); +} + +bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) { + const int64_t ne00 = op->src[0]->ne[0]; + const int64_t ne11 = op->src[1]->ne[1]; + + return !ggml_is_transposed(op->src[0]) && + !ggml_is_transposed(op->src[1]) && + has_simdgroup_mm && ne00 >= 64 && ne11 > 8; +} + +bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) { + const int64_t ne00 = op->src[0]->ne[0]; + const int64_t ne02 = op->src[0]->ne[2]; + const int64_t ne21 = op->src[2]->ne[1]; + + return has_simdgroup_mm && ne00 >= 64 && ne02 <= 1024 && ne21 >= 32; +} + +// the most src1 rows of the few-row MMA kernels +static constexpr int64_t GGML_METAL_MMA_ROWS_MAX = 16; + +// src1 rows per 8x8 simdgroup matrix tile of the few-row MMA kernels +static constexpr int64_t GGML_METAL_MMA_TILE_ROWS = 8; + +// weights per K step of the q5_K and generic few-row MMA kernels +static constexpr int64_t GGML_METAL_MMA_K_CHUNK = 64; + +enum ggml_metal_mma_kind ggml_metal_mul_mv_mma_kind(enum ggml_type type, int rt) { + if (type == GGML_TYPE_Q4_0 || (type == GGML_TYPE_Q8_0 && rt == 1)) { + return GGML_METAL_MMA_KIND_BLK; + } + return type == GGML_TYPE_Q5_K ? GGML_METAL_MMA_KIND_Q5_K : GGML_METAL_MMA_KIND_GEN; +} + +int ggml_metal_mul_mv_mma_rt(const struct ggml_tensor * op) { + return op->src[1]->ne[1] > GGML_METAL_MMA_TILE_ROWS ? 2 : 1; +} + +static bool ggml_metal_mul_mv_mma_type_supported(enum ggml_type type) { + switch (type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_BF16: + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_TQ2_0: + return true; + default: + return false; + } +} + +int64_t ggml_metal_mul_mv_mma_k_step(enum ggml_type type, int rt) { + if (!ggml_metal_mul_mv_mma_type_supported(type)) { + return 0; + } + return ggml_metal_mul_mv_mma_kind(type, rt) == GGML_METAL_MMA_KIND_BLK ? ggml_blck_size(type) : GGML_METAL_MMA_K_CHUNK; +} + +static bool ggml_metal_mul_mat_mma_type_ok(const struct ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + const int64_t step = ggml_metal_mul_mv_mma_k_step(src0->type, ggml_metal_mul_mv_mma_rt(op)); + + return step > 0 && src0->ne[0] % step == 0 && src0->nb[0] == ggml_type_size(src0->type); +} + +// the fewest src1 rows at which the few-row MMA kernels beat the mat-vec kernels (measured on an M3 Ultra) +static int64_t ggml_metal_mul_mv_mma_rows_min(enum ggml_type type) { + switch (type) { + case GGML_TYPE_F32: + return 6; + case GGML_TYPE_TQ2_0: + return 5; + case GGML_TYPE_BF16: + return 4; + case GGML_TYPE_F16: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q2_K: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + return 3; + default: + return 2; + } +} + +bool ggml_metal_op_mul_mat_use_mma(const struct ggml_tensor * op) { + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + + // the batch shape goes into int16 function constants + const bool batch_ok = src1->ne[2] <= INT16_MAX && src1->ne[2]/src0->ne[2] <= INT16_MAX && src1->ne[3]/src0->ne[3] <= INT16_MAX; + + return ggml_metal_mul_mat_mma_type_ok(op) && batch_ok && + src1->type == GGML_TYPE_F32 && src1->ne[1] >= ggml_metal_mul_mv_mma_rows_min(src0->type) && src1->ne[1] <= GGML_METAL_MMA_ROWS_MAX && + !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && + src1->nb[0] == sizeof(float) && src1->nb[1] % 16 == 0 && src1->nb[2] % 16 == 0 && src1->nb[3] % 16 == 0; +} + +bool ggml_metal_op_mul_mat_may_use_mma(const struct ggml_tensor * op) { + return ggml_metal_mul_mv_mma_type_supported(op->src[0]->type) && op->src[1]->type == GGML_TYPE_F32; +} + // represents a memory range (i.e. an interval from a starting address p0 to an ending address p1 in a given buffer pb) // the type indicates whether it is a source range (i.e. ops read data from it) or a destination range (i.e. ops write data to it) struct ggml_mem_range { @@ -204,38 +356,63 @@ struct node_info { void add_fused(ggml_tensor * t) { fused.push_back(t); } + + bool is_output(const ggml_tensor * t) const { + if (t == node) { + return true; + } + for (const auto * f : fused) { + if (t == f) { + return true; + } + } + return false; + } }; static std::vector ggml_metal_graph_optimize_reorder(const std::vector & nodes) { // helper to add node src and dst ranges const auto & h_add = [](ggml_mem_ranges_t mrs, const node_info & node) { + // only external sources matter: sources produced by the fused group are internal for (int i = 0; i < GGML_MAX_SRC; i++) { - if (node.node->src[i]) { - if (!ggml_mem_ranges_add_src(mrs, node.node->src[i])) { + const ggml_tensor * src = node.node->src[i]; + if (src && !node.is_output(src)) { + if (!ggml_mem_ranges_add_src(mrs, src)) { return false; } } } - // keep track of the sources of the fused nodes as well for (const auto * fused : node.fused) { for (int i = 0; i < GGML_MAX_SRC; i++) { - if (fused->src[i]) { - if (!ggml_mem_ranges_add_src(mrs, fused->src[i])) { + const ggml_tensor * src = fused->src[i]; + if (src && !node.is_output(src)) { + if (!ggml_mem_ranges_add_src(mrs, src)) { return false; } } } } - return ggml_mem_ranges_add_dst(mrs, node.dst()); + // all fused tensors are produced by the fused kernel + if (!ggml_mem_ranges_add_dst(mrs, node.node)) { + return false; + } + for (const auto * fused : node.fused) { + if (!ggml_mem_ranges_add_dst(mrs, fused)) { + return false; + } + } + + return true; }; // helper to check if a node can run concurrently with the existing set of nodes const auto & h_check = [](ggml_mem_ranges_t mrs, const node_info & node) { for (int i = 0; i < GGML_MAX_SRC; i++) { - if (node.node->src[i]) { - if (!ggml_mem_ranges_check_src(mrs, node.node->src[i])) { + const ggml_tensor * src = node.node->src[i]; + if (src && !node.is_output(src)) { + if (!ggml_mem_ranges_check_src(mrs, src)) { return false; } } @@ -243,15 +420,25 @@ static std::vector ggml_metal_graph_optimize_reorder(const std::vectorsrc[i]) { - if (!ggml_mem_ranges_check_src(mrs, fused->src[i])) { + const ggml_tensor * src = fused->src[i]; + if (src && !node.is_output(src)) { + if (!ggml_mem_ranges_check_src(mrs, src)) { return false; } } } } - return ggml_mem_ranges_check_dst(mrs, node.dst()); + if (!ggml_mem_ranges_check_dst(mrs, node.node)) { + return false; + } + for (const auto * fused : node.fused) { + if (!ggml_mem_ranges_check_dst(mrs, fused)) { + return false; + } + } + + return true; }; // perform reorders only across these types of ops @@ -373,59 +560,31 @@ static std::vector ggml_metal_graph_optimize_reorder(const std::vectorn_nodes; - enum ggml_op ops[MAX_FUSE]; - std::vector nodes; nodes.reserve(gf->n_nodes); // fuse nodes: // we don't want to make reorders that break fusing, so we first pack all fusable tensors // and perform the reorder over the fused nodes. after the reorder is done, we unfuse + // + // the fusable sequences are declared in the fusion table (ggml-metal-fuse.cpp), so the + // packing here is driven by the same patterns that the op encoders will later use for (int i = 0; i < n; i++) { node_info node = { /*.node =*/ gf->nodes[i], /*.fused =*/ {}, }; - // fuse only ops that start with these operations - // can be expanded when needed - if (node.op() == GGML_OP_ADD || - node.op() == GGML_OP_NORM || - node.op() == GGML_OP_RMS_NORM) { - ops[0] = node.op(); - - int f = i + 1; - while (f < n && f < i + MAX_FUSE) { - // conservatively allow fusing only these ops - // can be expanded when needed - if (gf->nodes[f]->op != GGML_OP_ADD && - gf->nodes[f]->op != GGML_OP_MUL && - gf->nodes[f]->op != GGML_OP_NORM && - gf->nodes[f]->op != GGML_OP_RMS_NORM) { - break; - } - ops[f - i] = gf->nodes[f]->op; - f++; - } - - f -= i; - for (; f > 1; f--) { - if (ggml_can_fuse(gf, i, ops, f)) { - break; - } - } + const int f = ggml_metal_fusion_max(gf, i); - // add the fused tensors into the node info so we can unfuse them later - for (int k = 1; k < f; k++) { - ++i; + // add the fused tensors into the node info so we can unfuse them later + for (int k = 1; k < f; k++) { + ++i; - // the .dst() becomes the last fused tensor - node.add_fused(gf->nodes[i]); - } + // the .dst() becomes the last fused tensor + node.add_fused(gf->nodes[i]); } nodes.push_back(std::move(node)); diff --git a/ggml/src/ggml-metal/ggml-metal-common.h b/ggml/src/ggml-metal/ggml-metal-common.h index 3acbc6a..c86d92b 100644 --- a/ggml/src/ggml-metal/ggml-metal-common.h +++ b/ggml/src/ggml-metal/ggml-metal-common.h @@ -2,7 +2,10 @@ #pragma once +#include "ggml.h" + #include +#include #ifdef __cplusplus extern "C" { @@ -41,12 +44,30 @@ bool ggml_mem_ranges_add(ggml_mem_ranges_t mrs, const struct ggml_tensor * tenso // - new dst range overlaps with any existing range (src or dst) bool ggml_mem_ranges_check(ggml_mem_ranges_t mrs, const struct ggml_tensor * tensor); -// reorder the nodes in the graph to improve concurrency, while respecting fusion +// reorder the nodes in the graph to improve concurrency, while respecting the fusions of a device with props // // note: this implementation is generic and not specific to metal // if it proves to work well, we can start using it for other backends in the future void ggml_graph_optimize(struct ggml_cgraph * gf); +// mat-mat vs mat-vec dispatch; used by both supports_op and ggml_metal_op_mul_mat* +bool ggml_metal_op_mul_mat_use_fwht (const struct ggml_tensor * op, size_t max_tg_mem); +bool ggml_metal_op_mul_mat_use_mm (const struct ggml_tensor * op, bool has_simdgroup_mm); +bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm); + +bool ggml_metal_op_mul_mat_use_mma (const struct ggml_tensor * op); +bool ggml_metal_op_mul_mat_may_use_mma(const struct ggml_tensor * op); // graph structure only + +// the few-row MMA kernel for a src0 type and rt src1 tiles: per 32-weight block (q4_0, q8_0 with one tile), q5_K, or the generic 64-weight chunk kernel +enum ggml_metal_mma_kind { GGML_METAL_MMA_KIND_BLK, GGML_METAL_MMA_KIND_Q5_K, GGML_METAL_MMA_KIND_GEN }; +enum ggml_metal_mma_kind ggml_metal_mul_mv_mma_kind(enum ggml_type type, int rt); + +// the src1 tiles of the few-row MMA kernels for mat-mul op: one 8-row tile, or two above 8 rows +int ggml_metal_mul_mv_mma_rt(const struct ggml_tensor * op); + +// the weights of K per simdgroup step of the few-row MMA kernel for a src0 type and rt src1 tiles, 0 if none takes the type +int64_t ggml_metal_mul_mv_mma_k_step(enum ggml_type type, int rt); + #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-metal/ggml-metal-context.h b/ggml/src/ggml-metal/ggml-metal-context.h index abf4b06..b538b1a 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.h +++ b/ggml/src/ggml-metal/ggml-metal-context.h @@ -33,6 +33,7 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx); void ggml_metal_set_n_cb (ggml_metal_t ctx, int n_cb); void ggml_metal_set_abort_callback (ggml_metal_t ctx, ggml_abort_callback abort_callback, void * user_data); + bool ggml_metal_supports_family (ggml_metal_t ctx, int family); void ggml_metal_capture_next_compute(ggml_metal_t ctx); diff --git a/ggml/src/ggml-metal/ggml-metal-context.m b/ggml/src/ggml-metal/ggml-metal-context.m index 32d97cd..597c126 100644 --- a/ggml/src/ggml-metal/ggml-metal-context.m +++ b/ggml/src/ggml-metal/ggml-metal-context.m @@ -6,6 +6,7 @@ #import "ggml-metal-impl.h" #import "ggml-metal-common.h" #import "ggml-metal-ops.h" +#import "ggml-metal-fusion.h" #import @@ -29,22 +30,20 @@ ggml_metal_device_t dev; ggml_metal_library_t lib; - ggml_metal_event_t ev_cpy; // for async copies + ggml_metal_event_t ev_cpy; // for async copies + ggml_metal_event_t ev_sync; // destination completion signal dispatch_queue_t d_queue; // additional, inference-time compiled pipelines ggml_metal_pipelines_t pipelines_ext; - bool use_fusion; bool use_concurrency; bool use_graph_optimize; int debug_graph; - int debug_fusion; - // how many times a given op was fused - uint64_t fuse_cnt[GGML_OP_COUNT]; + struct ggml_metal_fusion_info * finfo; // capture state int capture_compute; @@ -69,6 +68,10 @@ // extra command buffers for things like getting, setting and copying tensors NSMutableArray * cmd_bufs_ext; + // buffers to release after async Metal operations complete + // if Metal released them, it would do so on a Metal-internal thread without an autorelease pool, which could cause leaks + NSMutableArray * buf_refs; + // the last command buffer queued into the Metal queue with operations relevant to the current Metal backend id cmd_buf_last; @@ -84,106 +87,109 @@ ggml_metal_t ggml_metal_init(ggml_metal_device_t dev) { GGML_LOG_INFO("%s: allocating\n", __func__); + @autoreleasepool { #if TARGET_OS_OSX && !GGML_METAL_NDEBUG - // Show all the Metal device instances in the system - NSArray * devices = MTLCopyAllDevices(); - for (id device in devices) { - GGML_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]); - } - [devices release]; // since it was created by a *Copy* C method + // Show all the Metal device instances in the system + NSArray * devices = MTLCopyAllDevices(); + for (id device in devices) { + GGML_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]); + } + [devices release]; // since it was created by a *Copy* C method #endif - // init context - ggml_metal_t res = calloc(1, sizeof(struct ggml_metal)); + // init context + ggml_metal_t res = calloc(1, sizeof(struct ggml_metal)); - id device = ggml_metal_device_get_obj(dev); + id device = ggml_metal_device_get_obj(dev); - GGML_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]); + GGML_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]); - // TODO: would it be better to have one queue for the backend and one queue for the device? - // the graph encoders and async ops would use the backend queue while the sync ops would use the device queue? - //res->queue = [device newCommandQueue]; [TAG_QUEUE_PER_BACKEND] - id queue = ggml_metal_device_get_queue(dev); - if (queue == nil) { - GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__); - return NULL; - } - - res->dev = dev; - res->lib = ggml_metal_device_get_library(dev); - if (res->lib == NULL) { - GGML_LOG_WARN("%s: the device does not have a precompiled Metal library - this is unexpected\n", __func__); - GGML_LOG_WARN("%s: will try to compile it on the fly\n", __func__); + // TODO: would it be better to have one queue for the backend and one queue for the device? + // the graph encoders and async ops would use the backend queue while the sync ops would use the device queue? + //res->queue = [device newCommandQueue]; [TAG_QUEUE_PER_BACKEND] + id queue = ggml_metal_device_get_queue(dev); + if (queue == nil) { + GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__); + free(res); + return NULL; + } - res->lib = ggml_metal_library_init(dev); + res->dev = dev; + res->lib = ggml_metal_device_get_library(dev); if (res->lib == NULL) { - GGML_LOG_ERROR("%s: error: failed to initialize the Metal library\n", __func__); + GGML_LOG_WARN("%s: the device does not have a precompiled Metal library - this is unexpected\n", __func__); + GGML_LOG_WARN("%s: will try to compile it on the fly\n", __func__); - free(res); + res->lib = ggml_metal_library_init(dev); + if (res->lib == NULL) { + GGML_LOG_ERROR("%s: error: failed to initialize the Metal library\n", __func__); - return NULL; - } - } + free(res); - res->ev_cpy = ggml_metal_device_event_init(dev); + return NULL; + } + } - const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev); + res->ev_cpy = ggml_metal_device_event_init(dev); + res->ev_sync = ggml_metal_device_event_init(dev); - snprintf(res->name, sizeof(res->name), "%s", props_dev->name); + const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev); - res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); + snprintf(res->name, sizeof(res->name), "%s", props_dev->name); - res->use_fusion = getenv("GGML_METAL_FUSION_DISABLE") == nil; - res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil; + res->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT); - { - const char * val = getenv("GGML_METAL_GRAPH_DEBUG"); - res->debug_graph = val ? atoi(val) : 0; - } + res->use_concurrency = getenv("GGML_METAL_CONCURRENCY_DISABLE") == nil; - { - const char * val = getenv("GGML_METAL_FUSION_DEBUG"); - res->debug_fusion = val ? atoi(val) : 0; - } + { + const char * val = getenv("GGML_METAL_GRAPH_DEBUG"); + res->debug_graph = val ? atoi(val) : 0; + } - res->use_graph_optimize = true; + res->use_graph_optimize = true; - if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) { - res->use_graph_optimize = false; - } + if (getenv("GGML_METAL_GRAPH_OPTIMIZE_DISABLE") != NULL) { + res->use_graph_optimize = false; + } - memset(res->fuse_cnt, 0, sizeof(res->fuse_cnt)); + res->finfo = ggml_metal_device_get_fusion_info(dev); + if (ggml_metal_fusion_info_stats(res->finfo)) { + ggml_metal_fusion_info_labels_init(res->finfo); + res->n_cb = 0; + } - GGML_LOG_INFO("%s: use fusion = %s\n", __func__, res->use_fusion ? "true" : "false"); - GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false"); - GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false"); + GGML_LOG_INFO("%s: use fusion = %s\n", __func__, ggml_metal_fusion_info_enabled(res->finfo) ? "true" : "false"); + GGML_LOG_INFO("%s: use concurrency = %s\n", __func__, res->use_concurrency ? "true" : "false"); + GGML_LOG_INFO("%s: use graph optimize = %s\n", __func__, res->use_graph_optimize ? "true" : "false"); - res->capture_compute = 0; - res->capture_started = false; - res->capture_scope = nil; + res->capture_compute = 0; + res->capture_started = false; + res->capture_scope = nil; - { - const char * val = getenv("GGML_METAL_CAPTURE_COMPUTE"); - if (val) { - res->capture_compute = atoi(val); + { + const char * val = getenv("GGML_METAL_CAPTURE_COMPUTE"); + if (val) { + res->capture_compute = atoi(val); + } } - } - res->has_error = false; + res->has_error = false; - res->gf = nil; - res->encode_async = nil; - for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) { - res->cmd_bufs[i].obj = nil; - } + res->gf = nil; + res->encode_async = nil; + for (int i = 0; i < GGML_METAL_MAX_COMMAND_BUFFERS; ++i) { + res->cmd_bufs[i].obj = nil; + } - res->cmd_bufs_ext = [[NSMutableArray alloc] init]; + res->cmd_bufs_ext = [[NSMutableArray alloc] init]; + res->buf_refs = [[NSMutableArray alloc] init]; - res->cmd_buf_last = nil; + res->cmd_buf_last = nil; - res->pipelines_ext = ggml_metal_pipelines_init(); + res->pipelines_ext = ggml_metal_pipelines_init(); - return res; + return res; + } } void ggml_metal_free(ggml_metal_t ctx) { @@ -204,20 +210,28 @@ void ggml_metal_free(ggml_metal_t ctx) { [ctx->cmd_bufs_ext removeAllObjects]; [ctx->cmd_bufs_ext release]; + @autoreleasepool { + [ctx->buf_refs removeAllObjects]; + [ctx->buf_refs release]; + } + if (ctx->pipelines_ext) { ggml_metal_pipelines_free(ctx->pipelines_ext); ctx->pipelines_ext = nil; } - if (ctx->debug_fusion > 0) { + if (ggml_metal_fusion_info_debug(ctx->finfo) > 0) { GGML_LOG_DEBUG("%s: fusion stats:\n", __func__); - for (int i = 0; i < GGML_OP_COUNT; i++) { - if (ctx->fuse_cnt[i] == 0) { + + const int n_fusions = ggml_metal_fusion_info_n_fusions(ctx->finfo); + for (int i = 0; i < n_fusions; i++) { + const uint64_t count = ggml_metal_fusion_info_count(ctx->finfo, i); + if (count == 0) { continue; } // note: cannot use ggml_log here - GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_op_name((enum ggml_op) i), ctx->fuse_cnt[i]); + GGML_LOG_DEBUG("%s: - %s: %" PRIu64 "\n", __func__, ggml_metal_fusion_info_label(ctx->finfo, i), count); } } @@ -228,6 +242,7 @@ void ggml_metal_free(ggml_metal_t ctx) { dispatch_release(ctx->d_queue); ggml_metal_device_event_free(ctx->dev, ctx->ev_cpy); + ggml_metal_device_event_free(ctx->dev, ctx->ev_sync); free(ctx); } @@ -292,6 +307,10 @@ void ggml_metal_synchronize(ggml_metal_t ctx) { [ctx->cmd_bufs_ext removeAllObjects]; } + + @autoreleasepool { + [ctx->buf_refs removeAllObjects]; + } } static struct ggml_metal_buffer_id ggml_metal_get_buffer_id(const struct ggml_tensor * t) { @@ -335,6 +354,8 @@ void ggml_metal_set_tensor_async(ggml_metal_t ctx, struct ggml_tensor * tensor, [encoder endEncoding]; [cmd_buf commit]; + + [ctx->buf_refs addObject:buf_src]; [buf_src release]; // do not wait here for completion @@ -379,6 +400,8 @@ void ggml_metal_get_tensor_async(ggml_metal_t ctx, const struct ggml_tensor * te [encoder endEncoding]; [cmd_buf commit]; + + [ctx->buf_refs addObject:buf_dst]; [buf_dst release]; // do not wait here for completion @@ -401,10 +424,23 @@ bool ggml_metal_cpy_tensor_async(ggml_metal_t ctx_src, ggml_metal_t ctx_dst, con return false; } + id dst_queue = ggml_metal_device_get_queue(ctx_dst->dev); + id sync_cmd_buf = [dst_queue commandBuffer]; + + ggml_metal_event_encode_signal(ctx_dst->ev_sync, sync_cmd_buf); + + [sync_cmd_buf commit]; + + [ctx_dst->cmd_bufs_ext addObject:sync_cmd_buf]; + ctx_dst->cmd_buf_last = sync_cmd_buf; + + [sync_cmd_buf retain]; + // queue the copy operation into the Metal context // this will be queued at the end, after any currently ongoing GPU operations id queue = ggml_metal_device_get_queue(ctx_src->dev); id cmd_buf = [queue commandBuffer]; + ggml_metal_event_encode_wait(ctx_dst->ev_sync, cmd_buf); id encoder = [cmd_buf blitCommandEncoder]; [encoder copyFromBuffer:bid_src.metal @@ -441,6 +477,10 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * return GGML_STATUS_FAILED; } + if (gf->n_nodes == 0) { + return GGML_STATUS_SUCCESS; + } + // number of nodes encoded by the main thread (empirically determined) const int n_main = MAX(64, 0.1*gf->n_nodes); @@ -460,10 +500,17 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * @autoreleasepool { ctx->gf = gf; - ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); - ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; + if (ctx->n_cb == 0) { + // single-threaded encoding: the whole graph is encoded by one command buffer + ctx->n_nodes_0 = gf->n_nodes; + ctx->n_nodes_1 = 0; + ctx->n_nodes_per_cb = 0; + } else { + ctx->n_nodes_0 = MIN(n_main, gf->n_nodes); + ctx->n_nodes_1 = gf->n_nodes - ctx->n_nodes_0; - ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + ctx->n_nodes_per_cb = (ctx->n_nodes_1 + ctx->n_cb - 1) / ctx->n_cb; + } if (ctx->capture_compute >= 0) { ctx->capture_compute--; @@ -471,8 +518,6 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * const bool use_capture = ctx->capture_compute == 0; if (use_capture) { - ctx->capture_compute = -1; - // make sure all previous computations have finished before starting the capture if (ctx->cmd_buf_last) { [ctx->cmd_buf_last waitUntilCompleted]; @@ -494,8 +539,14 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * descriptor.outputURL = [NSURL fileURLWithPath:path]; NSError * error = nil; - if (![[MTLCaptureManager sharedCaptureManager] startCaptureWithDescriptor:descriptor error:&error]) { - GGML_LOG_ERROR("%s: error: unable to start capture '%s'\n", __func__, [[error localizedDescription] UTF8String]); + const bool ok = [[MTLCaptureManager sharedCaptureManager] startCaptureWithDescriptor:descriptor error:&error]; + + [descriptor release]; + + if (!ok) { + GGML_LOG_ERROR("%s: error: unable to start capture '%s' (did you set METAL_CAPTURE_ENABLED=1 ?)\n", __func__, [[error localizedDescription] UTF8String]); + [ctx->capture_scope release]; + ctx->capture_scope = nil; } else { [ctx->capture_scope beginScope]; ctx->capture_started = true; @@ -555,6 +606,8 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * // enter here only when capturing in order to wait for all computation to finish // otherwise, we leave the graph to compute asynchronously if (use_capture && ctx->capture_started) { + enum ggml_status res = GGML_STATUS_SUCCESS; + // wait for completion and check status of each command buffer // needed to detect if the device ran out-of-memory for example (#1881) { @@ -568,11 +621,11 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * GGML_LOG_INFO("error: %s\n", [[cmd_buf error].localizedDescription UTF8String]); } - return GGML_STATUS_FAILED; + res = GGML_STATUS_FAILED; } } - for (int i = 0; i < n_cb; ++i) { + for (int i = 0; i < n_cb && res == GGML_STATUS_SUCCESS; ++i) { id cmd_buf = ctx->cmd_bufs[i].obj; [cmd_buf waitUntilCompleted]; @@ -582,8 +635,8 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * if (status == MTLCommandBufferStatusError) { GGML_LOG_INFO("error: %s\n", [[cmd_buf error].localizedDescription UTF8String]); } - - return GGML_STATUS_FAILED; + res = GGML_STATUS_FAILED; + break; } id next_buffer = (i + 1 < n_cb ? ctx->cmd_bufs[i + 1].obj : nil); @@ -598,7 +651,8 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * if (ctx->abort_callback && ctx->abort_callback(ctx->abort_callback_data)) { GGML_LOG_INFO("%s: command buffer %d aborted", __func__, i); - return GGML_STATUS_ABORTED; + res = GGML_STATUS_ABORTED; + break; } [next_buffer commit]; @@ -607,7 +661,14 @@ enum ggml_status ggml_metal_graph_compute(ggml_metal_t ctx, struct ggml_cgraph * [ctx->capture_scope endScope]; [[MTLCaptureManager sharedCaptureManager] stopCapture]; + [ctx->capture_scope release]; + ctx->capture_scope = nil; + ctx->capture_started = false; + + if (res != GGML_STATUS_SUCCESS) { + return res; + } } } @@ -661,6 +722,12 @@ ggml_metal_event_t ggml_metal_get_ev_cpy(ggml_metal_t ctx) { } void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { + // when fusion stats are collected the graph must be encoded by a single thread so the + // counters are race-free; override whatever the caller requested + if (ggml_metal_fusion_info_stats(ctx->finfo)) { + n_cb = 0; + } + if (ctx->n_cb != n_cb) { ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_COMMAND_BUFFERS); @@ -696,13 +763,12 @@ void ggml_metal_set_n_cb(ggml_metal_t ctx, int n_cb) { ctx->dev, cmd_buf, ctx->gf, + ctx->finfo, idx_start, idx_end, - ctx->use_fusion, ctx->use_concurrency, - ctx->capture_compute, - ctx->debug_graph, - ctx->debug_fusion); + ctx->capture_compute == 0, + ctx->debug_graph); for (int idx = 0; idx < ggml_metal_op_n_nodes(ctx_op); ++idx) { const int res = ggml_metal_op_encode(ctx_op, idx); diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 07d016d..f3f5d4d 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -1,6 +1,8 @@ #include "ggml-metal-device.h" +#include "ggml-metal-common.h" #include "ggml-metal-impl.h" +#include "ggml-metal-tuning.h" #include "ggml-impl.h" @@ -17,10 +19,10 @@ struct ggml_metal_device_deleter { typedef std::unique_ptr ggml_metal_device_ptr; -ggml_metal_device_t ggml_metal_device_get(int device) { +ggml_metal_device_t ggml_metal_device_get(int device, int n_devices) { static std::vector devs; - devs.emplace_back(ggml_metal_device_init(device)); + devs.emplace_back(ggml_metal_device_init(device, n_devices)); return devs.back().get(); } @@ -66,7 +68,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_base(ggml const char * op_str = "undefined"; switch (op) { case GGML_OP_ADD_ID: op_str = "add_id"; break; - case GGML_OP_CONCAT: op_str = "concat"; break; default: GGML_ABORT("fatal error"); }; @@ -161,11 +162,15 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_get_rows(ggml_me return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows(ggml_metal_library_t lib, ggml_type tidx, ggml_type tdst) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows(ggml_metal_library_t lib, const ggml_tensor * op) { char base[256]; char name[256]; - snprintf(base, 256, "kernel_set_rows_%s_%s", ggml_type_name(tdst), ggml_type_name(tidx)); + const auto tsrc = op->src[0]->type; + const auto tidx = op->src[1]->type; + const auto tdst = op->type; + + snprintf(base, 256, "kernel_set_rows_%s_%s_%s", ggml_type_name(tsrc), ggml_type_name(tidx), ggml_type_name(tdst)); snprintf(name, 256, "%s", base); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); @@ -211,6 +216,21 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat(ggml_meta return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat(ggml_metal_library_t lib, ggml_type tsrc) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_concat_%s", ggml_type_name(tsrc)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary(ggml_metal_library_t lib, const ggml_tensor * op) { char base[256]; char name[256]; @@ -299,6 +319,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu(ggml_metal_l case GGML_GLU_OP_SWIGLU_OAI: op_str = "swiglu_oai"; break; case GGML_GLU_OP_GEGLU_ERF: op_str = "geglu_erf"; break; case GGML_GLU_OP_GEGLU_QUICK: op_str = "geglu_quick"; break; + case GGML_GLU_OP_SWIGLU_CLAMP: op_str = "swiglu_clamp"; break; default: GGML_ABORT("fatal error"); } break; default: GGML_ABORT("fatal error"); @@ -459,7 +480,75 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max(ggml_me return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv(ggml_metal_library_t lib, const ggml_tensor * op) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer( + ggml_metal_library_t lib, + const ggml_tensor * op) { + GGML_ASSERT(op->op == GGML_OP_LIGHTNING_INDEXER); + + char base[256]; + char name[256]; + + const int16_t nh = op->src[0]->ne[1]; + + snprintf(base, 256, "kernel_lightning_indexer_%s", ggml_type_name(op->src[1]->type)); + snprintf(name, 256, "%s_nh=%d", base, nh); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + + ggml_metal_cv_set_int16(cv, nh, FC_LIGHTNING_INDEXER + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc(ggml_metal_library_t lib, const ggml_tensor * op) { + char name[256]; + const char * base = nullptr; + + switch (op->op) { + case GGML_OP_DSV4_HC_COMB: + base = "kernel_dsv4_hc_comb_f32"; + snprintf(name, 256, "%s", base); + break; + case GGML_OP_DSV4_HC_PRE: + base = ggml_get_op_params_i32(op, 1) != 0 ? "kernel_dsv4_hc_pre_gated_f32" : "kernel_dsv4_hc_pre_f32"; + snprintf(name, 256, "%s_n_hc=%d", base, (int) op->src[0]->ne[1]); + break; + case GGML_OP_DSV4_HC_POST: + base = op->src[3] ? "kernel_dsv4_hc_post_f32" : "kernel_dsv4_hc_post_nocomb_f32"; + snprintf(name, 256, "%s", base); + break; + default: + GGML_ABORT("fatal error"); + } + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = nullptr; + + if (op->op == GGML_OP_DSV4_HC_PRE) { + cv = ggml_metal_cv_init(); + ggml_metal_cv_set_int32(cv, (int32_t) op->src[0]->ne[1], FC_DSV4_HC + 0); + } + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + if (cv) { + ggml_metal_cv_free(cv); + } + } + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv( + ggml_metal_library_t lib, const ggml_tensor * op, int32_t nc, bool use_silu) { GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); @@ -476,17 +565,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv(ggml_me } snprintf(base, 256, "kernel_ssm_conv_%s_%s%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type), suffix); - snprintf(name, 256, "%s", base); + snprintf(name, 256, "%s_nc=%d_silu=%d", base, nc, use_silu ? 1 : 0); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool(cv, use_silu, FC_SSM_CONV + 1); + ggml_metal_cv_set_int32(cv, nc, FC_SSM_CONV + 2); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); } return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched(ggml_metal_library_t lib, const ggml_tensor * op, int ssm_conv_bs) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched( + ggml_metal_library_t lib, const ggml_tensor * op, int ssm_conv_bs, int32_t nc, bool use_silu) { GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); @@ -502,13 +598,15 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched } snprintf(base, 256, "kernel_ssm_conv_%s_%s_batched%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type), suffix); - snprintf(name, 256, "%s_ssm_conv_bs=%d", base, ssm_conv_bs); + snprintf(name, 256, "%s_ssm_conv_bs=%d_nc=%d_silu=%d", base, ssm_conv_bs, nc, use_silu ? 1 : 0); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { ggml_metal_cv_t cv = ggml_metal_cv_init(); ggml_metal_cv_set_int16(cv, ssm_conv_bs, FC_SSM_CONV + 0); + ggml_metal_cv_set_bool(cv, use_silu, FC_SSM_CONV + 1); + ggml_metal_cv_set_int32(cv, nc, FC_SSM_CONV + 2); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -518,7 +616,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan(ggml_metal_library_t lib, const ggml_tensor * op) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan(ggml_metal_library_t lib, const ggml_tensor * op, bool tail) { GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); char base[256]; @@ -526,7 +624,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan(ggml_me const int nsg = (ne00 + 31)/32; - snprintf(base, 256, "kernel_ssm_scan_%s", ggml_type_name(op->src[0]->type)); + snprintf(base, 256, "kernel_ssm_scan_%s%s", ggml_type_name(op->src[0]->type), tail ? "_tail" : ""); snprintf(name, 256, "%s_nsg=%d", base, nsg); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); @@ -539,7 +637,28 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan(ggml_me // - sgptg floats for shared_x_dt (nsg) // - sgptg floats for shared_dA (nsg) // Total: nsg * (32 + 2) floats - res.smem = (32 + 2)*sizeof(float)*nsg; + res.smem = GGML_PAD((32 + 2)*sizeof(float)*nsg, 16); + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan_ssd_mma(ggml_metal_library_t lib, const ggml_tensor * op) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_ssm_scan_ssd_mma_%s", ggml_type_name(op->src[0]->type)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + // acs/exp(acs)/state-decay vectors + dtX + SAM rows + two 8x8 tiles per simdgroup + res.smem = (3*OP_SSM_SCAN_SSD_CS + + OP_SSM_SCAN_SSD_CS*OP_SSM_SCAN_SSD_HD + + OP_SSM_SCAN_SSD_NSG*8*OP_SSM_SCAN_SSD_CS + + OP_SSM_SCAN_SSD_NSG*2*8*8)*sizeof(float); return res; } @@ -590,6 +709,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_gated_delta_net( const int ne20 = op->src[2]->ne[0]; // S_v const int ne21 = op->src[2]->ne[1]; // H const int ne30 = op->src[3]->ne[0]; // G + // state is src[5], 4D [S_v, S_v, H_v, n_seqs] (s0 only); K is op param 0. + const int K = ggml_get_op_params_i32(op, 0); const int nsg = op->src[2]->ne[0]/32; @@ -598,7 +719,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_gated_delta_net( GGML_ASSERT(ne20 % 32 == 0); snprintf(base, 256, "kernel_gated_delta_net_%s_%d", ggml_type_name(op->src[0]->type), nsg); - snprintf(name, 256, "%s_ne20=%d_ne30=%d", base, ne20, ne30); + snprintf(name, 256, "%s_ne20=%d_ne30=%d_K=%d", base, ne20, ne30, K); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { @@ -606,6 +727,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_gated_delta_net( ggml_metal_cv_set_int16(cv, ne20, FC_GATED_DELTA_NET + 0); ggml_metal_cv_set_int16(cv, ne30, FC_GATED_DELTA_NET + 1); + ggml_metal_cv_set_int16(cv, K, FC_GATED_DELTA_NET + 2); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -647,19 +769,30 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_solve_tri(ggml_m return res; } -ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext(ggml_metal_library_t lib, ggml_type tsrc0, ggml_type tsrc1, int nsg, int nxpsg, int r1ptg) { +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext(ggml_metal_library_t lib, const ggml_tensor * op, int nsg, int nxpsg, int r1ptg) { char base[256]; char name[256]; + const ggml_type tsrc0 = op->src[0]->type; + const ggml_type tsrc1 = op->src[1]->type; + const int ne12 = op->src[1]->ne[2]; + const int r2 = ne12 / op->src[0]->ne[2]; + const int r3 = op->src[1]->ne[3] / op->src[0]->ne[3]; + + GGML_ASSERT(ne12 <= INT16_MAX && r2 <= INT16_MAX && r3 <= INT16_MAX); + snprintf(base, 256, "kernel_mul_mv_ext_%s_%s_r1_%d", ggml_type_name(tsrc0), ggml_type_name(tsrc1), r1ptg); - snprintf(name, 256, "%s_nsg=%d_nxpsg=%d", base, nsg, nxpsg); + snprintf(name, 256, "%s_nsg=%d_nxpsg=%d_ne12=%d_r2=%d_r3=%d", base, nsg, nxpsg, ne12, r2, r3); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { ggml_metal_cv_t cv = ggml_metal_cv_init(); - ggml_metal_cv_set_int16(cv, nsg, FC_MUL_MV + 0); - ggml_metal_cv_set_int16(cv, nxpsg, FC_MUL_MV + 1); + ggml_metal_cv_set_int16(cv, nsg, FC_MUL_MV + 0); + ggml_metal_cv_set_int16(cv, nxpsg, FC_MUL_MV + 1); + ggml_metal_cv_set_int16(cv, (int16_t) ne12, FC_MUL_MV + 2); + ggml_metal_cv_set_int16(cv, (int16_t) r2, FC_MUL_MV + 3); + ggml_metal_cv_set_int16(cv, (int16_t) r3, FC_MUL_MV + 4); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -669,6 +802,125 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext(ggml_ return res; } +// threadgroups needed to fill the GPU +// TODO: dedup +static constexpr int64_t GGML_METAL_MIN_THREADGROUPS = 128; + +// few-row MMA mat-mul: 8x8 simdgroup matrices for the src1 rows of ggml_metal_mul_mat_use_mma +static constexpr int GGML_METAL_MMA_TILE = 8; +static constexpr int GGML_METAL_MMA_NT_MAX = 4; +static constexpr int GGML_METAL_MMA_SMEM_MAX = 16384; + +// simdgroups per threadgroup for at most FEW_ROWS, at most MANY_ROWS, and more src0 rows +static constexpr int64_t GGML_METAL_MMA_FEW_ROWS = 64; +static constexpr int64_t GGML_METAL_MMA_MANY_ROWS = 6144; +static constexpr int GGML_METAL_MMA_NSG_FEW_ROWS = 32; +static constexpr int GGML_METAL_MMA_NSG_MID_ROWS = 16; +static constexpr int GGML_METAL_MMA_NSG_MANY_ROWS = 8; + +struct ggml_metal_mma_tiling { + int nsg; // simdgroups per threadgroup, each over a slice of K + int nt; // 8-row src0 tiles per threadgroup + int rt; // 8-row src1 tiles per threadgroup +}; + +// halves n while it is above limit +static int ggml_metal_halve_to_limit(int n, int64_t limit) { + while (n > 1 && n > limit) { + n /= 2; + } + return n; +} + +static size_t ggml_metal_mul_mv_mma_smem(int nsg, int nt, int rt) { + // one 8x8 float simdgroup matrix per output tile + constexpr size_t tile_bytes = 8*8*sizeof(float); + return (size_t) nsg*nt*rt*tile_bytes; +} + +// fewer src0 rows need more simdgroups per threadgroup (a finer K split) to fill the GPU. +// the tiles become narrower when the K-slice reduction buffer is too large or when too few threadgroups fill the GPU. +static ggml_metal_mma_tiling ggml_metal_op_mul_mat_mma_tiling(const ggml_tensor * op) { + const ggml_type type = op->src[0]->type; + const int64_t ne00 = op->src[0]->ne[0]; + const int64_t ne01 = op->src[0]->ne[1]; + + ggml_metal_mma_tiling res; + res.rt = ggml_metal_mul_mv_mma_rt(op); + + const int64_t n_steps = ne00/ggml_metal_mul_mv_mma_k_step(type, res.rt); + + int nsg = GGML_METAL_MMA_NSG_MANY_ROWS; + if (ne01 <= GGML_METAL_MMA_FEW_ROWS) { + nsg = GGML_METAL_MMA_NSG_FEW_ROWS; + } else if (ne01 <= GGML_METAL_MMA_MANY_ROWS) { + nsg = GGML_METAL_MMA_NSG_MID_ROWS; + } + res.nsg = ggml_metal_halve_to_limit(nsg, n_steps); + + const int64_t nt_smem = GGML_METAL_MMA_SMEM_MAX/(int64_t) ggml_metal_mul_mv_mma_smem(res.nsg, 1, res.rt); + const int64_t nt_rows = ne01/(GGML_METAL_MIN_THREADGROUPS*GGML_METAL_MMA_TILE); + res.nt = ggml_metal_halve_to_limit(GGML_METAL_MMA_NT_MAX, std::min(nt_smem, nt_rows)); + + return res; +} + +// the pipeline for the tiling, with fewer simdgroups when the device cannot run that many threads +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_mma_auto(ggml_metal_library_t lib, const ggml_tensor * op, bool add) { + ggml_metal_mma_tiling tiling = ggml_metal_op_mul_mat_mma_tiling(op); + + auto pipeline = ggml_metal_library_get_pipeline_mul_mv_mma(lib, op, tiling.nsg, tiling.nt, tiling.rt, add); + while (tiling.nsg > 1 && ggml_metal_pipeline_max_theads_per_threadgroup(pipeline) < tiling.nsg*32) { + tiling.nsg /= 2; + pipeline = ggml_metal_library_get_pipeline_mul_mv_mma(lib, op, tiling.nsg, tiling.nt, tiling.rt, add); + } + + return pipeline; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_mma(ggml_metal_library_t lib, const ggml_tensor * op, int nsg, int nt, int rt, bool add) { + char base[256]; + char name[256]; + + const ggml_type tsrc0 = op->src[0]->type; + const ggml_type tsrc1 = op->src[1]->type; + const int ne12 = op->src[1]->ne[2]; + const int r2 = ne12 / op->src[0]->ne[2]; + const int r3 = op->src[1]->ne[3] / op->src[0]->ne[3]; + + GGML_ASSERT(ne12 <= INT16_MAX && r2 <= INT16_MAX && r3 <= INT16_MAX); + + // the specialized kernels unroll over a compile-time row length + const int ne00 = ggml_metal_mul_mv_mma_kind(tsrc0, rt) != GGML_METAL_MMA_KIND_GEN ? op->src[0]->ne[0] : 0; + + snprintf(base, 256, "kernel_mul_mv_mma_%s_%s_nt%d_rt%d", ggml_type_name(tsrc0), ggml_type_name(tsrc1), nt, rt); + snprintf(name, 256, "%s_nsg=%d_ne12=%d_r2=%d_r3=%d_ne00=%d_add=%d", base, nsg, ne12, r2, r3, ne00, add); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + + ggml_metal_cv_set_int32(cv, ne00, FC_MUL_MV_MMA + 4); + ggml_metal_cv_set_int16(cv, nsg, FC_MUL_MV_MMA + 0); + ggml_metal_cv_set_int16(cv, (int16_t) ne12, FC_MUL_MV_MMA + 1); + ggml_metal_cv_set_int16(cv, (int16_t) r2, FC_MUL_MV_MMA + 2); + ggml_metal_cv_set_int16(cv, (int16_t) r3, FC_MUL_MV_MMA + 3); + ggml_metal_cv_set_bool (cv, add, FC_MUL_MV_MMA + 5); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + res.nsg = nsg; + res.smem = ggml_metal_mul_mv_mma_smem(nsg, nt, rt); + + res.nr0 = GGML_METAL_MMA_TILE*nt; + res.nr1 = GGML_METAL_MMA_TILE*rt; + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_metal_library_t lib, const ggml_tensor * op) { char base[256]; char name[256]; @@ -677,10 +929,25 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_meta const ggml_type tsrc1 = op->src[1]->type; const bool bc_inp = op->src[0]->ne[0] % 32 != 0; - const bool bc_out = op->ne[0] % 64 != 0 || op->ne[1] % 32 != 0; + + constexpr int NRA = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y; + constexpr int NRB = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X; + + const bool has_tensor = ggml_metal_device_get_props(ggml_metal_library_get_device(lib))->has_tensor; + + const bool bc_out = has_tensor + ? (op->ne[0] % NRA != 0 || op->ne[1] % NRB != 0) + : (op->ne[0] % 64 != 0 || op->ne[1] % 32 != 0); + + GGML_ASSERT(op->src[1]->ne[2] <= INT16_MAX && op->src[1]->ne[3] <= INT16_MAX); + const int16_t ne12 = (int16_t) op->src[1]->ne[2]; + const int16_t ne13 = (int16_t) op->src[1]->ne[3]; + const int16_t r2 = (int16_t) (ne12 / op->src[0]->ne[2]); + const int16_t r3 = (int16_t) (ne13 / op->src[0]->ne[3]); snprintf(base, 256, "kernel_mul_mm_%s_%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1)); - snprintf(name, 256, "%s_bci=%d_bco=%d", base, bc_inp, bc_out); + snprintf(name, 256, "%s_bci=%d_bco=%d_ne12=%d_ne13=%d_r2=%d_r3=%d", + base, bc_inp, bc_out, ne12, ne13, r2, r3); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { @@ -688,14 +955,30 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm(ggml_meta ggml_metal_cv_set_bool(cv, bc_inp, FC_MUL_MM + 0); ggml_metal_cv_set_bool(cv, bc_out, FC_MUL_MM + 1); + ggml_metal_cv_set_int16(cv, ne12, FC_MUL_MM + 2); + ggml_metal_cv_set_int16(cv, ne13, FC_MUL_MM + 3); + ggml_metal_cv_set_int16(cv, r2, FC_MUL_MM + 4); + ggml_metal_cv_set_int16(cv, r3, FC_MUL_MM + 5); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); ggml_metal_cv_free(cv); } - // when the output size is not multiple of 64x32, we need extra smem to prevent out-of-bounds writes - res.smem = bc_out ? 8192 : 4096 + 2048; + if (has_tensor) { + res.nr0 = NRA; + res.nr1 = NRB; + + const size_t smem_a = NRA * N_MM_NK_TOTAL * sizeof(ggml_fp16_t); + res.smem = smem_a; + } else { + res.nr0 = 64; + res.nr1 = 32; + + res.smem = bc_out ? 8192 : (4096 + 2048); + } + + res.nsg = N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y; return res; } @@ -718,6 +1001,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta const char * suffix = ""; + bool split = false; + // use custom matrix x vector kernel switch (tsrc0) { case GGML_TYPE_F32: @@ -742,6 +1027,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_Q1_0; nr0 = N_R0_Q1_0; } break; + case GGML_TYPE_Q2_0: + { + nsg = N_SG_Q2_0; + nr0 = N_R0_Q2_0; + } break; case GGML_TYPE_Q4_0: { nsg = N_SG_Q4_0; @@ -804,39 +1094,82 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { nsg = N_SG_IQ3_XXS; nr0 = N_R0_IQ3_XXS; smem = 256*4+128; + + // split the rows across threads when there are fewer than 32 chunks per row + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_S: { nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { @@ -850,6 +1183,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta nr0 = N_R0_IQ4_XS; smem = 32*sizeof(float); } break; + case GGML_TYPE_TQ2_0: + { + nsg = N_SG_TQ2_0; + nr0 = N_R0_TQ2_0; + } break; default: { GGML_LOG_ERROR("Asserting on type %d\n", (int) tsrc0); @@ -857,14 +1195,22 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta } }; + GGML_ASSERT(ne12 <= INT16_MAX && ne13 <= INT16_MAX); + const int16_t r2 = (int16_t) (ne12 / ne02); + const int16_t r3 = (int16_t) (ne13 / ne03); + snprintf(base, 256, "kernel_mul_mv_%s_%s%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1), suffix); - snprintf(name, 256, "%s_nsg=%d", base, nsg); + snprintf(name, 256, "%s_nsg=%d_ne12=%d_r2=%d_r3=%d_split=%d", base, nsg, ne12, r2, r3, split); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { ggml_metal_cv_t cv = ggml_metal_cv_init(); - ggml_metal_cv_set_int16(cv, nsg, FC_MUL_MV + 0); + ggml_metal_cv_set_int16(cv, nsg, FC_MUL_MV + 0); + ggml_metal_cv_set_int16(cv, (int16_t) ne12, FC_MUL_MV + 2); + ggml_metal_cv_set_int16(cv, r2, FC_MUL_MV + 3); + ggml_metal_cv_set_int16(cv, r3, FC_MUL_MV + 4); + ggml_metal_cv_set_bool (cv, split, FC_MUL_MV + 5); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -888,10 +1234,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(g ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_int32(cv, ne02, FC_MUL_MM + 7); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); } - res.smem = (size_t) ne02*ne20*sizeof(uint16_t); + // per expert: entry count + base offset, followed by the staging of the ids of ntg tokens + res.smem = 2*(size_t) ne02*sizeof(uint32_t) + (size_t) ne02*ne20*sizeof(uint16_t); + res.smem = GGML_PAD(res.smem, 16); + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_zero(ggml_metal_library_t lib) { + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, "kernel_mul_mm_id_zero"); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, "kernel_mul_mm_id_zero", "kernel_mul_mm_id_zero", nullptr); + } return res; } @@ -942,6 +1304,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m const char * suffix = ""; + bool split = false; + // use custom matrix x vector kernel switch (tsrc0) { case GGML_TYPE_F32: @@ -959,6 +1323,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_Q1_0; nr0 = N_R0_Q1_0; } break; + case GGML_TYPE_Q2_0: + { + nsg = N_SG_Q2_0; + nr0 = N_R0_Q2_0; + } break; case GGML_TYPE_Q4_0: { nsg = N_SG_Q4_0; @@ -1021,39 +1390,82 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nsg = N_SG_IQ2_XXS; nr0 = N_R0_IQ2_XXS; smem = 256*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_XS: { nsg = N_SG_IQ2_XS; nr0 = N_R0_IQ2_XS; smem = 512*8+128; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_XS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_XXS: { nsg = N_SG_IQ3_XXS; nr0 = N_R0_IQ3_XXS; smem = 256*4+128; + + // split the rows across threads when there are fewer than 32 chunks per row + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_XXS_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ3_S: { nsg = N_SG_IQ3_S; nr0 = N_R0_IQ3_S; smem = 512*4; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ3_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ2_S: { nsg = N_SG_IQ2_S; nr0 = N_R0_IQ2_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ2_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_S: { nsg = N_SG_IQ1_S; nr0 = N_R0_IQ1_S; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_S_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ1_M: { nsg = N_SG_IQ1_M; nr0 = N_R0_IQ1_M; + + const int nb32 = ne00/32; + if (nb32 < 32 && (32 % nb32) == 0) { + nr0 = N_R0_IQ1_M_SPLIT; + split = true; + } } break; case GGML_TYPE_IQ4_NL: { @@ -1067,6 +1479,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m nr0 = N_R0_IQ4_XS; smem = 32*sizeof(float); } break; + case GGML_TYPE_TQ2_0: + { + nsg = N_SG_TQ2_0; + nr0 = N_R0_TQ2_0; + } break; default: { GGML_LOG_ERROR("Asserting on type %d\n", (int)op->src[2]->type); @@ -1075,13 +1492,17 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m }; snprintf(base, 256, "kernel_mul_mv_id_%s_%s%s", ggml_type_name(tsrc0), ggml_type_name(tsrc1), suffix); - snprintf(name, 256, "%s_nsg=%d", base, nsg); + snprintf(name, 256, "%s_nsg=%d_split=%d", base, nsg, split); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { ggml_metal_cv_t cv = ggml_metal_cv_init(); ggml_metal_cv_set_int16(cv, nsg, FC_MUL_MV + 0); + ggml_metal_cv_set_int16(cv, 1, FC_MUL_MV + 2); + ggml_metal_cv_set_int16(cv, 1, FC_MUL_MV + 3); + ggml_metal_cv_set_int16(cv, 1, FC_MUL_MV + 4); + ggml_metal_cv_set_bool (cv, split, FC_MUL_MV + 5); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -1117,6 +1538,28 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax(ggml_meta return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax_part(ggml_metal_library_t lib) { + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, "kernel_argmax_part_f32"); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, "kernel_argmax_part_f32", "kernel_argmax_part_f32", nullptr); + } + + res.smem = 32*(sizeof(float) + sizeof(int32_t)); + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax_final(ggml_metal_library_t lib) { + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, "kernel_argmax_final_f32"); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, "kernel_argmax_final_f32", "kernel_argmax_final_f32", nullptr); + } + + res.smem = 32*(sizeof(float) + sizeof(int32_t)); + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_ARGSORT); @@ -1169,7 +1612,22 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge(gg return res; } -// note: reuse the argsort kernel for top_k +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht(ggml_metal_library_t lib, int n, ggml_type tsrc) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_fwht_%s_%d", ggml_type_name(tsrc), n); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + +// note: reuse the argsort kernel for the bitonic top_k fallback ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_TOP_K); @@ -1197,6 +1655,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k(ggml_metal return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_radix(ggml_metal_library_t lib, const ggml_tensor * op) { + assert(op->op == GGML_OP_TOP_K); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_top_k_%s_%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->type)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_TOP_K); @@ -1223,6 +1698,49 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge(ggml return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe( + ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_topk_moe_f32"); + snprintf(name, 256, "%s_n_expert=%d_top_k=%d_with_norm=%d", base, n_expert, top_k, with_norm ? 1 : 0); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool (cv, with_norm, FC_TOPK_MOE + 0); + ggml_metal_cv_set_int32(cv, n_expert, FC_TOPK_MOE + 1); + ggml_metal_cv_set_int32(cv, top_k, FC_TOPK_MOE + 2); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_moe_reduce(ggml_metal_library_t lib, int32_t n_expert_used) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_moe_reduce_f32"); + snprintf(name, 256, "%s_n_expert_used=%d", base, n_expert_used); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_int32(cv, n_expert_used, FC_MOE_REDUCE + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_pad( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -1266,6 +1784,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_p return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16( + ggml_metal_library_t lib, + const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + char base[256]; + + snprintf(base, 256, "kernel_flash_attn_ext_kv_%s_f16", ggml_type_name(op->src[1]->type)); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, base); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, base, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -1309,6 +1844,43 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_b return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_tensor( + ggml_metal_library_t lib, + const ggml_tensor * op, + bool has_mask, + bool has_sinks, + bool has_bias, + bool has_scap) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + char base[256]; + char name[256]; + + const int32_t dk = (int32_t) op->src[1]->ne[0]; + const int32_t dv = (int32_t) op->src[2]->ne[0]; + + const char * type = op->src[1]->type == GGML_TYPE_BF16 ? "bf16" : "f16"; + + snprintf(base, 256, "kernel_flash_attn_ext_tensor_%s_dk%d_dv%d", type, dk, dv); + snprintf(name, 256, "%s_mask=%d_sinks=%d_bias=%d_scap=%d", base, has_mask, has_sinks, has_bias, has_scap); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + + ggml_metal_cv_set_bool(cv, has_mask, FC_FLASH_ATTN_EXT_TENSOR + 0); + ggml_metal_cv_set_bool(cv, has_sinks, FC_FLASH_ATTN_EXT_TENSOR + 1); + ggml_metal_cv_set_bool(cv, has_bias, FC_FLASH_ATTN_EXT_TENSOR + 2); + ggml_metal_cv_set_bool(cv, has_scap, FC_FLASH_ATTN_EXT_TENSOR + 3); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( ggml_metal_library_t lib, const ggml_tensor * op, @@ -1317,7 +1889,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( bool has_bias, bool has_scap, bool has_kvpad, - int32_t nsg) { + int32_t nsg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); char base[256]; @@ -1326,15 +1901,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( const int32_t dk = (int32_t) op->src[1]->ne[0]; const int32_t dv = (int32_t) op->src[2]->ne[0]; - const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0]; - const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0]; + const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type); // do bounds checks for the mask? const bool bc_mask = op->src[3] && (op->src[3]->ne[1] % 8 != 0); snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d", "flash_attn_ext", - ggml_type_name(op->src[1]->type), + type, dk, dv); @@ -1374,6 +1948,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx( + ggml_metal_library_t lib, + const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + assert(op->src[3]); + + char name[256]; + + snprintf(name, 256, "kernel_flash_attn_ext_vec_idx"); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr); + } + + GGML_UNUSED(op); + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec( ggml_metal_library_t lib, const ggml_tensor * op, @@ -1382,8 +1976,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v bool has_bias, bool has_scap, bool has_kvpad, + bool has_sparse, + int32_t nqpsg, + int32_t ne, int32_t nsg, - int32_t nwg) { + int32_t nwg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); char base[256]; @@ -1392,22 +1992,28 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v const int32_t dk = (int32_t) op->src[1]->ne[0]; const int32_t dv = (int32_t) op->src[2]->ne[0]; - const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0]; - const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0]; + const char * type = use_kv_f16 ? "f16" : ggml_type_name(op->src[1]->type); - snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d", + char qne_suffix[16] = {0}; + if (!(nqpsg == 1 && ne == ggml_metal_tuning::fa_vec_baseline_ne(dk, dv))) { + snprintf(qne_suffix, sizeof(qne_suffix), "_q%d_ne%d", nqpsg, ne); + } + + snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d%s", "flash_attn_ext_vec", - ggml_type_name(op->src[1]->type), + type, dk, - dv); + dv, + qne_suffix); - snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d", + snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_sparse=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d", base, has_mask, has_sinks, has_bias, has_scap, has_kvpad, + has_sparse, ns10, ns20, nsg, nwg); @@ -1420,7 +2026,8 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v ggml_metal_cv_set_bool(cv, has_sinks, FC_FLASH_ATTN_EXT_VEC + 1); ggml_metal_cv_set_bool(cv, has_bias, FC_FLASH_ATTN_EXT_VEC + 2); ggml_metal_cv_set_bool(cv, has_scap, FC_FLASH_ATTN_EXT_VEC + 3); - ggml_metal_cv_set_bool(cv, has_kvpad, FC_FLASH_ATTN_EXT_VEC + 4); + ggml_metal_cv_set_bool(cv, has_kvpad, FC_FLASH_ATTN_EXT_VEC + 4); + ggml_metal_cv_set_bool(cv, has_sparse, FC_FLASH_ATTN_EXT_VEC + 5); ggml_metal_cv_set_int32(cv, ns10, FC_FLASH_ATTN_EXT_VEC + 20); ggml_metal_cv_set_int32(cv, ns20, FC_FLASH_ATTN_EXT_VEC + 21); @@ -1625,7 +2232,48 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm(ggml_metal_ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool(cv, false, FC_NORM + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } + + res.smem = 32*sizeof(float); + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm_scale(ggml_metal_library_t lib, const ggml_tensor * op) { + assert(op->op == GGML_OP_NORM || op->op == GGML_OP_RMS_NORM); + + GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); + + char base[256]; + char name[256]; + + const char * suffix = ""; + if (op->ne[0] % 4 == 0) { + suffix = "_4"; + } + + switch (op->op) { + case GGML_OP_NORM: snprintf(base, 256, "kernel_norm_mul_f32%s", suffix); break; + case GGML_OP_RMS_NORM: snprintf(base, 256, "kernel_rms_norm_mul_f32%s", suffix); break; + default: GGML_ABORT("fatal error"); + } + + snprintf(name, 256, "%s_use_scale", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + ggml_metal_cv_set_bool(cv, true, FC_NORM + 0); + + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); } res.smem = 32*sizeof(float); @@ -1634,7 +2282,9 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm(ggml_metal_ } ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope(ggml_metal_library_t lib, const ggml_tensor * op) { - assert(op->op == GGML_OP_ROPE); + assert(op->op == GGML_OP_ROPE || op->op == GGML_OP_ROPE_BACK); + + const bool is_back = op->op == GGML_OP_ROPE_BACK; char base[256]; char name[256]; @@ -1658,13 +2308,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope(ggml_metal_ snprintf(base, 256, "kernel_rope_norm_%s", ggml_type_name(op->src[0]->type)); } - snprintf(name, 256, "%s_imrope=%d", base, is_imrope ? 1 : 0); + snprintf(name, 256, "%s_imrope=%d_is_back=%d", base, is_imrope ? 1 : 0, is_back ? 1 : 0); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { ggml_metal_cv_t cv = ggml_metal_cv_init(); ggml_metal_cv_set_bool(cv, is_imrope, FC_ROPE + 0); + ggml_metal_cv_set_bool(cv, is_back, FC_ROPE + 1); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); @@ -1677,14 +2328,24 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope(ggml_metal_ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_im2col(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_IM2COL); + GGML_TENSOR_LOCALS(int64_t, ne0, op->src[0], ne); + GGML_ASSERT(ggml_is_contiguous(op->src[1])); GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); GGML_ASSERT(op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32); + const bool is_2D = ((const int32_t *)(op->op_params))[6] == 1; + const int64_t KH = is_2D ? ne01 : 1; + const int64_t KW = ne00; + char base[256]; char name[256]; - snprintf(base, 256, "kernel_im2col_%s", ggml_type_name(op->type)); + if (KH*KW <= 1024) { + snprintf(base, 256, "kernel_im2col_%s", ggml_type_name(op->type)); + } else { + snprintf(base, 256, "kernel_im2col_ext_%s", ggml_type_name(op->type)); + } snprintf(name, 256, "%s", base); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); @@ -1718,6 +2379,43 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_1 return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d(ggml_metal_library_t lib, const ggml_tensor * op) { + assert(op->op == GGML_OP_COL2IM_1D); + + GGML_ASSERT(ggml_is_contiguous(op->src[0])); + GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_col2im_1d_%s", ggml_type_name(op->src[0]->type)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake(ggml_metal_library_t lib, enum ggml_type type) { + GGML_ASSERT(type == GGML_TYPE_F32 || type == GGML_TYPE_F16 || type == GGML_TYPE_BF16); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_snake_%s", ggml_type_name(type)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_2d(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_CONV_TRANSPOSE_2D); @@ -1763,6 +2461,29 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d(ggml_met return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) { + assert(op->op == GGML_OP_CONV_2D_DW); + + GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_conv_2d_dw%s_%s_%s", + tiled ? "_tiled" : "", + ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_CONV_3D); @@ -1842,15 +2563,21 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad(ggml_metal_l char base[256]; char name[256]; + const bool circular = ggml_get_op_params_i32(op, 8) != 0; + snprintf(base, 256, "kernel_pad_%s", ggml_type_name(op->src[0]->type)); - snprintf(name, 256, "%s", base); + snprintf(name, 256, "%s_circular=%d", base, circular); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); - if (res.pipeline) { - return res; - } + if (!res.pipeline) { + ggml_metal_cv_t cv = ggml_metal_cv_init(); + + ggml_metal_cv_set_bool(cv, circular, FC_PAD + 0); - res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + res = ggml_metal_library_compile_pipeline(lib, base, name, cv); + + ggml_metal_cv_free(cv); + } return res; } @@ -1940,6 +2667,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_sgd(ggm return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back(ggml_metal_library_t lib, const ggml_tensor * op) { + assert(op->op == GGML_OP_SILU_BACK); + + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_silu_back_%s", ggml_type_name(op->src[0]->type)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_memset(ggml_metal_library_t lib, const ggml_tensor * op) { GGML_ASSERT(op->type == GGML_TYPE_I64); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index b423501..6d77c57 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -102,6 +102,8 @@ ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev void ggml_metal_library_free(ggml_metal_library_t lib); +ggml_metal_device_t ggml_metal_library_get_device(ggml_metal_library_t lib); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline (ggml_metal_library_t lib, const char * name); struct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_metal_library_t lib, const char * base, const char * name, ggml_metal_cv_t cv); @@ -110,10 +112,12 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cpy struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_1d (ggml_metal_library_t lib, const struct ggml_tensor * op, enum ggml_op_pool op_pool); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pool_2d (ggml_metal_library_t lib, const struct ggml_tensor * op, enum ggml_op_pool op_pool); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_get_rows (ggml_metal_library_t lib, enum ggml_type tsrc); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows (ggml_metal_library_t lib, enum ggml_type tidx, enum ggml_type tdst); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diag (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat (ggml_metal_library_t lib, enum ggml_type tsrc); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat (ggml_metal_library_t lib, enum ggml_type tsrc); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum_rows (ggml_metal_library_t lib, const struct ggml_tensor * op); @@ -121,33 +125,50 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_bl struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_cumsum_add (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_tri (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_soft_max (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_lightning_indexer (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_dsv4_hc (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t nc, bool use_silu); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_conv_batched (ggml_metal_library_t lib, const struct ggml_tensor * op, int ssm_conv_bs, int32_t nc, bool use_silu); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tail); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan_ssd_mma (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rwkv (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_gated_delta_net (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_solve_tri (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext (ggml_metal_library_t lib, enum ggml_type tsrc0, enum ggml_type tsrc1, int nsg, int nxpsg, int r1ptg); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext (ggml_metal_library_t lib, const struct ggml_tensor * op, int nsg, int nxpsg, int r1ptg); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_mma (ggml_metal_library_t lib, const struct ggml_tensor * op, int nsg, int nt, int rt, bool add); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_mma_auto (ggml_metal_library_t lib, const struct ggml_tensor * op, bool add); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0 (ggml_metal_library_t lib, int ne02, int ne20); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_zero (ggml_metal_library_t lib); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax_part (ggml_metal_library_t lib); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax_final (ggml_metal_library_t lib); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht (ggml_metal_library_t lib, int n, enum ggml_type tsrc); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_radix (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); -struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse ); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_topk_moe (ggml_metal_library_t lib, int32_t n_expert, int32_t top_k, bool with_norm); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_moe_reduce (ggml_metal_library_t lib, int32_t n_expert_used); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin_one (ggml_metal_library_t lib, enum ggml_op op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_l2_norm (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_group_norm (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm_scale (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_im2col (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_1d (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_transpose_2d (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad (ggml_metal_library_t lib, const struct ggml_tensor * op); @@ -166,12 +187,24 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_mask, int32_t ncpsg); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16( + ggml_metal_library_t lib, + const struct ggml_tensor * op); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_blk( ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t nqptg, int32_t ncpsg); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_tensor( + ggml_metal_library_t lib, + const struct ggml_tensor * op, + bool has_mask, + bool has_sinks, + bool has_bias, + bool has_scap); + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext( ggml_metal_library_t lib, const struct ggml_tensor * op, @@ -180,7 +213,14 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_bias, bool has_scap, bool has_kvpad, - int32_t nsg); + int32_t nsg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20); + +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx( + ggml_metal_library_t lib, + const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec( ggml_metal_library_t lib, @@ -190,8 +230,14 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_att bool has_bias, bool has_scap, bool has_kvpad, + bool has_sparse, + int32_t nqpsg, + int32_t ne, int32_t nsg, - int32_t nwg); + int32_t nwg, + bool use_kv_f16, + int32_t ns10, + int32_t ns20); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_vec_reduce( ggml_metal_library_t lib, @@ -206,15 +252,44 @@ typedef void * ggml_metal_rset_t; // a collection of residency sets (non-owning) typedef struct ggml_metal_rsets * ggml_metal_rsets_t; -ggml_metal_rsets_t ggml_metal_rsets_init(void); +ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev); void ggml_metal_rsets_free(ggml_metal_rsets_t rsets); // // device // +enum ggml_metal_device_id { + GGML_METAL_DEVICE_GENERIC = 0, + + GGML_METAL_DEVICE_M1, + GGML_METAL_DEVICE_M1_PRO, + GGML_METAL_DEVICE_M1_MAX, + GGML_METAL_DEVICE_M1_ULTRA, + GGML_METAL_DEVICE_M2, + GGML_METAL_DEVICE_M2_PRO, + GGML_METAL_DEVICE_M2_MAX, + GGML_METAL_DEVICE_M2_ULTRA, + GGML_METAL_DEVICE_M3, + GGML_METAL_DEVICE_M3_PRO, + GGML_METAL_DEVICE_M3_MAX, + GGML_METAL_DEVICE_M3_ULTRA, + GGML_METAL_DEVICE_M4, + GGML_METAL_DEVICE_M4_PRO, + GGML_METAL_DEVICE_M4_MAX, + GGML_METAL_DEVICE_M5, + GGML_METAL_DEVICE_M5_PRO, + GGML_METAL_DEVICE_M5_MAX, + GGML_METAL_DEVICE_M5_ULTRA, + GGML_METAL_DEVICE_A18_PRO, +}; + +const char * ggml_metal_device_id_token(enum ggml_metal_device_id id); + struct ggml_metal_device_props { int device; + int device_phys; + int device_virt; char name[128]; char desc[128]; @@ -230,7 +305,8 @@ struct ggml_metal_device_props { bool use_residency_sets; bool use_shared_buffers; - bool supports_gpu_family_apple7; + enum ggml_metal_device_id device_id; + int gpu_family; int op_offload_min_batch_size; }; @@ -240,10 +316,10 @@ typedef struct ggml_metal_event * ggml_metal_event_t; void ggml_metal_event_encode_signal(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf); void ggml_metal_event_encode_wait (ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf); -ggml_metal_device_t ggml_metal_device_init(int device); +ggml_metal_device_t ggml_metal_device_init(int device, int n_devices); void ggml_metal_device_free(ggml_metal_device_t dev); -ggml_metal_device_t ggml_metal_device_get(int device); +ggml_metal_device_t ggml_metal_device_get(int device, int n_devices); void * ggml_metal_device_get_obj (ggml_metal_device_t dev); // id void * ggml_metal_device_get_queue(ggml_metal_device_t dev); // id @@ -264,6 +340,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev); +struct ggml_metal_fusion_info; + +// the device-owned fusion debugging context (NULL unless fusion debugging is enabled) +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev); + // // device buffers // @@ -280,6 +361,7 @@ bool ggml_metal_buffer_is_shared(ggml_metal_buffer_t buf); void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); void ggml_metal_buffer_set_tensor (ggml_metal_buffer_t buf, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); void ggml_metal_buffer_get_tensor (ggml_metal_buffer_t buf, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); +bool ggml_metal_buffer_cpy_tensor (ggml_metal_buffer_t buf, const struct ggml_tensor * src, struct ggml_tensor * dst); void ggml_metal_buffer_clear (ggml_metal_buffer_t buf, uint8_t value); // finds the Metal buffer that contains the tensor data on the GPU device diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 27cb168..d1bce8b 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1,6 +1,10 @@ #import "ggml-metal-device.h" +#import "ggml-metal-fusion.h" #import "ggml-impl.h" +#import "ggml-backend-impl.h" +#import "ggml-metal-impl.h" +#import "ggml-metal-common.h" #include @@ -24,6 +28,9 @@ static const NSInteger MTLGPUFamilyMetal3_GGML = 5001; static const NSInteger MTLGPUFamilyMetal4_GGML = 5002; +// MTLLanguageVersion4_0 is not present in older SDKs +static const NSUInteger MTLLanguageVersion4_0_GGML = 4 << 16; + #if !GGML_METAL_EMBED_LIBRARY // Here to assist with NSBundle Path Hack @interface GGMLMetalClass : NSObject @@ -93,169 +100,505 @@ int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_wi return pipeline.pipeline->obj.maxTotalThreadsPerThreadgroup; } +// +// MTLLibrary collection (one library per op-source, compiled separately) +// + +// Single source of truth for the per-kind metal libraries. The order here +// defines the enum values and every per-kind table below, so adding a library +// is a one-line change here (plus adding its source to CMakeLists.txt). +// X(suffix, name): name is both the kernels/.metal basename and the +// ggml_metallib__{start,end} embed-symbol stem. +#define GGML_METAL_LIBS \ + X(FA_AUX, fa_aux) \ + X(FA_F16, fa_f16) \ + X(FA_F32, fa_f32) \ + X(FA_Q4_0, fa_q4_0) \ + X(FA_Q4_1, fa_q4_1) \ + X(FA_Q5_0, fa_q5_0) \ + X(FA_Q5_1, fa_q5_1) \ + X(FA_Q8_0, fa_q8_0) \ + X(FA_TENSOR, fa_tensor) \ + X(FA_VEC_F16, fa_vec_f16) \ + X(FA_VEC_F32, fa_vec_f32) \ + X(FA_VEC_Q4_0, fa_vec_q4_0) \ + X(FA_VEC_Q4_1, fa_vec_q4_1) \ + X(FA_VEC_Q5_0, fa_vec_q5_0) \ + X(FA_VEC_Q5_1, fa_vec_q5_1) \ + X(FA_VEC_Q8_0, fa_vec_q8_0) \ + X(MUL_MV, mul_mv) \ + X(MUL_MV_MMA, mul_mv_mma) \ + X(MUL_MM, mul_mm) \ + X(QUANTIZE, quantize) \ + X(SOFTMAX, softmax) \ + X(NORM, norm) \ + X(UNARY, unary) \ + X(BINBCAST, binbcast) \ + X(REDUCE, reduce) \ + X(TRI, tri) \ + X(SSM, ssm) \ + X(WKV, wkv) \ + X(GATED_DELTA_NET, gated_delta_net)\ + X(SOLVE_TRI, solve_tri) \ + X(ROPE, rope) \ + X(CONV, conv) \ + X(UPSCALE, upscale) \ + X(ARGSORT, argsort) \ + X(POOL, pool) \ + X(MISC, misc) + +enum ggml_metal_lib_kind { +#define X(e, s) GGML_METAL_LIB_##e, + GGML_METAL_LIBS +#undef X + GGML_METAL_LIB_COUNT, +}; + +static const char * const k_lib_names[GGML_METAL_LIB_COUNT] = { +#define X(e, s) [GGML_METAL_LIB_##e] = #s, + GGML_METAL_LIBS +#undef X +}; + struct ggml_metal_library { - id obj; - id device; + // Per-kind compiled libraries. When single_library is true, the whole library + // (e.g. a pre-compiled default.metallib or a from-source build) lives at + // objs[0] and the remaining slots are nil. + id objs[GGML_METAL_LIB_COUNT]; + bool single_library; // true: combined library at objs[0]; false: per-kind libs in objs[*] + + // Routing table: kernel function name -> objs[] index, populated from each + // compiled library's -[MTLLibrary functionNames]. The actual compiled + // libraries are the single source of truth for which library owns a kernel, + // so adding kernels later requires no manual routing maintenance. + // nil in single_library mode (everything resolves to objs[0]). + NSMutableDictionary * fn_to_lib; + + // kernels from a second metallib, resolved ahead of the combined library + NSSet * override_fns; + ggml_metal_device_t dev; ggml_metal_pipelines_t pipelines; // cache of compiled pipelines NSLock * lock; }; -ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) { - id library = nil; - id device = ggml_metal_device_get_obj(dev); +// Build the fn_to_lib routing table by querying each compiled library's public +// function names. Call once after all per-kind libraries have been compiled. +static void ggml_metal_library_build_index(ggml_metal_library_t lib) { + @autoreleasepool { + NSMutableDictionary * index = [[NSMutableDictionary alloc] init]; + for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) { + for (NSString * fname in [lib->objs[kind] functionNames]) { + index[fname] = @(kind); + } + } + lib->fn_to_lib = index; + } +} - // load library - // - // - first check if the library is embedded - // - then check if the library is in the bundle - // - if not found, load the source and compile it - // - if that fails, return NULL - // - // TODO: move to a function - { - const int64_t t_start = ggml_time_us(); +// note: defined below, after struct ggml_metal_device +static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev); - NSError * error = nil; - NSString * src = nil; +// the tensor API headers are exposed to the shader compiler only at Metal language version 4.0 +static void ggml_metal_compile_options_set_lang(MTLCompileOptions * options, bool has_tensor) { + if (!has_tensor) { + return; + } -#if GGML_METAL_EMBED_LIBRARY - GGML_LOG_INFO("%s: using embedded metal library\n", __func__); + options.languageVersion = (MTLLanguageVersion) MTLLanguageVersion4_0_GGML; +} - extern const char ggml_metallib_start[]; - extern const char ggml_metallib_end[]; +// Parse a `#include "name"` line. Returns the quoted name in *include_name on +// success. Whitespace-tolerant; ignores `#include <...>` (system headers). +static bool ggml_metal_library_parse_quoted_include(NSString * line, NSString ** include_name) { + NSScanner * scanner = [NSScanner scannerWithString:line]; + scanner.charactersToBeSkipped = [NSCharacterSet whitespaceCharacterSet]; - src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding]; -#else + if (![scanner scanString:@"#" intoString:NULL] || + ![scanner scanString:@"include" intoString:NULL] || + ![scanner scanString:@"\"" intoString:NULL]) { + return false; + } -#ifdef SWIFT_PACKAGE - NSBundle * bundle = SWIFTPM_MODULE_BUNDLE; -#else - NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; -#endif + NSString * name = nil; + if (![scanner scanUpToString:@"\"" intoString:&name]) { + return false; + } - NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"]; - if (path_lib == nil) { - // Try to find the resource in the directory where the current binary located. - NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0]; - NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent]; - - NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, @"default.metallib"]]; - if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) { - GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]); - - NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error]; - if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) { - // Optionally, if this is a symlink, try to resolve it. - path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error]; - if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) { - // It is a relative path, adding the binary directory as directory prefix. - path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]]; - } - if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) { - // Link to the resource could not be resolved. - path_lib_default = nil; - } else { - GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]); - } + if (include_name) { + *include_name = name; + } + return true; +} + +// Recursively inline `#include "name"` directives. System includes (<...>), +// `#if/#else/#endif`, and other preprocessor lines are passed through to the +// Metal compiler unchanged. `#pragma once` is dropped since `seen` already +// guards against double-inclusion. +static bool ggml_metal_library_flatten_file(NSMutableString * dst, NSString * path, + NSArray * search_paths, + NSMutableSet * seen, NSError ** error) { + NSString * key = [path stringByStandardizingPath]; + if ([seen containsObject:key]) { + return true; + } + [seen addObject:key]; + + NSString * src = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:error]; + if (!src) { + return false; + } + + NSFileManager * fm = [NSFileManager defaultManager]; + for (NSString * line in [src componentsSeparatedByString:@"\n"]) { + NSString * trimmed = [line stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]]; + if ([trimmed isEqualToString:@"#pragma once"]) { + continue; + } + + NSString * include_name = nil; + if (ggml_metal_library_parse_quoted_include(line, &include_name)) { + NSString * resolved = nil; + for (NSString * dir in search_paths) { + NSString * candidate = [dir stringByAppendingPathComponent:include_name]; + if ([fm isReadableFileAtPath:candidate]) { + resolved = candidate; + break; } - } else { - // The resource couldn't be found in the binary's directory. - path_lib_default = nil; } - - path_lib = path_lib_default; + if (!resolved) { + if (error) { + NSString * msg = [NSString stringWithFormat:@"could not resolve include \"%@\" from '%@'", include_name, path]; + *error = [NSError errorWithDomain:@"ggml-metal-source-flatten" code:1 + userInfo:@{NSLocalizedDescriptionKey: msg}]; + } + return false; + } + if (!ggml_metal_library_flatten_file(dst, resolved, search_paths, seen, error)) { + return false; + } + continue; } - if (path_lib != nil) { - // pre-compiled library found - NSURL * libURL = [NSURL fileURLWithPath:path_lib]; - GGML_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]); + [dst appendString:line]; + [dst appendString:@"\n"]; + } - library = [device newLibraryWithURL:libURL error:&error]; - if (error) { - GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); - return nil; + return true; +} + +static NSString * ggml_metal_library_flatten_source(NSString * path_source, NSError ** error) { + // Search paths cover both runtime layout (build/bin/kernels + build/bin) + // and source-tree layout (ggml/src/ggml-metal/kernels + ggml/src/ggml-metal + ggml/src). + NSString * path_kernels = [path_source stringByDeletingLastPathComponent]; + NSString * path_base = [path_kernels stringByDeletingLastPathComponent]; + NSArray * search_paths = @[ + path_kernels, + path_base, + [path_base stringByDeletingLastPathComponent], + ]; + + NSMutableString * src = [[NSMutableString alloc] init]; + NSMutableSet * seen = [NSMutableSet set]; + + if (!ggml_metal_library_flatten_file(src, path_source, search_paths, seen, error)) { + [src release]; + return nil; + } + return src; +} + +// Compile all per-kind libraries in parallel. `source_for_kind` returns the MSL +// source for a kind (the helper takes ownership and releases it), or nil with +// *err set on failure. On success the objs[] slots are populated and the routing +// index is built; on any failure every error is logged and false is returned +// (the caller is responsible for freeing `res`). +static bool ggml_metal_library_compile_all( + ggml_metal_library_t res, + id device, + NSDictionary * prep, + NSString * (^source_for_kind)(int kind, NSError ** err), + const char * origin) { + const int64_t t_start = ggml_time_us(); + + int64_t * t_per_lib = calloc(GGML_METAL_LIB_COUNT, sizeof(int64_t)); + NSError ** err_per_lib = calloc(GGML_METAL_LIB_COUNT, sizeof(NSError *)); + __block atomic_bool any_failure = false; + + dispatch_group_t group = dispatch_group_create(); + dispatch_queue_t queue = dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0); + + for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) { + dispatch_group_async(group, queue, ^{ + + const int64_t t0 = ggml_time_us(); + + NSError * error = nil; + + NSString * src = source_for_kind(kind, &error); + if (!src) { + err_per_lib[kind] = [error retain]; + atomic_store(&any_failure, true); + return; } - } else { - GGML_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__); - NSString * path_source; - NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"]; + id lib = nil; - GGML_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, path_resource ? [path_resource UTF8String] : "nil"); + @autoreleasepool { + MTLCompileOptions * options = [MTLCompileOptions new]; + options.preprocessorMacros = prep; + ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(res->dev)->has_tensor); - if (path_resource) { - path_source = [path_resource stringByAppendingPathComponent:@"ggml-metal.metal"]; - } else { - path_source = [bundle pathForResource:@"ggml-metal" ofType:@"metal"]; + lib = [device newLibraryWithSource:src options:options error:&error]; + + [options release]; + + // retain the error before the autorelease pool drains it + if (!lib) { + err_per_lib[kind] = [error retain]; + } } - if (path_source == nil) { - GGML_LOG_WARN("%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\n", __func__); - path_source = @"ggml-metal.metal"; + [src release]; + + t_per_lib[kind] = ggml_time_us() - t0; + + if (!lib) { + atomic_store(&any_failure, true); + return; } - GGML_LOG_INFO("%s: loading '%s'\n", __func__, [path_source UTF8String]); + res->objs[kind] = lib; + }); + } + dispatch_group_wait(group, DISPATCH_TIME_FOREVER); + dispatch_release(group); + + const bool ok = !atomic_load(&any_failure); + + if (ok) { + const int64_t t_total = ggml_time_us() - t_start; + int64_t t_max = 0; + for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) { + GGML_LOG_DEBUG("%s: compiled '%s' library in %.3f sec\n", + __func__, k_lib_names[kind], t_per_lib[kind] / 1e6); + if (t_per_lib[kind] > t_max) t_max = t_per_lib[kind]; + } + GGML_LOG_INFO("%s: loaded %d libraries from %s in %.3f sec (max single = %.3f sec)\n", + __func__, GGML_METAL_LIB_COUNT, origin, t_total / 1e6, t_max / 1e6); - src = [NSString stringWithContentsOfFile:path_source encoding:NSUTF8StringEncoding error:&error]; - if (error) { - GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); - return nil; + ggml_metal_library_build_index(res); + } else { + for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) { + if (err_per_lib[kind]) { + GGML_LOG_ERROR("%s: failed to build '%s' library: %s\n", __func__, + k_lib_names[kind], [[err_per_lib[kind] description] UTF8String]); + [err_per_lib[kind] release]; } } -#endif + } - if (!library) { - @autoreleasepool { - // dictionary of preprocessor macros - NSMutableDictionary * prep = [NSMutableDictionary dictionary]; + free(err_per_lib); + free(t_per_lib); - if (ggml_metal_device_get_props(dev)->has_bfloat) { - [prep setObject:@"1" forKey:@"GGML_METAL_HAS_BF16"]; - } + return ok; +} + +// look for .metallib as a bundle resource, then next to the running binary +static NSString * ggml_metal_find_metallib(NSBundle * bundle, NSString * name) { + NSError * error = nil; - if (ggml_metal_device_get_props(dev)->has_tensor) { - [prep setObject:@"1" forKey:@"GGML_METAL_HAS_TENSOR"]; + NSString * path_lib = [bundle pathForResource:name ofType:@"metallib"]; + if (path_lib == nil) { + // Try to find the resource in the directory where the current binary located. + NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0]; + NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent]; + + NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, [name stringByAppendingPathExtension:@"metallib"]]]; + if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) { + GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]); + + NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error]; + if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) { + // Optionally, if this is a symlink, try to resolve it. + path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error]; + if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) { + // It is a relative path, adding the binary directory as directory prefix. + path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]]; } + if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) { + // Link to the resource could not be resolved. + path_lib_default = nil; + } else { + GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]); + } + } + } else { + // The resource couldn't be found in the binary's directory. + path_lib_default = nil; + } + + path_lib = path_lib_default; + } + + return path_lib; +} + +ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) { + id device = ggml_metal_device_get_obj(dev); + ggml_metal_library_t res = calloc(1, sizeof(struct ggml_metal_library)); + res->dev = dev; + res->pipelines = ggml_metal_pipelines_init(); + res->lock = [NSLock new]; + + // shared MTLCompileOptions preprocessor macros (matches the build-time defines) + NSMutableDictionary * prep = [NSMutableDictionary dictionary]; + if (ggml_metal_device_get_props(dev)->has_bfloat) { + [prep setObject:@"1" forKey:@"GGML_METAL_HAS_BF16"]; + } + if (ggml_metal_device_get_props(dev)->has_tensor) { + [prep setObject:@"1" forKey:@"GGML_METAL_HAS_TENSOR"]; + } #if GGML_METAL_EMBED_LIBRARY - [prep setObject:@"1" forKey:@"GGML_METAL_EMBED_LIBRARY"]; + [prep setObject:@"1" forKey:@"GGML_METAL_EMBED_LIBRARY"]; #endif - MTLCompileOptions * options = [MTLCompileOptions new]; - options.preprocessorMacros = prep; +#if GGML_METAL_EMBED_LIBRARY + GGML_LOG_INFO("%s: using embedded metal library\n", __func__); - //[options setFastMathEnabled:false]; + // start/end symbols emitted by CMake (see CMakeLists.txt), one pair per kind +#define X(e, s) extern const char ggml_metallib_##s##_start[]; extern const char ggml_metallib_##s##_end[]; + GGML_METAL_LIBS +#undef X - library = [device newLibraryWithSource:src options:options error:&error]; - if (error) { - GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); - return nil; - } + static const char * const lib_start[GGML_METAL_LIB_COUNT] = { +#define X(e, s) [GGML_METAL_LIB_##e] = ggml_metallib_##s##_start, + GGML_METAL_LIBS +#undef X + }; + static const char * const lib_end[GGML_METAL_LIB_COUNT] = { +#define X(e, s) [GGML_METAL_LIB_##e] = ggml_metallib_##s##_end, + GGML_METAL_LIBS +#undef X + }; -#if !__has_feature(objc_arc) - [options release]; + const bool ok = ggml_metal_library_compile_all(res, device, prep, + ^NSString * (int kind, NSError ** err) { + (void) err; + return [[NSString alloc] initWithBytes:lib_start[kind] + length:(lib_end[kind] - lib_start[kind]) + encoding:NSUTF8StringEncoding]; + }, "embedded data"); + + if (!ok) { + ggml_metal_library_free(res); + return NULL; + } + + return res; +#else +#ifdef SWIFT_PACKAGE + NSBundle * bundle = SWIFTPM_MODULE_BUNDLE; +#else + NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]]; #endif - } + + const int64_t t_start = ggml_time_us(); + + NSError * error = nil; + NSString * path_lib = ggml_metal_find_metallib(bundle, @"default"); + + if (path_lib != nil) { + // pre-compiled library found: a single combined default.metallib + NSURL * libURL = [NSURL fileURLWithPath:path_lib]; + GGML_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]); + + res->objs[0] = [device newLibraryWithURL:libURL error:&error]; + res->single_library = true; + if (!res->objs[0]) { + GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]); + ggml_metal_library_free(res); + return NULL; } -#if GGML_METAL_EMBED_LIBRARY - [src release]; -#endif // GGML_METAL_EMBED_LIBRARY + // the tensor API kernels are built into a separate metallib + if (ggml_metal_device_get_props(dev)->has_tensor) { + NSString * path_mm = ggml_metal_find_metallib(bundle, @"ggml-tensor"); + + id lib_mm = nil; + if (path_mm != nil) { + lib_mm = [device newLibraryWithURL:[NSURL fileURLWithPath:path_mm] error:&error]; + if (!lib_mm && error) { + GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]); + } + } + + if (lib_mm) { + GGML_LOG_INFO("%s: loaded '%s'\n", __func__, [path_mm UTF8String]); + + res->objs[GGML_METAL_LIB_MUL_MM] = [lib_mm retain]; + res->override_fns = [[NSSet setWithArray:[lib_mm functionNames]] retain]; + } else { + GGML_LOG_INFO("%s: ggml-tensor.metallib not found - disabling the tensor API\n", __func__); + + ggml_metal_device_disable_tensor(dev); + } + } GGML_LOG_INFO("%s: loaded in %.3f sec\n", __func__, (ggml_time_us() - t_start) / 1e6); + return res; } - ggml_metal_library_t res = calloc(1, sizeof(struct ggml_metal_library)); + // no pre-compiled metallib: fall back to compiling each kernel source separately + GGML_LOG_INFO("%s: default.metallib not found, loading kernel sources\n", __func__); - res->obj = library; - res->device = device; - res->pipelines = ggml_metal_pipelines_init(); - res->lock = [NSLock new]; + NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"]; + if (path_resource) { + GGML_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, [path_resource UTF8String]); + } + + // resolve each kind's source path up front (file lookup/logging stays on the calling thread) + NSString ** path_per_kind = calloc(GGML_METAL_LIB_COUNT, sizeof(NSString *)); + for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) { + NSString * rel = [NSString stringWithFormat:@"kernels/%s.metal", k_lib_names[kind]]; + + NSString * path_source = nil; + if (path_resource) { + path_source = [path_resource stringByAppendingPathComponent:rel]; + } else { + NSString * stem = [NSString stringWithFormat:@"kernels/%s", k_lib_names[kind]]; + path_source = [bundle pathForResource:stem ofType:@"metal"]; + } + + if (path_source == nil || ![[NSFileManager defaultManager] isReadableFileAtPath:path_source]) { + GGML_LOG_WARN("%s: could not locate %s in bundle, falling back to cwd\n", __func__, [rel UTF8String]); + path_source = rel; + } + + GGML_LOG_DEBUG("%s: loading '%s'\n", __func__, [path_source UTF8String]); + + path_per_kind[kind] = [path_source retain]; + } + + const bool ok = ggml_metal_library_compile_all(res, device, prep, + ^NSString * (int kind, NSError ** err) { + return ggml_metal_library_flatten_source(path_per_kind[kind], err); + }, "source"); + + for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) { + [path_per_kind[kind] release]; + } + free(path_per_kind); + + if (!ok) { + ggml_metal_library_free(res); + return NULL; + } return res; +#endif } ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev, const char * source, bool verbose) { @@ -283,6 +626,7 @@ ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev MTLCompileOptions * options = [MTLCompileOptions new]; options.preprocessorMacros = prep; + ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(dev)->has_tensor); library = [device newLibraryWithSource:src options:options error:&error]; if (error) { @@ -317,10 +661,11 @@ ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev return NULL; } - res->obj = library; - res->device = device; - res->pipelines = ggml_metal_pipelines_init(); - res->lock = [NSLock new]; + res->objs[0] = library; + res->single_library = true; + res->dev = dev; + res->pipelines = ggml_metal_pipelines_init(); + res->lock = [NSLock new]; return res; } @@ -330,8 +675,18 @@ void ggml_metal_library_free(ggml_metal_library_t lib) { return; } - if (lib->obj) { - [lib->obj release]; + for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) { + if (lib->objs[kind]) { + [lib->objs[kind] release]; + } + } + + if (lib->fn_to_lib) { + [lib->fn_to_lib release]; + } + + if (lib->override_fns) { + [lib->override_fns release]; } ggml_metal_pipelines_free(lib->pipelines); @@ -341,6 +696,10 @@ void ggml_metal_library_free(ggml_metal_library_t lib) { free(lib); } +ggml_metal_device_t ggml_metal_library_get_device(ggml_metal_library_t lib) { + return lib->dev; +} + struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline(ggml_metal_library_t lib, const char * name) { [lib->lock lock]; @@ -388,11 +747,30 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_ GGML_LOG_DEBUG("%s: compiling pipeline: base = '%s', name = '%s'\n", __func__, base, name); + // route to the library that actually defines this kernel; fn_to_lib is + // built from -[MTLLibrary functionNames] so it's always in sync + int lib_idx = 0; + if (lib->override_fns && [lib->override_fns containsObject:base_func]) { + lib_idx = GGML_METAL_LIB_MUL_MM; + } else if (!lib->single_library) { + NSNumber * idx = lib->fn_to_lib[base_func]; + if (!idx) { + [lib->lock unlock]; + + GGML_LOG_ERROR("%s: kernel not found in any metal library: base = '%s', name = '%s'\n", __func__, base, name); + + return res; + } + lib_idx = [idx intValue]; + } + + id mtl_lib = lib->objs[lib_idx]; + id mtl_function; if (!cv) { - mtl_function = [lib->obj newFunctionWithName:base_func]; + mtl_function = [mtl_lib newFunctionWithName:base_func]; } else { - mtl_function = [lib->obj newFunctionWithName:base_func constantValues:cv->obj error:&error]; + mtl_function = [mtl_lib newFunctionWithName:base_func constantValues:cv->obj error:&error]; } if (!mtl_function) { [lib->lock unlock]; @@ -405,7 +783,8 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_ return res; } - id obj = [lib->device newComputePipelineStateWithFunction:mtl_function error:&error]; + id device = ggml_metal_device_get_obj(lib->dev); + id obj = [device newComputePipelineStateWithFunction:mtl_function error:&error]; [mtl_function release]; @@ -476,7 +855,9 @@ void ggml_metal_encoder_free(ggml_metal_encoder_t encoder) { } void ggml_metal_encoder_debug_group_push(ggml_metal_encoder_t encoder, const char * name) { - [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]]; + @autoreleasepool { + [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]]; + } } void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) { @@ -484,6 +865,10 @@ void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) { } void ggml_metal_encoder_set_pipeline(ggml_metal_encoder_t encoder, struct ggml_metal_pipeline_with_params pipeline) { + if (!pipeline.pipeline) { + GGML_ABORT("%s: nil Metal pipeline (missing kernel; see compile_pipeline log above)\n", __func__); + } + [encoder->obj setComputePipelineState:pipeline.pipeline->obj]; } @@ -496,6 +881,9 @@ void ggml_metal_encoder_set_buffer(ggml_metal_encoder_t encoder, struct ggml_met } void ggml_metal_encoder_set_threadgroup_memory_size(ggml_metal_encoder_t encoder, size_t size, int idx) { + // ref: https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setthreadgroupmemorylength(_:index:) + GGML_ASSERT(size % 16 == 0); + [encoder->obj setThreadgroupMemoryLength:size atIndex:idx]; } @@ -525,6 +913,9 @@ void ggml_metal_encoder_end_encoding(ggml_metal_encoder_t encoder) { struct ggml_metal_device_props props; + // shared fusion debugging context + struct ggml_metal_fusion_info * finfo; + // virtual address for GPU memory allocations atomic_uintptr_t addr_virt; }; @@ -541,6 +932,8 @@ void ggml_metal_encoder_end_encoding(ggml_metal_encoder_t encoder) { // number of seconds since the last graph computation // keep the residency sets wired for that amount of time to avoid being collected by the OS int keep_alive_s; + int loops_per_s; + int time_per_loop_ms; // background heartbeat thread to keep the residency sets alive atomic_bool d_stop; @@ -549,7 +942,32 @@ void ggml_metal_encoder_end_encoding(ggml_metal_encoder_t encoder) { dispatch_group_t d_group; }; -ggml_metal_rsets_t ggml_metal_rsets_init(void) { +#if defined(GGML_METAL_HAS_RESIDENCY_SETS) +static void ggml_metal_dummy_work(ggml_metal_device_t dev) { + if (dev->mtl_queue == nil) { + return; + } + + @autoreleasepool { + // perform a minimal dummy operation on the GPU + id buf = [dev->mtl_device newBufferWithLength:1 options:MTLResourceStorageModePrivate]; + id cmd_buf = [dev->mtl_queue commandBuffer]; + + { + id encoder = [cmd_buf blitCommandEncoder]; + + [encoder fillBuffer:buf range:NSMakeRange(0, 1) value:0]; + + [encoder endEncoding]; + } + + [cmd_buf commit]; + [buf release]; + } +} +#endif + +ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev) { ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets)); res->lock = [[NSLock alloc] init]; @@ -567,10 +985,13 @@ ggml_metal_rsets_t ggml_metal_rsets_init(void) { res->keep_alive_s = 3*60; } + res->time_per_loop_ms = 5; + res->loops_per_s = 1000/res->time_per_loop_ms; + GGML_LOG_INFO("%s: creating a residency set collection (keep_alive = %d s)\n", __func__, res->keep_alive_s); atomic_store_explicit(&res->d_stop, false, memory_order_relaxed); - atomic_store_explicit(&res->d_loop, 2*res->keep_alive_s, memory_order_relaxed); + atomic_store_explicit(&res->d_loop, res->loops_per_s*res->keep_alive_s, memory_order_relaxed); res->d_group = dispatch_group_create(); @@ -593,13 +1014,21 @@ ggml_metal_rsets_t ggml_metal_rsets_init(void) { [res->lock unlock]; } - // half a second - usleep(500 * 1000); + usleep(res->time_per_loop_ms * 1000); } } #endif }); +#if defined(GGML_METAL_HAS_RESIDENCY_SETS) + if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) { + // workaround for residency set memory not being released if no GPU operation occurs + // https://developer.apple.com/forums/thread/839089 + // https://github.com/ggml-org/llama.cpp/issues/25937 + ggml_metal_dummy_work(dev); + } +#endif + return res; } @@ -622,239 +1051,320 @@ void ggml_metal_rsets_free(ggml_metal_rsets_t rsets) { free(rsets); } -ggml_metal_device_t ggml_metal_device_init(int device) { +static const struct { + const char * name; + const char * token; + enum ggml_metal_device_id id; +} k_metal_devices[] = { +#define DEV(name, id) { name, #id, id } + DEV("M1", GGML_METAL_DEVICE_M1), + DEV("M1 Pro", GGML_METAL_DEVICE_M1_PRO), + DEV("M1 Max", GGML_METAL_DEVICE_M1_MAX), + DEV("M1 Ultra", GGML_METAL_DEVICE_M1_ULTRA), + DEV("M2", GGML_METAL_DEVICE_M2), + DEV("M2 Pro", GGML_METAL_DEVICE_M2_PRO), + DEV("M2 Max", GGML_METAL_DEVICE_M2_MAX), + DEV("M2 Ultra", GGML_METAL_DEVICE_M2_ULTRA), + DEV("M3", GGML_METAL_DEVICE_M3), + DEV("M3 Pro", GGML_METAL_DEVICE_M3_PRO), + DEV("M3 Max", GGML_METAL_DEVICE_M3_MAX), + DEV("M3 Ultra", GGML_METAL_DEVICE_M3_ULTRA), + DEV("M4", GGML_METAL_DEVICE_M4), + DEV("M4 Pro", GGML_METAL_DEVICE_M4_PRO), + DEV("M4 Max", GGML_METAL_DEVICE_M4_MAX), + DEV("M5", GGML_METAL_DEVICE_M5), + DEV("M5 Pro", GGML_METAL_DEVICE_M5_PRO), + DEV("M5 Max", GGML_METAL_DEVICE_M5_MAX), + DEV("M5 Ultra", GGML_METAL_DEVICE_M5_ULTRA), + DEV("A18 Pro", GGML_METAL_DEVICE_A18_PRO), +#undef DEV +}; + +static enum ggml_metal_device_id ggml_metal_device_id_parse(const char * name) { + if (!name) { + return GGML_METAL_DEVICE_GENERIC; + } + + static const char prefix[] = "Apple "; + if (strncmp(name, prefix, sizeof(prefix) - 1) != 0) { + return GGML_METAL_DEVICE_GENERIC; + } + const char * suffix = name + sizeof(prefix) - 1; + + for (size_t i = 0; i < sizeof(k_metal_devices)/sizeof(k_metal_devices[0]); ++i) { + if (strcmp(suffix, k_metal_devices[i].name) == 0) { + return k_metal_devices[i].id; + } + } + return GGML_METAL_DEVICE_GENERIC; +} + +const char * ggml_metal_device_id_token(enum ggml_metal_device_id id) { + for (size_t i = 0; i < sizeof(k_metal_devices)/sizeof(k_metal_devices[0]); ++i) { + if (k_metal_devices[i].id == id) { + return k_metal_devices[i].token; + } + } + return "GGML_METAL_DEVICE_GENERIC"; +} + +ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) { ggml_metal_device_t dev = calloc(1, sizeof(struct ggml_metal_device)); assert(dev != NULL); - if (dev->mtl_device == nil) { - dev->mtl_device = MTLCreateSystemDefaultDevice(); + @autoreleasepool { + if (dev->mtl_device == nil) { + dev->mtl_device = MTLCreateSystemDefaultDevice(); - if (dev->mtl_device) { - dev->mtl_queue = [dev->mtl_device newCommandQueue]; - if (dev->mtl_queue == nil) { - GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__); - } + if (dev->mtl_device) { + dev->mtl_queue = [dev->mtl_device newCommandQueue]; + if (dev->mtl_queue == nil) { + GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__); + } - dev->addr_virt = 0x000000400ULL; + dev->addr_virt = 0x000000400ULL; - dev->props.device = device; - dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7]; - dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML]; + dev->props.device = device; - dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7]; - dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory; + // the Metal backend uses the system default device as the single physical device; + // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES + dev->props.device_phys = 0; + dev->props.device_virt = device; - dev->props.has_bfloat = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML]; - dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6]; - if (getenv("GGML_METAL_BF16_DISABLE") != NULL) { - dev->props.has_bfloat = false; - } + dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7]; + dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML]; - dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML]; - if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) { - dev->props.has_tensor = false; - } + dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7]; + dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory; - // note: disable the tensor API by default for old chips because with the current implementation it is not useful - // - M2 Ultra: ~5% slower - // - M4, M4 Max: no significant difference - // - // TODO: try to update the tensor API kernels to at least match the simdgroup performance - if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL && - ![[dev->mtl_device name] containsString:@"M5"] && - ![[dev->mtl_device name] containsString:@"M6"] && - ![[dev->mtl_device name] containsString:@"A19"] && - ![[dev->mtl_device name] containsString:@"A20"]) { - GGML_LOG_WARN("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__); - dev->props.has_tensor = false; - } + dev->props.has_bfloat = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML]; + dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6]; + if (getenv("GGML_METAL_BF16_DISABLE") != NULL) { + dev->props.has_bfloat = false; + } - // double-check that the tensor API compiles - if (dev->props.has_tensor) { - const char * src_tensor_f16 = "\n" - "#include \n" - "#include \n" - "#include \n" - " \n" - "using namespace metal; \n" - "using namespace mpp::tensor_ops; \n" - " \n" - "kernel void dummy_kernel( \n" - " tensor> A [[buffer(0)]], \n" - " tensor> B [[buffer(1)]], \n" - " device float * C [[buffer(2)]], \n" - " uint2 tgid [[threadgroup_position_in_grid]]) \n" - "{ \n" - " auto tA = A.slice(0, (int)tgid.y); \n" - " auto tB = B.slice((int)tgid.x, 0); \n" - " \n" - " matmul2d< \n" - " matmul2d_descriptor(16, 16, dynamic_extent), \n" - " execution_simdgroups<4>> mm; \n" - " \n" - " auto cT = mm.get_destination_cooperative_tensor(); \n" - " \n" - " auto sA = tA.slice(0, 0); \n" - " auto sB = tB.slice(0, 0); \n" - " mm.run(sB, sA, cT); \n" - " \n" - " auto tC = tensor, tensor_inline>(C, dextents(4, 4)); \n" - " \n" - " cT.store(tC); \n" - "}"; - - GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__); - ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false); - if (lib == NULL) { - GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__); + dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML]; + if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) { dev->props.has_tensor = false; - } else { - struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil); - if (!ppl.pipeline) { + } + + // note: disable the tensor API by default for old chips because with the current implementation it is not useful + // - M2 Ultra: ~5% slower + // - M4, M4 Max: no significant difference + // + // TODO: try to update the tensor API kernels to at least match the simdgroup performance + if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL && + ![[dev->mtl_device name] containsString:@"M5"] && + ![[dev->mtl_device name] containsString:@"M6"] && + ![[dev->mtl_device name] containsString:@"A19"] && + ![[dev->mtl_device name] containsString:@"A20"]) { + GGML_LOG_INFO("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__); + dev->props.has_tensor = false; + } + + // double-check that the tensor API compiles + if (dev->props.has_tensor) { + const char * src_tensor_f16 = "\n" + "#include \n" + "#include \n" + "#include \n" + " \n" + "using namespace metal; \n" + "using namespace mpp::tensor_ops; \n" + " \n" + "kernel void dummy_kernel( \n" + " tensor> A [[buffer(0)]], \n" + " tensor> B [[buffer(1)]], \n" + " device float * C [[buffer(2)]], \n" + " uint2 tgid [[threadgroup_position_in_grid]]) \n" + "{ \n" + " auto tA = A.slice(0, (int)tgid.y); \n" + " auto tB = B.slice((int)tgid.x, 0); \n" + " \n" + " matmul2d< \n" + " matmul2d_descriptor(16, 16, dynamic_extent), \n" + " execution_simdgroups<4>> mm; \n" + " \n" + " auto cT = mm.get_destination_cooperative_tensor(); \n" + " \n" + " auto sA = tA.slice(0, 0); \n" + " auto sB = tB.slice(0, 0); \n" + " mm.run(sB, sA, cT); \n" + " \n" + " auto tC = tensor, tensor_inline>(C, dextents(16, 16)); \n" + " \n" + " cT.store(tC); \n" + "}"; + + GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__); + ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false); + if (lib == NULL) { GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__); dev->props.has_tensor = false; - } + } else { + struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil); + if (!ppl.pipeline) { + GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__); + dev->props.has_tensor = false; + } - ggml_metal_library_free(lib); + ggml_metal_library_free(lib); + } } - } - // try to compile a dummy kernel to determine if the tensor API is supported for bfloat - if (dev->props.has_tensor && dev->props.has_bfloat) { - const char * src_tensor_bf16 = "\n" - "#include \n" - "#include \n" - "#include \n" - " \n" - "using namespace metal; \n" - "using namespace mpp::tensor_ops; \n" - " \n" - "kernel void dummy_kernel( \n" - " tensor> A [[buffer(0)]], \n" - " tensor> B [[buffer(1)]], \n" - " device float * C [[buffer(2)]], \n" - " uint2 tgid [[threadgroup_position_in_grid]]) \n" - "{ \n" - " auto tA = A.slice(0, (int)tgid.y); \n" - " auto tB = B.slice((int)tgid.x, 0); \n" - " \n" - " matmul2d< \n" - " matmul2d_descriptor(16, 16, dynamic_extent), \n" - " execution_simdgroups<4>> mm; \n" - " \n" - " auto cT = mm.get_destination_cooperative_tensor(); \n" - " \n" - " auto sA = tA.slice(0, 0); \n" - " auto sB = tB.slice(0, 0); \n" - " mm.run(sB, sA, cT); \n" - " \n" - " auto tC = tensor, tensor_inline>(C, dextents(4, 4)); \n" - " \n" - " cT.store(tC); \n" - "}"; - - GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__); - ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false); - if (lib == NULL) { - GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__); - dev->props.has_bfloat = false; - } else { - struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil); - if (!ppl.pipeline) { + // try to compile a dummy kernel to determine if the tensor API is supported for bfloat + if (dev->props.has_tensor && dev->props.has_bfloat) { + const char * src_tensor_bf16 = "\n" + "#include \n" + "#include \n" + "#include \n" + " \n" + "using namespace metal; \n" + "using namespace mpp::tensor_ops; \n" + " \n" + "kernel void dummy_kernel( \n" + " tensor> A [[buffer(0)]], \n" + " tensor> B [[buffer(1)]], \n" + " device float * C [[buffer(2)]], \n" + " uint2 tgid [[threadgroup_position_in_grid]]) \n" + "{ \n" + " auto tA = A.slice(0, (int)tgid.y); \n" + " auto tB = B.slice((int)tgid.x, 0); \n" + " \n" + " matmul2d< \n" + " matmul2d_descriptor(16, 16, dynamic_extent), \n" + " execution_simdgroups<4>> mm; \n" + " \n" + " auto cT = mm.get_destination_cooperative_tensor(); \n" + " \n" + " auto sA = tA.slice(0, 0); \n" + " auto sB = tB.slice(0, 0); \n" + " mm.run(sB, sA, cT); \n" + " \n" + " auto tC = tensor, tensor_inline>(C, dextents(16, 16)); \n" + " \n" + " cT.store(tC); \n" + "}"; + + GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__); + ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false); + if (lib == NULL) { GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__); dev->props.has_bfloat = false; - } + } else { + struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil); + if (!ppl.pipeline) { + GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__); + dev->props.has_bfloat = false; + } - ggml_metal_library_free(lib); + ggml_metal_library_free(lib); + } } - } - dev->props.use_residency_sets = true; + dev->props.use_residency_sets = true; #if defined(GGML_METAL_HAS_RESIDENCY_SETS) - dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil; + dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil; #endif - dev->props.use_shared_buffers = dev->props.has_unified_memory; -#if TARGET_OS_OSX - // In case of eGPU, shared memory may be preferable. - dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal; + dev->props.use_shared_buffers = dev->props.has_unified_memory; +#if TARGET_OS_OSX && TARGET_CPU_X86_64 + // In case of eGPU, shared memory may be preferable. + dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal; #endif - if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) { - dev->props.use_shared_buffers = false; - } - if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) { - dev->props.use_shared_buffers = true; - } + if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) { + dev->props.use_shared_buffers = false; + } + if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) { + dev->props.use_shared_buffers = true; + } - dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7]; + dev->props.device_id = ggml_metal_device_id_parse([[dev->mtl_device name] UTF8String]); - dev->props.op_offload_min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32; + dev->props.op_offload_min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32; - dev->props.max_buffer_size = dev->mtl_device.maxBufferLength; - dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength; - if (@available(macOS 10.12, iOS 16.0, *)) { - dev->props.max_working_set_size = dev->mtl_device.recommendedMaxWorkingSetSize; - } else { - dev->props.max_working_set_size = dev->mtl_device.maxBufferLength; - } + dev->props.max_buffer_size = dev->mtl_device.maxBufferLength; + dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength; + if (@available(macOS 10.12, iOS 16.0, *)) { + dev->props.max_working_set_size = dev->mtl_device.recommendedMaxWorkingSetSize; + } else { + dev->props.max_working_set_size = dev->mtl_device.maxBufferLength; + } - snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device); - snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", [[dev->mtl_device name] UTF8String]); + { + const char * val = getenv("GGML_METAL_FUSION_DEBUG"); + dev->finfo = ggml_metal_fusion_info_init( + getenv("GGML_METAL_FUSION_DISABLE") == nil, + val ? atoi(val) : 0); + } - dev->library = ggml_metal_library_init(dev); - if (!dev->library) { - GGML_LOG_ERROR("%s: error: failed to create library\n", __func__); - } + snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device); + const char * gpu_name = [[dev->mtl_device name] UTF8String]; + if (n_devices > 1) { + snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)", + gpu_name, dev->props.device_phys, dev->props.device_virt); + } else { + snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name); + } - if (dev->props.use_residency_sets) { - dev->rsets = ggml_metal_rsets_init(); - } else { - dev->rsets = nil; - } + dev->library = ggml_metal_library_init(dev); + if (!dev->library) { + GGML_LOG_ERROR("%s: error: failed to create library\n", __func__); + } - // print MTL GPU family: - GGML_LOG_INFO("%s: GPU name: %s\n", __func__, dev->props.name); + if (dev->props.use_residency_sets) { + dev->rsets = ggml_metal_rsets_init(dev); + } else { + dev->rsets = nil; + } - // determine max supported GPU family - // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf - // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf - { - for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) { - if ([dev->mtl_device supportsFamily:i]) { - GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i); - break; + // print MTL GPU family: + GGML_LOG_INFO("%s: GPU name: %s (%s)\n", __func__, dev->props.name, dev->props.desc); + + // determine max supported GPU family + // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf + // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf + { + for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) { + if ([dev->mtl_device supportsFamily:i]) { + dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1; + GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, dev->props.gpu_family, i); + break; + } } - } - for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) { - if ([dev->mtl_device supportsFamily:i]) { - GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i); - break; +#if TARGET_CPU_X86_64 + for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) { + if ([dev->mtl_device supportsFamily:i]) { + GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i); + break; + } } - } +#endif - for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) { - if ([dev->mtl_device supportsFamily:i]) { - GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i); - break; + for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) { + if ([dev->mtl_device supportsFamily:i]) { + GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i); + break; + } } } - } - GGML_LOG_INFO("%s: simdgroup reduction = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false"); - GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm ? "true" : "false"); - GGML_LOG_INFO("%s: has unified memory = %s\n", __func__, dev->props.has_unified_memory ? "true" : "false"); - GGML_LOG_INFO("%s: has bfloat = %s\n", __func__, dev->props.has_bfloat ? "true" : "false"); - GGML_LOG_INFO("%s: has tensor = %s\n", __func__, dev->props.has_tensor ? "true" : "false"); - GGML_LOG_INFO("%s: use residency sets = %s\n", __func__, dev->props.use_residency_sets ? "true" : "false"); - GGML_LOG_INFO("%s: use shared buffers = %s\n", __func__, dev->props.use_shared_buffers ? "true" : "false"); + GGML_LOG_INFO("%s: simdgroup reduction = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false"); + GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm ? "true" : "false"); + GGML_LOG_INFO("%s: has unified memory = %s\n", __func__, dev->props.has_unified_memory ? "true" : "false"); + GGML_LOG_INFO("%s: has bfloat = %s\n", __func__, dev->props.has_bfloat ? "true" : "false"); + GGML_LOG_INFO("%s: has tensor = %s\n", __func__, dev->props.has_tensor ? "true" : "false"); + GGML_LOG_INFO("%s: use residency sets = %s\n", __func__, dev->props.use_residency_sets ? "true" : "false"); + GGML_LOG_INFO("%s: use shared buffers = %s\n", __func__, dev->props.use_shared_buffers ? "true" : "false"); #if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15) - if (@available(macOS 10.12, iOS 16.0, *)) { - GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6); - } + if (@available(macOS 10.12, iOS 16.0, *)) { + GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6); + } #endif + } } } @@ -864,19 +1374,23 @@ ggml_metal_device_t ggml_metal_device_init(int device) { void ggml_metal_device_free(ggml_metal_device_t dev) { assert(dev != NULL); - ggml_metal_rsets_free(dev->rsets); + @autoreleasepool { + ggml_metal_fusion_info_free(dev->finfo); - ggml_metal_library_free(dev->library); - dev->library = NULL; + ggml_metal_rsets_free(dev->rsets); - if (dev->mtl_queue) { - [dev->mtl_queue release]; - dev->mtl_queue = nil; - } + ggml_metal_library_free(dev->library); + dev->library = NULL; - if (dev->mtl_device) { - [dev->mtl_device release]; - dev->mtl_device = nil; + if (dev->mtl_queue) { + [dev->mtl_queue release]; + dev->mtl_queue = nil; + } + + if (dev->mtl_device) { + [dev->mtl_device release]; + dev->mtl_device = nil; + } } free(dev); @@ -927,17 +1441,17 @@ void ggml_metal_device_rsets_keep_alive(ggml_metal_device_t dev) { return; } - atomic_store_explicit(&dev->rsets->d_loop, 2*dev->rsets->keep_alive_s, memory_order_relaxed); + atomic_store_explicit(&dev->rsets->d_loop, dev->rsets->loops_per_s*dev->rsets->keep_alive_s, memory_order_relaxed); } struct ggml_metal_event { - void * obj; // id + void * obj; // id atomic_int value; }; void ggml_metal_event_encode_signal(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf_raw) { - id event = (id)ev->obj; + id event = (id)ev->obj; id cmd_buf = (id) cmd_buf_raw; @@ -945,7 +1459,7 @@ void ggml_metal_event_encode_signal(ggml_metal_event_t ev, ggml_metal_cmd_buf_t } void ggml_metal_event_encode_wait(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf_raw) { - id event = (id)ev->obj; + id event = (id)ev->obj; id cmd_buf = (id) cmd_buf_raw; @@ -953,7 +1467,7 @@ void ggml_metal_event_encode_wait(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cm } ggml_metal_event_t ggml_metal_device_event_init(ggml_metal_device_t dev) { - id event = [dev->mtl_device newEvent]; + id event = [dev->mtl_device newSharedEvent]; ggml_metal_event_t ev = calloc(1, sizeof(struct ggml_metal_event)); @@ -964,35 +1478,64 @@ ggml_metal_event_t ggml_metal_device_event_init(ggml_metal_device_t dev) { } void ggml_metal_device_event_free(ggml_metal_device_t dev, ggml_metal_event_t ev) { - id event = ev->obj; - [event release]; + @autoreleasepool { + id event = ev->obj; + [event release]; - free(ev); + free(ev); - GGML_UNUSED(dev); + GGML_UNUSED(dev); + } } void ggml_metal_device_event_synchronize(ggml_metal_device_t dev, ggml_metal_event_t ev) { - @autoreleasepool { - id event = ev->obj; - - id cmd_buf = [dev->mtl_queue commandBuffer]; - [cmd_buf encodeWaitForEvent:event value:atomic_load_explicit(&ev->value, memory_order_relaxed)]; - [cmd_buf commit]; - [cmd_buf waitUntilCompleted]; + id event = ev->obj; + const bool res = [event waitUntilSignaledValue:atomic_load_explicit(&ev->value, memory_order_relaxed) timeoutMS:60000]; + if (!res) { + GGML_ABORT("%s: failed to wait for event\n", __func__); } + + GGML_UNUSED(dev); } void ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t * total) { if (@available(macOS 10.12, iOS 16.0, *)) { - *total = dev->mtl_device.recommendedMaxWorkingSetSize; - *free = *total - dev->mtl_device.currentAllocatedSize; + *total = dev->mtl_device.recommendedMaxWorkingSetSize; + size_t cur = dev->mtl_device.currentAllocatedSize; + // it's possible to allocate more than `recommendedMaxWorkingSetSize` + *free = *total > cur ? *total - cur : 0; } else { *free = 0; *total = 0; } } +static bool ggml_metal_supports_mul_mat_op( + bool has_simdgroup_reduction, + const struct ggml_tensor * op, + bool src0_f16_has_mv, + bool mm_path) { + if (!has_simdgroup_reduction || + op->src[0]->type == GGML_TYPE_NVFP4 || + op->src[0]->type == GGML_TYPE_TQ1_0) { + return false; + } + + if (op->src[1]->type != GGML_TYPE_F16) { + return true; + } + + if (op->src[0]->type == GGML_TYPE_BF16) { + return false; + } + + if (src0_f16_has_mv && op->src[0]->type == GGML_TYPE_F16) { + return true; + } + + return mm_path; +} + bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tensor * op) { const bool has_simdgroup_mm = dev->props.has_simdgroup_mm; const bool has_simdgroup_reduction = dev->props.has_simdgroup_reduction; @@ -1048,6 +1591,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te default: return false; } + case GGML_OP_SILU_BACK: + return (op->src[0]->type == GGML_TYPE_F32) && + (op->src[1]->type == GGML_TYPE_F32) && + (op->type == GGML_TYPE_F32) && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op->src[1]) && + ggml_is_contiguous(op) && + ggml_are_same_shape(op->src[0], op->src[1]); case GGML_OP_GLU: switch (ggml_get_glu_op(op)) { case GGML_GLU_OP_REGLU: @@ -1056,7 +1607,8 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_GLU_OP_SWIGLU_OAI: case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: - return ggml_is_contiguous_1(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; + case GGML_GLU_OP_SWIGLU_CLAMP: + return ggml_is_contiguous_1(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); default: return false; } @@ -1065,13 +1617,40 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_VIEW: case GGML_OP_TRANSPOSE: case GGML_OP_PERMUTE: - case GGML_OP_CONCAT: return true; + case GGML_OP_CONCAT: + { + const enum ggml_type src0_type = op->src[0]->type; + const enum ggml_type src1_type = op->src[1]->type; + if (src0_type != src1_type || src0_type != op->type) { + return false; + } + switch (src0_type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_I8: + case GGML_TYPE_I16: + case GGML_TYPE_I32: + case GGML_TYPE_I64: + return true; + case GGML_TYPE_BF16: + return has_bfloat; + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return true; + default: + return false; + } + } case GGML_OP_ADD: case GGML_OP_SUB: case GGML_OP_MUL: case GGML_OP_DIV: case GGML_OP_ADD_ID: + return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->src[0]->type == op->src[1]->type); case GGML_OP_ACC: return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_REPEAT: @@ -1082,6 +1661,11 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_COL2IM_1D: + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) && + op->type == op->src[0]->type && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op); case GGML_OP_CONV_3D: return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && @@ -1109,6 +1693,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_RMS_NORM: return has_simdgroup_reduction && (ggml_is_contiguous_rows(op->src[0])); case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: return true; case GGML_OP_IM2COL: return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32); @@ -1117,6 +1702,10 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32); + case GGML_OP_CONV_2D_DW: + return op->src[1]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32); case GGML_OP_UPSCALE: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_POOL_1D: @@ -1124,22 +1713,17 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_POOL_2D: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_PAD: - // TODO: add circular padding support for metal, see https://github.com/ggml-org/llama.cpp/pull/16985 - if (ggml_get_op_params_i32(op, 8) != 0) { - return false; - } - - return (ggml_get_op_params_i32(op, 0) == 0) && (ggml_get_op_params_i32(op, 2) == 0) && - (ggml_get_op_params_i32(op, 4) == 0) && (ggml_get_op_params_i32(op, 6) == 0); case GGML_OP_PAD_REFLECT_1D: case GGML_OP_TIMESTEP_EMBEDDING: - case GGML_OP_LEAKY_RELU: return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_LEAKY_RELU: + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; case GGML_OP_ARGSORT: case GGML_OP_TOP_K: case GGML_OP_ARANGE: - case GGML_OP_ROLL: return true; + case GGML_OP_ROLL: + return ggml_is_contiguous(op->src[0]); case GGML_OP_FLASH_ATTN_EXT: // for new head sizes, add checks here if (op->src[0]->ne[0] != 32 && @@ -1158,6 +1742,15 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[0]->ne[0] != 576) { return false; } + // the kernels exist for K == V and for these K > V pairs only + if (op->src[1]->ne[0] != op->src[2]->ne[0] && + !(op->src[1]->ne[0] == 96 && op->src[2]->ne[0] == 64) && + !(op->src[1]->ne[0] == 128 && op->src[2]->ne[0] == 96) && + !(op->src[1]->ne[0] == 192 && op->src[2]->ne[0] == 128) && + !(op->src[1]->ne[0] == 320 && op->src[2]->ne[0] == 256) && + !(op->src[1]->ne[0] == 576 && op->src[2]->ne[0] == 512)) { + return false; + } if (op->src[1]->type != op->src[2]->type) { return false; } @@ -1179,18 +1772,93 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return false; } return has_simdgroup_mm; // TODO: over-restricted for vec-kernels - case GGML_OP_SSM_CONV: + case GGML_OP_LIGHTNING_INDEXER: + if (op->src[0]->ne[0] != OP_LIGHTNING_INDEXER_DK) { + return false; + } + if (!has_simdgroup_mm || + op->src[0]->type != GGML_TYPE_F32 || + op->src[2]->type != GGML_TYPE_F32 || + op->src[3]->type != GGML_TYPE_F16 || + op->type != GGML_TYPE_F32 || + !ggml_is_contiguous_rows(op->src[0]) || + !ggml_is_contiguous_rows(op->src[1]) || + !ggml_is_contiguous_rows(op->src[2]) || + !ggml_is_contiguous_rows(op->src[3])) { + return false; + } + switch (op->src[1]->type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return true; + case GGML_TYPE_BF16: + return has_bfloat; + default: + return false; + } + case GGML_OP_DSV4_HC_COMB: + return has_simdgroup_reduction && + op->src[0]->type == GGML_TYPE_F32 && + op->src[1]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + op->src[0]->ne[0] == 24 && + op->src[1]->ne[0] >= 3 && + op->src[2]->ne[0] == 24 && + ggml_is_contiguous_rows(op->src[0]) && + ggml_is_contiguous_rows(op->src[1]) && + ggml_is_contiguous_rows(op->src[2]); + case GGML_OP_DSV4_HC_PRE: + return has_simdgroup_reduction && + op->src[0]->type == GGML_TYPE_F32 && + op->src[1]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + ggml_is_contiguous_rows(op->src[0]) && + ggml_is_contiguous_rows(op->src[1]); + case GGML_OP_DSV4_HC_POST: + return has_simdgroup_reduction && + op->src[0]->type == GGML_TYPE_F32 && + op->src[1]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && + (op->src[3] == NULL || op->src[3]->type == GGML_TYPE_F32) && + op->type == GGML_TYPE_F32 && + op->src[1]->ne[1] == 4 && + op->src[2]->ne[0] == 4 && + (op->src[3] == NULL || (op->src[3]->ne[0] == 4 && op->src[3]->ne[1] == 4)) && + ggml_is_contiguous_rows(op->src[0]) && + ggml_is_contiguous_rows(op->src[1]) && + ggml_is_contiguous_rows(op->src[2]) && + (op->src[3] == NULL || ggml_is_contiguous_rows(op->src[3])); case GGML_OP_SSM_SCAN: return has_simdgroup_reduction; + case GGML_OP_SSM_CONV: + return has_simdgroup_reduction; case GGML_OP_RWKV_WKV6: case GGML_OP_RWKV_WKV7: return true; case GGML_OP_GATED_DELTA_NET: return has_simdgroup_reduction && op->src[2]->ne[0] % 32 == 0; case GGML_OP_SOLVE_TRI: + return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_MUL_MAT: + // the FWHT kernels read an F16 source directly; every other F16 src1 path + // still goes through ggml_metal_supports_mul_mat_op + if (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F16 && + ggml_metal_op_mul_mat_use_fwht(op, dev->props.max_theadgroup_memory_size)) { + return has_simdgroup_reduction; + } + return ggml_metal_supports_mul_mat_op( + has_simdgroup_reduction, op, true, + ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm)); case GGML_OP_MUL_MAT_ID: - return has_simdgroup_reduction && op->src[0]->type != GGML_TYPE_NVFP4; + return ggml_metal_supports_mul_mat_op( + has_simdgroup_reduction, op, false, + ggml_metal_op_mul_mat_id_use_mm(op, has_simdgroup_mm)); case GGML_OP_SET: case GGML_OP_CPY: case GGML_OP_DUP: @@ -1204,11 +1872,13 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_TYPE_BF16: case GGML_TYPE_Q8_0: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_TQ2_0: case GGML_TYPE_I32: return true; default: @@ -1231,11 +1901,13 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return false; } case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: + case GGML_TYPE_TQ2_0: switch (op->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: @@ -1250,9 +1922,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te }; } case GGML_OP_GET_ROWS: - return op->src[0]->type != GGML_TYPE_NVFP4; + return op->src[0]->type != GGML_TYPE_NVFP4 && + op->src[0]->type != GGML_TYPE_TQ1_0; case GGML_OP_SET_ROWS: { + if (op->src[0]->type == GGML_TYPE_F16) { + return op->type == GGML_TYPE_F16; + } + if (op->src[0]->type != GGML_TYPE_F32) { return false; } @@ -1267,6 +1944,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_TQ2_0: return true; default: return false; @@ -1286,6 +1964,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return &dev->props; } +static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) { + dev->props.has_tensor = false; +} + +struct ggml_metal_fusion_info * ggml_metal_device_get_fusion_info(ggml_metal_device_t dev) { + return dev->finfo; +} + // // device buffers // @@ -1392,6 +2078,7 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) { if (buf->rset) { [buf->rset endResidency]; [buf->rset removeAllAllocations]; + [buf->rset commit]; [buf->rset release]; } } @@ -1586,13 +2273,15 @@ ggml_metal_buffer_t ggml_metal_buffer_map(ggml_metal_device_t dev, void * ptr, s } void ggml_metal_buffer_free(ggml_metal_buffer_t buf) { - ggml_metal_device_rsets_rm(buf->dev, buf->rset); + @autoreleasepool { + ggml_metal_device_rsets_rm(buf->dev, buf->rset); - for (int i = 0; i < buf->n_buffers; i++) { - [buf->buffers[i].metal release]; - } + for (int i = 0; i < buf->n_buffers; i++) { + [buf->buffers[i].metal release]; + } - ggml_metal_buffer_rset_free(buf); + ggml_metal_buffer_rset_free(buf); + } if (buf->is_shared && buf->owned) { #if TARGET_OS_OSX @@ -1692,6 +2381,8 @@ void ggml_metal_buffer_set_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * dispatch_release(completion_semaphore); //[cmd_buf waitUntilCompleted]; + + [buf_src release]; } } @@ -1730,9 +2421,52 @@ void ggml_metal_buffer_get_tensor(ggml_metal_buffer_t buf, const struct ggml_ten [cmd_buf commit]; [cmd_buf waitUntilCompleted]; + + [buf_dst release]; } } +bool ggml_metal_buffer_cpy_tensor(ggml_metal_buffer_t buf_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) { + ggml_metal_buffer_t buf_src = (ggml_metal_buffer_t)src->buffer->context; + + const size_t size = ggml_nbytes(src); + + // if both buffers are shared, we can use memcpy directly + if (buf_dst->is_shared && buf_src->is_shared) { + memcpy(dst->data, src->data, size); + return true; + } + + // for private buffers, we need to use Metal blit commands + @autoreleasepool { + struct ggml_metal_buffer_id bid_src = ggml_metal_buffer_get_id(buf_src, src); + struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(buf_dst, dst); + + if (bid_src.metal == nil || bid_dst.metal == nil) { + return false; + } + + id cmd_buf = [buf_dst->dev->mtl_queue commandBufferWithUnretainedReferences]; + + { + id encoder = [cmd_buf blitCommandEncoder]; + + [encoder copyFromBuffer:bid_src.metal + sourceOffset:bid_src.offs + toBuffer:bid_dst.metal + destinationOffset:bid_dst.offs + size:size]; + + [encoder endEncoding]; + } + + [cmd_buf commit]; + [cmd_buf waitUntilCompleted]; + } + + return true; +} + void ggml_metal_buffer_clear(ggml_metal_buffer_t buf, uint8_t value) { if (buf->is_shared) { memset(buf->all_data, value, buf->all_size); diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp new file mode 100644 index 0000000..020cad2 --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.cpp @@ -0,0 +1,1214 @@ +#include "ggml-metal-fusion.h" + +#include "ggml-metal-common.h" +#include "ggml-metal-device.h" + +#include "ggml-backend-impl.h" + +#include +#include +#include +#include +#include +#include + +// derive the non-empty op sequence from the raw `ops_all` sequence +static std::vector ggml_metal_fusion_filter_ops(const std::vector & ops_all) { + std::vector ops; + + for (ggml_op op : ops_all) { + if (!ggml_op_is_empty(op)) { + ops.push_back(op); + } + } + + return ops; +} + +struct ggml_metal_fusion { + ggml_metal_fusion_id id; + + std::vector ops; // non-empty op sequence, derived from ops_all + std::vector ops_all; // full raw op sequence (may include empty RESHAPE/VIEW nodes) + std::vector outs; // additional fused output nodes, relative to ops + + // if unsafe: the generic chain/shape + ggml_can_fuse_subgraph checks are skipped and the + // check callback below is the sole validator (used for patterns that are not elision chains, + // e.g. the gdn + cache-cpy write-through fusion) + bool unsafe; + + // extra backend constraints on top of ggml_can_fuse_subgraph + // nodes[j] is the j-th node of the pattern; node_idxs[idx + j] is its raw graph index + bool (*check)(const struct ggml_metal_fusion * fusion, + const struct ggml_tensor * const * nodes, + const struct ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode); + + ggml_metal_fusion( + ggml_metal_fusion_id id, + const std::vector & ops_all, + const std::vector & outs, + bool unsafe, + bool (*check)(const struct ggml_metal_fusion * fusion, + const struct ggml_tensor * const * nodes, + const struct ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode)) + : id(id), + ops(ggml_metal_fusion_filter_ops(ops_all)), + ops_all(ops_all), + outs(outs), + unsafe(unsafe), + check(check) { + } +}; + +ggml_metal_fusion_id ggml_metal_fusion_get_id(const ggml_metal_fusion * fusion) { + return fusion->id; +} + +// ---- helpers ------------------------------------------------------------- + +// true if two tensors live in the same Metal buffer +static bool ggml_metal_fusion_same_buffer(const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b) { + return false; + } + + ggml_backend_buffer_t ba = a->view_src ? a->view_src->buffer : a->buffer; + ggml_backend_buffer_t bb = b->view_src ? b->view_src->buffer : b->buffer; + + ggml_metal_buffer_t ca = (ggml_metal_buffer_t) ba->context; + ggml_metal_buffer_t cb = (ggml_metal_buffer_t) bb->context; + + return ggml_metal_buffer_get_id(ca, a).metal == ggml_metal_buffer_get_id(cb, b).metal; +} + +// true if the memory of two tensors overlaps in the same Metal buffer +static bool ggml_metal_fusion_overlap(const ggml_tensor * a, const ggml_tensor * b) { + ggml_backend_buffer_t ba = a->view_src ? a->view_src->buffer : a->buffer; + ggml_backend_buffer_t bb = b->view_src ? b->view_src->buffer : b->buffer; + + const ggml_metal_buffer_id bid_a = ggml_metal_buffer_get_id((ggml_metal_buffer_t) ba->context, a); + const ggml_metal_buffer_id bid_b = ggml_metal_buffer_get_id((ggml_metal_buffer_t) bb->context, b); + + if (bid_a.metal == nullptr || bid_a.metal != bid_b.metal) { + return false; + } + + return bid_a.offs <= bid_b.offs + ? bid_b.offs - bid_a.offs < ggml_nbytes(a) + : bid_a.offs - bid_b.offs < ggml_nbytes(b); +} + +// ---- pattern checks ------------------------------------------------------ + +// NORM/RMS_NORM + MUL + ADD: the weight/bias of each fused step must match the norm input +// width, be contiguous rows, and the fused outputs must stay F32 +static bool ggml_metal_fusion_check_norm( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(mode); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + + GGML_ASSERT(fusion->ops.size() >= 2); + + if (fusion->id == GGML_METAL_FUSION_NORM_SCALE) { + GGML_ASSERT(fusion->ops.size() == 2); + + const ggml_tensor * scale = nodes[1]; + if (scale->op != GGML_OP_SCALE || scale->src[0] != nodes[0] || scale->src[1] || + scale->type != GGML_TYPE_F32) { + return false; + } + + return true; + } + + for (int j = 1; j < (int) fusion->ops.size(); j++) { + // the fused MUL/ADD must read the previous node as src0 + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + // the weight/bias must have the same row width as the norm input + if (nodes[j]->src[1]->ne[0] != nodes[0]->ne[0]) { + return false; + } + + if (!ggml_is_contiguous_rows(nodes[j]->src[1])) { + return false; + } + + if (nodes[j]->type != GGML_TYPE_F32) { + return false; + } + } + + return true; +} + +// SSM_CONV + UNARY (silu) +static bool ggml_metal_fusion_check_ssm_conv_silu( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + GGML_UNUSED(mode); + + const ggml_tensor * conv = nodes[0]; + const ggml_tensor * un = nodes[1]; + + if (conv->op != GGML_OP_SSM_CONV || un->op != GGML_OP_UNARY || un->src[0] != conv || un->src[1]) { + return false; + } + + if (ggml_get_unary_op(un) != GGML_UNARY_OP_SILU) { + return false; + } + + if (conv->type != GGML_TYPE_F32 || un->type != GGML_TYPE_F32 || !ggml_is_contiguous_rows(un)) { + return false; + } + + return true; +} + +// ADD x N: each ADD reads the previous ADD as src0, and all addends must share layout +// (and, in FULL mode, live in the same Metal buffer) +static bool ggml_metal_fusion_check_add_chain( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + GGML_ASSERT(fusion->ops.size() >= 2); + + for (int j = 1; j < (int) fusion->ops.size(); j++) { + if (nodes[j]->src[0] != nodes[j - 1]) { + return false; + } + + if (!ggml_are_same_layout(nodes[j]->src[1], nodes[j - 1]->src[1])) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!ggml_metal_fusion_same_buffer(nodes[j]->src[1], nodes[0]->src[1])) { + return false; + } + } + } + + return true; +} + +// GATED_DELTA_NET + CPY: the trailing cpy scatters the gdn state snapshots into the recurrent +// cache, so the gdn kernel writes them straight to the cache and the cpy is elided. +// mirrors ggml_metal_op_can_fuse_gdn_cache (PR #25788). the gdn output has other consumers (the +// attn scores view), so unlike the other patterns this is not an elision chain: the structural +// checks live entirely in this callback (unsafe = true). +static bool ggml_metal_fusion_check_gdn_cache( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + + const ggml_tensor * gdn = nodes[0]; + const ggml_tensor * cpy = nodes[1]; + + // the kernel skips the snapshot tail, so the gdn output must not be a graph output + if (gdn->type != GGML_TYPE_F32 || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + if (cpy->op != GGML_OP_CPY || (cpy->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return false; + } + + const int64_t S_v = gdn->src[2]->ne[0]; + const int64_t H = gdn->src[2]->ne[1]; + const int64_t n_tokens = gdn->src[2]->ne[2]; + const int64_t n_seqs = gdn->src[2]->ne[3]; + const int64_t K = ggml_get_op_params_i32(gdn, 0); + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + const int64_t D = S_v * S_v * H; + const int64_t n_written = std::min(n_tokens, K); + + const ggml_tensor * src = cpy->src[0]; // gdn snapshot tail view + const ggml_tensor * dst = cpy->src[1]; // cache view + + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != gdn || + src->view_offs != tail_off || !ggml_is_contiguous(src)) { + return false; + } + + const int64_t expected_ne[GGML_MAX_DIMS] = { D, n_seqs, n_written, 1 }; + if (dst->type != GGML_TYPE_F32 || + !std::equal(expected_ne, expected_ne + GGML_MAX_DIMS, dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || + dst->nb[1] != ggml_row_size(GGML_TYPE_F32, D)) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + // the cache must be allocated so the kernel can write straight to its buffer + if (dst->data == nullptr) { + return false; + } + } + + return true; +} + +// MUL + SIN + SQR + MUL + ADD (snake activation) +static bool ggml_metal_fusion_check_snake( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(fusion); + GGML_UNUSED(mode); + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + + const ggml_tensor * mul0 = nodes[0]; + const ggml_tensor * sin_node = nodes[1]; + const ggml_tensor * sqr = nodes[2]; + const ggml_tensor * mul1 = nodes[3]; + const ggml_tensor * add = nodes[4]; + + // x carries the full activation shape, a is the broadcast operand + const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + + // mul1 reads sqr and inv_b in either operand order + const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + + // closure check: the trailing add reads the same x as the leading mul + const ggml_tensor * x_in_add = (add->src[0] == mul1) ? add->src[1] : add->src[0]; + + // x is in the supported whitelist and every chain intermediate shares x's type. + // a and inv_b bind as device const float * in the kernel, so they stay F32. + const bool types_ok = + (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16 || x->type == GGML_TYPE_BF16) && + (a->type == GGML_TYPE_F32) && (inv_b->type == GGML_TYPE_F32) && + (mul0->type == x->type) && (sin_node->type == x->type) && + (sqr->type == x->type) && (mul1->type == x->type) && + (add->type == x->type); + + // a / inv_b collapse to [1, C, 1, 1], x and add stay 2D + const bool shape_ok = ggml_are_same_shape(a, inv_b) && a->ne[0] == 1 && a->ne[1] == x->ne[1]; + const bool dim_ok = + (x->ne[2] == 1) && (x->ne[3] == 1) && + (add->ne[2] == 1) && (add->ne[3] == 1) && + (a->ne[2] == 1) && (a->ne[3] == 1) && + (inv_b->ne[2] == 1) && (inv_b->ne[3] == 1); + + // kernel reads x[idx] and a[c] / inv_b[c] linearly, so every operand is contiguous + const bool contig_ok = + ggml_is_contiguous(x) && ggml_is_contiguous(add) && + ggml_is_contiguous(a) && ggml_is_contiguous(inv_b); + + return types_ok && shape_ok && dim_ok && contig_ok && x_in_add == x; +} + +#define GGML_METAL_TOPK_MOE_MAX_EXPERTS 1024 + +// SOFT_MAX + ARGSORT + GET_ROWS (plus optional norm/scale) for MoE routing. +// This is a multi-output elision chain: the fused kernel writes both the selected +// expert ids and the gathered/normalized routing weights. +static const std::vector ops_topk_moe = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS +}; +static const std::vector ops_topk_moe_scale = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_SCALE +}; +static const std::vector ops_topk_moe_norm = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE +}; +static const std::vector ops_topk_moe_norm_scale = { + GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS, + GGML_OP_RESHAPE, GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV, GGML_OP_RESHAPE, GGML_OP_SCALE +}; + +static bool ggml_metal_fusion_check_topk_moe( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_ASSERT(fusion->ops.size() >= 3); + GGML_UNUSED(nodes); + + const int n_ops = (int) fusion->ops.size(); + + const bool with_norm = n_ops >= 6; + const bool with_scale = n_ops == 4 || n_ops == 7; + + // the fusion table operates on the non-empty node sequence; the raw graph also + // contains the RESHAPE/VIEW nodes that the fused kernel elides. + const std::vector & ops_all = fusion->ops_all; + + const int raw_start = node_idxs[idx]; + int raw_end = node_idxs[idx + n_ops - 1]; + + // the norm variant ends with a RESHAPE that the non-empty sequence filters out; + // include it so the output use-count check sees the real final routing tensor + if (with_norm && !with_scale) { + if (raw_end + 1 >= gf->n_nodes) { + return false; + } + const ggml_tensor * trailing_reshape = gf->nodes[raw_end + 1]; + if (trailing_reshape->op != GGML_OP_RESHAPE || trailing_reshape->src[0] != gf->nodes[raw_end]) { + return false; + } + raw_end++; + } + + const int raw_count = raw_end - raw_start + 1; + if (raw_count != (int) ops_all.size()) { + return false; + } + + int raw_idxs[GGML_METAL_FUSION_MAX]; + for (int i = 0; i < raw_count; ++i) { + raw_idxs[i] = raw_start + i; + if (gf->nodes[raw_start + i]->op != ops_all[i]) { + return false; + } + } + + const ggml_tensor * softmax = gf->nodes[raw_start]; + const ggml_tensor * probs_reshaped = gf->nodes[raw_start + 1]; + const ggml_tensor * argsort = gf->nodes[raw_start + 2]; + const ggml_tensor * ids = gf->nodes[raw_start + 3]; + const ggml_tensor * get_rows = gf->nodes[raw_start + 4]; + const ggml_tensor * out = gf->nodes[raw_end]; + const ggml_tensor * logits = softmax->src[0]; + + // the fused kernel implements plain softmax only + float scale = 1.0f; + float max_bias = 0.0f; + memcpy(&scale, ((const int32_t *) softmax->op_params) + 0, sizeof(scale)); + memcpy(&max_bias, ((const int32_t *) softmax->op_params) + 1, sizeof(max_bias)); + if (scale != 1.0f || max_bias != 0.0f || softmax->src[1] || softmax->src[2]) { + return false; + } + + if (logits->type != GGML_TYPE_F32 || softmax->type != GGML_TYPE_F32 || + out->type != GGML_TYPE_F32 || ids->type != GGML_TYPE_I32) { + return false; + } + + const int64_t n_expert = logits->ne[0]; + const int64_t n_tokens = logits->ne[1]; + const int64_t n_expert_used = ids->ne[0]; + + // note: n_tokens == 0 (no-output batch) must match so that the packing stays shape-independent + if (n_expert <= 0 || n_expert_used <= 0 || n_expert_used > n_expert || + n_expert > GGML_METAL_TOPK_MOE_MAX_EXPERTS || n_expert_used > GGML_METAL_TOPK_MOE_MAX_EXPERTS) { + return false; + } + + if (logits->ne[2] != 1 || logits->ne[3] != 1 || + ids->ne[1] != n_tokens || ids->ne[2] != 1 || ids->ne[3] != 1 || + out->ne[0] != 1 || out->ne[1] != n_expert_used || out->ne[2] != n_tokens || out->ne[3] != 1) { + return false; + } + + if (!ggml_is_contiguous(logits) || !ggml_is_contiguous(out) || + ids->nb[0] != ggml_type_size(GGML_TYPE_I32) || + ids->nb[1] != ggml_type_size(GGML_TYPE_I32) * n_expert) { + return false; + } + + if (probs_reshaped->src[0] != softmax || argsort->src[0] != softmax || + ids->src[0] != argsort || get_rows->src[0] != probs_reshaped || get_rows->src[1] != ids) { + return false; + } + + if (with_norm) { + const ggml_tensor * weights_reshaped = gf->nodes[raw_start + 5]; + const ggml_tensor * sum_rows = gf->nodes[raw_start + 6]; + const ggml_tensor * clamp = gf->nodes[raw_start + 7]; + const ggml_tensor * div = gf->nodes[raw_start + 8]; + const ggml_tensor * out_reshaped = gf->nodes[raw_start + 9]; + + if (weights_reshaped->src[0] != get_rows || sum_rows->src[0] != weights_reshaped || + clamp->src[0] != sum_rows || div->src[0] != weights_reshaped || div->src[1] != clamp || + out_reshaped->src[0] != div) { + return false; + } + + if (with_scale) { + const ggml_tensor * scale_node = gf->nodes[raw_start + 10]; + if (scale_node->src[0] != out_reshaped) { + return false; + } + } + } else if (with_scale) { + const ggml_tensor * scale_node = gf->nodes[raw_start + 5]; + if (scale_node->src[0] != get_rows) { + return false; + } + } + + const int outputs[2] = { raw_start + 3, raw_end }; + if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, raw_count, ops_all.data(), outputs, 2)) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!logits->data || !out->data || !ids->data) { + return false; + } + } + + return true; +} + +#define GGML_METAL_MOE_REDUCE_MAX_EXPERTS 8 + +struct ggml_metal_moe_reduce_match { + const ggml_tensor * experts; + const ggml_tensor * weights; + const ggml_tensor * dst; + int node_count; +}; + +static bool ggml_metal_fusion_match_moe_reduce( + const ggml_cgraph * gf, int node_idx, const std::vector & ops_all, + ggml_metal_moe_reduce_match * match) { + if (match == nullptr || node_idx < 0 || node_idx + (int) ops_all.size() > gf->n_nodes) { + return false; + } + + const ggml_tensor * mul = gf->nodes[node_idx]; + if (mul->op != GGML_OP_MUL || mul->type != GGML_TYPE_F32) { + return false; + } + + // MUL, then one VIEW per expert, then one ADD per additional expert + const int raw_count = (int) ops_all.size(); + const int n_expert_used = raw_count / 2; + + if (n_expert_used < 2 || n_expert_used > GGML_METAL_MOE_REDUCE_MAX_EXPERTS || + raw_count != 2 * n_expert_used) { + return false; + } + + int n_views = 0; + while (node_idx + 1 + n_views < gf->n_nodes && + gf->nodes[node_idx + 1 + n_views]->op == GGML_OP_VIEW) { + n_views++; + } + + if (n_views != n_expert_used) { + return false; + } + + for (int i = n_expert_used + 1; i < raw_count; ++i) { + if (gf->nodes[node_idx + i]->op != GGML_OP_ADD) { + return false; + } + } + + int raw_idxs[GGML_METAL_FUSION_MAX]; + for (int i = 0; i < raw_count; ++i) { + raw_idxs[i] = node_idx + i; + if (gf->nodes[node_idx + i]->op != ops_all[i]) { + return false; + } + } + + const ggml_tensor * experts = mul->src[0]; + const ggml_tensor * weights = mul->src[1]; + const ggml_tensor * dst = gf->nodes[node_idx + raw_count - 1]; + + if (experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + const int64_t n_embd = experts->ne[0]; + const int64_t n_tokens = experts->ne[2]; + + // note: n_tokens == 0 (no-output batch) must match so that the packing stays shape-independent + if (n_embd <= 0 || experts->ne[1] != n_expert_used || experts->ne[3] != 1 || + weights->ne[0] != 1 || weights->ne[1] != n_expert_used || weights->ne[2] != n_tokens || weights->ne[3] != 1 || + dst->ne[0] != n_embd || dst->ne[1] != n_tokens || dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + + if (!ggml_is_contiguous(experts) || !ggml_is_contiguous(weights) || !ggml_is_contiguous(dst)) { + return false; + } + + for (int i = 1; i <= n_expert_used; ++i) { + const ggml_tensor * view = gf->nodes[node_idx + i]; + if (view->view_src != mul || view->src[0] != mul || + view->view_offs != (size_t) (i - 1) * mul->nb[1] || + view->ne[0] != n_embd || view->ne[1] != n_tokens || + view->nb[1] != mul->nb[2]) { + return false; + } + } + + const ggml_tensor * prev_add = nullptr; + for (int j = 1; j < n_expert_used; ++j) { + const ggml_tensor * add = gf->nodes[node_idx + n_expert_used + j]; + const ggml_tensor * rhs = gf->nodes[node_idx + j + 1]; + const ggml_tensor * lhs = j == 1 ? gf->nodes[node_idx + 1] : prev_add; + if (add->src[0] != lhs || add->src[1] != rhs) { + return false; + } + prev_add = add; + } + + const int outputs[1] = { node_idx + raw_count - 1 }; + if (!ggml_can_fuse_subgraph_ext(gf, raw_idxs, raw_count, ops_all.data(), outputs, 1)) { + return false; + } + + match->experts = experts; + match->weights = weights; + match->dst = dst; + match->node_count = raw_count; + return true; +} + +static bool ggml_metal_fusion_check_moe_reduce( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(nodes); + + ggml_metal_moe_reduce_match match; + if (!ggml_metal_fusion_match_moe_reduce(gf, node_idxs[idx], fusion->ops_all, &match)) { + return false; + } + + if ((int) fusion->ops.size() != match.experts->ne[1]) { + return false; + } + + const int raw_end = node_idxs[idx] + match.node_count - 1; + if (node_idxs[idx + (int) fusion->ops.size() - 1] != raw_end) { + return false; + } + + if (mode == GGML_METAL_FUSION_FULL) { + if (!match.experts->data || !match.weights->data || !match.dst->data) { + return false; + } + + // the fused kernel reads experts and weights while writing dst, so dst must not overlap either input + if (ggml_metal_fusion_overlap(match.dst, match.experts) || ggml_metal_fusion_overlap(match.dst, match.weights)) { + return false; + } + } + + return true; +} + +// true if t is or views a tensor in a buffer marked as weights, such as a bias; the model loader marks its buffers before +// any graph is optimized, and tensors in unmarked or not yet allocated buffers count as non-weights in both phases +static bool ggml_metal_tensor_is_weight(const struct ggml_tensor * t) { + const ggml_tensor * base = t->view_src != NULL ? t->view_src : t; + + return base->buffer != NULL && ggml_backend_buffer_get_usage(base->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; +} + +static const struct ggml_tensor * ggml_metal_mul_mat_add_operand(const struct ggml_tensor * mm, const struct ggml_tensor * add) { + if (add->op != GGML_OP_ADD || (add->src[0] == mm) == (add->src[1] == mm)) { + return NULL; + } + + const ggml_tensor * other = add->src[0] == mm ? add->src[1] : add->src[0]; + + const bool ok = other->type == GGML_TYPE_F32 && add->type == GGML_TYPE_F32 && !ggml_metal_tensor_is_weight(other); + + return ok ? other : NULL; +} + +static const struct ggml_tensor * ggml_metal_mul_mat_add_residual(const struct ggml_tensor * mm, const struct ggml_tensor * add) { + const ggml_tensor * res = ggml_metal_mul_mat_add_operand(mm, add); + + const bool ok = res != NULL && ggml_are_same_shape(res, mm) && + ggml_is_contiguous(res) && ggml_is_contiguous(mm) && ggml_is_contiguous(add); + + return ok ? res : NULL; +} + +// MUL_MAT + ADD of an f32 non-weight: the reorder packs it without reading row counts, so ubatch sizes share one order; +// the encoder fuses only a same-shape residual in the few-row MMA store, which the sum may overlap only in place +static bool ggml_metal_fusion_check_mul_mat_add( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + const ggml_cgraph * gf, + const int * node_idxs, + int idx, + ggml_metal_fusion_mode mode) { + GGML_UNUSED(gf); + GGML_UNUSED(node_idxs); + GGML_UNUSED(idx); + GGML_UNUSED(fusion); + + const ggml_tensor * mm = nodes[0]; + const ggml_tensor * add = nodes[1]; + + if (ggml_metal_mul_mat_add_operand(mm, add) == nullptr || + !ggml_metal_op_mul_mat_may_use_mma(mm)) { + return false; + } + + if (mode == GGML_METAL_FUSION_STRUCTURAL) { + return true; + } + + const ggml_tensor * res = ggml_metal_mul_mat_add_residual(mm, add); + + if (res == nullptr || !ggml_metal_op_mul_mat_use_mma(mm)) { + return false; + } + + return !ggml_metal_fusion_overlap(add, mm->src[0]) && !ggml_metal_fusion_overlap(add, mm->src[1]) && + (add->data == res->data || !ggml_metal_fusion_overlap(add, res)); +} + +// ---- patterns ------------------------------------------------------------ + +static const std::vector ops_norm_mul = { GGML_OP_NORM, GGML_OP_MUL }; +static const std::vector ops_norm_mul_add = { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD }; +static const std::vector ops_norm_scale = { GGML_OP_NORM, GGML_OP_SCALE }; +static const std::vector ops_rms_norm_mul = { GGML_OP_RMS_NORM, GGML_OP_MUL }; +static const std::vector ops_rms_norm_mul_add = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }; +static const std::vector ops_rms_norm_scale = { GGML_OP_RMS_NORM, GGML_OP_SCALE }; + +static const std::vector ops_add_2 = { GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector ops_add_3 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector ops_add_4 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector ops_add_5 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector ops_add_6 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector ops_add_7 = { GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD }; +static const std::vector ops_snake = { GGML_OP_MUL, GGML_OP_SIN, GGML_OP_SQR, GGML_OP_MUL, GGML_OP_ADD }; + +static const std::vector ops_gdn_cache = { GGML_OP_GATED_DELTA_NET, GGML_OP_CPY }; + +static const std::vector ops_ssm_conv_silu = { GGML_OP_SSM_CONV, GGML_OP_UNARY }; + +static const std::vector ops_moe_reduce_2 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_ADD +}; +static const std::vector ops_moe_reduce_3 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector ops_moe_reduce_4 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector ops_moe_reduce_5 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector ops_moe_reduce_6 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector ops_moe_reduce_7 = { + GGML_OP_MUL, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; +static const std::vector ops_moe_reduce_8 = { + GGML_OP_MUL, + GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, GGML_OP_VIEW, + GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD, GGML_OP_ADD +}; + +static const std::vector ops_mul_mat_add = { GGML_OP_MUL_MAT, GGML_OP_ADD }; + +static const std::vector ggml_metal_fusions = { + { GGML_METAL_FUSION_NORM_MUL, ops_norm_mul, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_SCALE, ops_norm_scale, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL, ops_rms_norm_mul, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_MUL_ADD, ops_rms_norm_mul_add, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_NORM_SCALE, ops_rms_norm_scale, {}, false, ggml_metal_fusion_check_norm }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_2, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_3, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_4, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_5, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_6, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_ADD_CHAIN, ops_add_7, {}, false, ggml_metal_fusion_check_add_chain }, + { GGML_METAL_FUSION_SNAKE, ops_snake, {}, false, ggml_metal_fusion_check_snake }, + { GGML_METAL_FUSION_GDN_CACHE, ops_gdn_cache, {}, true, ggml_metal_fusion_check_gdn_cache }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_TOPK_MOE, ops_topk_moe_norm_scale, {1}, true, ggml_metal_fusion_check_topk_moe }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_2, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_3, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_4, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_5, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_6, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_7, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_MOE_REDUCE, ops_moe_reduce_8, {}, true, ggml_metal_fusion_check_moe_reduce }, + { GGML_METAL_FUSION_SSM_CONV_SILU, ops_ssm_conv_silu, {}, false, ggml_metal_fusion_check_ssm_conv_silu }, + { GGML_METAL_FUSION_MUL_MAT_ADD, ops_mul_mat_add, {}, false, ggml_metal_fusion_check_mul_mat_add }, +}; + +// ---- alloc deps ----------------------------------------------------------- + +static bool ggml_metal_fusion_match_raw_pattern( + const ggml_cgraph * gf, int node_idx, const std::vector & ops) { + if (node_idx < 0 || node_idx + (int) ops.size() > gf->n_nodes) { + return false; + } + + for (int i = 0; i < (int) ops.size(); ++i) { + if (gf->nodes[node_idx + i]->op != ops[i]) { + return false; + } + } + + return true; +} + +static void ggml_metal_fusion_add_pattern_alloc_deps( + void * user_data, + void (*add_alloc_dep)(void *, ggml_tensor *, ggml_tensor *), + ggml_cgraph * gf, + const ggml_metal_fusion * fusion, + int node_idx) { + const int last_node = node_idx + (int) fusion->ops_all.size() - 1; + + // keep all external inputs alive until the fused output + std::set seen; + for (int j = 0; j < (int) fusion->ops_all.size(); ++j) { + ggml_tensor * node = gf->nodes[node_idx + j]; + for (int s = 0; s < GGML_MAX_SRC; ++s) { + ggml_tensor * src = node->src[s]; + if (src && seen.insert(src).second) { + add_alloc_dep(user_data, src, gf->nodes[last_node]); + } + } + seen.insert(node); + } +} + +void ggml_metal_fusion_add_alloc_deps( + void * user_data, + void (*add_alloc_dep)(void *, ggml_tensor *, ggml_tensor *), + ggml_cgraph * gf) { + for (int i = 0; i < gf->n_nodes; ++i) { + const ggml_metal_fusion * best = nullptr; + int best_raw = 0; + + for (const ggml_metal_fusion & fusion : ggml_metal_fusions) { + if ((int) fusion.ops_all.size() <= best_raw) { + continue; + } + if (ggml_metal_fusion_match_raw_pattern(gf, i, fusion.ops_all)) { + best = &fusion; + best_raw = (int) fusion.ops_all.size(); + } + } + + if (best) { + ggml_metal_fusion_add_pattern_alloc_deps(user_data, add_alloc_dep, gf, best, i); + i += best_raw - 1; + } + } +} + +// ---- shared fusion info --------------------------------------------------- + +static std::string ggml_metal_fusion_label(const ggml_metal_fusion * fusion) { + GGML_ASSERT(fusion != nullptr); + + std::string label; + for (int j = 0; j < (int) fusion->ops.size(); j++) { + if (j > 0) { + label += '+'; + } + label += ggml_op_name(fusion->ops[j]); + } + return label; +} + +struct ggml_metal_fusion_info { + std::vector labels; + std::vector counts; + bool enabled; + bool stats; + bool labels_set; + int debug; +}; + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug) { + ggml_metal_fusion_info * finfo = new ggml_metal_fusion_info; + finfo->enabled = enabled; + finfo->stats = debug > 0; + finfo->labels_set = false; + finfo->debug = debug; + + if (finfo->stats) { + ggml_metal_fusion_info_labels_init(finfo); + } + + return finfo; +} + +void ggml_metal_fusion_info_free(ggml_metal_fusion_info * finfo) { + delete finfo; +} + +bool ggml_metal_fusion_info_enabled(const ggml_metal_fusion_info * finfo) { + return finfo->enabled; +} + +bool ggml_metal_fusion_info_stats(const ggml_metal_fusion_info * finfo) { + return finfo->stats; +} + +int ggml_metal_fusion_info_debug(const ggml_metal_fusion_info * finfo) { + return finfo->debug; +} + +int ggml_metal_fusion_info_n_fusions(const ggml_metal_fusion_info * finfo) { + return (int) finfo->labels.size(); +} + +const char * ggml_metal_fusion_info_label(const ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->labels.size()); + return finfo->labels[idx].c_str(); +} + +uint64_t ggml_metal_fusion_info_count(const ggml_metal_fusion_info * finfo, int idx) { + GGML_ASSERT(idx >= 0 && idx < (int) finfo->counts.size()); + return finfo->counts[idx]; +} + +void ggml_metal_fusion_info_count_fusion(ggml_metal_fusion_info * finfo, const ggml_metal_fusion * fusion) { + if (!finfo->stats || fusion == nullptr) { + return; + } + + const ptrdiff_t idx = fusion - ggml_metal_fusions.data(); + if (idx >= 0 && idx < (ptrdiff_t) finfo->counts.size()) { + finfo->counts[idx]++; + } +} + +void ggml_metal_fusion_info_set_enabled(ggml_metal_fusion_info * finfo, bool enabled) { + finfo->enabled = enabled; +} + +void ggml_metal_fusion_info_labels_init(ggml_metal_fusion_info * finfo) { + if (finfo->labels_set) { + return; + } + + finfo->labels.clear(); + finfo->counts.assign(ggml_metal_fusions.size(), 0); + finfo->labels.reserve(ggml_metal_fusions.size()); + + for (const ggml_metal_fusion & fusion : ggml_metal_fusions) { + finfo->labels.emplace_back(ggml_metal_fusion_label(&fusion)); + } + + finfo->labels_set = true; +} + +void ggml_metal_fusion_info_stats_init(ggml_metal_fusion_info * finfo) { + finfo->stats = true; + ggml_metal_fusion_info_labels_init(finfo); +} + +void ggml_metal_fusion_info_stats_reset(ggml_metal_fusion_info * finfo) { + std::fill(finfo->counts.begin(), finfo->counts.end(), 0); +} + +int ggml_metal_fusion_info_stats_get(const ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n) { + const int n_fusions = (int) finfo->labels.size(); + + if (labels == nullptr) { + return n_fusions; + } + + const int n_fill = std::min(n, n_fusions); + for (int i = 0; i < n_fill; i++) { + labels[i] = finfo->labels[i].c_str(); + if (counts != nullptr) { + counts[i] = finfo->counts[i]; + } + } + + return n_fill; +} + +// ---- memory-range checks ------------------------------------------------- + +// reject fusions where an external source overlaps any fused output. the fused +// kernels elide intermediate nodes, so only sources that are not part of the +// fused subgraph can cause read/write races with the output. +static bool ggml_metal_fusion_check_memory_ranges( + const ggml_metal_fusion * fusion, + const ggml_tensor * const * nodes, + int node_count) { + // some fused kernels write through a tensor that also appears as a source (e.g. the gdn + // cache cpy), so a source that is the same memory as the output is not an external read + // source + auto same_memory = [](const ggml_tensor * a, const ggml_tensor * b) { + if (a->data && b->data && a->data == b->data) { + return true; + } + for (const ggml_tensor * v = a; v; v = v->view_src) { + if (v == b) { + return true; + } + } + for (const ggml_tensor * v = b; v; v = v->view_src) { + if (v == a) { + return true; + } + } + return false; + }; + + auto nodes_overlap = [](const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b || !a->data || !b->data || !a->buffer || !b->buffer) { + return false; + } + + if (a->buffer != b->buffer) { + return false; + } + + const int64_t a_start = (int64_t) a->data; + const int64_t a_end = a_start + ggml_backend_buft_get_alloc_size(a->buffer->buft, a); + const int64_t b_start = (int64_t) b->data; + const int64_t b_end = b_start + ggml_backend_buft_get_alloc_size(b->buffer->buft, b); + + return (b_start <= a_start && a_start < b_end) || + (a_start <= b_start && b_start < a_end); + }; + + auto is_intermediate = [](const ggml_tensor * src, const ggml_tensor * const * nodes, int j) { + for (int k = 0; k < j; ++k) { + if (src == nodes[k]) { + return true; + } + for (const ggml_tensor * view_src = src->view_src; view_src; view_src = view_src->view_src) { + if (view_src == nodes[k]) { + return true; + } + } + } + return false; + }; + + auto check_dst = [&](const ggml_tensor * dst) { + for (int j = 0; j < node_count; ++j) { + for (int s = 0; s < GGML_MAX_SRC; ++s) { + const ggml_tensor * src = nodes[j]->src[s]; + if (!src || src->op == GGML_OP_NONE || same_memory(src, dst)) { + continue; + } + + if (nodes_overlap(dst, src) && !is_intermediate(src, nodes, j)) { + return false; + } + } + } + return true; + }; + + if (!check_dst(nodes[node_count - 1])) { + return false; + } + + for (int offset : fusion->outs) { + GGML_ASSERT(offset >= 0 && offset < node_count); + if (!check_dst(nodes[offset])) { + return false; + } + } + + return true; +} + +// ---- queries ------------------------------------------------------------- + +// find the longest pattern matching the node sequence starting at idx +// (idx is a position in node_idxs, which maps to graph node indices) +const ggml_metal_fusion * ggml_metal_fusion_next( + const ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out) { + const ggml_metal_fusion * res = nullptr; + int best = 1; + + for (const ggml_metal_fusion & fusion : ggml_metal_fusions) { + const int n_ops = (int) fusion.ops.size(); + + // only look for a longer match than the current best + if (n_ops <= best) { + continue; + } + if (idx + n_ops > n_idxs) { + continue; + } + + const ggml_tensor * nodes[GGML_METAL_FUSION_MAX]; + + // the op sequence must match exactly + bool ok = true; + for (int j = 0; j < n_ops; j++) { + nodes[j] = gf->nodes[node_idxs[idx + j]]; + if (nodes[j]->op != fusion.ops[j]) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + if (!fusion.unsafe) { + // common element-wise chain constraints: each node reads the previous one, + // and all nodes have the same shape + for (int j = 1; j < n_ops && ok; j++) { + if (nodes[j]->src[0] != nodes[j - 1] && nodes[j]->src[1] != nodes[j - 1]) { + ok = false; + break; + } + if (!ggml_are_same_shape(nodes[j], nodes[j - 1])) { + ok = false; + break; + } + } + if (!ok) { + continue; + } + + // primary output is the last node; additional outputs come from fusion.outs + int outputs_buf[GGML_METAL_FUSION_MAX]; + outputs_buf[0] = node_idxs[idx + n_ops - 1]; + for (size_t i = 0; i < fusion.outs.size(); ++i) { + const int out_offset = fusion.outs[i]; + GGML_ASSERT(out_offset >= 0 && out_offset < n_ops); + outputs_buf[i + 1] = node_idxs[idx + out_offset]; + } + + const int n_outputs = 1 + (int) fusion.outs.size(); + + // structural subgraph checks (op sequence, elidable uses, view containment) + if (!ggml_can_fuse_subgraph_ext(gf, node_idxs + idx, n_ops, fusion.ops.data(), outputs_buf, n_outputs)) { + continue; + } + } + + // pattern-specific checks (the sole validator for unsafe patterns) + if (fusion.check && !fusion.check(&fusion, nodes, gf, node_idxs, idx, mode)) { + continue; + } + + // the compute phase has allocated tensors and can detect aliasing between + // external sources and fused outputs; the optimizer phase cannot do this yet + if (mode == GGML_METAL_FUSION_FULL && + !ggml_metal_fusion_check_memory_ranges(&fusion, nodes, n_ops)) { + continue; + } + + best = n_ops; + res = &fusion; + } + + *n_out = best; + + return res; +} + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused, chaining patterns back-to-back. matching runs on the same filtered (view +// transparent) node sequence that the compute phase uses, so the returned count is the raw index +// span from idx to the last matched node (intermediate views are packed along). +int ggml_metal_fusion_max(const ggml_cgraph * gf, int idx) { + // a view node cannot start a pattern - pack it alone + if (ggml_op_is_empty(gf->nodes[idx]->op)) { + return 1; + } + + // collect the non-view node indices starting at idx; 0-element tensors are included so + // that empty graphs pack like their non-empty counterparts (see ggml_metal_fusion_filter_ops) + int idxs[GGML_METAL_FUSION_MAX]; + int n_idxs = 0; + for (int i = idx; i < gf->n_nodes && n_idxs < GGML_METAL_FUSION_MAX; i++) { + if (!ggml_op_is_empty(gf->nodes[i]->op)) { + idxs[n_idxs++] = i; + } + } + if (n_idxs == 0) { + return 1; + } + + int total = 0; + int i_f = 0; + + while (i_f < n_idxs && total < GGML_METAL_FUSION_MAX) { + int len = 1; + const ggml_metal_fusion * fusion = ggml_metal_fusion_next(gf, idxs, n_idxs, i_f, GGML_METAL_FUSION_STRUCTURAL, &len); + if (!fusion || total + len > GGML_METAL_FUSION_MAX) { + break; + } + + total += len; + i_f += len; + } + + if (i_f == 0) { + return 1; + } + + // map the matched non-empty nodes back to the raw index span (views are included) + return std::min(GGML_METAL_FUSION_MAX, idxs[i_f - 1] - idx + 1); +} diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.h new file mode 100644 index 0000000..5d994c2 --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-fusion.h @@ -0,0 +1,102 @@ +// single source of truth for the fusions supported by the Metal backend +// +// every fusable subgraph is declared exactly once as a ggml_metal_fusion entry in +// the table in ggml-metal-fusion.cpp. both the graph optimizer (ggml_metal_fusion_max) +// and the op encoders (ggml_metal_fusion_next) consult this same table with the same device +// properties. a check that passes in FULL mode also passes in STRUCTURAL mode, so the encoders +// fuse only groups that the optimizer may pack. + +#pragma once + +#include "ggml-impl.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct ggml_metal_device_props; + +// the maximum number of nodes that can be fused in a single kernel +// (also the maximum length of a packed fusion group during graph optimization) +#define GGML_METAL_FUSION_MAX 16 + +typedef enum ggml_metal_fusion_mode { + // structural checks for the graph optimizer, which runs before the graph tensors are allocated (weights + // already are); a check may skip conditions that differ between batch sizes, and so accept more than FULL + GGML_METAL_FUSION_STRUCTURAL = 0, + // full checks, including buffer placement; used by the op encoders + GGML_METAL_FUSION_FULL, +} ggml_metal_fusion_mode; + +// identifier of each fusion pattern so the op encoders know which kernel to use +typedef enum ggml_metal_fusion_id { + GGML_METAL_FUSION_NONE = 0, + GGML_METAL_FUSION_NORM_MUL, // NORM/RMS_NORM + MUL + GGML_METAL_FUSION_NORM_MUL_ADD, // NORM/RMS_NORM + MUL + ADD + GGML_METAL_FUSION_NORM_SCALE, // NORM/RMS_NORM + SCALE + GGML_METAL_FUSION_ADD_CHAIN, // ADD x N (N in [2, 7]) + GGML_METAL_FUSION_SNAKE, // MUL + SIN + SQR + MUL + ADD + GGML_METAL_FUSION_GDN_CACHE, // GATED_DELTA_NET + CPY (write snapshots into the recurrent cache) + GGML_METAL_FUSION_TOPK_MOE, // SOFT_MAX + ARGSORT + GET_ROWS + norm/scale (MoE routing) + GGML_METAL_FUSION_MOE_REDUCE, // MUL + expert VIEWs + ADD chain (MoE output reduction) + GGML_METAL_FUSION_SSM_CONV_SILU, // SSM_CONV + UNARY (silu) + GGML_METAL_FUSION_MUL_MAT_ADD, // MUL_MAT + ADD (residual added in the few-row MMA store) +} ggml_metal_fusion_id; + +struct ggml_metal_fusion; // defined in ggml-metal-fusion.cpp + +typedef struct ggml_metal_fusion ggml_metal_fusion; + +// access the fusion identifier without exposing the full pattern definition +ggml_metal_fusion_id ggml_metal_fusion_get_id(const struct ggml_metal_fusion * fusion); + +// apply any alloc-dependencies required by the fused kernels during graph optimize +void ggml_metal_fusion_add_alloc_deps( + void * user_data, + void (*add_alloc_dep)(void *, struct ggml_tensor *, struct ggml_tensor *), + struct ggml_cgraph * gf); + +// ---- shared fusion info --------------------------------------------------- + +// shared fusion debugging context, owned by the device; newly created backend contexts for that +// device register with it so the fusion counters are race-free and accumulate across contexts. +struct ggml_metal_fusion_info; // defined in ggml-metal-fusion.cpp + +struct ggml_metal_fusion_info * ggml_metal_fusion_info_init(bool enabled, int debug); +void ggml_metal_fusion_info_free(struct ggml_metal_fusion_info * finfo); + +bool ggml_metal_fusion_info_enabled(const struct ggml_metal_fusion_info * finfo); +bool ggml_metal_fusion_info_stats (const struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_debug (const struct ggml_metal_fusion_info * finfo); + +int ggml_metal_fusion_info_n_fusions(const struct ggml_metal_fusion_info * finfo); +const char * ggml_metal_fusion_info_label (const struct ggml_metal_fusion_info * finfo, int idx); +uint64_t ggml_metal_fusion_info_count (const struct ggml_metal_fusion_info * finfo, int idx); + +void ggml_metal_fusion_info_count_fusion(struct ggml_metal_fusion_info * finfo, const struct ggml_metal_fusion * fusion); +void ggml_metal_fusion_info_set_enabled (struct ggml_metal_fusion_info * finfo, bool enabled); + +void ggml_metal_fusion_info_stats_init ( struct ggml_metal_fusion_info * finfo); +void ggml_metal_fusion_info_stats_reset( struct ggml_metal_fusion_info * finfo); +int ggml_metal_fusion_info_stats_get (const struct ggml_metal_fusion_info * finfo, const char ** labels, uint64_t * counts, int n); +void ggml_metal_fusion_info_labels_init( struct ggml_metal_fusion_info * finfo); + +// compute phase: longest fusion starting at idx (a position in node_idxs) that matches in `mode` on a device with +// props. returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes consumed. +const ggml_metal_fusion * ggml_metal_fusion_next( + const struct ggml_cgraph * gf, + const int * node_idxs, + int n_idxs, + int idx, + ggml_metal_fusion_mode mode, + int * n_out); + +// optimize phase: maximum number of nodes starting at idx (a raw sequential graph index) that +// could be fused on a device with props, chaining patterns back-to-back. returns at least 1. +int ggml_metal_fusion_max(const struct ggml_cgraph * gf, int idx); + +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 379a8b3..9db266f 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -1,6 +1,22 @@ #ifndef GGML_METAL_IMPL #define GGML_METAL_IMPL +// kernel parameters for mat-mat threadgroups +// +// TODO: become function constants + +#define SZ_SIMDGROUP 16 +#define N_MM_NK 2 +#define N_MM_NK_TOTAL (SZ_SIMDGROUP * N_MM_NK) + +#define N_MM_BLOCK_X 4 +#define N_MM_BLOCK_Y 2 +#define N_MM_SIMD_GROUP_X 2 +#define N_MM_SIMD_GROUP_Y 2 + +#define N_ARGMAX_NPART_MAX 256 +#define N_ARGMAX_CHUNK 4096 + // kernel parameters for mat-vec threadgroups // // N_R0: number of src0 rows to process per simdgroup @@ -11,6 +27,9 @@ #define N_R0_Q1_0 8 #define N_SG_Q1_0 2 +#define N_R0_Q2_0 8 +#define N_SG_Q2_0 2 + #define N_R0_Q4_0 4 #define N_SG_Q4_0 2 @@ -46,24 +65,31 @@ #define N_R0_IQ1_S 4 #define N_SG_IQ1_S 2 +#define N_R0_IQ1_S_SPLIT 8 #define N_R0_IQ1_M 4 #define N_SG_IQ1_M 2 +#define N_R0_IQ1_M_SPLIT 8 #define N_R0_IQ2_XXS 4 #define N_SG_IQ2_XXS 2 +#define N_R0_IQ2_XXS_SPLIT 8 #define N_R0_IQ2_XS 4 #define N_SG_IQ2_XS 2 +#define N_R0_IQ2_XS_SPLIT 8 #define N_R0_IQ2_S 4 #define N_SG_IQ2_S 2 +#define N_R0_IQ2_S_SPLIT 8 #define N_R0_IQ3_XXS 4 #define N_SG_IQ3_XXS 2 +#define N_R0_IQ3_XXS_SPLIT 8 #define N_R0_IQ3_S 4 #define N_SG_IQ3_S 2 +#define N_R0_IQ3_S_SPLIT 8 #define N_R0_IQ4_NL 2 #define N_SG_IQ4_NL 2 @@ -71,6 +97,9 @@ #define N_R0_IQ4_XS 2 #define N_SG_IQ4_XS 2 +#define N_R0_TQ2_0 4 +#define N_SG_TQ2_0 2 + // function constants offsets #define FC_FLASH_ATTN_EXT_PAD 100 #define FC_FLASH_ATTN_EXT_BLK 200 @@ -88,14 +117,33 @@ #define FC_SUM_ROWS 1400 #define FC_UPSCALE 1500 #define FC_GATED_DELTA_NET 1600 +#define FC_NORM 1700 +#define FC_TOPK_MOE 1800 +#define FC_MOE_REDUCE 1900 +#define FC_DSV4_HC 2000 +#define FC_PAD 2100 +#define FC_FLASH_ATTN_EXT_TENSOR 2200 +#define FC_LIGHTNING_INDEXER 2200 +#define FC_MUL_MV_MMA 2300 // op-specific constants #define OP_FLASH_ATTN_EXT_NQPSG 8 #define OP_FLASH_ATTN_EXT_NCPSG 64 +#define OP_FLASH_ATTN_EXT_TENSOR_NQPSG 32 +#define OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE 16 +#define OP_FLASH_ATTN_EXT_TENSOR_NCPSG 64 +#define OP_FLASH_ATTN_EXT_TENSOR_NSG 8 + #define OP_FLASH_ATTN_EXT_VEC_NQPSG 1 #define OP_FLASH_ATTN_EXT_VEC_NCPSG 32 +#define OP_LIGHTNING_INDEXER_DK 128 +#define OP_LIGHTNING_INDEXER_NHPTG 8 +#define OP_LIGHTNING_INDEXER_NKPSG 8 +#define OP_LIGHTNING_INDEXER_NSG 8 +#define OP_LIGHTNING_INDEXER_NBPTG 8 + #define OP_UNARY_NUM_SCALE 10 #define OP_UNARY_NUM_FILL 11 #define OP_UNARY_NUM_CLAMP 12 @@ -132,6 +180,10 @@ #define OP_SUM_ROWS_NUM_SUM_ROWS 10 #define OP_SUM_ROWS_NUM_MEAN 11 +#define OP_SSM_SCAN_SSD_CS 64 // Metal-specific; Chunk Size; 64 is largest multiple of 8 (simdgroup tile) fitting into 32 KiB Metal threadgroup mem limit (~26.75 KiB shared mem; see smem layout comment in kernel_ssm_scan_ssd_mma_f32) +#define OP_SSM_SCAN_SSD_HD 64 // Metal-specific; Head Dim the MMA kernel is specialized for (Mamba-2); use_mma gates on d_inner == this +#define OP_SSM_SCAN_SSD_NSG 4 // Metal-specific; Number of SimdGroups per threadgroup; NSG*32 == threads dispatched per threadgroup + // kernel argument structs // // - element counters (e.g. ne00) typically use int32_t to reduce register usage @@ -165,6 +217,7 @@ typedef struct { uint64_t nb2; uint64_t nb3; int32_t dim; + int32_t nc0; } ggml_metal_kargs_concat; typedef struct { @@ -303,6 +356,7 @@ typedef struct { uint64_t nb3; int32_t n_past; int32_t n_dims; + int32_t n_offs; int32_t n_ctx_orig; float freq_base; float freq_scale; @@ -315,8 +369,21 @@ typedef struct { int32_t sect_2; int32_t sect_3; bool src2; + bool inplace; } ggml_metal_kargs_rope; +typedef struct { + int32_t ne0; + int32_t ne1; + int32_t ne2; + int32_t ne3; + uint64_t nb0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; + int32_t nblocks; +} ggml_metal_kargs_flash_attn_ext_kv_f16; + typedef struct { int32_t ne11; int32_t ne_12_2; // assume K and V are same shape @@ -414,8 +481,21 @@ typedef struct { float m1; int32_t n_head_log2; float logit_softcap; + int32_t n_kv_max_padded; } ggml_metal_kargs_flash_attn_ext_vec; +typedef struct { + int32_t ne30; + int32_t ne31; + int32_t ne32; + int32_t ne33; + uint64_t nb31; + uint64_t nb32; + uint64_t nb33; + int32_t n_kv_max; + int32_t n_kv_max_padded; +} ggml_metal_kargs_flash_attn_ext_vec_idx; + typedef struct { int32_t nrows; } ggml_metal_kargs_flash_attn_ext_vec_reduce; @@ -481,7 +561,6 @@ typedef struct { } ggml_metal_kargs_mul_mv_ext; typedef struct { - int32_t ne02; int32_t ne10; int32_t ne11; // n_expert_used (bcast) uint64_t nb11; @@ -491,6 +570,13 @@ typedef struct { uint64_t nb21; } ggml_metal_kargs_mul_mm_id_map0; +typedef struct { + int32_t ne0; + uint64_t nb1; + uint64_t nb2; + uint64_t nb21; +} ggml_metal_kargs_mul_mm_id_zero; + typedef struct { int32_t ne00; int32_t ne02; @@ -503,7 +589,6 @@ typedef struct { uint64_t nb12; uint64_t nb13; int32_t ne20; - int32_t ne21; int32_t ne0; int32_t ne1; int16_t r2; @@ -548,6 +633,7 @@ typedef struct { uint64_t nbf1[3]; uint64_t nbf2[3]; uint64_t nbf3[3]; + float scale; } ggml_metal_kargs_norm; typedef struct { @@ -590,6 +676,21 @@ typedef struct { uint64_t nb1; } ggml_metal_kargs_conv_transpose_1d; +typedef struct { + int32_t T_in; + int32_t T_out; + int32_t OC; + int32_t K; + int32_t K_OC; + int32_t s0; + int32_t p0; +} ggml_metal_kargs_col2im_1d; + +typedef struct { + int32_t T; + int32_t C; +} ggml_metal_kargs_snake; + typedef struct { int32_t IC; int32_t IH; @@ -601,6 +702,7 @@ typedef struct { uint64_t nb0; uint64_t nb1; uint64_t nb2; + uint64_t nb3; } ggml_metal_kargs_conv_transpose_2d; typedef struct { @@ -633,6 +735,34 @@ typedef struct { int32_t d1; } ggml_metal_kargs_conv_2d; +typedef struct { + uint64_t nb00; // kernel strides + uint64_t nb01; + uint64_t nb02; + uint64_t nb10; // input strides + uint64_t nb11; + uint64_t nb12; + uint64_t nb13; + uint64_t nb0; // output strides + uint64_t nb1; + uint64_t nb2; + uint64_t nb3; + int32_t IW; // input width + int32_t IH; // input height + int32_t KW; // kernel width + int32_t KH; // kernel height + int32_t C; // channels (IC == OC for depthwise) + int32_t OW; // output width + int32_t OH; // output height + int32_t N; // batch size + int32_t s0; // stride x + int32_t s1; // stride y + int32_t p0; // padding x + int32_t p1; // padding y + int32_t d0; // dilation x + int32_t d1; // dilation y +} ggml_metal_kargs_conv_2d_dw; + typedef struct { uint64_t ofs0; uint64_t ofs1; @@ -792,7 +922,6 @@ typedef struct { uint64_t nb00; uint64_t nb01; uint64_t nb02; - int64_t ne10; int64_t ne11; uint64_t nb10; uint64_t nb11; @@ -810,7 +939,10 @@ typedef struct { int64_t n_head; int64_t n_group; int64_t n_seq_tokens; + int64_t n_seq_tokens_total; + int64_t token_offset; int64_t n_seqs; + int64_t K; uint64_t s_off; uint64_t nb00; uint64_t nb01; @@ -874,6 +1006,7 @@ typedef struct { uint64_t nb1; uint64_t nb2; uint64_t nb3; + uint64_t nb_out; // 0 => snapshots are appended after the attn scores (unfused) } ggml_metal_kargs_gated_delta_net; typedef struct { @@ -994,6 +1127,10 @@ typedef struct { uint64_t nb1; uint64_t nb2; uint64_t nb3; + int32_t lp0; + int32_t lp1; + int32_t lp2; + int32_t lp3; } ggml_metal_kargs_pad; typedef struct { @@ -1098,6 +1235,39 @@ typedef struct { int32_t len; } ggml_metal_kargs_argsort_merge; +typedef struct { + int32_t ne00; // number of columns (elements per row) + int32_t ne01; // rows + int32_t ne02; + int32_t ne03; + uint64_t nb01; // row stride in src0 + uint64_t nb02; + uint64_t nb03; + int32_t top_k; // k +} ggml_metal_kargs_top_k; + +// widths at or above this use the threadgroup FWHT kernel, one row per threadgroup +// with GGML_METAL_FWHT_TG_NT threads, instead of one row per simdgroup +#define GGML_METAL_FWHT_TG_MIN_N 512 +#define GGML_METAL_FWHT_TG_NT 256 + +typedef struct { + int32_t ne01; // n_tokens + uint64_t nb01; // logits row stride + uint64_t nb1_ids; // ids row stride + float clamp; + float scale; +} ggml_metal_kargs_topk_moe; + +typedef struct { + int32_t ne00; // n_embd + int32_t ne02; // n_tokens +} ggml_metal_kargs_moe_reduce; + +typedef struct { + int32_t nrows; +} ggml_metal_kargs_fwht; + typedef struct { int64_t ne0; float start; @@ -1108,6 +1278,68 @@ typedef struct { int64_t val; } ggml_metal_kargs_memset; +typedef struct { + int32_t n_kv; + int32_t n_batch; + int32_t mask_ne3; + uint64_t nb1; + uint64_t nb3; + uint64_t nbq1; + uint64_t nbq2; + uint64_t nbq3; + uint64_t nbk2; + uint64_t nbk3; + uint64_t nbw1; + uint64_t nbw3; + uint64_t nbm1; + uint64_t nbm3; +} ggml_metal_kargs_lightning_indexer; + +typedef struct { + int32_t n_tokens; + int32_t n_iter; + uint64_t nb_m0; + uint64_t nb_m1; + uint64_t nb_s0; + uint64_t nb_b0; + uint64_t nb_d0; + uint64_t nb_d1; + uint64_t nb_d2; + float eps; +} ggml_metal_kargs_dsv4_hc_comb; + +typedef struct { + int32_t n_embd; + int32_t n_tokens; + uint64_t nb_x0; + uint64_t nb_x1; + uint64_t nb_x2; + uint64_t nb_w0; + uint64_t nb_w1; + uint64_t nb_w2; + uint64_t nb_d0; + uint64_t nb_d1; + float scale; +} ggml_metal_kargs_dsv4_hc_pre; + +typedef struct { + int32_t n_embd; + int32_t n_tokens; + uint64_t nb_x0; + uint64_t nb_x1; + uint64_t nb_r0; + uint64_t nb_r1; + uint64_t nb_r2; + uint64_t nb_p0; + uint64_t nb_p1; + uint64_t nb_c0; + uint64_t nb_c1; + uint64_t nb_c2; + uint64_t nb_d0; + uint64_t nb_d1; + uint64_t nb_d2; +} ggml_metal_kargs_dsv4_hc_post; + typedef struct { int32_t ne00; int32_t ne01; @@ -1147,8 +1379,10 @@ typedef struct { } ggml_metal_kargs_pool_1d; typedef struct { - int64_t ne00; + int64_t ne00; uint64_t nb01; + int64_t npart; + int64_t ne01; } ggml_metal_kargs_argmax; typedef struct { @@ -1159,4 +1393,8 @@ typedef struct { int64_t np; } ggml_metal_kargs_opt_step_sgd; +typedef struct { + int64_t ne; +} ggml_metal_kargs_silu_back; + #endif // GGML_METAL_IMPL diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index e173527..0cebbe5 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -7,12 +7,18 @@ #include "ggml-metal-impl.h" #include "ggml-metal-common.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" +#include "ggml-metal-tuning.h" #include #include #include #include +// threadgroups needed to fill the GPU +// TODO: dedup +static constexpr int64_t GGML_METAL_MIN_THREADGROUPS = 128; + static ggml_metal_buffer_id ggml_metal_get_buffer_id(const ggml_tensor * t) { if (!t) { return { nullptr, 0 }; @@ -30,24 +36,22 @@ struct ggml_metal_op { ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { this->dev = dev; this->lib = ggml_metal_device_get_library(dev); this->enc = ggml_metal_encoder_init(cmd_buf, use_concurrency); this->mem_ranges = ggml_mem_ranges_init(debug_graph); + this->finfo = finfo; this->idx_start = idx_start; this->idx_end = idx_end; - this->use_fusion = use_fusion; this->use_concurrency = use_concurrency; this->use_capture = use_capture; this->debug_graph = debug_graph; - this->debug_fusion = debug_fusion; this->gf = gf; idxs.reserve(gf->n_nodes); @@ -77,15 +81,24 @@ struct ggml_metal_op { return ggml_graph_node(gf, idxs[i]); } - bool can_fuse(int i0, const ggml_op * ops, int n_ops) const { - assert(use_fusion); + // consult the fusion table for the longest pattern starting at i0 + // returns the matching pattern (nullptr if no fusion) and sets *n_out to the number of nodes + const ggml_metal_fusion * can_fuse(int i0, enum ggml_metal_fusion_mode mode, int * n_out) const { + assert(use_fusion()); assert(i0 >= 0 && i0 < n_nodes()); - if (i0 + n_ops > n_nodes()) { - return false; - } + return ggml_metal_fusion_next(gf, idxs.data(), (int) idxs.size(), i0, mode, n_out); + } - return ggml_can_fuse_ext(gf, idxs.data() + i0, ops, n_ops); + // whether to attempt fusion; the toggle lives in the shared fusion debugging context owned + // by the device (initialized from GGML_METAL_FUSION_DISABLE, overridable by the test) + bool use_fusion() const { + return ggml_metal_fusion_info_enabled(finfo); + } + + // record that a fusion fired, indexed by the matching table entry + void count_fusions(const ggml_metal_fusion * fusion) const { + ggml_metal_fusion_info_count_fusion(finfo, fusion); } ggml_metal_device_t dev; @@ -93,12 +106,13 @@ struct ggml_metal_op { ggml_metal_encoder_t enc; ggml_mem_ranges_t mem_ranges; - bool use_fusion; + // shared fusion debugging context + ggml_metal_fusion_info * finfo; + bool use_concurrency; bool use_capture; int debug_graph; - int debug_fusion; private: ggml_cgraph * gf; @@ -114,24 +128,22 @@ ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, ggml_cgraph * gf, + ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion) { + int debug_graph) { ggml_metal_op_t res = new ggml_metal_op( dev, cmd_buf, gf, + finfo, idx_start, idx_end, - use_fusion, use_concurrency, use_capture, - debug_graph, - debug_fusion); + debug_graph); return res; } @@ -218,7 +230,16 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { // otherwise, we add the new ranges to the encoding context and process the node concurrently // { - const bool is_concurrent = ggml_metal_op_concurrency_check(ctx, node); + bool is_concurrent = ggml_metal_op_concurrency_check(ctx, node); + + if (is_concurrent && ctx->use_fusion()) { + int n_fuse = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n_fuse); + if (fusion) { + // fused kernels write to the last node of the group, not necessarily to the first node's dst + is_concurrent = ggml_mem_ranges_check(ctx->mem_ranges, ctx->node(idx + n_fuse - 1)); + } + } if (!is_concurrent) { ggml_metal_op_concurrency_reset(ctx); @@ -256,7 +277,7 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { ggml_is_contiguous(node->src[3]), node->src[3]->name); } if (node) { - GGML_LOG_DEBUG("%s: node - %4s [%5lld, %5lld, %5lld, %5lld] [%5lld, %5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(node->type), ne0, ne1, ne2, ne3, nb0, nb1, nb2, nb3, + GGML_LOG_DEBUG("%s: node - %4s [%5lld, %5lld, %5lld, %5lld] [%5lld, %5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(node->type), ne0, ne1, ne2, ne3, nb0, nb1, nb2, nb3, node->name); } } @@ -299,6 +320,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_unary(ctx, idx); } break; + case GGML_OP_SILU_BACK: + { + n_fuse = ggml_metal_op_silu_back(ctx, idx); + } break; case GGML_OP_GLU: { n_fuse = ggml_metal_op_glu(ctx, idx); @@ -316,6 +341,16 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_cumsum(ctx, idx); } break; + case GGML_OP_LIGHTNING_INDEXER: + { + n_fuse = ggml_metal_op_lightning_indexer(ctx, idx); + } break; + case GGML_OP_DSV4_HC_COMB: + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + { + n_fuse = ggml_metal_op_dsv4_hc(ctx, idx); + } break; case GGML_OP_SOFT_MAX: { n_fuse = ggml_metal_op_soft_max(ctx, idx); @@ -375,6 +410,7 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { n_fuse = ggml_metal_op_norm(ctx, idx); } break; case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: { n_fuse = ggml_metal_op_rope(ctx, idx); } break; @@ -386,6 +422,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_conv_2d(ctx, idx); } break; + case GGML_OP_CONV_2D_DW: + { + n_fuse = ggml_metal_op_conv_2d_dw(ctx, idx); + } break; case GGML_OP_CONV_TRANSPOSE_1D: { n_fuse = ggml_metal_op_conv_transpose_1d(ctx, idx); @@ -394,6 +434,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { { n_fuse = ggml_metal_op_conv_transpose_2d(ctx, idx); } break; + case GGML_OP_COL2IM_1D: + { + n_fuse = ggml_metal_op_col2im_1d(ctx, idx); + } break; case GGML_OP_CONV_3D: { n_fuse = ggml_metal_op_conv_3d(ctx, idx); @@ -528,8 +572,45 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { const int32_t dim = ((const int32_t *) op->op_params)[0]; + const bool is_q = ggml_is_quantized(op->type); + + // for quantized types, concat is done at the block level (nb0 == type_size == block size) + int32_t ne00_arg = ne00; + int32_t ne10_arg = ne10; + int32_t ne0_arg = ne0; + if (is_q) { + const int32_t blck = ggml_blck_size(op->type); + GGML_ASSERT(ne00 % blck == 0); + GGML_ASSERT(ne10 % blck == 0); + GGML_ASSERT(ne0 % blck == 0); + ne00_arg = ne00/blck; + ne10_arg = ne10/blck; + ne0_arg = ne0/blck; + } + + int nth = std::min(256, ne0_arg); + + // when rows are small, we can batch them together in a single threadgroup + int nrptg = 1; + if (nth < 256) { + nrptg = std::min((256 + nth - 1) / nth, ne1); + if (nrptg * nth > 256) { + nrptg = 256 / nth; + } + } + + const int nw0 = (ne1 + nrptg - 1) / nrptg; + + // split long rows across threadgroups when there are too few rows to fill the GPU + const int64_t n_rows = (int64_t) nw0*ne2*ne3; + + int nc0 = 1; + if (nrptg == 1) { + nc0 = (int) std::max(1, std::min((ne0_arg + nth - 1)/nth, (GGML_METAL_MIN_THREADGROUPS + n_rows - 1)/n_rows)); + } + ggml_metal_kargs_concat args = { - /*.ne00 =*/ ne00, + /*.ne00 =*/ ne00_arg, /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, /*.ne03 =*/ ne03, @@ -537,7 +618,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, /*.nb03 =*/ nb03, - /*.ne10 =*/ ne10, + /*.ne10 =*/ ne10_arg, /*.ne11 =*/ ne11, /*.ne12 =*/ ne12, /*.ne13 =*/ ne13, @@ -545,7 +626,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { /*.nb11 =*/ nb11, /*.nb12 =*/ nb12, /*.nb13 =*/ nb13, - /*.ne0 =*/ ne0, + /*.ne0 =*/ ne0_arg, /*.ne1 =*/ ne1, /*.ne2 =*/ ne2, /*.ne3 =*/ ne3, @@ -554,9 +635,10 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { /*.nb2 =*/ nb2, /*.nb3 =*/ nb3, /*.dim =*/ dim, + /*.nc0 =*/ nc0, }; - auto pipeline = ggml_metal_library_get_pipeline_base(lib, GGML_OP_CONCAT); + auto pipeline = ggml_metal_library_get_pipeline_concat(lib, op->type); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); @@ -564,9 +646,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); - const int nth = std::min(1024, ne0); - - ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, nw0*nc0, ne2, ne3, nth, nrptg, 1); return 1; } @@ -582,6 +662,8 @@ int ggml_metal_op_repeat(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + // TODO: optimize for degenerate cases such as ggml_nelements(op->src[0]) == 1 and others + auto pipeline = ggml_metal_library_get_pipeline_repeat(lib, op->type); ggml_metal_kargs_repeat args = { @@ -816,9 +898,7 @@ int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, 1, 1, 1); } else { const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); - const int nth = MIN(args.ne00, nth_max); - const int nk0 = (args.ne00 + nth - 1)/nth; ggml_metal_encoder_dispatch_threadgroups(enc, nk0*ne01, ne02, ne03, nth, 1, 1); @@ -868,7 +948,7 @@ int ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) { const int64_t nrows = ggml_nrows(op->src[0]); - const int32_t nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00/2); + const int32_t nth = std::max(1, std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00/2)); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); @@ -915,7 +995,7 @@ int ggml_metal_op_sum(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); - ggml_metal_encoder_set_threadgroup_memory_size(enc, nsg * sizeof(float), 0); + ggml_metal_encoder_set_threadgroup_memory_size(enc, GGML_PAD(nsg * sizeof(float), 16), 0); ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, nth, 1, 1); @@ -1188,7 +1268,7 @@ int ggml_metal_op_set_rows(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); - auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op->src[1]->type, op->type); + auto pipeline = ggml_metal_library_get_pipeline_set_rows(lib, op); const int32_t nk0 = ne0/ggml_blck_size(op->type); @@ -1279,9 +1359,217 @@ int ggml_metal_op_diag(ggml_metal_op_t ctx, int idx) { return 1; } +int ggml_metal_op_lightning_indexer(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_encoder_t enc = ctx->enc; + + GGML_ASSERT(op->op == GGML_OP_LIGHTNING_INDEXER); + + const ggml_tensor * q = op->src[0]; + const ggml_tensor * k = op->src[1]; + const ggml_tensor * w = op->src[2]; + const ggml_tensor * m = op->src[3]; + + GGML_ASSERT(q->type == GGML_TYPE_F32); + GGML_ASSERT(k->type == GGML_TYPE_F32 || + k->type == GGML_TYPE_F16 || + k->type == GGML_TYPE_BF16 || + k->type == GGML_TYPE_Q4_0 || + k->type == GGML_TYPE_Q4_1 || + k->type == GGML_TYPE_Q5_0 || + k->type == GGML_TYPE_Q5_1 || + k->type == GGML_TYPE_Q8_0); + GGML_ASSERT(w->type == GGML_TYPE_F32); + GGML_ASSERT(m->type == GGML_TYPE_F16); + GGML_ASSERT(op->type == GGML_TYPE_F32); + + GGML_ASSERT(q->ne[0] == OP_LIGHTNING_INDEXER_DK); + + ggml_metal_kargs_lightning_indexer args = { + /*.n_kv =*/ (int32_t) k->ne[2], + /*.n_batch =*/ (int32_t) q->ne[2], + /*.mask_ne3 =*/ (int32_t) m->ne[3], + /*.nb1 =*/ op->nb[1], + /*.nb3 =*/ op->nb[3], + /*.nbq1 =*/ q->nb[1], + /*.nbq2 =*/ q->nb[2], + /*.nbq3 =*/ q->nb[3], + /*.nbk2 =*/ k->nb[2], + /*.nbk3 =*/ k->nb[3], + /*.nbw1 =*/ w->nb[1], + /*.nbw3 =*/ w->nb[3], + /*.nbm1 =*/ m->nb[1], + /*.nbm3 =*/ m->nb[3], + }; + + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(q), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(k), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(w), 3); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(m), 4); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5); + + const int nsg = OP_LIGHTNING_INDEXER_NSG; + const int nkptg = OP_LIGHTNING_INDEXER_NKPSG*nsg; + const int nbptg = OP_LIGHTNING_INDEXER_NBPTG; + + auto pipeline = ggml_metal_library_get_pipeline_lightning_indexer(ctx->lib, op); + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); + ggml_metal_encoder_dispatch_threadgroups(enc, + (k->ne[2] + nkptg - 1)/nkptg, + (q->ne[2] + nbptg - 1)/nbptg, + q->ne[3], 32, nsg, 1); + + return 1; +} + +int ggml_metal_op_dsv4_hc(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_encoder_t enc = ctx->enc; + auto pipeline = ggml_metal_library_get_pipeline_dsv4_hc(ctx->lib, op); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + + switch (op->op) { + case GGML_OP_DSV4_HC_COMB: + { + const ggml_tensor * mixes = op->src[0]; + const ggml_tensor * scale = op->src[1]; + const ggml_tensor * base = op->src[2]; + + GGML_ASSERT(mixes->type == GGML_TYPE_F32); + GGML_ASSERT(scale->type == GGML_TYPE_F32); + GGML_ASSERT(base->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + GGML_ASSERT(mixes->ne[0] == 24); + GGML_ASSERT(op->ne[0] == 4 && op->ne[1] == 4); + + ggml_metal_kargs_dsv4_hc_comb args = { + /*.n_tokens =*/ (int32_t) mixes->ne[1], + /*.n_iter =*/ ggml_get_op_params_i32(op, 1), + /*.nb_m0 =*/ mixes->nb[0], + /*.nb_m1 =*/ mixes->nb[1], + /*.nb_s0 =*/ scale->nb[0], + /*.nb_b0 =*/ base->nb[0], + /*.nb_d0 =*/ op->nb[0], + /*.nb_d1 =*/ op->nb[1], + /*.nb_d2 =*/ op->nb[2], + /*.eps =*/ ggml_get_op_params_f32(op, 0), + }; + + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(mixes), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(scale), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(base), 3); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4); + + // One SIMDgroup owns one 4x4 Sinkhorn matrix. Packing up to four + // independent tokens per threadgroup keeps both decode and prompt + // dispatches compact without any threadgroup-memory synchronization. + const int nsg = std::min(4, args.n_tokens); + ggml_metal_encoder_dispatch_threadgroups( + enc, (args.n_tokens + nsg - 1)/nsg, 1, 1, 32, nsg, 1); + } break; + case GGML_OP_DSV4_HC_PRE: + { + const ggml_tensor * x = op->src[0]; + const ggml_tensor * weights = op->src[1]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(weights->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + + ggml_metal_kargs_dsv4_hc_pre args = { + /*.n_embd =*/ (int32_t) x->ne[0], + /*.n_tokens =*/ (int32_t) x->ne[2], + /*.nb_x0 =*/ x->nb[0], + /*.nb_x1 =*/ x->nb[1], + /*.nb_x2 =*/ x->nb[2], + /*.nb_w0 =*/ weights->nb[0], + /*.nb_w1 =*/ weights->nb[1], + /*.nb_w2 =*/ weights->nb[2], + /*.nb_d0 =*/ op->nb[0], + /*.nb_d1 =*/ op->nb[1], + /*.scale =*/ ggml_get_op_params_f32(op, 0), + }; + + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(weights), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); + + const int n_tiles = (args.n_embd + 31)/32; + const int nsg = std::min(4, n_tiles); + ggml_metal_encoder_dispatch_threadgroups( + enc, (n_tiles + nsg - 1)/nsg, args.n_tokens, 1, 32, nsg, 1); + } break; + case GGML_OP_DSV4_HC_POST: + { + const ggml_tensor * x = op->src[0]; + const ggml_tensor * residual = op->src[1]; + const ggml_tensor * post = op->src[2]; + const ggml_tensor * comb = op->src[3]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(residual->type == GGML_TYPE_F32); + GGML_ASSERT(post->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + GGML_ASSERT(residual->ne[1] == 4); + + ggml_metal_kargs_dsv4_hc_post args = { + /*.n_embd =*/ (int32_t) x->ne[0], + /*.n_tokens =*/ (int32_t) x->ne[1], + /*.nb_x0 =*/ x->nb[0], + /*.nb_x1 =*/ x->nb[1], + /*.nb_r0 =*/ residual->nb[0], + /*.nb_r1 =*/ residual->nb[1], + /*.nb_r2 =*/ residual->nb[2], + /*.nb_p0 =*/ post->nb[0], + /*.nb_p1 =*/ post->nb[1], + /*.nb_c0 =*/ comb ? comb->nb[0] : 0, + /*.nb_c1 =*/ comb ? comb->nb[1] : 0, + /*.nb_c2 =*/ comb ? comb->nb[2] : 0, + /*.nb_d0 =*/ op->nb[0], + /*.nb_d1 =*/ op->nb[1], + /*.nb_d2 =*/ op->nb[2], + }; + + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(x), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(residual), 2); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(post), 3); + if (comb) { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(comb), 4); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 5); + } else { + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 4); + } + + const int n_tiles = (args.n_embd + 31)/32; + const int nsg = std::min(4, n_tiles); + ggml_metal_encoder_dispatch_threadgroups( + enc, (n_tiles + nsg - 1)/nsg, args.n_tokens, 1, 32, nsg, 1); + } break; + default: + GGML_ABORT("fatal error"); + } + + return 1; +} + int ggml_metal_op_soft_max(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); + if (ctx->use_fusion()) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_TOPK_MOE) { + return ggml_metal_op_topk_moe(ctx, idx); + } + } + ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; @@ -1382,6 +1670,20 @@ int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + int n_fuse = 1; + bool use_silu = false; + + if (ctx->use_fusion()) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_SSM_CONV_SILU) { + n_fuse = n; + use_silu = true; + + ctx->count_fusions(fusion); + } + } + ggml_metal_kargs_ssm_conv args = { /*.ne00 =*/ ne00, /*.ne01 =*/ ne01, @@ -1389,7 +1691,6 @@ int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { /*.nb00 =*/ nb00, /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, - /*.ne10 =*/ ne10, /*.ne11 =*/ ne11, /*.nb10 =*/ nb10, /*.nb11 =*/ nb11, @@ -1401,6 +1702,8 @@ int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { /*.nb2 =*/ nb2, }; + const ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(n_fuse > 1 ? ctx->node(idx + n_fuse - 1) : op); + // Use batched kernel for prefill (ne1 > 1) to reduce threadgroup dispatch overhead const bool use_batched = (ne1 > 1); @@ -1415,31 +1718,35 @@ int ggml_metal_op_ssm_conv(ggml_metal_op_t ctx, int idx) { else if (ne1 > 4 ) BATCH_SIZE = 8; else BATCH_SIZE = 2; - auto pipeline = ggml_metal_library_get_pipeline_ssm_conv_batched(lib, op, BATCH_SIZE); + auto pipeline = ggml_metal_library_get_pipeline_ssm_conv_batched(lib, op, BATCH_SIZE, (int32_t) ne10, use_silu); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); - ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); + ggml_metal_encoder_set_buffer(enc, bid_dst, 3); // Dispatch: ne01 rows, ceil(ne1/BATCH_SIZE) token batches, ne02 sequences // Each threadgroup has BATCH_SIZE threads, each handling one token const int n_token_batches = (ne1 + BATCH_SIZE - 1) / BATCH_SIZE; ggml_metal_encoder_dispatch_threadgroups(enc, ne01, n_token_batches, ne02, BATCH_SIZE, 1, 1); } else { - auto pipeline = ggml_metal_library_get_pipeline_ssm_conv(lib, op); + auto pipeline = ggml_metal_library_get_pipeline_ssm_conv(lib, op, (int32_t) ne10, use_silu); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[0]), 1); ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op->src[1]), 2); - ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 3); + ggml_metal_encoder_set_buffer(enc, bid_dst, 3); ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne1, ne02, 1, 1, 1); } - return 1; + if (n_fuse > 1 && ggml_metal_fusion_info_debug(ctx->finfo) > 1) { + GGML_LOG_DEBUG("%s: fuse: SSM_CONV + UNARY\n", __func__); + } + + return n_fuse; } int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) { @@ -1447,6 +1754,7 @@ int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; + const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -1481,6 +1789,10 @@ int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) { const int64_t n_group = ne41; const int64_t n_seq_tokens = ne12; const int64_t n_seqs = ne13; + const int64_t K = ggml_get_op_params_i32(op, 0); + + GGML_ASSERT(K >= 1); + GGML_ASSERT(ggml_nelements(op->src[1]) + K*d_state*d_inner*n_head*n_seqs == ggml_nelements(op)); ggml_metal_kargs_ssm_scan args = { /*.d_state =*/ d_state, @@ -1488,7 +1800,10 @@ int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) { /*.n_head =*/ n_head, /*.n_group =*/ n_group, /*.n_seq_tokens =*/ n_seq_tokens, + /*.n_seq_tokens_total =*/ n_seq_tokens, + /*.token_offset =*/ 0, /*.n_seqs =*/ n_seqs, + /*.K =*/ K, /*.s_off =*/ ggml_nelements(op->src[1]) * sizeof(float), /*.nb00 =*/ nb00, /*.nb01 =*/ nb01, @@ -1516,26 +1831,53 @@ int ggml_metal_op_ssm_scan(ggml_metal_op_t ctx, int idx) { /*.nb0 =*/ nb0, }; - auto pipeline = ggml_metal_library_get_pipeline_ssm_scan(lib, op); + constexpr int64_t CHUNK = OP_SSM_SCAN_SSD_CS; - GGML_ASSERT(d_state <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + const int64_t snap_reserve = K > 1 ? K : 0; // tokens reserved for sequential kernel rollback snapshots + const int64_t mma_tokens = ((n_seq_tokens - snap_reserve) / CHUNK) * CHUNK; // largest multiple of CHUNK that leaves snap_reserve for the tail + const bool use_mma = + mma_tokens > 0 && + ne30 == 1 && // checks that A tensor is set to scalar decay per head (A shape {1, n_head}) + props_dev->has_simdgroup_mm && // hardware check for M1 or newer + d_state % 8 == 0 && // d_state must be multiple of 8 to align with simdgroup_float 8x8 tiles + d_inner == OP_SSM_SCAN_SSD_HD; // mma kernel is specialized for the Mamba-2 head dim; this checks it - const size_t smem = pipeline.smem; + const auto dispatch = [&](ggml_metal_pipeline_with_params pipeline, int64_t nth, int64_t n_tg_x) { + GGML_ASSERT(nth <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + GGML_ASSERT(pipeline.smem <= props_dev->max_theadgroup_memory_size); - ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), 3); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), 4); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), 5); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), 6); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), 7); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 8); + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), 4); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), 5); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), 6); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[6]), 7); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 8); + ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, n_tg_x, n_head, n_seqs, nth, 1, 1); + }; - ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + if (!use_mma) { + dispatch(ggml_metal_library_get_pipeline_ssm_scan(lib, op, false), d_state, d_inner); + return 1; + } - ggml_metal_encoder_dispatch_threadgroups(enc, d_inner, n_head, n_seqs, d_state, 1, 1); + args.n_seq_tokens = mma_tokens; + dispatch( + ggml_metal_library_get_pipeline_ssm_scan_ssd_mma(lib, op), + OP_SSM_SCAN_SSD_NSG*32, + 1); + + if (mma_tokens < n_seq_tokens) { + ggml_metal_op_concurrency_reset(ctx); + + args.n_seq_tokens = n_seq_tokens - mma_tokens; + args.token_offset = mma_tokens; + dispatch(ggml_metal_library_get_pipeline_ssm_scan(lib, op, true), d_state, d_inner); + } return 1; } @@ -1587,6 +1929,8 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; + const bool use_fusion = ctx->use_fusion(); + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -1599,6 +1943,31 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { auto pipeline = ggml_metal_library_get_pipeline_gated_delta_net(lib, op); + // when fused with the trailing cache cpy, the snapshots are written straight into the + // recurrent cache and the cpy is skipped (see GGML_METAL_FUSION_GDN_CACHE) + ggml_metal_buffer_id bid_out = ggml_metal_get_buffer_id(op); + uint64_t nb_out = 0; + int n_fuse = 1; + + if (use_fusion) { + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_GDN_CACHE) { + const ggml_tensor * dst_cache = ctx->node(idx + 1)->src[1]; // cache view + + bid_out = ggml_metal_get_buffer_id(dst_cache); + nb_out = dst_cache->nb[2]/sizeof(float); + n_fuse = 2; + + ctx->count_fusions(fusion); + + if (debug_fusion > 1) { + GGML_LOG_DEBUG("%s: fuse: GATED_DELTA_NET + CPY\n", __func__); + } + } + } + int ida = 0; ggml_metal_kargs_gated_delta_net args = { @@ -1637,23 +2006,25 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) { /*.nb1 =*/ nb1, /*.nb2 =*/ nb2, /*.nb3 =*/ nb3, + /*.nb_out =*/ nb_out, }; ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++); // args ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++); // q ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), ida++); // k ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[2]), ida++); // v ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[3]), ida++); // gate ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[4]), ida++); // beta ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[5]), ida++); // state - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++); // dst (attn) + ggml_metal_encoder_set_buffer (enc, bid_out, ida++); // state_out const int nsg = pipeline.nsg; ggml_metal_encoder_dispatch_threadgroups(enc, op->src[2]->ne[0]/nsg, op->src[2]->ne[1], op->src[2]->ne[3], 32, nsg, 1); - return 1; + return n_fuse; } int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) { @@ -1788,7 +2159,7 @@ int ggml_metal_op_set(ggml_metal_op_t ctx, int idx) { nk0 = ne10/ggml_blck_size(op->type); } - int nth = std::min(nk0, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + int nth = std::min(nk0*ne11, 256); // when rows are small, we can batch them together in a single threadgroup int nrptg = 1; @@ -1799,7 +2170,7 @@ int ggml_metal_op_set(ggml_metal_op_t ctx, int idx) { nrptg = (nth + nk0 - 1)/nk0; nth = nk0; - if (nrptg*nth > ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { + if (nrptg*nth > 256) { nrptg--; } } @@ -1863,7 +2234,7 @@ int ggml_metal_op_cpy(ggml_metal_op_t ctx, int idx) { nk0 = ne00/ggml_blck_size(op->type); } - int nth = std::min(nk0, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + int nth = std::min(nk0*ne01, 256); // when rows are small, we can batch them together in a single threadgroup int nrptg = 1; @@ -1874,7 +2245,7 @@ int ggml_metal_op_cpy(ggml_metal_op_t ctx, int idx) { nrptg = (nth + nk0 - 1)/nk0; nth = nk0; - if (nrptg*nth > ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { + if (nrptg*nth > 256) { nrptg--; } } @@ -1948,19 +2319,65 @@ int ggml_metal_op_pool_1d(ggml_metal_op_t ctx, int idx) { auto pipeline = ggml_metal_library_get_pipeline_pool_1d(lib, op, op_pool); - const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), (int) np); - const int ntg = (np + nth - 1) / nth; + const int nth = std::min(256, std::max(1, k0)); + const int ntg = (int) np; + + const size_t smem = op_pool == GGML_OP_POOL_AVG ? + 2*32*sizeof(float) : + 32*sizeof(float); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args_pool_1d, sizeof(args_pool_1d), 0); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); return 1; } +int ggml_metal_op_fwht(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + ggml_tensor * src1 = op->src[1]; + + const int64_t n = src1->ne[0]; + const int64_t nrows = ggml_nrows(src1); + + ggml_metal_kargs_fwht args = { + /*.nrows = */ (int32_t) nrows, + }; + + auto pipeline = ggml_metal_library_get_pipeline_fwht(lib, n, src1->type); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(src1), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 2); + + const int th_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline); + const int simd_size = 32; + + if (n >= GGML_METAL_FWHT_TG_MIN_N) { + GGML_ASSERT(th_max >= GGML_METAL_FWHT_TG_NT); + ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, GGML_METAL_FWHT_TG_NT, 1, 1); + + return 1; + } + + int sg_per_tg = 2; + sg_per_tg = std::min(sg_per_tg, th_max/simd_size); + sg_per_tg = std::max(sg_per_tg, 1); + + const int64_t n_tg = (nrows + sg_per_tg - 1) / sg_per_tg; + ggml_metal_encoder_dispatch_threadgroups(enc, n_tg, 1, 1, 32*sg_per_tg, 1, 1); + + return 1; +} int ggml_metal_op_pool_2d(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -2022,93 +2439,250 @@ int ggml_metal_op_pool_2d(ggml_metal_op_t ctx, int idx) { return 1; } -int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { +// the number of nodes from idx on that the fusion table fuses as pattern id into one dispatch, 1 if it fuses none +static int ggml_metal_op_try_fusion(ggml_metal_op_t ctx, int idx, ggml_metal_fusion_id id) { + if (!ctx->use_fusion()) { + return 1; + } + + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + if (fusion == nullptr || ggml_metal_fusion_get_id(fusion) != id) { + return 1; + } + + ctx->count_fusions(fusion); + + if (ggml_metal_fusion_info_debug(ctx->finfo) > 1) { + GGML_LOG_DEBUG("%s: fuse: %s to %s, %d nodes\n", __func__, ggml_op_name(ctx->node(idx)->op), ggml_op_name(ctx->node(idx + n - 1)->op), n); + } + + return n; +} + +static int ggml_metal_op_mul_mat_mma(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev); - GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); GGML_TENSOR_LOCALS( int32_t, ne, op, ne); - GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); - GGML_ASSERT(ne00 == ne10); + // the MMA store adds the residual when the table fuses the ADD after this mat-mul + const int n_fuse = ggml_metal_op_try_fusion(ctx, idx, GGML_METAL_FUSION_MUL_MAT_ADD); + const bool fuse_add = n_fuse > 1; - GGML_ASSERT(ne12 % ne02 == 0); - GGML_ASSERT(ne13 % ne03 == 0); + const ggml_tensor * dst = op; + const ggml_tensor * res = dst; - const int16_t r2 = ne12/ne02; - const int16_t r3 = ne13/ne03; + if (fuse_add) { + dst = ctx->node(idx + n_fuse - 1); + // the residual is the other operand of the ADD, by identity: it can itself be a MUL_MAT output + res = dst->src[0] == op ? dst->src[1] : dst->src[0]; + } - // find the break-even point where the matrix-matrix kernel becomes more efficient compared - // to the matrix-vector kernel - const int ne11_mm_min = 8; + auto pipeline = ggml_metal_library_get_pipeline_mul_mv_mma_auto(lib, op, fuse_add); - // first try to use small-batch mat-mv kernels - // these should be efficient for BS [2, ~8] - if (op->src[1]->type == GGML_TYPE_F32 && (ne00%128 == 0) && - ( - ( - ( - op->src[0]->type == GGML_TYPE_F32 || // TODO: helper function - op->src[0]->type == GGML_TYPE_F16 || - op->src[0]->type == GGML_TYPE_BF16 || - op->src[0]->type == GGML_TYPE_Q1_0 || - op->src[0]->type == GGML_TYPE_Q4_0 || - op->src[0]->type == GGML_TYPE_Q4_1 || - op->src[0]->type == GGML_TYPE_Q5_0 || - op->src[0]->type == GGML_TYPE_Q5_1 || - op->src[0]->type == GGML_TYPE_Q8_0 || - op->src[0]->type == GGML_TYPE_MXFP4 || - op->src[0]->type == GGML_TYPE_IQ4_NL || - false) && (ne11 >= 2 && ne11 <= 8) - ) || - ( - ( - op->src[0]->type == GGML_TYPE_Q4_K || - op->src[0]->type == GGML_TYPE_Q5_K || - op->src[0]->type == GGML_TYPE_Q6_K || - op->src[0]->type == GGML_TYPE_Q2_K || - op->src[0]->type == GGML_TYPE_Q3_K || - false) && (ne11 >= 4 && ne11 <= 8) - ) - ) - ) { - // TODO: determine the optimal parameters based on grid utilization - // I still don't know why we should not always use the maximum available threads: - // - // nsg = pipeline.maxTotalThreadsPerThreadgroup / 32 - // - // my current hypothesis is that the work grid is not evenly divisible for different nsg - // values and there can be some tail effects when nsg is high. need to confirm this - // - const int nsg = 2; // num simdgroups per threadgroup + ggml_metal_kargs_mul_mv_ext args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.ne10 =*/ ne10, + /*.ne11 =*/ ne11, + /*.ne12 =*/ ne12, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, + /*.nb12 =*/ nb12, + /*.nb13 =*/ nb13, + /*.ne0 =*/ ne0, + /*.ne1 =*/ ne1, + /*.r2 =*/ (int16_t) (ne12/ne02), + /*.r3 =*/ (int16_t) (ne13/ne03), + }; - // num threads along row per simdgroup - int16_t nxpsg = 0; - if (ne00 % 256 == 0 && ne11 < 3) { - nxpsg = 16; - } else if (ne00 % 128 == 0) { - nxpsg = 8; - } else { - nxpsg = 4; - } + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(dst), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(res), 4); - const int16_t nypsg = 32/nxpsg; // num threads along col per simdgroup (i.e. a simdgroup processes that many src0 rows at a time) - const int16_t r0ptg = nypsg*nsg; // num src0 rows per threadgroup - int16_t r1ptg = 4; // num src1 rows per threadgroup + ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0); - // note: not sure how optimal are those across all different hardware. there might be something cleverer - switch (ne11) { - case 2: - r1ptg = 2; break; - case 3: - case 6: + const int rows0 = pipeline.nr0; + const int rows1 = pipeline.nr1; + + ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + rows0 - 1)/rows0, (ne11 + rows1 - 1)/rows1, ne12*ne13, 32, pipeline.nsg, 1); + + return n_fuse; +} + +// the generic mat-vec kernel, or its 2-row Q4_0 variant if nc +static int ggml_metal_op_mul_mat_mv(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); + GGML_TENSOR_LOCALS( int32_t, ne, op, ne); + + const int16_t r2 = ne12/ne02; + const int16_t r3 = ne13/ne03; + + auto pipeline = ggml_metal_library_get_pipeline_mul_mv(lib, op); + + const int nr0 = pipeline.nr0; + const int nr1 = pipeline.nr1; + const int nsg = pipeline.nsg; + + const size_t smem = pipeline.smem; + + ggml_metal_kargs_mul_mv args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.ne10 =*/ ne10, + /*.ne11 =*/ ne11, + /*.ne12 =*/ ne12, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, + /*.nb12 =*/ nb12, + /*.nb13 =*/ nb13, + /*.ne0 =*/ ne0, + /*.ne1 =*/ ne1, + /*.nr0 =*/ nr0, + /*.r2 =*/ r2, + /*.r3 =*/ r3, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); + + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + + if (op->src[0]->type == GGML_TYPE_F32 || + op->src[0]->type == GGML_TYPE_F16 || + op->src[0]->type == GGML_TYPE_BF16 || + op->src[0]->type == GGML_TYPE_Q8_0) { + ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + nr0 - 1)/(nr0)), ((ne11 + nr1 - 1)/nr1), ne12*ne13, 32, nsg, 1); + } else { + ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + nr0*nsg - 1)/(nr0*nsg)), ((ne11 + nr1 - 1)/nr1), ne12*ne13, 32, nsg, 1); + } + + return 1; +} + +int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev); + + if (ggml_metal_op_mul_mat_use_fwht(op, props_dev->max_theadgroup_memory_size)) { + return ggml_metal_op_fwht(ctx, idx); + } + + if (props_dev->has_simdgroup_mm && ggml_metal_op_mul_mat_use_mma(op)) { + return ggml_metal_op_mul_mat_mma(ctx, idx); + } + + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); + GGML_TENSOR_LOCALS( int32_t, ne, op, ne); + GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + + GGML_ASSERT(ne00 == ne10); + + GGML_ASSERT(ne12 % ne02 == 0); + GGML_ASSERT(ne13 % ne03 == 0); + + const int16_t r2 = ne12/ne02; + const int16_t r3 = ne13/ne03; + + // first try to use small-batch mat-mv kernels + // these should be efficient for BS [2, ~8] + if (op->src[1]->type == GGML_TYPE_F32 && (ne00%128 == 0) && + ( + ( + ( + op->src[0]->type == GGML_TYPE_F32 || // TODO: helper function + op->src[0]->type == GGML_TYPE_F16 || + op->src[0]->type == GGML_TYPE_BF16 || + op->src[0]->type == GGML_TYPE_Q1_0 || + op->src[0]->type == GGML_TYPE_Q2_0 || + op->src[0]->type == GGML_TYPE_Q4_0 || + op->src[0]->type == GGML_TYPE_Q4_1 || + op->src[0]->type == GGML_TYPE_Q5_0 || + op->src[0]->type == GGML_TYPE_Q5_1 || + op->src[0]->type == GGML_TYPE_Q8_0 || + op->src[0]->type == GGML_TYPE_MXFP4 || + op->src[0]->type == GGML_TYPE_IQ4_NL || + false) && (ne11 >= 2 && ne11 <= 8) + ) || + ( + ( + op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q6_K || + op->src[0]->type == GGML_TYPE_Q2_K || + op->src[0]->type == GGML_TYPE_Q3_K || + false) && (ne11 >= 4 && ne11 <= 8) + ) + ) + ) { + // TODO: determine the optimal parameters based on grid utilization + // I still don't know why we should not always use the maximum available threads: + // + // nsg = pipeline.maxTotalThreadsPerThreadgroup / 32 + // + // my current hypothesis is that the work grid is not evenly divisible for different nsg + // values and there can be some tail effects when nsg is high. need to confirm this + // + const int nsg = 2; // num simdgroups per threadgroup + + // num threads along row per simdgroup + int16_t nxpsg = 0; + if (ne00 % 256 == 0 && ne11 < 3) { + nxpsg = 16; + } else if (ne00 % 128 == 0) { + nxpsg = 8; + } else { + nxpsg = 4; + } + + const int16_t nypsg = 32/nxpsg; // num threads along col per simdgroup (i.e. a simdgroup processes that many src0 rows at a time) + const int16_t r0ptg = nypsg*nsg; // num src0 rows per threadgroup + int16_t r1ptg = 4; // num src1 rows per threadgroup + + // note: not sure how optimal are those across all different hardware. there might be something cleverer + switch (ne11) { + case 2: + r1ptg = 2; break; + case 3: + case 6: r1ptg = 3; break; case 4: case 7: @@ -2120,7 +2694,7 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { GGML_ABORT("unsupported ne11"); }; - auto pipeline = ggml_metal_library_get_pipeline_mul_mv_ext(lib, op->src[0]->type, op->src[1]->type, nsg, nxpsg, r1ptg); + auto pipeline = ggml_metal_library_get_pipeline_mul_mv_ext(lib, op, nsg, nxpsg, r1ptg); ggml_metal_kargs_mul_mv_ext args = { /*.ne00 =*/ ne00, @@ -2150,12 +2724,7 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + r0ptg - 1)/r0ptg), ((ne11 + r1ptg - 1)/r1ptg), ne12*ne13, 32, nsg, 1); - } else if ( - !ggml_is_transposed(op->src[0]) && - !ggml_is_transposed(op->src[1]) && - // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs - // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel - props_dev->has_simdgroup_mm && ne00 >= 64 && ne11 > ne11_mm_min) { + } else if (ggml_metal_op_mul_mat_use_mm(op, props_dev->has_simdgroup_mm)) { //GGML_LOG_INFO("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12); // some Metal matrix data types require aligned pointers @@ -2195,74 +2764,52 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { const size_t smem = pipeline.smem; ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - ggml_metal_encoder_dispatch_threadgroups(enc, ((ne11 + 31)/32), ((ne01 + 63)/64), ne12*ne13, 128, 1, 1); - } else { - auto pipeline = ggml_metal_library_get_pipeline_mul_mv(lib, op); const int nr0 = pipeline.nr0; const int nr1 = pipeline.nr1; const int nsg = pipeline.nsg; - const size_t smem = pipeline.smem; - - ggml_metal_kargs_mul_mv args = { - /*.ne00 =*/ ne00, - /*.ne01 =*/ ne01, - /*.ne02 =*/ ne02, - /*.nb00 =*/ nb00, - /*.nb01 =*/ nb01, - /*.nb02 =*/ nb02, - /*.nb03 =*/ nb03, - /*.ne10 =*/ ne10, - /*.ne11 =*/ ne11, - /*.ne12 =*/ ne12, - /*.nb10 =*/ nb10, - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, - /*.ne0 =*/ ne0, - /*.ne1 =*/ ne1, - /*.nr0 =*/ nr0, - /*.r2 =*/ r2, - /*.r3 =*/ r3, - }; + ggml_metal_encoder_dispatch_threadgroups(enc, ((ne11 + nr1 - 1) / nr1), ((ne01 + nr0 - 1) / nr0), ne12 * ne13, 32, nsg, 1); + } else { + return ggml_metal_op_mul_mat_mv(ctx, idx); + } - ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); + return 1; +} - ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); +// upper bound on the number of entry tiles emitted by kernel_mul_mm_id_map0: the full tiles +// over all entries plus at most one partial tile per expert +static int64_t ggml_metal_op_mul_mat_id_max_tiles(const ggml_tensor * op) { + const int64_t ne02 = op->src[0]->ne[2]; // n_expert + const int64_t ne20 = op->src[2]->ne[0]; // n_expert_used + const int64_t ne21 = op->src[2]->ne[1]; // n_token - if (op->src[0]->type == GGML_TYPE_F32 || - op->src[0]->type == GGML_TYPE_F16 || - op->src[0]->type == GGML_TYPE_BF16 || - op->src[0]->type == GGML_TYPE_Q8_0) { - ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + nr0 - 1)/(nr0)), ((ne11 + nr1 - 1)/nr1), ne12*ne13, 32, nsg, 1); - } else { - ggml_metal_encoder_dispatch_threadgroups(enc, ((ne01 + nr0*nsg - 1)/(nr0*nsg)), ((ne11 + nr1 - 1)/nr1), ne12*ne13, 32, nsg, 1); - } - } + const int64_t n_rows = ne20*ne21; + const int64_t tiles = (n_rows + 31)/32 + (ne02 < n_rows ? ne02 : n_rows) - 1; - return 1; + return n_rows < tiles ? n_rows : tiles; } size_t ggml_metal_op_mul_mat_id_extra_tpe(const ggml_tensor * op) { assert(op->op == GGML_OP_MUL_MAT_ID); - const int64_t ne02 = op->src[0]->ne[2]; // n_expert + // row map of kernel_mul_mm_id: total tile count + one + // (expert, first entry, valid rows) descriptor per tile + const int64_t max_tiles = ggml_metal_op_mul_mat_id_max_tiles(op); - return ggml_type_size(GGML_TYPE_I32)*ne02; + return ggml_type_size(GGML_TYPE_I32)*(1 + 3*max_tiles); } size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) { assert(op->op == GGML_OP_MUL_MAT_ID); const int64_t ne02 = op->src[0]->ne[2]; // n_expert + const int64_t ne20 = op->src[2]->ne[0]; // n_expert_used const int64_t ne21 = op->src[2]->ne[1]; // n_token - return ggml_type_size(GGML_TYPE_I32)*ne02*ne21; + // one entry per (expert used, token) pair of ids plus one padding slot per expert + // (see kernel_mul_mm_id_map0) + return ggml_type_size(GGML_TYPE_I32)*(ne20*ne21 + ne02); } int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { @@ -2299,13 +2846,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { const uint32_t r2 = 1; const uint32_t r3 = 1; - // find the break-even point where the matrix-matrix kernel becomes more efficient compared - // to the matrix-vector kernel - // ne20 = n_used_experts - // ne21 = n_rows (batch size) - const int ne21_mm_id_min = 32; - - if (props_dev->has_simdgroup_mm && ne00 >= 64 && (ne21 >= ne21_mm_id_min)) { + if (ggml_metal_op_mul_mat_id_use_mm(op, props_dev->has_simdgroup_mm)) { // some Metal matrix data types require aligned pointers // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5) //switch (op->src[0]->type) { @@ -2324,7 +2865,6 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { { ggml_metal_kargs_mul_mm_id_map0 args = { - ne02, ne10, ne11, // n_expert_used (bcast) nb11, @@ -2353,6 +2893,25 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, ne02, 1, 1); } + // zero the rows that no expert owns - disjoint from what kernel_mul_mm_id writes + { + ggml_metal_kargs_mul_mm_id_zero args = { + /*.ne0 =*/ ne0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb21 =*/ nb21, + }; + + auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_zero(lib); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_src2, 1); + ggml_metal_encoder_set_buffer (enc, bid_dst, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, ne20, ne21, 1, 32, 1, 1); + } + // this barrier is always needed because the next kernel has to wait for the id maps to be computed ggml_metal_op_concurrency_reset(ctx); @@ -2371,7 +2930,6 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { /*.nb12 =*/ nb12, /*.nb13 =*/ nb13, /*.ne20 =*/ ne20, // n_expert_used - /*.ne21 =*/ ne21, // n_tokens /*.ne0 =*/ ne0, /*.ne1 =*/ ne1, /*.r2 =*/ r2, @@ -2388,9 +2946,13 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) { const size_t smem = pipeline.smem; + GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - ggml_metal_encoder_dispatch_threadgroups(enc, (ne21 + 31)/32, (ne01 + 63)/64, ne02, 128, 1, 1); + const int max_tiles = (int) ggml_metal_op_mul_mat_id_max_tiles(op); + + ggml_metal_encoder_dispatch_threadgroups(enc, max_tiles, (ne01 + 63)/64, 1, 128, 1, 1); } } else { auto pipeline = ggml_metal_library_get_pipeline_mul_mv_id(lib, op); @@ -2509,6 +3071,155 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) { return (ne01 < 20) && (ne00 % 32 == 0); } +// ref: https://github.com/ggml-org/llama.cpp/pull/27390 +// dequantize the quantized KV cache to F16 before running the F16 flash attention kernels +static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + // depending on compute/bandwidth ratio, dequant to f16 kv is not always beneficial + // ref: https://github.com/ggml-org/llama.cpp/pull/27390#issuecomment-5355152767 + // TODO: tune per device + // large heads need the upfront dequant to fit the non-vec threadgroup memory + if (op->src[0]->ne[1] < 32 && + (op->src[0]->ne[0] < 512 || ggml_metal_op_flash_attn_ext_use_vec(op))) { + return false; + } + + switch (op->src[1]->type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + return true; + default: + return false; + } +} + +static bool ggml_metal_op_flash_attn_ext_use_tensor(const ggml_tensor * op, bool has_tensor) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + if (!has_tensor || ggml_metal_op_flash_attn_ext_use_vec(op)) { + return false; + } + + const int64_t ne01 = op->src[0]->ne[1]; + const int64_t ne02 = op->src[0]->ne[2]; + const int64_t ne03 = op->src[0]->ne[3]; + + const int64_t dk = op->src[1]->ne[0]; + const int64_t dv = op->src[2]->ne[0]; + + const bool dk_dv_ok = (dk == 64 && dv == 64) || + (dk == 128 && dv == 128) || + (dk == 192 && dv == 128) || + (dk == 256 && dv == 256) || + (dk == 512 && dv == 512) || + (dk == 576 && dv == 512); + + if (!dk_dv_ok) { + return false; + } + + // large heads use fewer queries per threadgroup, so that the queries fit in threadgroup memory + const int64_t nqptg = dk >= 512 ? OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE : OP_FLASH_ATTN_EXT_TENSOR_NQPSG; + + // few heads and small batches do not fill the GPU - the half8x8 kernel is faster there + // TODO: tune per device + if (((ne01 + nqptg - 1)/nqptg)*ne02*ne03*dk < 8192) { + return false; + } + + if (op->src[1]->type != GGML_TYPE_F16 && op->src[1]->type != GGML_TYPE_BF16 && !ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { + return false; + } + + return op->src[1]->ne[1] % OP_FLASH_ATTN_EXT_TENSOR_NCPSG == 0; +} + +// returns the n_kv_max hint if the sparse path is available for this op, or 0 otherwise +// the mask (src[3]) remains the single source of truth: finite entries are the valid KV positions, +// n_kv_max is only an upper bound on their number per mask row, used to size the index lists +static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + int32_t n_kv_max = 0; + memcpy(&n_kv_max, ((const int32_t *) op->op_params) + 4, sizeof(n_kv_max)); + + if (n_kv_max <= 0) { + return 0; + } + + // the sparse indices are gathered from the mask + if (!op->src[3]) { + return 0; + } + + // bound the size of the index lists + if (n_kv_max > 4096) { + return 0; + } + + // vec kernel instantiations exist for these (type, dk, dv) combinations only + const int64_t dk = op->src[1]->ne[0]; + const int64_t dv = op->src[2]->ne[0]; + + const bool dk_dv_ok = (dk == 32 && dv == 32) || + (dk == 64 && dv == 64) || + (dk == 96 && dv == 96) || + (dk == 96 && dv == 64) || + (dk == 128 && dv == 128) || + (dk == 128 && dv == 96) || + (dk == 192 && dv == 128) || + (dk == 192 && dv == 192) || + (dk == 256 && dv == 256) || + (dk == 320 && dv == 256) || + (dk == 512 && dv == 512) || + (dk == 576 && dv == 512); + + if (!dk_dv_ok) { + return 0; + } + + switch (op->src[1]->type) { + case GGML_TYPE_F16: + case GGML_TYPE_BF16: + case GGML_TYPE_F32: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + break; + default: + return 0; + } + + return n_kv_max; +} + +// in some models (e.g. MLA-based), V is a view of K (the first ne20 elements of each K row); +// the dequantized V is then a view of the dequantized K and does not need its own dequant or scratch +// - ref: https://github.com/ggml-org/llama.cpp/pull/13435 +static bool ggml_metal_op_flash_attn_ext_v_is_view_of_k(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + const ggml_tensor * K = op->src[1]; + const ggml_tensor * V = op->src[2]; + + return V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)); +} + +// size of the F16 dequantized K tensor; the dequantized V tensor follows it in the same scratch buffer +static size_t ggml_metal_op_flash_attn_ext_kv_f16_k_size(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + + return GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne10*ne11*ne12*ne13, 16); +} + size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); @@ -2524,6 +3235,18 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { size_t res = 0; const bool has_mask = op->src[3] != nullptr; + const bool use_kv_f16 = ggml_metal_op_flash_attn_ext_use_kv_f16(op); + + // when the KV is dequantized to F16, the pad kernel copies the tail chunk from the F16 scratch buffer + // note: when V is a view of K, the dequantized V is read from the dequantized K with K's row stride + const bool v_is_view_of_k = use_kv_f16 && ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + uint64_t nb11_pad = nb11; + uint64_t nb21_pad = nb21; + + if (use_kv_f16) { + nb11_pad = sizeof(ggml_fp16_t)*ne10; + nb21_pad = sizeof(ggml_fp16_t)*(v_is_view_of_k ? ne10 : ne20); + } // note: the non-vec kernel requires more extra memory, so always reserve for it GGML_ASSERT(OP_FLASH_ATTN_EXT_NCPSG >= OP_FLASH_ATTN_EXT_VEC_NCPSG); @@ -2536,8 +3259,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { if (has_kvpad) { res += OP_FLASH_ATTN_EXT_VEC_NCPSG*( - nb11*ne12*ne13 + - nb21*ne22*ne23 + + nb11_pad*ne12*ne13 + + nb21_pad*ne22*ne23 + (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0)); } } else { @@ -2546,8 +3269,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_pad(const ggml_tensor * op) { if (has_kvpad) { res += OP_FLASH_ATTN_EXT_NCPSG*( - nb11*ne12*ne13 + - nb21*ne22*ne23 + + nb11_pad*ne12*ne13 + + nb21_pad*ne22*ne23 + (has_mask ? ggml_type_size(GGML_TYPE_F16)*ne31*ne32*ne33 : 0)); } } @@ -2623,6 +3346,47 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) { return res; } +size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + // note: always reserve the temp buffer to avoid graph reallocations + //if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { + // return 0; + //} + + GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne); + + const size_t k_size = ggml_metal_op_flash_attn_ext_kv_f16_k_size(op); + + // when V is a view of K, the dequantized V is a view of the dequantized K + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + if (v_is_view_of_k) { + return k_size; + } + + const size_t v_size = GGML_PAD(sizeof(ggml_fp16_t)*(size_t) ne20*ne21*ne22*ne23, 16); + + return k_size + v_size; +} + +// size of the sparse index lists: one list of KV indices per mask row, +// padded with -1 up to a multiple of OP_FLASH_ATTN_EXT_VEC_NCPSG +size_t ggml_metal_op_flash_attn_ext_extra_idx(const ggml_tensor * op) { + assert(op->op == GGML_OP_FLASH_ATTN_EXT); + + GGML_TENSOR_LOCALS( int32_t, ne3, op->src[3], ne); + + const int n_kv_max = ggml_metal_op_flash_attn_ext_n_kv_max_sparse(op); + + if (n_kv_max <= 0) { + return 0; + } + + const int n_kv_max_padded = GGML_PAD(n_kv_max, OP_FLASH_ATTN_EXT_VEC_NCPSG); + + return GGML_PAD(sizeof(int32_t)*(size_t) n_kv_max_padded*ne31*ne32*ne33, 16); +} + int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -2697,7 +3461,215 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_buffer_id bid_tmp = bid_blk; bid_tmp.offs += ggml_metal_op_flash_attn_ext_extra_blk(op); - if (!ggml_metal_op_flash_attn_ext_use_vec(op)) { + ggml_metal_buffer_id bid_kv_f16 = bid_tmp; + bid_kv_f16.offs += ggml_metal_op_flash_attn_ext_extra_tmp(op); + + // sparse path: gather the finite mask entries into index lists and run the vec kernels over them + const int n_kv_max_sparse = ggml_metal_op_flash_attn_ext_n_kv_max_sparse(op); + const bool use_sparse = n_kv_max_sparse > 0; + const int n_kv_max_padded = use_sparse ? GGML_PAD(n_kv_max_sparse, OP_FLASH_ATTN_EXT_VEC_NCPSG) : 0; + + // the vec kernels dequantize the KV inline; no need for the F16 dequant pass in the sparse path + const bool use_kv_f16 = !use_sparse && ggml_metal_op_flash_attn_ext_use_kv_f16(op); + + ggml_metal_buffer_id bid_idx = bid_kv_f16; + bid_idx.offs += ggml_metal_op_flash_attn_ext_extra_kv_f16(op); + + ggml_metal_buffer_id bid_k = bid_src1; + ggml_metal_buffer_id bid_v = bid_src2; + + uint64_t nb10_attn = nb10; + uint64_t nb11_attn = nb11; + uint64_t nb12_attn = nb12; + uint64_t nb13_attn = nb13; + uint64_t nb20_attn = nb20; + uint64_t nb21_attn = nb21; + uint64_t nb22_attn = nb22; + uint64_t nb23_attn = nb23; + + if (use_kv_f16) { + assert(ggml_metal_op_flash_attn_ext_extra_kv_f16(op) != 0); + + const bool v_is_view_of_k = ggml_metal_op_flash_attn_ext_v_is_view_of_k(op); + + const int64_t nblocks1_64 = (ne10/ggml_blck_size(op->src[1]->type))*(int64_t) ne11*ne12*ne13; + GGML_ASSERT(nblocks1_64 <= INT32_MAX); + const int32_t nblocks1 = nblocks1_64; + + ggml_metal_buffer_id bid_v_f16 = bid_kv_f16; + bid_v_f16.offs += ggml_metal_op_flash_attn_ext_kv_f16_k_size(op); + + auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_kv_f16(lib, op); + const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 256); + + // K + ggml_metal_kargs_flash_attn_ext_kv_f16 args_k = { + /*.ne0 =*/ ne10, + /*.ne1 =*/ ne11, + /*.ne2 =*/ ne12, + /*.ne3 =*/ ne13, + /*.nb0 =*/ nb10, + /*.nb1 =*/ nb11, + /*.nb2 =*/ nb12, + /*.nb3 =*/ nb13, + /*.nblocks =*/ nblocks1, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args_k, sizeof(args_k), 0); + ggml_metal_encoder_set_buffer (enc, bid_src1, 1); + ggml_metal_encoder_set_buffer (enc, bid_kv_f16, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks1 + nth - 1)/nth, 1, 1, nth, 1, 1); + + // V (skip when V is a view of K: the dequantized V is a view of the dequantized K) + if (!v_is_view_of_k) { + const int64_t nblocks2_64 = (ne20/ggml_blck_size(op->src[2]->type))*(int64_t) ne21*ne22*ne23; + GGML_ASSERT(nblocks2_64 <= INT32_MAX); + const int32_t nblocks2 = nblocks2_64; + + ggml_metal_kargs_flash_attn_ext_kv_f16 args_v = { + /*.ne0 =*/ ne20, + /*.ne1 =*/ ne21, + /*.ne2 =*/ ne22, + /*.ne3 =*/ ne23, + /*.nb0 =*/ nb20, + /*.nb1 =*/ nb21, + /*.nb2 =*/ nb22, + /*.nb3 =*/ nb23, + /*.nblocks =*/ nblocks2, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args_v, sizeof(args_v), 0); + ggml_metal_encoder_set_buffer (enc, bid_src2, 1); + ggml_metal_encoder_set_buffer (enc, bid_v_f16, 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, (nblocks2 + nth - 1)/nth, 1, 1, nth, 1, 1); + } + + // the pad and attention kernels read the dequantized KV + ggml_metal_op_concurrency_reset(ctx); + + bid_k = bid_kv_f16; + bid_v = v_is_view_of_k ? bid_k : bid_v_f16; + + // contiguous F16 layout of the dequantized K + nb10_attn = sizeof(ggml_fp16_t); + nb11_attn = nb10_attn*ne10; + nb12_attn = nb11_attn*ne11; + nb13_attn = nb12_attn*ne12; + + // if V is a view of K, the dequantized V is read from the dequantized K with K's strides + if (v_is_view_of_k) { + nb20_attn = nb10_attn; + nb21_attn = nb11_attn; + nb22_attn = nb12_attn; + nb23_attn = nb13_attn; + } else { + // contiguous F16 layout of the dequantized V + nb20_attn = sizeof(ggml_fp16_t); + nb21_attn = nb20_attn*ne20; + nb22_attn = nb21_attn*ne21; + nb23_attn = nb22_attn*ne22; + } + } + + if (!use_sparse && ggml_metal_op_flash_attn_ext_use_tensor(op, props_dev->has_tensor)) { + // tensor API kernel + const int nqptg = ne00 >= 512 ? OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE : OP_FLASH_ATTN_EXT_TENSOR_NQPSG; // queries per threadgroup + const int ncpsg = OP_FLASH_ATTN_EXT_TENSOR_NCPSG; // cache values per threadgroup + const int nsg = OP_FLASH_ATTN_EXT_TENSOR_NSG; + + if (has_mask) { + assert(ggml_metal_op_flash_attn_ext_extra_blk(op) != 0); + + ggml_metal_kargs_flash_attn_ext_blk args0 = { + /*.ne01 =*/ ne01, + /*.ne30 =*/ ne30, + /*.ne31 =*/ ne31, + /*.ne32 =*/ ne32, + /*.ne33 =*/ ne33, + /*.nb31 =*/ nb31, + /*.nb32 =*/ nb32, + /*.nb33 =*/ nb33, + }; + + auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_blk(lib, op, nqptg, ncpsg); + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); + ggml_metal_encoder_set_buffer (enc, bid_src3, 1); + ggml_metal_encoder_set_buffer (enc, bid_blk, 2); + + const int32_t nblk1 = ((ne01 + nqptg - 1)/nqptg); + const int32_t nblk0 = ((ne30 + ncpsg - 1)/ncpsg); + + ggml_metal_encoder_dispatch_threadgroups(enc, nblk0, nblk1, ne32*ne33, 32, 1, 1); + + ggml_metal_op_concurrency_reset(ctx); + } + + const int32_t ns10 = nb11_attn/nb10_attn; + const int32_t ns20 = nb21_attn/nb20_attn; + + ggml_metal_kargs_flash_attn_ext args = { + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.ne03 =*/ ne03, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.ne11 =*/ ne11, + /*.ne_12_2 =*/ ne12, + /*.ne_12_3 =*/ ne13, + /*.ns10 =*/ ns10, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb13 =*/ nb13_attn, + /*.ns20 =*/ ns20, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + /*.nb23 =*/ nb23_attn, + /*.ne31 =*/ ne31, + /*.ne32 =*/ ne32, + /*.ne33 =*/ ne33, + /*.nb31 =*/ nb31, + /*.nb32 =*/ nb32, + /*.nb33 =*/ nb33, + /*.ne1 =*/ ne1, + /*.ne2 =*/ ne2, + /*.ne3 =*/ ne3, + /*.scale =*/ scale, + /*.max_bias =*/ max_bias, + /*.m0 =*/ m0, + /*.m1 =*/ m1, + /*.n_head_log2 =*/ n_head_log2, + /*.logit_softcap =*/ logit_softcap, + }; + + // shared memory layout: queries (16-bit), scores (float), probabilities (16-bit), row scale (float), rescale flag (int) + const size_t smem = GGML_PAD(nqptg*ne00*sizeof(ggml_fp16_t) + nqptg*ncpsg*(sizeof(float) + sizeof(ggml_fp16_t)) + nqptg*sizeof(float) + sizeof(int32_t), 16); + + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_tensor(lib, op, has_mask, has_sinks, has_bias, has_scap); + + GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_src0, 1); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_v, 3); + ggml_metal_encoder_set_buffer (enc, bid_src3, 4); + ggml_metal_encoder_set_buffer (enc, bid_src4, 5); + ggml_metal_encoder_set_buffer (enc, bid_blk, 6); + ggml_metal_encoder_set_buffer (enc, bid_dst, 7); + + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + + ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nqptg - 1)/nqptg, ne02, ne03, 32, nsg, 1); + } else if (!use_sparse && !ggml_metal_op_flash_attn_ext_use_vec(op)) { // half8x8 kernel const int nqptg = OP_FLASH_ATTN_EXT_NQPSG; // queries per threadgroup const int ncpsg = OP_FLASH_ATTN_EXT_NCPSG; // cache values per simdgroup @@ -2717,12 +3689,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ne11, /*.ne_12_2 =*/ne12, /*.ne_12_3 =*/ne13, - /*.nb11 =*/nb11, - /*.nb12 =*/nb12, - /*.nb13 =*/nb13, - /*.nb21 =*/nb21, - /*.nb22 =*/nb22, - /*.nb23 =*/nb23, + /*.nb11 =*/nb11_attn, + /*.nb12 =*/nb12_attn, + /*.nb13 =*/nb13_attn, + /*.nb21 =*/nb21_attn, + /*.nb22 =*/nb22_attn, + /*.nb23 =*/nb23_attn, /*.ne31 =*/ne31, /*.ne32 =*/ne32, /*.ne33 =*/ne33, @@ -2735,8 +3707,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_pipeline(enc, pipeline0); ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); - ggml_metal_encoder_set_buffer (enc, bid_src1, 1); - ggml_metal_encoder_set_buffer (enc, bid_src2, 2); + ggml_metal_encoder_set_buffer (enc, bid_k, 1); + ggml_metal_encoder_set_buffer (enc, bid_v, 2); ggml_metal_encoder_set_buffer (enc, bid_src3, 3); ggml_metal_encoder_set_buffer (enc, bid_pad, 4); @@ -2781,36 +3753,25 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_op_concurrency_reset(ctx); } - const int is_q = ggml_is_quantized(op->src[1]->type) ? 1 : 0; + const int is_q = !use_kv_f16 && ggml_is_quantized(op->src[1]->type) ? 1 : 0; - // 2*(2*ncpsg) - // ncpsg soft_max values + ncpsg mask values - // - // 16*32*(nsg) - // the shared memory needed for the simdgroups to load the KV cache - // each thread loads (dequantizes) 16 head elements, there are 32 threads in th SG - // -#define FATTN_SMEM(nsg) (GGML_PAD((nqptg*(ne00 + 2*GGML_PAD(ne20, 64) + 2*(2*ncpsg)) + is_q*(16*32*(nsg)))*(sizeof(float)/2), 16)) + // shared memory layout (halfs unless noted): + // queries/attn/result: Q*(DK + 2*PAD2(DV,64) + 4*C) + // quantized KV scratch: 16*32*NSG (only when is_q) + const int64_t dv_pad = GGML_PAD(ne20, 64); - //int64_t nsgmax = 4; - // - //if (is_q) { - // nsgmax = 2; - // while (true) { - // const size_t smem = FATTN_SMEM(nsgmax); - // if (smem > props_dev->max_theadgroup_memory_size) { - // break; - // } - // nsgmax *= 2; - // } - // nsgmax /= 2; - //} + auto fa_smem = [&](int32_t nsg) -> size_t { + const size_t smem_half = nqptg*(ne00 + 2*dv_pad + 4*ncpsg) + is_q*(16*32*nsg); + return GGML_PAD(smem_half*sizeof(ggml_fp16_t), 16); + }; // simdgroups per threadgroup (a.k.a. warps) - //nsg = ne01 <= nqptg ? MAX(4, MIN(nsgmax, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32))) : 4; int32_t nsg = ne00 >= 512 ? 8 : 4; - const size_t smem = FATTN_SMEM(nsg); + const size_t smem = fa_smem(nsg); + + const int32_t ns10 = nb11_attn/nb10_attn; + const int32_t ns20 = nb21_attn/nb20_attn; ggml_metal_kargs_flash_attn_ext args = { /*.ne01 =*/ ne01, @@ -2822,14 +3783,14 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ ne11, /*.ne_12_2 =*/ ne12, /*.ne_12_3 =*/ ne13, - /*.ns10 =*/ int32_t(nb11/nb10), - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, - /*.ns20 =*/ int32_t(nb21/nb20), - /*.nb21 =*/ nb21, - /*.nb22 =*/ nb22, - /*.nb23 =*/ nb23, + /*.ns10 =*/ ns10, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb13 =*/ nb13_attn, + /*.ns20 =*/ ns20, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + /*.nb23 =*/ nb23_attn, /*.ne31 =*/ ne31, /*.ne32 =*/ ne32, /*.ne33 =*/ ne33, @@ -2847,36 +3808,79 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.logit_softcap =*/ logit_softcap, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, use_kv_f16, ns10, ns20); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, bid_src0, 1); - ggml_metal_encoder_set_buffer (enc, bid_src1, 2); - ggml_metal_encoder_set_buffer (enc, bid_src2, 3); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_v, 3); ggml_metal_encoder_set_buffer (enc, bid_src3, 4); ggml_metal_encoder_set_buffer (enc, bid_src4, 5); ggml_metal_encoder_set_buffer (enc, bid_pad, 6); ggml_metal_encoder_set_buffer (enc, bid_blk, 7); ggml_metal_encoder_set_buffer (enc, bid_dst, 8); + GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); ggml_metal_encoder_dispatch_threadgroups(enc, (ne01 + nqptg - 1)/nqptg, ne02, ne03, 32, nsg, 1); -#undef FATTN_SMEM } else { // half4x4 kernel - const int nqptg = OP_FLASH_ATTN_EXT_VEC_NQPSG; // queries per threadgroup + // sparse: the index lists are per query row, so a threadgroup can share KV with Q == 1 only + auto cfg = use_sparse + ? ggml_metal_tuning::fa_vec_baseline_cfg((int) ne00, (int) ne20) + : ggml_metal_tuning::fa_vec_pick( + props_dev->gpu_family, + (int) op->src[1]->type, + (int) ne00, (int) ne20, // dk, dv (ne00 == dk for FA) + ne11, ne01); + + int nqptg = cfg.Q; // queries per threadgroup + const int ncpsg = OP_FLASH_ATTN_EXT_VEC_NCPSG; // cache values per simdgroup !! sync with kernel template arguments !! const int nhptg = 1; // heads per threadgroup GGML_ASSERT(nqptg <= 32); - GGML_ASSERT(nqptg % 1 == 0); + GGML_ASSERT(nqptg == 1 || nqptg == 2 || nqptg == 4); // only instantiated Q values GGML_ASSERT(ncpsg % 32 == 0); bool need_sync = false; - const bool has_kvpad = ne11 % ncpsg != 0; + const bool has_kvpad = !use_sparse && ne11 % ncpsg != 0; + + if (use_sparse) { + assert(ggml_metal_op_flash_attn_ext_extra_idx(op) != 0); + + GGML_ASSERT(ne30 == ne11); + + ggml_metal_kargs_flash_attn_ext_vec_idx args0 = { + /*.ne30 =*/ ne30, + /*.ne31 =*/ ne31, + /*.ne32 =*/ ne32, + /*.ne33 =*/ ne33, + /*.nb31 =*/ nb31, + /*.nb32 =*/ nb32, + /*.nb33 =*/ nb33, + /*.n_kv_max =*/ n_kv_max_sparse, + /*.n_kv_max_padded =*/ n_kv_max_padded, + }; + + auto pipeline0 = ggml_metal_library_get_pipeline_flash_attn_ext_vec_idx(lib, op); + + ggml_metal_encoder_set_pipeline(enc, pipeline0); + ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); + ggml_metal_encoder_set_buffer (enc, bid_src3, 1); + ggml_metal_encoder_set_buffer (enc, bid_idx, 2); + + int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline0), 256); + nth = std::max(32, (nth/32)*32); + + ggml_metal_encoder_dispatch_threadgroups(enc, ne31, ne32, ne33, nth, 1, 1); + + need_sync = true; + } if (has_kvpad) { assert(ggml_metal_op_flash_attn_ext_extra_pad(op) != 0); @@ -2885,12 +3889,12 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.ne11 =*/ne11, /*.ne_12_2 =*/ne12, /*.ne_12_3 =*/ne13, - /*.nb11 =*/nb11, - /*.nb12 =*/nb12, - /*.nb13 =*/nb13, - /*.nb21 =*/nb21, - /*.nb22 =*/nb22, - /*.nb23 =*/nb23, + /*.nb11 =*/nb11_attn, + /*.nb12 =*/nb12_attn, + /*.nb13 =*/nb13_attn, + /*.nb21 =*/nb21_attn, + /*.nb22 =*/nb22_attn, + /*.nb23 =*/nb23_attn, /*.ne31 =*/ne31, /*.ne32 =*/ne32, /*.ne33 =*/ne33, @@ -2903,8 +3907,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_set_pipeline(enc, pipeline0); ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0); - ggml_metal_encoder_set_buffer (enc, bid_src1, 1); - ggml_metal_encoder_set_buffer (enc, bid_src2, 2); + ggml_metal_encoder_set_buffer (enc, bid_k, 1); + ggml_metal_encoder_set_buffer (enc, bid_v, 2); ggml_metal_encoder_set_buffer (enc, bid_src3, 3); ggml_metal_encoder_set_buffer (enc, bid_pad, 4); @@ -2923,25 +3927,44 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { // note: for simplicity assume the K is larger or equal than V GGML_ASSERT(ne10 >= ne20); - // ne00 + 2*ncpsg*(nsg) - // for each query, we load it as f16 in shared memory (ne00) - // and store the soft_max values and the mask - // - // ne20*(nsg) - // each simdgroup has a full f32 head vector in shared mem to accumulate results - // -#define FATTN_SMEM(nsg) (GGML_PAD(((GGML_PAD(ne00, 128) + 4*ncpsg + 2*GGML_PAD(ne20, 128))*(nsg))*(sizeof(float)/2), 16)) + // shared memory layout (halfs unless noted): + // queries: Q*NSG*PAD2(ne00, 128) + // attn + mask: NSG*4*Q*C + // results: 2*NSG*Q*PAD2(ne20, 128) + // sparse idx: NSG*C ints (only when use_sparse) + const int64_t dk_pad = GGML_PAD(ne00, 128); + const int64_t dv_pad = GGML_PAD(ne20, 128); + + auto fa_vec_smem = [&](int64_t nsg, int32_t nqptg) -> size_t { + const size_t smem_half = (size_t) (dk_pad + 4*ncpsg + 2*dv_pad)*nqptg*nsg; + return GGML_PAD(smem_half*sizeof(ggml_fp16_t) + (use_sparse ? (size_t) nsg*ncpsg*sizeof(int) : 0), 16); + }; int64_t nsg = 1; // workgroups // each workgroup handles nsg*nkpsg cache values int32_t nwg = 1; - if (false) { - // for small KV caches, we could launch a single workgroup and write the results directly to dst/ - // however, this does not lead to significant improvement, so disabled - nwg = 1; - nsg = 4; + if (use_sparse) { + if (ne01 > 32) { + // large sparse batch + nwg = 1; + nsg = 1; + if (n_kv_max_padded == 640) { + nsg = 4; // 640 % (4*32) == 0 + } else { + while (2*nwg*nsg*ncpsg < n_kv_max_padded && nsg < 4) { + nsg *= 2; + } + } + } else { + // small sparse batch + nwg = 32; + nsg = 1; + while (2*nwg*nsg*ncpsg < n_kv_max_padded && nsg < 4) { + nsg *= 2; + } + } } else { nwg = 32; nsg = 1; @@ -2950,6 +3973,15 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { } } + // fall back to baseline (Q=1) if the tuned config exceeds threadgroup memory + if (fa_vec_smem(nsg, nqptg) > props_dev->max_theadgroup_memory_size) { + cfg = ggml_metal_tuning::fa_vec_baseline_cfg((int) ne00, (int) ne20); + nqptg = cfg.Q; // = 1 + } + + const int32_t ns10 = nb11_attn/nb10_attn; + const int32_t ns20 = nb21_attn/nb20_attn; + ggml_metal_kargs_flash_attn_ext_vec args = { /*.ne01 =*/ ne01, /*.ne02 =*/ ne02, @@ -2957,17 +3989,17 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.nb01 =*/ nb01, /*.nb02 =*/ nb02, /*.nb03 =*/ nb03, - /*.ne11 =*/ ne11, + /*.ne11 =*/ use_sparse ? n_kv_max_padded : ne11, /*.ne_12_2 =*/ ne12, /*.ne_12_3 =*/ ne13, - /*.ns10 =*/ int32_t(nb11/nb10), - /*.nb11 =*/ nb11, - /*.nb12 =*/ nb12, - /*.nb13 =*/ nb13, - /*.ns20 =*/ int32_t(nb21/nb20), - /*.nb21 =*/ nb21, - /*.nb22 =*/ nb22, - /*.nb23 =*/ nb23, + /*.ns10 =*/ ns10, + /*.nb11 =*/ nb11_attn, + /*.nb12 =*/ nb12_attn, + /*.nb13 =*/ nb13_attn, + /*.ns20 =*/ ns20, + /*.nb21 =*/ nb21_attn, + /*.nb22 =*/ nb22_attn, + /*.nb23 =*/ nb23_attn, /*.ne31 =*/ ne31, /*.ne32 =*/ ne32, /*.ne33 =*/ ne33, @@ -2983,28 +4015,27 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { /*.m1 =*/ m1, /*.n_head_log2 =*/ n_head_log2, /*.logit_softcap =*/ logit_softcap, + /*.n_kv_max_padded =*/ n_kv_max_padded, }; - auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, nsg, nwg); + auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_vec(lib, op, has_mask, has_sinks, has_bias, has_scap, has_kvpad, use_sparse, nqptg, cfg.NE, nsg, nwg, use_kv_f16, ns10, ns20); GGML_ASSERT(nsg*32 <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, bid_src0, 1); - ggml_metal_encoder_set_buffer (enc, bid_src1, 2); - ggml_metal_encoder_set_buffer (enc, bid_src2, 3); + ggml_metal_encoder_set_buffer (enc, bid_k, 2); + ggml_metal_encoder_set_buffer (enc, bid_v, 3); ggml_metal_encoder_set_buffer (enc, bid_src3, 4); ggml_metal_encoder_set_buffer (enc, bid_src4, 5); + ggml_metal_encoder_set_buffer (enc, use_sparse ? bid_idx : bid_src0, 8); - const size_t smem = FATTN_SMEM(nsg); + const size_t smem = fa_vec_smem(nsg, nqptg); - //printf("smem: %zu, max: %zu, nsg = %d, nsgmax = %d\n", smem, props_dev->max_theadgroup_memory_size, (int) nsg, (int) nsgmax); GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size); if (nwg == 1) { - assert(ggml_metal_op_flash_attn_ext_extra_tmp(op) == 0); - // using 1 workgroup -> write the result directly into dst ggml_metal_encoder_set_buffer(enc, bid_pad, 6); ggml_metal_encoder_set_buffer(enc, bid_dst, 7); @@ -3047,21 +4078,41 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) { ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, 32*nwg, 1, 1); } } -#undef FATTN_SMEM } return 1; } int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { + int n_fuse = 1; + const ggml_metal_fusion * fusion = nullptr; + + const bool use_fusion = ctx->use_fusion(); + + if (use_fusion) { + int n = 1; + fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); + n_fuse = n; + + // snake activation autofuse: mul -> sin -> sqr -> mul -> add + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_SNAKE) { + ctx->count_fusions(fusion); + return ggml_metal_op_snake_fused(ctx, idx); + } + + // MoE output reduction: experts * weights -> weighted sum + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_MOE_REDUCE) { + ctx->count_fusions(fusion); + return ggml_metal_op_moe_reduce(ctx, idx); + } + } + ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; - - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -3070,9 +4121,6 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); - GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); GGML_ASSERT(ggml_is_contiguous_rows(op->src[1])); @@ -3109,57 +4157,19 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { /*.o1 =*/ { bid_src1.offs }, }; - ggml_op fops[8]; - - int n_fuse = 1; - // c[0] = add(a, b[0]) // c[1] = add(c[0], b[1]) // c[2] = add(c[1], b[2]) // ... - if (use_fusion) { - fops[0] = GGML_OP_ADD; - fops[1] = GGML_OP_ADD; - fops[2] = GGML_OP_ADD; - fops[3] = GGML_OP_ADD; - fops[4] = GGML_OP_ADD; - fops[5] = GGML_OP_ADD; - fops[6] = GGML_OP_ADD; - fops[7] = GGML_OP_ADD; - - // note: in metal, we sometimes encode the graph in parallel so we have to avoid fusing ops - // across splits. idx_end indicates the last node in the current split - for (n_fuse = 0; n_fuse <= 6; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; - } - - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); - - if (f0 != f1->src[0]) { - break; - } - - // b[0] === b[1] === ... - if (!ggml_are_same_layout(f0->src[1], f1->src[1])) { - break; - } - - // only fuse ops if src1 is in the same Metal buffer - ggml_metal_buffer_id bid_fuse = ggml_metal_get_buffer_id(f1->src[1]); - if (bid_fuse.metal != bid_src1.metal) { - break; - } - - //ctx->fuse_cnt[ops[n_fuse + 1]->op]++; - - args.o1[n_fuse + 1] = bid_fuse.offs; + if (use_fusion && fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_ADD_CHAIN) { + // the offsets of the fused addends are relative to the start of the src1 buffer + for (int i = 1; i < n_fuse; i++) { + args.o1[i] = ggml_metal_get_buffer_id(ctx->node(idx + i)->src[1]).offs; } - ++n_fuse; + ctx->count_fusions(fusion); - if (debug_fusion > 1 && n_fuse > 1) { + if (debug_fusion > 1) { GGML_LOG_DEBUG("%s: fuse: ADD x %d\n", __func__, n_fuse); } } @@ -3212,6 +4222,36 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) { return n_fuse; } +int ggml_metal_op_silu_back(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + auto pipeline = ggml_metal_library_get_pipeline_silu_back(lib, op); + + const int64_t ne = ggml_nelements(op); + + ggml_metal_kargs_silu_back args = { + /*.ne =*/ ne, + }; + + int arg_idx{0}; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), arg_idx++); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), arg_idx++); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), arg_idx++); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), arg_idx++); + + const int nth = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne); + const int64_t n = (ne + nth - 1) / nth; + + ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, nth, 1, 1); + + return 1; +} + int ggml_metal_op_l2_norm(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -3337,9 +4377,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; - const bool use_fusion = ctx->use_fusion; + const bool use_fusion = ctx->use_fusion(); - const int debug_fusion = ctx->debug_fusion; + const int debug_fusion = ggml_metal_fusion_info_debug(ctx->finfo); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); @@ -3365,67 +4405,61 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { /*.nbf1 =*/ { nb01 }, /*.nbf2 =*/ { nb02 }, /*.nbf3 =*/ { nb03 }, + /*.scale =*/ 1.0f, }; - ggml_op fops[8]; - int n_fuse = 1; + bool fused_norm_scale = false; ggml_metal_buffer_id bid_fuse[2] = { bid_src0, bid_src0 }; // d[0] = norm(a) - // d[1] = mul(d[0], b) + // d[1] = mul(d[0], b) or scale(d[0]) // d[2] = add(d[1], c) if (use_fusion) { - fops[0] = op->op; - fops[1] = GGML_OP_MUL; - fops[2] = GGML_OP_ADD; - - for (n_fuse = 0; n_fuse <= 1; ++n_fuse) { - if (!ctx->can_fuse(idx + n_fuse, fops + n_fuse, 2)) { - break; - } - - ggml_tensor * f0 = ctx->node(idx + n_fuse); - ggml_tensor * f1 = ctx->node(idx + n_fuse + 1); - - if (f0 != f1->src[0]) { - break; - } + int n = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n); - if (f1->src[1]->ne[0] != op->ne[0]) { - break; - } + if (fusion && (ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_NORM_MUL || ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_NORM_MUL_ADD)) { + n_fuse = n; - if (!ggml_is_contiguous_rows(f1->src[1])) { - break; - } + ctx->count_fusions(fusion); - if (f1->type != GGML_TYPE_F32) { - break; - } + for (int i = 1; i < n_fuse; i++) { + const ggml_tensor * fn = ctx->node(idx + i); - //ctx->fuse_cnt[f1->op]++; + bid_fuse[i - 1] = ggml_metal_get_buffer_id(fn->src[1]); - bid_fuse[n_fuse] = ggml_metal_get_buffer_id(f1->src[1]); + args.nef1[i] = fn->src[1]->ne[1]; + args.nef2[i] = fn->src[1]->ne[2]; + args.nef3[i] = fn->src[1]->ne[3]; - args.nef1[n_fuse + 1] = f1->src[1]->ne[1]; - args.nef2[n_fuse + 1] = f1->src[1]->ne[2]; - args.nef3[n_fuse + 1] = f1->src[1]->ne[3]; + args.nbf1[i] = fn->src[1]->nb[1]; + args.nbf2[i] = fn->src[1]->nb[2]; + args.nbf3[i] = fn->src[1]->nb[3]; + } - args.nbf1[n_fuse + 1] = f1->src[1]->nb[1]; - args.nbf2[n_fuse + 1] = f1->src[1]->nb[2]; - args.nbf3[n_fuse + 1] = f1->src[1]->nb[3]; + if (debug_fusion > 1) { + if (n_fuse == 2) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); + } + if (n_fuse == 3) { + GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + } + } } - ++n_fuse; + if (fusion && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_NORM_SCALE) { + n_fuse = n; + fused_norm_scale = true; - if (debug_fusion > 1 && n_fuse > 1) { - if (n_fuse == 2) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL\n", __func__, ggml_op_name(op->op)); - } - if (n_fuse == 3) { - GGML_LOG_DEBUG("%s: fuse: %s + MUL + ADD\n", __func__, ggml_op_name(op->op)); + ctx->count_fusions(fusion); + + const ggml_tensor * scale_node = ctx->node(idx + 1); + args.scale = ggml_get_op_params_f32(scale_node, 0); + + if (debug_fusion > 1) { + GGML_LOG_DEBUG("%s: fuse: %s + SCALE\n", __func__, ggml_op_name(op->op)); } } } @@ -3442,7 +4476,9 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { } } - auto pipeline = ggml_metal_library_get_pipeline_norm(lib, op, n_fuse); + auto pipeline = fused_norm_scale ? + ggml_metal_library_get_pipeline_norm_scale(lib, op) : + ggml_metal_library_get_pipeline_norm(lib, op, n_fuse); int nth = 32; // SIMD width @@ -3451,7 +4487,7 @@ int ggml_metal_op_norm(ggml_metal_op_t ctx, int idx) { } nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); - nth = std::min(nth, args.ne00_t); + nth = std::min(nth, (args.ne00_t + 31)/32*32); const size_t smem = pipeline.smem; @@ -3514,6 +4550,11 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) { const int sect_2 = ((const int32_t *) op->op_params)[13]; const int sect_3 = ((const int32_t *) op->op_params)[14]; + const int n_offs = ((const int32_t *) op->op_params)[15]; + + // when dst aliases src0, the channels outside the rotated window already hold the correct data + const bool inplace = op->data == op->src[0]->data; + ggml_metal_kargs_rope args = { /*.ne00 =*/ ne00, /*.ne01 =*/ ne01, @@ -3533,6 +4574,7 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) { /*.nb3 =*/ nb3, /*.n_past =*/ n_past, /*.n_dims =*/ n_dims, + /*.n_offs =*/ n_offs, /*.n_ctx_orig =*/ n_ctx_orig, /*.freq_base =*/ freq_base, /*.freq_scale =*/ freq_scale, @@ -3545,6 +4587,7 @@ int ggml_metal_op_rope(ggml_metal_op_t ctx, int idx) { /* sect_2 =*/ sect_2, /* sect_3 =*/ sect_3, /* src2 =*/ op->src[2] != nullptr, + /* inplace =*/ inplace, }; auto pipeline = ggml_metal_library_get_pipeline_rope(lib, op); @@ -3621,16 +4664,26 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) { auto pipeline = ggml_metal_library_get_pipeline_im2col(lib, op); - GGML_ASSERT(KH*KW <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + if (KH*KW <= ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) { + const uint64_t ntptg0 = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)/(KH*KW), N); - const uint64_t ntptg0 = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)/(KH*KW), N); + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); - ggml_metal_encoder_set_pipeline(enc, pipeline); - ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 1); - ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + ggml_metal_encoder_dispatch_threadgroups(enc, IC, OH, OW, ntptg0, KH, KW); + } else { + const uint64_t n_threads = std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), N); + const int64_t quotient = N / n_threads + (N % n_threads > 0 ? 1 : 0); - ggml_metal_encoder_dispatch_threadgroups(enc, IC, OH, OW, ntptg0, KH, KW); + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, quotient * CHW, OH, OW, n_threads, 1, 1); + } return 1; } @@ -3713,6 +4766,86 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) { return 1; } +int ggml_metal_op_conv_2d_dw(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + GGML_TENSOR_LOCALS( int32_t, ne1, op->src[1], ne); + GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb); + GGML_TENSOR_LOCALS( int32_t, ne, op, ne); + GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + + GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32); + GGML_ASSERT(op->type == GGML_TYPE_F32); + GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32); + + const int32_t s0 = ((const int32_t *) op->op_params)[0]; + const int32_t s1 = ((const int32_t *) op->op_params)[1]; + const int32_t p0 = ((const int32_t *) op->op_params)[2]; + const int32_t p1 = ((const int32_t *) op->op_params)[3]; + const int32_t d0 = ((const int32_t *) op->op_params)[4]; + const int32_t d1 = ((const int32_t *) op->op_params)[5]; + + ggml_metal_kargs_conv_2d_dw args = { + /*.nb00 =*/ nb00, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb03, + /*.nb10 =*/ nb10, + /*.nb11 =*/ nb11, + /*.nb12 =*/ nb12, + /*.nb13 =*/ nb13, + /*.nb0 =*/ nb0, + /*.nb1 =*/ nb1, + /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, + /*.IW =*/ ne10, + /*.IH =*/ ne11, + /*.KW =*/ ne00, + /*.KH =*/ ne01, + /*.C =*/ ne12, + /*.OW =*/ ne0, + /*.OH =*/ ne1, + /*.N =*/ ne13, + /*.s0 =*/ s0, + /*.s1 =*/ s1, + /*.p0 =*/ p0, + /*.p1 =*/ p1, + /*.d0 =*/ d0, + /*.d1 =*/ d1, + }; + + const bool use_tiled = (nb12 < nb10); + + auto pipeline = ggml_metal_library_get_pipeline_conv_2d_dw(lib, op, use_tiled); + + int nth = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline); + nth = std::min(nth, 256); + nth = std::max(nth, 1); + + const int32_t OW = ne0; + const int32_t OH = ne1; + const int32_t C = ne12; + const int32_t N = ne13; + + const int tg_x = use_tiled ? (C + nth - 1) / nth : (OW + nth - 1) / nth; + const int tg_y = OH; + const int tg_z = use_tiled ? OW * N : C * N; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3); + + ggml_metal_encoder_dispatch_threadgroups(enc, tg_x, tg_y, tg_z, nth, 1, 1); + + return 1; +} + int ggml_metal_op_conv_3d(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -3829,6 +4962,96 @@ int ggml_metal_op_conv_transpose_1d(ggml_metal_op_t ctx, int idx) { return 1; } +int ggml_metal_op_col2im_1d(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + const int32_t s0 = ((const int32_t *)(op->op_params))[0]; + const int32_t OC = ((const int32_t *)(op->op_params))[1]; + const int32_t p0 = ((const int32_t *)(op->op_params))[2]; + + const int32_t K_OC = (int32_t) op->src[0]->ne[0]; + const int32_t T_in = (int32_t) op->src[0]->ne[1]; + const int32_t K = K_OC / OC; + const int32_t T_out = (int32_t) op->ne[0]; + + ggml_metal_kargs_col2im_1d args = { + /*.T_in =*/ T_in, + /*.T_out =*/ T_out, + /*.OC =*/ OC, + /*.K =*/ K, + /*.K_OC =*/ K_OC, + /*.s0 =*/ s0, + /*.p0 =*/ p0, + }; + + auto pipeline = ggml_metal_library_get_pipeline_col2im_1d(lib, op); + + const int total = T_out * OC; + const int nth = 256; + const int ntg = (total + nth - 1) / nth; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + + ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); + + return 1; +} + +// Dispatch the fused snake kernel from the matched mul -> sin -> sqr -> mul -> add chain. +// idx points at the leading mul. The caller has validated the chain. +int ggml_metal_op_snake_fused(ggml_metal_op_t ctx, int idx) { + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + const ggml_tensor * mul0 = ctx->node(idx + 0); + const ggml_tensor * sqr = ctx->node(idx + 2); + const ggml_tensor * mul1 = ctx->node(idx + 3); + ggml_tensor * add = ctx->node(idx + 4); + + const ggml_tensor * x = ggml_are_same_shape(mul0, mul0->src[0]) ? mul0->src[0] : mul0->src[1]; + const ggml_tensor * a = (x == mul0->src[0]) ? mul0->src[1] : mul0->src[0]; + const ggml_tensor * inv_b = (mul1->src[0] == sqr) ? mul1->src[1] : mul1->src[0]; + + const int T = (int) x->ne[0]; + const int C = (int) x->ne[1]; + const int total = T * C; + + // the encode loop pre-checked the leading mul only, check the rest of the chain + for (int i = 1; i < 5; ++i) { + if (!ggml_metal_op_concurrency_check(ctx, ctx->node(idx + i))) { + ggml_metal_op_concurrency_reset(ctx); + + break; + } + } + + auto pipeline = ggml_metal_library_get_pipeline_snake(lib, x->type); + + ggml_metal_kargs_snake args = { + /*.T =*/ T, + /*.C =*/ C, + }; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(x), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(a), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(inv_b), 3); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(add), 4); + + const int nth = 256; + const int ntg = (total + nth - 1) / nth; + ggml_metal_encoder_dispatch_threadgroups(enc, ntg, 1, 1, nth, 1, 1); + + return 5; +} + int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -3854,6 +5077,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) { const int32_t OW = op->ne[0]; const int32_t OH = op->ne[1]; const int32_t OC = op->ne[2]; + const int32_t N = op->src[1]->ne[3]; ggml_metal_kargs_conv_transpose_2d args = { /*.IC =*/ IC, @@ -3866,6 +5090,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) { /*.nb0 =*/ nb0, /*.nb1 =*/ nb1, /*.nb2 =*/ nb2, + /*.nb3 =*/ nb3, }; auto pipeline = ggml_metal_library_get_pipeline_conv_transpose_2d(lib, op); @@ -3880,7 +5105,7 @@ int ggml_metal_op_conv_transpose_2d(ggml_metal_op_t ctx, int idx) { const size_t smem = GGML_PAD(KW * KH * sizeof(float), 16); ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); - ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC, KW, KH, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, OW, OH, OC * N, KW, KH, 1); return 1; } @@ -4029,19 +5254,25 @@ int ggml_metal_op_pad(ggml_metal_op_t ctx, int idx) { /*.nb0 =*/ nb0, /*.nb1 =*/ nb1, /*.nb2 =*/ nb2, - /*.nb3 =*/ nb3 + /*.nb3 =*/ nb3, + /*.lp0 =*/ ggml_get_op_params_i32(op, 0), + /*.lp1 =*/ ggml_get_op_params_i32(op, 2), + /*.lp2 =*/ ggml_get_op_params_i32(op, 4), + /*.lp3 =*/ ggml_get_op_params_i32(op, 6), }; auto pipeline = ggml_metal_library_get_pipeline_pad(lib, op); - const int nth = std::min(1024, ne0); + const int nth_max = MIN(64, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + const int nth = MIN(args.ne0, nth_max); + const int nk0 = (args.ne0 + 1024 - 1)/1024; // note: 1024 is hardcoded in the kernel! ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); - ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1); + ggml_metal_encoder_dispatch_threadgroups(enc, nk0*ne1, ne2, ne3, nth, 1, 1); return 1; } @@ -4160,6 +5391,25 @@ int ggml_metal_op_timestep_embedding(ggml_metal_op_t ctx, int idx) { return 1; } +size_t ggml_metal_op_argmax_extra(const ggml_tensor * op) { + assert(op->op == GGML_OP_ARGMAX); + + const int64_t ne00 = op->src[0]->ne[0]; + const int64_t ne01 = op->src[0]->ne[1]; + + int npart = 1; + if (ne00 > N_ARGMAX_CHUNK) { + npart = (int) std::min(N_ARGMAX_NPART_MAX, (ne00 + N_ARGMAX_CHUNK - 1)/N_ARGMAX_CHUNK); + } + + if (npart == 1) { + return 0; + } + + // partial max + partial argmax for every row + return (sizeof(float) + sizeof(int32_t)) * npart * ne01; +} + int ggml_metal_op_argmax(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -4171,22 +5421,61 @@ int ggml_metal_op_argmax(ggml_metal_op_t ctx, int idx) { GGML_TENSOR_LOCALS( int32_t, ne, op, ne); GGML_TENSOR_LOCALS(uint64_t, nb, op, nb); + const int64_t nrows = ggml_nrows(op->src[0]); + + int npart = 1; + if (ne00 > N_ARGMAX_CHUNK) { + npart = (int) std::min(N_ARGMAX_NPART_MAX, (ne00 + N_ARGMAX_CHUNK - 1)/N_ARGMAX_CHUNK); + } + ggml_metal_kargs_argmax args = { - /*.ne00 = */ ne00, - /*.nb01 = */ nb01, + /*.ne00 = */ ne00, + /*.nb01 = */ nb01, + /*.npart = */ npart, + /*.ne01 = */ ne01, }; - auto pipeline = ggml_metal_library_get_pipeline_argmax(lib, op); + const size_t smem = 32*(sizeof(float) + sizeof(int32_t)); - const int64_t nrows = ggml_nrows(op->src[0]); + if (npart > 1) { + auto pipeline_part = ggml_metal_library_get_pipeline_argmax_part(lib); + auto pipeline_final = ggml_metal_library_get_pipeline_argmax_final(lib); + + ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op); + ggml_metal_buffer_id bid_tmp = bid_dst; + bid_tmp.offs += ggml_nbytes(op); + + ggml_metal_encoder_set_pipeline(enc, pipeline_part); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, bid_tmp, 2); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, npart, nrows, 1, 256, 1, 1); + + ggml_metal_op_concurrency_reset(ctx); + + int nth = 32; // SIMD width + while (nth < npart && nth < 256) { + nth *= 2; + } + + ggml_metal_encoder_set_pipeline(enc, pipeline_final); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, bid_tmp, 1); + ggml_metal_encoder_set_buffer (enc, bid_dst, 2); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0); + ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, nth, 1, 1); + + return 1; + } + + auto pipeline = ggml_metal_library_get_pipeline_argmax(lib, op); int nth = 32; // SIMD width while (nth < ne00 && nth*ne01*ne02*ne03 < 256) { nth *= 2; } - const size_t smem = pipeline.smem; - ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); @@ -4306,7 +5595,9 @@ int ggml_metal_op_argsort(ggml_metal_op_t ctx, int idx) { return 1; } -int ggml_metal_op_top_k(ggml_metal_op_t ctx, int idx) { +// bitonic-sort + merge fallback: efficient when k is small and there are few rows, +// where the single-workgroup-per-row radix-select cannot reach enough parallelism +static void ggml_metal_op_top_k_bitonic(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); ggml_metal_library_t lib = ctx->lib; @@ -4414,6 +5705,178 @@ int ggml_metal_op_top_k(ggml_metal_op_t ctx, int idx) { len <<= 1; } +} + +// radix-select: one workgroup per row. Maps each float to an order-preserving unsigned +// key, finds the k-th largest via 4 radix-8 histogram passes, then compacts the top-k +// indices. Fast for large k and/or many rows. +static void ggml_metal_op_top_k_radix(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + GGML_ASSERT(ggml_is_contiguous_rows(op->src[0])); + + GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); + GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb); + + auto pipeline = ggml_metal_library_get_pipeline_top_k_radix(lib, op); + + // one workgroup per row; radix-select the k-th largest value + const int nth = std::min(1024, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + + ggml_metal_kargs_top_k args = { + /*.ne00 =*/ ne00, + /*.ne01 =*/ ne01, + /*.ne02 =*/ ne02, + /*.ne03 =*/ ne03, + /*.nb01 =*/ nb01, + /*.nb02 =*/ nb02, + /*.nb03 =*/ nb03, + /*.top_k =*/ (int32_t) op->ne[0], + }; + + // shared memory: 256-entry histogram + bucket/above scalars + output counter + const size_t smem_histo = GGML_PAD(256*sizeof(uint32_t), 16); + const size_t smem_bucket = GGML_PAD( sizeof(uint32_t), 16); + const size_t smem_above = GGML_PAD( sizeof(uint32_t), 16); + const size_t smem_out = GGML_PAD( sizeof(uint32_t), 16); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 2); + + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_histo, 0); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_bucket, 1); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_above, 2); + ggml_metal_encoder_set_threadgroup_memory_size(enc, smem_out, 3); + + ggml_metal_encoder_dispatch_threadgroups(enc, ne01, ne02, ne03, nth, 1, 1); +} + +int ggml_metal_op_topk_moe(ggml_metal_op_t ctx, int idx) { + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + int n_fuse = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n_fuse); + if (!fusion || ggml_metal_fusion_get_id(fusion) != GGML_METAL_FUSION_TOPK_MOE) { + return 1; + } + + ggml_tensor * softmax = ctx->node(idx); + ggml_tensor * logits = softmax->src[0]; + ggml_tensor * get_rows = ctx->node(idx + 2); + ggml_tensor * ids = get_rows->src[1]; + ggml_tensor * weights = ctx->node(idx + n_fuse - 1); + + const int64_t n_expert = logits->ne[0]; + const int64_t n_tokens = logits->ne[1]; + const int64_t n_expert_used = ids->ne[0]; + + const bool with_norm = n_fuse >= 6; + const bool with_scale = n_fuse == 4 || n_fuse == 7; + + float clamp = -INFINITY; + if (with_norm) { + ggml_tensor * clamp_node = ctx->node(idx + 4); + clamp = ggml_get_op_params_f32(clamp_node, 0); + } + + float scale = 1.0f; + if (with_scale) { + ggml_tensor * scale_node = ctx->node(idx + n_fuse - 1); + scale = ggml_get_op_params_f32(scale_node, 0); + } + + ggml_metal_kargs_topk_moe args = { + /*.ne01 =*/ (int32_t) n_tokens, + /*.nb01 =*/ logits->nb[1], + /*.nb1_ids =*/ ids->nb[1], + /*.clamp =*/ clamp, + /*.scale =*/ scale, + }; + + auto pipeline = ggml_metal_library_get_pipeline_topk_moe(lib, (int32_t) n_expert, (int32_t) n_expert_used, with_norm); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(logits), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(weights), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(ids), 3); + + ggml_metal_encoder_dispatch_threadgroups(enc, (uint32_t) n_tokens, 1, 1, 32, 1, 1); + + ctx->count_fusions(fusion); + + if (ggml_metal_fusion_info_debug(ctx->finfo) > 1) { + GGML_LOG_DEBUG("%s: fuse: SOFT_MAX + ARGSORT + GET_ROWS\n", __func__); + } + + return n_fuse; +} + +int ggml_metal_op_moe_reduce(ggml_metal_op_t ctx, int idx) { + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + int n_fuse = 1; + const ggml_metal_fusion * fusion = ctx->can_fuse(idx, GGML_METAL_FUSION_FULL, &n_fuse); + if (!fusion || ggml_metal_fusion_get_id(fusion) != GGML_METAL_FUSION_MOE_REDUCE) { + return 1; + } + + ggml_tensor * mul = ctx->node(idx); + ggml_tensor * experts = mul->src[0]; + ggml_tensor * weights = mul->src[1]; + ggml_tensor * dst = ctx->node(idx + n_fuse - 1); + + ggml_metal_kargs_moe_reduce args = { + /*.ne00 =*/ (int32_t) experts->ne[0], + /*.ne02 =*/ (int32_t) experts->ne[2], + }; + + auto pipeline = ggml_metal_library_get_pipeline_moe_reduce(lib, (int32_t) experts->ne[1]); + + const int nth = std::min(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)); + const int n_col_tiles = (args.ne00 + nth - 1) / nth; + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(experts), 1); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(weights), 2); + ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(dst), 3); + + ggml_metal_encoder_dispatch_threadgroups(enc, (uint32_t) args.ne02, (uint32_t) n_col_tiles, 1, nth, 1, 1); + + ctx->count_fusions(fusion); + + if (ggml_metal_fusion_info_debug(ctx->finfo) > 1) { + GGML_LOG_DEBUG("%s: fuse: MOE_REDUCE\n", __func__); + } + + return n_fuse; +} + +int ggml_metal_op_top_k(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + // radix-select has a fixed single-workgroup-per-row cost (~50-60us) that is only + // amortized for long rows, many rows, or a large k; otherwise the bitonic path wins + const int ncols = op->src[0]->ne[0]; + const int k = op->ne[0]; + const int nrows = ggml_nrows(op->src[0]); + + const bool use_radix = + ncols > 2048 && (k > 64 || (nrows > 4 && ncols >= 8192)); + + if (use_radix) { + ggml_metal_op_top_k_radix(ctx, idx); + } else { + ggml_metal_op_top_k_bitonic(ctx, idx); + } return 1; } diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index 36c6107..e14a2dd 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -8,17 +8,18 @@ extern "C" { typedef struct ggml_metal_op * ggml_metal_op_t; +struct ggml_metal_fusion; // forward decl (ggml-metal-device.h) + ggml_metal_op_t ggml_metal_op_init( ggml_metal_device_t dev, ggml_metal_cmd_buf_t cmd_buf, struct ggml_cgraph * gf, + struct ggml_metal_fusion_info * finfo, int idx_start, int idx_end, - bool use_fusion, bool use_concurrency, bool use_capture, - int debug_graph, - int debug_fusion); + int debug_graph); void ggml_metal_op_free(ggml_metal_op_t ctx); @@ -42,6 +43,10 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_pad(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_blk(const struct ggml_tensor * op); size_t ggml_metal_op_flash_attn_ext_extra_tmp(const struct ggml_tensor * op); +size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const struct ggml_tensor * op); +size_t ggml_metal_op_flash_attn_ext_extra_idx(const struct ggml_tensor * op); + +size_t ggml_metal_op_argmax_extra(const struct ggml_tensor * op); int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx); int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx); @@ -54,6 +59,8 @@ int ggml_metal_op_cumsum (ggml_metal_op_t ctx, int idx); int ggml_metal_op_get_rows (ggml_metal_op_t ctx, int idx); int ggml_metal_op_set_rows (ggml_metal_op_t ctx, int idx); int ggml_metal_op_diag (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_lightning_indexer (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_dsv4_hc (ggml_metal_op_t ctx, int idx); int ggml_metal_op_soft_max (ggml_metal_op_t ctx, int idx); int ggml_metal_op_ssm_conv (ggml_metal_op_t ctx, int idx); int ggml_metal_op_ssm_scan (ggml_metal_op_t ctx, int idx); @@ -64,20 +71,25 @@ int ggml_metal_op_set (ggml_metal_op_t ctx, int idx); int ggml_metal_op_cpy (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pool_1d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pool_2d (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_fwht (ggml_metal_op_t ctx, int idx); int ggml_metal_op_mul_mat (ggml_metal_op_t ctx, int idx); int ggml_metal_op_mul_mat_id (ggml_metal_op_t ctx, int idx); int ggml_metal_op_add_id (ggml_metal_op_t ctx, int idx); int ggml_metal_op_flash_attn_ext (ggml_metal_op_t ctx, int idx); int ggml_metal_op_bin (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_silu_back (ggml_metal_op_t ctx, int idx); int ggml_metal_op_l2_norm (ggml_metal_op_t ctx, int idx); int ggml_metal_op_group_norm (ggml_metal_op_t ctx, int idx); int ggml_metal_op_norm (ggml_metal_op_t ctx, int idx); int ggml_metal_op_rope (ggml_metal_op_t ctx, int idx); int ggml_metal_op_im2col (ggml_metal_op_t ctx, int idx); int ggml_metal_op_conv_2d (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_conv_2d_dw (ggml_metal_op_t ctx, int idx); int ggml_metal_op_conv_3d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_conv_transpose_1d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_conv_transpose_2d (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_col2im_1d (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_snake_fused (ggml_metal_op_t ctx, int idx); int ggml_metal_op_upscale (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pad (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pad_reflect_1d (ggml_metal_op_t ctx, int idx); @@ -87,6 +99,8 @@ int ggml_metal_op_timestep_embedding(ggml_metal_op_t ctx, int idx); int ggml_metal_op_argmax (ggml_metal_op_t ctx, int idx); int ggml_metal_op_argsort (ggml_metal_op_t ctx, int idx); int ggml_metal_op_top_k (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_topk_moe (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_moe_reduce (ggml_metal_op_t ctx, int idx); int ggml_metal_op_tri (ggml_metal_op_t ctx, int idx); int ggml_metal_op_opt_step_adamw (ggml_metal_op_t ctx, int idx); int ggml_metal_op_opt_step_sgd (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp new file mode 100644 index 0000000..0fe5027 --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -0,0 +1,1175 @@ +#include "ggml-metal-tuning.h" + +#include +#include +#include + +namespace ggml_metal_tuning { + +int fa_vec_ne11_bucket(int64_t ne11) { + for (int i = 0; i < (int) std::size(FA_VEC_NE11_BUCKETS); ++i) { + if (ne11 < FA_VEC_NE11_BUCKETS[i]) { + return i; + } + } + return (int) std::size(FA_VEC_NE11_BUCKETS); +} + +int fa_vec_ne01_bucket(int64_t ne01) { + for (int i = 0; i < (int) std::size(FA_VEC_NE01_BUCKETS); ++i) { + if (ne01 < FA_VEC_NE01_BUCKETS[i]) { + return i; + } + } + return (int) std::size(FA_VEC_NE01_BUCKETS); +} + +int fa_vec_baseline_ne(int dk, int dv) { + if (dk == 32 && dv == 32) { + return 4; + } + if (dk == 64 && dv == 64) { + return 2; + } + if (dk == 96 && dv == 96) { + return 4; + } + if (dk == 96 && dv == 64) { + return 4; + } + if (dk == 128 && dv == 128) { + return 1; + } + if (dk == 128 && dv == 96) { + return 4; + } + if (dk == 192 && dv == 192) { + return 2; + } + if (dk == 192 && dv == 128) { + return 2; + } + if (dk == 256 && dv == 256) { + return 1; + } + if (dk == 320 && dv == 256) { + return 2; + } + if (dk == 512 && dv == 512) { + return 1; + } + if (dk == 576 && dv == 512) { + return 2; + } + return 4; // template default +} + +fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv) { + return { 1, (int8_t) fa_vec_baseline_ne(dk, dv) }; +} + +// Generated by `ggml-metal-tuning fa-vec`; do not hand-edit. +// Keyed by Apple GPU family, retagged from the per-SKU token the tuner emits, and pooled from +// the sweeps listed at the head of each segment. One row per kept bucket, plus +// per-(dtype,dk,dv) ne11-collapsed domain defaults (ne11_b = FA_VEC_NE11_DEFAULT, +// ne01_b = domain). To retune or add a device, see tools/tuning/README.md. +// See ggml-metal-tuning.h for the row/lookup semantics. +// ref: https://github.com/ggml-org/llama.cpp/pull/27824 +// https://github.com/ggml-org/llama.cpp/discussions/27668 +constexpr fa_vec_entry_t fa_vec_tuned_table[] = { + // Apple7 - M1, M1_ULTRA + { { 7, GGML_TYPE_F16, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 2, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 3 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_F16, 320, 256, 1, 1 }, { 1, 2 } }, + { { 7, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, + { { 7, GGML_TYPE_F16, 320, 256, 1, 3 }, { 1, 2 } }, + { { 7, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 256, 256, 3, 2 }, { 1, 1 } }, + { { 7, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 96, 96, 1, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, + { { 7, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, + { { 7, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 1, 2 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 1, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 1, 4 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 2, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 96, 96, 3, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 192, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 192, 128, 2, 3 }, { 4, 2 } }, + { { 7, GGML_TYPE_Q5_1, 192, 128, 3, 3 }, { 4, 2 } }, + { { 7, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 256, 256, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 7, GGML_TYPE_Q8_0, 32, 32, 3, 4 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, 2, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, + { { 7, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 2, 4 } }, + { { 7, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 7, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + // Apple8 - M2, M2_MAX, M2_PRO + { { 8, GGML_TYPE_F16, 64, 64, 1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 64, 64, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 2 } }, + { { 8, GGML_TYPE_F16, 64, 64, 3, 2 }, { 1, 2 } }, + { { 8, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_F16, 64, 64, 3, 4 }, { 1, 2 } }, + { { 8, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_F16, 192, 128, 1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 320, 256, 2, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 320, 256, 2, 3 }, { 1, 4 } }, + { { 8, GGML_TYPE_F16, 320, 256, 2, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 2, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, 1, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 320, 256, 3, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 1, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 1, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 2, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 192, 3, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 3, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 3 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 256, 256, 1, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 2, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 2, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 1, 2 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 1, 4 }, { 1, 2 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 2, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 192, 128, 3, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 128, 128, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 1, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 192, 128, 3, 4 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 320, 256, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 576, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q5_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 2, 4 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 2, 2 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 4, 4 } }, + { { 8, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 2 } }, + { { 8, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 1, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 1, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 2, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 3, 1 }, { 2, 4 } }, + { { 8, GGML_TYPE_Q8_0, 320, 256, 3, 3 }, { 4, 2 } }, + { { 8, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 8, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + // Apple9 - A18_PRO, M3_MAX, M3_PRO, M3_ULTRA, M4, M4_MAX, M4_PRO + { { 9, GGML_TYPE_F16, 32, 32, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 32, 32, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 32, 32, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 32, 32, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 64, 64, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 1, 3 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 1, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 96, 96, 2, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_F16, 128, 128, 1, 1 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 128, 128, 1, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 128, 128, 2, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 128, 128, 2, 4 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_F16, 192, 192, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 192, 128, 2, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 256, 256, 2, 0 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 256, 256, 3, 0 }, { 1, 2 } }, + { { 9, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, + { { 9, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, 2, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 64, 64, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 128, 128, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 192, 1, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 192, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 320, 256, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 320, 256, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 1, 3 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_0, 576, 512, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 32, 32, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 32, 32, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_1, 32, 32, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_1, 64, 64, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 96, 96, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 96, 96, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, 1, 4 }, { 1, 1 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, 2, 2 }, { 1, 1 } }, + { { 9, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 1, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 192, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 320, 256, 3, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q4_1, 576, 512, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 1, 2 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 1, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 1, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 2, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 2, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 128, 128, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, 1, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 256, 256, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 1, 2 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 1, 3 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_0, 576, 512, 1, 4 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 1, 2 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 1, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 128, 128, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, 1, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 256, 256, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 320, 256, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 320, 256, 1, 3 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_1, 320, 256, 3, 3 }, { 2, 2 } }, + { { 9, GGML_TYPE_Q5_1, 512, 512, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q5_1, 576, 512, 2, 4 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 64, 64, 3, 3 }, { 4, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 3, 2 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, 1, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, 2, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 192, 2, 3 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 512, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 3, 0 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 1, 1 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 1, 3 }, { 1, 2 } }, + { { 9, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } }, + // Apple10 - M5_MAX + { { 10, GGML_TYPE_F16, 32, 32, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 1, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 32, 32, 3, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, 3, 0 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 64, 64, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 64, 64, 3, 3 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 64, 64, 3, 4 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 96, 96, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 96, 96, 3, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 3, 0 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 128, 128, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 128, 128, 1, 2 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 1, 4 }, { 1, 1 } }, + { { 10, GGML_TYPE_F16, 128, 128, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 128, 128, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 192, 192, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 192, 192, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 192, 192, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 192, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_F16, 192, 128, 1, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 192, 128, 3, 2 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 256, 256, -1, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 256, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 256, 256, 1, 2 }, { 1, 1 } }, + { { 10, GGML_TYPE_F16, 256, 256, 1, 4 }, { 1, 1 } }, + { { 10, GGML_TYPE_F16, 320, 256, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 320, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 320, 256, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 320, 256, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 512, 512, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_F16, 512, 512, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 512, 512, 2, 3 }, { 4, 2 } }, + { { 10, GGML_TYPE_F16, 512, 512, 3, 3 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 512, 512, 3, 4 }, { 4, 1 } }, + { { 10, GGML_TYPE_F16, 576, 512, 2, 0 }, { 4, 4 } }, + { { 10, GGML_TYPE_F16, 576, 512, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 1, 1 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_F16, 576, 512, 2, 3 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 1, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 32, 32, 3, 4 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 64, 64, 3, 4 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 1, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 96, 96, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, 1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, 2, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 128, 128, 3, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 2, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 192, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 192, 128, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 1, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 2, 3 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 3, 3 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 256, 256, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, 2, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 320, 256, 3, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 2, 3 }, { 4, 1 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 3, 1 }, { 4, 1 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 3, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_0, 512, 512, 3, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 2, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 3, 1 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_0, 576, 512, 3, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 1, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 2, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 3, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 32, 32, 3, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, 3, 0 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 64, 64, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 1, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 2, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 96, 96, 3, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 1, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 2, 1 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 3, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 128, 128, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 1, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 1, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 192, 3, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q4_1, 192, 128, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 3, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 2, 3 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 256, 256, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 2, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 320, 256, 3, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, -1, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 1, 3 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 2, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 3, 1 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 512, 512, 3, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q4_1, 576, 512, 2, 0 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_1, 576, 512, 1, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q4_1, 576, 512, 2, 1 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q4_1, 576, 512, 3, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 32, 32, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 64, 64, 3, 2 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 96, 96, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 96, 96, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 96, 96, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 96, 96, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 1, 0 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 128, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 1, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 1, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 2, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 192, 3, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 192, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 1, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 1, 2 }, { 1, 1 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 2, 2 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 256, 256, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 2, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 320, 256, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 3, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 1, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 512, 512, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 2, 1 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 2, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_0, 576, 512, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 32, 32, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 32, 32, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 32, 32, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 32, 32, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 64, 64, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 96, 96, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 96, 96, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 96, 96, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 96, 96, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 128, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 3, 0 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 192, 3, 3 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 2, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, -1, 1 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 192, 128, 3, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 2, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 1, 2 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 2, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 256, 256, 3, 4 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, -1, 1 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, 2, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 320, 256, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 2, 0 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 3, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 1, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 1, 4 }, { 1, 1 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 2, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q5_1, 512, 512, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 1, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 2, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 2, 3 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q5_1, 576, 512, 3, 3 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 1, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 32, 32, 3, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 64, 64, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 64, 64, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 64, 64, 1, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 64, 64, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 96, 96, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 96, 96, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 96, 96, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 96, 96, 3, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 1, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 2, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 2, 4 }, { 2, 2 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 3, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 128, 128, 3, 2 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, 1, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, 2, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, 3, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 192, 3, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q8_0, 192, 128, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 128, -1, 1 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 192, 128, 1, 2 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q8_0, 192, 128, 3, 2 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 256, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 256, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 256, 256, 3, 3 }, { 2, 4 } }, + { { 10, GGML_TYPE_Q8_0, 320, 256, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 320, 256, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 0 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 1, 3 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 3 }, { 4, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 2, 4 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 3, 1 }, { 4, 1 } }, + { { 10, GGML_TYPE_Q8_0, 512, 512, 3, 2 }, { 4, 2 } }, + { { 10, GGML_TYPE_Q8_0, 576, 512, -1, 0 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 576, 512, -1, 1 }, { 1, 4 } }, + { { 10, GGML_TYPE_Q8_0, 576, 512, 1, 4 }, { 1, 2 } }, + { { 10, GGML_TYPE_Q8_0, 576, 512, 2, 1 }, { 4, 4 } }, +}; + +static bool g_override_set = false; +static fa_vec_cfg_t g_override_cfg = { 1, 4 }; + +void fa_vec_set_override(fa_vec_cfg_t cfg) { + g_override_cfg = cfg; + g_override_set = true; +} + +void fa_vec_clear_override() { + g_override_set = false; +} + +static const fa_vec_cfg_t * find_cfg(const fa_vec_entry_t * tbl, size_t n, const fa_vec_key_t & k) { + for (size_t i = 0; i < n; ++i) { + if (memcmp(&tbl[i].key, &k, sizeof(k)) == 0) { + return &tbl[i].cfg; + } + } + return nullptr; +} + +fa_vec_cfg_t fa_vec_pick(int gpu_family, int dtype, int dk, int dv, int64_t ne11, int64_t ne01) { + if (g_override_set) { + return g_override_cfg; + } + + const fa_vec_cfg_t baseline = fa_vec_baseline_cfg(dk, dv); + + const int ne11_b = fa_vec_ne11_bucket(ne11); + if (ne11_b == 0) { + return baseline; // short KV: attention is a small slice of the step, left to baseline + } + + const int ne01_b = fa_vec_ne01_bucket(ne01); + + fa_vec_key_t k{}; + k.family = (int8_t) gpu_family; + k.dtype = (int8_t) dtype; + k.dk = (int16_t) dk; + k.dv = (int16_t) dv; + k.ne11_b = (int8_t) ne11_b; + k.ne01_b = (int8_t) ne01_b; + + // exact bucket, then the ne01 domain default (ne11 collapsed) + if (auto * c = find_cfg(fa_vec_tuned_table, std::size(fa_vec_tuned_table), k)) { + return *c; + } + k.ne11_b = FA_VEC_NE11_DEFAULT; + k.ne01_b = (ne01_b == 0) ? FA_VEC_DOMAIN_DECODE : FA_VEC_DOMAIN_BATCH; + if (auto * c = find_cfg(fa_vec_tuned_table, std::size(fa_vec_tuned_table), k)) { + return *c; + } + + return baseline; +} + +} // namespace ggml_metal_tuning diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.h b/ggml/src/ggml-metal/ggml-metal-tuning.h new file mode 100644 index 0000000..d36a0b6 --- /dev/null +++ b/ggml/src/ggml-metal/ggml-metal-tuning.h @@ -0,0 +1,75 @@ +#pragma once + +#include "ggml.h" + +#include +#include + +namespace ggml_metal_tuning { + +// FA vec selection buckets. ne01 (query rows) splits decode (==1) from batch (>=2), the +// batch side refined into {2,3,4,5}: Q>1 reuses one K/V load across rows, so it only pays +// off once ne01 aligns with Q. ne11 (KV length) is bucketed too, as the Q>1 crossover is +// head-size dependent (small dk crosses late, large dk wins even at short KV). +constexpr int FA_VEC_NE11_BUCKETS[] = { 1024, 4096, 16384 }; +constexpr int FA_VEC_NE01_BUCKETS[] = { 2, 3, 4, 5 }; + +int fa_vec_ne11_bucket(int64_t ne11); +int fa_vec_ne01_bucket(int64_t ne01); + +// NE baked into each (dk,dv) baseline instantiation in kernels/fa_vec_*.metal. +// Hand-maintained mirror; keep in sync with those instantiations. +// The Metal test slice covers every legal config for dk=128 and dk=576. +int fa_vec_baseline_ne(int dk, int dv); + +// Tuned table has two row kinds. Exact rows key a (ne11_b, ne01_b) bucket. Default rows +// collapse ne11 over one ne01 domain: ne11_b == FA_VEC_NE11_DEFAULT and ne01_b holds the +// domain. fa_vec_pick tries exact bucket -> domain default -> baseline; short KV +// (ne11 < FA_VEC_NE11_BUCKETS[0]) always uses baseline. +constexpr int8_t FA_VEC_NE11_DEFAULT = -1; +constexpr int8_t FA_VEC_DOMAIN_DECODE = 0; // ne01 == 1 +constexpr int8_t FA_VEC_DOMAIN_BATCH = 1; // ne01 >= 2 + +struct fa_vec_key_t { + int8_t family; + int8_t dtype; + int16_t dk; + int16_t dv; + int8_t ne11_b; + int8_t ne01_b; +}; + +static_assert(sizeof(fa_vec_key_t) == 8, "fa_vec_key_t must be tightly packed for memcmp"); + +struct fa_vec_cfg_t { + int8_t Q; + int8_t NE; +}; + +struct fa_vec_entry_t { + fa_vec_key_t key; + fa_vec_cfg_t cfg; +}; + +// legal NE values for a (dk,dv): NL = 32/NE, require (dk/4)%NL==0 && (dv/4)%NL==0. +// single source shared by the offline tuner and test-backend-ops. +inline std::vector fa_vec_legal_ne(int dk, int dv) { + std::vector r; + for (int ne : { 1, 2, 4 }) { + const int nl = 32 / ne; + if ((dk / 4) % nl == 0 && (dv / 4) % nl == 0) { + r.push_back(ne); + } + } + return r; +} + +// test/tune-only override; when set, fa_vec_pick returns it directly. +void fa_vec_set_override(fa_vec_cfg_t cfg); +void fa_vec_clear_override(); +fa_vec_cfg_t fa_vec_baseline_cfg(int dk, int dv); + +// Keyed by Apple GPU family; an untuned family matches no row and gets the baseline. +fa_vec_cfg_t fa_vec_pick(int gpu_family, int dtype, int dk, int dv, int64_t ne11, int64_t ne01); + +} // namespace ggml_metal_tuning diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 4dbf8e6..d0771eb 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -4,8 +4,10 @@ #include "ggml-backend-impl.h" #include "ggml-metal-device.h" +#include "ggml-metal-fusion.h" #include "ggml-metal-context.h" #include "ggml-metal-ops.h" +#include "ggml-metal-tuning.h" #include #include @@ -17,6 +19,9 @@ // note: can be overridden with GGML_METAL_DEVICES env to simulate virtual devices static int g_devices = 1; +// forward declaration +static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer); + //////////////////////////////////////////////////////////////////////////////// // backend interface //////////////////////////////////////////////////////////////////////////////// @@ -68,11 +73,11 @@ static bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t bu GGML_ASSERT(ggml_metal_buffer_is_shared(ctx)); - GGML_UNUSED(buffer); - GGML_UNUSED(src); - GGML_UNUSED(dst); + if (!ggml_backend_buffer_is_metal(src->buffer)) { + return false; + } - return false; + return ggml_metal_buffer_cpy_tensor(ctx, src, dst); } static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer, uint8_t value) { @@ -84,17 +89,17 @@ static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer, } static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = { - /* .free_buffer = */ ggml_backend_metal_buffer_shared_free_buffer, - /* .get_base = */ ggml_backend_metal_buffer_shared_get_base, - /* .init_tensor = */ NULL, - /* .memset_tensor = */ ggml_backend_metal_buffer_shared_memset_tensor, - /* .set_tensor = */ ggml_backend_metal_buffer_shared_set_tensor, - /* .get_tensor = */ ggml_backend_metal_buffer_shared_get_tensor, - /* .set_tensor_2d = */ NULL, - /* .get_tensor_2d = */ NULL, - /* .cpy_tensor = */ ggml_backend_metal_buffer_shared_cpy_tensor, - /* .clear = */ ggml_backend_metal_buffer_shared_clear, - /* .reset = */ NULL, + /* .free_buffer = */ ggml_backend_metal_buffer_shared_free_buffer, + /* .get_base = */ ggml_backend_metal_buffer_shared_get_base, + /* .init_tensor = */ NULL, + /* .memset_tensor = */ ggml_backend_metal_buffer_shared_memset_tensor, + /* .set_tensor = */ ggml_backend_metal_buffer_shared_set_tensor, + /* .get_tensor = */ ggml_backend_metal_buffer_shared_get_tensor, + /* .set_tensor_2d = */ NULL, + /* .get_tensor_2d = */ NULL, + /* .cpy_tensor = */ ggml_backend_metal_buffer_shared_cpy_tensor, + /* .clear = */ ggml_backend_metal_buffer_shared_clear, + /* .reset = */ NULL, }; // private buffer @@ -144,11 +149,11 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t b GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx)); - GGML_UNUSED(buffer); - GGML_UNUSED(src); - GGML_UNUSED(dst); + if (!ggml_backend_buffer_is_metal(src->buffer)) { + return false; + } - return false; + return ggml_metal_buffer_cpy_tensor(ctx, src, dst); } static void ggml_backend_metal_buffer_private_clear(ggml_backend_buffer_t buffer, uint8_t value) { @@ -160,17 +165,17 @@ static void ggml_backend_metal_buffer_private_clear(ggml_backend_buffer_t buffer } static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = { - /* .free_buffer = */ ggml_backend_metal_buffer_private_free_buffer, - /* .get_base = */ ggml_backend_metal_buffer_private_get_base, - /* .init_tensor = */ NULL, - /* .memset_tensor = */ ggml_backend_metal_buffer_private_memset_tensor, - /* .set_tensor = */ ggml_backend_metal_buffer_private_set_tensor, - /* .get_tensor = */ ggml_backend_metal_buffer_private_get_tensor, - /* .get_tensor_2d_async = */ NULL, - /* .set_tensor_2d_async = */ NULL, - /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor, - /* .clear = */ ggml_backend_metal_buffer_private_clear, - /* .reset = */ NULL, + /* .free_buffer = */ ggml_backend_metal_buffer_private_free_buffer, + /* .get_base = */ ggml_backend_metal_buffer_private_get_base, + /* .init_tensor = */ NULL, + /* .memset_tensor = */ ggml_backend_metal_buffer_private_memset_tensor, + /* .set_tensor = */ ggml_backend_metal_buffer_private_set_tensor, + /* .get_tensor = */ ggml_backend_metal_buffer_private_get_tensor, + /* .set_tensor_2d = */ NULL, + /* .get_tensor_2d = */ NULL, + /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor, + /* .clear = */ ggml_backend_metal_buffer_private_clear, + /* .reset = */ NULL, }; static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) { @@ -200,6 +205,11 @@ static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_ba ggml_metal_device_t ctx_dev = (ggml_metal_device_t)buft->device->context; ggml_metal_buffer_t res = ggml_metal_buffer_init(ctx_dev, size, shared); + if (res == NULL) { + GGML_LOG_ERROR("%s: failed to allocate Metal buffer of %zu bytes (out of memory)\n", __func__, size); + return NULL; + } + ggml_backend_buffer_i buf_i = ggml_metal_buffer_is_shared(res) ? ggml_backend_metal_buffer_shared_i : ggml_backend_metal_buffer_private_i; @@ -222,6 +232,8 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_ res += ggml_metal_op_flash_attn_ext_extra_pad(tensor); res += ggml_metal_op_flash_attn_ext_extra_blk(tensor); res += ggml_metal_op_flash_attn_ext_extra_tmp(tensor); + res += ggml_metal_op_flash_attn_ext_extra_kv_f16(tensor); + res += ggml_metal_op_flash_attn_ext_extra_idx(tensor); } break; case GGML_OP_CUMSUM: case GGML_OP_ARGSORT: @@ -232,6 +244,10 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_ { res = 2*sizeof(int32_t)*ggml_nelements(tensor->src[0]); } break; + case GGML_OP_ARGMAX: + { + res += ggml_metal_op_argmax_extra(tensor); + } break; default: break; } @@ -297,12 +313,14 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi ggml_backend_buffer_type buft = { /* .iface = */ { - /* .get_name = */ ggml_backend_metal_buffer_type_shared_get_name, - /* .alloc_buffer = */ ggml_backend_metal_buffer_type_shared_alloc_buffer, - /* .get_alignment = */ ggml_backend_metal_buffer_type_shared_get_alignment, - /* .get_max_size = */ ggml_backend_metal_buffer_type_shared_get_max_size, - /* .get_alloc_size = */ ggml_backend_metal_buffer_type_shared_get_alloc_size, - /* .is_host = */ ggml_backend_metal_buffer_type_shared_is_host, + /* .get_name = */ ggml_backend_metal_buffer_type_shared_get_name, + /* .alloc_buffer = */ ggml_backend_metal_buffer_type_shared_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_metal_buffer_type_shared_get_alignment, + /* .get_max_size = */ ggml_backend_metal_buffer_type_shared_get_max_size, + /* .get_alloc_size = */ ggml_backend_metal_buffer_type_shared_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_metal_buffer_type_shared_is_host, }, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_metal_reg(), i), /* .context = */ raw_ctx, @@ -372,12 +390,14 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev ggml_backend_buffer_type buft = { /* .iface = */ { - /* .get_name = */ ggml_backend_metal_buffer_type_private_get_name, - /* .alloc_buffer = */ ggml_backend_metal_buffer_type_private_alloc_buffer, - /* .get_alignment = */ ggml_backend_metal_buffer_type_private_get_alignment, - /* .get_max_size = */ ggml_backend_metal_buffer_type_private_get_max_size, - /* .get_alloc_size = */ ggml_backend_metal_buffer_type_private_get_alloc_size, - /* .is_host = */ ggml_backend_metal_buffer_type_private_is_host, + /* .get_name = */ ggml_backend_metal_buffer_type_private_get_name, + /* .alloc_buffer = */ ggml_backend_metal_buffer_type_private_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_metal_buffer_type_private_get_alignment, + /* .get_max_size = */ ggml_backend_metal_buffer_type_private_get_max_size, + /* .get_alloc_size = */ ggml_backend_metal_buffer_type_private_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_metal_buffer_type_private_is_host, }, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_metal_reg(), i), /* .context = */ raw_ctx, @@ -450,12 +470,14 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi // https://github.com/ggml-org/llama.cpp/pull/15832#discussion_r2333177099 ggml_backend_buffer_type buft = { /* .iface = */ { - /* .get_name = */ ggml_backend_metal_buffer_type_mapped_get_name, - /* .alloc_buffer = */ ggml_backend_metal_buffer_type_mapped_alloc_buffer, - /* .get_alignment = */ ggml_backend_metal_buffer_type_mapped_get_alignment, - /* .get_max_size = */ ggml_backend_metal_buffer_type_mapped_get_max_size, - /* .get_alloc_size = */ ggml_backend_metal_buffer_type_mapped_get_alloc_size, - /* .is_host = */ ggml_backend_metal_buffer_type_mapped_is_host, + /* .get_name = */ ggml_backend_metal_buffer_type_mapped_get_name, + /* .alloc_buffer = */ ggml_backend_metal_buffer_type_mapped_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_metal_buffer_type_mapped_get_alignment, + /* .get_max_size = */ ggml_backend_metal_buffer_type_mapped_get_max_size, + /* .get_alloc_size = */ ggml_backend_metal_buffer_type_mapped_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_metal_buffer_type_mapped_is_host, }, /* .device = */ ggml_backend_reg_dev_get(ggml_backend_metal_reg(), i), /* .context = */ raw_ctx, @@ -548,7 +570,13 @@ static void ggml_backend_metal_event_wait(ggml_backend_t backend, ggml_backend_e ggml_metal_event_wait(ctx, ev); } -static void ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) { +static void ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) { + GGML_ASSERT(params && params->add_alloc_dep); + + // keep the MoE weighted-reduction inputs alive until the fused output so the + // allocator cannot reuse them while the fused kernel is still reading them + ggml_metal_fusion_add_alloc_deps(params->user_data, params->add_alloc_dep, cgraph); + ggml_metal_t ctx = (ggml_metal_t)backend->context; ggml_metal_graph_optimize(ctx, cgraph); @@ -567,8 +595,8 @@ static ggml_backend_i ggml_backend_metal_i = { /* .free = */ ggml_backend_metal_free, /* .set_tensor_async = */ ggml_backend_metal_set_tensor_async, /* .get_tensor_async = */ ggml_backend_metal_get_tensor_async, - /* .get_tensor_2d_async = */ NULL, /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, /* .cpy_tensor_async = */ ggml_backend_metal_cpy_tensor_async, // only needed for multi-GPU setups /* .synchronize = */ ggml_backend_metal_synchronize, /* .graph_plan_create = */ NULL, @@ -678,6 +706,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ true, /* .events = */ true, + /* .mmap_support = */ true, }; } @@ -865,10 +894,96 @@ static ggml_backend_feature * ggml_backend_metal_get_features(ggml_backend_reg_t GGML_UNUSED(reg); } +// test/tune-only override for the FA vec (Q, NE) selection, reached via proc_address. +static void ggml_backend_metal_tuning_set_fa_vec_override(int Q, int NE) { + ggml_metal_tuning::fa_vec_set_override({ (int8_t) Q, (int8_t) NE }); +} + +static void ggml_backend_metal_tuning_clear_fa_vec_override(void) { + ggml_metal_tuning::fa_vec_clear_override(); +} + +static int ggml_backend_metal_tuning_fa_vec_ne11_bucket(int64_t ne11) { + return ggml_metal_tuning::fa_vec_ne11_bucket(ne11); +} + +static int ggml_backend_metal_tuning_fa_vec_ne01_bucket(int64_t ne01) { + return ggml_metal_tuning::fa_vec_ne01_bucket(ne01); +} + +static int ggml_backend_metal_tuning_fa_vec_baseline_ne(int dk, int dv) { + return ggml_metal_tuning::fa_vec_baseline_ne(dk, dv); +} + +static const char * ggml_backend_metal_tuning_device_token(ggml_backend_dev_t dev) { + ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context; + + return ggml_metal_device_id_token(ggml_metal_device_get_props(ctx_dev)->device_id); +} + +// generic fusion debugging API (ad-hoc proc-address mechanism): the test resolves the device +// fusion context once and passes that opaque handle to the rest of the functions +typedef void * ggml_backend_fusion_t; + +static ggml_backend_fusion_t ggml_backend_metal_fusion_get(ggml_backend_dev_t dev) { + return ggml_metal_device_get_fusion_info((ggml_metal_device_t)dev->context); +} + +static void ggml_backend_metal_fusion_stats_init(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_init((struct ggml_metal_fusion_info *) finfo); +} + +static void ggml_backend_metal_fusion_stats_reset(ggml_backend_fusion_t finfo) { + ggml_metal_fusion_info_stats_reset((struct ggml_metal_fusion_info *) finfo); +} + +static int ggml_backend_metal_fusion_stats_get(ggml_backend_fusion_t finfo, const char ** labels, uint64_t * counts, int n) { + return ggml_metal_fusion_info_stats_get((struct ggml_metal_fusion_info *) finfo, labels, counts, n); +} + +static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, bool enabled) { + ggml_metal_fusion_info_set_enabled((struct ggml_metal_fusion_info *) finfo, enabled); +} + static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *)ggml_backend_metal_get_features; } + if (strcmp(name, "ggml_backend_metal_tuning_set_fa_vec_override") == 0) { + return (void *)ggml_backend_metal_tuning_set_fa_vec_override; + } + if (strcmp(name, "ggml_backend_metal_tuning_clear_fa_vec_override") == 0) { + return (void *)ggml_backend_metal_tuning_clear_fa_vec_override; + } + if (strcmp(name, "ggml_backend_metal_tuning_fa_vec_ne11_bucket") == 0) { + return (void *)ggml_backend_metal_tuning_fa_vec_ne11_bucket; + } + if (strcmp(name, "ggml_backend_metal_tuning_fa_vec_ne01_bucket") == 0) { + return (void *)ggml_backend_metal_tuning_fa_vec_ne01_bucket; + } + if (strcmp(name, "ggml_backend_metal_tuning_fa_vec_baseline_ne") == 0) { + return (void *)ggml_backend_metal_tuning_fa_vec_baseline_ne; + } + if (strcmp(name, "ggml_backend_metal_tuning_device_token") == 0) { + return (void *)ggml_backend_metal_tuning_device_token; + } + // generic fusion debugging API (ad-hoc proc-address mechanism, not part of the official + // ggml backend interface yet; a backend that adopts it exports these exact names) + if (strcmp(name, "ggml_backend_fusion_get") == 0) { + return (void *)ggml_backend_metal_fusion_get; + } + if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) { + return (void *)ggml_backend_metal_fusion_stats_init; + } + if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) { + return (void *)ggml_backend_metal_fusion_stats_reset; + } + if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) { + return (void *)ggml_backend_metal_fusion_stats_get; + } + if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) { + return (void *)ggml_backend_metal_fusion_set_enabled; + } return NULL; @@ -886,7 +1001,7 @@ static ggml_backend_dev_t ggml_backend_metal_device_init(ggml_backend_reg_t reg, return new ggml_backend_device { /* .iface = */ ggml_backend_metal_device_i, /* .reg = */ reg, - /* .context = */ ggml_metal_device_get(device), + /* .context = */ ggml_metal_device_get(device, g_devices), }; } @@ -918,6 +1033,10 @@ ggml_backend_reg_t ggml_backend_metal_reg(void) { static std::vector devs; if (!initialized) { + // workaround macOS limitation (kIOGPUCommandBufferCallbackErrorImpactingInteractivity) until proper fix becomes possible + // ref: https://github.com/ggml-org/llama.cpp/issues/20141#issuecomment-4272947703 + setenv("AGX_RELAX_CDM_CTXSTORE_TIMEOUT", "1", true); + static ggml_backend_metal_reg_ptr reg_ctx(ggml_backend_metal_reg_init()); for (int i = 0; i < g_devices; ++i) { diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal deleted file mode 100644 index 9f38c9d..0000000 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ /dev/null @@ -1,10592 +0,0 @@ -#define GGML_COMMON_DECL_METAL -#define GGML_COMMON_IMPL_METAL -#if defined(GGML_METAL_EMBED_LIBRARY) -__embed_ggml-common.h__ -#else -#include "ggml-common.h" -#endif -#include "ggml-metal-impl.h" - -#include - -#ifdef GGML_METAL_HAS_TENSOR -#include - -#include -#endif - -using namespace metal; - -#define MAX(x, y) ((x) > (y) ? (x) : (y)) -#define MIN(x, y) ((x) < (y) ? (x) : (y)) -#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; } - -#define PAD2(x, n) (((x) + (n) - 1) & ~((n) - 1)) - -#define FOR_UNROLL(x) _Pragma("clang loop unroll(full)") for (x) - -#define N_SIMDWIDTH 32 // assuming SIMD group size is 32 - -// ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf -// -// cmd: -// .../usr/bin/metal -dM -E -c ggml/src/ggml-metal/ggml-metal.metal -// .../usr/bin/metal -dM -E -c -target air64-apple-ios14.0 ggml/src/ggml-metal/ggml-metal.metal -// -#if __METAL_VERSION__ < 310 && defined(GGML_METAL_HAS_BF16) -#undef GGML_METAL_HAS_BF16 -#endif - -#if defined(GGML_METAL_HAS_BF16) -typedef matrix bfloat4x4; -typedef matrix bfloat2x4; -#endif - -constexpr constant static float kvalues_iq4nl_f[16] = { - -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f -}; - -constexpr constant static float kvalues_mxfp4_f[16] = { - 0, .5f, 1.f, 1.5f, 2.f, 3.f, 4.f, 6.f, -0, -.5f, -1.f, -1.5f, -2.f, -3.f, -4.f, -6.f -}; - -static inline int best_index_int8(int n, constant float * val, float x) { - if (x <= val[0]) return 0; - if (x >= val[n-1]) return n-1; - int ml = 0, mu = n-1; - while (mu-ml > 1) { - int mav = (ml+mu)/2; - if (x < val[mav]) mu = mav; else ml = mav; - } - return x - val[mu-1] < val[mu] - x ? mu-1 : mu; -} - -static inline float e8m0_to_fp32(uint8_t x) { - uint32_t bits; - - if (x == 0) { - bits = 0x00400000; - } else { - bits = (uint32_t) x << 23; - } - - return as_type(bits); -} - -static inline float dot(float x, float y) { - return x*y; -} - -static inline float sum(float x) { - return x; -} - -static inline float sum(float4 x) { - return x[0] + x[1] + x[2] + x[3]; -} - -// NOTE: this is not dequantizing - we are simply fitting the template -template -void dequantize_f32(device const float4x4 * src, short il, thread type4x4 & reg) { - reg = (type4x4)(*src); -} - -template -void dequantize_f32_t4(device const float4 * src, short il, thread type4 & reg) { - reg = (type4)(*src); -} - -template -void dequantize_f16(device const half4x4 * src, short il, thread type4x4 & reg) { - reg = (type4x4)(*src); -} - -template -void dequantize_f16_t4(device const half4 * src, short il, thread type4 & reg) { - reg = (type4)(*(src)); -} - -#if defined(GGML_METAL_HAS_BF16) -template -void dequantize_bf16(device const bfloat4x4 * src, short il, thread type4x4 & reg) { - reg = (type4x4)(*src); -} - -template -void dequantize_bf16_t4(device const bfloat4 * src, short il, thread type4 & reg) { - reg = (type4)(*(src)); -} -#endif - -template -void dequantize_q1_0(device const block_q1_0 * xb, short il, thread type4x4 & reg) { - device const uint8_t * qs = xb->qs; - const float d = xb->d; - const float neg_d = -d; - - const int byte_offset = il * 2; // il*16 bits = il*2 bytes - const uint8_t b0 = qs[byte_offset]; - const uint8_t b1 = qs[byte_offset + 1]; - - float4x4 reg_f; - - reg_f[0][0] = select(neg_d, d, bool(b0 & 0x01)); - reg_f[0][1] = select(neg_d, d, bool(b0 & 0x02)); - reg_f[0][2] = select(neg_d, d, bool(b0 & 0x04)); - reg_f[0][3] = select(neg_d, d, bool(b0 & 0x08)); - reg_f[1][0] = select(neg_d, d, bool(b0 & 0x10)); - reg_f[1][1] = select(neg_d, d, bool(b0 & 0x20)); - reg_f[1][2] = select(neg_d, d, bool(b0 & 0x40)); - reg_f[1][3] = select(neg_d, d, bool(b0 & 0x80)); - - reg_f[2][0] = select(neg_d, d, bool(b1 & 0x01)); - reg_f[2][1] = select(neg_d, d, bool(b1 & 0x02)); - reg_f[2][2] = select(neg_d, d, bool(b1 & 0x04)); - reg_f[2][3] = select(neg_d, d, bool(b1 & 0x08)); - reg_f[3][0] = select(neg_d, d, bool(b1 & 0x10)); - reg_f[3][1] = select(neg_d, d, bool(b1 & 0x20)); - reg_f[3][2] = select(neg_d, d, bool(b1 & 0x40)); - reg_f[3][3] = select(neg_d, d, bool(b1 & 0x80)); - - reg = (type4x4) reg_f; -} - -template -void dequantize_q1_0_t4(device const block_q1_0 * xb, short il, thread type4 & reg) { - const float d = xb->d; - const float neg_d = -d; - const int base = il * 4; - const uint8_t byte = xb->qs[base / 8]; - const int s = base % 8; - - float4 reg_f; - reg_f[0] = select(neg_d, d, bool((byte >> (s )) & 1)); - reg_f[1] = select(neg_d, d, bool((byte >> (s + 1)) & 1)); - reg_f[2] = select(neg_d, d, bool((byte >> (s + 2)) & 1)); - reg_f[3] = select(neg_d, d, bool((byte >> (s + 3)) & 1)); - - reg = (type4) reg_f; -} - -template -void dequantize_q4_0(device const block_q4_0 * xb, short il, thread type4x4 & reg) { - device const uint16_t * qs = ((device const uint16_t *)xb + 1); - const float d1 = il ? (xb->d / 16.h) : xb->d; - const float d2 = d1 / 256.f; - const float md = -8.h * xb->d; - const ushort mask0 = il ? 0x00F0 : 0x000F; - const ushort mask1 = mask0 << 8; - - float4x4 reg_f; - - for (int i = 0; i < 8; i++) { - reg_f[i/2][2*(i%2) + 0] = d1 * (qs[i] & mask0) + md; - reg_f[i/2][2*(i%2) + 1] = d2 * (qs[i] & mask1) + md; - } - - reg = (type4x4) reg_f; -} - -template -void dequantize_q4_0_t4(device const block_q4_0 * xb, short il, thread type4 & reg) { - device const uint16_t * qs = ((device const uint16_t *)xb + 1); - const float d1 = (il/4) ? (xb->d / 16.h) : xb->d; - const float d2 = d1 / 256.f; - const float md = -8.h * xb->d; - const ushort mask0 = (il/4) ? 0x00F0 : 0x000F; - const ushort mask1 = mask0 << 8; - - for (int i = 0; i < 2; i++) { - reg[2*i + 0] = d1 * (qs[2*(il%4) + i] & mask0) + md; - reg[2*i + 1] = d2 * (qs[2*(il%4) + i] & mask1) + md; - } -} - -void quantize_q1_0(device const float * src, device block_q1_0 & dst) { - float sum_abs = 0.0f; - for (int j = 0; j < QK1_0; j++) { - sum_abs += fabs(src[j]); - } - dst.d = sum_abs / QK1_0; - - for (int j = 0; j < QK1_0 / 8; j++) { - dst.qs[j] = 0; - } - for (int j = 0; j < QK1_0; j++) { - if (src[j] >= 0.0f) { - dst.qs[j / 8] |= (1 << (j % 8)); - } - } -} - -void quantize_q4_0(device const float * src, device block_q4_0 & dst) { -#pragma METAL fp math_mode(safe) - float amax = 0.0f; // absolute max - float max = 0.0f; - - for (int j = 0; j < QK4_0; j++) { - const float v = src[j]; - if (amax < fabs(v)) { - amax = fabs(v); - max = v; - } - } - - const float d = max / -8; - const float id = d ? 1.0f/d : 0.0f; - - dst.d = d; - - for (int j = 0; j < QK4_0/2; ++j) { - const float x0 = src[0 + j]*id; - const float x1 = src[QK4_0/2 + j]*id; - - const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f)); - const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f)); - - dst.qs[j] = xi0; - dst.qs[j] |= xi1 << 4; - } -} - -void quantize_q4_1(device const float * src, device block_q4_1 & dst) { -#pragma METAL fp math_mode(safe) - float min = FLT_MAX; - float max = -FLT_MAX; - - for (int j = 0; j < QK4_1; j++) { - const float v = src[j]; - if (min > v) min = v; - if (max < v) max = v; - } - - const float d = (max - min) / ((1 << 4) - 1); - const float id = d ? 1.0f/d : 0.0f; - - dst.d = d; - dst.m = min; - - for (int j = 0; j < QK4_1/2; ++j) { - const float x0 = (src[0 + j] - min)*id; - const float x1 = (src[QK4_1/2 + j] - min)*id; - - const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f)); - const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f)); - - dst.qs[j] = xi0; - dst.qs[j] |= xi1 << 4; - } -} - -void quantize_q5_0(device const float * src, device block_q5_0 & dst) { -#pragma METAL fp math_mode(safe) - float amax = 0.0f; // absolute max - float max = 0.0f; - - for (int j = 0; j < QK5_0; j++) { - const float v = src[j]; - if (amax < fabs(v)) { - amax = fabs(v); - max = v; - } - } - - const float d = max / -16; - const float id = d ? 1.0f/d : 0.0f; - - dst.d = d; - - uint32_t qh = 0; - for (int j = 0; j < QK5_0/2; ++j) { - const float x0 = src[0 + j]*id; - const float x1 = src[QK5_0/2 + j]*id; - - const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f)); - const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f)); - - dst.qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); - qh |= ((xi0 & 0x10u) >> 4) << (j + 0); - qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2); - } - - thread const uint8_t * qh8 = (thread const uint8_t *)&qh; - - for (int j = 0; j < 4; ++j) { - dst.qh[j] = qh8[j]; - } -} - -void quantize_q5_1(device const float * src, device block_q5_1 & dst) { -#pragma METAL fp math_mode(safe) - float max = src[0]; - float min = src[0]; - - for (int j = 1; j < QK5_1; j++) { - const float v = src[j]; - min = v < min ? v : min; - max = v > max ? v : max; - } - - const float d = (max - min) / 31; - const float id = d ? 1.0f/d : 0.0f; - - dst.d = d; - dst.m = min; - - uint32_t qh = 0; - for (int j = 0; j < QK5_1/2; ++j) { - const float x0 = (src[0 + j] - min)*id; - const float x1 = (src[QK5_1/2 + j] - min)*id; - - const uint8_t xi0 = (uint8_t)(x0 + 0.5f); - const uint8_t xi1 = (uint8_t)(x1 + 0.5f); - - dst.qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); - qh |= ((xi0 & 0x10u) >> 4) << (j + 0); - qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2); - } - - thread const uint8_t * qh8 = (thread const uint8_t *)&qh; - - for (int j = 0; j < 4; ++j) { - dst.qh[j] = qh8[j]; - } -} - -void quantize_q8_0(device const float * src, device block_q8_0 & dst) { -#pragma METAL fp math_mode(safe) - float amax = 0.0f; // absolute max - - for (int j = 0; j < QK8_0; j++) { - const float v = src[j]; - amax = MAX(amax, fabs(v)); - } - - const float d = amax / ((1 << 7) - 1); - const float id = d ? 1.0f/d : 0.0f; - - dst.d = d; - - for (int j = 0; j < QK8_0; ++j) { - const float x0 = src[j]*id; - - dst.qs[j] = round(x0); - } -} - -void quantize_iq4_nl(device const float * src, device block_iq4_nl & dst) { -#pragma METAL fp math_mode(safe) - float amax = 0.0f; // absolute max - float max = 0.0f; - - for (int j = 0; j < QK4_NL; j++) { - const float v = src[j]; - if (amax < fabs(v)) { - amax = fabs(v); - max = v; - } - } - - const float d = max / kvalues_iq4nl_f[0]; - const float id = d ? 1.0f/d : 0.0f; - - float sumqx = 0, sumq2 = 0; - for (int j = 0; j < QK4_NL/2; ++j) { - const float x0 = src[0 + j]*id; - const float x1 = src[QK4_NL/2 + j]*id; - - const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl_f, x0); - const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl_f, x1); - - dst.qs[j] = xi0 | (xi1 << 4); - - const float v0 = kvalues_iq4nl_f[xi0]; - const float v1 = kvalues_iq4nl_f[xi1]; - const float w0 = src[0 + j]*src[0 + j]; - const float w1 = src[QK4_NL/2 + j]*src[QK4_NL/2 + j]; - sumqx += w0*v0*src[j] + w1*v1*src[QK4_NL/2 + j]; - sumq2 += w0*v0*v0 + w1*v1*v1; - - } - - dst.d = sumq2 > 0 ? sumqx/sumq2 : d; -} - -template -void dequantize_q4_1(device const block_q4_1 * xb, short il, thread type4x4 & reg) { - device const uint16_t * qs = ((device const uint16_t *)xb + 2); - const float d1 = il ? (xb->d / 16.h) : xb->d; - const float d2 = d1 / 256.f; - const float m = xb->m; - const ushort mask0 = il ? 0x00F0 : 0x000F; - const ushort mask1 = mask0 << 8; - - float4x4 reg_f; - - for (int i = 0; i < 8; i++) { - reg_f[i/2][2*(i%2) + 0] = ((qs[i] & mask0) * d1) + m; - reg_f[i/2][2*(i%2) + 1] = ((qs[i] & mask1) * d2) + m; - } - - reg = (type4x4) reg_f; -} - -template -void dequantize_q4_1_t4(device const block_q4_1 * xb, short il, thread type4 & reg) { - device const uint16_t * qs = ((device const uint16_t *)xb + 2); - const float d1 = (il/4) ? (xb->d / 16.h) : xb->d; - const float d2 = d1 / 256.f; - const float m = xb->m; - const ushort mask0 = (il/4) ? 0x00F0 : 0x000F; - const ushort mask1 = mask0 << 8; - - for (int i = 0; i < 2; i++) { - reg[2*i + 0] = d1 * (qs[2*(il%4) + i] & mask0) + m; - reg[2*i + 1] = d2 * (qs[2*(il%4) + i] & mask1) + m; - } -} - -template -void dequantize_q5_0(device const block_q5_0 * xb, short il, thread type4x4 & reg) { - device const uint16_t * qs = ((device const uint16_t *)xb + 3); - const float d = xb->d; - const float md = -16.h * xb->d; - const ushort mask = il ? 0x00F0 : 0x000F; - - const uint32_t qh = *((device const uint32_t *)xb->qh); - - const int x_mv = il ? 4 : 0; - - const int gh_mv = il ? 12 : 0; - const int gh_bk = il ? 0 : 4; - - float4x4 reg_f; - - for (int i = 0; i < 8; i++) { - // extract the 5-th bits for x0 and x1 - const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10; - const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10; - - // combine the 4-bits from qs with the 5th bit - const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0); - const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1); - - reg_f[i/2][2*(i%2) + 0] = d * x0 + md; - reg_f[i/2][2*(i%2) + 1] = d * x1 + md; - } - - reg = (type4x4) reg_f; -} - -template -void dequantize_q5_0_t4(device const block_q5_0 * xb, short il, thread type4 & reg) { - device const uint16_t * qs = ((device const uint16_t *)xb + 3); - const float d = xb->d; - const float md = -16.h * xb->d; - const ushort mask = (il/4) ? 0x00F0 : 0x000F; - - const uint32_t qh = *((device const uint32_t *)xb->qh); - - const int x_mv = (il/4) ? 4 : 0; - - const int gh_mv = (il/4) ? 12 : 0; - const int gh_bk = (il/4) ? 0 : 4; - - for (int ii = 0; ii < 2; ii++) { - int i = 2*(il%4) + ii; - - // extract the 5-th bits for x0 and x1 - const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10; - const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10; - - // combine the 4-bits from qs with the 5th bit - const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0); - const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1); - - reg[2*ii + 0] = d * x0 + md; - reg[2*ii + 1] = d * x1 + md; - } -} - -template -void dequantize_q5_1(device const block_q5_1 * xb, short il, thread type4x4 & reg) { - device const uint16_t * qs = ((device const uint16_t *)xb + 4); - const float d = xb->d; - const float m = xb->m; - const ushort mask = il ? 0x00F0 : 0x000F; - - const uint32_t qh = *((device const uint32_t *)xb->qh); - - const int x_mv = il ? 4 : 0; - - const int gh_mv = il ? 12 : 0; - const int gh_bk = il ? 0 : 4; - - float4x4 reg_f; - - for (int i = 0; i < 8; i++) { - // extract the 5-th bits for x0 and x1 - const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10; - const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10; - - // combine the 4-bits from qs with the 5th bit - const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0); - const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1); - - reg_f[i/2][2*(i%2) + 0] = d * x0 + m; - reg_f[i/2][2*(i%2) + 1] = d * x1 + m; - } - - reg = (type4x4) reg_f; -} - -template -void dequantize_q5_1_t4(device const block_q5_1 * xb, short il, thread type4 & reg) { - device const uint16_t * qs = ((device const uint16_t *)xb + 4); - const float d = xb->d; - const float m = xb->m; - const ushort mask = (il/4) ? 0x00F0 : 0x000F; - - const uint32_t qh = *((device const uint32_t *)xb->qh); - - const int x_mv = (il/4) ? 4 : 0; - - const int gh_mv = (il/4) ? 12 : 0; - const int gh_bk = (il/4) ? 0 : 4; - - for (int ii = 0; ii < 2; ii++) { - int i = 2*(il%4) + ii; - - // extract the 5-th bits for x0 and x1 - const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10; - const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10; - - // combine the 4-bits from qs with the 5th bit - const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0); - const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1); - - reg[2*ii + 0] = d * x0 + m; - reg[2*ii + 1] = d * x1 + m; - } -} - -template -void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & reg) { - device const int8_t * qs = ((device const int8_t *)xb->qs); - const float d = xb->d; - - float4x4 reg_f; - - for (int i = 0; i < 16; i++) { - reg_f[i/4][i%4] = (qs[i + 16*il] * d); - } - - reg = (type4x4) reg_f; -} - -template -void dequantize_q8_0_t4(device const block_q8_0 *xb, short il, thread type4 & reg) { - device const int8_t * qs = ((device const int8_t *)xb->qs); - const float d = xb->d; - - for (int i = 0; i < 4; i++) { - reg[i] = (qs[4*(il%4) + i + 16*(il/4)] * d); - } -} - -template -void dequantize_mxfp4(device const block_mxfp4 * xb, short il, thread type4x4 & reg) { - device const uint8_t * q2 = (device const uint8_t *)xb->qs; - - const float d = e8m0_to_fp32(xb->e); - const uint8_t shr = il >= 1 ? 4 : 0; - - for (int i = 0; i < 4; ++i) { - reg[i][0] = d * kvalues_mxfp4_f[(q2[4*i + 0] >> shr) & 0x0F]; - reg[i][1] = d * kvalues_mxfp4_f[(q2[4*i + 1] >> shr) & 0x0F]; - reg[i][2] = d * kvalues_mxfp4_f[(q2[4*i + 2] >> shr) & 0x0F]; - reg[i][3] = d * kvalues_mxfp4_f[(q2[4*i + 3] >> shr) & 0x0F]; - } -} - -template -void dequantize_mxfp4_t4(device const block_mxfp4 * xb, short il, thread type4 & reg) { - device const uint8_t * q2 = (device const uint8_t *)xb->qs; - - const float d = e8m0_to_fp32(xb->e); - const short il4 = il%4; - - const uint8_t shr = il >= 4 ? 4 : 0; - - reg[0] = d * kvalues_mxfp4_f[(q2[4*il4 + 0] >> shr) & 0x0F]; - reg[1] = d * kvalues_mxfp4_f[(q2[4*il4 + 1] >> shr) & 0x0F]; - reg[2] = d * kvalues_mxfp4_f[(q2[4*il4 + 2] >> shr) & 0x0F]; - reg[3] = d * kvalues_mxfp4_f[(q2[4*il4 + 3] >> shr) & 0x0F]; -} - -template -void dequantize_q2_K(device const block_q2_K *xb, short il, thread type4x4 & reg) { - const float d = xb->d; - const float min = xb->dmin; - device const uint8_t * q = (device const uint8_t *)xb->qs; - float dl, ml; - uint8_t sc = xb->scales[il]; - - q = q + 32*(il/8) + 16*(il&1); - il = (il/2)%4; - - half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h); - uchar mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3); - dl = d * (sc & 0xF) * coef, ml = min * (sc >> 4); - for (int i = 0; i < 16; ++i) { - reg[i/4][i%4] = dl * (q[i] & mask) - ml; - } -} - -template -void dequantize_q3_K(device const block_q3_K *xb, short il, thread type4x4 & reg) { - const half d_all = xb->d; - device const uint8_t * q = (device const uint8_t *)xb->qs; - device const uint8_t * h = (device const uint8_t *)xb->hmask; - device const int8_t * scales = (device const int8_t *)xb->scales; - - q = q + 32 * (il/8) + 16 * (il&1); - h = h + 16 * (il&1); - uint8_t m = 1 << (il/2); - uint16_t kmask1 = (il/4)>1 ? ((il/4)>2 ? 192 : 48) : \ - ((il/4)>0 ? 12 : 3); - uint16_t kmask2 = il/8 ? 0xF0 : 0x0F; - uint16_t scale_2 = scales[il%8], scale_1 = scales[8 + il%4]; - int16_t dl_int = (il/4)&1 ? (scale_2&kmask2) | ((scale_1&kmask1) << 2) - : (scale_2&kmask2) | ((scale_1&kmask1) << 4); - float dl = il<8 ? d_all * (dl_int - 32.f) : d_all * (dl_int / 16.f - 32.f); - const float ml = 4.f * dl; - - il = (il/2) & 3; - const half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h); - const uint8_t mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3); - dl *= coef; - - for (int i = 0; i < 16; ++i) { - reg[i/4][i%4] = dl * (q[i] & mask) - (h[i] & m ? 0 : ml); - } -} - -static inline uchar2 get_scale_min_k4_just2(int j, int k, device const uchar * q) { - return j < 4 ? uchar2{uchar(q[j+0+k] & 63), uchar(q[j+4+k] & 63)} - : uchar2{uchar((q[j+4+k] & 0xF) | ((q[j-4+k] & 0xc0) >> 2)), uchar((q[j+4+k] >> 4) | ((q[j-0+k] & 0xc0) >> 2))}; -} - -template -void dequantize_q4_K(device const block_q4_K * xb, short il, thread type4x4 & reg) { - device const uchar * q = xb->qs; - - short is = (il/4) * 2; - q = q + (il/4) * 32 + 16 * (il&1); - il = il & 3; - const uchar2 sc = get_scale_min_k4_just2(is, il/2, xb->scales); - const float d = il < 2 ? xb->d : xb->d / 16.h; - const float min = xb->dmin; - const float dl = d * sc[0]; - const float ml = min * sc[1]; - - const ushort mask = il < 2 ? 0x0F : 0xF0; - for (int i = 0; i < 16; ++i) { - reg[i/4][i%4] = dl * (q[i] & mask) - ml; - } -} - -template -void dequantize_q5_K(device const block_q5_K *xb, short il, thread type4x4 & reg) { - device const uint8_t * q = xb->qs; - device const uint8_t * qh = xb->qh; - - short is = (il/4) * 2; - q = q + 32 * (il/4) + 16 * (il&1); - qh = qh + 16 * (il&1); - uint8_t ul = 1 << (il/2); - il = il & 3; - const uchar2 sc = get_scale_min_k4_just2(is, il/2, xb->scales); - const float d = il < 2 ? xb->d : xb->d / 16.f; - const float min = xb->dmin; - const float dl = d * sc[0]; - const float ml = min * sc[1]; - - const ushort mask = il<2 ? 0x0F : 0xF0; - const float qh_val = il<2 ? 16.f : 256.f; - for (int i = 0; i < 16; ++i) { - reg[i/4][i%4] = dl * ((q[i] & mask) + (qh[i] & ul ? qh_val : 0)) - ml; - } -} - -template -void dequantize_q6_K(device const block_q6_K *xb, short il, thread type4x4 & reg) { - const half d_all = xb->d; - device const uint16_t * ql = (device const uint16_t *)xb->ql; - device const uint16_t * qh = (device const uint16_t *)xb->qh; - device const int8_t * scales = (device const int8_t *)xb->scales; - - ql = ql + 32*(il/8) + 16*((il/2)&1) + 8*(il&1); - qh = qh + 16*(il/8) + 8*(il&1); - float sc = scales[(il%2) + 2 * ((il/2))]; - il = (il/2) & 3; - - const uint32_t kmask1 = il>1 ? (il>2 ? 0xC0C0C0C0 : 0x30303030) : (il>0 ? 0x0C0C0C0C : 0x03030303); - const uint32_t kmask2 = il>1 ? 0xF0F0F0F0 : 0x0F0F0F0F; - const float ml = d_all * sc * 32.f; - const float dl0 = d_all * sc; - const float dl1 = dl0 / 256.f; - const float dl2 = dl0 / (256.f * 256.f); - const float dl3 = dl0 / (256.f * 256.f * 256.f); - const uint8_t shr_h = il>2 ? 2 : 0; - const uint8_t shl_h = il>1 ? 0 : (il>0 ? 2 : 4); - const uint8_t shr_l = il>1 ? 4 : 0; - for (int i = 0; i < 4; ++i) { - const uint32_t low = (ql[2*i] | (uint32_t)(ql[2*i+1] << 16)) & kmask2; - const uint32_t high = (qh[2*i] | (uint32_t)(qh[2*i+1] << 16)) & kmask1; - const uint32_t q = ((high << shl_h) >> shr_h) | (low >> shr_l); - reg[i][0] = dl0 * ((half)(q & 0xFF)) - ml; - reg[i][1] = dl1 * ((float)(q & 0xFF00)) - ml; - reg[i][2] = dl2 * ((float)(q & 0xFF0000)) - ml; - reg[i][3] = dl3 * ((float)(q & 0xFF000000)) - ml; - } -} - -template -void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x4 & reg) { - // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 - const float d = xb->d; - const int ib32 = il/2; - il = il%2; - // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 - // each block of 32 needs 2 uint32_t's for the quants & scale, so 4 uint16_t's. - device const uint16_t * q2 = xb->qs + 4*ib32; - const uint32_t aux32_g = q2[0] | (q2[1] << 16); - const uint32_t aux32_s = q2[2] | (q2[3] << 16); - thread const uint8_t * aux8 = (thread const uint8_t *)&aux32_g; - const float dl = d * (0.5f + (aux32_s >> 28)) * 0.25f; - constant uint8_t * grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+0]); - uint8_t signs = ksigns_iq2xs[(aux32_s >> 14*il) & 127]; - for (int i = 0; i < 8; ++i) { - reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); - } - grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+1]); - signs = ksigns_iq2xs[(aux32_s >> (14*il+7)) & 127]; - for (int i = 0; i < 8; ++i) { - reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); - } -} - -template -void dequantize_iq2_xs(device const block_iq2_xs * xb, short il, thread type4x4 & reg) { - // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 - const float d = xb->d; - const int ib32 = il/2; - il = il%2; - // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 - device const uint16_t * q2 = xb->qs + 4*ib32; - const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f; - constant uint8_t * grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+0] & 511)); - uint8_t signs = ksigns_iq2xs[q2[2*il+0] >> 9]; - for (int i = 0; i < 8; ++i) { - reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); - } - grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+1] & 511)); - signs = ksigns_iq2xs[q2[2*il+1] >> 9]; - for (int i = 0; i < 8; ++i) { - reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); - } -} - -template -void dequantize_iq3_xxs(device const block_iq3_xxs * xb, short il, thread type4x4 & reg) { - // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 - const float d = xb->d; - const int ib32 = il/2; - il = il%2; - // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 - device const uint8_t * q3 = xb->qs + 8*ib32; - device const uint16_t * gas = (device const uint16_t *)(xb->qs + QK_K/4) + 2*ib32; - const uint32_t aux32 = gas[0] | (gas[1] << 16); - const float dl = d * (0.5f + (aux32 >> 28)) * 0.5f; - constant uint8_t * grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+0]); - constant uint8_t * grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+1]); - uint8_t signs = ksigns_iq2xs[(aux32 >> 14*il) & 127]; - for (int i = 0; i < 4; ++i) { - reg[0][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f); - reg[1][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f); - } - grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+2]); - grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+3]); - signs = ksigns_iq2xs[(aux32 >> (14*il+7)) & 127]; - for (int i = 0; i < 4; ++i) { - reg[2][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f); - reg[3][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f); - } -} - -template -void dequantize_iq3_s(device const block_iq3_s * xb, short il, thread type4x4 & reg) { - // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 - const float d = xb->d; - const int ib32 = il/2; - il = il%2; - // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 - device const uint8_t * qs = xb->qs + 8*ib32; - device const uint8_t * signs = xb->signs + 4*ib32 + 2*il; - const uint8_t qh = xb->qh[ib32] >> 4*il; - const float dl = d * (1 + 2*((xb->scales[ib32/2] >> 4*(ib32%2)) & 0xf)); - constant uint8_t * grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+0] | ((qh << 8) & 256))); - constant uint8_t * grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+1] | ((qh << 7) & 256))); - for (int i = 0; i < 4; ++i) { - reg[0][i] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i+0]); - reg[1][i] = dl * grid2[i] * select(1, -1, signs[0] & kmask_iq2xs[i+4]); - } - grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+2] | ((qh << 6) & 256))); - grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+3] | ((qh << 5) & 256))); - for (int i = 0; i < 4; ++i) { - reg[2][i] = dl * grid1[i] * select(1, -1, signs[1] & kmask_iq2xs[i+0]); - reg[3][i] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i+4]); - } -} - -template -void dequantize_iq2_s(device const block_iq2_s * xb, short il, thread type4x4 & reg) { - // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 - const float d = xb->d; - const int ib32 = il/2; - il = il%2; - // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 - device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; - device const uint8_t * signs = qs + QK_K/8; - const uint8_t qh = xb->qh[ib32] >> 4*il; - const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f; - constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[0] | ((qh << 8) & 0x300))); - constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[1] | ((qh << 6) & 0x300))); - for (int i = 0; i < 8; ++i) { - reg[i/4+0][i%4] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i]); - reg[i/4+2][i%4] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i]); - } -} - -template -void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) { - // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 - const int ib32 = il/2; - il = il%2; - const float d = xb->d; - device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; - device const uint16_t * qh = xb->qh; - const float dl = d * (2*((qh[ib32] >> 12) & 7) + 1); - const float ml = dl * (qh[ib32] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA); - const uint16_t h = qh[ib32] >> 6*il; - constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((h << 8) & 0x700))); - constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((h << 5) & 0x700))); - for (int i = 0; i < 4; ++i) { - reg[0][i] = dl * (grid1[i] & 0xf) + ml; - reg[1][i] = dl * (grid1[i] >> 4) + ml; - reg[2][i] = dl * (grid2[i] & 0xf) + ml; - reg[3][i] = dl * (grid2[i] >> 4) + ml; - } -} - -template -void dequantize_iq1_m(device const block_iq1_m * xb, short il, thread type4x4 & reg) { - // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 - const int ib32 = il/2; - il = il%2; - device const uint16_t * sc = (device const uint16_t *)xb->scales; - - iq1m_scale_t scale; - scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); - const float d = scale.f16; - - device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; - device const uint8_t * qh = xb->qh + 2*ib32 + il; - - const float dl = d * (2*((sc[ib32/2] >> (6*(ib32%2)+3*il)) & 7) + 1); - const float ml1 = dl * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); - const float ml2 = dl * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); - constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); - constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700))); - for (int i = 0; i < 4; ++i) { - reg[0][i] = dl * (grid1[i] & 0xf) + ml1; - reg[1][i] = dl * (grid1[i] >> 4) + ml1; - reg[2][i] = dl * (grid2[i] & 0xf) + ml2; - reg[3][i] = dl * (grid2[i] >> 4) + ml2; - } -} - -template -void dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4 & reg) { - device const uint16_t * q4 = (device const uint16_t *)xb->qs; - const float d = xb->d; - uint32_t aux32; - thread const uint8_t * q8 = (thread const uint8_t *)&aux32; - for (int i = 0; i < 4; ++i) { - aux32 = ((q4[2*i] | (q4[2*i+1] << 16)) >> 4*il) & 0x0f0f0f0f; - reg[i][0] = d * kvalues_iq4nl_f[q8[0]]; - reg[i][1] = d * kvalues_iq4nl_f[q8[1]]; - reg[i][2] = d * kvalues_iq4nl_f[q8[2]]; - reg[i][3] = d * kvalues_iq4nl_f[q8[3]]; - } -} - -template -void dequantize_iq4_nl_t4(device const block_iq4_nl * xb, short il, thread type4 & reg) { - device const uint16_t * q4 = (device const uint16_t *)xb->qs; - const float d = xb->d; - uint32_t aux32; - thread const uint8_t * q8 = (thread const uint8_t *)&aux32; - aux32 = ((q4[2*(il%4)] | (q4[2*(il%4)+1] << 16)) >> 4*(il/4)) & 0x0f0f0f0f; - reg[0] = d * kvalues_iq4nl_f[q8[0]]; - reg[1] = d * kvalues_iq4nl_f[q8[1]]; - reg[2] = d * kvalues_iq4nl_f[q8[2]]; - reg[3] = d * kvalues_iq4nl_f[q8[3]]; -} - -template -void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 & reg) { - // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 - const int ib32 = il/2; - il = il%2; - // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 - device const uint32_t * q4 = (device const uint32_t *)xb->qs + 4*ib32; - const int ls = ((xb->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((xb->scales_h >> 2*ib32) & 3) << 4); - const float d = (float)xb->d * (ls - 32); - uint32_t aux32; - thread const uint8_t * q8 = (thread const uint8_t *)&aux32; - for (int i = 0; i < 4; ++i) { - aux32 = (q4[i] >> 4*il) & 0x0f0f0f0f; - reg[i][0] = d * kvalues_iq4nl_f[q8[0]]; - reg[i][1] = d * kvalues_iq4nl_f[q8[1]]; - reg[i][2] = d * kvalues_iq4nl_f[q8[2]]; - reg[i][3] = d * kvalues_iq4nl_f[q8[3]]; - } -} - -enum ggml_sort_order { - GGML_SORT_ORDER_ASC, - GGML_SORT_ORDER_DESC, -}; - -constant float GELU_COEF_A = 0.044715f; -constant float GELU_QUICK_COEF = -1.702f; -constant float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; -constant float SQRT_2_INV = 0.70710678118654752440084436210484f; - -// based on Abramowitz and Stegun formula 7.1.26 or similar Hastings' approximation -// ref: https://www.johndcook.com/blog/python_erf/ -constant float p_erf = 0.3275911f; -constant float a1_erf = 0.254829592f; -constant float a2_erf = -0.284496736f; -constant float a3_erf = 1.421413741f; -constant float a4_erf = -1.453152027f; -constant float a5_erf = 1.061405429f; - -template -inline T erf_approx(T x) { - T sign_x = sign(x); - x = fabs(x); - T t = 1.0f / (1.0f + p_erf * x); - T y = 1.0f - (((((a5_erf * t + a4_erf) * t) + a3_erf) * t + a2_erf) * t + a1_erf) * t * exp(-x * x); - return sign_x * y; -} - -template T elu_approx(T x); - -template<> inline float elu_approx(float x) { - return (x > 0.f) ? x : (exp(x) - 1); -} - -template<> inline float4 elu_approx(float4 x) { - float4 res; - - res[0] = (x[0] > 0.0f) ? x[0] : (exp(x[0]) - 1.0f); - res[1] = (x[1] > 0.0f) ? x[1] : (exp(x[1]) - 1.0f); - res[2] = (x[2] > 0.0f) ? x[2] : (exp(x[2]) - 1.0f); - res[3] = (x[3] > 0.0f) ? x[3] : (exp(x[3]) - 1.0f); - - return res; -} - -constant short FC_unary_op [[function_constant(FC_UNARY + 0)]]; -constant bool FC_unary_cnt[[function_constant(FC_UNARY + 1)]]; - -template -kernel void kernel_unary_impl( - constant ggml_metal_kargs_unary & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { -#define FC_OP FC_unary_op -#define FC_CNT FC_unary_cnt - - device const T0 * src0_ptr; - device T * dst_ptr; - - int i0; - - if (FC_CNT) { - i0 = tgpig.x; - - src0_ptr = (device const T0 *) (src0); - dst_ptr = (device T *) (dst); - } else { - const int i03 = tgpig.z; - const int i02 = tgpig.y; - const int k0 = tgpig.x/args.ne01; - const int i01 = tgpig.x - k0*args.ne01; - - i0 = k0*ntg.x + tpitg.x; - - src0_ptr = (device const T0 *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); - dst_ptr = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1 ); - } - - { - //threadgroup_barrier(mem_flags::mem_none); - - if (!FC_CNT) { - if (i0 >= args.ne0) { - return; - } - } - - const TC x = (TC) src0_ptr[i0]; - - if (FC_OP == OP_UNARY_NUM_SCALE) { - dst_ptr[i0] = (T) (args.scale * x + args.bias); - } - - if (FC_OP == OP_UNARY_NUM_FILL) { - dst_ptr[i0] = (T) args.val; - } - - if (FC_OP == OP_UNARY_NUM_CLAMP) { - dst_ptr[i0] = (T) clamp(x, args.min, args.max); - } - - if (FC_OP == OP_UNARY_NUM_SQR) { - dst_ptr[i0] = (T) (x * x); - } - - if (FC_OP == OP_UNARY_NUM_SQRT) { - dst_ptr[i0] = (T) sqrt(x); - } - - if (FC_OP == OP_UNARY_NUM_SIN) { - dst_ptr[i0] = (T) sin(x); - } - - if (FC_OP == OP_UNARY_NUM_COS) { - dst_ptr[i0] = (T) cos(x); - } - - if (FC_OP == OP_UNARY_NUM_LOG) { - dst_ptr[i0] = (T) log(x); - } - - if (FC_OP == OP_UNARY_NUM_LEAKY_RELU) { - dst_ptr[i0] = (T) (TC(x > 0)*x + TC(x <= 0)*(x * args.slope)); - } - - if (FC_OP == OP_UNARY_NUM_TANH) { - dst_ptr[i0] = (T) precise::tanh(x); - } - - if (FC_OP == OP_UNARY_NUM_RELU) { - dst_ptr[i0] = (T) fmax(0, x); - } - - if (FC_OP == OP_UNARY_NUM_SIGMOID) { - dst_ptr[i0] = (T) (1 / (1 + exp(-x))); - } - - if (FC_OP == OP_UNARY_NUM_GELU) { - dst_ptr[i0] = (T) (0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x)))); - } - - if (FC_OP == OP_UNARY_NUM_GELU_ERF) { - dst_ptr[i0] = (T) (0.5*x*(1 + erf_approx(SQRT_2_INV*x))); - } - - if (FC_OP == OP_UNARY_NUM_GELU_QUICK) { - dst_ptr[i0] = (T) (x * (1/(1 + exp(GELU_QUICK_COEF*x)))); - } - - if (FC_OP == OP_UNARY_NUM_SILU) { - dst_ptr[i0] = (T) (x / (1 + exp(-x))); - } - - if (FC_OP == OP_UNARY_NUM_ELU) { - dst_ptr[i0] = (T) elu_approx(x); - } - - if (FC_OP == OP_UNARY_NUM_NEG) { - dst_ptr[i0] = (T) -x; - } - - if (FC_OP == OP_UNARY_NUM_ABS) { - dst_ptr[i0] = (T) fabs(x); - } - - if (FC_OP == OP_UNARY_NUM_SGN) { - dst_ptr[i0] = T(x > 0) - T(x < 0); - } - - if (FC_OP == OP_UNARY_NUM_STEP) { - dst_ptr[i0] = T(x > 0); - } - - if (FC_OP == OP_UNARY_NUM_HARDSWISH) { - dst_ptr[i0] = (T) (x * fmax(0, fmin(1, x/6 + 0.5))); - } - - if (FC_OP == OP_UNARY_NUM_HARDSIGMOID) { - dst_ptr[i0] = (T) fmax(0, fmin(1, x/6 + 0.5)); - } - - if (FC_OP == OP_UNARY_NUM_EXP) { - dst_ptr[i0] = (T) exp(x); - } - - if (FC_OP == OP_UNARY_NUM_SOFTPLUS) { - dst_ptr[i0] = (T) select(log(1 + exp(x)), x, x > 20); - } - - if (FC_OP == OP_UNARY_NUM_EXPM1) { - // TODO: precise implementation - dst_ptr[i0] = (T) (exp(x) - 1); - } - - if (FC_OP == OP_UNARY_NUM_FLOOR) { - dst_ptr[i0] = (T) floor(x); - } - - if (FC_OP == OP_UNARY_NUM_CEIL) { - dst_ptr[i0] = (T) ceil(x); - } - - if (FC_OP == OP_UNARY_NUM_ROUND) { - dst_ptr[i0] = (T) round(x); - } - - if (FC_OP == OP_UNARY_NUM_TRUNC) { - dst_ptr[i0] = (T) trunc(x); - } - - if (FC_OP == OP_UNARY_NUM_XIELU) { - const TC xi = x; - const TC gate = TC(xi > TC(0.0f)); - const TC clamped = fmin(xi, TC(args.val)); - const TC y_pos = TC(args.scale) * xi * xi + TC(args.bias) * xi; - const TC y_neg = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi; - dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg); - } - } - -#undef FC_OP -#undef FC_CNT -} - -typedef decltype(kernel_unary_impl) kernel_unary_t; - -template [[host_name("kernel_unary_f32_f32")]] kernel kernel_unary_t kernel_unary_impl; -template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl; -template [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl; -template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl; - -// OP: 0 - add, 1 - sub, 2 - mul, 3 - div -constant short FC_bin_op [[function_constant(FC_BIN + 0)]]; -constant short FC_bin_f [[function_constant(FC_BIN + 1)]]; -constant bool FC_bin_rb [[function_constant(FC_BIN + 2)]]; -constant bool FC_bin_cb [[function_constant(FC_BIN + 3)]]; - -template -kernel void kernel_bin_fuse_impl( - constant ggml_metal_kargs_bin & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { -#define FC_OP FC_bin_op -#define FC_F FC_bin_f -#define FC_RB FC_bin_rb -#define FC_CB FC_bin_cb - - if (FC_RB) { - // row broadcast - const uint i0 = tgpig.y*args.ne00 + tgpig.x; - const uint i1 = FC_CB ? tgpig.x%args.ne10 : tgpig.x; - - device const T0 * src0_row = (device const T0 *) (src0); - device T * dst_row = (device T *) (dst); - - if (FC_F == 1) { - device const T1 * src1_row = (device const T1 *) (src1 + args.o1[0]); - - if (FC_OP == 0) { - dst_row[i0] = src0_row[i0] + src1_row[i1]; - } - - if (FC_OP == 1) { - dst_row[i0] = src0_row[i0] - src1_row[i1]; - } - - if (FC_OP == 2) { - dst_row[i0] = src0_row[i0] * src1_row[i1]; - } - - if (FC_OP == 3) { - dst_row[i0] = src0_row[i0] / src1_row[i1]; - } - } else { - T0 res = src0_row[i0]; - - if (FC_OP == 0) { - FOR_UNROLL (short j = 0; j < FC_F; ++j) { - res += ((device const T1 *) (src1 + args.o1[j]))[i1]; - } - } - - if (FC_OP == 1) { - FOR_UNROLL (short j = 0; j < FC_F; ++j) { - res -= ((device const T1 *) (src1 + args.o1[j]))[i1]; - } - } - - if (FC_OP == 2) { - FOR_UNROLL (short j = 0; j < FC_F; ++j) { - res *= ((device const T1 *) (src1 + args.o1[j]))[i1]; - } - } - - if (FC_OP == 3) { - FOR_UNROLL (short j = 0; j < FC_F; ++j) { - res /= ((device const T1 *) (src1 + args.o1[j]))[i1]; - } - } - - dst_row[i0] = res; - } - } else { - const int i03 = tgpig.z; - const int i02 = tgpig.y; - const int i01 = tgpig.x; - - if (i01 >= args.ne01) { - return; - } - - const int i13 = i03%args.ne13; - const int i12 = i02%args.ne12; - const int i11 = i01%args.ne11; - - device const T0 * src0_ptr = (device const T0 *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + args.offs); - device T * dst_ptr = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1 + args.offs); - - if (FC_F == 1) { - device const T1 * src1_ptr = (device const T1 *) (src1 + args.o1[0] + i13*args.nb13 + i12*args.nb12 + i11*args.nb11); - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - const int i10 = FC_CB ? i0%args.ne10 : i0; - - if (FC_OP == 0) { - dst_ptr[i0] = src0_ptr[i0] + src1_ptr[i10]; - } - - if (FC_OP == 1) { - dst_ptr[i0] = src0_ptr[i0] - src1_ptr[i10]; - } - - if (FC_OP == 2) { - dst_ptr[i0] = src0_ptr[i0] * src1_ptr[i10]; - } - - if (FC_OP == 3) { - dst_ptr[i0] = src0_ptr[i0] / src1_ptr[i10]; - } - } - } else { - device const T1 * src1_ptr[8]; - FOR_UNROLL (short j = 0; j < FC_F; ++j) { - src1_ptr[j] = (device const T1 *) (src1 + args.o1[j] + i13*args.nb13 + i12*args.nb12 + i11*args.nb11); - } - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - const int i10 = FC_CB ? i0%args.ne10 : i0; - - T res = src0_ptr[i0]; - - if (FC_OP == 0) { - FOR_UNROLL (short j = 0; j < FC_F; ++j) { - res += src1_ptr[j][i10]; - } - } - - if (FC_OP == 1) { - FOR_UNROLL (short j = 0; j < FC_F; ++j) { - res -= src1_ptr[j][i10]; - } - } - - if (FC_OP == 2) { - FOR_UNROLL (short j = 0; j < FC_F; ++j) { - res *= src1_ptr[j][i10]; - } - } - - if (FC_OP == 3) { - FOR_UNROLL (short j = 0; j < FC_F; ++j) { - res /= src1_ptr[j][i10]; - } - } - - dst_ptr[i0] = res; - } - } - } - -#undef FC_OP -#undef FC_F -#undef FC_RB -#undef FC_CB -} - -typedef decltype(kernel_bin_fuse_impl) kernel_bin_fuse_t; - -template [[host_name("kernel_bin_fuse_f32_f32_f32")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; -template [[host_name("kernel_bin_fuse_f32_f32_f32_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; - -kernel void kernel_add_id( - constant ggml_metal_kargs_add_id & args, - device const char * src0, - device const char * src1, - device const char * src2, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int i1 = tgpig.x; - const int i2 = tgpig.y; - - const int i11 = *((device const int32_t *) (src2 + i1*sizeof(int32_t) + i2*args.nb21)); - - const size_t nb1 = args.ne0 * sizeof(float); - const size_t nb2 = args.ne1 * nb1; - - device float * dst_row = (device float *)((device char *)dst + i1*nb1 + i2*nb2); - device const float * src0_row = (device const float *)((device char *)src0 + i1*args.nb01 + i2*args.nb02); - device const float * src1_row = (device const float *)((device char *)src1 + i11*args.nb11); - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - dst_row[i0] = src0_row[i0] + src1_row[i0]; - } -} - -template -kernel void kernel_repeat( - constant ggml_metal_kargs_repeat & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int i3 = tgpig.z; - const int i2 = tgpig.y; - const int i1 = tgpig.x; - - const int i03 = i3%args.ne03; - const int i02 = i2%args.ne02; - const int i01 = i1%args.ne01; - - device const char * src0_ptr = src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01; - device char * dst_ptr = dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1; - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - const int i00 = i0%args.ne00; - *((device T *)(dst_ptr + i0*args.nb0)) = *((device T *)(src0_ptr + i00*args.nb00)); - } -} - -typedef decltype(kernel_repeat) kernel_repeat_t; - -template [[host_name("kernel_repeat_f32")]] kernel kernel_repeat_t kernel_repeat; -template [[host_name("kernel_repeat_f16")]] kernel kernel_repeat_t kernel_repeat; -template [[host_name("kernel_repeat_i32")]] kernel kernel_repeat_t kernel_repeat; -template [[host_name("kernel_repeat_i16")]] kernel kernel_repeat_t kernel_repeat; - -kernel void kernel_reglu_f32( - constant ggml_metal_kargs_glu & args, - device const char * src0, - device const char * src1, - device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { - const float x0 = src0_row[i0]; - const float x1 = src1_row[i0]; - - dst_row[i0] = x0*x1*(x0 > 0.0f); - } -} - -kernel void kernel_geglu_f32( - constant ggml_metal_kargs_glu & args, - device const char * src0, - device const char * src1, - device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { - const float x0 = src0_row[i0]; - const float x1 = src1_row[i0]; - - const float gelu = 0.5f*x0*(1.0f + precise::tanh(SQRT_2_OVER_PI*x0*(1.0f + GELU_COEF_A*x0*x0))); - - dst_row[i0] = gelu*x1; - } -} - -kernel void kernel_swiglu_f32( - constant ggml_metal_kargs_glu & args, - device const char * src0, - device const char * src1, - device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { - const float x0 = src0_row[i0]; - const float x1 = src1_row[i0]; - - const float silu = x0 / (1.0f + exp(-x0)); - - dst_row[i0] = silu*x1; - } -} - -kernel void kernel_swiglu_oai_f32( - constant ggml_metal_kargs_glu & args, - device const char * src0, - device const char * src1, - device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { - float x0 = src0_row[i0]; - float x1 = src1_row[i0]; - - x0 = min(x0, args.limit); - x1 = max(min(x1, args.limit), -args.limit); - - float out_glu = x0 / (1.0f + exp(-x0 * args.alpha)); - out_glu = out_glu * (1.0f + x1); - - dst_row[i0] = out_glu; - } -} - -kernel void kernel_geglu_erf_f32( - constant ggml_metal_kargs_glu & args, - device const char * src0, - device const char * src1, - device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { - const float x0 = src0_row[i0]; - const float x1 = src1_row[i0]; - - const float gelu_erf = 0.5f*x0*(1.0f+erf_approx(x0*SQRT_2_INV)); - - dst_row[i0] = gelu_erf*x1; - } -} - -kernel void kernel_geglu_quick_f32( - constant ggml_metal_kargs_glu & args, - device const char * src0, - device const char * src1, - device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; - device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; - device float * dst_row = (device float *) ((device char *) dst + tgpig*args.nb1); - - for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { - const float x0 = src0_row[i0]; - const float x1 = src1_row[i0]; - - const float gelu_quick = x0*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x0))); - - dst_row[i0] = gelu_quick*x1; - } -} - -kernel void kernel_op_sum_f32( - constant ggml_metal_kargs_sum & args, - device const float * src0, - device float * dst, - threadgroup float * shmem_f32 [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - - if (args.np == 0) { - return; - } - - // TODO: become function constant - const uint nsg = (ntg.x + 31) / 32; - - float sumf = 0; - - for (uint64_t i0 = tpitg.x; i0 < args.np; i0 += ntg.x) { - sumf += src0[i0]; - } - - sumf = simd_sum(sumf); - - if (tiisg == 0) { - shmem_f32[sgitg] = sumf; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - float total = 0; - - if (sgitg == 0) { - float v = 0; - - if (tpitg.x < nsg) { - v = shmem_f32[tpitg.x]; - } - - total = simd_sum(v); - - if (tpitg.x == 0) { - dst[0] = total; - } - } -} - -constant short FC_sum_rows_op [[function_constant(FC_SUM_ROWS + 0)]]; - -template -kernel void kernel_sum_rows_impl( - constant ggml_metal_kargs_sum_rows & args, - device const char * src0, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { -#define FC_OP FC_sum_rows_op - - const int i3 = tgpig.z; - const int i2 = tgpig.y; - const int i1 = tgpig.x; - - threadgroup T0 * shmem_t = (threadgroup T0 *) shmem; - - if (sgitg == 0) { - shmem_t[tiisg] = 0.0f; - } - - device const T0 * src_row = (device const T0 *) (src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03); - device T * dst_row = (device T *) (dst + i1*args.nb1 + i2*args.nb2 + i3*args.nb3); - - T0 sumf = T0(0.0f); - - for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) { - sumf += src_row[i0]; - } - - sumf = simd_sum(sumf); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - shmem_t[sgitg] = sumf; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - sumf = shmem_t[tiisg]; - sumf = simd_sum(sumf); - - if (tpitg.x == 0) { - if (FC_OP == OP_SUM_ROWS_NUM_MEAN) { - if (is_same::value) { - dst_row[0] = sum(sumf) / (4*args.ne00); - } else { - dst_row[0] = sum(sumf) / args.ne00; - } - } else { - dst_row[0] = sum(sumf); - } - } - -#undef FC_OP -} - -typedef decltype(kernel_sum_rows_impl) kernel_sum_rows_t; - -template [[host_name("kernel_sum_rows_f32_f32")]] kernel kernel_sum_rows_t kernel_sum_rows_impl; -template [[host_name("kernel_sum_rows_f32_f32_4")]] kernel kernel_sum_rows_t kernel_sum_rows_impl; - -template -kernel void kernel_cumsum_blk( - constant ggml_metal_kargs_cumsum_blk & args, - device const char * src0, - device char * tmp, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int ib = tgpig[0]/args.ne01; - - const int i00 = ib*ntg.x; - const int i01 = tgpig[0]%args.ne01; - const int i02 = tgpig[1]; - const int i03 = tgpig[2]; - - device const float * src0_row = (device const float *) (src0 + - args.nb01*i01 + - args.nb02*i02 + - args.nb03*i03); - - threadgroup float * shmem_f32 = (threadgroup float *) shmem; - - float v = 0.0f; - - if (i00 + tpitg.x < args.ne00) { - v = src0_row[i00 + tpitg.x]; - } - - float s = simd_prefix_inclusive_sum(v); - - if (tiisg == N_SIMDWIDTH - 1) { - shmem_f32[sgitg] = s; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (sgitg == 0) { - shmem_f32[tiisg] = simd_prefix_exclusive_sum(shmem_f32[tiisg]); - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - s += shmem_f32[sgitg]; - - device float * dst_row = (device float *) dst + - args.ne00*i01 + - args.ne00*args.ne01*i02 + - args.ne00*args.ne01*args.ne02*i03; - - if (i00 + tpitg.x < args.ne00) { - dst_row[i00 + tpitg.x] = s; - } - - if (args.outb && tpitg.x == ntg.x - 1) { - device float * tmp_row = (device float *) tmp + - args.net0*i01 + - args.net0*args.net1*i02 + - args.net0*args.net1*args.net2*i03; - - tmp_row[ib] = s; - } -} - -typedef decltype(kernel_cumsum_blk) kernel_cumsum_blk_t; - -template [[host_name("kernel_cumsum_blk_f32")]] kernel kernel_cumsum_blk_t kernel_cumsum_blk; - -template -kernel void kernel_cumsum_add( - constant ggml_metal_kargs_cumsum_add & args, - device const char * tmp, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int ib = tgpig[0]/args.ne01; - - if (ib == 0) { - return; - } - - const int i00 = ib*ntg.x; - const int i01 = tgpig[0]%args.ne01; - const int i02 = tgpig[1]; - const int i03 = tgpig[2]; - - device const float * tmp_row = (device const float *) (tmp + - args.nbt1*i01 + - args.nbt2*i02 + - args.nbt3*i03); - - device float * dst_row = (device float *) dst + - args.ne00*i01 + - args.ne00*args.ne01*i02 + - args.ne00*args.ne01*args.ne02*i03; - - if (i00 + tpitg.x < args.ne00) { - dst_row[i00 + tpitg.x] += tmp_row[ib - 1]; - } -} - -typedef decltype(kernel_cumsum_add) kernel_cumsum_add_t; - -template [[host_name("kernel_cumsum_add_f32")]] kernel kernel_cumsum_add_t kernel_cumsum_add; - - -template -bool _ggml_vec_tri_cmp(const int i, const int r); - -template<> -bool _ggml_vec_tri_cmp(const int i, const int r) { - return i < r; -} - -template<> -bool _ggml_vec_tri_cmp(const int i, const int r) { - return i <= r; -} - -template<> -bool _ggml_vec_tri_cmp(const int i, const int r) { - return i > r; -} - -template<> -bool _ggml_vec_tri_cmp(const int i, const int r) { - return i >= r; -} - -template -kernel void kernel_tri( - constant ggml_metal_kargs_tri & args, - device const char * src0, - device const char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int i3 = tgpig.z; - const int i2 = tgpig.y; - const int i1 = tgpig.x; - - if (i3 >= args.ne03 || i2 >= args.ne02 || i1 >= args.ne01) { - return; - } - - device const T * src_row = (device const T *) ((device const char *) src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03); - device T * dst_row = (device T *) ((device char *) dst + i1*args.nb1 + i2*args.nb2 + i3*args.nb3); - - // Each thread is a single element of the row if ne00 < max threads per - // threadgroup, so this will loop once for each index that this thread is - // responsible for - for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) { - // Use the comparison as a mask for branchless - dst_row[i0] = static_cast(_ggml_vec_tri_cmp(i0, i1)) * src_row[i0]; - } -} - -typedef decltype(kernel_tri) kernel_tri_t; - -template [[host_name("kernel_tri_f32_0")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_f32_1")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_f32_2")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_f32_3")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_f16_0")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_f16_1")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_f16_2")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_f16_3")]] kernel kernel_tri_t kernel_tri; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_tri_bf16_0")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_bf16_1")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_bf16_2")]] kernel kernel_tri_t kernel_tri; -template [[host_name("kernel_tri_bf16_3")]] kernel kernel_tri_t kernel_tri; -#endif - -template -kernel void kernel_soft_max( - constant ggml_metal_kargs_soft_max & args, - device const char * src0, - device const char * src1, - device const char * src2, - device char * dst, - threadgroup float * buf [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint sgitg[[simdgroup_index_in_threadgroup]], - uint tiisg[[thread_index_in_simdgroup]], - uint3 tptg[[threads_per_threadgroup]]) { - const int32_t i03 = tgpig.z; - const int32_t i02 = tgpig.y; - const int32_t i01 = tgpig.x; - - const int32_t i13 = i03%args.ne13; - const int32_t i12 = i02%args.ne12; - const int32_t i11 = i01; - - device const float * psrc0 = (device const float *) (src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); - device const T * pmask = src1 != src0 ? (device const T * ) (src1 + i11*args.nb11 + i12*args.nb12 + i13*args.nb13) : nullptr; - device const float * psrc2 = src2 != src0 ? (device const float *) (src2) : nullptr; - device float * pdst = (device float *) (dst + i01*args.nb1 + i02*args.nb2 + i03*args.nb3); - - float slope = 1.0f; - - // ALiBi - if (args.max_bias > 0.0f) { - const int32_t h = i02; - - const float base = h < args.n_head_log2 ? args.m0 : args.m1; - const int exp = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; - - slope = pow(base, exp); - } - - // parallel max - float lmax = psrc2 ? psrc2[i02] : -INFINITY; - - for (int i00 = tpitg.x; i00 < args.ne00; i00 += tptg.x) { - lmax = MAX(lmax, psrc0[i00]*args.scale + (pmask ? slope*pmask[i00] : 0.0f)); - } - - // find the max value in the block - float max_val = simd_max(lmax); - if (tptg.x > N_SIMDWIDTH) { - if (sgitg == 0) { - buf[tiisg] = -INFINITY; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - buf[sgitg] = max_val; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - max_val = buf[tiisg]; - max_val = simd_max(max_val); - } - - // parallel sum - float lsum = 0.0f; - for (int i00 = tpitg.x; i00 < args.ne00; i00 += tptg.x) { - const float exp_psrc0 = exp((psrc0[i00]*args.scale + (pmask ? slope*pmask[i00] : 0.0f)) - max_val); - lsum += exp_psrc0; - pdst[i00] = exp_psrc0; - } - - // This barrier fixes a failing test - // ref: https://github.com/ggml-org/ggml/pull/621#discussion_r1425156335 - threadgroup_barrier(mem_flags::mem_none); - - float sum = simd_sum(lsum); - - if (tptg.x > N_SIMDWIDTH) { - if (sgitg == 0) { - buf[tiisg] = 0.0f; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - buf[sgitg] = sum; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - sum = buf[tiisg]; - sum = simd_sum(sum); - } - - if (psrc2) { - sum += exp(psrc2[i02] - max_val); - } - - const float inv_sum = 1.0f/sum; - - for (int i00 = tpitg.x; i00 < args.ne00; i00 += tptg.x) { - pdst[i00] *= inv_sum; - } -} - -template -kernel void kernel_soft_max_4( - constant ggml_metal_kargs_soft_max & args, - device const char * src0, - device const char * src1, - device const char * src2, - device char * dst, - threadgroup float * buf [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint sgitg[[simdgroup_index_in_threadgroup]], - uint tiisg[[thread_index_in_simdgroup]], - uint3 tptg[[threads_per_threadgroup]]) { - const int32_t i03 = tgpig.z; - const int32_t i02 = tgpig.y; - const int32_t i01 = tgpig.x; - - const int32_t i13 = i03%args.ne13; - const int32_t i12 = i02%args.ne12; - const int32_t i11 = i01; - - device const float4 * psrc4 = (device const float4 *) (src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); - device const T * pmask = src1 != src0 ? (device const T * ) (src1 + i11*args.nb11 + i12*args.nb12 + i13*args.nb13) : nullptr; - device const float * psrc2 = src2 != src0 ? (device const float * ) (src2) : nullptr; - device float4 * pdst4 = (device float4 *) (dst + i01*args.nb1 + i02*args.nb2 + i03*args.nb3); - - float slope = 1.0f; - - if (args.max_bias > 0.0f) { - const int32_t h = i02; - - const float base = h < args.n_head_log2 ? args.m0 : args.m1; - const int exp = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; - - slope = pow(base, exp); - } - - // parallel max - float4 lmax4 = psrc2 ? psrc2[i02] : -INFINITY; - - for (int i00 = tpitg.x; i00 < args.ne00/4; i00 += tptg.x) { - lmax4 = fmax(lmax4, psrc4[i00]*args.scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))); - } - - const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3])); - - float max_val = simd_max(lmax); - if (tptg.x > N_SIMDWIDTH) { - if (sgitg == 0) { - buf[tiisg] = -INFINITY; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - buf[sgitg] = max_val; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - max_val = buf[tiisg]; - max_val = simd_max(max_val); - } - - // parallel sum - float4 lsum4 = 0.0f; - for (int i00 = tpitg.x; i00 < args.ne00/4; i00 += tptg.x) { - const float4 exp_psrc4 = exp((psrc4[i00]*args.scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))) - max_val); - lsum4 += exp_psrc4; - pdst4[i00] = exp_psrc4; - } - - const float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3]; - - // This barrier fixes a failing test - // ref: https://github.com/ggml-org/ggml/pull/621#discussion_r1425156335 - threadgroup_barrier(mem_flags::mem_none); - - float sum = simd_sum(lsum); - - if (tptg.x > N_SIMDWIDTH) { - if (sgitg == 0) { - buf[tiisg] = 0.0f; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - buf[sgitg] = sum; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - sum = buf[tiisg]; - sum = simd_sum(sum); - } - - if (psrc2) { - sum += exp(psrc2[i02] - max_val); - } - - const float inv_sum = 1.0f/sum; - - for (int i00 = tpitg.x; i00 < args.ne00/4; i00 += tptg.x) { - pdst4[i00] *= inv_sum; - } -} - -typedef decltype(kernel_soft_max) kernel_soft_max_t; -typedef decltype(kernel_soft_max_4) kernel_soft_max_4_t; - -template [[host_name("kernel_soft_max_f16")]] kernel kernel_soft_max_t kernel_soft_max; -template [[host_name("kernel_soft_max_f32")]] kernel kernel_soft_max_t kernel_soft_max; -template [[host_name("kernel_soft_max_f16_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4; -template [[host_name("kernel_soft_max_f32_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4; - -// ref: ggml.c:ggml_compute_forward_ssm_conv_f32 -kernel void kernel_ssm_conv_f32_f32( - constant ggml_metal_kargs_ssm_conv & args, - device const void * src0, - device const void * src1, - device float * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - const int64_t ir = tgpig.x; - const int64_t i2 = tgpig.y; - const int64_t i3 = tgpig.z; - - const int64_t nc = args.ne10; - //const int64_t ncs = args.ne00; - //const int64_t nr = args.ne01; - //const int64_t n_t = args.ne1; - //const int64_t n_s = args.ne2; - - device const float * s = (device const float *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02); - device const float * c = (device const float *) ((device const char *) src1 + ir*args.nb11); - device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); - - float sumf = 0.0f; - - for (int64_t i0 = 0; i0 < nc; ++i0) { - sumf += s[i0] * c[i0]; - } - - x[0] = sumf; -} - -kernel void kernel_ssm_conv_f32_f32_4( - constant ggml_metal_kargs_ssm_conv & args, - device const void * src0, - device const void * src1, - device float * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - const int64_t ir = tgpig.x; - const int64_t i2 = tgpig.y; - const int64_t i3 = tgpig.z; - - const int64_t nc = args.ne10; - //const int64_t ncs = args.ne00; - //const int64_t nr = args.ne01; - //const int64_t n_t = args.ne1; - //const int64_t n_s = args.ne2; - - device const float4 * s = (device const float4 *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02); - device const float4 * c = (device const float4 *) ((device const char *) src1 + ir*args.nb11); - device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); - - float sumf = 0.0f; - - for (int64_t i0 = 0; i0 < nc/4; ++i0) { - sumf += dot(s[i0], c[i0]); - } - - x[0] = sumf; -} - -constant short FC_ssm_conv_bs [[function_constant(FC_SSM_CONV + 0)]]; - -// Batched version: each threadgroup processes multiple tokens for better efficiency -// Thread layout: each thread handles one token, threadgroup covers BATCH_SIZE tokens -kernel void kernel_ssm_conv_f32_f32_batched( - constant ggml_metal_kargs_ssm_conv & args, - device const void * src0, - device const void * src1, - device float * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - // tgpig.x = row index (ir) - // tgpig.y = batch of tokens (i2_base / BATCH_SIZE) - // tgpig.z = sequence index (i3) - // tpitg.x = thread within batch (0..BATCH_SIZE-1) - const short BATCH_SIZE = FC_ssm_conv_bs; - - const int64_t ir = tgpig.x; - const int64_t i2_base = tgpig.y * BATCH_SIZE; - const int64_t i3 = tgpig.z; - const int64_t i2_off = tpitg.x; - const int64_t i2 = i2_base + i2_off; - - const int64_t nc = args.ne10; // conv kernel size (typically 4) - const int64_t n_t = args.ne1; // number of tokens - - // Bounds check for partial batches at the end - if (i2 >= n_t) { - return; - } - - // Load conv weights (shared across all tokens for this row) - device const float * c = (device const float *) ((device const char *) src1 + ir*args.nb11); - - // Load source for this specific token - device const float * s = (device const float *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02); - - // Output location for this token - device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); - - float sumf = 0.0f; - for (int64_t i0 = 0; i0 < nc; ++i0) { - sumf += s[i0] * c[i0]; - } - - x[0] = sumf; -} - -kernel void kernel_ssm_conv_f32_f32_batched_4( - constant ggml_metal_kargs_ssm_conv & args, - device const void * src0, - device const void * src1, - device float * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - // tgpig.x = row index (ir) - // tgpig.y = batch of tokens (i2_base / BATCH_SIZE) - // tgpig.z = sequence index (i3) - // tpitg.x = thread within batch (0..BATCH_SIZE-1) - const short BATCH_SIZE = FC_ssm_conv_bs; - - const int64_t ir = tgpig.x; - const int64_t i2_base = tgpig.y * BATCH_SIZE; - const int64_t i3 = tgpig.z; - const int64_t i2_off = tpitg.x; - const int64_t i2 = i2_base + i2_off; - - const int64_t nc = args.ne10; // conv kernel size (typically 4) - const int64_t n_t = args.ne1; // number of tokens - - // Bounds check for partial batches at the end - if (i2 >= n_t) { - return; - } - - // Load conv weights (shared across all tokens for this row) - device const float4 * c = (device const float4 *) ((device const char *) src1 + ir*args.nb11); - - // Load source for this specific token - device const float4 * s = (device const float4 *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02); - - // Output location for this token - device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); - - float sumf = 0.0f; - for (int64_t i0 = 0; i0 < nc/4; ++i0) { - sumf += dot(s[i0], c[i0]); - } - - x[0] = sumf; -} - -// ref: ggml.c:ggml_compute_forward_ssm_scan_f32, Mamba-2 part -// Optimized version: reduces redundant memory loads by having one thread load shared values -kernel void kernel_ssm_scan_f32( - constant ggml_metal_kargs_ssm_scan & args, - device const void * src0, - device const void * src1, - device const void * src2, - device const void * src3, - device const void * src4, - device const void * src5, - device const void * src6, - device float * dst, - threadgroup float * shared [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgptg[[simdgroups_per_threadgroup]], - uint3 tgpg[[threadgroups_per_grid]]) { - constexpr short NW = N_SIMDWIDTH; - - // Shared memory layout: - // [0..sgptg*NW-1]: partial sums for reduction (existing) - // [sgptg*NW..sgptg*NW+sgptg-1]: pre-computed x_dt values for each token in batch - // [sgptg*NW+sgptg..sgptg*NW+2*sgptg-1]: pre-computed dA values for each token in batch - threadgroup float * shared_sums = shared; - threadgroup float * shared_x_dt = shared + sgptg * NW; - threadgroup float * shared_dA = shared + sgptg * NW + sgptg; - - shared_sums[tpitg.x] = 0.0f; - - const int32_t i0 = tpitg.x; - const int32_t i1 = tgpig.x; - const int32_t ir = tgpig.y; // current head - const int32_t i3 = tgpig.z; // current seq - - const int32_t nc = args.d_state; - const int32_t nr = args.d_inner; - const int32_t nh = args.n_head; - const int32_t ng = args.n_group; - const int32_t n_t = args.n_seq_tokens; - - const int32_t s_off = args.s_off; - - device const int32_t * ids = (device const int32_t *) src6; - - device const float * s0_buff = (device const float *) ((device const char *) src0 + ir*args.nb02 + ids[i3]*args.nb03); - device float * s_buff = (device float *) ((device char *) dst + ir*args.nb02 + i3*args.nb03 + s_off); - - const int32_t i = i0 + i1*nc; - const int32_t g = ir / (nh / ng); // repeat_interleave - - float s0 = s0_buff[i]; - float s = 0.0f; - - device const float * A = (device const float *) ((device const char *) src3 + ir*args.nb31); // {ne30, nh} - - const float A0 = A[i0%args.ne30]; - - device const float * x = (device const float *)((device const char *) src1 + i1*args.nb10 + ir*args.nb11 + i3*args.nb13); // {dim, nh, nt, ns} - device const float * dt = (device const float *)((device const char *) src2 + ir*args.nb20 + i3*args.nb22); // {nh, nt, ns} - device const float * B = (device const float *)((device const char *) src4 + g*args.nb41 + i3*args.nb43); // {d_state, ng, nt, ns} - device const float * C = (device const float *)((device const char *) src5 + g*args.nb51 + i3*args.nb53); // {d_state, ng, nt, ns} - - device float * y = dst + (i1 + ir*(nr) + i3*(n_t*nh*nr)); // {dim, nh, nt, ns} - - for (int i2 = 0; i2 < n_t; i2 += sgptg) { - threadgroup_barrier(mem_flags::mem_threadgroup); - - // Pre-compute x_dt and dA for this batch of tokens - // Only first sgptg threads do the loads and expensive math - if (i0 < sgptg && i2 + i0 < n_t) { - // ns12 and ns21 are element strides (nb12/nb10, nb21/nb20) - device const float * x_t = x + i0 * args.ns12; - device const float * dt_t = dt + i0 * args.ns21; - - const float dt0 = dt_t[0]; - const float dtsp = dt0 <= 20.0f ? log(1.0f + exp(dt0)) : dt0; - shared_x_dt[i0] = x_t[0] * dtsp; - shared_dA[i0] = dtsp; // Store dtsp, compute exp(dtsp * A0) per-thread since A0 varies - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - for (int t = 0; t < sgptg && i2 + t < n_t; t++) { - const float x_dt = shared_x_dt[t]; - const float dA = exp(shared_dA[t] * A0); - - s = (s0 * dA) + (B[i0] * x_dt); - - const float sumf = simd_sum(s * C[i0]); - - if (tiisg == 0) { - shared_sums[t*NW + sgitg] = sumf; - } - - // recurse - s0 = s; - - B += args.ns42; - C += args.ns52; - } - - // Advance pointers for next batch - x += sgptg * args.ns12; - dt += sgptg * args.ns21; - - threadgroup_barrier(mem_flags::mem_threadgroup); - - const float sumf = simd_sum(shared_sums[sgitg*NW + tiisg]); - - if (tiisg == 0 && i2 + sgitg < n_t) { - y[sgitg*nh*nr] = sumf; - } - - y += sgptg*nh*nr; - } - - s_buff[i] = s; -} - -kernel void kernel_rwkv_wkv6_f32( - device const float * k, - device const float * v, - device const float * r, - device const float * tf, - device const float * td, - device const float * state_in, - device float * dst, - constant uint & B, - constant uint & T, - constant uint & C, - constant uint & H, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const uint head_size = 64; // TODO: support head_size = 128 - const uint batch_id = tgpig.x / H; - const uint head_id = tgpig.x % H; - const uint tid = tpitg.x; - - if (batch_id >= B || head_id >= H) { - return; - } - - const uint state_size = C * head_size; - const uint n_seq_tokens = T / B; - - threadgroup float _k[head_size]; - threadgroup float _r[head_size]; - threadgroup float _tf[head_size]; - threadgroup float _td[head_size]; - - float state[head_size]; - - for (uint i = 0; i < head_size; i++) { - state[i] = state_in[batch_id * state_size + head_id * head_size * head_size - + i * head_size + tid]; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - _tf[tid] = tf[head_id * head_size + tid]; - threadgroup_barrier(mem_flags::mem_threadgroup); - - const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; - const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; - - for (uint t = start_t; t < end_t; t += C) { - threadgroup_barrier(mem_flags::mem_threadgroup); - _k[tid] = k[t]; - _r[tid] = r[t]; - _td[tid] = td[t]; - threadgroup_barrier(mem_flags::mem_threadgroup); - - const float v_val = v[t]; - float y = 0.0; - - for (uint j = 0; j < head_size; j += 4) { - float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); - float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); - float4 tf_vec = float4(_tf[j], _tf[j+1], _tf[j+2], _tf[j+3]); - float4 td_vec = float4(_td[j], _td[j+1], _td[j+2], _td[j+3]); - float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); - - float4 kv = k_vec * v_val; - - float4 temp = tf_vec * kv + s_vec; - y += dot(r_vec, temp); - - s_vec = s_vec * td_vec + kv; - state[j] = s_vec[0]; - state[j+1] = s_vec[1]; - state[j+2] = s_vec[2]; - state[j+3] = s_vec[3]; - } - - dst[t] = y; - } - - for (uint i = 0; i < head_size; i++) { - dst[T * C + batch_id * state_size + head_id * head_size * head_size - + i * head_size + tid] = state[i]; - } -} - -kernel void kernel_rwkv_wkv7_f32( - device const float * r, - device const float * w, - device const float * k, - device const float * v, - device const float * a, - device const float * b, - device const float * state_in, - device float * dst, - constant uint & B, - constant uint & T, - constant uint & C, - constant uint & H, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const uint head_size = 64; // TODO: support head_size = 128 - const uint batch_id = tgpig.x / H; - const uint head_id = tgpig.x % H; - const uint tid = tpitg.x; - - if (batch_id >= B || head_id >= H) { - return; - } - - const uint state_size = C * head_size; - const uint n_seq_tokens = T / B; - - threadgroup float _r[head_size]; - threadgroup float _w[head_size]; - threadgroup float _k[head_size]; - threadgroup float _a[head_size]; - threadgroup float _b[head_size]; - - float state[head_size]; - - for (uint i = 0; i < head_size; i++) { - state[i] = state_in[batch_id * state_size + head_id * head_size * head_size - + tid * head_size + i]; - } - - const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; - const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; - - for (uint t = start_t; t < end_t; t += C) { - threadgroup_barrier(mem_flags::mem_threadgroup); - _r[tid] = r[t]; - _w[tid] = w[t]; - _k[tid] = k[t]; - _a[tid] = a[t]; - _b[tid] = b[t]; - threadgroup_barrier(mem_flags::mem_threadgroup); - - const float v_val = v[t]; - float y = 0.0, sa = 0.0; - - float4 sa_vec(0.0); - - for (uint j = 0; j < head_size; j += 4) { - float4 a_vec = float4(_a[j], _a[j+1], _a[j+2], _a[j+3]); - float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); - sa_vec += a_vec * s_vec; - } - sa = sa_vec[0] + sa_vec[1] + sa_vec[2] + sa_vec[3]; - - for (uint j = 0; j < head_size; j += 4) { - float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); - float4 w_vec = float4(_w[j], _w[j+1], _w[j+2], _w[j+3]); - float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); - float4 b_vec = float4(_b[j], _b[j+1], _b[j+2], _b[j+3]); - float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); - - float4 kv = k_vec * v_val; - - s_vec = s_vec * w_vec + kv + sa * b_vec; - y += dot(s_vec, r_vec); - - state[j] = s_vec[0]; - state[j+1] = s_vec[1]; - state[j+2] = s_vec[2]; - state[j+3] = s_vec[3]; - } - - dst[t] = y; - } - - for (uint i = 0; i < head_size; i++) { - dst[T * C + batch_id * state_size + head_id * head_size * head_size - + tid * head_size + i] = state[i]; - } -} - -constant short FC_gated_delta_net_ne20 [[function_constant(FC_GATED_DELTA_NET + 0)]]; -constant short FC_gated_delta_net_ne30 [[function_constant(FC_GATED_DELTA_NET + 1)]]; - -#if 1 -template -kernel void kernel_gated_delta_net_impl( - constant ggml_metal_kargs_gated_delta_net & args, - device const char * q, - device const char * k, - device const char * v, - device const char * g, - device const char * b, - device const char * s, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { -#define S_v FC_gated_delta_net_ne20 -#define G FC_gated_delta_net_ne30 - - const uint tx = tpitg.x; - const uint ty = tpitg.y; - - const uint i23 = tgpig.z; // B - const uint i21 = tgpig.y; // H - const uint i20 = tgpig.x*NSG + ty; - - const uint i01 = i21 % args.ne01; - const uint i11 = i21 % args.ne11; - - const float scale = 1.0f / sqrt((float)S_v); - - // state is stored transposed: M[i20][is] = S[is][i20], so row i20 is contiguous - device const float * s_ptr = (device const float *) (s) + (i23*args.ne21 + i21)*S_v*S_v + i20*S_v; - - float ls[NSG]; - - FOR_UNROLL (short j = 0; j < NSG; j++) { - const short is = tx*NSG + j; - ls[j] = s_ptr[is]; - } - - device float * dst_attn = (device float *) (dst) + (i23*args.ne22*args.ne21 + i21)*S_v + i20; - - device const float * q_ptr = (device const float *) (q + i23*args.nb03 + i01*args.nb01); - device const float * k_ptr = (device const float *) (k + i23*args.nb13 + i11*args.nb11); - device const float * v_ptr = (device const float *) (v + i23*args.nb23 + i21*args.nb21); - - device const float * b_ptr = (device const float *) (b) + (i23*args.ne22*args.ne21 + i21); - device const float * g_ptr = (device const float *) (g) + (i23*args.ne22*args.ne21 + i21)*G; - - for (short t = 0; t < args.ne22; t++) { - float s_k = 0.0f; - - if (G == 1) { - const float g_exp = exp(g_ptr[0]); - - FOR_UNROLL (short j = 0; j < NSG; j++) { - const short is = tx*NSG + j; - ls[j] *= g_exp; - - s_k += ls[j]*k_ptr[is]; - } - } else { - // KDA - FOR_UNROLL (short j = 0; j < NSG; j++) { - const short is = tx*NSG + j; - ls[j] *= exp(g_ptr[is]); - - s_k += ls[j]*k_ptr[is]; - } - } - - s_k = simd_sum(s_k); - - const float d = (v_ptr[i20] - s_k)*b_ptr[0]; - - float y = 0.0f; - - FOR_UNROLL (short j = 0; j < NSG; j++) { - const short is = tx*NSG + j; - ls[j] += k_ptr[is]*d; - - y += ls[j]*q_ptr[is]; - } - - y = simd_sum(y); - - if (tx == 0) { - dst_attn[t*args.ne21*S_v] = y*scale; - } - - q_ptr += args.ns02; - k_ptr += args.ns12; - v_ptr += args.ns22; - - b_ptr += args.ne21; - g_ptr += args.ne21*G; - } - - device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20*S_v; - - FOR_UNROLL (short j = 0; j < NSG; j++) { - const short is = tx*NSG + j; - dst_state[is] = ls[j]; - } - -#undef S_v -#undef G -} - -typedef decltype(kernel_gated_delta_net_impl<4>) kernel_gated_delta_net_t; - -template [[host_name("kernel_gated_delta_net_f32_1")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl<1>; -template [[host_name("kernel_gated_delta_net_f32_2")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl<2>; -template [[host_name("kernel_gated_delta_net_f32_4")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl<4>; - -#else -// a simplified version of the above -// no performance improvement, so keep the above version for now - -template -kernel void kernel_gated_delta_net_impl( - constant ggml_metal_kargs_gated_delta_net & args, - device const char * q, - device const char * k, - device const char * v, - device const char * g, - device const char * b, - device const char * s, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { -#define S_v FC_gated_delta_net_ne20 -#define G FC_gated_delta_net_ne30 - - const uint tx = tpitg.x; - const uint ty = tpitg.y; - - const uint i23 = tgpig.z; // B - const uint i21 = tgpig.y; // H - const uint i20 = tgpig.x*NSG + ty; - - const uint i01 = i21 % args.ne01; - const uint i11 = i21 % args.ne11; - - const float scale = 1.0f / sqrt((float)S_v); - - device const float * s_ptr = (device const float *) (s) + (i23*args.ne21 + i21)*S_v*S_v + i20; - - float lsf[NSG]; - - FOR_UNROLL (short j = 0; j < NSG; j++) { - const short is = tx*NSG + j; - lsf[j] = s_ptr[is*S_v]; - } - - thread T * ls = (thread T *) (lsf); - - device float * dst_attn = (device float *) (dst) + (i23*args.ne22*args.ne21 + i21)*S_v + i20; - - device const float * q_ptr = (device const float *) (q + i23*args.nb03 + i01*args.nb01); - device const float * k_ptr = (device const float *) (k + i23*args.nb13 + i11*args.nb11); - device const float * v_ptr = (device const float *) (v + i23*args.nb23 + i21*args.nb21); - - device const float * b_ptr = (device const float *) (b) + (i23*args.ne22*args.ne21 + i21); - device const float * g_ptr = (device const float *) (g) + (i23*args.ne22*args.ne21 + i21)*G; - - for (short t = 0; t < args.ne22; t++) { - device const T * qt_ptr = (device const T *) (q_ptr); - device const T * kt_ptr = (device const T *) (k_ptr); - device const T * gt_ptr = (device const T *) (g_ptr); - - if (G == 1) { - *ls *= exp(g_ptr[0]); - } else { - // KDA - *ls *= exp(gt_ptr[tx]); - } - - const float s_k = simd_sum(dot(*ls, kt_ptr[tx])); - - const float d = (v_ptr[i20] - s_k)*b_ptr[0]; - - *ls += kt_ptr[tx]*d; - - const float y = simd_sum(dot(*ls, qt_ptr[tx])); - - if (tx == 0) { - *dst_attn = y*scale; - } - - q_ptr += args.ns02; - k_ptr += args.ns12; - v_ptr += args.ns22; - - b_ptr += args.ne21; - g_ptr += args.ne21*G; - - dst_attn += args.ne21*S_v; - } - - device float * dst_state = (device float *) (dst) + args.ne23*args.ne22*args.ne21*S_v + (i23*args.ne21 + i21)*S_v*S_v + i20; - device T * dstt_state = (device T *) (dst_state); - - FOR_UNROLL (short j = 0; j < NSG; j++) { - const short is = tx*NSG + j; - dst_state[is*S_v] = lsf[j]; - } - -#undef S_v -#undef G -} - -typedef decltype(kernel_gated_delta_net_impl) kernel_gated_delta_net_t; - -template [[host_name("kernel_gated_delta_net_f32_1")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl; -template [[host_name("kernel_gated_delta_net_f32_2")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl; -template [[host_name("kernel_gated_delta_net_f32_4")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl; -#endif - -constant short FC_solve_tri_nsg [[function_constant(FC_SOLVE_TRI + 0)]]; -constant short FC_solve_tri_n [[function_constant(FC_SOLVE_TRI + 1)]]; -constant short FC_solve_tri_k [[function_constant(FC_SOLVE_TRI + 2)]]; - -kernel void kernel_solve_tri_f32( - constant ggml_metal_kargs_solve_tri & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - ushort3 tgpig[[threadgroup_position_in_grid]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - constexpr short NW = N_SIMDWIDTH; - - const short NSG = FC_solve_tri_nsg; - const short N = FC_solve_tri_n; - const short K = FC_solve_tri_k; - const short NP = PAD2(N, NW); - - const int32_t i03 = tgpig.z; - const int32_t i02 = tgpig.y; - const int32_t i01 = tgpig.x*NSG + sgitg; - - threadgroup float * sh0 = (threadgroup float *) shmem; - - device const float * src0_ptr = (device const float *)(src0 + i02 * args.nb02 + i03 * args.nb03) + sgitg*N; - device const float * src1_ptr = (device const float *)(src1 + i02 * args.nb12 + i03 * args.nb13) + i01; - device float * dst_ptr = (device float *)(dst + i02 * args.nb2 + i03 * args.nb3) + i01; - - for (short rr = 0; rr < N; rr += NSG) { - threadgroup_barrier(mem_flags::mem_threadgroup); - - { - threadgroup float * sh0_cur = sh0 + sgitg*NP; - - for (short t = 0; t*NW < N; ++t) { - const short idx = t*NW + tiisg; - sh0_cur[idx] = src0_ptr[idx]; - } - - src0_ptr += NSG*N; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (i01 >= args.ne10) { - continue; - } - - for (short ir = 0; ir < NSG && rr + ir < N; ++ir) { - const short r = rr + ir; - - threadgroup float * sh0_cur = sh0 + ir*NP; - - float sum = 0.0f; - - for (short t = 0; t*NW < r; ++t) { - const short idx = t*NW + tiisg; - sum += sh0_cur[idx] * dst_ptr[idx*K] * (idx < r); - } - - sum = simd_sum(sum); - - if (tiisg == 0) { - const float diag = sh0_cur[r]; - - dst_ptr[r*K] = (src1_ptr[r*K] - sum) / diag; - } - } - } -} - -kernel void kernel_argmax_f32( - constant ggml_metal_kargs_argmax & args, - device const char * src0, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint sgitg[[simdgroup_index_in_threadgroup]], - uint tiisg[[thread_index_in_simdgroup]], - uint ntg[[threads_per_threadgroup]]) { - device const float * x_row = (device const float *) ((device const char *) src0 + tgpig * args.nb01); - - float lmax = -INFINITY; - int32_t larg = -1; - - for (int i00 = tpitg; i00 < args.ne00; i00 += ntg) { - if (x_row[i00] > lmax) { - lmax = x_row[i00]; - larg = i00; - } - } - - // find the argmax value in the block - float max_val = simd_max(lmax); - int32_t arg_val = simd_max(select(-1, larg, lmax == max_val)); - - device int32_t * dst_i32 = (device int32_t *) dst; - - threadgroup float * shared_maxval = (threadgroup float *) shmem; - threadgroup int32_t * shared_argmax = (threadgroup int32_t *) shmem + N_SIMDWIDTH; - - if (ntg > N_SIMDWIDTH) { - if (sgitg == 0) { - shared_maxval[tiisg] = -INFINITY; - shared_argmax[tiisg] = -1; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - shared_maxval[sgitg] = max_val; - shared_argmax[sgitg] = arg_val; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - max_val = shared_maxval[tiisg]; - arg_val = shared_argmax[tiisg]; - - float max_val_reduced = simd_max(max_val); - int32_t arg_val_reduced = simd_max(select(-1, arg_val, max_val == max_val_reduced)); - - dst_i32[tgpig] = arg_val_reduced; - - return; - } - - dst_i32[tgpig] = arg_val; -} - -// F == 1 : norm (no fuse) -// F == 2 : norm + mul -// F == 3 : norm + mul + add -template -kernel void kernel_norm_fuse_impl( - constant ggml_metal_kargs_norm & args, - device const char * src0, - device const char * src1_0, - device const char * src1_1, - device char * dst, - threadgroup float * shmem_f32 [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - if (sgitg == 0) { - shmem_f32[tiisg] = 0.0f; - } - - const int i01 = tgpig.x; - const int i02 = tgpig.y; - const int i03 = tgpig.z; - - device const T * x = (device const T *) (src0 + i03*args.nbf3[0] + i02*args.nbf2[0] + i01*args.nbf1[0]); - - device const T * f0 = (device const T *) (src1_0 + (i03%args.nef3[1])*args.nbf3[1] + (i02%args.nef2[1])*args.nbf2[1] + (i01%args.nef1[1])*args.nbf1[1]); - device const T * f1 = (device const T *) (src1_1 + (i03%args.nef3[2])*args.nbf3[2] + (i02%args.nef2[2])*args.nbf2[2] + (i01%args.nef1[2])*args.nbf1[2]); - - T sumft(0.0f); - - float sumf = 0.0f; - - for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { - sumft += x[i00]; - } - sumf = dot(sumft, T(1.0f)); - sumf = simd_sum(sumf); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - shmem_f32[sgitg] = sumf; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - sumf = shmem_f32[tiisg]; - sumf = simd_sum(sumf); - - const float mean = sumf/args.ne00; - - device T * y = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1); - - sumf = 0.0f; - for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { - y[i00] = x[i00] - mean; - sumf += dot(y[i00], y[i00]); - } - sumf = simd_sum(sumf); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - shmem_f32[sgitg] = sumf; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - sumf = shmem_f32[tiisg]; - sumf = simd_sum(sumf); - - const float variance = sumf/args.ne00; - - const float scale = 1.0f/sqrt(variance + args.eps); - for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { - if (F == 1) { - y[i00] = (y[i00]*scale); - } - if (F == 2) { - y[i00] = (y[i00]*scale)*f0[i00]; - } - if (F == 3) { - y[i00] = (y[i00]*scale)*f0[i00] + f1[i00]; - } - } -} - -typedef decltype(kernel_norm_fuse_impl) kernel_norm_fuse_t; - -template [[host_name("kernel_norm_f32")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; -template [[host_name("kernel_norm_mul_f32")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; -template [[host_name("kernel_norm_mul_add_f32")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; - -template [[host_name("kernel_norm_f32_4")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; -template [[host_name("kernel_norm_mul_f32_4")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; -template [[host_name("kernel_norm_mul_add_f32_4")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; - -// F == 1 : rms_norm (no fuse) -// F == 2 : rms_norm + mul -// F == 3 : rms_norm + mul + add -template -kernel void kernel_rms_norm_fuse_impl( - constant ggml_metal_kargs_norm & args, - device const char * src0, - device const char * src1_0, - device const char * src1_1, - device char * dst, - threadgroup float * shmem_f32 [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - if (sgitg == 0) { - shmem_f32[tiisg] = 0.0f; - } - - const int i01 = tgpig.x; - const int i02 = tgpig.y; - const int i03 = tgpig.z; - - device const T * x = (device const T *) (src0 + i03*args.nbf3[0] + i02*args.nbf2[0] + i01*args.nbf1[0]); - - device const T * f0 = (device const T *) (src1_0 + (i03%args.nef3[1])*args.nbf3[1] + (i02%args.nef2[1])*args.nbf2[1] + (i01%args.nef1[1])*args.nbf1[1]); - device const T * f1 = (device const T *) (src1_1 + (i03%args.nef3[2])*args.nbf3[2] + (i02%args.nef2[2])*args.nbf2[2] + (i01%args.nef1[2])*args.nbf1[2]); - - float sumf = 0.0f; - - // parallel sum - for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { - sumf += dot(x[i00], x[i00]); - } - sumf = simd_sum(sumf); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - shmem_f32[sgitg] = sumf; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - sumf = shmem_f32[tiisg]; - sumf = simd_sum(sumf); - - const float mean = sumf/args.ne00; - const float scale = 1.0f/sqrt(mean + args.eps); - - device T * y = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1); - for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { - if (F == 1) { - y[i00] = (x[i00]*scale); - } - if (F == 2) { - y[i00] = (x[i00]*scale)*f0[i00]; - } - if (F == 3) { - y[i00] = (x[i00]*scale)*f0[i00] + f1[i00]; - } - } -} - -typedef decltype(kernel_rms_norm_fuse_impl) kernel_rms_norm_fuse_t; - -template [[host_name("kernel_rms_norm_f32")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; -template [[host_name("kernel_rms_norm_mul_f32")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; -template [[host_name("kernel_rms_norm_mul_add_f32")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; - -template [[host_name("kernel_rms_norm_f32_4")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; -template [[host_name("kernel_rms_norm_mul_f32_4")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; -template [[host_name("kernel_rms_norm_mul_add_f32_4")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; - -template -kernel void kernel_l2_norm_impl( - constant ggml_metal_kargs_l2_norm & args, - device const char * src0, - device char * dst, - threadgroup float * shmem_f32 [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int i03 = tgpig.z; - const int i02 = tgpig.y; - const int i01 = tgpig.x; - - if (sgitg == 0) { - shmem_f32[tiisg] = 0.0f; - } - - device const T0 * x = (device const T0 *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); - device T * y = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1); - - float sumf = 0.0f; - - // parallel sum - for (int i00 = tpitg.x; i00 < args.ne00; i00 += ntg.x) { - sumf += dot(x[i00], x[i00]); - } - sumf = simd_sum(sumf); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - shmem_f32[sgitg] = sumf; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - sumf = shmem_f32[tiisg]; - sumf = simd_sum(sumf); - - const float scale = 1.0f/max(sqrt(sumf), args.eps); - - for (int i00 = tpitg.x; i00 < args.ne00; i00 += ntg.x) { - y[i00] = x[i00] * scale; - } -} - -typedef decltype(kernel_l2_norm_impl) kernel_l2_norm_t; - -template [[host_name("kernel_l2_norm_f32_f32")]] kernel kernel_l2_norm_t kernel_l2_norm_impl; -template [[host_name("kernel_l2_norm_f32_f32_4")]] kernel kernel_l2_norm_t kernel_l2_norm_impl; - -kernel void kernel_group_norm_f32( - constant ggml_metal_kargs_group_norm & args, - device const float * src0, - device float * dst, - threadgroup float * buf [[threadgroup(0)]], - uint tgpig[[threadgroup_position_in_grid]], - uint tpitg[[thread_position_in_threadgroup]], - uint sgitg[[simdgroup_index_in_threadgroup]], - uint tiisg[[thread_index_in_simdgroup]], - uint ntg[[threads_per_threadgroup]]) { - const int64_t ne = args.ne00*args.ne01*args.ne02; - const int64_t gs = args.ne00*args.ne01*((args.ne02 + args.ngrp - 1) / args.ngrp); - - int start = tgpig * gs; - int end = start + gs; - - start += tpitg; - - if (end >= ne) { - end = ne; - } - - float tmp = 0.0f; // partial sum for thread in warp - - for (int j = start; j < end; j += ntg) { - tmp += src0[j]; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - tmp = simd_sum(tmp); - if (ntg > N_SIMDWIDTH) { - if (sgitg == 0) { - buf[tiisg] = 0.0f; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - buf[sgitg] = tmp; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - tmp = buf[tiisg]; - tmp = simd_sum(tmp); - } - - const float mean = tmp / gs; - tmp = 0.0f; - - for (int j = start; j < end; j += ntg) { - float xi = src0[j] - mean; - dst[j] = xi; - tmp += xi * xi; - } - - tmp = simd_sum(tmp); - if (ntg > N_SIMDWIDTH) { - if (sgitg == 0) { - buf[tiisg] = 0.0f; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tiisg == 0) { - buf[sgitg] = tmp; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - tmp = buf[tiisg]; - tmp = simd_sum(tmp); - } - - const float variance = tmp / gs; - const float scale = 1.0f/sqrt(variance + args.eps); - for (int j = start; j < end; j += ntg) { - dst[j] *= scale; - } -} - -// Q1_0 dot product: dot = d * (2 * Σ(yl[i] where bit=1) - sumy) -inline float block_q_n_dot_y(device const block_q1_0 * qb_curr, float sumy, thread float * yl, int il) { - device const uint8_t * qs = qb_curr->qs + il / 8; - const uint8_t b0 = qs[0]; - const uint8_t b1 = qs[1]; - - float acc = 0.0f; - - acc += select(0.0f, yl[ 0], bool(b0 & 0x01)); - acc += select(0.0f, yl[ 1], bool(b0 & 0x02)); - acc += select(0.0f, yl[ 2], bool(b0 & 0x04)); - acc += select(0.0f, yl[ 3], bool(b0 & 0x08)); - acc += select(0.0f, yl[ 4], bool(b0 & 0x10)); - acc += select(0.0f, yl[ 5], bool(b0 & 0x20)); - acc += select(0.0f, yl[ 6], bool(b0 & 0x40)); - acc += select(0.0f, yl[ 7], bool(b0 & 0x80)); - - acc += select(0.0f, yl[ 8], bool(b1 & 0x01)); - acc += select(0.0f, yl[ 9], bool(b1 & 0x02)); - acc += select(0.0f, yl[10], bool(b1 & 0x04)); - acc += select(0.0f, yl[11], bool(b1 & 0x08)); - acc += select(0.0f, yl[12], bool(b1 & 0x10)); - acc += select(0.0f, yl[13], bool(b1 & 0x20)); - acc += select(0.0f, yl[14], bool(b1 & 0x40)); - acc += select(0.0f, yl[15], bool(b1 & 0x80)); - - return qb_curr->d * (2.0f * acc - sumy); -} - -// function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i]) -// il indicates where the q4 quants begin (0 or QK4_0/4) -// we assume that the yl's have been multiplied with the appropriate scale factor -// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096) -inline float block_q_n_dot_y(device const block_q4_0 * qb_curr, float sumy, thread float * yl, int il) { - float d = qb_curr->d; - - float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; - - device const uint16_t * qs = ((device const uint16_t *) qb_curr + 1 + il/2); - - for (int i = 0; i < 8; i += 2) { - acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F); - acc[1] += yl[i + 1] * (qs[i / 2] & 0x0F00); - acc[2] += yl[i + 8] * (qs[i / 2] & 0x00F0); - acc[3] += yl[i + 9] * (qs[i / 2] & 0xF000); - } - - return d * (sumy * -8.f + acc[0] + acc[1] + acc[2] + acc[3]); -} - -// function for calculate inner product between half a q4_1 block and 16 floats (yl), sumy is SUM(yl[i]) -// il indicates where the q4 quants begin (0 or QK4_0/4) -// we assume that the yl's have been multiplied with the appropriate scale factor -// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096) -inline float block_q_n_dot_y(device const block_q4_1 * qb_curr, float sumy, thread float * yl, int il) { - float d = qb_curr->d; - float m = qb_curr->m; - - float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; - - device const uint16_t * qs = ((device const uint16_t *) qb_curr + 2 + il/2); - - for (int i = 0; i < 8; i+=2) { - acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F); - acc[1] += yl[i + 1] * (qs[i / 2] & 0x0F00); - acc[2] += yl[i + 8] * (qs[i / 2] & 0x00F0); - acc[3] += yl[i + 9] * (qs[i / 2] & 0xF000); - } - - return d * (acc[0] + acc[1] + acc[2] + acc[3]) + sumy * m; -} - -// function for calculate inner product between half a q5_0 block and 16 floats (yl), sumy is SUM(yl[i]) -// il indicates where the q5 quants begin (0 or QK5_0/4) -// we assume that the yl's have been multiplied with the appropriate scale factor -// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096) -inline float block_q_n_dot_y(device const block_q5_0 * qb_curr, float sumy, thread float * yl, int il) { - float d = qb_curr->d; - - float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; - - device const uint16_t * qs = ((device const uint16_t *)qb_curr + 3 + il/2); - const uint32_t qh = *((device const uint32_t *)qb_curr->qh); - - for (int i = 0; i < 8; i+=2) { - acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010)); - acc[1] += yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000)); - acc[2] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100)); - acc[3] += yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000)); - } - - return d * (sumy * -16.f + acc[0] + acc[1] + acc[2] + acc[3]); -} - -// function for calculate inner product between half a q5_1 block and 16 floats (yl), sumy is SUM(yl[i]) -// il indicates where the q5 quants begin (0 or QK5_1/4) -// we assume that the yl's have been multiplied with the appropriate scale factor -// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096) -inline float block_q_n_dot_y(device const block_q5_1 * qb_curr, float sumy, thread float * yl, int il) { - float d = qb_curr->d; - float m = qb_curr->m; - - float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; - - device const uint16_t * qs = ((device const uint16_t *)qb_curr + 4 + il/2); - const uint32_t qh = *((device const uint32_t *)qb_curr->qh); - - for (int i = 0; i < 8; i+=2) { - acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010)); - acc[1] += yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000)); - acc[2] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100)); - acc[3] += yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000)); - } - - return d * (acc[0] + acc[1] + acc[2] + acc[3]) + sumy * m; -} - -template -static inline void helper_mv_reduce_and_write( - device float * dst_f32, - float sumf[NR0], - const int r0, - const int ne01, - ushort tiisg, - ushort sgitg, - threadgroup char * shmem) { - constexpr short NW = N_SIMDWIDTH; - - threadgroup float * shmem_f32[NR0]; - - for (short row = 0; row < NR0; ++row) { - shmem_f32[row] = (threadgroup float *) shmem + NW*row; - - if (sgitg == 0) { - shmem_f32[row][tiisg] = 0.0f; - } - - sumf[row] = simd_sum(sumf[row]); - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - for (short row = 0; row < NR0; ++row) { - if (tiisg == 0) { - shmem_f32[row][sgitg] = sumf[row]; - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - for (short row = 0; row < NR0 && r0 + row < ne01; ++row) { - float tot = simd_sum(shmem_f32[row][tiisg]); - - if (tiisg == 0 && sgitg == 0) { - dst_f32[r0 + row] = tot; - } - } -} - -constant short FC_mul_mv_nsg [[function_constant(FC_MUL_MV + 0)]]; -constant short FC_mul_mv_nxpsg [[function_constant(FC_MUL_MV + 1)]]; - -template -void mul_vec_q_n_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - constexpr short NW = N_SIMDWIDTH; - constexpr short NQ = 16; - - const int nb = args.ne00/QK4_0; - - const int r0 = (tgpig.x*NSG + sgitg)*NR0; - //const int r0 = tgpig.x*NR0; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - //const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - //device const block_q_type * x = (device const block_q_type *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - // pointers to src0 rows - device const block_q_type * ax[NR0]; - FOR_UNROLL (int row = 0; row < NR0; ++row) { - const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - - ax[row] = (device const block_q_type *) ((device char *) src0 + offset0); - } - - float sumf[NR0] = {0.f}; - - const short ix = (tiisg/(NW/NQ)); - const short il = (tiisg%(NW/NQ))*8; - - //const int ib0 = sgitg*NQ + ix; - const int ib0 = ix; - - float yl[16]; // src1 vector cache - - //device const float * yb = y + ix*QK4_0 + il; - device const float * yb = y + ib0*QK4_0 + il; - - // each thread in a SIMD group deals with half a block. - //for (int ib = ib0; ib < nb; ib += NSG*NQ) { - for (int ib = ib0; ib < nb; ib += NQ) { - float sumy[2] = { 0.f, 0.f }; - - FOR_UNROLL (short i = 0; i < 8; i += 2) { - sumy[0] += yb[i + 0] + yb[i + 1]; - yl[i + 0] = yb[i + 0]; - yl[i + 1] = yb[i + 1]/256.f; - - sumy[1] += yb[i + 16] + yb[i + 17]; - yl[i + 8] = yb[i + 16]/16.f; - yl[i + 9] = yb[i + 17]/4096.f; - } - - FOR_UNROLL (short row = 0; row < NR0; row++) { - sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy[0] + sumy[1], yl, il); - } - - yb += QK4_0 * 16; - //yb += NSG*NQ*QK4_0; - } - - device float * dst_f32 = (device float *) dst + im*args.ne0*args.ne1 + r1*args.ne0; - - //helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem); - - for (int row = 0; row < NR0; ++row) { - const float tot = simd_sum(sumf[row]); - - if (tiisg == 0 && r0 + row < args.ne01) { - dst_f32[r0 + row] = tot; - } - } -} - -template -void kernel_mul_mv_q1_0_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK1_0; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset1 = r1*args.nb11 + (i12)*args.nb12 + (i13)*args.nb13; - - device const float * y = (device const float *) (src1 + offset1); - - device const block_q1_0 * ax[nr0]; - for (int row = 0; row < nr0; ++row) { - const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - ax[row] = (device const block_q1_0 *) ((device char *) src0 + offset0); - } - - float yl[16]; - float sumf[nr0] = {0.f}; - - const short ix = (tiisg/8); - const short il = (tiisg%8)*16; - - device const float * yb = y + ix*QK1_0 + il; - - for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) { - float sumy = 0.f; - - FOR_UNROLL (short i = 0; i < 16; i++) { - yl[i] = yb[i]; - sumy += yb[i]; - } - - FOR_UNROLL (short row = 0; row < nr0; row++) { - sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy, yl, il); - } - - yb += QK1_0 * (N_SIMDWIDTH/8); - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0; ++row) { - const float tot = simd_sum(sumf[row]); - - if (tiisg == 0 && first_row + row < args.ne01) { - dst_f32[first_row + row] = tot; - } - } -} - -[[host_name("kernel_mul_mv_q1_0_f32")]] -kernel void kernel_mul_mv_q1_0_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_q1_0_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -} - -kernel void kernel_mul_mv_q4_0_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -kernel void kernel_mul_mv_q4_1_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -kernel void kernel_mul_mv_q5_0_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -kernel void kernel_mul_mv_q5_1_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_q8_0_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - constexpr short NW = N_SIMDWIDTH; - constexpr short NQ = 8; - - const int nb = args.ne00/QK8_0; - - const int r0 = tgpig.x*NR0; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - //const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - //device const block_q8_0 * x = (device const block_q8_0 *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - // pointers to src0 rows - device const block_q8_0 * ax[NR0]; - FOR_UNROLL (short row = 0; row < NR0; ++row) { - const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - - ax[row] = (device const block_q8_0 *) ((device char *) src0 + offset0); - } - - float sumf[NR0] = { 0.f }; - - const short ix = tiisg/(NW/NQ); - const short il = tiisg%(NW/NQ); - - const int ib0 = sgitg*NQ + ix; - - float yl[NQ]; - - device const float * yb = y + ib0*QK8_0 + il*NQ; - - // each thread in a SIMD group deals with NQ quants at a time - for (int ib = ib0; ib < nb; ib += NSG*NQ) { - for (short i = 0; i < NQ; ++i) { - yl[i] = yb[i]; - } - - for (short row = 0; row < NR0; row++) { - device const int8_t * qs = ax[row][ib].qs + il*NQ; - - float sumq = 0.f; - FOR_UNROLL (short i = 0; i < NQ; ++i) { - sumq += qs[i] * yl[i]; - } - - sumf[row] += sumq*ax[row][ib].d; - } - - yb += NSG*NQ*QK8_0; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem); -} - -[[host_name("kernel_mul_mv_q8_0_f32")]] -kernel void kernel_mul_mv_q8_0_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_q8_0_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -// mat-vec kernel processing in chunks of float4 -// chpb - chunks per quantization block -template -void kernel_mul_mv_ext_q4_f32_impl( - constant ggml_metal_kargs_mul_mv_ext & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - const short NSG = FC_mul_mv_nsg; - const short nxpsg = FC_mul_mv_nxpsg; - - const short chpt = 4; // chunks per thread - - //const short nxpsg = (32); - const short nypsg = (32/nxpsg); - - const short tx = tiisg%nxpsg; - const short ty = tiisg/nxpsg; - - const int i01 = tgpig.x*(nypsg*NSG) + nypsg*sgitg + ty; - const int i11 = tgpig.y*r1ptg; - const int i1m = tgpig.z; - - const int i12 = i1m%args.ne12; - const int i13 = i1m/args.ne12; - - const uint64_t offset0 = i01*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = i11*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const q_t * xq = (i01 < args.ne01) ? (device const q_t *) (src0 + offset0) + tx/chpb : (device const q_t *) src0; - - device const float4 * y4[r1ptg]; - - for (int ir1 = 0; ir1 < r1ptg; ++ir1) { - y4[ir1] = (i11 + ir1 < args.ne11) ? (device const float4 *) (src1 + offset1 + ir1*args.nb11) + tx : (device const float4 *) src1; - } - - float sumf[r1ptg] = { [ 0 ... r1ptg - 1 ] = 0.0f }; - - short cch = tx%chpb; // current chunk index - - for (int ich = tx; 4*ich < args.ne00; ich += chpt*nxpsg) { - float4 lx[chpt]; - -#pragma unroll(chpt) - for (short ch = 0; ch < chpt; ++ch) { - deq_t4(xq, cch, lx[ch]); - - cch += nxpsg; - if (cch >= chpb) { - xq += cch/chpb; - cch %= chpb; - } - } - -#pragma unroll(chpt) - for (short ch = 0; ch < chpt; ++ch) { -#pragma unroll(r1ptg) - for (short ir1 = 0; ir1 < r1ptg; ++ir1) { - sumf[ir1] += dot(lx[ch], y4[ir1][ch*nxpsg]); - } - } - -#pragma unroll(r1ptg) - for (short ir1 = 0; ir1 < r1ptg; ++ir1) { - y4[ir1] += chpt*nxpsg; - } - } - - // reduce only the threads in each row - for (short ir1 = 0; ir1 < r1ptg; ++ir1) { - if (nxpsg >= 32) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 16); - } - if (nxpsg >= 16) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 8); - } - if (nxpsg >= 8) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 4); - } - if (nxpsg >= 4) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 2); - } - if (nxpsg >= 2) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 1); - } - - //sumf[ir1] = simd_sum(sumf[ir1]); - } - - if (tx == 0) { - for (short ir1 = 0; ir1 < r1ptg && i11 + ir1 < args.ne11; ++ir1) { - device float * dst_f32 = (device float *) dst + (uint64_t)i1m*args.ne0*args.ne1 + (uint64_t)(i11 + ir1)*args.ne0; - - if (i01 < args.ne01) { - dst_f32[i01] = sumf[ir1]; - } - } - } -} - -// mat-vec kernel processing in chunks of float4x4 -template -void kernel_mul_mv_ext_q4x4_f32_impl( - constant ggml_metal_kargs_mul_mv_ext & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - const short NSG = FC_mul_mv_nsg; - const short nxpsg = FC_mul_mv_nxpsg; - - const short chpt = 1; - - //const short nxpsg = (32); - const short nypsg = (32/nxpsg); - - const short tx = tiisg%nxpsg; - const short ty = tiisg/nxpsg; - - const int i01 = tgpig.x*(nypsg*NSG) + nypsg*sgitg + ty; - const int i11 = tgpig.y*r1ptg; - const int i1m = tgpig.z; - - const int i12 = i1m%args.ne12; - const int i13 = i1m/args.ne12; - - const uint64_t offset0 = i01*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = i11*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const q_t * xq = (i01 < args.ne01) ? (device const q_t *) (src0 + offset0) + tx/chpb : (device const q_t *) src0; - - device const float4x4 * y4x4[r1ptg]; - - for (int ir1 = 0; ir1 < r1ptg; ++ir1) { - y4x4[ir1] = (i11 + ir1 < args.ne11) ? (device const float4x4 *) (src1 + offset1 + ir1*args.nb11) + tx : (device const float4x4 *) src1; - } - - float sumf[r1ptg] = { [ 0 ... r1ptg - 1 ] = 0.0f }; - - short cch = tx%chpb; - - for (int ich = tx; 16*ich < args.ne00; ich += chpt*nxpsg) { - float4x4 lx[chpt]; - -#pragma unroll(chpt) - for (short ch = 0; ch < chpt; ++ch) { - deq_t4x4(xq, cch, lx[ch]); - - cch += nxpsg; - if (cch >= chpb) { - xq += cch/chpb; - cch %= chpb; - } - } - -#pragma unroll(chpt) - for (short ch = 0; ch < chpt; ++ch) { -#pragma unroll(r1ptg) - for (short ir1 = 0; ir1 < r1ptg; ++ir1) { - sumf[ir1] += - dot(lx[ch][0], y4x4[ir1][ch*nxpsg][0]) + - dot(lx[ch][1], y4x4[ir1][ch*nxpsg][1]) + - dot(lx[ch][2], y4x4[ir1][ch*nxpsg][2]) + - dot(lx[ch][3], y4x4[ir1][ch*nxpsg][3]); - - } - } - -#pragma unroll(r1ptg) - for (short ir1 = 0; ir1 < r1ptg; ++ir1) { - y4x4[ir1] += chpt*nxpsg; - } - } - - for (short ir1 = 0; ir1 < r1ptg; ++ir1) { - if (nxpsg >= 32) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 16); - } - if (nxpsg >= 16) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 8); - } - if (nxpsg >= 8) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 4); - } - if (nxpsg >= 4) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 2); - } - if (nxpsg >= 2) { - sumf[ir1] += simd_shuffle_down(sumf[ir1], 1); - } - - //sumf[ir1] = simd_sum(sumf[ir1]); - } - - if (tx == 0) { - for (short ir1 = 0; ir1 < r1ptg && i11 + ir1 < args.ne11; ++ir1) { - device float * dst_f32 = (device float *) dst + (uint64_t)i1m*args.ne0*args.ne1 + (uint64_t)(i11 + ir1)*args.ne0; - - if (i01 < args.ne01) { - dst_f32[i01] = sumf[ir1]; - } - } - } -} - -// dispatchers needed for compile-time nxpsg -// epb - elements per quantization block -template -kernel void kernel_mul_mv_ext_q4_f32_disp( - constant ggml_metal_kargs_mul_mv_ext & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_ext_q4_f32_impl(args, src0, src1, dst, tgpig, tiisg, sgitg); -} - -template -kernel void kernel_mul_mv_ext_q4x4_f32_disp( - constant ggml_metal_kargs_mul_mv_ext & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_ext_q4x4_f32_impl(args, src0, src1, dst, tgpig, tiisg, sgitg); -} - -typedef decltype(kernel_mul_mv_ext_q4_f32_disp <2, block_q8_0, 32, dequantize_q8_0_t4>) mul_mv_ext_q4_f32_t; -typedef decltype(kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>) mul_mv_ext_q4x4_f32_t; - -template [[host_name("kernel_mul_mv_ext_f32_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, float4, 4, dequantize_f32_t4>; -template [[host_name("kernel_mul_mv_ext_f32_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, float4, 4, dequantize_f32_t4>; -template [[host_name("kernel_mul_mv_ext_f32_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, float4, 4, dequantize_f32_t4>; -template [[host_name("kernel_mul_mv_ext_f32_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, float4, 4, dequantize_f32_t4>; - -template [[host_name("kernel_mul_mv_ext_f16_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, half4, 4, dequantize_f16_t4>; -template [[host_name("kernel_mul_mv_ext_f16_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, half4, 4, dequantize_f16_t4>; -template [[host_name("kernel_mul_mv_ext_f16_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, half4, 4, dequantize_f16_t4>; -template [[host_name("kernel_mul_mv_ext_f16_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, half4, 4, dequantize_f16_t4>; - -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, bfloat4, 4, dequantize_bf16_t4>; -template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, bfloat4, 4, dequantize_bf16_t4>; -template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, bfloat4, 4, dequantize_bf16_t4>; -template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, bfloat4, 4, dequantize_bf16_t4>; -#endif - -template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q1_0, 128, dequantize_q1_0_t4>; -template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q1_0, 128, dequantize_q1_0_t4>; -template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q1_0, 128, dequantize_q1_0_t4>; -template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q1_0, 128, dequantize_q1_0_t4>; - -template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_0, 32, dequantize_q4_0_t4>; -template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_0, 32, dequantize_q4_0_t4>; -template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_0, 32, dequantize_q4_0_t4>; -template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q4_0, 32, dequantize_q4_0_t4>; - -template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_1, 32, dequantize_q4_1_t4>; -template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_1, 32, dequantize_q4_1_t4>; -template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_1, 32, dequantize_q4_1_t4>; -template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q4_1, 32, dequantize_q4_1_t4>; - -template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_0, 32, dequantize_q5_0_t4>; -template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_0, 32, dequantize_q5_0_t4>; -template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_0, 32, dequantize_q5_0_t4>; -template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q5_0, 32, dequantize_q5_0_t4>; - -template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_1, 32, dequantize_q5_1_t4>; -template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_1, 32, dequantize_q5_1_t4>; -template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_1, 32, dequantize_q5_1_t4>; -template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q5_1, 32, dequantize_q5_1_t4>; - -template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q8_0, 32, dequantize_q8_0_t4>; -template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q8_0, 32, dequantize_q8_0_t4>; -template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q8_0, 32, dequantize_q8_0_t4>; -template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q8_0, 32, dequantize_q8_0_t4>; - -template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_mxfp4, 32, dequantize_mxfp4_t4>; -template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_mxfp4, 32, dequantize_mxfp4_t4>; -template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_mxfp4, 32, dequantize_mxfp4_t4>; -template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_mxfp4, 32, dequantize_mxfp4_t4>; - -template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_iq4_nl, 32, dequantize_iq4_nl_t4>; -template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_iq4_nl, 32, dequantize_iq4_nl_t4>; -template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_iq4_nl, 32, dequantize_iq4_nl_t4>; -template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_iq4_nl, 32, dequantize_iq4_nl_t4>; - -template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>; -template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q4_K, 256, dequantize_q4_K>; -template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q4_K, 256, dequantize_q4_K>; -template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q4_K, 256, dequantize_q4_K>; - -template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q5_K, 256, dequantize_q5_K>; -template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q5_K, 256, dequantize_q5_K>; -template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q5_K, 256, dequantize_q5_K>; -template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q5_K, 256, dequantize_q5_K>; - -template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q6_K, 256, dequantize_q6_K>; -template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q6_K, 256, dequantize_q6_K>; -template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q6_K, 256, dequantize_q6_K>; -template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q6_K, 256, dequantize_q6_K>; - -template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q2_K, 256, dequantize_q2_K>; -template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q2_K, 256, dequantize_q2_K>; -template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q2_K, 256, dequantize_q2_K>; -template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q2_K, 256, dequantize_q2_K>; - -template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q3_K, 256, dequantize_q3_K>; -template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q3_K, 256, dequantize_q3_K>; -template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q3_K, 256, dequantize_q3_K>; -template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q3_K, 256, dequantize_q3_K>; - -template -void kernel_mul_mv_t_t_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - constexpr short NW = N_SIMDWIDTH; - constexpr short NB = 32; - constexpr short NF = 8; - - const int nb = args.ne00/NB; - - const int r0 = tgpig.x*NR0; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - //const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - //device const T0 * x = (device const T0 *) (src0 + offset0); - device const T1 * y = (device const T1 *) (src1 + offset1); - - // pointers to src0 rows - device const T0 * ax [NR0]; - FOR_UNROLL (short row = 0; row < NR0; ++row) { - const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - - ax[row] = (device const T0 *) ((device char *) src0 + offset0); - } - - float sumf[NR0] = { 0.f }; - - const short ix = tiisg/(NW/NF); - const short il = tiisg%(NW/NF); - - const int ib0 = sgitg*NF + ix; - - T1 yl[NF]; - - device const T1 * yb = y + (ib0*NB + il*NF); - - for (int ib = ib0; ib < nb; ib += NSG*NF) { - for (short i = 0; i < NF; ++i) { - yl[i] = yb[i]; - } - - for (short row = 0; row < NR0; row++) { - device const T0 * xb = ax[row] + (ib*NB + il*NF); - - float sumq = 0.f; - FOR_UNROLL (short i = 0; i < NF; ++i) { - sumq += xb[i] * yl[i]; - } - - sumf[row] += sumq; - } - - yb += NSG*NF*NW; - } - - for (int i = nb*NB + sgitg*NW + tiisg; i < args.ne00; i += NW*NSG) { - for (short row = 0; row < NR0; row++) { - sumf[row] += ax[row][i] * y[i]; - } - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem); -} - -template -void kernel_mul_mv_t_t_disp( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - switch (args.nr0) { - //case 1: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; - case 2: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; - //case 3: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; - //case 4: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; - } -} - -template -kernel void kernel_mul_mv_t_t( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_t_t_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -typedef decltype(kernel_mul_mv_t_t) mul_mv_t_t; - -template [[host_name("kernel_mul_mv_f32_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t; -template [[host_name("kernel_mul_mv_f16_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t; -template [[host_name("kernel_mul_mv_f16_f16")]] kernel mul_mv_t_t kernel_mul_mv_t_t; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mv_bf16_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t; -template [[host_name("kernel_mul_mv_bf16_bf16")]] kernel mul_mv_t_t kernel_mul_mv_t_t; -#endif - -template -void kernel_mul_mv_t_t_4_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - constexpr short NW = N_SIMDWIDTH; - constexpr short NB = 32; - constexpr short NF = 16; - constexpr short NF4 = NF/4; - - const int nb = args.ne00/NB; - - const int r0 = tgpig.x*NR0; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - //const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const T1 * y = (device const T1 *) (src1 + offset1); - device const T14 * y4 = (device const T14 *) (src1 + offset1); - - // pointers to src0 rows - device const T0 * ax [NR0]; - device const T04 * ax4[NR0]; - FOR_UNROLL (short row = 0; row < NR0; ++row) { - const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - - ax [row] = (device const T0 *) ((device char *) src0 + offset0); - ax4[row] = (device const T04 *) ((device char *) src0 + offset0); - } - - float sumf[NR0] = { 0.f }; - - const short ix = tiisg/(NW/NF); - const short il = tiisg%(NW/NF); - - const int ib0 = sgitg*NF + ix; - - T14 yl4[NF4]; - - device const T14 * yb4 = y4 + (ib0*NB + il*NF)/4; - - for (int ib = ib0; ib < nb; ib += NSG*NF) { - for (short i = 0; i < NF4; ++i) { - yl4[i] = yb4[i]; - } - - for (short row = 0; row < NR0; row++) { - device const T04 * xb4 = ax4[row] + (ib*NB + il*NF)/4; - - float sumq = 0.f; - FOR_UNROLL (short i = 0; i < NF4; ++i) { - sumq += dot(float4(xb4[i]), float4(yl4[i])); - } - - sumf[row] += sumq; - } - - yb4 += NSG*NF*NW/4; - } - - for (int i = nb*NB + sgitg*NW + tiisg; i < args.ne00; i += NW*NSG) { - for (short row = 0; row < NR0; row++) { - sumf[row] += ax[row][i] * y[i]; - } - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem); -} - -template -void kernel_mul_mv_t_t_4_disp( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - switch (args.nr0) { - //case 1: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; - case 2: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; - //case 3: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; - //case 4: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; - }; -} - -template -kernel void kernel_mul_mv_t_t_4( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_t_t_4_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -typedef decltype(kernel_mul_mv_t_t_4) mul_mv_t_t_4; - -template [[host_name("kernel_mul_mv_f32_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; -template [[host_name("kernel_mul_mv_f16_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; -template [[host_name("kernel_mul_mv_f16_f16_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mv_bf16_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; -template [[host_name("kernel_mul_mv_bf16_bf16_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; -#endif - -template -void kernel_mul_mv_t_t_short_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig, - ushort tiisg) { - const int r0 = tgpig.x*32 + tiisg; - const int r1 = tgpig.y; - const int im = tgpig.z; - - if (r0 >= args.ne01) { - return; - } - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = r0*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - - device const T0 * x = (device const T0 *) (src0 + offset0); - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1; - - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const T1 * y = (device const T1 *) (src1 + offset1); - - float res = 0.0f; - - for (int i = 0; i < args.ne00; ++i) { - res += (float) x[i] * (float) y[i]; - } - - dst_f32[(uint64_t)r1*args.ne0 + r0] = res; -} - -template -kernel void kernel_mul_mv_t_t_short( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]]) { - kernel_mul_mv_t_t_short_impl( - args, - src0, - src1, - dst, - tgpig, - tiisg); -} - -typedef decltype(kernel_mul_mv_t_t_short) mul_mv_t_t_short_t; - -template [[host_name("kernel_mul_mv_f32_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; -template [[host_name("kernel_mul_mv_f16_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; -template [[host_name("kernel_mul_mv_f16_f16_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mv_bf16_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; -template [[host_name("kernel_mul_mv_bf16_bf16_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; -#endif - -constant bool FC_rope_is_imrope [[function_constant(FC_ROPE + 0)]]; - -static float rope_yarn_ramp(const float low, const float high, const int i0) { - const float y = (i0 / 2 - low) / max(0.001f, high - low); - return 1.0f - min(1.0f, max(0.0f, y)); -} - -// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn -// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng. -static void rope_yarn( - float theta_extrap, float freq_scale, float corr_dims[2], int i0, float ext_factor, float mscale, - thread float * cos_theta, thread float * sin_theta) { - // Get n-d rotational scaling corrected for extrapolation - float theta_interp = freq_scale * theta_extrap; - float theta = theta_interp; - if (ext_factor != 0.0f) { - float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor; - theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix; - - // Get n-d magnitude scaling corrected for interpolation - mscale *= 1.0f + 0.1f * log(1.0f / freq_scale); - } - *cos_theta = cos(theta) * mscale; - *sin_theta = sin(theta) * mscale; -} - -// Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get -// `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))` -static float rope_yarn_corr_factor(int n_dims, int n_ctx_orig, float n_rot, float base) { - return n_dims * log(n_ctx_orig / (n_rot * 2 * M_PI_F)) / (2 * log(base)); -} - -static void rope_yarn_corr_dims( - int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2] -) { - // start and end correction dims - dims[0] = max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_fast, freq_base))); - dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_slow, freq_base))); -} - -template -kernel void kernel_rope_norm( - constant ggml_metal_kargs_rope & args, - device const char * src0, - device const char * src1, - device const char * src2, - device char * dst, - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 tptg [[threads_per_threadgroup]], - uint3 tgpig[[threadgroup_position_in_grid]]) { - const int i3 = tgpig[2]; - const int i2 = tgpig[1]; - const int i1 = tgpig[0]; - - float corr_dims[2]; - rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims); - - device const int32_t * pos = (device const int32_t *) src1; - - const float theta_base = (float) pos[i2]; - const float inv_ndims = -1.f/args.n_dims; - - float cos_theta; - float sin_theta; - - for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) { - if (i0 < args.n_dims) { - const int ic = i0/2; - - const float theta = theta_base * pow(args.freq_base, inv_ndims*i0); - - const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f; - - rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, i0, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta); - - device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - const float x0 = src[0]; - const float x1 = src[1]; - - dst_data[0] = x0*cos_theta - x1*sin_theta; - dst_data[1] = x0*sin_theta + x1*cos_theta; - } else { - device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - dst_data[0] = src[0]; - dst_data[1] = src[1]; - } - } -} - -template -kernel void kernel_rope_neox( - constant ggml_metal_kargs_rope & args, - device const char * src0, - device const char * src1, - device const char * src2, - device char * dst, - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 tptg [[threads_per_threadgroup]], - uint3 tgpig[[threadgroup_position_in_grid]]) { - const int i3 = tgpig[2]; - const int i2 = tgpig[1]; - const int i1 = tgpig[0]; - - float corr_dims[2]; - rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims); - - device const int32_t * pos = (device const int32_t *) src1; - - const float theta_base = (float) pos[i2]; - const float inv_ndims = -1.f/args.n_dims; - - float cos_theta; - float sin_theta; - - for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) { - if (i0 < args.n_dims) { - const int ic = i0/2; - - const float theta = theta_base * pow(args.freq_base, inv_ndims*i0); - - const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f; - - rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, i0, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta); - - device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + ic*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + ic*args.nb0); - - const float x0 = src[0]; - const float x1 = src[args.n_dims/2]; - - dst_data[0] = x0*cos_theta - x1*sin_theta; - dst_data[args.n_dims/2] = x0*sin_theta + x1*cos_theta; - } else { - device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - dst_data[0] = src[0]; - dst_data[1] = src[1]; - } - } -} - -template -kernel void kernel_rope_multi( - constant ggml_metal_kargs_rope & args, - device const char * src0, - device const char * src1, - device const char * src2, - device char * dst, - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 tptg [[threads_per_threadgroup]], - uint3 tgpig[[threadgroup_position_in_grid]]) { - const int i3 = tgpig[2]; - const int i2 = tgpig[1]; - const int i1 = tgpig[0]; - - float corr_dims[2]; - rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims); - - device const int32_t * pos = (device const int32_t *) src1; - - const float inv_ndims = -1.f/args.n_dims; - - float cos_theta; - float sin_theta; - - for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) { - if (i0 < args.n_dims) { - const int ic = i0/2; - - // mrope theta calculations - // note: the rest is the same as kernel_rope_neox - const int sect_dims = args.sect_0 + args.sect_1 + args.sect_2 + args.sect_3; - const int sec_w01 = args.sect_0 + args.sect_1; // end of section 1 - const int sec_w012 = args.sect_0 + args.sect_1 + args.sect_2; // end of section 2 - const int sector = ic % sect_dims; - - float theta_base; - if (FC_rope_is_imrope) { - if (sector % 3 == 1 && sector < 3 * args.sect_1) { // h - theta_base = (float) pos[i2 + args.ne02 * 1]; - } else if (sector % 3 == 2 && sector < 3 * args.sect_2) { // w - theta_base = (float) pos[i2 + args.ne02 * 2]; - } else if (sector % 3 == 0 && sector < 3 * args.sect_0) { // t - theta_base = (float) pos[i2 + args.ne02 * 0]; - } else { // e - theta_base = (float) pos[i2 + args.ne02 * 3]; - } - } else { - if (sector < args.sect_0) { - theta_base = (float) pos[i2]; - } else if (sector < sec_w01) { - theta_base = (float) pos[i2 + args.ne02 * 1]; - } else if (sector < sec_w012) { - theta_base = (float) pos[i2 + args.ne02 * 2]; - } else { - theta_base = (float) pos[i2 + args.ne02 * 3]; - } - } - // end of mrope - - const float theta = theta_base * pow(args.freq_base, inv_ndims*i0); - - const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f; - - rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, i0, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta); - - device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + ic*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + ic*args.nb0); - - const float x0 = src[0]; - const float x1 = src[args.n_dims/2]; - - dst_data[0] = x0*cos_theta - x1*sin_theta; - dst_data[args.n_dims/2] = x0*sin_theta + x1*cos_theta; - } else { - device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - dst_data[0] = src[0]; - dst_data[1] = src[1]; - } - } -} - -template -kernel void kernel_rope_vision( - constant ggml_metal_kargs_rope & args, - device const char * src0, - device const char * src1, - device const char * src2, - device char * dst, - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 tptg [[threads_per_threadgroup]], - uint3 tgpig[[threadgroup_position_in_grid]]) { - const int i3 = tgpig[2]; - const int i2 = tgpig[1]; - const int i1 = tgpig[0]; - - float corr_dims[2]; - rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims); - - device const int32_t * pos = (device const int32_t *) src1; - - const float inv_ndims = -1.f/args.n_dims; - - float cos_theta; - float sin_theta; - - for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) { - if (i0 < 2*args.n_dims) { // different from kernel_rope_multi - const int ic = i0/2; - - // mrope theta calculations (only support 2 dimensions) - const int sect_dims = args.sect_0 + args.sect_1; - const int sector = ic % sect_dims; - - float p; - float theta_base; - if (sector < args.sect_1) { - p = (float) sector; - theta_base = (float) pos[i2]; - } else { - p = (float) sector - args.sect_0; - theta_base = (float) pos[i2 + args.ne02]; - } - - const float theta = theta_base * pow(args.freq_base, 2.0f * inv_ndims * p); - // end of mrope - - const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f; - - rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, i0, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta); - - device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + ic*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + ic*args.nb0); - - const float x0 = src[0]; - const float x1 = src[args.n_dims]; // different from kernel_rope_multi - - dst_data[0] = x0*cos_theta - x1*sin_theta; - dst_data[args.n_dims] = x0*sin_theta + x1*cos_theta; // different from kernel_rope_multi - } else { - device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); - device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - dst_data[0] = src[0]; - dst_data[1] = src[1]; - } - } -} - -typedef decltype(kernel_rope_norm) kernel_rope_norm_t; -typedef decltype(kernel_rope_neox) kernel_rope_neox_t; -typedef decltype(kernel_rope_multi) kernel_rope_multi_t; -typedef decltype(kernel_rope_vision) kernel_rope_vision_t; - -template [[host_name("kernel_rope_norm_f32")]] kernel kernel_rope_norm_t kernel_rope_norm; -template [[host_name("kernel_rope_norm_f16")]] kernel kernel_rope_norm_t kernel_rope_norm; - -template [[host_name("kernel_rope_neox_f32")]] kernel kernel_rope_neox_t kernel_rope_neox; -template [[host_name("kernel_rope_neox_f16")]] kernel kernel_rope_neox_t kernel_rope_neox; - -template [[host_name("kernel_rope_multi_f32")]] kernel kernel_rope_multi_t kernel_rope_multi; -template [[host_name("kernel_rope_multi_f16")]] kernel kernel_rope_multi_t kernel_rope_multi; - -template [[host_name("kernel_rope_vision_f32")]] kernel kernel_rope_vision_t kernel_rope_vision; -template [[host_name("kernel_rope_vision_f16")]] kernel kernel_rope_vision_t kernel_rope_vision; - -typedef void (im2col_t)( - constant ggml_metal_kargs_im2col & args, - device const float * x, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]); - -template -kernel void kernel_im2col( - constant ggml_metal_kargs_im2col & args, - device const float * x, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { -// const int64_t IC = tgpg[0]; - const int64_t OH = tgpg[1]; - const int64_t OW = tgpg[2]; - - const int64_t KH = ntg[1]; - const int64_t KW = ntg[2]; - - int64_t in = tpitg[0]; - const int64_t ikh = tpitg[1]; - const int64_t ikw = tpitg[2]; - - const int64_t iic = tgpig[0]; - const int64_t ioh = tgpig[1]; - const int64_t iow = tgpig[2]; - - const int64_t iiw = iow*args.s0 + ikw*args.d0 - args.p0; - const int64_t iih = ioh*args.s1 + ikh*args.d1 - args.p1; - - int64_t offset_dst = (in*OH*OW + ioh*OW + iow)*args.CHW + (iic*(KH*KW) + ikh*KW + ikw); - - device T * pdst = (device T *) (dst); - - if (iih < 0 || iih >= args.IH || iiw < 0 || iiw >= args.IW) { - while (in < args.N) { - pdst[offset_dst] = 0.0f; - offset_dst += ntg[0]*args.CHW*OH*OW; - - in += ntg[0]; - } - } else { - int64_t offset_src = in*args.ofs0 + iic*args.ofs1 + iih*args.IW + iiw; - - while (in < args.N) { - pdst[offset_dst] = x[offset_src]; - - offset_dst += ntg[0]*args.CHW*OH*OW; - offset_src += ntg[0]*args.ofs0; - - in += ntg[0]; - } - } -} - -template [[host_name("kernel_im2col_f32")]] kernel im2col_t kernel_im2col; -template [[host_name("kernel_im2col_f16")]] kernel im2col_t kernel_im2col; - -// TODO: obsolete -- remove -//typedef void (im2col_ext_t)( -// constant ggml_metal_kargs_im2col & args, -// device const float * x, -// device char * dst, -// uint3 tgpig[[threadgroup_position_in_grid]], -// uint3 tgpg[[threadgroups_per_grid]], -// uint3 tpitg[[thread_position_in_threadgroup]], -// uint3 ntg[[threads_per_threadgroup]]); -// -//template -//kernel void kernel_im2col_ext( -// constant ggml_metal_kargs_im2col & args, -// device const float * x, -// device char * dst, -// uint3 tgpig[[threadgroup_position_in_grid]], -// uint3 tgpg[[threadgroups_per_grid]], // tgpg[0] = D x IC x KH x KW, CHW = IC x KH x KW -// uint3 tpitg[[thread_position_in_threadgroup]], -// uint3 ntg[[threads_per_threadgroup]]) { // [M, 1, 1] -// const int64_t KHW = (int64_t)args.KHW; -// -// const int64_t d = tgpig[0] / args.CHW; -// const int64_t chw = tgpig[0] % args.CHW; -// const int64_t tgpig_0 = chw / KHW; // 0 ~ (IC - 1) -// const int64_t HW = tgpig[0] % KHW; -// -// const int64_t tpitg_0 = (d * ntg[0]) + tpitg[0]; -// if (tpitg_0 >= args.N) { -// return; -// } -// -// const int64_t tpitg_1 = HW / args.KW; -// const int64_t tpitg_2 = HW % args.KW; -// -// const int64_t iiw = tgpig[2] * args.s0 + tpitg_2 * args.d0 - args.p0; -// const int64_t iih = tgpig[1] * args.s1 + tpitg_1 * args.d1 - args.p1; -// -// const int64_t offset_dst = -// (tpitg_0 * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * args.CHW + -// (tgpig_0 * KHW + tpitg_1 * args.KW + tpitg_2); -// -// device T * pdst = (device T *) (dst); -// -// if (iih < 0 || iih >= args.IH || iiw < 0 || iiw >= args.IW) { -// pdst[offset_dst] = 0.0f; -// } else { -// const int64_t offset_src = tpitg_0 * args.ofs0 + tgpig_0 * args.ofs1; -// pdst[offset_dst] = x[offset_src + iih * args.IW + iiw]; -// } -//} -// -//template [[host_name("kernel_im2col_ext_f32")]] kernel im2col_ext_t kernel_im2col_ext; -//template [[host_name("kernel_im2col_ext_f16")]] kernel im2col_ext_t kernel_im2col_ext; - -template -kernel void kernel_conv_2d( - constant ggml_metal_kargs_conv_2d & args, - device const char * weights, - device const char * src, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const uint threads_per_tg = ntg.x * ntg.y * ntg.z; - const uint tg_index = (tgpig.z * tgpg.y + tgpig.y) * tgpg.x + tgpig.x; - const uint local_thread = tpitg.z * (ntg.x * ntg.y) + tpitg.y * ntg.x + tpitg.x; - const uint thread_index = tg_index * threads_per_tg + local_thread; - const uint64_t total_threads = (uint64_t) threads_per_tg * tgpg.x * tgpg.y * tgpg.z; - const uint64_t total_outputs = (uint64_t) args.N * args.OC * args.OH * args.OW; - - for (uint64_t index = thread_index; index < total_outputs; index += total_threads) { - uint64_t tmp = index; - - const int32_t ow = tmp % args.OW; tmp /= args.OW; - const int32_t oh = tmp % args.OH; tmp /= args.OH; - const int32_t oc = tmp % args.OC; tmp /= args.OC; - const int32_t n = tmp; - - float acc = 0.0f; - - const int32_t base_x = ow*args.s0 - args.p0; - const int32_t base_y = oh*args.s1 - args.p1; - - int32_t ky_start = 0; - if (base_y < 0) { - ky_start = (-base_y + args.d1 - 1)/args.d1; - } - int32_t ky_end = args.KH; - const int32_t y_max = args.IH - 1 - base_y; - if (y_max < 0) { - ky_end = ky_start; - } else if (base_y + (args.KH - 1)*args.d1 >= args.IH) { - ky_end = min(ky_end, y_max/args.d1 + 1); - } - - int32_t kx_start = 0; - if (base_x < 0) { - kx_start = (-base_x + args.d0 - 1)/args.d0; - } - int32_t kx_end = args.KW; - const int32_t x_max = args.IW - 1 - base_x; - if (x_max < 0) { - kx_end = kx_start; - } else if (base_x + (args.KW - 1)*args.d0 >= args.IW) { - kx_end = min(kx_end, x_max/args.d0 + 1); - } - - if (ky_start < ky_end && kx_start < kx_end) { - const uint64_t src_base_n = (uint64_t) n * args.nb13; - const uint64_t w_base_oc = (uint64_t) oc * args.nb03; - - for (int32_t ic = 0; ic < args.IC; ++ic) { - const uint64_t src_base_nc = src_base_n + (uint64_t) ic * args.nb12; - const uint64_t w_base_ocic = w_base_oc + (uint64_t) ic * args.nb02; - - for (int32_t ky = ky_start; ky < ky_end; ++ky) { - const int32_t iy = base_y + ky*args.d1; - const uint64_t src_base_row = src_base_nc + (uint64_t) iy * args.nb11; - const uint64_t w_base_row = w_base_ocic + (uint64_t) ky * args.nb01; - - for (int32_t kx = kx_start; kx < kx_end; ++kx) { - const int32_t ix = base_x + kx*args.d0; - const uint64_t src_offs = src_base_row + (uint64_t) ix * args.nb10; - const uint64_t w_offs = w_base_row + (uint64_t) kx * args.nb00; - - const float x = *(device const float *)(src + src_offs); - const float w = (float) (*(device const TK *)(weights + w_offs)); - - acc += x * w; - } - } - } - } - - const uint64_t dst_offs = - (uint64_t) n * args.nb3 + - (uint64_t) oc * args.nb2 + - (uint64_t) oh * args.nb1 + - (uint64_t) ow * args.nb0; - - *(device float *)(dst + dst_offs) = acc; - } -} - -template [[host_name("kernel_conv_2d_f32_f32")]] -kernel void kernel_conv_2d( - constant ggml_metal_kargs_conv_2d & args, - device const char * weights, - device const char * src, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]); - -template [[host_name("kernel_conv_2d_f16_f32")]] -kernel void kernel_conv_2d( - constant ggml_metal_kargs_conv_2d & args, - device const char * weights, - device const char * src, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]); - -typedef void (conv_transpose_1d_t)( - constant ggml_metal_kargs_conv_transpose_1d & args, - device const float * src0, - device const float * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]]); - -template -kernel void kernel_conv_transpose_1d( - constant ggml_metal_kargs_conv_transpose_1d & args, - device const T * src0, - device const float * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]]) { - - float v = 0.0f; - - for (int64_t c = 0; c < args.IC; c++) { - const int32_t kernel_offset = c * tgpg[1] * args.K + args.K * tgpig[1]; - const int32_t input_offset = c * args.IL; - - for (int64_t i = 0; i < args.IL; i++) { - if (tgpig[0] >= i * args.s0 && tgpig[0] < i * args.s0 + args.K) { - v += src0[kernel_offset + tgpig[0] - i * args.s0] * src1[input_offset + i]; - } - } - } - - device float * dst_ptr = (device float *) (dst + tgpig[0] * args.nb0 + tgpig[1] * args.nb1); - - dst_ptr[0] = v; -} - -template [[host_name("kernel_conv_transpose_1d_f32_f32")]] -kernel void kernel_conv_transpose_1d( - constant ggml_metal_kargs_conv_transpose_1d & args, - device const float * src0, - device const float * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]]); - -template [[host_name("kernel_conv_transpose_1d_f16_f32")]] -kernel void kernel_conv_transpose_1d( - constant ggml_metal_kargs_conv_transpose_1d & args, - device const half * src0, - device const float * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]]); - - -typedef void (conv_transpose_2d_t)( - constant ggml_metal_kargs_conv_transpose_2d & args, - device const float * src0, - device const float * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]]); - -template -kernel void kernel_conv_transpose_2d( - constant ggml_metal_kargs_conv_transpose_2d & args, - device const T * src0, - device const float * src1, - device char * dst, - threadgroup float * shared_sum [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const int64_t out_x = tgpig[0]; - const int64_t out_y = tgpig[1]; - const int64_t out_c = tgpig[2]; - - const int64_t kw = tpitg[0]; - const int64_t kh = tpitg[1]; - - float v = 0.0f; - - for (int64_t in_c = 0; in_c < args.IC; in_c++) { - int64_t in_y = out_y - kh; - - if (in_y < 0 || in_y % args.s0) continue; - - in_y /= args.s0; - - if (in_y >= args.IH) continue; - - int64_t in_x = out_x - kw; - - if (in_x < 0 || in_x % args.s0) continue; - - in_x /= args.s0; - - if (in_x >= args.IW) continue; - - const int64_t input_idx = (args.IW * args.IH) * in_c + (args.IW) * in_y + in_x; - const int64_t kernel_idx = (args.KH * args.KW * args.OC) * in_c + (args.KH * args.KW) * out_c + (args.KW) * kh + kw; - - v += (float)src0[kernel_idx] * src1[input_idx]; - } - - const uint tid = tpitg.y * ntg.x + tpitg.x; - shared_sum[tid] = v; - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (tid == 0) { - float total = 0.0f; - const uint num_threads = ntg.x * ntg.y; - for (uint i = 0; i < num_threads; i++) { - total += shared_sum[i]; - } - - device float * dst_ptr = (device float *) (dst + out_x*args.nb0 + out_y * args.nb1 + out_c*args.nb2); - dst_ptr[0] = total; - } -} - -template [[host_name("kernel_conv_transpose_2d_f32_f32")]] -kernel void kernel_conv_transpose_2d( - constant ggml_metal_kargs_conv_transpose_2d & args, - device const float * src0, - device const float * src1, - device char * dst, - threadgroup float * shared_sum [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]); - -template [[host_name("kernel_conv_transpose_2d_f16_f32")]] -kernel void kernel_conv_transpose_2d( - constant ggml_metal_kargs_conv_transpose_2d & args, - device const half * src0, - device const float * src1, - device char * dst, - threadgroup float * shared_sum [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]); - -constant bool FC_upscale_aa [[function_constant(FC_UPSCALE + 0)]]; - -kernel void kernel_upscale_nearest_f32( - constant ggml_metal_kargs_upscale & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const int64_t i3 = tgpig.z; - const int64_t i2 = tgpig.y; - const int64_t i1 = tgpig.x; - - const int64_t i03 = i3/args.sf3; - const int64_t i02 = i2/args.sf2; - const int64_t i01 = i1/args.sf1; - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - const int64_t i00 = i0/args.sf0; - - device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00); - device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - dst_ptr[0] = src0_ptr[0]; - } -} - -static inline float bilinear_tri(float x) { - return MAX(0.0f, 1.0f - fabs(x)); -} - -kernel void kernel_upscale_bilinear_f32( - constant ggml_metal_kargs_upscale & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const int64_t i3 = tgpig.z; - const int64_t i2 = tgpig.y; - const int64_t i1 = tgpig.x; - - const int64_t i03 = i3 / args.sf3; - const int64_t i02 = i2 / args.sf2; - - const float f01 = ((float)i1 + args.poffs) / args.sf1 - args.poffs; - const int64_t i01 = MAX(0, MIN(args.ne01 - 1, (int64_t)floor(f01))); - const int64_t i01p = MAX(0, MIN(args.ne01 - 1, i01 + 1)); - const float fd1 = MAX(0.0f, MIN(1.0f, f01 - (float)i01)); - - src0 += i03*args.nb03 + i02*args.nb02; - - device float * dst_ptr = (device float *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1); - - if (FC_upscale_aa) { - const float support0 = MAX(1.0f, 1.0f / args.sf0); - const float invscale0 = 1.0f / support0; - const float support1 = MAX(1.0f, 1.0f / args.sf1); - const float invscale1 = 1.0f / support1; - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - const float f00 = ((float)i0 + args.poffs) / args.sf0 - args.poffs; - - int64_t x_min = MAX((int64_t)0, (int64_t)floor(f00 - support0 + args.poffs)); - int64_t x_max = MIN(args.ne00, (int64_t)ceil (f00 + support0 + args.poffs)); - - int64_t y_min = MAX((int64_t)0, (int64_t)floor(f01 - support1 + args.poffs)); - int64_t y_max = MIN(args.ne01, (int64_t)ceil (f01 + support1 + args.poffs)); - - float sum = 0.0f; - float wsum = 0.0f; - - for (int64_t sy = y_min; sy < y_max; ++sy) { - const float wy = MAX(0.0f, 1.0f - fabs((float)sy - f01) * invscale1); - for (int64_t sx = x_min; sx < x_max; ++sx) { - const float wx = MAX(0.0f, 1.0f - fabs((float)sx - f00) * invscale0); - const float w = wx * wy; - const device const float * src_ptr = (device const float *)(src0 + sy*args.nb01 + sx*args.nb00); - sum += (*src_ptr) * w; - wsum += w; - } - } - - const float v = (wsum > 0.0f) ? (sum / wsum) : 0.0f; - dst_ptr[i0] = v; - } - } else { - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - const float f00 = ((float)i0 + args.poffs) / args.sf0 - args.poffs; - const int64_t i00 = MAX(0, MIN(args.ne00 - 1, (int64_t)floor(f00))); - const int64_t i00p = MAX(0, MIN(args.ne00 - 1, i00 + 1)); - const float fd0 = MAX(0.0f, MIN(1.0f, f00 - (float)i00)); - - device const float * src00 = (device const float *)(src0 + i01*args.nb01 + i00*args.nb00); - device const float * src10 = (device const float *)(src0 + i01*args.nb01 + i00p*args.nb00); - device const float * src01 = (device const float *)(src0 + i01p*args.nb01 + i00*args.nb00); - device const float * src11 = (device const float *)(src0 + i01p*args.nb01 + i00p*args.nb00); - - const float v = - (*src00) * (1.0f - fd0) * (1.0f - fd1) + - (*src10) * fd0 * (1.0f - fd1) + - (*src01) * (1.0f - fd0) * fd1 + - (*src11) * fd0 * fd1; - - dst_ptr[i0] = v; - } - } -} - -template -kernel void kernel_conv_3d( - constant ggml_metal_kargs_conv_3d & args, - device const char * src0, // Weights [IC * OC, KD, KH, KW] - device const char * src1, // Inputs [IC * N, ID, IH, IW] - device char * dst, // Outputs [OC * N, OD, OH, OW] - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]]) { - - // 1. Un-flatten the spatial dimension from Grid X - int64_t spatial_idx = tgpig.x * 32 + tpitg.x; - - if (spatial_idx >= args.OW * args.OH * args.OD) { - return; // Thread falls outside the spatial volume - } - - int64_t od = spatial_idx / (args.OW * args.OH); - int64_t oh = (spatial_idx / args.OW) % args.OH; - int64_t ow = spatial_idx % args.OW; - - // 2. Map Y to Channels, Z to Batch - int64_t oc = tgpig.y; - int64_t batch_idx = tgpig.z; - - // 3. Calculate anchor coordinates in the Input volume - int64_t i_w_base = ow * args.s0 - args.p0; - int64_t i_h_base = oh * args.s1 - args.p1; - int64_t i_d_base = od * args.s2 - args.p2; - - float sum = 0.0f; - - // 4. Gather Loop (Iterate over Input Channels -> Depth -> Height -> Width) - for (int64_t ic = 0; ic < args.IC; ++ic) { - - // ggml packs batch and channel together in the 4th dimension - int64_t src_cn_idx = batch_idx * args.IC + ic; - int64_t w_cn_idx = oc * args.IC + ic; - - for (int64_t kz = 0; kz < args.KD; ++kz) { - int64_t id = i_d_base + kz * args.d2; - if (id < 0 || id >= args.ID) continue; // Boundary check (Padding) - - for (int64_t ky = 0; ky < args.KH; ++ky) { - int64_t ih = i_h_base + ky * args.d1; - if (ih < 0 || ih >= args.IH) continue; - - for (int64_t kx = 0; kx < args.KW; ++kx) { - int64_t iw = i_w_base + kx * args.d0; - if (iw < 0 || iw >= args.IW) continue; - - // Convert multi-dimensional coordinates to flat byte offsets - int64_t w_idx = kx*args.nb00 + ky*args.nb01 + kz*args.nb02 + w_cn_idx*args.nb03; - int64_t i_idx = iw*args.nb10 + ih*args.nb11 + id*args.nb12 + src_cn_idx*args.nb13; - - // Dereference memory and cast weights to f32 if they were f16 - float w_val = (float)*(device const T*)((device const char*)src0 + w_idx); - float i_val = *(device const float*)((device const char*)src1 + i_idx); - - sum += w_val * i_val; - } - } - } - } - - // 5. Write the accumulated value out to RAM - int64_t dst_cn_idx = batch_idx * args.OC + oc; - int64_t d_idx = ow*args.nb0 + oh*args.nb1 + od*args.nb2 + dst_cn_idx*args.nb3; - - *(device float*)(dst + d_idx) = sum; -} - -// Explicit instantiations so the JIT compiler can find them by name -template [[host_name("kernel_conv_3d_f32_f32")]] -kernel void kernel_conv_3d( - constant ggml_metal_kargs_conv_3d & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]]); - -// Explicit instantiation for f16 weights -template [[host_name("kernel_conv_3d_f16_f32")]] -kernel void kernel_conv_3d( - constant ggml_metal_kargs_conv_3d & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]]); - - -static inline float bicubic_weight1(float x) { - const float a = -0.75f; - return ((a + 2) * x - (a + 3)) * x * x + 1; -} - -static inline float bicubic_weight2(float x) { - const float a = -0.75f; - return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; -} - -kernel void kernel_upscale_bicubic_f32( - constant ggml_metal_kargs_upscale & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const int64_t i3 = tgpig.z; - const int64_t i2 = tgpig.y; - const int64_t i1 = tgpig.x; - - const int64_t i03 = i3 / args.sf3; - const int64_t i02 = i2 / args.sf2; - - const float f01 = ((float)i1 + args.poffs) / args.sf1 - args.poffs; - const int64_t i01 = (int64_t)floor(f01); - const float fd1 = f01 - (float)i01; - - const float w_y0 = bicubic_weight2(fd1 + 1.0f); - const float w_y1 = bicubic_weight1(fd1); - const float w_y2 = bicubic_weight1(1.0f - fd1); - const float w_y3 = bicubic_weight2(2.0f - fd1); - - const device const char * src_slice = src0 + i03 * args.nb03 + i02 * args.nb02; - - device float * dst_ptr = (device float *)(dst + i3 * args.nb3 + i2 * args.nb2 + i1 * args.nb1); - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - const float f00 = ((float)i0 + args.poffs) / args.sf0 - args.poffs; - const int64_t i00 = (int64_t)floor(f00); - const float fd0 = f00 - (float)i00; - - const float w_x0 = bicubic_weight2(fd0 + 1.0f); - const float w_x1 = bicubic_weight1(fd0); - const float w_x2 = bicubic_weight1(1.0f - fd0); - const float w_x3 = bicubic_weight2(2.0f - fd0); - - float sum = 0.0f; - - for (int dy = -1; dy <= 2; ++dy) { - const int64_t iy = MAX(0, MIN(args.ne01 - 1, i01 + dy)); - const float wy = (dy == -1) ? w_y0 : (dy == 0) ? w_y1 : (dy == 1) ? w_y2 : w_y3; - - for (int dx = -1; dx <= 2; ++dx) { - const int64_t ix = MAX(0, MIN(args.ne00 - 1, i00 + dx)); - const float wx = (dx == -1) ? w_x0 : (dx == 0) ? w_x1 : (dx == 1) ? w_x2 : w_x3; - - const device const float * src_ptr = (device const float *)(src_slice + iy * args.nb01 + ix * args.nb00); - sum += (*src_ptr) * wx * wy; - } - } - - dst_ptr[i0] = sum; - } -} - -kernel void kernel_roll_f32( - constant ggml_metal_kargs_roll & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const int64_t i3 = tgpig.z; - const int64_t i2 = tgpig.y; - const int64_t i1 = tgpig.x; - - device const float * src0_ptr = (device const float *) src0; - device float * dst_ptr = (device float *) dst; - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - // apply shifts and wrap around - int64_t i00 = i0 - args.s0; - int64_t i01 = i1 - args.s1; - int64_t i02 = i2 - args.s2; - int64_t i03 = i3 - args.s3; - - if (i00 < 0) { i00 += args.ne00; } else if (i00 >= args.ne00) { i00 -= args.ne00; } - if (i01 < 0) { i01 += args.ne01; } else if (i01 >= args.ne01) { i01 -= args.ne01; } - if (i02 < 0) { i02 += args.ne02; } else if (i02 >= args.ne02) { i02 -= args.ne02; } - if (i03 < 0) { i03 += args.ne03; } else if (i03 >= args.ne03) { i03 -= args.ne03; } - - int64_t src_idx = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00 + i00; - int64_t dst_idx = i3 *args.ne2 *args.ne1 *args.ne0 + i2 *args.ne1 *args.ne0 + i1 *args.ne0 + i0; - - dst_ptr[dst_idx] = src0_ptr[src_idx]; - } -} - -kernel void kernel_pad_f32( - constant ggml_metal_kargs_pad & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const int64_t i3 = tgpig.z; - const int64_t i2 = tgpig.y; - const int64_t i1 = tgpig.x; - - const int64_t i03 = i3; - const int64_t i02 = i2; - const int64_t i01 = i1; - - device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); - device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1); - - if (i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) { - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - if (i0 < args.ne00) { - dst_ptr[i0] = src0_ptr[i0]; - } else { - dst_ptr[i0] = 0.0f; - } - } - - return; - } - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - dst_ptr[i0] = 0.0f; - } -} - -kernel void kernel_pad_reflect_1d_f32( - constant ggml_metal_kargs_pad_reflect_1d & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tgpg[[threadgroups_per_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - const int64_t i3 = tgpig.z; - const int64_t i2 = tgpig.y; - const int64_t i1 = tgpig.x; - - const int64_t i03 = i3; - const int64_t i02 = i2; - const int64_t i01 = i1; - - device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); - device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1); - - if (i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) { - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - if (i0 < args.p0) { - dst_ptr[i0] = src0_ptr[args.p0 - i0]; - } else if (i0 < args.ne0 - args.p1) { - dst_ptr[i0] = src0_ptr[i0 - args.p0]; - } else { - dst_ptr[i0] = src0_ptr[(args.ne0 - args.p1 - args.p0) - (args.p1 + 1 - (args.ne0 - i0)) - 1]; - } - } - } -} - -kernel void kernel_arange_f32( - constant ggml_metal_kargs_arange & args, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - device float * dst_ptr = (device float *) dst; - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - dst_ptr[i0] = args.start + args.step * i0; - } -} - -kernel void kernel_timestep_embedding_f32( - constant ggml_metal_kargs_timestep_embedding & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint3 tpitg[[thread_position_in_threadgroup]], - uint3 ntg[[threads_per_threadgroup]]) { - - int i = tgpig.x; - device float * embed_data = (device float *)(dst + i*args.nb1); - - int half_ = args.dim / 2; - for (int j = tpitg.x; j < half_; j += ntg.x) { - float timestep = ((device float *)src0)[i]; - float freq = (float)exp(-log((float)args.max_period) * j / half_); - float arg = timestep * freq; - embed_data[j ] = cos(arg); - embed_data[j + half_] = sin(arg); - } - - if (args.dim % 2 != 0 && tpitg.x == 0) { - embed_data[2 * half_] = 0.f; - } -} - -// bitonic sort implementation following the CUDA kernels as reference -typedef void (argsort_t)( - constant ggml_metal_kargs_argsort & args, - device const char * src0, - device int32_t * dst, - threadgroup int32_t * shmem_i32 [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]); - -template -kernel void kernel_argsort_f32_i32( - constant ggml_metal_kargs_argsort & args, - device const char * src0, - device int32_t * dst, - threadgroup int32_t * shmem_i32 [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - // bitonic sort - const int col = tpitg[0]; - const int ib = tgpig[0] / args.ne01; - - const int i00 = ib*ntg.x; - const int i01 = tgpig[0] % args.ne01; - const int i02 = tgpig[1]; - const int i03 = tgpig[2]; - - device const float * src0_row = (device const float *) (src0 + args.nb01*i01 + args.nb02*i02 + args.nb03*i03); - - // initialize indices - shmem_i32[col] = i00 + col; - - threadgroup_barrier(mem_flags::mem_threadgroup); - - for (int k = 2; k <= ntg.x; k *= 2) { - for (int j = k / 2; j > 0; j /= 2) { - int ixj = col ^ j; - if (ixj > col) { - if ((col & k) == 0) { - if (shmem_i32[col] >= args.ne00 || - (shmem_i32[ixj] < args.ne00 && (order == GGML_SORT_ORDER_ASC ? - src0_row[shmem_i32[col]] > src0_row[shmem_i32[ixj]] : - src0_row[shmem_i32[col]] < src0_row[shmem_i32[ixj]])) - ) { - SWAP(shmem_i32[col], shmem_i32[ixj]); - } - } else { - if (shmem_i32[ixj] >= args.ne00 || - (shmem_i32[col] < args.ne00 && (order == GGML_SORT_ORDER_ASC ? - src0_row[shmem_i32[col]] < src0_row[shmem_i32[ixj]] : - src0_row[shmem_i32[col]] > src0_row[shmem_i32[ixj]])) - ) { - SWAP(shmem_i32[col], shmem_i32[ixj]); - } - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - } - } - - const int64_t i0 = ib*args.top_k; - - // copy the result to dst without the padding - if (i0 + col < args.ne0 && col < args.top_k) { - dst += i0 + args.ne0*i01 + args.ne0*args.ne1*i02 + args.ne0*args.ne1*args.ne2*i03; - - dst[col] = shmem_i32[col]; - } -} - -template [[host_name("kernel_argsort_f32_i32_asc")]] kernel argsort_t kernel_argsort_f32_i32; -template [[host_name("kernel_argsort_f32_i32_desc")]] kernel argsort_t kernel_argsort_f32_i32; - -typedef void (argsort_merge_t)( - constant ggml_metal_kargs_argsort_merge & args, - device const char * src0, - device const int32_t * tmp, - device int32_t * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]); - -template -kernel void kernel_argsort_merge_f32_i32( - constant ggml_metal_kargs_argsort_merge & args, - device const char * src0, - device const int32_t * tmp, - device int32_t * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - - const int im = tgpig[0] / args.ne01; - const int i01 = tgpig[0] % args.ne01; - const int i02 = tgpig[1]; - const int i03 = tgpig[2]; - - const int start = im * (2 * args.len); - - const int len0 = MIN(args.len, MAX(0, args.ne0 - (int)(start))); - const int len1 = MIN(args.len, MAX(0, args.ne0 - (int)(start + args.len))); - - const int total = len0 + len1; - - device const int32_t * tmp0 = tmp + start - + i01*args.ne0 - + i02*args.ne0*args.ne01 - + i03*args.ne0*args.ne01*args.ne02; - - device const int32_t * tmp1 = tmp0 + args.len; - - dst += start - + i01*args.top_k - + i02*args.top_k*args.ne01 - + i03*args.top_k*args.ne01*args.ne02; - - device const float * src0_row = (device const float *)(src0 - + args.nb01*i01 - + args.nb02*i02 - + args.nb03*i03); - - if (total == 0) { - return; - } - - const int chunk = (total + ntg.x - 1) / ntg.x; - - const int k0 = tpitg.x * chunk; - const int k1 = MIN(MIN(k0 + chunk, total), args.top_k); - - if (k0 >= args.top_k) { - return; - } - - if (k0 >= total) { - return; - } - - int low = k0 > len1 ? k0 - len1 : 0; - int high = MIN(k0, len0); - - // binary-search partition (i, j) such that i + j = k - while (low < high) { - const int mid = (low + high) >> 1; - - const int32_t idx0 = tmp0[mid]; - const int32_t idx1 = tmp1[k0 - mid - 1]; - - const float val0 = src0_row[idx0]; - const float val1 = src0_row[idx1]; - - bool take_left; - if (order == GGML_SORT_ORDER_ASC) { - take_left = (val0 <= val1); - } else { - take_left = (val0 >= val1); - } - - if (take_left) { - low = mid + 1; - } else { - high = mid; - } - } - - int i = low; - int j = k0 - i; - - // keep the merge fronts into registers - int32_t idx0 = 0; - float val0 = 0.0f; - if (i < len0) { - idx0 = tmp0[i]; - val0 = src0_row[idx0]; - } - - int32_t idx1 = 0; - float val1 = 0.0f; - if (j < len1) { - idx1 = tmp1[j]; - val1 = src0_row[idx1]; - } - - for (int k = k0; k < k1; ++k) { - int32_t out_idx; - - if (i >= len0) { - while (k < k1) { - dst[k++] = tmp1[j++]; - } - break; - } else if (j >= len1) { - while (k < k1) { - dst[k++] = tmp0[i++]; - } - break; - } else { - bool take_left; - - if (order == GGML_SORT_ORDER_ASC) { - take_left = (val0 <= val1); - } else { - take_left = (val0 >= val1); - } - - if (take_left) { - out_idx = idx0; - ++i; - if (i < len0) { - idx0 = tmp0[i]; - val0 = src0_row[idx0]; - } - } else { - out_idx = idx1; - ++j; - if (j < len1) { - idx1 = tmp1[j]; - val1 = src0_row[idx1]; - } - } - } - - dst[k] = out_idx; - } -} - -template [[host_name("kernel_argsort_merge_f32_i32_asc")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32; -template [[host_name("kernel_argsort_merge_f32_i32_desc")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32; - -constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]]; - -constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]]; - -// pad the last chunk of C elements of k and v into a an extra pad buffer -kernel void kernel_flash_attn_ext_pad( - constant ggml_metal_kargs_flash_attn_ext_pad & args, - device const char * k, - device const char * v, - device const char * mask, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int32_t C = FC_flash_attn_ext_pad_ncpsg; - - device char * k_pad = dst; - device char * v_pad = k_pad + args.nb11*C*args.ne_12_2*args.ne_12_3; - device char * mask_pad = v_pad + args.nb21*C*args.ne_12_2*args.ne_12_3; - - const int32_t icp = args.ne11 % C; - const int32_t ic0 = args.ne11 - icp; - - const int32_t i1 = tgpig[0]; - const int32_t i2 = tgpig[1]; - const int32_t i3 = tgpig[2]; - - if (i2 < args.ne_12_2 && i3 < args.ne_12_3) { - device const char * k_src = k + args.nb11*(ic0 + i1) + args.nb12*i2 + args.nb13*i3; - device const char * v_src = v + args.nb21*(ic0 + i1) + args.nb22*i2 + args.nb23*i3; - - device char * k_dst = k_pad + args.nb11*i1 + args.nb11*C*i2 + args.nb11*C*args.ne_12_2*i3; - device char * v_dst = v_pad + args.nb21*i1 + args.nb21*C*i2 + args.nb21*C*args.ne_12_2*i3; - - if (i1 >= icp) { - // here it is not important the exact value that will be used as we rely on masking out the scores in the attention - for (uint64_t i = tiitg; i < args.nb11; i += ntg.x) { - k_dst[i] = 0; - } - for (uint64_t i = tiitg; i < args.nb21; i += ntg.x) { - v_dst[i] = 0; - } - } else { - for (uint64_t i = tiitg; i < args.nb11; i += ntg.x) { - k_dst[i] = k_src[i]; - } - for (uint64_t i = tiitg; i < args.nb21; i += ntg.x) { - v_dst[i] = v_src[i]; - } - } - } - - if (FC_flash_attn_ext_pad_has_mask) { - if (i2 < args.ne32 && i3 < args.ne33) { - for (int ib = i1; ib < args.ne31; ib += C) { - device const half * mask_src = (device const half *)(mask + args.nb31*ib + args.nb32*i2 + args.nb33*i3) + ic0; - device half * mask_dst = (device half *)(mask_pad) + C*ib + C*args.ne31*i2 + C*args.ne31*args.ne32*i3; - - for (int i = tiitg; i < C; i += ntg.x) { - if (i >= icp) { - mask_dst[i] = -MAXHALF; - } else { - mask_dst[i] = mask_src[i]; - } - } - } - } - } -} - -constant int32_t FC_flash_attn_ext_blk_nqptg [[function_constant(FC_FLASH_ATTN_EXT_BLK + 24)]]; -constant int32_t FC_flash_attn_ext_blk_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_BLK + 25)]]; - -// scan the blocks of the mask that are not masked -// 0 - masked (i.e. full of -INF, skip) -// 1 - not masked (i.e. at least one element of the mask is not -INF) -// 2 - all zero -kernel void kernel_flash_attn_ext_blk( - constant ggml_metal_kargs_flash_attn_ext_blk & args, - device const char * mask, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]]) { - // block size C x Q - const int32_t Q = FC_flash_attn_ext_blk_nqptg; - const int32_t C = FC_flash_attn_ext_blk_ncpsg; - - constexpr short NW = N_SIMDWIDTH; - - const int32_t i3 = tgpig[2]/args.ne32; - const int32_t i2 = tgpig[2]%args.ne32; - const int32_t i1 = tgpig[1]; - const int32_t i0 = tgpig[0]; - - char res = i0*C + C > args.ne30 ? 1 : 0; - - device const half * mask_src = (device const half *) (mask + (i1*Q)*args.nb31 + i2*args.nb32 + i3*args.nb33) + i0*C + tiisg; - - // detailed check of the elements of the block - if ((C > NW || Q > 1) && res == 0) { - half mmin = MAXHALF; - half mmax = -MAXHALF; - - FOR_UNROLL (short j = 0; j < Q; ++j) { - FOR_UNROLL (short ii = 0; ii < C/NW; ++ii) { - mmin = min(mmin, mask_src[ii*NW]); - mmax = max(mmax, mask_src[ii*NW]); - } - - mask_src += args.nb31/2; - } - - mmin = simd_min(mmin); - mmax = simd_max(mmax); - - if (mmax > -MAXHALF) { - if (mmin == 0.0 && mmax == 0.0) { - res = 2; - } else { - res = 1; - } - } - } - - const int32_t nblk1 = ((args.ne01 + Q - 1)/Q); - const int32_t nblk0 = ((args.ne30 + C - 1)/C); - - if (tiisg == 0) { - dst[((i3*args.ne32 + i2)*nblk1 + i1)*nblk0 + i0] = res; - } -} - -constant bool FC_flash_attn_ext_has_mask [[function_constant(FC_FLASH_ATTN_EXT + 0)]]; -constant bool FC_flash_attn_ext_has_sinks [[function_constant(FC_FLASH_ATTN_EXT + 1)]]; -constant bool FC_flash_attn_ext_has_bias [[function_constant(FC_FLASH_ATTN_EXT + 2)]]; -constant bool FC_flash_attn_ext_has_scap [[function_constant(FC_FLASH_ATTN_EXT + 3)]]; -constant bool FC_flash_attn_ext_has_kvpad [[function_constant(FC_FLASH_ATTN_EXT + 4)]]; - -constant bool FC_flash_attn_ext_bc_mask [[function_constant(FC_FLASH_ATTN_EXT + 10)]]; - -//constant float FC_flash_attn_ext_scale [[function_constant(FC_FLASH_ATTN_EXT + 10)]]; -//constant float FC_flash_attn_ext_max_bias [[function_constant(FC_FLASH_ATTN_EXT + 11)]]; -//constant float FC_flash_attn_ext_logit_softcap [[function_constant(FC_FLASH_ATTN_EXT + 12)]]; - -constant int32_t FC_flash_attn_ext_ns10 [[function_constant(FC_FLASH_ATTN_EXT + 20)]]; -constant int32_t FC_flash_attn_ext_ns20 [[function_constant(FC_FLASH_ATTN_EXT + 21)]]; -constant int32_t FC_flash_attn_ext_nsg [[function_constant(FC_FLASH_ATTN_EXT + 22)]]; - -// ref: https://arxiv.org/pdf/2307.08691.pdf -template< - typename q_t, // query types in shared memory - typename q4_t, - typename q8x8_t, - typename k_t, // key types in shared memory - typename k4x4_t, - typename k8x8_t, - typename v_t, // value types in shared memory - typename v4x4_t, - typename v8x8_t, - typename qk_t, // Q*K types - typename qk8x8_t, - typename s_t, // soft-max types - typename s2_t, - typename s8x8_t, - typename o_t, // attention accumulation types - typename o4_t, - typename o8x8_t, - typename kd4x4_t, // key type in device memory - short nl_k, - void (*deq_k)(device const kd4x4_t *, short, thread k4x4_t &), - typename vd4x4_t, // value type in device memory - short nl_v, - void (*deq_v)(device const vd4x4_t *, short, thread v4x4_t &), - short DK, // K head size - short DV, // V head size - short Q, // queries per threadgroup - short C, // cache items per threadgroup - short NSG> // number of simd groups -void kernel_flash_attn_ext_impl( - constant ggml_metal_kargs_flash_attn_ext & args, - device const char * q, - device const char * k, - device const char * v, - device const char * mask, - device const char * sinks, - device const char * pad, - device const char * blk, - device char * dst, - threadgroup half * shmem_f16, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const ushort iq3 = tgpig[2]; - const ushort iq2 = tgpig[1]; - const ushort iq1 = tgpig[0]*Q; - -#define NS10 (FC_flash_attn_ext_ns10) -#define NS20 (FC_flash_attn_ext_ns20) - - // note: I had some concerns that using this instead of the ugly macros above was affecting performance - // need to re-check carefully and if no regressions are observerd - remove the macros - // the concerns is that maybe using const variables requires extra registers? but not sure if the compiler - // is clever enough to avoid this. unfortunately, using constexpr is not possible with FC - //const short NS10 = FC_flash_attn_ext_ns10; - //const short NS20 = FC_flash_attn_ext_ns20; - - constexpr short KV = 8; - - constexpr short DK4 = DK/4; - constexpr short DK8 = DK/8; - constexpr short DK16 = DK/16; - constexpr short DV4 = DV/4; - //constexpr short DV8 = DV/8; - constexpr short DV16 = DV/16; - - constexpr short PV = PAD2(DV, 64); - constexpr short PV4 = PV/4; - constexpr short PV8 = PV/8; - //constexpr short PV16 = PV/16; - - constexpr short NW = N_SIMDWIDTH; - constexpr short NQ = Q/NSG; - constexpr short SH = 2*C; // shared memory per simdgroup (s_t == float) - - constexpr short TS = 2*SH; - constexpr short T = DK + 2*PV; // shared memory size per query in (half) - - threadgroup q_t * sq = (threadgroup q_t *) (shmem_f16 + 0*T); // holds the query data - threadgroup q4_t * sq4 = (threadgroup q4_t *) (shmem_f16 + 0*T); // same as above but in q4_t - threadgroup o_t * so = (threadgroup o_t *) (shmem_f16 + 0*T + Q*DK); // the result for all queries in 8x8 matrices (the O matrix from the paper) - threadgroup o4_t * so4 = (threadgroup o4_t *) (shmem_f16 + 0*T + Q*DK); - threadgroup s_t * ss = (threadgroup s_t *) (shmem_f16 + Q*T); // scratch buffer for attention, mask and diagonal matrix - threadgroup s2_t * ss2 = (threadgroup s2_t *) (shmem_f16 + Q*T); // same as above but in s2_t - - threadgroup k_t * sk = (threadgroup k_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // scratch buffer to load K in shared memory - threadgroup k4x4_t * sk4x4 = (threadgroup k4x4_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // same as above but in k4x4_t - - threadgroup v_t * sv = (threadgroup v_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // scratch buffer to load V in shared memory - threadgroup v4x4_t * sv4x4 = (threadgroup v4x4_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // same as above but in v4x4_t - - // mask storage in shared mem - threadgroup half2 * sm2 = (threadgroup half2 *) (shmem_f16 + Q*T + 2*C); - - // per-query mask pointers - device const half2 * pm2[NQ]; - - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - pm2[jj] = (device const half2 *) ((device const char *) mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33); - } - - { - const int32_t nblk1 = ((args.ne01 + Q - 1)/Q); - const int32_t nblk0 = ((args.ne11 + C - 1)/C); - - blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0; - } - - { - q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03; - - const short ikv2 = iq2/(args.ne02/args.ne_12_2); - const short ikv3 = iq3/(args.ne03/args.ne_12_3); - - k += ikv2*args.nb12 + ikv3*args.nb13; - v += ikv2*args.nb22 + ikv3*args.nb23; - } - - // load heads from Q to shared memory - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - device const float4 * q4 = (device const float4 *) ((device const char *) q + j*args.nb01); - - for (short i = tiisg; i < DK4; i += NW) { - if (iq1 + j < args.ne01) { - sq4[j*DK4 + i] = (q4_t) q4[i]; - } else { - sq4[j*DK4 + i] = 0; - } - } - } - - // zero out - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - for (short i = tiisg; i < DV4; i += NW) { - so4[j*PV4 + i] = 0; - } - - for (short i = tiisg; i < SH; i += NW) { - ss[j*SH + i] = 0.0f; - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - float S[NQ] = { [0 ... NQ-1] = 0.0f }; - - { - float M[NQ] = { [0 ... NQ-1] = -FLT_MAX/2 }; - - float slope = 1.0f; - - // ALiBi - if (FC_flash_attn_ext_has_bias) { - const short h = iq2; - - const float base = h < args.n_head_log2 ? args.m0 : args.m1; - const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; - - slope = pow(base, exph); - } - - // loop over the KV cache - // each simdgroup handles blocks of Q rows and C columns - for (int ic0 = 0; ; ++ic0) { - int ic = ic0*C; - if (ic >= args.ne11) { - break; - } - - // the last partial chunk uses the pad buffer as source - if (FC_flash_attn_ext_has_kvpad && ic + C > args.ne11) { - k = pad; - v = k + args.nb11*C*args.ne_12_2*args.ne_12_3; - mask = v + args.nb21*C*args.ne_12_2*args.ne_12_3; - - const short ikv2 = iq2/(args.ne02/args.ne_12_2); - const short ikv3 = iq3/(args.ne03/args.ne_12_3); - - k += (ikv2 + ikv3*args.ne_12_2)*args.nb11*C; - v += (ikv2 + ikv3*args.ne_12_2)*args.nb21*C; - - if (!FC_flash_attn_ext_has_mask) { - threadgroup half * sm = (threadgroup half *) (sm2); - - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - for (short i = tiisg; i < C; i += NW) { - if (ic + i >= args.ne11) { - sm[2*j*SH + i] = -MAXHALF; - } - } - } - } else { - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - pm2[jj] = (device const half2 *) ((device const half *) mask + - (iq1 + j)*C + - (iq2%args.ne32)*(C*args.ne31) + - (iq3%args.ne33)*(C*args.ne31*args.ne32)); - } - } - - ic = 0; - } - - char blk_cur = 1; - - // read the mask into shared mem - if (FC_flash_attn_ext_has_mask) { - blk_cur = blk[ic0]; - - if (blk_cur == 0) { - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - pm2[jj] += NW; - } - - continue; - } - - if (blk_cur == 1) { - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - if (FC_flash_attn_ext_bc_mask) { - sm2[j*SH + tiisg] = (iq1 + j) < args.ne31 ? pm2[jj][tiisg] : half2(-MAXHALF, -MAXHALF); - } else { - sm2[j*SH + tiisg] = pm2[jj][tiisg]; - } - - pm2[jj] += NW; - } - } else if (blk_cur == 2) { - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - pm2[jj] += NW; - } - } - -#if 0 - // note: old -INF block optimization - obsoleted by pre-computing non-masked blocks - - threadgroup_barrier(mem_flags::mem_threadgroup); - - // used to detect blocks full of -INF - // skip only when the entire threadgroup is masked - half2 smax2(-MAXHALF/2, -MAXHALF/2); - - FOR_UNROLL (short j = 0; j < Q; ++j) { - smax2 = max(smax2, sm2[j*SH + tiisg]); - } - - smax2 = simd_max(smax2); - - if (max(smax2[0], smax2[1]) <= -MAXHALF/2) { - // this barrier is important - threadgroup_barrier(mem_flags::mem_threadgroup); - - continue; - } -#endif - } - - // Q*K^T - // this is compile-time check, so it does not have runtime overhead - if (is_same::value) { - // we can read directly from global memory - device const k_t * pk = (device const k_t *) (k + ic*args.nb11); - threadgroup const q_t * pq = sq; - threadgroup s_t * ps = ss; - - pk += sgitg*(8*NS10); - ps += sgitg*(8*1); - - static_assert((C/8) % NSG == 0, ""); - - constexpr short NC = (C/8)/NSG; - - FOR_UNROLL (short cc = 0; cc < NC; ++cc) { - qk8x8_t mqk = make_filled_simdgroup_matrix((qk_t) 0.0f); - - if (DK % 16 != 0) { - k8x8_t mk; - q8x8_t mq; - - FOR_UNROLL (short i = 0; i < DK8; ++i) { - simdgroup_barrier(mem_flags::mem_none); - - simdgroup_load(mk, pk + 8*i, NS10, 0, true); - simdgroup_load(mq, pq + 8*i, DK); - - simdgroup_barrier(mem_flags::mem_none); - - simdgroup_multiply_accumulate(mqk, mq, mk, mqk); - } - } else { - k8x8_t mk[2]; - q8x8_t mq[2]; - - // note: too much unroll can tank the performance for large heads - #pragma unroll (MIN(DK8/2, 4*NSG)) - for (short i = 0; i < DK8/2; ++i) { - simdgroup_barrier(mem_flags::mem_none); - - simdgroup_load(mq[0], pq + 0*8 + 16*i, DK); - simdgroup_load(mq[1], pq + 1*8 + 16*i, DK); - - simdgroup_load(mk[0], pk + 0*8 + 16*i, NS10, 0, true); - simdgroup_load(mk[1], pk + 1*8 + 16*i, NS10, 0, true); - - simdgroup_barrier(mem_flags::mem_none); - - simdgroup_multiply_accumulate(mqk, mq[0], mk[0], mqk); - simdgroup_multiply_accumulate(mqk, mq[1], mk[1], mqk); - } - } - - simdgroup_store(mqk, ps, SH, 0, false); - - pk += 8*(NSG*NS10); - ps += 8*(NSG); - } - } else { - // TODO: this is the quantized K cache branch - not optimized yet - for (short ccc = 0; ccc < (C/8)/NSG; ++ccc) { - const short cc = ccc*NSG + sgitg; - - const short tx = tiisg%4; - const short ty = tiisg/4; - - qk8x8_t mqk = make_filled_simdgroup_matrix((qk_t) 0.0f); - - for (short ii = 0; ii < DK16; ii += 4) { - device const kd4x4_t * pk4x4 = (device const kd4x4_t *) (k + ((ic + 8*cc + ty)*args.nb11)); - - if (DK16%4 == 0) { - // the head is evenly divisible by 4*16 = 64, so no need for bound checks - { - k4x4_t tmp; - deq_k(pk4x4 + (ii + tx)/nl_k, (ii + tx)%nl_k, tmp); - sk4x4[4*ty + tx] = tmp; - } - - simdgroup_barrier(mem_flags::mem_threadgroup); - - FOR_UNROLL (short k = 0; k < 4; ++k) { - k8x8_t mk; - q8x8_t mq; - - simdgroup_load(mk, sk + 16*k + 0*8, 4*16, 0, true); // transpose - simdgroup_load(mq, sq + (2*(ii + k) + 0)*8, DK); - simdgroup_multiply_accumulate(mqk, mq, mk, mqk); - - simdgroup_load(mk, sk + 16*k + 1*8, 4*16, 0, true); // transpose - simdgroup_load(mq, sq + (2*(ii + k) + 1)*8, DK); - simdgroup_multiply_accumulate(mqk, mq, mk, mqk); - } - } else { - if (ii + tx < DK16) { - k4x4_t tmp; - deq_k(pk4x4 + (ii + tx)/nl_k, (ii + tx)%nl_k, tmp); - sk4x4[4*ty + tx] = tmp; - } - - simdgroup_barrier(mem_flags::mem_threadgroup); - - for (short k = 0; k < 4 && ii + k < DK16; ++k) { - k8x8_t mk; - q8x8_t mq; - - simdgroup_load(mk, sk + 16*k + 0*8, 4*16, 0, true); // transpose - simdgroup_load(mq, sq + (2*(ii + k) + 0)*8, DK); - simdgroup_multiply_accumulate(mqk, mq, mk, mqk); - - simdgroup_load(mk, sk + 16*k + 1*8, 4*16, 0, true); // transpose - simdgroup_load(mq, sq + (2*(ii + k) + 1)*8, DK); - simdgroup_multiply_accumulate(mqk, mq, mk, mqk); - } - } - } - - simdgroup_store(mqk, ss + 8*cc, SH, 0, false); - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - // online softmax - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - const float m = M[jj]; - - // scale and apply the logitcap / mask - float2 s2 = ss2[j*SH/2 + tiisg]*args.scale; - - if (FC_flash_attn_ext_has_scap) { - s2 = args.logit_softcap*precise::tanh(s2); - } - - // mqk = mqk + slope*mask - if (blk_cur != 2) { - if (FC_flash_attn_ext_has_bias) { - s2 += s2_t(sm2[j*SH + tiisg])*slope; - } else { - s2 += s2_t(sm2[j*SH + tiisg]); - } - } - - M[jj] = simd_max(max(M[jj], max(s2[0], s2[1]))); - - const float ms = exp(m - M[jj]); - const float2 vs2 = exp(s2 - M[jj]); - - S[jj] = S[jj]*ms + simd_sum(vs2[0] + vs2[1]); - - // the P matrix from the paper (Q rows, C columns) - ss2[j*SH/2 + tiisg] = vs2; - - if (DV4 % NW == 0) { - FOR_UNROLL (short ii = 0; ii < DV4/NW; ++ii) { - const short i = ii*NW + tiisg; - - so4[j*PV4 + i] *= ms; - } - } else { - for (short i = tiisg; i < DV4; i += NW) { - so4[j*PV4 + i] *= ms; - } - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - // O = O + (Q*K^T)*V - { - // we can read directly from global memory - if (is_same::value) { - static_assert(PV8 % NSG == 0, ""); - - constexpr short NO = PV8/NSG; - - o8x8_t lo[NO]; - - { - auto sot = so + 8*sgitg; - - FOR_UNROLL (short ii = 0; ii < NO; ++ii) { - simdgroup_load(lo[ii], sot, PV, 0, false); - - sot += 8*NSG; - } - } - - { - device const v_t * pv = (device const v_t *) (v + ic*args.nb21); - - pv += 8*sgitg; - - if (DV <= 64) { - FOR_UNROLL (short cc = 0; cc < C/8; ++cc) { - s8x8_t vs; - simdgroup_load(vs, ss + 8*cc, SH, 0, false); - - FOR_UNROLL (short ii = 0; ii < NO/2; ++ii) { - v8x8_t mv[2]; - - simdgroup_load(mv[0], pv + 0*NSG + 16*ii*NSG, NS20, 0, false); - simdgroup_load(mv[1], pv + 8*NSG + 16*ii*NSG, NS20, 0, false); - - simdgroup_multiply_accumulate(lo[2*ii + 0], vs, mv[0], lo[2*ii + 0]); - simdgroup_multiply_accumulate(lo[2*ii + 1], vs, mv[1], lo[2*ii + 1]); - } - - pv += 8*NS20; - } - } else { - constexpr short NC = (C/8)/2; - - FOR_UNROLL (short cc = 0; cc < NC; ++cc) { - s8x8_t vs[2]; - - simdgroup_load(vs[0], ss + 16*cc + 0, SH, 0, false); - simdgroup_load(vs[1], ss + 16*cc + 8, SH, 0, false); - - FOR_UNROLL (short ii = 0; ii < NO/2; ++ii) { - v8x8_t mv[4]; - - simdgroup_load(mv[0], pv + 0*NSG + 16*ii*NSG + 0*8*NS20, NS20, 0, false); - simdgroup_load(mv[1], pv + 8*NSG + 16*ii*NSG + 0*8*NS20, NS20, 0, false); - simdgroup_load(mv[2], pv + 0*NSG + 16*ii*NSG + 1*8*NS20, NS20, 0, false); - simdgroup_load(mv[3], pv + 8*NSG + 16*ii*NSG + 1*8*NS20, NS20, 0, false); - - simdgroup_multiply_accumulate(lo[2*ii + 0], vs[0], mv[0], lo[2*ii + 0]); - simdgroup_multiply_accumulate(lo[2*ii + 1], vs[0], mv[1], lo[2*ii + 1]); - simdgroup_multiply_accumulate(lo[2*ii + 0], vs[1], mv[2], lo[2*ii + 0]); - simdgroup_multiply_accumulate(lo[2*ii + 1], vs[1], mv[3], lo[2*ii + 1]); - } - - pv += 2*8*NS20; - } - } - } - - { - auto sot = so + 8*sgitg; - - FOR_UNROLL (short ii = 0; ii < NO; ++ii) { - simdgroup_store(lo[ii], sot, PV, 0, false); - - sot += 8*NSG; - } - } - } else { - // TODO: this is the quantized V cache branch - not optimized yet - - const short tx = tiisg%4; - const short ty = tiisg/4; - - for (short cc = 0; cc < C/8; ++cc) { - s8x8_t vs; - simdgroup_load(vs, ss + 8*cc, SH, 0, false); - - for (short ii = 4*sgitg; ii < DV16; ii += 4*NSG) { - device const vd4x4_t * pv4x4 = (device const vd4x4_t *) (v + ((ic + 8*cc + ty)*args.nb21)); - - if (DV16%4 == 0) { - // no need for bound checks - { - v4x4_t tmp; - deq_v(pv4x4 + (ii + tx)/nl_v, (ii + tx)%nl_v, tmp); - sv4x4[4*ty + tx] = tmp; - } - - simdgroup_barrier(mem_flags::mem_threadgroup); - - FOR_UNROLL (short k = 0; k < 4; ++k) { - v8x8_t mv[2]; - o8x8_t lo[2]; - - simdgroup_load(mv[0], sv + 16*k + 0*8, 4*16, 0, false); - simdgroup_load(mv[1], sv + 16*k + 1*8, 4*16, 0, false); - simdgroup_load(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false); - simdgroup_load(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false); - - simdgroup_multiply_accumulate(lo[0], vs, mv[0], lo[0]); - simdgroup_multiply_accumulate(lo[1], vs, mv[1], lo[1]); - - simdgroup_store(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false); - simdgroup_store(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false); - } - } else { - if (ii + tx < DV16) { - v4x4_t tmp; - deq_v(pv4x4 + (ii + tx)/nl_v, (ii + tx)%nl_v, tmp); - sv4x4[4*ty + tx] = tmp; - } - - simdgroup_barrier(mem_flags::mem_threadgroup); - - for (short k = 0; k < 4 && ii + k < DV16; ++k) { - v8x8_t mv[2]; - o8x8_t lo[2]; - - simdgroup_load(mv[0], sv + 16*k + 0*8, 4*16, 0, false); - simdgroup_load(mv[1], sv + 16*k + 1*8, 4*16, 0, false); - simdgroup_load(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false); - simdgroup_load(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false); - - simdgroup_multiply_accumulate(lo[0], vs, mv[0], lo[0]); - simdgroup_multiply_accumulate(lo[1], vs, mv[1], lo[1]); - - simdgroup_store(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false); - simdgroup_store(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false); - } - } - } - } - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - if (FC_flash_attn_ext_has_sinks) { - FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - - const float m = M[jj]; - const float s = tiisg == 0 ? ((device const float *) sinks)[iq2] : -FLT_MAX/2; - - M[jj] = simd_max(max(M[jj], s)); - - const float ms = exp(m - M[jj]); - const float vs = exp(s - M[jj]); - - S[jj] = S[jj]*ms + simd_sum(vs); - - for (short i = tiisg; i < DV4; i += NW) { - so4[j*PV4 + i] *= ms; - } - } - } - } - - // store to global memory - for (short jj = 0; jj < NQ; ++jj) { - const short j = jj*NSG + sgitg; - if (iq1 + j >= args.ne01) { - break; - } - - device float4 * dst4 = (device float4 *) dst + ((uint64_t)iq3*args.ne2*args.ne1 + iq2 + (uint64_t)(iq1 + j)*args.ne1)*DV4; - - const float scale = S[jj] == 0.0 ? 0.0f : 1.0f/S[jj]; - - if (DV4 % NW == 0) { - FOR_UNROLL (short ii = 0; ii < DV4/NW; ++ii) { - const short i = ii*NW + tiisg; - - dst4[i] = (float4) so4[j*PV4 + i]*scale; - } - } else { - for (short i = tiisg; i < DV4; i += NW) { - dst4[i] = (float4) so4[j*PV4 + i]*scale; - } - } - } - -#undef NS10 -#undef NS20 -} - -template< - typename q_t, // query types in shared memory - typename q4_t, - typename q8x8_t, - typename k_t, // key types in shared memory - typename k4x4_t, - typename k8x8_t, - typename v_t, // value types in shared memory - typename v4x4_t, - typename v8x8_t, - typename qk_t, // Q*K types - typename qk8x8_t, - typename s_t, // soft-max types - typename s2_t, - typename s8x8_t, - typename o_t, // attention accumulation types - typename o4_t, - typename o8x8_t, - typename kd4x4_t, // key type in device memory - short nl_k, - void (*deq_k)(device const kd4x4_t *, short, thread k4x4_t &), - typename vd4x4_t, // value type in device memory - short nl_v, - void (*deq_v)(device const vd4x4_t *, short, thread v4x4_t &), - short DK, // K head size - short DV, // V head size - short Q = OP_FLASH_ATTN_EXT_NQPSG, // queries per threadgroup - short C = OP_FLASH_ATTN_EXT_NCPSG> // cache items per threadgroup -kernel void kernel_flash_attn_ext( - constant ggml_metal_kargs_flash_attn_ext & args, - device const char * q, - device const char * k, - device const char * v, - device const char * mask, - device const char * sinks, - device const char * pad, - device const char * blk, - device char * dst, - threadgroup half * shmem_f16 [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { -#define FWD_TMPL q_t, q4_t, q8x8_t, k_t, k4x4_t, k8x8_t, v_t, v4x4_t, v8x8_t, qk_t, qk8x8_t, s_t, s2_t, s8x8_t, o_t, o4_t, o8x8_t, kd4x4_t, nl_k, deq_k, vd4x4_t, nl_v, deq_v, DK, DV, Q, C -#define FWD_ARGS args, q, k, v, mask, sinks, pad, blk, dst, shmem_f16, tgpig, tiisg, sgitg - switch (FC_flash_attn_ext_nsg) { - // note: disabled cases to reduce library load time - //case 1: kernel_flash_attn_ext_impl(FWD_ARGS); break; - //case 2: kernel_flash_attn_ext_impl(FWD_ARGS); break; - case 4: kernel_flash_attn_ext_impl(FWD_ARGS); break; - case 8: kernel_flash_attn_ext_impl(FWD_ARGS); break; - } -#undef FWD_TMPL -#undef FWD_ARGS -} - -// TODO: this is quite ugly. in the future these types will be hardcoded in the kernel, but for now keep them as -// template to be able to explore different combinations -// -#define FA_TYPES \ - half, half4, simdgroup_half8x8, \ - half, half4x4, simdgroup_half8x8, \ - half, half4x4, simdgroup_half8x8, \ - float, simdgroup_float8x8, \ - float, float2, simdgroup_float8x8, \ - float, float4, simdgroup_float8x8 - //half, half4, simdgroup_half8x8 - -#define FA_TYPES_BF \ - bfloat, bfloat4, simdgroup_bfloat8x8, \ - bfloat, bfloat4x4, simdgroup_bfloat8x8, \ - bfloat, bfloat4x4, simdgroup_bfloat8x8, \ - float, simdgroup_float8x8, \ - float, float2, simdgroup_float8x8, \ - half, half4, simdgroup_half8x8 - //float, float4, simdgroup_float8x8 - -#define FA_TYPES_F32 \ - half, half4, simdgroup_half8x8, \ - float, float4x4, simdgroup_float8x8, \ - float, float4x4, simdgroup_float8x8, \ - float, simdgroup_float8x8, \ - float, float2, simdgroup_float8x8, \ - float, float4, simdgroup_float8x8 - //half, half4, simdgroup_half8x8 - -typedef decltype(kernel_flash_attn_ext) flash_attn_ext_t; - -template [[host_name("kernel_flash_attn_ext_f32_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f32_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; - -template [[host_name("kernel_flash_attn_ext_f16_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_f16_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; - -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_bf16_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_bf16_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -#endif - -template [[host_name("kernel_flash_attn_ext_q4_0_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_0_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; - -template [[host_name("kernel_flash_attn_ext_q4_1_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q4_1_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; - -template [[host_name("kernel_flash_attn_ext_q5_0_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_0_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; - -template [[host_name("kernel_flash_attn_ext_q5_1_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q5_1_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; - -template [[host_name("kernel_flash_attn_ext_q8_0_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; -template [[host_name("kernel_flash_attn_ext_q8_0_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; - -#undef FA_TYPES -#undef FA_TYPES_BF -#undef FA_TYPES_F32 - -constant bool FC_flash_attn_ext_vec_has_mask [[function_constant(FC_FLASH_ATTN_EXT_VEC + 0)]]; -constant bool FC_flash_attn_ext_vec_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_VEC + 1)]]; -constant bool FC_flash_attn_ext_vec_has_bias [[function_constant(FC_FLASH_ATTN_EXT_VEC + 2)]]; -constant bool FC_flash_attn_ext_vec_has_scap [[function_constant(FC_FLASH_ATTN_EXT_VEC + 3)]]; -constant bool FC_flash_attn_ext_vec_has_kvpad [[function_constant(FC_FLASH_ATTN_EXT_VEC + 4)]]; - -//constant float FC_flash_attn_ext_vec_scale [[function_constant(FC_FLASH_ATTN_EXT_VEC + 10)]]; -//constant float FC_flash_attn_ext_vec_max_bias [[function_constant(FC_FLASH_ATTN_EXT_VEC + 11)]]; -//constant float FC_flash_attn_ext_vec_logit_softcap [[function_constant(FC_FLASH_ATTN_EXT_VEC + 12)]]; - -constant int32_t FC_flash_attn_ext_vec_ns10 [[function_constant(FC_FLASH_ATTN_EXT_VEC + 20)]]; -constant int32_t FC_flash_attn_ext_vec_ns20 [[function_constant(FC_FLASH_ATTN_EXT_VEC + 21)]]; -constant int32_t FC_flash_attn_ext_vec_nsg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 22)]]; -constant int32_t FC_flash_attn_ext_vec_nwg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 23)]]; - -template< - typename q4_t, // query types in shared memory - typename k4_t, // key types in shared memory - typename v4_t, // value types in shared memory - typename qk_t, // Q*K types - typename s_t, // soft-max types - typename s4_t, - typename o4_t, // attention accumulation types - typename kd4_t, // key type in device memory - short nl_k, - void (*deq_k_t4)(device const kd4_t *, short, thread k4_t &), - typename vd4_t, // value type in device memory - short nl_v, - void (*deq_v_t4)(device const vd4_t *, short, thread v4_t &), - short DK, // K head size - short DV, // V head size - short NE = 4, // head elements per thread - short Q = OP_FLASH_ATTN_EXT_VEC_NQPSG, // queries per threadgroup - short C = OP_FLASH_ATTN_EXT_VEC_NCPSG> // cache items per threadgroup -kernel void kernel_flash_attn_ext_vec( - constant ggml_metal_kargs_flash_attn_ext_vec & args, - device const char * q, - device const char * k, - device const char * v, - device const char * mask, - device const char * sinks, - device const char * pad, - device char * dst, - threadgroup half * shmem_f16 [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - static_assert(DK % 32 == 0, "DK must be divisible by 32"); - static_assert(DV % 32 == 0, "DV must be divisible by 32"); - -#define NWG (FC_flash_attn_ext_vec_nwg) -#define NSG (FC_flash_attn_ext_vec_nsg) - -#define NS10 (FC_flash_attn_ext_vec_ns10) -#define NS20 (FC_flash_attn_ext_vec_ns20) - - const short iwg = tgpig[2]%NWG; - - const ushort iq3 = tgpig[2]/NWG; - const ushort iq2 = tgpig[1]; - const ushort iq1 = tgpig[0]; - - constexpr short DK4 = DK/4; - constexpr short DV4 = DV/4; - - constexpr short PK = PAD2(DK, 128); - constexpr short PK4 = PK/4; - - constexpr short PV = PAD2(DV, 128); - constexpr short PV4 = PV/4; - - constexpr short NW = N_SIMDWIDTH; - constexpr short NL = NW/NE; // note: this can be adjusted to support different head sizes and simdgroup work loads - constexpr short SH = 4*C; // shared memory per simdgroup - - static_assert(DK4 % NL == 0, "DK4 must be divisible by NL"); - static_assert(DV4 % NL == 0, "DV4 must be divisible by NL"); - - //const short T = PK + NSG*SH; // shared memory size per query in (half) - - //threadgroup q_t * sq = (threadgroup q_t *) (shmem_f16 + 0*PK); // holds the query data - threadgroup q4_t * sq4 = (threadgroup q4_t *) (shmem_f16 + 0*PK); // same as above but in q4_t - threadgroup s_t * ss = (threadgroup s_t *) (shmem_f16 + sgitg*SH + NSG*PK); // scratch buffer for attention - threadgroup s4_t * ss4 = (threadgroup s4_t *) (shmem_f16 + sgitg*SH + NSG*PK); // same as above but in s4_t - threadgroup half * sm = (threadgroup half *) (shmem_f16 + sgitg*SH + 2*C + NSG*PK); // scratch buffer for mask - threadgroup o4_t * so4 = (threadgroup o4_t *) (shmem_f16 + 2*sgitg*PV + NSG*PK + NSG*SH); // scratch buffer for the results - - // store the result for all queries in shared memory (the O matrix from the paper) - so4 += tiisg; - - { - q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03; - - const short ikv2 = iq2/(args.ne02/args.ne_12_2); - const short ikv3 = iq3/(args.ne03/args.ne_12_3); - - k += ikv2*args.nb12 + ikv3*args.nb13; - v += ikv2*args.nb22 + ikv3*args.nb23; - } - - // load heads from Q to shared memory - device const float4 * q4 = (device const float4 *) ((device const char *) q); - - if (iq1 < args.ne01) { - for (short i = tiisg; i < PK4; i += NW) { - if (i < DK4) { - sq4[i] = (q4_t) q4[i]; - } else { - sq4[i] = (q4_t) 0.0f; - } - } - } - - // zero out so - for (short i = 0; i < DV4/NL; ++i) { - so4[i*NL] = (o4_t) 0.0f; - } - - // zero out shared memory SH - for (short i = tiisg; i < SH/4; i += NW) { - ss4[i] = (s4_t) 0.0f; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - { - float S = 0.0f; - float M = -FLT_MAX/2; - - // thread indices inside the simdgroup - const short tx = tiisg%NL; - const short ty = tiisg/NL; - - // pointer to the mask - device const half * pm = (device const half *) (mask + iq1*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33); - - float slope = 1.0f; - - // ALiBi - if (FC_flash_attn_ext_vec_has_bias) { - const short h = iq2; - - const float base = h < args.n_head_log2 ? args.m0 : args.m1; - const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; - - slope = pow(base, exph); - } - - // loop over the KV cache - // each simdgroup handles blocks of Q rows and C columns - for (int ic0 = iwg*NSG + sgitg; ; ic0 += NWG*NSG) { - int ic = ic0*C; - if (ic >= args.ne11) { - break; - } - - // the last partial chunk uses the pad buffer as source - if (FC_flash_attn_ext_vec_has_kvpad && ic + C > args.ne11) { - k = pad; - v = k + args.nb11*C*args.ne_12_2*args.ne_12_3; - mask = v + args.nb21*C*args.ne_12_2*args.ne_12_3; - - const short ikv2 = iq2/(args.ne02/args.ne_12_2); - const short ikv3 = iq3/(args.ne03/args.ne_12_3); - - k += (ikv2 + ikv3*args.ne_12_2)*args.nb11*C; - v += (ikv2 + ikv3*args.ne_12_2)*args.nb21*C; - - if (!FC_flash_attn_ext_vec_has_mask) { - if (ic + tiisg >= args.ne11) { - sm[tiisg] = -MAXHALF; - } - } else { - pm = (device const half *) (mask) + - iq1*C + - (iq2%args.ne32)*(C*args.ne31) + - (iq3%args.ne33)*(C*args.ne31*args.ne32); - } - - ic = 0; - } - - if (FC_flash_attn_ext_vec_has_mask) { - sm[tiisg] = pm[ic + tiisg]; - } - - // skip -INF blocks - if (simd_max(sm[tiisg]) <= -MAXHALF) { - continue; - } - - // Q*K^T - { - device const k4_t * pk4 = (device const k4_t *) (k + ic*args.nb11); - threadgroup const q4_t * pq4 = sq4; - - pk4 += ty*NS10/4 + tx; - pq4 += tx; - - qk_t mqk[C/NE] = { [ 0 ... C/NE - 1] = 0.0f }; - - // each simdgroup processes 1 query and NE (NW/NL) cache elements - FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { - if (is_same::value) { - FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) { - mqk[cc] += dot((float4) pk4[cc*NE*NS10/4 + ii*NL], (float4) pq4[ii*NL]); - } - } else { - device const kd4_t * pk = (device const kd4_t *) (k + ((ic + NE*cc + ty)*args.nb11)); - - k4_t mk; - - FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) { - const short i = ii*NL + tx; - - deq_k_t4(pk + i/nl_k, i%nl_k, mk); - - mqk[cc] += dot((float4) mk, (float4) sq4[i]); - } - } - - if (NE == 1) { - mqk[cc] = simd_sum(mqk[cc]); - } else { - // simdgroup reduce (NE = 4) - // [ 0 .. 7] -> [ 0] - // [ 8 .. 15] -> [ 8] - // [16 .. 23] -> [16] - // [24 .. 31] -> [24] - if (NE <= 1) { - mqk[cc] += simd_shuffle_down(mqk[cc], 16); - } - if (NE <= 2) { - mqk[cc] += simd_shuffle_down(mqk[cc], 8); - } - if (NE <= 4) { - mqk[cc] += simd_shuffle_down(mqk[cc], 4); - } - if (NE <= 8) { - mqk[cc] += simd_shuffle_down(mqk[cc], 2); - } - if (NE <= 16) { - mqk[cc] += simd_shuffle_down(mqk[cc], 1); - } - - // broadcast - mqk[cc] = simd_shuffle(mqk[cc], NL*ty); - } - } - - if (FC_flash_attn_ext_vec_has_mask && - !FC_flash_attn_ext_vec_has_scap && - !FC_flash_attn_ext_vec_has_bias) { - ss[NE*tx + ty] = fma(mqk[tx], args.scale, (qk_t) sm[NE*tx + ty]); - } else { - mqk[tx] *= args.scale; - - if (FC_flash_attn_ext_vec_has_scap) { - mqk[tx] = args.logit_softcap*precise::tanh(mqk[tx]); - } - - if (FC_flash_attn_ext_vec_has_bias) { - mqk[tx] += (qk_t) sm[NE*tx + ty]*slope; - } else { - mqk[tx] += (qk_t) sm[NE*tx + ty]; - } - - ss[NE*tx + ty] = mqk[tx]; - } - } - - simdgroup_barrier(mem_flags::mem_threadgroup); - - // online softmax - { - const float m = M; - const float s = ss[tiisg]; - - M = simd_max(max(M, s)); - - const float ms = exp(m - M); - const float vs = exp(s - M); - - S = S*ms + simd_sum(vs); - - // the P matrix from the paper (Q rows, C columns) - ss[tiisg] = vs; - - // O = diag(ms)*O - if ((DV4/NL % NW == 0) || ty == 0) { - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - so4[ii*NL] *= ms; - } - } - } - - simdgroup_barrier(mem_flags::mem_threadgroup); - - // O = O + (Q*K^T)*V - { - o4_t lo[DV4/NL]; - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - lo[ii] = 0.0f; - } - - if (is_same::value) { - device const v4_t * pv4 = (device const v4_t *) (v + ic*args.nb21); - - pv4 += ty*NS20/4 + tx; - - const auto sst = ss + ty; - - FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - lo[ii] += o4_t(float4(pv4[cc*NE*NS20/4 + ii*NL])*float4(sst[cc*NE])); - } - } - } else { - FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { - device const vd4_t * pv4 = (device const vd4_t *) (v + ((ic + NE*cc + ty)*args.nb21)); - - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - const short i = ii*NL + tx; - - v4_t mv; - deq_v_t4(pv4 + i/nl_v, i%nl_v, mv); - - lo[ii] += o4_t(float4(mv)*float4(ss[NE*cc + ty])); - } - } - } - - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - if (NE > 1) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 16); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 16); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 16); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 16); - } - - if (NE > 2) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 8); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 8); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 8); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 8); - } - - if (NE > 4) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 4); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 4); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 4); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 4); - } - - if (NE > 8) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 2); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 2); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 2); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 2); - } - - if (NE > 16) { - lo[ii][0] += simd_shuffle_down(lo[ii][0], 1); - lo[ii][1] += simd_shuffle_down(lo[ii][1], 1); - lo[ii][2] += simd_shuffle_down(lo[ii][2], 1); - lo[ii][3] += simd_shuffle_down(lo[ii][3], 1); - } - } - - if ((DV4/NL % NW == 0) || ty == 0) { - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - so4[ii*NL] += lo[ii]; - } - } - } - } - - if (FC_flash_attn_ext_vec_has_sinks && sgitg == 0 && iwg == 0) { - const float m = M; - const float s = tiisg == 0 ? ((device const float *) sinks)[iq2] : -FLT_MAX/2; - - M = simd_max(max(M, s)); - - const float ms = exp(m - M); - const float vs = exp(s - M); - - S = S*ms + simd_sum(vs); - - if ((DV4/NL % NW == 0) || ty == 0) { - FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { - so4[ii*NL] *= ms; - } - } - } - - // these are needed for reducing the results from the simdgroups (reuse the ss buffer) - if (tiisg == 0) { - ss[0] = (s_t) S; - ss[1] = (s_t) M; - } - } - - so4 -= tiisg; - - threadgroup_barrier(mem_flags::mem_threadgroup); - - // parallel reduce - for (short r = NSG/2; r > 0; r >>= 1) { - if (sgitg < r) { - const float S0 = ss[ 0]; - const float S1 = ss[r*(SH/2) + 0]; - - const float M0 = ss[ 1]; - const float M1 = ss[r*(SH/2) + 1]; - - const float M = max(M0, M1); - - const float ms0 = exp(M0 - M); - const float ms1 = exp(M1 - M); - - const float S = S0*ms0 + S1*ms1; - - if (tiisg == 0) { - ss[0] = S; - ss[1] = M; - } - - // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1 - for (short i = tiisg; i < DV4; i += NW) { - so4[i] = so4[i]*ms0 + so4[i + r*PV4]*ms1; - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - // final rescale with 1/S and store to global memory - if (sgitg == 0) { - const int64_t nrows = args.ne3*args.ne2*args.ne1; - const int64_t rid = iq3*args.ne2*args.ne1 + iq2 + iq1*args.ne1; - - device float4 * dst4 = (device float4 *) dst; - device float * dst1 = (device float *) dst + nrows*DV*NWG; // the S and M are stored after the results - - const float S = NWG == 1 ? (ss[0] == 0.0f ? 0.0f : 1.0f/ss[0]) : 1.0f; - - // interleave the workgroup data - for (short i = tiisg; i < DV4; i += NW) { - dst4[rid*DV4*NWG + NWG*i + iwg] = (float4) so4[i]*S; - } - - // store S and M - if (NWG > 1) { - if (tiisg == 0) { - dst1[rid*(2*NWG) + 2*iwg + 0] = ss[0]; - dst1[rid*(2*NWG) + 2*iwg + 1] = ss[1]; - } - } - } - -#undef NWG -#undef NSG -#undef NS10 -#undef NS20 -} - -// note: I think the s_t can be half instead of float, because the Q*K scaling is done before storing to shared mem -// in the other (non-vec) kernel, we need s_t to also be float because we scale during the soft_max -// -#define FA_TYPES \ - half4, \ - half4, \ - half4, \ - float, \ - float, float4, \ - float4 - -#define FA_TYPES_F32 \ - half4, \ - float4, \ - float4, \ - float, \ - float, float4, \ - float4 - -typedef decltype(kernel_flash_attn_ext_vec) flash_attn_ext_vec_t; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -template [[host_name("kernel_flash_attn_ext_vec_f32_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_flash_attn_ext_vec_bf16_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -#endif -template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; -template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; - -#undef FA_TYPES -#undef FA_TYPES_F32 - -constant int32_t FC_flash_attn_ext_vec_reduce_DV [[function_constant(FC_FLASH_ATTN_EXT_VEC_REDUCE + 0)]]; -constant int32_t FC_flash_attn_ext_vec_reduce_NWG [[function_constant(FC_FLASH_ATTN_EXT_VEC_REDUCE + 1)]]; - -kernel void kernel_flash_attn_ext_vec_reduce( - constant ggml_metal_kargs_flash_attn_ext_vec_reduce & args, - device const char * htmp, - device char * dst, - uint tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { -#define NWG (FC_flash_attn_ext_vec_reduce_NWG) -#define DV (FC_flash_attn_ext_vec_reduce_DV) - - const uint64_t rid = tgpig; - - const short iwg = tiisg; - - device const float * ss = (device const float *) htmp + (uint64_t)args.nrows*DV*NWG; - - float S = ss[rid*(2*NWG) + 2*iwg + 0]; - float M = ss[rid*(2*NWG) + 2*iwg + 1]; - - const float m = simd_max(M); - const float ms = exp(M - m); - - S = simd_sum(S*ms); - S = S == 0.0f ? 0.0f : 1.0f/S; - - const short DV4 = DV/4; - - device const float4 * htmp4 = (device const float4 *) htmp + rid*DV4*NWG; - device float4 * dst4 = (device float4 *) dst + rid*DV4; - - for (short i = sgitg; i < DV4; i += NWG) { - const float4 v = simd_sum(htmp4[i*NWG + iwg]*ms); - - if (iwg == 0) { - dst4[i] = v*S; - } - } - -#undef NWG -#undef DV -} - -template -kernel void kernel_cpy_t_t( - constant ggml_metal_kargs_cpy & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int i03 = tgpig[2]; - const int i02 = tgpig[1]; - const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0]; - const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; - - const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00; - - const int64_t i3 = n/(args.ne2*args.ne1*args.ne0); - const int64_t i2 = (n - i3*args.ne2*args.ne1*args.ne0)/(args.ne1*args.ne0); - const int64_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0; - const int64_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0); - - device T1 * dst_data = (device T1 *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - for (int64_t i00 = iw0*ntg[0] + tiitg%ntg[0]; i00 < args.ne00; ) { - device const T0 * src = (device T0 *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00); - dst_data[i00] = (T1) src[0]; - break; - } -} - -typedef decltype(kernel_cpy_t_t) kernel_cpy_t; - -template [[host_name("kernel_cpy_f32_f32")]] kernel kernel_cpy_t kernel_cpy_t_t; -template [[host_name("kernel_cpy_f32_f16")]] kernel kernel_cpy_t kernel_cpy_t_t; -template [[host_name("kernel_cpy_f32_i32")]] kernel kernel_cpy_t kernel_cpy_t_t; -template [[host_name("kernel_cpy_i32_f32")]] kernel kernel_cpy_t kernel_cpy_t_t; -template [[host_name("kernel_cpy_i32_i32")]] kernel kernel_cpy_t kernel_cpy_t_t; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_cpy_f32_bf16")]] kernel kernel_cpy_t kernel_cpy_t_t; -#endif -template [[host_name("kernel_cpy_f16_f32")]] kernel kernel_cpy_t kernel_cpy_t_t; -template [[host_name("kernel_cpy_f16_f16")]] kernel kernel_cpy_t kernel_cpy_t_t; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_cpy_bf16_f32")]] kernel kernel_cpy_t kernel_cpy_t_t; -template [[host_name("kernel_cpy_bf16_bf16")]] kernel kernel_cpy_t kernel_cpy_t_t; -#endif - -template -kernel void kernel_cpy_f32_q( - constant ggml_metal_kargs_cpy & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int i03 = tgpig[2]; - const int i02 = tgpig[1]; - const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0]; - const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; - - const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00; - - const int64_t i3 = n / (args.ne2*args.ne1*args.ne0); - const int64_t i2 = (n - i3*args.ne2*args.ne1*args.ne0) / (args.ne1*args.ne0); - const int64_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0) / args.ne0; - const int64_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0)/QK; - - device block_q * dst_data = (device block_q *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - for (int64_t i00 = iw0*ntg[0] + tiitg%ntg[0]; i00 < args.nk0; ) { - device const float * src = (device const float *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + (i00*QK)*args.nb00); - - quantize_func(src, dst_data[i00]); - - break; - } -} - -typedef decltype(kernel_cpy_f32_q) cpy_f_q_t; - -template [[host_name("kernel_cpy_f32_q8_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; -template [[host_name("kernel_cpy_f32_q1_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; -template [[host_name("kernel_cpy_f32_q4_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; -template [[host_name("kernel_cpy_f32_q4_1")]] kernel cpy_f_q_t kernel_cpy_f32_q; -template [[host_name("kernel_cpy_f32_q5_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; -template [[host_name("kernel_cpy_f32_q5_1")]] kernel cpy_f_q_t kernel_cpy_f32_q; -template [[host_name("kernel_cpy_f32_iq4_nl")]] kernel cpy_f_q_t kernel_cpy_f32_q; - -template -kernel void kernel_cpy_q_f32( - constant ggml_metal_kargs_cpy & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const int i03 = tgpig[2]; - const int i02 = tgpig[1]; - const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0]; - const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; - - const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00; - - const int64_t i3 = n/(args.ne2*args.ne1*args.ne0); - const int64_t i2 = (n - i3*args.ne2*args.ne1*args.ne0)/(args.ne1*args.ne0); - const int64_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0; - const int64_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0); - - device const block_q * src_data = (device const block_q *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); - device T4x4 * dst_data = (device T4x4 *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - for (int64_t i00 = iw0*ntg[0] + tiitg%ntg[0]; i00 < args.nk0; ) { - T4x4 temp; - dequantize_func(src_data + i00/nl, i00%nl, temp); - dst_data[i00] = temp; - - break; - } -} - -typedef decltype(kernel_cpy_q_f32) cpy_q_f_t; - -template [[host_name("kernel_cpy_q1_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q4_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q4_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q5_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q5_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q8_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; - -template [[host_name("kernel_cpy_q1_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q4_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q4_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q5_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q5_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; -template [[host_name("kernel_cpy_q8_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; - -kernel void kernel_concat( - constant ggml_metal_kargs_concat & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - - const int i3 = tgpig.z; - const int i2 = tgpig.y; - const int i1 = tgpig.x; - - int o[4] = {0, 0, 0, 0}; - o[args.dim] = args.dim == 0 ? args.ne00 : (args.dim == 1 ? args.ne01 : (args.dim == 2 ? args.ne02 : args.ne03)); - - device const float * x; - - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { - if (i0 < args.ne00 && i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) { - x = (device const float *)(src0 + (i3 )*args.nb03 + (i2 )*args.nb02 + (i1 )*args.nb01 + (i0 )*args.nb00); - } else { - x = (device const float *)(src1 + (i3 - o[3])*args.nb13 + (i2 - o[2])*args.nb12 + (i1 - o[1])*args.nb11 + (i0 - o[0])*args.nb10); - } - - device float * y = (device float *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); - - *y = *x; - } -} - -template -void kernel_mul_mv_q2_K_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_q2_K * x = (device const block_q2_K *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - float yl[32]; - float sumf[nr0]={0.f}; - - const short ix = tiisg/8; // 0...3 - const short it = tiisg%8; // 0...7 - const short iq = it/4; // 0 or 1 - const short ir = it%4; // 0...3 - const short is = (8*ir)/16;// 0 or 1 - - device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir; - - for (int ib = ix; ib < nb; ib += 4) { - float4 sumy = {0.f, 0.f, 0.f, 0.f}; - for (short i = 0; i < 8; ++i) { - yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; - yl[i+ 8] = y4[i+32]; sumy[1] += yl[i+ 8]; - yl[i+16] = y4[i+64]; sumy[2] += yl[i+16]; - yl[i+24] = y4[i+96]; sumy[3] += yl[i+24]; - } - - device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is; - device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir; - device const half * dh = &x[ib].d; - - for (short row = 0; row < nr0; row++) { - float4 acc1 = {0.f, 0.f, 0.f, 0.f}; - float4 acc2 = {0.f, 0.f, 0.f, 0.f}; - for (int i = 0; i < 8; i += 2) { - acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003); - acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300); - acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c); - acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00); - acc1[2] += yl[i+16] * (qs[i/2] & 0x0030); - acc2[2] += yl[i+17] * (qs[i/2] & 0x3000); - acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0); - acc2[3] += yl[i+25] * (qs[i/2] & 0xc000); - } - float dall = dh[0]; - float dmin = dh[1] * 1.f/16.f; - sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f + - (acc1[1] + 1.f/256.f * acc2[1]) * (sc[2] & 0xF) * 1.f/ 4.f + - (acc1[2] + 1.f/256.f * acc2[2]) * (sc[4] & 0xF) * 1.f/16.f + - (acc1[3] + 1.f/256.f * acc2[3]) * (sc[6] & 0xF) * 1.f/64.f) - - dmin * (sumy[0] * (sc[0] & 0xF0) + sumy[1] * (sc[2] & 0xF0) + sumy[2] * (sc[4] & 0xF0) + sumy[3] * (sc[6] & 0xF0)); - - qs += args.nb01/2; - sc += args.nb01; - dh += args.nb01/2; - } - - y4 += 4 * QK_K; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all; - } - } -} - -[[host_name("kernel_mul_mv_q2_K_f32")]] -kernel void kernel_mul_mv_q2_K_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_q2_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_q3_K_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_q3_K * x = (device const block_q3_K *) (src0 + offset0); - device const float * yy = (device const float *) (src1 + offset1); - - float yl[32]; - - //const uint16_t kmask1 = 0x3030; - //const uint16_t kmask2 = 0x0f0f; - - const short tid = tiisg/4; - const short ix = tiisg%4; - const short ip = tid/4; // 0 or 1 - const short il = 2*((tid%4)/2); // 0 or 2 - const short ir = tid%2; - const short l0 = 8*ir; - - // One would think that the Metal compiler would figure out that ip and il can only have - // 4 possible states, and optimize accordingly. Well, no. It needs help, and we do it - // with these two tales. - // - // Possible masks for the high bit - const ushort4 mm[4] = {{0x0001, 0x0100, 0x0002, 0x0200}, // ip = 0, il = 0 - {0x0004, 0x0400, 0x0008, 0x0800}, // ip = 0, il = 2 - {0x0010, 0x1000, 0x0020, 0x2000}, // ip = 1, il = 0 - {0x0040, 0x4000, 0x0080, 0x8000}}; // ip = 1, il = 2 - - // Possible masks for the low 2 bits - const int4 qm[2] = {{0x0003, 0x0300, 0x000c, 0x0c00}, {0x0030, 0x3000, 0x00c0, 0xc000}}; - - const ushort4 hm = mm[2*ip + il/2]; - - const short shift = 2*il; - - const float v1 = il == 0 ? 4.f : 64.f; - const float v2 = 4.f * v1; - - const uint16_t s_shift1 = 4*ip; - const uint16_t s_shift2 = s_shift1 + il; - - const short q_offset = 32*ip + l0; - const short y_offset = 128*ip + 32*il + l0; - - device const float * y1 = yy + ix*QK_K + y_offset; - - uint32_t scales32, aux32; - thread uint16_t * scales16 = (thread uint16_t *)&scales32; - thread const int8_t * scales = (thread const int8_t *)&scales32; - - float sumf1[nr0] = {0.f}; - float sumf2[nr0] = {0.f}; - - for (int i = ix; i < nb; i += 4) { - for (short l = 0; l < 8; ++l) { - yl[l+ 0] = y1[l+ 0]; - yl[l+ 8] = y1[l+16]; - yl[l+16] = y1[l+32]; - yl[l+24] = y1[l+48]; - } - - device const uint16_t * q = (device const uint16_t *)(x[i].qs + q_offset); - device const uint16_t * h = (device const uint16_t *)(x[i].hmask + l0); - device const uint16_t * a = (device const uint16_t *)(x[i].scales); - device const half * dh = &x[i].d; - - for (short row = 0; row < nr0; ++row) { - const float d_all = (float)dh[0]; - - scales16[0] = a[4]; - scales16[1] = a[5]; - aux32 = ((scales32 >> s_shift2) << 4) & 0x30303030; - scales16[0] = a[il+0]; - scales16[1] = a[il+1]; - scales32 = ((scales32 >> s_shift1) & 0x0f0f0f0f) | aux32; - - float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; - for (short l = 0; l < 8; l += 2) { - const int32_t qs = q[l/2]; - s1 += yl[l+0] * (qs & qm[il/2][0]); - s2 += yl[l+1] * (qs & qm[il/2][1]); - s3 += ((h[l/2] & hm[0]) ? 0.f : yl[l+0]) + ((h[l/2] & hm[1]) ? 0.f : yl[l+1]); - s4 += yl[l+16] * (qs & qm[il/2][2]); - s5 += yl[l+17] * (qs & qm[il/2][3]); - s6 += ((h[l/2] & hm[2]) ? 0.f : yl[l+16]) + ((h[l/2] & hm[3]) ? 0.f : yl[l+17]); - } - float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); - float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); - sumf1[row] += d1 * (scales[0] - 32); - sumf2[row] += d2 * (scales[2] - 32); - - s1 = s2 = s3 = s4 = s5 = s6 = 0; - for (short l = 0; l < 8; l += 2) { - const int32_t qs = q[l/2+8]; - s1 += yl[l+8] * (qs & qm[il/2][0]); - s2 += yl[l+9] * (qs & qm[il/2][1]); - s3 += ((h[l/2+8] & hm[0]) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm[1]) ? 0.f : yl[l+9]); - s4 += yl[l+24] * (qs & qm[il/2][2]); - s5 += yl[l+25] * (qs & qm[il/2][3]); - s6 += ((h[l/2+8] & hm[2]) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm[3]) ? 0.f : yl[l+25]); - } - d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); - d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); - sumf1[row] += d1 * (scales[1] - 32); - sumf2[row] += d2 * (scales[3] - 32); - - q += args.nb01/2; - h += args.nb01/2; - a += args.nb01/2; - dh += args.nb01/2; - } - - y1 += 4 * QK_K; - } - - for (int row = 0; row < nr0; ++row) { - const float sumf = (sumf1[row] + 0.25f * sumf2[row]) / (1 << shift); - sumf1[row] = simd_sum(sumf); - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - if (tiisg == 0) { - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - dst_f32[first_row + row] = sumf1[row]; - } - } -} - -[[host_name("kernel_mul_mv_q3_K_f32")]] -kernel void kernel_mul_mv_q3_K_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_q3_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_q4_K_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - constexpr uint16_t kmask1 = 0x3f3f; - constexpr uint16_t kmask2 = 0x0f0f; - constexpr uint16_t kmask3 = 0xc0c0; - - const short ix = tiisg/8; // 0...3 - const short it = tiisg%8; // 0...7 - const short iq = it/4; // 0 or 1 - const short ir = it%4; // 0...3 - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_q4_K * x = (device const block_q4_K *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - float yl[16]; - float yh[16]; - - float sumf[nr0]={0.f}; - - device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir; - - uint16_t sc16[4]; - thread const uint8_t * sc8 = (thread const uint8_t *)sc16; - - for (int ib = ix; ib < nb; ib += 4) { - float4 sumy = {0.f, 0.f, 0.f, 0.f}; - - for (short i = 0; i < 8; ++i) { - yl[i+0] = y4[i+ 0]; sumy[0] += yl[i+0]; - yl[i+8] = y4[i+ 32]; sumy[1] += yl[i+8]; - yh[i+0] = y4[i+128]; sumy[2] += yh[i+0]; - yh[i+8] = y4[i+160]; sumy[3] += yh[i+8]; - } - - device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq; - device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir; - device const half * dh = &x[ib].d; - - for (short row = 0; row < nr0; row++) { - sc16[0] = sc[0] & kmask1; - sc16[1] = sc[2] & kmask1; - sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2); - sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2); - - device const uint16_t * q2 = q1 + 32; - - float4 acc1 = {0.f, 0.f, 0.f, 0.f}; - float4 acc2 = {0.f, 0.f, 0.f, 0.f}; - - FOR_UNROLL (short i = 0; i < 4; ++i) { - acc1[0] += yl[2*i + 0] * (q1[i] & 0x000F); - acc1[1] += yl[2*i + 1] * (q1[i] & 0x0F00); - acc1[2] += yl[2*i + 8] * (q1[i] & 0x00F0); - acc1[3] += yl[2*i + 9] * (q1[i] & 0xF000); - acc2[0] += yh[2*i + 0] * (q2[i] & 0x000F); - acc2[1] += yh[2*i + 1] * (q2[i] & 0x0F00); - acc2[2] += yh[2*i + 8] * (q2[i] & 0x00F0); - acc2[3] += yh[2*i + 9] * (q2[i] & 0xF000); - } - - sumf[row] += dh[0] * ((acc1[0] + 1.f/256.f * acc1[1]) * sc8[0] + - (acc1[2] + 1.f/256.f * acc1[3]) * sc8[1] * 1.f/16.f + - (acc2[0] + 1.f/256.f * acc2[1]) * sc8[4] + - (acc2[2] + 1.f/256.f * acc2[3]) * sc8[5] * 1.f/16.f) - - dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]); - - q1 += args.nb01/2; - sc += args.nb01/2; - dh += args.nb01/2; - } - - y4 += 4 * QK_K; - } - - device float * dst_f32 = (device float *) dst + (int64_t)im*args.ne0*args.ne1 + (int64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all; - } - } -} - -[[host_name("kernel_mul_mv_q4_K_f32")]] -kernel void kernel_mul_mv_q4_K_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_q4_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_q5_K_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_q5_K * x = (device const block_q5_K *) (src0 + offset0); - device const float * yy = (device const float *) (src1 + offset1); - - float sumf[nr0]={0.f}; - - float yl[16], yh[16]; - - constexpr uint16_t kmask1 = 0x3f3f; - constexpr uint16_t kmask2 = 0x0f0f; - constexpr uint16_t kmask3 = 0xc0c0; - - const short tid = tiisg/4; - const short ix = tiisg%4; - const short iq = tid/4; - const short ir = tid%4; - - const short l0 = 8*ir; - const short q_offset = 32*iq + l0; - const short y_offset = 64*iq + l0; - - const uint8_t hm1 = 1u << (2*iq); - const uint8_t hm2 = hm1 << 1; - const uint8_t hm3 = hm1 << 4; - const uint8_t hm4 = hm2 << 4; - - uint16_t sc16[4]; - thread const uint8_t * sc8 = (thread const uint8_t *)sc16; - - device const float * y1 = yy + ix*QK_K + y_offset; - - for (int i = ix; i < nb; i += 4) { - device const uint8_t * q1 = x[i].qs + q_offset; - device const uint8_t * qh = x[i].qh + l0; - device const half * dh = &x[i].d; - device const uint16_t * a = (device const uint16_t *)x[i].scales + iq; - - device const float * y2 = y1 + 128; - float4 sumy = {0.f, 0.f, 0.f, 0.f}; - for (short l = 0; l < 8; ++l) { - yl[l+0] = y1[l+ 0]; sumy[0] += yl[l+0]; - yl[l+8] = y1[l+32]; sumy[1] += yl[l+8]; - yh[l+0] = y2[l+ 0]; sumy[2] += yh[l+0]; - yh[l+8] = y2[l+32]; sumy[3] += yh[l+8]; - } - - for (short row = 0; row < nr0; ++row) { - device const uint8_t * q2 = q1 + 64; - - sc16[0] = a[0] & kmask1; - sc16[1] = a[2] & kmask1; - sc16[2] = ((a[4] >> 0) & kmask2) | ((a[0] & kmask3) >> 2); - sc16[3] = ((a[4] >> 4) & kmask2) | ((a[2] & kmask3) >> 2); - - float4 acc1 = {0.f}; - float4 acc2 = {0.f}; - FOR_UNROLL (short l = 0; l < 8; ++l) { - uint8_t h = qh[l]; - acc1[0] += yl[l+0] * (q1[l] & 0x0F); - acc1[1] += yl[l+8] * (q1[l] & 0xF0); - acc1[2] += yh[l+0] * (q2[l] & 0x0F); - acc1[3] += yh[l+8] * (q2[l] & 0xF0); - acc2[0] += h & hm1 ? yl[l+0] : 0.f; - acc2[1] += h & hm2 ? yl[l+8] : 0.f; - acc2[2] += h & hm3 ? yh[l+0] : 0.f; - acc2[3] += h & hm4 ? yh[l+8] : 0.f; - } - - sumf[row] += dh[0] * (sc8[0] * (acc1[0] + 16.f*acc2[0]) + - sc8[1] * (acc1[1]/16.f + 16.f*acc2[1]) + - sc8[4] * (acc1[2] + 16.f*acc2[2]) + - sc8[5] * (acc1[3]/16.f + 16.f*acc2[3])) - - dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]); - - q1 += args.nb01; - qh += args.nb01; - dh += args.nb01/2; - a += args.nb01/2; - } - - y1 += 4 * QK_K; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - const float tot = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = tot; - } - } -} - -[[host_name("kernel_mul_mv_q5_K_f32")]] -kernel void kernel_mul_mv_q5_K_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_q5_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_q6_K_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - constexpr uint8_t kmask1 = 0x03; - constexpr uint8_t kmask2 = 0x0C; - constexpr uint8_t kmask3 = 0x30; - constexpr uint8_t kmask4 = 0xC0; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_q6_K * x = (device const block_q6_K *) (src0 + offset0); - device const float * yy = (device const float *) (src1 + offset1); - - float sumf[nr0] = { 0.f }; - - float yl[16]; - - const short tid = tiisg/2; - const short ix = tiisg%2; - const short ip = tid/8; // 0 or 1 - const short il = tid%8; - const short l0 = 4*il; - const short is = 8*ip + l0/16; - - const short y_offset = 128*ip + l0; - const short q_offset_l = 64*ip + l0; - const short q_offset_h = 32*ip + l0; - - for (int i = ix; i < nb; i += 2) { - device const uint8_t * q1 = x[i].ql + q_offset_l; - device const uint8_t * q2 = q1 + 32; - device const uint8_t * qh = x[i].qh + q_offset_h; - device const int8_t * sc = x[i].scales + is; - device const half * dh = &x[i].d; - - device const float * y = yy + i * QK_K + y_offset; - - for (short l = 0; l < 4; ++l) { - yl[4*l + 0] = y[l + 0]; - yl[4*l + 1] = y[l + 32]; - yl[4*l + 2] = y[l + 64]; - yl[4*l + 3] = y[l + 96]; - } - - for (short row = 0; row < nr0; ++row) { - float4 sums = {0.f, 0.f, 0.f, 0.f}; - - FOR_UNROLL (short l = 0; l < 4; ++l) { - sums[0] += yl[4*l + 0] * ((int8_t)((q1[l] & 0xF) | ((qh[l] & kmask1) << 4)) - 32); - sums[1] += yl[4*l + 1] * ((int8_t)((q2[l] & 0xF) | ((qh[l] & kmask2) << 2)) - 32); - sums[2] += yl[4*l + 2] * ((int8_t)((q1[l] >> 4) | ((qh[l] & kmask3) << 0)) - 32); - sums[3] += yl[4*l + 3] * ((int8_t)((q2[l] >> 4) | ((qh[l] & kmask4) >> 2)) - 32); - } - - sumf[row] += dh[0] * (sums[0] * sc[0] + sums[1] * sc[2] + sums[2] * sc[4] + sums[3] * sc[6]); - - q1 += args.nb01; - q2 += args.nb01; - qh += args.nb01; - sc += args.nb01; - dh += args.nb01/2; - } - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all; - } - } -} - -[[host_name("kernel_mul_mv_q6_K_f32")]] -kernel void kernel_mul_mv_q6_K_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_q6_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -} - -// ======================= "True" 2-bit - -template -void kernel_mul_mv_iq2_xxs_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - float yl[32]; - float sumf[nr0]={0.f}; - - const int nb32 = nb * (QK_K / 32); - - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); - threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); - { - int nval = 4; - int pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xxs_grid[pos + i]; - nval = 2; - pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - const int ix = tiisg; - - device const float * y4 = y + 32 * ix; - - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { - for (short i = 0; i < 32; ++i) { - yl[i] = y4[i]; - } - - const int ibl = ib32 / (QK_K / 32); - const int ib = ib32 % (QK_K / 32); - - device const block_iq2_xxs * xr = x + ibl; - device const uint16_t * q2 = xr->qs + 4 * ib; - device const half * dh = &xr->d; - - for (short row = 0; row < nr0; row++) { - const float db = dh[0]; - device const uint8_t * aux8 = (device const uint8_t *)q2; - const uint32_t aux32 = q2[2] | (q2[3] << 16); - const float d = db * (0.5f + (aux32 >> 28)); - - float sum = 0; - for (short l = 0; l < 4; ++l) { - const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + aux8[l]); - const uint8_t signs = ssigns[(aux32 >> 7*l) & 127]; - for (short j = 0; j < 8; ++j) { - sum += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); - } - } - sumf[row] += d * sum; - - dh += args.nb01/2; - q2 += args.nb01/2; - } - - y4 += 32 * 32; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all * 0.25f; - } - } -} - -[[host_name("kernel_mul_mv_iq2_xxs_f32")]] -kernel void kernel_mul_mv_iq2_xxs_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_iq2_xs_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - float yl[32]; - float sumf[nr0]={0.f}; - - const int nb32 = nb * (QK_K / 32); - - threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); - threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512); - { - int nval = 8; - int pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xs_grid[pos + i]; - nval = 2; - pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - const int ix = tiisg; - - device const float * y4 = y + 32 * ix; - - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { - for (short i = 0; i < 32; ++i) { - yl[i] = y4[i]; - } - - const int ibl = ib32 / (QK_K / 32); - const int ib = ib32 % (QK_K / 32); - - device const block_iq2_xs * xr = x + ibl; - device const uint16_t * q2 = xr->qs + 4 * ib; - device const uint8_t * sc = xr->scales + ib; - device const half * dh = &xr->d; - - for (short row = 0; row < nr0; row++) { - const float db = dh[0]; - const uint8_t ls1 = sc[0] & 0xf; - const uint8_t ls2 = sc[0] >> 4; - const float d1 = db * (0.5f + ls1); - const float d2 = db * (0.5f + ls2); - - float sum1 = 0, sum2 = 0; - for (short l = 0; l < 2; ++l) { - const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511)); - const uint8_t signs = ssigns[(q2[l] >> 9)]; - for (short j = 0; j < 8; ++j) { - sum1 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); - } - } - for (short l = 2; l < 4; ++l) { - const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511)); - const uint8_t signs = ssigns[(q2[l] >> 9)]; - for (short j = 0; j < 8; ++j) { - sum2 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); - } - } - sumf[row] += d1 * sum1 + d2 * sum2; - - dh += args.nb01/2; - q2 += args.nb01/2; - sc += args.nb01; - } - - y4 += 32 * 32; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all * 0.25f; - } - } -} - -[[host_name("kernel_mul_mv_iq2_xs_f32")]] -kernel void kernel_mul_mv_iq2_xs_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_iq3_xxs_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_iq3_xxs * x = (device const block_iq3_xxs *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - float yl[32]; - float sumf[nr0]={0.f}; - - const int nb32 = nb * (QK_K / 32); - - threadgroup uint32_t * svalues = (threadgroup uint32_t *)(shmem); - threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); - { - int nval = 4; - int pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) svalues[pos + i] = iq3xxs_grid[pos + i]; - nval = 2; - pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - const int ix = tiisg; - - device const float * y4 = y + 32 * ix; - - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { - for (short i = 0; i < 32; ++i) { - yl[i] = y4[i]; - } - - const int ibl = ib32 / (QK_K / 32); - const int ib = ib32 % (QK_K / 32); - - device const block_iq3_xxs * xr = x + ibl; - device const uint8_t * q3 = xr->qs + 8 * ib; - device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib; - device const half * dh = &xr->d; - - for (short row = 0; row < nr0; row++) { - const float db = dh[0]; - const uint32_t aux32 = gas[0] | (gas[1] << 16); - const float d = db * (0.5f + (aux32 >> 28)); - - float2 sum = {0}; - for (short l = 0; l < 4; ++l) { - const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(svalues + q3[2*l+0]); - const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(svalues + q3[2*l+1]); - const uint8_t signs = ssigns[(aux32 >> 7*l) & 127]; - for (short j = 0; j < 4; ++j) { - sum[0] += yl[8*l + j + 0] * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); - sum[1] += yl[8*l + j + 4] * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); - } - } - sumf[row] += d * (sum[0] + sum[1]); - - dh += args.nb01/2; - q3 += args.nb01; - gas += args.nb01/2; - } - - y4 += 32 * 32; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all * 0.5f; - } - } -} - -[[host_name("kernel_mul_mv_iq3_xxs_f32")]] -kernel void kernel_mul_mv_iq3_xxs_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_iq3_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_iq3_s_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_iq3_s * x = (device const block_iq3_s *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - float yl[32]; - float sumf[nr0]={0.f}; - - const int nb32 = nb * (QK_K / 32); - - threadgroup uint32_t * svalues = (threadgroup uint32_t *) shmem; - { - int nval = 8; - int pos = (32*sgitg + tiisg)*nval; - for (int i = 0; i < nval; ++i) svalues[pos + i] = iq3s_grid[pos + i]; - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - const int ix = tiisg; - - device const float * y4 = y + 32 * ix; - - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { - for (short i = 0; i < 32; ++i) { - yl[i] = y4[i]; - } - - const int ibl = ib32 / (QK_K / 32); - const int ib = ib32 % (QK_K / 32); - - device const block_iq3_s * xr = x + ibl; - device const uint8_t * qs = xr->qs + 8 * ib; - device const uint8_t * qh = xr->qh + ib; - device const uint8_t * sc = xr->scales + (ib/2); - device const uint8_t * signs = xr->signs + 4 * ib; - device const half * dh = &xr->d; - - for (short row = 0; row < nr0; row++) { - const float db = dh[0]; - const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf)); - - float2 sum = {0}; - for (short l = 0; l < 4; ++l) { - const threadgroup uint32_t * table1 = qh[0] & kmask_iq2xs[2*l+0] ? svalues + 256 : svalues; - const threadgroup uint32_t * table2 = qh[0] & kmask_iq2xs[2*l+1] ? svalues + 256 : svalues; - const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(table1 + qs[2*l+0]); - const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(table2 + qs[2*l+1]); - for (short j = 0; j < 4; ++j) { - sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l] & kmask_iq2xs[j+0]); - sum[1] += yl[8*l + j + 4] * grid2[j] * select(1, -1, signs[l] & kmask_iq2xs[j+4]); - } - } - sumf[row] += d * (sum[0] + sum[1]); - - dh += args.nb01/2; - qs += args.nb01; - qh += args.nb01; - sc += args.nb01; - signs += args.nb01; - } - - y4 += 32 * 32; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all; - } - } -} - -[[host_name("kernel_mul_mv_iq3_s_f32")]] -kernel void kernel_mul_mv_iq3_s_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_iq2_s_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_iq2_s * x = (device const block_iq2_s *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - float yl[32]; - float sumf[nr0]={0.f}; - - const int nb32 = nb * (QK_K / 32); - - //threadgroup uint64_t * svalues = (threadgroup uint64_t *) shmem; - //{ - // int nval = 32; - // int pos = (32*sgitg + tiisg)*nval; - // for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2s_grid[pos + i]; - // threadgroup_barrier(mem_flags::mem_threadgroup); - //} - - const short ix = tiisg; - - device const float * y4 = y + 32 * ix; - - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { - for (short i = 0; i < 32; ++i) { - yl[i] = y4[i]; - } - - const int ibl = ib32 / (QK_K / 32); - const int ib = ib32 % (QK_K / 32); - - device const block_iq2_s * xr = x + ibl; - device const uint8_t * qs = xr->qs + 4 * ib; - device const uint8_t * qh = xr->qh + ib; - device const uint8_t * sc = xr->scales + ib; - device const uint8_t * signs = qs + QK_K/8; - device const half * dh = &xr->d; - - for (short row = 0; row < nr0; row++) { - const float db = dh[0]; - const float d1 = db * (0.5f + (sc[0] & 0xf)); - const float d2 = db * (0.5f + (sc[0] >> 4)); - - float2 sum = {0}; - for (short l = 0; l < 2; ++l) { - //const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(svalues + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300))); - //const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(svalues + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300))); - constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300))); - constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300))); - for (short j = 0; j < 8; ++j) { - sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l+0] & kmask_iq2xs[j]); - sum[1] += yl[8*l + j + 16] * grid2[j] * select(1, -1, signs[l+2] & kmask_iq2xs[j]); - } - } - sumf[row] += d1 * sum[0] + d2 * sum[1]; - - dh += args.nb01/2; - qs += args.nb01; - qh += args.nb01; - sc += args.nb01; - signs += args.nb01; - } - - y4 += 32 * 32; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all * 0.25f; - } - } -} - -[[host_name("kernel_mul_mv_iq2_s_f32")]] -kernel void kernel_mul_mv_iq2_s_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_iq1_s_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - float yl[32]; - float sumf[nr0]={0.f}; - - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - - device const float * y4 = y + 32 * ix; - - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { - float sumy = 0; - for (short i = 0; i < 32; ++i) { - yl[i] = y4[i]; - sumy += yl[i]; - } - - const int ibl = ib32 / (QK_K / 32); - const int ib = ib32 % (QK_K / 32); - - device const block_iq1_s * xr = x + ibl; - device const uint8_t * qs = xr->qs + 4 * ib; - device const uint16_t * qh = xr->qh + ib; - device const half * dh = &xr->d; - - for (short row = 0; row < nr0; row++) { - constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); - constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700))); - constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700))); - constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[0] >> 1) & 0x700))); - - float sum = 0; - for (short j = 0; j < 4; ++j) { - sum += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4) - + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4) - + yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4) - + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4); - } - sumf[row] += (float)dh[0] * (sum + sumy * (qh[0] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA)) * (2*((qh[0] >> 12) & 7) + 1); - - dh += args.nb01/2; - qs += args.nb01; - qh += args.nb01/2; - } - - y4 += 32 * 32; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all; - } - } -} - -[[host_name("kernel_mul_mv_iq1_s_f32")]] -kernel void kernel_mul_mv_iq1_s_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_iq1_m_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - const int nb = args.ne00/QK_K; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * nr0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - float yl[32]; - float sumf[nr0]={0.f}; - - const int nb32 = nb * (QK_K / 32); - - const short ix = tiisg; - - device const float * y4 = y + 32 * ix; - - iq1m_scale_t scale; - - for (int ib32 = ix; ib32 < nb32; ib32 += 32) { - float4 sumy = {0.f}; - for (short i = 0; i < 8; ++i) { - yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; - yl[i+ 8] = y4[i+ 8]; sumy[1] += yl[i+ 8]; - yl[i+16] = y4[i+16]; sumy[2] += yl[i+16]; - yl[i+24] = y4[i+24]; sumy[3] += yl[i+24]; - } - - const int ibl = ib32 / (QK_K / 32); - const int ib = ib32 % (QK_K / 32); - - device const block_iq1_m * xr = x + ibl; - device const uint8_t * qs = xr->qs + 4 * ib; - device const uint8_t * qh = xr->qh + 2 * ib; - device const uint16_t * sc = (device const uint16_t *)xr->scales; - - for (short row = 0; row < nr0; row++) { - scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); - - constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); - constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700))); - constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[1] << 8) & 0x700))); - constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[1] << 4) & 0x700))); - - float2 sum = {0.f}; - for (short j = 0; j < 4; ++j) { - sum[0] += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4) - + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4); - sum[1] += yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4) - + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4); - } - const float delta1 = sumy[0] * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[1] * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); - const float delta2 = sumy[2] * (qh[1] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[3] * (qh[1] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); - - sumf[row] += (float)scale.f16 * ((sum[0] + delta1) * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 7) + 1) + - (sum[1] + delta2) * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 7) + 1)); - - sc += args.nb01/2; - qs += args.nb01; - qh += args.nb01; - } - - y4 += 32 * 32; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all; - } - } -} - -[[host_name("kernel_mul_mv_iq1_m_f32")]] -kernel void kernel_mul_mv_iq1_m_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_iq4_nl_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - threadgroup float * shmem_f32 = (threadgroup float *) shmem; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * NR0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_iq4_nl * x = (device const block_iq4_nl *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - const int nb = args.ne00/QK4_NL; - const int ns01 = args.nb01/args.nb00; - - const short ix = tiisg/2; // 0...15 - const short it = tiisg%2; // 0 or 1 - - shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16]; - threadgroup_barrier(mem_flags::mem_threadgroup); - - float4 yl[4]; - float sumf[NR0]={0.f}; - - device const float * yb = y + ix*QK4_NL + it*8; - - uint32_t aux32[2]; - thread const uint8_t * q8 = (thread const uint8_t *)aux32; - - float4 qf1, qf2; - - // [TAG_MUL_MV_WEIRD] - for (int ib = ix; ib < nb && ib < ns01; ib += 16) { - device const float4 * y4 = (device const float4 *)yb; - yl[0] = y4[0]; - yl[1] = y4[4]; - yl[2] = y4[1]; - yl[3] = y4[5]; - - for (short row = 0; row < NR0; row++) { - device const block_iq4_nl & xb = x[row*ns01 + ib]; - device const uint16_t * q4 = (device const uint16_t *)(xb.qs + 8*it); - - float4 acc1 = {0.f}, acc2 = {0.f}; - - aux32[0] = q4[0] | (q4[1] << 16); - aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f; - aux32[0] &= 0x0f0f0f0f; - qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]}; - qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; - acc1 += yl[0] * qf1; - acc2 += yl[1] * qf2; - - aux32[0] = q4[2] | (q4[3] << 16); - aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f; - aux32[0] &= 0x0f0f0f0f; - qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]}; - qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; - acc1 += yl[2] * qf1; - acc2 += yl[3] * qf2; - - acc1 += acc2; - - sumf[row] += (float)xb.d * (acc1[0] + acc1[1] + acc1[2] + acc1[3]); - } - - yb += 16 * QK4_NL; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all; - } - } -} - -[[host_name("kernel_mul_mv_iq4_nl_f32")]] -kernel void kernel_mul_mv_iq4_nl_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_iq4_nl_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_iq4_xs_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - threadgroup float * shmem_f32 = (threadgroup float *) shmem; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - const int first_row = (r0 * NSG + sgitg) * NR0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_iq4_xs * x = (device const block_iq4_xs *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - const int nb = args.ne00/QK_K; - const int ns01 = args.nb01/args.nb00; - - const short ix = tiisg/16; // 0 or 1 - const short it = tiisg%16; // 0...15 - const short ib = it/2; - const short il = it%2; - - shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16]; - threadgroup_barrier(mem_flags::mem_threadgroup); - - float4 yl[4]; - float sumf[NR0]={0.f}; - - device const float * yb = y + ix * QK_K + ib * 32 + il * 8; - - uint32_t aux32[2]; - thread const uint8_t * q8 = (thread const uint8_t *)aux32; - - float4 qf1, qf2; - - // [TAG_MUL_MV_WEIRD] - for (int ibl = ix; ibl < nb && ibl < ns01; ibl += 2) { - device const float4 * y4 = (device const float4 *)yb; - yl[0] = y4[0]; - yl[1] = y4[4]; - yl[2] = y4[1]; - yl[3] = y4[5]; - - for (short row = 0; row < NR0; ++row) { - device const block_iq4_xs & xb = x[row*ns01 + ibl]; - device const uint32_t * q4 = (device const uint32_t *)(xb.qs + 16*ib + 8*il); - - float4 acc1 = {0.f}, acc2 = {0.f}; - - aux32[0] = (q4[0] ) & 0x0f0f0f0f; - aux32[1] = (q4[0] >> 4) & 0x0f0f0f0f; - qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]}; - qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; - acc1 += yl[0] * qf1; - acc2 += yl[1] * qf2; - - aux32[0] = (q4[1] ) & 0x0f0f0f0f; - aux32[1] = (q4[1] >> 4) & 0x0f0f0f0f; - qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]}; - qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; - acc1 += yl[2] * qf1; - acc2 += yl[3] * qf2; - - acc1 += acc2; - - const int ls = (((xb.scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((xb.scales_h >> 2*ib) & 3) << 4)) - 32; - sumf[row] += (float)xb.d * ls * (acc1[0] + acc1[1] + acc1[2] + acc1[3]); - } - - yb += 2 * QK_K; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all; - } - } -} - -[[host_name("kernel_mul_mv_iq4_xs_f32")]] -kernel void kernel_mul_mv_iq4_xs_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_iq4_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -template -void kernel_mul_mv_mxfp4_f32_impl( - args_t args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg) { - const short NSG = FC_mul_mv_nsg; - - threadgroup float * shmem_f32 = (threadgroup float *) shmem; - - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - - const int first_row = (r0 * NSG + sgitg) * NR0; - - const uint i12 = im%args.ne12; - const uint i13 = im/args.ne12; - - const uint64_t offset0 = first_row*args.nb01 + (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; - - device const block_mxfp4 * x = (device const block_mxfp4 *) (src0 + offset0); - device const float * y = (device const float *) (src1 + offset1); - - const int nb = args.ne00/QK_MXFP4; - const int ns01 = args.nb01/args.nb00; // this can be larger than nb for permuted src0 tensors - - const short ix = tiisg/2; // 0...15 - const short it = tiisg%2; // 0 or 1 - - shmem_f32[tiisg] = kvalues_mxfp4_f[tiisg%16]; - threadgroup_barrier(mem_flags::mem_threadgroup); - - float4 yl[4]; - float sumf[NR0]={0.f}; - - device const float * yb = y + ix*QK_MXFP4 + it*8; - - // note: just the check `ib < nb` is enough, but adding the redundant `&& ib < ns01` check makes the kernel a bit faster - // no idea why that is - needs some deeper investigation [TAG_MUL_MV_WEIRD] - for (int ib = ix; ib < nb && ib < ns01; ib += 16) { - device const float4 * y4 = (device const float4 *) yb; - - yl[0] = y4[0]; - yl[1] = y4[4]; - yl[2] = y4[1]; - yl[3] = y4[5]; - - FOR_UNROLL (short row = 0; row < NR0; row++) { - device const block_mxfp4 & xb = x[row*ns01 + ib]; - device const uint8_t * q2 = (device const uint8_t *)(xb.qs + 8*it); - - float4 acc1 = yl[0]*float4(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]); - float4 acc2 = yl[1]*float4(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]); - float4 acc3 = yl[2]*float4(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]); - float4 acc4 = yl[3]*float4(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]); - - acc1 = (acc1 + acc3) + (acc2 + acc4); - - sumf[row] += e8m0_to_fp32(xb.e) * ((acc1[0] + acc1[1]) + (acc1[2] + acc1[3])); - } - - yb += 16 * QK_MXFP4; - } - - device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; - - for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) { - float sum_all = simd_sum(sumf[row]); - if (tiisg == 0) { - dst_f32[first_row + row] = sum_all; - } - } -} - -[[host_name("kernel_mul_mv_mxfp4_f32")]] -kernel void kernel_mul_mv_mxfp4_f32( - constant ggml_metal_kargs_mul_mv & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - kernel_mul_mv_mxfp4_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -template -kernel void kernel_get_rows_q( - constant ggml_metal_kargs_get_rows & args, - device const void * src0, - device const void * src1, - device void * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 ntg [[threads_per_threadgroup]]) { - const int32_t iw0 = tgpig.x/args.ne10; - const int32_t i10 = tgpig.x%args.ne10; - const int32_t i11 = tgpig.y; - const int32_t i12 = tgpig.z; - - const int32_t r = ((const device int32_t *) ((const device char *) src1 + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0]; - - const int32_t i02 = i11; - const int32_t i03 = i12; - - auto psrc = (device const block_q *) ((const device char *) src0 + i03*args.nb03 + i02*args.nb02 + r*args.nb01); - auto pdst = (device float4x4 *) (( device char *) dst + i12*args.nb3 + i11*args.nb2 + i10*args.nb1); - - for (int ind = iw0*ntg.x + tiitg; ind < args.ne00t;) { - float4x4 temp; - dequantize_func(psrc + ind/nl, ind%nl, temp); - pdst[ind] = temp; - - break; - } -} - -template -kernel void kernel_get_rows_f( - constant ggml_metal_kargs_get_rows & args, - device const void * src0, - device const void * src1, - device void * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort3 ntg [[threads_per_threadgroup]]) { - const int32_t iw0 = tgpig.x/args.ne10; - const int32_t i10 = tgpig.x%args.ne10; - const int32_t i11 = tgpig.y; - const int32_t i12 = tgpig.z; - - const int32_t r = ((const device int32_t *) ((const device char *) src1 + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0]; - - const int32_t i02 = i11; - const int32_t i03 = i12; - - auto psrc = (const device T0 *) ((const device char *) src0 + i03*args.nb03 + i02*args.nb02 + r*args.nb01); - auto pdst = ( device T *) (( device char *) dst + i12*args.nb3 + i11*args.nb2 + i10*args.nb1); - - for (int ind = iw0*ntg.x + tiitg; ind < args.ne00t;) { - pdst[ind] = psrc[ind]; - - break; - } -} - -template -kernel void kernel_set_rows_q32( - constant ggml_metal_kargs_set_rows & args, - device const void * src0, - device const void * src1, - device float * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint tiitg[[thread_index_in_threadgroup]], - uint3 tptg [[threads_per_threadgroup]]) { - const int32_t i03 = tgpig.z; - const int32_t i02 = tgpig.y; - - const int32_t i12 = i03%args.ne12; - const int32_t i11 = i02%args.ne11; - - const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x; - if (i01 >= args.ne01) { - return; - } - - const int32_t i10 = i01; - const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0]; - - device block_q * dst_row = ( device block_q *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3); - const device float * src_row = (const device float *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); - - for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) { - quantize_func(src_row + 32*ind, dst_row[ind]); - } -} - -template -kernel void kernel_set_rows_f( - constant ggml_metal_kargs_set_rows & args, - device const void * src0, - device const void * src1, - device float * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - uint tiitg[[thread_index_in_threadgroup]], - uint3 tptg [[threads_per_threadgroup]]) { - const int32_t i03 = tgpig.z; - const int32_t i02 = tgpig.y; - - const int32_t i12 = i03%args.ne12; - const int32_t i11 = i02%args.ne11; - - const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x; - if (i01 >= args.ne01) { - return; - } - - const int32_t i10 = i01; - const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0]; - - device T * dst_row = ( device T *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3); - const device float * src_row = (const device float *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); - - for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) { - dst_row[ind] = (T) src_row[ind]; - } -} - -kernel void kernel_diag_f32( - constant ggml_metal_kargs_diag & args, - device const char * src0, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]]) { - constexpr short NW = N_SIMDWIDTH; - - const int32_t i3 = tgpig.z; - const int32_t i2 = tgpig.y; - const int32_t i1 = tgpig.x; - - device const float * src0_ptr = (device const float *)(src0 + i2*args.nb02 + i3*args.nb03); - device float * dst_ptr = (device float *)(dst + i1*args.nb01 + i2*args.nb2 + i3*args.nb3); - - for (int i0 = tiitg; i0 < args.ne0; i0 += NW) { - dst_ptr[i0] = i0 == i1 ? src0_ptr[i0] : 0.0f; - } -} - -constant bool FC_mul_mm_bc_inp [[function_constant(FC_MUL_MM + 0)]]; -constant bool FC_mul_mm_bc_out [[function_constant(FC_MUL_MM + 1)]]; - -// each block_q contains 16*nl weights -template -kernel void kernel_mul_mm( - constant ggml_metal_kargs_mul_mm & args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - - threadgroup S0 * sa = (threadgroup S0 *)(shmem); - threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096); - -#ifdef GGML_METAL_HAS_TENSOR - threadgroup float * sc = (threadgroup float *)(shmem); -#endif - - constexpr int NR0 = 64; - constexpr int NR1 = 32; - - constexpr int NK = 32; - constexpr int NL0 = NK/16; - constexpr int NL1 = NK/8; - - const int im = tgpig.z; - const int r0 = tgpig.y*NR0; - const int r1 = tgpig.x*NR1; - - // if this block is of 64x32 shape or smaller - const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0; - const short nr1 = (args.ne1 - r1 < NR1) ? (args.ne1 - r1) : NR1; - - // a thread shouldn't load data outside of the matrix - const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63 - const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31 - - const short il0 = (tiitg % NL0); - - short il = il0; - - const int i12 = im%args.ne12; - const int i13 = im/args.ne12; - - const uint64_t offset0 = (i12/args.r2)*args.nb02 + (i13/args.r3)*args.nb03; - const short offset1 = il0/nl; - - device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1; - - const short iy = 8*(tiitg % NL1); - - device const T1 * y = (device const T1 *)(src1 - + args.nb13*i13 - + args.nb12*i12 - + args.nb11*(r1 + lr1) - + args.nb10*iy); - -#ifndef GGML_METAL_HAS_TENSOR - S0_8x8 ma[4]; - S1_8x8 mb[2]; - - simdgroup_float8x8 mc[8]; - - for (short i = 0; i < 8; i++){ - mc[i] = make_filled_simdgroup_matrix(0.f); - } -#else - auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0)); - auto tB = tensor, tensor_inline>(sb, dextents(NR1, NK )); - - mpp::tensor_ops::matmul2d< - mpp::tensor_ops::matmul2d_descriptor(NR1, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), - execution_simdgroups<4>> mm; - - auto cT = mm.get_destination_cooperative_tensor(); -#endif - - for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { -#ifndef GGML_METAL_HAS_TENSOR - // load data and store to threadgroup memory - if (is_same::value && FC_mul_mm_bc_inp) { - threadgroup_barrier(mem_flags::mem_threadgroup); - - // no need for dequantization - for (short i = 0; i < 16; i++) { - const short sx = 2*il0 + i/8; - const short sy = (tiitg/NL0)/8; - - //const short lx = i%8; - //const short ly = (tiitg/NL0)%8; - const short lx = (tiitg/NL0)%8; - const short ly = i%8; - - const short ib = 8*sx + sy; - - *(sa + 64*ib + 8*ly + lx) = loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0; - } - } else { - S0_4x4 temp_a; - dequantize_func(x, il, temp_a); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - FOR_UNROLL (short i = 0; i < 16; i++) { - const short sx = 2*il0 + i/8; - const short sy = (tiitg/NL0)/8; - - //const short lx = i%8; - //const short ly = (tiitg/NL0)%8; - const short lx = (tiitg/NL0)%8; - const short ly = i%8; - - const short ib = 8*sx + sy; - - // NOTE: this is massively slower.. WTF? - //sa[64*ib + 8*ly + lx] = temp_a[i/4][i%4]; - - *(sa + 64*ib + 8*ly + lx) = temp_a[i/4][i%4]; - } - } - - if (FC_mul_mm_bc_inp) { - for (short i = 0; i < 8; ++i) { - const short sx = (tiitg%NL1); - const short sy = (tiitg/NL1)/8; - - const short lx = i; - const short ly = (tiitg/NL1)%8; - //const short lx = (tiitg/NL1)%8; - //const short ly = i; - - const short ib = 4*sx + sy; - - *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; - } - } else { - const short sx = (tiitg%NL1); - const short sy = (tiitg/NL1)/8; - - //const short dx = sx; - //const short dy = sy; - - const short ly = (tiitg/NL1)%8; - - const short ib = 4*sx + sy; - - *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y)); - } -#else - // load data and store to threadgroup memory - if (is_same::value && FC_mul_mm_bc_inp) { - threadgroup_barrier(mem_flags::mem_threadgroup); - - // no need for dequantization - for (short i = 0; i < 16; i++) { - const short sx = 2*il0 + i/8; - const short sy = (tiitg/NL0)/8; - - const short lx = i%8; - const short ly = (tiitg/NL0)%8; - //const short lx = (tiitg/NL0)%8; - //const short ly = i%8; - - *(sa + NK*(8*sy + ly) + 8*sx + lx) = loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0; - } - } else { - S0_4x4 temp_a; - dequantize_func(x, il, temp_a); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - FOR_UNROLL (short i = 0; i < 16; i++) { - const short sx = 2*il0 + i/8; - const short sy = (tiitg/NL0)/8; - - const short lx = i%8; - const short ly = (tiitg/NL0)%8; - //const short lx = (tiitg/NL0)%8; - //const short ly = i%8; - - *(sa + NK*(8*sy + ly) + 8*sx + lx) = temp_a[i/4][i%4]; - } - } - - if (FC_mul_mm_bc_inp) { - for (short i = 0; i < 8; ++i) { - const short sx = (tiitg%NL1); - const short sy = (tiitg/NL1)/8; - - const short lx = i; - const short ly = (tiitg/NL1)%8; - //const short lx = (tiitg/NL1)%8; - //const short ly = i; - - *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; - } - } else { - const short sx = (tiitg%NL1); - const short sy = (tiitg/NL1)/8; - - //const short lx = i; - const short ly = (tiitg/NL1)%8; - //const short lx = (tiitg/NL1)%8; - //const short ly = i; - - *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)(*((device T1_2x4 *) y)); - } -#endif - - il = (il + 2 < nl) ? il + 2 : il % 2; - x = (il < 2) ? x + (2 + nl - 1)/nl : x; - - y += NK; - - threadgroup_barrier(mem_flags::mem_threadgroup); - -#ifndef GGML_METAL_HAS_TENSOR - // load matrices from threadgroup memory and conduct outer products - threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); - threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); - - FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 4; i++) { - simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); - } - - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 2; i++) { - simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); - } - - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 8; i++){ - simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); - } - - lsma += 8*64; - lsmb += 4*64; - } -#else - auto sA = tA.slice(0, 0); - auto sB = tB.slice(0, 0); - - mm.run(sB, sA, cT); -#endif - } - - if (!FC_mul_mm_bc_out || (r0 + NR0 <= args.ne0 && r1 + NR1 <= args.ne1)) { - // if no bounds checks on the output are needed, we can directly write to device memory -#ifdef GGML_METAL_HAS_TENSOR - device float * C = (device float *) dst + - r0 + \ - r1 * args.ne0 + im*args.ne1*args.ne0; - - auto tC = tensor, tensor_inline>(C, dextents(args.ne0, NR1)); - cT.store(tC); -#else - device float * C = (device float *) dst + - (r0 + 32*(sgitg & 1)) + \ - (r1 + 16*(sgitg >> 1)) * args.ne0 + im*args.ne1*args.ne0; - - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], C + 8*(i%4) + 8*args.ne0*(i/4), args.ne0, 0, false); - } -#endif - } else { - // block is smaller than 64x32, we should avoid writing data outside of the matrix - threadgroup_barrier(mem_flags::mem_threadgroup); - - threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; - -#ifdef GGML_METAL_HAS_TENSOR - auto tC = tensor, tensor_inline>(sc, dextents(NR0, NR1)); - cT.store(tC); -#else - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); - } -#endif - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (sgitg == 0) { - for (int j = tiitg; j < nr1; j += NR1) { - device float * D = (device float *) dst + r0 + (r1 + j)*args.ne0 + im*args.ne1*args.ne0; - device float4 * D4 = (device float4 *) D; - - threadgroup float * C = temp_str + (j*NR0); - threadgroup float4 * C4 = (threadgroup float4 *) C; - - int i = 0; - for (; i < nr0/4; i++) { - *(D4 + i) = *(C4 + i); - } - - i *= 4; - for (; i < nr0; i++) { - *(D + i) = *(C + i); - } - } - } - } -} - -template // n_expert_used -kernel void kernel_mul_mm_id_map0( - constant ggml_metal_kargs_mul_mm_id_map0 & args, - device const char * src2, - device char * htpe, - device char * hids, - threadgroup char * shmem [[threadgroup(0)]], - ushort tpitg[[thread_position_in_threadgroup]], - ushort ntg[[threads_per_threadgroup]]) { - const short ide = tpitg; // expert id - - uint32_t n_all = 0; - - device int32_t * ids_i32 = (device int32_t *) hids + ide*args.ne21; - - for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens - if (i21 + tpitg < args.ne21) { - device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21); - - threadgroup uint16_t * sids = (threadgroup uint16_t *) shmem + tpitg*ne20; - - #pragma unroll(ne20) - for (short i20 = 0; i20 < ne20; i20++) { - sids[i20] = src2_i32[i20]; - } - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - for (short t = 0; t < ntg; t++) { - if (i21 + t >= args.ne21) { - break; - } - - threadgroup const uint16_t * sids = (threadgroup const uint16_t *) shmem + t*ne20; - - short sel = 0; - #pragma unroll(ne20) - for (short i20 = 0; i20 < ne20; i20++) { - sel += (sids[i20] == ide)*(i20 + 1); - } - - ids_i32[n_all] = (i21 + t)*ne20 + sel - 1; - - n_all += sel > 0; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - } - - device uint32_t * tpe_u32 = (device uint32_t *) (htpe); - tpe_u32[ide] = n_all; -} - -typedef decltype(kernel_mul_mm_id_map0<1>) kernel_mul_mm_id_map0_t; - -template [[host_name("kernel_mul_mm_id_map0_ne20_1" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<1>; -template [[host_name("kernel_mul_mm_id_map0_ne20_2" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<2>; -template [[host_name("kernel_mul_mm_id_map0_ne20_4" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<4>; -template [[host_name("kernel_mul_mm_id_map0_ne20_5" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<5>; -template [[host_name("kernel_mul_mm_id_map0_ne20_6" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<6>; -template [[host_name("kernel_mul_mm_id_map0_ne20_8" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<8>; -template [[host_name("kernel_mul_mm_id_map0_ne20_10")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<10>; -template [[host_name("kernel_mul_mm_id_map0_ne20_16")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<16>; -template [[host_name("kernel_mul_mm_id_map0_ne20_22")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<22>; - -template -kernel void kernel_mul_mm_id( - constant ggml_metal_kargs_mul_mm_id & args, - device const char * src0, - device const char * src1, - device const char * htpe, - device const char * hids, - device char * dst, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - threadgroup S0 * sa = (threadgroup S0 *)(shmem); - threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096); - -#ifdef GGML_METAL_HAS_TENSOR - threadgroup float * sc = (threadgroup float *)(shmem); -#endif - - constexpr int NR0 = 64; - constexpr int NR1 = 32; - - constexpr int NK = 32; - constexpr int NL0 = NK/16; - constexpr int NL1 = NK/8; - - const int im = tgpig.z; // expert - const int r0 = tgpig.y*NR0; - const int r1 = tgpig.x*NR1; - - device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe); - device const int32_t * ids_i32 = (device const int32_t *) (hids); - - const int32_t neh1 = tpe_u32[im]; - - if (r1 >= neh1) { - return; - } - - // if this block is of 64x32 shape or smaller - const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0; - const short nr1 = ( neh1 - r1 < NR1) ? ( neh1 - r1) : NR1; - - // a thread shouldn't load data outside of the matrix - const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63 - const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31 - - const short il0 = (tiitg % NL0); - - short il = il0; - - const int id = ids_i32[im*args.ne21 + r1 + lr1]; - - const short i11 = (id % args.ne20) % args.ne11; - const short i12 = (id / args.ne20); - const short i13 = 0; - - const uint64_t offset0 = im*args.nb02 + i13*args.nb03; - const short offset1 = il0/nl; - - device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1; - - const short iy = 8*(tiitg % NL1); - - device const T1 * y = (device const T1 *)(src1 - + args.nb13*i13 - + args.nb12*i12 - + args.nb11*i11 - + args.nb10*iy); - -#ifndef GGML_METAL_HAS_TENSOR - S0_8x8 ma[4]; - S1_8x8 mb[2]; - - simdgroup_float8x8 mc[8]; - - for (short i = 0; i < 8; i++){ - mc[i] = make_filled_simdgroup_matrix(0.f); - } -#else - auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0)); - auto tB = tensor, tensor_inline>(sb, dextents(NR1, NK )); - - mpp::tensor_ops::matmul2d< - mpp::tensor_ops::matmul2d_descriptor(NR1, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), - execution_simdgroups<4>> mm; - - auto cT = mm.get_destination_cooperative_tensor(); -#endif - - for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { -#ifndef GGML_METAL_HAS_TENSOR - // load data and store to threadgroup memory - if (is_same::value && FC_mul_mm_bc_inp) { - threadgroup_barrier(mem_flags::mem_threadgroup); - - // no need for dequantization - for (short i = 0; i < 16; i++) { - const short sx = 2*il0 + i/8; - const short sy = (tiitg/NL0)/8; - - //const short lx = i%8; - //const short ly = (tiitg/NL0)%8; - const short lx = (tiitg/NL0)%8; - const short ly = i%8; - - const short ib = 8*sx + sy; - - *(sa + 64*ib + 8*ly + lx) = loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0; - } - } else { - S0_4x4 temp_a; - dequantize_func(x, il, temp_a); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - FOR_UNROLL (short i = 0; i < 16; i++) { - const short sx = 2*il0 + i/8; - const short sy = (tiitg/NL0)/8; - - //const short lx = i%8; - //const short ly = (tiitg/NL0)%8; - const short lx = (tiitg/NL0)%8; - const short ly = i%8; - - const short ib = 8*sx + sy; - - // NOTE: this is massively slower.. WTF? - //sa[64*ib + 8*ly + lx] = temp_a[i/4][i%4]; - - *(sa + 64*ib + 8*ly + lx) = temp_a[i/4][i%4]; - } - } - - if (FC_mul_mm_bc_inp) { - for (short i = 0; i < 8; ++i) { - const short sx = (tiitg%NL1); - const short sy = (tiitg/NL1)/8; - - const short lx = i; - const short ly = (tiitg/NL1)%8; - //const short lx = (tiitg/NL1)%8; - //const short ly = i; - - const short ib = 4*sx + sy; - - *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; - } - } else { - const short sx = (tiitg%NL1); - const short sy = (tiitg/NL1)/8; - - //const short dx = sx; - //const short dy = sy; - - const short ly = (tiitg/NL1)%8; - - const short ib = 4*sx + sy; - - *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y)); - } -#else - // load data and store to threadgroup memory - if (is_same::value && FC_mul_mm_bc_inp) { - threadgroup_barrier(mem_flags::mem_threadgroup); - - // no need for dequantization - for (short i = 0; i < 16; i++) { - const short sx = 2*il0 + i/8; - const short sy = (tiitg/NL0)/8; - - const short lx = i%8; - const short ly = (tiitg/NL0)%8; - //const short lx = (tiitg/NL0)%8; - //const short ly = i%8; - - *(sa + NK*(8*sy + ly) + 8*sx + lx) = loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0; - } - } else { - S0_4x4 temp_a; - dequantize_func(x, il, temp_a); - - threadgroup_barrier(mem_flags::mem_threadgroup); - - FOR_UNROLL (short i = 0; i < 16; i++) { - const short sx = 2*il0 + i/8; - const short sy = (tiitg/NL0)/8; - - const short lx = i%8; - const short ly = (tiitg/NL0)%8; - //const short lx = (tiitg/NL0)%8; - //const short ly = i%8; - - *(sa + NK*(8*sy + ly) + 8*sx + lx) = temp_a[i/4][i%4]; - } - } - - if (FC_mul_mm_bc_inp) { - for (short i = 0; i < 8; ++i) { - const short sx = (tiitg%NL1); - const short sy = (tiitg/NL1)/8; - - const short lx = i; - const short ly = (tiitg/NL1)%8; - //const short lx = (tiitg/NL1)%8; - //const short ly = i; - - *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; - } - } else { - const short sx = (tiitg%NL1); - const short sy = (tiitg/NL1)/8; - - //const short lx = i; - const short ly = (tiitg/NL1)%8; - //const short lx = (tiitg/NL1)%8; - //const short ly = i; - - *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)(*((device T1_2x4 *) y)); - } -#endif - - il = (il + 2 < nl) ? il + 2 : il % 2; - x = (il < 2) ? x + (2 + nl - 1)/nl : x; - - y += NK; - - threadgroup_barrier(mem_flags::mem_threadgroup); - -#ifndef GGML_METAL_HAS_TENSOR - // load matrices from threadgroup memory and conduct outer products - threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); - threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); - - FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 4; i++) { - simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); - } - - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 2; i++) { - simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); - } - - simdgroup_barrier(mem_flags::mem_none); - - FOR_UNROLL (short i = 0; i < 8; i++){ - simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); - } - - lsma += 8*64; - lsmb += 4*64; - } -#else - auto sA = tA.slice(0, 0); - auto sB = tB.slice(0, 0); - - mm.run(sB, sA, cT); -#endif - } - - // block is smaller than 64x32, we should avoid writing data outside of the matrix - threadgroup_barrier(mem_flags::mem_threadgroup); - -#ifdef GGML_METAL_HAS_TENSOR - auto tC = tensor, tensor_inline>(sc, dextents(NR0, NR1)); - cT.store(tC); -#else - threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; - - for (short i = 0; i < 8; i++) { - simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); - } -#endif - - threadgroup_barrier(mem_flags::mem_threadgroup); - - for (short j = sgitg; j < nr1; j += 4) { - const int id = ids_i32[im*args.ne21 + r1 + j]; - - const short ide = id % args.ne20; - const short idt = id / args.ne20; - - device float * D = (device float *) dst + r0 + ide*args.ne0 + idt*args.ne1*args.ne0; - device float4 * D4 = (device float4 *) D; - - threadgroup float * C = (threadgroup float *) shmem + j*NR0; - threadgroup float4 * C4 = (threadgroup float4 *) C; - - int i = tiisg; - for (; i < nr0/4; i += 32) { - *(D4 + i) = *(C4 + i); - } - - i = (4*(nr0/4)) + tiisg; - for (; i < nr0; i += 32) { - *(D + i) = *(C + i); - } - } -} - -#define QK_NL 16 - -// -// get rows -// - -typedef decltype(kernel_get_rows_f) get_rows_f_t; - -template [[host_name("kernel_get_rows_f32")]] kernel get_rows_f_t kernel_get_rows_f; -template [[host_name("kernel_get_rows_f16")]] kernel get_rows_f_t kernel_get_rows_f; -template [[host_name("kernel_get_rows_i32")]] kernel get_rows_f_t kernel_get_rows_f; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_get_rows_bf16")]] kernel get_rows_f_t kernel_get_rows_f; -#endif - -typedef decltype(kernel_get_rows_q) get_rows_q_t; - -template [[host_name("kernel_get_rows_q1_0")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q4_0")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q4_1")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q5_0")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q5_1")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q8_0")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_mxfp4")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q2_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q3_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q4_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q5_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq1_m")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_q_t kernel_get_rows_q; -template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_q_t kernel_get_rows_q; - -// -// set rows -// - -typedef decltype(kernel_set_rows_f) set_rows_f_t; - -template [[host_name("kernel_set_rows_f32_i64")]] kernel set_rows_f_t kernel_set_rows_f; -template [[host_name("kernel_set_rows_f32_i32")]] kernel set_rows_f_t kernel_set_rows_f; -template [[host_name("kernel_set_rows_f16_i64")]] kernel set_rows_f_t kernel_set_rows_f; -template [[host_name("kernel_set_rows_f16_i32")]] kernel set_rows_f_t kernel_set_rows_f; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_set_rows_bf16_i64")]] kernel set_rows_f_t kernel_set_rows_f; -template [[host_name("kernel_set_rows_bf16_i32")]] kernel set_rows_f_t kernel_set_rows_f; -#endif - -typedef decltype(kernel_set_rows_q32) set_rows_q32_t; - -template [[host_name("kernel_set_rows_q8_0_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q8_0_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q4_0_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q4_0_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q4_1_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q4_1_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q5_0_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q5_0_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q5_1_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_q5_1_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_iq4_nl_i64")]] kernel set_rows_q32_t kernel_set_rows_q32; -template [[host_name("kernel_set_rows_iq4_nl_i32")]] kernel set_rows_q32_t kernel_set_rows_q32; - -// -// matrix-matrix multiplication -// - -typedef decltype(kernel_mul_mm) mul_mm_t; - -template [[host_name("kernel_mul_mm_f32_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_mm; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm; -#endif -template [[host_name("kernel_mul_mm_q1_0_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q4_1_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q5_0_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q5_1_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q8_0_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_mxfp4_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q2_K_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q3_K_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q5_K_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq2_xxs_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq3_s_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq2_s_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq1_m_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mul_mm_t kernel_mul_mm; - -template [[host_name("kernel_mul_mm_f32_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_f16_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q1_0_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q4_0_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q4_1_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q5_0_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q5_1_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q8_0_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_mxfp4_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q2_K_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q3_K_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q4_K_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q5_K_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_q6_K_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq2_xxs_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq2_xs_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq3_xxs_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq3_s_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq2_s_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq1_s_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq1_m_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq4_nl_f16")]] kernel mul_mm_t kernel_mul_mm; -template [[host_name("kernel_mul_mm_iq4_xs_f16")]] kernel mul_mm_t kernel_mul_mm; - -// -// indirect matrix-matrix multiplication -// - -typedef decltype(kernel_mul_mm_id) mul_mm_id; - -template [[host_name("kernel_mul_mm_id_f32_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_f16_f32")]] kernel mul_mm_id kernel_mul_mm_id; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mm_id_bf16_f32")]] kernel mul_mm_id kernel_mul_mm_id; -#endif -template [[host_name("kernel_mul_mm_id_q1_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q4_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q4_1_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q5_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q5_1_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q8_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_mxfp4_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q2_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q3_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q4_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q5_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq3_s_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq2_s_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq1_m_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mul_mm_id kernel_mul_mm_id; - -template [[host_name("kernel_mul_mm_id_f32_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_f16_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q1_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q4_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q4_1_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q5_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q5_1_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q8_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_mxfp4_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q2_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q3_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q4_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q5_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_q6_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq2_xxs_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq2_xs_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq3_xxs_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq3_s_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq2_s_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq1_s_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq1_m_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq4_nl_f16")]] kernel mul_mm_id kernel_mul_mm_id; -template [[host_name("kernel_mul_mm_id_iq4_xs_f16")]] kernel mul_mm_id kernel_mul_mm_id; - -// -// matrix-vector multiplication -// - -typedef void (kernel_mul_mv_disp_t)( - ggml_metal_kargs_mul_mv args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig, - ushort tiisg); - -typedef void (kernel_mul_mv2_disp_t)( - ggml_metal_kargs_mul_mv args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiisg, - ushort sgitg); - -template -void mmv_fn( - ggml_metal_kargs_mul_mv args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiitg, - ushort tiisg, - ushort sgitg) { - disp_fn(args, src0, src1, dst, tgpig, tiisg); -} - -template -void mmv_fn( - ggml_metal_kargs_mul_mv args, - device const char * src0, - device const char * src1, - device char * dst, - threadgroup char * shmem, - uint3 tgpig, - ushort tiitg, - ushort tiisg, - ushort sgitg) { - disp_fn(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); -} - -typedef decltype(mmv_fn>) mul_mv_disp_fn_t; - -template -kernel void kernel_mul_mv_id( - constant ggml_metal_kargs_mul_mv_id & args, - device const char * src0s, - device const char * src1, - device char * dst, - device const char * ids, - threadgroup char * shmem [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort tiitg[[thread_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]]) { - const int iid1 = tgpig.z/args.nei0; - const int idx = tgpig.z%args.nei0; - - tgpig.z = 0; - - const int32_t i02 = ((device const int32_t *) (ids + iid1*args.nbi1))[idx]; - - const int64_t i11 = idx % args.ne11; - const int64_t i12 = iid1; - - const int64_t i1 = idx; - const int64_t i2 = i12; - - device const char * src0_cur = src0s + i02*args.nb02; - device const char * src1_cur = src1 + i11*args.nb11 + i12*args.nb12; - - device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float); - - ggml_metal_kargs_mul_mv args0 = { - /*.ne00 =*/ args.ne00, - /*.ne01 =*/ args.ne01, - /*.ne02 =*/ 1, // args.ne02, - /*.nb00 =*/ args.nb00, - /*.nb01 =*/ args.nb01, - /*.nb02 =*/ args.nb02, - /*.nb03 =*/ args.nb02, // args.ne02 == 1 - /*.ne10 =*/ args.ne10, - /*.ne11 =*/ 1, // args.ne11, - /*.ne12 =*/ 1, // args.ne12, - /*.nb10 =*/ args.nb10, - /*.nb11 =*/ args.nb11, - /*.nb12 =*/ args.nb12, - /*.nb13 =*/ args.nb12, // ne12 == 1 - /*.ne0 =*/ args.ne0, - /*.ne1 =*/ 1, // args.ne1, - /*.nr0 =*/ args.nr0, - /*.r2 =*/ 1, - /*.r3 =*/ 1, - }; - - disp_fn( - args0, - /* src0 */ src0_cur, - /* src1 */ src1_cur, - /* dst */ dst_cur, - shmem, - tgpig, - tiitg, - tiisg, - sgitg); -} - -typedef decltype(kernel_mul_mv_id>>) kernel_mul_mv_id_t; - -typedef decltype(kernel_mul_mv_id>>) kernel_mul_mv_id_4_t; - -template [[host_name("kernel_mul_mv_id_f32_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_f16_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mv_id_bf16_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -#endif -template [[host_name("kernel_mul_mv_id_f32_f32_4")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_f16_f32_4")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id>>; -#if defined(GGML_METAL_HAS_BF16) -template [[host_name("kernel_mul_mv_id_bf16_f32_4")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id>>; -#endif - -template [[host_name("kernel_mul_mv_id_q8_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; - -template [[host_name("kernel_mul_mv_id_q1_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_q4_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_q4_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_q5_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_q5_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; - -template [[host_name("kernel_mul_mv_id_mxfp4_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; - -template [[host_name("kernel_mul_mv_id_q2_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_q3_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_q4_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_q5_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_q6_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; -template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; - -kernel void kernel_pool_2d_max_f32( - constant ggml_metal_kargs_pool_2d & args, - device const float * src0, - device float * dst, - uint gid[[thread_position_in_grid]]) { - - if (gid >= args.np) { - return; - } - - const int idx = gid; - const int I_HW = args.IH * args.IW; - const int O_HW = args.OH * args.OW; - const int nc = idx / O_HW; - const int cur_oh = idx % O_HW / args.OW; - const int cur_ow = idx % O_HW % args.OW; - - device const float * i_ptr = src0 + nc * I_HW; - device float * o_ptr = dst + nc * O_HW; - - const int start_h = cur_oh * args.s1 - args.p1; - const int bh = MAX(0, start_h); - const int eh = MIN(args.IH, start_h + args.k1); - const int start_w = cur_ow * args.s0 - args.p0; - const int bw = MAX(0, start_w); - const int ew = MIN(args.IW, start_w + args.k0); - - float res = -INFINITY; - - for (int i = bh; i < eh; i += 1) { - for (int j = bw; j < ew; j += 1) { - res = MAX(res, i_ptr[i * args.IW + j]); - } - } - - o_ptr[cur_oh * args.OW + cur_ow] = res; -} - -kernel void kernel_pool_2d_avg_f32( - constant ggml_metal_kargs_pool_2d & args, - device const float * src0, - device float * dst, - uint gid[[thread_position_in_grid]]) { - - if (gid >= args.np) { - return; - } - - const int idx = gid; - const int I_HW = args.IH * args.IW; - const int O_HW = args.OH * args.OW; - const int nc = idx / O_HW; - const int cur_oh = idx % O_HW / args.OW; - const int cur_ow = idx % O_HW % args.OW; - - device const float * i_ptr = src0 + nc * I_HW; - device float * o_ptr = dst + nc * O_HW; - - const int start_h = cur_oh * args.s1 - args.p1; - const int bh = MAX(0, start_h); - const int eh = MIN(args.IH, start_h + args.k1); - const int start_w = cur_ow * args.s0 - args.p0; - const int bw = MAX(0, start_w); - const int ew = MIN(args.IW, start_w + args.k0); - // const float scale = 1. / ((eh - bh) * (ew - bw)); - const float scale = 1. / (args.k0 * args.k1); - - float res = 0; - - for (int i = bh; i < eh; i += 1) { - for (int j = bw; j < ew; j += 1) { - float cur = i_ptr[i * args.IW + j]; - res += cur * scale; - } - } - - o_ptr[cur_oh * args.OW + cur_ow] = res; -} - - -kernel void kernel_pool_1d_max_f32( - constant ggml_metal_kargs_pool_1d & args, - device const float * src, - device float * dst, - uint gid [[thread_position_in_grid]] -) { - - if (gid >= args.np) { - return; - } - - const int ow = (int)gid % args.OW; - const int row = (int)gid / args.OW; - - const int base = ow * args.s0 - args.p0; - - float acc = -INFINITY; - - const int src_off = row * args.IW; - const int dst_off = row * args.OW; - - for (int ki = 0; ki < args.k0; ++ki) { - int j = base + ki; - if (j < 0 || j >= args.IW){ - continue; - } - float v = src[src_off + j]; - acc = max(acc, v); - } - - dst[dst_off + ow] = acc; -} - -kernel void kernel_pool_1d_avg_f32( - constant ggml_metal_kargs_pool_1d & args, - device const float * src, - device float * dst, - uint gid [[thread_position_in_grid]] -) { - - if (gid >= args.np) { - return; - } - - const int ow = (int)gid % args.OW; - const int row = (int)gid / args.OW; - - const int base = ow * args.s0 - args.p0; - - float acc = 0.0f; - int cnt = 0; - - const int src_off = row * args.IW; - const int dst_off = row * args.OW; - - for (int ki = 0; ki < args.k0; ++ki) { - const int j = base + ki; - if (j < 0 || j >= args.IW) { - continue; - } - acc += src[src_off + j]; - cnt += 1; - } - - dst[dst_off + ow] = (cnt > 0) ? (acc / (float)cnt) : 0.0f; -} - -kernel void kernel_opt_step_adamw_f32( - constant ggml_metal_kargs_opt_step_adamw & args, - device float * x, - device const float * g, - device float * g_m, - device float * g_v, - device const float * pars, - uint gid[[thread_position_in_grid]]) { - - if (gid >= args.np) { - return; - } - - const float alpha = pars[0]; - const float beta1 = pars[1]; - const float beta2 = pars[2]; - const float eps = pars[3]; - const float wd = pars[4]; - const float beta1h = pars[5]; - const float beta2h = pars[6]; - - const float gi = g[gid]; - const float gmi = g_m[gid] * beta1 + gi * (1.0f - beta1); - const float gvi = g_v[gid] * beta2 + gi * gi * (1.0f - beta2); - - g_m[gid] = gmi; - g_v[gid] = gvi; - - const float mh = gmi * beta1h; - const float vh = sqrt(gvi * beta2h) + eps; - - x[gid] = x[gid] * (1.0f - alpha * wd) - alpha * mh / vh; -} - -kernel void kernel_opt_step_sgd_f32( - constant ggml_metal_kargs_opt_step_sgd & args, - device float * x, - device const float * g, - device const float * pars, - uint gid[[thread_position_in_grid]]) { - - if (gid >= args.np) { - return; - } - - x[gid] = x[gid] * (1.0f - pars[0] * pars[1]) - pars[0] * g[gid]; -} - -template -kernel void kernel_memset( - constant ggml_metal_kargs_memset & args, - device T * dst, - uint tpig[[thread_position_in_grid]]) { - dst[tpig] = args.val; -} - -typedef decltype(kernel_memset) kernel_memset_t; - -template [[host_name("kernel_memset_i64")]] kernel kernel_memset_t kernel_memset; - -constant short FC_count_equal_nsg [[function_constant(FC_COUNT_EQUAL + 0)]]; - -template -kernel void kernel_count_equal( - constant ggml_metal_kargs_count_equal & args, - device const char * src0, - device const char * src1, - device atomic_int * dst, - threadgroup int32_t * shmem_i32 [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort sgitg[[simdgroup_index_in_threadgroup]], - ushort tiisg[[thread_index_in_simdgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { - const short NSG = FC_count_equal_nsg; - - const int i3 = tgpig.z; - const int i2 = tgpig.y; - const int i1 = tgpig.x; - - if (i3 >= args.ne03 || i2 >= args.ne02 || i1 >= args.ne01) { - return; - } - - int sum = 0; - - device const char * base0 = src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03; - device const char * base1 = src1 + i1*args.nb11 + i2*args.nb12 + i3*args.nb13; - - for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) { - const T v0 = *(device const T *)(base0 + i0*args.nb00); - const T v1 = *(device const T *)(base1 + i0*args.nb10); - sum += (v0 == v1); - } - - sum = simd_sum(sum); - - if (tiisg == 0) { - shmem_i32[sgitg] = sum; - } - - threadgroup_barrier(mem_flags::mem_threadgroup); - - if (sgitg == 0) { - float v = 0.0f; - if (tpitg.x < NSG) { - v = shmem_i32[tpitg.x]; - } - - float total = simd_sum(v); - if (tpitg.x == 0) { - atomic_fetch_add_explicit(dst, (int32_t) total, memory_order_relaxed); - } - } -} - -typedef decltype(kernel_count_equal) kernel_count_equal_t; - -template [[host_name("kernel_count_equal_i32")]] kernel kernel_count_equal_t kernel_count_equal; diff --git a/ggml/src/ggml-metal/kernels/argsort.metal b/ggml/src/ggml-metal/kernels/argsort.metal new file mode 100644 index 0000000..5231b83 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/argsort.metal @@ -0,0 +1,479 @@ +#include "common.h" + +constant bool FC_topk_moe_with_norm [[function_constant(FC_TOPK_MOE + 0)]]; +constant int FC_topk_moe_n_expert [[function_constant(FC_TOPK_MOE + 1)]]; +constant int FC_topk_moe_top_k [[function_constant(FC_TOPK_MOE + 2)]]; + +constant int FC_moe_reduce_n_expert_used [[function_constant(FC_MOE_REDUCE + 0)]]; + +// bitonic sort implementation following the CUDA kernels as reference +typedef void (argsort_t)( + constant ggml_metal_kargs_argsort & args, + device const char * src0, + device int32_t * dst, + threadgroup int32_t * shmem_i32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]); + +template +kernel void kernel_argsort_f32_i32( + constant ggml_metal_kargs_argsort & args, + device const char * src0, + device int32_t * dst, + threadgroup int32_t * shmem_i32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + // bitonic sort + const int col = tpitg[0]; + const int ib = tgpig[0] / args.ne01; + + const int i00 = ib*ntg.x; + const int i01 = tgpig[0] % args.ne01; + const int i02 = tgpig[1]; + const int i03 = tgpig[2]; + + device const float * src0_row = (device const float *) (src0 + args.nb01*i01 + args.nb02*i02 + args.nb03*i03); + + // initialize indices + shmem_i32[col] = i00 + col; + + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (int k = 2; k <= ntg.x; k *= 2) { + for (int j = k / 2; j > 0; j /= 2) { + int ixj = col ^ j; + if (ixj > col) { + if ((col & k) == 0) { + if (shmem_i32[col] >= args.ne00 || + (shmem_i32[ixj] < args.ne00 && (order == GGML_SORT_ORDER_ASC ? + src0_row[shmem_i32[col]] > src0_row[shmem_i32[ixj]] : + src0_row[shmem_i32[col]] < src0_row[shmem_i32[ixj]])) + ) { + SWAP(shmem_i32[col], shmem_i32[ixj]); + } + } else { + if (shmem_i32[ixj] >= args.ne00 || + (shmem_i32[col] < args.ne00 && (order == GGML_SORT_ORDER_ASC ? + src0_row[shmem_i32[col]] < src0_row[shmem_i32[ixj]] : + src0_row[shmem_i32[col]] > src0_row[shmem_i32[ixj]])) + ) { + SWAP(shmem_i32[col], shmem_i32[ixj]); + } + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } + } + + const int64_t i0 = ib*args.top_k; + + // copy the result to dst without the padding + if (i0 + col < args.ne0 && col < args.top_k) { + dst += i0 + args.ne0*i01 + args.ne0*args.ne1*i02 + args.ne0*args.ne1*args.ne2*i03; + + dst[col] = shmem_i32[col]; + } +} + +template [[host_name("kernel_argsort_f32_i32_asc")]] kernel argsort_t kernel_argsort_f32_i32; +template [[host_name("kernel_argsort_f32_i32_desc")]] kernel argsort_t kernel_argsort_f32_i32; + +typedef void (argsort_merge_t)( + constant ggml_metal_kargs_argsort_merge & args, + device const char * src0, + device const int32_t * tmp, + device int32_t * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]); + +template +kernel void kernel_argsort_merge_f32_i32( + constant ggml_metal_kargs_argsort_merge & args, + device const char * src0, + device const int32_t * tmp, + device int32_t * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + + const int im = tgpig[0] / args.ne01; + const int i01 = tgpig[0] % args.ne01; + const int i02 = tgpig[1]; + const int i03 = tgpig[2]; + + const int start = im * (2 * args.len); + + const int len0 = MIN(args.len, MAX(0, args.ne0 - (int)(start))); + const int len1 = MIN(args.len, MAX(0, args.ne0 - (int)(start + args.len))); + + const int total = len0 + len1; + + device const int32_t * tmp0 = tmp + start + + i01*args.ne0 + + i02*args.ne0*args.ne01 + + i03*args.ne0*args.ne01*args.ne02; + + device const int32_t * tmp1 = tmp0 + args.len; + + dst += start + + i01*args.top_k + + i02*args.top_k*args.ne01 + + i03*args.top_k*args.ne01*args.ne02; + + device const float * src0_row = (device const float *)(src0 + + args.nb01*i01 + + args.nb02*i02 + + args.nb03*i03); + + if (total == 0) { + return; + } + + const int chunk = (total + ntg.x - 1) / ntg.x; + + const int k0 = tpitg.x * chunk; + const int k1 = MIN(MIN(k0 + chunk, total), args.top_k); + + if (k0 >= args.top_k) { + return; + } + + if (k0 >= total) { + return; + } + + int low = k0 > len1 ? k0 - len1 : 0; + int high = MIN(k0, len0); + + // binary-search partition (i, j) such that i + j = k + while (low < high) { + const int mid = (low + high) >> 1; + + const int32_t idx0 = tmp0[mid]; + const int32_t idx1 = tmp1[k0 - mid - 1]; + + const float val0 = src0_row[idx0]; + const float val1 = src0_row[idx1]; + + bool take_left; + if (order == GGML_SORT_ORDER_ASC) { + take_left = (val0 <= val1); + } else { + take_left = (val0 >= val1); + } + + if (take_left) { + low = mid + 1; + } else { + high = mid; + } + } + + int i = low; + int j = k0 - i; + + // keep the merge fronts into registers + int32_t idx0 = 0; + float val0 = 0.0f; + if (i < len0) { + idx0 = tmp0[i]; + val0 = src0_row[idx0]; + } + + int32_t idx1 = 0; + float val1 = 0.0f; + if (j < len1) { + idx1 = tmp1[j]; + val1 = src0_row[idx1]; + } + + for (int k = k0; k < k1; ++k) { + int32_t out_idx; + + if (i >= len0) { + while (k < k1) { + dst[k++] = tmp1[j++]; + } + break; + } else if (j >= len1) { + while (k < k1) { + dst[k++] = tmp0[i++]; + } + break; + } else { + bool take_left; + + if (order == GGML_SORT_ORDER_ASC) { + take_left = (val0 <= val1); + } else { + take_left = (val0 >= val1); + } + + if (take_left) { + out_idx = idx0; + ++i; + if (i < len0) { + idx0 = tmp0[i]; + val0 = src0_row[idx0]; + } + } else { + out_idx = idx1; + ++j; + if (j < len1) { + idx1 = tmp1[j]; + val1 = src0_row[idx1]; + } + } + } + + dst[k] = out_idx; + } +} + +template [[host_name("kernel_argsort_merge_f32_i32_asc")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32; +template [[host_name("kernel_argsort_merge_f32_i32_desc")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32; + +static inline uint ggml_top_k_f2ui(float x) { + uint y = as_type(x); + if ((y & 0x80000000u) != 0u) { + y ^= 0xFFFFFFFFu; // negative floats: flip all bits + } else { + y |= 0x80000000u; // positive floats: set the sign bit + } + return y; +} + +kernel void kernel_top_k_f32_i32( + constant ggml_metal_kargs_top_k & args, + device const char * src0, + device int32_t * dst, + threadgroup atomic_uint * histo [[threadgroup(0)]], + threadgroup uint * sh_bucket [[threadgroup(1)]], + threadgroup uint * sh_above [[threadgroup(2)]], + threadgroup atomic_uint * out_count [[threadgroup(3)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + + const uint ncols = args.ne00; + const uint top_k = args.top_k; + const uint i01 = tgpig[0]; + const uint i02 = tgpig[1]; + const uint i03 = tgpig[2]; + + device const float * src0_row = (device const float *) (src0 + args.nb01*i01 + args.nb02*i02 + args.nb03*i03); + + device int32_t * dst_row = dst + top_k*(i01 + args.ne01*i02 + args.ne01*args.ne02*i03); + + const uint tid = tpitg.x; + const uint ntg_x = ntg.x; + + uint prefix = 0; // fixed high bits of the threshold key + uint desired = top_k; // count still needed from the candidate range + + for (int shift = 24; shift >= 0; shift -= 8) { + for (uint i = tid; i < 256; i += ntg_x) { + atomic_store_explicit(&histo[i], 0u, memory_order_relaxed); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + const uint hi_mask = (shift + 8 >= 32) ? 0u : (0xFFFFFFFFu << uint(shift + 8)); + const uint prefix_hi = prefix & hi_mask; + + for (uint i = tid; i < ncols; i += ntg_x) { + const uint key = ggml_top_k_f2ui(src0_row[i]); + if ((key & hi_mask) == prefix_hi) { + atomic_fetch_add_explicit(&histo[(key >> uint(shift)) & 0xFFu], 1u, memory_order_relaxed); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + // top-down scan for the bucket holding the k-th value + if (tid == 0) { + uint acc = 0; + uint b = 0; + for (int bb = 255; bb >= 0; --bb) { + const uint c = atomic_load_explicit(&histo[bb], memory_order_relaxed); + if (acc + c >= desired) { + b = uint(bb); + break; + } + acc += c; + } + *sh_bucket = b; + *sh_above = acc; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + prefix |= *sh_bucket << uint(shift); + desired -= *sh_above; + + // ensure every thread has consumed sh_bucket/sh_above before the next pass + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + if (tid == 0) { + atomic_store_explicit(out_count, 0u, memory_order_relaxed); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + // emit everything above the threshold, then fill the rest from ties + const uint threshold = prefix; + + for (uint i = tid; i < ncols; i += ntg_x) { + if (ggml_top_k_f2ui(src0_row[i]) > threshold) { + const uint pos = atomic_fetch_add_explicit(out_count, 1u, memory_order_relaxed); + dst_row[pos] = (int32_t) i; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint i = tid; i < ncols; i += ntg_x) { + if (ggml_top_k_f2ui(src0_row[i]) == threshold) { + const uint pos = atomic_fetch_add_explicit(out_count, 1u, memory_order_relaxed); + if (pos < top_k) { + dst_row[pos] = (int32_t) i; + } + } + } +} + +// fused SOFT_MAX + top-k + GET_ROWS (+ optional norm/scale) for MoE routing. +// One SIMDgroup handles one token row; n_expert is limited to 1024 by the host. +kernel void kernel_topk_moe_f32( + constant ggml_metal_kargs_topk_moe & args, + device const char * src0, + device float * weights, + device int32_t * ids, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]]) { + const int row = (int) tgpig.x; + if (row >= args.ne01) { + return; + } + + const int n_expert = FC_topk_moe_n_expert; + const int top_k = FC_topk_moe_top_k; + const int lane = (int) tiisg; + const int n_per_lane = (n_expert + 31) / 32; + + device const float * logits_row = (device const float *) (src0 + row * args.nb01); + device float * weights_row = weights + row * top_k; + device int32_t * ids_row = ids + row * (args.nb1_ids / sizeof(int32_t)); + + float wt[32]; + float output_weights[32]; + FOR_UNROLL (int i = 0; i < 32; ++i) { + wt[i] = -INFINITY; + output_weights[i] = 0.0f; + } + + for (int i = lane; i < n_expert; i += 32) { + const float v = logits_row[i]; + wt[i / 32] = isnan(v) ? -FLT_MAX : v; + } + + // softmax over the expert logits + float max_val = -INFINITY; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + max_val = max(max_val, wt[i]); + } + max_val = simd_max(max_val); + + float sum_val = 0.0f; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] = exp(wt[i] - max_val); + sum_val += wt[i]; + } + sum_val = simd_sum(sum_val); + + const float inv_sum = 1.0f / sum_val; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + wt[i] *= inv_sum; + } + + float wt_sum = 0.0f; + + for (int k = 0; k < top_k; ++k) { + float best_val = -INFINITY; + int best_expert = -1; + + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + const int expert = lane + i * 32; + if (expert < n_expert && (wt[i] > best_val || (wt[i] == best_val && expert < best_expert))) { + best_val = wt[i]; + best_expert = expert; + } + } + + FOR_UNROLL (int mask = 16; mask > 0; mask >>= 1) { + const float val = simd_shuffle_xor(best_val, mask); + const int expert = simd_shuffle_xor(best_expert, mask); + if (val > best_val || (val == best_val && expert < best_expert)) { + best_val = val; + best_expert = expert; + } + } + + if ((best_expert & 31) == lane) { + wt[best_expert / 32] = -INFINITY; + } + + if ((k & 31) == lane) { + output_weights[k / 32] = best_val; + } + + if ((best_expert & 31) == lane) { + ids_row[k] = best_expert; + if (FC_topk_moe_with_norm) { + wt_sum += best_val; + } + } + } + + if (FC_topk_moe_with_norm) { + wt_sum = simd_sum(wt_sum); + wt_sum = max(wt_sum, args.clamp); + const float inv = 1.0f / wt_sum; + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + output_weights[i] *= inv; + } + } + + FOR_UNROLL (int i = 0; i < n_per_lane; ++i) { + const int idx = i * 32 + lane; + if (idx < top_k) { + weights_row[idx] = output_weights[i] * args.scale; + } + } +} + +// fused MoE expert weighting + reduction: weighted = sum(experts[e] * weights[e]). +// The host guarantees all tensors are contiguous F32. +kernel void kernel_moe_reduce_f32( + constant ggml_metal_kargs_moe_reduce & args, + device const float * experts, + device const float * weights, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int64_t token = tgpig.x; + const int64_t col = (int64_t) tgpig.y * ntg.x + tpitg.x; + if (token >= args.ne02 || col >= args.ne00) { + return; + } + + const int n_expert_used = FC_moe_reduce_n_expert_used; + + const int64_t base = token * (int64_t) n_expert_used * args.ne00 + col; + float sum = 0.0f; + FOR_UNROLL (int e = 0; e < n_expert_used; ++e) { + sum += experts[base + e * args.ne00] * weights[token * n_expert_used + e]; + } + dst[token * args.ne00 + col] = sum; +} diff --git a/ggml/src/ggml-metal/kernels/binbcast.metal b/ggml/src/ggml-metal/kernels/binbcast.metal new file mode 100644 index 0000000..b367b3e --- /dev/null +++ b/ggml/src/ggml-metal/kernels/binbcast.metal @@ -0,0 +1,236 @@ +#include "common.h" + +// OP: 0 - add, 1 - sub, 2 - mul, 3 - div +constant short FC_bin_op [[function_constant(FC_BIN + 0)]]; +constant short FC_bin_f [[function_constant(FC_BIN + 1)]]; +constant bool FC_bin_rb [[function_constant(FC_BIN + 2)]]; +constant bool FC_bin_cb [[function_constant(FC_BIN + 3)]]; + +template +kernel void kernel_bin_fuse_impl( + constant ggml_metal_kargs_bin & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { +#define FC_OP FC_bin_op +#define FC_F FC_bin_f +#define FC_RB FC_bin_rb +#define FC_CB FC_bin_cb + + if (FC_RB) { + // row broadcast + const uint i0 = tgpig.y*args.ne00 + tgpig.x; + const uint i1 = FC_CB ? tgpig.x%args.ne10 : tgpig.x; + + device const T0 * src0_row = (device const T0 *) (src0); + device T * dst_row = (device T *) (dst); + + if (FC_F == 1) { + device const T1 * src1_row = (device const T1 *) (src1 + args.o1[0]); + + if (FC_OP == 0) { + dst_row[i0] = src0_row[i0] + src1_row[i1]; + } + + if (FC_OP == 1) { + dst_row[i0] = src0_row[i0] - src1_row[i1]; + } + + if (FC_OP == 2) { + dst_row[i0] = src0_row[i0] * src1_row[i1]; + } + + if (FC_OP == 3) { + dst_row[i0] = src0_row[i0] / src1_row[i1]; + } + } else { + T0 res = src0_row[i0]; + + if (FC_OP == 0) { + FOR_UNROLL (short j = 0; j < FC_F; ++j) { + res += ((device const T1 *) (src1 + args.o1[j]))[i1]; + } + } + + if (FC_OP == 1) { + FOR_UNROLL (short j = 0; j < FC_F; ++j) { + res -= ((device const T1 *) (src1 + args.o1[j]))[i1]; + } + } + + if (FC_OP == 2) { + FOR_UNROLL (short j = 0; j < FC_F; ++j) { + res *= ((device const T1 *) (src1 + args.o1[j]))[i1]; + } + } + + if (FC_OP == 3) { + FOR_UNROLL (short j = 0; j < FC_F; ++j) { + res /= ((device const T1 *) (src1 + args.o1[j]))[i1]; + } + } + + dst_row[i0] = res; + } + } else { + const int i03 = tgpig.z; + const int i02 = tgpig.y; + const int i01 = tgpig.x; + + if (i01 >= args.ne01) { + return; + } + + const int i13 = i03%args.ne13; + const int i12 = i02%args.ne12; + const int i11 = i01%args.ne11; + + device const T0 * src0_ptr = (device const T0 *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + args.offs); + device T * dst_ptr = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1 + args.offs); + + if (FC_F == 1) { + device const T1 * src1_ptr = (device const T1 *) (src1 + args.o1[0] + i13*args.nb13 + i12*args.nb12 + i11*args.nb11); + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + const int i10 = FC_CB ? i0%args.ne10 : i0; + + if (FC_OP == 0) { + dst_ptr[i0] = src0_ptr[i0] + src1_ptr[i10]; + } + + if (FC_OP == 1) { + dst_ptr[i0] = src0_ptr[i0] - src1_ptr[i10]; + } + + if (FC_OP == 2) { + dst_ptr[i0] = src0_ptr[i0] * src1_ptr[i10]; + } + + if (FC_OP == 3) { + dst_ptr[i0] = src0_ptr[i0] / src1_ptr[i10]; + } + } + } else { + device const T1 * src1_ptr[8]; + FOR_UNROLL (short j = 0; j < FC_F; ++j) { + src1_ptr[j] = (device const T1 *) (src1 + args.o1[j] + i13*args.nb13 + i12*args.nb12 + i11*args.nb11); + } + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + const int i10 = FC_CB ? i0%args.ne10 : i0; + + T res = src0_ptr[i0]; + + if (FC_OP == 0) { + FOR_UNROLL (short j = 0; j < FC_F; ++j) { + res += src1_ptr[j][i10]; + } + } + + if (FC_OP == 1) { + FOR_UNROLL (short j = 0; j < FC_F; ++j) { + res -= src1_ptr[j][i10]; + } + } + + if (FC_OP == 2) { + FOR_UNROLL (short j = 0; j < FC_F; ++j) { + res *= src1_ptr[j][i10]; + } + } + + if (FC_OP == 3) { + FOR_UNROLL (short j = 0; j < FC_F; ++j) { + res /= src1_ptr[j][i10]; + } + } + + dst_ptr[i0] = res; + } + } + } + +#undef FC_OP +#undef FC_F +#undef FC_RB +#undef FC_CB +} + +typedef decltype(kernel_bin_fuse_impl) kernel_bin_fuse_t; + +template [[host_name("kernel_bin_fuse_f32_f32_f32")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; +template [[host_name("kernel_bin_fuse_f32_f32_f32_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; +template [[host_name("kernel_bin_fuse_f16_f16_f16")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; +template [[host_name("kernel_bin_fuse_f16_f16_f16_4")]] kernel kernel_bin_fuse_t kernel_bin_fuse_impl; + +kernel void kernel_add_id( + constant ggml_metal_kargs_add_id & args, + device const char * src0, + device const char * src1, + device const char * src2, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int i1 = tgpig.x; + const int i2 = tgpig.y; + + const int i11 = *((device const int32_t *) (src2 + i1*sizeof(int32_t) + i2*args.nb21)); + + const size_t nb1 = args.ne0 * sizeof(float); + const size_t nb2 = args.ne1 * nb1; + + device float * dst_row = (device float *)((device char *)dst + i1*nb1 + i2*nb2); + device const float * src0_row = (device const float *)((device char *)src0 + i1*args.nb01 + i2*args.nb02); + + if (i11 == -1) { + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + dst_row[i0] = src0_row[i0]; + } + return; + } + + device const float * src1_row = (device const float *)((device char *)src1 + i11*args.nb11); + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + dst_row[i0] = src0_row[i0] + src1_row[i0]; + } +} + +template +kernel void kernel_repeat( + constant ggml_metal_kargs_repeat & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int i3 = tgpig.z; + const int i2 = tgpig.y; + const int i1 = tgpig.x; + + const int i03 = i3%args.ne03; + const int i02 = i2%args.ne02; + const int i01 = i1%args.ne01; + + device const char * src0_ptr = src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01; + device char * dst_ptr = dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1; + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + const int i00 = i0%args.ne00; + *((device T *)(dst_ptr + i0*args.nb0)) = *((device T *)(src0_ptr + i00*args.nb00)); + } +} + +typedef decltype(kernel_repeat) kernel_repeat_t; + +template [[host_name("kernel_repeat_f32")]] kernel kernel_repeat_t kernel_repeat; +template [[host_name("kernel_repeat_f16")]] kernel kernel_repeat_t kernel_repeat; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_repeat_bf16")]] kernel kernel_repeat_t kernel_repeat; +#endif +template [[host_name("kernel_repeat_i32")]] kernel kernel_repeat_t kernel_repeat; +template [[host_name("kernel_repeat_i16")]] kernel kernel_repeat_t kernel_repeat; diff --git a/ggml/src/ggml-metal/kernels/common.h b/ggml/src/ggml-metal/kernels/common.h new file mode 100644 index 0000000..c4d6743 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/common.h @@ -0,0 +1,126 @@ +#pragma once + +#include "ggml-metal-impl.h" + +#include + +#ifdef GGML_METAL_HAS_TENSOR +#include + +#include +#endif + +using namespace metal; + +#define MAX(x, y) ((x) > (y) ? (x) : (y)) +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; } + +#define PAD2(x, n) (((x) + (n) - 1) & ~((n) - 1)) + +#define FOR_UNROLL(x) _Pragma("clang loop unroll(full)") for (x) + +#define N_SIMDWIDTH 32 // assuming SIMD group size is 32 + +// ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf +// +// cmd: +// .../usr/bin/metal -dM -E -c ggml/src/ggml-metal/kernels/.metal +// .../usr/bin/metal -dM -E -c -target air64-apple-ios14.0 ggml/src/ggml-metal/kernels/.metal +// +#if __METAL_VERSION__ < 310 && defined(GGML_METAL_HAS_BF16) +#undef GGML_METAL_HAS_BF16 +#endif + +#if defined(GGML_METAL_HAS_BF16) +typedef matrix bfloat4x4; +typedef matrix bfloat2x4; +#endif + +constexpr constant static float kvalues_iq4nl_f[16] = { + -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f +}; + +constexpr constant static float kvalues_mxfp4_f[16] = { + 0, .5f, 1.f, 1.5f, 2.f, 3.f, 4.f, 6.f, -0, -.5f, -1.f, -1.5f, -2.f, -3.f, -4.f, -6.f +}; + +static inline int best_index_int8(int n, constant float * val, float x) { + if (x <= val[0]) return 0; + if (x >= val[n-1]) return n-1; + int ml = 0, mu = n-1; + while (mu-ml > 1) { + int mav = (ml+mu)/2; + if (x < val[mav]) mu = mav; else ml = mav; + } + return x - val[mu-1] < val[mu] - x ? mu-1 : mu; +} + +static inline float e8m0_to_fp32(uint8_t x) { + uint32_t bits; + + if (x == 0) { + bits = 0x00400000; + } else { + bits = (uint32_t) x << 23; + } + + return as_type(bits); +} + +static inline float dot(float x, float y) { + return x*y; +} + +static inline float sum(float x) { + return x; +} + +static inline float sum(float4 x) { + return x[0] + x[1] + x[2] + x[3]; +} + +enum ggml_sort_order { + GGML_SORT_ORDER_ASC, + GGML_SORT_ORDER_DESC, +}; + +constant float GELU_COEF_A = 0.044715f; +constant float GELU_QUICK_COEF = -1.702f; +constant float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; +constant float SQRT_2_INV = 0.70710678118654752440084436210484f; + +// based on Abramowitz and Stegun formula 7.1.26 or similar Hastings' approximation +// ref: https://www.johndcook.com/blog/python_erf/ +constant float p_erf = 0.3275911f; +constant float a1_erf = 0.254829592f; +constant float a2_erf = -0.284496736f; +constant float a3_erf = 1.421413741f; +constant float a4_erf = -1.453152027f; +constant float a5_erf = 1.061405429f; + +template +inline T erf_approx(T x) { + T sign_x = sign(x); + x = fabs(x); + T t = 1.0f / (1.0f + p_erf * x); + T y = 1.0f - (((((a5_erf * t + a4_erf) * t) + a3_erf) * t + a2_erf) * t + a1_erf) * t * exp(-x * x); + return sign_x * y; +} + +template T elu_approx(T x); + +template<> inline float elu_approx(float x) { + return (x > 0.f) ? x : (exp(x) - 1); +} + +template<> inline float4 elu_approx(float4 x) { + float4 res; + + res[0] = (x[0] > 0.0f) ? x[0] : (exp(x[0]) - 1.0f); + res[1] = (x[1] > 0.0f) ? x[1] : (exp(x[1]) - 1.0f); + res[2] = (x[2] > 0.0f) ? x[2] : (exp(x[2]) - 1.0f); + res[3] = (x[3] > 0.0f) ? x[3] : (exp(x[3]) - 1.0f); + + return res; +} diff --git a/ggml/src/ggml-metal/kernels/conv.metal b/ggml/src/ggml-metal/kernels/conv.metal new file mode 100644 index 0000000..a5d5aa9 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/conv.metal @@ -0,0 +1,724 @@ +#include "common.h" + +typedef void (im2col_t)( + constant ggml_metal_kargs_im2col & args, + device const float * x, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template +kernel void kernel_im2col( + constant ggml_metal_kargs_im2col & args, + device const float * x, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { +// const int64_t IC = tgpg[0]; + const int64_t OH = tgpg[1]; + const int64_t OW = tgpg[2]; + + const int64_t KH = ntg[1]; + const int64_t KW = ntg[2]; + + int64_t in = tpitg[0]; + const int64_t ikh = tpitg[1]; + const int64_t ikw = tpitg[2]; + + const int64_t iic = tgpig[0]; + const int64_t ioh = tgpig[1]; + const int64_t iow = tgpig[2]; + + const int64_t iiw = iow*args.s0 + ikw*args.d0 - args.p0; + const int64_t iih = ioh*args.s1 + ikh*args.d1 - args.p1; + + int64_t offset_dst = (in*OH*OW + ioh*OW + iow)*args.CHW + (iic*(KH*KW) + ikh*KW + ikw); + + device T * pdst = (device T *) (dst); + + if (iih < 0 || iih >= args.IH || iiw < 0 || iiw >= args.IW) { + while (in < args.N) { + pdst[offset_dst] = 0.0f; + offset_dst += ntg[0]*args.CHW*OH*OW; + + in += ntg[0]; + } + } else { + int64_t offset_src = in*args.ofs0 + iic*args.ofs1 + iih*args.IW + iiw; + + while (in < args.N) { + pdst[offset_dst] = x[offset_src]; + + offset_dst += ntg[0]*args.CHW*OH*OW; + offset_src += ntg[0]*args.ofs0; + + in += ntg[0]; + } + } +} + +template [[host_name("kernel_im2col_f32")]] kernel im2col_t kernel_im2col; +template [[host_name("kernel_im2col_f16")]] kernel im2col_t kernel_im2col; + +// TODO: optimize +typedef void (im2col_ext_t)( + constant ggml_metal_kargs_im2col & args, + device const float * x, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template +kernel void kernel_im2col_ext( + constant ggml_metal_kargs_im2col & args, + device const float * x, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]], // tgpg[0] = D x IC x KH x KW, CHW = IC x KH x KW + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { // [M, 1, 1] + const int64_t KHW = (int64_t)args.KHW; + + const int64_t d = tgpig[0] / args.CHW; + const int64_t chw = tgpig[0] % args.CHW; + const int64_t tgpig_0 = chw / KHW; // 0 ~ (IC - 1) + const int64_t HW = tgpig[0] % KHW; + + const int64_t tpitg_0 = (d * ntg[0]) + tpitg[0]; + if (tpitg_0 >= args.N) { + return; + } + + const int64_t tpitg_1 = HW / args.KW; + const int64_t tpitg_2 = HW % args.KW; + + const int64_t iiw = tgpig[2] * args.s0 + tpitg_2 * args.d0 - args.p0; + const int64_t iih = tgpig[1] * args.s1 + tpitg_1 * args.d1 - args.p1; + + const int64_t offset_dst = + (tpitg_0 * tgpg[1] * tgpg[2] + tgpig[1] * tgpg[2] + tgpig[2]) * args.CHW + + (tgpig_0 * KHW + tpitg_1 * args.KW + tpitg_2); + + device T * pdst = (device T *) (dst); + + if (iih < 0 || iih >= args.IH || iiw < 0 || iiw >= args.IW) { + pdst[offset_dst] = 0.0f; + } else { + const int64_t offset_src = tpitg_0 * args.ofs0 + tgpig_0 * args.ofs1; + pdst[offset_dst] = x[offset_src + iih * args.IW + iiw]; + } +} + +template [[host_name("kernel_im2col_ext_f32")]] kernel im2col_ext_t kernel_im2col_ext; +template [[host_name("kernel_im2col_ext_f16")]] kernel im2col_ext_t kernel_im2col_ext; + +template +kernel void kernel_col2im_1d( + constant ggml_metal_kargs_col2im_1d & args, + device const T * col, + device T * dst, + uint tgpig [[threadgroup_position_in_grid]], + uint tpitg [[thread_position_in_threadgroup]], + uint ntg [[threads_per_threadgroup]]) { + + const int idx = tgpig * ntg + tpitg; + if (idx >= args.T_out * args.OC) { + return; + } + + const int t_out = idx % args.T_out; + const int oc = idx / args.T_out; + const int t_abs = t_out + args.p0; // absolute position in uncropped signal + + int t_in_min = (t_abs - args.K + args.s0) / args.s0; // ceil((t_abs - K + 1) / s0) + if (t_in_min < 0) { + t_in_min = 0; + } + int t_in_max = t_abs / args.s0; + if (t_in_max >= args.T_in) { + t_in_max = args.T_in - 1; + } + + float sum = 0.0f; + for (int t_in = t_in_min; t_in <= t_in_max; t_in++) { + const int k = t_abs - t_in * args.s0; + sum += float(col[(oc * args.K + k) + t_in * args.K_OC]); + } + + dst[t_out + oc * args.T_out] = T(sum); +} + +template [[host_name("kernel_col2im_1d_f32")]] kernel void kernel_col2im_1d(constant ggml_metal_kargs_col2im_1d &, device const float *, device float *, uint, uint, uint); +template [[host_name("kernel_col2im_1d_f16")]] kernel void kernel_col2im_1d(constant ggml_metal_kargs_col2im_1d &, device const half *, device half *, uint, uint, uint); +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_col2im_1d_bf16")]] kernel void kernel_col2im_1d(constant ggml_metal_kargs_col2im_1d &, device const bfloat *, device bfloat *, uint, uint, uint); +#endif + +template +kernel void kernel_conv_2d( + constant ggml_metal_kargs_conv_2d & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const uint threads_per_tg = ntg.x * ntg.y * ntg.z; + const uint tg_index = (tgpig.z * tgpg.y + tgpig.y) * tgpg.x + tgpig.x; + const uint local_thread = tpitg.z * (ntg.x * ntg.y) + tpitg.y * ntg.x + tpitg.x; + const uint thread_index = tg_index * threads_per_tg + local_thread; + const uint64_t total_threads = (uint64_t) threads_per_tg * tgpg.x * tgpg.y * tgpg.z; + const uint64_t total_outputs = (uint64_t) args.N * args.OC * args.OH * args.OW; + + for (uint64_t index = thread_index; index < total_outputs; index += total_threads) { + uint64_t tmp = index; + + const int32_t ow = tmp % args.OW; tmp /= args.OW; + const int32_t oh = tmp % args.OH; tmp /= args.OH; + const int32_t oc = tmp % args.OC; tmp /= args.OC; + const int32_t n = tmp; + + float acc = 0.0f; + + const int32_t base_x = ow*args.s0 - args.p0; + const int32_t base_y = oh*args.s1 - args.p1; + + int32_t ky_start = 0; + if (base_y < 0) { + ky_start = (-base_y + args.d1 - 1)/args.d1; + } + int32_t ky_end = args.KH; + const int32_t y_max = args.IH - 1 - base_y; + if (y_max < 0) { + ky_end = ky_start; + } else if (base_y + (args.KH - 1)*args.d1 >= args.IH) { + ky_end = min(ky_end, y_max/args.d1 + 1); + } + + int32_t kx_start = 0; + if (base_x < 0) { + kx_start = (-base_x + args.d0 - 1)/args.d0; + } + int32_t kx_end = args.KW; + const int32_t x_max = args.IW - 1 - base_x; + if (x_max < 0) { + kx_end = kx_start; + } else if (base_x + (args.KW - 1)*args.d0 >= args.IW) { + kx_end = min(kx_end, x_max/args.d0 + 1); + } + + if (ky_start < ky_end && kx_start < kx_end) { + const uint64_t src_base_n = (uint64_t) n * args.nb13; + const uint64_t w_base_oc = (uint64_t) oc * args.nb03; + + for (int32_t ic = 0; ic < args.IC; ++ic) { + const uint64_t src_base_nc = src_base_n + (uint64_t) ic * args.nb12; + const uint64_t w_base_ocic = w_base_oc + (uint64_t) ic * args.nb02; + + for (int32_t ky = ky_start; ky < ky_end; ++ky) { + const int32_t iy = base_y + ky*args.d1; + const uint64_t src_base_row = src_base_nc + (uint64_t) iy * args.nb11; + const uint64_t w_base_row = w_base_ocic + (uint64_t) ky * args.nb01; + + for (int32_t kx = kx_start; kx < kx_end; ++kx) { + const int32_t ix = base_x + kx*args.d0; + const uint64_t src_offs = src_base_row + (uint64_t) ix * args.nb10; + const uint64_t w_offs = w_base_row + (uint64_t) kx * args.nb00; + + const float x = *(device const float *)(src + src_offs); + const float w = (float) (*(device const TK *)(weights + w_offs)); + + acc += x * w; + } + } + } + } + + const uint64_t dst_offs = + (uint64_t) n * args.nb3 + + (uint64_t) oc * args.nb2 + + (uint64_t) oh * args.nb1 + + (uint64_t) ow * args.nb0; + + *(device float *)(dst + dst_offs) = acc; + } +} + +template [[host_name("kernel_conv_2d_f32_f32")]] +kernel void kernel_conv_2d( + constant ggml_metal_kargs_conv_2d & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template [[host_name("kernel_conv_2d_f16_f32")]] +kernel void kernel_conv_2d( + constant ggml_metal_kargs_conv_2d & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +typedef void (conv_transpose_1d_t)( + constant ggml_metal_kargs_conv_transpose_1d & args, + device const float * src0, + device const float * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]]); + +template +kernel void kernel_conv_transpose_1d( + constant ggml_metal_kargs_conv_transpose_1d & args, + device const T * src0, + device const float * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]]) { + + // For output position j on the time axis, only input positions + // i such that i*s0 <= j < i*s0 + K + // contribute -- i.e. i in [ceil((j - K + 1)/s0), floor(j/s0)] + // intersected with [0, IL-1]. That's at most ceil(K/s0) values + // (typically 2 for stride==K/2 transposed convs). + const int32_t j = tgpig[0]; + const int32_t s0 = args.s0; + const int32_t K = args.K; + const int32_t IL = args.IL; + + int32_t i_min; + { + int32_t a = j - K + 1; + i_min = a <= 0 ? 0 : (a + s0 - 1) / s0; // ceil(a/s0) for a>0 + } + int32_t i_max = j / s0; + if (i_max > IL - 1) i_max = IL - 1; + + float v = 0.0f; + if (i_min <= i_max) { + for (int64_t c = 0; c < args.IC; c++) { + const int32_t kernel_offset = c * tgpg[1] * K + K * tgpig[1]; + const int32_t input_offset = c * IL; + + for (int32_t i = i_min; i <= i_max; i++) { + v += float(src0[kernel_offset + j - i * s0]) * src1[input_offset + i]; + } + } + } + + device float * dst_ptr = (device float *) (dst + tgpig[0] * args.nb0 + tgpig[1] * args.nb1); + + dst_ptr[0] = v; +} + +template [[host_name("kernel_conv_transpose_1d_f32_f32")]] +kernel void kernel_conv_transpose_1d( + constant ggml_metal_kargs_conv_transpose_1d & args, + device const float * src0, + device const float * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]]); + +template [[host_name("kernel_conv_transpose_1d_f16_f32")]] +kernel void kernel_conv_transpose_1d( + constant ggml_metal_kargs_conv_transpose_1d & args, + device const half * src0, + device const float * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]]); + + +typedef void (conv_transpose_2d_t)( + constant ggml_metal_kargs_conv_transpose_2d & args, + device const float * src0, + device const float * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]]); + +template +kernel void kernel_conv_transpose_2d( + constant ggml_metal_kargs_conv_transpose_2d & args, + device const T * src0, + device const float * src1, + device char * dst, + threadgroup float * shared_sum [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int64_t out_x = tgpig[0]; + const int64_t out_y = tgpig[1]; + const int64_t batch = tgpig[2] / args.OC; + const int64_t out_c = tgpig[2] % args.OC; + + const int64_t kw = tpitg[0]; + const int64_t kh = tpitg[1]; + + float v = 0.0f; + + for (int64_t in_c = 0; in_c < args.IC; in_c++) { + int64_t in_y = out_y - kh; + + if (in_y < 0 || in_y % args.s0) continue; + + in_y /= args.s0; + + if (in_y >= args.IH) continue; + + int64_t in_x = out_x - kw; + + if (in_x < 0 || in_x % args.s0) continue; + + in_x /= args.s0; + + if (in_x >= args.IW) continue; + + const int64_t input_idx = (args.IW * args.IH) * (args.IC * batch + in_c) + (args.IW) * in_y + in_x; + const int64_t kernel_idx = (args.KH * args.KW * args.OC) * in_c + (args.KH * args.KW) * out_c + (args.KW) * kh + kw; + + v += (float)src0[kernel_idx] * src1[input_idx]; + } + + const uint tid = tpitg.y * ntg.x + tpitg.x; + shared_sum[tid] = v; + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tid == 0) { + float total = 0.0f; + const uint num_threads = ntg.x * ntg.y; + for (uint i = 0; i < num_threads; i++) { + total += shared_sum[i]; + } + + device float * dst_ptr = (device float *) (dst + batch*args.nb3 + out_c*args.nb2 + out_y * args.nb1 + out_x*args.nb0); + dst_ptr[0] = total; + } +} + +template [[host_name("kernel_conv_transpose_2d_f32_f32")]] +kernel void kernel_conv_transpose_2d( + constant ggml_metal_kargs_conv_transpose_2d & args, + device const float * src0, + device const float * src1, + device char * dst, + threadgroup float * shared_sum [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template [[host_name("kernel_conv_transpose_2d_f16_f32")]] +kernel void kernel_conv_transpose_2d( + constant ggml_metal_kargs_conv_transpose_2d & args, + device const half * src0, + device const float * src1, + device char * dst, + threadgroup float * shared_sum [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +// grid: x = C tile, y = OH, z = OW * N (for channel-contiguous layouts) +template +kernel void kernel_conv_2d_dw_tiled( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int32_t c = (int32_t)(tgpig.x * ntg.x + tpitg.x); + if (c >= args.C) { + return; + } + + const int32_t oh = tgpig.y; + const int32_t own = tgpig.z; + const int32_t ow = own % args.OW; + const int32_t n = own / args.OW; + + const int32_t base_y = oh*args.s1 - args.p1; + + int32_t ky_start = 0; + if (base_y < 0) { + ky_start = (-base_y + args.d1 - 1)/args.d1; + } + int32_t ky_end = args.KH; + const int32_t y_max = args.IH - 1 - base_y; + if (y_max < 0) { + ky_end = ky_start; + } else if (base_y + (args.KH - 1)*args.d1 >= args.IH) { + ky_end = min(ky_end, y_max/args.d1 + 1); + } + + const int32_t base_x = ow*args.s0 - args.p0; + + int32_t kx_start = 0; + if (base_x < 0) { + kx_start = (-base_x + args.d0 - 1)/args.d0; + } + int32_t kx_end = args.KW; + const int32_t x_max = args.IW - 1 - base_x; + if (x_max < 0) { + kx_end = kx_start; + } else if (base_x + (args.KW - 1)*args.d0 >= args.IW) { + kx_end = min(kx_end, x_max/args.d0 + 1); + } + + float acc = 0.0f; + + if (ky_start < ky_end && kx_start < kx_end) { + const uint64_t w_base = (uint64_t) c * args.nb02; + const uint64_t src_base = (uint64_t) n * args.nb13 + (uint64_t) c * args.nb12; + + for (int32_t ky = ky_start; ky < ky_end; ++ky) { + const int32_t iy = base_y + ky*args.d1; + const uint64_t src_row = src_base + (uint64_t) iy * args.nb11; + const uint64_t w_row = w_base + (uint64_t) ky * args.nb01; + + for (int32_t kx = kx_start; kx < kx_end; ++kx) { + const int32_t ix = base_x + kx*args.d0; + const float x = *(device const float *)(src + src_row + (uint64_t) ix * args.nb10); + const float w = (float)(*(device const TK *)(weights + w_row + (uint64_t) kx * args.nb00)); + acc += x * w; + } + } + } + + const uint64_t dst_offs = + (uint64_t) n * args.nb3 + + (uint64_t) c * args.nb2 + + (uint64_t) oh * args.nb1 + + (uint64_t) ow * args.nb0; + + *(device float *)(dst + dst_offs) = acc; +} + +// grid: x = OW tile, y = OH, z = C * N (for spatially-contiguous layouts) +template +kernel void kernel_conv_2d_dw( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int32_t oh = tgpig.y; + const int32_t cn = tgpig.z; + const int32_t c = cn % args.C; + const int32_t n = cn / args.C; + + const int32_t base_y = oh*args.s1 - args.p1; + + int32_t ky_start = 0; + if (base_y < 0) { + ky_start = (-base_y + args.d1 - 1)/args.d1; + } + int32_t ky_end = args.KH; + const int32_t y_max = args.IH - 1 - base_y; + if (y_max < 0) { + ky_end = ky_start; + } else if (base_y + (args.KH - 1)*args.d1 >= args.IH) { + ky_end = min(ky_end, y_max/args.d1 + 1); + } + + const uint64_t w_base = (uint64_t) c * args.nb02; + const uint64_t src_base = (uint64_t) n * args.nb13 + (uint64_t) c * args.nb12; + + const int32_t ow = (int32_t)(tgpig.x * ntg.x + tpitg.x); + if (ow >= args.OW) { + return; + } + + float acc = 0.0f; + + const int32_t base_x = ow*args.s0 - args.p0; + + int32_t kx_start = 0; + if (base_x < 0) { + kx_start = (-base_x + args.d0 - 1)/args.d0; + } + int32_t kx_end = args.KW; + const int32_t x_max = args.IW - 1 - base_x; + if (x_max < 0) { + kx_end = kx_start; + } else if (base_x + (args.KW - 1)*args.d0 >= args.IW) { + kx_end = min(kx_end, x_max/args.d0 + 1); + } + + if (ky_start < ky_end && kx_start < kx_end) { + for (int32_t ky = ky_start; ky < ky_end; ++ky) { + const int32_t iy = base_y + ky*args.d1; + const uint64_t src_row = src_base + (uint64_t) iy * args.nb11; + const uint64_t w_row = w_base + (uint64_t) ky * args.nb01; + + for (int32_t kx = kx_start; kx < kx_end; ++kx) { + const int32_t ix = base_x + kx*args.d0; + const float x = *(device const float *)(src + src_row + (uint64_t) ix * args.nb10); + const float w = (float)(*(device const TK *)(weights + w_row + (uint64_t) kx * args.nb00)); + acc += x * w; + } + } + } + + const uint64_t dst_offs = + (uint64_t) n * args.nb3 + + (uint64_t) c * args.nb2 + + (uint64_t) oh * args.nb1 + + (uint64_t) ow * args.nb0; + + *(device float *)(dst + dst_offs) = acc; +} + +template [[host_name("kernel_conv_2d_dw_f32_f32")]] +kernel void kernel_conv_2d_dw( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template [[host_name("kernel_conv_2d_dw_f16_f32")]] +kernel void kernel_conv_2d_dw( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template [[host_name("kernel_conv_2d_dw_tiled_f32_f32")]] +kernel void kernel_conv_2d_dw_tiled( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template [[host_name("kernel_conv_2d_dw_tiled_f16_f32")]] +kernel void kernel_conv_2d_dw_tiled( + constant ggml_metal_kargs_conv_2d_dw & args, + device const char * weights, + device const char * src, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]); + +template +kernel void kernel_conv_3d( + constant ggml_metal_kargs_conv_3d & args, + device const char * src0, // Weights [IC * OC, KD, KH, KW] + device const char * src1, // Inputs [IC * N, ID, IH, IW] + device char * dst, // Outputs [OC * N, OD, OH, OW] + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]]) { + + // 1. Un-flatten the spatial dimension from Grid X + int64_t spatial_idx = tgpig.x * 32 + tpitg.x; + + if (spatial_idx >= args.OW * args.OH * args.OD) { + return; // Thread falls outside the spatial volume + } + + int64_t od = spatial_idx / (args.OW * args.OH); + int64_t oh = (spatial_idx / args.OW) % args.OH; + int64_t ow = spatial_idx % args.OW; + + // 2. Map Y to Channels, Z to Batch + int64_t oc = tgpig.y; + int64_t batch_idx = tgpig.z; + + // 3. Calculate anchor coordinates in the Input volume + int64_t i_w_base = ow * args.s0 - args.p0; + int64_t i_h_base = oh * args.s1 - args.p1; + int64_t i_d_base = od * args.s2 - args.p2; + + float sum = 0.0f; + + // 4. Gather Loop (Iterate over Input Channels -> Depth -> Height -> Width) + for (int64_t ic = 0; ic < args.IC; ++ic) { + + // ggml packs batch and channel together in the 4th dimension + int64_t src_cn_idx = batch_idx * args.IC + ic; + int64_t w_cn_idx = oc * args.IC + ic; + + for (int64_t kz = 0; kz < args.KD; ++kz) { + int64_t id = i_d_base + kz * args.d2; + if (id < 0 || id >= args.ID) continue; // Boundary check (Padding) + + for (int64_t ky = 0; ky < args.KH; ++ky) { + int64_t ih = i_h_base + ky * args.d1; + if (ih < 0 || ih >= args.IH) continue; + + for (int64_t kx = 0; kx < args.KW; ++kx) { + int64_t iw = i_w_base + kx * args.d0; + if (iw < 0 || iw >= args.IW) continue; + + // Convert multi-dimensional coordinates to flat byte offsets + int64_t w_idx = kx*args.nb00 + ky*args.nb01 + kz*args.nb02 + w_cn_idx*args.nb03; + int64_t i_idx = iw*args.nb10 + ih*args.nb11 + id*args.nb12 + src_cn_idx*args.nb13; + + // Dereference memory and cast weights to f32 if they were f16 + float w_val = (float)*(device const T*)((device const char*)src0 + w_idx); + float i_val = *(device const float*)((device const char*)src1 + i_idx); + + sum += w_val * i_val; + } + } + } + } + + // 5. Write the accumulated value out to RAM + int64_t dst_cn_idx = batch_idx * args.OC + oc; + int64_t d_idx = ow*args.nb0 + oh*args.nb1 + od*args.nb2 + dst_cn_idx*args.nb3; + + *(device float*)(dst + d_idx) = sum; +} + +// Explicit instantiations so the JIT compiler can find them by name +template [[host_name("kernel_conv_3d_f32_f32")]] +kernel void kernel_conv_3d( + constant ggml_metal_kargs_conv_3d & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]]); + +// Explicit instantiation for f16 weights +template [[host_name("kernel_conv_3d_f16_f32")]] +kernel void kernel_conv_3d( + constant ggml_metal_kargs_conv_3d & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]]); diff --git a/ggml/src/ggml-metal/kernels/dequantize.h b/ggml/src/ggml-metal/kernels/dequantize.h new file mode 100644 index 0000000..54e00fa --- /dev/null +++ b/ggml/src/ggml-metal/kernels/dequantize.h @@ -0,0 +1,739 @@ +#pragma once + +#include "common.h" + +#define GGML_COMMON_DECL_METAL +#define GGML_COMMON_IMPL_METAL +#if defined(GGML_METAL_EMBED_LIBRARY) +__embed_ggml-common.h__ +#else +#include "ggml-common.h" +#endif + +#define QK_NL 16 // shared by mul_mm and get_rows_q instantiations + +// NOTE: this is not dequantizing - we are simply fitting the template +template +void dequantize_f32(device const float4x4 * src, short il, thread type4x4 & reg) { + reg = (type4x4)(*src); +} + +template +void dequantize_f32_t4(device const float4 * src, short il, thread type4 & reg) { + reg = (type4)(*src); +} + +template +void dequantize_f16(device const half4x4 * src, short il, thread type4x4 & reg) { + reg = (type4x4)(*src); +} + +template +void dequantize_f16_t4(device const half4 * src, short il, thread type4 & reg) { + reg = (type4)(*(src)); +} + +#if defined(GGML_METAL_HAS_BF16) +template +void dequantize_bf16(device const bfloat4x4 * src, short il, thread type4x4 & reg) { + reg = (type4x4)(*src); +} + +template +void dequantize_bf16_t4(device const bfloat4 * src, short il, thread type4 & reg) { + reg = (type4)(*(src)); +} +#endif + +template +void dequantize_q1_0(device const block_q1_0 * xb, short il, thread type4x4 & reg) { + device const uint8_t * qs = xb->qs; + const float d = xb->d; + const float neg_d = -d; + + const int byte_offset = il * 2; // il*16 bits = il*2 bytes + const uint8_t b0 = qs[byte_offset]; + const uint8_t b1 = qs[byte_offset + 1]; + + float4x4 reg_f; + + reg_f[0][0] = select(neg_d, d, bool(b0 & 0x01)); + reg_f[0][1] = select(neg_d, d, bool(b0 & 0x02)); + reg_f[0][2] = select(neg_d, d, bool(b0 & 0x04)); + reg_f[0][3] = select(neg_d, d, bool(b0 & 0x08)); + reg_f[1][0] = select(neg_d, d, bool(b0 & 0x10)); + reg_f[1][1] = select(neg_d, d, bool(b0 & 0x20)); + reg_f[1][2] = select(neg_d, d, bool(b0 & 0x40)); + reg_f[1][3] = select(neg_d, d, bool(b0 & 0x80)); + + reg_f[2][0] = select(neg_d, d, bool(b1 & 0x01)); + reg_f[2][1] = select(neg_d, d, bool(b1 & 0x02)); + reg_f[2][2] = select(neg_d, d, bool(b1 & 0x04)); + reg_f[2][3] = select(neg_d, d, bool(b1 & 0x08)); + reg_f[3][0] = select(neg_d, d, bool(b1 & 0x10)); + reg_f[3][1] = select(neg_d, d, bool(b1 & 0x20)); + reg_f[3][2] = select(neg_d, d, bool(b1 & 0x40)); + reg_f[3][3] = select(neg_d, d, bool(b1 & 0x80)); + + reg = (type4x4) reg_f; +} + +template +void dequantize_q1_0_t4(device const block_q1_0 * xb, short il, thread type4 & reg) { + const float d = xb->d; + const float neg_d = -d; + const int base = il * 4; + const uint8_t byte = xb->qs[base / 8]; + const int s = base % 8; + + float4 reg_f; + reg_f[0] = select(neg_d, d, bool((byte >> (s )) & 1)); + reg_f[1] = select(neg_d, d, bool((byte >> (s + 1)) & 1)); + reg_f[2] = select(neg_d, d, bool((byte >> (s + 2)) & 1)); + reg_f[3] = select(neg_d, d, bool((byte >> (s + 3)) & 1)); + + reg = (type4) reg_f; +} + +template +void dequantize_q2_0(device const block_q2_0 * xb, short il, thread type4x4 & reg) { + device const uint8_t * qs = xb->qs; + const float d = xb->d; + + const int byte_offset = il * 4; // il*16 elements = il*4 bytes (4 elements per byte) + float4x4 reg_f; + + for (int i = 0; i < 4; i++) { + const uint8_t b = qs[byte_offset + i]; + reg_f[i][0] = ((float)((b >> 0) & 3) - 1.0f) * d; + reg_f[i][1] = ((float)((b >> 2) & 3) - 1.0f) * d; + reg_f[i][2] = ((float)((b >> 4) & 3) - 1.0f) * d; + reg_f[i][3] = ((float)((b >> 6) & 3) - 1.0f) * d; + } + + reg = (type4x4) reg_f; +} + +template +void dequantize_q2_0_t4(device const block_q2_0 * xb, short il, thread type4 & reg) { + const float d = xb->d; + const uint8_t b = xb->qs[il]; + + float4 reg_f; + reg_f[0] = ((float)((b >> 0) & 3) - 1.0f) * d; + reg_f[1] = ((float)((b >> 2) & 3) - 1.0f) * d; + reg_f[2] = ((float)((b >> 4) & 3) - 1.0f) * d; + reg_f[3] = ((float)((b >> 6) & 3) - 1.0f) * d; + + reg = (type4) reg_f; +} + +template +void dequantize_q4_0(device const block_q4_0 * xb, short il, thread type4x4 & reg) { + device const uint16_t * qs = ((device const uint16_t *)xb + 1); + const float d1 = il ? (xb->d / 16.h) : xb->d; + const float d2 = d1 / 256.f; + const float md = -8.h * xb->d; + const ushort mask0 = il ? 0x00F0 : 0x000F; + const ushort mask1 = mask0 << 8; + + float4x4 reg_f; + + for (int i = 0; i < 8; i++) { + reg_f[i/2][2*(i%2) + 0] = d1 * (qs[i] & mask0) + md; + reg_f[i/2][2*(i%2) + 1] = d2 * (qs[i] & mask1) + md; + } + + reg = (type4x4) reg_f; +} + +template +void dequantize_q4_0_t4(device const block_q4_0 * xb, short il, thread type4 & reg) { + device const uint16_t * qs = ((device const uint16_t *)xb + 1); + const float d1 = (il/4) ? (xb->d / 16.h) : xb->d; + const float d2 = d1 / 256.f; + const float md = -8.h * xb->d; + const ushort mask0 = (il/4) ? 0x00F0 : 0x000F; + const ushort mask1 = mask0 << 8; + + for (int i = 0; i < 2; i++) { + reg[2*i + 0] = d1 * (qs[2*(il%4) + i] & mask0) + md; + reg[2*i + 1] = d2 * (qs[2*(il%4) + i] & mask1) + md; + } +} + + + +template +void dequantize_q4_1(device const block_q4_1 * xb, short il, thread type4x4 & reg) { + device const uint16_t * qs = ((device const uint16_t *)xb + 2); + const float d1 = il ? (xb->d / 16.h) : xb->d; + const float d2 = d1 / 256.f; + const float m = xb->m; + const ushort mask0 = il ? 0x00F0 : 0x000F; + const ushort mask1 = mask0 << 8; + + float4x4 reg_f; + + for (int i = 0; i < 8; i++) { + reg_f[i/2][2*(i%2) + 0] = ((qs[i] & mask0) * d1) + m; + reg_f[i/2][2*(i%2) + 1] = ((qs[i] & mask1) * d2) + m; + } + + reg = (type4x4) reg_f; +} + +template +void dequantize_q4_1_t4(device const block_q4_1 * xb, short il, thread type4 & reg) { + device const uint16_t * qs = ((device const uint16_t *)xb + 2); + const float d1 = (il/4) ? (xb->d / 16.h) : xb->d; + const float d2 = d1 / 256.f; + const float m = xb->m; + const ushort mask0 = (il/4) ? 0x00F0 : 0x000F; + const ushort mask1 = mask0 << 8; + + for (int i = 0; i < 2; i++) { + reg[2*i + 0] = d1 * (qs[2*(il%4) + i] & mask0) + m; + reg[2*i + 1] = d2 * (qs[2*(il%4) + i] & mask1) + m; + } +} + +template +void dequantize_q5_0(device const block_q5_0 * xb, short il, thread type4x4 & reg) { + device const uint16_t * qs = ((device const uint16_t *)xb + 3); + const float d = xb->d; + const float md = -16.h * xb->d; + const ushort mask = il ? 0x00F0 : 0x000F; + + const uint32_t qh = *((device const uint32_t *)xb->qh); + + const int x_mv = il ? 4 : 0; + + const int gh_mv = il ? 12 : 0; + const int gh_bk = il ? 0 : 4; + + float4x4 reg_f; + + for (int i = 0; i < 8; i++) { + // extract the 5-th bits for x0 and x1 + const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10; + const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10; + + // combine the 4-bits from qs with the 5th bit + const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0); + const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1); + + reg_f[i/2][2*(i%2) + 0] = d * x0 + md; + reg_f[i/2][2*(i%2) + 1] = d * x1 + md; + } + + reg = (type4x4) reg_f; +} + +template +void dequantize_q5_0_t4(device const block_q5_0 * xb, short il, thread type4 & reg) { + device const uint16_t * qs = ((device const uint16_t *)xb + 3); + const float d = xb->d; + const float md = -16.h * xb->d; + const ushort mask = (il/4) ? 0x00F0 : 0x000F; + + const uint32_t qh = *((device const uint32_t *)xb->qh); + + const int x_mv = (il/4) ? 4 : 0; + + const int gh_mv = (il/4) ? 12 : 0; + const int gh_bk = (il/4) ? 0 : 4; + + for (int ii = 0; ii < 2; ii++) { + int i = 2*(il%4) + ii; + + // extract the 5-th bits for x0 and x1 + const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10; + const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10; + + // combine the 4-bits from qs with the 5th bit + const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0); + const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1); + + reg[2*ii + 0] = d * x0 + md; + reg[2*ii + 1] = d * x1 + md; + } +} + +template +void dequantize_q5_1(device const block_q5_1 * xb, short il, thread type4x4 & reg) { + device const uint16_t * qs = ((device const uint16_t *)xb + 4); + const float d = xb->d; + const float m = xb->m; + const ushort mask = il ? 0x00F0 : 0x000F; + + const uint32_t qh = *((device const uint32_t *)xb->qh); + + const int x_mv = il ? 4 : 0; + + const int gh_mv = il ? 12 : 0; + const int gh_bk = il ? 0 : 4; + + float4x4 reg_f; + + for (int i = 0; i < 8; i++) { + // extract the 5-th bits for x0 and x1 + const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10; + const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10; + + // combine the 4-bits from qs with the 5th bit + const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0); + const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1); + + reg_f[i/2][2*(i%2) + 0] = d * x0 + m; + reg_f[i/2][2*(i%2) + 1] = d * x1 + m; + } + + reg = (type4x4) reg_f; +} + +template +void dequantize_q5_1_t4(device const block_q5_1 * xb, short il, thread type4 & reg) { + device const uint16_t * qs = ((device const uint16_t *)xb + 4); + const float d = xb->d; + const float m = xb->m; + const ushort mask = (il/4) ? 0x00F0 : 0x000F; + + const uint32_t qh = *((device const uint32_t *)xb->qh); + + const int x_mv = (il/4) ? 4 : 0; + + const int gh_mv = (il/4) ? 12 : 0; + const int gh_bk = (il/4) ? 0 : 4; + + for (int ii = 0; ii < 2; ii++) { + int i = 2*(il%4) + ii; + + // extract the 5-th bits for x0 and x1 + const uint8_t xh_0 = ((qh >> (gh_mv + 2*i )) << gh_bk) & 0x10; + const uint8_t xh_1 = ((qh >> (gh_mv + 2*i+1)) << gh_bk) & 0x10; + + // combine the 4-bits from qs with the 5th bit + const int32_t x0 = ((((qs[i] ) & mask) >> x_mv) | xh_0); + const int32_t x1 = ((((qs[i] >> 8) & mask) >> x_mv) | xh_1); + + reg[2*ii + 0] = d * x0 + m; + reg[2*ii + 1] = d * x1 + m; + } +} + +template +void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & reg) { + device const packed_char4 * qs = (device const packed_char4 *) xb->qs; + const float d = xb->d; + + float4x4 reg_f; + + for (int i = 0; i < 4; ++i) { + reg_f[i] = float4(qs[4*il + i]) * d; + } + + reg = (type4x4) reg_f; +} + +template +void dequantize_q8_0_t4(device const block_q8_0 *xb, short il, thread type4 & reg) { + device const packed_char4 * qs = (device const packed_char4 *) xb->qs; + const float d = xb->d; + + reg = (type4) (float4(qs[il]) * d); +} + +template +void dequantize_mxfp4(device const block_mxfp4 * xb, short il, thread type4x4 & reg) { + device const uint8_t * q2 = (device const uint8_t *)xb->qs; + + const float d = e8m0_to_fp32(xb->e); + const uint8_t shr = il >= 1 ? 4 : 0; + + float4x4 reg_f; + + for (int i = 0; i < 4; ++i) { + reg_f[i][0] = d * kvalues_mxfp4_f[(q2[4*i + 0] >> shr) & 0x0F]; + reg_f[i][1] = d * kvalues_mxfp4_f[(q2[4*i + 1] >> shr) & 0x0F]; + reg_f[i][2] = d * kvalues_mxfp4_f[(q2[4*i + 2] >> shr) & 0x0F]; + reg_f[i][3] = d * kvalues_mxfp4_f[(q2[4*i + 3] >> shr) & 0x0F]; + } + + reg = (type4x4) reg_f; +} + +template +void dequantize_mxfp4_t4(device const block_mxfp4 * xb, short il, thread type4 & reg) { + device const uint8_t * q2 = (device const uint8_t *)xb->qs; + + const float d = e8m0_to_fp32(xb->e); + const short il4 = il%4; + + const uint8_t shr = il >= 4 ? 4 : 0; + + reg[0] = d * kvalues_mxfp4_f[(q2[4*il4 + 0] >> shr) & 0x0F]; + reg[1] = d * kvalues_mxfp4_f[(q2[4*il4 + 1] >> shr) & 0x0F]; + reg[2] = d * kvalues_mxfp4_f[(q2[4*il4 + 2] >> shr) & 0x0F]; + reg[3] = d * kvalues_mxfp4_f[(q2[4*il4 + 3] >> shr) & 0x0F]; +} + +template +void dequantize_q2_K(device const block_q2_K *xb, short il, thread type4x4 & reg) { + const float d = xb->d; + const float min = xb->dmin; + device const uint8_t * q = (device const uint8_t *)xb->qs; + float dl, ml; + uint8_t sc = xb->scales[il]; + + q = q + 32*(il/8) + 16*(il&1); + il = (il/2)%4; + + half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h); + uchar mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3); + dl = d * (sc & 0xF) * coef, ml = min * (sc >> 4); + for (int i = 0; i < 16; ++i) { + reg[i/4][i%4] = dl * (q[i] & mask) - ml; + } +} + +template +void dequantize_q3_K(device const block_q3_K *xb, short il, thread type4x4 & reg) { + const half d_all = xb->d; + device const uint8_t * q = (device const uint8_t *)xb->qs; + device const uint8_t * h = (device const uint8_t *)xb->hmask; + device const int8_t * scales = (device const int8_t *)xb->scales; + + q = q + 32 * (il/8) + 16 * (il&1); + h = h + 16 * (il&1); + uint8_t m = 1 << (il/2); + uint16_t kmask1 = (il/4)>1 ? ((il/4)>2 ? 192 : 48) : \ + ((il/4)>0 ? 12 : 3); + uint16_t kmask2 = il/8 ? 0xF0 : 0x0F; + uint16_t scale_2 = scales[il%8], scale_1 = scales[8 + il%4]; + int16_t dl_int = (il/4)&1 ? (scale_2&kmask2) | ((scale_1&kmask1) << 2) + : (scale_2&kmask2) | ((scale_1&kmask1) << 4); + float dl = il<8 ? d_all * (dl_int - 32.f) : d_all * (dl_int / 16.f - 32.f); + const float ml = 4.f * dl; + + il = (il/2) & 3; + const half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h); + const uint8_t mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3); + dl *= coef; + + for (int i = 0; i < 16; ++i) { + reg[i/4][i%4] = dl * (q[i] & mask) - (h[i] & m ? 0 : ml); + } +} + +static inline uchar2 get_scale_min_k4_just2(int j, int k, device const uchar * q) { + return j < 4 ? uchar2{uchar(q[j+0+k] & 63), uchar(q[j+4+k] & 63)} + : uchar2{uchar((q[j+4+k] & 0xF) | ((q[j-4+k] & 0xc0) >> 2)), uchar((q[j+4+k] >> 4) | ((q[j-0+k] & 0xc0) >> 2))}; +} + +template +void dequantize_q4_K(device const block_q4_K * xb, short il, thread type4x4 & reg) { + device const uchar * q = xb->qs; + + short is = (il/4) * 2; + q = q + (il/4) * 32 + 16 * (il&1); + il = il & 3; + const uchar2 sc = get_scale_min_k4_just2(is, il/2, xb->scales); + const float d = il < 2 ? xb->d : xb->d / 16.h; + const float min = xb->dmin; + const float dl = d * sc[0]; + const float ml = min * sc[1]; + + const ushort mask = il < 2 ? 0x0F : 0xF0; + for (int i = 0; i < 16; ++i) { + reg[i/4][i%4] = dl * (q[i] & mask) - ml; + } +} + +template +void dequantize_q5_K(device const block_q5_K *xb, short il, thread type4x4 & reg) { + device const uint8_t * q = xb->qs; + device const uint8_t * qh = xb->qh; + + short is = (il/4) * 2; + q = q + 32 * (il/4) + 16 * (il&1); + qh = qh + 16 * (il&1); + uint8_t ul = 1 << (il/2); + il = il & 3; + const uchar2 sc = get_scale_min_k4_just2(is, il/2, xb->scales); + const float d = il < 2 ? xb->d : xb->d / 16.f; + const float min = xb->dmin; + const float dl = d * sc[0]; + const float ml = min * sc[1]; + + const ushort mask = il<2 ? 0x0F : 0xF0; + const float qh_val = il<2 ? 16.f : 256.f; + for (int i = 0; i < 16; ++i) { + reg[i/4][i%4] = dl * ((q[i] & mask) + (qh[i] & ul ? qh_val : 0)) - ml; + } +} + +template +void dequantize_q6_K(device const block_q6_K *xb, short il, thread type4x4 & reg) { + const half d_all = xb->d; + device const uint16_t * ql = (device const uint16_t *)xb->ql; + device const uint16_t * qh = (device const uint16_t *)xb->qh; + device const int8_t * scales = (device const int8_t *)xb->scales; + + ql = ql + 32*(il/8) + 16*((il/2)&1) + 8*(il&1); + qh = qh + 16*(il/8) + 8*(il&1); + float sc = scales[(il%2) + 2 * ((il/2))]; + il = (il/2) & 3; + + const uint32_t kmask1 = il>1 ? (il>2 ? 0xC0C0C0C0 : 0x30303030) : (il>0 ? 0x0C0C0C0C : 0x03030303); + const uint32_t kmask2 = il>1 ? 0xF0F0F0F0 : 0x0F0F0F0F; + const float ml = d_all * sc * 32.f; + const float dl0 = d_all * sc; + const float dl1 = dl0 / 256.f; + const float dl2 = dl0 / (256.f * 256.f); + const float dl3 = dl0 / (256.f * 256.f * 256.f); + const uint8_t shr_h = il>2 ? 2 : 0; + const uint8_t shl_h = il>1 ? 0 : (il>0 ? 2 : 4); + const uint8_t shr_l = il>1 ? 4 : 0; + for (int i = 0; i < 4; ++i) { + const uint32_t low = (ql[2*i] | (uint32_t)(ql[2*i+1] << 16)) & kmask2; + const uint32_t high = (qh[2*i] | (uint32_t)(qh[2*i+1] << 16)) & kmask1; + const uint32_t q = ((high << shl_h) >> shr_h) | (low >> shr_l); + reg[i][0] = dl0 * ((half)(q & 0xFF)) - ml; + reg[i][1] = dl1 * ((float)(q & 0xFF00)) - ml; + reg[i][2] = dl2 * ((float)(q & 0xFF0000)) - ml; + reg[i][3] = dl3 * ((float)(q & 0xFF000000)) - ml; + } +} + +template +void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const float d = xb->d; + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + // each block of 32 needs 2 uint32_t's for the quants & scale, so 4 uint16_t's. + device const uint16_t * q2 = xb->qs + 4*ib32; + const uint32_t aux32_g = q2[0] | (q2[1] << 16); + const uint32_t aux32_s = q2[2] | (q2[3] << 16); + thread const uint8_t * aux8 = (thread const uint8_t *)&aux32_g; + const float dl = d * (0.5f + (aux32_s >> 28)) * 0.25f; + constant uint8_t * grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+0]); + uint8_t signs = ksigns_iq2xs[(aux32_s >> 14*il) & 127]; + for (int i = 0; i < 8; ++i) { + reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); + } + grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+1]); + signs = ksigns_iq2xs[(aux32_s >> (14*il+7)) & 127]; + for (int i = 0; i < 8; ++i) { + reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); + } +} + +template +void dequantize_iq2_xs(device const block_iq2_xs * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const float d = xb->d; + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + device const uint16_t * q2 = xb->qs + 4*ib32; + const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f; + constant uint8_t * grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+0] & 511)); + uint8_t signs = ksigns_iq2xs[q2[2*il+0] >> 9]; + for (int i = 0; i < 8; ++i) { + reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); + } + grid = (constant uint8_t *)(iq2xs_grid + (q2[2*il+1] & 511)); + signs = ksigns_iq2xs[q2[2*il+1] >> 9]; + for (int i = 0; i < 8; ++i) { + reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f); + } +} + +template +void dequantize_iq3_xxs(device const block_iq3_xxs * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const float d = xb->d; + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + device const uint8_t * q3 = xb->qs + 8*ib32; + device const uint16_t * gas = (device const uint16_t *)(xb->qs + QK_K/4) + 2*ib32; + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float dl = d * (0.5f + (aux32 >> 28)) * 0.5f; + constant uint8_t * grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+0]); + constant uint8_t * grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+1]); + uint8_t signs = ksigns_iq2xs[(aux32 >> 14*il) & 127]; + for (int i = 0; i < 4; ++i) { + reg[0][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f); + reg[1][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f); + } + grid1 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+2]); + grid2 = (constant uint8_t *)(iq3xxs_grid + q3[4*il+3]); + signs = ksigns_iq2xs[(aux32 >> (14*il+7)) & 127]; + for (int i = 0; i < 4; ++i) { + reg[2][i] = dl * grid1[i] * (signs & kmask_iq2xs[i+0] ? -1.f : 1.f); + reg[3][i] = dl * grid2[i] * (signs & kmask_iq2xs[i+4] ? -1.f : 1.f); + } +} + +template +void dequantize_iq3_s(device const block_iq3_s * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const float d = xb->d; + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + device const uint8_t * qs = xb->qs + 8*ib32; + device const uint8_t * signs = xb->signs + 4*ib32 + 2*il; + const uint8_t qh = xb->qh[ib32] >> 4*il; + const float dl = d * (1 + 2*((xb->scales[ib32/2] >> 4*(ib32%2)) & 0xf)); + constant uint8_t * grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+0] | ((qh << 8) & 256))); + constant uint8_t * grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+1] | ((qh << 7) & 256))); + for (int i = 0; i < 4; ++i) { + reg[0][i] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i+0]); + reg[1][i] = dl * grid2[i] * select(1, -1, signs[0] & kmask_iq2xs[i+4]); + } + grid1 = (constant uint8_t *)(iq3s_grid + (qs[4*il+2] | ((qh << 6) & 256))); + grid2 = (constant uint8_t *)(iq3s_grid + (qs[4*il+3] | ((qh << 5) & 256))); + for (int i = 0; i < 4; ++i) { + reg[2][i] = dl * grid1[i] * select(1, -1, signs[1] & kmask_iq2xs[i+0]); + reg[3][i] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i+4]); + } +} + +template +void dequantize_iq2_s(device const block_iq2_s * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const float d = xb->d; + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; + device const uint8_t * signs = qs + QK_K/8; + const uint8_t qh = xb->qh[ib32] >> 4*il; + const float dl = d * (0.5f + ((xb->scales[ib32] >> 4*il) & 0xf)) * 0.25f; + constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[0] | ((qh << 8) & 0x300))); + constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[1] | ((qh << 6) & 0x300))); + for (int i = 0; i < 8; ++i) { + reg[i/4+0][i%4] = dl * grid1[i] * select(1, -1, signs[0] & kmask_iq2xs[i]); + reg[i/4+2][i%4] = dl * grid2[i] * select(1, -1, signs[1] & kmask_iq2xs[i]); + } +} + +template +void dequantize_iq1_s(device const block_iq1_s * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const int ib32 = il/2; + il = il%2; + const float d = xb->d; + device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; + device const uint16_t * qh = xb->qh; + const float dl = d * (2*((qh[ib32] >> 12) & 7) + 1); + const float ml = dl * (qh[ib32] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA); + const uint16_t h = qh[ib32] >> 6*il; + constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((h << 8) & 0x700))); + constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((h << 5) & 0x700))); + for (int i = 0; i < 4; ++i) { + reg[0][i] = dl * (grid1[i] & 0xf) + ml; + reg[1][i] = dl * (grid1[i] >> 4) + ml; + reg[2][i] = dl * (grid2[i] & 0xf) + ml; + reg[3][i] = dl * (grid2[i] >> 4) + ml; + } +} + +template +void dequantize_iq1_m(device const block_iq1_m * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const int ib32 = il/2; + il = il%2; + device const uint16_t * sc = (device const uint16_t *)xb->scales; + + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); + const float d = scale.f16; + + device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; + device const uint8_t * qh = xb->qh + 2*ib32 + il; + + const float dl = d * (2*((sc[ib32/2] >> (6*(ib32%2)+3*il)) & 7) + 1); + const float ml1 = dl * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); + const float ml2 = dl * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); + constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); + constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700))); + for (int i = 0; i < 4; ++i) { + reg[0][i] = dl * (grid1[i] & 0xf) + ml1; + reg[1][i] = dl * (grid1[i] >> 4) + ml1; + reg[2][i] = dl * (grid2[i] & 0xf) + ml2; + reg[3][i] = dl * (grid2[i] >> 4) + ml2; + } +} + +template +void dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4 & reg) { + device const uint16_t * q4 = (device const uint16_t *)xb->qs; + const float d = xb->d; + uint32_t aux32; + thread const uint8_t * q8 = (thread const uint8_t *)&aux32; + for (int i = 0; i < 4; ++i) { + aux32 = ((q4[2*i] | (q4[2*i+1] << 16)) >> 4*il) & 0x0f0f0f0f; + reg[i][0] = d * kvalues_iq4nl_f[q8[0]]; + reg[i][1] = d * kvalues_iq4nl_f[q8[1]]; + reg[i][2] = d * kvalues_iq4nl_f[q8[2]]; + reg[i][3] = d * kvalues_iq4nl_f[q8[3]]; + } +} + +template +void dequantize_iq4_nl_t4(device const block_iq4_nl * xb, short il, thread type4 & reg) { + device const uint16_t * q4 = (device const uint16_t *)xb->qs; + const float d = xb->d; + uint32_t aux32; + thread const uint8_t * q8 = (thread const uint8_t *)&aux32; + aux32 = ((q4[2*(il%4)] | (q4[2*(il%4)+1] << 16)) >> 4*(il/4)) & 0x0f0f0f0f; + reg[0] = d * kvalues_iq4nl_f[q8[0]]; + reg[1] = d * kvalues_iq4nl_f[q8[1]]; + reg[2] = d * kvalues_iq4nl_f[q8[2]]; + reg[3] = d * kvalues_iq4nl_f[q8[3]]; +} + +template +void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 & reg) { + // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 + const int ib32 = il/2; + il = il%2; + // il = 0 or 1. il = 0 processes the first 16 quants in a block of 32, il = 1 the second 16 + device const uint32_t * q4 = (device const uint32_t *)xb->qs + 4*ib32; + const int ls = ((xb->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((xb->scales_h >> 2*ib32) & 3) << 4); + const float d = (float)xb->d * (ls - 32); + uint32_t aux32; + thread const uint8_t * q8 = (thread const uint8_t *)&aux32; + for (int i = 0; i < 4; ++i) { + aux32 = (q4[i] >> 4*il) & 0x0f0f0f0f; + reg[i][0] = d * kvalues_iq4nl_f[q8[0]]; + reg[i][1] = d * kvalues_iq4nl_f[q8[1]]; + reg[i][2] = d * kvalues_iq4nl_f[q8[2]]; + reg[i][3] = d * kvalues_iq4nl_f[q8[3]]; + } +} + +template +void dequantize_tq2_0(device const block_tq2_0 * xb, short il, thread type4x4 & reg) { + device const uint8_t * qs = xb->qs; + const float d = xb->d; + + float4x4 reg_f; + + // 2 bits per element, 4 elements per byte, 128 elements per 32-byte group + const short base = il * 16; + for (int k = 0; k < 16; k++) { + const int i = base + k; + const int byte = ((i >> 7) & 1) * 32 + (i & 31); + const int l = (i >> 5) & 3; + reg_f[k/4][k%4] = d * (float)(((qs[byte] >> (2*l)) & 3) - 1); + } + + reg = (type4x4) reg_f; +} diff --git a/ggml/src/ggml-metal/kernels/fa_aux.metal b/ggml/src/ggml-metal/kernels/fa_aux.metal new file mode 100644 index 0000000..ebee1be --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_aux.metal @@ -0,0 +1,485 @@ +#include "common.h" +#include "dequantize.h" + +// dequantize a quantized KV cache tensor to contiguous F16 before running the F16 flash attention kernels +// - one thread per block; dispatched separately for K and V +// - ref: https://github.com/ggml-org/llama.cpp/pull/27390 +template < + typename block_t, + short QK, + void (*deq_t4x4)(device const block_t *, short, thread float4x4 &)> +kernel void kernel_flash_attn_ext_kv_f16( + constant ggml_metal_kargs_flash_attn_ext_kv_f16 & args, + device const char * x, + device half * x_dst, + uint gid [[thread_position_in_grid]]) { + if (gid >= (uint) args.nblocks) { + return; + } + + const uint nb = args.ne0/QK; + const uint i0 = gid%nb; + uint ib = gid/nb; + const uint i1 = ib%args.ne1; + ib /= args.ne1; + const uint i2 = ib%args.ne2; + const uint i3 = ib/args.ne2; + + const uint64_t offs = i0*args.nb0 + i1*args.nb1 + i2*args.nb2 + i3*args.nb3; + + device const block_t * src = (device const block_t *) (x + offs); + device half4 * dst = (device half4 *) x_dst + (QK/4)*gid; + + for (short i = 0; i < QK/16; ++i) { + float4x4 reg; + deq_t4x4(src, i, reg); + dst[4*i + 0] = (half4) reg[0]; + dst[4*i + 1] = (half4) reg[1]; + dst[4*i + 2] = (half4) reg[2]; + dst[4*i + 3] = (half4) reg[3]; + } +} + +typedef decltype(kernel_flash_attn_ext_kv_f16) kernel_flash_attn_ext_kv_f16_t; + +template [[host_name("kernel_flash_attn_ext_kv_q4_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q4_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q5_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q5_1_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; +template [[host_name("kernel_flash_attn_ext_kv_q8_0_f16")]] kernel kernel_flash_attn_ext_kv_f16_t kernel_flash_attn_ext_kv_f16; + +constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]]; + +constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]]; + +// pad the last chunk of C elements of k and v into a an extra pad buffer +kernel void kernel_flash_attn_ext_pad( + constant ggml_metal_kargs_flash_attn_ext_pad & args, + device const char * k, + device const char * v, + device const char * mask, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int32_t C = FC_flash_attn_ext_pad_ncpsg; + + device char * k_pad = dst; + device char * v_pad = k_pad + args.nb11*C*args.ne_12_2*args.ne_12_3; + device char * mask_pad = v_pad + args.nb21*C*args.ne_12_2*args.ne_12_3; + + const int32_t icp = args.ne11 % C; + const int32_t ic0 = args.ne11 - icp; + + const int32_t i1 = tgpig[0]; + const int32_t i2 = tgpig[1]; + const int32_t i3 = tgpig[2]; + + if (i2 < args.ne_12_2 && i3 < args.ne_12_3) { + device const char * k_src = k + args.nb11*(ic0 + i1) + args.nb12*i2 + args.nb13*i3; + device const char * v_src = v + args.nb21*(ic0 + i1) + args.nb22*i2 + args.nb23*i3; + + device char * k_dst = k_pad + args.nb11*i1 + args.nb11*C*i2 + args.nb11*C*args.ne_12_2*i3; + device char * v_dst = v_pad + args.nb21*i1 + args.nb21*C*i2 + args.nb21*C*args.ne_12_2*i3; + + if (i1 >= icp) { + // here it is not important the exact value that will be used as we rely on masking out the scores in the attention + for (uint64_t i = tiitg; i < args.nb11; i += ntg.x) { + k_dst[i] = 0; + } + for (uint64_t i = tiitg; i < args.nb21; i += ntg.x) { + v_dst[i] = 0; + } + } else { + for (uint64_t i = tiitg; i < args.nb11; i += ntg.x) { + k_dst[i] = k_src[i]; + } + for (uint64_t i = tiitg; i < args.nb21; i += ntg.x) { + v_dst[i] = v_src[i]; + } + } + } + + if (FC_flash_attn_ext_pad_has_mask) { + if (i2 < args.ne32 && i3 < args.ne33) { + for (int ib = i1; ib < args.ne31; ib += C) { + device const half * mask_src = (device const half *)(mask + args.nb31*ib + args.nb32*i2 + args.nb33*i3) + ic0; + device half * mask_dst = (device half *)(mask_pad) + C*ib + C*args.ne31*i2 + C*args.ne31*args.ne32*i3; + + for (int i = tiitg; i < C; i += ntg.x) { + if (i >= icp) { + mask_dst[i] = -MAXHALF; + } else { + mask_dst[i] = mask_src[i]; + } + } + } + } + } +} + +constant int32_t FC_flash_attn_ext_blk_nqptg [[function_constant(FC_FLASH_ATTN_EXT_BLK + 24)]]; +constant int32_t FC_flash_attn_ext_blk_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_BLK + 25)]]; + +// scan the blocks of the mask that are not masked +// 0 - masked (i.e. full of -INF, skip) +// 1 - not masked (i.e. at least one element of the mask is not -INF) +// 2 - all zero +kernel void kernel_flash_attn_ext_blk( + constant ggml_metal_kargs_flash_attn_ext_blk & args, + device const char * mask, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]]) { + // block size C x Q + const int32_t Q = FC_flash_attn_ext_blk_nqptg; + const int32_t C = FC_flash_attn_ext_blk_ncpsg; + + constexpr short NW = N_SIMDWIDTH; + + const int32_t i3 = tgpig[2]/args.ne32; + const int32_t i2 = tgpig[2]%args.ne32; + const int32_t i1 = tgpig[1]; + const int32_t i0 = tgpig[0]; + + char res = i0*C + C > args.ne30 || i1*Q + Q > args.ne31 ? 1 : 0; + + device const half * mask_src = (device const half *) (mask + (i1*Q)*args.nb31 + i2*args.nb32 + i3*args.nb33) + i0*C + tiisg; + + // detailed check of the elements of the block + if ((C > NW || Q > 1) && res == 0) { + half mmin = MAXHALF; + half mmax = -MAXHALF; + + FOR_UNROLL (short j = 0; j < Q; ++j) { + FOR_UNROLL (short ii = 0; ii < C/NW; ++ii) { + mmin = min(mmin, mask_src[ii*NW]); + mmax = max(mmax, mask_src[ii*NW]); + } + + mask_src += args.nb31/2; + } + + mmin = simd_min(mmin); + mmax = simd_max(mmax); + + if (mmax > -MAXHALF) { + if (mmin == 0.0 && mmax == 0.0) { + res = 2; + } else { + res = 1; + } + } + } + + const int32_t nblk1 = ((args.ne01 + Q - 1)/Q); + const int32_t nblk0 = ((args.ne30 + C - 1)/C); + + if (tiisg == 0) { + dst[((i3*args.ne32 + i2)*nblk1 + i1)*nblk0 + i0] = res; + } +} +// compress the finite entries of each KQ mask row into a list of KV indices (ascending order), +// padded with -1 up to n_kv_max_padded (a multiple of OP_FLASH_ATTN_EXT_VEC_NCPSG) +// one threadgroup per mask row; the mask remains the single source of truth for the values +kernel void kernel_flash_attn_ext_vec_idx( + constant ggml_metal_kargs_flash_attn_ext_vec_idx & args, + device const half * mask, + device int * idx, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr short NW = N_SIMDWIDTH; + constexpr short NLOCAL = 32; // max finite positions kept in registers per thread + + const int i1 = tgpig[0]; + const int i2 = tgpig[1]; + const int i3 = tgpig[2]; + + device const half * pm = (device const half *) ((device const char *) mask + i1*args.nb31 + i2*args.nb32 + i3*args.nb33); + device int * pidx = idx + (((int64_t)i3*args.ne32 + i2)*args.ne31 + i1)*args.n_kv_max_padded; + + const int n = args.ne30; + const int q = n/ntg.x; + const int r = n%ntg.x; + + // each thread handles a contiguous slice of the mask row + const int r0 = q*tiitg + min((int) tiitg, r); + const int r1 = r0 + q + (tiitg < r ? 1 : 0); + + // count the finite entries in the slice and keep their positions in registers (single mask read) + int cnt = 0; // total finite entries in the slice + int nloc = 0; // finite entries kept in registers + int local[NLOCAL]; + for (int i = r0; i < r1; ++i) { + if (isfinite((float) pm[i])) { + if (nloc < NLOCAL) { + local[nloc] = i; + nloc++; + } + cnt++; + } + } + + const short sgitg = tiitg/NW; + const short tiisg = tiitg%NW; + + threadgroup int tcount[8]; + + // simd_sum is a collective: all lanes must evaluate it + const int sg_sum = simd_sum(cnt); + if (tiisg == 0) { + tcount[sgitg] = sg_sum; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + int total = 0; + for (short s = 0; s < ntg.x/NW; ++s) { + total += tcount[s]; + } + + // base offset of this thread's slice in the output list (exclusive scan within the simdgroup) + int sg_base = 0; + for (short s = 0; s < sgitg; ++s) { + sg_base += tcount[s]; + } + + // exclusive prefix scan of the per-thread counts within the simdgroup + int incl = cnt; + for (int d = 1; d < NW; d <<= 1) { + const int v = simd_shuffle_up(incl, d); + if (tiisg >= d) { + incl += v; + } + } + const int base = sg_base + (incl - cnt); + + // write the finite positions in order; if the hint is violated, keep only the first n_kv_max entries + int j = 0; + for (; j < nloc && base + j < args.n_kv_max; ++j) { + pidx[base + j] = local[j]; + } + + // a dense mask may have more than NLOCAL finite entries in a slice; re-read the mask to write the rest + if (cnt > nloc && base + nloc < args.n_kv_max) { + int j2 = 0; + for (int i = r0; i < r1; ++i) { + if (isfinite((float) pm[i])) { + if (j2 >= nloc) { + pidx[base + j2] = i; + } + j2++; + if (base + j2 >= args.n_kv_max) { + break; + } + } + } + } + + // pad the tail of the list with -1 + const int count = min(total, args.n_kv_max); + for (int i = count + tiitg; i < args.n_kv_max_padded; i += ntg.x) { + pidx[i] = -1; + } +} + +constant int32_t FC_flash_attn_ext_vec_reduce_DV [[function_constant(FC_FLASH_ATTN_EXT_VEC_REDUCE + 0)]]; +constant int32_t FC_flash_attn_ext_vec_reduce_NWG [[function_constant(FC_FLASH_ATTN_EXT_VEC_REDUCE + 1)]]; + +kernel void kernel_flash_attn_ext_vec_reduce( + constant ggml_metal_kargs_flash_attn_ext_vec_reduce & args, + device const char * htmp, + device char * dst, + uint tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { +#define NWG (FC_flash_attn_ext_vec_reduce_NWG) +#define DV (FC_flash_attn_ext_vec_reduce_DV) + + const uint64_t rid = tgpig; + + const short iwg = tiisg; + + device const float * ss = (device const float *) htmp + (uint64_t)args.nrows*DV*NWG; + + float S = ss[rid*(2*NWG) + 2*iwg + 0]; + float M = ss[rid*(2*NWG) + 2*iwg + 1]; + + const float m = simd_max(M); + const float ms = exp(M - m); + + S = simd_sum(S*ms); + S = S == 0.0f ? 0.0f : 1.0f/S; + + const short DV4 = DV/4; + + device const float4 * htmp4 = (device const float4 *) htmp + rid*DV4*NWG; + device float4 * dst4 = (device float4 *) dst + rid*DV4; + + for (short i = sgitg; i < DV4; i += NWG) { + const float4 v = simd_sum(htmp4[i*NWG + iwg]*ms); + + if (iwg == 0) { + dst4[i] = v*S; + } + } + +#undef NWG +#undef DV +} + +constant short FC_lightning_indexer_nh [[function_constant(FC_LIGHTNING_INDEXER + 0)]]; + +template< + typename kd4x4_t, + short nl_k, + void (*deq_k)(device const kd4x4_t *, short, thread half4x4 &)> +kernel void kernel_lightning_indexer( + constant ggml_metal_kargs_lightning_indexer & args, + device const char * q, + device const char * k, + device const char * w, + device const char * m, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + constexpr short DK = OP_LIGHTNING_INDEXER_DK; + const short NH = FC_lightning_indexer_nh; + constexpr short NHPTG = OP_LIGHTNING_INDEXER_NHPTG; + constexpr short NKPSG = OP_LIGHTNING_INDEXER_NKPSG; + constexpr short NSG = OP_LIGHTNING_INDEXER_NSG; + constexpr short NBPTG = OP_LIGHTNING_INDEXER_NBPTG; + + constexpr short DK4 = DK/4; + constexpr short DK8 = DK/8; + constexpr short DK16 = DK/16; + + constexpr short NK = NKPSG*NSG; // keys per threadgroup + constexpr short NTG = 32*NSG; // threads per threadgroup + + const int i_stream = tgpig.z; + const int i_kv_0 = tgpig.x*NK; // first key of this threadgroup + const int i_kv = i_kv_0 + sgitg*NKPSG; // first key of this simdgroup + + threadgroup half sk[NK * DK16 * 16]; + threadgroup half4x4 * sk4x4 = (threadgroup half4x4 *) sk; + + for (short i = tiitg; i < NK*DK16; i += NTG) { + const short ik = i/DK16; + const short i16 = i%DK16; + + half4x4 tmp; + + if (i_kv_0 + ik < args.n_kv) { + device const kd4x4_t * kr = (device const kd4x4_t *) (k + (i_kv_0 + ik)*args.nbk2 + i_stream*args.nbk3); + + deq_k(kr + i16/nl_k, i16%nl_k, tmp); + } else { + FOR_UNROLL (short j = 0; j < 4; ++j) { + tmp[j] = half4(0.0h); + } + } + + sk4x4[i] = tmp; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // K tile of this simdgroup, transposed to [DK, NKPSG] + simdgroup_half8x8 mk[DK8]; + + FOR_UNROLL (short i = 0; i < DK8; ++i) { + simdgroup_load(mk[i], sk + sgitg*NKPSG*DK + 8*i, DK, 0, true); + } + + threadgroup half4 sq4[NHPTG*DK4]; + threadgroup half * sq = (threadgroup half *) sq4; + + threadgroup float sw [NHPTG]; + threadgroup float sqk[NSG*NHPTG*NKPSG]; + + const int i_batch_0 = tgpig.y*NBPTG; + const int n_batch = min((int) NBPTG, args.n_batch - i_batch_0); + + for (short ib = 0; ib < n_batch; ++ib) { + const int i_batch = i_batch_0 + ib; + + device const char * pq = q + i_batch*args.nbq2 + i_stream*args.nbq3; + device const char * pw = w + i_batch*args.nbw1 + i_stream*args.nbw3; + + float score = 0.0f; + + FOR_UNROLL (short i_head = 0; i_head < NH; i_head += NHPTG) { + // stage the Q tile [DK, NHPTG] and the (prescaled) head weights, heads past NH are zero + for (short i = tiitg; i < NHPTG*DK4; i += NTG) { + const short ih = i/DK4; + const short i4 = i%DK4; + + if (i_head + ih < NH) { + device const float4 * q4 = (device const float4 *) (pq + (i_head + ih)*args.nbq1); + + sq4[ih*DK4 + i4] = half4(q4[i4]); + } else { + sq4[ih*DK4 + i4] = half4(0.0h); + } + } + + if (tiitg < NHPTG) { + sw[tiitg] = i_head + tiitg < NH ? ((device const float *) pw)[i_head + tiitg] : 0.0f; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + simdgroup_float8x8 mqk = make_filled_simdgroup_matrix(0.0f); + + FOR_UNROLL (short i = 0; i < DK8; ++i) { + simdgroup_half8x8 mq; + + simdgroup_load(mq, sq + 8*i, DK, 0, false); + simdgroup_multiply_accumulate(mqk, mq, mk[i], mqk); + } + + threadgroup float * pqk = sqk + sgitg*NHPTG*NKPSG; + + simdgroup_store(mqk, pqk, NKPSG, 0, false); + simdgroup_barrier(mem_flags::mem_threadgroup); + + // one lane per key: ReLU, apply the head weight and accumulate over the head tile + if (tiisg < NKPSG) { + FOR_UNROLL (short ih = 0; ih < NHPTG; ++ih) { + score += max(pqk[ih*NKPSG + tiisg], 0.0f)*sw[ih]; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + if (tiisg < NKPSG) { + const int ik = i_kv + tiisg; + if (ik < args.n_kv) { + device const half * pm = (device const half *) (m + i_batch*args.nbm1 + (i_stream % args.mask_ne3)*args.nbm3); + device float * pd = (device float *) (dst + i_batch*args.nb1 + i_stream*args.nb3); + + pd[ik] = score + (float) pm[ik]; + } + } + } +} + +typedef decltype(kernel_lightning_indexer) kernel_lightning_indexer_t; + +template [[host_name("kernel_lightning_indexer_f32")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_f16")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; + +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_lightning_indexer_bf16")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +#endif + +template [[host_name("kernel_lightning_indexer_q4_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_q4_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_q5_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_q5_1")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; +template [[host_name("kernel_lightning_indexer_q8_0")]] kernel kernel_lightning_indexer_t kernel_lightning_indexer; diff --git a/ggml/src/ggml-metal/kernels/fa_common.metal b/ggml/src/ggml-metal/kernels/fa_common.metal new file mode 100644 index 0000000..5e08bf8 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_common.metal @@ -0,0 +1,710 @@ +constant bool FC_flash_attn_ext_has_mask [[function_constant(FC_FLASH_ATTN_EXT + 0)]]; +constant bool FC_flash_attn_ext_has_sinks [[function_constant(FC_FLASH_ATTN_EXT + 1)]]; +constant bool FC_flash_attn_ext_has_bias [[function_constant(FC_FLASH_ATTN_EXT + 2)]]; +constant bool FC_flash_attn_ext_has_scap [[function_constant(FC_FLASH_ATTN_EXT + 3)]]; +constant bool FC_flash_attn_ext_has_kvpad [[function_constant(FC_FLASH_ATTN_EXT + 4)]]; + +constant bool FC_flash_attn_ext_bc_mask [[function_constant(FC_FLASH_ATTN_EXT + 10)]]; + +//constant float FC_flash_attn_ext_scale [[function_constant(FC_FLASH_ATTN_EXT + 10)]]; +//constant float FC_flash_attn_ext_max_bias [[function_constant(FC_FLASH_ATTN_EXT + 11)]]; +//constant float FC_flash_attn_ext_logit_softcap [[function_constant(FC_FLASH_ATTN_EXT + 12)]]; + +constant int32_t FC_flash_attn_ext_ns10 [[function_constant(FC_FLASH_ATTN_EXT + 20)]]; +constant int32_t FC_flash_attn_ext_ns20 [[function_constant(FC_FLASH_ATTN_EXT + 21)]]; +constant int32_t FC_flash_attn_ext_nsg [[function_constant(FC_FLASH_ATTN_EXT + 22)]]; + +// ref: https://arxiv.org/pdf/2307.08691.pdf +template< + typename q_t, // query types in shared memory + typename q4_t, + typename q8x8_t, + typename k_t, // key types in shared memory + typename k4x4_t, + typename k8x8_t, + typename v_t, // value types in shared memory + typename v4x4_t, + typename v8x8_t, + typename qk_t, // Q*K types + typename qk8x8_t, + typename s_t, // soft-max types + typename s2_t, + typename s8x8_t, + typename o_t, // attention accumulation types + typename o4_t, + typename o8x8_t, + typename kd4x4_t, // key type in device memory + short nl_k, + void (*deq_k)(device const kd4x4_t *, short, thread k4x4_t &), + typename vd4x4_t, // value type in device memory + short nl_v, + void (*deq_v)(device const vd4x4_t *, short, thread v4x4_t &), + short DK, // K head size + short DV, // V head size + short Q, // queries per threadgroup + short C, // cache items per threadgroup + short NSG> // number of simd groups +void kernel_flash_attn_ext_impl( + constant ggml_metal_kargs_flash_attn_ext & args, + device const char * q, + device const char * k, + device const char * v, + device const char * mask, + device const char * sinks, + device const char * pad, + device const char * blk, + device char * dst, + threadgroup half * shmem_f16, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const ushort iq3 = tgpig[2]; + const ushort iq2 = tgpig[1]; + const ushort iq1 = tgpig[0]*Q; + +#define NS10 (FC_flash_attn_ext_ns10) +#define NS20 (FC_flash_attn_ext_ns20) + + // note: I had some concerns that using this instead of the ugly macros above was affecting performance + // need to re-check carefully and if no regressions are observerd - remove the macros + // the concerns is that maybe using const variables requires extra registers? but not sure if the compiler + // is clever enough to avoid this. unfortunately, using constexpr is not possible with FC + //const short NS10 = FC_flash_attn_ext_ns10; + //const short NS20 = FC_flash_attn_ext_ns20; + + constexpr short KV = 8; + + constexpr short DK4 = DK/4; + constexpr short DK8 = DK/8; + constexpr short DK16 = DK/16; + constexpr short DV4 = DV/4; + //constexpr short DV8 = DV/8; + constexpr short DV16 = DV/16; + + constexpr short PV = PAD2(DV, 64); + constexpr short PV4 = PV/4; + constexpr short PV8 = PV/8; + //constexpr short PV16 = PV/16; + + constexpr short NW = N_SIMDWIDTH; + constexpr short NQ = Q/NSG; + constexpr short SH = 2*C; // shared memory per simdgroup (s_t == float) + + constexpr short TS = 2*SH; + constexpr short T = DK + 2*PV; // shared memory size per query in (half) + + threadgroup q_t * sq = (threadgroup q_t *) (shmem_f16 + 0*T); // holds the query data + threadgroup q4_t * sq4 = (threadgroup q4_t *) (shmem_f16 + 0*T); // same as above but in q4_t + threadgroup o_t * so = (threadgroup o_t *) (shmem_f16 + 0*T + Q*DK); // the result for all queries in 8x8 matrices (the O matrix from the paper) + threadgroup o4_t * so4 = (threadgroup o4_t *) (shmem_f16 + 0*T + Q*DK); + threadgroup s_t * ss = (threadgroup s_t *) (shmem_f16 + Q*T); // scratch buffer for attention, mask and diagonal matrix + threadgroup s2_t * ss2 = (threadgroup s2_t *) (shmem_f16 + Q*T); // same as above but in s2_t + + threadgroup k_t * sk = (threadgroup k_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // scratch buffer to load K in shared memory + threadgroup k4x4_t * sk4x4 = (threadgroup k4x4_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // same as above but in k4x4_t + + threadgroup v_t * sv = (threadgroup v_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // scratch buffer to load V in shared memory + threadgroup v4x4_t * sv4x4 = (threadgroup v4x4_t *) (shmem_f16 + sgitg*(4*16*KV) + Q*T + Q*TS); // same as above but in v4x4_t + + // mask storage in shared mem + threadgroup half2 * sm2 = (threadgroup half2 *) (shmem_f16 + Q*T + 2*C); + + // per-query mask pointers + device const half2 * pm2[NQ]; + + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + pm2[jj] = (device const half2 *) ((device const char *) mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33); + } + + { + const int32_t nblk1 = ((args.ne01 + Q - 1)/Q); + const int32_t nblk0 = ((args.ne11 + C - 1)/C); + + blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0; + } + + { + q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03; + + const short ikv2 = iq2/(args.ne02/args.ne_12_2); + const short ikv3 = iq3/(args.ne03/args.ne_12_3); + + k += ikv2*args.nb12 + ikv3*args.nb13; + v += ikv2*args.nb22 + ikv3*args.nb23; + } + + // load heads from Q to shared memory + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + device const float4 * q4 = (device const float4 *) ((device const char *) q + j*args.nb01); + + for (short i = tiisg; i < DK4; i += NW) { + if (iq1 + j < args.ne01) { + sq4[j*DK4 + i] = (q4_t) q4[i]; + } else { + sq4[j*DK4 + i] = 0; + } + } + } + + // zero out + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + for (short i = tiisg; i < DV4; i += NW) { + so4[j*PV4 + i] = 0; + } + + for (short i = tiisg; i < SH; i += NW) { + ss[j*SH + i] = 0.0f; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + float S[NQ] = { [0 ... NQ-1] = 0.0f }; + + { + float M[NQ] = { [0 ... NQ-1] = -FLT_MAX/2 }; + + float slope = 1.0f; + + // ALiBi + if (FC_flash_attn_ext_has_bias) { + const short h = iq2; + + const float base = h < args.n_head_log2 ? args.m0 : args.m1; + const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; + + slope = pow(base, exph); + } + + // loop over the KV cache + // each simdgroup handles blocks of Q rows and C columns + for (int ic0 = 0; ; ++ic0) { + int ic = ic0*C; + if (ic >= args.ne11) { + break; + } + + // the last partial chunk uses the pad buffer as source + if (FC_flash_attn_ext_has_kvpad && ic + C > args.ne11) { + k = pad; + v = k + args.nb11*C*args.ne_12_2*args.ne_12_3; + mask = v + args.nb21*C*args.ne_12_2*args.ne_12_3; + + const short ikv2 = iq2/(args.ne02/args.ne_12_2); + const short ikv3 = iq3/(args.ne03/args.ne_12_3); + + k += (ikv2 + ikv3*args.ne_12_2)*args.nb11*C; + v += (ikv2 + ikv3*args.ne_12_2)*args.nb21*C; + + if (!FC_flash_attn_ext_has_mask) { + threadgroup half * sm = (threadgroup half *) (sm2); + + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + for (short i = tiisg; i < C; i += NW) { + if (ic + i >= args.ne11) { + sm[2*j*SH + i] = -MAXHALF; + } + } + } + } else { + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + pm2[jj] = (device const half2 *) ((device const half *) mask + + (iq1 + j)*C + + (iq2%args.ne32)*(C*args.ne31) + + (iq3%args.ne33)*(C*args.ne31*args.ne32)); + } + } + + ic = 0; + } + + char blk_cur = 1; + + // read the mask into shared mem + if (FC_flash_attn_ext_has_mask) { + blk_cur = blk[ic0]; + + if (blk_cur == 0) { + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + pm2[jj] += NW; + } + + continue; + } + + if (blk_cur == 1) { + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + if (FC_flash_attn_ext_bc_mask) { + sm2[j*SH + tiisg] = (iq1 + j) < args.ne31 ? pm2[jj][tiisg] : half2(-MAXHALF, -MAXHALF); + } else { + sm2[j*SH + tiisg] = pm2[jj][tiisg]; + } + + pm2[jj] += NW; + } + } else if (blk_cur == 2) { + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + pm2[jj] += NW; + } + } + +#if 0 + // note: old -INF block optimization - obsoleted by pre-computing non-masked blocks + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // used to detect blocks full of -INF + // skip only when the entire threadgroup is masked + half2 smax2(-MAXHALF/2, -MAXHALF/2); + + FOR_UNROLL (short j = 0; j < Q; ++j) { + smax2 = max(smax2, sm2[j*SH + tiisg]); + } + + smax2 = simd_max(smax2); + + if (max(smax2[0], smax2[1]) <= -MAXHALF/2) { + // this barrier is important + threadgroup_barrier(mem_flags::mem_threadgroup); + + continue; + } +#endif + } + + // Q*K^T + // this is compile-time check, so it does not have runtime overhead + if (is_same::value) { + // we can read directly from global memory + device const k_t * pk = (device const k_t *) (k + ic*args.nb11); + threadgroup const q_t * pq = sq; + threadgroup s_t * ps = ss; + + pk += sgitg*(8*NS10); + ps += sgitg*(8*1); + + static_assert((C/8) % NSG == 0, ""); + + constexpr short NC = (C/8)/NSG; + + FOR_UNROLL (short cc = 0; cc < NC; ++cc) { + qk8x8_t mqk = make_filled_simdgroup_matrix((qk_t) 0.0f); + + if (DK % 16 != 0) { + k8x8_t mk; + q8x8_t mq; + + FOR_UNROLL (short i = 0; i < DK8; ++i) { + simdgroup_barrier(mem_flags::mem_none); + + simdgroup_load(mk, pk + 8*i, NS10, 0, true); + simdgroup_load(mq, pq + 8*i, DK); + + simdgroup_barrier(mem_flags::mem_none); + + simdgroup_multiply_accumulate(mqk, mq, mk, mqk); + } + } else { + k8x8_t mk[2]; + q8x8_t mq[2]; + + // note: too much unroll can tank the performance for large heads + #pragma unroll (MIN(DK8/2, 4*NSG)) + for (short i = 0; i < DK8/2; ++i) { + simdgroup_barrier(mem_flags::mem_none); + + simdgroup_load(mq[0], pq + 0*8 + 16*i, DK); + simdgroup_load(mq[1], pq + 1*8 + 16*i, DK); + + simdgroup_load(mk[0], pk + 0*8 + 16*i, NS10, 0, true); + simdgroup_load(mk[1], pk + 1*8 + 16*i, NS10, 0, true); + + simdgroup_barrier(mem_flags::mem_none); + + simdgroup_multiply_accumulate(mqk, mq[0], mk[0], mqk); + simdgroup_multiply_accumulate(mqk, mq[1], mk[1], mqk); + } + } + + simdgroup_store(mqk, ps, SH, 0, false); + + pk += 8*(NSG*NS10); + ps += 8*(NSG); + } + } else { + // TODO: this is the quantized K cache branch - not optimized yet + for (short ccc = 0; ccc < (C/8)/NSG; ++ccc) { + const short cc = ccc*NSG + sgitg; + + const short tx = tiisg%4; + const short ty = tiisg/4; + + qk8x8_t mqk = make_filled_simdgroup_matrix((qk_t) 0.0f); + + for (short ii = 0; ii < DK16; ii += 4) { + device const kd4x4_t * pk4x4 = (device const kd4x4_t *) (k + ((ic + 8*cc + ty)*args.nb11)); + + if (DK16%4 == 0) { + // the head is evenly divisible by 4*16 = 64, so no need for bound checks + { + k4x4_t tmp; + deq_k(pk4x4 + (ii + tx)/nl_k, (ii + tx)%nl_k, tmp); + sk4x4[4*ty + tx] = tmp; + } + + simdgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short k = 0; k < 4; ++k) { + k8x8_t mk; + q8x8_t mq; + + simdgroup_load(mk, sk + 16*k + 0*8, 4*16, 0, true); // transpose + simdgroup_load(mq, sq + (2*(ii + k) + 0)*8, DK); + simdgroup_multiply_accumulate(mqk, mq, mk, mqk); + + simdgroup_load(mk, sk + 16*k + 1*8, 4*16, 0, true); // transpose + simdgroup_load(mq, sq + (2*(ii + k) + 1)*8, DK); + simdgroup_multiply_accumulate(mqk, mq, mk, mqk); + } + } else { + if (ii + tx < DK16) { + k4x4_t tmp; + deq_k(pk4x4 + (ii + tx)/nl_k, (ii + tx)%nl_k, tmp); + sk4x4[4*ty + tx] = tmp; + } + + simdgroup_barrier(mem_flags::mem_threadgroup); + + for (short k = 0; k < 4 && ii + k < DK16; ++k) { + k8x8_t mk; + q8x8_t mq; + + simdgroup_load(mk, sk + 16*k + 0*8, 4*16, 0, true); // transpose + simdgroup_load(mq, sq + (2*(ii + k) + 0)*8, DK); + simdgroup_multiply_accumulate(mqk, mq, mk, mqk); + + simdgroup_load(mk, sk + 16*k + 1*8, 4*16, 0, true); // transpose + simdgroup_load(mq, sq + (2*(ii + k) + 1)*8, DK); + simdgroup_multiply_accumulate(mqk, mq, mk, mqk); + } + } + } + + simdgroup_store(mqk, ss + 8*cc, SH, 0, false); + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // online softmax + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + const float m = M[jj]; + + // scale and apply the logitcap / mask + float2 s2 = ss2[j*SH/2 + tiisg]*args.scale; + + if (FC_flash_attn_ext_has_scap) { + s2 = args.logit_softcap*precise::tanh(s2); + } + + // mqk = mqk + slope*mask + if (blk_cur != 2) { + if (FC_flash_attn_ext_has_bias) { + s2 += s2_t(sm2[j*SH + tiisg])*slope; + } else { + s2 += s2_t(sm2[j*SH + tiisg]); + } + } + + M[jj] = simd_max(max(M[jj], max(s2[0], s2[1]))); + + const float ms = exp(m - M[jj]); + const float2 vs2 = exp(s2 - M[jj]); + + S[jj] = S[jj]*ms + simd_sum(vs2[0] + vs2[1]); + + // the P matrix from the paper (Q rows, C columns) + ss2[j*SH/2 + tiisg] = vs2; + + if (DV4 % NW == 0) { + FOR_UNROLL (short ii = 0; ii < DV4/NW; ++ii) { + const short i = ii*NW + tiisg; + + so4[j*PV4 + i] *= ms; + } + } else { + for (short i = tiisg; i < DV4; i += NW) { + so4[j*PV4 + i] *= ms; + } + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // O = O + (Q*K^T)*V + { + // we can read directly from global memory + if (is_same::value) { + static_assert(PV8 % NSG == 0, ""); + + constexpr short NO = PV8/NSG; + + o8x8_t lo[NO]; + + { + auto sot = so + 8*sgitg; + + FOR_UNROLL (short ii = 0; ii < NO; ++ii) { + simdgroup_load(lo[ii], sot, PV, 0, false); + + sot += 8*NSG; + } + } + + { + device const v_t * pv = (device const v_t *) (v + ic*args.nb21); + + pv += 8*sgitg; + + if (DV <= 64) { + FOR_UNROLL (short cc = 0; cc < C/8; ++cc) { + s8x8_t vs; + simdgroup_load(vs, ss + 8*cc, SH, 0, false); + + FOR_UNROLL (short ii = 0; ii < NO/2; ++ii) { + v8x8_t mv[2]; + + simdgroup_load(mv[0], pv + 0*NSG + 16*ii*NSG, NS20, 0, false); + simdgroup_load(mv[1], pv + 8*NSG + 16*ii*NSG, NS20, 0, false); + + simdgroup_multiply_accumulate(lo[2*ii + 0], vs, mv[0], lo[2*ii + 0]); + simdgroup_multiply_accumulate(lo[2*ii + 1], vs, mv[1], lo[2*ii + 1]); + } + + pv += 8*NS20; + } + } else { + constexpr short NC = (C/8)/2; + + FOR_UNROLL (short cc = 0; cc < NC; ++cc) { + s8x8_t vs[2]; + + simdgroup_load(vs[0], ss + 16*cc + 0, SH, 0, false); + simdgroup_load(vs[1], ss + 16*cc + 8, SH, 0, false); + + FOR_UNROLL (short ii = 0; ii < NO/2; ++ii) { + v8x8_t mv[4]; + + simdgroup_load(mv[0], pv + 0*NSG + 16*ii*NSG + 0*8*NS20, NS20, 0, false); + simdgroup_load(mv[1], pv + 8*NSG + 16*ii*NSG + 0*8*NS20, NS20, 0, false); + simdgroup_load(mv[2], pv + 0*NSG + 16*ii*NSG + 1*8*NS20, NS20, 0, false); + simdgroup_load(mv[3], pv + 8*NSG + 16*ii*NSG + 1*8*NS20, NS20, 0, false); + + simdgroup_multiply_accumulate(lo[2*ii + 0], vs[0], mv[0], lo[2*ii + 0]); + simdgroup_multiply_accumulate(lo[2*ii + 1], vs[0], mv[1], lo[2*ii + 1]); + simdgroup_multiply_accumulate(lo[2*ii + 0], vs[1], mv[2], lo[2*ii + 0]); + simdgroup_multiply_accumulate(lo[2*ii + 1], vs[1], mv[3], lo[2*ii + 1]); + } + + pv += 2*8*NS20; + } + } + } + + { + auto sot = so + 8*sgitg; + + FOR_UNROLL (short ii = 0; ii < NO; ++ii) { + simdgroup_store(lo[ii], sot, PV, 0, false); + + sot += 8*NSG; + } + } + } else { + // TODO: this is the quantized V cache branch - not optimized yet + + const short tx = tiisg%4; + const short ty = tiisg/4; + + for (short cc = 0; cc < C/8; ++cc) { + s8x8_t vs; + simdgroup_load(vs, ss + 8*cc, SH, 0, false); + + for (short ii = 4*sgitg; ii < DV16; ii += 4*NSG) { + device const vd4x4_t * pv4x4 = (device const vd4x4_t *) (v + ((ic + 8*cc + ty)*args.nb21)); + + if (DV16%4 == 0) { + // no need for bound checks + { + v4x4_t tmp; + deq_v(pv4x4 + (ii + tx)/nl_v, (ii + tx)%nl_v, tmp); + sv4x4[4*ty + tx] = tmp; + } + + simdgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short k = 0; k < 4; ++k) { + v8x8_t mv[2]; + o8x8_t lo[2]; + + simdgroup_load(mv[0], sv + 16*k + 0*8, 4*16, 0, false); + simdgroup_load(mv[1], sv + 16*k + 1*8, 4*16, 0, false); + simdgroup_load(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false); + simdgroup_load(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false); + + simdgroup_multiply_accumulate(lo[0], vs, mv[0], lo[0]); + simdgroup_multiply_accumulate(lo[1], vs, mv[1], lo[1]); + + simdgroup_store(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false); + simdgroup_store(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false); + } + } else { + if (ii + tx < DV16) { + v4x4_t tmp; + deq_v(pv4x4 + (ii + tx)/nl_v, (ii + tx)%nl_v, tmp); + sv4x4[4*ty + tx] = tmp; + } + + simdgroup_barrier(mem_flags::mem_threadgroup); + + for (short k = 0; k < 4 && ii + k < DV16; ++k) { + v8x8_t mv[2]; + o8x8_t lo[2]; + + simdgroup_load(mv[0], sv + 16*k + 0*8, 4*16, 0, false); + simdgroup_load(mv[1], sv + 16*k + 1*8, 4*16, 0, false); + simdgroup_load(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false); + simdgroup_load(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false); + + simdgroup_multiply_accumulate(lo[0], vs, mv[0], lo[0]); + simdgroup_multiply_accumulate(lo[1], vs, mv[1], lo[1]); + + simdgroup_store(lo[0], so + 8*(2*(ii + k) + 0), PV, 0, false); + simdgroup_store(lo[1], so + 8*(2*(ii + k) + 1), PV, 0, false); + } + } + } + } + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + if (FC_flash_attn_ext_has_sinks) { + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + const float m = M[jj]; + const float s = tiisg == 0 ? ((device const float *) sinks)[iq2] : -FLT_MAX/2; + + M[jj] = simd_max(max(M[jj], s)); + + const float ms = exp(m - M[jj]); + const float vs = exp(s - M[jj]); + + S[jj] = S[jj]*ms + simd_sum(vs); + + for (short i = tiisg; i < DV4; i += NW) { + so4[j*PV4 + i] *= ms; + } + } + } + } + + // store to global memory + for (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + if (iq1 + j >= args.ne01) { + break; + } + + device float4 * dst4 = (device float4 *) dst + ((uint64_t)iq3*args.ne2*args.ne1 + iq2 + (uint64_t)(iq1 + j)*args.ne1)*DV4; + + const float scale = S[jj] == 0.0 ? 0.0f : 1.0f/S[jj]; + + if (DV4 % NW == 0) { + FOR_UNROLL (short ii = 0; ii < DV4/NW; ++ii) { + const short i = ii*NW + tiisg; + + dst4[i] = (float4) so4[j*PV4 + i]*scale; + } + } else { + for (short i = tiisg; i < DV4; i += NW) { + dst4[i] = (float4) so4[j*PV4 + i]*scale; + } + } + } + +#undef NS10 +#undef NS20 +} + +template< + typename q_t, // query types in shared memory + typename q4_t, + typename q8x8_t, + typename k_t, // key types in shared memory + typename k4x4_t, + typename k8x8_t, + typename v_t, // value types in shared memory + typename v4x4_t, + typename v8x8_t, + typename qk_t, // Q*K types + typename qk8x8_t, + typename s_t, // soft-max types + typename s2_t, + typename s8x8_t, + typename o_t, // attention accumulation types + typename o4_t, + typename o8x8_t, + typename kd4x4_t, // key type in device memory + short nl_k, + void (*deq_k)(device const kd4x4_t *, short, thread k4x4_t &), + typename vd4x4_t, // value type in device memory + short nl_v, + void (*deq_v)(device const vd4x4_t *, short, thread v4x4_t &), + short DK, // K head size + short DV, // V head size + short Q = OP_FLASH_ATTN_EXT_NQPSG, // queries per threadgroup + short C = OP_FLASH_ATTN_EXT_NCPSG> // cache items per threadgroup +kernel void kernel_flash_attn_ext( + constant ggml_metal_kargs_flash_attn_ext & args, + device const char * q, + device const char * k, + device const char * v, + device const char * mask, + device const char * sinks, + device const char * pad, + device const char * blk, + device char * dst, + threadgroup half * shmem_f16 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { +#define FWD_TMPL q_t, q4_t, q8x8_t, k_t, k4x4_t, k8x8_t, v_t, v4x4_t, v8x8_t, qk_t, qk8x8_t, s_t, s2_t, s8x8_t, o_t, o4_t, o8x8_t, kd4x4_t, nl_k, deq_k, vd4x4_t, nl_v, deq_v, DK, DV, Q, C +#define FWD_ARGS args, q, k, v, mask, sinks, pad, blk, dst, shmem_f16, tgpig, tiisg, sgitg + switch (FC_flash_attn_ext_nsg) { + // note: disabled cases to reduce library load time + //case 1: kernel_flash_attn_ext_impl(FWD_ARGS); break; + //case 2: kernel_flash_attn_ext_impl(FWD_ARGS); break; + case 4: kernel_flash_attn_ext_impl(FWD_ARGS); break; + case 8: kernel_flash_attn_ext_impl(FWD_ARGS); break; + } +#undef FWD_TMPL +#undef FWD_ARGS +} diff --git a/ggml/src/ggml-metal/kernels/fa_f16.metal b/ggml/src/ggml-metal/kernels/fa_f16.metal new file mode 100644 index 0000000..6e30b09 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_f16.metal @@ -0,0 +1,77 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_common.metal" + +// TODO: this is quite ugly. in the future these types will be hardcoded in the kernel, but for now keep them as +// template to be able to explore different combinations + +#define FA_TYPES \ + half, half4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +#define FA_TYPES_BF \ + bfloat, bfloat4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + half, half4, simdgroup_half8x8 + //float, float4, simdgroup_float8x8 + +#define FA_TYPES_F32 \ + half, half4, simdgroup_half8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +typedef decltype(kernel_flash_attn_ext) flash_attn_ext_t; + +template [[host_name("kernel_flash_attn_ext_f16_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; + +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_bf16_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +#endif + +#undef FA_TYPES +#undef FA_TYPES_BF +#undef FA_TYPES_F32 diff --git a/ggml/src/ggml-metal/kernels/fa_f32.metal b/ggml/src/ggml-metal/kernels/fa_f32.metal new file mode 100644 index 0000000..449079e --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_f32.metal @@ -0,0 +1,54 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_common.metal" + +#define FA_TYPES \ + half, half4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +#define FA_TYPES_BF \ + bfloat, bfloat4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + half, half4, simdgroup_half8x8 + //float, float4, simdgroup_float8x8 + +#define FA_TYPES_F32 \ + half, half4, simdgroup_half8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +typedef decltype(kernel_flash_attn_ext) flash_attn_ext_t; + +template [[host_name("kernel_flash_attn_ext_f32_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; + +#undef FA_TYPES +#undef FA_TYPES_BF +#undef FA_TYPES_F32 diff --git a/ggml/src/ggml-metal/kernels/fa_q4_0.metal b/ggml/src/ggml-metal/kernels/fa_q4_0.metal new file mode 100644 index 0000000..6b05b5d --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_q4_0.metal @@ -0,0 +1,54 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_common.metal" + +#define FA_TYPES \ + half, half4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +#define FA_TYPES_BF \ + bfloat, bfloat4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + half, half4, simdgroup_half8x8 + //float, float4, simdgroup_float8x8 + +#define FA_TYPES_F32 \ + half, half4, simdgroup_half8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +typedef decltype(kernel_flash_attn_ext) flash_attn_ext_t; + +template [[host_name("kernel_flash_attn_ext_q4_0_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; + +#undef FA_TYPES +#undef FA_TYPES_BF +#undef FA_TYPES_F32 diff --git a/ggml/src/ggml-metal/kernels/fa_q4_1.metal b/ggml/src/ggml-metal/kernels/fa_q4_1.metal new file mode 100644 index 0000000..87fac70 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_q4_1.metal @@ -0,0 +1,54 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_common.metal" + +#define FA_TYPES \ + half, half4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +#define FA_TYPES_BF \ + bfloat, bfloat4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + half, half4, simdgroup_half8x8 + //float, float4, simdgroup_float8x8 + +#define FA_TYPES_F32 \ + half, half4, simdgroup_half8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +typedef decltype(kernel_flash_attn_ext) flash_attn_ext_t; + +template [[host_name("kernel_flash_attn_ext_q4_1_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; + +#undef FA_TYPES +#undef FA_TYPES_BF +#undef FA_TYPES_F32 diff --git a/ggml/src/ggml-metal/kernels/fa_q5_0.metal b/ggml/src/ggml-metal/kernels/fa_q5_0.metal new file mode 100644 index 0000000..cd0e31f --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_q5_0.metal @@ -0,0 +1,54 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_common.metal" + +#define FA_TYPES \ + half, half4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +#define FA_TYPES_BF \ + bfloat, bfloat4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + half, half4, simdgroup_half8x8 + //float, float4, simdgroup_float8x8 + +#define FA_TYPES_F32 \ + half, half4, simdgroup_half8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +typedef decltype(kernel_flash_attn_ext) flash_attn_ext_t; + +template [[host_name("kernel_flash_attn_ext_q5_0_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; + +#undef FA_TYPES +#undef FA_TYPES_BF +#undef FA_TYPES_F32 diff --git a/ggml/src/ggml-metal/kernels/fa_q5_1.metal b/ggml/src/ggml-metal/kernels/fa_q5_1.metal new file mode 100644 index 0000000..4d74951 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_q5_1.metal @@ -0,0 +1,54 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_common.metal" + +#define FA_TYPES \ + half, half4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +#define FA_TYPES_BF \ + bfloat, bfloat4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + half, half4, simdgroup_half8x8 + //float, float4, simdgroup_float8x8 + +#define FA_TYPES_F32 \ + half, half4, simdgroup_half8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +typedef decltype(kernel_flash_attn_ext) flash_attn_ext_t; + +template [[host_name("kernel_flash_attn_ext_q5_1_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; + +#undef FA_TYPES +#undef FA_TYPES_BF +#undef FA_TYPES_F32 diff --git a/ggml/src/ggml-metal/kernels/fa_q8_0.metal b/ggml/src/ggml-metal/kernels/fa_q8_0.metal new file mode 100644 index 0000000..bb29a26 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_q8_0.metal @@ -0,0 +1,54 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_common.metal" + +#define FA_TYPES \ + half, half4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + half, half4x4, simdgroup_half8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +#define FA_TYPES_BF \ + bfloat, bfloat4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + bfloat, bfloat4x4, simdgroup_bfloat8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + half, half4, simdgroup_half8x8 + //float, float4, simdgroup_float8x8 + +#define FA_TYPES_F32 \ + half, half4, simdgroup_half8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, float4x4, simdgroup_float8x8, \ + float, simdgroup_float8x8, \ + float, float2, simdgroup_float8x8, \ + float, float4, simdgroup_float8x8 + //half, half4, simdgroup_half8x8 + +typedef decltype(kernel_flash_attn_ext) flash_attn_ext_t; + +template [[host_name("kernel_flash_attn_ext_q8_0_dk32_dv32" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk40_dv40" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk48_dv48" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk64_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk72_dv72" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk80_dv80" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk320_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk512_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk576_dv512")]] kernel flash_attn_ext_t kernel_flash_attn_ext; + +#undef FA_TYPES +#undef FA_TYPES_BF +#undef FA_TYPES_F32 diff --git a/ggml/src/ggml-metal/kernels/fa_tensor.metal b/ggml/src/ggml-metal/kernels/fa_tensor.metal new file mode 100644 index 0000000..4a9ed30 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_tensor.metal @@ -0,0 +1,298 @@ +#include "common.h" + +#ifdef GGML_METAL_HAS_TENSOR + +constant bool FC_flash_attn_ext_tensor_has_mask [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 0)]]; +constant bool FC_flash_attn_ext_tensor_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 1)]]; +constant bool FC_flash_attn_ext_tensor_has_bias [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 2)]]; +constant bool FC_flash_attn_ext_tensor_has_scap [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 3)]]; + +// ref: https://arxiv.org/pdf/2307.08691.pdf +template< + typename T, // Q, K, V and P types + typename T4, + short DK, // K head size + short DV, // V head size + short Q = OP_FLASH_ATTN_EXT_TENSOR_NQPSG, // queries per threadgroup + short C = OP_FLASH_ATTN_EXT_TENSOR_NCPSG, // cache items per threadgroup + short NSG = OP_FLASH_ATTN_EXT_TENSOR_NSG> // number of simd groups +kernel void kernel_flash_attn_ext_tensor( + constant ggml_metal_kargs_flash_attn_ext & args, + device const char * q, + device const char * k, + device const char * v, + device const char * mask, + device const char * sinks, + device const char * blk, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiisg [[thread_index_in_simdgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + constexpr short NW = N_SIMDWIDTH; + constexpr short NT = NW*NSG; + constexpr short NQ = Q/NSG; + constexpr short NC = C/NW; // columns per thread + + static_assert(DK % 4 == 0, "DK must be divisible by 4"); + static_assert(Q % NSG == 0, "Q must be divisible by NSG"); + static_assert(C % NW == 0, "C must be divisible by NW"); + + const int iq3 = tgpig[2]; + const int iq2 = tgpig[1]; + const int iq1 = tgpig[0]*Q; + + const short tiitg = sgitg*NW + tiisg; + + threadgroup T * sq = (threadgroup T *) shmem; // [Q, DK] queries + threadgroup float * ss = (threadgroup float *) (sq + Q*DK); // [Q, C] scores + threadgroup T * sp = (threadgroup T *) (ss + Q*C); // [Q, C] probabilities + threadgroup float * sr = (threadgroup float *) (sp + Q*C); // [Q] per-row scale of O + threadgroup int * sf = (threadgroup int *) (sr + Q); // [1] last iteration (ic0 + 1) that rescaled O + + q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03; + + { + const int ikv2 = iq2/(args.ne02/args.ne_12_2); + const int ikv3 = iq3/(args.ne03/args.ne_12_3); + + k += ikv2*args.nb12 + ikv3*args.nb13; + v += ikv2*args.nb22 + ikv3*args.nb23; + } + + // with softcap the scale is small (scale/softcap), so it is applied to the scores to keep the precision of Q + const float qscale = FC_flash_attn_ext_tensor_has_scap ? 1.0f : args.scale; + + // load the queries, with the scale folded in + for (int i = tiitg; i < Q*DK/4; i += NT) { + const int j = i/(DK/4); + + float4 q4 = 0.0f; + if (iq1 + j < args.ne01) { + q4 = ((device const float4 *) (q + j*args.nb01))[i%(DK/4)]; + } + + ((threadgroup T4 *) sq)[i] = (T4) (q4*qscale); + } + + device const half * pm[NQ]; + + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + pm[jj] = (device const half *) (mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33); + } + + { + const int nblk1 = (args.ne01 + Q - 1)/Q; + const int nblk0 = (args.ne11 + C - 1)/C; + + blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0; + } + + float M[NQ]; + float S[NQ]; + + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + M[jj] = -FLT_MAX/2; + S[jj] = 0.0f; + } + + float slope = 1.0f; + + // ALiBi + if (FC_flash_attn_ext_tensor_has_bias) { + const short h = iq2; + + const float base = h < args.n_head_log2 ? args.m0 : args.m1; + const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; + + slope = pow(base, exph); + } + + const int sk = args.ns10; + const int sv = args.ns20; + + auto tq = tensor(sq, dextents(DK, Q)); + auto ts = tensor(ss, dextents(C, Q)); + auto tp = tensor(sp, dextents(C, Q)); + + mpp::tensor_ops::matmul2d< + mpp::tensor_ops::matmul2d_descriptor(Q, C, DK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply), + execution_simdgroups> mm_qk; + + mpp::tensor_ops::matmul2d< + mpp::tensor_ops::matmul2d_descriptor(Q, DV, C, false, false, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), + execution_simdgroups> mm_pv; + + auto tv0 = tensor((device T *) v, dextents(DV, C), array({1, sv})); + + // the O matrix from the paper + auto co = mm_pv.template get_destination_cooperative_tensor(); + + FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) { + if (co.is_valid_element(i)) { + co[i] = 0.0f; + } + } + + if (tiitg == 0) { + sf[0] = 0; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // the host guarantees ne11 % C == 0 + for (int ic0 = 0, ic = 0; ic < args.ne11; ++ic0, ic += C) { + char blk_cur = 1; + + if (FC_flash_attn_ext_tensor_has_mask) { + blk_cur = blk[ic0]; + + if (blk_cur == 0) { + continue; + } + } + + // Q*K^T + { + auto tk = tensor((device T *) (k + (uint64_t) ic*args.nb11), dextents(DK, C), array({1, sk})); + + mm_qk.run(tq, tk, ts); + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // online softmax + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + float s[NC]; + + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + s[ii] = ss[j*C + ii*NW + tiisg]; + } + + if (FC_flash_attn_ext_tensor_has_scap) { + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + s[ii] = args.logit_softcap*precise::tanh(s[ii]*args.scale); + } + } + + if (FC_flash_attn_ext_tensor_has_mask && blk_cur != 2 && iq1 + j < args.ne31) { + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + s[ii] += slope*(float) pm[jj][ic + ii*NW + tiisg]; + } + } + + float m = M[jj]; + + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + m = max(m, s[ii]); + } + + m = simd_max(m); + + // lazy rescaling: move the running max only when it grows by more than 8 (e^8 fits in half) + float ms = 1.0f; + + if (m > M[jj] + 8.0f) { + ms = exp(M[jj] - m); + M[jj] = m; + + if (tiisg == 0) { + sf[0] = ic0 + 1; + } + } + + float sum = 0.0f; + + FOR_UNROLL (short ii = 0; ii < NC; ++ii) { + // the sum uses the same rounded values as P*V + const T p = (T) exp(s[ii] - M[jj]); + + sp[j*C + ii*NW + tiisg] = p; + + sum += (float) p; + } + + S[jj] = S[jj]*ms + simd_sum(sum); + + if (tiisg == 0) { + sr[j] = ms; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // O = diag(ms)*O + P*V + if (sf[0] == ic0 + 1) { + FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) { + if (co.is_valid_element(i)) { + co[i] *= sr[co.get_multidimensional_index(i)[1]]; + } + } + } + + { + auto tv = tensor((device T *) (v + (uint64_t) ic*args.nb21), dextents(DV, C), array({1, sv})); + + mm_pv.run(tp, tv, co); + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + FOR_UNROLL (short jj = 0; jj < NQ; ++jj) { + const short j = jj*NSG + sgitg; + + // the sink only adds to the denominator - its rescale of O is folded into the final scale + float ms = 1.0f; + + if (FC_flash_attn_ext_tensor_has_sinks) { + const float s = ((device const float *) sinks)[iq2]; + const float m = max(M[jj], s); + + ms = exp(M[jj] - m); + + S[jj] = S[jj]*ms + exp(s - m); + } + + if (tiisg == 0) { + sr[j] = S[jj] == 0.0f ? 0.0f : ms/S[jj]; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) { + if (co.is_valid_element(i)) { + co[i] *= sr[co.get_multidimensional_index(i)[1]]; + } + } + + // store to global memory - rows past ne01 are clipped by the tensor extents + device float * pdst = (device float *) dst + ((uint64_t) iq3*args.ne2*args.ne1 + iq2 + (uint64_t) iq1*args.ne1)*DV; + + auto td = tensor(pdst, dextents(DV, args.ne01 - iq1), array({1, args.ne1*DV})); + + co.store(td); +} + +typedef decltype(kernel_flash_attn_ext_tensor) flash_attn_ext_tensor_t; + +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk64_dv64" )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_f16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; + +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk64_dv64" )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; +template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor; + +#endif // GGML_METAL_HAS_TENSOR diff --git a/ggml/src/ggml-metal/kernels/fa_vec_common.metal b/ggml/src/ggml-metal/kernels/fa_vec_common.metal new file mode 100644 index 0000000..46270ba --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_vec_common.metal @@ -0,0 +1,650 @@ +constant bool FC_flash_attn_ext_vec_has_mask [[function_constant(FC_FLASH_ATTN_EXT_VEC + 0)]]; +constant bool FC_flash_attn_ext_vec_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_VEC + 1)]]; +constant bool FC_flash_attn_ext_vec_has_bias [[function_constant(FC_FLASH_ATTN_EXT_VEC + 2)]]; +constant bool FC_flash_attn_ext_vec_has_scap [[function_constant(FC_FLASH_ATTN_EXT_VEC + 3)]]; +constant bool FC_flash_attn_ext_vec_has_kvpad [[function_constant(FC_FLASH_ATTN_EXT_VEC + 4)]]; + +//constant float FC_flash_attn_ext_vec_scale [[function_constant(FC_FLASH_ATTN_EXT_VEC + 10)]]; +//constant float FC_flash_attn_ext_vec_max_bias [[function_constant(FC_FLASH_ATTN_EXT_VEC + 11)]]; +//constant float FC_flash_attn_ext_vec_logit_softcap [[function_constant(FC_FLASH_ATTN_EXT_VEC + 12)]]; + +constant int32_t FC_flash_attn_ext_vec_ns10 [[function_constant(FC_FLASH_ATTN_EXT_VEC + 20)]]; +constant int32_t FC_flash_attn_ext_vec_ns20 [[function_constant(FC_FLASH_ATTN_EXT_VEC + 21)]]; +constant int32_t FC_flash_attn_ext_vec_nsg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 22)]]; +constant int32_t FC_flash_attn_ext_vec_nwg [[function_constant(FC_FLASH_ATTN_EXT_VEC + 23)]]; +constant bool FC_flash_attn_ext_vec_has_sparse [[function_constant(FC_FLASH_ATTN_EXT_VEC + 5)]]; +template< + typename q4_t, // query types in shared memory + typename k4_t, // key types in shared memory + typename v4_t, // value types in shared memory + typename qk_t, // Q*K types + typename s_t, // soft-max types + typename s4_t, + typename o4_t, // attention accumulation types + typename kd4_t, // key type in device memory + short nl_k, + void (*deq_k_t4)(device const kd4_t *, short, thread k4_t &), + typename vd4_t, // value type in device memory + short nl_v, + void (*deq_v_t4)(device const vd4_t *, short, thread v4_t &), + short DK, // K head size + short DV, // V head size + short NE = 4, // head elements per thread + short Q = OP_FLASH_ATTN_EXT_VEC_NQPSG, // queries per threadgroup + short C = OP_FLASH_ATTN_EXT_VEC_NCPSG> // cache items per threadgroup + +kernel void kernel_flash_attn_ext_vec( + constant ggml_metal_kargs_flash_attn_ext_vec & args, + device const char * q, + device const char * k, + device const char * v, + device const char * mask, + device const char * sinks, + device const char * pad, + device char * dst, + device const char * idx, + threadgroup half * shmem_f16 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + static_assert(DK % 32 == 0, "DK must be divisible by 32"); + static_assert(DV % 32 == 0, "DV must be divisible by 32"); + +#define NWG (FC_flash_attn_ext_vec_nwg) +#define NSG (FC_flash_attn_ext_vec_nsg) + +#define NS10 (FC_flash_attn_ext_vec_ns10) +#define NS20 (FC_flash_attn_ext_vec_ns20) + + const short iwg = tgpig[2]%NWG; + + const ushort iq3 = tgpig[2]/NWG; + const ushort iq2 = tgpig[1]; + const ushort iq1 = tgpig[0]; + + constexpr short DK4 = DK/4; + constexpr short DV4 = DV/4; + + constexpr short PK = PAD2(DK, 128); + constexpr short PK4 = PK/4; + + constexpr short PV = PAD2(DV, 128); + constexpr short PV4 = PV/4; + + constexpr short NW = N_SIMDWIDTH; + constexpr short NL = NW/NE; // note: this can be adjusted to support different head sizes and simdgroup work loads + constexpr short SH = 4*Q*C; // shared memory per simdgroup + + const int SMEM_Q = Q*NSG*PK; + const int SMEM_S = NSG*SH; + const int SMEM_O = 2*NSG*Q*PV; + const int SMEM = SMEM_Q + SMEM_S + SMEM_O; + + static_assert(DK4 % NL == 0, "DK4 must be divisible by NL"); + static_assert(DV4 % NL == 0, "DV4 must be divisible by NL"); + + threadgroup q4_t * sq4 = (threadgroup q4_t *) shmem_f16; // holds the query data + threadgroup s_t * ss = (threadgroup s_t *) (shmem_f16 + SMEM_Q + sgitg*SH); // scratch buffer for attention + threadgroup s4_t * ss4 = (threadgroup s4_t *) (shmem_f16 + SMEM_Q + sgitg*SH); // same as above but in s4_t + threadgroup half * sm = (threadgroup half *) (shmem_f16 + SMEM_Q + sgitg*SH + 2*Q*C); // scratch buffer for mask + threadgroup o4_t * so4 = (threadgroup o4_t *) (shmem_f16 + SMEM_Q + SMEM_S + 2*sgitg*Q*PV); // scratch buffer for the results + + // sparse indices for the current block + threadgroup int * spidx = FC_flash_attn_ext_vec_has_sparse + ? (threadgroup int *) (shmem_f16 + SMEM) + sgitg*C + : nullptr; + + // store the result for all queries in shared memory (the O matrix from the paper) + so4 += tiisg; + + { + q += iq1*Q*args.nb01 + iq2*args.nb02 + iq3*args.nb03; + + const short ikv2 = iq2/(args.ne02/args.ne_12_2); + const short ikv3 = iq3/(args.ne03/args.ne_12_3); + + k += ikv2*args.nb12 + ikv3*args.nb13; + v += ikv2*args.nb22 + ikv3*args.nb23; + } + + // load Q query rows to shared memory + { + for (short qq = 0; qq < Q; ++qq) { + const int iq1_q = iq1*Q + qq; + device const float4 * q4 = (device const float4 *) ((device const char *) q + qq*args.nb01); + if (iq1_q < args.ne01) { + for (short i = tiisg; i < PK4; i += NW) { + if (i < DK4) { + sq4[qq*PK4 + i] = (q4_t) q4[i]; + } else { + sq4[qq*PK4 + i] = (q4_t) 0.0f; + } + } + } else { + for (short i = tiisg; i < PK4; i += NW) { + sq4[qq*PK4 + i] = (q4_t) 0.0f; + } + } + } + } + + // zero out so + for (short qq = 0; qq < Q; ++qq) { + for (short i = 0; i < DV4/NL; ++i) { + so4[qq*DV4 + i*NL] = (o4_t) 0.0f; + } + } + + // zero out shared memory SH + for (short i = tiisg; i < SH/4; i += NW) { + ss4[i] = (s4_t) 0.0f; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + { + float S[Q]; + float M[Q]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + S[qq] = 0.0f; + M[qq] = -FLT_MAX/2; + } + + // thread indices inside the simdgroup + const short tx = tiisg%NL; + const short ty = tiisg/NL; + + // pointer to the mask + device const half * pm_base = (device const half *) (mask + iq1*Q*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33); + + // sparse indices: the list of finite mask entries per query row + // the sparse path requires Q == 1 (enforced by the host) + device const int * pidx = nullptr; + if (FC_flash_attn_ext_vec_has_sparse) { + pidx = (device const int *) idx + + ((int64_t)(iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*args.ne31*args.n_kv_max_padded + (iq1%args.ne31)*args.n_kv_max_padded; + } + + float slope = 1.0f; + + // ALiBi + if (FC_flash_attn_ext_vec_has_bias) { + const short h = iq2; + + const float base = h < args.n_head_log2 ? args.m0 : args.m1; + const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; + + slope = pow(base, exph); + } + + // loop over the KV cache + // each simdgroup handles blocks of Q rows and C columns + for (int ic0 = iwg*NSG + sgitg; ; ic0 += NWG*NSG) { + int ic = ic0*C; + if (ic >= args.ne11) { + break; + } + + device const half * pm[Q]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + // padded query rows clamp to row 0 of the mask to avoid OOB; their scores + // are forced to -inf below, so the values never affect the result. + pm[qq] = pm_base + ((iq1*Q + qq) < args.ne01 ? qq*(args.nb31/sizeof(half)) : -iq1*Q*(args.nb31/sizeof(half))); + } + + // the last partial chunk uses the pad buffer as source + if (FC_flash_attn_ext_vec_has_kvpad && ic + C > args.ne11) { + k = pad; + v = k + args.nb11*C*args.ne_12_2*args.ne_12_3; + mask = v + args.nb21*C*args.ne_12_2*args.ne_12_3; + + const short ikv2 = iq2/(args.ne02/args.ne_12_2); + const short ikv3 = iq3/(args.ne03/args.ne_12_3); + + k += (ikv2 + ikv3*args.ne_12_2)*args.nb11*C; + v += (ikv2 + ikv3*args.ne_12_2)*args.nb21*C; + + if (!FC_flash_attn_ext_vec_has_mask) { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if (ic + tiisg >= args.ne11) { + sm[qq*C + tiisg] = -MAXHALF; + } + } + } else { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + pm[qq] = (device const half *) (mask) + + (iq1*Q + qq)*C + + (iq2%args.ne32)*(C*args.ne31) + + (iq3%args.ne33)*(C*args.ne31*args.ne32); + } + } + + ic = 0; + } + + // load the sparse KV indices for the current block into shared memory + if (FC_flash_attn_ext_vec_has_sparse) { + FOR_UNROLL (short ii = 0; ii < C/NW; ++ii) { + const short i = ii*NW + tiisg; + + spidx[i] = pidx[ic + i]; + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } + + if (FC_flash_attn_ext_vec_has_mask) { + if (FC_flash_attn_ext_vec_has_sparse) { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + const int i11 = spidx[tiisg]; + if ((iq1*Q + qq) < args.ne01 && i11 >= 0) { + sm[qq*C + tiisg] = pm[qq][i11]; + } else { + sm[qq*C + tiisg] = -MAXHALF; + } + } + } else { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if ((iq1*Q + qq) < args.ne01) { + sm[qq*C + tiisg] = pm[qq][ic + tiisg]; + } else { + sm[qq*C + tiisg] = -MAXHALF; + } + } + } + } else { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if ((iq1*Q + qq) >= args.ne01) { + sm[qq*C + tiisg] = -MAXHALF; + } + } + } + + // skip -INF mask + { + bool any_finite = false; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if (simd_max(sm[qq*C + tiisg]) > -MAXHALF) { + any_finite = true; + } + } + if (!any_finite) { + continue; + } + } + + // Q*K^T + { + device const k4_t * pk4 = nullptr; + + if (!FC_flash_attn_ext_vec_has_sparse) { + pk4 = (device const k4_t *) (k + ic*args.nb11); + + pk4 += ty*NS10/4 + tx; + } + + qk_t mqk[Q][C/NE]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { + mqk[qq][cc] = 0.0f; + } + } + + // each simdgroup processes Q queries and NE (NW/NL) cache elements + FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { + if (FC_flash_attn_ext_vec_has_sparse) { + // the KV rows are gathered from the index list; -1 entries are padding + const int i11 = spidx[NE*cc + ty]; + if (i11 >= 0) { + if (is_same::value) { + device const k4_t * pk4s = (device const k4_t *) (k + i11*args.nb11) + tx; + FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) { + const k4_t k_elem = pk4s[ii*NL]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + mqk[qq][cc] += dot((float4) k_elem, (float4) sq4[qq*PK4 + ii*NL + tx]); + } + } + } else { + device const kd4_t * pk = (device const kd4_t *) (k + i11*args.nb11); + + k4_t mk; + + FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) { + const short i = ii*NL + tx; + + deq_k_t4(pk + i/nl_k, i%nl_k, mk); + + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + mqk[qq][cc] += dot((float4) mk, (float4) sq4[qq*PK4 + i]); + } + } + } + } + } else if (is_same::value) { + FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) { + const k4_t k_elem = pk4[cc*NE*NS10/4 + ii*NL]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + mqk[qq][cc] += dot((float4) k_elem, (float4) sq4[qq*PK4 + ii*NL + tx]); + } + } + } else { + device const kd4_t * pk = (device const kd4_t *) (k + ((ic + NE*cc + ty)*args.nb11)); + + k4_t mk; + + FOR_UNROLL (short ii = 0; ii < DK4/NL; ++ii) { + const short i = ii*NL + tx; + + deq_k_t4(pk + i/nl_k, i%nl_k, mk); + + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + mqk[qq][cc] += dot((float4) mk, (float4) sq4[qq*PK4 + i]); + } + } + } + + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if (NE == 1) { + mqk[qq][cc] = simd_sum(mqk[qq][cc]); + } else { + // simdgroup reduce (NE = 4) + // [ 0 .. 7] -> [ 0] + // [ 8 .. 15] -> [ 8] + // [16 .. 23] -> [16] + // [24 .. 31] -> [24] + if (NE <= 1) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 16); + } + if (NE <= 2) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 8); + } + if (NE <= 4) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 4); + } + if (NE <= 8) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 2); + } + if (NE <= 16) { + mqk[qq][cc] += simd_shuffle_down(mqk[qq][cc], 1); + } + + // broadcast + mqk[qq][cc] = simd_shuffle(mqk[qq][cc], NL*ty); + } + } + } + + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + if (FC_flash_attn_ext_vec_has_mask && + !FC_flash_attn_ext_vec_has_scap && + !FC_flash_attn_ext_vec_has_bias) { + ss[qq*C + NE*tx + ty] = fma(mqk[qq][tx], args.scale, (qk_t) sm[qq*C + NE*tx + ty]); + } else { + mqk[qq][tx] *= args.scale; + + if (FC_flash_attn_ext_vec_has_scap) { + mqk[qq][tx] = args.logit_softcap*precise::tanh(mqk[qq][tx]); + } + + if (FC_flash_attn_ext_vec_has_bias) { + mqk[qq][tx] += (qk_t) sm[qq*C + NE*tx + ty]*slope; + } else { + mqk[qq][tx] += (qk_t) sm[qq*C + NE*tx + ty]; + } + + ss[qq*C + NE*tx + ty] = mqk[qq][tx]; + } + } + } + + simdgroup_barrier(mem_flags::mem_threadgroup); + + // online softmax + { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + const float m = M[qq]; + const float s = ss[qq*C + tiisg]; + + M[qq] = simd_max(max(M[qq], s)); + + const float ms = exp(m - M[qq]); + const float vs = exp(s - M[qq]); + + S[qq] = S[qq]*ms + simd_sum(vs); + + // the P matrix from the paper (Q rows, C columns) + ss[qq*C + tiisg] = vs; + + // O = diag(ms)*O + if ((DV4/NL % NW == 0) || ty == 0) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + so4[qq*DV4 + ii*NL] *= ms; + } + } + } + } + + simdgroup_barrier(mem_flags::mem_threadgroup); + + // O = O + (Q*K^T)*V + { + o4_t lo[Q][DV4/NL]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + lo[qq][ii] = 0.0f; + } + } + + if (FC_flash_attn_ext_vec_has_sparse) { + FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { + // the KV rows are gathered from the index list; -1 entries are padding + const int i11 = spidx[NE*cc + ty]; + if (i11 >= 0) { + if (is_same::value) { + device const v4_t * pv4 = (device const v4_t *) (v + i11*args.nb21); + + pv4 += tx; + + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + const v4_t v_elem = pv4[ii*NL]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + lo[qq][ii] += o4_t(float4(v_elem)*float4(ss[qq*C + cc*NE + ty])); + } + } + } else { + device const vd4_t * pv4 = (device const vd4_t *) (v + i11*args.nb21); + + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + const short i = ii*NL + tx; + + v4_t mv; + + deq_v_t4(pv4 + i/nl_v, i%nl_v, mv); + + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + lo[qq][ii] += o4_t(float4(mv)*float4(ss[qq*C + cc*NE + ty])); + } + } + } + } + } + } else if (is_same::value) { + device const v4_t * pv4 = (device const v4_t *) (v + ic*args.nb21); + + pv4 += ty*NS20/4 + tx; + + FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + const v4_t v_elem = pv4[cc*NE*NS20/4 + ii*NL]; + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + lo[qq][ii] += o4_t(float4(v_elem)*float4(ss[qq*C + cc*NE + ty])); + } + } + } + } else { + FOR_UNROLL (short cc = 0; cc < C/NE; ++cc) { + device const vd4_t * pv4 = (device const vd4_t *) (v + ((ic + NE*cc + ty)*args.nb21)); + + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + const short i = ii*NL + tx; + + v4_t mv; + deq_v_t4(pv4 + i/nl_v, i%nl_v, mv); + + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + lo[qq][ii] += o4_t(float4(mv)*float4(ss[qq*C + NE*cc + ty])); + } + } + } + } + + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + if (NE > 1) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 16); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 16); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 16); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 16); + } + + if (NE > 2) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 8); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 8); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 8); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 8); + } + + if (NE > 4) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 4); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 4); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 4); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 4); + } + + if (NE > 8) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 2); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 2); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 2); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 2); + } + + if (NE > 16) { + lo[qq][ii][0] += simd_shuffle_down(lo[qq][ii][0], 1); + lo[qq][ii][1] += simd_shuffle_down(lo[qq][ii][1], 1); + lo[qq][ii][2] += simd_shuffle_down(lo[qq][ii][2], 1); + lo[qq][ii][3] += simd_shuffle_down(lo[qq][ii][3], 1); + } + } + } + + if ((DV4/NL % NW == 0) || ty == 0) { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + so4[qq*DV4 + ii*NL] += lo[qq][ii]; + } + } + } + } + } + + if (FC_flash_attn_ext_vec_has_sinks && sgitg == 0 && iwg == 0) { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + const float m = M[qq]; + const float s = tiisg == 0 ? ((device const float *) sinks)[iq2] : -FLT_MAX/2; + + M[qq] = simd_max(max(M[qq], s)); + + const float ms = exp(m - M[qq]); + const float vs = exp(s - M[qq]); + + S[qq] = S[qq]*ms + simd_sum(vs); + + if ((DV4/NL % NW == 0) || ty == 0) { + FOR_UNROLL (short ii = 0; ii < DV4/NL; ++ii) { + so4[qq*DV4 + ii*NL] *= ms; + } + } + } + } + + // these are needed for reducing the results from the simdgroups (reuse the ss buffer) + if (tiisg == 0) { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + ss[2*qq + 0] = (s_t) S[qq]; + ss[2*qq + 1] = (s_t) M[qq]; + } + } + } + + so4 -= tiisg; + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // parallel reduce + for (short r = NSG/2; r > 0; r >>= 1) { + if (sgitg < r) { + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + const float S0 = ss[ 2*qq + 0]; + const float S1 = ss[r*(SH/2) + 2*qq + 0]; + + const float M0 = ss[ 2*qq + 1]; + const float M1 = ss[r*(SH/2) + 2*qq + 1]; + + const float Mx = max(M0, M1); + + const float ms0 = exp(M0 - Mx); + const float ms1 = exp(M1 - Mx); + + const float Sx = S0*ms0 + S1*ms1; + + if (tiisg == 0) { + ss[2*qq + 0] = Sx; + ss[2*qq + 1] = Mx; + } + + // O_0 = diag(ms0)*O_0 + diag(ms1)*O_1 + for (short i = tiisg; i < DV4; i += NW) { + so4[qq*DV4 + i] = so4[qq*DV4 + i]*ms0 + so4[qq*DV4 + i + r*Q*PV4]*ms1; + } + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + // final rescale with 1/S and store to global memory + if (sgitg == 0) { + const int64_t nrows = args.ne3*args.ne2*args.ne1; + + device float4 * dst4 = (device float4 *) dst; + device float * dst1 = (device float *) dst + nrows*DV*NWG; // the S and M are stored after the results + + FOR_UNROLL (short qq = 0; qq < Q; ++qq) { + const int iq1_q = iq1*Q + qq; + if (iq1_q >= args.ne01) { + continue; + } + + const int64_t rid = iq3*args.ne2*args.ne1 + iq2 + iq1_q*args.ne1; + + const float Sval = NWG == 1 ? (ss[2*qq + 0] == 0.0f ? 0.0f : 1.0f/ss[2*qq + 0]) : 1.0f; + + // interleave the workgroup data + for (short i = tiisg; i < DV4; i += NW) { + dst4[rid*DV4*NWG + NWG*i + iwg] = (float4) so4[qq*DV4 + i]*Sval; + } + + // store S and M + if (NWG > 1) { + if (tiisg == 0) { + dst1[rid*(2*NWG) + 2*iwg + 0] = ss[2*qq + 0]; + dst1[rid*(2*NWG) + 2*iwg + 1] = ss[2*qq + 1]; + } + } + } + } + +#undef NWG +#undef NSG +#undef NS10 +#undef NS20 +} diff --git a/ggml/src/ggml-metal/kernels/fa_vec_f16.metal b/ggml/src/ggml-metal/kernels/fa_vec_f16.metal new file mode 100644 index 0000000..bea54f9 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_vec_f16.metal @@ -0,0 +1,129 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_vec_common.metal" + +#define FA_TYPES \ + half4, \ + half4, \ + half4, \ + float, \ + float, float4, \ + float4 + +#define FA_TYPES_F32 \ + half4, \ + float4, \ + float4, \ + float, \ + float, float4, \ + float4 + +typedef decltype(kernel_flash_attn_ext_vec) flash_attn_ext_vec_t; + +template [[host_name("kernel_flash_attn_ext_vec_f16_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +#endif + +#undef FA_TYPES +#undef FA_TYPES_F32 + diff --git a/ggml/src/ggml-metal/kernels/fa_vec_f32.metal b/ggml/src/ggml-metal/kernels/fa_vec_f32.metal new file mode 100644 index 0000000..18f8629 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_vec_f32.metal @@ -0,0 +1,38 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_vec_common.metal" + +#define FA_TYPES \ + half4, \ + half4, \ + half4, \ + float, \ + float, float4, \ + float4 + +#define FA_TYPES_F32 \ + half4, \ + float4, \ + float4, \ + float, \ + float, float4, \ + float4 + +typedef decltype(kernel_flash_attn_ext_vec) flash_attn_ext_vec_t; + +template [[host_name("kernel_flash_attn_ext_vec_f32_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; + +#undef FA_TYPES +#undef FA_TYPES_F32 + diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal b/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal new file mode 100644 index 0000000..6bf5fad --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal @@ -0,0 +1,95 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_vec_common.metal" + +#define FA_TYPES \ + half4, \ + half4, \ + half4, \ + float, \ + float, float4, \ + float4 + +#define FA_TYPES_F32 \ + half4, \ + float4, \ + float4, \ + float, \ + float, float4, \ + float4 + +typedef decltype(kernel_flash_attn_ext_vec) flash_attn_ext_vec_t; + +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; + +#undef FA_TYPES +#undef FA_TYPES_F32 + diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal b/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal new file mode 100644 index 0000000..37277ee --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal @@ -0,0 +1,95 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_vec_common.metal" + +#define FA_TYPES \ + half4, \ + half4, \ + half4, \ + float, \ + float, float4, \ + float4 + +#define FA_TYPES_F32 \ + half4, \ + float4, \ + float4, \ + float, \ + float, float4, \ + float4 + +typedef decltype(kernel_flash_attn_ext_vec) flash_attn_ext_vec_t; + +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; + +#undef FA_TYPES +#undef FA_TYPES_F32 + diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal b/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal new file mode 100644 index 0000000..26868a2 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal @@ -0,0 +1,95 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_vec_common.metal" + +#define FA_TYPES \ + half4, \ + half4, \ + half4, \ + float, \ + float, float4, \ + float4 + +#define FA_TYPES_F32 \ + half4, \ + float4, \ + float4, \ + float, \ + float, float4, \ + float4 + +typedef decltype(kernel_flash_attn_ext_vec) flash_attn_ext_vec_t; + +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; + +#undef FA_TYPES +#undef FA_TYPES_F32 + diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal b/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal new file mode 100644 index 0000000..9d39dc5 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal @@ -0,0 +1,95 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_vec_common.metal" + +#define FA_TYPES \ + half4, \ + half4, \ + half4, \ + float, \ + float, float4, \ + float4 + +#define FA_TYPES_F32 \ + half4, \ + float4, \ + float4, \ + float, \ + float, float4, \ + float4 + +typedef decltype(kernel_flash_attn_ext_vec) flash_attn_ext_vec_t; + +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; + +#undef FA_TYPES +#undef FA_TYPES_F32 + diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal b/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal new file mode 100644 index 0000000..7686ccc --- /dev/null +++ b/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal @@ -0,0 +1,95 @@ +#include "common.h" +#include "dequantize.h" +#include "fa_vec_common.metal" + +#define FA_TYPES \ + half4, \ + half4, \ + half4, \ + float, \ + float, float4, \ + float4 + +#define FA_TYPES_F32 \ + half4, \ + float4, \ + float4, \ + float, \ + float, float4, \ + float4 + +typedef decltype(kernel_flash_attn_ext_vec) flash_attn_ext_vec_t; + +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk32_dv32")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk32_dv32_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk32_dv32_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk64_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv64_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk96_dv64_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk256_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk320_dv256_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q1_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q2_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk512_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk576_dv512_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; + +#undef FA_TYPES +#undef FA_TYPES_F32 + diff --git a/ggml/src/ggml-metal/kernels/gated_delta_net.metal b/ggml/src/ggml-metal/kernels/gated_delta_net.metal new file mode 100644 index 0000000..5e4861e --- /dev/null +++ b/ggml/src/ggml-metal/kernels/gated_delta_net.metal @@ -0,0 +1,264 @@ +#include "common.h" + +constant short FC_gated_delta_net_ne20 [[function_constant(FC_GATED_DELTA_NET + 0)]]; +constant short FC_gated_delta_net_ne30 [[function_constant(FC_GATED_DELTA_NET + 1)]]; +constant short FC_gated_delta_net_K [[function_constant(FC_GATED_DELTA_NET + 2)]]; + +#if 1 +template +kernel void kernel_gated_delta_net_impl( + constant ggml_metal_kargs_gated_delta_net & args, + device const char * q, + device const char * k, + device const char * v, + device const char * g, + device const char * b, + device const char * s, + device char * dst, + device char * dst_fuse, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { +#define S_v FC_gated_delta_net_ne20 +#define G FC_gated_delta_net_ne30 +#define K FC_gated_delta_net_K + + const uint tx = tpitg.x; + const uint ty = tpitg.y; + + const uint i23 = tgpig.z; // B (n_seqs) + const uint i21 = tgpig.y; // H (head) + const uint i20 = tgpig.x*NSG + ty; // row within S_v + + const uint i01 = i21 % args.ne01; + const uint i11 = i21 % args.ne11; + + const float scale = 1.0f / sqrt((float)S_v); + + // input state layout [S_v, S_v, H, n_seqs] (s0 only): per-seq stride is H*D. + // state is stored transposed: M[i20][is] = S[is][i20], so row i20 is contiguous + const uint state_in_base = (i23*args.ne21 + i21)*S_v*S_v + i20*S_v; + device const float * s_ptr = (device const float *) (s) + state_in_base; + + float ls[NSG]; + + FOR_UNROLL (short j = 0; j < NSG; j++) { + const short is = tx*NSG + j; + ls[j] = s_ptr[is]; + } + + device float * dst_attn = (device float *) (dst) + (i23*args.ne22*args.ne21 + i21)*S_v + i20; + + device const float * q_ptr = (device const float *) (q + i23*args.nb03 + i01*args.nb01); + device const float * k_ptr = (device const float *) (k + i23*args.nb13 + i11*args.nb11); + device const float * v_ptr = (device const float *) (v + i23*args.nb23 + i21*args.nb21); + + device const float * b_ptr = (device const float *) (b) + (i23*args.ne22*args.ne21 + i21); + device const float * g_ptr = (device const float *) (g) + (i23*args.ne22*args.ne21 + i21)*G; + + // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. + // When n_tokens < K, only slots 0..n_tokens-1 are written; older slots are caller-owned. + + // output state base offset: after attention scores + const uint attn_size = args.ne22 * args.ne21 * S_v * args.ne23; + // output state per-slot size: S_v * S_v * H * n_seqs + const uint state_size_per_snap = S_v * S_v * args.ne21 * args.ne23; + // per-(seq,head) offset within a slot + const uint state_out_base = (i23*args.ne21 + i21)*S_v*S_v + i20*S_v; + + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + attn_size; + const uint slot_stride = fused ? (uint)args.nb_out : state_size_per_snap; + + for (short t = 0; t < args.ne22; t++) { + float s_k = 0.0f; + + if (G == 1) { + const float g_exp = exp(g_ptr[0]); + + FOR_UNROLL (short j = 0; j < NSG; j++) { + const short is = tx*NSG + j; + ls[j] *= g_exp; + + s_k += ls[j]*k_ptr[is]; + } + } else { + // KDA + FOR_UNROLL (short j = 0; j < NSG; j++) { + const short is = tx*NSG + j; + ls[j] *= exp(g_ptr[is]); + + s_k += ls[j]*k_ptr[is]; + } + } + + s_k = simd_sum(s_k); + + const float d = (v_ptr[i20] - s_k)*b_ptr[0]; + + float y = 0.0f; + + FOR_UNROLL (short j = 0; j < NSG; j++) { + const short is = tx*NSG + j; + ls[j] += k_ptr[is]*d; + + y += ls[j]*q_ptr[is]; + } + + y = simd_sum(y); + + if (tx == 0) { + dst_attn[t*args.ne21*S_v] = y*scale; + } + + q_ptr += args.ns02; + k_ptr += args.ns12; + v_ptr += args.ns22; + + b_ptr += args.ne21; + g_ptr += args.ne21*G; + + if (K > 1) { + const int target_slot = (int)args.ne22 - 1 - (int)t; + if (target_slot >= 0 && target_slot < (int)K) { + device float * dst_state = (device float *)state_out + (uint)target_slot * slot_stride + state_out_base; + FOR_UNROLL (short j = 0; j < NSG; j++) { + const short is = tx*NSG + j; + dst_state[is] = ls[j]; + } + } + } + } + + if (K == 1) { + device float * dst_state = (device float *)state_out + state_out_base; + FOR_UNROLL (short j = 0; j < NSG; j++) { + const short is = tx*NSG + j; + dst_state[is] = ls[j]; + } + } + +#undef S_v +#undef G +#undef K +} + +typedef decltype(kernel_gated_delta_net_impl<4>) kernel_gated_delta_net_t; + +template [[host_name("kernel_gated_delta_net_f32_1")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl<1>; +template [[host_name("kernel_gated_delta_net_f32_2")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl<2>; +template [[host_name("kernel_gated_delta_net_f32_4")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl<4>; + +#else +// a simplified version of the above +// no performance improvement, so keep the above version for now + +template +kernel void kernel_gated_delta_net_impl( + constant ggml_metal_kargs_gated_delta_net & args, + device const char * q, + device const char * k, + device const char * v, + device const char * g, + device const char * b, + device const char * s, + device char * dst, + device char * dst_fuse, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { +#define S_v FC_gated_delta_net_ne20 +#define G FC_gated_delta_net_ne30 + + const uint tx = tpitg.x; + const uint ty = tpitg.y; + + const uint i23 = tgpig.z; // B + const uint i21 = tgpig.y; // H + const uint i20 = tgpig.x*NSG + ty; + + const uint i01 = i21 % args.ne01; + const uint i11 = i21 % args.ne11; + + const float scale = 1.0f / sqrt((float)S_v); + + device const float * s_ptr = (device const float *) (s) + (i23*args.ne21 + i21)*S_v*S_v + i20; + + float lsf[NSG]; + + FOR_UNROLL (short j = 0; j < NSG; j++) { + const short is = tx*NSG + j; + lsf[j] = s_ptr[is*S_v]; + } + + thread T * ls = (thread T *) (lsf); + + device float * dst_attn = (device float *) (dst) + (i23*args.ne22*args.ne21 + i21)*S_v + i20; + + device const float * q_ptr = (device const float *) (q + i23*args.nb03 + i01*args.nb01); + device const float * k_ptr = (device const float *) (k + i23*args.nb13 + i11*args.nb11); + device const float * v_ptr = (device const float *) (v + i23*args.nb23 + i21*args.nb21); + + device const float * b_ptr = (device const float *) (b) + (i23*args.ne22*args.ne21 + i21); + device const float * g_ptr = (device const float *) (g) + (i23*args.ne22*args.ne21 + i21)*G; + + for (short t = 0; t < args.ne22; t++) { + device const T * qt_ptr = (device const T *) (q_ptr); + device const T * kt_ptr = (device const T *) (k_ptr); + device const T * gt_ptr = (device const T *) (g_ptr); + + if (G == 1) { + *ls *= exp(g_ptr[0]); + } else { + // KDA + *ls *= exp(gt_ptr[tx]); + } + + const float s_k = simd_sum(dot(*ls, kt_ptr[tx])); + + const float d = (v_ptr[i20] - s_k)*b_ptr[0]; + + *ls += kt_ptr[tx]*d; + + const float y = simd_sum(dot(*ls, qt_ptr[tx])); + + if (tx == 0) { + *dst_attn = y*scale; + } + + q_ptr += args.ns02; + k_ptr += args.ns12; + v_ptr += args.ns22; + + b_ptr += args.ne21; + g_ptr += args.ne21*G; + + dst_attn += args.ne21*S_v; + } + + // when fused with the cache cpy, write the snapshots straight into the cache buffer using + // the slot stride; otherwise append them after the attn scores (nb_out == 0) + const bool fused = args.nb_out > 0; + const device float * state_out = fused ? (device float *)dst_fuse : (device float *)dst + args.ne23*args.ne22*args.ne21*S_v; + const uint slot_stride = fused ? (uint)args.nb_out : S_v*S_v; + + device float * dst_state = (device float *)state_out + (i23*args.ne21 + i21)*slot_stride + i20; + device T * dstt_state = (device T *) (dst_state); + + FOR_UNROLL (short j = 0; j < NSG; j++) { + const short is = tx*NSG + j; + dst_state[is*S_v] = lsf[j]; + } + +#undef S_v +#undef G +} + +typedef decltype(kernel_gated_delta_net_impl) kernel_gated_delta_net_t; + +template [[host_name("kernel_gated_delta_net_f32_1")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl; +template [[host_name("kernel_gated_delta_net_f32_2")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl; +template [[host_name("kernel_gated_delta_net_f32_4")]] kernel kernel_gated_delta_net_t kernel_gated_delta_net_impl; +#endif diff --git a/ggml/src/ggml-metal/kernels/misc.metal b/ggml/src/ggml-metal/kernels/misc.metal new file mode 100644 index 0000000..90d3fbe --- /dev/null +++ b/ggml/src/ggml-metal/kernels/misc.metal @@ -0,0 +1,908 @@ +#include "common.h" + +kernel void kernel_argmax_f32( + constant ggml_metal_kargs_argmax & args, + device const char * src0, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint ntg[[threads_per_threadgroup]]) { + device const float * x_row = (device const float *) ((device const char *) src0 + tgpig * args.nb01); + + float lmax = -INFINITY; + int32_t larg = -1; + + for (int i00 = tpitg; i00 < args.ne00; i00 += ntg) { + if (x_row[i00] > lmax) { + lmax = x_row[i00]; + larg = i00; + } + } + + // find the argmax value in the block + float max_val = simd_max(lmax); + int32_t arg_val = simd_max(select(-1, larg, lmax == max_val)); + + device int32_t * dst_i32 = (device int32_t *) dst; + + threadgroup float * shared_maxval = (threadgroup float *) shmem; + threadgroup int32_t * shared_argmax = (threadgroup int32_t *) shmem + N_SIMDWIDTH; + + if (ntg > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_maxval[tiisg] = -INFINITY; + shared_argmax[tiisg] = -1; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shared_maxval[sgitg] = max_val; + shared_argmax[sgitg] = arg_val; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + max_val = shared_maxval[tiisg]; + arg_val = shared_argmax[tiisg]; + + float max_val_reduced = simd_max(max_val); + int32_t arg_val_reduced = simd_max(select(-1, arg_val, max_val == max_val_reduced)); + + dst_i32[tgpig] = arg_val_reduced; + + return; + } + + dst_i32[tgpig] = arg_val; +} + +kernel void kernel_argmax_part_f32( + constant ggml_metal_kargs_argmax & args, + device const char * src0, + device char * tmp, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int row = tgpig.y; + const int part = tgpig.x; + const int ne00 = (int) args.ne00; + const int npart = (int) args.npart; + const int chunk = (ne00 + npart - 1) / npart; + const int start = part * chunk; + const int end = min(start + chunk, ne00); + + device const float * x_row = (device const float *) ((device const char *) src0 + row * args.nb01); + + float lmax = -INFINITY; + int32_t larg = -1; + + for (int i00 = start + tpitg.x; i00 < end; i00 += ntg.x) { + if (x_row[i00] > lmax) { + lmax = x_row[i00]; + larg = i00; + } + } + + // find the argmax value in the block + float max_val = simd_max(lmax); + int32_t arg_val = simd_max(select(-1, larg, lmax == max_val)); + + threadgroup float * shared_maxval = (threadgroup float *) shmem; + threadgroup int32_t * shared_argmax = (threadgroup int32_t *) shmem + N_SIMDWIDTH; + + if (ntg.x > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_maxval[tiisg] = -INFINITY; + shared_argmax[tiisg] = -1; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shared_maxval[sgitg] = max_val; + shared_argmax[sgitg] = arg_val; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + max_val = shared_maxval[tiisg]; + arg_val = shared_argmax[tiisg]; + + float max_val_reduced = simd_max(max_val); + int32_t arg_val_reduced = simd_max(select(-1, arg_val, max_val == max_val_reduced)); + + max_val = max_val_reduced; + arg_val = arg_val_reduced; + } + + if (tpitg.x == 0) { + device float * tmp_max = (device float *) tmp; + device int32_t * tmp_arg = (device int32_t *) tmp + npart * (int) args.ne01; + + tmp_max[row*npart + part] = max_val; + tmp_arg[row*npart + part] = arg_val; + } +} + +kernel void kernel_argmax_final_f32( + constant ggml_metal_kargs_argmax & args, + device const char * tmp, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int row = tgpig.x; + const int npart = (int) args.npart; + + device const float * tmp_max = (device const float *) tmp; + device const int32_t * tmp_arg = (device const int32_t *) tmp + npart * (int) args.ne01; + + float lmax = -INFINITY; + int32_t larg = -1; + + for (int i = tpitg.x; i < npart; i += ntg.x) { + if (tmp_max[row*npart + i] > lmax) { + lmax = tmp_max[row*npart + i]; + larg = tmp_arg[row*npart + i]; + } + } + + // find the argmax value in the block + float max_val = simd_max(lmax); + int32_t arg_val = simd_max(select(-1, larg, lmax == max_val)); + + threadgroup float * shared_maxval = (threadgroup float *) shmem; + threadgroup int32_t * shared_argmax = (threadgroup int32_t *) shmem + N_SIMDWIDTH; + + if (ntg.x > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_maxval[tiisg] = -INFINITY; + shared_argmax[tiisg] = -1; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shared_maxval[sgitg] = max_val; + shared_argmax[sgitg] = arg_val; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + max_val = shared_maxval[tiisg]; + arg_val = shared_argmax[tiisg]; + + float max_val_reduced = simd_max(max_val); + int32_t arg_val_reduced = simd_max(select(-1, arg_val, max_val == max_val_reduced)); + + max_val = max_val_reduced; + arg_val = arg_val_reduced; + } + + if (tpitg.x == 0) { + ((device int32_t *) dst)[row] = arg_val; + } +} + +kernel void kernel_diag_f32( + constant ggml_metal_kargs_diag & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]]) { + constexpr short NW = N_SIMDWIDTH; + + const int32_t i3 = tgpig.z; + const int32_t i2 = tgpig.y; + const int32_t i1 = tgpig.x; + + device const float * src0_ptr = (device const float *)(src0 + i2*args.nb02 + i3*args.nb03); + device float * dst_ptr = (device float *)(dst + i1*args.nb01 + i2*args.nb2 + i3*args.nb3); + + for (int i0 = tiitg; i0 < args.ne0; i0 += NW) { + dst_ptr[i0] = i0 == i1 ? src0_ptr[i0] : 0.0f; + } +} + +kernel void kernel_roll_f32( + constant ggml_metal_kargs_roll & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int64_t i3 = tgpig.z; + const int64_t i2 = tgpig.y; + const int64_t i1 = tgpig.x; + + device const float * src0_ptr = (device const float *) src0; + device float * dst_ptr = (device float *) dst; + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + // apply shifts and wrap around + int64_t i00 = i0 - args.s0; + int64_t i01 = i1 - args.s1; + int64_t i02 = i2 - args.s2; + int64_t i03 = i3 - args.s3; + + if (i00 < 0) { i00 += args.ne00; } else if (i00 >= args.ne00) { i00 -= args.ne00; } + if (i01 < 0) { i01 += args.ne01; } else if (i01 >= args.ne01) { i01 -= args.ne01; } + if (i02 < 0) { i02 += args.ne02; } else if (i02 >= args.ne02) { i02 -= args.ne02; } + if (i03 < 0) { i03 += args.ne03; } else if (i03 >= args.ne03) { i03 -= args.ne03; } + + int64_t src_idx = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00 + i00; + int64_t dst_idx = i3 *args.ne2 *args.ne1 *args.ne0 + i2 *args.ne1 *args.ne0 + i1 *args.ne0 + i0; + + dst_ptr[dst_idx] = src0_ptr[src_idx]; + } +} + +constant bool FC_pad_circular [[function_constant(FC_PAD + 0)]]; + +// circular means on a torus, so the coordinates wrap around +static inline int32_t wrap_around(int32_t coord, int32_t size) { + return (coord + size) % size; +} + +kernel void kernel_pad_f32( + constant ggml_metal_kargs_pad & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int32_t i3 = tgpig.z; + const int32_t i2 = tgpig.y; + const int32_t k0 = tgpig.x/args.ne1; + const int32_t i1 = tgpig.x - k0*args.ne1; + + const int32_t ne00 = args.ne00; + const int32_t ne01 = args.ne01; + const int32_t ne02 = args.ne02; + const int32_t ne03 = args.ne03; + + int32_t i01 = i1 - args.lp1; + int32_t i02 = i2 - args.lp2; + int32_t i03 = i3 - args.lp3; + + if (FC_pad_circular) { + i01 = wrap_around(i01, ne01); + i02 = wrap_around(i02, ne02); + i03 = wrap_around(i03, ne03); + } + + device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1); + + // the row lies in the padded region, so no source row backs it + if (i01 < 0 || i01 >= ne01 || + i02 < 0 || i02 >= ne02 || + i03 < 0 || i03 >= ne03) { + for (int32_t l0 = 0; l0 < 1024; l0 += ntg.x) { + const int32_t i0 = k0*1024 + tpitg.x + l0; + if (i0 >= args.ne0) { + break; + } + + dst_ptr[i0] = 0.0f; + } + + return; + } + + device const char * src0_row = src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01; + + for (int32_t l0 = 0; l0 < 1024; l0 += ntg.x) { + const int32_t i0 = k0*1024 + tpitg.x + l0; + if (i0 >= args.ne0) { + break; + } + + int32_t i00 = i0 - args.lp0; + + if (FC_pad_circular) { + i00 = wrap_around(i00, ne00); + } + + dst_ptr[i0] = i00 >= 0 && i00 < ne00 ? *((device const float *) (src0_row + i00*args.nb00)) : 0.0f; + } +} + +// TODO: this is slow - optimize +kernel void kernel_pad_reflect_1d_f32( + constant ggml_metal_kargs_pad_reflect_1d & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tgpg[[threadgroups_per_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int64_t i3 = tgpig.z; + const int64_t i2 = tgpig.y; + const int64_t i1 = tgpig.x; + + const int64_t i03 = i3; + const int64_t i02 = i2; + const int64_t i01 = i1; + + device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); + device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1); + + if (i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) { + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + if (i0 < args.p0) { + dst_ptr[i0] = src0_ptr[args.p0 - i0]; + } else if (i0 < args.ne0 - args.p1) { + dst_ptr[i0] = src0_ptr[i0 - args.p0]; + } else { + dst_ptr[i0] = src0_ptr[(args.ne0 - args.p1 - args.p0) - (args.p1 + 1 - (args.ne0 - i0)) - 1]; + } + } + } +} + +kernel void kernel_arange_f32( + constant ggml_metal_kargs_arange & args, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + device float * dst_ptr = (device float *) dst; + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + dst_ptr[i0] = args.start + args.step * i0; + } +} + +kernel void kernel_timestep_embedding_f32( + constant ggml_metal_kargs_timestep_embedding & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + int i = tgpig.x; + device float * embed_data = (device float *)(dst + i*args.nb1); + + int half_ = args.dim / 2; + for (int j = tpitg.x; j < half_; j += ntg.x) { + float timestep = ((device float *)src0)[i]; + float freq = (float)exp(-log((float)args.max_period) * j / half_); + float arg = timestep * freq; + embed_data[j ] = cos(arg); + embed_data[j + half_] = sin(arg); + } + + if (args.dim % 2 != 0 && tpitg.x == 0) { + embed_data[2 * half_] = 0.f; + } +} + +kernel void kernel_opt_step_adamw_f32( + constant ggml_metal_kargs_opt_step_adamw & args, + device float * x, + device const float * g, + device float * g_m, + device float * g_v, + device const float * pars, + uint gid[[thread_position_in_grid]]) { + + if (gid >= args.np) { + return; + } + + const float alpha = pars[0]; + const float beta1 = pars[1]; + const float beta2 = pars[2]; + const float eps = pars[3]; + const float wd = pars[4]; + const float beta1h = pars[5]; + const float beta2h = pars[6]; + + const float gi = g[gid]; + const float gmi = g_m[gid] * beta1 + gi * (1.0f - beta1); + const float gvi = g_v[gid] * beta2 + gi * gi * (1.0f - beta2); + + g_m[gid] = gmi; + g_v[gid] = gvi; + + const float mh = gmi * beta1h; + const float vh = sqrt(gvi * beta2h) + eps; + + x[gid] = x[gid] * (1.0f - alpha * wd) - alpha * mh / vh; +} + +kernel void kernel_opt_step_sgd_f32( + constant ggml_metal_kargs_opt_step_sgd & args, + device float * x, + device const float * g, + device const float * pars, + uint gid[[thread_position_in_grid]]) { + + if (gid >= args.np) { + return; + } + + x[gid] = x[gid] * (1.0f - pars[0] * pars[1]) - pars[0] * g[gid]; +} + +template +kernel void kernel_memset( + constant ggml_metal_kargs_memset & args, + device T * dst, + uint tpig[[thread_position_in_grid]]) { + dst[tpig] = args.val; +} + +typedef decltype(kernel_memset) kernel_memset_t; + +template [[host_name("kernel_memset_i64")]] kernel kernel_memset_t kernel_memset; + +constant short FC_count_equal_nsg [[function_constant(FC_COUNT_EQUAL + 0)]]; + +template +kernel void kernel_count_equal( + constant ggml_metal_kargs_count_equal & args, + device const char * src0, + device const char * src1, + device atomic_int * dst, + threadgroup int32_t * shmem_i32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const short NSG = FC_count_equal_nsg; + + const int i3 = tgpig.z; + const int i2 = tgpig.y; + const int i1 = tgpig.x; + + if (i3 >= args.ne03 || i2 >= args.ne02 || i1 >= args.ne01) { + return; + } + + int sum = 0; + + device const char * base0 = src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03; + device const char * base1 = src1 + i1*args.nb11 + i2*args.nb12 + i3*args.nb13; + + for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) { + const T v0 = *(device const T *)(base0 + i0*args.nb00); + const T v1 = *(device const T *)(base1 + i0*args.nb10); + sum += (v0 == v1); + } + + sum = simd_sum(sum); + + if (tiisg == 0) { + shmem_i32[sgitg] = sum; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (sgitg == 0) { + float v = 0.0f; + if (tpitg.x < NSG) { + v = shmem_i32[tpitg.x]; + } + + float total = simd_sum(v); + if (tpitg.x == 0) { + atomic_fetch_add_explicit(dst, (int32_t) total, memory_order_relaxed); + } + } +} + +typedef decltype(kernel_count_equal) kernel_count_equal_t; + +template [[host_name("kernel_count_equal_i32")]] kernel kernel_count_equal_t kernel_count_equal; + +template +kernel void kernel_snake( + constant ggml_metal_kargs_snake & args, + device const T * x, + device const float * a, + device const float * inv_b, + device T * dst, + uint tgpig [[threadgroup_position_in_grid]], + uint tpitg [[thread_position_in_threadgroup]], + uint ntg [[threads_per_threadgroup]]) { + + const int idx = tgpig * ntg + tpitg; + if (idx >= args.T * args.C) { + return; + } + + const int c = idx / args.T; // x is [T, C], a / inv_b collapse to [1, C] + const float xi = float(x[idx]); + const float si = sin(a[c] * xi); + dst[idx] = T(xi + si * si * inv_b[c]); +} + +template [[host_name("kernel_snake_f32")]] kernel void kernel_snake(constant ggml_metal_kargs_snake &, device const float *, device const float *, device const float *, device float *, uint, uint, uint); +template [[host_name("kernel_snake_f16")]] kernel void kernel_snake(constant ggml_metal_kargs_snake &, device const half *, device const float *, device const float *, device half *, uint, uint, uint); +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_snake_bf16")]] kernel void kernel_snake(constant ggml_metal_kargs_snake &, device const bfloat *, device const float *, device const float *, device bfloat *, uint, uint, uint); +#endif + +template +kernel void kernel_fwht( + constant ggml_metal_kargs_fwht & args, + device const src_t * src, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + + constexpr int NW = N_SIMDWIDTH; + constexpr int NE = N / NW; + + const float scale = 1.0f / sqrt((float) N); + + const int sg_per_tg = ntg.x / NW; + const int64_t r = tgpig.x * sg_per_tg + sgitg; + if (r >= args.nrows) { + return; + } + + src += r * N; + dst += r * N; + + const int lane = tiisg; + + float reg[NE]; + for (int i = 0; i < NE; i++) { + reg[i] = float(src[i*NW + lane])*scale; + } + for (int i = 1; i < NW; i *= 2) { + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = simd_shuffle_xor(val, i); + reg[j] = val2 - val + 2*((lane & i) == 0)*val; + } + } + + for (int i = NW; i < N; i *= 2) { + const int step = i / NW; + for (int j = 0; j < NE; j += (2 * step)) { + for (int k = 0; k < step; k++) { + const float x = reg[j + k ]; + const float y = reg[j + k + step]; + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + + for (int i = 0; i < NE; i++) { + dst[i*NW + lane] = reg[i]; + } +} + +// Wide blocks: one row per threadgroup instead of per simdgroup, so each thread keeps +// N/NT values rather than N/32. Butterflies below the simdgroup width still shuffle; +// those up to NT go through threadgroup memory; the rest stay in registers. +// TODO: try avoiding branch https://github.com/ggml-org/llama.cpp/pull/29094#discussion_r4049563223 +// TODO: try to unroll loops +template +kernel void kernel_fwht_tg( + constant ggml_metal_kargs_fwht & args, + device const src_t * src, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + + constexpr int NW = N_SIMDWIDTH; + constexpr int NE = N / NT; + + threadgroup float shmem[N]; + + const float scale = 1.0f / sqrt((float) N); + + const int64_t r = tgpig.x; + if (r >= args.nrows) { + return; + } + + src += r * N; + dst += r * N; + + const int tid = sgitg * NW + tiisg; + + float reg[NE]; + for (int i = 0; i < NE; i++) { + reg[i] = float(src[i*NT + tid])*scale; + } + + for (int i = 1; i < NW; i *= 2) { + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = simd_shuffle_xor(val, i); + reg[j] = (tid & i) == 0 ? val2 + val : val2 - val; + } + } + + for (int i = NW; i < NT; i *= 2) { + for (int j = 0; j < NE; j++) { + shmem[j*NT + tid] = reg[j]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = shmem[j*NT + (tid ^ i)]; + reg[j] = (tid & i) == 0 ? val2 + val : val2 - val; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + for (int i = NT; i < N; i *= 2) { + const int step = i / NT; + for (int j = 0; j < NE; j += (2 * step)) { + for (int k = 0; k < step; k++) { + const float x = reg[j + k ]; + const float y = reg[j + k + step]; + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + + for (int i = 0; i < NE; i++) { + dst[i*NT + tid] = reg[i]; + } +} + +typedef decltype(kernel_fwht<64, float>) kernel_fwht_f32_t; +typedef decltype(kernel_fwht<64, half>) kernel_fwht_f16_t; + +template [[host_name("kernel_fwht_f32_64")]] kernel kernel_fwht_f32_t kernel_fwht<64, float>; +template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_f32_t kernel_fwht<128, float>; +template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_f32_t kernel_fwht<256, float>; + +template [[host_name("kernel_fwht_f16_64")]] kernel kernel_fwht_f16_t kernel_fwht<64, half>; +template [[host_name("kernel_fwht_f16_128")]] kernel kernel_fwht_f16_t kernel_fwht<128, half>; +template [[host_name("kernel_fwht_f16_256")]] kernel kernel_fwht_f16_t kernel_fwht<256, half>; + +template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_f32_t kernel_fwht_tg<512, GGML_METAL_FWHT_TG_NT, float>; +template [[host_name("kernel_fwht_f32_1024")]] kernel kernel_fwht_f32_t kernel_fwht_tg<1024, GGML_METAL_FWHT_TG_NT, float>; +template [[host_name("kernel_fwht_f32_2048")]] kernel kernel_fwht_f32_t kernel_fwht_tg<2048, GGML_METAL_FWHT_TG_NT, float>; +template [[host_name("kernel_fwht_f32_4096")]] kernel kernel_fwht_f32_t kernel_fwht_tg<4096, GGML_METAL_FWHT_TG_NT, float>; +template [[host_name("kernel_fwht_f32_8192")]] kernel kernel_fwht_f32_t kernel_fwht_tg<8192, GGML_METAL_FWHT_TG_NT, float>; + +template [[host_name("kernel_fwht_f16_512")]] kernel kernel_fwht_f16_t kernel_fwht_tg<512, GGML_METAL_FWHT_TG_NT, half>; +template [[host_name("kernel_fwht_f16_1024")]] kernel kernel_fwht_f16_t kernel_fwht_tg<1024, GGML_METAL_FWHT_TG_NT, half>; +template [[host_name("kernel_fwht_f16_2048")]] kernel kernel_fwht_f16_t kernel_fwht_tg<2048, GGML_METAL_FWHT_TG_NT, half>; +template [[host_name("kernel_fwht_f16_4096")]] kernel kernel_fwht_f16_t kernel_fwht_tg<4096, GGML_METAL_FWHT_TG_NT, half>; +template [[host_name("kernel_fwht_f16_8192")]] kernel kernel_fwht_f16_t kernel_fwht_tg<8192, GGML_METAL_FWHT_TG_NT, half>; + +constant int FC_dsv4_hc_n_hc [[function_constant(FC_DSV4_HC + 0)]]; + +kernel void kernel_dsv4_hc_comb_f32( + constant ggml_metal_kargs_dsv4_hc_comb & args, + device const char * mixes, + device const char * scale, + device const char * base, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr ushort hc = 4; + constexpr ushort comb_offset = 2*hc; + + const int it = tgpig.x*ntg.y + sgitg; + if (it >= args.n_tokens) { + return; + } + + float scale_lane = 0.0f; + if (tiisg == 0) { + scale_lane = *(device const float *) (scale + 2*args.nb_s0); + } + const float scale_comb = simd_shuffle(scale_lane, 0); + + float v = 0.0f; + if (tiisg < hc*hc) { + v = *(device const float *) (mixes + (comb_offset + tiisg)*args.nb_m0 + it*args.nb_m1)*scale_comb + + *(device const float *) (base + (comb_offset + tiisg)*args.nb_b0); + } + + // Softmax across destinations (the four contiguous lanes for each source). + float vmax = max(v, simd_shuffle_xor(v, 1)); + vmax = max(vmax, simd_shuffle_xor(vmax, 2)); + v = exp(v - vmax); + + float sum = v + simd_shuffle_xor(v, 1); + sum += simd_shuffle_xor(sum, 2); + v = v/sum + args.eps; + + // Normalize columns: equal destination indices are four lanes apart. + sum = v + simd_shuffle_xor(v, 4); + sum += simd_shuffle_xor(sum, 8); + v /= sum + args.eps; + + for (int i = 1; i < args.n_iter; ++i) { + sum = v + simd_shuffle_xor(v, 1); + sum += simd_shuffle_xor(sum, 2); + v /= sum + args.eps; + + sum = v + simd_shuffle_xor(v, 4); + sum += simd_shuffle_xor(sum, 8); + v /= sum + args.eps; + } + + if (tiisg < hc*hc) { + const ushort idst = tiisg & 3; + const ushort isrc = tiisg >> 2; + *(device float *) (dst + idst*args.nb_d0 + isrc*args.nb_d1 + it*args.nb_d2) = v; + } +} + +kernel void kernel_dsv4_hc_pre_f32( + constant ggml_metal_kargs_dsv4_hc_pre & args, + device const char * x, + device const char * weights, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + if (i0 >= args.n_embd) { + return; + } + + device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2; + float result = 0.0f; + FOR_UNROLL (int ih = 0; ih < FC_dsv4_hc_n_hc; ++ih) { + const float xv = *(device const float *) (xb + ih*args.nb_x1); + const float wv = *(device const float *) (weights + ih*args.nb_w0 + it*args.nb_w1); + result = fma(xv, wv, result); + } + + *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result; +} + +kernel void kernel_dsv4_hc_pre_gated_f32( + constant ggml_metal_kargs_dsv4_hc_pre & args, + device const char * x, + device const char * gate, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + if (i0 >= args.n_embd) { + return; + } + + device const char * xb = x + i0*args.nb_x0 + it*args.nb_x2; + device const char * gb = gate + i0*args.nb_w0 + it*args.nb_w2; + float result = 0.0f; + FOR_UNROLL (int ih = 0; ih < FC_dsv4_hc_n_hc; ++ih) { + const float g = 1.0f/(1.0f + exp(-*(device const float *) (gb + ih*args.nb_w1))); + const float xv = *(device const float *) (xb + ih*args.nb_x1); + result = fma(xv, g, result); + } + + *(device float *) (dst + i0*args.nb_d0 + it*args.nb_d1) = args.scale*result; +} + +kernel void kernel_dsv4_hc_post_nocomb_f32( + constant ggml_metal_kargs_dsv4_hc_post & args, + device const char * x, + device const char * residual, + device const char * post, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr ushort hc = 4; + + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + float post_lane = 0.0f; + if (tiisg < hc) { + post_lane = *(device const float *) (post + tiisg*args.nb_p0 + it*args.nb_p1); + } + + float post_reg[hc]; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + post_reg[idst] = simd_shuffle(post_lane, idst); + } + + if (i0 >= args.n_embd) { + return; + } + + const float xv = *(device const float *) (x + i0*args.nb_x0 + it*args.nb_x1); + device const char * rb = residual + i0*args.nb_r0 + it*args.nb_r2; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + const float rv = *(device const float *) (rb + idst*args.nb_r1); + *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = xv*post_reg[idst] + rv; + } +} + +kernel void kernel_dsv4_hc_post_f32( + constant ggml_metal_kargs_dsv4_hc_post & args, + device const char * x, + device const char * residual, + device const char * post, + device const char * comb, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr ushort hc = 4; + + const int it = tgpig.y; + const int i0 = ((int) tgpig.x*ntg.y + sgitg)*32 + tiisg; + + float coeff_lane = 0.0f; + if (tiisg < hc) { + coeff_lane = *(device const float *) (post + tiisg*args.nb_p0 + it*args.nb_p1); + } else if (tiisg < hc + hc*hc) { + const ushort idx = tiisg - hc; + const ushort idst = idx & 3; + const ushort isrc = idx >> 2; + coeff_lane = *(device const float *) (comb + idst*args.nb_c0 + isrc*args.nb_c1 + it*args.nb_c2); + } + + float post_reg[hc]; + float comb_reg[hc][hc]; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + post_reg[idst] = simd_shuffle(coeff_lane, idst); + } + FOR_UNROLL (ushort isrc = 0; isrc < hc; ++isrc) { + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + comb_reg[isrc][idst] = simd_shuffle(coeff_lane, hc + idst + hc*isrc); + } + } + + if (i0 >= args.n_embd) { + return; + } + + const float xv = *(device const float *) (x + i0*args.nb_x0 + it*args.nb_x1); + float result[hc]; + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + result[idst] = xv*post_reg[idst]; + } + + device const char * rb = residual + i0*args.nb_r0 + it*args.nb_r2; + FOR_UNROLL (ushort isrc = 0; isrc < hc; ++isrc) { + const float rv = *(device const float *) (rb + isrc*args.nb_r1); + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + result[idst] = fma(rv, comb_reg[isrc][idst], result[idst]); + } + } + + FOR_UNROLL (ushort idst = 0; idst < hc; ++idst) { + *(device float *) (dst + i0*args.nb_d0 + idst*args.nb_d1 + it*args.nb_d2) = result[idst]; + } +} diff --git a/ggml/src/ggml-metal/kernels/mul_mm.metal b/ggml/src/ggml-metal/kernels/mul_mm.metal new file mode 100644 index 0000000..edcc061 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/mul_mm.metal @@ -0,0 +1,1017 @@ +#include "common.h" +#include "dequantize.h" + +constant bool FC_mul_mm_bc_inp [[function_constant(FC_MUL_MM + 0)]]; +constant bool FC_mul_mm_bc_out [[function_constant(FC_MUL_MM + 1)]]; +constant short FC_mul_mm_ne12 [[function_constant(FC_MUL_MM + 2)]]; +constant short FC_mul_mm_ne13 [[function_constant(FC_MUL_MM + 3)]]; +constant short FC_mul_mm_r2 [[function_constant(FC_MUL_MM + 4)]]; +constant short FC_mul_mm_r3 [[function_constant(FC_MUL_MM + 5)]]; +constant int32_t FC_mul_mm_id_map0_ne02 [[function_constant(FC_MUL_MM + 7)]]; + +// each block_q contains 16*nl weights +#ifdef GGML_METAL_HAS_TENSOR +template< + typename SA, typename SA_4x4, typename SA_8x8, + typename SB, typename SB_2x4, typename SB_8x8, + typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread SA_4x4 &), + typename T0, typename T0_4x4, typename T1, typename T1_2x4> +kernel void kernel_mul_mm( + constant ggml_metal_kargs_mul_mm & args, + device const char * srcA, + device const char * srcB, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig [[threadgroup_position_in_grid]], + ushort tiitg [[thread_index_in_threadgroup]], + ushort sgitg [[simdgroup_index_in_threadgroup]]) { + (void) sgitg; + + // Matrix dimensions: A(M,K) x B(K,N) -> C(M,N) + const int K = args.ne00; + const int M = args.ne0; + const int N = args.ne1; + + // Batch dimension handling + const int im = tgpig.z; + const int i12 = im % FC_mul_mm_ne12; + const int i13 = im / FC_mul_mm_ne12; + + // Batch offsets for srcA and srcB + const uint64_t offset0 = (i12/FC_mul_mm_r2)*args.nb02 + (i13/FC_mul_mm_r3)*args.nb03; + + // Tile dimensions + constexpr int NRB = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X; + constexpr int NRA = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y; + + // Tile offsets in output matrix + const int ra = tgpig.y * NRA; + const int rb = tgpig.x * NRB; + + // Threadgroup memory for dequantized A tile only + threadgroup SA * sa = (threadgroup SA *)(shmem); + + // Work-item count for A loading + constexpr int A_WORK_ITEMS = NRA * N_MM_NK; + constexpr int NUM_THREADS = N_SIMDWIDTH * N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y; + + // tA wraps threadgroup memory + auto tA = tensor(sa, dextents(N_MM_NK_TOTAL, NRA)); + + // tB wraps device memory directly + device T1 * ptrB = (device T1 *)(srcB + args.nb12*i12 + args.nb13*i13); + const int strideB = args.nb11 / sizeof(T1); + auto tB = tensor(ptrB, dextents(K, N), array({1, strideB})); + + // Configure matmul operation + // note: K is dynamic_extent (clamped to the valid range in PHASE 2), since a static + // N_MM_NK_TOTAL K tile would read src1 out of bounds when K % N_MM_NK_TOTAL != 0 + // ref: https://github.com/ggml-org/llama.cpp/pull/27064 + mpp::tensor_ops::matmul2d< + mpp::tensor_ops::matmul2d_descriptor( + NRB, NRA, static_cast(dynamic_extent), false, true, true, + mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), + execution_simdgroups> mm; + + auto cT = mm.get_destination_cooperative_tensor(); + + // Accumulate partial results over K dimension + for (int loop_k = 0; loop_k < K; loop_k += N_MM_NK_TOTAL) { + // === PHASE 1: Dequantization of A into threadgroup memory === + for (int work = tiitg; work < A_WORK_ITEMS; work += NUM_THREADS) { + const int row = work / N_MM_NK; + const int k_chunk = work % N_MM_NK; + const int k_pos = loop_k + k_chunk * 16; + const short k_base = k_chunk * 16; + + // Bounds check: skip device read if row is out of matrix bounds + if (ra + row < M) { + if (is_same::value && FC_mul_mm_bc_inp) { + // Element-wise reads when K is not aligned (nb01 not aligned for half4x4/float4x4). + // MSL spec Table 2.5: half4x4 requires 8-byte alignment. When K is odd, + // nb01 = K*2 is not 8-byte aligned, so odd-row pointers are misaligned. + // Mirrors the legacy kernel's existing guard. + device const T0 * row_ptr = (device const T0 *)(srcA + args.nb01 * (ra + row) + offset0); + + FOR_UNROLL (short i = 0; i < 16; i++) { + sa[row * N_MM_NK_TOTAL + (k_base + i)] = (k_pos + i < K) ? (SA) row_ptr[k_pos + i] : (SA)0; + } + } else { + const int block_idx = k_pos / (16 * nl); + const short il = (k_pos / 16) % nl; + + device const block_q * row_ptr = (device const block_q *)(srcA + args.nb01 * (ra + row) + offset0); + + SA_4x4 temp_a; + dequantize_func(row_ptr + block_idx, il, temp_a); + + FOR_UNROLL (short i = 0; i < 16; i++) { + // Zero-pad A for K positions beyond valid range (handles partial K iterations) + sa[row * N_MM_NK_TOTAL + (k_base + i)] = (k_pos + i < K) ? temp_a[i/4][i%4] : (SA)0; + } + } + } else { + // Zero-pad rows beyond matrix bounds + FOR_UNROLL (short i = 0; i < 16; i++) { + sa[row * N_MM_NK_TOTAL + (k_base + i)] = (SA)0; + } + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // === PHASE 2: Tensor matmul === + // Clamp the K extent of both operand tensors to the remaining valid K range so + // the dynamic-K op never reads past the K extent of src1 (or the staged A tile). + const int kExt = min(N_MM_NK_TOTAL, K - loop_k); + + auto tAv = tensor(sa, dextents(kExt, NRA), array({1, N_MM_NK_TOTAL})); + auto tBv = tensor(ptrB + loop_k + rb * strideB, dextents(kExt, N - rb), array({1, strideB})); + + mm.run(tBv, tAv, cT); + + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + // Store result tile to output matrix (with batch offset) + // a slice is not used because its byte offset is int32 and wraps at 2 GiB + // ref: https://github.com/ggml-org/llama.cpp/pull/28748 + const int tileM = min(NRA, M - ra); + const int tileN = min(NRB, N - rb); + + device float * dstTile = (device float *)dst + (uint64_t) im * N * M + (uint64_t) rb * M + ra; + + auto tD = tensor(dstTile, dextents(tileM, tileN), array({1, M})); + cT.store(tD); +} + +#else + +template< + typename S0, typename S0_4x4, typename S0_8x8, + typename S1, typename S1_2x4, typename S1_8x8, + typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread S0_4x4 &), + typename T0, typename T0_4x4, typename T1, typename T1_2x4> +kernel void kernel_mul_mm( + constant ggml_metal_kargs_mul_mm & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + threadgroup S0 * sa = (threadgroup S0 *)(shmem); + threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096); + + constexpr int NR0 = 64; + constexpr int NR1 = 32; + + constexpr int NK = 32; + constexpr int NL0 = NK/16; + constexpr int NL1 = NK/8; + + const int im = tgpig.z; + const int r0 = tgpig.y*NR0; + const int r1 = tgpig.x*NR1; + + // if this block is of 64x32 shape or smaller + const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0; + const short nr1 = (args.ne1 - r1 < NR1) ? (args.ne1 - r1) : NR1; + + // a thread shouldn't load data outside of the matrix + const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63 + const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31 + + const short il0 = (tiitg % NL0); + + short il = il0; + + const int i12 = im % FC_mul_mm_ne12; + const int i13 = im / FC_mul_mm_ne12; + + const uint64_t offset0 = (i12/FC_mul_mm_r2)*args.nb02 + (i13/FC_mul_mm_r3)*args.nb03; + const short offset1 = il0/nl; + + device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1; + + const short iy = 8*(tiitg % NL1); + + device const T1 * y = (device const T1 *)(src1 + + args.nb13*i13 + + args.nb12*i12 + + args.nb11*(r1 + lr1) + + args.nb10*iy); + + S0_8x8 ma[4]; + S1_8x8 mb[2]; + + simdgroup_float8x8 mc[8]; + + for (short i = 0; i < 8; i++){ + mc[i] = make_filled_simdgroup_matrix(0.f); + } + + for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { + // load data and store to threadgroup memory + if (is_same::value && FC_mul_mm_bc_inp) { + threadgroup_barrier(mem_flags::mem_threadgroup); + + // no need for dequantization + for (short i = 0; i < 16; i++) { + const short sx = 2*il0 + i/8; + const short sy = (tiitg/NL0)/8; + + //const short lx = i%8; + //const short ly = (tiitg/NL0)%8; + const short lx = (tiitg/NL0)%8; + const short ly = i%8; + + const short ib = 8*sx + sy; + + *(sa + 64*ib + 8*ly + lx) = loop_k + 16*il + i < args.ne00 ? (S0) *((device T0 *) x + i) : 0; + } + } else { + S0_4x4 temp_a; + dequantize_func(x, il, temp_a); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short i = 0; i < 16; i++) { + const short sx = 2*il0 + i/8; + const short sy = (tiitg/NL0)/8; + + //const short lx = i%8; + //const short ly = (tiitg/NL0)%8; + const short lx = (tiitg/NL0)%8; + const short ly = i%8; + + const short ib = 8*sx + sy; + + // NOTE: this is massively slower.. WTF? + //sa[64*ib + 8*ly + lx] = temp_a[i/4][i%4]; + + *(sa + 64*ib + 8*ly + lx) = temp_a[i/4][i%4]; + } + } + + if (FC_mul_mm_bc_inp) { + for (short i = 0; i < 8; ++i) { + const short sx = (tiitg%NL1); + const short sy = (tiitg/NL1)/8; + + const short lx = i; + const short ly = (tiitg/NL1)%8; + //const short lx = (tiitg/NL1)%8; + //const short ly = i; + + const short ib = 4*sx + sy; + + *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; + } + } else { + const short sx = (tiitg%NL1); + const short sy = (tiitg/NL1)/8; + + //const short dx = sx; + //const short dy = sy; + + const short ly = (tiitg/NL1)%8; + + const short ib = 4*sx + sy; + + *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y)); + } + + il = (il + 2 < nl) ? il + 2 : il % 2; + x = (il < 2) ? x + (2 + nl - 1)/nl : x; + + y += NK; + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // load matrices from threadgroup memory and conduct outer products + threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); + threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); + + FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 4; i++) { + simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + } + + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 2; i++) { + simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); + } + + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 8; i++){ + simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); + } + + lsma += 8*64; + lsmb += 4*64; + } + } + + if (!FC_mul_mm_bc_out || (r0 + NR0 <= args.ne0 && r1 + NR1 <= args.ne1)) { + // if no bounds checks on the output are needed, we can directly write to device memory + device float * C = (device float *) dst + + (r0 + 32*(sgitg & 1)) + \ + (uint64_t) (r1 + 16*(sgitg >> 1))*args.ne0 + (uint64_t) im*args.ne1*args.ne0; + + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], C + 8*(i%4) + (uint64_t) 8*args.ne0*(i/4), args.ne0, 0, false); + } + } else { + // block is smaller than 64x32, we should avoid writing data outside of the matrix + threadgroup_barrier(mem_flags::mem_threadgroup); + + threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (sgitg == 0) { + for (int j = tiitg; j < nr1; j += NR1) { + device float * D = (device float *) dst + r0 + (uint64_t) (r1 + j)*args.ne0 + (uint64_t) im*args.ne1*args.ne0; + device float4 * D4 = (device float4 *) D; + + threadgroup float * C = temp_str + (j*NR0); + threadgroup float4 * C4 = (threadgroup float4 *) C; + + int i = 0; + for (; i < nr0/4; i++) { + *(D4 + i) = *(C4 + i); + } + + i *= 4; + for (; i < nr0; i++) { + *(D + i) = *(C + i); + } + } + } + } +} + +#endif // GGML_METAL_HAS_TENSOR + +template // n_expert_used +kernel void kernel_mul_mm_id_map0( + constant ggml_metal_kargs_mul_mm_id_map0 & args, + device const char * src2, + device char * htpe, + device char * hids, + threadgroup char * shmem [[threadgroup(0)]], + ushort tpitg[[thread_position_in_threadgroup]], + ushort ntg[[threads_per_threadgroup]]) { + const short ide = tpitg; // expert id + + // per expert: entry count (pass 1), replaced by the base offset of the slice (pass 2) + threadgroup uint32_t * cnt = (threadgroup uint32_t *) (shmem); + threadgroup uint32_t * bas = cnt + FC_mul_mm_id_map0_ne02; + + // staging of the ids of ntg tokens + threadgroup uint16_t * sids = (threadgroup uint16_t *) (bas + FC_mul_mm_id_map0_ne02); + + // pass 1: count the entries of this expert - every occurrence of this expert within the + // ids is an entry (duplicate ids produce multiple entries for the same token) + uint32_t n_all = 0; + + for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens + if (i21 + tpitg < args.ne21) { + device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21); + + // 0xFFFF marks a skipped slot, it never matches an expert since ne02 fits in a threadgroup + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + const int32_t id = src2_i32[i20]; + sids[tpitg*ne20 + i20] = id == -1 ? 0xFFFF : (uint16_t) id; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (short t = 0; t < ntg; t++) { + if (i21 + t >= args.ne21) { + break; + } + + threadgroup const uint16_t * sids_t = sids + t*ne20; + + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + n_all += sids_t[i20] == ide; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + cnt[ide] = n_all; + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // exclusive prefix sum over the counts: the entries of this expert start at bas[ide] - + // every slice is padded by one entry for the uniform dummy store of pass 2 - and emit the + // entry tile descriptors of kernel_mul_mm_id: (expert, first entry, valid rows) per tile + if (tpitg == 0) { + constexpr uint32_t NTILE = 32; // must match NR1 of kernel_mul_mm_id + + uint32_t base = 0; + uint32_t tile = 0; + + device uint32_t * tpe_u32 = (device uint32_t *) (htpe); + + for (int e = 0; e < FC_mul_mm_id_map0_ne02; ++e) { + const uint32_t n = cnt[e]; + + bas[e] = base; + + for (uint32_t k = 0; k < n; k += NTILE) { + tpe_u32[1 + 3*tile ] = e; + tpe_u32[1 + 3*tile + 1] = base + k; + tpe_u32[1 + 3*tile + 2] = n - k < NTILE ? n - k : NTILE; + + tile++; + } + + base += n + 1; + } + + tpe_u32[0] = tile; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + // pass 2: write the entries of this expert into its slice of the compact list + device int32_t * ids_i32 = (device int32_t *) (hids); + + uint32_t ient = bas[ide]; + + for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens + if (i21 + tpitg < args.ne21) { + device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21); + + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + sids[tpitg*ne20 + i20] = src2_i32[i20]; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (short t = 0; t < ntg; t++) { + if (i21 + t >= args.ne21) { + break; + } + + threadgroup const uint16_t * sids_t = sids + t*ne20; + + short first = -1; + short n_ents = 0; + + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + const short hit = sids_t[i20] == ide; + + first = hit && first < 0 ? i20 : first; + n_ents += hit; + } + + // one store per token for every thread, regardless of the match - conditional + // stores of the divergent lanes are very slow - the dummy value of the tokens + // without a match lands on the padding slot of this slice and is never read + ids_i32[ient] = first >= 0 ? (i21 + t)*ne20 + first : 0; + + if (n_ents > 1) { + // duplicate ids: the remaining matches of this token + short j = 1; + + FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) { + if (sids_t[i20] == ide && i20 != first) { + ids_i32[ient + j] = (i21 + t)*ne20 + i20; + j += 1; + } + } + } + + ient += n_ents; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + } +} + +// kernel_mul_mm_id only writes the rows that an expert owns, so the skipped rows are zeroed here +kernel void kernel_mul_mm_id_zero( + constant ggml_metal_kargs_mul_mm_id_zero & args, + device const char * src2, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int i20 = tgpig.x; // slot + const int i21 = tgpig.y; // token + + const int32_t id = ((device const int32_t *) (src2 + i21*args.nb21))[i20]; + + if (id != -1) { + return; + } + + device float * dst_row = (device float *) (dst + i20*args.nb1 + i21*args.nb2); + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + dst_row[i0] = 0.0f; + } +} + +typedef decltype(kernel_mul_mm_id_map0<1>) kernel_mul_mm_id_map0_t; + +template [[host_name("kernel_mul_mm_id_map0_ne20_1" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<1>; +template [[host_name("kernel_mul_mm_id_map0_ne20_2" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<2>; +template [[host_name("kernel_mul_mm_id_map0_ne20_4" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<4>; +template [[host_name("kernel_mul_mm_id_map0_ne20_5" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<5>; +template [[host_name("kernel_mul_mm_id_map0_ne20_6" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<6>; +template [[host_name("kernel_mul_mm_id_map0_ne20_8" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<8>; +template [[host_name("kernel_mul_mm_id_map0_ne20_9" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<9>; +template [[host_name("kernel_mul_mm_id_map0_ne20_10")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<10>; +template [[host_name("kernel_mul_mm_id_map0_ne20_16")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<16>; +template [[host_name("kernel_mul_mm_id_map0_ne20_22")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<22>; + +// TODO: refactor into two seperate functions to avoid ifdef branching: with and without tensor support +template +kernel void kernel_mul_mm_id( + constant ggml_metal_kargs_mul_mm_id & args, + device const char * src0, + device const char * src1, + device const char * htpe, + device const char * hids, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + threadgroup S0 * sa = (threadgroup S0 *)(shmem); + threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096); + +#ifdef GGML_METAL_HAS_TENSOR + threadgroup float * sc = (threadgroup float *)(shmem); +#endif + + constexpr int NR0 = 64; + constexpr int NR1 = 32; + + constexpr int NK = 32; + constexpr int NL0 = NK/16; + constexpr int NL1 = NK/8; + + const int tile = tgpig.x; // entry tile, described by the prepass + const int r0 = tgpig.y*NR0; + + device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe); + device const int32_t * ids_i32 = (device const int32_t *) (hids); + + // the prepass emits one (expert, first entry, valid rows) descriptor per tile that has + // work - the grid is sized by a static upper bound, so the tail early exits + if (tile >= (int) tpe_u32[0]) { + return; + } + + const int im = tpe_u32[1 + 3*tile]; // expert + const int idb = tpe_u32[1 + 3*tile + 1]; // first entry of this tile in hids + + // number of entries in this tile, always <= NR1 + const short nr1 = (short) ((tpe_u32[1 + 3*tile + 2] < NR1) ? tpe_u32[1 + 3*tile + 2] : NR1); + + // if this block is of 64x32 shape or smaller + const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0; + + // a thread shouldn't load data outside of the matrix + const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63 + + const short il0 = (tiitg % NL0); + + const short i13 = 0; + + const uint64_t offset0 = im*args.nb02 + i13*args.nb03; + const short offset1 = il0/nl; + + const short iy = 8*(tiitg % NL1); + + // skip the upper half of the token tile when the expert did not fill it + constexpr short NR1H = NR1/2; + +#ifndef GGML_METAL_HAS_TENSOR + S0_8x8 ma[4]; + S1_8x8 mb[2]; + + simdgroup_float8x8 mc[8]; +#else + auto tA = tensor, tensor_inline>(sa, dextents(NK, NR0)); + + // sb is [NR1][NK] row-major + auto tB0 = tensor, tensor_inline>(sb, dextents(NK, NR1H)); + auto tB1 = tensor, tensor_inline>(sb + NR1H*NK, dextents(NK, NR1H)); + + mpp::tensor_ops::matmul2d< + mpp::tensor_ops::matmul2d_descriptor(NR1H, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate), + execution_simdgroups<4>> mm; + +#endif + + const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31 + + short il = il0; + + const int id = ids_i32[idb + lr1]; + + const short i11 = (id % args.ne20) % args.ne11; + const int i12 = (id / args.ne20); + + device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1; + + device const T1 * y = (device const T1 *)(src1 + + args.nb13*i13 + + args.nb12*i12 + + args.nb11*i11 + + args.nb10*iy); + + const bool has_hi = nr1 > NR1H; + +#ifndef GGML_METAL_HAS_TENSOR + for (short i = 0; i < 8; i++){ + mc[i] = make_filled_simdgroup_matrix(0.f); + } + + // simdgroups 2,3 own rows NR1H..NR1-1 + const bool sg_active = has_hi || sgitg < 2; +#else + auto cT0 = mm.get_destination_cooperative_tensor(); + auto cT1 = mm.get_destination_cooperative_tensor(); +#endif + + for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) { +#ifndef GGML_METAL_HAS_TENSOR + // load data and store to threadgroup memory + if (is_same::value && FC_mul_mm_bc_inp) { + threadgroup_barrier(mem_flags::mem_threadgroup); + + // no need for dequantization + for (short i = 0; i < 16; i++) { + const short sx = 2*il0 + i/8; + const short sy = (tiitg/NL0)/8; + + //const short lx = i%8; + //const short ly = (tiitg/NL0)%8; + const short lx = (tiitg/NL0)%8; + const short ly = i%8; + + const short ib = 8*sx + sy; + + *(sa + 64*ib + 8*ly + lx) = loop_k + 16*il + i < args.ne00 ? (S0) *((device T0 *) x + i) : (S0) 0; + } + } else { + S0_4x4 temp_a; + dequantize_func(x, il, temp_a); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short i = 0; i < 16; i++) { + const short sx = 2*il0 + i/8; + const short sy = (tiitg/NL0)/8; + + //const short lx = i%8; + //const short ly = (tiitg/NL0)%8; + const short lx = (tiitg/NL0)%8; + const short ly = i%8; + + const short ib = 8*sx + sy; + + // NOTE: this is massively slower.. WTF? + //sa[64*ib + 8*ly + lx] = temp_a[i/4][i%4]; + + *(sa + 64*ib + 8*ly + lx) = temp_a[i/4][i%4]; + } + } + + if (FC_mul_mm_bc_inp) { + for (short i = 0; i < 8; ++i) { + const short sx = (tiitg%NL1); + const short sy = (tiitg/NL1)/8; + + const short lx = i; + const short ly = (tiitg/NL1)%8; + //const short lx = (tiitg/NL1)%8; + //const short ly = i; + + const short ib = 4*sx + sy; + + *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; + } + } else { + const short sx = (tiitg%NL1); + const short sy = (tiitg/NL1)/8; + + //const short dx = sx; + //const short dy = sy; + + const short ly = (tiitg/NL1)%8; + + const short ib = 4*sx + sy; + + *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y)); + } +#else + // load data and store to threadgroup memory + if (is_same::value && FC_mul_mm_bc_inp) { + threadgroup_barrier(mem_flags::mem_threadgroup); + + // no need for dequantization + for (short i = 0; i < 16; i++) { + const short sx = 2*il0 + i/8; + const short sy = (tiitg/NL0)/8; + + const short lx = i%8; + const short ly = (tiitg/NL0)%8; + //const short lx = (tiitg/NL0)%8; + //const short ly = i%8; + + *(sa + NK*(8*sy + ly) + 8*sx + lx) = loop_k + 16*il + i < args.ne00 ? (S0) *((device T0 *) x + i) : 0; + } + } else { + S0_4x4 temp_a; + dequantize_func(x, il, temp_a); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + FOR_UNROLL (short i = 0; i < 16; i++) { + const short sx = 2*il0 + i/8; + const short sy = (tiitg/NL0)/8; + + const short lx = i%8; + const short ly = (tiitg/NL0)%8; + //const short lx = (tiitg/NL0)%8; + //const short ly = i%8; + + *(sa + NK*(8*sy + ly) + 8*sx + lx) = temp_a[i/4][i%4]; + } + } + + if (FC_mul_mm_bc_inp) { + for (short i = 0; i < 8; ++i) { + const short sx = (tiitg%NL1); + const short sy = (tiitg/NL1)/8; + + const short lx = i; + const short ly = (tiitg/NL1)%8; + //const short lx = (tiitg/NL1)%8; + //const short ly = i; + + *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0; + } + } else { + const short sx = (tiitg%NL1); + const short sy = (tiitg/NL1)/8; + + //const short lx = i; + const short ly = (tiitg/NL1)%8; + //const short lx = (tiitg/NL1)%8; + //const short ly = i; + + *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)(*((device T1_2x4 *) y)); + } +#endif + + il = (il + 2 < nl) ? il + 2 : il % 2; + x = (il < 2) ? x + (2 + nl - 1)/nl : x; + + y += NK; + + threadgroup_barrier(mem_flags::mem_threadgroup); + +#ifndef GGML_METAL_HAS_TENSOR + if (sg_active) { + // load matrices from threadgroup memory and conduct outer products + threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2)); + threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2)); + + FOR_UNROLL (short ik = 0; ik < NK/8; ik++) { + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 4; i++) { + simdgroup_load(ma[i], lsma + 64*i, 8, 0, false); + } + + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 2; i++) { + simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false); + } + + simdgroup_barrier(mem_flags::mem_none); + + FOR_UNROLL (short i = 0; i < 8; i++){ + simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]); + } + + lsma += 8*64; + lsmb += 4*64; + } + } +#else + auto sA = tA.slice(0, 0); + auto sB0 = tB0.slice(0, 0); + + mm.run(sB0, sA, cT0); + + if (has_hi) { + auto sB1 = tB1.slice(0, 0); + + mm.run(sB1, sA, cT1); + } +#endif + } + + // block is smaller than 64x32, we should avoid writing data outside of the matrix + threadgroup_barrier(mem_flags::mem_threadgroup); + +#ifdef GGML_METAL_HAS_TENSOR + auto tC0 = tensor, tensor_inline>(sc, dextents(NR0, NR1H)); + cT0.store(tC0); + + if (has_hi) { + auto tC1 = tensor, tensor_inline>(sc + NR1H*NR0, dextents(NR0, NR1H)); + cT1.store(tC1); + } +#else + if (sg_active) { + threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0; + + for (short i = 0; i < 8; i++) { + simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false); + } + } +#endif + + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (short j = sgitg; j < nr1; j += 4) { + const int id = ids_i32[idb + j]; + + const short ide = id % args.ne20; + const int idt = id / args.ne20; + + device float * D = (device float *) dst + r0 + (uint64_t) ide*args.ne0 + (uint64_t) idt*args.ne1*args.ne0; + device float4 * D4 = (device float4 *) D; + + threadgroup float * C = (threadgroup float *) shmem + j*NR0; + threadgroup float4 * C4 = (threadgroup float4 *) C; + + int i = tiisg; + for (; i < nr0/4; i += 32) { + *(D4 + i) = *(C4 + i); + } + + i = (4*(nr0/4)) + tiisg; + for (; i < nr0; i += 32) { + *(D + i) = *(C + i); + } + } +} + +// +// matrix-matrix multiplication +// + +typedef decltype(kernel_mul_mm) mul_mm_t; + +template [[host_name("kernel_mul_mm_f32_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_f16_f32")]] kernel mul_mm_t kernel_mul_mm; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mm_bf16_f32")]] kernel mul_mm_t kernel_mul_mm; +#endif +template [[host_name("kernel_mul_mm_q1_0_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q2_0_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_1_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q5_0_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q5_1_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q8_0_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_mxfp4_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q2_K_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q3_K_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q5_K_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq2_xxs_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq2_xs_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq3_xxs_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq3_s_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq2_s_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq1_m_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_tq2_0_f32")]] kernel mul_mm_t kernel_mul_mm; + +template [[host_name("kernel_mul_mm_f32_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_f16_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q1_0_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q2_0_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_0_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_1_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q5_0_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q5_1_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q8_0_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_mxfp4_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q2_K_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q3_K_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q4_K_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q5_K_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_q6_K_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq2_xxs_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq2_xs_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq3_xxs_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq3_s_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq2_s_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq1_s_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq1_m_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq4_nl_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_iq4_xs_f16")]] kernel mul_mm_t kernel_mul_mm; +template [[host_name("kernel_mul_mm_tq2_0_f16")]] kernel mul_mm_t kernel_mul_mm; + +// +// indirect matrix-matrix multiplication +// + +typedef decltype(kernel_mul_mm_id) mul_mm_id; + +template [[host_name("kernel_mul_mm_id_f32_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_f16_f32")]] kernel mul_mm_id kernel_mul_mm_id; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mm_id_bf16_f32")]] kernel mul_mm_id kernel_mul_mm_id; +#endif +template [[host_name("kernel_mul_mm_id_q1_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q2_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q4_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q4_1_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q5_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q5_1_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q8_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mm_id_mxfp4_f32")]] kernel mul_mm_id kernel_mul_mm_id; +#else +template [[host_name("kernel_mul_mm_id_mxfp4_f32")]] kernel mul_mm_id kernel_mul_mm_id; +#endif +template [[host_name("kernel_mul_mm_id_q2_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q3_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q4_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q5_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq2_xxs_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq2_xs_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq3_xxs_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq3_s_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq2_s_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq1_m_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_tq2_0_f32")]] kernel mul_mm_id kernel_mul_mm_id; + +template [[host_name("kernel_mul_mm_id_f32_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_f16_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q1_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q2_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q4_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q4_1_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q5_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q5_1_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q8_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mm_id_mxfp4_f16")]] kernel mul_mm_id kernel_mul_mm_id; +#else +template [[host_name("kernel_mul_mm_id_mxfp4_f16")]] kernel mul_mm_id kernel_mul_mm_id; +#endif +template [[host_name("kernel_mul_mm_id_q2_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q3_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q4_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q5_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_q6_K_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq2_xxs_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq2_xs_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq3_xxs_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq3_s_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq2_s_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq1_s_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq1_m_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq4_nl_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_iq4_xs_f16")]] kernel mul_mm_id kernel_mul_mm_id; +template [[host_name("kernel_mul_mm_id_tq2_0_f16")]] kernel mul_mm_id kernel_mul_mm_id; diff --git a/ggml/src/ggml-metal/kernels/mul_mv.metal b/ggml/src/ggml-metal/kernels/mul_mv.metal new file mode 100644 index 0000000..7ad07a2 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/mul_mv.metal @@ -0,0 +1,3408 @@ +#include "common.h" +#include "dequantize.h" +// Q1_0 dot product: dot = d * (2 * Σ(yl[i] where bit=1) - sumy) +inline float block_q_n_dot_y(device const block_q1_0 * qb_curr, float sumy, thread float * yl, int il) { + device const uint8_t * qs = qb_curr->qs + il / 8; + const uint8_t b0 = qs[0]; + const uint8_t b1 = qs[1]; + + float acc = 0.0f; + + acc += select(0.0f, yl[ 0], bool(b0 & 0x01)); + acc += select(0.0f, yl[ 1], bool(b0 & 0x02)); + acc += select(0.0f, yl[ 2], bool(b0 & 0x04)); + acc += select(0.0f, yl[ 3], bool(b0 & 0x08)); + acc += select(0.0f, yl[ 4], bool(b0 & 0x10)); + acc += select(0.0f, yl[ 5], bool(b0 & 0x20)); + acc += select(0.0f, yl[ 6], bool(b0 & 0x40)); + acc += select(0.0f, yl[ 7], bool(b0 & 0x80)); + + acc += select(0.0f, yl[ 8], bool(b1 & 0x01)); + acc += select(0.0f, yl[ 9], bool(b1 & 0x02)); + acc += select(0.0f, yl[10], bool(b1 & 0x04)); + acc += select(0.0f, yl[11], bool(b1 & 0x08)); + acc += select(0.0f, yl[12], bool(b1 & 0x10)); + acc += select(0.0f, yl[13], bool(b1 & 0x20)); + acc += select(0.0f, yl[14], bool(b1 & 0x40)); + acc += select(0.0f, yl[15], bool(b1 & 0x80)); + + return qb_curr->d * (2.0f * acc - sumy); +} + +// Q2_0 dot: d * (sum_lo(y) + 2*sum_hi(y) - sumy) via per-bit conditional adds +inline float block_q_n_dot_y(device const block_q2_0 * qb_curr, float sumy, thread float * yl, int il) { + device const uint8_t * qs = qb_curr->qs + (il / 4); + const uint8_t b0 = qs[0]; + const uint8_t b1 = qs[1]; + const uint8_t b2 = qs[2]; + const uint8_t b3 = qs[3]; + + // Accumulate where low bit is set (bits 0,2,4,6 of each byte) + float acc_lo = 0.0f; + acc_lo += select(0.0f, yl[ 0], bool(b0 & 0x01)); + acc_lo += select(0.0f, yl[ 1], bool(b0 & 0x04)); + acc_lo += select(0.0f, yl[ 2], bool(b0 & 0x10)); + acc_lo += select(0.0f, yl[ 3], bool(b0 & 0x40)); + acc_lo += select(0.0f, yl[ 4], bool(b1 & 0x01)); + acc_lo += select(0.0f, yl[ 5], bool(b1 & 0x04)); + acc_lo += select(0.0f, yl[ 6], bool(b1 & 0x10)); + acc_lo += select(0.0f, yl[ 7], bool(b1 & 0x40)); + acc_lo += select(0.0f, yl[ 8], bool(b2 & 0x01)); + acc_lo += select(0.0f, yl[ 9], bool(b2 & 0x04)); + acc_lo += select(0.0f, yl[10], bool(b2 & 0x10)); + acc_lo += select(0.0f, yl[11], bool(b2 & 0x40)); + acc_lo += select(0.0f, yl[12], bool(b3 & 0x01)); + acc_lo += select(0.0f, yl[13], bool(b3 & 0x04)); + acc_lo += select(0.0f, yl[14], bool(b3 & 0x10)); + acc_lo += select(0.0f, yl[15], bool(b3 & 0x40)); + + // Accumulate where high bit is set (bits 1,3,5,7 of each byte) + float acc_hi = 0.0f; + acc_hi += select(0.0f, yl[ 0], bool(b0 & 0x02)); + acc_hi += select(0.0f, yl[ 1], bool(b0 & 0x08)); + acc_hi += select(0.0f, yl[ 2], bool(b0 & 0x20)); + acc_hi += select(0.0f, yl[ 3], bool(b0 & 0x80)); + acc_hi += select(0.0f, yl[ 4], bool(b1 & 0x02)); + acc_hi += select(0.0f, yl[ 5], bool(b1 & 0x08)); + acc_hi += select(0.0f, yl[ 6], bool(b1 & 0x20)); + acc_hi += select(0.0f, yl[ 7], bool(b1 & 0x80)); + acc_hi += select(0.0f, yl[ 8], bool(b2 & 0x02)); + acc_hi += select(0.0f, yl[ 9], bool(b2 & 0x08)); + acc_hi += select(0.0f, yl[10], bool(b2 & 0x20)); + acc_hi += select(0.0f, yl[11], bool(b2 & 0x80)); + acc_hi += select(0.0f, yl[12], bool(b3 & 0x02)); + acc_hi += select(0.0f, yl[13], bool(b3 & 0x08)); + acc_hi += select(0.0f, yl[14], bool(b3 & 0x20)); + acc_hi += select(0.0f, yl[15], bool(b3 & 0x80)); + + return qb_curr->d * (acc_lo + 2.0f * acc_hi - sumy); +} + +// function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i]) +// il indicates where the q4 quants begin (0 or QK4_0/4) +// we assume that the yl's have been multiplied with the appropriate scale factor +// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096) +inline float block_q_n_dot_y(device const block_q4_0 * qb_curr, float sumy, thread float * yl, int il) { + float d = qb_curr->d; + + float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; + + device const uint16_t * qs = ((device const uint16_t *) qb_curr + 1 + il/2); + + for (int i = 0; i < 8; i += 2) { + acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F); + acc[1] += yl[i + 1] * (qs[i / 2] & 0x0F00); + acc[2] += yl[i + 8] * (qs[i / 2] & 0x00F0); + acc[3] += yl[i + 9] * (qs[i / 2] & 0xF000); + } + + return d * (sumy * -8.f + acc[0] + acc[1] + acc[2] + acc[3]); +} + +// function for calculate inner product between half a q4_1 block and 16 floats (yl), sumy is SUM(yl[i]) +// il indicates where the q4 quants begin (0 or QK4_0/4) +// we assume that the yl's have been multiplied with the appropriate scale factor +// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096) +inline float block_q_n_dot_y(device const block_q4_1 * qb_curr, float sumy, thread float * yl, int il) { + float d = qb_curr->d; + float m = qb_curr->m; + + float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; + + device const uint16_t * qs = ((device const uint16_t *) qb_curr + 2 + il/2); + + for (int i = 0; i < 8; i+=2) { + acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F); + acc[1] += yl[i + 1] * (qs[i / 2] & 0x0F00); + acc[2] += yl[i + 8] * (qs[i / 2] & 0x00F0); + acc[3] += yl[i + 9] * (qs[i / 2] & 0xF000); + } + + return d * (acc[0] + acc[1] + acc[2] + acc[3]) + sumy * m; +} + +// function for calculate inner product between half a q5_0 block and 16 floats (yl), sumy is SUM(yl[i]) +// il indicates where the q5 quants begin (0 or QK5_0/4) +// we assume that the yl's have been multiplied with the appropriate scale factor +// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096) +inline float block_q_n_dot_y(device const block_q5_0 * qb_curr, float sumy, thread float * yl, int il) { + float d = qb_curr->d; + + float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; + + device const uint16_t * qs = ((device const uint16_t *)qb_curr + 3 + il/2); + const uint32_t qh = *((device const uint32_t *)qb_curr->qh); + + for (int i = 0; i < 8; i+=2) { + acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010)); + acc[1] += yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000)); + acc[2] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100)); + acc[3] += yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000)); + } + + return d * (sumy * -16.f + acc[0] + acc[1] + acc[2] + acc[3]); +} + +// function for calculate inner product between half a q5_1 block and 16 floats (yl), sumy is SUM(yl[i]) +// il indicates where the q5 quants begin (0 or QK5_1/4) +// we assume that the yl's have been multiplied with the appropriate scale factor +// that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096) +inline float block_q_n_dot_y(device const block_q5_1 * qb_curr, float sumy, thread float * yl, int il) { + float d = qb_curr->d; + float m = qb_curr->m; + + float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; + + device const uint16_t * qs = ((device const uint16_t *)qb_curr + 4 + il/2); + const uint32_t qh = *((device const uint32_t *)qb_curr->qh); + + for (int i = 0; i < 8; i+=2) { + acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010)); + acc[1] += yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000)); + acc[2] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100)); + acc[3] += yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000)); + } + + return d * (acc[0] + acc[1] + acc[2] + acc[3]) + sumy * m; +} + +template +static inline void helper_mv_reduce_and_write( + device float * dst_f32, + float sumf[NR0], + const int r0, + const int ne01, + ushort tiisg, + ushort sgitg, + threadgroup char * shmem) { + constexpr short NW = N_SIMDWIDTH; + + threadgroup float * shmem_f32[NR0]; + + for (short row = 0; row < NR0; ++row) { + shmem_f32[row] = (threadgroup float *) shmem + NW*row; + + if (sgitg == 0) { + shmem_f32[row][tiisg] = 0.0f; + } + + sumf[row] = simd_sum(sumf[row]); + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (short row = 0; row < NR0; ++row) { + if (tiisg == 0) { + shmem_f32[row][sgitg] = sumf[row]; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (short row = 0; row < NR0 && r0 + row < ne01; ++row) { + float tot = simd_sum(shmem_f32[row][tiisg]); + + if (tiisg == 0 && sgitg == 0) { + dst_f32[r0 + row] = tot; + } + } +} + +constant short FC_mul_mv_nsg [[function_constant(FC_MUL_MV + 0)]]; +constant short FC_mul_mv_nxpsg [[function_constant(FC_MUL_MV + 1)]]; +constant short FC_mul_mv_ne12 [[function_constant(FC_MUL_MV + 2)]]; +constant short FC_mul_mv_r2 [[function_constant(FC_MUL_MV + 3)]]; +constant short FC_mul_mv_r3 [[function_constant(FC_MUL_MV + 4)]]; +constant bool FC_mul_mv_split [[function_constant(FC_MUL_MV + 5)]]; + +template +void mul_vec_q_n_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + constexpr short NW = N_SIMDWIDTH; + constexpr short NQ = 16; + + const int nb = args.ne00/QK4_0; + + const int r0 = (tgpig.x*NSG + sgitg)*NR0; + //const int r0 = tgpig.x*NR0; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + //const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + //device const block_q_type * x = (device const block_q_type *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + // pointers to src0 rows + device const block_q_type * ax[NR0]; + FOR_UNROLL (int row = 0; row < NR0; ++row) { + const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + + ax[row] = (device const block_q_type *) ((device char *) src0 + offset0); + } + + float sumf[NR0] = {0.f}; + + const short ix = (tiisg/(NW/NQ)); + const short il = (tiisg%(NW/NQ))*8; + + //const int ib0 = sgitg*NQ + ix; + const int ib0 = ix; + + float yl[16]; // src1 vector cache + + //device const float * yb = y + ix*QK4_0 + il; + device const float * yb = y + ib0*QK4_0 + il; + + // each thread in a SIMD group deals with half a block. + //for (int ib = ib0; ib < nb; ib += NSG*NQ) { + for (int ib = ib0; ib < nb; ib += NQ) { + float sumy[2] = { 0.f, 0.f }; + + FOR_UNROLL (short i = 0; i < 8; i += 2) { + sumy[0] += yb[i + 0] + yb[i + 1]; + yl[i + 0] = yb[i + 0]; + yl[i + 1] = yb[i + 1]/256.f; + + sumy[1] += yb[i + 16] + yb[i + 17]; + yl[i + 8] = yb[i + 16]/16.f; + yl[i + 9] = yb[i + 17]/4096.f; + } + + FOR_UNROLL (short row = 0; row < NR0; row++) { + sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy[0] + sumy[1], yl, il); + } + + yb += QK4_0 * 16; + //yb += NSG*NQ*QK4_0; + } + + device float * dst_f32 = (device float *) dst + im*args.ne0*args.ne1 + r1*args.ne0; + + //helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem); + + for (int row = 0; row < NR0; ++row) { + const float tot = simd_sum(sumf[row]); + + if (tiisg == 0 && r0 + row < args.ne01) { + dst_f32[r0 + row] = tot; + } + } +} + +template +void kernel_mul_mv_q1_0_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK1_0; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset1 = r1*args.nb11 + (i12)*args.nb12 + (i13)*args.nb13; + + device const float * y = (device const float *) (src1 + offset1); + + device const block_q1_0 * ax[nr0]; + for (int row = 0; row < nr0; ++row) { + const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + ax[row] = (device const block_q1_0 *) ((device char *) src0 + offset0); + } + + float yl[16]; + float sumf[nr0] = {0.f}; + + const short ix = (tiisg/8); + const short il = (tiisg%8)*16; + + device const float * yb = y + ix*QK1_0 + il; + + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) { + float sumy = 0.f; + + FOR_UNROLL (short i = 0; i < 16; i++) { + yl[i] = yb[i]; + sumy += yb[i]; + } + + FOR_UNROLL (short row = 0; row < nr0; row++) { + sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy, yl, il); + } + + yb += QK1_0 * (N_SIMDWIDTH/8); + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0; ++row) { + const float tot = simd_sum(sumf[row]); + + if (tiisg == 0 && first_row + row < args.ne01) { + dst_f32[first_row + row] = tot; + } + } +} + +[[host_name("kernel_mul_mv_q1_0_f32")]] +kernel void kernel_mul_mv_q1_0_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_q1_0_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_q2_0_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK2_0; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset1 = r1*args.nb11 + (i12)*args.nb12 + (i13)*args.nb13; + + device const float * y = (device const float *) (src1 + offset1); + + device const block_q2_0 * ax[nr0]; + for (int row = 0; row < nr0; ++row) { + const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + ax[row] = (device const block_q2_0 *) ((device char *) src0 + offset0); + } + + float yl[16]; + float sumf[nr0] = {0.f}; + + // group 64: 4 sub-blocks of 16 weights per Q2_0 block + const short ix = (tiisg/4); + const short il = (tiisg%4)*16; + + device const float * yb = y + ix*QK2_0 + il; + + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) { + float sumy = 0.f; + + FOR_UNROLL (short i = 0; i < 16; i++) { + yl[i] = yb[i]; + sumy += yb[i]; + } + + FOR_UNROLL (short row = 0; row < nr0; row++) { + sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy, yl, il); + } + + yb += QK2_0 * (N_SIMDWIDTH/4); + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0; ++row) { + const float tot = simd_sum(sumf[row]); + + if (tiisg == 0 && first_row + row < args.ne01) { + dst_f32[first_row + row] = tot; + } + } +} + +[[host_name("kernel_mul_mv_q2_0_f32")]] +kernel void kernel_mul_mv_q2_0_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_q2_0_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +kernel void kernel_mul_mv_q4_0_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +kernel void kernel_mul_mv_q4_1_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +kernel void kernel_mul_mv_q5_0_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +kernel void kernel_mul_mv_q5_1_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + mul_vec_q_n_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_q8_0_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + constexpr short NW = N_SIMDWIDTH; + constexpr short NQ = 8; + + const int nb = args.ne00/QK8_0; + + const int r0 = tgpig.x*NR0; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + //const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + //device const block_q8_0 * x = (device const block_q8_0 *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + // pointers to src0 rows + device const block_q8_0 * ax[NR0]; + FOR_UNROLL (short row = 0; row < NR0; ++row) { + const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + + ax[row] = (device const block_q8_0 *) ((device char *) src0 + offset0); + } + + float sumf[NR0] = { 0.f }; + + const short ix = tiisg/(NW/NQ); + const short il = tiisg%(NW/NQ); + + const int ib0 = sgitg*NQ + ix; + + float yl[NQ]; + + device const float * yb = y + ib0*QK8_0 + il*NQ; + + // each thread in a SIMD group deals with NQ quants at a time + for (int ib = ib0; ib < nb; ib += NSG*NQ) { + for (short i = 0; i < NQ; ++i) { + yl[i] = yb[i]; + } + + for (short row = 0; row < NR0; row++) { + device const int8_t * qs = ax[row][ib].qs + il*NQ; + + float sumq = 0.f; + FOR_UNROLL (short i = 0; i < NQ; ++i) { + sumq += qs[i] * yl[i]; + } + + sumf[row] += sumq*ax[row][ib].d; + } + + yb += NSG*NQ*QK8_0; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem); +} + +[[host_name("kernel_mul_mv_q8_0_f32")]] +kernel void kernel_mul_mv_q8_0_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_q8_0_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +// mat-vec kernel processing in chunks of float4 +// chpb - chunks per quantization block +template +void kernel_mul_mv_ext_q4_f32_impl( + constant ggml_metal_kargs_mul_mv_ext & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + const short NSG = FC_mul_mv_nsg; + const short nxpsg = FC_mul_mv_nxpsg; + + const short chpt = 4; // chunks per thread + + //const short nxpsg = (32); + const short nypsg = (32/nxpsg); + + const short tx = tiisg%nxpsg; + const short ty = tiisg/nxpsg; + + const int i01 = tgpig.x*(nypsg*NSG) + nypsg*sgitg + ty; + const int i11 = tgpig.y*r1ptg; + const int i1m = tgpig.z; + + const int i12 = i1m%FC_mul_mv_ne12; + const int i13 = i1m/FC_mul_mv_ne12; + + const uint64_t offset0 = i01*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = i11*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const q_t * xq = (i01 < args.ne01) ? (device const q_t *) (src0 + offset0) + tx/chpb : (device const q_t *) src0; + + device const float4 * y4[r1ptg]; + + for (int ir1 = 0; ir1 < r1ptg; ++ir1) { + y4[ir1] = (i11 + ir1 < args.ne11) ? (device const float4 *) (src1 + offset1 + ir1*args.nb11) + tx : (device const float4 *) src1; + } + + float sumf[r1ptg] = { [ 0 ... r1ptg - 1 ] = 0.0f }; + + short cch = tx%chpb; // current chunk index + + for (int ich = tx; 4*ich < args.ne00; ich += chpt*nxpsg) { + float4 lx[chpt]; + +#pragma unroll(chpt) + for (short ch = 0; ch < chpt; ++ch) { + deq_t4(xq, cch, lx[ch]); + + cch += nxpsg; + if (cch >= chpb) { + xq += cch/chpb; + cch %= chpb; + } + } + +#pragma unroll(chpt) + for (short ch = 0; ch < chpt; ++ch) { +#pragma unroll(r1ptg) + for (short ir1 = 0; ir1 < r1ptg; ++ir1) { + sumf[ir1] += dot(lx[ch], y4[ir1][ch*nxpsg]); + } + } + +#pragma unroll(r1ptg) + for (short ir1 = 0; ir1 < r1ptg; ++ir1) { + y4[ir1] += chpt*nxpsg; + } + } + + // reduce only the threads in each row + for (short ir1 = 0; ir1 < r1ptg; ++ir1) { + if (nxpsg >= 32) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 16); + } + if (nxpsg >= 16) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 8); + } + if (nxpsg >= 8) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 4); + } + if (nxpsg >= 4) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 2); + } + if (nxpsg >= 2) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 1); + } + + //sumf[ir1] = simd_sum(sumf[ir1]); + } + + if (tx == 0) { + for (short ir1 = 0; ir1 < r1ptg && i11 + ir1 < args.ne11; ++ir1) { + device float * dst_f32 = (device float *) dst + (uint64_t)i1m*args.ne0*args.ne1 + (uint64_t)(i11 + ir1)*args.ne0; + + if (i01 < args.ne01) { + dst_f32[i01] = sumf[ir1]; + } + } + } +} + +// mat-vec kernel processing in chunks of float4x4 +template +void kernel_mul_mv_ext_q4x4_f32_impl( + constant ggml_metal_kargs_mul_mv_ext & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + const short NSG = FC_mul_mv_nsg; + const short nxpsg = FC_mul_mv_nxpsg; + + const short chpt = 1; + + //const short nxpsg = (32); + const short nypsg = (32/nxpsg); + + const short tx = tiisg%nxpsg; + const short ty = tiisg/nxpsg; + + const int i01 = tgpig.x*(nypsg*NSG) + nypsg*sgitg + ty; + const int i11 = tgpig.y*r1ptg; + const int i1m = tgpig.z; + + const int i12 = i1m%FC_mul_mv_ne12; + const int i13 = i1m/FC_mul_mv_ne12; + + const uint64_t offset0 = i01*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = i11*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const q_t * xq = (i01 < args.ne01) ? (device const q_t *) (src0 + offset0) + tx/chpb : (device const q_t *) src0; + + device const float4x4 * y4x4[r1ptg]; + + for (int ir1 = 0; ir1 < r1ptg; ++ir1) { + y4x4[ir1] = (i11 + ir1 < args.ne11) ? (device const float4x4 *) (src1 + offset1 + ir1*args.nb11) + tx : (device const float4x4 *) src1; + } + + float sumf[r1ptg] = { [ 0 ... r1ptg - 1 ] = 0.0f }; + + short cch = tx%chpb; + + for (int ich = tx; 16*ich < args.ne00; ich += chpt*nxpsg) { + float4x4 lx[chpt]; + +#pragma unroll(chpt) + for (short ch = 0; ch < chpt; ++ch) { + deq_t4x4(xq, cch, lx[ch]); + + cch += nxpsg; + if (cch >= chpb) { + xq += cch/chpb; + cch %= chpb; + } + } + +#pragma unroll(chpt) + for (short ch = 0; ch < chpt; ++ch) { +#pragma unroll(r1ptg) + for (short ir1 = 0; ir1 < r1ptg; ++ir1) { + sumf[ir1] += + dot(lx[ch][0], y4x4[ir1][ch*nxpsg][0]) + + dot(lx[ch][1], y4x4[ir1][ch*nxpsg][1]) + + dot(lx[ch][2], y4x4[ir1][ch*nxpsg][2]) + + dot(lx[ch][3], y4x4[ir1][ch*nxpsg][3]); + + } + } + +#pragma unroll(r1ptg) + for (short ir1 = 0; ir1 < r1ptg; ++ir1) { + y4x4[ir1] += chpt*nxpsg; + } + } + + for (short ir1 = 0; ir1 < r1ptg; ++ir1) { + if (nxpsg >= 32) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 16); + } + if (nxpsg >= 16) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 8); + } + if (nxpsg >= 8) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 4); + } + if (nxpsg >= 4) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 2); + } + if (nxpsg >= 2) { + sumf[ir1] += simd_shuffle_down(sumf[ir1], 1); + } + + //sumf[ir1] = simd_sum(sumf[ir1]); + } + + if (tx == 0) { + for (short ir1 = 0; ir1 < r1ptg && i11 + ir1 < args.ne11; ++ir1) { + device float * dst_f32 = (device float *) dst + (uint64_t)i1m*args.ne0*args.ne1 + (uint64_t)(i11 + ir1)*args.ne0; + + if (i01 < args.ne01) { + dst_f32[i01] = sumf[ir1]; + } + } + } +} + +// dispatchers needed for compile-time nxpsg +// epb - elements per quantization block +template +kernel void kernel_mul_mv_ext_q4_f32_disp( + constant ggml_metal_kargs_mul_mv_ext & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_ext_q4_f32_impl(args, src0, src1, dst, tgpig, tiisg, sgitg); +} + +template +kernel void kernel_mul_mv_ext_q4x4_f32_disp( + constant ggml_metal_kargs_mul_mv_ext & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_ext_q4x4_f32_impl(args, src0, src1, dst, tgpig, tiisg, sgitg); +} + +typedef decltype(kernel_mul_mv_ext_q4_f32_disp <2, block_q8_0, 32, dequantize_q8_0_t4>) mul_mv_ext_q4_f32_t; +typedef decltype(kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>) mul_mv_ext_q4x4_f32_t; + +template [[host_name("kernel_mul_mv_ext_f32_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, float4, 4, dequantize_f32_t4>; +template [[host_name("kernel_mul_mv_ext_f32_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, float4, 4, dequantize_f32_t4>; +template [[host_name("kernel_mul_mv_ext_f32_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, float4, 4, dequantize_f32_t4>; +template [[host_name("kernel_mul_mv_ext_f32_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, float4, 4, dequantize_f32_t4>; + +template [[host_name("kernel_mul_mv_ext_f16_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, half4, 4, dequantize_f16_t4>; +template [[host_name("kernel_mul_mv_ext_f16_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, half4, 4, dequantize_f16_t4>; +template [[host_name("kernel_mul_mv_ext_f16_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, half4, 4, dequantize_f16_t4>; +template [[host_name("kernel_mul_mv_ext_f16_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, half4, 4, dequantize_f16_t4>; + +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, bfloat4, 4, dequantize_bf16_t4>; +template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, bfloat4, 4, dequantize_bf16_t4>; +template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, bfloat4, 4, dequantize_bf16_t4>; +template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, bfloat4, 4, dequantize_bf16_t4>; +#endif + +template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q1_0, 128, dequantize_q1_0_t4>; +template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q1_0, 128, dequantize_q1_0_t4>; +template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q1_0, 128, dequantize_q1_0_t4>; +template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q1_0, 128, dequantize_q1_0_t4>; + +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q2_0, 64, dequantize_q2_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q2_0, 64, dequantize_q2_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q2_0, 64, dequantize_q2_0_t4>; +template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q2_0, 64, dequantize_q2_0_t4>; + +template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_0, 32, dequantize_q4_0_t4>; +template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_0, 32, dequantize_q4_0_t4>; +template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_0, 32, dequantize_q4_0_t4>; +template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q4_0, 32, dequantize_q4_0_t4>; + +template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_1, 32, dequantize_q4_1_t4>; +template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_1, 32, dequantize_q4_1_t4>; +template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_1, 32, dequantize_q4_1_t4>; +template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q4_1, 32, dequantize_q4_1_t4>; + +template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_0, 32, dequantize_q5_0_t4>; +template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_0, 32, dequantize_q5_0_t4>; +template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_0, 32, dequantize_q5_0_t4>; +template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q5_0, 32, dequantize_q5_0_t4>; + +template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_1, 32, dequantize_q5_1_t4>; +template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_1, 32, dequantize_q5_1_t4>; +template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_1, 32, dequantize_q5_1_t4>; +template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q5_1, 32, dequantize_q5_1_t4>; + +template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q8_0, 32, dequantize_q8_0_t4>; +template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q8_0, 32, dequantize_q8_0_t4>; +template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q8_0, 32, dequantize_q8_0_t4>; +template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q8_0, 32, dequantize_q8_0_t4>; + +template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_mxfp4, 32, dequantize_mxfp4_t4>; +template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_mxfp4, 32, dequantize_mxfp4_t4>; +template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_mxfp4, 32, dequantize_mxfp4_t4>; +template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_mxfp4, 32, dequantize_mxfp4_t4>; + +template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_iq4_nl, 32, dequantize_iq4_nl_t4>; +template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_iq4_nl, 32, dequantize_iq4_nl_t4>; +template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_iq4_nl, 32, dequantize_iq4_nl_t4>; +template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_iq4_nl, 32, dequantize_iq4_nl_t4>; + +template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>; +template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q4_K, 256, dequantize_q4_K>; +template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q4_K, 256, dequantize_q4_K>; +template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q4_K, 256, dequantize_q4_K>; + +template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q5_K, 256, dequantize_q5_K>; +template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q5_K, 256, dequantize_q5_K>; +template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q5_K, 256, dequantize_q5_K>; +template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q5_K, 256, dequantize_q5_K>; + +template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q6_K, 256, dequantize_q6_K>; +template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q6_K, 256, dequantize_q6_K>; +template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q6_K, 256, dequantize_q6_K>; +template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q6_K, 256, dequantize_q6_K>; + +template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q2_K, 256, dequantize_q2_K>; +template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q2_K, 256, dequantize_q2_K>; +template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q2_K, 256, dequantize_q2_K>; +template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q2_K, 256, dequantize_q2_K>; + +template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q3_K, 256, dequantize_q3_K>; +template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q3_K, 256, dequantize_q3_K>; +template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q3_K, 256, dequantize_q3_K>; +template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q3_K, 256, dequantize_q3_K>; + +template +void kernel_mul_mv_t_t_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + constexpr short NW = N_SIMDWIDTH; + constexpr short NB = 32; + constexpr short NF = 8; + + const int nb = args.ne00/NB; + + const int r0 = tgpig.x*NR0; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + //const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + //device const T0 * x = (device const T0 *) (src0 + offset0); + device const T1 * y = (device const T1 *) (src1 + offset1); + + // pointers to src0 rows + device const T0 * ax [NR0]; + FOR_UNROLL (short row = 0; row < NR0; ++row) { + const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + + ax[row] = (device const T0 *) ((device char *) src0 + offset0); + } + + float sumf[NR0] = { 0.f }; + + const short ix = tiisg/(NW/NF); + const short il = tiisg%(NW/NF); + + const int ib0 = sgitg*NF + ix; + + T1 yl[NF]; + + device const T1 * yb = y + (ib0*NB + il*NF); + + for (int ib = ib0; ib < nb; ib += NSG*NF) { + for (short i = 0; i < NF; ++i) { + yl[i] = yb[i]; + } + + for (short row = 0; row < NR0; row++) { + device const T0 * xb = ax[row] + (ib*NB + il*NF); + + float sumq = 0.f; + FOR_UNROLL (short i = 0; i < NF; ++i) { + sumq += xb[i] * yl[i]; + } + + sumf[row] += sumq; + } + + yb += NSG*NF*NW; + } + + for (int i = nb*NB + sgitg*NW + tiisg; i < args.ne00; i += NW*NSG) { + for (short row = 0; row < NR0; row++) { + sumf[row] += ax[row][i] * y[i]; + } + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem); +} + +template +void kernel_mul_mv_t_t_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + switch (args.nr0) { + //case 1: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; + case 2: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; + //case 3: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; + //case 4: kernel_mul_mv_t_t_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; + } +} + +template +kernel void kernel_mul_mv_t_t( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_t_t_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +typedef decltype(kernel_mul_mv_t_t) mul_mv_t_t; + +template [[host_name("kernel_mul_mv_f32_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t; +template [[host_name("kernel_mul_mv_f16_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t; +template [[host_name("kernel_mul_mv_f16_f16")]] kernel mul_mv_t_t kernel_mul_mv_t_t; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mv_bf16_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t; +template [[host_name("kernel_mul_mv_bf16_bf16")]] kernel mul_mv_t_t kernel_mul_mv_t_t; +template [[host_name("kernel_mul_mv_f32_bf16")]] kernel mul_mv_t_t kernel_mul_mv_t_t; +#endif + +template +void kernel_mul_mv_t_t_4_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + constexpr short NW = N_SIMDWIDTH; + constexpr short NB = 32; + constexpr short NF = 16; + constexpr short NF4 = NF/4; + + const int nb = args.ne00/NB; + + const int r0 = tgpig.x*NR0; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + //const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const T1 * y = (device const T1 *) (src1 + offset1); + device const T14 * y4 = (device const T14 *) (src1 + offset1); + + // pointers to src0 rows + device const T0 * ax [NR0]; + device const T04 * ax4[NR0]; + FOR_UNROLL (short row = 0; row < NR0; ++row) { + const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + + ax [row] = (device const T0 *) ((device char *) src0 + offset0); + ax4[row] = (device const T04 *) ((device char *) src0 + offset0); + } + + float sumf[NR0] = { 0.f }; + + const short ix = tiisg/(NW/NF); + const short il = tiisg%(NW/NF); + + const int ib0 = sgitg*NF + ix; + + T14 yl4[NF4]; + + device const T14 * yb4 = y4 + (ib0*NB + il*NF)/4; + + for (int ib = ib0; ib < nb; ib += NSG*NF) { + for (short i = 0; i < NF4; ++i) { + yl4[i] = yb4[i]; + } + + for (short row = 0; row < NR0; row++) { + device const T04 * xb4 = ax4[row] + (ib*NB + il*NF)/4; + + float sumq = 0.f; + FOR_UNROLL (short i = 0; i < NF4; ++i) { + sumq += dot(float4(xb4[i]), float4(yl4[i])); + } + + sumf[row] += sumq; + } + + yb4 += NSG*NF*NW/4; + } + + for (int i = nb*NB + sgitg*NW + tiisg; i < args.ne00; i += NW*NSG) { + for (short row = 0; row < NR0; row++) { + sumf[row] += ax[row][i] * y[i]; + } + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + helper_mv_reduce_and_write(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem); +} + +template +void kernel_mul_mv_t_t_4_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + switch (args.nr0) { + //case 1: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; + case 2: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; + //case 3: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; + //case 4: kernel_mul_mv_t_t_4_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break; + }; +} + +template +kernel void kernel_mul_mv_t_t_4( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_t_t_4_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +typedef decltype(kernel_mul_mv_t_t_4) mul_mv_t_t_4; + +template [[host_name("kernel_mul_mv_f32_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; +template [[host_name("kernel_mul_mv_f16_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; +template [[host_name("kernel_mul_mv_f16_f16_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mv_bf16_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; +template [[host_name("kernel_mul_mv_bf16_bf16_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; +template [[host_name("kernel_mul_mv_f32_bf16_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4; +#endif + +template +void kernel_mul_mv_t_t_short_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig, + ushort tiisg) { + const int r0 = tgpig.x*32 + tiisg; + const int r1 = tgpig.y; + const int im = tgpig.z; + + if (r0 >= args.ne01) { + return; + } + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + + device const T0 * x = (device const T0 *) (src0 + offset0); + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1; + + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const T1 * y = (device const T1 *) (src1 + offset1); + + float res = 0.0f; + + for (int i = 0; i < args.ne00; ++i) { + res += (float) x[i] * (float) y[i]; + } + + dst_f32[(uint64_t)r1*args.ne0 + r0] = res; +} + +template +kernel void kernel_mul_mv_t_t_short( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]]) { + kernel_mul_mv_t_t_short_impl( + args, + src0, + src1, + dst, + tgpig, + tiisg); +} + +typedef decltype(kernel_mul_mv_t_t_short) mul_mv_t_t_short_t; + +template [[host_name("kernel_mul_mv_f32_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; +template [[host_name("kernel_mul_mv_f16_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; +template [[host_name("kernel_mul_mv_f16_f16_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mv_bf16_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; +template [[host_name("kernel_mul_mv_bf16_bf16_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; +template [[host_name("kernel_mul_mv_f32_bf16_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short; +#endif + +template +void kernel_mul_mv_q2_K_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_q2_K * x = (device const block_q2_K *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + float yl[32]; + float sumf[nr0]={0.f}; + + const short ix = tiisg/8; // 0...3 + const short it = tiisg%8; // 0...7 + const short iq = it/4; // 0 or 1 + const short ir = it%4; // 0...3 + const short is = (8*ir)/16;// 0 or 1 + + device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir; + + for (int ib = ix; ib < nb; ib += 4) { + float4 sumy = {0.f, 0.f, 0.f, 0.f}; + for (short i = 0; i < 8; ++i) { + yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; + yl[i+ 8] = y4[i+32]; sumy[1] += yl[i+ 8]; + yl[i+16] = y4[i+64]; sumy[2] += yl[i+16]; + yl[i+24] = y4[i+96]; sumy[3] += yl[i+24]; + } + + device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is; + device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir; + device const half * dh = &x[ib].d; + + for (short row = 0; row < nr0; row++) { + float4 acc1 = {0.f, 0.f, 0.f, 0.f}; + float4 acc2 = {0.f, 0.f, 0.f, 0.f}; + for (int i = 0; i < 8; i += 2) { + acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003); + acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300); + acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c); + acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00); + acc1[2] += yl[i+16] * (qs[i/2] & 0x0030); + acc2[2] += yl[i+17] * (qs[i/2] & 0x3000); + acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0); + acc2[3] += yl[i+25] * (qs[i/2] & 0xc000); + } + float dall = dh[0]; + float dmin = dh[1] * 1.f/16.f; + sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f + + (acc1[1] + 1.f/256.f * acc2[1]) * (sc[2] & 0xF) * 1.f/ 4.f + + (acc1[2] + 1.f/256.f * acc2[2]) * (sc[4] & 0xF) * 1.f/16.f + + (acc1[3] + 1.f/256.f * acc2[3]) * (sc[6] & 0xF) * 1.f/64.f) - + dmin * (sumy[0] * (sc[0] & 0xF0) + sumy[1] * (sc[2] & 0xF0) + sumy[2] * (sc[4] & 0xF0) + sumy[3] * (sc[6] & 0xF0)); + + qs += args.nb01/2; + sc += args.nb01; + dh += args.nb01/2; + } + + y4 += 4 * QK_K; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; + } + } +} + +[[host_name("kernel_mul_mv_q2_K_f32")]] +kernel void kernel_mul_mv_q2_K_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_q2_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_q3_K_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_q3_K * x = (device const block_q3_K *) (src0 + offset0); + device const float * yy = (device const float *) (src1 + offset1); + + float yl[32]; + + //const uint16_t kmask1 = 0x3030; + //const uint16_t kmask2 = 0x0f0f; + + const short tid = tiisg/4; + const short ix = tiisg%4; + const short ip = tid/4; // 0 or 1 + const short il = 2*((tid%4)/2); // 0 or 2 + const short ir = tid%2; + const short l0 = 8*ir; + + // One would think that the Metal compiler would figure out that ip and il can only have + // 4 possible states, and optimize accordingly. Well, no. It needs help, and we do it + // with these two tales. + // + // Possible masks for the high bit + const ushort4 mm[4] = {{0x0001, 0x0100, 0x0002, 0x0200}, // ip = 0, il = 0 + {0x0004, 0x0400, 0x0008, 0x0800}, // ip = 0, il = 2 + {0x0010, 0x1000, 0x0020, 0x2000}, // ip = 1, il = 0 + {0x0040, 0x4000, 0x0080, 0x8000}}; // ip = 1, il = 2 + + // Possible masks for the low 2 bits + const int4 qm[2] = {{0x0003, 0x0300, 0x000c, 0x0c00}, {0x0030, 0x3000, 0x00c0, 0xc000}}; + + const ushort4 hm = mm[2*ip + il/2]; + + const short shift = 2*il; + + const float v1 = il == 0 ? 4.f : 64.f; + const float v2 = 4.f * v1; + + const uint16_t s_shift1 = 4*ip; + const uint16_t s_shift2 = s_shift1 + il; + + const short q_offset = 32*ip + l0; + const short y_offset = 128*ip + 32*il + l0; + + device const float * y1 = yy + ix*QK_K + y_offset; + + uint32_t scales32, aux32; + thread uint16_t * scales16 = (thread uint16_t *)&scales32; + thread const int8_t * scales = (thread const int8_t *)&scales32; + + float sumf1[nr0] = {0.f}; + float sumf2[nr0] = {0.f}; + + for (int i = ix; i < nb; i += 4) { + for (short l = 0; l < 8; ++l) { + yl[l+ 0] = y1[l+ 0]; + yl[l+ 8] = y1[l+16]; + yl[l+16] = y1[l+32]; + yl[l+24] = y1[l+48]; + } + + device const uint16_t * q = (device const uint16_t *)(x[i].qs + q_offset); + device const uint16_t * h = (device const uint16_t *)(x[i].hmask + l0); + device const uint16_t * a = (device const uint16_t *)(x[i].scales); + device const half * dh = &x[i].d; + + for (short row = 0; row < nr0; ++row) { + const float d_all = (float)dh[0]; + + scales16[0] = a[4]; + scales16[1] = a[5]; + aux32 = ((scales32 >> s_shift2) << 4) & 0x30303030; + scales16[0] = a[il+0]; + scales16[1] = a[il+1]; + scales32 = ((scales32 >> s_shift1) & 0x0f0f0f0f) | aux32; + + float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; + for (short l = 0; l < 8; l += 2) { + const int32_t qs = q[l/2]; + s1 += yl[l+0] * (qs & qm[il/2][0]); + s2 += yl[l+1] * (qs & qm[il/2][1]); + s3 += ((h[l/2] & hm[0]) ? 0.f : yl[l+0]) + ((h[l/2] & hm[1]) ? 0.f : yl[l+1]); + s4 += yl[l+16] * (qs & qm[il/2][2]); + s5 += yl[l+17] * (qs & qm[il/2][3]); + s6 += ((h[l/2] & hm[2]) ? 0.f : yl[l+16]) + ((h[l/2] & hm[3]) ? 0.f : yl[l+17]); + } + float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (scales[0] - 32); + sumf2[row] += d2 * (scales[2] - 32); + + s1 = s2 = s3 = s4 = s5 = s6 = 0; + for (short l = 0; l < 8; l += 2) { + const int32_t qs = q[l/2+8]; + s1 += yl[l+8] * (qs & qm[il/2][0]); + s2 += yl[l+9] * (qs & qm[il/2][1]); + s3 += ((h[l/2+8] & hm[0]) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm[1]) ? 0.f : yl[l+9]); + s4 += yl[l+24] * (qs & qm[il/2][2]); + s5 += yl[l+25] * (qs & qm[il/2][3]); + s6 += ((h[l/2+8] & hm[2]) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm[3]) ? 0.f : yl[l+25]); + } + d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (scales[1] - 32); + sumf2[row] += d2 * (scales[3] - 32); + + q += args.nb01/2; + h += args.nb01/2; + a += args.nb01/2; + dh += args.nb01/2; + } + + y1 += 4 * QK_K; + } + + for (int row = 0; row < nr0; ++row) { + const float sumf = (sumf1[row] + 0.25f * sumf2[row]) / (1 << shift); + sumf1[row] = simd_sum(sumf); + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + if (tiisg == 0) { + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + dst_f32[first_row + row] = sumf1[row]; + } + } +} + +[[host_name("kernel_mul_mv_q3_K_f32")]] +kernel void kernel_mul_mv_q3_K_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_q3_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_q4_K_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + constexpr uint16_t kmask1 = 0x3f3f; + constexpr uint16_t kmask2 = 0x0f0f; + constexpr uint16_t kmask3 = 0xc0c0; + + const short ix = tiisg/8; // 0...3 + const short it = tiisg%8; // 0...7 + const short iq = it/4; // 0 or 1 + const short ir = it%4; // 0...3 + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_q4_K * x = (device const block_q4_K *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + float yl[16]; + float yh[16]; + + float sumf[nr0]={0.f}; + + device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir; + + uint16_t sc16[4]; + thread const uint8_t * sc8 = (thread const uint8_t *)sc16; + + for (int ib = ix; ib < nb; ib += 4) { + float4 sumy = {0.f, 0.f, 0.f, 0.f}; + + for (short i = 0; i < 8; ++i) { + yl[i+0] = y4[i+ 0]; sumy[0] += yl[i+0]; + yl[i+8] = y4[i+ 32]; sumy[1] += yl[i+8]; + yh[i+0] = y4[i+128]; sumy[2] += yh[i+0]; + yh[i+8] = y4[i+160]; sumy[3] += yh[i+8]; + } + + device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq; + device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir; + device const half * dh = &x[ib].d; + + for (short row = 0; row < nr0; row++) { + sc16[0] = sc[0] & kmask1; + sc16[1] = sc[2] & kmask1; + sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2); + sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2); + + device const uint16_t * q2 = q1 + 32; + + float4 acc1 = {0.f, 0.f, 0.f, 0.f}; + float4 acc2 = {0.f, 0.f, 0.f, 0.f}; + + FOR_UNROLL (short i = 0; i < 4; ++i) { + acc1[0] += yl[2*i + 0] * (q1[i] & 0x000F); + acc1[1] += yl[2*i + 1] * (q1[i] & 0x0F00); + acc1[2] += yl[2*i + 8] * (q1[i] & 0x00F0); + acc1[3] += yl[2*i + 9] * (q1[i] & 0xF000); + acc2[0] += yh[2*i + 0] * (q2[i] & 0x000F); + acc2[1] += yh[2*i + 1] * (q2[i] & 0x0F00); + acc2[2] += yh[2*i + 8] * (q2[i] & 0x00F0); + acc2[3] += yh[2*i + 9] * (q2[i] & 0xF000); + } + + sumf[row] += dh[0] * ((acc1[0] + 1.f/256.f * acc1[1]) * sc8[0] + + (acc1[2] + 1.f/256.f * acc1[3]) * sc8[1] * 1.f/16.f + + (acc2[0] + 1.f/256.f * acc2[1]) * sc8[4] + + (acc2[2] + 1.f/256.f * acc2[3]) * sc8[5] * 1.f/16.f) - + dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]); + + q1 += args.nb01/2; + sc += args.nb01/2; + dh += args.nb01/2; + } + + y4 += 4 * QK_K; + } + + device float * dst_f32 = (device float *) dst + (int64_t)im*args.ne0*args.ne1 + (int64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; + } + } +} + +[[host_name("kernel_mul_mv_q4_K_f32")]] +kernel void kernel_mul_mv_q4_K_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_q4_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_q5_K_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_q5_K * x = (device const block_q5_K *) (src0 + offset0); + device const float * yy = (device const float *) (src1 + offset1); + + float sumf[nr0]={0.f}; + + float yl[16], yh[16]; + + constexpr uint16_t kmask1 = 0x3f3f; + constexpr uint16_t kmask2 = 0x0f0f; + constexpr uint16_t kmask3 = 0xc0c0; + + const short tid = tiisg/4; + const short ix = tiisg%4; + const short iq = tid/4; + const short ir = tid%4; + + const short l0 = 8*ir; + const short q_offset = 32*iq + l0; + const short y_offset = 64*iq + l0; + + const uint8_t hm1 = 1u << (2*iq); + const uint8_t hm2 = hm1 << 1; + const uint8_t hm3 = hm1 << 4; + const uint8_t hm4 = hm2 << 4; + + uint16_t sc16[4]; + thread const uint8_t * sc8 = (thread const uint8_t *)sc16; + + device const float * y1 = yy + ix*QK_K + y_offset; + + for (int i = ix; i < nb; i += 4) { + device const uint8_t * q1 = x[i].qs + q_offset; + device const uint8_t * qh = x[i].qh + l0; + device const half * dh = &x[i].d; + device const uint16_t * a = (device const uint16_t *)x[i].scales + iq; + + device const float * y2 = y1 + 128; + float4 sumy = {0.f, 0.f, 0.f, 0.f}; + for (short l = 0; l < 8; ++l) { + yl[l+0] = y1[l+ 0]; sumy[0] += yl[l+0]; + yl[l+8] = y1[l+32]; sumy[1] += yl[l+8]; + yh[l+0] = y2[l+ 0]; sumy[2] += yh[l+0]; + yh[l+8] = y2[l+32]; sumy[3] += yh[l+8]; + } + + for (short row = 0; row < nr0; ++row) { + device const uint8_t * q2 = q1 + 64; + + sc16[0] = a[0] & kmask1; + sc16[1] = a[2] & kmask1; + sc16[2] = ((a[4] >> 0) & kmask2) | ((a[0] & kmask3) >> 2); + sc16[3] = ((a[4] >> 4) & kmask2) | ((a[2] & kmask3) >> 2); + + float4 acc1 = {0.f}; + float4 acc2 = {0.f}; + FOR_UNROLL (short l = 0; l < 8; ++l) { + uint8_t h = qh[l]; + acc1[0] += yl[l+0] * (q1[l] & 0x0F); + acc1[1] += yl[l+8] * (q1[l] & 0xF0); + acc1[2] += yh[l+0] * (q2[l] & 0x0F); + acc1[3] += yh[l+8] * (q2[l] & 0xF0); + acc2[0] += h & hm1 ? yl[l+0] : 0.f; + acc2[1] += h & hm2 ? yl[l+8] : 0.f; + acc2[2] += h & hm3 ? yh[l+0] : 0.f; + acc2[3] += h & hm4 ? yh[l+8] : 0.f; + } + + sumf[row] += dh[0] * (sc8[0] * (acc1[0] + 16.f*acc2[0]) + + sc8[1] * (acc1[1]/16.f + 16.f*acc2[1]) + + sc8[4] * (acc1[2] + 16.f*acc2[2]) + + sc8[5] * (acc1[3]/16.f + 16.f*acc2[3])) - + dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]); + + q1 += args.nb01; + qh += args.nb01; + dh += args.nb01/2; + a += args.nb01/2; + } + + y1 += 4 * QK_K; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + const float tot = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = tot; + } + } +} + +[[host_name("kernel_mul_mv_q5_K_f32")]] +kernel void kernel_mul_mv_q5_K_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_q5_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_q6_K_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + constexpr uint8_t kmask1 = 0x03; + constexpr uint8_t kmask2 = 0x0C; + constexpr uint8_t kmask3 = 0x30; + constexpr uint8_t kmask4 = 0xC0; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_q6_K * x = (device const block_q6_K *) (src0 + offset0); + device const float * yy = (device const float *) (src1 + offset1); + + float sumf[nr0] = { 0.f }; + + float yl[16]; + + const short tid = tiisg/2; + const short ix = tiisg%2; + const short ip = tid/8; // 0 or 1 + const short il = tid%8; + const short l0 = 4*il; + const short is = 8*ip + l0/16; + + const short y_offset = 128*ip + l0; + const short q_offset_l = 64*ip + l0; + const short q_offset_h = 32*ip + l0; + + for (int i = ix; i < nb; i += 2) { + device const uint8_t * q1 = x[i].ql + q_offset_l; + device const uint8_t * q2 = q1 + 32; + device const uint8_t * qh = x[i].qh + q_offset_h; + device const int8_t * sc = x[i].scales + is; + device const half * dh = &x[i].d; + + device const float * y = yy + i * QK_K + y_offset; + + for (short l = 0; l < 4; ++l) { + yl[4*l + 0] = y[l + 0]; + yl[4*l + 1] = y[l + 32]; + yl[4*l + 2] = y[l + 64]; + yl[4*l + 3] = y[l + 96]; + } + + for (short row = 0; row < nr0; ++row) { + float4 sums = {0.f, 0.f, 0.f, 0.f}; + + FOR_UNROLL (short l = 0; l < 4; ++l) { + sums[0] += yl[4*l + 0] * ((int8_t)((q1[l] & 0xF) | ((qh[l] & kmask1) << 4)) - 32); + sums[1] += yl[4*l + 1] * ((int8_t)((q2[l] & 0xF) | ((qh[l] & kmask2) << 2)) - 32); + sums[2] += yl[4*l + 2] * ((int8_t)((q1[l] >> 4) | ((qh[l] & kmask3) << 0)) - 32); + sums[3] += yl[4*l + 3] * ((int8_t)((q2[l] >> 4) | ((qh[l] & kmask4) >> 2)) - 32); + } + + sumf[row] += dh[0] * (sums[0] * sc[0] + sums[1] * sc[2] + sums[2] * sc[4] + sums[3] * sc[6]); + + q1 += args.nb01; + q2 += args.nb01; + qh += args.nb01; + sc += args.nb01; + dh += args.nb01/2; + } + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; + } + } +} + +[[host_name("kernel_mul_mv_q6_K_f32")]] +kernel void kernel_mul_mv_q6_K_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_q6_K_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +// ======================= "True" 2-bit + +template +void kernel_mul_mv_iq2_xxs_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + float yl[32]; + float sumf[nr0]={0.f}; + + threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); + threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); + { + int nval = 4; + int pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xxs_grid[pos + i]; + nval = 2; + pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + device const float * y4 = y + 32 * ix; + + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { + for (short i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq2_xxs * xr = x + ibl; + device const uint16_t * q2 = xr->qs + 4 * ib; + device const half * dh = &xr->d; + + for (short row = row0; row < row1; row++) { + const float db = dh[0]; + device const uint8_t * aux8 = (device const uint8_t *)q2; + const uint32_t aux32 = q2[2] | (q2[3] << 16); + const float d = db * (0.5f + (aux32 >> 28)); + + float sum = 0; + for (short l = 0; l < 4; ++l) { + const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + aux8[l]); + const uint8_t signs = ssigns[(aux32 >> 7*l) & 127]; + for (short j = 0; j < 8; ++j) { + sum += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + } + sumf[row] += d * sum; + + dh += args.nb01/2; + q2 += args.nb01/2; + } + + y4 += 32 * ntx; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all * 0.25f; + } + } +} + +template +void kernel_mul_mv_iq2_xxs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + +[[host_name("kernel_mul_mv_iq2_xxs_f32")]] +kernel void kernel_mul_mv_iq2_xxs_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + kernel_mul_mv_iq2_xxs_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_iq2_xs_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + float yl[32]; + float sumf[nr0]={0.f}; + + threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem); + threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512); + { + int nval = 8; + int pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xs_grid[pos + i]; + nval = 2; + pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + device const float * y4 = y + 32 * ix; + + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { + for (short i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq2_xs * xr = x + ibl; + device const uint16_t * q2 = xr->qs + 4 * ib; + device const uint8_t * sc = xr->scales + ib; + device const half * dh = &xr->d; + + for (short row = row0; row < row1; row++) { + const float db = dh[0]; + const uint8_t ls1 = sc[0] & 0xf; + const uint8_t ls2 = sc[0] >> 4; + const float d1 = db * (0.5f + ls1); + const float d2 = db * (0.5f + ls2); + + float sum1 = 0, sum2 = 0; + for (short l = 0; l < 2; ++l) { + const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511)); + const uint8_t signs = ssigns[(q2[l] >> 9)]; + for (short j = 0; j < 8; ++j) { + sum1 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + } + for (short l = 2; l < 4; ++l) { + const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511)); + const uint8_t signs = ssigns[(q2[l] >> 9)]; + for (short j = 0; j < 8; ++j) { + sum2 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + } + sumf[row] += d1 * sum1 + d2 * sum2; + + dh += args.nb01/2; + q2 += args.nb01/2; + sc += args.nb01; + } + + y4 += 32 * ntx; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all * 0.25f; + } + } +} + +template +void kernel_mul_mv_iq2_xs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + +[[host_name("kernel_mul_mv_iq2_xs_f32")]] +kernel void kernel_mul_mv_iq2_xs_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq2_xs_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +// FC_mul_mv_split: for nb32 < 32 (nb32 divides 32), 32/nb32 threads share each chunk and each takes a slice of the rows +template +void kernel_mul_mv_iq3_xxs_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_iq3_xxs * x = (device const block_iq3_xxs *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + float yl[32]; + float sumf[nr0]={0.f}; + + threadgroup uint32_t * svalues = (threadgroup uint32_t *)(shmem); + threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256); + { + int nval = 4; + int pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) svalues[pos + i] = iq3xxs_grid[pos + i]; + nval = 2; + pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + device const float * y4 = y + 32 * ix; + + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { + for (short i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq3_xxs * xr = x + ibl; + device const uint8_t * q3 = xr->qs + 8 * ib; + device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib; + device const half * dh = &xr->d; + + for (short row = row0; row < row1; row++) { + const float db = dh[0]; + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float d = db * (0.5f + (aux32 >> 28)); + + float2 sum = {0}; + for (short l = 0; l < 4; ++l) { + const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(svalues + q3[2*l+0]); + const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(svalues + q3[2*l+1]); + const uint8_t signs = ssigns[(aux32 >> 7*l) & 127]; + for (short j = 0; j < 4; ++j) { + sum[0] += yl[8*l + j + 0] * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); + sum[1] += yl[8*l + j + 4] * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + } + } + sumf[row] += d * (sum[0] + sum[1]); + + dh += args.nb01/2; + q3 += args.nb01; + gas += args.nb01/2; + } + + y4 += 32 * ntx; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all * 0.5f; + } + } +} + +template +void kernel_mul_mv_iq3_xxs_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq3_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq3_xxs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + +[[host_name("kernel_mul_mv_iq3_xxs_f32")]] +kernel void kernel_mul_mv_iq3_xxs_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq3_xxs_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_iq3_s_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_iq3_s * x = (device const block_iq3_s *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + float yl[32]; + float sumf[nr0]={0.f}; + + threadgroup uint32_t * svalues = (threadgroup uint32_t *) shmem; + { + int nval = 8; + int pos = (32*sgitg + tiisg)*nval; + for (int i = 0; i < nval; ++i) svalues[pos + i] = iq3s_grid[pos + i]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + device const float * y4 = y + 32 * ix; + + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { + for (short i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq3_s * xr = x + ibl; + device const uint8_t * qs = xr->qs + 8 * ib; + device const uint8_t * qh = xr->qh + ib; + device const uint8_t * sc = xr->scales + (ib/2); + device const uint8_t * signs = xr->signs + 4 * ib; + device const half * dh = &xr->d; + + for (short row = row0; row < row1; row++) { + const float db = dh[0]; + const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf)); + + float2 sum = {0}; + for (short l = 0; l < 4; ++l) { + const threadgroup uint32_t * table1 = qh[0] & kmask_iq2xs[2*l+0] ? svalues + 256 : svalues; + const threadgroup uint32_t * table2 = qh[0] & kmask_iq2xs[2*l+1] ? svalues + 256 : svalues; + const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(table1 + qs[2*l+0]); + const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(table2 + qs[2*l+1]); + for (short j = 0; j < 4; ++j) { + sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l] & kmask_iq2xs[j+0]); + sum[1] += yl[8*l + j + 4] * grid2[j] * select(1, -1, signs[l] & kmask_iq2xs[j+4]); + } + } + sumf[row] += d * (sum[0] + sum[1]); + + dh += args.nb01/2; + qs += args.nb01; + qh += args.nb01; + sc += args.nb01; + signs += args.nb01; + } + + y4 += 32 * ntx; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; + } + } +} + +template +void kernel_mul_mv_iq3_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq3_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + +[[host_name("kernel_mul_mv_iq3_s_f32")]] +kernel void kernel_mul_mv_iq3_s_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq3_s_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_iq2_s_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_iq2_s * x = (device const block_iq2_s *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + float yl[32]; + float sumf[nr0]={0.f}; + + //threadgroup uint64_t * svalues = (threadgroup uint64_t *) shmem; + //{ + // int nval = 32; + // int pos = (32*sgitg + tiisg)*nval; + // for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2s_grid[pos + i]; + // threadgroup_barrier(mem_flags::mem_threadgroup); + //} + + device const float * y4 = y + 32 * ix; + + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { + for (short i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq2_s * xr = x + ibl; + device const uint8_t * qs = xr->qs + 4 * ib; + device const uint8_t * qh = xr->qh + ib; + device const uint8_t * sc = xr->scales + ib; + device const uint8_t * signs = qs + QK_K/8; + device const half * dh = &xr->d; + + for (short row = row0; row < row1; row++) { + const float db = dh[0]; + const float d1 = db * (0.5f + (sc[0] & 0xf)); + const float d2 = db * (0.5f + (sc[0] >> 4)); + + float2 sum = {0}; + for (short l = 0; l < 2; ++l) { + //const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(svalues + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300))); + //const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(svalues + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300))); + constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300))); + constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300))); + for (short j = 0; j < 8; ++j) { + sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l+0] & kmask_iq2xs[j]); + sum[1] += yl[8*l + j + 16] * grid2[j] * select(1, -1, signs[l+2] & kmask_iq2xs[j]); + } + } + sumf[row] += d1 * sum[0] + d2 * sum[1]; + + dh += args.nb01/2; + qs += args.nb01; + qh += args.nb01; + sc += args.nb01; + signs += args.nb01; + } + + y4 += 32 * ntx; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all * 0.25f; + } + } +} + +template +void kernel_mul_mv_iq2_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq2_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + +[[host_name("kernel_mul_mv_iq2_s_f32")]] +kernel void kernel_mul_mv_iq2_s_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq2_s_f32_disp(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_iq1_s_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + float yl[32]; + float sumf[nr0]={0.f}; + + device const float * y4 = y + 32 * ix; + + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { + float sumy = 0; + for (short i = 0; i < 32; ++i) { + yl[i] = y4[i]; + sumy += yl[i]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq1_s * xr = x + ibl; + device const uint8_t * qs = xr->qs + 4 * ib; + device const uint16_t * qh = xr->qh + ib; + device const half * dh = &xr->d; + + for (short row = row0; row < row1; row++) { + constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); + constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700))); + constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700))); + constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[0] >> 1) & 0x700))); + + float sum = 0; + for (short j = 0; j < 4; ++j) { + sum += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4) + + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4) + + yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4) + + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4); + } + sumf[row] += (float)dh[0] * (sum + sumy * (qh[0] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA)) * (2*((qh[0] >> 12) & 7) + 1); + + dh += args.nb01/2; + qs += args.nb01; + qh += args.nb01/2; + } + + y4 += 32 * ntx; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; + } + } +} + +template +void kernel_mul_mv_iq1_s_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_s_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + +[[host_name("kernel_mul_mv_iq1_s_f32")]] +kernel void kernel_mul_mv_iq1_s_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq1_s_f32_disp(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_iq1_m_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const int nb32 = nb * (QK_K / 32); + + const short ntx = FC_mul_mv_split ? nb32 : 32; + const short nrep = 32 / ntx; + + const short ix = tiisg % ntx; + const short irep = tiisg / ntx; + + const short row0 = (nr0 * irep ) / nrep; + const short row1 = (nr0 * (irep + 1)) / nrep; + + const uint64_t offset0 = (first_row + row0)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + float yl[32]; + float sumf[nr0]={0.f}; + + device const float * y4 = y + 32 * ix; + + iq1m_scale_t scale; + + for (int ib32 = ix; ib32 < nb32; ib32 += ntx) { + float4 sumy = {0.f}; + for (short i = 0; i < 8; ++i) { + yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; + yl[i+ 8] = y4[i+ 8]; sumy[1] += yl[i+ 8]; + yl[i+16] = y4[i+16]; sumy[2] += yl[i+16]; + yl[i+24] = y4[i+24]; sumy[3] += yl[i+24]; + } + + const int ibl = ib32 / (QK_K / 32); + const int ib = ib32 % (QK_K / 32); + + device const block_iq1_m * xr = x + ibl; + device const uint8_t * qs = xr->qs + 4 * ib; + device const uint8_t * qh = xr->qh + 2 * ib; + device const uint16_t * sc = (device const uint16_t *)xr->scales; + + for (short row = row0; row < row1; row++) { + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); + + constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); + constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700))); + constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[1] << 8) & 0x700))); + constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[1] << 4) & 0x700))); + + float2 sum = {0.f}; + for (short j = 0; j < 4; ++j) { + sum[0] += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4) + + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4); + sum[1] += yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4) + + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4); + } + const float delta1 = sumy[0] * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[1] * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); + const float delta2 = sumy[2] * (qh[1] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[3] * (qh[1] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); + + sumf[row] += (float)scale.f16 * ((sum[0] + delta1) * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 7) + 1) + + (sum[1] + delta2) * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 7) + 1)); + + sc += args.nb01/2; + qs += args.nb01; + qh += args.nb01; + } + + y4 += 32 * ntx; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; + } + } +} + +template +void kernel_mul_mv_iq1_m_f32_disp( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + if (FC_mul_mv_split) { + kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } else { + kernel_mul_mv_iq1_m_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); + } +} + +[[host_name("kernel_mul_mv_iq1_m_f32")]] +kernel void kernel_mul_mv_iq1_m_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq1_m_f32_disp(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_iq4_nl_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + threadgroup float * shmem_f32 = (threadgroup float *) shmem; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * NR0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_iq4_nl * x = (device const block_iq4_nl *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + const int nb = args.ne00/QK4_NL; + const int ns01 = args.nb01/args.nb00; + + const short ix = tiisg/2; // 0...15 + const short it = tiisg%2; // 0 or 1 + + shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + float4 yl[4]; + float sumf[NR0]={0.f}; + + device const float * yb = y + ix*QK4_NL + it*8; + + uint32_t aux32[2]; + thread const uint8_t * q8 = (thread const uint8_t *)aux32; + + float4 qf1, qf2; + + // [TAG_MUL_MV_WEIRD] + for (int ib = ix; ib < nb && ib < ns01; ib += 16) { + device const float4 * y4 = (device const float4 *)yb; + yl[0] = y4[0]; + yl[1] = y4[4]; + yl[2] = y4[1]; + yl[3] = y4[5]; + + for (short row = 0; row < NR0; row++) { + device const block_iq4_nl & xb = x[row*ns01 + ib]; + device const uint16_t * q4 = (device const uint16_t *)(xb.qs + 8*it); + + float4 acc1 = {0.f}, acc2 = {0.f}; + + aux32[0] = q4[0] | (q4[1] << 16); + aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f; + aux32[0] &= 0x0f0f0f0f; + qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]}; + qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; + acc1 += yl[0] * qf1; + acc2 += yl[1] * qf2; + + aux32[0] = q4[2] | (q4[3] << 16); + aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f; + aux32[0] &= 0x0f0f0f0f; + qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]}; + qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; + acc1 += yl[2] * qf1; + acc2 += yl[3] * qf2; + + acc1 += acc2; + + sumf[row] += (float)xb.d * (acc1[0] + acc1[1] + acc1[2] + acc1[3]); + } + + yb += 16 * QK4_NL; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; + } + } +} + +[[host_name("kernel_mul_mv_iq4_nl_f32")]] +kernel void kernel_mul_mv_iq4_nl_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq4_nl_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_iq4_xs_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + threadgroup float * shmem_f32 = (threadgroup float *) shmem; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + const int first_row = (r0 * NSG + sgitg) * NR0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_iq4_xs * x = (device const block_iq4_xs *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + const int nb = args.ne00/QK_K; + const int ns01 = args.nb01/args.nb00; + + const short ix = tiisg/16; // 0 or 1 + const short it = tiisg%16; // 0...15 + const short ib = it/2; + const short il = it%2; + + shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + float4 yl[4]; + float sumf[NR0]={0.f}; + + device const float * yb = y + ix * QK_K + ib * 32 + il * 8; + + uint32_t aux32[2]; + thread const uint8_t * q8 = (thread const uint8_t *)aux32; + + float4 qf1, qf2; + + // [TAG_MUL_MV_WEIRD] + for (int ibl = ix; ibl < nb && ibl < ns01; ibl += 2) { + device const float4 * y4 = (device const float4 *)yb; + yl[0] = y4[0]; + yl[1] = y4[4]; + yl[2] = y4[1]; + yl[3] = y4[5]; + + for (short row = 0; row < NR0; ++row) { + device const block_iq4_xs & xb = x[row*ns01 + ibl]; + device const uint32_t * q4 = (device const uint32_t *)(xb.qs + 16*ib + 8*il); + + float4 acc1 = {0.f}, acc2 = {0.f}; + + aux32[0] = (q4[0] ) & 0x0f0f0f0f; + aux32[1] = (q4[0] >> 4) & 0x0f0f0f0f; + qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]}; + qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; + acc1 += yl[0] * qf1; + acc2 += yl[1] * qf2; + + aux32[0] = (q4[1] ) & 0x0f0f0f0f; + aux32[1] = (q4[1] >> 4) & 0x0f0f0f0f; + qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]}; + qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]}; + acc1 += yl[2] * qf1; + acc2 += yl[3] * qf2; + + acc1 += acc2; + + const int ls = (((xb.scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((xb.scales_h >> 2*ib) & 3) << 4)) - 32; + sumf[row] += (float)xb.d * ls * (acc1[0] + acc1[1] + acc1[2] + acc1[3]); + } + + yb += 2 * QK_K; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; + } + } +} + +[[host_name("kernel_mul_mv_iq4_xs_f32")]] +kernel void kernel_mul_mv_iq4_xs_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_iq4_xs_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_mxfp4_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + threadgroup float * shmem_f32 = (threadgroup float *) shmem; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * NR0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const block_mxfp4 * x = (device const block_mxfp4 *) (src0 + offset0); + device const float * y = (device const float *) (src1 + offset1); + + const int nb = args.ne00/QK_MXFP4; + const int ns01 = args.nb01/args.nb00; // this can be larger than nb for permuted src0 tensors + + const short ix = tiisg/2; // 0...15 + const short it = tiisg%2; // 0 or 1 + + shmem_f32[tiisg] = kvalues_mxfp4_f[tiisg%16]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + float4 yl[4]; + float sumf[NR0]={0.f}; + + device const float * yb = y + ix*QK_MXFP4 + it*8; + + // note: just the check `ib < nb` is enough, but adding the redundant `&& ib < ns01` check makes the kernel a bit faster + // no idea why that is - needs some deeper investigation [TAG_MUL_MV_WEIRD] + for (int ib = ix; ib < nb && ib < ns01; ib += 16) { + device const float4 * y4 = (device const float4 *) yb; + + yl[0] = y4[0]; + yl[1] = y4[4]; + yl[2] = y4[1]; + yl[3] = y4[5]; + + FOR_UNROLL (short row = 0; row < NR0; row++) { + device const block_mxfp4 & xb = x[row*ns01 + ib]; + device const uint8_t * q2 = (device const uint8_t *)(xb.qs + 8*it); + + float4 acc1 = yl[0]*float4(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]); + float4 acc2 = yl[1]*float4(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]); + float4 acc3 = yl[2]*float4(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]); + float4 acc4 = yl[3]*float4(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]); + + acc1 = (acc1 + acc3) + (acc2 + acc4); + + sumf[row] += e8m0_to_fp32(xb.e) * ((acc1[0] + acc1[1]) + (acc1[2] + acc1[3])); + } + + yb += 16 * QK_MXFP4; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) { + float sum_all = simd_sum(sumf[row]); + if (tiisg == 0) { + dst_f32[first_row + row] = sum_all; + } + } +} + +[[host_name("kernel_mul_mv_mxfp4_f32")]] +kernel void kernel_mul_mv_mxfp4_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_mxfp4_f32_impl(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +template +void kernel_mul_mv_tq2_0_f32_impl( + args_t args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg) { + const short NSG = FC_mul_mv_nsg; + + const int nb = args.ne00/QK_K; + + const int r0 = tgpig.x; + const int r1 = tgpig.y; + const int im = tgpig.z; + + const int first_row = (r0 * NSG + sgitg) * nr0; + + const uint i12 = im%FC_mul_mv_ne12; + const uint i13 = im/FC_mul_mv_ne12; + + const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13; + + device const float * y = (device const float *) (src1 + offset1); + + device const block_tq2_0 * ax[nr0]; + for (int row = 0; row < nr0; ++row) { + const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03; + ax[row] = (device const block_tq2_0 *) ((device char *) src0 + offset0); + } + + float sumf[nr0] = {0.f}; + + // 8 threads per block, NBLOCK blocks per pass, 2 halves per block per pass + constexpr short NBLOCK = 4; + + constexpr short NB = N_SIMDWIDTH/NBLOCK; // threads per block + + const short blk = tiisg / NB; // 0..NBLOCK-1, block handled by this thread + const short htg = tiisg % NB; // 0..NB-1, thread within block (0..7) + + // byte and y base offsets within the block (32 elements per thread, 4 per byte) + device const float4 * yb4 = (device const float4 *)(y + 4*htg + blk*QK_K); + + // hoisted per-byte coefficients (from y) and total y-sum, shared across rows + // ref: https://github.com/ggml-org/llama.cpp/pull/26980 + float4 coef[4]; + + for (int ib = blk; ib < nb; ib += NBLOCK) { + FOR_UNROLL (short h0 = 0; h0 < 2; ++h0) { + const float4 y0 = yb4[ 0 + 32*h0]; + const float4 y1 = yb4[ 8 + 32*h0]; + const float4 y2 = yb4[16 + 32*h0]; + const float4 y3 = yb4[24 + 32*h0]; + + float sumy = 0.f; + FOR_UNROLL (short j = 0; j < 4; ++j) { + coef[j] = float4( + y0[j], + y1[j] - 4.0f*y0[j], + y2[j] - 4.0f*y1[j], + y3[j] - 4.0f*y2[j]); + + sumy += (y0[j] + y1[j]) + (y2[j] + y3[j]); + } + + FOR_UNROLL (short row = 0; row < nr0; ++row) { + device const block_tq2_0 & xb = ax[row][ib]; + device const uchar * qs = xb.qs + 4*htg + 32*h0; + + float sum = -sumy; + FOR_UNROLL (short j = 0; j < 4; ++j) { + // express the 2-bit field shifts (v>>2, v>>4, v>>6) as float floor ops + const float v = (float)qs[j]; + + const float f0 = v; + const float f1 = floor(v*0.25f); // v>>2 + const float f2 = floor(v*0.0625); // v>>4 + const float f3 = floor(v*0.015625); // v>>6 + + sum += coef[j][0]*f0 + coef[j][1]*f1 + coef[j][2]*f2 + coef[j][3]*f3; + } + + sumf[row] += xb.d * sum; + } + } + + yb4 += QK_K * NBLOCK / 4; + } + + device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0; + + for (int row = 0; row < nr0; ++row) { + const float tot = simd_sum(sumf[row]); + if (tiisg == 0 && first_row + row < args.ne01) { + dst_f32[first_row + row] = tot; + } + } +} + +[[host_name("kernel_mul_mv_tq2_0_f32")]] +kernel void kernel_mul_mv_tq2_0_f32( + constant ggml_metal_kargs_mul_mv & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + + kernel_mul_mv_tq2_0_f32_impl(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg); +} + +// +// matrix-vector multiplication +// + +typedef void (kernel_mul_mv_disp_t)( + ggml_metal_kargs_mul_mv args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig, + ushort tiisg); + +typedef void (kernel_mul_mv2_disp_t)( + ggml_metal_kargs_mul_mv args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiisg, + ushort sgitg); + +template +void mmv_fn( + ggml_metal_kargs_mul_mv args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiitg, + ushort tiisg, + ushort sgitg) { + disp_fn(args, src0, src1, dst, tgpig, tiisg); +} + +template +void mmv_fn( + ggml_metal_kargs_mul_mv args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem, + uint3 tgpig, + ushort tiitg, + ushort tiisg, + ushort sgitg) { + disp_fn(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); +} + +typedef decltype(mmv_fn>) mul_mv_disp_fn_t; + +template +kernel void kernel_mul_mv_id( + constant ggml_metal_kargs_mul_mv_id & args, + device const char * src0s, + device const char * src1, + device char * dst, + device const char * ids, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort3 ntg [[threads_per_threadgroup]]) { + const int iid1 = tgpig.z/args.nei0; + const int idx = tgpig.z%args.nei0; + + tgpig.z = 0; + + const int32_t i02 = ((device const int32_t *) (ids + iid1*args.nbi1))[idx]; + + const int64_t i11 = idx % args.ne11; + const int64_t i12 = iid1; + + const int64_t i1 = idx; + const int64_t i2 = i12; + + device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float); + + if (i02 == -1) { + device float * dst_f32 = (device float *) dst_cur; + const ushort nth = ntg.x*ntg.y*ntg.z; + for (int i0 = tiitg; i0 < args.ne0; i0 += nth) { + dst_f32[i0] = 0.0f; + } + return; + } + + device const char * src0_cur = src0s + i02*args.nb02; + device const char * src1_cur = src1 + i11*args.nb11 + i12*args.nb12; + + ggml_metal_kargs_mul_mv args0 = { + /*.ne00 =*/ args.ne00, + /*.ne01 =*/ args.ne01, + /*.ne02 =*/ 1, // args.ne02, + /*.nb00 =*/ args.nb00, + /*.nb01 =*/ args.nb01, + /*.nb02 =*/ args.nb02, + /*.nb03 =*/ args.nb02, // args.ne02 == 1 + /*.ne10 =*/ args.ne10, + /*.ne11 =*/ 1, // args.ne11, + /*.ne12 =*/ 1, // args.ne12, + /*.nb10 =*/ args.nb10, + /*.nb11 =*/ args.nb11, + /*.nb12 =*/ args.nb12, + /*.nb13 =*/ args.nb12, // ne12 == 1 + /*.ne0 =*/ args.ne0, + /*.ne1 =*/ 1, // args.ne1, + /*.nr0 =*/ args.nr0, + /*.r2 =*/ 1, + /*.r3 =*/ 1, + }; + + disp_fn( + args0, + /* src0 */ src0_cur, + /* src1 */ src1_cur, + /* dst */ dst_cur, + shmem, + tgpig, + tiitg, + tiisg, + sgitg); +} + +typedef decltype(kernel_mul_mv_id>>) kernel_mul_mv_id_t; + +typedef decltype(kernel_mul_mv_id>>) kernel_mul_mv_id_4_t; + +template [[host_name("kernel_mul_mv_id_f32_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_f16_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mv_id_bf16_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +#endif +template [[host_name("kernel_mul_mv_id_f32_f32_4")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_f16_f32_4")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id>>; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_mul_mv_id_bf16_f32_4")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id>>; +#endif + +template [[host_name("kernel_mul_mv_id_q8_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; + +template [[host_name("kernel_mul_mv_id_q1_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q4_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q4_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q5_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q5_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; + +template [[host_name("kernel_mul_mv_id_mxfp4_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; + +template [[host_name("kernel_mul_mv_id_q2_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q3_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q4_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q5_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_q6_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; +template [[host_name("kernel_mul_mv_id_tq2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>>; diff --git a/ggml/src/ggml-metal/kernels/mul_mv_mma.metal b/ggml/src/ggml-metal/kernels/mul_mv_mma.metal new file mode 100644 index 0000000..5cf245d --- /dev/null +++ b/ggml/src/ggml-metal/kernels/mul_mv_mma.metal @@ -0,0 +1,627 @@ +#include "common.h" +#include "dequantize.h" + +constant short FC_mul_mv_mma_nsg [[function_constant(FC_MUL_MV_MMA + 0)]]; +constant short FC_mul_mv_mma_ne12 [[function_constant(FC_MUL_MV_MMA + 1)]]; +constant short FC_mul_mv_mma_r2 [[function_constant(FC_MUL_MV_MMA + 2)]]; +constant short FC_mul_mv_mma_r3 [[function_constant(FC_MUL_MV_MMA + 3)]]; +constant int FC_mul_mv_mma_ne00 [[function_constant(FC_MUL_MV_MMA + 4)]]; +constant bool FC_mul_mv_mma_add [[function_constant(FC_MUL_MV_MMA + 5)]]; + +// a lane of a few-row MMA tile: A fragment row fm, B fragment columns fn and fn + 1 +struct mul_mv_mma_tile { + short fm; + short fn; + int i01; + int i11; + int i1m; + uint64_t offset0; + uint64_t offset1; +}; + +// the A fragment row and the first B fragment column that lane l holds in an 8x8 simdgroup matrix +inline short mul_mv_mma_lane_fm(ushort l) { return ((l/4) & 4) + ((l/2) % 4); } +inline short mul_mv_mma_lane_fn(ushort l) { return ((l/4) & 2)*2 + (l % 2)*2; } + +template +inline mul_mv_mma_tile mul_mv_mma_tile_init(constant ggml_metal_kargs_mul_mv_ext & args, uint3 tgpig, ushort tiisg) { + mul_mv_mma_tile tile; + tile.fm = mul_mv_mma_lane_fm(tiisg); + tile.fn = mul_mv_mma_lane_fn(tiisg); + tile.i01 = tgpig.x*(8*NT); + tile.i11 = tgpig.y*(8*RT); + tile.i1m = tgpig.z; + + const int i12 = tile.i1m%FC_mul_mv_mma_ne12; + const int i13 = tile.i1m/FC_mul_mv_mma_ne12; + + tile.offset0 = (i12/FC_mul_mv_mma_r2)*args.nb02 + (i13/FC_mul_mv_mma_r3)*args.nb03; + tile.offset1 = i12*args.nb12 + i13*args.nb13; + + return tile; +} + +// the src0 row of A fragment row fm in 8-row tile t, clamped to the last row +inline device const char * mul_mv_mma_src0_row( + thread const mul_mv_mma_tile & tile, constant ggml_metal_kargs_mul_mv_ext & args, device const char * src0, short t) { + const int r = min(tile.i01 + 8*t + tile.fm, args.ne01 - 1); + return src0 + tile.offset0 + (uint64_t) r*args.nb01; +} + +// the src1 row of B fragment column fn + e in 8-row tile rt, clamped to the last row +inline device const float * mul_mv_mma_src1_row( + thread const mul_mv_mma_tile & tile, constant ggml_metal_kargs_mul_mv_ext & args, device const char * src1, short rt, short e) { + const int r = min(tile.i11 + 8*rt + tile.fn + e, args.ne11 - 1); + return (device const float *) (src1 + tile.offset1 + (uint64_t) r*args.nb11); +} + +constexpr constant static ushort mma_f16_1024_bits = 0x6400; +constexpr constant static half mma_f16_1024 = 1024.0h; + +// the halves 1024 + q for integers q < 1024: exact normal values, unlike the subnormal q*2^-24 that Metal may flush to zero +inline half2 mul_mv_mma_1024_plus(ushort2 q) { + return as_type(q | mma_f16_1024_bits); +} + +// adds up the K slices of the NSG simdgroups for an 8*NT x 8*RT output tile and writes it, plus the residual in src2 if FC_mul_mv_mma_add +template +inline void mul_mv_mma_store( + thread float (&acc)[RT][NT][2], + constant ggml_metal_kargs_mul_mv_ext & args, + device const char * src2, + device char * dst, + threadgroup char * shmem, + thread const mul_mv_mma_tile & tile, ushort tiisg, ushort sgitg) { + const short NSG = FC_mul_mv_mma_nsg; + + threadgroup float * red = (threadgroup float *) shmem; + + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short t = 0; t < NT; ++t) { + red[((sgitg*RT + rt)*NT + t)*64 + 2*tiisg + 0] = acc[rt][t][0]; + red[((sgitg*RT + rt)*NT + t)*64 + 2*tiisg + 1] = acc[rt][t][1]; + } + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + device float * dst_f32 = (device float *) dst + (uint64_t) tile.i1m*args.ne0*args.ne1; + device const float * res_f32 = (device const float *) src2 + (uint64_t) tile.i1m*args.ne0*args.ne1; + + for (short idx = sgitg*32 + tiisg; idx < RT*NT*64; idx += NSG*32) { + float sum = 0.0f; + for (short sg = 0; sg < NSG; ++sg) { + sum += red[sg*(RT*NT*64) + idx]; + } + + const short rt = idx/(NT*64); + const short t = (idx/64) % NT; + const short l = (idx % 64)/2; + const short e = idx % 2; + + const int r0 = tile.i01 + 8*t + mul_mv_mma_lane_fm(l); + const int r1 = tile.i11 + 8*rt + mul_mv_mma_lane_fn(l) + e; + + if (r0 < args.ne01 && r1 < args.ne11) { + const uint64_t i = (uint64_t) r1*args.ne0 + r0; + dst_f32[i] = FC_mul_mv_mma_add ? sum + res_f32[i] : sum; + } + } +} + +// the per-type parts of kernel_mul_mv_mma_blk for 32-weight blocks. a src1 block splits into halves b0 and b1, +// and MMA step s uses half b1 when b1_step(s) and the .y value of a pair when y_step(s) +// q4_0: A lane (m, j) holds qs ushorts j and j + 1 (j even); a high nibble stays in place as 16*q, so b1 is divided by 16. +// B lane k = fm holds src1 values 2*k, 2*k + 1 (b0, low nibbles) and 2*k + 16, 2*k + 17 (b1, high nibbles) of a block +struct mul_mv_mma_q4_0 { + typedef block_q4_0 block; + typedef ushort2 quants; + + // weights per block, and the float2 offset of b1 in a src1 block + enum { qk = QK4_0, b1 = QK4_0/4 }; + + static short a_off(short fn) { return 1 + fn; } + static short b_off(short fm) { return 2*fm; } + + static quants load(device const ushort * qs) { return ushort2(qs[0], qs[1]); } + static quants prep(quants q) { return q; } + static float2 prep_b1(float2 v) { + constexpr float hi_scale = 1.0f/16; + return v*hi_scale; + } + + static bool b1_step(short s) { return s % 2 != 0; } + static bool y_step (short s) { return s >= 2; } + + static half2 frag(quants q, short s) { + constexpr ushort lo_mask = 0x000F; + constexpr ushort hi_mask = 0x00F0; + constexpr half lo_zero = 8.0h; + constexpr half hi_zero = 16*lo_zero; + + const ushort2 qq = s < 2 ? q : q >> 8; + return s % 2 == 0 ? mul_mv_mma_1024_plus(qq & lo_mask) - (mma_f16_1024 + lo_zero) : mul_mv_mma_1024_plus(qq & hi_mask) - (mma_f16_1024 + hi_zero); + } +}; + +// q8_0: A lane (m, j) holds qs bytes 4*j .. 4*j + 7 (j even); flipping the sign bit of a quant byte gives the unsigned q + 128. +// B lane k = fm holds src1 values b, b + 1 (b0) and b + 4, b + 5 (b1) of a block, b = 8*(k/2) + 2*(k%2) +struct mul_mv_mma_q8_0 { + typedef block_q8_0 block; + typedef ushort4 quants; + + // weights per block, and the float2 offset of b1 in a src1 block + enum { qk = QK8_0, b1 = 2 }; + + static short a_off(short fn) { return 1 + 2*fn; } + static short b_off(short fm) { return 8*(fm/2) + 2*(fm%2); } + + static quants load(device const ushort * qs) { return ushort4(qs[0], qs[1], qs[2], qs[3]); } + static quants prep(quants q) { + constexpr ushort sign_bits = 0x8080; + return q ^ ushort4(sign_bits); + } + static float2 prep_b1(float2 v) { return v; } + + static bool b1_step(short s) { return s >= 2; } + static bool y_step (short s) { return s % 2 != 0; } + + static half2 frag(quants q, short s) { + constexpr ushort byte_mask = 0x00FF; + constexpr half q_bias = mma_f16_1024 + 128.0h; + + const ushort2 w = s < 2 ? q.xy : q.zw; + const ushort2 qq = s % 2 == 0 ? w : w >> 8; + return mul_mv_mma_1024_plus(qq & byte_mask) - q_bias; + } +}; + +template +inline void load_mma_blk_a(device const ushort * const x[NT], int off, short fn, thread typename Q::quants * q, thread float * d) { + FOR_UNROLL (short t = 0; t < NT; ++t) { + device const ushort * qs = x[t] + off; + q[t] = Q::load(qs); + d[t] = as_type(*(qs - Q::a_off(fn))); + } +} + +template +inline void load_mma_blk_b(device const float2 * const y[RT][2], int ib, thread float2 (*b0)[2], thread float2 (*b1)[2]) { + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short e = 0; e < 2; ++e) { + b0[rt][e] = y[rt][e][ib*(Q::qk/2)]; + b1[rt][e] = y[rt][e][ib*(Q::qk/2) + Q::b1]; + } + } +} + +// few-row mat-mat (2..16 src1 rows) on 8x8 simdgroup matrices for 32-weight block types: a threadgroup reads each weight once +// for 8*NT src0 rows x 8*RT src1 rows, and its NSG simdgroups split K +template +kernel void kernel_mul_mv_mma_blk( + constant ggml_metal_kargs_mul_mv_ext & args, + device const char * src0, + device const char * src1, + device char * dst, + device const char * src2, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + const short NSG = FC_mul_mv_mma_nsg; + + const mul_mv_mma_tile tile = mul_mv_mma_tile_init(args, tgpig, tiisg); + + device const ushort * x[NT]; + FOR_UNROLL (short t = 0; t < NT; ++t) { + x[t] = (device const ushort *) mul_mv_mma_src0_row(tile, args, src0, t) + Q::a_off(tile.fn); + } + + device const float2 * y[RT][2]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short e = 0; e < 2; ++e) { + y[rt][e] = (device const float2 *) (mul_mv_mma_src1_row(tile, args, src1, rt, e) + Q::b_off(tile.fm)); + } + } + + float acc[RT][NT][2] = {}; + + const int nb = FC_mul_mv_mma_ne00/Q::qk; + + // a block is d, then the quants + constexpr short us_blk = sizeof(typename Q::block)/2; + + typename Q::quants q[NT]; + float d[NT]; + float2 b0[RT][2]; + float2 b1[RT][2]; + + const int ib0 = min((int) sgitg, nb - 1); + load_mma_blk_a(x, ib0*us_blk, tile.fn, q, d); + load_mma_blk_b(y, ib0, b0, b1); + + for (int ib = sgitg; ib < nb; ib += NSG) { + typename Q::quants qc[NT]; + float dc[NT]; + float2 b0c[RT][2]; + float2 b1c[RT][2]; + + FOR_UNROLL (short t = 0; t < NT; ++t) { + qc[t] = Q::prep(q[t]); + dc[t] = d[t]; + } + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short e = 0; e < 2; ++e) { + b0c[rt][e] = b0[rt][e]; + b1c[rt][e] = Q::prep_b1(b1[rt][e]); + } + } + + const int ibn = min(ib + NSG, nb - 1); + load_mma_blk_a(x, ibn*us_blk, tile.fn, q, d); + load_mma_blk_b(y, ibn, b0, b1); + + simdgroup_float8x8 mp[RT][NT]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short t = 0; t < NT; ++t) { + mp[rt][t] = make_filled_simdgroup_matrix(0.0f); + } + } + + FOR_UNROLL (short s = 0; s < 4; ++s) { + simdgroup_float8x8 mb[RT]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + const float2 v0 = Q::b1_step(s) ? b1c[rt][0] : b0c[rt][0]; + const float2 v1 = Q::b1_step(s) ? b1c[rt][1] : b0c[rt][1]; + mb[rt].thread_elements()[0] = Q::y_step(s) ? v0.y : v0.x; + mb[rt].thread_elements()[1] = Q::y_step(s) ? v1.y : v1.x; + } + + FOR_UNROLL (short t = 0; t < NT; ++t) { + const half2 h = Q::frag(qc[t], s); + + simdgroup_half8x8 ma; + ma.thread_elements()[0] = h.x; + ma.thread_elements()[1] = h.y; + + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + simdgroup_multiply_accumulate(mp[rt][t], ma, mb[rt], mp[rt][t]); + } + } + } + + FOR_UNROLL (short t = 0; t < NT; ++t) { + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + acc[rt][t][0] = fma(dc[t], mp[rt][t].thread_elements()[0], acc[rt][t][0]); + acc[rt][t][1] = fma(dc[t], mp[rt][t].thread_elements()[1], acc[rt][t][1]); + } + } + } + + mul_mv_mma_store(acc, args, src2, dst, shmem, tile, tiisg, sgitg); +} + +typedef decltype(kernel_mul_mv_mma_blk<4, 1, mul_mv_mma_q4_0>) mul_mv_mma_t; + +template [[host_name("kernel_mul_mv_mma_q4_0_f32_nt1_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_blk<1, 1, mul_mv_mma_q4_0>; +template [[host_name("kernel_mul_mv_mma_q4_0_f32_nt2_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_blk<2, 1, mul_mv_mma_q4_0>; +template [[host_name("kernel_mul_mv_mma_q4_0_f32_nt4_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_blk<4, 1, mul_mv_mma_q4_0>; +template [[host_name("kernel_mul_mv_mma_q4_0_f32_nt1_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_blk<1, 2, mul_mv_mma_q4_0>; +template [[host_name("kernel_mul_mv_mma_q4_0_f32_nt2_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_blk<2, 2, mul_mv_mma_q4_0>; +template [[host_name("kernel_mul_mv_mma_q4_0_f32_nt4_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_blk<4, 2, mul_mv_mma_q4_0>; + +template [[host_name("kernel_mul_mv_mma_q8_0_f32_nt1_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_blk<1, 1, mul_mv_mma_q8_0>; +template [[host_name("kernel_mul_mv_mma_q8_0_f32_nt2_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_blk<2, 1, mul_mv_mma_q8_0>; +template [[host_name("kernel_mul_mv_mma_q8_0_f32_nt4_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_blk<4, 1, mul_mv_mma_q8_0>; + +// q5_K scale and min of sub-block j from the 12 packed bytes, held as 3 words +inline float2 mul_mv_mma_q5_K_scale_min(thread const uint * w, short j) { + if (j < 4) { + return float2((w[0] >> 8*j) & 63, (w[1] >> 8*j) & 63); + } + const short k = 8*(j - 4); + return float2(((w[2] >> k) & 0xF) | (((w[0] >> (k + 6)) & 3) << 4), ((w[2] >> (k + 4)) & 0xF) | (((w[1] >> (k + 6)) & 3) << 4)); +} + +// the A fragments of one qs/qh word: lo holds the 5-bit quants q of the low-nibble sub-block of a pair, hi holds 16*q of the high-nibble sub-block. +// step e0 takes bytes 0 and 2 of the word, step e1 takes bytes 1 and 3; hs holds the qh bits of the pair. +inline void mul_mv_mma_q5_K_frags(uint q, uint hs, thread half2 * lo, thread half2 * hi) { + const ushort2 qw = as_type(q); + const ushort2 hw = as_type(hs); + + lo[0] = mul_mv_mma_1024_plus((qw & 0x000F) | ((hw << 4) & 0x0010)) - mma_f16_1024; + lo[1] = mul_mv_mma_1024_plus(((qw >> 8) & 0x000F) | ((hw >> 4) & 0x0010)) - mma_f16_1024; + hi[0] = mul_mv_mma_1024_plus((qw & 0x00F0) | ((hw << 7) & 0x0100)) - mma_f16_1024; + hi[1] = mul_mv_mma_1024_plus(((qw >> 8) & 0x00F0) | ((hw >> 1) & 0x0100)) - mma_f16_1024; +} + +struct mul_mv_mma_q5_K_a { + uint2 q; + uint2 h; + uint sc[3]; + uint dm; +}; + +template +inline void load_q5_K_mma_a(device const block_q5_K * const x[NT], int ip, short fn, thread mul_mv_mma_q5_K_a * a) { + constexpr short pairs = QK_K/64; + + FOR_UNROLL (short t = 0; t < NT; ++t) { + device const block_q5_K * xb = x[t] + ip/pairs; + device const uint * sp = (device const uint *) xb->scales; + + a[t].q = *((device const uint2 *) (xb->qs + 32*(ip%pairs)) + fn/2); + a[t].h = *((device const uint2 *) xb->qh + fn/2); + a[t].sc[0] = sp[0]; + a[t].sc[1] = sp[1]; + a[t].sc[2] = sp[2]; + a[t].dm = *((device const uint *) xb); + } +} + +// xs[rt][e][h]: src1 values of sub-block h of the pair, for the steps (b, b + 1) and (b + 4, b + 5) +template +inline void load_q5_K_mma_b(device const float2 * const y[RT][2], int ip, thread float2 (*xs)[2][2][2]) { + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short e = 0; e < 2; ++e) { + FOR_UNROLL (short h = 0; h < 2; ++h) { + xs[rt][e][h][0] = y[rt][e][ip*32 + h*16 + 0]; + xs[rt][e][h][1] = y[rt][e][ip*32 + h*16 + 2]; + } + } + } +} + +// few-row mat-mat for q5_K over pairs of 32-weight sub-blocks, laid out like mul_mv_mma_q4_0: A lane (m, j) holds qs and qh bytes 4*j .. 4*j + 7 of a pair (j even). +// the 1/16 of the in-place high nibbles and the sub-block scales go into the accumulation; the mins are removed with the src1 sums. +template +kernel void kernel_mul_mv_mma_q5_K_f32( + constant ggml_metal_kargs_mul_mv_ext & args, + device const char * src0, + device const char * src1, + device char * dst, + device const char * src2, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + const short NSG = FC_mul_mv_mma_nsg; + + constexpr float hi_scale = 1.0f/16; + constexpr short pairs = QK_K/64; + + const mul_mv_mma_tile tile = mul_mv_mma_tile_init(args, tgpig, tiisg); + + device const block_q5_K * x[NT]; + FOR_UNROLL (short t = 0; t < NT; ++t) { + x[t] = (device const block_q5_K *) mul_mv_mma_src0_row(tile, args, src0, t); + } + + // B lane k = fm holds src1 values b, b + 1, b + 4, b + 5 of each sub-block, b = 8*(k/2) + 2*(k%2) + device const float2 * y[RT][2]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short e = 0; e < 2; ++e) { + y[rt][e] = (device const float2 *) (mul_mv_mma_src1_row(tile, args, src1, rt, e) + 8*(tile.fm/2) + 2*(tile.fm%2)); + } + } + + float acc[RT][NT][2] = {}; + + const int np = FC_mul_mv_mma_ne00/64; + + mul_mv_mma_q5_K_a an[NT]; + float2 xn[RT][2][2][2]; + + const int ip0 = min((int) sgitg, np - 1); + load_q5_K_mma_a(x, ip0, tile.fn, an); + load_q5_K_mma_b(y, ip0, xn); + + for (int ip = sgitg; ip < np; ip += NSG) { + const short p = ip%pairs; + + mul_mv_mma_q5_K_a ac[NT]; + float2 xs[RT][2][2][2]; + FOR_UNROLL (short t = 0; t < NT; ++t) { + ac[t] = an[t]; + } + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short e = 0; e < 2; ++e) { + FOR_UNROLL (short h = 0; h < 2; ++h) { + xs[rt][e][h][0] = xn[rt][e][h][0]; + xs[rt][e][h][1] = xn[rt][e][h][1]; + } + } + } + + const int ipn = min(ip + NSG, np - 1); + load_q5_K_mma_a(x, ipn, tile.fn, an); + load_q5_K_mma_b(y, ipn, xn); + + float c[RT][2][2]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short e = 0; e < 2; ++e) { + FOR_UNROLL (short h = 0; h < 2; ++h) { + float u = xs[rt][e][h][0].x + xs[rt][e][h][0].y + xs[rt][e][h][1].x + xs[rt][e][h][1].y; + u += simd_shuffle_xor(u, 2); + u += simd_shuffle_xor(u, 4); + u += simd_shuffle_xor(u, 16); + c[rt][e][h] = u; + } + } + } + + FOR_UNROLL (short t = 0; t < NT; ++t) { + const float2 dm = float2(as_type(ac[t].dm)); + const float2 sm0 = mul_mv_mma_q5_K_scale_min(ac[t].sc, 2*p + 0); + const float2 sm1 = mul_mv_mma_q5_K_scale_min(ac[t].sc, 2*p + 1); + + half2 a[2][2][2]; + mul_mv_mma_q5_K_frags(ac[t].q.x, ac[t].h.x >> 2*p, a[0][0], a[1][0]); + mul_mv_mma_q5_K_frags(ac[t].q.y, ac[t].h.y >> 2*p, a[0][1], a[1][1]); + + FOR_UNROLL (short hh = 0; hh < 2; ++hh) { + simdgroup_float8x8 mp[RT]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + mp[rt] = make_filled_simdgroup_matrix(0.0f); + } + + FOR_UNROLL (short s = 0; s < 4; ++s) { + simdgroup_half8x8 ma; + ma.thread_elements()[0] = a[hh][s/2][s%2].x; + ma.thread_elements()[1] = a[hh][s/2][s%2].y; + + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + simdgroup_float8x8 mb; + mb.thread_elements()[0] = s % 2 == 0 ? xs[rt][0][hh][s/2].x : xs[rt][0][hh][s/2].y; + mb.thread_elements()[1] = s % 2 == 0 ? xs[rt][1][hh][s/2].x : xs[rt][1][hh][s/2].y; + + simdgroup_multiply_accumulate(mp[rt], ma, mb, mp[rt]); + } + } + + const float2 smh = hh == 0 ? sm0 : sm1; + const float dsc = dm.x*smh.x*(hh == 0 ? 1.0f : hi_scale); + const float dmn = dm.y*smh.y; + + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + acc[rt][t][0] = fma(dsc, mp[rt].thread_elements()[0], fma(-dmn, c[rt][0][hh], acc[rt][t][0])); + acc[rt][t][1] = fma(dsc, mp[rt].thread_elements()[1], fma(-dmn, c[rt][1][hh], acc[rt][t][1])); + } + } + } + } + + mul_mv_mma_store(acc, args, src2, dst, shmem, tile, tiisg, sgitg); +} + +template [[host_name("kernel_mul_mv_mma_q5_K_f32_nt1_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_q5_K_f32<1, 1>; +template [[host_name("kernel_mul_mv_mma_q5_K_f32_nt2_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_q5_K_f32<2, 1>; +template [[host_name("kernel_mul_mv_mma_q5_K_f32_nt4_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_q5_K_f32<4, 1>; +template [[host_name("kernel_mul_mv_mma_q5_K_f32_nt1_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_q5_K_f32<1, 2>; +template [[host_name("kernel_mul_mv_mma_q5_K_f32_nt2_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_q5_K_f32<2, 2>; +template [[host_name("kernel_mul_mv_mma_q5_K_f32_nt4_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_q5_K_f32<4, 2>; + +// few-row mat-mat for any type with a 16-weight dequantizer: a lane dequantizes 16 consecutive weights of a 64-weight chunk once for all src1 rows. +// MMA step s at MMA-k index j reads chunk weight 16*(j/2) + 8*(j%2) + s. +template +kernel void kernel_mul_mv_mma_gen( + constant ggml_metal_kargs_mul_mv_ext & args, + device const char * src0, + device const char * src1, + device char * dst, + device const char * src2, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]]) { + const short NSG = FC_mul_mv_mma_nsg; + + const mul_mv_mma_tile tile = mul_mv_mma_tile_init(args, tgpig, tiisg); + + device const block_q * x[NT]; + FOR_UNROLL (short t = 0; t < NT; ++t) { + x[t] = (device const block_q *) mul_mv_mma_src0_row(tile, args, src0, t); + } + + // B lane j = fm reads chunk values 16*(fm/2) + 8*(fm%2) .. +7 + device const float4 * y[RT][2]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short e = 0; e < 2; ++e) { + y[rt][e] = (device const float4 *) mul_mv_mma_src1_row(tile, args, src1, rt, e) + 4*(tile.fm/2) + 2*(tile.fm%2); + } + } + + simdgroup_float8x8 mc[RT][NT]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short t = 0; t < NT; ++t) { + mc[rt][t] = make_filled_simdgroup_matrix(0.0f); + } + } + + const int nch = args.ne00/64; + + for (int g = sgitg; g < nch; g += NSG) { + simdgroup_float8x8 mb[RT][8]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + const float4 a0 = y[rt][0][16*g + 0]; + const float4 a1 = y[rt][0][16*g + 1]; + const float4 b0 = y[rt][1][16*g + 0]; + const float4 b1 = y[rt][1][16*g + 1]; + FOR_UNROLL (short s = 0; s < 4; ++s) { + mb[rt][s ].thread_elements()[0] = a0[s]; + mb[rt][s ].thread_elements()[1] = b0[s]; + mb[rt][s + 4].thread_elements()[0] = a1[s]; + mb[rt][s + 4].thread_elements()[1] = b1[s]; + } + } + + const int ci = 4*g + tile.fn/2; + + FOR_UNROLL (short t = 0; t < NT; ++t) { + float4x4 w; + dequantize_func(x[t] + ci/nl, ci%nl, w); + + FOR_UNROLL (short s = 0; s < 8; ++s) { + simdgroup_float8x8 ma; + ma.thread_elements()[0] = w[s/4 ][s%4]; + ma.thread_elements()[1] = w[s/4 + 2][s%4]; + + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + simdgroup_multiply_accumulate(mc[rt][t], ma, mb[rt][s], mc[rt][t]); + } + } + } + } + + float acc[RT][NT][2]; + FOR_UNROLL (short rt = 0; rt < RT; ++rt) { + FOR_UNROLL (short t = 0; t < NT; ++t) { + acc[rt][t][0] = mc[rt][t].thread_elements()[0]; + acc[rt][t][1] = mc[rt][t].thread_elements()[1]; + } + } + + mul_mv_mma_store(acc, args, src2, dst, shmem, tile, tiisg, sgitg); +} + +#define MUL_MV_MMA_GEN(tname, bq, nl, deq) \ +template [[host_name("kernel_mul_mv_mma_" tname "_f32_nt1_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_gen<1, 1, bq, nl, deq>; \ +template [[host_name("kernel_mul_mv_mma_" tname "_f32_nt2_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_gen<2, 1, bq, nl, deq>; \ +template [[host_name("kernel_mul_mv_mma_" tname "_f32_nt4_rt1")]] kernel mul_mv_mma_t kernel_mul_mv_mma_gen<4, 1, bq, nl, deq>; \ +template [[host_name("kernel_mul_mv_mma_" tname "_f32_nt1_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_gen<1, 2, bq, nl, deq>; \ +template [[host_name("kernel_mul_mv_mma_" tname "_f32_nt2_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_gen<2, 2, bq, nl, deq>; \ +template [[host_name("kernel_mul_mv_mma_" tname "_f32_nt4_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_gen<4, 2, bq, nl, deq>; + +// q8_0 with 9..16 src1 rows: the per-block scaling above is slower than dequantizing to f32 +template [[host_name("kernel_mul_mv_mma_q8_0_f32_nt1_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_gen<1, 2, block_q8_0, 2, dequantize_q8_0>; +template [[host_name("kernel_mul_mv_mma_q8_0_f32_nt2_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_gen<2, 2, block_q8_0, 2, dequantize_q8_0>; +template [[host_name("kernel_mul_mv_mma_q8_0_f32_nt4_rt2")]] kernel mul_mv_mma_t kernel_mul_mv_mma_gen<4, 2, block_q8_0, 2, dequantize_q8_0>; + +MUL_MV_MMA_GEN("f32", float4x4, 1, dequantize_f32) +MUL_MV_MMA_GEN("f16", half4x4, 1, dequantize_f16) +MUL_MV_MMA_GEN("q4_1", block_q4_1, 2, dequantize_q4_1) +MUL_MV_MMA_GEN("q5_0", block_q5_0, 2, dequantize_q5_0) +MUL_MV_MMA_GEN("q5_1", block_q5_1, 2, dequantize_q5_1) +MUL_MV_MMA_GEN("q4_K", block_q4_K, QK_NL, dequantize_q4_K) +MUL_MV_MMA_GEN("q6_K", block_q6_K, QK_NL, dequantize_q6_K) + +#if defined(GGML_METAL_HAS_BF16) +MUL_MV_MMA_GEN("bf16", bfloat4x4, 1, dequantize_bf16) +#endif +MUL_MV_MMA_GEN("q1_0", block_q1_0, 8, dequantize_q1_0) +MUL_MV_MMA_GEN("q2_0", block_q2_0, 4, dequantize_q2_0) +MUL_MV_MMA_GEN("mxfp4", block_mxfp4, 2, dequantize_mxfp4) +MUL_MV_MMA_GEN("q2_K", block_q2_K, QK_NL, dequantize_q2_K) +MUL_MV_MMA_GEN("q3_K", block_q3_K, QK_NL, dequantize_q3_K) +MUL_MV_MMA_GEN("iq2_xxs", block_iq2_xxs, QK_NL, dequantize_iq2_xxs) +MUL_MV_MMA_GEN("iq2_xs", block_iq2_xs, QK_NL, dequantize_iq2_xs) +MUL_MV_MMA_GEN("iq2_s", block_iq2_s, QK_NL, dequantize_iq2_s) +MUL_MV_MMA_GEN("iq3_xxs", block_iq3_xxs, QK_NL, dequantize_iq3_xxs) +MUL_MV_MMA_GEN("iq3_s", block_iq3_s, QK_NL, dequantize_iq3_s) +MUL_MV_MMA_GEN("iq1_s", block_iq1_s, QK_NL, dequantize_iq1_s) +MUL_MV_MMA_GEN("iq1_m", block_iq1_m, QK_NL, dequantize_iq1_m) +MUL_MV_MMA_GEN("iq4_nl", block_iq4_nl, 2, dequantize_iq4_nl) +MUL_MV_MMA_GEN("iq4_xs", block_iq4_xs, QK_NL, dequantize_iq4_xs) +MUL_MV_MMA_GEN("tq2_0", block_tq2_0, QK_NL, dequantize_tq2_0) + +#undef MUL_MV_MMA_GEN diff --git a/ggml/src/ggml-metal/kernels/norm.metal b/ggml/src/ggml-metal/kernels/norm.metal new file mode 100644 index 0000000..c76d0c7 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/norm.metal @@ -0,0 +1,318 @@ +#include "common.h" + +constant bool FC_norm_use_scale [[function_constant(FC_NORM + 0)]]; + +// F == 1 : norm (no fuse) +// F == 2 : norm + mul +// F == 3 : norm + mul + add +template +kernel void kernel_norm_fuse_impl( + constant ggml_metal_kargs_norm & args, + device const char * src0, + device const char * src1_0, + device const char * src1_1, + device char * dst, + threadgroup float * shmem_f32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + if (sgitg == 0) { + shmem_f32[tiisg] = 0.0f; + } + + const int i01 = tgpig.x; + const int i02 = tgpig.y; + const int i03 = tgpig.z; + + device const T * x = (device const T *) (src0 + i03*args.nbf3[0] + i02*args.nbf2[0] + i01*args.nbf1[0]); + + device const T * f0 = (device const T *) (src1_0 + (i03%args.nef3[1])*args.nbf3[1] + (i02%args.nef2[1])*args.nbf2[1] + (i01%args.nef1[1])*args.nbf1[1]); + device const T * f1 = (device const T *) (src1_1 + (i03%args.nef3[2])*args.nbf3[2] + (i02%args.nef2[2])*args.nbf2[2] + (i01%args.nef1[2])*args.nbf1[2]); + + T sumft(0.0f); + + float sumf = 0.0f; + + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + sumft += x[i00]; + } + sumf = dot(sumft, T(1.0f)); + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_f32[tiisg]; + sumf = simd_sum(sumf); + + const float mean = sumf/args.ne00; + + device T * y = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1); + + sumf = 0.0f; + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + y[i00] = x[i00] - mean; + sumf += dot(y[i00], y[i00]); + } + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_f32[tiisg]; + sumf = simd_sum(sumf); + + const float variance = sumf/args.ne00; + + const float scale = 1.0f/sqrt(variance + args.eps); + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + if (F == 1) { + y[i00] = (y[i00]*scale); + } + if (F == 2) { + if (FC_norm_use_scale) { + y[i00] = (y[i00]*scale) * args.scale; + } else { + y[i00] = (y[i00]*scale)*f0[i00]; + } + } + if (F == 3) { + y[i00] = (y[i00]*scale)*f0[i00] + f1[i00]; + } + } +} + +typedef decltype(kernel_norm_fuse_impl) kernel_norm_fuse_t; + +template [[host_name("kernel_norm_f32")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; +template [[host_name("kernel_norm_mul_f32")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; +template [[host_name("kernel_norm_mul_add_f32")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; + +template [[host_name("kernel_norm_f32_4")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; +template [[host_name("kernel_norm_mul_f32_4")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; +template [[host_name("kernel_norm_mul_add_f32_4")]] kernel kernel_norm_fuse_t kernel_norm_fuse_impl; + +// F == 1 : rms_norm (no fuse) +// F == 2 : rms_norm + mul +// F == 3 : rms_norm + mul + add +template +kernel void kernel_rms_norm_fuse_impl( + constant ggml_metal_kargs_norm & args, + device const char * src0, + device const char * src1_0, + device const char * src1_1, + device char * dst, + threadgroup float * shmem_f32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + if (sgitg == 0) { + shmem_f32[tiisg] = 0.0f; + } + + const int i01 = tgpig.x; + const int i02 = tgpig.y; + const int i03 = tgpig.z; + + device const T * x = (device const T *) (src0 + i03*args.nbf3[0] + i02*args.nbf2[0] + i01*args.nbf1[0]); + + device const T * f0 = (device const T *) (src1_0 + (i03%args.nef3[1])*args.nbf3[1] + (i02%args.nef2[1])*args.nbf2[1] + (i01%args.nef1[1])*args.nbf1[1]); + device const T * f1 = (device const T *) (src1_1 + (i03%args.nef3[2])*args.nbf3[2] + (i02%args.nef2[2])*args.nbf2[2] + (i01%args.nef1[2])*args.nbf1[2]); + + float sumf = 0.0f; + + // parallel sum + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + sumf += dot(x[i00], x[i00]); + } + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_f32[tiisg]; + sumf = simd_sum(sumf); + + const float mean = sumf/args.ne00; + const float scale = 1.0f/sqrt(mean + args.eps); + + device T * y = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1); + for (int i00 = tpitg.x; i00 < args.ne00_t; i00 += ntg.x) { + if (F == 1) { + y[i00] = (x[i00]*scale); + } + if (F == 2) { + if (FC_norm_use_scale) { + y[i00] = (x[i00]*scale) * args.scale; + } else { + y[i00] = (x[i00]*scale)*f0[i00]; + } + } + if (F == 3) { + y[i00] = (x[i00]*scale)*f0[i00] + f1[i00]; + } + } +} + +typedef decltype(kernel_rms_norm_fuse_impl) kernel_rms_norm_fuse_t; + +template [[host_name("kernel_rms_norm_f32")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; +template [[host_name("kernel_rms_norm_mul_f32")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; +template [[host_name("kernel_rms_norm_mul_add_f32")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; + +template [[host_name("kernel_rms_norm_f32_4")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; +template [[host_name("kernel_rms_norm_mul_f32_4")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; +template [[host_name("kernel_rms_norm_mul_add_f32_4")]] kernel kernel_rms_norm_fuse_t kernel_rms_norm_fuse_impl; + +template +kernel void kernel_l2_norm_impl( + constant ggml_metal_kargs_l2_norm & args, + device const char * src0, + device char * dst, + threadgroup float * shmem_f32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int i03 = tgpig.z; + const int i02 = tgpig.y; + const int i01 = tgpig.x; + + if (sgitg == 0) { + shmem_f32[tiisg] = 0.0f; + } + + device const T0 * x = (device const T0 *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); + device T * y = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1); + + float sumf = 0.0f; + + // parallel sum + for (int i00 = tpitg.x; i00 < args.ne00; i00 += ntg.x) { + sumf += dot(x[i00], x[i00]); + } + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_f32[tiisg]; + sumf = simd_sum(sumf); + + const float scale = 1.0f/max(sqrt(sumf), args.eps); + + for (int i00 = tpitg.x; i00 < args.ne00; i00 += ntg.x) { + y[i00] = x[i00] * scale; + } +} + +typedef decltype(kernel_l2_norm_impl) kernel_l2_norm_t; + +template [[host_name("kernel_l2_norm_f32_f32")]] kernel kernel_l2_norm_t kernel_l2_norm_impl; +template [[host_name("kernel_l2_norm_f32_f32_4")]] kernel kernel_l2_norm_t kernel_l2_norm_impl; + +kernel void kernel_group_norm_f32( + constant ggml_metal_kargs_group_norm & args, + device const float * src0, + device float * dst, + threadgroup float * buf [[threadgroup(0)]], + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint ntg[[threads_per_threadgroup]]) { + const int64_t ne = args.ne00*args.ne01*args.ne02; + const int64_t gs = args.ne00*args.ne01*((args.ne02 + args.ngrp - 1) / args.ngrp); + + int start = tgpig * gs; + int end = start + gs; + + start += tpitg; + + if (end >= ne) { + end = ne; + } + + float tmp = 0.0f; // partial sum for thread in warp + + for (int j = start; j < end; j += ntg) { + tmp += src0[j]; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + tmp = simd_sum(tmp); + if (ntg > N_SIMDWIDTH) { + if (sgitg == 0) { + buf[tiisg] = 0.0f; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + buf[sgitg] = tmp; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + tmp = buf[tiisg]; + tmp = simd_sum(tmp); + } + + const float mean = tmp / gs; + tmp = 0.0f; + + for (int j = start; j < end; j += ntg) { + float xi = src0[j] - mean; + dst[j] = xi; + tmp += xi * xi; + } + + tmp = simd_sum(tmp); + if (ntg > N_SIMDWIDTH) { + if (sgitg == 0) { + buf[tiisg] = 0.0f; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + buf[sgitg] = tmp; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + tmp = buf[tiisg]; + tmp = simd_sum(tmp); + } + + const float variance = tmp / gs; + const float scale = 1.0f/sqrt(variance + args.eps); + for (int j = start; j < end; j += ntg) { + dst[j] *= scale; + } +} diff --git a/ggml/src/ggml-metal/kernels/pool.metal b/ggml/src/ggml-metal/kernels/pool.metal new file mode 100644 index 0000000..3dc0261 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/pool.metal @@ -0,0 +1,200 @@ +#include "common.h" + +kernel void kernel_pool_2d_max_f32( + constant ggml_metal_kargs_pool_2d & args, + device const float * src0, + device float * dst, + uint gid[[thread_position_in_grid]]) { + + if (gid >= args.np) { + return; + } + + const int idx = gid; + const int I_HW = args.IH * args.IW; + const int O_HW = args.OH * args.OW; + const int nc = idx / O_HW; + const int cur_oh = idx % O_HW / args.OW; + const int cur_ow = idx % O_HW % args.OW; + + device const float * i_ptr = src0 + nc * I_HW; + device float * o_ptr = dst + nc * O_HW; + + const int start_h = cur_oh * args.s1 - args.p1; + const int bh = MAX(0, start_h); + const int eh = MIN(args.IH, start_h + args.k1); + const int start_w = cur_ow * args.s0 - args.p0; + const int bw = MAX(0, start_w); + const int ew = MIN(args.IW, start_w + args.k0); + + float res = -INFINITY; + + for (int i = bh; i < eh; i += 1) { + for (int j = bw; j < ew; j += 1) { + res = MAX(res, i_ptr[i * args.IW + j]); + } + } + + o_ptr[cur_oh * args.OW + cur_ow] = res; +} + +kernel void kernel_pool_2d_avg_f32( + constant ggml_metal_kargs_pool_2d & args, + device const float * src0, + device float * dst, + uint gid[[thread_position_in_grid]]) { + + if (gid >= args.np) { + return; + } + + const int idx = gid; + const int I_HW = args.IH * args.IW; + const int O_HW = args.OH * args.OW; + const int nc = idx / O_HW; + const int cur_oh = idx % O_HW / args.OW; + const int cur_ow = idx % O_HW % args.OW; + + device const float * i_ptr = src0 + nc * I_HW; + device float * o_ptr = dst + nc * O_HW; + + const int start_h = cur_oh * args.s1 - args.p1; + const int bh = MAX(0, start_h); + const int eh = MIN(args.IH, start_h + args.k1); + const int start_w = cur_ow * args.s0 - args.p0; + const int bw = MAX(0, start_w); + const int ew = MIN(args.IW, start_w + args.k0); + // const float scale = 1. / ((eh - bh) * (ew - bw)); + const float scale = 1. / (args.k0 * args.k1); + + float res = 0; + + for (int i = bh; i < eh; i += 1) { + for (int j = bw; j < ew; j += 1) { + float cur = i_ptr[i * args.IW + j]; + res += cur * scale; + } + } + + o_ptr[cur_oh * args.OW + cur_ow] = res; +} + + +kernel void kernel_pool_1d_max_f32( + constant ggml_metal_kargs_pool_1d & args, + device const float * src, + device float * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint ntg[[threads_per_threadgroup]]) { + if (tgpig >= args.np) { + return; + } + + const int out = (int) tgpig; + const int ow = out % (int) args.OW; + const int row = out / (int) args.OW; + + const int base = ow * args.s0 - args.p0; + const int src_off = row * args.IW; + const int dst_off = row * args.OW; + + float lmax = -INFINITY; + + for (int ki = tpitg; ki < args.k0; ki += ntg) { + const int j = base + ki; + if (j >= 0 && j < args.IW) { + lmax = max(lmax, src[src_off + j]); + } + } + + float max_val = simd_max(lmax); + + threadgroup float * shared_max = (threadgroup float *) shmem; + + if (ntg > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_max[tiisg] = -INFINITY; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shared_max[sgitg] = max_val; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + max_val = shared_max[tiisg]; + max_val = simd_max(max_val); + } + + if (tpitg == 0) { + dst[dst_off + ow] = max_val; + } +} + +kernel void kernel_pool_1d_avg_f32( + constant ggml_metal_kargs_pool_1d & args, + device const float * src, + device float * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint ntg[[threads_per_threadgroup]]) { + if (tgpig >= args.np) { + return; + } + + const int out = (int) tgpig; + const int ow = out % (int) args.OW; + const int row = out / (int) args.OW; + + const int base = ow * args.s0 - args.p0; + const int src_off = row * args.IW; + const int dst_off = row * args.OW; + + float lsum = 0.0f; + int lcnt = 0; + + for (int ki = tpitg; ki < args.k0; ki += ntg) { + const int j = base + ki; + if (j >= 0 && j < args.IW) { + lsum += src[src_off + j]; + lcnt += 1; + } + } + + float sum = simd_sum(lsum); + float cnt = simd_sum((float) lcnt); + + threadgroup float * shared_sum = (threadgroup float *) shmem; + threadgroup float * shared_cnt = (threadgroup float *) shmem + N_SIMDWIDTH; + + if (ntg > N_SIMDWIDTH) { + if (sgitg == 0) { + shared_sum[tiisg] = 0.0f; + shared_cnt[tiisg] = 0.0f; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shared_sum[sgitg] = sum; + shared_cnt[sgitg] = cnt; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + sum = shared_sum[tiisg]; + cnt = shared_cnt[tiisg]; + + sum = simd_sum(sum); + cnt = simd_sum(cnt); + } + + if (tpitg == 0) { + dst[dst_off + ow] = (cnt > 0.0f) ? (sum / cnt) : 0.0f; + } +} diff --git a/ggml/src/ggml-metal/kernels/quantize.h b/ggml/src/ggml-metal/kernels/quantize.h new file mode 100644 index 0000000..0741b22 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/quantize.h @@ -0,0 +1,262 @@ +#pragma once + +#include "common.h" + +void quantize_q1_0(device const float * src, device block_q1_0 & dst) { + float sum_abs = 0.0f; + for (int j = 0; j < QK1_0; j++) { + sum_abs += fabs(src[j]); + } + dst.d = sum_abs / QK1_0; + + for (int j = 0; j < QK1_0 / 8; j++) { + dst.qs[j] = 0; + } + for (int j = 0; j < QK1_0; j++) { + if (src[j] >= 0.0f) { + dst.qs[j / 8] |= (1 << (j % 8)); + } + } +} + +void quantize_q2_0(device const float * src, device block_q2_0 & dst) { + float amax = 0.0f; + for (int j = 0; j < QK2_0; j++) { + float a = fabs(src[j]); + if (a > amax) amax = a; + } + const float d = amax; + dst.d = d; + + const float id = d > 0.0f ? 1.0f / d : 0.0f; + + for (int j = 0; j < QK2_0 / 4; j++) { + dst.qs[j] = 0; + } + for (int j = 0; j < QK2_0; j++) { + int q = (int)round(src[j] * id) + 1; + q = max(0, min(3, q)); + dst.qs[j / 4] |= (q << (2 * (j % 4))); + } +} + +void quantize_q4_0(device const float * src, device block_q4_0 & dst) { +#pragma METAL fp math_mode(safe) + float amax = 0.0f; // absolute max + float max = 0.0f; + + for (int j = 0; j < QK4_0; j++) { + const float v = src[j]; + if (amax < fabs(v)) { + amax = fabs(v); + max = v; + } + } + + const float d = max / -8; + const float id = d ? 1.0f/d : 0.0f; + + dst.d = d; + + for (int j = 0; j < QK4_0/2; ++j) { + const float x0 = src[0 + j]*id; + const float x1 = src[QK4_0/2 + j]*id; + + const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f)); + const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f)); + + dst.qs[j] = xi0; + dst.qs[j] |= xi1 << 4; + } +} + +void quantize_q4_1(device const float * src, device block_q4_1 & dst) { +#pragma METAL fp math_mode(safe) + float min = FLT_MAX; + float max = -FLT_MAX; + + for (int j = 0; j < QK4_1; j++) { + const float v = src[j]; + if (min > v) min = v; + if (max < v) max = v; + } + + const float d = (max - min) / ((1 << 4) - 1); + const float id = d ? 1.0f/d : 0.0f; + + dst.d = d; + dst.m = min; + + for (int j = 0; j < QK4_1/2; ++j) { + const float x0 = (src[0 + j] - min)*id; + const float x1 = (src[QK4_1/2 + j] - min)*id; + + const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f)); + const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f)); + + dst.qs[j] = xi0; + dst.qs[j] |= xi1 << 4; + } +} + +void quantize_q5_0(device const float * src, device block_q5_0 & dst) { +#pragma METAL fp math_mode(safe) + float amax = 0.0f; // absolute max + float max = 0.0f; + + for (int j = 0; j < QK5_0; j++) { + const float v = src[j]; + if (amax < fabs(v)) { + amax = fabs(v); + max = v; + } + } + + const float d = max / -16; + const float id = d ? 1.0f/d : 0.0f; + + dst.d = d; + + uint32_t qh = 0; + for (int j = 0; j < QK5_0/2; ++j) { + const float x0 = src[0 + j]*id; + const float x1 = src[QK5_0/2 + j]*id; + + const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f)); + const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f)); + + dst.qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); + qh |= ((xi0 & 0x10u) >> 4) << (j + 0); + qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2); + } + + thread const uint8_t * qh8 = (thread const uint8_t *)&qh; + + for (int j = 0; j < 4; ++j) { + dst.qh[j] = qh8[j]; + } +} + +void quantize_q5_1(device const float * src, device block_q5_1 & dst) { +#pragma METAL fp math_mode(safe) + float max = src[0]; + float min = src[0]; + + for (int j = 1; j < QK5_1; j++) { + const float v = src[j]; + min = v < min ? v : min; + max = v > max ? v : max; + } + + const float d = (max - min) / 31; + const float id = d ? 1.0f/d : 0.0f; + + dst.d = d; + dst.m = min; + + uint32_t qh = 0; + for (int j = 0; j < QK5_1/2; ++j) { + const float x0 = (src[0 + j] - min)*id; + const float x1 = (src[QK5_1/2 + j] - min)*id; + + const uint8_t xi0 = (uint8_t)(x0 + 0.5f); + const uint8_t xi1 = (uint8_t)(x1 + 0.5f); + + dst.qs[j] = (xi0 & 0xf) | ((xi1 & 0xf) << 4); + qh |= ((xi0 & 0x10u) >> 4) << (j + 0); + qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2); + } + + thread const uint8_t * qh8 = (thread const uint8_t *)&qh; + + for (int j = 0; j < 4; ++j) { + dst.qh[j] = qh8[j]; + } +} + +void quantize_q8_0(device const float * src, device block_q8_0 & dst) { +#pragma METAL fp math_mode(safe) + float amax = 0.0f; // absolute max + + for (int j = 0; j < QK8_0; j++) { + const float v = src[j]; + amax = MAX(amax, fabs(v)); + } + + const float d = amax / ((1 << 7) - 1); + const float id = d ? 1.0f/d : 0.0f; + + dst.d = d; + + for (int j = 0; j < QK8_0; ++j) { + const float x0 = src[j]*id; + + dst.qs[j] = round(x0); + } +} + +void quantize_iq4_nl(device const float * src, device block_iq4_nl & dst) { +#pragma METAL fp math_mode(safe) + float amax = 0.0f; // absolute max + float max = 0.0f; + + for (int j = 0; j < QK4_NL; j++) { + const float v = src[j]; + if (amax < fabs(v)) { + amax = fabs(v); + max = v; + } + } + + const float d = max / kvalues_iq4nl_f[0]; + const float id = d ? 1.0f/d : 0.0f; + + float sumqx = 0, sumq2 = 0; + for (int j = 0; j < QK4_NL/2; ++j) { + const float x0 = src[0 + j]*id; + const float x1 = src[QK4_NL/2 + j]*id; + + const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl_f, x0); + const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl_f, x1); + + dst.qs[j] = xi0 | (xi1 << 4); + + const float v0 = kvalues_iq4nl_f[xi0]; + const float v1 = kvalues_iq4nl_f[xi1]; + const float w0 = src[0 + j]*src[0 + j]; + const float w1 = src[QK4_NL/2 + j]*src[QK4_NL/2 + j]; + sumqx += w0*v0*src[j] + w1*v1*src[QK4_NL/2 + j]; + sumq2 += w0*v0*v0 + w1*v1*v1; + + } + + dst.d = sumq2 > 0 ? sumqx/sumq2 : d; +} + +void quantize_tq2_0(device const float * src, device block_tq2_0 & dst) { +#pragma METAL fp math_mode(safe) + float amax = 0.0f; // absolute max + + for (int j = 0; j < QK_K; j++) { + const float v = src[j]; + amax = MAX(amax, fabs(v)); + } + + const float d = amax; + const float id = d ? 1.0f/d : 0.0f; + + dst.d = (half) d; + + for (int j = 0; j < QK_K/4; j += 32) { + for (int m = 0; m < 32; ++m) { + uint8_t q = 0; + for (int n = 0; n < 4; ++n) { + // -1, 0, 1 -> 0, 1, 2 + int xi = (int)round(src[m + n*32] * id) + 1; + q += (uint8_t)((xi & 3) << (2*n)); + } + dst.qs[j + m] = q; + } + src += 4*32; + } +} diff --git a/ggml/src/ggml-metal/kernels/quantize.metal b/ggml/src/ggml-metal/kernels/quantize.metal new file mode 100644 index 0000000..73679b9 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/quantize.metal @@ -0,0 +1,490 @@ +#include "common.h" +#include "dequantize.h" +#include "quantize.h" + +template +kernel void kernel_cpy_t_t( + constant ggml_metal_kargs_cpy & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int32_t i03 = tgpig[2]; + const int32_t i02 = tgpig[1]; + const int32_t i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tpitg.y; + const int32_t iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; + + if (i01 >= args.ne01) { + return; + } + + const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00; + + const int32_t i3 = n/(args.ne2*args.ne1*args.ne0); + const int32_t i2 = (n - i3*args.ne2*args.ne1*args.ne0)/(args.ne1*args.ne0); + const int32_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0; + const int32_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0); + + device T1 * dst_data = (device T1 *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + for (int32_t i00 = iw0*ntg[0] + tpitg.x; i00 < args.ne00;) { + device const T0 * src = (device T0 *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00); + dst_data[i00] = (T1) src[0]; + break; + } +} + +typedef decltype(kernel_cpy_t_t) kernel_cpy_t; + +template [[host_name("kernel_cpy_f32_f32")]] kernel kernel_cpy_t kernel_cpy_t_t; +template [[host_name("kernel_cpy_f32_f16")]] kernel kernel_cpy_t kernel_cpy_t_t; +template [[host_name("kernel_cpy_f32_i32")]] kernel kernel_cpy_t kernel_cpy_t_t; +template [[host_name("kernel_cpy_i32_f32")]] kernel kernel_cpy_t kernel_cpy_t_t; +template [[host_name("kernel_cpy_i32_i32")]] kernel kernel_cpy_t kernel_cpy_t_t; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_cpy_f32_bf16")]] kernel kernel_cpy_t kernel_cpy_t_t; +#endif +template [[host_name("kernel_cpy_f16_f32")]] kernel kernel_cpy_t kernel_cpy_t_t; +template [[host_name("kernel_cpy_f16_f16")]] kernel kernel_cpy_t kernel_cpy_t_t; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_cpy_bf16_f32")]] kernel kernel_cpy_t kernel_cpy_t_t; +template [[host_name("kernel_cpy_bf16_bf16")]] kernel kernel_cpy_t kernel_cpy_t_t; +#endif + +template +kernel void kernel_cpy_f32_q( + constant ggml_metal_kargs_cpy & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int32_t i03 = tgpig[2]; + const int32_t i02 = tgpig[1]; + const int32_t i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tpitg.y; + const int32_t iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; + + if (i01 >= args.ne01) { + return; + } + + const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00; + + const int32_t i3 = n / (args.ne2*args.ne1*args.ne0); + const int32_t i2 = (n - i3*args.ne2*args.ne1*args.ne0) / (args.ne1*args.ne0); + const int32_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0) / args.ne0; + const int32_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0)/QK; + + device block_q * dst_data = (device block_q *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + for (int32_t i00 = iw0*ntg[0] + tpitg.x; i00 < args.nk0;) { + device const float * src = (device const float *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + (i00*QK)*args.nb00); + + quantize_func(src, dst_data[i00]); + + break; + } +} + +typedef decltype(kernel_cpy_f32_q) cpy_f_q_t; + +template [[host_name("kernel_cpy_f32_q8_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_q1_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_q2_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_q4_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_q4_1")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_q5_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_q5_1")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_iq4_nl")]] kernel cpy_f_q_t kernel_cpy_f32_q; +template [[host_name("kernel_cpy_f32_tq2_0")]] kernel cpy_f_q_t kernel_cpy_f32_q; + +template +kernel void kernel_cpy_q_f32( + constant ggml_metal_kargs_cpy & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int32_t i03 = tgpig[2]; + const int32_t i02 = tgpig[1]; + const int32_t i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tpitg.y; + const int32_t iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0; + + if (i01 >= args.ne01) { + return; + } + + const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00; + + const int32_t i3 = n/(args.ne2*args.ne1*args.ne0); + const int32_t i2 = (n - i3*args.ne2*args.ne1*args.ne0)/(args.ne1*args.ne0); + const int32_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0; + const int32_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0); + + device const block_q * src_data = (device const block_q *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); + device T4x4 * dst_data = (device T4x4 *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + for (int32_t i00 = iw0*ntg[0] + tpitg.x; i00 < args.nk0;) { + T4x4 temp; + dequantize_func(src_data + i00/nl, i00%nl, temp); + dst_data[i00] = temp; + + break; + } +} + +typedef decltype(kernel_cpy_q_f32) cpy_q_f_t; + +template [[host_name("kernel_cpy_q1_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q2_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q4_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q4_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q5_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q5_1_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q8_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; + +template [[host_name("kernel_cpy_tq2_0_f32")]] kernel cpy_q_f_t kernel_cpy_q_f32; + +template [[host_name("kernel_cpy_q1_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q4_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q4_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q5_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q5_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template [[host_name("kernel_cpy_q8_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; + +template [[host_name("kernel_cpy_tq2_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; + +template +kernel void kernel_concat( + constant ggml_metal_kargs_concat & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + + const int i3 = tgpig.z; + const int i2 = tgpig.y; + const int i1 = ntg.y == 1 ? tgpig.x/args.nc0 : tgpig.x*ntg.y + tpitg.y; + const int ic = ntg.y == 1 ? tgpig.x%args.nc0 : 0; + + if (i1 >= args.ne1) { + return; + } + + int o[4] = {0, 0, 0, 0}; + o[args.dim] = args.dim == 0 ? args.ne00 : (args.dim == 1 ? args.ne01 : (args.dim == 2 ? args.ne02 : args.ne03)); + + // chunk ic of nc0 along the row + const int n0 = (args.ne0 + args.nc0 - 1)/args.nc0; + const int i0e = min(args.ne0, (ic + 1)*n0); + + for (int i0 = ic*n0 + tpitg.x; i0 < i0e; i0 += ntg.x) { + device const T * x; + + if (i0 < args.ne00 && i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) { + x = (device const T *)(src0 + (i3 )*args.nb03 + (i2 )*args.nb02 + (i1 )*args.nb01 + (i0 )*args.nb00); + } else { + x = (device const T *)(src1 + (i3 - o[3])*args.nb13 + (i2 - o[2])*args.nb12 + (i1 - o[1])*args.nb11 + (i0 - o[0])*args.nb10); + } + + device T * y = (device T *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + *y = *x; + } +} + +typedef decltype(kernel_concat) kernel_concat_t; + +template [[host_name("kernel_concat_f32")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_f16")]] kernel kernel_concat_t kernel_concat; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_concat_bf16")]] kernel kernel_concat_t kernel_concat; +#endif +template [[host_name("kernel_concat_i8")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_i16")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_i32")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_i64")]] kernel kernel_concat_t kernel_concat; + +template +kernel void kernel_concat_q( + constant ggml_metal_kargs_concat & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + + // note: for quantized types, the args are in units of blocks (nb0 == type_size) + const int i3 = tgpig.z; + const int i2 = tgpig.y; + const int i1 = ntg.y == 1 ? tgpig.x/args.nc0 : tgpig.x*ntg.y + tpitg.y; + const int ic = ntg.y == 1 ? tgpig.x%args.nc0 : 0; + + if (i1 >= args.ne1) { + return; + } + + int o[4] = {0, 0, 0, 0}; + o[args.dim] = args.dim == 0 ? args.ne00 : (args.dim == 1 ? args.ne01 : (args.dim == 2 ? args.ne02 : args.ne03)); + + // chunk ic of nc0 along the row + const int n0 = (args.ne0 + args.nc0 - 1)/args.nc0; + const int i0e = min(args.ne0, (ic + 1)*n0); + + for (int i0 = ic*n0 + tpitg.x; i0 < i0e; i0 += ntg.x) { + device const block_q * x; + + if (i0 < args.ne00 && i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) { + x = (device const block_q *)(src0 + (i3 )*args.nb03 + (i2 )*args.nb02 + (i1 )*args.nb01 + (i0 )*args.nb00); + } else { + x = (device const block_q *)(src1 + (i3 - o[3])*args.nb13 + (i2 - o[2])*args.nb12 + (i1 - o[1])*args.nb11 + (i0 - o[0])*args.nb10); + } + + device block_q * y = (device block_q *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + *y = *x; + } +} + +typedef decltype(kernel_concat_q) kernel_concat_q_t; + +template [[host_name("kernel_concat_q4_0")]] kernel kernel_concat_q_t kernel_concat_q; +template [[host_name("kernel_concat_q4_1")]] kernel kernel_concat_q_t kernel_concat_q; +template [[host_name("kernel_concat_q5_0")]] kernel kernel_concat_q_t kernel_concat_q; +template [[host_name("kernel_concat_q5_1")]] kernel kernel_concat_q_t kernel_concat_q; +template [[host_name("kernel_concat_q8_0")]] kernel kernel_concat_q_t kernel_concat_q; + +template +kernel void kernel_get_rows_q( + constant ggml_metal_kargs_get_rows & args, + device const void * src0, + device const void * src1, + device void * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 ntg [[threads_per_threadgroup]]) { + const int32_t iw0 = tgpig.x/args.ne10; + const int32_t i10 = tgpig.x%args.ne10; + const int32_t i11 = tgpig.y; + const int32_t i12 = tgpig.z; + + const int32_t r = ((const device int32_t *) ((const device char *) src1 + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0]; + + const int32_t i02 = i11; + const int32_t i03 = i12; + + auto psrc = (device const block_q *) ((const device char *) src0 + i03*args.nb03 + i02*args.nb02 + r*args.nb01); + auto pdst = (device float4x4 *) (( device char *) dst + i12*args.nb3 + i11*args.nb2 + i10*args.nb1); + + for (int ind = iw0*ntg.x + tiitg; ind < args.ne00t;) { + float4x4 temp; + dequantize_func(psrc + ind/nl, ind%nl, temp); + pdst[ind] = temp; + + break; + } +} + +template +kernel void kernel_get_rows_f( + constant ggml_metal_kargs_get_rows & args, + device const void * src0, + device const void * src1, + device void * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 ntg [[threads_per_threadgroup]]) { + const int32_t iw0 = tgpig.x/args.ne10; + const int32_t i10 = tgpig.x%args.ne10; + const int32_t i11 = tgpig.y; + const int32_t i12 = tgpig.z; + + const int32_t r = ((const device int32_t *) ((const device char *) src1 + i12*args.nb12 + i11*args.nb11 + i10*args.nb10))[0]; + + const int32_t i02 = i11; + const int32_t i03 = i12; + + auto psrc = (const device T0 *) ((const device char *) src0 + i03*args.nb03 + i02*args.nb02 + r*args.nb01); + auto pdst = ( device T *) (( device char *) dst + i12*args.nb3 + i11*args.nb2 + i10*args.nb1); + + for (int ind = iw0*ntg.x + tiitg; ind < args.ne00t;) { + pdst[ind] = psrc[ind]; + + break; + } +} + +typedef decltype(kernel_get_rows_f) get_rows_f_t; + +template [[host_name("kernel_get_rows_f32")]] kernel get_rows_f_t kernel_get_rows_f; +template [[host_name("kernel_get_rows_f16")]] kernel get_rows_f_t kernel_get_rows_f; +template [[host_name("kernel_get_rows_i32")]] kernel get_rows_f_t kernel_get_rows_f; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_get_rows_bf16")]] kernel get_rows_f_t kernel_get_rows_f; +#endif + +typedef decltype(kernel_get_rows_q) get_rows_q_t; + +template [[host_name("kernel_get_rows_q1_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q2_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q4_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q4_1")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q5_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q5_1")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q8_0")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_mxfp4")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q2_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q3_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q4_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q5_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_q6_K")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq2_xxs")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq2_xs")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq3_xxs")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq3_s")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq1_m")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_q_t kernel_get_rows_q; +template [[host_name("kernel_get_rows_tq2_0")]] kernel get_rows_q_t kernel_get_rows_q; + +template +kernel void kernel_set_rows_q( + constant ggml_metal_kargs_set_rows & args, + device const void * src0, + device const void * src1, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiitg[[thread_index_in_threadgroup]], + uint3 tptg [[threads_per_threadgroup]]) { + const int32_t i03 = tgpig.z; + const int32_t i02 = tgpig.y; + + const int32_t i12 = i03%args.ne12; + const int32_t i11 = i02%args.ne11; + + const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x; + if (i01 >= args.ne01) { + return; + } + + const int32_t i10 = i01; + const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0]; + + device block_q * dst_row = ( device block_q *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3); + const device TS * src_row = (const device TS *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); + + for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) { + quantize_func(src_row + QK*ind, dst_row[ind]); + } +} + +template +kernel void kernel_set_rows_q32( + constant ggml_metal_kargs_set_rows & args, + device const void * src0, + device const void * src1, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiitg[[thread_index_in_threadgroup]], + uint3 tptg [[threads_per_threadgroup]]) { + const int32_t i03 = tgpig.z; + const int32_t i02 = tgpig.y; + + const int32_t i12 = i03%args.ne12; + const int32_t i11 = i02%args.ne11; + + const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x; + if (i01 >= args.ne01) { + return; + } + + const int32_t i10 = i01; + const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0]; + + device block_q * dst_row = ( device block_q *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3); + const device TS * src_row = (const device TS *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); + + for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) { + quantize_func(src_row + 32*ind, dst_row[ind]); + } +} + +template +kernel void kernel_set_rows_f( + constant ggml_metal_kargs_set_rows & args, + device const void * src0, + device const void * src1, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint tiitg[[thread_index_in_threadgroup]], + uint3 tptg [[threads_per_threadgroup]]) { + const int32_t i03 = tgpig.z; + const int32_t i02 = tgpig.y; + + const int32_t i12 = i03%args.ne12; + const int32_t i11 = i02%args.ne11; + + const int32_t i01 = tgpig.x*tptg.y + tiitg/tptg.x; + if (i01 >= args.ne01) { + return; + } + + const int32_t i10 = i01; + const TI i1 = ((const device TI *) ((const device char *) src1 + i10*args.nb10 + i11*args.nb11 + i12*args.nb12))[0]; + + device TD * dst_row = ( device TD *) (( device char *) dst + i1*args.nb1 + i02*args.nb2 + i03*args.nb3); + const device TS * src_row = (const device TS *) ((const device char *) src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); + + for (int ind = tiitg%tptg.x; ind < args.nk0; ind += tptg.x) { + dst_row[ind] = (TD) src_row[ind]; + } +} + +typedef decltype(kernel_set_rows_f) set_rows_f_t; + +template [[host_name("kernel_set_rows_f32_i64_f32")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f32_i32_f32")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f32_i64_f16")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f32_i32_f16")]] kernel set_rows_f_t kernel_set_rows_f; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_set_rows_f32_i64_bf16")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f32_i32_bf16")]] kernel set_rows_f_t kernel_set_rows_f; +#endif + +template [[host_name("kernel_set_rows_f16_i64_f16")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_f16_i32_f16")]] kernel set_rows_f_t kernel_set_rows_f; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_set_rows_bf16_i64_bf16")]] kernel set_rows_f_t kernel_set_rows_f; +template [[host_name("kernel_set_rows_bf16_i32_bf16")]] kernel set_rows_f_t kernel_set_rows_f; +#endif + +typedef decltype(kernel_set_rows_q32) set_rows_q32_t; + +template [[host_name("kernel_set_rows_f32_i64_q8_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q8_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_q4_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q4_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_q4_1")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q4_1")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_q5_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q5_0")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_q5_1")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_q5_1")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i64_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32; +template [[host_name("kernel_set_rows_f32_i32_iq4_nl")]] kernel set_rows_q32_t kernel_set_rows_q32; + +typedef decltype(kernel_set_rows_q) set_rows_qK_t; + +template [[host_name("kernel_set_rows_f32_i64_tq2_0")]] kernel set_rows_qK_t kernel_set_rows_q; +template [[host_name("kernel_set_rows_f32_i32_tq2_0")]] kernel set_rows_qK_t kernel_set_rows_q; + diff --git a/ggml/src/ggml-metal/kernels/reduce.metal b/ggml/src/ggml-metal/kernels/reduce.metal new file mode 100644 index 0000000..0af9e4f --- /dev/null +++ b/ggml/src/ggml-metal/kernels/reduce.metal @@ -0,0 +1,228 @@ +#include "common.h" + +kernel void kernel_op_sum_f32( + constant ggml_metal_kargs_sum & args, + device const float * src0, + device float * dst, + threadgroup float * shmem_f32 [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + + if (args.np == 0) { + return; + } + + // TODO: become function constant + const uint nsg = (ntg.x + 31) / 32; + + float sumf = 0; + + for (uint64_t i0 = tpitg.x; i0 < args.np; i0 += ntg.x) { + sumf += src0[i0]; + } + + sumf = simd_sum(sumf); + + if (tiisg == 0) { + shmem_f32[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + float total = 0; + + if (sgitg == 0) { + float v = 0; + + if (tpitg.x < nsg) { + v = shmem_f32[tpitg.x]; + } + + total = simd_sum(v); + + if (tpitg.x == 0) { + dst[0] = total; + } + } +} + +constant short FC_sum_rows_op [[function_constant(FC_SUM_ROWS + 0)]]; + +template +kernel void kernel_sum_rows_impl( + constant ggml_metal_kargs_sum_rows & args, + device const char * src0, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { +#define FC_OP FC_sum_rows_op + + const int i3 = tgpig.z; + const int i2 = tgpig.y; + const int i1 = tgpig.x; + + threadgroup T0 * shmem_t = (threadgroup T0 *) shmem; + + if (sgitg == 0) { + shmem_t[tiisg] = 0.0f; + } + + device const T0 * src_row = (device const T0 *) (src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03); + device T * dst_row = (device T *) (dst + i1*args.nb1 + i2*args.nb2 + i3*args.nb3); + + T0 sumf = T0(0.0f); + + for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) { + sumf += src_row[i0]; + } + + sumf = simd_sum(sumf); + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + shmem_t[sgitg] = sumf; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sumf = shmem_t[tiisg]; + sumf = simd_sum(sumf); + + if (tpitg.x == 0) { + if (FC_OP == OP_SUM_ROWS_NUM_MEAN) { + if (is_same::value) { + dst_row[0] = sum(sumf) / (4*args.ne00); + } else { + dst_row[0] = sum(sumf) / args.ne00; + } + } else { + dst_row[0] = sum(sumf); + } + } + +#undef FC_OP +} + +typedef decltype(kernel_sum_rows_impl) kernel_sum_rows_t; + +template [[host_name("kernel_sum_rows_f32_f32")]] kernel kernel_sum_rows_t kernel_sum_rows_impl; +template [[host_name("kernel_sum_rows_f32_f32_4")]] kernel kernel_sum_rows_t kernel_sum_rows_impl; + +template +kernel void kernel_cumsum_blk( + constant ggml_metal_kargs_cumsum_blk & args, + device const char * src0, + device char * tmp, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int ib = tgpig[0]/args.ne01; + + const int i00 = ib*ntg.x; + const int i01 = tgpig[0]%args.ne01; + const int i02 = tgpig[1]; + const int i03 = tgpig[2]; + + device const float * src0_row = (device const float *) (src0 + + args.nb01*i01 + + args.nb02*i02 + + args.nb03*i03); + + threadgroup float * shmem_f32 = (threadgroup float *) shmem; + + float v = 0.0f; + + if (i00 + tpitg.x < args.ne00) { + v = src0_row[i00 + tpitg.x]; + } + + float s = simd_prefix_inclusive_sum(v); + + if (tiisg == N_SIMDWIDTH - 1) { + shmem_f32[sgitg] = s; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (sgitg == 0) { + shmem_f32[tiisg] = simd_prefix_exclusive_sum(shmem_f32[tiisg]); + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + s += shmem_f32[sgitg]; + + device float * dst_row = (device float *) dst + + args.ne00*i01 + + args.ne00*args.ne01*i02 + + args.ne00*args.ne01*args.ne02*i03; + + if (i00 + tpitg.x < args.ne00) { + dst_row[i00 + tpitg.x] = s; + } + + if (args.outb && tpitg.x == ntg.x - 1) { + device float * tmp_row = (device float *) tmp + + args.net0*i01 + + args.net0*args.net1*i02 + + args.net0*args.net1*args.net2*i03; + + tmp_row[ib] = s; + } +} + +typedef decltype(kernel_cumsum_blk) kernel_cumsum_blk_t; + +template [[host_name("kernel_cumsum_blk_f32")]] kernel kernel_cumsum_blk_t kernel_cumsum_blk; + +template +kernel void kernel_cumsum_add( + constant ggml_metal_kargs_cumsum_add & args, + device const char * tmp, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int ib = tgpig[0]/args.ne01; + + if (ib == 0) { + return; + } + + const int i00 = ib*ntg.x; + const int i01 = tgpig[0]%args.ne01; + const int i02 = tgpig[1]; + const int i03 = tgpig[2]; + + device const float * tmp_row = (device const float *) (tmp + + args.nbt1*i01 + + args.nbt2*i02 + + args.nbt3*i03); + + device float * dst_row = (device float *) dst + + args.ne00*i01 + + args.ne00*args.ne01*i02 + + args.ne00*args.ne01*args.ne02*i03; + + if (i00 + tpitg.x < args.ne00) { + dst_row[i00 + tpitg.x] += tmp_row[ib - 1]; + } +} + +typedef decltype(kernel_cumsum_add) kernel_cumsum_add_t; + +template [[host_name("kernel_cumsum_add_f32")]] kernel kernel_cumsum_add_t kernel_cumsum_add; diff --git a/ggml/src/ggml-metal/kernels/rope.metal b/ggml/src/ggml-metal/kernels/rope.metal new file mode 100644 index 0000000..401ceac --- /dev/null +++ b/ggml/src/ggml-metal/kernels/rope.metal @@ -0,0 +1,333 @@ +#include "common.h" + +constant bool FC_rope_is_imrope [[function_constant(FC_ROPE + 0)]]; +constant bool FC_rope_is_back [[function_constant(FC_ROPE + 1)]]; + +static float rope_yarn_ramp(const float low, const float high, const int i0) { + const float y = (i0 / 2 - low) / max(0.001f, high - low); + return 1.0f - min(1.0f, max(0.0f, y)); +} + +// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn +// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng. +static void rope_yarn( + float theta_extrap, float freq_scale, float corr_dims[2], int i0, float ext_factor, float mscale, + thread float * cos_theta, thread float * sin_theta) { + // Get n-d rotational scaling corrected for extrapolation + float theta_interp = freq_scale * theta_extrap; + float theta = theta_interp; + if (ext_factor != 0.0f) { + float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor; + theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix; + + // Get n-d magnitude scaling corrected for interpolation + mscale *= 1.0f + 0.1f * log(1.0f / freq_scale); + } + *cos_theta = cos(theta) * mscale; + *sin_theta = sin(theta) * mscale; + if (FC_rope_is_back) { + *sin_theta *= -1.0f; + } +} + +// Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get +// `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))` +static float rope_yarn_corr_factor(int n_dims, int n_ctx_orig, float n_rot, float base) { + return n_dims * log(n_ctx_orig / (n_rot * 2 * M_PI_F)) / (2 * log(base)); +} + +static void rope_yarn_corr_dims( + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2] +) { + // start and end correction dims + dims[0] = max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_fast, freq_base))); + dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_slow, freq_base))); +} + +template +kernel void kernel_rope_norm( + constant ggml_metal_kargs_rope & args, + device const char * src0, + device const char * src1, + device const char * src2, + device char * dst, + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 tptg [[threads_per_threadgroup]], + uint3 tgpig[[threadgroup_position_in_grid]]) { + const int i3 = tgpig[2]; + const int i2 = tgpig[1]; + const int i1 = tgpig[0]; + + float corr_dims[2]; + rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims); + + device const int32_t * pos = (device const int32_t *) src1; + + const float theta_base = (float) pos[i2]; + const float inv_ndims = -1.f/args.n_dims; + + float cos_theta; + float sin_theta; + + for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) { + if (i0 >= args.n_offs && i0 < args.n_offs + args.n_dims) { + const int iw = i0 - args.n_offs; // relative idx + const int ic = iw/2; + + const float theta = theta_base * pow(args.freq_base, inv_ndims*iw); + + const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f; + + rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, iw, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta); + + device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); + device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + const float x0 = src[0]; + const float x1 = src[1]; + + dst_data[0] = x0*cos_theta - x1*sin_theta; + dst_data[1] = x0*sin_theta + x1*cos_theta; + } else { + if (args.inplace) { + continue; + } + + device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); + device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + dst_data[0] = src[0]; + dst_data[1] = src[1]; + } + } +} + +template +kernel void kernel_rope_neox( + constant ggml_metal_kargs_rope & args, + device const char * src0, + device const char * src1, + device const char * src2, + device char * dst, + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 tptg [[threads_per_threadgroup]], + uint3 tgpig[[threadgroup_position_in_grid]]) { + const int i3 = tgpig[2]; + const int i2 = tgpig[1]; + const int i1 = tgpig[0]; + + float corr_dims[2]; + rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims); + + device const int32_t * pos = (device const int32_t *) src1; + + const float theta_base = (float) pos[i2]; + const float inv_ndims = -1.f/args.n_dims; + + float cos_theta; + float sin_theta; + + for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) { + if (i0 >= args.n_offs && i0 < args.n_offs + args.n_dims) { + const int iw = i0 - args.n_offs; // relative idx + const int ic = iw/2; + + const float theta = theta_base * pow(args.freq_base, inv_ndims*iw); + + const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f; + + rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, iw, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta); + + device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + (args.n_offs + ic)*args.nb00); + device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + (args.n_offs + ic)*args.nb0); + + const float x0 = src[0]; + const float x1 = src[args.n_dims/2]; + + dst_data[0] = x0*cos_theta - x1*sin_theta; + dst_data[args.n_dims/2] = x0*sin_theta + x1*cos_theta; + } else { + if (args.inplace) { + continue; + } + + device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); + device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + dst_data[0] = src[0]; + dst_data[1] = src[1]; + } + } +} + +template +kernel void kernel_rope_multi( + constant ggml_metal_kargs_rope & args, + device const char * src0, + device const char * src1, + device const char * src2, + device char * dst, + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 tptg [[threads_per_threadgroup]], + uint3 tgpig[[threadgroup_position_in_grid]]) { + const int i3 = tgpig[2]; + const int i2 = tgpig[1]; + const int i1 = tgpig[0]; + + float corr_dims[2]; + rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims); + + device const int32_t * pos = (device const int32_t *) src1; + + const float inv_ndims = -1.f/args.n_dims; + + float cos_theta; + float sin_theta; + + for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) { + if (i0 >= args.n_offs && i0 < args.n_offs + args.n_dims) { + const int iw = i0 - args.n_offs; // relative idx + const int ic = iw/2; + + // mrope theta calculations + // note: the rest is the same as kernel_rope_neox + const int sect_dims = args.sect_0 + args.sect_1 + args.sect_2 + args.sect_3; + const int sec_w01 = args.sect_0 + args.sect_1; // end of section 1 + const int sec_w012 = args.sect_0 + args.sect_1 + args.sect_2; // end of section 2 + const int sector = ic % sect_dims; + + float theta_base; + if (FC_rope_is_imrope) { + if (sector % 3 == 1 && sector < 3 * args.sect_1) { // h + theta_base = (float) pos[i2 + args.ne02 * 1]; + } else if (sector % 3 == 2 && sector < 3 * args.sect_2) { // w + theta_base = (float) pos[i2 + args.ne02 * 2]; + } else if (sector % 3 == 0 && sector < 3 * args.sect_0) { // t + theta_base = (float) pos[i2 + args.ne02 * 0]; + } else { // e + theta_base = (float) pos[i2 + args.ne02 * 3]; + } + } else { + if (sector < args.sect_0) { + theta_base = (float) pos[i2]; + } else if (sector < sec_w01) { + theta_base = (float) pos[i2 + args.ne02 * 1]; + } else if (sector < sec_w012) { + theta_base = (float) pos[i2 + args.ne02 * 2]; + } else { + theta_base = (float) pos[i2 + args.ne02 * 3]; + } + } + // end of mrope + + const float theta = theta_base * pow(args.freq_base, inv_ndims*iw); + + const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f; + + rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, iw, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta); + + device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + (args.n_offs + ic)*args.nb00); + device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + (args.n_offs + ic)*args.nb0); + + const float x0 = src[0]; + const float x1 = src[args.n_dims/2]; + + dst_data[0] = x0*cos_theta - x1*sin_theta; + dst_data[args.n_dims/2] = x0*sin_theta + x1*cos_theta; + } else { + if (args.inplace) { + continue; + } + + device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); + device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + dst_data[0] = src[0]; + dst_data[1] = src[1]; + } + } +} + +template +kernel void kernel_rope_vision( + constant ggml_metal_kargs_rope & args, + device const char * src0, + device const char * src1, + device const char * src2, + device char * dst, + ushort tiitg[[thread_index_in_threadgroup]], + ushort3 tptg [[threads_per_threadgroup]], + uint3 tgpig[[threadgroup_position_in_grid]]) { + const int i3 = tgpig[2]; + const int i2 = tgpig[1]; + const int i1 = tgpig[0]; + + float corr_dims[2]; + rope_yarn_corr_dims(args.n_dims, args.n_ctx_orig, args.freq_base, args.beta_fast, args.beta_slow, corr_dims); + + device const int32_t * pos = (device const int32_t *) src1; + + const float inv_ndims = -1.f/args.n_dims; + + float cos_theta; + float sin_theta; + + for (int i0 = 2*tiitg; i0 < args.ne0; i0 += 2*tptg.x) { + if (i0 < 2*args.n_dims) { // different from kernel_rope_multi + const int ic = i0/2; + + // mrope theta calculations (only support 2 dimensions) + const int sect_dims = args.sect_0 + args.sect_1; + const int sector = ic % sect_dims; + + float p; + float theta_base; + if (sector < args.sect_1) { + p = (float) sector; + theta_base = (float) pos[i2]; + } else { + p = (float) sector - args.sect_0; + theta_base = (float) pos[i2 + args.ne02]; + } + + const float theta = theta_base * pow(args.freq_base, 2.0f * inv_ndims * p); + // end of mrope + + const float freq_factor = args.src2 ? ((device const float *) src2)[ic] : 1.0f; + + rope_yarn(theta/freq_factor, args.freq_scale, corr_dims, i0, args.ext_factor, args.attn_factor, &cos_theta, &sin_theta); + + device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + ic*args.nb00); + device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + ic*args.nb0); + + const float x0 = src[0]; + const float x1 = src[args.n_dims]; // different from kernel_rope_multi + + dst_data[0] = x0*cos_theta - x1*sin_theta; + dst_data[args.n_dims] = x0*sin_theta + x1*cos_theta; // different from kernel_rope_multi + } else { + device const T * const src = (device T *)(src0 + i3*args.nb03 + i2*args.nb02 + i1*args.nb01 + i0*args.nb00); + device T * dst_data = (device T *)( dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + dst_data[0] = src[0]; + dst_data[1] = src[1]; + } + } +} + +typedef decltype(kernel_rope_norm) kernel_rope_norm_t; +typedef decltype(kernel_rope_neox) kernel_rope_neox_t; +typedef decltype(kernel_rope_multi) kernel_rope_multi_t; +typedef decltype(kernel_rope_vision) kernel_rope_vision_t; + +template [[host_name("kernel_rope_norm_f32")]] kernel kernel_rope_norm_t kernel_rope_norm; +template [[host_name("kernel_rope_norm_f16")]] kernel kernel_rope_norm_t kernel_rope_norm; + +template [[host_name("kernel_rope_neox_f32")]] kernel kernel_rope_neox_t kernel_rope_neox; +template [[host_name("kernel_rope_neox_f16")]] kernel kernel_rope_neox_t kernel_rope_neox; + +template [[host_name("kernel_rope_multi_f32")]] kernel kernel_rope_multi_t kernel_rope_multi; +template [[host_name("kernel_rope_multi_f16")]] kernel kernel_rope_multi_t kernel_rope_multi; + +template [[host_name("kernel_rope_vision_f32")]] kernel kernel_rope_vision_t kernel_rope_vision; +template [[host_name("kernel_rope_vision_f16")]] kernel kernel_rope_vision_t kernel_rope_vision; diff --git a/ggml/src/ggml-metal/kernels/softmax.metal b/ggml/src/ggml-metal/kernels/softmax.metal new file mode 100644 index 0000000..f32fe29 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/softmax.metal @@ -0,0 +1,223 @@ +#include "common.h" + +template +kernel void kernel_soft_max( + constant ggml_metal_kargs_soft_max & args, + device const char * src0, + device const char * src1, + device const char * src2, + device char * dst, + threadgroup float * buf [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint3 tptg[[threads_per_threadgroup]]) { + const int32_t i03 = tgpig.z; + const int32_t i02 = tgpig.y; + const int32_t i01 = tgpig.x; + + const int32_t i13 = i03%args.ne13; + const int32_t i12 = i02%args.ne12; + const int32_t i11 = i01; + + device const float * psrc0 = (device const float *) (src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); + device const T * pmask = src1 != src0 ? (device const T * ) (src1 + i11*args.nb11 + i12*args.nb12 + i13*args.nb13) : nullptr; + device const float * psrc2 = src2 != src0 ? (device const float *) (src2) : nullptr; + device float * pdst = (device float *) (dst + i01*args.nb1 + i02*args.nb2 + i03*args.nb3); + + float slope = 1.0f; + + // ALiBi + if (args.max_bias > 0.0f) { + const int32_t h = i02; + + const float base = h < args.n_head_log2 ? args.m0 : args.m1; + const int exp = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; + + slope = pow(base, exp); + } + + // parallel max + float lmax = psrc2 ? psrc2[i02] : -INFINITY; + + for (int i00 = tpitg.x; i00 < args.ne00; i00 += tptg.x) { + lmax = MAX(lmax, psrc0[i00]*args.scale + (pmask ? slope*pmask[i00] : 0.0f)); + } + + // find the max value in the block + float max_val = simd_max(lmax); + if (tptg.x > N_SIMDWIDTH) { + if (sgitg == 0) { + buf[tiisg] = -INFINITY; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + buf[sgitg] = max_val; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + max_val = buf[tiisg]; + max_val = simd_max(max_val); + } + + // parallel sum + float lsum = 0.0f; + for (int i00 = tpitg.x; i00 < args.ne00; i00 += tptg.x) { + const float exp_psrc0 = exp((psrc0[i00]*args.scale + (pmask ? slope*pmask[i00] : 0.0f)) - max_val); + lsum += exp_psrc0; + pdst[i00] = exp_psrc0; + } + + // This barrier fixes a failing test + // ref: https://github.com/ggml-org/ggml/pull/621#discussion_r1425156335 + threadgroup_barrier(mem_flags::mem_none); + + float sum = simd_sum(lsum); + + if (tptg.x > N_SIMDWIDTH) { + if (sgitg == 0) { + buf[tiisg] = 0.0f; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + buf[sgitg] = sum; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sum = buf[tiisg]; + sum = simd_sum(sum); + } + + if (psrc2) { + sum += exp(psrc2[i02] - max_val); + } + + const float inv_sum = 1.0f/sum; + + for (int i00 = tpitg.x; i00 < args.ne00; i00 += tptg.x) { + pdst[i00] *= inv_sum; + } +} + +template +kernel void kernel_soft_max_4( + constant ggml_metal_kargs_soft_max & args, + device const char * src0, + device const char * src1, + device const char * src2, + device char * dst, + threadgroup float * buf [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint sgitg[[simdgroup_index_in_threadgroup]], + uint tiisg[[thread_index_in_simdgroup]], + uint3 tptg[[threads_per_threadgroup]]) { + const int32_t i03 = tgpig.z; + const int32_t i02 = tgpig.y; + const int32_t i01 = tgpig.x; + + const int32_t i13 = i03%args.ne13; + const int32_t i12 = i02%args.ne12; + const int32_t i11 = i01; + + device const float4 * psrc4 = (device const float4 *) (src0 + i01*args.nb01 + i02*args.nb02 + i03*args.nb03); + device const T * pmask = src1 != src0 ? (device const T * ) (src1 + i11*args.nb11 + i12*args.nb12 + i13*args.nb13) : nullptr; + device const float * psrc2 = src2 != src0 ? (device const float * ) (src2) : nullptr; + device float4 * pdst4 = (device float4 *) (dst + i01*args.nb1 + i02*args.nb2 + i03*args.nb3); + + float slope = 1.0f; + + if (args.max_bias > 0.0f) { + const int32_t h = i02; + + const float base = h < args.n_head_log2 ? args.m0 : args.m1; + const int exp = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1; + + slope = pow(base, exp); + } + + // parallel max + float4 lmax4 = psrc2 ? psrc2[i02] : -INFINITY; + + for (int i00 = tpitg.x; i00 < args.ne00/4; i00 += tptg.x) { + lmax4 = fmax(lmax4, psrc4[i00]*args.scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))); + } + + const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3])); + + float max_val = simd_max(lmax); + if (tptg.x > N_SIMDWIDTH) { + if (sgitg == 0) { + buf[tiisg] = -INFINITY; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + buf[sgitg] = max_val; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + max_val = buf[tiisg]; + max_val = simd_max(max_val); + } + + // parallel sum + float4 lsum4 = 0.0f; + for (int i00 = tpitg.x; i00 < args.ne00/4; i00 += tptg.x) { + const float4 exp_psrc4 = exp((psrc4[i00]*args.scale + (float4)((pmask ? slope*pmask[i00] : 0.0f))) - max_val); + lsum4 += exp_psrc4; + pdst4[i00] = exp_psrc4; + } + + const float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3]; + + // This barrier fixes a failing test + // ref: https://github.com/ggml-org/ggml/pull/621#discussion_r1425156335 + threadgroup_barrier(mem_flags::mem_none); + + float sum = simd_sum(lsum); + + if (tptg.x > N_SIMDWIDTH) { + if (sgitg == 0) { + buf[tiisg] = 0.0f; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiisg == 0) { + buf[sgitg] = sum; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + sum = buf[tiisg]; + sum = simd_sum(sum); + } + + if (psrc2) { + sum += exp(psrc2[i02] - max_val); + } + + const float inv_sum = 1.0f/sum; + + for (int i00 = tpitg.x; i00 < args.ne00/4; i00 += tptg.x) { + pdst4[i00] *= inv_sum; + } +} + +typedef decltype(kernel_soft_max) kernel_soft_max_t; +typedef decltype(kernel_soft_max_4) kernel_soft_max_4_t; + +template [[host_name("kernel_soft_max_f16")]] kernel kernel_soft_max_t kernel_soft_max; +template [[host_name("kernel_soft_max_f32")]] kernel kernel_soft_max_t kernel_soft_max; +template [[host_name("kernel_soft_max_f16_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4; +template [[host_name("kernel_soft_max_f32_4")]] kernel kernel_soft_max_4_t kernel_soft_max_4; diff --git a/ggml/src/ggml-metal/kernels/solve_tri.metal b/ggml/src/ggml-metal/kernels/solve_tri.metal new file mode 100644 index 0000000..50f16fa --- /dev/null +++ b/ggml/src/ggml-metal/kernels/solve_tri.metal @@ -0,0 +1,75 @@ +#include "common.h" + +constant short FC_solve_tri_nsg [[function_constant(FC_SOLVE_TRI + 0)]]; +constant short FC_solve_tri_n [[function_constant(FC_SOLVE_TRI + 1)]]; +constant short FC_solve_tri_k [[function_constant(FC_SOLVE_TRI + 2)]]; + +kernel void kernel_solve_tri_f32( + constant ggml_metal_kargs_solve_tri & args, + device const char * src0, + device const char * src1, + device char * dst, + threadgroup char * shmem [[threadgroup(0)]], + ushort3 tgpig[[threadgroup_position_in_grid]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + constexpr short NW = N_SIMDWIDTH; + + const short NSG = FC_solve_tri_nsg; + const short N = FC_solve_tri_n; + const short K = FC_solve_tri_k; + const short NP = PAD2(N, NW); + + const int32_t i03 = tgpig.z; + const int32_t i02 = tgpig.y; + const int32_t i01 = tgpig.x*NSG + sgitg; + + threadgroup float * sh0 = (threadgroup float *) shmem; + + device const float * src0_ptr = (device const float *)(src0 + i02 * args.nb02 + i03 * args.nb03) + sgitg*N; + device const float * src1_ptr = (device const float *)(src1 + i02 * args.nb12 + i03 * args.nb13) + i01; + device float * dst_ptr = (device float *)(dst + i02 * args.nb2 + i03 * args.nb3) + i01; + + for (short rr = 0; rr < N; rr += NSG) { + threadgroup_barrier(mem_flags::mem_threadgroup); + + { + threadgroup float * sh0_cur = sh0 + sgitg*NP; + + for (short t = 0; t*NW < N; ++t) { + const short idx = t*NW + tiisg; + sh0_cur[idx] = src0_ptr[idx]; + } + + src0_ptr += NSG*N; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (i01 >= args.ne10) { + continue; + } + + for (short ir = 0; ir < NSG && rr + ir < N; ++ir) { + const short r = rr + ir; + + threadgroup float * sh0_cur = sh0 + ir*NP; + + float sum = 0.0f; + + for (short t = 0; t*NW < r; ++t) { + const short idx = t*NW + tiisg; + sum += sh0_cur[idx] * dst_ptr[idx*K] * (idx < r); + } + + sum = simd_sum(sum); + + if (tiisg == 0) { + const float diag = sh0_cur[r]; + + dst_ptr[r*K] = (src1_ptr[r*K] - sum) / diag; + } + } + } +} diff --git a/ggml/src/ggml-metal/kernels/ssm.metal b/ggml/src/ggml-metal/kernels/ssm.metal new file mode 100644 index 0000000..b21c53b --- /dev/null +++ b/ggml/src/ggml-metal/kernels/ssm.metal @@ -0,0 +1,476 @@ +#include "common.h" + +constant bool FC_ssm_conv_silu [[function_constant(FC_SSM_CONV + 1)]]; +constant int FC_ssm_conv_nc [[function_constant(FC_SSM_CONV + 2)]]; + +// ref: ggml.c:ggml_compute_forward_ssm_conv_f32 +kernel void kernel_ssm_conv_f32_f32( + constant ggml_metal_kargs_ssm_conv & args, + device const void * src0, + device const void * src1, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int64_t ir = tgpig.x; + const int64_t i2 = tgpig.y; + const int64_t i3 = tgpig.z; + + const int64_t nc = FC_ssm_conv_nc; + //const int64_t ncs = args.ne00; + //const int64_t nr = args.ne01; + //const int64_t n_t = args.ne1; + //const int64_t n_s = args.ne2; + + device const float * s = (device const float *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02); + device const float * c = (device const float *) ((device const char *) src1 + ir*args.nb11); + device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); + + float sumf = 0.0f; + + FOR_UNROLL (int64_t i0 = 0; i0 < nc; ++i0) { + sumf += s[i0] * c[i0]; + } + + x[0] = FC_ssm_conv_silu ? sumf/(1.0f + exp(-sumf)) : sumf; +} + +kernel void kernel_ssm_conv_f32_f32_4( + constant ggml_metal_kargs_ssm_conv & args, + device const void * src0, + device const void * src1, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + const int64_t ir = tgpig.x; + const int64_t i2 = tgpig.y; + const int64_t i3 = tgpig.z; + + const int64_t nc = FC_ssm_conv_nc; + //const int64_t ncs = args.ne00; + //const int64_t nr = args.ne01; + //const int64_t n_t = args.ne1; + //const int64_t n_s = args.ne2; + + device const float4 * s = (device const float4 *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02); + device const float4 * c = (device const float4 *) ((device const char *) src1 + ir*args.nb11); + device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); + + float sumf = 0.0f; + + FOR_UNROLL (int64_t i0 = 0; i0 < nc/4; ++i0) { + sumf += dot(s[i0], c[i0]); + } + + x[0] = FC_ssm_conv_silu ? sumf/(1.0f + exp(-sumf)) : sumf; +} + +constant short FC_ssm_conv_bs [[function_constant(FC_SSM_CONV + 0)]]; + +// Batched version: each threadgroup processes multiple tokens for better efficiency +// Thread layout: each thread handles one token, threadgroup covers BATCH_SIZE tokens +kernel void kernel_ssm_conv_f32_f32_batched( + constant ggml_metal_kargs_ssm_conv & args, + device const void * src0, + device const void * src1, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + // tgpig.x = row index (ir) + // tgpig.y = batch of tokens (i2_base / BATCH_SIZE) + // tgpig.z = sequence index (i3) + // tpitg.x = thread within batch (0..BATCH_SIZE-1) + const short BATCH_SIZE = FC_ssm_conv_bs; + + const int64_t ir = tgpig.x; + const int64_t i2_base = tgpig.y * BATCH_SIZE; + const int64_t i3 = tgpig.z; + const int64_t i2_off = tpitg.x; + const int64_t i2 = i2_base + i2_off; + + const int64_t nc = FC_ssm_conv_nc; // conv kernel size (typically 4) + const int64_t n_t = args.ne1; // number of tokens + + // Bounds check for partial batches at the end + if (i2 >= n_t) { + return; + } + + // Load conv weights (shared across all tokens for this row) + device const float * c = (device const float *) ((device const char *) src1 + ir*args.nb11); + + // Load source for this specific token + device const float * s = (device const float *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02); + + // Output location for this token + device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); + + float sumf = 0.0f; + FOR_UNROLL (int64_t i0 = 0; i0 < nc; ++i0) { + sumf += s[i0] * c[i0]; + } + + x[0] = FC_ssm_conv_silu ? sumf/(1.0f + exp(-sumf)) : sumf; +} + +kernel void kernel_ssm_conv_f32_f32_batched_4( + constant ggml_metal_kargs_ssm_conv & args, + device const void * src0, + device const void * src1, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + // tgpig.x = row index (ir) + // tgpig.y = batch of tokens (i2_base / BATCH_SIZE) + // tgpig.z = sequence index (i3) + // tpitg.x = thread within batch (0..BATCH_SIZE-1) + const short BATCH_SIZE = FC_ssm_conv_bs; + + const int64_t ir = tgpig.x; + const int64_t i2_base = tgpig.y * BATCH_SIZE; + const int64_t i3 = tgpig.z; + const int64_t i2_off = tpitg.x; + const int64_t i2 = i2_base + i2_off; + + const int64_t nc = FC_ssm_conv_nc; // conv kernel size (typically 4) + const int64_t n_t = args.ne1; // number of tokens + + // Bounds check for partial batches at the end + if (i2 >= n_t) { + return; + } + + // Load conv weights (shared across all tokens for this row) + device const float4 * c = (device const float4 *) ((device const char *) src1 + ir*args.nb11); + + // Load source for this specific token + device const float4 * s = (device const float4 *) ((device const char *) src0 + ir*args.nb01 + i2*args.nb00 + i3*args.nb02); + + // Output location for this token + device float * x = (device float *) ((device char *) dst + ir*args.nb0 + i2*args.nb1 + i3*args.nb2); + + float sumf = 0.0f; + FOR_UNROLL (int64_t i0 = 0; i0 < nc/4; ++i0) { + sumf += dot(s[i0], c[i0]); + } + + x[0] = FC_ssm_conv_silu ? sumf/(1.0f + exp(-sumf)) : sumf; +} + +// ref: ggml.c:ggml_compute_forward_ssm_scan_f32, Mamba-2 part +// Optimized version: reduces redundant memory loads by having one thread load shared values +// TAIL == false is the whole-sequence / decode path: token_offset folds away at compile time. +template +kernel void kernel_ssm_scan_impl( + constant ggml_metal_kargs_ssm_scan & args, + device const void * src0, + device const void * src1, + device const void * src2, + device const void * src3, + device const void * src4, + device const void * src5, + device const void * src6, + device float * dst, + threadgroup float * shared [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort sgptg[[simdgroups_per_threadgroup]], + uint3 tgpg[[threadgroups_per_grid]]) { + constexpr short NW = N_SIMDWIDTH; + + // Shared memory layout: + // [0..sgptg*NW-1]: partial sums for reduction (existing) + // [sgptg*NW..sgptg*NW+sgptg-1]: pre-computed x_dt values for each token in batch + // [sgptg*NW+sgptg..sgptg*NW+2*sgptg-1]: pre-computed dA values for each token in batch + threadgroup float * shared_sums = shared; + threadgroup float * shared_x_dt = shared + sgptg * NW; + threadgroup float * shared_dA = shared + sgptg * NW + sgptg; + + shared_sums[tpitg.x] = 0.0f; + + const int32_t i0 = tpitg.x; + const int32_t i1 = tgpig.x; + const int32_t ir = tgpig.y; // current head + const int32_t i3 = tgpig.z; // current seq + + const int32_t nc = args.d_state; + const int32_t nr = args.d_inner; + const int32_t nh = args.n_head; + const int32_t ng = args.n_group; + const int32_t n_t = args.n_seq_tokens; + const int32_t n_s = args.n_seqs; + const int32_t K = args.K; + const int32_t n_t_total = TAIL ? args.n_seq_tokens_total : n_t; + const int32_t t_off = TAIL ? args.token_offset : 0; + + const int32_t s_off = args.s_off; + + device const int32_t * ids = (device const int32_t *) src6; + + device float * s_buff = (device float *) ((device char *) dst + ir*args.nb02 + i3*args.nb03 + s_off); + device const float * s0_buff = t_off != 0 ? + s_buff : + (device const float *) ((device const char *) src0 + ir*args.nb02 + ids[i3]*args.nb03); + + const int32_t i = i0 + i1*nc; + const int32_t g = ir / (nh / ng); // repeat_interleave + + float s0 = s0_buff[i]; + float s = 0.0f; + + device const float * A = (device const float *) ((device const char *) src3 + ir*args.nb31); // {ne30, nh} + + const float A0 = A[i0%args.ne30]; + + device const float * x = (device const float *)((device const char *) src1 + i1*args.nb10 + ir*args.nb11 + t_off*args.nb12 + i3*args.nb13); // {dim, nh, nt, ns} + device const float * dt = (device const float *)((device const char *) src2 + ir*args.nb20 + t_off*args.nb21 + i3*args.nb22); // {nh, nt, ns} + device const float * B = (device const float *)((device const char *) src4 + g*args.nb41 + t_off*args.nb42 + i3*args.nb43); // {d_state, ng, nt, ns} + device const float * C = (device const float *)((device const char *) src5 + g*args.nb51 + t_off*args.nb52 + i3*args.nb53); // {d_state, ng, nt, ns} + + device float * y = dst + (i1 + ir*nr + t_off*nh*nr + i3*(n_t_total*nh*nr)); // {dim, nh, nt, ns} + + for (int i2 = 0; i2 < n_t; i2 += sgptg) { + threadgroup_barrier(mem_flags::mem_threadgroup); + + // Pre-compute x_dt and dA for this batch of tokens + // Only first sgptg threads do the loads and expensive math + if (i0 < sgptg && i2 + i0 < n_t) { + // ns12 and ns21 are element strides (nb12/nb10, nb21/nb20) + device const float * x_t = x + i0 * args.ns12; + device const float * dt_t = dt + i0 * args.ns21; + + const float dt0 = dt_t[0]; + const float dtsp = dt0 <= 20.0f ? log(1.0f + exp(dt0)) : dt0; + shared_x_dt[i0] = x_t[0] * dtsp; + shared_dA[i0] = dtsp; // Store dtsp, compute exp(dtsp * A0) per-thread since A0 varies + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (int t = 0; t < sgptg && i2 + t < n_t; t++) { + const float x_dt = shared_x_dt[t]; + const float dA = exp(shared_dA[t] * A0); + + s = (s0 * dA) + (B[i0] * x_dt); + + const float sumf = simd_sum(s * C[i0]); + + if (tiisg == 0) { + shared_sums[t*NW + sgitg] = sumf; + } + + // recurse + s0 = s; + + const int32_t slot = n_t - 1 - (i2 + t); + if (slot > 0 && slot < K) { + device float * s_snapshot = (device float *) ((device char *) s_buff + (int64_t) slot*n_s*args.nb03); + s_snapshot[i] = s; + } + + B += args.ns42; + C += args.ns52; + } + + // Advance pointers for next batch + x += sgptg * args.ns12; + dt += sgptg * args.ns21; + + threadgroup_barrier(mem_flags::mem_threadgroup); + + const float sumf = simd_sum(shared_sums[sgitg*NW + tiisg]); + + if (tiisg == 0 && i2 + sgitg < n_t) { + y[sgitg*nh*nr] = sumf; + } + + y += sgptg*nh*nr; + } + + s_buff[i] = s; +} + +typedef decltype(kernel_ssm_scan_impl) kernel_ssm_scan_t; + +template [[host_name("kernel_ssm_scan_f32")]] kernel kernel_ssm_scan_t kernel_ssm_scan_impl; +template [[host_name("kernel_ssm_scan_f32_tail")]] kernel kernel_ssm_scan_t kernel_ssm_scan_impl; + +// Chunked SSD SSM scan via Metal simdgroup MMatrix Multiply-Accumulate (simdgroup_float8x8) fast path. +// One threadgroup per (head, sequence) and tokens are processed in chunks. +// C*B^T computed in each chunk one time and reused across the head_dim channel tiles. +kernel void kernel_ssm_scan_ssd_mma_f32( + constant ggml_metal_kargs_ssm_scan & args, + device const void * src0, + device const void * src1, + device const void * src2, + device const void * src3, + device const void * src4, + device const void * src5, + device const void * src6, + device float * dst, + threadgroup float * shared [[threadgroup(0)]], + uint3 tgpig[[threadgroup_position_in_grid]], + ushort tiitg[[thread_index_in_threadgroup]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]]) { + constexpr short CS = OP_SSM_SCAN_SSD_CS; + constexpr short TC = 8; // Tile Count of each edge in a simdgroup 8x8 tile + constexpr short HD = OP_SSM_SCAN_SSD_HD; + constexpr short NSG = OP_SSM_SCAN_SSD_NSG; + + // acs/exp(acs)/state-decay vectors, dtX[CS][HD], four private SAM row tiles [8][CS], + // and two 8x8 scratch tiles per simdgroup. Total: 26.75 KiB. + threadgroup float * shared_acs = shared; + threadgroup float * shared_exp_acs = shared + CS; + threadgroup float * shared_state_decay = shared + 2*CS; + threadgroup float * shared_dtx = shared + 3*CS; + threadgroup float * shared_sam = shared + 3*CS + CS*HD; + threadgroup float * sam_rows = shared_sam + sgitg*TC*CS; + threadgroup float * shared_tile = shared_sam + NSG*TC*CS; + threadgroup float * tile0 = shared_tile + sgitg*2*TC*TC; + threadgroup float * tile1 = tile0 + TC*TC; + + const int32_t ir = tgpig.y; // current head + const int32_t i3 = tgpig.z; // current seq + + const int32_t nc = args.d_state; + const int32_t nr = args.d_inner; + const int32_t nh = args.n_head; + const int32_t ng = args.n_group; + const int32_t n_t = args.n_seq_tokens; + const int32_t n_t_total = args.n_seq_tokens_total; + const int32_t g = ir / (nh / ng); + + device const int32_t * ids = (device const int32_t *) src6; + + device const float * s0_buff = (device const float *) ((device const char *) src0 + ir*args.nb02 + ids[i3]*args.nb03); + device float * s_buff = (device float *) ((device char *) dst + ir*args.nb02 + i3*args.nb03 + args.s_off); + + device const float * A = (device const float *) ((device const char *) src3 + ir*args.nb31); + device const float * x = (device const float *) ((device const char *) src1 + ir*args.nb11 + i3*args.nb13); + device const float * dt = (device const float *) ((device const char *) src2 + ir*args.nb20 + i3*args.nb22); + device const float * B = (device const float *) ((device const char *) src4 + g*args.nb41 + i3*args.nb43); + device const float * C = (device const float *) ((device const char *) src5 + g*args.nb51 + i3*args.nb53); + + device float * y = dst + (ir*nr + i3*(n_t_total*nh*nr)); + + for (int32_t t0 = 0; t0 < n_t; t0 += CS) { + for (int32_t idx = tiitg; idx < CS*HD; idx += NSG*N_SIMDWIDTH) { + const int32_t t = idx / HD; + const int32_t c = idx % HD; + const float dt0 = dt[(t0 + t) * (int32_t) args.ns21]; + const float dtsp = dt0 <= 20.0f ? log(1.0f + exp(dt0)) : dt0; + shared_dtx[idx] = x[(t0 + t) * (int32_t) args.ns12 + c] * dtsp; + } + if (tiitg < CS) { + const float dt0 = dt[(t0 + tiitg) * (int32_t) args.ns21]; + const float dtsp = dt0 <= 20.0f ? log(1.0f + exp(dt0)) : dt0; + shared_acs[tiitg] = dtsp * A[0]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + if (tiitg == 0) { + float acc = 0.0f; + for (short t = 0; t < CS; ++t) { + acc += shared_acs[t]; + shared_acs[t] = acc; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (tiitg < CS) { + shared_exp_acs[tiitg] = exp(shared_acs[tiitg]); + shared_state_decay[tiitg] = exp(shared_acs[CS - 1] - shared_acs[tiitg]); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + device const float * state = t0 == 0 ? s0_buff : s_buff; + + // Build one 8x64 row tile of SAM per simdgroup, then reuse it across every channel tile. + for (short ib = sgitg; ib < CS/TC; ib += NSG) { + for (short jb = 0; jb <= ib; ++jb) { + simdgroup_float8x8 cb = make_filled_simdgroup_matrix(0.0f); + + for (int32_t k0 = 0; k0 < nc; k0 += TC) { + simdgroup_float8x8 mc; + simdgroup_float8x8 mb; + simdgroup_load(mc, C + (t0 + ib*TC)*(int32_t) args.ns52 + k0, args.ns52); + simdgroup_load(mb, B + (t0 + jb*TC)*(int32_t) args.ns42 + k0, args.ns42, 0, true); + simdgroup_multiply_accumulate(cb, mc, mb, cb); + } + + threadgroup float * sam = sam_rows + jb*TC; + simdgroup_store(cb, sam, CS); + simdgroup_barrier(mem_flags::mem_threadgroup); + for (short e = tiisg; e < TC*TC; e += N_SIMDWIDTH) { + const short ri = e / TC; + const short rj = e % TC; + const short i = ib*TC + ri; + const short j = jb*TC + rj; + sam[ri*CS + rj] = j <= i ? + sam[ri*CS + rj] * exp(shared_acs[i] - shared_acs[j]) : 0.0f; + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } + + for (short ch = 0; ch < HD/TC; ++ch) { + simdgroup_float8x8 y_diag = make_filled_simdgroup_matrix(0.0f); + simdgroup_float8x8 y_inter = make_filled_simdgroup_matrix(0.0f); + + for (short jb = 0; jb <= ib; ++jb) { + simdgroup_float8x8 sam; + simdgroup_float8x8 mdtx; + simdgroup_load(sam, sam_rows + jb*TC, CS); + simdgroup_load(mdtx, shared_dtx + jb*TC*HD + ch*TC, HD); + simdgroup_multiply_accumulate(y_diag, sam, mdtx, y_diag); + } + + for (int32_t k0 = 0; k0 < nc; k0 += TC) { + simdgroup_float8x8 mc; + simdgroup_float8x8 ms; + simdgroup_load(mc, C + (t0 + ib*TC)*(int32_t) args.ns52 + k0, args.ns52); + simdgroup_load(ms, state + ch*TC*nc + k0, nc, 0, true); + simdgroup_multiply_accumulate(y_inter, mc, ms, y_inter); + } + + simdgroup_store(y_diag, tile0, TC); + simdgroup_store(y_inter, tile1, TC); + simdgroup_barrier(mem_flags::mem_threadgroup); + for (short e = tiisg; e < TC*TC; e += N_SIMDWIDTH) { + const short ri = e / TC; + const short ci = e % TC; + const int32_t token = t0 + ib*TC + ri; + y[token*nh*nr + ch*TC + ci] = + tile0[e] + shared_exp_acs[ib*TC + ri] * tile1[e]; + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } + } + + // All simdgroups must finish reading s_buff before any thread overwrites it. + threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup); + + // Keep the carried-state reduction in token order. Reassociating this particular product + // with MMA compounds rounding differences at every chunk boundary; CB, y_diag, and C*S + // remain on the matrix unit. + const float chunk_decay = exp(shared_acs[CS - 1]); + for (int32_t idx = tiitg; idx < nc*HD; idx += NSG*N_SIMDWIDTH) { + const int32_t ci = idx / nc; + const int32_t si = idx % nc; + float state_c = 0.0f; + for (short t = 0; t < CS; ++t) { + state_c += shared_state_decay[t] * + B[(t0 + t)*(int32_t) args.ns42 + si] * + shared_dtx[t*HD + ci]; + } + s_buff[idx] = chunk_decay * state[idx] + state_c; + } + + // All state tiles must be visible before the next chunk consumes s_buff as S_prev. + threadgroup_barrier(mem_flags::mem_device | mem_flags::mem_threadgroup); + } +} diff --git a/ggml/src/ggml-metal/kernels/tri.metal b/ggml/src/ggml-metal/kernels/tri.metal new file mode 100644 index 0000000..862f786 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/tri.metal @@ -0,0 +1,69 @@ +#include "common.h" + +template +bool _ggml_vec_tri_cmp(const int i, const int r); + +template<> +bool _ggml_vec_tri_cmp(const int i, const int r) { + return i < r; +} + +template<> +bool _ggml_vec_tri_cmp(const int i, const int r) { + return i <= r; +} + +template<> +bool _ggml_vec_tri_cmp(const int i, const int r) { + return i > r; +} + +template<> +bool _ggml_vec_tri_cmp(const int i, const int r) { + return i >= r; +} + +template +kernel void kernel_tri( + constant ggml_metal_kargs_tri & args, + device const char * src0, + device const char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + const int i3 = tgpig.z; + const int i2 = tgpig.y; + const int i1 = tgpig.x; + + if (i3 >= args.ne03 || i2 >= args.ne02 || i1 >= args.ne01) { + return; + } + + device const T * src_row = (device const T *) ((device const char *) src0 + i1*args.nb01 + i2*args.nb02 + i3*args.nb03); + device T * dst_row = (device T *) ((device char *) dst + i1*args.nb1 + i2*args.nb2 + i3*args.nb3); + + // Each thread is a single element of the row if ne00 < max threads per + // threadgroup, so this will loop once for each index that this thread is + // responsible for + for (int64_t i0 = tpitg.x; i0 < args.ne00; i0 += ntg.x) { + // Use the comparison as a mask for branchless + dst_row[i0] = static_cast(_ggml_vec_tri_cmp(i0, i1)) * src_row[i0]; + } +} + +typedef decltype(kernel_tri) kernel_tri_t; + +template [[host_name("kernel_tri_f32_0")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_f32_1")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_f32_2")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_f32_3")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_f16_0")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_f16_1")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_f16_2")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_f16_3")]] kernel kernel_tri_t kernel_tri; +#if defined(GGML_METAL_HAS_BF16) +template [[host_name("kernel_tri_bf16_0")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_bf16_1")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_bf16_2")]] kernel kernel_tri_t kernel_tri; +template [[host_name("kernel_tri_bf16_3")]] kernel kernel_tri_t kernel_tri; +#endif diff --git a/ggml/src/ggml-metal/kernels/unary.metal b/ggml/src/ggml-metal/kernels/unary.metal new file mode 100644 index 0000000..e50a648 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/unary.metal @@ -0,0 +1,400 @@ +#include "common.h" + +constant short FC_unary_op [[function_constant(FC_UNARY + 0)]]; +constant bool FC_unary_cnt[[function_constant(FC_UNARY + 1)]]; + +template +kernel void kernel_unary_impl( + constant ggml_metal_kargs_unary & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { +#define FC_OP FC_unary_op +#define FC_CNT FC_unary_cnt + + device const T0 * src0_ptr; + device T * dst_ptr; + + int i0; + + if (FC_CNT) { + i0 = tgpig.x; + + src0_ptr = (device const T0 *) (src0); + dst_ptr = (device T *) (dst); + } else { + const int i03 = tgpig.z; + const int i02 = tgpig.y; + const int k0 = tgpig.x/args.ne01; + const int i01 = tgpig.x - k0*args.ne01; + + i0 = k0*ntg.x + tpitg.x; + + src0_ptr = (device const T0 *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01); + dst_ptr = (device T *) (dst + i03*args.nb3 + i02*args.nb2 + i01*args.nb1 ); + } + + { + //threadgroup_barrier(mem_flags::mem_none); + + if (!FC_CNT) { + if (i0 >= args.ne0) { + return; + } + } + + const TC x = (TC) src0_ptr[i0]; + + if (FC_OP == OP_UNARY_NUM_SCALE) { + dst_ptr[i0] = (T) (args.scale * x + args.bias); + } + + if (FC_OP == OP_UNARY_NUM_FILL) { + dst_ptr[i0] = (T) args.val; + } + + if (FC_OP == OP_UNARY_NUM_CLAMP) { + dst_ptr[i0] = (T) clamp(x, args.min, args.max); + } + + if (FC_OP == OP_UNARY_NUM_SQR) { + dst_ptr[i0] = (T) (x * x); + } + + if (FC_OP == OP_UNARY_NUM_SQRT) { + dst_ptr[i0] = (T) sqrt(x); + } + + if (FC_OP == OP_UNARY_NUM_SIN) { + dst_ptr[i0] = (T) sin(x); + } + + if (FC_OP == OP_UNARY_NUM_COS) { + dst_ptr[i0] = (T) cos(x); + } + + if (FC_OP == OP_UNARY_NUM_LOG) { + dst_ptr[i0] = (T) log(x); + } + + if (FC_OP == OP_UNARY_NUM_LEAKY_RELU) { + dst_ptr[i0] = (T) (TC(x > 0)*x + TC(x <= 0)*(x * args.slope)); + } + + if (FC_OP == OP_UNARY_NUM_TANH) { + dst_ptr[i0] = (T) precise::tanh(x); + } + + if (FC_OP == OP_UNARY_NUM_RELU) { + dst_ptr[i0] = (T) fmax(0, x); + } + + if (FC_OP == OP_UNARY_NUM_SIGMOID) { + dst_ptr[i0] = (T) (1 / (1 + exp(-x))); + } + + if (FC_OP == OP_UNARY_NUM_GELU) { + dst_ptr[i0] = (T) (0.5*x*(1 + precise::tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x)))); + } + + if (FC_OP == OP_UNARY_NUM_GELU_ERF) { + dst_ptr[i0] = (T) (0.5*x*(1 + erf_approx(SQRT_2_INV*x))); + } + + if (FC_OP == OP_UNARY_NUM_GELU_QUICK) { + dst_ptr[i0] = (T) (x * (1/(1 + exp(GELU_QUICK_COEF*x)))); + } + + if (FC_OP == OP_UNARY_NUM_SILU) { + dst_ptr[i0] = (T) (x / (1 + exp(-x))); + } + + if (FC_OP == OP_UNARY_NUM_ELU) { + dst_ptr[i0] = (T) elu_approx(x); + } + + if (FC_OP == OP_UNARY_NUM_NEG) { + dst_ptr[i0] = (T) -x; + } + + if (FC_OP == OP_UNARY_NUM_ABS) { + dst_ptr[i0] = (T) fabs(x); + } + + if (FC_OP == OP_UNARY_NUM_SGN) { + dst_ptr[i0] = T(x > 0) - T(x < 0); + } + + if (FC_OP == OP_UNARY_NUM_STEP) { + dst_ptr[i0] = T(x > 0); + } + + if (FC_OP == OP_UNARY_NUM_HARDSWISH) { + dst_ptr[i0] = (T) (x * fmax(0, fmin(1, x/6 + 0.5))); + } + + if (FC_OP == OP_UNARY_NUM_HARDSIGMOID) { + dst_ptr[i0] = (T) fmax(0, fmin(1, x/6 + 0.5)); + } + + if (FC_OP == OP_UNARY_NUM_EXP) { + dst_ptr[i0] = (T) exp(x); + } + + if (FC_OP == OP_UNARY_NUM_SOFTPLUS) { + dst_ptr[i0] = (T) select(log(1 + exp(x)), x, x > 20); + } + + if (FC_OP == OP_UNARY_NUM_EXPM1) { + // TODO: precise implementation + dst_ptr[i0] = (T) (exp(x) - 1); + } + + if (FC_OP == OP_UNARY_NUM_FLOOR) { + dst_ptr[i0] = (T) floor(x); + } + + if (FC_OP == OP_UNARY_NUM_CEIL) { + dst_ptr[i0] = (T) ceil(x); + } + + if (FC_OP == OP_UNARY_NUM_ROUND) { + dst_ptr[i0] = (T) round(x); + } + + if (FC_OP == OP_UNARY_NUM_TRUNC) { + dst_ptr[i0] = (T) trunc(x); + } + + if (FC_OP == OP_UNARY_NUM_XIELU) { + const TC xi = x; + const TC gate = TC(xi > TC(0.0f)); + const TC clamped = fmin(xi, TC(args.val)); + const TC y_pos = TC(args.scale) * xi * xi + TC(args.bias) * xi; + const TC y_neg = (exp(clamped) - TC(1.0f) - xi) * TC(args.slope) + TC(args.bias) * xi; + dst_ptr[i0] = (T) (gate * y_pos + (TC(1.0f) - gate) * y_neg); + } + } + +#undef FC_OP +#undef FC_CNT +} + +typedef decltype(kernel_unary_impl) kernel_unary_t; + +template [[host_name("kernel_unary_f32_f32")]] kernel kernel_unary_t kernel_unary_impl; +template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_unary_impl; +template [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl; +template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl; + +kernel void kernel_silu_back_f32( + constant ggml_metal_kargs_silu_back & args, + device const float * dy, + device const float * x, + device float * dx, + uint gid [[thread_position_in_grid]]) { + if (gid >= args.ne) { + return; + } + + const float s = 1.0f / (1.0f + exp(-x[gid])); + dx[gid] = dy[gid] * s * (1.0f + x[gid] * (1.0f - s)); +} + +template +kernel void kernel_reglu( + constant ggml_metal_kargs_glu & args, + device const char * src0, + device const char * src1, + device char * dst, + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint ntg[[threads_per_threadgroup]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); + + for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + const float x0 = src0_row[i0]; + const float x1 = src1_row[i0]; + + dst_row[i0] = (T)(x0*x1*(x0 > 0.0f)); + } +} + +typedef decltype(kernel_reglu) kernel_reglu_t; + +template [[host_name("kernel_reglu_f32")]] kernel kernel_reglu_t kernel_reglu; +template [[host_name("kernel_reglu_f16")]] kernel kernel_reglu_t kernel_reglu; + +template +kernel void kernel_geglu( + constant ggml_metal_kargs_glu & args, + device const char * src0, + device const char * src1, + device char * dst, + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint ntg[[threads_per_threadgroup]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); + + for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + const float x0 = src0_row[i0]; + const float x1 = src1_row[i0]; + + const float gelu = 0.5f*x0*(1.0f + precise::tanh(SQRT_2_OVER_PI*x0*(1.0f + GELU_COEF_A*x0*x0))); + + dst_row[i0] = (T)(gelu*x1); + } +} + +typedef decltype(kernel_geglu) kernel_geglu_t; + +template [[host_name("kernel_geglu_f32")]] kernel kernel_geglu_t kernel_geglu; +template [[host_name("kernel_geglu_f16")]] kernel kernel_geglu_t kernel_geglu; + +template +kernel void kernel_swiglu( + constant ggml_metal_kargs_glu & args, + device const char * src0, + device const char * src1, + device char * dst, + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint ntg[[threads_per_threadgroup]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); + + for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + const float x0 = src0_row[i0]; + const float x1 = src1_row[i0]; + + const float silu = x0 / (1.0f + exp(-x0)); + + dst_row[i0] = (T)(silu*x1); + } +} + +typedef decltype(kernel_swiglu) kernel_swiglu_t; + +template [[host_name("kernel_swiglu_f32")]] kernel kernel_swiglu_t kernel_swiglu; +template [[host_name("kernel_swiglu_f16")]] kernel kernel_swiglu_t kernel_swiglu; + +template +kernel void kernel_swiglu_oai( + constant ggml_metal_kargs_glu & args, + device const char * src0, + device const char * src1, + device char * dst, + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint ntg[[threads_per_threadgroup]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); + + for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + float x0 = src0_row[i0]; + float x1 = src1_row[i0]; + + x0 = min(x0, args.limit); + x1 = max(min(x1, args.limit), -args.limit); + + float out_glu = x0 / (1.0f + exp(-x0 * args.alpha)); + out_glu = out_glu * (1.0f + x1); + + dst_row[i0] = (T)out_glu; + } +} + +typedef decltype(kernel_swiglu_oai) kernel_swiglu_oai_t; + +template [[host_name("kernel_swiglu_oai_f32")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai; +template [[host_name("kernel_swiglu_oai_f16")]] kernel kernel_swiglu_oai_t kernel_swiglu_oai; + +template +kernel void kernel_swiglu_clamp( + constant ggml_metal_kargs_glu & args, + device const char * src0, + device const char * src1, + device char * dst, + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint ntg[[threads_per_threadgroup]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); + + for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + const float gate = min((float) src0_row[i0], args.limit); + const float up = clamp((float) src1_row[i0], -args.limit, args.limit); + + dst_row[i0] = (T)(gate / (1.0f + exp(-gate)) * up); + } +} + +typedef decltype(kernel_swiglu_clamp) kernel_swiglu_clamp_t; + +template [[host_name("kernel_swiglu_clamp_f32")]] kernel kernel_swiglu_clamp_t kernel_swiglu_clamp; +template [[host_name("kernel_swiglu_clamp_f16")]] kernel kernel_swiglu_clamp_t kernel_swiglu_clamp; + +template +kernel void kernel_geglu_erf( + constant ggml_metal_kargs_glu & args, + device const char * src0, + device const char * src1, + device char * dst, + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint ntg[[threads_per_threadgroup]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); + + for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + const float x0 = src0_row[i0]; + const float x1 = src1_row[i0]; + + const float gelu_erf = 0.5f*x0*(1.0f+erf_approx(x0*SQRT_2_INV)); + + dst_row[i0] = (T)(gelu_erf*x1); + } +} + +typedef decltype(kernel_geglu_erf) kernel_geglu_erf_t; + +template [[host_name("kernel_geglu_erf_f32")]] kernel kernel_geglu_erf_t kernel_geglu_erf; +template [[host_name("kernel_geglu_erf_f16")]] kernel kernel_geglu_erf_t kernel_geglu_erf; + +template +kernel void kernel_geglu_quick( + constant ggml_metal_kargs_glu & args, + device const char * src0, + device const char * src1, + device char * dst, + uint tgpig[[threadgroup_position_in_grid]], + uint tpitg[[thread_position_in_threadgroup]], + uint ntg[[threads_per_threadgroup]]) { + device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00; + device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10; + device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1); + + for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) { + const float x0 = src0_row[i0]; + const float x1 = src1_row[i0]; + + const float gelu_quick = x0*(1.0f/(1.0f+exp(GELU_QUICK_COEF*x0))); + + dst_row[i0] = (T)(gelu_quick*x1); + } +} + +typedef decltype(kernel_geglu_quick) kernel_geglu_quick_t; + +template [[host_name("kernel_geglu_quick_f32")]] kernel kernel_geglu_quick_t kernel_geglu_quick; +template [[host_name("kernel_geglu_quick_f16")]] kernel kernel_geglu_quick_t kernel_geglu_quick; diff --git a/ggml/src/ggml-metal/kernels/upscale.metal b/ggml/src/ggml-metal/kernels/upscale.metal new file mode 100644 index 0000000..8bac130 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/upscale.metal @@ -0,0 +1,179 @@ +#include "common.h" + +constant bool FC_upscale_aa [[function_constant(FC_UPSCALE + 0)]]; + +kernel void kernel_upscale_nearest_f32( + constant ggml_metal_kargs_upscale & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int64_t i3 = tgpig.z; + const int64_t i2 = tgpig.y; + const int64_t i1 = tgpig.x; + + const int64_t i03 = i3/args.sf3; + const int64_t i02 = i2/args.sf2; + const int64_t i01 = i1/args.sf1; + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + const int64_t i00 = i0/args.sf0; + + device const float * src0_ptr = (device const float *) (src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00); + device float * dst_ptr = (device float *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + + dst_ptr[0] = src0_ptr[0]; + } +} + +static inline float bilinear_tri(float x) { + return MAX(0.0f, 1.0f - fabs(x)); +} + +kernel void kernel_upscale_bilinear_f32( + constant ggml_metal_kargs_upscale & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int64_t i3 = tgpig.z; + const int64_t i2 = tgpig.y; + const int64_t i1 = tgpig.x; + + const int64_t i03 = i3 / args.sf3; + const int64_t i02 = i2 / args.sf2; + + const float f01 = ((float)i1 + args.poffs) / args.sf1 - args.poffs; + const int64_t i01 = MAX(0, MIN(args.ne01 - 1, (int64_t)floor(f01))); + const int64_t i01p = MAX(0, MIN(args.ne01 - 1, i01 + 1)); + const float fd1 = MAX(0.0f, MIN(1.0f, f01 - (float)i01)); + + src0 += i03*args.nb03 + i02*args.nb02; + + device float * dst_ptr = (device float *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1); + + if (FC_upscale_aa) { + const float support0 = MAX(1.0f, 1.0f / args.sf0); + const float invscale0 = 1.0f / support0; + const float support1 = MAX(1.0f, 1.0f / args.sf1); + const float invscale1 = 1.0f / support1; + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + const float f00 = ((float)i0 + args.poffs) / args.sf0 - args.poffs; + + int64_t x_min = MAX((int64_t)0, (int64_t)floor(f00 - support0 + args.poffs)); + int64_t x_max = MIN(args.ne00, (int64_t)ceil (f00 + support0 + args.poffs)); + + int64_t y_min = MAX((int64_t)0, (int64_t)floor(f01 - support1 + args.poffs)); + int64_t y_max = MIN(args.ne01, (int64_t)ceil (f01 + support1 + args.poffs)); + + float sum = 0.0f; + float wsum = 0.0f; + + for (int64_t sy = y_min; sy < y_max; ++sy) { + const float wy = MAX(0.0f, 1.0f - fabs((float)sy - f01) * invscale1); + for (int64_t sx = x_min; sx < x_max; ++sx) { + const float wx = MAX(0.0f, 1.0f - fabs((float)sx - f00) * invscale0); + const float w = wx * wy; + device const float * src_ptr = (device const float *)(src0 + sy*args.nb01 + sx*args.nb00); + sum += (*src_ptr) * w; + wsum += w; + } + } + + const float v = (wsum > 0.0f) ? (sum / wsum) : 0.0f; + dst_ptr[i0] = v; + } + } else { + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + const float f00 = ((float)i0 + args.poffs) / args.sf0 - args.poffs; + const int64_t i00 = MAX(0, MIN(args.ne00 - 1, (int64_t)floor(f00))); + const int64_t i00p = MAX(0, MIN(args.ne00 - 1, i00 + 1)); + const float fd0 = MAX(0.0f, MIN(1.0f, f00 - (float)i00)); + + device const float * src00 = (device const float *)(src0 + i01*args.nb01 + i00*args.nb00); + device const float * src10 = (device const float *)(src0 + i01*args.nb01 + i00p*args.nb00); + device const float * src01 = (device const float *)(src0 + i01p*args.nb01 + i00*args.nb00); + device const float * src11 = (device const float *)(src0 + i01p*args.nb01 + i00p*args.nb00); + + const float v = + (*src00) * (1.0f - fd0) * (1.0f - fd1) + + (*src10) * fd0 * (1.0f - fd1) + + (*src01) * (1.0f - fd0) * fd1 + + (*src11) * fd0 * fd1; + + dst_ptr[i0] = v; + } + } +} + +static inline float bicubic_weight1(float x) { + const float a = -0.75f; + return ((a + 2) * x - (a + 3)) * x * x + 1; +} + +static inline float bicubic_weight2(float x) { + const float a = -0.75f; + return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; +} + +kernel void kernel_upscale_bicubic_f32( + constant ggml_metal_kargs_upscale & args, + device const char * src0, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const int64_t i3 = tgpig.z; + const int64_t i2 = tgpig.y; + const int64_t i1 = tgpig.x; + + const int64_t i03 = i3 / args.sf3; + const int64_t i02 = i2 / args.sf2; + + const float f01 = ((float)i1 + args.poffs) / args.sf1 - args.poffs; + const int64_t i01 = (int64_t)floor(f01); + const float fd1 = f01 - (float)i01; + + const float w_y0 = bicubic_weight2(fd1 + 1.0f); + const float w_y1 = bicubic_weight1(fd1); + const float w_y2 = bicubic_weight1(1.0f - fd1); + const float w_y3 = bicubic_weight2(2.0f - fd1); + + const device char * src_slice = src0 + i03 * args.nb03 + i02 * args.nb02; + + device float * dst_ptr = (device float *)(dst + i3 * args.nb3 + i2 * args.nb2 + i1 * args.nb1); + + for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + const float f00 = ((float)i0 + args.poffs) / args.sf0 - args.poffs; + const int64_t i00 = (int64_t)floor(f00); + const float fd0 = f00 - (float)i00; + + const float w_x0 = bicubic_weight2(fd0 + 1.0f); + const float w_x1 = bicubic_weight1(fd0); + const float w_x2 = bicubic_weight1(1.0f - fd0); + const float w_x3 = bicubic_weight2(2.0f - fd0); + + float sum = 0.0f; + + for (int dy = -1; dy <= 2; ++dy) { + const int64_t iy = MAX(0, MIN(args.ne01 - 1, i01 + dy)); + const float wy = (dy == -1) ? w_y0 : (dy == 0) ? w_y1 : (dy == 1) ? w_y2 : w_y3; + + for (int dx = -1; dx <= 2; ++dx) { + const int64_t ix = MAX(0, MIN(args.ne00 - 1, i00 + dx)); + const float wx = (dx == -1) ? w_x0 : (dx == 0) ? w_x1 : (dx == 1) ? w_x2 : w_x3; + + device const float * src_ptr = (device const float *)(src_slice + iy * args.nb01 + ix * args.nb00); + sum += (*src_ptr) * wx * wy; + } + } + + dst_ptr[i0] = sum; + } +} diff --git a/ggml/src/ggml-metal/kernels/wkv.metal b/ggml/src/ggml-metal/kernels/wkv.metal new file mode 100644 index 0000000..8767581 --- /dev/null +++ b/ggml/src/ggml-metal/kernels/wkv.metal @@ -0,0 +1,179 @@ +#include "common.h" + +kernel void kernel_rwkv_wkv6_f32( + device const float * k, + device const float * v, + device const float * r, + device const float * tf, + device const float * td, + device const float * state_in, + device float * dst, + constant uint & B, + constant uint & T, + constant uint & C, + constant uint & H, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const uint head_size = 64; // TODO: support head_size = 128 + const uint batch_id = tgpig.x / H; + const uint head_id = tgpig.x % H; + const uint tid = tpitg.x; + + if (batch_id >= B || head_id >= H) { + return; + } + + const uint state_size = C * head_size; + const uint n_seq_tokens = T / B; + + threadgroup float _k[head_size]; + threadgroup float _r[head_size]; + threadgroup float _tf[head_size]; + threadgroup float _td[head_size]; + + float state[head_size]; + + for (uint i = 0; i < head_size; i++) { + state[i] = state_in[batch_id * state_size + head_id * head_size * head_size + + i * head_size + tid]; + } + + threadgroup_barrier(mem_flags::mem_threadgroup); + _tf[tid] = tf[head_id * head_size + tid]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; + const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; + + for (uint t = start_t; t < end_t; t += C) { + threadgroup_barrier(mem_flags::mem_threadgroup); + _k[tid] = k[t]; + _r[tid] = r[t]; + _td[tid] = td[t]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + const float v_val = v[t]; + float y = 0.0; + + for (uint j = 0; j < head_size; j += 4) { + float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + float4 tf_vec = float4(_tf[j], _tf[j+1], _tf[j+2], _tf[j+3]); + float4 td_vec = float4(_td[j], _td[j+1], _td[j+2], _td[j+3]); + float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); + + float4 kv = k_vec * v_val; + + float4 temp = tf_vec * kv + s_vec; + y += dot(r_vec, temp); + + s_vec = s_vec * td_vec + kv; + state[j] = s_vec[0]; + state[j+1] = s_vec[1]; + state[j+2] = s_vec[2]; + state[j+3] = s_vec[3]; + } + + dst[t] = y; + } + + for (uint i = 0; i < head_size; i++) { + dst[T * C + batch_id * state_size + head_id * head_size * head_size + + i * head_size + tid] = state[i]; + } +} + +kernel void kernel_rwkv_wkv7_f32( + device const float * r, + device const float * w, + device const float * k, + device const float * v, + device const float * a, + device const float * b, + device const float * state_in, + device float * dst, + constant uint & B, + constant uint & T, + constant uint & C, + constant uint & H, + uint3 tgpig[[threadgroup_position_in_grid]], + uint3 tpitg[[thread_position_in_threadgroup]], + uint3 ntg[[threads_per_threadgroup]]) { + + const uint head_size = 64; // TODO: support head_size = 128 + const uint batch_id = tgpig.x / H; + const uint head_id = tgpig.x % H; + const uint tid = tpitg.x; + + if (batch_id >= B || head_id >= H) { + return; + } + + const uint state_size = C * head_size; + const uint n_seq_tokens = T / B; + + threadgroup float _r[head_size]; + threadgroup float _w[head_size]; + threadgroup float _k[head_size]; + threadgroup float _a[head_size]; + threadgroup float _b[head_size]; + + float state[head_size]; + + for (uint i = 0; i < head_size; i++) { + state[i] = state_in[batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i]; + } + + const uint start_t = batch_id * n_seq_tokens * C + head_id * head_size + tid; + const uint end_t = (batch_id + 1) * n_seq_tokens * C + head_id * head_size + tid; + + for (uint t = start_t; t < end_t; t += C) { + threadgroup_barrier(mem_flags::mem_threadgroup); + _r[tid] = r[t]; + _w[tid] = w[t]; + _k[tid] = k[t]; + _a[tid] = a[t]; + _b[tid] = b[t]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + const float v_val = v[t]; + float y = 0.0, sa = 0.0; + + float4 sa_vec(0.0); + + for (uint j = 0; j < head_size; j += 4) { + float4 a_vec = float4(_a[j], _a[j+1], _a[j+2], _a[j+3]); + float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); + sa_vec += a_vec * s_vec; + } + sa = sa_vec[0] + sa_vec[1] + sa_vec[2] + sa_vec[3]; + + for (uint j = 0; j < head_size; j += 4) { + float4 r_vec = float4(_r[j], _r[j+1], _r[j+2], _r[j+3]); + float4 w_vec = float4(_w[j], _w[j+1], _w[j+2], _w[j+3]); + float4 k_vec = float4(_k[j], _k[j+1], _k[j+2], _k[j+3]); + float4 b_vec = float4(_b[j], _b[j+1], _b[j+2], _b[j+3]); + float4 s_vec = float4(state[j], state[j+1], state[j+2], state[j+3]); + + float4 kv = k_vec * v_val; + + s_vec = s_vec * w_vec + kv + sa * b_vec; + y += dot(s_vec, r_vec); + + state[j] = s_vec[0]; + state[j+1] = s_vec[1]; + state[j+2] = s_vec[2]; + state[j+3] = s_vec[3]; + } + + dst[t] = y; + } + + for (uint i = 0; i < head_size; i++) { + dst[T * C + batch_id * state_size + head_id * head_size * head_size + + tid * head_size + i] = state[i]; + } +} diff --git a/ggml/src/ggml-musa/CMakeLists.txt b/ggml/src/ggml-musa/CMakeLists.txt index cc53c81..721621a 100644 --- a/ggml/src/ggml-musa/CMakeLists.txt +++ b/ggml/src/ggml-musa/CMakeLists.txt @@ -21,7 +21,10 @@ if (MUSAToolkit_FOUND) message(STATUS "MUSA Toolkit found") if (NOT DEFINED MUSA_ARCHITECTURES) - set(MUSA_ARCHITECTURES "21;22;31") + set(MUSA_ARCHITECTURES "22;31") + endif() + if ("21" IN_LIST MUSA_ARCHITECTURES) + message(WARNING "MUSA architecture 21 (MTT S70, MTT S80, MTT S3000) is deprecated and no longer tested") endif() message(STATUS "Using MUSA architectures: ${MUSA_ARCHITECTURES}") @@ -43,17 +46,8 @@ if (MUSAToolkit_FOUND) add_compile_definitions(GGML_MUSA_MUDNN_COPY) endif() - if (GGML_CUDA_FA_ALL_QUANTS) - file(GLOB SRCS "../ggml-cuda/template-instances/fattn-vec*.cu") - list(APPEND GGML_SOURCES_MUSA ${SRCS}) - add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS) - else() - list(APPEND GGML_SOURCES_MUSA - ../ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu - ../ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu) - endif() + ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR}/../ggml-cuda SRCS) + list(APPEND GGML_SOURCES_MUSA ${SRCS}) set_source_files_properties(${GGML_SOURCES_MUSA} PROPERTIES LANGUAGE CXX) foreach(SOURCE ${GGML_SOURCES_MUSA}) @@ -75,7 +69,6 @@ if (MUSAToolkit_FOUND) endif() add_compile_definitions(GGML_USE_MUSA) - add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE}) if (GGML_MUSA_GRAPHS) add_compile_definitions(GGML_MUSA_GRAPHS) diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 772fc53..57620e5 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -5,6 +5,8 @@ set(TARGET_NAME ggml-opencl) ggml_add_backend_library(${TARGET_NAME} ggml-opencl.cpp + cl-program-cache.cpp + cl-program-cache.h ../../include/ggml-opencl.h) target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCL_LIBRARIES}) target_include_directories(${TARGET_NAME} PRIVATE ${OpenCL_INCLUDE_DIRS}) @@ -31,6 +33,11 @@ if (GGML_OPENCL_EMBED_KERNELS) target_include_directories(${TARGET_NAME} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/autogenerated") endif () +if (GGML_OPENCL_USE_ADRENO_BIN_KERNELS) + message(STATUS "OpenCL will use precompiled binary kernels for Adreno (improved performance on some platforms)") + add_compile_definitions(GGML_OPENCL_USE_ADRENO_BIN_KERNELS) +endif () + function(ggml_opencl_add_kernel KNAME) set(KERN_HDR ${CMAKE_CURRENT_BINARY_DIR}/autogenerated/${KNAME}.cl.h) set(KERN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/kernels/${KNAME}.cl) @@ -56,7 +63,11 @@ endfunction() set(GGML_OPENCL_KERNELS add add_id + moe_add_id_glu + argmax argsort + top_k + lightning_indexer tri fill clamp @@ -66,21 +77,23 @@ set(GGML_OPENCL_KERNELS diag div gelu - gemv_noshuffle_general - gemv_noshuffle get_rows glu group_norm solve_tri im2col_f32 im2col_f16 + im2col_3d_f32 + im2col_3d_f16 mean - mul_mat_Ab_Bi_8x4 mul_mv_f16_f16 mul_mv_f16_f32_1row mul_mv_f16_f32_l4 mul_mv_f16_f32 + mul_mv_f16_f32_mrow mul_mv_f32_f32 + mul_mv_q1_0_f32 + mul_mv_q1_0_f32_flat mul_mv_q4_0_f32 mul_mv_q4_0_f32_v mul_mv_q4_0_f32_8x_flat @@ -88,14 +101,26 @@ set(GGML_OPENCL_KERNELS mul_mv_q4_0_f32_1d_16x_flat mul_mv_q4_1_f32 mul_mv_q4_1_f32_flat + gemm_noshuffle_q2_k_q8_1_dp4a + gemv_noshuffle_q2_k_f32 + gemm_noshuffle_q2_k_f32 + mul_mv_q2_k_f32 + mul_mv_q2_k_f32_flat + mul_mm_q2_k_f32_l4_lm mul_mv_q4_k_f32 mul_mv_q4_k_f32_flat + mul_mv_q5_0_f32 + mul_mv_q5_0_f32_flat + mul_mv_q5_1_f32 + mul_mv_q5_1_f32_flat mul_mv_q5_k_f32 mul_mv_q5_k_f32_flat mul_mv_q6_k_f32 mul_mv_q6_k_f32_flat mul_mv_q8_0_f32 mul_mv_q8_0_f32_flat + mul_mv_iq4_nl_f32 + mul_mv_iq4_nl_f32_flat mul_mv_mxfp4_f32 mul_mv_mxfp4_f32_flat mul_mv_id_q4_0_f32_8x_flat @@ -103,26 +128,88 @@ set(GGML_OPENCL_KERNELS mul_mv_id_q8_0_f32_flat mul_mv_id_mxfp4_f32 mul_mv_id_mxfp4_f32_flat + gemm_moe_q4_0_f32_ns + gemm_moe_q4_0_q8_1_dp4a + gemv_moe_q4_0_f32_ns + gemm_moe_q8_0_f32_ns + gemm_moe_q4_1_f32_ns + gemv_moe_q4_1_f32_ns + gemm_moe_q5_0_f32_ns + gemv_moe_q5_0_f32_ns + gemm_moe_q5_1_f32_ns + gemv_moe_q5_1_f32_ns + gemm_moe_q4_k_f32_ns + gemm_moe_q4_k_q8_1_dp4a + gemm_moe_q6_k_q8_1_dp4a + gemm_moe_q8_1_dp4a + moe_reorder_quant_a_q8_1 + transpose_activation_q8 + gemm_noshuffle_q4_k_q8_1_dp4a + gemm_noshuffle_q5_k_q8_1_dp4a + gemm_noshuffle_q6_k_q8_1_dp4a + gemm_noshuffle_q8_0_q8_1_dp4a + gemm_noshuffle_q5_0_q8_1_dp4a + gemm_noshuffle_iq4_nl_q8_1_dp4a + gemm_noshuffle_q4_0_q8_1_dp4a + gemm_coopk_q4_0_q8_1_dp4a + gemm_coopk_q4_k_q8_1_dp4a + gemm_coopk_q6_k_q8_1_dp4a + quant_a_q8_1 + gemv_moe_q4_k_f32_ns + gemm_moe_q5_k_f32_ns + gemv_moe_q5_k_f32_ns + gemm_moe_q6_k_f32_ns + gemv_moe_q6_k_f32_ns gemm_moe_mxfp4_f32 gemv_moe_mxfp4_f32 + gemm_moe_mxfp4_f32_ns + gemm_moe_mxfp4_q8_1_dp4a + gemv_moe_mxfp4_f32_ns + moe_reorder_b + moe_combine + moe_sort_by_expert mul_mm_f32_f32_l4_lm mul_mm_f16_f32_l4_lm + mul_mm_q1_0_f32_l4_lm mul_mm_q4_0_f32_l4_lm mul_mm_q4_1_f32_l4_lm + mul_mm_q5_0_f32_l4_lm + mul_mm_q5_1_f32_l4_lm mul_mm_q8_0_f32_l4_lm + mul_mm_iq4_nl_f32_l4_lm mul_mm_q4_k_f32_l4_lm mul_mm_q5_k_f32_l4_lm mul_mm_q6_k_f32_l4_lm - mul_mm_q8_0_f32_8x4 + gemv_noshuffle_q1_0_f32 + gemm_noshuffle_q1_0_f32 + gemv_noshuffle_q4_0_f32 + gemv_noshuffle_q4_0_f32_spec + gemm_noshuffle_q4_0_f32 + gemv_noshuffle_q4_0_f32_32b_trans gemv_noshuffle_q4_1_f32 gemm_noshuffle_q4_1_f32 - gemv_noshuffle_general_q8_0_f32 + gemv_noshuffle_q5_0_f32 + gemm_noshuffle_q5_0_f32 + gemv_noshuffle_q5_1_f32 + gemm_noshuffle_q5_1_f32 + gemv_noshuffle_iq4_nl_f32 + gemm_noshuffle_iq4_nl_f32 + gemv_noshuffle_q8_0_f32 + gemm_noshuffle_q8_0_f32 gemv_noshuffle_q4_k_f32 + gemv_noshuffle_q4_k_f32_o4 + gemv_noshuffle_q4_k_f32_tiled gemm_noshuffle_q4_k_f32 + gemv_noshuffle_q4_k_f32_32b_trans gemv_noshuffle_q6_k_f32 + gemv_noshuffle_q6_k_f32_o4 + gemv_noshuffle_q6_k_f32_tiled gemm_noshuffle_q6_k_f32 + gemm_noshuffle_q6_k_f32_tiled + gemv_noshuffle_q6_k_f32_32b_trans gemv_noshuffle_q5_k_f32 gemm_noshuffle_q5_k_f32 + gemv_noshuffle_q5_k_f32_32b_trans mul neg norm @@ -141,6 +228,8 @@ set(GGML_OPENCL_KERNELS sqr sqrt ssm_conv + ssm_scan + gated_delta_net sub sum_rows cumsum @@ -151,6 +240,8 @@ set(GGML_OPENCL_KERNELS tanh exp expm1 + abs + unary_ext softplus pad repeat @@ -158,11 +249,20 @@ set(GGML_OPENCL_KERNELS mul_mm_f16_f32_kq_kqv conv2d conv2d_f16_f32 + conv2_dw + flash_attn_repack + flash_attn_pre_f16 flash_attn_f32_f16 + flash_attn_f32_q8_0 + flash_attn_f32_q4_0 flash_attn_f16 flash_attn_f32 ) +if (GGML_OPENCL_USE_ADRENO_KERNELS) + list(APPEND GGML_OPENCL_KERNELS gemm_xmem_f16_f32_os8 sdpa_xmem_f32_f16_os8) +endif () + foreach (K ${GGML_OPENCL_KERNELS}) ggml_opencl_add_kernel(${K}) endforeach() diff --git a/ggml/src/ggml-opencl/cl-program-cache.cpp b/ggml/src/ggml-opencl/cl-program-cache.cpp new file mode 100644 index 0000000..1a4a281 --- /dev/null +++ b/ggml/src/ggml-opencl/cl-program-cache.cpp @@ -0,0 +1,453 @@ +// Match the version setup ggml-opencl.cpp uses, so any cl.h declarations we +// touch are consistent across this backend's translation units. +#define CL_TARGET_OPENCL_VERSION GGML_OPENCL_TARGET_VERSION +#define CL_USE_DEPRECATED_OPENCL_1_2_APIS + +#include "cl-program-cache.h" + +#include "ggml-impl.h" // GGML_LOG_INFO / WARN + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +# include +# define ggml_getpid() ((int) GetCurrentProcessId()) +#else +# include +# define ggml_getpid() ((int) getpid()) +#endif + +namespace fs = std::filesystem; + +// ---------------------------------------------------------------------------- +// SHA-256 (FIPS 180-4). Self-contained, ~80 lines, public-domain reference. +// Hot path is a few KB of source per kernel ⇒ <1 ms total per process init. +// ---------------------------------------------------------------------------- + +namespace { + +struct sha256_ctx { + uint32_t state[8]; + uint64_t bitlen; + uint8_t buf[64]; + size_t buf_len; +}; + +const uint32_t K256[64] = { + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2, +}; + +inline uint32_t rotr32(uint32_t x, unsigned n) { return (x >> n) | (x << (32 - n)); } + +void sha256_compress(uint32_t state[8], const uint8_t block[64]) { + uint32_t w[64]; + for (int i = 0; i < 16; ++i) { + w[i] = ((uint32_t)block[i*4 ] << 24) | + ((uint32_t)block[i*4 + 1] << 16) | + ((uint32_t)block[i*4 + 2] << 8) | + ((uint32_t)block[i*4 + 3] ); + } + for (int i = 16; i < 64; ++i) { + uint32_t s0 = rotr32(w[i-15], 7) ^ rotr32(w[i-15], 18) ^ (w[i-15] >> 3); + uint32_t s1 = rotr32(w[i-2], 17) ^ rotr32(w[i-2], 19) ^ (w[i-2] >> 10); + w[i] = w[i-16] + s0 + w[i-7] + s1; + } + + uint32_t a = state[0],b = state[1],c = state[2],d = state[3],e = state[4],f = state[5],g = state[6],h = state[7]; + + for (int i = 0; i < 64; ++i) { + uint32_t S1 = rotr32(e, 6) ^ rotr32(e, 11) ^ rotr32(e, 25); + uint32_t ch = (e & f) ^ ((~e) & g); + uint32_t t1 = h + S1 + ch + K256[i] + w[i]; + uint32_t S0 = rotr32(a, 2) ^ rotr32(a, 13) ^ rotr32(a, 22); + uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = S0 + maj; + h = g; g = f; f = e; e = d + t1; + d = c; c = b; b = a; a = t1 + t2; + } + state[0]+=a; state[1]+=b; state[2]+=c; state[3]+=d; + state[4]+=e; state[5]+=f; state[6]+=g; state[7]+=h; +} + +void sha256_init(sha256_ctx & c) { + c.state[0]=0x6a09e667; c.state[1]=0xbb67ae85; c.state[2]=0x3c6ef372; c.state[3]=0xa54ff53a; + c.state[4]=0x510e527f; c.state[5]=0x9b05688c; c.state[6]=0x1f83d9ab; c.state[7]=0x5be0cd19; + c.bitlen = 0; + c.buf_len = 0; +} + +void sha256_update(sha256_ctx & c, const void * data, size_t len) { + const uint8_t * p = (const uint8_t *) data; + c.bitlen += (uint64_t) len * 8; + if (c.buf_len > 0) { + size_t n = 64 - c.buf_len; + if (n > len) { n = len; } + memcpy(c.buf + c.buf_len, p, n); + c.buf_len += n; + p += n; + len -= n; + if (c.buf_len == 64) { + sha256_compress(c.state, c.buf); + c.buf_len = 0; + } + } + while (len >= 64) { + sha256_compress(c.state, p); + p += 64; + len -= 64; + } + if (len > 0) { + memcpy(c.buf, p, len); + c.buf_len = len; + } +} + +void sha256_final(sha256_ctx & c, uint8_t out[32]) { + uint64_t bitlen = c.bitlen; + c.buf[c.buf_len++] = 0x80; + if (c.buf_len > 56) { + while (c.buf_len < 64) { c.buf[c.buf_len++] = 0; } + sha256_compress(c.state, c.buf); + c.buf_len = 0; + } + while (c.buf_len < 56) { c.buf[c.buf_len++] = 0; } + for (int i = 7; i >= 0; --i) { c.buf[c.buf_len++] = (uint8_t) (bitlen >> (i * 8)); } + sha256_compress(c.state, c.buf); + for (int i = 0; i < 8; ++i) { + out[i*4 ] = (uint8_t) (c.state[i] >> 24); + out[i*4 + 1] = (uint8_t) (c.state[i] >> 16); + out[i*4 + 2] = (uint8_t) (c.state[i] >> 8); + out[i*4 + 3] = (uint8_t) (c.state[i] ); + } +} + +std::string sha256_hex(const uint8_t digest[32]) { + static const char hex[] = "0123456789abcdef"; + std::string s(64, '0'); + for (int i = 0; i < 32; ++i) { + s[i*2 ] = hex[digest[i] >> 4]; + s[i*2 + 1] = hex[digest[i] & 0xf]; + } + return s; +} + +std::string compute_key(const std::string & key_suffix, + const char * source, + const std::string & compile_opts) { + sha256_ctx c; + sha256_init(c); + + static const uint8_t sep = 0; + sha256_update(c, source, strlen(source)); + sha256_update(c, &sep, 1); + sha256_update(c, compile_opts.data(), compile_opts.size()); + sha256_update(c, &sep, 1); + sha256_update(c, key_suffix.data(), key_suffix.size()); + + uint8_t digest[32]; + sha256_final(c, digest); + return sha256_hex(digest); +} + +bool make_dir_recursive(const std::string & path) { + if (path.empty()) { return false; } + // create_directories() already creates missing parents. It returns false + // (with ec clear) when the directory is already there, so re-check. + const fs::path p = fs::u8path(path); + std::error_code ec; + if (fs::create_directories(p, ec)) { return true; } + std::error_code ec_stat; + return fs::is_directory(p, ec_stat); +} + +std::string default_cache_dir() { +#if defined(_WIN32) + const char * base = std::getenv("LOCALAPPDATA"); + if (!base || !*base) { base = std::getenv("APPDATA"); } + if (!base || !*base) { base = std::getenv("TEMP"); } + if (!base || !*base) { base = "."; } + return std::string(base) + "\\llama.cpp\\cl-cache"; +#elif defined(__APPLE__) + const char * home = std::getenv("HOME"); + if (!home || !*home) { home = "."; } + return std::string(home) + "/Library/Caches/llama.cpp/cl-cache"; +#else + // The throwing overload aborts the process when no usable temp directory + // exists (e.g. Android app contexts with TMPDIR unset); an empty return + // here just disables the cache instead. + std::error_code ec; + const fs::path tmp_path = fs::temp_directory_path(ec); + if (ec || tmp_path.empty()) { return {}; } + return tmp_path.string() + "/llama.cpp/cl-cache"; +#endif +} + +// Query a NUL-terminated string from clGetDeviceInfo / clGetPlatformInfo. +template +std::string query_string(GetInfoFn fn, Object obj, cl_uint name) { + size_t sz = 0; + if (fn(obj, name, 0, nullptr, &sz) != CL_SUCCESS || sz == 0) { + return {}; + } + std::string s(sz, '\0'); + if (fn(obj, name, sz, &s[0], nullptr) != CL_SUCCESS) { + return {}; + } + if (!s.empty() && s.back() == '\0') { + s.pop_back(); + } + return s; +} + +std::string compute_key_suffix(cl_device_id device) { + cl_platform_id platform = nullptr; + clGetDeviceInfo(device, CL_DEVICE_PLATFORM, sizeof(platform), &platform, nullptr); + + std::string s; + s.reserve(512); + s += query_string(clGetDeviceInfo, device, CL_DEVICE_NAME); s.push_back('\0'); + s += query_string(clGetDeviceInfo, device, CL_DRIVER_VERSION); s.push_back('\0'); + s += query_string(clGetDeviceInfo, device, CL_DEVICE_VERSION); s.push_back('\0'); + if (platform) { + s += query_string(clGetPlatformInfo, platform, CL_PLATFORM_VERSION); s.push_back('\0'); + } + s += "fmt=" + std::to_string(CL_PROGRAM_CACHE_FORMAT_VERSION); + return s; +} + +const uint8_t MAGIC[8] = { 'G','G','M','L','C','L','B','C' }; + +bool read_all(const std::string & path, std::vector & out) { + std::ifstream f(fs::u8path(path), std::ios::binary); + if (!f) { return false; } + f.seekg(0, std::ios::end); + std::streamsize sz = f.tellg(); + if (sz < 0) { return false; } + f.seekg(0, std::ios::beg); + out.resize((size_t) sz); + if (sz > 0) { f.read((char *) out.data(), sz); } + return f.good() || f.eof(); +} + +bool write_atomic(const std::string & path, const uint8_t * data, size_t len) { + const fs::path dst = fs::u8path(path); + const fs::path tmp = fs::u8path(path + ".tmp." + std::to_string(ggml_getpid())); + { + std::ofstream f(tmp, std::ios::binary | std::ios::trunc); + if (!f) { return false; } + f.write((const char *) data, (std::streamsize) len); + if (!f.good()) { + std::error_code ec_rm; + fs::remove(tmp, ec_rm); + return false; + } + } + + std::error_code ec; + fs::rename(tmp, dst, ec); + if (ec) { + std::error_code ec_rm; + fs::remove(tmp, ec_rm); + return false; + } + return true; +} + +} // namespace + +static bool cache_debug_enabled() { + static int cached = -1; + if (cached < 0) { + const char * e = std::getenv("GGML_OPENCL_KERNEL_CACHE_DEBUG"); + cached = (e && *e) ? 1 : 0; + } + return cached != 0; +} + +static std::string opts_preview(const std::string & opts, size_t n = 120) { + if (opts.size() <= n) { return opts; } + return opts.substr(0, n) + "..."; +} + +// Running cache tally (diagnostic; plain ints — a benign race in the rare +// multi-threaded lazy-compile case at worst miscounts by one). +static int g_cache_hits = 0, g_cache_misses = 0, g_cache_saves = 0; + +// Debug trace directly to stderr +static void cache_debug_line(const char * kind, const std::string & key, + const char * source, const std::string & opts) { + if (!cache_debug_enabled()) { return; } + fprintf(stderr, "ggml_opencl: cache %-4s [h=%d m=%d s=%d] key=%s src=%zuB opts='%s'\n", + kind, g_cache_hits, g_cache_misses, g_cache_saves, + key.substr(0, 16).c_str(), strlen(source), opts_preview(opts).c_str()); + fflush(stderr); +} + +cl_program_cache_state cl_program_cache_init(cl_device_id device) { + cl_program_cache_state st; + + const char * env = std::getenv("GGML_OPENCL_KERNEL_CACHE_DIR"); + if (env && (!std::strcmp(env, "0") || !std::strcmp(env, "off") || + !std::strcmp(env, "none") || !std::strcmp(env, "disable") || + !std::strcmp(env, "disabled"))) { + if (cache_debug_enabled()) { + fprintf(stderr, "ggml_opencl: kernel cache disabled by GGML_OPENCL_KERNEL_CACHE_DIR=%s\n", env); + fflush(stderr); + } + return st; + } + + std::string dir; + if (!env || !*env || !std::strcmp(env, "1") || !std::strcmp(env, "default")) { + dir = default_cache_dir(); + if (dir.empty()) { + GGML_LOG_INFO("ggml_opencl: kernel cache disabled (no usable default cache directory)\n"); + return st; + } + } else { + dir = env; + } + + if (!make_dir_recursive(dir)) { + GGML_LOG_INFO("ggml_opencl: kernel cache disabled (cannot create directory '%s')\n", dir.c_str()); + return st; + } + + st.dir = dir; + st.key_suffix = compute_key_suffix(device); + GGML_LOG_INFO("ggml_opencl: kernel cache enabled at '%s'\n", st.dir.c_str()); + if (cache_debug_enabled()) { + fprintf(stderr, "ggml_opencl: kernel cache enabled at '%s' " + "(GGML_OPENCL_KERNEL_CACHE_DIR=off to disable)\n", st.dir.c_str()); + fflush(stderr); + } + return st; +} + +cl_program cl_program_cache_try_load( + const cl_program_cache_state & state, + cl_context context, + cl_device_id device, + const char * source, + const std::string & compile_opts) { + + if (state.dir.empty() || !source) { return nullptr; } + + const std::string key = compute_key(state.key_suffix, source, compile_opts); + const std::string path = state.dir + "/" + key + ".clbin"; + + std::vector file; + if (!read_all(path, file)) { + ++g_cache_misses; + cache_debug_line("MISS", key, source, compile_opts); + return nullptr; + } + if (file.size() < 16 || std::memcmp(file.data(), MAGIC, 8) != 0) { return nullptr; } + + uint32_t fmt = + ((uint32_t) file[ 8]) | ((uint32_t) file[ 9] << 8) | + ((uint32_t) file[10] << 16) | ((uint32_t) file[11] << 24); + if (fmt != CL_PROGRAM_CACHE_FORMAT_VERSION) { return nullptr; } + + const size_t hdr_len = 16; + const unsigned char * bin = file.data() + hdr_len; + const size_t bin_len = file.size() - hdr_len; + + cl_int err = CL_SUCCESS; + cl_int bin_err = CL_SUCCESS; + cl_program p = clCreateProgramWithBinary(context, 1, &device, &bin_len, &bin, &bin_err, &err); + if (err != CL_SUCCESS || bin_err != CL_SUCCESS || p == nullptr) { + if (p) { clReleaseProgram(p); } + return nullptr; + } + + err = clBuildProgram(p, 0, nullptr, compile_opts.c_str(), nullptr, nullptr); + if (err != CL_SUCCESS) { + clReleaseProgram(p); + return nullptr; + } + ++g_cache_hits; + cache_debug_line("HIT", key, source, compile_opts); + return p; +} + +void cl_program_cache_try_save( + const cl_program_cache_state & state, + cl_program program, + cl_device_id /*device*/, + const char * source, + const std::string & compile_opts) { + + if (state.dir.empty() || !program || !source) { + return; + } + + cl_uint n_dev = 0; + if (clGetProgramInfo(program, CL_PROGRAM_NUM_DEVICES, sizeof(n_dev), &n_dev, nullptr) != CL_SUCCESS || n_dev == 0) { + return; + } + + std::vector sizes(n_dev); + if (clGetProgramInfo(program, CL_PROGRAM_BINARY_SIZES, sizeof(size_t) * n_dev, sizes.data(), nullptr) != CL_SUCCESS) { + return; + } + if (sizes.empty() || sizes[0] == 0) { + return; + } + + std::vector> binaries(n_dev); + std::vector bin_ptrs(n_dev); + for (cl_uint i = 0; i < n_dev; ++i) { + binaries[i].resize(sizes[i]); + bin_ptrs[i] = binaries[i].data(); + } + if (clGetProgramInfo(program, CL_PROGRAM_BINARIES, sizeof(unsigned char *) * n_dev, bin_ptrs.data(), nullptr) != CL_SUCCESS) { + return; + } + + // We only care about the first device's binary — that's the one we'd + // re-load with on a future cache hit. Multi-device contexts aren't a + // pattern this backend uses today. + const std::vector & bin = binaries[0]; + + std::vector file; + file.reserve(16 + bin.size()); + file.insert(file.end(), MAGIC, MAGIC + 8); + uint32_t fmt = CL_PROGRAM_CACHE_FORMAT_VERSION; + file.push_back((uint8_t) (fmt & 0xff)); + file.push_back((uint8_t) ((fmt >> 8) & 0xff)); + file.push_back((uint8_t) ((fmt >> 16) & 0xff)); + file.push_back((uint8_t) ((fmt >> 24) & 0xff)); + file.push_back(0); file.push_back(0); file.push_back(0); file.push_back(0); // reserved + file.insert(file.end(), bin.begin(), bin.end()); + + const std::string key = compute_key(state.key_suffix, source, compile_opts); + const std::string path = state.dir + "/" + key + ".clbin"; + if (!write_atomic(path, file.data(), file.size())) { + GGML_LOG_INFO("ggml_opencl: kernel cache: failed to write '%s'\n", path.c_str()); + } else { + ++g_cache_saves; + cache_debug_line("SAVE", key, source, compile_opts); + } +} diff --git a/ggml/src/ggml-opencl/cl-program-cache.h b/ggml/src/ggml-opencl/cl-program-cache.h new file mode 100644 index 0000000..49aa2d1 --- /dev/null +++ b/ggml/src/ggml-opencl/cl-program-cache.h @@ -0,0 +1,75 @@ +// On-disk cache for OpenCL cl_program binaries. Lets a fresh process skip the +// expensive clBuildProgram-from-source step when a binary for the exact same +// (source, compile options, device, driver, platform) was previously saved. +// +// Activation: default on via GGML_OPENCL_KERNEL_CACHE_DIR: +// unset / empty / "1" / "default" : platform default cache dir +// (%LOCALAPPDATA%\llama.cpp\cl-cache, +// ~/Library/Caches/llama.cpp/cl-cache, +// /llama.cpp/cl-cache elsewhere) +// "0" / "off" / "none" / "disable(d)" : disabled (all functions no-op) +// any other value : used verbatim as the cache path +// If the chosen directory cannot be created/used, the cache silently disables +// itself for the process and falls back to source compile. +// GGML_OPENCL_KERNEL_CACHE_DEBUG=1 prints a HIT/MISS/SAVE trace (with a running +// tally) straight to stderr — visible even in tools that filter INFO/WARN logs; +// redirect stderr to record it. +// +// Cache key (SHA-256 hex): +// sha256(source_bytes || '\x00' || +// compile_opts || '\x00' || +// CL_DEVICE_NAME || '\x00' || +// CL_DRIVER_VERSION || '\x00' || +// CL_PLATFORM_VERSION || '\x00' || +// CL_PROGRAM_CACHE_FORMAT_VERSION) +// +// The key fully captures everything that can affect the produced binary, +// without needing the host source revision (a kernel source change shows up +// in source_bytes; a compile-option change shows up in compile_opts). +// +// File layout per cache entry: /.clbin +// bytes [0..7] : magic "GGMLCLBC" +// bytes [8..11] : uint32_t format version (CL_PROGRAM_CACHE_FORMAT_VERSION) +// bytes [12..15] : uint32_t reserved (0) +// bytes [16..] : raw cl_program binary as returned by +// clGetProgramInfo(CL_PROGRAM_BINARIES) +// +// Concurrency: writes go to .tmp. then atomic rename. On race, +// last-writer-wins. No locks. + +#pragma once + +#include +#include + +// Bumped manually if host-side OpenCL API usage changes in a way that +// affects compile semantics but does not show up in source_bytes / +// compile_opts (e.g. switching from clCreateProgramWithSource to +// clCompileProgram + clLinkProgram, or changing how multiple sources +// are concatenated). Most commits — including kernel changes — do NOT +// require bumping this; the source bytes already capture those. +#define CL_PROGRAM_CACHE_FORMAT_VERSION 1u + +struct cl_program_cache_state { + // Empty string means cache is disabled. + std::string dir; + // Concatenated device/driver/platform identity + cache format version, + // computed once at init and folded into every key. + std::string key_suffix; +}; + +cl_program_cache_state cl_program_cache_init(cl_device_id device); + +cl_program cl_program_cache_try_load( + const cl_program_cache_state & state, + cl_context context, + cl_device_id device, + const char * source, + const std::string & compile_opts); + +void cl_program_cache_try_save( + const cl_program_cache_state & state, + cl_program program, + cl_device_id device, + const char * source, + const std::string & compile_opts); diff --git a/ggml/src/ggml-opencl/fa_tune.h b/ggml/src/ggml-opencl/fa_tune.h new file mode 100644 index 0000000..447f0cf --- /dev/null +++ b/ggml/src/ggml-opencl/fa_tune.h @@ -0,0 +1,92 @@ +#pragma once + +// Flash-attention per-(dk,dv) tile tuning for the Adreno OpenCL backend. +// Isolated from ggml-opencl.cpp so the tuning numbers are easy to find and +// edit; the FA dispatch and kernel-compile logic stay in the main file. +// This header is a file section — it is #included exactly once, at the point +// in ggml-opencl.cpp where the ggml logging macros are already in scope. + +// Per-(dk, dv) FA config; shared by dispatch and supports_op. +struct ggml_opencl_fa_dim { + int dk; int dv; int bm; int bn; int n_split; int nkv_split_threshold; +}; + +// Split variant fires when n_kv >= threshold (threshold=0 -> always split). +// Default tuning covers Adreno 7xx/8xx mobile and X1-series laptop GPUs. +static const ggml_opencl_fa_dim g_fa_dims_adreno_default[] = { + { 40, 40, 64, 32, 1, 0}, { 64, 64, 64, 32, 2, 64}, + { 80, 80, 64, 32, 2, 64}, { 96, 96, 64, 32, 2, 64}, + {112, 112, 64, 32, 2, 64}, {128, 128, 64, 32, 2, 64}, + {192, 128, 16, 16, 1, 0}, + {192, 192, 16, 16, 1, 0}, + {256, 256, 16, 16, 16, 0}, + {512, 512, 8, 16, 64, 0}, +}; + +struct ggml_opencl_fa_dim_table { + const ggml_opencl_fa_dim * data; + size_t count; + + const ggml_opencl_fa_dim * begin() const { return data; } + const ggml_opencl_fa_dim * end() const { return data + count; } +}; + +// Mutable copy of the active table; GGML_OPENCL_FA_TUNE patches entries here +// at backend init without touching the const source table. +static ggml_opencl_fa_dim g_fa_dims_runtime[ + sizeof(g_fa_dims_adreno_default) / sizeof(g_fa_dims_adreno_default[0])]; + +static ggml_opencl_fa_dim_table g_opencl_fa_dims = { + g_fa_dims_adreno_default, + sizeof(g_fa_dims_adreno_default) / sizeof(g_fa_dims_adreno_default[0]), +}; + +// GGML_OPENCL_FA_TUNE=dk:dv:bm:bn:nsplit:thr[,…] — patches matching entries +// in the active table at backend init, before the first FA kernel compiles. +// Unmatched (dk,dv) pairs are warned and ignored. +static void ggml_opencl_fa_apply_env_overrides() { + const char * e = std::getenv("GGML_OPENCL_FA_TUNE"); + if (!e || !e[0]) { + return; + } + + std::string s = e; + size_t pos = 0; + while (pos < s.size()) { + size_t comma = s.find(',', pos); + std::string entry = s.substr(pos, comma == std::string::npos ? std::string::npos : comma - pos); + int dk, dv, bm, bn, nsplit, thr; + if (std::sscanf(entry.c_str(), "%d:%d:%d:%d:%d:%d", &dk, &dv, &bm, &bn, &nsplit, &thr) == 6) { + bool patched = false; + for (size_t i = 0; i < g_opencl_fa_dims.count; ++i) { + ggml_opencl_fa_dim & d = g_fa_dims_runtime[i]; + if (d.dk == dk && d.dv == dv) { + d.bm = bm; d.bn = bn; d.n_split = nsplit; d.nkv_split_threshold = thr; + GGML_LOG_INFO("ggml_opencl: FA tune override DK=%d DV=%d -> bm=%d bn=%d n_split=%d thr=%d\n", + dk, dv, bm, bn, nsplit, thr); + patched = true; + break; + } + } + if (!patched) { + GGML_LOG_WARN("ggml_opencl: FA tune override DK=%d DV=%d ignored (no matching dim)\n", dk, dv); + } + } else { + GGML_LOG_WARN("ggml_opencl: FA tune override entry malformed: '%s'\n", entry.c_str()); + } + if (comma == std::string::npos) break; + pos = comma + 1; + } +} + +// Copy the default table into the mutable runtime buffer and apply any +// GGML_OPENCL_FA_TUNE overrides. A per-generation table can be added here +// once it has been tuned on hardware. +static void ggml_cl_init_fa_dims_table() { + const size_t count = sizeof(g_fa_dims_adreno_default) / sizeof(g_fa_dims_adreno_default[0]); + for (size_t i = 0; i < count; ++i) { + g_fa_dims_runtime[i] = g_fa_dims_adreno_default[i]; + } + g_opencl_fa_dims = { g_fa_dims_runtime, count }; + ggml_opencl_fa_apply_env_overrides(); +} diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 8bc7ae6..176a6e5 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -13,6 +13,24 @@ #include "ggml-backend-impl.h" #include "ggml.h" +#include "cl-program-cache.h" + +#ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS +#include "libdl.h" +#ifdef _WIN32 +#define KERNEL_LIB_NAME "adreno-opencl-kernels.dll" +#else +#define KERNEL_LIB_NAME "libadreno-opencl-kernels.so" +#endif // _WIN32 +#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS + +typedef const void * (*get_adreno_bin_kernel_func_t)( + const char * name, + const char * gpu_name, + const char * compiler_ver, + size_t * out_size +); + #include #include @@ -28,6 +46,10 @@ #include #include #include +#include +#include +#include +#include #undef MIN #undef MAX @@ -53,6 +75,10 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor); +static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor); +static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor); +static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + // See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1. // Precompute mp (m' in the paper) and L such that division // can be computed using a multiply (high 32b of 64b result) @@ -92,13 +118,16 @@ enum GPU_FAMILY { enum ADRENO_GPU_GEN { ADRENO_UNKNOWN, + A6X, A7X, A8X, X1E, + X2E, }; enum ADRENO_CL_COMPILER_TYPE { E031, + E17, DX, }; @@ -175,39 +204,67 @@ static ggml_cl_version get_opencl_platform_version(cl_platform_id platform) { return parse_cl_version(param_value); } +// Returns the DEVICE's OpenCL version. On an error returns ggml_cl_version with all zeroes. +static ggml_cl_version get_opencl_device_version(cl_device_id device) { + size_t param_size; + if (clGetDeviceInfo(device, CL_DEVICE_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { + return {}; + } + std::unique_ptr param_storage(new char[param_size]); + if (clGetDeviceInfo(device, CL_DEVICE_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { + return {}; + } + + auto param_value = std::string_view(param_storage.get(), param_size); + const std::string version_prefix = "OpenCL "; // "OpenCL . " + if (param_value.find(version_prefix) != 0) { + return {}; + } + param_value.remove_prefix(version_prefix.length()); + return parse_cl_version(param_value); +} + // Return a version to use in OpenCL C compilation. On an error returns ggml_cl_version with all zeroes. static ggml_cl_version get_opencl_c_version(ggml_cl_version platform_version, cl_device_id device) { size_t param_size; #if CL_TARGET_OPENCL_VERSION >= 300 - if (platform_version.major >= 3) { - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size)); - if (!param_size) { - return {}; - } - - std::unique_ptr versions(new cl_name_version[param_size]); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr)); - unsigned versions_count = param_size / sizeof(cl_name_version); + // CL_DEVICE_OPENCL_C_ALL_VERSIONS is an OpenCL 3.0 *device* query, so gating it on the + // *platform* version is not enough: a 3.0 platform can expose 2.0 devices, where the + // query returns CL_INVALID_VALUE and the old CL_CHECK aborted during backend init. + // Gate on the device version, and treat a failure as "fall back to the legacy query" + // rather than fatal -- a device may advertise 3.0 and still refuse the property. + const ggml_cl_version device_version = get_opencl_device_version(device); + if (platform_version.major >= 3 && device_version.major >= 3) { + cl_int err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, 0, nullptr, ¶m_size); + if (err == CL_SUCCESS && param_size) { + std::unique_ptr versions(new cl_name_version[param_size]); + err = clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_ALL_VERSIONS, param_size, versions.get(), nullptr); + if (err == CL_SUCCESS) { + unsigned versions_count = param_size / sizeof(cl_name_version); + + cl_version version_max = 0; + for (unsigned i = 0; i < versions_count; i++) { + version_max = std::max(versions[i].version, version_max); + } - cl_version version_max = 0; - for (unsigned i = 0; i < versions_count; i++) { - version_max = std::max(versions[i].version, version_max); + return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; + } } - - return { CL_VERSION_MAJOR(version_max), CL_VERSION_MINOR(version_max) }; + // fall through to CL_DEVICE_OPENCL_C_VERSION below } #else GGML_UNUSED(platform_version); #endif // CL_TARGET_OPENCL_VERSION >= 300 - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size)); - if (!param_size) { + if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, 0, nullptr, ¶m_size) != CL_SUCCESS || !param_size) { return {}; } std::unique_ptr param_storage(new char[param_size]); - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr)); + if (clGetDeviceInfo(device, CL_DEVICE_OPENCL_C_VERSION, param_size, param_storage.get(), nullptr) != CL_SUCCESS) { + return {}; + } auto param_value = std::string_view(param_storage.get(), param_size); const std::string version_prefix = "OpenCL C "; // Suffix: "XX.YY " @@ -220,14 +277,30 @@ static ggml_cl_version get_opencl_c_version(ggml_cl_version platform_version, cl } static ADRENO_GPU_GEN get_adreno_gpu_gen(const char *device_name) { + if (strstr(device_name, "610") || strstr(device_name, "612") || + strstr(device_name, "613") || strstr(device_name, "615") || + strstr(device_name, "616") || strstr(device_name, "618") || + strstr(device_name, "619") || strstr(device_name, "620") || + strstr(device_name, "623") || strstr(device_name, "630") || + strstr(device_name, "640") || + strstr(device_name, "642") || strstr(device_name, "643") || + strstr(device_name, "644") || strstr(device_name, "650") || + strstr(device_name, "660") || strstr(device_name, "663") || + strstr(device_name, "680") || strstr(device_name, "685") || + strstr(device_name, "690")) { + return ADRENO_GPU_GEN::A6X; + } + if (strstr(device_name, "730") || strstr(device_name, "740") || strstr(device_name, "750")) { return ADRENO_GPU_GEN::A7X; } - if (strstr(device_name, "830") || - strstr(device_name, "840")) { + if (strstr(device_name, "810") || + strstr(device_name, "830") || + strstr(device_name, "840") || + strstr(device_name, "850")) { return ADRENO_GPU_GEN::A8X; } @@ -235,6 +308,10 @@ static ADRENO_GPU_GEN get_adreno_gpu_gen(const char *device_name) { return ADRENO_GPU_GEN::X1E; } + if (strstr(device_name, "X2")) { + return ADRENO_GPU_GEN::X2E; + } + return ADRENO_GPU_GEN::ADRENO_UNKNOWN; } @@ -247,6 +324,17 @@ static ggml_cl_compiler_version get_adreno_cl_compiler_version(const char *drive size_t compiler_minor_offset = 8; size_t compiler_patch_offset = 11; + if (compiler_ver_pos == std::string::npos) { + compiler_ver_pos = driver_ver_str.find("E17"); + if (compiler_ver_pos != std::string::npos) { + type = ADRENO_CL_COMPILER_TYPE::E17; + compiler_ver_len = 12; + compiler_major_offset = 4; + compiler_minor_offset = 7; + compiler_patch_offset = 10; + } + } + if (compiler_ver_pos == std::string::npos) { compiler_ver_pos = driver_ver_str.find("DX"); if (compiler_ver_pos == std::string::npos) { @@ -255,6 +343,8 @@ static ggml_cl_compiler_version get_adreno_cl_compiler_version(const char *drive type = ADRENO_CL_COMPILER_TYPE::DX; compiler_ver_len = 11; compiler_major_offset = 3; + compiler_minor_offset = 6; + compiler_patch_offset = 9; } std::string compiler_ver_str = driver_ver_str.substr(compiler_ver_pos, compiler_ver_len); @@ -357,6 +447,10 @@ static void populateProfilingInfo( struct ggml_backend_opencl_context; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_adreno_xmem_attn_release_scratch(ggml_backend_opencl_context * backend_ctx); +#endif + // backend device context struct ggml_backend_opencl_device_context { cl_platform_id platform; @@ -367,14 +461,180 @@ struct ggml_backend_opencl_device_context { cl_device_type device_type; std::string device_version; - // Initialized by ggml_cl2_init(). + // Initialized by ggml_cl_init(). ggml_backend_opencl_context * backend_ctx = nullptr; // Initialized by ggml_backend_opencl_device_get_buffer_type() ggml_backend_buffer_type buffer_type; cl_context context = nullptr; + + GPU_FAMILY gpu_family = GPU_FAMILY::UNKNOWN; + ADRENO_GPU_GEN adreno_gen = ADRENO_GPU_GEN::ADRENO_UNKNOWN; + + std::regex *opfilter = nullptr; // regex of ops to not claim + std::string opfilter_str = ""; // regex string for opfilter + size_t global_mem_size = 0; +}; + +// Lazily-compiled flash-attention kernels and their per-(dk,dv) tile metadata. +// One map per (Q/KV dtype, decode/prefill, split) combination; the int maps +// hold tile dims (bm/bn), workgroup sizes and the n_kv split thresholds. +struct ggml_opencl_fa_kernels { + // f16 Q / f16 KV + std::map, cl_kernel> f16; + std::map, cl_kernel> f16_q1; + // f32 Q / f32 KV + std::map, cl_kernel> f32; + std::map, cl_kernel> f32_q1; + // f32 Q / f16 KV (mixed) + std::map, cl_kernel> f32_f16; + std::map, cl_kernel> f32_f16_split; // N_SPLIT>1 variant + std::map, cl_kernel> f32_f16_split_k_img; // DK=512 prefill split, K via image1d_buffer_t + std::map, cl_kernel> f32_f16_q1; + std::map, cl_kernel> f32_f16_q1_split; // flash-decoding K-split + // vec decode + std::map, cl_kernel> f32_f16_q1_vec; + bool f32_f16_vec_512_attempted = false; + std::map, cl_kernel> f32_f16_mq_decode; + std::map, size_t> f32_f16_mq_decode_wg; + std::map, int> f32_f16_mq_decode_hs; + std::set> f32_f16_mq_decode_attempted; + ggml_cl_buffer fd_partial; + cl_uint compute_units = 0; + // kv-head-coalesced vec decode + std::map, cl_kernel> f32_f16_q1_vec_mq; + // kv-head-coalesced + flash-decoding split + std::map, cl_kernel> f32_f16_q1_vec_mq_split; + // MQ_GQA=8 specializations + std::map, cl_kernel> f32_f16_q1_vec_mq_g8; + std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8; + // k-image variant of MQ_G8 vec_mq_split + std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8_k_img; + // k-image variant of MQ_GQA=4 vec_mq_split + std::map, cl_kernel> f32_f16_q1_vec_mq_split_k_img; + // Cluster-parallel decode + std::map, cl_kernel> f32_f16_q1_vec_mq_split_c8; + std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8_c8; + // NSG_SPLIT=2 specializations (WG=128): the c8 kernel's register footprint + // caps its per-kernel WG at 128 on X2, below the stock 256/192 requirement. + // 2 subgroups × FA_CL_NCL streams still gives 16 in-flight rows per WG. + std::map, cl_kernel> f32_f16_q1_vec_mq_split_c8_ns2; + std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8_c8_ns2; + // FA_CL_C=32 / MQ_GQA=8 / NSG_SPLIT=2 specialization for the DK=DV=256 + // GQA=8 class (Qwen3.5/3.6-35B-A3B: 16 Q heads, 2 KV heads). o_acc = + // DV_VEC/32 × 8 = 128B/lane (in budget); the baseline fa1 path for this + // shape has NO MQ/FD at all and pays an 8× KV re-read per Q head. + std::map, cl_kernel> f32_f16_q1_vec_mq_split_g8_c32; + // alternative decode + std::map, cl_kernel> f32_f16_q1_local_tile; + // hybrid local-tile + MQ + FD-split kernel for DK=DV=128 only + std::map, cl_kernel> f32_f16_q1_local_mq_split; + std::map, cl_kernel> f32_f16_q1_local_mq_split_g8; + std::map, int> f32_f16_bm; + std::map, int> f32_f16_bn; + std::map, int> f32_f16_wg_size; + std::map, int> f32_f16_split_wg_size; + std::map, int> f32_f16_split_nkv_threshold; + // f32 Q / native q8_0 KV + std::map, cl_kernel> f32_q8_0_q1; // decode + std::map, cl_kernel> f32_q8_0_q1_vec; // DV-split + multi-subgroup decode + std::map, cl_kernel> f32_q8_0_q1_split; // flash-decoding pass 1 + // KV-head-coalesced + flash-decoding split for q8_0 KV + std::map, cl_kernel> f32_q8_0_q1_vec_mq_split; + std::map, cl_kernel> f32_q8_0_q1_vec_mq_split_g8; + // Cluster-parallel q8_0 decode + std::map, cl_kernel> f32_q8_0_q1_vec_mq_split_c8; + std::map, cl_kernel> f32_q8_0; // prefill (baseline) + std::map, cl_kernel> f32_q8_0_split; // N_SPLIT>1 variant + std::map, int> f32_q8_0_split_wg_size; // wg_size = bm*n_split + std::map, int> f32_q8_0_split_nkv_threshold; // use split when n_kv >= this + std::map, int> f32_q8_0_split_bm; // per-split BLOCK_M + // f32 Q / native q4_0 KV + std::map, cl_kernel> f32_q4_0_q1; + std::map, cl_kernel> f32_q4_0_q1_vec; // DV-split + multi-subgroup decode + std::map, cl_kernel> f32_q4_0_q1_split; + // kv-head-coalesced + flash-decoding split for q4_0 kv (dp4a K dot) + std::map, cl_kernel> f32_q4_0_q1_vec_mq_split; + std::map, cl_kernel> f32_q4_0_q1_vec_mq_split_g8; + // Cluster-parallel q4_0 decode + std::map, cl_kernel> f32_q4_0_q1_vec_mq_split_g8_c8; + std::map, cl_kernel> f32_q4_0_q1_vec_mq_split_c8; + std::map, cl_kernel> f32_q4_0; + std::map, cl_kernel> f32_q4_0_split; + std::map, int> f32_q4_0_split_wg_size; + std::map, int> f32_q4_0_split_nkv_threshold; + std::map, int> f32_q4_0_split_bm; + // shared: flash-decoding merge + prefill prepass (kv-pad, mask-pad, blk class) + std::map, cl_kernel> f32_merge; + std::map, cl_kernel> kv_pad_f16; + std::map, cl_kernel> mask_pad_f16; + std::map, cl_kernel> blk_f16; + // generic prefill tile dims (f16 / f32 paths) + std::map, int> bm; + std::map, int> bn; + // attempted (variant, (dk, dv)) + // all attempted FA kernels appear here, but those not registered failed compilation + std::set>> variant_attempted; + + // FA bin kernels +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel_flash_attn_f32_f16_bin; + + cl_kernel kernel_repack_q_for_wmm; + cl_kernel kernel_repack_k_for_wmm; + cl_kernel kernel_repack_v_for_wmm; + cl_kernel kernel_repack_mask_for_wmm; +#endif +}; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +struct ggml_cl_adreno_xmem_attn_scratch { + cl_mem q_img = nullptr; + cl_mem k_img = nullptr; + cl_mem v_img = nullptr; + cl_mem out_img = nullptr; + cl_mem k_transpose_buf = nullptr; + cl_mem k_transpose_img1d = nullptr; + cl_mem k_packed_buf = nullptr; + cl_mem v_packed_buf = nullptr; + cl_mem score_buf = nullptr; + cl_mem prob_buf = nullptr; + cl_mem score_img1d = nullptr; + cl_mem prob_img1d = nullptr; + cl_mem softmax_stats_img2d = nullptr; + cl_mem xmem_qk = nullptr; + cl_mem xmem_pv = nullptr; + + int n_q = 0; + int n_kv = 0; + int n_kv_padded = 0; + int d_head_q = 0; + int d_head_v = 0; + int q_width = 0; + int kv_heads_total = 0; +}; + +struct ggml_cl_adreno_xmem_attn_state { + bool compiled = false; + bool logged = false; + + cl_kernel kernel_q_f32_to_img_scaled = nullptr; + cl_kernel kernel_kv_f32_to_img_gqa = nullptr; + cl_kernel kernel_kv_f16_to_img_gqa = nullptr; + cl_kernel kernel_img_to_f32 = nullptr; + cl_kernel kernel_k_gather = nullptr; + cl_kernel kernel_pack_k = nullptr; + cl_kernel kernel_qk_gemm = nullptr; + cl_kernel kernel_softmax_reduce_basic = nullptr; + cl_kernel kernel_softmax_apply_basic = nullptr; + cl_kernel kernel_mask_scores = nullptr; + cl_kernel kernel_pack_v = nullptr; + cl_kernel kernel_pv_gemm = nullptr; + + ggml_cl_adreno_xmem_attn_scratch scratch; }; +#endif // backend context struct ggml_backend_opencl_context { @@ -383,41 +643,138 @@ struct ggml_backend_opencl_context { cl_device_id device; std::string device_name; + ggml_cl_version platform_version; + ggml_cl_version opencl_c_version; + + // argmax and argsort are loaded in supports_op; argsort availability depends on the compiled kernel's workgroup limit. + bool kernels_loaded_argmax = false; + bool kernels_loaded_argsort = false; + // rest of the kernels are currently always loaded in alloc_buffer. + bool kernels_loaded = false; + std::string driver_version; GPU_FAMILY gpu_family; ADRENO_GPU_GEN adreno_gen; cl_int alignment; + size_t global_mem_size; size_t max_alloc_size; size_t max_workgroup_size; bool fp16_support; bool has_vector_subgroup_broadcast; + bool has_khr_subgroup_shuffle = false; + bool has_subgroup_shuffle = false; // cl_khr_subgroup_shuffle or cl_qcom_subgroup_shuffle + bool has_integer_dot = false; // cl_khr_integer_dot_product or cl_qcom_dot_product8 + bool has_qcom_subgroup_shuffle = false; // specifically cl_qcom_subgroup_shuffle bool disable_fusion; + bool fuse_mm_glu = true; // opt-out GGML_OPENCL_FUSE_MM_GLU=0 (byte-identical gate+up GEMV + GLU, q4_K FFN) + bool fuse_rms_add = true; // opt-out GGML_OPENCL_FUSE_RMS_ADD=0 (fused rms_norm*w + residual) + bool f16_mrow = true; // opt-out GGML_OPENCL_F16_MROW=0 (multi-row-per-WG f16 decode GEMV for attn proj + lm_head) + int f16_mrow_rpt = 1; // GGML_OPENCL_F16_MROW_RPT={1,2,4,8,16} rows-per-subgroup register blocking + + // ragged moe, use int to directly pass to kernel + cl_uint adreno_use_moe_ragged; + cl_uint adreno_moe_ragged_skip_gran; + cl_uint adreno_use_moe_ragged_dp4; + + // whether fuse moe combine + cl_uint fuse_moe_combine; + + // whether to fold the MoE bias adds into swiglu_oai + cl_uint fuse_moe_bias_glu; + + // whether to fold the MoE down-projection bias add into the combine + cl_uint fuse_moe_bias_combine; bool adreno_has_large_buffer; bool adreno_use_large_buffer; + bool adreno_use_bin_kernels; + get_adreno_bin_kernel_func_t get_adreno_bin_kernel_func = nullptr; ggml_cl_compiler_version adreno_cl_compiler_version; + // The q6_K flat mul_mat codegen workarounds are needed by old E031 compilers only. + bool q6_k_flat_old_compiler; + + std::string kernel_compile_opts; // cached for lazy-compiled kernels. int adreno_wave_size; cl_bool non_uniform_workgroups; size_t image_max_buffer_size; + size_t image2d_max_width; + size_t image2d_max_height; + + cl_device_svm_capabilities svm_caps; cl_context context; cl_command_queue queue; + // On-disk compiled-program cache (see GGML_OPENCL_KERNEL_CACHE_DIR). + cl_program_cache_state program_cache; + bool program_cache_initialized = false; + // prealloc buffers for transposing weights and activations ggml_cl_buffer prealloc_quant_trans; ggml_cl_buffer prealloc_scales_trans; ggml_cl_buffer prealloc_act_trans; + // q8_1-quantized reordered MoE activations for the dp4a prefill GEMM. + ggml_cl_buffer prealloc_moe_qa; // int8 quants [tok_slots * ne00] + ggml_cl_buffer prealloc_moe_da; // per-block d [tok_slots * ne00/32] (half) + ggml_cl_buffer prealloc_moe_sa; // per-block s [tok_slots * ne00/32] (half) + // per-tile transposed qa/da and on-chip scratch for the ldck dp4a MoE GEMM + ggml_cl_buffer prealloc_moe_qa_trans; // int8 quants [tok_slots * ne00] + ggml_cl_buffer prealloc_moe_da_trans; // per-block d [tok_slots * ne00/32] (float) + ggml_cl_buffer prealloc_moe_qda_on_chip; + // scratch copy of the router weights to avoid dst aliasing + ggml_cl_buffer prealloc_moe_combine_w; + ggml_cl_buffer prealloc_splitk_partial; // [ksplit * M] partials for split-K GEMV + + // pool of persistent image1d_buffer views over kv-cache layers, keyed by + // (parent buffer, offset within parent) + // used by the img-variant KQ/KQV dispatch paths to avoid per-call + // clCreateSubBuffer + clCreateImage + pending-release-queue on long-context decode + struct ImagePoolKey { + uintptr_t buf; + uint64_t offset; + bool operator<(const ImagePoolKey & o) const { + if (buf != o.buf) return buf < o.buf; + return offset < o.offset; + } + }; + struct ImagePoolEntry { + cl_mem sub_buffer = nullptr; + cl_mem image = nullptr; + size_t k_bytes = 0; + cl_channel_type channel_data_type = CL_FLOAT; + }; + std::map kq_img_pool; + std::map kqv_img_pool; + + // pool for the on-device f16 buffer for kv-cache with non-FA quantized-K (q8_0/q4_0) + std::map dequant_f16_pool; // prealloc buffers for src0 and src1 ggml_cl_buffer prealloc_src0; ggml_cl_buffer prealloc_src1; + ggml_cl_buffer prealloc_top_k; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer prealloc_adreno_xmem_const; + bool adreno_xmem_gemm_enabled = false; +#endif + + // prealloc buffers for MoE router table preprocess + bool toggle_reorder = false; + ggml_cl_buffer prealloc_post_router; + ggml_cl_buffer prealloc_emap; + ggml_cl_buffer prealloc_hist; + ggml_cl_buffer prealloc_tile_offset; + ggml_cl_buffer prealloc_total_tiles; + ggml_cl_buffer prealloc_slot_counter; cl_program program_add; cl_program program_add_id; + cl_program program_moe_add_id_glu; cl_program program_clamp; cl_program program_cvt; cl_program program_diag_mask_inf; @@ -476,6 +833,7 @@ struct ggml_backend_opencl_context { cl_program program_mul_mv_id_mxfp4_f32_flat; cl_program program_mul_mm_f32_f32_l4_lm; cl_program program_mul_mm_f16_f32_l4_lm; + cl_program program_mul_mm_f16_f32_l4_lm_n8; cl_program program_mul_mm_q8_0_f32_l4_lm; cl_kernel kernel_add, kernel_add_row, kernel_add_f16, kernel_add_row_f16; @@ -483,6 +841,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_div, kernel_div_row, kernel_div_f16, kernel_div_row_f16; cl_kernel kernel_sub, kernel_sub_row, kernel_sub_f16, kernel_sub_row_f16; cl_kernel kernel_add_id; + cl_kernel kernel_add_id_add_id_swiglu_oai; cl_kernel kernel_scale_f32, kernel_scale_f32_4; cl_kernel kernel_sqr_cont_f32, kernel_sqr_cont_f32_4, kernel_sqr_cont_f16, kernel_sqr_cont_f16_4; cl_kernel kernel_sqrt_cont_f32, kernel_sqrt_cont_f32_4, kernel_sqrt_cont_f16, kernel_sqrt_cont_f16_4; @@ -495,49 +854,97 @@ struct ggml_backend_opencl_context { cl_kernel kernel_sigmoid_f32, kernel_sigmoid_f16; cl_kernel kernel_tri; cl_kernel kernel_fill; - cl_kernel kernel_clamp; - cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_geglu_erf, kernel_geglu_quick, - kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_geglu_erf_f16, kernel_geglu_quick_f16; + cl_kernel kernel_clamp, kernel_clamp_nc; + cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_swiglu_clamp, kernel_geglu_erf, + kernel_geglu_quick, kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_swiglu_clamp_f16, + kernel_geglu_erf_f16, kernel_geglu_quick_f16; cl_kernel kernel_norm, kernel_norm_mul_add; cl_kernel kernel_rms_norm, kernel_rms_norm_mul; + cl_kernel kernel_rms_norm_mul_add = nullptr; // fused rms_norm(x)*w + b (residual) cl_kernel kernel_l2_norm_f32; cl_kernel kernel_group_norm, kernel_group_norm_mul_add; cl_kernel kernel_diag_mask_inf, kernel_diag_mask_inf_8; cl_kernel kernel_diag_f32; cl_kernel kernel_soft_max, kernel_soft_max_4; cl_kernel kernel_soft_max_f16, kernel_soft_max_4_f16; - std::map, cl_kernel> kernels_flash_attn_f16; - std::map, cl_kernel> kernels_flash_attn_f16_q1; - std::map, cl_kernel> kernels_flash_attn_f32; - std::map, cl_kernel> kernels_flash_attn_f32_q1; - std::map, cl_kernel> kernels_flash_attn_f32_f16; - std::map, cl_kernel> kernels_flash_attn_f32_f16_q1; - std::map, int> kernels_flash_attn_bm; - std::map, int> kernels_flash_attn_bn; - cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0; + ggml_opencl_fa_kernels fa; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_adreno_xmem_attn_state adreno_xmem_attn; +#endif + cl_kernel kernel_top_k_f32, kernel_top_k_merge_f32; + cl_kernel kernel_lightning_indexer; + cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_i32, kernel_get_rows_q4_0; cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32; + cl_kernel kernel_set_rows_q8_0_i64, kernel_set_rows_q8_0_i32; + cl_kernel kernel_set_rows_q8_0_soa_i64, kernel_set_rows_q8_0_soa_i32; + cl_kernel kernel_set_rows_q4_0_i64, kernel_set_rows_q4_0_i32; + cl_kernel kernel_set_rows_q4_0_soa_i64, kernel_set_rows_q4_0_soa_i32; cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16; cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16; - cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32; + cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32, kernel_cpy_i32_f32, kernel_cpy_f32_i32; + cl_kernel kernel_cpy_f32_f32_pack = nullptr; + cl_kernel kernel_cpy_f32_f32_flat = nullptr; cl_kernel kernel_mul_mat_f32_f32; cl_kernel kernel_mul_mat_f16_f16; cl_kernel kernel_mul_mat_f16_f32_1row; + cl_program program_mul_mv_f16_f32_mrow; + cl_kernel kernel_mul_mat_f16_f32_mrow = nullptr; // multi-row decode GEMV (attn proj + lm_head) + cl_kernel kernel_mul_mat_f16_f32_mrow_r2 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_mrow_r4 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_mrow_h8 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_mrow_h8r2 = nullptr; cl_kernel kernel_mul_mat_f16_f32; cl_kernel kernel_mul_mat_f16_f32_l4; + cl_kernel kernel_mul_mat_f16_f32_l4_dr; + cl_kernel kernel_mul_mat_f16_f32_l4_dr_ls; + cl_kernel kernel_mul_mat_f16_f32_l4_dr_lq; + cl_kernel kernel_mul_mat_f16_f32_l4_x8 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_pair = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa4 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa4_img = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_y8 = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_y8_gqa = nullptr; + cl_kernel kernel_mul_mat_f16_f32_l4_y8_gqa_img = nullptr; cl_kernel kernel_mul_mat_f16_f32_tiled; + cl_kernel kernel_adreno_xmem_pack_src_f32; + cl_kernel kernel_adreno_xmem_prepack_weight_f16; + cl_kernel kernel_gemm_xmem_f16_f32_os8; + cl_kernel kernel_adreno_xmem_store_dst_f32; cl_kernel kernel_mul_mm_f16_f32_kqv; cl_kernel kernel_mul_mm_f16_f32_kq; cl_kernel kernel_mul_mat_q4_0_f32, kernel_mul_mat_q4_0_f32_v; + cl_kernel kernel_convert_block_q1_0, kernel_restore_block_q1_0; cl_kernel kernel_convert_block_q4_0, kernel_restore_block_q4_0; + cl_kernel kernel_convert_block_q4_0_trans4_ns, kernel_restore_block_q4_0_trans4_ns; cl_kernel kernel_convert_block_q4_1, kernel_restore_block_q4_1; + cl_kernel kernel_convert_block_q4_1_trans4_ns, kernel_restore_block_q4_1_trans4_ns; + cl_kernel kernel_convert_block_q5_0, kernel_restore_block_q5_0; + cl_kernel kernel_convert_block_q5_0_trans4_ns, kernel_restore_block_q5_0_trans4_ns; + cl_kernel kernel_convert_block_q5_1, kernel_restore_block_q5_1; + cl_kernel kernel_convert_block_q5_1_trans4_ns, kernel_restore_block_q5_1_trans4_ns; + cl_kernel kernel_convert_block_q4_k_trans4_ns, kernel_restore_block_q4_k_trans4_ns; + cl_kernel kernel_convert_block_q5_k_trans4_ns, kernel_restore_block_q5_k_trans4_ns; + cl_kernel kernel_convert_block_q6_k_trans4_ns, kernel_restore_block_q6_k_trans4_ns; cl_kernel kernel_convert_block_mxfp4, kernel_convert_block_mxfp4_trans, kernel_restore_block_mxfp4, kernel_restore_block_mxfp4_trans; + cl_kernel kernel_convert_block_mxfp4_trans4_ns, kernel_restore_block_mxfp4_trans4_ns; cl_kernel kernel_convert_block_q8_0, kernel_restore_block_q8_0, kernel_restore_block_q8_0_trans; + cl_kernel kernel_dequant_q8_0_f16_view_aos; + cl_kernel kernel_dequant_q8_0_f32_view_aos; + cl_kernel kernel_dequant_q4_0_f16_view_aos; + cl_kernel kernel_dequant_q4_0_f32_view_aos; cl_kernel kernel_convert_block_q6_K_noshuffle, kernel_restore_block_q6_K_noshuffle; + cl_kernel kernel_convert_bf16_to_f16, kernel_convert_f16_to_bf16; cl_kernel kernel_mul_mat_q4_0_f32_8x_flat; cl_kernel kernel_convert_block_q4_0_noshuffle; cl_kernel kernel_restore_block_q4_0_noshuffle; cl_kernel kernel_convert_block_q4_1_noshuffle; cl_kernel kernel_restore_block_q4_1_noshuffle; + cl_kernel kernel_convert_block_q5_0_noshuffle; + cl_kernel kernel_restore_block_q5_0_noshuffle; + cl_kernel kernel_convert_block_q5_1_noshuffle; + cl_kernel kernel_restore_block_q5_1_noshuffle; cl_kernel kernel_convert_block_q4_K_noshuffle; cl_kernel kernel_restore_block_q4_K_noshuffle; cl_kernel kernel_convert_block_q4_K, kernel_restore_block_q4_K; @@ -545,9 +952,29 @@ struct ggml_backend_opencl_context { cl_kernel kernel_convert_block_q5_K_noshuffle; cl_kernel kernel_restore_block_q5_K_noshuffle; cl_kernel kernel_convert_block_q6_K, kernel_restore_block_q6_K; + cl_kernel kernel_convert_block_iq4_nl, kernel_restore_block_iq4_nl; + cl_kernel kernel_convert_block_iq4_nl_noshuffle; + cl_kernel kernel_restore_block_iq4_nl_noshuffle; + cl_kernel kernel_mul_mv_q1_0_f32, kernel_mul_mv_q1_0_f32_flat; cl_kernel kernel_mul_mat_q4_0_f32_1d_8x_flat, kernel_mul_mat_q4_0_f32_1d_16x_flat; cl_kernel kernel_mul_mv_q4_1_f32; cl_kernel kernel_mul_mv_q4_1_f32_flat; + cl_kernel kernel_mul_mv_q5_0_f32; + cl_kernel kernel_mul_mv_q5_0_f32_flat; + cl_kernel kernel_mul_mv_q5_1_f32; + cl_kernel kernel_mul_mv_q5_1_f32_flat; + cl_kernel kernel_convert_block_q2_K, kernel_restore_block_q2_K; + cl_kernel kernel_mul_mv_q2_K_f32_flat; + cl_kernel kernel_mul_mm_q2_k_f32_l4_lm; + cl_kernel kernel_convert_block_q2_k_noshuffle; + cl_kernel kernel_restore_block_q2_k_noshuffle; + cl_kernel kernel_gemv_noshuffle_q2_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q2_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q2_k_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow = nullptr; + int q2_k_mv_rows = 4; + int q2_k_mv_nsg = 4; + cl_kernel kernel_mul_mv_q2_K_f32; cl_kernel kernel_mul_mv_q4_K_f32; cl_kernel kernel_mul_mv_q4_K_f32_flat; cl_kernel kernel_mul_mv_q5_K_f32; @@ -556,8 +983,12 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mv_q6_K_f32_flat; cl_kernel kernel_mul_mv_mxfp4_f32, kernel_mul_mv_mxfp4_f32_flat; cl_kernel kernel_mul_mv_q8_0_f32, kernel_mul_mv_q8_0_f32_flat; + cl_kernel kernel_mul_mv_iq4_nl_f32; + cl_kernel kernel_mul_mv_iq4_nl_f32_flat; cl_kernel kernel_solve_tri_f32; cl_kernel kernel_im2col_f32, kernel_im2col_f16; + cl_kernel kernel_im2col_3d_f32, kernel_im2col_3d_f16; + cl_kernel kernel_argmax_f32_i32; cl_kernel kernel_argsort_f32_i32; cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4; cl_kernel kernel_cumsum_blk, kernel_cumsum_add; @@ -571,36 +1002,94 @@ struct ggml_backend_opencl_context { cl_kernel kernel_exp_f16, kernel_exp_f16_4, kernel_exp_f16_nc; cl_kernel kernel_expm1_f32, kernel_expm1_f32_4, kernel_expm1_f32_nc; cl_kernel kernel_expm1_f16, kernel_expm1_f16_4, kernel_expm1_f16_nc; + cl_kernel kernel_abs_f32, kernel_abs_f32_4, kernel_abs_f32_nc; + cl_kernel kernel_abs_f16, kernel_abs_f16_4, kernel_abs_f16_nc; + cl_kernel kernel_sgn_f32, kernel_sgn_f32_4, kernel_sgn_f32_nc, kernel_sgn_f16, kernel_sgn_f16_4, kernel_sgn_f16_nc; + cl_kernel kernel_step_f32, kernel_step_f32_4, kernel_step_f32_nc, kernel_step_f16, kernel_step_f16_4, kernel_step_f16_nc; + cl_kernel kernel_elu_f32, kernel_elu_f32_4, kernel_elu_f32_nc, kernel_elu_f16, kernel_elu_f16_4, kernel_elu_f16_nc; + cl_kernel kernel_hardswish_f32, kernel_hardswish_f32_4, kernel_hardswish_f32_nc, kernel_hardswish_f16, kernel_hardswish_f16_4, kernel_hardswish_f16_nc; + cl_kernel kernel_hardsigmoid_f32, kernel_hardsigmoid_f32_4, kernel_hardsigmoid_f32_nc, kernel_hardsigmoid_f16, kernel_hardsigmoid_f16_4, kernel_hardsigmoid_f16_nc; + cl_kernel kernel_floor_f32, kernel_floor_f32_4, kernel_floor_f32_nc, kernel_floor_f16, kernel_floor_f16_4, kernel_floor_f16_nc; + cl_kernel kernel_ceil_f32, kernel_ceil_f32_4, kernel_ceil_f32_nc, kernel_ceil_f16, kernel_ceil_f16_4, kernel_ceil_f16_nc; + cl_kernel kernel_round_f32, kernel_round_f32_4, kernel_round_f32_nc, kernel_round_f16, kernel_round_f16_4, kernel_round_f16_nc; + cl_kernel kernel_trunc_f32, kernel_trunc_f32_4, kernel_trunc_f32_nc, kernel_trunc_f16, kernel_trunc_f16_4, kernel_trunc_f16_nc; cl_kernel kernel_softplus_f32, kernel_softplus_f32_4, kernel_softplus_f32_nc; cl_kernel kernel_softplus_f16, kernel_softplus_f16_4, kernel_softplus_f16_nc; cl_kernel kernel_upscale; cl_kernel kernel_upscale_bilinear; - cl_kernel kernel_concat_f32; + cl_kernel kernel_concat_b1, kernel_concat_b2, kernel_concat_b4, kernel_concat_b8, kernel_concat_b4_pack; cl_kernel kernel_conv_2d_f16; cl_kernel kernel_conv_2d_f32; cl_kernel kernel_conv_2d_f16_f32; + cl_kernel kernel_conv2d_dw_f32_f32, kernel_conv2d_dwf16_f32; cl_kernel kernel_ssm_conv_f32_f32, kernel_ssm_conv_f32_f32_4; + // [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1. + // tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4). + cl_kernel kernel_gated_delta_net_f32[4][2][2] = {}; + cl_kernel kernel_ssm_scan_f32 = nullptr; + cl_kernel kernel_ssm_scan_f32_mamba2_d128 = nullptr; + cl_kernel kernel_ssm_scan_f32_mamba2_d256 = nullptr; + cl_kernel kernel_timestep_embedding; + cl_kernel kernel_gemv_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns_bin; + cl_kernel kernel_gemm_moe_q8_0_f32_ns; + cl_kernel kernel_gemv_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns_bin; + cl_kernel kernel_gemv_moe_q5_0_f32_ns, kernel_gemm_moe_q5_0_f32_ns; + cl_kernel kernel_gemv_moe_q5_1_f32_ns, kernel_gemm_moe_q5_1_f32_ns; + cl_kernel kernel_gemv_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns_bin; + cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV (opt-in) + cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) prefill GEMM variant + cl_kernel kernel_moe_reorder_quant_a_q8_1; // fused reorder + q8_1 quant for the dp4a GEMM + cl_kernel kernel_transpose_activation_q8; // per-tile transpose of q8_1 activations for the ldck dp4a GEMM + cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr; // generic dp4a MoE GEMM (MOE_QT=80), opt-in + cl_kernel kernel_moe_expand_scale_q8_0 = nullptr; // q8_0 per-block d -> uniform scale[16] + cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr; // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in + cl_kernel kernel_moe_expand_scale_q5_0 = nullptr; // q5_0 d -> uniform scale[2]/min[1] per 32-block + cl_kernel kernel_gemm_moe_q8_1_dp4a_q5k = nullptr; // generic dp4a MoE GEMM (MOE_QT=5, q5_K), opt-in + cl_kernel kernel_moe_expand_scale_q5_K = nullptr; // q5_K 6-bit s[] -> uniform scale[16]/min[8] + cl_kernel kernel_gemv_moe_q5_k_f32_ns, kernel_gemm_moe_q5_k_f32_ns; + cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns_bin; + cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) q6_K MoE prefill GEMM cl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32; + cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns_bin; + cl_kernel kernel_gemv_moe_mxfp4_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV + cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr; // dp4a (int8) mxfp4 MoE prefill GEMM + cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMM + cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) mxfp4 MoE prefill GEMM + cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr; // binary dp4a (int8) mxfp4 MoE prefill GEMM using ldck + cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_0 MoE prefill GEMM + cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_k MoE prefill GEMM + cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q6_k MoE prefill GEMM + cl_kernel kernel_moe_reorder_b; + cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter; + cl_kernel kernel_moe_scatter_stable = nullptr; // deterministic slot assignment + cl_kernel kernel_moe_zero_dst; + cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum + cl_kernel kernel_moe_combine_bias_f32 = nullptr; // same, with the down-projection bias add folded in cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat; cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat; cl_kernel kernel_mul_mv_id_mxfp4_f32; cl_kernel kernel_mul_mv_id_mxfp4_f32_flat; cl_kernel kernel_mul_mm_f32_f32_l4_lm; + cl_kernel kernel_gemv_f32_f32_mc; // multi-column (small-N) f32 GEMV for spec/MTP verify cl_kernel kernel_mul_mm_f16_f32_l4_lm; + cl_kernel kernel_mul_mm_f16_f32_l4_lm_n8; + cl_kernel kernel_mul_mm_q1_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_1_f32_l4_lm; + cl_kernel kernel_mul_mm_q5_0_f32_l4_lm; + cl_kernel kernel_mul_mm_q5_1_f32_l4_lm; cl_kernel kernel_mul_mm_q8_0_f32_l4_lm; cl_kernel kernel_mul_mm_q4_k_f32_l4_lm; cl_kernel kernel_mul_mm_q5_k_f32_l4_lm; cl_kernel kernel_mul_mm_q6_k_f32_l4_lm; + cl_kernel kernel_mul_mm_iq4_nl_f32_l4_lm; std::vector profiling_info; + std::vector profiling_results; - void write_profiling_info() { - FILE * fperf = fopen("cl_profiling.csv", "w"); - if (!fperf) { - GGML_LOG_ERROR("Failed to open cl_profiling.csv\n"); + void flush_profiling_batch() { + if (profiling_info.empty()) { return; } @@ -624,6 +1113,7 @@ struct ggml_backend_opencl_context { CL_CHECK(clGetEventProfilingInfo( info.evt, CL_PROFILING_COMMAND_COMPLETE, sizeof(cl_ulong), &cmd_complete, NULL)); CL_CHECK(clReleaseEvent(info.evt)); + info.evt = nullptr; char kernel_name[512]; CL_CHECK(clGetKernelInfo(info.kernel, CL_KERNEL_FUNCTION_NAME, @@ -641,10 +1131,26 @@ struct ggml_backend_opencl_context { info.cmd_complete_duration_ns = cmd_complete - cmd_end; info.cmd_total_duration_ns = cmd_complete - cmd_queued; } + profiling_results.insert(profiling_results.end(), + std::make_move_iterator(profiling_info.begin()), + std::make_move_iterator(profiling_info.end())); + profiling_info.clear(); + } + + void write_profiling_info() { + if (profiling_results.empty()) { + return; + } // Dump a csv + FILE * fperf = fopen("cl_profiling.csv", "w"); + if (!fperf) { + GGML_LOG_ERROR("Failed to open cl_profiling.csv\n"); + return; + } + fprintf(fperf, "op name, kernel name, exec duration (ms), global size, local size, output size\n"); - for (const ProfilingInfo & info : profiling_info) { + for (const ProfilingInfo & info : profiling_results) { fprintf(fperf, "%s,%s,%f,%zux%zux%zu,%zux%zux%zu,%zux%zux%zux%zu\n", info.op_name.c_str(), info.kernel_name.c_str(), info.cmd_duration_ns/1.e6f, @@ -655,14 +1161,14 @@ struct ggml_backend_opencl_context { fclose(fperf); // Dump a simple chrome trace - FILE* ftrace = fopen("cl_trace.json", "w"); + FILE * ftrace = fopen("cl_trace.json", "w"); if (!ftrace) { GGML_LOG_ERROR("Failed to open cl_trace.json\n"); return; } fprintf(ftrace, "[\n"); - for (const ProfilingInfo & info : profiling_info) { + for (const ProfilingInfo & info : profiling_results) { fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"B\", \"ts\": %" PRIu64 ", \"pid\": \"\", \"tid\": \"Host\"},\n", info.kernel_name.c_str(), info.cmd_queued/1000); fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"E\", \"ts\": %" PRIu64 ", \"pid\": \"\", \"tid\": \"Host\"},\n", @@ -673,6 +1179,7 @@ struct ggml_backend_opencl_context { fprintf(ftrace, "{\"name\": \"%s\", \"cat\": \"OpenCL\", \"ph\": \"E\", \"ts\": %" PRIu64 ", \"pid\": \"\", \"tid\": \"Device\"},\n", info.kernel_name.c_str(), info.cmd_end/1000); } + fprintf(ftrace, "]\n"); fclose(ftrace); } @@ -687,18 +1194,47 @@ struct ggml_backend_opencl_context { } void enqueue_ndrange_kernel(cl_kernel kernel, cl_uint work_dim, size_t *global_work_size, size_t *local_work_size, const ggml_tensor * tensor) { + // From the spec on clEnqueueNDRangeKernel: + // If the device associated with command_queue is an OpenCL 2.1 or newer device, + // and global_work_size is NULL or the value in any passed dimension is zero, + // then the kernel command will trivially succeed after its event dependencies + // are satisfied and subsequently update its completion event. + // So this ensures such cases always return trivially without causing errors in + // case of an older device. + for (cl_uint i = 0; i < work_dim; i++) { + if (global_work_size[i] == 0) { + return; + } + } #ifdef GGML_OPENCL_PROFILING cl_event evt; CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, work_dim, NULL, global_work_size, local_work_size, 0, NULL, &evt)); profiling_info.emplace_back(); populateProfilingInfo(profiling_info.back(), evt, kernel, work_dim, global_work_size, local_work_size, tensor); + if (profiling_info.size() >= 2048) { + flush_profiling_batch(); + } #else GGML_UNUSED(tensor); CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, work_dim, NULL, global_work_size, local_work_size, 0, NULL, NULL)); #endif } + const void * get_adreno_bin_kernel(const std::string &kernel_name, size_t *bin_size) const { + if (!get_adreno_bin_kernel_func) { + return nullptr; + } + + size_t sz = 0; + const void * kernel_bin = get_adreno_bin_kernel_func( + kernel_name.c_str(), device_name.c_str(), driver_version.c_str(), &sz); + if (bin_size) { + *bin_size = sz; + } + return kernel_bin; + } + #ifdef GGML_OPENCL_USE_ADRENO_KERNELS // Transpose kernels cl_program program_transpose; @@ -712,36 +1248,138 @@ struct ggml_backend_opencl_context { cl_kernel kernel_transpose_16_4x1; // Gemm and Gemv related programs, kernels, etc - cl_program program_CL_gemm; - cl_program program_CL_gemv_general; - cl_program program_CL_gemv_4096_1_11008; - cl_program program_CL_gemv_4096_1_4096; - cl_program program_CL_gemv_11008_1_4096; - cl_program program_CL_gemv_32000_1_4096; - cl_kernel CL_mul_mat_Ab_Bi_8x4; - cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_general; - cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_11008; - cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_4096; - cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_11008_1_4096; - cl_kernel CL_mul_mat_vec_q4_0_f32_1d_4x_flat_32000_1_4096; + cl_kernel kernel_gemm_noshuffle_q4_0_f32; + cl_kernel kernel_gemv_noshuffle_q4_0_f32; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) + cl_kernel kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_32b_trans; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; + cl_kernel kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; cl_kernel kernel_gemv_noshuffle_q4_1_f32; + cl_kernel kernel_gemv_noshuffle_q4_1_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q4_1_f32; - cl_kernel kernel_mul_mm_q8_0_f32_8x4; - cl_kernel CL_mul_mat_vec_q8_0_f32; + cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin; + cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in) + cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = nullptr; + cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in) + cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr; + cl_kernel kernel_gemv_noshuffle_q8_0_f32; + cl_kernel kernel_gemv_noshuffle_q8_0_f32_splitk; // split-K across WGs (small-M decode) + cl_kernel kernel_gemm_noshuffle_q1_0_f32; + cl_kernel kernel_gemv_noshuffle_q1_0_f32; + cl_kernel kernel_gemv_noshuffle_q1_0_f32_nsg1 = nullptr; + int min_m_gemv_nsg1_q1_0 = 0; cl_kernel kernel_gemv_noshuffle_q4_k_f32; + cl_kernel kernel_gemv_noshuffle_q4_k_f32_o4; // 4-output-per-WI, long-vocab lm_head + cl_kernel kernel_gemv_noshuffle_q4_k_f32_tiled; // tiled-wide layout (opt-in) + cl_kernel kernel_gemv_noshuffle_q4_k_f32_splitk; // split-K across WGs (small-M decode) + cl_kernel kernel_gemv_splitk_reduce_f32; // sums split-K per-slice partials + cl_kernel kernel_gemv_noshuffle_q4_k_f32_glu; // fused gate+up GEMV + GLU (FFN) + cl_kernel kernel_convert_block_q4_k_tiled_ns; // tiled-wide convert (opt-in) + cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q4_k_f32; + cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans; + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = nullptr; + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow = nullptr; + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg = nullptr; + int tile_n_dp4a_q4_k = 32; + int tile_n_dp4a_narrow_q4_k = 32; + int max_n_dp4a_narrow_q4_k = 16; + cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM + cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = nullptr; + cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM + cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = nullptr; + cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass + cl_kernel kernel_gemm_noshuffle_q4_k_f32_r1; + cl_kernel kernel_gemm_noshuffle_q4_k_f32_kimg; + cl_kernel kernel_gemm_noshuffle_q4_k_f32_cok; cl_kernel kernel_gemv_noshuffle_q6_K_f32; + cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4; + cl_kernel kernel_gemv_noshuffle_q6_K_f32_o4_global; // weights via __global (opt-in) + cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled; // tiled-wide layout (opt-in) + cl_kernel kernel_gemv_noshuffle_q6_K_f32_tiled_mc3; // tiled multi-column (N=3) verify lm_head + cl_kernel kernel_gemm_noshuffle_q6_K_f32_tiled; // batched (N>1) over the tiled layout + cl_kernel kernel_convert_block_q6_k_tiled_ns; // tiled-wide convert (opt-in) + cl_kernel kernel_gemv_noshuffle_q6_K_f32_mc3; // multi-column (N=3) verify GEMV cl_kernel kernel_gemm_noshuffle_q6_K_f32; + cl_kernel kernel_gemm_noshuffle_q6_K_f32_cok; + cl_kernel kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q6_k_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q5_k_f32; + cl_kernel kernel_gemv_noshuffle_q5_k_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP) cl_kernel kernel_gemm_noshuffle_q5_k_f32; + cl_kernel kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin; + cl_kernel kernel_gemv_noshuffle_q5_k_f32_32b_trans; + cl_kernel kernel_gemv_noshuffle_q5_0_f32; + cl_kernel kernel_gemm_noshuffle_q5_0_f32; + cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_0 prefill GEMM + cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = nullptr; // q5_0 dense dp4a, qs plane via texture (opt-in) + cl_kernel kernel_gemv_noshuffle_q5_1_f32; + cl_kernel kernel_gemm_noshuffle_q5_1_f32; + cl_kernel kernel_gemv_noshuffle_iq4_nl_f32; + cl_kernel kernel_gemm_noshuffle_iq4_nl_f32; + cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = nullptr; + cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr; + cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr; + int nsg_coopk_dp4a_q4_0 = 8; + int rows_coopk_dp4a_q4_0 = 4; + cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_c4 = nullptr; + cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_c2 = nullptr; + int nsg_coopk_dp4a_q4_k = 8; + int rows_coopk_dp4a_q4_k = 4; + cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 = nullptr; + cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 = nullptr; + int nsg_coopk_dp4a_q4_k_bin_c4 = 8; + int nsg_coopk_dp4a_q4_k_bin_c2 = 8; + cl_kernel kernel_gemm_coopk_q6_k_q8_1_dp4a_c4 = nullptr; + cl_kernel kernel_gemm_coopk_q6_k_q8_1_dp4a_c2 = nullptr; + int nsg_coopk_dp4a_q6_k = 8; + int rows_coopk_dp4a_q6_k = 4; #endif // GGML_OPENCL_USE_ADRENO_KERNELS void free() { + clFinish(queue); + ref_count--; if (ref_count == 0) { + if (fa.fd_partial.buffer) { + CL_CHECK(clReleaseMemObject(fa.fd_partial.buffer)); + fa.fd_partial.buffer = nullptr; + fa.fd_partial.size = 0; + } #ifdef GGML_OPENCL_PROFILING + flush_profiling_batch(); write_profiling_info(); - profiling_info.clear(); + profiling_results.clear(); +#endif + // release pooled image1d_buffer views over KV cache layers. + for (auto & kv : kq_img_pool) { + if (kv.second.image) { CL_CHECK(clReleaseMemObject(kv.second.image)); } + if (kv.second.sub_buffer) { CL_CHECK(clReleaseMemObject(kv.second.sub_buffer)); } + } + kq_img_pool.clear(); + for (auto & kv : kqv_img_pool) { + if (kv.second.image) { CL_CHECK(clReleaseMemObject(kv.second.image)); } + if (kv.second.sub_buffer) { CL_CHECK(clReleaseMemObject(kv.second.sub_buffer)); } + } + kqv_img_pool.clear(); + for (auto & kv : dequant_f16_pool) { + if (kv.second.image) { CL_CHECK(clReleaseMemObject(kv.second.image)); } + } + dequant_f16_pool.clear(); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_adreno_xmem_attn_release_scratch(this); #endif } } @@ -749,21 +1387,30 @@ struct ggml_backend_opencl_context { // All registered devices with a default device in the front. static std::vector g_ggml_backend_opencl_devices; +// All device contexts associated with the devices above. +// The devices live as long as the process, so do the contexts. +static std::vector> g_ggml_backend_opencl_dev_ctxs; inline std::string read_file(const std::string &path) { - std::ifstream ifs(path); - if (!ifs) { - return ""; - } - std::string text; - ifs.seekg(0, std::ios::end); - text.resize(ifs.tellg()); - ifs.seekg(0, std::ios::beg); - ifs.read(&text[0], text.size()); - return text; -} - -static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts) { + std::ifstream ifs(path); + if (!ifs) { + return ""; + } + std::string text; + ifs.seekg(0, std::ios::end); + text.resize(ifs.tellg()); + ifs.seekg(0, std::ios::beg); + ifs.read(&text[0], text.size()); + return text; +} + +// fatal=false returns NULL on compile failure instead of aborting; used for +// optional FA variants that may exhaust the Adreno compiler at large DK. +// when the compiler returns CL_OUT_OF_HOST_MEMORY/CL_OUT_OF_RESOURCES (seen with DK>=256/512) +// for FA programs, do clFinish the queue to free up resources, then rebuild (up to 3x) +// if retry_queue is provided +static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts, bool fatal, const char *tag = nullptr, cl_command_queue retry_queue = nullptr) { + if (tag) { GGML_LOG_INFO("ggml_opencl: compiling %s\n", tag); } cl_program p; char *program_log; size_t program_size; @@ -772,9 +1419,74 @@ static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, co program_size = strlen(program_buffer); - p = clCreateProgramWithSource(ctx, 1, (const char**)&program_buffer, &program_size, &err); + const int max_attempts = retry_queue ? 3 : 1; + for (int attempt = 0; attempt < max_attempts; ++attempt) { + p = clCreateProgramWithSource(ctx, 1, (const char**)&program_buffer, &program_size, &err); + if(err < 0) { + GGML_LOG_ERROR("OpenCL error creating program"); + if (fatal) exit(1); + return NULL; + } + + err = clBuildProgram(p, 0, NULL, compile_opts.c_str(), NULL, NULL); + if (err == CL_SUCCESS) { + return p; + } + + const bool transient = (err == CL_OUT_OF_HOST_MEMORY || err == CL_OUT_OF_RESOURCES); + if (retry_queue && transient && attempt + 1 < max_attempts) { + clReleaseProgram(p); + GGML_LOG_WARN("ggml_opencl: transient compile failure (err=%d)%s%s — clFinish + retry (%d/%d)\n", + err, tag ? " building " : "", tag ? tag : "", attempt + 2, max_attempts); + clFinish(retry_queue); // drain in-flight ops holding driver host-heap + continue; + } + + clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size); + program_log = (char*) malloc(log_size + 1); + program_log[log_size] = '\0'; + clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL); + GGML_LOG_ERROR("ggml_opencl: kernel compile error (err=%d)%s%s:\n\n%s\n", err, tag ? " building " : "", tag ? tag : "", program_log); + free(program_log); + clReleaseProgram(p); + if (fatal) { + exit(1); + } + return nullptr; + } + return NULL; +} + +static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts) { + cl_context ctx = backend_ctx->context; + cl_device_id dev = backend_ctx->device; + + // Try the on-disk binary cache first. Falls through silently on miss or + // any failure; never blocks the build path. Disabled cache => nullptr. + cl_program p_cached = cl_program_cache_try_load( + backend_ctx->program_cache, ctx, dev, program_buffer, compile_opts); + if (p_cached != nullptr) { + return p_cached; + } + + cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true); + + // Best-effort save of the freshly-built binary (no-op if cache disabled). + if (p != nullptr) { + cl_program_cache_try_save(backend_ctx->program_cache, p, dev, program_buffer, compile_opts); + } + return p; +} + +static cl_program build_program_from_binary(cl_context ctx, cl_device_id dev, const char* program_buffer, const std::string &compile_opts, size_t bin_size = 0) { + cl_program p; + char *program_log; + size_t log_size; + int err; + + p = clCreateProgramWithBinary(ctx, 1, &dev, &bin_size, (const unsigned char**)&program_buffer, NULL, &err); if(err < 0) { - GGML_LOG_ERROR("OpenCL error creating program"); + GGML_LOG_ERROR("OpenCL error creating program from binary"); exit(1); } @@ -792,106 +1504,384 @@ static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, co return p; } -static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_version opencl_c_version) { - cl_int err; +static void load_cl_kernels_argmax(ggml_backend_opencl_context *backend_ctx) { + auto opencl_c_std = + std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor); + std::string compile_opts = std::string("-cl-std=") + opencl_c_std; + + if (!backend_ctx->kernels_loaded_argmax) { + cl_int err; +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "argmax.cl.h" + }; +#else + const std::string kernel_src = read_file("argmax.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_argmax_f32_i32 = clCreateKernel(prog, "kernel_argmax_f32_i32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + backend_ctx->kernels_loaded_argmax = true; + } +} + +static std::string ggml_cl_q2_k_mm_options(const ggml_backend_opencl_context * backend_ctx) { + const bool x2 = backend_ctx->gpu_family == ADRENO && + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + int bk = x2 ? 16 : 32; + bool half = x2; + if (const char * env = getenv("GGML_OPENCL_LM_BK")) { + const int value = atoi(env); + if (value == 16 || value == 32) { + bk = value; + } + } + if (const char * env = getenv("GGML_OPENCL_LM_HALF")) { + half = atoi(env) != 0; + } + return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0); +} +static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) { // compiler options for general kernels auto opencl_c_std = - std::string("CL") + std::to_string(opencl_c_version.major) + "." + std::to_string(opencl_c_version.minor); + std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor); std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable -cl-unsafe-math-optimizations" " -cl-finite-math-only -cl-fast-relaxed-math"; - if (backend_ctx->adreno_use_large_buffer) { - compile_opts += " -qcom-enable-large-buffer "; - } - - GGML_LOG_INFO("ggml_opencl: loading OpenCL kernels"); - - // add - { + // argsort + if (!backend_ctx->kernels_loaded_argsort) { + cl_int err; #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "add.cl.h" + #include "argsort.cl.h" }; #else - const std::string kernel_src = read_file("add.cl"); + const std::string kernel_src = read_file("argsort.cl"); #endif - backend_ctx->program_add = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + backend_ctx->program_argsort_f32_i32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_add = clCreateKernel(backend_ctx->program_add, "kernel_add", &err), err)); - CL_CHECK((backend_ctx->kernel_add_row = clCreateKernel(backend_ctx->program_add, "kernel_add_row", &err), err)); - CL_CHECK((backend_ctx->kernel_add_f16 = clCreateKernel(backend_ctx->program_add, "kernel_add_f16", &err), err)); - CL_CHECK((backend_ctx->kernel_add_row_f16 = clCreateKernel(backend_ctx->program_add, "kernel_add_row_f16", &err), err)); - GGML_LOG_CONT("."); + CL_CHECK((backend_ctx->kernel_argsort_f32_i32 = clCreateKernel(backend_ctx->program_argsort_f32_i32, "kernel_argsort_f32_i32", &err), err)); + backend_ctx->kernels_loaded_argsort = true; } +} - // add_id - { -#ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "add_id.cl.h" - }; +static bool use_adreno_bin_kernels(ggml_backend_opencl_context * backend_ctx) { +#ifndef GGML_OPENCL_USE_ADRENO_BIN_KERNELS + GGML_UNUSED(backend_ctx); + return false; #else - const std::string kernel_src = read_file("add_id.cl"); -#endif - backend_ctx->program_add_id = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { + return false; + } + return backend_ctx->adreno_use_bin_kernels; +#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS +} - CL_CHECK((backend_ctx->kernel_add_id = clCreateKernel(backend_ctx->program_add_id, "kernel_add_id", &err), err)); - GGML_LOG_CONT("."); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_coopk_dp4a_enabled(const ggml_backend_opencl_context * backend_ctx) { + static const char * const env = getenv("GGML_OPENCL_COOPK_DP4A"); + if (env && *env && atoi(env) == 0) { + return false; + } + if (!backend_ctx->has_integer_dot) { + return false; + } + if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X) { + return false; + } + if (backend_ctx->adreno_cl_compiler_version.type == ADRENO_CL_COMPILER_TYPE::E17) { + return false; + } } + return true; +} - // tri +static bool ggml_cl_coopk_dp4a_narrow_on(const ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * op, int rows) { + if (!ggml_cl_coopk_dp4a_enabled(backend_ctx)) { + return false; + } + // Each lane handles rows output rows; subgroups split K for the same rows. + // Full row groups and 32-value K blocks avoid tail checks in the kernels. + return op->ne[1] >= 2 && op->ne[1] <= 4 + && rows > 0 && (op->src[0]->ne[1] % (64 * rows)) == 0 + && (op->src[0]->ne[0] % 32) == 0; +} + +// COOPK_NSG sizes the local reduction buffer, so a smaller workgroup requires a rebuild. +static cl_program ggml_cl_build_program_coopk(ggml_backend_opencl_context * backend_ctx, + const char * kernel_src, const char * kernel_name, + const std::string & opts_base, int & nsg) { + while (nsg >= 2) { + const std::string opts = opts_base + " -DCOOPK_NSG=" + std::to_string(nsg); + cl_program program = build_program_from_source(backend_ctx, kernel_src, opts); + cl_int err; + cl_kernel kernel = clCreateKernel(program, kernel_name, &err); + CL_CHECK(err); + size_t max_workgroup_size = 0; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, + sizeof(max_workgroup_size), &max_workgroup_size, NULL)); + CL_CHECK(clReleaseKernel(kernel)); + if ((size_t) 64 * nsg <= max_workgroup_size) { + return program; + } + CL_CHECK(clReleaseProgram(program)); + nsg /= 2; + } + return nullptr; +} + +static void ggml_cl_coopk_build_narrow(ggml_backend_opencl_context * backend_ctx, + const std::string & kernel_src, const char * kernel_name, + int rows, int & nsg, cl_kernel & kernel_c4, cl_kernel & kernel_c2) { + const std::string base_opts = backend_ctx->kernel_compile_opts + + " -DCOOPK_ROWS=" + std::to_string(rows); + for (int cols = 4; cols >= 2; cols -= 2) { + int nsg_eff = nsg; + cl_program program = ggml_cl_build_program_coopk( + backend_ctx, kernel_src.c_str(), kernel_name, + base_opts + " -DCOOPK_COLS=" + std::to_string(cols), nsg_eff); + if (program == nullptr) { + return; + } + cl_int err; + cl_kernel kernel = clCreateKernel(program, kernel_name, &err); + if (err != CL_SUCCESS) { kernel = nullptr; } + CL_CHECK(clReleaseProgram(program)); + if (cols == 4) { + kernel_c4 = kernel; + nsg = nsg_eff; + if (kernel == nullptr) { + return; + } + } else { + // Both widths share one stored subgroup count for launch. + if (kernel != nullptr && nsg_eff != nsg) { + CL_CHECK(clReleaseKernel(kernel)); + kernel = nullptr; + } + kernel_c2 = kernel; + } + } +} + +static void load_cl_kernels_gemm_coopk(ggml_backend_opencl_context * backend_ctx) { + if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO || !ggml_cl_coopk_dp4a_enabled(backend_ctx)) { + return; + } + + // Q4_0 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "tri.cl.h" + #include "gemm_coopk_q4_0_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("tri.cl"); + const std::string kernel_src = read_file("gemm_coopk_q4_0_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_tri = clCreateKernel(prog, "kernel_tri_f32", &err), err)); + ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q4_0_q8_1_dp4a", + backend_ctx->rows_coopk_dp4a_q4_0, backend_ctx->nsg_coopk_dp4a_q4_0, + backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4, + backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2); GGML_LOG_CONT("."); - - CL_CHECK(clReleaseProgram(prog)); } - // fill + // Q4_K { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "fill.cl.h" + #include "gemm_coopk_q4_k_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("fill.cl"); + const std::string kernel_src = read_file("gemm_coopk_q4_k_q8_1_dp4a.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_fill = clCreateKernel(prog, "kernel_fill_f32", &err), err)); + ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q4_k_q8_1_dp4a", + backend_ctx->rows_coopk_dp4a_q4_k, backend_ctx->nsg_coopk_dp4a_q4_k, + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4, + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2); + + const char * env = getenv("GGML_OPENCL_COOPK_Q4_K_BIN"); + if (use_adreno_bin_kernels(backend_ctx) && (!env || !*env || atoi(env) != 0)) { + const std::string base_opts = backend_ctx->kernel_compile_opts + + " -DCOOPK_Q_BIN=1 -DCOOPK_ROWS=" + std::to_string(backend_ctx->rows_coopk_dp4a_q4_k); + for (int cols = 4; cols >= 2; cols -= 2) { + int & nsg = (cols == 4) ? backend_ctx->nsg_coopk_dp4a_q4_k_bin_c4 + : backend_ctx->nsg_coopk_dp4a_q4_k_bin_c2; + int nsg_eff = nsg; + cl_program program = ggml_cl_build_program_coopk( + backend_ctx, kernel_src.c_str(), "kernel_gemm_coopk_q4_k_q8_1_dp4a", + base_opts + " -DCOOPK_COLS=" + std::to_string(cols), nsg_eff); + if (program == nullptr) { + continue; + } + cl_int err; + cl_kernel kernel = clCreateKernel(program, "kernel_gemm_coopk_q4_k_q8_1_dp4a", &err); + if (err != CL_SUCCESS) { kernel = nullptr; } + CL_CHECK(clReleaseProgram(program)); + if (cols == 4) { + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 = kernel; + } else { + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 = kernel; + } + nsg = nsg_eff; + } + } GGML_LOG_CONT("."); - - CL_CHECK(clReleaseProgram(prog)); } - // clamp + // Q6_K { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "clamp.cl.h" + #include "gemm_coopk_q6_k_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("clamp.cl"); + const std::string kernel_src = read_file("gemm_coopk_q6_k_q8_1_dp4a.cl"); #endif - backend_ctx->program_clamp = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q6_k_q8_1_dp4a", + backend_ctx->rows_coopk_dp4a_q6_k, backend_ctx->nsg_coopk_dp4a_q6_k, + backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4, + backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2); + GGML_LOG_CONT("."); + } +} + +#endif + +static std::string ggml_opencl_make_compile_opts(const ggml_backend_opencl_context * backend_ctx) { + auto opencl_c_std = + std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor); + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable -cl-unsafe-math-optimizations" + " -cl-finite-math-only -cl-fast-relaxed-math"; + + if (backend_ctx->adreno_use_large_buffer) { + compile_opts += " -qcom-enable-large-buffer "; + } + + return compile_opts; +} + +static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { + if (backend_ctx->kernels_loaded) { + return; + } + + cl_int err; + const std::string & compile_opts = backend_ctx->kernel_compile_opts; + const std::string opencl_c_std = "CL" + std::to_string(backend_ctx->opencl_c_version.major) + + "." + std::to_string(backend_ctx->opencl_c_version.minor); + + GGML_LOG_INFO("ggml_opencl: loading OpenCL kernels"); + + // add + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "add.cl.h" + }; +#else + const std::string kernel_src = read_file("add.cl"); +#endif + backend_ctx->program_add = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_add = clCreateKernel(backend_ctx->program_add, "kernel_add", &err), err)); + CL_CHECK((backend_ctx->kernel_add_row = clCreateKernel(backend_ctx->program_add, "kernel_add_row", &err), err)); + CL_CHECK((backend_ctx->kernel_add_f16 = clCreateKernel(backend_ctx->program_add, "kernel_add_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_add_row_f16 = clCreateKernel(backend_ctx->program_add, "kernel_add_row_f16", &err), err)); + GGML_LOG_CONT("."); + } + + // add_id + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "add_id.cl.h" + }; +#else + const std::string kernel_src = read_file("add_id.cl"); +#endif + backend_ctx->program_add_id = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_add_id = clCreateKernel(backend_ctx->program_add_id, "kernel_add_id", &err), err)); + GGML_LOG_CONT("."); + } + + // moe_add_id_glu + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_add_id_glu.cl.h" + }; +#else + const std::string kernel_src = read_file("moe_add_id_glu.cl"); +#endif + backend_ctx->program_moe_add_id_glu = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_add_id_add_id_swiglu_oai = + clCreateKernel(backend_ctx->program_moe_add_id_glu, "kernel_add_id_add_id_swiglu_oai", &err), err)); + GGML_LOG_CONT("."); + } + + // tri + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "tri.cl.h" + }; +#else + const std::string kernel_src = read_file("tri.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_tri = clCreateKernel(prog, "kernel_tri_f32", &err), err)); + GGML_LOG_CONT("."); + + CL_CHECK(clReleaseProgram(prog)); + } + + // fill + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "fill.cl.h" + }; +#else + const std::string kernel_src = read_file("fill.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_fill = clCreateKernel(prog, "kernel_fill_f32", &err), err)); + GGML_LOG_CONT("."); + + CL_CHECK(clReleaseProgram(prog)); + } + + // clamp + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "clamp.cl.h" + }; +#else + const std::string kernel_src = read_file("clamp.cl"); +#endif + backend_ctx->program_clamp = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_clamp = clCreateKernel(backend_ctx->program_clamp, "kernel_clamp", &err), err)); + CL_CHECK((backend_ctx->kernel_clamp_nc = clCreateKernel(backend_ctx->program_clamp, "kernel_clamp_nc", &err), err)); GGML_LOG_CONT("."); } @@ -904,14 +1894,28 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve #else const std::string kernel_src = read_file("cpy.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + const bool no_cpy_pack = backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X; + + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), + no_cpy_pack ? compile_opts + " -DGGML_CL_NO_CPY_PACK" : compile_opts); CL_CHECK((backend_ctx->kernel_cpy_f16_f16 = clCreateKernel(prog, "kernel_cpy_f16_f16", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f16_f32 = clCreateKernel(prog, "kernel_cpy_f16_f32", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f32_f16 = clCreateKernel(prog, "kernel_cpy_f32_f16", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f32_f32 = clCreateKernel(prog, "kernel_cpy_f32_f32", &err), err)); + if (!no_cpy_pack) { + CL_CHECK((backend_ctx->kernel_cpy_f32_f32_pack = clCreateKernel(prog, "kernel_cpy_f32_f32_pack", &err), err)); + } + { // optional: without it ggml_cl_cpy keeps the row-mapped kernel + cl_int err_flat = CL_SUCCESS; + cl_kernel k = clCreateKernel(prog, "kernel_cpy_f32_f32_flat", &err_flat); + if (err_flat == CL_SUCCESS) { + backend_ctx->kernel_cpy_f32_f32_flat = k; + } + } CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_cpy_i32_f32 = clCreateKernel(prog, "kernel_cpy_i32_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_cpy_f32_i32 = clCreateKernel(prog, "kernel_cpy_f32_i32", &err), err)); GGML_LOG_CONT("."); } @@ -925,23 +1929,59 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("cvt.cl"); #endif backend_ctx->program_cvt = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_convert_block_q1_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q1_0", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q1_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q1_0", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_0_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_0", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q4_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q4_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_0_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_1_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_1_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_1 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_1", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_1 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_1", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q4_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_1_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q4_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_1_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_0", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_0", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q5_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_0_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q5_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_0_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q5_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_1_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q5_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_1_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q5_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_0_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q5_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_0_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q5_1 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_1", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q5_1 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_1", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q5_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_1_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q5_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_1_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q4_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_k_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q4_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_k_trans4_ns", &err), err)); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK((backend_ctx->kernel_convert_block_q4_k_tiled_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_k_tiled_ns", &err), err)); +#endif + CL_CHECK((backend_ctx->kernel_convert_block_q5_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_k_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q5_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_k_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q6_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_k_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q6_k_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_k_trans4_ns", &err), err)); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK((backend_ctx->kernel_convert_block_q6_k_tiled_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_k_tiled_ns", &err), err)); +#endif CL_CHECK((backend_ctx->kernel_convert_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_mxfp4_trans = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4_trans", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_mxfp4_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4_trans4_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_mxfp4_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_mxfp4_trans4_ns", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_mxfp4_trans = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_mxfp4_trans", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_mxfp4", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q8_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q8_0", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q8_0_trans = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q8_0_trans", &err), err)); + CL_CHECK((backend_ctx->kernel_dequant_q8_0_f16_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q8_0_f16_view_aos", &err), err)); + CL_CHECK((backend_ctx->kernel_dequant_q8_0_f32_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q8_0_f32_view_aos", &err), err)); + CL_CHECK((backend_ctx->kernel_dequant_q4_0_f16_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q4_0_f16_view_aos", &err), err)); + CL_CHECK((backend_ctx->kernel_dequant_q4_0_f32_view_aos = clCreateKernel(backend_ctx->program_cvt, "kernel_dequant_q4_0_f32_view_aos", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q4_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q4_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_K_noshuffle", &err), err)); @@ -950,10 +1990,25 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve CL_CHECK((backend_ctx->kernel_restore_block_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q6_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_K_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq4_nl = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_nl", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq4_nl = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_nl", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq4_nl_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_nl_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq4_nl_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_nl_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_bf16_to_f16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_bf16_to_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_f16_to_bf16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_f16_to_bf16", &err), err)); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK((backend_ctx->kernel_moe_expand_scale_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q8_0", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_0", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_K", &err), err)); +#endif GGML_LOG_CONT("."); } @@ -967,7 +2022,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("diag_mask_inf.cl"); #endif backend_ctx->program_diag_mask_inf = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_diag_mask_inf_8 = clCreateKernel(backend_ctx->program_diag_mask_inf, "kernel_diag_mask_inf_8", &err), err)); CL_CHECK((backend_ctx->kernel_diag_mask_inf = clCreateKernel(backend_ctx->program_diag_mask_inf, "kernel_diag_mask_inf", &err), err)); @@ -984,7 +2039,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("diag.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_diag_f32 = clCreateKernel(prog, "kernel_diag_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1001,7 +2056,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("gelu.cl"); #endif backend_ctx->program_gelu = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gelu = clCreateKernel(backend_ctx->program_gelu, "kernel_gelu", &err), err)); CL_CHECK((backend_ctx->kernel_gelu_4 = clCreateKernel(backend_ctx->program_gelu, "kernel_gelu_4", &err), err)); @@ -1022,22 +2077,55 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("glu.cl"); #endif backend_ctx->program_glu = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_geglu = clCreateKernel(backend_ctx->program_glu, "kernel_geglu", &err), err)); CL_CHECK((backend_ctx->kernel_reglu = clCreateKernel(backend_ctx->program_glu, "kernel_reglu", &err), err)); CL_CHECK((backend_ctx->kernel_swiglu = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu", &err), err)); CL_CHECK((backend_ctx->kernel_swiglu_oai = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_oai", &err), err)); + CL_CHECK((backend_ctx->kernel_swiglu_clamp = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_clamp", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_erf = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_erf", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_quick = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_quick", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_f16", &err), err)); CL_CHECK((backend_ctx->kernel_reglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_reglu_f16", &err), err)); CL_CHECK((backend_ctx->kernel_swiglu_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_swiglu_clamp_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_swiglu_clamp_f16", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_erf_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_erf_f16", &err), err)); CL_CHECK((backend_ctx->kernel_geglu_quick_f16 = clCreateKernel(backend_ctx->program_glu, "kernel_geglu_quick_f16", &err), err)); GGML_LOG_CONT("."); } + // lightning_indexer + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "lightning_indexer.cl.h" + }; +#else + const std::string kernel_src = read_file("lightning_indexer.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_lightning_indexer = clCreateKernel(prog, "kernel_lightning_indexer", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // top_k + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "top_k.cl.h" + }; +#else + const std::string kernel_src = read_file("top_k.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_top_k_f32 = clCreateKernel(prog, "kernel_top_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_top_k_merge_f32 = clCreateKernel(prog, "kernel_top_k_merge_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // get_rows { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1048,9 +2136,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("get_rows.cl"); #endif backend_ctx->program_get_rows = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_get_rows_f32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_get_rows_i32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_i32", &err), err)); CL_CHECK((backend_ctx->kernel_get_rows_f16 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f16", &err), err)); CL_CHECK((backend_ctx->kernel_get_rows_q4_0 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_q4_0", &err), err)); GGML_LOG_CONT("."); @@ -1066,7 +2155,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("solve_tri.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_solve_tri_f32 = clCreateKernel(prog, "kernel_solve_tri_f32", &err), err)); GGML_LOG_CONT("."); @@ -1083,7 +2172,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("im2col_f32.cl"); #endif backend_ctx->program_im2col_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_im2col_f32 = clCreateKernel(backend_ctx->program_im2col_f32, "kernel_im2col_f32", &err), err)); GGML_LOG_CONT("."); @@ -1099,12 +2188,46 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("im2col_f16.cl"); #endif backend_ctx->program_im2col_f16 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_im2col_f16 = clCreateKernel(backend_ctx->program_im2col_f16, "kernel_im2col_f16", &err), err)); GGML_LOG_CONT("."); } + // im2col_3d_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "im2col_3d_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("im2col_3d_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_im2col_3d_f32 = clCreateKernel(prog, "kernel_im2col_3d_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // im2col_3d_f16 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "im2col_3d_f16.cl.h" + }; +#else + const std::string kernel_src = read_file("im2col_3d_f16.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_im2col_3d_f16 = clCreateKernel(prog, "kernel_im2col_3d_f16", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_q4_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1115,7 +2238,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q4_0_f32.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32, "kernel_mul_mat_q4_0_f32", &err), err)); GGML_LOG_CONT("."); @@ -1131,7 +2254,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q4_0_f32_v.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32_v = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_v = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_v, "kernel_mul_mat_q4_0_f32_v", &err), err)); GGML_LOG_CONT("."); @@ -1147,7 +2270,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q4_0_f32_8x_flat.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32_8x_flat = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_8x_flat, "kernel_mul_mat_q4_0_f32_8x_flat", &err), err)); GGML_LOG_CONT("."); @@ -1158,6 +2281,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve // those compiler versions since it is anyway not used for Adreno. if (backend_ctx->gpu_family != ADRENO || backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) || + backend_ctx->adreno_cl_compiler_version.type == E17 || backend_ctx->adreno_cl_compiler_version.type == DX) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { @@ -1167,7 +2291,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q4_0_f32_1d_8x_flat.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32_1d_8x_flat = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_1d_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_1d_8x_flat, "kernel_mul_mat_q4_0_f32_1d_8x_flat", &err), err)); GGML_LOG_CONT("."); @@ -1187,7 +2311,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q4_0_f32_1d_16x_flat.cl"); #endif backend_ctx->program_mul_mv_q4_0_f32_1d_16x_flat = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_q4_0_f32_1d_16x_flat = clCreateKernel(backend_ctx->program_mul_mv_q4_0_f32_1d_16x_flat, "kernel_mul_mat_q4_0_f32_1d_16x_flat", &err), err)); GGML_LOG_CONT("."); @@ -1203,7 +2327,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q4_1_f32.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q4_1_f32 = clCreateKernel(prog, "kernel_mul_mv_q4_1_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1220,13 +2344,60 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q4_1_f32_flat.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q4_1_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q4_1_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } + // mul_mv_q2_k_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q2_k_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q2_k_f32_flat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_q2_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q2_K_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_q2_k_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_q2_k_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_q2_k_f32_l4_lm.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + ggml_cl_q2_k_mm_options(backend_ctx)); + CL_CHECK((backend_ctx->kernel_mul_mm_q2_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q2_k_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_q2_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q2_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q2_k_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q2_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q2_K_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_q4_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1237,7 +2408,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q4_k_f32.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q4_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q4_K_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1254,13 +2425,81 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q4_k_f32_flat.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q4_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q4_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } + // mul_mv_q5_0_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q5_0_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q5_0_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q5_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_q5_0_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q5_0_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q5_0_f32_flat.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q5_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_0_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_q5_1_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q5_1_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q5_1_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q5_1_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_1_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_q5_1_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q5_1_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q5_1_f32_flat.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q5_1_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_1_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_q5_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1271,7 +2510,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q5_k_f32.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q5_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q5_K_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1288,7 +2527,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q5_k_f32_flat.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q5_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q5_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1304,7 +2543,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q6_k_f32.cl"); #endif backend_ctx->program_mul_mv_q6_K = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32 = clCreateKernel(backend_ctx->program_mul_mv_q6_K, "kernel_mul_mv_q6_K_f32", &err), err)); GGML_LOG_CONT("."); @@ -1319,8 +2558,14 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve #else const std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl"); #endif + // The codegen workarounds in this kernel are a measured 13-20% loss on + // compilers that do not need them, so only the affected ones build them; + // everyone else gets the original source. + const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler + ? compile_opts + " -DADRENO_OLD_COMPILER=1" + : compile_opts; cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1337,7 +2582,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q8_0_f32.cl"); #endif backend_ctx->program_mul_mv_q8_0_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32, "kernel_mul_mv_q8_0_f32", &err), err)); GGML_LOG_CONT("."); @@ -1353,39 +2598,107 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_q8_0_f32_flat.cl"); #endif backend_ctx->program_mul_mv_q8_0_f32_flat = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32_flat, "kernel_mul_mv_q8_0_f32_flat", &err), err)); GGML_LOG_CONT("."); } - // mul_mv_mxfp4_f32 + // mul_mv_q1_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_mxfp4_f32.cl.h" + #include "mul_mv_q1_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_mxfp4_f32.cl"); + const std::string kernel_src = read_file("mul_mv_q1_0_f32.cl"); #endif - backend_ctx->program_mul_mv_mxfp4_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32, "kernel_mul_mv_mxfp4_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mv_mxfp4_f32_flat + // mul_mv_q1_0_f32_flat { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "mul_mv_mxfp4_f32_flat.cl.h" + #include "mul_mv_q1_0_f32_flat.cl.h" }; #else - const std::string kernel_src = read_file("mul_mv_mxfp4_f32_flat.cl"); + const std::string kernel_src = read_file("mul_mv_q1_0_f32_flat.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_iq4_nl_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq4_nl_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq4_nl_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_nl_f32 = clCreateKernel(prog, "kernel_mul_mv_iq4_nl_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_iq4_nl_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq4_nl_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq4_nl_f32_flat.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_nl_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq4_nl_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_mxfp4_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_mxfp4_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_mxfp4_f32.cl"); +#endif + backend_ctx->program_mul_mv_mxfp4_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32, "kernel_mul_mv_mxfp4_f32", &err), err)); + GGML_LOG_CONT("."); + } + + // mul_mv_mxfp4_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_mxfp4_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_mxfp4_f32_flat.cl"); #endif backend_ctx->program_mul_mv_mxfp4_f32_flat = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32_flat, "kernel_mul_mv_mxfp4_f32_flat", &err), err)); GGML_LOG_CONT("."); @@ -1401,7 +2714,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_f16_f16.cl"); #endif backend_ctx->program_mul_mv_f16_f16 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f16 = clCreateKernel(backend_ctx->program_mul_mv_f16_f16, "kernel_mul_mat_f16_f16", &err), err)); GGML_LOG_CONT("."); @@ -1417,12 +2730,32 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_f16_f32_1row.cl"); #endif backend_ctx->program_mul_mv_f16_f32_1row = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_1row = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_1row, "kernel_mul_mat_f16_f32_1row", &err), err)); GGML_LOG_CONT("."); } + // mul_mv_f16_f32_mrow (multi-row decode GEMV) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_f16_f32_mrow.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_f16_f32_mrow.cl"); +#endif + backend_ctx->program_mul_mv_f16_f32_mrow = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r2", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_r4", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_mrow, "kernel_mul_mat_f16_f32_mrow_h8r2", &err), err)); + GGML_LOG_CONT("."); + } + // mul_mv_f16_f32_l4 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1433,9 +2766,59 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_f16_f32_l4.cl"); #endif backend_ctx->program_mul_mv_f16_f32_l4 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr", &err), err)); + if (backend_ctx->gpu_family == ADRENO) { + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_ls", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_lq", &err), err)); + } + + cl_int err_x8 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8", &err_x8); + if (err_x8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8 = nullptr; } + + cl_int err_x8p = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_pair", &err_x8p); + if (err_x8p != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair = nullptr; } + + cl_int err_x8g = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4", &err_x8g); + if (err_x8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 = nullptr; } + + cl_int err_x8gi = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa4_img", &err_x8gi); + if (err_x8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img = nullptr; } + + cl_int err_x8gi_r4 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img", &err_x8gi_r4); + if (err_x8gi_r4 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img = nullptr; } + + cl_int err_r2dk256 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img", &err_r2dk256); + if (err_r2dk256 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img = nullptr; } + + cl_int err_y8 = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8", &err_y8); + if (err_y8 != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8 = nullptr; } + + cl_int err_y8g = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa", &err_y8g); + if (err_y8g != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa = nullptr; } + + cl_int err_y8gi = CL_SUCCESS; + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = + clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_y8_gqa_img", &err_y8gi); + if (err_y8gi != CL_SUCCESS) { backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img = nullptr; } GGML_LOG_CONT("."); } @@ -1449,7 +2832,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_f16_f32.cl"); #endif backend_ctx->program_mul_mv_f16_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32, "kernel_mul_mat_f16_f32", &err), err)); GGML_LOG_CONT("."); @@ -1465,7 +2848,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_f32_f32.cl"); #endif backend_ctx->program_mul_mv_f32_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f32_f32 = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mat_f32_f32", &err), err)); GGML_LOG_CONT("."); @@ -1481,12 +2864,81 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mat_f16_f32.cl"); #endif backend_ctx->program_mul_mat_f16_f32_tiled = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_tiled = clCreateKernel(backend_ctx->program_mul_mat_f16_f32_tiled, "mul_mat_f16_f32", &err), err)); GGML_LOG_CONT("."); } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // gemm_xmem_f16_f32_os8 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_xmem_f16_f32_os8.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_xmem_f16_f32_os8.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_adreno_xmem_pack_src_f32 = + clCreateKernel(prog, "adreno_xmem_pack_src_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_adreno_xmem_prepack_weight_f16 = + clCreateKernel(prog, "adreno_xmem_prepack_weight_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_xmem_f16_f32_os8 = + clCreateKernel(prog, "kernel_gemm_xmem_f16_f32_os8", &err), err)); + CL_CHECK((backend_ctx->kernel_adreno_xmem_store_dst_f32 = + clCreateKernel(prog, "adreno_xmem_store_dst_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno xmem SDPA + if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "sdpa_xmem_f32_f16_os8.cl.h" + }; +#else + const std::string kernel_src = read_file("sdpa_xmem_f32_f16_os8.cl"); +#endif + cl_program program = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + auto & xmem_attn = backend_ctx->adreno_xmem_attn; + CL_CHECK((xmem_attn.kernel_q_f32_to_img_scaled = + clCreateKernel(program, "adreno_xmem_attn_q_f32_to_img_scaled", &err), err)); + CL_CHECK((xmem_attn.kernel_kv_f32_to_img_gqa = + clCreateKernel(program, "adreno_xmem_attn_kv_f32_to_img_gqa", &err), err)); + CL_CHECK((xmem_attn.kernel_kv_f16_to_img_gqa = + clCreateKernel(program, "adreno_xmem_attn_kv_f16_to_img_gqa", &err), err)); + CL_CHECK((xmem_attn.kernel_img_to_f32 = + clCreateKernel(program, "adreno_xmem_attn_img_to_f32", &err), err)); + CL_CHECK((xmem_attn.kernel_k_gather = + clCreateKernel(program, "adreno_xmem_attn_k_gather", &err), err)); + CL_CHECK((xmem_attn.kernel_pack_k = + clCreateKernel(program, "adreno_xmem_attn_pack_k", &err), err)); + CL_CHECK((xmem_attn.kernel_qk_gemm = + clCreateKernel(program, "adreno_xmem_attn_qk_gemm", &err), err)); + CL_CHECK((xmem_attn.kernel_softmax_reduce_basic = + clCreateKernel(program, "adreno_xmem_attn_softmax_reduce_basic", &err), err)); + CL_CHECK((xmem_attn.kernel_softmax_apply_basic = + clCreateKernel(program, "adreno_xmem_attn_softmax_apply_basic", &err), err)); + CL_CHECK((xmem_attn.kernel_mask_scores = + clCreateKernel(program, "adreno_xmem_attn_mask_scores", &err), err)); + CL_CHECK((xmem_attn.kernel_pack_v = + clCreateKernel(program, "adreno_xmem_attn_pack_v", &err), err)); + CL_CHECK((xmem_attn.kernel_pv_gemm = + clCreateKernel(program, "adreno_xmem_attn_pv_gemm", &err), err)); + CL_CHECK(clReleaseProgram(program)); + xmem_attn.compiled = true; + GGML_LOG_CONT("."); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + // mul_mm_f32_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1497,9 +2949,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mm_f32_f32_l4_lm.cl"); #endif backend_ctx->program_mul_mm_f32_f32_l4_lm = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_f32_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_mul_mm_f32_f32_l4_lm", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_f32_f32_mc = clCreateKernel(backend_ctx->program_mul_mm_f32_f32_l4_lm, "kernel_gemv_f32_f32_mc", &err), err)); GGML_LOG_CONT("."); } @@ -1513,9 +2966,14 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mm_f16_f32_l4_lm.cl"); #endif backend_ctx->program_mul_mm_f16_f32_l4_lm = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_l4_lm, "kernel_mul_mm_f16_f32_l4_lm", &err), err)); + + const std::string narrow_opts = compile_opts + " -DBN=8 -DTN=1 -DKERNEL_NAME_LM=kernel_mul_mm_f16_f32_l4_lm_n8"; + backend_ctx->program_mul_mm_f16_f32_l4_lm_n8 = + build_program_from_source(backend_ctx, kernel_src.c_str(), narrow_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_l4_lm_n8 = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_l4_lm_n8, "kernel_mul_mm_f16_f32_l4_lm_n8", &err), err)); GGML_LOG_CONT("."); } @@ -1529,7 +2987,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mm_q4_0_f32_l4_lm.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_0_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); @@ -1545,12 +3003,44 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mm_q4_1_f32_l4_lm.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_1_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } + // mul_mm_q5_0_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_q5_0_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_q5_0_f32_l4_lm.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_0_f32_l4_lm", &err), err)); + GGML_LOG_CONT("."); + } + + // mul_mm_q5_1_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_q5_1_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_q5_1_f32_l4_lm.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_1_f32_l4_lm", &err), err)); + GGML_LOG_CONT("."); + } + // mul_mm_q8_0_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1561,12 +3051,46 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mm_q8_0_f32_l4_lm.cl"); #endif backend_ctx->program_mul_mm_q8_0_f32_l4_lm = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm = clCreateKernel(backend_ctx->program_mul_mm_q8_0_f32_l4_lm, "kernel_mul_mm_q8_0_f32_l4_lm", &err), err)); GGML_LOG_CONT("."); } + // mul_mm_q1_0_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_q1_0_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_q1_0_f32_l4_lm.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q1_0_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_iq4_nl_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_iq4_nl_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_iq4_nl_f32_l4_lm.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq4_nl_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mm_q4_k_f32_l4_lm { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1577,7 +3101,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mm_q4_k_f32_l4_lm.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q4_k_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1594,7 +3118,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mm_q6_k_f32_l4_lm.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q6_k_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1611,7 +3135,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mm_q5_k_f32_l4_lm.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q5_k_f32_l4_lm", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1628,9 +3152,9 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mm_f16_f32_kq_kqv.cl"); #endif backend_ctx->program_mul_mm_f16_f32_kqv = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts+" -DKQV "); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts+" -DKQV "); backend_ctx->program_mul_mm_f16_f32_kq = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kqv = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kqv, "mul_mm_f16_f32_kqv", &err), err)); CL_CHECK((backend_ctx->kernel_mul_mm_f16_f32_kq = clCreateKernel(backend_ctx->program_mul_mm_f16_f32_kq, "mul_mm_f16_f32_kq", &err), err)); @@ -1647,7 +3171,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul.cl"); #endif backend_ctx->program_mul = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul = clCreateKernel(backend_ctx->program_mul, "kernel_mul", &err), err)); CL_CHECK((backend_ctx->kernel_mul_row = clCreateKernel(backend_ctx->program_mul, "kernel_mul_row", &err), err)); @@ -1666,7 +3190,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("norm.cl"); #endif backend_ctx->program_norm = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_norm = clCreateKernel(backend_ctx->program_norm, "kernel_norm", &err), err)); CL_CHECK((backend_ctx->kernel_norm_mul_add = clCreateKernel(backend_ctx->program_norm, "kernel_norm_mul_add", &err), err)); @@ -1683,7 +3207,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("relu.cl"); #endif backend_ctx->program_relu = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_relu = clCreateKernel(backend_ctx->program_relu, "kernel_relu", &err), err)); GGML_LOG_CONT("."); @@ -1699,10 +3223,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("rms_norm.cl"); #endif backend_ctx->program_rms_norm = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_rms_norm = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm", &err), err)); CL_CHECK((backend_ctx->kernel_rms_norm_mul = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul", &err), err)); + CL_CHECK((backend_ctx->kernel_rms_norm_mul_add = clCreateKernel(backend_ctx->program_rms_norm, "kernel_rms_norm_mul_add", &err), err)); GGML_LOG_CONT("."); } @@ -1716,7 +3241,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("l2_norm.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_l2_norm_f32 = clCreateKernel(prog, "kernel_l2_norm_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); @@ -1733,7 +3258,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("rope.cl"); #endif backend_ctx->program_rope = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_rope_norm_f32 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_norm_f32", &err), err)); CL_CHECK((backend_ctx->kernel_rope_norm_f16 = clCreateKernel(backend_ctx->program_rope, "kernel_rope_norm_f16", &err), err)); @@ -1756,7 +3281,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("scale.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_scale_f32 = clCreateKernel(prog, "kernel_scale_f32", &err), err)); CL_CHECK((backend_ctx->kernel_scale_f32_4 = clCreateKernel(prog, "kernel_scale_f32_4", &err), err)); @@ -1774,7 +3299,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("silu.cl"); #endif backend_ctx->program_silu = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_silu = clCreateKernel(backend_ctx->program_silu, "kernel_silu", &err), err)); CL_CHECK((backend_ctx->kernel_silu_4 = clCreateKernel(backend_ctx->program_silu, "kernel_silu_4", &err), err)); @@ -1791,7 +3316,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("softmax_f32.cl"); #endif backend_ctx->program_softmax_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_soft_max = clCreateKernel(backend_ctx->program_softmax_f32, "kernel_soft_max", &err), err)); GGML_LOG_CONT("."); @@ -1807,7 +3332,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("softmax_f16.cl"); #endif backend_ctx->program_softmax_f16 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_soft_max_f16 = clCreateKernel(backend_ctx->program_softmax_f16, "kernel_soft_max_f16", &err), err)); GGML_LOG_CONT("."); @@ -1823,7 +3348,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("softmax_4_f32.cl"); #endif backend_ctx->program_softmax_4_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_soft_max_4 = clCreateKernel(backend_ctx->program_softmax_4_f32, "kernel_soft_max_4", &err), err)); GGML_LOG_CONT("."); @@ -1839,95 +3364,12 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("softmax_4_f16.cl"); #endif backend_ctx->program_softmax_4_f16 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_soft_max_4_f16 = clCreateKernel(backend_ctx->program_softmax_4_f16, "kernel_soft_max_4_f16", &err), err)); GGML_LOG_CONT("."); } - // flash_attn - { - #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src_f16 { - #include "flash_attn_f16.cl.h" - }; - const std::string kernel_src_f32 { - #include "flash_attn_f32.cl.h" - }; - const std::string kernel_src_f32_f16 { - #include "flash_attn_f32_f16.cl.h" - }; - #else - const std::string kernel_src_f16 = read_file("flash_attn_f16.cl"); - const std::string kernel_src_f32 = read_file("flash_attn_f32.cl"); - const std::string kernel_src_f32_f16 = read_file("flash_attn_f32_f16.cl"); - #endif - - if (!kernel_src_f16.empty() && !kernel_src_f32.empty() && !kernel_src_f32_f16.empty()) { - const struct { int dk; int dv; int bm; int bn; } fa_dims[] = { - { 40, 40, 32, 32}, { 64, 64, 64, 64}, { 80, 80, 64, 32}, { 96, 96, 64, 32}, - {112, 112, 32, 32}, {128, 128, 32, 32}, {192, 128, 16, 16}, - {192, 192, 16, 16}, {256, 256, 16, 16}, - }; - - for (size_t i = 0; i < sizeof(fa_dims)/sizeof(fa_dims[0]); ++i) { - const int dk = fa_dims[i].dk; - const int dv = fa_dims[i].dv; - const int bm = fa_dims[i].bm; - const int bn = fa_dims[i].bn; - std::string OPTS = compile_opts + - " -D DK=" + std::to_string(dk) + - " -D DV=" + std::to_string(dv) + - " -D BLOCK_M=" + std::to_string(bm) + - " -D BLOCK_N=" + std::to_string(bn); - - cl_program prog_f16 = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f16.c_str(), OPTS); - cl_kernel k_f16, k_f16_q1; - CL_CHECK((k_f16 = clCreateKernel(prog_f16, "flash_attn_f16", &err), err)); - CL_CHECK((k_f16_q1 = clCreateKernel(prog_f16, "flash_attn_f16_q1", &err), err)); - backend_ctx->kernels_flash_attn_f16[{dk, dv}] = k_f16; - backend_ctx->kernels_flash_attn_f16_q1[{dk, dv}] = k_f16_q1; - CL_CHECK(clReleaseProgram(prog_f16)); - - cl_program prog_f32 = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f32.c_str(), OPTS); - cl_kernel k_f32, k_f32_q1; - CL_CHECK((k_f32 = clCreateKernel(prog_f32, "flash_attn_f32", &err), err)); - CL_CHECK((k_f32_q1 = clCreateKernel(prog_f32, "flash_attn_f32_q1", &err), err)); - backend_ctx->kernels_flash_attn_f32[{dk, dv}] = k_f32; - backend_ctx->kernels_flash_attn_f32_q1[{dk, dv}] = k_f32_q1; - CL_CHECK(clReleaseProgram(prog_f32)); - - cl_program prog_f32_f16 = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f32_f16.c_str(), OPTS); - cl_kernel k_f32_f16, k_f32_f16_q1; - CL_CHECK((k_f32_f16 = clCreateKernel(prog_f32_f16, "flash_attn_f32_f16", &err), err)); - CL_CHECK((k_f32_f16_q1 = clCreateKernel(prog_f32_f16, "flash_attn_f32_f16_q1", &err), err)); - backend_ctx->kernels_flash_attn_f32_f16[{dk, dv}] = k_f32_f16; - backend_ctx->kernels_flash_attn_f32_f16_q1[{dk, dv}] = k_f32_f16_q1; - CL_CHECK(clReleaseProgram(prog_f32_f16)); - - backend_ctx->kernels_flash_attn_bm[{dk, dv}] = bm; - backend_ctx->kernels_flash_attn_bn[{dk, dv}] = bn; - } - GGML_LOG_CONT("."); - } - } - - // argsort - { -#ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "argsort.cl.h" - }; -#else - const std::string kernel_src = read_file("argsort.cl"); -#endif - backend_ctx->program_argsort_f32_i32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); - - CL_CHECK((backend_ctx->kernel_argsort_f32_i32 = clCreateKernel(backend_ctx->program_argsort_f32_i32, "kernel_argsort_f32_i32", &err), err)); - GGML_LOG_CONT("."); - } - // div { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -1941,7 +3383,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve " -cl-mad-enable -cl-finite-math-only "; backend_ctx->program_div = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_div = clCreateKernel(backend_ctx->program_div, "kernel_div", &err), err)); CL_CHECK((backend_ctx->kernel_div_row = clCreateKernel(backend_ctx->program_div, "kernel_div_row", &err), err)); @@ -1960,7 +3402,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("sqr.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sqr_cont_f32 = clCreateKernel(prog, "kernel_sqr_cont_f32", &err), err)); CL_CHECK((backend_ctx->kernel_sqr_cont_f32_4 = clCreateKernel(prog, "kernel_sqr_cont_f32_4", &err), err)); @@ -1981,7 +3423,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("sqrt.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sqrt_cont_f32 = clCreateKernel(prog, "kernel_sqrt_cont_f32", &err), err)); CL_CHECK((backend_ctx->kernel_sqrt_cont_f32_4 = clCreateKernel(prog, "kernel_sqrt_cont_f32_4", &err), err)); @@ -2002,7 +3444,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mean.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mean_f32 = clCreateKernel(prog, "kernel_mean_f32", &err), err)); CL_CHECK((backend_ctx->kernel_mean_f32_4 = clCreateKernel(prog, "kernel_mean_f32_4", &err), err)); @@ -2021,7 +3463,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("sub.cl"); #endif backend_ctx->program_sub = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sub = clCreateKernel(backend_ctx->program_sub, "kernel_sub", &err), err)); CL_CHECK((backend_ctx->kernel_sub_row = clCreateKernel(backend_ctx->program_sub, "kernel_sub_row", &err), err)); @@ -2040,7 +3482,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("sum_rows.cl"); #endif backend_ctx->program_sum_rows_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sum_rows_f32 = clCreateKernel(backend_ctx->program_sum_rows_f32, "kernel_sum_rows_f32", &err), err)); CL_CHECK((backend_ctx->kernel_sum_rows_f32_4 = clCreateKernel(backend_ctx->program_sum_rows_f32, "kernel_sum_rows_f32_4", &err), err)); @@ -2057,7 +3499,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("cumsum.cl"); #endif cl_program prog; - prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_cumsum_blk = clCreateKernel(prog, "kernel_cumsum_blk", &err), err)); CL_CHECK((backend_ctx->kernel_cumsum_add = clCreateKernel(prog, "kernel_cumsum_add", &err), err)); @@ -2075,7 +3517,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("sigmoid.cl"); #endif backend_ctx->program_sigmoid = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_sigmoid_f32 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f32", &err), err)); CL_CHECK((backend_ctx->kernel_sigmoid_f16 = clCreateKernel(backend_ctx->program_sigmoid, "kernel_sigmoid_f16", &err), err)); @@ -2092,7 +3534,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("group_norm.cl"); #endif backend_ctx->program_group_norm = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_group_norm = clCreateKernel(backend_ctx->program_group_norm, "kernel_group_norm", &err), err)); CL_CHECK((backend_ctx->kernel_group_norm_mul_add = clCreateKernel(backend_ctx->program_group_norm, "kernel_group_norm_mul_add", &err), err)); @@ -2109,7 +3551,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("repeat.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_repeat_f32 = clCreateKernel(prog, "kernel_repeat_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); @@ -2126,7 +3568,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve #endif if (!kernel_src.empty()) { backend_ctx->program_pad = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_pad = clCreateKernel(backend_ctx->program_pad, "kernel_pad", &err), err)); GGML_LOG_CONT("."); } else { @@ -2146,7 +3588,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("tanh.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_tanh_f32 = clCreateKernel(prog, "kernel_tanh_f32", &err), err)); CL_CHECK((backend_ctx->kernel_tanh_f32_4 = clCreateKernel(prog, "kernel_tanh_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_tanh_f32_nc = clCreateKernel(prog, "kernel_tanh_f32_nc", &err), err)); @@ -2167,7 +3609,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("neg.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_neg_f32 = clCreateKernel(prog, "kernel_neg_f32", &err), err)); CL_CHECK((backend_ctx->kernel_neg_f32_4 = clCreateKernel(prog, "kernel_neg_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_neg_f32_nc = clCreateKernel(prog, "kernel_neg_f32_nc", &err), err)); @@ -2188,7 +3630,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("exp.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_exp_f32 = clCreateKernel(prog, "kernel_exp_f32", &err), err)); CL_CHECK((backend_ctx->kernel_exp_f32_4 = clCreateKernel(prog, "kernel_exp_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_exp_f32_nc = clCreateKernel(prog, "kernel_exp_f32_nc", &err), err)); @@ -2209,7 +3651,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("expm1.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_expm1_f32 = clCreateKernel(prog, "kernel_expm1_f32", &err), err)); CL_CHECK((backend_ctx->kernel_expm1_f32_4 = clCreateKernel(prog, "kernel_expm1_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_expm1_f32_nc = clCreateKernel(prog, "kernel_expm1_f32_nc", &err), err)); @@ -2220,6 +3662,59 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve GGML_LOG_CONT("."); } + // abs + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "abs.cl.h" + }; +#else + const std::string kernel_src = read_file("abs.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_abs_f32 = clCreateKernel(prog, "kernel_abs_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f32_4 = clCreateKernel(prog, "kernel_abs_f32_4", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f32_nc = clCreateKernel(prog, "kernel_abs_f32_nc", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f16 = clCreateKernel(prog, "kernel_abs_f16", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f16_4 = clCreateKernel(prog, "kernel_abs_f16_4", &err), err)); + CL_CHECK((backend_ctx->kernel_abs_f16_nc = clCreateKernel(prog, "kernel_abs_f16_nc", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // unary_ext (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "unary_ext.cl.h" + }; +#else + const std::string kernel_src = read_file("unary_ext.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); +#define CL_UNARY_EXT_K(op) \ + CL_CHECK((backend_ctx->kernel_##op##_f32 = clCreateKernel(prog, "kernel_" #op "_f32", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f32_4 = clCreateKernel(prog, "kernel_" #op "_f32_4", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f32_nc = clCreateKernel(prog, "kernel_" #op "_f32_nc", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f16 = clCreateKernel(prog, "kernel_" #op "_f16", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f16_4 = clCreateKernel(prog, "kernel_" #op "_f16_4", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f16_nc = clCreateKernel(prog, "kernel_" #op "_f16_nc", &err), err)); + CL_UNARY_EXT_K(sgn) + CL_UNARY_EXT_K(step) + CL_UNARY_EXT_K(elu) + CL_UNARY_EXT_K(hardswish) + CL_UNARY_EXT_K(hardsigmoid) + CL_UNARY_EXT_K(floor) + CL_UNARY_EXT_K(ceil) + CL_UNARY_EXT_K(round) + CL_UNARY_EXT_K(trunc) +#undef CL_UNARY_EXT_K + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // softplus { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -2230,7 +3725,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("softplus.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_softplus_f32 = clCreateKernel(prog, "kernel_softplus_f32", &err), err)); CL_CHECK((backend_ctx->kernel_softplus_f32_4 = clCreateKernel(prog, "kernel_softplus_f32_4", &err), err)); CL_CHECK((backend_ctx->kernel_softplus_f32_nc = clCreateKernel(prog, "kernel_softplus_f32_nc", &err), err)); @@ -2252,15 +3747,15 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve #endif if (!kernel_src.empty()) { backend_ctx->program_upscale = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_upscale = clCreateKernel(backend_ctx->program_upscale, "kernel_upscale", &err), err)); if (backend_ctx->program_upscale) { - cl_int err_bilinear; - backend_ctx->kernel_upscale_bilinear = clCreateKernel(backend_ctx->program_upscale, "kernel_upscale_bilinear", &err_bilinear); - if (err_bilinear != CL_SUCCESS) { + cl_int err_bilinear; + backend_ctx->kernel_upscale_bilinear = clCreateKernel(backend_ctx->program_upscale, "kernel_upscale_bilinear", &err_bilinear); + if (err_bilinear != CL_SUCCESS) { GGML_LOG_WARN("ggml_opencl: kernel_upscale_bilinear not found in upscale.cl. Bilinear upscale will not be available. Error: %d\n", err_bilinear); backend_ctx->kernel_upscale_bilinear = nullptr; - } + } } else { backend_ctx->kernel_upscale_bilinear = nullptr; } @@ -2283,8 +3778,12 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("concat.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_concat_f32 = clCreateKernel(prog, "kernel_concat_f32", &err), err)); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_concat_b1 = clCreateKernel(prog, "kernel_concat_b1", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b2 = clCreateKernel(prog, "kernel_concat_b2", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b4 = clCreateKernel(prog, "kernel_concat_b4", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b8 = clCreateKernel(prog, "kernel_concat_b8", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b4_pack = clCreateKernel(prog, "kernel_concat_b4_pack", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -2301,7 +3800,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve #endif if (!kernel_src.empty()) { backend_ctx->program_tsembd = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_timestep_embedding = clCreateKernel(backend_ctx->program_tsembd, "kernel_timestep_embedding", &err), err)); GGML_LOG_CONT("."); } else { @@ -2321,17 +3820,25 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("set_rows.cl"); #endif backend_ctx->program_set_rows = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_set_rows_f32_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f32_i64", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_f32_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f32_i32", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_f16_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i64", &err), err)); CL_CHECK((backend_ctx->kernel_set_rows_f16_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_f16_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q8_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q8_0_soa_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i64 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i64", &err), err)); + CL_CHECK((backend_ctx->kernel_set_rows_q4_0_soa_i32 = clCreateKernel(backend_ctx->program_set_rows, "kernel_set_rows_q4_0_soa_i32", &err), err)); GGML_LOG_CONT("."); } - // conv2d - { + // conv2d + { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "conv2d.cl.h" @@ -2345,11 +3852,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve #endif if (!kernel_src.empty()) { backend_ctx->program_conv_2d_f16 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), (std::string(compile_opts) + " -DUSE_FP16=1").c_str()); + build_program_from_source(backend_ctx, kernel_src.c_str(), (std::string(compile_opts) + " -DUSE_FP16=1").c_str()); CL_CHECK((backend_ctx->kernel_conv_2d_f16 = clCreateKernel(backend_ctx->program_conv_2d_f16, "kernel_conv_2d", &err), err)); GGML_LOG_CONT("."); backend_ctx->program_conv_2d_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_conv_2d_f32 = clCreateKernel(backend_ctx->program_conv_2d_f32, "kernel_conv_2d", &err), err)); GGML_LOG_CONT("."); } else { @@ -2361,7 +3868,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve } if (!kernel_src_f16_f32.empty()) { backend_ctx->program_conv_2d_f16_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_f16_f32.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src_f16_f32.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_conv_2d_f16_f32 = clCreateKernel(backend_ctx->program_conv_2d_f16_f32, "kernel_conv_2d", &err), err)); GGML_LOG_CONT("."); } else { @@ -2371,6 +3878,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve } } + // conv2_dw + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "conv2_dw.cl.h" + }; +#else + const std::string kernel_src = read_file("conv2_dw.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_conv2d_dw_f32_f32 = clCreateKernel(prog, "kernel_conv2d_dw_f32_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_conv2d_dwf16_f32 = clCreateKernel(prog, "kernel_conv2d_dwf16_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // ssm_conv { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -2381,7 +3905,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("ssm_conv.cl"); #endif cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32 = clCreateKernel(prog, "kernel_ssm_conv_f32_f32", &err), err)); CL_CHECK((backend_ctx->kernel_ssm_conv_f32_f32_4 = clCreateKernel(prog, "kernel_ssm_conv_f32_f32_4", &err), err)); @@ -2389,6 +3913,148 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve GGML_LOG_CONT("."); } + // ssm_scan + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "ssm_scan.cl.h" + }; +#else + const std::string kernel_src = read_file("ssm_scan.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_ssm_scan_f32 = clCreateKernel(prog, "kernel_ssm_scan_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d128 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d128", &err), err)); + CL_CHECK((backend_ctx->kernel_ssm_scan_f32_mamba2_d256 = clCreateKernel(prog, "kernel_ssm_scan_f32_mamba2_d256", &err), err)); + + cl_kernel * kernels[] = { + &backend_ctx->kernel_ssm_scan_f32_mamba2_d128, + &backend_ctx->kernel_ssm_scan_f32_mamba2_d256 + }; + + // specialized kernels use subgroups and assume subgroup size is 64, + // if device does not support subgroups or subgroup size is not 64, + // release these kernels + for (int i = 0; i < 2; ++i) { + size_t subgroup_size = 0; +#if CL_TARGET_OPENCL_VERSION >= 210 + const size_t local_work_size[] = { 64, 1 }; + const cl_int subgroup_err = clGetKernelSubGroupInfo(*kernels[i], backend_ctx->device, CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, + sizeof(local_work_size), local_work_size, sizeof(subgroup_size), &subgroup_size, nullptr); + if (subgroup_err != CL_SUCCESS) { + subgroup_size = 0; + } +#endif + // The specialized kernels reduce over one 64-lane subgroup. + if (subgroup_size != 64) { + CL_CHECK(clReleaseKernel(*kernels[i])); + *kernels[i] = nullptr; + } + } + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gated_delta_net: one kernel per (S_V, KDA, tgpp) triple. + { + #ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gated_delta_net.cl.h" + }; + #else + const std::string kernel_src = read_file("gated_delta_net.cl"); + #endif + + const int gdn_sizes[4] = { 16, 32, 64, 128 }; + const int sg_size = backend_ctx->gpu_family == GPU_FAMILY::ADRENO ? 64 : backend_ctx->gpu_family == GPU_FAMILY::INTEL ? 32 : -1; + if (sg_size < 0) { + GGML_LOG_ERROR("Unsupported GPU Family: only Adreno and Intel are supported.\n"); + exit(1); + } + + for (int si = 0; si < 4; si++) { + const int S_V = gdn_sizes[si]; + + // MUST match the dispatcher heuristic in ggml_cl_gated_delta_net exactly. + int lanes_per_column; + if (S_V >= 128) { + lanes_per_column = 8; + } else { + lanes_per_column = std::min(S_V, sg_size); + } + + // Round LANES_PER_COLUMN down until it is: + // * power-of-two + // * divides both S_V and sg_size + while (lanes_per_column > 1 && + (((lanes_per_column & (lanes_per_column - 1)) != 0) || + (S_V % lanes_per_column) != 0 || + (sg_size % lanes_per_column) != 0)) { + lanes_per_column >>= 1; + } + + GGML_ASSERT(lanes_per_column >= 1); + GGML_ASSERT(((lanes_per_column & (lanes_per_column - 1)) == 0)); + GGML_ASSERT((S_V % lanes_per_column) == 0); + GGML_ASSERT((sg_size % lanes_per_column) == 0); + + const bool is_partial_reduce = (lanes_per_column != 1) && (lanes_per_column < sg_size); + int use_qcom_shuffle = 0; + if (is_partial_reduce) { + if (backend_ctx->has_qcom_subgroup_shuffle) { + use_qcom_shuffle = 1; + } + } + for (int kda = 0; kda < 2; kda++) { + for (int tgpp = 0; tgpp < 2; tgpp++) { + const int cpl = (tgpp == 0) ? 1 : 4; + const int spw = (tgpp == 0) ? 1 : 1; + + std::string opts = compile_opts; + opts += " -DS_V=" + std::to_string(S_V); + opts += " -DKDA=" + std::to_string(kda); + opts += " -DSUBGROUP_SIZE=" + std::to_string(sg_size); + opts += " -DLANES_PER_COLUMN=" + std::to_string(lanes_per_column); + opts += " -DCOLS_PER_LANE_GROUP=" + std::to_string(cpl); + opts += " -DUSE_QCOM_SUBGROUP_SHUFFLE=" + std::to_string(use_qcom_shuffle); + + // Since spw=1 is found to be optimal, SUBGROUPS_PER_WG > 1 code in + // the kernel is removed. If you want to experiment with spw > 1, + // Please remember to implement code to handle it. + opts += " -DSUBGROUPS_PER_WG=" + std::to_string(spw); + + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + + CL_CHECK((backend_ctx->kernel_gated_delta_net_f32[si][kda][tgpp] = + clCreateKernel(prog, "kernel_gated_delta_net", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + } + } + } + GGML_LOG_CONT("."); + } + + // moe_combine (fused router-weight mul + cross-expert sum) + { + #ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_combine.cl.h" + }; + #else + const std::string kernel_src = read_file("moe_combine.cl"); + #endif + cl_program prog = build_program_from_source( + backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_moe_combine_f32 = + clCreateKernel(prog, "kernel_moe_combine_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_combine_bias_f32 = + clCreateKernel(prog, "kernel_moe_combine_bias_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_id_q4_0_f32_8x_flat { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -2399,7 +4065,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_id_q4_0_f32_8x_flat.cl"); #endif backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_q4_0_f32_8x_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q4_0_f32_8x_flat, "kernel_mul_mv_id_q4_0_f32_8x_flat", &err), err)); GGML_LOG_CONT("."); @@ -2415,7 +4081,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_id_q8_0_f32.cl"); #endif backend_ctx->program_mul_mv_id_q8_0_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32, "kernel_mul_mv_id_q8_0_f32", &err), err)); GGML_LOG_CONT("."); @@ -2431,7 +4097,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_id_q8_0_f32_flat.cl"); #endif backend_ctx->program_mul_mv_id_q8_0_f32_flat = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_q8_0_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_q8_0_f32_flat, "kernel_mul_mv_id_q8_0_f32_flat", &err), err)); GGML_LOG_CONT("."); @@ -2447,7 +4113,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_id_mxfp4_f32.cl"); #endif backend_ctx->program_mul_mv_id_mxfp4_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32 = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32, "kernel_mul_mv_id_mxfp4_f32", &err), err)); GGML_LOG_CONT("."); @@ -2463,7 +4129,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("mul_mv_id_mxfp4_f32_flat.cl"); #endif backend_ctx->program_mul_mv_id_mxfp4_f32_flat = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32_flat, "kernel_mul_mv_id_mxfp4_f32_flat", &err), err)); GGML_LOG_CONT("."); @@ -2481,7 +4147,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("transpose.cl"); #endif backend_ctx->program_transpose = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_transpose_32_16 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32_16", &err), err)); CL_CHECK((backend_ctx->kernel_transpose_32 = clCreateKernel(backend_ctx->program_transpose, "kernel_transpose_32", &err), err)); @@ -2493,28 +4159,153 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve GGML_LOG_CONT("."); } - // gemv_noshuffle_general + // gemm_noshuffle_q1_0_f32 { - std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q1_0_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q1_0_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q1_0_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); + +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src_CL_gemv_general { + #include "gemv_noshuffle_q1_0_f32.cl.h" + }; +#else + const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q1_0_f32.cl"); +#endif + + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + + const char * nsg1_min_m_env = getenv("GGML_OPENCL_Q1_0_GEMV_NSG1_MIN_M"); + backend_ctx->min_m_gemv_nsg1_q1_0 = nsg1_min_m_env ? atoi(nsg1_min_m_env) : + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? 8192 : 0); + if (backend_ctx->min_m_gemv_nsg1_q1_0 > 0) { + const std::string nsg1_opts = CL_gemv_compile_opts + " -DN_SIMDGROUP=1"; + cl_program prog_nsg1 = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), nsg1_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q1_0_f32_nsg1 = clCreateKernel(prog_nsg1, "kernel_gemv_noshuffle_q1_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog_nsg1)); + } + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q2_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q2_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q2_k_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q2_k_q8_1_dp4a + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q2_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q2_k_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q2_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q2_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q2_k_f32.cl"); +#endif + if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_R")) { + const int rows = atoi(env); + if (rows == 1 || rows == 2 || rows == 4) { + backend_ctx->q2_k_mv_rows = rows; + } + } + if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->q2_k_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DQ2_K_MV_R=" + std::to_string(backend_ctx->q2_k_mv_rows) + + " -DQ2_K_MV_NSG=" + std::to_string(backend_ctx->q2_k_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q2_k_f32", &err); + CL_CHECK(err); + CL_CHECK(clReleaseProgram(prog)); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->q2_k_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_q2_k_f32 = kernel; + break; + } + CL_CHECK(clReleaseKernel(kernel)); + GGML_ASSERT(backend_ctx->q2_k_mv_nsg > 1); + backend_ctx->q2_k_mv_nsg /= 2; + } + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_general + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X) { + CL_gemv_compile_opts += " -DGGML_CL_A6X_CONSTFOLD_FIX"; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src_CL_gemv_general { - #include "gemv_noshuffle_general.cl.h" + #include "gemv_noshuffle_q4_0_f32.cl.h" }; #else - const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_general.cl"); + const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q4_0_f32.cl"); #endif - backend_ctx->program_CL_gemv_general = build_program_from_source( - backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_general = clCreateKernel(backend_ctx->program_CL_gemv_general, "kernel_gemv_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_mc3", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -2528,20 +4319,20 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src_CL_gemv { - #include "gemv_noshuffle.cl.h" + #include "gemv_noshuffle_q4_0_f32_spec.cl.h" }; #else - const std::string kernel_src_CL_gemv = read_file("gemv_noshuffle.cl"); + const std::string kernel_src_CL_gemv = read_file("gemv_noshuffle_q4_0_f32_spec.cl"); #endif - backend_ctx->program_CL_gemv_4096_1_4096 = build_program_from_source( - backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_4096 = clCreateKernel(backend_ctx->program_CL_gemv_4096_1_4096, "kernel_gemv_noshuffle", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); // Gemv 2048, 16384 @@ -2552,12 +4343,12 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - backend_ctx->program_CL_gemv_4096_1_11008 = build_program_from_source( - backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_11008 = clCreateKernel(backend_ctx->program_CL_gemv_4096_1_11008, "kernel_gemv_noshuffle", &err), err)); + prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); // Gemv 5504, 44032 @@ -2568,12 +4359,12 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve " -DSIMDGROUP_WIDTH=" + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - backend_ctx->program_CL_gemv_11008_1_4096 = build_program_from_source( - backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_11008_1_4096 = clCreateKernel(backend_ctx->program_CL_gemv_11008_1_4096, "kernel_gemv_noshuffle", &err), err)); + prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); // Gemv 16000, 128000 @@ -2585,12 +4376,12 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - backend_ctx->program_CL_gemv_32000_1_4096 = build_program_from_source( - backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_32000_1_4096 = clCreateKernel(backend_ctx->program_CL_gemv_32000_1_4096, "kernel_gemv_noshuffle", &err), err)); + prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -2598,26 +4389,76 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src_CL_gemm { - #include "mul_mat_Ab_Bi_8x4.cl.h" + #include "gemm_noshuffle_q4_0_f32.cl.h" }; #else - const std::string kernel_src_CL_gemm = read_file("mul_mat_Ab_Bi_8x4.cl"); + const std::string kernel_src_CL_gemm = read_file("gemm_noshuffle_q4_0_f32.cl"); #endif - backend_ctx->program_CL_gemm = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_CL_gemm.c_str(), compile_opts); - CL_CHECK((backend_ctx->CL_mul_mat_Ab_Bi_8x4 = clCreateKernel(backend_ctx->program_CL_gemm, "kernel_mul_mat_Ab_Bi_8x4", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemm.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } + backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = nullptr; + { + { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_0_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_0_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans = + clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } + } + } + // gemm_noshuffle_q4_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { #include "gemm_noshuffle_q4_1_f32.cl.h" - }; + }; #else const std::string kernel_src = read_file("gemm_noshuffle_q4_1_f32.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_1_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); @@ -2639,70 +4480,82 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve const std::string kernel_src = read_file("gemv_noshuffle_q4_1_f32.cl"); #endif - cl_program prog = build_program_from_source( - backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_gemv_compile_opts); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_1_f32_mc3", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // mul_mm_q8_0_f32_8x4 + // gemm_noshuffle_q5_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src_q8_8x4_gemm { - #include "mul_mm_q8_0_f32_8x4.cl.h" - }; + const std::string kernel_src { + #include "gemm_noshuffle_q5_0_f32.cl.h" + }; #else - const std::string kernel_src_q8_8x4_gemm = read_file("mul_mm_q8_0_f32_8x4.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q5_0_f32.cl"); #endif - backend_ctx->program_CL_gemm = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src_q8_8x4_gemm.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_mul_mm_q8_0_f32_8x4 = clCreateKernel(backend_ctx->program_CL_gemm, "kernel_mul_mm_q8_0_f32_8x4", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_general_q8_0_f32 + // gemm_noshuffle_q5_0_q8_1_dp4a (dp4a dense q5_0 prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_0_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q5_0_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -DSIMDGROUP_WIDTH=" + - std::to_string(backend_ctx->adreno_wave_size); + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src_CL_gemv_general { - #include "gemv_noshuffle_general_q8_0_f32.cl.h" + const std::string kernel_src { + #include "gemv_noshuffle_q5_0_f32.cl.h" }; #else - const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_general_q8_0_f32.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_q5_0_f32.cl"); #endif - - cl_program prog = build_program_from_source( - backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->CL_mul_mat_vec_q8_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_0_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q4_k_f32 + // gemm_noshuffle_q5_1_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q4_k_f32.cl.h" - }; + #include "gemm_noshuffle_q5_1_f32.cl.h" + }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q4_k_f32.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q5_1_f32.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_1_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q4_k_f32 + // gemv_noshuffle_q5_1_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; @@ -2712,1380 +4565,5293 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q4_k_f32.cl.h" + #include "gemv_noshuffle_q5_1_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32.cl"); + const std::string kernel_src = read_file("gemv_noshuffle_q5_1_f32.cl"); #endif - - cl_program prog = build_program_from_source( - backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_gemv_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_1_f32", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - std::string CL_moe_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable " - " -cl-fast-relaxed-math"; - - // gemv_moe_mxfp4_f32 + // gemm_noshuffle_iq4_nl_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_moe_mxfp4_f32.cl.h" + #include "gemm_noshuffle_iq4_nl_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_moe_mxfp4_f32.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_f32.cl"); #endif - backend_ctx->program_gemv_moe_mxfp4_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemv_moe_mxfp4_f32, "kernel_gemv_moe_mxfp4_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_moe_mxfp4_f32 - { + // gemm_noshuffle_iq4_nl_q8_1_dp4a (dp4a dense IQ4_NL prefill GEMM) + if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_moe_mxfp4_f32.cl.h" + #include "gemm_noshuffle_iq4_nl_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemm_moe_mxfp4_f32.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_q8_1_dp4a.cl"); #endif - backend_ctx->program_gemm_moe_mxfp4_f32 = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemm_moe_mxfp4_f32, "kernel_gemm_moe_mxfp4_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q6_k_f32 - { + // gemm_noshuffle_q4_0_q8_1_dp4a (dp4a dense q4_0 prefill GEMM) + if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemv_noshuffle_q6_k_f32.cl.h" + #include "gemm_noshuffle_q4_0_q8_1_dp4a.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q4_0_q8_1_dp4a.cl"); #endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemv_noshuffle_iq4_nl_f32 + { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_gemv_compile_opts); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_iq4_nl_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_iq4_nl_f32.cl"); +#endif - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_iq4_nl_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q6_k_f32 + // mul_mm_q8_0_f32_8x4 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q6_k_f32.cl.h" + #include "gemm_noshuffle_q8_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q8_0_f32.cl"); #endif - cl_program prog = - build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); - - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemv_noshuffle_q5_k_f32 + // gemm_noshuffle_q8_0_f32_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_f32_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } + + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } + + // gemv_noshuffle_general_q8_0_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + - " -cl-mad-enable "; + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); if (backend_ctx->has_vector_subgroup_broadcast) { - CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } #ifdef GGML_OPENCL_EMBED_KERNELS - const std::string kernel_src { - #include "gemv_noshuffle_q5_k_f32.cl.h" + const std::string kernel_src_CL_gemv_general { + #include "gemv_noshuffle_q8_0_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemv_noshuffle_q5_k_f32.cl"); + const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q8_0_f32.cl"); #endif - cl_program prog = build_program_from_source( - backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_gemv_compile_opts); + cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts); - CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q8_0_f32_splitk", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } - // gemm_noshuffle_q5_k_f32 + // gemm_noshuffle_q4_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS const std::string kernel_src { - #include "gemm_noshuffle_q5_k_f32.cl.h" + #include "gemm_noshuffle_q4_k_f32.cl.h" }; #else - const std::string kernel_src = read_file("gemm_noshuffle_q5_k_f32.cl"); + const std::string kernel_src = read_file("gemm_noshuffle_q4_k_f32.cl"); #endif - cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_f32", &err), err)); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_r1", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_kimg", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_cok", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - GGML_LOG_CONT("\n"); -} - -// XXX static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { -// XXX static bool initialized = false; -// XXX static ggml_backend_opencl_context *backend_ctx = nullptr; -static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev); - -namespace /* anonymous */ { -extern struct ggml_backend_device_i ggml_backend_opencl_device_i; -} + // gemm_noshuffle_q4_k_q8_1_dp4a (dp4a dense prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_k_q8_1_dp4a.cl"); +#endif + // X1 uses a smaller N tile to reduce local memory and register use. + int tile_n = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32; + if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) { tile_n = atoi(e); } + GGML_ASSERT(tile_n >= 4 && tile_n <= 64 && tile_n % 4 == 0); + backend_ctx->tile_n_dp4a_q4_k = tile_n; + std::string dp4a_opts = compile_opts + " -DTILESIZE_N=" + std::to_string(tile_n); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), dp4a_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); -// Look for available and suitable devices. -static std::vector ggml_opencl_probe_devices(ggml_backend_reg * reg) { - std::vector found_devices; + // Compile a smaller N tile to avoid unused columns in small batches. + int tile_n_narrow = MIN(tile_n, 16); + if (const char * e = getenv("GGML_OPENCL_Q4_K_DP4A_TS_NARROW")) { tile_n_narrow = atoi(e); } + GGML_ASSERT(tile_n_narrow >= 4 && tile_n_narrow <= 64 && tile_n_narrow % 4 == 0); + if (const char * e = getenv("GGML_OPENCL_Q4_K_DP4A_NARROW_MAX")) { + backend_ctx->max_n_dp4a_narrow_q4_k = atoi(e); + } + backend_ctx->tile_n_dp4a_narrow_q4_k = tile_n_narrow; + if (tile_n_narrow != tile_n) { + std::string narrow_opts = compile_opts + " -DTILESIZE_N=" + std::to_string(tile_n_narrow); + cl_program narrow_prog = build_program_from_source(backend_ctx, kernel_src.c_str(), narrow_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow = clCreateKernel(narrow_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg = clCreateKernel(narrow_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err)); + CL_CHECK(clReleaseProgram(narrow_prog)); + } + GGML_LOG_CONT("."); + } -#ifdef GGML_OPENCL_PROFILING - GGML_LOG_INFO("ggml_opencl: OpenCL profiling enabled\n"); + // gemm_noshuffle_q8_0_q8_1_dp4a (dp4a dense q8_0 prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q8_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q8_0_q8_1_dp4a.cl"); #endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - struct cl_device; - struct cl_platform { - cl_platform_id id; - unsigned number; - char name[128]; - char vendor[128]; - struct cl_device * devices; - unsigned n_devices; - struct cl_device * default_device; - }; - - struct cl_device { - struct cl_platform * platform; - cl_device_id id; - unsigned number; - cl_device_type type; - char name[128]; - char version[128]; - }; - - enum { NPLAT = 16, NDEV = 16 }; - - struct cl_platform platforms[NPLAT]; - unsigned n_platforms = 0; - struct cl_device devices[NDEV]; - unsigned n_devices = 0; - struct cl_device * default_device = NULL; - unsigned default_platform_number = 0; + // gemm_noshuffle_q5_k_q8_1_dp4a (dp4a dense prefill GEMM for q5_K) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_k_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - cl_platform_id platform_ids[NPLAT]; - if (clGetPlatformIDs(NPLAT, platform_ids, &n_platforms) != CL_SUCCESS) { - GGML_LOG_ERROR("ggml_opencl: platform IDs not available.\n"); - return found_devices; + // gemm_noshuffle_q6_k_q8_1_dp4a (dp4a dense prefill GEMM for q6_K ffn_down/output) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q6_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q6_k_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - for (unsigned i = 0; i < n_platforms; i++) { - struct cl_platform * p = &platforms[i]; - p->number = i; - p->id = platform_ids[i]; - CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL)); - CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL)); + // quant_a_q8_1 (plain activation q8_1 pre-pass for the dense dp4a GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "quant_a_q8_1.cl.h" + }; +#else + const std::string kernel_src = read_file("quant_a_q8_1.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_quant_a_q8_1 = clCreateKernel(prog, "kernel_quant_a_q8_1", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - cl_device_id device_ids[NDEV]; - cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices); - if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) { - p->n_devices = 0; - } else { - CL_CHECK(clGetDeviceIDsError); + // gemv_noshuffle_q4_k_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X) { + CL_gemv_compile_opts += " -DGGML_CL_A6X_CONSTFOLD_FIX"; + } + // Opt-in: dequant-once-per-block mc3 verify GEMV (factors q4_K dequant + // out of the 3-column loop; byte-identical, lower spill). A/B vs the + // shipped inline mc3 in the same binary. + if (getenv("GGML_OPENCL_Q4K_MC3_DQ")) { + CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_ONCE "; + } + // Opt-in: LDS-staged dequant mc3 verify GEMV (stages the dequantized + // q4_K weights in __local instead of private regs that spill to slow + // global on Adreno; byte-identical). A/B vs inline + dequant-once. + if (getenv("GGML_OPENCL_Q4K_MC3_LDS")) { + CL_gemv_compile_opts += " -DQ4K_MC3_DEQUANT_LDS "; } - p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL; - p->default_device = NULL; - for (unsigned j = 0; j < p->n_devices; j++) { - struct cl_device * d = &devices[n_devices]; - d->number = n_devices++; - d->id = device_ids[j]; - d->platform = p; - CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_NAME, sizeof(d->name), &d->name, NULL)); - CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_TYPE, sizeof(d->type), &d->type, NULL)); - CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_VERSION, sizeof(d->version), &d->version, NULL)); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32.cl"); +#endif - if (p->default_device == NULL && d->type == CL_DEVICE_TYPE_GPU) { - p->default_device = d; - } - } + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - if (default_device == NULL && p->default_device != NULL) { - default_device = p->default_device; - default_platform_number = i; - } + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_mc3", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_splitk", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_splitk_reduce_f32 = clCreateKernel(prog, "kernel_gemv_splitk_reduce_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_glu", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (n_devices == 0) { - GGML_LOG_ERROR("ggml_opencl: could find any OpenCL devices.\n"); - return found_devices; + // gemv_noshuffle_q4_k_f32_o4 — 4-output-per-WI variant for the long-vocab + // q4_K lm_head/embed GEMV (shares one activation read across 4 output rows). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_o4.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_o4.cl"); +#endif + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; + } + cl_program prog = build_program_from_source( + backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_o4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - char * user_platform_string = getenv("GGML_OPENCL_PLATFORM"); - char * user_device_string = getenv("GGML_OPENCL_DEVICE"); - int user_platform_number = -1; - int user_device_number = -1; - cl_device * candidate_devices = nullptr; - unsigned n_candidate_devices = 0; - - unsigned n; - if (user_platform_string != NULL && sscanf(user_platform_string, " %u", &n) == 1 && n < n_platforms) { - user_platform_number = (int)n; - } - if (user_device_string != NULL && sscanf(user_device_string, " %u", &n) == 1 && n < n_devices) { - user_device_number = (int)n; + // gemv_noshuffle_q4_k_f32_tiled — tiled-wide canonical layout, default ON + // (opt out: GGML_OPENCL_Q4K_GEMV_TILED=0; separate convert + GEMV; weights via __global). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_tiled.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_tiled.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled = + clCreateKernel(prog, "kernel_gemv_noshuffle_q4_k_f32_tiled", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (user_platform_number != -1 && user_device_number != -1) { - cl_platform* platform = &platforms[user_platform_number]; - if ((unsigned)user_device_number >= platform->n_devices) { - GGML_LOG_ERROR("ggml_opencl: invalid device number %d\n", user_device_number); - exit(1); + + backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = nullptr; + { + if (backend_ctx->has_vector_subgroup_broadcast) { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q4_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q4_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = + clCreateKernel(prog, "gemv_noshuffle_q4_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - default_device = &platform->devices[user_device_number]; - candidate_devices = platform->devices; - n_candidate_devices = platform->n_devices; - } else { - // Choose a platform by matching a substring. - if (user_platform_number == -1 && user_platform_string != NULL && user_platform_string[0] != 0) { - for (unsigned i = 0; i < n_platforms; i++) { - struct cl_platform * p = &platforms[i]; - if (strstr(p->name, user_platform_string) != NULL || - strstr(p->vendor, user_platform_string) != NULL) { - user_platform_number = (int)i; - break; - } + + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); } - if (user_platform_number == -1) { - GGML_LOG_ERROR("ggml_opencl: no platform matching '%s' was found.\n", user_platform_string); - exit(1); + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); } } + } - int platform_idx = user_platform_number != -1 ? user_platform_number : default_platform_number; - struct cl_platform * p = &platforms[platform_idx]; - candidate_devices = p->devices; - n_candidate_devices = p->n_devices; - default_device = p->default_device; - if (n_candidate_devices == 0) { - GGML_LOG_ERROR("ggml_opencl: selected platform '%s' does not have any devices.\n", p->name); - exit(1); + backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = nullptr; + { + if (backend_ctx->has_vector_subgroup_broadcast) { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = + clCreateKernel(prog, "kernel_gemv_noshuffle_q6_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (user_device_number == -1 && user_device_string != NULL && user_device_string[0] != 0) { - for (unsigned i = 0; i < n_candidate_devices; i++) { - struct cl_device * d = &candidate_devices[i]; - if (strstr(d->name, user_device_string) != NULL) { - user_device_number = d->number; - break; - } + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); } - if (user_device_number == -1) { - GGML_LOG_ERROR("ggml_opencl: no device matching '%s' was found.\n", user_device_string); - exit(1); + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); } } - if (user_device_number != -1) { - candidate_devices = &devices[user_device_number]; - n_candidate_devices = 1; - default_device = &candidate_devices[0]; + } + + backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = nullptr; + backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = nullptr; + { + if (backend_ctx->has_vector_subgroup_broadcast) { + std::string opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -DSIMDGROUP_WIDTH=" + + std::to_string(backend_ctx->adreno_wave_size); +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q5_k_f32_32b_trans.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q5_k_f32_32b_trans.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = + clCreateKernel(prog, "gemv_noshuffle_q5_k_f32_32b_trans", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - GGML_ASSERT(n_candidate_devices > 0); + if (use_adreno_bin_kernels(backend_ctx)) { + size_t bin_size = 0; + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_f32_32b_trans_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } - if (default_device == NULL) { - default_device = &candidate_devices[0]; + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } } } - GGML_ASSERT(n_candidate_devices != 0 && candidate_devices); + std::string CL_moe_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable " + " -cl-fast-relaxed-math"; - // Put the default device in front. - for (unsigned i = 1; i < n_candidate_devices; i++) { - if (&candidate_devices[i] == default_device) { - std::swap(candidate_devices[0], candidate_devices[i]); - default_device = &candidate_devices[0]; - break; - } + // gemv_moe_q4_1_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q4_1_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q4_1_f32_ns.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_moe_q4_1_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_1_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - GGML_LOG_INFO("ggml_opencl: selected platform: '%s'\n", default_device->platform->name); + // gemm_moe_q4_1_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_1_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_1_f32_ns.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - std::vector device_ids; - for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { - device_ids.push_back(dev->id); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_1_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_1_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - cl_int err; - cl_context shared_context; - cl_context_properties properties[] = { (intptr_t) CL_CONTEXT_PLATFORM, (intptr_t) default_device->platform->id, 0 }; + // gemm_moe_q4_1_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_1_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_1_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } - CL_CHECK( - (shared_context = clCreateContext(properties, device_ids.size(), device_ids.data(), NULL, NULL, &err), err)); + // gemv_moe_mxfp4_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_mxfp4_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_mxfp4_f32.cl"); +#endif + backend_ctx->program_gemv_moe_mxfp4_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { - GGML_LOG_INFO("\nggml_opencl: device: '%s (%s)'\n", dev->name, dev->version); + CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemv_moe_mxfp4_f32, "kernel_gemv_moe_mxfp4_f32", &err), err)); + GGML_LOG_CONT("."); + } - auto dev_ctx = std::unique_ptr(new ggml_backend_opencl_device_context{ - /*.platform =*/dev->platform->id, - /*.platform_nane =*/dev->platform->name, - /*.device =*/dev->id, - /*.device_name =*/dev->name, - /*.device_type =*/dev->type, - /*.device_version =*/dev->version, - /*.backend_ctx =*/nullptr, - /*.buffer_type =*/{}, - /*.context =*/shared_context, - }); + // gemm_moe_mxfp4_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_mxfp4_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_mxfp4_f32.cl"); +#endif + backend_ctx->program_gemm_moe_mxfp4_f32 = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - found_devices.push_back(ggml_backend_device{ - /* .iface = */ ggml_backend_opencl_device_i, - /* .reg = */ reg, - /* .context = */ dev_ctx.get(), - }); + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32 = clCreateKernel(backend_ctx->program_gemm_moe_mxfp4_f32, "kernel_gemm_moe_mxfp4_f32", &err), err)); + GGML_LOG_CONT("."); + } - if (!ggml_cl2_init(&found_devices.back())) { - found_devices.pop_back(); - GGML_LOG_INFO("ggml_opencl: drop unsupported device.\n"); - continue; - } + // gemv_moe_q4_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q4_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q4_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - dev_ctx.release(); + CL_CHECK((backend_ctx->kernel_gemv_moe_q4_0_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (found_devices.size()) { - auto * dev_ctx = static_cast(found_devices.front().context); - GGML_LOG_INFO("ggml_opencl: default device: '%s (%s)'\n", dev_ctx->device_name.c_str(), - dev_ctx->device_version.c_str()); + // gemm_moe_q4_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - if (dev_ctx->device_type != CL_DEVICE_TYPE_GPU) { - GGML_LOG_WARN("ggml_opencl: warning, the default device is not a GPU: '%s'.\n", - dev_ctx->device_name.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_moe_q4_0_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } } } - return found_devices; -} + // gemm_moe_q8_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q8_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q8_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); -// Initialize device if it is supported (returns nullptr if it is not). -static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) { - GGML_ASSERT(dev); - GGML_ASSERT(dev->context); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q8_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; - GGML_ASSERT(dev_ctx->platform); - GGML_ASSERT(dev_ctx->device); + // gemv_moe_q5_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q5_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q5_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - if (dev_ctx->backend_ctx) { - return dev_ctx->backend_ctx; + CL_CHECK((backend_ctx->kernel_gemv_moe_q5_0_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - auto backend_ctx = std::make_unique(); - backend_ctx->device = dev_ctx->device; - backend_ctx->gpu_family = GPU_FAMILY::UNKNOWN; + // gemm_moe_q5_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q5_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q5_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // ref_count get increased in ggml_backend_opencl_device_init - // This function is also used to retrieve backend context, so we don't want - // to increase ref_count for each call. We only want to increase ref_count - // when the associated device is initialized - backend_ctx->ref_count = 0; + CL_CHECK((backend_ctx->kernel_gemm_moe_q5_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - if (strstr(dev_ctx->device_name.c_str(), "Adreno") || - strstr(dev_ctx->device_name.c_str(), "Qualcomm") || - strstr(dev_ctx->device_version.c_str(), "Adreno")) { - backend_ctx->gpu_family = GPU_FAMILY::ADRENO; - // Usually device version contains the detailed device name - backend_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_version.c_str()); - if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN) { - backend_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_name.c_str()); - } + // gemv_moe_q5_1_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q5_1_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q5_1_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // Use wave size of 64 for all Adreno GPUs. - backend_ctx->adreno_wave_size = 64; - } else if (strstr(dev_ctx->device_name.c_str(), "Intel")) { - backend_ctx->gpu_family = GPU_FAMILY::INTEL; - } else { - GGML_LOG_ERROR("Unsupported GPU: %s\n", dev_ctx->device_name.c_str()); - backend_ctx->gpu_family = GPU_FAMILY::UNKNOWN; - return nullptr; + CL_CHECK((backend_ctx->kernel_gemv_moe_q5_1_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_1_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { - GGML_LOG_ERROR("ggml_opencl: Adreno-specific kernels should not be enabled for non-Adreno GPUs; " - "run on an Adreno GPU or recompile with CMake option `-DGGML_OPENCL_USE_ADRENO_KERNELS=OFF`\n"); - return nullptr; + // gemm_moe_q5_1_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q5_1_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q5_1_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q5_1_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_1_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } + + // gemv_moe_q4_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q4_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q4_k_f32_ns.cl"); #endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // Populate backend device name - backend_ctx->device_name = dev_ctx->device_name; + CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns_wimg", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - // A local ref of cl_device_id for convenience - cl_device_id device = backend_ctx->device; + // gemm_moe_q4_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q4_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - // Check device OpenCL version, OpenCL 2.0 or above is required - ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, device); - if (opencl_c_version.major < 2) { - GGML_LOG_ERROR("ggml_opencl: OpenCL 2.0 or above is required\n"); - return nullptr; + // gemm_moe_q4_k_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_k_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_k_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } } - // Check driver version - size_t driver_version_str_size; - clGetDeviceInfo(device, CL_DRIVER_VERSION, 0, NULL, &driver_version_str_size); - char *driver_version = (char *)alloca(driver_version_str_size + 1); - clGetDeviceInfo(device, CL_DRIVER_VERSION, driver_version_str_size, driver_version, NULL); - driver_version[driver_version_str_size] = '\0'; - GGML_LOG_INFO("ggml_opencl: OpenCL driver: %s\n", driver_version); - backend_ctx->driver_version = driver_version; + // gemm_moe_q4_k_q8_1_dp4a (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_k_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - backend_ctx->adreno_cl_compiler_version = get_adreno_cl_compiler_version(driver_version); - backend_ctx->has_vector_subgroup_broadcast = - (backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) || - (backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17); - GGML_LOG_INFO("ggml_opencl: vector subgroup broadcast support: %s\n", - backend_ctx->has_vector_subgroup_broadcast ? "true" : "false"); + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - size_t ext_str_size; - clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size); - char *ext_buffer = (char *)alloca(ext_str_size + 1); - clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer, NULL); - ext_buffer[ext_str_size] = '\0'; // ensure it is null terminated - // Check if ext_buffer contains cl_khr_fp16 - backend_ctx->fp16_support = strstr(ext_buffer, "cl_khr_fp16") != NULL; - GGML_LOG_INFO("ggml_opencl: device FP16 support: %s\n", backend_ctx->fp16_support ? "true" : "false"); - // check Adreno large buffer support - backend_ctx->adreno_has_large_buffer = strstr(ext_buffer, "cl_qcom_large_buffer") != NULL; + // gemm_moe_q4_k_q8_1_dp4a_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr; - // fp16 is required - if (!backend_ctx->fp16_support) { - GGML_LOG_ERROR("ggml_opencl: device does not support FP16\n"); - return nullptr; + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_k_q8_1_dp4a_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } } - // If OpenCL 3.0 is supported, then check for cl_khr_subgroups, which becomes - // optional in OpenCL 3.0 (cl_khr_subgroup is mandatory in OpenCL 2.x) - if (opencl_c_version.major == 3 && strstr(ext_buffer, "cl_khr_subgroups") == NULL && - strstr(ext_buffer, "cl_intel_subgroups") == NULL) { - GGML_LOG_ERROR("ggml_opencl: device does not support subgroups (cl_khr_subgroups or cl_intel_subgroups) " - "(note that subgroups is an optional feature in OpenCL 3.0)\n"); - return nullptr; + // gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_mxfp4_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_mxfp4_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - cl_uint base_align_in_bits; - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &base_align_in_bits, NULL)); - GGML_ASSERT(base_align_in_bits % 8u == 0); - backend_ctx->alignment = base_align_in_bits / 8u; - GGML_LOG_INFO("ggml_opencl: mem base addr align: %u\n", backend_ctx->alignment); + // gemm_moe_mxfp4_q8_1_dp4a_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; - clGetDeviceInfo(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(size_t), &backend_ctx->max_alloc_size, NULL); - GGML_LOG_INFO("ggml_opencl: max mem alloc size: %zu MB\n", backend_ctx->max_alloc_size/1024/1024); + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - clGetDeviceInfo(device, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, sizeof(size_t), &backend_ctx->image_max_buffer_size, NULL); - GGML_LOG_INFO("ggml_opencl: device max image buffer size (pixels): %lu\n", backend_ctx->image_max_buffer_size); + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } - clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), &backend_ctx->max_workgroup_size, NULL); - GGML_LOG_INFO("ggml_opencl: device max workgroup size: %lu\n", backend_ctx->max_workgroup_size); + // gemm_moe_mxfp4_q8_1_dp4a_ldck_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr; - // Check SVM. - cl_device_svm_capabilities svm_caps; - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_SVM_CAPABILITIES, sizeof(cl_device_svm_capabilities), &svm_caps, 0)); - GGML_LOG_INFO("ggml_opencl: SVM coarse grain buffer support: %s\n", - svm_caps & CL_DEVICE_SVM_COARSE_GRAIN_BUFFER ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: SVM fine grain buffer support: %s\n", - svm_caps & CL_DEVICE_SVM_FINE_GRAIN_BUFFER ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: SVM fine grain system support: %s\n", - svm_caps & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM ? "true" : "false"); - GGML_LOG_INFO("ggml_opencl: SVM atomics support: %s\n", - svm_caps & CL_DEVICE_SVM_ATOMICS ? "true" : "false"); + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - if (opencl_c_version.major >= 3) { - // Assume it is not available for 3.0, since it is optional in 3.0. - // If compiling against 3.0, then we can query. - backend_ctx->non_uniform_workgroups = false; -#if CL_TARGET_OPENCL_VERSION >= 300 - CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_NON_UNIFORM_WORK_GROUP_SUPPORT, sizeof(cl_bool), - &backend_ctx->non_uniform_workgroups, 0)); -#endif - } else { - GGML_ASSERT(opencl_c_version.major == 2); - // Non-uniform workgroup sizes is mandatory feature in v2.x. - backend_ctx->non_uniform_workgroups = true; + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } } - // Print out configurations -#ifdef GGML_OPENCL_SOA_Q - GGML_LOG_INFO("ggml_opencl: flattening quantized weights representation as struct of arrays (GGML_OPENCL_SOA_Q)\n"); -#endif // GGML_OPENCL_SOA_Q + // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_0_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - GGML_LOG_INFO("ggml_opencl: using kernels optimized for Adreno (GGML_OPENCL_USE_ADRENO_KERNELS)\n"); -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - // determine whether to use large buffer for Adreno - backend_ctx->adreno_use_large_buffer = getenv("GGML_OPENCL_ADRENO_USE_LARGE_BUFFER") != nullptr && - backend_ctx->gpu_family == GPU_FAMILY::ADRENO; - if (backend_ctx->adreno_use_large_buffer) { - if (!backend_ctx->adreno_has_large_buffer) { - GGML_LOG_INFO("ggml_opencl: Adreno large buffer requested but not supported by driver, will use regular buffer\n"); - backend_ctx->adreno_use_large_buffer = false; - } else { - GGML_LOG_INFO("ggml_opencl: Adreno large buffer enabled\n"); + // gemm_moe_q4_0_q8_1_dp4a_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_0_q8_1_dp4a_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } } } - cl_int err; + // gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q8_1_dp4a.cl"); +#endif + const std::string opts80 = CL_moe_compile_opts + " -DMOE_QT=80"; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), opts80.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 = clCreateKernel(prog, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); - // A local ref of cl_context for convenience - cl_context context = backend_ctx->context = dev_ctx->context; + const std::string opts50 = CL_moe_compile_opts + " -DMOE_QT=50"; + cl_program prog50 = + build_program_from_source(backend_ctx, kernel_src.c_str(), opts50.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 = clCreateKernel(prog50, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog50)); + + const std::string opts5 = CL_moe_compile_opts + " -DMOE_QT=5"; + cl_program prog5 = + build_program_from_source(backend_ctx, kernel_src.c_str(), opts5.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k = clCreateKernel(prog5, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog5)); + GGML_LOG_CONT("."); + } - //CL_CHECK((queue = clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err), - // (err != CL_INVALID_QUEUE_PROPERTIES && err != CL_INVALID_VALUE ? err : - // (queue = clCreateCommandQueue(context, device, 0, &err), err) - //))); - cl_command_queue_properties command_queue_props = 0; -#ifdef GGML_OPENCL_PROFILING - command_queue_props |= CL_QUEUE_PROFILING_ENABLE; + // moe_reorder_quant_a_q8_1 (fused reorder + q8_1 quant) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_reorder_quant_a_q8_1.cl.h" + }; +#else + const std::string kernel_src = read_file("moe_reorder_quant_a_q8_1.cl"); #endif - CL_CHECK((backend_ctx->queue = clCreateCommandQueue(context, device, command_queue_props, &err), err)); + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // Load kernels - load_cl_kernels(backend_ctx.get(), opencl_c_version); + CL_CHECK((backend_ctx->kernel_moe_reorder_quant_a_q8_1 = clCreateKernel(prog, "kernel_moe_reorder_quant_a_q8_1", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Allocate intermediate buffers and images - size_t required_A_q_d_bytes = 311164928; - size_t required_A_s_d_bytes = 38895616; - size_t required_B_d_bytes = 45088768; + // transpose_activation_q8 (per-tile transpose of q8_1 activations for the ldck dp4a GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "transpose_activation_q8.cl.h" + }; +#else + const std::string kernel_src = read_file("transpose_activation_q8.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - // Ensure buffer sizes do not exceed the maximum allocation size - size_t max_A_q_d_bytes = MIN(required_A_q_d_bytes, backend_ctx->max_alloc_size); - size_t max_A_s_d_bytes = MIN(required_A_s_d_bytes, backend_ctx->max_alloc_size); - size_t max_B_d_bytes = MIN(required_B_d_bytes, backend_ctx->max_alloc_size); - if (required_A_q_d_bytes > backend_ctx->max_alloc_size) { - GGML_LOG_WARN("ggml_opencl: A_q_d buffer size reduced from %zu to %zu due to device limitations.\n", - required_A_q_d_bytes, max_A_q_d_bytes); + CL_CHECK((backend_ctx->kernel_transpose_activation_q8 = clCreateKernel(prog, "kernel_transpose_activation_q8", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (required_A_s_d_bytes > backend_ctx->max_alloc_size) { - GGML_LOG_WARN("ggml_opencl: A_s_d buffer size reduced from %zu to %zu due to device limitations.\n", - required_A_s_d_bytes, max_A_s_d_bytes); + + // gemv_moe_q5_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q5_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q5_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemv_moe_q5_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q5_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - if (required_B_d_bytes > backend_ctx->max_alloc_size) { - GGML_LOG_WARN("ggml_opencl: B_d buffer size reduced from %zu to %zu due to device limitations.\n", - required_B_d_bytes, max_B_d_bytes); + + // gemm_moe_q5_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q5_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q5_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q5_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q5_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - backend_ctx->prealloc_quant_trans.allocate(context, max_A_q_d_bytes); - backend_ctx->prealloc_scales_trans.allocate(context, max_A_s_d_bytes); - backend_ctx->prealloc_act_trans.allocate(context, max_B_d_bytes); -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + // gemv_moe_q6_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_q6_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_q6_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - backend_ctx->disable_fusion = getenv("GGML_OPENCL_DISABLE_FUSION") != nullptr; + CL_CHECK((backend_ctx->kernel_gemv_moe_q6_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q6_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - dev_ctx->backend_ctx = backend_ctx.release(); - return dev_ctx->backend_ctx; -} + // gemm_moe_q6_k_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q6_k_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q6_k_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); -static void ggml_cl2_free(ggml_backend_t backend) { - ggml_backend_opencl_context * ctx = (ggml_backend_opencl_context *) backend->context; - ctx->free(); + CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q6_k_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - // The CL context is shared by all backends, release it if all backends have been released - bool should_release_opencl = true; - for (auto device : g_ggml_backend_opencl_devices) { - ggml_backend_opencl_device_context * ctx_dev = (ggml_backend_opencl_device_context *) device.context; - if (ctx_dev->backend_ctx->ref_count > 0) { - should_release_opencl = false; + // gemm_moe_q6_k_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q6_k_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_q6_k_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } } } - if (should_release_opencl) { - CL_CHECK(clReleaseContext(ctx->context)); - } -} + // gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM) + if (backend_ctx->has_integer_dot) { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q6_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q6_k_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS -static void transpose_2d( - ggml_backend_opencl_context * backend_ctx, - cl_kernel kernel, - cl_mem src, cl_mem dst, size_t size, - cl_int stride, cl_int rows, - bool blocking = true -) { - static ggml_cl_buffer buf; + CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - cl_event evt; - cl_int err; + // gemm_moe_q6_k_q8_1_dp4a_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr; - buf.allocate(backend_ctx->context, size); + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q6_k_q8_1_dp4a_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); - cl_mem trans; - cl_buffer_region region; + CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } - region.origin = 0; - region.size = size; - CL_CHECK((trans = clCreateSubBuffer( - buf.buffer, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + // gemv_moe_mxfp4_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_moe_mxfp4_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_moe_mxfp4_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &stride)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &rows)); + CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns_wimg", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - size_t local_size[3] = {64, 1, 1}; - size_t global_size[3] = {(size_t)stride, (size_t)rows, 1};; - CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, - global_size, local_size, 0, NULL, NULL)); + // gemm_moe_mxfp4_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_mxfp4_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_mxfp4_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); - if (blocking) { - CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseEvent(evt)); - } else { - CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, NULL)); + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - CL_CHECK(clReleaseMemObject(trans)); -} + // gemm_moe_mxfp4_f32_ns_bin + { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_f32_ns_ila", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_f32_ns_ila", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } -static void transpose_2d_as_8b( - ggml_backend_opencl_context * backend_ctx, - cl_mem src, cl_mem dst, size_t size, - cl_int stride, cl_int rows, - bool blocking = true -) { - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_8_buf, - src, dst, size, stride, rows, blocking); -} + // moe_reorder_b + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_reorder_b.cl.h" + }; +#else + const std::string kernel_src = read_file("moe_reorder_b.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); -static void transpose_2d_as_16b( - ggml_backend_opencl_context * backend_ctx, - cl_mem src, cl_mem dst, size_t size, - cl_int stride, cl_int rows, - bool blocking = true -) { - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, - src, dst, size, stride, rows, blocking); -} + CL_CHECK((backend_ctx->kernel_moe_reorder_b = clCreateKernel(prog, "kernel_moe_reorder_b", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } -static void transpose_2d_as_32b( - ggml_backend_opencl_context * backend_ctx, - cl_mem src, cl_mem dst, size_t size, - cl_int stride, cl_int rows, - bool blocking = true -) { - transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, - src, dst, size, stride, rows, blocking); -} -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + // moe_sort_by_expert + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_sort_by_expert.cl.h" + }; +#else + const std::string kernel_src = read_file("moe_sort_by_expert.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_zero_dst = clCreateKernel(prog, "kernel_moe_zero_dst", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_scatter_stable = clCreateKernel(prog, "kernel_moe_scatter_stable", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } -//------------------------------------------------------------------------------ -// Tensor extra management -//------------------------------------------------------------------------------ -struct ggml_tensor_extra_cl { - // The buffer object that holds the data. - cl_mem data_device; - // The offset into the buffer object. This is primarily for scratch buffer - // and view operation. - // NB: this offset no longer includes view offset (view_offs). Whenever this - // offset is used, view_offs should be considered. - cl_ulong offset; - // The actual size of the cl_mem object. This is needed when returning the - // block to the pool. - size_t actual_size; + // gemv_noshuffle_q6_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32.cl"); +#endif - void reset() { - data_device = nullptr; - offset = 0; - actual_size = 0; - } -}; + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; + } -// Additional tensor extra structs for quantized tensors. -// These tensors are loaded from files and should not be allocated in scratch -- -// they should always be allocated from the pool. Hence, they do not have an -// `offset`, which indicate their locations in the scratch buffer. -struct ggml_tensor_extra_cl_q4_0 { - // Quantized values. - cl_mem q = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem q_img = nullptr; - // Scales. - cl_mem d = nullptr; - // Scales in image1d_buffer_t. - cl_mem d_img = nullptr; - // Size of quantized values. - size_t size_q = 0; - // Size of scales. - size_t size_d = 0; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - ~ggml_tensor_extra_cl_q4_0() { - reset(); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_mc3", &err), err)); + if (getenv("GGML_OPENCL_MC3_PROBE")) { + cl_ulong pm6 = 0, pm4 = 0; size_t wg6 = 0, wg4 = 0, mult = 0; + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm6), &pm6, NULL); + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg6), &wg6, NULL); + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, sizeof(pm4), &pm4, NULL); + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(wg4), &wg4, NULL); + clGetKernelWorkGroupInfo(backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3, backend_ctx->device, CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE, sizeof(mult), &mult, NULL); + fprintf(stderr, "[MC3-PROBE] q4K_mc3 private=%llu wg_cap=%zu | q6K_mc3 private=%llu wg_cap=%zu | pref_mult=%zu\n", + (unsigned long long)pm4, wg4, (unsigned long long)pm6, wg6, mult); + fflush(stderr); + } + GGML_LOG_CONT("."); } - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; + // gemv_noshuffle_q6_k_f32_o4 — 4-output-per-WI variant, opt-in via + // GGML_OPENCL_Q6K_GEMV_O4=1 (~3x fewer dispatches on long-vocab lm_head). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32_o4.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_o4.cl"); +#endif + + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAT "; } - // Currently, q_img and d_img are only initialized when SMALL_ALLOC is - // enabled. They point to the images in ggml_backend_opencl_buffer_context. - // So, there is no need to release them here. - // TODO: initialize them for non SMALL_PATH path, or remove them. - q_img = nullptr; - d_img = nullptr; - size_q = 0; - size_d = 0; - } -}; -struct ggml_tensor_extra_cl_q4_1 { - // Quantized values. - cl_mem q = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem q_img = nullptr; - // Scales. - cl_mem d = nullptr; - // Scales in image1d_buffer_t. - cl_mem d_img = nullptr; - // Min - cl_mem m = nullptr; - // Min in image1d_buffer_t. - cl_mem m_img = nullptr; - // Size of quantized values. - size_t size_q = 0; - // Size of scales. - size_t size_d = 0; - // Size of min values. - size_t size_m = 0; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - ~ggml_tensor_extra_cl_q4_1() { - reset(); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 = clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_o4", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + + // Global-read variant: weights read from __global coalesced instead of + // image1d_buffer (the texture cache caps the streaming lm_head read + // bandwidth). Opt-in via GGML_OPENCL_Q6K_GEMV_O4_GLOBAL. + cl_program prog_g = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts + " -DQ6K_O4_GLOBAL"); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global = + clCreateKernel(prog_g, "kernel_gemv_noshuffle_q6_K_f32_o4_global", &err), err)); + CL_CHECK(clReleaseProgram(prog_g)); + GGML_LOG_CONT("."); } - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } - if (m != nullptr) { - CL_CHECK(clReleaseMemObject(m)); - m = nullptr; - } - // Currently, q_img and d_img are only initialized when SMALL_ALLOC is - // enabled. They point to the images in ggml_backend_opencl_buffer_context. - // So, there is no need to release them here. - // TODO: initialize them for non SMALL_PATH path, or remove them. - q_img = nullptr; - d_img = nullptr; - m_img = nullptr; - size_q = 0; - size_d = 0; - size_m = 0; + // gemv_noshuffle_q6_k_f32_tiled — tiled-wide canonical layout, default ON + // (opt out: GGML_OPENCL_Q6K_GEMV_TILED=0; separate convert + GEMV; weights via __global). + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q6_k_f32_tiled.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q6_k_f32_tiled.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled = + clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 = + clCreateKernel(prog, "kernel_gemv_noshuffle_q6_K_f32_tiled_mc3", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } -}; -struct ggml_tensor_extra_cl_mxfp4 { - // Quantized values. - cl_mem q = nullptr; - // Quantized values in image1d_buffer_t. - cl_mem q_img = nullptr; - // Scales in E8M0. - cl_mem e = nullptr; - // Scales in image1d_buffer_t. - cl_mem e_img = nullptr; - // Size of quantized values. - size_t size_q = 0; - // Size of scales. - size_t size_e = 0; + // gemm_noshuffle_q6_k_f32_tiled — batched (N>1) GEMM over the same tiled-wide + // canonical layout, so batched lm_head/embed stays correct + on GPU. + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q6_k_f32_tiled.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32_tiled.cl"); +#endif + std::string compile_opts = std::string("-cl-std=") + opencl_c_std + " -cl-mad-enable "; + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled = + clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_tiled", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - ~ggml_tensor_extra_cl_mxfp4() { - reset(); + // gemm_noshuffle_q6_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q6_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_K_f32_cok", &err), err)); + GGML_LOG_CONT("."); } - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (e != nullptr) { - CL_CHECK(clReleaseMemObject(e)); - e = nullptr; - } - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q_img)); - q = nullptr; + // gemv_noshuffle_q5_k_f32 + { + std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + + " -cl-mad-enable "; + if (backend_ctx->has_vector_subgroup_broadcast) { + CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST "; } - // Currently, q_img and d_img are not used. They can be image1d_buffer_t - // that wraps around q and d to utilize image access path. - q_img = nullptr; - e_img = nullptr; - size_q = 0; - size_e = 0; - } -}; -struct ggml_tensor_extra_cl_q8_0 { - cl_mem q = nullptr; - cl_mem q_img = nullptr; +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q5_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q5_k_f32.cl"); +#endif - cl_mem d = nullptr; - cl_mem d_img = nullptr; + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), CL_gemv_compile_opts); - size_t size_q = 0; - size_t size_d = 0; + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_k_f32_mc3", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } - ~ggml_tensor_extra_cl_q8_0() { - reset(); + // gemm_noshuffle_q5_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_k_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); } - void reset() { - // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. - // They must be properly released so that the original buffer can be - // properly released to avoid memory leak. - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } - // Currently, q_img and d_img are not used. They can be image1d_buffer_t - // that wraps around q and d to utilize image access path. - q_img = nullptr; - d_img = nullptr; - size_q = 0; - size_d = 0; - } -}; - -struct ggml_tensor_extra_cl_q4_K { - // Quantized values - cl_mem q = nullptr; - // Scales for each super block. - cl_mem s = nullptr; - // Scales - cl_mem d = nullptr; - // Min - cl_mem dm = nullptr; + // repack + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "flash_attn_repack.cl.h" + }; +#else + const std::string kernel_src = read_file("flash_attn_repack.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - ~ggml_tensor_extra_cl_q4_K() { - reset(); + CL_CHECK((backend_ctx->fa.kernel_repack_q_for_wmm = clCreateKernel(prog, "kernel_repack_q_for_wmm", &err), err)); + CL_CHECK((backend_ctx->fa.kernel_repack_k_for_wmm = clCreateKernel(prog, "kernel_repack_k_for_wmm", &err), err)); + CL_CHECK((backend_ctx->fa.kernel_repack_v_for_wmm = clCreateKernel(prog, "kernel_repack_v_for_wmm", &err), err)); + CL_CHECK((backend_ctx->fa.kernel_repack_mask_for_wmm = clCreateKernel(prog, "kernel_repack_mask_for_wmm", &err), err)); + GGML_LOG_CONT("."); } - void reset() { - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (s != nullptr) { - CL_CHECK(clReleaseMemObject(s)); - s = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } - if (dm != nullptr) { - CL_CHECK(clReleaseMemObject(dm)); - dm = nullptr; + // kernel_flash_attn_f32_f16_bin + { + size_t bin_size = 0; + backend_ctx->fa.kernel_flash_attn_f32_f16_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("flash_attn_f32_f16_wmm", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->fa.kernel_flash_attn_f32_f16_bin = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } } } -}; - -struct ggml_tensor_extra_cl_q5_K { - // Lower 4 bits of quantized weights. - cl_mem q = nullptr; - // Upper 1 bit of quantized weights. - cl_mem qh = nullptr; - // Scales for each block. - cl_mem s = nullptr; - // Scales for each super block. - cl_mem d = nullptr; - // Min for each super block. - cl_mem dm = nullptr; + load_cl_kernels_gemm_coopk(backend_ctx); +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + GGML_LOG_CONT("\n"); + backend_ctx->kernels_loaded = true; +} - size_t size_q = 0; - size_t size_qh = 0; - size_t size_s = 0; - size_t size_d = 0; - size_t size_dm = 0; +static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev); +static bool ggml_opencl_is_device_supported(ggml_backend_dev_t dev); + +// FA per-(dk,dv) tile tuning table + GGML_OPENCL_FA_TUNE override parsing. +#include "fa_tune.h" + +// FA variant key for the per-(dk,dv,variant) lazy compile cache. +// Kernel built on first dispatch to reduce kernel loading time. +// NB - a warmup run is recommended to get all necessary FA variants compiled +// before actual runs. +enum ggml_opencl_fa_variant { + FA_VARIANT_PRE = 0, // prepass kernels (kv_pad, mask_pad, blk) + FA_VARIANT_F16 = 1, + FA_VARIANT_F32 = 2, + FA_VARIANT_F32_F16 = 3, + FA_VARIANT_Q8_0 = 4, + FA_VARIANT_Q4_0 = 5, + FA_VARIANT_F32_F16_SPLIT = 6, + FA_VARIANT_Q8_0_SPLIT = 7, + FA_VARIANT_Q4_0_SPLIT = 8, +}; - ~ggml_tensor_extra_cl_q5_K() { - reset(); +static std::string ggml_opencl_fa_kernel_src(ggml_opencl_fa_variant v) { +#ifdef GGML_OPENCL_EMBED_KERNELS + switch (v) { + case FA_VARIANT_F16: + return std::string{ + #include "flash_attn_f16.cl.h" + }; + case FA_VARIANT_F32: + return std::string{ + #include "flash_attn_f32.cl.h" + }; + case FA_VARIANT_F32_F16: + case FA_VARIANT_F32_F16_SPLIT: + return std::string{ + #include "flash_attn_f32_f16.cl.h" + }; + case FA_VARIANT_PRE: + return std::string{ + #include "flash_attn_pre_f16.cl.h" + }; + case FA_VARIANT_Q8_0: + case FA_VARIANT_Q8_0_SPLIT: + return std::string{ + #include "flash_attn_f32_q8_0.cl.h" + }; + case FA_VARIANT_Q4_0: + case FA_VARIANT_Q4_0_SPLIT: + return std::string{ + #include "flash_attn_f32_q4_0.cl.h" + }; } + return {}; +#else + switch (v) { + case FA_VARIANT_F16: return read_file("flash_attn_f16.cl"); + case FA_VARIANT_F32: return read_file("flash_attn_f32.cl"); + case FA_VARIANT_F32_F16: + case FA_VARIANT_F32_F16_SPLIT: return read_file("flash_attn_f32_f16.cl"); + case FA_VARIANT_PRE: return read_file("flash_attn_pre_f16.cl"); + case FA_VARIANT_Q8_0: + case FA_VARIANT_Q8_0_SPLIT: return read_file("flash_attn_f32_q8_0.cl"); + case FA_VARIANT_Q4_0: + case FA_VARIANT_Q4_0_SPLIT: return read_file("flash_attn_f32_q4_0.cl"); + } + return {}; +#endif +} - void reset() { - if (q != nullptr) { - CL_CHECK(clReleaseMemObject(q)); - q = nullptr; - } - if (qh != nullptr) { - CL_CHECK(clReleaseMemObject(qh)); - qh = nullptr; - } - if (s != nullptr) { - CL_CHECK(clReleaseMemObject(s)); - s = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } - if (dm != nullptr) { - CL_CHECK(clReleaseMemObject(dm)); - dm = nullptr; - } +static std::string ggml_opencl_fa_compile_opts(ggml_backend_opencl_context * backend_ctx, + const ggml_opencl_fa_dim * cfg, + ggml_opencl_fa_variant variant) { + std::string opts = backend_ctx->kernel_compile_opts + + " -D DK=" + std::to_string(cfg->dk) + + " -D DV=" + std::to_string(cfg->dv) + + " -D BLOCK_M=" + std::to_string(cfg->bm) + + " -D BLOCK_N=" + std::to_string(cfg->bn); - size_q = 0; - size_qh = 0; - size_s = 0; - size_d = 0; - size_dm = 0; + if (backend_ctx->gpu_family == INTEL) { + opts += " -D FA_SG=32"; + } + + const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || + variant == FA_VARIANT_Q8_0_SPLIT || + variant == FA_VARIANT_Q4_0_SPLIT; + if (is_split) { + opts += " -D N_SPLIT=" + std::to_string(cfg->n_split); + } + // Shuffle define for the split tile paths AND the cluster-parallel decode + // kernel (q1_vec_mq_split_c8) in the plain F32_F16 program. Without it the + // c8 kernel is compiled out (HAS_SUBGROUP_SHUFFLE guard) and dispatch + // falls back to the baseline mq_split. + if ((is_split || variant == FA_VARIANT_F32_F16) && backend_ctx->has_subgroup_shuffle) { + opts += backend_ctx->has_qcom_subgroup_shuffle + ? " -D cl_qcom_subgroup_shuffle=1" + : " -D cl_khr_subgroup_shuffle=1"; + } + // X1E drops the explicit sub-group size pin on the c8 kernels, compiler + // routes the fp16-heavy kernel to a slow variant with explicit subgroup size + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) { + opts += " -D FA_C8_NO_SG_PIN"; + } + // Transposed K tile in local memory: the KV rows the QK loop walks together become + // adjacent, so a group of them is ONE 128-bit local read instead of several narrow + // ones. The QK loop is LDS-read-issue-bound (a wrong-math probe that kept every FMA/dp4a + // but removed the LDS reads ran the kernel ~40% faster), so this is worth up to +26% on + // fa=1 prefill. Output is bit-identical -- only the layout moves. + // + // DK <= 128 only. At DK=256 (gemma-3-4b) it measures 1-2% NEGATIVE and reproduces across + // rounds; padding the row stride does not recover it, so the cause is not a simple bank + // conflict and the wider tile does not want this layout. + // + // Default on within that gate; GGML_OPENCL_FA_K_LDS_T=0 restores the row-major tile. + { + const char * e = getenv("GGML_OPENCL_FA_K_LDS_T"); + if ((e == nullptr || e[0] != '0') && cfg->dk <= 128) { + opts += " -D FA_K_LDS_T"; + } } -}; + return opts; +} -struct ggml_tensor_extra_cl_q6_K { - // Lower 4 bits of quantized weights. - cl_mem ql = nullptr; - // Upper 2 bits of quantized weights. - cl_mem qh = nullptr; - // Scales for each block. - cl_mem s = nullptr; - // Scales for each super block. - cl_mem d = nullptr; +// only register when the kernel's required dispatch workgroup size is within +// the limit of the device's maximum workgroup size +static bool ggml_opencl_fa_kernel_fits_wg(ggml_backend_opencl_context * backend_ctx, + cl_kernel kernel, size_t required_wg, + const char * name, int dk, int dv) { + if (kernel == NULL) { return false; } + const size_t dev_max = backend_ctx->max_workgroup_size; + if (dev_max < required_wg) { + GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d requires WG %zu > device max %zu; skipping registration (will fall back)\n", + name, dk, dv, required_wg, dev_max); + return false; + } + size_t kwg = 0; + cl_int err = clGetKernelWorkGroupInfo(kernel, backend_ctx->device, + CL_KERNEL_WORK_GROUP_SIZE, + sizeof(kwg), &kwg, NULL); + if (err != CL_SUCCESS) { + GGML_LOG_INFO("ggml_opencl: clGetKernelWorkGroupInfo failed for %s DK=%d DV=%d (err=%d); skipping registration\n", + name, dk, dv, err); + return false; + } + if (kwg < required_wg) { + GGML_LOG_INFO("ggml_opencl: %s DK=%d DV=%d per-kernel max %zu < required %zu; skipping registration (will fall back)\n", + name, dk, dv, kwg, required_wg); + return false; + } + return true; +} - size_t size_ql = 0; - size_t size_qh = 0; - size_t size_s = 0; - size_t size_d = 0; +// Log private memory for an FA kernel. Enable via `GGML_OPENCL_FA_LOG_SPILL=1`. +// On Adreno non-zero private_mem means spilling to global memory due to resource +// constraint and usually causes performance degradation. +// (per-work-item, no cache locality) — a strong signal to pick a config +// with smaller per-thread state (e.g. larger N_SPLIT). +static void ggml_opencl_log_fa_kernel_spill(ggml_backend_opencl_context * backend_ctx, + cl_kernel kernel, const char * name, int dk, int dv) { + static const bool enabled = []{ + const char * e = std::getenv("GGML_OPENCL_FA_LOG_SPILL"); + return e && e[0] && e[0] != '0'; + }(); + + if (!enabled || kernel == nullptr) { + return; + } - ~ggml_tensor_extra_cl_q6_K() { - reset(); + cl_ulong priv_mem = 0; + if (clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_PRIVATE_MEM_SIZE, + sizeof(priv_mem), &priv_mem, NULL) == CL_SUCCESS) { + const char * tag = priv_mem > 0 ? "SPILL" : "ok"; + GGML_LOG_INFO("ggml_opencl: [%s] %s DK=%d DV=%d private_mem=%llu bytes\n", + tag, name, dk, dv, (unsigned long long) priv_mem); } +} - void reset() { - if (ql != nullptr) { - CL_CHECK(clReleaseMemObject(ql)); - ql = nullptr; - } - if (qh != nullptr) { - CL_CHECK(clReleaseMemObject(qh)); - qh = nullptr; - } - if (s != nullptr) { - CL_CHECK(clReleaseMemObject(s)); - s = nullptr; - } - if (d != nullptr) { - CL_CHECK(clReleaseMemObject(d)); - d = nullptr; - } +static void ggml_opencl_ensure_fa_pre_kernels(ggml_backend_opencl_context * backend_ctx, int dk, int dv) { + const std::pair dk_dv = {dk, dv}; - size_ql = 0; - size_qh = 0; - size_s = 0; - size_d = 0; + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; break; + } } -}; -//------------------------------------------------------------------------------ -// Backend API -//------------------------------------------------------------------------------ + if (cfg == nullptr) { + GGML_ABORT("ggml_opencl: no flash_attn config for DK=%d DV=%d", dk, dv); + } -// -// backend -// -static const char * ggml_backend_opencl_name(ggml_backend_t backend) { - return "OpenCL"; + // BM-tile metadata is consumed by the prefill dispatch (n_q_blocks / wg + // sizing) regardless of whether the prepass kernels are needed for this + // n_kv — set it unconditionally + backend_ctx->fa.f32_f16_bm[{dk, dv}] = cfg->bm; + backend_ctx->fa.f32_f16_bn[{dk, dv}] = cfg->bn; + backend_ctx->fa.f32_f16_wg_size[{dk, dv}] = cfg->bm; + backend_ctx->fa.bm[{dk, dv}] = cfg->bm; + backend_ctx->fa.bn[{dk, dv}] = cfg->bn; - UNUSED(backend); -} + if (backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0) { return; } -static void ggml_backend_opencl_free(ggml_backend_t backend) { - ggml_cl2_free(backend); + GGML_LOG_INFO("ggml_opencl: lazy-compiling flash_attn prepass for DK=%d DV=%d\n", dk, dv); + cl_int err; + const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_PRE); + const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_PRE); + // retry when kernel compile fails + cl_program prog_pre_f16 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts, + /*fatal=*/false, "fa prepass f16", backend_ctx->queue); + if (!prog_pre_f16) { return; } + cl_kernel k_kv_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_kv_pad_f16", &err); + if (err != CL_SUCCESS) { clReleaseProgram(prog_pre_f16); return; } + cl_kernel k_mask_pad_f16 = clCreateKernel(prog_pre_f16, "flash_attn_mask_pad_f16", &err); + if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseProgram(prog_pre_f16); return; } + cl_kernel k_blk_f16 = clCreateKernel(prog_pre_f16, "flash_attn_blk_f16", &err); + if (err != CL_SUCCESS) { clReleaseKernel(k_kv_pad_f16); clReleaseKernel(k_mask_pad_f16); clReleaseProgram(prog_pre_f16); return; } + backend_ctx->fa.kv_pad_f16[{dk, dv}] = k_kv_pad_f16; + backend_ctx->fa.mask_pad_f16[{dk, dv}] = k_mask_pad_f16; + backend_ctx->fa.blk_f16[{dk, dv}] = k_blk_f16; + clReleaseProgram(prog_pre_f16); } -static void ggml_backend_opencl_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - GGML_UNUSED(backend); - GGML_UNUSED(tensor); - GGML_UNUSED(data); - GGML_UNUSED(offset); - GGML_UNUSED(size); -} +static bool ggml_opencl_ensure_fa_f32_f16_vec_512(ggml_backend_opencl_context * backend_ctx) { + const std::pair key = {512, 512}; + auto & fa = backend_ctx->fa; + if (fa.f32_f16_q1_vec.count(key) > 0) { + return true; + } + if (fa.f32_f16_vec_512_attempted || backend_ctx->kernel_compile_opts.empty()) { + return false; + } + fa.f32_f16_vec_512_attempted = true; -static void ggml_backend_opencl_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - GGML_UNUSED(backend); - GGML_UNUSED(tensor); - GGML_UNUSED(data); - GGML_UNUSED(offset); - GGML_UNUSED(size); -} + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == 512 && d.dv == 512) { + cfg = &d; + break; + } + } + if (cfg == nullptr) { + return false; + } -static bool ggml_backend_opencl_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { - GGML_UNUSED(backend); - GGML_UNUSED(src); - GGML_UNUSED(dst); - return false; + // Compile only vec decode and merge to stay within the Adreno compiler's memory limit. + const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_F32_F16) + + " -D FA_DECODE_ONLY -D FA_VEC_ONLY"; + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, + ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts, + /*fatal=*/false, "fa f32_f16 decode512 vec", backend_ctx->queue); + if (!prog) { + return false; + } + cl_int err; + cl_kernel vec = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec", &err); + if (err != CL_SUCCESS) { + clReleaseProgram(prog); + return false; + } + cl_kernel merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); + clReleaseProgram(prog); + if (err != CL_SUCCESS) { + clReleaseKernel(vec); + return false; + } + if (!ggml_opencl_fa_kernel_fits_wg(backend_ctx, vec, 256, "flash_attn_f32_f16_q1_vec", 512, 512) || + !ggml_opencl_fa_kernel_fits_wg(backend_ctx, merge, 128, "flash_attn_f32_merge", 512, 512)) { + clReleaseKernel(vec); + clReleaseKernel(merge); + return false; + } + fa.f32_f16_q1_vec[key] = vec; + if (fa.f32_merge.count(key) > 0) { + clReleaseKernel(merge); + } else { + fa.f32_merge[key] = merge; + } + return true; } -static void ggml_backend_opencl_synchronize(ggml_backend_t backend) { - auto * backend_ctx = static_cast(backend->context); +static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context * backend_ctx, int dk, int dv, int gqa) { + const std::tuple key = {dk, dv, gqa}; + auto & fa = backend_ctx->fa; + if (fa.f32_f16_mq_decode.count(key) > 0 || fa.f32_f16_mq_decode_attempted.count(key) > 0 || + backend_ctx->kernel_compile_opts.empty()) { + return; + } + if (gqa == 4 && dk != 128 && fa.f32_f16_q1_vec_mq_split.count({dk, dv}) > 0) { + fa.f32_f16_mq_decode[key] = fa.f32_f16_q1_vec_mq_split.at({dk, dv}); + fa.f32_f16_mq_decode_wg[key] = 256; + fa.f32_f16_mq_decode_hs[key] = 1; + return; + } + fa.f32_f16_mq_decode_attempted.insert(key); + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; + break; + } + } + if (cfg == nullptr) { + return; + } - cl_event evt; - CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, 0, nullptr, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseEvent(evt)); + const bool cluster = dk == 64 || dk == 128; + const int head_sub = cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1); + const int nsg_max = dk == 64 ? 1 : (dk == 128 || (gqa == 8 && dk == 256) ? 2 : 4); + const char * kernel_name = cluster ? "flash_attn_f32_f16_q1_vec_mq_split_c8" : "flash_attn_f32_f16_q1_vec_mq_split"; + const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16); + const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_F32_F16) + + " -D FA_MQ_ONLY -D MQ_GQA=" + std::to_string(gqa / head_sub) + + " -D FA_HEAD_SUB=" + std::to_string(head_sub) + + (cluster ? " -D MQ_NSG=" + std::to_string(nsg_max) + " -D FA_CL_C=16" : " -D FA_MQ_SPLIT_ONLY") + + (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : "") + + (gqa == 8 && dk == 256 ? " -D FA_Q1_Q_REG" : ""); + for (int nsg = nsg_max; nsg >= 1; nsg /= 2) { + const size_t wg = 64 * nsg; + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), + opts + " -D MQ_NSG_SPLIT=" + std::to_string(nsg), + /*fatal=*/false, "fa f32_f16 mq decode", backend_ctx->queue); + if (!prog) { + continue; + } + cl_int err; + cl_kernel kernel = clCreateKernel(prog, kernel_name, &err); + clReleaseProgram(prog); + if (err != CL_SUCCESS) { + continue; + } + if (!ggml_opencl_fa_kernel_fits_wg(backend_ctx, kernel, wg, kernel_name, dk, dv)) { + clReleaseKernel(kernel); + continue; + } + fa.f32_f16_mq_decode[key] = kernel; + fa.f32_f16_mq_decode_wg[key] = wg; + fa.f32_f16_mq_decode_hs[key] = head_sub; + return; + } } -// Synchronizes the 'backend_ctx's device with others so that commands -// enqueued to it won't start until commands in the other devices have -// completed. -static void sync_with_other_backends(ggml_backend_opencl_context * backend_ctx) { - if (g_ggml_backend_opencl_devices.size() < 2) - return; // No other devices to synchronize with. +// DK=512 prefill BM-tile +static bool ggml_opencl_ensure_fa_f32_f16_prefill_512(ggml_backend_opencl_context * backend_ctx, bool split) { + const int dk = 512, dv = 512; + const std::pair dk_dv = {dk, dv}; + auto & target = split ? backend_ctx->fa.f32_f16_split : backend_ctx->fa.f32_f16; + if (target.count(dk_dv) > 0) { return true; } - std::vector events; - events.reserve(g_ggml_backend_opencl_devices.size()); + static bool failed[2] = { false, false }; + if (failed[split ? 1 : 0]) { return false; } - for (ggml_backend_device & backend_dev : g_ggml_backend_opencl_devices) { - auto * other_backend_ctx = ggml_cl2_init(&backend_dev); - if (backend_ctx != other_backend_ctx) { - cl_event ev; - CL_CHECK(clEnqueueMarkerWithWaitList(other_backend_ctx->queue, 0, nullptr, &ev)); - CL_CHECK(clFlush(other_backend_ctx->queue)); - events.push_back(ev); - } + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { cfg = &d; break; } } + if (cfg == nullptr) { failed[split ? 1 : 0] = true; return false; } + if (split && cfg->n_split <= 1) { failed[1] = true; return false; } - CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, events.size(), events.data(), nullptr)); - for (auto ev : events) { - CL_CHECK(clReleaseEvent(ev)); + const ggml_opencl_fa_variant variant = split ? FA_VARIANT_F32_F16_SPLIT : FA_VARIANT_F32_F16; + std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + " -D FA_PREFILL_ONLY"; + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, + ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts, + /*fatal=*/false, split ? "fa f32_f16 prefill512 split" : "fa f32_f16 prefill512", + backend_ctx->queue); + if (!prog) { failed[split ? 1 : 0] = true; return false; } + + cl_int err; + cl_kernel k = clCreateKernel(prog, "flash_attn_f32_f16", &err); + if (err != CL_SUCCESS) { clReleaseProgram(prog); failed[split ? 1 : 0] = true; return false; } + target[dk_dv] = k; + if (split) { + backend_ctx->fa.f32_f16_split_wg_size[dk_dv] = cfg->bm * cfg->n_split; + backend_ctx->fa.f32_f16_split_nkv_threshold[dk_dv] = cfg->nkv_split_threshold; + } + ggml_opencl_log_fa_kernel_spill(backend_ctx, k, + split ? "flash_attn_f32_f16 (prefill512 split)" : "flash_attn_f32_f16 (prefill512)", dk, dv); + clReleaseProgram(prog); + + // determine whether to use the K-image variant of the split tile + static const char * pkimg_build_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); + const bool pkimg_build = (pkimg_build_env != NULL) && (pkimg_build_env[0] != '0'); + if (split && pkimg_build && backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) == 0) { + std::string opts_img = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant) + + " -D FA_PREFILL_ONLY -D FA_K_IMG -D FA_TILE_NAME=flash_attn_f32_f16_k_img"; + cl_program prog_img = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, + ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts_img, + /*fatal=*/false, "fa f32_f16 prefill512 split k_img", backend_ctx->queue); + if (prog_img) { + cl_int err_img; + cl_kernel k_img = clCreateKernel(prog_img, "flash_attn_f32_f16_k_img", &err_img); + if (err_img == CL_SUCCESS) { + backend_ctx->fa.f32_f16_split_k_img[dk_dv] = k_img; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_img, + "flash_attn_f32_f16 (prefill512 split k_img)", dk, dv); + } + clReleaseProgram(prog_img); + } } + return true; } -static void sync_with_other_backends(ggml_backend_t backend) { - auto * backend_ctx = static_cast(backend->context); - sync_with_other_backends(backend_ctx); -} +// Compile one (variant, dk, dv); memoised. false = compiler rejected. +static bool ggml_opencl_ensure_fa_variant(ggml_backend_opencl_context * backend_ctx, int dk, int dv, ggml_opencl_fa_variant variant) { + const std::pair dk_dv = {dk, dv}; -static bool ggml_opencl_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { - if (!ggml_can_fuse(cgraph, node_idx, ops)) { + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; break; + } + } + if (cfg == nullptr) { return false; } - if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { - const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; - const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + // if a variant has already been compiled + switch (variant) { + case FA_VARIANT_F16: { + if (backend_ctx->fa.f16.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_F32: { + if (backend_ctx->fa.f32.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_F32_F16: { + // The DK=512 decode-only program does not create the f32_f16 + // prefill kernel; check the q1 kernel instead so that repeated + // calls return a consistent result. + const bool decode_only = (dk == 512); + if (decode_only ? (backend_ctx->fa.f32_f16_q1.count(dk_dv) > 0) + : (backend_ctx->fa.f32_f16.count(dk_dv) > 0)) { + return true; + } + break; + } + case FA_VARIANT_Q8_0: { + if (backend_ctx->fa.f32_q8_0.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_Q4_0: { + if (backend_ctx->fa.f32_q4_0.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_F32_F16_SPLIT: { + if (backend_ctx->fa.f32_f16_split.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_Q8_0_SPLIT: { + if (backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_Q4_0_SPLIT: { + if (backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { + return true; + } + break; + } + case FA_VARIANT_PRE: { + ggml_opencl_ensure_fa_pre_kernels(backend_ctx, dk, dv); + return true; + } + } - GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + // not registered but attempted - meaning these kernels failed to compile + const auto attempt_key = std::make_pair(variant, dk_dv); + if (backend_ctx->fa.variant_attempted.count(attempt_key)) { + return false; + } + backend_ctx->fa.variant_attempted.insert(attempt_key); - // rms_norm only supports f32 - if (mul->src[0]->type != GGML_TYPE_F32 || - mul->src[1]->type != GGML_TYPE_F32 || - mul->type != GGML_TYPE_F32) { - return false; - } + const bool is_split = variant == FA_VARIANT_F32_F16_SPLIT || + variant == FA_VARIANT_Q8_0_SPLIT || + variant == FA_VARIANT_Q4_0_SPLIT; + const bool is_quant = variant == FA_VARIANT_Q8_0 || variant == FA_VARIANT_Q8_0_SPLIT || + variant == FA_VARIANT_Q4_0 || variant == FA_VARIANT_Q4_0_SPLIT; + if (is_quant && (dk % 32 != 0 || dv % 32 != 0)) { + return false; + } + if (is_split && cfg->n_split <= 1) { + return false; + } + if ((variant == FA_VARIANT_Q8_0_SPLIT || variant == FA_VARIANT_Q4_0_SPLIT) && + ((dk / 32) % cfg->n_split != 0 || (dv / 4) % cfg->n_split != 0)) { + return false; + } - // if rms_norm is the B operand, then we don't handle broadcast - if (rms_norm == mul->src[1] && - !ggml_are_same_shape(mul->src[0], rms_norm)) { - return false; - } + const std::string src = ggml_opencl_fa_kernel_src(variant); + if (src.empty()) { return false; } + std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, variant); + + // bypass kernels for DK=512 + const bool fa_decode_only = (variant == FA_VARIANT_F32_F16 && dk == 512); + if (fa_decode_only) { + opts += " -D FA_DECODE_ONLY -D FA_DECODE_MINIMAL"; + } + + // c8 cluster width (GGML_OPENCL_FA_CL_C overrides): value = GQA4 cluster + // width (kernel default 8); the g8 programs use 2x the value (default 16). + // Wider clusters halve per-lane o_acc at the cost of position streams per + // subgroup + static const int fa_cl_c_env = []{ + const char * e = std::getenv("GGML_OPENCL_FA_CL_C"); + const int x = (e && e[0]) ? atoi(e) : 0; + return (x == 8 || x == 16 || x == 32) ? x : 0; // 0 = per-gen default + }(); + // X2E needs 16 to keep per-lane o_acc at 128B (the compiler spills the + // kernel-default width); X1E does not spill, but C=16 is still a measured + // +28-30% DK128-GQA4 decode win there (X1-85, kv 4096/8192), neutral on + // DK64 / GQA1 / quant-KV. + const int fa_cl_c_gqa4 = fa_cl_c_env ? fa_cl_c_env + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E ? 16 : 0); + const std::string opts_cl_c_gqa4 = fa_cl_c_gqa4 + ? " -D FA_CL_C=" + std::to_string(fa_cl_c_gqa4) : std::string(); + const std::string fa_cl_c_g8_val = std::to_string(fa_cl_c_gqa4 ? fa_cl_c_gqa4 * 2 : 16); + + const char * tag = nullptr; + switch (variant) { + case FA_VARIANT_F16: tag = "fa f16"; break; + case FA_VARIANT_F32: tag = "fa f32"; break; + case FA_VARIANT_F32_F16: tag = "fa f32_f16"; break; + case FA_VARIANT_Q8_0: tag = "fa q8_0"; break; + case FA_VARIANT_Q4_0: tag = "fa q4_0"; break; + case FA_VARIANT_F32_F16_SPLIT: tag = "fa f32_f16 split"; break; + case FA_VARIANT_Q8_0_SPLIT: tag = "fa q8_0 split"; break; + case FA_VARIANT_Q4_0_SPLIT: tag = "fa q4_0 split"; break; + default: break; + } + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts + opts_cl_c_gqa4, + /*fatal=*/false, tag, backend_ctx->queue); + if (!prog) { return false; } - // rms_norm assumes contiguous rows - if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) { - return false; + cl_int err; + switch (variant) { + case FA_VARIANT_F16: { + cl_kernel k, kq1; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f16", &err), err)); + CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f16_q1", &err), err)); + backend_ctx->fa.f16[{dk, dv}] = k; + backend_ctx->fa.f16_q1[{dk, dv}] = kq1; + break; } - } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { - const ggml_tensor *norm = cgraph->nodes[node_idx]; - const ggml_tensor *mul = cgraph->nodes[node_idx+1]; - const ggml_tensor *add = cgraph->nodes[node_idx+2]; - const ggml_tensor *w = mul->src[0] == norm ? mul->src[1] : mul->src[0]; - const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; - - // norm fusion only supports F32 - if (norm->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { - return false; + case FA_VARIANT_F32: { + cl_kernel k, kq1; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32", &err), err)); + CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_q1", &err), err)); + backend_ctx->fa.f32[{dk, dv}] = k; + backend_ctx->fa.f32_q1[{dk, dv}] = kq1; + break; } + case FA_VARIANT_F32_F16: { + cl_kernel kq1; + // BM-tile prefill kernel is excluded from the decode-only (DK=512) + if (!fa_decode_only) { + cl_kernel k; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); + backend_ctx->fa.f32_f16[{dk, dv}] = k; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k, "flash_attn_f32_f16", dk, dv); + } + CL_CHECK((kq1 = clCreateKernel(prog, "flash_attn_f32_f16_q1", &err), err)); + backend_ctx->fa.f32_f16_q1[{dk, dv}] = kq1; + ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, "flash_attn_f32_f16_q1", dk, dv); + cl_kernel k_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_split", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.f32_f16_q1_split[{dk, dv}] = k_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, "flash_attn_f32_f16_q1_split", dk, dv); + } + // q1_vec decode kernel (DV-split + subgroup reduce) + cl_kernel k_q1_vec = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, + "flash_attn_f32_f16_q1_vec", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec[{dk, dv}] = k_q1_vec; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, "flash_attn_f32_f16_q1_vec", dk, dv); + } else { + clReleaseKernel(k_q1_vec); + } + } + // KV-head-coalesced vec for high-GQA small models + cl_kernel k_q1_vec_mq = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq, 256, + "flash_attn_f32_f16_q1_vec_mq", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq[{dk, dv}] = k_q1_vec_mq; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq, "flash_attn_f32_f16_q1_vec_mq", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq); + } + } + // KV-head-coalesced + flash-decoding split, reuses merge kernel + cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, + "flash_attn_f32_f16_q1_vec_mq_split", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split[{dk, dv}] = k_q1_vec_mq_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, "flash_attn_f32_f16_q1_vec_mq_split", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split); + } + } + // K-image variant of MQ_GQA=4 split + cl_kernel k_q1_vec_mq_split_k_img = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_k_img, 256, + "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img[{dk, dv}] = k_q1_vec_mq_split_k_img; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_k_img, "flash_attn_f32_f16_q1_vec_mq_split_k_img", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_k_img); + } + } + // Cluster-parallel decode variant + cl_kernel k_q1_vec_mq_split_c8 = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_c8, 256, + "flash_attn_f32_f16_q1_vec_mq_split_c8", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_c8[{dk, dv}] = k_q1_vec_mq_split_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_c8); + } + } + cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; + } + // local-tile decode variant + if (dk == 128 && dv == 128) { + cl_kernel k_lt = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_tile", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lt, 128, + "flash_attn_f32_f16_q1_local_tile", dk, dv)) { + backend_ctx->fa.f32_f16_q1_local_tile[{dk, dv}] = k_lt; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lt, "flash_attn_f32_f16_q1_local_tile", dk, dv); + } else { + clReleaseKernel(k_lt); + } + } + // hybrid local-tile + MQ + FD-split + cl_kernel k_lmq = clCreateKernel(prog, "flash_attn_f32_f16_q1_local_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq, 64, + "flash_attn_f32_f16_q1_local_mq_split", dk, dv)) { + backend_ctx->fa.f32_f16_q1_local_mq_split[{dk, dv}] = k_lmq; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq, "flash_attn_f32_f16_q1_local_mq_split", dk, dv); + } else { + clReleaseKernel(k_lmq); + } + } + } - if (norm->src[0]->ne[0] % 4 != 0) { - return false; + // second compile of the same source with -DMQ_GQA=8. + // FA_MQ_ONLY keeps only the vec_mq kernels so that the program + // compiles within the Adreno compiler's memory budget at DK>=256. + // FA_CL_C for the g8 program: MQ_GQA=8 doubles the c8 kernel's + // per-lane o_acc, so widen the cluster to keep the register + // footprint inside the 192-thread WG cap (see fa_cl_c_gqa4 above + // for the per-gen default). + const std::string opts_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3 -D FA_MQ_ONLY -D FA_CL_C=" + fa_cl_c_g8_val; + cl_program prog_g8 = fa_decode_only ? nullptr : build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8, + /*fatal=*/false, "fa f32_f16 MQ_GQA=8", backend_ctx->queue); + if (prog_g8) { + const size_t mq_g8_required_wg = 192; // Q1_WG_SIZE(64) * MQ_NSG_SPLIT(3) + cl_kernel k_q1_vec_mq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_g8, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_g8[{dk, dv}] = k_q1_vec_mq_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_g8, "flash_attn_f32_f16_q1_vec_mq_g8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_g8); + } + } + cl_kernel k_q1_vec_mq_split_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq_split (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8[{dk, dv}] = k_q1_vec_mq_split_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8, "flash_attn_f32_f16_q1_vec_mq_split_g8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_g8); + } + } + // K-image variant + cl_kernel k_q1_vec_mq_split_g8_k_img = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split_k_img", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8_k_img, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq_split_k_img (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img[{dk, dv}] = k_q1_vec_mq_split_g8_k_img; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8_k_img, "flash_attn_f32_f16_q1_vec_mq_split_g8_k_img", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_g8_k_img); + } + } + // Cluster-parallel decode, MQ_GQA=8 / FA_CL_C=16 specialization + cl_kernel k_q1_vec_mq_split_g8_c8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split_g8_c8, mq_g8_required_wg, + "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8[{dk, dv}] = k_q1_vec_mq_split_g8_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_g8_c8", dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split_g8_c8); + } + } + // hybrid local-tile + MQ_GQA=8 + if (dk == 128 && dv == 128) { + cl_kernel k_lmq_g8 = clCreateKernel(prog_g8, "flash_attn_f32_f16_q1_local_mq_split", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_lmq_g8, 64, + "flash_attn_f32_f16_q1_local_mq_split (g8)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_local_mq_split_g8[{dk, dv}] = k_lmq_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_lmq_g8, "flash_attn_f32_f16_q1_local_mq_split_g8", dk, dv); + } else { + clReleaseKernel(k_lmq_g8); + } + } + } + clReleaseProgram(prog_g8); + } + // NSG_SPLIT=2 programs for the cluster-parallel kernel: its register + // footprint caps the per-kernel WG at 128 on X2 (< the stock 256/192 + // requirement), so it can never register from the stock programs. + // With FA_CL_NCL position streams per subgroup, 2 subgroups still + // carry 16 in-flight rows per WG (baseline WG has 4). FA_MQ_ONLY + // keeps these compiles minimal; skipped when the stock program c8 + // registered (some other device) or shuffles are absent. + if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && + backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count({dk, dv}) == 0) { + const std::string opts_c8_ns2 = opts + " -D FA_MQ_ONLY -D MQ_GQA=4 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2" + opts_cl_c_gqa4; + cl_program prog_c8 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8_ns2, + /*fatal=*/false, "fa f32_f16 c8 NSG2", backend_ctx->queue); + if (prog_c8) { + cl_kernel k_c8 = clCreateKernel(prog_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + // WG = MQ_NSG(2) × Q1_WG_SIZE(=FA_SG): 128 Adreno (64), 64 Intel (32). + const size_t c8_ns2_wg = backend_ctx->gpu_family == INTEL ? 64 : 128; + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8, c8_ns2_wg, + "flash_attn_f32_f16_q1_vec_mq_split_c8 (ns2)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2[{dk, dv}] = k_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8_ns2", dk, dv); + } else { + clReleaseKernel(k_c8); + } + } + clReleaseProgram(prog_c8); + } + } + // FA_CL_C=32 g8 program for the DK=DV=256 GQA=8 + if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && + dk == 256 && dv == 256) { + const std::string opts_g8_c32 = opts + " -D FA_MQ_ONLY -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2 -D FA_CL_C=32"; + cl_program prog_g8_c32 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8_c32, + /*fatal=*/false, "fa f32_f16 c32 g8 d256 NSG2", backend_ctx->queue); + if (prog_g8_c32) { + cl_kernel k_g8_c32 = clCreateKernel(prog_g8_c32, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8_c32, 128, + "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8 c32 d256)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32[{dk, dv}] = k_g8_c32; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8_c32, "flash_attn_f32_f16_q1_vec_mq_split_g8_c32", dk, dv); + } else { + clReleaseKernel(k_g8_c32); + } + } + clReleaseProgram(prog_g8_c32); + } + } + if (!fa_decode_only && backend_ctx->has_subgroup_shuffle && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count({dk, dv}) == 0) { + const std::string opts_g8_c8_ns2 = opts + " -D FA_MQ_ONLY -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2 -D FA_CL_C=" + fa_cl_c_g8_val; + cl_program prog_g8_c8 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_g8_c8_ns2, + /*fatal=*/false, "fa f32_f16 c8 g8 NSG2", backend_ctx->queue); + if (prog_g8_c8) { + cl_kernel k_g8_c8 = clCreateKernel(prog_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8_c8, 128, + "flash_attn_f32_f16_q1_vec_mq_split_c8 (g8 ns2)", dk, dv)) { + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2[{dk, dv}] = k_g8_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8_c8, "flash_attn_f32_f16_q1_vec_mq_split_g8_c8_ns2", dk, dv); + } else { + clReleaseKernel(k_g8_c8); + } + } + clReleaseProgram(prog_g8_c8); + } + } + break; } + case FA_VARIANT_Q8_0: + case FA_VARIANT_Q4_0: { + const bool is_q8 = variant == FA_VARIANT_Q8_0; + const std::string base = is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0"; + const std::string name_q1 = base + "_q1"; + const std::string name_q1_split = base + "_q1_split"; + auto & m_q1 = is_q8 ? backend_ctx->fa.f32_q8_0_q1 : backend_ctx->fa.f32_q4_0_q1; + auto & m_prefill = is_q8 ? backend_ctx->fa.f32_q8_0 : backend_ctx->fa.f32_q4_0; + auto & m_q1_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_split : backend_ctx->fa.f32_q4_0_q1_split; + + cl_kernel k, kq1; + CL_CHECK((kq1 = clCreateKernel(prog, name_q1.c_str(), &err), err)); + CL_CHECK((k = clCreateKernel(prog, base.c_str(), &err), err)); + m_q1[{dk, dv}] = kq1; + m_prefill[{dk, dv}] = k; + ggml_opencl_log_fa_kernel_spill(backend_ctx, kq1, name_q1.c_str(), dk, dv); + ggml_opencl_log_fa_kernel_spill(backend_ctx, k, base.c_str(), dk, dv); + cl_kernel k_split = clCreateKernel(prog, name_q1_split.c_str(), &err); + if (err == CL_SUCCESS) { + m_q1_split[{dk, dv}] = k_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_split, name_q1_split.c_str(), dk, dv); + } - if (!ggml_is_contiguous(norm->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { - return false; - } - } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_GROUP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { - const ggml_tensor *gn = cgraph->nodes[node_idx]; - const ggml_tensor *mul = cgraph->nodes[node_idx+1]; - const ggml_tensor *add = cgraph->nodes[node_idx+2]; - const ggml_tensor *w = mul->src[0] == gn ? mul->src[1] : mul->src[0]; - const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + // DV-split decode variant (q1_vec) + auto & m_q1_vec = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec : backend_ctx->fa.f32_q4_0_q1_vec; + const std::string name_q1_vec = name_q1 + "_vec"; + cl_kernel k_q1_vec = clCreateKernel(prog, name_q1_vec.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec, 256, + name_q1_vec.c_str(), dk, dv)) { + m_q1_vec[{dk, dv}] = k_q1_vec; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec, name_q1_vec.c_str(), dk, dv); + } else { + clReleaseKernel(k_q1_vec); + } + } - if (gn->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { - return false; + // KV-head-coalesced + flash-decoding split + auto & m_mq_split = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split + : backend_ctx->fa.f32_q4_0_q1_vec_mq_split; + const std::string name_mq_split = name_q1 + "_vec_mq_split"; + cl_kernel k_q1_vec_mq_split = clCreateKernel(prog, name_mq_split.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_q1_vec_mq_split, 256, + name_mq_split.c_str(), dk, dv)) { + m_mq_split[{dk, dv}] = k_q1_vec_mq_split; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_q1_vec_mq_split, name_mq_split.c_str(), dk, dv); + } else { + clReleaseKernel(k_q1_vec_mq_split); + } + } + if (!backend_ctx->fa.f32_merge.count({dk, dv})) { + cl_kernel k_merge = clCreateKernel(prog, "flash_attn_f32_merge", &err); + if (err == CL_SUCCESS) { + backend_ctx->fa.f32_merge[{dk, dv}] = k_merge; + } + } + // Second compile with MQ_GQA=8, MQ_NSG=3, MQ_NSG_SPLIT=3 + auto & m_mq_split_g8 = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8 + : backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8; + const std::string opts_mq_g8 = opts + " -D MQ_GQA=8 -D MQ_NSG=3 -D MQ_NSG_SPLIT=3"; + cl_program prog_mq_g8 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_mq_g8, + /*fatal=*/false, is_q8 ? "fa q8_0 MQ_GQA=8" : "fa q4_0 MQ_GQA=8", + backend_ctx->queue); + if (prog_mq_g8) { + const size_t mq_g8_required_wg = 192; + cl_kernel k_g8 = clCreateKernel(prog_mq_g8, name_mq_split.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_g8, mq_g8_required_wg, + name_mq_split.c_str(), dk, dv)) { + m_mq_split_g8[{dk, dv}] = k_g8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_g8, name_mq_split.c_str(), dk, dv); + } else { + clReleaseKernel(k_g8); + } + } + clReleaseProgram(prog_mq_g8); + } + // GQA=4 cluster-parallel program (NSG_SPLIT=2 / WG=128) + if (backend_ctx->has_subgroup_shuffle) { + auto & m_c8_gqa4 = is_q8 ? backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8 + : backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8; + const std::string name_c8_gqa4 = name_q1 + "_vec_mq_split_c8"; + const std::string opts_c8_gqa4 = opts + " -D MQ_GQA=4 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2" + opts_cl_c_gqa4; + cl_program prog_c8_gqa4 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8_gqa4, + /*fatal=*/false, is_q8 ? "fa q8_0 c8 GQA4 NSG2" : "fa q4_0 c8 GQA4 NSG2", + backend_ctx->queue); + if (prog_c8_gqa4) { + cl_kernel k_c8_gqa4 = clCreateKernel(prog_c8_gqa4, name_c8_gqa4.c_str(), &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8_gqa4, 128, + name_c8_gqa4.c_str(), dk, dv)) { + m_c8_gqa4[{dk, dv}] = k_c8_gqa4; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8_gqa4, name_c8_gqa4.c_str(), dk, dv); + } else { + clReleaseKernel(k_c8_gqa4); + } + } + clReleaseProgram(prog_c8_gqa4); + } + } + // Cluster-parallel q4_0 decode kernel + if (!is_q8 && backend_ctx->has_subgroup_shuffle) { + const std::string opts_c8 = opts + " -D MQ_GQA=8 -D MQ_NSG=2 -D MQ_NSG_SPLIT=2"; + cl_program prog_c8 = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts_c8, + /*fatal=*/false, "fa q4_0 c8 NSG2", backend_ctx->queue); + if (prog_c8) { + cl_kernel k_c8 = clCreateKernel(prog_c8, "flash_attn_f32_q4_0_q1_vec_mq_split_c8", &err); + if (err == CL_SUCCESS) { + if (ggml_opencl_fa_kernel_fits_wg(backend_ctx, k_c8, 128, + "flash_attn_f32_q4_0_q1_vec_mq_split_c8 (g8 ns2)", dk, dv)) { + backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8[{dk, dv}] = k_c8; + ggml_opencl_log_fa_kernel_spill(backend_ctx, k_c8, "flash_attn_f32_q4_0_q1_vec_mq_split_g8_c8", dk, dv); + } else { + clReleaseKernel(k_c8); + } + } + clReleaseProgram(prog_c8); + } + } + break; } - - if (!ggml_is_contiguous(gn->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { - return false; + case FA_VARIANT_F32_F16_SPLIT: { + cl_kernel k; + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_f16", &err), err)); + backend_ctx->fa.f32_f16_split[{dk, dv}] = k; + backend_ctx->fa.f32_f16_split_wg_size[{dk, dv}] = cfg->bm * cfg->n_split; + backend_ctx->fa.f32_f16_split_nkv_threshold[{dk, dv}] = cfg->nkv_split_threshold; + break; } + case FA_VARIANT_Q8_0_SPLIT: + case FA_VARIANT_Q4_0_SPLIT: { + const bool is_q8 = variant == FA_VARIANT_Q8_0_SPLIT; + cl_kernel k; + CL_CHECK((k = clCreateKernel(prog, is_q8 ? "flash_attn_f32_q8_0" : "flash_attn_f32_q4_0", &err), err)); + auto & split = is_q8 ? backend_ctx->fa.f32_q8_0_split : backend_ctx->fa.f32_q4_0_split; + auto & split_wg = is_q8 ? backend_ctx->fa.f32_q8_0_split_wg_size : backend_ctx->fa.f32_q4_0_split_wg_size; + auto & split_bm = is_q8 ? backend_ctx->fa.f32_q8_0_split_bm : backend_ctx->fa.f32_q4_0_split_bm; + auto & split_thresh = is_q8 ? backend_ctx->fa.f32_q8_0_split_nkv_threshold : backend_ctx->fa.f32_q4_0_split_nkv_threshold; + split[{dk, dv}] = k; + split_wg[{dk, dv}] = cfg->bm * cfg->n_split; + split_bm[{dk, dv}] = cfg->bm; + split_thresh[{dk, dv}] = 0; // quant prefill: always split + break; + } + default: + break; } - + CL_CHECK(clReleaseProgram(prog)); return true; } -static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor); -static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); -static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); - -static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; +// Compile a quant FA split kernel with a hand-picked (BLOCK_M, N_SPLIT) that +// overrides the default fa_dims tuning, for the DK values where the default +// N_SPLIT is degenerate for quant prefill: +// DK=256: default N_SPLIT=16 leaves DK/32=8 blocks -> 0 blocks/split. +// Override N_SPLIT=8 (1 block/split), BLOCK_M=16. +// DK=96 : DK/32 = 3 blocks, not divisible by the default N_SPLIT=2 -> +// override N_SPLIT=3. BLOCK_M must be 16, not 32: the N_SPLIT=3 +// QK-partial reduction uses sub_group_shuffle, so all 3 split +// threads of a query must land in one subgroup — WG_SIZE = +// BLOCK_M*N_SPLIT must be <= the 64-lane Adreno subgroup (16*3=48). +static bool ggml_opencl_ensure_fa_quant_split_override( + ggml_backend_opencl_context * backend_ctx, + int dk, int dv, int quant_bm, int quant_n_split, bool is_q8_0 +) { + const std::pair dk_dv = {dk, dv}; + if (is_q8_0 && backend_ctx->fa.f32_q8_0_split.count(dk_dv)) { + return true; + } + if (!is_q8_0 && backend_ctx->fa.f32_q4_0_split.count(dk_dv)) { + return true; + } - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; + const ggml_opencl_fa_variant variant = is_q8_0 ? FA_VARIANT_Q8_0_SPLIT : FA_VARIANT_Q4_0_SPLIT; + const auto attempt_key = std::make_pair(variant, dk_dv); + if (backend_ctx->fa.variant_attempted.count(attempt_key)) { + return false; + } - // NOTE: this may oversynchronize by synchronizing with - // backends/devices which don't compute 'cgraph's - // dependencies. - sync_with_other_backends(backend); + backend_ctx->fa.variant_attempted.insert(attempt_key); - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { - continue; + std::string shuffle_opts; + if (backend_ctx->has_subgroup_shuffle) { + shuffle_opts = backend_ctx->has_qcom_subgroup_shuffle + ? " -D cl_qcom_subgroup_shuffle=1" + : " -D cl_khr_subgroup_shuffle=1"; + } + const ggml_opencl_fa_dim * cfg = nullptr; + for (const auto & d : g_opencl_fa_dims) { + if (d.dk == dk && d.dv == dv) { + cfg = &d; break; } + } + if (cfg == nullptr) { + return false; + } - if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { - continue; - } + // BLK_PREPASS_BM is the prepass-kernel BLOCK_M, needed so the quant kernel + // indexes the blk[] classification buffer correctly. + std::string opts = backend_ctx->kernel_compile_opts + shuffle_opts + + " -D DK=" + std::to_string(dk) + + " -D DV=" + std::to_string(dv) + + " -D BLOCK_M=" + std::to_string(quant_bm) + + " -D BLOCK_N=" + std::to_string(cfg->bn) + + " -D N_SPLIT=" + std::to_string(quant_n_split) + + " -D BLK_PREPASS_BM=" + std::to_string(cfg->bm); - if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { - ggml_opencl_op_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); - i += 2; - continue; - } - if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { - ggml_opencl_op_group_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); - i += 2; - continue; - } - if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { - ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]); - i++; - continue; - } + const std::string src = ggml_opencl_fa_kernel_src(variant); + if (src.empty()) { + return false; + } - bool ok = ggml_cl_compute_forward(backend, node); - if (!ok) { - GGML_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); - } - GGML_ASSERT(ok); + const std::string tag = std::string("fa ") + (is_q8_0 ? "q8_0" : "q4_0") + + " split DK=" + std::to_string(dk); + cl_program prog = build_program_from_source_ex( + backend_ctx->context, backend_ctx->device, src.c_str(), opts, + /*fatal=*/false, tag.c_str(), backend_ctx->queue); + if (!prog) { return false; } + cl_int err; + cl_kernel k; + if (is_q8_0) { + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q8_0", &err), err)); + backend_ctx->fa.f32_q8_0_split[dk_dv] = k; + backend_ctx->fa.f32_q8_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; + backend_ctx->fa.f32_q8_0_split_bm[dk_dv] = quant_bm; + backend_ctx->fa.f32_q8_0_split_nkv_threshold[dk_dv] = 0; + } else { + CL_CHECK((k = clCreateKernel(prog, "flash_attn_f32_q4_0", &err), err)); + backend_ctx->fa.f32_q4_0_split[dk_dv] = k; + backend_ctx->fa.f32_q4_0_split_wg_size[dk_dv] = quant_bm * quant_n_split; + backend_ctx->fa.f32_q4_0_split_bm[dk_dv] = quant_bm; + backend_ctx->fa.f32_q4_0_split_nkv_threshold[dk_dv] = 0; } + CL_CHECK(clReleaseProgram(prog)); + return true; +} - return GGML_STATUS_SUCCESS; +namespace /* anonymous */ { +extern struct ggml_backend_device_i ggml_backend_opencl_device_i; } -static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { - ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *)dev->context; - ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; +// Look for available and suitable devices. +static std::vector ggml_opencl_probe_devices(ggml_backend_reg * reg) { + std::vector found_devices; - switch (op->op) { - case GGML_OP_NONE: - return true; - case GGML_OP_GET_ROWS: - switch (op->src[0]->type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - return true; - case GGML_TYPE_Q4_0: -#ifdef GGML_OPENCL_SOA_Q - // We do not support flattened Q4_0 (and possibly other Q's) - return false; -#else // GGML_OPENCL_SOA_Q - return true; -#endif // GGML_OPENCL_SOA_Q - default: - return false; +#ifdef GGML_OPENCL_PROFILING + GGML_LOG_INFO("ggml_opencl: OpenCL profiling enabled\n"); +#endif + + struct cl_device; + struct cl_platform { + cl_platform_id id; + unsigned number; + char name[128]; + char vendor[128]; + struct cl_device * devices; + unsigned n_devices; + struct cl_device * default_device; + }; + + struct cl_device { + struct cl_platform * platform; + cl_device_id id; + unsigned number; + cl_device_type type; + char name[128]; + char version[128]; + }; + + enum { NPLAT = 16, NDEV = 16 }; + + struct cl_platform platforms[NPLAT]; + unsigned n_platforms = 0; + struct cl_device devices[NDEV]; + unsigned n_devices = 0; + struct cl_device * default_device = NULL; + unsigned default_platform_number = 0; + + cl_platform_id platform_ids[NPLAT]; + if (clGetPlatformIDs(NPLAT, platform_ids, &n_platforms) != CL_SUCCESS) { + GGML_LOG_ERROR("ggml_opencl: platform IDs not available.\n"); + return found_devices; + } + + for (unsigned i = 0; i < n_platforms; i++) { + struct cl_platform * p = &platforms[i]; + p->number = i; + p->id = platform_ids[i]; + CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL)); + CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL)); + + cl_device_id device_ids[NDEV]; + cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices); + if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) { + p->n_devices = 0; + } else { + CL_CHECK(clGetDeviceIDsError); + } + p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL; + p->default_device = NULL; + + for (unsigned j = 0; j < p->n_devices; j++) { + struct cl_device * d = &devices[n_devices]; + d->number = n_devices++; + d->id = device_ids[j]; + d->platform = p; + CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_NAME, sizeof(d->name), &d->name, NULL)); + CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_TYPE, sizeof(d->type), &d->type, NULL)); + CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_VERSION, sizeof(d->version), &d->version, NULL)); + + if (p->default_device == NULL && d->type == CL_DEVICE_TYPE_GPU) { + p->default_device = d; } - case GGML_OP_SET_ROWS: - { - // TODO: add support - // ref: https://github.com/ggml-org/llama.cpp/pull/14274 -#pragma message("TODO: implement BF16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, IQ4_NL support (https://github.com/ggml-org/llama.cpp/pull/14661)") - if (op->src[0]->type != GGML_TYPE_F32) { - return false; - } - switch (op->type) { - case GGML_TYPE_F16: - case GGML_TYPE_F32: - return (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); - default: - return false; + } + + if (default_device == NULL && p->default_device != NULL) { + default_device = p->default_device; + default_platform_number = i; + } + } + + if (n_devices == 0) { + GGML_LOG_ERROR("ggml_opencl: could find any OpenCL devices.\n"); + return found_devices; + } + + char * user_platform_string = getenv("GGML_OPENCL_PLATFORM"); + char * user_device_string = getenv("GGML_OPENCL_DEVICE"); + int user_platform_number = -1; + int user_device_number = -1; + cl_device * candidate_devices = nullptr; + unsigned n_candidate_devices = 0; + + unsigned n; + if (user_platform_string != NULL && sscanf(user_platform_string, " %u", &n) == 1 && n < n_platforms) { + user_platform_number = (int)n; + } + if (user_device_string != NULL && sscanf(user_device_string, " %u", &n) == 1 && n < n_devices) { + user_device_number = (int)n; + } + if (user_platform_number != -1 && user_device_number != -1) { + cl_platform* platform = &platforms[user_platform_number]; + if ((unsigned)user_device_number >= platform->n_devices) { + GGML_LOG_ERROR("ggml_opencl: invalid device number %d\n", user_device_number); + exit(1); + } + default_device = &platform->devices[user_device_number]; + candidate_devices = platform->devices; + n_candidate_devices = platform->n_devices; + } else { + // Choose a platform by matching a substring. + if (user_platform_number == -1 && user_platform_string != NULL && user_platform_string[0] != 0) { + for (unsigned i = 0; i < n_platforms; i++) { + struct cl_platform * p = &platforms[i]; + if (strstr(p->name, user_platform_string) != NULL || + strstr(p->vendor, user_platform_string) != NULL) { + user_platform_number = (int)i; + break; } } - case GGML_OP_CPY: - case GGML_OP_DUP: - case GGML_OP_CONT: - switch (op->src[0]->type) { - case GGML_TYPE_F32: - switch (op->type) { - case GGML_TYPE_F16: - case GGML_TYPE_F32: - return true; - default: - return false; - } - case GGML_TYPE_F16: - switch (op->type) { - case GGML_TYPE_F16: - case GGML_TYPE_F32: - return true; - default: - return false; - } - case GGML_TYPE_I32: - switch (op->type) { - case GGML_TYPE_I32: - return true; - default: - return false; - } - default: - return false; + if (user_platform_number == -1) { + GGML_LOG_ERROR("ggml_opencl: no platform matching '%s' was found.\n", user_platform_string); + exit(1); } - case GGML_OP_SET: { - return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32) && - op->type == op->src[0]->type && - op->type == op->src[1]->type; } - case GGML_OP_SCALE: - return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); - case GGML_OP_ADD: - if (op->type == GGML_TYPE_F16) { - const bool src0_ok = op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32; - const bool src1_ok = op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32; - if (src0_ok && src1_ok) { - return true; + + int platform_idx = user_platform_number != -1 ? user_platform_number : default_platform_number; + struct cl_platform * p = &platforms[platform_idx]; + candidate_devices = p->devices; + n_candidate_devices = p->n_devices; + default_device = p->default_device; + if (n_candidate_devices == 0) { + GGML_LOG_ERROR("ggml_opencl: selected platform '%s' does not have any devices.\n", p->name); + exit(1); + } + + if (user_device_number == -1 && user_device_string != NULL && user_device_string[0] != 0) { + for (unsigned i = 0; i < n_candidate_devices; i++) { + struct cl_device * d = &candidate_devices[i]; + if (strstr(d->name, user_device_string) != NULL) { + user_device_number = d->number; + break; } } - case GGML_OP_MUL: - case GGML_OP_DIV: - case GGML_OP_SUB: - return (op->src[0]->type == op->src[1]->type) && - (op->src[0]->type == op->type) && - (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); - case GGML_OP_ADD_ID: - return op->src[0]->type == GGML_TYPE_F32; - case GGML_OP_SQR: - case GGML_OP_SQRT: - return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && - ggml_is_contiguous(op->src[0]); - case GGML_OP_UNARY: - switch (ggml_get_unary_op(op)) { - case GGML_UNARY_OP_GELU: - case GGML_UNARY_OP_SILU: - case GGML_UNARY_OP_RELU: - case GGML_UNARY_OP_GELU_ERF: - case GGML_UNARY_OP_GELU_QUICK: - return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; - case GGML_UNARY_OP_SIGMOID: - return ggml_is_contiguous(op->src[0]); - case GGML_UNARY_OP_TANH: - case GGML_UNARY_OP_NEG: - case GGML_UNARY_OP_EXP: - return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; - case GGML_UNARY_OP_EXPM1: - return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; - case GGML_UNARY_OP_SOFTPLUS: - return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; - default: - return false; - } - case GGML_OP_GLU: - switch (ggml_get_glu_op(op)) { - case GGML_GLU_OP_GEGLU: - case GGML_GLU_OP_REGLU: - case GGML_GLU_OP_SWIGLU: - case GGML_GLU_OP_SWIGLU_OAI: - case GGML_GLU_OP_GEGLU_ERF: - case GGML_GLU_OP_GEGLU_QUICK: - return ggml_is_contiguous_1(op->src[0]) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); - default: - return false; - } - case GGML_OP_TRI: - return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); - case GGML_OP_FILL: - return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); - case GGML_OP_CLAMP: - return op->src[0]->type == GGML_TYPE_F32; - case GGML_OP_SOFT_MAX: - case GGML_OP_NORM: - return true; - case GGML_OP_RMS_NORM: - return op->ne[0] % 4 == 0 && ggml_is_contiguous_rows(op->src[0]); - case GGML_OP_L2_NORM: - return ggml_is_contiguous_rows(op->src[0]); - case GGML_OP_REPEAT: - return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; // Assuming F32 for now, can be expanded - case GGML_OP_PAD: - // TODO: add circular padding support for opencl, see https://github.com/ggml-org/llama.cpp/pull/16985 - if (ggml_get_op_params_i32(op, 8) != 0) { - return false; + if (user_device_number == -1) { + GGML_LOG_ERROR("ggml_opencl: no device matching '%s' was found.\n", user_device_string); + exit(1); } - return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; - case GGML_OP_UPSCALE: { - ggml_scale_mode mode = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & 0xFF); - const bool antialias = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & GGML_SCALE_FLAG_ANTIALIAS); - return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && - (mode == GGML_SCALE_MODE_NEAREST || mode == GGML_SCALE_MODE_BILINEAR) && !antialias; + } + if (user_device_number != -1) { + candidate_devices = &devices[user_device_number]; + n_candidate_devices = 1; + default_device = &candidate_devices[0]; + } + + GGML_ASSERT(n_candidate_devices > 0); + + if (default_device == NULL) { + default_device = &candidate_devices[0]; + } + } + + GGML_ASSERT(n_candidate_devices != 0 && candidate_devices); + + // Put the default device in front. + for (unsigned i = 1; i < n_candidate_devices; i++) { + if (&candidate_devices[i] == default_device) { + std::swap(candidate_devices[0], candidate_devices[i]); + default_device = &candidate_devices[0]; + break; + } + } + + GGML_LOG_INFO("ggml_opencl: selected platform: '%s'\n", default_device->platform->name); + + std::vector device_ids; + for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { + device_ids.push_back(dev->id); + } + + cl_int err; + cl_context shared_context; + cl_context_properties properties[] = { (intptr_t) CL_CONTEXT_PLATFORM, (intptr_t) default_device->platform->id, 0 }; + + CL_CHECK( + (shared_context = clCreateContext(properties, device_ids.size(), device_ids.data(), NULL, NULL, &err), err)); + + for (auto dev = candidate_devices, dev_end = candidate_devices + n_candidate_devices; dev != dev_end; dev++) { + GGML_LOG_INFO("\nggml_opencl: device: '%s (%s)'\n", dev->name, dev->version); + + auto dev_ctx = std::unique_ptr(new ggml_backend_opencl_device_context{ + /*.platform =*/dev->platform->id, + /*.platform_nane =*/dev->platform->name, + /*.device =*/dev->id, + /*.device_name =*/dev->name, + /*.device_type =*/dev->type, + /*.device_version =*/dev->version, + /*.backend_ctx =*/nullptr, + /*.buffer_type =*/{}, + /*.context =*/shared_context, + }); + + found_devices.push_back(ggml_backend_device{ + /* .iface = */ ggml_backend_opencl_device_i, + /* .reg = */ reg, + /* .context = */ dev_ctx.get(), + }); + + if (!ggml_opencl_is_device_supported(&found_devices.back())) { + found_devices.pop_back(); + GGML_LOG_WARN("ggml_opencl: drop unsupported device '%s'.\n", dev->name); + continue; + } + + g_ggml_backend_opencl_dev_ctxs.push_back(std::move(dev_ctx)); + } + + if (found_devices.size()) { + auto * dev_ctx = static_cast(found_devices.front().context); + GGML_LOG_INFO("ggml_opencl: default device: '%s (%s)'\n", dev_ctx->device_name.c_str(), + dev_ctx->device_version.c_str()); + + if (dev_ctx->device_type != CL_DEVICE_TYPE_GPU) { + GGML_LOG_WARN("ggml_opencl: warning, the default device is not a GPU: '%s'.\n", + dev_ctx->device_name.c_str()); + } + } + + return found_devices; +} + +static void ggml_opencl_print_backend_info(ggml_backend_opencl_device_context * dev_ctx) { + GGML_ASSERT(dev_ctx); + GGML_ASSERT(dev_ctx->backend_ctx); + + auto * backend_ctx = dev_ctx->backend_ctx; + + GGML_LOG_INFO("ggml_opencl: OpenCL device: %s\n", + backend_ctx->device_name.c_str()); + GGML_LOG_INFO("ggml_opencl: OpenCL driver: %s\n", + backend_ctx->driver_version.c_str()); + GGML_LOG_INFO("ggml_opencl: vector subgroup broadcast support: %s\n", + backend_ctx->has_vector_subgroup_broadcast ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: subgroup shuffle support: %s\n", + backend_ctx->has_subgroup_shuffle ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: device FP16 support: %s\n", + backend_ctx->fp16_support ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: khr dot product support: %s\n", + backend_ctx->has_integer_dot ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: mem base addr align: %u\n", + backend_ctx->alignment); + GGML_LOG_INFO("ggml_opencl: global mem size: %zu MB\n", + backend_ctx->global_mem_size/1024/1024); + GGML_LOG_INFO("ggml_opencl: max mem alloc size: %zu MB\n", + backend_ctx->max_alloc_size/1024/1024); + GGML_LOG_INFO("ggml_opencl: device max image buffer size (pixels): %zu\n", + backend_ctx->image_max_buffer_size); + GGML_LOG_INFO("ggml_opencl: device max image2d size: %zu x %zu\n", + backend_ctx->image2d_max_width, backend_ctx->image2d_max_height); + GGML_LOG_INFO("ggml_opencl: device max workgroup size: %zu\n", + backend_ctx->max_workgroup_size); + GGML_LOG_INFO("ggml_opencl: SVM coarse grain buffer support: %s\n", + backend_ctx->svm_caps & CL_DEVICE_SVM_COARSE_GRAIN_BUFFER ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: SVM fine grain buffer support: %s\n", + backend_ctx->svm_caps & CL_DEVICE_SVM_FINE_GRAIN_BUFFER ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: SVM fine grain system support: %s\n", + backend_ctx->svm_caps & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: SVM atomics support: %s\n", + backend_ctx->svm_caps & CL_DEVICE_SVM_ATOMICS ? "true" : "false"); + GGML_LOG_INFO("ggml_opencl: cl_qcom_subgroup_shuffle support: %s\n", + backend_ctx->has_qcom_subgroup_shuffle ? "true" : "false"); + + // Print out configurations +#ifdef GGML_OPENCL_SOA_Q + GGML_LOG_INFO("ggml_opencl: flattening quantized weights representation as struct of arrays (GGML_OPENCL_SOA_Q)\n"); +#endif // GGML_OPENCL_SOA_Q + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_LOG_INFO("ggml_opencl: using kernels optimized for Adreno (GGML_OPENCL_USE_ADRENO_KERNELS)\n"); + if (backend_ctx->adreno_xmem_gemm_enabled) { + GGML_LOG_INFO("ggml_opencl: Adreno xmem F16xF32 GEMM enabled (temporary weight prepack)\n"); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + if (backend_ctx->adreno_use_large_buffer) { + if (!backend_ctx->adreno_has_large_buffer) { + GGML_LOG_INFO("ggml_opencl: Adreno large buffer requested but not supported by driver, will use regular buffer\n"); + backend_ctx->adreno_use_large_buffer = false; + } else { + GGML_LOG_INFO("ggml_opencl: Adreno large buffer enabled\n"); + } + } + + if (dev_ctx->opfilter) { + // for information only, the actual regex object is created in ggml_opencl_is_device_supported + GGML_LOG_INFO("ggml_opencl: opfilter regex = \"%s\"\n", dev_ctx->opfilter_str.c_str()); + } +} + +// check if device should be accepted +static bool ggml_opencl_is_device_supported(ggml_backend_dev_t dev) { + GGML_ASSERT(dev); + GGML_ASSERT(dev->context); + + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + GGML_ASSERT(dev_ctx->platform); + GGML_ASSERT(dev_ctx->device); + + if (strstr(dev_ctx->device_name.c_str(), "Adreno") || + strstr(dev_ctx->device_name.c_str(), "Qualcomm") || + strstr(dev_ctx->device_version.c_str(), "Adreno")) { + dev_ctx->gpu_family = GPU_FAMILY::ADRENO; + + // Usually device version contains the detailed device name + dev_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_version.c_str()); + if (dev_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN) { + dev_ctx->adreno_gen = get_adreno_gpu_gen(dev_ctx->device_name.c_str()); + } + } else if (strstr(dev_ctx->device_name.c_str(), "Intel")) { + dev_ctx->gpu_family = GPU_FAMILY::INTEL; + } else { + GGML_LOG_WARN("ggml_opencl: unsupported GPU '%s'.\n", dev_ctx->device_name.c_str()); + dev_ctx->gpu_family = GPU_FAMILY::UNKNOWN; + return false; + } + + ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform); + + // Check device OpenCL version, OpenCL 2.0 or above is required + ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, dev_ctx->device); + if (opencl_c_version.major < 2) { + GGML_LOG_WARN("ggml_opencl: OpenCL 2.0 or above is required\n"); + return false; + } + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (dev_ctx->gpu_family != GPU_FAMILY::ADRENO) { + GGML_LOG_WARN("ggml_opencl: Adreno-specific kernels should not be enabled for non-Adreno GPUs; " + "run on an Adreno GPU or recompile with CMake option `-DGGML_OPENCL_USE_ADRENO_KERNELS=OFF`\n"); + return false; + } +#endif + + size_t ext_str_size; + CL_CHECK(clGetDeviceInfo(dev_ctx->device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size)); + + std::vector ext_buffer(ext_str_size + 1); + CL_CHECK(clGetDeviceInfo(dev_ctx->device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer.data(), NULL)); + ext_buffer[ext_str_size] = '\0'; + + // Check if ext_buffer contains cl_khr_fp16 + bool fp16_support = strstr(ext_buffer.data(), "cl_khr_fp16") != NULL; + if (!fp16_support) { + GGML_LOG_WARN("ggml_opencl: device does not support FP16\n"); + return false; + } + + // If OpenCL 3.0 is supported, then check for cl_khr_subgroups, which becomes + // optional in OpenCL 3.0 (cl_khr_subgroup is mandatory in OpenCL 2.x) + if (opencl_c_version.major == 3 && strstr(ext_buffer.data(), "cl_khr_subgroups") == NULL && + strstr(ext_buffer.data(), "cl_intel_subgroups") == NULL) { + GGML_LOG_WARN("ggml_opencl: device does not support subgroups (cl_khr_subgroups or cl_intel_subgroups) " + "(note that subgroups is an optional feature in OpenCL 3.0)\n"); + return false; + } + + clGetDeviceInfo(dev_ctx->device, CL_DEVICE_GLOBAL_MEM_SIZE, sizeof(size_t), &dev_ctx->global_mem_size, NULL); + + const char * str_opfilter = getenv("GGML_OPENCL_OPFILTER"); + if (str_opfilter) { + dev_ctx->opfilter_str = str_opfilter; + dev_ctx->opfilter = new std::regex(str_opfilter, std::regex_constants::icase); + } + + return true; +} + +// Initialize device if it is supported (returns nullptr if it is not). +static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { + GGML_ASSERT(dev); + GGML_ASSERT(dev->context); + + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + GGML_ASSERT(dev_ctx->platform); + GGML_ASSERT(dev_ctx->device); + + if (dev_ctx->backend_ctx) { + return dev_ctx->backend_ctx; + } + + auto backend_ctx = std::make_unique(); + backend_ctx->device = dev_ctx->device; + backend_ctx->gpu_family = GPU_FAMILY::UNKNOWN; + + // ref_count get increased in ggml_backend_opencl_device_init + // This function is also used to retrieve backend context, so we don't want + // to increase ref_count for each call. We only want to increase ref_count + // when the associated device is initialized + backend_ctx->ref_count = 0; + + backend_ctx->gpu_family = dev_ctx->gpu_family; + backend_ctx->adreno_gen = dev_ctx->adreno_gen; + if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) { + ggml_cl_init_fa_dims_table(); + + // Use wave size of 64 for all Adreno GPUs. + backend_ctx->adreno_wave_size = 64; + } + + // Populate backend device name + backend_ctx->device_name = dev_ctx->device_name; + + // A local ref of cl_device_id for convenience + cl_device_id device = backend_ctx->device; + + ggml_cl_version platform_version = get_opencl_platform_version(dev_ctx->platform); + ggml_cl_version opencl_c_version = get_opencl_c_version(platform_version, device); + + backend_ctx->platform_version = platform_version; + backend_ctx->opencl_c_version = opencl_c_version; + + // Check driver version + size_t driver_version_str_size; + CL_CHECK(clGetDeviceInfo(device, CL_DRIVER_VERSION, 0, NULL, &driver_version_str_size)); + std::vector driver_version(driver_version_str_size + 1); + CL_CHECK(clGetDeviceInfo(device, CL_DRIVER_VERSION, driver_version_str_size, driver_version.data(), NULL)); + driver_version[driver_version_str_size] = '\0'; + backend_ctx->driver_version = driver_version.data(); + + backend_ctx->adreno_cl_compiler_version = get_adreno_cl_compiler_version(driver_version.data()); + backend_ctx->has_vector_subgroup_broadcast = + (backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) || + (backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17) || + (backend_ctx->adreno_cl_compiler_version.type == E17); + + // The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a + // property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41 + // (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts + // that do not need the workarounds do not pay for them. The explicit type check is + // required: newer_than_or_same() is false for every non-E031 compiler, so negating it + // alone would enable the workarounds on E17/DX. + backend_ctx->q6_k_flat_old_compiler = + backend_ctx->adreno_cl_compiler_version.type == E031 && + !backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0); + + size_t ext_str_size; + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size)); + std::vector ext_buffer(ext_str_size + 1); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer.data(), NULL)); + ext_buffer[ext_str_size] = '\0'; // ensure it is null terminated + + // check support for qcom_subgroup_shuffle + if (strstr(ext_buffer.data(), "cl_qcom_subgroup_shuffle") != NULL) { + backend_ctx->has_qcom_subgroup_shuffle = true; + } + + // Check if ext_buffer contains cl_khr_fp16 + backend_ctx->fp16_support = strstr(ext_buffer.data(), "cl_khr_fp16") != NULL; + + // check Adreno large buffer support + backend_ctx->adreno_has_large_buffer = strstr(ext_buffer.data(), "cl_qcom_large_buffer") != NULL; + + // subgroup shuffle support (N_SPLIT>1 FA kernel) + backend_ctx->has_qcom_subgroup_shuffle = strstr(ext_buffer.data(), "cl_qcom_subgroup_shuffle") != NULL; + backend_ctx->has_khr_subgroup_shuffle = strstr(ext_buffer.data(), "cl_khr_subgroup_shuffle") != NULL; + backend_ctx->has_subgroup_shuffle = backend_ctx->has_khr_subgroup_shuffle || backend_ctx->has_qcom_subgroup_shuffle; + + // check for cl_khr_integer_dot_product + // cl_qcom_dot_product8 uses signed * unsigned + // while cl_khr_integer_dot_product uses signed * signed -- we stick with khr for now + backend_ctx->has_integer_dot = + strstr(ext_buffer.data(), "cl_khr_integer_dot_product") != NULL; + + cl_uint base_align_in_bits; + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &base_align_in_bits, NULL)); + GGML_ASSERT(base_align_in_bits % 8u == 0); + backend_ctx->alignment = base_align_in_bits / 8u; + + backend_ctx->global_mem_size = dev_ctx->global_mem_size; + + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(size_t), &backend_ctx->max_alloc_size, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, sizeof(size_t), &backend_ctx->image_max_buffer_size, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image2d_max_width, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image2d_max_height, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), &backend_ctx->max_workgroup_size, NULL)); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_SVM_CAPABILITIES, sizeof(cl_device_svm_capabilities), &backend_ctx->svm_caps, 0)); + + if (opencl_c_version.major >= 3) { + // Assume it is not available for 3.0, since it is optional in 3.0. + // If compiling against 3.0, then we can query. + backend_ctx->non_uniform_workgroups = false; +#if CL_TARGET_OPENCL_VERSION >= 300 + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_NON_UNIFORM_WORK_GROUP_SUPPORT, sizeof(cl_bool), + &backend_ctx->non_uniform_workgroups, 0)); +#endif + } else { + GGML_ASSERT(opencl_c_version.major == 2); + // Non-uniform workgroup sizes is mandatory feature in v2.x. + backend_ctx->non_uniform_workgroups = true; + } + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno xmem F16xF32 GEMM, default on adreno, opt out with GGML_OPENCL_ADRENO_XMEM_GEMM=0. + // This helps models with f16 attention weights, e.g., gpt-oss-20b-f16 + { + const char * xmem_env = getenv("GGML_OPENCL_ADRENO_XMEM_GEMM"); + backend_ctx->adreno_xmem_gemm_enabled = backend_ctx->gpu_family == GPU_FAMILY::ADRENO && + (xmem_env ? atoi(xmem_env) != 0 : true); + } +#endif + + // determine whether to use large buffer for Adreno + backend_ctx->adreno_use_large_buffer = getenv("GGML_OPENCL_ADRENO_USE_LARGE_BUFFER") != nullptr && + backend_ctx->gpu_family == GPU_FAMILY::ADRENO; + + backend_ctx->kernel_compile_opts = ggml_opencl_make_compile_opts(backend_ctx.get()); + CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_COMPUTE_UNITS, + sizeof(backend_ctx->fa.compute_units), &backend_ctx->fa.compute_units, NULL)); + + // ragged moe, unspecified or non-zero means enabled, set to 0 to disable + static const char * ragged_fp16_env = getenv("GGML_OPENCL_MOE_RAGGED_FP16"); + backend_ctx->adreno_use_moe_ragged = (ragged_fp16_env == NULL) ? 1 : (atoi(ragged_fp16_env) != 0); + + // ragged moe, tile-skip granularity (columns per skip-group): 8 = quarter (default), + // 16 = half (legacy), 32 = disabled. Override with GGML_OPENCL_MOE_RAGGED_GRAN={8,16,32} + static const char * ragged_gran_env = getenv("GGML_OPENCL_MOE_RAGGED_GRAN"); + backend_ctx->adreno_moe_ragged_skip_gran = (ragged_gran_env != NULL) ? atoi(ragged_gran_env) : 8; + + // whether fuse moe combine + static const char * fuse_moe_bias_glu_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_GLU"); + backend_ctx->fuse_moe_bias_glu = fuse_moe_bias_glu_env == NULL ? 1 : (atoi(fuse_moe_bias_glu_env) != 0); + + static const char * fuse_moe_bias_combine_env = getenv("GGML_OPENCL_FUSE_MOE_BIAS_COMBINE"); + backend_ctx->fuse_moe_bias_combine = fuse_moe_bias_combine_env == NULL ? 1 : (atoi(fuse_moe_bias_combine_env) != 0); + + static const char * fuse_moe_combine_env = getenv("GGML_OPENCL_FUSE_MOE_COMBINE"); + backend_ctx->fuse_moe_combine = fuse_moe_combine_env == NULL ? 1 : (atoi(fuse_moe_combine_env) != 0); + + // ragged moe dp4 variant + static const char * ragged_dp4_env = getenv("GGML_OPENCL_MOE_RAGGED"); + backend_ctx->adreno_use_moe_ragged_dp4 = ragged_dp4_env == NULL ? 1 : (atoi(ragged_dp4_env) != 0); + +#ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS + // try loading adreno binary kernels if enabled + // if fails to load, builtin kernels will be used + { + dl_handle * kernel_lib_handle = dl_load_library(KERNEL_LIB_NAME); + backend_ctx->adreno_use_bin_kernels = false; + + if (kernel_lib_handle) { + backend_ctx->get_adreno_bin_kernel_func = (get_adreno_bin_kernel_func_t)dl_get_sym(kernel_lib_handle, "get_adreno_kernels"); + if (backend_ctx->get_adreno_bin_kernel_func) { + GGML_LOG_INFO("ggml_opencl: loaded bin kernel library %s\n", KERNEL_LIB_NAME); + backend_ctx->adreno_use_bin_kernels = true; + } else { + GGML_LOG_INFO("ggml_opencl: bin kernel library %s is invalid, will use builtin kernels\n", KERNEL_LIB_NAME); + } + } else { + GGML_LOG_INFO("ggml_opencl: failed to load %s, will use builtin kernels\n", KERNEL_LIB_NAME); + } + } +#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS + + cl_int err; + + // A local ref of cl_context for convenience + cl_context context = backend_ctx->context = dev_ctx->context; + + //CL_CHECK((queue = clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err), + // (err != CL_INVALID_QUEUE_PROPERTIES && err != CL_INVALID_VALUE ? err : + // (queue = clCreateCommandQueue(context, device, 0, &err), err) + //))); + cl_command_queue_properties command_queue_props = 0; +#ifdef GGML_OPENCL_PROFILING + command_queue_props |= CL_QUEUE_PROFILING_ENABLE; +#endif + CL_CHECK((backend_ctx->queue = clCreateCommandQueue(context, device, command_queue_props, &err), err)); + + // delay kernel loading until the first buffer is created + // load_cl_kernels(backend_ctx.get()); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Allocate intermediate buffers and images + size_t required_A_q_d_bytes = 311164928; + size_t required_A_s_d_bytes = 38895616; + size_t required_B_d_bytes = 45088768; + + // Ensure buffer sizes do not exceed the maximum allocation size + size_t max_A_q_d_bytes = MIN(required_A_q_d_bytes, backend_ctx->max_alloc_size); + size_t max_A_s_d_bytes = MIN(required_A_s_d_bytes, backend_ctx->max_alloc_size); + size_t max_B_d_bytes = MIN(required_B_d_bytes, backend_ctx->max_alloc_size); + if (required_A_q_d_bytes > backend_ctx->max_alloc_size) { + GGML_LOG_WARN("ggml_opencl: A_q_d buffer size reduced from %zu to %zu due to device limitations.\n", + required_A_q_d_bytes, max_A_q_d_bytes); + } + if (required_A_s_d_bytes > backend_ctx->max_alloc_size) { + GGML_LOG_WARN("ggml_opencl: A_s_d buffer size reduced from %zu to %zu due to device limitations.\n", + required_A_s_d_bytes, max_A_s_d_bytes); + } + if (required_B_d_bytes > backend_ctx->max_alloc_size) { + GGML_LOG_WARN("ggml_opencl: B_d buffer size reduced from %zu to %zu due to device limitations.\n", + required_B_d_bytes, max_B_d_bytes); + } + + backend_ctx->prealloc_quant_trans.allocate(context, max_A_q_d_bytes); + backend_ctx->prealloc_scales_trans.allocate(context, max_A_s_d_bytes); + backend_ctx->prealloc_act_trans.allocate(context, max_B_d_bytes); +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + backend_ctx->disable_fusion = getenv("GGML_OPENCL_DISABLE_FUSION") != nullptr; + if (const char * env = getenv("GGML_OPENCL_FUSE_MM_GLU")) { + backend_ctx->fuse_mm_glu = atoi(env) != 0; + } + if (const char * env = getenv("GGML_OPENCL_FUSE_RMS_ADD")) { + backend_ctx->fuse_rms_add = atoi(env) != 0; + } + if (const char * env = getenv("GGML_OPENCL_F16_MROW")) { + backend_ctx->f16_mrow = atoi(env) != 0; + } + if (const char * env = getenv("GGML_OPENCL_F16_MROW_RPT")) { + const int v = atoi(env); + backend_ctx->f16_mrow_rpt = (v == 2 || v == 4 || v == 8 || v == 16) ? v : 1; + } + + dev_ctx->backend_ctx = backend_ctx.release(); + return dev_ctx->backend_ctx; +} + +static void ggml_cl_free(ggml_backend_t backend) { + ggml_backend_opencl_context * ctx = (ggml_backend_opencl_context *) backend->context; + ctx->free(); +} + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void transpose_2d( + ggml_backend_opencl_context * backend_ctx, + cl_kernel kernel, + cl_mem src, cl_mem dst, size_t size, + cl_int stride, cl_int rows, + bool blocking = true, + bool auto_local = false // let driver pick local size for non-uniform workgroups +) { + static ggml_cl_buffer buf; + + cl_event evt; + cl_int err; + + buf.allocate(backend_ctx->context, size); + + cl_mem trans; + cl_buffer_region region; + + region.origin = 0; + region.size = size; + CL_CHECK((trans = clCreateSubBuffer( + buf.buffer, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &stride)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &rows)); + + size_t local_size[3] = {64, 1, 1}; + size_t global_size[3] = {(size_t)stride, (size_t)rows, 1};; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, + global_size, auto_local ? NULL : local_size, 0, NULL, NULL)); + + if (blocking) { + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + } else { + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, trans, dst, 0, 0, size, 0, NULL, NULL)); + } + + CL_CHECK(clReleaseMemObject(trans)); +} + +static void transpose_2d_as_8b( + ggml_backend_opencl_context * backend_ctx, + cl_mem src, cl_mem dst, size_t size, + cl_int stride, cl_int rows, + bool blocking = true, + bool auto_local = false +) { + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_8_buf, + src, dst, size, stride, rows, blocking, auto_local); +} + +static void transpose_2d_as_16b( + ggml_backend_opencl_context * backend_ctx, + cl_mem src, cl_mem dst, size_t size, + cl_int stride, cl_int rows, + bool blocking = true +) { + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, + src, dst, size, stride, rows, blocking); +} + +static void transpose_2d_as_32b( + ggml_backend_opencl_context * backend_ctx, + cl_mem src, cl_mem dst, size_t size, + cl_int stride, cl_int rows, + bool blocking = true +) { + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, + src, dst, size, stride, rows, blocking); +} +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +//------------------------------------------------------------------------------ +// Tensor extra management +//------------------------------------------------------------------------------ +struct ggml_tensor_extra_cl { + // The buffer object that holds the data. + cl_mem data_device; + // The offset into the buffer object. This is primarily for scratch buffer + // and view operation. + // NB: this offset no longer includes view offset (view_offs). Whenever this + // offset is used, view_offs should be considered. + cl_ulong offset; + // The actual size of the cl_mem object. This is needed when returning the + // block to the pool. + size_t actual_size; + + void reset() { + data_device = nullptr; + offset = 0; + actual_size = 0; + } +}; + +struct ggml_tensor_extra_cl_q1_0 { + cl_mem q = nullptr; + cl_mem q_img = nullptr; + + cl_mem d = nullptr; + cl_mem d_img = nullptr; + + size_t size_q = 0; + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q1_0() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + q_img = nullptr; + d_img = nullptr; + size_q = 0; + size_d = 0; + } +}; + +// Additional tensor extra structs for quantized tensors. +// These tensors are loaded from files and should not be allocated in scratch -- +// they should always be allocated from the pool. Hence, they do not have an +// `offset`, which indicate their locations in the scratch buffer. +struct ggml_tensor_extra_cl_q4_0 { + // Quantized values. + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Scales. + cl_mem d = nullptr; + // Scales in image1d_buffer_t. + cl_mem d_img = nullptr; + // Size of quantized values. + size_t size_q = 0; + // Size of scales. + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q4_0() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + if (d_img != nullptr) { + CL_CHECK(clReleaseMemObject(d_img)); + d_img = nullptr; + } + size_q = 0; + size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_q4_1 { + // Quantized values. + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Scales. + cl_mem d = nullptr; + // Scales in image1d_buffer_t. + cl_mem d_img = nullptr; + // Min + cl_mem m = nullptr; + // Min in image1d_buffer_t. + cl_mem m_img = nullptr; + // Size of quantized values. + size_t size_q = 0; + // Size of scales. + size_t size_d = 0; + // Size of min values. + size_t size_m = 0; + + ~ggml_tensor_extra_cl_q4_1() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (m != nullptr) { + CL_CHECK(clReleaseMemObject(m)); + m = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + // Currently, q_img and d_img are only initialized when SMALL_ALLOC is + // enabled. They point to the images in ggml_backend_opencl_buffer_context. + // So, there is no need to release them here. + // TODO: initialize them for non SMALL_PATH path, or remove them. + d_img = nullptr; + m_img = nullptr; + size_q = 0; + size_d = 0; + size_m = 0; + } +}; + +struct ggml_tensor_extra_cl_q5_0 { + // Quantized values. + cl_mem qs = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem qs_img = nullptr; + // 5-th bit values. + cl_mem qh = nullptr; + // 5-th bit values in image1d_buffer_t. + cl_mem qh_img = nullptr; + // Scales. + cl_mem d = nullptr; + // Scales in image1d_buffer_t. + cl_mem d_img = nullptr; + // Uniform per-32-block scale (2/block) + min (1/block, = d*16 for the -16 centering) + // for the generic dp4a MoE GEMM. Built from d. + cl_mem scale = nullptr; + cl_mem min = nullptr; + // Size of quantized values. + size_t size_qs = 0; + // Size of 5-th bit values. + size_t size_qh = 0; + // Size of scales. + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q5_0() { + reset(); + } + + void reset() { + if (qs != nullptr) { + CL_CHECK(clReleaseMemObject(qs)); + qs = nullptr; + } + if (qh != nullptr) { + CL_CHECK(clReleaseMemObject(qh)); + qh = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (qs_img != nullptr) { + CL_CHECK(clReleaseMemObject(qs_img)); + qs_img = nullptr; + } + if (scale != nullptr) { + CL_CHECK(clReleaseMemObject(scale)); + scale = nullptr; + } + if (min != nullptr) { + CL_CHECK(clReleaseMemObject(min)); + min = nullptr; + } + + qh_img = nullptr; + d_img = nullptr; + size_qs = 0; + size_qh = 0; + size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_q5_1 { + // Quantized values. + cl_mem qs = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem qs_img = nullptr; + // 5-th bit values. + cl_mem qh = nullptr; + // 5-th bit values in image1d_buffer_t. + cl_mem qh_img = nullptr; + // Scales. + cl_mem d = nullptr; + // Scales in image1d_buffer_t. + cl_mem d_img = nullptr; + // Min + cl_mem m = nullptr; + // Min in image1d_buffer_t. + cl_mem m_img = nullptr; + // Size of quantized values. + size_t size_qs = 0; + // Size of 5-th bit values. + size_t size_qh = 0; + // Size of scales. + size_t size_d = 0; + // Size of min values. + size_t size_m = 0; + + ~ggml_tensor_extra_cl_q5_1() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (qs != nullptr) { + CL_CHECK(clReleaseMemObject(qs)); + qs = nullptr; + } + if (qh != nullptr) { + CL_CHECK(clReleaseMemObject(qh)); + qh = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (m != nullptr) { + CL_CHECK(clReleaseMemObject(m)); + m = nullptr; + } + if (qs_img != nullptr) { + CL_CHECK(clReleaseMemObject(qs_img)); + qs_img = nullptr; + } + // qh_img, d_img, and m_img are not currently allocated separately. + // TODO: initialize them for non SMALL_PATH path, or remove them. + qh_img = nullptr; + d_img = nullptr; + m_img = nullptr; + size_qs = 0; + size_qh = 0; + size_d = 0; + size_m = 0; + } +}; + +struct ggml_tensor_extra_cl_mxfp4 { + // Quantized values. + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Scales in E8M0. + cl_mem e = nullptr; + // Scales in image1d_buffer_t. + cl_mem e_img = nullptr; + // Size of quantized values. + size_t size_q = 0; + // Size of scales. + size_t size_e = 0; + + ~ggml_tensor_extra_cl_mxfp4() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (e != nullptr) { + CL_CHECK(clReleaseMemObject(e)); + e = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + // Currently, e_img is not used. They can be image1d_buffer_t + // that wraps around q and d to utilize image access path. + e_img = nullptr; + size_q = 0; + size_e = 0; + } +}; + +struct ggml_tensor_extra_cl_q8_0 { + cl_mem q = nullptr; + cl_mem q_img = nullptr; + + cl_mem d = nullptr; + cl_mem d_img = nullptr; + + // Uniform per-16-segment scale (16/superblock) for the generic dp4a MoE GEMM. + // Expanded from d at set_tensor; the int8 codes are reused from q. + // q8_0 is symmetric so no min buffer (has_min=0). + cl_mem scale = nullptr; + + size_t size_q = 0; + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q8_0() { + reset(); + } + + void reset() { + // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer. + // They must be properly released so that the original buffer can be + // properly released to avoid memory leak. + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (scale != nullptr) { + CL_CHECK(clReleaseMemObject(scale)); + scale = nullptr; + } + // Currently, q_img and d_img are not used. They can be image1d_buffer_t + // that wraps around q and d to utilize image access path. + q_img = nullptr; + d_img = nullptr; + size_q = 0; + size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_iq4_nl { + cl_mem q = nullptr; + cl_mem q_img = nullptr; + + cl_mem d = nullptr; + cl_mem d_img = nullptr; + + size_t size_q = 0; + size_t size_d = 0; + + ~ggml_tensor_extra_cl_iq4_nl() { + reset(); + } + + void reset() { + if (q != nullptr) { CL_CHECK(clReleaseMemObject(q)); q = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + q_img = nullptr; + d_img = nullptr; + size_q = 0; + size_d = 0; + } +}; + +struct ggml_tensor_extra_cl_q4_K { + // Quantized values + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Scales for each super block. + cl_mem s = nullptr; + // Scales + cl_mem d = nullptr; + // Min + cl_mem dm = nullptr; + + ~ggml_tensor_extra_cl_q4_K() { + reset(); + } + + void reset() { + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (s != nullptr) { + CL_CHECK(clReleaseMemObject(s)); + s = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (dm != nullptr) { + CL_CHECK(clReleaseMemObject(dm)); + dm = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + } +}; + +struct ggml_tensor_extra_cl_q5_K { + // Lower 4 bits of quantized weights. + cl_mem q = nullptr; + // Quantized values in image1d_buffer_t. + cl_mem q_img = nullptr; + // Upper 1 bit of quantized weights. + cl_mem qh = nullptr; + // Scales for each block. + cl_mem s = nullptr; + // Scales for each super block. + cl_mem d = nullptr; + // Min for each super block. + cl_mem dm = nullptr; + // Uniform per-32-block scale (2/block) + min (1/block, = dm*mn) decoded from the + // 6-bit packed s[] for the generic dp4a MoE GEMM kernel_gemm_moe_q8_1_dp4a. + // Built from s/d/dm at set_tensor; q/qh are reused as-is. + cl_mem scale = nullptr; + cl_mem min = nullptr; + + size_t size_q = 0; + size_t size_qh = 0; + size_t size_s = 0; + size_t size_d = 0; + size_t size_dm = 0; + + ~ggml_tensor_extra_cl_q5_K() { + reset(); + } + + void reset() { + if (q != nullptr) { + CL_CHECK(clReleaseMemObject(q)); + q = nullptr; + } + if (qh != nullptr) { + CL_CHECK(clReleaseMemObject(qh)); + qh = nullptr; + } + if (s != nullptr) { + CL_CHECK(clReleaseMemObject(s)); + s = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (dm != nullptr) { + CL_CHECK(clReleaseMemObject(dm)); + dm = nullptr; + } + if (q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q_img)); + q_img = nullptr; + } + if (scale != nullptr) { + CL_CHECK(clReleaseMemObject(scale)); + scale = nullptr; + } + if (min != nullptr) { + CL_CHECK(clReleaseMemObject(min)); + min = nullptr; + } + + size_q = 0; + size_qh = 0; + size_s = 0; + size_d = 0; + size_dm = 0; + } +}; + +struct ggml_tensor_extra_cl_q2_K { + bool noshuffle = false; + cl_mem qs = nullptr; // quantized values + cl_mem sc = nullptr; // one uchar per 16 weights: scale nibble + min nibble + cl_mem dm = nullptr; // half2 per super-block: d, dmin + + size_t size_qs = 0, size_sc = 0, size_dm = 0; + + ~ggml_tensor_extra_cl_q2_K() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + if (dm != nullptr) { CL_CHECK(clReleaseMemObject(dm)); dm = nullptr; } + size_qs = size_sc = size_dm = 0; + } +}; + +struct ggml_tensor_extra_cl_q6_K { + // Lower 4 bits of quantized weights. + cl_mem ql = nullptr; + // Lower 4 bits as image1d_buffer_t + cl_mem ql_img = nullptr; + // Upper 2 bits of quantized weights. + cl_mem qh = nullptr; + // Upper 2 bits as image1d_buffer_t + cl_mem qh_img = nullptr; + // Scales for each block. + cl_mem s = nullptr; + // Scales for each super block. + cl_mem d = nullptr; + + size_t size_ql = 0; + size_t size_qh = 0; + size_t size_s = 0; + size_t size_d = 0; + + ~ggml_tensor_extra_cl_q6_K() { + reset(); + } + + void reset() { + if (ql != nullptr) { + CL_CHECK(clReleaseMemObject(ql)); + ql = nullptr; + } + if (qh != nullptr) { + CL_CHECK(clReleaseMemObject(qh)); + qh = nullptr; + } + if (s != nullptr) { + CL_CHECK(clReleaseMemObject(s)); + s = nullptr; + } + if (d != nullptr) { + CL_CHECK(clReleaseMemObject(d)); + d = nullptr; + } + if (ql_img != nullptr) { + CL_CHECK(clReleaseMemObject(ql_img)); + ql_img = nullptr; + } + if (qh_img != nullptr) { + CL_CHECK(clReleaseMemObject(qh_img)); + qh_img = nullptr; + } + + size_ql = 0; + size_qh = 0; + size_s = 0; + size_d = 0; + } +}; + +//------------------------------------------------------------------------------ +// Backend API +//------------------------------------------------------------------------------ + +// +// backend +// +static const char * ggml_backend_opencl_name(ggml_backend_t backend) { + return "OpenCL"; + + UNUSED(backend); +} + +static void ggml_backend_opencl_free(ggml_backend_t backend) { + ggml_cl_free(backend); +} + +static void ggml_backend_opencl_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + GGML_UNUSED(backend); + GGML_UNUSED(tensor); + GGML_UNUSED(data); + GGML_UNUSED(offset); + GGML_UNUSED(size); +} + +static void ggml_backend_opencl_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + GGML_UNUSED(backend); + GGML_UNUSED(tensor); + GGML_UNUSED(data); + GGML_UNUSED(offset); + GGML_UNUSED(size); +} + +static bool ggml_backend_opencl_cpy_tensor_async(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst) { + GGML_UNUSED(backend); + GGML_UNUSED(src); + GGML_UNUSED(dst); + return false; +} + +static void ggml_backend_opencl_synchronize(ggml_backend_t backend) { + auto * backend_ctx = static_cast(backend->context); + + cl_event evt; + CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, 0, nullptr, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); +} + +// Synchronizes the 'backend_ctx's device with others so that commands +// enqueued to it won't start until commands in the other devices have +// completed. +static void sync_with_other_backends(ggml_backend_opencl_context * backend_ctx) { + if (g_ggml_backend_opencl_devices.size() < 2) { + return; // No other devices to synchronize with. + } + + std::vector events; + events.reserve(g_ggml_backend_opencl_devices.size()); + + for (ggml_backend_device & backend_dev : g_ggml_backend_opencl_devices) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) backend_dev.context; + auto * other_backend_ctx = dev_ctx->backend_ctx; + + if (backend_ctx != other_backend_ctx) { + cl_event ev; + CL_CHECK(clEnqueueMarkerWithWaitList(other_backend_ctx->queue, 0, nullptr, &ev)); + CL_CHECK(clFlush(other_backend_ctx->queue)); + events.push_back(ev); + } + } + + CL_CHECK(clEnqueueBarrierWithWaitList(backend_ctx->queue, events.size(), events.data(), nullptr)); + for (auto ev : events) { + CL_CHECK(clReleaseEvent(ev)); + } +} + +static void sync_with_other_backends(ggml_backend_t backend) { + auto * backend_ctx = static_cast(backend->context); + sync_with_other_backends(backend_ctx); +} + +// True if two tensors share a device buffer with overlapping byte ranges. The pool +// allocator may place a fused op's output over a sequentially-dead input (safe for the +// original separate kernels, but a read/write race inside one fused kernel). +static bool ggml_cl_tensors_overlap(const ggml_tensor * x, const ggml_tensor * y) { + ggml_tensor_extra_cl * ex = (ggml_tensor_extra_cl *)x->extra; + ggml_tensor_extra_cl * ey = (ggml_tensor_extra_cl *)y->extra; + if (!ex || !ey || ex->data_device != ey->data_device) { return false; } + const cl_ulong xo = ex->offset + x->view_offs, xe = xo + ggml_nbytes(x); + const cl_ulong yo = ey->offset + y->view_offs, ye = yo + ggml_nbytes(y); + return xo < ye && yo < xe; +} + +// Detect the MoE combine epilogue: router-weight MUL ([n_embd,k,nt] * [1,k,nt]) followed +// by k VIEWs of it and a (k-1)-long ADD reduction chain producing [n_embd, nt]. When it +// matches (and the output does not alias the inputs), the whole subgraph collapses to one +// weighted-sum-across-experts kernel. +static bool ggml_opencl_can_fuse_moe_combine(const struct ggml_cgraph * cgraph, int node_idx, + const ggml_tensor ** out_final_add) { + const ggml_tensor * mul = cgraph->nodes[node_idx]; + if (mul->op != GGML_OP_MUL) { return false; } + const ggml_tensor * experts = mul->src[0]; + const ggml_tensor * weights = mul->src[1]; + if (!experts || !weights) { return false; } + if (experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { return false; } + + const int64_t n_embd = experts->ne[0]; + const int64_t k = experts->ne[1]; + const int64_t nt = experts->ne[2]; + if (k < 2 || k > 64 || experts->ne[3] != 1 || n_embd % 4 != 0) { return false; } + if (weights->ne[0] != 1 || weights->ne[1] != k || weights->ne[2] != nt || weights->ne[3] != 1) { return false; } + if (mul->ne[0] != n_embd || mul->ne[1] != k || mul->ne[2] != nt) { return false; } + // the fused kernel needs contiguous experts/weights and a contiguous 2D dst + if (!ggml_is_contiguous(experts) || !ggml_is_contiguous(weights)) { return false; } + + const int n_nodes = 1 + (int)k + (int)(k - 1); // MUL + k*VIEW + (k-1)*ADD + if (n_nodes >= 32) { return false; } + if (node_idx + n_nodes > cgraph->n_nodes) { return false; } + + enum ggml_op ops[1 + 64 + 63]; + int n = 0; + ops[n++] = GGML_OP_MUL; + for (int j = 0; j < (int)k; ++j) { ops[n++] = GGML_OP_VIEW; } + for (int j = 0; j < (int)k - 1; ++j) { ops[n++] = GGML_OP_ADD; } + const int outs[] = { node_idx + n_nodes - 1 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { return false; } + + for (int j = 0; j < (int)k; ++j) { + const ggml_tensor * vw = cgraph->nodes[node_idx + 1 + j]; + if (vw->op != GGML_OP_VIEW || vw->src[0] != mul || vw->ne[0] != n_embd || vw->ne[1] != nt) { return false; } + } + const ggml_tensor * final_add = cgraph->nodes[node_idx + n_nodes - 1]; + if (final_add->op != GGML_OP_ADD || final_add->type != GGML_TYPE_F32 || + final_add->ne[0] != n_embd || final_add->ne[1] != nt || final_add->ne[2] != 1) { return false; } + if (!ggml_is_contiguous(final_add)) { return false; } + // the fused kernel reads experts + writes final_add in one pass; bail if the + // pool allocator overlapped the output with the (large) experts input -- would race. + // The small weights input is copied to a private scratch in the dispatch, so its own + // aliasing with the output is handled there and does not block the fusion. + if (ggml_cl_tensors_overlap(experts, final_add)) { return false; } + + *out_final_add = final_add; + return true; +} + +// Detect the gpt-oss MoE bias+activation epilogue on the PREFILL path: +// {MUL_MAT_ID(gate), ADD_ID(gate_bias), MUL_MAT_ID(up), ADD_ID(up_bias), GLU(swiglu_oai)}. +// The two matmuls still run as their own dispatches (the prefill GEMM is the vendor's); +// what collapses is the epilogue — both add_id passes are in-place read-modify-writes of a +// tensor the GLU immediately reads again, so they are three full passes over the same +// [n_ff, n_expert_used, n_tokens] f32 tensor where one suffices. +// +// The decode counterpart is handled by the mxfp4 fused GEMV arm in ggml_opencl_can_fuse, +// which folds the matmul too; this one deliberately fires only when that cannot (ne[2] > 1). +static bool ggml_opencl_can_fuse_moe_bias_glu(const struct ggml_cgraph * cgraph, int node_idx) { + if (node_idx + 4 >= cgraph->n_nodes) { + return false; + } + + const enum ggml_op mg_ops[] = { GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_MUL_MAT_ID, GGML_OP_ADD_ID, GGML_OP_GLU }; + const int mg_out[] = { node_idx + 4 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, 5, mg_ops, mg_out, 1)) { + return false; + } + + const ggml_tensor * gmm = cgraph->nodes[node_idx]; + const ggml_tensor * gad = cgraph->nodes[node_idx+1]; + const ggml_tensor * umm = cgraph->nodes[node_idx+2]; + const ggml_tensor * uad = cgraph->nodes[node_idx+3]; + const ggml_tensor * glu = cgraph->nodes[node_idx+4]; + + if (ggml_get_glu_op(glu) != GGML_GLU_OP_SWIGLU_OAI) { + return false; + } + // Prefill only — at one token the mxfp4 arm above folds the matmul as well. + if (gmm->src[1]->ne[2] == 1) { + return false; + } + // Wiring: both matmuls share the activation and the expert selection, each add_id + // biases its own matmul, and the GLU consumes the two biased results as separate + // operands (so the same-buffer ne00_off/ne10_off split path is not in play). + if (gad->src[0] != gmm || uad->src[0] != umm || + glu->src[0] != gad || glu->src[1] != uad || + umm->src[1] != gmm->src[1] || umm->src[2] != gmm->src[2]) { + return false; + } + // A swapped GLU would exchange the gate/up roles the fused kernel hard-codes. + if (ggml_get_op_params_i32(glu, 1)) { + return false; + } + if (gad->type != GGML_TYPE_F32 || uad->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { + return false; + } + if (!gad->src[1] || gad->src[1]->type != GGML_TYPE_F32 || + !uad->src[1] || uad->src[1]->type != GGML_TYPE_F32) { + return false; + } + if (!gad->src[2] || gad->src[2]->type != GGML_TYPE_I32 || uad->src[2] != gad->src[2]) { + return false; + } + // Full width on both operands: the kernel writes one output element per input pair. + if (!ggml_are_same_shape(gad, uad) || glu->ne[0] != gad->ne[0] || + glu->ne[1] != gad->ne[1] || glu->ne[2] != gad->ne[2] || glu->ne[3] != gad->ne[3]) { + return false; + } + if (gad->ne[3] != 1) { + return false; + } + // The destination is addressed by (expert slot, token) rather than the GLU's flat row + // walk; those agree only for a contiguous destination. + if (!ggml_is_contiguous(glu) || !ggml_is_contiguous(gmm) || !ggml_is_contiguous(umm)) { + return false; + } + return true; +} + +static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + +// Runs the gate and up matmuls unchanged, then one kernel in place of +// add_id(gate) + add_id(up) + swiglu_oai. See ggml_opencl_can_fuse_moe_bias_glu. +static void ggml_cl_moe_bias_glu_fused(ggml_backend_t backend, ggml_tensor * gate_mm, const ggml_tensor * gate_add, + ggml_tensor * up_mm, const ggml_tensor * up_add, const ggml_tensor * glu) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_cl_mul_mat_id(backend, gate_mm->src[0], gate_mm->src[1], gate_mm); + ggml_cl_mul_mat_id(backend, up_mm->src[0], up_mm->src[1], up_mm); + + const ggml_tensor * gbias = gate_add->src[1]; + const ggml_tensor * ubias = up_add->src[1]; + const ggml_tensor * ids = gate_add->src[2]; + + ggml_tensor_extra_cl * eg = (ggml_tensor_extra_cl *)gate_mm->extra; + ggml_tensor_extra_cl * egb = (ggml_tensor_extra_cl *)gbias->extra; + ggml_tensor_extra_cl * eu = (ggml_tensor_extra_cl *)up_mm->extra; + ggml_tensor_extra_cl * eub = (ggml_tensor_extra_cl *)ubias->extra; + ggml_tensor_extra_cl * ei = (ggml_tensor_extra_cl *)ids->extra; + ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)glu->extra; + + cl_ulong off_g = eg->offset + gate_mm->view_offs; + cl_ulong off_gb = egb->offset + gbias->view_offs; + cl_ulong off_u = eu->offset + up_mm->view_offs; + cl_ulong off_ub = eub->offset + ubias->view_offs; + cl_ulong off_i = ei->offset + ids->view_offs; + cl_ulong off_d = ed->offset + glu->view_offs; + + const cl_ulong nb01_g = gate_mm->nb[1]; + const cl_ulong nb02_g = gate_mm->nb[2]; + const cl_ulong nb01_u = up_mm->nb[1]; + const cl_ulong nb02_u = up_mm->nb[2]; + const cl_ulong nb11_g = gbias->nb[1]; + const cl_ulong nb11_u = ubias->nb[1]; + const cl_ulong nb21 = ids->nb[1]; + const cl_ulong nbd1 = glu->nb[1]; + const cl_ulong nbd2 = glu->nb[2]; + + const int ne0 = (int)glu->ne[0]; + const float alpha = ggml_get_op_params_f32(glu, 2); + const float limit = ggml_get_op_params_f32(glu, 3); + + cl_kernel kernel = backend_ctx->kernel_add_id_add_id_swiglu_oai; + + int i = 0; + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eg->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_g)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &egb->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_gb)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eu->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_u)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &eub->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_ub)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &ei->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_i)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_mem), &ed->data_device)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &off_d)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_g)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_g)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb01_u)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb02_u)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_g)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb11_u)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd1)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(cl_ulong), &nbd2)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float), &limit)); + CL_CHECK(clSetKernelArg(kernel, i++, sizeof(float), &alpha)); + + const int nth = MIN(ne0, (int) backend_ctx->get_kernel_workgroup_size(kernel)); + size_t global_work_size[] = { (size_t)glu->ne[1]*nth, (size_t)glu->ne[2], 1 }; + size_t local_work_size[] = { (size_t)nth, 1, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, glu); +} + +// Fusion B: the MoE down-projection bias add feeding the combine. +// +// The graph runs ADD_ID(down_bias) and then immediately the combine subgraph +// {MUL(router weights), k VIEWs, k-1 ADDs}, and the ADD_ID's only consumer is that +// MUL. Since the ADD_ID is an in-place read-modify-write of a tensor the combine +// reads once more, the bias can be added inside the combine instead, dropping a +// full pass over [n_embd, k, n_tokens]. +// +// Shape checks for the combine tail are delegated to ggml_opencl_can_fuse_moe_combine +// (which also owns the n_nodes >= 32 bail and the experts/dst aliasing bail); what is +// added here is the ADD_ID wiring plus a subgraph check over the WHOLE run, so that +// the intermediate bias result is confirmed not to escape. +static bool ggml_opencl_can_fuse_moe_bias_combine(const struct ggml_cgraph * cgraph, int node_idx, + const ggml_tensor ** out_final_add) { + if (node_idx + 1 >= cgraph->n_nodes) { + return false; + } + const ggml_tensor * add = cgraph->nodes[node_idx]; + if (add->op != GGML_OP_ADD_ID) { + return false; + } + const ggml_tensor * mul = cgraph->nodes[node_idx+1]; + if (mul->op != GGML_OP_MUL || mul->src[0] != add) { + return false; + } + + const ggml_tensor * final_add = NULL; + if (!ggml_opencl_can_fuse_moe_combine(cgraph, node_idx+1, &final_add)) { + return false; + } + + const ggml_tensor * raw = add->src[0]; + const ggml_tensor * bias = add->src[1]; + const ggml_tensor * ids = add->src[2]; + if (!raw || !bias || !ids) { + return false; + } + if (raw->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 || + ids->type != GGML_TYPE_I32 || add->type != GGML_TYPE_F32) { + return false; + } + // The combine reads the raw matmul output with the strides it computed from the + // add_id result, so the two must have the same layout. + if (!ggml_are_same_shape(raw, add) || !ggml_is_contiguous(raw)) { + return false; + } + if (raw->nb[1] != add->nb[1] || raw->nb[2] != add->nb[2]) { + return false; + } + // ids is indexed as [expert slot, token]; the combine walks the same two axes. + if (ids->ne[0] < add->ne[1] || ids->ne[1] < add->ne[2]) { + return false; + } + + // Whole-run escape check: ADD_ID + MUL + k VIEWs + (k-1) ADDs, only the last node escapes. + const int k = (int)add->ne[1]; + const int n_nodes = 2 + k + (k - 1); + if (n_nodes >= 32 || node_idx + n_nodes > cgraph->n_nodes) { + return false; + } + enum ggml_op ops[32]; + int n = 0; + ops[n++] = GGML_OP_ADD_ID; + ops[n++] = GGML_OP_MUL; + for (int j = 0; j < k; ++j) ops[n++] = GGML_OP_VIEW; + for (int j = 0; j < k - 1; ++j) ops[n++] = GGML_OP_ADD; + const int outs[] = { node_idx + n_nodes - 1 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { + return false; + } + + *out_final_add = final_add; + return true; +} + + +// Fusion B dispatch: the combine, reading the RAW matmul output and adding the +// per-expert bias row inline. See ggml_opencl_can_fuse_moe_bias_combine. +static void ggml_cl_moe_bias_combine_fused(ggml_backend_t backend, const ggml_tensor * add, + const ggml_tensor * mul, const ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + + const ggml_tensor * experts = add->src[0]; // raw matmul output, bias not yet applied + const ggml_tensor * bias = add->src[1]; + const ggml_tensor * ids = add->src[2]; + const ggml_tensor * weights = mul->src[1]; + + ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra; + ggml_tensor_extra_cl * eb = (ggml_tensor_extra_cl *)bias->extra; + ggml_tensor_extra_cl * ei = (ggml_tensor_extra_cl *)ids->extra; + ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra; + ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong off_e = ee->offset + experts->view_offs; + cl_ulong off_b = eb->offset + bias->view_offs; + cl_ulong off_i = ei->offset + ids->view_offs; + cl_ulong off_w = ew->offset + weights->view_offs; + cl_ulong off_d = ed->offset + dst->view_offs; + + const int n_embd4 = (int)(experts->ne[0] / 4); + const int k = (int)experts->ne[1]; + const int nt = (int)experts->ne[2]; + const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float)); + const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float)); + const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float)); + const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float)); + const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float)); + const cl_ulong nb_b1 = bias->nb[1]; + const cl_ulong nb_i1 = ids->nb[1]; + + const size_t w_bytes = ggml_nbytes(weights); + backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer, + off_w, 0, w_bytes, 0, NULL, NULL)); + cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer; + cl_ulong w_off = 0; + + cl_kernel kernel = backend_ctx->kernel_moe_combine_bias_f32; + int a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &eb->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_b)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ei->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_i)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_b1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb_i1)); + + size_t lws[2] = { 64, 1 }; + size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); +} + + +static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor * mul, const ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + const ggml_tensor * experts = mul->src[0]; + const ggml_tensor * weights = mul->src[1]; + + ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra; + ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra; + ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong off_e = ee->offset + experts->view_offs; + cl_ulong off_w = ew->offset + weights->view_offs; + cl_ulong off_d = ed->offset + dst->view_offs; + + const int n_embd4 = (int)(experts->ne[0] / 4); + const int k = (int)experts->ne[1]; + const int nt = (int)experts->ne[2]; + const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float)); + const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float)); + const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float)); + const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float)); + const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float)); + + // The router weights are tiny ([1,k,nt]) and may share a pool buffer with the output; + // copy them into a private scratch so the fused kernel never reads aliased memory. + const size_t w_bytes = ggml_nbytes(weights); + backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer, + off_w, 0, w_bytes, 0, NULL, NULL)); + cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer; + cl_ulong w_off = 0; + + cl_kernel kernel = backend_ctx->kernel_moe_combine_f32; + int a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1)); + + size_t lws[2] = { 64, 1 }; + size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); +} + +inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below (used by the GLU-subgraph fuse check) +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); +inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor); // defined below + +static bool ggml_opencl_can_fuse(const ggml_backend_opencl_context * backend_ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { + + // glu(mul_mat(Wg,x), mul_mat(Wu,x)) — the FFN gate/up GEMVs + GLU. This is a + // non-linear subgraph (up does NOT consume gate), so the contiguous + // ggml_can_fuse below rejects it; use ggml_can_fuse_subgraph with the glu as + // the sole output and validate the edges explicitly. q4_K decode only; + // byte-identical to the per-op path. + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_MUL_MAT && + ops.begin()[1] == GGML_OP_MUL_MAT && ops.begin()[2] == GGML_OP_GLU) { + const enum ggml_op glu_ops[] = { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }; + const int glu_out[] = { node_idx + 2 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, 3, glu_ops, glu_out, 1)) { + return false; + } + + const ggml_tensor *gate = cgraph->nodes[node_idx]; + const ggml_tensor *up = cgraph->nodes[node_idx+1]; + const ggml_tensor *glu = cgraph->nodes[node_idx+2]; + + // decode GEMV path only (single token); prefill GEMM is separate + if (gate->ne[1] != 1 || up->ne[1] != 1) { + return false; + } + // both projections must be q4_K weights, f32 activation/output + if (gate->src[0]->type != GGML_TYPE_Q4_K || up->src[0]->type != GGML_TYPE_Q4_K || + gate->src[1]->type != GGML_TYPE_F32 || up->src[1]->type != GGML_TYPE_F32 || + gate->type != GGML_TYPE_F32 || up->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { + return false; + } + // gate and up must share the same activation and have matching shape/stride + if (gate->src[1] != up->src[1] || + !ggml_are_same_shape(gate->src[0], up->src[0]) || + !ggml_are_same_stride(gate->src[0], up->src[0])) { + return false; + } + // GLU must read gate as src[0] and up as src[1], no swap (the fused + // epilogue applies the activation to gate, multiplies by up) + if (glu->src[0] != gate || glu->src[1] != up) { + return false; + } + if (ggml_get_op_params_i32(glu, 1) /* swapped */) { + return false; + } + // SWIGLU_OAI carries extra alpha/limit params -> not handled by the fused kernel + if (ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU_OAI) { + return false; + } + // the fused kernel reads the standard noshuffle image layout; the tiled + // layout packs weights differently -> defer those to the per-op path + if (use_q4k_tiled(backend_ctx, gate->src[0]) || use_q4k_tiled(backend_ctx, up->src[0])) { + return false; + } + // q4_K bin kernel requires 32b transposed layout, not compatible with the fused gemv + if (use_q4_k_bin_kernels(backend_ctx, gate->src[0]) || use_q4_k_bin_kernels(backend_ctx, up->src[0])) { + return false; + } + // that noshuffle layout is only produced at set_tensor time when + // use_adreno_kernels() accepts the weight (ne0 >= 512 && ne1 >= 512). + // Smaller weights stay in the plain q4_K layout, which this kernel would + // misread -> defer them to the per-op path. Real FFN gate/up weights are + // far above the threshold, so production dispatch is unchanged. + if (!use_adreno_kernels(backend_ctx, gate->src[0]) || + !use_adreno_kernels(backend_ctx, up->src[0])) { + return false; + } + return true; + } + + if (!ggml_can_fuse(cgraph, node_idx, ops)) { + return false; + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { + const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + + // rms_norm only supports f32 + if (mul->src[0]->type != GGML_TYPE_F32 || + mul->src[1]->type != GGML_TYPE_F32 || + mul->type != GGML_TYPE_F32) { + return false; + } + + // if rms_norm is the B operand, then we don't handle broadcast + if (rms_norm == mul->src[1] && + !ggml_are_same_shape(mul->src[0], rms_norm)) { + return false; + } + + // rms_norm assumes contiguous rows + if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) { + return false; + } + } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { + const ggml_tensor *norm = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + const ggml_tensor *add = cgraph->nodes[node_idx+2]; + const ggml_tensor *w = mul->src[0] == norm ? mul->src[1] : mul->src[0]; + const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + + // norm fusion only supports F32 + if (norm->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { + return false; + } + + if (norm->src[0]->ne[0] % 4 != 0) { + return false; + } + + if (!ggml_is_contiguous(norm->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { + return false; + } + } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { + // rms_norm(x) * w + b, fused (residual). Mirrors the RMS_NORM+MUL gate + // plus the residual-add operand's constraints. + const ggml_tensor *rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + const ggml_tensor *add = cgraph->nodes[node_idx+2]; + const ggml_tensor *w = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0]; + const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + + if (w->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 || + b->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32) { + return false; + } + if (rms_norm->src[0]->ne[0] % 4 != 0) { + return false; + } + // if rms_norm is the B operand of mul, broadcast is not handled + if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) { + return false; + } + // the residual must match the normed output shape (no add broadcast) + if (!ggml_are_same_shape(b, add)) { + return false; + } + // rms_norm assumes contiguous rows + if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1]) || + !ggml_is_contiguous_rows(b)) { + return false; + } + } else if (ops.size() == 3 && ops.begin()[0] == GGML_OP_GROUP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { + const ggml_tensor *gn = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx+1]; + const ggml_tensor *add = cgraph->nodes[node_idx+2]; + const ggml_tensor *w = mul->src[0] == gn ? mul->src[1] : mul->src[0]; + const ggml_tensor *b = add->src[0] == mul ? add->src[1] : add->src[0]; + + if (gn->src[0]->type != GGML_TYPE_F32 || w->type != GGML_TYPE_F32 || b->type != GGML_TYPE_F32) { + return false; + } + + if (!ggml_is_contiguous(gn->src[0]) || !ggml_is_contiguous(w) || !ggml_is_contiguous(b)) { + return false; + } + } + + return true; +} + +static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor); +static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); +static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor); + +static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor * gate_tensor, ggml_tensor * up_tensor, ggml_tensor * glu_tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_ASSERT(gate_tensor && up_tensor && glu_tensor); + + const ggml_tensor * Wg = gate_tensor->src[0]; + const ggml_tensor * Wu = up_tensor->src[0]; + const ggml_tensor * src1 = gate_tensor->src[1]; // == up_tensor->src[1] + const ggml_tensor * dst = glu_tensor; + + GGML_ASSERT(Wg && Wg->extra); + GGML_ASSERT(Wu && Wu->extra); + GGML_ASSERT(src1 && src1->extra); + GGML_ASSERT(dst && dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_K * extra_g = (ggml_tensor_extra_cl_q4_K *)Wg->extra; + ggml_tensor_extra_cl_q4_K * extra_u = (ggml_tensor_extra_cl_q4_K *)Wu->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int K = Wg->ne[0]; // ne00 + const int M = Wg->ne[1]; // ne01 (= ffn intermediate width) + const int N = 1; // decode GEMV + + const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0; + const int glu_op = (int)ggml_get_glu_op(dst); + + cl_context context = backend_ctx->context; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; + + // q images for the two weight matrices (standard noshuffle layout) + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * K / 2 / 4; + img_desc.buffer = extra_g->q; + cl_mem qg_img = nullptr, qu_img = nullptr; + CL_CHECK((qg_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_desc.buffer = extra_u->q; + CL_CHECK((qu_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // shared activation image (one column at decode) + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + cl_mem b_sub_buf = nullptr, b_img = nullptr; + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_glu; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qg_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_g->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_g->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra_g->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &qu_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra_u->d)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_u->dm)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extra_u->s)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &glu_op)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_uchar), &mask_hi2)); + + // K-split = nsg_y subgroups. HARD-CAP at 8 (512 work-items): the fused + // kernel's cross-subgroup reduce uses a float4 reduceLM (gate+up packed) = + // 2x the LDS of the base GEMV's float2 reduce, so 16 co-resident subgroups + // exceed the per-CU LDS budget on X2 and the WG barrier DEADLOCKS -> GPU TDR + // (reproduced on upstream gemma-4 E4B decode, K=2560 M=10240). This used to + // be masked: get_kernel_workgroup_size reported 896 for this kernel (so the + // cap loop fell to 8), but it now returns 1024 and the Adreno per-kernel WG + // query is unreliable (over-reports), so cap explicitly instead of trusting + // it. nsg_y < 16 also means the cross-subgroup accumulation grouping differs + // from the standalone wide (nsg=16) GEMV, so the output is coherent but NOT + // byte-identical to the per-op path. Keep the maxwg query as a further floor + // for any driver that reports < 512. + size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); + size_t nsg_y = 8; + while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } + size_t local_work_size[3] = { 64, nsg_y, 1 }; + size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 }; + + if (getenv("GGML_OPENCL_FUSE_DEBUG")) { + static int dbg = 0; + if (dbg < 3) { fprintf(stderr, "[FUSE_MM_GLU] fired #%d K=%d M=%d glu_op=%d nsg=%zu maxwg=%zu\n", ++dbg, K, M, glu_op, nsg_y, maxwg); fflush(stderr); } + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(qg_img)); + CL_CHECK(clReleaseMemObject(qu_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +#else + GGML_UNUSED(backend); + GGML_UNUSED(gate_tensor); + GGML_UNUSED(up_tensor); + GGML_UNUSED(glu_tensor); +#endif +} + + +static void ggml_opencl_op_rms_norm_mul_add_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { + GGML_ASSERT(rms_norm_tensor && mul_tensor && add_tensor); + + const ggml_tensor * src0 = rms_norm_tensor->src[0]; + const ggml_tensor * src1 = mul_tensor->src[0] == rms_norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; + const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; + const ggml_tensor * dst = add_tensor; + + GGML_ASSERT(src0 && src0->extra); + GGML_ASSERT(src1 && src1->extra); + GGML_ASSERT(src2 && src2->extra); + GGML_ASSERT(dst && dst->extra); + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + float eps; + memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); + + const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; + const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; + const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; + const cl_ulong nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + GGML_ASSERT(ne00 % 4 == 0); + + size_t sgs; + if (backend_ctx->gpu_family == ADRENO) sgs = 64; + else if (backend_ctx->gpu_family == INTEL) sgs = 32; + else GGML_ASSERT(false && "Unsupported GPU"); + + cl_kernel kernel = backend_ctx->kernel_rms_norm_mul_add; + + int nth = sgs; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (nth < ne00 && nth < max_workgroup_size) nth *= 2; + nth = MIN(nth, max_workgroup_size); + nth = MIN(nth, ne00); + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(float)*sgs, NULL)); + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + + // NOTE: this may oversynchronize by synchronizing with + // backends/devices which don't compute 'cgraph's + // dependencies. + sync_with_other_backends(backend); + + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + continue; + } + + if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; + } + + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + ggml_opencl_op_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + ggml_opencl_op_group_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } + // Fuse the MoE combine: router-weight mul + cross-expert add chain -> + // one weighted-sum-across-experts kernel. + // Fold the gpt-oss MoE bias epilogue: add_id(gate_bias) + add_id(up_bias) + + // glu(swiglu_oai) -> one kernel, leaving the two matmuls as their own dispatches. + // Both add_ids are in-place passes over a tensor the GLU reads again, so this + // drops two full read+write passes per layer. Opt out GGML_OPENCL_FUSE_MOE_BIAS_GLU=0. + if (backend_ctx->fuse_moe_bias_glu && !backend_ctx->disable_fusion && + ggml_opencl_can_fuse_moe_bias_glu(cgraph, i)) { + ggml_cl_moe_bias_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2], + cgraph->nodes[i+3], cgraph->nodes[i+4]); + i += 4; + continue; + } + + // Fold the MoE down-projection bias into the combine: add_id(down_bias) + the whole + // combine subgraph -> one kernel. Checked before the plain combine arm so the longer + // pattern wins. Opt out GGML_OPENCL_FUSE_MOE_BIAS_COMBINE=0. + if (backend_ctx->fuse_moe_bias_combine && backend_ctx->fuse_moe_combine && + !backend_ctx->disable_fusion) { + const ggml_tensor * bias_combine_out = nullptr; + if (ggml_opencl_can_fuse_moe_bias_combine(cgraph, i, &bias_combine_out)) { + ggml_cl_moe_bias_combine_fused(backend, node, cgraph->nodes[i+1], bias_combine_out); + i += 2 * (int)node->ne[1]; // ADD_ID + MUL + k VIEWs + (k-1) ADDs + continue; + } + } + + if (backend_ctx->fuse_moe_combine && !backend_ctx->disable_fusion) { + const ggml_tensor * combine_out = nullptr; + if (ggml_opencl_can_fuse_moe_combine(cgraph, i, &combine_out)) { + ggml_cl_moe_combine_fused(backend, node, combine_out); + i += 2 * (int)node->ne[1] - 1; // skip the k VIEWs + (k-1) ADDs + continue; + } + } + + // Fuse rms_norm + mul(weight) + add(residual). Checked before the + // rms_norm+mul fuse so the 3-op pattern wins over its 2-op prefix. + // Default on, opt-out GGML_OPENCL_FUSE_RMS_ADD=0. + if (!backend_ctx->disable_fusion && backend_ctx->fuse_rms_add && + ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + ggml_opencl_op_rms_norm_mul_add_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { + ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]); + i++; + continue; + } + // Fuse mul_mat(Wg,x) + mul_mat(Wu,x) + glu — fold the FFN's two decode + // GEMVs and the GLU into one dispatch. q4_K only (guarded below); the + // fused kernel uses the same accumulation/reduction order and the same + // scalar GLU formula -> coherent. Default on, opt-out GGML_OPENCL_FUSE_MM_GLU=0. +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // The fused executor (ggml_cl_mul_mat_q4_k_glu_fused) is image-path / + // Adreno-only (GGML_ABORT on the non-Adreno #else); gate the dispatch to + // match so the FFN GLU subgraph stays dormant on Intel/other drivers. + if (backend_ctx->fuse_mm_glu && !backend_ctx->disable_fusion && + ggml_opencl_can_fuse(backend_ctx, cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU })) { + ggml_cl_mul_mat_q4_k_glu_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]); + i += 2; + continue; + } +#endif + + bool ok = ggml_cl_compute_forward(backend, node); + if (!ok) { + GGML_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); + } + GGML_ASSERT(ok); + } + + return GGML_STATUS_SUCCESS; +} + +// The optimized gemm and gemv kernels are used for large matrices without batch. +// tensor is the quantized weights matrix. +inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + int64_t threshold_ne0 = 512; + int64_t threshold_ne1 = 512; + if (!backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) && + backend_ctx->adreno_cl_compiler_version.type != DX) { + threshold_ne0 = 128; + threshold_ne1 = 128; + } + bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && + tensor->ne[2] == 1 && tensor->ne[3] == 1; + + // the transposed layout needs K % 32 == 0 and M % 4 == 0 + if (tensor->ne[0] % 32 != 0 || tensor->ne[1] % 4 != 0) { + return false; + } + + // The noshuffle layout packs 2 rows per 32-bit texel and the GEMV reads it at an + // ne1/2 texel stride with an exact-cover dispatch, so it is only addressable when + // ne1 is a multiple of 64; an unaligned ne1 truncates the stride and the weight is + // read misaligned. That is a property of the layout, not of one quant -- q4_K, q5_K + // and q8_0 read the same packing as q6_K. The bound is 64, not 128: a q8_0 attention + // weight of ne1 = 2880 is a multiple of 64 but not 128 and is correct. + switch (tensor->type) { + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q8_0: + return threashold_ok && tensor->ne[1] % 64 == 0; + default: + break; + } + return threashold_ok; +} + +static bool adreno_e17_compiler_quirks(const ggml_backend_opencl_context *backend_ctx) { + if (!backend_ctx || backend_ctx->gpu_family != GPU_FAMILY::ADRENO || + backend_ctx->adreno_cl_compiler_version.type != ADRENO_CL_COMPILER_TYPE::E17) { + return false; + } + const char * env = getenv("GGML_OPENCL_ADRENO_E17_QUIRKS"); + return !(env && env[0] == '0'); +} + +static bool ggml_cl_q2_k_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_Q2_K_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_Q2_K && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + // The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*. + // Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights. + // So, exclude A6x and A7x from using Adreno MoE kernels for now. + // The quants that have a general mul_mat_id kernel fallback to the general version; the + // rest fallback to CPU. + if (backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN)) { + return false; + } + + if (adreno_e17_compiler_quirks(backend_ctx)) { + return false; + } + + int ne01 = tensor->ne[1]; + return (((strstr(tensor->name, "ffn") != NULL) && (strstr(tensor->name, "exps") != NULL)) || (strstr(tensor->name, "as") != NULL)) && (ne01 % 32 == 0); +} + +// Device default for the tiled-wide lm_head/embed GEMV layout: ON for X2E and A8X. +// +// These kernels were previously off everywhere on the grounds that they compute +// wrong values at multi-superblock K. They do not: that NMSE ~2 came from the +// backend having no get_tensor restore path for the tiled layout, so +// test-backend-ops (which builds its CPU reference by copying the weights back +// out of the backend) compared a correct GPU result against a reference +// dequantized from tiled bytes. With the restore path added, MUL_MAT passes with +// the tiled kernels on, unmodified, on both devices. +// +// Perf, Qwen3-4B-Q4_K_M (q6_K lm_head 151936x2560), tg128, matched pairs with +// alternating lead, tiled vs o4: +// +// A8X +11.9% 6/6 pairs positive, order bias -0.06% (16.93 vs 15.14 tok/s) +// X2E +6.9% 4/4 pairs positive, order bias -0.03% (35.24 vs 32.87 tok/s) +// +// Measure this one on a COLD device. These kernels are far more clock-sensitive +// than the o4 route they replace: on a heat-soaked A8X (CPU cap at 1.5-1.9 GHz) +// tiled pins at ~14.2 tok/s while o4 still makes ~14.9, which reads as a 4-5% +// LOSS and inverts the ranking. The same box, after a reboot and a gate that +// waits for policy6 to return to 4396800, reports the +11.9% above with no +// order bias. A7X regresses hard on this layout and stays off. +// GGML_OPENCL_{Q4K,Q6K}_GEMV_TILED forces either way (=0 off, any other value on). +inline bool tiled_gemv_default_on(const ggml_backend_opencl_context *backend_ctx) { + return backend_ctx && (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X); +} + +// Tiled-wide q6_K GEMV (default OFF; GGML_OPENCL_Q6K_GEMV_TILED forces either +// way: =0 off everywhere, any other value on everywhere). +// Both the convert (set_tensor) and the GEMV dispatch must agree on this so the +// buffer layout matches the kernel. +inline bool q6k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { + static const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_TILED"); + if (e && e[0] != '\0') { + return e[0] != '0'; + } + return tiled_gemv_default_on(backend_ctx); +} + +// Only the long-vocab lm_head/embed shapes use the tiled layout; ne01 % 64 == 0 +// is required by the 64-row tiling (no row padding in the buffers). +// use_adreno_kernels is required: only the Adreno GEMV path can read the tiled +// layout, so converting a weight it would decline (e.g. ne00 < 512) leaves the +// generic kernel reading tiled bytes as plain SOA. +inline bool use_q6k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + return q6k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q6_K && + tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && + use_adreno_kernels(backend_ctx, tensor); +} + +// q4_K analog of the tiled-wide lm_head/embed GEMV (default OFF; +// GGML_OPENCL_Q4K_GEMV_TILED forces either way: =0 off, else on). Same gate. +inline bool q4k_gemv_tiled_enabled(const ggml_backend_opencl_context *backend_ctx) { + static const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_TILED"); + if (e && e[0] != '\0') { + return e[0] != '0'; + } + return tiled_gemv_default_on(backend_ctx); +} +inline bool use_q4k_tiled(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + return q4k_gemv_tiled_enabled(backend_ctx) && tensor->type == GGML_TYPE_Q4_K && + tensor->ne[1] >= 32768 && tensor->ne[1] % 64 == 0 && + use_adreno_kernels(backend_ctx, tensor); +} + +inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + + bool adreno_kernel = use_adreno_kernels(backend_ctx, tensor); + + size_t elem_num = tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]; + + // The 2D weight transpose (transpose_2d_as_*) tiles rows by 4 over a 2D matrix, + // so it requires K(ne0)%32==0, M(ne1)%4==0 and ne2==ne3==1. + const bool shape_ok = (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 4 == 0) && + (tensor->ne[2] == 1) && (tensor->ne[3] == 1); + + return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27 +} + +inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + if (!use_adreno_kernels(backend_ctx, tensor)) { + return false; + } + + const size_t elem_num = ggml_nelements(tensor); + const size_t q_img_width = elem_num / 8; + const size_t qh_img_width = elem_num / 16; + const bool shape_ok = tensor->ne[0] % 32 == 0 && tensor->ne[1] % 4 == 0 && + tensor->ne[2] == 1 && tensor->ne[3] == 1; + + return shape_ok && q_img_width <= backend_ctx->image_max_buffer_size && + qh_img_width <= backend_ctx->image_max_buffer_size; +} + +inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) { + return false; + } + + // bin kernels require ne2 == 1 and ne3 == 1 for weights + return tensor->ne[2] == 1 && tensor->ne[3] == 1 && + (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool use_fa_bin_kernels_prefill(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * q, const ggml_tensor * k, const ggml_tensor * v) { + if (backend_ctx->fa.kernel_flash_attn_f32_f16_bin == nullptr) { + return false; + } + + const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; + const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + + const int n_q = q->ne[1]; + const int dk = q->ne[0]; + const int dv = v->ne[0]; + + constexpr bool prefill_only = true; + + return (backend_ctx->gpu_family == GPU_FAMILY::ADRENO && + (is_mixed || is_q8_0) && (dk == dv) + && (dk == 64 || dk == 128 || dk == 256 || dk == 512) + && (!prefill_only || n_q != 1)); +} +#endif + +// The flat-GEMV large-m escape is OPT-IN (GGML_OPENCL_FLAT_LARGE_M=1) because it +// is SLOWER than the route it replaces, not because it is unsafe. It was first +// parked on the theory that it out-of-bounds-writes at vocab-scale shapes; that +// was a misattribution (the test-backend-ops dst sentinel was tripped by the o4 +// GEMV's unguarded tail store, fixed separately - and at the shape it was blamed +// for, k=1536, this predicate returns false anyway, so the flat route never ran). +// +// The escape's original rationale, "gemv_noshuffle perf drops for large M", +// predates the o4 kernel, which now covers the same long-vocab shapes and beats +// this route on every device measured (Qwen3-4B-Q4_K_M, q6_K lm_head +// 151936x2560, tg128, matched pairs vs o4): A8X -10.3% (0/3 pairs), X2E -3.7% +// (0/3). Keep it reachable for shapes o4 declines, but do not default it on. +static inline bool flat_large_m_enabled() { + static const char * e = getenv("GGML_OPENCL_FLAT_LARGE_M"); + static const bool en = e != nullptr && atoi(e) != 0; + return en; +} + +// The noshuffle Q4_K weight image stores eight weights per uint texel. +static inline bool q4_K_weight_image_fits(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + const size_t texels = (size_t) ggml_nelements(tensor) / 8; + return texels != 0 && texels <= backend_ctx->image_max_buffer_size; +} + +static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + if (tensor->ne[1] % 4 != 0 && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + return true; + } + + if (tensor->ne[2] != 1 || tensor->ne[3] != 1 || use_q4k_tiled(backend_ctx, tensor)) { + return false; + } + + // The image limit is a correctness guard, independent of the large-M opt-in. + if (!q4_K_weight_image_fits(backend_ctx, tensor)) { + return true; + } + + // gemv_noshuffle variant perf drops for large M, use flat variant for large M. + // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. + // note that this forces large M weights to use LM GEMM. + return flat_large_m_enabled() && tensor->ne[1] >= 32768; +} + +static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { + // NOTE on ordering: the ne01 % 128 escape below is a CORRECTNESS guard, not a + // performance one, so it must be reachable regardless of flat_large_m_enabled(). + // The opt-in gate therefore sits after it, and after the tiled deferral. + // gemv_noshuffle variant perf drops for large M, use flat variant for large M. + // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. + // q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold. + // note that this forces large M weights to use LM GEMM. + // When this branch's tiled-canonical lm_head/embed layout is active, the weight is + // converted to the 64-row tiled layout, which the flat gemv would misread as + // garbage. use_q6k_tiled owns these large-M weights (it requires ne01 % 64 == 0, + // so it never claims an odd-vocab weight), so defer to it first. + if (use_q6k_tiled(backend_ctx, tensor)) { + return false; + } + // The noshuffle (transposed-weight) layout packs 2 rows per 32-bit texel and the + // gemv reads it with a ne01/2 texel stride and an exact-cover dispatch of + // ceil(ne01/2 / 64)*64 work-items with no store guard; the gemm uses 4-row tiles. + // It is therefore only correct for ne01 % 128 == 0: an odd ne01 (e.g. granitemoe + // lm_head [1536, 49155] -- odd vocab) truncates the texel stride, misaligning every + // odd column of the transposed layout (gross garbage) and dropping the last row; + // other non-multiples over-dispatch and write past the end of dst. Route such + // tensors to the flat GEMV + regular convert; the matching GEMM (ne1>1) falls back + // to CPU (see supports_op). All standard even-vocab/hidden dims are multiples of + // 128 and keep the noshuffle path. + if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + return true; + } + + if (!flat_large_m_enabled()) { + return false; + } + + // The gemv_noshuffle slowdown tracks TOTAL weight size, not ne0 alone; ne0 >= 2048 is a + // proxy for "large weight" that misses a narrow-hidden vocab-scale lm_head. + // Add a direct size escape so such weights also take the flat path, without changing + // which weights ne0 >= 2048 already routes there. + // The size escape is not taken on the A7X since its compiler miscompiles the flat K-quant GEMV + return tensor->ne[1] >= 32768 + && (tensor->ne[0] >= 2048 || (backend_ctx->adreno_gen != ADRENO_GPU_GEN::A7X && ggml_nbytes(tensor) >= (256ull << 20))) + && tensor->ne[2] == 1 && tensor->ne[3] == 1; +} + +inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin) { + return false; + } + // bin kernels require ne2 == 1 and ne3 == 1 for weights + return tensor->ne[2] == 1 && tensor->ne[3] == 1 && + (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && + !use_q6k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) { + return false; + } + // bin kernels require ne2 == 1 and ne3 == 1 for weights + return tensor->ne[2] == 1 && tensor->ne[3] == 1 && + (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) && + !use_q4k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +inline bool use_q5_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (!backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans || + !backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin) { + return false; + } + return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + +static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *)dev->context; + ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; + + // reject ops that match the opfilter regex + if (dev_ctx->opfilter && std::regex_match(std::string(ggml_op_desc(op)), *dev_ctx->opfilter)) { + return false; + } + + switch (op->op) { + case GGML_OP_NONE: + return true; + case GGML_OP_GET_ROWS: + switch (op->src[0]->type) { + case GGML_TYPE_I32: + return op->type == GGML_TYPE_I32; + case GGML_TYPE_F32: + case GGML_TYPE_F16: + return true; + case GGML_TYPE_Q4_0: +#ifdef GGML_OPENCL_SOA_Q + // We do not support flattened Q4_0 (and possibly other Q's) + return false; +#else // GGML_OPENCL_SOA_Q + return true; +#endif // GGML_OPENCL_SOA_Q + default: + return false; + } + case GGML_OP_SET_ROWS: + { + // TODO: add support + // ref: https://github.com/ggml-org/llama.cpp/pull/14274 +#pragma message("TODO: implement BF16, Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, IQ4_NL support (https://github.com/ggml-org/llama.cpp/pull/14661)") + if (op->src[0]->type != GGML_TYPE_F32) { + return false; + } + switch (op->type) { + case GGML_TYPE_F16: + case GGML_TYPE_F32: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_0: + return (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); + default: + return false; + } + } + case GGML_OP_CPY: + case GGML_OP_DUP: + case GGML_OP_CONT: + switch (op->src[0]->type) { + case GGML_TYPE_F32: + switch (op->type) { + case GGML_TYPE_I32: + case GGML_TYPE_F16: + case GGML_TYPE_F32: + return true; + default: + return false; + } + case GGML_TYPE_F16: + switch (op->type) { + case GGML_TYPE_F16: + case GGML_TYPE_F32: + return true; + default: + return false; + } + case GGML_TYPE_I32: + switch (op->type) { + case GGML_TYPE_F32: + case GGML_TYPE_I32: + return true; + default: + return false; + } + default: + return false; + } + case GGML_OP_SET: { + return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32) && + op->type == op->src[0]->type && + op->type == op->src[1]->type; + } + case GGML_OP_SCALE: + return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); + case GGML_OP_ADD: + if (op->type == GGML_TYPE_F16) { + const bool src0_ok = op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32; + const bool src1_ok = op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32; + if (src0_ok && src1_ok) { + return true; + } + } + case GGML_OP_MUL: + case GGML_OP_DIV: + case GGML_OP_SUB: + return (op->src[0]->type == op->src[1]->type) && + (op->src[0]->type == op->type) && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); + case GGML_OP_ADD_ID: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_SQR: + case GGML_OP_SQRT: + return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + ggml_is_contiguous(op->src[0]); + case GGML_OP_UNARY: + switch (ggml_get_unary_op(op)) { + case GGML_UNARY_OP_GELU: + case GGML_UNARY_OP_SILU: + case GGML_UNARY_OP_RELU: + case GGML_UNARY_OP_GELU_ERF: + case GGML_UNARY_OP_GELU_QUICK: + return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; + case GGML_UNARY_OP_SIGMOID: + return ggml_is_contiguous(op->src[0]); + case GGML_UNARY_OP_TANH: + case GGML_UNARY_OP_NEG: + case GGML_UNARY_OP_EXP: + // Adreno F16 exp/expm1 overflow even post-half->float convert. + return op->src[0]->type == GGML_TYPE_F32; + case GGML_UNARY_OP_EXPM1: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_UNARY_OP_ABS: + case GGML_UNARY_OP_SGN: + case GGML_UNARY_OP_STEP: + case GGML_UNARY_OP_ELU: + case GGML_UNARY_OP_HARDSWISH: + case GGML_UNARY_OP_HARDSIGMOID: + case GGML_UNARY_OP_FLOOR: + case GGML_UNARY_OP_CEIL: + case GGML_UNARY_OP_ROUND: + case GGML_UNARY_OP_TRUNC: + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; + case GGML_UNARY_OP_SOFTPLUS: + return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; + default: + return false; + } + case GGML_OP_GLU: + switch (ggml_get_glu_op(op)) { + case GGML_GLU_OP_GEGLU: + case GGML_GLU_OP_REGLU: + case GGML_GLU_OP_SWIGLU: + case GGML_GLU_OP_SWIGLU_OAI: + case GGML_GLU_OP_GEGLU_ERF: + case GGML_GLU_OP_GEGLU_QUICK: + case GGML_GLU_OP_SWIGLU_CLAMP: + return ggml_is_contiguous_1(op->src[0]) && (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); + default: + return false; + } + case GGML_OP_TRI: + return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); + case GGML_OP_FILL: + return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); + case GGML_OP_CLAMP: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_SOFT_MAX: + case GGML_OP_NORM: + return true; + case GGML_OP_RMS_NORM: + return op->ne[0] % 4 == 0 && ggml_is_contiguous_rows(op->src[0]); + case GGML_OP_L2_NORM: + return ggml_is_contiguous_rows(op->src[0]); + case GGML_OP_REPEAT: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; // Assuming F32 for now, can be expanded + case GGML_OP_PAD: + // TODO: add circular padding support for opencl, see https://github.com/ggml-org/llama.cpp/pull/16985 + if (ggml_get_op_params_i32(op, 8) != 0) { + return false; + } + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_UPSCALE: { + ggml_scale_mode mode = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & 0xFF); + const bool antialias = (ggml_scale_mode)(ggml_get_op_params_i32(op, 0) & GGML_SCALE_FLAG_ANTIALIAS); + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + (mode == GGML_SCALE_MODE_NEAREST || mode == GGML_SCALE_MODE_BILINEAR) && !antialias; } case GGML_OP_CONV_2D: return (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16) || (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) || (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); + case GGML_OP_CONV_2D_DW: + return op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && + op->src[1]->nb[0] == sizeof(float) && op->nb[0] == sizeof(float) && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16); case GGML_OP_SSM_CONV: return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32); + case GGML_OP_SSM_SCAN: { + if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || + op->src[1]->type != GGML_TYPE_F32 || op->src[2]->type != GGML_TYPE_F32 || + op->src[3]->type != GGML_TYPE_F32 || op->src[4]->type != GGML_TYPE_F32 || + op->src[5]->type != GGML_TYPE_F32 || op->src[6]->type != GGML_TYPE_I32) { + return false; + } + + const int64_t d_state = op->src[0]->ne[0]; + return d_state >= 1 && d_state <= 256 && (d_state & (d_state - 1)) == 0; + } + case GGML_OP_GATED_DELTA_NET: + { + // Match the Vulkan backend: only F32 -> F32, S_v in {16, 32, 64, 128}. + if (op->src[0]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) { + return false; + } + const int64_t S_v = op->src[2]->ne[0]; + return S_v == 16 || S_v == 32 || S_v == 64 || S_v == 128; + } case GGML_OP_CONCAT: - return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + { + const ggml_type t = op->src[0]->type; + return op->src[1]->type == t && op->type == t && + !ggml_is_quantized(t) && ggml_blck_size(t) == 1 && + (ggml_type_size(t) == 1 || ggml_type_size(t) == 2 || + ggml_type_size(t) == 4 || ggml_type_size(t) == 8); + } case GGML_OP_TIMESTEP_EMBEDDING: return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; case GGML_OP_GROUP_NORM: @@ -4093,2471 +9859,7522 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_MUL_MAT: if (op->src[0]->type == GGML_TYPE_F16) { return true; - } else if (op->src[0]->type == GGML_TYPE_F32) { - return op->src[1]->type == GGML_TYPE_F32; - } else if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_1 || - op->src[0]->type == GGML_TYPE_MXFP4 || - op->src[0]->type == GGML_TYPE_Q4_K || - op->src[0]->type == GGML_TYPE_Q5_K || - op->src[0]->type == GGML_TYPE_Q6_K) { - return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); - } else if (op->src[0]->type == GGML_TYPE_Q8_0) { - return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_BF16) { + return true; + } else if (op->src[0]->type == GGML_TYPE_F32) { + return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_Q1_0) { + return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_Q4_0) { + // Non-contig src0 routes through on-device dequant-to-f16. + return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_Q2_K) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_q2_k_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && + ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_Q4_1 || + op->src[0]->type == GGML_TYPE_Q5_0 || op->src[0]->type == GGML_TYPE_Q5_1 || + op->src[0]->type == GGML_TYPE_MXFP4 || + op->src[0]->type == GGML_TYPE_IQ4_NL || + op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q6_K) { + // The E031.41 compiler (usually with A7x) miscompiles the flat K-quant + // GEMV kernels (kernel_mul_mv_q*_K_f32_flat) and makes lm_head run much + // slower than it should. So, make it fallback to CPU to preserve performance + // for this compiler series. + static const char * a7x_lmhead_env = getenv("GGML_OPENCL_A7X_LMHEAD_CPU"); + static const bool a7x_lmhead_cpu = (a7x_lmhead_env == nullptr || a7x_lmhead_env[0] != '0'); + if (a7x_lmhead_cpu && + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q6_K) && + op->src[0]->ne[1] >= 32768) { // vocab-scale weight; no FFN/attn weight is this tall + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_Q8_0) { + // ggml_cl_mul_mat_q8_0_f32_adreno now honors src1/dst view_offs (the + // activation sub-buffer starts at offset1 and the kernels take offsetd), + // so a broadcast q8_0 matmul (src1 batch > src0 batch, e.g. Qwen3.5-9B-UD + // / Qwen3.6-35B q8_0 GDN ssm_out) runs on GPU via the per-slice broadcast + // iteration in ggml_cl_mul_mat. No special-casing needed. + return op->src[1]->type == GGML_TYPE_F32; + } + return false; + case GGML_OP_MUL_MAT_ID: +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32 && + use_adreno_moe_kernels(backend_ctx, op->src[0])) { + return false; + } +#endif + if (op->src[0]->type == GGML_TYPE_Q4_0 || + op->src[0]->type == GGML_TYPE_Q8_0 || + op->src[0]->type == GGML_TYPE_MXFP4) { + if (op->src[1]->type == GGML_TYPE_F32) { + return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); + } + } + // q4_0, q8_0 and mxfp4 have general MUL_MAT_ID support, + // the quantizations here currently do not - they are only supported by Adreno with certain shapes + if (op->src[0]->type == GGML_TYPE_Q4_1 || + op->src[0]->type == GGML_TYPE_Q5_0 || + op->src[0]->type == GGML_TYPE_Q5_1 || + op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q6_K) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (op->src[1]->type == GGML_TYPE_F32) { + return use_adreno_moe_kernels(backend_ctx, op->src[0]) + && ggml_is_contiguous(op->src[0]) + && ggml_is_contiguous(op->src[1]); + } +#endif + return false; + } + return false; + case GGML_OP_RESHAPE: + case GGML_OP_VIEW: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + return true; + case GGML_OP_DIAG: + return true; + case GGML_OP_DIAG_MASK_INF: + return op->ne[3] == 1; + case GGML_OP_ROPE: { + const int mode = ((const int32_t *) op->op_params)[2]; + const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; + const bool is_vision = mode == GGML_ROPE_TYPE_VISION; + if (is_mrope && !is_vision) { + if (op->src[0]->type == GGML_TYPE_F32 || + op->src[0]->type == GGML_TYPE_F16) { + return true; + } + return false; + } + if (is_vision) { + if (op->src[0]->type == GGML_TYPE_F32 || + op->src[0]->type == GGML_TYPE_F16) { + return true; + } + return false; + } + return true; + } + case GGML_OP_SOLVE_TRI: + return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); + case GGML_OP_IM2COL: + return true; + case GGML_OP_IM2COL_3D: + return true; + case GGML_OP_ARGMAX: + load_cl_kernels_argmax(backend_ctx); + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_I32 && + ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op) && + backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_argmax_f32_i32) >= 64; + case GGML_OP_LIGHTNING_INDEXER: + return op->src[0]->type == GGML_TYPE_F32 && + (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_BF16) && + op->src[2]->type == GGML_TYPE_F32 && op->src[3]->type == GGML_TYPE_F16 && + op->src[0]->nb[0] == sizeof(float) && op->src[1]->nb[0] == ggml_type_size(op->src[1]->type) && + op->src[2]->nb[0] == sizeof(float) && op->src[3]->nb[0] == sizeof(ggml_fp16_t); + case GGML_OP_TOP_K: + return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_I32; + case GGML_OP_ARGSORT: { + load_cl_kernels_argsort(backend_ctx); + + cl_kernel kernel = backend_ctx->kernel_argsort_f32_i32; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + + int cols = 1; + while (cols < op->ne[0]) { + cols *= 2; + } + + return cols <= max_workgroup_size && op->src[0]->type == GGML_TYPE_F32; + } + case GGML_OP_SUM_ROWS: + case GGML_OP_CUMSUM: + return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); + case GGML_OP_MEAN: + return op->src[0]->type == GGML_TYPE_F32; + case GGML_OP_FLASH_ATTN_EXT: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_fa_bin_kernels_prefill(backend_ctx, op->src[0], op->src[1], op->src[2])) { + return true; + } +#endif + // The E17 compilers segfault while building FA kernels, skip E17 for now + if (adreno_e17_compiler_quirks(backend_ctx)) { + return false; + } + const ggml_tensor * q = op->src[0]; + const ggml_tensor * k = op->src[1]; + const ggml_tensor * v = op->src[2]; + + const int dk = q->ne[0]; + const int dv = v->ne[0]; + + const struct { int dk; int dv; } supported_dims[] = { + { 40, 40}, { 64, 64}, { 80, 80}, { 96, 96}, + {112, 112}, {128, 128}, {192, 128}, + {192, 192}, {256, 256}, + {512, 512}, + }; + + bool dims_supported = false; + for (size_t i = 0; i < sizeof(supported_dims)/sizeof(supported_dims[0]); ++i) { + if (supported_dims[i].dk == dk && supported_dims[i].dv == dv) { + dims_supported = true; + break; + } + } + if (!dims_supported) { + return false; + } + + const bool is_f32_f32 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 && + v->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + const bool is_f16_f16 = q->type == GGML_TYPE_F16 && k->type == GGML_TYPE_F16 && + v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16; + const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && + v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32; + + const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && + v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 && + dk % 32 == 0 && dv % 32 == 0; + const bool is_f32_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && + v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 && + dk % 32 == 0 && dv % 32 == 0; + + // A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram + // building the flash_attn programs whose KV path is mixed-type or + // dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it + // is DK-independent). It is a driver crash, not codegen-wrong-output, so + // it cannot be caught in-process (fatal=false only handles clean compile + // errors). The uniform f16_f16 / f32_f32 programs compile fine on this + // compiler, so decline only the KV-convert variants; ggml then runs + // those (f16-KV / quant-KV) attention layers on the CPU backend. + // Negative compiler carve-out, same idiom as the Intel DK=512 decline + // below and the X1E driver-quirk guards. + if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { + return false; + } + + // Asymmetric KV: host-dequants both sides to F32, uses f32 kernel. + auto is_kv_type_ok = [](ggml_type t) { + return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 || + t == GGML_TYPE_Q4_0 || t == GGML_TYPE_Q8_0; + }; + const bool is_f32_asym = q->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + k->type != v->type && + is_kv_type_ok(k->type) && is_kv_type_ok(v->type); + + const bool kv_combo_ok = is_f32_f32 || is_f16_f16 || is_f32_f16 || + is_f32_q8_0 || is_f32_q4_0 || is_f32_asym; + if (!kv_combo_ok) { + return false; + } + + // Some compilers for A7x (Adreno 740, compiler E031.41) crashes when + // building FA kernels with mixed or quant types (f32_f16, f32_q8_0, f32_q4_0) + // Here we skip all A7x for these kernels to avoid crash + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X && + (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) { + return false; + } + + if (dk == 512) { + if (backend_ctx->gpu_family == INTEL) { + return false; + } + if (!is_f32_f16) { + return false; + } + if (q->ne[1] == 1) { + const char * decode_env = getenv("GGML_OPENCL_FA_DK512_DECODE"); + if ((decode_env && decode_env[0] == '0') || + backend_ctx->gpu_family != ADRENO || k->ne[2] <= 0 || + (q->ne[2] / k->ne[2] != 4 && q->ne[2] / k->ne[2] != 8) || q->ne[2] % k->ne[2] != 0 || + !ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)) { + return false; + } + } else { + // prefill, BM-tile in its own FA_PREFILL_ONLY program + if (!ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false)) { + return false; + } + } + } + return true; + } + default: + return false; + } +} + +// Forward declaration - implementation appears later in the file. +static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type); + +static ggml_guid_t ggml_backend_opencl_guid() { + static ggml_guid guid = { 0xde, 0xe0, 0x70, 0xa2, 0x73, 0x4e, 0x4d, 0xbc, 0xb0, 0xc7, 0x4f, 0xd4, 0x6d, 0x4e, 0x90, 0xfe }; + return &guid; +} + +static ggml_backend_i ggml_backend_opencl_i = { + /* .get_name = */ ggml_backend_opencl_name, + /* .free = */ ggml_backend_opencl_free, + /* .set_tensor_async = */ NULL, /* ggml_backend_opencl_set_tensor_async */ + /* .get_tensor_async = */ NULL, /* ggml_backend_opencl_get_tensor_async */ + /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, + /* .cpy_tensor_async = */ NULL, /* ggml_backend_opencl_cpy_tensor_async */ + /* .synchronize = */ ggml_backend_opencl_synchronize, + /* .graph_plan_create = */ NULL, + /* .graph_plan_free = */ NULL, + /* .graph_plan_update = */ NULL, + /* .graph_plan_compute = */ NULL, + /* .graph_compute = */ ggml_backend_opencl_graph_compute, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .graph_optimize = */ NULL, +}; + +ggml_backend_t ggml_backend_opencl_init(void) { + ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_opencl_reg(), 0); + ggml_backend_opencl_context *backend_ctx = ggml_cl_init(dev); + backend_ctx->ref_count++; + + ggml_backend_t backend = new ggml_backend { + /* .guid = */ ggml_backend_opencl_guid(), + /* .iface = */ ggml_backend_opencl_i, + /* .device = */ dev, + /* .context = */ backend_ctx + }; + + return backend; +} + +bool ggml_backend_is_opencl(ggml_backend_t backend) { + return backend && backend->iface.get_name == ggml_backend_opencl_name; +} + +// +// buffer +// +struct ggml_backend_opencl_buffer_context { + // A buffer context can hold multiple cl_mem objects. This is for flattening + // quantized weights and should be used with GGML_OPENCL_SMALL_ALLOC where + // each tensor is allocated a separate buffer. When flattening is enabled + // with small allocation, each tensor is backed by two cl_mem objects (for + // quants and scales) packed into a backend_opencl_buffer. + ggml_backend_opencl_buffer_context(cl_mem buf) + : name("OpenCL") { + buffer.push_back(buf); + } + + ~ggml_backend_opencl_buffer_context() { + for (cl_mem buf : buffer) { + CL_CHECK(clReleaseMemObject(buf)); + } + for (cl_mem im : img) { + CL_CHECK(clReleaseMemObject(im)); + } + + // Delete all extras to trigger their destructors + for (ggml_tensor_extra_cl * e : temp_tensor_extras) { + delete e; + } + for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0) { + delete e; + } + for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1) { + delete e; + } + for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0) { + delete e; + } + for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1) { + delete e; + } + for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4) { + delete e; + } + for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0) { + delete e; + } + for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0) { + delete e; + } + for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl) { + delete e; + } + for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K) { + delete e; + } + for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K) { + delete e; + } + for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K) { + delete e; + } + for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) { + delete e; + } + for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { + delete e; + } + for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { + delete e; + } + } + + ggml_tensor_extra_cl * ggml_opencl_alloc_temp_tensor_extra() { + ggml_tensor_extra_cl * extra; + if (temp_tensor_extras.empty()) { + extra = new ggml_tensor_extra_cl(); + } else { + extra = temp_tensor_extras.back(); + temp_tensor_extras.pop_back(); + } + + temp_tensor_extras_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q1_0 * ggml_opencl_alloc_temp_tensor_extra_q1_0() { + ggml_tensor_extra_cl_q1_0 * extra; + if (temp_tensor_extras_q1_0.empty()) { + extra = new ggml_tensor_extra_cl_q1_0(); + } else { + extra = temp_tensor_extras_q1_0.back(); + temp_tensor_extras_q1_0.pop_back(); + } + + temp_tensor_extras_q1_0_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q4_0 * ggml_opencl_alloc_temp_tensor_extra_q4_0() { + ggml_tensor_extra_cl_q4_0 * extra; + if (temp_tensor_extras_q4_0.empty()) { + extra = new ggml_tensor_extra_cl_q4_0(); + } else { + extra = temp_tensor_extras_q4_0.back(); + temp_tensor_extras_q4_0.pop_back(); + } + + temp_tensor_extras_q4_0_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q4_1 * ggml_opencl_alloc_temp_tensor_extra_q4_1() { + ggml_tensor_extra_cl_q4_1 * extra; + if (temp_tensor_extras_q4_1.empty()) { + extra = new ggml_tensor_extra_cl_q4_1(); + } else { + extra = temp_tensor_extras_q4_1.back(); + temp_tensor_extras_q4_1.pop_back(); + } + + temp_tensor_extras_q4_1_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q5_0 * ggml_opencl_alloc_temp_tensor_extra_q5_0() { + ggml_tensor_extra_cl_q5_0 * extra; + if (temp_tensor_extras_q5_0.empty()) { + extra = new ggml_tensor_extra_cl_q5_0(); + } else { + extra = temp_tensor_extras_q5_0.back(); + temp_tensor_extras_q5_0.pop_back(); + } + + temp_tensor_extras_q5_0_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q5_1 * ggml_opencl_alloc_temp_tensor_extra_q5_1() { + ggml_tensor_extra_cl_q5_1 * extra; + if (temp_tensor_extras_q5_1.empty()) { + extra = new ggml_tensor_extra_cl_q5_1(); + } else { + extra = temp_tensor_extras_q5_1.back(); + temp_tensor_extras_q5_1.pop_back(); + } + + temp_tensor_extras_q5_1_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_mxfp4 * ggml_opencl_alloc_temp_tensor_extra_mxfp4() { + ggml_tensor_extra_cl_mxfp4 * extra; + if (temp_tensor_extras_mxfp4.empty()) { + extra = new ggml_tensor_extra_cl_mxfp4(); + } else { + extra = temp_tensor_extras_mxfp4.back(); + temp_tensor_extras_mxfp4.pop_back(); + } + + temp_tensor_extras_mxfp4_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q8_0 * ggml_opencl_alloc_temp_tensor_extra_q8_0() { + ggml_tensor_extra_cl_q8_0 * extra; + if (temp_tensor_extras_q8_0.empty()) { + extra = new ggml_tensor_extra_cl_q8_0(); + } else { + extra = temp_tensor_extras_q8_0.back(); + temp_tensor_extras_q8_0.pop_back(); + } + + temp_tensor_extras_q8_0_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_iq4_nl * ggml_opencl_alloc_temp_tensor_extra_iq4_nl() { + ggml_tensor_extra_cl_iq4_nl * extra; + if (temp_tensor_extras_iq4_nl.empty()) { + extra = new ggml_tensor_extra_cl_iq4_nl(); + } else { + extra = temp_tensor_extras_iq4_nl.back(); + temp_tensor_extras_iq4_nl.pop_back(); + } + + temp_tensor_extras_iq4_nl_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q4_K * ggml_opencl_alloc_temp_tensor_extra_q4_K() { + ggml_tensor_extra_cl_q4_K * extra; + if (temp_tensor_extras_q4_K.empty()) { + extra = new ggml_tensor_extra_cl_q4_K(); + } else { + extra = temp_tensor_extras_q4_K.back(); + temp_tensor_extras_q4_K.pop_back(); + } + + temp_tensor_extras_q4_K_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q5_K * ggml_opencl_alloc_temp_tensor_extra_q5_K() { + ggml_tensor_extra_cl_q5_K * extra; + if (temp_tensor_extras_q5_K.empty()) { + extra = new ggml_tensor_extra_cl_q5_K(); + } else { + extra = temp_tensor_extras_q5_K.back(); + temp_tensor_extras_q5_K.pop_back(); + } + + temp_tensor_extras_q5_K_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q2_K * ggml_opencl_alloc_temp_tensor_extra_q2_K() { + ggml_tensor_extra_cl_q2_K * extra; + if (temp_tensor_extras_q2_K.empty()) { + extra = new ggml_tensor_extra_cl_q2_K(); + } else { + extra = temp_tensor_extras_q2_K.back(); + temp_tensor_extras_q2_K.pop_back(); + } + + temp_tensor_extras_q2_K_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() { + ggml_tensor_extra_cl_q6_K * extra; + if (temp_tensor_extras_q6_K.empty()) { + extra = new ggml_tensor_extra_cl_q6_K(); + } else { + extra = temp_tensor_extras_q6_K.back(); + temp_tensor_extras_q6_K.pop_back(); + } + + temp_tensor_extras_q6_K_in_use.push_back(extra); + + extra->reset(); + return extra; + } + + void reset() { + for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { + temp_tensor_extras.push_back(e); + } + temp_tensor_extras_in_use.clear(); + + for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) { + temp_tensor_extras_q1_0.push_back(e); + } + temp_tensor_extras_q1_0_in_use.clear(); + + for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { + temp_tensor_extras_q4_0.push_back(e); + } + temp_tensor_extras_q4_0_in_use.clear(); + + for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) { + temp_tensor_extras_q4_1.push_back(e); + } + temp_tensor_extras_q4_1_in_use.clear(); + + for (ggml_tensor_extra_cl_q5_0 * e : temp_tensor_extras_q5_0_in_use) { + temp_tensor_extras_q5_0.push_back(e); + } + temp_tensor_extras_q5_0_in_use.clear(); + + for (ggml_tensor_extra_cl_q5_1 * e : temp_tensor_extras_q5_1_in_use) { + temp_tensor_extras_q5_1.push_back(e); + } + temp_tensor_extras_q5_1_in_use.clear(); + + for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { + temp_tensor_extras_mxfp4.push_back(e); + } + temp_tensor_extras_mxfp4_in_use.clear(); + + for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { + temp_tensor_extras_q8_0.push_back(e); + } + temp_tensor_extras_q8_0_in_use.clear(); + + for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl_in_use) { + temp_tensor_extras_iq4_nl.push_back(e); + } + temp_tensor_extras_iq4_nl_in_use.clear(); + + for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { + temp_tensor_extras_q4_K.push_back(e); + } + temp_tensor_extras_q4_K_in_use.clear(); + + for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { + temp_tensor_extras_q5_K.push_back(e); + } + temp_tensor_extras_q5_K_in_use.clear(); + + for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { + temp_tensor_extras_q6_K.push_back(e); + } + temp_tensor_extras_q6_K_in_use.clear(); + + for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) { + temp_tensor_extras_q2_K.push_back(e); + } + temp_tensor_extras_q2_K_in_use.clear(); + + q8_0_soa_tensors.clear(); + q4_0_soa_tensors.clear(); + } + + // Pools for extras. Available extras are in `temp_tensor_extras`. Extras + // being used are in `temp_tensor_extras_in_use`. At the first run, new + // extras get created and put in `in_use`. When the buffer is reset via + // the `reset` callback, all extras in `in_use` get moved to available extras + // for reuse. + std::vector temp_tensor_extras; + std::vector temp_tensor_extras_in_use; + std::vector temp_tensor_extras_q1_0; + std::vector temp_tensor_extras_q1_0_in_use; + std::vector temp_tensor_extras_q4_0; + std::vector temp_tensor_extras_q4_0_in_use; + std::vector temp_tensor_extras_q4_1; + std::vector temp_tensor_extras_q4_1_in_use; + std::vector temp_tensor_extras_q5_0; + std::vector temp_tensor_extras_q5_0_in_use; + std::vector temp_tensor_extras_q5_1; + std::vector temp_tensor_extras_q5_1_in_use; + std::vector temp_tensor_extras_mxfp4; + std::vector temp_tensor_extras_mxfp4_in_use; + std::vector temp_tensor_extras_q8_0; + std::vector temp_tensor_extras_q8_0_in_use; + std::vector temp_tensor_extras_iq4_nl; + std::vector temp_tensor_extras_iq4_nl_in_use; + std::vector temp_tensor_extras_q4_K; + std::vector temp_tensor_extras_q4_K_in_use; + std::vector temp_tensor_extras_q5_K; + std::vector temp_tensor_extras_q5_K_in_use; + std::vector temp_tensor_extras_q2_K; + std::vector temp_tensor_extras_q2_K_in_use; + std::vector temp_tensor_extras_q6_K; + std::vector temp_tensor_extras_q6_K_in_use; + + // q8_0 tensors with AoS->SoA layout conversion installed by set_tensor. + // Two types of tensors get SOA'ed - normal weights and MoE weights. + // In Q8_0's case, we only have normal weights. If we ever have Q8_0 as MoE + // weights, they need to be added to this set in `set_tensors`. + std::unordered_set q8_0_soa_tensors; + + // Same for q4_0. KV-cache q4_0 tensors are allocated but never pass + // through set_tensor, so they stay AoS and aren't in this set. + // In Q4_0's case, in addition to normal weights, we have MoE weights. + std::unordered_set q4_0_soa_tensors; + + // The buffer_context is initially created by ggml_backend_buft_alloc_buffer + // before any tensor is initialized (at the beginning of alloc_tensor_range). + // Hence, there is always a buffer object in this vector. When each tensor is + // being initialized, this original buffer object will be released if both + // flattening and small allocation are enabled, and additional buffer + // objects will be created in init_tensor to represent flattened quantized + // weights. + std::vector buffer; + // These are image1d_buffer_t objects that wrap around the quants and scales. + // For Q4_0 quantization, there should be two of them - one for quants and + // one for scales. They should be populated only when flattening and small + // allocation are enabled. + std::vector img; + std::string name; +}; + +static void ggml_backend_opencl_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + delete ctx; +} + +static void * ggml_backend_opencl_buffer_get_base(ggml_backend_buffer_t buffer) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; + return (void *) (uintptr_t) dev_ctx->backend_ctx->alignment; +} + +static enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + + if (tensor->view_src != nullptr) { + GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); + + ggml_tensor_extra_cl * view_extra = (ggml_tensor_extra_cl *) tensor->view_src->extra; + GGML_ASSERT(view_extra && "view_extra is nullptr?"); + + // Reuse extra of the parent tensor. The offset of this view tensor + // becomes `extra->offset + view_offs` and needs to be calculated when + // it is used. This changes is needed because of the change to + // ggml_alloc.c in https://github.com/ggml-org/llama.cpp/pull/7640. + // `buffer` passed in here will always be `tensor->buffer`. It is OK + // to allocate extras from the same buffer context for ordinary + // intermediate tensors. But for views into kv cache tensors, doing so + // would mess up the extras used by kv cache. + // Before #7640, `buffer` is for intermediate tensors, which is always + // different from that of kv cache tensors. + // + // NB: now extra->offset no longer accounts for view_offs. + // NB: this should not apply to weight tensors (for end-to-end runs, but + // may apply for test-backend-ops). + // FIXME: if any unexpected results are seen, double check the offset - + // there could be other places that need fix. + tensor->extra = view_extra; + } else { + { + size_t offset = (char *) tensor->data - (char *) ggml_backend_opencl_buffer_get_base(buffer); + + ggml_tensor_extra_cl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra(); + extra->offset = offset; + extra->data_device = ctx->buffer[0]; + extra->actual_size = ggml_nbytes(tensor); + + tensor->extra = extra; + } + } + return GGML_STATUS_SUCCESS; +} + +// Allocate a temporary upload buffer of `nbytes` and populate it with `data` +// from host. On Adreno X1-85 the device-only pool intermittently fails to +// allocate at hundreds of MB once model weights fragment the heap (observed +// on Qwen3.5-9B output.weight Q6_K at 834 MB). Three-step retry: +// 1. CL_MEM_READ_WRITE alloc + clEnqueueWriteBuffer (normal fast path). +// 2. clFinish + retry (drains in-flight allocs that may be holding heap; +// mirrors the proven pattern at the FD-split partial buffer alloc). +// 3. CL_MEM_ALLOC_HOST_PTR + map(WRITE_INVALIDATE) + memcpy + unmap — +// different memory pool (host-pinned); true zero-copy on Adreno per +// QCOM guidance. (CL_MEM_USE_HOST_PTR is NOT zero-copy on Adreno: the +// driver triggers an internal copy because arbitrary host pages aren't +// guaranteed mappable/coherent, AND it draws from the same exhausted +// device pool — so it doesn't solve the problem.) +// Returns the ready-to-read buffer (caller must clReleaseMemObject) or NULL +// if all three strategies fail. The buffer is opaque to the caller — it can +// be passed as a kernel argument like any normal cl_mem. +static cl_mem ggml_cl_create_temp_upload_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const void * data, + const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + const cl_int werr = clEnqueueWriteBuffer(queue, buf, CL_TRUE, 0, nbytes, data, 0, NULL, NULL); + if (werr == CL_SUCCESS) { + return buf; + } + clReleaseMemObject(buf); + } + buf = clCreateBuffer(context, + CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_WRITE_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; + } + void * mapped = clEnqueueMapBuffer(queue, buf, CL_TRUE, + CL_MAP_WRITE_INVALIDATE_REGION, 0, nbytes, 0, NULL, NULL, &err); + if (err != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + memcpy(mapped, data, nbytes); + const cl_int uerr = clEnqueueUnmapMemObject(queue, buf, mapped, 0, NULL, NULL); + if (uerr != CL_SUCCESS) { + clReleaseMemObject(buf); + return NULL; + } + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); + } + return buf; +} + +// Allocate a temporary download buffer of `nbytes`. The caller runs a kernel +// that writes into it, then reads it back to host via clEnqueueReadBuffer (or +// equivalent). Mirrors ggml_cl_create_temp_upload_buffer; the host-pinned +// fallback flags are flipped (CL_MEM_WRITE_ONLY | HOST_READ_ONLY) and the +// helper doesn't populate the buffer. +static cl_mem ggml_cl_create_temp_download_buffer( + cl_context context, cl_command_queue queue, + size_t nbytes, const char * tensor_name_for_log) +{ + cl_int err; + cl_mem buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + if (err != CL_SUCCESS) { + clFinish(queue); + buf = clCreateBuffer(context, CL_MEM_READ_WRITE, nbytes, NULL, &err); + } + if (err == CL_SUCCESS) { + return buf; + } + buf = clCreateBuffer(context, + CL_MEM_WRITE_ONLY | CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_READ_ONLY, + nbytes, NULL, &err); + if (err != CL_SUCCESS) { + return NULL; + } + if (tensor_name_for_log) { + GGML_LOG_INFO("ggml_opencl: %s download (%.1f MiB) — device alloc failed, using CL_MEM_ALLOC_HOST_PTR fallback\n", + tensor_name_for_log, nbytes / 1024.0 / 1024.0); + } + return buf; +} + +static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; + ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; + + cl_context context = backend_ctx->context; + cl_command_queue queue = backend_ctx->queue; + +#ifdef GGML_OPENCL_SOA_Q + if (tensor->type == GGML_TYPE_Q2_K) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q2_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q2_K(); + extra->noshuffle = ggml_cl_q2_k_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * (blck/4); + const size_t size_sc = n_blk * (blck/16); + const size_t size_dm = n_blk * 2 * sizeof(ggml_fp16_t); + GGML_ASSERT(size_qs + size_sc + size_dm == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + auto previous_origin = region.origin; + + region.origin = align_to(previous_origin + size_qs, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + region.origin = align_to(previous_origin + size_sc, backend_ctx->alignment); + region.size = size_dm; + CL_CHECK((extra->dm = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_q2_k_noshuffle + : backend_ctx->kernel_convert_block_q2_K; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/16, m, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->dm, extra->dm, size_dm, k/(int)blck, m, true, true); + } +#endif + + extra->size_qs = size_qs; + extra->size_sc = size_sc; + extra->size_dm = size_dm; + + tensor->extra = extra; + return; + } + + if (tensor->type == GGML_TYPE_Q1_0) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q1_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q1_0(); + + // q1_0 block = ggml_half d + (QK1_0/8) quant bytes = 2 + 16 = 18 bytes + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, then quants. + cl_buffer_region region; + + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_convert_block_q1_0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + + // q is uint32 (32 sign bits each); d is one half per 128-block. +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { + int M = tensor->ne[1]; // ne01 + int K = tensor->ne[0]; // ne00 + + GGML_ASSERT(K % 128 == 0); + GGML_ASSERT(M % 4 == 0); + GGML_ASSERT(tensor->ne[2] == 1); + GGML_ASSERT(tensor->ne[3] == 1); + + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/32, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/128, M); + } // end transpose +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + return; + } + // We separate the quantized bits and scale from block_q4_0 by using an + // additional kernel, where each thread handles a block. We first read the + // original weights into a temporary buffer, then create two separate + // buffers for quantized bits and scales, which are then populated by the + // conversion kernel. + if (tensor->type == GGML_TYPE_Q4_0) { + // Views can't SoA-ify here — parent owns the layout (see q8_0 guard). + // A view of an AoS tensor (the KV cache, written by a state restore) + // is written in place. + if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { + if (!ggml_cl_is_q4_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueWriteBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + } + return; + } + // Tensors should have been preallocated, therefore they should + // already have ggml_tensor_extra_cl as extra. + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q4_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_0(); + + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + + // We consider the specified offset arg as always, although For weights + // the offset arg should be 0 (we do not assert this). + //GGML_ASSERT(offset == 0); + + // We create subbuffers from the original tensor buffer for scales and + // quants - i.e., scales and quants are aliases into the buffer object + // that backs the original tensor. This is a cleaner way to adapt to the + // new memory management. + // In the old code, we allocate new buffers for scales and quants + // respectively, which could still be done but would result in double + // allocation; properly deallocating the preallocated buffer that backs + // the tensors is tricky and would leak the backend specific information + // into the general backend code. + // Does this create misaligned subbuffers (alignment is 1024) in certain + // cases ? + cl_buffer_region region; + + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, then quants. + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe q4_0 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + // Create image for Q + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + tensor->extra = extra; + // MoE tensors are also SOA'ed + ctx->q4_0_soa_tensors.insert(tensor); + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; + + // The optimized kernels need weights in natural order, so unshuffle. + if (use_adreno_kernels(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q4_0_noshuffle; + } +#else + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + ctx->q4_0_soa_tensors.insert(tensor); + + // transpose the weights and scales +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Only do transpose for large, non batched matrix + // TODO: use preallocated images instead of sub-buffer then image + if (use_adreno_kernels(backend_ctx, tensor)) { + int M = tensor->ne[1]; + int K = tensor->ne[0]; + + GGML_ASSERT(K % 32 == 0); + + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K / 8, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K / 32, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + wimg_fmt = { CL_R, CL_HALF_FLOAT }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 32; + wimg_desc.buffer = extra->d; + CL_CHECK((extra->d_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q and d as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + return; + } + if (tensor->type == GGML_TYPE_Q4_1) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q4_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_1(); + + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + GGML_ASSERT(size_d + size_m + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + + cl_buffer_region region; + + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, mins, then quants. + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for mins. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_m; + extra->m = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_m, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe q4_1 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + // Create image for Q + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + tensor->extra = extra; + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + // normal q4_1 repack +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; + + if (use_adreno_kernels(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q4_1_noshuffle; + } +#else + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor)) { + + int M = tensor->ne[1]; + int K = tensor->ne[0]; + + GGML_ASSERT(K % 32 == 0); + + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + // Transpose m as ushort + transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + return; + } + if (tensor->type == GGML_TYPE_Q5_0) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q5_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_0(); + + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_qs = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(int32_t); + GGML_ASSERT(size_d + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + + cl_buffer_region region; + + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for qh. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_qh; + extra->qh = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for qs. + region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); + region.size = size_qs; + extra->qs = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe q5_0 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + // Create image for Q + cl_image_format img_format_qs = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_qs = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->qs } + }; + extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err); + tensor->extra = extra; + + // Generic dp4a MoE path + { + static const char * q5dp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A"); + const bool q5dp4a = q5dp4a_env ? (atoi(q5dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q5dp4a && ne02 > 1 && (ne00 % 32 == 0)) { + size_t nb32 = (size_t)ne00 / 32; + size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; + size_t mn_elems = (size_t)ne02 * ne01 * nb32; + extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_0; + CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); + CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->min)); + CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(ek, 4, sizeof(int), &ne01)); + size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; + size_t el[3] = { 64, 1, 1 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + } + } + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + + int M = tensor->ne[1]; + int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); + + // Transpose qs as ushort + transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M); + // Transpose qh as uchar + transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + return; + } + if (tensor->type == GGML_TYPE_Q5_1) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q5_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_1(); + + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_qs = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(int32_t); + GGML_ASSERT(size_d + size_m + size_qs + size_qh == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + + cl_buffer_region region; + + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, mins, then quants. + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for mins. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_m; + extra->m = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for qh. + region.origin = align_to(previous_origin + size_m, backend_ctx->alignment); + region.size = size_qh; + extra->qh = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for qs. + region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); + region.size = size_qs; + extra->qs = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe q5_1 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + // Create image for Q + cl_image_format img_format_qs = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_qs = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->qs } + }; + extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err); + tensor->extra = extra; + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + + int M = tensor->ne[1]; + int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); + + // Transpose qs as ushort + transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M); + // Transpose qh as uchar + transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + // Transpose m as ushort + transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M); + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_mxfp4 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_mxfp4(); + + size_t size_e = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(char); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, then quants. + cl_buffer_region region; + + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_e; + extra->e = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_e, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno moe mxfp4 kernel needs special transpose and unshuffling + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + tensor->extra = extra; + + // Create image for Q + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + tensor->extra = extra; + + return; + } + +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); + + size_t global_work_size[3] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[3] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + // Create image for Q + cl_image_format img_format_q = {CL_RG, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast(ggml_nelements(tensor)/32*2), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + tensor->extra = extra; + + return; + } + if (tensor->type == GGML_TYPE_Q8_0) { + // Views share the parent's buffer; parent owns SoA conversion. + // A view of an AoS tensor (the KV cache, written by a state restore) + // is written in place. + if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) { + if (!ggml_cl_is_q8_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueWriteBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + } + return; + } + + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q8_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q8_0(); + + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)*sizeof(char)); + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + + // The original tensor memory is divided into scales and quants, i.e., + // we first store scales, then quants. + cl_buffer_region region; + + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_convert_block_q8_0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + ctx->q8_0_soa_tensors.insert(tensor); + + // Generic dp4a MoE path (opt-in GGML_OPENCL_Q8_MOE_DP4A) +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + { + static const char * q8dp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A"); + const bool q8dp4a = q8dp4a_env ? (atoi(q8dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q8dp4a && tensor->ne[2] > 1 && (tensor->ne[0] % 32 == 0)) { + int ne00 = (int)tensor->ne[0]; + int ne01 = (int)tensor->ne[1]; + int ne02 = (int)tensor->ne[2]; + size_t nb32 = (size_t)ne00 / 32; + size_t scale_elems = (size_t)ne02 * ne01 * nb32 * 2; // 2 per-16-seg scales / 32-block + extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, scale_elems * sizeof(cl_half), NULL, &err); + CL_CHECK(err); + cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q8_0; + CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); + CL_CHECK(clSetKernelArg(ek, 2, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne01)); + size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; + size_t el[3] = { 64, 1, 1 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + } + } +#endif + + // Transpose the weights and scales +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { + + int M = tensor->ne[1]; // ne01 + int K = tensor->ne[0]; // ne00 + + GGML_ASSERT(K % 32 == 0); + GGML_ASSERT(M % 4 == 0); + GGML_ASSERT(tensor->ne[2] == 1); + GGML_ASSERT(tensor->ne[3] == 1); + + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } // end transpose +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + return; + } + if (tensor->type == GGML_TYPE_IQ4_NL) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq4_nl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq4_nl(); + + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/2); + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "set_tensor: temp upload buffer alloc failed"); + + cl_buffer_region region; + + // Create subbuffer for scales. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + #ifdef GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_iq4_nl; + if (use_adreno_kernels(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_iq4_nl_noshuffle; + } + #else + cl_kernel kernel = backend_ctx->kernel_convert_block_iq4_nl; + #endif + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64)*64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor)) { + int M = tensor->ne[1]; + int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); + + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); + } +#endif + return; + } + if (tensor->type == GGML_TYPE_Q4_K) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q4_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_K(); + + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3 * ggml_blck_size(tensor->type) / 64); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + GGML_ASSERT(size_d + size_dm + size_s + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q4_K set_tensor: temp upload buffer alloc failed"); + + cl_buffer_region region; + + // Create subbuffer for d. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for mins. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_dm; + extra->dm = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for s. + region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); + region.size = size_s; + extra->s = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for quants. + region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_k_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + CL_CHECK(err); + tensor->extra = extra; + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed/ + // output q4_K weight (token_embd.weight, ne1=vocab) is NOT matched by + // use_adreno_moe_kernels, so it lands here in the general branch. Produce + // the final 64-row-tiled canonical layout directly into q/d/dm/s (buffer + // sizes already match), read back by kernel_gemv_noshuffle_q4_k_f32_tiled. + if (use_q4k_tiled(backend_ctx, tensor)) { + cl_kernel tk = backend_ctx->kernel_convert_block_q4_k_tiled_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + + CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(tk, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(tk, 6, sizeof(int), &ne01)); + + size_t gws[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t lws[] = {64, 1, 1}; + + cl_event tevt; + CL_CHECK(clEnqueueNDRangeKernel(queue, tk, 3, NULL, gws, lws, 0, NULL, &tevt)); + CL_CHECK(clWaitForEvents(1, &tevt)); + CL_CHECK(clReleaseMemObject(data_device)); + + extra->q_img = nullptr; + tensor->extra = extra; + return; + } + + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q4_K_noshuffle; + } +#else + cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + tensor->extra = extra; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { + + int M = tensor->ne[1]; + int K = tensor->ne[0]; + + GGML_ASSERT(K % 32 == 0); + + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose quants as 32-bit words (M-first) + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + } + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); + transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); + + // Transpose s as uchar + transpose_2d_as_8b(backend_ctx, extra->s, extra->s, size_s, K/256*12, M, true, true); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + return; + } + if (tensor->type == GGML_TYPE_Q5_K) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q5_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_K(); + + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/8; + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3*ggml_blck_size(tensor->type)/64); + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + GGML_ASSERT(size_q + size_qh + size_s + size_d + size_dm == ggml_nbytes(tensor) && + "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q5_K set_tensor: temp upload buffer alloc failed"); + + cl_buffer_region region; + + // Create subbuffer for d. + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_d; + extra->d = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + auto previous_origin = region.origin; + + // Create subbuffer for dm. + region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); + region.size = size_dm; + extra->dm = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for s. + region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); + region.size = size_s; + extra->s = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for q (lower 4 bits) + region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); + region.size = size_q; + extra->q = clCreateSubBuffer( + extra_orig->data_device, CL_MEM_READ_WRITE, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + previous_origin = region.origin; + + // Create subbuffer for qh (upper 1 bit) + region.origin = align_to(previous_origin + size_q, backend_ctx->alignment); + region.size = size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK(err); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_k_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + cl_image_format img_format_q = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_q = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->q } + }; + extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); + CL_CHECK(err); + tensor->extra = extra; + + // Generic dp4a MoE path + { + static const char * q5kdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A"); + const bool q5kdp4a = q5kdp4a_env ? (atoi(q5kdp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q5kdp4a && ne02 > 1 && (ne00 % 256 == 0)) { + size_t nb32 = (size_t)ne00 / 32; + size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; + size_t mn_elems = (size_t)ne02 * ne01 * nb32; + extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_K; + CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(ek, 3, sizeof(cl_mem), &extra->scale)); + CL_CHECK(clSetKernelArg(ek, 4, sizeof(cl_mem), &extra->min)); + CL_CHECK(clSetKernelArg(ek, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(ek, 6, sizeof(int), &ne01)); + size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), (size_t)(ne00 / 256), (size_t)ne02 }; + size_t el[3] = { 64, 1, 1 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + } + } + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; + if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle; + } +#else + cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; +#endif + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + extra->size_q = size_q; + extra->size_qh = size_qh; + extra->size_s = size_s; + extra->size_d = size_d; + extra->size_dm = size_dm; + + tensor->extra = extra; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { + + int M = tensor->ne[1]; + int K = tensor->ne[0]; + + GGML_ASSERT(K % 32 == 0); + + if (use_q5_k_bin_kernels(backend_ctx, tensor)) { + cl_int err; + cl_image_format wimg_fmt; + cl_image_desc wimg_desc; + + // transpose q as 32-bit words (M-first); qh/d/dm stay in their existing layout + // (both new ILA kernels read qh via the existing [K/8][M] uchar plane directly). + GGML_ASSERT(M % 64 == 0); + transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/8, M); + + wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = (size_t)M * K / 8; + wimg_desc.buffer = extra->q; + CL_CHECK((extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + // Transpose s as uchar + transpose_2d_as_8b(backend_ctx, extra->s, extra->s, size_s, K/256*12, M, true, true); + } else { + // Transpose q as ushort + transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); + } + transpose_2d_as_8b (backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); + transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + return; + } + if (tensor->type == GGML_TYPE_Q6_K) { + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + + // Allocate the new extra and create aliases from the original. + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q6_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q6_K(); + + size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && + "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = ggml_cl_create_temp_upload_buffer(context, queue, ggml_nbytes(tensor), data, tensor->name); + GGML_ASSERT(data_device != NULL && "q6_K set_tensor: temp upload buffer alloc failed"); + + cl_buffer_region region; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Adreno MoE Q6_K kernel needs special transposed layout + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + size_t moe_size_ql = (size_t)(ggml_nelements(tensor) / 8) * sizeof(uint32_t); // 4 bits per element + size_t moe_size_qh = (size_t)(ggml_nelements(tensor) / 16) * sizeof(uint32_t); // 2 bits per element + size_t moe_size_s = size_s; + size_t moe_size_d = size_d; + + // Subbuffer for ql + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = moe_size_ql; + CL_CHECK((extra->ql = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + auto previous_origin = region.origin; + + // Subbuffer for qh + region.origin = align_to(previous_origin + moe_size_ql, backend_ctx->alignment); + region.size = moe_size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + // Subbuffer for scales + region.origin = align_to(previous_origin + moe_size_qh, backend_ctx->alignment); + region.size = moe_size_s; + CL_CHECK((extra->s = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + // Subbuffer for d + region.origin = align_to(previous_origin + moe_size_s, backend_ctx->alignment); + region.size = moe_size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_trans4_ns; + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + // Create image for ql + cl_image_format img_format_ql = {CL_R, CL_UNSIGNED_INT32}; + cl_image_desc img_desc_ql = { + CL_MEM_OBJECT_IMAGE1D_BUFFER, + static_cast(ggml_nelements(tensor) / 8), + 0, 0, 0, 0, 0, 0, 0, + { extra->ql } + }; + extra->ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_ql, &img_desc_ql, NULL, &err); + tensor->extra = extra; + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + // Subbuffer for ql + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_ql; + CL_CHECK((extra->ql = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + auto previous_origin = region.origin; + + // Subbuffer for qh + region.origin = align_to(previous_origin + size_ql, backend_ctx->alignment); + region.size = size_qh; + CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + // Subbuffer for scales + region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); + region.size = size_s; + CL_CHECK((extra->s = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + // Create subbuffer for d. + region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Tiled-wide convert for the long-vocab lm_head/embed (opt-in). The embed + // /output q6_K weight (e.g. token_embd.weight, ne1=vocab) is NOT matched by + // use_adreno_moe_kernels, so it lands here in the general branch. Produce + // the final 64-row-tiled canonical layout directly into ql/qh/s/d (buffer + // sizes already match), read back by kernel_gemv_noshuffle_q6_K_f32_tiled. + // Bypasses the plain-SOA convert + per-array transpose below. + if (use_q6k_tiled(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_convert_block_q6_k_tiled_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + + size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + extra->size_ql = size_ql; + extra->size_qh = size_qh; + extra->size_s = size_s; + extra->size_d = size_d; + tensor->extra = extra; + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + // Flatten the weights + cl_kernel kernel; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + kernel = backend_ctx->kernel_convert_block_q6_K; + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { + kernel = backend_ctx->kernel_convert_block_q6_K_noshuffle; + } +#else + kernel = backend_ctx->kernel_convert_block_q6_K; +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + cl_uchar mask = 0xff; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64)*64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + + extra->size_ql = size_ql; + extra->size_qh = size_qh; + extra->size_s = size_s; + extra->size_d = size_d; + + tensor->extra = extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { + cl_int M = tensor->ne[1]; // ne01 + cl_int K = tensor->ne[0]; // ne00 + + if (use_q6_k_bin_kernels(backend_ctx, tensor)) { + GGML_ASSERT(K % 256 == 0); + GGML_ASSERT(M % 64 == 0); + + transpose_2d_as_32b(backend_ctx, extra->ql, extra->ql, size_ql, K/8, M); + transpose_2d_as_32b(backend_ctx, extra->qh, extra->qh, size_qh, K/16, M); + + cl_image_format wimg_fmt = { CL_R, CL_UNSIGNED_INT32 }; + cl_image_desc wimg_desc; + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = static_cast(ggml_nelements(tensor) / 8); + wimg_desc.buffer = extra->ql; + CL_CHECK((extra->ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + + memset(&wimg_desc, 0, sizeof(wimg_desc)); + wimg_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + wimg_desc.image_width = static_cast(ggml_nelements(tensor) / 16); + wimg_desc.buffer = extra->qh; + CL_CHECK((extra->qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &wimg_fmt, &wimg_desc, NULL, &err), err)); + } else { + // Transpose ql as ushort + transpose_2d_as_16b(backend_ctx, + extra->ql, extra->ql, size_ql, K/4, M); + + // Transpose qh as uchar + transpose_2d_as_8b(backend_ctx, + extra->qh, extra->qh, size_qh, K/4, M); + + // Transpose s as ushort + transpose_2d_as_16b(backend_ctx, + extra->s, extra->s, size_s, K/16/2, M); + } + // Transpose d as ushort + transpose_2d_as_16b(backend_ctx, + extra->d, extra->d, size_d, K/256, M); + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + return; + } +#endif // GGML_OPENCL_SOA_Q + + // convert bf16 to f16 and store as f16 in device buffer + if (tensor->type == GGML_TYPE_BF16) { + GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0 + && "Offset and size must be multiples of 2 for bf16 tensors"); + + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); + + cl_ulong n_elements = size / sizeof(ggml_fp16_t); + cl_ulong off_dst = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, + size, const_cast(data), &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_convert_bf16_to_f16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &off_dst)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseMemObject(data_device)); + CL_CHECK(clReleaseEvent(evt)); + + return; + } + + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); + + // a view reuses its parent's extra, whose offset does not include view_offs + CL_CHECK(clEnqueueWriteBuffer( + queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + + GGML_UNUSED(buffer); +} + +static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + GGML_ASSERT(tensor->extra); + + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; + ggml_backend_opencl_context *backend_ctx = dev_ctx->backend_ctx; + + cl_context context = backend_ctx->context; + cl_command_queue queue = backend_ctx->queue; + + // Make sure all previously submitted commands in other devices are finished. + sync_with_other_backends(backend_ctx); + +#ifdef GGML_OPENCL_SOA_Q + if (tensor->type == GGML_TYPE_Q2_K) { + ggml_tensor_extra_cl_q2_K * extra = (ggml_tensor_extra_cl_q2_K *)tensor->extra; + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, sc = extra->sc, dm = extra->dm; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_sc, buf_dm; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_sc.allocate(context, extra->size_sc); + buf_dm.allocate(context, extra->size_dm); + qs = buf_qs.buffer; + sc = buf_sc.buffer; + dm = buf_dm.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/16, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->dm, dm, extra->size_dm, m, k/(int)blck, true, true); + } +#endif + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q2_k_noshuffle + : backend_ctx->kernel_restore_block_q2_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } + + if (tensor->type == GGML_TYPE_Q1_0) { + ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { + ggml_cl_buffer buf_trans_q; + ggml_cl_buffer buf_trans_d; + ggml_cl_buffer buf_unpacked; + + int M = tensor->ne[1]; + int K = tensor->ne[0]; + + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8); + + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/32); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/128); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } +#endif + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + // In end-to-end runs, get_tensor is usually used to get back the logits, + // where we can simply do clEnqueueReadBuffer since they are f32. + // However, in test-backend-ops, the GPU graph is copied to the CPU backend, + // which requires reading back quantized weight tensors. + // To properly support this, we need to restore block_q4_0 struct arrays + // from the flattened buffers. + if (tensor->type == GGML_TYPE_Q4_0) { + // KV-cache q4_0 stays AoS — direct readback, no SoA restore. + if (!ggml_cl_is_q4_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueReadBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + return; + } + // SoA extra lives on the parent tensor — follow view_src. + const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; + ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *)extra_src->extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_trans4_ns; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (use_adreno_kernels(backend_ctx, tensor)) { + ggml_cl_buffer buf_trans_q; + ggml_cl_buffer buf_trans_d; + ggml_cl_buffer buf_unpacked; + + cl_int M = tensor->ne[1]; // ne01 + cl_int K = tensor->ne[0]; // ne00 + + GGML_ASSERT(K % 32 == 0); + GGML_ASSERT(M % 4 == 0); + + size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; + size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + if (use_q4_0_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K / 4); + } + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } +#endif + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_Q4_1) { + ggml_tensor_extra_cl_q4_1 * extra = (ggml_tensor_extra_cl_q4_1 *)tensor->extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (use_adreno_kernels(backend_ctx, tensor)) { + static ggml_cl_buffer buf_trans_q; + static ggml_cl_buffer buf_trans_m; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_unpacked; + + cl_int M = tensor->ne[1]; + cl_int K = tensor->ne[0]; + + GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); + + size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; + size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + GGML_ASSERT(size_d + size_q + size_m == ggml_nbytes(tensor) && "Incorrect tensor size"); + + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_m.allocate(backend_ctx->context, size_m); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + // transpose q, d, m back + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_m.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0)); + + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } +#endif + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_Q5_0) { + ggml_tensor_extra_cl_q5_0 * extra = (ggml_tensor_extra_cl_q5_0 *)tensor->extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + // TODO: use ggml_cl_buffer to manage this temporary buffer + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (use_adreno_kernels(backend_ctx, tensor)) { + ggml_cl_buffer buf_trans_qs; + ggml_cl_buffer buf_trans_qh; + ggml_cl_buffer buf_trans_d; + ggml_cl_buffer buf_unpacked; + + cl_int M = tensor->ne[1]; + cl_int K = tensor->ne[0]; + + GGML_ASSERT(K % 32 == 0); + + size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; + size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t); + size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + + buf_trans_qs.allocate(backend_ctx->context, size_qs); + buf_trans_qh.allocate(backend_ctx->context, size_qh); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4); + transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0)); + + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_Q5_1) { + ggml_tensor_extra_cl_q5_1 * extra = (ggml_tensor_extra_cl_q5_1 *)tensor->extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + // TODO: use ggml_cl_buffer to manage this temporary buffer + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + + if (use_adreno_kernels(backend_ctx, tensor)) { + ggml_cl_buffer buf_trans_qs; + ggml_cl_buffer buf_trans_qh; + ggml_cl_buffer buf_trans_d; + ggml_cl_buffer buf_trans_m; + ggml_cl_buffer buf_unpacked; + + cl_int M = tensor->ne[1]; + cl_int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); + + size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; + size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t); + size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + + buf_trans_qs.allocate(backend_ctx->context, size_qs); + buf_trans_qh.allocate(backend_ctx->context, size_qh); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_trans_m.allocate(backend_ctx->context, size_m); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + // Transpose back: from col-major to row-major + transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4); + transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_m.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); + + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { + ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; + size_t local_work_size[3] = {64, 2, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_Q8_0) { + // KV-cache q8_0 stays AoS (see Q4_0 branch). + if (!ggml_cl_is_q8_0_soa(tensor)) { + ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra; + CL_CHECK(clEnqueueReadBuffer( + queue, extra_aos->data_device, CL_TRUE, + extra_aos->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + return; + } + // SoA extra lives on the parent — follow view_src. + const ggml_tensor * extra_src = tensor->view_src != nullptr ? tensor->view_src : tensor; + ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)extra_src->extra; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (enable_adreno_trans_weight(backend_ctx, tensor)) { + cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + GGML_ASSERT(tensor->ne[2] == 1); + GGML_ASSERT(tensor->ne[3] == 1); + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); + + size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), 1, 1}; + size_t local_work_size[3] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } +#endif + cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_IQ4_NL) { + ggml_tensor_extra_cl_iq4_nl * extra = (ggml_tensor_extra_cl_iq4_nl *)tensor->extra; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_kernels(backend_ctx, tensor)) { + static ggml_cl_buffer buf_trans_q; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_unpacked; + + cl_int M = tensor->ne[1]; + cl_int K = tensor->ne[0]; + GGML_ASSERT(K % 32 == 0); + + size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*(ggml_blck_size(tensor->type)/2); + size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); + GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + // transpose q, d back + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + cl_kernel kernel = backend_ctx->kernel_restore_block_iq4_nl_noshuffle; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)n_blk, 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } +#endif + cl_kernel kernel = backend_ctx->kernel_restore_block_iq4_nl; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)n_blk, 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_Q4_K) { + ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Undo the 64-row-tiled canonical pack (kernel_convert_block_q4_k_tiled_ns). + // Without this, a read-back of a tiled weight returns the tiled bytes + // reinterpreted as block_q4_K -- which is how test-backend-ops builds its + // CPU reference (ggml_backend_graph_copy -> tensor_get), so the tiled path + // "failed" the suite while computing the correct product. + if (use_q4k_tiled(backend_ctx, tensor)) { + const int ne00v = tensor->ne[0]; + const int ne01v = tensor->ne[1]; + const int nbv = ne00v / 256; + const size_t n_blk = (size_t)nbv * ne01v; + + std::vector tq(n_blk*32); + std::vector td(n_blk), tdm(n_blk); + std::vector ts(n_blk*12); + CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, tq.size()*4, tq.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->dm, CL_TRUE, 0, tdm.size()*2, tdm.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); + + std::vector rebuilt(ggml_nbytes(tensor), 0); + for (int i01 = 0; i01 < ne01v; ++i01) { + const int rt = i01/64, rit = i01%64; + for (int i00 = 0; i00 < nbv; ++i00) { + uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*144; + const int tb = rt*nbv + i00; + const size_t si = (size_t)tb*64 + rit; + + memcpy(b + 0, &td [si], 2); + memcpy(b + 2, &tdm[si], 2); + memcpy(b + 4, &ts[si*12], 12); + + uint32_t qw[32]; + for (int gr = 0; gr < 8; ++gr) { + const size_t base = ((size_t)tb*8 + gr)*64 + rit; + for (int j = 0; j < 4; ++j) qw[gr*4 + j] = tq[base*4 + j]; + } + uint8_t * q = b + 16; + for (int e = 0; e < 256; ++e) { + const int g = e>>6, w = e&63, h = w>>5, l = w&31; + const uint32_t code = (qw[e>>3] >> ((e&7)*4)) & 0xF; + q[g*32 + l] |= (uint8_t)(h ? (code << 4) : code); + } + } + } + memcpy(data, rebuilt.data() + offset, size); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_k_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor)) { + int M = tensor->ne[1]; + int K = tensor->ne[0]; + + size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*12; + + static ggml_cl_buffer buf_trans_q; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_trans_dm; + static ggml_cl_buffer buf_trans_s; + + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_trans_dm.allocate(backend_ctx->context, size_dm); + buf_trans_s.allocate(backend_ctx->context, size_s); + + // Transpose q, d, dm, s back + if (use_q4_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + } + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); + transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); + transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_s.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_dm.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_Q5_K) { + ggml_tensor_extra_cl_q5_K * extra = (ggml_tensor_extra_cl_q5_K *)tensor->extra; + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_k_trans4_ns; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) { + int M = tensor->ne[1]; + int K = tensor->ne[0]; + + size_t size_q = extra->size_q; + size_t size_qh = extra->size_qh; + size_t size_s = extra->size_s; + size_t size_d = extra->size_d; + size_t size_dm = extra->size_dm; + + static ggml_cl_buffer buf_trans_q; + static ggml_cl_buffer buf_trans_qh; + static ggml_cl_buffer buf_trans_s; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_trans_dm; + + buf_trans_q.allocate(backend_ctx->context, size_q); + buf_trans_qh.allocate(backend_ctx->context, size_qh); + buf_trans_s.allocate(backend_ctx->context, size_s); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_trans_dm.allocate(backend_ctx->context, size_dm); + + // bin kernel transposes s but src kernel does not + cl_mem buf_s = extra->s; + + if (use_q5_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/8); + transpose_2d_as_8b (backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/256*12, true, true); + buf_s = buf_trans_s.buffer; + } else { + transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); + } + transpose_2d_as_8b (backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); + transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_trans_dm.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (tensor->type == GGML_TYPE_Q6_K) { + ggml_tensor_extra_cl_q6_K * extra = (ggml_tensor_extra_cl_q6_K *)tensor->extra; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // Undo the 64-row-tiled canonical pack (kernel_convert_block_q6_k_tiled_ns). + // See the q4_K tiled restore above for why a read-back path is required. + if (use_q6k_tiled(backend_ctx, tensor)) { + const int ne00v = tensor->ne[0]; + const int ne01v = tensor->ne[1]; + const int nbv = ne00v / 256; + const size_t n_blk = (size_t)nbv * ne01v; + + std::vector tql(n_blk*32), tqh(n_blk*16); + std::vector ts(n_blk*16); + std::vector td(n_blk); + CL_CHECK(clEnqueueReadBuffer(queue, extra->ql, CL_TRUE, 0, tql.size()*4, tql.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->qh, CL_TRUE, 0, tqh.size()*4, tqh.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->s, CL_TRUE, 0, ts.size(), ts.data(), 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, td.size()*2, td.data(), 0, NULL, NULL)); + + std::vector rebuilt(ggml_nbytes(tensor), 0); + for (int i01 = 0; i01 < ne01v; ++i01) { + const int rt = i01/64, rit = i01%64; + for (int i00 = 0; i00 < nbv; ++i00) { + uint8_t * b = rebuilt.data() + ((size_t)i00 + (size_t)i01*nbv)*210; + const int tb = rt*nbv + i00; + const size_t si = (size_t)tb*64 + rit; + + uint32_t qlw[32], qhw[16]; + for (int g = 0; g < 8; ++g) { + const size_t base = ((size_t)tb*8 + g)*64 + rit; + for (int j = 0; j < 4; ++j) qlw[g*4 + j] = tql[base*4 + j]; + } + for (int g = 0; g < 4; ++g) { + const size_t base = ((size_t)tb*4 + g)*64 + rit; + for (int j = 0; j < 4; ++j) qhw[g*4 + j] = tqh[base*4 + j]; + } + + uint8_t * ql = b; + uint8_t * qh = b + 128; + for (int e = 0; e < 256; ++e) { + const int n = (e >= 128) ? 1 : 0; + const int within = e - n*128, q = within/32, l = within%32; + const int off_ql = n*64, off_qh = n*32; + const uint8_t low4 = (qlw[e>>3] >> ((e&7)*4)) & 0xF; + const uint8_t hi2 = (qhw[e>>4] >> ((e&15)*2)) & 0x3; + if (q == 0) ql[off_ql + l] |= low4; + else if (q == 1) ql[off_ql + l + 32] |= low4; + else if (q == 2) ql[off_ql + l] |= (uint8_t)(low4 << 4); + else ql[off_ql + l + 32] |= (uint8_t)(low4 << 4); + qh[off_qh + l] |= (uint8_t)(hi2 << (q*2)); + } + memcpy(b + 192, &ts[si*16], 16); + memcpy(b + 208, &td[si], 2); + } + } + memcpy(data, rebuilt.data() + offset, size); + return; + } + if (use_adreno_moe_kernels(backend_ctx, tensor)) { + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_kernel kernel = backend_ctx->kernel_restore_block_q6_k_trans4_ns; + + cl_uchar mask_0F = 0x0F; + cl_uchar mask_F0 = 0xF0; + + int ne00 = tensor->ne[0]; + int ne01 = tensor->ne[1]; + int ne02 = tensor->ne[2]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uchar), &mask_F0)); + + size_t global_work_size[] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 256), static_cast(ne02)}; + size_t local_work_size[] = {64, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } + if (use_adreno_kernels(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor)) { + static ggml_cl_buffer buf_trans_ql; + static ggml_cl_buffer buf_trans_qh; + static ggml_cl_buffer buf_trans_s; + static ggml_cl_buffer buf_trans_d; + static ggml_cl_buffer buf_unpacked; + + cl_int M = tensor->ne[1]; // ne01 + cl_int K = tensor->ne[0]; // ne00 + + GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); + + size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; + size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; + size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; + size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + buf_trans_ql.allocate(backend_ctx->context, size_ql); + buf_trans_qh.allocate(backend_ctx->context, size_qh); + buf_trans_d.allocate(backend_ctx->context, size_d); + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + cl_mem s_buffer; + if (use_q6_k_bin_kernels(backend_ctx, tensor)) { + transpose_2d_as_32b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/8); + transpose_2d_as_32b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/16); + // s is left row-major, untransposed, for the binary layout. + s_buffer = extra->s; + } else { + // transpose ql, qh, s and d back + buf_trans_s.allocate(backend_ctx->context, size_s); + transpose_2d_as_16b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/4); + transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/4); + transpose_2d_as_16b(backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/16/2); + s_buffer = buf_trans_s.buffer; + } + transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); + + // unpack + cl_uchar mask = 0xFF; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_ql.buffer)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s_buffer)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)n_blk, 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + cl_mem data_device = ggml_cl_create_temp_download_buffer(context, queue, ggml_nbytes(tensor), tensor->name); + GGML_ASSERT(data_device != NULL && "get_tensor: temp download buffer alloc failed"); + + cl_uchar mask = 0xFF; + cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); + cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + + size_t global_work_size[] = {(size_t)n_blk, 1, 1}; + size_t local_work_size[] = {1, 1, 1}; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, offset, + size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + return; + } +#endif // GGML_OPENCL_SOA_Q + + if (tensor->type == GGML_TYPE_BF16) { + GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0 + && "Offset and size must be multiples of 2 for bf16 tensors"); + + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); + + cl_ulong n_elements = size / sizeof(ggml_fp16_t); + cl_ulong off_src = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, size, NULL, &err); + CL_CHECK(err); + + cl_kernel kernel = backend_ctx->kernel_convert_f16_to_bf16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + + CL_CHECK(clEnqueueReadBuffer( + queue, data_device, CL_TRUE, 0, size, data, 0, NULL, NULL)); + CL_CHECK(clReleaseMemObject(data_device)); + + return; + } + + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + + CL_CHECK(clEnqueueReadBuffer( + queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset, + size, data, 0, NULL, NULL)); + + GGML_UNUSED(buffer); +} + +static void ggml_backend_opencl_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer->buft->device->context; + ggml_backend_opencl_context * backend_ctx = dev_ctx->backend_ctx; + + cl_command_queue queue = backend_ctx->queue; + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + for (cl_mem buf : ctx->buffer) { + CL_CHECK(clEnqueueFillBuffer(queue, buf, &value, sizeof(value), 0, buffer->size, 0, NULL, NULL)); + } + CL_CHECK(clFinish(queue)); +} + +static void ggml_backend_opencl_buffer_reset(ggml_backend_buffer_t buffer) { + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ctx->reset(); +} + +static ggml_backend_buffer_i ggml_backend_opencl_buffer_interface = { + /* .free_buffer = */ ggml_backend_opencl_buffer_free_buffer, + /* .get_base = */ ggml_backend_opencl_buffer_get_base, + /* .init_tensor = */ ggml_backend_opencl_buffer_init_tensor, + /* .memset_tensor = */ NULL, + /* .set_tensor = */ ggml_backend_opencl_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_opencl_buffer_get_tensor, + /* .set_tensor_2d = */ NULL, + /* .get_tensor_2d = */ NULL, + /* .cpy_tensor = */ NULL, + /* .clear = */ ggml_backend_opencl_buffer_clear, + /* .reset = */ ggml_backend_opencl_buffer_reset, +}; + +// +// buffer type +// + +static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type) { + return "OpenCL"; + + GGML_UNUSED(buffer_type); +} + +static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buffer_type, size_t size) { + ggml_backend_opencl_context *backend_ctx = ggml_cl_init(buffer_type->device); + + if (!backend_ctx->program_cache_initialized) { + backend_ctx->program_cache = cl_program_cache_init(backend_ctx->device); + backend_ctx->program_cache_initialized = true; + } + load_cl_kernels(backend_ctx); + + // clCreateBuffer returns -61 for size 0 + size = std::max(size, (size_t)1); + + cl_int err; + cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + // On Adreno X1-85 the device pool intermittently fails at hundreds of MB + // once the heap fragments (e.g. graph-allocator compute-buffer reserve + // after model load). Four-step retry: + // 1. normal alloc (fast path) + // 2. clFinish + retry (drains in-flight allocs) + // 3. cl_qcom_large_buffer (X2-class driver only, OpenCL 3.0 only) + // 4. ALLOC_HOST_PTR (host-pinned pool) — last-resort fallback. This + // buffer backs compute scratch read/written by every kernel in the + // graph, so kernel accesses fall to host memory and runtime perf + // degrades meaningfully. Better than failing to load, but the user + // should see the warning and consider -ngl reduction. + if (err != CL_SUCCESS) { + clFinish(backend_ctx->queue); + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); + } +#if GGML_OPENCL_TARGET_VERSION >= 300 + // clCreateBufferWithProperties and cl_mem_properties are OpenCL 3.0. Drivers older than + // that do not export the symbol, so a build targeting them fails to link. The large + // buffer extension is only ever enabled on drivers that are well past 3.0, so this path + // is dead there anyway. + if (err != CL_SUCCESS && backend_ctx->adreno_use_large_buffer) { + cl_mem_properties props[] = { 0x41A6 /* CL_LARGE_BUFFER_QCOM */, 1, 0 }; + mem = clCreateBufferWithProperties(backend_ctx->context, props, CL_MEM_READ_WRITE, size, NULL, &err); + } +#endif + if (err != CL_SUCCESS) { + mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, size, NULL, &err); + if (err == CL_SUCCESS) { + GGML_LOG_WARN("%s: %.2f MiB allocated via CL_MEM_ALLOC_HOST_PTR fallback — " + "device pool exhausted; runtime perf will be degraded. " + "Consider lowering -ngl or context size.\n", + __func__, size / 1024.0 / 1024.0); + } + } + + if (err != CL_SUCCESS) { + GGML_LOG_ERROR("%s: failed to allocate %.2f MiB (err=%d). " + "Consider reducing -ngl, lowering -c / -ub, or using quantized KV cache.\n", + __func__, size / 1024.0 / 1024.0, err); + return nullptr; + } + + ggml_backend_opencl_buffer_context * ctx = new ggml_backend_opencl_buffer_context(mem); + + return ggml_backend_buffer_init(buffer_type, ggml_backend_opencl_buffer_interface, ctx, size); +} + +static size_t ggml_backend_opencl_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer_type->device->context; + return dev_ctx->backend_ctx->alignment; +} + +static size_t ggml_backend_opencl_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) { + static size_t max_size = -1; + if (max_size == (size_t)-1) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buffer_type->device->context; + max_size = dev_ctx->backend_ctx->max_alloc_size; + } + return max_size; +} + +static bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { + return ggml_backend_is_opencl(backend); + + UNUSED(buft); +} + +static size_t ggml_backend_opencl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + size_t size = ggml_nbytes(tensor); +#ifdef GGML_OPENCL_SOA_Q + // set_tensor carves quantized weights into per-component subbuffers (d/q, + // ql/qh/s/d, ...) whose origins are each rounded up to the device base + // alignment. When a component's size is not a multiple of the alignment + // (e.g. q6_K [1536,49155]: size_s = 49155*96 leaves a 96-byte gap at 128-byte + // alignment), the aligned carve extends past ggml_nbytes and the last + // subbuffer would overlap the next tensor in the pool. Reserve the worst-case + // carve slack: at most 5 components (q5_K), i.e. 4 aligned gaps. + if (ggml_is_quantized(tensor->type)) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buft->device->context; + size += 4 * dev_ctx->backend_ctx->alignment; + } +#endif // GGML_OPENCL_SOA_Q + return size; +} + +static ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = { + /* .get_name = */ ggml_backend_opencl_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_opencl_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_opencl_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ NULL, +}; + +// +// backend device +// + +static const char * ggml_backend_opencl_device_get_name(ggml_backend_dev_t dev) { + return "GPUOpenCL"; + + GGML_UNUSED(dev); +} + +static const char * ggml_backend_opencl_device_get_description(ggml_backend_dev_t dev) { + ggml_backend_opencl_device_context *dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + return dev_ctx->device_name.c_str(); +} + +static void ggml_backend_opencl_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + + static const size_t opencl_extra_margin = 1024ull*1024ull*1024ull; + + // OpenCL does not provide reliable currently-free device memory. + // Use total/global memory as a best-effort upper bound. + // Improved safety: Reduce by a 1GiB extra margin for common --fit + *total = dev_ctx->global_mem_size; + *free = *total > opencl_extra_margin ? *total - opencl_extra_margin : 0; +} + +static enum ggml_backend_dev_type ggml_backend_opencl_device_get_type(ggml_backend_dev_t dev) { + return GGML_BACKEND_DEVICE_TYPE_GPU; + + GGML_UNUSED(dev); +} + +static void ggml_backend_opencl_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { + props->name = ggml_backend_opencl_device_get_name(dev); + props->description = ggml_backend_opencl_device_get_description(dev); + props->type = ggml_backend_opencl_device_get_type(dev); + ggml_backend_opencl_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->caps = ggml_backend_dev_caps { + /* .async = */ false, + /* .host_buffer = */ false, + /* .buffer_from_host_ptr = */ false, + /* .events = */ false, + /* .mmap_support = */ false, + }; +} + +static ggml_backend_t ggml_backend_opencl_device_init(ggml_backend_dev_t dev, const char * params) { + ggml_backend_opencl_context * backend_ctx = ggml_cl_init(dev); + // Getting a new reference to the backend, increase ref_count + backend_ctx->ref_count++; + + ggml_backend_t backend = new ggml_backend { + /* .guid = */ ggml_backend_opencl_guid(), + /* .interface = */ ggml_backend_opencl_i, + /* .device = */ dev, + /* .context = */ backend_ctx, + }; + + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) dev->context; + ggml_opencl_print_backend_info(dev_ctx); + return backend; + + GGML_UNUSED(params); +} + +static ggml_backend_buffer_type_t ggml_backend_opencl_device_get_buffer_type(ggml_backend_dev_t dev) { + auto * dev_ctx = static_cast(dev->context); + + dev_ctx->buffer_type = ggml_backend_buffer_type{ + /* .iface = */ ggml_backend_opencl_buffer_type_interface, + /* .device = */ dev, + /* .context = */ nullptr, + }; + + return &dev_ctx->buffer_type; +} + +static ggml_backend_buffer_t ggml_backend_opencl_device_buffer_from_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { + GGML_UNUSED(dev); + GGML_UNUSED(ptr); + GGML_UNUSED(size); + GGML_UNUSED(max_tensor_size); + return nullptr; +} + +static bool ggml_backend_opencl_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + ggml_cl_init(dev); + return ggml_opencl_supports_op(dev, op); +} + +static bool ggml_backend_opencl_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { + // Check 'dev' and 'buffer_type' are not objects belonging to this backend. + if (dev->iface.get_name != ggml_backend_opencl_device_get_name || + buft->iface.get_name != ggml_backend_opencl_buffer_type_get_name) { + return false; + } + + // Check cl_context is the same. clEnqueue* commands may not use + // buffers from another cl_context. + ggml_backend_opencl_context * backend_ctx0 = ggml_cl_init(dev); + ggml_backend_opencl_context * backend_ctx1 = ggml_cl_init(buft->device); + return backend_ctx0->context == backend_ctx1->context; +} + +namespace /* anonymous */ { +struct ggml_backend_device_i ggml_backend_opencl_device_i = { + /* .get_name = */ ggml_backend_opencl_device_get_name, + /* .get_description = */ ggml_backend_opencl_device_get_description, + /* .get_memory = */ ggml_backend_opencl_device_get_memory, + /* .get_type = */ ggml_backend_opencl_device_get_type, + /* .get_props = */ ggml_backend_opencl_device_get_props, + /* .init_backend = */ ggml_backend_opencl_device_init, + /* .get_buffer_type = */ ggml_backend_opencl_device_get_buffer_type, + /* .get_host_buffer_type = */ NULL, + /* .buffer_from_host_ptr = */ ggml_backend_opencl_device_buffer_from_ptr, + /* .supports_op = */ ggml_backend_opencl_device_supports_op, + /* .supports_buft = */ ggml_backend_opencl_device_supports_buft, + /* .offload_op = */ NULL, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_synchronize = */ NULL, +}; +} + +// Backend registry + +static const char * ggml_backend_opencl_reg_get_name(ggml_backend_reg_t reg) { + return "OpenCL"; + + GGML_UNUSED(reg); +} + +static size_t ggml_backend_opencl_reg_device_count(ggml_backend_reg_t reg) { + return g_ggml_backend_opencl_devices.size(); + + GGML_UNUSED(reg); +} + +static ggml_backend_dev_t ggml_backend_opencl_reg_device_get(ggml_backend_reg_t reg, size_t index) { + GGML_ASSERT(index < ggml_backend_opencl_reg_device_count(reg)); + + return &g_ggml_backend_opencl_devices[index]; + + GGML_UNUSED(reg); + GGML_UNUSED(index); +} + +static struct ggml_backend_reg_i ggml_backend_opencl_reg_i = { + /* .get_name = */ ggml_backend_opencl_reg_get_name, + /* .device_count = */ ggml_backend_opencl_reg_device_count, + /* .device_get = */ ggml_backend_opencl_reg_device_get, + /* .get_proc_address = */ NULL, +}; + +ggml_backend_reg_t ggml_backend_opencl_reg(void) { + static std::mutex mutex; + static ggml_backend_reg reg; + static bool initialized = false; + std::lock_guard lock(mutex); + + if (initialized) { + return ® + } + initialized = true; + + g_ggml_backend_opencl_devices = ggml_opencl_probe_devices(®); + + reg = ggml_backend_reg{ + /* .api_version = */ GGML_BACKEND_API_VERSION, + /* .iface = */ ggml_backend_opencl_reg_i, + /* .context = */ NULL, + }; + + return ® +} + +GGML_BACKEND_DL_IMPL(ggml_backend_opencl_reg) + +//------------------------------------------------------------------------------ +// Debugging utils +//------------------------------------------------------------------------------ +#if 0 +#define QK4_0 32 +typedef struct { + ggml_fp16_t d; // delta + uint8_t qs[QK4_0 / 2]; // nibbles / quants +} block_q4_0; +static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, + "wrong q4_0 block size/padding"); + +#define QK_MXFP4 32 + +#include +#ifdef __cplusplus +#include "half.hpp" +#endif + +static void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tensor) { + void * buf = malloc(ggml_nbytes(tensor)); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_command_queue queue = backend_ctx->queue; +#ifdef GGML_OPENCL_SOA_Q + void * buf_q; + void * buf_d; +#endif + + // Make sure everything is done. + CL_CHECK(clFinish(queue)); + +#ifdef GGML_OPENCL_SOA_Q + if (tensor->type == GGML_TYPE_Q4_0) { + ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *) tensor->extra; + GGML_ASSERT(extra); + + size_t size_q = ggml_nelements(tensor)/QK4_0 * QK4_0/2; + size_t size_d = ggml_nelements(tensor)/QK4_0 * sizeof(ggml_fp16_t); + GGML_ASSERT(size_q + size_d == ggml_nbytes(tensor)); + buf_q = malloc(size_q); + buf_d = malloc(size_d); + + CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, size_d, buf_d, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); + } else if (tensor->type == GGML_TYPE_MXFP4) { + ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *) tensor->extra; + GGML_ASSERT(extra); + + size_t size_q = ggml_nelements(tensor)/QK_MXFP4 * QK_MXFP4/2; + size_t size_e = ggml_nelements(tensor)/QK_MXFP4 * sizeof(char); + GGML_ASSERT(size_q + size_e == ggml_nbytes(tensor)); + buf_q = malloc(size_q); + buf_d = malloc(size_e); + + CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); + CL_CHECK(clEnqueueReadBuffer(queue, extra->e, CL_TRUE, 0, size_e, buf_d, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); + } else { + // Read out the tensor from GPU memory. + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); + + CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, + extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); + } +#else + // Read out the tensor from GPU memory. + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra); + + CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, + extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); + CL_CHECK(clFinish(queue)); +#endif // GGML_OPENCL_SOA_Q + + // Open file and dump. + char fname[512]; + snprintf(fname, sizeof(fname), "./tensor-dumps/%s.txt", tensor->name); + FILE * f = fopen(fname, "w"); + if (!f) { + printf("Failed to open %s\n", fname); + return; + } + + if (tensor->type == GGML_TYPE_F32) { + float * data = (float *) buf; + for (int i = 0; i < ggml_nelements(tensor); ++i) { + if (isnan(data[i])) { + printf("NaN found: %s\n", tensor->name); + break; + } + fprintf(f, "%f\n", data[i]); + } + } else if (tensor->type == GGML_TYPE_I32) { + int * data = (int *) buf; + for (int i = 0; i < ggml_nelements(tensor); ++i) { + if (isnan(data[i])) { + printf("NaN found: %s\n", tensor->name); + break; } - return false; - case GGML_OP_MUL_MAT_ID: - if (op->src[0]->type == GGML_TYPE_Q4_0 || - op->src[0]->type == GGML_TYPE_Q8_0 || - op->src[0]->type == GGML_TYPE_MXFP4) { - if (op->src[1]->type == GGML_TYPE_F32) { - return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); - } + fprintf(f, "%d\n", data[i]); + } + } else if (tensor->type == GGML_TYPE_F16) { +#ifdef __cplusplus + half_float::half * data = (half_float::half *) buf; + for (int i = 0; i < ggml_nelements(tensor); ++i) { + if (std::isnan(data[i])) { + printf("NaN found: %s\n", tensor->name); + break; } - return false; - case GGML_OP_RESHAPE: - case GGML_OP_VIEW: - case GGML_OP_PERMUTE: - case GGML_OP_TRANSPOSE: - return true; - case GGML_OP_DIAG: - return true; - case GGML_OP_DIAG_MASK_INF: - return op->ne[3] == 1; - case GGML_OP_ROPE: { - const int mode = ((const int32_t *) op->op_params)[2]; - const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE; - const bool is_vision = mode == GGML_ROPE_TYPE_VISION; - if (is_mrope && !is_vision) { - if (op->src[0]->type == GGML_TYPE_F32 || - op->src[0]->type == GGML_TYPE_F16) { - return true; - } - return false; + fprintf(f, "%f\n", float(data[i])); + } +#endif + } else if (tensor->type == GGML_TYPE_Q4_0) { +#ifdef GGML_OPENCL_SOA_Q + ggml_fp16_t * data_d = (ggml_fp16_t *)buf_d; + unsigned char * data_q = (unsigned char *)buf_q; + + for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { + fprintf(f, "%04x, ", data_d[i]); + for (int k = 0; k < QK4_0/2; ++k) { + fprintf(f, "%02x, ", data_q[k]); } - if (is_vision) { - if (op->src[0]->type == GGML_TYPE_F32 || - op->src[0]->type == GGML_TYPE_F16) { - return true; - } - return false; + fprintf(f, "\n"); + data_q += QK4_0/2; + } + free(buf_d); + free(buf_q); +#else + block_q4_0 * data = (block_q4_0 *) buf; + for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { + fprintf(f, "%04x, ", data[i].d); + for (int k = 0; k < QK4_0/2; ++k) { + fprintf(f, "%02x, ", data[i].qs[k]); } - return true; + fprintf(f, "\n"); + } +#endif // GGML_OPENCL_SOA_Q + } + free(buf); + fflush(f); + fclose(f); +} +#else +#define dump_tensor(tensor) +#endif + +//------------------------------------------------------------------------------ +// Ops +//------------------------------------------------------------------------------ + +static bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { + const int64_t ne10 = src1->ne[0]; + + const int64_t ne0 = dst->ne[0]; + const int64_t ne1 = dst->ne[1]; + + // TODO: find the optimal values for these + return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && + src1->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F32 && + (ne0 >= 32 && ne1 >= 32 && ne10 >= 32); +} + +// Copy a noncontiguous tensor to contiguous tensor. ne[] remains the same but +// nb[] is recalculated such that tensor is contiguous. +static void ggml_cl_copy_to_contiguous(ggml_backend_t backend, const ggml_tensor * src, cl_mem dst, + cl_ulong &nb0, cl_ulong &nb1, cl_ulong &nb2, cl_ulong &nb3) { + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + const int tensor_type_size = ggml_type_size(src->type); + + const int ne00 = src->ne[0]; + const int ne01 = src->ne[1]; + const int ne02 = src->ne[2]; + const int ne03 = src->ne[3]; + + const cl_ulong nb00 = src->nb[0]; + const cl_ulong nb01 = src->nb[1]; + const cl_ulong nb02 = src->nb[2]; + const cl_ulong nb03 = src->nb[3]; + + const int ne0 = src->ne[0]; + const int ne1 = src->ne[1]; + const int ne2 = src->ne[2]; + const int ne3 = src->ne[3]; + + nb0 = tensor_type_size; + nb1 = tensor_type_size*ne00; + nb2 = tensor_type_size*ne00*ne01; + nb3 = tensor_type_size*ne00*ne01*ne02; + + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *)src->extra; + + cl_ulong offset0 = extra->offset + src->view_offs; + cl_ulong offsetd = 0; + + cl_kernel kernel; + + switch (src->type) { + case GGML_TYPE_F32: + kernel = backend_ctx->kernel_cpy_f32_f32; + break; + case GGML_TYPE_F16: + case GGML_TYPE_BF16: // stored as f16 on device + kernel = backend_ctx->kernel_cpy_f16_f16; + break; + default: + GGML_ASSERT(false && "not implemented"); + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &dst)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb3)); + + const int nth = MIN(64, ne00); + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src); +} + +static void ggml_cl_nop(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + UNUSED(backend); + UNUSED(src0); + UNUSED(src1); + UNUSED(dst); +} + +static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(int, ne1, src1, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + cl_kernel kernel; + + switch (src0->type) { + case GGML_TYPE_F32: + kernel = backend_ctx->kernel_get_rows_f32; + break; + case GGML_TYPE_I32: + kernel = backend_ctx->kernel_get_rows_i32; + break; + case GGML_TYPE_F16: + kernel = backend_ctx->kernel_get_rows_f16; + break; + case GGML_TYPE_Q4_0: + kernel = backend_ctx->kernel_get_rows_q4_0; + break; + default: + GGML_ASSERT(false && "not implemented"); + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); + + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + int nth = 1; + while (nth < ne00 && 2*nth <= max_workgroup_size) { + nth *= 2; + } + + int nchunks = 1; + if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32) { + const int chunk_target = nth * 4; + nchunks = (ne00 + chunk_target - 1) / chunk_target; + nchunks = MAX(1, MIN(nchunks, 64)); + } + + size_t global_work_size[] = {(size_t)ne10*nth*nchunks, (size_t)ne11, (size_t)ne12}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +// check if a Q8_0 tensor has been SOA'ed in set_tensor +// we store SOA'ed tensors in a map in set_tensor, check against that map +static bool ggml_cl_is_q8_0_soa(const ggml_tensor * tensor) { + if (tensor == nullptr || tensor->type != GGML_TYPE_Q8_0 || tensor->buffer == nullptr) { + return false; + } + auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; + if (ctx == nullptr) { + return false; + } + const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; + return ctx->q8_0_soa_tensors.count(key) > 0; +} + +// check if a Q4_0 tensor has been SOA'ed in set_tensor +// we store SOA'ed tensors in a map in set_tensor, check against that map +static bool ggml_cl_is_q4_0_soa(const ggml_tensor * tensor) { + if (tensor == nullptr || tensor->type != GGML_TYPE_Q4_0 || tensor->buffer == nullptr) { + return false; + } + auto * ctx = (ggml_backend_opencl_buffer_context *) tensor->buffer->context; + if (ctx == nullptr) { + return false; + } + const ggml_tensor * key = tensor->view_src != nullptr ? tensor->view_src : tensor; + return ctx->q4_0_soa_tensors.count(key) > 0; +} + +static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32); + + // ne0 = ne00 + // ne2 = ne02 + // ne3 = ne03 + + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + + GGML_TENSOR_LOCALS(int, ne1, src1, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); + + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + + const int nblk0 = ne0/ggml_blck_size(dst->type); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + + const bool q8_0_soa = dst->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(dst); + const bool q4_0_soa = dst->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(dst); + const bool is_soa = q8_0_soa || q4_0_soa; + + cl_kernel kernel; + + if (q8_0_soa) { + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q8_0_soa_i64 + : backend_ctx->kernel_set_rows_q8_0_soa_i32; + } else if (q4_0_soa) { + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q4_0_soa_i64 + : backend_ctx->kernel_set_rows_q4_0_soa_i32; + } else { + switch (dst->type) { + case GGML_TYPE_F32: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_f32_i64 + : backend_ctx->kernel_set_rows_f32_i32; + break; + case GGML_TYPE_F16: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_f16_i64 + : backend_ctx->kernel_set_rows_f16_i32; + break; + case GGML_TYPE_Q8_0: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q8_0_i64 + : backend_ctx->kernel_set_rows_q8_0_i32; + break; + case GGML_TYPE_Q4_0: + kernel = (src1->type == GGML_TYPE_I64) + ? backend_ctx->kernel_set_rows_q4_0_i64 + : backend_ctx->kernel_set_rows_q4_0_i32; + break; + default: + GGML_ABORT("not implemented"); + } + } + + fastdiv_vals ne11_ = init_fastdiv_values(ne11); + fastdiv_vals ne12_ = init_fastdiv_values(ne12); + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + + if (is_soa) { + // The q/d subbuffers in q8_0/q4_0 extras are interchangeable here. + // For views (e.g. ggml_set_rows' `out`), follow view_src for the SoA extra. + const ggml_tensor * soa_src = dst->view_src != nullptr ? dst->view_src : dst; + cl_mem q_mem = nullptr; + cl_mem d_mem = nullptr; + if (q8_0_soa) { + ggml_tensor_extra_cl_q8_0 * e = (ggml_tensor_extra_cl_q8_0 *)soa_src->extra; + q_mem = e->q; + d_mem = e->d; + } else { + ggml_tensor_extra_cl_q4_0 * e = (ggml_tensor_extra_cl_q4_0 *)soa_src->extra; + q_mem = e->q; + d_mem = e->d; + } + cl_ulong offset_q = 0; + cl_ulong offset_d = 0; + const int ne1_dst = dst->ne[1]; + const int ne2_dst = dst->ne[2]; + const int ne3_dst = dst->ne[3]; + + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &q_mem)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_q)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &d_mem)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_d)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(fastdiv_vals), &ne11_)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(fastdiv_vals), &ne12_)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &nblk0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1_dst)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne2_dst)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne3_dst)); + } else { + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(fastdiv_vals), &ne11_)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(fastdiv_vals), &ne12_)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &nblk0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb3)); + } + + int nth0 = 64; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 32; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + } + + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (nth0 < nblk0 && nth0 < max_workgroup_size) { + nth0 *= 2; + } + + int rows_per_workgroup = 1; + if (nth0 > nblk0) { + rows_per_workgroup = nth0 / nblk0; + nth0 = nblk0; + } + + size_t global_work_size[] = { + (size_t)(ne01 + rows_per_workgroup - 1)/rows_per_workgroup*nth0, + (size_t)ne02*rows_per_workgroup, + (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth0, (size_t)rows_per_workgroup, 1}; + + // ne01 == 0 makes global_work_size[0] zero here; enqueue_ndrange_kernel drops the empty range. + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; + + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; + + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; + + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + cl_kernel kernel; + + const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0; + + if (bcast_row) { + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ne11 == 1); + } + + if (dst->type == GGML_TYPE_F32) { + GGML_ASSERT(src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32); + if (bcast_row) { + kernel = backend_ctx->kernel_add_row; + const int ne = ne00 / 4; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + } else { + kernel = backend_ctx->kernel_add; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); + } + } else if (dst->type == GGML_TYPE_F16) { + GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); + const int type_src0 = (src0->type == GGML_TYPE_F32); + const int type_src1 = (src1->type == GGML_TYPE_F32); + if (bcast_row) { + kernel = backend_ctx->kernel_add_row_f16; + const int ne = ne00 / 4; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &type_src0)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &type_src1)); + } else { + kernel = backend_ctx->kernel_add_f16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &type_src0)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(int), &type_src1)); } - case GGML_OP_SOLVE_TRI: - return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); - case GGML_OP_IM2COL: - return true; - case GGML_OP_ARGSORT: { - cl_kernel kernel = backend_ctx->kernel_argsort_f32_i32; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + } else { + GGML_ASSERT(false && "unsupported data types for add"); + } + + if (bcast_row) { + int n = ggml_nelements(dst)/4; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size_ptr, dst); + } else { + unsigned int nth = MIN(64, ne0); + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } +} + +static void ggml_cl_add_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + const ggml_tensor * src2 = dst->src[2]; + GGML_ASSERT(src2); + GGML_ASSERT(src2->extra); + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(src2->type == GGML_TYPE_I32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_ASSERT(ggml_is_contiguous_rows(src0)); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + + const cl_ulong nb11 = src1->nb[1]; + + const cl_ulong nb21 = src2->nb[1]; + + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + cl_kernel kernel = backend_ctx->kernel_add_id; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); + + int nth = MIN(ne00, (int) backend_ctx->get_kernel_workgroup_size(kernel)); + size_t global_work_size[] = { (size_t)ne01*nth, (size_t)ne02, 1 }; + size_t local_work_size[] = { (size_t)nth, 1, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_mul(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; UNUSED(ne13); + + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; UNUSED(nb13); + + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; + + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + bool bcast_row = false; + cl_kernel kernel; + + if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { + GGML_ASSERT(ggml_is_contiguous(src0)); + + // src1 is a row + GGML_ASSERT(ne11 == 1); + + bcast_row = true; + int ne = ne00 / 4; + + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_mul_row; + } else { + kernel = backend_ctx->kernel_mul_row_f16; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_mul; + } else { + kernel = backend_ctx->kernel_mul_f16; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); + } + + if (bcast_row) { + int n = ggml_nelements(dst)/4; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + } else { + unsigned int nth = MIN(64, ne0); + size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } +} + +static void ggml_cl_div(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - int cols = 1; - while (cols < op->ne[0]) { - cols *= 2; - } + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - return cols <= max_workgroup_size && op->src[0]->type == GGML_TYPE_F32; + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; + + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; + + const int ne0 = dst->ne[0]; + + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + bool bcast_row = false; + cl_kernel kernel; + + if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { + GGML_ASSERT(ggml_is_contiguous(src0)); + + // src1 is a row + GGML_ASSERT(ne11 == 1); + + bcast_row = true; + int ne = ne00 / 4; + + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_div_row; + } else { + kernel = backend_ctx->kernel_div_row_f16; } - case GGML_OP_SUM_ROWS: - case GGML_OP_CUMSUM: - return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); - case GGML_OP_MEAN: - return op->src[0]->type == GGML_TYPE_F32; - case GGML_OP_FLASH_ATTN_EXT: - { - const ggml_tensor * q = op->src[0]; - const ggml_tensor * k = op->src[1]; - const ggml_tensor * v = op->src[2]; - const int dk = q->ne[0]; - const int dv = v->ne[0]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_div; + } else { + kernel = backend_ctx->kernel_div_f16; + } - const struct { int dk; int dv; } supported_dims[] = { - { 40, 40}, { 64, 64}, { 80, 80}, { 96, 96}, - {112, 112}, {128, 128}, {192, 128}, - {192, 192}, {256, 256}, - }; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); + } - bool dims_supported = false; - for (size_t i = 0; i < sizeof(supported_dims)/sizeof(supported_dims[0]); ++i) { - if (supported_dims[i].dk == dk && supported_dims[i].dv == dv) { - dims_supported = true; - break; - } - } - if (!dims_supported) { - return false; - } + if (bcast_row) { + int n = ggml_nelements(dst)/4; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - const bool is_f32_f32 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 && - v->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; - const bool is_f16_f16 = q->type == GGML_TYPE_F16 && k->type == GGML_TYPE_F16 && - v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16; - const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && - v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + unsigned int nth = MIN(64, ne0); + size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {nth, 1, 1}; - return is_f32_f32 || is_f16_f16 || is_f32_f16; - } - default: - return false; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } } -// Forward declaration - implementation appears later in the file. -static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type); +static void ggml_cl_sub(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); -static ggml_guid_t ggml_backend_opencl_guid() { - static ggml_guid guid = { 0xde, 0xe0, 0x70, 0xa2, 0x73, 0x4e, 0x4d, 0xbc, 0xb0, 0xc7, 0x4f, 0xd4, 0x6d, 0x4e, 0x90, 0xfe }; - return &guid; -} + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); -static ggml_backend_i ggml_backend_opencl_i = { - /* .get_name = */ ggml_backend_opencl_name, - /* .free = */ ggml_backend_opencl_free, - /* .set_tensor_async = */ NULL, /* ggml_backend_opencl_set_tensor_async */ - /* .get_tensor_async = */ NULL, /* ggml_backend_opencl_get_tensor_async */ - /* .cpy_tensor_async = */ NULL, /* ggml_backend_opencl_cpy_tensor_async */ - /* .get_tensor_2d_async = */ NULL, - /* .set_tensor_2d_async = */ NULL, - /* .synchronize = */ ggml_backend_opencl_synchronize, - /* .graph_plan_create = */ NULL, - /* .graph_plan_free = */ NULL, - /* .graph_plan_update = */ NULL, - /* .graph_plan_compute = */ NULL, - /* .graph_compute = */ ggml_backend_opencl_graph_compute, - /* .event_record = */ NULL, - /* .event_wait = */ NULL, - /* .graph_optimize = */ NULL, -}; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; -ggml_backend_t ggml_backend_opencl_init(void) { - ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_opencl_reg(), 0); - ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(dev); + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - ggml_backend_t backend = new ggml_backend { - /* .guid = */ ggml_backend_opencl_guid(), - /* .iface = */ ggml_backend_opencl_i, - /* .device = */ dev, - /* .context = */ backend_ctx - }; + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; - return backend; -} + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; -bool ggml_backend_is_opencl(ggml_backend_t backend) { - return backend && backend->iface.get_name == ggml_backend_opencl_name; -} + const int ne0 = dst->ne[0]; -// -// buffer -// -struct ggml_backend_opencl_buffer_context { - // A buffer context can hold multiple cl_mem objects. This is for flattening - // quantized weights and should be used with GGML_OPENCL_SMALL_ALLOC where - // each tensor is allocated a separate buffer. When flattening is enabled - // with small allocation, each tensor is backed by two cl_mem objects (for - // quants and scales) packed into a backend_opencl_buffer. - ggml_backend_opencl_buffer_context(cl_mem buf) - : name("OpenCL") { - buffer.push_back(buf); - } + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; - ~ggml_backend_opencl_buffer_context() { - for (cl_mem buf : buffer) { - CL_CHECK(clReleaseMemObject(buf)); - } - for (cl_mem im : img) { - CL_CHECK(clReleaseMemObject(im)); - } + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - // Delete all extras to trigger their destructors - for (ggml_tensor_extra_cl * e : temp_tensor_extras) { - delete e; - } - for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0) { - delete e; - } - for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4) { - delete e; - } - for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0) { - delete e; - } - for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K) { - delete e; - } - for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K) { - delete e; - } - for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { - delete e; - } - for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { - delete e; - } - for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { - delete e; - } - } + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + bool bcast_row = false; + cl_kernel kernel; - ggml_tensor_extra_cl * ggml_opencl_alloc_temp_tensor_extra() { - ggml_tensor_extra_cl * extra; - if (temp_tensor_extras.empty()) { - extra = new ggml_tensor_extra_cl(); - } else { - extra = temp_tensor_extras.back(); - temp_tensor_extras.pop_back(); - } + if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { + GGML_ASSERT(ggml_is_contiguous(src0)); - temp_tensor_extras_in_use.push_back(extra); + // src1 is a row + GGML_ASSERT(ne11 == 1); - extra->reset(); - return extra; - } + bcast_row = true; + int ne = ne00 / 4; - ggml_tensor_extra_cl_q4_0 * ggml_opencl_alloc_temp_tensor_extra_q4_0() { - ggml_tensor_extra_cl_q4_0 * extra; - if (temp_tensor_extras_q4_0.empty()) { - extra = new ggml_tensor_extra_cl_q4_0(); + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sub_row; } else { - extra = temp_tensor_extras_q4_0.back(); - temp_tensor_extras_q4_0.pop_back(); + kernel = backend_ctx->kernel_sub_row_f16; } - temp_tensor_extras_q4_0_in_use.push_back(extra); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sub; + } else { + kernel = backend_ctx->kernel_sub_f16; + } - extra->reset(); - return extra; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); } - ggml_tensor_extra_cl_q4_1 * ggml_opencl_alloc_temp_tensor_extra_q4_1() { - ggml_tensor_extra_cl_q4_1 * extra; - if (temp_tensor_extras_q4_1.empty()) { - extra = new ggml_tensor_extra_cl_q4_1(); - } else { - extra = temp_tensor_extras_q4_1.back(); - temp_tensor_extras_q4_1.pop_back(); - } + if (bcast_row) { + int n = ggml_nelements(dst)/4; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - temp_tensor_extras_q4_1_in_use.push_back(extra); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + unsigned int nth = MIN(64, ne0); + size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {nth, 1, 1}; - extra->reset(); - return extra; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } +} - ggml_tensor_extra_cl_mxfp4 * ggml_opencl_alloc_temp_tensor_extra_mxfp4() { - ggml_tensor_extra_cl_mxfp4 * extra; - if (temp_tensor_extras_mxfp4.empty()) { - extra = new ggml_tensor_extra_cl_mxfp4(); - } else { - extra = temp_tensor_extras_mxfp4.back(); - temp_tensor_extras_mxfp4.pop_back(); - } +static void ggml_cl_sqr(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + UNUSED(src1); - temp_tensor_extras_mxfp4_in_use.push_back(extra); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - extra->reset(); - return extra; - } + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q8_0 * ggml_opencl_alloc_temp_tensor_extra_q8_0() { - ggml_tensor_extra_cl_q8_0 * extra; - if (temp_tensor_extras_q8_0.empty()) { - extra = new ggml_tensor_extra_cl_q8_0(); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + cl_kernel kernel; + + // Currently assumes src0 is contiguous + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sqr_cont_f32_4; } else { - extra = temp_tensor_extras_q8_0.back(); - temp_tensor_extras_q8_0.pop_back(); + kernel = backend_ctx->kernel_sqr_cont_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sqr_cont_f32; + } else { + kernel = backend_ctx->kernel_sqr_cont_f16; } + } - temp_tensor_extras_q8_0_in_use.push_back(extra); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - extra->reset(); - return extra; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } - ggml_tensor_extra_cl_q4_K * ggml_opencl_alloc_temp_tensor_extra_q4_K() { - ggml_tensor_extra_cl_q4_K * extra; - if (temp_tensor_extras_q4_K.empty()) { - extra = new ggml_tensor_extra_cl_q4_K(); - } else { - extra = temp_tensor_extras_q4_K.back(); - temp_tensor_extras_q4_K.pop_back(); - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); +} - temp_tensor_extras_q4_K_in_use.push_back(extra); +static void ggml_cl_sqrt(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + UNUSED(src1); - extra->reset(); - return extra; - } + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl_q5_K * ggml_opencl_alloc_temp_tensor_extra_q5_K() { - ggml_tensor_extra_cl_q5_K * extra; - if (temp_tensor_extras_q5_K.empty()) { - extra = new ggml_tensor_extra_cl_q5_K(); - } else { - extra = temp_tensor_extras_q5_K.back(); - temp_tensor_extras_q5_K.pop_back(); - } + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - temp_tensor_extras_q5_K_in_use.push_back(extra); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - extra->reset(); - return extra; - } + cl_kernel kernel; - ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() { - ggml_tensor_extra_cl_q6_K * extra; - if (temp_tensor_extras_q6_K.empty()) { - extra = new ggml_tensor_extra_cl_q6_K(); + // Currently assumes src0 is contiguous + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sqrt_cont_f32_4; } else { - extra = temp_tensor_extras_q6_K.back(); - temp_tensor_extras_q6_K.pop_back(); + kernel = backend_ctx->kernel_sqrt_cont_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sqrt_cont_f32; + } else { + kernel = backend_ctx->kernel_sqrt_cont_f16; } + } - temp_tensor_extras_q6_K_in_use.push_back(extra); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - extra->reset(); - return extra; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } - void reset() { - for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) { - temp_tensor_extras.push_back(e); - } - temp_tensor_extras_in_use.clear(); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); +} + +static void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_UNUSED(src1); + + GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; - for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) { - temp_tensor_extras_q4_0.push_back(e); - } - temp_tensor_extras_q4_0_in_use.clear(); + cl_kernel kernel; - for (ggml_tensor_extra_cl_q4_1 * e : temp_tensor_extras_q4_1_in_use) { - temp_tensor_extras_q4_1.push_back(e); - } - temp_tensor_extras_q4_1_in_use.clear(); + const bool is_c4 = ne00 % 4 == 0; + if (is_c4) { + kernel = backend_ctx->kernel_mean_f32_4; + } else { + kernel = backend_ctx->kernel_mean_f32; + } - for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) { - temp_tensor_extras_mxfp4.push_back(e); - } - temp_tensor_extras_mxfp4_in_use.clear(); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb3)); - for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) { - temp_tensor_extras_q8_0.push_back(e); - } - temp_tensor_extras_q8_0_in_use.clear(); + size_t global_work_size[] = {64 * (size_t)ne01, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)64, 1, 1}; - for (ggml_tensor_extra_cl_q4_K * e : temp_tensor_extras_q4_K_in_use) { - temp_tensor_extras_q4_K.push_back(e); - } - temp_tensor_extras_q4_K_in_use.clear(); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K_in_use) { - temp_tensor_extras_q5_K.push_back(e); +static void ggml_cl_ssm_scan(ggml_backend_t backend, ggml_tensor * dst) { + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(dst->src[0]); + GGML_ASSERT(dst->src[0]->extra); + GGML_ASSERT(dst->src[1]); + GGML_ASSERT(dst->src[1]->extra); + GGML_ASSERT(dst->src[2]); + GGML_ASSERT(dst->src[2]->extra); + GGML_ASSERT(dst->src[3]); + GGML_ASSERT(dst->src[3]->extra); + GGML_ASSERT(dst->src[4]); + GGML_ASSERT(dst->src[4]->extra); + GGML_ASSERT(dst->src[5]); + GGML_ASSERT(dst->src[5]->extra); + GGML_ASSERT(dst->src[6]); + GGML_ASSERT(dst->src[6]->extra); + + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) dst->src[0]->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) dst->src[1]->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) dst->src[2]->extra; + ggml_tensor_extra_cl * extra3 = (ggml_tensor_extra_cl *) dst->src[3]->extra; + ggml_tensor_extra_cl * extra4 = (ggml_tensor_extra_cl *) dst->src[4]->extra; + ggml_tensor_extra_cl * extra5 = (ggml_tensor_extra_cl *) dst->src[5]->extra; + ggml_tensor_extra_cl * extra6 = (ggml_tensor_extra_cl *) dst->src[6]->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + dst->src[0]->view_offs; + const cl_ulong offset1 = extra1->offset + dst->src[1]->view_offs; + const cl_ulong offset2 = extra2->offset + dst->src[2]->view_offs; + const cl_ulong offset3 = extra3->offset + dst->src[3]->view_offs; + const cl_ulong offset4 = extra4->offset + dst->src[4]->view_offs; + const cl_ulong offset5 = extra5->offset + dst->src[5]->view_offs; + const cl_ulong offset6 = extra6->offset + dst->src[6]->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + + const ggml_tensor * s = dst->src[0]; + const ggml_tensor * x = dst->src[1]; + const ggml_tensor * dt = dst->src[2]; + const ggml_tensor * A = dst->src[3]; + const ggml_tensor * B = dst->src[4]; + const ggml_tensor * C = dst->src[5]; + + const cl_ulong s_nb1 = s->nb[1]; + const cl_ulong s_nb2 = s->nb[2]; + const cl_ulong s_nb3 = s->nb[3]; + const cl_ulong x_nb1 = x->nb[1]; + const cl_ulong x_nb2 = x->nb[2]; + const cl_ulong x_nb3 = x->nb[3]; + const cl_ulong dt_nb1 = dt->nb[1]; + const cl_ulong dt_nb2 = dt->nb[2]; + const cl_ulong A_nb1 = A->nb[1]; + const cl_ulong B_nb1 = B->nb[1]; + const cl_ulong B_nb2 = B->nb[2]; + const cl_ulong B_nb3 = B->nb[3]; + const cl_ulong C_nb1 = C->nb[1]; + const cl_ulong C_nb2 = C->nb[2]; + const cl_ulong C_nb3 = C->nb[3]; + + const cl_uint A_ne0 = A->ne[0]; + const cl_uint d_state = s->ne[0]; + const cl_int head_dim = x->ne[0]; + const cl_int n_head = x->ne[1]; + const cl_int n_group = B->ne[1]; + const cl_int n_tokens = x->ne[2]; + const cl_uint n_seqs = x->ne[3]; + const cl_uint K = ggml_get_op_params_i32(dst, 0); + const cl_ulong s_off_bytes = (cl_ulong) ggml_nelements(x) * sizeof(float); + + cl_kernel kernel = backend_ctx->kernel_ssm_scan_f32; + size_t nth = d_state; + if (A_ne0 == 1 && K == 1) { + cl_kernel kernel_mamba2 = nullptr; + if (d_state == 128) { + kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d128; + } else if (d_state == 256) { + kernel_mamba2 = backend_ctx->kernel_ssm_scan_f32_mamba2_d256; } - temp_tensor_extras_q5_K_in_use.clear(); - - for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { - temp_tensor_extras_q6_K.push_back(e); + if (kernel_mamba2 != nullptr) { + kernel = kernel_mamba2; + nth = 64; } - temp_tensor_extras_q6_K_in_use.clear(); } - // Pools for extras. Available extras are in `temp_tensor_extras`. Extras - // being used are in `temp_tensor_extras_in_use`. At the first run, new - // extras get created and put in `in_use`. When the buffer is reset via - // the `reset` callback, all extras in `in_use` get moved to available extras - // for reuse. - std::vector temp_tensor_extras; - std::vector temp_tensor_extras_in_use; - std::vector temp_tensor_extras_q4_0; - std::vector temp_tensor_extras_q4_0_in_use; - std::vector temp_tensor_extras_q4_1; - std::vector temp_tensor_extras_q4_1_in_use; - std::vector temp_tensor_extras_mxfp4; - std::vector temp_tensor_extras_mxfp4_in_use; - std::vector temp_tensor_extras_q8_0; - std::vector temp_tensor_extras_q8_0_in_use; - std::vector temp_tensor_extras_q4_K; - std::vector temp_tensor_extras_q4_K_in_use; - std::vector temp_tensor_extras_q5_K; - std::vector temp_tensor_extras_q5_K_in_use; - std::vector temp_tensor_extras_q6_K; - std::vector temp_tensor_extras_q6_K_in_use; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra3->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extra4->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offset4)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), &extra5->data_device)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &offset5)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), &extra6->data_device)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &offset6)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &s_nb2)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &s_nb3)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &x_nb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &x_nb3)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &dt_nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &dt_nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &A_nb1)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &B_nb2)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &B_nb3)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(cl_ulong), &C_nb2)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &C_nb3)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &s_off_bytes)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_int), &head_dim)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_int), &n_group)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_int), &n_tokens)); + + if (kernel == backend_ctx->kernel_ssm_scan_f32) { + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &s_nb1)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &x_nb1)); + CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &B_nb1)); + CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &C_nb1)); + CL_CHECK(clSetKernelArg(kernel, 36, sizeof(cl_uint), &A_ne0)); + CL_CHECK(clSetKernelArg(kernel, 37, sizeof(cl_uint), &d_state)); + CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_uint), &n_seqs)); + CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_uint), &K)); + CL_CHECK(clSetKernelArg(kernel, 40, d_state * sizeof(float), nullptr)); + } - // The buffer_context is initially created by ggml_backend_buft_alloc_buffer - // before any tensor is initialized (at the beginning of alloc_tensor_range). - // Hence, there is always a buffer object in this vector. When each tensor is - // being initialized, this original buffer object will be released if both - // flattening and small allocation are enabled, and additional buffer - // objects will be created in init_tensor to represent flattened quantized - // weights. - std::vector buffer; - // These are image1d_buffer_t objects that wrap around the quants and scales. - // For Q4_0 quantization, there should be two of them - one for quants and - // one for scales. They should be populated only when flattening and small - // allocation are enabled. - std::vector img; - std::string name; -}; + size_t global_work_size[] = { + (size_t) head_dim * (size_t) n_head * nth, + (size_t) n_seqs, + }; + size_t local_work_size[] = { nth, 1 }; -static void ggml_backend_opencl_buffer_free_buffer(ggml_backend_buffer_t buffer) { - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - delete ctx; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } -static void * ggml_backend_opencl_buffer_get_base(ggml_backend_buffer_t buffer) { - ggml_backend_opencl_context * backend_ctx = ggml_cl2_init(buffer->buft->device); - return (void *) (uintptr_t) backend_ctx->alignment; -} +static void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); -static enum ggml_status ggml_backend_opencl_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_cl2_init(buffer->buft->device); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - if (tensor->view_src != nullptr) { - GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - ggml_tensor_extra_cl * view_extra = (ggml_tensor_extra_cl *) tensor->view_src->extra; - GGML_ASSERT(view_extra && "view_extra is nullptr?"); + int ne01 = src0->ne[1]; + cl_ulong nb00 = src0->nb[0]; + cl_ulong nb01 = src0->nb[1]; + cl_ulong nb02 = src0->nb[2]; - // Reuse extra of the parent tensor. The offset of this view tensor - // becomes `extra->offset + view_offs` and needs to be calculated when - // it is used. This changes is needed because of the change to - // ggml_alloc.c in https://github.com/ggml-org/llama.cpp/pull/7640. - // `buffer` passed in here will always be `tensor->buffer`. It is OK - // to allocate extras from the same buffer context for ordinary - // intermediate tensors. But for views into kv cache tensors, doing so - // would mess up the extras used by kv cache. - // Before #7640, `buffer` is for intermediate tensors, which is always - // different from that of kv cache tensors. - // - // NB: now extra->offset no longer accounts for view_offs. - // NB: this should not apply to weight tensors (for end-to-end runs, but - // may apply for test-backend-ops). - // FIXME: if any unexpected results are seen, double check the offset - - // there could be other places that need fix. - tensor->extra = view_extra; - } else { - { - size_t offset = (char *) tensor->data - (char *) ggml_backend_opencl_buffer_get_base(buffer); + int ne10 = src1->ne[0]; + cl_ulong nb11 = src1->nb[1]; - ggml_tensor_extra_cl * extra = ctx->ggml_opencl_alloc_temp_tensor_extra(); - extra->offset = offset; - extra->data_device = ctx->buffer[0]; - extra->actual_size = ggml_nbytes(tensor); + int ne1 = dst->ne[1]; + int ne2 = dst->ne[2]; + cl_ulong nb0 = dst->nb[0]; + cl_ulong nb1 = dst->nb[1]; + cl_ulong nb2 = dst->nb[2]; - tensor->extra = extra; - } + cl_kernel kernel = backend_ctx->kernel_ssm_conv_f32_f32; + + if (ne10 % 4 == 0) { + kernel = backend_ctx->kernel_ssm_conv_f32_f32_4; } - return GGML_STATUS_SUCCESS; -} -// The optimized gemm and gemv kernels are used for large matrices without batch. -// tensor is the quantized weights matrix. -inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - int64_t threshold_ne0 = 512; - int64_t threshold_ne1 = 512; - if (!backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 38, 11, 0) && - backend_ctx->adreno_cl_compiler_version.type != DX) { - threshold_ne0 = 128; - threshold_ne1 = 128; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2)); + + size_t global_work_size[] = {(size_t)ne01, (size_t)ne1, (size_t)ne2}; + size_t local_work_size[] = {64, 1, 1}; + + size_t * local_work_size_ptr = local_work_size; + if (ne01 % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } - return tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && - tensor->ne[2] == 1 && tensor->ne[3] == 1; -} -inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - GGML_UNUSED(backend_ctx); - int ne01 = tensor->ne[1]; - return ((strstr(tensor->name, "ffn") != NULL) || (strstr(tensor->name, "as") != NULL)) && (ne01 % 64 == 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { - - bool adreno_kernel = use_adreno_kernels(backend_ctx, tensor); +static void ggml_cl_gelu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - size_t elem_num = tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]; + UNUSED(src1); - return ((elem_num < 128 * 1024 * 1024) && adreno_kernel); // max element num: 2**27 -} + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; -static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { - ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(buffer->buft->device); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_context context = backend_ctx->context; - cl_command_queue queue = backend_ctx->queue; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; -#ifdef GGML_OPENCL_SOA_Q - // We separate the quantized bits and scale from block_q4_0 by using an - // additional kernel, where each thread handles a block. We first read the - // original weights into a temporary buffer, then create two separate - // buffers for quantized bits and scales, which are then populated by the - // conversion kernel. - if (tensor->type == GGML_TYPE_Q4_0) { - // Tensors should have been preallocated, therefore they should - // already have ggml_tensor_extra_cl as extra. - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + cl_kernel kernel; - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q4_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_0(); + int n = ggml_nelements(dst); - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + if (n % 4 == 0) { + kernel = backend_ctx->kernel_gelu_4; + n /= 4; + } else { + kernel = backend_ctx->kernel_gelu; + } - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - // We consider the specified offset arg as always, although For weights - // the offset arg should be 0 (we do not assert this). - //GGML_ASSERT(offset == 0); + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - // We create subbuffers from the original tensor buffer for scales and - // quants - i.e., scales and quants are aliases into the buffer object - // that backs the original tensor. This is a cleaner way to adapt to the - // new memory management. - // In the old code, we allocate new buffers for scales and quants - // respectively, which could still be done but would result in double - // allocation; properly deallocating the preallocated buffer that backs - // the tensors is tricky and would leak the backend specific information - // into the general backend code. - // Does this create misaligned subbuffers (alignment is 1024) in certain - // cases ? - cl_buffer_region region; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - // The original tensor memory is divided into scales and quants, i.e., - // we first store scales, then quants. - // Create subbuffer for scales. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; +static void ggml_cl_gelu_erf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + UNUSED(src1); - //cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; - #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - // The optimized kernels need weights in natural order, so unshuffle. - if (use_adreno_kernels(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q4_0_noshuffle; - } - #else - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_0; - #endif // GGML_OPENCL_USE_ADRENO_KERNELS - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + cl_kernel kernel; - tensor->extra = extra; + int n = ggml_nelements(dst); - // transpose the weights and scales - #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // Only do transpose for large, non batched matrix - // TODO: use preallocated images instead of sub-buffer then image - if (use_adreno_kernels(backend_ctx, tensor)) { - // <----------------------------------------------------------------------------------> // - // start transpose - // <----------------------------------------------------------------------------------> // - int M = tensor->ne[1]; // ne01 - int K = tensor->ne[0]; // ne00 + if (n % 4 == 0) { + kernel = backend_ctx->kernel_gelu_erf_4; + n /= 4; + } else { + kernel = backend_ctx->kernel_gelu_erf; + } - //For matrix-vector multiplication kernel, we assume K is a multiple of 32 - GGML_ASSERT(K % 32 == 0); - //For transpose kernels, we assume K is a multiple of 4 (satisfied by prior assert), and M is a multiple of 4 - GGML_ASSERT(M % 4 == 0); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - // transpose is out of place, so we need to allocate transposed buffers - // <----------------------------------------------------------------------------------> // - // use sub_buffer of max buffer size instead + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - size_t q_size_bytes = K * M / 8 * sizeof(float); - backend_ctx->prealloc_quant_trans.allocate(context, q_size_bytes); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - cl_buffer_region region; - region.origin = 0; - region.size = q_size_bytes; - cl_mem qT_d = clCreateSubBuffer( - backend_ctx->prealloc_quant_trans.buffer, - 0, - CL_BUFFER_CREATE_TYPE_REGION, - ®ion, - &err); - CL_CHECK(err); +static void ggml_cl_gelu_quick(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - bool K_tile_trans = true; - if ((K / 32) % 4 != 0){ - K_tile_trans =false; - } + UNUSED(src1); - size_t d_size_bytes = M * (K / 32) * 2; - backend_ctx->prealloc_scales_trans.allocate(context, d_size_bytes); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - region.origin = 0; - region.size = d_size_bytes; - cl_mem dT_d = clCreateSubBuffer( - backend_ctx->prealloc_scales_trans.buffer, - 0, - CL_BUFFER_CREATE_TYPE_REGION, - ®ion, - &err); - CL_CHECK(err); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - // <----------------------------------------------------------------------------------> // + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_kernel kernel; - // create images from the buffers - // <----------------------------------------------------------------------------------> // - cl_mem q_d_image1D; - cl_mem d_d_image1D; - cl_mem qT_d_image1D; - cl_mem dT_d_image1D; + int n = ggml_nelements(dst); - cl_image_format img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT }; - cl_image_desc img_desc_1d; + if (n % 4 == 0) { + kernel = backend_ctx->kernel_gelu_quick_4; + n /= 4; + } else { + kernel = backend_ctx->kernel_gelu_quick; + } - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = M * K / 4 / 4; - img_desc_1d.buffer = extra->q; - q_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err); - CL_CHECK(err); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT }; - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = M * K / 4 / 4; - img_desc_1d.buffer = qT_d; - qT_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err); - CL_CHECK(err); + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - if (K_tile_trans) { - img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT }; - img_desc_1d.image_width = M * K / 32 / 4; - } else { - img_fmt_1d = { CL_R, CL_HALF_FLOAT }; - img_desc_1d.image_width = M * K / 32; - } - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.buffer = extra->d; - d_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err); - CL_CHECK(err); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT }; - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = M * K / 32 / 4; - img_desc_1d.buffer = dT_d; - dT_d_image1D = clCreateImage(context, 0, &img_fmt_1d, &img_desc_1d, NULL, &err); - CL_CHECK(err); - // <----------------------------------------------------------------------------------> // - - // set up and call the transpose kernels - // <----------------------------------------------------------------------------------> // - // weights - int height_q = M / 4; - int width_q = K / 4 / 4; - kernel = backend_ctx->kernel_transpose_16; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_d_image1D)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qT_d_image1D)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_q)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_q)); - - size_t local_size_q[3] = {4, 16, 1}; - size_t global_size_q[3] = {static_cast(width_q), static_cast(height_q), 1}; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_size_q, local_size_q, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); +static void ggml_cl_silu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - // scales - int height_s = M / 4; - int width_s = K / 32 / 4; + UNUSED(src1); - kernel = backend_ctx->kernel_transpose_16; - if (!K_tile_trans) { - kernel = backend_ctx->kernel_transpose_16_4x1; - width_s = K / 32; - } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &d_d_image1D)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &dT_d_image1D)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_s)); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - size_t local_size_s[3] = {4, 16, 1}; - size_t global_size_s[3] = {static_cast(width_s), static_cast(height_s), 1}; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_size_s, local_size_s, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - // <----------------------------------------------------------------------------------> // + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - // copy transposed buffer contents to original buffers - // <----------------------------------------------------------------------------------> // - // weights - CL_CHECK(clEnqueueCopyBuffer(queue, qT_d, extra->q, 0, 0, q_size_bytes, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // scales - CL_CHECK(clEnqueueCopyBuffer(queue, dT_d, extra->d, 0, 0, d_size_bytes, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - // <----------------------------------------------------------------------------------> // + cl_kernel kernel; - // deallocate transpose buffers - // <----------------------------------------------------------------------------------> // - CL_CHECK(clReleaseMemObject(qT_d)); - CL_CHECK(clReleaseMemObject(dT_d)); + int n = ggml_nelements(dst); - // deallocate temporary images - CL_CHECK(clReleaseMemObject(q_d_image1D)); - CL_CHECK(clReleaseMemObject(d_d_image1D)); - CL_CHECK(clReleaseMemObject(qT_d_image1D)); - CL_CHECK(clReleaseMemObject(dT_d_image1D)); - // <----------------------------------------------------------------------------------> // - // end transpose - // <----------------------------------------------------------------------------------> // - } - #endif // GGML_OPENCL_USE_ADRENO_KERNELS + if (n % 4 == 0) { + kernel = backend_ctx->kernel_silu_4; + n /= 4; + } else { + kernel = backend_ctx->kernel_silu; + } - return; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. } - if (tensor->type == GGML_TYPE_Q4_1) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q4_1 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_1(); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); +} - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_m = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - GGML_ASSERT(size_d + size_m + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); +static void ggml_cl_relu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + UNUSED(src1); - cl_buffer_region region; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - // The original tensor memory is divided into scales and quants, i.e., - // we first store scales, mins, then quants. - // Create subbuffer for scales. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - // Create subbuffer for mins. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_m; - extra->m = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_m, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + cl_kernel kernel = backend_ctx->kernel_relu; - #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - if (use_adreno_kernels(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q4_1_noshuffle; - } - #else - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_1; - #endif // GGML_OPENCL_USE_ADRENO_KERNELS - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->m)); + const int64_t n = ggml_nelements(dst); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); +} + +static void ggml_cl_sigmoid(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - tensor->extra = extra; + UNUSED(src1); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - int M = tensor->ne[1]; - int K = tensor->ne[0]; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - GGML_ASSERT(K % 32 == 0); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // Transpose q as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); - // Transpose m as ushort - transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M); - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - return; + cl_kernel kernel; + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_sigmoid_f32; + } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { + kernel = backend_ctx->kernel_sigmoid_f16; + } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { + // bf16 converted to f16 + kernel = backend_ctx->kernel_sigmoid_f16; + } else { + GGML_ASSERT(false && "Unsupported data types for sigmoid (input and output must be both f32 or f16)"); } - if (tensor->type == GGML_TYPE_MXFP4) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_mxfp4 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_mxfp4(); - - size_t size_e = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(char); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - // The original tensor memory is divided into scales and quants, i.e., - // we first store scales, then quants. - cl_buffer_region region; + const int64_t n = ggml_nelements(dst); - // Create subbuffer for scales. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_e; - extra->e = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_e, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. + } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4_trans; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); +} - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); +static void ggml_cl_tri(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + UNUSED(src1); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); - tensor->extra = extra; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - return; - } -#endif - cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e)); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - size_t global_work_size[3] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[3] = {64, 1, 1}; + const int tri_type = ggml_get_op_params_i32(dst, 0); + const int64_t n = ggml_nelements(dst); + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + cl_kernel kernel = backend_ctx->kernel_tri; - // Create image for Q - cl_image_format img_format_q = {CL_RG, CL_UNSIGNED_INT32}; - cl_image_desc img_desc_q = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast(ggml_nelements(tensor)/32*2), - 0, 0, 0, 0, 0, 0, 0, - { extra->q } - }; - extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err); - tensor->extra = extra; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &tri_type)); - return; - } - if (tensor->type == GGML_TYPE_Q8_0) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + size_t local_work_size[1] = { 256 }; + size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q8_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q8_0(); + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); +} - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)*sizeof(char)); - GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); +static void ggml_cl_fill(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + UNUSED(src0); + UNUSED(src1); - // The original tensor memory is divided into scales and quants, i.e., - // we first store scales, then quants. - cl_buffer_region region; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - // Create subbuffer for scales. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + float v = 0.0f; + memcpy(&v, ((int32_t *) dst->op_params), sizeof(float)); - cl_kernel kernel = backend_ctx->kernel_convert_block_q8_0; + const int64_t n = ggml_nelements(dst); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); + cl_kernel kernel = backend_ctx->kernel_fill; - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(float), &v)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &n)); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + size_t local_work_size[1] = { 256 }; + size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; - tensor->extra = extra; + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); +} - // Transpose the weights and scales -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (enable_adreno_trans_weight(backend_ctx, tensor)) { +static void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - int M = tensor->ne[1]; // ne01 - int K = tensor->ne[0]; // ne00 + UNUSED(src1); - GGML_ASSERT(K % 32 == 0); - GGML_ASSERT(M % 4 == 0); - GGML_ASSERT(tensor->ne[2] == 1); - GGML_ASSERT(tensor->ne[3] == 1); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M); - } // end transpose -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - return; - } - if (tensor->type == GGML_TYPE_Q4_K) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q4_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q4_K(); + float min; + float max; + memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float)); + memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float)); - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3 * ggml_blck_size(tensor->type) / 64); - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - GGML_ASSERT(size_d + size_dm + size_s + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + if (!ggml_is_contiguous(src0)) { + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + cl_kernel kernel = backend_ctx->kernel_clamp_nc; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); - CL_CHECK(clEnqueueWriteBuffer( - queue, data_device, CL_TRUE, 0, - ggml_nbytes(tensor), data, 0, NULL, NULL)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &min)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &max)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb3)); + + const int nth = 64; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - cl_buffer_region region; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } - // Create subbuffer for d. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; + cl_kernel kernel = backend_ctx->kernel_clamp; - // Create subbuffer for mins. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_dm; - extra->dm = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &min)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &max)); - // Create subbuffer for s. - region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); - region.size = size_s; - extra->s = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; + const int64_t n = ggml_nelements(dst); - // Create subbuffer for quants. - region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; - if (use_adreno_kernels(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q4_K_noshuffle; - } - #else - cl_kernel kernel = backend_ctx->kernel_convert_block_q4_K; - #endif + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. + } - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); +} - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); +static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + UNUSED(src1); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - tensor->extra = extra; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - int M = tensor->ne[1]; - int K = tensor->ne[0]; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - GGML_ASSERT(K % 32 == 0); + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); - // Transpose q, d, dm as ushort - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); - transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - return; - } - if (tensor->type == GGML_TYPE_Q5_K) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q5_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q5_K(); + int nth = 1; + while (nth < ne00 && nth < 64) { + nth *= 2; + } - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/8; - size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(3*ggml_blck_size(tensor->type)/64); - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - GGML_ASSERT(size_q + size_qh + size_s + size_d + size_dm == ggml_nbytes(tensor) && - "Incorrect tensor size"); + cl_kernel kernel = backend_ctx->kernel_norm; - cl_int err; - cl_mem data_device; - CL_CHECK((data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err), err)); - CL_CHECK(clEnqueueWriteBuffer(queue, data_device, CL_TRUE, 0, ggml_nbytes(tensor), data, 0, NULL, NULL)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float)*nth, NULL)); - cl_buffer_region region; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - // Create subbuffer for d. - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_d; - extra->d = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - auto previous_origin = region.origin; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - // Create subbuffer for dm. - region.origin = align_to(previous_origin + size_d, backend_ctx->alignment); - region.size = size_dm; - extra->dm = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; +static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - // Create subbuffer for s. - region.origin = align_to(previous_origin + size_dm, backend_ctx->alignment); - region.size = size_s; - extra->s = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; + UNUSED(src1); - // Create subbuffer for q (lower 4 bits) - region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); - region.size = size_q; - extra->q = clCreateSubBuffer( - extra_orig->data_device, CL_MEM_READ_WRITE, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - CL_CHECK(err); - previous_origin = region.origin; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - // Create subbuffer for qh (upper 1 bit) - region.origin = align_to(previous_origin + size_q, backend_ctx->alignment); - region.size = size_qh; - CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - CL_CHECK(err); + //ggml_backend_opencl_device_context * dev_ctx = + // (ggml_backend_opencl_device_context *)backend->device->context; - #ifdef GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; - if (use_adreno_kernels(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle; - } - #else - cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; - #endif + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + const int ne00 = src0 ? src0->ne[0] : 0; + const int ne01 = src0 ? src0->ne[1] : 0; + const int ne02 = src0 ? src0->ne[2] : 0; + const int ne03 = src0 ? src0->ne[3] : 0; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + const cl_ulong nb01 = src0 ? src0->nb[1] : 0; + const cl_ulong nb02 = src0 ? src0->nb[2] : 0; + const cl_ulong nb03 = src0 ? src0->nb[3] : 0; - extra->size_q = size_q; - extra->size_qh = size_qh; - extra->size_s = size_s; - extra->size_d = size_d; - extra->size_dm = size_dm; + GGML_ASSERT(ne00 % 4 == 0); - tensor->extra = extra; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { + const int nth = 64; - int M = tensor->ne[1]; - int K = tensor->ne[0]; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - GGML_ASSERT(K % 32 == 0); + cl_kernel kernel = backend_ctx->kernel_rms_norm; - // Transpose q, d, dm as ushort, qh as uchar - transpose_2d_as_16b(backend_ctx, extra->q, extra->q, size_q, K/4, M); - transpose_2d_as_8b (backend_ctx, extra->qh, extra->qh, size_qh, K/8, M); - transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/256, M); - transpose_2d_as_16b(backend_ctx, extra->dm, extra->dm, size_dm, K/256, M); - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - return; + // Note, this kernel declares local memory in kernel args and the size + // depends on subgroup size. + // Note, this requires OpenCL 2.1 and above + // For now we use fixed subgroup size to simplify support for OpenCL 2.0. + size_t sgs; + //CL_CHECK(clGetKernelSubGroupInfo(kernel, dev_ctx->device, + // CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, + // sizeof(local_work_size), local_work_size, + // sizeof(size_t), &sgs, NULL)); + if (backend_ctx->gpu_family == ADRENO) { + sgs = 64; + } else if (backend_ctx->gpu_family == INTEL) { + sgs = 32; + } else { + GGML_ASSERT(false && "Unsupported GPU"); } - if (tensor->type == GGML_TYPE_Q6_K) { - ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; - GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); - - // Allocate the new extra and create aliases from the original. - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ggml_tensor_extra_cl_q6_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q6_K(); - size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; - size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && - "Incorrect tensor size"); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); + // This is local memory - the size depends on subgroup size. + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); - cl_int err; - cl_mem data_device; - CL_CHECK((data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err), err)); - CL_CHECK(clEnqueueWriteBuffer(queue, data_device, CL_TRUE, 0, ggml_nbytes(tensor), data, 0, NULL, NULL)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - cl_buffer_region region; +static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor) { + GGML_ASSERT(mul_tensor); + GGML_ASSERT(rms_norm_tensor); - // Subbuffer for ql - region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); - region.size = size_ql; - CL_CHECK((extra->ql = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - auto previous_origin = region.origin; + // src0 is the src of rms_norm, src1 is the other src of mul (one being rms_norm) + const ggml_tensor * src0 = rms_norm_tensor->src[0]; + const ggml_tensor * src1; + if (mul_tensor->src[0] == rms_norm_tensor) { + src1 = mul_tensor->src[1]; + } else if (mul_tensor->src[1] == rms_norm_tensor) { + src1 = mul_tensor->src[0]; + } else { + GGML_ASSERT(false && "Invalid args for rms_norm and mul"); + } + const ggml_tensor * dst = mul_tensor; - // Subbuffer for qh - region.origin = align_to(previous_origin + size_ql, backend_ctx->alignment); - region.size = size_qh; - CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - previous_origin = region.origin; + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - // Subbuffer for scales - region.origin = align_to(previous_origin + size_qh, backend_ctx->alignment); - region.size = size_s; - CL_CHECK((extra->s = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - previous_origin = region.origin; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - // Create subbuffer for d. - region.origin = align_to(previous_origin + size_s, backend_ctx->alignment); - region.size = size_d; - CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - previous_origin = region.origin; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // Flatten the weights - cl_kernel kernel; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - kernel = backend_ctx->kernel_convert_block_q6_K; - if (use_adreno_kernels(backend_ctx, tensor)) { - kernel = backend_ctx->kernel_convert_block_q6_K_noshuffle; - } -#else - kernel = backend_ctx->kernel_convert_block_q6_K; -#endif // GGML_OPENCL_USE_ADRENO_KERNELS + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - cl_uchar mask = 0xff; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->ql)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + float eps; + memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); - size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64)*64, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clReleaseMemObject(data_device)); + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - extra->size_ql = size_ql; - extra->size_qh = size_qh; - extra->size_s = size_s; - extra->size_d = size_d; + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; - tensor->extra = extra; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - cl_int M = tensor->ne[1]; // ne01 - cl_int K = tensor->ne[0]; // ne00 + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; - // Transpose ql as ushort - transpose_2d_as_16b(backend_ctx, - extra->ql, extra->ql, size_ql, K/4, M); + GGML_ASSERT(ne00 % 4 == 0); - // Transpose qh as uchar - transpose_2d_as_8b(backend_ctx, - extra->qh, extra->qh, size_qh, K/4, M); + size_t sgs; + if (backend_ctx->gpu_family == ADRENO) { + sgs = 64; + } else if (backend_ctx->gpu_family == INTEL) { + sgs = 32; + } else { + GGML_ASSERT(false && "Unsupported GPU"); + } - // Transpose s as ushort - transpose_2d_as_16b(backend_ctx, - extra->s, extra->s, size_s, K/16/2, M); + cl_kernel kernel = backend_ctx->kernel_rms_norm_mul; - // Transpose d as ushort - transpose_2d_as_16b(backend_ctx, - extra->d, extra->d, size_d, K/256, M); - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - return; + int nth = sgs; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (nth < ne00 && nth < max_workgroup_size) { + nth *= 2; } -#endif // GGML_OPENCL_SOA_Q + nth = MIN(nth, max_workgroup_size); + nth = MIN(nth, ne00); - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra); + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - CL_CHECK(clEnqueueWriteBuffer( - queue, extra->data_device, CL_TRUE, extra->offset + offset, - size, data, 0, NULL, NULL)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float)*sgs, NULL)); - GGML_UNUSED(buffer); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { - GGML_ASSERT(tensor->extra); +static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { + GGML_ASSERT(norm_tensor && mul_tensor && add_tensor); - ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(buffer->buft->device); + const ggml_tensor * src0 = norm_tensor->src[0]; + const ggml_tensor * src1 = mul_tensor->src[0] == norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; + const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; + const ggml_tensor * dst = add_tensor; - cl_context context = backend_ctx->context; - cl_command_queue queue = backend_ctx->queue; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - // Make sure all previously submitted commands in other devices are finished. - sync_with_other_backends(backend_ctx); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; -#ifdef GGML_OPENCL_SOA_Q - // In end-to-end runs, get_tensor is usually used to get back the logits, - // where we can simply do clEnqueueReadBuffer since they are f32. - // However, in test-backend-ops, the GPU graph is copied to the CPU backend, - // which requires reading back quantized weight tensors. - // To properly support this, we need to restore block_q4_0 struct arrays - // from the flattened buffers. - if (tensor->type == GGML_TYPE_Q4_0) { - ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *)tensor->extra; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - cl_int err; - cl_kernel kernel; + float eps; + memcpy(&eps, norm_tensor->op_params, sizeof(float)); - cl_int M = tensor->ne[1]; // ne01 - cl_int K = tensor->ne[0]; // ne00 + const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; + const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; + const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; + const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; + const cl_ulong nbd1 = dst->nb[1], nbd2 = dst->nb[2], nbd3 = dst->nb[3]; - GGML_ASSERT(K % 32 == 0); - GGML_ASSERT(M % 4 == 0); + size_t sgs; + if (backend_ctx->gpu_family == ADRENO) sgs = 64; + else if (backend_ctx->gpu_family == INTEL) sgs = 32; + else GGML_ASSERT(false && "Unsupported GPU"); - size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; - size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); - GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size"); + cl_kernel kernel = backend_ctx->kernel_norm_mul_add; + + int nth = sgs; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (nth < ne00/4 && nth < max_workgroup_size) nth *= 2; + nth = MIN(nth, max_workgroup_size); + nth = MIN(nth, ne00/4); - cl_mem buf_trans_q; - cl_mem buf_trans_d; + size_t gws[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t lws[] = {(size_t)nth, 1, 1}; + size_t num_subgroups = (nth + sgs - 1) / sgs; - CL_CHECK((buf_trans_q = clCreateBuffer(context, CL_MEM_READ_WRITE, - size_q, NULL, &err), err)); - CL_CHECK((buf_trans_d = clCreateBuffer(context, CL_MEM_READ_WRITE, - size_d, NULL, &err), err)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nbd1)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nbd2)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nbd3)); + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_float2) * num_subgroups, NULL)); - kernel = backend_ctx->kernel_transpose_16_buf; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); +} - // transpose q back - cl_int stride_k_q = K/4; - size_t local_size_q[3] = {64, 1, 1}; - size_t global_size_q[3] = {(size_t)M, (size_t)stride_k_q, 1}; +static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { + GGML_ASSERT(gn_tensor && mul_tensor && add_tensor); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &M)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &stride_k_q)); + const ggml_tensor * src0 = gn_tensor->src[0]; + const ggml_tensor * src1 = mul_tensor->src[0] == gn_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; + const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; + const ggml_tensor * dst = add_tensor; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_size_q, local_size_q, 0, NULL, NULL)); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - // transpose scales back - cl_int stride_k_d = K/32; - size_t local_size_d[3] = {64, 1, 1}; - size_t global_size_d[3] = {(size_t)M, (size_t)stride_k_d, 1}; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_int), &M)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &stride_k_d)); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_size_d, local_size_d, 0, NULL, NULL)); + int groups; + float eps; + memcpy(&groups, gn_tensor->op_params, sizeof(int)); + memcpy(&eps, (char *)gn_tensor->op_params + sizeof(int), sizeof(float)); - // unpack - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_kernel kernel = backend_ctx->kernel_group_norm_mul_add; + int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + int ne = ggml_nelements(src0); + int group_size = ne / groups; - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + size_t lws[] = { (size_t)MIN(max_workgroup_size, group_size) }; + size_t gws[] = { (size_t)groups * lws[0] }; - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &group_size)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(float), &eps)); - kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + backend_ctx->enqueue_ndrange_kernel(kernel, 1, gws, lws, dst); +} - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, NULL)); +static void ggml_cl_group_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - // read back to host - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); + UNUSED(src1); - CL_CHECK(clReleaseMemObject(data_device)); - CL_CHECK(clReleaseMemObject(buf_trans_q)); - CL_CHECK(clReleaseMemObject(buf_trans_d)); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - return; - } -#endif + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + int32_t n_groups = ((const int32_t *) dst->op_params)[0]; + int32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + n_groups - 1) / n_groups); + float eps = ((const float *) dst->op_params)[1]; - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne = ne00*ne01*ne02; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } - if (tensor->type == GGML_TYPE_Q4_1) { - ggml_tensor_extra_cl_q4_1 * extra = (ggml_tensor_extra_cl_q4_1 *)tensor->extra; + cl_kernel kernel = backend_ctx->kernel_group_norm; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - static ggml_cl_buffer buf_trans_q; - static ggml_cl_buffer buf_trans_m; - static ggml_cl_buffer buf_trans_d; - static ggml_cl_buffer buf_unpacked; + size_t sgs = 64; + if (backend_ctx->gpu_family == ADRENO) { + sgs = 64; + } else if (backend_ctx->gpu_family == INTEL) { + sgs = 32; + } else { + GGML_ASSERT(false && "Unsupported GPU"); + } - cl_int M = tensor->ne[1]; - cl_int K = tensor->ne[0]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &group_size)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(float), &eps)); - GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); + size_t global_work_size[] = {(size_t)n_groups*sgs, 1, 1}; + size_t local_work_size[] = {(size_t)sgs, 1, 1}; - size_t size_q = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2; - size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); - size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t); - GGML_ASSERT(size_d + size_q + size_m == ggml_nbytes(tensor) && "Incorrect tensor size"); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - buf_trans_q.allocate(backend_ctx->context, size_q); - buf_trans_m.allocate(backend_ctx->context, size_m); - buf_trans_d.allocate(backend_ctx->context, size_d); - buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); +static void ggml_cl_l2_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - // transpose q, d, m back - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32); - transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32); + UNUSED(src1); - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_m.buffer)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0)); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); - return; - } -#endif + float eps; + memcpy(&eps, dst->op_params, sizeof(float)); - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &data_device)); + size_t sgs; + if (backend_ctx->gpu_family == ADRENO) { + sgs = 64; + } else if (backend_ctx->gpu_family == INTEL) { + sgs = 32; + } else { + GGML_ASSERT(false && "Unsupported GPU"); + } - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + cl_kernel kernel = backend_ctx->kernel_l2_norm_f32; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; + int nth = sgs; + while (nth < ne00 && nth < (int)backend_ctx->get_kernel_workgroup_size(kernel)) { + nth *= 2; } - if (tensor->type == GGML_TYPE_MXFP4) { - ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_moe_kernels(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4_trans; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - int ne02 = tensor->ne[2]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); +static void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), static_cast(ne00 / 32), static_cast(ne02)}; - size_t local_work_size[3] = {64, 2, 1}; + UNUSED(src1); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } -#endif - cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; - } - if (tensor->type == GGML_TYPE_Q8_0) { - ggml_tensor_extra_cl_q8_0 * extra = (ggml_tensor_extra_cl_q8_0 *)tensor->extra; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (enable_adreno_trans_weight(backend_ctx, tensor)) { - cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - int ne00 = tensor->ne[0]; - int ne01 = tensor->ne[1]; - GGML_ASSERT(tensor->ne[2] == 1); - GGML_ASSERT(tensor->ne[3] == 1); + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); + cl_kernel kernel; - size_t global_work_size[3] = {static_cast(((ne01 + 63) / 64) * 64), 1, 1}; - size_t local_work_size[3] = {64, 1, 1}; + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_tanh_f32_4; + } else { + kernel = backend_ctx->kernel_tanh_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_tanh_f32; + } else { + kernel = backend_ctx->kernel_tanh_f16; + } + } - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } -#endif - cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + } else { + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_tanh_f32_nc; + } else { + kernel = backend_ctx->kernel_tanh_f16_nc; + } - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + + int nth = 64; + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } - if (tensor->type == GGML_TYPE_Q4_K) { - ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; +} - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); +static void ggml_cl_neg(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + UNUSED(src1); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - int M = tensor->ne[1]; - int K = tensor->ne[0]; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - size_t size_dm = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - static ggml_cl_buffer buf_trans_q; - static ggml_cl_buffer buf_trans_d; - static ggml_cl_buffer buf_trans_dm; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - buf_trans_q.allocate(backend_ctx->context, size_q); - buf_trans_d.allocate(backend_ctx->context, size_d); - buf_trans_dm.allocate(backend_ctx->context, size_dm); + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); - // Transpose q, d, dm back - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); - transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); + cl_kernel kernel; - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_dm.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_neg_f32_4; + } else { + kernel = backend_ctx->kernel_neg_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_neg_f32; + } else { + kernel = backend_ctx->kernel_neg_f16; + } + } - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; + size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_neg_f32_nc; + } else { + kernel = backend_ctx->kernel_neg_f16_nc; } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_restore_block_q4_K; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + int nth = 64; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } - if (tensor->type == GGML_TYPE_Q5_K) { - ggml_tensor_extra_cl_q5_K * extra = (ggml_tensor_extra_cl_q5_K *)tensor->extra; +} - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); +static void ggml_cl_exp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - cl_uchar mask_0F = 0x0F; - cl_uchar mask_F0 = 0xF0; + UNUSED(src1); -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - int M = tensor->ne[1]; - int K = tensor->ne[0]; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - size_t size_q = extra->size_q; - size_t size_qh = extra->size_qh; - size_t size_d = extra->size_d; - size_t size_dm = extra->size_dm; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - static ggml_cl_buffer buf_trans_q; - static ggml_cl_buffer buf_trans_qh; - static ggml_cl_buffer buf_trans_d; - static ggml_cl_buffer buf_trans_dm; + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - buf_trans_q.allocate(backend_ctx->context, size_q); - buf_trans_qh.allocate(backend_ctx->context, size_qh); - buf_trans_d.allocate(backend_ctx->context, size_d); - buf_trans_dm.allocate(backend_ctx->context, size_dm); + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); - // Reverse transpose q, qh, d, dm - transpose_2d_as_16b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/4); - transpose_2d_as_8b (backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); - transpose_2d_as_16b(backend_ctx, extra->dm, buf_trans_dm.buffer, size_dm, M, K/256); + cl_kernel kernel; - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_trans_dm.buffer)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_exp_f32_4; + } else { + kernel = backend_ctx->kernel_exp_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_exp_f32; + } else { + kernel = backend_ctx->kernel_exp_f16; + } + } - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; + size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_exp_f32_nc; + } else { + kernel = backend_ctx->kernel_exp_f16_nc; } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_kernel kernel = backend_ctx->kernel_restore_block_q5_K; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_0F)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uchar), &mask_F0)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + int nth = 64; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } - if (tensor->type == GGML_TYPE_Q6_K) { - ggml_tensor_extra_cl_q6_K * extra = (ggml_tensor_extra_cl_q6_K *)tensor->extra; +} -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - if (use_adreno_kernels(backend_ctx, tensor)) { - static ggml_cl_buffer buf_trans_ql; - static ggml_cl_buffer buf_trans_qh; - static ggml_cl_buffer buf_trans_s; - static ggml_cl_buffer buf_trans_d; - static ggml_cl_buffer buf_unpacked; +static void ggml_cl_expm1(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - cl_int M = tensor->ne[1]; // ne01 - cl_int K = tensor->ne[0]; // ne00 + UNUSED(src1); - GGML_ASSERT(K % ggml_blck_size(tensor->type) == 0); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - size_t size_ql = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2; - size_t size_qh = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/4; - size_t size_s = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/16; - size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t); - GGML_ASSERT(size_ql + size_qh + size_s + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - buf_trans_ql.allocate(backend_ctx->context, size_ql); - buf_trans_qh.allocate(backend_ctx->context, size_qh); - buf_trans_s.allocate(backend_ctx->context, size_s); - buf_trans_d.allocate(backend_ctx->context, size_d); - buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // transpose ql, qh, s and d back - transpose_2d_as_16b(backend_ctx, extra->ql, buf_trans_ql.buffer, size_ql, M, K/4); - transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/4); - transpose_2d_as_16b(backend_ctx, extra->s, buf_trans_s.buffer, size_s, M, K/16/2); - transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/256); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - // unpack - cl_uchar mask = 0xFF; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K_noshuffle; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_ql.buffer)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_s.buffer)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_d.buffer)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - size_t global_work_size[] = {(size_t)n_blk, 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + cl_kernel kernel; + + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_expm1_f32_4; + } else { + kernel = backend_ctx->kernel_expm1_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_expm1_f32; + } else { + kernel = backend_ctx->kernel_expm1_f16; + } + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - return; + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - cl_int err; - cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, - ggml_nbytes(tensor), NULL, &err); - CL_CHECK(err); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + } else { + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_expm1_f32_nc; + } else { + kernel = backend_ctx->kernel_expm1_f16_nc; + } - cl_uchar mask = 0xFF; - cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type); - cl_kernel kernel = backend_ctx->kernel_restore_block_q6_K; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->ql)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &n_blk)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - size_t global_work_size[] = {(size_t)n_blk, 1, 1}; - size_t local_work_size[] = {1, 1, 1}; + int nth = 64; - cl_event evt; - CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, - global_work_size, local_work_size, 0, NULL, &evt)); - CL_CHECK(clWaitForEvents(1, &evt)); - CL_CHECK(clEnqueueReadBuffer( - queue, data_device, CL_TRUE, offset, - size, data, 0, NULL, NULL)); - CL_CHECK(clReleaseMemObject(data_device)); - return; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -#endif // GGML_OPENCL_SOA_Q +} - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; +static void ggml_cl_abs(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - CL_CHECK(clEnqueueReadBuffer( - queue, extra->data_device, CL_TRUE, extra->offset + tensor->view_offs + offset, - size, data, 0, NULL, NULL)); + UNUSED(src1); - GGML_UNUSED(buffer); -} + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; -static void ggml_backend_opencl_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { - ggml_backend_dev_t dev = buffer->buft->device; - ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(dev); - cl_command_queue queue = backend_ctx->queue; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - for (cl_mem buf : ctx->buffer) { - CL_CHECK(clEnqueueFillBuffer(queue, buf, &value, sizeof(value), 0, buffer->size, 0, NULL, NULL)); - } - CL_CHECK(clFinish(queue)); -} + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; -static void ggml_backend_opencl_buffer_reset(ggml_backend_buffer_t buffer) { - ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; - ctx->reset(); -} + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; -static ggml_backend_buffer_i ggml_backend_opencl_buffer_interface = { - /* .free_buffer = */ ggml_backend_opencl_buffer_free_buffer, - /* .get_base = */ ggml_backend_opencl_buffer_get_base, - /* .init_tensor = */ ggml_backend_opencl_buffer_init_tensor, - /* .memset_tensor = */ NULL, - /* .set_tensor = */ ggml_backend_opencl_buffer_set_tensor, - /* .get_tensor = */ ggml_backend_opencl_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, - /* .get_tensor_2d = */ NULL, - /* .cpy_tensor = */ NULL, - /* .clear = */ ggml_backend_opencl_buffer_clear, - /* .reset = */ ggml_backend_opencl_buffer_reset, -}; + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; -// -// buffer type -// + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; -static const char * ggml_backend_opencl_buffer_type_get_name(ggml_backend_buffer_type_t buffer_type) { - return "OpenCL"; + cl_kernel kernel; - GGML_UNUSED(buffer_type); -} + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_abs_f32_4; + } else { + kernel = backend_ctx->kernel_abs_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_abs_f32; + } else { + kernel = backend_ctx->kernel_abs_f16; + } + } -static ggml_backend_buffer_t ggml_backend_opencl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buffer_type, size_t size) { - ggml_backend_opencl_context *backend_ctx = ggml_cl2_init(buffer_type->device); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - // clCreateBuffer returns -61 for size 0 - size = std::max(size, (size_t)1); + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; - cl_int err; - cl_mem mem = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size, NULL, &err); - if (err != CL_SUCCESS && backend_ctx->adreno_use_large_buffer) { - cl_mem_properties props[] = { 0x41A6 /* CL_LARGE_BUFFER_QCOM */, 1, 0 }; - mem = clCreateBufferWithProperties(backend_ctx->context, props, CL_MEM_READ_WRITE, size, NULL, &err); - } + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } - if (err != CL_SUCCESS) { - GGML_LOG_INFO("%s: failed to allocate %.2f MiB\n", __func__, size / 1024.0 / 1024.0); - return nullptr; - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + } else { + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_abs_f32_nc; + } else { + kernel = backend_ctx->kernel_abs_f16_nc; + } - ggml_backend_opencl_buffer_context * ctx = new ggml_backend_opencl_buffer_context(mem); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - return ggml_backend_buffer_init(buffer_type, ggml_backend_opencl_buffer_interface, ctx, size); -} + int nth = 64; -static size_t ggml_backend_opencl_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) { - ggml_backend_opencl_context * backend_ctx = ggml_cl2_init(buffer_type->device); - return backend_ctx->alignment; -} + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; -static size_t ggml_backend_opencl_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) { - static size_t max_size = -1; - if (max_size == (size_t)-1) { - ggml_backend_opencl_context * backend_ctx = ggml_cl2_init(buffer_type->device); - max_size = backend_ctx->max_alloc_size; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } - return max_size; } -static bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { - return ggml_backend_is_opencl(backend); +// Shared driver for the extended unary ops (unary_ext.cl), same selection as +// ggml_cl_abs: contiguous picks the vec4 kernel when the element count is a +// multiple of 4 (else scalar); non-contiguous uses the stride-addressed kernel. +static void ggml_cl_unary_ext(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst, + cl_kernel k_f32, cl_kernel k_f32_4, cl_kernel k_f32_nc, + cl_kernel k_f16, cl_kernel k_f16_4, cl_kernel k_f16_nc) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - UNUSED(buft); -} + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; -static ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = { - /* .get_name = */ ggml_backend_opencl_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_opencl_buffer_type_get_max_size, - /* .get_alloc_size = */ NULL, - /* .is_host = */ NULL, -}; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; -// -// backend device -// + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; -static const char * ggml_backend_opencl_device_get_name(ggml_backend_dev_t dev) { - return "GPUOpenCL"; + const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const cl_ulong nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const cl_ulong nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; - GGML_UNUSED(dev); -} + const bool is_f16 = (src0->type == GGML_TYPE_F16); + cl_kernel kernel; -static const char * ggml_backend_opencl_device_get_description(ggml_backend_dev_t dev) { - ggml_backend_opencl_device_context *dev_ctx = (ggml_backend_opencl_device_context *) dev->context; - return dev_ctx->device_name.c_str(); + if (ggml_is_contiguous(src0)) { + int n = ggml_nelements(dst); + if (n % 4 == 0) { + kernel = is_f16 ? k_f16_4 : k_f32_4; + n /= 4; + } else { + kernel = is_f16 ? k_f16 : k_f32; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + } else { + kernel = is_f16 ? k_f16_nc : k_f32_nc; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + + int nth = 64; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } -static void ggml_backend_opencl_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { - // no memory to report - *free = 0; - *total = 0; - - GGML_UNUSED(dev); +#define GGML_CL_UNARY_EXT_WRAP(FN, OP) \ +static void FN(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { \ + UNUSED(src1); \ + ggml_backend_opencl_context *c = (ggml_backend_opencl_context *)backend->context; \ + ggml_cl_unary_ext(backend, src0, dst, c->kernel_##OP##_f32, c->kernel_##OP##_f32_4, c->kernel_##OP##_f32_nc, \ + c->kernel_##OP##_f16, c->kernel_##OP##_f16_4, c->kernel_##OP##_f16_nc); \ } -static enum ggml_backend_dev_type ggml_backend_opencl_device_get_type(ggml_backend_dev_t dev) { - return GGML_BACKEND_DEVICE_TYPE_GPU; +GGML_CL_UNARY_EXT_WRAP(ggml_cl_sgn, sgn) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_step, step) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_elu, elu) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardswish, hardswish) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardsigmoid, hardsigmoid) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_floor, floor) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_ceil, ceil) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_round, round) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_trunc, trunc) - GGML_UNUSED(dev); -} +#undef GGML_CL_UNARY_EXT_WRAP -static void ggml_backend_opencl_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { - props->name = ggml_backend_opencl_device_get_name(dev); - props->description = ggml_backend_opencl_device_get_description(dev); - props->type = ggml_backend_opencl_device_get_type(dev); - ggml_backend_opencl_device_get_memory(dev, &props->memory_free, &props->memory_total); - props->caps = ggml_backend_dev_caps { - /* .async = */ false, - /* .host_buffer = */ false, - /* .buffer_from_host_ptr = */ false, - /* .events = */ false, - }; -} +static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); -static ggml_backend_t ggml_backend_opencl_device_init(ggml_backend_dev_t dev, const char * params) { - ggml_backend_opencl_context * backend_ctx = ggml_cl2_init(dev); - // Getting a new reference to the backend, increase ref_count - backend_ctx->ref_count++; + UNUSED(src1); - ggml_backend_t backend = new ggml_backend { - /* .guid = */ ggml_backend_opencl_guid(), - /* .interface = */ ggml_backend_opencl_i, - /* .device = */ dev, - /* .context = */ backend_ctx, - }; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - return backend; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - GGML_UNUSED(params); -} + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; -static ggml_backend_buffer_type_t ggml_backend_opencl_device_get_buffer_type(ggml_backend_dev_t dev) { - auto * dev_ctx = static_cast(dev->context); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - dev_ctx->buffer_type = ggml_backend_buffer_type{ - /* .iface = */ ggml_backend_opencl_buffer_type_interface, - /* .device = */ dev, - /* .context = */ nullptr, - }; + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - return &dev_ctx->buffer_type; -} + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; -static ggml_backend_buffer_t ggml_backend_opencl_device_buffer_from_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { - GGML_UNUSED(dev); - GGML_UNUSED(ptr); - GGML_UNUSED(size); - GGML_UNUSED(max_tensor_size); - return nullptr; -} + cl_kernel kernel; -static bool ggml_backend_opencl_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { - return ggml_opencl_supports_op(dev, op); -} + if (ggml_is_contiguous(src0)) { + // Handle contiguous input + int n = ggml_nelements(dst); + if (n % 4 == 0) { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_softplus_f32_4; + } else { + kernel = backend_ctx->kernel_softplus_f16_4; + } + n /= 4; + } else { + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_softplus_f32; + } else { + kernel = backend_ctx->kernel_softplus_f16; + } + } -static bool ggml_backend_opencl_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { - // Check 'dev' and 'buffer_type' are not objects belonging to this backend. - if (dev->iface.get_name != ggml_backend_opencl_device_get_name || - buft->iface.get_name != ggml_backend_opencl_buffer_type_get_name) { - return false; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - // Check cl_context is the same. clEnqueue* commands may not use - // buffers from another cl_context. - ggml_backend_opencl_context * backend_ctx0 = ggml_cl2_init(dev); - ggml_backend_opencl_context * backend_ctx1 = ggml_cl2_init(buft->device); - return backend_ctx0->context == backend_ctx1->context; -} + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; -namespace /* anonymous */ { -struct ggml_backend_device_i ggml_backend_opencl_device_i = { - /* .get_name = */ ggml_backend_opencl_device_get_name, - /* .get_description = */ ggml_backend_opencl_device_get_description, - /* .get_memory = */ ggml_backend_opencl_device_get_memory, - /* .get_type = */ ggml_backend_opencl_device_get_type, - /* .get_props = */ ggml_backend_opencl_device_get_props, - /* .init_backend = */ ggml_backend_opencl_device_init, - /* .get_buffer_type = */ ggml_backend_opencl_device_get_buffer_type, - /* .get_host_buffer_type = */ NULL, - /* .buffer_from_host_ptr = */ ggml_backend_opencl_device_buffer_from_ptr, - /* .supports_op = */ ggml_backend_opencl_device_supports_op, - /* .supports_buft = */ ggml_backend_opencl_device_supports_buft, - /* .offload_op = */ NULL, - /* .event_new = */ NULL, - /* .event_free = */ NULL, - /* .event_synchronize = */ NULL, -}; -} + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } -// Backend registry + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + } else { + // Handle non-contiguous input + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_softplus_f32_nc; + } else { + kernel = backend_ctx->kernel_softplus_f16_nc; + } -static const char * ggml_backend_opencl_reg_get_name(ggml_backend_reg_t reg) { - return "OpenCL"; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); - GGML_UNUSED(reg); -} + int nth = 64; -static size_t ggml_backend_opencl_reg_device_count(ggml_backend_reg_t reg) { - return g_ggml_backend_opencl_devices.size(); + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - GGML_UNUSED(reg); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } -static ggml_backend_dev_t ggml_backend_opencl_reg_device_get(ggml_backend_reg_t reg, size_t index) { - GGML_ASSERT(index < ggml_backend_opencl_reg_device_count(reg)); +static void ggml_cl_repeat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1_shape_def, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(dst->type == src0->type); - return &g_ggml_backend_opencl_devices[index]; + UNUSED(src1_shape_def); - GGML_UNUSED(reg); - GGML_UNUSED(index); -} + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; -static struct ggml_backend_reg_i ggml_backend_opencl_reg_i = { - /* .get_name = */ ggml_backend_opencl_reg_get_name, - /* .device_count = */ ggml_backend_opencl_reg_device_count, - /* .device_get = */ ggml_backend_opencl_reg_device_get, - /* .get_proc_address = */ NULL, -}; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; -ggml_backend_reg_t ggml_backend_opencl_reg(void) { - static std::mutex mutex; - static ggml_backend_reg reg; - static bool initialized = false; - std::lock_guard lock(mutex); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - if (initialized) { - return ® - } - initialized = true; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - g_ggml_backend_opencl_devices = ggml_opencl_probe_devices(®); + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - reg = ggml_backend_reg{ - /* .api_version = */ GGML_BACKEND_API_VERSION, - /* .iface = */ ggml_backend_opencl_reg_i, - /* .context = */ NULL, - }; + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; - return ® -} + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; -GGML_BACKEND_DL_IMPL(ggml_backend_opencl_reg) + cl_kernel kernel = backend_ctx->kernel_repeat_f32; -//------------------------------------------------------------------------------ -// Debugging utils -//------------------------------------------------------------------------------ -#if 0 -#define QK4_0 32 -typedef struct { - ggml_fp16_t d; // delta - uint8_t qs[QK4_0 / 2]; // nibbles / quants -} block_q4_0; -static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, - "wrong q4_0 block size/padding"); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); -#define QK_MXFP4 32 + int nth = 64; + + size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; -#include -#ifdef __cplusplus -#include "half.hpp" -#endif + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} -static void dump_tensor(ggml_backend_t backend, const struct ggml_tensor * tensor) { - void * buf = malloc(ggml_nbytes(tensor)); +static void ggml_cl_pad(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - cl_command_queue queue = backend_ctx->queue; -#ifdef GGML_OPENCL_SOA_Q - void * buf_q; - void * buf_d; -#endif - // Make sure everything is done. - CL_CHECK(clFinish(queue)); + if (backend_ctx->kernel_pad == nullptr) { + GGML_LOG_WARN("%s: pad kernel not available, skipping OpenCL execution.\n", __func__); + return; + } -#ifdef GGML_OPENCL_SOA_Q - if (tensor->type == GGML_TYPE_Q4_0) { - ggml_tensor_extra_cl_q4_0 * extra = (ggml_tensor_extra_cl_q4_0 *) tensor->extra; - GGML_ASSERT(extra); + ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; - size_t size_q = ggml_nelements(tensor)/QK4_0 * QK4_0/2; - size_t size_d = ggml_nelements(tensor)/QK4_0 * sizeof(ggml_fp16_t); - GGML_ASSERT(size_q + size_d == ggml_nbytes(tensor)); - buf_q = malloc(size_q); - buf_d = malloc(size_d); + cl_ulong off_src0 = extra_src0->offset + src0->view_offs; + cl_ulong off_dst = extra_dst->offset + dst->view_offs; - CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, size_d, buf_d, 0, NULL, NULL)); - CL_CHECK(clFinish(queue)); - } else if (tensor->type == GGML_TYPE_MXFP4) { - ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *) tensor->extra; - GGML_ASSERT(extra); + const int s_ne0 = src0->ne[0]; + const int s_ne1 = src0->ne[1]; + const int s_ne2 = src0->ne[2]; + const int s_ne3 = src0->ne[3]; - size_t size_q = ggml_nelements(tensor)/QK_MXFP4 * QK_MXFP4/2; - size_t size_e = ggml_nelements(tensor)/QK_MXFP4 * sizeof(char); - GGML_ASSERT(size_q + size_e == ggml_nbytes(tensor)); - buf_q = malloc(size_q); - buf_d = malloc(size_e); + const int s_nb0 = src0->nb[0]; + const int s_nb1 = src0->nb[1]; + const int s_nb2 = src0->nb[2]; + const int s_nb3 = src0->nb[3]; - CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL)); - CL_CHECK(clEnqueueReadBuffer(queue, extra->e, CL_TRUE, 0, size_e, buf_d, 0, NULL, NULL)); - CL_CHECK(clFinish(queue)); - } else { - // Read out the tensor from GPU memory. - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra); + const int d_ne0 = dst->ne[0]; + const int d_ne1 = dst->ne[1]; + const int d_ne2 = dst->ne[2]; + const int d_ne3 = dst->ne[3]; - CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, - extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); - CL_CHECK(clFinish(queue)); - } -#else - // Read out the tensor from GPU memory. - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; - GGML_ASSERT(extra); + const int d_nb0 = dst->nb[0]; + const int d_nb1 = dst->nb[1]; + const int d_nb2 = dst->nb[2]; + const int d_nb3 = dst->nb[3]; - CL_CHECK(clEnqueueReadBuffer(queue, extra->data_device, CL_TRUE, - extra->offset, ggml_nbytes(tensor), buf, 0, NULL, NULL)); - CL_CHECK(clFinish(queue)); -#endif // GGML_OPENCL_SOA_Q + const int lp0 = ((const int*)(dst->op_params))[0]; + const int rp0 = ((const int*)(dst->op_params))[1]; + const int lp1 = ((const int*)(dst->op_params))[2]; + const int rp1 = ((const int*)(dst->op_params))[3]; + const int lp2 = ((const int*)(dst->op_params))[4]; + const int rp2 = ((const int*)(dst->op_params))[5]; + const int lp3 = ((const int*)(dst->op_params))[6]; + const int rp3 = ((const int*)(dst->op_params))[7]; - // Open file and dump. - char fname[512]; - snprintf(fname, sizeof(fname), "./tensor-dumps/%s.txt", tensor->name); - FILE * f = fopen(fname, "w"); - if (!f) { - printf("Failed to open %s\n", fname); - return; - } + cl_kernel kernel = backend_ctx->kernel_pad; - if (tensor->type == GGML_TYPE_F32) { - float * data = (float *) buf; - for (int i = 0; i < ggml_nelements(tensor); ++i) { - if (isnan(data[i])) { - printf("NaN found: %s\n", tensor->name); - break; - } - fprintf(f, "%f\n", data[i]); - } - } else if (tensor->type == GGML_TYPE_I32) { - int * data = (int *) buf; - for (int i = 0; i < ggml_nelements(tensor); ++i) { - if (isnan(data[i])) { - printf("NaN found: %s\n", tensor->name); - break; - } - fprintf(f, "%d\n", data[i]); - } - } else if (tensor->type == GGML_TYPE_F16) { -#ifdef __cplusplus - half_float::half * data = (half_float::half *) buf; - for (int i = 0; i < ggml_nelements(tensor); ++i) { - if (std::isnan(data[i])) { - printf("NaN found: %s\n", tensor->name); - break; - } - fprintf(f, "%f\n", float(data[i])); - } -#endif - } else if (tensor->type == GGML_TYPE_Q4_0) { -#ifdef GGML_OPENCL_SOA_Q - ggml_fp16_t * data_d = (ggml_fp16_t *)buf_d; - unsigned char * data_q = (unsigned char *)buf_q; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &s_ne0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &s_ne1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &s_ne2)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &s_ne3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &s_nb0)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &s_nb1)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &s_nb2)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &s_nb3)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &d_ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &d_ne1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &d_ne2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &d_ne3)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &d_nb0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &d_nb1)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &d_nb2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &d_nb3)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &lp0)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &rp0)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &lp1)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &rp1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &lp2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &rp2)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &lp3)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(int), &rp3)); - for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { - fprintf(f, "%04x, ", data_d[i]); - for (int k = 0; k < QK4_0/2; ++k) { - fprintf(f, "%02x, ", data_q[k]); - } - fprintf(f, "\n"); - data_q += QK4_0/2; - } - free(buf_d); - free(buf_q); -#else - block_q4_0 * data = (block_q4_0 *) buf; - for (int i = 0; i < ggml_nelements(tensor)/QK4_0; ++i) { - fprintf(f, "%04x, ", data[i].d); - for (int k = 0; k < QK4_0/2; ++k) { - fprintf(f, "%02x, ", data[i].qs[k]); - } - fprintf(f, "\n"); - } -#endif // GGML_OPENCL_SOA_Q + size_t lws0 = 64; + size_t gws0 = (( (size_t)d_ne0 + lws0 - 1 ) / lws0) * lws0; + + size_t global_work_size[] = { gws0, (size_t)d_ne1, (size_t)d_ne2*d_ne3 }; + size_t local_work_size[] = { lws0, 1, 1 }; + + size_t * local_work_size_ptr = local_work_size; + if (d_ne0 % lws0 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; } - free(buf); - fflush(f); - fclose(f); + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -#else -#define dump_tensor(tensor) -#endif -//------------------------------------------------------------------------------ -// Ops -//------------------------------------------------------------------------------ +static void ggml_cl_upscale(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); -static bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) { - const int64_t ne10 = src1->ne[0]; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - const int64_t ne0 = dst->ne[0]; - const int64_t ne1 = dst->ne[1]; + const int mode_flags = (ggml_scale_mode) ggml_get_op_params_i32(dst, 0); + const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF); + cl_kernel kernel = nullptr; - // TODO: find the optimal values for these - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && - src1->type == GGML_TYPE_F32 && - dst->type == GGML_TYPE_F32 && - (ne0 >= 32 && ne1 >= 32 && ne10 >= 32); -} + if (mode == GGML_SCALE_MODE_NEAREST) { + kernel = backend_ctx->kernel_upscale; + if (kernel == nullptr) { + GGML_LOG_WARN("%s: nearest upscale kernel not available, skipping OpenCL execution.\n", __func__); + return; + } + } else if (mode == GGML_SCALE_MODE_BILINEAR) { + kernel = backend_ctx->kernel_upscale_bilinear; + if (kernel == nullptr) { + GGML_LOG_WARN("%s: bilinear upscale kernel not available, skipping OpenCL execution.\n", __func__); + return; + } + } else { + GGML_LOG_WARN("%s: unsupported upscale mode %d, skipping OpenCL execution.\n", __func__, mode); + return; + } -// Copy a noncontiguous tensor to contiguous tensor. ne[] remains the same but -// nb[] is recalculated such that tensor is contiguous. -static void ggml_cl_copy_to_contiguous(ggml_backend_t backend, const ggml_tensor * src, cl_mem dst, - cl_ulong &nb0, cl_ulong &nb1, cl_ulong &nb2, cl_ulong &nb3) { - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; - const int tensor_type_size = ggml_type_size(src->type); + cl_ulong off_src0 = extra_src0->offset + src0->view_offs; + cl_ulong off_dst = extra_dst->offset + dst->view_offs; - const int ne00 = src->ne[0]; - const int ne01 = src->ne[1]; - const int ne02 = src->ne[2]; - const int ne03 = src->ne[3]; + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - const cl_ulong nb00 = src->nb[0]; - const cl_ulong nb01 = src->nb[1]; - const cl_ulong nb02 = src->nb[2]; - const cl_ulong nb03 = src->nb[3]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - const int ne0 = src->ne[0]; - const int ne1 = src->ne[1]; - const int ne2 = src->ne[2]; - const int ne3 = src->ne[3]; + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; - nb0 = tensor_type_size; - nb1 = tensor_type_size*ne00; - nb2 = tensor_type_size*ne00*ne01; - nb3 = tensor_type_size*ne00*ne01*ne02; + float sf0 = (float)ne0 / ne00; + float sf1 = (float)ne1 / ne01; + float sf2 = (float)ne2 / ne02; + float sf3 = (float)ne3 / ne03; - ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *)src->extra; + float pixel_offset = 0.5f; - cl_ulong offset0 = extra->offset + src->view_offs; - cl_ulong offsetd = 0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb03)); - cl_kernel kernel; + if (mode == GGML_SCALE_MODE_NEAREST) { + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &sf0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &sf1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf3)); + } else if (mode == GGML_SCALE_MODE_BILINEAR) { + if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) { + sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0; + sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1; + pixel_offset = 0.0f; + } - switch (src->type) { - case GGML_TYPE_F32: - kernel = backend_ctx->kernel_cpy_f32_f32; - break; - case GGML_TYPE_F16: - kernel = backend_ctx->kernel_cpy_f16_f16; - break; - default: - GGML_ASSERT(false && "not implemented"); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne3)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf0)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf1)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(float), &sf2)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(float), &sf3)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float), &pixel_offset)); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &dst)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb3)); - - const int nth = MIN(64, ne00); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + size_t dst_total_elements = (size_t)ne0 * ne1 * ne2 * ne3; + if (dst_total_elements == 0) { + return; + } + size_t global_work_size[] = { dst_total_elements, 1, 1 }; + size_t local_work_size_pref = 256; + size_t local_work_size[] = { MIN(local_work_size_pref, dst_total_elements), 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src); -} + size_t * local_work_size_ptr = local_work_size; + if (dst_total_elements % local_work_size[0] != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } -static void ggml_cl_nop(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - UNUSED(backend); - UNUSED(src0); - UNUSED(src1); - UNUSED(dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); } -static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - GGML_TENSOR_LOCALS(int, ne1, src1, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); - GGML_TENSOR_LOCALS(int, ne, dst, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; @@ -6567,22 +17384,52 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_kernel kernel; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - switch (src0->type) { - case GGML_TYPE_F32: - kernel = backend_ctx->kernel_get_rows_f32; - break; - case GGML_TYPE_F16: - kernel = backend_ctx->kernel_get_rows_f16; - break; - case GGML_TYPE_Q4_0: - kernel = backend_ctx->kernel_get_rows_q4_0; - break; - default: - GGML_ASSERT(false && "not implemented"); + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; + + const cl_ulong nb10 = src1->nb[0]; + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; + + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; + const int ne3 = dst->ne[3]; + + const cl_ulong nb0 = dst->nb[0]; + const cl_ulong nb1 = dst->nb[1]; + const cl_ulong nb2 = dst->nb[2]; + const cl_ulong nb3 = dst->nb[3]; + + const cl_int dim = ((const int32_t *) dst->op_params)[0]; + GGML_ASSERT(dim >= 0 && dim <= 3); + + int nth = MIN(64, ne0); + + const size_t ts = ggml_type_size(dst->type); + // the pack kernel copies 4-byte elements, so it is only valid for those. + const bool concat_pack = (dim == 0 && ne0 < 32 && ts == 4); + cl_kernel kernel; + if (concat_pack) { + kernel = backend_ctx->kernel_concat_b4_pack; + } else { + switch (ts) { + case 1: kernel = backend_ctx->kernel_concat_b1; break; + case 2: kernel = backend_ctx->kernel_concat_b2; break; + case 4: kernel = backend_ctx->kernel_concat_b4; break; + case 8: kernel = backend_ctx->kernel_concat_b8; break; + default: GGML_ABORT("unsupported concat element size: %zu", ts); + } } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); @@ -6592,939 +17439,2385 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); - - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - int nth = 1; - while (nth < ne00 && 2*nth <= max_workgroup_size) { - nth *= 2; - } + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_int), &dim)); - size_t global_work_size[] = {(size_t)ne10*nth, (size_t)ne11, (size_t)ne12}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + if (concat_pack) { + // packed kernel needs the dst dims to unflatten its 1-D row index. + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne2)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &ne3)); + + const int maxwg = (int)backend_ctx->get_kernel_workgroup_size(kernel); + const int base = MIN(64, maxwg); + const int tpr = MIN(ne0, base); // threads per row + const int rpw = MAX(1, base / tpr); // rows per workgroup + const int lsz = tpr * rpw; + const int nrows = ne1*ne2*ne3; + const int nwg = (nrows + rpw - 1) / rpw; + size_t global_work_size[] = {(size_t)nwg*lsz, 1, 1}; + size_t local_work_size[] = {(size_t)lsz, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); + } else { + size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } -static void ggml_cl_set_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_timestep_embedding(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32); + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); - // ne0 = ne00 - // ne2 = ne02 - // ne3 = ne03 + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + if (backend_ctx->kernel_timestep_embedding == nullptr) { + GGML_LOG_WARN("%s: timestep_embedding kernel not available, skipping OpenCL execution.\n", __func__); + return; + } - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; + cl_ulong off_src0 = extra_src0->offset + src0->view_offs; + cl_ulong off_dst = extra_dst->offset + dst->view_offs; - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; + const int logical_dim = dst->op_params[0]; + const int max_period = dst->op_params[1]; + const int dst_nb1_bytes = dst->nb[1]; - const int ne0 = dst->ne[0]; + cl_kernel kernel = backend_ctx->kernel_timestep_embedding; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &dst_nb1_bytes)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &logical_dim)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &max_period)); - const int nblk0 = ne0/ggml_blck_size(dst->type); + size_t gws0 = (size_t)(((logical_dim + 1) / 2) + 1); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + size_t gws1 = (size_t)src0->ne[0]; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + size_t global_work_size[] = {gws0, gws1, 1}; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, NULL, dst); +} - cl_kernel kernel; +struct ggml_cl_flash_attn_temp_buffer { + cl_mem data = nullptr; - switch (dst->type) { - case GGML_TYPE_F32: - if (src1->type == GGML_TYPE_I64) { - kernel = backend_ctx->kernel_set_rows_f32_i64; - } else { - kernel = backend_ctx->kernel_set_rows_f32_i32; - } - break; - case GGML_TYPE_F16: - if (src1->type == GGML_TYPE_I64) { - kernel = backend_ctx->kernel_set_rows_f16_i64; - } else { - kernel = backend_ctx->kernel_set_rows_f16_i32; - } - break; - default: - GGML_ABORT("not implemented"); + ~ggml_cl_flash_attn_temp_buffer() { + if (data != nullptr) { + CL_CHECK(clReleaseMemObject(data)); + data = nullptr; + } } +}; - fastdiv_vals ne11_ = init_fastdiv_values(ne11); - fastdiv_vals ne12_ = init_fastdiv_values(ne12); +// Resolve the source buffer + strides for an FA KV tensor: keep the +// caller-supplied AoS buffer if non-NULL, else fall back to tensor->extra. +static void ggml_cl_flash_attn_resolve_src( + const ggml_tensor * tensor, + cl_mem & buf, + cl_ulong & offset, + cl_ulong & nb1, + cl_ulong & nb2, + cl_ulong & nb3) { + if (buf != NULL) { + return; + } + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra && extra->data_device); + buf = extra->data_device; + offset = extra->offset + tensor->view_offs; + nb1 = tensor->nb[1]; + nb2 = tensor->nb[2]; + nb3 = tensor->nb[3]; +} - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(fastdiv_vals), &ne11_)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(fastdiv_vals), &ne12_)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &nblk0)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb3)); +// Read a (possibly strided-view) tensor from device into a tight host buffer. +// dim 0 is always tight; a strided view is gathered row-by-row. +static void ggml_cl_flash_attn_read_tensor_host( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + cl_mem src_buffer, cl_ulong src_offset, + cl_ulong src_nb1, cl_ulong src_nb2, cl_ulong src_nb3, + size_t row_bytes, void * dst, size_t total_bytes +) { + const bool contiguous_layout = + src_nb1 == row_bytes && + src_nb2 == row_bytes * (cl_ulong) tensor->ne[1] && + src_nb3 == src_nb2 * (cl_ulong) tensor->ne[2]; + + if (contiguous_layout) { + CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, + src_offset, total_bytes, dst, 0, NULL, NULL)); + return; + } - int nth0 = 64; - if (backend_ctx->gpu_family == INTEL) { - nth0 = 32; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; + size_t dst_off = 0; + for (int64_t i3 = 0; i3 < tensor->ne[3]; ++i3) { + for (int64_t i2 = 0; i2 < tensor->ne[2]; ++i2) { + for (int64_t i1 = 0; i1 < tensor->ne[1]; ++i1) { + const cl_ulong row_src_off = src_offset + + (cl_ulong) i3 * src_nb3 + + (cl_ulong) i2 * src_nb2 + + (cl_ulong) i1 * src_nb1; + CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, src_buffer, CL_TRUE, + row_src_off, row_bytes, + (uint8_t *) dst + dst_off, 0, NULL, NULL)); + dst_off += row_bytes; + } + } } + GGML_ASSERT(dst_off == total_bytes); +} - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - while (nth0 < nblk0 && nth0 < max_workgroup_size) { - nth0 *= 2; +// forward decl: used by the FA decode dispatch (K-image variant) below. +static cl_mem ggml_cl_img_pool_get_or_create( + ggml_backend_opencl_context * backend_ctx, + std::map & pool, + cl_mem data_device, + cl_ulong offset0, + size_t required_bytes, + cl_channel_type channel_data_type); + +// Rebuild AoS q8_0/q4_0 bytes from a SoA tensor into a temp buffer. +// Returns false if the tensor is not SoA-quantised (already AoS). +static bool ggml_cl_flash_attn_reconstruct_aos( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & out_buf, + cl_ulong & out_offset, + cl_ulong & out_nb1, + cl_ulong & out_nb2, + cl_ulong & out_nb3 +) { + if (tensor == nullptr) { + return false; + } + const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0 && ggml_cl_is_q8_0_soa(tensor); + const bool is_q4_0 = tensor->type == GGML_TYPE_Q4_0 && ggml_cl_is_q4_0_soa(tensor); + if (!is_q8_0 && !is_q4_0) { + return false; } - int rows_per_workgroup = 1; - if (nth0 > nblk0) { - rows_per_workgroup = nth0 / nblk0; - nth0 = nblk0; + // For views, SoA extra is on view_src (view->extra is pre-SoA). + // Noshuffle layout only applies to 2D weights, as determined by `use_adreno_kernels`, + // where ne2 == 1 and ne3 == 1 -- these are never FA inputs. + // Therefore, we use `restore_block_qk_0` kernels, not `restore_block_qk_0_noshuffle`. + const ggml_tensor * soa_src = tensor->view_src ? tensor->view_src : tensor; + cl_mem extra_q = NULL; + cl_mem extra_d = NULL; + if (is_q8_0) { + auto * e = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; + GGML_ASSERT(e && e->q && e->d); + extra_q = e->q; + extra_d = e->d; + } else { + auto * e = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; + GGML_ASSERT(e && e->q && e->d); + extra_q = e->q; + extra_d = e->d; } - size_t global_work_size[] = { - (size_t)(ne01 + rows_per_workgroup - 1)/rows_per_workgroup*nth0, - (size_t)ne02*rows_per_workgroup, - (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth0, (size_t)rows_per_workgroup, 1}; + // Reconstruct the whole parent; view offsets then work naturally. + const size_t parent_nbytes = ggml_nbytes(soa_src); + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); + CL_CHECK(err); + + cl_kernel kernel = is_q8_0 ? backend_ctx->kernel_restore_block_q8_0 + : backend_ctx->kernel_restore_block_q4_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra_d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &temp.data)); + + const size_t n_blocks = (size_t) ggml_nelements(soa_src) / ggml_blck_size(soa_src->type); + size_t global_work_size[] = { n_blocks, 1, 1 }; + size_t local_work_size[] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, + global_work_size, local_work_size, 0, NULL, NULL)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + out_buf = temp.data; + out_offset = tensor->view_offs; + out_nb1 = tensor->nb[1]; + out_nb2 = tensor->nb[2]; + out_nb3 = tensor->nb[3]; + return true; } -static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); +// GPU dequant of a contiguous q4_0/q8_0 KV tensor to f16/f32. Caller supplies +// src_buf when reconstructing from SoA. Returns false for non-contig layouts +// (the kernel indexes blocks tightly within ne[0]) so the caller can fall back +// to the host path. +static bool ggml_cl_flash_attn_dequant_kv_gpu( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_type target_type, + cl_mem in_src_buf, + cl_ulong in_src_offset, + cl_ulong in_src_nb1, + cl_ulong in_src_nb2, + cl_ulong in_src_nb3, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & out_buf, + cl_ulong & out_offset, + cl_ulong & out_nb1, + cl_ulong & out_nb2, + cl_ulong & out_nb3 +) { + GGML_ASSERT(tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q4_0); + GGML_ASSERT(target_type == GGML_TYPE_F16 || target_type == GGML_TYPE_F32); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + const bool is_q8_0 = tensor->type == GGML_TYPE_Q8_0; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + cl_mem src_buf = in_src_buf; + cl_ulong src_offset = in_src_offset; + cl_ulong src_nb1 = in_src_nb1; + cl_ulong src_nb2 = in_src_nb2; + cl_ulong src_nb3 = in_src_nb3; + ggml_cl_flash_attn_resolve_src(tensor, src_buf, src_offset, src_nb1, src_nb2, src_nb3); - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; + if (tensor->nb[0] != (cl_ulong) ggml_type_size(tensor->type)) { + return false; + } - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; + const size_t n_blocks = (size_t) ggml_nelements(tensor) / 32; // block size is 32 + const size_t elem_size = ggml_type_size(target_type); + const size_t out_bytes = n_blocks * 32 * elem_size; + const cl_int nblk0_arg = (cl_int) (tensor->ne[0] / 32); + const cl_int ne1_arg = (cl_int) tensor->ne[1]; + const cl_int ne2_arg = (cl_int) tensor->ne[2]; + const cl_int ne3_arg = (cl_int) tensor->ne[3]; - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); + CL_CHECK(err); - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + cl_kernel kernel; + if (target_type == GGML_TYPE_F16) { + kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f16_view_aos + : backend_ctx->kernel_dequant_q4_0_f16_view_aos; + } else { + kernel = is_q8_0 ? backend_ctx->kernel_dequant_q8_0_f32_view_aos + : backend_ctx->kernel_dequant_q4_0_f32_view_aos; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &src_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &src_offset)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &src_nb1)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &src_nb2)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &src_nb3)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &nblk0_arg)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne1_arg)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne2_arg)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne3_arg)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), &temp.data)); + + size_t global_ws[3] = { (size_t) nblk0_arg, (size_t) ne1_arg, (size_t) ne2_arg * (size_t) ne3_arg }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, + global_ws, NULL, 0, NULL, NULL)); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + out_buf = temp.data; + out_offset = 0; + out_nb1 = (cl_ulong) tensor->ne[0] * elem_size; + out_nb2 = out_nb1 * (cl_ulong) tensor->ne[1]; + out_nb3 = out_nb2 * (cl_ulong) tensor->ne[2]; + return true; +} - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; +static bool ggml_cl_flash_attn_prepare_quantized_tensor( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_type target_type, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & data_device, + cl_ulong & offset, + cl_ulong & nb1, + cl_ulong & nb2, + cl_ulong & nb3 +) { + if (!ggml_is_quantized(tensor->type)) { + return false; + } - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + // Caller-supplied AoS buffer wins over tensor->extra when present. + cl_mem src_buffer = data_device; + cl_ulong src_offset = offset; + cl_ulong src_nb1 = nb1; + cl_ulong src_nb2 = nb2; + cl_ulong src_nb3 = nb3; + ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); - cl_kernel kernel; + const int64_t n = ggml_nelements(tensor); + const size_t row_bytes = (size_t) (tensor->ne[0] / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); + // tight-packed byte count (ggml_nbytes includes stride gaps). + const size_t total_bytes = (size_t) (n / ggml_blck_size(tensor->type)) * ggml_type_size(tensor->type); + std::vector host_quant(total_bytes); - const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0; + sync_with_other_backends(backend_ctx); + ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, + src_nb1, src_nb2, src_nb3, + row_bytes, host_quant.data(), total_bytes); - if (bcast_row) { - GGML_ASSERT(ggml_is_contiguous(src0)); - GGML_ASSERT(ne11 == 1); - } + std::vector host_f32(n); + ggml_get_type_traits(tensor->type)->to_float(host_quant.data(), host_f32.data(), n); - if (dst->type == GGML_TYPE_F32) { - GGML_ASSERT(src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32); - if (bcast_row) { - kernel = backend_ctx->kernel_add_row; - const int ne = ne00 / 4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); - } else { - kernel = backend_ctx->kernel_add; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); - } - } else if (dst->type == GGML_TYPE_F16) { - GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32); - const int type_src0 = (src0->type == GGML_TYPE_F32); - const int type_src1 = (src1->type == GGML_TYPE_F32); - if (bcast_row) { - kernel = backend_ctx->kernel_add_row_f16; - const int ne = ne00 / 4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &type_src0)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &type_src1)); - } else { - kernel = backend_ctx->kernel_add_f16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &type_src0)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(int), &type_src1)); - } + const size_t bytes_per_elem = ggml_type_size(target_type); + const size_t buffer_size = (size_t) n * bytes_per_elem; + + std::vector host_linear(buffer_size); + if (target_type == GGML_TYPE_F32) { + memcpy(host_linear.data(), host_f32.data(), buffer_size); } else { - GGML_ASSERT(false && "unsupported data types for add"); + GGML_ASSERT(target_type == GGML_TYPE_F16); + ggml_fp32_to_fp16_row(host_f32.data(), (ggml_fp16_t *) host_linear.data(), n); } - if (bcast_row) { - int n = ggml_nelements(dst)/4; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; - - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, buffer_size, NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, buffer_size, host_linear.data(), 0, NULL, NULL)); - backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size_ptr, dst); - } else { - unsigned int nth = MIN(64, ne0); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {nth, 1, 1}; + data_device = temp.data; + offset = 0; + nb1 = (cl_ulong) (tensor->ne[0] * bytes_per_elem); + nb2 = (cl_ulong) (tensor->ne[1] * nb1); + nb3 = (cl_ulong) (tensor->ne[2] * nb2); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: OpenCL flash attention dequantizes GPU-resident quantized KV cache into temporary linear buffers; performance may be poor\n"); + warned = true; } + + return true; } -static void ggml_cl_add_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); +// Host-side F16 -> F32 for the asymmetric-KV F32 fallback path. +static bool ggml_cl_flash_attn_convert_f16_to_f32( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + ggml_cl_flash_attn_temp_buffer & temp, + cl_mem & data_device, + cl_ulong & offset, + cl_ulong & nb1, + cl_ulong & nb2, + cl_ulong & nb3 +) { + if (tensor->type != GGML_TYPE_F16) { + return false; + } - const ggml_tensor * src2 = dst->src[2]; - GGML_ASSERT(src2); - GGML_ASSERT(src2->extra); + cl_mem src_buffer = data_device; + cl_ulong src_offset = offset; + cl_ulong src_nb1 = nb1; + cl_ulong src_nb2 = nb2; + cl_ulong src_nb3 = nb3; + ggml_cl_flash_attn_resolve_src(tensor, src_buffer, src_offset, src_nb1, src_nb2, src_nb3); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(src2->type == GGML_TYPE_I32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + const int64_t n = ggml_nelements(tensor); + const size_t row_bytes = (size_t) tensor->ne[0] * sizeof(ggml_fp16_t); + const size_t total_bytes = (size_t) n * sizeof(ggml_fp16_t); + std::vector host_f16(total_bytes); - GGML_ASSERT(ggml_is_contiguous_rows(src0)); + sync_with_other_backends(backend_ctx); + ggml_cl_flash_attn_read_tensor_host(backend_ctx, tensor, src_buffer, src_offset, + src_nb1, src_nb2, src_nb3, + row_bytes, host_f16.data(), total_bytes); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; + std::vector host_f32(n); + ggml_fp16_to_fp32_row((const ggml_fp16_t *) host_f16.data(), host_f32.data(), n); - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; + const size_t f32_bytes = (size_t) n * sizeof(float); + cl_int err; + temp.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, f32_bytes, NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer(backend_ctx->queue, temp.data, CL_TRUE, 0, + f32_bytes, host_f32.data(), 0, NULL, NULL)); - const cl_ulong nb11 = src1->nb[1]; + data_device = temp.data; + offset = 0; + nb1 = (cl_ulong) (tensor->ne[0] * sizeof(float)); + nb2 = (cl_ulong) (tensor->ne[1] * nb1); + nb3 = (cl_ulong) (tensor->ne[2] * nb2); - const cl_ulong nb21 = src2->nb[1]; + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: OpenCL flash attention asymmetric KV converts an F16 cache to F32 host-side; performance may be poor\n"); + warned = true; + } - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; + return true; +} - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; +// Flash-Decoding (K-split) dispatch thresholds. FD fires for non-causal +// attention with n_kv >= FD_MIN_N_KV and d_head <= FD_MAX_DK; the KV range is +// split into ~n_kv/FD_KV_PER_SPLIT partials, clamped to [FD_MIN_SPLITS, +// FD_MAX_SPLITS]. Multi-query FD is restricted to small heads +// (d_head <= FD_MAX_DK_MULTI) and capped at FD_MAX_N_Q_MULTI queries. +static constexpr int FD_MIN_N_KV = 2048; +static constexpr int FD_KV_PER_SPLIT = 2048; +// f16 KV decode wants more splits than the 2048 default; quantized KV keeps 2048. +static constexpr int FD_KV_PER_SPLIT_F16 = 512; +static constexpr int FD_MIN_SPLITS = 2; +static constexpr int FD_MAX_SPLITS = 16; +static constexpr int FD_MAX_DK = 128; +static constexpr int FD_MAX_DK_MULTI = 64; +static constexpr int FD_MAX_N_Q_MULTI = 8; +// MQ FD split-groups have few subgroups (MQ_NSG_SPLIT), so use a smaller +// kv_per_split to keep the softmax recurrence short; non-MQ keeps FD_KV_PER_SPLIT. +static constexpr int FD_MQ_KV_PER_SPLIT = 256; +static constexpr int FD_MQ_MAX_SPLITS = 128; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +struct ggml_cl_adreno_xmem_attn_schedule { + int qk_lws0 = 256; + int qk_lws2 = 1; + int softmax_reduce_lws0 = 256; + int softmax_apply_lws0 = 64; + int softmax_apply_lws2 = 4; + int pv_lws0 = 64; + int pv_lws2 = 4; +}; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offset2 = extra2->offset + src2->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; +static inline size_t ggml_cl_round_up(size_t x, size_t a) { + return ((x + a - 1) / a) * a; +} - cl_kernel kernel = backend_ctx->kernel_add_id; +static inline int ggml_cl_round_up_div(int x, int y) { + return (x + y - 1) / y; +} - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb21)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); +static inline void ggml_cl_set_arg_int4(cl_kernel kernel, cl_uint index, int x, int y, int z, int w) { + struct { int x, y, z, w; } value { x, y, z, w }; + CL_CHECK(clSetKernelArg(kernel, index, sizeof(value), &value)); +} - int nth = MIN(ne00, (int) backend_ctx->get_kernel_workgroup_size(kernel)); - size_t global_work_size[] = { (size_t)ne01*nth, (size_t)ne02, 1 }; - size_t local_work_size[] = { (size_t)nth, 1, 1 }; +static cl_mem ggml_cl_make_image2d_half4(cl_context context, cl_mem_flags flags, size_t width, size_t height) { + cl_int err = CL_SUCCESS; + cl_image_format format = { CL_RGBA, CL_HALF_FLOAT }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE2D; + desc.image_width = width; + desc.image_height = height; + cl_mem image = clCreateImage(context, flags, &format, &desc, nullptr, &err); + CL_CHECK(err); + return image; +} - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +static cl_mem ggml_cl_make_image1d_buffer_half4(cl_context context, cl_mem_flags flags, size_t width, cl_mem backing_buffer) { + cl_int err = CL_SUCCESS; + cl_image_format format = { CL_RGBA, CL_HALF_FLOAT }; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = width; + desc.buffer = backing_buffer; + cl_mem image = clCreateImage(context, flags, &format, &desc, nullptr, &err); + CL_CHECK(err); + return image; } -static void ggml_cl_mul(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); +static void ggml_cl_release_mem(cl_mem & mem) { + if (mem != nullptr) { + CL_CHECK(clReleaseMemObject(mem)); + mem = nullptr; + } +} - GGML_ASSERT(src0->type == src1->type); - GGML_ASSERT(src0->type == dst->type); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); +static void ggml_cl_adreno_xmem_attn_release_scratch(ggml_backend_opencl_context * backend_ctx) { + auto & s = backend_ctx->adreno_xmem_attn.scratch; + ggml_cl_release_mem(s.q_img); + ggml_cl_release_mem(s.k_img); + ggml_cl_release_mem(s.v_img); + ggml_cl_release_mem(s.out_img); + ggml_cl_release_mem(s.k_transpose_img1d); + ggml_cl_release_mem(s.k_transpose_buf); + ggml_cl_release_mem(s.k_packed_buf); + ggml_cl_release_mem(s.v_packed_buf); + ggml_cl_release_mem(s.score_img1d); + ggml_cl_release_mem(s.prob_img1d); + ggml_cl_release_mem(s.score_buf); + ggml_cl_release_mem(s.prob_buf); + ggml_cl_release_mem(s.softmax_stats_img2d); + ggml_cl_release_mem(s.xmem_qk); + ggml_cl_release_mem(s.xmem_pv); + s = {}; +} - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; +static ggml_cl_adreno_xmem_attn_schedule ggml_cl_adreno_xmem_attn_select_schedule( + const ggml_backend_opencl_context * backend_ctx, + int n_q, + int n_kv, + int heads_total, + int q_width, + int gqa_ratio) { + const bool big_h = heads_total >= 8; + ggml_cl_adreno_xmem_attn_schedule sched; + + if (gqa_ratio == 1) { + if (n_q >= 512) { sched.qk_lws0 = 512; } + else if (n_q >= 256) { sched.qk_lws0 = 128; } + else { sched.qk_lws0 = 64; } + sched.qk_lws2 = (big_h && n_q >= 512) ? 2 : 1; + } else { + if (q_width >= 2048) { sched.qk_lws0 = 512; } + else if (q_width >= 256) { sched.qk_lws0 = 128; } + else { sched.qk_lws0 = 64; } + sched.qk_lws2 = MIN(8, (int) backend_ctx->max_workgroup_size / sched.qk_lws0); + } + + if (n_kv >= 2048) { sched.softmax_reduce_lws0 = 1024; } + else if (n_kv >= 512) { sched.softmax_reduce_lws0 = big_h ? 256 : 512; } + else { sched.softmax_reduce_lws0 = 256; } + + if (n_kv < 256) { sched.softmax_apply_lws0 = 64; } + else { sched.softmax_apply_lws0 = big_h ? 128 : 64; } + sched.softmax_apply_lws2 = n_kv >= 512 ? 8 : 4; + + if (n_q < 256) { sched.pv_lws0 = 64; } + else { sched.pv_lws0 = big_h ? 128 : 64; } + sched.pv_lws2 = big_h ? 8 : (n_q <= 256 ? 8 : 4); + + const int max_wg = (int) backend_ctx->max_workgroup_size; + auto fix = [&](int & l0, int & l2) { + while (l0 * l2 > max_wg) { + if (l2 > 1) { l2 /= 2; } + else if (l0 > 32) { l0 /= 2; } + else { break; } + } + }; + fix(sched.qk_lws0, sched.qk_lws2); + fix(sched.softmax_apply_lws0, sched.softmax_apply_lws2); + fix(sched.pv_lws0, sched.pv_lws2); + while (sched.softmax_reduce_lws0 > max_wg) { + sched.softmax_reduce_lws0 /= 2; + } - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + return sched; +} - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; UNUSED(ne13); +static bool ggml_cl_adreno_xmem_attn_prepare( + ggml_backend_opencl_context * backend_ctx, + int n_q, + int n_kv, + int d_head_q, + int d_head_v, + int n_head, + int n_head_kv, + int n_batch) { + auto & s = backend_ctx->adreno_xmem_attn.scratch; + const int gqa_ratio = n_head / n_head_kv; + const int q_width = n_q * gqa_ratio; + const int kv_heads_total = n_head_kv * n_batch; + const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); + if (s.q_img != nullptr && + s.n_q == n_q && + s.n_kv == n_kv && + s.n_kv_padded == n_kv_padded && + s.d_head_q == d_head_q && + s.d_head_v == d_head_v && + s.q_width == q_width && + s.kv_heads_total == kv_heads_total) { + return true; + } + + ggml_cl_adreno_xmem_attn_release_scratch(backend_ctx); + + const int qpack = d_head_q / 4; + const int vpack = d_head_v / 4; + const int npack = n_kv_padded / 4; + const size_t q_img_h = (size_t) kv_heads_total * qpack; + const size_t v_img_h = (size_t) kv_heads_total * vpack; + + s.q_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) q_width, q_img_h); + s.k_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) n_kv_padded, q_img_h); + s.v_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) n_kv_padded, v_img_h); + s.out_img = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, (size_t) q_width, v_img_h); + + const size_t k_transpose_half4_elems = (size_t) npack * kv_heads_total * d_head_q; + s.k_transpose_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_transpose_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); + GGML_ASSERT(s.k_transpose_buf != nullptr); + s.k_transpose_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, k_transpose_half4_elems, s.k_transpose_buf); + + const size_t k_groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_q, 16); + const size_t v_groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_v, 16); + const size_t k_packed_half4_elems = (size_t) n_kv_padded * k_groups16 * 4; + const size_t v_packed_half4_elems = (size_t) n_kv_padded * v_groups16 * 4; + s.k_packed_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_packed_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); + s.v_packed_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, v_packed_half4_elems * sizeof(uint16_t) * 4, nullptr, nullptr); + GGML_ASSERT(s.k_packed_buf != nullptr && s.v_packed_buf != nullptr); + + const size_t score_half4_elems = (size_t) npack * kv_heads_total * q_width; + const size_t score_bytes = score_half4_elems * sizeof(uint16_t) * 4; + s.score_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, score_bytes, nullptr, nullptr); + s.prob_buf = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, score_bytes, nullptr, nullptr); + GGML_ASSERT(s.score_buf != nullptr && s.prob_buf != nullptr); + s.score_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, score_half4_elems, s.score_buf); + s.prob_img1d = ggml_cl_make_image1d_buffer_half4(backend_ctx->context, CL_MEM_READ_ONLY, score_half4_elems, s.prob_buf); + s.softmax_stats_img2d = ggml_cl_make_image2d_half4(backend_ctx->context, CL_MEM_READ_WRITE, + (size_t) q_width, (size_t) kv_heads_total); + s.xmem_qk = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY, 6144, nullptr, nullptr); + s.xmem_pv = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY, 6144, nullptr, nullptr); + GGML_ASSERT(s.softmax_stats_img2d != nullptr && s.xmem_qk != nullptr && s.xmem_pv != nullptr); + + s.n_q = n_q; + s.n_kv = n_kv; + s.n_kv_padded = n_kv_padded; + s.d_head_q = d_head_q; + s.d_head_v = d_head_v; + s.q_width = q_width; + s.kv_heads_total = kv_heads_total; + return true; +} - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; UNUSED(nb13); +static bool ggml_cl_adreno_xmem_attn_can_use( + const ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * q, + const ggml_tensor * k, + const ggml_tensor * dst) { + static const char * xmem_sdpa_env = getenv("GGML_OPENCL_XMEM_SDPA"); + if (xmem_sdpa_env == nullptr || xmem_sdpa_env[0] == '0') { + return false; + } - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + if (!backend_ctx->adreno_xmem_attn.compiled || backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { + return false; + } + if (q->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || + (k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_F32) || + (v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_F32)) { + return false; + } + if (sinks != nullptr) { + return false; + } + if (q->nb[0] != ggml_type_size(q->type) || k->nb[0] != ggml_type_size(k->type) || + v->nb[0] != ggml_type_size(v->type) || dst->nb[0] != ggml_type_size(dst->type)) { + return false; + } + if (mask != nullptr && (mask->type != GGML_TYPE_F16 || mask->nb[0] != sizeof(ggml_fp16_t))) { + return false; + } - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const int n_q = q->ne[1]; + const int n_kv = k->ne[1]; + const int d_head_q = q->ne[0]; + const int d_head_v = v->ne[0]; + const int n_head = q->ne[2]; + const int n_head_kv = k->ne[2]; + const int n_batch = q->ne[3]; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + if (n_q <= 1 || n_kv <= 0 || n_kv > 8192) { + return false; + } + if (d_head_q != k->ne[0] || d_head_v != v->ne[0] || k->ne[1] != v->ne[1] || k->ne[3] != v->ne[3]) { + return false; + } + if (q->ne[3] != k->ne[3]) { + return false; + } + if (n_head_kv <= 0 || n_head % n_head_kv != 0 || k->ne[2] != v->ne[2]) { + return false; + } + if (dst->ne[0] != d_head_v || dst->ne[1] != n_head || dst->ne[2] != n_q || dst->ne[3] != n_batch) { + return false; + } + if ((d_head_q % 8) != 0 || (d_head_v % 32) != 0) { + return false; + } + if (mask != nullptr && + (mask->ne[0] < n_kv || mask->ne[1] < n_q || mask->ne[2] <= 0 || mask->ne[3] <= 0)) { + return false; + } - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + float params[3]; + memcpy(params, dst->op_params, sizeof(params)); + if (params[1] != 0.0f || params[2] != 0.0f) { + return false; + } - bool bcast_row = false; - cl_kernel kernel; + const int gqa_ratio = n_head / n_head_kv; + const int q_width = n_q * gqa_ratio; + const int kv_heads_total = n_head_kv * n_batch; + const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); + const int qpack = d_head_q / 4; + const int vpack = d_head_v / 4; + const int npack = n_kv_padded / 4; - if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { - GGML_ASSERT(ggml_is_contiguous(src0)); + if ((size_t) q_width > backend_ctx->image2d_max_width || + (size_t) n_kv_padded > backend_ctx->image2d_max_width) { + return false; + } + if ((size_t) kv_heads_total * (size_t) qpack > backend_ctx->image2d_max_height || + (size_t) kv_heads_total * (size_t) vpack > backend_ctx->image2d_max_height) { + return false; + } + if ((size_t) npack * (size_t) kv_heads_total * (size_t) d_head_q > backend_ctx->image_max_buffer_size || + (size_t) npack * (size_t) kv_heads_total * (size_t) q_width > backend_ctx->image_max_buffer_size) { + return false; + } - // src1 is a row - GGML_ASSERT(ne11 == 1); + return true; +} - bcast_row = true; - int ne = ne00 / 4; +static void ggml_cl_adreno_xmem_attn_run( + ggml_backend_t backend, + const ggml_tensor * q, + const ggml_tensor * k, + ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + auto & xstate = backend_ctx->adreno_xmem_attn; + auto & s = xstate.scratch; + if (!xstate.logged) { + GGML_LOG_INFO("ggml_opencl: using Adreno xmem attention path\n"); + xstate.logged = true; + } - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_mul_row; - } else { - kernel = backend_ctx->kernel_mul_row_f16; - } + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_mul; - } else { - kernel = backend_ctx->kernel_mul_f16; - } + ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *) q->extra; + ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *) k->extra; + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *) v->extra; + ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *) dst->extra; + ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *) mask->extra : nullptr; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nb3)); + const cl_ulong offset_q = extra_q->offset + q->view_offs; + const cl_ulong offset_k = extra_k->offset + k->view_offs; + const cl_ulong offset_v = extra_v->offset + v->view_offs; + const cl_ulong offset_o = extra_o->offset + dst->view_offs; + const cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; + + const int n_q = q->ne[1]; + const int n_kv = k->ne[1]; + const int d_head_q = q->ne[0]; + const int d_head_v = v->ne[0]; + const int n_head = q->ne[2]; + const int n_head_kv = k->ne[2]; + const int n_batch = q->ne[3]; + const int heads_total = n_head * n_batch; + const int gqa_ratio = n_head / n_head_kv; + const int q_width = n_q * gqa_ratio; + const int kv_heads_total = n_head_kv * n_batch; + const int n_kv_padded = (int) ggml_cl_round_up((size_t) n_kv, 32); + const int qpack = d_head_q / 4; + const int opack = d_head_v / 4; + const int npack = n_kv_padded / 4; + const float scale = ((const float *) dst->op_params)[0]; + + GGML_ASSERT(ggml_cl_adreno_xmem_attn_prepare( + backend_ctx, n_q, n_kv, d_head_q, d_head_v, n_head, n_head_kv, n_batch)); + const ggml_cl_adreno_xmem_attn_schedule sched = + ggml_cl_adreno_xmem_attn_select_schedule( + backend_ctx, n_q, n_kv_padded, heads_total, q_width, gqa_ratio); + + { + size_t gws[3] = {ggml_cl_round_up((size_t) n_q, 8), (size_t) heads_total, (size_t) qpack}; + size_t lws[3] = {8, 1, (size_t) ((qpack <= 32) ? qpack : 1)}; + cl_kernel kernel = xstate.kernel_q_f32_to_img_scaled; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.q_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &d_head_q)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &q->nb[1])); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &q->nb[2])); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &q->nb[3])); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } - if (bcast_row) { - int n = ggml_nelements(dst)/4; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + { + size_t gws[3] = {(size_t) n_kv_padded, (size_t) kv_heads_total, (size_t) qpack}; + size_t lws[3] = {8, 1, (size_t) ((qpack <= 32) ? qpack : 1)}; + cl_kernel kernel = k->type == GGML_TYPE_F16 ? + xstate.kernel_kv_f16_to_img_gqa : xstate.kernel_kv_f32_to_img_gqa; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_k->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_k)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.k_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_q)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_kv_padded)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &k->nb[1])); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &k->nb[2])); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &k->nb[3])); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. + { + size_t gws[3] = {(size_t) n_kv_padded, (size_t) kv_heads_total, (size_t) opack}; + size_t lws[3] = {8, 1, (size_t) ((opack <= 32) ? opack : 1)}; + cl_kernel kernel = v->type == GGML_TYPE_F16 ? + xstate.kernel_kv_f16_to_img_gqa : xstate.kernel_kv_f32_to_img_gqa; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_v->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.v_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_v)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_kv_padded)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &v->nb[1])); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &v->nb[2])); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &v->nb[3])); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + + { + size_t gws[3] = {(size_t) d_head_q, (size_t) kv_heads_total, (size_t) npack}; + size_t lws[3] = {(size_t) MIN(64, d_head_q), (size_t) (kv_heads_total >= 2 ? 2 : 1), (size_t) MIN(8, npack)}; + if (lws[0] * lws[1] * lws[2] > backend_ctx->max_workgroup_size) { + lws[1] = 1; } + cl_kernel kernel = xstate.kernel_k_gather; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.k_transpose_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_img)); + ggml_cl_set_arg_int4(kernel, 2, n_kv_padded, kv_heads_total, npack, d_head_q); + ggml_cl_set_arg_int4(kernel, 3, qpack, 0, 0, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + { + const size_t groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_q, 16); + const size_t packed_linear = (size_t) n_kv_padded * groups16; + const size_t lws0 = MIN((size_t) 1024, backend_ctx->max_workgroup_size); + size_t gws[3] = {ggml_cl_round_up(packed_linear, lws0), 1, 1}; + size_t lws[3] = {lws0, 1, 1}; + cl_kernel kernel = xstate.kernel_pack_k; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.k_packed_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_transpose_img1d)); + ggml_cl_set_arg_int4(kernel, 2, 8, (int) packed_linear, qpack, d_head_q); + ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, kv_heads_total, kv_heads_total, npack); + ggml_cl_set_arg_int4(kernel, 4, d_head_q, 0, 0, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); - } else { - unsigned int nth = MIN(64, ne0); - size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {nth, 1, 1}; + { + size_t lws[3] = {(size_t) sched.qk_lws0, 1, (size_t) sched.qk_lws2}; + const int slices_per_group = sched.qk_lws2 * 8; + const size_t groups_z = (size_t) ggml_cl_round_up_div(npack, slices_per_group); + const size_t groups_x = (size_t) ggml_cl_round_up_div(q_width, sched.qk_lws0); + size_t gws[3] = { + lws[0] * groups_z, + groups_x, + (size_t) kv_heads_total * lws[2], + }; + + cl_kernel kernel = xstate.kernel_qk_gemm; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.score_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.k_packed_buf)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.xmem_qk)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &s.q_img)); + ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, npack, q_width, 32); + ggml_cl_set_arg_int4(kernel, 5, qpack, 0, 0, kv_heads_total); + ggml_cl_set_arg_int4(kernel, 6, qpack, 1, 1, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + cl_mem softmax_input_img = s.score_img1d; + cl_mem softmax_output_buf = s.prob_buf; + cl_mem pv_prob_img = s.prob_img1d; + + if (mask != nullptr) { + const cl_ulong mask_nb1 = mask->nb[1]; + const cl_ulong mask_nb2 = mask->nb[2]; + const cl_ulong mask_nb3 = mask->nb[3]; + const int mask_ne2 = mask->ne[2]; + const int mask_ne3 = mask->ne[3]; + size_t lws[3] = {(size_t) sched.softmax_apply_lws0, 1, (size_t) sched.softmax_apply_lws2}; + size_t gws[3] = { + ggml_cl_round_up((size_t) q_width, lws[0]), + (size_t) kv_heads_total, + ggml_cl_round_up((size_t) npack, lws[2]), + }; + cl_kernel kernel = xstate.kernel_mask_scores; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.prob_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.score_img1d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_mask->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &q_width)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_kv_padded)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &kv_heads_total)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &mask_ne3)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + + softmax_input_img = s.prob_img1d; + softmax_output_buf = s.score_buf; + pv_prob_img = s.score_img1d; + } + + { + size_t lws[3] = {(size_t) sched.softmax_reduce_lws0, 1, 1}; + size_t gws[3] = {ggml_cl_round_up((size_t) q_width, lws[0]), (size_t) kv_heads_total, 1}; + cl_kernel kernel = xstate.kernel_softmax_reduce_basic; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &softmax_input_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.softmax_stats_img2d)); + ggml_cl_set_arg_int4(kernel, 2, kv_heads_total, 1, q_width, n_kv); + ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, q_width, 0, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + { + size_t lws[3] = {(size_t) sched.softmax_apply_lws0, 1, (size_t) sched.softmax_apply_lws2}; + size_t gws[3] = { + ggml_cl_round_up((size_t) q_width, lws[0]), + (size_t) kv_heads_total, + ggml_cl_round_up((size_t) npack, lws[2]), + }; + cl_kernel kernel = xstate.kernel_softmax_apply_basic; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &softmax_output_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &softmax_input_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.softmax_stats_img2d)); + ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, npack, q_width, 1); + ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, q_width, n_kv, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + { + const size_t groups16 = (size_t) ggml_cl_round_up_div(kv_heads_total * d_head_v, 16); + const size_t packed_linear = (size_t) n_kv_padded * groups16; + const size_t lws0 = MIN((size_t) 1024, backend_ctx->max_workgroup_size); + size_t gws[3] = {ggml_cl_round_up(packed_linear, lws0), 1, 1}; + size_t lws[3] = {lws0, 1, 1}; + cl_kernel kernel = xstate.kernel_pack_v; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.v_packed_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.v_img)); + ggml_cl_set_arg_int4(kernel, 2, 8, (int) packed_linear, npack, n_kv_padded); + ggml_cl_set_arg_int4(kernel, 3, kv_heads_total, kv_heads_total, opack, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + + { + size_t lws[3] = {(size_t) sched.pv_lws0, 1, (size_t) sched.pv_lws2}; + const int blocks = ggml_cl_round_up_div(opack, 8); + const size_t groups_z = (size_t) ggml_cl_round_up_div(blocks, sched.pv_lws2); + const size_t groups_x = (size_t) ggml_cl_round_up_div(q_width, sched.pv_lws0); + size_t gws[3] = { + lws[0] * groups_z, + groups_x, + (size_t) kv_heads_total * lws[2], + }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + cl_kernel kernel = xstate.kernel_pv_gemm; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &s.v_packed_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s.xmem_pv)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &pv_prob_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &s.out_img)); + ggml_cl_set_arg_int4(kernel, 4, kv_heads_total, opack, q_width, 32); + ggml_cl_set_arg_int4(kernel, 5, npack, 0, 0, kv_heads_total); + ggml_cl_set_arg_int4(kernel, 6, kv_heads_total * q_width, npack, q_width, 1); + ggml_cl_set_arg_int4(kernel, 7, 1, 0, 0, 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); + } + + { + size_t gws[3] = {ggml_cl_round_up((size_t) n_q, 8), (size_t) heads_total, (size_t) opack}; + size_t lws[3] = {8, 1, (size_t) ((opack <= 32) ? opack : 1)}; + cl_kernel kernel = xstate.kernel_img_to_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_o->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_o)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &s.out_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &d_head_v)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &dst->nb[1])); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &dst->nb[2])); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &dst->nb[3])); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); } } -static void ggml_cl_div(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + GGML_ASSERT(q->extra); + GGML_ASSERT(k->extra); + GGML_ASSERT(v->extra); GGML_ASSERT(dst->extra); + if (mask) { + GGML_ASSERT(mask->extra); + } + if (sinks) { + GGML_ASSERT(sinks->extra); + } - GGML_ASSERT(src0->type == src1->type); - GGML_ASSERT(src0->type == dst->type); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_context context = backend_ctx->context; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + const int n_q = q->ne[1]; + const int n_kv = k->ne[1]; + const int d_head_q = q->ne[0]; + const int d_head_v = v->ne[0]; + const int n_head = q->ne[2]; + const int n_head_kv = k->ne[2]; + const int n_batch = q->ne[3]; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + const std::pair dk_dv = {d_head_q, d_head_v}; + cl_kernel kernel = backend_ctx->fa.kernel_flash_attn_f32_f16_bin; + GGML_ASSERT(kernel != NULL); - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; + ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; + ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *)k->extra; + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; + ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; + ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; + cl_ulong offset_q = extra_q->offset + q->view_offs; + cl_ulong offset_o = extra_o->offset + dst->view_offs; - const int ne0 = dst->ne[0]; + cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; + cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; + cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; + cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + const cl_ulong q_nb1 = q->nb[1]; + const cl_ulong q_nb2 = q->nb[2]; + const cl_ulong q_nb3 = q->nb[3]; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_mem k_data_device = extra_k->data_device; + cl_ulong offset_k = extra_k->offset + k->view_offs; + cl_ulong k_nb1 = k->nb[1]; + cl_ulong k_nb2 = k->nb[2]; + cl_ulong k_nb3 = k->nb[3]; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + cl_mem v_data_device = extra_v->data_device; + cl_ulong offset_v = extra_v->offset + v->view_offs; + cl_ulong v_nb1 = v->nb[1]; + cl_ulong v_nb2 = v->nb[2]; + cl_ulong v_nb3 = v->nb[3]; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + const cl_ulong o_nb1 = dst->nb[1]; + const cl_ulong o_nb2 = dst->nb[2]; + const cl_ulong o_nb3 = dst->nb[3]; - bool bcast_row = false; - cl_kernel kernel; + const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; + const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; + const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; + const int mask_ne2 = mask ? mask->ne[2] : 0; + const int mask_ne3 = mask ? mask->ne[3] : 0; - if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { - GGML_ASSERT(ggml_is_contiguous(src0)); + float * params = (float *)dst->op_params; + float scale = params[0]; + float max_bias = params[1]; + float logit_softcap = params[2]; - // src1 is a row - GGML_ASSERT(ne11 == 1); + const int is_causal = (mask == NULL && n_q > 1 && n_q == n_kv); // redundant n_q > 1 check ? - bcast_row = true; - int ne = ne00 / 4; + const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; + const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; + const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_div_row; - } else { - kernel = backend_ctx->kernel_div_row_f16; - } + const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_div; - } else { - kernel = backend_ctx->kernel_div_f16; - } + ggml_cl_flash_attn_temp_buffer temp_k; + ggml_cl_flash_attn_temp_buffer temp_v; + ggml_cl_flash_attn_temp_buffer temp_k_aos; + ggml_cl_flash_attn_temp_buffer temp_v_aos; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); - } + if (is_q8_0) { + ggml_cl_flash_attn_reconstruct_aos( + backend_ctx, k, temp_k_aos, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); - if (bcast_row) { - int n = ggml_nelements(dst)/4; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + ggml_cl_flash_attn_reconstruct_aos( + backend_ctx, v, temp_v_aos, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - unsigned int nth = MIN(64, ne0); - size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {nth, 1, 1}; + bool k_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, k, GGML_TYPE_F16, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, + temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + bool v_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, v, GGML_TYPE_F16, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, + temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + + GGML_ASSERT(k_done && v_done); } -} -static void ggml_cl_sub(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + // Allocate input/output memory buffers + cl_mem mem_matrixQ; + cl_mem mem_matrixK; + cl_mem mem_matrixV; + cl_mem mem_matrixO; + cl_buffer_region region; + cl_int err; - GGML_ASSERT(src0->type == src1->type); - GGML_ASSERT(src0->type == dst->type); - GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + region.origin = offset_q; + region.size = ggml_nbytes(q); + mem_matrixQ = clCreateSubBuffer(extra_q->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + region.origin = offset_k; + region.size = is_q8_0 ? (size_t) k_nb3 * (size_t) k->ne[3] : ggml_nbytes(k); + mem_matrixK = clCreateSubBuffer(k_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + region.origin = offset_v; + region.size = is_q8_0 ? (size_t) v_nb3 * (size_t) v->ne[3] : ggml_nbytes(v); + mem_matrixV = clCreateSubBuffer(v_data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; + region.origin = offset_o; + region.size = ggml_nbytes(dst); + mem_matrixO = clCreateSubBuffer(extra_o->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; + cl_image_format img_fmt_1d = { CL_RGBA, CL_FLOAT}; + cl_image_desc img_desc_1d; - const int ne0 = dst->ne[0]; + // use image 1d buffer used as fallback when on mask is applied + cl_mem mem_tex_mask_fallback_1dbuf; + img_fmt_1d = { CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_1d.image_width = 1; + img_desc_1d.buffer = mem_matrixK; + mem_tex_mask_fallback_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); + CL_CHECK(err); - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + cl_mem mem_tex_matrixO_1dbuf; + img_fmt_1d = { CL_RGBA, CL_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_1d.image_width = ggml_nbytes(dst) / 4 / 4; + img_desc_1d.buffer = mem_matrixO; + mem_tex_matrixO_1dbuf = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &err); + CL_CHECK(err); + + // The bin kernel requires 2d (or 3d) buffers packed for data loading/multiplication. + // These repack kernels launch across all buffers to ensure compatibility + cl_mem mem_tex_matrixMask_1dbuf = NULL; + cl_mem mem_matrixMask = NULL; + cl_mem mem_matrixMask_padded = NULL; + cl_ulong mask_nb1_padded = mask_nb1, mask_nb2_padded = mask_nb2, mask_nb3_padded = mask_nb3; + if (extra_mask) { + // allocate mem_matrixMask w/ new padded size + size_t n_kv_padded = GGML_PAD(n_kv, 4); + size_t mask_nb_padded = n_kv_padded * sizeof(cl_half) * mask->ne[1] * mask->ne[2] * mask->ne[3]; + + // apply offset and create subBuffer for mask + region.origin = offset_mask; + region.size = ggml_nbytes(mask); + mem_matrixMask = clCreateSubBuffer(extra_mask->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + { + // create padded mask to contain all data + mem_matrixMask_padded = clCreateBuffer(context, CL_MEM_ALLOC_HOST_PTR, mask_nb_padded, NULL, &err); + CL_CHECK(err); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // pass extra_mask->data_device, mem_matrixMask to kernel for copying/padding + mask_nb1_padded = (cl_ulong)n_kv_padded * sizeof(cl_half); + mask_nb2_padded = mask_nb1_padded * (cl_ulong)mask->ne[1]; + mask_nb3_padded = mask_nb2_padded * (cl_ulong)mask->ne[2]; + + cl_kernel repack_mask = backend_ctx->fa.kernel_repack_mask_for_wmm; + CL_CHECK(clSetKernelArg(repack_mask, 0, sizeof(cl_mem), &mem_matrixMask)); + CL_CHECK(clSetKernelArg(repack_mask, 1, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(repack_mask, 2, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(repack_mask, 3, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(repack_mask, 4, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(repack_mask, 5, sizeof(cl_mem), &mem_matrixMask_padded)); + CL_CHECK(clSetKernelArg(repack_mask, 6, sizeof(cl_ulong), &mask_nb1_padded)); + CL_CHECK(clSetKernelArg(repack_mask, 7, sizeof(cl_ulong), &mask_nb2_padded)); + CL_CHECK(clSetKernelArg(repack_mask, 8, sizeof(cl_ulong), &mask_nb3_padded)); + + size_t repack_mask_gws[3] = {(size_t)n_kv, (size_t)mask->ne[1], (size_t)mask_ne2 * (size_t)mask->ne[3]}; + backend_ctx->enqueue_ndrange_kernel(repack_mask, 3, repack_mask_gws, NULL, dst); + } - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + // use image 1d buffer for matrix Mask (padded row stride) + cl_image_format img_fmt_mask_1d = { CL_RGBA, CL_HALF_FLOAT}; + cl_image_desc img_desc_mask_1d; + memset(&img_desc_mask_1d, 0, sizeof(img_desc_mask_1d)); + img_desc_mask_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_mask_1d.image_width = mask_nb_padded / 2 / 4; + img_desc_mask_1d.buffer = mem_matrixMask_padded; + mem_tex_matrixMask_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_mask_1d, &img_desc_mask_1d, NULL, &err); + CL_CHECK(err); + } - bool bcast_row = false; - cl_kernel kernel; + // WMM QK uses repacked 3D images. + // Q image: rows, heads, packed depth. + cl_image_format img_fmt_3d = { CL_RGBA, CL_HALF_FLOAT }; + cl_image_desc img_desc_3d; - if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) { - GGML_ASSERT(ggml_is_contiguous(src0)); + memset(&img_desc_3d, 0, sizeof(img_desc_3d)); + img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; + img_desc_3d.image_width = (size_t)n_q; + img_desc_3d.image_height = (size_t)n_batch * (size_t)n_head; + img_desc_3d.image_depth = (size_t)d_head_q / 4; + cl_mem img_q_wmm = NULL; + img_q_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + CL_CHECK(err); - // src1 is a row - GGML_ASSERT(ne11 == 1); + { + cl_kernel repack_q = backend_ctx->fa.kernel_repack_q_for_wmm; + CL_CHECK(clSetKernelArg(repack_q, 0, sizeof(cl_mem), &mem_matrixQ)); + CL_CHECK(clSetKernelArg(repack_q, 1, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(repack_q, 2, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(repack_q, 3, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(repack_q, 4, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(repack_q, 5, sizeof(cl_mem), &img_q_wmm)); + + size_t repack_q_gws[3] = {(size_t)d_head_q / 4, (size_t)n_q, (size_t)n_batch * (size_t)n_head}; + backend_ctx->enqueue_ndrange_kernel(repack_q, 3, repack_q_gws, NULL, dst); + } + + // K image: columns, row groups, KV heads. + const size_t n_kv_row4 = ((size_t)n_kv + 3) / 4; + + memset(&img_desc_3d, 0, sizeof(img_desc_3d)); + img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; + img_desc_3d.image_width = (size_t)d_head_q; + img_desc_3d.image_height = n_kv_row4; + img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; + cl_mem img_k_wmm = NULL; + img_k_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + CL_CHECK(err); - bcast_row = true; - int ne = ne00 / 4; + { + cl_kernel repack_k = backend_ctx->fa.kernel_repack_k_for_wmm; + CL_CHECK(clSetKernelArg(repack_k, 0, sizeof(cl_mem), &mem_matrixK)); + CL_CHECK(clSetKernelArg(repack_k, 1, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(repack_k, 2, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(repack_k, 3, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(repack_k, 4, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(repack_k, 5, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(repack_k, 6, sizeof(cl_mem), &img_k_wmm)); + + size_t repack_k_gws[3] = {(size_t)d_head_q, n_kv_row4, (size_t)n_batch * (size_t)n_head_kv}; + backend_ctx->enqueue_ndrange_kernel(repack_k, 3, repack_k_gws, NULL, dst); + } + + // V image: kv-rows (contracted), packed head-dim groups, KV heads. + memset(&img_desc_3d, 0, sizeof(img_desc_3d)); + img_desc_3d.image_type = CL_MEM_OBJECT_IMAGE3D; + img_desc_3d.image_width = (size_t)n_kv; + img_desc_3d.image_height = (size_t)d_head_v / 4; + img_desc_3d.image_depth = (size_t)n_batch * (size_t)n_head_kv; + cl_mem img_v_wmm = NULL; + img_v_wmm = clCreateImage(context, CL_MEM_READ_WRITE, &img_fmt_3d, &img_desc_3d, NULL, &err); + CL_CHECK(err); - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sub_row; - } else { - kernel = backend_ctx->kernel_sub_row_f16; - } + { + cl_kernel repack_v = backend_ctx->fa.kernel_repack_v_for_wmm; + CL_CHECK(clSetKernelArg(repack_v, 0, sizeof(cl_mem), &mem_matrixV)); + CL_CHECK(clSetKernelArg(repack_v, 1, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(repack_v, 2, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(repack_v, 3, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(repack_v, 4, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(repack_v, 5, sizeof(cl_mem), &img_v_wmm)); + + size_t repack_v_gws[3] = {(size_t)d_head_v / 4, (size_t)n_kv, (size_t)n_batch * (size_t)n_head_kv}; + backend_ctx->enqueue_ndrange_kernel(repack_v, 3, repack_v_gws, NULL, dst); + } + + cl_int enable_mask = (extra_mask) ? 1 : 0; + mask_buffer = extra_mask ? mem_tex_matrixMask_1dbuf : mem_tex_mask_fallback_1dbuf; + + cl_mem mem_sinksBuf = NULL; + cl_mem mem_tex_sinks_1dbuf = NULL; + cl_int enable_sinks = (sinks_buffer != NULL) ? 1 : 0; + if (enable_sinks) { + region.origin = offset_sinks; + region.size = ggml_nbytes(sinks); + mem_sinksBuf = clCreateSubBuffer(extra_sinks->data_device, CL_MEM_READ_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne)); + cl_image_format img_fmt_sinks_1d = { CL_R, CL_FLOAT }; + cl_image_desc img_desc_sinks_1d; + memset(&img_desc_sinks_1d, 0, sizeof(img_desc_sinks_1d)); + img_desc_sinks_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_sinks_1d.image_width = (size_t)n_head; + img_desc_sinks_1d.buffer = mem_sinksBuf; + mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_1d, &img_desc_sinks_1d, NULL, &err); + CL_CHECK(err); } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sub; - } else { - kernel = backend_ctx->kernel_sub_f16; - } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); + // The image obj cannot be null so we back with buffer of size 1 and use matrixK to back because it always exists + cl_image_format img_fmt_sinks_fallback = { CL_R, CL_FLOAT }; + cl_image_desc img_desc_sinks_fallback; + memset(&img_desc_sinks_fallback, 0, sizeof(img_desc_sinks_fallback)); + img_desc_sinks_fallback.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_sinks_fallback.image_width = 1; + img_desc_sinks_fallback.buffer = mem_matrixK; + mem_tex_sinks_1dbuf = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_sinks_fallback, &img_desc_sinks_fallback, NULL, &err); + CL_CHECK(err); } - if (bcast_row) { - int n = ggml_nelements(dst)/4; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + cl_uint arg = 0; + + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_matrixO_1dbuf)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &is_causal)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &o_nb3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_log2_val)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &logit_softcap)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_mask)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb1_padded)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb2_padded)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &mask_nb3_padded)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &mask_ne3)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &mem_tex_sinks_1dbuf)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &enable_sinks)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_q_wmm)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_k_wmm)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &img_v_wmm)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(int), &d_head_q)); + + size_t global_work_size[3], local_work_size[3]; + + const int n_waves_v = d_head_q / 64; + + local_work_size[0] = 64; + local_work_size[1] = n_waves_v; + local_work_size[2] = 1; + + global_work_size[0] = 64; + global_work_size[1] = ((n_q + 64 - 1) / 64) * n_waves_v; + global_work_size[2] = n_batch * n_head; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - unsigned int nth = MIN(64, ne0); - size_t global_work_size[] = {ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {nth, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(mem_tex_matrixO_1dbuf)); + CL_CHECK(clReleaseMemObject(img_q_wmm)); + CL_CHECK(clReleaseMemObject(img_k_wmm)); + CL_CHECK(clReleaseMemObject(img_v_wmm)); + + if (mem_tex_matrixMask_1dbuf) { + CL_CHECK(clReleaseMemObject(mem_tex_matrixMask_1dbuf)); + } + if (mem_matrixMask) { + CL_CHECK(clReleaseMemObject(mem_matrixMask)); + } + if (mem_matrixMask_padded) { + CL_CHECK(clReleaseMemObject(mem_matrixMask_padded)); + } + if (mem_tex_sinks_1dbuf) { + CL_CHECK(clReleaseMemObject(mem_tex_sinks_1dbuf)); } + if (mem_sinksBuf) { + CL_CHECK(clReleaseMemObject(mem_sinksBuf)); + } + CL_CHECK(clReleaseMemObject(mem_matrixQ)); + CL_CHECK(clReleaseMemObject(mem_matrixK)); + CL_CHECK(clReleaseMemObject(mem_matrixV)); + CL_CHECK(clReleaseMemObject(mem_matrixO)); } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_sqr(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); +static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { + const ggml_tensor * v = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + + GGML_ASSERT(q->extra); + GGML_ASSERT(k->extra); + GGML_ASSERT(v->extra); GGML_ASSERT(dst->extra); - UNUSED(src1); + + if (mask) { + GGML_ASSERT(mask->extra); + } + if (sinks) { + GGML_ASSERT(sinks->extra); + } ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + const int n_q = q->ne[1]; + const int n_kv = k->ne[1]; + const int d_head_q = q->ne[0]; + const int d_head_v = v->ne[0]; + const int n_head = q->ne[2]; + const int n_head_kv = k->ne[2]; + const int n_batch = q->ne[3]; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (ggml_cl_adreno_xmem_attn_can_use(backend_ctx, q, k, dst)) { + ggml_cl_adreno_xmem_attn_run(backend, q, k, dst); + return; + } +#endif - cl_kernel kernel; + // Compile DK512 decode separately from the prefill programs. + const bool fa_decode_only_512 = (d_head_q == 512); - // Currently assumes src0 is contiguous - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sqr_cont_f32_4; + // per-variant lazy compile for this (dk, dv) + // DK=512 decode (n_q==1) needs no prepass + // DK=512 prefill (n_q>1) does, so compile it only when needed + if (!fa_decode_only_512 || n_q > 1) { + ggml_opencl_ensure_fa_pre_kernels(backend_ctx, d_head_q, d_head_v); + } + + cl_kernel kernel = NULL; + bool use_prefill_k_img = false; // K is image1d_buffer_t for DK=512 prefill + + const bool is_f16 = q->type == GGML_TYPE_F16; + const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16; + const bool is_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 && v->type == GGML_TYPE_Q8_0; + const bool is_q4_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q4_0 && v->type == GGML_TYPE_Q4_0; + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_fa_bin_kernels_prefill(backend_ctx, q, k, v)) { + // We support the prefill path of flash attn with a specialized d_head = 64/128/256 + ggml_cl_flash_attn_prefill_bin(backend, q, k, dst); + return; + } +#endif + + if (is_f16) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F16); + } else if (is_mixed) { + if (fa_decode_only_512 && n_q == 1) { + GGML_ASSERT(ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)); } else { - kernel = backend_ctx->kernel_sqr_cont_f16_4; + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16); } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sqr_cont_f32; + if (fa_decode_only_512) { + // DK=512: the BM-tile prefill kernels are specifically compiled from + // FA_PREFILL_ONLY + if (n_q > 1) { + ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false); + ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/true); + } } else { - kernel = backend_ctx->kernel_sqr_cont_f16; + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16_SPLIT); + } + } else if (is_q8_0) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0); + if (d_head_q == 96 && d_head_v == 96) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/true); + } else if (d_head_q == 256 && d_head_v == 256) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/true); + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q8_0_SPLIT); + } + } else if (is_q4_0) { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0); + if (d_head_q == 96 && d_head_v == 96) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 96, 96, /*quant_bm=*/16, /*quant_n_split=*/3, /*is_q8_0=*/false); + } else if (d_head_q == 256 && d_head_v == 256) { + ggml_opencl_ensure_fa_quant_split_override(backend_ctx, 256, 256, /*quant_bm=*/16, /*quant_n_split=*/8, /*is_q8_0=*/false); + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_Q4_0_SPLIT); } + } else { + ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + const std::pair dk_dv = {d_head_q, d_head_v}; + const int mq_gqa = n_head_kv > 0 ? n_head / n_head_kv : 0; + const std::tuple mq_decode_key = {d_head_q, d_head_v, mq_gqa}; + const bool mq_decode_shape = backend_ctx->gpu_family == ADRENO && is_mixed && n_q == 1 && + d_head_q == d_head_v && n_head_kv > 0 && n_head % n_head_kv == 0 && + ((((d_head_q == 64 && mq_gqa == 8) || (d_head_q == 128 && mq_gqa == 4)) && + backend_ctx->has_subgroup_shuffle) || + ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8))); + if (mq_decode_shape && n_kv >= 32) { + ggml_opencl_ensure_fa_f32_f16_mq_decode(backend_ctx, d_head_q, d_head_v, mq_gqa); + } + const bool use_native_q8_0_q1 = is_q8_0 && n_q == 1 && + backend_ctx->fa.f32_q8_0_q1.count(dk_dv) > 0; + // Native q8_0 prefill — reads q8_0 directly, wg_size = cfg->bm. + const bool use_native_q8_0 = is_q8_0 && n_q > 1 && + backend_ctx->fa.f32_q8_0.count(dk_dv) > 0; + const bool use_native_q4_0_q1 = is_q4_0 && n_q == 1 && + backend_ctx->fa.f32_q4_0_q1.count(dk_dv) > 0; + const bool use_native_q4_0 = is_q4_0 && n_q > 1 && + backend_ctx->fa.f32_q4_0.count(dk_dv) > 0; + const int block_m = n_q > 1 + ? (is_mixed ? backend_ctx->fa.f32_f16_bm.at(dk_dv) : backend_ctx->fa.bm.at(dk_dv)) + : 0; + // block_n is only used by the n_q > 1 prefill path; its map is not + // populated for DK=512 decode, so do not read it for decode. + const int block_n = (n_q > 1) + ? (is_mixed ? backend_ctx->fa.f32_f16_bn.at(dk_dv) + : backend_ctx->fa.bn.at(dk_dv)) + : 0; + // Pick split variant only when n_kv crosses the per-(dk,dv) threshold. + // the N_SPLIT>1 prefill tile reduces DK partials via subgroup shuffle, + // on Intel it uses the non-split BM tile and does not depend on subgroup size + const bool use_split_kernel = (n_q > 1 && is_mixed && + backend_ctx->gpu_family != INTEL && + backend_ctx->fa.f32_f16_split.count(dk_dv) > 0 && + n_kv >= backend_ctx->fa.f32_f16_split_nkv_threshold.at(dk_dv)); + const bool use_split_q8_0 = (use_native_q8_0 && backend_ctx->gpu_family != INTEL && + backend_ctx->fa.f32_q8_0_split.count(dk_dv) > 0 && + n_kv >= backend_ctx->fa.f32_q8_0_split_nkv_threshold.at(dk_dv)); + const bool use_split_q4_0 = (use_native_q4_0 && backend_ctx->gpu_family != INTEL && + backend_ctx->fa.f32_q4_0_split.count(dk_dv) > 0 && + n_kv >= backend_ctx->fa.f32_q4_0_split_nkv_threshold.at(dk_dv)); + const int wg_size_fa = (n_q > 1 && is_mixed) + ? (use_split_kernel + ? backend_ctx->fa.f32_f16_split_wg_size.at(dk_dv) + : backend_ctx->fa.f32_f16_wg_size.at(dk_dv)) + : block_m; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; + ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; + ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } + // SoA q8_0/q4_0 K/V: data_device aliases the `q` subbuffer; reconstruct + // AoS into a temp buffer below. AoS tensors use extra_k/v->data_device. + const bool k_soa = ggml_cl_is_q8_0_soa(k) || ggml_cl_is_q4_0_soa(k); + const bool v_soa = ggml_cl_is_q8_0_soa(v) || ggml_cl_is_q4_0_soa(v); + ggml_tensor_extra_cl * extra_k = k_soa ? nullptr : (ggml_tensor_extra_cl *)k->extra; + ggml_tensor_extra_cl * extra_v = v_soa ? nullptr : (ggml_tensor_extra_cl *)v->extra; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); -} + cl_ulong offset_q = extra_q->offset + q->view_offs; + cl_ulong offset_k = k_soa ? 0 : extra_k->offset + k->view_offs; + cl_ulong offset_v = v_soa ? 0 : extra_v->offset + v->view_offs; + cl_ulong offset_o = extra_o->offset + dst->view_offs; + cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; + cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; + cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; + cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; -static void ggml_cl_sqrt(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - UNUSED(src1); + const cl_ulong q_nb1 = q->nb[1]; + const cl_ulong q_nb2 = q->nb[2]; + const cl_ulong q_nb3 = q->nb[3]; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_ulong k_nb1 = k->nb[1]; + cl_ulong k_nb2 = k->nb[2]; + cl_ulong k_nb3 = k->nb[3]; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong v_nb1 = v->nb[1]; + cl_ulong v_nb2 = v->nb[2]; + cl_ulong v_nb3 = v->nb[3]; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + const cl_ulong o_nb1 = dst->nb[1]; + const cl_ulong o_nb2 = dst->nb[2]; + const cl_ulong o_nb3 = dst->nb[3]; - cl_kernel kernel; + const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; + const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; + const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; + const int mask_ne2 = mask ? mask->ne[2] : 0; + const int mask_ne3 = mask ? mask->ne[3] : 0; - // Currently assumes src0 is contiguous - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sqrt_cont_f32_4; + float scale; + float max_bias; + float logit_softcap; + + const float * params = (const float *)dst->op_params; + scale = params[0]; + max_bias = params[1]; + logit_softcap = params[2]; + + bool use_q1_vec = false; + bool use_q1_vec_mq = false; + bool use_local_tile = false; + // KV-head-coalesced gate: gqa_ratio == compile-time MQ_GQA + // restricts to DK=DV=256 for now due to local memory size + const int gqa_ratio_dispatch = n_head_kv > 0 ? (n_head / n_head_kv) : 0; + if (n_q == 1) { + if (use_native_q8_0_q1) { + if (d_head_v >= 256 && + backend_ctx->fa.f32_q8_0_q1_vec.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_q8_0_q1_vec.at(dk_dv); + use_q1_vec = true; + } else { + kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); + } + } else if (use_native_q4_0_q1) { + // q4_0 vec kernel uses per-lane dp4a (cl_khr_integer_dot_product) + const char * q4vec_env = getenv("GGML_OPENCL_FA_Q4_VEC"); + const bool q4vec_off = (q4vec_env != NULL) && (q4vec_env[0] == '0'); + if (!q4vec_off && d_head_v >= 256 && + backend_ctx->fa.f32_q4_0_q1_vec.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_q4_0_q1_vec.at(dk_dv); + use_q1_vec = true; + } else { + kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); + } + } else if (is_mixed) { + static const char * lt_env = getenv("GGML_OPENCL_FA_LOCAL_TILE"); + static const bool lt_on = (lt_env != NULL) && (lt_env[0] != '0'); + if (lt_on && d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_f16_q1_local_tile.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_f16_q1_local_tile.at(dk_dv); + use_local_tile = true; + } else { + static const char * f16_vec_dk128_env = getenv("GGML_OPENCL_FA_F16_VEC_DK128"); + static const bool f16_vec_dk128_off = (f16_vec_dk128_env != NULL) && (f16_vec_dk128_env[0] == '0'); + const int dv_gate = f16_vec_dk128_off ? 256 : 128; + if (d_head_v >= dv_gate && + backend_ctx->fa.f32_f16_q1_vec.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_f16_q1_vec.at(dk_dv); + use_q1_vec = true; + } else { + kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); + } + } + } else if (is_f16) { + kernel = backend_ctx->fa.f16_q1.at(dk_dv); } else { - kernel = backend_ctx->kernel_sqrt_cont_f16_4; + kernel = backend_ctx->fa.f32_q1.at(dk_dv); } - n /= 4; } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sqrt_cont_f32; + if (use_native_q8_0) { + kernel = use_split_q8_0 + ? backend_ctx->fa.f32_q8_0_split.at(dk_dv) + : backend_ctx->fa.f32_q8_0.at(dk_dv); + } else if (use_native_q4_0) { + kernel = use_split_q4_0 + ? backend_ctx->fa.f32_q4_0_split.at(dk_dv) + : backend_ctx->fa.f32_q4_0.at(dk_dv); + } else if (is_mixed) { + if (use_split_kernel) { + // DK=512 prefill: opt-in texture-cache K reads (image1d_buffer_t). + static const char * pkimg_env = getenv("GGML_OPENCL_FA_PREFILL_K_IMG"); + const bool pkimg_on = (pkimg_env != NULL) && (pkimg_env[0] != '0'); + if (d_head_q == 512 && pkimg_on && + backend_ctx->fa.f32_f16_split_k_img.count(dk_dv) > 0) { + kernel = backend_ctx->fa.f32_f16_split_k_img.at(dk_dv); + use_prefill_k_img = true; + } else { + kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); + } + } else { + kernel = backend_ctx->fa.f32_f16.at(dk_dv); + } + } else if (is_f16) { + kernel = backend_ctx->fa.f16.at(dk_dv); } else { - kernel = backend_ctx->kernel_sqrt_cont_f16; + kernel = backend_ctx->fa.f32.at(dk_dv); } } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + // Intel goes to the basic q1 kernel + if (backend_ctx->gpu_family == INTEL && n_q == 1) { + use_q1_vec = use_q1_vec_mq = use_local_tile = false; + if (is_mixed && backend_ctx->fa.f32_f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_f16_q1.at(dk_dv); } + else if (is_f16 && backend_ctx->fa.f16_q1.count(dk_dv)) { kernel = backend_ctx->fa.f16_q1.at(dk_dv); } + else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q8_0_q1.at(dk_dv); } + else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q4_0_q1.at(dk_dv); } + else if (backend_ctx->fa.f32_q1.count(dk_dv)) { kernel = backend_ctx->fa.f32_q1.at(dk_dv); } + } + GGML_ASSERT(kernel != NULL); - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + ggml_cl_flash_attn_temp_buffer temp_k; + ggml_cl_flash_attn_temp_buffer temp_v; + ggml_cl_flash_attn_temp_buffer temp_k_pad; + ggml_cl_flash_attn_temp_buffer temp_v_pad; + ggml_cl_flash_attn_temp_buffer temp_mask_pad; + ggml_cl_flash_attn_temp_buffer temp_blk; + const ggml_type kv_target_type = is_f16 ? GGML_TYPE_F16 : GGML_TYPE_F32; + + cl_mem k_data_device = k_soa ? NULL : extra_k->data_device; + cl_mem v_data_device = v_soa ? NULL : extra_v->data_device; + + // SoA q8_0/q4_0 -> reconstruct AoS for downstream kernels that expect + // tight records (no-op when k/v is already AoS). + ggml_cl_flash_attn_temp_buffer temp_k_aos; + ggml_cl_flash_attn_temp_buffer temp_v_aos; + ggml_cl_flash_attn_reconstruct_aos(backend_ctx, k, temp_k_aos, + k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + ggml_cl_flash_attn_reconstruct_aos(backend_ctx, v, temp_v_aos, + v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + + // currently FA kernels support KV cache with f16, f32, q4_0 and q8_0. + // there two cases that these kernels cannot cover, + // 1. KV cache types are q4_0 or q8_0, but the FA kernels fail to compile + // 2. KV cache types not currently supported by an FA kernel, e.g., q4_1 + // these two cases are supported here by dequantizing to f32/f16 and this + // causes performance degradation. + // For q4_0 or q8_0 cases that fail kernel compilation, dequant happens in GPU; + // for types that do not have FA kernels, dequant happens on host. + if (!use_native_q8_0_q1 && !use_native_q8_0 && + !use_native_q4_0_q1 && !use_native_q4_0) { + // for q4_0, q8_0 FA kernels that fail to compile + bool k_done = false; + bool v_done = false; + if (k->type == GGML_TYPE_Q8_0 || k->type == GGML_TYPE_Q4_0) { + k_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, k, kv_target_type, k_data_device, offset_k, k_nb1, k_nb2, k_nb3, + temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + } + if (v->type == GGML_TYPE_Q8_0 || v->type == GGML_TYPE_Q4_0) { + v_done = ggml_cl_flash_attn_dequant_kv_gpu( + backend_ctx, v, kv_target_type, v_data_device, offset_v, v_nb1, v_nb2, v_nb3, + temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + } + if (!k_done) { + ggml_cl_flash_attn_prepare_quantized_tensor( + backend_ctx, k, kv_target_type, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + } + if (!v_done) { + ggml_cl_flash_attn_prepare_quantized_tensor( + backend_ctx, v, kv_target_type, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + } + // Asymmetric KV on the F32 fallback path: convert the F16 side to F32 + // too. (Symmetric F16 / mixed paths handle F16 directly.) + if (kv_target_type == GGML_TYPE_F32 && !is_mixed && !is_f16) { + ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, k, temp_k, k_data_device, offset_k, k_nb1, k_nb2, k_nb3); + ggml_cl_flash_attn_convert_f16_to_f32(backend_ctx, v, temp_v, v_data_device, offset_v, v_nb1, v_nb2, v_nb3); + } + } - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; + cl_mem k_pad_buffer = NULL; + cl_mem v_pad_buffer = NULL; + cl_mem mask_pad_buffer = NULL; + cl_mem blk_buffer = NULL; + cl_ulong mask_pad_nb1 = 0; + cl_ulong mask_pad_nb2 = 0; + cl_ulong mask_pad_nb3 = 0; + + // Flash-Decoding K-split decision. Resolved here, before the prefill + // prepass, because KV-pad and blk prepass are pure overhead when FD fires. + // Do not infer causality from tensor shapes: a NULL mask means full + // (bidirectional) attention, e.g. ViT encoders, where n_q == n_kv as well. + // Causal attention in llama.cpp always comes with an explicit KQ mask. + // Inferring is_causal here corrupted mmproj output on OpenCL (see #23800). + const int is_causal = 0; + const int fd_max_n_q = (d_head_q <= FD_MAX_DK_MULTI) ? FD_MAX_N_Q_MULTI : 1; + cl_kernel fd_k_split = NULL; + bool use_fd_mq = false; + bool use_fd_mq_decode = false; + int fd_head_sub = 1; + size_t fd_mq_wg = 256; // MQ_GQA=4 kernel: Q1_WG_SIZE(64) * MQ_NSG_SPLIT(4) + bool use_fa_k_img = false; // K bound as image1d_buffer_t instead of (buf, offset) + + { + const char * mq_env = getenv("GGML_OPENCL_FA_MQ"); + const bool mq_enabled = (mq_env == NULL) ? true : (mq_env[0] != '0'); + const bool mq_kv_ok = is_mixed || is_q8_0 || is_q4_0; + + const char * lmq_env = getenv("GGML_OPENCL_FA_LOCAL_MQ_SPLIT"); + const bool lmq_on = (lmq_env != NULL) && (lmq_env[0] != '0'); + + static const char * vec_nq_env = getenv("GGML_OPENCL_FA_VEC_NQ"); + static const int N_MAX_VEC_NQ = (vec_nq_env != NULL && vec_nq_env[0] != '\0') + ? atoi(vec_nq_env) : 1; + + const bool nq_in_vec_range = (n_q >= 1) && (n_q <= N_MAX_VEC_NQ); + const bool nq1_only = (n_q == 1); + + // Cluster-parallel decode default on for Adreno X2E/X1E + static const int c8_env_state = []{ + const char * e = getenv("GGML_OPENCL_FA_C8"); + if (e == NULL || e[0] == '\0') { return -1; } + return (e[0] != '0') ? 1 : 0; + }(); + const bool c8_default_on = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E; + const bool c8_f16_on = (c8_env_state >= 0) ? (c8_env_state == 1) : c8_default_on; + // Quant-KV (q4_0/q8_0) GQA4 c8: default-on X2E + X1E + const bool c8_quant_on = (c8_env_state >= 0) ? (c8_env_state == 1) : c8_default_on; + if (mq_enabled && mq_kv_ok && nq_in_vec_range && !is_causal && + backend_ctx->gpu_family != INTEL && + !use_local_tile && + n_kv >= (mq_decode_shape ? 32 : FD_MIN_N_KV) && + backend_ctx->fa.f32_merge.count(dk_dv) > 0) { + if (mq_decode_shape && backend_ctx->fa.f32_f16_mq_decode.count(mq_decode_key) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_mq_decode.at(mq_decode_key); + fd_mq_wg = backend_ctx->fa.f32_f16_mq_decode_wg.at(mq_decode_key); + fd_head_sub = backend_ctx->fa.f32_f16_mq_decode_hs.at(mq_decode_key); + use_fd_mq = true; + use_fd_mq_decode = true; + } else if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && + gqa_ratio_dispatch == 8 && + backend_ctx->fa.f32_f16_q1_local_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 64; + } else if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 && + gqa_ratio_dispatch == 4 && + backend_ctx->fa.f32_f16_q1_local_mq_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 64; + } else if (nq1_only && is_mixed && gqa_ratio_dispatch == 4 && + ((d_head_q == 256 && d_head_v == 256) || + (d_head_q == 128 && d_head_v == 128)) && + backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { + const bool k_img_on = d_head_q == 128 && d_head_v == 128 && + getenv("GGML_OPENCL_FA_K_IMG") != NULL && + getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && + backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.count(dk_dv) > 0; + // Cluster-parallel decode + const bool c8_env = d_head_q == 128 && d_head_v == 128 && c8_f16_on; + if (c8_env && backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.at(dk_dv); + use_fd_mq = true; + } else if (c8_env && backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 128; + } else if (k_img_on) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_k_img.at(dk_dv); + use_fd_mq = true; + use_fa_k_img = true; + } else { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); + use_fd_mq = true; + } + // Cluster-parallel decode, DK=DV=256 GQA=8 + } else if (nq1_only && is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 256 && d_head_v == 256 && + c8_env_state == 1 && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c32.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 128; + // Cluster-parallel decode for the g8 + } else if (is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + c8_f16_on && + (backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count(dk_dv) > 0 || + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2.count(dk_dv) > 0)) { + if (backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8.at(dk_dv); + fd_mq_wg = 192; + } else { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_c8_ns2.at(dk_dv); + fd_mq_wg = 128; + } + use_fd_mq = true; + } else if (is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + getenv("GGML_OPENCL_FA_K_IMG") != NULL && + getenv("GGML_OPENCL_FA_K_IMG")[0] != '0' && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8_k_img.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + use_fa_k_img = true; + } else if (is_mixed && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + } else if (nq1_only && is_q8_0 && gqa_ratio_dispatch == 4 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q8_0_q1_vec_mq_split.count(dk_dv) > 0) { + // Cluster-parallel q8_0 GQA4 + if (c8_quant_on && + backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split_c8.at(dk_dv); + fd_mq_wg = 128; + } else { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_vec_mq_split.at(dk_dv); + } + use_fd_mq = true; + } else if (nq1_only && is_q4_0) { + const char * q4_mq_env = getenv("GGML_OPENCL_FA_Q4_MQ"); + const bool q4_mq_on = (q4_mq_env != NULL) && (q4_mq_env[0] != '0'); + // Cluster-parallel q4_0 + const bool q4_c8_on = c8_env_state == 1 && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8.count(dk_dv) > 0; + if (q4_c8_on && gqa_ratio_dispatch == 8 && + d_head_q == 64 && d_head_v == 64) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8_c8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 128; + } else if (q4_mq_on && gqa_ratio_dispatch == 8 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_g8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 192; + } else if (gqa_ratio_dispatch == 4 && + d_head_q == 128 && d_head_v == 128 && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split.count(dk_dv) > 0) { + // Cluster-parallel q4_0 GQA4 + if (c8_quant_on && + backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split_c8.at(dk_dv); + fd_mq_wg = 128; + } else { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_vec_mq_split.at(dk_dv); + } + use_fd_mq = true; + } + } + } + } + // Intel cluster-parallel decode FA + if (fd_k_split == NULL && backend_ctx->gpu_family == INTEL && n_q == 1 && !is_causal && + is_mixed && gqa_ratio_dispatch == 4 && d_head_q == 128 && d_head_v == 128 && + n_kv >= FD_MIN_N_KV && + getenv("GGML_OPENCL_FA_C8") != NULL && getenv("GGML_OPENCL_FA_C8")[0] != '0' && + backend_ctx->fa.f32_merge.count(dk_dv) > 0) { + if (backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 128; + } else if (backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_c8_ns2.at(dk_dv); + use_fd_mq = true; + fd_mq_wg = 64; + } + } + if (fd_k_split == NULL && + n_q >= 1 && n_q <= fd_max_n_q && n_kv >= FD_MIN_N_KV && !is_causal && + d_head_q <= FD_MAX_DK && + backend_ctx->fa.f32_merge.count(dk_dv) > 0) { + if (is_mixed && backend_ctx->fa.f32_f16_q1_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_f16_q1_split.at(dk_dv); + } else if (is_q8_0 && backend_ctx->fa.f32_q8_0_q1_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q8_0_q1_split.at(dk_dv); + } else if (is_q4_0 && backend_ctx->fa.f32_q4_0_q1_split.count(dk_dv) > 0) { + fd_k_split = backend_ctx->fa.f32_q4_0_q1_split.at(dk_dv); + } } + const bool use_fd = (fd_k_split != NULL); + + const int n_q_blocks = n_q > 1 ? (n_q + block_m - 1) / block_m : 0; + const int n_kv_blocks = (n_kv > 0 && block_n > 0) ? (n_kv + block_n - 1) / block_n : 0; + // KV pad + blk prepass are pure overhead when FD will fire — skip them. + const bool use_mixed_prepass = is_mixed && n_q > 1 && !use_fd; + // make sure prepass kernels are compiled + const bool have_kv_pad = backend_ctx->fa.kv_pad_f16.count(dk_dv) > 0; + const bool have_blk = backend_ctx->fa.blk_f16.count(dk_dv) > 0; + const bool use_kv_pad = use_mixed_prepass && (n_kv % block_n != 0) && have_kv_pad; + // blk prepass: per-KV-tile mask class (0=masked, 1=mixed, 2=unmasked). + // Consumed identically by f32_f16, q8_0 and q4_0 prefill kernels. + const bool use_quant_prepass = (use_native_q8_0 || use_native_q4_0) && !use_fd; + const bool use_blk_mask = (use_mixed_prepass || use_quant_prepass) && mask_buffer != NULL && have_blk; + + if (use_kv_pad) { + cl_int err; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); -} + const size_t k_pad_size = (size_t) k_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; + temp_k_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, k_pad_size, NULL, &err); + CL_CHECK(err); + k_pad_buffer = temp_k_pad.data; -static void ggml_cl_mean(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - GGML_UNUSED(src1); + const size_t v_pad_size = (size_t) v_nb1 * (size_t) block_n * (size_t) n_head_kv * (size_t) n_batch; + temp_v_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, v_pad_size, NULL, &err); + CL_CHECK(err); + v_pad_buffer = temp_v_pad.data; + + cl_kernel kernel_kv_pad = backend_ctx->fa.kv_pad_f16.at(dk_dv); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 0, sizeof(cl_mem), &k_data_device)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 1, sizeof(cl_ulong), &offset_k)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 2, sizeof(cl_mem), &v_data_device)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 3, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 4, sizeof(cl_mem), &k_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 5, sizeof(cl_mem), &v_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 6, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 7, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 8, sizeof(int), &n_batch)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 9, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 10, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 11, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 12, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 13, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel_kv_pad, 14, sizeof(cl_ulong), &v_nb3)); + + size_t global_work_size[] = { (size_t) block_n, (size_t) n_head_kv, (size_t) n_batch }; + backend_ctx->enqueue_ndrange_kernel(kernel_kv_pad, 3, global_work_size, NULL, dst); + + if (mask_buffer != NULL) { + mask_pad_nb1 = (cl_ulong) block_n * (cl_ulong) sizeof(ggml_fp16_t); + mask_pad_nb2 = (cl_ulong) n_q * mask_pad_nb1; + mask_pad_nb3 = (cl_ulong) mask_ne2 * mask_pad_nb2; + + const size_t mask_pad_size = (size_t) mask_ne3 * (size_t) mask_pad_nb3; + temp_mask_pad.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, mask_pad_size, NULL, &err); + CL_CHECK(err); + mask_pad_buffer = temp_mask_pad.data; + + cl_kernel kernel_mask_pad = backend_ctx->fa.mask_pad_f16.at(dk_dv); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 0, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 1, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 2, sizeof(cl_mem), &mask_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 3, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 5, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 6, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 7, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 8, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel_mask_pad, 9, sizeof(int), &mask_ne3)); + + size_t global_work_size_mask[] = { (size_t) block_n, (size_t) n_q, (size_t) (mask_ne2 * mask_ne3) }; + backend_ctx->enqueue_ndrange_kernel(kernel_mask_pad, 3, global_work_size_mask, NULL, dst); + } + } - GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); + if (use_blk_mask) { + cl_int err; + const size_t blk_size = (size_t) n_kv_blocks * (size_t) n_q_blocks * (size_t) mask_ne2 * (size_t) mask_ne3; + temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); + if (err != CL_SUCCESS) { + // Flush before retry — reclaim deferred driver deallocations. + CL_CHECK(clFinish(backend_ctx->queue)); + temp_blk.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, blk_size, NULL, &err); + } + CL_CHECK(err); + blk_buffer = temp_blk.data; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_kernel kernel_blk = backend_ctx->fa.blk_f16.at(dk_dv); + CL_CHECK(clSetKernelArg(kernel_blk, 0, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel_blk, 1, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel_blk, 2, sizeof(cl_mem), &blk_buffer)); + CL_CHECK(clSetKernelArg(kernel_blk, 3, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel_blk, 4, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel_blk, 5, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel_blk, 6, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel_blk, 7, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel_blk, 8, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel_blk, 9, sizeof(int), &mask_ne3)); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + size_t global_work_size_blk[] = { (size_t) n_kv_blocks, (size_t) n_q_blocks, (size_t) (mask_ne2 * mask_ne3) }; + backend_ctx->enqueue_ndrange_kernel(kernel_blk, 3, global_work_size_blk, NULL, dst); + } - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; + const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; + const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + if (use_fd) { + static const int fd_env_kv_per_split = []{ + const char * e = getenv("GGML_OPENCL_FD_KV_PER_SPLIT"); + return (e && e[0]) ? atoi(e) : 0; + }(); + static const int fd_env_max_splits = []{ + const char * e = getenv("GGML_OPENCL_FD_MAX_SPLITS"); + return (e && e[0]) ? atoi(e) : 0; + }(); + + int fd_kv_per_split = use_fd_mq ? FD_MQ_KV_PER_SPLIT + : (is_mixed ? FD_KV_PER_SPLIT_F16 : FD_KV_PER_SPLIT); + int fd_max_splits = use_fd_mq ? FD_MQ_MAX_SPLITS : FD_MAX_SPLITS; + if (fd_env_kv_per_split > 0) { fd_kv_per_split = fd_env_kv_per_split; } + if (fd_env_max_splits > 0) { fd_max_splits = fd_env_max_splits; } + int n_splits = (n_kv + fd_kv_per_split - 1) / fd_kv_per_split; + if (n_splits < FD_MIN_SPLITS) { n_splits = FD_MIN_SPLITS; } + if (n_splits > fd_max_splits) { n_splits = fd_max_splits; } + if (use_fd_mq_decode) { + const size_t wg_per_split = (size_t) n_head_kv * n_batch; + const size_t wg_target = 4 * (size_t) backend_ctx->fa.compute_units; + while (wg_per_split * n_splits < wg_target && n_splits < fd_max_splits && + n_kv / (n_splits + 1) >= 32) { + n_splits++; + } + } + const int kv_per_split = (n_kv + n_splits - 1) / n_splits; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + const int fa_partial_floats = 2 + d_head_v; + const size_t partial_size_bytes = + (size_t) n_batch * n_head * n_q * n_splits * fa_partial_floats * sizeof(float); - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + ggml_cl_flash_attn_temp_buffer temp_partial; + cl_mem partial_buffer; + cl_int err; + if (use_fd_mq_decode) { + auto & pool = backend_ctx->fa.fd_partial; + if (partial_size_bytes > pool.size) { + CL_CHECK(clFinish(backend_ctx->queue)); + pool.allocate(backend_ctx->context, partial_size_bytes); + } + partial_buffer = pool.buffer; + } else { + temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, + partial_size_bytes, NULL, &err); + if (err != CL_SUCCESS) { + CL_CHECK(clFinish(backend_ctx->queue)); + temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, + partial_size_bytes, NULL, &err); + } + CL_CHECK(err); + partial_buffer = temp_partial.data; + } - cl_kernel kernel; + cl_kernel k_split = fd_k_split; + int argi = 0; + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &extra_q->data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_q)); + if (use_fa_k_img) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + + (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + + (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + + (size_t)d_head_q * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; + const size_t k_pixels = k_bytes >> 3; + cl_mem k_img = nullptr; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + k_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); + } + + // if image creation fails, fallback to buffer based kernels + if (k_img == nullptr) { + if (gqa_ratio_dispatch == 4 && + backend_ctx->fa.f32_f16_q1_vec_mq_split.count(dk_dv) > 0) { + k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split.at(dk_dv); + } else { + k_split = backend_ctx->fa.f32_f16_q1_vec_mq_split_g8.at(dk_dv); + } + use_fa_k_img = false; + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); + } else { + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_img)); + } + } else { + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &k_data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_k)); + } + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &v_data_device)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_log2_val)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(float), &logit_softcap)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &mask_ne3)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem), &partial_buffer)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &n_splits)); + CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int), &kv_per_split)); + + // MQ split kernel uses MQ_NSG_SPLIT subgroups and one WG per (kv_head, batch, split) + // matches Q1_WG_SIZE * NSG (MQ_GQA=4 -> 256; MQ_GQA=8 -> 192) + const size_t fd_wg = use_fd_mq ? fd_mq_wg : 64; + const size_t fd_head_dim = use_fd_mq + ? (size_t)(n_head_kv * fd_head_sub * n_batch) + : (size_t)(n_head * n_batch); + size_t fd_lws[3] = { fd_wg, 1, 1 }; + // gid(2) packs q_idx * n_splits + split_idx. + size_t fd_gws[3] = { fd_wg, fd_head_dim, (size_t)(n_splits * n_q) }; + backend_ctx->enqueue_ndrange_kernel(k_split, 3, fd_gws, fd_lws, dst); + + cl_kernel k_merge = backend_ctx->fa.f32_merge.at(dk_dv); + argi = 0; + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &partial_buffer)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &extra_o->data_device)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_o)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_splits)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb1)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb2)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &o_nb3)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem), &sinks_buffer)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_sinks)); + CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int), &n_q)); + + const size_t merge_wg = (size_t) (d_head_v / 4); // one lane per float4 + size_t merge_lws[3] = { merge_wg, 1, 1 }; + size_t merge_gws[3] = { merge_wg, (size_t)(n_head * n_batch), (size_t) n_q }; + backend_ctx->enqueue_ndrange_kernel(k_merge, 3, merge_gws, merge_lws, dst); + return; + } + + cl_mem prefill_k_img = nullptr; + if (use_prefill_k_img) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(n_kv > 0 ? n_kv - 1 : 0) * (size_t)k_nb1 + + (size_t)(n_head_kv > 0 ? n_head_kv - 1 : 0) * (size_t)k_nb2 + + (size_t)(n_batch > 0 ? n_batch - 1 : 0) * (size_t)k_nb3 + + (size_t)d_head_q * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 7) & ~(size_t)7; + const size_t k_pixels = k_bytes >> 3; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + prefill_k_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + k_data_device, offset_k, k_bytes, CL_HALF_FLOAT); + } + if (prefill_k_img == nullptr) { + kernel = backend_ctx->fa.f32_f16_split.at(dk_dv); + use_prefill_k_img = false; + } + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), use_prefill_k_img ? &prefill_k_img : &k_data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_k)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &v_data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_v)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_o->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_o)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_q)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_kv)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &is_causal)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_head)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &q_nb1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &q_nb2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &q_nb3)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &k_nb1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &k_nb2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &k_nb3)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &v_nb1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &v_nb2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &v_nb3)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &o_nb1)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &o_nb2)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &o_nb3)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(float), &max_bias)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(float), &m0)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(float), &m1)); + CL_CHECK(clSetKernelArg(kernel, 28, sizeof(int), &n_head_log2_val)); + CL_CHECK(clSetKernelArg(kernel, 29, sizeof(float), &logit_softcap)); + CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &n_head_kv)); + CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_mem), &mask_buffer)); + CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &offset_mask)); + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &mask_nb1)); + CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &mask_nb2)); + CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &mask_nb3)); + CL_CHECK(clSetKernelArg(kernel, 36, sizeof(int), &mask_ne2)); + CL_CHECK(clSetKernelArg(kernel, 37, sizeof(int), &mask_ne3)); + CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_mem), &sinks_buffer)); + CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_ulong), &offset_sinks)); + if (n_q > 1 && is_mixed) { + CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &k_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel, 41, sizeof(cl_mem), &v_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel, 42, sizeof(cl_mem), &mask_pad_buffer)); + CL_CHECK(clSetKernelArg(kernel, 43, sizeof(cl_mem), &blk_buffer)); + CL_CHECK(clSetKernelArg(kernel, 44, sizeof(int), &n_kv_blocks)); + CL_CHECK(clSetKernelArg(kernel, 45, sizeof(cl_ulong), &mask_pad_nb1)); + CL_CHECK(clSetKernelArg(kernel, 46, sizeof(cl_ulong), &mask_pad_nb2)); + CL_CHECK(clSetKernelArg(kernel, 47, sizeof(cl_ulong), &mask_pad_nb3)); + } else if (use_native_q8_0 || use_native_q4_0) { + // arg 40 = blk classification buffer (NULL disables prepass opt). + CL_CHECK(clSetKernelArg(kernel, 40, sizeof(cl_mem), &blk_buffer)); + } - const bool is_c4 = ne00 % 4 == 0; - if (is_c4) { - kernel = backend_ctx->kernel_mean_f32_4; + if (n_q == 1) { + if (use_local_tile) { + const size_t lt_wg = 128; + size_t local_work_size[] = { lt_wg, 1, 1 }; + size_t global_work_size[] = { lt_wg, (size_t) n_head, (size_t) n_batch }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + // q1_vec dispatches with NSG subgroups + const size_t q1_wg = backend_ctx->gpu_family == INTEL ? 32 : 64; + const size_t wg_size = use_q1_vec ? 256 : q1_wg; + const size_t head_dim_global = use_q1_vec_mq + ? (size_t)(n_head_kv * n_batch) + : (size_t)(n_head * n_batch); + size_t local_work_size[] = { wg_size, 1 }; + size_t global_work_size[] = { wg_size, head_dim_global }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + } + } else if (use_native_q8_0 || use_native_q4_0) { + // Native quant prefill. The split variant may override BLOCK_M + // (e.g. DK=96 quant uses BM=16). + const bool use_split = use_native_q8_0 ? use_split_q8_0 : use_split_q4_0; + int bm; + size_t wg_size; + if (use_split) { + bm = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_bm.at(dk_dv) + : backend_ctx->fa.f32_q4_0_split_bm.at(dk_dv); + wg_size = use_native_q8_0 ? backend_ctx->fa.f32_q8_0_split_wg_size.at(dk_dv) + : backend_ctx->fa.f32_q4_0_split_wg_size.at(dk_dv); + } else { + bm = backend_ctx->fa.bm.at(dk_dv); + wg_size = (size_t) bm; + } + size_t local_work_size[] = { wg_size, 1 }; + size_t global_work_size[] = { (size_t)((n_q + bm - 1) / bm) * wg_size, (size_t)(n_head * n_batch) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } else { - kernel = backend_ctx->kernel_mean_f32; + const size_t wg_size = (size_t) wg_size_fa; + size_t local_work_size[] = { wg_size, 1 }; + size_t global_work_size[] = { (size_t)((n_q + block_m - 1) / block_m) * wg_size, (size_t)(n_head * n_batch) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb3)); - - size_t global_work_size[] = {64 * (size_t)ne01, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)64, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } -static void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - +static void ggml_cl_mul_mat_f16_f32_tiled(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; @@ -7535,2138 +19828,3841 @@ static void ggml_cl_ssm_conv(ggml_backend_t backend, const ggml_tensor * src0, c cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - int ne01 = src0->ne[1]; - cl_ulong nb00 = src0->nb[0]; - cl_ulong nb01 = src0->nb[1]; - cl_ulong nb02 = src0->nb[2]; - - int ne10 = src1->ne[0]; - cl_ulong nb11 = src1->nb[1]; + const int M = src0->ne[1]; + const int N = src1->ne[1]; + const int K = src0->ne[0]; - int ne1 = dst->ne[1]; - int ne2 = dst->ne[2]; - cl_ulong nb0 = dst->nb[0]; - cl_ulong nb1 = dst->nb[1]; - cl_ulong nb2 = dst->nb[2]; + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_tiled; - cl_kernel kernel = backend_ctx->kernel_ssm_conv_f32_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); - if (ne10 % 4 == 0) { - kernel = backend_ctx->kernel_ssm_conv_f32_f32_4; - } + // Tiling parameters. These need to be tuned for optimal performance. + // They must match the #defines in the kernel mul_mat_f16_f32.cl. + // + // OPWM / OPWN: Output tile size per Work-Group. A work-group computes a tile of size OPWM x OPWN. + // TPWM / TPWN: Threads per Work-group. This is the work-group size. + // OPTM / OPTN: Output elements per Thread. Each thread computes OPTM x OPTN elements. + // + // The following relationships must hold: + // OPWM = TPWM * OPTM + // OPWN = TPWN * OPTN + // + const int OPWM = 64; + const int OPWN = 64; + const int TPWM = 16; + const int TPWN = 8; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2)); + size_t local_work_size[2] = { TPWM, TPWN }; + size_t global_work_size[2] = { + (size_t) ((M + OPWM - 1) / OPWM) * TPWM, + (size_t) ((N + OPWN - 1) / OPWN) * TPWN, + }; - size_t global_work_size[] = {(size_t)ne01, (size_t)ne1, (size_t)ne2}; - size_t local_work_size[] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); +} - size_t * local_work_size_ptr = local_work_size; - if (ne01 % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_can_use_adreno_xmem_gemm_f16_f32( + const ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * src0, + const ggml_tensor * src1, + const ggml_tensor * dst) { + if (!backend_ctx->adreno_xmem_gemm_enabled) { + return false; } - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) { + return false; + } + if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16) || + src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { + return false; + } + if (src0->ne[2] != 1 || src0->ne[3] != 1 || + src1->ne[2] != 1 || src1->ne[3] != 1 || + dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + const int K = src0->ne[0]; + const int M = src0->ne[1]; + const int N = src1->ne[1]; + if (src1->ne[0] != K || dst->ne[0] != M || dst->ne[1] != N) { + return false; + } + if (N <= 1 || M < 64 || N < 16 || K < 64) { + return false; + } + if ((K % 8) != 0) { + return false; + } + const int kpack = K / 4; + const int npack = CEIL_DIV(M, 4); + if (static_cast(N) > backend_ctx->image2d_max_width || + static_cast(kpack) > backend_ctx->image2d_max_height) { + return false; + } + if (static_cast(N) > backend_ctx->image2d_max_width || + static_cast(npack) > backend_ctx->image2d_max_height) { + return false; + } + return true; } -static void ggml_cl_gelu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - - UNUSED(src1); - - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; +static void ggml_cl_mul_mat_f16_f32_adreno_xmem( + ggml_backend_t backend, + const ggml_tensor * src0, + const ggml_tensor * src1, + ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; - - cl_kernel kernel; - - int n = ggml_nelements(dst); - - if (n % 4 == 0) { - kernel = backend_ctx->kernel_gelu_4; - n /= 4; - } else { - kernel = backend_ctx->kernel_gelu; - } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offset1 = extra1->offset + src1->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + const int K = src0->ne[0]; + const int M = src0->ne[1]; + const int N = src1->ne[1]; + const int kpack = K / 4; + const int npack = CEIL_DIV(M, 4); + const int os = 8; + // Pad weights to the 32-row tiles read by the xmem kernel. + const int npack_padded = CEIL_DIV(npack, os)*os; + + const size_t xmem_bytes = 6144; + const size_t weight_bytes = static_cast(kpack) * static_cast(npack_padded) * 4u * sizeof(cl_half4); + + backend_ctx->prealloc_adreno_xmem_const.allocate(backend_ctx->context, xmem_bytes); + + cl_int err = CL_SUCCESS; + cl_image_format fmt = {}; + fmt.image_channel_order = CL_RGBA; + fmt.image_channel_data_type = CL_HALF_FLOAT; + + cl_image_desc desc_src = {}; + desc_src.image_type = CL_MEM_OBJECT_IMAGE2D; + desc_src.image_width = static_cast(N); + desc_src.image_height = static_cast(kpack); + cl_mem src_img = clCreateImage(backend_ctx->context, CL_MEM_READ_WRITE, &fmt, &desc_src, nullptr, &err); + CL_CHECK(err); + + cl_image_desc desc_dst = {}; + desc_dst.image_type = CL_MEM_OBJECT_IMAGE2D; + desc_dst.image_width = static_cast(N); + desc_dst.image_height = static_cast(npack); + cl_mem dst_img = clCreateImage(backend_ctx->context, CL_MEM_READ_WRITE, &fmt, &desc_dst, nullptr, &err); + CL_CHECK(err); + + cl_mem weights = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, weight_bytes, nullptr, &err); + CL_CHECK(err); + + cl_kernel prepack = backend_ctx->kernel_adreno_xmem_prepack_weight_f16; + CL_CHECK(clSetKernelArg(prepack, 0, sizeof(cl_mem), &weights)); + CL_CHECK(clSetKernelArg(prepack, 1, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(prepack, 2, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(prepack, 3, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(prepack, 4, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(prepack, 5, sizeof(int), &kpack)); + CL_CHECK(clSetKernelArg(prepack, 6, sizeof(int), &npack_padded)); + CL_CHECK(clSetKernelArg(prepack, 7, sizeof(int), &os)); + size_t lws = 256; + size_t max_wg = backend_ctx->get_kernel_workgroup_size(prepack); + if (lws > max_wg) { + lws = max_wg; + } + size_t gws = CEIL_DIV(static_cast(kpack) * static_cast(npack_padded), lws) * lws; + backend_ctx->enqueue_ndrange_kernel(prepack, 1, &gws, &lws, dst); + + cl_kernel pack_src = backend_ctx->kernel_adreno_xmem_pack_src_f32; + CL_CHECK(clSetKernelArg(pack_src, 0, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(pack_src, 1, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(pack_src, 2, sizeof(cl_mem), &src_img)); + CL_CHECK(clSetKernelArg(pack_src, 3, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(pack_src, 4, sizeof(int), &N)); + size_t pack_src_lws[2] = { 16, 16 }; + size_t pack_src_gws[2] = { + CEIL_DIV(static_cast(N), pack_src_lws[0])*pack_src_lws[0], + CEIL_DIV(static_cast(kpack), pack_src_lws[1])*pack_src_lws[1] + }; + backend_ctx->enqueue_ndrange_kernel(pack_src, 2, pack_src_gws, pack_src_lws, dst); + + cl_kernel gemm = backend_ctx->kernel_gemm_xmem_f16_f32_os8; + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &weights)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &backend_ctx->prealloc_adreno_xmem_const.buffer)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &src_img)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &dst_img)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(int), &npack)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(int), &kpack)); + const size_t z_values = CEIL_DIV(static_cast(npack), static_cast(os)); + size_t gemm_lws[3] = { 64, 1, 1 }; + size_t gemm_gws[3] = { + z_values*gemm_lws[0], + CEIL_DIV(static_cast(N), gemm_lws[0]), + 1 + }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, gemm_gws, gemm_lws, dst); + + cl_kernel store_dst = backend_ctx->kernel_adreno_xmem_store_dst_f32; + CL_CHECK(clSetKernelArg(store_dst, 0, sizeof(cl_mem), &dst_img)); + CL_CHECK(clSetKernelArg(store_dst, 1, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(store_dst, 2, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(store_dst, 3, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(store_dst, 4, sizeof(int), &N)); + size_t store_lws[2] = { 16, 16 }; + size_t store_gws[2] = { + CEIL_DIV(static_cast(N), store_lws[0])*store_lws[0], + CEIL_DIV(static_cast(npack), store_lws[1])*store_lws[1] + }; + backend_ctx->enqueue_ndrange_kernel(store_dst, 2, store_gws, store_lws, dst); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(weights)); + CL_CHECK(clReleaseMemObject(dst_img)); + CL_CHECK(clReleaseMemObject(src_img)); } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_gelu_erf(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - - UNUSED(src1); - +static void ggml_cl_conv_2d(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS; ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - cl_kernel kernel; - - int n = ggml_nelements(dst); - - if (n % 4 == 0) { - kernel = backend_ctx->kernel_gelu_erf_4; - n /= 4; - } else { - kernel = backend_ctx->kernel_gelu_erf; - } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} + const cl_uint Cout = ne03; + const cl_uint Cin = ne02; + const cl_uint N = ne13; + const cl_uint KW = ne00; + const cl_uint KH = ne01; + const cl_uint W = ne10; + const cl_uint H = ne11; + const cl_uint OW = ne0; + const cl_uint OH = ne1; + + const cl_uint s0 = dst->op_params[0]; + const cl_uint s1 = dst->op_params[1]; + const cl_uint p0 = dst->op_params[2]; + const cl_uint p1 = dst->op_params[3]; + const cl_uint d0 = dst->op_params[4]; + const cl_uint d1 = dst->op_params[5]; + + const cl_uint cl_nb00 = nb00/ggml_type_size(src0->type); + const cl_uint cl_nb01 = nb01/ggml_type_size(src0->type); + const cl_uint cl_nb02 = nb02/ggml_type_size(src0->type); + const cl_uint cl_nb03 = nb03/ggml_type_size(src0->type); + const cl_uint cl_nb10 = nb10/ggml_type_size(src1->type); + const cl_uint cl_nb11 = nb11/ggml_type_size(src1->type); + const cl_uint cl_nb12 = nb12/ggml_type_size(src1->type); + const cl_uint cl_nb13 = nb13/ggml_type_size(src1->type); + const cl_uint cl_nb1 = nb1/ggml_type_size(dst->type); + const cl_uint cl_nb2 = nb2/ggml_type_size(dst->type); + const cl_uint cl_nb3 = nb3/ggml_type_size(dst->type); -static void ggml_cl_gelu_quick(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + const int64_t NPQ = (int64_t)N * OW * OH; - UNUSED(src1); + const uint32_t BS_K = 64; + const uint32_t BS_NPQ = 64; + const uint32_t BS_CRS = 16; + const uint32_t VEC_SIZE = 4; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const uint32_t TS_K = 4; + const uint32_t TS_NPQ = 8; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + const uint32_t WG_K = BS_K / TS_K; + const uint32_t WG_NPQ = BS_NPQ / TS_NPQ; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + auto splitWork = [](uint32_t work_size, uint32_t block_size) { return (block_size + work_size - 1) / block_size; }; + const uint32_t NB_K = splitWork(Cout, BS_K); + const uint32_t NB_NPQ = splitWork(NPQ, BS_NPQ); cl_kernel kernel; + size_t shmem_size; - int n = ggml_nelements(dst); - - if (n % 4 == 0) { - kernel = backend_ctx->kernel_gelu_quick_4; - n /= 4; + if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { + kernel = backend_ctx->kernel_conv_2d_f16; + shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_half4)); + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_conv_2d_f32; + shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_float) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_conv_2d_f16_f32; + shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); } else { - kernel = backend_ctx->kernel_gelu_quick; + GGML_ASSERT(false && "Unsupported data type combination for conv2d"); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + cl_uint idx = 0; + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, idx++, shmem_size, NULL)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cout)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cin)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &N)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &W)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb00)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb01)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb02)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb03)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb10)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb11)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb12)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb13)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb2)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb3)); - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + size_t global_work_size[] = { (size_t)NB_K * WG_K, (size_t)NB_NPQ * WG_NPQ, 1 }; + size_t local_work_size[] = { (size_t)WG_K, (size_t)WG_NPQ, 1 }; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); } -static void ggml_cl_silu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); +static void ggml_cl_conv2_dw(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS; + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_long offset0 = extra0->offset + src0->view_offs; + const cl_long offset1 = extra1->offset + src1->view_offs; + const cl_long offsetd = extrad->offset + dst->view_offs; + + const cl_long C = ne12; + const cl_long N = ne13; + const cl_long KW = ne00; + const cl_long KH = ne01; + const cl_long W = ne10; + const cl_long H = ne11; + const cl_long OW = ne0; + const cl_long OH = ne1; + + const cl_int s0 = dst->op_params[0]; + const cl_int s1 = dst->op_params[1]; + const cl_int p0 = dst->op_params[2]; + const cl_int p1 = dst->op_params[3]; + const cl_int d0 = dst->op_params[4]; + const cl_int d1 = dst->op_params[5]; + + const cl_long cl_nb01 = nb01/ggml_type_size(src0->type); + const cl_long cl_nb03 = nb03/ggml_type_size(src0->type); + const cl_long cl_nb11 = nb11/ggml_type_size(src1->type); + const cl_long cl_nb12 = nb12/ggml_type_size(src1->type); + const cl_long cl_nb13 = nb13/ggml_type_size(src1->type); + const cl_long cl_nb1 = nb1/ggml_type_size(dst->type); + const cl_long cl_nb2 = nb2/ggml_type_size(dst->type); + const cl_long cl_nb3 = nb3/ggml_type_size(dst->type); - UNUSED(src1); + cl_kernel kernel; + if (src0->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_conv2d_dw_f32_f32; + } else if (src0->type == GGML_TYPE_F16) { + kernel = backend_ctx->kernel_conv2d_dwf16_f32; + } else { + GGML_ASSERT(false && "Unsupported data type combination for conv2_dw"); + } + + cl_uint idx = 0; + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &offset0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &offset1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &C)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &N)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &KW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &KH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &W)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &H)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &OW)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &OH)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &s0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &s1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &p0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &p1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &d0)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_int), &d1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb01)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb03)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb11)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb12)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb13)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb2)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_long), &cl_nb3)); + + const size_t total = (size_t) N * C * OH * OW; + const size_t local_size = MIN((size_t) 256, backend_ctx->get_kernel_workgroup_size(kernel)); + size_t global_work_size[] = { CEIL_DIV(total, local_size) * local_size }; + size_t local_work_size[] = { local_size }; + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); +} + +// is_kq selects which of the two products this call is, and it is decided by the +// CALLER -- the two admission arms in ggml_cl_mul_mat, each of which knows which +// one it matched. It used to be re-derived here from nb01 > nb02, i.e. "K is +// head-major, V^T is not". That discriminator COLLAPSES at n_head_kv == 1, where +// the two strides are equal because there is only one head to order, so nothing +// here could tell a KQ from a KQV. Pass it in rather than infer it. +static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool is_kq) { ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; - cl_kernel kernel; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; - int n = ggml_nelements(dst); + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; - if (n % 4 == 0) { - kernel = backend_ctx->kernel_silu_4; - n /= 4; - } else { - kernel = backend_ctx->kernel_silu; - } + const cl_ulong nb10 = src1->nb[0]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + GGML_ASSERT(ne00 == ne10); - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } + cl_kernel kernel; + cl_context context = backend_ctx->context; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); -} + cl_int status; + cl_image_format img_fmt_1d; + cl_image_desc img_desc_1d; + cl_buffer_region region; + cl_mem A_image1d; + cl_mem A_sub_buffer; + cl_mem B_sub_buffer; + cl_mem D_image1d; + cl_mem D_sub_buffer; -static void ggml_cl_relu(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + int M = ne01; + int N = ne1; + int K = ne00; - UNUSED(src1); + kernel = is_kq ? backend_ctx->kernel_mul_mm_f16_f32_kq + : backend_ctx->kernel_mul_mm_f16_f32_kqv; + // create sub-buffer for A + // <--------------------------------------------> // + extra0 = src0->view_src ? (ggml_tensor_extra_cl *)src0->view_src->extra : (ggml_tensor_extra_cl *)src0->extra; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + region.origin = (extra0->offset + src0->view_offs); + if (is_kq) { + // KQ + region.size = nb01 * ne01; + } else { + // KQV + region.size = nb02 * ne02; + } - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + A_sub_buffer = clCreateSubBuffer((extra0->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + // <--------------------------------------------> // - cl_kernel kernel = backend_ctx->kernel_relu; + // create sub-buffer for B + // <--------------------------------------------> // + region.origin = (extra1->offset + src1->view_offs); + region.size = nb10 * ne10 * ne11 * ne12; + B_sub_buffer = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + // <--------------------------------------------> // - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + img_fmt_1d = {CL_RGBA, CL_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + if (is_kq) { + img_desc_1d.image_width = (nb01 * ne01 / 4)/4; + } + else { + img_desc_1d.image_width = (nb02 * ne02 / 4)/4; + } + img_desc_1d.buffer = A_sub_buffer; + A_image1d = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); + CL_CHECK(status); - const int64_t n = ggml_nelements(dst); + // create sub-buffer for output C + // <--------------------------------------------> // + region.origin = (extrad->offset + dst->view_offs); + region.size = ne0 * ne1 * dst->ne[2] * dst->nb[0]; // size of C in bytes + D_sub_buffer = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + // <--------------------------------------------> // - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + // create image for C output + // <--------------------------------------------> // + img_fmt_1d = {CL_R, CL_FLOAT}; + memset(&img_desc_1d, 0, sizeof(img_desc_1d)); + img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc_1d.image_width = ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4; + img_desc_1d.buffer = D_sub_buffer; + D_image1d = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); + CL_CHECK(status); + // <--------------------------------------------> // - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } + int offset_src0 = 0; + int offset_src1 = 0; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + // set kernel args + // <--------------------------------------------> // + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_sub_buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &D_image1d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &extrad->offset)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &nb01)); + + size_t global_work_size[3] = {64, static_cast(((M+63)/64)), static_cast(((N+31)/32)*ne12)}; + size_t local_work_size[3] = {64, 1, 2}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + // deallocate sub buffers and images + // <--------------------------------------------> // + CL_CHECK(clReleaseMemObject(A_image1d)); + CL_CHECK(clReleaseMemObject(D_image1d)); + CL_CHECK(clReleaseMemObject(A_sub_buffer)); + CL_CHECK(clReleaseMemObject(B_sub_buffer)); + CL_CHECK(clReleaseMemObject(D_sub_buffer)); } -static void ggml_cl_sigmoid(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - UNUSED(src1); + GGML_ASSERT(src0->type == GGML_TYPE_Q1_0); + GGML_ASSERT(src1->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; + GGML_ASSERT(src1->view_offs == 0); + GGML_ASSERT(dst->view_offs == 0); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + + const int ne10 = src1->ne[0]; + const int ne12 = src1->ne[2]; + + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 == ne10); + GGML_ASSERT((ne00 % 128) == 0); + GGML_ASSERT(ne0 == ne01); + + cl_context context = backend_ctx->context; cl_kernel kernel; - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_sigmoid_f32; - } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { - kernel = backend_ctx->kernel_sigmoid_f16; - } else { - GGML_ASSERT(false && "Unsupported data types for sigmoid (input and output must be both f32 or f16)"); - } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - const int64_t n = ggml_nelements(dst); + int M = ne01; + int N = ne1; + int K = ne00; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + if (ne1 == 1) { + cl_mem q_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } + // image for q (uint32: each texel packs 32 sign bits) + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 32; + img_desc.buffer = extra0_q1_0->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); -} + // create a sub_buffer for B + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); -static void ggml_cl_tri(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - UNUSED(src1); + // Tall M has enough row parallelism to skip the K split and its reduction. + const bool use_nsg1 = backend_ctx->min_m_gemv_nsg1_q1_0 > 0 && M >= backend_ctx->min_m_gemv_nsg1_q1_0; + kernel = use_nsg1 ? backend_ctx->kernel_gemv_noshuffle_q1_0_f32_nsg1 : backend_ctx->kernel_gemv_noshuffle_q1_0_f32; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + int r2 = 1; + int r3 = 1; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &extra1->offset)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &extrad->offset)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + size_t wavesize = backend_ctx->adreno_wave_size; + const size_t nsg = use_nsg1 ? 1 : 4; + size_t local_work_size[] = { wavesize, nsg, 1 }; + size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, nsg, 1 }; - const int tri_type = ggml_get_op_params_i32(dst, 0); - const int64_t n = ggml_nelements(dst); - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - cl_kernel kernel = backend_ctx->kernel_tri; + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } else { + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &tri_type)); + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - size_t local_work_size[1] = { 256 }; - size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); -} + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; + } -static void ggml_cl_fill(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - UNUSED(src0); - UNUSED(src1); + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = K/4; + int padded_height_B = (N + padding)/4; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - cl_ulong offsetd = extrad->offset + dst->view_offs; + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - float v = 0.0f; - memcpy(&v, ((int32_t *) dst->op_params), sizeof(float)); + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - const int64_t n = ggml_nelements(dst); + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q1_0_f32; + int padded_N = N + padding; - cl_kernel kernel = backend_ctx->kernel_fill; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(float), &v)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(float), &n)); + size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; + size_t local_work_size[] = { 2, 128, 1 }; - size_t local_work_size[1] = { 256 }; - size_t global_work_size[1] = { ((size_t)n + local_work_size[0] - 1) / local_work_size[0] * local_work_size[0] }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -static void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - UNUSED(src1); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - float min; - float max; - memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float)); - memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float)); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - cl_kernel kernel = backend_ctx->kernel_clamp; + const int ne1 = dst->ne[1]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &min)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &max)); + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - const int64_t n = ggml_nelements(dst); + cl_context context = backend_ctx->context; + cl_kernel kernel; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; // Let driver choose the work-group sizes. - } + int M = ne01; + int N = ne1; + int K = ne00; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); -} + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; -static void ggml_cl_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - UNUSED(src1); + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &M)); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[3] = { wavesize, 4, 1 }; + size_t global_work_size[3] = { (size_t)CEIL_DIV(M, 64) * 64, 4, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + const int gemm_tile_n = 64; + int N_pad = (N + gemm_tile_n - 1) & ~(gemm_tile_n - 1); + + cl_mem a_img = extra0_q4_0->q_img; + cl_mem s_img = extra0_q4_0->d_img; + GGML_ASSERT(a_img && s_img && "ILA Q4_0 weight images missing; set_tensor should have built them"); + + static const char * q4_0_bin_dp4a_env = getenv("GGML_OPENCL_Q4_0_BIN_DP4A"); + bool q4_0_bin_dp4a_on = q4_0_bin_dp4a_env + ? (atoi(q4_0_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q4_0_bin_dp4a_on = backend_ctx->has_integer_dot && q4_0_bin_dp4a_on; + + if (q4_0_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 1, 1 }; + size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - float eps; - memcpy(&eps, dst->op_params, sizeof(float)); + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; + } - const int ne00 = src0 ? src0->ne[0] : 0; - const int ne01 = src0 ? src0->ne[1] : 0; - const int ne02 = src0 ? src0->ne[2] : 0; - const int ne03 = src0 ? src0->ne[3] : 0; + // Pad B through a zero-filled scratch buffer when N needs + // padding, since the GEMM kernel always reads a full N-tile. + const bool need_pad = N_pad > N; + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + if (need_pad) { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, + (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), + 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, + offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + } else { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + } - const cl_ulong nb01 = src0 ? src0->nb[1] : 0; - const cl_ulong nb02 = src0 ? src0->nb[2] : 0; - const cl_ulong nb03 = src0 ? src0->nb[3] : 0; + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = need_pad ? (size_t)K * N_pad : (size_t)K * N; + img_desc.buffer = need_pad ? b_padded : b_sub_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - const int nth = MIN(64, ne00); + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_kernel kernel = backend_ctx->kernel_norm; + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth, NULL)); + int line_stride_matrix_A_in_bytes = M * 4; + int line_stride_matrix_S_in_bytes = M * 2; + int line_stride_matrix_B_in_bytes = K * 4; + int line_stride_matrix_C_in_bytes = M * 4; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + int c_offset_for_kernel = 0; + int b_offset_for_kernel = 0; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &s_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &b_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_A_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_S_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_B_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_matrix_C_in_bytes)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - UNUSED(src1); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - //ggml_backend_opencl_device_context * dev_ctx = - // (ggml_backend_opencl_device_context *)backend->device->context; - - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - float eps; - memcpy(&eps, dst->op_params, sizeof(float)); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; - const int ne00 = src0 ? src0->ne[0] : 0; - const int ne01 = src0 ? src0->ne[1] : 0; - const int ne02 = src0 ? src0->ne[2] : 0; - const int ne03 = src0 ? src0->ne[3] : 0; + const int ne10 = src1->ne[0]; + const int ne12 = src1->ne[2]; - const cl_ulong nb01 = src0 ? src0->nb[1] : 0; - const cl_ulong nb02 = src0 ? src0->nb[2] : 0; - const cl_ulong nb03 = src0 ? src0->nb[3] : 0; + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; - GGML_ASSERT(ne00 % 4 == 0); + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - const int nth = MIN(64, ne00); + cl_context context = backend_ctx->context; + cl_kernel kernel; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - cl_kernel kernel = backend_ctx->kernel_rms_norm; + int M = ne01; + int N = ne1; + int K = ne00; - // Note, this kernel declares local memory in kernel args and the size - // depends on subgroup size. - // Note, this requires OpenCL 2.1 and above - // For now we use fixed subgroup size to simplify support for OpenCL 2.0. - size_t sgs; - //CL_CHECK(clGetKernelSubGroupInfo(kernel, dev_ctx->device, - // CL_KERNEL_MAX_SUB_GROUP_SIZE_FOR_NDRANGE, - // sizeof(local_work_size), local_work_size, - // sizeof(size_t), &sgs, NULL)); - if (backend_ctx->gpu_family == ADRENO) { - sgs = 64; - } else if (backend_ctx->gpu_family == INTEL) { - sgs = 32; - } else { - GGML_ASSERT(false && "Unsupported GPU"); + // Multi-column (N=3) verify GEMV for q4_0: route the spec/MTP verify batch + // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- + // zone (gemm_noshuffle_q4_0 is ~50% of MTP decode on a Q4_0 model since q4_0 + // weights have no cok/mc3, unlike q4_K). Reuses the ne1==1 GEMV image setup + // (activation image already sized by N=ne1). Byte-identical. Opt-in via + // GGML_OPENCL_Q40_MC3=1. Per-layer only (ne01 < 32768); q4_0 lm_head doesn't + // occur (token_embd/output stay Q6_K), guard kept for parity with q4_K mc3. + static const bool q40_mc3 = (getenv("GGML_OPENCL_Q40_MC3") != nullptr); + const bool use_q40_mc3 = q40_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + + const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_q40_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q40_MC3 is bypassed by Q4_0 binary kernels\n"); + warned = true; + } + } + // This bin layout is not supported by CoopK. + ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst); + return; } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); - // This is local memory - the size depends on subgroup size. - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); + if (ne1 == 1 || use_q40_mc3) { + cl_mem q_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} + // image for q + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_0->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); -static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor * rms_norm_tensor, ggml_tensor * mul_tensor) { - GGML_ASSERT(mul_tensor); - GGML_ASSERT(rms_norm_tensor); + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // src0 is the src of rms_norm, src1 is the other src of mul (one being rms_norm) - const ggml_tensor * src0 = rms_norm_tensor->src[0]; - const ggml_tensor * src1; - if (mul_tensor->src[0] == rms_norm_tensor) { - src1 = mul_tensor->src[1]; - } else if (mul_tensor->src[1] == rms_norm_tensor) { - src1 = mul_tensor->src[0]; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + if (use_q40_mc3) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1)); + } else { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32; + if (M == 4096 && K == 4096) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096; + } else if (M == 4096 && K == 11008) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_11008; + } else if (M == 11008 && K == 4096) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_11008_1_4096; + } else if (M == 32000 && K == 4096) { + kernel = backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32000_1_4096; + } + + int r2 = 1; + int r3 = 1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + } + + // Small-M mc3 verify is occupancy/latency-bound (too few WGs at small M, so + // its bandwidth falls well short of the FFN matmuls'). Use 8 subgroups (512-WI WGs, half the + // per-lane K-walk) for small M. Layout stride is fixed (4 uints/block), so only + // the K-split count changes; the mc3 kernel reads it via get_local_size(1). The + // ne1==1 base kernel hardcodes N_SIMDGROUP=4, so it always stays at 4. + int mc3_nsg = (use_q40_mc3 && ne01 < 4096) ? 8 : 4; + if (use_q40_mc3) { + // Register pressure can lower the kernel limit below the device limit. + const size_t max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); + while (mc3_nsg > 1 && (size_t)(64 * mc3_nsg) > max_workgroup_size) { + mc3_nsg /= 2; + } + } + size_t local_work_size[3] = {64, (size_t)mc3_nsg, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, (size_t)mc3_nsg, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); } else { - GGML_ASSERT(false && "Invalid args for rms_norm and mul"); - } - const ggml_tensor * dst = mul_tensor; + // CoopK uses the same untiled weight layout as gemm_noshuffle_q4_0_f32, with K-major Q8_1 activations (K contiguous). + if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_0) + && backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 != nullptr + && ((N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4) != nullptr) { + cl_mem b_sub_buf = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const int cols = (N <= 2) ? 2 : 4; + const size_t n_blocks_padded = (size_t)cols * (K / 32); + const size_t size_qa = (size_t)cols * K * sizeof(cl_char); + const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size; + const size_t size_da_prev = backend_ctx->prealloc_moe_da.size; + backend_ctx->prealloc_moe_qa.allocate(context, size_qa); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half)); + if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) { + const cl_char z8 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer, + &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL)); + } + if (backend_ctx->prealloc_moe_da.size != size_da_prev) { + const cl_half z16 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL)); + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL)); + } - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + cl_int n_blocks = (cl_int)((size_t)N * (K / 32)); + cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks)); + size_t local_work_size_quant[1] = { 64 }; + size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst); + + cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4; + const cl_int n_padded = cols; + CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_int), &n_padded)); + CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &N)); + + const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q4_0; + size_t local_work_size[3] = { 64, nsg, 1 }; + size_t global_work_size[3] = { (size_t)(M / backend_ctx->rows_coopk_dp4a_q4_0), nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + static const char * q4_0_act_vload4_env = getenv("GGML_OPENCL_Q4_0_DP4A_ACT_VLOAD4"); + const bool q4_0_act_vload4_on = q4_0_act_vload4_env == nullptr || atoi(q4_0_act_vload4_env) != 0; + // Enable dense prefill by default on X2E when the uint4 activation tile is enabled. + static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A"); + bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env + ? (atoi(q4_0_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && q4_0_act_vload4_on); + // dot prod has to be available + q4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on; + + if (q4_0_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_kernel dk = q4_0_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + CL_CHECK(clReleaseMemObject(a_sub)); + return; + } - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; + cl_mem d_sub_buf = nullptr; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - float eps; - memcpy(&eps, rms_norm_tensor->op_params, sizeof(float)); + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; + } - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; + // subbuffer for output + region.origin = extrad->offset; // Specify the starting offset (in bytes) + region.size = M * N * sizeof(float); // Specify the size of the sub-buffer + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = K/4; + int padded_height_B = (N + padding)/4; - GGML_ASSERT(ne00 % 4 == 0); + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - size_t sgs; - if (backend_ctx->gpu_family == ADRENO) { - sgs = 64; - } else if (backend_ctx->gpu_family == INTEL) { - sgs = 32; - } else { - GGML_ASSERT(false && "Unsupported GPU"); - } + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { + local_work_size_t[0]=4; + local_work_size_t[1]=8; + } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { + local_work_size_t[0]=2; + local_work_size_t[1]=8; + } else if(ne0 == 4096 && ne1 == 128 && ne10 == 11008) { + local_work_size_t[0]=1; + local_work_size_t[1]=8; + } else if(ne0 == 32000 && ne1 == 128 && ne10 == 4096) { + local_work_size_t[0]=2; + local_work_size_t[1]=8; + } + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - cl_kernel kernel = backend_ctx->kernel_rms_norm_mul; + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q4_0_f32; + int padded_N = N + padding; - int nth = sgs; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - while (nth < ne00 && nth < max_workgroup_size) { - nth *= 2; - } - nth = MIN(nth, max_workgroup_size); - nth = MIN(nth, ne00); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d_sub_buf)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne1)); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; + if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { + local_work_size[0] = 1; + local_work_size[1] = 128; + } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { + local_work_size[0] = 2; + local_work_size[1] = 64; + } else if (ne0 == 4096 && ne1 == 128 && ne10 == 11008) { + local_work_size[0] = 2; + local_work_size[1] = 64; + } else if (ne0 == 32000 && ne1 == 128 && ne10 == 4096) { + local_work_size[0] = 2; + local_work_size[1] = 64; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float)*sgs, NULL)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { - GGML_ASSERT(norm_tensor && mul_tensor && add_tensor); +static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - const ggml_tensor * src0 = norm_tensor->src[0]; - const ggml_tensor * src1 = mul_tensor->src[0] == norm_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; - const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; - const ggml_tensor * dst = add_tensor; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offset2 = extra2->offset + src2->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - - float eps; - memcpy(&eps, norm_tensor->op_params, sizeof(float)); - - const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; - const cl_ulong nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; - const int ne10 = src1->ne[0], ne11 = src1->ne[1], ne12 = src1->ne[2], ne13 = src1->ne[3]; - const cl_ulong nb11 = src1->nb[1], nb12 = src1->nb[2], nb13 = src1->nb[3]; - const int ne20 = src2->ne[0], ne21 = src2->ne[1], ne22 = src2->ne[2], ne23 = src2->ne[3]; - const cl_ulong nb21 = src2->nb[1], nb22 = src2->nb[2], nb23 = src2->nb[3]; - const cl_ulong nbd1 = dst->nb[1], nbd2 = dst->nb[2], nbd3 = dst->nb[3]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - size_t sgs; - if (backend_ctx->gpu_family == ADRENO) sgs = 64; - else if (backend_ctx->gpu_family == INTEL) sgs = 32; - else GGML_ASSERT(false && "Unsupported GPU"); + const int ne1 = dst->ne[1]; - cl_kernel kernel = backend_ctx->kernel_norm_mul_add; + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - int nth = sgs; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - while (nth < ne00/4 && nth < max_workgroup_size) nth *= 2; - nth = MIN(nth, max_workgroup_size); - nth = MIN(nth, ne00/4); + cl_context context = backend_ctx->context; + cl_kernel kernel; - size_t gws[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t lws[] = {(size_t)nth, 1, 1}; - size_t num_subgroups = (nth + sgs - 1) / sgs; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne13)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne20)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &ne21)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &ne22)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &ne23)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(cl_ulong), &nb21)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(cl_ulong), &nb22)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(cl_ulong), &nb23)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(cl_ulong), &nbd1)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(cl_ulong), &nbd2)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_ulong), &nbd3)); - CL_CHECK(clSetKernelArg(kernel, 32, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_float2) * num_subgroups, NULL)); + int M = ne01; + int N = ne1; + int K = ne00; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst); -} + // Multi-column (N=3) verify GEMV for q4_1: route the spec/MTP verify batch + // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- + // zone (gemm_noshuffle_q4_1). Reuses the ne1==1 GEMV image setup. Opt-in via + // GGML_OPENCL_Q41_MC3=1. Per-layer only (ne01 < 32768). + static const bool q41_mc3 = (getenv("GGML_OPENCL_Q41_MC3") != nullptr); + const bool use_q41_mc3 = q41_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); -static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor) { - GGML_ASSERT(gn_tensor && mul_tensor && add_tensor); + if (ne1 == 1 || use_q41_mc3) { + cl_mem q_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - const ggml_tensor * src0 = gn_tensor->src[0]; - const ggml_tensor * src1 = mul_tensor->src[0] == gn_tensor ? mul_tensor->src[1] : mul_tensor->src[0]; - const ggml_tensor * src2 = add_tensor->src[0] == mul_tensor ? add_tensor->src[1] : add_tensor->src[0]; - const ggml_tensor * dst = add_tensor; + // image for q + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_1->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offset2 = extra2->offset + src2->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + kernel = use_q41_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_1_f32_mc3 + : backend_ctx->kernel_gemv_noshuffle_q4_1_f32; - int groups; - float eps; - memcpy(&groups, gn_tensor->op_params, sizeof(int)); - memcpy(&eps, (char *)gn_tensor->op_params + sizeof(int), sizeof(float)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + if (use_q41_mc3) { + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); // n_cols + } - cl_kernel kernel = backend_ctx->kernel_group_norm_mul_add; - int max_workgroup_size = backend_ctx->get_kernel_workgroup_size(kernel); - int ne = ggml_nelements(src0); - int group_size = ne / groups; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; - size_t lws[] = { (size_t)MIN(max_workgroup_size, group_size) }; - size_t gws[] = { (size_t)groups * lws[0] }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &group_size)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(float), &eps)); + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; - backend_ctx->enqueue_ndrange_kernel(kernel, 1, gws, lws, dst); -} + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); -static void ggml_cl_group_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - UNUSED(src1); + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; + } - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = K/4; + int padded_height_B = (N + padding)/4; - int32_t n_groups = ((const int32_t *) dst->op_params)[0]; - int32_t group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + n_groups - 1) / n_groups); - float eps = ((const float *) dst->op_params)[1]; + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne = ne00*ne01*ne02; + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - cl_kernel kernel = backend_ctx->kernel_group_norm; + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q4_1_f32; + int padded_N = N + padding; - size_t sgs = 64; - if (backend_ctx->gpu_family == ADRENO) { - sgs = 64; - } else if (backend_ctx->gpu_family == INTEL) { - sgs = 32; - } else { - GGML_ASSERT(false && "Unsupported GPU"); - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &group_size)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(float), &eps)); + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; - size_t global_work_size[] = {(size_t)n_groups*sgs, 1, 1}; - size_t local_work_size[] = {(size_t)sgs, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -static void ggml_cl_l2_norm(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - UNUSED(src1); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)src0->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - float eps; - memcpy(&eps, dst->op_params, sizeof(float)); - - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - - size_t sgs; - if (backend_ctx->gpu_family == ADRENO) { - sgs = 64; - } else if (backend_ctx->gpu_family == INTEL) { - sgs = 32; - } else { - GGML_ASSERT(false && "Unsupported GPU"); - } + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - cl_kernel kernel = backend_ctx->kernel_l2_norm_f32; + const int ne1 = dst->ne[1]; - int nth = sgs; - while (nth < ne00 && nth < (int)backend_ctx->get_kernel_workgroup_size(kernel)) { - nth *= 2; - } + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(float), &eps)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float)*nth/sgs, NULL)); + cl_context context = backend_ctx->context; + cl_kernel kernel; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} + int M = ne01; + int N = ne1; + int K = ne00; -static void ggml_cl_tanh(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + if (ne1 == 1) { + cl_mem qs_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - UNUSED(src1); + // image for qs + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q5_0->qs; + CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + kernel = backend_ctx->kernel_gemv_noshuffle_q5_0_f32; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - cl_kernel kernel; + CL_CHECK(clReleaseMemObject(qs_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + // dp4a (int8) dense q5_0 prefill GEMM, default off + static const char * q5_dense_dp4a_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A"); + static const char * q5_dense_wimg_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A_WIMG"); + const bool q5_dense_wimg_on = q5_dense_wimg_env && (atoi(q5_dense_wimg_env) != 0); + bool q5_dense_dp4a_on = q5_dense_wimg_on + ? true + : (q5_dense_dp4a_env && (atoi(q5_dense_dp4a_env) != 0)); + // dot prod has to be available + q5_dense_dp4a_on = backend_ctx->has_integer_dot && q5_dense_dp4a_on; + + if (q5_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + // optional qs texture (image1d_buffer over the nibble plane; the same + // CL_R/UINT32 view, width M*K/8, the GEMV path builds). + cl_mem q5_qs_img = nullptr; + bool use_wimg = q5_dense_wimg_on; + if (use_wimg) { + const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels (2 ushorts/texel) + if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { + use_wimg = false; + } else { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = tex; + img_desc.buffer = extra0_q5_0->qs; + q5_qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); + if (err != CL_SUCCESS || q5_qs_img == nullptr) { use_wimg = false; q5_qs_img = nullptr; } + } + } - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_tanh_f32_4; + cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg + : backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a; + int ai = 0; + if (use_wimg) { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q5_qs_img)); } else { - kernel = backend_ctx->kernel_tanh_f16_4; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qs)); } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_tanh_f32; - } else { - kernel = backend_ctx->kernel_tanh_f16; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + if (q5_qs_img != nullptr) { + CL_CHECK(clReleaseMemObject(q5_qs_img)); } + CL_CHECK(clReleaseMemObject(a_sub)); + return; } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; + cl_mem d_sub_buf = nullptr; - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); - } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_tanh_f32_nc; - } else { - kernel = backend_ctx->kernel_tanh_f16_nc; + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + + // subbuffer for output + region.origin = extrad->offset; + region.size = M * N * sizeof(float); + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; } + int width_B = K/4; + int padded_height_B = (N + padding)/4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - int nth = 64; + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q5_0_f32; + int padded_N = N + padding; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_sub_buf)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); + + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -static void ggml_cl_neg(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q5_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - UNUSED(src1); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)src0->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - GGML_TENSOR_LOCALS(int, ne, dst, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - cl_kernel kernel; + const int ne1 = dst->ne[1]; - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_neg_f32_4; - } else { - kernel = backend_ctx->kernel_neg_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_neg_f32; - } else { - kernel = backend_ctx->kernel_neg_f16; - } - } + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); + cl_context context = backend_ctx->context; + cl_kernel kernel; - size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_neg_f32_nc; - } else { - kernel = backend_ctx->kernel_neg_f16_nc; - } + int M = ne01; + int N = ne1; + int K = ne00; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + if (ne1 == 1) { + cl_mem qs_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - int nth = 64; + // image for qs + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q5_1->qs; + CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } -} + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); -static void ggml_cl_exp(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + kernel = backend_ctx->kernel_gemv_noshuffle_q5_1_f32; - UNUSED(src1); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + CL_CHECK(clReleaseMemObject(qs_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; + cl_mem d_sub_buf = nullptr; - GGML_TENSOR_LOCALS(int, ne0, src0, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); - GGML_TENSOR_LOCALS(int, ne, dst, ne); - GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_kernel kernel; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_exp_f32_4; - } else { - kernel = backend_ctx->kernel_exp_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_exp_f32; - } else { - kernel = backend_ctx->kernel_exp_f16; - } + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &n)); + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - size_t global_work_size[] = {(size_t)CEIL_DIV(n, 64)*64, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_exp_f32_nc; - } else { - kernel = backend_ctx->kernel_exp_f16_nc; + // subbuffer for output + region.origin = extrad->offset; + region.size = M * N * sizeof(float); + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; } + int width_B = K/4; + int padded_height_B = (N + padding)/4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - int nth = 64; + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q5_1_f32; + int padded_N = N + padding; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_sub_buf)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); + + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -static void ggml_cl_expm1(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - UNUSED(src1); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)src0->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + const int ne1 = dst->ne[1]; - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + GGML_ASSERT(ne00 % 32 == 0); + cl_context context = backend_ctx->context; cl_kernel kernel; - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_expm1_f32_4; - } else { - kernel = backend_ctx->kernel_expm1_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_expm1_f32; - } else { - kernel = backend_ctx->kernel_expm1_f16; - } - } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; - - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } - - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); - } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_expm1_f32_nc; - } else { - kernel = backend_ctx->kernel_expm1_f16_nc; - } + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + int M = ne01; + int N = ne1; + int K = ne00; - int nth = 64; + if (ne1 == 1) { + cl_mem q_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + // image for q + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_iq4_nl->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } -} + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); -static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - UNUSED(src1); + kernel = backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + // dp4a (int8) dense IQ4_NL prefill GEMM. Quantizes the [N,K] activations to + // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch + // (ne1>8) only + static const char * iq4nl_dense_dp4a_env = getenv("GGML_OPENCL_IQ4NL_DENSE_DP4A"); + bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env + ? (atoi(iq4nl_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // dot prod has to be available + iq4nl_dense_dp4a_on = backend_ctx->has_integer_dot && iq4nl_dense_dp4a_on; + + if (iq4nl_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + CL_CHECK(clReleaseMemObject(a_sub)); + return; + } - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_kernel kernel; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - if (ggml_is_contiguous(src0)) { - // Handle contiguous input - int n = ggml_nelements(dst); - if (n % 4 == 0) { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_softplus_f32_4; - } else { - kernel = backend_ctx->kernel_softplus_f16_4; - } - n /= 4; - } else { - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_softplus_f32; - } else { - kernel = backend_ctx->kernel_softplus_f16; - } + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - size_t global_work_size[] = {(size_t)n, 1, 1}; - size_t local_work_size[] = {64, 1, 1}; + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - size_t * local_work_size_ptr = local_work_size; - if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; } + int width_B = K/4; + int padded_height_B = (N + padding)/4; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); - } else { - // Handle non-contiguous input - if (src0->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_softplus_f32_nc; - } else { - kernel = backend_ctx->kernel_softplus_f16_nc; - } + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32; + int padded_N = N + padding; - int nth = 64; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1)); - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -static void ggml_cl_repeat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1_shape_def, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(dst->type == src0->type); - UNUSED(src1_shape_def); + GGML_ASSERT(src0->type == GGML_TYPE_Q8_0); + GGML_ASSERT(src1->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; - - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; - - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; - - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; - - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; - - cl_kernel kernel = backend_ctx->kernel_repeat_f32; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb3)); - - int nth = 64; - - size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // SoA extra lives on view_src (view->extra is pre-SoA). + const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; + ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; -static void ggml_cl_pad(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + const int ne10 = src1->ne[0]; + const int ne12 = src1->ne[2]; - if (backend_ctx->kernel_pad == nullptr) { - GGML_LOG_WARN("%s: pad kernel not available, skipping OpenCL execution.\n", __func__); - return; - } + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; - ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; + GGML_ASSERT(ne00 == ne10); + GGML_ASSERT((ne00 % 32) == 0); + GGML_ASSERT(ne0 == ne01); - cl_ulong off_src0 = extra_src0->offset + src0->view_offs; - cl_ulong off_dst = extra_dst->offset + dst->view_offs; + cl_context context = backend_ctx->context; + cl_kernel kernel; - const int s_ne0 = src0->ne[0]; - const int s_ne1 = src0->ne[1]; - const int s_ne2 = src0->ne[2]; - const int s_ne3 = src0->ne[3]; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - const int s_nb0 = src0->nb[0]; - const int s_nb1 = src0->nb[1]; - const int s_nb2 = src0->nb[2]; - const int s_nb3 = src0->nb[3]; + int M = ne01; + int N = ne1; + int K = ne00; - const int d_ne0 = dst->ne[0]; - const int d_ne1 = dst->ne[1]; - const int d_ne2 = dst->ne[2]; - const int d_ne3 = dst->ne[3]; + if (ne1 == 1) { + cl_mem q_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - const int d_nb0 = dst->nb[0]; - const int d_nb1 = dst->nb[1]; - const int d_nb2 = dst->nb[2]; - const int d_nb3 = dst->nb[3]; + // image for q + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 4; + img_desc.buffer = extra0_q8_0->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - const int lp0 = ((const int*)(dst->op_params))[0]; - const int rp0 = ((const int*)(dst->op_params))[1]; - const int lp1 = ((const int*)(dst->op_params))[2]; - const int rp1 = ((const int*)(dst->op_params))[3]; - const int lp2 = ((const int*)(dst->op_params))[4]; - const int rp2 = ((const int*)(dst->op_params))[5]; - const int lp3 = ((const int*)(dst->op_params))[6]; - const int rp3 = ((const int*)(dst->op_params))[7]; + // create a sub_buffer for B + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_kernel kernel = backend_ctx->kernel_pad; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &s_ne0)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &s_ne1)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &s_ne2)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &s_ne3)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &s_nb0)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &s_nb1)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &s_nb2)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &s_nb3)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &d_ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &d_ne1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &d_ne2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &d_ne3)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &d_nb0)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &d_nb1)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &d_nb2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &d_nb3)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &lp0)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &rp0)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &lp1)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &rp1)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &lp2)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &rp2)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &lp3)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(int), &rp3)); + // Split-K for small-M decode GEMVs. The base kernel puts one output row + // per lane and splits K only inside one workgroup, so M is the sole source + // of workgroup parallelism: gpt-oss's K and V projections are M=512 = 8 + // workgroups on a 16-CU X2, and the kernel measures 48 GB/s where the + // M=2880/4096 projections in the same decode graph reach 122-123. Mirrors + // the q4_0/q4_K split-K above and reuses their reduce kernel. + // + // Enabled where it is measured to win, like the q4_K gate: X2-90 +2.8% + // tg32 @d4096 on gpt-oss; Adreno 840 (12 CU) NEUTRAL on Llama-3.2-3B-Q8_0 + // (0.0% @d4096 -- its K/V proj is M=1024 = 16 workgroups, which already + // fills 12 CUs). Unmeasured on X1E/A7X/A6X and the q4_K split-K measured + // -0.7% on X1E, so the default is not widened on absence of evidence. + static const bool q8_splitk_env_set = []{ + const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); + return e && e[0] != '\0'; + }(); + static const bool q8_splitk_env_on = []{ + const char * e = std::getenv("GGML_OPENCL_Q8_GEMV_SPLITK"); + return !(e && e[0] == '0'); + }(); + const bool q8_splitk_on = q8_splitk_env_set + ? q8_splitk_env_on + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q8_splitk_on && backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk && + ne01 <= 1024 && ne01 % 64 == 0) { + const int nsg = 8; + const int ksplit = 8; // -> 8 * M/64 workgroups + const size_t gx = (size_t) CEIL_DIV(ne01, 64) * 64; + + backend_ctx->prealloc_splitk_partial.allocate( + backend_ctx->context, (size_t) ksplit * ne01 * sizeof(float)); + cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer; + + cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q8_0_f32_splitk; + CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_int), &ne01)); + size_t lsk[3] = { 64, (size_t) nsg, 1 }; + size_t gsk[3] = { gx, (size_t) (nsg * ksplit), 1 }; + backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst); + + cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32; + CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit)); + size_t lr[3] = { 64, 1, 1 }; + size_t gr[3] = { (size_t) CEIL_DIV(ne01, 64) * 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst); - size_t lws0 = 64; - size_t gws0 = (( (size_t)d_ne0 + lws0 - 1 ) / lws0) * lws0; + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } - size_t global_work_size[] = { gws0, (size_t)d_ne1, (size_t)d_ne2*d_ne3 }; - size_t local_work_size[] = { lws0, 1, 1 }; + kernel = backend_ctx->kernel_gemv_noshuffle_q8_0_f32; - size_t * local_work_size_ptr = local_work_size; - if (d_ne0 % lws0 != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } + int r2 = 1; + int r3 = 1; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); -} + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); -static void ggml_cl_upscale(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + size_t wavesize = backend_ctx->adreno_wave_size; + size_t local_work_size[] = { wavesize, 4, 1 }; + size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - const int mode_flags = (ggml_scale_mode) ggml_get_op_params_i32(dst, 0); - const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF); - cl_kernel kernel = nullptr; + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } else { + static const char * q8_bin_dp4a_env = getenv("GGML_OPENCL_Q8_0_BIN_DP4A"); + const bool q8_bin_dp4a_on = backend_ctx->has_integer_dot && + (q8_bin_dp4a_env ? (atoi(q8_bin_dp4a_env) != 0) : true); + + if (q8_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin + && M % 64 == 0) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + + const size_t N_pad = ((size_t)N + 31) / 32 * 32; + const size_t n_blocks_pad = N_pad * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_pad * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_pad * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem q_img = nullptr; + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * (size_t)K / 4; + img_desc.buffer = extra0_q8_0->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + cl_mem d_sub_buf = nullptr; + cl_mem d_img = nullptr; + region.origin = extrad->offset; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin; + cl_uint ne00_u = (cl_uint)K; + cl_uint ne01_u = (cl_uint)M; + CL_CHECK(clSetKernelArg(dk, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(dk, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(dk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, 4, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(dk, 5, sizeof(cl_uint), &ne00_u)); + CL_CHECK(clSetKernelArg(dk, 6, sizeof(cl_uint), &ne01_u)); + CL_CHECK(clSetKernelArg(dk, 7, sizeof(int), &N)); + + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); - if (mode == GGML_SCALE_MODE_NEAREST) { - kernel = backend_ctx->kernel_upscale; - if (kernel == nullptr) { - GGML_LOG_WARN("%s: nearest upscale kernel not available, skipping OpenCL execution.\n", __func__); + CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + CL_CHECK(clReleaseMemObject(a_sub)); return; } - } else if (mode == GGML_SCALE_MODE_BILINEAR) { - kernel = backend_ctx->kernel_upscale_bilinear; - if (kernel == nullptr) { - GGML_LOG_WARN("%s: bilinear upscale kernel not available, skipping OpenCL execution.\n", __func__); + + // dp4a dense q8_0 prefill GEMM. Quantizes the [N,K] activations to + // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch + // (ne1>8) only; q8_0 weights are already int8 (no requant) and symmetric + // (no min term) + static const char * q8_dense_dp4a_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A"); + static const char * q8_dense_wimg_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A_WIMG"); + const bool q8_dense_wimg_on = q8_dense_wimg_env && (atoi(q8_dense_wimg_env) != 0); + + const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr); + // bin kernel takes precedence + bool q8_dense_dp4a_on = q8_dense_wimg_on + ? true + : q8_dense_dp4a_env + ? (atoi(q8_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && !q8_bin_loaded); + // dot prod has to be available + q8_dense_dp4a_on = backend_ctx->has_integer_dot && q8_dense_dp4a_on; + + if (q8_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + // optional weight texture, the same CL_R/UINT32 view, width M*K/4 + cl_mem q8_q_img = nullptr; + bool use_wimg = q8_dense_wimg_on; + if (use_wimg) { + const size_t tex = (size_t)M * (size_t)K / 4; // uint32 texels + if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { + use_wimg = false; + } else { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = tex; + img_desc.buffer = extra0_q8_0->q; + q8_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); + if (err != CL_SUCCESS || q8_q_img == nullptr) { use_wimg = false; q8_q_img = nullptr; } + } + } + + static const char * q8_0_act_vload4_env = getenv("GGML_OPENCL_Q8_0_DP4A_ACT_VLOAD4"); + const bool q8_0_act_vload4_on = q8_0_act_vload4_env == nullptr || atoi(q8_0_act_vload4_env) != 0; + cl_kernel dk = use_wimg + ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg + : q8_0_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a; + int ai = 0; + if (use_wimg) { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img)); + } else { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->q)); + } + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + if (q8_q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q8_q_img)); + } + CL_CHECK(clReleaseMemObject(a_sub)); return; } - } else { - GGML_LOG_WARN("%s: unsupported upscale mode %d, skipping OpenCL execution.\n", __func__, mode); - return; - } - ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; + // use bin kernel if available + if (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin) { + int K_pad = K; + + cl_mem b_sub_buf = nullptr; + cl_mem d_sub_buf = nullptr; + + cl_mem a_img = nullptr; + cl_mem s_img = nullptr; + cl_mem b_img = nullptr; + cl_mem d_img = nullptr; + + // subbuffer for activations + region.origin = offset1; + region.size = K_pad * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // Create subbuffer and image1d_buffer for dst + region.origin = (extrad->offset); // + dst->view_offs; + region.size = M * N * sizeof(float); + CL_CHECK((d_sub_buf = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // create an image for A + img_fmt = { CL_R, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 4; // Divide by 4 for char -> float + img_desc.buffer = extra0_q8_0->q; + CL_CHECK((a_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // create an image for Scale + img_fmt = { CL_R, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 32; // Block size is 32 + img_desc.buffer = extra0_q8_0->d; + CL_CHECK((s_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - cl_ulong off_src0 = extra_src0->offset + src0->view_offs; - cl_ulong off_dst = extra_dst->offset + dst->view_offs; + // create an image for B from sub_buffer + img_fmt = {CL_R, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K_pad * N; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // img for d + img_fmt = {CL_R, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * N; + img_desc.buffer = d_sub_buf; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin; + + bool layoutA_Mfirst = true; + bool layoutS_Mfirst = true; + bool layoutB_Nfirst = false; + bool layoutC_Mfirst = true; + + cl_uint lineStrideMatrixAinBytes = layoutA_Mfirst ? M * 4 : K; // int8 + cl_uint lineStrideMatrixSinBytes = layoutS_Mfirst ? M * 2 : (K / 32) * 2; // fp16 + cl_uint lineStrideMatrixBinBytes = layoutB_Nfirst ? N * 4 : K_pad * 4; // fp32 + cl_uint lineStrideMatrixCinBytes = layoutC_Mfirst ? M * 4 : N * 4; // fp32 + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &a_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &s_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &extra1->offset)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &extrad->offset)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &lineStrideMatrixAinBytes)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &lineStrideMatrixSinBytes)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &lineStrideMatrixBinBytes)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &lineStrideMatrixCinBytes)); + + size_t global_work_size[] = { 64, (size_t)CEIL_DIV(M, 64), (size_t)CEIL_DIV(N, 64)}; + size_t local_work_size[] = { 64, 2, 2 }; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + CL_CHECK(clReleaseMemObject(a_img)); + CL_CHECK(clReleaseMemObject(s_img)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(d_img)); + return; + } - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; - float sf0 = (float)ne0 / ne00; - float sf1 = (float)ne1 / ne01; - float sf2 = (float)ne2 / ne02; - float sf3 = (float)ne3 / ne03; + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - float pixel_offset = 0.5f; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb03)); + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; + } - if (mode == GGML_SCALE_MODE_NEAREST) { - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &sf0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &sf1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf3)); - } else if (mode == GGML_SCALE_MODE_BILINEAR) { - if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) { - sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0; - sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1; - pixel_offset = 0.0f; + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; } + int width_B = K/4; + int padded_height_B = (N + padding)/4; - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne2)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne3)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(float), &sf0)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(float), &sf1)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(float), &sf2)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(float), &sf3)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float), &pixel_offset)); - } + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - size_t dst_total_elements = (size_t)ne0 * ne1 * ne2 * ne3; - if (dst_total_elements == 0) { - return; - } - size_t global_work_size[] = { dst_total_elements, 1, 1 }; - size_t local_work_size_pref = 256; - size_t local_work_size[] = { MIN(local_work_size_pref, dst_total_elements), 1, 1}; + // gemm + kernel = backend_ctx->kernel_gemm_noshuffle_q8_0_f32; + int padded_N = N + padding; - size_t * local_work_size_ptr = local_work_size; - if (dst_total_elements % local_work_size[0] != 0 && !backend_ctx->non_uniform_workgroups) { - local_work_size_ptr = nullptr; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; + size_t local_work_size[] = { 2, 128, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; - cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; const int ne00 = src0->ne[0]; const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; - - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; - - const cl_ulong nb10 = src1->nb[0]; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; - const int ne0 = dst->ne[0]; const int ne1 = dst->ne[1]; - const int ne2 = dst->ne[2]; - const int ne3 = dst->ne[3]; - - const cl_ulong nb0 = dst->nb[0]; - const cl_ulong nb1 = dst->nb[1]; - const cl_ulong nb2 = dst->nb[2]; - const cl_ulong nb3 = dst->nb[3]; - - const cl_int dim = ((const int32_t *) dst->op_params)[0]; - GGML_ASSERT(dim >= 0 && dim <= 3); - int nth = MIN(64, ne0); + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - cl_kernel kernel = backend_ctx->kernel_concat_f32; + cl_context context = backend_ctx->context; + cl_kernel kernel; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb0)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb1)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nb2)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nb3)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_int), &dim)); + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - size_t global_work_size[] = {(size_t)ne1*nth, (size_t)ne2, (size_t)ne3}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + int M = ne01; + int N = ne1; + int K = ne00; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); -} + // CoopK handles with _bin use the bin weight layout (uint-transposed q), with K-major Q8_1 activations (K contiguous). + // This switch disables only this bin-layout path; weight conversion is unchanged. + static const char * env_coopk_bin = getenv("GGML_OPENCL_COOPK_Q4_K_BIN"); + const bool use_coopk_bin = !env_coopk_bin || !*env_coopk_bin || atoi(env_coopk_bin) != 0; + if (use_coopk_bin && ne1 >= 2 && ne1 <= 4 && + ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_k) && + backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 != nullptr && + ((ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 + : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4) != nullptr) { + cl_mem b_sub_buf = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); -static void ggml_cl_timestep_embedding(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + // Allocate all kernel columns; padded columns have independent accumulators and are not stored. + const int cols = (ne1 <= 2) ? 2 : 4; + const size_t size_qa = (size_t)cols * K * sizeof(cl_char); + const size_t n_blocks_padded = (size_t)cols * (K / 32); + const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size; + const size_t size_da_prev = backend_ctx->prealloc_moe_da.size; + backend_ctx->prealloc_moe_qa.allocate(context, size_qa); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half)); + if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) { + const cl_char z8 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer, + &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL)); + } + if (backend_ctx->prealloc_moe_da.size != size_da_prev) { + const cl_half z16 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL)); + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL)); + } - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_int n_blocks = (cl_int)((size_t)N * (K / 32)); + cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks)); + size_t local_work_size_quant[1] = { 64 }; + size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst); + + cl_kernel kernel_coopk = (ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 + : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4; + const int nsg_selected = (ne1 <= 2) ? backend_ctx->nsg_coopk_dp4a_q4_k_bin_c2 + : backend_ctx->nsg_coopk_dp4a_q4_k_bin_c4; + const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0; + const cl_int n_padded = cols; + CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_k->q)); + CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &n_padded)); + CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_int), &ne1)); + CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel_coopk, 14, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel_coopk, 15, sizeof(cl_uchar), &mask_hi2)); + + // Each width can compile with a different subgroup count. + const size_t nsg = (size_t)nsg_selected; + size_t local_work_size[3] = { 64, nsg, 1 }; + size_t global_work_size[3] = { (size_t)(ne01 / backend_ctx->rows_coopk_dp4a_q4_k), nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst); - if (backend_ctx->kernel_timestep_embedding == nullptr) { - GGML_LOG_WARN("%s: timestep_embedding kernel not available, skipping OpenCL execution.\n", __func__); + CL_CHECK(clReleaseMemObject(b_sub_buf)); return; } - ggml_tensor_extra_cl * extra_src0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *)dst->extra; + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - cl_ulong off_src0 = extra_src0->offset + src0->view_offs; - cl_ulong off_dst = extra_dst->offset + dst->view_offs; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - const int logical_dim = dst->op_params[0]; - const int max_period = dst->op_params[1]; - const int dst_nb1_bytes = dst->nb[1]; + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - cl_kernel kernel = backend_ctx->kernel_timestep_embedding; + kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_src0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_dst->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &off_dst)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &dst_nb1_bytes)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &logical_dim)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &max_period)); + size_t local_work_size[3] = { 64, 8, 1 }; + size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - size_t gws0 = (size_t)(((logical_dim + 1) / 2) + 1); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + const int gemm_tile_n = 64; + int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + + static const char * q4_k_bin_dp4a_env = getenv("GGML_OPENCL_Q4_K_BIN_DP4A"); + bool q4_k_bin_dp4a_on = q4_k_bin_dp4a_env + ? (atoi(q4_k_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q4_k_bin_dp4a_on = backend_ctx->has_integer_dot && q4_k_bin_dp4a_on; + + if (q4_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N)); + + size_t local_work_size[3] = { 64, 1, 1 }; + size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - size_t gws1 = (size_t)src0->ne[0]; + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; + } - size_t global_work_size[] = {gws0, gws1, 1}; + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + cl_mem b_buf = nullptr; + if (N_pad == N) { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + b_buf = b_sub_buf; + } else { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_padded; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, NULL, dst); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst) { - const ggml_tensor * v = dst->src[2]; - const ggml_tensor * mask = dst->src[3]; - const ggml_tensor * sinks = dst->src[4]; - GGML_ASSERT(q->extra); - GGML_ASSERT(k->extra); - GGML_ASSERT(v->extra); +static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - if (mask) { - GGML_ASSERT(mask->extra); - } - if (sinks) { - GGML_ASSERT(sinks->extra); - } ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - const int n_q = q->ne[1]; - const int n_kv = k->ne[1]; - const int d_head_q = q->ne[0]; - const int d_head_v = v->ne[0]; - const int n_head = q->ne[2]; - const int n_head_kv = k->ne[2]; - const int n_batch = q->ne[3]; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; - cl_kernel kernel = NULL; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - const bool is_f16 = q->type == GGML_TYPE_F16; - const bool is_mixed = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16; - const std::pair dk_dv = {d_head_q, d_head_v}; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - if (n_q == 1) { - if (is_mixed) { - kernel = backend_ctx->kernels_flash_attn_f32_f16_q1.at(dk_dv); - } else if (is_f16) { - kernel = backend_ctx->kernels_flash_attn_f16_q1.at(dk_dv); - } else { - kernel = backend_ctx->kernels_flash_attn_f32_q1.at(dk_dv); - } - } else { - if (is_mixed) { - kernel = backend_ctx->kernels_flash_attn_f32_f16.at(dk_dv); - } else if (is_f16) { - kernel = backend_ctx->kernels_flash_attn_f16.at(dk_dv); - } else { - kernel = backend_ctx->kernels_flash_attn_f32.at(dk_dv); - } - } - GGML_ASSERT(kernel != NULL); + const int ne1 = dst->ne[1]; - ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *)q->extra; - ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *)k->extra; - ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *)v->extra; - ggml_tensor_extra_cl * extra_o = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl * extra_mask = mask ? (ggml_tensor_extra_cl *)mask->extra : NULL; - ggml_tensor_extra_cl * extra_sinks = sinks ? (ggml_tensor_extra_cl *)sinks->extra : NULL; + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - cl_ulong offset_q = extra_q->offset + q->view_offs; - cl_ulong offset_k = extra_k->offset + k->view_offs; - cl_ulong offset_v = extra_v->offset + v->view_offs; - cl_ulong offset_o = extra_o->offset + dst->view_offs; - cl_mem mask_buffer = extra_mask ? extra_mask->data_device : NULL; - cl_ulong offset_mask = extra_mask ? extra_mask->offset + mask->view_offs : 0; - cl_mem sinks_buffer = extra_sinks ? extra_sinks->data_device : NULL; - cl_ulong offset_sinks = extra_sinks ? extra_sinks->offset + sinks->view_offs : 0; + cl_context context = backend_ctx->context; + cl_kernel kernel; - const cl_ulong q_nb1 = q->nb[1], q_nb2 = q->nb[2], q_nb3 = q->nb[3]; - const cl_ulong k_nb1 = k->nb[1], k_nb2 = k->nb[2], k_nb3 = k->nb[3]; - const cl_ulong v_nb1 = v->nb[1], v_nb2 = v->nb[2], v_nb3 = v->nb[3]; - const cl_ulong o_nb1 = dst->nb[1], o_nb2 = dst->nb[2], o_nb3 = dst->nb[3]; - const cl_ulong mask_nb1 = mask ? mask->nb[1] : 0; - const cl_ulong mask_nb2 = mask ? mask->nb[2] : 0; - const cl_ulong mask_nb3 = mask ? mask->nb[3] : 0; - const int mask_ne2 = mask ? mask->ne[2] : 0; - const int mask_ne3 = mask ? mask->ne[3] : 0; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; - float scale, max_bias, logit_softcap; - const float * params = (const float *)dst->op_params; - scale = params[0]; - max_bias = params[1]; - logit_softcap = params[2]; + int M = ne01; + int N = ne1; + int K = ne00; - const int is_causal = (mask == NULL && n_q > 1 && n_q == n_kv); + cl_uchar mask_d6 = 0x3F; + cl_uchar mask_d4 = 0x0F; + cl_uchar mask_hi2 = 0xC0; - const int n_head_log2_val = n_head > 0 ? 1u << (int)floorf(log2f((float)n_head)) : 0; - const float n_head_log2_f = n_head_log2_val > 0 ? (float)n_head_log2_val : 1.0f; - const float m0 = powf(2.0f, -(max_bias) / n_head_log2_f); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2_f); + // Multi-column verify GEMV: route the spec/MTP verify batch (ne1==3 = 2 + // drafts + 1 bonus) onto the efficient GEMV path (subgroup-broadcast, no + // transpose) instead of the transposed-GEMM dead-zone. Reuses the ne1==1 + // GEMV setup (the activation image is already sized by N=ne1). Byte- + // identical. Opt-in via GGML_OPENCL_Q4K_MC3=1 while validating. + static const bool q4k_mc3 = (getenv("GGML_OPENCL_Q4K_MC3") != nullptr); + // Per-layer only (ne01 < 32768): the batched large-vocab lm_head at ne1==3 + // is left to the existing routing (corrupts on the Adreno GEMV path; x2- + // unified routes batched Q6_K lm_head to CPU). Per-layer mc3 is byte-identical. + const bool use_mc3 = q4k_mc3 && (ne1 == 3) && (ne01 < 32768); + + const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q4K_MC3 is bypassed by Q4_K binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q4_k_f32_adreno_ila(backend, src0, src1, dst); + return; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra_q->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset_q)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra_k->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset_k)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra_v->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset_v)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra_o->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset_o)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(float), &scale)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n_q)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n_kv)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &is_causal)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_head)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &q_nb1)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &q_nb2)); CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &q_nb3)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &k_nb1)); CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &k_nb2)); CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &k_nb3)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &v_nb1)); CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &v_nb2)); CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &v_nb3)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &o_nb1)); CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &o_nb2)); CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &o_nb3)); - CL_CHECK(clSetKernelArg(kernel, 25, sizeof(float), &max_bias)); - CL_CHECK(clSetKernelArg(kernel, 26, sizeof(float), &m0)); - CL_CHECK(clSetKernelArg(kernel, 27, sizeof(float), &m1)); - CL_CHECK(clSetKernelArg(kernel, 28, sizeof(int), &n_head_log2_val)); - CL_CHECK(clSetKernelArg(kernel, 29, sizeof(float), &logit_softcap)); - CL_CHECK(clSetKernelArg(kernel, 30, sizeof(int), &n_head_kv)); - CL_CHECK(clSetKernelArg(kernel, 31, sizeof(cl_mem), &mask_buffer)); - CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &offset_mask)); - CL_CHECK(clSetKernelArg(kernel, 33, sizeof(cl_ulong), &mask_nb1)); - CL_CHECK(clSetKernelArg(kernel, 34, sizeof(cl_ulong), &mask_nb2)); - CL_CHECK(clSetKernelArg(kernel, 35, sizeof(cl_ulong), &mask_nb3)); - CL_CHECK(clSetKernelArg(kernel, 36, sizeof(int), &mask_ne2)); - CL_CHECK(clSetKernelArg(kernel, 37, sizeof(int), &mask_ne3)); - CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_mem), &sinks_buffer)); - CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_ulong), &offset_sinks)); + if (ne1 == 1 || use_mc3) { + cl_mem q_img = nullptr; + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - if (n_q == 1) { - const size_t wg_size = 64; - size_t local_work_size[] = { wg_size, 1 }; - size_t global_work_size[] = { wg_size, (size_t)(n_head * n_batch) }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); - } else { - const int block_m = backend_ctx->kernels_flash_attn_bm.at(dk_dv); - const size_t wg_size = block_m; - size_t local_work_size[] = { wg_size, 1 }; - size_t global_work_size[] = { (size_t)((n_q + block_m - 1) / block_m) * wg_size, (size_t)(n_head * n_batch) }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); - } -} + const bool use_tiled = !use_mc3 && use_q4k_tiled(backend_ctx, src0); -static void ggml_cl_mul_mat_f16_f32_tiled(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // image for q (not needed for the tiled path, which reads __global) + if (!use_tiled) { + img_fmt = { CL_R, CL_UNSIGNED_INT32}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_k->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + } - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - const int M = src0->ne[1]; - const int N = src1->ne[1]; - const int K = src0->ne[0]; + // 4-output-per-WI o4 variant for the long-vocab lm_head/embed GEMV + // (ne01 = vocab ~256K on Gemma): shares one activation read across 4 + // output rows. Gated to large ne01 (lm_head/embed). Default on; opt-out + // GGML_OPENCL_Q4K_GEMV_O4=0. (Skipped when mc3 handles the ne1==3 verify.) + static const bool q4k_o4_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_O4"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + const bool use_q4k_o4 = !use_tiled && !use_mc3 && q4k_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); + // Split-K across workgroups for small-M decode GEMVs. A single-token GEMV + // makes only CEIL_DIV(M/2,64) workgroups; even with the wide intra-WG split + // (16 subgroups) those all land on ONE CU, so small-M matmuls under-fill the + // 16 CUs and their bandwidth falls well short of what the large-M FFN matmuls + // reach. Adding a `ksplit` second grid dim that spreads K across WGs (+ a + // reduce pass) fills the CUs. Gate is M<=2560: the tiny M<=1024 ones only + // break even (the reduce dispatch eats the kernel win), but the big-K M=2560 + // cases (ffn_down, attn_output) make the per-call win dwarf the reduce, and + // are byte-identical. ffn_gate/up (large M) fill the CUs already and are excluded. + // + // DEVICE-GATED. Split-K buys GPU time by spending an extra kernel LAUNCH (the + // reduce), so it only pays where launches are cheap. That is a per-device + // property and it does not travel from the X2-90 this was tuned on. Measured + // with one binary, env A/B (tg32, GGML_OPENCL_Q4K_GEMV_SPLITK=0/1): + // + // X2-90 +3.36% gemma-4 E4B (the number this gate was built on) + // 840 -1.3% Qwen3.5-4B-Q4_K_M 14.00 -> 13.85 + // 850 -20.0% Qwen3-1.7B-Q4_K_M 6.97 -> 5.58 (6 interleaved reps) + // + // The kernel is not the problem. On the 850 split-K makes the GPU strictly + // faster -- total busy 537 -> 485 ms, this GEMV 43.7 -> 34.0 us/call (-22%) -- + // and still costs a fifth of decode, because the +3696 reduce dispatches cost + // ~550 us of HOST round-trip each against 2.7 us of GPU work (~200x; that part + // is ~95% host-bound at decode). The 840 pays the same tax at ~42 us/dispatch. + // Break-even needs launch cost below the ~9.7 us/call the split actually saves, + // so this is not a "the 850 is slow" adjustment that a faster part would fix -- + // the 840 is 13x cheaper per launch and still loses. + // + // Enabled where it is measured to win, i.e. X2E only. The X1-85 was measured + // afterwards and is NOT a win either: Qwen3.5-4B-Q4_K_M tg32, split-K off + // 17.98/18.10/18.19 vs on 18.03/17.91/17.97 = -0.7%, so X1E stays excluded on + // evidence rather than on absence of it. Do not widen this without a NEW + // measurement. The env still forces either way so every device stays measurable. + static const bool splitk_env_set = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); + return e && e[0] != '\0'; + }(); + static const bool splitk_env_on = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_SPLITK"); + return !(e && e[0] == '0'); + }(); + const bool splitk_wg_env = splitk_env_set + ? splitk_env_on + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // Gate: small-M decode GEMVs that under-fill the 16 CUs even with the wide + // intra-WG split (all 16 subgroups land on one CU). M<=2560 covers Kcur/Vcur + // (M=1024), Qcur (2048), attn_output + ffn_down (2560). The tiny ones + // (M<=1024) only break even (reduce dispatch eats the kernel win), but the + // big-K M=2560 cases (ffn_down K=10240 @182us, attn_output @42us) have a + // large per-call win that dwarfs the ~5us reduce, so extending to 2560 nets + // positive end-to-end. ffn_gate/up (M=10240) already fill the CUs -> excluded. + const bool use_splitk = splitk_wg_env && !use_tiled && !use_q4k_o4 && !use_mc3 && ne01 <= 2560; + + if (use_splitk) { + const int nsg = 8; + const int ksplit = (ne01 <= 512) ? 8 : 4; // -> ~32 total WGs + const size_t gx = (size_t)CEIL_DIV(ne01/2, 64) * 64; + + backend_ctx->prealloc_splitk_partial.allocate( + backend_ctx->context, (size_t)ksplit * ne01 * sizeof(float)); + cl_mem partial = backend_ctx->prealloc_splitk_partial.buffer; + + cl_kernel ks = backend_ctx->kernel_gemv_noshuffle_q4_k_f32_splitk; + CL_CHECK(clSetKernelArg(ks, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(ks, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(ks, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(ks, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(ks, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(ks, 5, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(ks, 6, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(ks, 7, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(ks, 8, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(ks, 9, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(ks, 10, sizeof(cl_uchar), &mask_hi2)); + size_t lsk[3] = {64, (size_t)nsg, 1}; + size_t gsk[3] = {gx, (size_t)(nsg * ksplit), 1}; + backend_ctx->enqueue_ndrange_kernel(ks, 3, gsk, lsk, dst); + + cl_kernel kr = backend_ctx->kernel_gemv_splitk_reduce_f32; + CL_CHECK(clSetKernelArg(kr, 0, sizeof(cl_mem), &partial)); + CL_CHECK(clSetKernelArg(kr, 1, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kr, 2, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kr, 3, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kr, 4, sizeof(cl_int), &ksplit)); + size_t lr[3] = {64, 1, 1}; + size_t gr[3] = {(size_t)CEIL_DIV(ne01, 64) * 64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kr, 3, gr, lr, dst); + + if (q_img) CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + return; + } - cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_tiled; + kernel = use_mc3 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_mc3 + : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_tiled + : use_q4k_o4 ? backend_ctx->kernel_gemv_noshuffle_q4_k_f32_o4 + : backend_ctx->kernel_gemv_noshuffle_q4_k_f32; + + if (use_tiled) { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + } else { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_hi2)); + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + // Wide K-split for the decode GEMV: the default 4-subgroup K-split leaves + // each Adreno SP with only ~4 waves, too few to hide LPDDR weight-load + // latency, so even the large FFN matmuls run well below the achievable + // bandwidth. Widen to 16 subgroups/WG (= the 1024-lane Adreno WG max) so + // each SP holds enough in-flight memory requests. Prefill is unaffected (the + // GEMM path is separate) and coherence-identical (greedy output unchanged). + // Applies to the plain base + // GEMV only; tiled/o4/mc3 keep 4 (their reductions are hard-coded to 4). + // Layout-safe: the base kernel derives its K-split from get_local_size(1) + // and the packed block stride is a physical constant (independent of it). + // Opt-out: GGML_OPENCL_Q4K_GEMV_WIDE=0. + static const bool splitk_wide_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q4K_GEMV_WIDE"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + const bool splitk_wide = splitk_wide_env && !use_tiled && !use_q4k_o4 && !use_mc3; + size_t nsg_y = splitk_wide ? 16 : 4; + // Cap the wide K-split by the kernel's real max WG. X1-class drivers cap + // this GEMV at 768 (< 64*16 = 1024), so an uncapped lws aborts the + // dispatch with CL_INVALID_WORK_GROUP_SIZE (-54) and breaks ALL q4_K + // decode for M>2560. nsg_y is a pure K-split (the base kernel reads it + // from get_local_size(1); the packed block stride is a physical constant), + // so halving it stays coherent — just a narrower split. X2 keeps 16 + // (maxwg 1024); X1 falls to 8. + if (splitk_wide) { + const size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); + while (nsg_y > 4 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } + } + size_t local_work_size[3] = {64, nsg_y, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(use_tiled ? ne01 : (use_q4k_o4 ? ne01/4 : ne01/2), 64)*64, nsg_y, 1}; - // Tiling parameters. These need to be tuned for optimal performance. - // They must match the #defines in the kernel mul_mat_f16_f32.cl. - // - // OPWM / OPWN: Output tile size per Work-Group. A work-group computes a tile of size OPWM x OPWN. - // TPWM / TPWN: Threads per Work-group. This is the work-group size. - // OPTM / OPTN: Output elements per Thread. Each thread computes OPTM x OPTN elements. - // - // The following relationships must hold: - // OPWM = TPWM * OPTM - // OPWN = TPWN * OPTN - // - const int OPWM = 64; - const int OPWN = 64; - const int TPWM = 16; - const int TPWN = 8; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - size_t local_work_size[2] = { TPWM, TPWN }; - size_t global_work_size[2] = { - (size_t) ((M + OPWM - 1) / OPWM) * TPWM, - (size_t) ((N + OPWN - 1) / OPWN) * TPWN, - }; + if (q_img) CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + + cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; + + // subbuffer for activations + region.origin = offset1; + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); -} + // CoopK handles without _bin use the same untiled weight layout as gemm_noshuffle_q4_k_f32. + // Activations use K-major Q8_1 SoA (K contiguous). + if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_k) + && backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4 != nullptr + && ((N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4) != nullptr) { + const int cols = (N <= 2) ? 2 : 4; + const size_t size_qa = (size_t)cols * K * sizeof(cl_char); + const size_t n_blocks_padded = (size_t)cols * (K / 32); + const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size; + const size_t size_da_prev = backend_ctx->prealloc_moe_da.size; + backend_ctx->prealloc_moe_qa.allocate(context, size_qa); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half)); + if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) { + const cl_char z8 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer, + &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL)); + } + if (backend_ctx->prealloc_moe_da.size != size_da_prev) { + const cl_half z16 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL)); + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL)); + } -static void ggml_cl_conv_2d(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_TENSOR_BINARY_OP_LOCALS; - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + cl_int n_blocks = (cl_int)((size_t)N * (K / 32)); + cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks)); + size_t local_work_size_quant[1] = { 64 }; + size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst); + + cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4; + const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0; + const cl_int n_padded = cols; + CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_k->q)); + CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &n_padded)); + CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel_coopk, 14, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel_coopk, 15, sizeof(cl_uchar), &mask_hi2)); + + const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q4_k; + size_t local_work_size[3] = { 64, nsg, 1 }; + size_t global_work_size[3] = { (size_t)(M / backend_ctx->rows_coopk_dp4a_q4_k), nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // image for activations + img_fmt = {CL_RGBA, CL_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + // pad N to multiple of 8 + int extra_elements = N % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; + } - const cl_uint Cout = ne03; const cl_uint Cin = ne02; const cl_uint N = ne13; - const cl_uint KW = ne00; const cl_uint KH = ne01; const cl_uint W = ne10; const cl_uint H = ne11; const cl_uint OW = ne0; const cl_uint OH = ne1; + // subbuffer for transposed activations + region.origin = 0; + region.size = K * (N + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - const cl_uint s0 = dst->op_params[0]; const cl_uint s1 = dst->op_params[1]; - const cl_uint p0 = dst->op_params[2]; const cl_uint p1 = dst->op_params[3]; - const cl_uint d0 = dst->op_params[4]; const cl_uint d1 = dst->op_params[5]; + // image for transposed activations + img_fmt = {CL_RGBA, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = K * (N + padding) / 4; + img_desc.buffer = b_sub_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); - const cl_uint cl_nb01 = nb01/ggml_type_size(src0->type); const cl_uint cl_nb02 = nb02/ggml_type_size(src0->type); const cl_uint cl_nb03 = nb03/ggml_type_size(src0->type); - const cl_uint cl_nb11 = nb11/ggml_type_size(src1->type); const cl_uint cl_nb12 = nb12/ggml_type_size(src1->type); const cl_uint cl_nb13 = nb13/ggml_type_size(src1->type); - const cl_uint cl_nb1 = nb1/ggml_type_size(dst->type); const cl_uint cl_nb2 = nb2/ggml_type_size(dst->type); const cl_uint cl_nb3 = nb3/ggml_type_size(dst->type); + // transpose activations + int height_B = N/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = K/4; + int padded_height_B = (N + padding)/4; - const int64_t NPQ = (int64_t)N * OW * OH; + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - const uint32_t BS_K = 64; - const uint32_t BS_NPQ = 64; - const uint32_t BS_CRS = 16; - const uint32_t VEC_SIZE = 4; + size_t local_work_size_t[2] = { 1, 16 }; + size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - const uint32_t TS_K = 4; - const uint32_t TS_NPQ = 8; + // dp4a (int8) dense prefill GEMM and weight via texture + static const char * q4k_dense_dp4a_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A"); + static const char * q4k_dense_wimg_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A_WIMG"); + + const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0); + bool q4k_dense_dp4a_on = q4k_dense_wimg_on + ? true + : q4k_dense_dp4a_env + ? (atoi(q4k_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + + // dp4 has to be available + q4k_dense_dp4a_on = backend_ctx->has_integer_dot && q4k_dense_dp4a_on; + + // Min N for the dp4a prefill GEMM, default 9, i.e., ne1 > 8 + static const char * q4k_dp4a_minn_env = getenv("GGML_OPENCL_Q4K_DP4A_MINN"); + const int q4k_dp4a_minn = q4k_dp4a_minn_env ? atoi(q4k_dp4a_minn_env) : 9; + + if (q4k_dense_dp4a_on && N >= q4k_dp4a_minn && (K % 32 == 0) && (M % 64 == 0)) { + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + // check if weights go through texture + cl_mem q4k_q_img = nullptr; + bool use_wimg = q4k_dense_wimg_on; + if (use_wimg) { + const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels = bytes/4 + if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { + use_wimg = false; + } else { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = tex; + img_desc.buffer = extra0_q4_k->q; + q4k_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); + if (err != CL_SUCCESS || q4k_q_img == nullptr) { + use_wimg = false; + q4k_q_img = nullptr; + } + } + } - const uint32_t WG_K = BS_K / TS_K; - const uint32_t WG_NPQ = BS_NPQ / TS_NPQ; + const bool use_narrow = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow != nullptr + && N <= backend_ctx->max_n_dp4a_narrow_q4_k; + static const char * q4_k_act_vload4_env = getenv("GGML_OPENCL_Q4_K_DP4A_ACT_VLOAD4"); + const bool q4_k_act_vload4_on = q4_k_act_vload4_env == nullptr || atoi(q4_k_act_vload4_env) != 0; + cl_kernel dk = use_narrow + ? (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg + : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow) + : use_wimg + ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg + : q4_k_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a; + int ai = 0; + if (use_wimg) { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img)); + } else { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->q)); + } + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); + const int tile_n = use_narrow ? backend_ctx->tile_n_dp4a_narrow_q4_k : backend_ctx->tile_n_dp4a_q4_k; + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, tile_n) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + if (q4k_q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q4k_q_img)); + } + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + return; + } - auto splitWork = [](uint32_t work_size, uint32_t block_size) { return (block_size + work_size - 1) / block_size; }; - const uint32_t NB_K = splitWork(Cout, BS_K); - const uint32_t NB_NPQ = splitWork(NPQ, BS_NPQ); + // gemm + // Small-batch (medium n_q) occupancy fix: at ne1<=8 the 2x8 grid is + // (1, ceil(M/2)) -> ~M/256 workgroups, which under-occupies the SP and + // makes the GEMM much slower than the ne1==1 GEMV at the same weight + // traffic. The _r1 (1-row) kernel doubles the M-axis workgroup count + // and removes the accumulator spill. Opt-in via env while validating. + static const bool q4k_gemm_r1 = (getenv("GGML_OPENCL_Q4K_GEMM_R1") != nullptr); + static const bool q4k_gemm_kimg = (getenv("GGML_OPENCL_Q4K_GEMM_KIMG") != nullptr); + // Cooperative-K (intra-WG K-split + reduction) for the small-batch + // (n_q in [2..8]) path: DEFAULT ON, opt out with GGML_OPENCL_Q4K_GEMM_COK=0. + // Byte-identical greedy output; large-batch (ne1>8) untouched. + static const char * q4k_cok_env = getenv("GGML_OPENCL_Q4K_GEMM_COK"); + static const bool q4k_gemm_cok = (q4k_cok_env == nullptr) || (atoi(q4k_cok_env) != 0); + const bool use_cok = q4k_gemm_cok && (ne1 <= 8); + const bool use_r1 = !use_cok && q4k_gemm_r1 && (ne1 <= 8); + // Weights-as-image (L1/TPL1) for the small-batch weight-read-bound path. + const bool use_kimg = !use_cok && !use_r1 && q4k_gemm_kimg && (ne1 <= 8); - cl_kernel kernel; - size_t shmem_size; + cl_mem q_img = nullptr; + if (use_kimg) { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q4_k->q; + CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + } - if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { - kernel = backend_ctx->kernel_conv_2d_f16; - shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_half4)); - } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_conv_2d_f32; - shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_float) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); - } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) { - kernel = backend_ctx->kernel_conv_2d_f16_f32; - shmem_size = (size_t)(BS_K * BS_CRS * sizeof(cl_half) + BS_CRS * (BS_NPQ / VEC_SIZE) * sizeof(cl_float4)); - } else { - GGML_ASSERT(false && "Unsupported data type combination for conv2d"); - } + kernel = use_cok ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_cok + : use_r1 ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_r1 + : use_kimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_f32_kimg + : backend_ctx->kernel_gemm_noshuffle_q4_k_f32; + int padded_N = N + padding; - cl_uint idx = 0; - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra1->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extrad->data_device)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, idx++, shmem_size, NULL)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cout)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &Cin)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &N)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KW)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &KH)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &W)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OW)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &OH)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &p1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d0)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &d1)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb01)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb02)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb03)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb11)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb12)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb13)); - CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb1)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb2)); CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &cl_nb3)); + if (use_kimg) { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); + } else { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); + } + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_hi2)); - size_t global_work_size[] = { (size_t)NB_K * WG_K, (size_t)NB_NPQ * WG_NPQ, 1 }; - size_t local_work_size[] = { (size_t)WG_K, (size_t)WG_NPQ, 1 }; + size_t global_work_size[3]; + size_t local_work_size[3]; + if (use_cok) { + // (COK_SG lanes x COK_NSG subgroups): one row per lane, K split + // across the COK_NSG subgroups. ne01 is a multiple of 64. + global_work_size[0] = (size_t)ne01; // rows + global_work_size[1] = 8; // COK_NSG + global_work_size[2] = 1; + local_work_size[0] = 64; // COK_SG + local_work_size[1] = 8; // COK_NSG + local_work_size[2] = 1; + } else if (use_r1) { + // 1 row per WI (opt-in occupancy experiment). + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)ne01; + global_work_size[2] = 1; + local_work_size[0] = 1; + local_work_size[1] = 128; + local_work_size[2] = 1; + } else if (use_kimg) { + // kimg is a 2-row tile (opt-in weights-as-image experiment). + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)CEIL_DIV(ne01, 2); + global_work_size[2] = 1; + local_work_size[0] = 1; + local_work_size[1] = 128; + local_work_size[2] = 1; + } else { + // Default: x2-unified base kernel is the 4-row (gx<<2) tile. + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); + global_work_size[2] = 1; + local_work_size[0] = 1; + local_work_size[1] = 128; + local_work_size[2] = 1; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + if (q_img) CL_CHECK(clReleaseMemObject(q_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif } -static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - const cl_ulong nb10 = src1->nb[0]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; + const int ne1 = dst->ne[1]; - GGML_ASSERT(ne00 == ne10); + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - cl_kernel kernel; cl_context context = backend_ctx->context; + cl_kernel kernel; - cl_int status; - cl_image_format img_fmt_1d; - cl_image_desc img_desc_1d; - cl_buffer_region region; - cl_mem A_image1d; - cl_mem A_sub_buffer; - cl_mem B_sub_buffer; - cl_mem D_image1d; - cl_mem D_sub_buffer; - - int M = ne01; - int N = ne1; - int K = ne00; - - if (nb01 > nb02) { - // KQ - kernel = backend_ctx->kernel_mul_mm_f16_f32_kq; - } else { - // KQV - kernel = backend_ctx->kernel_mul_mm_f16_f32_kqv; - } - // create sub-buffer for A - // <--------------------------------------------> // - extra0 = src0->view_src ? (ggml_tensor_extra_cl *)src0->view_src->extra : (ggml_tensor_extra_cl *)src0->extra; - - region.origin = (extra0->offset); - if (nb01 > nb02) { - // KQ - region.size = nb01 * ne01; - } else { - // KQV - region.size = nb02 * ne02; - } - - A_sub_buffer = clCreateSubBuffer((extra0->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); - CL_CHECK(status); + cl_int err; + cl_buffer_region region; + cl_image_format img_fmt; + cl_image_desc img_desc; - // <--------------------------------------------> // + const int M = ne01; + const int N = ne1; + const int K = ne00; - // create sub-buffer for B - // <--------------------------------------------> // - region.origin = (extra1->offset); - region.size = nb10 * ne10 * ne11 * ne12; - B_sub_buffer = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); - CL_CHECK(status); - // <--------------------------------------------> // + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - img_fmt_1d = {CL_RGBA, CL_FLOAT}; - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - if (nb01 > nb02) { - img_desc_1d.image_width = (nb01 * ne01 / 4)/4; - } - else { - img_desc_1d.image_width = (nb02 * ne02 / 4)/4; - } - img_desc_1d.buffer = A_sub_buffer; - A_image1d = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); - CL_CHECK(status); + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // create sub-buffer for output C - // <--------------------------------------------> // - region.origin = (extrad->offset); - region.size = ne0 * ne1 * dst->ne[2] * dst->nb[0]; // size of C in bytes - D_sub_buffer = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); - CL_CHECK(status); - // <--------------------------------------------> // + img_fmt = { CL_RGBA, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // create image for C output - // <--------------------------------------------> // - img_fmt_1d = {CL_R, CL_FLOAT}; - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4; - img_desc_1d.buffer = D_sub_buffer; - D_image1d = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt_1d, &img_desc_1d, NULL, &status); - CL_CHECK(status); - // <--------------------------------------------> // + kernel = backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - int offset_src0 = 0; - int offset_src1 = 0; + size_t local_work_size[3] = { 64, 8, 1 }; + size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - // set kernel args - // <--------------------------------------------> // - cl_uint k_arg = 0; - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src0)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_sub_buffer)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset_src1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &D_image1d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &extrad->offset)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &nb01)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } else { + const int gemm_tile_n = 64; + int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + + static const char * q6_k_bin_dp4a_env = getenv("GGML_OPENCL_Q6_K_BIN_DP4A"); + bool q6_k_bin_dp4a_on = q6_k_bin_dp4a_env + ? (atoi(q6_k_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q6_k_bin_dp4a_on = backend_ctx->has_integer_dot && q6_k_bin_dp4a_on; + + if (q6_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 1, 1 }; + size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - size_t global_work_size[3] = {64, static_cast(((M+63)/64)), static_cast(((N+31)/32)*ne12)}; - size_t local_work_size[3] = {64, 1, 2}; + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + cl_mem b_buf = nullptr; + if (N_pad == N) { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + b_buf = b_sub_buf; + } else { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_padded; + } - // deallocate sub buffers and images - // <--------------------------------------------> // - CL_CHECK(clReleaseMemObject(A_image1d)); - CL_CHECK(clReleaseMemObject(D_image1d)); - CL_CHECK(clReleaseMemObject(A_sub_buffer)); - CL_CHECK(clReleaseMemObject(B_sub_buffer)); - CL_CHECK(clReleaseMemObject(D_sub_buffer)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); + } } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -9677,170 +23673,661 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; - const int ne1 = dst->ne[1]; + const int ne1 = dst->ne[1]; GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; - cl_int err; - cl_image_format img_fmt; - cl_image_desc img_desc; - cl_buffer_region region; + cl_int err; + cl_buffer_region region; + cl_image_format img_fmt; + cl_image_desc img_desc; - int M = ne01; - int N = ne1; - int K = ne00; + // subbuffer and image for activation + // Multi-column verify GEMV: route the spec/MTP verify q6_K matmuls (ne1==3) + // onto the efficient GEMV path instead of the transposed-GEMM dead-zone. + // Reuses the ne1==1 image setup (activation image sized by N=ne1). Byte- + // identical. Opt-in via GGML_OPENCL_Q6K_MC3=1 while validating. + static const bool q6k_mc3 = (getenv("GGML_OPENCL_Q6K_MC3") != nullptr); + // Per-layer only (ne01 < 32768): batched large-vocab lm_head stays on the + // existing path (x2-unified routes batched Q6_K lm_head to CPU; the Adreno + // GEMV corrupts it). Per-layer mc3 is byte-identical. + const bool use_q6k_mc3 = q6k_mc3 && (ne1 == 3) && (ne01 < 32768); + // Batched verify lm_head/embed (ne1==3, tiled layout): multi-column tiled + // GEMV — streams the large lm_head weight once across the 3 verify columns + // (the #1 MTP bottleneck; mc3 above can't, it reads the noshuffle layout). + const bool use_q6k_tiled_mc = q6k_mc3 && (ne1 == 3) && (ne01 >= 32768) && use_q6k_tiled(backend_ctx, src0); + + const bool use_bin = use_q6_k_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_q6k_mc3 || use_q6k_tiled_mc) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q6K_MC3 is bypassed by Q6_K binary kernels\n"); + warned = true; + } + } + // This bin layout is not supported by CoopK. + ggml_cl_mul_mat_q6_K_f32_adreno_ila(backend, src0, src1, dst); + return; + } - if (ne1 == 1) { - cl_mem q_img = nullptr; - cl_mem b_sub_buf = nullptr; + if (ne1 == 1 || use_q6k_mc3 || use_q6k_tiled_mc) { + cl_mem ql_img = nullptr; + cl_mem qh_img = nullptr; + cl_mem b_sub_buffer = nullptr; cl_mem b_img = nullptr; - // image for q - img_fmt = { CL_R, CL_UNSIGNED_INT32}; + // o4 = 4-output-per-WI variant for long-vocab lm_head/embed; gated to + // ne01 >= 32768 so per-layer q6_K (ne01=hidden 2-8K) keeps the 2-output + // kernel (o4 regresses there). o4_global reads the weights from __global + // coalesced instead of image1d_buffer -- the texture cache caps the + // read-once-per-token lm_head bandwidth, while __global reaches the higher + // rate the rest of the model gets. Both default ON; opt out via + // GGML_OPENCL_Q6K_GEMV_O4 / GGML_OPENCL_Q6K_GEMV_O4_GLOBAL = 0. + static const bool gemv_o4_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + static const bool o4_global_env = []{ + const char * e = std::getenv("GGML_OPENCL_Q6K_GEMV_O4_GLOBAL"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + const bool use_tiled = !use_q6k_mc3 && use_q6k_tiled(backend_ctx, src0); + const bool use_o4 = !use_tiled && !use_q6k_mc3 && gemv_o4_env && (ne01 % 4 == 0) && (ne01 >= 32768); + const bool use_o4_global = use_o4 && o4_global_env; + + // ql/qh image views are only needed when NOT reading weights from global. + if (!use_o4_global && !use_tiled) { + // image for ql + img_fmt.image_channel_order = CL_R; + img_fmt.image_channel_data_type = CL_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne01 * ne00 / 8; + img_desc.buffer = extra0_q6_K->ql; + CL_CHECK((ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // image for qh + img_fmt.image_channel_order = CL_R; + img_fmt.image_channel_data_type = CL_HALF_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne01 * ne00 / 8; + img_desc.buffer = extra0_q6_K->qh; + CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + } + + region.origin = offset1; + region.size = ne00 * ne1 * sizeof(float); + CL_CHECK((b_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt.image_channel_order = CL_RGBA; + img_fmt.image_channel_data_type = CL_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q4_1->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + img_desc.image_width = ne00 * ne1 / 4; + img_desc.buffer = b_sub_buffer; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // subbuffer for activations + kernel = use_q6k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_mc3 + : use_q6k_tiled_mc ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled_mc3 + : use_tiled ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_tiled + : use_o4_global ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4_global + : use_o4 ? backend_ctx->kernel_gemv_noshuffle_q6_K_f32_o4 + : backend_ctx->kernel_gemv_noshuffle_q6_K_f32; + + if (use_o4_global || use_tiled) { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + } else { + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &ql_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); + } + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + + const size_t gws_x = use_tiled + ? (size_t) CEIL_DIV(ne01, 64) * 64 + : use_o4 + ? (size_t) CEIL_DIV(ne01/4, 64) * 64 + : (size_t) CEIL_DIV(ne01/2, 64) * 64; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {gws_x, 4, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + if (ql_img) CL_CHECK(clReleaseMemObject(ql_img)); + if (qh_img) CL_CHECK(clReleaseMemObject(qh_img)); + CL_CHECK(clReleaseMemObject(b_sub_buffer)); + CL_CHECK(clReleaseMemObject(b_img)); + } else { + // Tiled-layout batched GEMM. When the weight was converted to the 64-row + // tiled canonical layout (use_q6k_tiled — the default for lm_head/embed), + // the plain noshuffle GEMM below reads it as plain-transposed and produces + // garbage. Use the batched GEMM that matches the decode tiled GEMV's + // layout; it reads the f32 activation directly (column-major, no transpose). + if (use_q6k_tiled(backend_ctx, src0)) { + cl_mem b_sub_buf_t = nullptr; + cl_mem b_img_t = nullptr; + + region.origin = offset1; + region.size = ne00 * ne1 * sizeof(float); + CL_CHECK((b_sub_buf_t = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt.image_channel_order = CL_RGBA; + img_fmt.image_channel_data_type = CL_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne00 * ne1 / 4; + img_desc.buffer = b_sub_buf_t; + CL_CHECK((b_img_t = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + cl_kernel kt = backend_ctx->kernel_gemm_noshuffle_q6_K_f32_tiled; + CL_CHECK(clSetKernelArg(kt, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kt, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kt, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kt, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kt, 4, sizeof(cl_mem), &b_img_t)); + CL_CHECK(clSetKernelArg(kt, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kt, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kt, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kt, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kt, 9, sizeof(int), &ne1)); + + // Must match the kernel: NTILES=4 64-row tiles per work-group (256 rows), + // BN=8 output columns per work-group. + const int BN_T = 16; + const int WROWS = 4 * 64; // NTILES * TILE_ROWS + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01, WROWS) * 64, 4, (size_t)CEIL_DIV(ne1, BN_T)}; + backend_ctx->enqueue_ndrange_kernel(kt, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img_t)); + CL_CHECK(clReleaseMemObject(b_sub_buf_t)); + return; + } + + cl_mem b_sub_buf; + cl_mem b_buf_trans; + cl_mem b_img; + cl_mem b_img_trans; + + // subbuffer for activation region.origin = offset1; - region.size = K * N * sizeof(float); + region.size = ne00 * ne1 * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; + // CoopK uses the same untiled weight layout as gemm_noshuffle_q6_K_f32, with K-major Q8_1 activations (K contiguous). + // Keep output and embedding weights on the existing path for logit stability. + if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q6_k) + && backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4 != nullptr + && strncmp(src0->name, "output", 6) != 0 + && strncmp(src0->name, "token_embd", 10) != 0 + && ((ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4) != nullptr) { + const int N = (int)ne1, K = (int)ne00; + const int cols = (N <= 2) ? 2 : 4; + const size_t n_blocks_padded = (size_t)cols * (K / 32); + const size_t size_qa = (size_t)cols * K * sizeof(cl_char); + const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size; + const size_t size_da_prev = backend_ctx->prealloc_moe_da.size; + backend_ctx->prealloc_moe_qa.allocate(context, size_qa); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half)); + if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) { + const cl_char z8 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer, + &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL)); + } + if (backend_ctx->prealloc_moe_da.size != size_da_prev) { + const cl_half z16 = 0; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL)); + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer, + &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL)); + } + + cl_int n_blocks = (cl_int)((size_t)N * (K / 32)); + cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks)); + size_t local_work_size_quant[1] = { 64 }; + size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst); + + cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2 + : backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4; + const cl_ushort mask_f000 = 0xF000; + const cl_uchar mask_c0 = 0xC0; + const cl_int n_padded = cols; + const cl_int m = (cl_int)ne01, k = (cl_int)ne00, n = (cl_int)ne1; + CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_int), &m)); + CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &n_padded)); + CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &k)); + CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &n)); + CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_ushort), &mask_f000)); + CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_c0)); + + const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q6_k; + size_t local_work_size[3] = { 64, nsg, 1 }; + size_t global_work_size[3] = { (size_t)(ne01 / backend_ctx->rows_coopk_dp4a_q6_k), nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } + + // dp4a (int8) dense q6_K prefill GEMM + static const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A"); + bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr) + ? (atoi(q6k_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen != ADRENO_GPU_GEN::X1E); + // dot prod has to be available + q6k_dense_dp4a_on = backend_ctx->has_integer_dot && q6k_dense_dp4a_on; + + const bool is_output_w_dp4a = strncmp(src0->name, "output", 6) == 0 || + strncmp(src0->name, "token_embd", 10) == 0; + + if (q6k_dense_dp4a_on && !is_output_w_dp4a && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { + const int M = ne01, N = ne1, K = ne00; + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + static const char * q6_k_act_vload4_env = getenv("GGML_OPENCL_Q6_K_DP4A_ACT_VLOAD4"); + const bool q6_k_act_vload4_on = q6_k_act_vload4_env == nullptr || atoi(q6_k_act_vload4_env) != 0; + cl_kernel dk = q6_k_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } + + // image for activation + img_fmt.image_channel_order = CL_RGBA; + img_fmt.image_channel_data_type = CL_FLOAT; memset(&img_desc, 0, sizeof(img_desc)); img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; + img_desc.image_width = ne00 * ne1 / 4; img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->kernel_gemv_noshuffle_q4_1_f32; + // pad N to multiple of 8 + int extra_elements = ne1 % 8; + int padding = 0; + if (extra_elements > 0){ + padding = 8 - extra_elements; + } + + // subbuffer for transposed activation + region.origin = 0; + region.size = ne00 * (ne1 + padding) * sizeof(float)/2; + backend_ctx->prealloc_act_trans.allocate(context, region.size); + CL_CHECK((b_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + // image for transposed activation + img_fmt.image_channel_order = CL_RGBA; + img_fmt.image_channel_data_type = CL_HALF_FLOAT; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = ne00 * (ne1 + padding) / 4; + img_desc.buffer = b_buf_trans; + CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + + // transpose activation + int height_B = ne1/4; + if (height_B == 0) { + height_B = 1; + } + int width_B = ne00/4; + int padded_height_B = (ne1 + padding) / 4; + + kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + + size_t local_size_t[2] = { 1, 16 }; + size_t global_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst); + + // gemm + // Cooperative-K small-batch (n_q in [2..8]) path: intra-WG K-split, + // mirrors the q4_K _cok path (batched serving). OPT-IN + // (GGML_OPENCL_Q6K_GEMM_COK=1), DEFAULT OFF: q6_K is the tied lm_head/ + // output projection, so the K-reassociation perturbs final logits and + // greedy is NOT byte-identical (op-tests pass, output coherent, but not + // bit-exact). It is also NEUTRAL on end-to-end MTP (q4_K cok already + // captured that; the MTP bottleneck moved off the GEMMs). Keep opt-in + // for batched serving until PPL-validated on a non-GDN q6_K model. + static const char * q6k_cok_env = getenv("GGML_OPENCL_Q6K_GEMM_COK"); + static const bool q6k_gemm_cok = (q6k_cok_env != nullptr) && (atoi(q6k_cok_env) != 0); + const bool use_q6k_cok = q6k_gemm_cok && (ne1 <= 8); + kernel = use_q6k_cok ? backend_ctx->kernel_gemm_noshuffle_q6_K_f32_cok + : backend_ctx->kernel_gemm_noshuffle_q6_K_f32; + int padded_N = ne1 + padding; + + cl_ushort mask_f000 = 0xF000; + cl_uchar mask_c0 = 0xC0; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ushort),&mask_f000)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_c0)); + + size_t global_work_size[3]; + size_t local_work_size[3]; + if (use_q6k_cok) { + global_work_size[0] = (size_t)ne01; // rows (1 per lane) + global_work_size[1] = 8; // COK_NSG + global_work_size[2] = 1; + local_work_size[0] = 64; // COK_SG + local_work_size[1] = 8; // COK_NSG + local_work_size[2] = 1; + } else { + global_work_size[0] = (size_t)CEIL_DIV(ne1, 8); + global_work_size[1] = (size_t)CEIL_DIV(ne01, 4); + global_work_size[2] = 1; + local_work_size[0] = 2; + local_work_size[1] = 128; + local_work_size[2] = 1; + } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + } +#else + GGML_UNUSED(backend); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); +#endif +} + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static void ggml_cl_mul_mat_q5_K_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl_q5_K * extra0_q5_k = (ggml_tensor_extra_cl_q5_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + + const int ne1 = dst->ne[1]; + + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); + cl_context context = backend_ctx->context; + cl_kernel kernel; - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + cl_int err; + cl_buffer_region region; + cl_image_format img_fmt; + cl_image_desc img_desc; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + const int M = ne01; + const int N = ne1; + const int K = ne00; - CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { - cl_mem b_sub_buf = nullptr; - cl_mem b_sub_buf_trans = nullptr; - cl_mem b_img = nullptr; - cl_mem b_img_trans = nullptr; + if (ne1 == 1) { + cl_mem b_sub_buf = nullptr; + cl_mem b_img = nullptr; - // subbuffer for activations region.origin = offset1; - region.size = K * N * sizeof(float); + region.size = (size_t)K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; + img_fmt = { CL_RGBA, CL_FLOAT }; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N / 4; + img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // pad N to multiple of 8 - int extra_elements = N % 8; - int padding = 0; - if (extra_elements > 0){ - padding = 8 - extra_elements; - } - - // subbuffer for transposed activations - region.origin = 0; - region.size = K * (N + padding) * sizeof(float)/2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + kernel = backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne01)); - // image for transposed activations - img_fmt = {CL_RGBA, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * (N + padding) / 4; - img_desc.buffer = b_sub_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + size_t local_work_size[3] = { 64, 8, 1 }; + size_t global_work_size[3] = { (size_t)ne01, 8, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - // transpose activations - int height_B = N/4; - if (height_B == 0) { - height_B = 1; + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } else { + static const char * q5_k_bin_dp4a_env = getenv("GGML_OPENCL_Q5_K_BIN_DP4A"); + bool q5_k_bin_dp4a_on = q5_k_bin_dp4a_env + ? (atoi(q5_k_bin_dp4a_env) != 0) + : true; + // dot prod has to be available + q5_k_bin_dp4a_on = backend_ctx->has_integer_dot && q5_k_bin_dp4a_on; + + if (q5_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin) { + const int dp4a_N_pad = CEIL_DIV(N, 32) * 32; + const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32); + + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_mem b_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_int tb = (cl_int)((size_t)N * (K / 32)); + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_mem d_sub = nullptr; + cl_mem d_img = nullptr; + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin; + + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_int), &N)); + + size_t local_work_size_dp4a[3] = { 64, 1, 1 }; + size_t global_work_size_dp4a[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size_dp4a, local_work_size_dp4a, dst); + + CL_CHECK(clReleaseMemObject(b_sub)); + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub)); + return; } - int width_B = K/4; - int padded_height_B = (N + padding)/4; - - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - - size_t local_work_size_t[2] = { 1, 16 }; - size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); - // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q4_1_f32; - int padded_N = N + padding; + const int gemm_tile_n = 64; + int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1)); + cl_mem b_sub_buf = nullptr; + cl_mem b_padded = nullptr; + cl_mem b_buf = nullptr; + if (N_pad == N) { + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + b_buf = b_sub_buf; + } else { + CL_CHECK((b_padded = clCreateBuffer(context, CL_MEM_READ_WRITE, (size_t)K * N_pad * sizeof(float), NULL, &err), err)); + const float zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_padded, &zero, sizeof(zero), 0, (size_t)K * N_pad * sizeof(float), 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, extra1->data_device, b_padded, offset1, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_padded; + } - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {1, 128, 1}; + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img; + CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + region.origin = offsetd; + region.size = (size_t)M * N * sizeof(float); + cl_mem d_sub_buf; + CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)M * N; + img_desc.buffer = d_sub_buf; + cl_mem d_img; + CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &d_img)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 2 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_work_size[3] = { 64, m_tiles, (size_t)CEIL_DIV(N_pad, gemm_tile_n) }; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); + if (b_sub_buf) { + CL_CHECK(clReleaseMemObject(b_sub_buf)); + } + if (b_padded) { + CL_CHECK(clReleaseMemObject(b_padded)); + } + CL_CHECK(clReleaseMemObject(d_img)); + CL_CHECK(clReleaseMemObject(d_sub_buf)); } -#else - GGML_UNUSED(backend); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(dst); -#endif } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS -static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -9849,150 +24336,168 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_Q8_0); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra; + ggml_tensor_extra_cl_q5_K * extra0_q5_k = (ggml_tensor_extra_cl_q5_K *)src0->extra; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - GGML_ASSERT(src1->view_offs == 0); - GGML_ASSERT(dst->view_offs == 0); - - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - - const int ne10 = src1->ne[0]; - const int ne12 = src1->ne[2]; - - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne1 = dst->ne[1]; - GGML_ASSERT(ne00 == ne10); - GGML_ASSERT((ne00 % 32) == 0); - GGML_ASSERT(ne0 == ne01); + GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); cl_context context = backend_ctx->context; cl_kernel kernel; - cl_int err; - cl_image_format img_fmt; - cl_image_desc img_desc; - cl_buffer_region region; + cl_int err; + cl_image_format img_fmt; + cl_image_desc img_desc; + cl_buffer_region region; int M = ne01; int N = ne1; int K = ne00; - if (ne1 == 1) { - cl_mem q_img = nullptr; + cl_uchar mask_d6 = 0x3F; + cl_uchar mask_d4 = 0x0F; + cl_uchar mask_hi2 = 0xC0; + + // Multi-column (N=3) verify GEMV for q5_K: route the spec/MTP verify batch + // (ne1==3) onto the efficient GEMV path instead of the transposed-GEMM dead- + // zone (gemm_noshuffle_q5_k, the #2 chunk of MTP decode on a Q4_0-mix model + // after q4_0 mc3). Reuses the ne1==1 GEMV image setup (q + qh + activations). + // Opt-in via GGML_OPENCL_Q5K_MC3=1. Per-layer only (ne01 < 32768). + static const bool q5k_mc3 = (getenv("GGML_OPENCL_Q5K_MC3") != nullptr); + const bool use_q5k_mc3 = q5k_mc3 && (ne1 >= 2 && ne1 <= 4) && (ne01 < 32768); + + const bool use_bin = use_q5_k_bin_kernels(backend_ctx, src0); + + if (use_bin) { + if (use_q5k_mc3) { + static bool warned = false; + if (!warned) { + GGML_LOG_WARN("ggml_opencl: GGML_OPENCL_Q5K_MC3 is bypassed by Q5_K binary kernels\n"); + warned = true; + } + } + ggml_cl_mul_mat_q5_K_f32_adreno_ila(backend, src0, src1, dst); + return; + } + + if (ne1 == 1 || use_q5k_mc3) { + cl_mem q_img = nullptr; + cl_mem qh_img = nullptr; cl_mem b_sub_buf = nullptr; cl_mem b_img = nullptr; - // image for q - img_fmt = { CL_R, CL_UNSIGNED_INT32}; + // image for q (CL_R, CL_UNSIGNED_INT32): width = M*K/2/4 + img_fmt = {CL_R, CL_UNSIGNED_INT32}; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 4; - img_desc.buffer = extra0_q8_0->q; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 2 / 4; + img_desc.buffer = extra0_q5_k->q; CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - // create a sub_buffer for B + // image for qh (CL_R, CL_HALF_FLOAT): width = M*K/16 + img_fmt = {CL_R, CL_HALF_FLOAT}; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = M * K / 16; + img_desc.buffer = extra0_q5_k->qh; + CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + // subbuffer for activations region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + region.size = K * N * sizeof(float); + CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); - // image for activations + // image for activations (CL_RGBA, CL_FLOAT): width = K*N/4 img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; + img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); - kernel = backend_ctx->CL_mul_mat_vec_q8_0_f32; - - int r2 = 1; - int r3 = 1; + kernel = use_q5k_mc3 ? backend_ctx->kernel_gemv_noshuffle_q5_k_f32_mc3 + : backend_ctx->kernel_gemv_noshuffle_q5_k_f32; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &extra1->offset)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &extrad->offset)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_hi2)); + if (use_q5k_mc3) { + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_int), &ne1)); // n_cols + } - size_t wavesize = backend_ctx->adreno_wave_size; - size_t local_work_size[] = { wavesize, 4, 1 }; - size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 }; + size_t local_work_size[3] = {64, 4, 1}; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(qh_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img)); } else { - cl_mem b_sub_buf = nullptr; + cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; - cl_mem b_img = nullptr; - cl_mem b_img_trans = nullptr; + cl_mem b_img = nullptr; + cl_mem b_img_trans = nullptr; // subbuffer for activations region.origin = offset1; - region.size = K * N * sizeof(float); + region.size = K * N * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for activations img_fmt = {CL_RGBA, CL_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; + img_desc.buffer = b_sub_buf; CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); // pad N to multiple of 8 int extra_elements = N % 8; int padding = 0; - if (extra_elements > 0){ + if (extra_elements > 0) { padding = 8 - extra_elements; } // subbuffer for transposed activations region.origin = 0; - region.size = K * (N + padding) * sizeof(float)/2; + region.size = K * (N + padding) * sizeof(float) / 2; backend_ctx->prealloc_act_trans.allocate(context, region.size); CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); // image for transposed activations img_fmt = {CL_RGBA, CL_HALF_FLOAT}; memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; img_desc.image_width = K * (N + padding) / 4; - img_desc.buffer = b_sub_buf_trans; + img_desc.buffer = b_sub_buf_trans; CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); // transpose activations - int height_B = N/4; - if (height_B == 0) { - height_B = 1; - } - int width_B = K/4; - int padded_height_B = (N + padding)/4; + int height_B = N / 4; + if (height_B == 0) height_B = 1; + int width_B = K / 4; + int padded_height_B = (N + padding) / 4; kernel = backend_ctx->kernel_transpose_32_16; CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); @@ -10001,33 +24506,98 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - size_t local_work_size_t[2] = { 1, 16 }; - size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; + size_t local_work_size_t[2] = {1, 16}; + size_t global_work_size_t[2] = {(size_t)width_B, (size_t)padded_height_B}; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + // dp4a (int8) dense q5_K prefill GEMM + static const char * q5k_dense_dp4a_env = getenv("GGML_OPENCL_Q5K_DENSE_DP4A"); + bool q5k_dense_dp4a_on = q5k_dense_dp4a_env + ? (atoi(q5k_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // dot prod has to be available + q5k_dense_dp4a_on = backend_ctx->has_integer_dot && q5k_dense_dp4a_on; + + if (q5k_dense_dp4a_on && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { + const int Mm = ne01, Nn = ne1, Kk = ne00; + const size_t n_blocks = (size_t)Nn * (Kk / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)Nn * Kk * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + static const char * q5_k_act_vload4_env = getenv("GGML_OPENCL_Q5_K_DP4A_ACT_VLOAD4"); + const bool q5_k_act_vload4_on = q5_k_act_vload4_env == nullptr || atoi(q5_k_act_vload4_env) != 0; + cl_kernel dk = q5_k_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Mm)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Nn)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Kk)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(Mm / 64), (size_t)CEIL_DIV(Nn, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + return; + } + // gemm - kernel = backend_ctx->kernel_mul_mm_q8_0_f32_8x4; + kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32; int padded_N = N + padding; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &padded_N)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_hi2)); - size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 }; - size_t local_work_size[] = { 2, 128, 1 }; + size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; + size_t local_work_size[3] = {1, 128, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_img_trans)); } #else GGML_UNUSED(backend); @@ -10037,8 +24607,425 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t #endif } -static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +// Dequant a possibly-strided q4_0/q8_0 tensor to tight-packed f16. Returns a +// temp cl_mem the caller must release. SoA inputs are reconstructed into a +// temp AoS buffer reported via *extra_reconstruct (also caller-released). +// this is for quantized K cache without FA. +static cl_mem ggml_cl_mul_mat_dequant_quant_to_f16( + ggml_backend_opencl_context * backend_ctx, + const ggml_tensor * tensor, + cl_mem * extra_reconstruct /* out, may be NULL */ +) { + GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0); + + if (extra_reconstruct) { + *extra_reconstruct = NULL; + } + + cl_mem src_buf; + cl_ulong src_offset; + cl_ulong src_nb1; + cl_ulong src_nb2; + cl_ulong src_nb3; + + uintptr_t pool_key_buf = 0; + cl_ulong pool_key_off = (cl_ulong) tensor->view_offs; + + const bool is_soa = tensor->type == GGML_TYPE_Q4_0 + ? ggml_cl_is_q4_0_soa(tensor) + : ggml_cl_is_q8_0_soa(tensor); + + cl_mem aos = nullptr; + if (is_soa) { + // Reconstruct full parent AoS; view's own nb[] then index it correctly. + const ggml_tensor * parent = tensor->view_src ? tensor->view_src : tensor; + const ggml_tensor * soa_src = parent; + const size_t block_bytes = (size_t) ggml_type_size(tensor->type); + const size_t blck_size = (size_t) ggml_blck_size(tensor->type); + const size_t parent_row_blocks = (size_t) parent->ne[0] / blck_size; + const size_t parent_row_bytes = parent_row_blocks * block_bytes; + const size_t parent_nbytes = (size_t) ggml_nelements(parent) / blck_size * block_bytes; + + cl_int err; + aos = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, parent_nbytes, NULL, &err); + CL_CHECK(err); + + // large q4_0/q8_0 WEIGHTS are stored transposed and small weights + // (and the AoS KV-cache, handled in the else branch above) are not. + // choose a proper restore kernel based on this. + bool restored = false; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + const int p_ne00 = (int) parent->ne[0]; + const int p_ne01 = (int) parent->ne[1]; + if (tensor->type == GGML_TYPE_Q8_0 && enable_adreno_trans_weight(backend_ctx, parent)) { + auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; + pool_key_buf = (uintptr_t) extra->q; + cl_kernel kernel = backend_ctx->kernel_restore_block_q8_0_trans; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_int), &p_ne00)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_int), &p_ne01)); + size_t gws[] = { (size_t)(((p_ne01 + 63) / 64) * 64), 1, 1 }; + size_t lws[] = { 64, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); + restored = true; + } else if (tensor->type == GGML_TYPE_Q4_0 && + use_adreno_kernels(backend_ctx, parent) && + !use_adreno_moe_kernels(backend_ctx, parent)) { + auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; + pool_key_buf = (uintptr_t) extra->q; + const size_t size_q = (size_t) ggml_nelements(parent) / blck_size * (blck_size / 2); + const size_t size_d = (size_t) ggml_nelements(parent) / blck_size * sizeof(ggml_fp16_t); + cl_int err2 = CL_SUCCESS; + cl_mem buf_tq = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_q, NULL, &err2); CL_CHECK(err2); + cl_mem buf_td = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, size_d, NULL, &err2); CL_CHECK(err2); + transpose_2d_as_16b(backend_ctx, extra->q, buf_tq, size_q, p_ne01, p_ne00 / 4); + transpose_2d_as_16b(backend_ctx, extra->d, buf_td, size_d, p_ne01, p_ne00 / 32); + cl_uchar mask_0F = 0x0F, mask_F0 = 0xF0; + cl_kernel kernel = backend_ctx->kernel_restore_block_q4_0_noshuffle; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_tq)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_td)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_uchar), &mask_0F)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_F0)); + const size_t n_blk = parent_nbytes / block_bytes; + size_t gws[] = { n_blk, 1, 1 }; + size_t lws[] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws, lws, 0, NULL, NULL)); + + CL_CHECK(clReleaseMemObject(buf_tq)); + CL_CHECK(clReleaseMemObject(buf_td)); + restored = true; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + if (!restored) { + cl_kernel kernel; + if (tensor->type == GGML_TYPE_Q8_0) { + auto * extra = (ggml_tensor_extra_cl_q8_0 *) soa_src->extra; + kernel = backend_ctx->kernel_restore_block_q8_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + pool_key_buf = (uintptr_t) extra->q; + } else { + auto * extra = (ggml_tensor_extra_cl_q4_0 *) soa_src->extra; + kernel = backend_ctx->kernel_restore_block_q4_0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &aos)); + pool_key_buf = (uintptr_t) extra->q; + } + + const size_t n_blocks = parent_nbytes / block_bytes; + size_t gws_rec[] = { n_blocks, 1, 1 }; + size_t lws_rec[] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, kernel, 3, NULL, gws_rec, lws_rec, 0, NULL, NULL)); + } + + (void) parent_row_blocks; + (void) parent_row_bytes; + src_buf = aos; + src_offset = tensor->view_offs; + src_nb1 = tensor->nb[1]; + src_nb2 = tensor->nb[2]; + src_nb3 = tensor->nb[3]; + + if (extra_reconstruct) { + *extra_reconstruct = aos; + } + } else { + auto * extra = (ggml_tensor_extra_cl *) tensor->extra; + GGML_ASSERT(extra && extra->data_device); + src_buf = extra->data_device; + src_offset = extra->offset + tensor->view_offs; + src_nb1 = tensor->nb[1]; + src_nb2 = tensor->nb[2]; + src_nb3 = tensor->nb[3]; + pool_key_buf = (uintptr_t) extra->data_device; + pool_key_off = (cl_ulong) src_offset; + } + + const cl_int nblk0 = (cl_int) (tensor->ne[0] / ggml_blck_size(tensor->type)); + const cl_int ne1_ = (cl_int) tensor->ne[1]; + const cl_int ne2_ = (cl_int) tensor->ne[2]; + const cl_int ne3_ = (cl_int) tensor->ne[3]; + + const size_t out_bytes = (size_t) ggml_nelements(tensor) * sizeof(ggml_fp16_t); + + // reuse a pooled f16 buffer for this KV-cache view across decode steps instead of + // allocating new one per attention op + cl_mem out = nullptr; + { + auto & pool = backend_ctx->dequant_f16_pool; + ggml_backend_opencl_context::ImagePoolKey key{pool_key_buf, (uint64_t) pool_key_off}; + auto it = pool.find(key); + if (it != pool.end() && it->second.k_bytes >= out_bytes && it->second.image) { + out = it->second.image; + } else { + if (it != pool.end()) { + if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } + pool.erase(it); + } + cl_int err = CL_SUCCESS; + out = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE, out_bytes, NULL, &err); + CL_CHECK(err); + ggml_backend_opencl_context::ImagePoolEntry entry; + entry.image = out; + entry.k_bytes = out_bytes; + pool[key] = entry; + } + } + + cl_kernel dq_kernel = tensor->type == GGML_TYPE_Q8_0 + ? backend_ctx->kernel_dequant_q8_0_f16_view_aos + : backend_ctx->kernel_dequant_q4_0_f16_view_aos; + + CL_CHECK(clSetKernelArg(dq_kernel, 0, sizeof(cl_mem), &src_buf)); + CL_CHECK(clSetKernelArg(dq_kernel, 1, sizeof(cl_ulong), &src_offset)); + CL_CHECK(clSetKernelArg(dq_kernel, 2, sizeof(cl_ulong), &src_nb1)); + CL_CHECK(clSetKernelArg(dq_kernel, 3, sizeof(cl_ulong), &src_nb2)); + CL_CHECK(clSetKernelArg(dq_kernel, 4, sizeof(cl_ulong), &src_nb3)); + CL_CHECK(clSetKernelArg(dq_kernel, 5, sizeof(cl_int), &nblk0)); + CL_CHECK(clSetKernelArg(dq_kernel, 6, sizeof(cl_int), &ne1_)); + CL_CHECK(clSetKernelArg(dq_kernel, 7, sizeof(cl_int), &ne2_)); + CL_CHECK(clSetKernelArg(dq_kernel, 8, sizeof(cl_int), &ne3_)); + CL_CHECK(clSetKernelArg(dq_kernel, 9, sizeof(cl_mem), &out)); + + size_t gws[3] = { (size_t) nblk0, (size_t) ne1_, (size_t) (ne2_ * ne3_) }; + size_t lws[3] = { 1, 1, 1 }; + CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, dq_kernel, 3, NULL, gws, lws, 0, NULL, NULL)); + + // release the reconstructed aos if + // 1. it was actually reconstructed + // 2. the caller didn't request it to be returned + // src_buf may refer to aos, so we should release after this enqueue + if (aos && !extra_reconstruct) { + CL_CHECK(clReleaseMemObject(aos)); + } + return out; +} + +// look up or create a pooled image1d_buffer over a KV-cache view. +static cl_mem ggml_cl_img_pool_get_or_create( + ggml_backend_opencl_context * backend_ctx, + std::map & pool, + cl_mem data_device, + cl_ulong offset0, + size_t required_bytes, + cl_channel_type channel_data_type +) { + ggml_backend_opencl_context::ImagePoolKey key{(uintptr_t)data_device, (uint64_t)offset0}; + auto it = pool.find(key); + if (it != pool.end() + && it->second.k_bytes >= required_bytes + && it->second.channel_data_type == channel_data_type + && it->second.image != nullptr) { + return it->second.image; + } + + // need to create or recreate and release any stale entry first. + if (it != pool.end()) { + if (it->second.image) { CL_CHECK(clReleaseMemObject(it->second.image)); } + if (it->second.sub_buffer) {CL_CHECK(clReleaseMemObject(it->second.sub_buffer)); } + pool.erase(it); + } + + cl_int status = CL_SUCCESS; + cl_buffer_region region = {}; + region.origin = (size_t)offset0; + region.size = required_bytes; + cl_mem sub = clCreateSubBuffer(data_device, 0, + CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + if (status != CL_SUCCESS) { + return nullptr; + } + + const size_t pixel_size = (channel_data_type == CL_HALF_FLOAT) ? 8 : 16; + cl_image_format fmt = {CL_RGBA, channel_data_type}; + cl_image_desc desc = {}; + desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + desc.image_width = required_bytes / pixel_size; + desc.buffer = sub; + cl_mem img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, + &fmt, &desc, NULL, &status); + if (status != CL_SUCCESS) { + CL_CHECK(clReleaseMemObject(sub)); + return nullptr; + } + + ggml_backend_opencl_context::ImagePoolEntry entry; + entry.sub_buffer = sub; + entry.image = img; + entry.k_bytes = required_bytes; + entry.channel_data_type = channel_data_type; + pool[key] = entry; + return img; +} + #ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_mul_mat_q2_K_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 16 ? 8 : 32; + cl_kernel gemm = n <= 16 ? backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_Q2_K_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->dm)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + +static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q2_k_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->dm)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_d)); + const size_t rows = 64*(size_t)backend_ctx->q2_k_mv_rows; + const size_t nsg = backend_ctx->q2_k_mv_nsg; + size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_q2_K_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_q2_k_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} +#endif + +static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); GGML_ASSERT(src1); @@ -10046,2250 +25033,3772 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t GGML_ASSERT(dst); GGML_ASSERT(dst->extra); + // bf16 is stored as f16 on device + const enum ggml_type src0t = (src0->type == GGML_TYPE_BF16) ? GGML_TYPE_F16 : src0->type; + const enum ggml_type src1t = src1->type; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // quant kv without FA + // used for non-contiguous src0 (the usual head-major permuted K view when n_head_kv>1) + // AND for the contiguous case that occurs when n_head_kv==1 (e.g. Gemma-4 E2B) + // Q4_0 bin kernels use a special weight layout, which the restore below does not support. + bool q4_0_bin_layout = false; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + q4_0_bin_layout = src0t == GGML_TYPE_Q4_0 && src0->view_src == nullptr && + ggml_is_contiguous(src0) && + src0->ne[2] == 1 && src0->ne[3] == 1 && + use_adreno_kernels(backend_ctx, src0) && + !use_adreno_moe_kernels(backend_ctx, src0) && + use_q4_0_bin_kernels(backend_ctx, src0); +#endif + if ((src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q8_0) && !q4_0_bin_layout && + (!ggml_is_contiguous(src0) || src1->ne[2] > src0->ne[2])) { + cl_mem f16_buf = ggml_cl_mul_mat_dequant_quant_to_f16(backend_ctx, src0, nullptr); + + ggml_tensor fake_src0 = *src0; + ggml_tensor_extra_cl fake_extra = {}; + fake_extra.data_device = f16_buf; + fake_extra.offset = 0; + fake_src0.type = GGML_TYPE_F16; + fake_src0.extra = &fake_extra; + fake_src0.view_src = nullptr; + fake_src0.view_offs = 0; + fake_src0.nb[0] = sizeof(ggml_fp16_t); + fake_src0.nb[1] = fake_src0.nb[0] * src0->ne[0]; + fake_src0.nb[2] = fake_src0.nb[1] * src0->ne[1]; + fake_src0.nb[3] = fake_src0.nb[2] * src0->ne[2]; + + ggml_cl_mul_mat(backend, &fake_src0, src1, dst); + return; + } + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q4_K * extra0_q4_k = (ggml_tensor_extra_cl_q4_K *)src0->extra; + cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; cl_ulong offsetd = extrad->offset + dst->view_offs; - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; +#ifdef GGML_OPENCL_SOA_Q + // view->extra stays pre-SoA; cast to the SoA struct would SIGSEGV. + // Follow view_src to reach the real SoA extra. + const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; + ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)soa0_src->extra; + ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; + ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; + ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra; + ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; + ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; +#endif - const int ne1 = dst->ne[1]; + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + GGML_TENSOR_LOCALS(int, ne1, src1, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb1, src1, nb); + GGML_TENSOR_LOCALS(int, ne, dst, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb, dst, nb); - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + int r2 = ne12/ne02; + int r3 = ne13/ne03; + + GGML_ASSERT(ne00 == ne10); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // adreno GEMM/GEMV kernels do not support broadcast, assuming ne2 and ne3 are 1 for src1 + // so we handle broadcast here + if ((ne12 > 1 || ne13 > 1) && ne02 == 1 && ne03 == 1 && + src0t != GGML_TYPE_F16 && src0t != GGML_TYPE_F32) { + for (int i13 = 0; i13 < ne13; ++i13) { + for (int i12 = 0; i12 < ne12; ++i12) { + ggml_tensor s1 = *src1; + s1.ne[2] = 1; s1.ne[3] = 1; + s1.view_offs = src1->view_offs + (size_t)i12*nb12 + (size_t)i13*nb13; + ggml_tensor d = *dst; + d.ne[2] = 1; d.ne[3] = 1; + d.view_offs = dst->view_offs + (size_t)i12*nb2 + (size_t)i13*nb3; + d.src[1] = &s1; + ggml_cl_mul_mat(backend, src0, &s1, &d); + } + } + return; + } +#endif + + int nth0 = 32; + int nth1 = 1; + int nrows = 1; + // The number of values produced by each subgroup + int ndst = 4; - cl_context context = backend_ctx->context; cl_kernel kernel; - cl_int err; - cl_image_format img_fmt; - cl_image_desc img_desc; - cl_buffer_region region; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32){ + // Two tiling assumptions these kernels make but nothing enforced: + // + // ne00 % TILESIZE_K(16): the K loop has no tail, so a K that does not + // divide folds 1-15 rows of whatever follows the operands into every + // output. + // + // ne01 % TILESIZE_M(64): mm_store_c_N guards the n direction with its + // `mask` argument but nothing guards m -- the store walks all 64 rows + // of the tile at a stride of M. When M does not divide, the last tile + // does not run off the end of the buffer, it writes 64 - (M % 64) + // values ON TOP OF the next column, so the result is silently wrong. + // Reachable on the KQV side for any head size >= 64 that is not a + // multiple of it (80, 96, 112). + // + // Attention shapes in the graph satisfy both -- head sizes are multiples + // of 64 and n_kv is padded -- which is why this has stayed latent. + // Declining leaves the odd shapes on the generic GEMM, which handles them. + if (ne01 >= 64 && ne1 >= 32 && ne00 >= 16 && + (ne00 % 16) == 0 && (ne01 % 64) == 0 && (ne12 % ne02) == 0 && + // the KQ/KQV image kernels do not handle dim 3 (multi-stream batches) + ne03 == 1 && ne13 == 1 && + // dst is wrapped with image1d_buffer, the size limit applies, also src0 + (ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4 <= backend_ctx->image_max_buffer_size)) { + // For KQ. + // + // Layout admission, mirroring the KQV arm below. The KQ kernel takes + // no stride arguments for A or B: it derives them as K*D_A*2 and + // K*D_B*4, i.e. it assumes both operands pack exactly D heads of K + // elements per row. Every real KV-cache view and permuted-Q view + // does, but a view spanning part of a wider allocation does not, and + // the kernel then walks the wrong rows with nothing to range-check + // it. Gate on the packed layout itself rather than on the stride + // ORDERING, which a wider parent satisfies just as well. + const bool kq_packed_a = (nb01 == (cl_ulong)ne00 * ne02 * ggml_type_size(src0t)) && + (nb02 == (cl_ulong)ne00 * ggml_type_size(src0t)); + const bool kq_packed_b = (nb11 == (cl_ulong)ne10 * ne12 * ggml_type_size(src1t)) && + (nb12 == (cl_ulong)ne10 * ggml_type_size(src1t)); + // + // ggml_is_permuted(src0) stands in for "K is head-major", but it is + // only a proxy and it COLLAPSES at n_head_kv == 1: with a single + // head there is no head stride to be out of order, so nb01 == nb02 + // and the view reports itself unpermuted. Such a KQ was declined + // here and fell through to the generic GEMM (gemma-4 E2B, and any + // other multi-query model). The packed check above is the contract + // the kernel actually needs -- it pins both strides exactly -- so + // require permutedness only where there is more than one head for + // it to mean anything. + // + // Default on; GGML_OPENCL_KQ_NHEAD_KV1=0 restores the old proxy so + // the two routings can be compared in one binary. + static const char * kq_nhkv1_env = getenv("GGML_OPENCL_KQ_NHEAD_KV1"); + static const bool kq_nhkv1_on = + (kq_nhkv1_env == nullptr || kq_nhkv1_env[0] != '0'); + if ((ggml_is_permuted(src0) || (ne02 == 1 && kq_nhkv1_on)) && ggml_is_permuted(src1) && + kq_packed_a && kq_packed_b && + ((nb01 * ne01 / 4)/4 <= backend_ctx->image_max_buffer_size) && + nb00 <= nb02 && + nb02 <= nb01 && + nb01 <= nb03 && + nb10 <= nb12 && + nb12 <= nb11 && + nb11 <= nb13) { + ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst, /*is_kq =*/ true); + return; + } + // For KQV. Reaching this arm is what makes the op a KQV; the callee + // is told so explicitly rather than re-deriving it from the strides + // the arm above has already ruled on. + if (!ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && + ((nb02 * ne02 / 4)/4 <= backend_ctx->image_max_buffer_size)) { + ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst, /*is_kq =*/ false); + return; + } + } + + static const char * mm_kq_gqa_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_IMG"); + static const bool mm_kq_gqa_img_on = (mm_kq_gqa_img_env == nullptr || mm_kq_gqa_img_env[0] != '0'); + static const char * mm_kq_gqa_r4_img_env = getenv("GGML_OPENCL_MM_KQ_GQA_R4_IMG"); + static const bool mm_kq_gqa_r4_img_on = (mm_kq_gqa_r4_img_env == nullptr || mm_kq_gqa_r4_img_env[0] != '0'); + const bool img_r4_gate = + mm_kq_gqa_r4_img_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img != nullptr && + ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 4 && (ne13 / ne03) == 1; + if (mm_kq_gqa_img_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img != nullptr && + ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 128 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + + const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; + const size_t k_pixels = k_bytes >> 4; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4_img; + cl_mem K_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + extra0->data_device, offset0, k_bytes, CL_FLOAT); + if (K_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } + } + + // r2=4 specialization + if (img_r4_gate) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; + const size_t k_pixels = k_bytes >> 4; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img; + cl_mem K_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + extra0->data_device, offset0, k_bytes, CL_FLOAT); + if (K_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } + } + + // DK=256, r2=2 specialization + static const char * mm_kq_r2_dk256_env = getenv("GGML_OPENCL_MM_KQ_GQA_R2_DK256_IMG"); + static const bool mm_kq_r2_dk256_on = (mm_kq_r2_dk256_env != nullptr && mm_kq_r2_dk256_env[0] != '0'); + if (mm_kq_r2_dk256_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img != nullptr && + ne11 == 1 && ne01 >= 64 && (ne01 % 16) == 0 && ne00 == 256 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 2 && (ne13 / ne03) == 1) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t k_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + const size_t k_bytes = (k_bytes_span + 15) & ~(size_t)15; + const size_t k_pixels = k_bytes >> 4; + if (k_pixels > 0 && k_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img; + cl_mem K_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kq_img_pool, + extra0->data_device, offset0, k_bytes, CL_FLOAT); + if (K_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &K_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } + } + + // GQA-coalesced KQV for decode using image1d_buffer_t + static const char * mm_kqv_gqa_img_env = getenv("GGML_OPENCL_MM_KQV_GQA_IMG"); + static const bool mm_kqv_gqa_img_on = (mm_kqv_gqa_img_env != nullptr && mm_kqv_gqa_img_env[0] != '0'); + if (mm_kqv_gqa_img_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img != nullptr && + ne11 == 1 && ne01 == 128 && + (ne12 % ne02) == 0 && (ne12 / ne02) == 8 && (ne13 / ne03) == 1) { + const size_t nb00_bytes = sizeof(uint16_t); + const size_t v_bytes_span = + (size_t)(ne01 > 0 ? ne01 - 1 : 0) * (size_t)nb01 + + (size_t)(ne02 > 0 ? ne02 - 1 : 0) * (size_t)nb02 + + (size_t)(ne03 > 0 ? ne03 - 1 : 0) * (size_t)nb03 + + (size_t)ne00 * nb00_bytes; + const size_t v_bytes = (v_bytes_span + 7) & ~(size_t)7; + const size_t v_pixels = v_bytes >> 3; + if (v_pixels > 0 && v_pixels <= backend_ctx->image_max_buffer_size) { + cl_kernel kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa_img; + cl_mem V_img = ggml_cl_img_pool_get_or_create( + backend_ctx, backend_ctx->kqv_img_pool, + extra0->data_device, offset0, v_bytes, CL_HALF_FLOAT); + if (V_img != nullptr) { + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &V_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); + + const int nth0_d = 64; + const int64_t n_wg_x = ne01 / 8; + size_t global_work_size[] = {(size_t)n_wg_x * nth0_d, (size_t)1, (size_t)ne02 * ne13}; + size_t local_work_size[] = {(size_t)nth0_d, (size_t)1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + } + } + } + + if (ne01 && ne1 && use_adreno_kernels(backend_ctx, src0)) { + // NOTE: Kernels using image1d_buffer_t (e.g., src0_q) would normally require + // a limit check, but q4_0 / q4_1 tensors are very unlikely to exceed that + // limit, so the check is omitted. + + // q1_0 x fp32 + if (src0t == GGML_TYPE_Q1_0 && src1t == GGML_TYPE_F32 && + enable_adreno_trans_weight(backend_ctx, src0)) { + ggml_cl_mul_mat_q1_0_f32_adreno(backend, src0, src1, dst); + return; + } + + // q4_0 x fp32 + if(src0t == GGML_TYPE_Q4_0 && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_q4_0_f32_adreno(backend, src0, src1, dst); + return; + } + + // q4_1 x fp32 + if (src0t == GGML_TYPE_Q4_1 && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_q4_1_f32_adreno(backend, src0, src1, dst); + return; + } + + // q5_0 x fp32 + if (src0t == GGML_TYPE_Q5_0 && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_q5_0_f32_adreno(backend, src0, src1, dst); + return; + } + + // q5_1 x fp32 + if (src0t == GGML_TYPE_Q5_1 && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_q5_1_f32_adreno(backend, src0, src1, dst); + return; + } + + // iq4_nl x fp32 + if (src0t == GGML_TYPE_IQ4_NL && src1t == GGML_TYPE_F32) { + ggml_cl_mul_mat_iq4_nl_f32_adreno(backend, src0, src1, dst); + return; + } + + // q8_0 x fp32 + if (src0t == GGML_TYPE_Q8_0 && src1t == GGML_TYPE_F32 && + enable_adreno_trans_weight(backend_ctx, src0)) { + ggml_cl_mul_mat_q8_0_f32_adreno(backend, src0, src1, dst); + return; + } + + // q2_k x fp32 + if (src0t == GGML_TYPE_Q2_K && src1t == GGML_TYPE_F32 && extra0_q2_K->noshuffle) { + ggml_cl_mul_mat_q2_k_f32_adreno(backend, src0, src1, dst); + return; + } + + // q4_k x fp32 + if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) { + ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst); + return; + } + + // q6_K x fp32 + if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q6_K(backend_ctx, src0)) { + ggml_cl_mul_mat_q6_K_f32_adreno(backend, src0, src1, dst); + return; + } + + // q5_K x fp32 + if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32 && + enable_adreno_trans_weight_q5_K(backend_ctx, src0)) { + ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst); + return; + } + } // if (ne01 && ne1) +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + + // GEMM using local memory + // Current BK = 16, so ne00 % 16 == 0 + // + // Certain A7X compiler (E031.41) executes kernel_mul_mm_f32_f32_l4_lm poorly; + // matrices with ne11 <= 8 appears OK. + // Fallback to the MV style kernels for A7x and ne11 > 8. + // Override with GGML_OPENCL_A7X_F32_LM_BYPASS=0. + static const char * a7x_f32lm_env = getenv("GGML_OPENCL_A7X_F32_LM_BYPASS"); + static const bool a7x_f32lm_bypass = (a7x_f32lm_env == nullptr || a7x_f32lm_env[0] != '0'); + if (src1t == GGML_TYPE_F32 && + ne00 % 16 == 0 && + ne11 > 1 && + !(a7x_f32lm_bypass && src0t == GGML_TYPE_F32 && ne11 > 8 && + backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) { + switch(src0t) { + case GGML_TYPE_F32: { + // Small-N f32 GEMV for the spec/MTP verify batch: the tiled GEMM + // below always computes a full 64x64 tile, so at ne11=3 with a + // skinny f32 weight (GDN ssm_alpha/ssm_beta, M=32) it launches one + // under-occupied WG at ~2.3% tile utilization. Route to a per-output + // (m,n) GEMV (64-thread WG, K-split + __local reduce) instead. + // Opt-in GGML_OPENCL_F32_MC=1; 2D contiguous, small N + skinny M only. + static const bool f32_mc = (getenv("GGML_OPENCL_F32_MC") != nullptr); + if (f32_mc && ne11 >= 2 && ne11 <= 8 && ne01 <= 512 && (ne00 % 4 == 0) && + ne02 == 1 && ne12 == 1 && ne13 == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { + cl_kernel kmc = backend_ctx->kernel_gemv_f32_f32_mc; + int stride_a = ne00, stride_b = ne00, stride_d = ne01; + CL_CHECK(clSetKernelArg(kmc, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kmc, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kmc, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kmc, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kmc, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kmc, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kmc, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kmc, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kmc, 8, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kmc, 9, sizeof(int), &stride_a)); + CL_CHECK(clSetKernelArg(kmc, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(kmc, 11, sizeof(int), &stride_d)); + size_t gws[3] = {64, (size_t)ne01 * (size_t)ne11, 1}; + size_t lws[3] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kmc, 3, gws, lws, dst); + return; + } + kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + cl_mem mem_src0 = extra0->data_device; + cl_mem mem_src1 = extra1->data_device; + + cl_ulong nb00_cont = nb00; + cl_ulong nb01_cont = nb01; + cl_ulong nb02_cont = nb02; + cl_ulong nb03_cont = nb03; - int M = ne01; - int N = ne1; - int K = ne00; + cl_ulong nb10_cont = nb10; + cl_ulong nb11_cont = nb11; + cl_ulong nb12_cont = nb12; + cl_ulong nb13_cont = nb13; - cl_uchar mask_d6 = 0x3F; - cl_uchar mask_d4 = 0x0F; - cl_uchar mask_hi2 = 0xC0; + cl_ulong offset0_cont = offset0; + cl_ulong offset1_cont = offset1; - if (ne1 == 1) { - cl_mem q_img = nullptr; - cl_mem b_sub_buf = nullptr; - cl_mem b_img = nullptr; + if (!ggml_is_contiguous(src0)) { + backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); + ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, + nb00_cont, nb01_cont, nb02_cont, nb03_cont); + mem_src0 = backend_ctx->prealloc_src0.buffer; + offset0_cont = 0; + } - // image for q - img_fmt = { CL_R, CL_UNSIGNED_INT32}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q4_k->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + if (!ggml_is_contiguous(src1)) { + backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); + ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, + nb10_cont, nb11_cont, nb12_cont, nb13_cont); + mem_src1 = backend_ctx->prealloc_src1.buffer; + offset1_cont = 0; + } - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - kernel = backend_ctx->kernel_gemv_noshuffle_q4_k_f32; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_F16: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (ggml_cl_can_use_adreno_xmem_gemm_f16_f32(backend_ctx, src0, src1, dst)) { + ggml_cl_mul_mat_f16_f32_adreno_xmem(backend, src0, src1, dst); + return; + } +#endif + static const char * mm_narrow_n_env = getenv("GGML_OPENCL_MM_NARROW_N"); + const bool use_narrow_n = (mm_narrow_n_env == nullptr || mm_narrow_n_env[0] != '0') && ne11 >= 2 && ne11 <= 8; + kernel = use_narrow_n ? backend_ctx->kernel_mul_mm_f16_f32_l4_lm_n8 : backend_ctx->kernel_mul_mm_f16_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->s)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_hi2)); + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + cl_mem mem_src0 = extra0->data_device; + cl_mem mem_src1 = extra1->data_device; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + cl_ulong nb00_cont = nb00; + cl_ulong nb01_cont = nb01; + cl_ulong nb02_cont = nb02; + cl_ulong nb03_cont = nb03; - CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { + cl_ulong nb10_cont = nb10; + cl_ulong nb11_cont = nb11; + cl_ulong nb12_cont = nb12; + cl_ulong nb13_cont = nb13; - cl_mem b_sub_buf = nullptr; - cl_mem b_sub_buf_trans = nullptr; - cl_mem b_img = nullptr; - cl_mem b_img_trans = nullptr; + cl_ulong offset0_cont = offset0; + cl_ulong offset1_cont = offset1; - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + if (!ggml_is_contiguous(src0)) { + backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); + ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, + nb00_cont, nb01_cont, nb02_cont, nb03_cont); + mem_src0 = backend_ctx->prealloc_src0.buffer; + offset0_cont = 0; + } - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + if (!ggml_is_contiguous(src1)) { + backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); + ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, + nb10_cont, nb11_cont, nb12_cont, nb13_cont); + mem_src1 = backend_ctx->prealloc_src1.buffer; + offset1_cont = 0; + } - // pad N to multiple of 8 - int extra_elements = N % 8; - int padding = 0; - if (extra_elements > 0){ - padding = 8 - extra_elements; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - // subbuffer for transposed activations - region.origin = 0; - region.size = K * (N + padding) * sizeof(float)/2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + // BN=8/TN=1 keeps 128 threads for small batches, including attention KQ/KQV. + const int bn = use_narrow_n ? 8 : 64; + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, bn)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - // image for transposed activations - img_fmt = {CL_RGBA, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * (N + padding) / 4; - img_desc.buffer = b_sub_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q1_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - // transpose activations - int height_B = N/4; - if (height_B == 0) { - height_B = 1; - } - int width_B = K/4; - int padded_height_B = (N + padding)/4; + kernel = backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - size_t local_work_size_t[2] = { 1, 16 }; - size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32; - int padded_N = N + padding; + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_k->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_k->s)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_k->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_hi2)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q4_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {1, 128, 1}; + kernel = backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); - } -#else - GGML_UNUSED(backend); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(dst); -#endif -} + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; -static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q4_1: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + kernel = backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - const int ne1 = dst->ne[1]; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - cl_context context = backend_ctx->context; - cl_kernel kernel; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q5_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - cl_int err; - cl_buffer_region region; - cl_image_format img_fmt; - cl_image_desc img_desc; + kernel = backend_ctx->kernel_mul_mm_q5_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - // subbuffer and image for activation - if (ne1 == 1) { - cl_mem ql_img = nullptr; - cl_mem qh_img = nullptr; - cl_mem b_sub_buffer = nullptr; - cl_mem b_img = nullptr; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - // image for ql - img_fmt.image_channel_order = CL_R; - img_fmt.image_channel_data_type = CL_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne01 * ne00 / 8; - img_desc.buffer = extra0_q6_K->ql; - CL_CHECK((ql_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); - // image for qh - img_fmt.image_channel_order = CL_R; - img_fmt.image_channel_data_type = CL_HALF_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne01 * ne00 / 8; - img_desc.buffer = extra0_q6_K->qh; - CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - region.origin = offset1; - region.size = ne00 * ne1 * sizeof(float); - CL_CHECK((b_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q5_1: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - img_fmt.image_channel_order = CL_RGBA; - img_fmt.image_channel_data_type = CL_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne00 * ne1 / 4; - img_desc.buffer = b_sub_buffer; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + kernel = backend_ctx->kernel_mul_mm_q5_1_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - kernel = backend_ctx->kernel_gemv_noshuffle_q6_K_f32; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &ql_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q8_0: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - CL_CHECK(clReleaseMemObject(ql_img)); - CL_CHECK(clReleaseMemObject(qh_img)); - CL_CHECK(clReleaseMemObject(b_sub_buffer)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { - cl_mem b_sub_buf; - cl_mem b_buf_trans; - cl_mem b_img; - cl_mem b_img_trans; + kernel = backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - // subbuffer for activation - region.origin = offset1; - region.size = ne00 * ne1 * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - // image for activation - img_fmt.image_channel_order = CL_RGBA; - img_fmt.image_channel_data_type = CL_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne00 * ne1 / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - // pad N to multiple of 8 - int extra_elements = ne1 % 8; - int padding = 0; - if (extra_elements > 0){ - padding = 8 - extra_elements; - } + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - // subbuffer for transposed activation - region.origin = 0; - region.size = ne00 * (ne1 + padding) * sizeof(float)/2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - CL_CHECK((b_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_IQ4_NL: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - // image for transposed activation - img_fmt.image_channel_order = CL_RGBA; - img_fmt.image_channel_data_type = CL_HALF_FLOAT; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = ne00 * (ne1 + padding) / 4; - img_desc.buffer = b_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + kernel = backend_ctx->kernel_mul_mm_iq4_nl_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - // transpose activation - int height_B = ne1/4; - if (height_B == 0) { - height_B = 1; - } - int width_B = ne00/4; - int padded_height_B = (ne1 + padding) / 4; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); - size_t local_size_t[2] = { 1, 16 }; - size_t global_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst); + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q6_K_f32; - int padded_N = ne1 + padding; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q2_K: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - cl_ushort mask_f000 = 0xF000; - cl_uchar mask_c0 = 0xC0; + kernel = backend_ctx->kernel_mul_mm_q2_k_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ushort),&mask_f000)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_c0)); + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {2, 128, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q2_K->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_buf_trans)); - CL_CHECK(clReleaseMemObject(b_img_trans)); - } -#else - GGML_UNUSED(backend); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(dst); -#endif -} + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; -static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q4_K: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + kernel = backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm; + // (BM*BN)/(TM*TN): Intel uses an 8x8 microtile (WG=64), others 4x8 (WG=128) + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - ggml_tensor_extra_cl_q5_K * extra0_q5_k = (ggml_tensor_extra_cl_q5_K *)src0->extra; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_K->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_K->s)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_K->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_K->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne1 = dst->ne[1]; + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q5_K: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } - cl_context context = backend_ctx->context; - cl_kernel kernel; + kernel = backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; // Intel 8x8 microtile - cl_int err; - cl_image_format img_fmt; - cl_image_desc img_desc; - cl_buffer_region region; + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - int M = ne01; - int N = ne1; - int K = ne00; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_K->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_K->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); - cl_uchar mask_d6 = 0x3F; - cl_uchar mask_d4 = 0x0F; - cl_uchar mask_hi2 = 0xC0; + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - if (ne1 == 1) { - cl_mem q_img = nullptr; - cl_mem qh_img = nullptr; - cl_mem b_sub_buf = nullptr; - cl_mem b_img = nullptr; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + case GGML_TYPE_Q6_K: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm; + nth0 = 128; // calculated as (BM*BN)/(TM*TN) - // image for q (CL_R, CL_UNSIGNED_INT32): width = M*K/2/4 - img_fmt = {CL_R, CL_UNSIGNED_INT32}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 2 / 4; - img_desc.buffer = extra0_q5_k->q; - CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; - // image for qh (CL_R, CL_HALF_FLOAT): width = M*K/16 - img_fmt = {CL_R, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = M * K / 16; - img_desc.buffer = extra0_q5_k->qh; - CL_CHECK((qh_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; - // image for activations (CL_RGBA, CL_FLOAT): width = K*N/4 - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } + default: + break; + } + } - kernel = backend_ctx->kernel_gemv_noshuffle_q5_k_f32; + if (src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32 && + src0->ne[1] > 32 && // M > 32 + src1->ne[1] > 32 && // N > 32 + src0->ne[0] > 32 && // K > 32 + src0->ne[2] == 1 && src0->ne[3] == 1 && + src1->ne[2] == 1 && src1->ne[3] == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && + backend_ctx->kernel_mul_mat_f16_f32_tiled != NULL) { + ggml_cl_mul_mat_f16_f32_tiled(backend, src0, src1, dst); + return; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &qh_img)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->s)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_hi2)); + if (!ggml_is_transposed(src0) && + !ggml_is_transposed(src1) && + src1t == GGML_TYPE_F32 && + ne00%32 == 0 && + ne11 > 2) { +#ifdef GGML_OPENCL_SOA_Q + // Set up kernel. + switch(src0t) { + case GGML_TYPE_Q4_0: + // This should have been satisfied. + GGML_ASSERT(ne11 == ne1); + GGML_ASSERT(ne01 == ne0); - size_t local_work_size[3] = {64, 4, 1}; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1}; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_16x_flat; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; - CL_CHECK(clReleaseMemObject(q_img)); - CL_CHECK(clReleaseMemObject(qh_img)); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_img)); - } else { - cl_mem b_sub_buf = nullptr; - cl_mem b_sub_buf_trans = nullptr; - cl_mem b_img = nullptr; - cl_mem b_img_trans = nullptr; + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_8x_flat; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // subbuffer for activations - region.origin = offset1; - region.size = K * N * sizeof(float); - CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); + break; + default: + break; + } - // image for activations - img_fmt = {CL_RGBA, CL_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * N / 4; - img_desc.buffer = b_sub_buf; - CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + // Launch kernel. + if (src0t == GGML_TYPE_Q4_0) { + size_t global_work_size[] = {(size_t)(ne01 + 7)/8*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - // pad N to multiple of 8 - int extra_elements = N % 8; - int padding = 0; - if (extra_elements > 0) { - padding = 8 - extra_elements; + if (backend_ctx->gpu_family == INTEL) { + // Set global size for Intel. It uses 16x output values. + global_work_size[0] = (size_t)(ne01 + 15)/16*nth0; + global_work_size[1] = (size_t)ne11*nth1; + global_work_size[2] = (size_t)ne12*ne13; + } + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; } +#else // GGML_OPENCL_SOA_Q + // TODO: add block_q4_0 variant. +#endif // GGML_OPENCL_SOA_Q + } - // subbuffer for transposed activations - region.origin = 0; - region.size = K * (N + padding) * sizeof(float) / 2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + // use custom matrix x vector kernel + bool use_f16_mrow = false; + switch (src0t) { + case GGML_TYPE_F32: + //GGML_ASSERT(ne02 == ne12); + GGML_ASSERT(src1t == GGML_TYPE_F32); + kernel = backend_ctx->kernel_mul_mat_f32_f32; + nrows = 4; - // image for transposed activations - img_fmt = {CL_RGBA, CL_HALF_FLOAT}; - memset(&img_desc, 0, sizeof(img_desc)); - img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc.image_width = K * (N + padding) / 4; - img_desc.buffer = b_sub_buf_trans; - CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err)); + if (backend_ctx->gpu_family == INTEL) { + nth0 = 32; + nth1 = 1; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // transpose activations - int height_B = N / 4; - if (height_B == 0) height_B = 1; - int width_B = K / 4; - int padded_height_B = (N + padding) / 4; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); + break; + case GGML_TYPE_F16: + //GGML_ASSERT(ne02 == ne12); + if (backend_ctx->gpu_family == INTEL) { + nth0 = 32; + nth1 = 1; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); + if (src1t == GGML_TYPE_F32) { + // heuristic for packing more work for Adreno + const bool adreno_use_lane_split = + backend_ctx->gpu_family == ADRENO && + ne11 == 1 && + ne01 >= 8 && + ne00 % 4 == 0 && + r3 == 1 && r2 >= 1 && r2 <= 8 && + (ne12 % r2) == 0; + + if (ne11 * ne12 < 4) { + // Decode (single token): the legacy _1row runs one 64-lane + // subgroup per WG (one output row), under-utilizing BW. Route the + // wide f16 weight matmuls (attn proj + lm_head) to the multi-row + // variant: MROW rows per WG -> more loads in flight + activation + // staged once in __local. ne00<=8192 bounds the LDS. The mrow WG + // is 64 x MROW = 1024 work-items (> Intel's 512 max) and reduces + // within a 64-wide subgroup, so skip on Intel. + if (backend_ctx->f16_mrow && backend_ctx->gpu_family != INTEL && + backend_ctx->kernel_mul_mat_f16_f32_mrow != nullptr && + ne00 >= 128 && ne01 >= 8 && ne00 % 4 == 0 && ne00 <= 8192) { + // The register-blocked / half8 variants cast the src0 row pointer to + // half4 / half8 (8- and 16-byte loads) with no scalar fallback inside + // the kernel. ne00 % 4 == 0 constrains the element count per row, NOT + // the byte stride between rows: a permuted or strided src0 (or a view + // at an odd offset) can leave nb01/nb02/nb03 unaligned. Only take them + // when every row this dispatch touches is aligned; the base mrow kernel + // re-checks per row and falls back to its scalar loop. + const cl_ulong row_addr_bits = offset0 | nb01 | nb02 | nb03; + const bool aligned8 = (row_addr_bits & 7) == 0; + const bool aligned16 = (row_addr_bits & 15) == 0; + + // Register-blocked variants: each subgroup does RPT rows (more + // weight loads in flight per lane). 8/16 use half8 (128-bit) + // loads, gated on ne00 % 8 == 0. + const int rpt = backend_ctx->f16_mrow_rpt; + if (rpt == 16 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2; + } else if (rpt == 8 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8; + } else if (rpt == 4 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r4; + } else if (rpt == 2 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r2; + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow; + } + use_f16_mrow = true; + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32_1row; + } + } else if (adreno_use_lane_split && ne00 >= 64 && ne00 <= 128) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq; + nrows = 1; + } else if (adreno_use_lane_split && r2 >= 2 && ne00 > 128 && ne00 <= 256) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls; + nrows = 1; + } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { + // multi-output decode variants when Q is a single row + static const char * mm_force_l4_env = getenv("GGML_OPENCL_MM_F16_FORCE_L4"); + static const bool mm_force_l4_on = (mm_force_l4_env != nullptr && mm_force_l4_env[0] != '0'); + const bool can_multi_out = !mm_force_l4_on && ne11 == 1 && ne01 >= 64 && ne01 % 8 == 0; + // paired-K-row variant that doubles per-wave-cycle + static const char * mm_kq_pair_env = getenv("GGML_OPENCL_MM_KQ_PAIR"); + static const bool mm_kq_pair_on = (mm_kq_pair_env != nullptr && mm_kq_pair_env[0] != '0'); + // GQA-coalesced variant that reads each K-row once and + // emits gqa_ratio outputs + static const char * mm_kq_gqa_env = getenv("GGML_OPENCL_MM_KQ_GQA"); + static const bool mm_kq_gqa_on = (mm_kq_gqa_env != nullptr && mm_kq_gqa_env[0] != '0'); + // GQA-coalesced KQV variant (DK=128/r2=8/r3=1) that reads + // each V slab once per K-head and emits all r2 Q-heads + static const char * mm_kqv_gqa_env = getenv("GGML_OPENCL_MM_KQV_GQA"); + static const bool mm_kqv_gqa_on = (mm_kqv_gqa_env != nullptr && mm_kqv_gqa_env[0] != '0'); + if (can_multi_out && (ne01 % 16) == 0 && ne00 == 128 && r2 == 8 && r3 == 1 && mm_kq_gqa_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4; + nrows = 1; + } else if (can_multi_out && ne00 <= 256 && mm_kq_pair_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair; + nrows = 1; + } else if (can_multi_out && ne00 <= 256 && + backend_ctx->kernel_mul_mat_f16_f32_l4_x8 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_x8; + nrows = 1; + } else if (can_multi_out && ne01 == 128 && r2 == 8 && r3 == 1 && mm_kqv_gqa_on && + backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa; + nrows = 1; + } else if (can_multi_out && + backend_ctx->kernel_mul_mat_f16_f32_l4_y8 != nullptr) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_y8; + nrows = 1; + } else if (ne11 == 1) { + // Decode shapes that don't satisfy the x8/y8 row + // constraints (ne01 < 64 or ne01 % 8 != 0) fall back to + // upstream's 4-output _dr kernel. + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr; + nrows = 1; // not used by this kernel + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4; + nrows = ne11; + } + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32; + nrows = 4; + } + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f16; + nrows = 4; + } - size_t local_work_size_t[2] = {1, 16}; - size_t global_work_size_t[2] = {(size_t)width_B, (size_t)padded_height_B}; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); + if (use_f16_mrow) { + const int MROW = 16; // must match MROW in mul_mv_f16_f32_mrow.cl + // rows-per-subgroup multiplier for the selected variant: + // 1/2/4 -> half4 register blocking; 8 -> half8(1 row); 16 -> half8(2 rows) + const int rpt = backend_ctx->f16_mrow_rpt; + int rmul; + if (rpt == 16) rmul = (ne00 % 8 == 0) ? 2 : 1; + else if (rpt == 8) rmul = 1; + else rmul = rpt; // 1,2,4 + const int rows_per_wg = MROW * rmul; + // __local activation buffer: ne00 floats, rounded up for float4 access + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float) * ((ne00 + 3) / 4 * 4), nullptr)); + size_t mrow_global[] = { (size_t)((ne01 + rows_per_wg - 1) / rows_per_wg) * 64, (size_t)ne11 * MROW, (size_t)ne12 * ne13 }; + size_t mrow_local[] = { 64, (size_t)MROW, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, mrow_global, mrow_local, dst); + return; + } + break; + case GGML_TYPE_Q1_0: { +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q1_0_f32_flat; - // gemm - kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32; - int padded_N = N + padding; + // nth0 - subgroup size + // nth1 - number of subgroups per workgroup + // ndst - number of output values per workgroup = output per subgroup * number of subgroups + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_k->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_k->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_k->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_k->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_k->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img_trans)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &padded_N)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_uchar), &mask_d6)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_uchar), &mask_d4)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_uchar), &mask_hi2)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q1_0_f32; - size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1}; - size_t local_work_size[3] = {1, 128, 1}; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_Q4_0: + // This should have been satisfied. + GGML_ASSERT(ne11 == ne1); + GGML_ASSERT(ne01 == ne0); - CL_CHECK(clReleaseMemObject(b_sub_buf)); - CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); - CL_CHECK(clReleaseMemObject(b_img)); - CL_CHECK(clReleaseMemObject(b_img_trans)); - } -#else - GGML_UNUSED(backend); - GGML_UNUSED(src0); - GGML_UNUSED(src1); - GGML_UNUSED(dst); -#endif -} +#ifdef GGML_OPENCL_SOA_Q + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; -static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; + ndst = 8; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; - const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT; - const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT; + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; + ndst =8; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#else // GGML_OPENCL_SOA_Q + if (backend_ctx->gpu_family == INTEL) { + // Use 1D local size. Each workgroup is a SIMD group. Each SIMD + // group produces N_DST (4 for Q4_0 kernel) values in the result. + // The number of workgroups on dim 0 (the leading dimension) is + // the nearest multiple of 4 that covers ne0 (equals ne01). + nth0 = 16; + nth1 = 1; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + kernel = backend_ctx->kernel_mul_mat_q4_0_f32; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + kernel = backend_ctx->kernel_mul_mat_q4_0_f32_v; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + case GGML_TYPE_Q4_1: { #ifdef GGML_OPENCL_SOA_Q - ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; - ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)src0->extra; - ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)src0->extra; - ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra; - ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)src0->extra; - ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)src0->extra; - ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)src0->extra; -#endif + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - const int ne00 = src0 ? src0->ne[0] : 0; - const int ne01 = src0 ? src0->ne[1] : 0; - const int ne02 = src0 ? src0->ne[2] : 0; - const int ne03 = src0 ? src0->ne[3] : 0; + kernel = backend_ctx->kernel_mul_mv_q4_1_f32_flat; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3)); +#else + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - const cl_ulong nb00 = src0 ? src0->nb[0] : 0; - const cl_ulong nb01 = src0 ? src0->nb[1] : 0; - const cl_ulong nb02 = src0 ? src0->nb[2] : 0; - const cl_ulong nb03 = src0 ? src0->nb[3] : 0; + kernel = backend_ctx->kernel_mul_mv_q4_1_f32; - const int ne10 = src1 ? src1->ne[0] : 0; - const int ne11 = src1 ? src1->ne[1] : 0; - const int ne12 = src1 ? src1->ne[2] : 0; - const int ne13 = src1 ? src1->ne[3] : 0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_Q5_0: { +#ifdef GGML_OPENCL_SOA_Q + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - const cl_ulong nb10 = src1 ? src1->nb[0] : 0; - const cl_ulong nb11 = src1 ? src1->nb[1] : 0; - const cl_ulong nb12 = src1 ? src1->nb[2] : 0; - const cl_ulong nb13 = src1 ? src1->nb[3] : 0; + kernel = backend_ctx->kernel_mul_mv_q5_0_f32_flat; - const int ne0 = dst ? dst->ne[0] : 0; - const int ne1 = dst ? dst->ne[1] : 0; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3)); +#else + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - int r2 = ne12/ne02; - int r3 = ne13/ne03; + kernel = backend_ctx->kernel_mul_mv_q5_0_f32; - GGML_ASSERT(ne00 == ne10); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_Q5_1: { +#ifdef GGML_OPENCL_SOA_Q + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - int nth0 = 32; - int nth1 = 1; - int nrows = 1; - // The number of values produced by each subgroup - int ndst = 4; + kernel = backend_ctx->kernel_mul_mv_q5_1_f32_flat; - cl_kernel kernel; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3)); +#else + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - cl_context context = backend_ctx->context; + kernel = backend_ctx->kernel_mul_mv_q5_1_f32; - if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32){ - if (ne01 >= 64 && ne1 >= 32 && ne00 >= 16 && (ne12 % ne02) == 0 && - // dst is wrapped with image1d_buffer, the size limit applies, also src0 - (ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4 <= backend_ctx->image_max_buffer_size)) { - // For KQ - if (ggml_is_permuted(src0) && ggml_is_permuted(src1) && - ((nb01 * ne01 / 4)/4 <= backend_ctx->image_max_buffer_size) && - nb00 <= nb02 && - nb02 <= nb01 && - nb01 <= nb03 && - nb10 <= nb12 && - nb12 <= nb11 && - nb11 <= nb13) { - ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst); - return; - } - // For KQV - if (!ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && - ((nb02 * ne02 / 4)/4 <= backend_ctx->image_max_buffer_size)) { - ggml_cl_mul_mat_kq_kqv_adreno(backend, src0, src1, dst); - return; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; } - } + case GGML_TYPE_Q8_0: { +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q8_0_f32_flat; - if (ne01 && ne1 && use_adreno_kernels(backend_ctx, src0)) { + // nth0 - subgroup size + // nth1 - number of subgroups per workgroup + // ndst - number of output values per workgroup = output per subgroup * number of subgroups + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // init CL objects - // <--------------------------------------------> // - cl_int status; - cl_image_format img_fmt_1d; - cl_image_desc img_desc_1d; - cl_buffer_region region; - cl_mem A_image1d = nullptr; - cl_mem B_image1d = nullptr; - cl_mem B_sub_buffer = nullptr; - cl_mem C_d = nullptr; - // for B transpose - cl_mem B_d = nullptr; - cl_mem B_d_input_image = nullptr; - // <--------------------------------------------> // + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q8_0_f32; - // define matrix dimensions - // <--------------------------------------------> // - int M = ne01; - int N = ne1; - int K = ne00; - int padding; - // <--------------------------------------------> // + // nth0 - subgroup size + // nth1 - number of subgroups per workgroup + // ndst - number of output values per workgroup = output per subgroup * number of subgroups + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // NOTE: Kernels using image1d_buffer_t (e.g., src0_q) would normally require - // a limit check, but q4_0 / q4_1 tensors are very unlikely to exceed that - // limit, so the check is omitted. + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_IQ4_NL: { +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq4_nl_f32_flat; - // q4_1 x fp32 - if (src0t == GGML_TYPE_Q4_1 && src1t == GGML_TYPE_F32) { - ggml_cl_mul_mat_q4_1_f32_adreno(backend, src0, src1, dst); - return; - } + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 8; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 8; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // q8_0 x fp32 - if (src0t == GGML_TYPE_Q8_0 && src1t == GGML_TYPE_F32 && - enable_adreno_trans_weight(backend_ctx, src0)) { - ggml_cl_mul_mat_q8_0_f32_adreno(backend, src0, src1, dst); - return; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq4_nl_f32; - // q4_k x fp32 - if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32) { - ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst); - return; - } + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // q6_K x fp32 - if (src0t == GGML_TYPE_Q6_K && src1t == GGML_TYPE_F32) { - ggml_cl_mul_mat_q6_K_f32_adreno(backend, src0, src1, dst); - return; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_Q2_K: { + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // q5_K x fp32 - if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32) { - ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst); - return; - } +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q2_K_f32_flat; + offset0 = src0->view_offs; - // q4_0 x fp32 - if(src0t == GGML_TYPE_Q4_0 && src1t == GGML_TYPE_F32) { - // TODO: remove duplicate definitions of image description + format -- move to top + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q2_K->dm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q2_K_f32; - // create an image for A - // <--------------------------------------------> // - if (N == 1) { - img_fmt_1d = { CL_R, CL_UNSIGNED_INT32}; - } else { - img_fmt_1d = { CL_R, CL_FLOAT}; - } - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.image_width = M * K / 2 / 4; // Divide by 4 for char -> float - img_desc_1d.buffer = extra0_q4_0->q; - A_image1d = clCreateImage( - context, - CL_MEM_READ_ONLY, - &img_fmt_1d, - &img_desc_1d, - NULL, - &status); - CL_CHECK(status); - // <--------------------------------------------> // + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: { +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q4_K_f32_flat; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 16; // 8->16 rows per subgroup — matches N_DST in mul_mv_q4_k_f32_flat.cl (32 spills) + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = 16; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // create a sub_buffer for B - // <--------------------------------------------> // - region.origin = (extra1->offset); - region.size = K * N * sizeof(float); - B_sub_buffer = clCreateSubBuffer( - extra1->data_device, - 0, - CL_BUFFER_CREATE_TYPE_REGION, - ®ion, - &status); - CL_CHECK(status); - // <--------------------------------------------> // - - // transpose activation for Skyler's gemm - if (N != 1) { - //how many extra elements beyond multiple of 8 - int extra_elements = N % 8; - - //how much padding to add - padding = 0; - if (extra_elements > 0){ - padding = 8 - extra_elements; - } - - // Specify the starting offset (in bytes) - region.origin = 0; - // Specify the size of the sub-buffer (divide by 2 for FP16) - region.size = K * (N + padding) * sizeof(float)/2; - backend_ctx->prealloc_act_trans.allocate(context, region.size); - - B_d = clCreateSubBuffer( - backend_ctx->prealloc_act_trans.buffer, - 0, - CL_BUFFER_CREATE_TYPE_REGION, - ®ion, - &status); - CL_CHECK(status); - - cl_image_format image_format_B_d_input = { CL_RGBA, CL_FLOAT }; - cl_image_desc image_desc_B_d_input = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast(K * N / 4), - 0, 0, 0, 0, 0, 0, 0, { B_sub_buffer } - }; - B_d_input_image = clCreateImage( - context, - 0, - &image_format_B_d_input, - &image_desc_B_d_input, - NULL, - &status); - CL_CHECK(status); - - cl_image_format image_format_B_d_output = { CL_RGBA, CL_HALF_FLOAT }; //(CL_HALF_FLOAT for FP16) - cl_image_desc image_desc_B_d_output = { - CL_MEM_OBJECT_IMAGE1D_BUFFER, - static_cast(K * (N + padding)/4), - 0, 0, 0, 0, 0, 0, 0, { B_d } - }; - B_image1d = clCreateImage( - context, - 0, - &image_format_B_d_output, - &image_desc_B_d_output, - NULL, - &status); - CL_CHECK(status); - - int height_B = N/4; - if (height_B == 0) { - height_B = 1; - } - int width_B = K/4; - int padded_height_B = (N + padding)/4; - - kernel = backend_ctx->kernel_transpose_32_16; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &B_d_input_image)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &B_image1d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B)); - - size_t local_size_t[2] = { 1, 16 }; - //WGS tuning - if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { - local_size_t[0]=4; - local_size_t[1]=8; - } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { - local_size_t[0]=2; - local_size_t[1]=8; - } else if(ne0 == 4096 && ne1 == 128 && ne10 == 11008) { - local_size_t[0]=1; - local_size_t[1]=8; - } else if(ne0 == 32000 && ne1 == 128 && ne10 == 4096) { - local_size_t[0]=2; - local_size_t[1]=8; - } - - size_t global_size_t[2] = { - static_cast(width_B), - static_cast(padded_height_B) - }; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_K->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_K->s)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_K->d)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_K->dm)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q4_K_f32; - backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_size_t, local_size_t, dst); - } else { - // no need to transpose B in other cases - // create an image for B from sub_buffer - // <--------------------------------------------> // - img_fmt_1d = {CL_RGBA, CL_FLOAT}; - - memset(&img_desc_1d, 0, sizeof(img_desc_1d)); - img_desc_1d.image_width = K * N / 4; - img_desc_1d.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; - img_desc_1d.buffer = B_sub_buffer; - B_image1d = clCreateImage( - context, - CL_MEM_READ_ONLY, - &img_fmt_1d, - &img_desc_1d, - NULL, - &status); - CL_CHECK(status); - // <--------------------------------------------> // - } - - // choose gemm or gemv kernel - // <--------------------------------------------> // - if (N == 1) { - kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_general; - if (M == 4096 && K == 4096) { - kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_4096; - } else if (M == 4096 && K == 11008) { - kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_4096_1_11008; - } else if (M == 11008 && K == 4096) { - kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_11008_1_4096; - } else if (M == 32000 && K == 4096) { - kernel = backend_ctx->CL_mul_mat_vec_q4_0_f32_1d_4x_flat_32000_1_4096; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); } - } else { - kernel = backend_ctx->CL_mul_mat_Ab_Bi_8x4; - } - // <--------------------------------------------> // - - // set kernel args - // <--------------------------------------------> // - cl_uint k_arg = 0; - if (N == 1) { - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &A_image1d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &B_image1d)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &extra1->offset)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_ulong), &extrad->offset)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &r3)); - } else { - region.origin = extrad->offset; // Specify the starting offset (in bytes) - region.size = M * N * sizeof(float); // Specify the size of the sub-buffer - C_d = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); - CL_CHECK(status); - - int padded_N = ne1 + padding; - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); //A_q_dextra0_q4_0->q - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); //A_s_d - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &B_image1d)); //B_d - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &C_d)); //C_d - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne01)); //M - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &padded_N)); //N with padding - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); //K - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne1)); //N without padding - } - // <--------------------------------------------> // - - // choose workgroup size - // <--------------------------------------------> // - size_t global_work_size[3] = { - 64, static_cast((M+63)/64), static_cast((N+31)/32)}; - size_t local_work_size[3] = {64, 2, 4}; - - global_work_size[0] = (size_t)(ceil((float)ne1/8)); - global_work_size[1] = (size_t)(ne01/4); - global_work_size[2] = (size_t)(1); - - local_work_size[0] = (size_t)(1); //4x32 for FP32 - local_work_size[1] = (size_t)(128); - local_work_size[2] = (size_t)(1); - - //WGS tuning - if (ne0 == 4096 && ne1 == 128 && ne10 == 4096) { - local_work_size[0] = 1; - local_work_size[1] = 128; - } else if (ne0 == 11008 && ne1 == 128 && ne10 == 4096) { - local_work_size[0] = 2; - local_work_size[1] = 64; - } else if (ne0 == 4096 && ne1 == 128 && ne10 == 11008) { - local_work_size[0] = 2; - local_work_size[1] = 64; - } else if (ne0 == 32000 && ne1 == 128 && ne10 == 4096) { - local_work_size[0] = 2; - local_work_size[1] = 64; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; } + case GGML_TYPE_Q5_K: { +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q5_K_f32_flat; - if (N == 1) { - size_t wavesize = backend_ctx->adreno_wave_size; - local_work_size[0] = wavesize; // localsize - local_work_size[1] = 4; // reduce factor - local_work_size[2] = 1; - - global_work_size[0] = (((M / 2) + wavesize - 1) / wavesize) * wavesize; - global_work_size[1] = 4; // reduce factor - global_work_size[2] = 1; - } - // <--------------------------------------------> // + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 8; // 4->8 rows per subgroup (2x activation reuse) + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = 16; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - // enqueue kernel with profiling - // <--------------------------------------------> // - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - // <--------------------------------------------> // + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_K->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_K->dm)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q5_K_f32; - // deallocate sub buffers and images - // <--------------------------------------------> // - CL_CHECK(clReleaseMemObject(A_image1d)); - CL_CHECK(clReleaseMemObject(B_sub_buffer)); - CL_CHECK(clReleaseMemObject(B_image1d)); + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - if (N != 1) { - CL_CHECK(clReleaseMemObject(B_d)); - CL_CHECK(clReleaseMemObject(B_d_input_image)); - CL_CHECK(clReleaseMemObject(C_d)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; } - // <--------------------------------------------> // - - return; - } - } // if (ne01 && ne1) -#endif // GGML_OPENCL_USE_ADRENO_KERNELS - - // GEMM using local memory - // Current BK = 16, so ne00 % 16 == 0 - if (src1t == GGML_TYPE_F32 && - ne00 % 16 == 0 && - ne11 > 1) { - switch(src0t) { - case GGML_TYPE_F32: { - kernel = backend_ctx->kernel_mul_mm_f32_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) - - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; - - cl_mem mem_src0 = extra0->data_device; - cl_mem mem_src1 = extra1->data_device; - - cl_ulong nb00_cont = nb00; - cl_ulong nb01_cont = nb01; - cl_ulong nb02_cont = nb02; - cl_ulong nb03_cont = nb03; + case GGML_TYPE_Q6_K: +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q6_K_f32_flat; - cl_ulong nb10_cont = nb10; - cl_ulong nb11_cont = nb11; - cl_ulong nb12_cont = nb12; - cl_ulong nb13_cont = nb13; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = 16; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - cl_ulong offset0_cont = offset0; - cl_ulong offset1_cont = offset1; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3)); + // The optimizer-barrier arg exists only in the ADRENO_OLD_COMPILER build of + // this kernel; conformant compilers get the original 17-arg signature. + if (backend_ctx->q6_k_flat_old_compiler) { + cl_uchar q6k_mask = 0xFF; // never 0xFE in prod; see the kernel note + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_uchar), &q6k_mask)); + } +#else + kernel = backend_ctx->kernel_mul_mv_q6_K_f32; - if (!ggml_is_contiguous(src0)) { - backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); - ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, - nb00_cont, nb01_cont, nb02_cont, nb03_cont); - mem_src0 = backend_ctx->prealloc_src0.buffer; - offset0_cont = 0; - } + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = 1; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = 1; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - if (!ggml_is_contiguous(src1)) { - backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); - ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, - nb10_cont, nb11_cont, nb12_cont, nb13_cont); - mem_src1 = backend_ctx->prealloc_src1.buffer; - offset1_cont = 0; - } + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + case GGML_TYPE_MXFP4: { +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_mxfp4_f32_flat; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + cl_mem q; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*2; - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; + q = extra0_mxfp4->q; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*2; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; + q = extra0_mxfp4->q_img; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); } - case GGML_TYPE_F16: { - kernel = backend_ctx->kernel_mul_mm_f16_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_mxfp4->e)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_mxfp4_f32; - cl_mem mem_src0 = extra0->data_device; - cl_mem mem_src1 = extra1->data_device; + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 2; + ndst = nth1*2; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 2; + ndst = nth1*2; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } - cl_ulong nb00_cont = nb00; - cl_ulong nb01_cont = nb01; - cl_ulong nb02_cont = nb02; - cl_ulong nb03_cont = nb03; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float)*nth0,nullptr)); +#endif + break; + } + default: + GGML_ASSERT(false && "not implemented"); + } - cl_ulong nb10_cont = nb10; - cl_ulong nb11_cont = nb11; - cl_ulong nb12_cont = nb12; - cl_ulong nb13_cont = nb13; + if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_MXFP4 || + src0t == GGML_TYPE_Q4_1 || + src0t == GGML_TYPE_Q5_0 || + src0t == GGML_TYPE_Q5_1 || + src0t == GGML_TYPE_Q8_0 || + src0t == GGML_TYPE_Q1_0 || + src0t == GGML_TYPE_IQ4_NL || + src0t == GGML_TYPE_Q2_K) { + // Each SIMD group produces N_DST values in the result. Assuming each + // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will + // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size + // (number of workgroups) will be a nearest multiple of + // N_DST*N_SIMDGROUP to cover the size of the dimension. Below, 4 is + // N_DST*N_SIMDGROUP (see the kernel for Q4_0 matmul). + size_t global_work_size[] = {(size_t)(ne01 + ndst-1)/ndst*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - cl_ulong offset0_cont = offset0; - cl_ulong offset1_cont = offset1; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (src0t == GGML_TYPE_Q4_K) { + size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - if (!ggml_is_contiguous(src0)) { - backend_ctx->prealloc_src0.allocate(backend_ctx->context, ggml_nbytes(src0)); - ggml_cl_copy_to_contiguous(backend, src0, backend_ctx->prealloc_src0.buffer, - nb00_cont, nb01_cont, nb02_cont, nb03_cont); - mem_src0 = backend_ctx->prealloc_src0.buffer; - offset0_cont = 0; - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (src0t == GGML_TYPE_Q3_K) { + GGML_ASSERT(false && "not implemented"); + } else if (src0t == GGML_TYPE_Q5_K) { + size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - if (!ggml_is_contiguous(src1)) { - backend_ctx->prealloc_src1.allocate(backend_ctx->context, ggml_nbytes(src1)); - ggml_cl_copy_to_contiguous(backend, src1, backend_ctx->prealloc_src1.buffer, - nb10_cont, nb11_cont, nb12_cont, nb13_cont); - mem_src1 = backend_ctx->prealloc_src1.buffer; - offset1_cont = 0; - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (src0t == GGML_TYPE_Q6_K) { + size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &mem_src0)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0_cont)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &mem_src1)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1_cont)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_x8 || + kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_x8_pair || + kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_y8) { + // multi-output decode variants: each WG processes 8 outputs along ne01, ne11 == 1 + const int64_t n_wg_x = ne01 / 8; + size_t global_work_size[] = {(size_t)n_wg_x*nth0, (size_t)nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_x8_gqa4) { + // GQA-coalesced KQ: one WG per K-head emits N_K_ROWS_GQA=16 K-rows * r2 Q-heads + const int64_t n_wg_x = ne01 / 16; + size_t global_work_size[] = {(size_t)n_wg_x*nth0, (size_t)nth1, (size_t)ne02*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_y8_gqa) { + // GQA-coalesced KQV: one WG per K-head emits 8 DV-rows * r2 Q-heads + const int64_t n_wg_x = ne01 / 8; + size_t global_work_size[] = {(size_t)n_wg_x*nth0, (size_t)nth1, (size_t)ne02*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr) { + const int NDST_DR = 4; + size_t global_work_size[] = {(size_t)CEIL_DIV(ne01, NDST_DR)*nth0, (size_t)nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls) { + size_t global_work_size[] = {(size_t)CEIL_DIV(ne01, 2)*nth0, (size_t)nth1, (size_t)ne02*ne03}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q4_0: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq) { + size_t global_work_size[] = {(size_t)CEIL_DIV(ne01, 4)*nth0, (size_t)nth1, (size_t)ne02*ne03}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - kernel = backend_ctx->kernel_mul_mm_q4_0_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + int64_t ny = (ne11 + nrows - 1)/nrows; - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; + size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } + } +} - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; +static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, int ne20) { + cl_int err; + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q4_1: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } + ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *)src->extra; + cl_ulong offset = extra->offset + src->view_offs; - kernel = backend_ctx->kernel_mul_mm_q4_1_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + const int ne21 = src->ne[1]; + const int nb21 = src->nb[1]; + const int ne02 = nb21 / src->nb[0]; + const int n_tile_size = 32; + const int max_post_router_tile = (ne20 * ne21 / n_tile_size) + ne02; - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; + cl_buffer_region region; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r3)); + backend_ctx->prealloc_post_router.allocate(backend_ctx->context, sizeof(int) * max_post_router_tile * n_tile_size); + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + cl_mem post_router_buf = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; + backend_ctx->prealloc_emap.allocate(backend_ctx->context, sizeof(short) * max_post_router_tile); + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + cl_mem emap_buf = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q8_0: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } + backend_ctx->prealloc_hist.allocate(backend_ctx->context, sizeof(int) * ne02); + region.origin = 0; + region.size = sizeof(int) * ne02; + cl_mem hist_buf = clCreateSubBuffer(backend_ctx->prealloc_hist.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - kernel = backend_ctx->kernel_mul_mm_q8_0_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + backend_ctx->prealloc_tile_offset.allocate(backend_ctx->context, sizeof(int) * ne02); + region.origin = 0; + region.size = sizeof(int) * ne02; + cl_mem tile_offset_buf = clCreateSubBuffer(backend_ctx->prealloc_tile_offset.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; + backend_ctx->prealloc_slot_counter.allocate(backend_ctx->context, sizeof(int) * ne02); + region.origin = 0; + region.size = sizeof(int) * ne02; + cl_mem slot_counter_buf = clCreateSubBuffer(backend_ctx->prealloc_slot_counter.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); + backend_ctx->prealloc_total_tiles.allocate(backend_ctx->context, sizeof(int)); + region.origin = 0; + region.size = sizeof(int); + cl_mem total_tiles_buf = clCreateSubBuffer(backend_ctx->prealloc_total_tiles.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + // Histogram + cl_kernel kernel = backend_ctx->kernel_moe_histogram; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &hist_buf)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset)); + + size_t histogram_global_size[] = {(size_t)(((ne21 + 63) / 64) * 64), static_cast(ne20), 1}; + size_t histogram_local_size[] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src); + + // Scan + kernel = backend_ctx->kernel_moe_scan; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &hist_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &tile_offset_buf)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &total_tiles_buf)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &slot_counter_buf)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &n_tile_size)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne02)); + + size_t scan_global_size[] = {1}; + size_t scan_local_size[] = {1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 1, scan_global_size, scan_local_size, src); + + // Fill + kernel = backend_ctx->kernel_moe_fill; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &post_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &total_tiles_buf)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &n_tile_size)); + + size_t fill_global_size[] = {(size_t)(((max_post_router_tile + 63) / 64) * 64), n_tile_size, 1}; + size_t fill_local_size[] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, fill_global_size, fill_local_size, src); + + // Scatter. The deterministic variant is the default: kernel_moe_scatter derives + // each token's slot from an atomic counter, so the packing inside an expert - and + // with it the output of the ragged prefill GEMM - changes from run to run. Set + // GGML_OPENCL_MOE_STABLE_SCATTER=0 to restore the atomic version. + static const bool stable_scatter = []{ + const char * e = getenv("GGML_OPENCL_MOE_STABLE_SCATTER"); + return !e || e[0] == '\0' || e[0] != '0'; + }(); + + if (stable_scatter) { + kernel = backend_ctx->kernel_moe_scatter_stable; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset)); + + // one workgroup (one wave) per expert; each ranks its own tokens + size_t scatter_global_size[] = {64, (size_t)ne02}; + size_t scatter_local_size[] = {64, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, scatter_global_size, scatter_local_size, src); + } else { + kernel = backend_ctx->kernel_moe_scatter; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &post_router_buf)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &emap_buf)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &tile_offset_buf)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &slot_counter_buf)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offset)); + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src); + } + + // [MOE_TILES] env-gated padding probe: read back total_tiles (= Sum_e + // ceil(k_e/n_tile_size)) and compare to the ideal tile count for the real + // routing count. Quantifies the per-expert tile-padding waste. Blocking + // readback perturbs timing -> diagnostic only. + if (getenv("GGML_OPENCL_MOE_TILES_DEBUG")) { + int h_total = 0; + clFinish(backend_ctx->queue); + CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, total_tiles_buf, CL_TRUE, 0, sizeof(int), &h_total, 0, NULL, NULL)); + const int routings = ne20 * ne21; + const int ideal = (routings + n_tile_size - 1) / n_tile_size; + const int slots = h_total * n_tile_size; + fprintf(stderr, "[MOE_TILES] routings=%d (ne20=%d ne21=%d nexp=%d) total_tiles=%d ideal=%d slots=%d pad=%.1f%%\n", + routings, ne20, ne21, ne02, h_total, ideal, slots, + routings > 0 ? 100.0 * (slots - routings) / routings : 0.0); + fflush(stderr); + } + + CL_CHECK(clReleaseMemObject(hist_buf)); + CL_CHECK(clReleaseMemObject(tile_offset_buf)); + CL_CHECK(clReleaseMemObject(total_tiles_buf)); + CL_CHECK(clReleaseMemObject(slot_counter_buf)); + CL_CHECK(clReleaseMemObject(post_router_buf)); + CL_CHECK(clReleaseMemObject(emap_buf)); +} - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; +static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q4_K: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } + const ggml_tensor * src2 = dst->src[2]; + GGML_ASSERT(src2); + GGML_ASSERT(src2->extra); - kernel = backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_K->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_K->s)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_K->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_K->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offset2 = extra2->offset + src2->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; + GGML_UNUSED(offset0); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q5_K: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } +#ifdef GGML_OPENCL_SOA_Q + // SoA extra lives on view_src (view->extra is pre-SoA). + const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0; + ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)soa0_src->extra; + ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)soa0_src->extra; + ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; + ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; + ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; + ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; + ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; - kernel = backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) +#endif - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; + // TODO: general MoE for the following types + (void)extra0_q4_1; + (void)extra0_q5_0; + (void)extra0_q5_1; + (void)extra0_q4_K; + (void)extra0_q5_K; + (void)extra0_q6_K; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_K->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_K->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_K->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; + const cl_ulong nb00 = src0->nb[0]; + const cl_ulong nb01 = src0->nb[1]; + const cl_ulong nb02 = src0->nb[2]; + const cl_ulong nb03 = src0->nb[3]; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - case GGML_TYPE_Q6_K: { - if (ne11 < 32) { - break; - } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { - break; - } + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; - kernel = backend_ctx->kernel_mul_mm_q6_k_f32_l4_lm; - nth0 = 128; // calculated as (BM*BN)/(TM*TN) + const cl_ulong nb11 = src1->nb[1]; + const cl_ulong nb12 = src1->nb[2]; + const cl_ulong nb13 = src1->nb[3]; - int batch_stride_a = ne00*ne01; - int batch_stride_b = ne10*ne11; - int batch_stride_d = ne0*ne1; + const int ne20 = src2->ne[0]; + const int ne21 = src2->ne[1]; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); // stride_a - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10)); // stride_b - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne01)); // stride_d - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); + const cl_ulong nb21 = src2->nb[1]; + const cl_ulong nb20 = src2->nb[0]; - // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed. - size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, 1, 1}; + UNUSED(nb20); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } - default: - break; - } - } + const int ne0 = dst->ne[0]; + const int ne1 = dst->ne[1]; + const int ne2 = dst->ne[2]; - if (src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32 && - src0->ne[1] > 32 && // M > 32 - src1->ne[1] > 32 && // N > 32 - src0->ne[0] > 32 && // K > 32 - src0->ne[2] == 1 && src0->ne[3] == 1 && - src1->ne[2] == 1 && src1->ne[3] == 1 && - ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && - backend_ctx->kernel_mul_mat_f16_f32_tiled != NULL) { - ggml_cl_mul_mat_f16_f32_tiled(backend, src0, src1, dst); - return; - } + GGML_UNUSED(ne2); - if (!ggml_is_transposed(src0) && - !ggml_is_transposed(src1) && - src1t == GGML_TYPE_F32 && - ne00%32 == 0 && - ne11 > 2) { -#ifdef GGML_OPENCL_SOA_Q - // Set up kernel. - switch(src0t) { - case GGML_TYPE_Q4_0: - // This should have been satisfied. - GGML_ASSERT(ne11 == ne1); - GGML_ASSERT(ne01 == ne0); + const int r2 = ne12/ne02; + const int r3 = ne13/ne03; + const int dst_rows = ne20*ne21; // ne20 = n_used_experts, ne21 = n_rows - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 1; + GGML_ASSERT(ne00 == ne10); - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_16x_flat; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; + // the matmul kernels only write the rows an expert owns, so zero the skipped rows here + { + cl_int err; - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_1d_8x_flat; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + cl_buffer_region region; + region.origin = offset2; + region.size = nb21 * ne21; + cl_mem buf_ids = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); - break; - default: - break; - } + region.origin = offsetd; + region.size = (size_t) ne0 * ne1 * ne2 * sizeof(float); + cl_mem buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); - // Launch kernel. - if (src0t == GGML_TYPE_Q4_0) { - size_t global_work_size[] = {(size_t)(ne01 + 7)/8*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + const cl_uint ids_stride = nb21 / sizeof(int); - if (backend_ctx->gpu_family == INTEL) { - // Set global size for Intel. It uses 16x output values. - global_work_size[0] = (size_t)(ne01 + 15)/16*nth0; - global_work_size[1] = (size_t)ne11*nth1; - global_work_size[2] = (size_t)ne12*ne13; - } + cl_kernel kernel_zero = backend_ctx->kernel_moe_zero_dst; + CL_CHECK(clSetKernelArg(kernel_zero, 0, sizeof(cl_mem), &buf_ids)); + CL_CHECK(clSetKernelArg(kernel_zero, 1, sizeof(cl_mem), &buf_dst)); + CL_CHECK(clSetKernelArg(kernel_zero, 2, sizeof(cl_uint), &ne21)); + CL_CHECK(clSetKernelArg(kernel_zero, 3, sizeof(cl_uint), &ne20)); + CL_CHECK(clSetKernelArg(kernel_zero, 4, sizeof(cl_uint), &ids_stride)); + CL_CHECK(clSetKernelArg(kernel_zero, 5, sizeof(cl_uint), &ne0)); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - return; - } -#else // GGML_OPENCL_SOA_Q - // TODO: add block_q4_0 variant. -#endif // GGML_OPENCL_SOA_Q + size_t zero_local[3] = {1, 1, 64}; + size_t zero_global[3] = {(size_t) ne21, (size_t) ne20, 64}; + backend_ctx->enqueue_ndrange_kernel(kernel_zero, 3, zero_global, zero_local, dst); + + CL_CHECK(clReleaseMemObject(buf_ids)); + CL_CHECK(clReleaseMemObject(buf_dst)); } - // use custom matrix x vector kernel - switch (src0t) { - case GGML_TYPE_F32: - //GGML_ASSERT(ne02 == ne12); - GGML_ASSERT(src1t == GGML_TYPE_F32); - kernel = backend_ctx->kernel_mul_mat_f32_f32; - nrows = 4; + int sgs = 32; // subgroup size + int nsg = 1; // number of subgroups + int nrows = 1; // number of row in src1 + int ndst = 4; // number of values produced by each subgroup - if (backend_ctx->gpu_family == INTEL) { - nth0 = 32; - nth1 = 1; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + const int n_tile_size = 32; + const int max_post_router_tile = (ne20 * ne21 / n_tile_size) + ne02; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); - break; - case GGML_TYPE_F16: - //GGML_ASSERT(ne02 == ne12); - if (backend_ctx->gpu_family == INTEL) { - nth0 = 32; - nth1 = 1; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + GGML_UNUSED(max_post_router_tile); + + cl_kernel kernel; + + // subgroup mat vec + switch (src0->type) { + case GGML_TYPE_Q4_0: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, src0)) { + cl_int status; - if (src1t == GGML_TYPE_F32) { - if (ne11 * ne12 < 4) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_1row; - } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4; - nrows = ne11; - } else { - kernel = backend_ctx->kernel_mul_mat_f16_f32; - nrows = 4; - } - } else { - kernel = backend_ctx->kernel_mul_mat_f16_f16; - nrows = 4; - } + size_t local_size[3] = {64, 2, 1}; + size_t global_size[3] = {64, 2, 1}; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb10)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3)); - break; - case GGML_TYPE_Q4_0: - // This should have been satisfied. - GGML_ASSERT(ne11 == ne1); - GGML_ASSERT(ne01 == ne0); + if (ne12 == 1) { // for gemv + kernel = backend_ctx->kernel_gemv_moe_q4_0_f32_ns; -#ifdef GGML_OPENCL_SOA_Q - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 1; + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; - ndst = 8; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; + // create a sub_buffer for src2 + cl_buffer_region region; + region.origin = offset2; + region.size = ne20 * ne21 * sizeof(int); + buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_8x_flat; - ndst =8; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + // set thread grid + global_size[0] = static_cast(((ne01 + 63) / 64) * 64); + global_size[1] = 4; + global_size[2] = static_cast(ne20); + local_size[1] = 4; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); -#else // GGML_OPENCL_SOA_Q - if (backend_ctx->gpu_family == INTEL) { - // Use 1D local size. Each workgroup is a SIMD group. Each SIMD - // group produces N_DST (4 for Q4_0 kernel) values in the result. - // The number of workgroups on dim 0 (the leading dimension) is - // the nearest multiple of 4 that covers ne0 (equals ne01). - nth0 = 16; - nth1 = 1; + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - kernel = backend_ctx->kernel_mul_mat_q4_0_f32; - ndst = 4; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; + // create image for src1 + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; + buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); - kernel = backend_ctx->kernel_mul_mat_q4_0_f32_v; - ndst = 4; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); -#endif // GGML_OPENCL_SOA_Q - break; - case GGML_TYPE_Q4_1: { -#ifdef GGML_OPENCL_SOA_Q - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 1; - ndst = 4; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; - ndst = 4; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + // launch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); - kernel = backend_ctx->kernel_mul_mv_q4_1_f32_flat; + // deallocate sub buffers and images + CL_CHECK(clReleaseMemObject(src1_sub_buffer)); + CL_CHECK(clReleaseMemObject(buf_src1_image)); + CL_CHECK(clReleaseMemObject(buf_src2)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_1->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_1->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_1->m)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r3)); -#else - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 1; - ndst = 4; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; - ndst = 4; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + } else { // for gemm + // dp4a (int8) prefill GEMM variant + static const char * q4_0_moe_dp4a_env = getenv("GGML_OPENCL_Q4_0_MOE_DP4A"); + + // It turns out that the prebuilt kernel only outperforms the dp4a variant (on X2-90) + // at very large routing counts, so we gate its use accordingly using moe_bin_min, + // which can be overridden via the GGML_OPENCL_MOE_BIN_MIN_ROUTINGS environment variable. + // The routing count is ne20 * ne21 (n_expert_used * n_tokens). + static const char * moe_bin_min_env = getenv("GGML_OPENCL_MOE_BIN_MIN_ROUTINGS"); + const int moe_bin_min = moe_bin_min_env ? atoi(moe_bin_min_env) : 4096; + + // whether bin kernels are available + const bool bin_available = backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin != nullptr; + const bool dp4a_bin_available = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin != nullptr; + + bool use_moe_dp4a = q4_0_moe_dp4a_env + ? (atoi(q4_0_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E + && (dp4a_bin_available || !bin_available + || (int)(ne20 * ne21) < moe_bin_min)); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; + + const bool use_bin_kernel = bin_available && !use_moe_dp4a; + + kernel = use_bin_kernel + ? backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin + : backend_ctx->kernel_gemm_moe_q4_0_f32_ns; + + // Reorder router if called from test-backend-ops or when new router is generated. + // Otherwise reuse the reordered result from previous mul_mat_id call. + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; + } - kernel = backend_ctx->kernel_mul_mv_q4_1_f32; + cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; + cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; + cl_mem buf_src2, buf_src2_emap; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); -#endif // GGML_OPENCL_SOA_Q - break; - } - case GGML_TYPE_Q8_0: { -#ifdef GGML_OPENCL_SOA_Q - kernel = backend_ctx->kernel_mul_mv_q8_0_f32_flat; + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - // nth0 - subgroup size - // nth1 - number of subgroups per workgroup - // ndst - number of output values per workgroup = output per subgroup * number of subgroups - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 2; - ndst = nth1*4; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 2; - ndst = nth1*4; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q8_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q8_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); -#else - kernel = backend_ctx->kernel_mul_mv_q8_0_f32; + // Reorder activations + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - // nth0 - subgroup size - // nth1 - number of subgroups per workgroup - // ndst - number of output values per workgroup = output per subgroup * number of subgroups - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 2; - ndst = nth1*4; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 2; - ndst = nth1*4; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + + if (!use_moe_dp4a) { + // Create image for reordered src1 + // Use pre-allocated placeholder + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1; + cl_image_desc image_desc_buf_src1; + image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (use_bin_kernel) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + } + + // MoE kernel prepare + // Create sub buffer for dst + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer( + extrad->data_device, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + // Create image for dst + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); + + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + // fused reorder + q8_1 quant straight from the original activations + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + // dp4a GEMM + cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a; + if (backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin) { + dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a_bin; + } + + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->q_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; + } + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->q_img)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); + + // set thread grid + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; + + // Dispatch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + } + return; + } // fallback to generic Q4_0 MoE kernel - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); -#endif // GGML_OPENCL_SOA_Q - break; - } - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: { -#ifdef GGML_OPENCL_SOA_Q - kernel = backend_ctx->kernel_mul_mv_q4_K_f32_flat; +#endif // GGML_OPENCL_USE_ADRENO_KERNELS + kernel = backend_ctx->kernel_mul_mv_id_q4_0_f32_8x_flat; if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 1; - ndst = 4; + sgs = 16; + nsg = 1; + ndst = 8; } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 2; - ndst = 16; + sgs = 64; + nsg = 1; + ndst = 8; } else { GGML_ASSERT(false && "TODO: Unknown GPU"); } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_K->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_K->s)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q4_K->d)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q4_K->dm)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3)); -#else - kernel = backend_ctx->kernel_mul_mv_q4_K_f32; - - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 1; - ndst = 4; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; - ndst = 4; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne20)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne21)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb21)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &r3)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); -#endif // GGML_OPENCL_SOA_Q break; } - case GGML_TYPE_Q5_K: { -#ifdef GGML_OPENCL_SOA_Q - kernel = backend_ctx->kernel_mul_mv_q5_K_f32_flat; + case GGML_TYPE_Q4_1: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, src0)) { + cl_int status; - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 1; - ndst = 4; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 2; - ndst = 16; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + size_t local_size[3] = {64, 2, 1}; + size_t global_size[3] = {64, 2, 1}; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_K->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_K->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_q5_K->dm)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); -#else - kernel = backend_ctx->kernel_mul_mv_q5_K_f32; + if (ne12 == 1) { // for gemv + kernel = backend_ctx->kernel_gemv_moe_q4_1_f32_ns; - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 1; - ndst = 4; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 1; - ndst = 4; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(int), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); -#endif // GGML_OPENCL_SOA_Q - break; + // create a sub_buffer for src2 + cl_buffer_region region; + region.origin = offset2; + region.size = ne20 * ne21 * sizeof(int); + buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // set thread grid + global_size[0] = static_cast(((ne01 + 63) / 64) * 64); + global_size[1] = 4; + global_size[2] = static_cast(ne20); + local_size[1] = 4; + + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // create image for src1 + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; + buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->q)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + + // launch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + // deallocate sub buffers and images + CL_CHECK(clReleaseMemObject(src1_sub_buffer)); + CL_CHECK(clReleaseMemObject(buf_src1_image)); + CL_CHECK(clReleaseMemObject(buf_src2)); + + } else { // for gemm + kernel = backend_ctx->kernel_gemm_moe_q4_1_f32_ns; + if (backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin) { + kernel = backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin; + } + + // Reorder router if called from test-backend-ops or when new router is generated. + // Otherwise reuse the reordered result from previous mul_mat_id call. + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; + } + + cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem buf_src2, buf_src2_emap; + + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Reorder activations + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Create image for reordered src1 + // Use pre-allocated placeholder + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1; + cl_image_desc image_desc_buf_src1; + image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_q4_1_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + + // MoE kernel prepare + // Create sub buffer for dst + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer( + extrad->data_device, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + // Create image for dst + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->q_img)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_1->m)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); + + // set thread grid + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; + + // Dispatch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + } + return; + } +#endif //GGML_OPENCL_USE_ADRENO_KERNELS } - case GGML_TYPE_Q6_K: -#ifdef GGML_OPENCL_SOA_Q - kernel = backend_ctx->kernel_mul_mv_q6_K_f32_flat; + case GGML_TYPE_Q5_0: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, src0)) { + cl_int status; + + size_t local_size[3] = {64, 2, 1}; + size_t global_size[3] = {64, 2, 1}; + + if (ne12 == 1) { // for gemv + kernel = backend_ctx->kernel_gemv_moe_q5_0_f32_ns; + + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; + + // create a sub_buffer for src2 + cl_buffer_region region; + region.origin = offset2; + region.size = ne20 * ne21 * sizeof(int); + buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // set thread grid + global_size[0] = static_cast(((ne01 + 63) / 64) * 64); + global_size[1] = 4; + global_size[2] = static_cast(ne20); + local_size[1] = 4; - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 2; - ndst = 4; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 2; - ndst = 4; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q6_K->ql)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q6_K->qh)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q6_K->s)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q6_K->d)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3)); -#else - kernel = backend_ctx->kernel_mul_mv_q6_K_f32; + // create image for src1 + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; + buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 2; - ndst = 1; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 2; - ndst = 1; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qs)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); -#endif // GGML_OPENCL_SOA_Q - break; - case GGML_TYPE_MXFP4: { -#ifdef GGML_OPENCL_SOA_Q - kernel = backend_ctx->kernel_mul_mv_mxfp4_f32_flat; + // launch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); - cl_mem q; - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 2; - ndst = nth1*2; + // deallocate sub buffers and images + CL_CHECK(clReleaseMemObject(src1_sub_buffer)); + CL_CHECK(clReleaseMemObject(buf_src1_image)); + CL_CHECK(clReleaseMemObject(buf_src2)); - q = extra0_mxfp4->q; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 2; - ndst = nth1*2; + } else { // for gemm + kernel = backend_ctx->kernel_gemm_moe_q5_0_f32_ns; - q = extra0_mxfp4->q_img; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + // Reorder router if called from test-backend-ops or when new router is generated. + // Otherwise reuse the reordered result from previous mul_mat_id call. + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; + } - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_mxfp4->e)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3)); -#else - kernel = backend_ctx->kernel_mul_mv_mxfp4_f32; + cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem buf_src2, buf_src2_emap; - if (backend_ctx->gpu_family == INTEL) { - nth0 = 16; - nth1 = 2; - ndst = nth1*2; - } else if (backend_ctx->gpu_family == ADRENO) { - nth0 = 64; - nth1 = 2; - ndst = nth1*2; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); - } + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb13)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float)*nth0,nullptr)); -#endif - break; + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Reorder activations + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Generic dp4a MoE GEMM + { + static const char * q5mdp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A"); + const bool q5mdp4a_on = q5mdp4a_env ? (atoi(q5mdp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + const bool use_q5_moe_dp4a = q5mdp4a_on + && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 != nullptr + && extra0_q5_0->scale != nullptr; + + if (use_q5_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + unsigned short map_ratio_q5 = ne20 / ne11; + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format dp_ifd = {CL_R, CL_FLOAT}; + cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}}; + cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status); + CL_CHECK(status); + + int ne00i = (int)ne00, ne01i = (int)ne01; + cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50; + int has_min_q5 = 1; + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qs_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->scale)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->min)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(dp_sub_buf_dst); + clReleaseMemObject(dp_buf_dst_image); + return; + } + } + + // Create image for reordered src1 + // Use pre-allocated placeholder + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1; + cl_image_desc image_desc_buf_src1; + image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + + // MoE kernel prepare + // Create sub buffer for dst + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer( + extrad->data_device, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + // Create image for dst + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qs_img)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); + + // set thread grid + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; + + // Dispatch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + } + return; + } +#endif //GGML_OPENCL_USE_ADRENO_KERNELS } - default: - GGML_ASSERT(false && "not implemented"); - } + case GGML_TYPE_Q5_1: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, src0)) { + cl_int status; - if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_MXFP4 || - src0t == GGML_TYPE_Q4_1 || - src0t == GGML_TYPE_Q8_0 || - src0t == GGML_TYPE_Q2_K) { - // Each SIMD group produces N_DST values in the result. Assuming each - // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will - // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size - // (number of workgroups) will be a nearest multiple of - // N_DST*N_SIMDGROUP to cover the size of the dimension. Below, 4 is - // N_DST*N_SIMDGROUP (see the kernel for Q4_0 matmul). - size_t global_work_size[] = {(size_t)(ne01 + ndst-1)/ndst*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + size_t local_size[3] = {64, 2, 1}; + size_t global_size[3] = {64, 2, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else if (src0t == GGML_TYPE_Q4_K) { - size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + if (ne12 == 1) { // for gemv + kernel = backend_ctx->kernel_gemv_moe_q5_1_f32_ns; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else if (src0t == GGML_TYPE_Q3_K) { - GGML_ASSERT(false && "not implemented"); - } else if (src0t == GGML_TYPE_Q5_K) { - size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else if (src0t == GGML_TYPE_Q6_K) { - size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + // create a sub_buffer for src2 + cl_buffer_region region; + region.origin = offset2; + region.size = ne20 * ne21 * sizeof(int); + buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else { - int64_t ny = (ne11 + nrows - 1)/nrows; + // set thread grid + global_size[0] = static_cast(((ne01 + 63) / 64) * 64); + global_size[1] = 4; + global_size[2] = static_cast(ne20); + local_size[1] = 4; - size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } -} + // create image for src1 + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; + buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); -static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_ASSERT(src0); - GGML_ASSERT(src0->extra); - GGML_ASSERT(src1); - GGML_ASSERT(src1->extra); - GGML_ASSERT(dst); - GGML_ASSERT(dst->extra); + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qs)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); - const ggml_tensor * src2 = dst->src[2]; - GGML_ASSERT(src2); - GGML_ASSERT(src2->extra); + // launch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); - ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + // deallocate sub buffers and images + CL_CHECK(clReleaseMemObject(src1_sub_buffer)); + CL_CHECK(clReleaseMemObject(buf_src1_image)); + CL_CHECK(clReleaseMemObject(buf_src2)); + } else { // for gemm + kernel = backend_ctx->kernel_gemm_moe_q5_1_f32_ns; - ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; - ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; - ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *)src2->extra; - ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + // Reorder router if called from test-backend-ops or when new router is generated. + // Otherwise reuse the reordered result from previous mul_mat_id call. + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; + } - cl_ulong offset0 = extra0->offset + src0->view_offs; - cl_ulong offset1 = extra1->offset + src1->view_offs; - cl_ulong offset2 = extra2->offset + src2->view_offs; - cl_ulong offsetd = extrad->offset + dst->view_offs; + cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem buf_src2, buf_src2_emap; - GGML_UNUSED(offset0); + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); -#ifdef GGML_OPENCL_SOA_Q - ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra; - ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)src0->extra; - ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra; -#endif + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - const int ne00 = src0->ne[0]; - const int ne01 = src0->ne[1]; - const int ne02 = src0->ne[2]; - const int ne03 = src0->ne[3]; + // Reorder activations + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Create image for reordered src1 + // Use pre-allocated placeholder + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1; + cl_image_desc image_desc_buf_src1; + image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + + // MoE kernel prepare + // Create sub buffer for dst + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer( + extrad->data_device, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + // Create image for dst + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qs_img)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->qh)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_1->m)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); + + // set thread grid + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; + + // Dispatch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + } + return; + } +#endif //GGML_OPENCL_USE_ADRENO_KERNELS + } + case GGML_TYPE_Q8_0: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // MoE GEMM for q8_0 at prefill (ne12>1) + // There is no corresponding gemv_moe, so the code path is different here + static const char * moe_gemm_q8_env = getenv("GGML_OPENCL_MOE_GEMM_Q8"); + const bool moe_gemm_q8 = moe_gemm_q8_env + ? (atoi(moe_gemm_q8_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (moe_gemm_q8 && use_adreno_moe_kernels(backend_ctx, src0) && ne12 > 1) { + cl_int status; - const cl_ulong nb00 = src0->nb[0]; - const cl_ulong nb01 = src0->nb[1]; - const cl_ulong nb02 = src0->nb[2]; - const cl_ulong nb03 = src0->nb[3]; + size_t local_size[3] = {64, 2, 1}; + size_t global_size[3] = {64, 2, 1}; - const int ne10 = src1->ne[0]; - const int ne11 = src1->ne[1]; - const int ne12 = src1->ne[2]; - const int ne13 = src1->ne[3]; + kernel = backend_ctx->kernel_gemm_moe_q8_0_f32_ns; - const cl_ulong nb11 = src1->nb[1]; - const cl_ulong nb12 = src1->nb[2]; - const cl_ulong nb13 = src1->nb[3]; + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; + } - const int ne20 = src2->ne[0]; - const int ne21 = src2->ne[1]; + cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem buf_src2, buf_src2_emap; - const cl_ulong nb21 = src2->nb[1]; - const cl_ulong nb20 = src2->nb[0]; + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - UNUSED(nb20); + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - const int ne0 = dst->ne[0]; - const int ne1 = dst->ne[1]; + // Reorder activations (group tokens by expert into tiles of 32) + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - const int r2 = ne12/ne02; - const int r3 = ne13/ne03; - const int dst_rows = ne20*ne21; // ne20 = n_used_experts, ne21 = n_rows + // Generic dp4a MoE GEMM + { + static const char * q8mdp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A"); + const bool q8mdp4a_on = q8mdp4a_env ? (atoi(q8mdp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + const bool use_q8_moe_dp4a = q8mdp4a_on + && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 != nullptr + && extra0_q8_0->scale != nullptr; + if (use_q8_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + unsigned short map_ratio_q8 = ne20 / ne11; + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q8)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + // dst image + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format dp_ifd = {CL_R, CL_FLOAT}; + cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}}; + cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status); + CL_CHECK(status); + + int ne00i = (int)ne00, ne01i = (int)ne01; + cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80; + int has_min_q8 = 0; + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat int8 codes [expert][row][K] + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // uniform scale[16] + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // dummy min (has_min=0, unread) + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q8)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(dp_sub_buf_dst); + clReleaseMemObject(dp_buf_dst_image); + return; + } + } - GGML_ASSERT(ne00 == ne10); + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); - int sgs = 32; // subgroup size - int nsg = 1; // number of subgroups - int nrows = 1; // number of row in src1 - int ndst = 4; // number of values produced by each subgroup + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + + // dst image + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); - cl_kernel kernel; + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat q8_0 quants + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->d)); // flat q8_0 scales + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; - // subgroup mat vec - switch (src0->type) { - case GGML_TYPE_Q4_0: { - kernel = backend_ctx->kernel_mul_mv_id_q4_0_f32_8x_flat; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); - if (backend_ctx->gpu_family == INTEL) { - sgs = 16; - nsg = 1; - ndst = 8; - } else if (backend_ctx->gpu_family == ADRENO) { - sgs = 64; - nsg = 1; - ndst = 8; - } else { - GGML_ASSERT(false && "TODO: Unknown GPU"); + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; } - - CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q4_0->q)); - CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q4_0->d)); - CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); - CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); - CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); - CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); - CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01)); - CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02)); - CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb00)); - CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); - CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne10)); - CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne11)); - CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12)); - CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11)); - CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12)); - CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne20)); - CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne21)); - CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nb21)); - CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne0)); - CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &ne1)); - CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r2)); - CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &r3)); - - break; - } - case GGML_TYPE_Q8_0: { +#endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_id_q8_0_f32_flat; @@ -12365,6 +28874,791 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, #endif // GGML_OPENCL_SOA_Q break; } + case GGML_TYPE_Q4_K: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, src0)) { + cl_int status; + + size_t local_size[3] = {64, 2, 1}; + size_t global_size[3] = {64, 2, 1}; + + if (ne12 == 1) { // for gemv + kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns; + + // Weight-as-texture MoE decode GEMV + static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG"); + const bool moe_decode_wimg_on = moe_decode_wimg_env + ? (atoi(moe_decode_wimg_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + const bool use_moe_decode_wimg = moe_decode_wimg_on + && backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg != nullptr + && extra0_q4_K->q_img != nullptr; + if (use_moe_decode_wimg) { + kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg; + } + + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; + + // create a sub_buffer for src2 + cl_buffer_region region; + region.origin = offset2; + region.size = ne20 * ne21 * sizeof(int); + buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // set thread grid + global_size[0] = static_cast(((ne01 + 63) / 64) * 64); + global_size[1] = 4; + global_size[2] = static_cast(ne20); + local_size[1] = 4; + + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // create image for src1 + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; + buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_q4_K->q_img : &extra0_q4_K->q)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->dm)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->s)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + + // launch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + // deallocate sub buffers and images + CL_CHECK(clReleaseMemObject(src1_sub_buffer)); + CL_CHECK(clReleaseMemObject(buf_src1_image)); + CL_CHECK(clReleaseMemObject(buf_src2)); + + } else { // for gemm + kernel = backend_ctx->kernel_gemm_moe_q4_k_f32_ns; + if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) { + kernel = backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin; + } + + // Reorder router if called from test-backend-ops or when new router is generated. + // Otherwise reuse the reordered result from previous mul_mat_id call. + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; + } + + cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; + cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; + cl_mem buf_src2, buf_src2_emap; + + // dp4a (int8) prefill GEMM variant + static const char * q4k_moe_dp4a_env = getenv("GGML_OPENCL_Q4K_MOE_DP4A"); + bool use_moe_dp4a = (q4k_moe_dp4a_env != nullptr) + ? (atoi(q4k_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; + // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel + if (backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin == nullptr) { + use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr; + } + + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Reorder activations + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + + if (!use_moe_dp4a) { + // Create image for reordered src1 + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + } + + // MoE kernel prepare + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer( + extrad->data_device, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + // Create image for dst + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); + + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + // fused reorder + q8_1 quant straight from the original + // activations (no intermediate f32 reorder buffer) + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + // dp4a GEMM + cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a; + if (backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin) { + dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin; + } + + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->q_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->d)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->dm)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->s)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; + } + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->q_img)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->dm)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->s)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); + + // set thread grid + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; + + // Dispatch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + } + return; + } +#endif //GGML_OPENCL_USE_ADRENO_KERNELS + } + case GGML_TYPE_Q5_K: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, src0)) { + cl_int status; + + size_t local_size[3] = {64, 2, 1}; + size_t global_size[3] = {64, 2, 1}; + + if (ne12 == 1) { // for gemv + kernel = backend_ctx->kernel_gemv_moe_q5_k_f32_ns; + + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; + + // create a sub_buffer for src2 + cl_buffer_region region; + region.origin = offset2; + region.size = ne20 * ne21 * sizeof(int); + buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // set thread grid + global_size[0] = static_cast(((ne01 + 63) / 64) * 64); + global_size[1] = 4; + global_size[2] = static_cast(ne20); + local_size[1] = 4; + + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // create image for src1 + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; + buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->q)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->qh)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->dm)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->s)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + + // launch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + // deallocate sub buffers and images + CL_CHECK(clReleaseMemObject(src1_sub_buffer)); + CL_CHECK(clReleaseMemObject(buf_src1_image)); + CL_CHECK(clReleaseMemObject(buf_src2)); + + } else { // for gemm + kernel = backend_ctx->kernel_gemm_moe_q5_k_f32_ns; + + // Reorder router if called from test-backend-ops or when new router is generated. + // Otherwise reuse the reordered result from previous mul_mat_id call. + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; + } + + cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem buf_src2, buf_src2_emap; + + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Reorder activations + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Generic dp4a MoE GEMM + { + static const char * q5kmdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A"); + const bool q5kmdp4a_on = q5kmdp4a_env ? (atoi(q5kmdp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + bool use_moe_dp4a = q5kmdp4a_on + && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k != nullptr + && extra0_q5_K->scale != nullptr; + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; + + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + unsigned short map_ratio_q5k = ne20 / ne11; + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5k)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format dp_ifd = {CL_R, CL_FLOAT}; + cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}}; + cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status); + CL_CHECK(status); + + int ne00i = (int)ne00, ne01i = (int)ne01; + cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k; + int has_min_q5k = 1; + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->q_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->qh)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->scale)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->min)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5k)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(dp_sub_buf_dst); + clReleaseMemObject(dp_buf_dst_image); + return; + } + } + + // Create image for reordered src1 + // Use pre-allocated placeholder + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + + // MoE kernel prepare + // Create sub buffer for dst + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer( + extrad->data_device, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + // Create image for dst + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->q_img)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->qh)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->s)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q5_K->dm)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); + + // set thread grid + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; + + // Dispatch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + } + return; + } +#endif //GGML_OPENCL_USE_ADRENO_KERNELS + } + case GGML_TYPE_Q6_K: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (use_adreno_moe_kernels(backend_ctx, src0)) { + cl_int status; + + size_t local_size[3] = {64, 2, 1}; + size_t global_size[3] = {64, 2, 1}; + + if (ne12 == 1) { // for gemv + kernel = backend_ctx->kernel_gemv_moe_q6_k_f32_ns; + + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; + + // create a sub_buffer for src2 + cl_buffer_region region; + region.origin = offset2; + region.size = ne20 * ne21 * sizeof(int); + buf_src2 = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // set thread grid + global_size[0] = static_cast(((ne01 + 63) / 64) * 64); + global_size[1] = 4; + global_size[2] = static_cast(ne20); + local_size[1] = 4; + + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // create image for src1 + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; + buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + + // launch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + // deallocate sub buffers and images + CL_CHECK(clReleaseMemObject(src1_sub_buffer)); + CL_CHECK(clReleaseMemObject(buf_src1_image)); + CL_CHECK(clReleaseMemObject(buf_src2)); + + } else { // for gemm + kernel = backend_ctx->kernel_gemm_moe_q6_k_f32_ns; + if (backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin) { + kernel = backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin; + } + + // Reorder router if called from test-backend-ops or when new router is generated. + // Otherwise reuse the reordered result from previous mul_mat_id call. + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; + } + + cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; + cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; + cl_mem buf_src2, buf_src2_emap; + + // dp4a (int8) q6_K MoE prefill GEMM + static const char * q6k_moe_dp4a_env = getenv("GGML_OPENCL_Q6K_MOE_DP4A"); + bool use_moe_dp4a = (q6k_moe_dp4a_env != nullptr) + ? (atoi(q6k_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E + || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; + // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel + if (backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin == nullptr) { + use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin == nullptr; + } + + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Reorder activations + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + + if (!use_moe_dp4a) { + // Create image for reordered src1 + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + } + + // MoE kernel prepare + // Create sub buffer for dst + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer( + extrad->data_device, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + // Create image for dst + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); + + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + // fused reorder + q8_1 quant from the original activations + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + cl_kernel dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a; + if (backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin) { + dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin; + } + + int qi = 0; + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; + } + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); + + // set thread grid + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; + + // Dispatch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + } + return; + } +#endif //GGML_OPENCL_USE_ADRENO_KERNELS + } case GGML_TYPE_MXFP4: { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (use_adreno_moe_kernels(backend_ctx, src0)) { @@ -12373,11 +29667,19 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, size_t local_size[3] = {64, 2, 1}; size_t global_size[3] = {64, 2, 1}; - cl_mem src1_sub_buffer, buf_src1_image, buf_src2; - - int tile_size = 320; if (ne12 == 1) { // for gemv - kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32; + kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns; + + // Weight-as-texture MoE decode GEMV (see q4_K _wimg) + static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG"); + const bool use_moe_decode_wimg = (moe_decode_wimg_env && (atoi(moe_decode_wimg_env) != 0)) + && backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg != nullptr + && extra0_mxfp4->q_img != nullptr; + if (use_moe_decode_wimg) { + kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg; + } + + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 cl_buffer_region region; @@ -12387,82 +29689,275 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, CL_CHECK(status); // set thread grid - global_size[0] = static_cast(ne01); + global_size[0] = static_cast(((ne01 + 63) / 64) * 64); global_size[1] = 4; global_size[2] = static_cast(ne20); local_size[1] = 4; + + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // create image for src1 + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; + buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_mxfp4->q_img : &extra0_mxfp4->q)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->e)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); + + // launch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + // deallocate sub buffers and images + CL_CHECK(clReleaseMemObject(src1_sub_buffer)); + CL_CHECK(clReleaseMemObject(buf_src1_image)); + CL_CHECK(clReleaseMemObject(buf_src2)); + } else { // for gemm - kernel = backend_ctx->kernel_gemm_moe_mxfp4_f32; - - // preprocess router table - int num_tiles_per_expert = (ne01 + tile_size - 1) / tile_size; - void * host_src2_reorder = malloc(ne20 * ne21 * 4 * num_tiles_per_expert * sizeof(short)); - void * host_src2 = malloc(ne21 * nb21); - CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, extra2->data_device, CL_TRUE, offset2, ne21 * nb21, host_src2, 0, NULL, NULL)); - int total_experts = nb21 / nb20; - int out_idx = 0; - for (int i_expert = 0; i_expert < ne02; i_expert++) { - for (int i_tile = 0; i_tile < num_tiles_per_expert; i_tile++) { - for (int j = 0; j < ne21; j++) { - for (int i = 0; i < ne20; i++) { - int expert = ((int *)host_src2)[j * total_experts + i]; - if (i_expert == expert) { - ((short *)host_src2_reorder)[out_idx] = static_cast(expert); - ((short *)host_src2_reorder)[out_idx + 1] = static_cast(j * ne11 + (i % ne11)); - ((short *)host_src2_reorder)[out_idx + 2] = static_cast(j * ne20 + i); - ((short *)host_src2_reorder)[out_idx + 3] = static_cast(i_tile); - out_idx += 4; - } - } - } - } + kernel = backend_ctx->kernel_gemm_moe_mxfp4_f32_ns; + if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) { + kernel = backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin; + } + + // Reorder router if called from test-backend-ops or when new router is generated. + // Otherwise reuse the reordered result from previous mul_mat_id call. + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; } - buf_src2 = clCreateBuffer(backend_ctx->context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR, ne20 * ne21 * 4 * num_tiles_per_expert * sizeof(short), host_src2_reorder, &status); + + cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; + cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; + cl_mem buf_src2, buf_src2_emap; + + // dp4a (int8) prefill GEMM variant + static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A"); + bool use_moe_dp4a = mxfp4_moe_dp4a_env + ? (atoi(mxfp4_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); + // dot prod has to be available + use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; + // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel + // ldck dp4a bin kernel (X1) has the same priority as normal bin kernel (X2/A8x), they are not expected to coexist. + const bool use_moe_dp4a_ldck = backend_ctx->has_integer_dot && + (!mxfp4_moe_dp4a_env || atoi(mxfp4_moe_dp4a_env) != 0) && + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr && + backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr && + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin != nullptr; + if (use_moe_dp4a_ldck) { + use_moe_dp4a = true; + } else if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) { + use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr; + } + + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + GGML_ASSERT(backend_ctx->prealloc_post_router.buffer); + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); - // set thread grid - global_size[0] = static_cast(tile_size); - global_size[2] = static_cast(ne20 * ne21 * num_tiles_per_expert); - } + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - // create a sub_buffer for src1 - cl_buffer_region region; - region.origin = offset1; - region.size = ne10 * ne11 * ne12 * sizeof(float); - src1_sub_buffer = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); - CL_CHECK(status); + // Reorder activations + // create a sub_buffer for src1 + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); - // create image for src1 - cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; - cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne10 * ne11 * ne12 / 4), 0,0,0,0,0,0,0, {src1_sub_buffer}}; - buf_src1_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); - CL_CHECK(status); + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + + if (!use_moe_dp4a) { + // Create image for reordered src1 + // Use pre-allocated placeholder + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1; + cl_image_desc image_desc_buf_src1; + image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + } - // Set kernel args - int arg_idx = 0; - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q)); - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->e)); - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extrad->data_device)); - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_ulong), &offsetd)); - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); - if (ne12 == 1) { - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne11)); - } else { - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &tile_size)); - } + // MoE kernel prepare + // Create sub buffer for dst + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer( + extrad->data_device, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + // Create image for dst + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); - // launch kernel - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + if (use_moe_dp4a_ldck) { + backend_ctx->prealloc_moe_qa_trans.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da_trans.allocate(backend_ctx->context, n_blocks * sizeof(cl_float)); + // matches max_constant_size of src1_a_cm in the ldck kernel + backend_ctx->prealloc_moe_qda_on_chip.allocate(backend_ctx->context, 2304); + } - // deallocate sub buffers and images - CL_CHECK(clReleaseMemObject(src1_sub_buffer)); - CL_CHECK(clReleaseMemObject(buf_src1_image)); - CL_CHECK(clReleaseMemObject(buf_src2)); + // fused reorder + q8_1 quant straight from the original + // activations (no intermediate f32 reorder buffer). mxfp4 has no + // min term so the GEMM ignores sa, but reorder_quant still writes it. + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + // dp4a GEMM + cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a; + if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin) { + dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin; + } else if (use_moe_dp4a_ldck) { + dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin; + + // per-tile transpose of qa/da for the ldck kernel + cl_kernel tk = backend_ctx->kernel_transpose_activation_q8; + CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa_trans.buffer)); + CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da_trans.buffer)); + CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(tk, 5, sizeof(cl_uint), &ne00)); + size_t tk_local[3] = { 32, 8, 1 }; + size_t tk_global[3] = { 32, (size_t)(ne00 / 4), (size_t)max_post_router_tile }; + backend_ctx->enqueue_ndrange_kernel(tk, 3, tk_global, tk_local, dst); + } + + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e)); + if (use_moe_dp4a_ldck) { + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa_trans.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da_trans.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qda_on_chip.buffer)); + } else { + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + } + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; + } + + // Set kernel args + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q_img)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->e)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_use_moe_ragged)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_uint), &backend_ctx->adreno_moe_ragged_skip_gran)); + + // set thread grid + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; + + // Dispatch kernel + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + } return; - } // else fallback to generic kernel + } // fallback to generic MoE mxfp4 kernel #endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_SOA_Q @@ -12642,16 +30137,53 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; + // A contiguous f32 -> f32 copy is a linear move. The kernel below maps one workgroup to + // each row, so a tensor with few long rows runs on a single compute unit; dispatch those + // over the whole device instead. GGML_OPENCL_CPY_FLAT=0 restores the row-mapped path. + static const bool cpy_flat_on = []{ + const char * e = getenv("GGML_OPENCL_CPY_FLAT"); + return !(e && e[0] == '0'); + }(); + if (cpy_flat_on && backend_ctx->kernel_cpy_f32_f32_flat != nullptr && + src0t == GGML_TYPE_F32 && src1t == GGML_TYPE_F32 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && + ggml_nelements(src0) == ggml_nelements(src1)) { + cl_kernel k = backend_ctx->kernel_cpy_f32_f32_flat; + const cl_ulong nelem = (cl_ulong) ggml_nelements(src0); + const cl_ulong n4 = nelem / 4; + + CL_CHECK(clSetKernelArg(k, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(k, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(k, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(k, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(k, 4, sizeof(cl_ulong), &nelem)); + CL_CHECK(clSetKernelArg(k, 5, sizeof(cl_ulong), &n4)); + + // one work item per float4, plus one for the trailing scalars + const size_t items = (size_t) n4 + ((nelem % 4) ? 1 : 0); + const size_t lsz = MIN((size_t) 64, backend_ctx->max_workgroup_size); + size_t global_work_size[] = { ((items + lsz - 1) / lsz) * lsz, 1, 1 }; + size_t local_work_size[] = { lsz, 1, 1 }; + + backend_ctx->enqueue_ndrange_kernel(k, 1, global_work_size, local_work_size, src1); + return; + } + cl_kernel kernel; switch (src0t) { case GGML_TYPE_F32: switch (src1t) { + case GGML_TYPE_I32: + kernel = backend_ctx->kernel_cpy_f32_i32; + break; case GGML_TYPE_F16: kernel = backend_ctx->kernel_cpy_f32_f16; break; case GGML_TYPE_F32: - kernel = backend_ctx->kernel_cpy_f32_f32; + kernel = (ne00 < 32 && backend_ctx->kernel_cpy_f32_f32_pack) + ? backend_ctx->kernel_cpy_f32_f32_pack + : backend_ctx->kernel_cpy_f32_f32; break; default: GGML_ASSERT(false && "not implemented"); @@ -12671,6 +30203,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const break; case GGML_TYPE_I32: switch (src1t) { + case GGML_TYPE_F32: + kernel = backend_ctx->kernel_cpy_i32_f32; + break; case GGML_TYPE_I32: kernel = backend_ctx->kernel_cpy_i32_i32; break; @@ -12703,12 +30238,27 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb12)); CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb13)); - const int nth = MIN(64, ne00); + if (kernel == backend_ctx->kernel_cpy_f32_f32_pack) { + const int maxwg = (int)backend_ctx->get_kernel_workgroup_size(kernel); + const int base = MIN(64, maxwg); + const int tpr = MIN(ne00, base); // threads per row + const int rpw = MAX(1, base / tpr); // rows per workgroup + const int lsz = tpr * rpw; // <= base <= maxwg + const int nrows = ne01*ne02*ne03; + const int nwg = (nrows + rpw - 1) / rpw; - size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; - size_t local_work_size[] = {(size_t)nth, 1, 1}; + size_t global_work_size[] = {(size_t)nwg*lsz, 1, 1}; + size_t local_work_size[] = {(size_t)lsz, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 1, global_work_size, local_work_size, src1); + } else { + const int nth = MIN(64, ne00); + + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src1); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, src1); + } } static void ggml_cl_dup(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -13089,6 +30639,7 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const const int n_dims = ((int *) dst->op_params)[1]; const int mode = ((int *) dst->op_params)[2]; const int n_ctx_orig = ((int32_t *) dst->op_params)[4]; + const int n_offs = ((int32_t *) dst->op_params)[15]; float freq_base; float freq_scale; @@ -13117,6 +30668,7 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const if (is_vision) { GGML_ASSERT(n_dims == ne00/2); + GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row } cl_kernel kernel; @@ -13208,6 +30760,12 @@ static void ggml_cl_rope(ggml_backend_t backend, const ggml_tensor * src0, const if (is_mrope && !is_vision) { CL_CHECK(clSetKernelArg(kernel, 34, sizeof(int), &is_imrope)); } + // norm and neox have n_offs after beta_slow, mrope has it after is_imrope + if (!is_mrope && !is_vision) { + CL_CHECK(clSetKernelArg(kernel, 33, sizeof(int), &n_offs)); + } else if (is_mrope && !is_vision) { + CL_CHECK(clSetKernelArg(kernel, 35, sizeof(int), &n_offs)); + } size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; size_t local_work_size[] = {(size_t)nth, 1, 1}; @@ -13364,6 +30922,287 @@ static void ggml_cl_im2col(ggml_backend_t backend, const ggml_tensor * src0, con backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } +static void ggml_cl_im2col_3d(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src1); + GGML_ASSERT(src1->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int32_t s0 = ((const int32_t*)(dst->op_params))[0]; + const int32_t s1 = ((const int32_t*)(dst->op_params))[1]; + const int32_t s2 = ((const int32_t*)(dst->op_params))[2]; + const int32_t p0 = ((const int32_t*)(dst->op_params))[3]; + const int32_t p1 = ((const int32_t*)(dst->op_params))[4]; + const int32_t p2 = ((const int32_t*)(dst->op_params))[5]; + const int32_t d0 = ((const int32_t*)(dst->op_params))[6]; + const int32_t d1 = ((const int32_t*)(dst->op_params))[7]; + const int32_t d2 = ((const int32_t*)(dst->op_params))[8]; + const cl_long IC = ((const int32_t*)(dst->op_params))[9]; + + const cl_long N = src1->ne[3] / IC; + const cl_long ID = src1->ne[2]; + const cl_long IH = src1->ne[1]; + const cl_long IW = src1->ne[0]; + + const cl_long KD = src0->ne[2]; + const cl_long KH = src0->ne[1]; + const cl_long KW = src0->ne[0]; + + const cl_long OD = dst->ne[3] / N; + const cl_long OH = dst->ne[2]; + const cl_long OW = dst->ne[1]; + + const size_t es = ggml_element_size(src1); + const cl_long stride_x = src1->nb[0] / es; + const cl_long stride_y = src1->nb[1] / es; + const cl_long stride_z = src1->nb[2] / es; + const cl_long stride_q = src1->nb[3] / es; + + const cl_long IC_KD_KH_KW = IC * KD * KH * KW; + + cl_kernel kernel; + + if (dst->type == GGML_TYPE_F16) { + kernel = backend_ctx->kernel_im2col_3d_f16; + } else { + kernel = backend_ctx->kernel_im2col_3d_f32; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_long), &IC)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_long), &ID)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_long), &IH)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_long), &IW)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_long), &KD)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_long), &KH)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_long), &KW)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_long), &OD)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_long), &OH)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_long), &OW)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_long), &N)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_long), &stride_x)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_long), &stride_y)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_long), &stride_z)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_long), &stride_q)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &s0)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &s1)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &s2)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &p0)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &p1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(int), &p2)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &d0)); + CL_CHECK(clSetKernelArg(kernel, 26, sizeof(int), &d1)); + CL_CHECK(clSetKernelArg(kernel, 27, sizeof(int), &d2)); + + const size_t local_col = 64; + const size_t num_blocks = (IC_KD_KH_KW + local_col - 1) / local_col; + size_t global_work_size[] = { num_blocks * local_col, (size_t)OW, (size_t)(N * OD * OH) }; + size_t local_work_size[] = { local_col, 1, 1 }; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_argmax(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + GGML_UNUSED(src1); + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_I32); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(dst)); + + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + const int ne00 = src0->ne[0]; + const int nrows = ggml_nrows(src0); + constexpr int nth = 64; + + cl_kernel kernel = backend_ctx->kernel_argmax_f32_i32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &nrows)); + + size_t global_work_size[] = { (size_t) nth, (size_t) CEIL_DIV(nrows, 2), 1 }; + size_t local_work_size[] = { (size_t) nth, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_lightning_indexer(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + cl_kernel kernel = backend_ctx->kernel_lightning_indexer; + const ggml_tensor * src2 = dst->src[2]; + const ggml_tensor * src3 = dst->src[3]; + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *) src1->extra; + ggml_tensor_extra_cl * extra2 = (ggml_tensor_extra_cl *) src2->extra; + ggml_tensor_extra_cl * extra3 = (ggml_tensor_extra_cl *) src3->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offset1 = extra1->offset + src1->view_offs; + const cl_ulong offset2 = extra2->offset + src2->view_offs; + const cl_ulong offset3 = extra3->offset + src3->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int head_size = src0->ne[0]; + const int n_heads = src0->ne[1]; + const int n_tokens = src0->ne[2]; + const int n_keys = src1->ne[2]; + const int n_masks = src3->ne[3]; + // Keep the local-memory reduction for small grids or few heads. + const int use_shuffle = head_size == 128 && backend_ctx->has_khr_subgroup_shuffle && n_heads >= 32 && ggml_nelements(dst) >= 1024; + // BF16 tensors are stored as F16 on the device. + const int type_k = src1->type == GGML_TYPE_F32 ? 0 : 1; + + const cl_ulong nbq1 = src0->nb[1]; + const cl_ulong nbq2 = src0->nb[2]; + const cl_ulong nbq3 = src0->nb[3]; + const cl_ulong nbk2 = src1->nb[2]; + const cl_ulong nbk3 = src1->nb[3]; + const cl_ulong nbw1 = src2->nb[1]; + const cl_ulong nbw3 = src2->nb[3]; + const cl_ulong nbm1 = src3->nb[1]; + const cl_ulong nbm3 = src3->nb[3]; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra3->data_device)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset3)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &head_size)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &n_heads)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &n_tokens)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &n_keys)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &n_masks)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &type_k)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nbq1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nbq2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nbq3)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nbk2)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(cl_ulong), &nbk3)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(cl_ulong), &nbw1)); + CL_CHECK(clSetKernelArg(kernel, 22, sizeof(cl_ulong), &nbw3)); + CL_CHECK(clSetKernelArg(kernel, 23, sizeof(cl_ulong), &nbm1)); + CL_CHECK(clSetKernelArg(kernel, 24, sizeof(cl_ulong), &nbm3)); + CL_CHECK(clSetKernelArg(kernel, 25, sizeof(int), &use_shuffle)); + + const int keys_per_group = use_shuffle ? 16 : 4; + size_t global_work_size[] = {(size_t) ((src1->ne[2] + keys_per_group - 1)/keys_per_group)*64, (size_t) src0->ne[2], (size_t) src0->ne[3]}; + size_t local_work_size[] = {64, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + +static void ggml_cl_top_k(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_UNUSED(src1); + + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *) src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong offset0 = extra0->offset + src0->view_offs; + const cl_ulong offsetd = extrad->offset + dst->view_offs; + + GGML_TENSOR_LOCALS(int, ne0, src0, ne); + GGML_TENSOR_LOCALS(cl_ulong, nb0, src0, nb); + + const int k = dst->ne[0]; + cl_kernel kernel = backend_ctx->kernel_top_k_f32; + const int max_wg = backend_ctx->get_kernel_workgroup_size(kernel); + + int nth = 1; + const int target = MIN(512, MAX(32, ne00/4)); + while (2*nth <= MIN(target, max_wg)) { + nth *= 2; + } + + const size_t n_rows = ggml_nrows(src0); + const bool tiled = ne00 > 32768 && k > 0 && k <= 256 && n_rows <= 4; + const int tile_width = tiled ? 4096 : ne00; + const int n_tiles = tiled ? 1 + (ne00 - 1)/tile_width : 1; + const int n_candidates = (n_tiles - 1)*k + MIN(k, ne00 - (n_tiles - 1)*tile_width); + cl_mem candidates = nullptr; + if (tiled) { + GGML_ASSERT(n_rows <= SIZE_MAX/sizeof(cl_uint2)/(size_t) n_candidates); + backend_ctx->prealloc_top_k.allocate(backend_ctx->context, n_rows*(size_t) n_candidates*sizeof(cl_uint2)); + candidates = backend_ctx->prealloc_top_k.buffer; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 12, MAX(259, 2*nth)*sizeof(cl_uint), nullptr)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_mem), &candidates)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &tile_width)); + + size_t global_work_size[] = {(size_t) nth*n_tiles, n_rows, 1}; + size_t local_work_size[] = {(size_t) nth, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + + if (tiled) { + // The global top-k is contained in the top-k of each tile. + kernel = backend_ctx->kernel_top_k_merge_f32; + const int max_wg_merge = backend_ctx->get_kernel_workgroup_size(kernel); + + int nth_merge = 1; + const int target_merge = MIN(512, MAX(32, n_candidates/4)); + while (2*nth_merge <= MIN(target_merge, max_wg_merge)) { + nth_merge *= 2; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &candidates)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &n_candidates)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 5, MAX(259, 2*nth_merge)*sizeof(cl_uint), nullptr)); + + global_work_size[0] = nth_merge; + local_work_size[0] = nth_merge; + + backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst); + } +} + static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -13408,6 +31247,13 @@ static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, co size_t local_work_size[] = {(size_t)ne00_padded, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + const int ne21 = dst->ne[1]; + if ((strstr(src0->name, "_moe") != NULL) && (ne21 != 1)) { + backend_ctx->toggle_reorder = true; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS } static void ggml_cl_sum_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -13624,6 +31470,13 @@ static void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, const case GGML_GLU_OP_SWIGLU_OAI: kernel = backend_ctx->kernel_swiglu_oai; break; + case GGML_GLU_OP_SWIGLU_CLAMP: + if (dst->type == GGML_TYPE_F32) { + kernel = backend_ctx->kernel_swiglu_clamp; + } else { + kernel = backend_ctx->kernel_swiglu_clamp_f16; + } + break; case GGML_GLU_OP_GEGLU_ERF: if (dst->type == GGML_TYPE_F32) { kernel = backend_ctx->kernel_geglu_erf; @@ -13679,19 +31532,200 @@ static void ggml_cl_glu(ggml_backend_t backend, const ggml_tensor * src0, const CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00_off)); CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10_off)); - if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI) { + if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI || ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_CLAMP) { CL_CHECK(clSetKernelArg(kernel, 12, sizeof(float), &limit)); + } + if (ggml_get_glu_op(dst) == GGML_GLU_OP_SWIGLU_OAI) { CL_CHECK(clSetKernelArg(kernel, 13, sizeof(float), &alpha)); } const size_t nrows = ggml_nrows(src0); - size_t nth = 512; + size_t nth = backend_ctx->max_workgroup_size < 512 ? backend_ctx->max_workgroup_size : 512; size_t global_work_size[] = {nrows*nth, 1, 1}; size_t local_work_size[] = {nth, 1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } +static void ggml_cl_gated_delta_net(ggml_backend_t backend, ggml_tensor * dst) { + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + const ggml_tensor * src_q = dst->src[0]; + const ggml_tensor * src_k = dst->src[1]; + const ggml_tensor * src_v = dst->src[2]; + const ggml_tensor * src_g = dst->src[3]; + const ggml_tensor * src_beta = dst->src[4]; + const ggml_tensor * src_state = dst->src[5]; + + GGML_ASSERT(src_q && src_q->extra); + GGML_ASSERT(src_k && src_k->extra); + GGML_ASSERT(src_v && src_v->extra); + GGML_ASSERT(src_g && src_g->extra); + GGML_ASSERT(src_beta && src_beta->extra); + GGML_ASSERT(src_state && src_state->extra); + + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *) backend->context; + + const cl_uint S_v = (cl_uint) src_v->ne[0]; + const cl_uint H_v = (cl_uint) src_v->ne[1]; + const cl_uint n_tokens = (cl_uint) src_v->ne[2]; + const cl_uint n_seqs = (cl_uint) src_v->ne[3]; + const cl_uint K = (cl_uint) ggml_get_op_params_i32(dst, 0); + + int si; + switch (S_v) { + case 16: si = 0; break; + case 32: si = 1; break; + case 64: si = 2; break; + case 128: si = 3; break; + default: + GGML_ASSERT(false && "ggml_cl_gated_delta_net: unsupported S_v"); + } + + const int kda = (src_g->ne[0] == (int64_t) S_v) ? 1 : 0; + + // TODO: Optimize when S_v!=128. Not necessary for now as Qwen3.5/6 are all S_v=128 + // token generation mode (tgpp=0): + // process 1 token at a time, so columns per lane (cpl) == 1 + // prompt processing mode (tgpp=1): + // cpl=4 to process 4 tokens for single-token. 4 is chosen for Adreno 750 as per + // work-item/thread has at most 128 registers. + // All Qwen3.5/6 models are S_v == 128, so LANES_PER_COLUMN == 8 + // such that ROWS_PER_LANE = 128/8 = 16 + // Variables in the kernel: + // k_reg, q_reg, g_exp are all 16 floats + // s_shard has cpl*ROWS_PER_LANE = 4*16 = 64 floats + // Total 112 registers used. + // subgroups_per_workgroup (spw) can be set to 1,2,4,8,16 for tg and 1,2,4 for pp + // for S_v=128. + // Empirically found that when spw=1, we get the best performance for both tg and pp + const int tgpp = (n_tokens == 1) ? 0 : 1; + const int cpl = (tgpp == 0) ? 1 : 4; + // spw needs adjustment when S_v != 128 + const int spw = (tgpp == 0) ? 1 : 1; + + cl_kernel kernel = backend_ctx->kernel_gated_delta_net_f32[si][kda][tgpp]; + GGML_ASSERT(kernel != nullptr); + + const cl_uint s_off = S_v * H_v * n_tokens * n_seqs; + + const cl_uint sq1 = (cl_uint)(src_q->nb[1] / sizeof(float)); + const cl_uint sq2 = (cl_uint)(src_q->nb[2] / sizeof(float)); + const cl_uint sq3 = (cl_uint)(src_q->nb[3] / sizeof(float)); + const cl_uint sv1 = (cl_uint)(src_v->nb[1] / sizeof(float)); + const cl_uint sv2 = (cl_uint)(src_v->nb[2] / sizeof(float)); + const cl_uint sv3 = (cl_uint)(src_v->nb[3] / sizeof(float)); + const cl_uint sb1 = (cl_uint)(src_beta->nb[1] / sizeof(float)); + const cl_uint sb2 = (cl_uint)(src_beta->nb[2] / sizeof(float)); + const cl_uint sb3 = (cl_uint)(src_beta->nb[3] / sizeof(float)); + + const cl_uint H_k = (cl_uint) src_q->ne[1]; + const cl_uint rq3 = (cl_uint)(src_v->ne[3] / src_q->ne[3]); + + const float scale = 1.0f / sqrtf((float) S_v); + + ggml_tensor_extra_cl * extra_q = (ggml_tensor_extra_cl *) src_q->extra; + ggml_tensor_extra_cl * extra_k = (ggml_tensor_extra_cl *) src_k->extra; + ggml_tensor_extra_cl * extra_v = (ggml_tensor_extra_cl *) src_v->extra; + ggml_tensor_extra_cl * extra_g = (ggml_tensor_extra_cl *) src_g->extra; + ggml_tensor_extra_cl * extra_beta = (ggml_tensor_extra_cl *) src_beta->extra; + ggml_tensor_extra_cl * extra_state = (ggml_tensor_extra_cl *) src_state->extra; + ggml_tensor_extra_cl * extra_dst = (ggml_tensor_extra_cl *) dst->extra; + + const cl_ulong off_q = extra_q->offset + src_q->view_offs; + const cl_ulong off_k = extra_k->offset + src_k->view_offs; + const cl_ulong off_v = extra_v->offset + src_v->view_offs; + const cl_ulong off_g = extra_g->offset + src_g->view_offs; + const cl_ulong off_beta = extra_beta->offset + src_beta->view_offs; + const cl_ulong off_state = extra_state->offset + src_state->view_offs; + const cl_ulong off_dst = extra_dst->offset + dst->view_offs; + + int idx = 0; + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_q->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_q)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_k->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_k)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_v->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_v)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_g->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_g)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_beta->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_beta)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_state->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_state)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_mem), &extra_dst->data_device)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_ulong), &off_dst)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H_v)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &n_tokens)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &n_seqs)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &s_off)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sq1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sq2)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sq3)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sv1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sv2)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sv3)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sb1)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sb2)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &sb3)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &H_k)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &rq3)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, idx++, sizeof(cl_uint), &K)); + + // Subgroup size is 64 for Adreno and 32 for Intel + const int sg_size = backend_ctx->gpu_family == GPU_FAMILY::ADRENO ? 64 : backend_ctx->gpu_family == GPU_FAMILY::INTEL ? 32 : -1; + if (sg_size < 0) { + GGML_LOG_ERROR("Unsupported GPU Family: only Adreno and Intel are supported.\n"); + exit(1); + } + + // For the subgroup-shuffle kernel, we can safely prefer 8 lanes/column for S_v>=128 + // For the subgroup-shuffle kernel: + // S_v >= 128 -> prefer 8 lanes/column (good occupancy & register pressure tradeoff) + // else -> min(S_v, subgroup_size) + int lanes_per_column; + if ((int)S_v >= 128) { + lanes_per_column = 8; + } else { + lanes_per_column = std::min((int)S_v, sg_size); + } + + // Max workgroup size for Adreno 750 is 1024 + const int wg_size = sg_size * spw; + + // Ensure lanes_per_column is a power-of-two and divides both S_v and subgroup_size. + // (Required for lane-group shuffle-xor reduction correctness.) + while (lanes_per_column > 1 && + (((lanes_per_column & (lanes_per_column - 1)) != 0) || + (((int)S_v % lanes_per_column) != 0) || + (sg_size % lanes_per_column) != 0)) { + lanes_per_column >>= 1; + } + GGML_ASSERT(lanes_per_column >= 1); + GGML_ASSERT(((lanes_per_column & (lanes_per_column - 1)) == 0)); + GGML_ASSERT(((int)S_v % lanes_per_column) == 0); + GGML_ASSERT((sg_size % lanes_per_column) == 0); + + const int cols_per_wg = spw * (sg_size / lanes_per_column) * cpl; + GGML_ASSERT(cols_per_wg > 0); + GGML_ASSERT(((int)S_v % cols_per_wg) == 0); + + size_t global_work_size[3]; + size_t local_work_size[3]; + + global_work_size[0] = (size_t) H_v * (size_t) wg_size; + global_work_size[1] = (size_t) n_seqs; + global_work_size[2] = (size_t) S_v / (size_t) cols_per_wg; + + local_work_size[0] = (size_t) wg_size; + local_work_size[1] = 1; + local_work_size[2] = 1; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + //------------------------------------------------------------------------------ // Op offloading //------------------------------------------------------------------------------ @@ -13850,6 +31884,48 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_expm1; break; + case GGML_UNARY_OP_ABS: + if (!any_on_device) { + return false; + } + func = ggml_cl_abs; + break; + case GGML_UNARY_OP_SGN: + if (!any_on_device) { return false; } + func = ggml_cl_sgn; + break; + case GGML_UNARY_OP_STEP: + if (!any_on_device) { return false; } + func = ggml_cl_step; + break; + case GGML_UNARY_OP_ELU: + if (!any_on_device) { return false; } + func = ggml_cl_elu; + break; + case GGML_UNARY_OP_HARDSWISH: + if (!any_on_device) { return false; } + func = ggml_cl_hardswish; + break; + case GGML_UNARY_OP_HARDSIGMOID: + if (!any_on_device) { return false; } + func = ggml_cl_hardsigmoid; + break; + case GGML_UNARY_OP_FLOOR: + if (!any_on_device) { return false; } + func = ggml_cl_floor; + break; + case GGML_UNARY_OP_CEIL: + if (!any_on_device) { return false; } + func = ggml_cl_ceil; + break; + case GGML_UNARY_OP_ROUND: + if (!any_on_device) { return false; } + func = ggml_cl_round; + break; + case GGML_UNARY_OP_TRUNC: + if (!any_on_device) { return false; } + func = ggml_cl_trunc; + break; case GGML_UNARY_OP_SOFTPLUS: if (!any_on_device) { return false; @@ -13907,8 +31983,8 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_group_norm; break; - case GGML_OP_REPEAT: - if (!any_on_device) { + case GGML_OP_REPEAT: + if (!any_on_device) { return false; } func = ggml_cl_repeat; @@ -13931,12 +32007,34 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_conv_2d; break; + case GGML_OP_CONV_2D_DW: + if (!any_on_device) { + return false; + } + func = ggml_cl_conv2_dw; + break; case GGML_OP_SSM_CONV: if (!any_on_device) { return false; } func = ggml_cl_ssm_conv; break; + case GGML_OP_SSM_SCAN: + if (!any_on_device) { + return false; + } + // SSM_SCAN has 7 source tensors, so it cannot use the standard + // (src0, src1, dst) func signature. Dispatch directly and return. + ggml_cl_ssm_scan(backend, tensor); + return true; + case GGML_OP_GATED_DELTA_NET: + if (!any_on_device) { + return false; + } + // GDN has 6 source tensors, so it cannot use the standard + // (src0, src1, dst) func signature. Dispatch directly and return. + ggml_cl_gated_delta_net(backend, tensor); + return true; case GGML_OP_CONCAT: if (!any_on_device) { return false; @@ -14012,6 +32110,24 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_im2col; break; + case GGML_OP_IM2COL_3D: + if (!any_on_device) { + return false; + } + func = ggml_cl_im2col_3d; + break; + case GGML_OP_ARGMAX: + if (!any_on_device) { + return false; + } + func = ggml_cl_argmax; + break; + case GGML_OP_LIGHTNING_INDEXER: + func = ggml_cl_lightning_indexer; + break; + case GGML_OP_TOP_K: + func = ggml_cl_top_k; + break; case GGML_OP_ARGSORT: if (!any_on_device) { return false; diff --git a/ggml/src/ggml-opencl/kernels/abs.cl b/ggml/src/ggml-opencl/kernels/abs.cl new file mode 100644 index 0000000..96e952c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/abs.cl @@ -0,0 +1,113 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +//------------------------------------------------------------------------------ +// abs +//------------------------------------------------------------------------------ + +kernel void kernel_abs_f32( + global const float * src0, + ulong offset0, + global float * dst, + ulong offsetd +) { + src0 = (global float*)((global char*)src0 + offset0); + dst = (global float*)((global char*)dst + offsetd); + + dst[get_global_id(0)] = fabs(src0[get_global_id(0)]); +} + +kernel void kernel_abs_f32_4( + global const float4 * src0, + ulong offset0, + global float4 * dst, + ulong offsetd +) { + src0 = (global float4*)((global char*)src0 + offset0); + dst = (global float4*)((global char*)dst + offsetd); + + dst[get_global_id(0)] = fabs(src0[get_global_id(0)]); +} + +kernel void kernel_abs_f16( + global const half * src0, + ulong offset0, + global half * dst, + ulong offsetd +) { + src0 = (global half*)((global char*)src0 + offset0); + dst = (global half*)((global char*)dst + offsetd); + + dst[get_global_id(0)] = fabs(src0[get_global_id(0)]); +} + +kernel void kernel_abs_f16_4( + global const half4 * src0, + ulong offset0, + global half4 * dst, + ulong offsetd +) { + src0 = (global half4*)((global char*)src0 + offset0); + dst = (global half4*)((global char*)dst + offsetd); + + dst[get_global_id(0)] = fabs(src0[get_global_id(0)]); +} + +kernel void kernel_abs_f32_nc( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + int ne00, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + dst = dst + offsetd; + + const int i3 = get_group_id(2); + const int i2 = get_group_id(1); + const int i1 = get_group_id(0); + + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { + global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); + global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + *y = fabs(*x); + } +} + +kernel void kernel_abs_f16_nc( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + int ne00, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + dst = dst + offsetd; + + const int i3 = get_group_id(2); + const int i2 = get_group_id(1); + const int i1 = get_group_id(0); + + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { + global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); + global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + *y = fabs(*x); + } +} diff --git a/ggml/src/ggml-opencl/kernels/add_id.cl b/ggml/src/ggml-opencl/kernels/add_id.cl index e9c6d55..e9aa5a1 100644 --- a/ggml/src/ggml-opencl/kernels/add_id.cl +++ b/ggml/src/ggml-opencl/kernels/add_id.cl @@ -34,6 +34,14 @@ kernel void kernel_add_id( global float * dst_row = (global float *)((global char *)dst + i1*nb1 + i2*nb2); global float * src0_row = (global float *)((global char *)src0 + i1*nb01 + i2*nb02); + + if (i11 == -1) { // skipped slot, add nothing + for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { + dst_row[i0] = src0_row[i0]; + } + return; + } + global float * src1_row = (global float *)((global char *)src1 + i11*nb11); for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { diff --git a/ggml/src/ggml-opencl/kernels/argmax.cl b/ggml/src/ggml-opencl/kernels/argmax.cl new file mode 100644 index 0000000..24ed0e6 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/argmax.cl @@ -0,0 +1,55 @@ +#define ARGMAX_LANES_PER_ROW 32 +#define ARGMAX_ROWS_PER_GROUP 2 +#define ARGMAX_BLOCK_SIZE (ARGMAX_LANES_PER_ROW * ARGMAX_ROWS_PER_GROUP) + +kernel void kernel_argmax_f32_i32( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + int ne00, + int nrows) { + const int local_id = get_local_id(0); + const int col = local_id % ARGMAX_LANES_PER_ROW; + const int row = get_group_id(1) * ARGMAX_ROWS_PER_GROUP + local_id / ARGMAX_LANES_PER_ROW; + + float max_value = -INFINITY; + int max_index = 0; + + if (row < nrows) { + global const float * src_row = (global const float *) (src0 + offset0) + row * ne00; + for (int i = col; i < ne00; i += ARGMAX_LANES_PER_ROW) { + const float value = src_row[i]; + if (value > max_value || (value == max_value && i > max_index)) { + max_value = value; + max_index = i; + } + } + } + + local float max_values[ARGMAX_BLOCK_SIZE]; + local int max_indices[ARGMAX_BLOCK_SIZE]; + + max_values[local_id] = max_value; + max_indices[local_id] = max_index; + + barrier(CLK_LOCAL_MEM_FENCE); + + const int row_offset = local_id / ARGMAX_LANES_PER_ROW * ARGMAX_LANES_PER_ROW; + for (int stride = ARGMAX_LANES_PER_ROW / 2; stride > 0; stride /= 2) { + if (col < stride) { + const float value = max_values[row_offset + col + stride]; + const int index = max_indices[row_offset + col + stride]; + if (value > max_values[row_offset + col] || (value == max_values[row_offset + col] && index > max_indices[row_offset + col])) { + max_values[row_offset + col] = value; + max_indices[row_offset + col] = index; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (col == 0 && row < nrows) { + global int * dst_row = (global int *) (dst + offsetd) + row; + dst_row[0] = max_indices[row_offset]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/clamp.cl b/ggml/src/ggml-opencl/kernels/clamp.cl index ae60324..366a068 100644 --- a/ggml/src/ggml-opencl/kernels/clamp.cl +++ b/ggml/src/ggml-opencl/kernels/clamp.cl @@ -18,3 +18,33 @@ kernel void kernel_clamp( min : (src0[get_global_id(0)] > max ? max : src0[get_global_id(0)]); } + +kernel void kernel_clamp_nc( + global const char * src0, + ulong offset0, + global char * dst, + ulong offsetd, + float min, + float max, + int ne00, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + dst = dst + offsetd; + + const int i3 = get_group_id(2); + const int i2 = get_group_id(1); + const int i1 = get_group_id(0); + + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { + const float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); + *(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = x < min ? min : (x > max ? max : x); + } +} diff --git a/ggml/src/ggml-opencl/kernels/concat.cl b/ggml/src/ggml-opencl/kernels/concat.cl index 0c1b3d7..8ecf746 100644 --- a/ggml/src/ggml-opencl/kernels/concat.cl +++ b/ggml/src/ggml-opencl/kernels/concat.cl @@ -1,4 +1,66 @@ -kernel void kernel_concat_f32( +// concat is a pure copy, so the kernels are keyed by element byte size +// (1/2/4/8) rather than logical type, matching the CUDA backend. + +#define KERNEL_CONCAT(SUFFIX, T) \ +kernel void kernel_concat_##SUFFIX( \ + global const char * src0, \ + ulong offset0, \ + global const char * src1, \ + ulong offset1, \ + global char * dst, \ + ulong offsetd, \ + int ne00, \ + int ne01, \ + int ne02, \ + int ne03, \ + ulong nb00, \ + ulong nb01, \ + ulong nb02, \ + ulong nb03, \ + ulong nb10, \ + ulong nb11, \ + ulong nb12, \ + ulong nb13, \ + int ne0, \ + ulong nb0, \ + ulong nb1, \ + ulong nb2, \ + ulong nb3, \ + int dim \ +) { \ + src0 = src0 + offset0; \ + src1 = src1 + offset1; \ + dst = dst + offsetd; \ + \ + const int i3 = get_group_id(2); \ + const int i2 = get_group_id(1); \ + const int i1 = get_group_id(0); \ + \ + int o[4] = {0, 0, 0, 0}; \ + o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03)); \ + \ + global const T * x; \ + \ + for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { \ + if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { \ + x = (global const T *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); \ + } else { \ + x = (global const T *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); \ + } \ + \ + global T * y = (global T *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); \ + \ + *y = *x; \ + } \ +} + +KERNEL_CONCAT(b1, char) +KERNEL_CONCAT(b2, short) +KERNEL_CONCAT(b4, int) +KERNEL_CONCAT(b8, long) + +// packed variant for the common dim==0, small-ne0 case (4-byte elements only). +kernel void kernel_concat_b4_pack( global const char * src0, ulong offset0, global const char * src1, @@ -22,29 +84,44 @@ kernel void kernel_concat_f32( ulong nb1, ulong nb2, ulong nb3, - int dim + int dim, + int ne1, + int ne2, + int ne3 ) { src0 = src0 + offset0; src1 = src1 + offset1; dst = dst + offsetd; - const int i3 = get_group_id(2); - const int i2 = get_group_id(1); - const int i1 = get_group_id(0); + int lsz = get_local_size(0); + int tpr = min(ne0, lsz); // threads per row + int rpw = lsz / tpr; // rows per workgroup + int lid = get_local_id(0); + int row = get_group_id(0)*rpw + lid / tpr; + int lane = lid - (lid / tpr) * tpr; + + int nrows = ne1*ne2*ne3; + if (row >= nrows) { + return; + } + + int i1 = row % ne1; + int t = row / ne1; + int i2 = t % ne2; + int i3 = t / ne2; int o[4] = {0, 0, 0, 0}; o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03)); - global const float * x; - - for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { + for (int i0 = lane; i0 < ne0; i0 += tpr) { + global const int * x; if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); + x = (global const int *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); } else { - x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); + x = (global const int *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); } - global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + global int * y = (global int *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); *y = *x; } diff --git a/ggml/src/ggml-opencl/kernels/conv2_dw.cl b/ggml/src/ggml-opencl/kernels/conv2_dw.cl new file mode 100644 index 0000000..2a35b9c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/conv2_dw.cl @@ -0,0 +1,139 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +kernel void kernel_conv2d_dw_f32_f32( + global const float * weights, + long weights_offset, + global const float * input, + long input_offset, + global float * output, + long output_offset, + long C, + long N, + long KW, + long KH, + long W, + long H, + long OW, + long OH, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, + long nb01, + long nb03, + long nb11, + long nb12, + long nb13, + long nb1, + long nb2, + long nb3 +) { + weights = (global const float *) ((global const char *) weights + weights_offset); + input = (global const float *) ((global const char *) input + input_offset); + output = (global float *) ((global char *) output + output_offset); + + const long gid = get_global_id(0); + const long OHW = OH * OW; + const long COHW = C * OHW; + const long total = N * COHW; + if (gid >= total) { + return; + } + + const long n = gid / COHW; + const long c_rem = gid - n * COHW; + const long c = c_rem / OHW; + const long oh = (c_rem - c * OHW) / OW; + const long ow = c_rem - c * OHW - oh * OW; + + float sum = 0.0f; + for (long kh = 0; kh < KH; ++kh) { + const long ih = oh * s1 + kh * d1 - p1; + if (ih < 0 || ih >= H) { + continue; + } + for (long kw = 0; kw < KW; ++kw) { + const long iw = ow * s0 + kw * d0 - p0; + if (iw < 0 || iw >= W) { + continue; + } + const long weight_idx = kw + kh * nb01 + c * nb03; + const long input_idx = iw + ih * nb11 + c * nb12 + n * nb13; + sum += weights[weight_idx] * input[input_idx]; + } + } + + const long output_idx = ow + oh * nb1 + c * nb2 + n * nb3; + output[output_idx] = sum; +} + +kernel void kernel_conv2d_dwf16_f32( + global const half * weights, + long weights_offset, + global const float * input, + long input_offset, + global float * output, + long output_offset, + long C, + long N, + long KW, + long KH, + long W, + long H, + long OW, + long OH, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, + long nb01, + long nb03, + long nb11, + long nb12, + long nb13, + long nb1, + long nb2, + long nb3 +) { + weights = (global const half *) ((global const char *) weights + weights_offset); + input = (global const float *) ((global const char *) input + input_offset); + output = (global float *) ((global char *) output + output_offset); + + const long gid = get_global_id(0); + const long OHW = OH * OW; + const long COHW = C * OHW; + const long total = N * COHW; + if (gid >= total) { + return; + } + + const long n = gid / COHW; + const long c_rem = gid - n * COHW; + const long c = c_rem / OHW; + const long oh = (c_rem - c * OHW) / OW; + const long ow = c_rem - c * OHW - oh * OW; + + float sum = 0.0f; + for (long kh = 0; kh < KH; ++kh) { + const long ih = oh * s1 + kh * d1 - p1; + if (ih < 0 || ih >= H) { + continue; + } + for (long kw = 0; kw < KW; ++kw) { + const long iw = ow * s0 + kw * d0 - p0; + if (iw < 0 || iw >= W) { + continue; + } + const long weight_idx = kw + kh * nb01 + c * nb03; + const long input_idx = iw + ih * nb11 + c * nb12 + n * nb13; + sum += (float) weights[weight_idx] * input[input_idx]; + } + } + + const long output_idx = ow + oh * nb1 + c * nb2 + n * nb3; + output[output_idx] = sum; +} diff --git a/ggml/src/ggml-opencl/kernels/conv2d.cl b/ggml/src/ggml-opencl/kernels/conv2d.cl index e339c90..8a04c2e 100644 --- a/ggml/src/ggml-opencl/kernels/conv2d.cl +++ b/ggml/src/ggml-opencl/kernels/conv2d.cl @@ -48,8 +48,8 @@ kernel void kernel_conv_2d( uint Cout, uint Cin, uint N, uint KW, uint KH, uint W, uint H, uint OW, uint OH, uint s0, uint s1, uint p0, uint p1, uint d0, uint d1, - uint nb01, uint nb02, uint nb03, - uint nb11, uint nb12, uint nb13, + uint nb00, uint nb01, uint nb02, uint nb03, + uint nb10, uint nb11, uint nb12, uint nb13, uint nb1, uint nb2, uint nb3 ) { global T_FLOAT* knl_data = (global T_FLOAT*) ((global char*)p_knl + off_knl); @@ -95,7 +95,7 @@ kernel void kernel_conv_2d( const uint Cin_idx = crs_g / (KW*KH); const uint KH_idx = (crs_g - Cin_idx*KW*KH) / KW; const uint KW_idx = crs_g - Cin_idx*KW*KH - KH_idx*KW; - const uint knl_idx = KW_idx + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03; + const uint knl_idx = KW_idx*nb00 + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03; Ash[k_l * BS_CRS + crs_l] = knl_data[knl_idx]; } else { Ash[k_l * BS_CRS + crs_l] = (T_FLOAT)0.0f; @@ -123,7 +123,7 @@ kernel void kernel_conv_2d( const int W_idx = (int)(OW_idx * s0 + KW_idx * d0 - p0); if (H_idx >= 0 && H_idx < H && W_idx >= 0 && W_idx < W) { - const uint src_idx = W_idx + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13; + const uint src_idx = W_idx * nb10 + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13; ((T_FLOAT*)&val)[v] = src_data[src_idx]; } } diff --git a/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl b/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl index cb05637..94788e7 100644 --- a/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +++ b/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl @@ -39,8 +39,8 @@ kernel void kernel_conv_2d( uint Cout, uint Cin, uint N, uint KW, uint KH, uint W, uint H, uint OW, uint OH, uint s0, uint s1, uint p0, uint p1, uint d0, uint d1, - uint nb01, uint nb02, uint nb03, - uint nb11, uint nb12, uint nb13, + uint nb00, uint nb01, uint nb02, uint nb03, + uint nb10, uint nb11, uint nb12, uint nb13, uint nb1, uint nb2, uint nb3 ) { global half* knl_data = (global half*) ((global char*)p_knl + off_knl); @@ -86,7 +86,7 @@ kernel void kernel_conv_2d( const uint Cin_idx = crs_g / (KW*KH); const uint KH_idx = (crs_g - Cin_idx*KW*KH) / KW; const uint KW_idx = crs_g - Cin_idx*KW*KH - KH_idx*KW; - const uint knl_idx = KW_idx + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03; + const uint knl_idx = KW_idx*nb00 + KH_idx*nb01 + Cin_idx*nb02 + k_g*nb03; Ash[k_l * BS_CRS + crs_l] = knl_data[knl_idx]; } else { Ash[k_l * BS_CRS + crs_l] = (half)0.0f; @@ -114,7 +114,7 @@ kernel void kernel_conv_2d( const int W_idx = (int)(OW_idx * s0 + KW_idx * d0 - p0); if (H_idx >= 0 && H_idx < H && W_idx >= 0 && W_idx < W) { - const uint src_idx = W_idx + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13; + const uint src_idx = W_idx * nb10 + H_idx * nb11 + Cin_idx * nb12 + N_idx * nb13; ((float*)&val)[v] = src_data[src_idx]; } } diff --git a/ggml/src/ggml-opencl/kernels/cpy.cl b/ggml/src/ggml-opencl/kernels/cpy.cl index 820aa53..18400d3 100644 --- a/ggml/src/ggml-opencl/kernels/cpy.cl +++ b/ggml/src/ggml-opencl/kernels/cpy.cl @@ -183,6 +183,67 @@ kernel void kernel_cpy_f32_f32( } } +#ifndef GGML_CL_NO_CPY_PACK +kernel void kernel_cpy_f32_f32_pack( + global float * src0, + ulong offset0, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne0, + int ne1, + int ne2, + int ne3, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global float*)((global char*)src0 + offset0); + dst = (global float*)((global char*)dst + offsetd); + + int lsz = get_local_size(0); + int tpr = min(ne00, lsz); // threads per row + int rpw = lsz / tpr; // rows per workgroup + int lid = get_local_id(0); + int row = get_group_id(0)*rpw + lid / tpr; + int lane = lid - (lid / tpr) * tpr; + + int nrows = ne01*ne02*ne03; + if (row >= nrows) { + return; + } + + int i01 = row % ne01; + int t = row / ne01; + int i02 = t % ne02; + int i03 = t / ne02; + + // linear index of the first element of this row, unflattened over dst dims + long n = (long)row * ne00; + int i3 = (int)(n / ((long)ne2*ne1*ne0)); + long rm = n - (long)i3*ne2*ne1*ne0; + int i2 = (int)(rm / ((long)ne1*ne0)); + rm -= (long)i2*ne1*ne0; + int i1 = (int)(rm / ne0); + int i0 = (int)(rm - (long)i1*ne0); + + global float * dst_data = (global float *) ((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + for (int i00 = lane; i00 < ne00; i00 += tpr) { + global const float * src = (global float *)((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00); + dst_data[i00] = src[0]; + } +} +#endif // GGML_CL_NO_CPY_PACK + kernel void kernel_cpy_i32_i32( global int * src0, ulong offset0, @@ -227,3 +288,113 @@ kernel void kernel_cpy_i32_i32( dst_data[i00] = src[0]; } } + +// Contiguous f32 copy, one work item per float4 over the whole tensor. The kernels above map +// one workgroup to each row, which leaves a tensor with few long rows on a single compute unit. +// vload4/vstore4 rather than a float4 cast: these buffers carry an arbitrary 4-byte view offset. +kernel void kernel_cpy_f32_f32_flat( + global float * src0, + ulong offset0, + global float * dst, + ulong offsetd, + ulong ne, + ulong n4 +) { + src0 = (global float*)((global char*)src0 + offset0); + dst = (global float*)((global char*)dst + offsetd); + + const ulong i = get_global_id(0); + + if (i < n4) { + vstore4(vload4(i, src0), i, dst); + } else if (i == n4) { + for (ulong t = n4 * 4; t < ne; ++t) { + dst[t] = src0[t]; + } + } +} + +kernel void kernel_cpy_i32_f32( + global int * src0, + ulong offset0, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne0, + int ne1, + int ne2, + int ne3, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global int *) ((global char *) src0 + offset0); + dst = (global float *) ((global char *) dst + offsetd); + + const ulong i01 = get_group_id(0); + const ulong i02 = get_group_id(1); + const ulong i03 = get_group_id(2); + + for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) { + const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00; + const ulong i0 = n % ne0; + const ulong i1 = n / ne0 % ne1; + const ulong i2 = n / ne0 / ne1 % ne2; + const ulong i3 = n / ne0 / ne1 / ne2; + + global const int * src = (global const int *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); + global float * out = (global float *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + out[0] = convert_float(src[0]); + } +} + + +kernel void kernel_cpy_f32_i32( + global float * src0, + ulong offset0, + global int * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne0, + int ne1, + int ne2, + int ne3, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global float *) ((global char *) src0 + offset0); + dst = (global int *) ((global char *) dst + offsetd); + + const ulong i01 = get_group_id(0); + const ulong i02 = get_group_id(1); + const ulong i03 = get_group_id(2); + + for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) { + const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00; + const ulong i0 = n % ne0; + const ulong i1 = n / ne0 % ne1; + const ulong i2 = n / ne0 / ne1 % ne2; + const ulong i3 = n / ne0 / ne1 / ne2; + + global const float * src = (global const float *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); + global int * out = (global int *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + out[0] = convert_int_rtz(src[0]); + } +} diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index 39af32d..f3309a2 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -27,6 +27,8 @@ #define QR5_1 2 #define QK8_0 32 #define QR8_0 1 +#define QK1_0 128 +#define QR1_0 1 #define QK_K 256 #define K_SCALE_SIZE (3 * QK_K / 64) #define K_QUANTS_PER_ITERATION 2 @@ -38,6 +40,14 @@ typedef ushort uint16_t; typedef int int32_t; typedef uint uint32_t; +//------------------------------------------------------------------------------ +// block_q1_0 +//------------------------------------------------------------------------------ +typedef struct { + half d; // delta + uchar qs[QK1_0/8]; // 1-bit signs (16 bytes) +} block_q1_0; + //------------------------------------------------------------------------------ // block_q4_0 //------------------------------------------------------------------------------ @@ -56,6 +66,25 @@ struct block_q4_1 { uchar qs[QK4_1 / 2]; // nibbles / quants }; +//------------------------------------------------------------------------------ +// block_q5_0 +//------------------------------------------------------------------------------ +struct block_q5_0 { + half d; // delta + uchar qh[4]; // 5-th bit of quants + uchar qs[QK5_0 / 2]; // nibbles / quants +}; + +//------------------------------------------------------------------------------ +// block_q5_1 +//------------------------------------------------------------------------------ +struct block_q5_1 { + half d; // delta + half m; // min + uchar qh[4]; // 5-th bit of quants + uchar qs[QK5_1 / 2]; // nibbles / quants +}; + //------------------------------------------------------------------------------ // block_q4_k //------------------------------------------------------------------------------ @@ -87,6 +116,95 @@ struct block_q6_K { half d; // super-block scale }; +//------------------------------------------------------------------------------ +// block_iq4_nl +//------------------------------------------------------------------------------ +#define QK4_NL 32 + +struct block_iq4_nl +{ + half d; + uint8_t qs[QK4_NL / 2]; +}; + +//------------------------------------------------------------------------------ +// bf16 to f16 +//------------------------------------------------------------------------------ +kernel void kernel_convert_bf16_to_f16( + global const ushort * src, + global half * dst, + ulong off_dst, + ulong n +) { + uint i = get_global_id(0); + if (i >= n) { + return; + } + + dst[i + off_dst] = (half) as_float((uint) src[i] << 16); +} + +//------------------------------------------------------------------------------ +// f16 to bf16 +//------------------------------------------------------------------------------ +kernel void kernel_convert_f16_to_bf16( + global const half * src, + ulong off_src, + global ushort * dst, + ulong n +) { + uint i = get_global_id(0); + if (i >= n) { + return; + } + + float f = (float) src[i + off_src]; + uint bits = as_uint(f); + if ((bits & 0x7fffffffu) > 0x7f800000u) { + // nan to quiet nan + dst[i] = (ushort)((bits >> 16) | 0x40u); + } else { + uint rounded = bits + 0x7fffu + ((bits >> 16) & 1u); + dst[i] = (ushort)(rounded >> 16); + } +} + +//------------------------------------------------------------------------------ +// kernel_convert_block_q1_0 +// Convert block_q1_0 (AOS) to 2 separate arrays (SOA): quant bytes + scales. +// q1_0 bits are stored in natural order (bit j of byte i -> weight 8*i + j) +//------------------------------------------------------------------------------ +kernel void kernel_convert_block_q1_0( + global block_q1_0 * src0, + global uchar * dst_q, + global half * dst_d +) { + global block_q1_0 * b = (global block_q1_0 *) src0 + get_global_id(0); + global uchar * q = (global uchar *) dst_q + (QK1_0/8)*get_global_id(0); + global half * d = (global half *) dst_d + get_global_id(0); + + *d = b->d; + + for (int i = 0; i < QK1_0/8; ++i) { + q[i] = b->qs[i]; + } +} + +kernel void kernel_restore_block_q1_0( + global uchar * src_q, + global half * src_d, + global block_q1_0 * dst +) { + global block_q1_0 * b = (global block_q1_0 *) dst + get_global_id(0); + global uchar * q = (global uchar *) src_q + (QK1_0/8)*get_global_id(0); + global half * d = (global half *) src_d + get_global_id(0); + + b->d = *d; + for (int i = 0; i < QK1_0/8; ++i) { + b->qs[i] = q[i]; + } +} + //------------------------------------------------------------------------------ // kernel_convert_block_q4_0 // Convert the block_q4_0 format to 2 separate arrays (AOS -> SOA). @@ -179,6 +297,100 @@ kernel void kernel_restore_block_q4_0_noshuffle( } } +kernel void kernel_convert_block_q4_0_trans4_ns( + global struct block_q4_0 * src0, + __global uint * dst_q, + __global half * dst_d, + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK4_0; + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + global struct block_q4_0 * b = src0 + src_blk_offset; + dst_d[dst_blk_offset] = b->d; + + // extract quantization and unshuffle + ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0]; + + ushort8 post_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK4_0 / 4; ++i) { + uchar x0 = pre_block_ptr[2*i + 0]; + uchar x1 = pre_block_ptr[2*i + 1]; + + post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + post_block_ptr[i + QK4_0 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + + uint4 q_block = as_uint4(post_block); + + uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + dst_q[offset] = q_block.x; + dst_q[offset + ne01] = q_block.y; + dst_q[offset + ne01 * 2] = q_block.z; + dst_q[offset + ne01 * 3] = q_block.w; +} + +kernel void kernel_restore_block_q4_0_trans4_ns( + __global uint * src_q, + __global half * src_d, + __global struct block_q4_0 * dst0, + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK4_0; + uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint src_d_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + __global struct block_q4_0 * b = dst0 + dst_blk_offset; + b->d = src_d[src_d_offset]; + + // collect transposed quantization parts for a block + uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + uint4 q_block; + q_block.x = src_q[src_q_offset]; + q_block.y = src_q[src_q_offset + ne01]; + q_block.z = src_q[src_q_offset + ne01 * 2]; + q_block.w = src_q[src_q_offset + ne01 * 3]; + + ushort8 post_block = as_ushort8(q_block); + ushort8 pre_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK4_0 / 4; ++i) { + uchar x0 = post_block_ptr[i + 0]; + uchar x1 = post_block_ptr[i + QK4_0 / 4]; + + pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + + ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block; +} + //------------------------------------------------------------------------------ // kernel_convert_block_q4_1 // Convert the block_q4_1 format to 2 separate arrays (AOS -> SOA). @@ -238,38 +450,1032 @@ kernel void kernel_convert_block_q4_1_noshuffle( uchar x0 = b->qs[2*i + 0]; uchar x1 = b->qs[2*i + 1]; - q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); - q[i + QK4_1/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + q[i + QK4_1/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + +#ifdef ADRENO_GPU + if (get_global_id(0) == 65536*4096) { + printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0)); + } +#endif + } +} + +kernel void kernel_restore_block_q4_1_noshuffle( + global uchar * src_q, + global half * src_d, + global half * src_m, + global struct block_q4_1 * dst, + uchar mask_0F, + uchar mask_F0 +) { + global struct block_q4_1 * b = (global struct block_q4_1 *) dst + get_global_id(0); + global uchar * q = (global uchar *) src_q + QK4_1/2*get_global_id(0); + global half * d = (global half *) src_d + get_global_id(0); + global half * m = (global half *) src_m + get_global_id(0); + + b->d = *d; + b->m = *m; + for (int i = 0; i < QK4_1/4; ++i) { + uchar x0 = q[i + 0 ] ; + uchar x1 = q[i + QK4_1/4]; + + b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4)); + b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0)); + } +} + +kernel void kernel_convert_block_q4_1_trans4_ns( + __global struct block_q4_1 * src0, + __global uint * dst_q, + __global half * dst_d, + __global half * dst_m, + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK4_1; + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + global struct block_q4_1 * b = src0 + src_blk_offset; + dst_d[dst_blk_offset] = b->d; + dst_m[dst_blk_offset] = b->m; + + // extract quantization and unshuffle + ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0]; + + ushort8 post_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK4_1 / 4; ++i) { + uchar x0 = pre_block_ptr[2*i + 0]; + uchar x1 = pre_block_ptr[2*i + 1]; + + post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + post_block_ptr[i + QK4_1 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + + uint4 q_block = as_uint4(post_block); + + uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + dst_q[offset] = q_block.x; + dst_q[offset + ne01] = q_block.y; + dst_q[offset + ne01 * 2] = q_block.z; + dst_q[offset + ne01 * 3] = q_block.w; +} + +kernel void kernel_restore_block_q4_1_trans4_ns( + __global uint * src_q, + __global half * src_d, + __global half * src_m, + __global struct block_q4_1 * dst0, + uint ne00, + uint ne01 +) { + int i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK4_1; + uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint src_dm_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + __global struct block_q4_1 * b = dst0 + dst_blk_offset; + b->d = src_d[src_dm_offset]; + b->m = src_m[src_dm_offset]; + + // collect transposed quantization parts for a block + uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + uint4 q_block; + q_block.x = src_q[src_q_offset]; + q_block.y = src_q[src_q_offset + ne01]; + q_block.z = src_q[src_q_offset + ne01 * 2]; + q_block.w = src_q[src_q_offset + ne01 * 3]; + + ushort8 post_block = as_ushort8(q_block); + ushort8 pre_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK4_0 / 4; ++i) { + uchar x0 = post_block_ptr[i + 0]; + uchar x1 = post_block_ptr[i + QK4_0 / 4]; + + pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + + ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block; +} + +//------------------------------------------------------------------------------ +// kernel_convert_block_q5_0 +// Convert the block_q5_0 format to 3 separate arrays (AOS -> SOA). +// This kernel does not deshuffle the bits. +//------------------------------------------------------------------------------ +kernel void kernel_convert_block_q5_0( + global struct block_q5_0 * src0, + global uchar * dst_qs, + global uint * dst_qh, + global half * dst_d, + ulong n_blk +) { + if (get_global_id(0) >= n_blk) { + return; + } + + global struct block_q5_0 * b = (global struct block_q5_0 *) src0 + get_global_id(0); + global uchar * qs = (global uchar *) dst_qs + (QK5_0/2)*get_global_id(0); + global uint * qh = (global uint *) dst_qh + get_global_id(0); + global half * d = (global half *) dst_d + get_global_id(0); + + *d = b->d; + *qh = *((global uint *)(b->qh)); + + for (int i = 0; i < QK5_0/2; ++i) { + qs[i] = b->qs[i]; + } +} + +kernel void kernel_restore_block_q5_0( + global uchar * src_qs, + global uint * src_qh, + global half * src_d, + global struct block_q5_0 * dst +) { + global struct block_q5_0 * b = (global struct block_q5_0 *) dst + get_global_id(0); + global uchar * qs = (global uchar *) src_qs + (QK5_0/2)*get_global_id(0); + global uint * qh = (global uint *) src_qh + get_global_id(0); + global half * d = (global half *) src_d + get_global_id(0); + + b->d = *d; + *((global uint *)(b->qh)) = *qh; + for (int i = 0; i < QK5_0/2; ++i) { + b->qs[i] = qs[i]; + } +} + +kernel void kernel_convert_block_q5_0_noshuffle( + global struct block_q5_0 * src0, + global uchar * dst_q, + global uint * dst_qh, + global half * dst_d +) { + global struct block_q5_0 * b = (global struct block_q5_0 *) src0 + get_global_id(0); + global uchar * q = (global uchar *) dst_q + QK5_0/2*get_global_id(0); + global uint * qh = (global uint *) dst_qh + get_global_id(0); + global half * d = (global half *) dst_d + get_global_id(0); + + *d = b->d; + *qh = *((global uint *)(b->qh)); + + for (int i = 0; i < QK5_0/4; ++i) { + uchar x0 = b->qs[2*i + 0]; + uchar x1 = b->qs[2*i + 1]; + + q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + q[i + QK5_0/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + +#ifdef ADRENO_GPU + if (get_global_id(0) == 65536*4096) { + printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0)); + } +#endif + } +} + +kernel void kernel_restore_block_q5_0_noshuffle( + global uchar * src_q, + global uint * src_qh, + global half * src_d, + global struct block_q5_0 * dst, + uchar mask_0F, + uchar mask_F0 +) { + global struct block_q5_0 * b = (global struct block_q5_0 *) dst + get_global_id(0); + global uchar * q = (global uchar *) src_q + QK5_0/2*get_global_id(0); + global uint * qh = (global uint *) src_qh + get_global_id(0); + global half * d = (global half *) src_d + get_global_id(0); + + b->d = *d; + *((global uint *)(b->qh)) = *qh; + + for (int i = 0; i < QK5_0/4; ++i) { + uchar x0 = q[i + 0 ]; + uchar x1 = q[i + QK5_0/4]; + + b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4)); + b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0)); + } +} + +kernel void kernel_convert_block_q5_0_trans4_ns( + __global struct block_q5_0 * src0, + __global uint * dst_qs, + __global uint * dst_qh, + __global half * dst_d, + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK5_0; + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + global struct block_q5_0 * b = src0 + src_blk_offset; + dst_d[dst_blk_offset] = b->d; + + dst_qh[dst_blk_offset] = ((global uint *)(&(b->qh[0])))[0]; + + // extract quantization and unshuffle + ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0]; + ushort8 post_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK5_0 / 4; ++i) { + uchar x0 = pre_block_ptr[2*i + 0]; + uchar x1 = pre_block_ptr[2*i + 1]; + + post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + post_block_ptr[i + QK5_0 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + + uint4 q_block = as_uint4(post_block); + + uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + dst_qs[offset] = q_block.x; + dst_qs[offset + ne01] = q_block.y; + dst_qs[offset + ne01 * 2] = q_block.z; + dst_qs[offset + ne01 * 3] = q_block.w; +} + +kernel void kernel_restore_block_q5_0_trans4_ns( + __global uint * src_qs, + __global uint * src_qh, + __global half * src_d, + __global struct block_q5_0 * dst0, + uint ne00, + uint ne01 +) { + int i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK5_0; + uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + __global struct block_q5_0 * b = dst0 + dst_blk_offset; + b->d = src_d[src_blk_offset]; + + ((__global uint *)(&(b->qh[0])))[0] = src_qh[src_blk_offset]; + + // collect transposed quantization parts for a block + uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + uint4 q_block; + q_block.x = src_qs[src_q_offset]; + q_block.y = src_qs[src_q_offset + ne01]; + q_block.z = src_qs[src_q_offset + ne01 * 2]; + q_block.w = src_qs[src_q_offset + ne01 * 3]; + + ushort8 post_block = as_ushort8(q_block); + ushort8 pre_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK5_0 / 4; ++i) { + uchar x0 = post_block_ptr[i + 0]; + uchar x1 = post_block_ptr[i + QK5_0 / 4]; + + pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + + ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block; +} + +//------------------------------------------------------------------------------ +// kernel_convert_block_q5_1 +// Convert the block_q5_1 format to 4 separate arrays (AOS -> SOA). +// This kernel does not deshuffle the bits. +//------------------------------------------------------------------------------ +kernel void kernel_convert_block_q5_1( + global struct block_q5_1 * src0, + global uchar * dst_qs, + global uint * dst_qh, + global half * dst_d, + global half * dst_m, + ulong n_blk +) { + if (get_global_id(0) >= n_blk) { + return; + } + + global struct block_q5_1 * b = (global struct block_q5_1 *) src0 + get_global_id(0); + global uchar * qs = (global uchar *) dst_qs + (QK5_1/2)*get_global_id(0); + global uint * qh = (global uint *) dst_qh + get_global_id(0); + global half * d = (global half *) dst_d + get_global_id(0); + global half * m = (global half *) dst_m + get_global_id(0); + + *d = b->d; + *m = b->m; + *qh = *((global uint *)(b->qh)); + + for (int i = 0; i < QK5_1/2; ++i) { + qs[i] = b->qs[i]; + } +} + +kernel void kernel_restore_block_q5_1( + global uchar * src_qs, + global uint * src_qh, + global half * src_d, + global half * src_m, + global struct block_q5_1 * dst +) { + global struct block_q5_1 * b = (global struct block_q5_1 *) dst + get_global_id(0); + global uchar * qs = (global uchar *) src_qs + (QK5_1/2)*get_global_id(0); + global uint * qh = (global uint *) src_qh + get_global_id(0); + global half * d = (global half *) src_d + get_global_id(0); + global half * m = (global half *) src_m + get_global_id(0); + + b->d = *d; + b->m = *m; + *((global uint *)(b->qh)) = *qh; + for (int i = 0; i < QK5_1/2; ++i) { + b->qs[i] = qs[i]; + } +} + +kernel void kernel_convert_block_q5_1_noshuffle( + global struct block_q5_1 * src0, + global uchar * dst_q, + global uint * dst_qh, + global half * dst_d, + global half * dst_m +) { + global struct block_q5_1 * b = (global struct block_q5_1 *) src0 + get_global_id(0); + global uchar * q = (global uchar *) dst_q + QK5_1/2*get_global_id(0); + global uint * qh = (global uint *) dst_qh + get_global_id(0); + global half * d = (global half *) dst_d + get_global_id(0); + global half * m = (global half *) dst_m + get_global_id(0); + + *d = b->d; + *m = b->m; + *qh = *((global uint *)(b->qh)); + + for (int i = 0; i < QK5_1/4; ++i) { + uchar x0 = b->qs[2*i + 0]; + uchar x1 = b->qs[2*i + 1]; + + q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + q[i + QK5_1/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + +#ifdef ADRENO_GPU + if (get_global_id(0) == 65536*4096) { + printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0)); + } +#endif + } +} + +kernel void kernel_restore_block_q5_1_noshuffle( + global uchar * src_q, + global uint * src_qh, + global half * src_d, + global half * src_m, + global struct block_q5_1 * dst, + uchar mask_0F, + uchar mask_F0 +) { + global struct block_q5_1 * b = (global struct block_q5_1 *) dst + get_global_id(0); + global uchar * q = (global uchar *) src_q + QK5_1/2*get_global_id(0); + global uint * qh = (global uint *) src_qh + get_global_id(0); + global half * d = (global half *) src_d + get_global_id(0); + global half * m = (global half *) src_m + get_global_id(0); + + b->d = *d; + b->m = *m; + *((global uint *)(b->qh)) = *qh; + + for (int i = 0; i < QK5_1/4; ++i) { + uchar x0 = q[i + 0 ]; + uchar x1 = q[i + QK5_1/4]; + + b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4)); + b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0)); + } +} + +kernel void kernel_convert_block_q5_1_trans4_ns( + __global struct block_q5_1 * src0, + __global uint * dst_qs, + __global uint * dst_qh, + __global half * dst_d, + __global half * dst_m, + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK5_1; + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + global struct block_q5_1 * b = src0 + src_blk_offset; + dst_d[dst_blk_offset] = b->d; + dst_m[dst_blk_offset] = b->m; + + dst_qh[dst_blk_offset] = ((global uint *)(&(b->qh[0])))[0]; + + // extract quantization and unshuffle + ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0]; + ushort8 post_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK5_1 / 4; ++i) { + uchar x0 = pre_block_ptr[2*i + 0]; + uchar x1 = pre_block_ptr[2*i + 1]; + + post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + post_block_ptr[i + QK5_1 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + + uint4 q_block = as_uint4(post_block); + + uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + dst_qs[offset] = q_block.x; + dst_qs[offset + ne01] = q_block.y; + dst_qs[offset + ne01 * 2] = q_block.z; + dst_qs[offset + ne01 * 3] = q_block.w; +} + +kernel void kernel_restore_block_q5_1_trans4_ns( + __global uint * src_qs, + __global uint * src_qh, + __global half * src_d, + __global half * src_m, + __global struct block_q5_1 * dst0, + uint ne00, + uint ne01 +) { + int i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK5_1; + uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + __global struct block_q5_1 * b = dst0 + dst_blk_offset; + b->d = src_d[src_blk_offset]; + b->m = src_m[src_blk_offset]; + + ((__global uint *)(&(b->qh[0])))[0] = src_qh[src_blk_offset]; + + // collect transposed quantization parts for a block + uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + uint4 q_block; + q_block.x = src_qs[src_q_offset]; + q_block.y = src_qs[src_q_offset + ne01]; + q_block.z = src_qs[src_q_offset + ne01 * 2]; + q_block.w = src_qs[src_q_offset + ne01 * 3]; + + ushort8 post_block = as_ushort8(q_block); + ushort8 pre_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK5_1 / 4; ++i) { + uchar x0 = post_block_ptr[i + 0]; + uchar x1 = post_block_ptr[i + QK5_1 / 4]; + + pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block; +} + +kernel void kernel_convert_block_q4_k_trans4_ns( + __global struct block_q4_K * src0, + __global uint * dst_q, + __global half * dst_d, + __global half * dst_dm, + __global uchar * dst_s, + uint ne00, + uint ne01, + uchar mask_0F, + uchar mask_F0 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK_K; + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + __global struct block_q4_K * b = src0 + src_blk_offset; + + dst_d [dst_blk_offset] = b->d; + dst_dm[dst_blk_offset] = b->dm; + + uint4 qv[8]; + uchar * qv_bytes = (uchar *)qv; + for (int i = 0; i < QK_K / 64; ++i) { + for (int j = 0; j < 16; ++j) { + uchar x0 = b->q[i*32 + 2*j]; + uchar x1 = b->q[i*32 + 2*j + 1]; + + qv_bytes[i*32 + j ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4); + qv_bytes[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0); + } + } + + uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01; + #pragma unroll + for (int p = 0; p < 8; ++p) { + uint4 v = qv[p]; + dst_q[base + (p * 4 + 0) * ne01] = v.x; + dst_q[base + (p * 4 + 1) * ne01] = v.y; + dst_q[base + (p * 4 + 2) * ne01] = v.z; + dst_q[base + (p * 4 + 3) * ne01] = v.w; + } + + __global uchar * s_dst = dst_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE; + #pragma unroll + for (int i = 0; i < K_SCALE_SIZE; ++i) { + s_dst[i] = b->s[i]; + } +} + +kernel void kernel_restore_block_q4_k_trans4_ns( + __global uint * src_q, + __global half * src_d, + __global half * src_dm, + __global uchar * src_s, + __global struct block_q4_K * dst0, + uint ne00, + uint ne01, + uchar mask_0F, + uchar mask_F0 +) { + uint i00 = get_global_id(1); // block index along K + uint i01 = get_global_id(0); // row index + uint i02 = get_global_id(2); // batch index + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK_K; + + uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + + __global struct block_q4_K * b = dst0 + dst_blk_offset; + + b->d = src_d[src_blk_offset]; + b->dm = src_dm[src_blk_offset]; + + __global uchar * s_src = src_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE; + for (int i = 0; i < K_SCALE_SIZE; ++i) { + b->s[i] = s_src[i]; + } + + uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01; + + uint4 qv[8]; + for (int p = 0; p < 8; ++p) { + qv[p].x = src_q[base + (p * 4 + 0) * ne01]; + qv[p].y = src_q[base + (p * 4 + 1) * ne01]; + qv[p].z = src_q[base + (p * 4 + 2) * ne01]; + qv[p].w = src_q[base + (p * 4 + 3) * ne01]; + } + + uchar * qv_bytes = (uchar *)qv; + for (int i = 0; i < QK_K / 64; ++i) { + for (int j = 0; j < 16; ++j) { + uchar lo = qv_bytes[i*32 + j]; + uchar hi = qv_bytes[i*32 + j + 16]; + b->q[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4)); + b->q[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0)); + } + } +} + +//------------------------------------------------------------------------------ +// kernel_convert_block_q4_k_tiled_ns +// +// Tiled-wide layout for the long-vocab q4_K lm_head/embed GEMV (decode path). +// Mirror of kernel_convert_block_q6_k_tiled_ns: recovers each weight's 4-bit +// code in CANONICAL ggml element order (e in [0,256)) and re-packs into 32 uints +// (8 codes/uint), stored TILED by 64 output rows so the matching GEMV +// (gemv_noshuffle_q4_k_f32_tiled) coalesces every weight load. The 12-byte +// packed scale block `s` and d/dm are stored per (row, K-block) tiled; the GEMV +// re-derives the 8 (scale,min) pairs via get_scale_min_k4, exactly like the o4 +// kernel. Both ends owned here -> correct by construction vs the reference q4_K +// dequant. Requires ne01 % 64 == 0 (gated host-side). Buffer sizes identical to +// the trans4_ns layout. +// +// q uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk+sb)*8 + g)*64 + rit +// s (12 bytes) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit, *12 +// d/dm (half) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit +// where rt = r/64, rit = r%64. +//------------------------------------------------------------------------------ +kernel void kernel_convert_block_q4_k_tiled_ns( + __global struct block_q4_K * src0, + __global uint * dst_q, // 32 uints / superblock (4-bit codes, 8 codes/uint) + __global half * dst_d, // 1 half / superblock + __global half * dst_dm, // 1 half / superblock + __global uchar * dst_s, // K_SCALE_SIZE (12) bytes / superblock + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); // K-block index (superblock along ne00) + uint i01 = get_global_id(0); // output row index (along ne01) + uint i02 = get_global_id(2); // batch + + uint ne00_blk = ne00 / QK_K; + + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + __global struct block_q4_K * b = src0 + src_blk_offset; + + uint rt = i01 / 64; + uint rit = i01 % 64; + uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00; + + // --- recover canonical 4-bit codes in e-order, pack 8 codes/uint --- + uint qw[32] = {0}; + for (uint e = 0; e < 256; ++e) { + uint g = e >> 6; // group 0..3 (q advances 32 bytes/group) + uint within = e & 63u; + uint hlf = within >> 5; // 0 = low nibble, 1 = high nibble + uint l = within & 31u; // 0..31 + uchar byte = b->q[g * 32u + l]; + uint code = (hlf == 0u) ? (uint)(byte & 0x0F) : (uint)(byte >> 4); + qw[e >> 3] |= code << ((e & 7u) * 4u); + } + + for (uint gr = 0; gr < 8; ++gr) { + uint base = (tile_blk * 8u + gr) * 64u + rit; // uint4 index + dst_q[base * 4u + 0u] = qw[gr * 4u + 0u]; + dst_q[base * 4u + 1u] = qw[gr * 4u + 1u]; + dst_q[base * 4u + 2u] = qw[gr * 4u + 2u]; + dst_q[base * 4u + 3u] = qw[gr * 4u + 3u]; + } + + // packed scales (12 bytes), tiled per (row, block) + __global uchar * s_dst = dst_s + (tile_blk * 64u + rit) * K_SCALE_SIZE; + #pragma unroll + for (int i = 0; i < K_SCALE_SIZE; ++i) { + s_dst[i] = b->s[i]; + } + + dst_d [tile_blk * 64u + rit] = b->d; + dst_dm[tile_blk * 64u + rit] = b->dm; +} + +kernel void kernel_convert_block_q5_k_trans4_ns( + __global struct block_q5_K * src0, + __global uint * dst_qs, + __global uint * dst_qh, + __global half * dst_d, + __global half * dst_dm, + __global uchar * dst_s, + uint ne00, + uint ne01, + uchar mask_0F, + uchar mask_F0 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK_K; + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + __global struct block_q5_K * b = src0 + src_blk_offset; + + dst_d [dst_blk_offset] = b->d; + dst_dm[dst_blk_offset] = b->dm; + + for (int k = 0; k < 8; k++) { + uchar b0 = 0, b1 = 0, b2 = 0, b3 = 0; + for (int bit = 0; bit < 8; bit++) { + b0 |= (uchar)(((b->qh[bit] >> k) & 1) << bit); + b1 |= (uchar)(((b->qh[8 + bit] >> k) & 1) << bit); + b2 |= (uchar)(((b->qh[16 + bit] >> k) & 1) << bit); + b3 |= (uchar)(((b->qh[24 + bit] >> k) & 1) << bit); + } + uint packed = (uint)b0 | ((uint)b1 << 8) | ((uint)b2 << 16) | ((uint)b3 << 24); + dst_qh[i01 + (i00 * 8 + k) * ne01 + i02 * ne00_blk * 8 * ne01] = packed; + } + + uint4 qv[8]; + uchar * qv_bytes = (uchar *)qv; + for (int i = 0; i < QK_K / 64; ++i) { + for (int j = 0; j < 16; ++j) { + uchar x0 = b->qs[i*32 + 2*j]; + uchar x1 = b->qs[i*32 + 2*j + 1]; + + qv_bytes[i*32 + j ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4); + qv_bytes[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0); + } + } + + uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01; + #pragma unroll + for (int p = 0; p < 8; ++p) { + uint4 v = qv[p]; + dst_qs[base + (p * 4 + 0) * ne01] = v.x; + dst_qs[base + (p * 4 + 1) * ne01] = v.y; + dst_qs[base + (p * 4 + 2) * ne01] = v.z; + dst_qs[base + (p * 4 + 3) * ne01] = v.w; + } + + __global uchar * s_dst = dst_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE; + #pragma unroll + for (int i = 0; i < K_SCALE_SIZE; ++i) { + s_dst[i] = b->s[i]; + } +} + +kernel void kernel_restore_block_q5_k_trans4_ns( + __global uint * src_qs, + __global uint * src_qh, + __global half * src_d, + __global half * src_dm, + __global uchar * src_s, + __global struct block_q5_K * dst0, + uint ne00, + uint ne01, + uchar mask_0F, + uchar mask_F0 +) { + uint i00 = get_global_id(1); // block index along K + uint i01 = get_global_id(0); // row index + uint i02 = get_global_id(2); // batch index + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK_K; + + uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + + __global struct block_q5_K * b = dst0 + dst_blk_offset; + + b->d = src_d[src_blk_offset]; + b->dm = src_dm[src_blk_offset]; + + for (int j = 0; j < 32; j++) b->qh[j] = 0; + for (int k = 0; k < 8; k++) { + uint packed = src_qh[i01 + (i00 * 8 + k) * ne01 + i02 * ne00_blk * 8 * ne01]; + uchar b0 = (uchar)(packed & 0xFF); + uchar b1 = (uchar)((packed >> 8) & 0xFF); + uchar b2 = (uchar)((packed >> 16) & 0xFF); + uchar b3 = (uchar)((packed >> 24) & 0xFF); + for (int bit = 0; bit < 8; bit++) { + b->qh[bit] |= (uchar)(((b0 >> bit) & 1) << k); + b->qh[8 + bit] |= (uchar)(((b1 >> bit) & 1) << k); + b->qh[16 + bit] |= (uchar)(((b2 >> bit) & 1) << k); + b->qh[24 + bit] |= (uchar)(((b3 >> bit) & 1) << k); + } + } + + __global uchar * s_src = src_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE; + for (int i = 0; i < K_SCALE_SIZE; ++i) { + b->s[i] = s_src[i]; + } + + uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01; + + uint4 qv[8]; + for (int p = 0; p < 8; ++p) { + qv[p].x = src_qs[base + (p * 4 + 0) * ne01]; + qv[p].y = src_qs[base + (p * 4 + 1) * ne01]; + qv[p].z = src_qs[base + (p * 4 + 2) * ne01]; + qv[p].w = src_qs[base + (p * 4 + 3) * ne01]; + } + + uchar * qv_bytes = (uchar *)qv; + for (int i = 0; i < QK_K / 64; ++i) { + for (int j = 0; j < 16; ++j) { + uchar lo = qv_bytes[i*32 + j]; + uchar hi = qv_bytes[i*32 + j + 16]; + b->qs[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4)); + b->qs[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0)); + } + } +} + +kernel void kernel_convert_block_q6_k_trans4_ns( + __global struct block_q6_K * src0, + __global uint * dst_ql, + __global uint * dst_qh, + __global half * dst_d, + __global char * dst_s, + uint ne00, + uint ne01, + uchar mask_0F, + uchar mask_F0 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK_K; + + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + __global struct block_q6_K * b = src0 + src_blk_offset; + + dst_d[dst_blk_offset] = b->d; + + uint4 qlv[8]; + uchar * qlv_bytes = (uchar *)qlv; + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 16; ++j) { + uchar x0 = b->ql[i*64 + 2*j]; + uchar x1 = b->ql[i*64 + 2*j + 1]; + uchar x2 = b->ql[i*64 + 32 + 2*j]; + uchar x3 = b->ql[i*64 + 32 + 2*j + 1]; + qlv_bytes[i*64 + j ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4); + qlv_bytes[i*64 + j + 16] = convert_uchar(x2 & mask_0F) | convert_uchar((x3 & mask_0F) << 4); + qlv_bytes[i*64 + j + 32] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0); + qlv_bytes[i*64 + j + 48] = convert_uchar((x2 & mask_F0) >> 4) | convert_uchar(x3 & mask_F0); + } + } + + uint ql_base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01; + + #pragma unroll + for (int p = 0; p < 8; ++p) { + uint4 v = qlv[p]; + dst_ql[ql_base + (p * 4 + 0) * ne01] = v.x; + dst_ql[ql_base + (p * 4 + 1) * ne01] = v.y; + dst_ql[ql_base + (p * 4 + 2) * ne01] = v.z; + dst_ql[ql_base + (p * 4 + 3) * ne01] = v.w; + } + + uint qhv[16] = {0}; + + for (int n = 0; n < 2; ++n) { + for (int l = 0; l < 32; ++l) { + uchar h = b->qh[n*32 + l]; + int u = l / 16; + int bit_pos = (l % 16) * 2; + qhv[(n*4 + 0)*2 + u] |= ((uint)((h >> 0) & 0x03)) << bit_pos; + qhv[(n*4 + 1)*2 + u] |= ((uint)((h >> 2) & 0x03)) << bit_pos; + qhv[(n*4 + 2)*2 + u] |= ((uint)((h >> 4) & 0x03)) << bit_pos; + qhv[(n*4 + 3)*2 + u] |= ((uint)((h >> 6) & 0x03)) << bit_pos; + } + } + + uint qh_base = i02 * ne00_blk * ne01 * 16 + i00 * ne01 * 16 + i01; + + for (int p = 0; p < 16; ++p) { + dst_qh[qh_base + p * ne01] = qhv[p]; + } + + __global char * s_dst = dst_s + (i02 * ne01 + i01) * ne00_blk * 16 + i00 * 16; + #pragma unroll + for (int i = 0; i < 16; ++i) { + s_dst[i] = b->scales[i]; + } +} + +kernel void kernel_restore_block_q6_k_trans4_ns( + __global uint * src_ql, + __global uint * src_qh, + __global half * src_d, + __global char * src_s, + __global struct block_q6_K * dst0, + uint ne00, + uint ne01, + uchar mask_0F, + uchar mask_F0 +) { + uint i00 = get_global_id(1); // block index along K + uint i01 = get_global_id(0); // row index + uint i02 = get_global_id(2); // batch index + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK_K; + + uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + + __global struct block_q6_K * b = dst0 + dst_blk_offset; + + b->d = src_d[src_blk_offset]; + + uint ql_base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01; + uint4 qlv[8]; + for (int p = 0; p < 8; ++p) { + qlv[p].x = src_ql[ql_base + (p * 4 + 0) * ne01]; + qlv[p].y = src_ql[ql_base + (p * 4 + 1) * ne01]; + qlv[p].z = src_ql[ql_base + (p * 4 + 2) * ne01]; + qlv[p].w = src_ql[ql_base + (p * 4 + 3) * ne01]; + } -#ifdef ADRENO_GPU - if (get_global_id(0) == 65536*4096) { - printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0)); + uchar * qlv_bytes = (uchar *)qlv; + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 16; ++j) { + uchar lo_02 = qlv_bytes[i*64 + j]; + uchar lo_13 = qlv_bytes[i*64 + j + 16]; + uchar hi_02 = qlv_bytes[i*64 + j + 32]; + uchar hi_13 = qlv_bytes[i*64 + j + 48]; + b->ql[i*64 + 2*j] = convert_uchar((lo_02 & mask_0F) | ((hi_02 & mask_0F) << 4)); + b->ql[i*64 + 2*j + 1] = convert_uchar(((lo_02 & mask_F0) >> 4) | (hi_02 & mask_F0)); + b->ql[i*64 + 32 + 2*j] = convert_uchar((lo_13 & mask_0F) | ((hi_13 & mask_0F) << 4)); + b->ql[i*64 + 32 + 2*j + 1] = convert_uchar(((lo_13 & mask_F0) >> 4) | (hi_13 & mask_F0)); } -#endif } -} -kernel void kernel_restore_block_q4_1_noshuffle( - global uchar * src_q, - global half * src_d, - global half * src_m, - global struct block_q4_1 * dst, - uchar mask_0F, - uchar mask_F0 -) { - global struct block_q4_1 * b = (global struct block_q4_1 *) dst + get_global_id(0); - global uchar * q = (global uchar *) src_q + QK4_1/2*get_global_id(0); - global half * d = (global half *) src_d + get_global_id(0); - global half * m = (global half *) src_m + get_global_id(0); + uint qh_base = i02 * ne00_blk * ne01 * 16 + i00 * ne01 * 16 + i01; + uint qhv[16]; + for (int p = 0; p < 16; ++p) { + qhv[p] = src_qh[qh_base + p * ne01]; + } - b->d = *d; - b->m = *m; - for (int i = 0; i < QK4_1/4; ++i) { - uchar x0 = q[i + 0 ] ; - uchar x1 = q[i + QK4_1/4]; + for (int n = 0; n < 2; ++n) { + for (int l = 0; l < 32; ++l) { + int u = l / 16; + int bit_pos = (l % 16) * 2; + uchar v0 = (uchar)((qhv[(n*4 + 0)*2 + u] >> bit_pos) & 0x03); + uchar v1 = (uchar)((qhv[(n*4 + 1)*2 + u] >> bit_pos) & 0x03); + uchar v2 = (uchar)((qhv[(n*4 + 2)*2 + u] >> bit_pos) & 0x03); + uchar v3 = (uchar)((qhv[(n*4 + 3)*2 + u] >> bit_pos) & 0x03); + b->qh[n*32 + l] = v0 | (v1 << 2) | (v2 << 4) | (v3 << 6); + } + } - b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4)); - b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0)); + __global char * s_src = src_s + (i02 * ne01 + i01) * ne00_blk * 16 + i00 * 16; + for (int i = 0; i < 16; ++i) { + b->scales[i] = s_src[i]; } } @@ -360,6 +1566,200 @@ kernel void kernel_restore_block_mxfp4_trans( b->e = src_e[src_blk_offset]; } +//------------------------------------------------------------------------------ +// kernel_convert_block_q6_k_tiled_ns +// +// Tiled-wide layout for the long-vocab q6_K lm_head/embed GEMV (decode path). +// Unlike *_trans4_ns (which mirrors the bit-interleave the legacy 2-output GEMV +// consumes), this kernel is correct-by-construction against the CANONICAL ggml +// q6_K dequant: it recovers each weight's 6-bit code in element order e in +// [0,256), then re-packs low-4-bits into 32 uints (8 codes/uint) and high-2-bits +// into 16 uints (16 codes/uint). The matching GEMV (gemv_noshuffle_q6_k_f32_tiled) +// unpacks the same order, so both ends are owned here. +// +// Storage is TILED by 64 output rows so the GEMV's 64-thread tile coalesces: +// ql uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk + sb)*8 + g)*64 + rit +// qh uint4 granule g: idx = ((rt*ne00_blk + sb)*4 + g)*64 + rit +// scales (char16) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit +// d (half) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit +// where rt = r/64, rit = r%64. Requires ne01 % 64 == 0 (gated host-side). +// Buffer sizes are byte-identical to the trans4_ns layout. +//------------------------------------------------------------------------------ +kernel void kernel_convert_block_q6_k_tiled_ns( + __global struct block_q6_K * src0, + __global uint * dst_ql, // 32 uints / superblock (low 4 bits, 8 codes/uint) + __global uint * dst_qh, // 16 uints / superblock (high 2 bits, 16 codes/uint) + __global half * dst_d, // 1 half / superblock + __global char * dst_s, // 16 chars/ superblock + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); // K-block index (superblock along ne00) + uint i01 = get_global_id(0); // output row index (along ne01) + uint i02 = get_global_id(2); // batch + + uint ne00_blk = ne00 / QK_K; + + // Source block: row-major over (i02, i01, i00). + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + __global struct block_q6_K * b = src0 + src_blk_offset; + + uint rt = i01 / 64; + uint rit = i01 % 64; + uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00; // tile-major (row-tile, K-block) + + // --- recover canonical 6-bit codes, pack into ql (4b) + qh (2b) in e-order --- + // 32 ql-uints (8 low-nibbles each) + 16 qh-uints (16 2-bit slots each). + uint qlw[32] = {0}; + uint qhw[16] = {0}; + + for (uint e = 0; e < 256; ++e) { + uint n = (e >= 128) ? 1u : 0u; // which 128-half + uint within = e - n * 128u; + uint q = within / 32u; // quadrant 0..3 + uint l = within % 32u; // 0..31 + + uint off_ql = n * 64u; // raw ql byte base for this half + uint off_qh = n * 32u; // raw qh byte base for this half + + uchar low4; + uchar qlb0 = b->ql[off_ql + l]; + uchar qlb1 = b->ql[off_ql + l + 32]; + if (q == 0) low4 = qlb0 & 0x0F; + else if (q == 1) low4 = qlb1 & 0x0F; + else if (q == 2) low4 = (qlb0 >> 4) & 0x0F; + else low4 = (qlb1 >> 4) & 0x0F; + + uchar hi2 = (b->qh[off_qh + l] >> (q * 2u)) & 0x03; + + // pack low4 (e-order): uint e/8, nibble (e%8) + qlw[e >> 3] |= ((uint)low4) << ((e & 7u) * 4u); + // pack hi2 (e-order): uint e/16, 2-bit slot (e%16) + qhw[e >> 4] |= ((uint)hi2) << ((e & 15u) * 2u); + } + + // --- write tiled --- + for (uint g = 0; g < 8; ++g) { + uint base = (tile_blk * 8u + g) * 64u + rit; // uint4 index + dst_ql[base * 4u + 0u] = qlw[g * 4u + 0u]; + dst_ql[base * 4u + 1u] = qlw[g * 4u + 1u]; + dst_ql[base * 4u + 2u] = qlw[g * 4u + 2u]; + dst_ql[base * 4u + 3u] = qlw[g * 4u + 3u]; + } + for (uint g = 0; g < 4; ++g) { + uint base = (tile_blk * 4u + g) * 64u + rit; // uint4 index + dst_qh[base * 4u + 0u] = qhw[g * 4u + 0u]; + dst_qh[base * 4u + 1u] = qhw[g * 4u + 1u]; + dst_qh[base * 4u + 2u] = qhw[g * 4u + 2u]; + dst_qh[base * 4u + 3u] = qhw[g * 4u + 3u]; + } + + // scales: 16 chars contiguous per (row, block), tiled + __global char * s_dst = dst_s + (tile_blk * 64u + rit) * 16u; + #pragma unroll + for (int i = 0; i < 16; ++i) { + s_dst[i] = b->scales[i]; + } + + // super-block scale + dst_d[tile_blk * 64u + rit] = b->d; +} + +kernel void kernel_convert_block_mxfp4_trans4_ns( + global struct block_mxfp4 * src0, + __global uint * dst_q, + __global uchar * dst_e, + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK_MXFP4; + uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + global struct block_mxfp4 * b = src0 + src_blk_offset; + dst_e[dst_blk_offset] = b->e; + + // extract quantization and unshuffle + ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0]; + + ushort8 post_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK_MXFP4 / 4; ++i) { + uchar x0 = pre_block_ptr[2*i + 0]; + uchar x1 = pre_block_ptr[2*i + 1]; + + post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + post_block_ptr[i + QK_MXFP4 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + + uint4 q_block = as_uint4(post_block); + + uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + dst_q[offset] = q_block.x; + dst_q[offset + ne01] = q_block.y; + dst_q[offset + ne01 * 2] = q_block.z; + dst_q[offset + ne01 * 3] = q_block.w; +} + +kernel void kernel_restore_block_mxfp4_trans4_ns( + __global uint * src_q, + __global uchar * src_e, + __global struct block_mxfp4 * dst0, + uint ne00, + uint ne01 +) { + uint i00 = get_global_id(1); + uint i01 = get_global_id(0); + uint i02 = get_global_id(2); + + if (i01 >= ne01) { + return; + } + + uint ne00_blk = ne00 / QK_MXFP4; + uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01; + uint src_d_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01; + + __global struct block_mxfp4 * b = dst0 + dst_blk_offset; + b->e = src_e[src_d_offset]; + + // collect transposed quantization parts for a block + uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01; + uint4 q_block; + q_block.x = src_q[src_q_offset]; + q_block.y = src_q[src_q_offset + ne01]; + q_block.z = src_q[src_q_offset + ne01 * 2]; + q_block.w = src_q[src_q_offset + ne01 * 3]; + + ushort8 post_block = as_ushort8(q_block); + ushort8 pre_block = (ushort8)(0); + + uchar * pre_block_ptr = (uchar *)(&pre_block); + uchar * post_block_ptr = (uchar *)(&post_block); + + for (int i = 0; i < QK_MXFP4 / 4; ++i) { + uchar x0 = post_block_ptr[i + 0]; + uchar x1 = post_block_ptr[i + QK_MXFP4 / 4]; + + pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4); + pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0); + } + + ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block; +} + + //------------------------------------------------------------------------------ // block_q8_0 //------------------------------------------------------------------------------ @@ -399,6 +1799,158 @@ kernel void kernel_restore_block_q8_0( } } +// View-aware AoS q8_0 -> f32 dequant (f32/f32 FA path). +kernel void kernel_dequant_q8_0_f32_view_aos( + global char * src, + ulong src_offset, + ulong src_nb1, + ulong src_nb2, + ulong src_nb3, + int nblk0, + int ne1, + int ne2, + int ne3, + global float * dst +) { + int blk_i0 = get_global_id(0); + int i1 = get_global_id(1); + int batch = get_global_id(2); + + if (blk_i0 >= nblk0) return; + if (i1 >= ne1) return; + + int i2 = batch % ne2; + int i3 = batch / ne2; + if (i3 >= ne3) return; + + global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK8_0); + float d = vload_half(0, (global half *)block); + global char * qs = block + 2; + + ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0; + global float * out = dst + (dst_row_base + blk_i0) * QK8_0; + + for (int i = 0; i < QK8_0; ++i) { + out[i] = d * (float)qs[i]; + } +} + +// View-aware AoS q8_0 -> f16 dequant. Rows tight, batch strides may be gapped. +kernel void kernel_dequant_q8_0_f16_view_aos( + global char * src, + ulong src_offset, + ulong src_nb1, + ulong src_nb2, + ulong src_nb3, + int nblk0, + int ne1, + int ne2, + int ne3, + global half * dst +) { + int blk_i0 = get_global_id(0); + int i1 = get_global_id(1); + int batch = get_global_id(2); + + if (blk_i0 >= nblk0) return; + if (i1 >= ne1) return; + + int i2 = batch % ne2; + int i3 = batch / ne2; + if (i3 >= ne3) return; + + global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK8_0); + float d = vload_half(0, (global half *)block); + global char * qs = block + 2; + + ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0; + global half * out = dst + (dst_row_base + blk_i0) * QK8_0; + + for (int i = 0; i < QK8_0; ++i) { + out[i] = (half)(d * (float)qs[i]); + } +} + +// View-aware AoS q4_0 -> f32 dequant (mirrors the q8_0 view variant). +kernel void kernel_dequant_q4_0_f32_view_aos( + global char * src, + ulong src_offset, + ulong src_nb1, + ulong src_nb2, + ulong src_nb3, + int nblk0, + int ne1, + int ne2, + int ne3, + global float * dst +) { + int blk_i0 = get_global_id(0); + int i1 = get_global_id(1); + int batch = get_global_id(2); + + if (blk_i0 >= nblk0) return; + if (i1 >= ne1) return; + + int i2 = batch % ne2; + int i3 = batch / ne2; + if (i3 >= ne3) return; + + global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK4_0/2); + float d = vload_half(0, (global half *)block); + global uchar * qs = (global uchar *)(block + 2); + + ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0; + global float * out = dst + (dst_row_base + blk_i0) * QK4_0; + + for (int i = 0; i < QK4_0/2; ++i) { + uchar byte = qs[i]; + int q0 = (int)(byte & 0x0F) - 8; + int q1 = (int)(byte >> 4) - 8; + out[i] = d * (float)q0; + out[i + QK4_0/2] = d * (float)q1; + } +} + +// View-aware AoS q4_0 -> f16 dequant (mirrors the q8_0 view variant). +kernel void kernel_dequant_q4_0_f16_view_aos( + global char * src, + ulong src_offset, + ulong src_nb1, + ulong src_nb2, + ulong src_nb3, + int nblk0, + int ne1, + int ne2, + int ne3, + global half * dst +) { + int blk_i0 = get_global_id(0); + int i1 = get_global_id(1); + int batch = get_global_id(2); + + if (blk_i0 >= nblk0) return; + if (i1 >= ne1) return; + + int i2 = batch % ne2; + int i3 = batch / ne2; + if (i3 >= ne3) return; + + global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK4_0/2); + float d = vload_half(0, (global half *)block); + global uchar * qs = (global uchar *)(block + 2); + + ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0; + global half * out = dst + (dst_row_base + blk_i0) * QK4_0; + + for (int i = 0; i < QK4_0/2; ++i) { + uchar byte = qs[i]; + int q0 = (int)(byte & 0x0F) - 8; + int q1 = (int)(byte >> 4) - 8; + out[i] = (half)(d * (float)q0); + out[i + QK4_0/2] = (half)(d * (float)q1); + } +} + kernel void kernel_restore_block_q8_0_trans( global uchar * src_q, global half * src_d, @@ -895,3 +2447,329 @@ kernel void kernel_restore_block_q6_K_noshuffle( b->scales[i] = s[i]; } } + +//------------------------------------------------------------------------------ +// kernel_convert_block_iq4_nl +// Convert the block_iq4_nl format to 2 separate arrays (AOS -> SOA). +//------------------------------------------------------------------------------ +kernel void kernel_convert_block_iq4_nl( + global struct block_iq4_nl * src0, + global uchar * dst_q, + global half * dst_d, + uchar mask_0F, + uchar mask_F0, + ulong n_blk +) { + if (get_global_id(0) >= n_blk) { + return; + } + global struct block_iq4_nl * b = (global struct block_iq4_nl *) src0 + get_global_id(0); + global uchar * q = (global uchar *) dst_q + QK4_NL/2*get_global_id(0); + global half * d = (global half *) dst_d + get_global_id(0); + + *d = b->d; + + for (int i = 0; i < QK4_NL/2; ++i) { + q[i] = b->qs[i]; + } +} + +kernel void kernel_restore_block_iq4_nl( + global uchar * src_q, + global half * src_d, + global struct block_iq4_nl * dst, + ulong n_blk +) { + if (get_global_id(0) >= n_blk) { + return; + } + global struct block_iq4_nl * b = (global struct block_iq4_nl *) dst + get_global_id(0); + global uchar * q = (global uchar *) src_q + QK4_NL/2*get_global_id(0); + global half * d = (global half *) src_d + get_global_id(0); + + b->d = *d; + + for (int i = 0; i < QK4_NL/2; ++i) { + b->qs[i] = q[i]; + } +} + +kernel void kernel_convert_block_iq4_nl_noshuffle( + global struct block_iq4_nl * src0, + global uchar * dst_q, + global half * dst_d, + uchar mask_0F, + uchar mask_F0, + ulong n_blk +) { + if (get_global_id(0) >= n_blk) { + return; + } + global struct block_iq4_nl * b = (global struct block_iq4_nl *) src0 + get_global_id(0); + global uchar * q = (global uchar *) dst_q + QK4_NL/2*get_global_id(0); + global half * d = (global half *) dst_d + get_global_id(0); + + *d = b->d; + for (int i = 0; i < QK4_NL/4; ++i) { + uchar x0 = b->qs[2*i + 0]; + uchar x1 = b->qs[2*i + 1]; + + q[i + 0 ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4); + q[i + QK4_NL/4] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0); + } +} + +kernel void kernel_restore_block_iq4_nl_noshuffle( + global uchar * src_q, + global half * src_d, + global struct block_iq4_nl * dst, + uchar mask_0F, + uchar mask_F0, + ulong n_blk +) { + if (get_global_id(0) >= n_blk) { + return; + } + global struct block_iq4_nl * b = (global struct block_iq4_nl *) dst + get_global_id(0); + global uchar * q = (global uchar *) src_q + QK4_NL/2*get_global_id(0); + global half * d = (global half *) src_d + get_global_id(0); + + b->d = *d; + for (int i = 0; i < QK4_NL/4; ++i) { + uchar x0 = q[i + 0 ]; + uchar x1 = q[i + QK4_NL/4]; + + b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4)); + b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0)); + } +} + +// --------------------------------------------------------------------------- +// kernel_moe_expand_scale_q8_0 +// +// Expand the q8_0 per-32-block scale d (one half/block, [expert][row][block]) into +// the UNIFORM scale[16] format the generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a, +// MOE_QT=80) consumes: 16 f16 per 256-superblock (per-16-element segment), where the +// two segments of each 32-block share the block's d. q8_0 is symmetric -> no min +// buffer (the GEMM runs with has_min=0). The int8 weight codes are reused verbatim +// from the existing flat q8_0 weight buffer (extra0_q8_0->q), so only the scale is +// rebuilt here. One work-item per (row, superblock, expert). +// --------------------------------------------------------------------------- +kernel void kernel_moe_expand_scale_q8_0( + global const half * src_d, // [expert][row][block], one scale per 32-block + global half * dst_scale, // [expert][row][block][2] (FLAT per-32-block) + int ne00, + int ne01 +) { + int row = get_global_id(0); + int blk = get_global_id(1); // 32-block index along K + int e = get_global_id(2); + if (row >= ne01) { return; } + + long nb = ne00 / 32; // 32-blocks per row (K only needs % 32 == 0) + half d = src_d[((long)e*ne01 + row)*nb + blk]; + long b = (((long)e*ne01 + row)*nb + blk) * 2; + dst_scale[b + 0] = d; + dst_scale[b + 1] = d; +} + +// --------------------------------------------------------------------------- +// kernel_moe_expand_scale_q5_0 +// +// q5_0 = symmetric, value = d*(code-16), code = nibble | (hi<<4) in 0..31. The +// generic dp4a MoE GEMM keeps the unsigned code and centers via the min term: +// scale*dp4a(code,a) - min*sum(a), scale = d, min = d*16. +// Reads the existing q5_0 d ([expert][block][row], one half/32-block, from the +// trans4 convert) and writes the FLAT per-32-block uniform scale[2]/min[1] in +// [expert][row][block] order (a transpose). One work-item per (row, block, expert). +// --------------------------------------------------------------------------- +kernel void kernel_moe_expand_scale_q5_0( + global const half * src_d, // [expert][block][row] + global half * dst_scale, // [expert][row][block][2] + global half * dst_min, // [expert][row][block] + int ne00, + int ne01 +) { + int row = get_global_id(0); + int blk = get_global_id(1); + int e = get_global_id(2); + if (row >= ne01) { return; } + + long nb = ne00 / 32; + half d = src_d[(long)e*nb*ne01 + (long)blk*ne01 + row]; // [expert][block][row] + long sb = (((long)e*ne01 + row)*nb + blk) * 2; + long mb = ((long)e*ne01 + row)*nb + blk; + dst_scale[sb + 0] = d; + dst_scale[sb + 1] = d; + dst_min[mb] = (half)((float)d * 16.0f); +} + +// --------------------------------------------------------------------------- +// kernel_moe_expand_scale_q5_K +// +// q5_K value = d*sv*code + (-dm*mn), with the 6-bit packed per-sub-block scale sv +// and min mn (8 sub-blocks of 32 per 256-superblock, decoded by get_scale_min_k4 +// from the 12-byte s[]). The generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a, +// MOE_QT=5) keeps the unsigned 5-bit code and applies scale/min via the uniform +// per-32-block buffers: +// acc += sc0*a_d*raw1 + sc1*a_d*raw2 - mn_u*a_s, +// sc0 = sc1 = d*sv (both per-16 segments of a 32-block share the sub-block scale), +// mn_u = dm*mn (positive; the GEMM subtracts it -> the -dm*mn min term). +// q5_K's q_img (low nibbles) + qh (hi-bit plane) are already in the layout the GEMM +// reads (same trans4_ns convert that feeds gemm_moe_q5_k_f32_ns), so only the scale +// is rebuilt here. +// +// One work-item per (row, superblock, expert); each emits 8 sub-blocks. +// --------------------------------------------------------------------------- +kernel void kernel_moe_expand_scale_q5_K( + global const uchar * src_s, // [expert][row][superblock][12] + global const half * src_d, // [expert][superblock][row] + global const half * src_dm, // [expert][superblock][row] + global half * dst_scale, // [expert][row][32block][2] + global half * dst_min, // [expert][row][32block] + int ne00, + int ne01 +) { + int row = get_global_id(0); + int sb = get_global_id(1); // superblock index along K + int e = get_global_id(2); + if (row >= ne01) { return; } + + long nsb = ne00 / 256; // superblocks per row + long nblk32 = ne00 / 32; // 32-blocks per row + + float d = (float)src_d [((long)e*nsb + sb)*ne01 + row]; + float dm = (float)src_dm[((long)e*nsb + sb)*ne01 + row]; + + __global const uchar * sc = src_s + ((long)e*ne01 + row)*nsb*12 + (long)sb*12; + + for (int j = 0; j < 8; ++j) { + uchar sv, mn; + // get_scale_min_k4 (6-bit packed scale/min for sub-block j of 8) + if (j < 4) { + sv = sc[j] & 63; + mn = sc[j+4] & 63; + } else { + sv = (sc[j+4] & 0x0F) | ((sc[j-4] & 0xC0) >> 2); + mn = ((sc[j+4] >> 4) & 0x0F) | ((sc[j] & 0xC0) >> 2); + } + long sub = (long)sb*8 + j; + long sbase = (((long)e*ne01 + row)*nblk32 + sub) * 2; + half s_val = (half)(d * (float)sv); + dst_scale[sbase + 0] = s_val; + dst_scale[sbase + 1] = s_val; + dst_min[((long)e*ne01 + row)*nblk32 + sub] = (half)(dm * (float)mn); + } +} + +struct block_q2_K { + uint8_t scales[QK_K/16]; // low nibble scale, high nibble min, one per 16 weights + uint8_t qs[QK_K/4]; // quants, 2 bits each + half d; // super-block scale for the quantized scales + half dmin; // super-block scale for the quantized mins +}; + +kernel void kernel_convert_block_q2_K( + global struct block_q2_K * src0, + global uchar * dst_qs, + global uchar * dst_sc, + global half * dst_dm, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q2_K * b = src0 + i; + for (int j = 0; j < 64; ++j) { + dst_qs[64*i + j] = b->qs[j]; + } + for (int j = 0; j < 16; ++j) { + dst_sc[16*i + j] = b->scales[j]; + } + dst_dm[2*i + 0] = b->d; + dst_dm[2*i + 1] = b->dmin; +} + +kernel void kernel_restore_block_q2_K( + global uchar * src_qs, + global uchar * src_sc, + global half * src_dm, + global struct block_q2_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q2_K * b = dst + i; + for (int j = 0; j < 64; ++j) { + b->qs[j] = src_qs[64*i + j]; + } + for (int j = 0; j < 16; ++j) { + b->scales[j] = src_sc[16*i + j]; + } + b->d = src_dm[2*i + 0]; + b->dmin = src_dm[2*i + 1]; +} + +kernel void kernel_convert_block_q2_k_noshuffle( + global struct block_q2_K * src0, + global uchar * dst_qs, // QK_K/4 per block + global uchar * dst_sc, // QK_K/16 per block + global half * dst_dm, // 2 per block (d, dmin) + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q2_K * b = (global struct block_q2_K *) src0 + i; + + dst_dm[2*i + 0] = b->d; + dst_dm[2*i + 1] = b->dmin; + + for (int j = 0; j < QK_K/16; ++j) { dst_sc[(QK_K/16) * i + j] = b->scales[j]; } + + for (int g = 0; g < QK_K/4; ++g) { + uint pk = 0; + for (int t = 0; t < 4; ++t) { + const int e = 4*g + t; + const int sb = e >> 5; + const int ee = e & 31; + pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t)); + } + dst_qs[(QK_K/4) * i + g] = (uchar)pk; + } +} + +kernel void kernel_restore_block_q2_k_noshuffle( + global uchar * src_qs, + global uchar * src_sc, + global half * src_dm, + global struct block_q2_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q2_K * b = (global struct block_q2_K *) dst + i; + + b->d = src_dm[2*i + 0]; + b->dmin = src_dm[2*i + 1]; + + for (int j = 0; j < QK_K/16; ++j) { b->scales[j] = src_sc[(QK_K/16) * i + j]; } + + for (int n2 = 0; n2 < 2; ++n2) { + for (int ee = 0; ee < 32; ++ee) { + uint v = 0; + for (int j = 0; j < 4; ++j) { + const int sb = 4*n2 + j; + const int e = 32*sb + ee; + v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j); + } + b->qs[32*n2 + ee] = (uchar)v; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl index 8f43c4f..cb6fd89 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl @@ -4,13 +4,30 @@ #define ACC_TYPE4 float4 #define DATA_TYPE half #define DATA_TYPE4 half4 -#define CONVERT_ACC4(x) convert_float4(x) -#define CONVERT_DATA4(x) convert_half4(x) +#define CONVERT_ACC4(x) ((float4)((float)(x).s0, (float)(x).s1, (float)(x).s2, (float)(x).s3)) +#define CONVERT_DATA4(x) ((half4)((half)(x).s0, (half)(x).s1, (half)(x).s2, (half)(x).s3)) #define DK_VEC (DK/4) #define DV_VEC (DV/4) #define WG_SIZE (BLOCK_M) -#define Q1_WG_SIZE 64 +// q1 reduces over a Q1_WG_SIZE-wide WG via work-group barriers; the launch WG +// must match. Defaults to the Adreno sg (64); host passes -D FA_SG=32 on Intel. +#ifndef FA_SG +#define FA_SG 64 +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +// Drop full unroll at DK>=192 — Adreno compiler host-memory budget. +#if DK >= 192 +#define FA_UNROLL +#else +#define FA_UNROLL _Pragma("unroll") +#endif inline float get_alibi_slope( const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1 @@ -81,18 +98,18 @@ __kernel void flash_attn_f16( if (my_query_row < n_q) { const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll + FA_UNROLL for (int i = 0; i < DK_VEC; ++i) { q_priv[i] = CONVERT_ACC4(q_ptr[i]); } } ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] = (ACC_TYPE4)(0.0f); } - ACC_TYPE m_i = -INFINITY; + ACC_TYPE m_i = FA_M_INIT; ACC_TYPE l_i = 0.0f; float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); @@ -101,6 +118,17 @@ __kernel void flash_attn_f16( __local DATA_TYPE4 l_v[BLOCK_N][DV_VEC]; for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) { +#if WG_SIZE > FA_SG + // WAR on l_k/l_v: a thread that finishes the compute below early — either + // it skipped it (my_query_row >= n_q, the continue) or its subgroup simply + // ran ahead — wraps around and reloads the tiles while another subgroup is + // still reading them. Any WG that is exactly one lockstep subgroup + // (WG_SIZE == FA_SG) cannot diverge and hides this; a WG spanning multiple + // subgroups (Intel sg=32, or BLOCK_M > 64 on Adreno) corrupts the result. + // All threads reach this each iteration (no-op on the first), so it does + // not diverge with the continue. Compiled out when WG == one subgroup. + barrier(CLK_LOCAL_MEM_FENCE); +#endif for (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) { const int row = i / DK_VEC; const int col = i % DK_VEC; @@ -125,49 +153,71 @@ __kernel void flash_attn_f16( continue; } - for (int j = 0; j < BLOCK_N; j += 2) { + for (int j = 0; j < BLOCK_N; j += 4) { const int k_row0 = k_start + j; const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); - #pragma unroll + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { - dot_acc0 = mad(q_priv[k], CONVERT_ACC4(l_k[j][k]), dot_acc0); - dot_acc1 = mad(q_priv[k], CONVERT_ACC4(l_k[j+1][k]), dot_acc1); + const ACC_TYPE4 qk = q_priv[k]; + dot_acc0 = mad(qk, CONVERT_ACC4(l_k[j][k]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_ACC4(l_k[j+1][k]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_ACC4(l_k[j+2][k]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_ACC4(l_k[j+3][k]), dot_acc3); + } + ACC_TYPE s0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; + ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale; + ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); } - ACC_TYPE score0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; - ACC_TYPE score1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; if (is_causal) { - if (k_row0 > (n_kv - n_q + my_query_row)) score0 = -INFINITY; - if (k_row1 > (n_kv - n_q + my_query_row)) score1 = -INFINITY; + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; } - - if (k_row0 >= n_kv) score0 = -INFINITY; - if (k_row1 >= n_kv) score1 = -INFINITY; + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; if (mask_base != NULL) { const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base + my_query_row * mask_nb1); - if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0]; - if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1]; - } - - if (logit_softcap > 0.0f) { - score0 = logit_softcap * tanh(score0 / logit_softcap); - score1 = logit_softcap * tanh(score1 / logit_softcap); + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE)mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE)mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3]; } - const ACC_TYPE m_new = max(m_i, max(score0, score1)); - const ACC_TYPE p0 = exp(score0 - m_new); - const ACC_TYPE p1 = exp(score1 - m_new); - const ACC_TYPE scale_prev = exp(m_i - m_new); + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p0 = native_exp(s0 - m_new); + const ACC_TYPE p1 = native_exp(s1 - m_new); + const ACC_TYPE p2 = native_exp(s2 - m_new); + const ACC_TYPE p3 = native_exp(s3 - m_new); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { - o_acc[i] = o_acc[i] * scale_prev + p0 * CONVERT_ACC4(l_v[j][i]) + p1 * CONVERT_ACC4(l_v[j+1][i]); + o_acc[i] = mad(p3, CONVERT_ACC4(l_v[j+3][i]), + mad(p2, CONVERT_ACC4(l_v[j+2][i]), + mad(p1, CONVERT_ACC4(l_v[j+1][i]), + mad(p0, CONVERT_ACC4(l_v[j][i]), + o_acc[i] * scale_prev)))); } - l_i = l_i * scale_prev + p0 + p1; + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; m_i = m_new; } } @@ -179,7 +229,7 @@ __kernel void flash_attn_f16( const ACC_TYPE m_final = max(m_i, m_sink); const ACC_TYPE scale_o = exp(m_i - m_final); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] *= scale_o; } @@ -191,12 +241,12 @@ __kernel void flash_attn_f16( global DATA_TYPE4 *o_row = (global DATA_TYPE4 *)(o_base + o_row_offset); if (l_i > 0.0f) { const ACC_TYPE l_inv = 1.0f / l_i; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = CONVERT_DATA4(o_acc[i] * l_inv); } } else { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = (DATA_TYPE4)(0.0f); } @@ -258,7 +308,7 @@ __kernel void flash_attn_f16_q1( ACC_TYPE4 q_priv[DK_VEC]; const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll + FA_UNROLL for (int i = 0; i < DK_VEC; ++i) { q_priv[i] = CONVERT_ACC4(q_ptr[i]); } @@ -270,30 +320,30 @@ __kernel void flash_attn_f16_q1( sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); } - ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : -INFINITY; + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } m_i = max(m_i, score); } __local ACC_TYPE local_m[Q1_WG_SIZE]; local_m[tid] = m_i; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); barrier(CLK_LOCAL_MEM_FENCE); @@ -301,7 +351,7 @@ __kernel void flash_attn_f16_q1( const ACC_TYPE m_final = local_m[0]; ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); ACC_TYPE l_i = 0.0f; @@ -311,21 +361,21 @@ __kernel void flash_attn_f16_q1( const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset); const global DATA_TYPE4* v_ptr = (const global DATA_TYPE4*)(v_base + v_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global DATA_TYPE* mask_ptr = (const global DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } const ACC_TYPE p = exp(score - m_final); l_i += p; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; i++) { o_acc[i] = mad(p, CONVERT_ACC4(v_ptr[i]), o_acc[i]); } @@ -335,7 +385,7 @@ __kernel void flash_attn_f16_q1( __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; local_l[tid] = l_i; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_l[tid] += local_l[tid + s]; barrier(CLK_LOCAL_MEM_FENCE); @@ -354,7 +404,7 @@ __kernel void flash_attn_f16_q1( for (int i = 0; i < DV_VEC; i++) { local_o_comp[tid] = o_acc[i]; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; barrier(CLK_LOCAL_MEM_FENCE); @@ -364,7 +414,7 @@ __kernel void flash_attn_f16_q1( } } } else if (tid == 0) { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_row[i] = (DATA_TYPE4)(0.0f); } } diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl index a6d7479..15b9cf3 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl @@ -11,7 +11,24 @@ #define DK_VEC (DK/4) #define DV_VEC (DV/4) #define WG_SIZE (BLOCK_M) -#define Q1_WG_SIZE 64 +// q1 reduces over a Q1_WG_SIZE-wide WG via work-group barriers; the launch WG +// must match. Defaults to the Adreno sg (64); host passes -D FA_SG=32 on Intel. +#ifndef FA_SG +#define FA_SG 64 +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +// Drop full unroll at DK>=192 — Adreno compiler host-memory budget. +#if DK >= 192 +#define FA_UNROLL +#else +#define FA_UNROLL _Pragma("unroll") +#endif inline float get_alibi_slope( const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1 @@ -82,18 +99,18 @@ __kernel void flash_attn_f32( if (my_query_row < n_q) { const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll + FA_UNROLL for (int i = 0; i < DK_VEC; ++i) { q_priv[i] = CONVERT_ACC4(q_ptr[i]); } } ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] = (ACC_TYPE4)(0.0f); } - ACC_TYPE m_i = -INFINITY; + ACC_TYPE m_i = FA_M_INIT; ACC_TYPE l_i = 0.0f; float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); @@ -102,6 +119,17 @@ __kernel void flash_attn_f32( __local DATA_TYPE4 l_v[BLOCK_N][DV_VEC]; for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) { +#if WG_SIZE > FA_SG + // WAR on l_k/l_v: a thread that finishes the compute below early — either + // it skipped it (my_query_row >= n_q, the continue) or its subgroup simply + // ran ahead — wraps around and reloads the tiles while another subgroup is + // still reading them. Any WG that is exactly one lockstep subgroup + // (WG_SIZE == FA_SG) cannot diverge and hides this; a WG spanning multiple + // subgroups (Intel sg=32, or BLOCK_M > 64 on Adreno) corrupts the result. + // All threads reach this each iteration (no-op on the first), so it does + // not diverge with the continue. Compiled out when WG == one subgroup. + barrier(CLK_LOCAL_MEM_FENCE); +#endif for (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) { const int row = i / DK_VEC; const int col = i % DK_VEC; @@ -126,49 +154,71 @@ __kernel void flash_attn_f32( continue; } - for (int j = 0; j < BLOCK_N; j += 2) { + for (int j = 0; j < BLOCK_N; j += 4) { const int k_row0 = k_start + j; const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); - #pragma unroll + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { - dot_acc0 = mad(q_priv[k], CONVERT_ACC4(l_k[j][k]), dot_acc0); - dot_acc1 = mad(q_priv[k], CONVERT_ACC4(l_k[j+1][k]), dot_acc1); + const ACC_TYPE4 qk = q_priv[k]; + dot_acc0 = mad(qk, CONVERT_ACC4(l_k[j][k]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_ACC4(l_k[j+1][k]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_ACC4(l_k[j+2][k]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_ACC4(l_k[j+3][k]), dot_acc3); + } + ACC_TYPE s0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; + ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale; + ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); } - ACC_TYPE score0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; - ACC_TYPE score1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; if (is_causal) { - if (k_row0 > (n_kv - n_q + my_query_row)) score0 = -INFINITY; - if (k_row1 > (n_kv - n_q + my_query_row)) score1 = -INFINITY; + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; } - - if (k_row0 >= n_kv) score0 = -INFINITY; - if (k_row1 >= n_kv) score1 = -INFINITY; + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); - if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0]; - if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1]; - } - - if (logit_softcap > 0.0f) { - score0 = logit_softcap * tanh(score0 / logit_softcap); - score1 = logit_softcap * tanh(score1 / logit_softcap); + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE)mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE)mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3]; } - const ACC_TYPE m_new = max(m_i, max(score0, score1)); - const ACC_TYPE p0 = exp(score0 - m_new); - const ACC_TYPE p1 = exp(score1 - m_new); - const ACC_TYPE scale_prev = exp(m_i - m_new); + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p0 = native_exp(s0 - m_new); + const ACC_TYPE p1 = native_exp(s1 - m_new); + const ACC_TYPE p2 = native_exp(s2 - m_new); + const ACC_TYPE p3 = native_exp(s3 - m_new); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { - o_acc[i] = o_acc[i] * scale_prev + p0 * CONVERT_ACC4(l_v[j][i]) + p1 * CONVERT_ACC4(l_v[j+1][i]); + o_acc[i] = mad(p3, CONVERT_ACC4(l_v[j+3][i]), + mad(p2, CONVERT_ACC4(l_v[j+2][i]), + mad(p1, CONVERT_ACC4(l_v[j+1][i]), + mad(p0, CONVERT_ACC4(l_v[j][i]), + o_acc[i] * scale_prev)))); } - l_i = l_i * scale_prev + p0 + p1; + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; m_i = m_new; } } @@ -180,7 +230,7 @@ __kernel void flash_attn_f32( const ACC_TYPE m_final = max(m_i, m_sink); const ACC_TYPE scale_o = exp(m_i - m_final); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] *= scale_o; } @@ -192,12 +242,12 @@ __kernel void flash_attn_f32( global DATA_TYPE4 *o_row = (global DATA_TYPE4 *)(o_base + o_row_offset); if (l_i > 0.0f) { const ACC_TYPE l_inv = 1.0f / l_i; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = CONVERT_DATA4(o_acc[i] * l_inv); } } else { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = (DATA_TYPE4)(0.0f); } @@ -259,7 +309,7 @@ __kernel void flash_attn_f32_q1( ACC_TYPE4 q_priv[DK_VEC]; const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; const global DATA_TYPE4* q_ptr = (const global DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll + FA_UNROLL for (int i = 0; i < DK_VEC; ++i) { q_priv[i] = CONVERT_ACC4(q_ptr[i]); } @@ -271,30 +321,30 @@ __kernel void flash_attn_f32_q1( sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); } - ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : -INFINITY; + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } m_i = max(m_i, score); } __local ACC_TYPE local_m[Q1_WG_SIZE]; local_m[tid] = m_i; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); barrier(CLK_LOCAL_MEM_FENCE); @@ -302,7 +352,7 @@ __kernel void flash_attn_f32_q1( const ACC_TYPE m_final = local_m[0]; ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); ACC_TYPE l_i = 0.0f; @@ -312,21 +362,21 @@ __kernel void flash_attn_f32_q1( const global DATA_TYPE4* k_ptr = (const global DATA_TYPE4*)(k_base + k_row_offset); const global DATA_TYPE4* v_ptr = (const global DATA_TYPE4*)(v_base + v_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { dot_acc = mad(q_priv[k], CONVERT_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } const ACC_TYPE p = exp(score - m_final); l_i += p; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; i++) { o_acc[i] = mad(p, CONVERT_ACC4(v_ptr[i]), o_acc[i]); } @@ -336,7 +386,7 @@ __kernel void flash_attn_f32_q1( __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; local_l[tid] = l_i; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_l[tid] += local_l[tid + s]; barrier(CLK_LOCAL_MEM_FENCE); @@ -355,7 +405,7 @@ __kernel void flash_attn_f32_q1( for (int i = 0; i < DV_VEC; i++) { local_o_comp[tid] = o_acc[i]; barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; barrier(CLK_LOCAL_MEM_FENCE); @@ -365,7 +415,7 @@ __kernel void flash_attn_f32_q1( } } } else if (tid == 0) { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_row[i] = (DATA_TYPE4)(0.0f); } } diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl index ec7361b..d1cc148 100644 --- a/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl @@ -1,5 +1,41 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#else +#define REQD_SUBGROUP_SIZE_64 +#endif + +// subgroup size for q1 kernels +#ifndef FA_SG +#define FA_SG 64 +#endif +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define REQD_FA_SG __attribute__((intel_reqd_sub_group_size(FA_SG))) +#else +#define REQD_FA_SG +#endif + +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#elif defined(cl_qcom_subgroup_shuffle) +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR +// name, so calling it is an implicit declaration and the program fails to build. +// Route it to the qcom builtin. +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) +#endif + #define ACC_TYPE float #define ACC_TYPE4 float4 #define Q_DATA_TYPE4 float4 @@ -12,8 +48,37 @@ #define DK_VEC (DK/4) #define DV_VEC (DV/4) + +#ifndef FA_PARTIAL_FLOATS +#define FA_PARTIAL_FLOATS (2 + DV) +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +// Drop full unroll at DK>=192 — Adreno compiler host-memory budget. +#if DK >= 192 +#define FA_UNROLL +#else +#define FA_UNROLL _Pragma("unroll") +#endif + +// N_SPLIT>1 splits DK/DV across threads to cut per-thread register use. +#ifndef N_SPLIT +#define N_SPLIT 1 +#endif + +#define SPLIT_DK_VEC (DK_VEC / N_SPLIT) +#define SPLIT_DV_VEC (DV_VEC / N_SPLIT) + +#if N_SPLIT > 1 +#define WG_SIZE (BLOCK_M * N_SPLIT) +#else #define WG_SIZE (BLOCK_M) -#define Q1_WG_SIZE 64 +#endif inline float get_alibi_slope( const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1 @@ -26,9 +91,20 @@ inline float get_alibi_slope( return pow(base, exph); } -__kernel void flash_attn_f32_f16( + +// Adreno compiler crashes when attempting to compile the entire program for DK=512, +// FA_DECODE_ONLY allows bypass the encoding kernel. +#if !defined(FA_DECODE_ONLY) && !defined(FA_MQ_ONLY) +#ifndef FA_TILE_NAME +#define FA_TILE_NAME flash_attn_f32_f16 +#endif +__kernel void FA_TILE_NAME( const global void * q_void, ulong q_offset, +#ifdef FA_K_IMG + __read_only image1d_buffer_t k_img, ulong k_offset_unused, +#else const global void * k_void, ulong k_offset, +#endif const global void * v_void, ulong v_offset, global void * o_void, ulong o_offset, const float scale, @@ -54,135 +130,510 @@ __kernel void flash_attn_f32_f16( const int mask_ne2, const int mask_ne3, const global void* sinks_void, - const ulong sinks_offset + const ulong sinks_offset, + const global void * k_pad_void, + const global void * v_pad_void, + const global void * mask_pad_void, + const global char * blk, + const int n_kv_blocks, + const ulong mask_pad_nb1, + const ulong mask_pad_nb2, + const ulong mask_pad_nb3 ) { const int tid = get_local_id(0); const int block_q_idx = get_group_id(0); const int head_batch_idx = get_global_id(1); - const int my_query_row = block_q_idx * BLOCK_M + tid; +#if N_SPLIT > 1 + const int q_lane = tid / N_SPLIT; + const int split_idx = tid % N_SPLIT; +#else + const int q_lane = tid; + const int split_idx = 0; +#endif + + const int my_query_row = block_q_idx * BLOCK_M + q_lane; + const int query_valid = my_query_row < n_q; const int batch_idx = head_batch_idx / n_head; const int head_idx = head_batch_idx % n_head; const int gqa_ratio = n_head / n_head_kv; const int head_kv_idx = head_idx / gqa_ratio; + const int mask_head_idx = mask_void != NULL ? head_idx % mask_ne2 : 0; + const int mask_batch_idx = mask_void != NULL ? batch_idx % mask_ne3 : 0; const global char* q_base = (const global char*)q_void + q_offset; +#ifndef FA_K_IMG const global char* k_base = (const global char*)k_void + k_offset; +#endif const global char* v_base = (const global char*)v_void + v_offset; global char* o_base = (global char*)o_void + o_offset; const global char* mask_base = NULL; if (mask_void != NULL) { - const int mask_head_idx = head_idx % mask_ne2; - const int mask_batch_idx = batch_idx % mask_ne3; mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; } + const global char* mask_pad_base = NULL; + if (mask_pad_void != NULL) { + mask_pad_base = (const global char*)mask_pad_void + mask_batch_idx * mask_pad_nb3 + mask_head_idx * mask_pad_nb2; + } + const global char* blk_base = NULL; + if (blk != NULL) { + const int n_q_blocks = (n_q + BLOCK_M - 1) / BLOCK_M; + blk_base = blk + (((mask_batch_idx * mask_ne2) + mask_head_idx) * n_q_blocks + block_q_idx) * n_kv_blocks; + } - ACC_TYPE4 q_priv[DK_VEC]; - if (my_query_row < n_q) { + ACC_TYPE4 q_priv[SPLIT_DK_VEC]; + const int dk_off = split_idx * SPLIT_DK_VEC; + if (query_valid) { const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll - for (int i = 0; i < DK_VEC; ++i) { - q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + FA_UNROLL + for (int i = 0; i < SPLIT_DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[dk_off + i]); + } + } else { + FA_UNROLL + for (int i = 0; i < SPLIT_DK_VEC; ++i) { + q_priv[i] = (ACC_TYPE4)(0.0f); } } - ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll - for (int i = 0; i < DV_VEC; ++i) { + ACC_TYPE4 o_acc[SPLIT_DV_VEC]; + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { o_acc[i] = (ACC_TYPE4)(0.0f); } - ACC_TYPE m_i = -INFINITY; + + ACC_TYPE m_i = FA_M_INIT; ACC_TYPE l_i = 0.0f; float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); +#ifdef FA_K_LDS_T + // K tile transposed: [dk vec][kv row] instead of [kv row][dk vec]. + // + // The QK loop walks 2 or 4 KV rows at a time against the same dk element. Row-major + // those are DK_VEC half4s apart, so each is its own 64-bit local read. Transposed they + // are adjacent, so a pair is one 128-bit read -- half the LDS issues for the same bytes, + // no extra registers, arithmetic untouched. + // + // This kernel looked like it should be FMA-bound (a half4 mad does ~4 ALU ops per LDS + // read, unlike the 1:1 of the dp4a loop), but it is NOT: a wrong-math probe that kept + // every FMA and removed the LDS reads ran it 38.6% faster (18.92 -> 11.62 ms/op). + // Explicitly 16-byte aligned: FA_LK_PAIR below reads two adjacent half4 as one float4, + // and the element type only obliges the compiler to align this array to 8. The indices + // are even so the offset is a multiple of 16, but the base has to be too, and relying + // on the compiler to over-align it is relying on luck. + __local KV_DATA_TYPE4 l_k[DK_VEC][BLOCK_N] __attribute__((aligned(16))); +#define FA_LK(ROW, C) l_k[C][ROW] + // Two adjacent KV rows as one 128-bit local read (half4 pair == 16 B). j is even and + // BLOCK_N is even, so &l_k[c][j] is 16 B past a 16 B-aligned base. +#define FA_LK_PAIR(C, J) as_half8(*(__local const float4 *)(&l_k[C][J])) +#else __local KV_DATA_TYPE4 l_k[BLOCK_N][DK_VEC]; +#define FA_LK(ROW, C) l_k[ROW][C] +#endif __local KV_DATA_TYPE4 l_v[BLOCK_N][DV_VEC]; +#if N_SPLIT > 1 && !defined(HAS_SUBGROUP_SHUFFLE) + __local ACC_TYPE local_partial[BLOCK_N][WG_SIZE]; + __local ACC_TYPE local_p[BLOCK_M][BLOCK_N]; + __local ACC_TYPE local_softmax_scale[BLOCK_M]; + __local ACC_TYPE local_l_inv[BLOCK_M]; +#endif + for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) { + char blk_cur = 1; + if (blk_base != NULL) { + blk_cur = blk_base[k_start / BLOCK_N]; + if (blk_cur == 0) continue; + } + + const int use_kv_pad = k_pad_void != NULL && k_start + BLOCK_N > n_kv; + const int k_tile_start = use_kv_pad ? 0 : k_start; + const ulong k_tile_nb2 = use_kv_pad ? (ulong) BLOCK_N * k_nb1 : k_nb2; + const ulong k_tile_nb3 = use_kv_pad ? (ulong) n_head_kv * k_tile_nb2 : k_nb3; + const ulong v_tile_nb2 = use_kv_pad ? (ulong) BLOCK_N * v_nb1 : v_nb2; + const ulong v_tile_nb3 = use_kv_pad ? (ulong) n_head_kv * v_tile_nb2 : v_nb3; +#ifdef FA_K_IMG + // K via texture cache for the bulk (aligned) tiles; the ragged last + // tile (use_kv_pad) still reads the f32-strided pad buffer from global. + const global char* k_tile_base = use_kv_pad ? (const global char*) k_pad_void : (const global char*) 0; + const int k_pitch_px_row = (int)(k_nb1 >> 3); + const int k_pitch_px_head = (int)(k_nb2 >> 3); + const int k_pitch_px_batch = (int)(k_nb3 >> 3); +#else + const global char* k_tile_base = use_kv_pad ? (const global char*) k_pad_void : k_base; +#endif + const global char* v_tile_base = use_kv_pad ? (const global char*) v_pad_void : v_base; + for (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) { const int row = i / DK_VEC; const int col = i % DK_VEC; - const int k_row_idx = k_start + row; - if (k_row_idx < n_kv) { - const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; - l_k[row][col] = ((__global KV_DATA_TYPE4*)(k_base + k_row_offset))[col]; + const int k_row_idx = k_tile_start + row; + if (use_kv_pad || k_row_idx < n_kv) { +#ifdef FA_K_IMG + if (use_kv_pad) { + const ulong k_row_offset = batch_idx * k_tile_nb3 + head_kv_idx * k_tile_nb2 + k_row_idx * k_nb1; + FA_LK(row, col) = ((__global KV_DATA_TYPE4*)(k_tile_base + k_row_offset))[col]; + } else { + const int k_row_px = batch_idx * k_pitch_px_batch + head_kv_idx * k_pitch_px_head + k_row_idx * k_pitch_px_row; + FA_LK(row, col) = read_imageh(k_img, k_row_px + col); + } +#else + const ulong k_row_offset = batch_idx * k_tile_nb3 + head_kv_idx * k_tile_nb2 + k_row_idx * k_nb1; + FA_LK(row, col) = ((__global KV_DATA_TYPE4*)(k_tile_base + k_row_offset))[col]; +#endif + } else { + FA_LK(row, col) = (KV_DATA_TYPE4)(0.0h); } } for (int i = tid; i < BLOCK_N * DV_VEC; i += WG_SIZE) { const int row = i / DV_VEC; const int col = i % DV_VEC; - const int v_row_idx = k_start + row; - if (v_row_idx < n_kv) { - const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1; - l_v[row][col] = ((__global KV_DATA_TYPE4*)(v_base + v_row_offset))[col]; + const int v_row_idx = k_tile_start + row; + if (use_kv_pad || v_row_idx < n_kv) { + const ulong v_row_offset = batch_idx * v_tile_nb3 + head_kv_idx * v_tile_nb2 + v_row_idx * v_nb1; + l_v[row][col] = ((__global KV_DATA_TYPE4*)(v_tile_base + v_row_offset))[col]; + } else { + l_v[row][col] = (KV_DATA_TYPE4)(0.0h); } } barrier(CLK_LOCAL_MEM_FENCE); - if (my_query_row >= n_q) { - continue; - } +#if N_SPLIT > 1 && defined(HAS_SUBGROUP_SHUFFLE) + { + const int dv_off = split_idx * SPLIT_DV_VEC; + for (int j = 0; j < BLOCK_N; j += 2) { + const int k_row0 = k_start + j; + const int k_row1 = k_start + j + 1; - for (int j = 0; j < BLOCK_N; j += 2) { - const int k_row0 = k_start + j; - const int k_row1 = k_start + j + 1; + ACC_TYPE partial0 = 0.0f; + ACC_TYPE partial1 = 0.0f; + FA_UNROLL + for (int k = 0; k < SPLIT_DK_VEC; k++) { + const ACC_TYPE4 qk = q_priv[k]; +#if defined(FA_K_LDS_T) + // 2 KV rows adjacent in the transposed tile: one 128-bit local read. + const half8 kk = FA_LK_PAIR(dk_off + k, j); + ACC_TYPE4 dot0 = qk * CONVERT_KV_ACC4(kk.lo); + ACC_TYPE4 dot1 = qk * CONVERT_KV_ACC4(kk.hi); +#else + ACC_TYPE4 dot0 = qk * CONVERT_KV_ACC4(l_k[j ][dk_off + k]); + ACC_TYPE4 dot1 = qk * CONVERT_KV_ACC4(l_k[j+1][dk_off + k]); +#endif + partial0 += dot0.s0 + dot0.s1 + dot0.s2 + dot0.s3; + partial1 += dot1.s0 + dot1.s1 + dot1.s2 + dot1.s3; + } - ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); - ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); - #pragma unroll - for (int k = 0; k < DK_VEC; k++) { - dot_acc0 = mad(q_priv[k], CONVERT_KV_ACC4(l_k[j][k]), dot_acc0); - dot_acc1 = mad(q_priv[k], CONVERT_KV_ACC4(l_k[j+1][k]), dot_acc1); - } - ACC_TYPE score0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; - ACC_TYPE score1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + FA_UNROLL + for (int step = 1; step < N_SPLIT; step <<= 1) { + partial0 += sub_group_shuffle_xor(partial0, step); + partial1 += sub_group_shuffle_xor(partial1, step); + } + + ACC_TYPE score0 = partial0 * scale; + ACC_TYPE score1 = partial1 * scale; + if (logit_softcap > 0.0f) { + score0 = logit_softcap * tanh(score0 / logit_softcap); + score1 = logit_softcap * tanh(score1 / logit_softcap); + } + + if (!query_valid) { score0 = FA_M_INIT; score1 = FA_M_INIT; } + if (is_causal) { + if (k_row0 > (n_kv - n_q + my_query_row)) score0 = FA_M_INIT; + if (k_row1 > (n_kv - n_q + my_query_row)) score1 = FA_M_INIT; + } + if (k_row0 >= n_kv) score0 = FA_M_INIT; + if (k_row1 >= n_kv) score1 = FA_M_INIT; - if (is_causal) { - if (k_row0 > (n_kv - n_q + my_query_row)) score0 = -INFINITY; - if (k_row1 > (n_kv - n_q + my_query_row)) score1 = -INFINITY; + if (query_valid && mask_base != NULL && blk_cur != 2) { + if (use_kv_pad && mask_pad_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = + (const global MASK_DATA_TYPE*)(mask_pad_base + my_query_row * mask_pad_nb1); + score0 += slope * (ACC_TYPE)mask_ptr[j]; + score1 += slope * (ACC_TYPE)mask_ptr[j + 1]; + } else { + const global MASK_DATA_TYPE* mask_ptr = + (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); + if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0]; + if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + } + } + + const ACC_TYPE m_new = max(m_i, max(score0, score1)); + // Whole tile masked (m_new == FA_M_INIT): force the exp() args + // far negative so the tile contributes 0, not exp(0)=1. + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE sp = native_exp(m_i - m_exp); + const ACC_TYPE p0 = native_exp(score0 - m_exp); + const ACC_TYPE p1 = native_exp(score1 - m_exp); + + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_acc[i] = o_acc[i] * sp + + p0 * CONVERT_KV_ACC4(l_v[j ][dv_off + i]) + + p1 * CONVERT_KV_ACC4(l_v[j+1][dv_off + i]); + } + l_i = l_i * sp + p0 + p1; + m_i = m_new; + } + } +#elif N_SPLIT > 1 + // N_SPLIT>1 fallback (no shuffle): 3-phase local-memory reduction. + // Phase 1 — partial dots for all BLOCK_N tokens. + for (int j = 0; j < BLOCK_N; ++j) { + ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); + FA_UNROLL + for (int k = 0; k < SPLIT_DK_VEC; k++) { + dot_acc = mad(q_priv[k], CONVERT_KV_ACC4(FA_LK(j, dk_off + k)), dot_acc); } + local_partial[j][tid] = + dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3; + } + barrier(CLK_LOCAL_MEM_FENCE); // 1 barrier: partial dots visible + + // Phase 2 — split_idx==0 reduces partial sums and computes block softmax. + if (split_idx == 0) { + if (query_valid) { + ACC_TYPE m_new = m_i; + for (int j = 0; j < BLOCK_N; ++j) { + const int k_row = k_start + j; + ACC_TYPE score = 0.0f; + FA_UNROLL + for (int s = 0; s < N_SPLIT; s++) { + score += local_partial[j][q_lane * N_SPLIT + s]; + } + score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + if (is_causal && k_row > (n_kv - n_q + my_query_row)) score = FA_M_INIT; + if (k_row >= n_kv) score = FA_M_INIT; + + if (mask_base != NULL && blk_cur != 2) { + if (use_kv_pad && mask_pad_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = + (const global MASK_DATA_TYPE*)(mask_pad_base + my_query_row * mask_pad_nb1); + score += slope * (ACC_TYPE)mask_ptr[j]; + } else { + const global MASK_DATA_TYPE* mask_ptr = + (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); + if (k_row < n_kv) score += slope * (ACC_TYPE)mask_ptr[k_row]; + } + } - if (k_row0 >= n_kv) score0 = -INFINITY; - if (k_row1 >= n_kv) score1 = -INFINITY; + m_new = max(m_new, score); + local_p[q_lane][j] = score; + } - if (mask_base != NULL) { - const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); - if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0]; - if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE sp = native_exp(m_i - m_exp); + ACC_TYPE l_new = l_i * sp; + for (int j = 0; j < BLOCK_N; ++j) { + const ACC_TYPE p = native_exp(local_p[q_lane][j] - m_exp); + local_p[q_lane][j] = p; + l_new += p; + } + local_softmax_scale[q_lane] = sp; + l_i = l_new; + m_i = m_new; + } else { + local_softmax_scale[q_lane] = 1.0f; + for (int j = 0; j < BLOCK_N; ++j) local_p[q_lane][j] = 0.0f; } + } + barrier(CLK_LOCAL_MEM_FENCE); - if (logit_softcap > 0.0f) { - score0 = logit_softcap * tanh(score0 / logit_softcap); - score1 = logit_softcap * tanh(score1 / logit_softcap); + // Phase 3 — V accumulate using broadcast probabilities. + { + const ACC_TYPE sp_block = local_softmax_scale[q_lane]; + const int dv_off = split_idx * SPLIT_DV_VEC; + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_acc[i] *= sp_block; + } + for (int j = 0; j < BLOCK_N; ++j) { + const ACC_TYPE p = local_p[q_lane][j]; + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_acc[i] = mad(p, CONVERT_KV_ACC4(l_v[j][dv_off + i]), o_acc[i]); + } } + } +#else + // N_SPLIT==1: j+=4 unroll. Requires BLOCK_N % 4 == 0. + if (query_valid) { + for (int j = 0; j < BLOCK_N; j += 4) { + const int k_row0 = k_start + j; + const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; - const ACC_TYPE m_new = max(m_i, max(score0, score1)); - const ACC_TYPE p0 = exp(score0 - m_new); - const ACC_TYPE p1 = exp(score1 - m_new); - const ACC_TYPE scale_prev = exp(m_i - m_new); + ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + FA_UNROLL + for (int k = 0; k < DK_VEC; k++) { + const ACC_TYPE4 qk = q_priv[k]; +#if defined(FA_K_LDS_T) + // 4 KV rows adjacent in the transposed tile: two 128-bit local reads + // instead of four 64-bit ones. + const half8 kk01 = FA_LK_PAIR(k, j); + const half8 kk23 = FA_LK_PAIR(k, j + 2); + dot_acc0 = mad(qk, CONVERT_KV_ACC4(kk01.lo), dot_acc0); + dot_acc1 = mad(qk, CONVERT_KV_ACC4(kk01.hi), dot_acc1); + dot_acc2 = mad(qk, CONVERT_KV_ACC4(kk23.lo), dot_acc2); + dot_acc3 = mad(qk, CONVERT_KV_ACC4(kk23.hi), dot_acc3); +#else + dot_acc0 = mad(qk, CONVERT_KV_ACC4(l_k[j][k]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_KV_ACC4(l_k[j+1][k]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_KV_ACC4(l_k[j+2][k]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_KV_ACC4(l_k[j+3][k]), dot_acc3); +#endif + } + ACC_TYPE s0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale; + ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale; + ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale; + ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } - #pragma unroll - for (int i = 0; i < DV_VEC; ++i) { - o_acc[i] = o_acc[i] * scale_prev + p0 * CONVERT_KV_ACC4(l_v[j][i]) + p1 * CONVERT_KV_ACC4(l_v[j+1][i]); + if (is_causal) { + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; + } + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; + + if (mask_base != NULL && blk_cur != 2) { + if (use_kv_pad && mask_pad_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_pad_base + my_query_row * mask_pad_nb1); + s0 += slope * (ACC_TYPE)mask_ptr[j]; + s1 += slope * (ACC_TYPE)mask_ptr[j + 1]; + s2 += slope * (ACC_TYPE)mask_ptr[j + 2]; + s3 += slope * (ACC_TYPE)mask_ptr[j + 3]; + } else { + const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1); + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE)mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE)mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE)mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3]; + } + } + + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + // Whole tile masked (m_new == FA_M_INIT): force the exp() args + // far negative so the tile contributes 0, not exp(0)=1. + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE scale_prev = native_exp(m_i - m_exp); + const ACC_TYPE p0 = native_exp(s0 - m_exp); + const ACC_TYPE p1 = native_exp(s1 - m_exp); + const ACC_TYPE p2 = native_exp(s2 - m_exp); + const ACC_TYPE p3 = native_exp(s3 - m_exp); + + FA_UNROLL + for (int i = 0; i < DV_VEC; ++i) { + o_acc[i] = mad(p3, CONVERT_KV_ACC4(l_v[j+3][i]), + mad(p2, CONVERT_KV_ACC4(l_v[j+2][i]), + mad(p1, CONVERT_KV_ACC4(l_v[j+1][i]), + mad(p0, CONVERT_KV_ACC4(l_v[j][i]), + o_acc[i] * scale_prev)))); + } + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; + m_i = m_new; + } + } +#endif + // End of tile: every thread must finish reading l_k/l_v before the + // next iteration's load overwrites them (WAR hazard on local memory). + barrier(CLK_LOCAL_MEM_FENCE); + } + + // Write output. +#if N_SPLIT > 1 && defined(HAS_SUBGROUP_SHUFFLE) + if (query_valid) { + ACC_TYPE sinks_sp = 1.0f; + if (sinks_void != NULL) { + const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); + const ACC_TYPE m_sink = sinks_ptr[head_idx]; + const ACC_TYPE m_final = max(m_i, m_sink); + sinks_sp = exp(m_i - m_final); + l_i = l_i * sinks_sp + exp(m_sink - m_final); + m_i = m_final; + } + const ACC_TYPE l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f; + const int dv_off = split_idx * SPLIT_DV_VEC; + const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1; + global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); + if (l_inv > 0.0f) { + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_row[dv_off + i] = CONVERT_O_DATA4(o_acc[i] * sinks_sp * l_inv); + } + } else { + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_row[dv_off + i] = (O_DATA_TYPE4)(0.0f); } - l_i = l_i * scale_prev + p0 + p1; - m_i = m_new; } } +#elif N_SPLIT > 1 + if (split_idx == 0) { + ACC_TYPE sinks_sp = 1.0f; + if (query_valid && sinks_void != NULL) { + const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); + const ACC_TYPE m_sink = sinks_ptr[head_idx]; + const ACC_TYPE m_final = max(m_i, m_sink); + sinks_sp = exp(m_i - m_final); + l_i = l_i * sinks_sp + exp(m_sink - m_final); + m_i = m_final; + } + local_softmax_scale[q_lane] = sinks_sp; + local_l_inv[q_lane] = (query_valid && l_i > 0.0f) ? (1.0f / l_i) : 0.0f; + } + barrier(CLK_LOCAL_MEM_FENCE); - if (my_query_row < n_q) { + if (query_valid) { + const ACC_TYPE sinks_sp = local_softmax_scale[q_lane]; + const ACC_TYPE l_inv = local_l_inv[q_lane]; + const int dv_off = split_idx * SPLIT_DV_VEC; + const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1; + global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); + if (l_inv > 0.0f) { + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_row[dv_off + i] = CONVERT_O_DATA4(o_acc[i] * sinks_sp * l_inv); + } + } else { + FA_UNROLL + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + o_row[dv_off + i] = (O_DATA_TYPE4)(0.0f); + } + } + } +#else + if (query_valid) { if (sinks_void != NULL) { const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); const ACC_TYPE m_sink = sinks_ptr[head_idx]; const ACC_TYPE m_final = max(m_i, m_sink); const ACC_TYPE scale_o = exp(m_i - m_final); - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_acc[i] *= scale_o; } @@ -194,19 +645,25 @@ __kernel void flash_attn_f32_f16( global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); if (l_i > 0.0f) { const ACC_TYPE l_inv = 1.0f / l_i; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = CONVERT_O_DATA4(o_acc[i] * l_inv); } } else { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) { o_row[i] = (O_DATA_TYPE4)(0.0f); } } } +#endif } +#endif // !FA_DECODE_ONLY +// allow bypassing decode kernels to avoid compiler crash for DK=512 on Adreno GPUs +#ifndef FA_PREFILL_ONLY +#if !defined(FA_MQ_ONLY) && !defined(FA_VEC_ONLY) +REQD_FA_SG __kernel void flash_attn_f32_f16_q1( const global void * q_void, ulong q_offset, const global void * k_void, ulong k_offset, @@ -247,7 +704,9 @@ __kernel void flash_attn_f32_f16_q1( const int head_kv_idx = head_idx / gqa_ratio; const global char* q_base = (const global char*)q_void + q_offset; +#ifndef FA_K_IMG const global char* k_base = (const global char*)k_void + k_offset; +#endif const global char* v_base = (const global char*)v_void + v_offset; global char* o_base = (global char*)o_void + o_offset; @@ -258,13 +717,16 @@ __kernel void flash_attn_f32_f16_q1( mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; } - ACC_TYPE4 q_priv[DK_VEC]; + // Q is uniform across WG threads (n_q=1). Share via local memory to + // avoid per-thread q_priv[DK_VEC] dynamic-indexed private array that + // spills to DDR on Adreno. + __local ACC_TYPE4 q_shared[DK_VEC]; const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset); - #pragma unroll - for (int i = 0; i < DK_VEC; ++i) { - q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + for (int i = tid; i < DK_VEC; i += Q1_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); } + sub_group_barrier(CLK_LOCAL_MEM_FENCE); float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); @@ -273,38 +735,30 @@ __kernel void flash_attn_f32_f16_q1( sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); } - ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : -INFINITY; + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; const global KV_DATA_TYPE4* k_ptr = (const global KV_DATA_TYPE4*)(k_base + k_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { - dot_acc = mad(q_priv[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); + dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } m_i = max(m_i, score); } - __local ACC_TYPE local_m[Q1_WG_SIZE]; - local_m[tid] = m_i; - barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll - for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { - if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); - barrier(CLK_LOCAL_MEM_FENCE); - } - const ACC_TYPE m_final = local_m[0]; + const ACC_TYPE m_final = sub_group_reduce_max(m_i); ACC_TYPE4 o_acc[DV_VEC]; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); ACC_TYPE l_i = 0.0f; @@ -314,39 +768,32 @@ __kernel void flash_attn_f32_f16_q1( const global KV_DATA_TYPE4* k_ptr = (const global KV_DATA_TYPE4*)(k_base + k_row_offset); const global KV_DATA_TYPE4* v_ptr = (const global KV_DATA_TYPE4*)(v_base + v_row_offset); ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); - #pragma unroll + FA_UNROLL for (int k = 0; k < DK_VEC; k++) { - dot_acc = mad(q_priv[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); + dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); } ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } if (mask_base != NULL) { const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); score += slope * (ACC_TYPE)mask_ptr[k_idx]; } - if (logit_softcap > 0.0f) { - score = logit_softcap * tanh(score / logit_softcap); - } const ACC_TYPE p = exp(score - m_final); l_i += p; - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; i++) { o_acc[i] = mad(p, CONVERT_KV_ACC4(v_ptr[i]), o_acc[i]); } } - __local ACC_TYPE local_l[Q1_WG_SIZE]; __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; - local_l[tid] = l_i; - barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll - for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { - if (tid < s) local_l[tid] += local_l[tid + s]; - barrier(CLK_LOCAL_MEM_FENCE); - } + const ACC_TYPE l_red = sub_group_reduce_add(l_i); const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); - ACC_TYPE l_final = local_l[0]; + ACC_TYPE l_final = l_red; if (sinks_ptr != NULL) { l_final += exp(sinks_ptr[head_idx] - m_final); @@ -356,18 +803,1973 @@ __kernel void flash_attn_f32_f16_q1( const ACC_TYPE l_inv = 1.0f / l_final; for (int i = 0; i < DV_VEC; i++) { local_o_comp[tid] = o_acc[i]; - barrier(CLK_LOCAL_MEM_FENCE); - #pragma unroll + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + FA_UNROLL for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; - barrier(CLK_LOCAL_MEM_FENCE); + sub_group_barrier(CLK_LOCAL_MEM_FENCE); } if (tid == 0) { o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv); } } } else if (tid == 0) { - #pragma unroll + FA_UNROLL for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f); } } + +#endif // !FA_MQ_ONLY (q1) +// decode variant for large DV (e.g. Gemma-4 DK=DV=512 global layers). +#define VEC_NSG 4 +#define VEC_WG_SIZE (Q1_WG_SIZE * VEC_NSG) +#define Q1V_DV_PER_THREAD ((DV_VEC + Q1_WG_SIZE - 1) / Q1_WG_SIZE) + +// allow bypassing the kernel to avoid compiler crash for DK=512 on Adreno GPUs +#if !defined(FA_DECODE_MINIMAL) && !defined(FA_MQ_ONLY) +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_vec( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; // subgroup index (0..VEC_NSG-1) + const int tid_sg = tid % Q1_WG_SIZE; // lane within subgroup + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + // Q is uniform across the WG — stage in __local once. All WG threads load. + __local ACC_TYPE4 q_shared[DK_VEC]; + { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + for (int i = tid; i < DK_VEC; i += VEC_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE * sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + } + + // per-thread DV slice within its subgroup + // DV=512 -> 2x float4 = 32 bytes; DV=256 -> 1x float4 - no spill + ACC_TYPE4 o_acc[Q1V_DV_PER_THREAD]; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + // each subgroup independently runs the FA-2 online softmax over its slice of n_kv. + // sinks are not folded into per-subgroup m_i — they're added once in + // the cross-subgroup merge to avoid double-counting. + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + const int kv_per_sg = (n_kv + VEC_NSG - 1) / VEC_NSG; + const int kv_start = sgid * kv_per_sg; + const int kv_end = min(n_kv, kv_start + kv_per_sg); + + for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_off); + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off); + + // Q*K^T: each thread accumulates its DK slice; subgroup-reduce the partial. + ACC_TYPE4 dot4 = (ACC_TYPE4)(0.0f); + for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) { + dot4 = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot4); + } + ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3; + ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + + // FA-2 online update. All threads in the subgroup see the same score, + // so m_i and l_i evolve identically across lanes within the subgroup. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p = native_exp(score - m_new); + + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + o_acc[idx] = mad(p, CONVERT_KV_ACC4(v_ptr[dv_idx]), o_acc[idx] * scale_prev); + } + l_i = l_i * scale_prev + p; + m_i = m_new; + } + + // Cross-subgroup merge via __local. Each subgroup publishes (m_i, l_i) + // and its o_acc slice; subgroup 0 then folds them into the final norm + // and writes the row. + __local ACC_TYPE sg_m[VEC_NSG]; + __local ACC_TYPE sg_l[VEC_NSG]; + __local ACC_TYPE4 sg_o[VEC_NSG][DV_VEC]; + + if (tid_sg == 0) { + sg_m[sgid] = m_i; + sg_l[sgid] = l_i; + } + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + // m_final = max over all subgroups' m_i, plus the sink (if any). + ACC_TYPE m_final = sg_m[0]; + #pragma unroll + for (int s = 1; s < VEC_NSG; ++s) { + m_final = max(m_final, sg_m[s]); + } + if (sinks_ptr != NULL) { + m_final = max(m_final, sinks_ptr[head_idx]); + } + + ACC_TYPE l_final = 0.0f; + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + l_final += sg_l[s] * native_exp(sg_m[s] - m_final); + } + if (sinks_ptr != NULL) { + l_final += native_exp(sinks_ptr[head_idx] - m_final); + } + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset); + + // Each thread in subgroup 0 writes its DV slice, folding all subgroups' + // contributions with the rescale factor. + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[s] - m_final); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + o_row[dv_idx] = CONVERT_O_DATA4(o_merged * l_inv); + } + } +} + +#endif // !FA_DECODE_MINIMAL + +#ifndef FA_DECODE_ONLY + +// flash_attn_f32_f16_q1_local_tile +// one WG per (q_idx, q_head) + +#define LT_KC 32 +#define LT_WG 128 + +#ifndef FA_MQ_ONLY // q1_local_tile excluded from the MQ-only (g8) program +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_local_tile( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void * sinks_void, + const ulong sinks_offset +) { + const int q_idx = get_global_id(0) / LT_WG; + const int head_idx = get_global_id(1); + const int batch_idx = get_global_id(2); + const int tid = get_local_id(0); + + const int gqa_ratio = n_head_kv > 0 ? (n_head / n_head_kv) : 1; + const int head_kv_idx = head_idx / gqa_ratio; + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + __local half k_tile[LT_KC * DK]; // 32*128*2 = 8 KB at DK=128 + __local half v_tile[LT_KC * DV]; // 8 KB + __local float red[LT_WG]; // 512 B reduction scratch + __local float score_shared; // broadcast score (each K-step) + + // Each thread owns one float of Q at index `tid` (assumes LT_WG == DK). + const global char * q_row_base = (const global char *) q_void + q_offset + + batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + float q_val = ((const global float *) q_row_base)[tid]; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 + + (ulong) q_idx * mask_nb1; + } + + float o_val = 0.0f; + float m_i = FA_M_INIT; + float l_i = 0.0f; + + for (int kb = 0; kb < n_kv; kb += LT_KC) { + const int tile_len = min(LT_KC, n_kv - kb); + + // Stage K and V tiles into __local. + for (int i = tid; i < tile_len * DK; i += LT_WG) { + const int j = i / DK; + const int d = i % DK; + const int kv_idx = kb + j; + const global char * k_row = (const global char *) k_void + k_offset + + batch_idx * k_nb3 + head_kv_idx * k_nb2 + + (ulong) kv_idx * k_nb1; + const global char * v_row = (const global char *) v_void + v_offset + + batch_idx * v_nb3 + head_kv_idx * v_nb2 + + (ulong) kv_idx * v_nb1; + k_tile[j * DK + d] = ((const global half *) k_row)[d]; + v_tile[j * DV + d] = ((const global half *) v_row)[d]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + for (int j = 0; j < tile_len; ++j) { + const int kv_idx = kb + j; + + // Q·K dot via __local tree-reduce. + red[tid] = q_val * convert_float(k_tile[j * DK + tid]); + barrier(CLK_LOCAL_MEM_FENCE); + for (int stride = LT_WG >> 1; stride > 0; stride >>= 1) { + if (tid < stride) { + red[tid] += red[tid + stride]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (tid == 0) { + float s = red[0] * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; + s += slope * (float) mask_ptr[kv_idx]; + } + score_shared = s; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const float s = score_shared; + const float m_new = fmax(m_i, s); + const float alpha = native_exp(m_i - m_new); + const float beta = native_exp(s - m_new); + + o_val = o_val * alpha + beta * convert_float(v_tile[j * DV + tid]); + l_i = l_i * alpha + beta; + m_i = m_new; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + // Fold attention sinks into the running (m, l, o), if present. + if (sinks_void != NULL) { + const global float * sinks_ptr = + (const global float *) ((const global char *) sinks_void + sinks_offset); + const float m_sink = sinks_ptr[head_idx]; + const float m_new = fmax(m_i, m_sink); + const float alpha = native_exp(m_i - m_new); + const float beta = native_exp(m_sink - m_new); + o_val = o_val * alpha; + l_i = l_i * alpha + beta; + m_i = m_new; + } + + const float l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f; + global float * o_row = (global float *) ((global char *) o_void + o_offset + + batch_idx * o_nb3 + head_idx * o_nb1 + + (ulong) q_idx * o_nb2); + o_row[tid] = o_val * l_inv; +} + +// flash_attn_f32_f16_q1_local_mq_split + +#define LMQ_WG 64 +#define LMQ_KC 32 +#define LMQ_DPL 2 // DK / LMQ_WG at DK=128 + +#endif // !FA_MQ_ONLY (q1_local_tile) +#ifndef MQ_GQA +#define MQ_GQA 4 +#endif + +#ifndef FA_PARTIAL_FLOATS +#define FA_PARTIAL_FLOATS (2 + DV) +#endif + +#ifndef FA_MQ_ONLY // q1_local_mq_split excluded from the MQ-only (g8) program +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_local_mq_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); // 0..LMQ_WG-1 + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + // Empty split — write sentinel for each Q-head so merge treats it as 0. + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + // Stage MQ_GQA Q rows in __local (MQ_GQA × DK floats). + __local float q_shared[MQ_GQA * DK]; + for (int i = tid; i < MQ_GQA * DK; i += LMQ_WG) { + const int h = i / DK; + const int d = i % DK; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_off = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global float * q_ptr = (const global float *) (q_base + q_row_off); + q_shared[h * DK + d] = q_ptr[d]; + } + + // K/V tile staging buffers (16 KB combined at DK=DV=128 KC=32). + __local half k_tile[LMQ_KC * DK]; + __local half v_tile[LMQ_KC * DV]; + + // Per-h state held in private registers. + float o_acc[MQ_GQA][LMQ_DPL]; + float m_i[MQ_GQA]; + float l_i[MQ_GQA]; + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + #pragma unroll + for (int p = 0; p < LMQ_DPL; ++p) o_acc[h][p] = 0.0f; + } + + // Per-h mask pointers. + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + barrier(CLK_LOCAL_MEM_FENCE); // Ensure Q staged before first dot. + + for (int kb = kv_start; kb < kv_end; kb += LMQ_KC) { + const int tile_len = min((int) LMQ_KC, kv_end - kb); + + // Cooperative load K + V tile. + for (int i = tid; i < tile_len * DK; i += LMQ_WG) { + const int j = i / DK; + const int d = i % DK; + const int kv_idx = kb + j; + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + (ulong) kv_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + (ulong) kv_idx * v_nb1; + k_tile[j * DK + d] = ((const global half *) k_row)[d]; + v_tile[j * DV + d] = ((const global half *) v_row)[d]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + // Process each cache row in the tile. + for (int j = 0; j < tile_len; ++j) { + const int kv_idx = kb + j; + + // Dot product per h: lane owns LMQ_DPL D-elements at (tid*LMQ_DPL..). + float score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + float contrib = 0.0f; + #pragma unroll + for (int p = 0; p < LMQ_DPL; ++p) { + const int d = tid * LMQ_DPL + p; + contrib += q_shared[h * DK + d] * (float) k_tile[j * DK + d]; + } + float s = sub_group_reduce_add(contrib) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (float) mask_ptr[kv_idx]; + } + score[h] = s; + } + + // Online softmax update + V accumulation per h. + float p_h[MQ_GQA]; + float sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const float m_new = fmax(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + #pragma unroll + for (int p = 0; p < LMQ_DPL; ++p) { + const int d = tid * LMQ_DPL + p; + const float v_val = (float) v_tile[j * DV + d]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][p] = o_acc[h][p] * sp_h[h] + p_h[h] * v_val; + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); // Before next tile load overwrites k/v_tile. + } + + // write partial records: one per (h, split) + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float * rec_o = rec + 2; + + if (tid == 0) { + rec[0] = m_i[h]; + rec[1] = l_i[h]; + } + #pragma unroll + for (int p = 0; p < LMQ_DPL; ++p) { + const int d = tid * LMQ_DPL + p; + rec_o[d] = o_acc[h][p]; + } + } +} + +#endif // !FA_MQ_ONLY (q1_local_mq_split) +#ifndef MQ_NSG +#define MQ_NSG 4 +#endif +#define MQ_WG_SIZE (Q1_WG_SIZE * MQ_NSG) + +#ifndef FA_MQ_SPLIT_ONLY +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_vec_mq( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; // subgroup 0..MQ_NSG-1 + const int tid_sg = tid % Q1_WG_SIZE; // lane 0..63 + const int kvhead_batch_idx = get_global_id(1); + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_WG_SIZE) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + // per-h ALiBi slope + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + // per-h mask row pointer + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + const global ACC_TYPE * sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + } + + // per-thread per-h DV slice. + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + // each subgroup independently sweeps its slice of n_kv. + const int kv_per_sg = (n_kv + MQ_NSG - 1) / MQ_NSG; + const int kv_start = sgid * kv_per_sg; + const int kv_end = min(n_kv, kv_start + kv_per_sg); + + for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_off); + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off); + + // Q*K^T: load each K stride once, dot against all MQ_GQA Q rows. + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) { + const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[k]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]); + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + score[h] = s; + } + + // FA-2 online softmax update — V load amortized across MQ_GQA heads. + // p, scale_prev are computed per h; the V vector is loaded once + // per dv stride and reused MQ_GQA times. + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[dv_idx]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_vec, o_acc[h][idx] * sp_h[h]); + } + } + } + + // cross subgroup merge + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG]; + __local ACC_TYPE4 sg_o[MQ_NSG][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + // each subgroup publishes its o_acc slice for head h. + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_final = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG; ++s) { + m_final = max(m_final, sg_m[h][s]); + } + if (sinks_ptr != NULL) { + m_final = max(m_final, sinks_ptr[head_idx]); + } + + ACC_TYPE l_final = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG; ++s) { + l_final += sg_l[h][s] * native_exp(sg_m[h][s] - m_final); + } + if (sinks_ptr != NULL) { + l_final += native_exp(sinks_ptr[head_idx] - m_final); + } + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset); + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_final); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + o_row[dv_idx] = CONVERT_O_DATA4(o_merged * l_inv); + } + } + // Barrier guards next h's overwrite of sg_o. + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +#endif // !FA_MQ_SPLIT_ONLY + +#ifndef MQ_NSG_SPLIT +#define MQ_NSG_SPLIT 4 +#endif +#define MQ_SPLIT_WG_SIZE (Q1_WG_SIZE * MQ_NSG_SPLIT) + +#ifndef FA_PARTIAL_FLOATS +#define FA_PARTIAL_FLOATS (2 + DV) +#endif + +#ifndef FA_HEAD_SUB +#define FA_HEAD_SUB 1 +#endif + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_vec_mq_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int hgroups = n_head_kv * FA_HEAD_SUB; + const int batch_idx = kvhead_batch_idx / hgroups; + const int hg = kvhead_batch_idx % hgroups; + const int head_kv_idx = hg / FA_HEAD_SUB; + const int head_sub = hg % FA_HEAD_SUB; +#define FA_MQS_HEAD_IDX(h) (head_kv_idx * (MQ_GQA * FA_HEAD_SUB) + head_sub * MQ_GQA + (h)) + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + // write sentinel for each of the MQ_GQA Q-heads so the + // merge pass treats this slot as dropped + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = FA_MQS_HEAD_IDX(h); + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + +#ifdef FA_Q1_Q_REG + ACC_TYPE4 q_reg[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = FA_MQS_HEAD_IDX(h); + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_reg[h] = (tid_sg < DK_VEC) ? CONVERT_Q_ACC4(q_ptr[tid_sg]) : (ACC_TYPE4)(0.0f); + } +#else + // stage MQ_GQA Q rows in __local once (uniform across WG) + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = FA_MQS_HEAD_IDX(h); + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); +#endif + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, FA_MQS_HEAD_IDX(h), n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = FA_MQS_HEAD_IDX(h); + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + // each subgroup independently sweeps its slice of the split's kv range. + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_off); + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off); + + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); +#ifdef FA_Q1_Q_REG + if (tid_sg < DK_VEC) { + const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[tid_sg]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_reg[h], k_vec, dot4[h]); + } + } +#else + for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) { + const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[k]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]); + } + } +#endif + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + score[h] = s; + } + + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[dv_idx]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_vec, o_acc[h][idx] * sp_h[h]); + } + } + } + + // per-h cross-subgroup merge + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = FA_MQS_HEAD_IDX(h); + + // fold per-subgroup (m, l) into split-level (m_c, l_c) + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + // each thread writes its DV slice of the merged O. + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +#undef FA_MQS_HEAD_IDX + +#ifndef FA_MQ_SPLIT_ONLY +// Cluster-parallel variant of _q1_vec_mq_split +// +// Tthe baseline keeps one 256B K row in flight per subgroup (32 lanes cooperate +// on one position, serialized by the reduce+exp chain). This kernel +// takes q1_split's memory-level parallelism at MQ's read-once traffic: +// - the 64-lane subgroup is split into FA_CL_NCL clusters of FA_CL_C lanes; +// - each cluster owns its own KV position stream (positions strided by +// FA_CL_NCL) with private per-cluster online-softmax state, hence FA_CL_NCL +// independent K rows in flight per subgroup, no cross-cluster serial chain; +// - within a cluster, lanes split DK for the dot (cluster-reduce via +// sub_group_shuffle_xor, steps < FA_CL_C stay inside the cluster) and +// split DV for o_acc (each lane owns dv indices {lic + FA_CL_C*i} — the +// same slice for every position, so accumulation is lane-local); +// - merge stage 1 folds the FA_CL_NCL cluster partials with cross-cluster +// shuffles (distances >= FA_CL_C); stage 2 is the baseline cross-subgroup +// LDS merge (o published by cluster 0's lanes, layout-identical to the +// baseline's sg_o). +// The KV sweep runs a UNIFORM trip count (max over clusters) with a clamped +// row address + FA_M_INIT score on the tail — keeps every shuffle convergent +// (p = exp(FA_M_INIT - m) underflows to 0, so clamped-row reads are inert). +// Register cost vs baseline: o_acc grows from DV_VEC/64 to DV_VEC/FA_CL_C +// float4 per lane per head — FA_CL_C=8 / MQ_GQA=4 => 16 float4 (256B). + +#ifdef HAS_SUBGROUP_SHUFFLE // cluster reduce/merge needs shuffles; absent -> kernel dropped, dispatch falls back + +#ifndef FA_CL_C +#define FA_CL_C 8 +#endif + +// The lane striping requires DK/DV to divide evenly across the cluster; +// otherwise (e.g. DK=40 with FA_CL_C=16 -> zero-size arrays) compile the +// kernel out — host soft-create falls back silently. +#if (DK_VEC % FA_CL_C) == 0 && (DV_VEC % FA_CL_C) == 0 +#define FA_CL_NCL (Q1_WG_SIZE / FA_CL_C) // clusters (position streams) per subgroup +#define FA_CL_DK (DK_VEC / FA_CL_C) // half4s of K per lane per row +#define FA_CL_DV (DV_VEC / FA_CL_C) // float4s of o_acc per lane per head + +// explicit "half" sub-group attribute routes this fp16-heavy kernel to a slow +// codegen path on the X1 compiler. X2 keeps the pin: its driver miscompile +// without it. +#ifdef FA_C8_NO_SG_PIN +#define FA_C8_SG_ATTR +#else +// REQD_FA_SG pins the HW subgroup on Intel (intel_reqd_sub_group_size(FA_SG), +// host passes -D FA_SG=32); empty on Adreno. REQD_SUBGROUP_SIZE_64 pins 64 on +// Adreno; empty on Intel. +#define FA_C8_SG_ATTR REQD_FA_SG REQD_SUBGROUP_SIZE_64 +#endif + +FA_C8_SG_ATTR +__kernel void flash_attn_f32_f16_q1_vec_mq_split_c8( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int cl = tid_sg / FA_CL_C; // cluster id + const int lic = tid_sg % FA_CL_C; // lane in cluster + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int hgroups = n_head_kv * FA_HEAD_SUB; + const int batch_idx = kvhead_batch_idx / hgroups; + const int hg = kvhead_batch_idx % hgroups; + const int head_kv_idx = hg / FA_HEAD_SUB; + const int head_sub = hg % FA_HEAD_SUB; +#define FA_HEAD_IDX(h) (head_kv_idx * (MQ_GQA * FA_HEAD_SUB) + head_sub * MQ_GQA + (h)) + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = FA_HEAD_IDX(h); + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + // Stage MQ_GQA Q rows in __local once (uniform across WG). + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = FA_HEAD_IDX(h); + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, FA_HEAD_IDX(h), n_head_log2, m0, m1); + } + +#ifdef FA_CL_MASK_BCAST + const global char * mask_base_b = NULL; + if (mask_void != NULL) { + mask_base_b = (const global char *) mask_void + mask_offset + + (batch_idx % mask_ne3) * mask_nb3 + (ulong) q_idx * mask_nb1; + } + const int mask_bcast = mask_base_b != NULL && mask_ne2 == 1; +#else + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = FA_HEAD_IDX(h); + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } +#endif + + // Per-CLUSTER online-softmax state (uniform across the cluster's lanes); + // o_acc holds this lane's DV slice {lic + FA_CL_C*i}. + ACC_TYPE4 o_acc[MQ_GQA][FA_CL_DV]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < FA_CL_DV; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + // Uniform trip count across the subgroup: every cluster runs n_iter + // iterations; tail positions clamp the row address and drop the score to + // FA_M_INIT so shuffles stay convergent and the contribution is exactly 0. + const int n_iter = (kv_hi - kv_lo + FA_CL_NCL - 1) / FA_CL_NCL; + const ulong kv_row_base = batch_idx * k_nb3 + head_kv_idx * k_nb2; + const ulong v_row_base = batch_idx * v_nb3 + head_kv_idx * v_nb2; + + for (int it = 0; it < n_iter; ++it) { + const int k_idx = kv_lo + cl + it * FA_CL_NCL; + const int valid = k_idx < kv_hi; + const int k_safe = valid ? k_idx : (kv_hi - 1); + + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + kv_row_base + (ulong) k_safe * k_nb1); + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_base + (ulong) k_safe * v_nb1); + +#if defined(FA_CL_MHRED) && MQ_GQA == 4 && FA_CL_C == 16 && FA_CL_DK == 1 && FA_CL_DV == 1 + ACC_TYPE mask_val = 0.0f; + if (mask_bcast) { + mask_val = (ACC_TYPE) ((const global MASK_DATA_TYPE *) mask_base_b)[k_safe]; + } + const ACC_TYPE4 k_vec_1 = CONVERT_KV_ACC4(k_ptr[lic]); + const ACC_TYPE4 v_vec_1 = CONVERT_KV_ACC4(v_ptr[lic]); + + // Reduce four heads with eight shuffles and keep each head's summation order. + const int mh_b0 = lic & 1; + const int mh_b1 = lic & 2; + + ACC_TYPE mh_p[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE4 d4 = mad(q_shared[h * DK_VEC + lic], k_vec_1, (ACC_TYPE4)(0.0f)); + mh_p[h] = d4.s0 + d4.s1 + d4.s2 + d4.s3; + } + + ACC_TYPE mh_r2[2]; + #pragma unroll + for (int j = 0; j < 2; ++j) { + const ACC_TYPE keep = mh_b0 ? mh_p[j + 2] : mh_p[j]; + const ACC_TYPE send = mh_b0 ? mh_p[j] : mh_p[j + 2]; + mh_r2[j] = keep + sub_group_shuffle_xor(send, 1); + } + ACC_TYPE mh_r1 = (mh_b1 ? mh_r2[1] : mh_r2[0]) + + sub_group_shuffle_xor(mh_b1 ? mh_r2[0] : mh_r2[1], 2); + mh_r1 += sub_group_shuffle_xor(mh_r1, 4); + mh_r1 += sub_group_shuffle_xor(mh_r1, 8); + + ACC_TYPE mh_e2[2]; + { + const ACC_TYPE other = sub_group_shuffle_xor(mh_r1, 2); + mh_e2[0] = mh_b1 ? other : mh_r1; + mh_e2[1] = mh_b1 ? mh_r1 : other; + } + ACC_TYPE mh_s[MQ_GQA]; + #pragma unroll + for (int j = 0; j < 2; ++j) { + const ACC_TYPE other = sub_group_shuffle_xor(mh_e2[j], 1); + mh_s[j] = mh_b0 ? other : mh_e2[j]; + mh_s[j + 2] = mh_b0 ? mh_e2[j] : other; + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE s = mh_s[h] * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_bcast) { + s += slope[h] * mask_val; + } else if (mask_base_b != NULL) { + const int mask_head_idx = FA_HEAD_IDX(h) % mask_ne2; + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base_b + mask_head_idx * mask_nb2); + s += slope[h] * (ACC_TYPE) mask_ptr[k_safe]; + } + const ACC_TYPE sc = valid ? s : FA_M_INIT; + const ACC_TYPE m_new = max(m_i[h], sc); + const ACC_TYPE sp = native_exp(m_i[h] - m_new); + const ACC_TYPE p = native_exp(sc - m_new); + l_i[h] = l_i[h] * sp + p; + m_i[h] = m_new; + o_acc[h][0] = mad(p, v_vec_1, o_acc[h][0] * sp); + } +#else + // Dot: this lane covers DK elements {lic + FA_CL_C*i} of the cluster's row. + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int i = 0; i < FA_CL_DK; ++i) { + const int kk = lic + FA_CL_C * i; + const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[kk]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + kk], k_vec, dot4[h]); + } + } + + // Cluster-reduce (xor steps < FA_CL_C stay inside the cluster) + score. + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE s = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + #pragma unroll + for (int step = 1; step < FA_CL_C; step <<= 1) { + s += sub_group_shuffle_xor(s, step); + } + s *= scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_safe]; + } + score[h] = valid ? s : FA_M_INIT; + } + + // Per-cluster online update — identical math to the baseline, but the + // serial chain is per cluster (depth n_iter, not kv_per_sg). + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + // V accumulate on this lane's DV slice (p = 0 on tail -> inert). + #pragma unroll + for (int i = 0; i < FA_CL_DV; ++i) { + const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[lic + FA_CL_C * i]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][i] = mad(p_h[h], v_vec, o_acc[h][i] * sp_h[h]); + } + } +#endif + } + + // Merge stage 1: fold the FA_CL_NCL cluster partials inside the subgroup. + // Lanes with equal lic across clusters hold the SAME dv slice, so a + // cross-cluster xor-reduce (distances FA_CL_C..Q1_WG_SIZE/2) sums o + // slice-wise; m/l fold the same way. All shuffles are subgroup-convergent. + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE m_c = m_i[h]; + #pragma unroll + for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) { + m_c = max(m_c, sub_group_shuffle_xor(m_c, step)); + } + const ACC_TYPE alpha = native_exp(m_i[h] - m_c); + ACC_TYPE l_c = l_i[h] * alpha; + #pragma unroll + for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) { + l_c += sub_group_shuffle_xor(l_c, step); + } + #pragma unroll + for (int i = 0; i < FA_CL_DV; ++i) { + ACC_TYPE4 o = o_acc[h][i] * alpha; + #pragma unroll + for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) { + o.s0 += sub_group_shuffle_xor(o.s0, step); + o.s1 += sub_group_shuffle_xor(o.s1, step); + o.s2 += sub_group_shuffle_xor(o.s2, step); + o.s3 += sub_group_shuffle_xor(o.s3, step); + } + o_acc[h][i] = o; + } + m_i[h] = m_c; + l_i[h] = l_c; + } + + // Merge stage 2: baseline cross-subgroup LDS merge. Cluster 0's lanes hold + // the subgroup's merged o (dv indices {lic + FA_CL_C*i}) — same sg_o layout + // and fold loop as q1_vec_mq_split. + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + if (cl == 0) { + #pragma unroll + for (int i = 0; i < FA_CL_DV; ++i) { + sg_o[sgid][lic + FA_CL_C * i] = o_acc[h][i]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = FA_HEAD_IDX(h); + + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +#undef FA_HEAD_IDX + +#endif // DK_VEC/DV_VEC divisible by FA_CL_C +#endif // HAS_SUBGROUP_SHUFFLE (q1_vec_mq_split_c8) + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_f16_q1_vec_mq_split_k_img( + const global void * q_void, ulong q_offset, + __read_only image1d_buffer_t k_img, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + // K pitches in pixel units, pixel = 1 half4 = 8 B -> byte_stride >> 3. + const int pitch_px_row = (int)(k_nb1 >> 3); + const int pitch_px_head = (int)(k_nb2 >> 3); + const int pitch_px_batch = (int)(k_nb3 >> 3); + + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) { + const int k_row_px = batch_idx * pitch_px_batch + + head_kv_idx * pitch_px_head + + k_idx * pitch_px_row; + + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off); + + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) { + const half4 k_h4 = read_imageh(k_img, k_row_px + k); + const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_h4); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]); + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + score[h] = s; + } + + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[dv_idx]); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_vec, o_acc[h][idx] * sp_h[h]); + } + } + } + + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} +#endif // !FA_MQ_SPLIT_ONLY +#endif // !FA_DECODE_ONLY + +#ifndef FA_MQ_ONLY // q1_split + merge excluded from the MQ-only (g8) program +#ifndef FA_VEC_ONLY +__kernel void flash_attn_f32_f16_q1_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + record_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (kv_start >= kv_end) { + // Empty split: leave sentinel partial for merge. + if (tid == 0) { + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 + + (ulong) q_idx * mask_nb1; + } + + // share Q via local memory (n_q=1 per split -> uniform across WG). + __local ACC_TYPE4 q_shared[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + for (int i = tid; i < DK_VEC; i += Q1_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + // pass 1a — split-local max. + ACC_TYPE m_i = FA_M_INIT; + for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) { + const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_offset); + ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int k = 0; k < DK_VEC; ++k) { + dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); + } + ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + m_i = max(m_i, score); + } + + const ACC_TYPE m_c = sub_group_reduce_max(m_i); + + // pass 1b — softmax-weighted V accumulate. + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + ACC_TYPE l_i = 0.0f; + + for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) { + const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_offset); + const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_offset); + ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int k = 0; k < DK_VEC; ++k) { + dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc); + } + ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + const ACC_TYPE p = exp(score - m_c); + l_i += p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) { + o_acc[i] = mad(p, CONVERT_KV_ACC4(v_ptr[i]), o_acc[i]); + } + } + + __local ACC_TYPE4 local_o[Q1_WG_SIZE]; + const ACC_TYPE l_c = sub_group_reduce_add(l_i); + + if (tid == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int i = 0; i < DV_VEC; ++i) { + local_o[tid] = o_acc[i]; + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o[tid] += local_o[tid + s]; + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + rec_o[i] = local_o[0]; + } + } +} + +#endif // !FA_VEC_ONLY + +// FD Pass 2: merge per-split partials into final O +// empty splits drop via exp(-INF)=0. +__kernel void flash_attn_f32_merge( + const global float * partial_void, + global void * o_void, + const ulong o_offset, + const int n_head, + const int n_splits, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const global void * sinks_void, + const ulong sinks_offset, + const int n_q +) { + const int lane = get_local_id(0); // 0..DV_VEC-1 + const int head_batch_idx = get_global_id(1); + const int q_idx = get_global_id(2); + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx_0 = (((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits; + const global float * rec0 = partial_void + record_idx_0 * record_stride; + + __local ACC_TYPE m_final_shared; + __local ACC_TYPE l_final_shared; + if (lane == 0) { + ACC_TYPE m = FA_M_INIT; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + m = max(m, m_c); + } + ACC_TYPE m_sink = 0.0f; + bool has_sink = false; + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + m_sink = sinks_ptr[head_idx]; + has_sink = true; + m = max(m, m_sink); + } + ACC_TYPE l = 0.0f; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + const ACC_TYPE l_c = rec0[c * record_stride + 1]; + if (m_c > FA_M_INIT) { + l += l_c * exp(m_c - m); + } + } + if (has_sink) { + l += exp(m_sink - m); + } + m_final_shared = m; + l_final_shared = l; + } + barrier(CLK_LOCAL_MEM_FENCE); + const ACC_TYPE m_final = m_final_shared; + const ACC_TYPE l_final = l_final_shared; + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + ACC_TYPE4 o = (ACC_TYPE4)(0.0f); + for (int c = 0; c < n_splits; ++c) { + const global float * rec_c = rec0 + c * record_stride; + const ACC_TYPE m_c = rec_c[0]; + if (m_c <= FA_M_INIT) continue; + const global float4 * rec_oc = (const global float4 *) (rec_c + 2); + const ACC_TYPE scale_c = exp(m_c - m_final); + o = mad((ACC_TYPE4)(scale_c), rec_oc[lane], o); + } + o = o * l_inv; + + const ulong o_row_offset = (ulong) batch_idx * o_nb3 + (ulong) q_idx * o_nb2 + (ulong) head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) ((global char *) o_void + o_offset + o_row_offset); + o_row[lane] = CONVERT_O_DATA4(o); +} +#endif // !FA_MQ_ONLY (q1_split + merge) +#endif // !FA_PREFILL_ONLY (decode kernels) diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl new file mode 100644 index 0000000..3e75dc9 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl @@ -0,0 +1,2011 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#define FA_HAVE_INT_DOT 1 +#endif + +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#elif defined(cl_qcom_subgroup_shuffle) +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR +// name, so calling it is an implicit declaration and the program fails to build. +// Route it to the qcom builtin. +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) +#endif + +// Flash attention: Q=f32, K=q4_0, V=q4_0. +// Block = half d + uchar qs[16]; qs[j] low/high nibble -> elem j / j+16. +// Dequant: val[i] = d * (nibble_i - 8). dp4a path runs on raw 0..15 nibbles +// and applies the -8*sum(q) correction once per block (needs Q q_sum). + +#define ACC_TYPE float +#define ACC_TYPE4 float4 +#define Q_DATA_TYPE4 float4 +#define O_DATA_TYPE4 float4 +#define MASK_DATA_TYPE half +#define CONVERT_Q_ACC4(x) (x) +#define CONVERT_O_DATA4(x) (x) + +#define DK_VEC (DK/4) +#define DV_VEC (DV/4) + +#ifndef FA_SG +#define FA_SG 64 +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +#define QK4_0 32 +#define Q4_0_BLOCK_SIZE 18 + +#define DK_Q4_BLOCKS (DK / QK4_0) +#define DV_Q4_BLOCKS (DV / QK4_0) + +inline float dot_q4_0_f32(const global char * block_ptr, ACC_TYPE4 * q_slice) { + float d = vload_half(0, (const global half *)block_ptr); + const global uchar * qs = (const global uchar *)(block_ptr + 2); + + float sum = 0.0f; + // Low nibbles -> elems 0..15. + #pragma unroll + for (int g = 0; g < 4; ++g) { + float4 nv = (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f); + sum += dot(q_slice[g], nv); + } + // High nibbles -> elems 16..31. + #pragma unroll + for (int g = 0; g < 4; ++g) { + float4 nv = (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 1] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 2] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 3] >> 4) - 8.0f); + sum += dot(q_slice[4 + g], nv); + } + return sum * d; +} + +#ifdef FA_HAVE_INT_DOT +inline uint pack_i8x4(char a, char b, char c, char d) { + return ((uint)(uchar)a) | + ((uint)(uchar)b) << 8 | + ((uint)(uchar)c) << 16 | + ((uint)(uchar)d) << 24; +} + +// Returns (qd, q_sum); q_sum feeds the -8*sum(q) bias correction. +typedef struct { + float qd; + int q_sum; +} q4_q_block_info; + +inline q4_q_block_info quant_q_block_int8_packed_q4(const ACC_TYPE4 * q_block, + uint * out_packed) { + float amax = 0.0f; + #pragma unroll + for (int i = 0; i < 8; ++i) { + float4 av = fabs(q_block[i]); + amax = fmax(amax, fmax(fmax(av.s0, av.s1), fmax(av.s2, av.s3))); + } + float qd = amax / 127.0f; + float qid = (amax > 0.0f) ? 127.0f / amax : 0.0f; + + int q_sum = 0; + #pragma unroll + for (int i = 0; i < 8; ++i) { + float4 v = q_block[i] * qid; + char a = (char)((int)round(v.s0)); + char b = (char)((int)round(v.s1)); + char c = (char)((int)round(v.s2)); + char d = (char)((int)round(v.s3)); + out_packed[i] = pack_i8x4(a, b, c, d); + q_sum += (int)a + (int)b + (int)c + (int)d; + } + q4_q_block_info info = { qd, q_sum }; + return info; +} + +// k_packed[0..3] = low nibbles (Q elems 0..15), k_packed[4..7] = high (16..31). +inline void pack_q4_0_nibbles(const global uchar * qs, uint * k_packed) { + #pragma unroll + for (int g = 0; g < 4; ++g) { + uchar b0 = qs[g*4 + 0]; + uchar b1 = qs[g*4 + 1]; + uchar b2 = qs[g*4 + 2]; + uchar b3 = qs[g*4 + 3]; + k_packed[g] = + ((uint)(b0 & 0x0F)) | + ((uint)(b1 & 0x0F)) << 8 | + ((uint)(b2 & 0x0F)) << 16 | + ((uint)(b3 & 0x0F)) << 24; + k_packed[4 + g] = + ((uint)(b0 >> 4)) | + ((uint)(b1 >> 4)) << 8 | + ((uint)(b2 >> 4)) << 16 | + ((uint)(b3 >> 4)) << 24; + } +} + +inline float dot_q4_0_int(const global char * k_block_ptr, + const uint * q_packed, + float q_d, + int q_sum) { + float kd = vload_half(0, (const global half *)k_block_ptr); + const global uchar * k_qs = (const global uchar *)(k_block_ptr + 2); + + uint k_packed[8]; + pack_q4_0_nibbles(k_qs, k_packed); + + int sum = 0; + #pragma unroll + for (int i = 0; i < 8; ++i) { + sum = dot_acc_sat_4x8packed_ss_int(q_packed[i], k_packed[i], sum); + } + // Correct raw-nibble sum: (nibble - 8) bias -> subtract 8 * q_sum. + return (float)(sum - 8 * q_sum) * q_d * kd; +} +#endif // FA_HAVE_INT_DOT + +inline void dequant_q4_0_f32(const global char * block_ptr, ACC_TYPE4 * out) { + float d = vload_half(0, (const global half *)block_ptr); + const global uchar * qs = (const global uchar *)(block_ptr + 2); + + #pragma unroll + for (int g = 0; g < 4; ++g) { + out[g] = d * (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f); + } + #pragma unroll + for (int g = 0; g < 4; ++g) { + out[4 + g] = d * (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 1] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 2] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 3] >> 4) - 8.0f); + } +} + +// max_bias<=0 returns 1.0 so score += 1.0 * mask[k] stays a no-op multiplier. +inline float get_alibi_slope(float max_bias, int head_idx, int n_head_log2, float m0, float m1) { + if (max_bias <= 0.0f) return 1.0f; + float base = (head_idx < n_head_log2) ? m0 : m1; + int exph = (head_idx < n_head_log2) ? (head_idx + 1) : (2*(head_idx - n_head_log2) + 1); + return pow(base, (float)exph); +} + +// q1 decode: one query row per WG, threads sweep KV positions. +__kernel void flash_attn_f32_q4_0_q1( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char* q_base = (const global char*)q_void + q_offset; + const global char* k_base = (const global char*)k_void + k_offset; + const global char* v_base = (const global char*)v_void + v_offset; + global char* o_base = (global char*)o_void + o_offset; + + const global char* mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + ACC_TYPE4 q_priv[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + +#ifdef FA_HAVE_INT_DOT + // Quantise Q once per thread: 8 uints + qd + q_sum per block. + uint q_packed[DK_Q4_BLOCKS * 8]; + float q_d_scale[DK_Q4_BLOCKS]; + int q_sum_arr[DK_Q4_BLOCKS]; + #pragma unroll + for (int b = 0; b < DK_Q4_BLOCKS; ++b) { + q4_q_block_info info = quant_q_block_int8_packed_q4(&q_priv[b * 8], &q_packed[b * 8]); + q_d_scale[b] = info.qd; + q_sum_arr[b] = info.q_sum; + } +#endif + + float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE* sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); + } + + // One-pass online softmax (FA-2): single sweep over kv positions, + // updating per-thread (m_i, l_i, o_acc) per K. Eliminates the second + // K read of the original two-pass implementation. + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; + ACC_TYPE l_i = 0.0f; + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { + const global char* k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char* v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + + ACC_TYPE score = 0.0f; + #pragma unroll + for (int b = 0; b < DK_Q4_BLOCKS; b++) { +#ifdef FA_HAVE_INT_DOT + score += dot_q4_0_int(k_row + b * Q4_0_BLOCK_SIZE, + &q_packed[b * 8], q_d_scale[b], q_sum_arr[b]); +#else + score += dot_q4_0_f32(k_row + b * Q4_0_BLOCK_SIZE, &q_priv[b * 8]); +#endif + } + score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + if (mask_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); + score += slope * (ACC_TYPE)mask_ptr[k_idx]; + } + + // Online softmax step. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE alpha = exp(m_i - m_new); + const ACC_TYPE p = exp(score - m_new); + + l_i = alpha * l_i + p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha; + + #pragma unroll + for (int b = 0; b < DV_Q4_BLOCKS; b++) { + ACC_TYPE4 v_dequant[8]; + dequant_q4_0_f32(v_row + b * Q4_0_BLOCK_SIZE, v_dequant); + #pragma unroll + for (int i = 0; i < 8; i++) { + o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]); + } + } + + m_i = m_new; + } + + // Cross-thread reduce: max(m_i) -> m_final, rescale per-thread l_i and + // o_acc by alpha = exp(m_i_thread - m_final) before sum-reduce. + __local ACC_TYPE local_m[Q1_WG_SIZE]; + local_m[tid] = m_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE m_final = local_m[0]; + + const ACC_TYPE alpha_final = exp(m_i - m_final); + l_i *= alpha_final; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final; + + __local ACC_TYPE local_l[Q1_WG_SIZE]; + __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; + local_l[tid] = l_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_l[tid] += local_l[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); + ACC_TYPE l_final = local_l[0]; + + if (sinks_ptr != NULL) { + l_final += exp(sinks_ptr[head_idx] - m_final); + } + + if (l_final > 0.0f) { + const ACC_TYPE l_inv = 1.0f / l_final; + for (int i = 0; i < DV_VEC; i++) { + local_o_comp[tid] = o_acc[i]; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv); + } + } + } else if (tid == 0) { + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f); + } +} + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#else +#define REQD_SUBGROUP_SIZE_64 +#endif + +#define VEC_NSG 4 +#define VEC_WG_SIZE (Q1_WG_SIZE * VEC_NSG) +#define Q1V_DV_PER_THREAD ((DV_VEC + Q1_WG_SIZE - 1) / Q1_WG_SIZE) + +// Dequant one float4 lane (0..7) from a q4_0 block. +// Lanes 0..3 → low nibbles of qs[0..15], lanes 4..7 → high nibbles. +inline float4 dequant_q4_0_lane(const global char * block_ptr, int lane) { + const float d = vload_half(0, (const global half *)block_ptr); + const global uchar * qs = (const global uchar *)(block_ptr + 2); + const int g = lane & 3; + const int shift = (lane < 4) ? 0 : 4; + return d * (float4)((float)((qs[g*4+0] >> shift) & 0x0F) - 8.0f, + (float)((qs[g*4+1] >> shift) & 0x0F) - 8.0f, + (float)((qs[g*4+2] >> shift) & 0x0F) - 8.0f, + (float)((qs[g*4+3] >> shift) & 0x0F) - 8.0f); +} + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_q4_0_q1_vec( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + __local ACC_TYPE4 q_shared[DK_VEC]; + { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + for (int i = tid; i < DK_VEC; i += VEC_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + +#ifdef FA_HAVE_INT_DOT + // quantize Q to int8-packed uints + per-block (qd, q_sum) once per WG for dp4a + // one thread per Q block, remaining threads idle this step + __local uint q_packed_shared[DK_Q4_BLOCKS * 8]; + __local float q_d_shared[DK_Q4_BLOCKS]; + __local int q_sum_shared[DK_Q4_BLOCKS]; + if (tid < DK_Q4_BLOCKS) { + ACC_TYPE4 q_block[8]; + #pragma unroll + for (int i = 0; i < 8; ++i) q_block[i] = q_shared[tid * 8 + i]; + uint packed[8]; + q4_q_block_info info = quant_q_block_int8_packed_q4(q_block, packed); + #pragma unroll + for (int i = 0; i < 8; ++i) q_packed_shared[tid * 8 + i] = packed[i]; + q_d_shared[tid] = info.qd; + q_sum_shared[tid] = info.q_sum; + } + barrier(CLK_LOCAL_MEM_FENCE); +#endif + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE * sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + } + + ACC_TYPE4 o_acc[Q1V_DV_PER_THREAD]; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + const int kv_per_sg = (n_kv + VEC_NSG - 1) / VEC_NSG; + const int kv_start = sgid * kv_per_sg; + const int kv_end = min(n_kv, kv_start + kv_per_sg); + + for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + +#ifdef FA_HAVE_INT_DOT + // per-lane dp4a: each lane packs 4 raw q4_0 nibbles into a uint, + // then dot_acc_sat_4x8packed_ss_int against the matching uint. + ACC_TYPE lane_contrib = 0.0f; + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane_in_block = qk % 8; + const int g = lane_in_block & 3; + const int shift = (lane_in_block < 4) ? 0 : 4; + const global char * k_block = k_row + block_idx * Q4_0_BLOCK_SIZE; + const float kd = vload_half(0, (const global half *)k_block); + const global uchar * k_qs = (const global uchar *)(k_block + 2); + const uchar b0 = k_qs[g*4 + 0]; + const uchar b1 = k_qs[g*4 + 1]; + const uchar b2 = k_qs[g*4 + 2]; + const uchar b3 = k_qs[g*4 + 3]; + const uint k_packed = ((uint)((b0 >> shift) & 0x0F)) | + ((uint)((b1 >> shift) & 0x0F)) << 8 | + ((uint)((b2 >> shift) & 0x0F)) << 16 | + ((uint)((b3 >> shift) & 0x0F)) << 24; + const uint q_packed_lane = q_packed_shared[block_idx * 8 + lane_in_block]; + const int raw_dot = dot_acc_sat_4x8packed_ss_int(q_packed_lane, k_packed, 0); + const float qd = q_d_shared[block_idx]; + const float block_scale = qd * kd; + float contrib = (float)raw_dot * block_scale; + if (lane_in_block == 0) { + // block bias correction is per-block + const int q_sum_b = q_sum_shared[block_idx]; + contrib -= 8.0f * block_scale * (float)q_sum_b; + } + lane_contrib += contrib; + } + ACC_TYPE score = sub_group_reduce_add(lane_contrib) * scale; +#else + ACC_TYPE4 dot4 = (ACC_TYPE4)(0.0f); + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane = qk % 8; + const float4 k_v = dequant_q4_0_lane(k_row + block_idx * Q4_0_BLOCK_SIZE, lane); + dot4 = mad(q_shared[qk], k_v, dot4); + } + ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3; + ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale; +#endif + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p = native_exp(score - m_new); + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + const int block_idx = dv / 8; + const int lane = dv % 8; + const float4 v_v = dequant_q4_0_lane(v_row + block_idx * Q4_0_BLOCK_SIZE, lane); + o_acc[idx] = mad(p, v_v, o_acc[idx] * scale_prev); + } + l_i = l_i * scale_prev + p; + m_i = m_new; + } + + __local ACC_TYPE sg_m[VEC_NSG]; + __local ACC_TYPE sg_l[VEC_NSG]; + __local ACC_TYPE4 sg_o[VEC_NSG][DV_VEC]; + + if (tid_sg == 0) { + sg_m[sgid] = m_i; + sg_l[sgid] = l_i; + } + { + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv] = o_acc[idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + ACC_TYPE m_final = sg_m[0]; + #pragma unroll + for (int s = 1; s < VEC_NSG; ++s) { + m_final = max(m_final, sg_m[s]); + } + if (sinks_ptr != NULL) { + m_final = max(m_final, sinks_ptr[head_idx]); + } + + ACC_TYPE l_final = 0.0f; + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + l_final += sg_l[s] * native_exp(sg_m[s] - m_final); + } + if (sinks_ptr != NULL) { + l_final += native_exp(sinks_ptr[head_idx] - m_final); + } + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset); + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[s] - m_final); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv], o_merged); + } + o_row[dv] = CONVERT_O_DATA4(o_merged * l_inv); + } + } +} + +// Flash-decoding split pass for q4_0 KV. Merge kernel is type-agnostic and +// shared with the f16/q8_0 FA kernels. +#define FA_PARTIAL_FLOATS (2 + DV) + +__kernel void flash_attn_f32_q4_0_q1_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + record_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (kv_start >= kv_end) { + if (tid == 0) { + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 + + (ulong) q_idx * mask_nb1; + } + + ACC_TYPE4 q_priv[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + +#ifdef FA_HAVE_INT_DOT + uint q_packed[DK_Q4_BLOCKS * 8]; + float q_d_scale[DK_Q4_BLOCKS]; + int q_sum_arr[DK_Q4_BLOCKS]; + #pragma unroll + for (int b = 0; b < DK_Q4_BLOCKS; ++b) { + q4_q_block_info info = quant_q_block_int8_packed_q4(&q_priv[b * 8], &q_packed[b * 8]); + q_d_scale[b] = info.qd; + q_sum_arr[b] = info.q_sum; + } +#endif + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + // One-pass online softmax (FA-2): single sweep over the split's K range. + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + ACC_TYPE score = 0.0f; + #pragma unroll + for (int b = 0; b < DK_Q4_BLOCKS; ++b) { +#ifdef FA_HAVE_INT_DOT + score += dot_q4_0_int(k_row + b * Q4_0_BLOCK_SIZE, + &q_packed[b * 8], q_d_scale[b], q_sum_arr[b]); +#else + score += dot_q4_0_f32(k_row + b * Q4_0_BLOCK_SIZE, &q_priv[b * 8]); +#endif + } + score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + + // Online softmax step. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE alpha = exp(m_i - m_new); + const ACC_TYPE p = exp(score - m_new); + + l_i = alpha * l_i + p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha; + + #pragma unroll + for (int b = 0; b < DV_Q4_BLOCKS; ++b) { + ACC_TYPE4 v_dequant[8]; + dequant_q4_0_f32(v_row + b * Q4_0_BLOCK_SIZE, v_dequant); + #pragma unroll + for (int i = 0; i < 8; ++i) { + o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]); + } + } + + m_i = m_new; + } + + // Cross-thread reduce: max(m_i) -> m_c, rescale per-thread l_i and o_acc + // by alpha = exp(m_i_thread - m_c) before sum-reduce. + __local ACC_TYPE local_m[Q1_WG_SIZE]; + local_m[tid] = m_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE m_c = local_m[0]; + + const ACC_TYPE alpha_final = exp(m_i - m_c); + l_i *= alpha_final; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final; + + __local ACC_TYPE local_l[Q1_WG_SIZE]; + __local ACC_TYPE4 local_o[Q1_WG_SIZE]; + local_l[tid] = l_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_l[tid] += local_l[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE l_c = local_l[0]; + + if (tid == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int i = 0; i < DV_VEC; ++i) { + local_o[tid] = o_acc[i]; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o[tid] += local_o[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + rec_o[i] = local_o[0]; + } + } +} + +// Prefill: q4_0 K/V, n_q > 1. BLOCK_M × BLOCK_N tiling. +// K in local as packed nibbles + per-block scale; V dequant -> half in local. +// Requires DK % QK4_0 == 0 and DV % QK4_0 == 0. +#define KV_DATA_TYPE4 half4 +#define CONVERT_KV_ACC4(x) convert_float4(x) + +#define DK_Q4_BLOCKS_PREFILL (DK / QK4_0) +#define DV_Q4_BLOCKS_PREFILL (DV / QK4_0) + +// N_SPLIT>1 splits DK/DV across N_SPLIT threads per query row; needs +// sub_group_shuffle_xor and DK_Q4_BLOCKS_PREFILL % N_SPLIT == 0. +#ifndef N_SPLIT +#define N_SPLIT 1 +#endif + +#if N_SPLIT > 1 +#define SPLIT_DK_VEC (DK_VEC / N_SPLIT) +#define SPLIT_DV_VEC (DV_VEC / N_SPLIT) +#define SPLIT_DK_Q4_BLOCKS (DK_Q4_BLOCKS_PREFILL / N_SPLIT) +#define WG_SIZE (BLOCK_M * N_SPLIT) +#else +#define SPLIT_DK_VEC DK_VEC +#define SPLIT_DV_VEC DV_VEC +#define SPLIT_DK_Q4_BLOCKS DK_Q4_BLOCKS_PREFILL +#define WG_SIZE BLOCK_M +#endif + +#ifndef MQ_GQA +#define MQ_GQA 4 +#endif +#ifndef MQ_NSG_SPLIT +#define MQ_NSG_SPLIT 4 +#endif +#define MQ_SPLIT_WG_SIZE_Q4 (Q1_WG_SIZE * MQ_NSG_SPLIT) + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_q4_0_q1_vec_mq_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q4) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + +#ifdef FA_HAVE_INT_DOT + __local uint q_packed_shared[MQ_GQA * DK_Q4_BLOCKS * 8]; + __local float q_d_shared[MQ_GQA * DK_Q4_BLOCKS]; + __local int q_sum_shared[MQ_GQA * DK_Q4_BLOCKS]; + { + const int active = MQ_GQA * DK_Q4_BLOCKS; + if (tid < active) { + const int h = tid / DK_Q4_BLOCKS; + const int block_id = tid % DK_Q4_BLOCKS; + ACC_TYPE4 q_block[8]; + #pragma unroll + for (int i = 0; i < 8; ++i) q_block[i] = q_shared[h * DK_VEC + block_id * 8 + i]; + uint packed[8]; + q4_q_block_info info = quant_q_block_int8_packed_q4(q_block, packed); + #pragma unroll + for (int i = 0; i < 8; ++i) q_packed_shared[(h * DK_Q4_BLOCKS + block_id) * 8 + i] = packed[i]; + q_d_shared[h * DK_Q4_BLOCKS + block_id] = info.qd; + q_sum_shared[h * DK_Q4_BLOCKS + block_id] = info.q_sum; + } + } + barrier(CLK_LOCAL_MEM_FENCE); +#endif + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + +#ifdef FA_HAVE_INT_DOT + ACC_TYPE lane_contrib[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) lane_contrib[h] = 0.0f; + + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane_in_block = qk % 8; + const int g = lane_in_block & 3; + const int shift = (lane_in_block < 4) ? 0 : 4; + const global char * k_block = k_row + block_idx * Q4_0_BLOCK_SIZE; + const float kd = vload_half(0, (const global half *)k_block); + const global uchar * k_qs = (const global uchar *)(k_block + 2); + const uchar b0 = k_qs[g*4 + 0]; + const uchar b1 = k_qs[g*4 + 1]; + const uchar b2 = k_qs[g*4 + 2]; + const uchar b3 = k_qs[g*4 + 3]; + const uint k_packed = ((uint)((b0 >> shift) & 0x0F)) | + ((uint)((b1 >> shift) & 0x0F)) << 8 | + ((uint)((b2 >> shift) & 0x0F)) << 16 | + ((uint)((b3 >> shift) & 0x0F)) << 24; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const uint q_packed_lane = q_packed_shared[(h * DK_Q4_BLOCKS + block_idx) * 8 + lane_in_block]; + const int raw_dot = dot_acc_sat_4x8packed_ss_int(q_packed_lane, k_packed, 0); + const float qd = q_d_shared[h * DK_Q4_BLOCKS + block_idx]; + const float block_scale = qd * kd; + float contrib = (float) raw_dot * block_scale; + if (lane_in_block == 0) { + const int q_sum_b = q_sum_shared[h * DK_Q4_BLOCKS + block_idx]; + contrib -= 8.0f * block_scale * (float) q_sum_b; + } + lane_contrib[h] += contrib; + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE s = sub_group_reduce_add(lane_contrib[h]) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + score[h] = s; + } +#else + // fallback float-dequant K dot + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane = qk % 8; + const float4 k_v = dequant_q4_0_lane(k_row + block_idx * Q4_0_BLOCK_SIZE, lane); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + qk], k_v, dot4[h]); + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + score[h] = s; + } +#endif + + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + const int block_idx = dv / 8; + const int lane = dv % 8; + const float4 v_v = dequant_q4_0_lane(v_row + block_idx * Q4_0_BLOCK_SIZE, lane); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_v, o_acc[h][idx] * sp_h[h]); + } + } + } + + // per-h cross-subgroup merge + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +// flash_attn_f32_q4_0_q1_vec_mq_split_c8 — cluster-parallel variant of the MQ +// split, port of flash_attn_f32_f16_q1_vec_mq_split_c8 +// Requires dp4a + subgroup shuffles + +#if defined(FA_HAVE_INT_DOT) && defined(HAS_SUBGROUP_SHUFFLE) + +#ifndef FA_CL_C +#define FA_CL_C 8 +#endif + +// Lane striping requires DK/DV to divide across the cluster (see f16 c8). +#if (DK_VEC % FA_CL_C) == 0 && (DV_VEC % FA_CL_C) == 0 +#define FA_CL_NCL (Q1_WG_SIZE / FA_CL_C) // clusters (position streams) per subgroup +#define FA_CL_DKQ (DK_VEC / FA_CL_C) // K quartets per lane per row +#define FA_CL_DVQ (DV_VEC / FA_CL_C) // V quartets (o_acc float4s) per lane per head + +#ifdef FA_C8_NO_SG_PIN +#define FA_C8_SG_ATTR_Q4 +#else +#define FA_C8_SG_ATTR_Q4 REQD_SUBGROUP_SIZE_64 +#endif + +FA_C8_SG_ATTR_Q4 +__kernel void flash_attn_f32_q4_0_q1_vec_mq_split_c8( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int cl = tid_sg / FA_CL_C; // cluster id + const int lic = tid_sg % FA_CL_C; // lane in cluster + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + // Stage MQ_GQA Q rows in __local as float4 (source for the quantize pass). + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q4) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + // Per-(h, block) int8-packed Q + (qd, q_sum), quantized once per WG. + __local uint q_packed_shared[MQ_GQA * DK_Q4_BLOCKS * 8]; + __local float q_d_shared[MQ_GQA * DK_Q4_BLOCKS]; + __local int q_sum_shared[MQ_GQA * DK_Q4_BLOCKS]; + { + const int active = MQ_GQA * DK_Q4_BLOCKS; + if (tid < active) { + const int h = tid / DK_Q4_BLOCKS; + const int block_id = tid % DK_Q4_BLOCKS; + ACC_TYPE4 q_block[8]; + #pragma unroll + for (int i = 0; i < 8; ++i) q_block[i] = q_shared[h * DK_VEC + block_id * 8 + i]; + uint packed[8]; + q4_q_block_info info = quant_q_block_int8_packed_q4(q_block, packed); + #pragma unroll + for (int i = 0; i < 8; ++i) q_packed_shared[(h * DK_Q4_BLOCKS + block_id) * 8 + i] = packed[i]; + q_d_shared[h * DK_Q4_BLOCKS + block_id] = info.qd; + q_sum_shared[h * DK_Q4_BLOCKS + block_id] = info.q_sum; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + // Per-CLUSTER online state; o_acc holds this lane's V quartets {lic + FA_CL_C*i}. + ACC_TYPE4 o_acc[MQ_GQA][FA_CL_DVQ]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < FA_CL_DVQ; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + // Uniform trip count; tail clamps the row address and drops the score to + // FA_M_INIT (p underflows to 0) so shuffles stay convergent. + const int n_iter = (kv_hi - kv_lo + FA_CL_NCL - 1) / FA_CL_NCL; + const ulong k_row_base = batch_idx * k_nb3 + head_kv_idx * k_nb2; + const ulong v_row_base = batch_idx * v_nb3 + head_kv_idx * v_nb2; + + for (int it = 0; it < n_iter; ++it) { + const int k_idx = kv_lo + cl + it * FA_CL_NCL; + const int valid = k_idx < kv_hi; + const int k_safe = valid ? k_idx : (kv_hi - 1); + + const global char * k_row = k_base + k_row_base + (ulong) k_safe * k_nb1; + const global char * v_row = v_base + v_row_base + (ulong) k_safe * v_nb1; + + // dp4a K dot over this lane's quartets of the cluster's row. + ACC_TYPE lane_contrib[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) lane_contrib[h] = 0.0f; + + #pragma unroll + for (int i = 0; i < FA_CL_DKQ; ++i) { + const int qk = lic + FA_CL_C * i; + const int block_idx = qk / 8; + const int lane_in_block = qk % 8; + const int g = lane_in_block & 3; + const int shift = (lane_in_block < 4) ? 0 : 4; + const global char * k_block = k_row + block_idx * Q4_0_BLOCK_SIZE; + const float kd = vload_half(0, (const global half *)k_block); + const global uchar * k_qs = (const global uchar *)(k_block + 2); + const uchar b0 = k_qs[g*4 + 0]; + const uchar b1 = k_qs[g*4 + 1]; + const uchar b2 = k_qs[g*4 + 2]; + const uchar b3 = k_qs[g*4 + 3]; + const uint k_packed = ((uint)((b0 >> shift) & 0x0F)) | + ((uint)((b1 >> shift) & 0x0F)) << 8 | + ((uint)((b2 >> shift) & 0x0F)) << 16 | + ((uint)((b3 >> shift) & 0x0F)) << 24; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const uint q_packed_lane = q_packed_shared[(h * DK_Q4_BLOCKS + block_idx) * 8 + lane_in_block]; + const int raw_dot = dot_acc_sat_4x8packed_ss_int(q_packed_lane, k_packed, 0); + const float qd = q_d_shared[h * DK_Q4_BLOCKS + block_idx]; + const float block_scale = qd * kd; + float contrib = (float) raw_dot * block_scale; + if (lane_in_block == 0) { + const int q_sum_b = q_sum_shared[h * DK_Q4_BLOCKS + block_idx]; + contrib -= 8.0f * block_scale * (float) q_sum_b; + } + lane_contrib[h] += contrib; + } + } + + // Cluster-reduce + score. + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE s = lane_contrib[h]; + #pragma unroll + for (int step = 1; step < FA_CL_C; step <<= 1) { + s += sub_group_shuffle_xor(s, step); + } + s *= scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_safe]; + } + score[h] = valid ? s : FA_M_INIT; + } + + // Per-cluster online update (serial chain depth n_iter, not kv_per_sg). + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + // V accumulate on this lane's quartets (p = 0 on tail -> inert). + #pragma unroll + for (int i = 0; i < FA_CL_DVQ; ++i) { + const int dv = lic + FA_CL_C * i; + const float4 v_v = dequant_q4_0_lane(v_row + (dv / 8) * Q4_0_BLOCK_SIZE, dv % 8); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][i] = mad(p_h[h], v_v, o_acc[h][i] * sp_h[h]); + } + } + } + + // Merge stage 1: fold cluster partials inside the subgroup via shuffles. + // Lanes with equal lic across clusters hold the SAME dv slice. + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE m_c = m_i[h]; + #pragma unroll + for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) { + m_c = max(m_c, sub_group_shuffle_xor(m_c, step)); + } + const ACC_TYPE alpha = native_exp(m_i[h] - m_c); + ACC_TYPE l_c = l_i[h] * alpha; + #pragma unroll + for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) { + l_c += sub_group_shuffle_xor(l_c, step); + } + #pragma unroll + for (int i = 0; i < FA_CL_DVQ; ++i) { + ACC_TYPE4 o = o_acc[h][i] * alpha; + #pragma unroll + for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) { + o.s0 += sub_group_shuffle_xor(o.s0, step); + o.s1 += sub_group_shuffle_xor(o.s1, step); + o.s2 += sub_group_shuffle_xor(o.s2, step); + o.s3 += sub_group_shuffle_xor(o.s3, step); + } + o_acc[h][i] = o; + } + m_i[h] = m_c; + l_i[h] = l_c; + } + + // Merge stage 2: baseline cross-subgroup LDS merge (o published by + // cluster 0's lanes; layout identical to the baseline sg_o). + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + if (cl == 0) { + #pragma unroll + for (int i = 0; i < FA_CL_DVQ; ++i) { + sg_o[sgid][lic + FA_CL_C * i] = o_acc[h][i]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +#endif // DK_VEC/DV_VEC divisible by FA_CL_C +#endif // FA_HAVE_INT_DOT && HAS_SUBGROUP_SHUFFLE (q1_vec_mq_split_c8) + +__kernel void flash_attn_f32_q4_0( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset, + // blk: per-(qblock,kvblock) class from flash_attn_blk_f16 + // (0=masked, 1=mixed, 2=unmasked). NULL disables the prepass opt. + const global void * blk_void +) { + const int tid = get_local_id(0); + const int block_q_idx = get_group_id(0); + const int head_batch_idx = get_global_id(1); + +#if N_SPLIT > 1 + const int q_lane = tid / N_SPLIT; + const int split_idx = tid % N_SPLIT; +#else + const int q_lane = tid; + const int split_idx = 0; +#endif + const int my_query_row = block_q_idx * BLOCK_M + q_lane; + const int query_valid = my_query_row < n_q; + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + const int mask_head_idx = mask_void != NULL ? head_idx % mask_ne2 : 0; + const int mask_batch_idx = mask_void != NULL ? batch_idx % mask_ne3 : 0; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + // BLK_PREPASS_BM may differ from this kernel's BLOCK_M; scale q-block idx. + #ifndef BLK_PREPASS_BM + #define BLK_PREPASS_BM BLOCK_M + #endif + const global char * blk_base = NULL; + int n_kv_blocks = 0; + if (blk_void != NULL) { + n_kv_blocks = (n_kv + BLOCK_N - 1) / BLOCK_N; + const int n_q_blocks_prepass = (n_q + BLK_PREPASS_BM - 1) / BLK_PREPASS_BM; + const int prepass_q_block = (block_q_idx * BLOCK_M) / BLK_PREPASS_BM; + blk_base = (const global char *) blk_void + + (((mask_batch_idx * mask_ne2) + mask_head_idx) * n_q_blocks_prepass + prepass_q_block) * n_kv_blocks; + } + + const int dk_off_vec = split_idx * SPLIT_DK_VEC; + ACC_TYPE4 q_priv[SPLIT_DK_VEC]; + if (query_valid) { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; + const global float4 * q_ptr = (const global float4 *) (q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < SPLIT_DK_VEC; ++i) { + q_priv[i] = q_ptr[dk_off_vec + i]; + } + } else { + #pragma unroll + for (int i = 0; i < SPLIT_DK_VEC; ++i) q_priv[i] = (ACC_TYPE4)(0.0f); + } + +#ifdef FA_HAVE_INT_DOT + uint q_packed_pf[SPLIT_DK_Q4_BLOCKS * 8]; + float q_d_pf[SPLIT_DK_Q4_BLOCKS]; + int q_sum_pf[SPLIT_DK_Q4_BLOCKS]; + #pragma unroll + for (int b = 0; b < SPLIT_DK_Q4_BLOCKS; ++b) { + q4_q_block_info info = quant_q_block_int8_packed_q4(&q_priv[b * 8], &q_packed_pf[b * 8]); + q_d_pf[b] = info.qd; + q_sum_pf[b] = info.q_sum; + } +#endif + + const int dv_off_vec = split_idx * SPLIT_DV_VEC; + ACC_TYPE4 o_acc[SPLIT_DV_VEC]; + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + +#ifdef FA_HAVE_INT_DOT +// Accessors so the staging code is layout-agnostic. +#ifdef FA_K_LDS_T +#define FA_K_PACKED(ROW, IDX) l_k_packed[IDX][ROW] +#define FA_K_SCALE(ROW, BLK) l_k_scale[BLK][ROW] +#else +#define FA_K_PACKED(ROW, IDX) l_k_packed[ROW][IDX] +#define FA_K_SCALE(ROW, BLK) l_k_scale[ROW][BLK] +#endif + +#ifdef FA_K_LDS_T + // K tile transposed: the 4 KV rows the QK loop walks together become adjacent, so each + // (block, group) step is ONE 128-bit local read instead of four 32-bit ones. The QK + // loop is LDS-read-issue-bound. + __local uint l_k_packed[DK_Q4_BLOCKS_PREFILL * 8][BLOCK_N]; + __local float l_k_scale [DK_Q4_BLOCKS_PREFILL][BLOCK_N]; +#else + __local uint l_k_packed[BLOCK_N][DK_Q4_BLOCKS_PREFILL * 8]; + __local float l_k_scale [BLOCK_N][DK_Q4_BLOCKS_PREFILL]; +#endif +#else + __local half4 l_k[BLOCK_N][DK_VEC]; +#endif + + __local half4 l_v[BLOCK_N][DV_VEC]; + + for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) { + // Skip fully-masked KV tiles (uniform branch across WG). + char blk_cur = 1; + if (blk_base != NULL) { + blk_cur = blk_base[k_start / BLOCK_N]; + if (blk_cur == 0) continue; + } + + { +#ifdef FA_HAVE_INT_DOT + const int k_blocks_per_row = DK_Q4_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * k_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / k_blocks_per_row; + const int blk = i % k_blocks_per_row; + const int k_row_idx = k_start + row; + if (k_row_idx < n_kv) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; + const global char * blk_ptr = k_base + k_row_off + blk * Q4_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + FA_K_SCALE(row, blk) = df; + uint k_packed[8]; + pack_q4_0_nibbles(qs, k_packed); + #pragma unroll + for (int j = 0; j < 8; ++j) { + FA_K_PACKED(row, blk * 8 + j) = k_packed[j]; + } + } else { + FA_K_SCALE(row, blk) = 0.0f; + #pragma unroll + for (int j = 0; j < 8; ++j) FA_K_PACKED(row, blk * 8 + j) = 0u; + } + } +#else + // Fallback: dequant q4_0 -> half in local memory. + const int k_blocks_per_row = DK_Q4_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * k_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / k_blocks_per_row; + const int blk = i % k_blocks_per_row; + const int k_row_idx = k_start + row; + if (k_row_idx < n_kv) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; + const global char * blk_ptr = k_base + k_row_off + blk * Q4_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + #pragma unroll + for (int g = 0; g < 4; ++g) { + float4 vlo = df * (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f); + float4 vhi = df * (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 1] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 2] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 3] >> 4) - 8.0f); + l_k[row][blk * 8 + g ] = (half4)((half)vlo.s0, (half)vlo.s1, (half)vlo.s2, (half)vlo.s3); + l_k[row][blk * 8 + 4 + g] = (half4)((half)vhi.s0, (half)vhi.s1, (half)vhi.s2, (half)vhi.s3); + } + } else { + #pragma unroll + for (int j = 0; j < 8; ++j) l_k[row][blk * 8 + j] = (half4)(0.0h); + } + } +#endif + } + // V tile load — dequant V -> half in local memory. + { + const int v_blocks_per_row = DV_Q4_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * v_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / v_blocks_per_row; + const int blk = i % v_blocks_per_row; + const int v_row_idx = k_start + row; + if (v_row_idx < n_kv) { + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1; + const global char * blk_ptr = v_base + v_row_off + blk * Q4_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + #pragma unroll + for (int g = 0; g < 4; ++g) { + float4 vlo = df * (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f, + (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f); + float4 vhi = df * (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 1] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 2] >> 4) - 8.0f, + (float)(int)(qs[g*4 + 3] >> 4) - 8.0f); + l_v[row][blk * 8 + g ] = (half4)((half)vlo.s0, (half)vlo.s1, (half)vlo.s2, (half)vlo.s3); + l_v[row][blk * 8 + 4 + g] = (half4)((half)vhi.s0, (half)vhi.s1, (half)vhi.s2, (half)vhi.s3); + } + } else { + #pragma unroll + for (int j = 0; j < 8; ++j) l_v[row][blk * 8 + j] = (half4)(0.0h); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + // QK dot + online softmax. N_SPLIT>1 reduces per-thread partials via shuffle_xor. +#if N_SPLIT > 1 + { +#else + if (query_valid) { +#endif + const int k_blk_base = split_idx * SPLIT_DK_Q4_BLOCKS; + for (int j = 0; j < BLOCK_N; j += 4) { + const int k_row0 = k_start + j; + const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; + + ACC_TYPE s0, s1, s2, s3; +#ifdef FA_HAVE_INT_DOT + s0 = 0.0f; s1 = 0.0f; s2 = 0.0f; s3 = 0.0f; + #pragma unroll + for (int b_local = 0; b_local < SPLIT_DK_Q4_BLOCKS; ++b_local) { + const int b = k_blk_base + b_local; + int sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; +#ifdef FA_K_LDS_T + // 4 KV rows are adjacent in the transposed tile: one 128-bit local + // read per (block, group) instead of four 32-bit ones. + #pragma unroll + for (int g = 0; g < 8; ++g) { + const uint qp = q_packed_pf[b_local * 8 + g]; + const uint4 kq4 = vload4(0, &l_k_packed[b * 8 + g][j]); + sum0 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s0, sum0); + sum1 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s1, sum1); + sum2 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s2, sum2); + sum3 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s3, sum3); + } +#else + #pragma unroll + for (int g = 0; g < 8; ++g) { + const uint qp = q_packed_pf[b_local * 8 + g]; + sum0 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j ][b * 8 + g], sum0); + sum1 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+1][b * 8 + g], sum1); + sum2 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+2][b * 8 + g], sum2); + sum3 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+3][b * 8 + g], sum3); + } +#endif + const float qd = q_d_pf[b_local]; + const int q_sum = q_sum_pf[b_local]; +#ifdef FA_K_LDS_T + const float4 ks4 = vload4(0, &l_k_scale[b][j]); + s0 += (float)(sum0 - 8 * q_sum) * qd * ks4.s0; + s1 += (float)(sum1 - 8 * q_sum) * qd * ks4.s1; + s2 += (float)(sum2 - 8 * q_sum) * qd * ks4.s2; + s3 += (float)(sum3 - 8 * q_sum) * qd * ks4.s3; +#else + s0 += (float)(sum0 - 8 * q_sum) * qd * l_k_scale[j ][b]; + s1 += (float)(sum1 - 8 * q_sum) * qd * l_k_scale[j+1][b]; + s2 += (float)(sum2 - 8 * q_sum) * qd * l_k_scale[j+2][b]; + s3 += (float)(sum3 - 8 * q_sum) * qd * l_k_scale[j+3][b]; +#endif + } +#else + ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int k = 0; k < SPLIT_DK_VEC; ++k) { + const ACC_TYPE4 qk = q_priv[k]; + const int k_abs = dk_off_vec + k; + dot_acc0 = mad(qk, CONVERT_KV_ACC4(l_k[j ][k_abs]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_KV_ACC4(l_k[j+1][k_abs]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_KV_ACC4(l_k[j+2][k_abs]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_KV_ACC4(l_k[j+3][k_abs]), dot_acc3); + } + s0 = dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3; + s1 = dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3; + s2 = dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3; + s3 = dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3; +#endif + +#if N_SPLIT > 1 + // Power-of-2 N_SPLIT: shuffle_xor butterfly. N_SPLIT=3 (DK=96): + // explicit 3-lane shuffle. + #if (N_SPLIT & (N_SPLIT - 1)) == 0 + #pragma unroll + for (int step = 1; step < N_SPLIT; step <<= 1) { + s0 += sub_group_shuffle_xor(s0, step); + s1 += sub_group_shuffle_xor(s1, step); + s2 += sub_group_shuffle_xor(s2, step); + s3 += sub_group_shuffle_xor(s3, step); + } + #else + const uint tri_base = (get_sub_group_local_id() / N_SPLIT) * N_SPLIT; + s0 = sub_group_shuffle(s0, tri_base + 0) + sub_group_shuffle(s0, tri_base + 1) + sub_group_shuffle(s0, tri_base + 2); + s1 = sub_group_shuffle(s1, tri_base + 0) + sub_group_shuffle(s1, tri_base + 1) + sub_group_shuffle(s1, tri_base + 2); + s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2); + s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2); + #endif +#endif + s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } + if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; } + + if (is_causal) { + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; + } + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; + + if (query_valid && mask_base != NULL && blk_cur != 2) { + const global MASK_DATA_TYPE * mask_ptr = + (const global MASK_DATA_TYPE *) (mask_base + my_query_row * mask_nb1); + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE) mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE) mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3]; + } + + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + // Whole tile masked (m_new == FA_M_INIT): force the exp() args + // far negative so the tile contributes 0, not exp(0)=1. + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE scale_prev = native_exp(m_i - m_exp); + const ACC_TYPE p0 = native_exp(s0 - m_exp); + const ACC_TYPE p1 = native_exp(s1 - m_exp); + const ACC_TYPE p2 = native_exp(s2 - m_exp); + const ACC_TYPE p3 = native_exp(s3 - m_exp); + + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + const int i_abs = dv_off_vec + i; + o_acc[i] = mad(p3, CONVERT_KV_ACC4(l_v[j+3][i_abs]), + mad(p2, CONVERT_KV_ACC4(l_v[j+2][i_abs]), + mad(p1, CONVERT_KV_ACC4(l_v[j+1][i_abs]), + mad(p0, CONVERT_KV_ACC4(l_v[j ][i_abs]), + o_acc[i] * scale_prev)))); + } + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; + m_i = m_new; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + // Write output. + if (query_valid) { + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + const ACC_TYPE m_sink = sinks_ptr[head_idx]; + const ACC_TYPE m_final = max(m_i, m_sink); + const ACC_TYPE scale_o = exp(m_i - m_final); + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] *= scale_o; + l_i = l_i * scale_o + exp(m_sink - m_final); + m_i = m_final; + } + const ACC_TYPE l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f; + const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1; + global float4 * o_row = (global float4 *) (o_base + o_row_offset); + if (l_inv > 0.0f) { + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = o_acc[i] * l_inv; + } else { + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = (float4)(0.0f); + } + } +} + +// FD Pass 2: merge split partials. Identical across q4_0/q8_0/f16; each FA +// source owns a copy since kernels compile per-source-program. +__kernel void flash_attn_f32_merge( + const global float * partial_void, + global void * o_void, + const ulong o_offset, + const int n_head, + const int n_splits, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const global void * sinks_void, + const ulong sinks_offset, + const int n_q +) { + const int lane = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int q_idx = get_global_id(2); + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx_0 = (((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits; + const global float * rec0 = partial_void + record_idx_0 * record_stride; + + __local ACC_TYPE m_final_shared; + __local ACC_TYPE l_final_shared; + if (lane == 0) { + ACC_TYPE m = FA_M_INIT; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + m = max(m, m_c); + } + ACC_TYPE m_sink = 0.0f; + bool has_sink = false; + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + m_sink = sinks_ptr[head_idx]; + has_sink = true; + m = max(m, m_sink); + } + ACC_TYPE l = 0.0f; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + const ACC_TYPE l_c = rec0[c * record_stride + 1]; + if (m_c > FA_M_INIT) { + l += l_c * exp(m_c - m); + } + } + if (has_sink) { + l += exp(m_sink - m); + } + m_final_shared = m; + l_final_shared = l; + } + barrier(CLK_LOCAL_MEM_FENCE); + const ACC_TYPE m_final = m_final_shared; + const ACC_TYPE l_final = l_final_shared; + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + ACC_TYPE4 o = (ACC_TYPE4)(0.0f); + for (int c = 0; c < n_splits; ++c) { + const global float * rec_c = rec0 + c * record_stride; + const ACC_TYPE m_c = rec_c[0]; + if (m_c <= FA_M_INIT) continue; + const global float4 * rec_oc = (const global float4 *) (rec_c + 2); + const ACC_TYPE scale_c = exp(m_c - m_final); + o = mad((ACC_TYPE4)(scale_c), rec_oc[lane], o); + } + o = o * l_inv; + + const ulong o_row_offset = (ulong) batch_idx * o_nb3 + (ulong) q_idx * o_nb2 + (ulong) head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) ((global char *) o_void + o_offset + o_row_offset); + o_row[lane] = CONVERT_O_DATA4(o); +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl new file mode 100644 index 0000000..75d0d9b --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl @@ -0,0 +1,1840 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#define FA_HAVE_INT_DOT 1 +#endif + +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#elif defined(cl_qcom_subgroup_shuffle) +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR +// name, so calling it is an implicit declaration and the program fails to build. +// Route it to the qcom builtin. +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) +#endif + +// Flash attention: Q=f32, K=q8_0, V=q8_0. + +#define ACC_TYPE float +#define ACC_TYPE4 float4 +#define Q_DATA_TYPE4 float4 +#define O_DATA_TYPE4 float4 +#define MASK_DATA_TYPE half +#define CONVERT_Q_ACC4(x) (x) +#define CONVERT_O_DATA4(x) (x) + +#define DK_VEC (DK/4) +#define DV_VEC (DV/4) + +#ifndef FA_SG +#define FA_SG 64 +#endif +#define Q1_WG_SIZE FA_SG + +// The kernels are built with -cl-finite-math-only. On some older Adreno GPUs, +// infinite operand can cause undefined behavior and miscompilation for exp. +// Therefore, a large negative value is used instead. +#define FA_M_INIT (-3.0e38f) + +// q8_0 block: 2B scale (half) + 32B int8 quants. +#define QK8_0 32 +#define Q8_0_BLOCK_SIZE 34 + +#define DK_Q8_BLOCKS (DK / QK8_0) +#define DV_Q8_BLOCKS (DV / QK8_0) + +inline float dot_q8_0_f32(const global char * block_ptr, ACC_TYPE4 * q_slice) { + float d = vload_half(0, (const global half *)block_ptr); + const global char * qs = block_ptr + 2; + + float sum = 0.0f; + #pragma unroll + for (int i = 0; i < 8; i++) { + float4 qv = (float4)((float)qs[i*4], (float)qs[i*4+1], (float)qs[i*4+2], (float)qs[i*4+3]); + sum += dot(q_slice[i], qv); + } + return sum * d; +} + +#ifdef FA_HAVE_INT_DOT +inline uint pack_i8x4(char a, char b, char c, char d) { + return ((uint)(uchar)a) | + ((uint)(uchar)b) << 8 | + ((uint)(uchar)c) << 16 | + ((uint)(uchar)d) << 24; +} + +inline float quant_q_block_int8_packed(const ACC_TYPE4 * q_block, + uint * out_packed) { + float amax = 0.0f; + #pragma unroll + for (int i = 0; i < 8; ++i) { + float4 av = fabs(q_block[i]); + amax = fmax(amax, fmax(fmax(av.s0, av.s1), fmax(av.s2, av.s3))); + } + float qd = amax / 127.0f; + float qid = (amax > 0.0f) ? 127.0f / amax : 0.0f; + + #pragma unroll + for (int i = 0; i < 8; ++i) { + float4 v = q_block[i] * qid; + char a = (char)((int)round(v.s0)); + char b = (char)((int)round(v.s1)); + char c = (char)((int)round(v.s2)); + char d = (char)((int)round(v.s3)); + out_packed[i] = pack_i8x4(a, b, c, d); + } + return qd; +} + +inline float dot_q8_0_int(const global char * k_block_ptr, + const uint * q_packed, + float q_d) { + float kd = vload_half(0, (const global half *)k_block_ptr); + const global uchar * k_qs = (const global uchar *)(k_block_ptr + 2); + + // k_qs is 2-byte aligned; pack chars per iteration rather than cast to uint*. + int sum = 0; + #pragma unroll + for (int i = 0; i < 8; ++i) { + uint k_packed = + (uint)k_qs[i*4 + 0] | + ((uint)k_qs[i*4 + 1]) << 8 | + ((uint)k_qs[i*4 + 2]) << 16 | + ((uint)k_qs[i*4 + 3]) << 24; + sum = dot_acc_sat_4x8packed_ss_int(q_packed[i], k_packed, sum); + } + return (float)sum * q_d * kd; +} +#endif // FA_HAVE_INT_DOT + +inline void dequant_q8_0_f32(const global char * block_ptr, ACC_TYPE4 * out) { + float d = vload_half(0, (const global half *)block_ptr); + const global char * qs = block_ptr + 2; + + #pragma unroll + for (int i = 0; i < 8; i++) { + out[i] = d * (float4)((float)qs[i*4], (float)qs[i*4+1], (float)qs[i*4+2], (float)qs[i*4+3]); + } +} + +// max_bias<=0 returns 1.0 so score += 1.0 * mask[k] stays a no-op multiplier. +inline float get_alibi_slope(float max_bias, int head_idx, int n_head_log2, float m0, float m1) { + if (max_bias <= 0.0f) return 1.0f; + float base = (head_idx < n_head_log2) ? m0 : m1; + int exph = (head_idx < n_head_log2) ? (head_idx + 1) : (2*(head_idx - n_head_log2) + 1); + return pow(base, (float)exph); +} + +// q1 decode: one query row per WG, threads sweep KV positions. +__kernel void flash_attn_f32_q8_0_q1( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char* q_base = (const global char*)q_void + q_offset; + const global char* k_base = (const global char*)k_void + k_offset; + const global char* v_base = (const global char*)v_void + v_offset; + global char* o_base = (global char*)o_void + o_offset; + + const global char* mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + ACC_TYPE4 q_priv[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + +#ifdef FA_HAVE_INT_DOT + // Quantise Q once per thread; q_priv stays as fp for the V accumulate. + uint q_packed[DK_Q8_BLOCKS * 8]; + float q_d_scale[DK_Q8_BLOCKS]; + #pragma unroll + for (int b = 0; b < DK_Q8_BLOCKS; ++b) { + q_d_scale[b] = quant_q_block_int8_packed(&q_priv[b * 8], &q_packed[b * 8]); + } +#endif + + float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE* sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset); + } + + // One-pass online softmax: per-thread maintains running (m_i, l_i, o_acc), + // updating each as new K positions are processed. Eliminates the second + // K read of the original two-pass implementation. After the loop, threads + // are merged via the standard FA-2 cross-thread reduction (rescale each + // thread's l_i and o_acc by alpha=exp(m_i_thread - m_final), then sum). + ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT; + ACC_TYPE l_i = 0.0f; + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) { + const global char* k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char* v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + + ACC_TYPE score = 0.0f; + #pragma unroll + for (int b = 0; b < DK_Q8_BLOCKS; b++) { +#ifdef FA_HAVE_INT_DOT + score += dot_q8_0_int(k_row + b * Q8_0_BLOCK_SIZE, + &q_packed[b * 8], q_d_scale[b]); +#else + score += dot_q8_0_f32(k_row + b * Q8_0_BLOCK_SIZE, &q_priv[b * 8]); +#endif + } + score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + if (mask_base != NULL) { + const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base); + score += slope * (ACC_TYPE)mask_ptr[k_idx]; + } + + // Online softmax step. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE alpha = exp(m_i - m_new); + const ACC_TYPE p = exp(score - m_new); + + l_i = alpha * l_i + p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha; + + #pragma unroll + for (int b = 0; b < DV_Q8_BLOCKS; b++) { + ACC_TYPE4 v_dequant[8]; + dequant_q8_0_f32(v_row + b * Q8_0_BLOCK_SIZE, v_dequant); + #pragma unroll + for (int i = 0; i < 8; i++) { + o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]); + } + } + + m_i = m_new; + } + + // Cross-thread reduce: max(m_i) -> m_final, then rescale per-thread l_i + // and o_acc by alpha = exp(m_i_thread - m_final) before sum-reduce. + __local ACC_TYPE local_m[Q1_WG_SIZE]; + local_m[tid] = m_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE m_final = local_m[0]; + + const ACC_TYPE alpha_final = exp(m_i - m_final); + l_i *= alpha_final; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final; + + __local ACC_TYPE local_l[Q1_WG_SIZE]; + __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE]; + local_l[tid] = l_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_l[tid] += local_l[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset); + ACC_TYPE l_final = local_l[0]; + + if (sinks_ptr != NULL) { + l_final += exp(sinks_ptr[head_idx] - m_final); + } + + if (l_final > 0.0f) { + const ACC_TYPE l_inv = 1.0f / l_final; + for (int i = 0; i < DV_VEC; i++) { + local_o_comp[tid] = o_acc[i]; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o_comp[tid] += local_o_comp[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv); + } + } + } else if (tid == 0) { + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f); + } +} + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#else +#define REQD_SUBGROUP_SIZE_64 +#endif + +#define VEC_NSG 4 +#define VEC_WG_SIZE (Q1_WG_SIZE * VEC_NSG) +#define Q1V_DV_PER_THREAD ((DV_VEC + Q1_WG_SIZE - 1) / Q1_WG_SIZE) + +inline float4 dequant_q8_0_lane(const global char * block_ptr, int lane) { + const float d = vload_half(0, (const global half *)block_ptr); + const global char * qs = block_ptr + 2 + lane * 4; + return d * (float4)((float)qs[0], (float)qs[1], (float)qs[2], (float)qs[3]); +} + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_q8_0_q1_vec( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int head_batch_idx = get_global_id(1); + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + __local ACC_TYPE4 q_shared[DK_VEC]; + { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + for (int i = tid; i < DK_VEC; i += VEC_WG_SIZE) { + q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + const global ACC_TYPE * sinks_ptr = NULL; + if (sinks_void != NULL) { + sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + } + + ACC_TYPE4 o_acc[Q1V_DV_PER_THREAD]; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + const int kv_per_sg = (n_kv + VEC_NSG - 1) / VEC_NSG; + const int kv_start = sgid * kv_per_sg; + const int kv_end = min(n_kv, kv_start + kv_per_sg); + + for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + + ACC_TYPE4 dot4 = (ACC_TYPE4)(0.0f); + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane = qk % 8; + const float4 k_v = dequant_q8_0_lane(k_row + block_idx * Q8_0_BLOCK_SIZE, lane); + dot4 = mad(q_shared[qk], k_v, dot4); + } + ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3; + ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base; + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE scale_prev = native_exp(m_i - m_new); + const ACC_TYPE p = native_exp(score - m_new); + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + const int block_idx = dv / 8; + const int lane = dv % 8; + const float4 v_v = dequant_q8_0_lane(v_row + block_idx * Q8_0_BLOCK_SIZE, lane); + o_acc[idx] = mad(p, v_v, o_acc[idx] * scale_prev); + } + l_i = l_i * scale_prev + p; + m_i = m_new; + } + + __local ACC_TYPE sg_m[VEC_NSG]; + __local ACC_TYPE sg_l[VEC_NSG]; + __local ACC_TYPE4 sg_o[VEC_NSG][DV_VEC]; + + if (tid_sg == 0) { + sg_m[sgid] = m_i; + sg_l[sgid] = l_i; + } + { + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv] = o_acc[idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + ACC_TYPE m_final = sg_m[0]; + #pragma unroll + for (int s = 1; s < VEC_NSG; ++s) { + m_final = max(m_final, sg_m[s]); + } + if (sinks_ptr != NULL) { + m_final = max(m_final, sinks_ptr[head_idx]); + } + + ACC_TYPE l_final = 0.0f; + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + l_final += sg_l[s] * native_exp(sg_m[s] - m_final); + } + if (sinks_ptr != NULL) { + l_final += native_exp(sinks_ptr[head_idx] - m_final); + } + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset); + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < VEC_NSG; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[s] - m_final); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv], o_merged); + } + o_row[dv] = CONVERT_O_DATA4(o_merged * l_inv); + } + } +} + +// Flash-decoding split pass for q8_0 KV. Partial record: [m, l, O[DV]]. +// Merge kernel from flash_attn_f32_f16.cl is type-agnostic and reused. +#define FA_PARTIAL_FLOATS (2 + DV) + +__kernel void flash_attn_f32_q8_0_q1_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + record_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (kv_start >= kv_end) { + // Empty split: leave sentinel partial for merge. + if (tid == 0) { + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + const int mask_head_idx = head_idx % mask_ne2; + const int mask_batch_idx = batch_idx % mask_ne3; + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 + + (ulong) q_idx * mask_nb1; + } + + ACC_TYPE4 q_priv[DK_VEC]; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < DK_VEC; ++i) { + q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]); + } + +#ifdef FA_HAVE_INT_DOT + uint q_packed[DK_Q8_BLOCKS * 8]; + float q_d_scale[DK_Q8_BLOCKS]; + #pragma unroll + for (int b = 0; b < DK_Q8_BLOCKS; ++b) { + q_d_scale[b] = quant_q_block_int8_packed(&q_priv[b * 8], &q_packed[b * 8]); + } +#endif + + const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + + // One-pass online softmax (FA-2): single sweep over the split's K range, + // updating per-thread (m_i, l_i, o_acc) per position. Eliminates the + // second K read of the original two-pass implementation. + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + ACC_TYPE4 o_acc[DV_VEC]; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + ACC_TYPE score = 0.0f; + #pragma unroll + for (int b = 0; b < DK_Q8_BLOCKS; ++b) { +#ifdef FA_HAVE_INT_DOT + score += dot_q8_0_int(k_row + b * Q8_0_BLOCK_SIZE, &q_packed[b * 8], q_d_scale[b]); +#else + score += dot_q8_0_f32(k_row + b * Q8_0_BLOCK_SIZE, &q_priv[b * 8]); +#endif + } + score *= scale; + if (logit_softcap > 0.0f) { + score = logit_softcap * tanh(score / logit_softcap); + } + if (mask_base != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base); + score += slope * (ACC_TYPE) mask_ptr[k_idx]; + } + + // Online softmax step. + const ACC_TYPE m_new = max(m_i, score); + const ACC_TYPE alpha = exp(m_i - m_new); + const ACC_TYPE p = exp(score - m_new); + + l_i = alpha * l_i + p; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha; + + #pragma unroll + for (int b = 0; b < DV_Q8_BLOCKS; ++b) { + ACC_TYPE4 v_dequant[8]; + dequant_q8_0_f32(v_row + b * Q8_0_BLOCK_SIZE, v_dequant); + #pragma unroll + for (int i = 0; i < 8; ++i) { + o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]); + } + } + + m_i = m_new; + } + + // Cross-thread reduce: max(m_i) -> m_c, then rescale per-thread l_i and + // o_acc by alpha = exp(m_i_thread - m_c) before sum-reduce. + __local ACC_TYPE local_m[Q1_WG_SIZE]; + local_m[tid] = m_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]); + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE m_c = local_m[0]; + + const ACC_TYPE alpha_final = exp(m_i - m_c); + l_i *= alpha_final; + #pragma unroll + for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final; + + __local ACC_TYPE local_l[Q1_WG_SIZE]; + __local ACC_TYPE4 local_o[Q1_WG_SIZE]; + local_l[tid] = l_i; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_l[tid] += local_l[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + const ACC_TYPE l_c = local_l[0]; + + if (tid == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int i = 0; i < DV_VEC; ++i) { + local_o[tid] = o_acc[i]; + barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) { + if (tid < s) local_o[tid] += local_o[tid + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (tid == 0) { + rec_o[i] = local_o[0]; + } + } +} + +// Prefill: q8_0 K/V, n_q > 1. BLOCK_M × BLOCK_N tiling. +// K path keeps packed int8 in local for dp4a QK dot; V path dequant -> half in local. +// Requires DK % QK8_0 == 0 and DV % QK8_0 == 0 (gated in supports_op). +#define KV_DATA_TYPE4 half4 +#define CONVERT_KV_ACC4(x) convert_float4(x) + +#define DK_Q8_BLOCKS_PREFILL (DK / QK8_0) +#define DV_Q8_BLOCKS_PREFILL (DV / QK8_0) + +// N_SPLIT>1 splits DK/DV across N_SPLIT threads per query row; needs +// sub_group_shuffle_xor and DK_Q8_BLOCKS_PREFILL % N_SPLIT == 0. +#ifndef N_SPLIT +#define N_SPLIT 1 +#endif + +#if N_SPLIT > 1 +#define SPLIT_DK_VEC (DK_VEC / N_SPLIT) +#define SPLIT_DV_VEC (DV_VEC / N_SPLIT) +#define SPLIT_DK_Q8_BLOCKS (DK_Q8_BLOCKS_PREFILL / N_SPLIT) +#define WG_SIZE (BLOCK_M * N_SPLIT) +#else +#define SPLIT_DK_VEC DK_VEC +#define SPLIT_DV_VEC DV_VEC +#define SPLIT_DK_Q8_BLOCKS DK_Q8_BLOCKS_PREFILL +#define WG_SIZE BLOCK_M +#endif + +// FA_V_STRATEGY: 0 = dequant V to half in local (default); 2 = keep packed +// int8 in local, dequant in the accumulate loop (smaller local, slightly slower). +#ifndef FA_V_STRATEGY +#define FA_V_STRATEGY 0 +#endif + +#ifndef MQ_GQA +#define MQ_GQA 4 +#endif +#ifndef MQ_NSG_SPLIT +#define MQ_NSG_SPLIT 4 +#endif +#define MQ_SPLIT_WG_SIZE_Q8 (Q1_WG_SIZE * MQ_NSG_SPLIT) + +REQD_SUBGROUP_SIZE_64 +__kernel void flash_attn_f32_q8_0_q1_vec_mq_split( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + // Empty split — write sentinel for each of the MQ_GQA Q-heads. + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q8) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) { + const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1; + const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1; + + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + + for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) { + const int block_idx = qk / 8; + const int lane = qk % 8; + const float4 k_v = dequant_q8_0_lane(k_row + block_idx * Q8_0_BLOCK_SIZE, lane); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + qk], k_v, dot4[h]); + } + } + + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_idx]; + } + score[h] = s; + } + + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + int idx = 0; + for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) { + const int block_idx = dv / 8; + const int lane = dv % 8; + const float4 v_v = dequant_q8_0_lane(v_row + block_idx * Q8_0_BLOCK_SIZE, lane); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][idx] = mad(p_h[h], v_v, o_acc[h][idx] * sp_h[h]); + } + } + } + + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + { + int idx = 0; + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) { + sg_o[sgid][dv_idx] = o_acc[h][idx]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +// flash_attn_f32_q8_0_q1_vec_mq_split_c8 — cluster-parallel variant of the MQ +// split above, port of the f16/q4_0 c8 kernels + +#ifdef HAS_SUBGROUP_SHUFFLE + +#ifndef FA_CL_C +#define FA_CL_C 8 +#endif + +// Lane striping requires DK/DV to divide across the cluster (see f16 c8). +#if (DK_VEC % FA_CL_C) == 0 && (DV_VEC % FA_CL_C) == 0 +#define FA_CL_NCL (Q1_WG_SIZE / FA_CL_C) // clusters (position streams) per subgroup +#define FA_CL_DKQ (DK_VEC / FA_CL_C) // K quartets per lane per row +#define FA_CL_DVQ (DV_VEC / FA_CL_C) // V quartets (o_acc float4s) per lane per head + +#ifdef FA_C8_NO_SG_PIN +#define FA_C8_SG_ATTR_Q8 +#else +#define FA_C8_SG_ATTR_Q8 REQD_SUBGROUP_SIZE_64 +#endif + +FA_C8_SG_ATTR_Q8 +__kernel void flash_attn_f32_q8_0_q1_vec_mq_split_c8( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + const float scale, + const int n_q, + const int n_kv, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void * mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + global float * partial_void, + const int n_splits, + const int kv_per_split +) { + const int tid = get_local_id(0); + const int sgid = tid / Q1_WG_SIZE; + const int tid_sg = tid % Q1_WG_SIZE; + const int cl = tid_sg / FA_CL_C; // cluster id + const int lic = tid_sg % FA_CL_C; // lane in cluster + const int kvhead_batch_idx = get_global_id(1); + const int split_q_idx = get_global_id(2); + const int split_idx = split_q_idx % n_splits; + const int q_idx = split_q_idx / n_splits; + + const int batch_idx = kvhead_batch_idx / n_head_kv; + const int head_kv_idx = kvhead_batch_idx % n_head_kv; + + const int kv_start = split_idx * kv_per_split; + const int kv_end = min(kv_start + kv_per_split, n_kv); + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + + if (kv_start >= kv_end) { + if (tid == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + rec[0] = FA_M_INIT; + rec[1] = 0.0f; + } + } + return; + } + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + + // Stage MQ_GQA Q rows in __local once (uniform across WG). + __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC]; + for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q8) { + const int h = i / DK_VEC; + const int k = i % DK_VEC; + const int head_idx = head_kv_idx * MQ_GQA + h; + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1; + const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset); + q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]); + } + barrier(CLK_LOCAL_MEM_FENCE); + + float slope[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1); + } + + const global char * mask_base[MQ_GQA]; + if (mask_void != NULL) { + const int mask_batch_idx = batch_idx % mask_ne3; + const global char * mask_base_b = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + + (ulong) q_idx * mask_nb1; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const int head_idx = head_kv_idx * MQ_GQA + h; + const int mask_head_idx = head_idx % mask_ne2; + mask_base[h] = mask_base_b + mask_head_idx * mask_nb2; + } + } else { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL; + } + + // Per-CLUSTER online state; o_acc holds this lane's V quartets {lic + FA_CL_C*i}. + ACC_TYPE4 o_acc[MQ_GQA][FA_CL_DVQ]; + ACC_TYPE m_i[MQ_GQA]; + ACC_TYPE l_i[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + m_i[h] = FA_M_INIT; + l_i[h] = 0.0f; + #pragma unroll + for (int i = 0; i < FA_CL_DVQ; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f); + } + + const int kv_len = kv_end - kv_start; + const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT; + const int kv_lo = kv_start + sgid * kv_per_sg; + const int kv_hi = min(kv_end, kv_lo + kv_per_sg); + + // Uniform trip count; tail clamps the row address and drops the score to + // FA_M_INIT (p underflows to 0) so shuffles stay convergent. + const int n_iter = (kv_hi - kv_lo + FA_CL_NCL - 1) / FA_CL_NCL; + const ulong k_row_base = batch_idx * k_nb3 + head_kv_idx * k_nb2; + const ulong v_row_base = batch_idx * v_nb3 + head_kv_idx * v_nb2; + + for (int it = 0; it < n_iter; ++it) { + const int k_idx = kv_lo + cl + it * FA_CL_NCL; + const int valid = k_idx < kv_hi; + const int k_safe = valid ? k_idx : (kv_hi - 1); + + const global char * k_row = k_base + k_row_base + (ulong) k_safe * k_nb1; + const global char * v_row = v_base + v_row_base + (ulong) k_safe * v_nb1; + + // Float-dequant K dot over this lane's quartets of the cluster's row. + ACC_TYPE4 dot4[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int i = 0; i < FA_CL_DKQ; ++i) { + const int qk = lic + FA_CL_C * i; + const float4 k_v = dequant_q8_0_lane(k_row + (qk / 8) * Q8_0_BLOCK_SIZE, qk % 8); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + dot4[h] = mad(q_shared[h * DK_VEC + qk], k_v, dot4[h]); + } + } + + // Cluster-reduce (xor steps < FA_CL_C stay inside the cluster) + score. + ACC_TYPE score[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE s = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3; + #pragma unroll + for (int step = 1; step < FA_CL_C; step <<= 1) { + s += sub_group_shuffle_xor(s, step); + } + s *= scale; + if (logit_softcap > 0.0f) { + s = logit_softcap * tanh(s / logit_softcap); + } + if (mask_base[h] != NULL) { + const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h]; + s += slope[h] * (ACC_TYPE) mask_ptr[k_safe]; + } + score[h] = valid ? s : FA_M_INIT; + } + + // Per-cluster online update (serial chain depth n_iter, not kv_per_sg). + ACC_TYPE p_h[MQ_GQA]; + ACC_TYPE sp_h[MQ_GQA]; + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + const ACC_TYPE m_new = max(m_i[h], score[h]); + sp_h[h] = native_exp(m_i[h] - m_new); + p_h[h] = native_exp(score[h] - m_new); + l_i[h] = l_i[h] * sp_h[h] + p_h[h]; + m_i[h] = m_new; + } + + // V accumulate on this lane's quartets (p = 0 on tail -> inert). + #pragma unroll + for (int i = 0; i < FA_CL_DVQ; ++i) { + const int dv = lic + FA_CL_C * i; + const float4 v_v = dequant_q8_0_lane(v_row + (dv / 8) * Q8_0_BLOCK_SIZE, dv % 8); + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + o_acc[h][i] = mad(p_h[h], v_v, o_acc[h][i] * sp_h[h]); + } + } + } + + // Merge stage 1: fold cluster partials inside the subgroup via shuffles. + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + ACC_TYPE m_c = m_i[h]; + #pragma unroll + for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) { + m_c = max(m_c, sub_group_shuffle_xor(m_c, step)); + } + const ACC_TYPE alpha = native_exp(m_i[h] - m_c); + ACC_TYPE l_c = l_i[h] * alpha; + #pragma unroll + for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) { + l_c += sub_group_shuffle_xor(l_c, step); + } + #pragma unroll + for (int i = 0; i < FA_CL_DVQ; ++i) { + ACC_TYPE4 o = o_acc[h][i] * alpha; + #pragma unroll + for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) { + o.s0 += sub_group_shuffle_xor(o.s0, step); + o.s1 += sub_group_shuffle_xor(o.s1, step); + o.s2 += sub_group_shuffle_xor(o.s2, step); + o.s3 += sub_group_shuffle_xor(o.s3, step); + } + o_acc[h][i] = o; + } + m_i[h] = m_c; + l_i[h] = l_c; + } + + // Merge stage 2: baseline cross-subgroup LDS merge (o published by + // cluster 0's lanes; layout identical to the baseline sg_o). + __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT]; + __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC]; + + if (tid_sg == 0) { + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + sg_m[h][sgid] = m_i[h]; + sg_l[h][sgid] = l_i[h]; + } + } + + #pragma unroll + for (int h = 0; h < MQ_GQA; ++h) { + if (cl == 0) { + #pragma unroll + for (int i = 0; i < FA_CL_DVQ; ++i) { + sg_o[sgid][lic + FA_CL_C * i] = o_acc[h][i]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sgid == 0) { + const int head_idx = head_kv_idx * MQ_GQA + h; + + ACC_TYPE m_c = sg_m[h][0]; + #pragma unroll + for (int s = 1; s < MQ_NSG_SPLIT; ++s) { + m_c = max(m_c, sg_m[h][s]); + } + ACC_TYPE l_c = 0.0f; + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c); + } + + const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) + * n_splits + split_idx); + global float * rec = partial_void + rec_idx * record_stride; + global float4 * rec_o = (global float4 *) (rec + 2); + + if (tid_sg == 0) { + rec[0] = (float) m_c; + rec[1] = (float) l_c; + } + for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) { + ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int s = 0; s < MQ_NSG_SPLIT; ++s) { + const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c); + o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged); + } + rec_o[dv_idx] = o_merged; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +#endif // DK_VEC/DV_VEC divisible by FA_CL_C +#endif // HAS_SUBGROUP_SHUFFLE (q1_vec_mq_split_c8) + +__kernel void flash_attn_f32_q8_0( + const global void * q_void, ulong q_offset, + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * o_void, ulong o_offset, + const float scale, + const int n_q, + const int n_kv, + const int is_causal, + const int n_head, + const ulong q_nb1, const ulong q_nb2, const ulong q_nb3, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const float max_bias, + const float m0, + const float m1, + const int n_head_log2, + const float logit_softcap, + const int n_head_kv, + const global void* mask_void, + const ulong mask_offset, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3, + const global void* sinks_void, + const ulong sinks_offset, + // blk: per-(qblock,kvblock) class from flash_attn_blk_f16 + // (0=masked, 1=mixed, 2=unmasked). NULL disables the prepass opt. + const global void * blk_void +) { + const int tid = get_local_id(0); + const int block_q_idx = get_group_id(0); + const int head_batch_idx = get_global_id(1); + +#if N_SPLIT > 1 + const int q_lane = tid / N_SPLIT; + const int split_idx = tid % N_SPLIT; +#else + const int q_lane = tid; + const int split_idx = 0; +#endif + const int my_query_row = block_q_idx * BLOCK_M + q_lane; + const int query_valid = my_query_row < n_q; + + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const int gqa_ratio = n_head / n_head_kv; + const int head_kv_idx = head_idx / gqa_ratio; + const int mask_head_idx = mask_void != NULL ? head_idx % mask_ne2 : 0; + const int mask_batch_idx = mask_void != NULL ? batch_idx % mask_ne3 : 0; + + const global char * q_base = (const global char *) q_void + q_offset; + const global char * k_base = (const global char *) k_void + k_offset; + const global char * v_base = (const global char *) v_void + v_offset; + global char * o_base = (global char *) o_void + o_offset; + + const global char * mask_base = NULL; + if (mask_void != NULL) { + mask_base = (const global char *) mask_void + mask_offset + + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2; + } + + // BLK_PREPASS_BM may differ from this kernel's BLOCK_M; scale q-block idx. + #ifndef BLK_PREPASS_BM + #define BLK_PREPASS_BM BLOCK_M + #endif + const global char * blk_base = NULL; + int n_kv_blocks = 0; + if (blk_void != NULL) { + n_kv_blocks = (n_kv + BLOCK_N - 1) / BLOCK_N; + const int n_q_blocks_prepass = (n_q + BLK_PREPASS_BM - 1) / BLK_PREPASS_BM; + const int prepass_q_block = (block_q_idx * BLOCK_M) / BLK_PREPASS_BM; + blk_base = (const global char *) blk_void + + (((mask_batch_idx * mask_ne2) + mask_head_idx) * n_q_blocks_prepass + prepass_q_block) * n_kv_blocks; + } + + const int dk_off_vec = split_idx * SPLIT_DK_VEC; + ACC_TYPE4 q_priv[SPLIT_DK_VEC]; + if (query_valid) { + const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1; + const global float4 * q_ptr = (const global float4 *) (q_base + q_row_offset); + #pragma unroll + for (int i = 0; i < SPLIT_DK_VEC; ++i) { + q_priv[i] = q_ptr[dk_off_vec + i]; + } + } else { + #pragma unroll + for (int i = 0; i < SPLIT_DK_VEC; ++i) q_priv[i] = (ACC_TYPE4)(0.0f); + } + +#ifdef FA_HAVE_INT_DOT + uint q_packed_pf[SPLIT_DK_Q8_BLOCKS * 8]; + float q_d_pf[SPLIT_DK_Q8_BLOCKS]; + #pragma unroll + for (int b = 0; b < SPLIT_DK_Q8_BLOCKS; ++b) { + q_d_pf[b] = quant_q_block_int8_packed(&q_priv[b * 8], &q_packed_pf[b * 8]); + } +#endif + + const int dv_off_vec = split_idx * SPLIT_DV_VEC; + ACC_TYPE4 o_acc[SPLIT_DV_VEC]; + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f); + + ACC_TYPE m_i = FA_M_INIT; + ACC_TYPE l_i = 0.0f; + + float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1); + +#ifdef FA_HAVE_INT_DOT +// Accessors so the staging code is layout-agnostic. +#ifdef FA_K_LDS_T +#define FA_K_PACKED(ROW, IDX) l_k_packed[IDX][ROW] +#define FA_K_SCALE(ROW, BLK) l_k_scale[BLK][ROW] +#else +#define FA_K_PACKED(ROW, IDX) l_k_packed[ROW][IDX] +#define FA_K_SCALE(ROW, BLK) l_k_scale[ROW][BLK] +#endif + +#ifdef FA_K_LDS_T + // K tile transposed: [block*8 + g][kv row] instead of [kv row][block*8 + g]. + // + // The QK loop walks 4 KV rows at a time against the same (b, g), so in the original + // layout those 4 values are BLOCK_N*8 uints apart and cost 4 separate 32-bit local + // reads. Transposed they are adjacent, so they are one 128-bit read -- 4x fewer LDS + // issues for the same bytes and no extra registers. That matters because the QK loop + // is LDS-read-issue-bound: a wrong-math probe that kept every dp4a but cut the LDS + // reads ran the whole kernel 41% faster (18.51 -> 10.91 ms/op), and deleting QK + // outright only reached 10.88 -- i.e. essentially ALL of QK's cost is these reads. + __local uint l_k_packed[DK_Q8_BLOCKS_PREFILL * 8][BLOCK_N]; + __local float l_k_scale [DK_Q8_BLOCKS_PREFILL][BLOCK_N]; +#else + __local uint l_k_packed[BLOCK_N][DK_Q8_BLOCKS_PREFILL * 8]; + __local float l_k_scale [BLOCK_N][DK_Q8_BLOCKS_PREFILL]; +#endif +#else + __local half4 l_k[BLOCK_N][DK_VEC]; +#endif + +#if FA_V_STRATEGY == 2 + __local uint l_v_packed[BLOCK_N][DV_Q8_BLOCKS_PREFILL * 8]; + __local float l_v_scale [BLOCK_N][DV_Q8_BLOCKS_PREFILL]; +#else + __local half4 l_v[BLOCK_N][DV_VEC]; +#endif + + for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) { + // Skip fully-masked KV tiles (uniform branch across WG). + char blk_cur = 1; + if (blk_base != NULL) { + blk_cur = blk_base[k_start / BLOCK_N]; + if (blk_cur == 0) continue; + } + + { +#ifdef FA_HAVE_INT_DOT + const int k_blocks_per_row = DK_Q8_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * k_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / k_blocks_per_row; + const int blk = i % k_blocks_per_row; + const int k_row_idx = k_start + row; + if (k_row_idx < n_kv) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; + const global char * blk_ptr = k_base + k_row_off + blk * Q8_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + FA_K_SCALE(row, blk) = df; + #pragma unroll + for (int j = 0; j < 8; ++j) { + uint k_packed = + (uint) qs[j*4 + 0] | + ((uint) qs[j*4 + 1]) << 8 | + ((uint) qs[j*4 + 2]) << 16 | + ((uint) qs[j*4 + 3]) << 24; + FA_K_PACKED(row, blk * 8 + j) = k_packed; + } + } else { + FA_K_SCALE(row, blk) = 0.0f; + #pragma unroll + for (int j = 0; j < 8; ++j) FA_K_PACKED(row, blk * 8 + j) = 0u; + } + } +#else + // Fallback: dequant q8_0 -> half in local memory. + const int k_blocks_per_row = DK / QK8_0; + const int n_blocks_total = BLOCK_N * k_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / k_blocks_per_row; + const int blk = i % k_blocks_per_row; + const int k_row_idx = k_start + row; + if (k_row_idx < n_kv) { + const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1; + const global char * blk_ptr = k_base + k_row_off + blk * Q8_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global char * qs = blk_ptr + 2; + #pragma unroll + for (int j = 0; j < 8; ++j) { + const float4 v = df * (float4)((float) qs[j*4 + 0], + (float) qs[j*4 + 1], + (float) qs[j*4 + 2], + (float) qs[j*4 + 3]); + l_k[row][blk * 8 + j] = (half4)((half) v.s0, (half) v.s1, (half) v.s2, (half) v.s3); + } + } else { + #pragma unroll + for (int j = 0; j < 8; ++j) l_k[row][blk * 8 + j] = (half4)(0.0h); + } + } +#endif + } + // V tile load — strategy-dependent. +#if FA_V_STRATEGY == 2 + { + // Int8 packed V in local memory + per-block scale. Accumulate + // step unpacks inline. + const int v_blocks_per_row = DV_Q8_BLOCKS_PREFILL; + const int n_blocks_total = BLOCK_N * v_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / v_blocks_per_row; + const int blk = i % v_blocks_per_row; + const int v_row_idx = k_start + row; + if (v_row_idx < n_kv) { + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1; + const global char * blk_ptr = v_base + v_row_off + blk * Q8_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global uchar * qs = (const global uchar *)(blk_ptr + 2); + l_v_scale[row][blk] = df; + #pragma unroll + for (int j = 0; j < 8; ++j) { + uint v_packed = + (uint) qs[j*4 + 0] | + ((uint) qs[j*4 + 1]) << 8 | + ((uint) qs[j*4 + 2]) << 16 | + ((uint) qs[j*4 + 3]) << 24; + l_v_packed[row][blk * 8 + j] = v_packed; + } + } else { + l_v_scale[row][blk] = 0.0f; + #pragma unroll + for (int j = 0; j < 8; ++j) l_v_packed[row][blk * 8 + j] = 0u; + } + } + } +#else + { + // Default: dequant V -> half in local memory. + const int v_blocks_per_row = DV / QK8_0; + const int n_blocks_total = BLOCK_N * v_blocks_per_row; + for (int i = tid; i < n_blocks_total; i += WG_SIZE) { + const int row = i / v_blocks_per_row; + const int blk = i % v_blocks_per_row; + const int v_row_idx = k_start + row; + if (v_row_idx < n_kv) { + const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1; + const global char * blk_ptr = v_base + v_row_off + blk * Q8_0_BLOCK_SIZE; + const float df = (float) vload_half(0, (const global half *) blk_ptr); + const global char * qs = blk_ptr + 2; + #pragma unroll + for (int j = 0; j < 8; ++j) { + const float4 v = df * (float4)((float) qs[j*4 + 0], + (float) qs[j*4 + 1], + (float) qs[j*4 + 2], + (float) qs[j*4 + 3]); + l_v[row][blk * 8 + j] = (half4)((half) v.s0, (half) v.s1, (half) v.s2, (half) v.s3); + } + } else { + #pragma unroll + for (int j = 0; j < 8; ++j) l_v[row][blk * 8 + j] = (half4)(0.0h); + } + } + } +#endif + barrier(CLK_LOCAL_MEM_FENCE); + + // QK dot + online softmax. N_SPLIT>1 reduces per-thread partials via shuffle_xor. +#if N_SPLIT > 1 + { +#else + if (query_valid) { +#endif + const int k_blk_base = split_idx * SPLIT_DK_Q8_BLOCKS; + for (int j = 0; j < BLOCK_N; j += 4) { + const int k_row0 = k_start + j; + const int k_row1 = k_start + j + 1; + const int k_row2 = k_start + j + 2; + const int k_row3 = k_start + j + 3; + + ACC_TYPE s0, s1, s2, s3; +#ifdef FA_HAVE_INT_DOT + // dp4a-accelerated QK dot over owned blocks. + s0 = 0.0f; s1 = 0.0f; s2 = 0.0f; s3 = 0.0f; + #pragma unroll + for (int b_local = 0; b_local < SPLIT_DK_Q8_BLOCKS; ++b_local) { + const int b = k_blk_base + b_local; + int sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; +#if defined(FA_K_LDS_T) + // The 4 KV rows are adjacent in the transposed tile, so each (b, g) + // step is ONE 128-bit local read instead of four 32-bit ones. + #pragma unroll + for (int g = 0; g < 8; ++g) { + const uint qp = q_packed_pf[b_local * 8 + g]; + const uint4 kq4 = vload4(0, &l_k_packed[b * 8 + g][j]); + sum0 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s0, sum0); + sum1 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s1, sum1); + sum2 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s2, sum2); + sum3 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s3, sum3); + } +#else + #pragma unroll + for (int g = 0; g < 8; ++g) { + const uint qp = q_packed_pf[b_local * 8 + g]; + sum0 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j ][b * 8 + g], sum0); + sum1 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+1][b * 8 + g], sum1); + sum2 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+2][b * 8 + g], sum2); + sum3 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+3][b * 8 + g], sum3); + } +#endif + const float qd = q_d_pf[b_local]; +#ifdef FA_K_LDS_T + const float4 ks4 = vload4(0, &l_k_scale[b][j]); + s0 += (float)sum0 * qd * ks4.s0; + s1 += (float)sum1 * qd * ks4.s1; + s2 += (float)sum2 * qd * ks4.s2; + s3 += (float)sum3 * qd * ks4.s3; +#else + s0 += (float)sum0 * qd * l_k_scale[j ][b]; + s1 += (float)sum1 * qd * l_k_scale[j+1][b]; + s2 += (float)sum2 * qd * l_k_scale[j+2][b]; + s3 += (float)sum3 * qd * l_k_scale[j+3][b]; +#endif + } +#else + ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f); + ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f); + #pragma unroll + for (int k = 0; k < SPLIT_DK_VEC; ++k) { + const ACC_TYPE4 qk = q_priv[k]; + const int k_abs = dk_off_vec + k; + dot_acc0 = mad(qk, CONVERT_KV_ACC4(l_k[j ][k_abs]), dot_acc0); + dot_acc1 = mad(qk, CONVERT_KV_ACC4(l_k[j+1][k_abs]), dot_acc1); + dot_acc2 = mad(qk, CONVERT_KV_ACC4(l_k[j+2][k_abs]), dot_acc2); + dot_acc3 = mad(qk, CONVERT_KV_ACC4(l_k[j+3][k_abs]), dot_acc3); + } + s0 = dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3; + s1 = dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3; + s2 = dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3; + s3 = dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3; +#endif + +#if N_SPLIT > 1 + // Power-of-2 N_SPLIT: shuffle_xor butterfly. N_SPLIT=3 (DK=96): 3-way shuffle. + #if (N_SPLIT & (N_SPLIT - 1)) == 0 + #pragma unroll + for (int step = 1; step < N_SPLIT; step <<= 1) { + s0 += sub_group_shuffle_xor(s0, step); + s1 += sub_group_shuffle_xor(s1, step); + s2 += sub_group_shuffle_xor(s2, step); + s3 += sub_group_shuffle_xor(s3, step); + } + #else + const uint tri_base = (get_sub_group_local_id() / N_SPLIT) * N_SPLIT; + s0 = sub_group_shuffle(s0, tri_base + 0) + sub_group_shuffle(s0, tri_base + 1) + sub_group_shuffle(s0, tri_base + 2); + s1 = sub_group_shuffle(s1, tri_base + 0) + sub_group_shuffle(s1, tri_base + 1) + sub_group_shuffle(s1, tri_base + 2); + s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2); + s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2); + #endif +#endif + s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale; + if (logit_softcap > 0.0f) { + s0 = logit_softcap * tanh(s0 / logit_softcap); + s1 = logit_softcap * tanh(s1 / logit_softcap); + s2 = logit_softcap * tanh(s2 / logit_softcap); + s3 = logit_softcap * tanh(s3 / logit_softcap); + } + if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; } + + if (is_causal) { + const int causal_limit = n_kv - n_q + my_query_row; + if (k_row0 > causal_limit) s0 = FA_M_INIT; + if (k_row1 > causal_limit) s1 = FA_M_INIT; + if (k_row2 > causal_limit) s2 = FA_M_INIT; + if (k_row3 > causal_limit) s3 = FA_M_INIT; + } + if (k_row0 >= n_kv) s0 = FA_M_INIT; + if (k_row1 >= n_kv) s1 = FA_M_INIT; + if (k_row2 >= n_kv) s2 = FA_M_INIT; + if (k_row3 >= n_kv) s3 = FA_M_INIT; + + if (query_valid && mask_base != NULL && blk_cur != 2) { + const global MASK_DATA_TYPE * mask_ptr = + (const global MASK_DATA_TYPE *) (mask_base + my_query_row * mask_nb1); + if (k_row0 < n_kv) s0 += slope * (ACC_TYPE) mask_ptr[k_row0]; + if (k_row1 < n_kv) s1 += slope * (ACC_TYPE) mask_ptr[k_row1]; + if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2]; + if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3]; + } + + const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3))); + // Whole tile masked (m_new == FA_M_INIT): force the exp() args + // far negative so the tile contributes 0, not exp(0)=1. + const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new; + const ACC_TYPE scale_prev = native_exp(m_i - m_exp); + const ACC_TYPE p0 = native_exp(s0 - m_exp); + const ACC_TYPE p1 = native_exp(s1 - m_exp); + const ACC_TYPE p2 = native_exp(s2 - m_exp); + const ACC_TYPE p3 = native_exp(s3 - m_exp); + +#if FA_V_STRATEGY == 2 + #pragma unroll + for (int b_local = 0; b_local < DV_Q8_BLOCKS_PREFILL / N_SPLIT; ++b_local) { + const int b_abs = split_idx * (DV_Q8_BLOCKS_PREFILL / N_SPLIT) + b_local; + const float d0 = l_v_scale[j ][b_abs]; + const float d1 = l_v_scale[j+1][b_abs]; + const float d2 = l_v_scale[j+2][b_abs]; + const float d3 = l_v_scale[j+3][b_abs]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + const int lane_abs = b_abs * 8 + g; + const int lane_local = b_local * 8 + g; + uint pk0 = l_v_packed[j ][lane_abs]; + uint pk1 = l_v_packed[j+1][lane_abs]; + uint pk2 = l_v_packed[j+2][lane_abs]; + uint pk3 = l_v_packed[j+3][lane_abs]; + float4 v0 = d0 * (float4)((float)(char)(pk0 & 0xff), (float)(char)((pk0>>8)&0xff), (float)(char)((pk0>>16)&0xff), (float)(char)((pk0>>24)&0xff)); + float4 v1 = d1 * (float4)((float)(char)(pk1 & 0xff), (float)(char)((pk1>>8)&0xff), (float)(char)((pk1>>16)&0xff), (float)(char)((pk1>>24)&0xff)); + float4 v2 = d2 * (float4)((float)(char)(pk2 & 0xff), (float)(char)((pk2>>8)&0xff), (float)(char)((pk2>>16)&0xff), (float)(char)((pk2>>24)&0xff)); + float4 v3 = d3 * (float4)((float)(char)(pk3 & 0xff), (float)(char)((pk3>>8)&0xff), (float)(char)((pk3>>16)&0xff), (float)(char)((pk3>>24)&0xff)); + o_acc[lane_local] = mad(p3, v3, + mad(p2, v2, + mad(p1, v1, + mad(p0, v0, + o_acc[lane_local] * scale_prev)))); + } + } +#else // FA_V_STRATEGY == 0 + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) { + const int i_abs = dv_off_vec + i; + o_acc[i] = mad(p3, CONVERT_KV_ACC4(l_v[j+3][i_abs]), + mad(p2, CONVERT_KV_ACC4(l_v[j+2][i_abs]), + mad(p1, CONVERT_KV_ACC4(l_v[j+1][i_abs]), + mad(p0, CONVERT_KV_ACC4(l_v[j ][i_abs]), + o_acc[i] * scale_prev)))); + } +#endif + l_i = l_i * scale_prev + p0 + p1 + p2 + p3; + m_i = m_new; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + // Write output. With N_SPLIT>1 each thread writes its SPLIT_DV_VEC slice. + if (query_valid) { + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + const ACC_TYPE m_sink = sinks_ptr[head_idx]; + const ACC_TYPE m_final = max(m_i, m_sink); + const ACC_TYPE scale_o = exp(m_i - m_final); + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] *= scale_o; + l_i = l_i * scale_o + exp(m_sink - m_final); + m_i = m_final; + } + const ACC_TYPE l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f; + const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1; + global float4 * o_row = (global float4 *) (o_base + o_row_offset); + if (l_inv > 0.0f) { + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = o_acc[i] * l_inv; + } else { + #pragma unroll + for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = (float4)(0.0f); + } + } +} + +// FD Pass 2: merge split partials. Identical across q4_0/q8_0/f16; each FA +// source owns a copy since kernels compile per-source-program. +__kernel void flash_attn_f32_merge( + const global float * partial_void, + global void * o_void, + const ulong o_offset, + const int n_head, + const int n_splits, + const ulong o_nb1, const ulong o_nb2, const ulong o_nb3, + const global void * sinks_void, + const ulong sinks_offset, + const int n_q +) { + const int lane = get_local_id(0); + const int head_batch_idx = get_global_id(1); + const int q_idx = get_global_id(2); + const int batch_idx = head_batch_idx / n_head; + const int head_idx = head_batch_idx % n_head; + + const ulong record_stride = (ulong) FA_PARTIAL_FLOATS; + const ulong record_idx_0 = (((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits; + const global float * rec0 = partial_void + record_idx_0 * record_stride; + + __local ACC_TYPE m_final_shared; + __local ACC_TYPE l_final_shared; + if (lane == 0) { + ACC_TYPE m = FA_M_INIT; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + m = max(m, m_c); + } + ACC_TYPE m_sink = 0.0f; + bool has_sink = false; + if (sinks_void != NULL) { + const global ACC_TYPE * sinks_ptr = + (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset); + m_sink = sinks_ptr[head_idx]; + has_sink = true; + m = max(m, m_sink); + } + ACC_TYPE l = 0.0f; + for (int c = 0; c < n_splits; ++c) { + const ACC_TYPE m_c = rec0[c * record_stride + 0]; + const ACC_TYPE l_c = rec0[c * record_stride + 1]; + if (m_c > FA_M_INIT) { + l += l_c * exp(m_c - m); + } + } + if (has_sink) { + l += exp(m_sink - m); + } + m_final_shared = m; + l_final_shared = l; + } + barrier(CLK_LOCAL_MEM_FENCE); + const ACC_TYPE m_final = m_final_shared; + const ACC_TYPE l_final = l_final_shared; + const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f; + + ACC_TYPE4 o = (ACC_TYPE4)(0.0f); + for (int c = 0; c < n_splits; ++c) { + const global float * rec_c = rec0 + c * record_stride; + const ACC_TYPE m_c = rec_c[0]; + if (m_c <= FA_M_INIT) continue; + const global float4 * rec_oc = (const global float4 *) (rec_c + 2); + const ACC_TYPE scale_c = exp(m_c - m_final); + o = mad((ACC_TYPE4)(scale_c), rec_oc[lane], o); + } + o = o * l_inv; + + const ulong o_row_offset = (ulong) batch_idx * o_nb3 + (ulong) q_idx * o_nb2 + (ulong) head_idx * o_nb1; + global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) ((global char *) o_void + o_offset + o_row_offset); + o_row[lane] = CONVERT_O_DATA4(o); +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl b/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl new file mode 100644 index 0000000..88ead4b --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl @@ -0,0 +1,156 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +__kernel void flash_attn_kv_pad_f16( + const global void * k_void, ulong k_offset, + const global void * v_void, ulong v_offset, + global void * k_pad_void, + global void * v_pad_void, + const int n_kv, + const int n_head_kv, + const int n_batch, + const ulong k_nb1, const ulong k_nb2, const ulong k_nb3, + const ulong v_nb1, const ulong v_nb2, const ulong v_nb3 +) { + const int row_idx = get_global_id(0); + const int head_kv_idx = get_global_id(1); + const int batch_idx = get_global_id(2); + + if (row_idx >= BLOCK_N || head_kv_idx >= n_head_kv || batch_idx >= n_batch) { + return; + } + + const int tail_start = n_kv - (n_kv % BLOCK_N); + const int src_row_idx = tail_start + row_idx; + + const global char * k_src = (const global char *) k_void + k_offset; + const global char * v_src = (const global char *) v_void + v_offset; + global char * k_pad = (global char *) k_pad_void; + global char * v_pad = (global char *) v_pad_void; + + const ulong k_dst_offset = ((ulong) batch_idx * (ulong) n_head_kv + (ulong) head_kv_idx) * ((ulong) BLOCK_N * k_nb1) + (ulong) row_idx * k_nb1; + const ulong v_dst_offset = ((ulong) batch_idx * (ulong) n_head_kv + (ulong) head_kv_idx) * ((ulong) BLOCK_N * v_nb1) + (ulong) row_idx * v_nb1; + + if (src_row_idx < n_kv) { + const ulong k_src_offset = (ulong) batch_idx * k_nb3 + (ulong) head_kv_idx * k_nb2 + (ulong) src_row_idx * k_nb1; + const ulong v_src_offset = (ulong) batch_idx * v_nb3 + (ulong) head_kv_idx * v_nb2 + (ulong) src_row_idx * v_nb1; + + for (ulong i = 0; i < k_nb1; ++i) { + k_pad[k_dst_offset + i] = k_src[k_src_offset + i]; + } + for (ulong i = 0; i < v_nb1; ++i) { + v_pad[v_dst_offset + i] = v_src[v_src_offset + i]; + } + } else { + for (ulong i = 0; i < k_nb1; ++i) { + k_pad[k_dst_offset + i] = 0; + } + for (ulong i = 0; i < v_nb1; ++i) { + v_pad[v_dst_offset + i] = 0; + } + } +} + +__kernel void flash_attn_mask_pad_f16( + const global void * mask_void, ulong mask_offset, + global void * mask_pad_void, + const int n_q, + const int n_kv, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3 +) { + const int col_idx = get_global_id(0); + const int q_row = get_global_id(1); + const int mask_slice = get_global_id(2); + + if (col_idx >= BLOCK_N || q_row >= n_q || mask_slice >= mask_ne2 * mask_ne3) { + return; + } + + const int tail_start = n_kv - (n_kv % BLOCK_N); + const int src_col_idx = tail_start + col_idx; + const int mask_head_idx = mask_slice % mask_ne2; + const int mask_batch_idx = mask_slice / mask_ne2; + + const global char * mask_src_base = (const global char *) mask_void + mask_offset + + (ulong) mask_batch_idx * mask_nb3 + + (ulong) mask_head_idx * mask_nb2 + + (ulong) q_row * mask_nb1; + const global half * mask_src = (const global half *) mask_src_base; + + global half * mask_pad = (global half *) mask_pad_void; + const ulong dst_idx = + (((ulong) mask_batch_idx * (ulong) mask_ne2 + (ulong) mask_head_idx) * (ulong) n_q + (ulong) q_row) * (ulong) BLOCK_N + + (ulong) col_idx; + + mask_pad[dst_idx] = src_col_idx < n_kv ? mask_src[src_col_idx] : (half) (-INFINITY); +} + +// Per-KV-tile mask class. 0=all -inf (skip tile), 1=mixed (apply mask), +// 2=all zero, no -inf (skip mask lookup). Causal diagonal tiles are class 1. +__kernel void flash_attn_blk_f16( + const global void * mask_void, ulong mask_offset, + global char * blk, + const int n_q, + const int n_kv, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3 +) { + const int kv_block_idx = get_global_id(0); + const int q_block_idx = get_global_id(1); + const int mask_slice = get_global_id(2); + + const int n_q_blocks = (n_q + BLOCK_M - 1) / BLOCK_M; + const int n_kv_blocks = (n_kv + BLOCK_N - 1) / BLOCK_N; + if (kv_block_idx >= n_kv_blocks || q_block_idx >= n_q_blocks || mask_slice >= mask_ne2 * mask_ne3) { + return; + } + + const int mask_head_idx = mask_slice % mask_ne2; + const int mask_batch_idx = mask_slice / mask_ne2; + const int q_start = q_block_idx * BLOCK_M; + const int k_start = kv_block_idx * BLOCK_N; + const int q_count = min(BLOCK_M, n_q - q_start); + const int k_count = min(BLOCK_N, n_kv - k_start); + + const half neg_max_half = (half) (-65504.0f); + char has_unmasked = 0; + char has_masked = 0; + char has_nonzero = 0; + + const global char * mask_base = (const global char *) mask_void + mask_offset + + (ulong) mask_batch_idx * mask_nb3 + + (ulong) mask_head_idx * mask_nb2; + + for (int qi = 0; qi < q_count; ++qi) { + const global half * mask_row = (const global half *) (mask_base + (ulong) (q_start + qi) * mask_nb1) + k_start; + for (int ki = 0; ki < k_count; ++ki) { + const half v = mask_row[ki]; + if (v <= neg_max_half) { + has_masked = 1; + } else { + has_unmasked = 1; + if (v != (half) 0.0f) { + has_nonzero = 1; + } + } + } + if (has_masked && has_unmasked) break; // mixed tile — short-circuit. + } + + char res; + if (has_unmasked == 0) { + res = 0; + } else if (has_masked || has_nonzero) { + res = 1; + } else { + res = 2; + } + + blk[((ulong) mask_slice * (ulong) n_q_blocks + (ulong) q_block_idx) * (ulong) n_kv_blocks + (ulong) kv_block_idx] = res; +} diff --git a/ggml/src/ggml-opencl/kernels/flash_attn_repack.cl b/ggml/src/ggml-opencl/kernels/flash_attn_repack.cl new file mode 100644 index 0000000..db78d56 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/flash_attn_repack.cl @@ -0,0 +1,92 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +__kernel void kernel_repack_mask_for_wmm( + const global half* mask_buf, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + global half* mask_buf_padded, + const ulong mask_nb1_padded, + const ulong mask_nb2_padded, + const ulong mask_nb3_padded +) { + int col = get_global_id(0); // 0 .. n_kv + int row = get_global_id(1); // 0 .. n_q + int slice = get_global_id(2); // 0 .. (n_head * n_batch) + + int head_idx = slice % mask_ne2; + int batch_idx = slice / mask_ne2; + + ulong src_off = (ulong)batch_idx * mask_nb3 + (ulong)head_idx * mask_nb2 + (ulong)row * mask_nb1; + ulong dst_off = (ulong)batch_idx * mask_nb3_padded + (ulong)head_idx * mask_nb2_padded + (ulong)row * mask_nb1_padded; + + mask_buf_padded[dst_off / 2 + col] = mask_buf[src_off / 2 + col]; +} + +__kernel void kernel_repack_q_for_wmm( + const global float* q_buf, + const ulong q_nb1, + const ulong q_nb2, + const ulong q_nb3, + const int n_head, + __write_only image3d_t img_q_wmm +) { + int k4 = get_global_id(0); + int row = get_global_id(1); + int slice = get_global_id(2); + int batch_idx = slice / n_head; + int head_idx = slice % n_head; + + + ulong elem_off = (batch_idx * q_nb3 + head_idx * q_nb2 + row * q_nb1) / 4 + (ulong)k4 * 4; + float4 v = vload4(elem_off / 4, q_buf); + + write_imageh(img_q_wmm, (int4)(row, slice, k4, 0), convert_half4(v)); +} + +__kernel void kernel_repack_k_for_wmm( + const global half* k_buf, + const ulong k_nb1, + const ulong k_nb2, + const ulong k_nb3, + const int n_head_kv, + const int n_kv, + __write_only image3d_t img_k_wmm +) { + int kk = get_global_id(0); + int row4 = get_global_id(1); + int slice = get_global_id(2); + int batch_idx = slice / n_head_kv; + int head_kv_idx = slice % n_head_kv; + + ulong base = batch_idx * k_nb3 + head_kv_idx * k_nb2; + int row0 = row4 * 4; + half4 v; + v.x = (row0 + 0 < n_kv) ? k_buf[(base + (ulong)(row0 + 0) * k_nb1) / 2 + kk] : (half)0; + v.y = (row0 + 1 < n_kv) ? k_buf[(base + (ulong)(row0 + 1) * k_nb1) / 2 + kk] : (half)0; + v.z = (row0 + 2 < n_kv) ? k_buf[(base + (ulong)(row0 + 2) * k_nb1) / 2 + kk] : (half)0; + v.w = (row0 + 3 < n_kv) ? k_buf[(base + (ulong)(row0 + 3) * k_nb1) / 2 + kk] : (half)0; + + write_imageh(img_k_wmm, (int4)(kk, row4, slice, 0), v); +} + +__kernel void kernel_repack_v_for_wmm( + const global half* v_buf, + const ulong v_nb1, + const ulong v_nb2, + const ulong v_nb3, + const int n_head_kv, + __write_only image3d_t img_v_wmm +) { + int hdim4 = get_global_id(0); // now fastest — walks contiguous memory + int row = get_global_id(1); + int slice = get_global_id(2); + int batch_idx = slice / n_head_kv; + int head_kv_idx = slice % n_head_kv; + + ulong row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + (ulong)row * v_nb1; + half4 v = vload4((row_off / 2 + (ulong)hdim4 * 4) / 4, v_buf); + + write_imageh(img_v_wmm, (int4)(row, hdim4, slice, 0), v); +} diff --git a/ggml/src/ggml-opencl/kernels/gated_delta_net.cl b/ggml/src/ggml-opencl/kernels/gated_delta_net.cl new file mode 100644 index 0000000..319c982 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gated_delta_net.cl @@ -0,0 +1,249 @@ +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#ifndef S_V +#define S_V 128 +#endif +#ifndef KDA +#define KDA 0 +#endif +#ifndef SUBGROUP_SIZE +#define SUBGROUP_SIZE 64 +#endif +#ifndef LANES_PER_COLUMN +#define LANES_PER_COLUMN 8 +#endif +#ifndef COLS_PER_LANE_GROUP +#define COLS_PER_LANE_GROUP 1 +#endif +#ifndef SUBGROUPS_PER_WG +#define SUBGROUPS_PER_WG 1 +#endif +#ifndef USE_QCOM_SUBGROUP_SHUFFLE +#define USE_QCOM_SUBGROUP_SHUFFLE 0 +#endif + +#define WG_SIZE (SUBGROUP_SIZE * SUBGROUPS_PER_WG) +#define LANE_GROUPS_PER_SG (SUBGROUP_SIZE / LANES_PER_COLUMN) +#define COLS_PER_SG (LANE_GROUPS_PER_SG * COLS_PER_LANE_GROUP) +#define COLS_PER_WG (SUBGROUPS_PER_WG * COLS_PER_SG) +#define ROWS_PER_LANE (S_V / LANES_PER_COLUMN) + +#if USE_QCOM_SUBGROUP_SHUFFLE +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#endif + +// XOR-based parallel sum +// This does a reduction across groups of LANES_PER_COLUMN +static inline float reduce_add_shmem(float partial, __local float * temp, uint lane) { +#if USE_QCOM_SUBGROUP_SHUFFLE + #pragma unroll + for (uint s = LANES_PER_COLUMN / 2u; s > 0u; s >>= 1u) { + partial += qcom_sub_group_shuffle_xor(partial, s, CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, partial); + } + return partial; +#else + temp[lane] = partial; + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + #pragma unroll + for (uint s = LANES_PER_COLUMN / 2u; s > 0u; s >>= 1u) { + float other = temp[lane ^ s]; + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + temp[lane] += other; + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + } + const float result = temp[lane]; + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + return result; +#endif +} + +#define REDUCE_PARTIAL(partial, temp_ptr, lid) \ + ((LANES_PER_COLUMN == 1u) ? (partial) : reduce_add_shmem((partial), (temp_ptr), (lid))) + +// force compiler to optimize kernel for a specific fixed work-group size +__attribute__((reqd_work_group_size(WG_SIZE, 1, 1))) +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_32 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gated_delta_net( + global const char * q_buf, ulong off_q, + global const char * k_buf, ulong off_k, + global const char * v_buf, ulong off_v, + global const char * g_buf, ulong off_g, + global const char * beta_buf, ulong off_beta, + global const char * state_buf, ulong off_state, + global char * dst_buf, ulong off_dst, + uint H_v, + uint n_tokens, + uint n_seqs, + uint s_off, + uint sq1, uint sq2, uint sq3, + uint sv1, uint sv2, uint sv3, + uint sb1, uint sb2, uint sb3, + uint H_k, + uint rq3, + float scale, + uint K) { + + global const float * data_q = (global const float *)(q_buf + off_q); + global const float * data_k = (global const float *)(k_buf + off_k); + global const float * data_v = (global const float *)(v_buf + off_v); + global const float * data_g = (global const float *)(g_buf + off_g); + global const float * data_beta = (global const float *)(beta_buf + off_beta); + global const float * data_state = (global const float *)(state_buf + off_state); + global float * data_dst = (global float *)(dst_buf + off_dst); + + const uint head_id = get_group_id(0); + const uint seq_id = get_group_id(1); + const uint tid = (uint)get_local_id(0); + + const uint sg_id = get_sub_group_id(); // subgroup id + const uint sg_lid = get_sub_group_local_id(); // subgroup lane id + + const uint lane = sg_lid % LANES_PER_COLUMN; + const uint lane_group = sg_lid / LANES_PER_COLUMN; + const uint wg_col_base = get_group_id(2) * COLS_PER_WG; + const uint sg_col_base = wg_col_base + sg_id * COLS_PER_SG; + + const uint iq1 = head_id % H_k; // head index for Q and K + const uint iq3 = seq_id / rq3; // seq index for Q and K + + const uint state_size = S_V * S_V; + // input state holds s0 only [S_v, S_v, H, n_seqs]: per-seq stride is H*D. + const uint state_base = (seq_id * H_v + head_id) * state_size; + const uint q_off_base = iq3 * sq3 + iq1 * sq1; + const uint v_off_base = seq_id * sv3 + head_id * sv1; + const uint gb_off_base = seq_id * sb3 + head_id * sb1; + const uint state_out_base = (seq_id * H_v + head_id) * state_size; + const uint state_size_per_snap = state_size * H_v * n_seqs; + + __local float reduce_temp[WG_SIZE]; + __local float * temp_ptr = reduce_temp + sg_id * SUBGROUP_SIZE; + + float s_shard[COLS_PER_LANE_GROUP][ROWS_PER_LANE]; + #pragma unroll + for (uint cg = 0; cg < COLS_PER_LANE_GROUP; cg++) { + const uint col = sg_col_base + cg * LANE_GROUPS_PER_SG + lane_group; + #pragma unroll + for (uint r = 0; r < ROWS_PER_LANE; r++) { + s_shard[cg][r] = data_state[state_base + col * S_V + r * LANES_PER_COLUMN + lane]; + } + } + + // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. + // When n_tokens < K only slots 0..n_tokens-1 are written; older slots are caller-owned. + uint attn_off = (seq_id * n_tokens * H_v + head_id) * S_V; + + for (uint t = 0; t < n_tokens; t++) { + const uint q_off = q_off_base + t * sq2; + const uint k_off = q_off; + const uint v_off = v_off_base + t * sv2; + const uint gb_off = gb_off_base + t * sb2; + const float beta_val = data_beta[gb_off]; + + float k_reg[ROWS_PER_LANE]; + float q_reg[ROWS_PER_LANE]; +#if KDA + float g_exp[ROWS_PER_LANE]; + #pragma unroll + for (uint r = 0; r < ROWS_PER_LANE; r++) { + const uint i = r * LANES_PER_COLUMN + lane; + k_reg[r] = data_k[k_off + i]; + q_reg[r] = data_q[q_off + i]; + g_exp[r] = exp(data_g[gb_off * S_V + i]); + } +#else + const float g_val = exp(data_g[gb_off]); + + #pragma unroll + for (uint r = 0; r < ROWS_PER_LANE; r++) { + const uint i = r * LANES_PER_COLUMN + lane; + k_reg[r] = data_k[k_off + i]; + q_reg[r] = data_q[q_off + i]; + } +#endif + + #pragma unroll + for (uint cg = 0; cg < COLS_PER_LANE_GROUP; cg++) { + const uint col = sg_col_base + cg * LANE_GROUPS_PER_SG + lane_group; + float v_val = data_v[v_off + col]; + + float kv_shard = 0.0f; + #pragma unroll + for (uint r = 0; r < ROWS_PER_LANE; r++) { +#if KDA + float gs = g_exp[r] * s_shard[cg][r]; + kv_shard += gs * k_reg[r]; +#else + kv_shard += s_shard[cg][r] * k_reg[r]; +#endif + } + +#if !KDA + kv_shard *= g_val; // Applied once instead of ROWS_PER_LANE times +#endif + + const float kv_col = REDUCE_PARTIAL(kv_shard, temp_ptr, sg_lid); + + const float delta_col = (v_val - kv_col) * beta_val; + + float attn_partial = 0.0f; + #pragma unroll + for (uint r = 0; r < ROWS_PER_LANE; r++) { +#if KDA + float gs = g_exp[r] * s_shard[cg][r]; +#else + float gs = g_val * s_shard[cg][r]; +#endif + s_shard[cg][r] = gs + k_reg[r] * delta_col; + attn_partial += s_shard[cg][r] * q_reg[r]; + } + const float attn_col = REDUCE_PARTIAL(attn_partial, temp_ptr, sg_lid); + + if (lane == 0) { + data_dst[attn_off + col] = attn_col * scale; + } + } + attn_off += S_V * H_v; + + if (K > 1u) { + const int target_slot = (int)n_tokens - 1 - (int)t; + if (target_slot >= 0 && target_slot < (int)K) { + #pragma unroll + for (uint cg = 0; cg < COLS_PER_LANE_GROUP; cg++) { + const uint col = sg_col_base + cg * LANE_GROUPS_PER_SG + lane_group; + const uint slot_base = s_off + (uint)target_slot * state_size_per_snap + state_out_base; + #pragma unroll + for (uint r = 0; r < ROWS_PER_LANE; r++) { + data_dst[slot_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[cg][r]; + } + } + } + } + } + + if (K == 1u) { + #pragma unroll + for (uint cg = 0; cg < COLS_PER_LANE_GROUP; cg++) { + const uint col = sg_col_base + cg * LANE_GROUPS_PER_SG + lane_group; + #pragma unroll + for (uint r = 0; r < ROWS_PER_LANE; r++) { + data_dst[s_off + state_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[cg][r]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl new file mode 100644 index 0000000..a359574 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl @@ -0,0 +1,205 @@ +// Cooperative-K DP4A GEMM for 2 to 4 columns with Q4_0 weights and Q8_1 activations. +// Named vector components avoid dynamic private-array indexing, which can cause register spills. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#ifndef COOPK_NSG +#define COOPK_NSG 4 +#endif +#define COOPK_SG 64 + +#ifndef COOPK_ROWS +#define COOPK_ROWS 4 +#endif +#ifndef COOPK_COLS +#define COOPK_COLS 4 +#endif + +#if COOPK_COLS == 4 +typedef float4 coopk_accv; +typedef int4 coopk_dotv; +#define COOPK_CONVF convert_float4 +#else +typedef float2 coopk_accv; +typedef int2 coopk_dotv; +#define COOPK_CONVF convert_float2 +#endif + +// Pack four Q4_0 weights as signed bytes after subtracting 8. +#define EXP40(l) \ + ( (((uint)((int)(( (l) & 0x000Fu) - 8) & 0xFF))) \ + | (((uint)((int)((((l) & 0x00F0u) >> 4) - 8) & 0xFF)) << 8) \ + | (((uint)((int)((((l) & 0x0F00u) >> 8) - 8) & 0xFF)) << 16) \ + | (((uint)((int)((((l) & 0xF000u) >> 12) - 8) & 0xFF)) << 24) ) + +#if COOPK_ROWS == 4 +#define COOPK_DOT(ci, t) \ + d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \ + d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); \ + d2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, d2.s##ci); \ + d3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, d3.s##ci); +#else +#define COOPK_DOT(ci, t) \ + d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \ + d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); +#endif + +#if COOPK_COLS == 4 +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t) +#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3) +#else +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) +#define COOPK_FOR_COLS(F) F(0) F(1) +#endif + +#if COOPK_ROWS == 4 +#define COOPK_KSTEP(t) \ + { \ + ushort4 bl = vload4(0, src0_q + row0 + (ku0 + t) * m); \ + const uint w0 = EXP40(bl.s0); \ + const uint w1 = EXP40(bl.s1); \ + const uint w2 = EXP40(bl.s2); \ + const uint w3 = EXP40(bl.s3); \ + COOPK_DOTS_AT(t) \ + } +#else +#define COOPK_KSTEP(t) \ + { \ + ushort2 bl = vload2(0, src0_q + row0 + (ku0 + t) * m); \ + const uint w0 = EXP40(bl.s0); \ + const uint w1 = EXP40(bl.s1); \ + COOPK_DOTS_AT(t) \ + } +#endif + +// Weights use the untiled Adreno noshuffle SoA layout, with the output row contiguous in each plane. +// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used. +kernel void kernel_gemm_coopk_q4_0_q8_1_dp4a( + global const ushort * src0_q, // four consecutive K nibbles per ushort, [k/4][m] + global const half * src0_d, // block scales, [k/32][m] + global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4] + global const half * src1_da, // activation scales, [n][k/32] + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + + const int gx = get_global_id(0); + const int sg = get_local_id(1); + const int lane = get_local_id(0); + + const int row0 = gx * COOPK_ROWS; + const int num_32blk = k / 32; + const int k_u = k >> 2; + const int k_b = k >> 5; + + // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding. + const int c0 = 0, c1 = 1; +#if COOPK_COLS == 4 + const int c2 = 2, c3 = 3; +#endif + + coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f); +#endif + + for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) { +#if COOPK_ROWS == 4 + half4 scale = vload4(0, src0_d + row0 + blk * m); +#else + half2 scale = vload2(0, src0_d + row0 + blk * m); +#endif + + coopk_accv da; + da.s0 = (float)src1_da[c0*k_b + blk]; + da.s1 = (float)src1_da[c1*k_b + blk]; +#if COOPK_COLS == 4 + da.s2 = (float)src1_da[c2*k_b + blk]; + da.s3 = (float)src1_da[c3*k_b + blk]; +#endif + + // Q4_0 uses one weight scale per 32 K values. + coopk_dotv d0 = (coopk_dotv)(0), d1 = (coopk_dotv)(0); +#if COOPK_ROWS == 4 + coopk_dotv d2 = (coopk_dotv)(0), d3 = (coopk_dotv)(0); +#endif + for (int half_idx = 0; half_idx < 2; ++half_idx) { + const int ku0 = (blk << 3) + half_idx * 4; + + uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0); + uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0); +#if COOPK_COLS == 4 + uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0); + uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0); +#endif + COOPK_KSTEP(0) + COOPK_KSTEP(1) + COOPK_KSTEP(2) + COOPK_KSTEP(3) + } + + acc0 += (float)scale.s0 * da * COOPK_CONVF(d0); + acc1 += (float)scale.s1 * da * COOPK_CONVF(d1); +#if COOPK_ROWS == 4 + acc2 += (float)scale.s2 * da * COOPK_CONVF(d2); + acc3 += (float)scale.s3 * da * COOPK_CONVF(d3); +#endif + } + + // Reuse one local buffer for all rows; the first barrier protects the previous row's reads. + local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)]; + coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f); +#endif + +#define COOPK_REDUCE(accv, outv) \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg == 0) { \ + coopk_accv sum = (accv); \ + for (int s = 0; s < COOPK_NSG - 1; s++) { \ + sum += reduceLM[s * COOPK_SG + lane]; \ + } \ + (outv) = sum; \ + } + + COOPK_REDUCE(acc0, out0) + COOPK_REDUCE(acc1, out1) +#if COOPK_ROWS == 4 + COOPK_REDUCE(acc2, out2) + COOPK_REDUCE(acc3, out3) +#endif + +#undef COOPK_REDUCE + +#if COOPK_ROWS == 4 +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \ + idx += m; \ + } +#else +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \ + idx += m; \ + } +#endif + + if (sg == 0) { + int idx = row0; + COOPK_FOR_COLS(COOPK_STORE_COL) + } + +#undef COOPK_STORE_COL +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl new file mode 100644 index 0000000..31dce62 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl @@ -0,0 +1,264 @@ +// Cooperative-K DP4A GEMM for 2 to 4 columns with Q4_K weights and Q8_1 activations. +// Named vector components avoid dynamic private-array indexing, which can cause register spills. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 +#define K_SCALE_SIZE 12 + +#ifndef COOPK_NSG +#define COOPK_NSG 4 +#endif +#define COOPK_SG 64 + +#ifndef COOPK_ROWS +#define COOPK_ROWS 4 +#endif +#ifndef COOPK_COLS +#define COOPK_COLS 4 +#endif + +#if COOPK_COLS == 4 +typedef float4 coopk_accv; +typedef int4 coopk_dotv; +#define COOPK_CONVF convert_float4 +#else +typedef float2 coopk_accv; +typedef int2 coopk_dotv; +#define COOPK_CONVF convert_float2 +#endif + +// Expand four Q4_K nibbles into bytes for DP4A. +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +#if COOPK_ROWS == 4 +#define COOPK_DOT(ci, t) \ + s0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, s0.s##ci); \ + s1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, s1.s##ci); \ + s2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, s2.s##ci); \ + s3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, s3.s##ci); +#else +#define COOPK_DOT(ci, t) \ + s0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, s0.s##ci); \ + s1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, s1.s##ci); +#endif + +#if COOPK_COLS == 4 +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t) +#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3) +#else +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) +#define COOPK_FOR_COLS(F) F(0) F(1) +#endif + +// The bin layout packs eight K values in each uint, with four nibbles in each 16-bit half. +// Since ku0 is a multiple of four, t / 2 selects the uint and t & 1 selects its half. +#ifdef COOPK_Q_BIN +#define COOPK_QBITS4(t) convert_ushort4(((t) & 1) ? (vload4(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) >> 16) \ + : (vload4(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) & 0xFFFFu)) +#define COOPK_QBITS2(t) convert_ushort2(((t) & 1) ? (vload2(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) >> 16) \ + : (vload2(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) & 0xFFFFu)) +#else +#define COOPK_QBITS4(t) vload4(0, src0_q + row0 + (ku0 + t) * m) +#define COOPK_QBITS2(t) vload2(0, src0_q + row0 + (ku0 + t) * m) +#endif + +#if COOPK_ROWS == 4 +#define COOPK_KSTEP(t) \ + { \ + ushort4 bits = COOPK_QBITS4(t); \ + const uint w0 = EXP4(bits.s0); \ + const uint w1 = EXP4(bits.s1); \ + const uint w2 = EXP4(bits.s2); \ + const uint w3 = EXP4(bits.s3); \ + COOPK_DOTS_AT(t) \ + } +#else +#define COOPK_KSTEP(t) \ + { \ + ushort2 bits = COOPK_QBITS2(t); \ + const uint w0 = EXP4(bits.s0); \ + const uint w1 = EXP4(bits.s1); \ + COOPK_DOTS_AT(t) \ + } +#endif + +inline void get_scale_min_k4_c(int j, global const uchar * q, int stride, + uchar * d, uchar * m, + uchar mask_d6, uchar mask_d4, uchar mask_hi2) { + if (j < 4) { + *d = q[j*stride] & mask_d6; + *m = q[(j + 4)*stride] & mask_d6; + } else { + *d = (q[(j + 4)*stride] & mask_d4) | ((q[(j - 4)*stride] >> 6) << 4); + *m = (q[(j + 4)*stride] >> 4) | ((q[(j )*stride] >> 6) << 4); + } +} + +// Weights use the untiled Adreno noshuffle SoA layout; COOPK_Q_BIN selects uint-transposed q instead of ushort-transposed q. +// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used. +kernel void kernel_gemm_coopk_q4_k_q8_1_dp4a( +#ifdef COOPK_Q_BIN + global const uint * src0_q, // eight consecutive K nibbles per uint, [k/8][m] +#else + global const ushort * src0_q, // four consecutive K nibbles per ushort, [k/4][m] +#endif + global const uchar * src0_s, // packed 6-bit scales/mins, [k/256][12][m] + global const half * src0_d, // super-block scales, [k/256][m] + global const half * src0_dm, // super-block mins, [k/256][m] + global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4] + global const half * src1_da, // activation scales, [n][k/32] + global const half * src1_sa, // scaled activation sums, [n][k/32] + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const int gx = get_global_id(0); // row group + const int sg = get_local_id(1); // K-split subgroup + const int lane = get_local_id(0); + + const int row0 = gx * COOPK_ROWS; + const int num_32blk = k / 32; + const int k_u = k >> 2; // K in uint (int8x4) units + const int k_b = k >> 5; // 32-blocks along K + + // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding. + const int nl = n - 1; + const int c0 = 0; + const int c1 = (1 < n) ? 1 : nl; +#if COOPK_COLS >= 4 + const int c2 = (2 < n) ? 2 : nl; + const int c3 = (3 < n) ? 3 : nl; +#endif + + coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f); +#endif + + for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) { + const int i = blk << 5; + const int sb_idx = blk >> 3; + const int sub_idx = blk & 7; + + global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * m + row0; + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4_c(sub_idx, sc + 0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4_c(sub_idx, sc + 1, m, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); +#if COOPK_ROWS == 4 + half4 dd = vload4(0, src0_d + row0 + sb_idx * m); + half4 dmm = vload4(0, src0_dm + row0 + sb_idx * m); + uchar sv2, mn2, sv3, mn3; + get_scale_min_k4_c(sub_idx, sc + 2, m, &sv2, &mn2, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4_c(sub_idx, sc + 3, m, &sv3, &mn3, mask_d6, mask_d4, mask_hi2); + const float sc2 = (float)dd.s2 * (float)sv2, mv2 = (float)dmm.s2 * (float)mn2; + const float sc3 = (float)dd.s3 * (float)sv3, mv3 = (float)dmm.s3 * (float)mn3; +#else + half2 dd = vload2(0, src0_d + row0 + sb_idx * m); + half2 dmm = vload2(0, src0_dm + row0 + sb_idx * m); +#endif + const float sc0 = (float)dd.s0 * (float)sv0, mv0 = (float)dmm.s0 * (float)mn0; + const float sc1 = (float)dd.s1 * (float)sv1, mv1 = (float)dmm.s1 * (float)mn1; + + // Q4_K scale and min values change every 32 K values. + coopk_dotv s0 = (coopk_dotv)(0), s1 = (coopk_dotv)(0); +#if COOPK_ROWS == 4 + coopk_dotv s2 = (coopk_dotv)(0), s3 = (coopk_dotv)(0); +#endif + + for (int uq = 0; uq < 2; ++uq) { + const int ku0 = (i >> 2) + uq * 4; + + uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0); + uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0); +#if COOPK_COLS >= 4 + uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0); + uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0); +#endif + COOPK_KSTEP(0) + COOPK_KSTEP(1) + COOPK_KSTEP(2) + COOPK_KSTEP(3) + } + + // Q4_K stores q * scale - min for each 32-value block. + // src1_sa already includes the activation scale, so the min term needs no extra factor of da. + coopk_accv da, sa; + da.s0 = (float)src1_da[c0*k_b + blk]; sa.s0 = (float)src1_sa[c0*k_b + blk]; + da.s1 = (float)src1_da[c1*k_b + blk]; sa.s1 = (float)src1_sa[c1*k_b + blk]; +#if COOPK_COLS >= 4 + da.s2 = (float)src1_da[c2*k_b + blk]; sa.s2 = (float)src1_sa[c2*k_b + blk]; + da.s3 = (float)src1_da[c3*k_b + blk]; sa.s3 = (float)src1_sa[c3*k_b + blk]; +#endif + + acc0 += sc0 * da * COOPK_CONVF(s0) - mv0 * sa; + acc1 += sc1 * da * COOPK_CONVF(s1) - mv1 * sa; +#if COOPK_ROWS == 4 + acc2 += sc2 * da * COOPK_CONVF(s2) - mv2 * sa; + acc3 += sc3 * da * COOPK_CONVF(s3) - mv3 * sa; +#endif + } + + // Reuse one local buffer for all rows; the first barrier protects the previous row's reads. + local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)]; + coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f); +#endif + +#define COOPK_REDUCE(accv, outv) \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg == 0) { \ + coopk_accv sum = (accv); \ + for (int s = 0; s < COOPK_NSG - 1; s++) { \ + sum += reduceLM[s * COOPK_SG + lane]; \ + } \ + (outv) = sum; \ + } + + COOPK_REDUCE(acc0, out0) + COOPK_REDUCE(acc1, out1) +#if COOPK_ROWS == 4 + COOPK_REDUCE(acc2, out2) + COOPK_REDUCE(acc3, out3) +#endif + +#undef COOPK_REDUCE + +#if COOPK_ROWS == 4 +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \ + idx += m; \ + } +#else +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \ + idx += m; \ + } +#endif + + if (sg == 0) { + int idx = row0; + COOPK_FOR_COLS(COOPK_STORE_COL) + } + +#undef COOPK_STORE_COL +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl new file mode 100644 index 0000000..4410a23 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl @@ -0,0 +1,216 @@ +// Cooperative-K DP4A GEMM for 2 to 4 columns with Q6_K weights and Q8_1 activations. +// Named vector components avoid dynamic private-array indexing, which can cause register spills. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#ifndef COOPK_NSG +#define COOPK_NSG 4 +#endif +#define COOPK_SG 64 + +#ifndef COOPK_ROWS +#define COOPK_ROWS 4 +#endif +#ifndef COOPK_COLS +#define COOPK_COLS 4 +#endif + +#if COOPK_COLS == 4 +typedef float4 coopk_accv; +typedef int4 coopk_dotv; +#define COOPK_CONVF convert_float4 +#else +typedef float2 coopk_accv; +typedef int2 coopk_dotv; +#define COOPK_CONVF convert_float2 +#endif + +// Combine low nibbles and high bits, subtract 32, and pack four signed bytes for DP4A. +#define EXP6(l, h, mf, mc) \ + ( (((uint)((int)((( (l) & 0x000Fu) | (((h) & 0x03u) << 4)) - 32) & 0xFF))) \ + | (((uint)((int)(((((l) & 0x00F0u) >> 4) | (((h) & 0x0Cu) << 2)) - 32) & 0xFF)) << 8) \ + | (((uint)((int)(((((l) & 0x0F00u) >> 8) | ((h) & 0x30u)) - 32) & 0xFF)) << 16) \ + | (((uint)((int)(((((l) & (mf)) >> 12) | (((h) & (mc)) >> 2)) - 32) & 0xFF)) << 24) ) + +#if COOPK_ROWS == 4 +#define COOPK_DOT(ci, t) \ + d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \ + d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); \ + d2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, d2.s##ci); \ + d3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, d3.s##ci); +#else +#define COOPK_DOT(ci, t) \ + d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \ + d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); +#endif + +#if COOPK_COLS == 4 +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t) +#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3) +#else +#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) +#define COOPK_FOR_COLS(F) F(0) F(1) +#endif + +#if COOPK_ROWS == 4 +#define COOPK_KSTEP(t) \ + { \ + ushort4 bl = vload4(0, src0_ql + row0 + (ku0 + t) * m); \ + uchar4 bh = vload4(0, src0_qh + row0 + (ku0 + t) * m); \ + const uint w0 = EXP6(bl.s0, bh.s0, mask_f000, mask_c0); \ + const uint w1 = EXP6(bl.s1, bh.s1, mask_f000, mask_c0); \ + const uint w2 = EXP6(bl.s2, bh.s2, mask_f000, mask_c0); \ + const uint w3 = EXP6(bl.s3, bh.s3, mask_f000, mask_c0); \ + COOPK_DOTS_AT(t) \ + } +#else +#define COOPK_KSTEP(t) \ + { \ + ushort2 bl = vload2(0, src0_ql + row0 + (ku0 + t) * m); \ + uchar2 bh = vload2(0, src0_qh + row0 + (ku0 + t) * m); \ + const uint w0 = EXP6(bl.s0, bh.s0, mask_f000, mask_c0); \ + const uint w1 = EXP6(bl.s1, bh.s1, mask_f000, mask_c0); \ + COOPK_DOTS_AT(t) \ + } +#endif + +// Weights use the untiled Adreno noshuffle SoA layout, with the output row contiguous in each plane. +// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used. +kernel void kernel_gemm_coopk_q6_k_q8_1_dp4a( + global const ushort * src0_ql, // four consecutive low nibbles per ushort, [k/4][m] + global const uchar * src0_qh, // four consecutive high 2-bit codes per uchar, [k/4][m] + global const ushort * src0_s, // two signed int8 scales per ushort, [k/32][m] + global const half * src0_d, // super-block scales, [k/256][m] + global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4] + global const half * src1_da, // activation scales, [n][k/32] + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + ushort mask_f000, + uchar mask_c0 +) { + dst = (global float *)((global char *)dst + offsetd); + + const int gx = get_global_id(0); + const int sg = get_local_id(1); + const int lane = get_local_id(0); + + const int row0 = gx * COOPK_ROWS; + const int num_32blk = k / 32; + const int k_u = k >> 2; + const int k_b = k >> 5; + + // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding. + const int c0 = 0, c1 = 1; +#if COOPK_COLS == 4 + const int c2 = 2, c3 = 3; +#endif + + coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f); +#endif + + for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) { + const int i = blk << 5; + + // Each ushort packs two signed scale bytes, one for each 16-value half of this 32-value block. + // The half-precision super-block scale is shared by 256 K values. + ushort4 spk = vload4(0, src0_s + row0 + blk * m); + half4 scd = vload4(0, src0_d + row0 + (i >> 8) * m); + + coopk_accv da; + da.s0 = (float)src1_da[c0*k_b + blk]; + da.s1 = (float)src1_da[c1*k_b + blk]; +#if COOPK_COLS == 4 + da.s2 = (float)src1_da[c2*k_b + blk]; + da.s3 = (float)src1_da[c3*k_b + blk]; +#endif + + // Apply each signed weight scale to its 16-value dot product. + for (int half_idx = 0; half_idx < 2; ++half_idx) { + const int ku0 = (i >> 2) + half_idx * 4; + + coopk_dotv d0 = (coopk_dotv)(0), d1 = (coopk_dotv)(0); +#if COOPK_ROWS == 4 + coopk_dotv d2 = (coopk_dotv)(0), d3 = (coopk_dotv)(0); +#endif + uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0); + uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0); +#if COOPK_COLS == 4 + uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0); + uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0); +#endif + COOPK_KSTEP(0) + COOPK_KSTEP(1) + COOPK_KSTEP(2) + COOPK_KSTEP(3) + + const float s0f = (float)(half_idx ? as_char2(spk.s0).s1 : as_char2(spk.s0).s0) * (float)scd.s0; + const float s1f = (float)(half_idx ? as_char2(spk.s1).s1 : as_char2(spk.s1).s0) * (float)scd.s1; + acc0 += s0f * da * COOPK_CONVF(d0); + acc1 += s1f * da * COOPK_CONVF(d1); +#if COOPK_ROWS == 4 + const float s2f = (float)(half_idx ? as_char2(spk.s2).s1 : as_char2(spk.s2).s0) * (float)scd.s2; + const float s3f = (float)(half_idx ? as_char2(spk.s3).s1 : as_char2(spk.s3).s0) * (float)scd.s3; + acc2 += s2f * da * COOPK_CONVF(d2); + acc3 += s3f * da * COOPK_CONVF(d3); +#endif + } + } + + // Reuse one local buffer for all rows; the first barrier protects the previous row's reads. + local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)]; + coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f); +#if COOPK_ROWS == 4 + coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f); +#endif + +#define COOPK_REDUCE(accv, outv) \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + if (sg == 0) { \ + coopk_accv sum = (accv); \ + for (int s = 0; s < COOPK_NSG - 1; s++) { \ + sum += reduceLM[s * COOPK_SG + lane]; \ + } \ + (outv) = sum; \ + } + + COOPK_REDUCE(acc0, out0) + COOPK_REDUCE(acc1, out1) +#if COOPK_ROWS == 4 + COOPK_REDUCE(acc2, out2) + COOPK_REDUCE(acc3, out3) +#endif + +#undef COOPK_REDUCE + +#if COOPK_ROWS == 4 +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \ + idx += m; \ + } +#else +#define COOPK_STORE_COL(ci) \ + if (idx < m*n_no_padding) { \ + vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \ + idx += m; \ + } +#endif + + if (sg == 0) { + int idx = row0; + COOPK_FOR_COLS(COOPK_STORE_COL) + } + +#undef COOPK_STORE_COL +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl new file mode 100644 index 0000000..10c8855 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl @@ -0,0 +1,376 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 + + +static inline half8 mxfp4_to_fp16_packed8(ushort2 fp4x8) { + ushort2 fp16_packed_a_0, fp16_packed_b_0, bias_a, bias_b, sign_a, sign_b; + fp16_packed_a_0.lo = (fp4x8.s0 << 9) & 0x0E00; + fp16_packed_a_0.hi = (fp4x8.s0 << 5) & 0x0E00; + fp16_packed_b_0.lo = (fp4x8.s0 << 1) & 0x0E00; + fp16_packed_b_0.hi = (fp4x8.s0 >> 3) & 0x0E00; + + bias_a.lo = (fp16_packed_a_0.lo != 0) ? 0x3800 : 0x0; + bias_a.hi = (fp16_packed_a_0.hi != 0) ? 0x3800 : 0x0; + bias_b.lo = (fp16_packed_b_0.lo != 0) ? 0x3800 : 0x0; + bias_b.hi = (fp16_packed_b_0.hi != 0) ? 0x3800 : 0x0; + + fp16_packed_a_0.lo = (fp16_packed_a_0.lo != 0x0200) ? fp16_packed_a_0.lo : 0x0; + fp16_packed_a_0.hi = (fp16_packed_a_0.hi != 0x0200) ? fp16_packed_a_0.hi : 0x0; + fp16_packed_b_0.lo = (fp16_packed_b_0.lo != 0x0200) ? fp16_packed_b_0.lo : 0x0; + fp16_packed_b_0.hi = (fp16_packed_b_0.hi != 0x0200) ? fp16_packed_b_0.hi : 0x0; + + sign_a.lo = (fp4x8.s0 << 12) & 0x8000; + sign_a.hi = (fp4x8.s0 << 8) & 0x8000; + sign_b.lo = (fp4x8.s0 << 4) & 0x8000; + sign_b.hi = fp4x8.s0 & 0x8000; + + fp16_packed_a_0 = sign_a + bias_a + fp16_packed_a_0; + fp16_packed_b_0 = sign_b + bias_b + fp16_packed_b_0; + + ushort2 fp16_packed_a_1, fp16_packed_b_1; + fp16_packed_a_1.lo = (fp4x8.s1 << 9) & 0x0E00; + fp16_packed_a_1.hi = (fp4x8.s1 << 5) & 0x0E00; + fp16_packed_b_1.lo = (fp4x8.s1 << 1) & 0x0E00; + fp16_packed_b_1.hi = (fp4x8.s1 >> 3) & 0x0E00; + + bias_a.lo = (fp16_packed_a_1.lo != 0) ? 0x3800 : 0x0; + bias_a.hi = (fp16_packed_a_1.hi != 0) ? 0x3800 : 0x0; + bias_b.lo = (fp16_packed_b_1.lo != 0) ? 0x3800 : 0x0; + bias_b.hi = (fp16_packed_b_1.hi != 0) ? 0x3800 : 0x0; + + fp16_packed_a_1.lo = (fp16_packed_a_1.lo != 0x0200) ? fp16_packed_a_1.lo : 0x0; + fp16_packed_a_1.hi = (fp16_packed_a_1.hi != 0x0200) ? fp16_packed_a_1.hi : 0x0; + fp16_packed_b_1.lo = (fp16_packed_b_1.lo != 0x0200) ? fp16_packed_b_1.lo : 0x0; + fp16_packed_b_1.hi = (fp16_packed_b_1.hi != 0x0200) ? fp16_packed_b_1.hi : 0x0; + + sign_a.lo = (fp4x8.s1 << 12) & 0x8000; + sign_a.hi = (fp4x8.s1 << 8) & 0x8000; + sign_b.lo = (fp4x8.s1 << 4) & 0x8000; + sign_b.hi = fp4x8.s1 & 0x8000; + + fp16_packed_a_1 = sign_a + bias_a + fp16_packed_a_1; + fp16_packed_b_1 = sign_b + bias_b + fp16_packed_b_1; + + return as_half8((ushort8)(fp16_packed_a_0, fp16_packed_b_0, fp16_packed_a_1, fp16_packed_b_1)); +} + + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + + +static inline half e8m0_to_fp16(uchar x) { + ushort bits; + bits = (ushort)(x) - (ushort)(112); + bits = ((bits & 0x00E0) != 0) ? 0x7C00 : (bits << 10); + return as_half(bits); +} + +static inline float e8m0_to_fp32(uchar x) { + int bits; + bits = (x == 0) ? 0x00400000 : ((uint) x << 23); + return as_float(bits); +} + + +__attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair +kernel void kernel_gemm_moe_mxfp4_f32_ns( + __read_only image1d_buffer_t src0_q, + __global uchar * src0_d, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global ushort * src2_emap, + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + uint is_ragged, + uint skip_gran +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + // Boundary check + if (block_id_n >= total_tiles[0]) { + return; + } + + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + // Loop along K axis, 32 elements (one block) for each iteration, divided into 2 sub-blocks + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + // First sub-block + uint q_sub_offset = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + uint s_sub_offset = row + ((ne01 * step) >> 5) + ((expert_id * ne00 * ne01) >> 5); + uint b_sub_offset = col * ne00 + step; + + // Load scale for current mxfp4 block + uint s_offset = s_sub_offset + get_global_id(0); + float s = e8m0_to_fp32(src0_d[s_offset]); + + // Load 16 fp4 (64-bits) in transposed layout + uint2 mxfp4x16; + mxfp4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + mxfp4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + // Cast the e8m0 scale to half to satisfy E17 compilers + reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * (half)s; + reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * (half)s; + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 8 elements reduction for better precision + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + + // Repeat for second sub-block + uint half_step = step + TILESIZE_K; + q_sub_offset = row + ((ne01 * half_step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + b_sub_offset = col * ne00 + half_step; + + // Load next 16 fp4 (64-bits) in transposed layout + mxfp4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + mxfp4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + // Cast the e8m0 scale to half to satisfy E17 compilers + reg_a.lo = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.lo)) * (half)s; + reg_a.hi = mxfp4_to_fp16_packed8(as_ushort2(mxfp4x16.hi)) * (half)s; + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 3-levels reduction for better precision + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + // Load poster router and share in LM + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + // Scatter results back to original position in output grid + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + // Store zero padding parts to the index of first output in tile, override correct result in the end + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl new file mode 100644 index 0000000..97fdc8e --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl @@ -0,0 +1,190 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_M 64 +#define TILESIZE_N 32 + +// 2*mxfp4_value as signed int8, packed 4 codes per uint. Divergent nibble +// lookups read a __constant *uint* array + shift, never a byte array +// (byte-indexed __constant loads serialize on Adreno and are far slower). +// idx 0-3: 0, 1, 2, 3 = 0x03020100 +// idx 4-7: 4, 6, 8, 12 = 0x0C080604 +// idx 8-11: 0, -1, -2, -3 = 0xFDFEFF00 (-1=0xFF,-2=0xFE,-3=0xFD) +// idx 12-15:-4, -6, -8,-12 = 0xF4F8FAFC (-4=0xFC,-6=0xFA,-8=0xF8,-12=0xF4) +__constant uint mxfp4_i8x4[4] = { + 0x03020100u, 0x0C080604u, 0xFDFEFF00u, 0xF4F8FAFCu +}; +inline uint mxfp4_code(uint n) { + return (mxfp4_i8x4[n >> 2] >> ((n & 3u) * 8u)) & 0xFFu; +} +// 4 nibbles in the low 16 bits of u -> 4 codebook int8, packed for dp4a. +inline uint mxfp4_pack(ushort u) { + return mxfp4_code((uint)( u & 0xF)) + | (mxfp4_code((uint)((u >> 4) & 0xF)) << 8) + | (mxfp4_code((uint)((u >> 8) & 0xF)) << 16) + | (mxfp4_code((uint)((u >> 12) & 0xF)) << 24); +} + +static inline float e8m0_to_fp32(uchar x) { + int bits; + bits = (x == 0) ? 0x00400000 : ((uint) x << 23); + return as_float(bits); +} + +// One token's dp4a dot (8 uints = 32 K elems) + mxfp4 block-scale epilogue. +// blk_scale already carries the 0.5 factor (== 0.5 * 2^e). +#define MOE_MXFP4_DP4A_T(t) do { \ + uint4 a0 = vload4(0, &sh_qa[t][0]); \ + uint4 a1 = vload4(0, &sh_qa[t][4]); \ + int raw = 0; \ + raw = dot_acc_sat_4x8packed_ss_int(qw[0], a0.s0, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[1], a0.s1, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[2], a0.s2, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[3], a0.s3, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[4], a1.s0, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[5], a1.s1, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[6], a1.s2, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[7], a1.s3, raw); \ + acc[t] += blk_scale * (float)sh_d[t] * (float)raw; \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_mxfp4_q8_1_dp4a( + __read_only image1d_buffer_t src0_q, // mxfp4 codes (transposed, packed nibbles) + __global uchar * src0_e, // e8m0 per-32-block scale + __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap, // tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged // 1: compute only real tokens per tile +) { + const uint block_id_m = get_global_id(1); // m_tile + const uint block_id_n = get_global_id(2); // n_tile + + if (block_id_n >= total_tiles[0]) { + return; + } + + const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile + + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + const uint num_blocks = ne00 >> 5; // blocks-of-32 per token + const uint row_idx = row + lid; + + const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units + + __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB + __local half sh_d[TILESIZE_N]; + + // Real token count for this tile. + // Real tokens are packed contiguously at the tile start; padded slots hold + // 0xFFFFFFFF (only the last tile of each expert is partial). is_ragged skips + // the dp4a/staging/scatter for padded slots; is_ragged==0 forces n_real=32. + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) { + sh_src2[lid] = src2[col + lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { + nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { + if (sh_src2[t] != 0xFFFFFFFFu) ++nr; + } + } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; // 32-block index along K + + // e8m0 block scale for this WI's row, this 32-block (folded x0.5) + const uint e_offset = row_idx + sub * ne01 + expert_id * num_blocks * ne01; + const float blk_scale = 0.5f * e8m0_to_fp32(src0_e[e_offset]); + + // repack this WI's 32 weight nibbles into 8 dp4a uints + const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint r0 = read_imageui(src0_q, qoff0 + lid).x; + const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x; + const uint r2 = read_imageui(src0_q, qoff1 + lid).x; + const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x; + uint qw[8]; + qw[0] = mxfp4_pack((ushort)(r0)); qw[1] = mxfp4_pack((ushort)(r0 >> 16)); + qw[2] = mxfp4_pack((ushort)(r1)); qw[3] = mxfp4_pack((ushort)(r1 >> 16)); + qw[4] = mxfp4_pack((ushort)(r2)); qw[5] = mxfp4_pack((ushort)(r2 >> 16)); + qw[6] = mxfp4_pack((ushort)(r3)); qw[7] = mxfp4_pack((ushort)(r3 >> 16)); + + // cooperatively stage the n_real-token x 32-K int8 activations + // Stage each token's 8 activation uints as two 128-bit uint4 loads/stores. + const uint vlim = (uint)n_real * 2; + for (uint idx = lid; idx < vlim; idx += 64) { + const uint t = idx >> 1; + const uint h = (idx & 1) << 2; // 0 or 4 + uint4 v = vload4(0, &src1_qa[(col + t) * ne00_u + (step >> 2) + h]); + vstore4(v, 0, &sh_qa[t][h]); + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * num_blocks + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + // Full tiles keep the fully-unrolled 32-wide loop; partial tiles run only n_real + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_MXFP4_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_MXFP4_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) { + return; + } + + // scatter results to original output rows (reuse sh_src2 from the top) + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) { + idx = sh_src2[0]; + } + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl new file mode 100644 index 0000000..58f7932 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl @@ -0,0 +1,324 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 + + +#define dequantize_q4_0(q4, a_f16, scale) \ + a_f16.s0 = (half)((q4.s0 & 0x000F) - 8) * scale; \ + a_f16.s1 = (half)(((q4.s0 & 0x00F0) >> 4) - 8) * scale; \ + a_f16.s2 = (half)(((q4.s0 & 0x0F00) >> 8) - 8) * scale; \ + a_f16.s3 = (half)(((q4.s0 & 0xF000) >> 12) - 8) * scale; \ + a_f16.s4 = (half)((q4.s1 & 0x000F) - 8) * scale; \ + a_f16.s5 = (half)(((q4.s1 & 0x00F0) >> 4) - 8) * scale; \ + a_f16.s6 = (half)(((q4.s1 & 0x0F00) >> 8) - 8) * scale; \ + a_f16.s7 = (half)(((q4.s1 & 0xF000) >> 12) - 8) * scale; \ + a_f16.s8 = (half)((q4.s2 & 0x000F) - 8) * scale; \ + a_f16.s9 = (half)(((q4.s2 & 0x00F0) >> 4) - 8) * scale; \ + a_f16.sa = (half)(((q4.s2 & 0x0F00) >> 8) - 8) * scale; \ + a_f16.sb = (half)(((q4.s2 & 0xF000) >> 12) - 8) * scale; \ + a_f16.sc = (half)((q4.s3 & 0x000F) - 8) * scale; \ + a_f16.sd = (half)(((q4.s3 & 0x00F0) >> 4) - 8) * scale; \ + a_f16.se = (half)(((q4.s3 & 0x0F00) >> 8) - 8) * scale; \ + a_f16.sf = (half)(((q4.s3 & 0xF000) >> 12) - 8) * scale; \ + + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + + +__attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair +kernel void kernel_gemm_moe_q4_0_f32_ns( + __read_only image1d_buffer_t src0_q, + __global half * src0_d, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global ushort * src2_emap, + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + uint is_ragged, + uint skip_gran +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + // Boundary check + if (block_id_n >= total_tiles[0]) { + return; + } + + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + // Loop along K axis, 32 elements (one block) for each iteration, divided into 2 sub-blocks + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + // First sub-block + uint q_sub_offset = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + uint s_sub_offset = row + ((ne01 * step) >> 5) + ((expert_id * ne00 * ne01) >> 5); + uint b_sub_offset = col * ne00 + step; + + // Load scale for current Q4_0 block + uint s_offset = s_sub_offset + get_global_id(0); + half s = src0_d[s_offset]; + + // Load 16 q (64-bits) in transposed layout + uint2 q4x16; + q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q4_0(as_ushort4(q4x16), reg_a, s); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 8 elements reduction for better precision + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + + // Repeat for second sub-block + uint half_step = step + TILESIZE_K; + q_sub_offset = row + ((ne01 * half_step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + b_sub_offset = col * ne00 + half_step; + + // Load next 16 q (64-bits) in transposed layout + q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q4_0(as_ushort4(q4x16), reg_a, s); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 3-levels reduction for better precision + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + // Load poster router and share in LM + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + // Scatter results back to original position in output grid + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + // Store zero padding parts to the index of first output in tile, override correct result in the end + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl new file mode 100644 index 0000000..5024720 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl @@ -0,0 +1,169 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_M 64 +#define TILESIZE_N 32 + +// Expand the 4 nibbles held in the low 16 bits of `u` into 4 bytes (one nibble +// per byte, value 0..15), packed for the int8 dp4a. The -8 zero-point is applied +// in the epilogue via the activation sum term (cheaper than biasing every byte). +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// One token's dp4a dot (8 uints = 32 K elems) + q4_0 scale/zero-point epilogue. +#define MOE_Q40_DP4A_T(t) do { \ + uint4 a0 = vload4(0, &sh_qa[t][0]); \ + uint4 a1 = vload4(0, &sh_qa[t][4]); \ + int raw = 0; \ + raw = dot_acc_sat_4x8packed_ss_int(qw[0], a0.s0, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[1], a0.s1, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[2], a0.s2, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[3], a0.s3, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[4], a1.s0, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[5], a1.s1, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[6], a1.s2, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[7], a1.s3, raw); \ + acc[t] += d_val * ((float)sh_d[t] * (float)raw - 8.0f * (float)sh_s[t]); \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q4_0_q8_1_dp4a( + __read_only image1d_buffer_t src0_q, // q4_0 weights (transposed, packed nibbles) + __global half * src0_d, // per-32-block scale + __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap,// tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged // 1: compute only real tokens per tile +) { + const uint block_id_m = get_global_id(1); // m_tile + const uint block_id_n = get_global_id(2); // n_tile + + if (block_id_n >= total_tiles[0]) { + return; + } + + const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile + + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + const uint num_blocks = ne00 >> 5; // blocks-of-32 per token + const uint row_idx = row + lid; + + const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units + + __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + + // Real-token count for this tile + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) { + sh_src2[lid] = src2[col + lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { + nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { + if (sh_src2[t] != 0xFFFFFFFFu) ++nr; + } + } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; // 32-block index along K + + // per-32-block scale for this WI's row + const uint d_offset = row_idx + sub * ne01 + expert_id * num_blocks * ne01; + const float d_val = (float)src0_d[d_offset]; + + // repack this WI's 32 weight nibbles into 8 dp4a uints + const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint r0 = read_imageui(src0_q, qoff0 + lid).x; + const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x; + const uint r2 = read_imageui(src0_q, qoff1 + lid).x; + const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x; + uint qw[8]; + qw[0] = EXP4(r0); qw[1] = EXP4(r0 >> 16); + qw[2] = EXP4(r1); qw[3] = EXP4(r1 >> 16); + qw[4] = EXP4(r2); qw[5] = EXP4(r2 >> 16); + qw[6] = EXP4(r3); qw[7] = EXP4(r3 >> 16); + + // cooperatively stage the n_real-token x 32-K int8 activations + // Stage each token's 8 activation uints as two 128-bit uint4 loads/stores. + const uint vlim = (uint)n_real * 2; + for (uint idx = lid; idx < vlim; idx += 64) { + const uint t = idx >> 1; + const uint h = (idx & 1) << 2; // 0 or 4 + uint4 v = vload4(0, &src1_qa[(col + t) * ne00_u + (step >> 2) + h]); + vstore4(v, 0, &sh_qa[t][h]); + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * num_blocks + sub]; + sh_s[lid] = src1_sa[(col + lid) * num_blocks + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q40_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_Q40_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) { + return; + } + + // scatter results to original output rows (reuse sh_src2 from the top) + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) { + idx = sh_src2[0]; + } + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl new file mode 100644 index 0000000..6593124 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl @@ -0,0 +1,326 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 + + +#define dequantize_q4_1(q4, a_f16, scale, m) \ + a_f16.s0 = (half)(q4.s0 & 0x000F) * scale + m; \ + a_f16.s1 = (half)((q4.s0 & 0x00F0) >> 4) * scale + m; \ + a_f16.s2 = (half)((q4.s0 & 0x0F00) >> 8) * scale + m; \ + a_f16.s3 = (half)((q4.s0 & 0xF000) >> 12) * scale + m; \ + a_f16.s4 = (half)(q4.s1 & 0x000F) * scale + m; \ + a_f16.s5 = (half)((q4.s1 & 0x00F0) >> 4) * scale + m; \ + a_f16.s6 = (half)((q4.s1 & 0x0F00) >> 8) * scale + m; \ + a_f16.s7 = (half)((q4.s1 & 0xF000) >> 12) * scale + m; \ + a_f16.s8 = (half)(q4.s2 & 0x000F) * scale + m; \ + a_f16.s9 = (half)((q4.s2 & 0x00F0) >> 4) * scale + m; \ + a_f16.sa = (half)((q4.s2 & 0x0F00) >> 8) * scale + m; \ + a_f16.sb = (half)((q4.s2 & 0xF000) >> 12) * scale + m; \ + a_f16.sc = (half)(q4.s3 & 0x000F) * scale + m; \ + a_f16.sd = (half)((q4.s3 & 0x00F0) >> 4) * scale + m; \ + a_f16.se = (half)((q4.s3 & 0x0F00) >> 8) * scale + m; \ + a_f16.sf = (half)((q4.s3 & 0xF000) >> 12) * scale + m; \ + + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + + +__attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair +kernel void kernel_gemm_moe_q4_1_f32_ns( + __read_only image1d_buffer_t src0_q, + __global half * src0_d, + __global half * src0_m, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global ushort * src2_emap, + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + uint is_ragged, + uint skip_gran +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + // Boundary check + if (block_id_n >= total_tiles[0]) { + return; + } + + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + // Loop along K axis, 32 elements (one block) for each iteration, divided into 2 sub-blocks + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + // First sub-block + uint q_sub_offset = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + uint s_sub_offset = row + ((ne01 * step) >> 5) + ((expert_id * ne00 * ne01) >> 5); + uint b_sub_offset = col * ne00 + step; + + // Load scale and m for current Q4_1 block + uint sm_offset = s_sub_offset + get_global_id(0); + half s = src0_d[sm_offset]; + half m = src0_m[sm_offset]; + + // Load 16 q (64-bits) in transposed layout + uint2 q4x16; + q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q4_1(as_ushort4(q4x16), reg_a, s, m); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 8 elements reduction for better precision + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + + // Repeat for second sub-block + uint half_step = step + TILESIZE_K; + q_sub_offset = row + ((ne01 * half_step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + b_sub_offset = col * ne00 + half_step; + + // Load next 16 q (64-bits) in transposed layout + q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q4_1(as_ushort4(q4x16), reg_a, s, m); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 3-levels reduction for better precision + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + // Load poster router and share in LM + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + // Scatter results back to original position in output grid + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + // Store zero padding parts to the index of first output in tile, override correct result in the end + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl new file mode 100644 index 0000000..f961896 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl @@ -0,0 +1,348 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 +#define QK_K 256 +#define K_SCALE_SIZE 12 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j+4] & 63; + } else { + *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2); + *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2); + } +} + +#define dequantize_q4_k(q4, a_f16, scale, minv) \ + a_f16.s0 = (half)((float)(q4.s0 & 0x000F) * scale - minv); \ + a_f16.s1 = (half)((float)((q4.s0 & 0x00F0) >> 4) * scale - minv); \ + a_f16.s2 = (half)((float)((q4.s0 & 0x0F00) >> 8) * scale - minv); \ + a_f16.s3 = (half)((float)((q4.s0 & 0xF000) >> 12) * scale - minv); \ + a_f16.s4 = (half)((float)(q4.s1 & 0x000F) * scale - minv); \ + a_f16.s5 = (half)((float)((q4.s1 & 0x00F0) >> 4) * scale - minv); \ + a_f16.s6 = (half)((float)((q4.s1 & 0x0F00) >> 8) * scale - minv); \ + a_f16.s7 = (half)((float)((q4.s1 & 0xF000) >> 12) * scale - minv); \ + a_f16.s8 = (half)((float)(q4.s2 & 0x000F) * scale - minv); \ + a_f16.s9 = (half)((float)((q4.s2 & 0x00F0) >> 4) * scale - minv); \ + a_f16.sa = (half)((float)((q4.s2 & 0x0F00) >> 8) * scale - minv); \ + a_f16.sb = (half)((float)((q4.s2 & 0xF000) >> 12) * scale - minv); \ + a_f16.sc = (half)((float)(q4.s3 & 0x000F) * scale - minv); \ + a_f16.sd = (half)((float)((q4.s3 & 0x00F0) >> 4) * scale - minv); \ + a_f16.se = (half)((float)((q4.s3 & 0x0F00) >> 8) * scale - minv); \ + a_f16.sf = (half)((float)((q4.s3 & 0xF000) >> 12) * scale - minv); \ + + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q4_k_f32_ns( + __read_only image1d_buffer_t src0_q, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global ushort * src2_emap, + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + uint is_ragged, + uint skip_gran +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + // Boundary check + if (block_id_n >= total_tiles[0]) { + return; + } + + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + uint num_superblocks = ne00 / QK_K; + uint scales_per_row = num_superblocks * K_SCALE_SIZE; + uint row_idx = row + get_global_id(0); + + // Loop along K axis, 32 elements per iteration (one sub-block), divided into 2 halves of 16 + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + uint sub = step / 32; + uint sb = sub / 8; + uint j = sub % 8; + + // Load d and dm for super-block + uint d_offset = row + sb * ne01 + expert_id * num_superblocks * ne01 + get_global_id(0); + half d_val = src0_d[d_offset]; + half dm_val = src0_dm[d_offset]; + + // Load sub-block scale and min + global const uchar * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(j, sc, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = (float)dm_val * (float)mn; + + // First sub-block (16 elements) + uint q_sub_offset = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + uint b_sub_offset = col * ne00 + step; + + // Load 16 q (64-bits) in transposed layout + uint2 q4x16; + q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q4_k(as_ushort4(q4x16), reg_a, scale, minv); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + + // Second half (next 16 elements, same sub-block scale) + uint half_step = step + TILESIZE_K; + q_sub_offset = row + ((ne01 * half_step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + b_sub_offset = col * ne00 + half_step; + + q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + dequantize_q4_k(as_ushort4(q4x16), reg_a, scale, minv); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + // Load post router and share in LM + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + // Scatter results back to original position in output grid + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + // Store zero padding parts to the index of first output in tile + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl new file mode 100644 index 0000000..9d968f3 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl @@ -0,0 +1,209 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// q4_K subblock (32 elems): w_i = scale*q_i - minv, q_i in [0,15], scale = +// d_super*sv6, minv = dmin_super*mn6. With activation block (a_d, a_s, qa[32]): +// Sum_i w_i * a_i = scale * a_d * dp4a(q, qa) - minv * a_s +// where a_s = a_d * Sum(qa) (the q8_1 "s" field) + +#define TILESIZE_M 64 +#define TILESIZE_N 32 +#define QK_K 256 +#define K_SCALE_SIZE 12 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j+4] & 63; + } else { + *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2); + *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2); + } +} + +// Expand the 4 nibbles held in the low 16 bits of `u` into 4 bytes (one nibble +// per byte, value 0..15), packed for the int8 dp4a. +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// One token's dp4a dot (8 uints = 32 K elems) + q4_K scale/min epilogue into acc[t]. +// The 8 activation uints are read as two 128-bit uint4 loads staged to private (Adreno +// wants 128-bit local reads, and a __local operand fed straight to the dp4a builtin is +// slower and can miscompile). +#define MOE_Q4K_DP4A_T(t) do { \ + uint4 a0 = vload4(0, &sh_qa[t][0]); \ + uint4 a1 = vload4(0, &sh_qa[t][4]); \ + int raw = 0; \ + raw = dot_acc_sat_4x8packed_ss_int(qw[0], a0.s0, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[1], a0.s1, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[2], a0.s2, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[3], a0.s3, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[4], a1.s0, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[5], a1.s1, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[6], a1.s2, raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[7], a1.s3, raw); \ + acc[t] += scale * (float)sh_d[t] * (float)raw - minv * (float)sh_s[t]; \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q4_k_q8_1_dp4a( + __read_only image1d_buffer_t src0_q, // q4_K weights (transposed, packed nibbles) + __global half * src0_d, // per-superblock scale + __global half * src0_dm, // per-superblock min + __global uchar * src0_s, // 6-bit scale/min codes + __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap,// tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged // 1: compute only real tokens per tile +) { + const uint block_id_m = get_global_id(1); // m_tile + const uint block_id_n = get_global_id(2); // n_tile + + if (block_id_n >= total_tiles[0]) { + return; + } + + const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile + + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + const uint num_superblocks = ne00 / QK_K; + const uint scales_per_row = num_superblocks * K_SCALE_SIZE; + const uint row_idx = row + lid; + + const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units + const uint ne00_b = ne00 >> 5; // blocks-of-32 per token + + __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + + // Real token count for this tile + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) { + sh_src2[lid] = src2[col + lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { + nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { + if (sh_src2[t] != 0xFFFFFFFFu) ++nr; + } + } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; // subblock index along K + const uint sb = sub >> 3; // superblock index + const uint j = sub & 7; // subblock within superblock + + // --- weight scale / min for this WI's row, this subblock --- + const uint d_offset = row + sb * ne01 + expert_id * num_superblocks * ne01 + lid; + const float d_val = (float)src0_d[d_offset]; + const float dm_val = (float)src0_dm[d_offset]; + + global const uchar * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(j, sc, &sv, &mn); + const float scale = d_val * (float)sv; + const float minv = dm_val * (float)mn; + + // --- repack this WI's 32 weight nibbles into 8 dp4a uints --- + const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint r0 = read_imageui(src0_q, qoff0 + lid).x; + const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x; + const uint r2 = read_imageui(src0_q, qoff1 + lid).x; + const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x; + uint qw[8]; + qw[0] = EXP4(r0); qw[1] = EXP4(r0 >> 16); + qw[2] = EXP4(r1); qw[3] = EXP4(r1 >> 16); + qw[4] = EXP4(r2); qw[5] = EXP4(r2 >> 16); + qw[6] = EXP4(r3); qw[7] = EXP4(r3 >> 16); + + // cooperatively stage the n_real-token x 32-K int8 activations to lm + // Stage each token's 8 activation uints as two 128-bit uint4 loads/stores. + const uint vlim = (uint)n_real * 2; + for (uint idx = lid; idx < vlim; idx += 64) { + const uint t = idx >> 1; + const uint h = (idx & 1) << 2; // 0 or 4 + uint4 v = vload4(0, &src1_qa[(col + t) * ne00_u + (step >> 2) + h]); + vstore4(v, 0, &sh_qa[t][h]); + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * ne00_b + sub]; + sh_s[lid] = src1_sa[(col + lid) * ne00_b + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + // dp4a - each real token sum over 8 uints (32 K), then scale/min + // Full tiles keep the fully-unrolled 32-wide loop; + // partial tiles run only n_real (saves the padded-slot dp4a + staging). + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q4K_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_Q4K_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) { + return; + } + + // scatter results to original output rows + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) { + idx = sh_src2[0]; + } + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl new file mode 100644 index 0000000..4b8dc52 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl @@ -0,0 +1,328 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 + + +#define dequantize_q5_0(qs5x16, qh5x16, a_f16, scale) \ + a_f16.s0 = (half)((( qs5x16.s0 & 0x000F) | (( qh5x16.s0 & 0x01) << 4)) - 16) * scale; \ + a_f16.s1 = (half)((((qs5x16.s0 & 0x00F0) >> 4 ) | (((qh5x16.s0 >> 1) & 0x01) << 4)) - 16) * scale; \ + a_f16.s2 = (half)((((qs5x16.s0 & 0x0F00) >> 8 ) | (((qh5x16.s0 >> 2) & 0x01) << 4)) - 16) * scale; \ + a_f16.s3 = (half)((((qs5x16.s0 & 0xF000) >> 12) | (((qh5x16.s0 >> 3) & 0x01) << 4)) - 16) * scale; \ + a_f16.s4 = (half)((( qs5x16.s1 & 0x000F) | (((qh5x16.s0 >> 4) & 0x01) << 4)) - 16) * scale; \ + a_f16.s5 = (half)((((qs5x16.s1 & 0x00F0) >> 4 ) | (((qh5x16.s0 >> 5) & 0x01) << 4)) - 16) * scale; \ + a_f16.s6 = (half)((((qs5x16.s1 & 0x0F00) >> 8 ) | (((qh5x16.s0 >> 6) & 0x01) << 4)) - 16) * scale; \ + a_f16.s7 = (half)((((qs5x16.s1 & 0xF000) >> 12) | (((qh5x16.s0 >> 7) & 0x01) << 4)) - 16) * scale; \ + a_f16.s8 = (half)((( qs5x16.s2 & 0x000F) | (( qh5x16.s1 & 0x01) << 4)) - 16) * scale; \ + a_f16.s9 = (half)((((qs5x16.s2 & 0x00F0) >> 4 ) | (((qh5x16.s1 >> 1) & 0x01) << 4)) - 16) * scale; \ + a_f16.sa = (half)((((qs5x16.s2 & 0x0F00) >> 8 ) | (((qh5x16.s1 >> 2) & 0x01) << 4)) - 16) * scale; \ + a_f16.sb = (half)((((qs5x16.s2 & 0xF000) >> 12) | (((qh5x16.s1 >> 3) & 0x01) << 4)) - 16) * scale; \ + a_f16.sc = (half)((( qs5x16.s3 & 0x000F) | (((qh5x16.s1 >> 4) & 0x01) << 4)) - 16) * scale; \ + a_f16.sd = (half)((((qs5x16.s3 & 0x00F0) >> 4 ) | (((qh5x16.s1 >> 5) & 0x01) << 4)) - 16) * scale; \ + a_f16.se = (half)((((qs5x16.s3 & 0x0F00) >> 8 ) | (((qh5x16.s1 >> 6) & 0x01) << 4)) - 16) * scale; \ + a_f16.sf = (half)((((qs5x16.s3 & 0xF000) >> 12) | (((qh5x16.s1 >> 7) & 0x01) << 4)) - 16) * scale; \ + + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + + +__attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair +kernel void kernel_gemm_moe_q5_0_f32_ns( + __read_only image1d_buffer_t src0_qs, + __global uint * src0_qh, + __global half * src0_d, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global ushort * src2_emap, + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + uint is_ragged, + uint skip_gran +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + // Boundary check + if (block_id_n >= total_tiles[0]) { + return; + } + + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + // Loop along K axis, 32 elements (one block) for each iteration, divided into 2 sub-blocks + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + // First sub-block + uint q_sub_offset = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + uint s_sub_offset = row + ((ne01 * step) >> 5) + ((expert_id * ne00 * ne01) >> 5); + uint b_sub_offset = col * ne00 + step; + + // Load scale for current Q5_0 block + uint blk_offset = s_sub_offset + get_global_id(0); + half s = src0_d[blk_offset]; + + // Load 32 qh (5-th bit of each Q5) for the entire block + uchar4 qhx32 = as_uchar4(src0_qh[blk_offset]); + + // Load 16 qs (half block) in transposed layout + uint2 qsx16; + qsx16.x = read_imageui(src0_qs, q_sub_offset + sub_block_id_m).x; + qsx16.y = read_imageui(src0_qs, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q5_0(as_ushort4(qsx16), qhx32.lo, reg_a, s); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 8 elements reduction for better precision + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + + // Repeat for second sub-block + uint half_step = step + TILESIZE_K; + q_sub_offset = row + ((ne01 * half_step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + b_sub_offset = col * ne00 + half_step; + + // Load next 16 qs in transposed layout + qsx16.x = read_imageui(src0_qs, q_sub_offset + sub_block_id_m).x; + qsx16.y = read_imageui(src0_qs, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q5_0(as_ushort4(qsx16), qhx32.hi, reg_a, s); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 3-levels reduction for better precision + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + // Load poster router and share in LM + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + // Scatter results back to original position in output grid + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + // Store zero padding parts to the index of first output in tile, override correct result in the end + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl new file mode 100644 index 0000000..ceb9770 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl @@ -0,0 +1,330 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 + + +#define dequantize_q5_1(qs5x16, qh5x16, a_f16, scale, m) \ + a_f16.s0 = (half)((( qs5x16.s0 & 0x000F) | (( qh5x16.s0 & 0x01) << 4)) * scale + m); \ + a_f16.s1 = (half)((((qs5x16.s0 & 0x00F0) >> 4 ) | (((qh5x16.s0 >> 1) & 0x01) << 4)) * scale + m); \ + a_f16.s2 = (half)((((qs5x16.s0 & 0x0F00) >> 8 ) | (((qh5x16.s0 >> 2) & 0x01) << 4)) * scale + m); \ + a_f16.s3 = (half)((((qs5x16.s0 & 0xF000) >> 12) | (((qh5x16.s0 >> 3) & 0x01) << 4)) * scale + m); \ + a_f16.s4 = (half)((( qs5x16.s1 & 0x000F) | (((qh5x16.s0 >> 4) & 0x01) << 4)) * scale + m); \ + a_f16.s5 = (half)((((qs5x16.s1 & 0x00F0) >> 4 ) | (((qh5x16.s0 >> 5) & 0x01) << 4)) * scale + m); \ + a_f16.s6 = (half)((((qs5x16.s1 & 0x0F00) >> 8 ) | (((qh5x16.s0 >> 6) & 0x01) << 4)) * scale + m); \ + a_f16.s7 = (half)((((qs5x16.s1 & 0xF000) >> 12) | (((qh5x16.s0 >> 7) & 0x01) << 4)) * scale + m); \ + a_f16.s8 = (half)((( qs5x16.s2 & 0x000F) | (( qh5x16.s1 & 0x01) << 4)) * scale + m); \ + a_f16.s9 = (half)((((qs5x16.s2 & 0x00F0) >> 4 ) | (((qh5x16.s1 >> 1) & 0x01) << 4)) * scale + m); \ + a_f16.sa = (half)((((qs5x16.s2 & 0x0F00) >> 8 ) | (((qh5x16.s1 >> 2) & 0x01) << 4)) * scale + m); \ + a_f16.sb = (half)((((qs5x16.s2 & 0xF000) >> 12) | (((qh5x16.s1 >> 3) & 0x01) << 4)) * scale + m); \ + a_f16.sc = (half)((( qs5x16.s3 & 0x000F) | (((qh5x16.s1 >> 4) & 0x01) << 4)) * scale + m); \ + a_f16.sd = (half)((((qs5x16.s3 & 0x00F0) >> 4 ) | (((qh5x16.s1 >> 5) & 0x01) << 4)) * scale + m); \ + a_f16.se = (half)((((qs5x16.s3 & 0x0F00) >> 8 ) | (((qh5x16.s1 >> 6) & 0x01) << 4)) * scale + m); \ + a_f16.sf = (half)((((qs5x16.s3 & 0xF000) >> 12) | (((qh5x16.s1 >> 7) & 0x01) << 4)) * scale + m); \ + + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + + +__attribute__((qcom_wave_pair_mode(1))) // 1=force single 2=force pair +kernel void kernel_gemm_moe_q5_1_f32_ns( + __read_only image1d_buffer_t src0_qs, + __global uint * src0_qh, + __global half * src0_d, + __global half * src0_m, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global ushort * src2_emap, + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + uint is_ragged, + uint skip_gran +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + // Boundary check + if (block_id_n >= total_tiles[0]) { + return; + } + + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + // Loop along K axis, 32 elements (one block) for each iteration, divided into 2 sub-blocks + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + // First sub-block + uint q_sub_offset = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + uint s_sub_offset = row + ((ne01 * step) >> 5) + ((expert_id * ne00 * ne01) >> 5); + uint b_sub_offset = col * ne00 + step; + + // Load scale and m for current Q5_1 block + uint blk_offset = s_sub_offset + get_global_id(0); + half s = src0_d[blk_offset]; + half m = src0_m[blk_offset]; + + // Load 32 qh (5-th bit of each Q5) for the entire block + uchar4 qhx32 = as_uchar4(src0_qh[blk_offset]); + + // Load 16 qs (half block) in transposed layout + uint2 qsx16; + qsx16.x = read_imageui(src0_qs, q_sub_offset + sub_block_id_m).x; + qsx16.y = read_imageui(src0_qs, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q5_1(as_ushort4(qsx16), qhx32.lo, reg_a, s, m); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 8 elements reduction for better precision + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + + // Repeat for second sub-block + uint half_step = step + TILESIZE_K; + q_sub_offset = row + ((ne01 * half_step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + b_sub_offset = col * ne00 + half_step; + + // Load next 16 qs in transposed layout + qsx16.x = read_imageui(src0_qs, q_sub_offset + sub_block_id_m).x; + qsx16.y = read_imageui(src0_qs, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B, each fiber out of 64 in a sub-group loads 8 elements + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + // Convert to half and store to LM to share within the subgroup + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q5_1(as_ushort4(qsx16), qhx32.hi, reg_a, s, m); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + // 32 16x16 fp16 dot product with 3-levels reduction for better precision + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + // Load poster router and share in LM + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + // Scatter results back to original position in output grid + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + // Store zero padding parts to the index of first output in tile, override correct result in the end + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl new file mode 100644 index 0000000..0cbea13 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl @@ -0,0 +1,356 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 +#define QK_K 256 +#define K_SCALE_SIZE 12 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j+4] & 63; + } else { + *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2); + *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2); + } +} + +#define dequantize_q5_k(qs5x16, qh5x16, a_f16, scale, m) \ + a_f16.s0 = (half)((float)(( qs5x16.s0 & 0x000F) | (( qh5x16.s0 & 0x01) << 4)) * scale + m); \ + a_f16.s1 = (half)((float)((((qs5x16.s0 & 0x00F0) >> 4 ) | (((qh5x16.s0 >> 1) & 0x01) << 4)) * scale + m)); \ + a_f16.s2 = (half)((float)((((qs5x16.s0 & 0x0F00) >> 8 ) | (((qh5x16.s0 >> 2) & 0x01) << 4)) * scale + m)); \ + a_f16.s3 = (half)((float)((((qs5x16.s0 & 0xF000) >> 12) | (((qh5x16.s0 >> 3) & 0x01) << 4)) * scale + m)); \ + a_f16.s4 = (half)((float)((( qs5x16.s1 & 0x000F) | (((qh5x16.s0 >> 4) & 0x01) << 4)) * scale + m)); \ + a_f16.s5 = (half)((float)((((qs5x16.s1 & 0x00F0) >> 4 ) | (((qh5x16.s0 >> 5) & 0x01) << 4)) * scale + m)); \ + a_f16.s6 = (half)((float)(((qs5x16.s1 & 0x0F00) >> 8 ) | (((qh5x16.s0 >> 6) & 0x01) << 4)) * scale + m); \ + a_f16.s7 = (half)((float)((((qs5x16.s1 & 0xF000) >> 12) | (((qh5x16.s0 >> 7) & 0x01) << 4)) * scale + m)); \ + a_f16.s8 = (half)((float)((( qs5x16.s2 & 0x000F) | (( qh5x16.s1 & 0x01) << 4)) * scale + m)); \ + a_f16.s9 = (half)((float)((((qs5x16.s2 & 0x00F0) >> 4 ) | (((qh5x16.s1 >> 1) & 0x01) << 4)) * scale + m)); \ + a_f16.sa = (half)((float)((((qs5x16.s2 & 0x0F00) >> 8 ) | (((qh5x16.s1 >> 2) & 0x01) << 4)) * scale + m)); \ + a_f16.sb = (half)((float)((((qs5x16.s2 & 0xF000) >> 12) | (((qh5x16.s1 >> 3) & 0x01) << 4)) * scale + m)); \ + a_f16.sc = (half)((float)((( qs5x16.s3 & 0x000F) | (((qh5x16.s1 >> 4) & 0x01) << 4)) * scale + m)); \ + a_f16.sd = (half)((float)((((qs5x16.s3 & 0x00F0) >> 4 ) | (((qh5x16.s1 >> 5) & 0x01) << 4)) * scale + m)); \ + a_f16.se = (half)((float)((((qs5x16.s3 & 0x0F00) >> 8 ) | (((qh5x16.s1 >> 6) & 0x01) << 4)) * scale + m)); \ + a_f16.sf = (half)((float)((((qs5x16.s3 & 0xF000) >> 12) | (((qh5x16.s1 >> 7) & 0x01) << 4)) * scale + m)); \ + + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q5_k_f32_ns( + __read_only image1d_buffer_t src0_q, + __global uint * src0_qh, + __global uchar * src0_s, + __global half * src0_d, + __global half * src0_dm, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global ushort * src2_emap, + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + uint is_ragged, + uint skip_gran +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + // Boundary check + if (block_id_n >= total_tiles[0]) { + return; + } + + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + uint num_superblocks = ne00 / QK_K; + uint scales_per_row = num_superblocks * K_SCALE_SIZE; + uint row_idx = row + get_global_id(0); + + // Loop along K axis, 32 elements per iteration (one sub-block), divided into 2 halves of 16 + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + uint sub = step / 32; + uint sb = sub / 8; + uint j = sub % 8; + + // Load d and dm for super-block + uint d_offset = row + sb * ne01 + expert_id * num_superblocks * ne01 + get_global_id(0); + half d_val = src0_d[d_offset]; + half dm_val = src0_dm[d_offset]; + + // Load sub-block scale and min + global const uchar * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(j, sc, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = -(float)dm_val * (float)mn; + + // qh is stored at sub-block granularity + uint qh_offset = row + sub * ne01 + expert_id * num_superblocks * 8 * ne01 + get_global_id(0); + uchar4 qhx32 = as_uchar4(src0_qh[qh_offset]); + + // First sub-block (16 elements) + uint q_sub_offset = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + uint b_sub_offset = col * ne00 + step; + + // Load 16 q (64-bits) in transposed layout + uint2 q4x16; + q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantization + dequantize_q5_k(as_ushort4(q4x16), qhx32.lo, reg_a, scale, minv); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + + // Second half + uint half_step = step + TILESIZE_K; + q_sub_offset = row + ((ne01 * half_step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + b_sub_offset = col * ne00 + half_step; + + q4x16.x = read_imageui(src0_q, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_q, q_sub_offset + sub_block_id_m + ne01).x; + + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + dequantize_q5_k(as_ushort4(q4x16), qhx32.hi, reg_a, scale, minv); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + // Load post router and share in LM + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + // Scatter results back to original position in output grid + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + // Store zero padding parts to the index of first output in tile + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl new file mode 100644 index 0000000..f32114f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl @@ -0,0 +1,335 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 +#define QK_K 256 + +#define dequantize_q6_k(qs16, qh16, a_f16, scale) \ + a_f16.s0 = (half)(((float)(( qs16.s0 & 0x000F) | ((uint)(( qh16 ) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.s1 = (half)(((float)((( qs16.s0 >> 4) & 0x000F) | ((uint)(( qh16 >> 2) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.s2 = (half)(((float)((( qs16.s0 >> 8) & 0x000F) | ((uint)(( qh16 >> 4) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.s3 = (half)(((float)((( qs16.s0 >>12) & 0x000F) | ((uint)(( qh16 >> 6) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.s4 = (half)(((float)(( qs16.s1 & 0x000F) | ((uint)(( qh16 >> 8) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.s5 = (half)(((float)((( qs16.s1 >> 4) & 0x000F) | ((uint)(( qh16 >> 10) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.s6 = (half)(((float)((( qs16.s1 >> 8) & 0x000F) | ((uint)(( qh16 >> 12) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.s7 = (half)(((float)((( qs16.s1 >>12) & 0x000F) | ((uint)(( qh16 >> 14) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.s8 = (half)(((float)(( qs16.s2 & 0x000F) | ((uint)(( qh16 >> 16) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.s9 = (half)(((float)((( qs16.s2 >> 4) & 0x000F) | ((uint)(( qh16 >> 18) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.sa = (half)(((float)((( qs16.s2 >> 8) & 0x000F) | ((uint)(( qh16 >> 20) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.sb = (half)(((float)((( qs16.s2 >>12) & 0x000F) | ((uint)(( qh16 >> 22) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.sc = (half)(((float)(( qs16.s3 & 0x000F) | ((uint)(( qh16 >> 24) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.sd = (half)(((float)((( qs16.s3 >> 4) & 0x000F) | ((uint)(( qh16 >> 26) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.se = (half)(((float)((( qs16.s3 >> 8) & 0x000F) | ((uint)(( qh16 >> 28) & 0x3) << 4)) - 32.f) * scale); \ + a_f16.sf = (half)(((float)((( qs16.s3 >>12) & 0x000F) | ((uint)(( qh16 >> 30) & 0x3) << 4)) - 32.f) * scale); \ + + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + +// Quarter-tile variant: computes 8 output columns (one skip-group) into a float8 +// accumulator. Same reduction order / flush cadence as dotx16_reduce8, so the +// non-skipped path is byte-identical; it just lets the caller skip empty +// 8-column groups at finer granularity. Uses a private half8 `acc8`. +#define dotx8_reduce4(a_reg, b_lm, c_reg, lm_offset) \ + acc8.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc8.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc8.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc8.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc8.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc8.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc8.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc8.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc8.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc8.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc8.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc8.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc8.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc8.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc8.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc8.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + c_reg += convert_float8(acc8); \ + acc8.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc8.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc8.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc8.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc8.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc8.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc8.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc8.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc8.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc8.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc8.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc8.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc8.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc8.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc8.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc8.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + c_reg += convert_float8(acc8); \ + + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q6_k_f32_ns( + __read_only image1d_buffer_t src0_ql, + __global uint * src0_qh, + __global char * src0_s, + __global half * src0_d, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global ushort * src2_emap, + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + uint is_ragged, + uint skip_gran +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + // Boundary check + if (block_id_n >= total_tiles[0]) { + return; + } + + // Ragged tile-skip: when is_ragged and the upper 16 token-slots of this tile are all + // padding (router 0xFFFFFFFF), skip the second (reg_c.hi) dotx16_reduce8 half -> ~half + // the GEMM dot for sparse tiles. Numerically identical (the skipped lanes are padding). + // Ragged tile-skip: tokens are packed contiguously per expert (moe_scatter fills + // lanes 0..V-1, moe_fill pre-pads the rest), so router padding (0xFFFFFFFF) is always + // trailing. Find the valid-token count V and round it UP to the skip granularity + // skip_gran (columns per skip-group: 8 = quarter, 16 = half/legacy, 32 = disabled). + // A 8-column group g is all-padding iff its first column (8*g) >= n_active, so its + // dotx8_reduce4 is skipped. Numerically identical (skipped lanes are padding). + uint n_active = TILESIZE_N; + if (is_ragged && skip_gran < TILESIZE_N) { + uint n_valid = TILESIZE_N; + for (uint _t = 0; _t < TILESIZE_N; ++_t) { + if (src2[block_id_n * TILESIZE_N + _t] == 0xFFFFFFFFu) { n_valid = _t; break; } + } + n_active = min((uint)TILESIZE_N, ((n_valid + skip_gran - 1) / skip_gran) * skip_gran); + } + // Group 0 (cols 0-7) always runs; groups 1-3 skip when fully padding. + bool skip_g1 = (8u >= n_active); + bool skip_g2 = (16u >= n_active); + bool skip_g3 = (24u >= n_active); + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + uint num_superblocks = ne00 / QK_K; + uint scales_per_row = num_superblocks * 16; + uint row_idx = row + get_global_id(0); + + // Loop along K axis, 32 elements per iteration (one sub-block), divided into 2 halves of 16 + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + uint sub = step / 32; // 32-element group index + uint sb = sub / 8; // super-block index + uint j = sub % 8; // group within super-block + + // Load d for super-block + uint d_offset = row + sb * ne01 + expert_id * num_superblocks * ne01 + get_global_id(0); + half d_val = src0_d[d_offset]; + + // Load sub-block scales + global const char * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * 16; + float scale0 = (float)d_val * (float)sc[j * 2]; + float scale1 = (float)d_val * (float)sc[j * 2 + 1]; + + uint qh_base = row + (sub * 2) * ne01 + expert_id * (num_superblocks * 16) * ne01 + get_global_id(0); + uint qh_first16 = src0_qh[qh_base]; + uint qh_second16 = src0_qh[qh_base + ne01]; + + // First half (16 elements) + uint q_sub_offset = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + uint b_sub_offset = col * ne00 + step; + + // Load 16 ql nibbles (2 uints) from image + uint2 q4x16; + q4x16.x = read_imageui(src0_ql, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_ql, q_sub_offset + sub_block_id_m + ne01).x; + + // Load 16x32 floats from matrix B + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + // Dequantize first 16 elements (scale0) + dequantize_q6_k(as_ushort4(q4x16), qh_first16, reg_a, scale0); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + half8 acc8; + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + + // Second half + uint half_step = step + TILESIZE_K; + q_sub_offset = row + ((ne01 * half_step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + b_sub_offset = col * ne00 + half_step; + + q4x16.x = read_imageui(src0_ql, q_sub_offset + sub_block_id_m).x; + q4x16.y = read_imageui(src0_ql, q_sub_offset + sub_block_id_m + ne01).x; + + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + dequantize_q6_k(as_ushort4(q4x16), qh_second16, reg_a, scale1); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + dotx8_reduce4(reg_a, shared_b, reg_c.lo.lo, 0); + if (!skip_g1) { dotx8_reduce4(reg_a, shared_b, reg_c.lo.hi, 8); } + if (!skip_g2) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.lo, 16); } + if (!skip_g3) { dotx8_reduce4(reg_a, shared_b, reg_c.hi.hi, 24); } + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + // Load post router and share in LM + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + // Scatter results back to original position in output grid + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + // Store zero padding parts to the index of first output in tile + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl new file mode 100644 index 0000000..4ffe9f8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl @@ -0,0 +1,200 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_N 32 +#define QK_K 256 + +// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, in bits 0-3). +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// 4 2-bit highs in byte `b` (8 bits) -> 4 bytes, value 0..3 in bits 4-5 +// (pre-multiplied by 16 so it ORs with the EXP4 nibble to form q6 in 0..63). +#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \ + (((uint)((b) & 0x0Cu)) << 10) | \ + (((uint)((b) & 0x30u)) << 16) | \ + (((uint)((b) & 0xC0u)) << 22) ) + +// q6 (0..63, bits 0-5 of each byte) -> (q6-32) as a signed int8 per byte. +// Flipping bit5 subtracts 32 in 6-bit two's complement; then replicate bit5 +// into bits 6-7 to sign-extend to int8. Per-byte, no inter-byte carry. +inline uint SIGN6(uint q6p) { + uint x = q6p ^ 0x20202020u; + uint s = x & 0x20202020u; + return x | (s << 1) | (s << 2); +} + +inline int dp4a_q6(uint qw0, uint qw1, uint qw2, uint qw3, + uint a0, uint a1, uint a2, uint a3) { + int raw = 0; + raw = dot_acc_sat_4x8packed_ss_int(qw0, a0, raw); + raw = dot_acc_sat_4x8packed_ss_int(qw1, a1, raw); + raw = dot_acc_sat_4x8packed_ss_int(qw2, a2, raw); + raw = dot_acc_sat_4x8packed_ss_int(qw3, a3, raw); + return raw; +} + +// One token's q6_K dp4a dot (two halves, per-16 scales) + epilogue into acc[t]. +#define MOE_Q6K_DP4A_T(t) do { \ + uint4 a0 = vload4(0, &sh_qa[t][0]); \ + uint4 a1 = vload4(0, &sh_qa[t][4]); \ + const int raw1 = dp4a_q6(qw[0], qw[1], qw[2], qw[3], a0.s0, a0.s1, a0.s2, a0.s3); \ + const int raw2 = dp4a_q6(qw[4], qw[5], qw[6], qw[7], a1.s0, a1.s1, a1.s2, a1.s3); \ + const float a_d = (float)sh_d[t]; \ + acc[t] += scale0 * a_d * (float)raw1 + scale1 * a_d * (float)raw2; \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q6_k_q8_1_dp4a( + __read_only image1d_buffer_t src0_ql, // q6_K low nibbles (image, q4_K-style layout) + __global uint * src0_qh, // q6_K high 2-bit (16 elems/uint) + __global char * src0_s, // int8 scales (one per 16 elems) + __global half * src0_d, // per-superblock scale + __global uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap, // tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged // 1: compute only real tokens per tile +) { + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + if (block_id_n >= total_tiles[0]) { + return; + } + + const uint lid = get_local_id(0); // 0..63 -> row within M-tile + + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * 64; + const uint col = block_id_n * TILESIZE_N; + + const uint num_superblocks = ne00 / QK_K; + const uint scales_per_row = num_superblocks * 16; + const uint row_idx = row + lid; + + const uint ne00_u = ne00 >> 2; + const uint ne00_b = ne00 >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + + // Real token count for this tile + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) { + sh_src2[lid] = src2[col + lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { + nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { + if (sh_src2[t] != 0xFFFFFFFFu) ++nr; + } + } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; + const uint sb = sub >> 3; + const uint j = sub & 7; + + const float d_val = (float)src0_d[row + sb * ne01 + expert_id * num_superblocks * ne01 + lid]; + global const char * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * 16; + const float scale0 = d_val * (float)sc[j * 2]; + const float scale1 = d_val * (float)sc[j * 2 + 1]; + + // high bits: one uint covers 16 elems; first/second 16 of this 32-block + const uint qh_base = row + (sub * 2) * ne01 + expert_id * (num_superblocks * 16) * ne01 + lid; + const uint qh1 = src0_qh[qh_base]; + const uint qh2 = src0_qh[qh_base + ne01]; + + // low nibbles: same image layout as q4_K (8 ushorts over the 32 K) + const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint r0 = read_imageui(src0_ql, qoff0 + lid).x; + const uint r1 = read_imageui(src0_ql, qoff0 + lid + ne01).x; + const uint r2 = read_imageui(src0_ql, qoff1 + lid).x; + const uint r3 = read_imageui(src0_ql, qoff1 + lid + ne01).x; + + uint qw[8]; + qw[0] = SIGN6(EXP4(r0) | EXP2((qh1) & 0xFFu)); + qw[1] = SIGN6(EXP4(r0 >> 16) | EXP2((qh1 >> 8) & 0xFFu)); + qw[2] = SIGN6(EXP4(r1) | EXP2((qh1 >> 16) & 0xFFu)); + qw[3] = SIGN6(EXP4(r1 >> 16) | EXP2((qh1 >> 24) & 0xFFu)); + qw[4] = SIGN6(EXP4(r2) | EXP2((qh2) & 0xFFu)); + qw[5] = SIGN6(EXP4(r2 >> 16) | EXP2((qh2 >> 8) & 0xFFu)); + qw[6] = SIGN6(EXP4(r3) | EXP2((qh2 >> 16) & 0xFFu)); + qw[7] = SIGN6(EXP4(r3 >> 16) | EXP2((qh2 >> 24) & 0xFFu)); + + // Stage each token's 8 activation uints as two 128-bit uint4 loads/stores. + const uint vlim = (uint)n_real * 2; + for (uint idx = lid; idx < vlim; idx += 64) { + const uint t = idx >> 1; + const uint h = (idx & 1) << 2; // 0 or 4 + uint4 v = vload4(0, &src1_qa[(col + t) * ne00_u + (step >> 2) + h]); + vstore4(v, 0, &sh_qa[t][h]); + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * ne00_b + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + // Full tiles keep the fully-unrolled 32-wide loop; partial tiles run n_real. + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q6K_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_Q6K_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) { + return; + } + + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) { + idx = sh_src2[0]; + } + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl new file mode 100644 index 0000000..dc0f0ed --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl @@ -0,0 +1,221 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 + +// q8_0: 16 signed int8 weights (one uint4 = 16 chars) -> half16, scaled. +#define dequantize_q8_0(q4, a_f16, scale) \ + a_f16 = convert_half16(as_char16(q4)) * scale; + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q8_0_f32_ns( + __global char * src0_q, // flat q8_0 quants [n_expert*ne01*ne00] + __global half * src0_d, // flat q8_0 scales [n_expert*ne01*nb] + __read_only image1d_buffer_t src1, // reordered activations (f32) + __global uint * src2, // post-router out indices + __global ushort * src2_emap,// expert per tile + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01 +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + if (block_id_n >= total_tiles[0]) { + return; + } + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + const uint nb = ne00 >> 5; // blocks per row (ne00/32) + const uint w_row = expert_id * ne01 + row + get_local_id(0); // this lane's output row + __global char * w_q = src0_q + (ulong)w_row * ne00; // char base for the row + __global half * w_d = src0_d + (ulong)w_row * nb; // scale base for the row + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + // Loop along K axis, 32 elements per iteration, split into 2 sub-blocks. + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + half s = w_d[step >> 5]; // one q8_0 scale per 32-element block + + // First sub-block: 16 weights (16 chars = one uint4) at K=step + uint4 q8x16 = *((__global uint4 *)(w_q + step)); + + uint b_sub_offset = col * ne00 + step; + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + dequantize_q8_0(q8x16, reg_a, s); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + half16 acc; + dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); + dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + + // Second sub-block: next 16 weights at K=step+16 + uint half_step = step + TILESIZE_K; + q8x16 = *((__global uint4 *)(w_q + half_step)); + b_sub_offset = col * ne00 + half_step; + + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + dequantize_q8_0(q8x16, reg_a, s); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); + dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl new file mode 100644 index 0000000..0fdd8d2 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl @@ -0,0 +1,233 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// Generic int8 dp4a MoE GEMM, specialized versions also exist +// MOE_QT: +// 4 (q4_K)/41(q4_1)/40(q4_0) NIBBLE image low nibbles -> EXP4 +// 5 (q5_K)/51(q5_1)/50(q5_0) NIBBLE+HI image nibbles + qh high-bit plane +// 6 (q6_K) Q6 image nibbles + qh 2-bit -> SIGN6((nibble|hi2)) +// 80(q8_0)/82(mxfp4) INT8 global int8 codes (mxfp4: convert applies kvalues LUT) + +#define TILESIZE_M 64 +#define TILESIZE_N 32 +#define QK_K 256 + +#ifndef MOE_QT +#define MOE_QT 4 +#endif + +// 4 nibbles in low 16 bits of u -> 4 bytes (value 0..15) +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) +// 4 2-bit highs in byte b -> 4 bytes, bits 4-5 (q6_K) +#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \ + (((uint)((b) & 0x0Cu)) << 10) | \ + (((uint)((b) & 0x30u)) << 16) | \ + (((uint)((b) & 0xC0u)) << 22) ) + +// q6 (0..63) -> (q6-32) signed int8/byte (no inter-byte carry) +inline uint SIGN6(uint q6p){ uint x=q6p^0x20202020u; uint s=x&0x20202020u; return x|(s<<1)|(s<<2); } + +// 4 high bits (one per element, in bits 0..3 of h) -> bit4 of each of 4 bytes (5-bit hi) +#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \ + (((uint)((h) & 0x2u)) << 11) | \ + (((uint)((h) & 0x4u)) << 18) | \ + (((uint)((h) & 0x8u)) << 25) ) + +// per-type weight params + per-32-step unpack into qw[8] (8 int8 uints) +#if MOE_QT == 4 || MOE_QT == 41 || MOE_QT == 40 + #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_q, + #define LOAD_QW(step, sub) \ + uint qw[8]; { \ + const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint r0=read_imageui(src0_q,qoff0+lid).x, r1=read_imageui(src0_q,qoff0+lid+ne01).x; \ + const uint r2=read_imageui(src0_q,qoff1+lid).x, r3=read_imageui(src0_q,qoff1+lid+ne01).x; \ + qw[0]=EXP4(r0); qw[1]=EXP4(r0>>16); qw[2]=EXP4(r1); qw[3]=EXP4(r1>>16); \ + qw[4]=EXP4(r2); qw[5]=EXP4(r2>>16); qw[6]=EXP4(r3); qw[7]=EXP4(r3>>16); } + +#elif MOE_QT == 5 || MOE_QT == 51 || MOE_QT == 50 + // low nibbles via image (q4_K layout) + high-bit plane src0_qh: 1 uint per 32-block + // (bit i = high bit of element i). qh laid out [expert][block][row] to match the + // existing q5_0 trans4 convert + #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_q, __global uint * src0_qh, + #define LOAD_QW(step, sub) \ + uint qw[8]; { \ + const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint r0=read_imageui(src0_q,qoff0+lid).x, r1=read_imageui(src0_q,qoff0+lid+ne01).x; \ + const uint r2=read_imageui(src0_q,qoff1+lid).x, r3=read_imageui(src0_q,qoff1+lid+ne01).x; \ + const uint h = src0_qh[row_idx + (sub)*ne01 + expert_id*(ne00>>5)*ne01]; \ + qw[0]=EXP4(r0)|EXP1(h); qw[1]=EXP4(r0>>16)|EXP1(h>>4); \ + qw[2]=EXP4(r1)|EXP1(h>>8); qw[3]=EXP4(r1>>16)|EXP1(h>>12); \ + qw[4]=EXP4(r2)|EXP1(h>>16); qw[5]=EXP4(r2>>16)|EXP1(h>>20); \ + qw[6]=EXP4(r3)|EXP1(h>>24); qw[7]=EXP4(r3>>16)|EXP1(h>>28); } + +#elif MOE_QT == 6 + #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_ql, __global uint * src0_qh, + #define LOAD_QW(step, sub) \ + uint qw[8]; { \ + const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint r0=read_imageui(src0_ql,qoff0+lid).x, r1=read_imageui(src0_ql,qoff0+lid+ne01).x; \ + const uint r2=read_imageui(src0_ql,qoff1+lid).x, r3=read_imageui(src0_ql,qoff1+lid+ne01).x; \ + const uint qhb = row + ((sub)*2)*ne01 + expert_id*((ne00>>5)*2)*ne01 + lid; \ + const uint qh1=src0_qh[qhb], qh2=src0_qh[qhb+ne01]; \ + qw[0]=SIGN6(EXP4(r0)|EXP2(qh1&0xFFu)); qw[1]=SIGN6(EXP4(r0>>16)|EXP2((qh1>>8)&0xFFu)); \ + qw[2]=SIGN6(EXP4(r1)|EXP2((qh1>>16)&0xFFu)); qw[3]=SIGN6(EXP4(r1>>16)|EXP2((qh1>>24)&0xFFu)); \ + qw[4]=SIGN6(EXP4(r2)|EXP2(qh2&0xFFu)); qw[5]=SIGN6(EXP4(r2>>16)|EXP2((qh2>>8)&0xFFu)); \ + qw[6]=SIGN6(EXP4(r3)|EXP2((qh2>>16)&0xFFu)); qw[7]=SIGN6(EXP4(r3>>16)|EXP2((qh2>>24)&0xFFu)); } + +#elif MOE_QT == 80 || MOE_QT == 82 + // 8-bit direct: int8 codes 8 uints / 32-block, [expert][row][8*sub]. mxfp4: the + // convert resolves kvalues_mxfp4[nibble] -> int8 and stores the e8m0_half scale. + #define WEIGHT_PARAMS __global uint * src0_q8, + #define LOAD_QW(step, sub) \ + uint qw[8]; { \ + const uint qb = (expert_id*ne01 + row_idx)*(ne00>>2) + (sub)*8; \ + qw[0]=src0_q8[qb+0]; qw[1]=src0_q8[qb+1]; qw[2]=src0_q8[qb+2]; qw[3]=src0_q8[qb+3]; \ + qw[4]=src0_q8[qb+4]; qw[5]=src0_q8[qb+5]; qw[6]=src0_q8[qb+6]; qw[7]=src0_q8[qb+7]; } +#else + #error "unknown MOE_QT" +#endif + +inline int dp4a4(uint w0,uint w1,uint w2,uint w3,uint a0,uint a1,uint a2,uint a3){ + int r=0; r=dot_acc_sat_4x8packed_ss_int(w0,a0,r); r=dot_acc_sat_4x8packed_ss_int(w1,a1,r); + r=dot_acc_sat_4x8packed_ss_int(w2,a2,r); r=dot_acc_sat_4x8packed_ss_int(w3,a3,r); return r; } + +// Q8_0 and MXFP4 have no min term. Do not multiply zero by an overflowing half activation sum. +#if MOE_QT == 80 || MOE_QT == 82 +#define MOE_MIN_TERM(t) 0.0f +#else +#define MOE_MIN_TERM(t) (mn*(float)sh_s[t]) +#endif + +// One token's two-half dp4a + uniform scale/min epilogue into acc[t]. +#define MOE_DP4A_T(t) do { \ + uint4 a0 = vload4(0, &sh_qa[t][0]); \ + uint4 a1 = vload4(0, &sh_qa[t][4]); \ + const int raw1 = dp4a4(qw[0],qw[1],qw[2],qw[3], a0.s0,a0.s1,a0.s2,a0.s3); \ + const int raw2 = dp4a4(qw[4],qw[5],qw[6],qw[7], a1.s0,a1.s1,a1.s2,a1.s3); \ + const float a_d = (float)sh_d[t]; \ + acc[t] += sc0*a_d*(float)raw1 + sc1*a_d*(float)raw2 - MOE_MIN_TERM(t); \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q8_1_dp4a( + WEIGHT_PARAMS // per-type native weight buffer(s) + __global half * src0_scale,// uniform f16 16/superblock (per-16), [expert,row] + __global half * src0_min, // uniform f16 8/superblock (per-32), [expert,row] + __global uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap, // tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged, + int has_min // 0 for symmetric types (q8_0/q6_K/q4_0/...): skip min read +) { + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + if (block_id_n >= total_tiles[0]) return; + + const uint lid = get_local_id(0); // 0..63 -> output row within M-tile + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + const uint row_idx = row + lid; + + // Scale/min are laid out FLAT per-32-block (2 per-16-segment scales + 1 min per + // 32-block), so K only needs to be a multiple of 32 — works for the 32-block + // types (q8_0/q5_0/q4_0/...) as well as the K-quants (K%256==0, same bytes). + const uint nblk32 = ne00 / 32; + const uint sc_per_row = nblk32 * 2; + const uint mn_per_row = nblk32; + const uint ne00_u = ne00 >> 2; + const uint ne00_b = ne00 >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) sh_src2[lid] = src2[col + lid]; + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) if (sh_src2[t] != 0xFFFFFFFFu) ++nr; } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; // 32-block index along K + + // uniform pre-decoded scale (2 per-16-seg) + min (1) for this row, this 32-block + __global half * scl = src0_scale + (expert_id*ne01 + row_idx)*sc_per_row + sub*2; + const float sc0 = (float)scl[0]; + const float sc1 = (float)scl[1]; + float mn = 0.0f; + if (has_min) mn = (float)src0_min[(expert_id*ne01 + row_idx)*mn_per_row + sub]; + + LOAD_QW(step, sub) + + // Stage each token's 8 activation uints as two 128-bit uint4 loads/stores. + const uint vlim = (uint)n_real * 2; + for (uint idx = lid; idx < vlim; idx += 64) { + const uint t = idx >> 1; + const uint h = (idx & 1) << 2; // 0 or 4 + uint4 v = vload4(0, &src1_qa[(col + t) * ne00_u + (step >> 2) + h]); + vstore4(v, 0, &sh_qa[t][h]); + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * ne00_b + sub]; + sh_s[lid] = src1_sa[(col + lid) * ne00_b + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) return; + + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) idx = sh_src2[0]; + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) write_imagef(dst, out_idx[t] + m_offset, acc[t]); + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl new file mode 100644 index 0000000..6869d82 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl @@ -0,0 +1,150 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#ifdef cl_qcom_reqd_sub_group_size +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +constant half kvalues_iq4nl[16] = { + (half)-127.f, (half)-104.f, (half)-83.f, (half)-65.f, + (half) -49.f, (half) -35.f, (half)-22.f, (half)-10.f, + (half) 1.f, (half) 13.f, (half) 25.f, (half) 38.f, + (half) 53.f, (half) 69.f, (half) 89.f, (half)113.f +}; + +// Packed LUT: 2 FP16 values per uint, 8 unique constant loads instead of 16 +constant uint iq4nl_packed[8] = { + 0xD680D7F0u, // idx 0,1: -127, -104 + 0xD410D530u, // idx 2,3: -83, -65 + 0xD060D220u, // idx 4,5: -49, -35 + 0xC900CD80u, // idx 6,7: -22, -10 + 0x4A803C00u, // idx 8,9: 1, 13 + 0x50C04E40u, // idx 10,11: 25, 38 + 0x545052A0u, // idx 12,13: 53, 69 + 0x57105590u // idx 14,15: 89, 113 +}; + +// Packed dequant: 1 uint constant load (8-way divergence) + shift + as_half +#define IQ4_NL_DEQUANT(nibble) as_half((ushort)(iq4nl_packed[(nibble) >> 1] >> (((nibble) & 1u) << 4))) + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif + +kernel void kernel_gemm_noshuffle_iq4_nl_f32( + global const ushort * src0_q, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + + int m_4 = m >> 2; + int n_4 = n >> 2; + + int gy = get_global_id(0); + int gx = get_global_id(1); + int gx_2 = gx << 2; + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + half8 B; + half4 dequantized_weights; + + global const ushort * weight_ptr = src0_q + gx_2; + global const half * scale_ptr = src0_d + gx_2; + + for (int i = 0; i < k; i += 4) { + B.s0123 = read_imageh(src1, gy*2 + (i)*(n_4)); + B.s4567 = read_imageh(src1, gy*2 + (i)*(n_4)+1); + + ushort4 bits4 = vload4(0, weight_ptr + (i/4)*(m)); + + half4 scale = vload4(0, scale_ptr + (i/32)*(m)); + + // j=0 + dequantized_weights.s0 = IQ4_NL_DEQUANT(bits4.s0 & 0x000Fu) * scale.s0; + dequantized_weights.s1 = IQ4_NL_DEQUANT(bits4.s1 & 0x000Fu) * scale.s1; + dequantized_weights.s2 = IQ4_NL_DEQUANT(bits4.s2 & 0x000Fu) * scale.s2; + dequantized_weights.s3 = IQ4_NL_DEQUANT(bits4.s3 & 0x000Fu) * scale.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + + // j=1 + B.s0123 = read_imageh(src1, gy*2 + (i+1)*(n_4)); + B.s4567 = read_imageh(src1, gy*2 + (i+1)*(n_4)+1); + dequantized_weights.s0 = IQ4_NL_DEQUANT((bits4.s0 >> 4) & 0x000Fu) * scale.s0; + dequantized_weights.s1 = IQ4_NL_DEQUANT((bits4.s1 >> 4) & 0x000Fu) * scale.s1; + dequantized_weights.s2 = IQ4_NL_DEQUANT((bits4.s2 >> 4) & 0x000Fu) * scale.s2; + dequantized_weights.s3 = IQ4_NL_DEQUANT((bits4.s3 >> 4) & 0x000Fu) * scale.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + + // j=2 + B.s0123 = read_imageh(src1, gy*2 + (i+2)*(n_4)); + B.s4567 = read_imageh(src1, gy*2 + (i+2)*(n_4)+1); + dequantized_weights.s0 = IQ4_NL_DEQUANT((bits4.s0 >> 8) & 0x000Fu) * scale.s0; + dequantized_weights.s1 = IQ4_NL_DEQUANT((bits4.s1 >> 8) & 0x000Fu) * scale.s1; + dequantized_weights.s2 = IQ4_NL_DEQUANT((bits4.s2 >> 8) & 0x000Fu) * scale.s2; + dequantized_weights.s3 = IQ4_NL_DEQUANT((bits4.s3 >> 8) & 0x000Fu) * scale.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + + // j=3 + B.s0123 = read_imageh(src1, gy*2 + (i+3)*(n_4)); + B.s4567 = read_imageh(src1, gy*2 + (i+3)*(n_4)+1); + dequantized_weights.s0 = IQ4_NL_DEQUANT((bits4.s0 >> 12) & 0x000Fu) * scale.s0; + dequantized_weights.s1 = IQ4_NL_DEQUANT((bits4.s1 >> 12) & 0x000Fu) * scale.s1; + dequantized_weights.s2 = IQ4_NL_DEQUANT((bits4.s2 >> 12) & 0x000Fu) * scale.s2; + dequantized_weights.s3 = IQ4_NL_DEQUANT((bits4.s3 >> 12) & 0x000Fu) * scale.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + } + + int idx = (gy<<3)*m + (gx<<2); + + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl new file mode 100644 index 0000000..2941289 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl @@ -0,0 +1,143 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// Weight layout, feature-major: +// src0_q[row + (k/4)*m] ushort = 4 nibbles (K = 4*grp .. +3) +// src0_d[row + (k/32)*m] half = per-32-block scale + +#define TILESIZE_N 32 + +// IQ4_NL non-linear codebook as signed int8, packed 4 codes per uint. +// divergent nibble lookups read a small __constant uint array + shift, +// never a byte array because byte-indexed __constant loads serialize on Adreno and tank perf +// idx 0-3: -127,-104,-83,-65 = 0x81,0x98,0xAD,0xBF +// idx 4-7: -49,-35,-22,-10 = 0xCF,0xDD,0xEA,0xF6 +// idx 8-11: 1, 13, 25, 38 = 0x01,0x0D,0x19,0x26 +// idx 12-15: 53, 69, 89,113 = 0x35,0x45,0x59,0x71 +__constant uint kvalues_iq4nl_i8x4[4] = { + 0xBFAD9881u, 0xF6EADDCFu, 0x26190D01u, 0x71594535u +}; + +// nibble (0..15) -> its codebook byte in the low 8 bits. +inline uint iq4nl_code(uint n) { + return (kvalues_iq4nl_i8x4[n >> 2] >> ((n & 3u) * 8u)) & 0xFFu; +} + +// 4 nibbles in low 16 bits of u -> 4 codebook int8, packed for dp4a. +inline uint iq4nl_pack(ushort u) { + return iq4nl_code((uint)( u & 0xF)) + | (iq4nl_code((uint)((u >> 4) & 0xF)) << 8) + | (iq4nl_code((uint)((u >> 8) & 0xF)) << 16) + | (iq4nl_code((uint)((u >> 12) & 0xF)) << 24); +} + +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a( + __global const ushort * src0_q, // IQ4_NL nibbles (4/ushort, feature-major) + __global const half * src0_d, // per-32-block scale, feature-major + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + // 8 weight uints (32 codebook int8) for this row, this 32-block. + const uint qsbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = iq4nl_pack(src0_q[qsbase + 0 * m]); + qw.s1 = iq4nl_pack(src0_q[qsbase + 1 * m]); + qw.s2 = iq4nl_pack(src0_q[qsbase + 2 * m]); + qw.s3 = iq4nl_pack(src0_q[qsbase + 3 * m]); + qw.s4 = iq4nl_pack(src0_q[qsbase + 4 * m]); + qw.s5 = iq4nl_pack(src0_q[qsbase + 5 * m]); + qw.s6 = iq4nl_pack(src0_q[qsbase + 6 * m]); + qw.s7 = iq4nl_pack(src0_q[qsbase + 7 * m]); + + // cooperatively stage the 32-token x 32-K int8 activations to lm + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl new file mode 100644 index 0000000..ba46cc7 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl @@ -0,0 +1,94 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +// each work-item computes a 4 (rows of A / m) x 8 (cols of B / n) output tile. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif +kernel void kernel_gemm_noshuffle_q1_0_f32( + global const uint * src0_q, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + int k, + int m, + int n, + int n_no_padding, + ulong offsetd +) { + int n_4 = n >> 2; + + int gy = get_global_id(0); + int gx = get_global_id(1); + int gx_2 = gx << 2; + dst = (global float *)((global char*)dst + offsetd); + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + half8 B; + + global const uint* wptr = src0_q + gx_2; + global const half* sptr = src0_d + gx_2; + + // 32 weights per uint32, 128 weights (one block / one scale) per 4 uint32. + for (int i = 0; i < k; i += 32) { + uint4 pack4 = vload4(0, wptr + (i / 32) * m); // 4 rows, 32 K-values each + half4 scale = vload4(0, sptr + (i / 128) * m); // 4 rows, one scale per 128 + + for (int j = 0; j < 32; ++j) { + B.s0123 = read_imageh(src1, gy * 2 + (i + j) * n_4); + B.s4567 = read_imageh(src1, gy * 2 + (i + j) * n_4 + 1); + + // sign bit -> +-1 (half arithmetic avoids unsigned underflow) + half4 wj = (half4)( + 2.0h * (half)((pack4.s0 >> j) & 1u) - 1.0h, + 2.0h * (half)((pack4.s1 >> j) & 1u) - 1.0h, + 2.0h * (half)((pack4.s2 >> j) & 1u) - 1.0h, + 2.0h * (half)((pack4.s3 >> j) & 1u) - 1.0h) * scale; + + c0 += B * wj.s0; + c1 += B * wj.s1; + c2 += B * wj.s2; + c3 += B * wj.s3; + } + } + + int idx = (gy << 3) * m + (gx << 2); + + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl new file mode 100644 index 0000000..2749e1c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl @@ -0,0 +1,83 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_q2_k_f32( + global const uchar * src0_qs, + global const uchar * src0_sc, + global const half * src0_dm, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const half8 dm = vload8(0, src0_dm + 2 * (row + (i / 256) * m)); + const uchar4 sc = vload4(0, src0_sc + row + (i / 16) * m); + const half4 scale = convert_half4(convert_float4(dm.s0246) * convert_float4(sc & (uchar4)(15))); + const half4 mval = convert_half4(convert_float4(dm.s1357) * convert_float4(sc >> (uchar4)(4))); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const half4 w = convert_half4((bits >> (uchar4)(2 * j)) & (uchar4)(3)) * scale - mval; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl new file mode 100644 index 0000000..7db7e6d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl @@ -0,0 +1,146 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint q2_k_pack(uint pk) { + return ( (pk ) & 3u) + | (((pk >> 2) & 3u) << 8) + | (((pk >> 4) & 3u) << 16) + | (((pk >> 6) & 3u) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +#define KQ_STAGE(p) vload4(0, (p)) + +inline int dot4_q8a_v(uint4 qw, uint4 a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q2_k_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uchar * src0_sc, + __global const half * src0_dm, // half pairs: d, dmin + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local float sh_s [TILESIZE_N][2]; // per-16 activation sums, in qa units + __local half sh_d [TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; + + const half2 dm = vload2(rrow + ib * (uint)m, src0_dm); + const float dv = (float)dm.s0; + const float dmv = (float)dm.s1; + + const uint scb = rrow + (sub << 1) * (uint)m; + const uint sc0 = (uint)src0_sc[scb + 0u * (uint)m]; + const uint sc1 = (uint)src0_sc[scb + 1u * (uint)m]; + const float dl0 = dv * (float)( sc0 & 0xFu); + const float ml0 = dmv * (float)( sc0 >> 4); + const float dl1 = dv * (float)( sc1 & 0xFu); + const float ml1 = dmv * (float)( sc1 >> 4); + + const uint qsb = rrow + (step >> 2) * (uint)m; + uint4 qlo, qhi; + qlo.s0 = q2_k_pack((uint)src0_qs[qsb + 0u * (uint)m]); + qlo.s1 = q2_k_pack((uint)src0_qs[qsb + 1u * (uint)m]); + qlo.s2 = q2_k_pack((uint)src0_qs[qsb + 2u * (uint)m]); + qlo.s3 = q2_k_pack((uint)src0_qs[qsb + 3u * (uint)m]); + qhi.s0 = q2_k_pack((uint)src0_qs[qsb + 4u * (uint)m]); + qhi.s1 = q2_k_pack((uint)src0_qs[qsb + 5u * (uint)m]); + qhi.s2 = q2_k_pack((uint)src0_qs[qsb + 6u * (uint)m]); + qhi.s3 = q2_k_pack((uint)src0_qs[qsb + 7u * (uint)m]); + + for (uint idx = lid; idx < TILESIZE_N * 2u; idx += 64u) { + const uint t = idx >> 1; + const uint h = idx & 1u; + const uint c = col_base + t; + const bool ok = c < (uint)n_no_padding; + const uint4 w = ok ? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (h << 2)) + : (uint4)(0u); + sh_qa4[t][h] = w; + int s = 0; + s = dot_acc_sat_4x8packed_ss_int(w.x, 0x01010101u, s); + s = dot_acc_sat_4x8packed_ss_int(w.y, 0x01010101u, s); + s = dot_acc_sat_4x8packed_ss_int(w.z, 0x01010101u, s); + s = dot_acc_sat_4x8packed_ss_int(w.w, 0x01010101u, s); + sh_s[t][h] = (float)s; + if (h == 0u) { + sh_d[t] = ok ? src1_da[c * k_b + sub] : (half)0; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) +#define Q2_K_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) - ml0 * sh_s[b][0] + \ + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])) - ml1 * sh_s[b][1]) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = Q2_K_COL(b+0); rf.s1 = Q2_K_COL(b+1); + rf.s2 = Q2_K_COL(b+2); rf.s3 = Q2_K_COL(b+3); + acc[g] += LD4(sh_d, b) * rf; + } +#undef Q2_K_COL +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl similarity index 99% rename from ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl rename to ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl index ecb577b..1593780 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl @@ -17,7 +17,7 @@ REQD_SUBGROUP_SIZE_128 #endif -kernel void kernel_mul_mat_Ab_Bi_8x4( +kernel void kernel_gemm_noshuffle_q4_0_f32( global const ushort * src0_q, // quantized A global const half * src0_d, // A scales __read_only image1d_buffer_t src1, // B (1d image) diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl new file mode 100644 index 0000000..6418d0f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl @@ -0,0 +1,253 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_N 32 + +// Expand the 4 nibbles in the low 16 bits of u into 4 bytes (value 0..15), +// packed for the int8 dp4a. The -8 zero-point is applied via the sum term. +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a( + __global const ushort * src0_q, // q4_0 nibbles (4/ushort, feature-major) + __global const half * src0_d, // per-32-block scale, feature-major + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + // 8 weight uints (32 nibbles) for this row, this 32-block. Feature-major: + // src0_q[row + (k/4 + u)*m], k/4 = step/4 (= step>>2). EXP4 -> dp4a int8. + const uint qsbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[qsbase + 0 * m]); + qw.s1 = EXP4(src0_q[qsbase + 1 * m]); + qw.s2 = EXP4(src0_q[qsbase + 2 * m]); + qw.s3 = EXP4(src0_q[qsbase + 3 * m]); + qw.s4 = EXP4(src0_q[qsbase + 4 * m]); + qw.s5 = EXP4(src0_q[qsbase + 5 * m]); + qw.s6 = EXP4(src0_q[qsbase + 6 * m]); + qw.s7 = EXP4(src0_q[qsbase + 7 * m]); + + // cooperatively stage the 32-token x 32-K int8 activations to LDS + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + // q4_0: w = d*(q-8) -> d_w * (a_d * dp4a(q,qa) - 8 * a_s) + acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b)); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4( + __global const ushort * src0_q, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + const uint qsbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[qsbase + 0 * m]); + qw.s1 = EXP4(src0_q[qsbase + 1 * m]); + qw.s2 = EXP4(src0_q[qsbase + 2 * m]); + qw.s3 = EXP4(src0_q[qsbase + 3 * m]); + qw.s4 = EXP4(src0_q[qsbase + 4 * m]); + qw.s5 = EXP4(src0_q[qsbase + 5 * m]); + qw.s6 = EXP4(src0_q[qsbase + 6 * m]); + qw.s7 = EXP4(src0_q[qsbase + 7 * m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + + acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b)); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl index 99fd1fd..c379a9a 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl @@ -4,13 +4,16 @@ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable #define ADRENO_GPU 1 #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) #endif #define QK_K 256 #define K_SCALE_SIZE 12 +// scales are transposed: consecutive codes of a row are `stride` apart inline void get_scale_min_k4( int j, global const uchar * q, + int stride, uchar * d, uchar * m, uchar mask_d6, @@ -18,11 +21,11 @@ inline void get_scale_min_k4( uchar mask_hi2 ) { if (j < 4) { - *d = q[j] & mask_d6; - *m = q[j+4] & mask_d6; + *d = q[j*stride] & mask_d6; + *m = q[(j+4)*stride] & mask_d6; } else { - *d = (q[j+4] & mask_d4) | ((q[j-4] & mask_hi2) >> 2); - *m = ((q[j+4] >> 4) & mask_d4) | ((q[j] & mask_hi2) >> 2); + *d = (q[(j+4)*stride] & mask_d4) | ((q[(j-4)*stride] & mask_hi2) >> 2); + *m = ((q[(j+4)*stride] >> 4) & mask_d4) | ((q[j*stride] & mask_hi2) >> 2); } } @@ -55,7 +58,6 @@ kernel void kernel_gemm_noshuffle_q4_k_f32( half8 B; half4 dequantized_weights; - int num_blocks_K = k / QK_K; global const ushort * weight_ptr = src0_q + gx_2; global const half * d_ptr = src0_d + gx_2; @@ -68,16 +70,16 @@ kernel void kernel_gemm_noshuffle_q4_k_f32( half4 d = vload4(0, d_ptr + sb_idx * m); half4 dm = vload4(0, dm_ptr + sb_idx * m); - global const uchar * sc0 = src0_s + (gx_2+0) * num_blocks_K * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; - global const uchar * sc1 = src0_s + (gx_2+1) * num_blocks_K * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; - global const uchar * sc2 = src0_s + (gx_2+2) * num_blocks_K * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; - global const uchar * sc3 = src0_s + (gx_2+3) * num_blocks_K * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; + global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + (gx_2+0); + global const uchar * sc1 = sc0 + 1; + global const uchar * sc2 = sc0 + 2; + global const uchar * sc3 = sc0 + 3; uchar sv0, mn0, sv1, mn1, sv2, mn2, sv3, mn3; - get_scale_min_k4(sub_idx, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); - get_scale_min_k4(sub_idx, sc1, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); - get_scale_min_k4(sub_idx, sc2, &sv2, &mn2, mask_d6, mask_d4, mask_hi2); - get_scale_min_k4(sub_idx, sc3, &sv3, &mn3, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(sub_idx, sc1, m, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(sub_idx, sc2, m, &sv2, &mn2, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(sub_idx, sc3, m, &sv3, &mn3, mask_d6, mask_d4, mask_hi2); half4 scale = convert_half4(convert_float4(d) * convert_float4((uchar4)(sv0, sv1, sv2, sv3))); half4 mval = convert_half4(convert_float4(dm) * convert_float4((uchar4)(mn0, mn1, mn2, mn3))); @@ -170,3 +172,319 @@ kernel void kernel_gemm_noshuffle_q4_k_f32( vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); } } + +// 1x8 per-WI tile (1 output row x 8 output cols). For the small-batch +// (medium n_q, e.g. MTP/spec verify) path where the 2x8 kernel is starved: +// at ne1<=8 the grid is (1, ceil(M/2)) -> only ~M/256 workgroups, leaving +// the SP under-occupied. 1 row per WI doubles the M-axis workgroup count +// (ceil(M/1)/128 vs ceil(M/2)/128) AND collapses the accumulators to a +// single half8 (16 regs, no spill), so more waves co-reside. Same weight +// traffic as 2x8 (rows never share weights); the win is pure occupancy. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif +kernel void kernel_gemm_noshuffle_q4_k_f32_r1( + global const ushort * src0_q, + global const uchar * src0_s, + global const half * src0_d, + global const half * src0_dm, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + int n_4 = n >> 2; + int gy = get_global_id(0); + int gx = get_global_id(1); // 1 row per WI + + half8 c0 = 0; + half8 B; + half dq; + + int num_blocks_K = k / QK_K; + + global const ushort * weight_ptr = src0_q + gx; + global const half * d_ptr = src0_d + gx; + global const half * dm_ptr = src0_dm + gx; + + for (int i = 0; i < k; i += 32) { + int sb_idx = i / QK_K; + int sub_idx = (i / 32) % 8; + + half dd = d_ptr [sb_idx * m]; + half dmm = dm_ptr[sb_idx * m]; + + global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + gx; + + uchar sv0, mn0; + get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + + half scale = convert_half(convert_float(dd) * (float)sv0); + half mval = convert_half(convert_float(dmm) * (float)mn0); + + for (int l = 0; l < 32; l += 4) { + int ki = i + l; + ushort bits = weight_ptr[(ki/4) * m]; + + B.s0123 = read_imageh(src1, gy*2 + (ki+0) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+0) * n_4); + dq = (bits & 0x000F) * scale - mval; + c0 += B * dq; + + B.s0123 = read_imageh(src1, gy*2 + (ki+1) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+1) * n_4); + dq = ((bits & 0x00F0) >> 4) * scale - mval; + c0 += B * dq; + + B.s0123 = read_imageh(src1, gy*2 + (ki+2) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+2) * n_4); + dq = ((bits & 0x0F00) >> 8) * scale - mval; + c0 += B * dq; + + B.s0123 = read_imageh(src1, gy*2 + (ki+3) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+3) * n_4); + dq = ((bits & 0xF000) >> 12) * scale - mval; + c0 += B * dq; + } + } + + // Output: 8 cols, 1 row per col-step. Scalar store, coalesced across + // neighbouring WIs (consecutive gx -> consecutive dst addresses). + int idx = (gy<<3)*m + gx; + if (idx < m*n_no_padding) { dst[idx] = c0.s0; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s1; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s2; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s3; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s4; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s5; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s6; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = c0.s7; } +} + +// 2x8 tile, but weights read through an image1d_buffer (CL_R/UINT32 over the +// same packed-q buffer) instead of a plain global buffer. The ne1==1 GEMV +// already does this and is much faster per weight byte than this GEMM at +// small n_q; the structural difference is the image path hits the dedicated +// TPL1 weight cache (L1) while the global path only reaches L2. At small n_q +// the forward is weight-read-bound, so L1-cached weights is the lever. +// The 2 adjacent rows the 2x8 tile reads as a ushort2 are exactly one uint32, +// so the vload2 becomes a single read_imageui at index gx + (ki/4)*(m/2). +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif +kernel void kernel_gemm_noshuffle_q4_k_f32_kimg( + read_only image1d_buffer_t src0_q_img, + global const uchar * src0_s, + global const half * src0_d, + global const half * src0_dm, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + int n_4 = n >> 2; + int m_2 = m >> 1; + int gy = get_global_id(0); + int gx = get_global_id(1); + int gx_2 = gx << 1; + + half8 c0 = 0, c1 = 0; + half8 B; + half2 dequantized_weights; + + int num_blocks_K = k / QK_K; + + global const half * d_ptr = src0_d + gx_2; + global const half * dm_ptr = src0_dm + gx_2; + + for (int i = 0; i < k; i += 32) { + int sb_idx = i / QK_K; + int sub_idx = (i / 32) % 8; + + half2 d = vload2(0, d_ptr + sb_idx * m); + half2 dm = vload2(0, dm_ptr + sb_idx * m); + + global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + (gx_2+0); + global const uchar * sc1 = sc0 + 1; + + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(sub_idx, sc1, m, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + + half2 scale = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); + half2 mval = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); + + for (int l = 0; l < 32; l += 4) { + int ki = i + l; + uint wpacked = read_imageui(src0_q_img, gx + (ki/4) * m_2).x; + ushort2 bits2 = (ushort2)((ushort)(wpacked & 0xFFFFu), (ushort)(wpacked >> 16)); + + // j=0 + B.s0123 = read_imageh(src1, gy*2 + (ki+0) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+0) * n_4); + dequantized_weights.s0 = (bits2.s0 & 0x000F) * scale.s0 - mval.s0; + dequantized_weights.s1 = (bits2.s1 & 0x000F) * scale.s1 - mval.s1; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + + // j=1 + B.s0123 = read_imageh(src1, gy*2 + (ki+1) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+1) * n_4); + dequantized_weights.s0 = ((bits2.s0 & 0x00F0) >> 4) * scale.s0 - mval.s0; + dequantized_weights.s1 = ((bits2.s1 & 0x00F0) >> 4) * scale.s1 - mval.s1; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + + // j=2 + B.s0123 = read_imageh(src1, gy*2 + (ki+2) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+2) * n_4); + dequantized_weights.s0 = ((bits2.s0 & 0x0F00) >> 8) * scale.s0 - mval.s0; + dequantized_weights.s1 = ((bits2.s1 & 0x0F00) >> 8) * scale.s1 - mval.s1; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + + // j=3 + B.s0123 = read_imageh(src1, gy*2 + (ki+3) * n_4); + B.s4567 = read_imageh(src1, gy*2+1 + (ki+3) * n_4); + dequantized_weights.s0 = ((bits2.s0 & 0xF000) >> 12) * scale.s0 - mval.s0; + dequantized_weights.s1 = ((bits2.s1 & 0xF000) >> 12) * scale.s1 - mval.s1; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + } + } + + int idx = (gy<<3)*m + (gx<<1); + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s0, c1.s0), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s1, c1.s1), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s2, c1.s2), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s3, c1.s3), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s4, c1.s4), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s5, c1.s5), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s6, c1.s6), 0, dst + idx); idx += m; } + if (idx+1 < m*n_no_padding) { vstore2((float2)(c0.s7, c1.s7), 0, dst + idx); } +} + +// Cooperative-K GEMM for the small-batch (n_q in [2..8]) path. Mirrors the +// ne1==1 GEMV's structure: a WG is (COK_SG lanes x COK_NSG subgroups); each +// lane owns ONE output row and computes its 8 (padded) columns, and the +// COK_NSG subgroups SPLIT the K reduction round-robin, combining via a +// __local reduction. This is the thing the per-WI GEMM lacked — at small n_q +// the old kernel had ~M/256 workgroups each walking all of K serially; this +// has M/64 workgroups AND COK_NSG-way K parallelism. Uses REQD_SUBGROUP_SIZE_64 +// + barrier (same safe reduction pattern as the GEMV; never sub_group_reduce +// at full width on X2 per the GDN miscompile note). +#define COK_NSG 8 +#define COK_SG 64 +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemm_noshuffle_q4_k_f32_cok( + global const ushort * src0_q, + global const uchar * src0_s, + global const half * src0_d, + global const half * src0_dm, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + int n_4 = n >> 2; + int gx = get_global_id(0); // output row + int sg = get_local_id(1); // subgroup index (K-split lane) + int lane = get_local_id(0); // lane within subgroup (0..COK_SG-1) + + int num_blocks_K = k / QK_K; + int num_32blk = k / 32; + + global const ushort * weight_ptr = src0_q + gx; + global const half * d_ptr = src0_d + gx; + global const half * dm_ptr = src0_dm + gx; + + half8 acc = 0; + half8 B; + half dq; + + for (int blk = sg; blk < num_32blk; blk += COK_NSG) { + int i = blk << 5; // blk * 32 + int sb_idx = blk >> 3; // (blk*32) / QK_K (QK_K = 256 = 32*8) + int sub_idx = blk & 7; // (i/32) % 8 + + half dd = d_ptr [sb_idx * m]; + half dmm = dm_ptr[sb_idx * m]; + + global const uchar * sc0 = src0_s + sb_idx * K_SCALE_SIZE * m + gx; + uchar sv0, mn0; + get_scale_min_k4(sub_idx, sc0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + half scale = convert_half(convert_float(dd) * (float)sv0); + half mval = convert_half(convert_float(dmm) * (float)mn0); + + for (int l = 0; l < 32; l += 4) { + int ki = i + l; + ushort bits = weight_ptr[(ki>>2) * m]; + + B.s0123 = read_imageh(src1, (ki+0) * n_4); + B.s4567 = read_imageh(src1, 1 + (ki+0) * n_4); + dq = (bits & 0x000F) * scale - mval; + acc += B * dq; + + B.s0123 = read_imageh(src1, (ki+1) * n_4); + B.s4567 = read_imageh(src1, 1 + (ki+1) * n_4); + dq = ((bits & 0x00F0) >> 4) * scale - mval; + acc += B * dq; + + B.s0123 = read_imageh(src1, (ki+2) * n_4); + B.s4567 = read_imageh(src1, 1 + (ki+2) * n_4); + dq = ((bits & 0x0F00) >> 8) * scale - mval; + acc += B * dq; + + B.s0123 = read_imageh(src1, (ki+3) * n_4); + B.s4567 = read_imageh(src1, 1 + (ki+3) * n_4); + dq = ((bits & 0xF000) >> 12) * scale - mval; + acc += B * dq; + } + } + + // cross-subgroup reduction over the K-split (float for accuracy) + local float8 reduceLM[COK_SG * (COK_NSG - 1)]; + if (sg > 0) { + reduceLM[(sg - 1) * COK_SG + lane] = convert_float8(acc); + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sg == 0) { + float8 sum = convert_float8(acc); + for (int s = 0; s < COK_NSG - 1; s++) { + sum += reduceLM[s * COK_SG + lane]; + } + int idx = gx; + if (idx < m*n_no_padding) { dst[idx] = sum.s0; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s1; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s2; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s3; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s4; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s5; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s6; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s7; } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl new file mode 100644 index 0000000..69cabb9 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl @@ -0,0 +1,419 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif +#define QK_K 256 +#define K_SCALE_SIZE 12 + +// scales are transposed: consecutive codes of a row are `stride` apart +inline void get_scale_min_k4( + int j, + global const uchar * q, + uint stride, + uchar * d, + uchar * m, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + if (j < 4) { + *d = q[j*stride] & mask_d6; + *m = q[(j+4)*stride] & mask_d6; + } else { + *d = (q[(j+4)*stride] & mask_d4) | ((q[(j-4)*stride] & mask_hi2) >> 2); + *m = ((q[(j+4)*stride] >> 4) & mask_d4) | ((q[j*stride] & mask_hi2) >> 2); + } +} + +// Expand the 4 nibbles in the low 16 bits of `u` into 4 bytes (one nibble per +// byte, value 0..15), packed for the int8 dp4a. +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// 32-K dp4a dot of one token's int8 activations (8 packed uints in lm) against the +// row's 8 packed weight uints. qw passed by value as a uint8 (register), not an array. +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a( + __global const ushort * src0_q, // q4_K weights (noshuffle, packed nibbles) + __global const uchar * src0_s, // 6-bit scale/min codes + __global const half * src0_d, // per-superblock scale + __global const half * src0_dm, // per-superblock min + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k, // K (== ne00) + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + + // One float4 vector-register accumulator per group of 4 tokens (NGROUPS = TILESIZE_N/4). +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { acc[g] = (float4)(0.0f); } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + // weight scale/min for this WI's row, this subblock + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * (uint)m + rrow; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, (uint)m, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + // repack this row's 32 weight nibbles into 8 dp4a uints. The packed q4_K + // layout stores one ushort = 4 consecutive-K nibbles for a row at + // src0_q[row + (K_group)*m], K_group = step/4 + u. + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[wbase + 0 * m]); + qw.s1 = EXP4(src0_q[wbase + 1 * m]); + qw.s2 = EXP4(src0_q[wbase + 2 * m]); + qw.s3 = EXP4(src0_q[wbase + 3 * m]); + qw.s4 = EXP4(src0_q[wbase + 4 * m]); + qw.s5 = EXP4(src0_q[wbase + 5 * m]); + qw.s6 = EXP4(src0_q[wbase + 6 * m]); + qw.s7 = EXP4(src0_q[wbase + 7 * m]); + + // cooperatively stage the 32-token x 32-K int8 activations to lm + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. Scatter each + // lane with a per-token padding guard (dst is non-contiguous in token). + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg( + __read_only image1d_buffer_t src0_q_img, // q4_K weights as uint32 texels (2 ushorts/texel) + __global const uchar * src0_s, // 6-bit scale/min codes + __global const half * src0_d, // per-superblock scale + __global const half * src0_dm, // per-superblock min + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k, // K (== ne00) + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + // Constant per WI: the ushort the row needs always sits in the same half of + // its uint32 texel (m even => index parity == rrow parity). Hoist the shift. + const uint sel = (rrow & 1u) * 16u; + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * (uint)m + rrow; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, (uint)m, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(read_imageui(src0_q_img, (int)((wbase + 0 * m) >> 1)).x >> sel); + qw.s1 = EXP4(read_imageui(src0_q_img, (int)((wbase + 1 * m) >> 1)).x >> sel); + qw.s2 = EXP4(read_imageui(src0_q_img, (int)((wbase + 2 * m) >> 1)).x >> sel); + qw.s3 = EXP4(read_imageui(src0_q_img, (int)((wbase + 3 * m) >> 1)).x >> sel); + qw.s4 = EXP4(read_imageui(src0_q_img, (int)((wbase + 4 * m) >> 1)).x >> sel); + qw.s5 = EXP4(read_imageui(src0_q_img, (int)((wbase + 5 * m) >> 1)).x >> sel); + qw.s6 = EXP4(read_imageui(src0_q_img, (int)((wbase + 6 * m) >> 1)).x >> sel); + qw.s7 = EXP4(read_imageui(src0_q_img, (int)((wbase + 7 * m) >> 1)).x >> sel); + + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4( + __global const ushort * src0_q, + __global const uchar * src0_s, + __global const half * src0_d, + __global const half * src0_dm, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * (uint)m + rrow; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, (uint)m, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[wbase + 0 * m]); + qw.s1 = EXP4(src0_q[wbase + 1 * m]); + qw.s2 = EXP4(src0_q[wbase + 2 * m]); + qw.s3 = EXP4(src0_q[wbase + 3 * m]); + qw.s4 = EXP4(src0_q[wbase + 4 * m]); + qw.s5 = EXP4(src0_q[wbase + 5 * m]); + qw.s6 = EXP4(src0_q[wbase + 6 * m]); + qw.s7 = EXP4(src0_q[wbase + 7 * m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl new file mode 100644 index 0000000..1d6bd48 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl @@ -0,0 +1,131 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif + +kernel void kernel_gemm_noshuffle_q5_0_f32( + global const ushort * src0_qs, // quantized A + global const uchar * src0_qh, // 5th bits + global const half * src0_d, // A scales + __read_only image1d_buffer_t src1, // B (1d image) + global float * dst, // C + int m, // M + int n, // N with padding + int k, // K + int n_no_padding // N without padding +) { + + int n_4 = n >> 2; + + int gy = get_global_id(0); + int gx = get_global_id(1); + int gx_2 = gx << 2; + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + half8 B; + half4 dequantized_weights; + + global const ushort * weight_ptr = src0_qs + gx_2; + global const uchar * qh_ptr = src0_qh + gx_2; + global const half * scale_ptr = src0_d + gx_2; + + for (int i = 0; i < k; i += 4) { + + B.s0123 = read_imageh(src1, gy*2 + i*n_4); + B.s4567 = read_imageh(src1, gy*2 + i*n_4 + 1); + + ushort4 bits4 = vload4(0, weight_ptr + (i >> 2)*m); + uchar4 bits1 = vload4(0, qh_ptr + (i >> 3)*m); + uchar4 qh = bits1 >> (uchar4)(i & 4); + + half4 scale = vload4(0, scale_ptr + (i >> 5)*m); + + // j=0 + dequantized_weights.s0 = (convert_half((bits4.s0 & 0x000F) | ((qh.s0 & 0x01) << 4)) - 16.0h) * scale.s0; + dequantized_weights.s1 = (convert_half((bits4.s1 & 0x000F) | ((qh.s1 & 0x01) << 4)) - 16.0h) * scale.s1; + dequantized_weights.s2 = (convert_half((bits4.s2 & 0x000F) | ((qh.s2 & 0x01) << 4)) - 16.0h) * scale.s2; + dequantized_weights.s3 = (convert_half((bits4.s3 & 0x000F) | ((qh.s3 & 0x01) << 4)) - 16.0h) * scale.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + + // j=1 + B.s0123 = read_imageh(src1, gy*2 + (i+1)*n_4); + B.s4567 = read_imageh(src1, gy*2 + (i+1)*n_4 + 1); + dequantized_weights.s0 = (convert_half(((bits4.s0 & 0x00F0) >> 4) | ((qh.s0 & 0x02) << 3)) - 16.0h) * scale.s0; + dequantized_weights.s1 = (convert_half(((bits4.s1 & 0x00F0) >> 4) | ((qh.s1 & 0x02) << 3)) - 16.0h) * scale.s1; + dequantized_weights.s2 = (convert_half(((bits4.s2 & 0x00F0) >> 4) | ((qh.s2 & 0x02) << 3)) - 16.0h) * scale.s2; + dequantized_weights.s3 = (convert_half(((bits4.s3 & 0x00F0) >> 4) | ((qh.s3 & 0x02) << 3)) - 16.0h) * scale.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + + // j=2 + B.s0123 = read_imageh(src1, gy*2 + (i+2)*n_4); + B.s4567 = read_imageh(src1, gy*2 + (i+2)*n_4 + 1); + dequantized_weights.s0 = (convert_half(((bits4.s0 & 0x0F00) >> 8) | ((qh.s0 & 0x04) << 2)) - 16.0h) * scale.s0; + dequantized_weights.s1 = (convert_half(((bits4.s1 & 0x0F00) >> 8) | ((qh.s1 & 0x04) << 2)) - 16.0h) * scale.s1; + dequantized_weights.s2 = (convert_half(((bits4.s2 & 0x0F00) >> 8) | ((qh.s2 & 0x04) << 2)) - 16.0h) * scale.s2; + dequantized_weights.s3 = (convert_half(((bits4.s3 & 0x0F00) >> 8) | ((qh.s3 & 0x04) << 2)) - 16.0h) * scale.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + + // j=3 + B.s0123 = read_imageh(src1, gy*2 + (i+3)*n_4); + B.s4567 = read_imageh(src1, gy*2 + (i+3)*n_4 + 1); + dequantized_weights.s0 = (convert_half(((bits4.s0 & 0xF000) >> 12) | ((qh.s0 & 0x08) << 1)) - 16.0h) * scale.s0; + dequantized_weights.s1 = (convert_half(((bits4.s1 & 0xF000) >> 12) | ((qh.s1 & 0x08) << 1)) - 16.0h) * scale.s1; + dequantized_weights.s2 = (convert_half(((bits4.s2 & 0xF000) >> 12) | ((qh.s2 & 0x08) << 1)) - 16.0h) * scale.s2; + dequantized_weights.s3 = (convert_half(((bits4.s3 & 0xF000) >> 12) | ((qh.s3 & 0x08) << 1)) - 16.0h) * scale.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + } + + int idx = (gy<<3)*m + (gx<<2); + + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl new file mode 100644 index 0000000..4d1c6bd --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl @@ -0,0 +1,235 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// Weight layout +// src0_qs[row + (k/4)*m] ushort = 4 low nibbles (K = 4*grp .. +3) +// src0_qh[row + (k/8)*m] uchar = 8 high bits (one per element) +// src0_d [row + (k/32)*m] half = per-32-block scale + +#define TILESIZE_N 32 + +// 4 nibbles in low 16 bits of u -> 4 bytes (value 0..15) +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) +// 4 high bits (one per element, in bits 0..3 of h) -> bit4 of each of 4 bytes +#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \ + (((uint)((h) & 0x2u)) << 11) | \ + (((uint)((h) & 0x4u)) << 18) | \ + (((uint)((h) & 0x8u)) << 25) ) + +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q5_0_q8_1_dp4a( + __global const ushort * src0_qs, // q5_0 low nibbles (4/ushort, feature-major) + __global const uchar * src0_qh, // q5_0 high-bit plane (8/uchar, feature-major) + __global const half * src0_d, // per-32-block scale, feature-major + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + const float minv = d_w * 16.0f; // -16 centering -> subtract via q8_1 sum + + // 8 weight uints (32 elements) for this row, this 32-block. + // nibbles: src0_qs[row + (step/4 + u)*m]; high bits: src0_qh[row + (step/8 + u/2)*m], + // 4-bit group selected by (u&1)*4. + const uint qsbase = rrow + (step >> 2) * (uint)m; + const uint qhbase = rrow + (step >> 3) * (uint)m; + uint8 qw; + #define QW(u) (EXP4(src0_qs[qsbase + (u) * m]) | \ + EXP1((uint)(src0_qh[qhbase + ((u) >> 1) * m] >> (((u) & 1u) * 4u)) & 0xFu)) + qw.s0 = QW(0); qw.s1 = QW(1); qw.s2 = QW(2); qw.s3 = QW(3); + qw.s4 = QW(4); qw.s5 = QW(5); qw.s6 = QW(6); qw.s7 = QW(7); + #undef QW + + // cooperatively stage the 32-token x 32-K int8 activations to lm + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg( + __read_only image1d_buffer_t src0_qs_img, // q5_0 low nibbles as uint32 texels (2 ushorts/texel) + __global const uchar * src0_qh, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint sel = (rrow & 1u) * 16u; // constant per WI: qs ushort half in its uint32 texel + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + const float minv = d_w * 16.0f; + + const uint qsbase = rrow + (step >> 2) * (uint)m; // ushort index + const uint qhbase = rrow + (step >> 3) * (uint)m; + uint8 qw; + // qs ushort via texture: uint32 texel = ushort_index>>1, half = sel. + #define QSU(u) ((read_imageui(src0_qs_img, (int)((qsbase + (u) * m) >> 1)).x >> sel) & 0xFFFFu) + #define QW(u) (EXP4(QSU(u)) | \ + EXP1((uint)(src0_qh[qhbase + ((u) >> 1) * m] >> (((u) & 1u) * 4u)) & 0xFu)) + qw.s0 = QW(0); qw.s1 = QW(1); qw.s2 = QW(2); qw.s3 = QW(3); + qw.s4 = QW(4); qw.s5 = QW(5); qw.s6 = QW(6); qw.s7 = QW(7); + #undef QW + #undef QSU + + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl new file mode 100644 index 0000000..94b4ef6 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_128 +#endif + +kernel void kernel_gemm_noshuffle_q5_1_f32( + global const ushort * src0_qs, // quantized A + global const uchar * src0_qh, // 5th bits + global const half * src0_d, // A scales + global const half * src0_m, // A mins + __read_only image1d_buffer_t src1, // B (1d image) + global float * dst, // C + int m, // M + int n, // N with padding + int k, // K + int n_no_padding // N without padding +) { + + int n_4 = n >> 2; + + int gy = get_global_id(0); + int gx = get_global_id(1); + int gx_2 = gx << 2; + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + half8 B; + half4 dequantized_weights; + + global const ushort * weight_ptr = src0_qs + gx_2; + global const uchar * qh_ptr = src0_qh + gx_2; + global const half * scale_ptr = src0_d + gx_2; + global const half * min_ptr = src0_m + gx_2; + + for (int i = 0; i < k; i += 4) { + + B.s0123 = read_imageh(src1, gy*2 + i*n_4); + B.s4567 = read_imageh(src1, gy*2 + i*n_4 + 1); + + ushort4 bits4 = vload4(0, weight_ptr + (i >> 2)*m); + uchar4 bits1 = vload4(0, qh_ptr + (i >> 3)*m); + uchar4 qh = bits1 >> (uchar4)(i & 4); + + half4 scale = vload4(0, scale_ptr + (i >> 5)*m); + half4 minv = vload4(0, min_ptr + (i >> 5)*m); + + // j=0 + dequantized_weights.s0 = convert_half((bits4.s0 & 0x000F) | ((qh.s0 & 0x01) << 4)) * scale.s0 + minv.s0; + dequantized_weights.s1 = convert_half((bits4.s1 & 0x000F) | ((qh.s1 & 0x01) << 4)) * scale.s1 + minv.s1; + dequantized_weights.s2 = convert_half((bits4.s2 & 0x000F) | ((qh.s2 & 0x01) << 4)) * scale.s2 + minv.s2; + dequantized_weights.s3 = convert_half((bits4.s3 & 0x000F) | ((qh.s3 & 0x01) << 4)) * scale.s3 + minv.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + + // j=1 + B.s0123 = read_imageh(src1, gy*2 + (i+1)*n_4); + B.s4567 = read_imageh(src1, gy*2 + (i+1)*n_4 + 1); + dequantized_weights.s0 = convert_half(((bits4.s0 & 0x00F0) >> 4) | ((qh.s0 & 0x02) << 3)) * scale.s0 + minv.s0; + dequantized_weights.s1 = convert_half(((bits4.s1 & 0x00F0) >> 4) | ((qh.s1 & 0x02) << 3)) * scale.s1 + minv.s1; + dequantized_weights.s2 = convert_half(((bits4.s2 & 0x00F0) >> 4) | ((qh.s2 & 0x02) << 3)) * scale.s2 + minv.s2; + dequantized_weights.s3 = convert_half(((bits4.s3 & 0x00F0) >> 4) | ((qh.s3 & 0x02) << 3)) * scale.s3 + minv.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + + // j=2 + B.s0123 = read_imageh(src1, gy*2 + (i+2)*n_4); + B.s4567 = read_imageh(src1, gy*2 + (i+2)*n_4 + 1); + dequantized_weights.s0 = convert_half(((bits4.s0 & 0x0F00) >> 8) | ((qh.s0 & 0x04) << 2)) * scale.s0 + minv.s0; + dequantized_weights.s1 = convert_half(((bits4.s1 & 0x0F00) >> 8) | ((qh.s1 & 0x04) << 2)) * scale.s1 + minv.s1; + dequantized_weights.s2 = convert_half(((bits4.s2 & 0x0F00) >> 8) | ((qh.s2 & 0x04) << 2)) * scale.s2 + minv.s2; + dequantized_weights.s3 = convert_half(((bits4.s3 & 0x0F00) >> 8) | ((qh.s3 & 0x04) << 2)) * scale.s3 + minv.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + + // j=3 + B.s0123 = read_imageh(src1, gy*2 + (i+3)*n_4); + B.s4567 = read_imageh(src1, gy*2 + (i+3)*n_4 + 1); + dequantized_weights.s0 = convert_half(((bits4.s0 & 0xF000) >> 12) | ((qh.s0 & 0x08) << 1)) * scale.s0 + minv.s0; + dequantized_weights.s1 = convert_half(((bits4.s1 & 0xF000) >> 12) | ((qh.s1 & 0x08) << 1)) * scale.s1 + minv.s1; + dequantized_weights.s2 = convert_half(((bits4.s2 & 0xF000) >> 12) | ((qh.s2 & 0x08) << 1)) * scale.s2 + minv.s2; + dequantized_weights.s3 = convert_half(((bits4.s3 & 0xF000) >> 12) | ((qh.s3 & 0x08) << 1)) * scale.s3 + minv.s3; + c0 += B * dequantized_weights.s0; + c1 += B * dequantized_weights.s1; + c2 += B * dequantized_weights.s2; + c3 += B * dequantized_weights.s3; + } + + int idx = (gy<<3)*m + (gx<<2); + + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if(idx+3 < m*n_no_padding){ + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl new file mode 100644 index 0000000..eef474b --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl @@ -0,0 +1,302 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_N 32 +#define QK_K 256 +#define K_SCALE_SIZE 12 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uchar * d, + uchar * m, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + if (j < 4) { + *d = q[j] & mask_d6; + *m = q[j+4] & mask_d6; + } else { + *d = (q[j+4] & mask_d4) | ((q[j-4] & mask_hi2) >> 2); + *m = ((q[j+4] >> 4) & mask_d4) | ((q[j] & mask_hi2) >> 2); + } +} + +// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, bits 0-3). +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// 4 high bits (one per element, in bits 0-3 of h) -> bit 4 of each of 4 bytes, +// so OR with EXP4 forms the 5-bit q5_K code 0..31. +#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \ + (((uint)((h) & 0x2u)) << 11) | \ + (((uint)((h) & 0x4u)) << 18) | \ + (((uint)((h) & 0x8u)) << 25) ) + +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a( + __global const ushort * src0_q, // q5_K low nibbles (transposed, ushort = 4 nibbles) + __global const uchar * src0_qh, // q5_K high bits (transposed, uchar = 8 elems/byte) + __global const uchar * src0_s, // 6-bit scale/min codes [row][superblock][12] + __global const half * src0_d, // per-superblock scale (transposed) + __global const half * src0_dm, // per-superblock min (transposed) + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k, // K (== ne00) + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint num_superblocks = (uint)k / QK_K; + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + // repack this row's 32 weights (nibble | high-bit) into 8 dp4a uints. + // ushort u -> 4 elements at K = step + u*4; its 4 high bits are nibble + // (u&1) of qh byte (step/8 + u/2). + const uint wbase = rrow + (step >> 2) * (uint)m; + const uint qhbase = rrow + (step >> 3) * (uint)m; + uint8 qw; +#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \ + | EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) ) + qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3); + qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7); +#undef QWU + + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4( + __global const ushort * src0_q, + __global const uchar * src0_qh, + __global const uchar * src0_s, + __global const half * src0_d, + __global const half * src0_dm, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint num_superblocks = (uint)k / QK_K; + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + const uint wbase = rrow + (step >> 2) * (uint)m; + const uint qhbase = rrow + (step >> 3) * (uint)m; + uint8 qw; +#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \ + | EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) ) + qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3); + qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7); +#undef QWU + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl index 3a9c624..141f6a2 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl @@ -5,6 +5,7 @@ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable #define ADRENO_GPU 1 #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) #endif #ifdef ADRENO_GPU @@ -138,3 +139,107 @@ kernel void kernel_gemm_noshuffle_q6_K_f32( vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); } } + +// Cooperative-K q6_K GEMM for the small-batch (n_q in [2..8]) path. Same idea +// as the q4_K _cok kernel: WG = (COK_SG lanes x COK_NSG subgroups), each lane +// owns ONE output row (half8 over the 8 padded cols), and the COK_NSG +// subgroups split the K iterations round-robin and combine via a __local +// reduction. Replaces the default 4-row-per-WI tile that walked all of K alone +// (~M/512 WGs + serial reduction) at small n_q. REQD_SUBGROUP_SIZE_64 + +// barrier (never sub_group_reduce at full width on X2). +#define COK_NSG 8 +#define COK_SG 64 +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemm_noshuffle_q6_K_f32_cok( + global const ushort * src0_ql, + global const uchar * src0_qh, + global const ushort * src0_s, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding, + ushort mask_f000, + uchar mask_c0 +) { + dst = (global float *)( (global char *)dst + offsetd ); + + int n_4 = n >> 2; + int gx = get_global_id(0); // output row + int sg = get_local_id(1); // subgroup index (K-split) + int lane = get_local_id(0); // lane within subgroup + + global const ushort * ptr_ql = src0_ql + gx; + global const uchar * ptr_qh = src0_qh + gx; + global const ushort * ptr_s = src0_s + gx; + global const half * ptr_d = src0_d + gx; + + half8 acc = 0; + half8 B; + half dq; + + int num_iter = k >> 2; // k/4 iterations, 4 k-values each + + for (int ib = sg; ib < num_iter; ib += COK_NSG) { + int i = ib << 2; // ib * 4 + + ushort bits4 = ptr_ql[ib * m]; // ql for row gx at this 4-block + uchar bits2 = ptr_qh[ib * m]; // qh + + ushort s_packed = ptr_s[(i >> 5) * m]; // (i/16/2) = i/32 + char2 sc2 = as_char2(s_packed); + char scale_s = (((i >> 4) & 1) == 0) ? sc2.s0 : sc2.s1; // (i/16)%2 + half scale_d = ptr_d[(i >> 8) * m]; // i/256 + + // j=0 + B.s0123 = read_imageh(src1, (i + 0)*n_4 + 0); + B.s4567 = read_imageh(src1, (i + 0)*n_4 + 1); + dq = (convert_half((bits4 & 0x000F) | ((bits2 & 0x03) << 4)) - 32.f) * scale_s * scale_d; + acc += B * dq; + + // j=1 + B.s0123 = read_imageh(src1, (i + 1)*n_4 + 0); + B.s4567 = read_imageh(src1, (i + 1)*n_4 + 1); + dq = (convert_half(((bits4 & 0x00F0) >> 4) | ((bits2 & 0x0C) << 2)) - 32.f) * scale_s * scale_d; + acc += B * dq; + + // j=2 + B.s0123 = read_imageh(src1, (i + 2)*n_4 + 0); + B.s4567 = read_imageh(src1, (i + 2)*n_4 + 1); + dq = (convert_half(((bits4 & 0x0F00) >> 8) | (bits2 & 0x30)) - 32.f) * scale_s * scale_d; + acc += B * dq; + + // j=3 + B.s0123 = read_imageh(src1, (i + 3)*n_4 + 0); + B.s4567 = read_imageh(src1, (i + 3)*n_4 + 1); + dq = (convert_half(((bits4 & mask_f000) >> 12) | ((bits2 & mask_c0) >> 2)) - 32.f) * scale_s * scale_d; + acc += B * dq; + } + + local float8 reduceLM[COK_SG * (COK_NSG - 1)]; + if (sg > 0) { + reduceLM[(sg - 1) * COK_SG + lane] = convert_float8(acc); + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (sg == 0) { + float8 sum = convert_float8(acc); + for (int s = 0; s < COK_NSG - 1; s++) { + sum += reduceLM[s * COK_SG + lane]; + } + int idx = gx; + if (idx < m*n_no_padding) { dst[idx] = sum.s0; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s1; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s2; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s3; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s4; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s5; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s6; idx += m; } + if (idx < m*n_no_padding) { dst[idx] = sum.s7; } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl new file mode 100644 index 0000000..ffd943a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl @@ -0,0 +1,136 @@ +// Batched (N>1) q6_K GEMM over the 64-row-TILED canonical layout produced by +// kernel_convert_block_q6_k_tiled_ns (cvt.cl). Companion to the decode kernel +// kernel_gemv_noshuffle_q6_K_f32_tiled: SAME pack, SAME canonical e-order +// dequant (correct by construction vs reference ggml q6_K), extended to N output +// columns. Makes the batched lm_head/embed (perplexity, spec-decode verify, +// batched serving) correct on GPU while keeping the tiled convert the fast decode +// GEMV depends on. +// +// One work-item owns one output ROW for a block of BN columns. A work-group is +// {64 lanes, NTILES subgroups} = NTILES*64 rows; the global z dimension tiles the +// N columns by BN. Each work-item computes its row's FULL K (no K-split, so no +// cross-subgroup reduction), which lets the whole work-group share one staged +// activation block: +// +// __local activation staging — the BN columns of the current superblock (BN*256 +// floats) are loaded into __local once per superblock, cooperatively by all +// NTILES*64 work-items, then every row reads its activation from __local. This +// removes the ~Nrows-fold redundant image reads of the first version (each lane +// re-read the activation), which made the batched GEMM ~2x slower than the plain +// noshuffle GEMM. +// +// Weights are read from __global (coalesced) — matching the decode kernel; the +// lm_head weight is streamed with little reuse where coalesced global beats the +// Adreno texture cache. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define NTILES 4 // 64-row tiles per work-group (NTILES*64 = 256 rows) +#define TILE_ROWS 64 +#define BN 16 // output columns handled per work-group (global z step) +#define WG_THREADS (NTILES * TILE_ROWS) + +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemm_noshuffle_q6_K_f32_tiled( + __global uint4 * src0_ql, // tiled: 8 uint4 granules / superblock + __global uint4 * src0_qh, // tiled: 4 uint4 granules / superblock + __global char * src0_s, // tiled: 16 chars / superblock + __global half * src0_d, // tiled: 1 half / superblock + read_only image1d_buffer_t src1, // activation [ne00, ne11] f32 (RGBA), column-major + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + int rit = get_local_id(0); // 0..63 (lane within a tile; coalesces weight loads) + int sg = get_local_id(1); // 0..NTILES-1 + int lid = sg * TILE_ROWS + rit; // 0..WG_THREADS-1 (flat local id) + int row = get_group_id(0) * WG_THREADS + lid; + int rt = row / TILE_ROWS; // global 64-row tile index + int col0 = get_global_id(2) * BN; // first output column of this block + + int nb = ne00 / 256; // superblocks per row + int act_col_stride = ne00 / 4; // activation float4 pixels per column + + const bool row_ok = row < ne01; + + // staged activation: BN columns x 256 elements for the current superblock + __local float lact[BN * 256]; + + float acc[BN]; + #pragma unroll + for (int j = 0; j < BN; ++j) acc[j] = 0.0f; + + for (int sb = 0; sb < nb; ++sb) { + // cooperatively stage BN columns' 256 activation elements (= BN*64 float4) + for (int p = lid; p < BN * 64; p += WG_THREADS) { + int j = p >> 6; // column within the BN block (p / 64) + int e4 = p & 63; // element-quad within the column (p % 64) + int c = col0 + j; + float4 v = (c < ne11) + ? read_imagef(src1, c * act_col_stride + sb * 64 + e4) + : (float4)(0.0f); + lact[p * 4 + 0] = v.x; + lact[p * 4 + 1] = v.y; + lact[p * 4 + 2] = v.z; + lact[p * 4 + 3] = v.w; // lact[j*256 + e], e = e4*4 + t + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (row_ok) { + int tile_blk = rt * nb + sb; // ne02 == 1 for lm_head/embed + + float dval = (float)src0_d[tile_blk * TILE_ROWS + rit]; + __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16; + + uint ql[32]; + uint qh[16]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit]; + ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w; + } + #pragma unroll + for (int g = 0; g < 4; ++g) { + uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit]; + qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w; + } + + // NOTE: the e loop (256) is deliberately NOT unrolled. Fully unrolling + // 256*BN MACs overflows the in-process Adreno compiler (host stack + // overflow at clBuildProgram, same class as the FA DK=512 OOM). + for (int e = 0; e < 256; ++e) { + uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF; + uint hi2 = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3; + int code = (int)(low4 | (hi2 << 4)) - 32; + int sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1); + float cs = (float)code * (float)sc[sidx] * dval; + #pragma unroll + for (int j = 0; j < BN; ++j) { + acc[j] += cs * lact[j * 256 + e]; + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_ok) { + dst = (global float*)((global char*)dst + offsetd); + #pragma unroll + for (int j = 0; j < BN; ++j) { + int c = col0 + j; + if (c < ne11) { + dst[(ulong)c * ne01 + row] = acc[j]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl new file mode 100644 index 0000000..f17403a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl @@ -0,0 +1,258 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_N 32 +#define QK_K 256 + +// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, in bits 0-3). +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// 4 2-bit highs in byte `b` -> 4 bytes, value 0..3 in bits 4-5 (pre-multiplied +// by 16 so it ORs with the EXP4 nibble to form q6 in 0..63). +#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \ + (((uint)((b) & 0x0Cu)) << 10) | \ + (((uint)((b) & 0x30u)) << 16) | \ + (((uint)((b) & 0xC0u)) << 22) ) + +// q6 (0..63, bits 0-5 of each byte) -> (q6-32) as a signed int8 per byte. +inline uint SIGN6(uint q6p) { + uint x = q6p ^ 0x20202020u; + uint s = x & 0x20202020u; + return x | (s << 1) | (s << 2); +} + +// 16-K dp4a dot: 4 packed weight uints against 4 packed int8 activation uints. +inline int dot4_q8a(uint w0, uint w1, uint w2, uint w3, + uint a0, uint a1, uint a2, uint a3) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(w0, a0, r); + r = dot_acc_sat_4x8packed_ss_int(w1, a1, r); + r = dot_acc_sat_4x8packed_ss_int(w2, a2, r); + r = dot_acc_sat_4x8packed_ss_int(w3, a3, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a( + __global const ushort * src0_ql, // q6_K low nibbles (noshuffle) + __global const uchar * src0_qh, // q6_K high 2-bit (uchar, 4 highs/elem) + __global const ushort * src0_s, // int8 scale codes (2 chars/ushort, per 16) + __global const half * src0_d, // per-superblock scale + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; // 32-block index along K + const uint sb_idx = step / QK_K; // superblock index + + // q6_K superblock scale + the two int8 sub-scales spanning this 32-block + const float dd = (float)src0_d[rrow + sb_idx * m]; + const char2 sc = as_char2(src0_s[rrow + sub * m]); + const float scale0 = dd * (float)sc.s0; // K step..step+15 + const float scale1 = dd * (float)sc.s1; // K step+16..step+31 + + // repack this row's 32 weights into 8 dp4a uints (4 K each). ql ushort + + // qh uchar are co-located at src0_*[row + (step/4 + u)*m]. + const uint wbase = rrow + (step >> 2) * (uint)m; + uint qw[8]; + #pragma unroll + for (int u = 0; u < 8; ++u) { + const uint o = wbase + (uint)u * (uint)m; + qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o])); + } + + // cooperatively stage the 32-token x 32-K int8 activations + scale + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + #define DOT_TOK(j) { \ + __local const uint * a = sh_qa[b + (j)]; \ + const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a[0], a[1], a[2], a[3]); \ + const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a[4], a[5], a[6], a[7]); \ + rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \ + } + DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3); + #undef DOT_TOK + const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]); + acc[g] += ad * rf; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +// Stage each 32-K activation block as two uint4 values. +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4( + __global const ushort * src0_ql, + __global const uchar * src0_qh, + __global const ushort * src0_s, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + + const float dd = (float)src0_d[rrow + sb_idx * m]; + const char2 sc = as_char2(src0_s[rrow + sub * m]); + const float scale0 = dd * (float)sc.s0; + const float scale1 = dd * (float)sc.s1; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint qw[8]; + #pragma unroll + for (int u = 0; u < 8; ++u) { + const uint o = wbase + (uint)u * (uint)m; + qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o])); + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + #define DOT_TOK(j) { \ + const uint4 a0 = sh_qa4[b + (j)][0]; \ + const uint4 a1 = sh_qa4[b + (j)][1]; \ + const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a0.x, a0.y, a0.z, a0.w); \ + const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a1.x, a1.y, a1.z, a1.w); \ + rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \ + } + DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3); + #undef DOT_TOK + const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]); + acc[g] += ad * rf; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_8x4.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl similarity index 98% rename from ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_8x4.cl rename to ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl index 51ce212..7f06a22 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_8x4.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl @@ -11,7 +11,7 @@ REQD_SUBGROUP_SIZE_128 #endif -kernel void kernel_mul_mm_q8_0_f32_8x4( +kernel void kernel_gemm_noshuffle_q8_0_f32( global const uint * src0_q, global const half * src0_d, __read_only image1d_buffer_t src1, diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl new file mode 100644 index 0000000..32ff7b4 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl @@ -0,0 +1,334 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// ne1<=8 keeps the f16 / bin small-batch path. + +#define TILESIZE_N 32 + +// 32-K dp4a dot of one token's int8 activations (8 packed uints in lm) against +// 8 packed weight uints. q8_0 weights are already dp4a-format signed int8. +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a( + __global const uint * src0_q, // q8_0 weights: signed int8, 4/uint, feature-major + __global const half * src0_d, // per-32-block scale, feature-major [row + (k/32)*m] + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + // 8 weight uints (32 int8) for this row, this 32-block. Feature-major: + // src0_q[row + (k/4 + u)*m], k/4 = step/4 (= step>>2). + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = src0_q[wbase + 0 * m]; + qw.s1 = src0_q[wbase + 1 * m]; + qw.s2 = src0_q[wbase + 2 * m]; + qw.s3 = src0_q[wbase + 3 * m]; + qw.s4 = src0_q[wbase + 4 * m]; + qw.s5 = src0_q[wbase + 5 * m]; + qw.s6 = src0_q[wbase + 6 * m]; + qw.s7 = src0_q[wbase + 7 * m]; + + // cooperatively stage the 32-token x 32-K int8 activations to LDS + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg( + __read_only image1d_buffer_t src0_q_img, // q8_0 weights as uint32 texels (4 int8/texel) + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = read_imageui(src0_q_img, (int)(wbase + 0 * m)).x; + qw.s1 = read_imageui(src0_q_img, (int)(wbase + 1 * m)).x; + qw.s2 = read_imageui(src0_q_img, (int)(wbase + 2 * m)).x; + qw.s3 = read_imageui(src0_q_img, (int)(wbase + 3 * m)).x; + qw.s4 = read_imageui(src0_q_img, (int)(wbase + 4 * m)).x; + qw.s5 = read_imageui(src0_q_img, (int)(wbase + 5 * m)).x; + qw.s6 = read_imageui(src0_q_img, (int)(wbase + 6 * m)).x; + qw.s7 = read_imageui(src0_q_img, (int)(wbase + 7 * m)).x; + + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4( + __global const uint * src0_q, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = src0_q[wbase + 0 * m]; + qw.s1 = src0_q[wbase + 1 * m]; + qw.s2 = src0_q[wbase + 2 * m]; + qw.s3 = src0_q[wbase + 3 * m]; + qw.s4 = src0_q[wbase + 4 * m]; + qw.s5 = src0_q[wbase + 5 * m]; + qw.s6 = src0_q[wbase + 6 * m]; + qw.s7 = src0_q[wbase + 7 * m]; + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl b/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl new file mode 100644 index 0000000..0f17df4 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl @@ -0,0 +1,238 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load : enable + +__constant sampler_t smp_zero = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST; + +__kernel void adreno_xmem_pack_src_f32( + __global const void * src_void, + ulong offset, + __write_only image2d_t src_img, + int K, + int N) { + const int x = get_global_id(0); + const int y = get_global_id(1); + const int kpack = K / 4; + + if (x >= N || y >= kpack) { + return; + } + + __global const float * src = (__global const float *)((__global const char *)src_void + offset); + const int base = x*K + y*4; + const half4 v = (half4)((half)src[base + 0], (half)src[base + 1], (half)src[base + 2], (half)src[base + 3]); + write_imageh(src_img, (int2)(x, y), v); +} + +__kernel void adreno_xmem_prepack_weight_f16( + __global half4 * dst, + __global const void * src_void, + ulong offset, + int K, + int M, + int kpack, + int npack, + int os) { + const int linear = get_global_id(0); + const int total = kpack*npack; + if (linear >= total) { + return; + } + + __global const half * src = (__global const half *)((__global const char *)src_void + offset); + + const int dst_ogroup = linear % os; + const int dst_o_sp_i = linear / os; + const int dst_i = dst_o_sp_i % kpack; + const int dst_o = dst_o_sp_i / kpack; + const int o_slice = dst_o*os + dst_ogroup; + const int k_base = dst_i*4; + + half4 w0 = (half4)(0.0h); + half4 w1 = (half4)(0.0h); + half4 w2 = (half4)(0.0h); + half4 w3 = (half4)(0.0h); + + const int o0 = o_slice*4 + 0; + const int o1 = o_slice*4 + 1; + const int o2 = o_slice*4 + 2; + const int o3 = o_slice*4 + 3; + + if (k_base + 0 < K) { + if (o0 < M) w0.s0 = src[o0*K + k_base + 0]; + if (o1 < M) w0.s1 = src[o1*K + k_base + 0]; + if (o2 < M) w0.s2 = src[o2*K + k_base + 0]; + if (o3 < M) w0.s3 = src[o3*K + k_base + 0]; + } + if (k_base + 1 < K) { + if (o0 < M) w1.s0 = src[o0*K + k_base + 1]; + if (o1 < M) w1.s1 = src[o1*K + k_base + 1]; + if (o2 < M) w1.s2 = src[o2*K + k_base + 1]; + if (o3 < M) w1.s3 = src[o3*K + k_base + 1]; + } + if (k_base + 2 < K) { + if (o0 < M) w2.s0 = src[o0*K + k_base + 2]; + if (o1 < M) w2.s1 = src[o1*K + k_base + 2]; + if (o2 < M) w2.s2 = src[o2*K + k_base + 2]; + if (o3 < M) w2.s3 = src[o3*K + k_base + 2]; + } + if (k_base + 3 < K) { + if (o0 < M) w3.s0 = src[o0*K + k_base + 3]; + if (o1 < M) w3.s1 = src[o1*K + k_base + 3]; + if (o2 < M) w3.s2 = src[o2*K + k_base + 3]; + if (o3 < M) w3.s3 = src[o3*K + k_base + 3]; + } + + dst[linear*4 + 0] = w0; + dst[linear*4 + 1] = w1; + dst[linear*4 + 2] = w2; + dst[linear*4 + 3] = w3; +} + +__attribute__((qcom_max_concurrent_subgroups(12))) +__kernel void kernel_gemm_xmem_f16_f32_os8( + __constant half8 * weights_buffer __attribute__((sub_group_uniform)), + __constant half8 * xmem_buffer __attribute__((max_constant_size((6144)))), + __read_only image2d_t src_img, + __write_only image2d_t dst_img, + int N, + int npack, + int kpack) { + const int X = get_group_id(1)*get_local_size(0) + get_local_id(0); + const int Z = get_group_id(0)*get_local_size(2) + get_local_id(2); + + if (Z*8 >= npack) { + return; + } + + half4 r0 = (half4)(0.0h); + half4 r1 = (half4)(0.0h); + half4 r2 = (half4)(0.0h); + half4 r3 = (half4)(0.0h); + half4 r4 = (half4)(0.0h); + half4 r5 = (half4)(0.0h); + half4 r6 = (half4)(0.0h); + half4 r7 = (half4)(0.0h); + + int f_offset = Z*kpack*32; + int subgroup_id = (int)(0x1F & qcom_get_physical_sub_group_id()); + subgroup_id = subgroup_id % 12; + const int c_offset = subgroup_id*32; + __constant half16 * weights_cache = (__constant half16 *)&xmem_buffer[c_offset]; + + int coord_s = 0; + do { + const half4 src0 = read_imageh(src_img, smp_zero, (int2)(X, coord_s)); + coord_s++; + const half4 src1 = read_imageh(src_img, smp_zero, (int2)(X, coord_s)); + coord_s++; + + qcom_sub_group_constant_load8(xmem_buffer, weights_buffer, c_offset, f_offset >> 1, 32); + f_offset += 64; + qcom_sub_group_sync(QCOM_CLK_CONST_LOAD_SYNC); + + r0 += src0.x * weights_cache[0].s0123; + r0 += src0.y * weights_cache[0].s4567; + r0 += src0.z * weights_cache[0].s89ab; + r0 += src0.w * weights_cache[0].scdef; + r1 += src0.x * weights_cache[1].s0123; + r1 += src0.y * weights_cache[1].s4567; + r1 += src0.z * weights_cache[1].s89ab; + r1 += src0.w * weights_cache[1].scdef; + r2 += src0.x * weights_cache[2].s0123; + r2 += src0.y * weights_cache[2].s4567; + r2 += src0.z * weights_cache[2].s89ab; + r2 += src0.w * weights_cache[2].scdef; + r3 += src0.x * weights_cache[3].s0123; + r3 += src0.y * weights_cache[3].s4567; + r3 += src0.z * weights_cache[3].s89ab; + r3 += src0.w * weights_cache[3].scdef; + r4 += src0.x * weights_cache[4].s0123; + r4 += src0.y * weights_cache[4].s4567; + r4 += src0.z * weights_cache[4].s89ab; + r4 += src0.w * weights_cache[4].scdef; + r5 += src0.x * weights_cache[5].s0123; + r5 += src0.y * weights_cache[5].s4567; + r5 += src0.z * weights_cache[5].s89ab; + r5 += src0.w * weights_cache[5].scdef; + r6 += src0.x * weights_cache[6].s0123; + r6 += src0.y * weights_cache[6].s4567; + r6 += src0.z * weights_cache[6].s89ab; + r6 += src0.w * weights_cache[6].scdef; + r7 += src0.x * weights_cache[7].s0123; + r7 += src0.y * weights_cache[7].s4567; + r7 += src0.z * weights_cache[7].s89ab; + r7 += src0.w * weights_cache[7].scdef; + + r0 += src1.x * weights_cache[8].s0123; + r0 += src1.y * weights_cache[8].s4567; + r0 += src1.z * weights_cache[8].s89ab; + r0 += src1.w * weights_cache[8].scdef; + r1 += src1.x * weights_cache[9].s0123; + r1 += src1.y * weights_cache[9].s4567; + r1 += src1.z * weights_cache[9].s89ab; + r1 += src1.w * weights_cache[9].scdef; + r2 += src1.x * weights_cache[10].s0123; + r2 += src1.y * weights_cache[10].s4567; + r2 += src1.z * weights_cache[10].s89ab; + r2 += src1.w * weights_cache[10].scdef; + r3 += src1.x * weights_cache[11].s0123; + r3 += src1.y * weights_cache[11].s4567; + r3 += src1.z * weights_cache[11].s89ab; + r3 += src1.w * weights_cache[11].scdef; + r4 += src1.x * weights_cache[12].s0123; + r4 += src1.y * weights_cache[12].s4567; + r4 += src1.z * weights_cache[12].s89ab; + r4 += src1.w * weights_cache[12].scdef; + r5 += src1.x * weights_cache[13].s0123; + r5 += src1.y * weights_cache[13].s4567; + r5 += src1.z * weights_cache[13].s89ab; + r5 += src1.w * weights_cache[13].scdef; + r6 += src1.x * weights_cache[14].s0123; + r6 += src1.y * weights_cache[14].s4567; + r6 += src1.z * weights_cache[14].s89ab; + r6 += src1.w * weights_cache[14].scdef; + r7 += src1.x * weights_cache[15].s0123; + r7 += src1.y * weights_cache[15].s4567; + r7 += src1.z * weights_cache[15].s89ab; + r7 += src1.w * weights_cache[15].scdef; + } while (coord_s < kpack); + + // Keep all lanes active until the subgroup loads and syncs are done. + if (X >= N) { + return; + } + + int coord_s_out = Z*8; + if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r0); coord_s_out++; } + if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r1); coord_s_out++; } + if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r2); coord_s_out++; } + if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r3); coord_s_out++; } + if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r4); coord_s_out++; } + if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r5); coord_s_out++; } + if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r6); coord_s_out++; } + if (coord_s_out < npack) { write_imageh(dst_img, (int2)(X, coord_s_out), r7); } +} + +__kernel void adreno_xmem_store_dst_f32( + __read_only image2d_t dst_img, + __global void * dst_void, + ulong offset, + int M, + int N) { + const int x = get_global_id(0); + const int y = get_global_id(1); + const int npack = (M + 3) / 4; + + if (x >= N || y >= npack) { + return; + } + + __global float * dst = (__global float *)((__global char *)dst_void + offset); + const half4 hv = read_imageh(dst_img, smp_zero, (int2)(x, y)); + const int m = y*4; + if (m + 0 < M) dst[x*M + m + 0] = (float)hv.s0; + if (m + 1 < M) dst[x*M + m + 1] = (float)hv.s1; + if (m + 2 < M) dst[x*M + m + 2] = (float)hv.s2; + if (m + 3 < M) dst[x*M + m + 3] = (float)hv.s3; +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl new file mode 100644 index 0000000..3f21c35 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl @@ -0,0 +1,263 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_MXFP4 32 +#define N_SIMDGROUP 4 +#define SIMDGROUP_WIDTH 64 + +static inline half8 mxfp4_to_fp16_packed8(ushort2 fp4x8) { + ushort2 fp16_packed_a_0, fp16_packed_b_0, bias_a, bias_b, sign_a, sign_b; + fp16_packed_a_0.lo = (fp4x8.s0 << 9) & 0x0E00; + fp16_packed_a_0.hi = (fp4x8.s0 << 5) & 0x0E00; + fp16_packed_b_0.lo = (fp4x8.s0 << 1) & 0x0E00; + fp16_packed_b_0.hi = (fp4x8.s0 >> 3) & 0x0E00; + + bias_a.lo = (fp16_packed_a_0.lo != 0) ? 0x3800 : 0x0; + bias_a.hi = (fp16_packed_a_0.hi != 0) ? 0x3800 : 0x0; + bias_b.lo = (fp16_packed_b_0.lo != 0) ? 0x3800 : 0x0; + bias_b.hi = (fp16_packed_b_0.hi != 0) ? 0x3800 : 0x0; + + fp16_packed_a_0.lo = (fp16_packed_a_0.lo != 0x0200) ? fp16_packed_a_0.lo : 0x0; + fp16_packed_a_0.hi = (fp16_packed_a_0.hi != 0x0200) ? fp16_packed_a_0.hi : 0x0; + fp16_packed_b_0.lo = (fp16_packed_b_0.lo != 0x0200) ? fp16_packed_b_0.lo : 0x0; + fp16_packed_b_0.hi = (fp16_packed_b_0.hi != 0x0200) ? fp16_packed_b_0.hi : 0x0; + + sign_a.lo = (fp4x8.s0 << 12) & 0x8000; + sign_a.hi = (fp4x8.s0 << 8) & 0x8000; + sign_b.lo = (fp4x8.s0 << 4) & 0x8000; + sign_b.hi = fp4x8.s0 & 0x8000; + + fp16_packed_a_0 = sign_a + bias_a + fp16_packed_a_0; + fp16_packed_b_0 = sign_b + bias_b + fp16_packed_b_0; + + ushort2 fp16_packed_a_1, fp16_packed_b_1; + fp16_packed_a_1.lo = (fp4x8.s1 << 9) & 0x0E00; + fp16_packed_a_1.hi = (fp4x8.s1 << 5) & 0x0E00; + fp16_packed_b_1.lo = (fp4x8.s1 << 1) & 0x0E00; + fp16_packed_b_1.hi = (fp4x8.s1 >> 3) & 0x0E00; + + bias_a.lo = (fp16_packed_a_1.lo != 0) ? 0x3800 : 0x0; + bias_a.hi = (fp16_packed_a_1.hi != 0) ? 0x3800 : 0x0; + bias_b.lo = (fp16_packed_b_1.lo != 0) ? 0x3800 : 0x0; + bias_b.hi = (fp16_packed_b_1.hi != 0) ? 0x3800 : 0x0; + + fp16_packed_a_1.lo = (fp16_packed_a_1.lo != 0x0200) ? fp16_packed_a_1.lo : 0x0; + fp16_packed_a_1.hi = (fp16_packed_a_1.hi != 0x0200) ? fp16_packed_a_1.hi : 0x0; + fp16_packed_b_1.lo = (fp16_packed_b_1.lo != 0x0200) ? fp16_packed_b_1.lo : 0x0; + fp16_packed_b_1.hi = (fp16_packed_b_1.hi != 0x0200) ? fp16_packed_b_1.hi : 0x0; + + sign_a.lo = (fp4x8.s1 << 12) & 0x8000; + sign_a.hi = (fp4x8.s1 << 8) & 0x8000; + sign_b.lo = (fp4x8.s1 << 4) & 0x8000; + sign_b.hi = fp4x8.s1 & 0x8000; + + fp16_packed_a_1 = sign_a + bias_a + fp16_packed_a_1; + fp16_packed_b_1 = sign_b + bias_b + fp16_packed_b_1; + + return as_half8((ushort8)(fp16_packed_a_0, fp16_packed_b_0, fp16_packed_a_1, fp16_packed_b_1)); +} + +static inline float e8m0_to_fp32(uchar x) { + int bits; + bits = (x == 0) ? 0x00400000 : ((uint) x << 23); + return as_float(bits); +} + + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_mxfp4_f32_ns( + __global uint * src0_q, + __global uchar * src0_e, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + uint expert_offset = expert_id * ne00 * ne01 / 32; + + __private float sum = 0.0f; // each thread calculate partial sum of one output + + // loop along ne00 in block granularity, skip 4 blocks every iter + for (uint ib00 = sgid; ib00 < (ne00 / QK_MXFP4); ib00 += N_SIMDGROUP) { + + // load one block of q + uint4 regQ; + uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01; + + regQ.s0 = src0_q[block_offset]; + regQ.s1 = src0_q[block_offset + ne01]; + regQ.s2 = src0_q[block_offset + ne01 * 2]; + regQ.s3 = src0_q[block_offset + ne01 * 3]; + + uint offset = i11 * ne00 / 4 + ib00 * 8; + + half8 fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s0)); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (offset + 0)); + float4 acc = shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 1)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s1)); + + shared_y4 = read_imagef(src1, (offset + 2)); + acc += shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 3)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + + fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s2)); + + shared_y4 = read_imagef(src1, (offset + 4)); + acc += shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 5)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + + fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s3)); + + shared_y4 = read_imagef(src1, (offset + 6)); + acc += shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 7)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + uchar regE = src0_e[ib00 * ne01 + i01 + expert_offset]; + sum += e8m0_to_fp32(regE) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + // 1 outputs per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } + +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_mxfp4_f32_ns_wimg( + __read_only image1d_buffer_t src0_q, + __global uchar * src0_e, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + uint expert_offset = expert_id * ne00 * ne01 / 32; + + __private float sum = 0.0f; + + for (uint ib00 = sgid; ib00 < (ne00 / QK_MXFP4); ib00 += N_SIMDGROUP) { + + uint4 regQ; + uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01; + + regQ.s0 = read_imageui(src0_q, (int)(block_offset)).x; + regQ.s1 = read_imageui(src0_q, (int)(block_offset + ne01)).x; + regQ.s2 = read_imageui(src0_q, (int)(block_offset + ne01 * 2)).x; + regQ.s3 = read_imageui(src0_q, (int)(block_offset + ne01 * 3)).x; + + uint offset = i11 * ne00 / 4 + ib00 * 8; + + half8 fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s0)); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (offset + 0)); + float4 acc = shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 1)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s1)); + + shared_y4 = read_imagef(src1, (offset + 2)); + acc += shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 3)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s2)); + + shared_y4 = read_imagef(src1, (offset + 4)); + acc += shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 5)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s3)); + + shared_y4 = read_imagef(src1, (offset + 6)); + acc += shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 7)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + uchar regE = src0_e[ib00 * ne01 + i01 + expert_offset]; + sum += e8m0_to_fp32(regE) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl new file mode 100644 index 0000000..f90970b --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl @@ -0,0 +1,123 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_Q4_0 32 +#define N_SIMDGROUP 4 +#define SIMDGROUP_WIDTH 64 + +static inline float8 q4_0_to_fp32_packed8(ushort2 q4x8) { + float8 fp32x8; + fp32x8.s0 = (float)((q4x8.s0 & 0x000F) - 8); + fp32x8.s1 = (float)(((q4x8.s0 & 0x00F0) >> 4) - 8); + fp32x8.s2 = (float)(((q4x8.s0 & 0x0F00) >> 8) - 8); + fp32x8.s3 = (float)(((q4x8.s0 & 0xF000) >> 12) - 8); + fp32x8.s4 = (float)((q4x8.s1 & 0x000F) - 8); + fp32x8.s5 = (float)(((q4x8.s1 & 0x00F0) >> 4) - 8); + fp32x8.s6 = (float)(((q4x8.s1 & 0x0F00) >> 8) - 8); + fp32x8.s7 = (float)(((q4x8.s1 & 0xF000) >> 12) - 8); + return fp32x8; +} + + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_q4_0_f32_ns( + __global uint * src0_q, + __global half * src0_d, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + uint expert_offset = expert_id * ne00 * ne01 / 32; + + __private float sum = 0.0f; // each thread calculate partial sum of one output + + // loop along ne00 in block granularity, skip 4 blocks every iter + for (uint ib00 = sgid; ib00 < (ne00 / QK_Q4_0); ib00 += N_SIMDGROUP) { + + // load one block of q + uint4 regQ; + uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01; + + regQ.s0 = src0_q[block_offset]; + regQ.s1 = src0_q[block_offset + ne01]; + regQ.s2 = src0_q[block_offset + ne01 * 2]; + regQ.s3 = src0_q[block_offset + ne01 * 3]; + + uint offset = i11 * ne00 / 4 + ib00 * 8; + + float8 fp32x8 = q4_0_to_fp32_packed8(as_ushort2(regQ.s0)); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (offset + 0)); + float4 acc = shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 1)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_0_to_fp32_packed8(as_ushort2(regQ.s1)); + + shared_y4 = read_imagef(src1, (offset + 2)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 3)); + acc += shared_y4 * fp32x8.hi; + + + fp32x8 = q4_0_to_fp32_packed8(as_ushort2(regQ.s2)); + + shared_y4 = read_imagef(src1, (offset + 4)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 5)); + acc += shared_y4 * fp32x8.hi; + + + fp32x8 = q4_0_to_fp32_packed8(as_ushort2(regQ.s3)); + + shared_y4 = read_imagef(src1, (offset + 6)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 7)); + acc += shared_y4 * fp32x8.hi; + + half regS = src0_d[ib00 * ne01 + i01 + expert_offset]; + sum += (float)(regS) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + // 1 outputs per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } + +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl new file mode 100644 index 0000000..44bc15f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl @@ -0,0 +1,126 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_Q4_1 32 +#define N_SIMDGROUP 4 +#define SIMDGROUP_WIDTH 64 + +static inline float8 q4_1_to_fp32_packed8(ushort2 q4x8, half s, half m) { + float8 fp32x8; + fp32x8.s0 = (float)((q4x8.s0 & 0x000F) * s + m); + fp32x8.s1 = (float)(((q4x8.s0 & 0x00F0) >> 4) * s + m); + fp32x8.s2 = (float)(((q4x8.s0 & 0x0F00) >> 8) * s + m); + fp32x8.s3 = (float)(((q4x8.s0 & 0xF000) >> 12) * s + m); + fp32x8.s4 = (float)((q4x8.s1 & 0x000F) * s + m); + fp32x8.s5 = (float)(((q4x8.s1 & 0x00F0) >> 4) * s + m); + fp32x8.s6 = (float)(((q4x8.s1 & 0x0F00) >> 8) * s + m); + fp32x8.s7 = (float)(((q4x8.s1 & 0xF000) >> 12) * s + m); + return fp32x8; +} + + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_q4_1_f32_ns( + __global uint * src0_q, + __global half * src0_d, + __global half * src0_m, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + uint expert_offset = expert_id * ne00 * ne01 / 32; + + __private float sum = 0.0f; // each thread calculate partial sum of one output + + // loop along ne00 in block granularity, skip 4 blocks every iter + for (uint ib00 = sgid; ib00 < (ne00 / QK_Q4_1); ib00 += N_SIMDGROUP) { + + // load one block of q + uint4 regQ; + uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01; + + regQ.s0 = src0_q[block_offset]; + regQ.s1 = src0_q[block_offset + ne01]; + regQ.s2 = src0_q[block_offset + ne01 * 2]; + regQ.s3 = src0_q[block_offset + ne01 * 3]; + + uint offset = i11 * ne00 / 4 + ib00 * 8; + + half regM = src0_m[ib00 * ne01 + i01 + expert_offset]; + half regS = src0_d[ib00 * ne01 + i01 + expert_offset]; + + float8 fp32x8 = q4_1_to_fp32_packed8(as_ushort2(regQ.s0), regS, regM); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (offset + 0)); + float4 acc = shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 1)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_1_to_fp32_packed8(as_ushort2(regQ.s1), regS, regM); + + shared_y4 = read_imagef(src1, (offset + 2)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 3)); + acc += shared_y4 * fp32x8.hi; + + + fp32x8 = q4_1_to_fp32_packed8(as_ushort2(regQ.s2), regS, regM); + + shared_y4 = read_imagef(src1, (offset + 4)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 5)); + acc += shared_y4 * fp32x8.hi; + + + fp32x8 = q4_1_to_fp32_packed8(as_ushort2(regQ.s3), regS, regM); + + shared_y4 = read_imagef(src1, (offset + 6)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 7)); + acc += shared_y4 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + // 1 outputs per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } + +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl new file mode 100644 index 0000000..290a315 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl @@ -0,0 +1,272 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define K_SCALE_SIZE 12 +#define N_SIMDGROUP 4 +#define SIMDGROUP_WIDTH 64 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j+4] & 63; + } else { + *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2); + *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2); + } +} + +static inline float8 q4_k_to_fp32_packed8(ushort2 q4x8, float scale, float minv) { + float8 fp32x8; + fp32x8.s0 = (q4x8.s0 & 0x000F) * scale - minv; + fp32x8.s1 = ((q4x8.s0 & 0x00F0) >> 4) * scale - minv; + fp32x8.s2 = ((q4x8.s0 & 0x0F00) >> 8) * scale - minv; + fp32x8.s3 = ((q4x8.s0 & 0xF000) >> 12) * scale - minv; + fp32x8.s4 = (q4x8.s1 & 0x000F) * scale - minv; + fp32x8.s5 = ((q4x8.s1 & 0x00F0) >> 4) * scale - minv; + fp32x8.s6 = ((q4x8.s1 & 0x0F00) >> 8) * scale - minv; + fp32x8.s7 = ((q4x8.s1 & 0xF000) >> 12) * scale - minv; + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_q4_k_f32_ns( + __global uint * src0_q, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + + int num_superblocks = ne00 / QK_K; + int num_subblocks = ne00 / 32; + int scales_per_row = num_superblocks * K_SCALE_SIZE; + + // Expert offsets in the transposed noshuffle layout + uint expert_q_offset = expert_id * (ne00 / 8) * ne01; + uint expert_d_offset = expert_id * num_superblocks * ne01; + + __private float sum = 0.0f; + + // Loop over sub-blocks of 32 elements, N_SIMDGROUP sub-blocks per iter + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; + uint j = ib % 8; + + // Load d and dmin for this super-block + half d_val = src0_d[expert_d_offset + sb * ne01 + i01]; + half dm_val = src0_dm[expert_d_offset + sb * ne01 + i01]; + + // Load sub-block scale and min + global const uchar * sc = src0_s + (expert_id * ne01 + i01) * scales_per_row + sb * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(j, sc, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = (float)dm_val * (float)mn; + + // Load 4 uints of quants (32 nibbles = 32 elements) + uint q_base = expert_q_offset + ib * ne01 * 4 + i01; + + uint4 regQ; + regQ.s0 = src0_q[q_base]; + regQ.s1 = src0_q[q_base + ne01]; + regQ.s2 = src0_q[q_base + ne01 * 2]; + regQ.s3 = src0_q[q_base + ne01 * 3]; + + // Load activations: 32 floats = 8 float4s + uint y_offset = i11 * ne00 / 4 + ib * 8; + + float8 fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s0), scale, minv); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (y_offset + 0)); + float4 acc = shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 1)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s1), scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 2)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 3)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s2), scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 4)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 5)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s3), scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 6)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 7)); + acc += shared_y4 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_q4_k_f32_ns_wimg( + __read_only image1d_buffer_t src0_q, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + + int num_superblocks = ne00 / QK_K; + int num_subblocks = ne00 / 32; + int scales_per_row = num_superblocks * K_SCALE_SIZE; + + uint expert_q_offset = expert_id * (ne00 / 8) * ne01; + uint expert_d_offset = expert_id * num_superblocks * ne01; + + __private float sum = 0.0f; + + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; + uint j = ib % 8; + + half d_val = src0_d[expert_d_offset + sb * ne01 + i01]; + half dm_val = src0_dm[expert_d_offset + sb * ne01 + i01]; + + global const uchar * sc = src0_s + (expert_id * ne01 + i01) * scales_per_row + sb * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(j, sc, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = (float)dm_val * (float)mn; + + uint q_base = expert_q_offset + ib * ne01 * 4 + i01; + + uint4 regQ; + regQ.s0 = read_imageui(src0_q, (int)(q_base)).x; + regQ.s1 = read_imageui(src0_q, (int)(q_base + ne01)).x; + regQ.s2 = read_imageui(src0_q, (int)(q_base + ne01 * 2)).x; + regQ.s3 = read_imageui(src0_q, (int)(q_base + ne01 * 3)).x; + + uint y_offset = i11 * ne00 / 4 + ib * 8; + + float8 fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s0), scale, minv); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (y_offset + 0)); + float4 acc = shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 1)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s1), scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 2)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 3)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s2), scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 4)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 5)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s3), scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 6)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 7)); + acc += shared_y4 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl new file mode 100644 index 0000000..83b186b --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl @@ -0,0 +1,126 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_Q5_0 32 +#define N_SIMDGROUP 4 +#define SIMDGROUP_WIDTH 64 + +static inline float8 q5_0_to_fp32_packed8(ushort2 qs5x8, uchar qh5x8) { + float8 fp32x8; + fp32x8.s0 = (float)((( qs5x8.s0 & 0x000F) | (( qh5x8 & 0x01) << 4)) - 16); + fp32x8.s1 = (float)((((qs5x8.s0 & 0x00F0) >> 4 ) | (((qh5x8 >> 1) & 0x01) << 4)) - 16); + fp32x8.s2 = (float)((((qs5x8.s0 & 0x0F00) >> 8 ) | (((qh5x8 >> 2) & 0x01) << 4)) - 16); + fp32x8.s3 = (float)((((qs5x8.s0 & 0xF000) >> 12) | (((qh5x8 >> 3) & 0x01) << 4)) - 16); + fp32x8.s4 = (float)((( qs5x8.s1 & 0x000F) | (((qh5x8 >> 4) & 0x01) << 4)) - 16); + fp32x8.s5 = (float)((((qs5x8.s1 & 0x00F0) >> 4 ) | (((qh5x8 >> 5) & 0x01) << 4)) - 16); + fp32x8.s6 = (float)((((qs5x8.s1 & 0x0F00) >> 8 ) | (((qh5x8 >> 6) & 0x01) << 4)) - 16); + fp32x8.s7 = (float)((((qs5x8.s1 & 0xF000) >> 12) | (((qh5x8 >> 7) & 0x01) << 4)) - 16); + return fp32x8; +} + + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_q5_0_f32_ns( + __global uint * src0_qs, + __global uint * src0_qh, + __global half * src0_d, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + uint ne00, + uint ne01, + uint ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + uint expert_offset = expert_id * ne00 * ne01 / 32; + + __private float sum = 0.0f; // each thread calculate partial sum of one output + + // loop along ne00 in block granularity, skip 4 blocks every iter + for (uint ib00 = sgid; ib00 < (ne00 / QK_Q5_0); ib00 += N_SIMDGROUP) { + + // load one block of q + uint4 regQ; + uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01; + + regQ.s0 = src0_qs[block_offset]; + regQ.s1 = src0_qs[block_offset + ne01]; + regQ.s2 = src0_qs[block_offset + ne01 * 2]; + regQ.s3 = src0_qs[block_offset + ne01 * 3]; + + uint offset = i11 * ne00 / 4 + ib00 * 8; + + uchar4 regQh = as_uchar4(src0_qh[ib00 * ne01 + i01 + expert_offset]); + half regS = src0_d[ib00 * ne01 + i01 + expert_offset]; + + float8 fp32x8 = q5_0_to_fp32_packed8(as_ushort2(regQ.s0), regQh.s0); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (offset + 0)); + float4 acc = shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 1)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q5_0_to_fp32_packed8(as_ushort2(regQ.s1), regQh.s1); + + shared_y4 = read_imagef(src1, (offset + 2)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 3)); + acc += shared_y4 * fp32x8.hi; + + + fp32x8 = q5_0_to_fp32_packed8(as_ushort2(regQ.s2), regQh.s2); + + shared_y4 = read_imagef(src1, (offset + 4)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 5)); + acc += shared_y4 * fp32x8.hi; + + + fp32x8 = q5_0_to_fp32_packed8(as_ushort2(regQ.s3), regQh.s3); + + shared_y4 = read_imagef(src1, (offset + 6)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 7)); + acc += shared_y4 * fp32x8.hi; + + sum += (float)(regS) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + // 1 outputs per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } + +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl new file mode 100644 index 0000000..2eba34e --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl @@ -0,0 +1,128 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_Q5_1 32 +#define N_SIMDGROUP 4 +#define SIMDGROUP_WIDTH 64 + +static inline float8 q5_1_to_fp32_packed8(ushort2 qs5x8, uchar qh5x8, half s, half m) { + float8 fp32x8; + fp32x8.s0 = (float)((( qs5x8.s0 & 0x000F) | (( qh5x8 & 0x01) << 4)) * s + m); + fp32x8.s1 = (float)((((qs5x8.s0 & 0x00F0) >> 4 ) | (((qh5x8 >> 1) & 0x01) << 4)) * s + m); + fp32x8.s2 = (float)((((qs5x8.s0 & 0x0F00) >> 8 ) | (((qh5x8 >> 2) & 0x01) << 4)) * s + m); + fp32x8.s3 = (float)((((qs5x8.s0 & 0xF000) >> 12) | (((qh5x8 >> 3) & 0x01) << 4)) * s + m); + fp32x8.s4 = (float)((( qs5x8.s1 & 0x000F) | (((qh5x8 >> 4) & 0x01) << 4)) * s + m); + fp32x8.s5 = (float)((((qs5x8.s1 & 0x00F0) >> 4 ) | (((qh5x8 >> 5) & 0x01) << 4)) * s + m); + fp32x8.s6 = (float)((((qs5x8.s1 & 0x0F00) >> 8 ) | (((qh5x8 >> 6) & 0x01) << 4)) * s + m); + fp32x8.s7 = (float)((((qs5x8.s1 & 0xF000) >> 12) | (((qh5x8 >> 7) & 0x01) << 4)) * s + m); + return fp32x8; +} + + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_q5_1_f32_ns( + __global uint * src0_qs, + __global uint * src0_qh, + __global half * src0_d, + __global half * src0_m, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + uint ne00, + uint ne01, + uint ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + uint expert_offset = expert_id * ne00 * ne01 / 32; + + __private float sum = 0.0f; // each thread calculate partial sum of one output + + // loop along ne00 in block granularity, skip 4 blocks every iter + for (uint ib00 = sgid; ib00 < (ne00 / QK_Q5_1); ib00 += N_SIMDGROUP) { + + // load one block of q + uint4 regQ; + uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01; + + regQ.s0 = src0_qs[block_offset]; + regQ.s1 = src0_qs[block_offset + ne01]; + regQ.s2 = src0_qs[block_offset + ne01 * 2]; + regQ.s3 = src0_qs[block_offset + ne01 * 3]; + + uint offset = i11 * ne00 / 4 + ib00 * 8; + + uchar4 regQh = as_uchar4(src0_qh[ib00 * ne01 + i01 + expert_offset]); + half regM = src0_m[ib00 * ne01 + i01 + expert_offset]; + half regS = src0_d[ib00 * ne01 + i01 + expert_offset]; + + float8 fp32x8 = q5_1_to_fp32_packed8(as_ushort2(regQ.s0), regQh.s0, regS, regM); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (offset + 0)); + float4 acc = shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 1)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q5_1_to_fp32_packed8(as_ushort2(regQ.s1), regQh.s1, regS, regM); + + shared_y4 = read_imagef(src1, (offset + 2)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 3)); + acc += shared_y4 * fp32x8.hi; + + + fp32x8 = q5_1_to_fp32_packed8(as_ushort2(regQ.s2), regQh.s2, regS, regM); + + shared_y4 = read_imagef(src1, (offset + 4)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 5)); + acc += shared_y4 * fp32x8.hi; + + + fp32x8 = q5_1_to_fp32_packed8(as_ushort2(regQ.s3), regQh.s3, regS, regM); + + shared_y4 = read_imagef(src1, (offset + 6)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (offset + 7)); + acc += shared_y4 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + // 1 outputs per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } + +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl new file mode 100644 index 0000000..a959f72 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl @@ -0,0 +1,163 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define K_SCALE_SIZE 12 +#define N_SIMDGROUP 4 +#define SIMDGROUP_WIDTH 64 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j+4] & 63; + } else { + *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2); + *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2); + } +} + +static inline float8 q5_k_to_fp32_packed8(ushort2 qs5x8, uchar qh5x8, half s, half m) { + float8 fp32x8; + fp32x8.s0 = (float)((( qs5x8.s0 & 0x000F) | (( qh5x8 & 0x01) << 4)) * s + m); + fp32x8.s1 = (float)((((qs5x8.s0 & 0x00F0) >> 4 ) | (((qh5x8 >> 1) & 0x01) << 4)) * s + m); + fp32x8.s2 = (float)((((qs5x8.s0 & 0x0F00) >> 8 ) | (((qh5x8 >> 2) & 0x01) << 4)) * s + m); + fp32x8.s3 = (float)((((qs5x8.s0 & 0xF000) >> 12) | (((qh5x8 >> 3) & 0x01) << 4)) * s + m); + fp32x8.s4 = (float)((( qs5x8.s1 & 0x000F) | (((qh5x8 >> 4) & 0x01) << 4)) * s + m); + fp32x8.s5 = (float)((((qs5x8.s1 & 0x00F0) >> 4 ) | (((qh5x8 >> 5) & 0x01) << 4)) * s + m); + fp32x8.s6 = (float)((((qs5x8.s1 & 0x0F00) >> 8 ) | (((qh5x8 >> 6) & 0x01) << 4)) * s + m); + fp32x8.s7 = (float)((((qs5x8.s1 & 0xF000) >> 12) | (((qh5x8 >> 7) & 0x01) << 4)) * s + m); + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_q5_k_f32_ns( + __global uint * src0_q, + __global uint * src0_qh, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + + int num_superblocks = ne00 / QK_K; + int num_subblocks = ne00 / 32; + int scales_per_row = num_superblocks * K_SCALE_SIZE; + + // Expert offsets in the transposed noshuffle layout + uint expert_q_offset = expert_id * (ne00 / 8) * ne01; + uint expert_d_offset = expert_id * num_superblocks * ne01; + + __private float sum = 0.0f; + + // Loop over sub-blocks of 32 elements, N_SIMDGROUP sub-blocks per iter + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; + uint j = ib % 8; + + // Load d and dmin for this super-block + half d_val = src0_d[expert_d_offset + sb * ne01 + i01]; + half dm_val = src0_dm[expert_d_offset + sb * ne01 + i01]; + + // sub_block index = sb * 8 + j + uint expert_qh_offset = expert_id * num_superblocks * 8 * ne01; + uchar4 regQh = as_uchar4(src0_qh[expert_qh_offset + (sb * 8 + j) * ne01 + i01]); + + // Load sub-block scale and min + global const uchar * sc = src0_s + (expert_id * ne01 + i01) * scales_per_row + sb * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(j, sc, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = -(float)dm_val * (float)mn; + + // Load 4 uints of quants (32 nibbles = 32 elements) + uint q_base = expert_q_offset + ib * ne01 * 4 + i01; + + uint4 regQ; + regQ.s0 = src0_q[q_base]; + regQ.s1 = src0_q[q_base + ne01]; + regQ.s2 = src0_q[q_base + ne01 * 2]; + regQ.s3 = src0_q[q_base + ne01 * 3]; + + // Load activations: 32 floats = 8 float4s + uint y_offset = i11 * ne00 / 4 + ib * 8; + + float8 fp32x8 = q5_k_to_fp32_packed8(as_ushort2(regQ.s0), regQh.s0, scale, minv); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (y_offset + 0)); + float4 acc = shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 1)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q5_k_to_fp32_packed8(as_ushort2(regQ.s1), regQh.s1, scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 2)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 3)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q5_k_to_fp32_packed8(as_ushort2(regQ.s2), regQh.s2, scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 4)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 5)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q5_k_to_fp32_packed8(as_ushort2(regQ.s3), regQh.s3, scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 6)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 7)); + acc += shared_y4 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl new file mode 100644 index 0000000..dcd34fb --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl @@ -0,0 +1,144 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define N_SIMDGROUP 4 +#define SIMDGROUP_WIDTH 64 + +static inline float8 q6_k_to_fp32_packed8(ushort2 ql8, ushort qh8, float d_scale) { + float8 fp32x8; + fp32x8.s0 = ((float)(( ql8.s0 & 0x000F) | ((uint)((qh8 ) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s1 = ((float)((( ql8.s0 >> 4) & 0x000F) | ((uint)((qh8 >> 2) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s2 = ((float)((( ql8.s0 >> 8) & 0x000F) | ((uint)((qh8 >> 4) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s3 = ((float)((( ql8.s0 >> 12)& 0x000F) | ((uint)((qh8 >> 6) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s4 = ((float)(( ql8.s1 & 0x000F) | ((uint)((qh8 >> 8) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s5 = ((float)((( ql8.s1 >> 4) & 0x000F) | ((uint)((qh8 >>10) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s6 = ((float)((( ql8.s1 >> 8) & 0x000F) | ((uint)((qh8 >>12) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s7 = ((float)((( ql8.s1 >> 12)& 0x000F) | ((uint)((qh8 >>14) & 0x3) << 4)) - 32.f) * d_scale; + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_q6_k_f32_ns( + __global uint * src0_ql, + __global uint * src0_qh, + __global char * src0_s, + __global half * src0_d, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } + + int num_superblocks = ne00 / QK_K; + int num_subblocks = ne00 / 32; // 8 sub-blocks of 32 per super-block + int scales_per_row = num_superblocks * 16; + + // Expert offsets in the transposed noshuffle layout + uint expert_ql_offset = expert_id * (ne00 / 8) * ne01; // 32 uints per super-block + uint expert_qh_offset = expert_id * (ne00 / 16) * ne01; // 16 uints per super-block + uint expert_d_offset = expert_id * num_superblocks * ne01; + + __private float sum = 0.0f; + + // Loop over sub-blocks of 32 elements, N_SIMDGROUP sub-blocks per iter + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; // super-block index + uint j = ib % 8; // 32-element group within super-block + + // Load d for this super-block + half d_val = src0_d[expert_d_offset + sb * ne01 + i01]; + + // Load 2 sub-block scales + global const char * sc = src0_s + (expert_id * ne01 + i01) * scales_per_row + sb * 16; + float scale0 = (float)d_val * (float)sc[j * 2]; + float scale1 = (float)d_val * (float)sc[j * 2 + 1]; + + // Load 4 uints of ql + uint ql_base = expert_ql_offset + (ib * 4) * ne01 + i01; + uint4 regQL; + regQL.s0 = src0_ql[ql_base]; + regQL.s1 = src0_ql[ql_base + ne01]; + regQL.s2 = src0_ql[ql_base + ne01 * 2]; + regQL.s3 = src0_ql[ql_base + ne01 * 3]; + + // Load 2 uints of qh + uint qh_base = expert_qh_offset + (ib * 2) * ne01 + i01; + uint2 regQH; + regQH.s0 = src0_qh[qh_base]; + regQH.s1 = src0_qh[qh_base + ne01]; + + // Load activations: 32 floats = 8 float4s + uint y_offset = i11 * ne00 / 4 + ib * 8; + + float8 fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s0), (ushort)(regQH.s0 & 0xFFFF), scale0); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (y_offset + 0)); + float4 acc = shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 1)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s1), (ushort)(regQH.s0 >> 16), scale0); + + shared_y4 = read_imagef(src1, (y_offset + 2)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 3)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s2), (ushort)(regQH.s1 & 0xFFFF), scale1); + + shared_y4 = read_imagef(src1, (y_offset + 4)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 5)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s3), (ushort)(regQH.s1 >> 16), scale1); + + shared_y4 = read_imagef(src1, (y_offset + 6)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 7)); + acc += shared_y4 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl new file mode 100644 index 0000000..e17ca09 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl @@ -0,0 +1,309 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK4_NL 32 +#define NSUBGROUPS 4 +#define SUBGROUP_SIZE 64 + +constant half kvalues_iq4nl[16] = { + (half)-127.f, (half)-104.f, (half)-83.f, (half)-65.f, + (half) -49.f, (half) -35.f, (half)-22.f, (half)-10.f, + (half) 1.f, (half) 13.f, (half) 25.f, (half) 38.f, + (half) 53.f, (half) 69.f, (half) 89.f, (half)113.f +}; + +// Packed LUT: 2 FP16 values per uint, 8 unique constant loads instead of 16 +constant uint iq4nl_packed[8] = { + 0xD680D7F0u, // idx 0,1: -127, -104 + 0xD410D530u, // idx 2,3: -83, -65 + 0xD060D220u, // idx 4,5: -49, -35 + 0xC900CD80u, // idx 6,7: -22, -10 + 0x4A803C00u, // idx 8,9: 1, 13 + 0x50C04E40u, // idx 10,11: 25, 38 + 0x545052A0u, // idx 12,13: 53, 69 + 0x57105590u // idx 14,15: 89, 113 +}; + +// Packed dequant: 1 uint constant load (8-way divergence) + shift + as_half +#define IQ4_NL_DEQUANT(nibble) as_half((ushort)(iq4nl_packed[(nibble) >> 1] >> (((nibble) & 1u) << 4))) + +#define dequantizeBlockAccum_ns_sgbroadcast_1_hi(total_sums, bits4, scale, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s0 & 0x000F)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s1 & 0x000F)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0x00F0) >> 4)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0x00F0) >> 4)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0x0F00) >> 8)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0x0F00) >> 8)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0xF000) >> 12)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0xF000) >> 12)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s2 & 0x000F)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s3 & 0x000F)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0x00F0) >> 4)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0x00F0) >> 4)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0x0F00) >> 8)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0x0F00) >> 8)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0xF000) >> 12)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0xF000) >> 12)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s4 & 0x000F)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s5 & 0x000F)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0x00F0) >> 4)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0x00F0) >> 4)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0x0F00) >> 8)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0x0F00) >> 8)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0xF000) >> 12)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0xF000) >> 12)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s6 & 0x000F)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s7 & 0x000F)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0x00F0) >> 4)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0x00F0) >> 4)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0x0F00) >> 8)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0x0F00) >> 8)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0xF000) >> 12)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0xF000) >> 12)) * scale.s1 * shared_y; \ + + +#define dequantizeBlockAccum_ns_sgbroadcast_1_lo(total_sums, bits4, scale, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s0 & 0x000F)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s1 & 0x000F)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0x00F0) >> 4)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0x00F0) >> 4)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0x0F00) >> 8)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0x0F00) >> 8)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0xF000) >> 12)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0xF000) >> 12)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s2 & 0x000F)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s3 & 0x000F)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0x00F0) >> 4)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0x00F0) >> 4)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0x0F00) >> 8)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0x0F00) >> 8)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0xF000) >> 12)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0xF000) >> 12)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s4 & 0x000F)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s5 & 0x000F)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0x00F0) >> 4)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0x00F0) >> 4)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0x0F00) >> 8)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0x0F00) >> 8)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0xF000) >> 12)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0xF000) >> 12)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s6 & 0x000F)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s7 & 0x000F)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0x00F0) >> 4)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0x00F0) >> 4)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0x0F00) >> 8)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0x0F00) >> 8)) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0xF000) >> 12)) * scale.s0 * shared_y; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0xF000) >> 12)) * scale.s1 * shared_y; \ + + +#define dequantizeBlockAccum_ns_sgbroadcast_8_hi(total_sums, bits4, scale, y) \ + float8 shared_y; \ + shared_y = sub_group_broadcast(y, 0); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s0 & 0x000F)) * scale.s0 * shared_y.s0; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0x00F0) >> 4)) * scale.s0 * shared_y.s1; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0x0F00) >> 8)) * scale.s0 * shared_y.s2; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0xF000) >> 12)) * scale.s0 * shared_y.s3; \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s2 & 0x000F)) * scale.s0 * shared_y.s4; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0x00F0) >> 4)) * scale.s0 * shared_y.s5; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0x0F00) >> 8)) * scale.s0 * shared_y.s6; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0xF000) >> 12)) * scale.s0 * shared_y.s7; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s1 & 0x000F)) * scale.s1 * shared_y.s0; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0x00F0) >> 4)) * scale.s1 * shared_y.s1; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0x0F00) >> 8)) * scale.s1 * shared_y.s2; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0xF000) >> 12)) * scale.s1 * shared_y.s3; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s3 & 0x000F)) * scale.s1 * shared_y.s4; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0x00F0) >> 4)) * scale.s1 * shared_y.s5; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0x0F00) >> 8)) * scale.s1 * shared_y.s6; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0xF000) >> 12)) * scale.s1 * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 1); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s4 & 0x000F)) * scale.s0 * shared_y.s0; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0x00F0) >> 4)) * scale.s0 * shared_y.s1; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0x0F00) >> 8)) * scale.s0 * shared_y.s2; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0xF000) >> 12)) * scale.s0 * shared_y.s3; \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s6 & 0x000F)) * scale.s0 * shared_y.s4; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0x00F0) >> 4)) * scale.s0 * shared_y.s5; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0x0F00) >> 8)) * scale.s0 * shared_y.s6; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0xF000) >> 12)) * scale.s0 * shared_y.s7; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s5 & 0x000F)) * scale.s1 * shared_y.s0; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0x00F0) >> 4)) * scale.s1 * shared_y.s1; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0x0F00) >> 8)) * scale.s1 * shared_y.s2; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0xF000) >> 12)) * scale.s1 * shared_y.s3; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s7 & 0x000F)) * scale.s1 * shared_y.s4; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0x00F0) >> 4)) * scale.s1 * shared_y.s5; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0x0F00) >> 8)) * scale.s1 * shared_y.s6; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0xF000) >> 12)) * scale.s1 * shared_y.s7; \ + + +#define dequantizeBlockAccum_ns_sgbroadcast_8_lo(total_sums, bits4, scale, y) \ + shared_y = sub_group_broadcast(y, 2); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s0 & 0x000F)) * scale.s0 * shared_y.s0; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0x00F0) >> 4)) * scale.s0 * shared_y.s1; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0x0F00) >> 8)) * scale.s0 * shared_y.s2; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s0 & 0xF000) >> 12)) * scale.s0 * shared_y.s3; \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s2 & 0x000F)) * scale.s0 * shared_y.s4; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0x00F0) >> 4)) * scale.s0 * shared_y.s5; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0x0F00) >> 8)) * scale.s0 * shared_y.s6; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s2 & 0xF000) >> 12)) * scale.s0 * shared_y.s7; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s1 & 0x000F)) * scale.s1 * shared_y.s0; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0x00F0) >> 4)) * scale.s1 * shared_y.s1; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0x0F00) >> 8)) * scale.s1 * shared_y.s2; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s1 & 0xF000) >> 12)) * scale.s1 * shared_y.s3; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s3 & 0x000F)) * scale.s1 * shared_y.s4; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0x00F0) >> 4)) * scale.s1 * shared_y.s5; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0x0F00) >> 8)) * scale.s1 * shared_y.s6; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s3 & 0xF000) >> 12)) * scale.s1 * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 3); \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s4 & 0x000F)) * scale.s0 * shared_y.s0; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0x00F0) >> 4)) * scale.s0 * shared_y.s1; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0x0F00) >> 8)) * scale.s0 * shared_y.s2; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s4 & 0xF000) >> 12)) * scale.s0 * shared_y.s3; \ + total_sums.s0 += IQ4_NL_DEQUANT((bits4.s6 & 0x000F)) * scale.s0 * shared_y.s4; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0x00F0) >> 4)) * scale.s0 * shared_y.s5; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0x0F00) >> 8)) * scale.s0 * shared_y.s6; \ + total_sums.s0 += IQ4_NL_DEQUANT(((bits4.s6 & 0xF000) >> 12)) * scale.s0 * shared_y.s7; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s5 & 0x000F)) * scale.s1 * shared_y.s0; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0x00F0) >> 4)) * scale.s1 * shared_y.s1; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0x0F00) >> 8)) * scale.s1 * shared_y.s2; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s5 & 0xF000) >> 12)) * scale.s1 * shared_y.s3; \ + total_sums.s1 += IQ4_NL_DEQUANT((bits4.s7 & 0x000F)) * scale.s1 * shared_y.s4; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0x00F0) >> 4)) * scale.s1 * shared_y.s5; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0x0F00) >> 8)) * scale.s1 * shared_y.s6; \ + total_sums.s1 += IQ4_NL_DEQUANT(((bits4.s7 & 0xF000) >> 12)) * scale.s1 * shared_y.s7; \ + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_iq4_nl_f32( + read_only image1d_buffer_t src0_q, + global half2 * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + + private uint4 regA; + private half2 regS; + private float8 regB; + + private float2 totalSum = (float2)(0.0f); + + // loop along K in block granularity, skip 4 blocks every iter + for (uint k = groupId; k < (K / QK4_NL); k += NSUBGROUPS) { + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows + // first 4 fibers in each wave load 8 B values to its private scope + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + + // load half weights for two blocks in consecutive rows + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB); +#endif // VECTOR_SUB_GROUP_BROADCAST + + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regB); +#endif // VECTOR_SUB_GROUP_BROADCAST + } + + // reduction in local memory, assumes #wave=4 + local float2 reduceLM[SUBGROUP_SIZE * 3]; + if (groupId == 1) { + reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum; + } + if (groupId == 2) { + reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum; + } + if (groupId == 3) { + reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid]; + } + if (groupId == 0) { + totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid]; + } + if (groupId == 0) { + totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid]; + } + + // 2 outputs per fiber in wave 0 + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; + } + +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl new file mode 100644 index 0000000..572f116 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl @@ -0,0 +1,135 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK1_0 128 +#ifndef N_SIMDGROUP +#define N_SIMDGROUP 4 +#endif + +#define dequantizeBlockAccum_q1(total, bits, scale, regB, lb) \ + total += (2.0f*(float)((bits >> 0) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+0); \ + total += (2.0f*(float)((bits >> 1) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+0); \ + total += (2.0f*(float)((bits >> 2) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+0); \ + total += (2.0f*(float)((bits >> 3) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+0); \ + total += (2.0f*(float)((bits >> 4) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+0); \ + total += (2.0f*(float)((bits >> 5) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+0); \ + total += (2.0f*(float)((bits >> 6) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+0); \ + total += (2.0f*(float)((bits >> 7) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+0); \ + total += (2.0f*(float)((bits >> 8) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+1); \ + total += (2.0f*(float)((bits >> 9) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+1); \ + total += (2.0f*(float)((bits >> 10) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+1); \ + total += (2.0f*(float)((bits >> 11) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+1); \ + total += (2.0f*(float)((bits >> 12) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+1); \ + total += (2.0f*(float)((bits >> 13) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+1); \ + total += (2.0f*(float)((bits >> 14) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+1); \ + total += (2.0f*(float)((bits >> 15) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+1); \ + total += (2.0f*(float)((bits >> 16) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+2); \ + total += (2.0f*(float)((bits >> 17) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+2); \ + total += (2.0f*(float)((bits >> 18) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+2); \ + total += (2.0f*(float)((bits >> 19) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+2); \ + total += (2.0f*(float)((bits >> 20) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+2); \ + total += (2.0f*(float)((bits >> 21) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+2); \ + total += (2.0f*(float)((bits >> 22) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+2); \ + total += (2.0f*(float)((bits >> 23) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+2); \ + total += (2.0f*(float)((bits >> 24) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+3); \ + total += (2.0f*(float)((bits >> 25) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+3); \ + total += (2.0f*(float)((bits >> 26) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+3); \ + total += (2.0f*(float)((bits >> 27) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+3); \ + total += (2.0f*(float)((bits >> 28) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+3); \ + total += (2.0f*(float)((bits >> 29) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+3); \ + total += (2.0f*(float)((bits >> 30) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+3); \ + total += (2.0f*(float)((bits >> 31) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+3); + + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q1_0_f32( + read_only image1d_buffer_t src0_q, + global half * src0_d, + read_only image1d_buffer_t src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = 4 * M; + + uint4 regA; + half regS; + float8 regB; + + float totalSum = 0.0f; + + #pragma unroll 1 + for (uint kb = groupId; kb < (K / QK1_0); kb += N_SIMDGROUP) { + regS = src0_d[gid_s + kb * LINE_STRIDE_A]; // each fiber loads its row's scale + + // first 16 fibers load 8 B values each -> 128 activations for this block + if (slid < 16) { + regB.s0123 = read_imagef(src1, (slid * 2 + kb * 32)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + kb * 32)); + } + + // load this row's 4 uint32 (128 sign bits) + regA.s0 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + + float scale = (float)regS; + dequantizeBlockAccum_q1(totalSum, regA.s0, scale, regB, 0); + dequantizeBlockAccum_q1(totalSum, regA.s1, scale, regB, 4); + dequantizeBlockAccum_q1(totalSum, regA.s2, scale, regB, 8); + dequantizeBlockAccum_q1(totalSum, regA.s3, scale, regB, 12); + } + +#if N_SIMDGROUP > 1 + // Sum the K slices from the other subgroups. + local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (groupId > 0) { + reduceLM[SIMDGROUP_WIDTH * (groupId - 1) + slid] = totalSum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) { + for (uint w = 0; w < N_SIMDGROUP - 1; ++w) { + totalSum += reduceLM[SIMDGROUP_WIDTH * w + slid]; + } + } +#endif + + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + // Guard the output row. The x-grid is padded to CEIL_DIV(M,wavesize)*wavesize, + // so when ne01 is not a multiple of the wave size the tail work-items run past + // row ne01 and would overrun dst into the adjacent tensor. No-op / byte-identical + // when ne01 is wave-aligned (no padding). + if (gid < M) { + dst[gid] = totalSum; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl new file mode 100644 index 0000000..efcc310 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl @@ -0,0 +1,231 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef Q2_K_MV_NSG +#define Q2_K_MV_NSG 8 +#endif + +#ifndef Q2_K_MV_R +#define Q2_K_MV_R 4 +#endif + +inline float4 q2_k_vals(uint pk) { + return (float4)((float)( pk & 3u), (float)((pk >> 2) & 3u), + (float)((pk >> 4) & 3u), (float)((pk >> 6) & 3u)); +} + +kernel void kernel_gemv_noshuffle_q2_k_f32( + global const uchar * src0_qs, + global const uchar * src0_sc, + global const half * src0_dm, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, // K + int ne01, // M + int ne10, // activation row stride, == K + int ne0 // dst row stride +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + + const uint mr = m / Q2_K_MV_R; // row groups + const uint j = get_group_id(0) * 64u + lid; // row group this lane owns + const uint row = j * Q2_K_MV_R; + +#if Q2_K_MV_R == 4 + float sumf0 = 0.f; + float sumf1 = 0.f; + float sumf2 = 0.f; + float sumf3 = 0.f; + + if (j < mr) { + global const uint * qsu = (global const uint *)src0_qs; + global const uint * scu = (global const uint *)src0_sc; + + for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) { + const half4 dma = vload4(2u*(j + ib*mr) + 0u, src0_dm); // d0, dmin0, d1, dmin1 + const half4 dmb = vload4(2u*(j + ib*mr) + 1u, src0_dm); // d2, dmin2, d3, dmin3 + + float ad0 = 0.f, am0 = 0.f, ad1 = 0.f, am1 = 0.f; + float ad2 = 0.f, am2 = 0.f, ad3 = 0.f, am3 = 0.f; + + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mr; + + for (uint h = 0; h < 2u; ++h) { + const uint scv = scu[j + (2u * (ib * 8u + sb) + h) * mr]; + + float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f; + float4 as = (float4)(0.f); + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = qsu[qsb + gg * mr]; // four rows, one load + const float4 yv = vload4(grp + gg, y); + as += yv; + a0 += dot(yv, q2_k_vals( qsv & 0xFFu)); + a1 += dot(yv, q2_k_vals((qsv >> 8) & 0xFFu)); + a2 += dot(yv, q2_k_vals((qsv >> 16) & 0xFFu)); + a3 += dot(yv, q2_k_vals((qsv >> 24) & 0xFFu)); + } + const float asum = as.s0 + as.s1 + as.s2 + as.s3; + + const uint s0 = scv & 0xFFu; + const uint s1 = (scv >> 8) & 0xFFu; + const uint s2 = (scv >> 16) & 0xFFu; + const uint s3 = (scv >> 24) & 0xFFu; + ad0 += (float)(s0 & 0xFu) * a0; am0 += (float)(s0 >> 4) * asum; + ad1 += (float)(s1 & 0xFu) * a1; am1 += (float)(s1 >> 4) * asum; + ad2 += (float)(s2 & 0xFu) * a2; am2 += (float)(s2 >> 4) * asum; + ad3 += (float)(s3 & 0xFu) * a3; am3 += (float)(s3 >> 4) * asum; + } + } + sumf0 += (float)dma.s0 * ad0 - (float)dma.s1 * am0; + sumf1 += (float)dma.s2 * ad1 - (float)dma.s3 * am1; + sumf2 += (float)dmb.s0 * ad2 - (float)dmb.s1 * am2; + sumf3 += (float)dmb.s2 * ad3 - (float)dmb.s3 * am3; + } + } + +#if Q2_K_MV_NSG > 1 + __local float4 part[Q2_K_MV_NSG][64]; + part[sgi][lid] = (float4)(sumf0, sumf1, sumf2, sumf3); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q2_K_MV_NSG; ++s) { + const float4 p = part[s][lid]; + sumf0 += p.s0; + sumf1 += p.s1; + sumf2 += p.s2; + sumf3 += p.s3; + } +#endif + + if (j < mr) { + global float * o = dst + (ulong)col * (uint)ne0 + row; + o[0] = sumf0; o[1] = sumf1; o[2] = sumf2; o[3] = sumf3; + } + +#elif Q2_K_MV_R == 2 + float sumf0 = 0.f; + float sumf1 = 0.f; + + if (j < mr) { + global const ushort * qsu = (global const ushort *)src0_qs; + global const ushort * scu = (global const ushort *)src0_sc; + + for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) { + const half4 dm = vload4(j + ib * mr, src0_dm); // d0, dmin0, d1, dmin1 + + float ad0 = 0.f, am0 = 0.f, ad1 = 0.f, am1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mr; + + for (uint h = 0; h < 2u; ++h) { + const uint scv = (uint)scu[j + (2u * (ib * 8u + sb) + h) * mr]; + const uint s0 = scv & 0xFFu; + const uint s1 = scv >> 8; + + float a0 = 0.f, a1 = 0.f; + float4 as = (float4)(0.f); + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = (uint)qsu[qsb + gg * mr]; + const float4 yv = vload4(grp + gg, y); + as += yv; + a0 += dot(yv, q2_k_vals( qsv & 0xFFu)); + a1 += dot(yv, q2_k_vals((qsv >> 8) & 0xFFu)); + } + const float asum = as.s0 + as.s1 + as.s2 + as.s3; + ad0 += (float)(s0 & 0xFu) * a0; am0 += (float)(s0 >> 4) * asum; + ad1 += (float)(s1 & 0xFu) * a1; am1 += (float)(s1 >> 4) * asum; + } + } + sumf0 += (float)dm.s0 * ad0 - (float)dm.s1 * am0; + sumf1 += (float)dm.s2 * ad1 - (float)dm.s3 * am1; + } + } + +#if Q2_K_MV_NSG > 1 + __local float2 part[Q2_K_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf0, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q2_K_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf0 += p.s0; + sumf1 += p.s1; + } +#endif + + if (j < mr) { + vstore2((float2)(sumf0, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } + +#else + float sumf0 = 0.f; + + if (j < mr) { + for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) { + const half2 dm = vload2(row + ib * m, src0_dm); + + float ad = 0.f, am = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = row + grp * m; + + for (uint h = 0; h < 2u; ++h) { + const uint sc = (uint)src0_sc[row + (2u * (ib * 8u + sb) + h) * m]; + + float a = 0.f; + float4 as = (float4)(0.f); + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const float4 yv = vload4(grp + gg, y); + as += yv; + const uint pkv = (uint)src0_qs[qsb + gg * m]; + a += dot(yv, q2_k_vals(pkv)); + } + const float asum = as.s0 + as.s1 + as.s2 + as.s3; + ad += (float)(sc & 0xFu) * a; am += (float)(sc >> 4) * asum; + } + } + sumf0 += (float)dm.s0 * ad - (float)dm.s1 * am; + } + } + +#if Q2_K_MV_NSG > 1 + __local float part[Q2_K_MV_NSG][64]; + part[sgi][lid] = sumf0; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q2_K_MV_NSG; ++s) { + sumf0 += part[s][lid]; + } +#endif + + if (j < mr) { + dst[(ulong)col * (uint)ne0 + row] = sumf0; + } +#endif +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl similarity index 67% rename from ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl rename to ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl index 469d3ed..6066ae3 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl @@ -7,6 +7,14 @@ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) #endif +// A6X compiler incorrectly constant-folds get_local_size() results; +// force runtime materialization via a no-op ALU round-trip. +#ifdef GGML_CL_A6X_CONSTFOLD_FIX +#define MATERIALIZE_WG(x) do { (x) *= 2u; if ((x) > 1u) (x) /= 2u; } while(0) +#else +#define MATERIALIZE_WG(x) +#endif + // assume #define QK4_0 32 #define N_SIMDGROUP 4 @@ -191,7 +199,7 @@ #ifdef ADRENO_GPU REQD_SUBGROUP_SIZE_64 #endif -__kernel void kernel_gemv_noshuffle( +__kernel void kernel_gemv_noshuffle_q4_0_f32( __read_only image1d_buffer_t src0_q, // quantized A global half2 * src0_d, // A scales __read_only image1d_buffer_t src1, // B @@ -216,6 +224,8 @@ __kernel void kernel_gemv_noshuffle( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = N_SIMDGROUP * M; __private uint4 regA; @@ -226,7 +236,7 @@ __kernel void kernel_gemv_noshuffle( // loop along K in block granularity, skip 4 blocks every iter for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) { - regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows // first 4 fibers in each wave load 8 B values to its private scope if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); @@ -234,25 +244,25 @@ __kernel void kernel_gemv_noshuffle( } // load half weights for two blocks in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; -#ifdef VECTOR_SUB_GROUP_BROADCAT + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB); -#endif // VECTOR_SUB_GROUP_BROADCAT +#endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; -#ifdef VECTOR_SUB_GROUP_BROADCAT + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regB); -#endif // VECTOR_SUB_GROUP_BROADCAT +#endif // VECTOR_SUB_GROUP_BROADCAST } // reduction in local memory, assumes #wave=4 @@ -268,7 +278,121 @@ __kernel void kernel_gemv_noshuffle( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; + } + +} + +// Multi-column (N in [2..4]) variant of the q4_0 decode GEMV, for the speculative +// / MTP verify batch (n_cols = 2..4 = drafted + bonus positions). Routes the small- +// batch verify OFF the transposed-GEMM dead-zone (gemm_noshuffle_q4_0) onto the +// efficient GEMV path. Each K-block's weights (regA hi+lo) are loaded ONCE and +// reused across the n_cols activation columns. Per-column accumulation is +// independent and identical to n_cols standalone GEMVs. n_cols==3 is byte-identical +// to the original mc3 (col3 disabled, slots 6/7 stay zero). Kept the _mc3 name. +#ifdef VECTOR_SUB_GROUP_BROADCAST +#define MC_DQ_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi +#define MC_DQ_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo +#else +#define MC_DQ_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi +#define MC_DQ_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo +#endif +// One column c: load this column's activation (own brace scope so the macros' +// `shared_y` decl is re-scoped), then dequant (hi+lo) against the shared weights. +#define MC_COL_Q40(ts, c) \ + { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \ + regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \ + MC_DQ_HI(ts, as_ushort8(regA_hi), regS, regB); \ + MC_DQ_LO(ts, as_ushort8(regA_lo), regS, regB); } + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q4_0_f32_mc3( + __read_only image1d_buffer_t src0_q, // quantized A + global half2 * src0_d, // A scales + __read_only image1d_buffer_t src1, // B (n_cols columns, col-major image) + global float * dst, // C (column-major [M x n_cols]) + ulong offsetd, + int ne00, // K + int ne01, // M + int n_cols) // N (2..4) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + // BLOCK_STRIDE_A is the LAYOUT stride between consecutive K-blocks = 4 uints + // per q4_0 block * M (set by the trans4_ns convert). The "4" is uints/block, NOT + // the subgroup count — keep it fixed so the K-split count (nsg) can vary. + uint BLOCK_STRIDE_A = N_SIMDGROUP * M; // = 4 * M (N_SIMDGROUP is the #define 4) + uint COL_STRIDE = K / 4; // float4 pixels per activation column + uint nsg = get_local_size(1); // runtime K-split (4 default, 8 small-M) + MATERIALIZE_WG(nsg); + + __private uint4 regA_hi, regA_lo; + __private half2 regS; + __private float8 regB; + + __private float2 ts0 = (float2)(0.0f); + __private float2 ts1 = (float2)(0.0f); + __private float2 ts2 = (float2)(0.0f); + __private float2 ts3 = (float2)(0.0f); + + for (uint k = groupId; k < (K / QK4_0); k += nsg) { + regS = src0_d[gid_s + k * LINE_STRIDE_A]; + + // weights loaded ONCE, reused across the columns + regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + MC_COL_Q40(ts0, 0); + MC_COL_Q40(ts1, 1); + if (n_cols > 2) MC_COL_Q40(ts2, 2); + if (n_cols > 3) MC_COL_Q40(ts3, 3); } + // cross-subgroup reduce over nsg subgroups: pack the (up to 4) columns' float2 + // into a float8. Generalized to runtime nsg (4 default, 8 for small-M). Each + // subgroup writes its partial; subgroup 0 sums the rest into its own acc. At + // nsg==4 this is byte-identical to the original (sums subgroups 1,2,3 in order). + __local float8 reduceLM[SIMDGROUP_WIDTH * 8]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1); + reduceLM[groupId * SIMDGROUP_WIDTH + slid] = acc; + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + for (uint g = 1; g < nsg; g++) { + acc += reduceLM[g * SIMDGROUP_WIDTH + slid]; + } + dst = (global float*)((global char*)dst + offsetd); + // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row + if (gid < LINE_STRIDE_A) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + } + } } +#undef MC_COL_Q40 +#undef MC_DQ_HI +#undef MC_DQ_LO diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl new file mode 100644 index 0000000..565285f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_32b_trans.cl @@ -0,0 +1,137 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK4_0 32 +#define N_SIMDGROUP 4 + +#define dequantizeBlockAccum_ila_1row_hi(total_sum, bits4, scale, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sum += ((bits4.s0 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sum += (((bits4.s0 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sum += (((bits4.s0 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sum += (((bits4.s0 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sum += ((bits4.s1 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sum += (((bits4.s1 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sum += (((bits4.s1 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sum += (((bits4.s1 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sum += ((bits4.s2 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sum += (((bits4.s2 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sum += (((bits4.s2 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sum += (((bits4.s2 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sum += ((bits4.s3 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sum += (((bits4.s3 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sum += (((bits4.s3 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sum += (((bits4.s3 & 0xF000) >> 12) - 8) * scale * shared_y; + +#define dequantizeBlockAccum_ila_1row_lo(total_sum, bits4, scale, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sum += ((bits4.s4 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sum += (((bits4.s4 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sum += (((bits4.s4 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sum += (((bits4.s4 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sum += ((bits4.s5 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sum += (((bits4.s5 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sum += (((bits4.s5 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sum += (((bits4.s5 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sum += ((bits4.s6 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sum += (((bits4.s6 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sum += (((bits4.s6 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sum += (((bits4.s6 & 0xF000) >> 12) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sum += ((bits4.s7 & 0x000F) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sum += (((bits4.s7 & 0x00F0) >> 4) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sum += (((bits4.s7 & 0x0F00) >> 8) - 8) * scale * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sum += (((bits4.s7 & 0xF000) >> 12) - 8) * scale * shared_y; + + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q4_0_f32_32b_trans( + __read_only image1d_buffer_t src0_q, + global half * src0_d, + __read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + __private uint4 regA; + __private half regS; + __private float8 regB; + __private float totalSum = 0.0f; + + for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) { + regS = src0_d[k * M + gid]; + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + regA.s0 = read_imageui(src0_q, ((k * 4 + 0) * M + gid)).x; + regA.s1 = read_imageui(src0_q, ((k * 4 + 1) * M + gid)).x; + regA.s2 = read_imageui(src0_q, ((k * 4 + 2) * M + gid)).x; + regA.s3 = read_imageui(src0_q, ((k * 4 + 3) * M + gid)).x; + + dequantizeBlockAccum_ila_1row_hi(totalSum, as_ushort8(regA), regS, regB); + dequantizeBlockAccum_ila_1row_lo(totalSum, as_ushort8(regA), regS, regB); + } + + __local float reduceLM[SIMDGROUP_WIDTH * 3]; + if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum; + if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum; + if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum; + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + if (gid < M) { + dst[gid] = totalSum; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl similarity index 90% rename from ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl rename to ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl index ee5c79f..63990c4 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl @@ -191,7 +191,7 @@ #ifdef ADRENO_GPU REQD_SUBGROUP_SIZE_64 #endif -__kernel void kernel_gemv_noshuffle( +__kernel void kernel_gemv_noshuffle_q4_0_f32( __read_only image1d_buffer_t src0_q, // quantized A global half2 * src0_d, // A scales __read_only image1d_buffer_t src1, // B @@ -210,6 +210,8 @@ __kernel void kernel_gemv_noshuffle( { uint groupId = get_local_id(1); uint gid = get_global_id(0); + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, (uint)LINE_STRIDE_A - 1u); ushort slid = get_sub_group_local_id(); __private uint4 regA; @@ -220,7 +222,7 @@ __kernel void kernel_gemv_noshuffle( // loop along K in block granularity, skip 4 blocks every iter for (uint k = groupId; k < (K / QK4_0); k += N_SIMDGROUP) { - regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows // first 4 fibers in each wave load 8 B values to its private scope if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); @@ -228,25 +230,25 @@ __kernel void kernel_gemv_noshuffle( } // load half weights for two blocks in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; -#ifdef VECTOR_SUB_GROUP_BROADCAT + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regB); -#endif // VECTOR_SUB_GROUP_BROADCAT +#endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; -#ifdef VECTOR_SUB_GROUP_BROADCAT + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regB); -#endif // VECTOR_SUB_GROUP_BROADCAT +#endif // VECTOR_SUB_GROUP_BROADCAST } // reduction in local memory, assumes #wave=4 @@ -262,7 +264,11 @@ __kernel void kernel_gemv_noshuffle( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows against the padded x-grid tail overrunning dst. + // The current shape specializations are all ne01 % 128 == 0 (no padding), so + // this is a no-op / byte-identical today; keep it in lockstep with the base kernel. + if (gid * 2 + 0 < ne01) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = totalSum.s1; } } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl index fdc1472..b0b4793 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl @@ -209,6 +209,8 @@ kernel void kernel_gemv_noshuffle_q4_1_f32( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; private uint4 regA; @@ -220,8 +222,8 @@ kernel void kernel_gemv_noshuffle_q4_1_f32( // loop along K in block granularity, skip 4 blocks every iter for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) { - regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of two rows - regM = src0_m[gid + k * LINE_STRIDE_A]; // each fiber loads min of two rows + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads scale of two rows + regM = src0_m[gid_s + k * LINE_STRIDE_A]; // each fiber loads min of two rows // first 4 fibers in each wave load 8 B values to its private scope if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); @@ -229,20 +231,20 @@ kernel void kernel_gemv_noshuffle_q4_1_f32( } // load half weights for two blocks in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAT dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB); #endif // VECTOR_SUB_GROUP_BROADCAT - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAT dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB); #else @@ -277,7 +279,112 @@ kernel void kernel_gemv_noshuffle_q4_1_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } + +// Multi-column (N in [2..4]) variant of the q4_1 decode GEMV (spec/MTP verify) = +// q4_0 mc3 + the q4_1 per-block min (regM; dequant = q*scale + minv). n_cols=2..4; +// routes the small-batch verify OFF the gemm_noshuffle_q4_1 dead-zone. n_cols==3 is +// byte-identical to the original mc3. NB: this file spells the vec-broadcast define +// BROADCAT (no S) — match it so the fast _8 path compiles. +#ifdef VECTOR_SUB_GROUP_BROADCAT +#define MC_DQ1_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi +#define MC_DQ1_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo +#else +#define MC_DQ1_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi +#define MC_DQ1_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo +#endif +#define MC_COL_Q41(ts, c) \ + { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \ + regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \ + MC_DQ1_HI(ts, as_ushort8(regA_hi), regS, regM, regB); \ + MC_DQ1_LO(ts, as_ushort8(regA_lo), regS, regM, regB); } +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_1_f32_mc3( + read_only image1d_buffer_t src0_q, + global half2 * src0_d, + global half2 * src0_m, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int n_cols) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + uint COL_STRIDE = K / 4; // float4 pixels per activation column + + private uint4 regA_hi, regA_lo; + private half2 regS, regM; + private float8 regB; + + private float2 ts0 = (float2)(0.0f); + private float2 ts1 = (float2)(0.0f); + private float2 ts2 = (float2)(0.0f); + private float2 ts3 = (float2)(0.0f); + + for (uint k = groupId; k < (K / QK4_0); k += NSUBGROUPS) { + regS = src0_d[gid_s + k * LINE_STRIDE_A]; + regM = src0_m[gid_s + k * LINE_STRIDE_A]; + + // weights loaded ONCE, reused across the columns + regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + MC_COL_Q41(ts0, 0); + MC_COL_Q41(ts1, 1); + if (n_cols > 2) MC_COL_Q41(ts2, 2); + if (n_cols > 3) MC_COL_Q41(ts3, 3); + } + + // cross-subgroup reduce: pack the (up to 4) columns' float2 into a float8. + local float8 reduceLM[SUBGROUP_SIZE * 3]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1); + if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; } + if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; } + if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + acc += reduceLM[SUBGROUP_SIZE * 0 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 1 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row + if (gid < LINE_STRIDE_A) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + } + } +} +#undef MC_COL_Q41 +#undef MC_DQ1_HI +#undef MC_DQ1_LO diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl index dd1e2b5..3dc30db 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl @@ -11,9 +11,19 @@ #define NSUBGROUPS 4 #define SUBGROUP_SIZE 64 +// A6X compiler incorrectly constant-folds get_local_size() results; +// force runtime materialization via a no-op ALU round-trip. +#ifdef GGML_CL_A6X_CONSTFOLD_FIX +#define MATERIALIZE_WG(x) do { (x) *= 2u; if ((x) > 1u) (x) /= 2u; } while(0) +#else +#define MATERIALIZE_WG(x) +#endif + +// scales are transposed: consecutive codes of a row are `stride` apart inline void get_scale_min_k4( int j, global const uchar * q, + uint stride, uchar * d, uchar * m, uchar mask_d6, @@ -21,11 +31,11 @@ inline void get_scale_min_k4( uchar mask_hi2 ) { if (j < 4) { - *d = q[j] & mask_d6; - *m = q[j+4] & mask_d6; + *d = q[j*stride] & mask_d6; + *m = q[(j+4)*stride] & mask_d6; } else { - *d = (q[j+4] & mask_d4) | ((q[j-4] & mask_hi2) >> 2); - *m = ((q[j+4] >> 4) & mask_d4) | ((q[j] & mask_hi2) >> 2); + *d = (q[(j+4)*stride] & mask_d4) | ((q[(j-4)*stride] & mask_hi2) >> 2); + *m = ((q[(j+4)*stride] >> 4) & mask_d4) | ((q[j*stride] & mask_hi2) >> 2); } } @@ -226,14 +236,39 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( uint groupId = get_local_id(1); uint gid = get_global_id(0); ushort slid = get_sub_group_local_id(); + // K-split factor = #subgroups in the WG. Read from the launch (NOT a compile + // constant) so small-M projections (Kcur/Vcur/Qcur) can dispatch a wider + // K-split (more waves/SP -> latency hiding) while large-M keeps 4. The + // physical weight layout stride below is INDEPENDENT of this (see BLOCK_STRIDE_A). + uint nsg = get_local_size(1); + MATERIALIZE_WG(nsg); uint K = ne00; uint M = ne01; uint LINE_STRIDE_A = M / 2; - uint BLOCK_STRIDE_A = NSUBGROUPS * M; + // Physical per-K-block stride in the packed image: 8 uints/block-row-pair * + // (M/2) row-pairs = 4*M uints. This is a layout constant, not tied to nsg. + uint BLOCK_STRIDE_A = 4 * M; uint scales_per_row = (K / QK_K) * 12; + // The x-grid is padded to CEIL_DIV(ne01/2,64)*64, so when ne01 % 128 != 0 the + // tail lanes hold gid >= ne01/2. The output stores below are guarded, but the + // input fetches are not: src0_d and src0_m are raw global half2 pointers, + // src0_s is a raw global uchar pointer, and read_imageui on an + // image1d_buffer_t is UNDEFINED out of range -- an image clamps only for + // SAMPLER reads, which these are not. Those lanes therefore read past the end + // of all three allocations. For a [2816, 2112] weight (2112 % 128 == 64) the + // top tail lane is gid = 1087 while only gid < 1056 is backed, and it runs + // 32 half2 past src0_d/src0_m, 31 uints past the quant image, and 63 bytes + // past src0_s. + // + // Clamp the row used for every fetch. The lanes stay ACTIVE, which the + // sub_group_broadcast in the dequant macros requires, and their results are + // still discarded by the existing output guard. No-op and byte-identical + // whenever ne01 % 128 == 0. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + private uint4 regA; private half2 regS; private half2 regM; @@ -241,19 +276,19 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( private float2 totalSum = (float2)(0.0f); - for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) { + for (uint k = groupId; k < (K / 32); k += nsg) { uint sb = k / 8; uint j = k % 8; - half2 d = src0_d[gid + sb * LINE_STRIDE_A]; - half2 dm = src0_m[gid + sb * LINE_STRIDE_A]; + half2 d = src0_d[gid_s + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid_s + sb * LINE_STRIDE_A]; - global const uchar * sc0 = src0_s + 2 * gid * scales_per_row + sb * 12; - global const uchar * sc1 = src0_s + (2 * gid + 1) * scales_per_row + sb * 12; + global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid_s; + global const uchar * sc1 = sc0 + 1; uchar sv0, mn0, sv1, mn1; - get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); - get_scale_min_k4(j, sc1, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); @@ -264,20 +299,20 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( } // load half weights for two blocks in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB); #endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB); #else @@ -285,34 +320,528 @@ kernel void kernel_gemv_noshuffle_q4_k_f32( #endif // VECTOR_SUB_GROUP_BROADCAST } - // reduction in local memory, assumes #wave=4 - local float2 reduceLM[SUBGROUP_SIZE * 3]; - if (groupId == 1) { - reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum; - } - if (groupId == 2) { - reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum; - } - if (groupId == 3) { - reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum; + // Cross-subgroup reduction in local memory. Generalized to nsg subgroups + // (was a hard-coded 4-wave unroll). Sized for up to 16 subgroups (the widest + // K-split we dispatch for small M). At nsg==4 the accumulation order is + // identical to the original unroll -> byte-identical for the large-M path. + local float2 reduceLM[SUBGROUP_SIZE * 15]; + if (groupId > 0) { + reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = totalSum; } barrier(CLK_LOCAL_MEM_FENCE); if (groupId == 0) { - totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid]; + for (uint i = 0; i < nsg - 1; ++i) { + totalSum += reduceLM[SUBGROUP_SIZE * i + slid]; + } } + + // 2 outputs per fiber in wave 0 if (groupId == 0) { - totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } - if (groupId == 0) { - totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid]; + +} + +// --- Fused gate+up GEMV + GLU epilogue (FFN) ------------------------------------ +// Folds the FFN's two decode GEMVs (ffn_gate, ffn_up) and the following GLU into a +// SINGLE dispatch: {MUL_MAT(Wg,x), MUL_MAT(Wu,x), GLU}. Both matmuls share the same +// activation x (ffn_norm), so the activation image read is issued ONCE per K-block +// and reused for the gate and up dot products (the per-op path re-reads it twice and +// also materializes the two full ffn-wide intermediates to global, which the GLU +// then re-reads). The gate/up partial sums are accumulated in the SAME per-fiber +// order and reduced in the SAME cross-subgroup order as the standalone GEMV, and the +// GLU formula is the exact scalar expression from kernels/glu.cl, so the output is +// BYTE-IDENTICAL to the per-op matmul+matmul+glu path -> safe to default on. +// glu_op: REGLU=0, GEGLU=1, SWIGLU=2, GEGLU_ERF=4, GEGLU_QUICK=5 (ggml_glu_op). +// Weights: src0g_* = gate (= GLU src[0]); src0u_* = up (= GLU src[1]). +#define GLU_GEGLU_COEF_A 0.044715f +#define GLU_SQRT_2_OVER_PI 0.79788456080286535587989211986876f +#define GLU_SQRT_2_INV 0.70710678118654752440084436210484f +#define GLU_QUICK_COEF -1.702f + +inline float glu_apply(int glu_op, float g, float u) { + float act; + if (glu_op == 1) { // GEGLU (tanh-approx gelu) + act = 0.5f*g*(1.0f + tanh(GLU_SQRT_2_OVER_PI*g*(1.0f + GLU_GEGLU_COEF_A*g*g))); + } else if (glu_op == 2) { // SWIGLU (silu) + act = g / (1.0f + exp(-g)); + } else if (glu_op == 0) { // REGLU + return g*u*(g > 0.0f); + } else if (glu_op == 4) { // GEGLU_ERF + act = 0.5f*g*(1.0f + erf(g*GLU_SQRT_2_INV)); + } else { // GEGLU_QUICK (glu_op == 5) + act = g*(1.0f/(1.0f + exp(GLU_QUICK_COEF*g))); } + return act*u; +} - // 2 outputs per fiber in wave 0 +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_glu( + read_only image1d_buffer_t src0g_q, + global half2 * src0g_d, + global half2 * src0g_m, + global uchar * src0g_s, + read_only image1d_buffer_t src0u_q, + global half2 * src0u_d, + global half2 * src0u_m, + global uchar * src0u_s, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int glu_op, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + uint nsg = get_local_size(1); + MATERIALIZE_WG(nsg); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = 4 * M; + + private uint4 regA; + private half2 regS, regM; + private float8 regB; + + private float2 gateSum = (float2)(0.0f); + private float2 upSum = (float2)(0.0f); + + // Two SEQUENTIAL K-loops (gate fully, then up). Keeping only one weight's + // working set live at a time holds the kernel's register footprint at ~the + // base single-weight GEMV's, so its max WG stays 1024 (16 subgroups) and the + // per-subgroup K-split matches the standalone wide GEMV exactly -> the gate + // and up partial sums are BYTE-IDENTICAL to the per-op path. The macro body + // is the base kernel's inner loop verbatim, parameterized by weight source. +#define Q4K_GLU_LOOP(SUM, Q, DD, MM, SS) \ + for (uint k = groupId; k < (K / 32); k += nsg) { \ + uint sb = k / 8; \ + uint j = k % 8; \ + half2 d = DD[gid_s + sb * LINE_STRIDE_A]; \ + half2 dm = MM[gid_s + sb * LINE_STRIDE_A]; \ + global const uchar * sc0 = SS + sb * 12 * M + 2 * gid_s; \ + global const uchar * sc1 = sc0 + 1; \ + uchar sv0, mn0, sv1, mn1; \ + get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); \ + get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); \ + regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); \ + regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); \ + if (slid < 4) { \ + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); \ + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); \ + } \ + regA.s0 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; \ + regA.s1 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; \ + regA.s2 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; \ + regA.s3 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; \ + DEQ_HI(SUM, as_ushort8(regA), regS, regM, regB); \ + regA.s0 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; \ + regA.s1 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; \ + regA.s2 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; \ + regA.s3 = read_imageui(Q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; \ + DEQ_LO(SUM, as_ushort8(regA), regS, regM, regB); \ + } + +#ifdef VECTOR_SUB_GROUP_BROADCAST +#define DEQ_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi +#define DEQ_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo +#else +#define DEQ_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi +#define DEQ_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo +#endif + + Q4K_GLU_LOOP(gateSum, src0g_q, src0g_d, src0g_m, src0g_s) + Q4K_GLU_LOOP(upSum, src0u_q, src0u_d, src0u_m, src0u_s) + +#undef DEQ_HI +#undef DEQ_LO +#undef Q4K_GLU_LOOP + + // Cross-subgroup reduction in local memory. Packs gate (xy) + up (zw) into a + // float4 so both reduce in one pass; summation order matches the base GEMV's + // per-channel loop -> byte-identical partial sums. + local float4 reduceLM[SUBGROUP_SIZE * 15]; + if (groupId > 0) { + reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = (float4)(gateSum, upSum); + } + barrier(CLK_LOCAL_MEM_FENCE); if (groupId == 0) { + for (uint i = 0; i < nsg - 1; ++i) { + float4 p = reduceLM[SUBGROUP_SIZE * i + slid]; + gateSum += p.xy; + upSum += p.zw; + } dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + if (gid < LINE_STRIDE_A) { + dst[gid * 2 + 0] = glu_apply(glu_op, gateSum.s0, upSum.s0); + dst[gid * 2 + 1] = glu_apply(glu_op, gateSum.s1, upSum.s1); + } + } +} + +// --- Split-K-across-workgroups decode GEMV (small-M projections) ---------------- +// A single-token GEMV makes only ceil(M/2/64) workgroups; a WG runs on one Adreno +// compute unit, so for small M (Kcur/Vcur, M=512 -> 4 WGs) most of the 16 CUs sit +// idle and the matmul is bandwidth-starved even with a wide intra-WG K-split. This +// variant adds a SECOND grid dimension of `ksplit` workgroups that each reduce a +// disjoint slice of K and write a per-slice partial; kernel_gemv_splitk_reduce_f32 +// then sums the partials into dst. Identical math/layout to the base kernel +// (physical block stride 4*M, get_scale_min_k4) -> coherent. Gated host-side to +// M<=1024 (M>=2048 +// already fills the CUs and the extra reduce dispatch only hurts). +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_splitk( + read_only image1d_buffer_t src0_q, + global half2 * src0_d, + global half2 * src0_m, + global uchar * src0_s, + read_only image1d_buffer_t src1, + global float * partial, // [ksplit * M], slice-major + int ne00, + int ne01, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + uint nsg = get_local_size(1); + MATERIALIZE_WG(nsg); + uint ksplit = get_num_groups(1); + uint kslice = get_group_id(1); + + uint K = ne00; + uint M = ne01; + uint LINE_STRIDE_A = M / 2; + uint BLOCK_STRIDE_A = 4 * M; // physical, independent of the K-split + + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + + private uint4 regA; + private half2 regS, regM; + private float8 regB; + private float2 totalSum = (float2)(0.0f); + + // each (kslice, subgroup) pair owns a disjoint set of K-blocks + for (uint k = kslice * nsg + groupId; k < (K / 32); k += ksplit * nsg) { + uint sb = k / 8; + uint j = k % 8; + half2 d = src0_d[gid_s + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid_s + sb * LINE_STRIDE_A]; + global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid_s; + global const uchar * sc1 = sc0 + 1; + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); + regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), regS, regM, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), regS, regM, regB); +#endif + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), regS, regM, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum, as_ushort8(regA), regS, regM, regB); +#endif + } + + local float2 reduceLM[SUBGROUP_SIZE * 15]; + if (groupId > 0) { + reduceLM[SUBGROUP_SIZE * (groupId - 1) + slid] = totalSum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) { + for (uint i = 0; i < nsg - 1; ++i) { + totalSum += reduceLM[SUBGROUP_SIZE * i + slid]; + } + // Keep tail stores inside this K-slice. + if (gid * 2 + 0 < M) partial[kslice * M + gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) partial[kslice * M + gid * 2 + 1] = totalSum.s1; + } +} + +// Sum the per-slice partials [ksplit * M] into dst[M]; applies the dst byte offset. +kernel void kernel_gemv_splitk_reduce_f32( + global float * partial, + global float * dst, + ulong offsetd, + int ne01, // M + int ksplit) +{ + uint r = get_global_id(0); + if (r >= (uint)ne01) return; + float acc = 0.0f; + for (uint s = 0; s < (uint)ksplit; ++s) { + acc += partial[s * (uint)ne01 + r]; } + dst = (global float*)((global char*)dst + offsetd); + dst[r] = acc; +} + +// --- Dequant-once macros for the mc3 verify GEMV (Q4K_MC3_DEQUANT_ONCE) --- +// The inline dequantizeBlockAccum_* macros recompute the dequantized weight +// ((code & mask)>>shift)*scale - minv ONCE PER COLUMN (3x), and the flat +// 32-FMA unroll spills ~430 B of temporaries. These macros split the work: +// DEQUANT_Q4K_BLOCK computes the 16 weights/row of one 32-block ONCE into a +// half2[] (row0 in .s0, row1 in .s1) — stored as half, the exact type the +// inline expression yields (int*half-half), so no extra rounding. MAC_Q4K_BLOCK +// then accumulates them against a column's broadcast activation in the SAME +// per-accumulator order as the inline macro. Each weight value and each +// accumulator's add-chain is bit-for-bit identical => byte-identical output, +// while the dequant ALU drops 3x->1x and the live set shrinks. Requires the +// Qualcomm vector sub_group_broadcast (float8); enabled opt-in on Adreno. +#define DEQ_Q4K_HALF2(b0, b1, msk, sh, scale, minv) \ + (half2)( ((b0 & msk) >> sh) * scale.s0 - minv.s0, \ + ((b1 & msk) >> sh) * scale.s1 - minv.s1 ) + +#define DEQUANT_Q4K_BLOCK(wq, bits, scale, minv) \ + wq[0] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x000F, 0, scale, minv); \ + wq[1] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x00F0, 4, scale, minv); \ + wq[2] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0x0F00, 8, scale, minv); \ + wq[3] = DEQ_Q4K_HALF2(bits.s0, bits.s1, 0xF000, 12, scale, minv); \ + wq[4] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x000F, 0, scale, minv); \ + wq[5] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x00F0, 4, scale, minv); \ + wq[6] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0x0F00, 8, scale, minv); \ + wq[7] = DEQ_Q4K_HALF2(bits.s2, bits.s3, 0xF000, 12, scale, minv); \ + wq[8] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x000F, 0, scale, minv); \ + wq[9] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x00F0, 4, scale, minv); \ + wq[10] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0x0F00, 8, scale, minv); \ + wq[11] = DEQ_Q4K_HALF2(bits.s4, bits.s5, 0xF000, 12, scale, minv); \ + wq[12] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x000F, 0, scale, minv); \ + wq[13] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x00F0, 4, scale, minv); \ + wq[14] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0x0F00, 8, scale, minv); \ + wq[15] = DEQ_Q4K_HALF2(bits.s6, bits.s7, 0xF000, 12, scale, minv); + +// ln0/ln1 = the two source lanes whose activation float8 this block consumes +// (0,1 for the hi block, 2,3 for the lo block — matching the inline _hi/_lo). +#define MAC_Q4K_BLOCK(ts, wq, y, ln0, ln1) { \ + float8 sy = sub_group_broadcast(y, ln0); \ + ts.s0 += wq[0].s0*sy.s0; ts.s0 += wq[1].s0*sy.s1; ts.s0 += wq[2].s0*sy.s2; ts.s0 += wq[3].s0*sy.s3; \ + ts.s0 += wq[4].s0*sy.s4; ts.s0 += wq[5].s0*sy.s5; ts.s0 += wq[6].s0*sy.s6; ts.s0 += wq[7].s0*sy.s7; \ + ts.s1 += wq[0].s1*sy.s0; ts.s1 += wq[1].s1*sy.s1; ts.s1 += wq[2].s1*sy.s2; ts.s1 += wq[3].s1*sy.s3; \ + ts.s1 += wq[4].s1*sy.s4; ts.s1 += wq[5].s1*sy.s5; ts.s1 += wq[6].s1*sy.s6; ts.s1 += wq[7].s1*sy.s7; \ + sy = sub_group_broadcast(y, ln1); \ + ts.s0 += wq[8].s0*sy.s0; ts.s0 += wq[9].s0*sy.s1; ts.s0 += wq[10].s0*sy.s2; ts.s0 += wq[11].s0*sy.s3; \ + ts.s0 += wq[12].s0*sy.s4; ts.s0 += wq[13].s0*sy.s5; ts.s0 += wq[14].s0*sy.s6; ts.s0 += wq[15].s0*sy.s7; \ + ts.s1 += wq[8].s1*sy.s0; ts.s1 += wq[9].s1*sy.s1; ts.s1 += wq[10].s1*sy.s2; ts.s1 += wq[11].s1*sy.s3; \ + ts.s1 += wq[12].s1*sy.s4; ts.s1 += wq[13].s1*sy.s5; ts.s1 += wq[14].s1*sy.s6; ts.s1 += wq[15].s1*sy.s7; \ +} + +// Multi-column (N=3) variant of the q4_K decode GEMV, for the speculative / +// MTP verify batch (ne1=3 = 2 drafts + 1 bonus). Stays on the efficient GEMV +// path (subgroup-broadcast activation, NSUBGROUPS K-split) instead of the +// transposed-GEMM dead-zone path. Each K-block's weights (regA_hi/regA_lo) are +// loaded ONCE and reused across all 3 activation columns — same weight traffic +// as one decode, ~3x the (cheap) dequant ALU. Per-column accumulation is +// independent and identical to 3 standalone GEMVs => byte-identical, so it does +// NOT perturb the lm_head logits / spec accept rate. +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_mc3( + read_only image1d_buffer_t src0_q, + global half2 * src0_d, + global half2 * src0_m, + global uchar * src0_s, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + uint COL_STRIDE = K / 4; // float4 pixels per activation column + + private uint4 regA_hi, regA_lo; + private half2 regS, regM; + private float8 regB; + + private float2 ts0 = (float2)(0.0f); + private float2 ts1 = (float2)(0.0f); + private float2 ts2 = (float2)(0.0f); + +#ifdef Q4K_MC3_DEQUANT_LDS + // One 16-half2 block buffer per WI (reused hi->lo): forces the dequantized + // weights into LDS instead of private arrays (which spill to slow global on + // Adreno). 64*NSUBGROUPS WIs * 16 half2 = 16 KB; each WI owns its own slot + // range (flat*16) -> no cross-lane sharing, no barrier needed. + local half2 wstage[SUBGROUP_SIZE * NSUBGROUPS * 16]; + local half2 * ws = wstage + (groupId * SUBGROUP_SIZE + slid) * 16; +#endif + + for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) { + uint sb = k / 8; + uint j = k % 8; + + half2 d = src0_d[gid_s + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid_s + sb * LINE_STRIDE_A]; + + global const uchar * sc0 = src0_s + sb * 12 * M + 2 * gid_s; + global const uchar * sc1 = sc0 + 1; + + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4(j, sc0, M, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1, M, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + + regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); + regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); + + // weights loaded ONCE, reused across the 3 columns + regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + +#ifdef Q4K_MC3_DEQUANT_ONCE + // Dequant the 32 weights/row (16 hi + 16 lo) ONCE into half2[] (byte- + // identical to the inline intermediate), then MAC against each column's + // activation. Drops the dequant ALU 3x->1x and the macro-temp spill. + half2 wq_hi[16], wq_lo[16]; + DEQUANT_Q4K_BLOCK(wq_hi, as_ushort8(regA_hi), regS, regM); + DEQUANT_Q4K_BLOCK(wq_lo, as_ushort8(regA_lo), regS, regM); + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts0, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts0, wq_lo, regB, 2, 3); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts1, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts1, wq_lo, regB, 2, 3); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts2, wq_hi, regB, 0, 1); MAC_Q4K_BLOCK(ts2, wq_lo, regB, 2, 3); } +#elif defined(Q4K_MC3_DEQUANT_LDS) + // LDS-staged dequant: dequant a 32-block ONCE into the per-WI LDS slot + // (hi pass then lo pass, overwriting), MAC each column from LDS. ts* + // receive hi-then-lo in the same order as DEQUANT_ONCE -> byte-identical. + // Activations reloaded per pass (cheap, imaged); only one regB + 0 weight + // regs live -> the weight working set lives in LDS, not spilled private. + DEQUANT_Q4K_BLOCK(ws, as_ushort8(regA_hi), regS, regM); + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts0, ws, regB, 0, 1); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts1, ws, regB, 0, 1); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts2, ws, regB, 0, 1); } + DEQUANT_Q4K_BLOCK(ws, as_ushort8(regA_lo), regS, regM); + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts0, ws, regB, 2, 3); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts1, ws, regB, 2, 3); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + MAC_Q4K_BLOCK(ts2, ws, regB, 2, 3); } +#else + // Per-column: load only this column's activation (single regB live at a + // time -> 1/3 the activation register pressure vs holding all 3) then + // dequant against the shared weights. Cuts the private-mem spill. +#ifdef VECTOR_SUB_GROUP_BROADCAST + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts0, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts0, as_ushort8(regA_lo), regS, regM, regB); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts1, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts1, as_ushort8(regA_lo), regS, regM, regB); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_8_hi(ts2, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_8_lo(ts2, as_ushort8(regA_lo), regS, regM, regB); } +#else + { if (slid < 4) { regB.s0123 = read_imagef(src1, 0*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts0, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts0, as_ushort8(regA_lo), regS, regM, regB); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 1*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts1, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts1, as_ushort8(regA_lo), regS, regM, regB); } + { if (slid < 4) { regB.s0123 = read_imagef(src1, 2*COL_STRIDE + slid*2 + k*8); + regB.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + dequantizeBlockAccum_ns_sgbroadcast_1_hi(ts2, as_ushort8(regA_hi), regS, regM, regB); + dequantizeBlockAccum_ns_sgbroadcast_1_lo(ts2, as_ushort8(regA_lo), regS, regM, regB); } +#endif +#endif // Q4K_MC3_DEQUANT_ONCE + } + + // cross-subgroup reduce: pack the 3 columns' float2 into a float8 (6 used). + local float8 reduceLM[SUBGROUP_SIZE * 3]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, 0.0f, 0.0f); + if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; } + if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; } + if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + acc += reduceLM[SUBGROUP_SIZE * 0 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 1 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // dst is column-major [M rows x 3 cols]: (row, col) at col*M + row + if (gid < LINE_STRIDE_A) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + } + } } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl new file mode 100644 index 0000000..2dbd943 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_32b_trans.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define K_SCALE_SIZE 12 +#define N_SIMDGROUP 8 +#define SIMDGROUP_WIDTH 64 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uint stride, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j*stride] & 63; + *m = q[(j+4)*stride] & 63; + } else { + *d = (q[(j+4)*stride] & 0x0F) | ((q[(j-4)*stride] & 0xC0) >> 2); + *m = ((q[(j+4)*stride] >> 4) & 0x0F) | ((q[j*stride] & 0xC0) >> 2); + } +} + +static inline float8 q4_k_to_fp32_packed8(ushort2 q4x8, float scale, float minv) { + float8 fp32x8; + fp32x8.s0 = (q4x8.s0 & 0x000F) * scale - minv; + fp32x8.s1 = ((q4x8.s0 & 0x00F0) >> 4) * scale - minv; + fp32x8.s2 = ((q4x8.s0 & 0x0F00) >> 8) * scale - minv; + fp32x8.s3 = ((q4x8.s0 & 0xF000) >> 12) * scale - minv; + fp32x8.s4 = (q4x8.s1 & 0x000F) * scale - minv; + fp32x8.s5 = ((q4x8.s1 & 0x00F0) >> 4) * scale - minv; + fp32x8.s6 = ((q4x8.s1 & 0x0F00) >> 8) * scale - minv; + fp32x8.s7 = ((q4x8.s1 & 0xF000) >> 12) * scale - minv; + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void gemv_noshuffle_q4_k_f32_32b_trans( + read_only image1d_buffer_t src0_q, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + uint i01 = get_global_id(0); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + int num_subblocks = ne00 / 32; + + __private float sum = 0.0f; + + // Loop over sub-blocks of 32 elements, N_SIMDGROUP sub-blocks per iter + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; + uint j = ib % 8; + + // Load d and dmin for this super-block + half d_val = src0_d[sb * ne01 + i01]; + half dm_val = src0_dm[sb * ne01 + i01]; + + // Load sub-block scale and min. s is transposed [nb][12][M]; stride ne01 per code. + global const uchar * sc = src0_s + sb * K_SCALE_SIZE * ne01 + i01; + uchar sv, mn; + get_scale_min_k4(j, sc, ne01, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = (float)dm_val * (float)mn; + + // Load 4 uints of quants (32 nibbles = 32 elements), column-major stride ne01 + uint q_base = ib * ne01 * 4 + i01; + + uint4 regQ; + regQ.s0 = read_imageui(src0_q, q_base).x; + regQ.s1 = read_imageui(src0_q, q_base + ne01).x; + regQ.s2 = read_imageui(src0_q, q_base + ne01 * 2).x; + regQ.s3 = read_imageui(src0_q, q_base + ne01 * 3).x; + + // Load activations: 32 floats = 8 float4s + uint y_offset = ib * 8; + + float4 y_local = (slid < 8) ? read_imagef(src1, (y_offset + slid)) : (float4)0.0f; + float4 y0 = sub_group_broadcast(y_local, 0); + float4 y1 = sub_group_broadcast(y_local, 1); + float4 y2 = sub_group_broadcast(y_local, 2); + float4 y3 = sub_group_broadcast(y_local, 3); + float4 y4 = sub_group_broadcast(y_local, 4); + float4 y5 = sub_group_broadcast(y_local, 5); + float4 y6 = sub_group_broadcast(y_local, 6); + float4 y7 = sub_group_broadcast(y_local, 7); + + float8 fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s0), scale, minv); + float4 acc = y0 * fp32x8.lo; + acc += y1 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s1), scale, minv); + acc += y2 * fp32x8.lo; + acc += y3 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s2), scale, minv); + acc += y4 * fp32x8.lo; + acc += y5 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s3), scale, minv); + acc += y6 * fp32x8.lo; + acc += y7 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory over N_SIMDGROUP subgroups + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid > 0) { + reduceLM[SIMDGROUP_WIDTH * (sgid - 1) + slid] = sum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) { + for (uint i = 0; i < N_SIMDGROUP - 1; ++i) { + sum += reduceLM[SIMDGROUP_WIDTH * i + slid]; + } + } + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl new file mode 100644 index 0000000..2c1c659 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl @@ -0,0 +1,350 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK_K 256 +#define NSUBGROUPS 4 +#define SUBGROUP_SIZE 64 + +// scales are transposed: consecutive codes of a row are `stride` apart +inline void get_scale_min_k4( + int j, + global const uchar * q, + uint stride, + uchar * d, + uchar * m, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + if (j < 4) { + *d = q[j*stride] & mask_d6; + *m = q[(j+4)*stride] & mask_d6; + } else { + *d = (q[(j+4)*stride] & mask_d4) | ((q[(j-4)*stride] & mask_hi2) >> 2); + *m = ((q[(j+4)*stride] >> 4) & mask_d4) | ((q[j*stride] & mask_hi2) >> 2); + } +} + +#define dequantizeBlockAccum_ns_sgbroadcast_1_hi(total_sums, bits4, scale, minv, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + + +#define dequantizeBlockAccum_ns_sgbroadcast_1_lo(total_sums, bits4, scale, minv, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y; \ + + +#define dequantizeBlockAccum_ns_sgbroadcast_8_hi(total_sums, bits4, scale, minv, y) \ + float8 shared_y; \ + shared_y = sub_group_broadcast(y, 0); \ + total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \ + total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \ + total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \ + total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \ + total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \ + total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \ + total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \ + total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \ + total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \ + total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \ + total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \ + total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \ + total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \ + total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \ + total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \ + total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 1); \ + total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \ + total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \ + total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \ + total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \ + total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \ + total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \ + total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \ + total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \ + total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \ + total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \ + total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \ + total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \ + total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \ + total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \ + total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \ + total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \ + + +#define dequantizeBlockAccum_ns_sgbroadcast_8_lo(total_sums, bits4, scale, minv, y) \ + shared_y = sub_group_broadcast(y, 2); \ + total_sums.s0 += ((bits4.s0 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \ + total_sums.s0 += (((bits4.s0 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \ + total_sums.s0 += (((bits4.s0 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \ + total_sums.s0 += (((bits4.s0 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \ + total_sums.s0 += ((bits4.s2 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \ + total_sums.s0 += (((bits4.s2 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \ + total_sums.s0 += (((bits4.s2 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \ + total_sums.s0 += (((bits4.s2 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \ + total_sums.s1 += ((bits4.s1 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \ + total_sums.s1 += (((bits4.s1 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \ + total_sums.s1 += (((bits4.s1 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \ + total_sums.s1 += (((bits4.s1 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \ + total_sums.s1 += ((bits4.s3 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \ + total_sums.s1 += (((bits4.s3 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \ + total_sums.s1 += (((bits4.s3 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \ + total_sums.s1 += (((bits4.s3 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 3); \ + total_sums.s0 += ((bits4.s4 & 0x000F) * scale.s0 - minv.s0) * shared_y.s0; \ + total_sums.s0 += (((bits4.s4 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s1; \ + total_sums.s0 += (((bits4.s4 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s2; \ + total_sums.s0 += (((bits4.s4 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s3; \ + total_sums.s0 += ((bits4.s6 & 0x000F) * scale.s0 - minv.s0) * shared_y.s4; \ + total_sums.s0 += (((bits4.s6 & 0x00F0) >> 4) * scale.s0 - minv.s0) * shared_y.s5; \ + total_sums.s0 += (((bits4.s6 & 0x0F00) >> 8) * scale.s0 - minv.s0) * shared_y.s6; \ + total_sums.s0 += (((bits4.s6 & 0xF000) >> 12) * scale.s0 - minv.s0) * shared_y.s7; \ + total_sums.s1 += ((bits4.s5 & 0x000F) * scale.s1 - minv.s1) * shared_y.s0; \ + total_sums.s1 += (((bits4.s5 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s1; \ + total_sums.s1 += (((bits4.s5 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s2; \ + total_sums.s1 += (((bits4.s5 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s3; \ + total_sums.s1 += ((bits4.s7 & 0x000F) * scale.s1 - minv.s1) * shared_y.s4; \ + total_sums.s1 += (((bits4.s7 & 0x00F0) >> 4) * scale.s1 - minv.s1) * shared_y.s5; \ + total_sums.s1 += (((bits4.s7 & 0x0F00) >> 8) * scale.s1 - minv.s1) * shared_y.s6; \ + total_sums.s1 += (((bits4.s7 & 0xF000) >> 12) * scale.s1 - minv.s1) * shared_y.s7; \ + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_o4( + read_only image1d_buffer_t src0_q, + global half2 * src0_d, + global half2 * src0_m, + global uchar * src0_s, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); // 4-output quad index + ushort slid = get_sub_group_local_id(); + + // Two consecutive pair-indices (each the same access pattern the 2-output + // kernel uses); together they cover 4 consecutive output rows. + // Keep tail lanes active for subgroup ops, but read valid weight pairs. + uint gid_a = min(gid * 2, (uint)(ne01 / 2) - 1u); + uint gid_b = min(gid * 2 + 1, (uint)(ne01 / 2) - 1u); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + + private uint4 regA; + private half2 regS_a, regS_b; + private half2 regM_a, regM_b; + private float8 regB; + + private float2 totalSum_a = (float2)(0.0f); + private float2 totalSum_b = (float2)(0.0f); + + for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) { + uint sb = k / 8; + uint j = k % 8; + + // pair a scales/mins + half2 d_a = src0_d[gid_a + sb * LINE_STRIDE_A]; + half2 dm_a = src0_m[gid_a + sb * LINE_STRIDE_A]; + global const uchar * sc0a = src0_s + sb * 12 * M + 2 * gid_a; + global const uchar * sc1a = sc0a + 1; + uchar sv0a, mn0a, sv1a, mn1a; + get_scale_min_k4(j, sc0a, M, &sv0a, &mn0a, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1a, M, &sv1a, &mn1a, mask_d6, mask_d4, mask_hi2); + regS_a = convert_half2(convert_float2(d_a) * convert_float2((uchar2)(sv0a, sv1a))); + regM_a = convert_half2(convert_float2(dm_a) * convert_float2((uchar2)(mn0a, mn1a))); + + // pair b scales/mins + half2 d_b = src0_d[gid_b + sb * LINE_STRIDE_A]; + half2 dm_b = src0_m[gid_b + sb * LINE_STRIDE_A]; + global const uchar * sc0b = src0_s + sb * 12 * M + 2 * gid_b; + global const uchar * sc1b = sc0b + 1; + uchar sv0b, mn0b, sv1b, mn1b; + get_scale_min_k4(j, sc0b, M, &sv0b, &mn0b, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1b, M, &sv1b, &mn1b, mask_d6, mask_d4, mask_hi2); + regS_b = convert_half2(convert_float2(d_b) * convert_float2((uchar2)(sv0b, sv1b))); + regM_b = convert_half2(convert_float2(dm_b) * convert_float2((uchar2)(mn0b, mn1b))); + + // activation: load once, reuse for both pairs + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + + // pair a (own block so _lo sees the shared_y declared by _hi) + { + regA.s0 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB); +#endif + regA.s0 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_a + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum_a, as_ushort8(regA), regS_a, regM_a, regB); +#endif + } + + // pair b + { + regA.s0 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB); +#endif + regA.s0 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_b + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB); +#else + dequantizeBlockAccum_ns_sgbroadcast_1_lo(totalSum_b, as_ushort8(regA), regS_b, regM_b, regB); +#endif + } + } + + // reduce 4 outputs (a.s0, a.s1, b.s0, b.s1) across the 4 subgroups + local float4 reduceLM[SUBGROUP_SIZE * 3]; + float4 acc = (float4)(totalSum_a.s0, totalSum_a.s1, totalSum_b.s0, totalSum_b.s1); + if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; } + if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; } + if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + acc += reduceLM[SUBGROUP_SIZE * 0 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 1 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // The dispatch rounds ne01/4 up to the subgroup width, so the tail + // quads past the last row must not store (they wrote 128 rows past + // dst on every ne01 % 256 == 128 vocab, e.g. 151936). + if (gid * 4 + 3 < (uint)ne01) { + vstore4(acc, 0, &(dst[gid * 4])); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl new file mode 100644 index 0000000..929538c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl @@ -0,0 +1,118 @@ +// Tiled-wide q4_K GEMV for the long-vocab lm_head/embed (decode path). +// +// Pairs with kernel_convert_block_q4_k_tiled_ns (cvt.cl): the weights are laid +// out CANONICALLY (4-bit code in element order e in [0,256)) and TILED by 64 +// output rows so the 64-thread lane group coalesces every weight load. Both the +// pack (convert) and the unpack (here) are owned by us -> correct by +// construction vs the reference ggml q4_K dequant. Same structure as the q6_K +// tiled GEMV; the only differences are the 4-bit dequant and the q4_K +// scale/min decode (get_scale_min_k4 from the packed 12-byte block). +// +// One work-item produces one output row. WG = {64 lanes, 4 subgroups}: the 64 +// lanes cover the 64 rows of one tile (coalesced uint4 reads), the 4 subgroups +// split the K-blocks and reduce through __local at the end. Weights read from +// __global (lm_head is streamed once per token; texture cache caps it below the +// coalesced-global rate). + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK_K 256 +#define NSUBGROUPS 4 +#define TILE_ROWS 64 + +// Decode one q4_K sub-block scale + min from the packed 12-byte block. +// Identical to the o4 kernel's helper (masks hard-coded: d6=0x3F, d4=0x0F, hi2=0xC0). +inline void q4k_scale_min(int j, __global const uchar * q, uchar * d, uchar * m) { + if (j < 4) { + *d = q[j] & 0x3F; + *m = q[j+4] & 0x3F; + } else { + *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2); + *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2); + } +} + +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q4_k_f32_tiled( + __global uint4 * src0_q, // tiled: 8 uint4 granules / superblock (4-bit codes) + __global half * src0_d, // tiled: 1 half / superblock + __global half * src0_dm, // tiled: 1 half / superblock + __global uchar * src0_s, // tiled: 12 bytes / superblock (packed scales) + read_only image1d_buffer_t src1, // activation (RGBA f32) + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); // subgroup index 0..3 (splits K) + int row = get_global_id(0); // output row along ne01 + int rt = row / TILE_ROWS; + int rit = row % TILE_ROWS; + + int nb = ne00 / QK_K; // superblocks per row + + float acc = 0.0f; + + for (int sb = grp; sb < nb; sb += NSUBGROUPS) { + int tile_blk = rt * nb + sb; // ne02 == 1 for lm_head/embed + + float dval = (float)src0_d [tile_blk * TILE_ROWS + rit]; + float dmval = (float)src0_dm[tile_blk * TILE_ROWS + rit]; + + // decode the 8 sub-block (scale, min) pairs + __global uchar * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 12; + float scale[8], minv[8]; + #pragma unroll + for (int is = 0; is < 8; ++is) { + uchar sd, sm; + q4k_scale_min(is, sc, &sd, &sm); + scale[is] = dval * (float)sd; + minv[is] = dmval * (float)sm; + } + + // 32 uints of 4-bit codes (8 codes/uint), e-order + uint q[32]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + uint4 v = src0_q[(tile_blk * 8 + g) * TILE_ROWS + rit]; + q[g*4+0] = v.x; q[g*4+1] = v.y; q[g*4+2] = v.z; q[g*4+3] = v.w; + } + + // dequant 256 codes in canonical e-order, MAC with activation. + int act_base = sb * 64; // activation float4 pixel base (256/4) + #pragma unroll + for (int e4 = 0; e4 < 64; ++e4) { + float4 a = read_imagef(src1, act_base + e4); + #pragma unroll + for (int t = 0; t < 4; ++t) { + int e = e4 * 4 + t; + uint code = (q[e >> 3] >> ((e & 7) * 4)) & 0xF; + int is = e >> 5; // sub-block index = e/32 + float av = (t == 0) ? a.x : (t == 1) ? a.y : (t == 2) ? a.z : a.w; + acc += ((float)code * scale[is] - minv[is]) * av; + } + } + } + + // reduce across the NSUBGROUPS subgroups (same rit, different K-subset) + local float reduce_lm[NSUBGROUPS * TILE_ROWS]; + reduce_lm[grp * TILE_ROWS + rit] = acc; + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + float total = reduce_lm[0 * TILE_ROWS + rit] + + reduce_lm[1 * TILE_ROWS + rit] + + reduce_lm[2 * TILE_ROWS + rit] + + reduce_lm[3 * TILE_ROWS + rit]; + dst = (global float*)((global char*)dst + offsetd); + dst[row] = total; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl new file mode 100644 index 0000000..d8537e1 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl @@ -0,0 +1,298 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK5_0 32 +#define NSUBGROUPS 4 +#define SUBGROUP_SIZE 64 + +#define dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_hi(total_sums, bits4, bits1, scale, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s0 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s4 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s0 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s4 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s0 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s4 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s0 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s4 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s0 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s4 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s0 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s4 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s0 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s4 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s0 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s4 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s1 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s5 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s1 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s5 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s1 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s5 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s1 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s5 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s1 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s5 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s1 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s5 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s1 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s5 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s1 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s5 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + + +#define dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_lo(total_sums, bits4, bits1, scale, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s2 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s6 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s2 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s6 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s2 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s6 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s2 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s6 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s2 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s6 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s2 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s6 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s2 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s6 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s2 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s6 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s3 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s7 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s3 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s7 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s3 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s7 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s3 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s7 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s3 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s7 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s3 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s7 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s3 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s7 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s3 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s7 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \ + + +#define dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_hi(total_sums, bits4, bits1, scale, y) \ + float8 shared_y; \ + shared_y = sub_group_broadcast(y, 0); \ + total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s0 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s0; \ + total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s0 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s1; \ + total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s0 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s2; \ + total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s0 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s3; \ + total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s0 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s4; \ + total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s0 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s5; \ + total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s0 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s6; \ + total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s0 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s7; \ + total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s4 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s0; \ + total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s4 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s1; \ + total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s4 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s2; \ + total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s4 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s3; \ + total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s4 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s4; \ + total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s4 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s5; \ + total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s4 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s6; \ + total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s4 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 1); \ + total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s1 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s0; \ + total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s1 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s1; \ + total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s1 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s2; \ + total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s1 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s3; \ + total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s1 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s4; \ + total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s1 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s5; \ + total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s1 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s6; \ + total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s1 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s7; \ + total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s5 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s0; \ + total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s5 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s1; \ + total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s5 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s2; \ + total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s5 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s3; \ + total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s5 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s4; \ + total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s5 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s5; \ + total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s5 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s6; \ + total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s5 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s7; \ + + +#define dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_lo(total_sums, bits4, bits1, scale, y) \ + shared_y = sub_group_broadcast(y, 2); \ + total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s2 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s0; \ + total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s2 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s1; \ + total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s2 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s2; \ + total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s2 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s3; \ + total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s2 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s4; \ + total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s2 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s5; \ + total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s2 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s6; \ + total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s2 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s7; \ + total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s6 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s0; \ + total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s6 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s1; \ + total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s6 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s2; \ + total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s6 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s3; \ + total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s6 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s4; \ + total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s6 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s5; \ + total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s6 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s6; \ + total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s6 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 3); \ + total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s3 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s0; \ + total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s3 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s1; \ + total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s3 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s2; \ + total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s3 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s3; \ + total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s3 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s4; \ + total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s3 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s5; \ + total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s3 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s6; \ + total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s3 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s7; \ + total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s7 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s0; \ + total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s7 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s1; \ + total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s7 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s2; \ + total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s7 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s3; \ + total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s7 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s4; \ + total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s7 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s5; \ + total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s7 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s6; \ + total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s7 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s7; \ + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q5_0_f32( + __read_only image1d_buffer_t src0_qs, // quantized A + global ushort * src0_qh, // 5th bits + global half2 * src0_d, // A scales + __read_only image1d_buffer_t src1, // B activations + global float * dst, + ulong offsetd, + int ne00, // K + int ne01) // M +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + + private uint4 regA; + private half2 regS; + private float8 regB; + + private float2 totalSum = (float2)(0.0f); + + for (uint k = groupId; k < (K / QK5_0); k += NSUBGROUPS) { + regS = src0_d[gid_s + k * LINE_STRIDE_A]; + + ushort4 qh_raw; + qh_raw.s0 = src0_qh[gid_s + (4*k + 0) * LINE_STRIDE_A]; + qh_raw.s1 = src0_qh[gid_s + (4*k + 1) * LINE_STRIDE_A]; + qh_raw.s2 = src0_qh[gid_s + (4*k + 2) * LINE_STRIDE_A]; + qh_raw.s3 = src0_qh[gid_s + (4*k + 3) * LINE_STRIDE_A]; + + uchar8 raw = as_uchar8(qh_raw); + uchar8 qh_bytes = (uchar8)(raw.s0, raw.s2, raw.s4, raw.s6, + raw.s1, raw.s3, raw.s5, raw.s7); + + // Load activations + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + + regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regB); +#else + dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regB); +#endif // VECTOR_SUB_GROUP_BROADCAST + + regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regB); +#else + dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regB); +#endif // VECTOR_SUB_GROUP_BROADCAST + } + + // reduction in local memory, assumes #wave=4 + local float2 reduceLM[SUBGROUP_SIZE * 3]; + if (groupId == 1) { + reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum; + } + if (groupId == 2) { + reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum; + } + if (groupId == 3) { + reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid]; + } + if (groupId == 0) { + totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid]; + } + if (groupId == 0) { + totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid]; + } + + // 2 outputs per fiber in wave 0 + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; + } + +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl new file mode 100644 index 0000000..82b4c74 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl @@ -0,0 +1,301 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define QK5_1 32 +#define NSUBGROUPS 4 +#define SUBGROUP_SIZE 64 + +#define dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_hi(total_sums, bits4, bits1, scale, minv, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s0 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s4 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s0 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s4 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s0 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s4 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s0 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s4 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s0 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s4 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s0 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s4 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s0 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s4 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s0 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s4 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s1 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s5 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s1 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s5 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s1 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s5 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s1 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s5 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s1 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s5 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s1 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s5 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s1 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s5 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s1 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s5 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + + +#define dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_lo(total_sums, bits4, bits1, scale, minv, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s2 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s6 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s2 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s6 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s2 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s6 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s2 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s6 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s2 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s6 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s2 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s6 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s2 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s6 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s2 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s6 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s3 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s7 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s3 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s7 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s3 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s7 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s3 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s7 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s3 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s7 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s3 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s7 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s3 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s7 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s3 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \ + total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s7 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \ + + +#define dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_hi(total_sums, bits4, bits1, scale, minv, y) \ + float8 shared_y; \ + shared_y = sub_group_broadcast(y, 0); \ + total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s0 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s0; \ + total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s0 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s1; \ + total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s0 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s2; \ + total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s0 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s3; \ + total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s0 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s4; \ + total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s0 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s5; \ + total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s0 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s6; \ + total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s0 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s7; \ + total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s4 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s0; \ + total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s4 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s1; \ + total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s4 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s2; \ + total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s4 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s3; \ + total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s4 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s4; \ + total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s4 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s5; \ + total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s4 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s6; \ + total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s4 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 1); \ + total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s1 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s0; \ + total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s1 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s1; \ + total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s1 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s2; \ + total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s1 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s3; \ + total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s1 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s4; \ + total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s1 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s5; \ + total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s1 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s6; \ + total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s1 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s7; \ + total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s5 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s0; \ + total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s5 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s1; \ + total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s5 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s2; \ + total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s5 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s3; \ + total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s5 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s4; \ + total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s5 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s5; \ + total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s5 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s6; \ + total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s5 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s7; \ + + +#define dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_lo(total_sums, bits4, bits1, scale, minv, y) \ + shared_y = sub_group_broadcast(y, 2); \ + total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s2 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s0; \ + total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s2 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s1; \ + total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s2 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s2; \ + total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s2 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s3; \ + total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s2 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s4; \ + total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s2 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s5; \ + total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s2 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s6; \ + total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s2 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s7; \ + total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s6 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s0; \ + total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s6 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s1; \ + total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s6 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s2; \ + total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s6 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s3; \ + total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s6 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s4; \ + total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s6 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s5; \ + total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s6 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s6; \ + total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s6 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 3); \ + total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s3 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s0; \ + total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s3 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s1; \ + total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s3 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s2; \ + total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s3 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s3; \ + total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s3 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s4; \ + total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s3 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s5; \ + total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s3 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s6; \ + total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s3 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s7; \ + total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s7 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s0; \ + total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s7 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s1; \ + total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s7 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s2; \ + total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s7 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s3; \ + total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s7 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s4; \ + total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s7 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s5; \ + total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s7 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s6; \ + total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s7 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s7; \ + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q5_1_f32( + __read_only image1d_buffer_t src0_qs, // quantized A + global ushort * src0_qh, // 5th bits + global half2 * src0_d, // A scales + global half2 * src0_m, // A mins + __read_only image1d_buffer_t src1, // B activations + global float * dst, + ulong offsetd, + int ne00, // K + int ne01) // M +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + + __private uint4 regA; + __private half2 regS; + __private half2 regM; + __private float8 regB; + + __private float2 totalSum = (float2)(0.0f); + + for (uint k = groupId; k < (K / QK5_1); k += NSUBGROUPS) { + regS = src0_d[gid_s + k * LINE_STRIDE_A]; + regM = src0_m[gid_s + k * LINE_STRIDE_A]; + + ushort4 qh_raw; + qh_raw.s0 = src0_qh[gid_s + (4*k + 0) * LINE_STRIDE_A]; + qh_raw.s1 = src0_qh[gid_s + (4*k + 1) * LINE_STRIDE_A]; + qh_raw.s2 = src0_qh[gid_s + (4*k + 2) * LINE_STRIDE_A]; + qh_raw.s3 = src0_qh[gid_s + (4*k + 3) * LINE_STRIDE_A]; + + uchar8 raw = as_uchar8(qh_raw); + uchar8 qh_bytes = (uchar8)(raw.s0, raw.s2, raw.s4, raw.s6, + raw.s1, raw.s3, raw.s5, raw.s7); + + // Load activations + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + + regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB); +#else + dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB); +#endif // VECTOR_SUB_GROUP_BROADCAST + + regA.s0 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_qs, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; +#ifdef VECTOR_SUB_GROUP_BROADCAST + dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB); +#else + dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB); +#endif // VECTOR_SUB_GROUP_BROADCAST + } + + // reduction in local memory, assumes #wave=4 + local float2 reduceLM[SUBGROUP_SIZE * 3]; + if (groupId == 1) { + reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum; + } + if (groupId == 2) { + reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum; + } + if (groupId == 3) { + reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid]; + } + if (groupId == 0) { + totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid]; + } + if (groupId == 0) { + totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid]; + } + + // 2 outputs per fiber in wave 0 + if (groupId == 0) { + dst = (global float*)((global char*)dst + offsetd); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; + } + +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl index c40db16..6570167 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl @@ -232,6 +232,8 @@ kernel void kernel_gemv_noshuffle_q5_k_f32( uint M = ne01; uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = NSUBGROUPS * M; uint LINE_STRIDE_A_QH = M / 2; @@ -250,11 +252,11 @@ kernel void kernel_gemv_noshuffle_q5_k_f32( uint sb = k / 8; uint j = k % 8; - half2 d = src0_d[gid + sb * LINE_STRIDE_A]; - half2 dm = src0_m[gid + sb * LINE_STRIDE_A]; + half2 d = src0_d[gid_s + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid_s + sb * LINE_STRIDE_A]; - global const uchar * sc0 = src0_s + 2 * gid * scales_per_row + sb * 12; - global const uchar * sc1 = src0_s + (2 * gid + 1) * scales_per_row + sb * 12; + global const uchar * sc0 = src0_s + 2 * gid_s * scales_per_row + sb * 12; + global const uchar * sc1 = src0_s + (2 * gid_s + 1) * scales_per_row + sb * 12; uchar sv0, mn0, sv1, mn1; get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); @@ -268,25 +270,25 @@ kernel void kernel_gemv_noshuffle_q5_k_f32( regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); } - regH.s0 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x); - regH.s1 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x); - regH.s2 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x); - regH.s3 = as_ushort(read_imageh(src0_qh, (gid + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x); + regH.s0 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x); + regH.s1 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x); + regH.s2 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x); + regH.s3 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x); - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_hi(totalSum, as_ushort8(regA), as_uchar8(regH), regS, regM, regB); #else dequantizeBlockAccum_ns_sgbroadcast_1_hi(totalSum, as_ushort8(regA), as_uchar8(regH), regS, regM, regB); #endif // VECTOR_SUB_GROUP_BROADCAST - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; #ifdef VECTOR_SUB_GROUP_BROADCAST dequantizeBlockAccum_ns_sgbroadcast_8_lo(totalSum, as_ushort8(regA), as_uchar8(regH), regS, regM, regB); #else @@ -321,6 +323,137 @@ kernel void kernel_gemv_noshuffle_q5_k_f32( // 2 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(totalSum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor. No-op / byte-identical when + // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding). + if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0; + if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1; } } + +// Multi-column (N in [2..4]) variant of the q5_K decode GEMV (spec/MTP verify) = +// q4_K mc3 + the high-bit qh plane (regH). n_cols = 2..4 (drafted + bonus); routes +// the small-batch verify OFF the gemm_noshuffle_q5_k dead-zone. n_cols==3 is byte- +// identical to the original mc3 (col3 disabled, float8 slots 6/7 stay zero). +#ifdef VECTOR_SUB_GROUP_BROADCAST +#define MC_DQ5_HI dequantizeBlockAccum_ns_sgbroadcast_8_hi +#define MC_DQ5_LO dequantizeBlockAccum_ns_sgbroadcast_8_lo +#else +#define MC_DQ5_HI dequantizeBlockAccum_ns_sgbroadcast_1_hi +#define MC_DQ5_LO dequantizeBlockAccum_ns_sgbroadcast_1_lo +#endif +#define MC_COL_Q5K(ts, c) \ + { if (slid < 4) { regB.s0123 = read_imagef(src1, (c)*COL_STRIDE + slid*2 + k*8); \ + regB.s4567 = read_imagef(src1, (c)*COL_STRIDE + 1 + slid*2 + k*8); } \ + MC_DQ5_HI(ts, as_ushort8(regA_hi), as_uchar8(regH), regS, regM, regB); \ + MC_DQ5_LO(ts, as_ushort8(regA_lo), as_uchar8(regH), regS, regM, regB); } +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q5_k_f32_mc3( + read_only image1d_buffer_t src0_q, + read_only image1d_buffer_t src0_qh, + global half2 * src0_d, + global half2 * src0_m, + global uchar * src0_s, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2, + int n_cols) +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = NSUBGROUPS * M; + uint LINE_STRIDE_A_QH = M / 2; + uint BLOCK_STRIDE_A_QH = NSUBGROUPS * M / 2; + uint scales_per_row = (K / QK_K) * 12; + uint COL_STRIDE = K / 4; // float4 pixels per activation column + + private uint4 regA_hi, regA_lo; + private ushort4 regH; + private half2 regS, regM; + private float8 regB; + + private float2 ts0 = (float2)(0.0f); + private float2 ts1 = (float2)(0.0f); + private float2 ts2 = (float2)(0.0f); + private float2 ts3 = (float2)(0.0f); + + for (uint k = groupId; k < (K / 32); k += NSUBGROUPS) { + uint sb = k / 8; + uint j = k % 8; + + half2 d = src0_d[gid_s + sb * LINE_STRIDE_A]; + half2 dm = src0_m[gid_s + sb * LINE_STRIDE_A]; + + global const uchar * sc0 = src0_s + 2 * gid_s * scales_per_row + sb * 12; + global const uchar * sc1 = src0_s + (2 * gid_s + 1) * scales_per_row + sb * 12; + + uchar sv0, mn0, sv1, mn1; + get_scale_min_k4(j, sc0, &sv0, &mn0, mask_d6, mask_d4, mask_hi2); + get_scale_min_k4(j, sc1, &sv1, &mn1, mask_d6, mask_d4, mask_hi2); + + regS = convert_half2(convert_float2(d) * convert_float2((uchar2)(sv0, sv1))); + regM = convert_half2(convert_float2(dm) * convert_float2((uchar2)(mn0, mn1))); + + // high-bit plane + weights loaded ONCE, reused across the columns + regH.s0 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 0)).x); + regH.s1 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 1)).x); + regH.s2 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 2)).x); + regH.s3 = as_ushort(read_imageh(src0_qh, (gid_s + k * BLOCK_STRIDE_A_QH + LINE_STRIDE_A_QH * 3)).x); + + regA_hi.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA_hi.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA_hi.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA_hi.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA_lo.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA_lo.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA_lo.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA_lo.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + MC_COL_Q5K(ts0, 0); + MC_COL_Q5K(ts1, 1); + if (n_cols > 2) MC_COL_Q5K(ts2, 2); + if (n_cols > 3) MC_COL_Q5K(ts3, 3); + } + + // cross-subgroup reduce: pack the (up to 4) columns' float2 into a float8. + local float8 reduceLM[SUBGROUP_SIZE * 3]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, ts3.s0, ts3.s1); + if (groupId == 1) { reduceLM[SUBGROUP_SIZE * 0 + slid] = acc; } + if (groupId == 2) { reduceLM[SUBGROUP_SIZE * 1 + slid] = acc; } + if (groupId == 3) { reduceLM[SUBGROUP_SIZE * 2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (groupId == 0) { + acc += reduceLM[SUBGROUP_SIZE * 0 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 1 + slid]; + acc += reduceLM[SUBGROUP_SIZE * 2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // dst is column-major [M rows x n_cols cols]: (row, col) at col*M + row + if (gid < LINE_STRIDE_A) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0 * M + gid * 2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1 * M + gid * 2])); + if (n_cols > 2) vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2 * M + gid * 2])); + if (n_cols > 3) vstore2((float2)(acc.s6, acc.s7), 0, &(dst[3 * M + gid * 2])); + } + } +} +#undef MC_COL_Q5K +#undef MC_DQ5_HI +#undef MC_DQ5_LO diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32_32b_trans.cl new file mode 100644 index 0000000..ecf1513 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32_32b_trans.cl @@ -0,0 +1,141 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define K_SCALE_SIZE 12 +#define N_SIMDGROUP 8 +#define SIMDGROUP_WIDTH 64 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uint stride, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j*stride] & 63; + *m = q[(j+4)*stride] & 63; + } else { + *d = (q[(j+4)*stride] & 0x0F) | ((q[(j-4)*stride] & 0xC0) >> 2); + *m = ((q[(j+4)*stride] >> 4) & 0x0F) | ((q[j*stride] & 0xC0) >> 2); + } +} + +static inline float8 q5_k_to_fp32_packed8(ushort2 q4x8, uint qh_byte, float scale, float minv) { + float8 fp32x8; + fp32x8.s0 = (float)(( q4x8.s0 & 0x000F) | (((qh_byte >> 0) & 1) << 4)) * scale - minv; + fp32x8.s1 = (float)(((q4x8.s0 >> 4) & 0x000F) | (((qh_byte >> 1) & 1) << 4)) * scale - minv; + fp32x8.s2 = (float)(((q4x8.s0 >> 8) & 0x000F) | (((qh_byte >> 2) & 1) << 4)) * scale - minv; + fp32x8.s3 = (float)(((q4x8.s0 >> 12) & 0x000F) | (((qh_byte >> 3) & 1) << 4)) * scale - minv; + fp32x8.s4 = (float)(( q4x8.s1 & 0x000F) | (((qh_byte >> 4) & 1) << 4)) * scale - minv; + fp32x8.s5 = (float)(((q4x8.s1 >> 4) & 0x000F) | (((qh_byte >> 5) & 1) << 4)) * scale - minv; + fp32x8.s6 = (float)(((q4x8.s1 >> 8) & 0x000F) | (((qh_byte >> 6) & 1) << 4)) * scale - minv; + fp32x8.s7 = (float)(((q4x8.s1 >> 12) & 0x000F) | (((qh_byte >> 7) & 1) << 4)) * scale - minv; + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void gemv_noshuffle_q5_k_f32_32b_trans( + read_only image1d_buffer_t src0_q, + __global uchar * src0_qh, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + uint i01 = get_global_id(0); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + int num_subblocks = ne00 / 32; + + __private float sum = 0.0f; + + // Loop over sub-blocks of 32 elements, N_SIMDGROUP sub-blocks per iter. + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; + uint j = ib % 8; + + // Load d and dmin for this super-block. + half d_val = src0_d[sb * ne01 + i01]; + half dm_val = src0_dm[sb * ne01 + i01]; + + // Load sub-block scale and min. s is transposed [nb][12][M]; stride ne01 per code. + global const uchar * sc = src0_s + sb * K_SCALE_SIZE * ne01 + i01; + uchar sv, mn; + get_scale_min_k4(j, sc, ne01, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = (float)dm_val * (float)mn; + + // Load 4 uints of quants (32 nibbles = 32 elements), column-major stride ne01. + uint q_base = ib * ne01 * 4 + i01; + + uint4 regQ; + regQ.s0 = read_imageui(src0_q, q_base).x; + regQ.s1 = read_imageui(src0_q, q_base + ne01).x; + regQ.s2 = read_imageui(src0_q, q_base + ne01 * 2).x; + regQ.s3 = read_imageui(src0_q, q_base + ne01 * 3).x; + + uint qh_grp = ib * 4; + uint qh_word = (uint)src0_qh[(qh_grp + 0) * ne01 + i01] + | ((uint)src0_qh[(qh_grp + 1) * ne01 + i01] << 8) + | ((uint)src0_qh[(qh_grp + 2) * ne01 + i01] << 16) + | ((uint)src0_qh[(qh_grp + 3) * ne01 + i01] << 24); + + // Load activations: 32 floats = 8 float4s. + uint y_offset = ib * 8; + + float4 y_local = (slid < 8) ? read_imagef(src1, (y_offset + slid)) : (float4)0.0f; + float4 y0 = sub_group_broadcast(y_local, 0); + float4 y1 = sub_group_broadcast(y_local, 1); + float4 y2 = sub_group_broadcast(y_local, 2); + float4 y3 = sub_group_broadcast(y_local, 3); + float4 y4 = sub_group_broadcast(y_local, 4); + float4 y5 = sub_group_broadcast(y_local, 5); + float4 y6 = sub_group_broadcast(y_local, 6); + float4 y7 = sub_group_broadcast(y_local, 7); + + float8 fp32x8 = q5_k_to_fp32_packed8(as_ushort2(regQ.s0), qh_word & 0xFF, scale, minv); + float4 acc = y0 * fp32x8.lo; + acc += y1 * fp32x8.hi; + + fp32x8 = q5_k_to_fp32_packed8(as_ushort2(regQ.s1), (qh_word >> 8) & 0xFF, scale, minv); + acc += y2 * fp32x8.lo; + acc += y3 * fp32x8.hi; + + fp32x8 = q5_k_to_fp32_packed8(as_ushort2(regQ.s2), (qh_word >> 16) & 0xFF, scale, minv); + acc += y4 * fp32x8.lo; + acc += y5 * fp32x8.hi; + + fp32x8 = q5_k_to_fp32_packed8(as_ushort2(regQ.s3), (qh_word >> 24) & 0xFF, scale, minv); + acc += y6 * fp32x8.lo; + acc += y7 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory over N_SIMDGROUP subgroups + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid > 0) { + reduceLM[SIMDGROUP_WIDTH * (sgid - 1) + slid] = sum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) { + for (uint i = 0; i < N_SIMDGROUP - 1; ++i) { + sum += reduceLM[SIMDGROUP_WIDTH * i + slid]; + } + } + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl index 6f89cf9..a6908a6 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl @@ -219,26 +219,28 @@ kernel void kernel_gemv_noshuffle_q6_K_f32( float2 total_sum = 0.0f; int line_stride_a = ne01 / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + int gid_s = min(gid, line_stride_a - 1); int block_stride_a = NSUBGROUPS * ne01; for (int k = grp; k < nb; k += NSUBGROUPS) { - reg_d = src0_d[gid + k/8 * line_stride_a]; - reg_s = as_char4(src0_s[gid + k * line_stride_a]); + reg_d = src0_d[gid_s + k/8 * line_stride_a]; + reg_s = as_char4(src0_s[gid_s + k * line_stride_a]); if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0 + slid*2 + k*8); reg_b.s4567 = read_imagef(src1, 1 + slid*2 + k*8); } - reg_a_l.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*0).x; - reg_a_l.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*1).x; - reg_a_l.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*2).x; - reg_a_l.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*3).x; + reg_a_l.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*0).x; + reg_a_l.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*1).x; + reg_a_l.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*2).x; + reg_a_l.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*3).x; - reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*0).x); - reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*1).x); - reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*2).x); - reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*3).x); + reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*0).x); + reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*1).x); + reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*2).x); + reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*3).x); #ifdef VECTOR_SUB_GROUP_BROADCAT dequantize_block_acc_bcast_8_hi(total_sum, as_ushort8(reg_a_l), as_uchar8(reg_a_h), reg_d, reg_s, reg_b); @@ -246,15 +248,15 @@ kernel void kernel_gemv_noshuffle_q6_K_f32( dequantize_block_acc_bcast_1_hi(total_sum, as_ushort8(reg_a_l), as_uchar8(reg_a_h), reg_d, reg_s, reg_b); #endif // VECTOR_SUB_GROUP_BROADCAT - reg_a_l.s0 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*4).x; - reg_a_l.s1 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*5).x; - reg_a_l.s2 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*6).x; - reg_a_l.s3 = read_imageui(src0_ql, gid + k*block_stride_a + line_stride_a*7).x; + reg_a_l.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*4).x; + reg_a_l.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*5).x; + reg_a_l.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*6).x; + reg_a_l.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*7).x; - reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*4).x); - reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*5).x); - reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*6).x); - reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid + k*block_stride_a + line_stride_a*7).x); + reg_a_h.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*4).x); + reg_a_h.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*5).x); + reg_a_h.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*6).x); + reg_a_h.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*7).x); #ifdef VECTOR_SUB_GROUP_BROADCAT dequantize_block_acc_bcast_8_lo(total_sum, as_ushort8(reg_a_l), as_uchar8(reg_a_h), reg_d, reg_s, reg_b); @@ -288,6 +290,126 @@ kernel void kernel_gemv_noshuffle_q6_K_f32( if (grp == 0) { dst = (global float*)((global char*)dst + offsetd); - vstore2(total_sum, 0, &(dst[gid * 2])); + // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64, + // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01 + // and would overrun dst into the adjacent tensor (garbage downstream). + // No-op / byte-identical when ne01 % 128 == 0 (no padding). + if (gid * 2 + 0 < ne01) dst[gid * 2 + 0] = total_sum.s0; + if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = total_sum.s1; + } +} + +// Multi-column (N=3) q6_K decode GEMV for the spec/MTP verify batch. Same idea +// as the q4_K mc3: stay on the efficient GEMV path (subgroup broadcast, no +// transpose) instead of the transposed-GEMM dead-zone. Each K-block's weights +// (ql/qh, hi+lo) are loaded ONCE and reused across all 3 activation columns. +// Per-column accumulation is independent and identical to 3 standalone GEMVs +// => byte-identical; does NOT perturb the lm_head logits / spec accept rate. +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q6_K_f32_mc3( + read_only image1d_buffer_t src0_ql, + read_only image1d_buffer_t src0_qh, + global half2 * src0_s, + global half2 * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); + int gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + + int nb = ne00 / 32; + int line_stride_a = ne01 / 2; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + int gid_s = min(gid, line_stride_a - 1); + int block_stride_a = NSUBGROUPS * ne01; + int COL_STRIDE = ne00 / 4; // float4 pixels per activation column + + uint4 ql_hi, ql_lo; + ushort4 qh_hi, qh_lo; + half2 reg_d; + char4 reg_s; + float8 reg_b; + + float2 ts0 = 0.0f, ts1 = 0.0f, ts2 = 0.0f; + + for (int k = grp; k < nb; k += NSUBGROUPS) { + reg_d = src0_d[gid_s + k/8 * line_stride_a]; + reg_s = as_char4(src0_s[gid_s + k * line_stride_a]); + + // weights loaded ONCE (hi: blocks 0-3, lo: blocks 4-7), reused x3 cols + ql_hi.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*0).x; + ql_hi.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*1).x; + ql_hi.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*2).x; + ql_hi.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*3).x; + qh_hi.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*0).x); + qh_hi.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*1).x); + qh_hi.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*2).x); + qh_hi.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*3).x); + + ql_lo.s0 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*4).x; + ql_lo.s1 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*5).x; + ql_lo.s2 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*6).x; + ql_lo.s3 = read_imageui(src0_ql, gid_s + k*block_stride_a + line_stride_a*7).x; + qh_lo.s0 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*4).x); + qh_lo.s1 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*5).x); + qh_lo.s2 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*6).x); + qh_lo.s3 = as_ushort(read_imageh(src0_qh, gid_s + k*block_stride_a + line_stride_a*7).x); + + // Per-column: load only this column's activation (single reg_b live) -> + // 1/3 the activation register pressure, cutting the private-mem spill. +#ifdef VECTOR_SUB_GROUP_BROADCAT + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_8_hi(ts0, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_8_lo(ts0, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 1*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_8_hi(ts1, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_8_lo(ts1, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 2*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_8_hi(ts2, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_8_lo(ts2, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } +#else + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 0*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 0*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_1_hi(ts0, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_1_lo(ts0, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 1*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 1*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_1_hi(ts1, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_1_lo(ts1, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } + { if (slid < 4) { reg_b.s0123 = read_imagef(src1, 2*COL_STRIDE + 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 2*COL_STRIDE + 1 + slid*2 + k*8); } + dequantize_block_acc_bcast_1_hi(ts2, as_ushort8(ql_hi), as_uchar8(qh_hi), reg_d, reg_s, reg_b); + dequantize_block_acc_bcast_1_lo(ts2, as_ushort8(ql_lo), as_uchar8(qh_lo), reg_d, reg_s, reg_b); } +#endif + } + + local float8 reduce_lm[SUBGROUP_SIZE * 3]; + float8 acc = (float8)(ts0.s0, ts0.s1, ts1.s0, ts1.s1, ts2.s0, ts2.s1, 0.0f, 0.0f); + if (grp == 1) { reduce_lm[SUBGROUP_SIZE*0 + slid] = acc; } + if (grp == 2) { reduce_lm[SUBGROUP_SIZE*1 + slid] = acc; } + if (grp == 3) { reduce_lm[SUBGROUP_SIZE*2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + acc += reduce_lm[SUBGROUP_SIZE*0 + slid]; + acc += reduce_lm[SUBGROUP_SIZE*1 + slid]; + acc += reduce_lm[SUBGROUP_SIZE*2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row + if (gid < line_stride_a) { + vstore2((float2)(acc.s0, acc.s1), 0, &(dst[0*ne01 + gid*2])); + vstore2((float2)(acc.s2, acc.s3), 0, &(dst[1*ne01 + gid*2])); + vstore2((float2)(acc.s4, acc.s5), 0, &(dst[2*ne01 + gid*2])); + } } } diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_32b_trans.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_32b_trans.cl new file mode 100644 index 0000000..2e1e2d7 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_32b_trans.cl @@ -0,0 +1,128 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable + +#define QK_K 256 +#define N_SIMDGROUP 8 +#define SIMDGROUP_WIDTH 64 + +static inline float8 q6_k_to_fp32_packed8(ushort2 ql8, ushort qh8, float d_scale) { + float8 fp32x8; + fp32x8.s0 = ((float)(( ql8.s0 & 0x000F) | ((uint)((qh8 ) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s1 = ((float)((( ql8.s0 >> 4) & 0x000F) | ((uint)((qh8 >> 2) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s2 = ((float)((( ql8.s0 >> 8) & 0x000F) | ((uint)((qh8 >> 4) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s3 = ((float)((( ql8.s0 >> 12)& 0x000F) | ((uint)((qh8 >> 6) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s4 = ((float)(( ql8.s1 & 0x000F) | ((uint)((qh8 >> 8) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s5 = ((float)((( ql8.s1 >> 4) & 0x000F) | ((uint)((qh8 >>10) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s6 = ((float)((( ql8.s1 >> 8) & 0x000F) | ((uint)((qh8 >>12) & 0x3) << 4)) - 32.f) * d_scale; + fp32x8.s7 = ((float)((( ql8.s1 >> 12)& 0x000F) | ((uint)((qh8 >>14) & 0x3) << 4)) - 32.f) * d_scale; + return fp32x8; +} + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_noshuffle_q6_k_f32_32b_trans( + __read_only image1d_buffer_t src0_ql, + __read_only image1d_buffer_t src0_qh, + __global char * src0_s, + __global half * src0_d, + __read_only image1d_buffer_t src1, + __global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + uint i01 = get_global_id(0); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + int num_superblocks = ne00 / QK_K; + int num_subblocks = ne00 / 32; // 2 sub-blocks of 16 processed per iter below + int scales_per_row = num_superblocks * 16; + + __private float sum = 0.0f; + + // Loop over 32-element groups (2 sub-blocks of 16 each), N_SIMDGROUP groups per iter. + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; // super-block index + uint j = ib % 8; // 32-element group within super-block (0..7) + + // Load d for this super-block. + half d_val = src0_d[sb * ne01 + i01]; + + // Load 2 sub-block scales (int8), one per 16 elements. + global const char * sc = src0_s + i01 * scales_per_row + sb * 16; + float scale0 = (float)d_val * (float)sc[j * 2]; + float scale1 = (float)d_val * (float)sc[j * 2 + 1]; + + // Load 4 uints of ql (32 elements, 4-bit each = 128 bits), column-major stride ne01. + uint ql_base = (ib * 4) * ne01 + i01; + uint4 regQL; + regQL.s0 = read_imageui(src0_ql, ql_base).x; + regQL.s1 = read_imageui(src0_ql, ql_base + ne01).x; + regQL.s2 = read_imageui(src0_ql, ql_base + ne01 * 2).x; + regQL.s3 = read_imageui(src0_ql, ql_base + ne01 * 3).x; + + // Load 2 uints of qh (32 elements, 2-bit each = 64 bits), column-major stride ne01. + uint qh_base = (ib * 2) * ne01 + i01; + uint2 regQH; + regQH.s0 = read_imageui(src0_qh, qh_base).x; + regQH.s1 = read_imageui(src0_qh, qh_base + ne01).x; + + // Load activations: 32 floats = 8 float4s. + uint y_offset = ib * 8; + + float4 y_local = (slid < 8) ? read_imagef(src1, (y_offset + slid)) : (float4)0.0f; + float4 y0 = sub_group_broadcast(y_local, 0); + float4 y1 = sub_group_broadcast(y_local, 1); + float4 y2 = sub_group_broadcast(y_local, 2); + float4 y3 = sub_group_broadcast(y_local, 3); + float4 y4v = sub_group_broadcast(y_local, 4); + float4 y5 = sub_group_broadcast(y_local, 5); + float4 y6 = sub_group_broadcast(y_local, 6); + float4 y7 = sub_group_broadcast(y_local, 7); + + // Dequantize elements 0..7 (scale0). + float8 fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s0), (ushort)(regQH.s0 & 0xFFFF), scale0); + + float4 acc = y0 * fp32x8.lo; + acc += y1 * fp32x8.hi; + + // Dequantize elements 8..15 (scale0). + fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s1), (ushort)(regQH.s0 >> 16), scale0); + + acc += y2 * fp32x8.lo; + acc += y3 * fp32x8.hi; + + // Dequantize elements 16..23 (scale1). + fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s2), (ushort)(regQH.s1 & 0xFFFF), scale1); + + acc += y4v * fp32x8.lo; + acc += y5 * fp32x8.hi; + + // Dequantize elements 24..31 (scale1). + fp32x8 = q6_k_to_fp32_packed8(as_ushort2(regQL.s3), (ushort)(regQH.s1 >> 16), scale1); + + acc += y6 * fp32x8.lo; + acc += y7 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + // reduction in local memory, assumes #subgroups=4 + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid > 0) { + reduceLM[SIMDGROUP_WIDTH * (sgid - 1) + slid] = sum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) { + for (uint i = 0; i < N_SIMDGROUP - 1; ++i) { + sum += reduceLM[SIMDGROUP_WIDTH * i + slid]; + } + } + + // 1 output per thread in subgroup 0 + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl new file mode 100644 index 0000000..22efb6c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl @@ -0,0 +1,373 @@ +// 4-output-per-WI variant of kernel_gemv_noshuffle_q6_K_f32. +// Each WI now produces 4 consecutive outputs (output quad). The activation +// fetch (reg_b) is shared across all 4 outputs, doubling per-WI ALU per +// activation broadcast and halving the WG count vs the 2-output kernel. +// +// Implementation: each K-block we fetch TWO sets of (scales + ql + qh) +// — one for the low pair (rows 0,1 of the quad) and one for the high pair +// (rows 2,3) — and invoke the existing 2-output dequant macros twice +// against the *same* reg_b. Identical data layout to the 2-output kernel, +// so the host only needs to halve the grid and double the gid-to-output +// mapping. +// +// Opt-in via the host dispatch when GGML_OPENCL_Q6K_GEMV_O4=1. + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define NSUBGROUPS 4 +#define SUBGROUP_SIZE 64 + +// Macros are identical to the 2-output kernel — they accept `total_sum` as +// a parameter so we can call them twice (once per pair) against different +// accumulators against the same reg_b. +#define dequantize_block_acc_bcast_8_hi(total_sum, bits4, bits2, cs, y) \ + float8 shared_y; \ + shared_y = sub_group_broadcast(y, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s0; \ + total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s1; \ + total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s0 * shared_y.s2; \ + total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s3; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s4; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s5; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s0 * shared_y.s6; \ + total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s7; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s0; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s1; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s2 * shared_y.s2; \ + total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s3; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s4; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s5; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s2 * shared_y.s6; \ + total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s0; \ + total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s1; \ + total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s0 * shared_y.s2; \ + total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s3; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y.s4; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y.s5; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s0 * shared_y.s6; \ + total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y.s7; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s0; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s1; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s2 * shared_y.s2; \ + total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s3; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y.s4; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y.s5; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s2 * shared_y.s6; \ + total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y.s7; \ + +#define dequantize_block_acc_bcast_8_lo(total_sum, bits4, bits2, cs, y) \ + shared_y = sub_group_broadcast(y, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s0; \ + total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s1; \ + total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s1 * shared_y.s2; \ + total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s3; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s4; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s5; \ + total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s1 * shared_y.s6; \ + total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s7; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s0; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s1; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s3 * shared_y.s2; \ + total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s3; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s4; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s5; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s3 * shared_y.s6; \ + total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s7; \ + shared_y = sub_group_broadcast(y, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s0; \ + total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s1; \ + total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s1 * shared_y.s2; \ + total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s3; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y.s4; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y.s5; \ + total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s1 * shared_y.s6; \ + total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y.s7; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s0; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s1; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s3 * shared_y.s2; \ + total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s3; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y.s4; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y.s5; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s3 * shared_y.s6; \ + total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y.s7; \ + +#define dequantize_block_acc_bcast_1_hi(total_sum, bits4, bits2, cs, y) \ + float shared_y; \ + shared_y = sub_group_broadcast(y.s0, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 0); \ + total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 0); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 0); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 0); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 0); \ + total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 1); \ + total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 1); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 1); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 1); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s2 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 1); \ + total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s0 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s2 * shared_y; \ + +#define dequantize_block_acc_bcast_1_lo(total_sum, bits4, bits2, cs, y) \ + shared_y = sub_group_broadcast(y.s0, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x000F) ) | ((bits2.s0 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x000F) ) | ((bits2.s1 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x00F0) >> 4) | ((bits2.s0 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x00F0) >> 4) | ((bits2.s1 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0x0F00) >> 8) | ((bits2.s0 & 0x30) )) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0x0F00) >> 8) | ((bits2.s1 & 0x30) )) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 2); \ + total_sum.s0 += ((float)(((bits4.s0 & 0xF000) >> 12) | ((bits2.s0 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s1 & 0xF000) >> 12) | ((bits2.s1 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 2); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x000F) ) | ((bits2.s2 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x000F) ) | ((bits2.s3 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 2); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x00F0) >> 4) | ((bits2.s2 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x00F0) >> 4) | ((bits2.s3 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 2); \ + total_sum.s0 += ((float)(((bits4.s2 & 0x0F00) >> 8) | ((bits2.s2 & 0x30) )) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0x0F00) >> 8) | ((bits2.s3 & 0x30) )) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 2); \ + total_sum.s0 += ((float)(((bits4.s2 & 0xF000) >> 12) | ((bits2.s2 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s3 & 0xF000) >> 12) | ((bits2.s3 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s0, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x000F) ) | ((bits2.s4 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x000F) ) | ((bits2.s5 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s1, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x00F0) >> 4) | ((bits2.s4 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x00F0) >> 4) | ((bits2.s5 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s2, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0x0F00) >> 8) | ((bits2.s4 & 0x30) )) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0x0F00) >> 8) | ((bits2.s5 & 0x30) )) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s3, 3); \ + total_sum.s0 += ((float)(((bits4.s4 & 0xF000) >> 12) | ((bits2.s4 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s5 & 0xF000) >> 12) | ((bits2.s5 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s4, 3); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x000F) ) | ((bits2.s6 & 0x03) << 4)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x000F) ) | ((bits2.s7 & 0x03) << 4)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s5, 3); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x00F0) >> 4) | ((bits2.s6 & 0x0C) << 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x00F0) >> 4) | ((bits2.s7 & 0x0C) << 2)) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s6, 3); \ + total_sum.s0 += ((float)(((bits4.s6 & 0x0F00) >> 8) | ((bits2.s6 & 0x30) )) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0x0F00) >> 8) | ((bits2.s7 & 0x30) )) - 32.f) * cs.s3 * shared_y; \ + shared_y = sub_group_broadcast(y.s7, 3); \ + total_sum.s0 += ((float)(((bits4.s6 & 0xF000) >> 12) | ((bits2.s6 & 0xC0) >> 2)) - 32.f) * cs.s1 * shared_y; \ + total_sum.s1 += ((float)(((bits4.s7 & 0xF000) >> 12) | ((bits2.s7 & 0xC0) >> 2)) - 32.f) * cs.s3 * shared_y; \ + +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +// Q6K_O4_GLOBAL: read the (read-once-per-token, no-reuse) lm_head/embed weights +// from __global coalesced instead of image1d_buffer. The texture cache caps the +// streaming (no-reuse) lm_head read bandwidth; global coalesced reaches the +// higher rate the rest of the model gets. src1 (activation) stays an image (it IS reused via +// the cross-subgroup broadcast). +#ifdef Q6K_O4_GLOBAL +#define Q6K_O4_NAME kernel_gemv_noshuffle_q6_K_f32_o4_global +#define QL_ARG __global uint * src0_ql +#define QH_ARG __global half * src0_qh +#define RD_QL(b,i) (b[i]) +#define RD_QH(b,i) as_ushort(b[i]) +#else +#define Q6K_O4_NAME kernel_gemv_noshuffle_q6_K_f32_o4 +#define QL_ARG read_only image1d_buffer_t src0_ql +#define QH_ARG read_only image1d_buffer_t src0_qh +#define RD_QL(b,i) (read_imageui(b,i).x) +#define RD_QH(b,i) as_ushort(read_imageh(b,i).x) +#endif +kernel void Q6K_O4_NAME( + QL_ARG, + QH_ARG, + global half2 * src0_s, + global half2 * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); + int gid = get_global_id(0); // 4-output-quad index + ushort slid = get_sub_group_local_id(); + + // Map quad index to the two pair-indices the existing 2-output access + // pattern uses (consecutive output pairs along ne01). NB: the two pairs are + // kept ADJACENT (gid*2, gid*2+1) on purpose -- a "stride-1" split (pairs + // ne01/4 apart) is slower because two distant cache-line streams have worse + // locality than the adjacent pair whose reads interleave into the same lines + // each iteration. + // Keep tail lanes active for subgroup ops, but read valid weight pairs. + int gid_a = min(gid * 2, ne01 / 2 - 1); + int gid_b = min(gid * 2 + 1, ne01 / 2 - 1); + + int nb = ne00 / 32; + + uint4 reg_a_l_a, reg_a_l_b; + ushort4 reg_a_h_a, reg_a_h_b; + half2 reg_d_a, reg_d_b; + char4 reg_s_a, reg_s_b; + float8 reg_b; + + float2 total_sum_a = 0.0f; + float2 total_sum_b = 0.0f; + + int line_stride_a = ne01 / 2; + int block_stride_a = NSUBGROUPS * ne01; + + for (int k = grp; k < nb; k += NSUBGROUPS) { + reg_d_a = src0_d[gid_a + k/8 * line_stride_a]; + reg_d_b = src0_d[gid_b + k/8 * line_stride_a]; + reg_s_a = as_char4(src0_s[gid_a + k * line_stride_a]); + reg_s_b = as_char4(src0_s[gid_b + k * line_stride_a]); + // Precompute the loop-invariant combined scale (sub-block scale * super-block d) + // once per pair instead of re-multiplying it for every one of the 256 elements. + float4 cs_a = (float4)((float)reg_s_a.s0*(float)reg_d_a.s0, (float)reg_s_a.s1*(float)reg_d_a.s0, + (float)reg_s_a.s2*(float)reg_d_a.s1, (float)reg_s_a.s3*(float)reg_d_a.s1); + float4 cs_b = (float4)((float)reg_s_b.s0*(float)reg_d_b.s0, (float)reg_s_b.s1*(float)reg_d_b.s0, + (float)reg_s_b.s2*(float)reg_d_b.s1, (float)reg_s_b.s3*(float)reg_d_b.s1); + + if (slid < 4) { + reg_b.s0123 = read_imagef(src1, 0 + slid*2 + k*8); + reg_b.s4567 = read_imagef(src1, 1 + slid*2 + k*8); + } + + // Pair a (output rows gid_a*2, gid_a*2+1): read hi+lo then dequant + // both in one block so the `_lo` macro can see the `shared_y` that + // `_hi` declared. Pair b follows in its own block — fresh shared_y. + { + reg_a_l_a.s0 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*0); + reg_a_l_a.s1 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*1); + reg_a_l_a.s2 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*2); + reg_a_l_a.s3 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*3); + reg_a_h_a.s0 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*0); + reg_a_h_a.s1 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*1); + reg_a_h_a.s2 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*2); + reg_a_h_a.s3 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*3); +#ifdef VECTOR_SUB_GROUP_BROADCAT + dequantize_block_acc_bcast_8_hi(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b); +#else + dequantize_block_acc_bcast_1_hi(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b); +#endif + + reg_a_l_a.s0 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*4); + reg_a_l_a.s1 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*5); + reg_a_l_a.s2 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*6); + reg_a_l_a.s3 = RD_QL(src0_ql, gid_a + k*block_stride_a + line_stride_a*7); + reg_a_h_a.s0 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*4); + reg_a_h_a.s1 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*5); + reg_a_h_a.s2 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*6); + reg_a_h_a.s3 = RD_QH(src0_qh, gid_a + k*block_stride_a + line_stride_a*7); +#ifdef VECTOR_SUB_GROUP_BROADCAT + dequantize_block_acc_bcast_8_lo(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b); +#else + dequantize_block_acc_bcast_1_lo(total_sum_a, as_ushort8(reg_a_l_a), as_uchar8(reg_a_h_a), cs_a, reg_b); +#endif + } + + { + reg_a_l_b.s0 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*0); + reg_a_l_b.s1 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*1); + reg_a_l_b.s2 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*2); + reg_a_l_b.s3 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*3); + reg_a_h_b.s0 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*0); + reg_a_h_b.s1 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*1); + reg_a_h_b.s2 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*2); + reg_a_h_b.s3 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*3); +#ifdef VECTOR_SUB_GROUP_BROADCAT + dequantize_block_acc_bcast_8_hi(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b); +#else + dequantize_block_acc_bcast_1_hi(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b); +#endif + + reg_a_l_b.s0 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*4); + reg_a_l_b.s1 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*5); + reg_a_l_b.s2 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*6); + reg_a_l_b.s3 = RD_QL(src0_ql, gid_b + k*block_stride_a + line_stride_a*7); + reg_a_h_b.s0 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*4); + reg_a_h_b.s1 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*5); + reg_a_h_b.s2 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*6); + reg_a_h_b.s3 = RD_QH(src0_qh, gid_b + k*block_stride_a + line_stride_a*7); +#ifdef VECTOR_SUB_GROUP_BROADCAT + dequantize_block_acc_bcast_8_lo(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b); +#else + dequantize_block_acc_bcast_1_lo(total_sum_b, as_ushort8(reg_a_l_b), as_uchar8(reg_a_h_b), cs_b, reg_b); +#endif + } + } + + // Cross-subgroup reduce. Same shape as the 2-output kernel but with the + // pair-a and pair-b accumulators concatenated into a single float4. + local float4 reduce_lm[SUBGROUP_SIZE * 3]; + float4 acc = (float4)(total_sum_a.s0, total_sum_a.s1, total_sum_b.s0, total_sum_b.s1); + if (grp == 1) { reduce_lm[SUBGROUP_SIZE*0 + slid] = acc; } + if (grp == 2) { reduce_lm[SUBGROUP_SIZE*1 + slid] = acc; } + if (grp == 3) { reduce_lm[SUBGROUP_SIZE*2 + slid] = acc; } + + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + acc += reduce_lm[SUBGROUP_SIZE*0 + slid]; + acc += reduce_lm[SUBGROUP_SIZE*1 + slid]; + acc += reduce_lm[SUBGROUP_SIZE*2 + slid]; + dst = (global float*)((global char*)dst + offsetd); + // The dispatch rounds ne01/4 up to the subgroup width, so the tail + // quads past the last row must not store (they wrote 128 rows past + // dst on every ne01 % 256 == 128 vocab, e.g. 151936). + if (gid * 4 + 3 < (uint)ne01) { + vstore4(acc, 0, &(dst[gid * 4])); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl new file mode 100644 index 0000000..c5049f3 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl @@ -0,0 +1,196 @@ +// Tiled-wide q6_K GEMV for the long-vocab lm_head/embed (decode path). +// +// Pairs with kernel_convert_block_q6_k_tiled_ns (cvt.cl): the weights are laid +// out CANONICALLY (6-bit code in element order e in [0,256)) and TILED by 64 +// output rows so the 64-thread lane group coalesces every weight load. Both the +// pack (convert) and the unpack (here) are owned by us — correct by construction +// against the reference ggml q6_K dequant, no bit-interleave reverse-engineering. +// +// One work-item produces one output row. A work-group is {64 lanes, 4 subgroups}: +// the 64 lanes cover the 64 rows of one tile (coalesced reads), the 4 subgroups +// split the K-blocks and reduce through __local at the end. +// +// Weights are read from __global (coalesced) rather than image1d_buffer: the +// lm_head is read once per token with no reuse, and the Adreno texture cache +// caps such a streaming read well below the coalesced-global rate +// (see opencl_q6k_gemv_o4_shipped / x2-90 roofline notes). + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#endif + +#define NSUBGROUPS 4 +#define TILE_ROWS 64 + +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q6_K_f32_tiled( + __global uint4 * src0_ql, // tiled: 8 uint4 granules / superblock + __global uint4 * src0_qh, // tiled: 4 uint4 granules / superblock + __global char * src0_s, // tiled: 16 chars / superblock + __global half * src0_d, // tiled: 1 half / superblock + read_only image1d_buffer_t src1, // activation (RGBA f32) + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); // subgroup index 0..3 (splits K) + int row = get_global_id(0); // output row along ne01 + int rt = row / TILE_ROWS; + int rit = row % TILE_ROWS; + + int nb = ne00 / 256; // superblocks per row + + float acc = 0.0f; + + for (int sb = grp; sb < nb; sb += NSUBGROUPS) { + int tile_blk = rt * nb + sb; // ne02 == 1 for lm_head/embed + + // d + 16 scales for this (row, superblock) + float dval = (float)src0_d[tile_blk * TILE_ROWS + rit]; + __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16; + + // 32 ql-uints (8 codes/uint) + 16 qh-uints (16 codes/uint) + uint ql[32]; + uint qh[16]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit]; + ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w; + } + #pragma unroll + for (int g = 0; g < 4; ++g) { + uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit]; + qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w; + } + + // dequant 256 codes in canonical e-order, MAC with activation. + int act_base = sb * 64; // activation float4 pixel base (256/4) + #pragma unroll + for (int e4 = 0; e4 < 64; ++e4) { + float4 a = read_imagef(src1, act_base + e4); + #pragma unroll + for (int t = 0; t < 4; ++t) { + int e = e4 * 4 + t; + uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF; + uint hi2 = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3; + int code = (int)(low4 | (hi2 << 4)) - 32; + int sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1); + float scale = (float)sc[sidx] * dval; + float av = (t == 0) ? a.x : (t == 1) ? a.y : (t == 2) ? a.z : a.w; + acc += (float)code * scale * av; + } + } + } + + // reduce across the NSUBGROUPS subgroups (same rit, different K-subset) + local float reduce_lm[NSUBGROUPS * TILE_ROWS]; + reduce_lm[grp * TILE_ROWS + rit] = acc; + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + float total = reduce_lm[0 * TILE_ROWS + rit] + + reduce_lm[1 * TILE_ROWS + rit] + + reduce_lm[2 * TILE_ROWS + rit] + + reduce_lm[3 * TILE_ROWS + rit]; + dst = (global float*)((global char*)dst + offsetd); + dst[row] = total; + } +} + +// Multi-column (N=3) variant of the tiled q6_K decode GEMV, for the speculative/ +// MTP VERIFY lm_head/embed (ne1=3 = 2 drafts + 1 bonus). Identical tiled weight +// layout + unpack as the ne1=1 kernel above; each WI computes 3 output columns, +// streaming the (large) lm_head weight ONCE per superblock and reusing it across +// the 3 verify activation columns (dequant once per code, MAC into 3 accs). This +// is the lm_head analogue of the per-layer mc3 GEMV; the multiply order matches +// the ne1=1 kernel, so each column is byte-identical to a standalone tiled GEMV. +#if defined(ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_gemv_noshuffle_q6_K_f32_tiled_mc3( + __global uint4 * src0_ql, + __global uint4 * src0_qh, + __global char * src0_s, + __global half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int ne00, + int ne01 +) { + int grp = get_local_id(1); + int row = get_global_id(0); + int rt = row / TILE_ROWS; + int rit = row % TILE_ROWS; + + int nb = ne00 / 256; + int col_stride = ne00 / 4; // activation float4 pixels per column + + float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f; + + for (int sb = grp; sb < nb; sb += NSUBGROUPS) { + int tile_blk = rt * nb + sb; + + float dval = (float)src0_d[tile_blk * TILE_ROWS + rit]; + __global char * sc = src0_s + (tile_blk * TILE_ROWS + rit) * 16; + + uint ql[32]; + uint qh[16]; + #pragma unroll + for (int g = 0; g < 8; ++g) { + uint4 v = src0_ql[(tile_blk * 8 + g) * TILE_ROWS + rit]; + ql[g*4+0] = v.x; ql[g*4+1] = v.y; ql[g*4+2] = v.z; ql[g*4+3] = v.w; + } + #pragma unroll + for (int g = 0; g < 4; ++g) { + uint4 v = src0_qh[(tile_blk * 4 + g) * TILE_ROWS + rit]; + qh[g*4+0] = v.x; qh[g*4+1] = v.y; qh[g*4+2] = v.z; qh[g*4+3] = v.w; + } + + int act_base = sb * 64; + #pragma unroll + for (int e4 = 0; e4 < 64; ++e4) { + float4 a0 = read_imagef(src1, 0*col_stride + act_base + e4); + float4 a1 = read_imagef(src1, 1*col_stride + act_base + e4); + float4 a2 = read_imagef(src1, 2*col_stride + act_base + e4); + #pragma unroll + for (int t = 0; t < 4; ++t) { + int e = e4 * 4 + t; + uint low4 = (ql[e >> 3] >> ((e & 7) * 4)) & 0xF; + uint hi2 = (qh[e >> 4] >> ((e & 15) * 2)) & 0x3; + int code = (int)(low4 | (hi2 << 4)) - 32; + int sidx = ((e >> 7) << 3) + (((e >> 5) & 3) << 1) + ((e >> 4) & 1); + float w = (float)code * ((float)sc[sidx] * dval); // dequant+scale once + float av0 = (t == 0) ? a0.x : (t == 1) ? a0.y : (t == 2) ? a0.z : a0.w; + float av1 = (t == 0) ? a1.x : (t == 1) ? a1.y : (t == 2) ? a1.z : a1.w; + float av2 = (t == 0) ? a2.x : (t == 1) ? a2.y : (t == 2) ? a2.z : a2.w; + acc0 += w * av0; + acc1 += w * av1; + acc2 += w * av2; + } + } + } + + local float4 reduce_lm[NSUBGROUPS * TILE_ROWS]; + reduce_lm[grp * TILE_ROWS + rit] = (float4)(acc0, acc1, acc2, 0.0f); + barrier(CLK_LOCAL_MEM_FENCE); + + if (grp == 0) { + float4 total = reduce_lm[0 * TILE_ROWS + rit] + + reduce_lm[1 * TILE_ROWS + rit] + + reduce_lm[2 * TILE_ROWS + rit] + + reduce_lm[3 * TILE_ROWS + rit]; + dst = (global float*)((global char*)dst + offsetd); + // dst column-major [ne01 rows x 3 cols]: (row, col) at col*ne01 + row + dst[0*ne01 + row] = total.x; + dst[1*ne01 + row] = total.y; + dst[2*ne01 + row] = total.z; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general_q8_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl similarity index 60% rename from ggml/src/ggml-opencl/kernels/gemv_noshuffle_general_q8_0_f32.cl rename to ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl index 9703b69..6109c96 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general_q8_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl @@ -118,6 +118,89 @@ elem = (char)((bits8.s7 & 0xFF000000) >> 24); \ total_sums += convert_int(elem) * scale * shared_y; \ +// ============================================================================ +// Split-K variant for small-M decode GEMVs. +// ---------------------------------------------------------------------------- +// The base kernel below puts one output row per lane and splits K only across +// the N_SIMDGROUP subgroups of a single workgroup, so M=512 yields M/64 = 8 +// workgroups -- half the compute units on a 16-CU X2 sit idle, and the kernel +// measures ~48 GB/s against the ~122 GB/s the larger projections reach in the +// same graph. Here each (kslice, subgroup) pair reduces a disjoint set of +// K-blocks into partial[kslice * M + row]; kernel_gemv_splitk_reduce_f32 (in +// gemv_noshuffle_q4_k_f32.cl) sums the slices. Same operand order within a +// slice as the base kernel; only the cross-slice grouping differs. +// +// Placed BEFORE the base kernel deliberately: on A6X no kernel may be defined +// after one that uses a subgroup builtin, or it silently miscompiles. +// ============================================================================ +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +__kernel void kernel_gemv_noshuffle_q8_0_f32_splitk( + __read_only image1d_buffer_t src0_q, // quantized A (weights) + global half * src0_d, // A scales + __read_only image1d_buffer_t src1, // B (activations) + global float * partial, // [ksplit * M], slice-major + int ne00, // K + int ne01) // M +{ + uint groupId = get_local_id(1); + uint gid = get_global_id(0); + ushort slid = get_sub_group_local_id(); + uint nsg = get_local_size(1); + uint ksplit = get_num_groups(1); + uint kslice = get_group_id(1); + + uint K = ne00; + uint M = ne01; + + uint LINE_STRIDE_A = M; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); + uint BLOCK_STRIDE_A = 8 * M; // physical, independent of the K-split + + __private uint8 regA; + __private half regS; + __private float8 regB; + __private float totalSum = (float)(0.0f); + + #pragma unroll 1 + for (uint k = kslice * nsg + groupId; k < (K / QK8_0); k += ksplit * nsg) { + regS = src0_d[gid_s + k * LINE_STRIDE_A]; + if (slid < 4) { + regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); + regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8)); + } + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s4 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s5 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s6 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s7 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, convert_float(regS), regB); + } + + // Intra-workgroup reduce across this K-slice's subgroups. Sized for + // nsg <= 8; the host never dispatches more. + __local float reduceLM[SIMDGROUP_WIDTH * 7]; + if (groupId > 0) { + reduceLM[SIMDGROUP_WIDTH * (groupId - 1) + slid] = totalSum; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (groupId == 0) { + for (uint i = 0; i < nsg - 1; ++i) { + totalSum += reduceLM[SIMDGROUP_WIDTH * i + slid]; + } + // x-grid is padded to CEIL_DIV(M,wave)*wave; guard the tail rows. + if (gid < M) { + partial[kslice * M + gid] = totalSum; + } + } +} + #ifdef ADRENO_GPU REQD_SUBGROUP_SIZE_64 #endif @@ -146,6 +229,8 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32( uint M = ne01; uint LINE_STRIDE_A = M; + // Keep tail lanes active for subgroup ops, but read a valid weight row. + uint gid_s = min(gid, LINE_STRIDE_A - 1); uint BLOCK_STRIDE_A = 8 * M; // 32 / 4 = 8 __private uint8 regA; @@ -157,7 +242,7 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32( // loop along K in block granularity, skip 4 blocks every iter #pragma unroll 1 /* tell compiler not to unroll */ for (uint k = groupId; k < (K / QK8_0); k += N_SIMDGROUP) { - regS = src0_d[gid + k * LINE_STRIDE_A]; // each fiber loads scale of one rows + regS = src0_d[gid_s + k * LINE_STRIDE_A]; // each fiber loads the scale of one row // first 4 fibers in each wave load 8 B values to its private scope if (slid < 4) { regB.s0123 = read_imagef(src1, (slid * 2 + k * 8)); @@ -165,16 +250,16 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32( } // load weights for one block in consecutive rows - regA.s0 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; - regA.s1 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; - regA.s2 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; - regA.s3 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; - regA.s4 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; - regA.s5 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; - regA.s6 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; - regA.s7 = read_imageui(src0_q, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; - - dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, regS, regB); + regA.s0 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x; + regA.s1 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x; + regA.s2 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x; + regA.s3 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x; + regA.s4 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x; + regA.s5 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x; + regA.s6 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x; + regA.s7 = read_imageui(src0_q, (gid_s + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x; + + dequantizeBlockAccum_ns_sgbroadcast_1(totalSum, regA, convert_float(regS), regB); } // reduction in local memory, assumes #wave=4 @@ -190,6 +275,10 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32( // 1 outputs per fiber in wave 0 if (groupId == 0) { dst = (global float*)((global char*)dst + offsetd); - dst[gid] = totalSum; + // Guard the output row. The x-grid is padded to CEIL_DIV(M,wavesize)*wavesize, + // so when ne01 is not a multiple of the wave size the tail work-items run past + // row ne01 and would overrun dst into the adjacent tensor. No-op / byte-identical + // when ne01 is wave-aligned (no padding). + if (gid < M) dst[gid] = totalSum; } } diff --git a/ggml/src/ggml-opencl/kernels/get_rows.cl b/ggml/src/ggml-opencl/kernels/get_rows.cl index c2962ed..f060e46 100644 --- a/ggml/src/ggml-opencl/kernels/get_rows.cl +++ b/ggml/src/ggml-opencl/kernels/get_rows.cl @@ -82,21 +82,74 @@ kernel void kernel_get_rows_f32( src1 = (global int*)((global char*)src1 + offset1); dst = (global float*)((global char*)dst + offsetd); - int i10 = get_group_id(0); - int i11 = get_group_id(1); - int i12 = get_group_id(2); + int nchunks = get_num_groups(0) / ne10; + int g = get_group_id(0); + int i10 = g / nchunks; + int chunk = g - i10 * nchunks; + int i11 = get_group_id(1); + int i12 = get_group_id(2); int r = ((global int *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0]; int i02 = i11; int i03 = i12; - for (int ind = get_local_id(0); ind < ne00; ind += get_local_size(0)) { - if (ind >= ne00) { - return; - } - ((global float *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1))[ind] = - ((global float *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03))[ind]; + global float * dst_row = (global float *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1); + global float * src_row = (global float *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03); + + int span = (ne00 + nchunks - 1) / nchunks; + int start = chunk * span; + int end = min(start + span, ne00); + + for (int ind = start + get_local_id(0); ind < end; ind += get_local_size(0)) { + dst_row[ind] = src_row[ind]; + } +} + +kernel void kernel_get_rows_i32( + global void * src0, + ulong offset0, + global int * src1, + ulong offset1, + global int * dst, + ulong offsetd, + int ne00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global void*)((global char*)src0 + offset0); + src1 = (global int*)((global char*)src1 + offset1); + dst = (global int*)((global char*)dst + offsetd); + + int nchunks = get_num_groups(0) / ne10; + int g = get_group_id(0); + int i10 = g / nchunks; + int chunk = g - i10 * nchunks; + int i11 = get_group_id(1); + int i12 = get_group_id(2); + + int r = ((global int *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0]; + + int i02 = i11; + int i03 = i12; + + global int * dst_row = (global int *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1); + global int * src_row = (global int *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03); + + int span = (ne00 + nchunks - 1) / nchunks; + int start = chunk * span; + int end = min(start + span, ne00); + + for (int ind = start + get_local_id(0); ind < end; ind += get_local_size(0)) { + dst_row[ind] = src_row[ind]; } } diff --git a/ggml/src/ggml-opencl/kernels/glu.cl b/ggml/src/ggml-opencl/kernels/glu.cl index 059a4bb..30bad00 100644 --- a/ggml/src/ggml-opencl/kernels/glu.cl +++ b/ggml/src/ggml-opencl/kernels/glu.cl @@ -243,6 +243,71 @@ kernel void kernel_swiglu_oai( } } +//------------------------------------------------------------------------------ +// swiglu_clamp +//------------------------------------------------------------------------------ +kernel void kernel_swiglu_clamp( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + ulong nb01, + ulong nb11, + int ne0, + ulong nb1, + int ne00_off, + int ne10_off, + float limit +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global char*)((global char*)dst + offsetd); + + global float * src0_row = (global float *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off; + global float * src1_row = (global float *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off; + global float * dst_row = (global float *) ((global char *) dst + get_group_id(0)*nb1); + + for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { + const float gate = min(src0_row[i0], limit); + const float up = clamp(src1_row[i0], -limit, limit); + + dst_row[i0] = gate / (1.0f + exp(-gate)) * up; + } +} + +kernel void kernel_swiglu_clamp_f16( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + ulong nb01, + ulong nb11, + int ne0, + ulong nb1, + int ne00_off, + int ne10_off, + float limit +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global char*)((global char*)dst + offsetd); + + global half * src0_row = (global half *) ((global char *) src0 + get_group_id(0)*nb01) + ne00_off; + global half * src1_row = (global half *) ((global char *) src1 + get_group_id(0)*nb11) + ne10_off; + global half * dst_row = (global half *) ((global char *) dst + get_group_id(0)*nb1); + + for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { + const float gate = min((float) src0_row[i0], limit); + const float up = clamp((float) src1_row[i0], -limit, limit); + + dst_row[i0] = (half) (gate / (1.0f + exp(-gate)) * up); + } +} + //------------------------------------------------------------------------------ // geglu_erf //------------------------------------------------------------------------------ diff --git a/ggml/src/ggml-opencl/kernels/im2col_3d_f16.cl b/ggml/src/ggml-opencl/kernels/im2col_3d_f16.cl new file mode 100644 index 0000000..a165e44 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/im2col_3d_f16.cl @@ -0,0 +1,90 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +kernel void kernel_im2col_3d_f16( + global float * src1, + ulong offset1, + global half * dst, + ulong offsetd, + long IC, + long ID, + long IH, + long IW, + long KD, + long KH, + long KW, + long OD, + long OH, + long OW, + long N, + long stride_x, + long stride_y, + long stride_z, + long stride_q, + int s0, + int s1, + int s2, + int p0, + int p1, + int p2, + int d0, + int d1, + int d2 +) { + const int ic = (int)IC; + const int kd = (int)KD; + const int kh = (int)KH; + const int kw = (int)KW; + const int od = (int)OD; + const int oh = (int)OH; + const int ow = (int)OW; + const int n = (int)N; + + const int KD_KH_KW = kd * kh * kw; + const int KH_KW = kh * kw; + const int IC_KD_KH_KW = ic * KD_KH_KW; + + const int i = (int)get_global_id(0); + if (i >= IC_KD_KH_KW) { + return; + } + + const int iow = (int)get_global_id(1); + if (iow >= ow) { + return; + } + + const int OD_OH = od * oh; + const int iz = (int)get_global_id(2); + if (iz >= n * OD_OH) { + return; + } + + src1 = (global float*)((global char*)src1 + offset1); + dst = (global half*)((global char*)dst + offsetd); + + const int iic = i / KD_KH_KW; + const int ikd = (i - iic * KD_KH_KW) / KH_KW; + const int ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / kw; + const int ikw = i % kw; + + const int in = iz / OD_OH; + const int iod = (iz - in * OD_OH) / oh; + const int ioh = iz % oh; + + const int iiw = iow * s0 + ikw * d0 - p0; + const int iih = ioh * s1 + ikh * d1 - p1; + const int iid = iod * s2 + ikd * d2 - p2; + + const long offset_dst = + (long)(iz * ow + iow) * (long)IC_KD_KH_KW + + (long)(iic * KD_KH_KW + ikd * KH_KW + ikh * kw + ikw); + + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) { + dst[offset_dst] = 0.0f; + } else { + const long offset_src = + (long)(in * ic + iic) * stride_q + (long)iid * stride_z + + (long)iih * stride_y + (long)iiw * stride_x; + dst[offset_dst] = convert_half_rte(src1[offset_src]); + } +} diff --git a/ggml/src/ggml-opencl/kernels/im2col_3d_f32.cl b/ggml/src/ggml-opencl/kernels/im2col_3d_f32.cl new file mode 100644 index 0000000..ea94d5a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/im2col_3d_f32.cl @@ -0,0 +1,90 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +kernel void kernel_im2col_3d_f32( + global float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + long IC, + long ID, + long IH, + long IW, + long KD, + long KH, + long KW, + long OD, + long OH, + long OW, + long N, + long stride_x, + long stride_y, + long stride_z, + long stride_q, + int s0, + int s1, + int s2, + int p0, + int p1, + int p2, + int d0, + int d1, + int d2 +) { + const int ic = (int)IC; + const int kd = (int)KD; + const int kh = (int)KH; + const int kw = (int)KW; + const int od = (int)OD; + const int oh = (int)OH; + const int ow = (int)OW; + const int n = (int)N; + + const int KD_KH_KW = kd * kh * kw; + const int KH_KW = kh * kw; + const int IC_KD_KH_KW = ic * KD_KH_KW; + + const int i = (int)get_global_id(0); + if (i >= IC_KD_KH_KW) { + return; + } + + const int iow = (int)get_global_id(1); + if (iow >= ow) { + return; + } + + const int OD_OH = od * oh; + const int iz = (int)get_global_id(2); + if (iz >= n * OD_OH) { + return; + } + + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + const int iic = i / KD_KH_KW; + const int ikd = (i - iic * KD_KH_KW) / KH_KW; + const int ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / kw; + const int ikw = i % kw; + + const int in = iz / OD_OH; + const int iod = (iz - in * OD_OH) / oh; + const int ioh = iz % oh; + + const int iiw = iow * s0 + ikw * d0 - p0; + const int iih = ioh * s1 + ikh * d1 - p1; + const int iid = iod * s2 + ikd * d2 - p2; + + const long offset_dst = + (long)(iz * ow + iow) * (long)IC_KD_KH_KW + + (long)(iic * KD_KH_KW + ikd * KH_KW + ikh * kw + ikw); + + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) { + dst[offset_dst] = 0.0f; + } else { + const long offset_src = + (long)(in * ic + iic) * stride_q + (long)iid * stride_z + + (long)iih * stride_y + (long)iiw * stride_x; + dst[offset_dst] = src1[offset_src]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/lightning_indexer.cl b/ggml/src/ggml-opencl/kernels/lightning_indexer.cl new file mode 100644 index 0000000..efd2f11 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/lightning_indexer.cl @@ -0,0 +1,111 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#endif + +kernel void kernel_lightning_indexer( + global const char * q, + ulong offset_q, + global const char * k, + ulong offset_k, + global const char * w, + ulong offset_w, + global const char * mask, + ulong offset_m, + global float * dst, + ulong offset_d, + int head_size, + int n_heads, + int n_tokens, + int n_keys, + int n_masks, + int type_k, + ulong nbq1, + ulong nbq2, + ulong nbq3, + ulong nbk2, + ulong nbk3, + ulong nbw1, + ulong nbw3, + ulong nbm1, + ulong nbm3, + int use_shuffle +) { + const int tid = get_local_id(0); + const int lane = tid & (use_shuffle ? 3 : 15); + const int key = use_shuffle ? get_group_id(0)*16 + tid/4 : get_group_id(0)*4 + tid/16; + const int token = get_group_id(1); + const int stream = get_group_id(2); + + q += offset_q + token*nbq2 + stream*nbq3; + k += offset_k + (ulong) key*nbk2 + stream*nbk3; + w += offset_w + token*nbw1 + stream*nbw3; + mask += offset_m + token*nbm1 + (stream % n_masks)*nbm3; + dst = (global float *) ((global char *) dst + offset_d); + +#ifdef cl_khr_subgroup_shuffle + if (use_shuffle) { + // Each group of four lanes keeps one key for all query heads. + float4 cached[8]; + #pragma unroll + for (int j = 0; j < 8; ++j) { + const int i = 4*(lane + j*4); + cached[j] = key < n_keys ? (type_k == 0 ? vload4(0, (global const float *) k + i) : convert_float4(vload4(0, (global const half *) k + i))) : (float4) (0.0f); + } + + float score = 0.0f; + for (int h = 0; h < n_heads; ++h) { + global const float * q_head = (global const float *) (q + h*nbq1); + float4 sum = (float4) (0.0f); + #pragma unroll + for (int j = 0; j < 8; ++j) { + sum = mad(vload4(0, q_head + 4*(lane + j*4)), cached[j], sum); + } + float value = sum.s0 + sum.s1 + sum.s2 + sum.s3; + value += sub_group_shuffle_xor(value, 2); + value += sub_group_shuffle_xor(value, 1); + if (lane == 0) { + score = mad(fmax(value, 0.0f), ((global const float *) w)[h], score); + } + } + if (lane == 0 && key < n_keys) { + dst[((ulong) stream*n_tokens + token)*n_keys + key] = score + (float) ((global const half *) mask)[key]; + } + return; + } +#endif + + local float sums[64]; + float score = 0.0f; + for (int h = 0; h < n_heads; ++h) { + global const float * q_head = (global const float *) (q + h*nbq1); + float sum = 0.0f; + if (key < n_keys) { + for (int i = lane; i < head_size; i += 16) { + float kv; + if (type_k == 0) { + kv = ((global const float *) k)[i]; + } else { + kv = (float) ((global const half *) k)[i]; + } + sum = mad(q_head[i], kv, sum); + } + } + sums[tid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + for (int stride = 8; stride > 0; stride /= 2) { + if (lane < stride) { + sums[tid] += sums[tid + stride]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + if (lane == 0) { + score = mad(fmax(sums[tid], 0.0f), ((global const float *) w)[h], score); + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (lane == 0 && key < n_keys) { + dst[((ulong) stream*n_tokens + token)*n_keys + key] = score + (float) ((global const half *) mask)[key]; + } +} diff --git a/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl b/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl new file mode 100644 index 0000000..6a8e4fb --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl @@ -0,0 +1,76 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +//------------------------------------------------------------------------------ +// add_id(gate) + add_id(up) + swiglu_oai, fused +// +// gpt-oss-class MoE FFNs run three full passes over the same +// [n_ff, n_expert_used, n_tokens] f32 tensor: a per-expert bias add on the gate +// matmul output, the same on the up matmul output, then swiglu_oai over the +// two. Both bias adds are in-place, so each costs a full read plus a full write +// of a tensor that is only read once more. Folding them into the swiglu pass +// leaves two reads and one write instead of six passes. +// +// Grouping matches kernel_add_id: group 0 = expert slot (i1), group 1 = token +// (i2). For a contiguous destination that addressing is identical to the flat +// row walk kernel_swiglu_oai uses, since row i1 + i2*ne1 sits at +// i1*nb1 + i2*ne1*nb1. +//------------------------------------------------------------------------------ +kernel void kernel_add_id_add_id_swiglu_oai( + global char * src_g, + ulong offset_g, + global char * src_gb, + ulong offset_gb, + global char * src_u, + ulong offset_u, + global char * src_ub, + ulong offset_ub, + global char * src_ids, + ulong offset_ids, + global char * dst, + ulong offsetd, + ulong nb01_g, + ulong nb02_g, + ulong nb01_u, + ulong nb02_u, + ulong nb11_g, + ulong nb11_u, + ulong nb21, + ulong nbd1, + ulong nbd2, + int ne0, + float limit, + float alpha +) { + src_g = (global char *)(src_g + offset_g); + src_gb = (global char *)(src_gb + offset_gb); + src_u = (global char *)(src_u + offset_u); + src_ub = (global char *)(src_ub + offset_ub); + src_ids = (global char *)(src_ids + offset_ids); + dst = (global char *)(dst + offsetd); + + const int i1 = get_group_id(0); + const int i2 = get_group_id(1); + + // The ids tensor is a view into a [n_expert, n_tokens] buffer, so its row + // stride is nb21 and the k selected ids are NOT contiguous per token. + const int i11 = *((global const int *) (src_ids + i1*sizeof(int) + i2*nb21)); + + global const float * g_row = (global const float *)(src_g + i1*nb01_g + i2*nb02_g); + global const float * u_row = (global const float *)(src_u + i1*nb01_u + i2*nb02_u); + global const float * gb_row = (global const float *)(src_gb + i11*nb11_g); + global const float * ub_row = (global const float *)(src_ub + i11*nb11_u); + global float * d_row = (global float *)(dst + i1*nbd1 + i2*nbd2); + + for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { + float x0 = g_row[i0] + gb_row[i0]; + float x1 = u_row[i0] + ub_row[i0]; + + x0 = min(x0, limit); + x1 = max(min(x1, limit), -limit); + + float out_glu = x0 / (1.0f + exp(-x0 * alpha)); + out_glu = out_glu * (1.0f + x1); + + d_row[i0] = out_glu; + } +} diff --git a/ggml/src/ggml-opencl/kernels/moe_combine.cl b/ggml/src/ggml-opencl/kernels/moe_combine.cl new file mode 100644 index 0000000..acd08db --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/moe_combine.cl @@ -0,0 +1,79 @@ +// Fused MoE combine epilogue: replaces the router-weight MUL + the (n_expert_used-1) +// cross-expert ADD chain with ONE weighted-sum-across-experts pass. +// dst[row, tok] = sum_e experts[row, e, tok] * weights[0, e, tok] +// experts: [n_embd, n_expert_used, n_tokens] f32 (contiguous after down-proj GEMM) +// weights: [1, n_expert_used, n_tokens] f32 +// dst: [n_embd, n_tokens] f32 +// One read of experts + one write of dst (eliminates the intermediate weighted +// buffer and the k-1 elementwise add round-trips). Vectorized float4 over rows. +// strides e1/e2/w1/w2/d1 are in ELEMENTS (floats). + +// Same weighted sum, with the per-expert bias add folded in. +// +// The MoE down projection's bias is applied by an in-place add_id whose only +// consumer is this combine, so it costs a full read plus a full write of a +// tensor that is read once more immediately afterwards. Reading the raw matmul +// output here and adding the bias row while it is already in registers removes +// that pass. Kept as a separate kernel so the unfused path is untouched. +__kernel void kernel_moe_combine_bias_f32( + __global const char * e_buf, ulong off_e, + __global const char * w_buf, ulong off_w, + __global const char * b_buf, ulong off_b, // per-expert bias rows + __global const char * i_buf, ulong off_i, // expert ids + __global char * d_buf, ulong off_d, + int n_embd4, // n_embd / 4 + int k, // n_expert_used + int n_tokens, + uint e1, uint e2, // experts strides (elements): per-expert, per-token + uint w1, uint w2, // weights strides (elements) + uint d1, // dst per-token stride (elements) + ulong nb_b1, // bias row stride (bytes) + ulong nb_i1) // ids row stride (bytes) - ids is a view, not packed +{ + const uint r4 = get_global_id(0); + const uint tok = get_global_id(1); + if (r4 >= (uint)n_embd4 || tok >= (uint)n_tokens) return; + + __global const float * E = (__global const float *)(e_buf + off_e) + tok*e2 + r4*4u; + __global const float * W = (__global const float *)(w_buf + off_w) + tok*w2; + __global const char * B = b_buf + off_b; + __global const char * I = i_buf + off_i + (ulong)tok*nb_i1; + + float4 acc = (float4)(0.0f); + for (int e = 0; e < k; ++e) { + const int i11 = *((__global const int *)(I + (ulong)e*sizeof(int))); + __global const float * Brow = (__global const float *)(B + (ulong)i11*nb_b1) + r4*4u; + const float4 v = vload4(0, E + (uint)e*e1) + vload4(0, Brow); + acc = mad(v, (float4)(W[(uint)e*w1]), acc); + } + + __global float * D = (__global float *)(d_buf + off_d) + tok*d1 + r4*4u; + vstore4(acc, 0, D); +} + +__kernel void kernel_moe_combine_f32( + __global const char * e_buf, ulong off_e, + __global const char * w_buf, ulong off_w, + __global char * d_buf, ulong off_d, + int n_embd4, // n_embd / 4 + int k, // n_expert_used + int n_tokens, + uint e1, uint e2, // experts strides (elements): per-expert, per-token + uint w1, uint w2, // weights strides (elements) + uint d1) // dst per-token stride (elements) +{ + const uint r4 = get_global_id(0); + const uint tok = get_global_id(1); + if (r4 >= (uint)n_embd4 || tok >= (uint)n_tokens) return; + + __global const float * E = (__global const float *)(e_buf + off_e) + tok*e2 + r4*4u; + __global const float * W = (__global const float *)(w_buf + off_w) + tok*w2; + + float4 acc = (float4)(0.0f); + for (int e = 0; e < k; ++e) { + acc = mad(vload4(0, E + (uint)e*e1), (float4)(W[(uint)e*w1]), acc); + } + + __global float * D = (__global float *)(d_buf + off_d) + tok*d1 + r4*4u; + vstore4(acc, 0, D); +} diff --git a/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl b/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl new file mode 100644 index 0000000..2f5c110 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl @@ -0,0 +1,32 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK4_0 32 + +kernel void kernel_moe_reorder_b( + global float4 * src, + global uint * router, + global float4 * dst, + global int * total_tiles, + uint K, + ushort map_ratio, + uint tile_size +) { + uint k_4 = get_global_id(0); + uint post_router_idx = get_global_id(1); + + if ((k_4 >= (K / 4)) || (post_router_idx >= total_tiles[0] * tile_size)) { + return; + } + + uint router_idx = router[post_router_idx]; + + // Padded slots need not be written at all. The MoE GEMMs accumulate per output + // column and scatter only the real columns, so whatever sits in a padded slot + // never reaches dst + if (router_idx == 0xFFFFFFFF) { + return; + } + + ushort activation_idx = router_idx / map_ratio; + dst[post_router_idx * K / 4 + k_4] = src[activation_idx * K / 4 + k_4]; +} diff --git a/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl new file mode 100644 index 0000000..0d16f3a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl @@ -0,0 +1,64 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +// Fused MoE activation reorder + q8_1 quantization for the dp4a prefill GEMM. +// Combines kernel_moe_reorder_b (gather src1 rows per the post-router map) with +// the q8_1 quant pre-pass, so the f32 reordered-activation tile buffer is never +// materialised (saves a full write + read of [tok_slots * ne00] floats). +// +// One work-item per (token_slot, 32-block). Padding lanes (router 0xFFFFFFFF) +// emit d=0,s=0,qs=0 so they contribute nothing to the GEMM, exactly as the +// reorder zero-fill did. Output layout matches kernel_moe_quant_a_q8_1: +// qa[token_slot*K + blk*32 + i], da/sa[token_slot*(K/32) + blk]. +__kernel void kernel_moe_reorder_quant_a_q8_1( + __global const float * src, // original activations (offset applied) + __global const uint * router, // post-router indices [tok_slots] + __global char * qa, + __global half * da, + __global half * sa, + __global const int * total_tiles, + uint K, + ushort map_ratio, + uint tile_size, + uint n_kblocks // K / 32 +) { + const uint blk = get_global_id(0); // 32-block along K + const uint tok = get_global_id(1); // token slot (post_router_idx) + + if (blk >= n_kblocks || tok >= (uint)total_tiles[0] * tile_size) { + return; + } + + const uint out_base = tok * K + blk * 32; + const uint bidx = tok * n_kblocks + blk; + + const uint router_idx = router[tok]; + + float v[32]; + float amax = 0.0f; + if (router_idx == 0xFFFFFFFF) { + #pragma unroll + for (int i = 0; i < 32; ++i) v[i] = 0.0f; + } else { + const uint act_idx = router_idx / map_ratio; + const uint in_base = act_idx * K + blk * 32; + #pragma unroll + for (int i = 0; i < 32; ++i) { + v[i] = src[in_base + i]; + amax = fmax(amax, fabs(v[i])); + } + } + + const float d = amax / 127.0f; + const float id = (amax > 0.0f) ? (127.0f / amax) : 0.0f; + + int sum = 0; + #pragma unroll + for (int i = 0; i < 32; ++i) { + const int q = (int)rint(v[i] * id); + qa[out_base + i] = (char)q; + sum += q; + } + + da[bidx] = (half)d; + sa[bidx] = (half)(d * (float)sum); +} diff --git a/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl new file mode 100644 index 0000000..5ea31af --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl @@ -0,0 +1,196 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +__kernel void kernel_moe_histogram( + __global const int * input, + __global int * hist, + uint N, + uint topK, + uint n_experts, + ulong offset +) { + input = (__global const int *)((__global const char *)input + offset); + + uint n = get_global_id(0); + uint k = get_global_id(1); + + if (n >= N || k >= topK) { + return; + } + + int expert_id = input[n * n_experts + k]; + if (expert_id == -1) { // skipped slot, owned by no expert + return; + } + atomic_inc(&hist[expert_id]); +} + +__kernel void kernel_moe_scan( + __global int * hist, + __global int * tile_offset, + __global int * total_tiles, + __global int * slot_counter, + int tile_size, + uint n_experts +) { + int offset = 0; + for (int v = 0; v < n_experts; v++) { + int count = hist[v]; + int tiles = (count + tile_size - 1) / tile_size; + tile_offset[v] = offset; + offset += tiles; + hist[v] = 0; + slot_counter[v] = 0; + } + + *total_tiles = offset; +} + +__kernel void kernel_moe_scatter( + __global const int * input, + __global int * post_router, + __global ushort * emap, + __global const int * tile_offset, + __global int * slot_counter, + int N, + int topK, + uint n_experts, + ulong offset +) { + input = (__global const int *)((__global const char *)input + offset); + + uint n = get_global_id(0); + uint k = get_global_id(1); + + if (n >= N || k >= topK) { + return; + } + + int val = input[n * n_experts + k]; + if (val == -1) { // skipped slot, owned by no expert + return; + } + + int local_slot = atomic_inc(&slot_counter[val]); + + int tile_idx = tile_offset[val] + (local_slot / 32); + int lane = local_slot % 32; + int out_pos = tile_idx * 32 + lane; + + post_router[out_pos] = n * topK + k; + emap[tile_idx] = val; +} + +// the gemm only writes the rows an expert owns, so the skipped rows (negative ids) are zeroed here +__kernel void kernel_moe_zero_dst( + __global const int * input, + __global float * dst, + uint N, + uint topK, + uint n_experts, + uint ne0 +) { + uint n = get_global_id(0); + uint k = get_global_id(1); + + if (n >= N || k >= topK) { + return; + } + + if (input[n * n_experts + k] != -1) { + return; + } + + __global float * dst_row = dst + ((ulong)n * topK + k) * ne0; + for (uint i = get_local_id(2); i < ne0; i += get_local_size(2)) { + dst_row[i] = 0.0f; + } +} + +// Deterministic replacement for kernel_moe_scatter. +// +// kernel_moe_scatter takes each token's slot from atomic_inc(slot_counter[expert]), +// so the token -> slot packing inside an expert depends on which work-item wins the +// atomic and changes from run to run. The ragged prefill GEMM path is sensitive to +// that packing (the non-ragged path is not, since its padded slots alias slot 0 and +// are overwritten last), which makes MoE prompt processing non-reproducible: the same +// binary on the same prompt returns one of several outputs. +// +// Here the slot is the token's rank in flat (n, k) order among the tokens routed to +// the same expert - a fixed function of the routing input. One workgroup per expert +// walks the flat routing list in blocks of 64 and ranks its own tokens with a +// workgroup scan, carrying a running count between blocks. Cost is one pass over the +// routing list per expert; the list is a few KiB and stays in cache. +__kernel void kernel_moe_scatter_stable( + __global const int * input, + __global int * post_router, + __global ushort * emap, + __global const int * tile_offset, + int N, + int topK, + uint n_experts, + ulong offset +) { + input = (__global const int *)((__global const char *)input + offset); + + const int e = get_group_id(1); + const int lid = get_local_id(0); + const int M = N * topK; + + __local int scan[64]; + __local int running; + + if (lid == 0) { + running = 0; + } + barrier(CLK_LOCAL_MEM_FENCE); + + for (int base = 0; base < M; base += 64) { + const int j = base + lid; + + int pred = 0; + if (j < M) { + const int n = j / topK; + const int k = j - n * topK; + pred = (input[n * (int)n_experts + k] == e) ? 1 : 0; + } + + scan[lid] = pred; + barrier(CLK_LOCAL_MEM_FENCE); + + // Hillis-Steele inclusive scan over the 64 lanes + for (int off = 1; off < 64; off <<= 1) { + int add = (lid >= off) ? scan[lid - off] : 0; + barrier(CLK_LOCAL_MEM_FENCE); + scan[lid] += add; + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (pred) { + const int local_slot = running + (scan[lid] - 1); // exclusive rank + const int tile_idx = tile_offset[e] + (local_slot >> 5); + const int lane = local_slot & 31; + + post_router[tile_idx * 32 + lane] = j; + emap[tile_idx] = (ushort)e; + } + + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 63) { + running += scan[63]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } +} + +__kernel void kernel_moe_fill( + __global int * post_router, + __global int * total_tiles, + int tile_size +) { + int tile_id = get_global_id(0); + int vec_id_in_tile = get_global_id(1); + + if (tile_id < total_tiles[0]) { + post_router[tile_id * tile_size + vec_id_in_tile] = 0xFFFFFFFF; + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl index 6982f8f..eba1778 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl @@ -4,12 +4,20 @@ #define LOAD_VEC_B 4 #define BM 64 +#ifndef BN #define BN 64 +#endif #define BK 16 #define TM 4 +#ifndef TN #define TN 8 +#endif -kernel void kernel_mul_mm_f16_f32_l4_lm( +#ifndef KERNEL_NAME_LM +#define KERNEL_NAME_LM kernel_mul_mm_f16_f32_l4_lm +#endif + +kernel void KERNEL_NAME_LM( global half4 * src0, ulong offset0, global float4 * src1, @@ -93,7 +101,8 @@ kernel void kernel_mul_mm_f16_f32_l4_lm( } } - for (int l = 0; l < BN; l += loadstride_b) { + // BN=8 uses only the threads that load a column inside the tile. + for (int l = 0; loadc_b + l < BN; l += loadstride_b) { if (ic*BN + loadc_b + l < ne11) { const int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl index d7d5ba6..9dc9862 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl @@ -145,3 +145,52 @@ kernel void kernel_mul_mm_f32_f32_l4_lm( } } } + +// Multi-column f32 GEMV for the small-N (spec/MTP verify) batch. The tiled GEMM +// above always computes a full BM x BN = 64 x 64 output tile, so at ne11=3 with a +// skinny weight (e.g. GDN ssm_alpha/ssm_beta, M=32) it launches ONE under-occupied +// workgroup at ~2.3% tile utilization. This kernel assigns one 64-thread workgroup +// per output element (m,n): the 64 threads split the K reduction (float4) and +// tree-reduce in __local (no subgroup ops -> portable). ne01*ne11 workgroups. +// Weight row is re-read per column (N small -> negligible). Summation order differs +// from the tiled GEMM (lane-strided + tree) -> f32-exact-ish, not bit-identical. +kernel void kernel_gemv_f32_f32_mc( + global float * src0, ulong offset0, // weight: row m at m*stride_a (elements) + global float * src1, ulong offset1, // activations: col n at n*stride_b + global float * dst, ulong offsetd, // dst [M x N] col-major: (m,n) at n*stride_d+m + int ne00, // K + int ne01, // M + int ne11, // N + int stride_a, // weight row stride (elements) = K + int stride_b, // activation col stride (elements) = K + int stride_d) // dst column stride (elements) = M +{ + src0 = (global float*)((global char*)src0 + offset0); + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + uint lane = get_local_id(0); // 0..63 + uint out = get_global_id(1); // 0 .. ne01*ne11 - 1 + uint m = out % (uint)ne01; + uint n = out / (uint)ne01; + + global float4 * wrow = (global float4*)(src0 + (ulong)m * (uint)stride_a); + global float4 * xcol = (global float4*)(src1 + (ulong)n * (uint)stride_b); + uint k4 = (uint)ne00 >> 2; + + float acc = 0.0f; + for (uint k = lane; k < k4; k += 64) { + float4 w = wrow[k]; + float4 x = xcol[k]; + acc += w.s0*x.s0 + w.s1*x.s1 + w.s2*x.s2 + w.s3*x.s3; + } + + local float red[64]; + red[lane] = acc; + barrier(CLK_LOCAL_MEM_FENCE); + for (uint s = 32; s > 0; s >>= 1) { + if (lane < s) red[lane] += red[lane + s]; + barrier(CLK_LOCAL_MEM_FENCE); + } + if (lane == 0) dst[(ulong)n * (uint)stride_d + m] = red[0]; +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl new file mode 100644 index 0000000..11ff7f8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl @@ -0,0 +1,171 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define LOAD_VEC_A 8 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#define TM 4 +#define TN 8 + +constant float kvalues_iq4nl[16] = { + -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, + 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f +}; + +kernel void kernel_mul_mm_iq4_nl_f32_l4_lm( + global uchar4 * src0_q, + global half * src0_d, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a; + int ib = idx / 4; + int iqs = idx % 4; + + float d = (float)src0_d[ib]; + global uchar4 * qs = src0_q + ib*4 + iqs; + uchar4 q = *qs; + // IQ4_NL: use lookup table instead of linear (nibble - 8) + float4 v1 = (float4)(kvalues_iq4nl[(q.s0 )&0x0F], kvalues_iq4nl[(q.s1 )&0x0F], + kvalues_iq4nl[(q.s2 )&0x0F], kvalues_iq4nl[(q.s3 )&0x0F])*d; + float4 v2 = (float4)(kvalues_iq4nl[(q.s0>>4)&0x0F], kvalues_iq4nl[(q.s1>>4)&0x0F], + kvalues_iq4nl[(q.s2>>4)&0x0F], kvalues_iq4nl[(q.s3>>4)&0x0F])*d; + + buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = v1.s3; + buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = v2.s0; + buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = v2.s1; + buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = v2.s2; + buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = v2.s3; + } else { + buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK / LOAD_VEC_A; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const int offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl new file mode 100644 index 0000000..67710fa --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl @@ -0,0 +1,156 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +// LOAD_VEC_A is 8 because one q1_0 quant byte expands to 8 weights along K. +#define LOAD_VEC_A 8 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#define TM 4 +#define TN 8 + +kernel void kernel_mul_mm_q1_0_f32_l4_lm( + global uchar * src0_q, + global half * src0_d, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a; + int ib = idx / 16; // 16 quant bytes per q1_0 block + + float d = (float)src0_d[ib]; + uint bits = src0_q[idx]; + + // use float to avoid unsigned underflow of (2*0 - 1). + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 0) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 1) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 2) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 3) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 4) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 4) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 5) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 5) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 6) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 6) & 1) - 1.0f); + buf_a[(loadr_a * LOAD_VEC_A + 7) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 7) & 1) - 1.0f); + } else { + for (int b = 0; b < LOAD_VEC_A; ++b) { + buf_a[(loadr_a * LOAD_VEC_A + b) * BM + loadc_a + l] = 0.0f; + } + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK / LOAD_VEC_A; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const int offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl new file mode 100644 index 0000000..e192bde --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl @@ -0,0 +1,203 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_q2_K { + uchar scales[QK_K/16]; + uchar qs[QK_K/4]; + half d; + half dmin; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#ifndef BK +#define BK 32 +#endif +#ifdef INTEL_GPU +#define TM 8 +#else +#define TM 4 +#endif +#define TN 8 + +#ifndef LM_HALF +#define LM_HALF 0 +#endif +#if LM_HALF +typedef half lm_st; +#define LM_LD4(p, i) convert_float4(vload4((i), (p))) +#else +typedef float lm_st; +#define LM_LD4(p, i) vload4((i), (p)) +#endif + +kernel void kernel_mul_mm_q2_k_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_sc, + global half * src0_dm, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local lm_st buf_a[BM * BK]; + local lm_st buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one q2_K super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float4 sums4[(TM/4) * TN]; + float4 cache_a4[TM/4]; + + for (int i = 0; i < (TM/4) * TN; i++) { + sums4[i] = (float4)(0.0f); + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_q2_K); + int e = idx % QK_K; + + // dequantize_row_q2_K walks 2 halves of 128, each as 4 shifts of + // 2 groups of 16. Recover that position from the element index. + int n = e >> 7; + int rem = e & 127; + int j = rem >> 5; + int s = (rem >> 4) & 1; + int l16 = rem & 15; + int isc = 8*n + 2*j + s; + int shift = 2*j; + + uchar sc = src0_sc[16*ib + isc]; + float d = (float)src0_dm[2*ib] * (float)(sc & 0xF); + float m = -(float)src0_dm[2*ib + 1] * (float)(sc >> 4); + + uchar4 q = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16); + float4 v1 = convert_float4((uchar4)((q.s0 >> shift) & 3, + (q.s1 >> shift) & 3, + (q.s2 >> shift) & 3, + (q.s3 >> shift) & 3)) * d + m; + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int a = 0; a < TM/4; a++) { + cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a); + } + + for (int cc = 0; cc < TN; cc++) { + const float cache_b = buf_b[(i) * BN + th_c * TN + cc]; + for (int a = 0; a < TM/4; a++) { + const int sums_idx = cc*(TM/4) + a; + sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int a = 0; a < TM/4; a++) { + const float4 v = sums4[cc * (TM/4) + a]; + const float vs[4] = { v.s0, v.s1, v.s2, v.s3 }; + for (int k = 0; k < 4; k++) { + if (dr + 4*a + k < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k]; + } + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl index 2235b1a..a9c649a 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl @@ -1,13 +1,23 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + #define LOAD_VEC_A 4 #define LOAD_VEC_B 4 #define BM 64 #define BN 64 #define BK 32 +#ifdef INTEL_GPU +// Intel Xe iGPU: 8x8 microtile (WG = BM*BN/(TM*TN) = 64) — ~+12% pp512 vs 4x8 +#define TM 8 +#define TN 8 +#else #define TM 4 #define TN 8 +#endif kernel void kernel_mul_mm_q4_k_f32_l4_lm( global uchar4 * src0_q, diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl new file mode 100644 index 0000000..1e980a4 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl @@ -0,0 +1,173 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define LOAD_VEC_A 8 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#define TM 4 +#define TN 8 + +kernel void kernel_mul_mm_q5_0_f32_l4_lm( + global uchar4 * src0_qs, + global uint * src0_qh, + global half * src0_d, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a; + int ib = idx / 4; + int iqs = idx % 4; + + float d = (float)src0_d[ib]; + uint qh_val = src0_qh[ib]; + + global uchar4 * qs_ptr = src0_qs + ib*4 + iqs; + uchar4 q = *qs_ptr; + + uint qh_lo = qh_val >> (iqs * 4); + uint qh_hi = qh_val >> (iqs * 4 + 16); + + uchar4 b_lo = (uchar4)((uchar)qh_lo, (uchar)(qh_lo >> 1), (uchar)(qh_lo >> 2), (uchar)(qh_lo >> 3)) & (uchar)1; + uchar4 b_hi = (uchar4)((uchar)qh_hi, (uchar)(qh_hi >> 1), (uchar)(qh_hi >> 2), (uchar)(qh_hi >> 3)) & (uchar)1; + + float4 v1 = (convert_float4((q & (uchar)0x0F) | (b_lo << (uchar)4)) - 16.0f) * d; + float4 v2 = (convert_float4((q >> (uchar)4) | (b_hi << (uchar)4)) - 16.0f) * d; + + buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = v1.s3; + buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = v2.s0; + buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = v2.s1; + buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = v2.s2; + buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = v2.s3; + } else { + buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK / LOAD_VEC_A; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const int offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl new file mode 100644 index 0000000..ba06be5 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl @@ -0,0 +1,175 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define LOAD_VEC_A 8 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#define TM 4 +#define TN 8 + +kernel void kernel_mul_mm_q5_1_f32_l4_lm( + global uchar4 * src0_qs, + global uint * src0_qh, + global half * src0_d, + global half * src0_m, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a; + int ib = idx / 4; + int iqs = idx % 4; + + float d = (float)src0_d[ib]; + float m = (float)src0_m[ib]; + uint qh_val = src0_qh[ib]; + + global uchar4 * qs = src0_qs + ib*4 + iqs; + uchar4 q = *qs; + + uint qh_lo = qh_val >> (iqs * 4); + uint qh_hi = qh_val >> (iqs * 4 + 16); + + uchar4 b_lo = (uchar4)((uchar)qh_lo, (uchar)(qh_lo >> 1), (uchar)(qh_lo >> 2), (uchar)(qh_lo >> 3)) & (uchar)1; + uchar4 b_hi = (uchar4)((uchar)qh_hi, (uchar)(qh_hi >> 1), (uchar)(qh_hi >> 2), (uchar)(qh_hi >> 3)) & (uchar)1; + + float4 v1 = convert_float4((q & (uchar)0x0F) | (b_lo << (uchar)4)) * d + m; + float4 v2 = convert_float4((q >> (uchar)4) | (b_hi << (uchar)4)) * d + m; + + buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = v1.s3; + buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = v2.s0; + buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = v2.s1; + buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = v2.s2; + buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = v2.s3; + } else { + buf_a[(loadr_a * 4 + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 3) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 16) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 17) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 18) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * 4 + 19) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK / LOAD_VEC_A; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const int offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl index 8e191f5..a343b5c 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl @@ -1,13 +1,23 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + #define LOAD_VEC_A 4 #define LOAD_VEC_B 4 #define BM 64 #define BN 64 #define BK 32 +#ifdef INTEL_GPU +// Intel Xe iGPU: 8x8 microtile (WG=64) +#define TM 8 +#define TN 8 +#else #define TM 4 #define TN 8 +#endif kernel void kernel_mul_mm_q5_k_f32_l4_lm( global uchar4 * src0_q, diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl index 9393b54..b4b03eb 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl @@ -64,7 +64,14 @@ kernel void kernel_mul_mat_f16_f16( global half * x = (global half *) (src0 + offset_src0); - if (ne00 < 128) { + // The vector path below casts the row pointers to half4, which must be 8-byte aligned. + // A row address is r0*nb01 + ..., and a permuted or strided src leaves nb01/nb11 + // unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned. Every + // src1 row this work-item walks is src1_base + r1*nb11, so require both. + const ulong src1_base = (ulong) (src1 + (i12)*nb12 + (i13)*nb13); + const bool row_aligned = (((ulong) x) & 7) == 0 && (src1_base & 7) == 0 && (nb11 & 7) == 0; + + if (ne00 < 128 || !row_aligned) { for (int row = 0; row < N_F16_F16; ++row) { int r1 = rb + row; if (r1 >= ne11) { diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl index e52d3c6..8f3ed9c 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl @@ -64,7 +64,14 @@ kernel void kernel_mul_mat_f16_f32( global half * x = (global half *) (src0 + offset_src0); - if (ne00 < 128) { + // The vector path below casts the row pointers to half4/float4, which must be 8- and + // 16-byte aligned. A row address is r0*nb01 + ..., and a permuted or strided src leaves + // nb01/nb11 unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned. + // Every src1 row this work-item walks is src1_base + r1*nb11, so require both. + const ulong src1_base = (ulong) (src1 + (i12)*nb12 + (i13)*nb13); + const bool row_aligned = (((ulong) x) & 7) == 0 && (src1_base & 15) == 0 && (nb11 & 15) == 0; + + if (ne00 < 128 || !row_aligned) { for (int row = 0; row < N_F16_F32; ++row) { int r1 = rb + row; if (r1 >= ne11) { diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl index 28d3021..eca4561 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl @@ -64,8 +64,15 @@ kernel void kernel_mul_mat_f16_f32_1row( global half * x = (global half *) (src0 + offset_src0); global float * y = (global float *) (src1 + offset_src1); + // The vector path below casts the row pointers to half4/float4, which must be 8- and + // 16-byte aligned. A row address is r0*nb01 + ..., and a permuted or strided src leaves + // nb01/nb11 unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned. + // Take the vector path only when the rows this work-item touches are actually aligned; + // the scalar loop has no such requirement. + const bool row_aligned = (((ulong) x) & 7) == 0 && (((ulong) y) & 15) == 0; + float sumf = 0; - if (ne00 < 128) { + if (ne00 < 128 || !row_aligned) { for (int i = get_sub_group_local_id(); i < ne00; i += get_max_sub_group_size()) { sumf += (float) x[i] * (float) y[i]; } diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl index cdf8197..97148d3 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl @@ -18,6 +18,18 @@ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) #endif +#ifdef cl_khr_subgroup_shuffle +#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +#elif defined(cl_qcom_subgroup_shuffle) +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define HAS_SUBGROUP_SHUFFLE 1 +// Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR +// name, so calling it is an implicit declaration and the program fails to build. +// Route it to the qcom builtin. +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) +#endif + // Assumes row size (ne00) is a multiple of 4 #ifdef ADRENO_GPU REQD_SUBGROUP_SIZE_64 @@ -82,3 +94,1144 @@ kernel void kernel_mul_mat_f16_f32_l4( } } } + +// Each subgroup produces DR_NDST outputs, assumes ne11 == 1 +#define MUL_MAT_F16_F32_L4_DR_NDST 4 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_dr( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + const int r0_base = get_group_id(0) * MUL_MAT_F16_F32_L4_DR_NDST; + const int im = get_group_id(2); + + const int i12 = im % ne12; + const int i13 = im / ne12; + + // assume ne11 == 1 + const ulong offset_src1 = i12*nb12 + i13*nb13; + global float4 * y4 = (global float4 *)(src1 + offset_src1); + + global half4 * x4[MUL_MAT_F16_F32_L4_DR_NDST]; + float sumf[MUL_MAT_F16_F32_L4_DR_NDST]; + + const ulong k_head_off = (i12/r2)*nb02 + (i13/r3)*nb03; + + #pragma unroll + for (int n = 0; n < MUL_MAT_F16_F32_L4_DR_NDST; ++n) { + int r0 = r0_base + n; + int r0c = r0 < ne01 ? r0 : 0; + ulong off = (ulong)r0c*nb01 + k_head_off; + x4[n] = (global half4 *)(src0 + off); + sumf[n] = 0.0f; + } + + const int n_chunks = ne00 / 4; + const int sg_size = get_max_sub_group_size(); + const int lid = get_sub_group_local_id(); + + for (int i = lid; i < n_chunks; i += sg_size) { + float4 q = y4[i]; + #pragma unroll + for (int n = 0; n < MUL_MAT_F16_F32_L4_DR_NDST; ++n) { + float4 k = convert_float4(x4[n][i]); + sumf[n] = mad(k.s0, q.s0, sumf[n]); + sumf[n] = mad(k.s1, q.s1, sumf[n]); + sumf[n] = mad(k.s2, q.s2, sumf[n]); + sumf[n] = mad(k.s3, q.s3, sumf[n]); + } + } + + #pragma unroll + for (int n = 0; n < MUL_MAT_F16_F32_L4_DR_NDST; ++n) { + float reduced = sub_group_reduce_add(sumf[n]); + int r0 = r0_base + n; + if (lid == 0 && r0 < ne01) { + dst[im*ne1*ne0 + r0] = reduced; + } + } +} + +// Kernels for decoding, Adreno only for now +#define MUL_MAT_F16_F32_L4_DR_LS_R2_MAX 8 + +#ifdef ADRENO_GPU +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) + +REQD_SUBGROUP_SIZE_64 +kernel void kernel_mul_mat_f16_f32_l4_dr_ls( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + const int r0_base = get_group_id(0) * 2; + const int kv_grp = get_group_id(2); // KV head group; im = kv_grp*r2 + q + + const int i12_kv = kv_grp % ne02; + const int i13_kv = kv_grp / ne02; + + const int lid = get_sub_group_local_id(); + const int subhalf = lid >> 5; // 0 or 1 (which K row in the WG) + const int intra = lid & 31; // 0..31 (lane within the half) + + const int r0 = r0_base + subhalf; + const int r0c = r0 < ne01 ? r0 : 0; // clamp OOB to row 0; skip write below + + // K row pointer for this lane (one K row per half-wave). + const ulong k_off = (ulong)r0c*nb01 + (ulong)i12_kv*nb02 + (ulong)i13_kv*nb03; + global half4 * x4 = (global half4 *)(src0 + k_off); + + global float4 * y4[MUL_MAT_F16_F32_L4_DR_LS_R2_MAX]; + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + const int i12_q = i12_kv*r2 + q; + const ulong q_off = (ulong)i12_q*nb12 + (ulong)i13_kv*nb13; + y4[q] = (global float4 *)(src1 + q_off); + } + + float partial[MUL_MAT_F16_F32_L4_DR_LS_R2_MAX]; + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + partial[q] = 0.0f; + } + + const int n_chunks = ne00 / 4; + + for (int i = intra; i < n_chunks; i += 32) { + float4 k = convert_float4(x4[i]); + + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + float4 v = y4[q][i]; + partial[q] = mad(k.s0, v.s0, partial[q]); + partial[q] = mad(k.s1, v.s1, partial[q]); + partial[q] = mad(k.s2, v.s2, partial[q]); + partial[q] = mad(k.s3, v.s3, partial[q]); + } + } + } + + // half-wave reduction + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + partial[q] += sub_group_shuffle_xor(partial[q], 1u); + partial[q] += sub_group_shuffle_xor(partial[q], 2u); + partial[q] += sub_group_shuffle_xor(partial[q], 4u); + partial[q] += sub_group_shuffle_xor(partial[q], 8u); + partial[q] += sub_group_shuffle_xor(partial[q], 16u); + } + } + + if (intra == 0 && r0 < ne01) { + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + const int im = i12_kv*r2 + q + i13_kv*ne12; + dst[im*ne1*ne0 + r0] = partial[q]; + } + } + } +} + +REQD_SUBGROUP_SIZE_64 +kernel void kernel_mul_mat_f16_f32_l4_dr_lq( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + const int r0_base = get_group_id(0) * 4; + const int kv_grp = get_group_id(2); + + const int i12_kv = kv_grp % ne02; + const int i13_kv = kv_grp / ne02; + + const int lid = get_sub_group_local_id(); + const int subq = lid >> 4; // 0..3 (which K row) + const int intra = lid & 15; // 0..15 (lane within quarter) + + const int r0 = r0_base + subq; + const int r0c = r0 < ne01 ? r0 : 0; + + const ulong k_off = (ulong)r0c*nb01 + (ulong)i12_kv*nb02 + (ulong)i13_kv*nb03; + global half4 * x4 = (global half4 *)(src0 + k_off); + + global float4 * y4[MUL_MAT_F16_F32_L4_DR_LS_R2_MAX]; + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + const int i12_q = i12_kv*r2 + q; + const ulong q_off = (ulong)i12_q*nb12 + (ulong)i13_kv*nb13; + y4[q] = (global float4 *)(src1 + q_off); + } + + float partial[MUL_MAT_F16_F32_L4_DR_LS_R2_MAX]; + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + partial[q] = 0.0f; + } + + const int n_chunks = ne00 / 4; + + for (int i = intra; i < n_chunks; i += 16) { + float4 k = convert_float4(x4[i]); + + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + float4 v = y4[q][i]; + partial[q] = mad(k.s0, v.s0, partial[q]); + partial[q] = mad(k.s1, v.s1, partial[q]); + partial[q] = mad(k.s2, v.s2, partial[q]); + partial[q] = mad(k.s3, v.s3, partial[q]); + } + } + } + + // quarter-wave reduction + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + partial[q] += sub_group_shuffle_xor(partial[q], 1u); + partial[q] += sub_group_shuffle_xor(partial[q], 2u); + partial[q] += sub_group_shuffle_xor(partial[q], 4u); + partial[q] += sub_group_shuffle_xor(partial[q], 8u); + } + } + + if (intra == 0 && r0 < ne01) { + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + const int im = i12_kv*r2 + q + i13_kv*ne12; + dst[im*ne1*ne0 + r0] = partial[q]; + } + } + } +} +#endif // ADRENO_GPU + +#define N_ROWS_PER_WG 8 +#define N_OUTS_PER_WG 8 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int sgs_sz = get_max_sub_group_size(); + + const int r0_base = get_group_id(0) * N_ROWS_PER_WG; + const int im = get_group_id(2); + + const int i12 = im % ne12; + const int i13 = im / ne12; + + const ulong offset_src1 = (i12) * nb12 + (i13) * nb13; + global float4 * y4 = (global float4 *)(src1 + offset_src1); + + __local float4 q_loc[64]; // ne00/4 max for sub_group_size 64 + if (sgs_lid < ne00 / 4) { + q_loc[sgs_lid] = y4[sgs_lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + #pragma unroll + for (int dr = 0; dr < N_ROWS_PER_WG; ++dr) { + const int r0 = r0_base + dr; + if (r0 >= ne01) return; + + const ulong offset_src0 = r0 * nb01 + (i12 / r2) * nb02 + (i13 / r3) * nb03; + global half4 * x4 = (global half4 *)(src0 + offset_src0); + + float sumf = 0.0f; + for (int i = sgs_lid; i < ne00 / 4; i += sgs_sz) { + const half4 k4 = x4[i]; + const float4 q = q_loc[i]; + sumf += convert_float(k4.s0) * q.s0 + + convert_float(k4.s1) * q.s1 + + convert_float(k4.s2) * q.s2 + + convert_float(k4.s3) * q.s3; + } + + const float all_sum = sub_group_reduce_add(sumf); + if (sgs_lid == 0) { + dst[im * ne1 * ne0 + r0] = all_sum; // ne11 == 1, so r1==0 + } + } +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_y8( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int sgs_sz = get_max_sub_group_size(); + + const int r0_base = get_group_id(0) * N_OUTS_PER_WG; + const int im = get_group_id(2); + + const int i12 = im % ne12; + const int i13 = im / ne12; + + const ulong offset_src1 = (i12) * nb12 + (i13) * nb13; + global float4 * y4 = (global float4 *)(src1 + offset_src1); + + global half4 * x4_o[N_OUTS_PER_WG]; + #pragma unroll + for (int o = 0; o < N_OUTS_PER_WG; ++o) { + const int r0 = r0_base + o; + const int r0c = (r0 < ne01) ? r0 : 0; + const ulong off = r0c * nb01 + (i12 / r2) * nb02 + (i13 / r3) * nb03; + x4_o[o] = (global half4 *)(src0 + off); + } + + float sum[N_OUTS_PER_WG] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; + + for (int i = sgs_lid; i < ne00 / 4; i += sgs_sz) { + const float4 q4 = y4[i]; + #pragma unroll + for (int o = 0; o < N_OUTS_PER_WG; ++o) { + const half4 v4 = x4_o[o][i]; + sum[o] += convert_float(v4.s0) * q4.s0 + + convert_float(v4.s1) * q4.s1 + + convert_float(v4.s2) * q4.s2 + + convert_float(v4.s3) * q4.s3; + } + } + + #pragma unroll + for (int o = 0; o < N_OUTS_PER_WG; ++o) { + const int r0 = r0_base + o; + const float s = sub_group_reduce_add(sum[o]); + if (sgs_lid == 0 && r0 < ne01) { + dst[im * ne1 * ne0 + r0] = s; + } + } +} + +#define N_OUTS_PAIR 8 +#define N_PAIRS_PAIR (N_OUTS_PAIR / 2) + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_pair( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int half_id = sgs_lid >> 5; // 0 = lower half, 1 = upper half + const int lane_h = sgs_lid & 31; // lane 0..31 within half + + const int r0_base = get_group_id(0) * N_OUTS_PAIR; + const int im = get_group_id(2); + + const int i12 = im % ne12; + const int i13 = im / ne12; + + const ulong offset_src1 = (i12) * nb12 + (i13) * nb13; + global float4 * y4 = (global float4 *)(src1 + offset_src1); + + __local float4 q_loc[64]; // ne00/4 max for sub_group_size 64 + if (sgs_lid < ne00 / 4) { + q_loc[sgs_lid] = y4[sgs_lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const int dk_vec = ne00 / 4; + + #pragma unroll + for (int p = 0; p < N_PAIRS_PAIR; ++p) { + const int r0 = r0_base + 2 * p + half_id; + + const ulong offset_src0 = r0 * nb01 + (i12 / r2) * nb02 + (i13 / r3) * nb03; + global half4 * x4 = (global half4 *)(src0 + offset_src0); + + float sumf = 0.0f; + for (int i = lane_h; i < dk_vec; i += 32) { + const half4 k4 = x4[i]; + const float4 q = q_loc[i]; + sumf += convert_float(k4.s0) * q.s0 + + convert_float(k4.s1) * q.s1 + + convert_float(k4.s2) * q.s2 + + convert_float(k4.s3) * q.s3; + } + + sumf += sub_group_shuffle_xor(sumf, 16); + sumf += sub_group_shuffle_xor(sumf, 8); + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_h == 0) { + dst[im * ne1 * ne0 + r0] = sumf; + } + } +} + +#define N_K_ROWS_GQA 16 +#define GQA_RATIO_GQA 8 +#define LANES_PER_QH 8 // 64 / GQA_RATIO_GQA +#define DK_VEC_GQA 32 // DK / 4 for DK=128 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_gqa4( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int q_id = sgs_lid >> 3; // 0..7: which Q-head (8 per WG) + const int lane_q = sgs_lid & 7; // 0..7: lane within Q-head partition + + const int r0_base = get_group_id(0) * N_K_ROWS_GQA; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; // K-head index (also K2 batch) + const int i03 = im_kv / ne02; // n13 batch index + + const int q_head_lo = i02 * GQA_RATIO_GQA; + + __local float4 q_loc[GQA_RATIO_GQA * DK_VEC_GQA]; // 4 × 32 = 128 float4 + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_GQA; ++qh) { + const int qh_idx = q_head_lo + qh; + global float4 * y4 = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + + if (sgs_lid < DK_VEC_GQA) { + q_loc[qh * DK_VEC_GQA + sgs_lid] = y4[sgs_lid]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + // K base offset for this WG. All 8 K-rows × 4 Q-heads share this K-head. + const ulong offset_src0_base = (i02) * nb02 + (i03 / r3) * nb03; + + #pragma unroll + for (int dr = 0; dr < N_K_ROWS_GQA; ++dr) { + const int r0 = r0_base + dr; + + const ulong offset_src0 = r0 * nb01 + offset_src0_base; + global half4 * x4 = (global half4 *)(src0 + offset_src0); + + float sumf = 0.0f; + #pragma unroll + for (int t = 0; t < 4; ++t) { + const int i = lane_q + t * LANES_PER_QH; // 8, 16, 24-step + const half4 k4 = x4[i]; + const float4 q = q_loc[q_id * DK_VEC_GQA + i]; + sumf += convert_float(k4.s0) * q.s0 + + convert_float(k4.s1) * q.s1 + + convert_float(k4.s2) * q.s2 + + convert_float(k4.s3) * q.s3; + } + + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_q == 0) { + const int im_out = i03 * ne12 + (q_head_lo + q_id); + dst[im_out * ne1 * ne0 + r0] = sumf; + } + } +} + +#define N_DV_ROWS_Y8GQA 8 +#define GQA_RATIO_Y8GQA 8 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_y8_gqa( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char *)((global char *)src0 + offset0); + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int sgs_sz = get_max_sub_group_size(); + + const int r0_base = get_group_id(0) * N_DV_ROWS_Y8GQA; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; // K-head index + const int i03 = im_kv / ne02; // n13 batch index + + // GQA Q-heads sharing this K-head. + const int q_head_lo = i02 * GQA_RATIO_Y8GQA; + + global float4 * y4_q[GQA_RATIO_Y8GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + const int qh_idx = q_head_lo + qh; + y4_q[qh] = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + } + + global half4 * x4_o[N_DV_ROWS_Y8GQA]; + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const int r0 = r0_base + o; + const int r0c = (r0 < ne01) ? r0 : 0; + const ulong off = r0c * nb01 + (i02) * nb02 + (i03 / r3) * nb03; + x4_o[o] = (global half4 *)(src0 + off); + } + + float sum[N_DV_ROWS_Y8GQA][GQA_RATIO_Y8GQA] = { {0.0f} }; + + for (int i = sgs_lid; i < ne00 / 4; i += sgs_sz) { + // load 8 V values (one per DV row), same K-head, K-pos = i. + half4 v[N_DV_ROWS_Y8GQA]; + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + v[o] = x4_o[o][i]; + } + + // load 8 softmax values (one per Q-head). + float4 q[GQA_RATIO_Y8GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + q[qh] = y4_q[qh][i]; + } + + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const float4 vf = (float4)(convert_float(v[o].s0), + convert_float(v[o].s1), + convert_float(v[o].s2), + convert_float(v[o].s3)); + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + sum[o][qh] += vf.s0 * q[qh].s0 + + vf.s1 * q[qh].s1 + + vf.s2 * q[qh].s2 + + vf.s3 * q[qh].s3; + } + } + } + + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const int r0 = r0_base + o; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + const float s = sub_group_reduce_add(sum[o][qh]); + if (sgs_lid == 0 && r0 < ne01) { + const int im_out = i03 * ne12 + (q_head_lo + qh); + dst[im_out * ne1 * ne0 + r0] = s; + } + } + } +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_gqa4_img( + __read_only image1d_buffer_t src0_img, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int q_id = sgs_lid >> 3; // 0..7: which Q-head (8 per WG) + const int lane_q = sgs_lid & 7; // 0..7: lane within Q-head partition + + const int r0_base = get_group_id(0) * N_K_ROWS_GQA; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; + const int i03 = im_kv / ne02; + + const int q_head_lo = i02 * GQA_RATIO_GQA; + + __local float4 q_loc[GQA_RATIO_GQA * DK_VEC_GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_GQA; ++qh) { + const int qh_idx = q_head_lo + qh; + global float4 * y4 = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + if (sgs_lid < DK_VEC_GQA) { + q_loc[qh * DK_VEC_GQA + sgs_lid] = y4[sgs_lid]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + const int pitch_px_row = (int)(nb01 >> 4); + const int pitch_px_head = (int)(nb02 >> 4); + const int pitch_px_n13 = (int)(nb03 >> 4); + + const int head_px_base = i02 * pitch_px_head + (i03 / r3) * pitch_px_n13; + + #pragma unroll + for (int dr = 0; dr < N_K_ROWS_GQA; ++dr) { + const int r0 = r0_base + dr; + const int row_px_base = r0 * pitch_px_row + head_px_base; + + float sumf = 0.0f; + #pragma unroll + for (int t = 0; t < 2; ++t) { + const int p = lane_q + t * LANES_PER_QH; // pixel idx in row, 0..15 + const half8 k8 = as_half8(read_imagef(src0_img, row_px_base + p)); + const int i0 = 2 * p; // first half4 idx + const float4 qa = q_loc[q_id * DK_VEC_GQA + i0 ]; + const float4 qb = q_loc[q_id * DK_VEC_GQA + i0 + 1]; + sumf += convert_float(k8.s0) * qa.s0 + + convert_float(k8.s1) * qa.s1 + + convert_float(k8.s2) * qa.s2 + + convert_float(k8.s3) * qa.s3 + + convert_float(k8.s4) * qb.s0 + + convert_float(k8.s5) * qb.s1 + + convert_float(k8.s6) * qb.s2 + + convert_float(k8.s7) * qb.s3; + } + + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_q == 0) { + const int im_out = i03 * ne12 + (q_head_lo + q_id); + dst[im_out * ne1 * ne0 + r0] = sumf; + } + } +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_y8_gqa_img( + __read_only image1d_buffer_t src0_img, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int sgs_sz = get_max_sub_group_size(); + + const int r0_base = get_group_id(0) * N_DV_ROWS_Y8GQA; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; + const int i03 = im_kv / ne02; + + const int q_head_lo = i02 * GQA_RATIO_Y8GQA; + + // Q (= softmax(KQ)) base pointers per Q-head + global float4 * y4_q[GQA_RATIO_Y8GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + const int qh_idx = q_head_lo + qh; + y4_q[qh] = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + } + + const int pitch_px_row = (int)(nb01 >> 3); + const int pitch_px_head = (int)(nb02 >> 3); + const int pitch_px_n13 = (int)(nb03 >> 3); + + const int head_px_base = i02 * pitch_px_head + (i03 / r3) * pitch_px_n13; + + // per-DV-row pixel base + int row_px_base[N_DV_ROWS_Y8GQA]; + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const int r0 = r0_base + o; + const int r0c = (r0 < ne01) ? r0 : 0; + row_px_base[o] = r0c * pitch_px_row + head_px_base; + } + + float sum[N_DV_ROWS_Y8GQA][GQA_RATIO_Y8GQA] = { {0.0f} }; + + for (int i = sgs_lid; i < ne00 / 4; i += sgs_sz) { + half4 v[N_DV_ROWS_Y8GQA]; + + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + v[o] = read_imageh(src0_img, row_px_base[o] + i); + } + + float4 q[GQA_RATIO_Y8GQA]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + q[qh] = y4_q[qh][i]; + } + // 64 mads. + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const float4 vf = (float4)(convert_float(v[o].s0), + convert_float(v[o].s1), + convert_float(v[o].s2), + convert_float(v[o].s3)); + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + sum[o][qh] += vf.s0 * q[qh].s0 + + vf.s1 * q[qh].s1 + + vf.s2 * q[qh].s2 + + vf.s3 * q[qh].s3; + } + } + } + + #pragma unroll + for (int o = 0; o < N_DV_ROWS_Y8GQA; ++o) { + const int r0 = r0_base + o; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_Y8GQA; ++qh) { + const float s = sub_group_reduce_add(sum[o][qh]); + if (sgs_lid == 0 && r0 < ne01) { + const int im_out = i03 * ne12 + (q_head_lo + qh); + dst[im_out * ne1 * ne0 + r0] = s; + } + } + } +} + +#define N_K_ROWS_GQA_R4 16 +#define GQA_RATIO_R4 4 +#define LANES_PER_QH_R4 16 // = 64 / GQA_RATIO_R4 +#define DK_VEC_R4 32 // DK / 4 for DK=128 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_gqa_r4_img( + __read_only image1d_buffer_t src0_img, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int q_id = sgs_lid >> 4; // 0..3 + const int lane_q = sgs_lid & 15; // 0..15 + + const int r0_base = get_group_id(0) * N_K_ROWS_GQA_R4; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; + const int i03 = im_kv / ne02; + + const int q_head_lo = i02 * GQA_RATIO_R4; + + __local float4 q_loc[GQA_RATIO_R4 * DK_VEC_R4]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_R4; ++qh) { + const int qh_idx = q_head_lo + qh; + global float4 * y4 = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + if (sgs_lid < DK_VEC_R4) { + q_loc[qh * DK_VEC_R4 + sgs_lid] = y4[sgs_lid]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + + const int pitch_px_row = (int)(nb01 >> 4); + const int pitch_px_head = (int)(nb02 >> 4); + const int pitch_px_n13 = (int)(nb03 >> 4); + + const int head_px_base = i02 * pitch_px_head + (i03 / r3) * pitch_px_n13; + + #pragma unroll + for (int dr = 0; dr < N_K_ROWS_GQA_R4; ++dr) { + const int r0 = r0_base + dr; + const int row_px_base = r0 * pitch_px_row + head_px_base; + + const int p = lane_q; + const half8 k8 = as_half8(read_imagef(src0_img, row_px_base + p)); + const int i0 = 2 * p; + const float4 qa = q_loc[q_id * DK_VEC_R4 + i0 ]; + const float4 qb = q_loc[q_id * DK_VEC_R4 + i0 + 1]; + + float sumf = + convert_float(k8.s0) * qa.s0 + + convert_float(k8.s1) * qa.s1 + + convert_float(k8.s2) * qa.s2 + + convert_float(k8.s3) * qa.s3 + + convert_float(k8.s4) * qb.s0 + + convert_float(k8.s5) * qb.s1 + + convert_float(k8.s6) * qb.s2 + + convert_float(k8.s7) * qb.s3; + + sumf += sub_group_shuffle_xor(sumf, 8); + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_q == 0) { + const int im_out = i03 * ne12 + (q_head_lo + q_id); + dst[im_out * ne1 * ne0 + r0] = sumf; + } + } +} + +#define N_K_ROWS_GQA_R2_DK256 16 +#define GQA_RATIO_R2 2 +#define LANES_PER_QH_R2 32 // = 64 / GQA_RATIO_R2 +#define DK_VEC_DK256 64 // DK / 4 for DK=256 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_x8_gqa_r2_dk256_img( + __read_only image1d_buffer_t src0_img, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char *)((global char *)src1 + offset1); + dst = (global float*)((global char *)dst + offsetd); + + const int sgs_lid = get_sub_group_local_id(); + const int q_id = sgs_lid >> 5; // 0..1 + const int lane_q = sgs_lid & 31; // 0..31 + + const int r0_base = get_group_id(0) * N_K_ROWS_GQA_R2_DK256; + const int im_kv = get_group_id(2); + + const int i02 = im_kv % ne02; + const int i03 = im_kv / ne02; + + const int q_head_lo = i02 * GQA_RATIO_R2; + + __local float4 q_loc[GQA_RATIO_R2 * DK_VEC_DK256]; + #pragma unroll + for (int qh = 0; qh < GQA_RATIO_R2; ++qh) { + const int qh_idx = q_head_lo + qh; + global float4 * y4 = (global float4 *)(src1 + qh_idx * nb12 + i03 * nb13); + q_loc[qh * DK_VEC_DK256 + sgs_lid] = y4[sgs_lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const int pitch_px_row = (int)(nb01 >> 4); + const int pitch_px_head = (int)(nb02 >> 4); + const int pitch_px_n13 = (int)(nb03 >> 4); + + const int head_px_base = i02 * pitch_px_head + (i03 / r3) * pitch_px_n13; + + #pragma unroll + for (int dr = 0; dr < N_K_ROWS_GQA_R2_DK256; ++dr) { + const int r0 = r0_base + dr; + const int row_px_base = r0 * pitch_px_row + head_px_base; + + const int p = lane_q; + const half8 k8 = as_half8(read_imagef(src0_img, row_px_base + p)); + const int i0 = 2 * p; + const float4 qa = q_loc[q_id * DK_VEC_DK256 + i0 ]; + const float4 qb = q_loc[q_id * DK_VEC_DK256 + i0 + 1]; + + float sumf = + convert_float(k8.s0) * qa.s0 + + convert_float(k8.s1) * qa.s1 + + convert_float(k8.s2) * qa.s2 + + convert_float(k8.s3) * qa.s3 + + convert_float(k8.s4) * qb.s0 + + convert_float(k8.s5) * qb.s1 + + convert_float(k8.s6) * qb.s2 + + convert_float(k8.s7) * qb.s3; + + sumf += sub_group_shuffle_xor(sumf, 16); + sumf += sub_group_shuffle_xor(sumf, 8); + sumf += sub_group_shuffle_xor(sumf, 4); + sumf += sub_group_shuffle_xor(sumf, 2); + sumf += sub_group_shuffle_xor(sumf, 1); + + if (lane_q == 0) { + const int im_out = i03 * ne12 + (q_head_lo + q_id); + dst[im_out * ne1 * ne0 + r0] = sumf; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl new file mode 100644 index 0000000..9a7627c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl @@ -0,0 +1,306 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +// Multi-row f16xf32 GEMV for the DECODE path (single token, ne11*ne12 small). +// The legacy kernel_mul_mat_f16_f32_1row runs ONE 64-lane subgroup per workgroup = +// one output row per WG, which caps memory-level parallelism at roughly half of +// LPDDR5x peak. This variant packs MROW subgroups per workgroup, each +// computing a distinct output row, so a WG keeps 64*MROW loads in flight. The +// activation column y (shared by every output row) is staged into __local ONCE per +// WG and reused across the MROW rows, cutting redundant activation reads. Used for +// the f16 attention projections (Q/K/V/O) and lm_head, which dominate decode. +// Numerically equivalent to _1row (same f16->f32 widening, same float4 partial sums, +// same subgroup-reduce order), so byte-identical to the per-op path. + +#define MROW 16 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3, + __local float * ysh +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + int r0 = get_group_id(0) * MROW + get_local_id(1); // output row + int r1 = get_group_id(1); // token (ne11) + int im = get_group_id(2); + int lid = get_sub_group_local_id(); // 0..63 + int nsg = get_local_size(1); // == MROW + + int i12 = im % ne12; + int i13 = im / ne12; + + ulong offset_src1 = r1*nb11 + (i12)*nb12 + (i13)*nb13; + global float * y = (global float *) (src1 + offset_src1); + + // Cooperatively stage the activation column (ne00 floats) into __local once per + // WG and reuse across the MROW rows. Staging is the actual win here: dropping it + // (each subgroup re-reading y from global) regresses below the 1-row kernel. + for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; i += nsg*get_sub_group_size()) { + ysh[i] = y[i]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (r0 >= ne01) { + return; + } + + ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + global half * x = (global half *) (src0 + offset_src0); + + // The vector path below casts the row pointer to half4, which must be 8-byte aligned. + // A row address is r0*nb01 + ..., and a permuted or strided src0 leaves nb01/nb02/nb03 + // unconstrained -- ne00 % 4 == 0 bounds the element count per row, not the byte stride + // between rows. Take the vector path only when this work-item's row is actually + // aligned; the scalar loop below has no such requirement. + const bool row_aligned = (((ulong) x) & 7) == 0; + + float sumf = 0.0f; + if (ne00 < 128 || !row_aligned) { + for (int i = lid; i < ne00; i += get_sub_group_size()) { + sumf += (float) x[i] * ysh[i]; + } + float all_sum = sub_group_reduce_add(sumf); + if (lid == 0) { + dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum; + } + } else { + global half4 * x4 = (global half4 *) x; + __local float4 * ysh4 = (__local float4 *) ysh; + for (int i = lid; i < ne00/4; i += get_sub_group_size()) { + float4 yv = ysh4[i]; + sumf += (float) x4[i].s0 * yv.s0; + sumf += (float) x4[i].s1 * yv.s1; + sumf += (float) x4[i].s2 * yv.s2; + sumf += (float) x4[i].s3 * yv.s3; + } + float all_sum = sub_group_reduce_add(sumf); + if (lid == 0) { + for (int i = 4*(ne00/4); i < ne00; ++i) { + all_sum += (float) x[i] * ysh[i]; + } + dst[im*ne1*ne0 + r1*ne0 + r0] = all_sum; + } + } +} + +// Register-blocked variant: each 64-lane subgroup accumulates RPT consecutive +// output rows instead of one. The staged activation is reused across all RPT rows, +// and each lane keeps RPT independent weight loads in flight per column step -> +// more memory-level parallelism on the streaming f16 weight read (the BW limiter), +// plus RPT fewer staging barriers per output row. Per-row reduction order is +// identical to _mrow, so byte-identical to the per-op path. Dispatch guarantees +// ne00 >= 128 and ne00 % 4 == 0, so only the half4 path is needed (no tail). +#define MROW_RB_BODY(RPT) \ + src0 = (global char*)((global char*)src0 + offset0); \ + src1 = (global char*)((global char*)src1 + offset1); \ + dst = (global float*)((global char*)dst + offsetd); \ + int r0b = (get_group_id(0) * get_local_size(1) + get_local_id(1)) * (RPT); \ + int r1 = get_group_id(1); \ + int im = get_group_id(2); \ + int lid = get_sub_group_local_id(); \ + int nsg = get_local_size(1); \ + int i12 = im % ne12; \ + int i13 = im / ne12; \ + ulong off_y = r1*nb11 + i12*nb12 + i13*nb13; \ + global float * y = (global float *) (src1 + off_y); \ + for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; \ + i += nsg*get_sub_group_size()) { \ + ysh[i] = y[i]; \ + } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + __local float4 * ysh4 = (__local float4 *) ysh; \ + global half4 * xr[RPT]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + int row = r0b + rr; \ + if (row > ne01 - 1) row = ne01 - 1; \ + xr[rr] = (global half4 *) (src0 + (ulong)row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); \ + } \ + float sumf[RPT]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) sumf[rr] = 0.0f; \ + for (int i = lid; i < ne00/4; i += get_sub_group_size()) { \ + float4 yv = ysh4[i]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + half4 xv = xr[rr][i]; \ + sumf[rr] += (float) xv.s0 * yv.s0 + (float) xv.s1 * yv.s1 \ + + (float) xv.s2 * yv.s2 + (float) xv.s3 * yv.s3; \ + } \ + } \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + float s = sub_group_reduce_add(sumf[rr]); \ + int row = r0b + rr; \ + if (lid == 0 && row < ne01) { \ + dst[im*ne1*ne0 + r1*ne0 + row] = s; \ + } \ + } + +// half8 (128-bit) load variant: Adreno's load/store unit issues 128-bit +// transactions, so half4 (64-bit) loads may leave the load path half-idle. This +// processes 8 weight elements per lane per step via half8. Accumulation groups +// elements in 8s rather than 4s, so it is NOT bit-identical to _1row (float add is +// non-associative) -- experimental BW probe, gate on ne00 % 8 == 0. +#define MROW_H8_BODY(RPT) \ + src0 = (global char*)((global char*)src0 + offset0); \ + src1 = (global char*)((global char*)src1 + offset1); \ + dst = (global float*)((global char*)dst + offsetd); \ + int r0b = (get_group_id(0) * get_local_size(1) + get_local_id(1)) * (RPT); \ + int r1 = get_group_id(1); \ + int im = get_group_id(2); \ + int lid = get_sub_group_local_id(); \ + int nsg = get_local_size(1); \ + int i12 = im % ne12; \ + int i13 = im / ne12; \ + ulong off_y = r1*nb11 + i12*nb12 + i13*nb13; \ + global float * y = (global float *) (src1 + off_y); \ + for (int i = get_local_id(1)*get_sub_group_size() + lid; i < ne00; \ + i += nsg*get_sub_group_size()) { \ + ysh[i] = y[i]; \ + } \ + barrier(CLK_LOCAL_MEM_FENCE); \ + __local float4 * ysh4 = (__local float4 *) ysh; \ + global half8 * xr[RPT]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + int row = r0b + rr; \ + if (row > ne01 - 1) row = ne01 - 1; \ + xr[rr] = (global half8 *) (src0 + (ulong)row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); \ + } \ + float sumf[RPT]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) sumf[rr] = 0.0f; \ + for (int i = lid; i < ne00/8; i += get_sub_group_size()) { \ + float4 y0 = ysh4[2*i]; \ + float4 y1 = ysh4[2*i + 1]; \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + half8 xv = xr[rr][i]; \ + sumf[rr] += (float) xv.s0 * y0.s0 + (float) xv.s1 * y0.s1 \ + + (float) xv.s2 * y0.s2 + (float) xv.s3 * y0.s3 \ + + (float) xv.s4 * y1.s0 + (float) xv.s5 * y1.s1 \ + + (float) xv.s6 * y1.s2 + (float) xv.s7 * y1.s3; \ + } \ + } \ + _Pragma("unroll") \ + for (int rr = 0; rr < (RPT); ++rr) { \ + float s = sub_group_reduce_add(sumf[rr]); \ + int row = r0b + rr; \ + if (lid == 0 && row < ne01) { \ + dst[im*ne1*ne0 + r1*ne0 + row] = s; \ + } \ + } + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow_h8( + global char * src0, ulong offset0, + global char * src1, ulong offset1, + global float * dst, ulong offsetd, + int ne00, int ne01, int ne02, + ulong nb00, ulong nb01, ulong nb02, ulong nb03, + int ne10, int ne11, int ne12, + ulong nb10, ulong nb11, ulong nb12, ulong nb13, + int ne0, int ne1, int r2, int r3, + __local float * ysh +) { + MROW_H8_BODY(1) +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow_h8r2( + global char * src0, ulong offset0, + global char * src1, ulong offset1, + global float * dst, ulong offsetd, + int ne00, int ne01, int ne02, + ulong nb00, ulong nb01, ulong nb02, ulong nb03, + int ne10, int ne11, int ne12, + ulong nb10, ulong nb11, ulong nb12, ulong nb13, + int ne0, int ne1, int r2, int r3, + __local float * ysh +) { + MROW_H8_BODY(2) +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow_r2( + global char * src0, ulong offset0, + global char * src1, ulong offset1, + global float * dst, ulong offsetd, + int ne00, int ne01, int ne02, + ulong nb00, ulong nb01, ulong nb02, ulong nb03, + int ne10, int ne11, int ne12, + ulong nb10, ulong nb11, ulong nb12, ulong nb13, + int ne0, int ne1, int r2, int r3, + __local float * ysh +) { + MROW_RB_BODY(2) +} + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_mrow_r4( + global char * src0, ulong offset0, + global char * src1, ulong offset1, + global float * dst, ulong offsetd, + int ne00, int ne01, int ne02, + ulong nb00, ulong nb01, ulong nb02, ulong nb03, + int ne10, int ne11, int ne12, + ulong nb10, ulong nb11, ulong nb12, ulong nb13, + int ne0, int ne1, int r2, int r3, + __local float * ysh +) { + MROW_RB_BODY(4) +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl index d50bd1f..4c0755e 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl @@ -172,6 +172,9 @@ kernel void kernel_mul_mv_id_mxfp4_f32( const int idx = get_group_id(2)%ne20; int i02 = ((global int *) (src2 + iid1*nb21))[idx]; + if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int i11 = idx % ne11; int i12 = iid1; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl index f65e86e..962ddd7 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl @@ -101,6 +101,9 @@ kernel void kernel_mul_mv_id_mxfp4_f32_flat( const int idx = get_group_id(2) % ne20; uint i02 = ((global uint *) (src2 + offset2 + iid1 * nb21))[idx]; + if ((int)i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int i11 = idx % ne11; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl index 7ccf41e..1331468 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl @@ -267,6 +267,9 @@ kernel void kernel_mul_mv_id_q4_0_f32_8x_flat( const int idx = get_group_id(2)%ne20; const int i02 = ((global int *)(src2 + iid1*nb21))[idx]; + if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } const int i11 = idx%ne11; const int i12 = iid1; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl index f37e83e..2c40a43 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl @@ -73,6 +73,9 @@ kernel void kernel_mul_mv_id_q8_0_f32( int idx = get_group_id(2)%ne20; int i02 = ((global int *) (src2 + iid1*nb21))[idx]; + if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int i11_ = idx % ne11; int i12_ = iid1; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl index fd3a071..3b1b39b 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl @@ -72,6 +72,9 @@ kernel void kernel_mul_mv_id_q8_0_f32_flat( int idx = (int)get_group_id(2)%ne20; int i02 = ((global int *) (src2 + iid1*nb21))[idx]; + if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst + return; + } int i11_ = idx % ne11; int i12_ = iid1; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl new file mode 100644 index 0000000..a6a325c --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl @@ -0,0 +1,164 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK4_NL 32 + +typedef char int8_t; +typedef uchar uint8_t; +typedef short int16_t; +typedef ushort uint16_t; +typedef int int32_t; +typedef uint uint32_t; + +constant float kvalues_iq4nl[16] = { + -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, + 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f +}; + +//------------------------------------------------------------------------------ +// block_iq4_nl +//------------------------------------------------------------------------------ +struct block_iq4_nl +{ + half d; + uint8_t qs[QK4_NL / 2]; +}; + +//------------------------------------------------------------------------------ +// mul_vec_q_n_f32 +//------------------------------------------------------------------------------ +// Compute inner product between half a block of iq4_nl and 16 floats (yl). +// il indicates where the quants begin (0 or 8). +inline float block_iq4_nl_dot_y( + global struct block_iq4_nl * qb_curr, + private float * yl, + int il +) { + float d = qb_curr->d; + float acc = 0.f; + global uchar * qs = qb_curr->qs + il; + for (int i = 0; i < 8; ++i) { + acc += yl[i] * kvalues_iq4nl[qs[i] & 0x0F]; + acc += yl[i+8] * kvalues_iq4nl[qs[i] >> 4]; + } + return d * acc; +} + +#ifdef INTEL_GPU +#define N_DST 4 // each subgroup group works on 4 rows +#define N_SUBGROUP 1 // number of subgroups in a thread group +#define N_SUBGROUP_SIZE 16 // assuming subgroup size is 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SUBGROUP 1 +#define N_SUBGROUP_SIZE 64 +#endif + +inline void mul_vec_q_n_f32( + global void * src0, + global float * src1, + global float * dst, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + + const ulong nb = ne00/QK4_NL; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0 * N_SUBGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + global struct block_iq4_nl * x = (global struct block_iq4_nl *) src0 + offset0; + global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1; + + float yl[16]; // src1 vector cache + float sumf[N_DST]={0.f}; + + int ix = get_sub_group_local_id()/2; + int il = 8*(get_sub_group_local_id()%2); + + global float * yb = y + ix * QK4_NL + il; + + // each thread in a SIMD group deals with half a block. + for (int ib = ix; ib < nb; ib += N_SUBGROUP_SIZE/2) { + for (int i = 0; i < 8; ++i) { + yl[i] = yb[i]; + yl[i+8] = yb[i+16]; + } + + for (int row = 0; row < N_DST; row++) { + sumf[row] += block_iq4_nl_dot_y(x+ib+row*nb, yl, il); + } + + yb += QK4_NL * (N_SUBGROUP_SIZE/2); + } + + float tot[N_DST] = { + sub_group_reduce_add(sumf[0]), sub_group_reduce_add(sumf[1]), + sub_group_reduce_add(sumf[2]), sub_group_reduce_add(sumf[3])}; + for (int row = 0; row < N_DST; ++row) { + if (get_sub_group_local_id() == 0 && first_row + row < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot[row]; + } + } +} + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq4_nl_f32( + global void * src0, + ulong offset0, + global float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global void*)((global char*)src0 + offset0); + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3); +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl new file mode 100644 index 0000000..8c5b3f5 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl @@ -0,0 +1,202 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK4_NL 32 + +typedef char int8_t; +typedef uchar uint8_t; +typedef short int16_t; +typedef ushort uint16_t; +typedef int int32_t; +typedef uint uint32_t; + +constant float kvalues_iq4nl[16] = { + -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f, + 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f +}; + +//------------------------------------------------------------------------------ +// block_iq4_nl +//------------------------------------------------------------------------------ +struct block_iq4_nl +{ + half d; + uint8_t qs[QK4_NL / 2]; +}; + +// Compute dot product between half a block of iq4_nl quants and activations. +// x points to the quant bytes, dh points to the scale. +// yl has 16 activation values: [0..7] for low nibbles, [8..15] for high nibbles. +// il indicates offset into the quant bytes (0 or 8). +inline float block_iq4_nl_dot_y_flat( + global uchar * x, + global half * dh, + private float * yl, + int il +) { + float d = *dh; + global uchar * qs = x + il; + float acc = 0.f; + for (int i = 0; i < 8; ++i) { + acc += yl[i] * kvalues_iq4nl[qs[i] & 0x0F]; + acc += yl[i+8] * kvalues_iq4nl[qs[i] >> 4]; + } + return d * acc; +} + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 8 // each subgroup works on 8 rows +#define N_SUBGROUP 1 // number of subgroups in a thread group +#define N_SUBGROUP_SIZE 16 // assuming subgroup size is 16 +#elif defined (ADRENO_GPU) +#define N_DST 8 +#define N_SUBGROUP 1 +#define N_SUBGROUP_SIZE 64 +#endif + +inline void mul_vec_q_n_f32_8x_flat( + global uchar * src0_q, + global half * src0_d, + global float * src1, + global float * dst, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + const ulong nb = ne00/QK4_NL; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0 * N_SUBGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + // The number of scales is the same as the number of blocks. + ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + // Each block contains QK4_NL/2 uchars, hence offset for qs is as follows. + ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_NL/2; + + global uchar * x = (global uchar *) src0_q + offset0_q; + global half * d = (global half *) src0_d + offset0_d; + global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1; + + float yl[16]; + float8 sumf = 0.f; + + int ix = get_sub_group_local_id()/2; + int il = 8*(get_sub_group_local_id()%2); + + global float * yb = y + ix*QK4_NL + il; + + for (int ib = ix; ib < nb; ib += N_SUBGROUP_SIZE/2) { + for (int i = 0; i < 8; ++i) { + yl[i] = yb[i]; + yl[i+8] = yb[i+16]; + } + + sumf.s0 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 0*nb*QK4_NL/2, d + ib + 0*nb, yl, il); + sumf.s1 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 1*nb*QK4_NL/2, d + ib + 1*nb, yl, il); + sumf.s2 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 2*nb*QK4_NL/2, d + ib + 2*nb, yl, il); + sumf.s3 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 3*nb*QK4_NL/2, d + ib + 3*nb, yl, il); + + sumf.s4 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 4*nb*QK4_NL/2, d + ib + 4*nb, yl, il); + sumf.s5 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 5*nb*QK4_NL/2, d + ib + 5*nb, yl, il); + sumf.s6 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 6*nb*QK4_NL/2, d + ib + 6*nb, yl, il); + sumf.s7 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 7*nb*QK4_NL/2, d + ib + 7*nb, yl, il); + + yb += QK4_NL * (N_SUBGROUP_SIZE/2); + } + + float8 tot = (float8)( + sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1), + sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3), + sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5), + sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7) + ); + + if (get_sub_group_local_id() == 0) { + if (first_row + 0 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0; + } + if (first_row + 1 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1; + } + if (first_row + 2 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2; + } + if (first_row + 3 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3; + } + + if (first_row + 4 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4; + } + if (first_row + 5 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5; + } + if (first_row + 6 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6; + } + if (first_row + 7 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7; + } + } +} + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq4_nl_f32_flat( + global uchar * src0_q, + global half * src0_d, + global float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + mul_vec_q_n_f32_8x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3); +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl index 3d5a923..2e31831 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl @@ -132,7 +132,7 @@ kernel void kernel_mul_mv_mxfp4_f32_flat( global float4 * y4 = (global float4 *)yb; #pragma unroll - for (short row = 0; row < N_R0_MXFP4; row++) { + for (short row = 0; row < N_R0_MXFP4 && first_row + row < ne0; row++) { uchar xb_e = x_e[row * nb + ib]; #ifdef SRC0Q_IMG ushort4 xb_q = as_ushort4(read_imageui(src0_q, (offset_q + row * nb + ib) * 2 + it).xy); diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl new file mode 100644 index 0000000..8746e2a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl @@ -0,0 +1,141 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK1_0 128 +typedef struct { + half d; + uchar qs[QK1_0/8]; +} block_q1_0; + +#define NB_Q1_0 16 + +#ifdef INTEL_GPU +#define N_R0_Q1_0 4 // number of rows each subgroup works on +#define N_SG_Q1_0 2 // number of subgroups in a work group +#define N_SIMDWIDTH 16 // subgroup size +#elif defined (ADRENO_GPU) +#define N_R0_Q1_0 4 +#define N_SG_Q1_0 2 +#define N_SIMDWIDTH 64 +#endif + +inline float block_q_1_0_dot_y(global block_q1_0 * qb, float sumy, float yl[NB_Q1_0], short il) { + global uchar * qs = qb->qs + il*2; + uint b0 = qs[0]; + uint b1 = qs[1]; + + float acc = 0.f; + acc += yl[ 0]*(float)((b0 >> 0) & 1) + yl[ 1]*(float)((b0 >> 1) & 1); + acc += yl[ 2]*(float)((b0 >> 2) & 1) + yl[ 3]*(float)((b0 >> 3) & 1); + acc += yl[ 4]*(float)((b0 >> 4) & 1) + yl[ 5]*(float)((b0 >> 5) & 1); + acc += yl[ 6]*(float)((b0 >> 6) & 1) + yl[ 7]*(float)((b0 >> 7) & 1); + + acc += yl[ 8]*(float)((b1 >> 0) & 1) + yl[ 9]*(float)((b1 >> 1) & 1); + acc += yl[10]*(float)((b1 >> 2) & 1) + yl[11]*(float)((b1 >> 3) & 1); + acc += yl[12]*(float)((b1 >> 4) & 1) + yl[13]*(float)((b1 >> 5) & 1); + acc += yl[14]*(float)((b1 >> 6) & 1) + yl[15]*(float)((b1 >> 7) & 1); + + return qb->d * (2.0f*acc - sumy); +} + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q1_0_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global char*)((global char*)dst + offsetd); + + int nb = ne00/QK1_0; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0*N_SG_Q1_0 + get_sub_group_id()) * N_R0_Q1_0; + + uint i12 = im%ne12; + uint i13 = im/ne12; + + ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13; + global float * y = (global float *) (src1 + offset_src1); + + // pointers to src0 rows + global block_q1_0 * ax[N_R0_Q1_0]; + for (int row = 0; row < N_R0_Q1_0; ++row) { + ulong offset_src0 = (first_row + row)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ax[row] = (global block_q1_0 *) ((global char *) src0 + offset_src0); + } + + float yl[NB_Q1_0]; + float sumf[N_R0_Q1_0] = { 0.f }; + + const short ix = get_sub_group_local_id()/8; + const short il = get_sub_group_local_id()%8; + + global float * yb = y + ix*QK1_0 + il*NB_Q1_0; + + // each thread handles NB_Q1_0 quants at a time + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) { + float sumy = 0.f; + for (short i = 0; i < NB_Q1_0; ++i) { + yl[i] = yb[i]; + sumy += yb[i]; + } + + for (short row = 0; row < N_R0_Q1_0; row++) { + sumf[row] += block_q_1_0_dot_y(ax[row] + ib, sumy, yl, il); + } + + yb += N_SIMDWIDTH*NB_Q1_0; + } + + global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0; + + for (int row = 0; row < N_R0_Q1_0; ++row) { + float tot = sub_group_reduce_add(sumf[row]); + + if (get_sub_group_local_id() == 0 && first_row + row < ne01) { + dst_f32[first_row + row] = tot; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl new file mode 100644 index 0000000..870e36d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl @@ -0,0 +1,190 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK1_0 128 +#define QK1_0_BYTES (QK1_0/8) // 16 quant bytes per block +#define QK1_0_BLK_BYTES (QK1_0_BYTES + 2) // d + qs in original tensor = 18 + +#define NB_Q1_0 16 // quants handled per thread (two qs bytes) + +#ifdef INTEL_GPU +#define N_R0_Q1_0 4 // number of rows each subgroup works on +#define N_SG_Q1_0 2 // number of subgroups in a work group +#define N_SIMDWIDTH 16 // subgroup size +#elif defined (ADRENO_GPU) +#define N_R0_Q1_0 4 +#define N_SG_Q1_0 2 +#define N_SIMDWIDTH 64 +#endif + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q1_0_f32_flat( + global char * src0_q, + global half * src0_d, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global char*)((global char*)src1 + offset1); + dst = (global char*)((global char*)dst + offsetd); + + int nb = ne00/QK1_0; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0*N_SG_Q1_0 + get_sub_group_id()) * N_R0_Q1_0; + + uint i12 = im%ne12; + uint i13 = im/ne12; + + ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13; + global float * y = (global float *) (src1 + offset_src1); + + // pointers to src0 rows (flat: q bytes + scales) + uint offset_src0_base = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + + global uchar * ax0, * ax1, * ax2, * ax3; + global half * ad0, * ad1, * ad2, * ad3; + uint offset_src0; + + offset_src0 = (offset_src0_base + 0*nb01) / QK1_0_BLK_BYTES; + ax0 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES); + ad0 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half)); + + offset_src0 = (offset_src0_base + 1*nb01) / QK1_0_BLK_BYTES; + ax1 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES); + ad1 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half)); + + offset_src0 = (offset_src0_base + 2*nb01) / QK1_0_BLK_BYTES; + ax2 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES); + ad2 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half)); + + offset_src0 = (offset_src0_base + 3*nb01) / QK1_0_BLK_BYTES; + ax3 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES); + ad3 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half)); + + const short ix = get_sub_group_local_id()/8; + const short il = get_sub_group_local_id()%8; + + global float * yb = y + ix*QK1_0 + il*NB_Q1_0; + + float8 yl_lo; + float8 yl_hi; + float4 sumf = 0.f; + + // each thread handles NB_Q1_0 = 16 quants (two qs bytes) at a time + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) { + yl_lo = vload8(0, yb); + yl_hi = vload8(0, yb + 8); + float sumy = yl_lo.s0 + yl_lo.s1 + yl_lo.s2 + yl_lo.s3 + + yl_lo.s4 + yl_lo.s5 + yl_lo.s6 + yl_lo.s7 + + yl_hi.s0 + yl_hi.s1 + yl_hi.s2 + yl_hi.s3 + + yl_hi.s4 + yl_hi.s5 + yl_hi.s6 + yl_hi.s7; + + uint b0, b1; + float acc; + + b0 = ax0[ib*QK1_0_BYTES + il*2 + 0]; + b1 = ax0[ib*QK1_0_BYTES + il*2 + 1]; + acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1) + + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1) + + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1) + + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1) + + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1) + + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1) + + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1) + + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1); + sumf.s0 += (float)ad0[ib] * (2.0f*acc - sumy); + + b0 = ax1[ib*QK1_0_BYTES + il*2 + 0]; + b1 = ax1[ib*QK1_0_BYTES + il*2 + 1]; + acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1) + + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1) + + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1) + + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1) + + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1) + + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1) + + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1) + + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1); + sumf.s1 += (float)ad1[ib] * (2.0f*acc - sumy); + + b0 = ax2[ib*QK1_0_BYTES + il*2 + 0]; + b1 = ax2[ib*QK1_0_BYTES + il*2 + 1]; + acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1) + + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1) + + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1) + + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1) + + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1) + + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1) + + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1) + + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1); + sumf.s2 += (float)ad2[ib] * (2.0f*acc - sumy); + + b0 = ax3[ib*QK1_0_BYTES + il*2 + 0]; + b1 = ax3[ib*QK1_0_BYTES + il*2 + 1]; + acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1) + + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1) + + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1) + + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1) + + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1) + + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1) + + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1) + + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1); + sumf.s3 += (float)ad3[ib] * (2.0f*acc - sumy); + + yb += N_SIMDWIDTH*NB_Q1_0; + } + + global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0; + + float4 tot = (float4)( + sub_group_reduce_add(sumf.s0), + sub_group_reduce_add(sumf.s1), + sub_group_reduce_add(sumf.s2), + sub_group_reduce_add(sumf.s3) + ); + + if (get_sub_group_local_id() == 0) { + if (first_row + 0 < ne01) dst_f32[first_row + 0] = tot.s0; + if (first_row + 1 < ne01) dst_f32[first_row + 1] = tot.s1; + if (first_row + 2 < ne01) dst_f32[first_row + 2] = tot.s2; + if (first_row + 3 < ne01) dst_f32[first_row + 3] = tot.s3; + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl new file mode 100644 index 0000000..7f53281 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl @@ -0,0 +1,161 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// block_q2_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q + b +typedef struct { + uchar scales[QK_K/16]; // scales and mins, quantized with 4 bits + uchar qs[QK_K/4]; // 2-bit quants + half d; // super-block scale for quantized scales + half dmin; // super-block scale for quantized mins +} block_q2_K; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// number of (super) blocks each subgroup processes +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q2_K_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // block index (inside super block) + int iq = it/4; // 0 or 1 - first or second half of the super block + int ir = it%4; // 0...3 - block index in the half super block + int is = (8*ir)/16; // 0 or 1 - scale pair + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_q2_K * x = (global block_q2_K *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 128 * iq + 8 * ir; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + float4 sumy = {0.f, 0.f, 0.f, 0.f}; + for (int i = 0; i < 8; ++i) { + yl[i+ 0] = y4[i+ 0]; sumy.s0 += yl[i+ 0]; + yl[i+ 8] = y4[i+32]; sumy.s1 += yl[i+ 8]; + yl[i+16] = y4[i+64]; sumy.s2 += yl[i+16]; + yl[i+24] = y4[i+96]; sumy.s3 += yl[i+24]; + } + + global uchar * sc = (global uchar *)x[ib].scales + 8*iq + is; + global ushort * qs = (global ushort *)x[ib].qs + 16 * iq + 4 * ir; + global half * dh = &x[ib].d; + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + float4 acc1 = {0.f, 0.f, 0.f, 0.f}; + float4 acc2 = {0.f, 0.f, 0.f, 0.f}; + for (int i = 0; i < 8; i += 2) { + acc1.s0 += yl[i+ 0] * (qs[i/2] & 0x0003); + acc2.s0 += yl[i+ 1] * (qs[i/2] & 0x0300); + acc1.s1 += yl[i+ 8] * (qs[i/2] & 0x000c); + acc2.s1 += yl[i+ 9] * (qs[i/2] & 0x0c00); + acc1.s2 += yl[i+16] * (qs[i/2] & 0x0030); + acc2.s2 += yl[i+17] * (qs[i/2] & 0x3000); + acc1.s3 += yl[i+24] * (qs[i/2] & 0x00c0); + acc2.s3 += yl[i+25] * (qs[i/2] & 0xc000); + } + + float dall = dh[0]; + float dmin = dh[1] * 1.f/16.f; + sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc2.s0) * (sc[0] & 0xF) * 1.f/ 1.f + + (acc1.s1 + 1.f/256.f * acc2.s1) * (sc[2] & 0xF) * 1.f/ 4.f + + (acc1.s2 + 1.f/256.f * acc2.s2) * (sc[4] & 0xF) * 1.f/16.f + + (acc1.s3 + 1.f/256.f * acc2.s3) * (sc[6] & 0xF) * 1.f/64.f) - + dmin * (sumy.s0 * (sc[0] & 0xF0) + sumy.s1 * (sc[2] & 0xF0) + + sumy.s2 * (sc[4] & 0xF0) + sumy.s3 * (sc[6] & 0xF0)); + + qs += nb01/2; + sc += nb01; + dh += nb01/2; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl new file mode 100644 index 0000000..c993abe --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl @@ -0,0 +1,162 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// struct block_q2_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q + b +struct block_q2_K { + uchar scales[QK_K/16]; // scales and mins, quantized with 4 bits + uchar qs[QK_K/4]; // 2-bit quants + half d; // super-block scale for quantized scales + half dmin; // super-block scale for quantized mins +}; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// number of (super) blocks each subgroup processes +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q2_K_f32_flat( + global uchar * src0_qs, + global uchar * src0_sc, + global half * src0_dm, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // block index (inside super block) + int iq = it/4; // 0 or 1 - first or second half of the super block + int ir = it%4; // 0...3 - block index in the half super block + int is = (8*ir)/16; // 0 or 1 - scale pair + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(struct block_q2_K); + ulong row_stride = nb01/sizeof(struct block_q2_K); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 128 * iq + 8 * ir; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + float4 sumy = {0.f, 0.f, 0.f, 0.f}; + for (int i = 0; i < 8; ++i) { + yl[i+ 0] = y4[i+ 0]; sumy.s0 += yl[i+ 0]; + yl[i+ 8] = y4[i+32]; sumy.s1 += yl[i+ 8]; + yl[i+16] = y4[i+64]; sumy.s2 += yl[i+16]; + yl[i+24] = y4[i+96]; sumy.s3 += yl[i+24]; + } + + global uchar * sc = (src0_sc + 16*(offset_src0 + ib)) + 8*iq + is; + global ushort * qs = (global ushort *)(src0_qs + 64*(offset_src0 + ib)) + 16 * iq + 4 * ir; + global half * dh = src0_dm + 2*(offset_src0 + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + float4 acc1 = {0.f, 0.f, 0.f, 0.f}; + float4 acc2 = {0.f, 0.f, 0.f, 0.f}; + for (int i = 0; i < 8; i += 2) { + acc1.s0 += yl[i+ 0] * (qs[i/2] & 0x0003); + acc2.s0 += yl[i+ 1] * (qs[i/2] & 0x0300); + acc1.s1 += yl[i+ 8] * (qs[i/2] & 0x000c); + acc2.s1 += yl[i+ 9] * (qs[i/2] & 0x0c00); + acc1.s2 += yl[i+16] * (qs[i/2] & 0x0030); + acc2.s2 += yl[i+17] * (qs[i/2] & 0x3000); + acc1.s3 += yl[i+24] * (qs[i/2] & 0x00c0); + acc2.s3 += yl[i+25] * (qs[i/2] & 0xc000); + } + + float dall = dh[0]; + float dmin = dh[1] * 1.f/16.f; + sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc2.s0) * (sc[0] & 0xF) * 1.f/ 1.f + + (acc1.s1 + 1.f/256.f * acc2.s1) * (sc[2] & 0xF) * 1.f/ 4.f + + (acc1.s2 + 1.f/256.f * acc2.s2) * (sc[4] & 0xF) * 1.f/16.f + + (acc1.s3 + 1.f/256.f * acc2.s3) * (sc[6] & 0xF) * 1.f/64.f) - + dmin * (sumy.s0 * (sc[0] & 0xF0) + sumy.s1 * (sc[2] & 0xF0) + + sumy.s2 * (sc[4] & 0xF0) + sumy.s3 * (sc[6] & 0xF0)); + + qs += row_stride*32; + sc += row_stride*16; + dh += row_stride*2; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl index 71ab989..4c3d5f0 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl @@ -1,3 +1,5 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + #ifdef cl_intel_required_subgroup_size #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable #define INTEL_GPU 1 diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl index d92fb96..5316bd3 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl @@ -40,7 +40,7 @@ typedef struct { #undef N_SIMDWIDTH #ifdef INTEL_GPU -#define N_DST 4 // number of rows each SIMD group works on +#define N_DST 16 // number of rows each SIMD group works on (Intel: 8->16, 2x further activation reuse; 32 spills registers) #define N_SIMDGROUP 1 // number of SIMD groups in a thread group #define N_SIMDWIDTH 16 // SIMD group size #elif defined (ADRENO_GPU) @@ -153,18 +153,27 @@ kernel void kernel_mul_mv_q4_K_f32_flat( global ushort * q2 = q1 + 32; - float4 acc1 = {0.f, 0.f, 0.f, 0.f}; - float4 acc2 = {0.f, 0.f, 0.f, 0.f}; - for (int i = 0; i < 8; i += 2) { - acc1.s0 += yl[i+0] * (q1[i/2] & 0x000F); - acc1.s1 += yl[i+1] * (q1[i/2] & 0x0F00); - acc1.s2 += yl[i+8] * (q1[i/2] & 0x00F0); - acc1.s3 += yl[i+9] * (q1[i/2] & 0xF000); - acc2.s0 += yh[i+0] * (q2[i/2] & 0x000F); - acc2.s1 += yh[i+1] * (q2[i/2] & 0x0F00); - acc2.s2 += yh[i+8] * (q2[i/2] & 0x00F0); - acc2.s3 += yh[i+9] * (q2[i/2] & 0xF000); - } + // Load the 4 q1 / 4 q2 quant ushorts as 2 uints each. 16-bit integer ops are + // disproportionately slow on the A7X (E031.41) compiler; keeping the dequant + // operands in 32-bit registers avoids the ushort path. q1/q2 are 4-byte aligned + // (ib*128 + (32*iq+8*ir) bytes; q1 += blk*128 bytes/row). Math is unchanged: + // w & 0x0F00 on the low/high halves equals the original ushort mask value. + global uint * q1u = (global uint *)q1; + global uint * q2u = (global uint *)q2; + uint a0 = q1u[0], a1 = q1u[1]; + uint b0 = q2u[0], b1 = q2u[1]; + uint w0 = a0 & 0xFFFF, w1 = a0 >> 16, w2 = a1 & 0xFFFF, w3 = a1 >> 16; + uint v0 = b0 & 0xFFFF, v1 = b0 >> 16, v2 = b1 & 0xFFFF, v3 = b1 >> 16; + + float4 acc1, acc2; + acc1.s0 = yl[0]*(w0&0x000F) + yl[ 2]*(w1&0x000F) + yl[ 4]*(w2&0x000F) + yl[ 6]*(w3&0x000F); + acc1.s1 = yl[1]*(w0&0x0F00) + yl[ 3]*(w1&0x0F00) + yl[ 5]*(w2&0x0F00) + yl[ 7]*(w3&0x0F00); + acc1.s2 = yl[8]*(w0&0x00F0) + yl[10]*(w1&0x00F0) + yl[12]*(w2&0x00F0) + yl[14]*(w3&0x00F0); + acc1.s3 = yl[9]*(w0&0xF000) + yl[11]*(w1&0xF000) + yl[13]*(w2&0xF000) + yl[15]*(w3&0xF000); + acc2.s0 = yh[0]*(v0&0x000F) + yh[ 2]*(v1&0x000F) + yh[ 4]*(v2&0x000F) + yh[ 6]*(v3&0x000F); + acc2.s1 = yh[1]*(v0&0x0F00) + yh[ 3]*(v1&0x0F00) + yh[ 5]*(v2&0x0F00) + yh[ 7]*(v3&0x0F00); + acc2.s2 = yh[8]*(v0&0x00F0) + yh[10]*(v1&0x00F0) + yh[12]*(v2&0x00F0) + yh[14]*(v3&0x00F0); + acc2.s3 = yh[9]*(v0&0xF000) + yh[11]*(v1&0xF000) + yh[13]*(v2&0xF000) + yh[15]*(v3&0xF000); float dall = *d; float dmin = *dm; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl new file mode 100644 index 0000000..6d8c9e8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl @@ -0,0 +1,241 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK5_0 32 + +struct block_q5_0 { + half d; + uchar qh[4]; + uchar qs[QK5_0 / 2]; +}; + +inline float block_q5_0_dot_y( + global const struct block_q5_0 * qb_curr, + float sumy, + float16 yl, + int il, + global const float * yb +) { + float d = qb_curr->d; + + float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f); + + global const ushort * qs = ((global const ushort *)((global const uchar *) qb_curr + 6 + il)); + + acc.s0 += yl.s0 * (qs[0] & 0x000F); + acc.s0 += yl.s1 * (qs[0] & 0x0F00); + acc.s0 += yl.s8 * (qs[0] & 0x00F0); + acc.s3 += yl.s9 * (qs[0] & 0xF000); + + acc.s0 += yl.s2 * (qs[1] & 0x000F); + acc.s1 += yl.s3 * (qs[1] & 0x0F00); + acc.s2 += yl.sa * (qs[1] & 0x00F0); + acc.s3 += yl.sb * (qs[1] & 0xF000); + + acc.s0 += yl.s4 * (qs[2] & 0x000F); + acc.s1 += yl.s5 * (qs[2] & 0x0F00); + acc.s2 += yl.sc * (qs[2] & 0x00F0); + acc.s3 += yl.sd * (qs[2] & 0xF000); + + acc.s0 += yl.s6 * (qs[3] & 0x000F); + acc.s1 += yl.s7 * (qs[3] & 0x0F00); + acc.s2 += yl.se * (qs[3] & 0x00F0); + acc.s3 += yl.sf * (qs[3] & 0xF000); + + uint qh_val = *((global const uint *)((global const uchar *) qb_curr + 2)); + uchar qh_lo = (uchar)((qh_val >> il) & 0xFF); + uchar qh_hi = (uchar)((qh_val >> (il + 16)) & 0xFF); + + float qh_sum = 0.0f; + qh_sum += yb[0] * (float)((qh_lo >> 0) & 1); + qh_sum += yb[1] * (float)((qh_lo >> 1) & 1); + qh_sum += yb[2] * (float)((qh_lo >> 2) & 1); + qh_sum += yb[3] * (float)((qh_lo >> 3) & 1); + qh_sum += yb[4] * (float)((qh_lo >> 4) & 1); + qh_sum += yb[5] * (float)((qh_lo >> 5) & 1); + qh_sum += yb[6] * (float)((qh_lo >> 6) & 1); + qh_sum += yb[7] * (float)((qh_lo >> 7) & 1); + qh_sum += yb[16] * (float)((qh_hi >> 0) & 1); + qh_sum += yb[17] * (float)((qh_hi >> 1) & 1); + qh_sum += yb[18] * (float)((qh_hi >> 2) & 1); + qh_sum += yb[19] * (float)((qh_hi >> 3) & 1); + qh_sum += yb[20] * (float)((qh_hi >> 4) & 1); + qh_sum += yb[21] * (float)((qh_hi >> 5) & 1); + qh_sum += yb[22] * (float)((qh_hi >> 6) & 1); + qh_sum += yb[23] * (float)((qh_hi >> 7) & 1); + + return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3 + 16.0f * qh_sum - 16.0f * sumy); +} + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // each subgroup works on 4 rows +#define N_SIMDGROUP 1 // number of subgroups in a thread group +#define N_SIMDWIDTH 16 // assuming subgroup size is 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +inline void mul_vec_q_n_f32( + global void * src0, + global float * src1, + global float * dst, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + const ulong nb = ne00/QK5_0; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + global struct block_q5_0 * x = (global struct block_q5_0 *) src0 + offset0; + global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1; + + float16 yl; + float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f); + + int ix = get_sub_group_local_id()/2; + int il = 8*(get_sub_group_local_id()%2); + + global float * yb = y + ix * QK5_0 + il; + + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) { + float sumy = 0; + + sumy += yb[0]; + sumy += yb[1]; + sumy += yb[2]; + sumy += yb[3]; + sumy += yb[4]; + sumy += yb[5]; + sumy += yb[6]; + sumy += yb[7]; + + sumy += yb[16]; + sumy += yb[17]; + sumy += yb[18]; + sumy += yb[19]; + sumy += yb[20]; + sumy += yb[21]; + sumy += yb[22]; + sumy += yb[23]; + + + yl.s0 = yb[0]; + yl.s1 = yb[1]/256.f; + + yl.s2 = yb[2]; + yl.s3 = yb[3]/256.f; + + yl.s4 = yb[4]; + yl.s5 = yb[5]/256.f; + + yl.s6 = yb[6]; + yl.s7 = yb[7]/256.f; + + yl.s8 = yb[16]/16.f; + yl.s9 = yb[17]/4096.f; + + yl.sa = yb[18]/16.f; + yl.sb = yb[19]/4096.f; + + yl.sc = yb[20]/16.f; + yl.sd = yb[21]/4096.f; + + yl.se = yb[22]/16.f; + yl.sf = yb[23]/4096.f; + + sumf.s0 += block_q5_0_dot_y(x+ib+0*nb, sumy, yl, il, yb); + sumf.s1 += block_q5_0_dot_y(x+ib+1*nb, sumy, yl, il, yb); + sumf.s2 += block_q5_0_dot_y(x+ib+2*nb, sumy, yl, il, yb); + sumf.s3 += block_q5_0_dot_y(x+ib+3*nb, sumy, yl, il, yb); + + yb += QK5_0 * (N_SIMDWIDTH/2); + } + + float4 tot = (float4)( + sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1), + sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3) + ); + + if (get_sub_group_local_id() == 0) { + if (first_row + 0 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0; + } + if (first_row + 1 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1; + } + if (first_row + 2 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2; + } + if (first_row + 3 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3; + } + } +} + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q5_0_f32( + global void * src0, + ulong offset0, + global float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global void*)((global char*)src0 + offset0); + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3); +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl new file mode 100644 index 0000000..34ec133 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl @@ -0,0 +1,243 @@ + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK5_0 32 + +inline float block_q5_0_dot_y_flat( + global const uchar * x, + global const uint * qh_ptr, + global const half * dh, + float sumy, + float16 yl, + int il, + global const float * yb +) { + float d = *dh; + global const ushort * qs = ((global const ushort *)(x + il)); + + float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f); + + acc.s0 += yl.s0 * (qs[0] & 0x000F); + acc.s0 += yl.s1 * (qs[0] & 0x0F00); + acc.s0 += yl.s8 * (qs[0] & 0x00F0); + acc.s3 += yl.s9 * (qs[0] & 0xF000); + + acc.s0 += yl.s2 * (qs[1] & 0x000F); + acc.s1 += yl.s3 * (qs[1] & 0x0F00); + acc.s2 += yl.sa * (qs[1] & 0x00F0); + acc.s3 += yl.sb * (qs[1] & 0xF000); + + acc.s0 += yl.s4 * (qs[2] & 0x000F); + acc.s1 += yl.s5 * (qs[2] & 0x0F00); + acc.s2 += yl.sc * (qs[2] & 0x00F0); + acc.s3 += yl.sd * (qs[2] & 0xF000); + + acc.s0 += yl.s6 * (qs[3] & 0x000F); + acc.s1 += yl.s7 * (qs[3] & 0x0F00); + acc.s2 += yl.se * (qs[3] & 0x00F0); + acc.s3 += yl.sf * (qs[3] & 0xF000); + + uint qh_val = *qh_ptr; + uchar qh_lo = (uchar)((qh_val >> il) & 0xFF); + uchar qh_hi = (uchar)((qh_val >> (il + 16)) & 0xFF); + + float qh_sum = 0.0f; + qh_sum += yb[0] * (float)((qh_lo >> 0) & 1); + qh_sum += yb[1] * (float)((qh_lo >> 1) & 1); + qh_sum += yb[2] * (float)((qh_lo >> 2) & 1); + qh_sum += yb[3] * (float)((qh_lo >> 3) & 1); + qh_sum += yb[4] * (float)((qh_lo >> 4) & 1); + qh_sum += yb[5] * (float)((qh_lo >> 5) & 1); + qh_sum += yb[6] * (float)((qh_lo >> 6) & 1); + qh_sum += yb[7] * (float)((qh_lo >> 7) & 1); + qh_sum += yb[16] * (float)((qh_hi >> 0) & 1); + qh_sum += yb[17] * (float)((qh_hi >> 1) & 1); + qh_sum += yb[18] * (float)((qh_hi >> 2) & 1); + qh_sum += yb[19] * (float)((qh_hi >> 3) & 1); + qh_sum += yb[20] * (float)((qh_hi >> 4) & 1); + qh_sum += yb[21] * (float)((qh_hi >> 5) & 1); + qh_sum += yb[22] * (float)((qh_hi >> 6) & 1); + qh_sum += yb[23] * (float)((qh_hi >> 7) & 1); + + return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3 + 16.0f * qh_sum - 16.0f * sumy); +} + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // each subgroup works on 4 rows +#define N_SIMDGROUP 1 // number of subgroups in a thread group +#define N_SIMDWIDTH 16 // assuming subgroup size is 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +inline void mul_vec_q_n_f32_flat( + global void * src0_qs, + global void * src0_qh, + global void * src0_d, + global float * src1, + global float * dst, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + const ulong nb = ne00/QK5_0; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + ulong offset0_qs = offset0 * (QK5_0/2); + + global uchar * x = (global uchar *) src0_qs + offset0_qs; + global uint * qh = (global uint *) src0_qh + offset0; + global half * d = (global half *) src0_d + offset0; + global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1; + + float16 yl; + float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f); + + int ix = get_sub_group_local_id()/2; + int il = 8*(get_sub_group_local_id()%2); + + global float * yb = y + ix * QK5_0 + il; + + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) { + float sumy = 0; + + sumy += yb[0]; + sumy += yb[1]; + sumy += yb[2]; + sumy += yb[3]; + sumy += yb[4]; + sumy += yb[5]; + sumy += yb[6]; + sumy += yb[7]; + + sumy += yb[16]; + sumy += yb[17]; + sumy += yb[18]; + sumy += yb[19]; + sumy += yb[20]; + sumy += yb[21]; + sumy += yb[22]; + sumy += yb[23]; + + + yl.s0 = yb[0]; + yl.s1 = yb[1]/256.f; + + yl.s2 = yb[2]; + yl.s3 = yb[3]/256.f; + + yl.s4 = yb[4]; + yl.s5 = yb[5]/256.f; + + yl.s6 = yb[6]; + yl.s7 = yb[7]/256.f; + + yl.s8 = yb[16]/16.f; + yl.s9 = yb[17]/4096.f; + + yl.sa = yb[18]/16.f; + yl.sb = yb[19]/4096.f; + + yl.sc = yb[20]/16.f; + yl.sd = yb[21]/4096.f; + + yl.se = yb[22]/16.f; + yl.sf = yb[23]/4096.f; + + sumf.s0 += block_q5_0_dot_y_flat(x + ib*(QK5_0/2) + 0*nb*(QK5_0/2), qh + ib + 0*nb, d + ib + 0*nb, sumy, yl, il, yb); + sumf.s1 += block_q5_0_dot_y_flat(x + ib*(QK5_0/2) + 1*nb*(QK5_0/2), qh + ib + 1*nb, d + ib + 1*nb, sumy, yl, il, yb); + sumf.s2 += block_q5_0_dot_y_flat(x + ib*(QK5_0/2) + 2*nb*(QK5_0/2), qh + ib + 2*nb, d + ib + 2*nb, sumy, yl, il, yb); + sumf.s3 += block_q5_0_dot_y_flat(x + ib*(QK5_0/2) + 3*nb*(QK5_0/2), qh + ib + 3*nb, d + ib + 3*nb, sumy, yl, il, yb); + + yb += QK5_0 * (N_SIMDWIDTH/2); + } + + float4 tot = (float4)( + sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1), + sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3) + ); + + if (get_sub_group_local_id() == 0) { + if (first_row + 0 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0; + } + if (first_row + 1 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1; + } + if (first_row + 2 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2; + } + if (first_row + 3 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3; + } + } +} + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q5_0_f32_flat( + global void * src0_qs, + global void * src0_qh, + global void * src0_d, + global float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + mul_vec_q_n_f32_flat(src0_qs, src0_qh, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3); +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl new file mode 100644 index 0000000..1480f67 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl @@ -0,0 +1,243 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK5_1 32 + +struct block_q5_1 { + half d; + half m; + uchar qh[4]; + uchar qs[QK5_1 / 2]; +}; + +inline float block_q5_1_dot_y( + global const struct block_q5_1 * qb_curr, + float sumy, + float16 yl, + int il, + global const float * yb +) { + float d = qb_curr->d; + float m = qb_curr->m; + + float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f); + + global const ushort * qs = ((global const ushort *)((global const uchar *) qb_curr + 8 + il)); + + acc.s0 += yl.s0 * (qs[0] & 0x000F); + acc.s0 += yl.s1 * (qs[0] & 0x0F00); + acc.s0 += yl.s8 * (qs[0] & 0x00F0); + acc.s3 += yl.s9 * (qs[0] & 0xF000); + + acc.s0 += yl.s2 * (qs[1] & 0x000F); + acc.s1 += yl.s3 * (qs[1] & 0x0F00); + acc.s2 += yl.sa * (qs[1] & 0x00F0); + acc.s3 += yl.sb * (qs[1] & 0xF000); + + acc.s0 += yl.s4 * (qs[2] & 0x000F); + acc.s1 += yl.s5 * (qs[2] & 0x0F00); + acc.s2 += yl.sc * (qs[2] & 0x00F0); + acc.s3 += yl.sd * (qs[2] & 0xF000); + + acc.s0 += yl.s6 * (qs[3] & 0x000F); + acc.s1 += yl.s7 * (qs[3] & 0x0F00); + acc.s2 += yl.se * (qs[3] & 0x00F0); + acc.s3 += yl.sf * (qs[3] & 0xF000); + + uint qh_val = *((global const uint *)((global const uchar *) qb_curr + 4)); + uchar qh_lo = (uchar)((qh_val >> il) & 0xFF); + uchar qh_hi = (uchar)((qh_val >> (il + 16)) & 0xFF); + + float qh_sum = 0.0f; + qh_sum += yb[0] * (float)((qh_lo >> 0) & 1); + qh_sum += yb[1] * (float)((qh_lo >> 1) & 1); + qh_sum += yb[2] * (float)((qh_lo >> 2) & 1); + qh_sum += yb[3] * (float)((qh_lo >> 3) & 1); + qh_sum += yb[4] * (float)((qh_lo >> 4) & 1); + qh_sum += yb[5] * (float)((qh_lo >> 5) & 1); + qh_sum += yb[6] * (float)((qh_lo >> 6) & 1); + qh_sum += yb[7] * (float)((qh_lo >> 7) & 1); + qh_sum += yb[16] * (float)((qh_hi >> 0) & 1); + qh_sum += yb[17] * (float)((qh_hi >> 1) & 1); + qh_sum += yb[18] * (float)((qh_hi >> 2) & 1); + qh_sum += yb[19] * (float)((qh_hi >> 3) & 1); + qh_sum += yb[20] * (float)((qh_hi >> 4) & 1); + qh_sum += yb[21] * (float)((qh_hi >> 5) & 1); + qh_sum += yb[22] * (float)((qh_hi >> 6) & 1); + qh_sum += yb[23] * (float)((qh_hi >> 7) & 1); + + return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3 + 16.0f * qh_sum) + sumy * m; +} + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // each subgroup works on 4 rows +#define N_SIMDGROUP 1 // number of subgroups in a thread group +#define N_SIMDWIDTH 16 // assuming subgroup size is 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +inline void mul_vec_q_n_f32( + global void * src0, + global float * src1, + global float * dst, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + const ulong nb = ne00/QK5_1; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + global struct block_q5_1 * x = (global struct block_q5_1 *) src0 + offset0; + global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1; + + float16 yl; + float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f); + + int ix = get_sub_group_local_id()/2; + int il = 8*(get_sub_group_local_id()%2); + + global float * yb = y + ix * QK5_1 + il; + + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) { + float sumy = 0; + + sumy += yb[0]; + sumy += yb[1]; + sumy += yb[2]; + sumy += yb[3]; + sumy += yb[4]; + sumy += yb[5]; + sumy += yb[6]; + sumy += yb[7]; + + sumy += yb[16]; + sumy += yb[17]; + sumy += yb[18]; + sumy += yb[19]; + sumy += yb[20]; + sumy += yb[21]; + sumy += yb[22]; + sumy += yb[23]; + + + yl.s0 = yb[0]; + yl.s1 = yb[1]/256.f; + + yl.s2 = yb[2]; + yl.s3 = yb[3]/256.f; + + yl.s4 = yb[4]; + yl.s5 = yb[5]/256.f; + + yl.s6 = yb[6]; + yl.s7 = yb[7]/256.f; + + yl.s8 = yb[16]/16.f; + yl.s9 = yb[17]/4096.f; + + yl.sa = yb[18]/16.f; + yl.sb = yb[19]/4096.f; + + yl.sc = yb[20]/16.f; + yl.sd = yb[21]/4096.f; + + yl.se = yb[22]/16.f; + yl.sf = yb[23]/4096.f; + + sumf.s0 += block_q5_1_dot_y(x+ib+0*nb, sumy, yl, il, yb); + sumf.s1 += block_q5_1_dot_y(x+ib+1*nb, sumy, yl, il, yb); + sumf.s2 += block_q5_1_dot_y(x+ib+2*nb, sumy, yl, il, yb); + sumf.s3 += block_q5_1_dot_y(x+ib+3*nb, sumy, yl, il, yb); + + yb += QK5_1 * (N_SIMDWIDTH/2); + } + + float4 tot = (float4)( + sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1), + sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3) + ); + + if (get_sub_group_local_id() == 0) { + if (first_row + 0 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0; + } + if (first_row + 1 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1; + } + if (first_row + 2 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2; + } + if (first_row + 3 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3; + } + } +} + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q5_1_f32( + global void * src0, + ulong offset0, + global float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global void*)((global char*)src0 + offset0); + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3); +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl new file mode 100644 index 0000000..57c2f14 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl @@ -0,0 +1,247 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_subgroups +#pragma OPENCL EXTENSION cl_intel_subgroups : enable +#else +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK5_1 32 + +inline float block_q5_1_dot_y_flat( + global const uchar * x, + global const uint * qh_ptr, + global const half * dh, + global const half * mh, + float sumy, + float16 yl, + int il, + global const float * yb +) { + float d = *dh; + float m = *mh; + global const ushort * qs = ((global const ushort *)(x + il)); + + float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f); + + acc.s0 += yl.s0 * (qs[0] & 0x000F); + acc.s0 += yl.s1 * (qs[0] & 0x0F00); + acc.s0 += yl.s8 * (qs[0] & 0x00F0); + acc.s3 += yl.s9 * (qs[0] & 0xF000); + + acc.s0 += yl.s2 * (qs[1] & 0x000F); + acc.s1 += yl.s3 * (qs[1] & 0x0F00); + acc.s2 += yl.sa * (qs[1] & 0x00F0); + acc.s3 += yl.sb * (qs[1] & 0xF000); + + acc.s0 += yl.s4 * (qs[2] & 0x000F); + acc.s1 += yl.s5 * (qs[2] & 0x0F00); + acc.s2 += yl.sc * (qs[2] & 0x00F0); + acc.s3 += yl.sd * (qs[2] & 0xF000); + + acc.s0 += yl.s6 * (qs[3] & 0x000F); + acc.s1 += yl.s7 * (qs[3] & 0x0F00); + acc.s2 += yl.se * (qs[3] & 0x00F0); + acc.s3 += yl.sf * (qs[3] & 0xF000); + + uint qh_val = *qh_ptr; + uchar qh_lo = (uchar)((qh_val >> il) & 0xFF); + uchar qh_hi = (uchar)((qh_val >> (il + 16)) & 0xFF); + + float qh_sum = 0.0f; + qh_sum += yb[0] * (float)((qh_lo >> 0) & 1); + qh_sum += yb[1] * (float)((qh_lo >> 1) & 1); + qh_sum += yb[2] * (float)((qh_lo >> 2) & 1); + qh_sum += yb[3] * (float)((qh_lo >> 3) & 1); + qh_sum += yb[4] * (float)((qh_lo >> 4) & 1); + qh_sum += yb[5] * (float)((qh_lo >> 5) & 1); + qh_sum += yb[6] * (float)((qh_lo >> 6) & 1); + qh_sum += yb[7] * (float)((qh_lo >> 7) & 1); + qh_sum += yb[16] * (float)((qh_hi >> 0) & 1); + qh_sum += yb[17] * (float)((qh_hi >> 1) & 1); + qh_sum += yb[18] * (float)((qh_hi >> 2) & 1); + qh_sum += yb[19] * (float)((qh_hi >> 3) & 1); + qh_sum += yb[20] * (float)((qh_hi >> 4) & 1); + qh_sum += yb[21] * (float)((qh_hi >> 5) & 1); + qh_sum += yb[22] * (float)((qh_hi >> 6) & 1); + qh_sum += yb[23] * (float)((qh_hi >> 7) & 1); + + return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3 + 16.0f * qh_sum) + sumy * m; +} + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 // each subgroup works on 4 rows +#define N_SIMDGROUP 1 // number of subgroups in a thread group +#define N_SIMDWIDTH 16 // assuming subgroup size is 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +inline void mul_vec_q_n_f32_flat( + global void * src0_qs, + global void * src0_qh, + global void * src0_d, + global void * src0_m, + global float * src1, + global float * dst, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + const ulong nb = ne00/QK5_1; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + ulong offset0_qs = offset0 * (QK5_1/2); + + global uchar * x = (global uchar *) src0_qs + offset0_qs; + global uint * qh = (global uint *) src0_qh + offset0; + global half * d = (global half *) src0_d + offset0; + global half * ms = (global half *) src0_m + offset0; + global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1; + + float16 yl; + float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f); + + int ix = get_sub_group_local_id()/2; + int il = 8*(get_sub_group_local_id()%2); + + global float * yb = y + ix * QK5_1 + il; + + for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) { + float sumy = 0; + + sumy += yb[0]; + sumy += yb[1]; + sumy += yb[2]; + sumy += yb[3]; + sumy += yb[4]; + sumy += yb[5]; + sumy += yb[6]; + sumy += yb[7]; + + sumy += yb[16]; + sumy += yb[17]; + sumy += yb[18]; + sumy += yb[19]; + sumy += yb[20]; + sumy += yb[21]; + sumy += yb[22]; + sumy += yb[23]; + + + yl.s0 = yb[0]; + yl.s1 = yb[1]/256.f; + + yl.s2 = yb[2]; + yl.s3 = yb[3]/256.f; + + yl.s4 = yb[4]; + yl.s5 = yb[5]/256.f; + + yl.s6 = yb[6]; + yl.s7 = yb[7]/256.f; + + yl.s8 = yb[16]/16.f; + yl.s9 = yb[17]/4096.f; + + yl.sa = yb[18]/16.f; + yl.sb = yb[19]/4096.f; + + yl.sc = yb[20]/16.f; + yl.sd = yb[21]/4096.f; + + yl.se = yb[22]/16.f; + yl.sf = yb[23]/4096.f; + + sumf.s0 += block_q5_1_dot_y_flat(x + ib*(QK5_1/2) + 0*nb*(QK5_1/2), qh + ib + 0*nb, d + ib + 0*nb, ms + ib + 0*nb, sumy, yl, il, yb); + sumf.s1 += block_q5_1_dot_y_flat(x + ib*(QK5_1/2) + 1*nb*(QK5_1/2), qh + ib + 1*nb, d + ib + 1*nb, ms + ib + 1*nb, sumy, yl, il, yb); + sumf.s2 += block_q5_1_dot_y_flat(x + ib*(QK5_1/2) + 2*nb*(QK5_1/2), qh + ib + 2*nb, d + ib + 2*nb, ms + ib + 2*nb, sumy, yl, il, yb); + sumf.s3 += block_q5_1_dot_y_flat(x + ib*(QK5_1/2) + 3*nb*(QK5_1/2), qh + ib + 3*nb, d + ib + 3*nb, ms + ib + 3*nb, sumy, yl, il, yb); + + yb += QK5_1 * (N_SIMDWIDTH/2); + } + + float4 tot = (float4)( + sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1), + sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3) + ); + + if (get_sub_group_local_id() == 0) { + if (first_row + 0 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0; + } + if (first_row + 1 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1; + } + if (first_row + 2 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2; + } + if (first_row + 3 < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3; + } + } +} + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q5_1_f32_flat( + global void * src0_qs, + global void * src0_qh, + global void * src0_d, + global void * src0_m, + global float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne10, + int ne12, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = (global float*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + mul_vec_q_n_f32_flat(src0_qs, src0_qh, src0_d, src0_m, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3); +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl index e353a72..ab2e1fa 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl @@ -38,7 +38,7 @@ typedef struct { #undef N_SIMDWIDTH #ifdef INTEL_GPU -#define N_DST 4 +#define N_DST 8 // Intel: 4->8 for 2x activation reuse (see mul_mv_q4_k_f32_flat.cl) #define N_SIMDGROUP 1 #define N_SIMDWIDTH 16 #elif defined(ADRENO_GPU) @@ -159,18 +159,59 @@ kernel void kernel_mul_mv_q5_K_f32_flat( global ushort * q2 = q1 + 32; - float4 acc1 = {0.f, 0.f, 0.f, 0.f}; - float4 acc2 = {0.f, 0.f, 0.f, 0.f}; - for (int i = 0; i < 8; i += 2) { - acc1.s0 += yl[i+0] * ((q1[i/2] & 0x000F) + (qh[i+0] & u1_lo ? 16.f : 0.f)); - acc1.s1 += yl[i+1] * ((q1[i/2] & 0x0F00) + (qh[i+1] & u1_lo ? 16.f*256.f : 0.f)); - acc1.s2 += yl[i+8] * ((q1[i/2] & 0x00F0) + (qh[i+0] & u2_lo ? 16.f*16.f : 0.f)); - acc1.s3 += yl[i+9] * ((q1[i/2] & 0xF000) + (qh[i+1] & u2_lo ? 16.f*4096.f: 0.f)); - acc2.s0 += yh[i+0] * ((q2[i/2] & 0x000F) + (qh[i+0] & u1_hi ? 16.f : 0.f)); - acc2.s1 += yh[i+1] * ((q2[i/2] & 0x0F00) + (qh[i+1] & u1_hi ? 16.f*256.f : 0.f)); - acc2.s2 += yh[i+8] * ((q2[i/2] & 0x00F0) + (qh[i+0] & u2_hi ? 16.f*16.f : 0.f)); - acc2.s3 += yh[i+9] * ((q2[i/2] & 0xF000) + (qh[i+1] & u2_hi ? 16.f*4096.f: 0.f)); - } + // Load the 4 q1 / 4 q2 quant ushorts as 2 uints each. 16-bit integer ops are + // disproportionately slow on the A7X (E031.41) compiler; keeping the dequant + // operands in 32-bit registers avoids the ushort path (same fix as q4_K flat). + // q1/q2 are 4-byte aligned; w & 0x0F00 on the low/high half of a uint equals the + // original ushort mask value, so the math is unchanged. The qh high-bit term is + // byte-indexed (qh[0..7]) and left as-is. + global uint * q1u = (global uint *)q1; + global uint * q2u = (global uint *)q2; + uint a0 = q1u[0], a1 = q1u[1], b0 = q2u[0], b1 = q2u[1]; + uint w0 = a0 & 0xFFFF, w1 = a0 >> 16, w2 = a1 & 0xFFFF, w3 = a1 >> 16; + uint v0 = b0 & 0xFFFF, v1 = b0 >> 16, v2 = b1 & 0xFFFF, v3 = b1 >> 16; + + float4 acc1, acc2; + acc1.s0 = + yl[0]*((w0&0x000F)+(qh[0]&u1_lo?16.f:0.f)) + + yl[2]*((w1&0x000F)+(qh[2]&u1_lo?16.f:0.f)) + + yl[4]*((w2&0x000F)+(qh[4]&u1_lo?16.f:0.f)) + + yl[6]*((w3&0x000F)+(qh[6]&u1_lo?16.f:0.f)); + acc1.s1 = + yl[1]*((w0&0x0F00)+(qh[1]&u1_lo?16.f*256.f:0.f)) + + yl[3]*((w1&0x0F00)+(qh[3]&u1_lo?16.f*256.f:0.f)) + + yl[5]*((w2&0x0F00)+(qh[5]&u1_lo?16.f*256.f:0.f)) + + yl[7]*((w3&0x0F00)+(qh[7]&u1_lo?16.f*256.f:0.f)); + acc1.s2 = + yl[ 8]*((w0&0x00F0)+(qh[0]&u2_lo?16.f*16.f:0.f)) + + yl[10]*((w1&0x00F0)+(qh[2]&u2_lo?16.f*16.f:0.f)) + + yl[12]*((w2&0x00F0)+(qh[4]&u2_lo?16.f*16.f:0.f)) + + yl[14]*((w3&0x00F0)+(qh[6]&u2_lo?16.f*16.f:0.f)); + acc1.s3 = + yl[ 9]*((w0&0xF000)+(qh[1]&u2_lo?16.f*4096.f:0.f)) + + yl[11]*((w1&0xF000)+(qh[3]&u2_lo?16.f*4096.f:0.f)) + + yl[13]*((w2&0xF000)+(qh[5]&u2_lo?16.f*4096.f:0.f)) + + yl[15]*((w3&0xF000)+(qh[7]&u2_lo?16.f*4096.f:0.f)); + acc2.s0 = + yh[0]*((v0&0x000F)+(qh[0]&u1_hi?16.f:0.f)) + + yh[2]*((v1&0x000F)+(qh[2]&u1_hi?16.f:0.f)) + + yh[4]*((v2&0x000F)+(qh[4]&u1_hi?16.f:0.f)) + + yh[6]*((v3&0x000F)+(qh[6]&u1_hi?16.f:0.f)); + acc2.s1 = + yh[1]*((v0&0x0F00)+(qh[1]&u1_hi?16.f*256.f:0.f)) + + yh[3]*((v1&0x0F00)+(qh[3]&u1_hi?16.f*256.f:0.f)) + + yh[5]*((v2&0x0F00)+(qh[5]&u1_hi?16.f*256.f:0.f)) + + yh[7]*((v3&0x0F00)+(qh[7]&u1_hi?16.f*256.f:0.f)); + acc2.s2 = + yh[ 8]*((v0&0x00F0)+(qh[0]&u2_hi?16.f*16.f:0.f)) + + yh[10]*((v1&0x00F0)+(qh[2]&u2_hi?16.f*16.f:0.f)) + + yh[12]*((v2&0x00F0)+(qh[4]&u2_hi?16.f*16.f:0.f)) + + yh[14]*((v3&0x00F0)+(qh[6]&u2_hi?16.f*16.f:0.f)); + acc2.s3 = + yh[ 9]*((v0&0xF000)+(qh[1]&u2_hi?16.f*4096.f:0.f)) + + yh[11]*((v1&0xF000)+(qh[3]&u2_hi?16.f*4096.f:0.f)) + + yh[13]*((v2&0xF000)+(qh[5]&u2_hi?16.f*4096.f:0.f)) + + yh[15]*((v3&0xF000)+(qh[7]&u2_hi?16.f*4096.f:0.f)); float dall = *d; float dmin = *dm; diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl index 86fe09c..2cca533 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl @@ -28,56 +28,91 @@ #define QK_K 256 +// ADRENO_OLD_COMPILER is defined by the host (-D) only for the Adreno E031 +// compilers older than E031.45, which miscompile several constructs this kernel +// used (confirmed on E031.38 and E031.41; E031.45 is clean). Every other +// compiler -- newer E031, E17, DX, Intel, and every non-Adreno device that +// builds this program -- takes the #else branches, which are the original +// source: the workarounds below cost ~13% on the q6_K flat n=1 GEMV where they +// are not needed. inline float block_q_6_K_dot_y_flat( global uchar * blk_ql, global uchar * blk_qh, global char * blk_scales, global half * blk_d, - global float * yy, int ib, int ip, int is, - int l0 + int l0, +#if defined(ADRENO_OLD_COMPILER) + int dbg, +#endif + float4 y0, + float4 y1, + float4 y2, + float4 y3 ) { - int y_offset = 128*ip + l0; int q_offset_l = 64*ip + l0; int q_offset_h = 32*ip + l0; global uchar * q1 = blk_ql + ib*128 + q_offset_l; global uchar * q2 = q1 + QK_K/8; global uchar * qh = blk_qh + ib*64 + q_offset_h; - global char * sc = blk_scales + ib*16 + is; - - global float * y = yy + ib * QK_K + y_offset; float dall = blk_d[ib]; - float sumf = 0; - float4 sums = {0.f, 0.f, 0.f, 0.f}; +#if defined(ADRENO_OLD_COMPILER) + // The vectorized dequant (int4/float4 bit-ops, convert_*4, dot()) and vload4 + // are miscompiled here -> garbage weights. Reconstruct the 6-bit weights and + // take the dot product scalar. q4_K/q5_K flat already use scalar paths, which + // is why q6_K was the only flat GEMV that failed. + // Scales are SIGNED int8; read as uchar and sign-extend arithmetically so the + // result does not depend on whether the compiler treats `char` as signed. + global uchar * sc = (global uchar *)(blk_scales + ib*16 + is); - sums.s0 += y[0+ 0] * ((float)((q1[0] & 0xF) | ((qh[0] & Q6_K_MASK1) << 4)) - 32.f); - sums.s1 += y[0+32] * ((float)((q2[0] & 0xF) | ((qh[0] & Q6_K_MASK2) << 2)) - 32.f); - sums.s2 += y[0+64] * ((float)((q1[0] >> 4) | ((qh[0] & Q6_K_MASK3) << 0)) - 32.f); - sums.s3 += y[0+96] * ((float)((q2[0] >> 4) | ((qh[0] & Q6_K_MASK4) >> 2)) - 32.f); + int s0 = (int)sc[0] - 256*(sc[0] >> 7); + int s2 = (int)sc[2] - 256*(sc[2] >> 7); + int s4 = (int)sc[4] - 256*(sc[4] >> 7); + int s6 = (int)sc[6] - 256*(sc[6] >> 7); - sums.s0 += y[1+ 0] * ((float)((q1[1] & 0xF) | ((qh[1] & Q6_K_MASK1) << 4)) - 32.f); - sums.s1 += y[1+32] * ((float)((q2[1] & 0xF) | ((qh[1] & Q6_K_MASK2) << 2)) - 32.f); - sums.s2 += y[1+64] * ((float)((q1[1] >> 4) | ((qh[1] & Q6_K_MASK3) << 0)) - 32.f); - sums.s3 += y[1+96] * ((float)((q2[1] >> 4) | ((qh[1] & Q6_K_MASK4) >> 2)) - 32.f); + // one 6-bit weight: low/high nibble of a ql byte OR'd with a 2-bit qh plane + // (plane p in {0,1,2,3} selects qh bits 2p..2p+1) placed at bits 4-5, minus 32. + #define Q6W(qb, sh, hb, p) ((float)((((int)(qb) >> (sh)) & 15) | ((((int)(hb) >> (2*(p))) & 3) << 4)) - 32.f) - sums.s0 += y[2+ 0] * ((float)((q1[2] & 0xF) | ((qh[2] & Q6_K_MASK1) << 4)) - 32.f); - sums.s1 += y[2+32] * ((float)((q2[2] & 0xF) | ((qh[2] & Q6_K_MASK2) << 2)) - 32.f); - sums.s2 += y[2+64] * ((float)((q1[2] >> 4) | ((qh[2] & Q6_K_MASK3) << 0)) - 32.f); - sums.s3 += y[2+96] * ((float)((q2[2] >> 4) | ((qh[2] & Q6_K_MASK4) >> 2)) - 32.f); + float d0 = y0.s0*Q6W(q1[0],0,qh[0],0) + y0.s1*Q6W(q1[1],0,qh[1],0) + y0.s2*Q6W(q1[2],0,qh[2],0) + y0.s3*Q6W(q1[3],0,qh[3],0); + float d1 = y1.s0*Q6W(q2[0],0,qh[0],1) + y1.s1*Q6W(q2[1],0,qh[1],1) + y1.s2*Q6W(q2[2],0,qh[2],1) + y1.s3*Q6W(q2[3],0,qh[3],1); + float d2 = y2.s0*Q6W(q1[0],4,qh[0],2) + y2.s1*Q6W(q1[1],4,qh[1],2) + y2.s2*Q6W(q1[2],4,qh[2],2) + y2.s3*Q6W(q1[3],4,qh[3],2); + float d3 = y3.s0*Q6W(q2[0],4,qh[0],3) + y3.s1*Q6W(q2[1],4,qh[1],3) + y3.s2*Q6W(q2[2],4,qh[2],3) + y3.s3*Q6W(q2[3],4,qh[3],3); + #undef Q6W - sums.s0 += y[3+ 0] * ((float)((q1[3] & 0xF) | ((qh[3] & Q6_K_MASK1) << 4)) - 32.f); - sums.s1 += y[3+32] * ((float)((q2[3] & 0xF) | ((qh[3] & Q6_K_MASK2) << 2)) - 32.f); - sums.s2 += y[3+64] * ((float)((q1[3] >> 4) | ((qh[3] & Q6_K_MASK3) << 0)) - 32.f); - sums.s3 += y[3+96] * ((float)((q2[3] >> 4) | ((qh[3] & Q6_K_MASK4) >> 2)) - 32.f); + if (dbg) printf("HELPER dall=%f s=[%d %d %d %d] d=[%f %f %f %f] ql0=%d qh0=%d y00=%f\n", + dall, s0, s2, s4, s6, d0, d1, d2, d3, (int)q1[0], (int)qh[0], y0.s0); + + return dall * (d0 * s0 + d1 * s2 + d2 * s4 + d3 * s6); +#else + global char * sc = blk_scales + ib*16 + is; - sumf += dall * (sums.s0 * sc[0] + sums.s1 * sc[2] + sums.s2 * sc[4] + sums.s3 * sc[6]); + // Vectorized loads: 3 uchar4 weight loads instead of 12 scalar byte reads. + // q_offset_l/h are 4-aligned, so these are aligned vector loads. + uchar4 q1v = vload4(0, q1); + uchar4 q2v = vload4(0, q2); + uchar4 qhv = vload4(0, qh); - return sumf; + int4 q1i = convert_int4(q1v); + int4 q2i = convert_int4(q2v); + int4 qhi = convert_int4(qhv); + + // Reconstruct the four 6-bit weight groups (low/high nibble of ql OR'd with the + // matching 2-bit plane of qh), same arithmetic as the scalar version, then dot() + // against the cached activation lanes. + float4 w0 = convert_float4((q1i & 0xF) | ((qhi & Q6_K_MASK1) << 4)) - 32.f; + float4 w1 = convert_float4((q2i & 0xF) | ((qhi & Q6_K_MASK2) << 2)) - 32.f; + float4 w2 = convert_float4((q1i >> 4) | ((qhi & Q6_K_MASK3) )) - 32.f; + float4 w3 = convert_float4((q2i >> 4) | ((qhi & Q6_K_MASK4) >> 2)) - 32.f; + + return dall * (dot(y0, w0) * sc[0] + dot(y1, w1) * sc[2] + + dot(y2, w2) * sc[4] + dot(y3, w3) * sc[6]); +#endif } #undef N_DST @@ -89,7 +124,7 @@ inline float block_q_6_K_dot_y_flat( #define N_SIMDGROUP 2 #define N_SIMDWIDTH 16 #elif defined (ADRENO_GPU) -#define N_DST 4 +#define N_DST 16 #define N_SIMDGROUP 2 #define N_SIMDWIDTH 64 #endif @@ -119,6 +154,11 @@ kernel void kernel_mul_mv_q6_K_f32_flat( int ne1, int r2, int r3 +#if defined(ADRENO_OLD_COMPILER) + , + uchar q6k_mask // runtime 0xFF; the host passes it so the compiler cannot + // constant-fold the printf guards below into nothing +#endif ) { src1 = (global float*)((global char*)src1 + offset1); dst = (global float*)((global char*)dst + offsetd); @@ -134,6 +174,22 @@ kernel void kernel_mul_mv_q6_K_f32_flat( int first_row = (N_SIMDGROUP * r0 + get_sub_group_id()) * N_DST; +#if defined(ADRENO_OLD_COMPILER) + // 64-bit `ulong` integer arithmetic is miscompiled here -> the base-pointer byte + // offsets came out wrong, so EVERY weight/scale read hit the wrong address. This + // was the primary cause of the q6_K flat failure (q5_K uses int offsets and is + // unaffected). Compute the block index in `int` and widen to `ulong` only inside + // the pointer expression: the byte offset stays 64-bit, but there is no ulong + // arithmetic chain to miscompile. The int index would overflow past ~2^31 blocks, + // which no realistic weight reaches -- but that is a narrowing, so keep it off the + // conformant path, which retains full ulong arithmetic. + int offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); + + global uchar * blk_ql = (global uchar *) src0_ql + (ulong)offset_src0 * 128; + global uchar * blk_qh = (global uchar *) src0_qh + (ulong)offset_src0 * 64; + global char * blk_scales = (global char *) src0_s + (ulong)offset_src0 * 16; + global half * blk_d = (global half *) src0_d + offset_src0; +#else ulong offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); ulong offset_src0_ql = offset_src0 * 128; ulong offset_src0_qh = offset_src0 * 64; @@ -144,51 +200,73 @@ kernel void kernel_mul_mv_q6_K_f32_flat( global uchar * blk_qh = (global uchar *) src0_qh + offset_src0_qh; global char * blk_scales = (global char *) src0_s + offset_src0_s; global half * blk_d = (global half *) src0_d + offset_src0_d; +#endif global float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1; - int tid = get_sub_group_local_id()/BLOCK_STRIDE; // first block_stride groups have tid=0 - int ix = get_sub_group_local_id()%BLOCK_STRIDE; // first block is 0..block_stride-1 + int tid = get_sub_group_local_id()%(N_SIMDWIDTH/BLOCK_STRIDE); // within-super-block part, 0..15 + int ix = get_sub_group_local_id()/(N_SIMDWIDTH/BLOCK_STRIDE); // super-block selector, 0..BLOCK_STRIDE-1 int ip = tid/8; // first or second half of (super) block (0 or 1) int il = tid%8; // each half has 8 parts, one per scale int n = 4; // 4 scales at a time (and 4 sums) int l0 = n*il; // offset into half-block, 0..28 int is = 8*ip + l0/16; // 0, 1, 8, 9 - float4 sumf = 0; + float sumf[N_DST]; + for (int row = 0; row < N_DST; row++) { + sumf[row] = 0.f; + } for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { - if (first_row + 0 < ne01) { - sumf.s0 += block_q_6_K_dot_y_flat(blk_ql + 0*nb*128, blk_qh + 0*nb*64, blk_scales + 0*nb*16, blk_d + 0*nb, yy, ib, ip, is, l0); - } - if (first_row + 1 < ne01) { - sumf.s1 += block_q_6_K_dot_y_flat(blk_ql + 1*nb*128, blk_qh + 1*nb*64, blk_scales + 1*nb*16, blk_d + 1*nb, yy, ib, ip, is, l0); - } - if (first_row + 2 < ne01) { - sumf.s2 += block_q_6_K_dot_y_flat(blk_ql + 2*nb*128, blk_qh + 2*nb*64, blk_scales + 2*nb*16, blk_d + 2*nb, yy, ib, ip, is, l0); - } - if (first_row + 3 < ne01) { - sumf.s3 += block_q_6_K_dot_y_flat(blk_ql + 3*nb*128, blk_qh + 3*nb*64, blk_scales + 3*nb*16, blk_d + 3*nb, yy, ib, ip, is, l0); + global float * y = yy + ib * QK_K + 128*ip + l0; +#if defined(ADRENO_OLD_COMPILER) + // vload4 of f32 is miscompiled here; index the lanes scalar instead. + float4 y0 = (float4)(y[ 0], y[ 1], y[ 2], y[ 3]); + float4 y1 = (float4)(y[32], y[33], y[34], y[35]); + float4 y2 = (float4)(y[64], y[65], y[66], y[67]); + float4 y3 = (float4)(y[96], y[97], y[98], y[99]); +#else + float4 y0 = vload4(0, y + 0); + float4 y1 = vload4(0, y + 32); + float4 y2 = vload4(0, y + 64); + float4 y3 = vload4(0, y + 96); +#endif + + for (int row = 0; row < N_DST; row++) { + if (first_row + row < ne01) { +#if defined(ADRENO_OLD_COMPILER) + int dbg = (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ib==0 && + ne00==256 && ne01==16 && get_sub_group_local_id()==0) ? 1 : 0; + sumf[row] += block_q_6_K_dot_y_flat( + blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb, + ib, ip, is, l0, dbg, y0, y1, y2, y3); +#else + sumf[row] += block_q_6_K_dot_y_flat( + blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb, + ib, ip, is, l0, y0, y1, y2, y3); +#endif + } } } - float4 tot = (float4)( - sub_group_reduce_add(sumf.s0), - sub_group_reduce_add(sumf.s1), - sub_group_reduce_add(sumf.s2), - sub_group_reduce_add(sumf.s3) - ); - if (get_sub_group_local_id() == 0) { - if (first_row + 0 < ne01) { - dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0; - } - if (first_row + 1 < ne01) { - dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1; - } - if (first_row + 2 < ne01) { - dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2; - } - if (first_row + 3 < ne01) { - dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3; +#if defined(ADRENO_OLD_COMPILER) + // Optimizer barrier. This compiler drops the sumf partials unless a side effect + // forces them to materialize. q6k_mask is a kernel arg the compiler cannot prove + // is never 0xFE (the host always passes 0xFF), so the printf survives compilation + // but never executes. FRAGILE: the exact set and placement of these guarded + // printfs is load-bearing on E031.41 -- removing any one re-breaks q6_K. + if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && ne00==256 && ne01==16 && get_sub_group_local_id()<16) { + printf("Q6KLANE lane=%d ip=%d il=%d is=%d l0=%d sumf0=%f\n", + get_sub_group_local_id(), ip, il, is, l0, sumf[0]); + } +#endif + for (int row = 0; row < N_DST; row++) { + float tot = sub_group_reduce_add(sumf[row]); + if (get_sub_group_local_id() == 0 && first_row + row < ne01) { + dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot; +#if defined(ADRENO_OLD_COMPILER) + if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ne00==256 && ne01==16) + printf("Q6KTOT tot=%f\n", tot); +#endif } } } diff --git a/ggml/src/ggml-opencl/kernels/norm.cl b/ggml/src/ggml-opencl/kernels/norm.cl index 170f822..a5ccac2 100644 --- a/ggml/src/ggml-opencl/kernels/norm.cl +++ b/ggml/src/ggml-opencl/kernels/norm.cl @@ -24,6 +24,7 @@ kernel void kernel_norm( int ne01, int ne02, int ne03, + ulong nb00, ulong nb01, ulong nb02, ulong nb03, @@ -43,7 +44,8 @@ kernel void kernel_norm( // parallel sum sum[get_local_id(0)] = 0.0f; for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) { - sum[get_local_id(0)] += x[i00]; + // this kernel handles float, nb00/4 translates byte offset to element offset + sum[get_local_id(0)] += x[i00*nb00/4]; } // reduce barrier(CLK_LOCAL_MEM_FENCE); @@ -60,7 +62,8 @@ kernel void kernel_norm( global float * y = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00; sum[get_local_id(0)] = 0.0f; for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) { - y[i00] = x[i00] - mean; + // this kernel handles float, nb00/4 translates byte offset to element offset + y[i00] = x[i00*nb00/4] - mean; sum[get_local_id(0)] += y[i00] * y[i00]; } diff --git a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl new file mode 100644 index 0000000..5b6dea7 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl @@ -0,0 +1,40 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +// Quantize a contiguous K-major [N, K] f32 activation buffer (K contiguous) into 32-value Q8_1 SoA blocks. +// Store int8 quants, per-block scale d, and scaled sum s = d * sum(qs); one work-item handles each block. +__kernel void kernel_quant_a_q8_1( + __global const float * src, // [N * K] + __global char * qa, // [N * K] + __global half * da, // [N * (K/32)] + __global half * sa, // [N * (K/32)] + int total_blocks // N * (K/32) +) { + const int blk = get_global_id(0); + if (blk >= total_blocks) { + return; + } + + const int base = blk * 32; + + float v[32]; + float amax = 0.0f; + #pragma unroll + for (int i = 0; i < 32; ++i) { + v[i] = src[base + i]; + amax = fmax(amax, fabs(v[i])); + } + + const float d = amax / 127.0f; + const float id = (amax > 0.0f) ? (127.0f / amax) : 0.0f; + + int sum = 0; + #pragma unroll + for (int i = 0; i < 32; ++i) { + const int q = (int)rint(v[i] * id); + qa[base + i] = (char)q; + sum += q; + } + + da[blk] = (half)d; + sa[blk] = (half)(d * (float)sum); +} diff --git a/ggml/src/ggml-opencl/kernels/rms_norm.cl b/ggml/src/ggml-opencl/kernels/rms_norm.cl index 4b18d17..9908562 100644 --- a/ggml/src/ggml-opencl/kernels/rms_norm.cl +++ b/ggml/src/ggml-opencl/kernels/rms_norm.cl @@ -188,3 +188,182 @@ kernel void kernel_rms_norm_mul( y[i00] = (x[i00] * scale) * f[i00%(ne10/4)]; } } + +//------------------------------------------------------------------------------ +// rms_norm + mul (norm weight) + add (residual), fused. Mirrors +// kernel_rms_norm_mul with an extra residual operand src2: computes +// y = (rmsnorm(x) * w) + g +// in one dispatch, removing one kernel launch + one global round-trip per +// residual block (the dominant per-layer adjacency on Gemma matformers). +//------------------------------------------------------------------------------ +kernel void kernel_rms_norm_mul_add( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * src2, + ulong offset2, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + int ne13, + ulong nb11, + ulong nb12, + ulong nb13, + int ne20, + int ne21, + int ne22, + int ne23, + ulong nb21, + ulong nb22, + ulong nb23, + ulong nb1, + ulong nb2, + ulong nb3, + float eps, + local float * sum +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + src2 = src2 + offset2; + dst = dst + offsetd; + + if (get_sub_group_id() == 0) { + sum[get_sub_group_local_id()] = 0.0f; + } + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0); + + global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01); + global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11); + global float4 * g = (global float4 *) (src2 + (i03%ne23)*nb23 + (i02%ne22)*nb22 + (i01%ne21)*nb21); + + float sumf = 0; + + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + sumf += dot(x[i00], x[i00]); + } + sumf = sub_group_reduce_add(sumf); + + barrier(CLK_LOCAL_MEM_FENCE); + + if (get_sub_group_local_id() == 0) { + sum[get_sub_group_id()] = sumf; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + sumf = sum[get_sub_group_local_id()]; + sumf = sub_group_reduce_add(sumf); + + float mean = sumf / ne00; + float scale = 1.0f/sqrt(mean + eps); + + global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1); + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + y[i00] = (x[i00] * scale) * f[i00%(ne10/4)] + g[i00%(ne20/4)]; + } +} + +//------------------------------------------------------------------------------ +// rms_norm + mul(norm weight) + add(residual) + mul(scalar scale), fused. +// Computes y = ((rmsnorm(x) * w) + g) * s, where s is a broadcast SCALAR (e.g. +// Gemma-4 layer_output_scale). Folds the trailing per-layer l_out scale-mul into +// the residual-norm kernel: one extra dispatch + global round-trip saved per +// layer. src3 points at the single scale value. +//------------------------------------------------------------------------------ +kernel void kernel_rms_norm_mul_add_scale( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * src2, + ulong offset2, + global char * src3, + ulong offset3, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + int ne13, + ulong nb11, + ulong nb12, + ulong nb13, + int ne20, + int ne21, + int ne22, + int ne23, + ulong nb21, + ulong nb22, + ulong nb23, + ulong nb1, + ulong nb2, + ulong nb3, + float eps, + local float * sum +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + src2 = src2 + offset2; + src3 = src3 + offset3; + dst = dst + offsetd; + + const float sc = *((global float *) src3); + + if (get_sub_group_id() == 0) { + sum[get_sub_group_local_id()] = 0.0f; + } + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0); + + global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01); + global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11); + global float4 * g = (global float4 *) (src2 + (i03%ne23)*nb23 + (i02%ne22)*nb22 + (i01%ne21)*nb21); + + float sumf = 0; + + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + sumf += dot(x[i00], x[i00]); + } + sumf = sub_group_reduce_add(sumf); + + barrier(CLK_LOCAL_MEM_FENCE); + + if (get_sub_group_local_id() == 0) { + sum[get_sub_group_id()] = sumf; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + sumf = sum[get_sub_group_local_id()]; + sumf = sub_group_reduce_add(sumf); + + float mean = sumf / ne00; + float scale = 1.0f/sqrt(mean + eps); + + global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1); + for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) { + y[i00] = ((x[i00] * scale) * f[i00%(ne10/4)] + g[i00%(ne20/4)]) * sc; + } +} diff --git a/ggml/src/ggml-opencl/kernels/rope.cl b/ggml/src/ggml-opencl/kernels/rope.cl index 82f4cd8..5b99d42 100644 --- a/ggml/src/ggml-opencl/kernels/rope.cl +++ b/ggml/src/ggml-opencl/kernels/rope.cl @@ -75,7 +75,8 @@ kernel void kernel_rope_norm_f32( float ext_factor, float attn_factor, float beta_fast, - float beta_slow + float beta_slow, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -94,14 +95,15 @@ kernel void kernel_rope_norm_f32( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - float theta = theta_base * pow(freq_base, inv_ndims*i0); + float theta = theta_base * pow(freq_base, inv_ndims*iw); float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); @@ -154,7 +156,8 @@ kernel void kernel_rope_norm_f16( float ext_factor, float attn_factor, float beta_fast, - float beta_slow + float beta_slow, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -173,14 +176,15 @@ kernel void kernel_rope_norm_f16( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - float theta = theta_base * pow(freq_base, inv_ndims*i0); + float theta = theta_base * pow(freq_base, inv_ndims*iw); float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); @@ -233,7 +237,8 @@ kernel void kernel_rope_neox_f32( float ext_factor, float attn_factor, float beta_fast, - float beta_slow + float beta_slow, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -252,17 +257,18 @@ kernel void kernel_rope_neox_f32( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - const float theta = theta_base * pow(freq_base, inv_ndims*i0); + const float theta = theta_base * pow(freq_base, inv_ndims*iw); const float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); - global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00); - global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0); + global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00); + global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + (n_offs + ic)*nb0); const float x0 = src[0]; const float x1 = src[n_dims/2]; @@ -312,7 +318,8 @@ kernel void kernel_rope_neox_f16( float ext_factor, float attn_factor, float beta_fast, - float beta_slow + float beta_slow, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -331,17 +338,18 @@ kernel void kernel_rope_neox_f16( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - const float theta = theta_base * pow(freq_base, inv_ndims*i0); + const float theta = theta_base * pow(freq_base, inv_ndims*iw); const float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); - global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00); - global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0); + global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00); + global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + (n_offs + ic)*nb0); const float x0 = src[0]; const float x1 = src[n_dims/2]; @@ -393,7 +401,8 @@ kernel void kernel_rope_multi_f32( float beta_fast, float beta_slow, int4 sections, - int is_imrope + int is_imrope, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); @@ -414,10 +423,11 @@ kernel void kernel_rope_multi_f32( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - const int sector = (i0 / 2) % sect_dims; + const int sector = ic % sect_dims; float theta_base = 0.0f; if (is_imrope) { @@ -445,14 +455,14 @@ kernel void kernel_rope_multi_f32( } } - const float theta = theta_base * pow(freq_base, inv_ndims*i0); + const float theta = theta_base * pow(freq_base, inv_ndims*iw); const float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); - global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00); - global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0); + global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00); + global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + (n_offs + ic)*nb0); const float x0 = src[0]; const float x1 = src[n_dims/2]; @@ -504,12 +514,13 @@ kernel void kernel_rope_multi_f16( float beta_fast, float beta_slow, int4 sections, - int is_imrope + int is_imrope, + int n_offs ) { src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); src2 = (global float*)((global char*)src2 + offset2); - dst = (global float*)((global char*)dst + offsetd); + dst = (global half*)((global char*)dst + offsetd); int i3 = get_group_id(2); int i2 = get_group_id(1); @@ -525,10 +536,11 @@ kernel void kernel_rope_multi_f16( float inv_ndims = -1.f/n_dims; for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) { - if (i0 < n_dims) { - int ic = i0/2; + if (i0 >= n_offs && i0 < n_offs + n_dims) { + int iw = i0 - n_offs; // relative idx + int ic = iw/2; - const int sector = (i0 / 2) % sect_dims; + const int sector = ic % sect_dims; float theta_base = 0.0f; if (is_imrope) { @@ -556,14 +568,14 @@ kernel void kernel_rope_multi_f16( } } - const float theta = theta_base * pow(freq_base, inv_ndims*i0); + const float theta = theta_base * pow(freq_base, inv_ndims*iw); const float freq_factor = src2 != src0 ? src2[ic] : 1.0f; - float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor); + float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor); - global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00); - global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0); + global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + (n_offs + ic)*nb00); + global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + (n_offs + ic)*nb0); const float x0 = src[0]; const float x1 = src[n_dims/2]; @@ -702,7 +714,7 @@ kernel void kernel_rope_vision_f16( src0 = (global void*)((global char*)src0 + offset0); src1 = (global int*)((global char*)src1 + offset1); src2 = (global float*)((global char*)src2 + offset2); - dst = (global float*)((global char*)dst + offsetd); + dst = (global half*)((global char*)dst + offsetd); int i3 = get_group_id(2); int i2 = get_group_id(1); diff --git a/ggml/src/ggml-opencl/kernels/sdpa_xmem_f32_f16_os8.cl b/ggml/src/ggml-opencl/kernels/sdpa_xmem_f32_f16_os8.cl new file mode 100644 index 0000000..26f0fbd --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/sdpa_xmem_f32_f16_os8.cl @@ -0,0 +1,871 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load : enable + +#define bool2 uchar2 +#define bool3 uchar3 +#define bool4 uchar4 + +__constant sampler_t smp_none = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_NONE | CLK_FILTER_NEAREST; +__constant sampler_t smp_zero = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST; + +__kernel void adreno_xmem_attn_q_f32_to_img_scaled(const global void * src_void, + ulong src_offset, + write_only image2d_t dst_image2d, + const float scale, + const int d_head, + const int n_q, + const int n_head, + const int n_head_kv, + const int n_batch, + const ulong src_nb1, + const ulong src_nb2, + const ulong src_nb3) { + const int x = get_global_id(0); + const int flat_h = get_global_id(1); + const int d = get_global_id(2); + + const int heads_total = n_head * n_batch; + const int kpack = d_head / 4; + + if (x >= n_q || flat_h >= heads_total || d >= kpack) { + return; + } + + const int batch = flat_h / n_head; + const int head = flat_h % n_head; + const int gqa = n_head / n_head_kv; + const int head_kv = head / gqa; + const int head_group = head - head_kv * gqa; + const int compact_h = batch * n_head_kv + head_kv; + const int compact_x = head_group * n_q + x; + const int c = d * 4; + + const global char * src_base = (const global char *) src_void + src_offset; + const global float * row_ptr = (const global float *) (src_base + batch * src_nb3 + head * src_nb2 + x * src_nb1); + + half4 out = (half4) (0.0h); + out.x = convert_half(row_ptr[c + 0] * scale); + if (c + 1 < d_head) { + out.y = convert_half(row_ptr[c + 1] * scale); + } + if (c + 2 < d_head) { + out.z = convert_half(row_ptr[c + 2] * scale); + } + if (c + 3 < d_head) { + out.w = convert_half(row_ptr[c + 3] * scale); + } + + write_imageh(dst_image2d, (int2) (compact_x, compact_h * kpack + d), out); +} + +__kernel void adreno_xmem_attn_kv_f32_to_img_gqa(const global void * src_void, + ulong src_offset, + write_only image2d_t dst_image2d, + const int d_head, + const int n_kv, + const int n_kv_padded, + const int n_head_kv, + const int n_batch, + const ulong src_nb1, + const ulong src_nb2, + const ulong src_nb3) { + const int x = get_global_id(0); + const int flat_h = get_global_id(1); + const int d = get_global_id(2); + + const int kv_heads_total = n_head_kv * n_batch; + const int kpack = d_head / 4; + + if (x >= n_kv_padded || flat_h >= kv_heads_total || d >= kpack) { + return; + } + + const int batch = flat_h / n_head_kv; + const int head_kv = flat_h % n_head_kv; + const int c = d * 4; + + half4 out = (half4) (0.0h); + if (x < n_kv) { + const global char * src_base = (const global char *) src_void + src_offset; + const global float * row_ptr = + (const global float *) (src_base + batch * src_nb3 + head_kv * src_nb2 + x * src_nb1); + out.x = convert_half(row_ptr[c + 0]); + if (c + 1 < d_head) { + out.y = convert_half(row_ptr[c + 1]); + } + if (c + 2 < d_head) { + out.z = convert_half(row_ptr[c + 2]); + } + if (c + 3 < d_head) { + out.w = convert_half(row_ptr[c + 3]); + } + } + + write_imageh(dst_image2d, (int2) (x, flat_h * kpack + d), out); +} + +__kernel void adreno_xmem_attn_kv_f16_to_img_gqa(const global void * src_void, + ulong src_offset, + write_only image2d_t dst_image2d, + const int d_head, + const int n_kv, + const int n_kv_padded, + const int n_head_kv, + const int n_batch, + const ulong src_nb1, + const ulong src_nb2, + const ulong src_nb3) { + const int x = get_global_id(0); + const int flat_h = get_global_id(1); + const int d = get_global_id(2); + + const int kv_heads_total = n_head_kv * n_batch; + const int kpack = d_head / 4; + + if (x >= n_kv_padded || flat_h >= kv_heads_total || d >= kpack) { + return; + } + + const int batch = flat_h / n_head_kv; + const int head_kv = flat_h % n_head_kv; + const int c = d * 4; + + half4 out = (half4) (0.0h); + if (x < n_kv) { + const global char * src_base = (const global char *) src_void + src_offset; + const global half * row_ptr = + (const global half *) (src_base + batch * src_nb3 + head_kv * src_nb2 + x * src_nb1); + out.x = row_ptr[c + 0]; + if (c + 1 < d_head) { + out.y = row_ptr[c + 1]; + } + if (c + 2 < d_head) { + out.z = row_ptr[c + 2]; + } + if (c + 3 < d_head) { + out.w = row_ptr[c + 3]; + } + } + + write_imageh(dst_image2d, (int2) (x, flat_h * kpack + d), out); +} + +__kernel void adreno_xmem_attn_img_to_f32(global void * dst_void, + ulong dst_offset, + read_only image2d_t src_image2d, + const int d_head, + const int n_q, + const int n_head, + const int n_head_kv, + const int n_batch, + const ulong dst_nb1, + const ulong dst_nb2, + const ulong dst_nb3) { + const int x = get_global_id(0); + const int flat_h = get_global_id(1); + const int d = get_global_id(2); + + const int heads_total = n_head * n_batch; + const int kpack = d_head / 4; + + if (x >= n_q || flat_h >= heads_total || d >= kpack) { + return; + } + + const int batch = flat_h / n_head; + const int head = flat_h % n_head; + const int gqa = n_head / n_head_kv; + const int head_kv = head / gqa; + const int head_group = head - head_kv * gqa; + const int compact_h = batch * n_head_kv + head_kv; + const int compact_x = head_group * n_q + x; + const int c = d * 4; + + global char * dst_base = (global char *) dst_void + dst_offset; + global float * row_ptr = (global float *) (dst_base + batch * dst_nb3 + x * dst_nb2 + head * dst_nb1); + + const half4 in_value = read_imageh(src_image2d, smp_zero, (int2) (compact_x, compact_h * kpack + d)); + row_ptr[c + 0] = convert_float(in_value.x); + if (c + 1 < d_head) { + row_ptr[c + 1] = convert_float(in_value.y); + } + if (c + 2 < d_head) { + row_ptr[c + 2] = convert_float(in_value.z); + } + if (c + 3 < d_head) { + row_ptr[c + 3] = convert_float(in_value.w); + } +} + +__kernel void adreno_xmem_attn_k_gather(global half4 * dst_tensor_buffer, + read_only image2d_t src_tensor_image2d, + const int4 shared_int4_0, + const int4 shared_int4_1) { + int X = get_global_id(0); + int Y = get_global_id(1); + int S = get_global_id(2); + if (X >= shared_int4_0.w || Y >= shared_int4_0.y || S >= shared_int4_0.z) { + return; + } + half temps[4]; + temps[0] = (half) (0.f); + temps[1] = (half) (0.f); + temps[2] = (half) (0.f); + temps[3] = (half) (0.f); + for (int i = 0; i < 4; ++i) { + int dst_channel = S * 4 + i; + if (dst_channel < shared_int4_0.x) { + int s_y = Y; + int s_x = dst_channel; + int s_c = X; + { + int slice_coord_TMP = (s_c) / 4; + int sub_ch_coord_TMP = (s_c) % 4; + half4 src_TMP = read_imageh(src_tensor_image2d, smp_zero, + (int2) ((s_x), ((s_y) *shared_int4_1.x + (slice_coord_TMP)))); + temps[i] = (half[4]){ src_TMP.x, src_TMP.y, src_TMP.z, src_TMP.w }[sub_ch_coord_TMP]; + }; + } + } + half4 result; + result.x = temps[0]; + result.y = temps[1]; + result.z = temps[2]; + result.w = temps[3]; + dst_tensor_buffer[(((S) *shared_int4_0.y + (Y)) * shared_int4_0.w + (X))] = result; +} + +__kernel void adreno_xmem_attn_pack_k(global half4 * dst_tensor_buffer, + read_only image1d_buffer_t src_image_buffer, + const int4 shared_int4_0, + const int4 shared_int4_1, + const int4 shared_int4_2) { + int linear_index = get_global_id(0); + if (linear_index >= shared_int4_0.y) { + return; + } + if (get_global_id(1) != 0) { + return; + } + if (get_global_id(2) != 0) { + return; + } + int dst_o_sp_i_ogroup = linear_index; + int dst_ogroup = dst_o_sp_i_ogroup % shared_int4_0.x; + int dst_o_sp_i = dst_o_sp_i_ogroup / shared_int4_0.x; + int dst_i = dst_o_sp_i % shared_int4_0.z; + int dst_o_sp = dst_o_sp_i / shared_int4_0.z; + int dst_sp = dst_o_sp % shared_int4_1.x; + int dst_o = dst_o_sp / shared_int4_1.x; + int i_slice = dst_i; + int o_slice = dst_o * shared_int4_0.x + dst_ogroup; + int spatial_linear = dst_sp; + int W = spatial_linear % shared_int4_1.y; + int H = spatial_linear / shared_int4_1.y; + half4 w0 = (half4) (0); + half4 w1 = (half4) (0); + half4 w2 = (half4) (0); + half4 w3 = (half4) (0); + + if (i_slice * 4 < shared_int4_0.w && o_slice < shared_int4_1.w) { + w0 = read_imageh(src_image_buffer, (((o_slice) *shared_int4_1.z + (W)) * shared_int4_2.x + (i_slice * 4))); + } + if (i_slice * 4 + 1 < shared_int4_0.w && o_slice < shared_int4_1.w) { + w1 = read_imageh(src_image_buffer, (((o_slice) *shared_int4_1.z + (W)) * shared_int4_2.x + (i_slice * 4 + 1))); + } + if (i_slice * 4 + 2 < shared_int4_0.w && o_slice < shared_int4_1.w) { + w2 = read_imageh(src_image_buffer, (((o_slice) *shared_int4_1.z + (W)) * shared_int4_2.x + (i_slice * 4 + 2))); + } + if (i_slice * 4 + 3 < shared_int4_0.w && o_slice < shared_int4_1.w) { + w3 = read_imageh(src_image_buffer, (((o_slice) *shared_int4_1.z + (W)) * shared_int4_2.x + (i_slice * 4 + 3))); + } + half4 r0 = w0; + half4 r1 = w1; + half4 r2 = w2; + half4 r3 = w3; + dst_tensor_buffer[linear_index * 4 + 0] = r0; + dst_tensor_buffer[linear_index * 4 + 1] = r1; + dst_tensor_buffer[linear_index * 4 + 2] = r2; + dst_tensor_buffer[linear_index * 4 + 3] = r3; +} + +__attribute__((qcom_max_concurrent_subgroups(12))) __kernel void adreno_xmem_attn_qk_gemm( + global half4 * dst_tensor_buffer, + constant half8 * weights_buffer __attribute__((sub_group_uniform)), + constant half8 * xmem_buffer __attribute__((max_constant_size((6144)))), + read_only image2d_t src_tensor_image2d, + const int4 shared_int4_0, + const int4 shared_int4_1, + const int4 shared_int4_2) { + int X = get_group_id(1) * get_local_size(0) + get_local_id(0); + int Y = get_group_id(2) * get_local_size(1) + get_local_id(1); + int Z = get_group_id(0) * get_local_size(2) + get_local_id(2); + if (X >= shared_int4_0.z || Y >= shared_int4_0.x) { + return; + } + if (Z * 8 >= shared_int4_0.y) { + return; + } + + half4 r0 = (half4) (0.f); + half4 r1 = (half4) (0.f); + half4 r2 = (half4) (0.f); + half4 r3 = (half4) (0.f); + half4 r4 = (half4) (0.f); + half4 r5 = (half4) (0.f); + half4 r6 = (half4) (0.f); + half4 r7 = (half4) (0.f); + int x_coord = mad24(X, shared_int4_2.y, shared_int4_1.y); + int y_coord = mad24(Y, shared_int4_2.z, shared_int4_1.z); + int coord_x, coord_y, coord_s; + int f_offset = (Z * shared_int4_1.w + Y) * shared_int4_1.x * 32; + + int subgroup_id = (int) ((0x1F & qcom_get_physical_sub_group_id())); + subgroup_id = subgroup_id % 12; + int c_offset = mul24(subgroup_id, shared_int4_0.w); + __constant half16 * weights_cache = (__constant half16 *) &xmem_buffer[c_offset]; + coord_y = Y; + coord_x = X; + coord_s = 0; + do { + half4 src0 = + read_imageh(src_tensor_image2d, smp_zero, (int2) ((coord_x), ((coord_y) *shared_int4_2.x + (coord_s)))); + coord_s++; + half4 src1 = + read_imageh(src_tensor_image2d, smp_zero, (int2) ((coord_x), ((coord_y) *shared_int4_2.x + (coord_s)))); + coord_s++; + qcom_sub_group_constant_load8(xmem_buffer, weights_buffer, c_offset, f_offset >> 1, 32); + f_offset += 64; + qcom_sub_group_sync(QCOM_CLK_CONST_LOAD_SYNC); + r0 += src0.x * weights_cache[0].s0123; + r0 += src0.y * weights_cache[0].s4567; + r0 += src0.z * weights_cache[0].s89ab; + r0 += src0.w * weights_cache[0].scdef; + r1 += src0.x * weights_cache[1].s0123; + r1 += src0.y * weights_cache[1].s4567; + r1 += src0.z * weights_cache[1].s89ab; + r1 += src0.w * weights_cache[1].scdef; + r2 += src0.x * weights_cache[2].s0123; + r2 += src0.y * weights_cache[2].s4567; + r2 += src0.z * weights_cache[2].s89ab; + r2 += src0.w * weights_cache[2].scdef; + r3 += src0.x * weights_cache[3].s0123; + r3 += src0.y * weights_cache[3].s4567; + r3 += src0.z * weights_cache[3].s89ab; + r3 += src0.w * weights_cache[3].scdef; + r4 += src0.x * weights_cache[4].s0123; + r4 += src0.y * weights_cache[4].s4567; + r4 += src0.z * weights_cache[4].s89ab; + r4 += src0.w * weights_cache[4].scdef; + r5 += src0.x * weights_cache[5].s0123; + r5 += src0.y * weights_cache[5].s4567; + r5 += src0.z * weights_cache[5].s89ab; + r5 += src0.w * weights_cache[5].scdef; + r6 += src0.x * weights_cache[6].s0123; + r6 += src0.y * weights_cache[6].s4567; + r6 += src0.z * weights_cache[6].s89ab; + r6 += src0.w * weights_cache[6].scdef; + r7 += src0.x * weights_cache[7].s0123; + r7 += src0.y * weights_cache[7].s4567; + r7 += src0.z * weights_cache[7].s89ab; + r7 += src0.w * weights_cache[7].scdef; + r0 += src1.x * weights_cache[8].s0123; + r0 += src1.y * weights_cache[8].s4567; + r0 += src1.z * weights_cache[8].s89ab; + r0 += src1.w * weights_cache[8].scdef; + r1 += src1.x * weights_cache[9].s0123; + r1 += src1.y * weights_cache[9].s4567; + r1 += src1.z * weights_cache[9].s89ab; + r1 += src1.w * weights_cache[9].scdef; + r2 += src1.x * weights_cache[10].s0123; + r2 += src1.y * weights_cache[10].s4567; + r2 += src1.z * weights_cache[10].s89ab; + r2 += src1.w * weights_cache[10].scdef; + r3 += src1.x * weights_cache[11].s0123; + r3 += src1.y * weights_cache[11].s4567; + r3 += src1.z * weights_cache[11].s89ab; + r3 += src1.w * weights_cache[11].scdef; + r4 += src1.x * weights_cache[12].s0123; + r4 += src1.y * weights_cache[12].s4567; + r4 += src1.z * weights_cache[12].s89ab; + r4 += src1.w * weights_cache[12].scdef; + r5 += src1.x * weights_cache[13].s0123; + r5 += src1.y * weights_cache[13].s4567; + r5 += src1.z * weights_cache[13].s89ab; + r5 += src1.w * weights_cache[13].scdef; + r6 += src1.x * weights_cache[14].s0123; + r6 += src1.y * weights_cache[14].s4567; + r6 += src1.z * weights_cache[14].s89ab; + r6 += src1.w * weights_cache[14].scdef; + r7 += src1.x * weights_cache[15].s0123; + r7 += src1.y * weights_cache[15].s4567; + r7 += src1.z * weights_cache[15].s89ab; + r7 += src1.w * weights_cache[15].scdef; + } while (coord_s < shared_int4_2.x); + + coord_s = mul24(Z, 8); + coord_x = X; + coord_y = Y; + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r0); + if (coord_s < 0) { + res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0)))); + } + dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res; + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r1); + if (coord_s < 0) { + res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0)))); + } + dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res; + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r2); + if (coord_s < 0) { + res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0)))); + } + dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res; + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r3); + if (coord_s < 0) { + res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0)))); + } + dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res; + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r4); + if (coord_s < 0) { + res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0)))); + } + dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res; + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r5); + if (coord_s < 0) { + res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0)))); + } + dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res; + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r6); + if (coord_s < 0) { + res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0)))); + } + dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res; + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r7); + if (coord_s < 0) { + res += read_imageh(src_tensor_image2d, smp_zero, (int2) ((0), ((0) * shared_int4_2.x + (0)))); + } + dst_tensor_buffer[(((coord_s) *shared_int4_0.x + (coord_y)) * shared_int4_0.z + (coord_x))] = res; + coord_s++; + } +} + +__kernel void adreno_xmem_attn_softmax_reduce_basic(read_only image1d_buffer_t src_tensor_image_buffer, + write_only image2d_t dst_tensor_image2d, + const int4 shared_int4_0, + const int4 shared_int4_1) { + int X = get_global_id(0); + int Y = get_global_id(1); + if (X >= shared_int4_0.z || Y >= shared_int4_0.x) { + return; + } + float sum = 0.0f; + int end_channel = shared_int4_0.w; + int end_slice = (end_channel + 3) / 4; + int start_channel = 0; + int start_slice = start_channel / 4; + bool need_per_channels_check = start_channel % 4 != 0 || end_channel % 4 != 0; + float maximum; + { + int slice_coord_TMP = (start_channel) / 4; + int sub_ch_coord_TMP = (start_channel) % 4; + float4 src_TMP = convert_float4( + read_imageh(src_tensor_image_buffer, ((slice_coord_TMP) *shared_int4_1.x + (Y)) * shared_int4_1.y + (X))); + maximum = (float[4]){ src_TMP.x, src_TMP.y, src_TMP.z, src_TMP.w }[sub_ch_coord_TMP]; + }; + for (int d = start_slice; d < end_slice; d += 1) { + float4 mask_dot = (float4) (1.f); + float4 src = + convert_float4(read_imageh(src_tensor_image_buffer, ((d) *shared_int4_1.x + (Y)) * shared_int4_1.y + (X))); + if (need_per_channels_check && (d == start_slice || d == end_slice - 1)) { + if (d * 4 + 0 < start_channel || d * 4 + 0 >= end_channel) { + mask_dot.x = 0.f; + src.x = maximum; + } + if (d * 4 + 1 < start_channel || d * 4 + 1 >= end_channel) { + mask_dot.y = 0.f; + src.y = maximum; + } + if (d * 4 + 2 < start_channel || d * 4 + 2 >= end_channel) { + mask_dot.z = 0.f; + src.z = maximum; + } + if (d * 4 + 3 < start_channel || d * 4 + 3 >= end_channel) { + mask_dot.w = 0.f; + src.w = maximum; + } + } + float new_max = max(src.x, src.y); + new_max = max(new_max, src.z); + new_max = max(new_max, src.w); + new_max = max(new_max, maximum); + float scale = native_exp(maximum - new_max); + maximum = new_max; + sum *= scale; + float4 exp_res = native_exp(src - maximum); + sum += dot(mask_dot, exp_res); + } + if (!isfinite(maximum) || sum == 0.0f) { + write_imageh(dst_tensor_image2d, (int2) (X, Y), (half4) (0.0h)); + return; + } + write_imageh(dst_tensor_image2d, (int2) (X, Y), + (half4) (convert_half(1.0f / sum), convert_half(maximum), 0.0h, 0.0h)); +} + +__kernel void adreno_xmem_attn_softmax_apply_basic(global half4 * dst_tensor_buffer, + read_only image1d_buffer_t src_tensor_image_buffer, + read_only image2d_t src_tensor_1_image2d, + const int4 shared_int4_0, + const int4 shared_int4_1) { + int X = get_global_id(0); + int Y = get_global_id(1); + int Z = get_global_id(2); + if (X >= shared_int4_0.z || Y >= shared_int4_0.x || Z >= shared_int4_0.y) { + return; + } + half4 src = read_imageh(src_tensor_image_buffer, ((Z) *shared_int4_1.x + (Y)) * shared_int4_1.y + (X)); + { + half4 src_final; + { + { + half4 exp_val = read_imageh(src_tensor_1_image2d, smp_zero, (int2) (X, Y)); + src_final = exp(src - exp_val.y) * exp_val.x; + const int k = Z * 4; + const int n_kv = shared_int4_1.z; + if (k + 0 >= n_kv) { + src_final.x = 0.0h; + } + if (k + 1 >= n_kv) { + src_final.y = 0.0h; + } + if (k + 2 >= n_kv) { + src_final.z = 0.0h; + } + if (k + 3 >= n_kv) { + src_final.w = 0.0h; + } + } + } + dst_tensor_buffer[(((Z) *shared_int4_0.x + (Y)) * shared_int4_0.z + (X))] = src_final; + }; +} + +__kernel void adreno_xmem_attn_mask_scores(global half4 * dst_score_tensor_buffer, + read_only image1d_buffer_t src_score_image_buffer, + const global half * mask, + const ulong mask_offset, + const int q_width, + const int n_q, + const int n_kv, + const int n_kv_padded, + const int kv_heads_total, + const int n_head, + const int n_head_kv, + const ulong mask_nb1, + const ulong mask_nb2, + const ulong mask_nb3, + const int mask_ne2, + const int mask_ne3) { + const int X = get_global_id(0); + const int Y = get_global_id(1); + const int Z = get_global_id(2); + const int npack = n_kv_padded / 4; + if (X >= q_width || Y >= kv_heads_total || Z >= npack) { + return; + } + + const int gqa = n_head / n_head_kv; + const int head_kv = Y % n_head_kv; + const int batch = Y / n_head_kv; + const int head_group = X / n_q; + const int q = X - head_group * n_q; + const int head = head_kv * gqa + head_group; + const int mask_head_idx = head % mask_ne2; + const int mask_batch_idx = batch % mask_ne3; + const global char * mask_base = (const global char *) mask + mask_offset; + const global half * mask_row = (const global half *) (mask_base + mask_batch_idx * mask_nb3 + + mask_head_idx * mask_nb2 + q * mask_nb1); + + const half4 score = read_imageh(src_score_image_buffer, ((Z * kv_heads_total + Y) * q_width + X)); + float vals[4] = { + convert_float(score.x), + convert_float(score.y), + convert_float(score.z), + convert_float(score.w), + }; + + for (int lane = 0; lane < 4; ++lane) { + const int k_idx = Z * 4 + lane; + if (k_idx >= n_kv) { + vals[lane] = -INFINITY; + } else { + vals[lane] += convert_float(mask_row[k_idx]); + } + } + + dst_score_tensor_buffer[((Z * kv_heads_total + Y) * q_width + X)] = + (half4) (convert_half(vals[0]), convert_half(vals[1]), convert_half(vals[2]), convert_half(vals[3])); +} + +__kernel void adreno_xmem_attn_pack_v(global half4 * dst_tensor_buffer, + read_only image2d_t src_image2d, + const int4 shared_int4_0, + const int4 shared_int4_1) { + int linear_index = get_global_id(0); + if (linear_index >= shared_int4_0.y) { + return; + } + if (get_global_id(1) != 0) { + return; + } + if (get_global_id(2) != 0) { + return; + } + int dst_o_sp_i_ogroup = linear_index; + int dst_ogroup = dst_o_sp_i_ogroup % shared_int4_0.x; + int dst_o_sp_i = dst_o_sp_i_ogroup / shared_int4_0.x; + int dst_i = dst_o_sp_i % shared_int4_0.z; + int dst_o_sp = dst_o_sp_i / shared_int4_0.z; + int dst_sp = dst_o_sp % shared_int4_1.x; + int dst_o = dst_o_sp / shared_int4_1.x; + int i_slice = dst_i; + int o_slice = dst_o * shared_int4_0.x + dst_ogroup; + int spatial_linear = dst_sp; + int W = spatial_linear % shared_int4_1.y; + int H = spatial_linear / shared_int4_1.y; + half4 w0 = (half4) (0); + half4 w1 = (half4) (0); + half4 w2 = (half4) (0); + half4 w3 = (half4) (0); + + if (i_slice * 4 < shared_int4_0.w && o_slice < shared_int4_1.z) { + w0 = read_imageh(src_image2d, smp_zero, (int2) ((i_slice * 4), ((W) *shared_int4_1.z + (o_slice)))); + } + if (i_slice * 4 + 1 < shared_int4_0.w && o_slice < shared_int4_1.z) { + w1 = read_imageh(src_image2d, smp_zero, (int2) ((i_slice * 4 + 1), ((W) *shared_int4_1.z + (o_slice)))); + } + if (i_slice * 4 + 2 < shared_int4_0.w && o_slice < shared_int4_1.z) { + w2 = read_imageh(src_image2d, smp_zero, (int2) ((i_slice * 4 + 2), ((W) *shared_int4_1.z + (o_slice)))); + } + if (i_slice * 4 + 3 < shared_int4_0.w && o_slice < shared_int4_1.z) { + w3 = read_imageh(src_image2d, smp_zero, (int2) ((i_slice * 4 + 3), ((W) *shared_int4_1.z + (o_slice)))); + } + half4 r0 = w0; + half4 r1 = w1; + half4 r2 = w2; + half4 r3 = w3; + dst_tensor_buffer[linear_index * 4 + 0] = r0; + dst_tensor_buffer[linear_index * 4 + 1] = r1; + dst_tensor_buffer[linear_index * 4 + 2] = r2; + dst_tensor_buffer[linear_index * 4 + 3] = r3; +} + +__attribute__((qcom_max_concurrent_subgroups(12))) __kernel void adreno_xmem_attn_pv_gemm( + constant half8 * weights_buffer __attribute__((sub_group_uniform)), + constant half8 * xmem_buffer __attribute__((max_constant_size((6144)))), + read_only image1d_buffer_t src_tensor_image_buffer, + write_only image2d_t dst_tensor_image2d, + const int4 shared_int4_0, + const int4 shared_int4_1, + const int4 shared_int4_2, + const int4 shared_int4_3) { + int X = get_group_id(1) * get_local_size(0) + get_local_id(0); + int Y = get_group_id(2) * get_local_size(1) + get_local_id(1); + int Z = get_group_id(0) * get_local_size(2) + get_local_id(2); + if (X >= shared_int4_0.z || Y >= shared_int4_0.x) { + return; + } + if (Z * 8 >= shared_int4_0.y) { + return; + } + + half4 r0 = (half4) (0.f); + half4 r1 = (half4) (0.f); + half4 r2 = (half4) (0.f); + half4 r3 = (half4) (0.f); + half4 r4 = (half4) (0.f); + half4 r5 = (half4) (0.f); + half4 r6 = (half4) (0.f); + half4 r7 = (half4) (0.f); + int x_coord = mad24(X, shared_int4_2.w, shared_int4_1.y); + int y_coord = mad24(Y, shared_int4_3.x, shared_int4_1.z); + int coord_x, coord_y, coord_s; + int f_offset = (Z * shared_int4_1.w + Y) * shared_int4_1.x * 32; + + int subgroup_id = (int) ((0x1F & qcom_get_physical_sub_group_id())); + subgroup_id = subgroup_id % 12; + int c_offset = mul24(subgroup_id, shared_int4_0.w); + __constant half16 * weights_cache = (__constant half16 *) &xmem_buffer[c_offset]; + coord_y = Y; + coord_x = X; + int addr = (((0) * shared_int4_1.w + (coord_y)) * shared_int4_2.z + (coord_x)); + int dz = shared_int4_2.x; + coord_s = 0; + do { + half4 src0 = read_imageh(src_tensor_image_buffer, addr); + addr += dz; + coord_s++; + half4 src1 = read_imageh(src_tensor_image_buffer, addr); + addr += dz; + coord_s++; + qcom_sub_group_constant_load8(xmem_buffer, weights_buffer, c_offset, f_offset >> 1, 32); + f_offset += 64; + qcom_sub_group_sync(QCOM_CLK_CONST_LOAD_SYNC); + r0 += src0.x * weights_cache[0].s0123; + r0 += src0.y * weights_cache[0].s4567; + r0 += src0.z * weights_cache[0].s89ab; + r0 += src0.w * weights_cache[0].scdef; + r1 += src0.x * weights_cache[1].s0123; + r1 += src0.y * weights_cache[1].s4567; + r1 += src0.z * weights_cache[1].s89ab; + r1 += src0.w * weights_cache[1].scdef; + r2 += src0.x * weights_cache[2].s0123; + r2 += src0.y * weights_cache[2].s4567; + r2 += src0.z * weights_cache[2].s89ab; + r2 += src0.w * weights_cache[2].scdef; + r3 += src0.x * weights_cache[3].s0123; + r3 += src0.y * weights_cache[3].s4567; + r3 += src0.z * weights_cache[3].s89ab; + r3 += src0.w * weights_cache[3].scdef; + r4 += src0.x * weights_cache[4].s0123; + r4 += src0.y * weights_cache[4].s4567; + r4 += src0.z * weights_cache[4].s89ab; + r4 += src0.w * weights_cache[4].scdef; + r5 += src0.x * weights_cache[5].s0123; + r5 += src0.y * weights_cache[5].s4567; + r5 += src0.z * weights_cache[5].s89ab; + r5 += src0.w * weights_cache[5].scdef; + r6 += src0.x * weights_cache[6].s0123; + r6 += src0.y * weights_cache[6].s4567; + r6 += src0.z * weights_cache[6].s89ab; + r6 += src0.w * weights_cache[6].scdef; + r7 += src0.x * weights_cache[7].s0123; + r7 += src0.y * weights_cache[7].s4567; + r7 += src0.z * weights_cache[7].s89ab; + r7 += src0.w * weights_cache[7].scdef; + r0 += src1.x * weights_cache[8].s0123; + r0 += src1.y * weights_cache[8].s4567; + r0 += src1.z * weights_cache[8].s89ab; + r0 += src1.w * weights_cache[8].scdef; + r1 += src1.x * weights_cache[9].s0123; + r1 += src1.y * weights_cache[9].s4567; + r1 += src1.z * weights_cache[9].s89ab; + r1 += src1.w * weights_cache[9].scdef; + r2 += src1.x * weights_cache[10].s0123; + r2 += src1.y * weights_cache[10].s4567; + r2 += src1.z * weights_cache[10].s89ab; + r2 += src1.w * weights_cache[10].scdef; + r3 += src1.x * weights_cache[11].s0123; + r3 += src1.y * weights_cache[11].s4567; + r3 += src1.z * weights_cache[11].s89ab; + r3 += src1.w * weights_cache[11].scdef; + r4 += src1.x * weights_cache[12].s0123; + r4 += src1.y * weights_cache[12].s4567; + r4 += src1.z * weights_cache[12].s89ab; + r4 += src1.w * weights_cache[12].scdef; + r5 += src1.x * weights_cache[13].s0123; + r5 += src1.y * weights_cache[13].s4567; + r5 += src1.z * weights_cache[13].s89ab; + r5 += src1.w * weights_cache[13].scdef; + r6 += src1.x * weights_cache[14].s0123; + r6 += src1.y * weights_cache[14].s4567; + r6 += src1.z * weights_cache[14].s89ab; + r6 += src1.w * weights_cache[14].scdef; + r7 += src1.x * weights_cache[15].s0123; + r7 += src1.y * weights_cache[15].s4567; + r7 += src1.z * weights_cache[15].s89ab; + r7 += src1.w * weights_cache[15].scdef; + } while (coord_s < shared_int4_2.y); + + coord_s = mul24(Z, 8); + coord_x = X; + coord_y = Y; + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r0); + if (coord_s < 0) { + res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0)); + } + write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res); + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r1); + if (coord_s < 0) { + res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0)); + } + write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res); + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r2); + if (coord_s < 0) { + res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0)); + } + write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res); + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r3); + if (coord_s < 0) { + res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0)); + } + write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res); + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r4); + if (coord_s < 0) { + res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0)); + } + write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res); + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r5); + if (coord_s < 0) { + res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0)); + } + write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res); + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r6); + if (coord_s < 0) { + res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0)); + } + write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res); + coord_s++; + } + if (coord_s < shared_int4_0.y) { + half4 res = convert_half4(r7); + if (coord_s < 0) { + res += read_imageh(src_tensor_image_buffer, ((0) * shared_int4_1.w + (0)) * shared_int4_2.z + (0)); + } + write_imageh(dst_tensor_image2d, (int2) ((coord_x), ((coord_y) *shared_int4_0.y + (coord_s))), res); + coord_s++; + } +} diff --git a/ggml/src/ggml-opencl/kernels/set_rows.cl b/ggml/src/ggml-opencl/kernels/set_rows.cl index fc3ff7a..4ad5af1 100644 --- a/ggml/src/ggml-opencl/kernels/set_rows.cl +++ b/ggml/src/ggml-opencl/kernels/set_rows.cl @@ -158,6 +158,239 @@ kernel void kernel_set_rows_f32_i32( } } +// f32 -> q8_0 quantize set_rows. Block = half d + char qs[32]. +#define QK8_0 32 + +inline void quantize_q8_0_block(global float * x, global char * qs, global half * d_out) { + float amax = 0.0f; + for (int j = 0; j < QK8_0; j++) { + amax = fmax(amax, fabs(x[j])); + } + + float d = amax / 127.0f; + float id = (d != 0.0f) ? 127.0f / amax : 0.0f; + + vstore_half(d, 0, d_out); + + for (int j = 0; j < QK8_0; j++) { + qs[j] = (char)((int)round(x[j] * id)); + } +} + +kernel void kernel_set_rows_q8_0_i64( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3); + global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK8_0; + global char * y = dst_row + blk * (2 + QK8_0); + + quantize_q8_0_block(x, y + 2, (global half *)y); + } +} + +kernel void kernel_set_rows_q8_0_i32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3); + global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK8_0; + global char * y = dst_row + blk * (2 + QK8_0); + + quantize_q8_0_block(x, y + 2, (global half *)y); + } +} + +// SoA q8_0 variants. dst_q: int8[QK8_0] per block; dst_d: fp16 scale per block. +// Layout matches kernel_convert_block_q8_0; block index follows dst element order. +kernel void kernel_set_rows_q8_0_soa_i64( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst_q, + ulong offset_q, + global char * dst_d, + ulong offset_d, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + int ne1_dst, + int ne2_dst, + int ne3_dst +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst_q = dst_q + offset_q; + dst_d = dst_d + offset_d; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0; + + global half * d_row = (global half *)(dst_d) + row_blk_base; + global char * q_row = (global char *)(dst_q) + row_blk_base * QK8_0; + global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK8_0; + global char * q = q_row + blk * QK8_0; + + quantize_q8_0_block(x, q, d_row + blk); + } +} + +kernel void kernel_set_rows_q8_0_soa_i32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst_q, + ulong offset_q, + global char * dst_d, + ulong offset_d, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + int ne1_dst, + int ne2_dst, + int ne3_dst +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst_q = dst_q + offset_q; + dst_d = dst_d + offset_d; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0; + + global half * d_row = (global half *)(dst_d) + row_blk_base; + global char * q_row = (global char *)(dst_q) + row_blk_base * QK8_0; + global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK8_0; + global char * q = q_row + blk * QK8_0; + + quantize_q8_0_block(x, q, d_row + blk); + } +} + kernel void kernel_set_rows_f16_i32( global char * src0, ulong offset0, @@ -206,3 +439,270 @@ kernel void kernel_set_rows_f16_i32( dst_row[ind] = src_row[ind]; } } + +// f32 -> q4_0 quantize set_rows. Block = half d + uchar qs[16] (shuffled +// nibbles: qs[j] low/high = elem j / j+16). +// Dequant: val[i] = d * (nibble_i - 8) +// nblk0 = number of q4_0 blocks per row = ne00 / 32. +#define QK4_0 32 +#define Q4_0_BLOCK_SIZE 18 + +inline void quantize_q4_0_block(global float * x, global uchar * qs, global half * d_out) { + // Find the signed value with the largest absolute magnitude (matches ggml ref). + float max = 0.0f; + float amax = 0.0f; + for (int j = 0; j < QK4_0; j++) { + float v = x[j]; + float a = fabs(v); + if (a > amax) { + amax = a; + max = v; + } + } + + float d = max / -8.0f; + float id = (d != 0.0f) ? 1.0f / d : 0.0f; + + vstore_half(d, 0, d_out); + + for (int j = 0; j < QK4_0/2; j++) { + float x0 = x[j] * id; + float x1 = x[j + QK4_0/2] * id; + + int i0 = (int)(x0 + 8.5f); + int i1 = (int)(x1 + 8.5f); + if (i0 < 0) i0 = 0; + if (i0 > 15) i0 = 15; + if (i1 < 0) i1 = 0; + if (i1 > 15) i1 = 15; + + qs[j] = (uchar)i0 | ((uchar)i1 << 4); + } +} + +kernel void kernel_set_rows_q4_0_i64( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3); + global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK4_0; + global char * y = dst_row + blk * Q4_0_BLOCK_SIZE; + global half * yd = (global half *)(y); + global uchar * yqs = (global uchar *)(y + 2); + + quantize_q4_0_block(x, yqs, yd); + } +} + +kernel void kernel_set_rows_q4_0_i32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3); + global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK4_0; + global char * y = dst_row + blk * Q4_0_BLOCK_SIZE; + global half * yd = (global half *)(y); + global uchar * yqs = (global uchar *)(y + 2); + + quantize_q4_0_block(x, yqs, yd); + } +} + +// SoA variants for q4_0 dst. Used when the backend has split block_q4_0 records +// into separate quant (dst_q) and scale (dst_d) sub-buffers — same pattern as +// the q8_0 SoA variants above. +// +// Layout (matches kernel_convert_block_q4_0, the "shuffled" variant): +// dst_q: contiguous 16 packed nibbles per block, block i at offset i * 16 bytes. +// dst_d: contiguous fp16 scales, block i at offset i * 2 bytes. +// Nibble layout inside each byte is unchanged from AoS: qs[j] low nibble = element j, +// qs[j] high nibble = element j+16. kernel_restore_block_q4_0 copies bytes as-is. +kernel void kernel_set_rows_q4_0_soa_i64( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst_q, + ulong offset_q, + global char * dst_d, + ulong offset_d, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + int ne1_dst, + int ne2_dst, + int ne3_dst +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst_q = dst_q + offset_q; + dst_d = dst_d + offset_d; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0; + + global half * d_row = (global half *)(dst_d) + row_blk_base; + global uchar * q_row = (global uchar *)(dst_q) + row_blk_base * (QK4_0/2); + global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK4_0; + global uchar * qs = q_row + blk * (QK4_0/2); + global half * d_bk = d_row + blk; + + quantize_q4_0_block(x, qs, d_bk); + } +} + +kernel void kernel_set_rows_q4_0_soa_i32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst_q, + ulong offset_q, + global char * dst_d, + ulong offset_d, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + uint4 ne11, + uint4 ne12, + ulong nb10, + ulong nb11, + ulong nb12, + int nblk0, + int ne1_dst, + int ne2_dst, + int ne3_dst +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst_q = dst_q + offset_q; + dst_d = dst_d + offset_d; + + int i03 = get_group_id(2); + int i02 = get_group_id(1); + int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1); + + if (i01 >= ne01) { + return; + } + + int i12 = fastmod(i03, ne12); + int i11 = fastmod(i02, ne11); + + int i10 = i01; + int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0]; + + long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0; + + global half * d_row = (global half *)(dst_d) + row_blk_base; + global uchar * q_row = (global uchar *)(dst_q) + row_blk_base * (QK4_0/2); + global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03); + + for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) { + global float * x = src_row + blk * QK4_0; + global uchar * qs = q_row + blk * (QK4_0/2); + global half * d_bk = d_row + blk; + + quantize_q4_0_block(x, qs, d_bk); + } +} diff --git a/ggml/src/ggml-opencl/kernels/ssm_scan.cl b/ggml/src/ggml-opencl/kernels/ssm_scan.cl new file mode 100644 index 0000000..1889b74 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/ssm_scan.cl @@ -0,0 +1,346 @@ +// Mamba2 fused SSM scan kernel. One workgroup per (head, dim, seq); WG size = +// 64 threads. Each thread owns c_factor = d_state/64 state elements in +// private registers; the state stays resident across the n_tokens t-loop +// +// References: +// ggml/src/ggml-cuda/ssm-scan.cu:117 ssm_scan_f32_group +// ggml/src/ggml-cpu/ops.cpp:9368 ggml_compute_forward_ssm_scan_f32 + +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_khr_subgroups +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +#if defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#else +#define REQD_SUBGROUP_SIZE_64 +#endif + +inline float softplus_f32(float x) { + return (x <= 20.0f) ? log(1.0f + exp(x)) : x; +} + +// d_state = 128 (most Mamba-2 models, e.g. mamba2-2.7B, Codestral-Mamba). +// WG = 64 threads, each holds 2 state elements (tid and tid+64). +REQD_SUBGROUP_SIZE_64 +kernel void kernel_ssm_scan_f32_mamba2_d128( + global const char * src0_base, ulong src0_off, + global const char * src1_base, ulong src1_off, + global const char * src2_base, ulong src2_off, + global const char * src3_base, ulong src3_off, + global const char * src4_base, ulong src4_off, + global const char * src5_base, ulong src5_off, + global const char * src6_base, ulong src6_off, + global char * dst_base, ulong dst_off, + ulong s0_nb2, ulong s0_nb3, + ulong x_nb2, ulong x_nb3, + ulong dt_nb1, ulong dt_nb2, + ulong A_nb1, + ulong B_nb2, ulong B_nb3, + ulong C_nb2, ulong C_nb3, + ulong s_off_bytes, + int head_dim, int n_head, int n_group, int n_tokens +) { + const int d_state = 128; + + const int tid = (int) get_local_id(0); + const int wg_x = (int) get_group_id(0); + const int seq_id = (int) get_group_id(1); + + const int head_id = wg_x / head_dim; + const int dim_id = wg_x - head_id * head_dim; + const int g = head_id / (n_head / n_group); + + src0_base += src0_off; + src1_base += src1_off; + src2_base += src2_off; + src3_base += src3_off; + src4_base += src4_off; + src5_base += src5_off; + src6_base += src6_off; + dst_base += dst_off; + + const int seq_slot = ((global const int *) src6_base)[seq_id]; + + const ulong state_base_off = (ulong)seq_slot * s0_nb3 + (ulong)head_id * s0_nb2 + + (ulong)dim_id * d_state * sizeof(float); + global const float * s0_warp = (global const float *)(src0_base + state_base_off); + const ulong state_out_off = (ulong)seq_id * s0_nb3 + (ulong)head_id * s0_nb2 + + (ulong)dim_id * d_state * sizeof(float); + global float * s_warp = (global float *)(dst_base + s_off_bytes + state_out_off); + + global const char * x_seq = src1_base + (ulong)seq_id * x_nb3; + global const char * dt_seq = src2_base + (ulong)seq_id * dt_nb2; + global const char * B_seq = src4_base + (ulong)seq_id * B_nb3 + (ulong)g * d_state * sizeof(float); + global const char * C_seq = src5_base + (ulong)seq_id * C_nb3 + (ulong)g * d_state * sizeof(float); + + const ulong y_dim_total = (ulong)n_head * head_dim; + global float * y_seq = (global float *)dst_base + + (ulong)seq_id * (ulong)n_tokens * y_dim_total; + + const float A_val = ((global const float *)src3_base)[(ulong)head_id * A_nb1 / sizeof(float)]; + + // c_factor = 2: each thread owns 2 state elements (tid and tid+64). + float state0 = s0_warp[tid]; + float state1 = s0_warp[tid + 64]; + + for (int t = 0; t < n_tokens; ++t) { + const float dt_h = ((global const float *)(dt_seq + (ulong)t * dt_nb1))[head_id]; + const float dt_softplus = softplus_f32(dt_h); + const float dA = exp(dt_softplus * A_val); + const float x_val = ((global const float *)(x_seq + (ulong)t * x_nb2))[(ulong)head_id * head_dim + dim_id]; + const float x_dt = x_val * dt_softplus; + + const float B0 = ((global const float *)(B_seq + (ulong)t * B_nb2))[tid]; + const float B1 = ((global const float *)(B_seq + (ulong)t * B_nb2))[tid + 64]; + const float C0 = ((global const float *)(C_seq + (ulong)t * C_nb2))[tid]; + const float C1 = ((global const float *)(C_seq + (ulong)t * C_nb2))[tid + 64]; + + state0 = state0 * dA + B0 * x_dt; + state1 = state1 * dA + B1 * x_dt; + const float partial = state0 * C0 + state1 * C1; + + const float sum = sub_group_reduce_add(partial); + if (tid == 0) { + y_seq[(ulong)t * y_dim_total + (ulong)head_id * head_dim + dim_id] = sum; + } + } + + s_warp[tid] = state0; + s_warp[tid + 64] = state1; +} + +// d_state = 256 (Falcon-H1). WG = 64 threads, each holds 4 state elements. +REQD_SUBGROUP_SIZE_64 +kernel void kernel_ssm_scan_f32_mamba2_d256( + global const char * src0_base, ulong src0_off, + global const char * src1_base, ulong src1_off, + global const char * src2_base, ulong src2_off, + global const char * src3_base, ulong src3_off, + global const char * src4_base, ulong src4_off, + global const char * src5_base, ulong src5_off, + global const char * src6_base, ulong src6_off, + global char * dst_base, ulong dst_off, + ulong s0_nb2, ulong s0_nb3, + ulong x_nb2, ulong x_nb3, + ulong dt_nb1, ulong dt_nb2, + ulong A_nb1, + ulong B_nb2, ulong B_nb3, + ulong C_nb2, ulong C_nb3, + ulong s_off_bytes, + int head_dim, int n_head, int n_group, int n_tokens +) { + const int d_state = 256; + + const int tid = (int) get_local_id(0); + const int wg_x = (int) get_group_id(0); + const int seq_id = (int) get_group_id(1); + + const int head_id = wg_x / head_dim; + const int dim_id = wg_x - head_id * head_dim; + const int g = head_id / (n_head / n_group); + + src0_base += src0_off; + src1_base += src1_off; + src2_base += src2_off; + src3_base += src3_off; + src4_base += src4_off; + src5_base += src5_off; + src6_base += src6_off; + dst_base += dst_off; + + const int seq_slot = ((global const int *) src6_base)[seq_id]; + + const ulong state_base_off = (ulong)seq_slot * s0_nb3 + (ulong)head_id * s0_nb2 + + (ulong)dim_id * d_state * sizeof(float); + global const float * s0_warp = (global const float *)(src0_base + state_base_off); + const ulong state_out_off = (ulong)seq_id * s0_nb3 + (ulong)head_id * s0_nb2 + + (ulong)dim_id * d_state * sizeof(float); + global float * s_warp = (global float *)(dst_base + s_off_bytes + state_out_off); + + global const char * x_seq = src1_base + (ulong)seq_id * x_nb3; + global const char * dt_seq = src2_base + (ulong)seq_id * dt_nb2; + global const char * B_seq = src4_base + (ulong)seq_id * B_nb3 + (ulong)g * d_state * sizeof(float); + global const char * C_seq = src5_base + (ulong)seq_id * C_nb3 + (ulong)g * d_state * sizeof(float); + + const ulong y_dim_total = (ulong)n_head * head_dim; + global float * y_seq = (global float *)dst_base + + (ulong)seq_id * (ulong)n_tokens * y_dim_total; + + const float A_val = ((global const float *)src3_base)[(ulong)head_id * A_nb1 / sizeof(float)]; + + // c_factor = 4: each thread owns 4 state elements. + float state0 = s0_warp[tid]; + float state1 = s0_warp[tid + 64]; + float state2 = s0_warp[tid + 128]; + float state3 = s0_warp[tid + 192]; + + for (int t = 0; t < n_tokens; ++t) { + const float dt_h = ((global const float *)(dt_seq + (ulong)t * dt_nb1))[head_id]; + const float dt_softplus = softplus_f32(dt_h); + const float dA = exp(dt_softplus * A_val); + const float x_val = ((global const float *)(x_seq + (ulong)t * x_nb2))[(ulong)head_id * head_dim + dim_id]; + const float x_dt = x_val * dt_softplus; + + global const float * B_t = (global const float *)(B_seq + (ulong)t * B_nb2); + global const float * C_t = (global const float *)(C_seq + (ulong)t * C_nb2); + + const float B0 = B_t[tid]; + const float B1 = B_t[tid + 64]; + const float B2 = B_t[tid + 128]; + const float B3 = B_t[tid + 192]; + const float C0 = C_t[tid]; + const float C1 = C_t[tid + 64]; + const float C2 = C_t[tid + 128]; + const float C3 = C_t[tid + 192]; + + state0 = state0 * dA + B0 * x_dt; + state1 = state1 * dA + B1 * x_dt; + state2 = state2 * dA + B2 * x_dt; + state3 = state3 * dA + B3 * x_dt; + const float partial = state0 * C0 + state1 * C1 + state2 * C2 + state3 * C3; + + const float sum = sub_group_reduce_add(partial); + if (tid == 0) { + y_seq[(ulong)t * y_dim_total + (ulong)head_id * head_dim + dim_id] = sum; + } + } + + s_warp[tid] = state0; + s_warp[tid + 64] = state1; + s_warp[tid + 128] = state2; + s_warp[tid + 192] = state3; +} + +kernel void kernel_ssm_scan_f32( + global const char * s_buf, + ulong s_off, + global const char * x_buf, + ulong x_off, + global const char * dt_buf, + ulong dt_off, + global const char * A_buf, + ulong A_off, + global const char * B_buf, + ulong B_off, + global const char * C_buf, + ulong C_off, + global const char * ids_buf, + ulong ids_off, + global char * dst_buf, + ulong dst_off, + ulong s_nb2, + ulong s_nb3, + ulong x_nb2, + ulong x_nb3, + ulong dt_nb1, + ulong dt_nb2, + ulong A_nb1, + ulong B_nb2, + ulong B_nb3, + ulong C_nb2, + ulong C_nb3, + ulong state_off, + int head_dim, + int n_head, + int n_group, + int n_tokens, + ulong s_nb1, + ulong x_nb1, + ulong B_nb1, + ulong C_nb1, + uint A_ne0, + uint d_state, + uint n_seqs, + uint K, + local float * reduce +) { + global const char * s_data = s_buf + s_off; + global const char * x_data = x_buf + x_off; + global const char * dt_data = dt_buf + dt_off; + global const char * A_data = A_buf + A_off; + global const char * B_data = B_buf + B_off; + global const char * C_data = C_buf + C_off; + global const int * ids_data = (global const int *) (ids_buf + ids_off); + global float * dst = (global float *) (dst_buf + dst_off); + const uint y_elems = state_off / sizeof(float); + + const uint tid = get_local_id(0); + const uint inner_idx = get_group_id(0); + const uint seq_idx = get_group_id(1); + const uint head_idx = inner_idx / head_dim; + const uint dim_idx = inner_idx - head_idx * head_dim; + const uint group_idx = head_idx / (n_head / n_group); + const uint state_slot = (uint) ids_data[seq_idx]; + + const ulong s_idx = (ulong) state_slot * s_nb3 + + (ulong) head_idx * s_nb2 + + (ulong) dim_idx * s_nb1 + + (ulong) tid * sizeof(float); + float state = *((global const float *) (s_data + s_idx)); + + const ulong A_idx = (ulong) head_idx * A_nb1 + + (ulong) (tid % A_ne0) * sizeof(float); + const float A_value = *((global const float *) (A_data + A_idx)); + + for (int token_idx = 0; token_idx < n_tokens; ++token_idx) { + const ulong x_idx = (ulong) head_idx * x_nb1 + + (ulong) token_idx * x_nb2 + + (ulong) seq_idx * x_nb3 + + (ulong) dim_idx * sizeof(float); + const ulong dt_idx = (ulong) token_idx * dt_nb1 + + (ulong) seq_idx * dt_nb2 + + (ulong) head_idx * sizeof(float); + const ulong B_idx = (ulong) group_idx * B_nb1 + + (ulong) token_idx * B_nb2 + + (ulong) seq_idx * B_nb3 + + (ulong) tid * sizeof(float); + const ulong C_idx = (ulong) group_idx * C_nb1 + + (ulong) token_idx * C_nb2 + + (ulong) seq_idx * C_nb3 + + (ulong) tid * sizeof(float); + + const float x_value = *((global const float *) (x_data + x_idx)); + const float dt_value = *((global const float *) (dt_data + dt_idx)); + const float B_value = *((global const float *) (B_data + B_idx)); + const float C_value = *((global const float *) (C_data + C_idx)); + const float dt_soft_plus = dt_value > 20.0f ? dt_value : log(1.0f + exp(dt_value)); + const float dA = exp(dt_soft_plus * A_value); + const float x_dt = x_value * dt_soft_plus; + + state = mad(state, dA, B_value * x_dt); + reduce[tid] = state * C_value; + barrier(CLK_LOCAL_MEM_FENCE); + + for (uint stride = d_state / 2; stride > 0; stride >>= 1) { + if (tid < stride) { + reduce[tid] += reduce[tid + stride]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (tid == 0) { + const uint y_idx = dim_idx + head_idx * head_dim + + token_idx * n_head * head_dim + + seq_idx * n_tokens * n_head * head_dim; + dst[y_idx] = reduce[0]; + } + + const uint snapshot_slot = n_tokens - 1 - token_idx; + if (snapshot_slot > 0 && snapshot_slot < K) { + const uint snapshot_idx = y_elems + tid + dim_idx * d_state + + head_idx * d_state * head_dim + + (snapshot_slot * n_seqs + seq_idx) * d_state * head_dim * n_head; + dst[snapshot_idx] = state; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const uint state_idx = y_elems + tid + dim_idx * d_state + + head_idx * d_state * head_dim + + seq_idx * d_state * head_dim * n_head; + dst[state_idx] = state; +} diff --git a/ggml/src/ggml-opencl/kernels/top_k.cl b/ggml/src/ggml-opencl/kernels/top_k.cl new file mode 100644 index 0000000..6b5126e --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/top_k.cl @@ -0,0 +1,207 @@ +#ifdef cl_khr_subgroups +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#endif + +uint top_k_key(float value) { + uint bits = as_uint(value); + if ((bits & 0x7fffffffu) == 0) { + bits = 0; + } + return bits & 0x80000000u ? ~bits : bits ^ 0x80000000u; +} + +uint4 top_k_keys(global const char * src, int i, int n, ulong nb0) { + if (nb0 == sizeof(float) && i + 3 < n) { + const float4 v = vload4(0, (global const float *) src + i); + return (uint4) (top_k_key(v.s0), top_k_key(v.s1), top_k_key(v.s2), top_k_key(v.s3)); + } + return (uint4) ( + i < n ? top_k_key(*(global const float *) (src + (ulong) i *nb0)) : 0, + i + 1 < n ? top_k_key(*(global const float *) (src + (ulong) (i + 1)*nb0)) : 0, + i + 2 < n ? top_k_key(*(global const float *) (src + (ulong) (i + 2)*nb0)) : 0, + i + 3 < n ? top_k_key(*(global const float *) (src + (ulong) (i + 3)*nb0)) : 0); +} + +void top_k_store(global int * dst, global uint2 * candidates, global const char * src, + ulong nb0, int indexed, int base, int i, uint pos) { + const int index = indexed ? *(global const int *) (src + (ulong) i*nb0 + sizeof(float)) : base + i; + if (candidates) { + candidates[pos] = (uint2) (as_uint(*(global const float *) (src + (ulong) i*nb0)), index); + } else { + dst[pos] = index; + } +} + +void top_k_select( + global const char * src, + global int * dst, + global uint2 * candidates, + ulong nb00, + int ne00, + int k, + int base, + int indexed, + local uint * hist +) { + const int tid = get_local_id(0); + const int nth = get_local_size(0); + if (k == ne00) { + for (int i = tid; i < ne00; i += nth) { + top_k_store(dst, candidates, src, nb00, indexed, base, i, i); + } + return; + } + + uint prefix = 0; + uint mask = 0; + uint rank = k; + for (int shift = 24; shift >= 0; shift -= 8) { + for (int i = tid; i < 256; i += nth) { + hist[i] = 0; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int i = 4*tid; i < ne00; i += 4*nth) { + const uint4 v = top_k_keys(src, i, ne00, nb00); + const uint keys[4] = {v.s0, v.s1, v.s2, v.s3}; + #pragma unroll + for (int j = 0; j < 4; ++j) { + const uint key = keys[j]; + if (i + j < ne00 && (key & mask) == prefix) { + atomic_inc(hist + ((key >> shift) & 255)); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if (tid == 0) { + uint above = 0; + for (int b = 255; b >= 0; --b) { + uint count = hist[b]; + if (above + count >= rank) { + hist[256] = b; + hist[257] = rank - above; + hist[258] = count; + break; + } + above += count; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + prefix |= hist[256] << shift; + rank = hist[257]; + const bool done = hist[258] == rank; + barrier(CLK_LOCAL_MEM_FENCE); + if (done) { + // The whole bucket is selected, so its low bits do not matter. + break; + } + mask |= 255u << shift; + } + + uint greater = 0; + uint equal = 0; + for (int i = 4*tid; i < ne00; i += 4*nth) { + const uint4 v = top_k_keys(src, i, ne00, nb00); + const uint keys[4] = {v.s0, v.s1, v.s2, v.s3}; + #pragma unroll + for (int j = 0; j < 4; ++j) { + greater += i + j < ne00 && keys[j] > prefix; + equal += i + j < ne00 && keys[j] == prefix; + } + } + uint pos_greater; + uint pos_equal; +#ifdef cl_khr_subgroups + const uint sg = get_sub_group_id(); + const uint nsg = get_num_sub_groups(); + const uint total_greater = sub_group_reduce_add(greater); + const uint total_equal = sub_group_reduce_add(equal); + pos_greater = sub_group_scan_exclusive_add(greater); + pos_equal = sub_group_scan_exclusive_add(equal); + if (get_sub_group_local_id() == 0) { + hist[sg] = total_greater; + hist[nsg + sg] = total_equal; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (uint i = 0; i < nsg; ++i) { + pos_equal += hist[i]; + if (i < sg) { + pos_greater += hist[i]; + pos_equal += hist[nsg + i]; + } + } +#else + hist[tid] = greater; + hist[nth + tid] = equal; + barrier(CLK_LOCAL_MEM_FENCE); + pos_greater = 0; + pos_equal = 0; + for (int i = 0; i < nth; ++i) { + pos_equal += hist[i]; + if (i < tid) { + pos_greater += hist[i]; + pos_equal += hist[nth + i]; + } + } +#endif + for (int i = 4*tid; i < ne00; i += 4*nth) { + const uint4 v = top_k_keys(src, i, ne00, nb00); + const uint keys[4] = {v.s0, v.s1, v.s2, v.s3}; + #pragma unroll + for (int j = 0; j < 4; ++j) { + if (i + j < ne00) { + if (keys[j] > prefix) { + top_k_store(dst, candidates, src, nb00, indexed, base, i + j, pos_greater++); + } else if (keys[j] == prefix && pos_equal < k) { + top_k_store(dst, candidates, src, nb00, indexed, base, i + j, pos_equal++); + } + } + } + } +} + +kernel void kernel_top_k_f32( + global const char * src, + ulong offset0, + global int * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int k_max, + local uint * hist, + global uint2 * candidates, + int tile_width +) { + const ulong row = get_group_id(1); + const int part = get_group_id(0); + const int parts = get_num_groups(0); + const int base = part*tile_width; + const int n = min(ne00 - base, tile_width); + const int k = min(k_max, n); + const int out_width = (parts - 1)*k_max + min(k_max, ne00 - (parts - 1)*tile_width); + src += offset0 + (row % ne01)*nb01 + (row / ne01 % ne02)*nb02 + (row / ne01 / ne02)*nb03 + (ulong) base*nb00; + if (candidates) { + candidates += row*out_width + part*k_max; + } else { + dst = (global int *) ((global char *) dst + offsetd) + row*out_width + part*k_max; + } + top_k_select(src, dst, candidates, nb00, n, k, base, 0, hist); +} + +kernel void kernel_top_k_merge_f32( + global const uint2 * candidates, + global int * dst, + ulong offsetd, + int n_candidates, + int k, + local uint * hist +) { + const ulong row = get_group_id(1); + candidates += row*n_candidates; + dst = (global int *) ((global char *) dst + offsetd) + row*k; + top_k_select((global const char *) candidates, dst, 0, sizeof(uint2), n_candidates, k, 0, 1, hist); +} diff --git a/ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl b/ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl new file mode 100644 index 0000000..34d54b5 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl @@ -0,0 +1,35 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#define tile_size 32 + +kernel void kernel_transpose_activation_q8( + __global uint * src1_qa, + __global uint * src1_qa_trans, + __global half * src1_sa, + __global float * src1_sa_trans, + __global const int * total_tiles, + uint K +) { + uint row_in_tile = get_global_id(0); + uint blk_k = get_group_id(1); + uint col_in_tile = get_local_id(1); + uint tile_n = get_global_id(2); + + if (blk_k >= (K / tile_size) || tile_n >= (uint)total_tiles[0]) { + return; + } + + uint src_idx = (tile_n * tile_size + row_in_tile) * K / 4 + blk_k * 8 + col_in_tile; + uint dst_idx = tile_n * tile_size * K / 4 + (blk_k * 8 + col_in_tile) * tile_size + row_in_tile; + + src1_qa_trans[dst_idx] = src1_qa[src_idx]; + + if (col_in_tile == 0) { + src_idx = (tile_n * tile_size + row_in_tile) * K / 32 + blk_k; + dst_idx = tile_n * tile_size * K / 32 + blk_k * tile_size + row_in_tile; + + src1_sa_trans[dst_idx] = (float)(src1_sa[src_idx]); + } + +} diff --git a/ggml/src/ggml-opencl/kernels/unary_ext.cl b/ggml/src/ggml-opencl/kernels/unary_ext.cl new file mode 100644 index 0000000..e86eadf --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/unary_ext.cl @@ -0,0 +1,85 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +//------------------------------------------------------------------------------ +// Extended elementwise unary ops, same variant shape as abs.cl: +// f32, f32_4 (vec4), f16, f16_4 (vec4), f32_nc, f16_nc (stride-addressed). +// +// sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc. +// +// Semantics match the ggml CPU reference (ggml.c). Values are computed in float +// (the f16 variants read/write half and convert), so the conditional ops match +// the CPU bit-for-bit within tolerance. SEXPR is the scalar form, VEXPR the +// float4 form (vector ternaries need select()). +//------------------------------------------------------------------------------ + +#define UNARY_EXT(NAME, SEXPR, VEXPR) \ +kernel void kernel_##NAME##_f32( \ + global const float * src0, ulong offset0, \ + global float * dst, ulong offsetd) { \ + src0 = (global float*)((global char*)src0 + offset0); \ + dst = (global float*)((global char*)dst + offsetd); \ + float x = src0[get_global_id(0)]; \ + dst[get_global_id(0)] = (SEXPR); \ +} \ +kernel void kernel_##NAME##_f32_4( \ + global const float4 * src0, ulong offset0, \ + global float4 * dst, ulong offsetd) { \ + src0 = (global float4*)((global char*)src0 + offset0); \ + dst = (global float4*)((global char*)dst + offsetd); \ + float4 x = src0[get_global_id(0)]; \ + dst[get_global_id(0)] = (VEXPR); \ +} \ +kernel void kernel_##NAME##_f16( \ + global const half * src0, ulong offset0, \ + global half * dst, ulong offsetd) { \ + src0 = (global half*)((global char*)src0 + offset0); \ + dst = (global half*)((global char*)dst + offsetd); \ + float x = src0[get_global_id(0)]; \ + dst[get_global_id(0)] = (SEXPR); \ +} \ +kernel void kernel_##NAME##_f16_4( \ + global const half4 * src0, ulong offset0, \ + global half4 * dst, ulong offsetd) { \ + src0 = (global half4*)((global char*)src0 + offset0); \ + dst = (global half4*)((global char*)dst + offsetd); \ + float4 x = convert_float4(src0[get_global_id(0)]); \ + dst[get_global_id(0)] = convert_half4(VEXPR); \ +} \ +kernel void kernel_##NAME##_f32_nc( \ + global const char * src0, ulong offset0, \ + global char * dst, ulong offsetd, \ + int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \ + ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \ + src0 = src0 + offset0; dst = dst + offsetd; \ + const int i3 = get_group_id(2); \ + const int i2 = get_group_id(1); \ + const int i1 = get_group_id(0); \ + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { \ + float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \ + *(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \ + } \ +} \ +kernel void kernel_##NAME##_f16_nc( \ + global const char * src0, ulong offset0, \ + global char * dst, ulong offsetd, \ + int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \ + ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \ + src0 = src0 + offset0; dst = dst + offsetd; \ + const int i3 = get_group_id(2); \ + const int i2 = get_group_id(1); \ + const int i1 = get_group_id(0); \ + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\ + float x = *(global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \ + *(global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \ + } \ +} + +UNARY_EXT(sgn, sign(x), sign(x)) +UNARY_EXT(step, x > 0.0f ? 1.0f : 0.0f, select((float4)0.0f, (float4)1.0f, x > 0.0f)) +UNARY_EXT(elu, x > 0.0f ? x : expm1(x), select(expm1(x), x, x > 0.0f)) +UNARY_EXT(hardswish, x * fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), x * fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f))) +UNARY_EXT(hardsigmoid, fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f))) +UNARY_EXT(floor, floor(x), floor(x)) +UNARY_EXT(ceil, ceil(x), ceil(x)) +UNARY_EXT(round, round(x), round(x)) +UNARY_EXT(trunc, trunc(x), trunc(x)) diff --git a/ggml/src/ggml-opencl/libdl.h b/ggml/src/ggml-opencl/libdl.h new file mode 100644 index 0000000..8ca5016 --- /dev/null +++ b/ggml/src/ggml-opencl/libdl.h @@ -0,0 +1,79 @@ +#pragma once + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +# include +#else +# include +# include +#endif +#include + +namespace fs = std::filesystem; + +#ifdef _WIN32 + +using dl_handle = std::remove_pointer_t; + +struct dl_handle_deleter { + void operator()(HMODULE handle) { + FreeLibrary(handle); + } +}; + +static inline dl_handle * dl_load_library(const fs::path & path) { + // suppress error dialogs for missing DLLs + DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); + SetErrorMode(old_mode | SEM_FAILCRITICALERRORS); + + HMODULE handle = LoadLibraryW(path.wstring().c_str()); + + SetErrorMode(old_mode); + + return handle; +} + +static inline void * dl_get_sym(dl_handle * handle, const char * name) { + DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); + SetErrorMode(old_mode | SEM_FAILCRITICALERRORS); + + void * p = (void *) GetProcAddress(handle, name); + + SetErrorMode(old_mode); + + return p; +} + +static inline const char * dl_error() { + return ""; +} + +#else + +using dl_handle = void; + +struct dl_handle_deleter { + void operator()(void * handle) { + dlclose(handle); + } +}; + +static inline dl_handle * dl_load_library(const fs::path & path) { + dl_handle * handle = dlopen(path.string().c_str(), RTLD_NOW | RTLD_LOCAL); + return handle; +} + +static inline void * dl_get_sym(dl_handle * handle, const char * name) { + return dlsym(handle, name); +} + +static inline const char * dl_error() { + const char *rslt = dlerror(); + return rslt != nullptr ? rslt : ""; +} + +#endif diff --git a/ggml/src/ggml-openvino/.clang-format b/ggml/src/ggml-openvino/.clang-format index a2a24d7..4a5c7c2 100644 --- a/ggml/src/ggml-openvino/.clang-format +++ b/ggml/src/ggml-openvino/.clang-format @@ -2,12 +2,7 @@ # Override root .clang-format AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false -Cpp11BracedListStyle: true -SpacesInContainerLiterals: false -BreakBeforeBraces: Attach AccessModifierOffset: -4 -IndentCaseBlocks: false -IndentCaseLabels: false Language: Cpp AlignAfterOpenBracket: Align diff --git a/ggml/src/ggml-openvino/CMakeLists.txt b/ggml/src/ggml-openvino/CMakeLists.txt index 175b585..347c24f 100644 --- a/ggml/src/ggml-openvino/CMakeLists.txt +++ b/ggml/src/ggml-openvino/CMakeLists.txt @@ -1,7 +1,7 @@ -find_package(OpenVINO REQUIRED) +find_package(OpenVINO REQUIRED COMPONENTS Runtime Threading) find_package(OpenCL REQUIRED) -include("${OpenVINO_DIR}/../3rdparty/tbb/lib/cmake/TBB/TBBConfig.cmake") +message(STATUS "Found OpenVINO: ${OpenVINO_DIR} (found version \"${OpenVINO_VERSION}\")") file(GLOB_RECURSE GGML_HEADERS_OPENVINO "*.h" "*.hpp") file(GLOB_RECURSE GGML_SOURCES_OPENVINO "*.cpp") @@ -11,7 +11,18 @@ ggml_add_backend_library(ggml-openvino ${GGML_HEADERS_OPENVINO} ) -target_link_libraries(ggml-openvino PRIVATE openvino::runtime TBB::tbb OpenCL::OpenCL) +target_link_libraries(ggml-openvino PRIVATE openvino::runtime openvino::threading OpenCL::OpenCL) + +# the OpenVINO RTTI macros take one argument and leave __VA_ARGS__ empty, which -Wpedantic reports +if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") + target_compile_options(ggml-openvino PRIVATE -Wno-gnu-zero-variadic-macro-arguments) +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_compile_options(ggml-openvino PRIVATE -Wno-pedantic) +endif() + +if (WIN32) + target_link_libraries(ggml-openvino PRIVATE psapi) +endif() if (GGML_OPENVINO) if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") diff --git a/ggml/src/ggml-openvino/README.md b/ggml/src/ggml-openvino/README.md new file mode 100644 index 0000000..6a23f62 --- /dev/null +++ b/ggml/src/ggml-openvino/README.md @@ -0,0 +1,44 @@ +# Compiled model cache + +`GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` exports compiled CPU/GPU graphs with their weights. It bypasses the plugin-level `GGML_OPENVINO_CACHE_DIR` and uses `OPTIMIZE_SPEED`, so weightless caching is disabled. + +One directory can hold blobs for different models and compilation settings. Run each intended workload once to export its dynamic graph: + +```sh +GGML_OPENVINO_DEVICE=GPU \ +GGML_OPENVINO_NATIVE_SOFTPLUS=1 \ +GGML_OPENVINO_DISABLE_KV_SLICE=1 \ +GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all \ +GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/path/to/qwen-cache \ +./build/ReleaseOV/bin/llama-bench -m /path/to/model.gguf -r 1 +``` + +`GGML_OPENVINO_SPILL_DIR` remains optional for this first run. Wait for the `model cache WROTE` message and completion of the workload before stopping it. Compatible prefill and decode graphs share one blob and manifest. A graph with different ports or incompatible shapes gets an exact entry instead; interrupted exports are not cache hits. + +On later runs, supply the same compilation settings and enable `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1`: + +```sh +GGML_OPENVINO_DEVICE=GPU \ +GGML_OPENVINO_NATIVE_SOFTPLUS=1 \ +GGML_OPENVINO_DISABLE_KV_SLICE=1 \ +GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all \ +GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/path/to/qwen-cache \ +GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \ +./build/ReleaseOV/bin/llama-bench -m /path/to/model.gguf -r 1 +``` + +Cache-only mode allocates backend address space without filling weight pages. On Windows, this also uses system commit capacity. The model-buffer size in the loader log is this virtual size. Weight uploads only record source identity; they do not read or requantize the weights. Graph conversion and compilation are skipped. Runtime buffers are still allocated and populated normally. + +Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. + +A missing or incompatible graph fails with an error instead of compiling with absent weights. The fingerprint uses the dynamic graph's topology, ports, model parameters, weights, settings, and OpenVINO version; changing only dynamic token or KV sizes does not require a new entry. A different workload can still require another graph; populate it first without cache-only mode. + +## Restrictions + +- Cache-only mode requires Linux or Windows and mmap loading (`--load-mode mmap`, or the default when all selected devices support mmap). Do not use tensor validation or mlock when trying to avoid weight reads. +- On Windows, the backend commits virtual memory for its buffers without touching weight pages. Large models can still reach the system commit limit. +- The model must execute entirely on OpenVINO, with dynamic CPU/GPU graphs and in-process caching enabled. Static/NPU execution and CPU fallback are unsupported in cache-only mode. +- GGUF metadata, tokenizer data, tensor descriptors, and graph construction are still needed. The llama.cpp loader is unchanged: depending on its prefetch settings, it may request pages with `MAP_POPULATE` or read-ahead on Linux, or `PrefetchVirtualMemory` on Windows. Non-mmap loading also reads the payload before the backend sees it. +- File identity, size, modification/change timestamps, tensor offsets, graph structure, settings, and OpenVINO version identify cache entries. Linux uses device/inode and Windows uses volume serial/file index. Replacing, copying, or modifying a GGUF invalidates its entries. This avoids reading weight bytes and ties the cache to the local source files. Keep those files unchanged throughout loading and inference. +- Use the same target device and compatible OpenVINO/plugin installation. Import support depends on the plugin; the tested CPU plugin cannot import MoE graphs containing `GatherMatmulCompressed`. GPU MoE and CPU dense graph imports were tested. +- Blobs contain weights and can approach model size for each compiled graph. Import still reads those blobs and initializes the device. diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 0938d22..10e82f3 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1,20 +1,17 @@ #include "ggml-decoder.h" -#include "ggml-backend-impl.h" -#include "ggml-backend.h" +#include "ggml-impl.h" #include "ggml-openvino-extra.h" #include "ggml-openvino.h" #include "ggml-quants.h" - -#include -#include +#include "ggml.h" +#include "utils.h" #include #include #include #include #include -#include #include #include #include @@ -29,13 +26,12 @@ #include #include #include -#include #include -#include #include #include #include #include +#include #include #include @@ -45,6 +41,7 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, std::map> & model_weights, bool is_static, bool is_stateful, + bool model_is_splitted, bool is_prefill, int prefill_chunk_size) : m_is_static(is_static), @@ -52,22 +49,23 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, m_is_prefill(is_prefill), m_naive(false), m_prefill_chunk_size(prefill_chunk_size), + m_model_is_splitted(model_is_splitted), m_cgraph(cgraph), m_model_weights(model_weights), m_model_params(model_params), m_compute_params(compute_params) { - if (auto * env = getenv("GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS"); env && std::string(env) != "0") { -#ifdef _WIN32 - _putenv_s("GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS", ""); -#else - unsetenv("GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS"); -#endif - print_tensor_address_map(cgraph); + static bool printed_address_map = false; + if (!printed_address_map) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS")) { + printed_address_map = true; + print_tensor_address_map(cgraph); + } } validate_cgraph(); set_input_output(); + compute_node_dynamic_dims(); compute_model_inputs(); compute_model_outputs(); @@ -89,6 +87,14 @@ void GgmlOvDecoder::update_io(ggml_cgraph * cgraph) { compute_model_outputs(); } +// llama keeps separate graphs for batches with and without outputs, so a cache hit can come from a +// graph built in other memory. The decoder then still points at the old graph's tensors. +bool GgmlOvDecoder::is_bound_to(const ggml_cgraph * cgraph) const { + return m_cgraph == cgraph && cgraph->n_nodes > 0 && m_node_info_list.size() == (size_t) cgraph->n_nodes && + m_node_info_list.front().node == cgraph->nodes[0] && + m_node_info_list.back().node == cgraph->nodes[cgraph->n_nodes - 1]; +} + GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, std::map> & model_weights) { m_cgraph = cgraph; m_model_weights = model_weights; @@ -102,27 +108,139 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, std::mapop == GGML_OP_SET_ROWS || node->op == GGML_OP_CPY || (node->op == GGML_OP_SCALE && node->view_src); +} + +bool is_same_shape(const ggml_tensor * a, const ggml_tensor * b) { + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (a->ne[i] != b->ne[i]) { + return false; + } + } + return true; +} + +bool is_conv_states_all_tensor(const ggml_tensor * tensor) { + return tensor != nullptr && strncmp(tensor->name, "conv_states_all", strlen("conv_states_all")) == 0; +} + +bool is_full_single_slot_writeback(const ggml_tensor * node) { + return node->view_src != nullptr && node->view_src->ne[1] == 1 && node->src[1] != nullptr && + node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src && + node->src[1]->view_offs == 0 && ggml_nbytes(node->src[1]) == ggml_nbytes(node->view_src); +} +} // namespace + +// MoE expert aggregation (build_moe_ffn in llama-graph.cpp): each expert plane is +// `ggml_view_2d(experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1])` and the planes +// are summed with a chain of ADDs: moe_out = ((view_0 + view_1) + view_2) + ... + view_{n-1}. +// Detected structurally by walking the ADD chain and checking every leaf is a same-shape, +// same-stride VIEW of one common base tensor, indexed by a distinct expert-plane offset, and +// that the chain covers every plane of that base (leaf count == base->ne[1]). Only the +// outermost ADD of the chain satisfies this (inner ADDs see fewer leaves than base->ne[1]). +bool is_moe_expert_sum_add(const ggml_tensor * node) { + std::vector leaves; + const ggml_tensor * cur = node; + while (cur->op == GGML_OP_ADD) { + if (cur->src[0] == nullptr || cur->src[1] == nullptr) { + return false; + } + leaves.push_back(cur->src[1]); + cur = cur->src[0]; + } + leaves.push_back(cur); + + const ggml_tensor * base = nullptr; + std::set plane_indices; + for (const ggml_tensor * leaf : leaves) { + if (leaf->op != GGML_OP_VIEW || leaf->src[0] == nullptr) { + return false; + } + const ggml_tensor * leaf_base = leaf->src[0]; + if (base == nullptr) { + base = leaf_base; + } else if (leaf_base != base) { + return false; + } + if (leaf->ne[0] != base->ne[0] || leaf->ne[1] != base->ne[2] || leaf->ne[2] != 1 || leaf->ne[3] != 1 || + leaf->nb[1] != base->nb[2]) { + return false; + } + if (base->nb[1] == 0 || leaf->view_offs % base->nb[1] != 0) { + return false; + } + int64_t plane = static_cast(leaf->view_offs / base->nb[1]); + if (plane < 0 || plane >= base->ne[1] || !plane_indices.insert(plane).second) { + return false; + } + } + + return base != nullptr && base->ne[1] > 1 && plane_indices.size() == static_cast(base->ne[1]); +} + +std::string GgmlOvDecoder::get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { + if (tensor == nullptr) { + return ""; + } + if ((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || is_kvcache(tensor, nullptr)) { + // Hash-table slots depend on tensor addresses and differ between contexts. + // Graph ordinals disambiguate duplicate names while keeping compiled-model + // ports identical for equivalent graphs in different contexts. + const auto * node = std::find(cgraph->nodes, cgraph->nodes + cgraph->n_nodes, tensor); + if (node != cgraph->nodes + cgraph->n_nodes) { + return std::string(tensor->name) + "#n" + std::to_string(node - cgraph->nodes); + } + const auto * leaf = std::find(cgraph->leafs, cgraph->leafs + cgraph->n_leafs, tensor); + if (leaf != cgraph->leafs + cgraph->n_leafs) { + return std::string(tensor->name) + "#l" + std::to_string(leaf - cgraph->leafs); + } + } + return tensor->name; +} + +static std::string get_tensor_ov_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { + return GgmlOvDecoder::get_tensor_name(cgraph, tensor); +} + +static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder, + const ggml_cgraph * cgraph, + const ggml_tensor * tensor, + const ggml_tensor * op) { + if (GgmlOvDecoder::is_inp_pos(tensor, op)) { + return "inp_pos"; + } + if (GgmlOvDecoder::is_inp_emb(tensor, op)) { + return "embd"; + } + if (GgmlOvDecoder::is_inp_mask(tensor, op)) { + // Give the two attention masks distinct OV parameter names. build_attn_inp_kq_mask() + // names the full-attention mask and the sliding-window mask identically, so keying a + // parameter off the name alone makes the second mask overwrite the first and both + // attention types read one parameter. Tell them apart by tensor identity, using the + // SWA classification computed in compute_llm_params(). An empty swa_layers set means + // there is only one mask in play and the plain name is correct. + const bool is_swa = decoder->is_swa_mask(tensor); + if (decoder->is_stateful()) { + return is_swa ? "self_kq_mask_swa" : "self_kq_mask"; + } + if (is_swa) { + return get_tensor_ov_name(cgraph, tensor) + "_swa"; + } + } + return get_tensor_ov_name(cgraph, tensor); +} + void GgmlOvDecoder::set_input_output() { for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) { - auto node = m_cgraph->nodes[node_n]; + auto * node = m_cgraph->nodes[node_n]; NodeInfo current_node_info; - auto node_name = std::string(node->name); - auto node_output_name = node_name; - auto * node_output = node; - if (node->op == GGML_OP_SET_ROWS) { - // SET_ROWS updates the tensor in place. For later ov op that uses the - // the view_src of SET_ROWS, we need to make sure they get the updated tensor - // by putting the view_src name in the tensor_map in - // /src/frontends/ggml/src/translate_session.cpp - node_output_name = std::string(node->view_src->name); - node_output = node->view_src; - } + auto node_name = get_tensor_ov_name(m_cgraph, node); current_node_info.node = node; current_node_info.node_name = node_name; - current_node_info.node_output = node_output; - current_node_info.node_output_name = node_output_name; current_node_info.node_op_case = 0; current_node_info.data_addr = node->data; @@ -131,12 +249,35 @@ void GgmlOvDecoder::set_input_output() { if (src == nullptr) { continue; } - auto src_name = std::string(src->name); + auto src_name = get_tensor_ov_name(m_cgraph, src); if (src->flags & GGML_TENSOR_FLAG_INPUT) { - src_name = get_graph_input_ov_name(src, node); + src_name = get_tensor_graph_input_ov_name(this, m_cgraph, src, node); } current_node_info.node_inputs[src_name] = src; current_node_info.node_inputs_names.push_back(src_name); + + if (src->op == GGML_OP_VIEW) { + // Traverse upward through nested VIEW operations + std::remove_reference_t view_chain; + auto * current = src; + + while (current != nullptr) { + auto current_name = get_tensor_ov_name(m_cgraph, current); + if (current->flags & GGML_TENSOR_FLAG_INPUT) { + current_name = get_tensor_graph_input_ov_name(this, m_cgraph, current, node); + } + view_chain.emplace_back(current_name, current); + // If current src is also a VIEW, continue traversing + if (current->src[0] != nullptr && current->src[0]->op == GGML_OP_VIEW) { + current = current->src[0]; + } else { + break; + } + } + + // Assign all collected view inputs to node_inputs_views + current_node_info.node_inputs_views[src_name] = view_chain; + } } m_node_info_list.push_back(current_node_info); @@ -147,7 +288,34 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { int op_case = 0; switch (node->op) { case GGML_OP_RESHAPE: { + if (m_naive) { + break; + } + auto name = std::string(node->name); auto * src = node->src[0]; + // Identify recurrent sequence reshapes before size checks, which are ambiguous for one token. + bool recurrent_sequence = false; + for (int i = 0; i < m_cgraph->n_nodes && !recurrent_sequence; ++i) { + const auto * consumer = m_cgraph->nodes[i]; + if (consumer->op == GGML_OP_MUL_MAT_ID && consumer->src[1] == node) { + return 1; + } else if (consumer->op == GGML_OP_SSM_CONV) { + const auto * concat = consumer->src[0]; + if (concat->op == GGML_OP_CONCAT) { + const auto * transposed = concat->src[1]; + recurrent_sequence = transposed->op == GGML_OP_TRANSPOSE && transposed->src[0] == node; + } + } else if (consumer->op == GGML_OP_UNARY && ggml_get_unary_op(consumer) == GGML_UNARY_OP_SOFTPLUS) { + const auto * biased = consumer->src[0]; + recurrent_sequence = biased->op == GGML_OP_ADD && biased->src[0] == node; + } + } + if (recurrent_sequence && node->ne[0] == src->ne[0] && node->ne[3] == 1) { + return 6; + } + if (node->ne[0] == src->ne[0] && node->ne[2] == 1 && node->ne[3] == 1) { + return 5; + } if (src->op == GGML_OP_RESHAPE && src->src[0]->ne[0] == node->ne[0] && src->src[0]->ne[1] == node->ne[1]) { op_case = 4; } else if (node->ne[0] * node->ne[1] == src->ne[0]) { @@ -157,20 +325,48 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { if (src->ne[2] * src->ne[3] == node->ne[1]) { op_case = 5; } - } else if (src->ne[0] * src->ne[1] == node->ne[1]) { + } else if (node->ne[0] == 1 && src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { op_case = 3; - } else if (src->ne[1] * src->ne[2] == node->ne[1]) { + } else if (name.find("linear_attn_qkv_mixed") == 0 || name.find("alpha") == 0) { op_case = 6; + } else if (name.find("linear_attn_out") == 0) { + op_case = 7; + } else if (name.find("state_predelta") == 0) { + op_case = 8; } - break; - } - case GGML_OP_CONT: { - if (node->src[0]->op == GGML_OP_PERMUTE) { - op_case = 1; - } else if (node->src[0]->op == GGML_OP_TRANSPOSE) { - op_case = 2; - } else if (node->src[0]->op == GGML_OP_VIEW) { - op_case = 3; + if (op_case == 1 && m_is_stateful) { + // Recurrent convolution and GDN gates retain their rank-4 layout. + // The gathered states may come through a view of the one gather of all states (see build_rs). + const auto * gather = src->op == GGML_OP_VIEW ? src->src[0] : src; + bool recurrent = gather->op == GGML_OP_GET_ROWS && is_recurrent_cache(gather->src[0]); + for (int i = 0; i < m_cgraph->n_nodes && !recurrent; ++i) { + const auto * consumer = m_cgraph->nodes[i]; + if (consumer->op == GGML_OP_GATED_DELTA_NET) { + for (int j : {3, 4}) { + const auto * gate = consumer->src[j]; + if (gate->op == GGML_OP_UNARY) { + gate = gate->src[0]; + } + recurrent = recurrent || gate == node; + } + } else if (consumer->op == GGML_OP_MUL) { + for (int j = 0; j < 2; ++j) { + const auto * gate = consumer->src[j]; + const auto * norm = consumer->src[1 - j]; + if (gate->op != GGML_OP_UNARY || gate->src[0] != node) { + continue; + } + if (norm->op == GGML_OP_MUL) { + norm = norm->src[0]; + } + recurrent = recurrent || (norm->op == GGML_OP_RMS_NORM && norm->src[0]->op == GGML_OP_VIEW && + norm->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET); + } + } + } + if (recurrent) { + op_case = 9; + } } break; } @@ -180,72 +376,295 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { } else if (node->src[0]->src[0]->op == GGML_OP_NONE) { // kv cache tensor std::string src_name(node->view_src->name); - int layer = extract_layer_from_name(src_name); - if (!is_swa_layer(layer)) { - op_case = 2; + int layer = extract_layer_from_name(src_name).value(); + if (ggml_is_contiguous(node->src[0])) { + // - 19: [ 64, 8, 256, 1] VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576] + // [ 512, 1024, 1, 1] 0: NONE cache_k_l0 [ 2, 1024, 1048576, 1048576] + // - 20: [ 64, 256, 8, 1] PERMUTE cache_k_l0 (view) (permuted) [ 2, 1024, 128, 1048576] + // [ 64, 8, 256, 1] 0: VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576] + if (!is_swa_layer(layer)) { + op_case = 3; + } else { + op_case = 4; + } } else { - op_case = 3; + // special case of cache v when `-fa off` + // - 17: [ 256, 8, 64, 1] VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576] + // [ 512, 1024, 1, 1] 0: NONE cache_v_l0 [ 2, 1024, 1048576, 1048576] + // - 18: [ 256, 64, 8, 1] PERMUTE cache_v_l0 (view) (permuted) [ 2, 2048, 131072, 1048576] + // [ 256, 8, 64, 1] 0: VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576] + if (!is_swa_layer(layer)) { + op_case = 5; + } else { + op_case = 6; + } } } else { // rope'ed query tensor - op_case = 4; + op_case = 2; } break; } case GGML_OP_MUL_MAT: { - if (node->src[0]->op == GGML_OP_CONT && node->src[0]->src[0]->op == GGML_OP_TRANSPOSE) { + if (node->src[1]->op == GGML_OP_SOFT_MAX) { + // In the case of `-fa off`, softmax is used, v_trans=true, the dynamic dim is ne[0] for cache_v op_case = 2; - } else if (node->src[0]->op == GGML_OP_VIEW && node->src[1]->op == GGML_OP_VIEW) { - op_case = 3; } break; } case GGML_OP_GET_ROWS: { - if (node->src[1]->op == GGML_OP_VIEW) { - op_case = 2; + // GET_ROWS gathering recurrent state cache rows via the inp->s_copy index list: + // src[0] is a reshape of cache_r/cache_s, src[1] is a view of the s_copy leaf, or the leaf itself + // when one gather covers all states (see build_rs). + // op_case 1/2: active/extra rows of a multi-slot cache + // op_case 3/4: active/extra rows of a single-slot cache + if (node->src[0]->op == GGML_OP_RESHAPE && node->src[0]->src[0] != nullptr && + is_recurrent_cache(node->src[0]->src[0])) { + const bool single_slot = node->src[0]->src[0]->ne[1] == 1; + if (node->src[1]->op == GGML_OP_VIEW) { + op_case = (node->src[1]->view_offs == 0 ? 1 : 2) + (single_slot ? 2 : 0); + } else if (single_slot) { + // a single-slot cache holds exactly the one state the gather selects + op_case = 3; + } } break; } case GGML_OP_ROPE: { - if (node->src[0]->op == GGML_OP_VIEW) { + const int mode = node->op_params[2]; + switch (mode) { + case GGML_ROPE_TYPE_NEOX: { + op_case = 1; + break; + } + case GGML_ROPE_TYPE_IMROPE: { op_case = 2; + break; + } + case GGML_ROPE_TYPE_VISION: { + op_case = 3; + break; + } + case GGML_ROPE_TYPE_MROPE: { + op_case = 4; + break; + } + default: + op_case = 0; + break; } break; } case GGML_OP_VIEW: { + if (!m_model_params.has_rs_rollback && node->src[0] != nullptr && + node->src[0]->op == GGML_OP_GATED_DELTA_NET) { + // The GDN translator publishes native attention/state outputs under these VIEW names. + op_case = 2; + break; + } + if (m_is_static && node->src[0] != nullptr && + (node->src[0]->op == GGML_OP_GATED_DELTA_NET || node->src[0]->op == GGML_OP_CONCAT)) { + // VIEW slicing a GATED_DELTA_NET combined [attn|state] output, or the conv_input + // CONCAT. The consuming CPY/RMS_NORM op recovers the true window at runtime via + // ssm_state_size / the fixed conv kernel width, so this VIEW must stay an identity + // pass-through of the full source here too (it already is on the dynamic path); + // otherwise the generic static-mode Slice below would bake in the *captured* + // cgraph's token count, which is wrong once the compiled static model runs with a + // different token count (prefill chunk size or 1). + op_case = 1; + break; + } if (node->src[0]->op == GGML_OP_VIEW) { auto * src = node->src[0]; if (ggml_nelements(node) != ggml_nelements(src)) { - throw std::runtime_error("Unsupported VIEW case"); + // throw std::runtime_error("Unsupported VIEW case"); + } + op_case = 0; + if (m_model_is_splitted && m_model_inputs.find(get_tensor_ov_name(m_cgraph, src)) != m_model_inputs.end()) { + op_case = 0; } - op_case = 2; } { auto * src = node->src[0]; - if ((ggml_nelements(node) != ggml_nelements(src)) && m_naive) { - // Compare each dimension of node and src, if only one dimension differs then op_case=3 + if (ggml_nelements(node) != ggml_nelements(src)) { + // Case 4: select one slice on src dim1 (via view offset), keep src dim2 as output dim1. + // Typical pattern: + // src: ne=[N, M, K, 1], nb=[b0, b1, b2, b3] + // dst: ne=[N, K, 1, 1], nb=[b0, b2, b3, b3] + if (node->ne[0] == src->ne[0] && node->ne[1] == src->ne[2] && node->ne[2] == 1 && + node->nb[0] == src->nb[0] && node->nb[1] == src->nb[2] && src->ne[1] > 1) { + op_case = 0; + break; + } + + // General case 3: shape differs from source (one or more dims) and is handled as VIEW slicing. int diff_count = 0; for (int i = 0; i < GGML_MAX_DIMS; i++) { if (node->ne[i] != src->ne[i]) { diff_count++; } + // if node ne[i] > src ne[i], case = 0 + if (node->ne[i] > src->ne[i]) { + return 0; + } } - if (diff_count == 1) { - op_case = 3; + if (diff_count >= 1) { + op_case = 0; + } + } + } + break; + } + case GGML_OP_RMS_NORM: { + if (node->src[0]->op == GGML_OP_VIEW) { + if (is_same_shape(node->src[0]->src[0], node->src[0])) { + op_case = 1; + } else if (!m_model_params.has_rs_rollback && + node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) { + // GDN attention is routed directly to this VIEW by get_output_names(). + op_case = 3; + } else if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) { + op_case = 2; + } + } + break; + } + case GGML_OP_POOL_2D: { + const ggml_op_pool pool_mode = static_cast(node->op_params[0]); + switch (pool_mode) { + case GGML_OP_POOL_MAX: { + op_case = 1; + break; + } + case GGML_OP_POOL_AVG: { + op_case = 2; + break; + } + default: + op_case = 0; + break; + } + break; + } + case GGML_OP_UPSCALE: { + const int32_t mode_flags = node->op_params[0]; + const ggml_scale_mode scale_mode = static_cast(mode_flags & 0xFF); + switch (scale_mode) { + case GGML_SCALE_MODE_NEAREST: { + op_case = 1; + break; + } + case GGML_SCALE_MODE_BILINEAR: { + op_case = 2; + break; + } + case GGML_SCALE_MODE_BICUBIC: { + op_case = 3; + break; + } + default: + op_case = 0; + break; + } + break; + } + case GGML_OP_CPY: { + if (node->src[0]->op == GGML_OP_VIEW) { + if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) { + if (!m_model_params.has_rs_rollback) { + // op_case 7 replaces a single-slot cache; op_case 10 writes native GDN state + // into an active range of a larger non-rollback cache. + op_case = is_full_single_slot_writeback(node) ? 7 : 10; + } else { + op_case = 1; + } + } else if (GgmlOvDecoder::is_conv_state_writeback(node)) { + op_case = is_full_single_slot_writeback(node) ? 8 : 2; + break; + } else if (is_conv_states_all_tensor(node->view_src) && node->src[1] != nullptr && + node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) { + op_case = 4; + break; + } else if (node->src[0]->src[0]->op == GGML_OP_GET_ROWS && node->src[1] != nullptr && + node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr && + is_recurrent_cache(node->src[1]->view_src) && node->src[1]->view_src->ne[1] == 1) { + // defrag remainder writeback of a single-slot cache, taken from a view of the one gather of + // all states (see build_rs) + op_case = 9; + } + } else if (node->src[0]->op == GGML_OP_GET_ROWS && node->src[1] != nullptr && + node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr && + is_recurrent_cache(node->src[1]->view_src)) { + // s_copy defrag remainder writeback: gathered extra state rows copied back into the cache + op_case = node->src[1]->view_src->ne[1] == 1 ? 9 : 3; + } else if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) { + // op_case 5: KV write for decoder self-attention (dynamic write offset) + // op_case 6: KV write for encoder self-attn or cross-attn (static offset) + const ggml_tensor * kv_buf = node->src[1]->view_src; + if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) { + op_case = 6; + // Forward-scan the graph for a FLASH_ATTN_EXT that reads from + // the same buffer. Having a mask (src[3] != nullptr) implies + // decoder self-attention and the write offset is dynamic. + for (int i = 0; i < m_cgraph->n_nodes; i++) { + const ggml_tensor * n = m_cgraph->nodes[i]; + if (n->op != GGML_OP_FLASH_ATTN_EXT) { + continue; + } + // K (src[1]) and V (src[2]) are 3-D views whose view_src is + // the flat KV buffer we are writing to. + if ((n->src[1] != nullptr && n->src[1]->view_src == kv_buf) || + (n->src[2] != nullptr && n->src[2]->view_src == kv_buf)) { + if (n->src[3] != nullptr) { + op_case = 5; // decoder self-attention: mask present + } + break; + } } } } break; } + case GGML_OP_ADD: { + if (is_moe_expert_sum_add(node)) { + // Outermost ADD of a MoE expert-plane sum chain: translated as a single + // ReduceSum over the base tensor instead of N-1 chained Adds over N Slices. + op_case = 1; + } + break; + } + case GGML_OP_SCALE: { + if (node->view_src && node->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) { + op_case = node->view_src->ne[1] == 1 ? 2 : 1; + } + break; + } + case GGML_OP_L2_NORM: { + if (std::string(node->name).find("predelta") != std::string::npos) { + op_case = 1; + } + break; + } + case GGML_OP_FLASH_ATTN_EXT: { + if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) { + const ggml_tensor * kv_buf = node->src[1]->view_src; + if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) { + op_case = (node->src[3] != nullptr) ? 1 : 2; + } + } + break; + } default: break; } return op_case; } -int extract_layer_from_name(const std::string & name) { +std::optional extract_layer_from_name(const std::string & name) { size_t pos1 = name.find("_l"); - assert(pos1 != std::string::npos); + if (pos1 == std::string::npos) { + return std::nullopt; + } pos1 += 2; size_t pos2 = name.find(' ', pos1); if (pos2 == std::string::npos) { @@ -256,34 +675,259 @@ int extract_layer_from_name(const std::string & name) { return layer; } +// Recover the sliding window width from ggml's own SWA mask. llama.cpp never passes n_swa to a +// backend, but fill_mask() writes it into the mask: a query row keeps exactly the cells inside +// its window, so the widest row counts min(pos + 1, n_swa) unmasked cells. Counting rather than +// looking for a contiguous band is what makes this work on the KV-cache mask, where columns are +// physical cache cells in arbitrary order, not positions. +// Assumes LLAMA_SWA_TYPE_STANDARD, the only type the caller reconstructs. +static int get_swa_window_from_mask(const ggml_tensor * mask) { + if (mask->data == nullptr || !ggml_backend_buffer_is_host(mask->buffer)) { + return -1; + } + if (mask->type != GGML_TYPE_F16 && mask->type != GGML_TYPE_F32) { + return -1; + } + + const int64_t n_kv = mask->ne[0]; + const int64_t n_tokens = mask->ne[1]; + int64_t window = 0; + + for (int64_t r = 0; r < n_tokens; r++) { + int64_t kept = 0; + for (int64_t c = 0; c < n_kv; c++) { + const size_t i = (size_t) r * n_kv + c; + const float v = mask->type == GGML_TYPE_F16 ? ggml_fp16_to_fp32(((const ggml_fp16_t *) mask->data)[i]) : + ((const float *) mask->data)[i]; + if (v > -INFINITY) { + kept++; + } + } + window = std::max(window, kept); + } + + return window > 0 ? (int) window : -1; +} + std::pair GgmlOvDecoder::compute_llm_params(ggml_cgraph * cgraph, bool is_static) { ModelParams model_params; ComputeParams compute_params; + auto get_attention_pattern_case = [](const ggml_tensor * node) -> int { + if (node == nullptr) { + return -1; + } + + switch (node->op) { + case GGML_OP_FLASH_ATTN_EXT: + if (node->src[0] == nullptr || node->src[1] == nullptr) { + return -1; + } + switch (node->src[1]->op) { + case GGML_OP_PERMUTE: + // case 0: src[1] is PERMUTE of a cache VIEW, mask required + if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr && + node->src[1]->src[0]->op == GGML_OP_VIEW) { + return 0; + } + break; + case GGML_OP_CPY: + // case 1: src[1] is CPY of a PERMUTE(VIEW), mask required + if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr && + node->src[1]->src[0]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->src[0] != nullptr && + node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) { + return 1; + } + break; + case GGML_OP_VIEW: + // cases 4/5/6: whisper - K is a direct non-contiguous VIEW_3D of a KV cache + if (node->src[1]->view_src != nullptr) { + if (node->src[3] != nullptr) { + return 4; // decoder self-attention + } + return 5; // cross-attention or encoder self-attention + } + break; + default: + break; + } + break; + case GGML_OP_SOFT_MAX: + // case 2: node op is SOFT_MAX, src 0 not null & op is MUL_MAT & the src 0 of MUL_MAT is PERMUTE & the permuted tensor src is the view of cache k + if (node->src[0] != nullptr && node->src[1] != nullptr && node->src[0]->op == GGML_OP_MUL_MAT && + node->src[0]->src[0] != nullptr && node->src[0]->src[1] != nullptr && + node->src[0]->src[0]->op == GGML_OP_PERMUTE && node->src[0]->src[0]->src[0] != nullptr && + node->src[0]->src[0]->src[0]->op == GGML_OP_VIEW) { + return 2; + } + // case 3: node op is SOFT_MAX, src 0 not null & op is ADD & the src 0 of ADD is MUL_MAT & the src 0 of MUL_MAT is PERMUTE + if (node->src[0]->op == GGML_OP_ADD && node->src[0]->src[0] != nullptr && + node->src[0]->src[0]->op == GGML_OP_MUL_MAT && node->src[0]->src[0]->src[0] != nullptr && + node->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) { + return 3; + } + break; + default: + break; + } + + return -1; + }; + + // Resolve the attention mask an attention node consumes, mirroring the src layout that + // get_attention_pattern_case() classifies. Used by the SWA pre-pass below. + auto get_attention_op_mask = [&get_attention_pattern_case](const ggml_tensor * node) -> const ggml_tensor * { + switch (get_attention_pattern_case(node)) { + case 0: + case 1: + return node->src[3]; + case 2: + case 3: + return node->src[1]; + default: + return nullptr; + } + }; + + // Pre-pass: classify sliding-window vs full-attention layers. + // + // An interleaved-SWA model keeps two KV caches and two attention masks, and hands each layer + // whichever pair matches its attention type. The mask tensor does not say which is which: both + // are named "attn_inp_kq_mask" by build_attn_inp_kq_mask(), and both carry the same n_kv because + // llama_kv_cache::get_n_kv() pads occupancy up to a common multiple. + // + // The KV cache does say. Each cache allocates cache_k_l once at load time with its own cell + // count: the windowed cache is sized from the window + // (PAD(min(size_base, n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256), see + // llama_kv_cache_iswa), the full-attention one spans the whole context. Read the LEAF buffer + // behind the VIEW rather than the VIEW itself: the leaf extent is a constant per layer, known + // from the first graph onwards, while the view grows with context depth and would invert the + // comparison at shallow depth. + // + // Layers whose leaf is smaller than the largest leaf are the windowed ones. When every layer + // reports the same extent there is no distinction to draw -- either the model has no windowed + // layers, or the window is at least as large as the context so the two caches coincide, in + // which case a windowed layer and a full-attention one compute the same thing. + // + // Getting this wrong is silent and severe: with the windowed layers classified as + // full-attention, permute's KV slicing uses attention_size instead of attention_size_swa. The + // two agree while the context is shorter than the window, then diverge, and the mask add fails + // shape inference ("Failed to broadcast-merge input shapes") partway into a long prompt. + { + std::map layer_extent; // layer -> leaf cache_k cell count + std::map layer_mask; // layer -> mask it consumes + int64_t max_extent = 0; + + for (int i = 0; i < cgraph->n_nodes; i++) { + const ggml_tensor * mask = get_attention_op_mask(cgraph->nodes[i]); + if (mask == nullptr) { + continue; + } + const ggml_tensor * cache_k_permute = nullptr; + switch (get_attention_pattern_case(cgraph->nodes[i])) { + case 0: cache_k_permute = cgraph->nodes[i]->src[1]; break; + case 1: cache_k_permute = cgraph->nodes[i]->src[1]->src[0]; break; + case 2: cache_k_permute = cgraph->nodes[i]->src[0]->src[0]; break; + default: cache_k_permute = cgraph->nodes[i]->src[0]->src[0]->src[0]; break; + } + const ggml_tensor * cache_k_view = cache_k_permute->src[0]; + if (cache_k_view->op != GGML_OP_VIEW) { + continue; + } + const ggml_tensor * leaf = cache_k_view->src[0]; + auto layer = extract_layer_from_name(leaf->name); + if (!layer.has_value()) { + continue; + } + layer_extent[layer.value()] = leaf->ne[1]; + layer_mask[layer.value()] = mask; + max_extent = std::max(max_extent, leaf->ne[1]); + } + + for (const auto & [layer, extent] : layer_extent) { + if (extent < max_extent) { + model_params.swa_layers.push_back(layer); + if (model_params.swa_mask == nullptr) { + model_params.swa_mask = layer_mask[layer]; + } + } + } + std::sort(model_params.swa_layers.begin(), model_params.swa_layers.end()); + + if (ggml_openvino_getenv_int("GGML_OPENVINO_LOG_SWA_LAYERS")) { + std::string per_layer; + for (const auto & [layer, extent] : layer_extent) { + per_layer += " " + std::to_string(layer) + ":" + std::to_string(extent) + + (extent < max_extent ? "(swa)" : ""); + } + GGML_LOG_WARN("ov-swa: attn_layers=%zu max_extent=%ld swa_layers=%zu |%s\n", layer_extent.size(), + (long) max_extent, model_params.swa_layers.size(), per_layer.c_str()); + } + } + + bool rope_seen = false; for (int i = 0; i < cgraph->n_nodes; i++) { - auto * node = cgraph->nodes[i]; - std::string name = std::string(node->name); - if (node->op == GGML_OP_FLASH_ATTN_EXT) { - model_params.n_heads = node->src[0]->ne[2]; - model_params.n_heads_kv = node->src[1]->ne[2]; - model_params.head_size = node->src[0]->ne[0]; + ggml_tensor * node = cgraph->nodes[i]; + const int attention_pattern_case = get_attention_pattern_case(node); + if (attention_pattern_case != -1) { + ggml_tensor * cache_k_permute = nullptr; + ggml_tensor * mask = nullptr; + + switch (attention_pattern_case) { + case 0: + cache_k_permute = node->src[1]; + mask = node->src[3]; + break; + case 1: + cache_k_permute = node->src[1]->src[0]; + mask = node->src[3]; + break; + case 2: + cache_k_permute = node->src[0]->src[0]; + mask = node->src[1]; + break; + case 3: + cache_k_permute = node->src[0]->src[0]->src[0]; + mask = node->src[1]; + break; + case 4: + case 5: { + // whisper: K is a direct VIEW_3D of the KV buffer, no PERMUTE node + auto * cache_k_view = node->src[1]; // VIEW_3D of kv_self.k or kv_cross.k` + compute_params.token_len_per_seq = node->src[0]->ne[1]; + if (attention_pattern_case == 4) { + compute_params.attention_size = cache_k_view->ne[1]; + } else { + compute_params.attention_size_static = cache_k_view->ne[1]; + } + continue; + } + default: + break; + } + + assert(cache_k_permute != nullptr); + + model_params.head_size = cache_k_permute->ne[0]; + model_params.n_heads_kv = cache_k_permute->ne[2]; compute_params.input_len = node->src[0]->ne[1]; + compute_params.token_len_per_seq = node->src[0]->ne[1]; - auto * cache_k_perm = node->src[1]; - if (cache_k_perm->op == GGML_OP_CPY) { - cache_k_perm = cache_k_perm->src[0]; + auto * cache_k_view = cache_k_permute->src[0]; + if (cache_k_view->op != GGML_OP_VIEW || mask == nullptr) { + continue; } - assert(cache_k_perm->op == GGML_OP_PERMUTE); - auto * cache_k_view = cache_k_perm->src[0]; - assert(cache_k_view->op == GGML_OP_VIEW); - auto * cache_k = cache_k_view->src[0]; - int layer = extract_layer_from_name(cache_k->name); - auto * mask = node->src[3]; - std::string mask_name(mask->name); + ggml_tensor * cache_k = cache_k_view->src[0]; + int layer = extract_layer_from_name(cache_k->name).value(); + + // Classified by the pre-pass above, which groups layers by mask tensor identity. The + // mask NAME cannot be used: build_attn_inp_kq_mask() gives both masks the same name. + const bool layer_is_swa = std::find(model_params.swa_layers.begin(), model_params.swa_layers.end(), + layer) != model_params.swa_layers.end(); model_params.kv_buffer_ctx_id = ggml_backend_openvino_buffer_get_ctx_id(cache_k->buffer); - if (mask_name.find("swa") != std::string::npos) { - model_params.swa_layers.push_back(layer); + model_params.n_heads_kv_per_layer[layer] = cache_k_permute->ne[2]; + if (layer_is_swa) { model_params.ctx_per_seq_swa = cache_k->ne[1]; } else { model_params.ctx_per_seq = cache_k->ne[1]; @@ -295,10 +939,10 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr size_t offset; memcpy(&offset, cache_k_view->op_params, sizeof(size_t)); compute_params.seq_active_start = offset / seq_size; - compute_params.token_len_per_seq = node->ne[2]; - if (mask_name.find("swa") != std::string::npos) { + if (layer_is_swa) { compute_params.attention_size_swa = mask->ne[0]; + compute_params.swa_window = get_swa_window_from_mask(mask); } else { compute_params.attention_size = mask->ne[0]; } @@ -307,20 +951,130 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr compute_params.attention_size_swa = model_params.ctx_per_seq_swa; compute_params.token_len_per_seq = 1; } - break; + } + + if (node->op == GGML_OP_MUL_MAT && node->src[0]->op == GGML_OP_PERMUTE && + node->src[0]->src[0]->op == GGML_OP_VIEW && is_kvcache(node->src[0]->view_src, node->view_src)) { + if (node->src[1]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->op == GGML_OP_VIEW && + node->src[1]->src[0]->src[0]->op == GGML_OP_ROPE) { + compute_params.attention_size = node->ne[0]; + } + } + + // if the node op is TRANSPOSE and its input is PERMUTE and the source of the PERMUTE is VIEW, then get the attention size with the TRANSPOSE node ne[0] (in case no GGML_OP_FLASH_ATTN_EXT) + if (node->op == GGML_OP_TRANSPOSE && node->src[0]->op == GGML_OP_PERMUTE && + node->src[0]->src[0]->op == GGML_OP_VIEW) { + compute_params.attention_size = node->ne[0]; + if (is_static) { + compute_params.attention_size = model_params.ctx_per_seq; + } } if (node->op == GGML_OP_ROPE) { - memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 15); + if (compute_params.token_len_per_seq == -1 && node->src[1] != nullptr) { + compute_params.token_len_per_seq = ggml_nelements(node->src[1]); + } + + // When multiple ROPE ops in the graph disagree on op_params (e.g. gemma4's + // mixed SWA/non-SWA layers with different n_dims or freq_base), we cannot + // share a single precomputed rope_sin/rope_cos. Track divergence so the + // translator falls back to per-op make_sin_cos in that case. + static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 16, "rope_params size"); + if (!rope_seen) { + memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 16); + rope_seen = true; + } else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 16) != 0) { + model_params.mixed_rope_params = true; + } + } + if (node->op == GGML_OP_GATED_DELTA_NET) { + model_params.state_size = node->src[0]->ne[0]; + } + if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_recurrent_cache(node->view_src)) { + if (model_params.n_rs_slots == -1) { + model_params.n_rs_slots = node->view_src->ne[1]; + } else { + GGML_ASSERT(model_params.n_rs_slots == node->view_src->ne[1]); + } + compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0]; + compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0]; + } + // Capture the destination slot block of every recurrent state cache writeback, plus the + // source window needed by conv state and packed GDN rollback writes. The active sequences + // occupy a contiguous slot block [begin, begin + n_seqs) of the cache; these offsets move + // with the batch, so they are fed to the cached model as runtime inputs. + if (node->op == GGML_OP_CPY && node->view_src != nullptr && is_recurrent_cache(node->view_src) && + node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) { + const bool is_conv = is_conv_state_writeback(node); + const bool is_gdn = node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr && + node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET; + const bool is_extra = node->src[0]->op == GGML_OP_GET_ROWS; + const bool is_gdn_rollback = is_gdn && is_same_shape(node->src[0], node->src[1]); + + const ggml_tensor * dest_view = node->src[1]; + const ggml_tensor * cache = node->view_src; + const size_t row_bytes = cache->ne[0] * ggml_type_size(cache->type); + if (is_gdn_rollback) { + // Rollback GDN exposes an already-flattened [state, seq, snapshot] VIEW and copies + // it to an identically-shaped cache VIEW. Non-rollback copies native 4-D state + // [value, key, head, seq] into flattened cache rows, so the shapes differ. This + // signature is local to the CPY and still works when fallback splits the graph. + model_params.has_rs_rollback = true; + } + if (row_bytes > 0 && (is_conv || is_gdn || is_extra) && !is_full_single_slot_writeback(node)) { + ComputeParams::RsWriteback writeback; + writeback.slot_begin = (int) (dest_view->view_offs / row_bytes); + if (is_conv) { + writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[0]); + } else if (is_gdn_rollback) { + writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[1]); + } + compute_params.rs_writebacks[get_tensor_ov_name(cgraph, node)] = writeback; + } + if ((is_conv || is_gdn) && !is_full_single_slot_writeback(node)) { + compute_params.s_copy_active_slot_len = (int) dest_view->ne[1]; + } } } + if (model_params.n_heads_kv == -1) { + for (int i = 0; i < cgraph->n_nodes; i++) { + const auto * node = cgraph->nodes[i]; + const ggml_tensor * mask = nullptr; + if (node->op == GGML_OP_SOFT_MAX) { + mask = node->src[1]; + } else if (node->op == GGML_OP_FLASH_ATTN_EXT) { + mask = node->src[3]; + } else { + continue; + } + if (mask == nullptr || mask->op != GGML_OP_NONE || !(mask->flags & GGML_TENSOR_FLAG_INPUT) || + node->src[0] == nullptr) { + continue; + } + model_params.is_cacheless_attn = true; + model_params.n_seq = 1; + model_params.ctx_per_seq = mask->ne[0]; + compute_params.input_len = node->src[0]->ne[1]; + compute_params.token_len_per_seq = compute_params.input_len; + break; + } + } + auto * output_tensor = cgraph->nodes[cgraph->n_nodes - 1]; compute_params.output_len = output_tensor->ne[1]; + if (model_params.is_cacheless_attn) { + for (int i = 0; i < cgraph->n_nodes; i++) { + const auto * node = cgraph->nodes[i]; + if (node->op == GGML_OP_GET_ROWS && is_output_idx(node->src[1], node)) { + compute_params.output_len = node->src[1]->ne[0]; + break; + } + } + } // for NPU, output_len is always 1 except for llama-perplexity if (is_static && compute_params.output_len == 0) { compute_params.output_len = 1; } model_params.ctx = model_params.ctx_per_seq * model_params.n_seq; - model_params.ctx_swa = model_params.ctx_per_seq_swa * model_params.n_seq; return {model_params, compute_params}; } @@ -330,22 +1084,34 @@ void GgmlOvDecoder::validate_cgraph() const { } } -ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, const ggml_tensor * input) const { +ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, + const ggml_tensor * input, + int dynamic_dim_index) const { if (m_naive) { - return input!= nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)}; + return input != nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)}; } - auto name = std::string(input->name); ov::PartialShape input_shape; if (is_inp_tok(input, op) || is_inp_pos(input, op)) { // tokens or positions int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1; + if (m_is_static && is_inp_pos(input, op)) { + // IMROPE stacks n_planes (t/h/w/e) position planes back to back + len *= get_inp_pos_n_planes(op); + } input_shape = ov::PartialShape{1, 1, 1, len}; } else if (is_output_idx(input, op)) { // output index input_shape = ov::PartialShape{1, 1, 1, m_is_static ? m_compute_params.output_len : -1}; + } else if (is_inp_mean(input, op)) { + input_shape = m_is_static ? ov::PartialShape{1, 1, input->ne[1], m_prefill_chunk_size} : + ov::PartialShape{1, 1, -1, -1}; + + } else if (is_inp_scale_rows(input, op)) { + input_shape = ov::PartialShape{1, 1, m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1, 1}; + } else if (is_inp_mask(input, op)) { // mask if (m_is_static) { @@ -356,21 +1122,39 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, co input_shape = ov::PartialShape{-1, 1, -1, -1}; } + } else if (is_recurrent_cache(input)) { + input_shape = ov::PartialShape{get_shape(input)}; + if (!m_is_static && !m_is_stateful && input->ne[1] > 1) { + input_shape[2] = -1; + } + } else if (is_kvcache(input, op)) { // kvcache input_shape = ov::PartialShape{get_shape(input)}; - if (!m_is_static) { + // Whisper.cpp uses a fixed size 1D KV buffer [N, 1, 1, 1] (GGML) or [1, 1, 1, N] (OV). + // the token fill level is handled by token_len_per_seq + dynamic mask input. + // skip dynamic dim and stateful reshape for this layout. + const bool is_flat_kv = (input->ne[1] == 1 && input->ne[2] == 1 && input->ne[3] == 1); + if (!m_is_static && !is_flat_kv) { // do not fix ctx size to make llama-bench work across test params input_shape[2] = -1; } - if (is_stateful()) { + if (is_stateful() && !is_flat_kv) { // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size] // to stateful layout [1, seq, n_heads_kv, head_size]. + // NOTE: Gemma4 uses per-layer-type KV shapes, so no single scalar describes every + // layer. E2B varies only the head size (sliding 256, full 512); 12B also varies the + // head COUNT (sliding 8 x 256, full 1 x 512). Take the head count for this tensor's + // own layer type and derive the head size from its own combined dim, so both layer + // types get the correct split. Using the model-level count split 12B's sliding + // states as 1 x 2048 and decoded garbage. assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 && - input_shape[2].is_dynamic() && - input_shape[3] == (m_model_params.n_heads_kv * m_model_params.head_size)); - input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv, - m_model_params.head_size}; + input_shape[2].is_dynamic() && input_shape[3].is_static()); + const int n_heads_kv = get_n_heads_kv_for_tensor(input); + assert(n_heads_kv > 0 && input_shape[3].get_length() % n_heads_kv == 0); + const int64_t combined_dim = input_shape[3].get_length(); // n_heads_kv * head_size + const int64_t head_size = combined_dim / n_heads_kv; + input_shape = {input_shape[0], ov::Dimension::dynamic(), n_heads_kv, head_size}; } } else if (is_kv_idx(input, op)) { @@ -378,45 +1162,120 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, co int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1; input_shape = ov::PartialShape{1, 1, 1, len}; + } else if (is_inp_s_copy(input, op) || is_s_copy_leaf(input)) { + // On NPU the total slot count (n_seq_max) is fixed at translation time, so the s_copy + // index list has a static length; on CPU/GPU it may change across compiles (defrag). + input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : ov::PartialShape{1, 1, 1, -1}; + } else { input_shape = ov::PartialShape{get_shape(input)}; } + if (dynamic_dim_index != -1 && m_model_is_splitted) { + input_shape[3 - dynamic_dim_index] = -1; + } + if (op->op == GGML_OP_SOFT_MAX && op->src[1] != nullptr && op->src[1]->op == GGML_OP_NONE && + op->src[1]->flags & GGML_TENSOR_FLAG_INPUT && op->src[1] == input) { + // for softmax input mask, the shape is [1, 1, seq_active, seq_active], where seq_active is determined by the input active sequence length instead of the kv cache sequence length + if (m_is_static) { + const int64_t seq_active = m_is_prefill ? m_prefill_chunk_size : 1; + input_shape[2] = seq_active; + input_shape[3] = seq_active; + } else { + input_shape[2] = -1; + input_shape[3] = -1; + } + } return input_shape; } +bool GgmlOvDecoder::is_s_copy_leaf(const ggml_tensor * tensor) const { + if (tensor == nullptr || tensor->op != GGML_OP_NONE || m_cgraph == nullptr) { + return false; + } + for (int i = 0; i < m_cgraph->n_nodes; i++) { + const ggml_tensor * node = m_cgraph->nodes[i]; + if (node->op != GGML_OP_GET_ROWS || node->src[0] == nullptr || node->src[1] == nullptr) { + continue; + } + // The index list may reach the s_copy leaf through one or more VIEWs. + const ggml_tensor * idx = node->src[1]; + while (idx != nullptr && idx->op == GGML_OP_VIEW) { + idx = idx->src[0]; + } + if (idx != tensor) { + continue; + } + // The gathered data must be a recurrent state cache (cache_r/cache_s). + const ggml_tensor * data = node->src[0]; + while (data != nullptr && (data->op == GGML_OP_VIEW || data->op == GGML_OP_RESHAPE)) { + data = data->src[0]; + } + if (data != nullptr && is_recurrent_cache(data)) { + return true; + } + } + return false; +} + void GgmlOvDecoder::add_extra_inputs() { // Extra inputs: // 1. `attention_size`, used in FLASH_ATTN where the shape of the matmul's are 256 aligned, // see llama_kv_cache_unified::get_n_kv and llama_kv_cache_unified::get_padding. // 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch - auto create_1d_input = [this](const std::string & name, int64_t value) { - if (m_is_static) { - auto constant = - std::make_shared(ov::element::i64, ov::Shape{1}, std::vector{value}); - constant->set_friendly_name(name); - m_model_extra_inputs[name] = constant; - } else { - auto param_node = std::make_shared(ov::element::i64, ov::Shape{1}); - param_node->set_friendly_name(name); - param_node->output(0).get_tensor().set_names({name}); - m_model_extra_inputs[name] = param_node; - - auto tensor = std::make_shared(ov::element::i64, ov::Shape{1}); - *tensor->data() = value; - m_model_extra_input_values[name] = tensor; - } + auto create_1d_input = [this](const std::string & name, int64_t value, bool force_parameter = false) { + m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, force_parameter || !m_is_static}; }; - create_1d_input("attention_size", m_compute_params.attention_size); + if (m_compute_params.attention_size != -1) { + create_1d_input("attention_size", m_compute_params.attention_size); + } + if (m_compute_params.attention_size_static != -1) { + create_1d_input("attention_size_static", m_compute_params.attention_size_static); + } if (m_compute_params.attention_size_swa != -1) { create_1d_input("attention_size_swa", m_compute_params.attention_size_swa); } + // only the stateful SWA mask consumes this + if (is_stateful() && m_compute_params.swa_window != -1) { + create_1d_input("swa_window", m_compute_params.swa_window); + } create_1d_input("n_seq_active", m_compute_params.n_seq_active); create_1d_input("seq_active_start", m_compute_params.seq_active_start); create_1d_input("seq_active_end", m_compute_params.seq_active_start + m_compute_params.n_seq_active); - create_1d_input("token_len_per_seq", m_compute_params.token_len_per_seq); - // create_1d_input("token_len", m_token_len_per_seq * m_n_seq_active); + if (m_compute_params.token_len_per_seq != -1) { + create_1d_input("token_len_per_seq", m_compute_params.token_len_per_seq); + } + // create_1d_input("token_len", m_compute_params.token_len_per_seq * m_compute_params.n_seq_active); + + if (m_compute_params.cache_rs_reset_idx != -1 && m_model_params.n_rs_slots != 1) { + // Whether/which cache slot to reset varies per compute call (e.g. a new sequence starting + // vs. continued decoding). can_reuse_statically() does not invalidate the cached static + // model on ComputeParams changes, so these must stay runtime Parameters even when static + // (scale.cpp op_case 1 only uses them in value comparisons, never as Slice bounds, so this + // does not reintroduce dynamic shapes). + create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx, /*force_parameter=*/true); + create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len, /*force_parameter=*/true); + } + + if (m_compute_params.s_copy_active_slot_len != -1) { + create_1d_input("s_copy_active_slot_len", m_compute_params.s_copy_active_slot_len); + if (m_is_static) { + // Number of real tokens in the current prefill chunk. The last chunk is padded with + // fabricated token ids; attention masks them out, but the recurrent (GDN/conv) path + // would otherwise fold them into cache_r/cache_s permanently. Varies per chunk, so it + // must stay a runtime Parameter; it is only compared against a Range or used as Gather + // indices, so it does not make any shape dynamic. + create_1d_input("chunk_valid_len", get_static_n_tokens(), /*force_parameter=*/true); + } + } + + for (const auto & [node_name, writeback] : m_compute_params.rs_writebacks) { + create_1d_input("rs_slot_begin_" + node_name, writeback.slot_begin); + if (!m_is_static && writeback.src_begin >= 0) { + create_1d_input("rs_src_begin_" + node_name, writeback.src_begin); + } + } } bool GgmlOvDecoder::node_is_used_as_src(const int node_idx) { @@ -439,14 +1298,11 @@ void GgmlOvDecoder::compute_model_inputs() { ggml_tensor * node = m_cgraph->nodes[i]; // the node op is NONE means this node maybe as input of later nodes, we should add it to model inputs for this node. if (node->op == GGML_OP_NONE && node_is_used_as_src(i)) { - std::string node_name(node->name); + std::string node_name = get_tensor_ov_name(m_cgraph, node); if (m_model_weights.find(node_name) == m_model_weights.end()) { m_inputs[node_name] = node; - auto param_node = - std::make_shared(get_ov_type(node), get_graph_input_shape(node, nullptr)); - param_node->set_friendly_name(node_name); - param_node->output(0).get_tensor().set_names({node_name}); - m_model_inputs[node_name] = param_node; + m_model_inputs[node_name] = {get_ov_type(node), + get_graph_input_shape(node, nullptr, m_node_dynamic_dims[node])}; } continue; } @@ -455,13 +1311,18 @@ void GgmlOvDecoder::compute_model_inputs() { if (src == nullptr) { continue; } - std::string src_name = std::string(src->name); + std::string src_name = get_tensor_ov_name(m_cgraph, src); if (src->flags & GGML_TENSOR_FLAG_INPUT) { - src_name = get_graph_input_ov_name(src, node); + src_name = get_tensor_graph_input_ov_name(this, m_cgraph, src, node); } if (m_model_weights.find(src_name) != m_model_weights.end()) { continue; } + // A view over a weight is served by the base tensor's Constant, never by a Parameter. + if (src->view_src != nullptr && + m_model_weights.find(get_tensor_ov_name(m_cgraph, src->view_src)) != m_model_weights.end()) { + continue; + } bool is_intermediate_node = false; for (const auto & node_info : m_node_info_list) { @@ -487,11 +1348,14 @@ void GgmlOvDecoder::compute_model_inputs() { m_model_params.kv_names.push_back(src_name); } } - ov::PartialShape param_shape = get_graph_input_shape(node, src); - auto param_node = std::make_shared(get_ov_type(src), param_shape); - param_node->set_friendly_name(src_name); - param_node->output(0).get_tensor().set_names({src_name}); - m_model_inputs[src_name] = param_node; + // Resolve nested VIEW nodes by following src[0] until the first non-VIEW tensor. + while (src->op == GGML_OP_VIEW && src->src[0] != nullptr) { + src = src->src[0]; + src_name = get_tensor_ov_name(m_cgraph, src); + } + m_inputs[src_name] = src; + m_model_inputs[src_name] = {get_ov_type(src), + get_graph_input_shape(node, src, m_node_dynamic_dims[src])}; } } } @@ -502,13 +1366,16 @@ void GgmlOvDecoder::compute_model_outputs() { for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) { auto * cur_node = m_cgraph->nodes[node_n]; // if the node op is NONE means this node is not used at all, we can skip it directly without adding to model outputs. - if (cur_node->op == GGML_OP_NONE) { + if (cur_node->op == GGML_OP_NONE || cur_node->op == GGML_OP_VIEW || cur_node->op == GGML_OP_RESHAPE) { + continue; + } + if (::is_inplace_op(cur_node) && ggml_nbytes(cur_node) == 0) { continue; } auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)]; if (cur_node_use_count == 0) { - // The output of SET_ROWS is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src. - if (cur_node != nullptr && cur_node->op == GGML_OP_SET_ROWS) { + // The output of in-place ops is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src. + if (cur_node != nullptr && ::is_inplace_op(cur_node) && ggml_nbytes(cur_node) > 0) { cur_node = cur_node->view_src; } } else { @@ -526,9 +1393,9 @@ void GgmlOvDecoder::compute_model_outputs() { } } if (cur_node != nullptr) { - std::string node_output_name(cur_node->name); - m_model_outputs[node_output_name] = cur_node; - m_model_output_names.push_back(node_output_name); + std::string cur_node_name = get_tensor_ov_name(m_cgraph, cur_node); + m_model_outputs[cur_node_name] = cur_node; + m_model_output_names.insert(cur_node_name); } } } @@ -556,7 +1423,7 @@ const ggml_tensor * GgmlOvDecoder::get_tensor_from_name(const std::string & name if (src == nullptr) { break; } - if (std::string(src->name) == name) { + if (get_tensor_ov_name(m_cgraph, src) == name) { return src; } } @@ -572,10 +1439,17 @@ std::map GgmlOvDecoder::get_kv_param_res_names() const return kv_param_res_names; } -std::map> GgmlOvDecoder::create_weight_nodes(ggml_cgraph * cgraph, bool naive) { - static std::mutex weights_mutex; - std::lock_guard lock(weights_mutex); +// MUL_MAT_ID's src[0] is the [k, m, n_expert] expert-weight tensor. It is always a constant per-expert +// weight table -- never a computed activation -- regardless of whether the backend happened to mark its +// buffer as GGML_BACKEND_BUFFER_USAGE_WEIGHTS (test-backend-ops, for example, never sets that usage +// flag, unlike real inference). Without this, non-quantized (F16/F32/BF16) expert weights would fall +// through the check below as "not a weight", get decoded as a Parameter/activation instead of a +// Constant, and crash GatherMatmul's "only constant weights are supported" check. +static bool is_mul_mat_id_expert_weight(const ggml_tensor * node, int src_index) { + return node->op == GGML_OP_MUL_MAT_ID && src_index == 0; +} +std::map> GgmlOvDecoder::create_weight_nodes(ggml_cgraph * cgraph, bool naive) { std::map> model_weights; auto * nodes = cgraph->nodes; auto n_nodes = cgraph->n_nodes; @@ -587,18 +1461,19 @@ std::map> GgmlOvDecoder::create_weight_no continue; } - std::string src_name(src->name); - if (is_rope_freqs_weight(src, node)) { + // A view over a weight is served by the base tensor's Constant. + ggml_tensor * base = src->view_src ? src->view_src : src; + std::string src_name = get_tensor_ov_name(cgraph, base); + if (is_rope_freqs_weight(base, node)) { src_name = "rope_freqs.weight"; } - if (!src->view_src) { - ggml_backend_buffer * buffer = src->buffer; - if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) { - if (model_weights.find(src_name) == model_weights.end()) { - auto weight_node = create_weight_node(src, naive); - weight_node->set_friendly_name(src_name); - model_weights[src_name] = weight_node; - } + ggml_backend_buffer * buffer = base->buffer; + if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(base->type) || + is_mul_mat_id_expert_weight(node, i)) { + if (model_weights.find(src_name) == model_weights.end()) { + auto weight_node = create_weight_node(base, naive); + weight_node->set_friendly_name(src_name); + model_weights[src_name] = weight_node; } } } @@ -606,6 +1481,41 @@ std::map> GgmlOvDecoder::create_weight_no return model_weights; } +// Process-lifetime cache for weight nodes built from NON-OpenVINO buffers (e.g. the +// token_embd.weight copy that lives in a CPU/mmap buffer and feeds GET_ROWS). Such +// tensors have no OV buffer context to own a cached extra, so without this they are +// re-extracted/re-requantized on every (re)compile — for token_embd that is a ~1-2 GB +// F32 dequant each time. Keyed by tensor->data, which is stable for the process and +// uniquely identifies the immutable weight bytes. OV-buffer weights keep using the +// per-tensor extra cache and never reach here. +static std::mutex g_nonov_weight_cache_mutex; +static std::unordered_map> g_nonov_weight_cache; + +std::set GgmlOvDecoder::collect_weight_names(ggml_cgraph * cgraph) { + // Mirrors the name-selection logic of create_weight_nodes() but builds no nodes, + // so topology checks don't trigger weight extraction/requantization. + std::set names; + for (int node_i = 0; node_i < cgraph->n_nodes; node_i++) { + auto * node = cgraph->nodes[node_i]; + for (int i = 0; i < GGML_MAX_SRC; i++) { + auto * src = node->src[i]; + if (src == nullptr) { + continue; + } + const ggml_tensor * base = src->view_src ? src->view_src : src; + std::string src_name(base->name); + if (is_rope_freqs_weight(base, node)) { + src_name = "rope_freqs.weight"; + } + ggml_backend_buffer * buffer = base->buffer; + if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(base->type)) { + names.insert(src_name); + } + } + } + return names; +} + std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor, bool naive) { const bool is_ov_buffer = ggml_backend_buffer_is_openvino(tensor->buffer); @@ -634,15 +1544,41 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor } } + // MUL_MAT_ID expert weights are 3D GGML tensors [k, m, n_expert]. + // Keep the full reversed 4D shape when materializing non-quantized constants, + // otherwise the expert dimension is collapsed and later Gather/MatMul logic + // only sees a single expert slice. + if (!ggml_is_quantized(tensor->type) && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) { + auto weight_tensor = ov::Tensor(get_ov_type(tensor), get_shape(tensor), tensor->data); + auto weight_node = std::make_shared(weight_tensor); + weight_node->set_friendly_name(tensor->name); + return weight_node; + } + + // Non-OV-buffer weights (CPU/mmap, e.g. the GET_ROWS token_embd copy) have no buffer + // context to cache an extra in, so memoize them here keyed by their (stable) data + // pointer to avoid re-extracting on every recompile. Opt-in via + // GGML_OPENVINO_REDUCE_COMPILE_MEM or GGML_OPENVINO_MEMORY_OPTIMIZE. Skip + // for `naive` (test/naive path) since use_bias changes the produced node. + const bool cacheable_nonov = ggml_openvino_reduce_compile_mem_enabled() && !is_ov_buffer && + !naive && tensor->data != nullptr; + if (cacheable_nonov) { + std::lock_guard lock(g_nonov_weight_cache_mutex); + auto it = g_nonov_weight_cache.find(tensor->data); + if (it != g_nonov_weight_cache.end()) { + return it->second; + } + } + // There are three cases where we need to create a new weight node: // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used // 3. test-backend-ops. buffers in test-backend-ops does not set USAGE_WEIGHT so backend_buffer_set_tensor will not create weight node // GGML_LOG_DEBUG("%s: creating new weight node for %s\n", __func__, tensor->name); - static const std::set weight_types = {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, - GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, - GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}; + static const std::set weight_types = {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, + GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1, GGML_TYPE_Q4_K, + GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_MXFP4}; if (weight_types.find(tensor->type) == weight_types.end()) { throw std::runtime_error("Unexpected weight tensor type: " + std::string(tensor->name) + " with type " + ggml_type_name(tensor->type)); @@ -671,6 +1607,12 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor ov_weight.weight_node->set_friendly_name(tensor->name); if (!is_ov_buffer) { + if (cacheable_nonov) { + std::lock_guard lock(g_nonov_weight_cache_mutex); + // Another thread may have inserted concurrently; keep the first. + auto [it, inserted] = g_nonov_weight_cache.emplace(tensor->data, ov_weight.weight_node); + return it->second; + } return ov_weight.weight_node; } @@ -689,7 +1631,7 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor void GgmlOvDecoder::dump_cgraph(const ggml_cgraph * cgraph, std::string & filename) { std::ofstream file(filename); if (!file.is_open()) { - std::cerr << "Failed to open file" << std::endl; + std::cerr << "Failed to open file" << '\n'; return; } @@ -795,11 +1737,11 @@ void print_tensor_address_map(const ggml_cgraph * cgraph) { } } for (const auto & pair : address_map) { - std::cout << "Address: " << pair.first << std::endl; + std::cout << "Address: " << pair.first << '\n'; for (const auto & name : pair.second) { std::cout << name << " ; "; } - std::cout << std::endl << std::endl; + std::cout << "\n\n"; } } @@ -850,6 +1792,161 @@ std::vector GgmlOvDecoder::get_input_stride(int node_idx, const std::str return get_stride(m_node_info_list[node_idx].node_inputs.at(name)); } +size_t GgmlOvDecoder::get_view_input_size(int node_idx, const std::string & name) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + return it->second.size(); + } + return 0; +} + +size_t GgmlOvDecoder::get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + return it->second[view_index].second->view_offs; + } + } + return 0; +} + +size_t GgmlOvDecoder::get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + return view_tensor->src[0]->view_offs; + } + } + } + return 0; +} + +std::vector GgmlOvDecoder::get_view_input_stride(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + return get_stride(it->second[view_index].second); + } + } + return {}; +} + +std::vector GgmlOvDecoder::get_view_input_src_stride(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + return get_stride(view_tensor->src[0]); + } + } + } + return {}; +} + +ov::Shape GgmlOvDecoder::get_view_input_ggml_shape(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + return get_shape(it->second[view_index].second); + } + } + return {}; +} + +ov::Shape GgmlOvDecoder::get_view_input_src_ggml_shape(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + return get_shape(view_tensor->src[0]); + } + } + } + return {}; +} + +ov::PartialShape GgmlOvDecoder::get_view_input_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * tensor = it->second[view_index].second; + ov::PartialShape shape = ov::PartialShape{get_shape(tensor)}; + + // Check if this tensor has a dynamic dimension + auto dynamic_it = m_node_dynamic_dims.find(tensor); + if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) { + int dynamic_dim_index = dynamic_it->second; + // GGML uses reverse indexing, so convert to OpenVINO indexing + shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1; + } + + return shape; + } + } + return {}; +} + +ov::PartialShape GgmlOvDecoder::get_view_input_src_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + auto * src_tensor = view_tensor->src[0]; + ov::PartialShape shape = ov::PartialShape{get_shape(src_tensor)}; + + // Check if this tensor has a dynamic dimension + auto dynamic_it = m_node_dynamic_dims.find(src_tensor); + if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) { + int dynamic_dim_index = dynamic_it->second; + // GGML uses reverse indexing, so convert to OpenVINO indexing + shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1; + } + + return shape; + } + } + } + return {}; +} + +std::string GgmlOvDecoder::get_view_input_name(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + return it->second[view_index].first; + } + } + return ""; +} + +std::string GgmlOvDecoder::get_view_input_src_name(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + return get_tensor_ov_name(m_cgraph, view_tensor->src[0]); + } + } + } + return ""; +} + ov::element::Type GgmlOvDecoder::get_input_type(int node_idx, const std::string & name) const { return get_ov_type(m_node_info_list[node_idx].node_inputs.at(name)); } @@ -867,7 +1964,7 @@ std::vector GgmlOvDecoder::get_input_names(int node_idx) const { } ov::PartialShape GgmlOvDecoder::get_output_shape(int node_idx) const { - auto * ggml_tensor = m_node_info_list[node_idx].node_output; + auto * ggml_tensor = m_node_info_list[node_idx].node; return ov::PartialShape(get_shape(ggml_tensor)); } @@ -875,8 +1972,55 @@ ov::element::Type GgmlOvDecoder::get_output_type(const int node_idx) const { return get_ov_type(m_node_info_list[node_idx].node); } +std::vector GgmlOvDecoder::get_output_stride(int node_idx) const { + auto * ggml_tensor = m_node_info_list[node_idx].node; + return get_stride(ggml_tensor); +} + std::vector GgmlOvDecoder::get_output_names(int node_idx) const { - return {m_node_info_list[node_idx].node_output_name}; + auto * node = m_node_info_list[node_idx].node; + if (node->op == GGML_OP_GATED_DELTA_NET && !m_model_params.has_rs_rollback) { + std::string attn_name; + std::string state_name; + for (int i = node_idx + 1; i < m_cgraph->n_nodes; i++) { + auto * consumer = m_cgraph->nodes[i]; + if (consumer->op != GGML_OP_VIEW || consumer->src[0] != node) { + continue; + } + // GGML packs [attention | state]. The attention VIEW starts at offset 0 and the + // state VIEW starts after the token-dependent attention segment. + auto & name = consumer->view_offs == 0 ? attn_name : state_name; + if (!name.empty()) { + return {m_node_info_list[node_idx].node_name}; + } + name = get_tensor_ov_name(m_cgraph, consumer); + } + if (!attn_name.empty() && !state_name.empty()) { + return {attn_name, state_name}; + } + } + return {m_node_info_list[node_idx].node_name}; +} + +std::string GgmlOvDecoder::get_inplace_op_src(int node_idx) const { + auto * node = m_node_info_list[node_idx].node; + if (!::is_inplace_op(node) || node->view_src == nullptr || ggml_nbytes(node) == 0) { + return ""; + } + const int op_case = m_node_info_list[node_idx].node_op_case; + if (node->op == GGML_OP_CPY && (op_case == 1 || op_case == 2 || op_case == 3) && + m_compute_params.s_copy_active_slot_len == -1) { + return ""; + } + return get_tensor_ov_name(m_cgraph, node->view_src); +} + +bool GgmlOvDecoder::is_view_like_alias_of(int node_idx, const std::string & view_src_name) const { + auto * node = m_node_info_list[node_idx].node; + if (node->view_src == nullptr || get_tensor_ov_name(m_cgraph, node->view_src) != view_src_name) { + return false; + } + return node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW; } const std::string & GgmlOvDecoder::get_op_name() const { @@ -884,6 +2028,14 @@ const std::string & GgmlOvDecoder::get_op_name() const { return unknown_name; } +int32_t GgmlOvDecoder::get_op_dynamic_dim(int node_idx) const { + auto it = m_node_dynamic_dims.find(m_node_info_list[node_idx].node); + if (it == m_node_dynamic_dims.end()) { + return -1; + } + return it->second; +} + const std::string & GgmlOvDecoder::get_op_name(int node_idx) const { return m_node_info_list[node_idx].node_name; } @@ -896,6 +2048,10 @@ int32_t * GgmlOvDecoder::get_output_op_params(int node_idx) const { return m_node_info_list[node_idx].node->op_params; } +size_t GgmlOvDecoder::get_output_op_offset(int node_idx) const { + return m_node_info_list[node_idx].node->view_offs; +} + void GgmlOvDecoder::visit_subgraph(std::function, int node_idx)> node_visitor) const { for (int node_idx = 0; node_idx < m_cgraph->n_nodes; node_idx++) { if (m_cgraph->nodes[node_idx]->op == GGML_OP_NONE) { @@ -906,63 +2062,14 @@ void GgmlOvDecoder::visit_subgraph(std::function ops = { - {GGML_OP_NONE, "GGML_OP_NONE" }, - {GGML_OP_ACC, "GGML_OP_ACC" }, - {GGML_OP_ADD, "GGML_OP_ADD" }, - {GGML_OP_ADD1, "GGML_OP_ADD1" }, - {GGML_OP_CONT, "GGML_OP_CONT" }, - {GGML_OP_DIV, "GGML_OP_DIV" }, - {GGML_OP_DUP, "GGML_OP_DUP" }, - {GGML_OP_GET_ROWS, "GGML_OP_GET_ROWS" }, - {GGML_OP_MUL, "GGML_OP_MUL" }, - {GGML_OP_MUL_MAT, "GGML_OP_MUL_MAT" }, - {GGML_OP_PERMUTE, "GGML_OP_PERMUTE" }, - {GGML_OP_RESHAPE, "GGML_OP_RESHAPE" }, - {GGML_OP_RMS_NORM, "GGML_OP_RMS_NORM" }, - {GGML_OP_ROPE, "GGML_OP_ROPE" }, - {GGML_OP_SCALE, "GGML_OP_SCALE" }, - {GGML_OP_SOFT_MAX, "GGML_OP_SOFT_MAX" }, - {GGML_OP_SUB, "GGML_OP_SUB" }, - {GGML_OP_TRANSPOSE, "GGML_OP_TRANSPOSE" }, - {GGML_OP_VIEW, "GGML_OP_VIEW" }, - {GGML_OP_SET_ROWS, "GGML_OP_SET_ROWS" }, - {GGML_OP_CPY, "GGML_OP_CPY" }, - {GGML_OP_FLASH_ATTN_EXT, "GGML_OP_FLASH_ATTN_EXT"}, - }; - static const std::map unary_ops = { - {GGML_UNARY_OP_ABS, "GGML_UNARY_OP_ABS" }, - {GGML_UNARY_OP_SGN, "GGML_UNARY_OP_SGN" }, - {GGML_UNARY_OP_NEG, "GGML_UNARY_OP_NEG" }, - {GGML_UNARY_OP_STEP, "GGML_UNARY_OP_STEP" }, - {GGML_UNARY_OP_TANH, "GGML_UNARY_OP_TANH" }, - {GGML_UNARY_OP_ELU, "GGML_UNARY_OP_ELU" }, - {GGML_UNARY_OP_RELU, "GGML_UNARY_OP_RELU" }, - {GGML_UNARY_OP_SIGMOID, "GGML_UNARY_OP_SIGMOID" }, - {GGML_UNARY_OP_GELU, "GGML_UNARY_OP_GELU" }, - {GGML_UNARY_OP_GELU_QUICK, "GGML_UNARY_OP_GELU_QUICK" }, - {GGML_UNARY_OP_SILU, "GGML_UNARY_OP_SILU" }, - {GGML_UNARY_OP_HARDSWISH, "GGML_UNARY_OP_HARDSWISH" }, - {GGML_UNARY_OP_HARDSIGMOID, "GGML_UNARY_OP_HARDSIGMOID"}, - {GGML_UNARY_OP_EXP, "GGML_UNARY_OP_EXP" }, - {GGML_UNARY_OP_COUNT, "GGML_UNARY_OP_COUNT" } - }; - static const std::map glu_ops = { - {GGML_GLU_OP_SWIGLU, "GGML_GLU_OP_SWIGLU"}, - {GGML_GLU_OP_GEGLU, "GGML_GLU_OP_GEGLU" }, - {GGML_GLU_OP_REGLU, "GGML_GLU_OP_REGLU" } - }; - switch (node->op) { case GGML_OP_UNARY: - return unary_ops.at(ggml_get_unary_op(node)); + return std::string("GGML_UNARY_OP_") + ggml_unary_op_name(ggml_get_unary_op(node)); case GGML_OP_GLU: - return glu_ops.at(ggml_get_glu_op(node)); + return std::string("GGML_GLU_OP_") + ggml_glu_op_name(ggml_get_glu_op(node)); default: - return ops.at(node->op); + return std::string("GGML_OP_") + ggml_op_name(node->op); } - static const std::string unknown_op = "UNKNOWN_GGML_OP"; - return unknown_op; } const std::string & GgmlOvDecoder::get_op_type(int node_idx) const { @@ -973,3 +2080,386 @@ const std::string & GgmlOvDecoder::get_op_type() const { static const std::string unknown_op = "UNKNOWN_GGML_OP"; return unknown_op; } + +void GgmlOvDecoder::compute_node_dynamic_dims() { + auto visit_node = [&](auto && self, ggml_tensor * node) -> void { + if (!node) { + return; + } + + if (node->op == GGML_OP_CPY) { + m_node_dynamic_dims[node] = -1; + } + + if (m_node_dynamic_dims.count(node)) { + return; + } + for (int i = 0; i < GGML_MAX_SRC; i++) { + ggml_tensor * src = node->src[i]; + if (src == nullptr) { + continue; + } + struct ggml_tensor * root_src = nullptr; + // if (src->org_src) { + // root_src = src->org_src; + // } + if (root_src) { + if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_output_idx(root_src, node)) { + m_node_dynamic_dims[root_src] = 0; + m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src]; + continue; + } + self(self, root_src); + m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src]; + } else { + if (is_inp_tok(src, node) || is_inp_pos(src, node) || is_output_idx(src, node)) { + m_node_dynamic_dims[src] = 0; + continue; + } + if (is_inp_scale_rows(src, node)) { + m_node_dynamic_dims[src] = 1; + continue; + } + if (node->op == GGML_OP_VIEW && src->op == GGML_OP_NONE && !is_stateful() && !m_model_is_splitted) { + m_node_dynamic_dims[src] = 1; + continue; + } + self(self, src); + } + } + switch (node->op) { + case GGML_OP_NONE: + m_node_dynamic_dims[node] = -1; + break; + case GGML_OP_GET_ROWS: + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[1]] != -1) { + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[1]]; + if (dynamic_dim_idx == 0) { + m_node_dynamic_dims[node] = 1; + } else { + auto dynamic_dim_stride = node->src[1]->nb[dynamic_dim_idx] / ggml_type_size(node->src[1]->type) * + ggml_type_size(node->src[0]->type); + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (dynamic_dim_stride == node->src[0]->nb[i]) { + m_node_dynamic_dims[node] = i; + break; + } + } + } + // OPENVINO_ASSERT(dynamic_dim_value == node->ne[m_node_dynamic_dims[node]], + // "Dynamic dim value mismatch for node: " + std::string(node->name) + + // " and its src[1]: " + std::string(node->src[1]->name)); + } + break; + case GGML_OP_MUL: + case GGML_OP_MUL_MAT: + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; + } + if (m_node_dynamic_dims[node->src[1]] != -1) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]]; + } + break; + case GGML_OP_PERMUTE: + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; + // auto dynamic_dim_value = node->src[0]->ne[dynamic_dim_idx]; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->op_params[i] == dynamic_dim_idx) { + m_node_dynamic_dims[node] = i; + break; + } + } + // OPENVINO_ASSERT(dynamic_dim_value == node->ne[m_node_dynamic_dims[node]], + // "Dynamic dim value mismatch for node: " + std::string(node->name) + + // " and its src[0]: " + std::string(node->src[0]->name)); + } + break; + case GGML_OP_VIEW: { + // Use stride-based matching: the stride of a VIEW dimension directly + // encodes which source dimension it indexes into, so it uniquely + // identifies the dynamic dim even when two dims share the same size. + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + if (node->src[0]->op == GGML_OP_NONE) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; + break; + } + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; + auto dynamic_dim_value = node->src[0]->ne[dynamic_dim_idx]; + auto dynamic_dim_stride = + node->src[0]->nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) * ggml_type_size(node->type); + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->nb[i] == dynamic_dim_stride) { + m_node_dynamic_dims[node] = i; + break; + } + } + if (m_node_dynamic_dims[node] != -1 && dynamic_dim_value != node->ne[m_node_dynamic_dims[node]]) { + m_node_dynamic_dims[node] = -1; + // an empty view, e.g. the extra states of a single-slot recurrent cache, always mismatches + if (ggml_nelements(node) > 0) { + GGML_LOG_WARN("ggml-openvino: dynamic dim value mismatch for VIEW node '%s', src[0]: '%s'\n", + node->name, node->src[0]->name); + } + } + } + break; + } + case GGML_OP_TRANSPOSE: + case GGML_OP_RESHAPE: { + if (is_same_shape(node->src[0], node)) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; + break; + } + // RESHAPE requires src[0] to be contiguous, so both src and result + // have standard compact strides: nb[i] = type_size * prod(ne[0..i-1]). + // Match src->nb[dynamic_dim] against result->nb[i] to find the output + // dimension whose flat-memory boundary aligns with the source dynamic + // boundary. This is unambiguous (result strides are strictly monotone) + // and handles merged-lower-dim cases that ne-value matching misses. + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; + auto dynamic_dim_stride = node->src[0]->nb[dynamic_dim_idx]; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->nb[i] == dynamic_dim_stride && node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) { + m_node_dynamic_dims[node] = i; + break; + } + } + if (m_node_dynamic_dims[node] == -1) { + GGML_LOG_WARN("ggml-openvino: cannot determine dynamic dim for RESHAPE node '%s'\n", node->name); + } + } + break; + } + case GGML_OP_FLASH_ATTN_EXT: { + // Output shape is hard-coded in ggml_flash_attn_ext as: + // ne = { v->ne[0], q->ne[2], q->ne[1], q->ne[3] } + // i.e. output dim 0 <- v dim 0 (head_size, static) + // output dim 1 <- q dim 2 (n_heads, static) + // output dim 2 <- q dim 1 (n_tokens, potentially dynamic) + // output dim 3 <- q dim 3 (batch, static) + // Using the fixed q-dim -> output-dim mapping table. + // q is src[0]; the mapping from q's dynamic dim to the output dim is: + // q dim 1 -> output dim 2 + // q dim 2 -> output dim 1 + // q dim 3 -> output dim 3 + // q dim 0 -> output dim 0 (head_size axis, unlikely to be dynamic) + constexpr int q_to_out[GGML_MAX_DIMS] = {0, 2, 1, 3}; + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + auto q_dynamic_dim = m_node_dynamic_dims[node->src[0]]; + m_node_dynamic_dims[node] = q_to_out[q_dynamic_dim]; + } + break; + } + case GGML_OP_CONT: + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; + if (ggml_are_same_shape(node, node->src[0])) { + m_node_dynamic_dims[node] = dynamic_dim_idx; + } else { + size_t src_logical_nb[GGML_MAX_DIMS]; + src_logical_nb[0] = ggml_type_size(node->src[0]->type); + src_logical_nb[1] = src_logical_nb[0] * (node->src[0]->ne[0] / ggml_blck_size(node->src[0]->type)); + for (int i = 2; i < GGML_MAX_DIMS; i++) { + src_logical_nb[i] = src_logical_nb[i - 1] * node->src[0]->ne[i - 1]; + } + + auto dynamic_dim_stride = src_logical_nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) * + ggml_type_size(node->type); + int matched_dim_count = 0; + int first_matched_dim = -1; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->nb[i] == dynamic_dim_stride && node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) { + if (first_matched_dim == -1) { + first_matched_dim = i; + } + m_node_dynamic_dims[node] = i; + matched_dim_count++; + } + } + if (matched_dim_count > 1 && node->src[0]->ne[dynamic_dim_idx] == 1) { + // Single-token capture: every trailing dim is size 1 with the same stride, so + // the match is ambiguous. The lowest index is the real axis; the rest are + // ggml's size-1 padding. Bailing out here would bake the captured token count + // into the static prefill model, which then runs with a different one. + m_node_dynamic_dims[node] = first_matched_dim; + } else if (matched_dim_count != 1) { + m_node_dynamic_dims[node] = -1; + GGML_LOG_WARN("ggml-openvino: cannot determine dynamic dim for CONT node '%s', src[0]: '%s'\n", + node->name, node->src[0]->name); + } + } + } + break; + case GGML_OP_CONCAT: + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->src[0]->ne[i] != node->ne[i]) { + m_node_dynamic_dims[node] = i; + break; + } + } + break; + case GGML_OP_SSM_CONV: + case GGML_OP_GATED_DELTA_NET: + m_node_dynamic_dims[node] = 1; + break; + case GGML_OP_RMS_NORM: + case GGML_OP_L2_NORM: + case GGML_OP_NORM: + case GGML_OP_ADD: + case GGML_OP_SUB: + case GGML_OP_GLU: + case GGML_OP_ROPE: + case GGML_OP_SCALE: + case GGML_OP_SOFT_MAX: + case GGML_OP_ARGSORT: + case GGML_OP_ADD_ID: + case GGML_OP_UNARY: + case GGML_OP_CUMSUM: + case GGML_OP_FILL: + case GGML_OP_SET: + case GGML_OP_DIAG: + case GGML_OP_TRI: + case GGML_OP_REPEAT: + case GGML_OP_DUP: + // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0]. + // DIV/CLAMP are used in the MoE routing-weight normalization + // (sum_rows -> clamp -> div). If they are left untracked here the dynamic + // (token) dim is lost there, the captured prefill token count gets baked into + // the downstream reshapes, and every decoder layer after layer 0 turns static + // (which then triggers the GPU in-place-concat KV-cache corruption). + case GGML_OP_DIV: + case GGML_OP_CLAMP: + case GGML_OP_PAD: + case GGML_OP_UPSCALE: + case GGML_OP_SIN: + case GGML_OP_COS: + case GGML_OP_LOG: + case GGML_OP_ROLL: + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; + break; + case GGML_OP_SUM_ROWS: + // SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the + // dynamic dim is preserved unless it was axis 0 (then it is summed away). + m_node_dynamic_dims[node] = + (m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]]; + break; + case GGML_OP_MUL_MAT_ID: + case GGML_OP_SOLVE_TRI: + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]]; + break; + case GGML_OP_CPY: + case GGML_OP_SET_ROWS: + case GGML_OP_SUM: + case GGML_OP_MEAN: + m_node_dynamic_dims[node] = -1; + break; + case GGML_OP_IM2COL: { + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[1]] != -1) { + const bool is_2D = node->op_params[6] == 1; + const int src_dyn = m_node_dynamic_dims[node->src[1]]; + if (is_2D) { + if (src_dyn == 0) { + m_node_dynamic_dims[node] = 1; // IW -> OW + } else if (src_dyn == 1) { + m_node_dynamic_dims[node] = 2; // IH -> OH + } else if (src_dyn == 3) { + m_node_dynamic_dims[node] = 3; // N -> N + } + } else { + if (src_dyn == 0) { + m_node_dynamic_dims[node] = 1; // IW -> OW + } else if (src_dyn == 2) { + m_node_dynamic_dims[node] = 2; // N -> N (1D: b->ne[2] is the batch/channel dim) + } + } + if (m_node_dynamic_dims[node] != -1) { + OPENVINO_ASSERT(node->src[1]->ne[src_dyn] == node->ne[m_node_dynamic_dims[node]], + "Dynamic dim value mismatch for IM2COL node: " + std::string(node->name) + + " and its src[1]: " + std::string(node->src[1]->name)); + } + } + break; + } + case GGML_OP_IM2COL_3D: { + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[1]] != -1) { + const int src_dyn = m_node_dynamic_dims[node->src[1]]; + if (src_dyn == 0) { + m_node_dynamic_dims[node] = 1; // IW -> OW + } else if (src_dyn == 1) { + m_node_dynamic_dims[node] = 2; // IH -> OH + } else if (src_dyn == 3) { + m_node_dynamic_dims[node] = 3; // N -> N + } + if (m_node_dynamic_dims[node] != -1) { + OPENVINO_ASSERT(node->src[1]->ne[src_dyn] == node->ne[m_node_dynamic_dims[node]], + "Dynamic dim value mismatch for IM2COL_3D node: " + std::string(node->name) + + " and its src[1]: " + std::string(node->src[1]->name)); + } + } + break; + } + default: + GGML_LOG_DEBUG("ggml-openvino: compute_node_dynamic_dims: unhandled op %s for node '%s'\n", + ggml_op_name(node->op), node->name); + break; + } + }; + + for (int i = 0; i < m_cgraph->n_nodes; i++) { + ggml_tensor * node = m_cgraph->nodes[i]; + visit_node(visit_node, node); + } + + // print the nodes in m_cgraph name & shape with the dynamic dim (the dynamic dim is the dimension with -1 in m_node_dynamic_dims) for debugging + if (0) { + for (int i = 0; i < m_cgraph->n_nodes; i++) { + ggml_tensor * node = m_cgraph->nodes[i]; + int dynamic_dim = m_node_dynamic_dims[node]; + std::cout << "[" << i << "] " << "node_name: " << node->name << " op: " << ggml_op_name(node->op) + << " shape: ["; + for (int j = 0; j < 4; j++) { + if (j == dynamic_dim) { + std::cout << "*"; + } else { + std::cout << node->ne[j]; + } + if (j < 3) { + std::cout << ", "; + } + } + std::cout << "]" << '\n'; + // print the src name & shape with the dynamic dim for debugging + for (int j = 0; j < GGML_MAX_SRC; j++) { + ggml_tensor * src = node->src[j]; + if (src == nullptr) { + continue; + } + int src_dynamic_dim = m_node_dynamic_dims[src]; + std::cout << " [" << j << "] src_name: " << src->name << " ["; + for (int k = 0; k < 4; k++) { + if (k == src_dynamic_dim) { + std::cout << "*"; + } else { + std::cout << src->ne[k]; + } + if (k < 3) { + std::cout << ", "; + } + } + std::cout << "]" << '\n'; + } + std::cout << '\n'; + } + } +} diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 3ae25dd..a614d29 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -1,6 +1,7 @@ #pragma once -#include "ggml-quants.h" +#include "ggml-backend-impl.h" +#include "ggml-backend.h" #include "ggml.h" #include "openvino/decoder.h" @@ -10,30 +11,51 @@ #include #include #include +#include +#include #include struct ModelParams { int ctx = -1; - int ctx_swa = -1; int ctx_per_seq = -1; int ctx_per_seq_swa = -1; int n_seq = 1; - int n_heads = -1; int n_heads_kv = -1; + // Per-layer KV head count. gemma-4 12B interleaves 8 x 256 sliding layers with 1 x 512 + // full-attention layers, so no single scalar describes every layer. Keyed by layer, not by + // layer TYPE, because the SWA classification depends on the context size (extents tie at a + // small -c) while the head count does not. + std::map n_heads_kv_per_layer; int head_size = -1; - int32_t rope_params[15]; + int state_size = -1; // for SSM molels, eg qwen35 + int32_t rope_params[16] = {}; + int n_rs_slots = -1; + bool has_rs_rollback = false; + bool mixed_rope_params = false; + bool is_cacheless_attn = false; std::vector swa_layers; + // The sliding-window mask tensor, identified in compute_llm_params() by grouping attention + // layers on the mask they consume. Only used to tell the two masks apart when naming OV + // parameters -- both carry the same tensor name. Null when the graph has a single mask. + const ggml_tensor * swa_mask = nullptr; std::vector kv_names; size_t kv_buffer_ctx_id = 0; bool same_rope_params(const ModelParams & other) const { - return memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0; + return mixed_rope_params == other.mixed_rope_params && + memcmp(rope_params, other.rope_params, sizeof(int32_t) * 16) == 0; } - bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other); } + bool can_reuse_dynamically(const ModelParams & other) const { + return same_rope_params(other) && n_rs_slots == other.n_rs_slots && + has_rs_rollback == other.has_rs_rollback; + } - bool can_reuse_statically(const ModelParams & other) const { return same_rope_params(other) && ctx == other.ctx; } + bool can_reuse_statically(const ModelParams & other) const { + return same_rope_params(other) && ctx == other.ctx && n_rs_slots == other.n_rs_slots && + has_rs_rollback == other.has_rs_rollback; + } bool kv_buffer_changed(const ModelParams & other) const { return kv_buffer_ctx_id != other.kv_buffer_ctx_id; } }; @@ -43,25 +65,80 @@ struct ComputeParams { int seq_active_start = 0; int attention_size = -1; int attention_size_swa = -1; + int attention_size_static = -1; // encoder/cross-attn KV fill level (whisper) + // Sliding window width, read back from the band of ggml's own SWA mask. ggml never passes + // n_swa down to a backend, but fill_mask() bakes it into the mask contents, so the widest + // unmasked row recovers it. Shorter than n_swa while the sequence is still short, which is + // harmless: every causal pair is inside the window then anyway. + int swa_window = -1; int input_len = -1; int token_len_per_seq = -1; int past_kv_len = -1; int output_len = 1; + + int cache_rs_reset_idx = -1; + int cache_rs_reset_len = -1; + // SSM/DeltaNet models otionally clear cache_r and cache_s of certain slots in the cgraph + // 3: [ 18432, 4, 1, 1] RESHAPE cache_r_l0 (reshaped) + // [ 18432, 4, 1, 1] 0: NONE cache_r_l0 + // 4: [ 18432, 1, 1, 1] VIEW cache_r_l0 (reshaped) (view) + // [ 18432, 4, 1, 1] 0: RESHAPE cache_r_l0 (reshaped) + // 5: [ 18432, 1, 1, 1] SCALE cache_r_l0 (reshaped) (view) (view) + // [ 18432, 1, 1, 1] 0: VIEW cache_r_l0 (reshaped) (view) + + int s_copy_active_slot_len = -1; + // SSM/DeltaNet models otionally reorder slots of state cache, to make the active slots contiguous + // leaf_5 is the inp->s_copy in llama-graph.cpp, eg if there are 8 slots in total and slot 3 and 7 + // are active in the current batch, leaf_5 will be [3, 7, 5, 6, 4] + // 6: [ 2, 1, 1, 1] VIEW (view) + // [ 2, 1, 1, 1] 0: NONE leaf_5 + // 7: [ 18432, 2, 1, 1] GET_ROWS conv_states-0 + // [ 18432, 4, 1, 1] 0: RESHAPE cache_r_l0 (reshaped) + // [ 2, 1, 1, 1] 1: VIEW (view) + // 8: [ 0, 1, 1, 1] VIEW (view) + // [ 2, 1, 1, 1] 0: NONE leaf_5 + // 9: [ 18432, 0, 1, 1] GET_ROWS node_9 + // [ 18432, 4, 1, 1] 0: RESHAPE cache_r_l0 (reshaped) + // [ 0, 1, 1, 1] 1: VIEW (view) + // 10: [ 18432, 0, 1, 1] VIEW cache_r_l0 (view) + // [ 18432, 4, 1, 1] 0: NONE cache_r_l0 + // 11: [ 18432, 0, 1, 1] CPY cache_r_l0 (view) (copy of ) + // [ 18432, 0, 1, 1] 0: GET_ROWS node_9 + // [ 18432, 0, 1, 1] 1: VIEW cache_r_l0 (view) + + struct RsWriteback { + int slot_begin = 0; // first cache slot written by the CPY + int src_begin = -1; // first source column copied by a conv-state CPY + }; + + std::map rs_writebacks; + // Destination slot offset of each state cache writeback CPY node, keyed by node name. It + // changes with the batch (kv head, active sequence count) and, with rollback enabled + // (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot. Passed to the + // cached model as a runtime input. Dynamic models also receive the source-side offset; static + // models use a fixed end-anchored offset in the translator. }; +// defined below; declared here because GgmlOvDecoder uses it inline +std::optional extract_layer_from_name(const std::string & name); + +// detects the MoE expert-plane-sum ADD chain (see definition); used by supports_op too +bool is_moe_expert_sum_add(const ggml_tensor * node); + class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { public: + static std::string get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor); struct NodeInfo { ggml_tensor * node; std::string node_name; std::string node_op_type; std::map node_inputs; + std::map>> node_inputs_views; std::vector node_inputs_names; - ggml_tensor * node_output; - std::string node_output_name; int node_op_case = 0; void * data_addr; }; + // Graph decoder GgmlOvDecoder(ggml_cgraph * cgraph, ModelParams & model_params, @@ -69,6 +146,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { std::map> & model_weights, bool is_static, bool is_stateful = false, + bool model_is_splitted = false, bool is_prefill = false, int prefill_chunk_size = 256); @@ -84,6 +162,42 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { virtual std::vector get_input_stride(int node_idx, const std::string & name) const override; + virtual size_t get_view_input_size(int node_idx, const std::string & name) const override; + + virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const override; + + virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const override; + + virtual std::vector get_view_input_stride(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual std::vector get_view_input_src_stride(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual ov::Shape get_view_input_ggml_shape(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual ov::Shape get_view_input_src_ggml_shape(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual ov::PartialShape get_view_input_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual ov::PartialShape get_view_input_src_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const override; + + virtual std::string get_view_input_src_name(int node_idx, + const std::string & name, + size_t view_index) const override; + virtual ov::element::Type get_input_type(int node_idx, const std::string & name) const override; virtual size_t get_input_size() const override; @@ -106,12 +220,20 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { virtual ov::element::Type get_output_type(int node_idx) const override; + virtual std::vector get_output_stride(int node_idx) const override; + virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const override; virtual int32_t * get_output_op_params(int node_idx) const override; + virtual size_t get_output_op_offset(int node_idx) const override; + virtual std::vector get_output_names(int node_idx) const override; + virtual std::string get_inplace_op_src(int node_idx) const override; + + virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const override; + virtual const std::string & get_op_type() const override; virtual const std::string & get_op_type(int node_idx) const override; @@ -120,38 +242,33 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { virtual const std::string & get_op_name(int node_idx) const override; - virtual void visit_subgraph(std::function, int node_idx)> node_visitor) const override; + virtual int32_t get_op_dynamic_dim(int node_idx) const override; + + virtual void visit_subgraph( + std::function, int node_idx)> node_visitor) const override; ggml_tensor * get_input_ggml_tensor(const std::string & name) const { return m_inputs.at(name); } virtual int get_op_case(int node_idx) const override { return m_node_info_list[node_idx].node_op_case; } - virtual const std::map> & get_model_inputs() const override { + virtual const std::map & get_model_inputs() const override { return m_model_inputs; } - virtual const std::map> & get_model_extra_inputs() const override { + virtual const std::map & get_model_extra_inputs() const override { return m_model_extra_inputs; } - virtual const std::map> & get_model_extra_input_values() const { - return m_model_extra_input_values; - } - virtual const std::map> & get_model_weights() const override { return m_model_weights; } - virtual std::vector get_model_output_names() const override { - return m_model_output_names; - } + virtual std::set get_model_output_names() const override { return m_model_output_names; } const std::map & get_model_outputs() const { return m_model_outputs; } virtual int get_ctx_size() const { return m_model_params.ctx; } - virtual int get_ctx_swa_size() const { return m_model_params.ctx_swa; } - virtual int get_ctx_per_seq() const { return m_model_params.ctx_per_seq; } virtual int get_ctx_per_seq_swa() const { return m_model_params.ctx_per_seq_swa; } @@ -163,19 +280,44 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { m_model_params.swa_layers.end(); } + // KV head count for one layer. Sliding and full layers can differ (gemma-4 12B), so callers + // that reinterpret a KV buffer must use this and not the model-level n_heads_kv. + int get_n_heads_kv_for_layer(int layer) const { + auto it = m_model_params.n_heads_kv_per_layer.find(layer); + return it != m_model_params.n_heads_kv_per_layer.end() ? it->second : m_model_params.n_heads_kv; + } + + // Same, for a KV cache tensor: its layer comes from the leaf name (cache_k_l). + int get_n_heads_kv_for_tensor(const ggml_tensor * kv_tensor) const { + if (auto layer = extract_layer_from_name(std::string(kv_tensor->name)); layer.has_value()) { + return get_n_heads_kv_for_layer(layer.value()); + } + return m_model_params.n_heads_kv; + } + int get_past_kv_len() const { return m_compute_params.past_kv_len; } int get_input_len() const { return m_compute_params.input_len; } virtual int32_t * get_rope_params() const override { return const_cast(m_model_params.rope_params); } + virtual bool has_mixed_rope_params() const override { return m_model_params.mixed_rope_params; } + + virtual int get_ssm_state_size() const override { return m_model_params.state_size; } + virtual std::map get_kv_param_res_names() const override; virtual bool is_static() const override { return m_is_static; } virtual bool is_stateful() const override { return m_is_stateful; } - ov::PartialShape get_graph_input_shape(const ggml_tensor * op, const ggml_tensor * input) const; + int get_static_n_tokens() const { return m_is_prefill ? m_prefill_chunk_size : 1; } + + virtual bool is_splited_model() const override { return m_model_is_splitted; } + + ov::PartialShape get_graph_input_shape(const ggml_tensor * op, + const ggml_tensor * input, + int dynamic_dim_index = -1) const; static void dump_cgraph(const ggml_cgraph * cgraph, std::string & filename); @@ -184,6 +326,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { static std::map> create_weight_nodes(ggml_cgraph * cgraph, bool naive = false); + // Collect just the set of weight-tensor names referenced by the graph, without + // building (or requantizing) any OV weight nodes. Used by topology checks like + // is_model_splitted that only need name membership. + static std::set collect_weight_names(ggml_cgraph * cgraph); + const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const; const ggml_tensor * get_tensor_from_name(const std::string & name) const; @@ -205,6 +352,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { bool m_is_prefill = false; bool m_naive = false; int m_prefill_chunk_size = 0; + bool m_model_is_splitted = false; // label the cgraph is splited or not static ov::Shape get_shape(const ggml_tensor * tensor); static std::vector get_stride(const ggml_tensor * tensor); @@ -213,54 +361,114 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { void add_extra_inputs(); void update_io(ggml_cgraph * cgraph); + bool is_bound_to(const ggml_cgraph * cgraph) const; - inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE; } - inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_ROPE && tensor == op->src[1]; } - inline static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) { + // IMROPE and VISION pack 4 stacked position planes (t/h/w/e) into inp_pos, each of length + // n_tokens; other modes carry a single position per token. + static int get_inp_pos_n_planes(const ggml_tensor * op) { + const int mode = op->op_params[2]; + return (mode == GGML_ROPE_TYPE_IMROPE || mode == GGML_ROPE_TYPE_VISION || (mode & GGML_ROPE_TYPE_MROPE)) ? 4 : 1; + } + + static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) { return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM; } - inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]); + static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { + return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) || + (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]); + } + + static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) { + return op->op == GGML_OP_MUL_MAT && tensor == op->src[1] && tensor->op == GGML_OP_NONE && + (tensor->flags & GGML_TENSOR_FLAG_INPUT) && tensor->type == GGML_TYPE_F32 && + op->src[0] != nullptr && op->src[0]->op != GGML_OP_NONE; } - inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { + // per-token embedding scale [1, n_tokens] (inp->scale_rows in llama-graph.cpp) + static bool is_inp_scale_rows(const ggml_tensor * tensor, const ggml_tensor * op) { + return op->op == GGML_OP_MUL && tensor == op->src[1] && strcmp(tensor->name, "inp_scale_rows") == 0; + } + + static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_ROPE && tensor == op->src[2]; } + inline static bool is_recurrent_cache(const ggml_tensor * tensor) { + return tensor != nullptr && (strncmp(tensor->name, "cache_r_l", strlen("cache_r_l")) == 0 || + strncmp(tensor->name, "cache_s_l", strlen("cache_s_l")) == 0 || + strncmp(tensor->name, "cache_ple_r_l", strlen("cache_ple_r_l")) == 0); + } + + inline static bool is_cache(const ggml_tensor * tensor, const ggml_tensor * op) { + return is_recurrent_cache(tensor) || is_kvcache(tensor, op); + } + inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_SET_ROWS && op->src[2] == tensor; + if (tensor == nullptr || is_recurrent_cache(tensor)) { + return false; + } + return (tensor->buffer != nullptr && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) || + (op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor); } - inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + inline static bool is_conv_state_writeback(const ggml_tensor * node) { + return node->op == GGML_OP_CPY && node->view_src != nullptr && is_recurrent_cache(node->view_src) && + node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr && + node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr && + node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src; + } + + static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor; } - inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE; + bool is_swa_mask(const ggml_tensor * tensor) const { + return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask; } - static std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) { - if (is_inp_tok(tensor, op)) { - return "inp_tokens"; - } + static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE && + op->src[1]->op == GGML_OP_NONE; + } + + // the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp) + static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) { + return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && + op->src[0]->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY; + } + + std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) const { if (is_inp_pos(tensor, op)) { return "inp_pos"; } if (is_inp_emb(tensor, op)) { return "embd"; } - if (is_output_idx(tensor, op)) { - return "inp_out_ids"; - } if (is_inp_mask(tensor, op)) { - return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa"; + // Give the two attention masks distinct OV parameter names. + // + // An interleaved-SWA model builds one full-attention mask and one sliding-window mask, + // but build_attn_inp_kq_mask() names them identically, so keying a parameter off + // tensor->name alone makes the second mask OVERWRITE the first in m_model_inputs: both + // attention types then read a single parameter, and the windowed layers silently run + // against an unbanded mask. Disambiguate using the SWA layer set computed in + // compute_llm_params(), which classifies by mask tensor identity rather than by name. + // + // When no SWA layer was found there is only one mask in play, so the plain name is + // correct and no _swa parameter is created. + if (m_model_params.swa_layers.empty()) { + return "self_kq_mask"; + } + return is_swa_mask(tensor) ? "self_kq_mask_swa" : "self_kq_mask"; } return tensor->name; } @@ -272,23 +480,28 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { void compute_model_inputs(); void compute_model_outputs(); + // True if tensor is the inp->s_copy index leaf gathered by a recurrent state cache GET_ROWS + // (possibly through a VIEW), so it gets a dynamic [1,1,1,-1] graph-input shape. + bool is_s_copy_leaf(const ggml_tensor * tensor) const; + + // Infer and propagate dynamic-dimension indices for all tensors in the GGML graph. + void compute_node_dynamic_dims(); + void validate_cgraph() const; ggml_cgraph * m_cgraph = nullptr; std::map m_inputs; - std::map> m_model_inputs; - std::map> m_model_extra_inputs; - std::map> m_model_extra_input_values; + std::map m_model_inputs; + std::map m_model_extra_inputs; std::map> m_model_weights; std::map m_model_outputs; - std::vector m_model_output_names; + std::set m_model_output_names; std::vector m_node_info_list; + std::map m_node_dynamic_dims; ModelParams m_model_params; ComputeParams m_compute_params; }; void print_tensor_address_map(const ggml_cgraph * cgraph); - -int extract_layer_from_name(const std::string & name); diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index cc3cb45..14a0b03 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -2,10 +2,15 @@ #include "ggml-impl.h" #include "ggml.h" +#include "model-cache.h" +#include +#include #include #include #include +#include +#include #include ov::Core & ov_singleton_core() { @@ -13,24 +18,149 @@ ov::Core & ov_singleton_core() { return core; } +static bool has_prefix(const std::string & s, const std::string & prefix) { + return s.size() >= prefix.size() && std::equal(prefix.begin(), prefix.end(), s.begin()); +} + +static bool is_virtual_routing_device(const std::string & device_name) { + return has_prefix(device_name, "AUTO") || has_prefix(device_name, "MULTI") || has_prefix(device_name, "HETERO"); +} + +static std::vector ov_enumerate_devices() { + std::vector result; + + for (const auto & device : ov_singleton_core().get_available_devices()) { + if (!is_virtual_routing_device(device)) { + result.push_back(device); + } + } + + if (result.empty()) { + result.push_back("CPU"); + } + + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + return result; +} + +std::string ggml_openvino_get_device_description(const std::string & device_name) { + std::string description = device_name; + try { + description = ov_singleton_core().get_property(device_name, ov::device::full_name); + } catch (...) { + return device_name; + } + + if (has_prefix(device_name, "NPU")) { + try { + const std::string arch = ov_singleton_core().get_property(device_name, "DEVICE_ARCHITECTURE").as(); + if (!arch.empty()) { + description += " (NPU " + arch + ")"; + } + } catch (...) { + } + } + + return description; +} + +// requested: GGML_OPENVINO_DEVICE, nullptr if unset. available_devices is never empty (see ov_enumerate_devices) +static std::string resolve_openvino_device_name(const std::vector & available_devices, + const char * requested) { + auto available = [&](const std::string & name) { + return std::find(available_devices.begin(), available_devices.end(), name) != available_devices.end(); + }; + if (requested == nullptr) { + return available("CPU") ? "CPU" : available_devices.front(); + } + if (!available(requested)) { + // No fallback to CPU (easy to miss) and no GPU -> GPU.0 alias (with iGPU + dGPU, GPU.0 is often the + // wrong one). List the devices here: --list-devices initializes this backend and would abort too. + std::string list; + for (const std::string & name : available_devices) { + list += "\n " + name + ": " + ggml_openvino_get_device_description(name); + } + GGML_ABORT("GGML OpenVINO Backend: GGML_OPENVINO_DEVICE=%s is not available. " + "Set it to one of the available OpenVINO devices:%s", + requested, list.c_str()); + } + return requested; +} + // ===================================================== // Device Configuration Implementations // ===================================================== void ggml_openvino_device_config::init() { + static std::mutex mutex; + std::lock_guard lock(mutex); if (initialized) { return; } - device_name = getenv("GGML_OPENVINO_DEVICE") ? getenv("GGML_OPENVINO_DEVICE") : "CPU"; - auto available_devices = ov_singleton_core().get_available_devices(); - if (std::find(available_devices.begin(), available_devices.end(), device_name) == available_devices.end()) { - GGML_LOG_WARN("GGML OpenVINO Backend: device %s is not available, fallback to CPU\n", device_name.c_str()); - device_name = "CPU"; + // Set up front: a failed OpenCL setup below is not retried on every call + initialized = true; + + // All recognized GGML_OPENVINO_* env vars. Their values are cached here + // once at backend init time and read back via ggml_openvino_getenv_str() + // (raw string) or ggml_openvino_getenv_int() (integer / boolean toggle). + static constexpr const char * env_var_names[] = { + // String values (use ggml_openvino_getenv_str) + "GGML_OPENVINO_DEVICE", + "GGML_OPENVINO_CACHE_DIR", + "GGML_OPENVINO_SPILL_DIR", + "GGML_OPENVINO_DEBUG_NODE", + "GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR", + "GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY", + "GGML_OPENVINO_NPU_COMPILE_CONFIG", + // Integer values (use ggml_openvino_getenv_int) + "GGML_OPENVINO_PREFILL_CHUNK_SIZE", + // Boolean toggles (treated as int flags via ggml_openvino_getenv_int) + "GGML_OPENVINO_STATEFUL_EXECUTION", + "GGML_OPENVINO_PROFILING", + "GGML_OPENVINO_DUMP_CGRAPH", + "GGML_OPENVINO_DUMP_IR", + "GGML_OPENVINO_DEBUG_INPUT", + "GGML_OPENVINO_DEBUG_OUTPUT", + // Force the static (NPU-shape) compute path on any device, e.g. GGML_OPENVINO_DEVICE=CPU, + // to test the static-shape translation without NPUW/real NPU hardware in the loop. + "GGML_OPENVINO_FORCE_STATIC", + "GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS", + "GGML_OPENVINO_ENABLE_CACHE", + "GGML_OPENVINO_DISABLE_CACHE", + "GGML_OPENVINO_DISABLE_KV_SLICE", + "GGML_OPENVINO_ENABLE_FALLBACK", + "GGML_OPENVINO_MANUAL_GQA_ATTN", + "GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN", + "GGML_OPENVINO_MOE_OP", + "GGML_OPENVINO_MEMORY_OPTIMIZE", + "GGML_OPENVINO_RELEASE_WEIGHTS", + "GGML_OPENVINO_REDUCE_COMPILE_MEM", + "GGML_OPENVINO_LOG_UNSUPPORTED_OPS", + "GGML_OPENVINO_LOG_SWA_LAYERS", + "GGML_OPENVINO_NATIVE_SOFTPLUS", + "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS", + "GGML_OPENVINO_REQUANT_KQUANT", + "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", + // Build the precise (but O(n_nodes)) graph cache key. Needed by op tests. + "GGML_OPENVINO_FULL_GRAPH_KEY", + }; + + for (const char * const & env_var : env_var_names) { + auto * env = getenv(env_var); + if (env) { + environment_variables[env_var] = env; + } } - is_npu = (device_name == "NPU"); - auto * cache_dir = getenv("GGML_OPENVINO_CACHE_DIR"); - if (device_name == "NPU") { + available_devices = ov_enumerate_devices(); + device_name = resolve_openvino_device_name(available_devices, ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE")); + is_npu = has_prefix(device_name, "NPU"); + + ggml_openvino_model_cache_init(); + + const char * cache_dir = ggml_openvino_getenv_str("GGML_OPENVINO_CACHE_DIR"); + if (has_prefix(device_name, "NPU")) { compile_config = { {"NPU_COMPILER_DYNAMIC_QUANTIZATION", "YES" }, {"NPU_USE_NPUW", "YES" }, @@ -42,55 +172,83 @@ void ggml_openvino_device_config::init() { {"NPUW_DQ", "YES" }, {"NPUW_DQ_FULL", "NO" }, }; - if (cache_dir) { + if (cache_dir && strlen(cache_dir) > 0) { compile_config["NPUW_CACHE_DIR"] = cache_dir; + compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE)); + } + const char * compilation_mode_params = + ggml_openvino_getenv_str("GGML_OPENVINO_NPU_COMPILE_CONFIG"); + if (compilation_mode_params && strlen(compilation_mode_params) > 0) { + compile_config["NPU_COMPILATION_MODE_PARAMS"] = compilation_mode_params; } - } else if (cache_dir) { - ov_singleton_core().set_property(ov::cache_dir(cache_dir)); + } else if (cache_dir && strlen(cache_dir) > 0) { + compile_config.insert(ov::cache_dir(cache_dir)); + compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE)); + } + + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2) { + compile_config.insert(ov::enable_profiling(true)); } // Initialize remote context with queue sharing for GPU - if (device_name == "GPU") { - // Create OpenCL context and queue - cl_int err; - cl_platform_id platform; - err = clGetPlatformIDs(1, &platform, nullptr); - if (err != CL_SUCCESS) { - GGML_LOG_ERROR("Failed to get OpenCL platform: %d\n", err); - return; + if (has_prefix(device_name, "GPU")) { + // Use the OpenCL context OpenVINO created for this device, so GPU.N gets its own device + cl_context cl_ctx; + try { + auto ov_ctx = ov_singleton_core().get_default_context(device_name).as(); + cl_ctx = ov_ctx.get(); + } catch (const std::exception & e) { + // The consumers of the remote context have no host fallback, and OpenVINO + // already reported the device as present. + GGML_ABORT("ggml-openvino: failed to get the OpenCL context for %s: %s", device_name.c_str(), e.what()); } + cl_int err; cl_device_id cl_device; - err = clGetDeviceIDs(platform, CL_DEVICE_TYPE_GPU, 1, &cl_device, nullptr); + err = clGetContextInfo(cl_ctx, CL_CONTEXT_DEVICES, sizeof(cl_device), &cl_device, nullptr); if (err != CL_SUCCESS) { - GGML_LOG_ERROR("Failed to get OpenCL device: %d\n", err); - return; + GGML_ABORT("ggml-openvino: failed to get the OpenCL device for %s: %d", device_name.c_str(), err); } - cl_context cl_ctx = clCreateContext(nullptr, 1, &cl_device, nullptr, nullptr, &err); + cl_platform_id cl_platform; + err = clGetDeviceInfo(cl_device, CL_DEVICE_PLATFORM, sizeof(cl_platform), &cl_platform, nullptr); if (err != CL_SUCCESS) { - GGML_LOG_ERROR("Failed to create OpenCL context: %d\n", err); - return; + GGML_ABORT("ggml-openvino: failed to get the OpenCL platform for %s: %d", device_name.c_str(), err); } - cl_queue = clCreateCommandQueueWithProperties(cl_ctx, cl_device, nullptr, &err); + cl_mem_fill_fn = + (clEnqueueMemFillINTEL_fn) clGetExtensionFunctionAddressForPlatform(cl_platform, "clEnqueueMemFillINTEL"); + cl_mem_cpy_fn = + (clEnqueueMemcpyINTEL_fn) clGetExtensionFunctionAddressForPlatform(cl_platform, "clEnqueueMemcpyINTEL"); + + cl_ulong device_max_alloc = 0; + err = clGetDeviceInfo(cl_device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(device_max_alloc), &device_max_alloc, + nullptr); + if (err == CL_SUCCESS) { + max_alloc_size = device_max_alloc; + } else { + // not fatal, ggml then allocates one buffer + GGML_LOG_WARN("Failed to get OpenCL max allocation size: %d\n", err); + } + + const cl_queue_properties profiling_properties[] = { + CL_QUEUE_PROPERTIES, + CL_QUEUE_PROFILING_ENABLE, + 0, + }; + const cl_queue_properties * queue_properties = + ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2 ? profiling_properties : nullptr; + cl_queue = clCreateCommandQueueWithProperties(cl_ctx, cl_device, queue_properties, &err); if (err != CL_SUCCESS) { - GGML_LOG_ERROR("Failed to create OpenCL command queue: %d\n", err); - clReleaseContext(cl_ctx); - return; + GGML_ABORT("ggml-openvino: failed to create the OpenCL queue for %s: %d", device_name.c_str(), err); } // Create OpenVINO remote context with queue sharing remote_context = ov::intel_gpu::ocl::ClContext(ov_singleton_core(), cl_queue); - - // Release the context (queue keeps a reference) - clReleaseContext(cl_ctx); - } else if (device_name == "NPU") { + } else if (has_prefix(device_name, "NPU")) { // remote tensor is not used for NPU yet // remote_context = ov_singleton_core().get_default_context(device_name); } - - initialized = true; } ggml_openvino_device_config::~ggml_openvino_device_config() { @@ -116,11 +274,58 @@ const std::string & ggml_openvino_get_device_name() { return ggml_openvino_get_device_config().device_name; } +std::vector ggml_openvino_get_available_devices() { + auto & config = ggml_openvino_get_device_config(); + config.init(); + return config.available_devices; +} + +// Get the value of a GGML_OPENVINO_* env var as a string. Returns +// default_value when the var is unset or set to an empty string. +const char * ggml_openvino_getenv_str(const char * var, const char * default_value) { + auto & env_map = ggml_openvino_get_device_config().environment_variables; + auto it = env_map.find(var); + return (it == env_map.end() || it->second.empty()) ? default_value : it->second.c_str(); +} + +// Get the value of a GGML_OPENVINO_* env var as an int (via std::atoi). +// Returns default_value (0) when the var is unset or empty. Used for both +// integer settings (e.g. GGML_OPENVINO_PREFILL_CHUNK_SIZE) and boolean +// toggles: "0" disables, any non-zero integer enables. +int ggml_openvino_getenv_int(const char * var, int default_value) { + const char * v = ggml_openvino_getenv_str(var, nullptr); + return v ? std::atoi(v) : default_value; +} + +bool ggml_openvino_reduce_compile_mem_enabled() { + const char * reduce_compile_mem = ggml_openvino_getenv_str("GGML_OPENVINO_REDUCE_COMPILE_MEM"); + if (reduce_compile_mem != nullptr) { + return ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") != 0; + } + return ggml_openvino_getenv_int("GGML_OPENVINO_MEMORY_OPTIMIZE") != 0; +} + +bool ggml_openvino_release_weights_enabled() { + const char * release_weights = ggml_openvino_getenv_str("GGML_OPENVINO_RELEASE_WEIGHTS"); + if (release_weights != nullptr) { + return ggml_openvino_is_gpu() && ggml_openvino_getenv_int("GGML_OPENVINO_RELEASE_WEIGHTS") != 0; + } + return ggml_openvino_is_gpu() && ggml_openvino_getenv_int("GGML_OPENVINO_MEMORY_OPTIMIZE") != 0; +} + // Check if running on NPU bool ggml_openvino_is_npu() { return ggml_openvino_get_device_config().is_npu; } +bool ggml_openvino_is_gpu() { + return has_prefix(ggml_openvino_get_device_name(), "GPU"); +} + +size_t ggml_openvino_max_alloc_size() { + return ggml_openvino_get_device_config().max_alloc_size; +} + // Get the remote context for the current device (returns empty optional for CPU) std::optional ggml_openvino_get_remote_context() { return ggml_openvino_get_device_config().remote_context; @@ -136,32 +341,14 @@ cl_command_queue ggml_openvino_get_cl_queue() { return ggml_openvino_get_device_config().cl_queue; } -// Get the clEnqueueMemFillINTEL function pointer (lazy load) +// Get the clEnqueueMemFillINTEL function pointer clEnqueueMemFillINTEL_fn ggml_openvino_get_clEnqueueMemFillINTEL() { - static clEnqueueMemFillINTEL_fn fn = nullptr; - static bool loaded = false; - if (!loaded) { - loaded = true; - cl_platform_id platform; - if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) { - fn = (clEnqueueMemFillINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueMemFillINTEL"); - } - } - return fn; + return ggml_openvino_get_device_config().cl_mem_fill_fn; } -// Get the clEnqueueMemcpyINTEL function pointer (lazy load) +// Get the clEnqueueMemcpyINTEL function pointer clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL() { - static clEnqueueMemcpyINTEL_fn fn = nullptr; - static bool loaded = false; - if (!loaded) { - loaded = true; - cl_platform_id platform; - if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) { - fn = (clEnqueueMemcpyINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueMemcpyINTEL"); - } - } - return fn; + return ggml_openvino_get_device_config().cl_mem_cpy_fn; } // Get requantization type for a tensor type (returns nullopt if no requant needed) @@ -170,7 +357,8 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * return std::nullopt; } if (strncmp(tensor->name, "token_embd.weight", 17) == 0) { - return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 : ExtraQuantType::Q8_0_C); + return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 : + ExtraQuantType::Q8_0_C); } if (strncmp(tensor->name, "output.weight", 13) == 0) { return ExtraQuantType::Q8_0_C; @@ -178,9 +366,79 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * if (ggml_openvino_is_npu()) { return ExtraQuantType::Q4_0_128; } + // By default Q6_K/Q5_K are requantized to Q8_0_C, which *inflates* 6- and 5-bit weights to 8 + // while the rest of the model stays at 4 bits, and Q4_K keeps its native group-32 layout + // (an f16 scale plus an f16 zero point per 32 weights = 0.125 B/weight of metadata). + // Decode of a large model is bandwidth-bound, so both cost throughput. + // + // GGML_OPENVINO_REQUANT_KQUANT selects a 4-bit target instead. Names are + // q4_[_all]: says whether a per-group zero point is kept, + // is the group size, and the _all suffix sends Q4_K down the same path (without it only + // Q6_K/Q5_K are touched): + // q4_sym128 Q6_K/Q5_K -> Q4_0_128 (u4, group 128, symmetric) + // q4_sym128_all and Q4_K too -- drops Q4_K's per-32 zero point, which costs some accuracy + // q4_asym64 Q6_K/Q5_K -> Q4_1_64 (u4, group 64, asymmetric) + // q4_asym64_all Q6_K/Q5_K and Q4_K -> Q4_1_64 (u4, group 64, asymmetric) + // native no requantization at all (keep Q6_K/Q5_K as they are) + // + // q4_asym64 leaves Q4_K at its native group 32. Use q4_asym64_all to keep the group size uniform. + const char * rq = ggml_openvino_getenv_str("GGML_OPENVINO_REQUANT_KQUANT"); + auto is_opt = [rq](const char * name) { + return rq && strcmp(rq, name) == 0; + }; + const bool sym128 = is_opt("q4_sym128"); + const bool sym128_all = is_opt("q4_sym128_all"); + const bool asym64_all = is_opt("q4_asym64_all"); + const bool asym64 = is_opt("q4_asym64"); + + if (tensor->type == GGML_TYPE_Q4_K) { + if (sym128_all) { + return ExtraQuantType::Q4_0_128; + } + if (asym64_all) { + return ExtraQuantType::Q4_1_64; + } + } + // MoE expert weights (3D, ne[2] = n_expert) stored as Q5_1/Q8_0 are the expert-side + // equivalent of Q6_K/Q5_K: kept at 8 bits by default while the rest of the model is at 4 + // (gemma-4 26B-A4B keeps its down projection there). Send them to 4 bits under the same + // option, at group 64 rather than 128: the down expert has k=704, which 64 divides + // (704/64 = 11) and 128 does not. + if (tensor->ne[2] > 1 && (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0)) { + if (sym128 || sym128_all) { + return ExtraQuantType::Q4_0_64; + } + if (asym64 || asym64_all) { + return ExtraQuantType::Q4_1_64; + } + // TODO: temporary workaround for a known OpenVINO GPU-plugin bug -- remove once the + // plugin computes grouped 8-bit GatherMatmulCompressed correctly. This costs accuracy + // (5/8-bit -> 4-bit) on any model it applies to, so it must not outlive the bug. + // + // On GPU these would otherwise stay in their native *grouped 8-bit* layout, which the GPU + // plugin's GatherMatmulCompressed computes incorrectly -- gemma-4 26B-A4B (whose down + // projection is Q5_1) produces garbage, while the same graph is correct on CPU. It is + // specific to grouped 8 bit: the gate/up experts are grouped u4 *with* a zero point and + // are fine, and Qwen3.5 / granite are fine because their Q5_K/Q6_K down projections + // already requantize to per-channel Q8_0_C (grouped=0). Sending these to grouped 4 bit + // avoids the broken layout and restores correct output. + // Opt out with GGML_OPENVINO_REQUANT_KQUANT=native. + if (ggml_openvino_is_gpu() && !is_opt("native")) { + return ExtraQuantType::Q4_0_64; + } + } switch (tensor->type) { case GGML_TYPE_Q6_K: case GGML_TYPE_Q5_K: + if (sym128 || sym128_all) { + return ExtraQuantType::Q4_0_128; + } + if (asym64 || asym64_all) { + return ExtraQuantType::Q4_1_64; + } + if (is_opt("native")) { + return std::nullopt; + } return ExtraQuantType::Q8_0_C; default: return std::nullopt; @@ -199,14 +457,31 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten return layout; } - // Only handle 2D weight tensors - if (tensor->ne[2] != 1 || tensor->ne[3] != 1) { + // Most quantized weights use the existing 2D extraction path. 3D expert weights for + // MUL_MAT_ID (MoE) are also supported, either as MXFP4 (packed, dedicated branch below) or via the + // generic sizing math below, which is shape-agnostic (based on total element count). Only reject 4D. + if (tensor->ne[3] != 1) { return layout; } + // 3D MoE expert weights that are not requantized (see below) always use the exact f16 + // zero-point extraction (see extract_quantized_weights), which needs a wider zp slot than + // the packed integer zero point -- must be kept in sync with that function so the buffer + // sizing here matches what process_weight_tensor actually writes. + const bool for_gather_matmul = tensor->ne[2] > 1; + int64_t n_elements = ggml_nelements(tensor); const size_t alignment = 64; // Good for SIMD + if (tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) { + layout.weights_per_block = 32; + layout.is_symmetric = true; + layout.weights_size = ggml_nbytes(tensor); + layout.weights_offset = 0; + layout.total_size = layout.weights_size; + return layout; + } + // Check if requantization is needed (NPU-specific) auto requant_type = ggml_openvino_get_requant_type(tensor, use_bias); if (requant_type.has_value()) { @@ -229,6 +504,16 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.weights_per_block = 128; layout.is_symmetric = true; break; + case ExtraQuantType::Q4_1_64: + layout.is_u4 = true; + layout.weights_per_block = 64; + layout.is_symmetric = false; + break; + case ExtraQuantType::Q4_0_64: + layout.is_u4 = true; + layout.weights_per_block = 64; + layout.is_symmetric = true; + break; case ExtraQuantType::Q4_0_C: layout.is_u4 = true; layout.weights_per_block = tensor->ne[0]; @@ -249,9 +534,7 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.weights_per_block = tensor->ne[0]; break; default: - layout.weights_per_block = -1; GGML_ABORT("Code of re-quantizing to channel-wise is not updated"); - break; } if (layout.is_requant) { @@ -259,10 +542,12 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements; int64_t n_blocks = n_elements / layout.weights_per_block; layout.scales_size = n_blocks * sizeof(uint16_t); - // For symmetric quantization, we only need one zp value (not one per block) - // Zero points are stored in U4 or U8 format matching the weight type - size_t n_zp_elements = layout.is_symmetric ? 1 : n_blocks; - layout.zp_size = layout.is_u4 ? ((n_zp_elements + 1) / 2) : n_zp_elements; + // For symmetric quantization, no zp needed (weights stored as signed) + if (layout.is_symmetric) { + layout.zp_size = 0; + } else { + layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks; + } layout.weights_offset = 0; layout.scales_offset = ((layout.weights_size + alignment - 1) / alignment) * alignment; @@ -279,6 +564,7 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.is_symmetric = false; switch (tensor->type) { + case GGML_TYPE_MXFP4: case GGML_TYPE_Q4_0: layout.is_u4 = true; layout.is_symmetric = true; @@ -293,6 +579,10 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.is_symmetric = true; break; + case GGML_TYPE_Q5_1: + // u8 weights (5-bit values), asymmetric (scale + zero point) + break; + case GGML_TYPE_Q6_K: layout.weights_per_block = 16; layout.is_symmetric = true; @@ -310,13 +600,20 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten // Weights: U4 = n_elements/2 bytes, U8 = n_elements bytes layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements; - // Scales: F16 per block + // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block. int64_t n_blocks = n_elements / layout.weights_per_block; - layout.scales_size = n_blocks * sizeof(uint16_t); // F16 = 2 bytes - // Zero points: U4 or U8 matching weight type - // For symmetric quantization, we only need one zp value (not one per block) - size_t n_zp_elements = layout.is_symmetric ? 1 : n_blocks; - layout.zp_size = layout.is_u4 ? ((n_zp_elements + 1) / 2) : n_zp_elements; + layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t)); + // For symmetric quantization, no zp needed (weights stored as signed). Asymmetric + // for_gather_matmul (3D MoE expert) weights use an exact f16 zero point (see + // extract_quantized_weights/make_int8_weights/make_int4_weights), which needs one f16 per + // block instead of a packed u4/u8 integer zero point. + if (layout.is_symmetric) { + layout.zp_size = 0; + } else if (use_bias || for_gather_matmul) { + layout.zp_size = n_blocks * sizeof(uint16_t); + } else { + layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks; + } // Layout in buffer: [weights | scales | zp] with alignment layout.weights_offset = 0; @@ -356,12 +653,11 @@ ggml_openvino_tensor_extra * ggml_openvino_create_tensor_extra(const ggml_tensor return nullptr; } - const auto & device_name = ggml_openvino_get_device_name(); auto remote_context = ggml_openvino_get_remote_context(); std::shared_ptr ov_tensor; if (is_remote) { - GGML_ASSERT(device_name == "GPU"); + GGML_ASSERT(ggml_openvino_is_gpu()); auto gpu_context = remote_context->as(); auto usm_tensor = gpu_context.create_tensor(element_type, shape, tensor->data); ov_tensor = std::make_shared(std::move(usm_tensor)); diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index cd0baf4..3a58b2d 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -15,7 +15,10 @@ #include // ExtraQuantType enum - defines requantization target formats -enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q8_0_C, Q8_0_32 }; +// Q4_1_64: u4, group 64, *true* asymmetric (per-group scale and zero point). Note that +// Q4_0_128/Q4_0_C are symmetric despite taking the unsigned branch of quantize_q4_0 -- that branch +// pins zp to 8 with d = max/-8, which is algebraically symmetric. +enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q4_0_64, Q8_0_C, Q8_0_32, Q4_1_64 }; ov::Core & ov_singleton_core(); @@ -60,11 +63,16 @@ clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL(); struct ggml_openvino_device_config { std::string device_name = "CPU"; + std::vector available_devices; bool is_npu = false; bool initialized = false; std::optional remote_context; + size_t max_alloc_size = SIZE_MAX; ov::AnyMap compile_config; + std::unordered_map environment_variables; cl_command_queue cl_queue = nullptr; + clEnqueueMemFillINTEL_fn cl_mem_fill_fn = nullptr; + clEnqueueMemcpyINTEL_fn cl_mem_cpy_fn = nullptr; void init(); ~ggml_openvino_device_config(); @@ -79,9 +87,50 @@ void ggml_openvino_init_device_config(); // Get the device name const std::string & ggml_openvino_get_device_name(); +// Get all available physical OpenVINO devices +std::vector ggml_openvino_get_available_devices(); + +// Human-readable device name, e.g. "Intel(R) AI Boost (NPU 4000)"; the device id if unavailable +std::string ggml_openvino_get_device_description(const std::string & device_name); + +// Environment variable accessors. All GGML_OPENVINO_* env vars are read once +// during backend init and cached on the device config; consumers must go +// through these helpers (never call ::getenv directly) so behavior stays +// consistent and centralized. +// +// Use ggml_openvino_getenv_str() for string / path values +// (e.g. GGML_OPENVINO_DEVICE, GGML_OPENVINO_CACHE_DIR). The optional +// default_value is returned when the var is unset or empty. +// +// Use ggml_openvino_getenv_int() for boolean toggles and integer settings. +// It returns std::atoi(value) when set, otherwise default_value. For +// boolean use, `if (ggml_openvino_getenv_int(name))` is true iff the value +// is a non-zero integer (so "0" disables, "1" enables). +const char * ggml_openvino_getenv_str(const char * var, const char * default_value = nullptr); +int ggml_openvino_getenv_int(const char * var, int default_value = 0); + +// Memory optimization toggles. GGML_OPENVINO_MEMORY_OPTIMIZE is an umbrella +// switch; the fine-grained env vars still override it when explicitly set. +bool ggml_openvino_reduce_compile_mem_enabled(); +bool ggml_openvino_release_weights_enabled(); + // Check if running on NPU bool ggml_openvino_is_npu(); +// Check if running on a GPU (GPU, GPU.0, GPU.1, ...) +bool ggml_openvino_is_gpu(); + +// Largest single memory object the device can allocate, SIZE_MAX when there is no known limit +size_t ggml_openvino_max_alloc_size(); + +// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS, GPU only). +// register: record a host weight buffer (idempotent per data pointer). +// release: madvise(MADV_DONTNEED) all registered buffers, dropping their RSS. +// released: true once release has run (used to fail-fast on post-release recompile). +void ggml_openvino_register_weight_buffer(void * data, size_t size); +void ggml_openvino_release_weight_buffers(); +bool ggml_openvino_weight_buffers_released(); + // Get requantization type for a tensor type (returns nullopt if no requant needed) std::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false); @@ -115,9 +164,9 @@ struct ggml_openvino_weight_extra : public ggml_openvino_extra_base { // Extra data for quantized weight tensors - stores extracted weights/scales/zp and weight node struct ggml_openvino_quantized_weight_extra : public ggml_openvino_extra_base { - ov::Tensor weights; // U4 or U8 extracted weights - ov::Tensor scales; // F16 scales - ov::Tensor zp; // U4 or U8 zero points (same type as weights) + ov::Tensor weights; // U4 or U8 extracted weights + ov::Tensor scales; // F16 scales + ov::Tensor zp; // U4 or U8 zero points (same type as weights) std::shared_ptr weight_node; // Pre-built OpenVINO weight subgraph ggml_openvino_quantized_weight_extra(ov::Tensor w, ov::Tensor s, ov::Tensor z, std::shared_ptr n) : @@ -132,8 +181,9 @@ struct ggml_openvino_quantized_weight_extra : public ggml_openvino_extra_base { struct ggml_openvino_tensor_extra : public ggml_openvino_extra_base { std::shared_ptr tensor; // For direct use with infer_request - explicit ggml_openvino_tensor_extra(std::shared_ptr t) - : ggml_openvino_extra_base(Type::TENSOR), tensor(std::move(t)) {} + explicit ggml_openvino_tensor_extra(std::shared_ptr t) : + ggml_openvino_extra_base(Type::TENSOR), + tensor(std::move(t)) {} }; // ===================================================== @@ -152,11 +202,11 @@ struct ggml_openvino_extracted_layout { size_t zp_size = 0; // Size of zero points in bytes (U4 or U8) bool is_u4; // true for U4 weights, false for U8 int64_t weights_per_block; // weights per scale/zp block - bool is_symmetric; // true for symmetric quantization + bool is_symmetric; // true for symmetric quantization // Requantization info - bool is_requant = false; // true if this tensor needs requantization - std::optional requant_type; // target requant type if is_requant + bool is_requant = false; // true if this tensor needs requantization + std::optional requant_type; // target requant type if is_requant }; // Calculate the buffer layout for extracted quantized data @@ -164,6 +214,9 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten ggml_openvino_tensor_extra * ggml_openvino_create_tensor_extra(const ggml_tensor * tensor, bool is_remote); +// Check if a tensor's buffer uses remote (device) memory (e.g. GPU USM) +bool ggml_openvino_buffer_is_remote(const ggml_tensor * tensor); + // Register an extra with the tensor's OpenVINO buffer context for proper lifetime management. // This sets tensor->extra and tracks the extra in the buffer context for cleanup. void ggml_openvino_buffer_register_extra(ggml_tensor * tensor, ggml_openvino_extra_base * extra); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 0c8d350..22b2f45 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -4,13 +4,19 @@ #include "ggml-backend.h" #include "ggml-impl.h" #include "ggml-openvino-extra.h" +#include "ggml-openvino/openvino/op_table.h" #include "ggml-openvino/utils.h" #include "ggml-quants.h" #include "ggml.h" +#include "model-cache.h" +#include #include -#include +#include +#include #include +#include +#include #include #include #include @@ -19,18 +25,22 @@ #include #include #include +#include #include +#include +#include #include #include #include -#if defined(_WIN32) +#ifdef _WIN32 # define WIN32_LEAN_AND_MEAN # ifndef NOMINMAX # define NOMINMAX # endif # include #else +# include # include #endif @@ -51,6 +61,7 @@ // - CPU repack buffer: tensor->extra stores tensor_traits with repacked data // ===================================================== +namespace { // Buffer context that manages per-tensor allocations (no contiguous buffer for weights) struct ggml_backend_openvino_buffer_context { int device; @@ -62,11 +73,17 @@ struct ggml_backend_openvino_buffer_context { size_t size; bool is_remote; + // File-backed spill or cache-only virtual memory. + void * spill_mapping = nullptr; + size_t spill_size = 0; + // Wrapping of the buffer std::shared_ptr ov_buffer; // Track all extras for cleanup std::map tensor_extras; + std::map weight_fingerprints; + std::vector source_mappings; // Used for re-allocation on device for kvcache void * data_prev; @@ -88,7 +105,7 @@ struct ggml_backend_openvino_buffer_context { const auto & device_name = ggml_openvino_get_device_name(); if (is_remote) { - GGML_ASSERT(device_name == "GPU"); + GGML_ASSERT(ggml_openvino_is_gpu()); auto remote_context = ggml_openvino_get_remote_context(); auto gpu_context = remote_context->as(); ov::intel_gpu::ocl::USMTensor usm_tensor = @@ -96,10 +113,73 @@ struct ggml_backend_openvino_buffer_context { data = usm_tensor.get(); ov_buffer = std::make_shared(std::move(usm_tensor)); } else { - data = ggml_aligned_malloc(size); - GGML_ASSERT(data); - memset(data, 0, size); - ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data); +#ifdef _WIN32 + if (ggml_openvino_model_cache_only()) { + data = spill_mapping = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + if (data == nullptr) { + return; + } + spill_size = size; + ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data); + } else +#else + if (ggml_openvino_model_cache_only()) { + void * m = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (m == MAP_FAILED) { + return; + } + data = spill_mapping = m; + spill_size = size; + ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data); + } else if (const char * spill_dir = ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) { + // Disk-backed weight buffer: back the repacked weights with a temp file via MAP_SHARED + // instead of anonymous memory. Anonymous pages can only be evicted to swap, so the + // repacked buffer stays pinned alongside the mmap'd source and both are resident at once + // -- that double residency is the load-time peak. File-backed pages are reclaimable: the + // kernel can write them back and drop them under pressure, then re-read on demand, so RSS + // becomes a working set rather than the whole buffer. The file is unlinked immediately, + // so it disappears when the process exits. + // + // The directory must be real storage. Pointing this at a tmpfs mount (/tmp on many + // systems) backs the "spill" with RAM and makes matters worse. + char path[PATH_MAX]; + snprintf(path, sizeof(path), "%s/ggml-ov-weights-%d-XXXXXX", spill_dir, (int) getpid()); + int fd = mkstemp(path); + if (fd < 0) { + GGML_LOG_ERROR("%s: mkstemp(%s) failed: %s\n", __func__, path, strerror(errno)); + return; + } + unlink(path); // anonymous-but-file-backed: freed on process exit + if (ftruncate(fd, (off_t) size) != 0) { + GGML_LOG_ERROR("%s: ftruncate(%zu) failed: %s\n", __func__, size, strerror(errno)); + close(fd); + return; + } + void * m = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + close(fd); // the mapping keeps the file alive + if (m == MAP_FAILED) { + GGML_LOG_ERROR("%s: mmap(%zu) failed: %s\n", __func__, size, strerror(errno)); + return; + } + data = m; + spill_mapping = m; + spill_size = size; + GGML_LOG_INFO("%s: weight buffer spilled to %s (%zu MB, file-backed)\n", __func__, spill_dir, + size / 1024 / 1024); + ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data); + } else +#endif + { +#ifdef _WIN32 + if (ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) { + GGML_LOG_WARN("%s: GGML_OPENVINO_SPILL_DIR is not supported on Windows, ignoring\n", __func__); + } +#endif + data = ggml_aligned_malloc(size); + GGML_ASSERT(data); + memset(data, 0, size); + ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data); + } } if (data == nullptr) { @@ -122,6 +202,15 @@ struct ggml_backend_openvino_buffer_context { delete pair.second; } tensor_extras.clear(); +#ifdef _WIN32 + if (spill_mapping != nullptr) { + VirtualFree(spill_mapping, 0, MEM_RELEASE); + } else +#else + if (spill_mapping != nullptr) { + munmap(spill_mapping, spill_size); + } else +#endif if (!is_remote && data != nullptr) { ggml_aligned_free(data, size); } @@ -133,6 +222,84 @@ struct ggml_backend_openvino_buffer_type_context { int device; std::string name; }; +} // namespace + +// ===================================================== +// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS) +// ===================================================== +// The OpenVINO weight Constants are zero-copy views into the host buffers +// allocated here (ggml_aligned_malloc, anonymous memory). On GPU the plugin +// holds its own device copy after compile_model, so the host pages are dead +// weight for inference and can be dropped to reclaim RSS (~weights size). +// +// We do NOT free the buffer (ggml owns its lifetime and tensors still point +// into it); instead madvise(MADV_DONTNEED) drops the resident pages while +// keeping the mapping valid. A later recompile would re-read these Constants +// from now-zeroed memory and produce garbage, so once released we fail fast +// if the cache-miss compile branch is reached again (see utils.cpp). +namespace { +struct ov_weight_buffer_registry { + std::mutex mutex; + // (data, size) of every non-remote weight buffer, for madvise. + std::vector> buffers; + bool released = false; +}; + +ov_weight_buffer_registry & ov_weight_registry() { + static ov_weight_buffer_registry reg; + return reg; +} +} // namespace + +void ggml_openvino_register_weight_buffer(void * data, size_t size) { + if (data == nullptr || size == 0) { + return; + } + auto & reg = ov_weight_registry(); + std::lock_guard lock(reg.mutex); + for (const auto & b : reg.buffers) { + if (b.first == data) { + return; // already registered + } + } + reg.buffers.emplace_back(data, size); +} + +bool ggml_openvino_weight_buffers_released() { + auto & reg = ov_weight_registry(); + std::lock_guard lock(reg.mutex); + return reg.released; +} + +void ggml_openvino_release_weight_buffers() { + auto & reg = ov_weight_registry(); + std::lock_guard lock(reg.mutex); + if (reg.released) { + return; + } + size_t total = 0; +#if !defined(_WIN32) + for (const auto & b : reg.buffers) { + // Align down/up to page boundaries so madvise only drops whole pages + // fully owned by this buffer. + const size_t page = (size_t) sysconf(_SC_PAGESIZE); + const uintptr_t ustart = reinterpret_cast(b.first); + const size_t offset_to_page = (page - (ustart & (page - 1))) & (page - 1); + if (b.second > offset_to_page) { + const size_t aligned_len = (b.second - offset_to_page) & ~(page - 1); + if (aligned_len > 0) { + char * astart = static_cast(b.first) + offset_to_page; + if (madvise(astart, aligned_len, MADV_DONTNEED) == 0) { + total += aligned_len; + } + } + } + } +#endif + reg.released = true; + GGML_LOG_INFO("%s: released %zu MB of host weight buffers (%zu buffers)\n", __func__, total / 1024 / 1024, + reg.buffers.size()); +} // Buffer interface functions static void ggml_backend_openvino_buffer_free_buffer(ggml_backend_buffer_t buffer) { @@ -145,13 +312,17 @@ static void * ggml_backend_openvino_buffer_get_base(ggml_backend_buffer_t buffer return ctx->data; } +static bool is_stateful_enabled() { + return ggml_openvino_getenv_int("GGML_OPENVINO_STATEFUL_EXECUTION") != 0; +} + static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { // GGML_LOG_DEBUG("%s: buffer usage=%d, tensor name=%s\n", __func__, buffer->usage, tensor->name); ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context; // Put kvcache on device memory for GPU (NPU memory is too small even for kvcache) - if (strncmp(tensor->name, "cache_", 6) == 0 && !ctx->is_remote && ggml_openvino_get_device_name() == "GPU" && - !getenv("GGML_OPENVINO_STATEFUL_EXECUTION")) { + if (strncmp(tensor->name, "cache_", 6) == 0 && !ctx->is_remote && ggml_openvino_is_gpu() && + !is_stateful_enabled()) { GGML_ASSERT(ctx->tensor_extras.empty()); auto device = ctx->device; auto size = ctx->size; @@ -166,6 +337,26 @@ static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_bu if (tensor->view_src != nullptr) { GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft); if (tensor->view_src->extra != nullptr) { + // The cached ov::Tensor carries the shape it was built with, so sharing view_src's + // extra hands out the wrong shape for a reshaping view (e.g. Vcur reshaped from + // [n_embd, n_tokens] to [head_size, n_heads_kv, n_tokens]). When such a view is a + // graph input, binding it fails the shape check. Give it its own extra instead; + // ggml_openvino_create_tensor_extra reads ne and data off the view, so the offset is + // handled too. Only safe for a contiguous view - the ov::Tensor assumes dense strides. + // Skip empty views: they have no data, and on GPU one can sit at the end of the USM buffer. + if (!ggml_are_same_shape(tensor, tensor->view_src) && ggml_is_contiguous(tensor) && + !ggml_is_quantized(tensor->type) && tensor->data != nullptr && ggml_nbytes(tensor) > 0) { + if (ggml_openvino_tensor_extra * extra = + ggml_openvino_create_tensor_extra(tensor, ctx->is_remote)) { + auto it = ctx->tensor_extras.find(tensor); + if (it != ctx->tensor_extras.end()) { + delete it->second; + } + ctx->tensor_extras[tensor] = extra; + tensor->extra = extra; + return GGML_STATUS_SUCCESS; + } + } tensor->extra = tensor->view_src->extra; } return GGML_STATUS_SUCCESS; @@ -201,7 +392,7 @@ static void ggml_backend_openvino_buffer_memset_tensor(ggml_backend_buffer_t buf // For remote (device) buffers, use OpenCL USM memfill cl_command_queue queue = ggml_openvino_get_cl_queue(); auto mem_fill_fn = ggml_openvino_get_clEnqueueMemFillINTEL(); - if (queue != nullptr && mem_fill_fn != nullptr) { + if (mem_fill_fn != nullptr) { uint8_t pattern = value; cl_int err = mem_fill_fn(queue, (char *) tensor->data + offset, &pattern, sizeof(pattern), size, 0, nullptr, nullptr); @@ -210,7 +401,7 @@ static void ggml_backend_openvino_buffer_memset_tensor(ggml_backend_buffer_t buf } clFinish(queue); } else { - GGML_LOG_ERROR("%s: no OpenCL queue or clEnqueueMemFillINTEL not available for GPU buffer\n", __func__); + GGML_LOG_ERROR("%s: clEnqueueMemFillINTEL not available for GPU buffer\n", __func__); } } else { memset((char *) tensor->data + offset, value, size); @@ -230,10 +421,23 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer bool is_weight_buffer = (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS); // Full tensor set: offset=0, full size, not a view bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr); - // 2D tensor (typical weight shape) + if (is_weight_buffer && ggml_openvino_getenv_str("GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR")) { + if (is_full_tensor_set) { + ctx->weight_fingerprints[tensor->data] = ggml_openvino_source_fingerprint(data, size, ctx->source_mappings); + } + if (ggml_openvino_model_cache_only()) { + if (!is_full_tensor_set) { + GGML_ABORT("ggml-openvino: cache-only mode requires whole mmap weight uploads"); + } + return; + } + } + // 2D tensor (typical weight shape), or a 3D quantized MoE expert weight (MUL_MAT_ID). Dense 3D + // expert weights are handled later in create_weight_node instead. bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1); + bool is_supported_weight_shape = is_2d || (tensor->ne[3] == 1 && ggml_is_quantized(tensor->type)); - if (is_weight_buffer && is_full_tensor_set && is_2d) { + if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) { try { auto result = process_weight_tensor(tensor, data, tensor->data); result.weight_node->set_friendly_name(tensor->name); @@ -269,6 +473,22 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer ctx->tensor_extras[tensor] = extra; tensor->extra = extra; + // Register the host buffer so its pages can be dropped after the GPU + // plugin has its own device copy (GGML_OPENVINO_RELEASE_WEIGHTS). + if (!ctx->is_remote) { + // Weights are set once at model load. Setting a weight after a release + // means a second model is loading while the first's compiled graph is + // pinned — that graph would be wrongly reused with this model's key. + // Fail loud rather than return silently-wrong results. + if (ggml_openvino_weight_buffers_released()) { + GGML_ABORT( + "ggml-openvino: loading a new model while GGML_OPENVINO_RELEASE_WEIGHTS pinned a previous " + "model's compiled graph. This mode supports a single model per process; unset it for " + "multi-model runs."); + } + ggml_openvino_register_weight_buffer(ctx->data, ctx->size); + } + } catch (const std::exception & e) { GGML_LOG_ERROR("%s: failed to process weight tensor for %s: %s\n", __func__, tensor->name, e.what()); memcpy((char *) tensor->data + offset, data, size); @@ -278,14 +498,14 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer if (ctx->is_remote) { cl_command_queue queue = ggml_openvino_get_cl_queue(); auto mem_cpy_fn = ggml_openvino_get_clEnqueueMemcpyINTEL(); - if (queue != nullptr && mem_cpy_fn != nullptr) { + if (mem_cpy_fn != nullptr) { cl_int err = mem_cpy_fn(queue, CL_TRUE, (char *) tensor->data + offset, data, size, 0, nullptr, nullptr); if (err != CL_SUCCESS) { GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL failed with error %d\n", __func__, err); } } else { - GGML_LOG_ERROR("%s: no OpenCL queue or clEnqueueMemcpyINTEL not available for GPU buffer\n", __func__); + GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL not available for GPU buffer\n", __func__); } } else { memcpy((char *) tensor->data + offset, data, size); @@ -315,18 +535,22 @@ static void ggml_backend_openvino_buffer_get_tensor(ggml_backend_buffer_t buffer GGML_ASSERT(tensor != nullptr && tensor->data != nullptr); ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context; + if (ggml_openvino_model_cache_only() && buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + GGML_ABORT("ggml-openvino: cannot read unloaded weights in cache-only mode"); + } + if (ctx->is_remote) { // For remote (device) buffers, use OpenCL USM memcpy (device-to-host) cl_command_queue queue = ggml_openvino_get_cl_queue(); auto mem_cpy_fn = ggml_openvino_get_clEnqueueMemcpyINTEL(); - if (queue != nullptr && mem_cpy_fn != nullptr) { + if (mem_cpy_fn != nullptr) { cl_int err = mem_cpy_fn(queue, CL_TRUE, data, (const char *) tensor->data + offset, size, 0, nullptr, nullptr); if (err != CL_SUCCESS) { GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL failed with error %d\n", __func__, err); } } else { - GGML_LOG_ERROR("%s: no OpenCL queue or clEnqueueMemcpyINTEL not available for GPU buffer\n", __func__); + GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL not available for GPU buffer\n", __func__); } } else { memcpy(data, (const char *) tensor->data + offset, size); @@ -344,8 +568,8 @@ static bool ggml_backend_openvino_buffer_cpy_tensor(ggml_backend_buffer_t buffer // For remote (device) buffers, use OpenCL USM memcpy cl_command_queue queue = ggml_openvino_get_cl_queue(); auto mem_cpy_fn = ggml_openvino_get_clEnqueueMemcpyINTEL(); - if (queue == nullptr || mem_cpy_fn == nullptr) { - GGML_LOG_ERROR("%s: no OpenCL queue or clEnqueueMemcpyINTEL not available for GPU buffer\n", __func__); + if (mem_cpy_fn == nullptr) { + GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL not available for GPU buffer\n", __func__); return false; } // Can copy from host to device @@ -362,11 +586,9 @@ static bool ggml_backend_openvino_buffer_cpy_tensor(ggml_backend_buffer_t buffer ggml_backend_openvino_buffer_context * src_ctx = (ggml_backend_openvino_buffer_context *) src->buffer->context; if (src_ctx->is_remote) { - cl_int err = - mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr); + cl_int err = mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr); if (err != CL_SUCCESS) { - GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL (device-to-device) failed with error %d\n", __func__, - err); + GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL (device-to-device) failed with error %d\n", __func__, err); return false; } return true; @@ -389,7 +611,7 @@ static void ggml_backend_openvino_buffer_clear(ggml_backend_buffer_t buffer, uin if (ctx->is_remote) { cl_command_queue queue = ggml_openvino_get_cl_queue(); auto mem_fill_fn = ggml_openvino_get_clEnqueueMemFillINTEL(); - if (queue != nullptr && mem_fill_fn != nullptr) { + if (mem_fill_fn != nullptr) { uint8_t pattern = value; cl_int err = mem_fill_fn(queue, ctx->data, &pattern, sizeof(pattern), ctx->size, 0, nullptr, nullptr); if (err != CL_SUCCESS) { @@ -397,8 +619,7 @@ static void ggml_backend_openvino_buffer_clear(ggml_backend_buffer_t buffer, uin } clFinish(queue); } else { - GGML_LOG_WARN("%s: no OpenCL queue or clEnqueueMemFillINTEL not available for GPU buffer clear\n", - __func__); + GGML_LOG_WARN("%s: clEnqueueMemFillINTEL not available for GPU buffer clear\n", __func__); } } else { memset(ctx->data, value, ctx->size); @@ -448,15 +669,16 @@ static size_t ggml_backend_openvino_buffer_type_get_alignment(ggml_backend_buffe static size_t ggml_backend_openvino_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { GGML_UNUSED(buft); - return SIZE_MAX; + // A GPU caps a single memory object, so let ggml split a large buffer into parts that fit + return ggml_openvino_max_alloc_size(); } static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { GGML_UNUSED(buft); - // For quantized 2D tensors (weights), we need extra space for extracted data - if (ggml_is_quantized(tensor->type) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + // For quantized weight tensors, we need extra space for extracted data. + if (!ggml_openvino_model_cache_only() && ggml_is_quantized(tensor->type) && tensor->ne[3] == 1) { ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor); if (layout.total_size > 0) { // GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n", @@ -470,12 +692,14 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff } static const ggml_backend_buffer_type_i ggml_backend_openvino_buffer_type_interface = { - /* .get_name = */ ggml_backend_openvino_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_openvino_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_openvino_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_openvino_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_openvino_buffer_type_get_alloc_size, - /* .is_host = */ nullptr, + /* .get_name = */ ggml_backend_openvino_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_openvino_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_openvino_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_openvino_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_openvino_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ nullptr, }; // Get buffer type for a specific device @@ -514,9 +738,7 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_buffer_type(in static const char * ggml_backend_openvino_host_buffer_type_get_name(ggml_backend_buffer_type_t buft) { ggml_backend_openvino_buffer_type_context * ctx = (ggml_backend_openvino_buffer_type_context *) buft->context; - static std::string name; - name = ctx->name + "_HOST"; - return name.c_str(); + return ctx->name.c_str(); } static bool ggml_backend_openvino_host_buffer_type_is_host(ggml_backend_buffer_type_t buft) { @@ -525,12 +747,14 @@ static bool ggml_backend_openvino_host_buffer_type_is_host(ggml_backend_buffer_t } static const ggml_backend_buffer_type_i ggml_backend_openvino_host_buffer_type_interface = { - /* .get_name = */ ggml_backend_openvino_host_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_openvino_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_openvino_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_openvino_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_openvino_buffer_type_get_alloc_size, - /* .is_host = */ ggml_backend_openvino_host_buffer_type_is_host, + /* .get_name = */ ggml_backend_openvino_host_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_openvino_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_openvino_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_openvino_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_openvino_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_openvino_host_buffer_type_is_host, }; GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_host_buffer_type(int device) { @@ -549,7 +773,7 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_host_buffer_ty for (int i = 0; i < device_count; i++) { buffer_type_contexts[i].device = i; - buffer_type_contexts[i].name = std::string(GGML_OPENVINO_NAME) + std::to_string(i); + buffer_type_contexts[i].name = std::string(GGML_OPENVINO_NAME) + std::to_string(i) + "_HOST"; buffer_types[i] = ggml_backend_buffer_type{ /* .iface = */ ggml_backend_openvino_host_buffer_type_interface, @@ -574,6 +798,17 @@ size_t ggml_backend_openvino_buffer_get_ctx_id(ggml_backend_buffer_t buffer) { return ctx->id; } +bool ggml_openvino_buffer_is_remote(const ggml_tensor * tensor) { + if (tensor == nullptr || tensor->buffer == nullptr) { + return false; + } + if (!ggml_backend_buffer_is_openvino(tensor->buffer)) { + return false; + } + auto * ctx = static_cast(tensor->buffer->context); + return ctx->is_remote; +} + void ggml_openvino_buffer_register_extra(ggml_tensor * tensor, ggml_openvino_extra_base * extra) { GGML_ASSERT(tensor != nullptr); GGML_ASSERT(tensor->buffer != nullptr); @@ -598,8 +833,38 @@ bool ggml_backend_buft_is_openvino_host(ggml_backend_buffer_type_t buft) { return buft->iface.get_name == ggml_backend_openvino_host_buffer_type_get_name; } +uint64_t ggml_backend_openvino_weight_fingerprint(const ggml_tensor * tensor) { + if (ggml_backend_buffer_is_openvino(tensor->buffer)) { + auto * ctx = static_cast(tensor->buffer->context); + auto it = ctx->weight_fingerprints.find(tensor->data); + if (it != ctx->weight_fingerprints.end()) { + return it->second; + } + GGML_ABORT("ggml-openvino: missing source identity for weight %s", tensor->name); + } + std::vector mappings; + return ggml_openvino_source_fingerprint(tensor->data, ggml_nbytes(tensor), mappings); +} + static void ggml_backend_openvino_free(ggml_backend_t backend) { ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context; + + if (ctx->runtime_context) { + auto r_ctx = std::static_pointer_cast(ctx->runtime_context); + auto cache = r_ctx->compiled_cache; + r_ctx->clear_caches(); + std::lock_guard cache_lock(cache->mutex); + if (--cache->backend_count == 0) { + // If host weight buffers were released (GGML_OPENVINO_RELEASE_WEIGHTS), the + // dropped pages can never be repopulated, so a recompile is impossible. Keep + // the compiled-model cache alive across backend teardown so the next context + // reuses it instead of recompiling against zeroed weights. + if (!ggml_openvino_weight_buffers_released()) { + cache->graphs.clear(); + } + } + } + delete ctx; delete backend; } @@ -634,7 +899,7 @@ static const ggml_backend_i ggml_backend_openvino_interface = { }; int ggml_backend_openvino_get_device_count() { - return 1; + return (int) ggml_openvino_get_available_devices().size(); } static ggml_guid_t ggml_backend_openvino_guid(void) { @@ -644,7 +909,14 @@ static ggml_guid_t ggml_backend_openvino_guid(void) { } static std::shared_ptr get_ov_runtime_context_ptr() { - static std::shared_ptr r_ctx = std::make_shared(); + // Share compiled models, but give every backend its own requests and KV state. + static auto cache = std::make_shared(); + auto r_ctx = std::make_shared(); + r_ctx->device = ggml_openvino_get_device_name(); + r_ctx->stateful = is_stateful_enabled() && !ggml_openvino_is_npu(); + r_ctx->compiled_cache = cache; + std::lock_guard cache_lock(cache->mutex); + ++cache->backend_count; return r_ctx; } @@ -668,10 +940,6 @@ GGML_BACKEND_API ggml_backend_t ggml_backend_openvino_init(int device) { return nullptr; } - std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context); - r_ctx->device = ggml_openvino_get_device_name(); - r_ctx->stateful = getenv("GGML_OPENVINO_STATEFUL_EXECUTION") && !ggml_openvino_is_npu(); - ggml_backend_t openvino_backend = new ggml_backend{ /* .guid = */ ggml_backend_openvino_guid(), /* .interface = */ ggml_backend_openvino_interface, @@ -686,11 +954,125 @@ GGML_BACKEND_API bool ggml_backend_is_openvino(ggml_backend_t backend) { return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_openvino_guid()); } +namespace { struct ggml_backend_openvino_device_context { int device; std::string name; + std::string ov_name; // OpenVINO device id: CPU, GPU, GPU.1, NPU, ... std::string description; + size_t total_memory; }; +} + +static bool ov_device_has_prefix(const std::string & s, const std::string & prefix) { + return s.size() >= prefix.size() && std::equal(prefix.begin(), prefix.end(), s.begin()); +} + +static bool ov_try_get_size_t_property(const std::string & device, const std::string & property, size_t & out) { + try { + const ov::Any value = ov_singleton_core().get_property(device, property); + if (value.is()) { + out = value.as(); + return true; + } + if (value.is()) { + out = (size_t) value.as(); + return true; + } + if (value.is()) { + out = (size_t) value.as(); + return true; + } + if (value.is()) { + const int64_t v = value.as(); + if (v >= 0) { + out = (size_t) v; + return true; + } + } + } catch (...) { + } + return false; +} + +// System memory available to new allocations (MemAvailable on Linux), SIZE_MAX if unknown +static size_t ov_system_available_memory() { +#ifdef _WIN32 + MEMORYSTATUSEX status; + status.dwLength = sizeof(status); + if (GlobalMemoryStatusEx(&status)) { + return (size_t) status.ullAvailPhys; + } +#else + if (FILE * f = fopen("/proc/meminfo", "r")) { + char line[256]; + unsigned long long kb = 0; + bool found = false; + while (!found && fgets(line, sizeof(line), f)) { + found = sscanf(line, "MemAvailable: %llu kB", &kb) == 1; + } + fclose(f); + if (found) { + return (size_t) std::min(kb * 1024, SIZE_MAX); + } + } +#endif + return SIZE_MAX; +} + +// iGPU and NPU allocate from system RAM, so their free memory can't exceed what the OS has available +static bool ov_device_shares_system_memory(const std::string & device) { + if (ov_device_has_prefix(device, "NPU")) { + return true; + } + if (!ov_device_has_prefix(device, "GPU")) { + return false; + } + try { + return ov_singleton_core().get_property(device, ov::device::type) == ov::device::Type::INTEGRATED; + } catch (...) { + return false; + } +} + +// usm_host / usm_shared allocations live in system RAM on a discrete GPU +static bool ov_gpu_stat_is_host_memory(const std::string & key) { + return key == "usm_host" || key == "usm_shared"; +} + +static bool ov_try_get_gpu_used_memory(const std::string & device, size_t & out) { + out = 0; + try { + const ov::Any stats_any = ov_singleton_core().get_property(device, "GPU_MEMORY_STATISTICS"); + if (stats_any.is>()) { + const auto stats = stats_any.as>(); + for (const auto & kv : stats) { + if (!ov_gpu_stat_is_host_memory(kv.first)) { + out += (size_t) kv.second; + } + } + return true; + } + if (stats_any.is()) { + const auto stats = stats_any.as(); + for (const auto & kv : stats) { + if (ov_gpu_stat_is_host_memory(kv.first)) { + continue; + } + if (kv.second.is()) { + out += kv.second.as(); + } else if (kv.second.is()) { + out += (size_t) kv.second.as(); + } else if (kv.second.is()) { + out += (size_t) kv.second.as(); + } + } + return true; + } + } catch (...) { + } + return false; +} static const char * ggml_backend_openvino_device_get_name(ggml_backend_dev_t dev) { ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context; @@ -703,27 +1085,45 @@ static const char * ggml_backend_openvino_device_get_description(ggml_backend_de } static void ggml_backend_openvino_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context; + + // total_memory is only set for GPU/NPU; used = this process's OpenVINO allocations on the device + size_t used = 0; + const bool known = ctx->total_memory > 0 && + (ov_device_has_prefix(ctx->ov_name, "GPU") ? + ov_try_get_gpu_used_memory(ctx->ov_name, used) : + ov_try_get_size_t_property(ctx->ov_name, "NPU_DEVICE_ALLOC_MEM_SIZE", used)); + if (known) { + *total = ctx->total_memory; + *free = (used >= *total) ? 0 : (*total - used); + } else { + // CPU, or a plugin without memory properties: report system memory #ifdef _WIN32 - MEMORYSTATUSEX status; - status.dwLength = sizeof(status); - GlobalMemoryStatusEx(&status); - *total = status.ullTotalPhys; - *free = status.ullAvailPhys; + MEMORYSTATUSEX status; + status.dwLength = sizeof(status); + GlobalMemoryStatusEx(&status); + *total = status.ullTotalPhys; + *free = status.ullAvailPhys; #else - long pages = sysconf(_SC_PHYS_PAGES); - long page_size = sysconf(_SC_PAGE_SIZE); - *total = pages * page_size; + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGE_SIZE); + *total = pages * page_size; - // "free" system memory is ill-defined, for practical purposes assume that all of it is free: - *free = *total; + // "free" system memory is ill-defined, for practical purposes assume that all of it is free: + *free = *total; #endif // _WIN32 + } - GGML_UNUSED(dev); + if (ov_device_shares_system_memory(ctx->ov_name)) { + *free = std::min(*free, ov_system_available_memory()); + } } static enum ggml_backend_dev_type ggml_backend_openvino_device_get_type(ggml_backend_dev_t dev) { - GGML_UNUSED(dev); - return GGML_BACKEND_DEVICE_TYPE_GPU; + ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context; + // Only the device selected by GGML_OPENVINO_DEVICE is offered for offload. The others are + // registered for discovery (--list-devices) only; llama.cpp skips IGPU devices when a GPU exists. + return ctx->ov_name == ggml_openvino_get_device_name() ? GGML_BACKEND_DEVICE_TYPE_GPU : GGML_BACKEND_DEVICE_TYPE_IGPU; } static void ggml_backend_openvino_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { @@ -737,12 +1137,19 @@ static void ggml_backend_openvino_device_get_props(ggml_backend_dev_t dev, ggml_ /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, /* .events = */ false, + /* .mmap_support = */ true, }; } static ggml_backend_t ggml_backend_openvino_device_init(ggml_backend_dev_t dev, const char * params) { GGML_UNUSED(params); ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context; + if (ctx->ov_name != ggml_openvino_get_device_name()) { + // Not an error: test-backend-ops initializes every device + GGML_LOG_WARN("%s: %s (OpenVINO %s) is not the selected device, no ops will run on it; " + "set GGML_OPENVINO_DEVICE=%s to use it\n", + __func__, ctx->name.c_str(), ctx->ov_name.c_str(), ctx->ov_name.c_str()); + } return ggml_backend_openvino_init(ctx->device); } @@ -768,63 +1175,329 @@ static bool has_view_op_input(const ggml_tensor * op) { return false; } +// OV slices whole elements per axis, so each stride must be a multiple of the next smaller one +// (e.g. a batch stride of m*nb[1] + pad bytes cannot be expressed and would be read wrongly). +static bool has_strides_on_element_grid(const ggml_tensor * t) { + std::vector strides; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (t->ne[i] > 1) { + strides.push_back(t->nb[i]); + } + } + std::sort(strides.begin(), strides.end()); + for (size_t i = 1; i < strides.size(); i++) { + if (strides[i - 1] == 0 || strides[i] % strides[i - 1] != 0) { + return false; + } + } + return true; +} + +static bool has_non_contiguous_view_input(const ggml_tensor * op) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (op->src[i] == nullptr) { + break; + } + if (op->src[i]->op == GGML_OP_VIEW && !ggml_is_contiguous(op->src[i])) { + return true; + } + } + return false; +} + static bool is_supported_flash_attn_pattern(const ggml_tensor * op) { - // pattern of q,k,v should be q->op==PERMUTE, q->src[0]->op==VIEW, q->src[0]->src[0]->view_src==nullptr + // Each Q/K/V input must follow one of: + // PERMUTE -> VIEW -> base (view_src==nullptr) (llama KV-cache path) + // PERMUTE -> RESHAPE -> base (view_src==nullptr) (whisper Q) + // VIEW -> base (view_src==nullptr) (whisper K/V from kv_pad) for (int i = 0; i < 3; i++) { const ggml_tensor * src = op->src[i]; - if (src->op != GGML_OP_PERMUTE || src->src[0] == nullptr || src->src[0]->op != GGML_OP_VIEW || - src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) { + if (src->op == GGML_OP_PERMUTE) { + if (src->src[0] == nullptr) { + return false; + } + if (src->src[0]->op != GGML_OP_VIEW && src->src[0]->op != GGML_OP_RESHAPE) { + return false; + } + if (src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) { + return false; + } + } else if (src->op == GGML_OP_VIEW) { + if (src->src[0] == nullptr || src->src[0]->view_src != nullptr) { + return false; + } + } else if (src->op == GGML_OP_CPY) { + if (src->src[0] == nullptr || src->src[0]->op != GGML_OP_PERMUTE || src->src[0]->src[0] == nullptr) { + return false; + } + } else { return false; } } return true; } -static bool is_op_unsupported_case(const ggml_tensor * op) { +static bool is_gemma3n_flash_attn_pattern(const ggml_tensor * op) { + if (!is_supported_flash_attn_pattern(op)) { + return false; + } + + const ggml_tensor * q_base = + op->src[0] != nullptr && op->src[0]->src[0] != nullptr ? op->src[0]->src[0]->src[0] : nullptr; + const ggml_tensor * k_base = + op->src[1] != nullptr && op->src[1]->src[0] != nullptr ? op->src[1]->src[0]->src[0] : nullptr; + const ggml_tensor * v_base = + op->src[2] != nullptr && op->src[2]->src[0] != nullptr ? op->src[2]->src[0]->src[0] : nullptr; + + if (q_base == nullptr || q_base->op != GGML_OP_ROPE) { + return false; + } + + // gemma3n direct attention path (no KV cache): q=ROPE, k=ROPE, v=RMS_NORM + // Only match this specific pattern to avoid falsely catching other models + // (e.g. Gemma4) that also use scale=1.0 with KV-cache backed attention. + const bool is_qkv_direct = + k_base != nullptr && v_base != nullptr && k_base->op == GGML_OP_ROPE && v_base->op == GGML_OP_RMS_NORM; + + return is_qkv_direct; +} + +static bool checked_mul_size(size_t a, size_t b, size_t & out) { + if (a == 0 || b == 0) { + out = 0; + return true; + } + if (a > SIZE_MAX / b) { + return false; + } + out = a * b; + return true; +} + +static bool tensor_view_fits_src_buffer(const ggml_tensor * tensor) { + if (tensor->view_src == nullptr) { + return true; + } + + const size_t src_nbytes = ggml_nbytes(tensor->view_src); + if (tensor->view_offs > src_nbytes) { + return false; + } + + const size_t tensor_nbytes = ggml_nbytes(tensor); + return tensor_nbytes <= src_nbytes - tensor->view_offs; +} + +static bool cpy_output_view_is_supported(const ggml_tensor * op) { + if (op->view_src == nullptr) { + return true; + } + + if (!tensor_view_fits_src_buffer(op)) { + return false; + } + + return ggml_nbytes(op) == 0 || ggml_is_contiguous(op) || GgmlOvDecoder::is_conv_state_writeback(op); +} + +static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) { + const ggml_tensor * as = op->src[0]; + const ggml_tensor * ids = op->src[2]; + if (as == nullptr || ids == nullptr) { + return true; + } + + // The MXFP4 MUL_MAT_ID translation (translate_mul_mat_id_mxfp4_packed in mul_mat_id.cpp) + // materializes selected expert weights with shape [n_tokens, n_used, rows, k]. Skip cases that + // would create a very large temporary and let the scheduler fall back instead. Every other weight + // type goes through GatherMatmul, which never materializes this temporary. + size_t tmp_elems = 1; + if (!checked_mul_size(tmp_elems, static_cast(ids->ne[1]), tmp_elems) || + !checked_mul_size(tmp_elems, static_cast(ids->ne[0]), tmp_elems) || + !checked_mul_size(tmp_elems, static_cast(as->ne[1]), tmp_elems) || + !checked_mul_size(tmp_elems, static_cast(as->ne[0]), tmp_elems)) { + return true; + } + + size_t tmp_bytes = 0; + if (!checked_mul_size(tmp_elems, sizeof(float), tmp_bytes)) { + return true; + } + + static constexpr size_t mul_mat_id_tmp_limit = 1ULL << 30; // 1 GiB + return tmp_bytes > mul_mat_id_tmp_limit; +} + +static bool tensor_name_starts_with(const ggml_tensor * tensor, const char * prefix) { + return tensor != nullptr && strncmp(tensor->name, prefix, strlen(prefix)) == 0; +} + +static bool is_msa_block_mask_expansion(const ggml_tensor * op) { + if (tensor_name_starts_with(op, "msa_")) { + return true; + } + + const ggml_tensor * src = op->src[0]; + while (src != nullptr && (src->op == GGML_OP_RESHAPE || src->op == GGML_OP_REPEAT)) { + if (tensor_name_starts_with(src, "msa_block_mask")) { + return true; + } + src = src->src[0]; + } + + return tensor_name_starts_with(src, "msa_block_mask"); +} + +namespace { +struct ggml_openvino_op_support { + bool is_supported = true; + std::string reason; + + operator bool() const { + return is_supported; + } +}; +} // namespace + +static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { + if (is_msa_block_mask_expansion(op)) { + return {false, "MSA block mask expansion is not supported"}; + } + switch (op->op) { + case GGML_OP_CONCAT: { + if (op->type == GGML_TYPE_I64) { + return {false, "CONCAT with I64 type is not supported"}; + } + // quantized inputs are dequantized, so the output cannot be written in the quantized type + if (ggml_is_quantized(op->type)) { + return {false, "CONCAT with quantized type is not supported"}; + } + if (ggml_openvino_is_gpu() && op->type == GGML_TYPE_BF16 && has_view_op_input(op)) { + return {false, "CONCAT with BF16 type and VIEW input is not supported on GPU"}; + } + break; + } + case GGML_OP_SET: { + const auto nb1 = static_cast(op->op_params[0]); + const auto nb2 = static_cast(op->op_params[1]); + const auto nb3 = static_cast(op->op_params[2]); + + // OpenVINO SET translation currently supports dst layouts that match src0 strides. + if (op->src[0] == nullptr || nb1 != op->src[0]->nb[1] || nb2 != op->src[0]->nb[2] || nb3 != op->src[0]->nb[3]) { + return {false, "SET op with dst nb1=" + std::to_string(nb1) + ", nb2=" + std::to_string(nb2) + ", nb3=" + std::to_string(nb3) + + " that does not match src0 strides nb[1]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[1]) : "null") + + ", nb[2]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[2]) : "null") + + ", nb[3]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[3]) : "null")}; + } + break; + } case GGML_OP_GET_ROWS: case GGML_OP_SET_ROWS: { if (op->ne[3] != 1) { - return true; + return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"}; + } + if (op->op == GGML_OP_GET_ROWS && ggml_openvino_is_gpu() && + op->src[0]->type == GGML_TYPE_BF16) { + return {false, "GET_ROWS with BF16 src0 is not supported on GPU"}; + } + if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || + op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) { + // These are all f16-arithmetic dequant rounding errors that intermittently exceed the + // tight 1e-7 NMSE threshold depending on the random test data (see ggml-quants.cpp + // make_int8_weights/make_int4_weights: dequant is done in f16, not f32, to keep the + // Convert/Subtract/Multiply chain fusable into GatherMatmulCompressed/FullyConnectedCompressed + // for the shared non-test code paths). + return {false, "GET_ROWS/SET_ROWS with ne[0] == 256 and type " + std::string(ggml_type_name(op->src[0]->type)) + + " rejected due to f16-arithmetic dequant rounding errors that intermittently exceed 1e-7 NMSE threshold"}; + } + break; + } + case GGML_OP_RESHAPE: { + if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) { + return {false, "RESHAPE for ffn_norm_exps is not supported"}; } break; } case GGML_OP_ADD: - case GGML_OP_MUL: { + case GGML_OP_MUL: + case GGML_OP_SUB: { if (op->src[1]->op == GGML_OP_PERMUTE) { - return true; + return {false, "ADD/MUL/SUB with PERMUTE src1 is not supported"}; + } + if (op->src[0]->type != op->src[1]->type && + (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16)) { + return {false, "ADD/MUL/SUB with BF16 and a different src1 type is not supported"}; + } + // >8-expert MoE ReduceSum drifts past the 1e-7 tolerance (f32 order vs CPU); intermittent. + if (op->op == GGML_OP_ADD && is_moe_expert_sum_add(op) && op->src[1]->src[0]->ne[1] > 8) { + return {false, "MoE expert-plane sum with more than 8 experts is not supported"}; } for (int i = 0; i < 4; i++) { if (op->src[0]->ne[i] != op->src[1]->ne[i] && (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1)) { - return true; + return {false, "ADD/MUL/SUB with incompatible broadcast shapes: src0->ne[" + std::to_string(i) + "]=" + + std::to_string(op->src[0]->ne[i]) + ", src1->ne[" + std::to_string(i) + "]=" + + std::to_string(op->src[1]->ne[i])}; } } break; } - case GGML_OP_SOFT_MAX: { - if (op->src[2] != nullptr) { - // GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with sinks\n"); - return true; + case GGML_OP_SCALE: { + if (op->type == GGML_TYPE_BF16) { + return {false, "SCALE with BF16 type is not supported"}; } - float scale = 1.0f; - float max_bias = 0.0f; - const auto * op_params = op->op_params; - memcpy(&scale, (const float *) op_params + 0, sizeof(float)); - memcpy(&max_bias, (const float *) op_params + 1, sizeof(float)); - if (max_bias > 0) { - // GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with max_bias > 0\n"); - return true; + break; + } + case GGML_OP_ADD_ID: { + // Keep support aligned with the CPU backend implementation, which only handles f32 inputs/output and i32 ids. + if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32 || + op->src[2]->type != GGML_TYPE_I32) { + return {false, "ADD_ID only supports F32 inputs/output and I32 ids"}; } break; } - case GGML_OP_FLASH_ATTN_EXT: { - if (op->src[4] != nullptr) { - // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with sinks\n"); - return true; + case GGML_OP_DIV: { + // The GPU plugin can fuse broadcast DIV into the preceding FFN GEMM path + // and produce infs for per-channel scale vectors. Keep those DIVs on CPU + // until the fused GPU kernel is reliable. (falied case llama-arch-test mpt) + if (ggml_openvino_is_gpu() && op->src[1]->ne[0] == op->ne[0] && + op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1) { + return {false, "DIV per-channel scale broadcast is not supported on GPU"}; } - if (!is_supported_flash_attn_pattern(op)) { - return true; + break; + } + case GGML_OP_POOL_2D: { + if (ggml_openvino_is_gpu()) { + const int32_t * params = op->op_params; + const int k0 = params[1]; + const int k1 = params[2]; + const int p0 = params[5]; + const int p1 = params[6]; + if ((p0 > 0 || p1 > 0) && (k0 < 3 || k1 < 3)) { + return {false, "POOL_2D with padding and kernel size < 3 is not supported on " + ggml_openvino_get_device_name()}; + } + } + break; + } + case GGML_OP_SUM: { + if (op->src[0]->op == GGML_OP_PERMUTE) { + return {false, "SUM with PERMUTE input is not supported"}; + } + break; + } + case GGML_OP_MEAN: { + if (op->src[0]->op == GGML_OP_PERMUTE && op->src[0]->src[0] != nullptr && op->src[0]->src[0]->op == GGML_OP_VIEW) { + return {false, "MEAN with PERMUTE of VIEW input is not supported"}; } + break; + } + case GGML_OP_SUM_ROWS: { + if (op->src[0]->op == GGML_OP_PERMUTE) { + return {false, "SUM_ROWS with PERMUTE input is not supported"}; + } + break; + } + case GGML_OP_FLASH_ATTN_EXT: { float scale = 1.0f; float max_bias = 0.0f; float logit_softcap = 0.0f; @@ -832,174 +1505,364 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { memcpy(&scale, (const float *) op_params + 0, sizeof(float)); memcpy(&max_bias, (const float *) op_params + 1, sizeof(float)); memcpy(&logit_softcap, (const float *) op_params + 2, sizeof(float)); + + // Keep gemma3n flash-attn pattern on CPU for GPU runs to avoid + // accuracy drift in the OpenVINO path. Restrict by scale=1.0 to avoid + // affecting non-gemma3n models such as Llama-3.2. + if (fabsf(scale - 1.0f) < 1e-6f && is_gemma3n_flash_attn_pattern(op)) { + return {false, "FLASH_ATTN_EXT gemma3n pattern on GPU is not supported"}; + } + + if (op->src[4] != nullptr) { + return {false, "FLASH_ATTN_EXT with sinks is not supported"}; + } + if (!is_supported_flash_attn_pattern(op)) { + return {false, "FLASH_ATTN_EXT unsupported attention pattern"}; + } if (max_bias > 0) { - // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with max_bias > 0\n"); - return true; + return {false, "FLASH_ATTN_EXT with max_bias > 0 (max_bias=" + std::to_string(max_bias) + ") is not supported"}; } if (logit_softcap != 0) { - // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with logit_softcap != 0\n"); - return true; + return {false, "FLASH_ATTN_EXT with logit_softcap != 0 (logit_softcap=" + std::to_string(logit_softcap) + ") is not supported"}; } break; } case GGML_OP_PERMUTE: { - if (op->type == GGML_TYPE_BF16) { - // err msg: [GPU] Could not find a suitable kernel for transpose - // GGML_LOG_WARN("OpenVINO backend does not support PERMUTE with BF16 type\n"); - return true; + if (op->type == GGML_TYPE_BF16 && ggml_openvino_is_gpu()) { + return {false, "PERMUTE with BF16 type is not supported on GPU"}; + } + break; + } + case GGML_OP_DUP: { + // translated as CONT, so only a plain copy + if (op->type != op->src[0]->type || !ggml_are_same_shape(op, op->src[0]) || !ggml_is_contiguous(op->src[0])) { + return {false, "DUP with type conversion or non-contiguous src is not supported"}; } break; } case GGML_OP_CPY: { - if (op->src[1] != op) { - // GGML_LOG_WARN("OpenVINO backend only supports CPY that is a cast\n"); - return true; + if (op->src[0]->type != GGML_TYPE_BF16 && op->src[1]->type == GGML_TYPE_BF16) { + return {false, "CPY with BF16 src[1] type is not supported"}; + } + if (ggml_openvino_is_npu() && (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16)) { + return {false, "CPY with BF16 is not supported is not supported on NPU"}; + } + // CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend. + if (ggml_is_quantized(op->type)) { + return {false, "CPY to quantized destination (e.g. f32 -> q4_0) is numerically unstable"}; + } + if (ggml_nelements(op->src[0]) != ggml_nelements(op->src[1])) { + return {false, "CPY with mismatched element counts is not supported: src0=" + std::to_string(ggml_nelements(op->src[0])) + + " != src1=" + std::to_string(ggml_nelements(op->src[1]))}; + } + // op test case with non-contiguous src or dst + if ((op->ne[0] == 3 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) || + (op->ne[0] == 1 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) || + (op->ne[0] == 2 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2)) { + return {false, "CPY with non-contiguous shape [" + std::to_string(op->ne[0]) + ", " + + std::to_string(op->ne[1]) + ", " + std::to_string(op->ne[2]) + ", " + + std::to_string(op->ne[3]) + "] is not supported"}; + } + if (!cpy_output_view_is_supported(op)) { + return {false, "CPY with non-contiguous output view is not supported"}; } break; } case GGML_OP_MUL_MAT: { - if (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16) { - // Has accuracy issue, try enabling this and see `test-backend-ops -o "MUL_MAT"` - // GGML_LOG_WARN("OpenVINO backend does not support MUL_MAT with two F16 tensors\n"); - return true; + if (ggml_openvino_is_gpu() && op->src[0] != nullptr && op->src[1] != nullptr && + ggml_is_quantized(op->src[0]->type) && strcmp(op->src[0]->name, "a") == 0 && + strcmp(op->src[1]->name, "b") == 0 && op->src[0]->ne[1] == 1 && op->src[1]->ne[1] == 64 && + op->src[0]->ne[0] == 256 && op->src[1]->ne[0] == 256) { + return {false, "MUL_MAT quantized benchmark test case on GPU is not supported"}; } - if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { - return true; + if (ggml_openvino_is_gpu() && op->type == GGML_TYPE_F32 && op->ne[0] == 1 && op->ne[1] == 1 && + (op->src[0]->buffer == nullptr || op->src[0]->buffer->usage != GGML_BACKEND_BUFFER_USAGE_WEIGHTS)) { + return {false, "MUL_MAT scalar dot product with non-weight src[0] on GPU is not supported"}; } - if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) { - return true; + // The GPU plugin fails to compile u4 weights with an f16 zero point for some row counts + // (clFinish CL_OUT_OF_RESOURCES). Op tests build Q4_1/Q4_K weights in that form; model weights use a + // u4 zero point. Op tests check support before allocating, while model loading checks with a dummy + // buffer, so only unbound weights are excluded. Remove once the GPU plugin is fixed. + if (ggml_openvino_is_gpu() && (op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q4_K) && + op->src[0]->buffer == nullptr) { + return {false, "MUL_MAT with unbound Q4_1/Q4_K src[0] on GPU is not supported"}; } - if (ggml_is_quantized(op->src[0]->type) && op->src[0]->ne[1] == 1) { - // MUL_MAT(type_a=q4_0,type_b=f32,m=1,n=2048,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1) - // triggers a bug in ov matmul_shape_inference.hpp - return true; + if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { + return {false, "MUL_MAT with incompatible broadcast on ne[3]: src0->ne[3]=" + std::to_string(op->src[0]->ne[3]) + + ", src1->ne[3]=" + std::to_string(op->src[1]->ne[3])}; } if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) { - return true; + return {false, "MUL_MAT with both inputs as VIEW is not supported"}; + } + break; + } + case GGML_OP_MUL_MAT_ID: { + // Single-expert (or empty) MUL_MAT_ID is a degenerate shape that stresses GatherMatmul edge + // cases and never occurs in real MoE; let it fall back to CPU. + if (op->src[0] != nullptr && op->src[0]->ne[2] <= 1) { + return {false, "MUL_MAT_ID with single-expert or empty ne[2] <= 1 (ne[2]=" + + std::to_string(op->src[0]->ne[2]) + ") is not supported"}; + } + if (ggml_openvino_is_gpu() && op->src[0] != nullptr && !ggml_is_quantized(op->src[0]->type)) { + return {false, "MUL_MAT_ID with non-quantized weights on GPU is not supported"}; + } + // The GPU plugin's GatherMatmul returns wrong values for the layouts test-backend-ops + // produces: it builds a rank-4 input layout ([n_used, n_tokens, k, 1]) instead of rank 3 + // and the kernel misreads it, silently returning garbage (NMSE ~86) rather than asserting. + // The same graph is correct on the CPU plugin, and correct on GPU for every real model, + // which always feeds experts from a bound tensor buffer. Standalone op-test tensors have + // no buffer at all, so use that to exclude them and let the scheduler run them on CPU. + if (ggml_openvino_is_gpu() && op->src[0] != nullptr && op->src[0]->buffer == nullptr) { + return {false, "MUL_MAT_ID with unbound expert tensors on GPU is not supported"}; + } + // Only MXFP4 still needs the large-temporary guard; every other quantized type goes + // through GatherMatmul, which never materializes the selected expert weights. + if (ggml_openvino_is_gpu() && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4 && + mul_mat_id_requires_large_tmp(op)) { + return {false, "MUL_MAT_ID with MXFP4 weights requires large temporary on GPU"}; } break; } case GGML_OP_ROPE: { - const int32_t * op_params = op->op_params; - const int n_dims = op_params[1]; - const int mode = op_params[2]; - if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX) { - // GGML_LOG_WARN("OpenVINO backend does not support ROPE with mode %d\n", mode); - return true; + if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) { + return {false, "ROPE on VIEW / non-contiguous input is not supported"}; } - if (n_dims != 0.0f && n_dims != op->src[0]->ne[0]) { - // GGML_LOG_WARN("OpenVINO backend does not support ROPE with n_dims %d != src[0]->ne[0] %ld\n", n_dims, - // op->src[0]->ne[0]); - return true; + break; + } + case GGML_OP_TRANSPOSE: { + if (op->type == GGML_TYPE_BF16) { + return {false, "TRANSPOSE with BF16 type is not supported"}; } - if (op->type != GGML_TYPE_F32) { - // GGML_LOG_WARN("OpenVINO backend does not support ROPE with type %s\n", ggml_type_name(op->type)); - return true; + break; + } + case GGML_OP_REPEAT: { + if (ggml_openvino_is_gpu() && op->type == GGML_TYPE_BF16) { + return {false, "REPEAT with BF16 type is not supported on GPU"}; } - float freq_scale; - float ext_factor; - memcpy(&freq_scale, op_params + 6, sizeof(float)); - memcpy(&ext_factor, op_params + 7, sizeof(float)); - if (ext_factor != 0.0f) { - // GGML_LOG_WARN("OpenVINO backend does not support ROPE with ext_factor %f != 0.0f\n", ext_factor); - return true; + break; + } + case GGML_OP_GATED_DELTA_NET: { + // enable after https://github.com/openvinotoolkit/openvino/pull/35917 is included in OV release + // return true; + // if (ggml_openvino_is_gpu() && op->src[0]->ne[2] > 1) { + // // CVS-186471 + // return true; + // } + if (op->src[2]->op == GGML_OP_PERMUTE) { + return {false, "GATED_DELTA_NET with PERMUTE src2 is not supported"}; } - if (op->src[0]->op == GGML_OP_VIEW) { - if (op->src[0]->view_src->ne[1] != op->src[0]->ne[2]) { - // GGML_LOG_WARN( - // "OpenVINO backend does not support ROPE with src[0]->view_src->ne[1] %ld != src[0]->ne[2] " - // "%ld\n", - // op->src[0]->view_src->ne[1], op->src[0]->ne[2]); - return true; - } + // kda (per-key-dimension gating) not supported by fused GatedDeltaNet op + if (op->src[3]->ne[0] != 1) { + return {false, "GATED_DELTA_NET with kda (per-key-dimension gating) is not supported"}; + } + // K > 1 (multiple state snapshots) not supported by fused op + if (((const int32_t *) op->op_params)[0] > 1) { + return {false, "GATED_DELTA_NET with K > 1 (multiple state snapshots) is not supported"}; } break; } - default: + case GGML_OP_SSM_CONV: { + // qwen3next is numerically unstable with OpenVINO SSM_CONV. + // Keep this op on CPU until the OpenVINO implementation is fixed. + // return true; break; } - if (op->op == GGML_OP_GET_ROWS) { - if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K)) { - // ERR = 0.000000306 > 0.000000100 GET_ROWS(type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0) - // ERR = 0.000000197 > 0.000000100 GET_ROWS(type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0) - return true; + case GGML_OP_VIEW: { + // Skip TOPK_MOE fused tests until it is fully supported. + // The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe. + if (strcmp(op->name, "selected_experts") == 0) { + return {false, "VIEW for selected_experts (argsort_top_k) is not supported"}; } + break; } - return false; + case GGML_OP_CONV_2D: + case GGML_OP_CONV_2D_DW: { + if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0) { + return {false, "CONV_2D kernel size must be positive"}; + } + if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) { + return {false, "CONV_2D with PERMUTE input is not supported"}; + } + if (has_non_contiguous_view_input(op)) { + return {false, "CONV_2D with non-contiguous view input is not supported"}; + } + const int32_t * params = op->op_params; + const int p0 = params[2]; + const int p1 = params[3]; + const int d0 = params[4]; + const int d1 = params[5]; + const int64_t dilated_kw = (int64_t) d0 * (op->src[0]->ne[0] - 1) + 1; + const int64_t dilated_kh = (int64_t) d1 * (op->src[0]->ne[1] - 1) + 1; + const int64_t padded_w = op->src[1]->ne[0] + 2 * p0; + const int64_t padded_h = op->src[1]->ne[1] + 2 * p1; + if (padded_w < dilated_kw || padded_h < dilated_kh) { + return {false, "CONV_2D padded input is smaller than kernel"}; + } + break; + } + case GGML_OP_CONV_3D: { + if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0 || op->src[0]->ne[2] <= 0) { + return {false, "CONV_3D kernel size must be positive"}; + } + if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) { + return {false, "CONV_3D with PERMUTE input is not supported"}; + } + if (has_non_contiguous_view_input(op)) { + return {false, "CONV_3D with non-contiguous view input is not supported"}; + } + const int32_t * params = op->op_params; + const int p0 = params[3]; + const int p1 = params[4]; + const int p2 = params[5]; + const int d0 = params[6]; + const int d1 = params[7]; + const int d2 = params[8]; + const int64_t dilated_kw = (int64_t) d0 * (op->src[0]->ne[0] - 1) + 1; + const int64_t dilated_kh = (int64_t) d1 * (op->src[0]->ne[1] - 1) + 1; + const int64_t dilated_kd = (int64_t) d2 * (op->src[0]->ne[2] - 1) + 1; + const int64_t padded_w = op->src[1]->ne[0] + 2 * p0; + const int64_t padded_h = op->src[1]->ne[1] + 2 * p1; + const int64_t padded_d = op->src[1]->ne[2] + 2 * p2; + if (padded_w < dilated_kw || padded_h < dilated_kh || padded_d < dilated_kd) { + return {false, "CONV_3D padded input is smaller than kernel"}; + } + break; + } + case GGML_OP_CONV_TRANSPOSE_1D: + case GGML_OP_CONV_TRANSPOSE_2D: { + if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0) { + return {false, "CONV_TRANSPOSE kernel size must be positive"}; + } + if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) { + return {false, "CONV_TRANSPOSE with PERMUTE input is not supported"}; + } + if (has_non_contiguous_view_input(op)) { + return {false, "CONV_TRANSPOSE with non-contiguous view input is not supported"}; + } + break; + } + case GGML_OP_IM2COL: { + if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0) { + return {false, "IM2COL kernel size must be positive"}; + } + break; + } + case GGML_OP_IM2COL_3D: { + if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0 || op->src[0]->ne[2] <= 0) { + return {false, "IM2COL_3D kernel size must be positive"}; + } + break; + } + default: + break; + } + return {true, ""}; } -static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { +static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(ggml_backend_dev_t dev, const ggml_tensor * op) { GGML_ASSERT(dev->reg != nullptr); - static std::set supported_types{GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, - GGML_TYPE_I32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, - GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K}; - - static const std::set supported_ops{GGML_OP_NONE, GGML_OP_ADD, GGML_OP_MUL, GGML_OP_MUL_MAT, GGML_OP_VIEW, - /*GGML_OP_CONT,*/ GGML_OP_RESHAPE, GGML_OP_PERMUTE, GGML_OP_TRANSPOSE, - GGML_OP_GET_ROWS, GGML_OP_ROPE, GGML_OP_RMS_NORM, GGML_OP_SCALE, - // softmax is not updated due to replaced by flash_attn_ext - // GGML_OP_SOFT_MAX, - GGML_OP_SET_ROWS, GGML_OP_FLASH_ATTN_EXT, GGML_OP_CPY}; - static const std::set supported_unary_ops{ - GGML_UNARY_OP_SILU, - }; - static const std::set supported_glu_ops{ - GGML_GLU_OP_SWIGLU, - GGML_GLU_OP_GEGLU, + ggml_backend_openvino_device_context * dev_ctx = (ggml_backend_openvino_device_context *) dev->context; + if (dev_ctx->ov_name != ggml_openvino_get_device_name()) { + // Data placed on a non-selected device (e.g. with -dev) can never run here; stop with a hint + // instead of the generic scheduler abort. Unallocated tensors (test-backend-ops) pass through. + for (int i = -1; i < GGML_MAX_SRC; i++) { + const ggml_tensor * t = i < 0 ? op : op->src[i]; + ggml_backend_buffer_t buf = t == nullptr ? nullptr : (t->view_src ? t->view_src->buffer : t->buffer); + if (buf != nullptr && + (ggml_backend_buft_is_openvino(buf->buft) || ggml_backend_buft_is_openvino_host(buf->buft)) && + ((ggml_backend_openvino_buffer_type_context *) buf->buft->context)->device == dev_ctx->device) { + GGML_ABORT("%s is not the selected OpenVINO device (%s). The OpenVINO device is chosen with the " + "GGML_OPENVINO_DEVICE environment variable, not -dev: set GGML_OPENVINO_DEVICE=%s", + dev_ctx->name.c_str(), ggml_openvino_get_device_name().c_str(), dev_ctx->ov_name.c_str()); + } + } + return {false, "device is not the selected OpenVINO device"}; + } + + static std::unordered_set supported_types{ + GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0, + GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, + GGML_TYPE_MXFP4}; + + // derive supported op sets from the op_table map, keys in + // the map use the full macro name (e.g. "GGML_OP_ADD"), while + // the ggml_*_op_name() helpers return only the trailing part (e.g. "ADD"). + // each set is built once and cached. + static const auto build_supported_sets = [] { + const auto & table = ov::frontend::ggml::get_supported_ops(); + std::unordered_set ops; + std::unordered_set unary_ops; + std::unordered_set glu_ops; + + // GGML_OP_NONE has no translator but is always safe to add to the supported set. + ops.insert(GGML_OP_NONE); + + for (int i = 0; i < GGML_OP_COUNT; ++i) { + const std::string key = std::string("GGML_OP_") + ggml_op_name(static_cast(i)); + if (table.count(key)) { + ops.insert(static_cast(i)); + } + } + for (int i = 0; i < GGML_UNARY_OP_COUNT; ++i) { + const std::string key = std::string("GGML_UNARY_OP_") + ggml_unary_op_name(static_cast(i)); + if (table.count(key)) { + unary_ops.insert(static_cast(i)); + } + } + for (int i = 0; i < GGML_GLU_OP_COUNT; ++i) { + const std::string key = std::string("GGML_GLU_OP_") + ggml_glu_op_name(static_cast(i)); + if (table.count(key)) { + glu_ops.insert(static_cast(i)); + } + } + return std::make_tuple(ops, unary_ops, glu_ops); }; + static const auto supported_sets = build_supported_sets(); + static const auto & supported_ops = std::get<0>(supported_sets); + static const auto & supported_unary_ops = std::get<1>(supported_sets); + static const auto & supported_glu_ops = std::get<2>(supported_sets); switch (op->op) { case GGML_OP_UNARY: { auto supported = supported_unary_ops.find(ggml_get_unary_op(op)) != supported_unary_ops.end(); if (!supported) { - // GGML_LOG_WARN("OpenVINO backend does not support unary op %s\n", ggml_unary_op_name(ggml_get_unary_op(op))); - return false; + return {false, "unary op " + std::string(ggml_unary_op_name(ggml_get_unary_op(op))) + " has no op translator"}; } - if (has_view_op_input(op)) { - // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n", - // ggml_unary_op_name(ggml_get_unary_op(op))); - return false; + if (op->type == GGML_TYPE_F32 && (ggml_get_unary_op(op) == GGML_UNARY_OP_EXP || + ggml_get_unary_op(op) == GGML_UNARY_OP_EXPM1)) { + return {false, "UNARY_EXP / UNARY_EXPM1 with F32 type is not supported"}; } break; } case GGML_OP_GLU: { auto supported = supported_glu_ops.find(ggml_get_glu_op(op)) != supported_glu_ops.end(); if (!supported) { - // GGML_LOG_WARN("OpenVINO backend does not support GLU op %s\n", ggml_glu_op_name(ggml_get_glu_op(op))); - return false; - } - if (has_view_op_input(op)) { - // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n", - // ggml_glu_op_name(ggml_get_glu_op(op))); - return false; + return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " has no op translator"}; } + // if (has_view_op_input(op)) { + // return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " with view input is not supported"}; + // } if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) { // triggers bug in ov gpu - return false; + return {false, "GLU op with odd src0 ne[0] and null src1 is not supported"}; } break; } default: { auto supported = supported_ops.find(op->op) != supported_ops.end(); if (!supported) { - // GGML_LOG_WARN("OpenVINO backend does not support op %s\n", ggml_op_name(op->op)); - return false; + return {false, "op " + std::string(ggml_op_name(op->op)) + " has no op translator"}; } - static std::set ops_not_support_view_input{ - GGML_OP_GET_ROWS, - GGML_OP_RMS_NORM, - }; + static std::set ops_not_support_view_input{}; if (ops_not_support_view_input.find(op->op) != ops_not_support_view_input.end() && has_view_op_input(op)) { - // GGML_LOG_WARN("OpenVINO backend does not support op %s with view input\n", ggml_op_name(op->op)); - return false; + return {false, "op " + std::string(ggml_op_name(op->op)) + " with VIEW input is not supported"}; } } } if (supported_types.find(op->type) == supported_types.end()) { - // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(op->type)); - return false; + return {false, "tensor type " + std::string(ggml_type_name(op->type)) + " is not supported"}; } for (int i = 0; i < GGML_MAX_SRC; i++) { auto * src = op->src[i]; @@ -1007,19 +1870,35 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con break; } if (supported_types.find(src->type) == supported_types.end()) { - // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type)); - return false; + return {false, "src[" + std::to_string(i) + "] type " + std::string(ggml_type_name(src->type)) + " is not supported"}; } - if (ggml_is_quantized(src->type) && src->ne[2] != 1) { - // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n"); - return false; + if (!has_strides_on_element_grid(src)) { + return {false, "src[" + std::to_string(i) + "] strides are not multiples of each other"}; + } + const bool is_supported_3d_moe_expert = + op->op == GGML_OP_MUL_MAT_ID && i == 0 && (src->type == GGML_TYPE_MXFP4 || src->ne[3] == 1); + if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert) { + return {false, "3D quantized tensor for src[" + std::to_string(i) + "] is not supported"}; } } - if (is_op_unsupported_case(op)) { - return false; + auto op_support_case = is_op_supported_case(op); + if (!op_support_case.is_supported) { + return op_support_case; } - return true; + return {true, ""}; +} + +static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + auto res = ggml_backend_openvino_device_supports_op_impl(dev, op); + if (!res.is_supported) { + static const bool log_unsupported = ggml_openvino_getenv_int("GGML_OPENVINO_LOG_UNSUPPORTED_OPS") != 0; + if (log_unsupported) { + GGML_LOG_WARN("OpenVINO op unsupported: op '%s' (%s), type %s: %s\n", + op->name, ggml_op_name(op->op), ggml_type_name(op->type), res.reason.c_str()); + } + } + return res.is_supported; } static bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { @@ -1045,9 +1924,11 @@ static const struct ggml_backend_device_i ggml_backend_openvino_device_interface /* .event_synchronize = */ NULL, }; +namespace { struct ggml_backend_openvino_reg_context { std::vector devices; }; +} static const char * ggml_backend_openvino_reg_get_name(ggml_backend_reg_t reg) { return GGML_OPENVINO_NAME; @@ -1087,15 +1968,26 @@ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_openvino_reg(void) { std::lock_guard lock(mutex); if (!initialized) { ggml_openvino_init(); + const std::vector openvino_devices = ggml_openvino_get_available_devices(); ggml_backend_openvino_reg_context * ctx = new ggml_backend_openvino_reg_context; for (int i = 0; i < ggml_backend_openvino_get_device_count(); i++) { ggml_backend_openvino_device_context * dev_ctx = new ggml_backend_openvino_device_context; dev_ctx->device = i; + // Not the raw OpenVINO id: "CPU" would shadow the ggml CPU backend in ggml_backend_dev_by_name dev_ctx->name = GGML_OPENVINO_NAME + std::to_string(i); - - dev_ctx->description = ov::get_openvino_version().description; + dev_ctx->ov_name = openvino_devices[i]; + // The device is chosen with GGML_OPENVINO_DEVICE, not -dev, so show the value to set + dev_ctx->description = "GGML_OPENVINO_DEVICE=" + dev_ctx->ov_name + + (dev_ctx->ov_name == ggml_openvino_get_device_name() ? " (selected)" : "") + + " - " + ggml_openvino_get_device_description(dev_ctx->ov_name); + dev_ctx->total_memory = 0; + if (ov_device_has_prefix(dev_ctx->ov_name, "GPU")) { + ov_try_get_size_t_property(dev_ctx->ov_name, "GPU_DEVICE_TOTAL_MEM_SIZE", dev_ctx->total_memory); + } else if (ov_device_has_prefix(dev_ctx->ov_name, "NPU")) { + ov_try_get_size_t_property(dev_ctx->ov_name, "NPU_DEVICE_TOTAL_MEM_SIZE", dev_ctx->total_memory); + } ggml_backend_dev_t dev = new ggml_backend_device{/* .interface = */ ggml_backend_openvino_device_interface, @@ -1114,3 +2006,5 @@ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_openvino_reg(void) { return ® } + +GGML_BACKEND_DL_IMPL(ggml_backend_openvino_reg) diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index dbf3864..824d244 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -2,6 +2,7 @@ #include "ggml-common.h" #include "ggml-impl.h" +#include "ggml-openvino-extra.h" #include "ggml.h" #include @@ -19,6 +20,8 @@ #include #include #include +#include +#include #include #include #include @@ -26,10 +29,20 @@ #include #include #include +#include #include #include #include +// From /src/common/transformations/include/transformations/utils/utils.hpp +namespace ov::op::util { +// From /src/common/transformations/include/transformations/utils/utils.hpp +bool get_single_value(const std::shared_ptr & const_node, + float & value, + bool check_value_range = true); +} // namespace ov::op::util + +namespace { void unpack_32_4(const uint8_t * data, uint8_t * dst) { std::fill_n(dst, 16, 0); for (int j = 0; j < 16; ++j) { @@ -44,8 +57,41 @@ void unpack_32_4(const uint8_t * data, uint8_t * dst) { } } +constexpr size_t MXFP4_BLOCK_SIZE = 32; +constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2; +constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE; + +void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) { + for (int j = 0; j < static_cast(MXFP4_BLOCK_QS_SIZE); j += 2) { + const uint8_t v0 = data[j] & 0x0F; + const uint8_t v1 = (data[j + 1] & 0x0F) << 4; + const uint8_t v16 = data[j] >> 4; + const uint8_t v17 = data[j + 1] & 0xF0; + dst[j / 2] = v0 | v1; + dst[MXFP4_BLOCK_SIZE / 4 + j / 2] = v16 | v17; + } +} + +void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr) { + GGML_ASSERT(tensor->type == GGML_TYPE_MXFP4); + GGML_ASSERT(weights_arr.get_element_type() == ov::element::f4e2m1); + GGML_ASSERT(scales_arr.get_element_type() == ov::element::f8e8m0); + + const auto * data = static_cast(tensor->data); + auto * weights = static_cast(weights_arr.data()); + auto * scales = scales_arr.data::value_type>(); + const size_t n_blocks = scales_arr.get_size(); + + ov::parallel_for(n_blocks, [&](size_t i) { + const uint8_t * block = data + i * MXFP4_BLOCK_BYTES; + pack_32_mxfp4_for_openvino(block + sizeof(uint8_t), weights + i * MXFP4_BLOCK_QS_SIZE); + scales[i] = ov::float8_e8m0::from_bits(block[0]); + }); +} + // Extracts (weight, scales, zp) from Q4_0 tensors. // Data layout is: |16 bit scale|32 x 4bit weights|. +// When zp_arr is empty (symmetric), weights are stored as signed i4 (value - 8). void extract_q4_0_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr, @@ -55,28 +101,32 @@ void extract_q4_0_data(const ggml_tensor * tensor, auto * data = static_cast(tensor->data); auto * weights = static_cast(weights_arr.data()); auto * scales = scales_arr.data::value_type>(); - auto * zp = static_cast(zp_arr.data()); - - bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization - // For Q4_0, zero point is always 8 - if (is_scalar_zp) { - zp[0] = 8 | (8 << 4); // Pack two 4-bit values - } + bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4); // Signed i4 path - ov::parallel_for(scales_arr.get_size(), [&](size_t i) { - scales[i] = ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block))); - // For asymmetric quantization, compute per-block zero points - if (!is_scalar_zp) { + if (!is_symmetric) { + auto * zp = static_cast(zp_arr.data()); + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + scales[i] = ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block))); // Pack two 4-bit zero points per byte if (i % 2 == 0) { zp[i / 2] = 8; // Lower nibble } else { zp[i / 2] |= (8 << 4); // Upper nibble } - } - unpack_32_4(data + i * bytes_per_block + 2, weights + i * 16); - }); + unpack_32_4(data + i * bytes_per_block + 2, weights + i * 16); + }); + } else { + // Symmetric: unpack as u4 then convert to i4 by subtracting 8 (XOR each nibble) + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + scales[i] = ov::float16::from_bits(*((uint16_t *) (data + i * bytes_per_block))); + unpack_32_4(data + i * bytes_per_block + 2, weights + i * 16); + // Convert u4 to i4: subtract 8 from each nibble. XOR 0x88 flips each nibble by 8. + for (int j = 0; j < 16; ++j) { + weights[i * 16 + j] ^= 0x88; + } + }); + } } // Extracts (weight, scales, zp) from Q4_1 tensors. @@ -121,8 +171,71 @@ void extract_q4_1_data(const ggml_tensor * tensor, } } +// Extracts (weight, scales, zp) from Q5_1 tensors. +// Data layout is: |16 bit scale|16 bit min|32 bit qh (5th bits)|32 x 4bit low nibbles|. +// Reconstructed quant q in [0,31]: q = (low nibble) | (qh_bit << 4). Dequant: w*d + m. +// Weights are stored as u8 (5-bit values do not fit u4), matching make_int8_weights. +void extract_q5_1_data(const ggml_tensor * tensor, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + bool use_bias) { + const uint64_t bytes_per_block = 24; // 2 scale + 2 min + 4 qh + 16 (32x0.5) weights + const int qk = 32; + + auto * data = static_cast(tensor->data); + auto * weights = static_cast(weights_arr.data()); // u8 weights, one byte per weight + auto * scales = scales_arr.data::value_type>(); + + // Read a 16-bit little-endian value without aliasing/const-qual violations. + auto read_u16 = [](const uint8_t * p) { + uint16_t v; + memcpy(&v, p, sizeof(v)); + return v; + }; + + auto unpack_block = [&](const uint8_t * block, uint8_t * dst) { + uint32_t qh; + memcpy(&qh, block + 4, sizeof(uint32_t)); + const uint8_t * qs = block + 8; + for (int j = 0; j < qk / 2; ++j) { + const uint8_t lo = qs[j] & 0x0F; + const uint8_t hi = qs[j] >> 4; + const uint8_t bit_lo = (qh >> j) & 1; + const uint8_t bit_hi = (qh >> (j + qk / 2)) & 1; + dst[j] = lo | (bit_lo << 4); // first 16 weights + dst[j + qk / 2] = hi | (bit_hi << 4); // last 16 weights + } + }; + + if (use_bias) { + // Store bias (min) directly as f16: dequant w*d + m + auto * bias = zp_arr.data::value_type>(); + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + const uint8_t * block = data + i * bytes_per_block; + float scale = static_cast(ov::float16::from_bits(read_u16(block))); + float min = static_cast(ov::float16::from_bits(read_u16(block + 2))); + scales[i] = ov::float16(scale); + bias[i] = ov::float16(min); + unpack_block(block, weights + i * qk); + }); + } else { + auto * zp = static_cast(zp_arr.data()); // u8 zero points + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + const uint8_t * block = data + i * bytes_per_block; + float scale = static_cast(ov::float16::from_bits(read_u16(block))); + float min = static_cast(ov::float16::from_bits(read_u16(block + 2))); + scales[i] = ov::float16(scale); + // zp = -min / scale (dequant: (w - zp) * s == w*s + min) + zp[i] = (scale != 0.0f) ? (uint8_t) std::lround(-min / scale) : 0; + unpack_block(block, weights + i * qk); + }); + } +} + // Extracts (weight, scales, zp) from Q8_0 tensors. // Data layout is: |16 bit scale|32 x 8bit weights|. +// When zp_arr is empty (symmetric), weights are stored as signed i8 directly. void extract_q8_0_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr, @@ -133,29 +246,30 @@ void extract_q8_0_data(const ggml_tensor * tensor, auto * data = static_cast(tensor->data); auto * weights = static_cast(weights_arr.data()); auto * scales = scales_arr.data::value_type>(); - auto * zp = static_cast(zp_arr.data()); - bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization + bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path - // For Q8_0, zero point is always 128 - if (is_scalar_zp) { - zp[0] = 128; - } - - ov::parallel_for(scales_arr.get_size(), [&](size_t i) { - uint8_t * block_data = data + i * bytes_per_block; - scales[i] = ov::float16::from_bits(*(uint16_t *) block_data); - // For asymmetric quantization, store per-block zero points - if (!is_scalar_zp) { + if (!is_symmetric) { + auto * zp = static_cast(zp_arr.data()); + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + uint8_t * block_data = data + i * bytes_per_block; + scales[i] = ov::float16::from_bits(*(uint16_t *) block_data); zp[i] = 128; - } - for (size_t j = 0; j < weights_per_block; ++j) { - uint8_t x = block_data[j + 2]; // j+2 to skip the scale bytes. - // Original data is in int8_t, so we add a bias of -128 and invert the first bit. - x ^= 1 << 7; - weights[i * weights_per_block + j] = x; - } - }); + for (size_t j = 0; j < weights_per_block; ++j) { + uint8_t x = block_data[j + 2]; + x ^= 1 << 7; // Convert int8 to uint8 by flipping sign bit + weights[i * weights_per_block + j] = x; + } + }); + } else { + // Symmetric: store original int8 values directly (no unsigned bias) + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + uint8_t * block_data = data + i * bytes_per_block; + scales[i] = ov::float16::from_bits(*(uint16_t *) block_data); + // Copy int8 weights as-is (the tensor element type is i8) + memcpy(weights + i * weights_per_block, block_data + 2, weights_per_block); + }); + } } void unpack_256_4(const uint8_t * data, uint8_t * dst) { @@ -256,47 +370,65 @@ void extract_q6_k_data(const ggml_tensor * tensor, auto * data = static_cast(tensor->data); auto * weights = static_cast(weights_arr.data()); auto * scales = scales_arr.data::value_type>(); - auto * zp = static_cast(zp_arr.data()); - - bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization - - // For Q6_K, zero point is always 32 - if (is_scalar_zp) { - zp[0] = 32; - } - - ov::parallel_for(n_super_block, [&](size_t i) { - uint8_t * block_data = data + i * bytes_per_block; - float scale_factor = - static_cast(ov::float16::from_bits(*((uint16_t *) block_data + 104))); // (128+64+16)/2 + bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path - for (size_t j = 0; j < 16; j++) { - scales[j + i * 16] = - ov::float16(scale_factor * static_cast(*((int8_t *) (block_data + 128 + 64 + j)))); - // For asymmetric quantization, store per-block zero points - if (!is_scalar_zp) { + if (!is_symmetric) { + auto * zp = static_cast(zp_arr.data()); + ov::parallel_for(n_super_block, [&](size_t i) { + uint8_t * block_data = data + i * bytes_per_block; + float scale_factor = static_cast(ov::float16::from_bits(*((uint16_t *) block_data + 104))); + for (size_t j = 0; j < 16; j++) { + scales[j + i * 16] = + ov::float16(scale_factor * static_cast(*((int8_t *) (block_data + 128 + 64 + j)))); zp[j + i * 16] = 32; } - } - - uint8_t * ql = block_data; - uint8_t * qh = block_data + 128; - - for (int64_t j = 0; j < 32; ++j) { - weights[i * 256 + j] = (ql[j] & 0xF) | (((qh[j] >> 0) & 3) << 4); - weights[i * 256 + j + 32] = (ql[32 + j] & 0xF) | (((qh[j] >> 2) & 3) << 4); - weights[i * 256 + j + 64] = (ql[j] >> 4) | (((qh[j] >> 4) & 3) << 4); - weights[i * 256 + j + 96] = (ql[32 + j] >> 4) | (((qh[j] >> 6) & 3) << 4); - weights[i * 256 + j + 128] = (ql[64 + j] & 0xF) | (((qh[32 + j] >> 0) & 3) << 4); - weights[i * 256 + j + 160] = (ql[96 + j] & 0xF) | (((qh[32 + j] >> 2) & 3) << 4); - weights[i * 256 + j + 192] = (ql[64 + j] >> 4) | (((qh[32 + j] >> 4) & 3) << 4); - weights[i * 256 + j + 224] = (ql[96 + j] >> 4) | (((qh[32 + j] >> 6) & 3) << 4); - } - }); + uint8_t * ql = block_data; + uint8_t * qh = block_data + 128; + for (int64_t j = 0; j < 32; ++j) { + weights[i * 256 + j] = (ql[j] & 0xF) | (((qh[j] >> 0) & 3) << 4); + weights[i * 256 + j + 32] = (ql[32 + j] & 0xF) | (((qh[j] >> 2) & 3) << 4); + weights[i * 256 + j + 64] = (ql[j] >> 4) | (((qh[j] >> 4) & 3) << 4); + weights[i * 256 + j + 96] = (ql[32 + j] >> 4) | (((qh[j] >> 6) & 3) << 4); + weights[i * 256 + j + 128] = (ql[64 + j] & 0xF) | (((qh[32 + j] >> 0) & 3) << 4); + weights[i * 256 + j + 160] = (ql[96 + j] & 0xF) | (((qh[32 + j] >> 2) & 3) << 4); + weights[i * 256 + j + 192] = (ql[64 + j] >> 4) | (((qh[32 + j] >> 4) & 3) << 4); + weights[i * 256 + j + 224] = (ql[96 + j] >> 4) | (((qh[32 + j] >> 6) & 3) << 4); + } + }); + } else { + // Symmetric: subtract 32 from each weight to store as signed i8 + ov::parallel_for(n_super_block, [&](size_t i) { + uint8_t * block_data = data + i * bytes_per_block; + float scale_factor = static_cast(ov::float16::from_bits(*((uint16_t *) block_data + 104))); + for (size_t j = 0; j < 16; j++) { + scales[j + i * 16] = + ov::float16(scale_factor * static_cast(*((int8_t *) (block_data + 128 + 64 + j)))); + } + uint8_t * ql = block_data; + uint8_t * qh = block_data + 128; + auto * signed_weights = reinterpret_cast(weights); + for (int64_t j = 0; j < 32; ++j) { + signed_weights[i * 256 + j] = static_cast((ql[j] & 0xF) | (((qh[j] >> 0) & 3) << 4)) - 32; + signed_weights[i * 256 + j + 32] = + static_cast((ql[32 + j] & 0xF) | (((qh[j] >> 2) & 3) << 4)) - 32; + signed_weights[i * 256 + j + 64] = static_cast((ql[j] >> 4) | (((qh[j] >> 4) & 3) << 4)) - 32; + signed_weights[i * 256 + j + 96] = + static_cast((ql[32 + j] >> 4) | (((qh[j] >> 6) & 3) << 4)) - 32; + signed_weights[i * 256 + j + 128] = + static_cast((ql[64 + j] & 0xF) | (((qh[32 + j] >> 0) & 3) << 4)) - 32; + signed_weights[i * 256 + j + 160] = + static_cast((ql[96 + j] & 0xF) | (((qh[32 + j] >> 2) & 3) << 4)) - 32; + signed_weights[i * 256 + j + 192] = + static_cast((ql[64 + j] >> 4) | (((qh[32 + j] >> 4) & 3) << 4)) - 32; + signed_weights[i * 256 + j + 224] = + static_cast((ql[96 + j] >> 4) | (((qh[32 + j] >> 6) & 3) << 4)) - 32; + } + }); + } } -static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { +inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { if (j < 4) { *d = q[j] & 63; *m = q[j + 4] & 63; @@ -383,143 +515,524 @@ void extract_q5_k_data(const ggml_tensor * tensor, // TODO Reorder for make_intX_weights +// If for_gather_matmul is true, weight may be N-D (e.g. 3D MoE expert weights [n_expert, rows, cols]). +// The dequantization chain below is built as usual but left in f16 (no final Convert to f32) -- +// ov::pass::MarkDequantization (registered in translate_session.cpp) marks the chain so it survives +// model-build-time ConstantFolding. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul directly +// on top of the resulting f16 chain. ov::Output make_int8_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, - size_t group_size, - bool use_bias) { + size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, + bool use_bias = false, + bool for_gather_matmul = false) { ov::Shape orig_shape = weight.get_shape(); + bool is_signed = (weight.get_element_type() == ov::element::i8); // Symmetric: signed weights, no ZP // Expand dimensions for scales and zp/bias auto scale_shape = scales.get_shape(); - auto zp_shape = zp.get_shape(); - bool is_scalar_zp = zp_shape.empty(); // Symmetric quantization - ov::Shape packed_shape = {orig_shape[0], orig_shape[1] / group_size, group_size}; + // Group the innermost (last) dimension. For 2D weights [rows, cols] this yields + // [rows, cols/group_size, group_size]; for 3D MoE experts [n_expert, rows, cols] this yields + // [n_expert, rows, cols/group_size, group_size]. + ov::Shape packed_shape = orig_shape; + packed_shape.back() /= group_size; + packed_shape.push_back(group_size); + const size_t group_dim = packed_shape.size() - 2; - if (packed_shape[1] == 1) { + if (packed_shape[group_dim] == 1) { // Requantized channel-wise case - packed_shape.erase(packed_shape.begin() + 1); + packed_shape.erase(packed_shape.begin() + group_dim); } else { scale_shape.push_back(1); scales.set_shape(scale_shape); - // For symmetric quantization, zp remains scalar (don't resize) - if (!is_scalar_zp) { + if (!is_signed && zp.get_size() > 0) { + auto zp_shape = zp.get_shape(); zp_shape.push_back(1); zp.set_shape(zp_shape); } } - // Create graph nodes - auto weights_node = std::make_shared(ov::element::u8, packed_shape, - static_cast(weight.data()), nullptr); - weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; auto scales_f16 = std::make_shared(scales); - auto weights_f16 = std::make_shared(weights_node, ov::element::f16); ov::Output result; - if (use_bias && !is_scalar_zp) { - // Bias path: w * s + b (zp tensor holds f16 bias values) - auto bias_f16 = std::make_shared(zp); - auto w_s = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY); + if (is_signed) { + // Signed path: q * s (no zero point subtraction needed) + auto weights_node = std::make_shared(ov::element::i8, packed_shape, + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + auto weights_f16 = std::make_shared(weights_node, ov::element::f16); + auto mul = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); + result = mul; } else { - // Zero point path: (w - zp) * s - auto zero_point = std::make_shared(zp); - float zp_value; - if (ov::op::util::get_single_value(zero_point, zp_value)) { - zero_point = ov::op::v0::Constant::create(zero_point->get_element_type(), {}, {zp_value}); + // Unsigned path + auto weights_node = std::make_shared(ov::element::u8, packed_shape, + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + auto weights_f16 = std::make_shared(weights_node, ov::element::f16); + + if (use_bias && zp.get_size() > 0) { + // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s, where the zero + // point is an exact f16 value zp = -bias/scale (the zp tensor holds bias values + // coming in). Algebraically equal to w*s + bias, but unlike an Add(bias) graph this + // matches CompressedWeightsBlock's pattern (Constant->Convert->Subtract->Multiply), + // so for_gather_matmul weights still fuse into GatherMatmulCompressed. Also avoids + // the round(min/scale) error of an integer zero point. Convert bias -> zero-point IN + // PLACE in the (possibly buffer-backed) zp tensor to avoid a duplicate allocation. + auto * bias_zp_data = zp.data(); + const auto * scale_data = scales.data(); + const size_t n = zp.get_size(); + for (size_t i = 0; i < n; i++) { + float s = static_cast(scale_data[i]); + float b = static_cast(bias_zp_data[i]); + bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f); + } + auto zero_point_f16 = std::make_shared(zp); + auto w_zp = + std::make_shared(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY); + result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); + } else { + // Zero point path: (w - zp) * s + auto zero_point = std::make_shared(zp); + float zp_value; + if (ov::op::util::get_single_value(zero_point, zp_value)) { + zero_point = ov::op::v0::Constant::create(zero_point->get_element_type(), {}, {zp_value}); + } + auto zero_point_f16 = std::make_shared(zero_point, ov::element::f16); + auto w_zp = + std::make_shared(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY); + auto mul = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); + result = mul; } - auto zero_point_f16 = std::make_shared(zero_point, ov::element::f16); - auto w_zp = - std::make_shared(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); } - if (packed_shape.size() != 2) { + if (packed_shape.size() != orig_shape.size()) { // If not requantized channel-wise case, reshape back to original shape auto final_shape = std::make_shared(ov::element::i64, ov::Shape{orig_shape.size()}, orig_shape); - result = std::make_shared(result, final_shape, false); + auto reshaped = std::make_shared(result, final_shape, false); + result = reshaped; } + if (for_gather_matmul) { + return result; + } return std::make_shared(result, ov::element::f32); } +// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights +// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as +// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in +// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see +// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul +// directly from the resulting f16 dequant chain. +// +// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp +// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point); +// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not +// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern. +// See make_int8_weights for the meaning of for_gather_matmul. ov::Output make_int4_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, - size_t group_size, - bool use_bias) { + size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, + bool use_bias = false, + bool for_gather_matmul = false) { ov::Shape orig_weight_shape = weight.get_shape(); + bool is_signed = (weight.get_element_type() == ov::element::i4); // Symmetric: signed weights, no ZP // Expand dimensions for scales and zp/bias ov::Shape scale_shape = scales.get_shape(); - auto zp_shape = zp.get_shape(); - bool is_scalar_zp = zp_shape.empty(); // Symmetric quantization - // Create INT4 weight tensor - ov::Shape packed_shape = {orig_weight_shape[0], orig_weight_shape[1] / group_size, group_size}; + // Create INT4 weight tensor. Group the innermost (last) dimension: for 2D weights + // [rows, cols] this yields [rows, cols/group_size, group_size]; for 3D MoE experts + // [n_expert, rows, cols] this yields [n_expert, rows, cols/group_size, group_size]. + ov::Shape packed_shape = orig_weight_shape; + packed_shape.back() /= group_size; + packed_shape.push_back(group_size); + const size_t group_dim = packed_shape.size() - 2; - if (packed_shape[1] == 1) { + if (packed_shape[group_dim] == 1) { // Requantized channel-wise case - packed_shape.erase(packed_shape.begin() + 1); + packed_shape.erase(packed_shape.begin() + group_dim); } else { scale_shape.push_back(1); scales.set_shape(scale_shape); - // For symmetric quantization, zp remains scalar (don't resize) - if (!is_scalar_zp) { + if (!is_signed && zp.get_size() > 0) { + auto zp_shape = zp.get_shape(); zp_shape.push_back(1); zp.set_shape(zp_shape); } } - auto weights_node = std::make_shared(ov::element::u4, packed_shape, - static_cast(weight.data()), nullptr); - weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; - auto weights_f16 = std::make_shared(weights_node, ov::element::f16); auto scales_f16 = std::make_shared(scales); ov::Output result; - if (use_bias && !is_scalar_zp) { - // Bias path: w * s + b (zp tensor holds f16 bias values) - auto bias_f16 = std::make_shared(zp); - auto w_s = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY); + if (is_signed) { + // Signed path: q * s (no zero point subtraction needed) + auto weights_node = std::make_shared(ov::element::i4, packed_shape, + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + auto weights_f16 = std::make_shared(weights_node, ov::element::f16); + auto mul = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); + result = mul; } else { - // Zero point path: (w - zp) * s - auto zero_points_node = std::make_shared(zp); - float zp_value; - if (ov::op::util::get_single_value(zero_points_node, zp_value)) { - zero_points_node = ov::op::v0::Constant::create(zero_points_node->get_element_type(), {}, {zp_value}); + // Unsigned path + auto weights_node = std::make_shared(ov::element::u4, packed_shape, + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + auto weights_f16 = std::make_shared(weights_node, ov::element::f16); + + if (use_bias && zp.get_size() > 0) { + // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s with an exact f16 + // zp = -bias/scale. Equivalent to w*s + bias but matches CompressedWeightsBlock's + // pattern so for_gather_matmul weights still fuse into GatherMatmulCompressed, and + // avoids the round(min/scale) error of an integer zp. Convert bias -> zero-point IN + // PLACE in the (possibly buffer-backed) zp tensor to avoid a duplicate allocation. + auto * bias_zp_data = zp.data(); + const auto * scale_data = scales.data(); + const size_t n = zp.get_size(); + for (size_t i = 0; i < n; i++) { + float s = static_cast(scale_data[i]); + float b = static_cast(bias_zp_data[i]); + bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f); + } + auto zero_points_f16 = std::make_shared(zp); + auto w_zp = + std::make_shared(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY); + result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); + } else { + // Zero point path: (w - zp) * s + auto zero_points_node = std::make_shared(zp); + float zp_value; + if (ov::op::util::get_single_value(zero_points_node, zp_value)) { + zero_points_node = ov::op::v0::Constant::create(zero_points_node->get_element_type(), {}, {zp_value}); + } + auto zero_points_f16 = std::make_shared(zero_points_node, ov::element::f16); + auto w_zp = + std::make_shared(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY); + auto mul = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); + result = mul; } - auto zero_points_f16 = std::make_shared(zero_points_node, ov::element::f16); - auto w_zp = - std::make_shared(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); } - if (packed_shape.size() != 2) { + if (packed_shape.size() != orig_weight_shape.size()) { // If not requantized channel-wise case, reshape back to original shape auto final_shape = std::make_shared(ov::element::i64, ov::Shape{orig_weight_shape.size()}, orig_weight_shape); - result = std::make_shared(result, final_shape, false); + auto reshaped = std::make_shared(result, final_shape, false); + result = reshaped; } + if (for_gather_matmul) { + return result; + } return std::make_shared(result, ov::element::f32); } +ov::Output make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales) { + const ov::Shape final_shape = weight.get_shape(); + GGML_ASSERT(!final_shape.empty()); + GGML_ASSERT(final_shape.back() % MXFP4_BLOCK_SIZE == 0); + + ov::Shape packed_shape = final_shape; + packed_shape.back() /= MXFP4_BLOCK_SIZE; + packed_shape.push_back(MXFP4_BLOCK_SIZE); + + ov::Shape scale_shape = packed_shape; + scale_shape.back() = 1; + scales.set_shape(scale_shape); + + auto weights_node = std::make_shared(ov::element::f4e2m1, packed_shape, + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + auto weights_f32 = std::make_shared(weights_node, ov::element::f32); + + auto scales_node = std::make_shared(scales); + auto scales_f32 = std::make_shared(scales_node, ov::element::f32); + ov::Output result = + std::make_shared(weights_f32, scales_f32, ov::op::AutoBroadcastType::NUMPY); + + auto final_shape_node = + std::make_shared(ov::element::i64, ov::Shape{final_shape.size()}, final_shape); + return std::make_shared(result, final_shape_node, false); +} + +ov::Output make_mxfp4_moe_packed_weights(ov::Tensor & weight) { + auto weights_node = std::make_shared(ov::element::u8, weight.get_shape(), + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + weights_node->get_rt_info()["__ggml_openvino_mxfp4_moe_packed"] = true; + return weights_node; +} + +void quantize_q4_0(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk) { + assert(k % qk == 0); + const int nb = k / qk; + + auto * weights = static_cast(weights_arr.data()); + auto * scales = scales_arr.data::value_type>(); + bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4); // Signed i4 path + + if (!is_symmetric) { + auto * zp = static_cast(zp_arr.data()); + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + float max = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + if (amax < fabsf(v)) { + amax = fabsf(v); + max = v; + } + } + const float d = max / -8; + if (d == 0) { + scales[i] = ov::float16(1.0f); + if (i % 2 == 0) { + zp[i / 2] = 8; + } else { + zp[i / 2] |= (8 << 4); + } + memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2); + continue; + } + const float id = 1.0f / d; + scales[i] = ov::float16(d); + if (i % 2 == 0) { + zp[i / 2] = 8; + } else { + zp[i / 2] |= (8 << 4); + } + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f)); + const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f)); + weights[i * qk / 2 + j] = xi0 | (xi1 << 4); + } + } + } else { + // Symmetric: produce signed i4 values in [-8, 7] + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + float max = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + if (amax < fabsf(v)) { + amax = fabsf(v); + max = v; + } + } + const float d = max / -8; + if (d == 0) { + scales[i] = ov::float16(1.0f); + // i4 value 0 packed: 0x00 + memset(weights + i * qk / 2, 0, qk / 2); + continue; + } + const float id = 1.0f / d; + scales[i] = ov::float16(d); + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + // Signed i4: range [-8, 7]. Quantize as round(x*id), then pack as 4-bit two's complement. + int8_t si0 = (int8_t) std::max(-8, std::min(7, (int) roundf(x0))); + int8_t si1 = (int8_t) std::max(-8, std::min(7, (int) roundf(x1))); + weights[i * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4); + } + } + } +} + +// Asymmetric u4 quantization with a per-group scale and zero point. +// +// Unlike quantize_q4_0's unsigned branch, which pins the zero point to 8 and is therefore +// symmetric, this keeps a real per-group zero point, so a group whose values are not centred on +// zero does not waste half its range. +void quantize_q4_1_asym(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk) { + assert(k % qk == 0); + const int nb = k / qk; + + auto * weights = static_cast(weights_arr.data()); + auto * scales = scales_arr.data::value_type>(); + auto * zp = static_cast(zp_arr.data()); + + // u4 zero points are packed two per byte, low nibble first, indexed by group -- the same + // convention as the unsigned branch of quantize_q4_0. + auto store_zp = [zp](int i, uint8_t v) { + if (i % 2 == 0) { + zp[i / 2] = v & 0x0F; + } else { + zp[i / 2] |= (uint8_t) ((v & 0x0F) << 4); + } + }; + + for (int i = 0; i < nb; i++) { + float vmin = x[i * qk]; + float vmax = x[i * qk]; + for (int j = 1; j < qk; j++) { + const float v = x[i * qk + j]; + vmin = std::min(vmin, v); + vmax = std::max(vmax, v); + } + // Include 0 in the range so an all-positive or all-negative group still represents zero + // exactly -- these are weights, so an exact zero matters. + vmin = std::min(vmin, 0.0f); + vmax = std::max(vmax, 0.0f); + + const float d = (vmax - vmin) / 15.0f; + if (d == 0.0f) { + scales[i] = ov::float16(1.0f); + store_zp(i, 0); + memset(weights + i * qk / 2, 0, qk / 2); + continue; + } + const float id = 1.0f / d; + + // The zero point is itself a 4-bit integer, so round it and dequantize as (q - zq) * d. + const int zq = std::max(0, std::min(15, (int) lroundf(-vmin * id))); + scales[i] = ov::float16(d); + store_zp(i, (uint8_t) zq); + + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + const uint8_t q0 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x0) + zq)); + const uint8_t q1 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x1) + zq)); + weights[i * qk / 2 + j] = (uint8_t) (q0 | (q1 << 4)); + } + } +} + +void quantize_q8_0(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk, + int64_t block_offset = 0) { + assert(k % qk == 0); + const int nb = k / qk; + + // block_offset lets a caller quantize a chunk of blocks into the right place in the + // output buffers (used for streaming requant). x points at this chunk's first block; + // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no + // nibble packing), so any block boundary is safe. + auto * weights = static_cast(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data::value_type>() + block_offset; + bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path + + if (!is_symmetric) { + auto * zp = static_cast(zp_arr.data()) + block_offset; + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + amax = std::max(amax, fabsf(v)); + } + const float d = amax / 127.0f; + const float id = d ? 1.0f / d : 0.0f; + scales[i] = ov::float16(d); + zp[i] = 128; + for (int j = 0; j < qk; ++j) { + const float x0 = x[i * qk + j] * id; + const int8_t xi0 = roundf(x0); + weights[i * qk + j] = (uint8_t) (xi0 + 128); + } + } + } else { + // Symmetric: store signed int8 values directly + auto * signed_weights = reinterpret_cast(weights); + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + amax = std::max(amax, fabsf(v)); + } + const float d = amax / 127.0f; + const float id = d ? 1.0f / d : 0.0f; + scales[i] = ov::float16(d); + for (int j = 0; j < qk; ++j) { + const float x0 = x[i * qk + j] * id; + signed_weights[i * qk + j] = (int8_t) roundf(x0); + } + } + } +} + +void quantize_q8_1(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk, + int64_t block_offset = 0) { + assert(k % qk == 0); + const int nb = k / qk; + + // See quantize_q8_0: block_offset places this chunk's output at the right block. + auto * weights = static_cast(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data::value_type>() + block_offset; + auto * zp = static_cast(zp_arr.data()) + block_offset; + for (int i = 0; i < nb; i++) { + float min = std::numeric_limits::max(); + float max = std::numeric_limits::lowest(); + + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + min = std::min(v, min); + max = std::max(v, max); + } + + const float d = (max - min) / ((1 << 8) - 1); + const float id = d ? 1.0f / d : 0.0f; + scales[i] = ov::float16(d); + // zp = -min / scale (Q8_1 is asymmetric) + zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0; + + for (int j = 0; j < qk; ++j) { + const float x0 = (x[i * qk + j] - min) * id; + const uint8_t xi0 = roundf(x0); + weights[i * qk + j] = xi0; + } + } +} + // Extract quantized weights from tensor and create weight subgraph +// If weights/scales/zp are provided (non-empty), uses them as output buffers +// Otherwise allocates new ov::Tensors internally +// Returns the weight node (make_int4_weights or make_int8_weights result) std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, - const void * data, + const void * data, // Source data pointer (may differ from tensor->data) ov::Tensor & weights, ov::Tensor & scales, ov::Tensor & zp, - bool use_bias) { + // Use an exact f16 zero point (vs. a rounded integer one); always + // used for for_gather_matmul (3D MoE expert) weights regardless of + // this flag, and also settable explicitly for test-backend-ops. + bool use_bias = false) { // Create a temporary tensor for extraction functions that read from tensor->data ggml_tensor temp_tensor = *tensor; temp_tensor.data = const_cast(data); + if (tensor->type == GGML_TYPE_MXFP4) { + extract_mxfp4_data(&temp_tensor, weights, scales); + auto result = make_mxfp4_weights(weights, scales).get_node_shared_ptr(); + result->set_friendly_name(tensor->name); + return result; + } + // Determine block size based on tensor type int64_t weights_per_block; bool is_u4; @@ -531,6 +1044,7 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, weights_per_block = 32; break; case GGML_TYPE_Q8_0: + case GGML_TYPE_Q5_1: case GGML_TYPE_Q5_K: is_u4 = false; weights_per_block = 32; @@ -544,6 +1058,13 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, std::string(ggml_type_name(tensor->type))); } + // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction + // (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point -- + // round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses + // into GatherMatmulCompressed since it stays a Subtract, not an Add. + const bool for_gather_matmul = tensor->ne[2] > 1; + use_bias = use_bias || for_gather_matmul; + // Extract quantized data switch (tensor->type) { case GGML_TYPE_Q4_0: @@ -555,6 +1076,9 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, case GGML_TYPE_Q4_K: extract_q4_k_data(&temp_tensor, weights, scales, zp, use_bias); break; + case GGML_TYPE_Q5_1: + extract_q5_1_data(&temp_tensor, weights, scales, zp, use_bias); + break; case GGML_TYPE_Q8_0: extract_q8_0_data(&temp_tensor, weights, scales, zp); break; @@ -568,12 +1092,13 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, throw std::runtime_error("Unsupported quantized type: " + std::string(ggml_type_name(tensor->type))); } - // Create the OpenVINO weight subgraph + // Create the OpenVINO weight subgraph. 3D expert weights (MoE) are routed through the + // GatherMatmul-oriented path: dequantized in f16, with constant folding disabled on the chain. ov::Output weight_node; if (is_u4) { - weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias); + weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul); } else { - weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias); + weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul); } auto result = weight_node.get_node_shared_ptr(); @@ -581,37 +1106,90 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, return result; } -// Requantize weights to target format, writing to provided buffers +// Requantize weights from tensor to target format, writing to provided buffers +// For F16 target, only weights buffer is used (scales/zp ignored) +// Returns the weight node std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, - const void * data, + const void * data, // Source data pointer ExtraQuantType requant_type, int64_t block_size, ov::Tensor & weights, ov::Tensor & scales, ov::Tensor & zp) { int64_t n_elements = ggml_nelements(tensor); - - // First dequantize to F32 - std::vector weights_f32(n_elements); - ggml_get_type_traits(tensor->type)->to_float(data, weights_f32.data(), n_elements); - - // Handle F16 case - just convert and create constant - if (requant_type == ExtraQuantType::F16) { - ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements); - auto result = std::make_shared(weights); - result->set_friendly_name(tensor->name); - return result; - } - - // Requantize to target quantized format - bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128); - - if (is_u4) { - quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); - } else if (requant_type == ExtraQuantType::Q8_1_C) { - quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size); + const int64_t ne0 = tensor->ne[0]; // elements per row + const int64_t n_rows = n_elements / ne0; + const auto * type_traits = ggml_get_type_traits(tensor->type); + const size_t src_row_bytes = ggml_row_size(tensor->type, ne0); + + bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128 || + requant_type == ExtraQuantType::Q4_0_64 || requant_type == ExtraQuantType::Q4_1_64); + + // Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM or + // GGML_OPENVINO_MEMORY_OPTIMIZE): instead of + // materializing the full n_elements F32 array (e.g. ~1 GB for token_embd), dequantize + // a chunk of complete rows into a small scratch and quantize/convert it straight into + // the output buffers, capping the transient F32 footprint at CHUNK_ROWS*ne0 floats. + // + // Only valid (and only used) for the Q8_0_C / Q8_1_C / F16 targets whose block size + // divides a row (channel-wise _C uses block_size == ne0) so no target block straddles + // a row boundary, and Q8/F16 have no cross-block packing. The u4 (Q4_0) path packs two + // weights per byte with running zp ORs that assume a single whole-array call, so it is + // never streamed. When the flag is off, behavior is identical to the original + // full-materialization path. + const bool stream_requant = ggml_openvino_reduce_compile_mem_enabled() && !is_u4 && + !(block_size > 0 && ne0 % block_size != 0); + + if (!stream_requant) { + // Full materialization (original behavior): dequantize the whole tensor to F32, + // then convert/quantize in one call. + std::vector weights_f32(n_elements); + type_traits->to_float(data, weights_f32.data(), n_elements); + if (requant_type == ExtraQuantType::F16) { + ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements); + auto result = std::make_shared(weights); + result->set_friendly_name(tensor->name); + return result; + } + if (requant_type == ExtraQuantType::Q4_1_64) { + quantize_q4_1_asym(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } else if (is_u4) { + quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } else if (requant_type == ExtraQuantType::Q8_1_C) { + quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } else { + quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } } else { - quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); + // Streaming path for Q8_0_C / Q8_1_C / F16 (covers token_embd, output.weight, + // and per-layer Q6_K/Q5_K requant — the large transient cases). + const int64_t CHUNK_ROWS = std::min(n_rows, 256); + std::vector scratch(CHUNK_ROWS * ne0); + // F16 destination: 2 bytes/element, advanced per chunk by r0*ne0 elements. + auto * f16_base = static_cast(weights.data()); + for (int64_t r0 = 0; r0 < n_rows; r0 += CHUNK_ROWS) { + const int64_t rows = std::min(CHUNK_ROWS, n_rows - r0); + const int64_t elems = rows * ne0; + const auto * src = static_cast(data) + r0 * src_row_bytes; + type_traits->to_float(src, scratch.data(), elems); + + if (requant_type == ExtraQuantType::F16) { + ggml_get_type_traits(GGML_TYPE_F16) + ->from_float_ref(scratch.data(), f16_base + (r0 * ne0) * sizeof(uint16_t), elems); + } else { + const int64_t block_offset = (r0 * ne0) / block_size; + if (requant_type == ExtraQuantType::Q8_1_C) { + quantize_q8_1(scratch.data(), weights, scales, zp, elems, block_size, block_offset); + } else { + quantize_q8_0(scratch.data(), weights, scales, zp, elems, block_size, block_offset); + } + } + } + if (requant_type == ExtraQuantType::F16) { + auto result = std::make_shared(weights); + result->set_friendly_name(tensor->name); + return result; + } } // Create the OpenVINO weight subgraph @@ -626,6 +1204,7 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, result->set_friendly_name(tensor->name); return result; } +} // namespace OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias) { GGML_ASSERT(tensor != nullptr); @@ -633,8 +1212,11 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo OvWeight result; - // Get 2D shape for weights [rows, cols] - ov::Shape node_shape = {static_cast(tensor->ne[1]), static_cast(tensor->ne[0])}; + // Get shape for weights: [rows, cols], or [n_expert, rows, cols] for 3D MoE expert weights. + ov::Shape node_shape = (tensor->ne[2] > 1) ? + ov::Shape{static_cast(tensor->ne[2]), static_cast(tensor->ne[1]), + static_cast(tensor->ne[0])} : + ov::Shape{static_cast(tensor->ne[1]), static_cast(tensor->ne[0])}; // Handle F16/F32/BF16 weights if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { @@ -676,6 +1258,35 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type)); } + // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see + // extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul" + // check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly. + // Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose + // zp sizing/type is unaffected by for_gather_matmul, so they are excluded here. + const bool for_gather_matmul = tensor->ne[2] > 1; + const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul); + + const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1); + if (is_3d_mxfp4_moe) { + ov::Shape packed_shape = {static_cast(tensor->ne[3]), + static_cast(tensor->ne[2]), + static_cast(tensor->ne[1]), + static_cast(tensor->ne[0] / MXFP4_BLOCK_SIZE), + MXFP4_BLOCK_BYTES}; + const size_t tensor_bytes = ggml_nbytes(tensor); + if (output_base_ptr) { + auto * buf_base = static_cast(output_base_ptr); + memcpy(buf_base + layout.weights_offset, data, tensor_bytes); + result.weights = ov::Tensor(ov::element::u8, packed_shape, buf_base + layout.weights_offset); + } else { + result.weights = ov::Tensor(ov::element::u8, packed_shape); + memcpy(result.weights.data(), data, tensor_bytes); + } + result.weight_node = make_mxfp4_moe_packed_weights(result.weights).get_node_shared_ptr(); + result.weight_node->set_friendly_name(tensor->name); + return result; + } + if (use_bias) { OPENVINO_ASSERT(!layout.is_requant, "use_bias is only used for test-backend-ops, which should not have requantization"); @@ -691,7 +1302,9 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo } else { result.weights = ov::Tensor(ov::element::f16, node_shape); } - ov::Tensor dummy_scales, dummy_zp; // Not used for F16 + // Not used for F16: + ov::Tensor dummy_scales; + ov::Tensor dummy_zp; result.weight_node = requantize_to_buffers(tensor, data, ExtraQuantType::F16, 0, result.weights, dummy_scales, dummy_zp); return result; @@ -699,24 +1312,53 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo // Quantized path (normal extraction or quantized requant) // Create weight/scale/zp tensors - shared between both paths - ov::element::Type weight_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; - ov::Shape scale_shape = {node_shape[0], node_shape[1] / layout.weights_per_block}; - ov::Shape zp_shape = layout.is_symmetric ? ov::Shape{} : scale_shape; + // For symmetric quantization, use signed types (i4/i8) and no ZP tensor + ov::element::Type weight_type; + if (tensor->type == GGML_TYPE_MXFP4) { + weight_type = ov::element::f4e2m1; + } else if (layout.is_symmetric) { + weight_type = layout.is_u4 ? ov::element::i4 : ov::element::i8; + } else { + weight_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; + } + ov::Shape scale_shape = node_shape; + scale_shape.back() /= layout.weights_per_block; + + if (tensor->type == GGML_TYPE_MXFP4) { + if (tensor->ne[2] == 1 && tensor->ne[3] == 1) { + node_shape = {static_cast(tensor->ne[1]), static_cast(tensor->ne[0])}; + } else { + node_shape.clear(); + for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) { + node_shape.push_back(static_cast(tensor->ne[i])); + } + } + + scale_shape = node_shape; + scale_shape.back() /= layout.weights_per_block; + } + + const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; + ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; + if (zp_is_f16) { + zp_type = ov::element::f16; + } if (output_base_ptr) { uint8_t * buf_base = static_cast(output_base_ptr); result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset); - result.scales = ov::Tensor(ov::element::f16, scale_shape, buf_base + layout.scales_offset); - result.zp = ov::Tensor(weight_type, zp_shape, buf_base + layout.zp_offset); + result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset); + if (!layout.is_symmetric) { + result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset); + } + // else: result.zp remains default-constructed (empty) for symmetric } else { result.weights = ov::Tensor(weight_type, node_shape); - result.scales = ov::Tensor(ov::element::f16, scale_shape); - if (use_bias && !layout.is_symmetric) { - // bias only has effect for asymmetric quant - result.zp = ov::Tensor(ov::element::f16, zp_shape); - } else { - result.zp = ov::Tensor(weight_type, zp_shape); + result.scales = ov::Tensor(scale_type, scale_shape); + if (!layout.is_symmetric) { + result.zp = ov::Tensor(zp_type, scale_shape); } + // else: result.zp remains default-constructed (empty) for symmetric } if (layout.is_requant && layout.requant_type.has_value()) { @@ -729,156 +1371,3 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo return result; } - -void quantize_q4_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk) { - assert(k % qk == 0); - const int nb = k / qk; - - auto * weights = static_cast(weights_arr.data()); - auto * scales = scales_arr.data::value_type>(); - auto * zp = static_cast(zp_arr.data()); - bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization - - // For Q4_0, zero point is always 8 - if (is_scalar_zp) { - zp[0] = 8 | (8 << 4); // Pack two 4-bit values - } - - for (int i = 0; i < nb; i++) { - float amax = 0.0f; // absolute max - float max = 0.0f; - - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - if (amax < fabsf(v)) { - amax = fabsf(v); - max = v; - } - } - - const float d = max / -8; - - if (d == 0) { - scales[i] = ov::float16(1.0f); - // zp is already set to 8 for symmetric, or set per-block for asymmetric - if (!is_scalar_zp) { - if (i % 2 == 0) { - zp[i / 2] = 8; - } else { - zp[i / 2] |= (8 << 4); - } - } - memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2); - continue; - } - - const float id = 1.0f / d; - scales[i] = ov::float16(d); - // For asymmetric quantization, store per-block zero points - if (!is_scalar_zp) { - if (i % 2 == 0) { - zp[i / 2] = 8; - } else { - zp[i / 2] |= (8 << 4); - } - } - - for (int j = 0; j < qk / 2; ++j) { - const float x0 = x[i * qk + 2 * j] * id; - const float x1 = x[i * qk + 2 * j + 1] * id; - const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f)); - const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f)); - weights[i * qk / 2 + j] = xi0 | (xi1 << 4); - } - } -} - -void quantize_q8_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk) { - assert(k % qk == 0); - const int nb = k / qk; - - auto * weights = static_cast(weights_arr.data()); - auto * scales = scales_arr.data::value_type>(); - auto * zp = static_cast(zp_arr.data()); - bool is_scalar_zp = (zp_arr.get_size() == 1); // Symmetric quantization - - // For Q8_0, zero point is always 128 - if (is_scalar_zp) { - zp[0] = 128; - } - - for (int i = 0; i < nb; i++) { - float amax = 0.0f; // absolute max - - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - if (amax < fabsf(v)) { - amax = fabsf(v); - } - } - - const float d = amax / 127.0f; - const float id = d ? 1.0f / d : 0.0f; - scales[i] = ov::float16(d); - // For asymmetric quantization, store per-block zero points - if (!is_scalar_zp) { - zp[i] = 128; - } - - for (int j = 0; j < qk; ++j) { - const float x0 = x[i * qk + j] * id; - const int8_t xi0 = roundf(x0); - weights[i * qk + j] = (uint8_t) (xi0 + 128); - } - } -} - -void quantize_q8_1(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk) { - assert(k % qk == 0); - const int nb = k / qk; - - auto * weights = static_cast(weights_arr.data()); - auto * scales = scales_arr.data::value_type>(); - auto * zp = static_cast(zp_arr.data()); - for (int i = 0; i < nb; i++) { - float min = std::numeric_limits::max(); - float max = std::numeric_limits::lowest(); - - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } - } - - const float d = (max - min) / ((1 << 8) - 1); - const float id = d ? 1.0f / d : 0.0f; - scales[i] = ov::float16(d); - // zp = -min / scale (Q8_1 is asymmetric) - zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0; - - for (int j = 0; j < qk; ++j) { - const float x0 = (x[i * qk + j] - min) * id; - const uint8_t xi0 = roundf(x0); - weights[i * qk + j] = xi0; - } - } -} diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index e4a0229..04fe021 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -2,84 +2,12 @@ #include "ggml-openvino-extra.h" // For ExtraQuantType #include "ggml.h" -#include +#include #include #include -void unpack_32_4(const uint8_t* data, uint8_t* dst); - -void extract_q4_0_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr); - -void extract_q4_1_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q8_0_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr); - -void unpack_256_4(const uint8_t* data, uint8_t* dst); - -void extract_q4_k_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q5_k_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q6_k_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr); - static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32; -ov::Output make_int8_weights(ov::Tensor & weight, - ov::Tensor & scales, - ov::Tensor & zp, - size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, - bool use_bias = false); - -ov::Output make_int4_weights(ov::Tensor & weight, - ov::Tensor & scales, - ov::Tensor & zp, - size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, - bool use_bias = false); - -// Extract quantized weights from tensor and create weight subgraph -// If weights/scales/zp are provided (non-empty), uses them as output buffers -// Otherwise allocates new ov::Tensors internally -// Returns the weight node (make_int4_weights or make_int8_weights result) -std::shared_ptr extract_quantized_weights( - const ggml_tensor * tensor, - const void * data, // Source data pointer (may differ from tensor->data) - ov::Tensor & weights, - ov::Tensor & scales, - ov::Tensor & zp, - bool use_bias = false); // Use fp bias instead of quantized zero_point (for test-backend-ops) - -// Requantize weights from tensor to target format, writing to provided buffers -// For F16 target, only weights buffer is used (scales/zp ignored) -// Returns the weight node -std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, - const void * data, // Source data pointer - ExtraQuantType requant_type, - int64_t block_size, - ov::Tensor & weights, - ov::Tensor & scales, - ov::Tensor & zp); - inline const char * extra_quant_type_name(ExtraQuantType t) { switch (t) { case ExtraQuantType::F16: @@ -94,6 +22,10 @@ inline const char * extra_quant_type_name(ExtraQuantType t) { return "Q8_0_32"; case ExtraQuantType::Q8_1_C: return "Q8_1_C"; + case ExtraQuantType::Q4_0_64: + return "Q4_0_64"; + case ExtraQuantType::Q4_1_64: + return "Q4_1_64"; default: return "unknown"; } @@ -120,34 +52,7 @@ OvWeight process_weight_tensor( const ggml_tensor * tensor, const void * data, // Source data pointer (may differ from tensor->data) void * output_base_ptr = nullptr, // Base pointer for output buffers (or nullptr for internal allocation) - bool use_bias = false); // Use fp bias instead of quantized zero_point, only used in test-backend-ops - -void quantize_q4_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk); -void quantize_q8_1(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk); -void quantize_q8_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk); - -namespace ov { -namespace op { -namespace util { -// From /src/common/transformations/include/transformations/utils/utils.hpp -bool get_single_value(const std::shared_ptr& const_node, - float& value, - bool check_value_range = true); -} // namespace util -} // namespace op -} // namespace ov + bool use_bias = false); // Use an exact f16 zero point (vs. a rounded integer one); + // always used for for_gather_matmul (3D MoE expert) weights + // regardless of this flag, and also settable explicitly for + // test-backend-ops. diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp new file mode 100644 index 0000000..c8c5fb5 --- /dev/null +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -0,0 +1,484 @@ +#include "model-cache.h" + +#include "ggml-backend-impl.h" +#include "ggml-backend.h" +#include "ggml-impl.h" +#include "ggml-openvino-extra.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +# include +# include +# include +#else +# include +#endif +#ifdef __linux__ +# include +#endif + +namespace { + +// 64-bit FNV-1a, the mixing primitive for all fingerprints here. +inline uint64_t fnv1a(uint64_t h, const void * data, size_t n) { + const uint8_t * p = static_cast(data); + for (size_t i = 0; i < n; ++i) { + h ^= p[i]; + h *= 0x100000001b3ull; + } + return h; +} + +inline uint64_t fnv1a_u64(uint64_t h, uint64_t v) { + return fnv1a(h, &v, sizeof(v)); +} + +constexpr uint64_t FNV_OFFSET = 0xcbf29ce484222325ull; + +// Fallback when source-file identity is unavailable outside cache-only mode. +constexpr size_t WEIGHT_SAMPLE_BYTES = 4096; + +// Is this src a model weight, mirroring create_weight_nodes()'s selection: +// non-view tensor whose buffer is USAGE_WEIGHTS or whose type is quantized. +bool is_weight_src(const ggml_tensor * src) { + if (src == nullptr || src->view_src != nullptr || src->buffer == nullptr) { + return false; + } + return src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type); +} + +// Weight metadata and source identity; do not read repacked or unloaded buffers. +uint64_t weight_fingerprint(const ggml_tensor * t) { + uint64_t h = FNV_OFFSET; + h = fnv1a(h, t->name, strlen(t->name)); + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + h = fnv1a_u64(h, static_cast(t->ne[i])); + } + h = fnv1a_u64(h, static_cast(t->type)); + const size_t nbytes = ggml_nbytes(t); + h = fnv1a_u64(h, nbytes); + return fnv1a_u64(h, ggml_backend_openvino_weight_fingerprint(t)); +} + +// Walk the cgraph and invoke fn(weight_tensor) for each distinct weight, in node +// order. De-duplicates by tensor pointer so a weight used by several nodes is +// fingerprinted once, deterministically. +template +void for_each_weight(const ggml_cgraph * cgraph, F && fn) { + std::vector seen; + for (int i = 0; i < cgraph->n_nodes; ++i) { + const ggml_tensor * node = cgraph->nodes[i]; + for (int s = 0; s < GGML_MAX_SRC; ++s) { + const ggml_tensor * src = node->src[s]; + if (!is_weight_src(src)) { + continue; + } + bool dup = false; + for (const auto * p : seen) { + if (p == src) { + dup = true; + break; + } + } + if (dup) { + continue; + } + seen.push_back(src); + fn(src); + } + } +} + +std::string ov_version_string() { + const ov::Version v = ov::get_openvino_version(); + return std::string(v.buildNumber ? v.buildNumber : "unknown"); +} + +std::string hex64(uint64_t v) { + char buf[17]; + snprintf(buf, sizeof(buf), "%016llx", static_cast(v)); + return std::string(buf); +} + +// Portable mkdir for a single path component. Returns true if the directory +// exists after the call (created now or already present). +bool make_dir(const std::string & path) { +#if defined(_WIN32) + int rc = _mkdir(path.c_str()); +#else + int rc = ::mkdir(path.c_str(), 0755); +#endif + if (rc == 0 || errno == EEXIST) { + return true; + } + return false; +} + +// Create `path` and any missing parents (like `mkdir -p`). Best-effort: +// returns true only if the full directory exists afterwards. +bool make_dirs(const std::string & path) { + if (path.empty()) { + return false; + } + std::string acc; + for (size_t i = 0; i < path.size(); ++i) { + const char c = path[i]; + acc.push_back(c); + const bool sep = (c == '/' +#if defined(_WIN32) + || c == '\\' +#endif + ); + // Create each intermediate component (skip a leading "/" root). + if (sep && acc.size() > 1) { + std::string component = acc.substr(0, acc.size() - 1); + if (!make_dir(component)) { + return false; + } + } + } + return make_dir(path); +} + +} // namespace + +bool ggml_openvino_model_cache_only() { + return ggml_openvino_getenv_int("GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY") != 0; +} + +static const char * cache_settings[] = { + "GGML_OPENVINO_REQUANT_KQUANT", + "GGML_OPENVINO_NATIVE_SOFTPLUS", + "GGML_OPENVINO_DISABLE_KV_SLICE", + "GGML_OPENVINO_MANUAL_GQA_ATTN", + "GGML_OPENVINO_STATEFUL_EXECUTION", + "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", + "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS", + "GGML_OPENVINO_REDUCE_COMPILE_MEM", + "GGML_OPENVINO_MEMORY_OPTIMIZE", + "GGML_OPENVINO_PROFILING", +}; + +void ggml_openvino_model_cache_init() { + const bool cache_only = ggml_openvino_model_cache_only(); + const std::string dir = ggml_openvino_model_cache_dir(); + if (dir.empty()) { + if (cache_only) { + GGML_ABORT("ggml-openvino: cache-only mode requires GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR"); + } + return; + } + if (cache_only && (ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC") || + ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE") || + ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK"))) { + GGML_ABORT("ggml-openvino: cache-only mode requires dynamic CPU/GPU execution with caching and without fallback"); + } +#if !defined(__linux__) && !defined(_WIN32) + if (cache_only) { + GGML_ABORT("ggml-openvino: cache-only mmap identification requires Linux or Windows"); + } +#endif + if (cache_only) { + auto & config = ggml_openvino_get_device_config(); + config.environment_variables.erase("GGML_OPENVINO_SPILL_DIR"); + config.environment_variables["GGML_OPENVINO_RELEASE_WEIGHTS"] = "0"; + } +} + +uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::vector & mappings) { + const uintptr_t address = reinterpret_cast(data); + auto contains = [&](const ggml_openvino_source_mapping & m) { + return address >= m.begin && address < m.end && size <= m.end - address; + }; + auto fingerprint = [&](const ggml_openvino_source_mapping & m) { + return fnv1a_u64(m.identity, m.offset + address - m.begin); + }; + for (const auto & m : mappings) { + if (contains(m)) { + return fingerprint(m); + } + } +#ifdef __linux__ + std::ifstream maps("/proc/self/maps"); + std::string line; + while (std::getline(maps, line)) { + unsigned long long begin, end, offset, inode; + unsigned int dev_major, dev_minor; + char permissions[5]; + int path_start = 0; + if (sscanf(line.c_str(), "%llx-%llx %4s %llx %x:%x %llu %n", &begin, &end, permissions, + &offset, &dev_major, &dev_minor, &inode, &path_start) != 7 || inode == 0) { + continue; + } + ggml_openvino_source_mapping m{uintptr_t(begin), uintptr_t(end), offset, FNV_OFFSET}; + if (!contains(m)) { + continue; + } + struct stat st; + const std::string path = line.substr(path_start); + if (stat(path.c_str(), &st) != 0 || !S_ISREG(st.st_mode) || uint64_t(st.st_ino) != inode || + major(st.st_dev) != dev_major || minor(st.st_dev) != dev_minor) { + break; + } + m.identity = fnv1a_u64(m.identity, st.st_dev); + m.identity = fnv1a_u64(m.identity, st.st_ino); + m.identity = fnv1a_u64(m.identity, st.st_size); + m.identity = fnv1a_u64(m.identity, st.st_mtim.tv_sec); + m.identity = fnv1a_u64(m.identity, st.st_mtim.tv_nsec); + m.identity = fnv1a_u64(m.identity, st.st_ctim.tv_sec); + m.identity = fnv1a_u64(m.identity, st.st_ctim.tv_nsec); + mappings.push_back(m); + return fingerprint(m); + } +#elif defined(_WIN32) + MEMORY_BASIC_INFORMATION memory; + if (VirtualQuery(data, &memory, sizeof(memory)) == sizeof(memory) && memory.Type == MEM_MAPPED) { + std::wstring name(MAX_PATH, L'\0'); + DWORD length = 0; + while (name.size() <= 32768) { + length = GetMappedFileNameW(GetCurrentProcess(), const_cast(data), name.data(), static_cast(name.size())); + if (length == 0 || length < name.size() - 1) { + break; + } + name.resize(name.size() * 2); + } + if (length > 0 && length < name.size() - 1) { + name.resize(length); + const std::wstring path = L"\\\\?\\GLOBALROOT" + name; + HANDLE file = CreateFileW(path.c_str(), FILE_READ_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (file != INVALID_HANDLE_VALUE) { + BY_HANDLE_FILE_INFORMATION info; + FILE_BASIC_INFO basic; + const bool valid = GetFileInformationByHandle(file, &info) && + GetFileInformationByHandleEx(file, FileBasicInfo, &basic, sizeof(basic)); + CloseHandle(file); + if (valid) { + const uint64_t file_size = (uint64_t(info.nFileSizeHigh) << 32) | info.nFileSizeLow; + const uintptr_t begin = reinterpret_cast(memory.AllocationBase); + if (file_size <= std::numeric_limits::max() - begin) { + ggml_openvino_source_mapping m{begin, begin + static_cast(file_size), 0, + fnv1a(FNV_OFFSET, "win32", 5)}; + if (contains(m)) { + m.identity = fnv1a_u64(m.identity, info.dwVolumeSerialNumber); + m.identity = fnv1a_u64(m.identity, (uint64_t(info.nFileIndexHigh) << 32) | info.nFileIndexLow); + m.identity = fnv1a_u64(m.identity, file_size); + m.identity = fnv1a_u64(m.identity, (uint64_t(info.ftLastWriteTime.dwHighDateTime) << 32) | + info.ftLastWriteTime.dwLowDateTime); + m.identity = fnv1a_u64(m.identity, static_cast(basic.ChangeTime.QuadPart)); + mappings.push_back(m); + return fingerprint(m); + } + } + } + } + } + } +#endif + if (ggml_openvino_model_cache_only()) { + GGML_ABORT("ggml-openvino: could not identify mapped GGUF weight; use --load-mode mmap"); + } + uint64_t h = FNV_OFFSET; + const size_t head = std::min(size, WEIGHT_SAMPLE_BYTES); + h = fnv1a(h, data, head); + if (size > head) { + const size_t tail = std::min(size - head, WEIGHT_SAMPLE_BYTES); + h = fnv1a(h, static_cast(data) + size - tail, tail); + } + return h; +} + +std::string ggml_openvino_model_cache_dir() { + const char * dir = ggml_openvino_getenv_str("GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR"); + if (!dir || strlen(dir) == 0) { + return std::string(); + } + std::string path(dir); + if (ggml_openvino_model_cache_only()) { + return path; + } + // Create the cache directory (and parents) on first use so callers don't + // have to pre-create it; a missing dir would otherwise silently disable the + // cache (manifest/blob writes fail with no directory to write into). + if (!make_dirs(path)) { + GGML_LOG_WARN("ggml-openvino: could not create model cache dir '%s' (errno=%d); caching disabled\n", + path.c_str(), errno); + return std::string(); + } + return path; +} + +std::string ggml_openvino_model_cache_temp_path(const std::string & path) { +#ifdef _WIN32 + const int pid = _getpid(); +#else + const int pid = getpid(); +#endif + return path + ".tmp." + std::to_string(pid) + "." + std::to_string(ggml_time_us()); +} + +uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph, + const std::string & device, + bool fa, + const int32_t * rope_params, + int rope_len, + uint64_t extra_cfg, + const std::string & graph_signature) { + uint64_t h = FNV_OFFSET; + h = fnv1a_u64(h, 2); + h = fnv1a(h, graph_signature.data(), graph_signature.size()); + for (const char * name : cache_settings) { + const char * value = ggml_openvino_getenv_str(name, ""); + h = fnv1a(h, value, strlen(value) + 1); + } + if (const char * debug_nodes = ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + h = fnv1a(h, "GGML_OPENVINO_DEBUG_NODE", sizeof("GGML_OPENVINO_DEBUG_NODE")); + h = fnv1a(h, debug_nodes, strlen(debug_nodes) + 1); + } + if (ggml_openvino_is_gpu() && ggml_openvino_getenv_int("GGML_OPENVINO_MOE_OP", 1) == 0) { + h = fnv1a(h, "GGML_OPENVINO_MOE_OP=0", sizeof("GGML_OPENVINO_MOE_OP=0")); + } + + // Topology: node count + each node's op and name (cheap, and distinguishes + // graphs that share weights but differ structurally). + h = fnv1a_u64(h, static_cast(cgraph->n_nodes)); + for (int i = 0; i < cgraph->n_nodes; ++i) { + const ggml_tensor * node = cgraph->nodes[i]; + h = fnv1a_u64(h, static_cast(node->op)); + h = fnv1a(h, node->name, strlen(node->name)); + } + + // Weights: the model identity. + for_each_weight(cgraph, [&](const ggml_tensor * t) { h = fnv1a_u64(h, weight_fingerprint(t)); }); + + // Device, model parameters, and backend configuration. + h = fnv1a(h, device.data(), device.size()); + h = fnv1a_u64(h, fa ? 1u : 0u); + if (rope_params && rope_len > 0) { + h = fnv1a(h, rope_params, sizeof(int32_t) * static_cast(rope_len)); + } + h = fnv1a_u64(h, extra_cfg); + const std::string ver = ov_version_string(); + h = fnv1a(h, ver.data(), ver.size()); + + return h; +} + +std::string ggml_openvino_model_cache_blob_path(const std::string & dir, uint64_t fingerprint) { + return dir + "/" + hex64(fingerprint) + ".blob"; +} + +std::string ggml_openvino_model_cache_manifest_path(const std::string & dir, uint64_t fingerprint) { + return dir + "/" + hex64(fingerprint) + ".manifest"; +} + +bool ggml_openvino_model_cache_write_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint, + const std::vector & inputs, + const std::vector & outputs) { + std::ofstream f(path, std::ios::trunc); + if (!f.is_open()) { + return false; + } + f << "fingerprint " << hex64(fingerprint) << "\n"; + f << "ov_version " << ov_version_string() << "\n"; + for_each_weight(cgraph, [&](const ggml_tensor * t) { + f << t->name << " " << t->ne[0] << " " << t->ne[1] << " " << t->ne[2] << " " << t->ne[3] << " " + << static_cast(t->type) << " " << hex64(weight_fingerprint(t)) << "\n"; + }); + f << "ports\n"; + for (const auto * names : { &inputs, &outputs }) { + f << names->size() << '\n'; + for (const auto & name : *names) { + f << std::quoted(name) << '\n'; + } + } + return f.good(); +} + +bool ggml_openvino_model_cache_verify_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint, + std::vector & inputs, + std::vector & outputs) { + std::ifstream f(path); + if (!f.is_open()) { + return false; + } + std::string tag; + std::string val; + // header: fingerprint + if (!(f >> tag >> val) || tag != "fingerprint" || val != hex64(fingerprint)) { + return false; + } + // header: ov_version + if (!(f >> tag >> val) || tag != "ov_version" || val != ov_version_string()) { + return false; + } + + // Build the expected per-weight lines from the live cgraph, then require an + // exact match (same set, same order) against the manifest. + std::vector expected; + for_each_weight(cgraph, [&](const ggml_tensor * t) { + expected.push_back(std::string(t->name) + " " + std::to_string(t->ne[0]) + " " + std::to_string(t->ne[1]) + + " " + std::to_string(t->ne[2]) + " " + std::to_string(t->ne[3]) + " " + + std::to_string(static_cast(t->type)) + " " + hex64(weight_fingerprint(t))); + }); + + size_t idx = 0; + std::string line; + std::getline(f, line); // consume rest of ov_version line + while (idx < expected.size() && std::getline(f, line)) { + if (line.empty()) { + continue; + } + if (idx >= expected.size() || line != expected[idx]) { + return false; + } + ++idx; + } + if (idx != expected.size()) { + return false; + } + if (!std::getline(f, line)) { + return true; + } + if (line != "ports") { + return false; + } + for (auto * names : { &inputs, &outputs }) { + size_t count; + if (!(f >> count) || count > 100000) { + return false; + } + for (size_t i = 0; i < count; ++i) { + std::string name; + if (!(f >> std::quoted(name))) { + return false; + } + names->push_back(name); + } + } + f >> std::ws; + return f.eof(); +} diff --git a/ggml/src/ggml-openvino/model-cache.h b/ggml/src/ggml-openvino/model-cache.h new file mode 100644 index 0000000..ee69fca --- /dev/null +++ b/ggml/src/ggml-openvino/model-cache.h @@ -0,0 +1,57 @@ +#pragma once + +// Compiled blobs include weights. Cache-only execution skips weight uploads and graph compilation. + +#include "ggml.h" + +#include +#include +#include + +bool ggml_openvino_model_cache_only(); +void ggml_openvino_model_cache_init(); + +struct ggml_openvino_source_mapping { + uintptr_t begin; + uintptr_t end; + uint64_t offset; + uint64_t identity; +}; + +// Identify mmap weights without reading their pages. Cache mappings for one buffer lifetime. +uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::vector & mappings); +uint64_t ggml_backend_openvino_weight_fingerprint(const ggml_tensor * tensor); + +// Returns the compiled-model cache directory, or empty if unset. +std::string ggml_openvino_model_cache_dir(); +std::string ggml_openvino_model_cache_temp_path(const std::string & path); + +// Hash graph structure, source weight identities, configuration, and OpenVINO version. +uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph, + const std::string & device, + bool fa, + const int32_t * rope_params, + int rope_len, + uint64_t extra_cfg, + const std::string & graph_signature); + +// Path to the compiled-blob file for a fingerprint (/.blob). +std::string ggml_openvino_model_cache_blob_path(const std::string & dir, uint64_t fingerprint); + +// Path to the sidecar manifest (/.manifest) holding the per-weight +// fingerprints, used to re-verify a hit before trusting the blob. +std::string ggml_openvino_model_cache_manifest_path(const std::string & dir, uint64_t fingerprint); + +// Record weight metadata and source identities. Returns false on I/O error. +bool ggml_openvino_model_cache_write_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint, + const std::vector & inputs, + const std::vector & outputs); + +// Require all weight metadata and source identities to match the manifest. +bool ggml_openvino_model_cache_verify_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint, + std::vector & inputs, + std::vector & outputs); diff --git a/ggml/src/ggml-openvino/openvino/decoder.h b/ggml/src/ggml-openvino/openvino/decoder.h index 3b8da2b..ec69752 100644 --- a/ggml/src/ggml-openvino/openvino/decoder.h +++ b/ggml/src/ggml-openvino/openvino/decoder.h @@ -3,31 +3,74 @@ #include #include #include +#include +#include #include +#include #include namespace ov { namespace frontend { namespace ggml { +struct ModelInputInfo { + element::Type type; + PartialShape shape; +}; + +struct ModelExtraInputInfo { + element::Type type; + Shape shape; + int64_t value; + bool is_parameter; +}; + class GgmlDecoder : public DecoderBase { public: - virtual ov::Any get_attribute(const std::string& name) const = 0; + virtual ov::Any get_attribute(const std::string & name) const = 0; + + virtual PartialShape get_input_shape(int node_idx, const std::string & name) const = 0; + + virtual std::vector get_input_stride(int node_idx, const std::string & name) const = 0; + + virtual size_t get_view_input_size(int node_idx, const std::string & name) const = 0; + + virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual std::vector get_view_input_stride(int node_idx, + const std::string & name, + size_t view_index) const = 0; + + virtual std::vector get_view_input_src_stride(int node_idx, + const std::string & name, + size_t view_index) const = 0; + + virtual Shape get_view_input_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0; - virtual PartialShape get_input_shape(int node_idx, const std::string& name) const = 0; + virtual Shape get_view_input_src_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0; - virtual std::vector get_input_stride(int node_idx, const std::string& name) const = 0; + virtual PartialShape get_view_input_ov_shape(int node_idx, const std::string & name, size_t view_index) const = 0; - virtual element::Type get_input_type(int node_idx, const std::string& name) const = 0; + virtual PartialShape get_view_input_src_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const = 0; + + virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual std::string get_view_input_src_name(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual element::Type get_input_type(int node_idx, const std::string & name) const = 0; virtual size_t get_input_size() const = 0; virtual size_t get_input_size(int node_idx) const = 0; virtual void get_input_node(size_t input_port_idx, - std::string& producer_name, - std::string& producer_output_port_name, - size_t& producer_output_port_index) const = 0; + std::string & producer_name, + std::string & producer_output_port_name, + size_t & producer_output_port_index) const = 0; virtual std::vector get_input_names(int node_idx) const = 0; @@ -35,30 +78,42 @@ class GgmlDecoder : public DecoderBase { virtual element::Type get_output_type(const int node_idx) const = 0; - virtual int32_t* get_input_op_params(int node_idx, const std::string& name) const = 0; + virtual std::vector get_output_stride(int node_idx) const = 0; + + virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const = 0; virtual int32_t * get_output_op_params(int node_idx) const = 0; + virtual size_t get_output_op_offset(int node_idx) const = 0; + virtual std::vector get_output_names(int node_idx) const = 0; - virtual const std::string& get_op_type() const = 0; + virtual std::string get_inplace_op_src(int node_idx) const = 0; + + virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const = 0; - virtual const std::string& get_op_type(int node_idx) const = 0; + virtual const std::string & get_op_type() const = 0; - virtual const std::string& get_op_name() const = 0; + virtual const std::string & get_op_type(int node_idx) const = 0; - virtual const std::string& get_op_name(int node_idx) const = 0; + virtual const std::string & get_op_name() const = 0; + + virtual const std::string & get_op_name(int node_idx) const = 0; virtual void visit_subgraph(std::function, int node_idx)> node_visitor) const = 0; virtual int get_op_case(int node_idx) const = 0; - virtual const std::map>& get_model_inputs() const = 0; - virtual const std::map>& get_model_extra_inputs() const = 0; - virtual const std::map>& get_model_weights() const = 0; - virtual std::vector get_model_output_names() const = 0; + virtual const std::map & get_model_inputs() const = 0; + virtual const std::map & get_model_extra_inputs() const = 0; + virtual const std::map> & get_model_weights() const = 0; + virtual std::set get_model_output_names() const = 0; + + virtual int32_t * get_rope_params() const = 0; + + virtual bool has_mixed_rope_params() const = 0; - virtual int32_t* get_rope_params() const = 0; + virtual int get_ssm_state_size() const = 0; virtual std::map get_kv_param_res_names() const = 0; @@ -66,7 +121,11 @@ class GgmlDecoder : public DecoderBase { virtual bool is_stateful() const = 0; + virtual bool is_splited_model() const = 0; + virtual int is_swa_layer(int layer) const = 0; + + virtual int32_t get_op_dynamic_dim(int node_idx) const = 0; }; } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/frontend.cpp b/ggml/src/ggml-openvino/openvino/frontend.cpp index c2ba14e..88de86f 100644 --- a/ggml/src/ggml-openvino/openvino/frontend.cpp +++ b/ggml/src/ggml-openvino/openvino/frontend.cpp @@ -3,6 +3,7 @@ #include "input_model.h" #include "op_table.h" #include "translate_session.h" +#include namespace ov { namespace frontend { @@ -11,7 +12,7 @@ namespace ggml { FrontEnd::FrontEnd() {} std::shared_ptr FrontEnd::convert(const InputModel::Ptr & model, bool naive) { - auto ggml_model = std::dynamic_pointer_cast(model); + auto ggml_model = ov::as_type_ptr(model); FRONT_END_GENERAL_CHECK(ggml_model, "Invalid input model"); std::shared_ptr converted_model; const auto & supported_ops = get_supported_ops(); diff --git a/ggml/src/ggml-openvino/openvino/frontend.h b/ggml/src/ggml-openvino/openvino/frontend.h index f1c6f0c..4e301d3 100644 --- a/ggml/src/ggml-openvino/openvino/frontend.h +++ b/ggml/src/ggml-openvino/openvino/frontend.h @@ -12,10 +12,9 @@ namespace ggml { class FrontEnd { public: - using Ptr = std::shared_ptr; FrontEnd(); - static std::shared_ptr convert(const InputModel::Ptr& model, bool naive = false); + static std::shared_ptr convert(const InputModel::Ptr & model, bool naive = false); }; } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/input_model.h b/ggml/src/ggml-openvino/openvino/input_model.h index ce84344..6ddcea9 100644 --- a/ggml/src/ggml-openvino/openvino/input_model.h +++ b/ggml/src/ggml-openvino/openvino/input_model.h @@ -1,9 +1,9 @@ #pragma once -#include - #include "decoder.h" +#include + namespace ov { namespace frontend { namespace ggml { @@ -16,9 +16,9 @@ class InputModel : public ov::frontend::InputModel { friend class ::ov::frontend::ggml::FrontEnd; public: - explicit InputModel(const std::shared_ptr& gdecoder); + explicit InputModel(const std::shared_ptr & gdecoder); - const std::shared_ptr& get_model_decoder() const; + const std::shared_ptr & get_model_decoder() const; private: std::shared_ptr m_decoder; diff --git a/ggml/src/ggml-openvino/openvino/node_context.h b/ggml/src/ggml-openvino/openvino/node_context.h index aa48412..c653bc0 100644 --- a/ggml/src/ggml-openvino/openvino/node_context.h +++ b/ggml/src/ggml-openvino/openvino/node_context.h @@ -1,11 +1,11 @@ #pragma once +#include "decoder.h" + #include #include #include -#include "decoder.h" - namespace ov { namespace frontend { namespace ggml { @@ -16,28 +16,26 @@ typedef std::map> TensorMap; class NodeContext : public frontend::NodeContext { public: - NodeContext(const std::shared_ptr& decoder, - std::shared_ptr& tensor_map, + NodeContext(const std::shared_ptr & decoder, + std::shared_ptr & tensor_map, int node_idx, - TranslateSession* translate_session = nullptr) - : ov::frontend::NodeContext(decoder->get_op_type(node_idx)), - m_decoder(decoder), - m_tensor_map(tensor_map), - m_node_idx(node_idx), - m_translate_session(translate_session) { + TranslateSession * translate_session = nullptr) : + ov::frontend::NodeContext(decoder->get_op_type(node_idx)), + m_decoder(decoder), + m_tensor_map(tensor_map), + m_node_idx(node_idx), + m_translate_session(translate_session) { m_input_names = decoder->get_input_names(m_node_idx); m_output_names = decoder->get_output_names(m_node_idx); } - TranslateSession* get_translate_session() const { - return m_translate_session; - } + TranslateSession * get_translate_session() const { return m_translate_session; } - const std::vector& get_input_names() const { return m_input_names; } + const std::vector & get_input_names() const { return m_input_names; } - size_t get_input_size() const override { - return m_decoder->get_input_size(m_node_idx); - } + const std::vector & get_output_names() const { return m_output_names; } + + size_t get_input_size() const override { return m_decoder->get_input_size(m_node_idx); } ov::element::Type get_input_type(size_t index) const { return m_decoder->get_input_type(m_node_idx, m_input_names[index]); @@ -55,57 +53,127 @@ class NodeContext : public frontend::NodeContext { PartialShape get_output_shape() const { return m_decoder->get_output_shape(m_node_idx); } - int32_t* get_input_op_params(size_t index) const { + int32_t * get_input_op_params(size_t index) const { return m_decoder->get_input_op_params(m_node_idx, m_input_names[index]); } - int32_t * get_output_op_params() const { return m_decoder->get_output_op_params(m_node_idx); } + size_t get_view_input_size(size_t index) const { + return m_decoder->get_view_input_size(m_node_idx, m_input_names[index]); + } + + size_t get_view_input_offset(size_t index, size_t view_index) const { + return m_decoder->get_view_input_offset(m_node_idx, m_input_names[index], view_index); + } + + size_t get_view_input_src_offset(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_offset(m_node_idx, m_input_names[index], view_index); + } + + std::vector get_view_input_stride(size_t index, size_t view_index) const { + return m_decoder->get_view_input_stride(m_node_idx, m_input_names[index], view_index); + } - ov::element::Type get_output_type() const { - return m_decoder->get_output_type(m_node_idx); + std::vector get_view_input_src_stride(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_stride(m_node_idx, m_input_names[index], view_index); } + ov::Shape get_view_input_ggml_shape(size_t index, size_t view_index) const { + return m_decoder->get_view_input_ggml_shape(m_node_idx, m_input_names[index], view_index); + } + + ov::Shape get_view_input_src_ggml_shape(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_ggml_shape(m_node_idx, m_input_names[index], view_index); + } + + ov::PartialShape get_view_input_ov_shape(size_t index, size_t view_index) const { + return m_decoder->get_view_input_ov_shape(m_node_idx, m_input_names[index], view_index); + } + + ov::PartialShape get_view_input_src_ov_shape(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_ov_shape(m_node_idx, m_input_names[index], view_index); + } + + std::string get_view_input_name(size_t index, size_t view_index) const { + return m_decoder->get_view_input_name(m_node_idx, m_input_names[index], view_index); + } + + std::string get_view_input_src_name(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_name(m_node_idx, m_input_names[index], view_index); + } + + int32_t get_op_dynamic_dim() const { return m_decoder->get_op_dynamic_dim(m_node_idx); } + + int32_t * get_output_op_params() const { return m_decoder->get_output_op_params(m_node_idx); } + + size_t get_output_op_offset() const { return m_decoder->get_output_op_offset(m_node_idx); } + + ov::element::Type get_output_type() const { return m_decoder->get_output_type(m_node_idx); } + + std::vector get_output_stride() const { return m_decoder->get_output_stride(m_node_idx); } + Output get_input(int idx) const override { + // Check if this input is a VIEW + size_t view_input_size = m_decoder->get_view_input_size(m_node_idx, m_input_names[idx]); + if (view_input_size > 0) { + // This is a VIEW input, get the base tensor name (last element in the chain) + std::string base_name = + m_decoder->get_view_input_src_name(m_node_idx, m_input_names[idx], view_input_size - 1); + // Check if the VIEW has been resolved (translate_view produced a Slice) + auto view_it = m_tensor_map->find(m_input_names[idx]); + if (!base_name.empty() && view_it != m_tensor_map->end()) { + auto base_it = m_tensor_map->find(base_name); + // A multi-output translator can publish a VIEW directly without materializing + // its packed parent (GatedDeltaNet attention/state). In that case the VIEW is the + // authoritative value. The node comparison retains the existing resolved-VIEW path. + if (base_it == m_tensor_map->end() || + view_it->second.get_node_shared_ptr() != base_it->second.get_node_shared_ptr()) { + return view_it->second; + } + return base_it->second; + } + if (!base_name.empty()) { + return m_tensor_map->at(base_name); + } + } + // Not a VIEW or failed to get base name, use the original logic return m_tensor_map->at(m_input_names[idx]); } - Output get_input(const std::string& name) const override { + Output get_input(const std::string & name) const override { if (m_tensor_map->find(name) == m_tensor_map->end()) { throw std::runtime_error("'" + name + "' not found in tensor map."); } return m_tensor_map->at(name); } - bool has_input(const std::string& name) const { - return m_tensor_map->find(name) != m_tensor_map->end(); - } + bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); } - const std::string& get_name() const override { - return m_decoder->get_op_name(m_node_idx); + void put_shared(const std::string & name, const Output & value) const { + m_tensor_map->insert({name, value}); } - ov::Any get_attribute_as_any(const std::string& name) const override { - return m_decoder->get_attribute(name); - } + const std::string & get_name() const override { return m_decoder->get_op_name(m_node_idx); } - int get_op_case() const { - return m_decoder->get_op_case(m_node_idx); - } + ov::Any get_attribute_as_any(const std::string & name) const override { return m_decoder->get_attribute(name); } + + int get_op_case() const { return m_decoder->get_op_case(m_node_idx); } bool is_static() const { return m_decoder->is_static(); } bool is_stateful() const { return m_decoder->is_stateful(); } + int get_ssm_state_size() const { return m_decoder->get_ssm_state_size(); } + private: std::shared_ptr m_decoder; - std::shared_ptr& m_tensor_map; + std::shared_ptr & m_tensor_map; int m_node_idx; - TranslateSession* m_translate_session; + TranslateSession * m_translate_session; std::vector m_input_names; std::vector m_output_names; }; -using CreatorFunction = std::function; +using CreatorFunction = std::function; } // namespace ggml } // namespace frontend diff --git a/ggml/src/ggml-openvino/openvino/op/add.cpp b/ggml/src/ggml-openvino/openvino/op/add.cpp new file mode 100644 index 0000000..84bc856 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/add.cpp @@ -0,0 +1,67 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_add(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + if (context.get_op_case() == 1) { + // MoE expert-plane sum (see is_moe_expert_sum_add): input 1 is a VIEW plane of the + // shared base tensor `experts` = [n_embd, n_expert_used, n_tokens, 1] (ggml order) -> + // [1, n_tokens, n_expert_used, n_embd] (OV order). The whole ADD chain is equivalent to + // reducing the expert axis (OV axis 2) of that base, so bypass the chain and the + // per-plane Slices entirely. + size_t view_size = context.get_view_input_size(1); + auto base_name = context.get_view_input_src_name(1, view_size - 1); + auto base = context.get_input(base_name); + + // Stateful models drop the leading batch dim, so the base is rank 3 and both axes + // below shift down by one. Take them from the actual rank: the expert axis is always + // second from last, and the token axis is re-added just before it. + const auto base_rank = base.get_partial_shape().rank(); + FRONT_END_OP_CONVERSION_CHECK(base_rank.is_static() && base_rank.get_length() >= 3, + "MoE expert sum needs a static rank of at least 3"); + const int64_t rank = base_rank.get_length(); + + auto reduced = std::make_shared( + base, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {rank - 2}), false); + auto res = std::make_shared( + reduced, ov::op::v0::Constant::create(ov::element::i64, {1}, {rank - 3})); + return rename_outputs_with_suffix({res}, context.get_name()); + } + + auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + // opset1::Add needs matching types (e.g. fused ADD_ADD mixes f16/f32); add in f32, cast once. + auto output_type = context.get_output_type(); + if (input_0.get_element_type() != input_1.get_element_type()) { + if (input_0.get_element_type() != ov::element::f32) { + input_0 = std::make_shared(input_0, ov::element::f32); + } + if (input_1.get_element_type() != ov::element::f32) { + input_1 = std::make_shared(input_1, ov::element::f32); + } + } + ov::Output res = std::make_shared(input_0, input_1); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/add_id.cpp b/ggml/src/ggml-openvino/openvino/op/add_id.cpp new file mode 100644 index 0000000..04b1d3f --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/add_id.cpp @@ -0,0 +1,88 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +static ov::Output reshape_add_id_input_to_2d(const ov::Output & input, + const ov::PartialShape & input_shape, + const std::vector & dims) { + const auto & actual_shape = input.get_partial_shape(); + if (actual_shape.rank().is_static() && actual_shape.rank().get_length() == 2) { + return input; + } + + if (input_shape.rank().is_static() && input_shape.rank().get_length() == 2) { + return input; + } + + auto shape = std::make_shared(input, ov::element::i64); + return std::make_shared(input, get_dimensions(shape, dims), false); +} + +OutputVector translate_add_id(const NodeContext & context) { + num_inputs_check(context, 3, 3); + + auto input = process_view_input_new(context, 0); + auto bias = process_view_input_new(context, 1); + auto ids = process_view_input_new(context, 2); + + // OpenVINO uses reversed GGML dimensions: + // input: [1, n_token, n_used, n_embd] + // bias: [1, 1, n_expert, n_embd] + // ids: [1, 1, n_token, n_used] + // Model bias constants may already be stored as [n_expert, n_embd]. + bias = reshape_add_id_input_to_2d(bias, context.get_input_shape(1), {2, 3}); + ids = reshape_add_id_input_to_2d(ids, context.get_input_shape(2), {2, 3}); + + if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { + ids = std::make_shared(ids, ov::element::i32); + } + + // an id of -1 marks a skipped slot: gather row 0 for it and zero the bias with a mask + auto zero = ov::op::v0::Constant::create(ids.get_element_type(), ov::Shape{}, {0}); + ov::Output keep = std::make_shared(ids, zero); + keep = std::make_shared(keep, bias.get_element_type()); + keep = std::make_shared(keep, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1})); + ids = std::make_shared(ids, zero); + + auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); + ov::Output selected_bias = std::make_shared(bias, ids, gather_axis); + selected_bias = std::make_shared(selected_bias, keep); + selected_bias = std::make_shared( + selected_bias, std::make_shared(input, ov::element::i64), false); + + if (selected_bias.get_element_type() != input.get_element_type()) { + selected_bias = std::make_shared(selected_bias, input.get_element_type()); + } + + ov::Output res = std::make_shared(input, selected_bias); + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/argsort.cpp b/ggml/src/ggml-openvino/openvino/op/argsort.cpp new file mode 100644 index 0000000..dd08725 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/argsort.cpp @@ -0,0 +1,51 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml.h" + +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_argsort(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + + const int32_t order = context.get_output_op_params()[0]; + + ov::op::v11::TopK::Mode mode; + switch (order) { + case GGML_SORT_ORDER_ASC: + mode = ov::op::v11::TopK::Mode::MIN; + break; + case GGML_SORT_ORDER_DESC: + mode = ov::op::v11::TopK::Mode::MAX; + break; + default: + FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported GGML_OP_ARGSORT order: ", order); + } + + // Stateful models drop the leading size-1 batch dim, so the expert axis is 2 there + // instead of 3 (same rank-3-vs-rank-4 split as get_rows.cpp / process_view_input). + const int axis = (context.is_stateful() && input.get_partial_shape().rank() == 3) ? 2 : 3; + + auto k = std::make_shared(get_dimensions(input.get_node_shared_ptr(), {axis}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + + auto topk = std::make_shared(input, k, axis, mode, ov::op::v11::TopK::SortType::SORT_VALUES, + context.get_output_type(), false); + + return rename_outputs_with_suffix({topk->output(1)}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/clamp.cpp b/ggml/src/ggml-openvino/openvino/op/clamp.cpp new file mode 100644 index 0000000..070ad33 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/clamp.cpp @@ -0,0 +1,33 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_clamp(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + + const int32_t * op_params = context.get_output_op_params(); + FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "CLAMP requires output op params"); + + float min; + float max; + std::memcpy(&min, reinterpret_cast(op_params) + 0, sizeof(float)); + std::memcpy(&max, reinterpret_cast(op_params) + 1, sizeof(float)); + + auto res = std::make_shared(input, min, max); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/concat.cpp b/ggml/src/ggml-openvino/openvino/op/concat.cpp new file mode 100644 index 0000000..4d36a66 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/concat.cpp @@ -0,0 +1,48 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_concat(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + const int32_t * op_params = context.get_output_op_params(); + FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "CONCAT requires output op params"); + + const auto output_shape = context.get_output_shape(); + FRONT_END_CHECK_IMPLEMENTED(output_shape.rank().is_static(), "CONCAT requires static output rank"); + + const auto rank = output_shape.rank().get_length(); + const int32_t ggml_dim = op_params[0]; + FRONT_END_CHECK_IMPLEMENTED(ggml_dim >= 0 && ggml_dim < rank, "CONCAT axis is out of range"); + + auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + const auto output_type = context.get_output_type(); + + if (input_0.get_element_type() != output_type) { + input_0 = std::make_shared(input_0, output_type); + } + if (input_1.get_element_type() != output_type) { + input_1 = std::make_shared(input_1, output_type); + } + + const auto axis = static_cast(rank - 1 - ggml_dim); + auto res = std::make_shared(OutputVector{input_0, input_1}, axis); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/cont.cpp b/ggml/src/ggml-openvino/openvino/op/cont.cpp index 6160dd7..9888f6b 100644 --- a/ggml/src/ggml-openvino/openvino/op/cont.cpp +++ b/ggml/src/ggml-openvino/openvino/op/cont.cpp @@ -3,12 +3,9 @@ #include "../op_table.h" #include "../utils.h" -#include -#include #include #include #include -#include namespace ov { namespace frontend { @@ -18,27 +15,19 @@ namespace op { OutputVector translate_cont(const NodeContext & context) { num_inputs_check(context, 1, 1); - int op_case = context.get_op_case(); - FRONT_END_CHECK_IMPLEMENTED(op_case == 1 || op_case == 2 || op_case == 3, "Unsupported CONT case"); - auto src_shape = context.get_input_shape(0).to_shape(); auto dst_shape = context.get_output_shape().to_shape(); - ov::Output res; - if (op_case == 1) { - // The input comes from a PERMUTE - throw std::runtime_error("Code of this case might be outdated"); - dst_shape[1] = -1; - res = std::make_shared( - context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {dst_shape.size()}, dst_shape), false); - } else if (op_case == 2) { - // The input comes from a TRANSPOSE - return {context.get_input(0)}; - } else { - // The input comes from a VIEW - res = process_view_input(context, 0); + if (context.get_op_dynamic_dim() != -1) { + dst_shape[3 - context.get_op_dynamic_dim()] = -1; } + auto input = process_view_input_new(context, 0); + + ov::Output res; + res = std::make_shared( + input, ov::op::v0::Constant::create(ov::element::i64, {dst_shape.size()}, dst_shape), false); + return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/conv.cpp b/ggml/src/ggml-openvino/openvino/op/conv.cpp new file mode 100644 index 0000000..94173dc --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/conv.cpp @@ -0,0 +1,233 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_conv_2d(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + ov::Output kernel = process_view_input_new(context, 0); + ov::Output input = process_view_input_new(context, 1); + + if (kernel.get_element_type() != input.get_element_type()) { + kernel = std::make_shared(kernel, input.get_element_type()); + } + + const int32_t * params = context.get_output_op_params(); + const int s0 = params[0]; + const int s1 = params[1]; + const int p0 = params[2]; + const int p1 = params[3]; + const int d0 = params[4]; + const int d1 = params[5]; + + ov::Strides strides{static_cast(s1), static_cast(s0)}; + ov::CoordinateDiff pads_begin{static_cast(p1), static_cast(p0)}; + ov::CoordinateDiff pads_end{static_cast(p1), static_cast(p0)}; + ov::Strides dilations{static_cast(d1), static_cast(d0)}; + + ov::Output res = std::make_shared( + input, kernel, strides, pads_begin, pads_end, dilations, ov::op::PadType::EXPLICIT); + + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_conv_2d_dw(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + ov::Output kernel = process_view_input_new(context, 0); + ov::Output input = process_view_input_new(context, 1); + + if (kernel.get_element_type() != input.get_element_type()) { + kernel = std::make_shared(kernel, input.get_element_type()); + } + + const int32_t * params = context.get_output_op_params(); + const int s0 = params[0]; + const int s1 = params[1]; + const int p0 = params[2]; + const int p1 = params[3]; + const int d0 = params[4]; + const int d1 = params[5]; + + // Reshape kernel from [C, 1, KH, KW] to [C, 1, 1, KH, KW] for 2D GroupConvolution + auto unsqueeze_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {1}); + auto kernel_5d = std::make_shared(kernel, unsqueeze_axis); + + ov::Strides strides{static_cast(s1), static_cast(s0)}; + ov::CoordinateDiff pads_begin{static_cast(p1), static_cast(p0)}; + ov::CoordinateDiff pads_end{static_cast(p1), static_cast(p0)}; + ov::Strides dilations{static_cast(d1), static_cast(d0)}; + + ov::Output res = std::make_shared( + input, kernel_5d, strides, pads_begin, pads_end, dilations, ov::op::PadType::EXPLICIT); + + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_conv_transpose_1d(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + ov::Output kernel = process_view_input_new(context, 0); + ov::Output input = process_view_input_new(context, 1); + + if (kernel.get_element_type() != input.get_element_type()) { + kernel = std::make_shared(kernel, input.get_element_type()); + } + + const int32_t * params = context.get_output_op_params(); + const int s0 = params[0]; + const int p0 = params[1]; + const int d0 = params[2]; + + const auto kernel_shape = context.get_input_shape(0).to_shape(); // [1, Cin, Cout, K] + const int64_t Cin = kernel_shape[1]; + const int64_t Cout = kernel_shape[2]; + const int64_t K = kernel_shape[3]; + + const auto input_shape = context.get_input_shape(1).to_shape(); // [1, N, Cin, L] + const int64_t N = input_shape[0] * input_shape[1]; + const int64_t L = input_shape[3]; + + auto kernel_3d = std::make_shared( + kernel, ov::op::v0::Constant::create(ov::element::i64, {3}, {Cin, Cout, K}), false); + auto input_3d = std::make_shared( + input, ov::op::v0::Constant::create(ov::element::i64, {3}, {N, Cin, L}), false); + + ov::Strides strides{static_cast(s0)}; + ov::CoordinateDiff pads_begin{static_cast(p0)}; + ov::CoordinateDiff pads_end{static_cast(p0)}; + ov::Strides dilations{static_cast(d0)}; + + auto conv_tr = std::make_shared( + input_3d, kernel_3d, strides, pads_begin, pads_end, dilations); + + const auto out_shape = context.get_output_shape().to_shape(); + auto out_shape_const = ov::op::v0::Constant::create( + ov::element::i64, {4}, {static_cast(out_shape[0]), static_cast(out_shape[1]), + static_cast(out_shape[2]), static_cast(out_shape[3])}); + ov::Output res = std::make_shared(conv_tr, out_shape_const, false); + + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_conv_transpose_2d(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + ov::Output kernel = process_view_input_new(context, 0); + ov::Output input = process_view_input_new(context, 1); + + if (kernel.get_element_type() != input.get_element_type()) { + kernel = std::make_shared(kernel, input.get_element_type()); + } + + const int32_t * params = context.get_output_op_params(); + const int stride = params[0]; + + ov::Strides strides{static_cast(stride), static_cast(stride)}; + ov::CoordinateDiff pads_begin{0, 0}; + ov::CoordinateDiff pads_end{0, 0}; + ov::Strides dilations{1, 1}; + + ov::Output res = std::make_shared( + input, kernel, strides, pads_begin, pads_end, dilations); + + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_conv_3d(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + ov::Output kernel = process_view_input_new(context, 0); + ov::Output input = process_view_input_new(context, 1); + + if (kernel.get_element_type() != input.get_element_type()) { + kernel = std::make_shared(kernel, input.get_element_type()); + } + + const int32_t * params = context.get_output_op_params(); + const int s0 = params[0]; + const int s1 = params[1]; + const int s2 = params[2]; + const int p0 = params[3]; + const int p1 = params[4]; + const int p2 = params[5]; + const int d0 = params[6]; + const int d1 = params[7]; + const int d2 = params[8]; + const int c = params[9]; + const int n = params[10]; + const int oc = params[11]; + + const auto kshape = context.get_input_shape(0).to_shape(); // [c*oc, KD, KH, KW] + const int64_t KD = kshape[1]; + const int64_t KH = kshape[2]; + const int64_t KW = kshape[3]; + + const auto inshape = context.get_input_shape(1).to_shape(); // [c*n, ID, IH, IW] + const int64_t ID = inshape[1]; + const int64_t IH = inshape[2]; + const int64_t IW = inshape[3]; + + auto kernel_5d = std::make_shared( + kernel, ov::op::v0::Constant::create(ov::element::i64, {5}, {static_cast(oc), static_cast(c), KD, KH, KW}), false); + auto input_5d = std::make_shared( + input, ov::op::v0::Constant::create(ov::element::i64, {5}, {static_cast(n), static_cast(c), ID, IH, IW}), false); + + ov::Strides strides{static_cast(s2), static_cast(s1), static_cast(s0)}; + ov::CoordinateDiff pads_begin{static_cast(p2), static_cast(p1), static_cast(p0)}; + ov::CoordinateDiff pads_end{static_cast(p2), static_cast(p1), static_cast(p0)}; + ov::Strides dilations{static_cast(d2), static_cast(d1), static_cast(d0)}; + + auto conv = std::make_shared( + input_5d, kernel_5d, strides, pads_begin, pads_end, dilations, ov::op::PadType::EXPLICIT); + + const auto out_shape = context.get_output_shape().to_shape(); // [oc*n, OD, OH, OW] + auto out_shape_const = ov::op::v0::Constant::create( + ov::element::i64, {4}, {static_cast(out_shape[0]), static_cast(out_shape[1]), + static_cast(out_shape[2]), static_cast(out_shape[3])}); + ov::Output res = std::make_shared(conv, out_shape_const, false); + + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/cpy.cpp b/ggml/src/ggml-openvino/openvino/op/cpy.cpp index 8311172..2c7299a 100644 --- a/ggml/src/ggml-openvino/openvino/op/cpy.cpp +++ b/ggml/src/ggml-openvino/openvino/op/cpy.cpp @@ -2,8 +2,27 @@ #include "../op_table.h" #include "../utils.h" +#include +#include +#include #include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace ov { namespace frontend { @@ -11,7 +30,281 @@ namespace ggml { namespace op { OutputVector translate_cpy(const NodeContext & context) { - auto res = std::make_shared(context.get_input(0), context.get_output_type()); + auto op_case = context.get_op_case(); + auto input_shape = context.get_input_shape(0); + auto output_shape = context.get_input_shape(1); + + if (op_case == 4) { + auto src = process_view_input_new(context, 0); + auto base = context.get_input(1); + + int64_t n_elems = 1; + for (const auto & dim : context.get_output_shape().to_shape()) { + n_elems *= static_cast(dim); + } + + const auto output_stride = context.get_output_stride(); + const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back(); + FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY conv state view update has invalid element size"); + + const int64_t begin_val = static_cast(context.get_output_op_offset() / elem_size); + const int64_t end_val = begin_val + n_elems; + + auto flat_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, 1, -1}); + src = std::make_shared(src, flat_shape, false); + if (src.get_element_type() != context.get_output_type()) { + src = std::make_shared(src, context.get_output_type()); + } + + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}); + auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}); + auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX}); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + + auto head_part = std::make_shared(base, zero, begin, one, axis); + auto tail_part = std::make_shared(base, end, int_max, one, axis); + auto res = std::make_shared(ov::OutputVector{head_part, src, tail_part}, 3); + return rename_outputs_with_suffix({res}, context.get_name()); + } + + // Recurrent state cache writeback into a slot block of the cache. Where the block starts is a + // runtime input, so the cached model works for any kv head and active sequence count. The + // result is the full updated cache. + // op_case 1: gated-delta-net state, op_case 2: conv state, op_case 3: defrag remainder. + if (op_case == 3) { + // With -np 1 (and generally whenever there is no defrag remainder) this GET_ROWS gathers + // zero rows: nothing to write back, and the cache is unchanged. NPU rejects zero-size + // tensors, so short-circuit instead of building a degenerate Slice/Concat chain. + bool is_empty = false; + if (input_shape.rank().is_static()) { + for (const auto & d : input_shape) { + if (d.is_static() && d.get_length() == 0) { + is_empty = true; + break; + } + } + } + if (is_empty) { + return {context.get_input(1)}; + } + } + // op_case 7/8/9 are the single-slot variants; op_case 10 writes native GDN state into a + // multi-slot cache without rollback snapshots. + const bool single_slot_assign = op_case >= 7 && op_case <= 9; + const bool direct_gdn_state = op_case == 7 || op_case == 10; + int writeback_case = op_case; + if (op_case == 10) { + writeback_case = 1; + } else if (single_slot_assign) { + writeback_case = op_case - 6; + } + const std::string slot_begin_name = "rs_slot_begin_" + context.get_name(); + const bool slice_assign = writeback_case >= 1 && writeback_case <= 3 && + (single_slot_assign || context.has_input(slot_begin_name)); + if (slice_assign) { + if (single_slot_assign && writeback_case == 3) { + return {context.get_input(1)}; + } + + const int64_t slot_axis = 2; + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {slot_axis}); + auto feature = ov::op::v0::Constant::create(ov::element::i64, {4}, + std::vector{1, 1, -1, output_shape[3].get_length()}); + + ov::Output src; + ov::Output begin; + if (!single_slot_assign) { + begin = context.get_input(slot_begin_name); + } + auto base = context.get_input(1); + if (writeback_case == 1) { + if (direct_gdn_state) { + // Non-rollback GDN publishes state directly as [active_slots, heads, value_dim, + // key_dim]. Flatten each active slot before replacing or updating the cache. + src = std::make_shared(context.get_input(0), feature, false); + } else { + // Multi-slot rollback still consumes GGML's packed [attention | state snapshots] + // layout. Slice the state block using the runtime source offset. + auto src_begin = context.get_input("rs_src_begin_" + context.get_name()); + auto state_part = + std::make_shared(context.get_input(0), src_begin, int_max, one, axis); + src = std::make_shared(state_part, feature, false); + } + } else if (writeback_case == 2) { + // conv_input is [previous conv state | new tokens]; the snapshot is the conv_kernel_size - 1 + // columns ending at the last *valid* token. Gather (rather than Slice) keeps the output + // shape static even though the window start is a runtime value. + auto window_size = (int64_t) input_shape[3].get_length(); + ov::Output window; + auto col_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + const std::string src_begin_name = "rs_src_begin_" + context.get_name(); + if (context.has_input(src_begin_name)) { + auto src_begin = context.get_input(src_begin_name); + auto src_end = std::make_shared( + src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size})); + window = std::make_shared(context.get_input(0), src_begin, src_end, one, col_axis); + } else if (context.has_input("chunk_valid_len")) { + std::vector offsets(window_size); + std::iota(offsets.begin(), offsets.end(), 0); + auto indices = std::make_shared( + ov::op::v0::Constant::create(ov::element::i64, {(size_t) window_size}, offsets), + context.get_input("chunk_valid_len")); + window = std::make_shared(context.get_input(0), indices, col_axis); + } else { + auto window_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-window_size}); + window = + std::make_shared(context.get_input(0), window_begin, int_max, one, col_axis); + } + const auto base_shape = base.get_partial_shape(); + FRONT_END_OP_CONVERSION_CHECK(base_shape.rank().is_static() && base_shape.rank().get_length() == 4, + "CPY conv state cache update requires rank-4 base cache"); + FRONT_END_OP_CONVERSION_CHECK(base_shape[3].is_static(), + "CPY conv state cache update requires static feature size"); + FRONT_END_OP_CONVERSION_CHECK(input_shape.rank().is_static() && input_shape.rank().get_length() == 4 && + input_shape[2].is_static() && input_shape[3].is_static(), + "CPY conv state cache update requires static source feature view"); + + const int64_t full_feature_size = base_shape[3].get_length(); + const int64_t update_feature_size = input_shape[2].get_length() * input_shape[3].get_length(); + const auto output_stride = context.get_output_stride(); + const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back(); + FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, + "CPY conv state cache update has invalid element size"); + const int64_t feature_begin = static_cast(context.get_output_op_offset() / elem_size) % + full_feature_size; + const int64_t feature_end = feature_begin + update_feature_size; + FRONT_END_OP_CONVERSION_CHECK(feature_begin >= 0 && feature_end <= full_feature_size, + "CPY conv state cache update feature range is out of bounds"); + + auto partial_feature = ov::op::v0::Constant::create( + ov::element::i64, {4}, std::vector{1, 1, -1, update_feature_size}); + src = std::make_shared(window, partial_feature, false); + if (src.get_element_type() != context.get_output_type()) { + src = std::make_shared(src, context.get_output_type()); + } + + if (single_slot_assign) { + return rename_outputs_with_suffix({src}, context.get_name()); + } + + auto src_len = std::make_shared( + std::make_shared(src, ov::element::i64), axis, + ov::op::v0::Constant::create(ov::element::i64, {}, {0})); + auto slot_end = std::make_shared(begin, src_len); + auto active_slots = std::make_shared(base, begin, slot_end, one, axis); + + auto feature_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto feature_begin_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_begin}); + auto feature_end_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_end}); + auto feature_head = std::make_shared(active_slots, zero, feature_begin_node, one, + feature_axis); + auto feature_tail = std::make_shared(active_slots, feature_end_node, int_max, one, + feature_axis); + src = std::make_shared(ov::OutputVector{feature_head, src, feature_tail}, 3); + } else { + // op_case 3: gathered remainder rows already have the cache slot layout [1, 1, extra, feature] + src = context.get_input(0); + } + + if (src.get_element_type() != context.get_output_type()) { + src = std::make_shared(src, context.get_output_type()); + } + + if (single_slot_assign) { + return rename_outputs_with_suffix({src}, context.get_name()); + } + + auto src_len = + std::make_shared(std::make_shared(src, ov::element::i64), axis, + ov::op::v0::Constant::create(ov::element::i64, {}, {0})); + auto end = std::make_shared(begin, src_len); + auto head_part = std::make_shared(base, zero, begin, one, axis); + auto tail_part = std::make_shared(base, end, int_max, one, axis); + auto res = std::make_shared(ov::OutputVector{head_part, src, tail_part}, slot_axis); + return rename_outputs_with_suffix({res}, context.get_name()); + } + + auto input = process_view_input_new(context, 0); + + if (op_case == 5 || op_case == 6) { + auto input_shape = context.get_input_shape(0); + auto output_shape = context.get_output_shape(); + auto dst_ggml_shape = context.get_view_input_ggml_shape(1, 0); + auto dst_stride = context.get_view_input_stride(1, 0); + size_t offset_bytes = context.get_view_input_offset(1, 0); + auto n_state = (int64_t) context.get_input_shape(0)[3].get_length(); + auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state}); + auto kv_buf = context.get_input(1); // shape {1,1,1,N} + + Output token_len_per_seq; + Output n_write_dyn; + if (context.has_input("token_len_per_seq")) { + token_len_per_seq = context.get_input("token_len_per_seq"); + n_write_dyn = std::make_shared(token_len_per_seq, n_state_c); + } else { + n_write_dyn = ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) dst_ggml_shape[3]}); + } + size_t elem_size = dst_stride[3]; + FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY KV cache view update has invalid element size"); + int64_t start_elem = (int64_t) (offset_bytes / elem_size); + // op_case 5: decoder self-attention – write offset advances each step. + // op_case 6: encoder self-attn or cross-attn – offset fixed at compile time. + const bool is_decoder_self_attn = (op_case == 5); + auto ones_c = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{1, 1, 1}); + auto new_shape = std::make_shared(ov::OutputVector{ones_c, n_write_dyn}, 0); + + auto reshaped = std::make_shared(input, new_shape, false); + auto data = std::make_shared(reshaped, context.get_output_type()); + // Indices [start_elem .. start_elem + n_write) on axis 3 of {1,1,1,N} + // For decoder self-attention the write offset advances each step, so compute it + // dynamically from the model inputs: start = (attention_size - token_len_per_seq) * n_state. + // For encoder self-attn and cross-attn the offset is fixed at graph-compile time. + ov::Output start; + if (is_decoder_self_attn && context.has_input("attention_size") && context.has_input("token_len_per_seq")) { + auto attention_size_in = context.get_input("attention_size"); + auto token_len_in = context.get_input("token_len_per_seq"); + auto past_tokens = std::make_shared(attention_size_in, token_len_in); + auto new_start = std::make_shared(past_tokens, n_state_c); + start = std::make_shared( + new_start, ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem})); + } else { + start = ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem}); + } + auto start_squeezed = std::make_shared(start); + auto end = std::make_shared(start_squeezed, n_write_dyn); + auto end_squeezed = std::make_shared(end); + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto step_squeezed = std::make_shared(step); + auto indices = + std::make_shared(start_squeezed, end_squeezed, step_squeezed, ov::element::i64); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + + auto kv_updated = std::make_shared(kv_buf, indices, data, axis); + return rename_outputs_with_suffix({kv_updated}, context.get_name()); + } + + if (input_shape != output_shape) { + auto new_shape = ov::op::v0::Constant::create( + ov::element::i64, {static_cast(output_shape.rank().get_length())}, output_shape.to_shape()); + input = std::make_shared(input, new_shape, false); + } + + ov::Output res; + if (context.get_input_type(0) != context.get_output_type()) { + res = std::make_shared(input, context.get_output_type()); + } else { + res = input; + } + + if (res.get_node_shared_ptr() == context.get_input(0).get_node_shared_ptr()) { + return {res}; + } + return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/cumsum.cpp b/ggml/src/ggml-openvino/openvino/op/cumsum.cpp new file mode 100644 index 0000000..0a414b2 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/cumsum.cpp @@ -0,0 +1,29 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML cumsum computes prefix sum along dim 0 (the innermost/fastest dimension). +// In OV layout the dims are reversed: ggml [ne0, ne1, ne2, ne3] → OV [ne3, ne2, ne1, ne0], +// so ggml dim 0 maps to OV axis 3 (last axis). +OutputVector translate_cumsum(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto x = context.get_input(0); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {}, {3}); + auto res = std::make_shared(x, axis); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/diag.cpp b/ggml/src/ggml-openvino/openvino/op/diag.cpp new file mode 100644 index 0000000..05e0648 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/diag.cpp @@ -0,0 +1,37 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML DIAG takes a 1D vector (ne0, 1, ne2, ne3) and produces a diagonal matrix +// of shape (ne0, ne0, ne2, ne3). +// In OV layout (ggml [ne0, ne1, ne2, ne3] → OV [ne3, ne2, ne1, ne0]): +// input: [ne3, ne2, 1, ne0] +// output: [ne3, ne2, ne0, ne0] +// The diagonal: output[..., i, j] = input[..., 0, j] if i == j, else 0. +OutputVector translate_diag(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto x = process_view_input_new(context, 0); // OV shape: [ne3, ne2, 1, ne0] + + auto n = get_dimensions(x.get_node_shared_ptr(), {3}); + auto zero_diag = ov::op::v0::Constant::create(ov::element::i64, {}, {0}); + + auto eye = std::make_shared(n, n, zero_diag, x.get_element_type()); + auto res = std::make_shared(x, eye); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/div.cpp b/ggml/src/ggml-openvino/openvino/op/div.cpp new file mode 100644 index 0000000..2089ffd --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/div.cpp @@ -0,0 +1,138 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +namespace { + +bool is_silu_div_pattern(const ov::Output & numerator, + const ov::Output & denominator, + const NodeContext & context) { + if (context.get_input_size() != 2) { + return false; + } + + const auto * unary_op = reinterpret_cast(context.get_input_op_params(0)); + if (unary_op == nullptr || *unary_op != GGML_UNARY_OP_SILU) { + return false; + } + + const auto denom_node = denominator.get_node_shared_ptr(); + + if (auto swish = ov::as_type_ptr(numerator.get_node_shared_ptr())) { + return swish->input_value(0).get_node_shared_ptr() == denom_node; + } + return false; +} + +ov::Output repeat_input_to_match(const NodeContext & context, + const ov::Output & input, + const ov::Output & target, + size_t input_index) { + const auto input_shape = context.get_input_shape(input_index); + const auto target_shape = context.get_input_shape(0); + + if (input_shape == target_shape) { + return input; + } + + if (input_shape.rank().is_static() && target_shape.rank().is_static()) { + const auto rank = static_cast(input_shape.rank().get_length()); + std::vector repeats(rank, 1); + bool needs_repeat = false; + + for (size_t axis = 0; axis < rank; ++axis) { + FRONT_END_OP_CONVERSION_CHECK(input_shape[axis].is_static() && target_shape[axis].is_static(), + "DIV repeat requires static dimensions on both inputs"); + + const int64_t input_dim = input_shape[axis].get_length(); + const int64_t target_dim = target_shape[axis].get_length(); + + FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && target_dim > 0 && target_dim % input_dim == 0, + "DIV input shape ", input_shape, " cannot repeat to match ", target_shape); + + repeats[axis] = target_dim / input_dim; + needs_repeat = needs_repeat || repeats[axis] != 1; + } + + if (!needs_repeat) { + return input; + } + + auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats); + return std::make_shared(input, repeats_node); + } + + auto input_shape_node = std::make_shared(input, ov::element::i64); + auto target_shape_node = std::make_shared(target, ov::element::i64); + auto repeats_node = std::make_shared(target_shape_node, input_shape_node); + return std::make_shared(input, repeats_node); +} + +} // namespace + +OutputVector translate_div(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + + if (is_silu_div_pattern(input_0, input_1, context)) { + ov::Output res = std::make_shared(input_1); + if (res.get_element_type() != context.get_output_type()) { + res = std::make_shared(res, context.get_output_type()); + } + return rename_outputs_with_suffix({res}, context.get_name()); + } + + input_1 = repeat_input_to_match(context, input_1, input_0, 1); + + const auto output_type = context.get_output_type(); + const bool use_f32_compute = input_0.get_element_type() != ov::element::f32 || + input_1.get_element_type() != ov::element::f32 || output_type != ov::element::f32; + + if (use_f32_compute) { + input_0 = std::make_shared(input_0, ov::element::f32); + input_1 = std::make_shared(input_1, ov::element::f32); + } + + ov::Output res = std::make_shared(input_0, input_1); + if (use_f32_compute) { + // Keep the reciprocal/divide path in FP32. Without this hint, the GPU + // plugin can still compress the subgraph back to FP16 and overflow on + // small shexp gate values (e.g. silu(x) / x in qwen2moe). + ov::mark_as_precision_sensitive(res.get_node_shared_ptr()->input(0)); + ov::mark_as_precision_sensitive(res.get_node_shared_ptr()->input(1)); + } + if (res.get_element_type() != output_type) { + auto output_convert = std::make_shared(res, output_type); + if (use_f32_compute) { + ov::mark_as_precision_sensitive(output_convert->input(0)); + } + res = output_convert; + } + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/fill.cpp b/ggml/src/ggml-openvino/openvino/op/fill.cpp new file mode 100644 index 0000000..87358e1 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/fill.cpp @@ -0,0 +1,34 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML FILL sets all elements of a tensor to a constant value. +// The constant is stored as a float in op_params[0]. +OutputVector translate_fill(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + float c; + memcpy(&c, context.get_output_op_params(), sizeof(float)); + + auto shape = context.get_input_shape(0).to_shape(); + + auto val = ov::op::v0::Constant::create(context.get_output_type(), {}, {c}); + auto target_shape = ov::op::v0::Constant::create(ov::element::i64, {shape.size()}, + std::vector(shape.begin(), shape.end())); + auto res = std::make_shared(val, target_shape); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp index 42602a7..44c6dbc 100644 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp @@ -1,15 +1,22 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" +#include "ggml-openvino/ggml-openvino-extra.h" +#include #include #include +#include #include #include #include #include +#include +#include #include #include +#include +#include #include #include #include @@ -18,13 +25,62 @@ namespace ov { namespace frontend { namespace ggml { namespace op { +static ov::Output reshape_flat_kv(const ov::Output & kv_flat, + size_t view_offset_bytes, + size_t nb1_bytes, + int64_t n_head, + int64_t head_size, + const ov::Output & attention_size) { + int64_t n_state = n_head * head_size; + int64_t layer_start_elem = (int64_t) (view_offset_bytes / (nb1_bytes / n_state)); + // Dynamic slice: [layer_start_elem, layer_start_elem + n_kv * n_state) + auto start_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {layer_start_elem}); + auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state}); + // end = start + attention_size * n_state (both static + dynamic) + auto kv_len_elems = std::make_shared(attention_size, n_state_c); + auto end_c = std::make_shared(start_c, kv_len_elems); + auto step_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto sliced = std::make_shared(kv_flat, start_c, end_c, step_c, axis_c); + + // KV cache is laid out as {n_kv, n_head, head_size} in memory + // Reshape to {1, n_kv, n_head, head_size}, then transpose to {1, n_head, n_kv, head_size} + // as required by SDPA. + auto one_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto n_head_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_head}); + auto head_size_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_size}); + // reshape: {n_kv*n_state} -> {1, n_kv, n_head, head_size} + auto new_shape = + std::make_shared(ov::OutputVector{one_c, attention_size, n_head_c, head_size_c}, 0); + auto reshaped = std::make_shared(sliced, new_shape, false); + // transpose: {1, n_kv, n_head, head_size} -> {1, n_head, n_kv, head_size} + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}); + auto ret = std::make_shared(reshaped, perm); + return ret; +} OutputVector translate_flash_attn_ext(const NodeContext & context) { - num_inputs_check(context, 4, 4); + num_inputs_check(context, 3, 4); + const bool has_mask = context.get_input_size() == 4; auto q_f32 = context.get_input(0); auto k = context.get_input(1); auto v = context.get_input(2); - auto mask = context.get_input(3); + const int op_case = context.get_op_case(); + + if (op_case == 1 || op_case == 2) { + int64_t n_state_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[3]; + int64_t n_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[1]; + size_t nb1 = context.get_view_input_stride(1, 0)[2]; + size_t offset = context.get_view_input_offset(1, 0); + ov::Output attention_size; + if (op_case == 1) { + attention_size = context.get_input("attention_size"); + } else { + attention_size = context.get_input("attention_size_static"); + } + k = reshape_flat_kv(k, offset, nb1, n_head, n_state_head, attention_size); + v = reshape_flat_kv(v, offset, nb1, n_head, n_state_head, attention_size); + } float * params = reinterpret_cast(context.get_output_op_params()); float scale = params[0]; @@ -34,36 +90,123 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { auto q = std::make_shared(q_f32, ov::element::f16); auto scale_node = std::make_shared(ov::element::f16, ov::Shape{}, std::vector{scale}); - ov::Output mask_sliced, res; - std::string mask_name = "KQ_mask_sliced"; - if (context.get_input_names()[3].find("swa") != std::string::npos) { - mask_name = "KQ_mask_swa_sliced"; - } - if (context.has_input(mask_name)) { - mask_sliced = context.get_input(mask_name); - } else { - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - auto token_len = get_dimensions(q, {2}); - mask_sliced = std::make_shared(mask, zero, token_len, one, two); + ov::Output res; + + // For stateful + ov::Output mask; + if (has_mask) { + mask = context.get_input(3); + std::string mask_name = "KQ_mask_sliced"; + if (context.get_input_names()[3].find("swa") != std::string::npos) { + mask_name = "KQ_mask_swa_sliced"; + } + if (context.has_input(mask_name)) { + mask = context.get_input(mask_name); + } + if (mask.get_element_type() != ov::element::f16) { + mask = std::make_shared(mask, ov::element::f16); + } } - if (mask_sliced.get_element_type() != ov::element::f16) { - mask_sliced = std::make_shared(mask_sliced, ov::element::f16); + //auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output kv) { + // int64_t factor = num_heads / num_heads_kv; + // if (factor > 1 && num_heads_kv > 1) { + auto q_shape = context.get_input_shape(0).to_shape(); + auto k_shape = context.get_input_shape(1).to_shape(); + const int64_t num_heads = q_shape[1]; + const int64_t num_heads_kv = k_shape[1]; + const int64_t head_size = q_shape[3]; + const int64_t factor = num_heads / num_heads_kv; + + // Manual GQA attention: enabled by default on GPU in stateless mode. + // Set GGML_OPENVINO_MANUAL_GQA_ATTN to a positive value (e.g. 1) to force-enable, + // or to 0 to force-disable. Unset falls back to the device-based default. + static const bool manual_gqa_enabled = []() { + const char * env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN"); + if (env != nullptr) { + return ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0; + } + return ggml_openvino_is_gpu(); + }(); + const bool use_manual_gqa_attention = + manual_gqa_enabled && factor > 1 && num_heads_kv > 1 && !context.is_stateful(); + + if (use_manual_gqa_attention) { + // Q, K, V arrive as [B, n_heads(_kv), S, head_size], where B is the active + // batch (n_seq_active) and may be > 1 (llama-perplexity, llama-server -np > 1) + // or dynamic. Reshape to + // K_r: [B, num_heads_kv, 1, S, head_size] + // Q_r: [B, num_heads_kv, factor, S_q, head_size] + // and let MatMul broadcast across the factor dim without materialising + // an expanded K/V. The leading 0 + special_zero=true copies B at runtime, + // so this is correct for B == 1, B > 1, and dynamic B alike. Only the head + // dims and head_size are baked in as literals; the sequence dim stays -1. + auto k_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, + std::vector{0, num_heads_kv, 1, -1, head_size}); + auto v_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, + std::vector{0, num_heads_kv, 1, -1, head_size}); + auto q_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, + std::vector{0, num_heads_kv, factor, -1, head_size}); + + auto k_r = std::make_shared(k, k_5d_shape, true); + auto v_r = std::make_shared(v, v_5d_shape, true); + auto q_r = std::make_shared(q, q_5d_shape, true); + + // QK^T → [B, num_heads_kv, factor, S_q, S_k] + auto qk = std::make_shared(q_r, k_r, /*tA=*/false, /*tB=*/true); + auto qk_scaled = std::make_shared(qk, scale_node); + + // Mask arrives as [B, 1, S_q, S_k]. Unsqueeze a factor axis at position 2 to + // get [B, 1, 1, S_q, S_k], which NUMPY-broadcasts cleanly against the + // [B, num_heads_kv, factor, S_q, S_k] scores: B==B, then 1→num_heads_kv and + // 1→factor on the head dims. + ov::Output qk_masked; + if (has_mask) { + auto mask_unsq1 = + std::make_shared(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2})); + qk_masked = std::make_shared(qk_scaled, mask_unsq1); + } else { + qk_masked = qk_scaled; + } + + auto softmax = std::make_shared(qk_masked, /*axis=*/-1); + + // softmax @ V → [B, num_heads_kv, factor, S_q, head_size] + auto attn = std::make_shared(softmax, v_r); + + // Reshape back to [B, num_heads, S_q, head_size] (combine num_heads_kv * factor). + // Leading 0 + special_zero=true copies B at runtime. + auto out_4d_shape = + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, num_heads, -1, head_size}); + auto out_4d = std::make_shared(attn, out_4d_shape, true); + + // The standard SDPA path's downstream is Transpose(0,2,1,3) → Convert(f32). + // Replicate it here so callers see the same output layout/dtype. + res = std::make_shared( + out_4d, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); + res = std::make_shared(res, ov::element::f32); + return rename_outputs_with_suffix({res}, context.get_name()); } - auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output kv) { - int64_t factor = num_heads / num_heads_kv; - if (factor > 1 && num_heads_kv > 1) { - ov::Output kv_broadcast_shape, kv_unsqueezed, new_kv_shape; + // Default path: explicit Broadcast → SDPA. Kept as the fallback because + // (a) it goes through the GPU plugin's micro-SDPA fast path (FlashAttention + // tiles via DPAS), and (b) the manual path above is still being validated. + auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output kv) { + int64_t f = n_heads / n_heads_kv; + if (f > 1 && n_heads_kv > 1) { + ov::Output kv_broadcast_shape; + ov::Output kv_unsqueezed; + ov::Output new_kv_shape; auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2}); kv_unsqueezed = std::make_shared(kv, unsqueeze_axes); - kv_broadcast_shape = ov::op::v0::Constant::create( - ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1}); + kv_broadcast_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, + {(int64_t) 1, (int64_t) 1, f, (int64_t) 1, (int64_t) 1}); new_kv_shape = - ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, num_heads, (int64_t) -1, head_size}); + ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, n_heads, (int64_t) -1, hs}); + // ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1}); + //new_kv_shape = + // ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, num_heads, (int64_t) -1, head_size}); kv = std::make_shared(kv_unsqueezed, kv_broadcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); @@ -72,14 +215,23 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { return kv; }; - auto q_shape = context.get_input_shape(0).to_shape(); - auto k_shape = context.get_input_shape(1).to_shape(); - k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k); - v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v); + //auto q_shape = context.get_input_shape(0).to_shape(); + //auto k_shape = context.get_input_shape(1).to_shape(); + //k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k); + //v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v); + k = tile_kv(num_heads, num_heads_kv, head_size, k); + v = tile_kv(num_heads, num_heads_kv, head_size, v); - auto sdpa = std::make_shared(q, k, v, mask_sliced, scale_node, false); - res = std::make_shared(sdpa, - ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); + constexpr auto causal = false; + if (has_mask) { + auto sdpa = std::make_shared(q, k, v, mask, scale_node, causal); + res = std::make_shared( + sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); + } else { + auto sdpa = std::make_shared(q, k, v, scale_node, causal); + res = std::make_shared( + sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); + } res = std::make_shared(res, ov::element::f32); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp new file mode 100644 index 0000000..9cedf68 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp @@ -0,0 +1,407 @@ +#include "gated_delta_net.hpp" + +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml-openvino/ggml-openvino-extra.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +static OutputVector translate_gated_delta_net_ref(const NodeContext & context); + +static bool match_gdn_l2_norm(const Output & normalized, Output & input, float & eps) { + // Match the RMSNorm decomposition emitted by translate_rms_norm, followed by GGML SCALE. + const auto scale = ov::as_type_ptr(normalized.get_node_shared_ptr()); + if (!scale) { + return false; + } + const auto factor = ov::as_type_ptr(scale->get_input_node_shared_ptr(1)); + const auto rms = ov::as_type_ptr(scale->get_input_node_shared_ptr(0)); + if (!factor || ov::shape_size(factor->get_shape()) != 1 || !rms) { + return false; + } + const auto x = rms->input_value(0); + const auto & shape = x.get_partial_shape(); + if (x.get_element_type() != ov::element::f32 || shape.rank() != 4 || shape[3].is_dynamic() || + shape[3].get_length() <= 0 || normalized.get_partial_shape() != shape) { + return false; + } + const float dim = static_cast(shape[3].get_length()); + if (factor->cast_vector()[0] != 1.0f / std::sqrt(dim)) { + return false; + } + const auto reciprocal = ov::as_type_ptr(rms->get_input_node_shared_ptr(1)); + if (!reciprocal) { + return false; + } + const auto one = ov::as_type_ptr(reciprocal->get_input_node_shared_ptr(0)); + const auto root = ov::as_type_ptr(reciprocal->get_input_node_shared_ptr(1)); + if (!one || ov::shape_size(one->get_shape()) != 1 || one->cast_vector()[0] != 1.0f || !root) { + return false; + } + const auto add = ov::as_type_ptr(root->get_input_node_shared_ptr(0)); + if (!add) { + return false; + } + const auto mean = ov::as_type_ptr(add->get_input_node_shared_ptr(0)); + const auto rms_eps = ov::as_type_ptr(add->get_input_node_shared_ptr(1)); + if (!mean || !mean->get_keep_dims() || !rms_eps || ov::shape_size(rms_eps->get_shape()) != 1) { + return false; + } + const auto axes = ov::as_type_ptr(mean->get_input_node_shared_ptr(1)); + const auto square = ov::as_type_ptr(mean->get_input_node_shared_ptr(0)); + if (!axes || axes->cast_vector() != std::vector{-1} || !square || + square->input_value(0) != x || square->input_value(1) != x) { + return false; + } + // RMSNorm(x, rms_eps) / sqrt(D) = x / sqrt(sum(x*x) + D*rms_eps). + eps = dim * rms_eps->cast_vector()[0]; + if (!std::isfinite(eps) || eps <= 0.0f) { + return false; + } + input = x; + return true; +} + +OutputVector translate_gated_delta_net(const NodeContext & context) { + auto v_shape = context.get_input_shape(2).to_shape(); // [B, T, H_v, S_v] + auto q_shape = context.get_input_shape(0).to_shape(); // [B, T, H_k, S_k] + + // Fused GatedDeltaNet op only supports scalar gate (kda=0). + // Fall back to reference implementation for per-key-dimension gating. + // if (kda) { + // return translate_gated_delta_net_ref(context); + // } + + // const int64_t B = v_shape[0]; + // const int64_t T = v_shape[1]; + const int64_t H_v = v_shape[2]; + const int64_t S_v = v_shape[3]; + const int64_t H_k = q_shape[2]; + // const int64_t S_k = q_shape[3]; + + auto q = context.get_input(0); + auto k = context.get_input(1); + auto v = process_view_input(context, 2, H_v * S_v, 3); + auto g = context.get_input(3); + auto beta = context.get_input(4); + auto state = context.get_input(5); + + Output raw_q, raw_k; + float q_eps = 1e-6f, k_eps = 1e-6f; + const bool fuse_qk_l2norm = ggml_openvino_is_gpu() && + match_gdn_l2_norm(q, raw_q, q_eps) && match_gdn_l2_norm(k, raw_k, k_eps); + if (fuse_qk_l2norm) { + // Keep head tiling below; GDN applies normalization per head and keeps its attention scale. + q = raw_q; + k = raw_k; + } + + // ggml maps GQA heads in tiled order, while OV GDN maps repeated heads in grouped order. + if (H_v != H_k) { + const int64_t repeat = H_v / H_k; + auto repeats = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, repeat, 1}); + q = std::make_shared(q, repeats); + k = std::make_shared(k, repeats); + } + + if (context.get_view_input_size(2)) { + // Same as l2_norm case 1 + v = std::make_shared(v, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + auto v_shape = context.get_input_shape(2).to_shape(); + std::vector reshape_pattern = {0, 0, (int64_t) v_shape[2], (int64_t) v_shape[3]}; + v = std::make_shared( + v, ov::op::v0::Constant::create(ov::element::i64, {4}, reshape_pattern), true); + } + + // ggml state layout (OV notation): [B, H_v, value_dim, key_dim] + // GatedDeltaNet op expects: [B, H_v, key_dim, value_dim] + auto state_perm = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, 1, 3, 2}); + state = std::make_shared(state, state_perm); + + g = std::make_shared(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3})); + beta = std::make_shared(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3})); + + if (context.has_input("chunk_valid_len")) { + // The last prefill chunk is padded with fabricated tokens. The recurrence is + // S_t = S_{t-1} * exp(g_t) + k_t (x) ((v_t - S_{t-1}^T k_t) * beta_t) + // so forcing g = 0 and beta = 0 makes a padded step an exact identity and keeps the final + // state equal to the state after the last real token. Attention output at those positions + // is garbage but never read. + const auto & g_ps = g.get_partial_shape(); + FRONT_END_OP_CONVERSION_CHECK(g_ps.rank().is_static() && g_ps.rank().get_length() == 3 && g_ps[1].is_static(), + "GATED_DELTA_NET pad masking requires a static token dimension"); + const int64_t n_tokens = g_ps[1].get_length(); + std::vector positions(n_tokens); + std::iota(positions.begin(), positions.end(), 0); + auto valid = std::make_shared( + ov::op::v0::Constant::create(ov::element::i64, {(size_t) n_tokens}, positions), + context.get_input("chunk_valid_len")); + auto mask = std::make_shared( + std::make_shared(valid, g.get_element_type()), + ov::op::v0::Constant::create(ov::element::i64, {2}, std::vector{0, 2})); + g = std::make_shared(g, mask); + beta = std::make_shared(beta, mask); + } + + // std::cout << "GatedDeltaNet input shapes: q=" << q.get_partial_shape() << ", k=" << k.get_partial_shape() + // << ", v=" << v.get_partial_shape() << ", g=" << g.get_partial_shape() + // << ", beta=" << beta.get_partial_shape() << ", state=" << state.get_partial_shape() << std::endl; + + auto gdn = std::make_shared(q, k, v, state, g, beta, fuse_qk_l2norm, q_eps, k_eps); + auto attn_4d = gdn->output(0); + auto state_4d = gdn->output(1); // [B, H_v, key_dim, value_dim] + + // std::cout << "GatedDeltaNet output shapes: attn=" << gdn->output(0).get_partial_shape() + // << ", new_state=" << gdn->output(1).get_partial_shape() << std::endl; + + // Transpose output state back to ggml layout [B, H_v, value_dim, key_dim] + auto state_transposed = std::make_shared(state_4d, state_perm); + if (context.get_output_names().size() == 2) { + // The canonical graph consumes the packed GGML result only through separate attention + // and state VIEWs. Publish the native outputs under those VIEW names to avoid + // flatten -> concat -> reshape -> slice -> reshape chains. This also works for B > 1. + return rename_outputs_with_suffix({attn_4d, state_transposed}, context.get_name()); + } + + auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto attn = std::make_shared(attn_4d, flat_shape_1d, false); + auto new_state = std::make_shared(state_transposed, flat_shape_1d, false); + auto packed = std::make_shared(ov::OutputVector{attn, new_state}, 0); + auto out_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, + std::vector{1, 1, -1 /*T * B + S_v * B*/, S_v * H_v}); + auto res = std::make_shared(packed, out_shape, false); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +static OutputVector translate_gated_delta_net_ref(const NodeContext & context) { + num_inputs_check(context, 6, 6); + + // Inputs (OV shapes are reversed from ggml): + // ggml: q[S_k, H_k, T, B], k[S_k, H_k, T, B], v[S_v, H_v, T, B] + // OV: q[B, T, H_k, S_k], k[B, T, H_k, S_k], v[B, T, H_v, S_v] + // ggml: g[1 or S_v, H_v, T, B], beta[1, H_v, T, B] + // OV: g[B, T, H_v, 1 or S_v], beta[B, T, H_v, 1] + // ggml: state[S_v, S_v, H_v, B] + // OV: state[B, H_v, S_v, S_v] + auto q = process_view_input_new(context, 0); + auto k = process_view_input_new(context, 1); + auto v = process_view_input_new(context, 2); + auto g = process_view_input_new(context, 3); + auto beta = process_view_input_new(context, 4); + auto state = process_view_input_new(context, 5); + + auto v_shape = context.get_input_shape(2).to_shape(); // [B, T, H_v, S_v] + auto q_shape = context.get_input_shape(0).to_shape(); // [B, T, H_k, S_k] + auto g_shape = context.get_input_shape(3).to_shape(); // [B, T, H_v, 1 or S_v] + + const int64_t B = v_shape[0]; + const int64_t T = v_shape[1]; + const int64_t H_v = v_shape[2]; + const int64_t S_v = v_shape[3]; + const int64_t H_k = q_shape[2]; + const bool kda = (g_shape[3] == (size_t) S_v); + + const int64_t rq1 = H_v / H_k; // head repeat factor + const float scale = 1.0f / std::sqrt((float) S_v); + + auto axis_1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis_2 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); + + // Transpose inputs from [B, T, H, S] to [B, H, T, S] for easier per-head processing + auto perm_0213 = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, 2, 1, 3}); + auto q_t = std::make_shared(q, perm_0213); // [B, H_k, T, S_k] + auto k_t = std::make_shared(k, perm_0213); // [B, H_k, T, S_k] + auto v_t = std::make_shared(v, perm_0213); // [B, H_v, T, S_v] + auto g_t = std::make_shared(g, perm_0213); // [B, H_v, T, 1 or S_v] + auto beta_t = std::make_shared(beta, perm_0213); // [B, H_v, T, 1] + + // Broadcast Q, K heads to match V heads if GQA is used (H_v > H_k) + ov::Output q_bh = q_t; + ov::Output k_bh = k_t; + if (rq1 > 1) { + auto q_unsq = std::make_shared(q_t, axis_2); // [B, H_k, 1, T, S] + auto k_unsq = std::make_shared(k_t, axis_2); // [B, H_k, 1, T, S] + + auto bcast_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector{1, 1, rq1, 1, 1}); + auto q_bcast = + std::make_shared(q_unsq, bcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); + auto k_bcast = + std::make_shared(k_unsq, bcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); + + // Transpose [B, H_k, rq1, T, S] -> [B, rq1, H_k, T, S] so that reshape merges + // as [rq1, H_k] giving repeat-blocks pattern matching CPU: iq1 = iv1 % H_k + auto perm_5d = ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector{0, 2, 1, 3, 4}); + auto q_transposed = std::make_shared(q_bcast, perm_5d); + auto k_transposed = std::make_shared(k_bcast, perm_5d); + + auto new_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, T, S_v}); + q_bh = std::make_shared(q_transposed, new_shape, false); + k_bh = std::make_shared(k_transposed, new_shape, false); + } + + // Merge batch and head dims: [B*H_v, T, S_v] + auto merge_bh = [&](const ov::Output & x, int64_t last_dim) { + auto shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{B * H_v, T, last_dim}); + return std::make_shared(x, shape, false); + }; + + auto q_m = merge_bh(q_bh, S_v); // [B*H_v, T, S_v] + auto k_m = merge_bh(k_bh, S_v); // [B*H_v, T, S_v] + auto v_m = merge_bh(v_t, S_v); // [B*H_v, T, S_v] + auto g_m = merge_bh(g_t, kda ? S_v : 1); // [B*H_v, T, 1 or S_v] + auto beta_m = merge_bh(beta_t, 1); // [B*H_v, T, 1] + + // State: [B, H_v, S_v, S_v] -> [B*H_v, S_v, S_v] + auto state_shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{B * H_v, S_v, S_v}); + auto state_m = std::make_shared(state, state_shape, false); + + auto scale_const = ov::op::v0::Constant::create(ov::element::f32, {}, std::vector{scale}); + + // --- Build Loop body --- + // Body parameters (no iteration counter needed, use -1 in special ports) + auto body_state = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_q = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_k = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_v = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_g = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_beta = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_iter = std::make_shared(ov::element::i64, ov::Shape{1}); + + // Condition output (always true - we rely on trip_count for termination) + auto body_cond_out = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, std::vector{true}); + + // Gather current token from invariant inputs using iteration counter + auto q_t_cur = std::make_shared(body_q, body_iter, axis_1); // [B*H_v, 1, S_v] + auto k_t_cur = std::make_shared(body_k, body_iter, axis_1); // [B*H_v, 1, S_v] + auto v_t_cur = std::make_shared(body_v, body_iter, axis_1); // [B*H_v, 1, S_v] + auto g_t_cur = std::make_shared(body_g, body_iter, axis_1); // [B*H_v, 1, 1 or S_v] + auto b_t_cur = std::make_shared(body_beta, body_iter, axis_1); // [B*H_v, 1, 1] + + // Squeeze token dim + auto q_cur = std::make_shared(q_t_cur, axis_1); // [B*H_v, S_v] + auto k_cur = std::make_shared(k_t_cur, axis_1); // [B*H_v, S_v] + auto v_cur = std::make_shared(v_t_cur, axis_1); // [B*H_v, S_v] + auto g_cur = std::make_shared(g_t_cur, axis_1); // [B*H_v, 1 or S_v] + auto b_cur = std::make_shared(b_t_cur, axis_1); // [B*H_v, 1] + + // Step 1: Apply decay gate to state + auto exp_g = std::make_shared(g_cur); // [B*H_v, 1 or S_v] + auto exp_g_unsq = std::make_shared(exp_g, axis_1); // [B*H_v, 1, 1 or S_v] + auto state_decayed = std::make_shared(body_state, exp_g_unsq); // [B*H_v, S_v, S_v] + + // Step 2: delta = (v - S @ k) * beta + auto k_col = std::make_shared(k_cur, axis_2); // [B*H_v, S_v, 1] + auto sk = std::make_shared(state_decayed, k_col, false, false); // [B*H_v, S_v, 1] + auto sk_sq = std::make_shared(sk, axis_2); // [B*H_v, S_v] + auto v_minus_sk = std::make_shared(v_cur, sk_sq); // [B*H_v, S_v] + auto delta = std::make_shared(v_minus_sk, b_cur); // [B*H_v, S_v] + + // Step 3: state += outer(delta, k) + auto delta_col = std::make_shared(delta, axis_2); // [B*H_v, S_v, 1] + auto k_row = std::make_shared(k_cur, axis_1); // [B*H_v, 1, S_v] + auto outer_prod = std::make_shared(delta_col, k_row, false, false); // [B*H_v, S_v, S_v] + auto state_updated = std::make_shared(state_decayed, outer_prod); // [B*H_v, S_v, S_v] + + // Step 4: attn_out = S @ q * scale + auto q_col = std::make_shared(q_cur, axis_2); // [B*H_v, S_v, 1] + auto sq = std::make_shared(state_updated, q_col, false, false); // [B*H_v, S_v, 1] + auto sq_squeezed = std::make_shared(sq, axis_2); // [B*H_v, S_v] + auto attn_out = std::make_shared(sq_squeezed, scale_const); // [B*H_v, S_v] + + // Unsqueeze attn_out to [B*H_v, 1, S_v] for scan output concatenation + auto attn_out_unsq = std::make_shared(attn_out, axis_1); // [B*H_v, 1, S_v] + + // --- Assemble Loop --- + // Body: results = [condition, state_updated, attn_out_unsq] + auto body = std::make_shared( + ov::OutputVector{body_cond_out, state_updated, attn_out_unsq}, + ov::ParameterVector{body_iter, body_state, body_q, body_k, body_v, body_g, body_beta}); + + auto trip_count = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, std::vector{T}); + auto exec_cond = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, std::vector{true}); + + auto loop = std::make_shared(trip_count, exec_cond); + loop->set_function(body); + loop->set_special_body_ports(ov::op::v5::Loop::SpecialBodyPorts{0, 0}); + + // Carried state: feeds back from body output 1 to body_state param + loop->set_merged_input(body_state, state_m, state_updated); + // Invariant inputs: passed through unchanged each iteration + loop->set_invariant_input(body_q, q_m); + loop->set_invariant_input(body_k, k_m); + loop->set_invariant_input(body_v, v_m); + loop->set_invariant_input(body_g, g_m); + loop->set_invariant_input(body_beta, beta_m); + + // Loop outputs: + // 1) Final state (last iteration value of state_updated) + auto final_state_out = loop->get_iter_value(state_updated, -1); // [B*H_v, S_v, S_v] + // 2) Concatenated attention outputs across all iterations along axis 1 + auto attn_concat_out = loop->get_concatenated_slices(attn_out_unsq, 0, 1, 1, -1, 1); // [B*H_v, T, S_v] + + // --- Pack outputs to match ggml layout --- + // ggml output ne = {S_v*H, T*B + S_v*B, 1, 1} -> OV [1, 1, T*B+S_v*B, S_v*H_v] + // attn: [B, T, H_v, S_v] row-major, state: [B, H_v, S_v, S_v] row-major + + // attn: [B*H_v, T, S_v] -> [B, H_v, T, S_v] -> transpose to [B, T, H_v, S_v] -> flatten + auto attn_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, T, S_v}); + auto attn_4d = std::make_shared(attn_concat_out, attn_4d_shape, false); + auto attn_perm = std::make_shared(attn_4d, perm_0213); // [B, T, H_v, S_v] + + auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{-1}); + auto attn_1d = std::make_shared(attn_perm, flat_shape_1d, false); + + // state: [B*H_v, S_v, S_v] -> [B, H_v, S_v, S_v] -> flatten + auto state_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, S_v, S_v}); + auto state_4d = std::make_shared(final_state_out, state_4d_shape, false); + if (context.get_output_names().size() == 2) { + // Match the fused translator's direct attention/state contract. + return rename_outputs_with_suffix({attn_perm, state_4d}, context.get_name()); + } + + auto state_1d = std::make_shared(state_4d, flat_shape_1d, false); + + // Concat [attn | state] and reshape to final output + auto packed = std::make_shared(ov::OutputVector{attn_1d, state_1d}, 0); + auto out_shape = + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, T * B + S_v * B, S_v * H_v}); + auto res = std::make_shared(packed, out_shape, false); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp new file mode 100644 index 0000000..20a4cfd --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include "openvino/op/op.hpp" + +namespace ov::op::internal { +/// \note GatedDeltaNet op class is under development and subject to change +/// +/// \brief Operator performing Gated Delta Net computation +/// \ingroup ov_ops_cpp_api +class OPENVINO_API GatedDeltaNet : public ov::op::Op { +public: + OPENVINO_OP("GatedDeltaNet") + + GatedDeltaNet() = default; + /// \brief Constructs a GatedDeltaNet operation. + /// + /// \param query Query tensor input. + /// \param key Key tensor input. + /// \param value Value tensor input. + /// \param recurrent_state Initial recurrent state tensor. + /// \param gate Gate tensor controlling state decay/update. + /// \param beta Beta tensor scaling the delta update. + /// \param fuse_qk_l2norm Enables fusing q/k L2-normalization into this op. + /// \param q_l2_norm_eps Epsilon used for query L2-normalization when fusion is enabled. + /// \param k_l2_norm_eps Epsilon used for key L2-normalization when fusion is enabled. + GatedDeltaNet(const Output& query, + const Output& key, + const Output& value, + const Output& recurrent_state, + const Output& gate, + const Output& beta, + const bool fuse_qk_l2norm = false, + const float q_l2_norm_eps = 1e-6F, + const float k_l2_norm_eps = 1e-6F); + + /// \brief Constructs a GatedDeltaNet operation from input vector. + /// + /// \param args Input tensor vector in order: query, key, value, recurrent_state, gate, beta. + /// \param fuse_qk_l2norm Enables fusing q/k L2-normalization into this op. + /// \param q_l2_norm_eps Epsilon used for query L2-normalization when fusion is enabled. + /// \param k_l2_norm_eps Epsilon used for key L2-normalization when fusion is enabled. + GatedDeltaNet(const ov::OutputVector& args, + const bool fuse_qk_l2norm = false, + const float q_l2_norm_eps = 1e-6F, + const float k_l2_norm_eps = 1e-6F); + void validate_and_infer_types() override; + bool visit_attributes(AttributeVisitor& visitor) override; + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; + bool get_fuse_qk_l2norm() const { + return m_fuse_qk_l2norm; + } + float get_q_l2_norm_eps() const { + return m_q_l2_norm_eps; + } + float get_k_l2_norm_eps() const { + return m_k_l2_norm_eps; + } + +private: + bool m_fuse_qk_l2norm = false; + float m_q_l2_norm_eps = 1e-6F; + float m_k_l2_norm_eps = 1e-6F; +}; + +} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp b/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp new file mode 100644 index 0000000..39bd744 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// Local mirror of OpenVINO's internal ov::op::internal::GatherMatmul op. +// +// The op class body (validate_and_infer_types / clone_with_new_inputs) is +// provided by the linked libopenvino.so; only the declaration is needed here so +// the backend can construct the node directly (same approach as GatedDeltaNet). +// The class layout must stay in sync with +// openvino/src/common/transformations/include/ov_ops/gather_matmul.hpp +// +// \note GatherMatmul op class is under development and subject to change. + +#pragma once + +#include "openvino/op/op.hpp" + +namespace ov::op::internal { + +class OPENVINO_API GatherMatmul : public ov::op::Op { +public: + OPENVINO_OP("GatherMatmul") + + GatherMatmul() = default; + + GatherMatmul(const ov::Output& A, + const ov::Output& B, + const ov::Output& indices, + const ov::Output& bias); + + GatherMatmul(const ov::Output& A, const ov::Output& B, const ov::Output& indices); + + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; + + void validate_and_infer_types() override; + +private: + // the weights matrix B is expected to have the transposed form [group, N, K] + static constexpr bool transp_a = false; + static constexpr bool transp_b = true; +}; + +} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp index 49f51b7..200821d 100644 --- a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp @@ -2,11 +2,17 @@ #include "../op_table.h" #include "../utils.h" +#include #include #include +#include +#include +#include #include #include #include +#include +#include #include #include @@ -18,21 +24,69 @@ namespace op { OutputVector translate_get_rows(const NodeContext & context) { num_inputs_check(context, 2, 2); - int op_case = context.get_op_case(); - Output res; - auto data = context.get_input(0); - auto indices = context.get_input(1); + Output data; + int64_t row_offset = 0; + if (context.get_view_input_size(0) > 0 && context.get_input(0).get_partial_shape().rank() == 2) { + // A row range view over a 2D weight Constant folds into the gather indices, + // which keeps the dequantization subgraph intact for the plugins. + const auto view_shape = context.get_view_input_ggml_shape(0, 0); + const auto view_stride = context.get_view_input_stride(0, 0); + const size_t view_offset = context.get_view_input_offset(0, 0); + const size_t row_bytes = view_stride[2]; + data = context.get_input(0); + FRONT_END_OP_CONVERSION_CHECK(row_bytes > 0 && view_offset % row_bytes == 0 && + data.get_partial_shape()[1].compatible(view_shape[3]), + "GET_ROWS: view over a weight must be a row range"); + row_offset = static_cast(view_offset / row_bytes); + } else { + data = process_view_input_new(context, 0); + } - if (op_case == 2) { - // The input comes from a VIEW - indices = process_view_input(context, 1); + auto op_case = context.get_op_case(); + if (op_case == 3 || op_case == 4) { + return {data}; + } + + ov::Output indices; + if ((op_case == 1 || op_case == 2) && context.has_input("s_copy_active_slot_len")) { + // Recurrent state reorder (inp->s_copy): slice the active (op_case 1) or extra (op_case 2) + // segment from the s_copy index list at runtime, instead of baking the static view offset, + // so the cached IR works for any number of active sequences. + auto s_copy = context.get_input(1); + auto len = context.get_input("s_copy_active_slot_len"); + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + if (op_case == 1) { + auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + indices = std::make_shared(s_copy, begin, len, step, axis); + } else { + auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX}); + indices = std::make_shared(s_copy, len, end, step, axis); + } + } else { + indices = process_view_input_new(context, 1); } // data[1,b,x,y] ind[1,1,b,x'] test-backend-ops case // data[x,y] ind[1,1,1,x'] normal case - indices = - std::make_shared(indices, ov::op::v0::Constant::create(ov::element::i64, {2}, {0, 1})); + // Squeeze the leading dims down to [b,x']. Stateful models drop one rank, so a hardcoded + // {0,1} would also strip the batch dim whenever b == 1 (every decode step). + const auto indices_rank = indices.get_partial_shape().rank(); + FRONT_END_OP_CONVERSION_CHECK(indices_rank.is_static(), "Expected static rank for GET_ROWS indices"); + std::vector indices_squeeze_axes; + for (int64_t i = 0; i + 2 < indices_rank.get_length(); ++i) { + indices_squeeze_axes.push_back(i); + } + if (!indices_squeeze_axes.empty()) { + indices = std::make_shared( + indices, ov::op::v0::Constant::create(ov::element::i64, {indices_squeeze_axes.size()}, + indices_squeeze_axes)); + } + if (row_offset != 0) { + indices = std::make_shared( + indices, ov::op::v0::Constant::create(indices.get_element_type(), {}, {row_offset})); + } if (data.get_partial_shape().rank() == 4) { if (!(data.get_partial_shape()[1].is_dynamic()) && data.get_partial_shape()[1].get_length() == 1) { // Work-around for a bug in ov cpu plugin for test-backend-ops @@ -44,7 +98,62 @@ OutputVector translate_get_rows(const NodeContext & context) { auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); data = std::make_shared(data, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - res = std::make_shared(data, indices, axis, 1); + // data: [batch, rows, ...], indices: [batch, n] - this is a batched gather + // (batch_dims=1) along the rows axis. The data and indices batch dims are + // logically equal (both == n_tokens) but reach this node through independent + // reshapes, so the GPU plugin's gather shape inference cannot prove + // data.shape[0] == indices.shape[0] and rejects the node. We must tie both + // batch dims to the SAME value, and crucially that value must stay DYNAMIC. + const auto data_ps = data.get_partial_shape(); + const auto idx_ps = indices.get_partial_shape(); + const bool data_batch_static = data_ps.rank().is_static() && data_ps[0].is_static(); + const bool idx_batch_dynamic = idx_ps.rank().is_dynamic() || idx_ps[0].is_dynamic(); + + if (data_batch_static && idx_batch_dynamic) { + // MoE per-expert-scale path: `data` is a statically-tiled REPEAT + // (ggml_repeat_4d(scale, 1, n_expert, n_tokens, 1)) whose batch dim is a + // compile-time-constant n_tokens, and every batch slice is IDENTICAL (it was + // tiled from a single [1, n_expert, 1] scale). `indices` (selected_experts) + // carries the genuinely dynamic token dim. Broadcasting indices up to the + // static data batch (the naive fix) would freeze the token dim to the + // captured prefill length, and that static value then flows through the + // gather into the residual stream, making every following decoder layer + // static -> triggers the GPU in-place-concat KV-cache corruption (only + // layer 0 stays dynamic). A static->dynamic Broadcast cannot expand, so + // instead collapse the redundant data batch to 1 and broadcast 1->dynamic to + // match the indices batch. Mathematically identical (the slices are equal), + // and the whole graph stays dynamic. + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto data_b1 = std::make_shared(data, zero, one, one, axis0); // [1, rows, ...] + + auto idx_shape = std::make_shared(indices, ov::element::i64); + auto idx_batch = get_dimensions(idx_shape, {0}); // [batch] (dynamic) + auto data_b1_shape = std::make_shared(data_b1, ov::element::i64); + const auto rank = data_ps.rank().get_length(); + std::vector rest_axes; + for (int a = 1; a < rank; ++a) { + rest_axes.push_back(a); + } + auto data_rest = get_dimensions(data_b1_shape, rest_axes); // [rows, ...] + auto data_target = std::make_shared(ov::OutputVector{idx_batch, data_rest}, 0); + data = + std::make_shared(data_b1, data_target, ov::op::BroadcastType::BIDIRECTIONAL); + res = std::make_shared(data, indices, axis, 1); + } else { + // General case: tie the indices batch to the data batch (the data batch is + // already dynamic, e.g. the routing-weights gather whose data comes from the + // activations). Broadcast indices to [data_batch, indices_n]. + auto data_shape = std::make_shared(data, ov::element::i64); + auto data_batch = get_dimensions(data_shape, {0}); // [batch] + auto idx_shape = std::make_shared(indices, ov::element::i64); + auto idx_n = get_dimensions(idx_shape, {1}); // [n] + auto idx_target = std::make_shared(ov::OutputVector{data_batch, idx_n}, 0); + indices = std::make_shared(indices, idx_target, + ov::op::BroadcastType::BIDIRECTIONAL); + res = std::make_shared(data, indices, axis, 1); + } } } else if (context.is_stateful() && data.get_partial_shape().rank() == 3) { auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); diff --git a/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp index d9fa4c2..a54870d 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -21,23 +22,26 @@ OutputVector translate_glu_geglu(const NodeContext & context) { ov::Output src0; ov::Output src1; if (context.get_input_size() == 2) { - src0 = context.get_input(0); - src1 = context.get_input(1); + // Inputs may be VIEW slices of a combined gate_up tensor (MoE experts): + // resolve them so each half has its real sliced shape, not the base tensor. + src0 = process_view_input_new(context, 0); + src1 = process_view_input_new(context, 1); } else { // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2. // Both halves are nc elements; if the dimension is odd, the last element is dropped. // Use Slice instead of Split to handle odd dimensions correctly. - auto combined = context.get_input(0); + // Resolve a VIEW input (e.g. non-contiguous slice) to its real shape first. + auto combined = process_view_input_new(context, 0); auto combined_shape = combined.get_partial_shape(); int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length(); int64_t nc = last_dim_val / 2; - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); + auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); + auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); src0 = std::make_shared(combined, start0, stop0, step, axis); src1 = std::make_shared(combined, start1, stop1, step, axis); @@ -49,6 +53,16 @@ OutputVector translate_glu_geglu(const NodeContext & context) { std::swap(src0, src1); } + if (context.is_static()) { + // TODO: Temporary solution for NPU accuracy issue due to fp16 overflow + // To be removed once permanent solution is implemented + // Justification: + // For |x| > 5, GELU(x) ≈ max(x, 0) (behaves like ReLU) + // So Clamp(-10, 10) only affects values where GELU would return ≈ x anyway. + // The only loss: values > 10 get mapped to 10 instead of x. + // In practice, FFN intermediates rarely exceed 10 after GEGLU gating. + src0 = std::make_shared(src0, -10.0, 10.0); + } auto gelu = std::make_shared(src0); auto res = std::make_shared(gelu, src1); diff --git a/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp new file mode 100644 index 0000000..385d75f --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp @@ -0,0 +1,62 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_glu_geglu_quick(const NodeContext & context) { + num_inputs_check(context, 1, 2); + + ov::Output src0; + ov::Output src1; + if (context.get_input_size() == 2) { + src0 = process_view_input_new(context, 0); + src1 = process_view_input_new(context, 1); + } else { + // split along last axis, nc = ne[0] / 2 + auto combined = process_view_input_new(context, 0); + auto combined_shape = combined.get_partial_shape(); + int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length(); + int64_t nc = last_dim_val / 2; + + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); + auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); + auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); + + src0 = std::make_shared(combined, start0, stop0, step, axis); + src1 = std::make_shared(combined, start1, stop1, step, axis); + } + + int32_t * params = context.get_output_op_params(); + const int32_t swapped = params[1]; + if (swapped) { + std::swap(src0, src1); + } + + // GELU_QUICK(x) = x * sigmoid(1.702 * x) + // Create the constant in the same type as src0 to avoid f16/f32 mismatch. + auto input_type = src0.get_element_type(); + auto coef = ov::op::v0::Constant::create(input_type, ov::Shape{}, {1.702f}); + auto gated = std::make_shared(src0, coef); + auto res = std::make_shared(gated, src1); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp index 00ed795..2c206a5 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp @@ -3,41 +3,48 @@ #include "../utils.h" #include +#include #include #include +#include +#include #include +#include #include -#include #include +#include namespace ov { namespace frontend { namespace ggml { namespace op { -OutputVector translate_glu_swiglu(const NodeContext & context) { +static std::pair, ov::Output> get_glu_inputs(const NodeContext & context) { num_inputs_check(context, 1, 2); ov::Output src0; ov::Output src1; if (context.get_input_size() == 2) { - src0 = context.get_input(0); - src1 = context.get_input(1); + // Inputs may be VIEW slices of a combined gate_up tensor (MoE experts): + // resolve them so each half has its real sliced shape, not the base tensor. + src0 = process_view_input_new(context, 0); + src1 = process_view_input_new(context, 1); } else { // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2. // Both halves are nc elements; if the dimension is odd, the last element is dropped. // Use Slice instead of Split to handle odd dimensions correctly. - auto combined = context.get_input(0); + // Resolve a VIEW input (e.g. non-contiguous slice) to its real shape first. + auto combined = process_view_input_new(context, 0); auto combined_shape = combined.get_partial_shape(); int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length(); int64_t nc = last_dim_val / 2; - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); + auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); + auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); src0 = std::make_shared(combined, start0, stop0, step, axis); src1 = std::make_shared(combined, start1, stop1, step, axis); @@ -49,13 +56,75 @@ OutputVector translate_glu_swiglu(const NodeContext & context) { std::swap(src0, src1); } - auto sigmoid = std::make_shared(src0); - auto silu = std::make_shared(src0, sigmoid); + return {src0, src1}; +} + +OutputVector translate_glu_swiglu(const NodeContext & context) { + auto [src0, src1] = get_glu_inputs(context); + + auto silu = std::make_shared(src0); auto res = std::make_shared(silu, src1); return rename_outputs_with_suffix({res}, context.get_name()); } +OutputVector translate_glu_swiglu_oai(const NodeContext & context) { + auto [src0, src1] = get_glu_inputs(context); + + const int32_t * params = context.get_output_op_params(); + const float alpha = reinterpret_cast(params)[2]; + const float limit = reinterpret_cast(params)[3]; + + // Compute in f32: alpha_const/one are f32, and Swish/Add need matching input types. + auto output_type = context.get_output_type(); + if (src0.get_element_type() != ov::element::f32) { + src0 = std::make_shared(src0, ov::element::f32); + } + if (src1.get_element_type() != ov::element::f32) { + src1 = std::make_shared(src1, ov::element::f32); + } + + auto gate = std::make_shared(src0, -std::numeric_limits::infinity(), limit); + auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha}); + auto out_glu = std::make_shared(gate, alpha_const); + + auto up = std::make_shared(src1, -limit, limit); + auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f}); + auto up_plus_one = std::make_shared(up, one); + ov::Output res = std::make_shared(out_glu, up_plus_one); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_glu_swiglu_clamp(const NodeContext & context) { + auto [src0, src1] = get_glu_inputs(context); + + const int32_t * params = context.get_output_op_params(); + const float limit = reinterpret_cast(params)[3]; + + // Compute in f32: f16 Swish/Clamp rounding drifts past the 1e-7 test tolerance. + auto output_type = context.get_output_type(); + if (src0.get_element_type() != ov::element::f32) { + src0 = std::make_shared(src0, ov::element::f32); + } + if (src1.get_element_type() != ov::element::f32) { + src1 = std::make_shared(src1, ov::element::f32); + } + + auto gate = std::make_shared(src0, -std::numeric_limits::infinity(), limit); + auto silu = std::make_shared(gate); + auto up = std::make_shared(src1, -limit, limit); + ov::Output res = std::make_shared(silu, up); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + } // namespace op } // namespace ggml } // namespace frontend diff --git a/ggml/src/ggml-openvino/openvino/op/im2col.cpp b/ggml/src/ggml-openvino/openvino/op/im2col.cpp new file mode 100644 index 0000000..469d1de --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/im2col.cpp @@ -0,0 +1,239 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_im2col(const NodeContext & context) { + num_inputs_check(context, 2, 2); + const int32_t * params = context.get_output_op_params(); + int32_t s0 = params[0]; + int32_t s1 = params[1]; + int32_t p0 = params[2]; + int32_t p1 = params[3]; + int32_t d0 = params[4]; + int32_t d1 = params[5]; + bool is_2D = params[6] == 1; + ov::Output res; + + ov::Output image = context.get_input(1); + const ov::Shape kernel_shape = context.get_input(0).get_shape(); + + const size_t IC = is_2D ? kernel_shape[1] : kernel_shape[2]; + const size_t KH = is_2D ? kernel_shape[2] : 1; + const size_t KW = kernel_shape[3]; + + int32_t stride_w = s0; + int32_t stride_h = is_2D ? s1 : 1; + int32_t pad_w = p0; + int32_t pad_h = is_2D ? p1 : 0; + int32_t dil_w = d0; + int32_t dil_h = is_2D ? d1 : 1; + + if (!is_2D) { + // GGML input shape: [IW, IC, N, 1] + // OpenVINO input shape: [1, N, IC, IW] + // Reshape image to: [N, IC, 1, IW] + const ov::Shape image_shape = image.get_shape(); + const size_t N = image_shape[1]; + const size_t IW = image_shape[3]; + auto image_reshape_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, + std::vector{static_cast(N), static_cast(IC), 1, static_cast(IW)}); + image = std::make_shared(image, image_reshape_shape, false); + } + + const ov::Shape patch_sizes = {KH, KW}; + const ov::Strides strides = {static_cast(stride_h), static_cast(stride_w)}; + const ov::Shape rates = {static_cast(dil_h), static_cast(dil_w)}; + + auto pads_begin = + ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector{0, 0, pad_h, pad_w}); + auto pads_end = + ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector{0, 0, pad_h, pad_w}); + + auto pad = std::make_shared(image, pads_begin, pads_end, ov::op::PadMode::CONSTANT); + auto patches = + std::make_shared(pad, patch_sizes, strides, rates, ov::op::PadType::VALID); + + // [N, KH*KW*IC, OH, OW] → [N, OH, OW, KH*KW*IC] + auto perm1 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector{0, 2, 3, 1}); + auto t1 = std::make_shared(patches, perm1); + + // [N, OH, OW, KH*KW*IC] → [N, OH, OW, KH*KW, IC] + const ov::Shape out_shape = t1->get_output_shape(0); + const size_t N = out_shape[0]; + const size_t OH = out_shape[1]; + const size_t OW = out_shape[2]; + auto reshape1_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{5}, + std::vector{static_cast(N), static_cast(OH), static_cast(OW), + static_cast(KH * KW), static_cast(IC)}); + auto r1 = std::make_shared(t1, reshape1_shape, false); + + // [N, OH, OW, KH*KW, IC] → [N, OH, OW, IC, KH*KW] + auto perm2 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, std::vector{0, 1, 2, 4, 3}); + auto t2 = std::make_shared(r1, perm2); + + // flatten back to [N, OH, OW, IC*KH*KW] + auto r2_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, + std::vector{static_cast(N), static_cast(OH), static_cast(OW), + static_cast(IC * KH * KW)}); + res = std::make_shared(t2, r2_shape, false); + + if (!is_2D) { + // [N, 1, OW, IC * KW] -> [1, N, OW, IC * KW] + auto final_reshape_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, + std::vector{1, static_cast(N), static_cast(OW), static_cast(IC * KW)}); + res = std::make_shared(res, final_reshape_shape, false); + } + + auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_im2col_3d(const NodeContext & context) { + num_inputs_check(context, 2, 2); + const int32_t * params = context.get_output_op_params(); + int32_t s0 = params[0]; + int32_t s1 = params[1]; + int32_t s2 = params[2]; + int32_t p0 = params[3]; + int32_t p1 = params[4]; + int32_t p2 = params[5]; + int32_t d0 = params[6]; + int32_t d1 = params[7]; + int32_t d2 = params[8]; + int32_t IC = params[9]; + + ov::Output image = process_view_input_new(context, 1); + const ov::Shape kernel_shape = context.get_input(0).get_shape(); + const ov::Shape image_shape = image.get_shape(); + const ov::Shape out_shape = context.get_output_shape().to_shape(); + + const size_t KD = kernel_shape[1]; + const size_t KH = kernel_shape[2]; + const size_t KW = kernel_shape[3]; + + const size_t N = image_shape[0] / static_cast(IC); + const size_t ID = image_shape[1]; + const size_t IH = image_shape[2]; + const size_t IW = image_shape[3]; + + const size_t OD = (ID + 2 * p2 - d2 * (KD - 1) - 1) / s2 + 1; + const size_t OH = (IH + 2 * p1 - d1 * (KH - 1) - 1) / s1 + 1; + const size_t OW = (IW + 2 * p0 - d0 * (KW - 1) - 1) / s0 + 1; + + if (N == 0 || OD == 0 || OH == 0 || OW == 0) { + auto output_type = context.get_output_type(); + ov::Output res = ov::op::v0::Constant::create( + output_type, ov::Shape{N * OD, OH, OW, static_cast(IC * KD * KH * KW)}, {}); + return rename_outputs_with_suffix({res}, context.get_name()); + } + + const size_t IH_pad = IH + 2 * p1; + const size_t IW_pad = IW + 2 * p0; + + auto image_5d_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{5}, + std::vector{static_cast(N), static_cast(IC), static_cast(ID), + static_cast(IH), static_cast(IW)}); + auto image_5d = std::make_shared(image, image_5d_shape, false); + + auto pads_begin = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{5}, std::vector{0, 0, p2, p1, p0}); + auto pads_end = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{5}, std::vector{0, 0, p2, p1, p0}); + auto pad_3d = std::make_shared(image_5d, pads_begin, pads_end, ov::op::PadMode::CONSTANT); + + const ov::Shape patch_sizes = {KH, KW}; + const ov::Strides strides = {static_cast(s1), static_cast(s0)}; + const ov::Shape rates = {static_cast(d1), static_cast(d0)}; + + ov::OutputVector kd_slices; + kd_slices.reserve(KD); + + auto perm_nod = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, {0, 2, 1, 3, 4}); + auto reshape_4d_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, + std::vector{static_cast(N * OD), static_cast(IC), + static_cast(IH_pad), static_cast(IW_pad)}); + auto perm1 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, {0, 2, 3, 1}); + auto r1_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{5}, + std::vector{static_cast(N * OD), static_cast(OH), static_cast(OW), + static_cast(KH * KW), static_cast(IC)}); + auto perm2 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, {0, 1, 2, 4, 3}); + auto r2_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{6}, + std::vector{static_cast(N * OD), static_cast(OH), static_cast(OW), + static_cast(IC), 1, static_cast(KH * KW)}); + auto step_c = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {static_cast(s2)}); + auto axes_c = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {2}); + + for (size_t ikd = 0; ikd < KD; ++ikd) { + auto start_c = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{1}, {static_cast(ikd * d2)}); + auto stop_c = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{1}, {static_cast(ikd * d2 + OD * s2)}); + auto depth_slice = std::make_shared(pad_3d, start_c, stop_c, step_c, axes_c); + auto depth_slice_trans = std::make_shared(depth_slice, perm_nod); + auto depth_slice_4d = std::make_shared(depth_slice_trans, reshape_4d_shape, false); + + auto patches = std::make_shared( + depth_slice_4d, patch_sizes, strides, rates, ov::op::PadType::VALID); + auto t1 = std::make_shared(patches, perm1); + auto r1 = std::make_shared(t1, r1_shape, false); + auto t2 = std::make_shared(r1, perm2); + auto r2 = std::make_shared(t2, r2_shape, false); + kd_slices.push_back(r2); + } + + ov::Output res; + if (KD == 1) { + res = kd_slices[0]; + } else { + res = std::make_shared(kd_slices, 4); + } + + auto final_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, + std::vector{static_cast(N * OD), static_cast(OH), static_cast(OW), + static_cast(IC * KD * KH * KW)}); + res = std::make_shared(res, final_shape, false); + + auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp b/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp new file mode 100644 index 0000000..6e0485d --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp @@ -0,0 +1,61 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_l2_norm(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input_node = process_view_input_new(context, 0); + + if (context.get_op_case() == 1) { + // 92: [ 128, 16, 1, 2] VIEW q_conv-1 + // [ 6144, 1, 2, 1] 0: UNARY conv_output_silu-1 + // 93: [ 128, 16, 1, 2] L2_NORM q_conv_predelta-1 + // [ 128, 16, 1, 2] 0: VIEW q_conv-1 + auto output_shape = context.get_output_shape().to_shape(); + input_node = process_view_input(context, 0, output_shape[2] * output_shape[3], 3); + input_node = + std::make_shared(input_node, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + + std::vector reshape_pattern = {0, 0, (int64_t) output_shape[2], (int64_t) output_shape[3]}; + input_node = std::make_shared( + input_node, ov::op::v0::Constant::create(ov::element::i64, {4}, reshape_pattern), true); + } + + auto squared = std::make_shared(input_node, input_node); + + auto sum_squared = std::make_shared( + squared, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); + + auto l2_norm = std::make_shared(sum_squared); + + float eps; + memcpy(&eps, context.get_output_op_params(), sizeof(float)); + + auto eps_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps}); + auto clamped_norm = std::make_shared(l2_norm, eps_const); + + auto res = std::make_shared(input_node, clamped_norm); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/mean.cpp b/ggml/src/ggml-openvino/openvino/op/mean.cpp new file mode 100644 index 0000000..1e9b10b --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/mean.cpp @@ -0,0 +1,27 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_mean(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + auto res = std::make_shared(input, axis, true); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp b/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp new file mode 100644 index 0000000..07e94c6 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp @@ -0,0 +1,90 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// Local mirror of OpenVINO's internal ov::op::internal::MOE and MOECompressed ops. +// +// The class bodies are provided by the linked libopenvino.so; only the declarations are +// needed here so the backend can construct the node directly (same approach as +// GatherMatmul and GatedDeltaNet). The class layout must stay in sync with +// openvino/src/core/dev_api/openvino/op/moe.hpp +// openvino/src/common/transformations/include/ov_ops/moe_compressed.hpp +// +// \note MOE op classes are under development and subject to change. + +#pragma once + +#include + +#include "openvino/core/type/element_type.hpp" +#include "openvino/op/op.hpp" + +namespace ov::op::internal { + +class OPENVINO_API MOE : public ov::op::Op { +public: + OPENVINO_OP("MOE") + + MOE() = default; + + MOE(const OutputVector & args) : Op(args) {} + + enum class Expert_type { GEMM2_BIAS_SWIGLU_CLAMP, GEMM3_SWIGLU }; + + enum class Activation_type { SWIGLU, GEGLU_TANH, GEGLU_ERF }; + + struct Config { + Expert_type expert_type{ Expert_type::GEMM2_BIAS_SWIGLU_CLAMP }; + float expert_alpha{ 0.0f }; + float expert_beta{ 1.0f }; + size_t gate_idx{ 0 }; + Activation_type activation_type{ Activation_type::SWIGLU }; + }; + + MOE(const OutputVector & args, const Config & config); + + const Config & get_config() const; + void set_config(const Config & config); + + bool visit_attributes(AttributeVisitor & visitor) override; + void validate_and_infer_types() override; + std::shared_ptr clone_with_new_inputs(const OutputVector & new_args) const override; + +private: + Config m_config; +}; + +class OPENVINO_API MOECompressed : public MOE { +public: + OPENVINO_OP("MOECompressed", "", ov::op::internal::MOE) + + MOECompressed() = default; + + struct Config : public MOE::Config { + size_t hidden_size = 0; + size_t inter_size = 0; + size_t num_expert = 0; + size_t num_shared_expert = 0; + size_t top_k = 0; + // numeric_limits::max() means per_channel compression (single group) + size_t group_size = 0; + bool has_batch_dim = false; + bool has_zp = false; + ov::element::Type out_type = ov::element::dynamic; + std::optional scale_factor; + }; + + MOECompressed(const OutputVector & args, const Config & config); + + const Config & get_config() const { return m_config; } + + void set_scale_factor(float scale_factor) { m_config.scale_factor = scale_factor; } + + bool visit_attributes(AttributeVisitor & visitor) override; + void validate_and_infer_types() override; + std::shared_ptr clone_with_new_inputs(const OutputVector & new_args) const override; + +protected: + Config m_config; +}; + +} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp new file mode 100644 index 0000000..52c3f5e --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp @@ -0,0 +1,254 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "gather_matmul.hpp" +#include "ggml-openvino/ggml-openvino-extra.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +namespace { + +std::shared_ptr const_i64(const std::vector & values) { + return ov::op::v0::Constant::create(ov::element::i64, ov::Shape{values.size()}, values); +} + +ov::Output slice_axis(const ov::Output & input, int64_t axis, int64_t begin, int64_t end) { + return std::make_shared(input, const_i64({begin}), const_i64({end}), const_i64({1}), + const_i64({axis})); +} + +// GGML tensors are rank 4, but stateful models drop the leading size-1 batch dim, so +// activations and ids arrive one rank lower. Pick the trailing dims by actual rank. +std::vector trailing_dims(const ov::Output & input, int count) { + const auto rank = input.get_partial_shape().rank(); + FRONT_END_OP_CONVERSION_CHECK(rank.is_static(), "Expected static rank for MUL_MAT_ID input"); + const int rank_len = static_cast(rank.get_length()); + FRONT_END_OP_CONVERSION_CHECK(rank_len >= count, "MUL_MAT_ID input rank is too low"); + + std::vector dims; + for (int i = rank_len - count; i < rank_len; ++i) { + dims.push_back(i); + } + return dims; +} + +ov::Output static_shape_dims_or_shapeof(const ov::Output & input, + const std::vector & dims) { + const auto & partial_shape = input.get_partial_shape(); + if (partial_shape.is_static()) { + std::vector values; + values.reserve(dims.size()); + for (const int64_t dim : dims) { + values.push_back(partial_shape[dim].get_length()); + } + return const_i64(values); + } + + auto shape = std::make_shared(input, ov::element::i64); + return get_dimensions(shape, dims); +} + +ov::Output translate_mul_mat_id_mxfp4_packed(const NodeContext & context, + ov::Output expert_weights, + ov::Output activations, + ov::Output ids) { + auto packed_shape = expert_weights.get_partial_shape().to_shape(); + FRONT_END_OP_CONVERSION_CHECK(packed_shape.size() == 5 && packed_shape[4] == 17, + "Expected packed MXFP4 expert weights with shape [1, n_expert, m, k_blocks, 17]"); + + const int64_t n_expert = static_cast(packed_shape[1]); + const int64_t rows = static_cast(packed_shape[2]); + const int64_t k_blocks = static_cast(packed_shape[3]); + const int64_t qk = 32; + const int64_t cols = k_blocks * qk; + + auto packed_shape_4d = const_i64({n_expert, rows, k_blocks, 17}); + expert_weights = std::make_shared(expert_weights, packed_shape_4d, false); + + auto activations_shape_4d = std::make_shared(activations, ov::element::i64); + auto ids_shape_4d = std::make_shared(ids, ov::element::i64); + auto activations_shape_3d = get_dimensions(activations_shape_4d, {1, 2, 3}); + auto ids_shape_2d = get_dimensions(ids_shape_4d, {2, 3}); + + activations = std::make_shared(activations, activations_shape_3d, false); + ids = std::make_shared(ids, ids_shape_2d, false); + if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { + ids = std::make_shared(ids, ov::element::i32); + } + + auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); + + static const std::vector f4e2m1_lut = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, + -0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f}; + std::vector e8m0_lut(256); + for (size_t i = 0; i < e8m0_lut.size(); ++i) { + uint32_t bits = static_cast(i) << 23; + memcpy(&e8m0_lut[i], &bits, sizeof(float)); + } + e8m0_lut[0] = std::numeric_limits::min() / 2.0f; + e8m0_lut[255] = std::numeric_limits::quiet_NaN(); + + auto f4_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{f4e2m1_lut.size()}, f4e2m1_lut); + auto scale_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{e8m0_lut.size()}, e8m0_lut); + + auto selected_packed_weights = std::make_shared(expert_weights, ids, gather_axis); + auto scale_byte = slice_axis(selected_packed_weights, 4, 0, 1); + auto qs = slice_axis(selected_packed_weights, 4, 1, 17); + auto low = std::make_shared( + qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {0x0F}), ov::op::AutoBroadcastType::NUMPY); + auto high_shift = std::make_shared( + qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {4}), ov::op::AutoBroadcastType::NUMPY); + auto nibbles = std::make_shared(ov::OutputVector{low, high_shift}, 4); + auto nibble_indices = std::make_shared(nibbles, ov::element::i32); + auto weights_f32 = std::make_shared(f4_lut, nibble_indices, gather_axis); + + auto scale_indices = std::make_shared(scale_byte, ov::element::i32); + auto scales_f32 = std::make_shared(scale_lut, scale_indices, gather_axis); + ov::Output selected_weights = std::make_shared(weights_f32, scales_f32, + ov::op::AutoBroadcastType::NUMPY); + + auto ids_shape = std::make_shared(ids, ov::element::i64); + auto selected_weights_target_dims = std::make_shared( + ov::OutputVector{get_dimensions(ids_shape, {0, 1}), const_i64({rows, cols})}, 0); + selected_weights = std::make_shared(selected_weights, selected_weights_target_dims, false); + + auto activations_shape = std::make_shared(activations, ov::element::i64); + ov::Output acts_target_dims = std::make_shared( + ov::OutputVector{ + get_dimensions(activations_shape, {0}), + get_dimensions(ids_shape, {1}), + get_dimensions(activations_shape, {2}), + }, + 0); + ov::Output acts_broadcasted = + std::make_shared(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL); + + auto activations_expanded = std::make_shared(acts_broadcasted, const_i64({2})); + ov::Output result = + std::make_shared(activations_expanded, selected_weights, false, true); + + auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {rows}); + auto result_target_dims = std::make_shared( + ov::OutputVector{batch_dim, get_dimensions(ids_shape, {0, 1}), row_dim}, 0); + result = std::make_shared(result, result_target_dims, false); + + const auto output_type = context.get_output_type(); + if (result.get_element_type() != output_type) { + result = std::make_shared(result, output_type); + } + return result; +} + +} // namespace + +OutputVector translate_mul_mat_id(const NodeContext & context) { + num_inputs_check(context, 3, 3); + + auto expert_weights = process_view_input_new(context, 0); + auto activations = process_view_input_new(context, 1); + auto ids = process_view_input_new(context, 2); + + if (activations.get_partial_shape().rank() == 3) { + activations = std::make_shared(activations, const_i64({0})); + } + if (ids.get_partial_shape().rank() == 3) { + ids = std::make_shared(ids, const_i64({0})); + } + + if (expert_weights.get_element_type() == ov::element::u8 && expert_weights.get_partial_shape().rank().is_static() && + expert_weights.get_partial_shape().rank().get_length() == 5) { + return rename_outputs_with_suffix({translate_mul_mat_id_mxfp4_packed(context, expert_weights, activations, ids)}, + context.get_name()); + } + + // General (non-packed) path: dense F32/F16/BF16 weights, or the f16 dequantization chain for + // quantized MoE experts (see extract_quantized_weights / make_int4_weights / make_int8_weights in + // ggml-quants.cpp). Routed through ov::op::internal::GatherMatmul instead of a naive + // Gather+Broadcast+MatMul, so the selected expert's full weight matrix is never materialized per + // token. The CPU plugin's ConvertGatherMatmulToGatherMatmulCompressed pass (run during + // compile_model) fuses the dequantization chain feeding GatherMatmul's B input into a + // GatherMatmulCompressed node automatically, as long as MarkDequantization has marked the chain -- + // see translate_session.cpp's apply_transformations for the MarkDequantization registration. + // + // OpenVINO sees GGML tensors in reversed dimension order: + // activations: [1, n_tokens, n_used_or_1, k] + // ids: [1, 1, n_tokens, n_used] + // expert_weights is either [1, n_expert, m, k] (4D, e.g. non-quantized weights without a + // pre-built extra) or already [n_expert, m, k] (3D, weights routed through + // process_weight_tensor) -- GatherMatmul's B input expects the latter. + auto expert_weights_rank = expert_weights.get_partial_shape().rank(); + FRONT_END_OP_CONVERSION_CHECK(expert_weights_rank.is_static(), + "Expected static rank for MUL_MAT_ID expert weights"); + if (expert_weights_rank.get_length() == 4) { + auto expert_weights_shape_3d = static_shape_dims_or_shapeof(expert_weights, {1, 2, 3}); + expert_weights = std::make_shared(expert_weights, expert_weights_shape_3d, false); + } + + auto activations_shape_3d = static_shape_dims_or_shapeof(activations, trailing_dims(activations, 3)); + auto ids_shape_2d = static_shape_dims_or_shapeof(ids, trailing_dims(ids, 2)); + + activations = std::make_shared(activations, activations_shape_3d, false); + ids = std::make_shared(ids, ids_shape_2d, false); + + if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { + ids = std::make_shared(ids, ov::element::i32); + } + + const auto output_type = context.get_output_type(); + const auto activations_type = ggml_openvino_is_gpu() ? ov::element::f16 : ov::element::f32; + if (activations.get_element_type() != activations_type) { + activations = std::make_shared(activations, activations_type); + } + + // GatherMatmul's A input is [n_used_or_1, n_tokens, k]; activations_3d is + // [n_tokens, n_used_or_1, k]. + auto activations_transpose_order = const_i64({1, 0, 2}); + ov::Output activations_for_gather = + std::make_shared(activations, activations_transpose_order); + + ov::Output result = std::make_shared(activations_for_gather, expert_weights, ids); + + // result is [n_used, n_tokens, m]; GGML expects [1, n_tokens, n_used, m], except on the + // stateful path where the leading batch dim is dropped. + auto result_transpose_order = const_i64({1, 0, 2}); + result = std::make_shared(result, result_transpose_order); + if (!context.is_stateful()) { + auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + result = std::make_shared(result, unsqueeze_axes); + } + + if (result.get_element_type() != output_type) { + result = std::make_shared(result, output_type); + } + + return rename_outputs_with_suffix({result}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp index 38edec8..9d4315a 100644 --- a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp @@ -29,20 +29,11 @@ OutputVector translate_mulmat(const NodeContext & context) { int op_case = context.get_op_case(); - ov::Output res; - ov::Output B = context.get_input(0); - ov::Output A = context.get_input(1); + ov::Output B = process_view_input_new(context, 0); + ov::Output A = process_view_input_new(context, 1); - bool transpose_b = true; - if (op_case == 2) { - B = B.get_node_shared_ptr()->input_value(0); - transpose_b = false; - } else if (op_case == 3) { - B = process_view_input(context, 0); - A = process_view_input(context, 1); - } if (A.get_element_type() != B.get_element_type()) { - B = std::make_shared(context.get_input(0), context.get_input_type(1)); + B = std::make_shared(B, context.get_input_type(1)); } auto B_shape = context.get_input_shape(0).to_shape(); @@ -55,6 +46,7 @@ OutputVector translate_mulmat(const NodeContext & context) { auto batch_small = A_batch_larger ? B_batch : A_batch; Output Z = A_batch_larger ? B : A; + auto Z_shape = A_batch_larger ? B_shape : A_shape; int64_t factor = batch_large / batch_small; if (factor > 1 && batch_small > 1) { auto batch_large_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{batch_large}); @@ -67,7 +59,11 @@ OutputVector translate_mulmat(const NodeContext & context) { auto broadcast_shape = ov::op::v0::Constant::create( ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1}); auto new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 0, batch_large, (int64_t) -1, (int64_t) A_shape[3]}); + {(int64_t) 0, batch_large, (int64_t) -1, (int64_t) Z_shape[3]}); + if (op_case == 2) { + new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, + {(int64_t) 0, batch_large, (int64_t) Z_shape[2], (int64_t) -1}); + } auto Z_broadcasted = std::make_shared(Z_unsqueezed, broadcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); @@ -79,7 +75,13 @@ OutputVector translate_mulmat(const NodeContext & context) { A = Z; } - res = std::make_shared(A, B, false, transpose_b); + bool transpose_b = true; + ov::Output res = std::make_shared(A, B, false, transpose_b); + + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/norm.cpp b/ggml/src/ggml-openvino/openvino/op/norm.cpp new file mode 100644 index 0000000..8660c65 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/norm.cpp @@ -0,0 +1,31 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_norm(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input_node = process_view_input_new(context, 0); + float eps; + memcpy(&eps, context.get_output_op_params(), sizeof(float)); + + auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + auto res = std::make_shared(input_node, axes, true, eps, ov::op::MVNEpsMode::INSIDE_SQRT); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/pad.cpp b/ggml/src/ggml-openvino/openvino/op/pad.cpp new file mode 100644 index 0000000..ae3d7be --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/pad.cpp @@ -0,0 +1,94 @@ +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +namespace { + +ov::Output translate_circular_pad(ov::Output input, + const std::array & pads, + const ov::Shape & input_shape) { + ov::Output result = std::move(input); + + const std::array pads_begin = {pads[6], pads[4], pads[2], pads[0]}; + const std::array pads_end = {pads[7], pads[5], pads[3], pads[1]}; + + for (size_t axis = 0; axis < input_shape.size(); ++axis) { + const int64_t input_dim = static_cast(input_shape[axis]); + const int64_t pad_begin = pads_begin[axis]; + const int64_t pad_end = pads_end[axis]; + + if (pad_begin == 0 && pad_end == 0) { + continue; + } + + FRONT_END_CHECK_IMPLEMENTED(input_dim > 0, "Circular PAD requires static non-zero input dimensions"); + + std::vector indices(static_cast(input_dim + pad_begin + pad_end)); + for (int64_t index = 0; index < static_cast(indices.size()); ++index) { + int64_t wrapped = (index - pad_begin) % input_dim; + if (wrapped < 0) { + wrapped += input_dim; + } + indices[static_cast(index)] = wrapped; + } + + auto gather_indices = ov::op::v0::Constant::create(ov::element::i64, {indices.size()}, indices); + auto gather_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {axis}); + result = std::make_shared(result, gather_indices, gather_axis); + } + + return result; +} + +} // namespace + +OutputVector translate_pad(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + if (context.get_input_shape(0) == context.get_output_shape()) { + return rename_outputs_with_suffix({input}, context.get_name()); + } + + const int32_t * op_params = context.get_output_op_params(); + FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "PAD requires output op params"); + + const std::array pads = {op_params[0], op_params[1], op_params[2], op_params[3], + op_params[4], op_params[5], op_params[6], op_params[7]}; + const bool circular = op_params[8] != 0; + + if (circular) { + auto res = translate_circular_pad(input, pads, context.get_input_shape(0).to_shape()); + return rename_outputs_with_suffix({res}, context.get_name()); + } + + const std::vector pads_begin = {pads[6], pads[4], pads[2], pads[0]}; + const std::vector pads_end = {pads[7], pads[5], pads[3], pads[1]}; + + auto pads_begin_node = ov::op::v0::Constant::create(ov::element::i64, {pads_begin.size()}, pads_begin); + auto pads_end_node = ov::op::v0::Constant::create(ov::element::i64, {pads_end.size()}, pads_end); + auto pad_value = ov::op::v0::Constant::create(context.get_input_type(0), ov::Shape{}, {0}); + auto res = + std::make_shared(input, pads_begin_node, pads_end_node, pad_value, ov::op::PadMode::CONSTANT); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index 4c800f9..df4f038 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace ov { namespace frontend { @@ -22,16 +23,44 @@ OutputVector translate_permute(const NodeContext & context) { num_inputs_check(context, 1, 1); int op_case = context.get_op_case(); - FRONT_END_CHECK_IMPLEMENTED(op_case == 1 || op_case == 2 || op_case == 3 || op_case == 4, - "Unsupported PERMUTE case"); + FRONT_END_CHECK_IMPLEMENTED(op_case != 0, "Unsupported PERMUTE case"); + // op_case 1 is trivial permute + // op_case 2 is to permute Q. It has a preceding VIEW that reshapes Q to restore the sequqence dimension + // op_case 3 4 it to permute KV cache in the default layout + // op_case 5 6 is to permute V cache when `-fa off`, where v_trans=true ov::Output res; - auto src = context.get_input(0); - auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}); - + ov::Output src; + if (op_case == 3 || op_case == 4 || op_case == 5 || op_case == 6) { + src = context.get_input(0); + } else { + src = process_view_input_new(context, 0); + } + std::vector perm_values{0, 2, 1, 3}; + const int32_t * op_params = context.get_output_op_params(); + if (op_params != nullptr) { + for (size_t input_axis = 0; input_axis < perm_values.size(); ++input_axis) { + const size_t output_axis = static_cast(op_params[input_axis]); + perm_values[perm_values.size() - 1 - output_axis] = + static_cast(perm_values.size() - 1 - input_axis); + } + } if (op_case == 1 || context.is_stateful()) { + // The stateful path carries hidden-state tensors in a rank-3 layout (the + // leading batch dim is dropped, e.g. Gemma4's per-layer-embedding path). The + // perm above is rank-4; when the actual input is rank-3, drop the batch axis + // (perm[0], which is always the identity 0 here) and shift the rest down by 1 + // so the transpose order matches the input rank. + std::vector perm_used = perm_values; + const auto & src_ps = src.get_partial_shape(); + if (src_ps.rank().is_static() && src_ps.rank().get_length() == 3 && perm_values.size() == 4 && + perm_values[0] == 0) { + perm_used = {perm_values[1] - 1, perm_values[2] - 1, perm_values[3] - 1}; + } + auto perm = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{perm_used.size()}, perm_used); res = std::make_shared(src, perm); - } else if (op_case == 4) { + } else if (op_case == 2) { + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); auto output_shape = context.get_output_shape().to_shape(); auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]}); auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); @@ -50,17 +79,22 @@ OutputVector translate_permute(const NodeContext & context) { auto reshaped = std::make_shared(src, new_shape, true); res = std::make_shared(reshaped, perm); } else { + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); auto cache_shape = src.get_partial_shape(); auto output_shape = context.get_output_shape().to_shape(); int64_t head_size = output_shape[3]; int64_t n_heads = output_shape[1]; + if (op_case == 5 || op_case == 6) { + head_size = output_shape[2]; + n_heads = output_shape[1]; + } int64_t ctx_per_seq = cache_shape[2].is_static() ? cache_shape[2].get_length() : -1; int64_t n_seq = cache_shape[1].get_length(); Output attention_size; if (!context.has_input("attention_size")) { attention_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[2]}); - } else if (op_case == 2) { + } else if (op_case == 3 || op_case == 5) { attention_size = context.get_input("attention_size"); } else { attention_size = context.get_input("attention_size_swa"); @@ -80,18 +114,41 @@ OutputVector translate_permute(const NodeContext & context) { seq_active_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {seq_active_end_val}); } - // 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size] + // 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size] (for `-fa off` [n_seq, n_heads, head_size, ctx_per_seq]) // 2. slice out the active sequences // 3. slice out the attention part in each sequence - // 4. permute + // 4. permute (skip for `-fa off`) auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto src_reshaped = std::make_shared( - src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), false); - auto slice1 = std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); - auto slice2 = std::make_shared(slice1, zero, attention_size, one, one); - res = std::make_shared(slice2, perm); + if (op_case == 3 || op_case == 4) { + auto src_reshaped = std::make_shared( + src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), + false); + ov::Output after_seq_slice; + if (n_seq == 1) { + after_seq_slice = src_reshaped; + } else { + after_seq_slice = + std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); + } + auto slice2 = std::make_shared(after_seq_slice, zero, attention_size, one, one); + res = std::make_shared(slice2, perm); + } else { + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto src_reshaped = std::make_shared( + src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}), + false); + ov::Output after_seq_slice; + if (n_seq == 1) { + after_seq_slice = src_reshaped; + } else { + after_seq_slice = + std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); + } + auto slice2 = std::make_shared(after_seq_slice, zero, attention_size, one, three); + res = slice2; + } } return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp b/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp new file mode 100644 index 0000000..fb63331 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp @@ -0,0 +1,53 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_pool_2d(const NodeContext & context) { + num_inputs_check(context, 1, 1); + const int32_t * params = context.get_output_op_params(); + + const int k0 = params[1]; + const int k1 = params[2]; + const int s0 = params[3]; + const int s1 = params[4]; + const int p0 = params[5]; + const int p1 = params[6]; + + const int op_case = context.get_op_case(); + ov::Output input = context.get_input(0); + ov::Strides strides{static_cast(s1), static_cast(s0)}; + ov::Shape pads_begin{static_cast(p1), static_cast(p0)}; + ov::Shape pads_end{static_cast(p1), static_cast(p0)}; + ov::Shape kernel{static_cast(k1), static_cast(k0)}; + ov::Output res; + + switch (op_case) { + case 1: // GGML_OP_POOL_MAX + { + res = std::make_shared(input, strides, pads_begin, pads_end, kernel); + break; + } + case 2: // GGML_OP_POOL_AVG + { + res = std::make_shared(input, strides, pads_begin, pads_end, kernel, false); + break; + } + default: + break; + } + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/repeat.cpp b/ggml/src/ggml-openvino/openvino/op/repeat.cpp new file mode 100644 index 0000000..b7aeaa2 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/repeat.cpp @@ -0,0 +1,47 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML_OP_REPEAT tiles src[0] to fill the destination shape. Every destination +// dimension is an integer multiple of the corresponding source dimension. +OutputVector translate_repeat(const NodeContext & context) { + num_inputs_check(context, 1, 2); + + auto input = process_view_input_new(context, 0); + + const auto input_shape = context.get_input_shape(0).to_shape(); + const auto output_shape = context.get_output_shape().to_shape(); + + std::vector repeats(4, 1); + for (size_t axis = 0; axis < 4; ++axis) { + const int64_t input_dim = input_shape[axis]; + const int64_t output_dim = output_shape[axis]; + + FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && output_dim > 0 && output_dim % input_dim == 0, + "REPEAT input shape ", input_shape, " cannot tile to match ", output_shape); + + repeats[axis] = output_dim / input_dim; + } + + auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats); + ov::Output res = std::make_shared(input, repeats_node); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/reshape.cpp b/ggml/src/ggml-openvino/openvino/op/reshape.cpp index efd9a5a..5ed27b1 100644 --- a/ggml/src/ggml-openvino/openvino/op/reshape.cpp +++ b/ggml/src/ggml-openvino/openvino/op/reshape.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include namespace ov { @@ -20,58 +19,62 @@ namespace op { OutputVector translate_reshape(const NodeContext & context) { num_inputs_check(context, 1, 1); - if (context.get_input_shape(0) == context.get_output_shape()) { + if (context.get_input(0).get_partial_shape().is_static() && + context.get_input_shape(0) == context.get_output_shape()) { return {context.get_input(0)}; } - int op_case = context.get_op_case(); - FRONT_END_CHECK_IMPLEMENTED( - op_case == 1 || op_case == 2 || op_case == 3 || op_case == 4 || op_case == 5 || op_case == 6, - "Unsupported RESHAPE case"); - - auto output_shape = context.get_output_shape().to_shape(); + const int op_case = context.get_op_case(); + const auto output_shape = context.get_output_shape().to_shape(); + std::vector shape(output_shape.begin(), output_shape.end()); std::shared_ptr new_shape_node; - if (op_case == 1) { - if (context.is_stateful()) { - new_shape_node = ov::op::v0::Constant::create( - ov::element::i64, {3}, - std::vector{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - } else { - new_shape_node = ov::op::v0::Constant::create( - ov::element::i64, {4}, - std::vector{(int64_t) output_shape[0], -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); + switch (op_case) { + case 0: + break; + case 1: + case 9: + shape[1] = -1; + if (context.is_stateful() && op_case == 1) { + shape.erase(shape.begin()); } - } else if (op_case == 2) { - new_shape_node = ov::op::v0::Constant::create( - ov::element::i64, {4}, - std::vector{(int64_t) output_shape[0], (int64_t) output_shape[1], -1, (int64_t) output_shape[3]}); - - } else if (op_case == 3) { - throw std::runtime_error("might be outdated RESHAPE case"); - new_shape_node = ov::op::v0::Constant::create( - ov::element::i64, {4}, std::vector{(int64_t) output_shape[0], (int64_t) output_shape[1], -1, 1}); - - } else if (op_case == 4) { + break; + case 2: + case 3: + shape[2] = -1; + if (op_case == 3) { + shape[3] = 1; + } + break; + case 4: return {context.get_input(0).get_node_shared_ptr()->input_value(0)}; - - } else if (op_case == 5) { - if (context.is_stateful()) { - std::vector shape_vec = {1, -1, (int64_t) context.get_output_shape().to_shape()[3]}; - new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {3}, shape_vec); + case 5: + case 7: + shape = {1, 1, -1, shape[3]}; + if (context.is_stateful() && op_case == 5) { + shape.erase(shape.begin()); + } + break; + case 6: + // Recurrent inputs keep the active sequence count separate from the token count. + if (context.has_input("s_copy_active_slot_len")) { + auto n_slot_active_len = context.get_input("s_copy_active_slot_len"); + auto emb_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {shape[3]}); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + new_shape_node = + std::make_shared(ov::OutputVector{one, n_slot_active_len, neg_one, emb_size}, 0); } else { - std::vector shape_vec = {1, 1, -1, (int64_t) context.get_output_shape().to_shape()[3]}; - new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {4}, shape_vec); + shape = {1, 1, -1, shape[3]}; } - - // // Alternative - // auto token_len = context.get_input("token_len"); - // auto emb_size = - // ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) context.get_output_shape().to_shape()[3]}); - // auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - // new_shape_node = std::make_shared(ov::OutputVector{one, one, token_len, emb_size}, 0); - - } else if (op_case == 6) { - new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {4}, context.get_output_shape().to_shape()); + break; + case 8: + shape[0] = -1; + break; + default: + FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported RESHAPE case: ", op_case); + } + if (!new_shape_node) { + new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {shape.size()}, shape); } auto res = std::make_shared(context.get_input(0), new_shape_node, false); return rename_outputs_with_suffix({res}, context.get_name()); diff --git a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp index 72cf922..980fa3c 100644 --- a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp @@ -7,8 +7,11 @@ #include #include #include +#include #include #include +#include +#include #include namespace ov { @@ -19,9 +22,49 @@ namespace op { OutputVector translate_rms_norm(const NodeContext & context) { num_inputs_check(context, 1, 1); - auto input_node = context.get_input(0); - auto square = std::make_shared( - input_node, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f})); + auto op_case = context.get_op_case(); + + ov::Output input_node; + if (op_case == 3) { + // Flatten sequence and token dimensions to match the gate layout. + auto input_shape = context.get_input_shape(0).to_shape(); + input_node = std::make_shared( + context.get_input(0), + ov::op::v0::Constant::create( + ov::element::i64, {4}, std::vector{1, -1, (int64_t) input_shape[2], (int64_t) input_shape[3]}), + false); + } else if (op_case == 1) { + input_node = process_view_input_new(context, 0); + } else if (op_case == 2) { + auto ssm_state_size = context.get_ssm_state_size(); + // The GDN op packs [attn | new_state] along the row axis; the state occupies the last + // ssm_state_size * n_seqs rows. Slice it off (scaling by the active sequence count) to keep + // just the attention output. + ov::Output state_end; + if (context.has_input("s_copy_active_slot_len")) { + auto len = context.get_input("s_copy_active_slot_len"); + auto state_rows = std::make_shared( + ov::op::v0::Constant::create(ov::element::i64, {1}, {ssm_state_size}), len); + state_end = std::make_shared(state_rows); + } else { + state_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {-ssm_state_size}); + } + auto gdn_attn_output = std::make_shared( + context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), state_end, + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {2})); + + auto input_shape = context.get_input_shape(0).to_shape(); + input_node = std::make_shared( + gdn_attn_output, + ov::op::v0::Constant::create( + ov::element::i64, {4}, std::vector{1, -1, (int64_t) input_shape[2], (int64_t) input_shape[3]}), + false); + + } else { + input_node = process_view_input_new(context, 0); + } + auto square = std::make_shared(input_node, input_node); auto mean = std::make_shared( square, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); diff --git a/ggml/src/ggml-openvino/openvino/op/roll.cpp b/ggml/src/ggml-openvino/openvino/op/roll.cpp new file mode 100644 index 0000000..e8d1b8e --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/roll.cpp @@ -0,0 +1,36 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_roll(const NodeContext & context) { + num_inputs_check(context, 1, 1); + const int32_t * params = context.get_output_op_params(); + + int64_t s0 = params[0]; + int64_t s1 = params[1]; + int64_t s2 = params[2]; + int64_t s3 = params[3]; + + auto input = context.get_input(0); + + auto shift = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, std::vector{s3, s2, s1, s0}); + auto axes = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, std::vector{0, 1, 2, 3}); + + auto roll = std::make_shared(input, shift, axes); + return rename_outputs_with_suffix({roll}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/rope.cpp b/ggml/src/ggml-openvino/openvino/op/rope.cpp index 26dc2d2..2c7fe62 100644 --- a/ggml/src/ggml-openvino/openvino/op/rope.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rope.cpp @@ -7,15 +7,17 @@ #include #include #include +#include #include #include +#include #include #include -#include #include #include #include #include +#include #include namespace ov { @@ -30,9 +32,20 @@ OutputVector translate_rope(const NodeContext & context) { ov::Output res; - auto data_node = context.get_input(0).get_node_shared_ptr(); + auto data_node = process_view_input_new(context, 0).get_node_shared_ptr(); auto output_shape = context.get_output_shape().to_shape(); int32_t * op_params = context.get_output_op_params(); + const int mode = op_case; + const int64_t head_dim = static_cast(output_shape[3]); + const int64_t configured_n_dims = static_cast(op_params[1]); + const int64_t n_dims = configured_n_dims == 0 ? head_dim : configured_n_dims; + const int64_t n_offs = static_cast(op_params[15]); + + constexpr int TYPE_NORMAL = 0; + constexpr int TYPE_NEOX = 1; + constexpr int TYPE_IMROPE = 2; + constexpr int TYPE_VISION = 3; + constexpr int TYPE_MROPE = 4; Output cos_theta_node; Output sin_theta_node; @@ -40,78 +53,187 @@ OutputVector translate_rope(const NodeContext & context) { cos_theta_node = context.get_input("rope_cos"); sin_theta_node = context.get_input("rope_sin"); } else { - auto inp_pos = context.get_input(1).get_node_shared_ptr(); - std::shared_ptr rope_freqs_weight; + std::string cache_key = "rope_sin_cos"; + for (int i = 0; i < 15; i++) { + cache_key += "_" + std::to_string(op_params[i]); + } if (context.get_input_size() == 3) { - rope_freqs_weight = context.get_input(2).get_node_shared_ptr(); + cache_key += "_ff_" + context.get_input_names()[2]; + } + if (context.has_input(cache_key + "_cos")) { + cos_theta_node = context.get_input(cache_key + "_cos"); + sin_theta_node = context.get_input(cache_key + "_sin"); + } else { + auto inp_pos = context.get_input(1).get_node_shared_ptr(); + std::shared_ptr rope_freqs_weight; + if (context.get_input_size() == 3) { + rope_freqs_weight = context.get_input(2).get_node_shared_ptr(); + } + auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode, false, head_dim); + sin_theta_node = sin_cos.first; + cos_theta_node = sin_cos.second; + context.put_shared(cache_key + "_cos", cos_theta_node); + context.put_shared(cache_key + "_sin", sin_theta_node); } - auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight); - sin_theta_node = sin_cos.first; - cos_theta_node = sin_cos.second; } - if (op_case == 2) { - // The input comes from a VIEW - int slice_len = output_shape[2] * output_shape[3]; - data_node = process_view_input(context, 0, slice_len).get_node_shared_ptr(); - if (context.is_stateful()) { - auto data_shape = ov::op::v0::Constant::create( - ov::element::i64, {3}, std::vector{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared(data_node, data_shape, false); + auto output_type = context.get_output_type(); + if (data_node->get_element_type() != ov::element::f32) { + data_node = std::make_shared(data_node, ov::element::f32); + } + + const int64_t total_rope_dims = (mode == TYPE_VISION) ? (2 * n_dims) : n_dims; + FRONT_END_OP_CONVERSION_CHECK(n_offs >= 0 && (n_offs % 2 == 0), + "ROPE expects non-negative even n_offs"); + FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && total_rope_dims + n_offs <= head_dim && (n_dims % 2 == 0), + "ROPE expects even n_dims with total_rope_dims + n_offs <= head_dim"); + + // RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin + // tables are already built rank-4 ([1, S, 1, head_size/2]) for both modes. In + // stateful mode the data arrives rank-3 ([S, n_heads, head_size]), so lift it + // to rank-4 ([1, S, n_heads, head_size]) here. Stateful RoPE already produced + // rank-4 output, so downstream attention is unaffected. + if (context.is_stateful()) { + const int64_t batch = static_cast(output_shape[0]); + auto r4_shape = ov::op::v0::Constant::create( + ov::element::i64, {4}, + std::vector{batch, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); + data_node = std::make_shared(data_node, r4_shape, false); + } + // For TYPE_NORMAL rope (both stateful and stateless) we emit the Flux-style + // interleaved pattern below so the GPU plugin's RoPEFusionFlux matcher folds it + // into ov::op::internal::RoPE. + if (mode == TYPE_NORMAL) { + auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + + const int64_t batch = static_cast(output_shape[0]); + const int64_t n_heads = static_cast(output_shape[2]); + const int64_t half = n_dims / 2; + auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); + auto rot_data = std::make_shared(data_node, rot_start, rot_end, step_one, axis_last); + + auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); + + auto paired_shape = ov::op::v0::Constant::create( + ov::element::i64, {5}, std::vector{batch, -1, n_heads, half, 2}); + auto x_paired = std::make_shared(rot_data, paired_shape, false); + + auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}); + auto data_split = std::make_shared(x_paired, split_axis, 2); + Output x0 = data_split->outputs()[0]; + Output x1 = data_split->outputs()[1]; + + auto x1_neg = std::make_shared(x1, neg_one_f); + auto x_rotated_paired = std::make_shared(ov::OutputVector{x1_neg, x0}, -1); + + auto flat_shape = + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{batch, -1, n_heads, n_dims}); + auto x_rotated = + std::make_shared(x_rotated_paired, flat_shape, false); + + // Expand cos/sin from [..., n_dims/2] to [..., n_dims] by repeating each + // entry twice. Use special_zero on the final Reshape so the seq dim passes + // through dynamically. Final rank is 4 to satisfy the matcher's predicate. + auto expand_cos_sin = [&](const Output& cs) { + auto cs_unsq = std::make_shared( + cs, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1})); + auto bcast_target = ov::op::v0::Constant::create( + ov::element::i64, {5}, std::vector{1, 1, 1, half, 2}); + auto bcast = std::make_shared( + cs_unsq, bcast_target, ov::op::BroadcastType::BIDIRECTIONAL); + auto flat = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, 0, 0, n_dims}); + return std::make_shared(bcast, flat, true); + }; + Output cos_full = expand_cos_sin(cos_theta_node); + Output sin_full = expand_cos_sin(sin_theta_node); + + auto y1 = std::make_shared(rot_data, cos_full); + auto y2 = std::make_shared(x_rotated, sin_full); + auto rotated = std::make_shared(y1, y2); + + ov::OutputVector concat_parts; + if (n_offs > 0) { + auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto head = std::make_shared(data_node, head_start, head_end, step_one, axis_last); + concat_parts.push_back(head); + } + concat_parts.push_back(rotated); + if (n_offs + n_dims < head_dim) { + auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); + auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); + auto tail = std::make_shared(data_node, tail_start, tail_end, step_one, axis_last); + concat_parts.push_back(tail); + } + if (concat_parts.size() == 1) { + res = rotated; + } else { + res = std::make_shared(concat_parts, -1); + } + } else if (mode == TYPE_NEOX || mode == TYPE_IMROPE || mode == TYPE_MROPE || mode == TYPE_VISION) { + const int64_t half = (mode == TYPE_VISION) ? n_dims : (n_dims / 2); + const int64_t rot_dims = 2 * half; + + if (mode != TYPE_NEOX) { + auto cos_sin_shape = std::make_shared(ov::element::i64, ov::Shape{4}, + std::vector{1, -1, 1, half}); + cos_theta_node = std::make_shared(cos_theta_node, cos_sin_shape, true); + sin_theta_node = std::make_shared(sin_theta_node, cos_sin_shape, true); + } + + auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + + Output rot_data = data_node; + if (n_offs > 0 || n_offs + rot_dims < head_dim) { + auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + rot_dims}); + rot_data = std::make_shared(data_node, rot_start, rot_end, step_one, axis_last); + } + + auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); + + auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {3}); + auto split_lengths = ov::op::v0::Constant::create(ov::element::i64, {2}, {half, half}); + auto data_split = std::make_shared(rot_data, split_axis, split_lengths); + Output x1 = data_split->outputs()[0]; + Output x2 = data_split->outputs()[1]; + + auto x2_neg = std::make_shared(x2, neg_one_f); + auto x_rotate_half = std::make_shared(ov::OutputVector{x2_neg, x1}, -1); + + auto cos_full = std::make_shared(ov::OutputVector{cos_theta_node, cos_theta_node}, -1); + auto sin_full = std::make_shared(ov::OutputVector{sin_theta_node, sin_theta_node}, -1); + + auto y1 = std::make_shared(rot_data, cos_full); + auto y2 = std::make_shared(x_rotate_half, sin_full); + auto rotated = std::make_shared(y1, y2); + + ov::OutputVector concat_parts; + if (n_offs > 0) { + auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto head = std::make_shared(data_node, head_start, head_end, step_one, axis_last); + concat_parts.push_back(head); + } + concat_parts.push_back(rotated); + if (n_offs + rot_dims < head_dim) { + auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + rot_dims}); + auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); + auto tail = std::make_shared(data_node, tail_start, tail_end, step_one, axis_last); + concat_parts.push_back(tail); + } + if (concat_parts.size() == 1) { + res = rotated; } else { - auto data_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared(data_node, data_shape, false); + res = std::make_shared(concat_parts, -1); } } - const int mode = op_params[2]; - constexpr int ROPE_TYPE_NORMAL = 0; - constexpr int ROPE_TYPE_NEOX = 2; - - if (mode == ROPE_TYPE_NORMAL) { - auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); - Output even_slice; - Output odd_slice; - int32_t unsqueeze_dim = context.is_stateful() ? 3 : 4; - even_slice = std::make_shared(data_node, zero, end, two, neg_one); - odd_slice = std::make_shared(data_node, one, end, two, neg_one); - - Output first_half = - std::make_shared(std::make_shared(even_slice, cos_theta_node), - std::make_shared(odd_slice, sin_theta_node)); - Output second_half = - std::make_shared(std::make_shared(even_slice, sin_theta_node), - std::make_shared(odd_slice, cos_theta_node)); - - first_half = std::make_shared(first_half, - ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); - second_half = std::make_shared(second_half, - ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); - auto stack = std::make_shared(OutputVector{first_half, second_half}, unsqueeze_dim); - - auto data_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - res = std::make_shared(stack, data_shape, false); - } else if (mode == ROPE_TYPE_NEOX) { - auto data_split = std::make_shared( - data_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}), 2); - Output slice_data_node_0 = data_split->outputs()[0]; - Output slice_data_node_1 = data_split->outputs()[1]; - - auto first_half_node = std::make_shared( - std::make_shared(slice_data_node_0, cos_theta_node), - std::make_shared(slice_data_node_1, sin_theta_node)); - - auto second_half_node = std::make_shared( - std::make_shared(slice_data_node_0, sin_theta_node), - std::make_shared(slice_data_node_1, cos_theta_node)); - - res = std::make_shared(ov::OutputVector{first_half_node, second_half_node}, -1); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); } return rename_outputs_with_suffix({res}, context.get_name()); diff --git a/ggml/src/ggml-openvino/openvino/op/scale.cpp b/ggml/src/ggml-openvino/openvino/op/scale.cpp index 0f3d800..2947ee0 100644 --- a/ggml/src/ggml-openvino/openvino/op/scale.cpp +++ b/ggml/src/ggml-openvino/openvino/op/scale.cpp @@ -2,9 +2,24 @@ #include "../op_table.h" #include "../utils.h" +#include #include +#include #include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include #include namespace ov { @@ -21,6 +36,40 @@ OutputVector translate_scale(const NodeContext & context) { memcpy(&bias, (float *) context.get_output_op_params() + 1, sizeof(float)); auto scale_node = std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale}); + + if (context.get_op_case() == 2) { + return {context.get_input(0)}; + } + + if (context.get_op_case() == 1 && context.has_input("cache_rs_reset_len")) { + auto cache_rs_reset_idx = context.get_input("cache_rs_reset_idx"); + auto cache_rs_reset_len = context.get_input("cache_rs_reset_len"); + + auto cache_rs = context.get_input(0); + + auto cache_shape = std::make_shared(cache_rs, ov::element::i64); + auto n_slots_1d = std::make_shared( + cache_shape, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {2}), + ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0})); + auto n_slots = std::make_shared(n_slots_1d); + + auto iota = std::make_shared( + ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0}), n_slots, + ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}), ov::element::i64); + + auto idx_plus_len = std::make_shared(cache_rs_reset_idx, cache_rs_reset_len); + auto less_than_idx = std::make_shared(iota, cache_rs_reset_idx); + auto greater_equal_idx_plus_len = std::make_shared(iota, idx_plus_len); + auto keep_mask = std::make_shared(less_than_idx, greater_equal_idx_plus_len); + + auto keep_mask_f32 = std::make_shared(keep_mask, ov::element::f32); + auto keep_mask_reshape = std::make_shared( + keep_mask_f32, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {1})); + + auto cleared_cache_rs = std::make_shared(cache_rs, keep_mask_reshape); + return rename_outputs_with_suffix({cleared_cache_rs}, context.get_name()); + } + auto scaled = std::make_shared(context.get_input(0), scale_node); std::shared_ptr res; diff --git a/ggml/src/ggml-openvino/openvino/op/set.cpp b/ggml/src/ggml-openvino/openvino/op/set.cpp new file mode 100644 index 0000000..9b18ccf --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/set.cpp @@ -0,0 +1,76 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML SET writes src1 into a view of src0 and returns the updated tensor. +OutputVector translate_set(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + auto dst = process_view_input_new(context, 0); + auto src = process_view_input_new(context, 1); + + src = std::make_shared(src, context.get_output_type()); + + const auto dst_stride = context.get_input_stride(0); + FRONT_END_OP_CONVERSION_CHECK(dst_stride.size() >= 4, "SET requires 4D destination strides"); + + const auto * op_params = reinterpret_cast(context.get_output_op_params()); + const size_t offset = static_cast(op_params[3]); + + const size_t elem_size = dst_stride.back(); + FRONT_END_OP_CONVERSION_CHECK(elem_size != 0 && offset % elem_size == 0, + "SET offset must be aligned to destination element size"); + + const int64_t offset_elems = static_cast(offset / elem_size); + + auto dst_flat = std::make_shared( + dst, + ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}), + false); + + auto src_flat = std::make_shared( + src, + ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}), + false); + + auto src_shape = std::make_shared(src_flat, ov::element::i64); + auto src_len = std::make_shared( + src_shape, + ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), + false); + + auto start = ov::op::v0::Constant::create(ov::element::i64, {}, {offset_elems}); + auto stop = std::make_shared(start, src_len); + auto step = ov::op::v0::Constant::create(ov::element::i64, {}, {1}); + + auto indices = std::make_shared(start, stop, step, ov::element::i64); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {}, {0}); + + auto updated_flat = std::make_shared(dst_flat, indices, src_flat, axis); + + auto dst_shape = std::make_shared(dst, ov::element::i64); + auto res = std::make_shared(updated_flat, dst_shape, false); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 136e426..3b606c8 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -7,12 +7,15 @@ #include #include #include +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -28,21 +31,18 @@ namespace op { OutputVector translate_set_rows(const NodeContext & context) { num_inputs_check(context, 3, 3); - auto data = context.get_input(0); - auto indices = context.get_input(1); - auto dst = context.get_input(2); + auto data = process_view_input_new(context, 0); + auto indices = process_view_input_new(context, 1); + auto dst = process_view_input_new(context, 2); data = std::make_shared(data, context.get_output_type()); - auto dst_shape = context.get_output_shape().to_shape(); + const auto indices_shape = context.get_input_shape(1); + const bool multidim_indices = indices_shape.rank().is_static() && + indices_shape.rank().get_length() == 4 && + ((indices_shape[1].is_static() && indices_shape[1].get_length() > 1) || + (indices_shape[2].is_static() && indices_shape[2].get_length() > 1)); - auto ind_squeezed = - std::make_shared(indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); - auto data_reshaped = std::make_shared( - data, - ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) dst_shape[3]}), - false); auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2}); Output res; @@ -53,11 +53,31 @@ OutputVector translate_set_rows(const NodeContext & context) { data = std::make_shared( data, ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, dim2, dim3}), false); res = std::make_shared(OutputVector{dst, data}, concat_axis); + } else if (multidim_indices) { + auto updates_shape = std::make_shared(data, ov::element::i64); + + auto indices_rank3 = std::make_shared( + indices, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto indices_rank4_shape = std::make_shared(OutputVector{get_dimensions(updates_shape, {0, 1, 2}), one}, 0); + auto indices_rank4 = std::make_shared(indices_rank3, indices_rank4_shape, false); + auto broadcasted_indices = std::make_shared(indices_rank4, updates_shape); + + res = std::make_shared(dst, broadcasted_indices, data, axes); } else { + auto row_size = context.get_input_shape(2)[3].get_length(); + auto ind_squeezed = std::make_shared( + indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); + auto data_reshaped = std::make_shared( + data, + ov::op::v0::Constant::create(ov::element::i64, {4}, + {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), + false); res = std::make_shared(dst, ind_squeezed, data_reshaped, axes); } - if (auto dst_reshape = std::dynamic_pointer_cast(dst.get_node_shared_ptr())) { + auto dst_reshape = ov::as_type_ptr(dst.get_node_shared_ptr()); + if (!multidim_indices && dst_reshape) { // Fix the case of multiple sequences, reshape back to original shape [1, n_seq, ctx_per_seq, emb] // ctx_per_seq is not fixed due to llama-bench compatibility auto dst_shape_partial = dst_reshape->get_input_partial_shape(0); diff --git a/ggml/src/ggml-openvino/openvino/op/softmax.cpp b/ggml/src/ggml-openvino/openvino/op/softmax.cpp index 9f63308..b391d3f 100644 --- a/ggml/src/ggml-openvino/openvino/op/softmax.cpp +++ b/ggml/src/ggml-openvino/openvino/op/softmax.cpp @@ -2,17 +2,19 @@ #include "../op_table.h" #include "../utils.h" -#include +#include #include +#include #include -#include -#include +#include +#include #include #include #include #include -#include #include +#include +#include #include #include #include @@ -22,63 +24,138 @@ namespace frontend { namespace ggml { namespace op { -OutputVector translate_soft_max(const NodeContext & context) { - // TODO code is outdated - num_inputs_check(context, 1, 2); +static bool is_static_one(const ov::Dimension & dim) { + return dim.is_static() && dim.get_length() == 1; +} + +static bool same_static_dim(const ov::Dimension & lhs, const ov::Dimension & rhs) { + return lhs.is_static() && rhs.is_static() && lhs.get_length() == rhs.get_length(); +} + +static bool is_attention_sinks_input_shape(const ov::PartialShape & candidate, const ov::PartialShape & logits_shape) { + if (candidate.rank().is_dynamic() || logits_shape.rank().is_dynamic() || candidate.rank().get_length() != 4 || + logits_shape.rank().get_length() != 4) { + return false; + } + + return is_static_one(candidate[0]) && is_static_one(candidate[1]) && is_static_one(candidate[2]) && + same_static_dim(candidate[3], logits_shape[1]); +} - auto input_node = context.get_input(0).get_node_shared_ptr(); - ov::Output res; +// Reimplementation of GGML_OP_SOFT_MAX semantics for OpenVINO backend: +// 1) logits = src0 * scale +// 2) logits += mask (if provided) +// 3) append attention sinks as hidden logits (if provided) +// 4) softmax over the last dimension and remove the hidden sink column +OutputVector translate_soft_max(const NodeContext & context) { + num_inputs_check(context, 1, 3); float scale = 1.0f; float max_bias = 0.0f; - auto * op_params = context.get_output_op_params(); - memcpy(&scale, (float *) op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) op_params + 1, sizeof(float)); - auto src0_shape = context.get_input_shape(0).get_shape(); - const uint32_t h = src0_shape[2]; - const uint32_t n_head = src0_shape[0]; - const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); - - const float m0 = powf(2.0f, -(max_bias) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - const float slope = - (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2 * (h - n_head_log2) + 1) : 1.0f; - - auto scale_node = std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale}); - auto scaled_input = std::make_shared(input_node, scale_node); - - if (context.get_input_size() < 2) { - res = std::make_shared(scaled_input, 2); - return rename_outputs_with_suffix({res}, context.get_name()); + memcpy(&scale, (float *) context.get_output_op_params() + 0, sizeof(float)); + memcpy(&max_bias, (float *) context.get_output_op_params() + 1, sizeof(float)); + + ov::Output logits = context.get_input(0); + const bool second_input_is_sinks = + context.get_input_size() == 2 && is_attention_sinks_input_shape(context.get_input_shape(1), context.get_output_shape()); + const bool has_mask = context.get_input_size() > 1 && !second_input_is_sinks; + const bool has_sinks = second_input_is_sinks || context.get_input_size() > 2; + const size_t sinks_input_idx = second_input_is_sinks ? 1 : 2; + + // Apply scale first: logits = src0 * scale + if (scale != 1.0f) { + auto scale_const = + std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale}); + logits = std::make_shared(logits, scale_const); } - ov::Output mask_node_sliced; - if (context.has_input("KQ_mask_sliced")) { - mask_node_sliced = context.get_input("KQ_mask_sliced"); - } else { - auto token_len = get_dimensions(input_node, {1}); - auto mask_node = context.get_input(1); + FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && !has_mask), + "OpenVINO softmax ALiBi path requires mask input"); + + // Optional mask add: logits += mask + // For max_bias > 0 (ALiBi), apply per-head slope to mask before adding. + if (has_mask) { + ov::Output mask = context.get_input(1); + + // For stateful + std::string mask_name = "KQ_mask_sliced"; + if (context.get_input_names()[1].find("swa") != std::string::npos) { + mask_name = "KQ_mask_swa_sliced"; + } + if (context.has_input(mask_name)) { + mask = context.get_input(mask_name); + } + + if (mask.get_element_type() != logits.get_element_type()) { + mask = std::make_shared(mask, logits.get_element_type()); + } + + if (max_bias > 0.0f) { + auto out_shape = context.get_output_shape().to_shape(); + FRONT_END_CHECK_IMPLEMENTED(out_shape.size() == 4, "OpenVINO softmax ALiBi path expects rank-4 tensor"); + + const uint32_t n_head = static_cast(out_shape[1]); + FRONT_END_CHECK_IMPLEMENTED(n_head > 0, "OpenVINO softmax ALiBi path expects n_head > 0"); + + const uint32_t n_head_log2 = 1u << static_cast(std::floor(std::log2(static_cast(n_head)))); + const float m0 = std::pow(2.0f, -(max_bias) / static_cast(n_head_log2)); + const float m1 = std::pow(2.0f, -(max_bias / 2.0f) / static_cast(n_head_log2)); + + std::vector slopes(n_head); + for (uint32_t h = 0; h < n_head; ++h) { + slopes[h] = h < n_head_log2 ? std::pow(m0, static_cast(h + 1)) : + std::pow(m1, static_cast(2 * (h - n_head_log2) + 1)); + } + + ov::Output slope_node = + std::make_shared(ov::element::f32, ov::Shape{n_head}, slopes); + if (slope_node.get_element_type() != mask.get_element_type()) { + slope_node = std::make_shared(slope_node, mask.get_element_type()); + } + + auto slope_shape = std::make_shared( + ov::element::i64, ov::Shape{4}, std::vector{1, static_cast(n_head), 1, 1}); + auto slope_4d = std::make_shared(slope_node, slope_shape, false); + mask = std::make_shared(mask, slope_4d); + } + + logits = std::make_shared(logits, mask); + } + + ov::Output softmax_input = logits; + if (has_sinks) { + ov::Output sinks = context.get_input(sinks_input_idx); + if (sinks.get_element_type() != logits.get_element_type()) { + sinks = std::make_shared(sinks, logits.get_element_type()); + } + + auto sink_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, {1, -1, 1, 1}); + auto sinks_4d = std::make_shared(sinks, sink_shape, false); + + auto logits_shape = std::make_shared(logits, ov::element::i64); auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - mask_node_sliced = std::make_shared(mask_node, zero, token_len, one, one); - } + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto four = ov::op::v0::Constant::create(ov::element::i64, {1}, {4}); + auto shape_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - if (mask_node_sliced.get_element_type() != context.get_output_type()) { - mask_node_sliced = std::make_shared(mask_node_sliced, context.get_output_type()); - } + auto sink_prefix_shape = std::make_shared(logits_shape, zero, three, one, shape_axis); + auto sink_last_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto sink_broadcast_shape = std::make_shared( + ov::OutputVector{sink_prefix_shape, sink_last_dim}, 0); + auto sink_column = std::make_shared(sinks_4d, sink_broadcast_shape, + ov::op::BroadcastType::BIDIRECTIONAL); + softmax_input = std::make_shared(ov::OutputVector{logits, sink_column}, 3); - Output slope_mask; - if (slope != 1.0f) { - auto slope_node = - std::make_shared(ov::element::f32, ov::Shape{}, std::vector{slope}); - slope_mask = std::make_shared(mask_node_sliced, slope_node); - throw std::runtime_error("Slope != 1.0f in softmax has not been tested, verify it before use."); - } - slope_mask = mask_node_sliced; + auto softmax_with_sink = std::make_shared(softmax_input, -1); + auto original_last_dim = std::make_shared(logits_shape, three, four, one, shape_axis); + auto res = std::make_shared(softmax_with_sink, zero, original_last_dim, one, three); - auto input_slope_mask_node = std::make_shared(scaled_input, slope_mask); + return rename_outputs_with_suffix({res}, context.get_name()); + } - res = std::make_shared(input_slope_mask_node, 2); + // Softmax along last dimension (equivalent to ggml softmax over ne[0]). + auto res = std::make_shared(softmax_input, -1); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp b/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp new file mode 100644 index 0000000..840233f --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp @@ -0,0 +1,108 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML SOLVE_TRI: solve Ax = B for lower-triangular A via forward substitution. +// Currently only lower, right, non-unitriangular variant is implemented. +// +// ggml layout: A [n, n, B1, B2], B [k, n, B1, B2] → X [k, n, B1, B2] +// OV layout: A [B2, B1, n, n], B [B2, B1, n, k] → X [B2, B1, n, k] +// +// Forward substitution row i: +// x[i] = (b[i] - sum_{t(A_shape[2]); + + // Initial X: zeros with shape of B + auto B_shape_node = std::make_shared(B, ov::element::i64); + auto zero_f32 = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); + auto X_init = std::make_shared(zero_f32, B_shape_node); + + // --- Loop body parameters --- + // body_iter: iteration counter injected by the Loop op (i64, shape {1}) + auto body_iter = std::make_shared(ov::element::i64, ov::Shape{1}); + auto body_X = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(4)); + auto body_A = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(4)); + auto body_B_p = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(4)); + + auto c_axis2 = ov::op::v0::Constant::create(ov::element::i64, {1}, {int64_t(2)}); + auto c_axis3 = ov::op::v0::Constant::create(ov::element::i64, {1}, {int64_t(3)}); + auto c_axis2_scalar = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(2)}); + + // b_i = B[..., i, :] [B2, B1, 1, k] + auto b_i = std::make_shared(body_B_p, body_iter, c_axis2); + + // A_row_i = A[..., i, :] [B2, B1, 1, n] + auto A_row_i = std::make_shared(body_A, body_iter, c_axis2); + + // sum_i = A_row_i @ X [B2, B1, 1, k] + // (lower-tri zeros + unfilled-X zeros make this equal to the partial sum) + auto sum_i = std::make_shared(A_row_i, body_X, false, false); + + // diag_i = A[..., i, i] [B2, B1, 1, 1] + auto diag_i = std::make_shared(A_row_i, body_iter, c_axis3); + + // x_i = (b_i - sum_i) / diag_i [B2, B1, 1, k] + auto x_i = std::make_shared( + std::make_shared(b_i, sum_i), diag_i); + + // X_updated: scatter x_i into body_X at row i along axis 2 + auto X_updated = std::make_shared(body_X, body_iter, x_i, c_axis2_scalar); + + auto body_cond = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, {true}); + + auto body = std::make_shared( + ov::OutputVector{body_cond, X_updated}, + ov::ParameterVector{body_iter, body_X, body_A, body_B_p}); + + // --- Assemble Loop --- + auto trip_count = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, std::vector{n}); + auto exec_cond = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, {true}); + + auto loop = std::make_shared(trip_count, exec_cond); + loop->set_function(body); + // iter_counter_body_param_idx=0 (body_iter), exec_condition_body_result_idx=0 (body_cond) + loop->set_special_body_ports(ov::op::v5::Loop::SpecialBodyPorts{0, 0}); + + // Carried state: X feeds back from X_updated each iteration + loop->set_merged_input(body_X, X_init, X_updated); + // Invariant inputs passed through unchanged + loop->set_invariant_input(body_A, A); + loop->set_invariant_input(body_B_p, B); + + // Final output: value of X_updated after the last iteration + auto X_final = loop->get_iter_value(X_updated, -1); + + return rename_outputs_with_suffix({X_final}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/sqr.cpp b/ggml/src/ggml-openvino/openvino/op/sqr.cpp new file mode 100644 index 0000000..be01fdc --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/sqr.cpp @@ -0,0 +1,35 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_sqr(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, input); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_sqrt(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp b/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp new file mode 100644 index 0000000..352fd90 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp @@ -0,0 +1,61 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_ssm_conv(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + auto sx = context.get_input(0); // conv state + input: OV shape [1, n_s, d_inner, ncs] + auto c = context.get_input(1); // conv1d weight: OV shape [1, 1, d_inner, d_conv] + + auto sx_shape = context.get_input_shape(0).to_shape(); // [1, n_s, d_inner, ncs] + auto c_shape = context.get_input_shape(1).to_shape(); // [1, 1, d_inner, d_conv] + + // int64_t n_s = sx_shape[1]; + int64_t d_inner = sx_shape[2]; + // int64_t ncs = sx_shape[3]; // d_conv - 1 + n_t + int64_t d_conv = c_shape[3]; + // int64_t n_t = ncs - d_conv + 1; + + // Reshape sx from [1, n_s, d_inner, ncs] to [n_s, d_inner, ncs] for 1D GroupConvolution + auto sx_reshaped = + std::make_shared(sx, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + + // Reshape c from [1, 1, d_inner, d_conv] to [d_inner, 1, 1, d_conv] + // GroupConvolution filter: [groups, out_channels/groups, in_channels/groups, kernel_size] + auto c_new_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{d_inner, 1, 1, d_conv}); + auto c_reshaped = std::make_shared(c, c_new_shape, false); + + // Depthwise 1D convolution: groups=d_inner, stride=1, no padding, no dilation + // Input: [n_s, d_inner, ncs], Filter: [d_inner, 1, 1, d_conv] + // Output: [n_s, d_inner, n_t] + auto conv = std::make_shared( + sx_reshaped, c_reshaped, ov::Strides{1}, ov::CoordinateDiff{0}, ov::CoordinateDiff{0}, ov::Strides{1}); + + // Transpose from [n_s, d_inner, n_t] to [n_s, n_t, d_inner] + auto perm = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{0, 2, 1}); + auto transposed = std::make_shared(conv, perm); + + // Reshape to output shape [1, n_s, n_t, d_inner] + auto res = + std::make_shared(transposed, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/sum.cpp b/ggml/src/ggml-openvino/openvino/op/sum.cpp new file mode 100644 index 0000000..c480ee4 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/sum.cpp @@ -0,0 +1,30 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_sum(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, {0, 1, 2, 3}); + auto res = std::make_shared(input, axes, true); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp b/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp new file mode 100644 index 0000000..d04e644 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp @@ -0,0 +1,27 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_sum_rows(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared( + input, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/transpose.cpp b/ggml/src/ggml-openvino/openvino/op/transpose.cpp index 8e62e83..0651a41 100644 --- a/ggml/src/ggml-openvino/openvino/op/transpose.cpp +++ b/ggml/src/ggml-openvino/openvino/op/transpose.cpp @@ -12,8 +12,37 @@ namespace op { OutputVector translate_transpose(const NodeContext & context) { num_inputs_check(context, 1, 1); + // Compute permute order from input/output shape and stride information + // so it adapts to different input and output layouts. + auto input_stride = context.get_input_stride(0); + auto output_stride = context.get_output_stride(); + + // Compute permute order by matching output and input stride rankings. + // Build pairs. + std::vector> output_stride_dims; + std::vector> input_stride_dims; + + for (int i = 0; i < 4; ++i) { + output_stride_dims.push_back({output_stride[i], i}); + input_stride_dims.push_back({input_stride[i], i}); + } + + // Sort by stride in descending order. + std::sort(output_stride_dims.rbegin(), output_stride_dims.rend()); + std::sort(input_stride_dims.rbegin(), input_stride_dims.rend()); + + // Build permute order. + std::vector permute_order(4); + for (int i = 0; i < 4; ++i) { + int output_dim = output_stride_dims[i].second; + int input_dim = input_stride_dims[i].second; + permute_order[output_dim] = input_dim; + } + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared( - context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 1, 3, 2})); + input, ov::op::v0::Constant::create(ov::element::i64, {4}, permute_order)); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/tri.cpp b/ggml/src/ggml-openvino/openvino/op/tri.cpp new file mode 100644 index 0000000..9b7774a --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/tri.cpp @@ -0,0 +1,82 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML TRI zeroes out elements outside a triangular region of a square matrix. +// The type param (stored in op_params[0]) maps to ggml_tri_type: +// 0 = UPPER_DIAG : keep where col >= row +// 1 = UPPER : keep where col > row +// 2 = LOWER_DIAG : keep where col <= row +// 3 = LOWER : keep where col < row +// +// In OV layout (ggml [ne0, ne1, ne2, ne3] → OV [ne3, ne2, ne1, ne0]): +// ggml dim 0 (ne0, cols) → OV axis 3 +// ggml dim 1 (ne1, rows) → OV axis 2 +// The matrix is square so ne0 == ne1. +OutputVector translate_tri(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto x = context.get_input(0); // OV shape: [ne3, ne2, ne1, ne0] + + int32_t tri_type = context.get_output_op_params()[0]; + + auto shape = context.get_input_shape(0).to_shape(); + int64_t n = static_cast(shape[3]); // ne0 == ne1 + + // Build index range [0, 1, ..., n-1] + auto start = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(0)}); + auto stop = ov::op::v0::Constant::create(ov::element::i64, {}, {n}); + auto step = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(1)}); + auto range = std::make_shared(start, stop, step, ov::element::i64); + + // col_idx shape [1, 1, 1, n] — broadcasts over batch and row dims + auto col_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, 1, n}); + auto col_idx = std::make_shared(range, col_shape, false); + + // row_idx shape [1, 1, n, 1] — broadcasts over batch and col dims + auto row_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, n, 1}); + auto row_idx = std::make_shared(range, row_shape, false); + + // Build boolean mask: true where element should be kept + std::shared_ptr mask; + switch (tri_type) { + case 0: // UPPER_DIAG: col >= row + mask = std::make_shared(col_idx, row_idx); + break; + case 1: // UPPER: col > row + mask = std::make_shared(col_idx, row_idx); + break; + case 2: // LOWER_DIAG: col <= row + mask = std::make_shared(col_idx, row_idx); + break; + case 3: // LOWER: col < row + mask = std::make_shared(col_idx, row_idx); + break; + default: + throw std::runtime_error("translate_tri: invalid tri_type " + std::to_string(tri_type)); + } + + auto zero = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); + auto res = std::make_shared(mask, x, zero); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/unary.cpp b/ggml/src/ggml-openvino/openvino/op/unary.cpp new file mode 100644 index 0000000..1bb7aa2 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/unary.cpp @@ -0,0 +1,138 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml-openvino/ggml-openvino-extra.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_unary_gelu(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, ov::op::GeluApproximationMode::TANH); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_unary_gelu_erf(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, ov::op::GeluApproximationMode::ERF); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_unary_gelu_quick(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto scale = ov::op::v0::Constant::create(input.get_element_type(), ov::Shape{}, {1.702f}); + auto mul = std::make_shared(input, scale); + auto sig = std::make_shared(mul); + auto res = std::make_shared(input, sig); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_unary_elu(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, 1.0); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_unary_hardsigmoid(const NodeContext & context) { + num_inputs_check(context, 1, 1); + // compute in f32 like the ggml reference: 1/6 is not exact in f16/bf16 (NPU cannot take the f32 path) + auto input = process_view_input_new(context, 0); + const auto type = ggml_openvino_is_npu() ? input.get_element_type() : ov::element::f32; + ov::Output x = input; + if (type != input.get_element_type()) { + x = std::make_shared(input, type); + } + auto alpha = ov::op::v0::Constant::create(type, ov::Shape{}, {1.0f / 6.0f}); + auto beta = ov::op::v0::Constant::create(type, ov::Shape{}, {0.5f}); + ov::Output res = std::make_shared(x, alpha, beta); + if (type != input.get_element_type()) { + res = std::make_shared(res, input.get_element_type()); + } + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_unary_step(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto zero = ov::op::v0::Constant::create(input.get_element_type(), ov::Shape{}, {0.0f}); + auto cond = std::make_shared(input, zero); + auto res = std::make_shared(cond, input.get_element_type()); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_unary_round(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, ov::op::v5::Round::RoundMode::HALF_AWAY_FROM_ZERO); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_unary_expm1(const NodeContext & context) { + num_inputs_check(context, 1, 1); + // compute in f32 like the ggml reference: exp(x) - 1 in f16 loses the small-x digits (NPU cannot take the f32 path) + auto input = process_view_input_new(context, 0); + const auto type = ggml_openvino_is_npu() ? input.get_element_type() : ov::element::f32; + ov::Output x = input; + if (type != input.get_element_type()) { + x = std::make_shared(input, type); + } + auto exp = std::make_shared(x); + auto one = ov::op::v0::Constant::create(type, ov::Shape{}, {1.0f}); + ov::Output res = std::make_shared(exp, one); + if (type != input.get_element_type()) { + res = std::make_shared(res, input.get_element_type()); + } + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_unary_softplus(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + if (ggml_openvino_getenv_int("GGML_OPENVINO_NATIVE_SOFTPLUS") != 0) { + return translate_1to1_match_1_input(context); + } + + auto input = process_view_input_new(context, 0); + const auto element_type = input.get_element_type(); + auto one = ov::op::v0::Constant::create(element_type, ov::Shape{}, {1.0f}); + + auto positive = std::make_shared(input); + auto abs = std::make_shared(input); + auto neg_abs = std::make_shared(abs); + auto exp_neg_abs = std::make_shared(neg_abs); + auto log_term = std::make_shared(std::make_shared(one, exp_neg_abs)); + auto res = std::make_shared(positive, log_term); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp b/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp deleted file mode 100644 index 037e0b9..0000000 --- a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_unary_silu(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = context.get_input(0); - auto sigmoid = std::make_shared(input); - auto res = std::make_shared(input, sigmoid); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/upscale.cpp b/ggml/src/ggml-openvino/openvino/op/upscale.cpp new file mode 100644 index 0000000..03f3a91 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/upscale.cpp @@ -0,0 +1,181 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_upscale(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + using Interpolate = ov::op::v4::Interpolate; + auto input = process_view_input_new(context, 0); + + const auto input_shape = context.get_input_shape(0).to_shape(); + const auto output_shape = context.get_output_shape().to_shape(); + + if (input_shape == output_shape) { + return rename_outputs_with_suffix({input}, context.get_name()); + } + + ov::Output res = input; + + // Resample batch / channel dimensions (ne[3] and ne[2], corresponding to OV axes 0 and 1) + // using nearest-neighbor index mapping: i0_d = floor(i_d * in_d / out_d). + for (size_t axis = 0; axis < 2; ++axis) { + const size_t in_dim = input_shape[axis]; + const size_t out_dim = output_shape[axis]; + if (in_dim != out_dim) { + std::vector indices(out_dim); + for (size_t i = 0; i < out_dim; ++i) { + indices[i] = static_cast((i * in_dim) / out_dim); + } + auto indices_node = ov::op::v0::Constant::create(ov::element::i64, {indices.size()}, indices); + auto axis_node = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {axis}); + res = std::make_shared(res, indices_node, axis_node); + } + } + + // Spatial interpolation for ne[1] and ne[0] (OV axes 2 and 3) + if (input_shape[2] != output_shape[2] || input_shape[3] != output_shape[3]) { + const int32_t * op_params = context.get_output_op_params(); + const int32_t mode_flags = op_params != nullptr ? op_params[0] : 0; + + const int op_case = context.get_op_case(); + const bool align_corners = (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) != 0; + const bool antialias = (mode_flags & GGML_SCALE_FLAG_ANTIALIAS) != 0; + + if (op_case == 2 && antialias) { // GGML_SCALE_MODE_BILINEAR with antialias + const size_t H_in = input_shape[2]; + const size_t W_in = input_shape[3]; + const size_t H_out = output_shape[2]; + const size_t W_out = output_shape[3]; + + const float pixel_offset = 0.5f; + + // Height projection: [H_out, H_in] + std::vector Wy_data(H_out * H_in, 0.0f); + const float sf1 = static_cast(H_out) / H_in; + const float support1 = std::max(1.0f, 1.0f / sf1); + const float invscale1 = 1.0f / support1; + + for (size_t i1 = 0; i1 < H_out; ++i1) { + const float y = (static_cast(i1) + pixel_offset) / sf1; + const auto y_start = static_cast(y - support1 + pixel_offset); + const size_t y_min = y_start > 0 ? static_cast(y_start) : 0; + const auto y_end = static_cast(y + support1 + pixel_offset); + const size_t y_max = y_end > 0 ? std::min(static_cast(y_end), H_in) : 0; + + float total_weight = 0.0f; + for (size_t sy = y_min; sy < y_max; ++sy) { + float diff = std::abs((static_cast(sy) - y + pixel_offset) * invscale1); + float weight = std::max(1.0f - diff, 0.0f); + Wy_data[i1 * H_in + sy] = weight; + total_weight += weight; + } + if (total_weight > 0.0f) { + for (size_t sy = y_min; sy < y_max; ++sy) { + Wy_data[i1 * H_in + sy] /= total_weight; + } + } + } + + // Width projection: [W_in, W_out] + std::vector Wx_data(W_in * W_out, 0.0f); + const float sf0 = static_cast(W_out) / W_in; + const float support0 = std::max(1.0f, 1.0f / sf0); + const float invscale0 = 1.0f / support0; + + for (size_t i0 = 0; i0 < W_out; ++i0) { + const float x = (static_cast(i0) + pixel_offset) / sf0; + const auto x_start = static_cast(x - support0 + pixel_offset); + const size_t x_min = x_start > 0 ? static_cast(x_start) : 0; + const auto x_end = static_cast(x + support0 + pixel_offset); + const size_t x_max = x_end > 0 ? std::min(static_cast(x_end), W_in) : 0; + + float total_weight = 0.0f; + for (size_t sx = x_min; sx < x_max; ++sx) { + float diff = std::abs((static_cast(sx) - x + pixel_offset) * invscale0); + float weight = std::max(1.0f - diff, 0.0f); + Wx_data[sx * W_out + i0] = weight; + total_weight += weight; + } + if (total_weight > 0.0f) { + for (size_t sx = x_min; sx < x_max; ++sx) { + Wx_data[sx * W_out + i0] /= total_weight; + } + } + } + + auto Wy_node = ov::op::v0::Constant::create(ov::element::f32, {H_out, H_in}, Wy_data); + auto Wx_node = ov::op::v0::Constant::create(ov::element::f32, {W_in, W_out}, Wx_data); + + auto res_y = std::make_shared(Wy_node, res); + res = std::make_shared(res_y, Wx_node); + } else { + Interpolate::InterpolateAttrs attrs; + attrs.shape_calculation_mode = Interpolate::ShapeCalcMode::SIZES; + attrs.antialias = antialias; + attrs.pads_begin = {0, 0, 0, 0}; + attrs.pads_end = {0, 0, 0, 0}; + + switch (op_case) { + case 1: // GGML_SCALE_MODE_NEAREST + attrs.mode = Interpolate::InterpolateMode::NEAREST; + attrs.nearest_mode = Interpolate::NearestMode::FLOOR; + attrs.coordinate_transformation_mode = Interpolate::CoordinateTransformMode::ASYMMETRIC; + break; + case 2: // GGML_SCALE_MODE_BILINEAR + attrs.mode = Interpolate::InterpolateMode::LINEAR; + attrs.coordinate_transformation_mode = align_corners + ? Interpolate::CoordinateTransformMode::ALIGN_CORNERS + : Interpolate::CoordinateTransformMode::HALF_PIXEL; + break; + case 3: // GGML_SCALE_MODE_BICUBIC + attrs.mode = Interpolate::InterpolateMode::CUBIC; + attrs.cube_coeff = -0.75; + attrs.coordinate_transformation_mode = align_corners + ? Interpolate::CoordinateTransformMode::ALIGN_CORNERS + : Interpolate::CoordinateTransformMode::HALF_PIXEL; + break; + default: + FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported upscale op_case: ", op_case); + } + + std::vector target_shape_vec = { + static_cast(output_shape[2]), + static_cast(output_shape[3]), + }; + std::vector scales_vec = { + static_cast(output_shape[2]) / static_cast(input_shape[2]), + static_cast(output_shape[3]) / static_cast(input_shape[3]), + }; + + auto target_shape_node = ov::op::v0::Constant::create(ov::element::i64, {2}, target_shape_vec); + auto scales_node = ov::op::v0::Constant::create(ov::element::f32, {2}, scales_vec); + auto axes_node = ov::op::v0::Constant::create(ov::element::i64, {2}, {2, 3}); + + res = std::make_shared( + res, target_shape_node, scales_node, axes_node, attrs); + } + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp index 8528d25..486c2ff 100644 --- a/ggml/src/ggml-openvino/openvino/op/view.cpp +++ b/ggml/src/ggml-openvino/openvino/op/view.cpp @@ -1,6 +1,13 @@ #include "../op_table.h" #include "../utils.h" + +#include +#include +#include #include +#include +#include + namespace ov { namespace frontend { namespace ggml { @@ -9,42 +16,241 @@ namespace op { OutputVector translate_view(const NodeContext & context) { num_inputs_check(context, 1, 1); - if (context.get_op_case() == 2) { - auto dst_shape = context.get_output_shape().to_shape(); - return rename_outputs_with_suffix({process_view_input(context, 0, dst_shape[2] * dst_shape[3])}, - context.get_name()); + if (context.get_op_case() == 1) { + // Static-mode identity pass-through for VIEWs over a GATED_DELTA_NET combined output or + // the conv_input CONCAT; the consuming op (CPY/RMS_NORM) does its own runtime-correct + // slicing on the full tensor (see ggml-decoder.cpp compute_op_case, GGML_OP_VIEW). + return {context.get_input(0)}; + } + + if (!context.is_static()) { + // On the stateless/non-static path VIEW is normally a no-op (consumers re-slice). + // EXCEPTION: the MoE expert aggregation slices each expert plane out of + // ffn_moe_weighted [n_embd, n_expert_used, n_tokens] with ggml_view_2d and then + // sums the planes with a chain of ADDs (llama-graph.cpp). Those ADDs read this + // VIEW node directly from the tensor map and do NOT re-slice, so a no-op here + // makes every plane the full tensor and the expert sum collapses. Materialize the + // single-expert slice here. Gated by name (ffn_moe_weighted...view) so it can't + // affect any other view. + const std::string & vname = context.get_name(); + if (vname.find("ffn_moe_weighted") != std::string::npos) { + auto src_ps = context.get_input_shape(0); + auto dst_ps = context.get_output_shape(); + if (src_ps.rank().is_static() && dst_ps.rank().is_static() && src_ps.rank() == dst_ps.rank() && + src_ps.is_static() && dst_ps.is_static()) { + auto sst = context.get_input_stride(0); + auto dst = context.get_output_stride(); + size_t voff = context.get_output_op_offset(); + auto ss = src_ps.to_shape(); + auto dd = dst_ps.to_shape(); + const size_t nd = ss.size(); + // Stateful models drop the leading size-1 batch dim, so the real OV tensor can + // be one rank lower than the ggml shape metadata above. Axis indices derived + // from that metadata must be shifted down by the difference before they are + // used as OV axes. Bail out if the dims we would drop are not all 1. + const auto in_rank = context.get_input(0).get_partial_shape().rank(); + const int axis_shift = + in_rank.is_static() ? (int) nd - (int) in_rank.get_length() : 0; + bool shift_ok = axis_shift >= 0 && (size_t) axis_shift < nd; + for (int a = 0; a < axis_shift && shift_ok; ++a) { + shift_ok = (ss[a] == 1 && dd[a] == 1); + } + if (shift_ok && sst.size() == nd && dst.size() == nd) { + // Map each dst axis of size>1 to a src axis with equal (size,stride); + // the unmatched src axis of size>1 is the indexed expert axis. + // dst_to_src[d] records which src axis each dst axis came from, so we can + // later pull the dynamic (token) dim from the right source axis at runtime. + std::vector used(nd, false); + std::vector dst_to_src(nd, -1); + bool ok = true; + for (size_t d = 0; d < nd; ++d) { + if (dd[d] == 1) { + continue; + } + int found = -1; + for (size_t s = 0; s < nd; ++s) { + if (!used[s] && ss[s] == dd[d] && sst[s] == dst[d]) { + found = (int) s; + break; + } + } + if (found < 0) { + ok = false; + break; + } + used[found] = true; + dst_to_src[d] = found; + } + int dropped = -1; + if (ok) { + for (size_t s = 0; s < nd; ++s) { + if (!used[s] && ss[s] > 1) { + if (dropped >= 0) { + ok = false; + break; + } + dropped = (int) s; + } + } + } + if (ok && dropped >= 0) { + const size_t dstr = sst[dropped]; + const int64_t dsz = (int64_t) ss[dropped]; + if (dstr > 0 && voff % dstr == 0) { + const int64_t sel = (int64_t) (voff / dstr); + if (sel >= 0 && sel < dsz) { + ov::Output sl = std::make_shared( + context.get_input(0), + ov::op::v0::Constant::create(ov::element::i64, {1}, {sel}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {sel + 1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, + {dropped - axis_shift})); + // Build the reshape target from the (concrete) dst shape, but + // keep the dynamic token axis dynamic instead of freezing it + // to the captured n_tokens. Without this the constant dst + // shape bakes in the prefill token count and the static value + // flows downstream, turning every later decoder layer static + // (the GPU in-place-concat KV-cache bug). The token axis is + // PERMUTED between the sliced input and the dst (e.g. input + // [1,tok,expert,emb] -> dst [1,1,tok,emb]), so special_zero + // (which copies the same-position dim) is not enough: pull the + // dynamic dim from the correct SOURCE axis via ShapeOf+Gather + // and place it at the dst token position. + const int32_t dyn = context.get_op_dynamic_dim(); // output ggml axis, -1 if none + // still in ggml metadata axis space; get_shape() reverses ggml order + int dst_ov_axis = (dyn != -1) ? ((int) nd - 1 - (int) dyn) : -1; + int src_ov_axis = (dst_ov_axis >= axis_shift && dst_ov_axis < (int) nd) + ? dst_to_src[dst_ov_axis] + : -1; + if (dst_ov_axis >= 0 && src_ov_axis >= axis_shift) { + // target = concat of per-axis scalars; the token axis is a + // runtime Gather of the slice's shape, the rest are constants. + auto sl_shape = std::make_shared(sl, ov::element::i64); + auto tok_dim = std::make_shared( + sl_shape, + ov::op::v0::Constant::create(ov::element::i64, {1}, + {src_ov_axis - axis_shift}), + ov::op::v0::Constant::create(ov::element::i64, {}, {0})); + ov::OutputVector parts; + for (int a = axis_shift; a < (int) nd; ++a) { + if (a == dst_ov_axis) { + parts.push_back(tok_dim); + } else { + parts.push_back(ov::op::v0::Constant::create( + ov::element::i64, {1}, {(int64_t) dd[a]})); + } + } + auto dc = std::make_shared(parts, 0); + auto rs = std::make_shared(sl, dc, false); + return rename_outputs_with_suffix({rs}, context.get_name()); + } + std::vector dd_ov(dd.begin() + axis_shift, dd.end()); + auto dc = ov::op::v0::Constant::create(ov::element::i64, {dd_ov.size()}, dd_ov); + auto rs = std::make_shared(sl, dc, false); + return rename_outputs_with_suffix({rs}, context.get_name()); + } + } + } + } + } + } + return {context.get_input(0)}; } - // op_case 3 - if (context.get_op_case() == 3) { - auto input = context.get_input(0); - auto input_ov_shape = input.get_partial_shape(); - auto input_llama_shape = context.get_input_shape(0).to_shape(); + auto input = context.get_input(0); + auto src_shape = context.get_input_shape(0); + auto dst_shape = context.get_output_shape(); + + if (src_shape.rank().is_dynamic() || dst_shape.rank().is_dynamic()) { + return {input}; + } - // if the input ov shape size is different from the input llama shape size, it means the input is already reshaped and we need to reshape it back to the original shape before slicing - if (input_ov_shape.size() != input_llama_shape.size()) { - input = std::make_shared(input, ov::op::v0::Constant::create(ov::element::i64, {input_llama_shape.size()}, input_llama_shape), false); + int64_t src_elems = 1; + int64_t dst_elems = 1; + for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) { + if (src_shape[i].is_dynamic()) { + return {input}; } + src_elems *= src_shape[i].get_length(); + } + for (int64_t i = 0; i < dst_shape.rank().get_length(); ++i) { + if (dst_shape[i].is_dynamic()) { + return {input}; + } + dst_elems *= dst_shape[i].get_length(); + } + + if (dst_elems >= src_elems) { + return {input}; + } - auto dst_shape = context.get_output_shape().to_shape(); + auto src_stride = context.get_input_stride(0); + auto dst_stride = context.get_output_stride(); + size_t view_offset = context.get_output_op_offset(); - // find the index of dst_shape that is different from input shape, and use that index to slice the input - int slice_dim = -1; - for (size_t i = 0; i < dst_shape.size(); ++i) { - if (dst_shape[i] != input_llama_shape[i]) { - slice_dim = i; + bool same_stride = (src_stride.size() == dst_stride.size()); + if (same_stride) { + for (size_t i = 0; i < src_stride.size(); ++i) { + if (src_stride[i] != dst_stride[i]) { + same_stride = false; break; } } + } + + if (!same_stride) { + return {input}; + } - auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {dst_shape[slice_dim]}); - auto stride = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim}); - auto sliced = std::make_shared(input, begin, end, stride, axes); - return {sliced}; + auto src_ov_shape = src_shape.to_shape(); + auto dst_ov_shape = dst_shape.to_shape(); + size_t ndims = src_ov_shape.size(); + if (dst_ov_shape.size() != ndims) { + return {input}; } - return {context.get_input(0)}; + + std::vector diff_dims; + for (size_t i = 0; i < ndims; ++i) { + if (src_ov_shape[i] != dst_ov_shape[i]) { + diff_dims.push_back(static_cast(i)); + } + } + + if (diff_dims.size() != 1) { + return {input}; + } + + int slice_dim = diff_dims[0]; + int64_t dim_size = static_cast(src_ov_shape[slice_dim]); + + size_t ov_stride_for_dim = 1; + for (size_t i = slice_dim + 1; i < ndims; ++i) { + ov_stride_for_dim *= src_ov_shape[i]; + } + size_t elem_size = src_stride.back(); + if (elem_size == 0) { + elem_size = 1; + } + + int64_t begin_val = 0; + if (ov_stride_for_dim > 0 && elem_size > 0) { + begin_val = static_cast((view_offset / elem_size) / ov_stride_for_dim); + } + int64_t end_val = begin_val + static_cast(dst_ov_shape[slice_dim]); + + if (begin_val < 0 || end_val > dim_size) { + return {input}; + } + + auto sliced = + std::make_shared(input, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim})); + + sliced->set_friendly_name(context.get_output_name()); + return {sliced->output(0)}; } } // namespace op diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index beadafe..23b3a2c 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -2,12 +2,28 @@ #include "utils.h" +#include #include +#include +#include #include +#include +#include #include +#include +#include +#include #include #include +#include +#include +#include +#include +#include +#include #include +#include +#include namespace ov { namespace frontend { @@ -16,28 +32,88 @@ namespace ggml { std::unordered_map get_supported_ops() { using namespace ov::op; return { - {"GGML_OP_ADD", op::translate_1to1_match_2_inputs }, - {"GGML_OP_ADD1", op::translate_1to1_match_2_inputs }, - {"GGML_OP_CONT", op::translate_cont }, - {"GGML_OP_DIV", op::translate_1to1_match_2_inputs }, - {"GGML_OP_GET_ROWS", op::translate_get_rows }, - {"GGML_OP_MUL", op::translate_1to1_match_2_inputs}, - {"GGML_OP_MUL_MAT", op::translate_mulmat }, - {"GGML_OP_PERMUTE", op::translate_permute }, - {"GGML_OP_RESHAPE", op::translate_reshape }, - {"GGML_OP_RMS_NORM", op::translate_rms_norm }, - {"GGML_OP_ROPE", op::translate_rope }, - {"GGML_OP_SCALE", op::translate_scale }, - {"GGML_OP_SOFT_MAX", op::translate_soft_max }, - {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, - {"GGML_OP_TRANSPOSE", op::translate_transpose }, - {"GGML_UNARY_OP_SILU", op::translate_unary_silu }, - {"GGML_OP_VIEW", op::translate_view }, - {"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu }, - {"GGML_GLU_OP_GEGLU", op::translate_glu_geglu }, - {"GGML_OP_SET_ROWS", op::translate_set_rows }, - {"GGML_OP_CPY", op::translate_cpy }, - {"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext }, + {"GGML_OP_ADD", op::translate_add }, + {"GGML_OP_ADD1", op::translate_1to1_match_2_inputs }, + {"GGML_OP_ADD_ID", op::translate_add_id }, + {"GGML_OP_CONCAT", op::translate_concat }, + {"GGML_OP_CONT", op::translate_cont }, + {"GGML_OP_DUP", op::translate_cont }, + {"GGML_OP_DIV", op::translate_div }, + {"GGML_OP_FILL", op::translate_fill }, + {"GGML_OP_GET_ROWS", op::translate_get_rows }, + {"GGML_OP_IM2COL", op::translate_im2col }, + {"GGML_OP_IM2COL_3D", op::translate_im2col_3d }, + {"GGML_OP_MUL", op::translate_1to1_match_2_inputs}, + {"GGML_OP_MUL_MAT", op::translate_mulmat }, + {"GGML_OP_MUL_MAT_ID", op::translate_mul_mat_id }, + {"GGML_OP_PERMUTE", op::translate_permute }, + {"GGML_OP_RESHAPE", op::translate_reshape }, + {"GGML_OP_RMS_NORM", op::translate_rms_norm }, + {"GGML_OP_NORM", op::translate_norm }, + {"GGML_OP_L2_NORM", op::translate_l2_norm }, + {"GGML_OP_SUM_ROWS", op::translate_sum_rows }, + {"GGML_OP_ROPE", op::translate_rope }, + {"GGML_OP_SCALE", op::translate_scale }, + {"GGML_OP_SQR", op::translate_sqr }, + {"GGML_OP_SQRT", op::translate_sqrt }, + {"GGML_OP_SOFT_MAX", op::translate_soft_max }, + {"GGML_OP_ARGSORT", op::translate_argsort }, + {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, + {"GGML_OP_TRANSPOSE", op::translate_transpose }, + {"GGML_OP_SIN", op::translate_1to1_match_1_input }, + {"GGML_OP_COS", op::translate_1to1_match_1_input }, + {"GGML_OP_LOG", op::translate_1to1_match_1_input }, + {"GGML_OP_MEAN", op::translate_mean }, + {"GGML_OP_SUM", op::translate_sum }, + {"GGML_UNARY_OP_GELU", op::translate_unary_gelu }, + {"GGML_UNARY_OP_GELU_ERF", op::translate_unary_gelu_erf }, + {"GGML_UNARY_OP_GELU_QUICK", op::translate_unary_gelu_quick }, + {"GGML_UNARY_OP_ELU", op::translate_unary_elu }, + {"GGML_UNARY_OP_HARDSWISH", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_HARDSIGMOID", op::translate_unary_hardsigmoid }, + {"GGML_UNARY_OP_STEP", op::translate_unary_step }, + {"GGML_UNARY_OP_ABS", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_SGN", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_FLOOR", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_CEIL", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_ROUND", op::translate_unary_round }, + {"GGML_UNARY_OP_EXPM1", op::translate_unary_expm1 }, + {"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_SILU", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_SOFTPLUS", op::translate_unary_softplus }, + {"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_EXP", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input }, + {"GGML_OP_VIEW", op::translate_view }, + {"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu }, + {"GGML_GLU_OP_SWIGLU_OAI", op::translate_glu_swiglu_oai }, + {"GGML_GLU_OP_SWIGLU_CLAMP", op::translate_glu_swiglu_clamp }, + {"GGML_GLU_OP_GEGLU", op::translate_glu_geglu }, + {"GGML_GLU_OP_GEGLU_QUICK", op::translate_glu_geglu_quick }, + {"GGML_OP_SET_ROWS", op::translate_set_rows }, + {"GGML_OP_CPY", op::translate_cpy }, + {"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext }, + {"GGML_OP_CLAMP", op::translate_clamp }, + {"GGML_OP_PAD", op::translate_pad }, + {"GGML_OP_SSM_CONV", op::translate_ssm_conv }, + {"GGML_OP_GATED_DELTA_NET", op::translate_gated_delta_net }, + {"GGML_OP_REPEAT", op::translate_repeat }, + {"GGML_OP_CUMSUM", op::translate_cumsum }, + {"GGML_OP_FILL", op::translate_fill }, + {"GGML_OP_DIAG", op::translate_diag }, + {"GGML_OP_TRI", op::translate_tri }, + {"GGML_OP_SET", op::translate_set }, + {"GGML_OP_POOL_2D", op::translate_pool_2d }, + {"GGML_OP_ROLL", op::translate_roll }, + {"GGML_OP_UPSCALE", op::translate_upscale }, + {"GGML_OP_CONV_2D", op::translate_conv_2d }, + {"GGML_OP_CONV_2D_DW", op::translate_conv_2d_dw }, + {"GGML_OP_CONV_TRANSPOSE_1D", op::translate_conv_transpose_1d }, + {"GGML_OP_CONV_TRANSPOSE_2D", op::translate_conv_transpose_2d }, + {"GGML_OP_CONV_3D", op::translate_conv_3d }, + // solve_tri has accuracy issues on GPU + // {"GGML_OP_SOLVE_TRI", op::translate_solve_tri }, }; } diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index 37f7631..6ed7549 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -8,29 +8,73 @@ namespace ggml { namespace op { -#define GGML_OP_CONVERTER(op) OutputVector op(const NodeContext& context) +#define GGML_OP_CONVERTER(op) OutputVector op(const NodeContext & context) GGML_OP_CONVERTER(translate_add); GGML_OP_CONVERTER(translate_cont); +GGML_OP_CONVERTER(translate_concat); +GGML_OP_CONVERTER(translate_add_id); +GGML_OP_CONVERTER(translate_div); +GGML_OP_CONVERTER(translate_fill); GGML_OP_CONVERTER(translate_get_rows); -GGML_OP_CONVERTER(translate_mul); +GGML_OP_CONVERTER(translate_im2col); +GGML_OP_CONVERTER(translate_im2col_3d); GGML_OP_CONVERTER(translate_mulmat); +GGML_OP_CONVERTER(translate_mul_mat_id); GGML_OP_CONVERTER(translate_permute); GGML_OP_CONVERTER(translate_reshape); GGML_OP_CONVERTER(translate_rms_norm); +GGML_OP_CONVERTER(translate_norm); +GGML_OP_CONVERTER(translate_l2_norm); +GGML_OP_CONVERTER(translate_sum_rows); +GGML_OP_CONVERTER(translate_sqr); GGML_OP_CONVERTER(translate_rope); GGML_OP_CONVERTER(translate_scale); -GGML_OP_CONVERTER(translate_unary_silu); +GGML_OP_CONVERTER(translate_sqrt); +GGML_OP_CONVERTER(translate_unary_softplus); GGML_OP_CONVERTER(translate_soft_max); GGML_OP_CONVERTER(translate_transpose); GGML_OP_CONVERTER(translate_view); GGML_OP_CONVERTER(translate_glu_swiglu); +GGML_OP_CONVERTER(translate_glu_swiglu_oai); +GGML_OP_CONVERTER(translate_glu_swiglu_clamp); GGML_OP_CONVERTER(translate_glu_geglu); +GGML_OP_CONVERTER(translate_glu_geglu_quick); GGML_OP_CONVERTER(translate_set_rows); GGML_OP_CONVERTER(translate_cpy); +GGML_OP_CONVERTER(translate_argsort); GGML_OP_CONVERTER(translate_flash_attn_ext); +GGML_OP_CONVERTER(translate_clamp); +GGML_OP_CONVERTER(translate_pad); +GGML_OP_CONVERTER(translate_ssm_conv); +GGML_OP_CONVERTER(translate_gated_delta_net); +GGML_OP_CONVERTER(translate_repeat); +GGML_OP_CONVERTER(translate_cumsum); +GGML_OP_CONVERTER(translate_fill); +GGML_OP_CONVERTER(translate_set); +GGML_OP_CONVERTER(translate_diag); +GGML_OP_CONVERTER(translate_tri); +GGML_OP_CONVERTER(translate_solve_tri); +GGML_OP_CONVERTER(translate_pool_2d); +GGML_OP_CONVERTER(translate_roll); +GGML_OP_CONVERTER(translate_upscale); +GGML_OP_CONVERTER(translate_mean); +GGML_OP_CONVERTER(translate_sum); +GGML_OP_CONVERTER(translate_unary_gelu); +GGML_OP_CONVERTER(translate_unary_gelu_erf); +GGML_OP_CONVERTER(translate_unary_gelu_quick); +GGML_OP_CONVERTER(translate_unary_elu); +GGML_OP_CONVERTER(translate_unary_hardsigmoid); +GGML_OP_CONVERTER(translate_unary_step); +GGML_OP_CONVERTER(translate_unary_round); +GGML_OP_CONVERTER(translate_unary_expm1); +GGML_OP_CONVERTER(translate_conv_2d); +GGML_OP_CONVERTER(translate_conv_2d_dw); +GGML_OP_CONVERTER(translate_conv_transpose_1d); +GGML_OP_CONVERTER(translate_conv_transpose_2d); +GGML_OP_CONVERTER(translate_conv_3d); -} // namespace op +} // namespace op std::unordered_map get_supported_ops(); diff --git a/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.cpp b/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.cpp new file mode 100644 index 0000000..eae5b2c --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.cpp @@ -0,0 +1,96 @@ +#include "align_eltwise_ranks.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +namespace { + +// True for an RMS-norm output, x * (1 / sqrt(mean(x^2) + eps)) as translate_rms_norm builds it, optionally +// scaled by the norm weight. +bool is_rms_norm_output(const ov::Output & value, int depth = 1) { + const auto * node = value.get_node(); + if (!ov::is_type(node)) { + return false; + } + for (const auto & input : node->input_values()) { + const auto * src = input.get_node(); + if (ov::is_type(src) && ov::is_type(src->get_input_node_ptr(1))) { + return true; + } + if (depth > 0 && is_rms_norm_output(input, depth - 1)) { + return true; + } + } + return false; +} + +} // namespace + +AlignEltwiseOperandRanks::AlignEltwiseOperandRanks() { + auto eltwise_m = ov::pass::pattern::wrap_type(); + + const auto callback = [this](ov::pass::pattern::Matcher & m) { + auto node = m.get_match_root(); + if (node->get_input_size() != 2) { + return false; + } + + auto lhs = node->input_value(0); + auto rhs = node->input_value(1); + + // A tensor-vs-Constant mismatch is the norm's own eps / 1-over-sqrt arithmetic. That + // folds into the `rms` primitive itself instead of becoming a fused eltwise post-op, + // so it is not affected and is left alone. + if (ov::is_type(lhs.get_node()) || + ov::is_type(rhs.get_node())) { + return false; + } + + const auto lhs_rank = lhs.get_partial_shape().rank(); + const auto rhs_rank = rhs.get_partial_shape().rank(); + if (lhs_rank.is_dynamic() || rhs_rank.is_dynamic() || lhs_rank == rhs_rank) { + return false; + } + + const size_t shorter_idx = lhs_rank.get_length() < rhs_rank.get_length() ? 0 : 1; + const auto & shorter = shorter_idx == 0 ? lhs : rhs; + + // The defect is with the norm output as the higher-rank operand. When the norm output is the + // lower-rank one (gemma-3 adds it to a rank-4 residual), unsqueezing it puts the Unsqueeze between + // `rms` and its post-op, and the GPU plugin then computes the layer wrongly. + if (is_rms_norm_output(shorter)) { + return false; + } + const int64_t diff = std::abs(lhs_rank.get_length() - rhs_rank.get_length()); + + std::vector axes(static_cast(diff)); + std::iota(axes.begin(), axes.end(), 0); + auto unsqueeze = std::make_shared( + shorter, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ axes.size() }, axes)); + + node->input(shorter_idx).replace_source_output(unsqueeze->output(0)); + register_new_node(unsqueeze); + return true; + }; + + register_matcher( + std::make_shared(eltwise_m, "ov::frontend::ggml::pass::AlignEltwiseOperandRanks"), + callback); +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.h b/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.h new file mode 100644 index 0000000..04f59cc --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/align_eltwise_ranks.h @@ -0,0 +1,39 @@ +#pragma once + +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +// Give a binary eltwise op's two operands the same rank, by unsqueezing leading axes onto the +// shorter one. +// +// This is semantically a no-op -- NUMPY broadcasting already left-pads the lower-rank operand +// with 1s, and the result rank is max(rank_a, rank_b) either way. It exists purely to work +// around an OpenVINO GPU-plugin defect: an eltwise op whose operands differ in rank is computed +// incorrectly once the plugin fuses it as a post-op into an `rms` primitive. gemma-4 dense hits +// this under stateful execution, where the layer tail adds a rank-3 residual to the rank-4 +// RMS-norm output; the model then degenerates into repeated tokens on GPU while CPU is correct. +// Equalising the ranks keeps the fusion and makes it compute the right answer. +// +// Deliberately a graph pass rather than something the op translators do: rank is load-bearing +// during translation (several translators and later passes read operand ranks), and rewriting +// operands mid-translate breaks the attention path. Running after the graph is complete avoids +// that entirely. +// +// Only applies when both operands are real tensors -- a tensor-vs-Constant mismatch is the +// RMS norm's own eps/rsqrt arithmetic, which folds into the `rms` primitive rather than +// becoming a fused post-op, and is not affected by the defect. The norm output itself is never +// unsqueezed: when it is the lower-rank operand (gemma-3), that breaks the fused path instead. +class AlignEltwiseOperandRanks : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::AlignEltwiseOperandRanks") + AlignEltwiseOperandRanks(); +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/eliminate_zp.cpp b/ggml/src/ggml-openvino/openvino/pass/eliminate_zp.cpp deleted file mode 100644 index ed2a3ab..0000000 --- a/ggml/src/ggml-openvino/openvino/pass/eliminate_zp.cpp +++ /dev/null @@ -1,123 +0,0 @@ -#include "eliminate_zp.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -EliminateZeroPoints::EliminateZeroPoints() { - // Find pattern: - // (Multiply Any(scale) - // (Subtract (Convert Constant(data))) - // (Convert Constant(zero_point))) - // where zero_point is a scalar - // If data is u4 and zp value is 8 (q4_0), Replace the Subtract with an i4 Constant whose value is data - zp_val - // If data is u8 and zp value is 128 (q8_0) or 32 (q6_k), Replace the Subtract with an i8 Constant - - auto m_data_constant = ov::pass::pattern::wrap_type(); - auto m_data_convert = ov::pass::pattern::wrap_type({m_data_constant}); - - auto m_zp_constant = ov::pass::pattern::wrap_type(); - auto m_zp_convert = ov::pass::pattern::wrap_type({m_zp_constant}); - - auto m_subtract = ov::pass::pattern::wrap_type({m_data_convert, m_zp_convert}); - auto m_scale = ov::pass::pattern::any_input(); - auto m_multiply = ov::pass::pattern::wrap_type({m_scale, m_subtract}); - - const auto callback = [=](ov::pass::pattern::Matcher & m) { - const auto & pattern_map = m.get_pattern_value_map(); - - auto multiply_node = - std::dynamic_pointer_cast(pattern_map.at(m_multiply).get_node_shared_ptr()); - auto subtract_node = - std::dynamic_pointer_cast(pattern_map.at(m_subtract).get_node_shared_ptr()); - auto data_constant = - std::dynamic_pointer_cast(pattern_map.at(m_data_constant).get_node_shared_ptr()); - auto zp_constant = - std::dynamic_pointer_cast(pattern_map.at(m_zp_constant).get_node_shared_ptr()); - - if (!multiply_node || !subtract_node || !data_constant || !zp_constant) { - return false; - } - - if (ov::shape_size(zp_constant->get_shape()) != 1) { - return false; - } - - auto data_type = data_constant->get_element_type(); - auto zp_data = zp_constant->cast_vector(); - - if (zp_data.empty()) { - return false; - } - - int zp_value = zp_data[0]; - - bool should_eliminate = false; - ov::element::Type target_type; - - if (data_type == ov::element::u4 && zp_value == 8) { - should_eliminate = true; - target_type = ov::element::i4; - } else if (data_type == ov::element::u8 && (zp_value == 128 || zp_value == 32)) { - should_eliminate = true; - target_type = ov::element::i8; - } - - if (!should_eliminate) { - return false; - } - - auto data_shape = data_constant->get_shape(); - size_t total_elements = ov::shape_size(data_shape); - - std::shared_ptr new_constant; - - // TODO improve performance - if (data_type == ov::element::u4) { - auto data_values = data_constant->cast_vector(); - std::vector adjusted_values(total_elements); - - ov::parallel_for(total_elements, [&](size_t i) { - adjusted_values[i] = static_cast(static_cast(data_values[i]) - 8); - }); - - new_constant = std::make_shared(target_type, data_shape, adjusted_values); - } else if (data_type == ov::element::u8) { - auto data_values = data_constant->cast_vector(); - std::vector adjusted_values(total_elements); - - ov::parallel_for(total_elements, [&, zp_value](size_t i) { - adjusted_values[i] = static_cast(static_cast(data_values[i]) - zp_value); - }); - - new_constant = std::make_shared(target_type, data_shape, adjusted_values); - } - - auto new_convert = - std::make_shared(new_constant, subtract_node->get_output_element_type(0)); - ov::replace_node(subtract_node, new_convert); - - return true; - }; - - register_matcher( - std::make_shared(m_multiply, "ov::frontend::ggml::pass::EliminateZeroPoints"), - callback); -} - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.cpp new file mode 100644 index 0000000..9cf5416 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.cpp @@ -0,0 +1,70 @@ +#include "fuse_argsort_topk.h" + +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +FuseArgsortTopK::FuseArgsortTopK() { + // GGML argsort materializes the full ordering before a VIEW keeps the prefix. + // Let TopK produce only that prefix when all index consumers are such views. + auto pattern = ov::pass::pattern::wrap_type(); + + const auto callback = [](ov::pass::pattern::Matcher & m) { + auto topk = ov::as_type_ptr(m.get_match_root()); + if (!topk->output(0).get_target_inputs().empty() || topk->output(1).get_target_inputs().empty() || + topk->get_sort_type() != ov::op::v11::TopK::SortType::SORT_VALUES) { + return false; + } + + const auto shape = topk->get_output_partial_shape(1); + if (shape.rank().is_dynamic()) { + return false; + } + const int64_t rank = shape.rank().get_length(); + const int64_t axis = topk->get_axis(); + if (shape[axis].is_dynamic()) { + return false; + } + + int64_t prefix = -1; + for (const auto & input : topk->output(1).get_target_inputs()) { + const auto * slice = ov::as_type(input.get_node()); + if (!slice || input.get_index() != 0 || slice->get_input_size() != 5) { + return false; + } + int64_t values[4]; + for (size_t i = 0; i < 4; ++i) { + auto value = ov::as_type_ptr(slice->get_input_node_shared_ptr(i + 1)); + if (!value || ov::shape_size(value->get_shape()) != 1) { + return false; + } + values[i] = value->cast_vector()[0]; + } + const int64_t slice_axis = values[3] < 0 ? values[3] + rank : values[3]; + if (values[0] != 0 || values[2] != 1 || slice_axis != axis || values[1] <= 0 || + values[1] >= shape[axis].get_length() || (prefix != -1 && prefix != values[1])) { + return false; + } + prefix = values[1]; + } + + // ggml_argsort_top_k sorts the full row, then exposes only its first k indices. + // Keep other TopK forms unchanged because their ordering can be observable. + topk->set_argument(1, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {prefix})); + topk->validate_and_infer_types(); + return true; + }; + + register_matcher(std::make_shared(pattern, "ov::frontend::ggml::pass::FuseArgsortTopK"), callback); +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.h b/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.h new file mode 100644 index 0000000..c248ebe --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +class FuseArgsortTopK : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseArgsortTopK") + FuseArgsortTopK(); +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp new file mode 100644 index 0000000..3041a6c --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp @@ -0,0 +1,553 @@ +#include "fuse_moe_compressed.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../op/gather_matmul.hpp" +#include "../op/moe_compressed.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +namespace { + +struct dequant_inputs { + ov::Output weight; + ov::Output scale; + ov::Output zp; + bool has_zp = false; + bool ok = false; +}; + +// Peel the chain built by make_int4_weights/make_int8_weights back to its Constant inputs. +// Grouped weights keep the pre-Reshape rank-4 form [n_expert, n, k/group, group] with scale +// and zp at [n_expert, n, k/group, 1], which is the layout MOECompressed expects. Channel-wise +// weights stay rank-3 with a rank-3 scale and carry no zp. +dequant_inputs unwrap_dequant(const ov::Output & b) { + dequant_inputs res; + + auto node = b.get_node_shared_ptr(); + while (ov::is_type(node) || ov::is_type(node)) { + node = node->get_input_node_shared_ptr(0); + } + + auto mul = ov::as_type_ptr(node); + if (!mul) { + return res; + } + res.scale = mul->input_value(1); + + auto lhs = mul->get_input_node_shared_ptr(0); + if (auto sub = ov::as_type_ptr(lhs)) { + // Take the zero point down to its Constant: an integer zp is wrapped in a Convert to f16, + // and the op wants the integer form. A natively quantized expert instead carries an exact + // f16 zp (-min/scale) with no integer behind it, which the MoE kernel does not accept. + auto zp_node = sub->get_input_node_shared_ptr(1); + while (ov::is_type(zp_node)) { + zp_node = zp_node->get_input_node_shared_ptr(0); + } + res.zp = zp_node->output(0); + res.has_zp = true; + lhs = sub->get_input_node_shared_ptr(0); + } + while (ov::is_type(lhs)) { + lhs = lhs->get_input_node_shared_ptr(0); + } + if (!ov::is_type(lhs)) { + return res; + } + + res.weight = lhs->output(0); + res.ok = res.scale.get_partial_shape().is_static() && res.weight.get_partial_shape().is_static(); + return res; +} + +size_t logical_k(const ov::Shape & shape) { + return shape.size() == 4 ? shape[2] * shape[3] : shape.back(); +} + +// Slice a [n_expert, m, ...] compressed weight/scale/zp Constant along axis 1 (m), [begin, end). +// Built by copying raw bytes directly instead of a graph Slice op: CommonOptimizations rewrites +// v8::Slice into v1::StridedSlice for constant folding regardless of disable_constant_folding +// (the rewrite doesn't carry the marking over), and that reference evaluator crashes on +// sub-byte (u4/i4) element types. Each axis-1 "row" (the trailing dims) is confirmed +// byte-aligned here -- weight/scale/zp trailing sizes are always whole groups -- so a +// byte-range memcpy per row is exact for both regular and sub-byte element types. +ov::Output slice_experts_dim1(const ov::Output & tensor, int64_t begin, int64_t end) { + auto constant = ov::as_type_ptr(tensor.get_node_shared_ptr()); + OPENVINO_ASSERT(constant, "slice_experts_dim1 expects a Constant input"); + + const auto & shape = constant->get_shape(); + OPENVINO_ASSERT(shape.size() >= 2, "slice_experts_dim1 expects rank >= 2"); + const auto & type = constant->get_element_type(); + + size_t row_elems = 1; + for (size_t i = 2; i < shape.size(); ++i) { + row_elems *= shape[i]; + } + const size_t row_bits = row_elems * type.bitwidth(); + OPENVINO_ASSERT(row_bits % 8 == 0, "slice_experts_dim1: row is not byte-aligned"); + const size_t row_bytes = row_bits / 8; + + ov::Shape out_shape = shape; + out_shape[1] = static_cast(end - begin); + + std::vector out_data(shape[0] * static_cast(end - begin) * row_bytes); + const auto * src = static_cast(constant->get_data_ptr()); + const size_t src_row_bytes = shape[1] * row_bytes; + for (size_t e = 0; e < shape[0]; ++e) { + std::memcpy(out_data.data() + e * static_cast(end - begin) * row_bytes, + src + e * src_row_bytes + static_cast(begin) * row_bytes, + static_cast(end - begin) * row_bytes); + } + + return std::make_shared(type, out_shape, out_data.data()); +} + +} // namespace + +FuseMoeCompressed::FuseMoeCompressed() { + using namespace ov::pass::pattern; + + // The gate and up projections each get their own Reshape/Transpose of the hidden state and + // their own Reshape of the routing ids, so every branch needs its own sub-pattern. On GPU + // mul_mat_id also converts the activations to f16 before the op and back to f32 after it, + // so those Converts are matched as optional. + auto hidden_gate_m = any_input(); + auto a_gate_reshape_m = wrap_type({ hidden_gate_m, any_input() }); + auto a_gate_m = + wrap_type({ optional({ a_gate_reshape_m }), any_input() }); + auto hidden_up_m = any_input(); + auto a_up_m = wrap_type( + { optional({ wrap_type({ hidden_up_m, any_input() }) }), + any_input() }); + + auto gate_w_m = any_input(); + auto up_w_m = any_input(); + auto down_w_m = any_input(); + auto ids_gate_m = any_input(); + auto ids_up_m = any_input(); + auto ids_down_m = any_input(); + + auto bgm_gate_m = wrap_type({ a_gate_m, gate_w_m, ids_gate_m, any_input() }); + auto gate_u_m = optional({ wrap_type( + { wrap_type({ bgm_gate_m, any_input() }), any_input() }) }); + + auto silu_m = wrap_type({ gate_u_m }); + + auto bgm_up_m = wrap_type({ a_up_m, up_w_m, ids_up_m, any_input() }); + auto up_u_m = optional({ wrap_type( + { wrap_type({ bgm_up_m, any_input() }), any_input() }) }); + auto swiglu_m = wrap_type({ silu_m, up_u_m }); + + auto d_t_m = wrap_type( + { optional({ wrap_type({ swiglu_m, any_input() }) }), + any_input() }); + auto bgm_down_m = wrap_type({ d_t_m, down_w_m, ids_down_m, any_input() }); + auto down_u_m = optional({ wrap_type( + { wrap_type({ bgm_down_m, any_input() }), any_input() }) }); + + auto routing_m = any_input(); + auto weighted_m = wrap_type({ down_u_m, routing_m }); + auto root_m = wrap_type({ weighted_m, any_input() }); + + const auto callback = [=](Matcher & m) { + auto & pm = m.get_pattern_value_map(); + + const auto gate = unwrap_dequant(pm.at(gate_w_m)); + const auto up = unwrap_dequant(pm.at(up_w_m)); + const auto down = unwrap_dequant(pm.at(down_w_m)); + if (!gate.ok || !up.ok || !down.ok) { + return false; + } + + const auto gate_shape = gate.weight.get_shape(); + const auto up_shape = up.weight.get_shape(); + const auto down_shape = down.weight.get_shape(); + if (gate_shape != up_shape || gate_shape.size() < 3 || down_shape.size() < 3) { + return false; + } + + // MOECompressed carries one group_size and one has_zp for all three projections, so a + // model whose down-proj is quantized differently from gate/up cannot be described. This + // happens when ggml requantizes Q5_K/Q6_K experts to channel-wise int8. + if (gate.has_zp != down.has_zp || gate_shape.size() != down_shape.size()) { + return false; + } + + // The kernel only takes an integer zero point (moe_3gemm_swiglu_opt validate_impl). + if (gate.has_zp) { + static const std::set int_zp_types = { ov::element::u4, ov::element::i4, + ov::element::u8, ov::element::i8 }; + if (int_zp_types.count(gate.zp.get_element_type()) == 0 || + int_zp_types.count(down.zp.get_element_type()) == 0) { + return false; + } + } + + // Config holds a single group_size for all three projections. + const auto group_of = [](const dequant_inputs & w) { + const auto s = w.weight.get_shape(); + return s.size() == 4 ? s[3] : logical_k(s); + }; + if (group_of(gate) != group_of(up) || group_of(gate) != group_of(down)) { + return false; + } + + // all three branches must route the same hidden state through the same experts + if (pm.at(hidden_gate_m) != pm.at(hidden_up_m)) { + return false; + } + + auto ids = pm.at(ids_down_m); + const auto ids_pshape = ids.get_partial_shape(); + if (ids_pshape.rank().is_dynamic() || ids_pshape[ids_pshape.rank().get_length() - 1].is_dynamic()) { + return false; + } + const size_t top_k = ids_pshape[ids_pshape.rank().get_length() - 1].get_length(); + + // routing weights arrive as [1, n_tokens, top_k, 1]; the op wants [..., top_k] + auto routing = pm.at(routing_m); + const auto routing_pshape = routing.get_partial_shape(); + if (routing_pshape.rank().is_dynamic() || routing_pshape.rank().get_length() != 4 || + routing_pshape[3] != 1) { + return false; + } + // MOE requires routing weights and ids to have the same shape. Drop the trailing 1 of the + // routing weights and give the ids the leading batch dim, so both become [1, n_tokens, top_k]. + routing = std::make_shared( + routing, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 3 })); + if (ids_pshape.rank().get_length() == 2) { + ids = std::make_shared( + ids, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); + } + if (routing.get_partial_shape() != ids.get_partial_shape()) { + return false; + } + + const size_t down_k = logical_k(down_shape); + const auto down_scale_shape = down.scale.get_shape(); + const size_t down_groups = down_scale_shape.size() >= 3 ? down_scale_shape[2] : 1; + + ov::op::internal::MOECompressed::Config config; + config.expert_type = ov::op::internal::MOE::Expert_type::GEMM3_SWIGLU; + config.activation_type = ov::op::internal::MOE::Activation_type::SWIGLU; + config.expert_alpha = 0.0f; + config.expert_beta = 1.0f; + config.gate_idx = 0; + config.hidden_size = logical_k(gate_shape); + config.inter_size = gate_shape[1]; + config.num_expert = gate_shape[0]; + config.num_shared_expert = 0; + config.top_k = top_k; + config.group_size = down_groups <= 1 ? std::numeric_limits::max() : down_k / down_groups; + config.has_batch_dim = true; + config.has_zp = gate.has_zp; + // dynamic makes the output follow the hidden state, so the plugin can lower this region + // to f16 together with the rest of the graph + config.out_type = ov::element::dynamic; + + auto absent_zp = [] { + auto zp = std::make_shared(ov::element::dynamic, ov::Shape{ 0 }); + ov::pass::disable_constant_folding(zp); + return zp->output(0); + }; + + // MOE takes its output type from the hidden state. Transpose the activations before the + // f16 Convert that mul_mat_id adds on GPU, so the op stays f32 like the block it replaces + // and the plugin can lower the whole region uniformly. + const auto a_transpose = pm.at(a_gate_m).get_node_shared_ptr(); + ov::Output hidden = + std::make_shared(pm.at(a_gate_reshape_m), a_transpose->input_value(1)); + + const ov::OutputVector args = { + hidden, routing, ids, + gate.weight, gate.scale, gate.has_zp ? gate.zp : absent_zp(), + up.weight, up.scale, up.has_zp ? up.zp : absent_zp(), + down.weight, down.scale, down.has_zp ? down.zp : absent_zp(), + }; + + auto moe = std::make_shared(args, config); + + // MOE takes its output type from the hidden state, which is f16 on GPU, while the rest of + // the ggml graph works in f32. + ov::Output result = moe->output(0); + const auto root_type = m.get_match_root()->get_output_element_type(0); + if (result.get_element_type() != root_type) { + result = std::make_shared(result, root_type); + } + + result.get_node_shared_ptr()->set_friendly_name(m.get_match_root()->get_friendly_name()); + ov::copy_runtime_info(m.get_matched_nodes(), result.get_node_shared_ptr()); + ov::replace_node(m.get_match_root(), result.get_node_shared_ptr()); + register_new_node(moe); + return true; + }; + + register_matcher(std::make_shared(root_m, "ov::frontend::ggml::pass::FuseMoeCompressed"), callback); +} + +FuseMoeCompressedFusedGateUp::FuseMoeCompressedFusedGateUp() { + using namespace ov::pass::pattern; + + // A single GatherMatmul computes the fused gate+up projection; the gate/up split happens + // AFTER the GEMM via two Slice ops on the last (feature) axis (see get_glu_inputs in + // op/glu_geglu.cpp), unlike FuseMoeCompressed's two-separate-GatherMatmul models. + auto hidden_m = any_input(); + auto a_reshape_m = wrap_type({ hidden_m, any_input() }); + auto a_m = wrap_type({ optional({ a_reshape_m }), any_input() }); + + auto gate_up_w_m = any_input(); + auto ids_gate_up_m = any_input(); + auto bgm_fused_m = wrap_type({ a_m, gate_up_w_m, ids_gate_up_m, any_input() }); + // Two graph shapes reach here. The rank-4 (stateless) graph restores the batch dim with an + // Unsqueeze after the Transpose and may Convert afterwards: + // GatherMatmul -> Transpose -> Unsqueeze -> [Convert] -> Slice + // The rank-3 (stateful) graph never drops to a batch dim at all, so there is no Unsqueeze + // here - it appears later, between the GEGLU and the down Reshape - and the Convert sits on + // the other side of the Transpose: + // GatherMatmul -> Convert -> Transpose -> Slice + // optional({a, b}) matches T(a, b) or bare a, so one pattern covers both. + auto gu_t_m = + wrap_type({ optional({ bgm_fused_m }), any_input() }); + auto gu_u_m = + optional({ optional({ gu_t_m, any_input() }) }); + + auto gate_slice_m = wrap_type({ gu_u_m, any_input(), any_input(), any_input(), any_input() }); + auto up_slice_m = wrap_type({ gu_u_m, any_input(), any_input(), any_input(), any_input() }); + auto gelu_m = wrap_type({ gate_slice_m }); + auto geglu_m = wrap_type({ gelu_m, up_slice_m }); + + // The rank-3 graph inserts the Unsqueeze the gate/up branch lacked right here, before the + // Reshape that feeds the down projection; the rank-4 graph goes straight from the GEGLU + // into the Reshape. + auto d_t_m = wrap_type( + { optional({ wrap_type( + { optional({ geglu_m, any_input() }), any_input() }) }), + any_input() }); + auto down_w_m = any_input(); + auto ids_down_m = any_input(); + auto bgm_down_m = wrap_type({ d_t_m, down_w_m, ids_down_m, any_input() }); + // Same two shapes as the gate/up branch above. + auto down_t_m = + wrap_type({ optional({ bgm_down_m }), any_input() }); + auto down_u_m = + optional({ optional({ down_t_m, any_input() }) }); + + // gemma-4 applies an extra per-expert output scale to the down projection before the + // router-weight multiply (llama-graph.cpp's ffn_down_exps.scale); FuseMoeCompressed's + // Qwen-shaped models have no such scale, so this node is specific to this pattern. + auto down_scale_m = any_input(); + auto down_scaled_m = wrap_type({ down_u_m, down_scale_m }); + + auto routing_m = any_input(); + auto weighted_m = wrap_type({ down_scaled_m, routing_m }); + auto root_m = wrap_type({ weighted_m, any_input() }); + + const auto callback = [=](Matcher & m) { + auto & pm = m.get_pattern_value_map(); + + const auto gate_up = unwrap_dequant(pm.at(gate_up_w_m)); + const auto down = unwrap_dequant(pm.at(down_w_m)); + if (!gate_up.ok || !down.ok) { + return false; + } + + // oneDNN's weight-decompression GEMM needs a real zero point, so a symmetric (no zp) + // projection cannot be fused here -- e.g. the GPU-only Q4_0 fallback in + // ggml_openvino_get_extracted_layout for grouped 8-bit expert weights. + if (!gate_up.has_zp || !down.has_zp) { + return false; + } + + const auto gate_up_shape = gate_up.weight.get_shape(); + const auto down_shape = down.weight.get_shape(); + if (gate_up_shape.size() < 3 || down_shape.size() < 3 || gate_up_shape.size() != down_shape.size()) { + return false; + } + + // The op reads the zero point straight off a weight port, so it must already be an + // integer Constant. Requantized experts (GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all) + // are; a native Q4_K expert keeps an exact f16 zp and is left to the unfused path + // rather than rounded here -- rounding would put a live node on that port and the + // expert GEMM would read the wrong zero point. + static const std::set int_types = { ov::element::u4, ov::element::i4, + ov::element::u8, ov::element::i8 }; + if (int_types.count(gate_up.weight.get_element_type()) == 0 || + int_types.count(down.weight.get_element_type()) == 0 || + int_types.count(gate_up.zp.get_element_type()) == 0 || + int_types.count(down.zp.get_element_type()) == 0) { + return false; + } + + const auto group_of = [](const dequant_inputs & w) { + const auto s = w.weight.get_shape(); + return s.size() == 4 ? s[3] : logical_k(s); + }; + if (group_of(gate_up) != group_of(down)) { + return false; + } + + auto gelu_node = ov::as_type_ptr(pm.at(gelu_m).get_node_shared_ptr()); + if (!gelu_node || gelu_node->get_approximation_mode() != ov::op::GeluApproximationMode::ERF) { + return false; + } + + // Read the actual gate/up split point from the matched Slice nodes instead of assuming + // inter_size/2, so this stays correct if the model's ffn_dim convention ever changes. + auto gate_slice = ov::as_type_ptr(pm.at(gate_slice_m).get_node_shared_ptr()); + auto up_slice = ov::as_type_ptr(pm.at(up_slice_m).get_node_shared_ptr()); + auto gate_end_c = ov::as_type_ptr(gate_slice->input_value(2).get_node_shared_ptr()); + auto up_begin_c = ov::as_type_ptr(up_slice->input_value(1).get_node_shared_ptr()); + if (!gate_end_c || !up_begin_c) { + return false; + } + const auto gate_end = gate_end_c->cast_vector().at(0); + const auto up_begin = up_begin_c->cast_vector().at(0); + if (gate_end != up_begin || gate_end <= 0 || gate_end >= static_cast(gate_up_shape[1])) { + return false; + } + const int64_t mid = gate_end; + const int64_t full = static_cast(gate_up_shape[1]); + + auto ids = pm.at(ids_down_m); + const auto ids_pshape = ids.get_partial_shape(); + if (ids_pshape.rank().is_dynamic() || ids_pshape[ids_pshape.rank().get_length() - 1].is_dynamic()) { + return false; + } + const size_t top_k = ids_pshape[ids_pshape.rank().get_length() - 1].get_length(); + + // Routing weights arrive as [1, n_tokens, top_k, 1] on the rank-4 (stateless) graph and + // as [n_tokens, top_k, 1] on the rank-3 (stateful) one, which is the same thing without + // the leading batch dim. Normalise the rank-3 form up to the rank-4 one so everything + // below - and the op's own config - stays in the shape that is already validated on the + // stateless path; the batch dim is taken back off the result at the end. + const auto routing_in = pm.at(routing_m); + const auto routing_pshape = routing_in.get_partial_shape(); + const auto routing_rank = routing_pshape.rank(); + if (routing_rank.is_dynamic() || (routing_rank.get_length() != 4 && routing_rank.get_length() != 3) || + routing_pshape[routing_rank.get_length() - 1] != 1) { + return false; + } + const bool batchless = routing_rank.get_length() == 3; + + // Fold gemma-4's per-expert output scale into the routing weights: the reduction is + // sum_e(routing[e] * scale[e] * down_out[e]), and MOECompressed only takes one + // per-expert weight, so pre-multiply it into routing here (same [.., top_k, 1] shape). + auto down_scale = pm.at(down_scale_m); + if (down_scale.get_partial_shape() != routing_pshape) { + return false; + } + ov::Output routing = std::make_shared(routing_in, down_scale); + if (batchless) { + routing = std::make_shared( + routing, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); + } + routing = std::make_shared( + routing, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 3 })); + if (ids_pshape.rank().get_length() == 2) { + ids = std::make_shared( + ids, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); + } + if (routing.get_partial_shape() != ids.get_partial_shape()) { + return false; + } + + const size_t down_k = logical_k(down_shape); + const auto down_scale_shape = down.scale.get_shape(); + const size_t down_groups = down_scale_shape.size() >= 3 ? down_scale_shape[2] : 1; + + // Split the fused gate_up weight/scale/zp on the output axis. slice_experts_dim1 copies + // raw bytes out of the Constant, so each half stays a Constant -- which is what the op + // needs on its weight ports. + const ov::Output gate_weight = slice_experts_dim1(gate_up.weight, 0, mid); + const ov::Output up_weight = slice_experts_dim1(gate_up.weight, mid, full); + const ov::Output gate_scale = slice_experts_dim1(gate_up.scale, 0, mid); + const ov::Output up_scale = slice_experts_dim1(gate_up.scale, mid, full); + const ov::Output gate_zp = slice_experts_dim1(gate_up.zp, 0, mid); + const ov::Output up_zp = slice_experts_dim1(gate_up.zp, mid, full); + + ov::op::internal::MOECompressed::Config config; + config.expert_type = ov::op::internal::MOE::Expert_type::GEMM3_SWIGLU; + config.activation_type = ov::op::internal::MOE::Activation_type::GEGLU_ERF; + config.expert_alpha = 0.0f; + config.expert_beta = 1.0f; + config.gate_idx = 0; + config.hidden_size = logical_k(gate_up_shape); + config.inter_size = static_cast(mid); + config.num_expert = gate_up_shape[0]; + config.num_shared_expert = 0; + config.top_k = top_k; + config.group_size = down_groups <= 1 ? std::numeric_limits::max() : down_k / down_groups; + config.has_batch_dim = true; + config.has_zp = true; + config.out_type = ov::element::dynamic; + + // Rebuild the activations Transpose before mul_mat_id's f16 GPU Convert, so the op + // stays f32 like the block it replaces (mirrors FuseMoeCompressed). + const auto a_transpose = pm.at(a_m).get_node_shared_ptr(); + ov::Output hidden = + std::make_shared(pm.at(a_reshape_m), a_transpose->input_value(1)); + + // w0 is the activated (gate) lane and w1 the multiplied (up) lane -- see + // moe_3gemm_swiglu_opt.cpp: "scratch.up = up(x) * silu(gate(x))". + const ov::OutputVector args = { + hidden, routing, ids, + gate_weight, gate_scale, gate_zp, + up_weight, up_scale, up_zp, + down.weight, down.scale, down.zp, + }; + + auto moe = std::make_shared(args, config); + + ov::Output result = moe->output(0); + // The op was fed the batched form, so it produces [1, n_tokens, hidden]. On the rank-3 + // graph the ReduceSum being replaced is [n_tokens, hidden], so drop the batch dim again. + if (batchless) { + result = std::make_shared( + result, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); + } + const auto root_type = m.get_match_root()->get_output_element_type(0); + if (result.get_element_type() != root_type) { + result = std::make_shared(result, root_type); + } + if (result.get_partial_shape() != m.get_match_root()->get_output_partial_shape(0)) { + return false; + } + + result.get_node_shared_ptr()->set_friendly_name(m.get_match_root()->get_friendly_name()); + ov::copy_runtime_info(m.get_matched_nodes(), result.get_node_shared_ptr()); + ov::replace_node(m.get_match_root(), result.get_node_shared_ptr()); + register_new_node(moe); + return true; + }; + + register_matcher( + std::make_shared(root_m, "ov::frontend::ggml::pass::FuseMoeCompressedFusedGateUp"), callback); +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h new file mode 100644 index 0000000..5c574eb --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h @@ -0,0 +1,30 @@ +#include "openvino/pass/matcher_pass.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +// Folds the MoE expert block emitted for MUL_MAT_ID (3 GatherMatmul + SwiGLU + routing +// weighting + expert reduction) into a single ov::op::internal::MOECompressed. +class FuseMoeCompressed : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseMoeCompressed") + FuseMoeCompressed(); +}; + +// Folds the MoE expert block emitted for a model whose gate and up projections share one +// fused MUL_MAT_ID weight (gemma-4: one GatherMatmul + Slice/Slice split, GEGLU activation) +// into a single ov::op::internal::MOECompressed, GEMM3_SWIGLU/GEGLU_ERF. Splits the fused +// weight/scale/zp into gate/up halves so it lands on the same 3-GEMM expert-grouped kernel +// FuseMoeCompressed uses for models with separate gate/up weights. +class FuseMoeCompressedFusedGateUp : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseMoeCompressedFusedGateUp") + FuseMoeCompressedFusedGateUp(); +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.cpp new file mode 100644 index 0000000..d6ed7f5 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.cpp @@ -0,0 +1,153 @@ +#include "fuse_moe_router.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +namespace { + +bool constant_is(const ov::Output & output, const std::vector & values) { + auto node = ov::as_type_ptr(output.get_node_shared_ptr()); + return node && node->get_element_type().is_integral_number() && node->cast_vector() == values; +} + +} // namespace + +FuseMoeRouter::FuseMoeRouter() { + using namespace ov::pass::pattern; + using namespace ov::op; + + // Match the GGML rank-4 routing chain. The GPU plugin recognizes the resulting + // Softmax -> TopK -> ReduceSum -> Divide subgraph as MoERouterFused. + auto logits = wrap_type(); + auto softmax = wrap_type({logits}); + auto topk = wrap_type({softmax, any_input()}); + topk->set_output_size(2); + auto ids = wrap_type({topk->output(1), any_input(), any_input(), any_input(), any_input()}); + auto ids_2d = wrap_type({ids, any_input()}); + auto probs = wrap_type({softmax, any_input()}); + auto data = wrap_type({probs, any_input()}); + auto data_batch = wrap_type({wrap_type({data}), any_input(), any_input()}); + auto ids_count = wrap_type({wrap_type({ids_2d}), any_input(), any_input()}); + auto target = wrap_type({data_batch, ids_count}); + auto broadcast = wrap_type({ids_2d, target}); + auto gather = wrap_type({data, broadcast, any_input()}); + auto weights_4d = wrap_type({gather, any_input()}); + auto weights = wrap_type({weights_4d, any_input()}); + auto sum = wrap_type({weights, any_input()}); + auto clamp = wrap_type({sum}); + auto tile = wrap_type({clamp, any_input()}); + auto norm = wrap_type({weights, tile}); + + const auto callback = [=](Matcher & m) { + const auto & pm = m.get_pattern_value_map(); + const auto node = [&](const std::shared_ptr & p) { return pm.at(p).get_node_shared_ptr(); }; + const auto input_is = [&](const std::shared_ptr & p, size_t i, const std::vector & v) { + return constant_is(node(p)->input_value(i), v); + }; + auto mm = ov::as_type_ptr(node(logits)); + auto sm = ov::as_type_ptr(node(softmax)); + auto tk = ov::as_type_ptr(node(topk)); + auto reduce = ov::as_type_ptr(node(sum)); + auto limit = ov::as_type_ptr(node(clamp)); + const auto shape = mm->get_output_partial_shape(0); + if (shape.rank() != 4 || shape[0] != 1 || shape[1] != 1 || shape[3].is_dynamic() || + mm->get_transpose_a() || mm->get_input_partial_shape(0).rank() != 4 || + mm->get_input_partial_shape(1).rank() != 2 || (sm->get_axis() != -1 && sm->get_axis() != 3) || + tk->get_axis() != 3 || tk->get_mode() != v11::TopK::Mode::MAX || + tk->get_sort_type() != v11::TopK::SortType::SORT_VALUES || tk->get_stable() || + tk->get_index_element_type() != ov::element::i32 || + !tk->output(0).get_target_inputs().empty()) { + return false; + } + const int64_t experts = shape[3].get_length(); + auto end = ov::as_type_ptr(node(ids)->get_input_node_shared_ptr(2)); + if (!end || !end->get_element_type().is_integral_number() || ov::shape_size(end->get_shape()) != 1) { + return false; + } + const int64_t k = end->cast_vector()[0]; + const auto sorted_shape = tk->get_output_partial_shape(1); + if (k <= 0 || k > experts || sorted_shape[3].is_dynamic() || sorted_shape[3].get_length() < k || + !input_is(probs, 1, {1, -1, experts, 1}) || + !input_is(data, 1, {0}) || !input_is(ids_2d, 1, {0, 1}) || + !input_is(weights_4d, 1, {0}) || !input_is(weights, 1, {1, 1, -1, k}) || + ov::as_type_ptr(node(probs))->get_special_zero() || + ov::as_type_ptr(node(weights))->get_special_zero() || + !input_is(gather, 2, {1}) || ov::as_type_ptr(node(gather))->get_batch_dims() != 1 || + !input_is(data_batch, 1, {0}) || !input_is(data_batch, 2, {0}) || + !input_is(ids_count, 1, {1}) || !input_is(ids_count, 2, {0}) || + ov::as_type_ptr(node(data_batch))->get_batch_dims() != 0 || + ov::as_type_ptr(node(ids_count))->get_batch_dims() != 0 || + ov::as_type_ptr(node(target))->get_axis() != 0 || + ov::as_type_ptr(node(broadcast))->get_broadcast_spec().m_type != ov::op::BroadcastType::BIDIRECTIONAL || + !reduce->get_keep_dims() || (!input_is(sum, 1, {-1}) && !input_is(sum, 1, {3})) || + !input_is(tile, 1, {1, 1, 1, k}) || + ov::as_type_ptr(node(norm))->get_autob().m_type != ov::op::AutoBroadcastType::NUMPY) { + return false; + } + // The top-k softmax sum is at least k / experts. Leave margin for rounding. + // This guard is only valid for unbiased softmax routing. + if (!(limit->get_min() <= 0.5 * double(k) / experts && limit->get_max() >= 2.0)) { + return false; + } + std::vector> slices; + for (const auto & input : tk->output(1).get_target_inputs()) { + auto slice = ov::as_type_ptr(input.get_node()->shared_from_this()); + if (!slice || input.get_index() != 0 || slice->get_input_size() != 5 || + !constant_is(slice->input_value(1), {0}) || !constant_is(slice->input_value(2), {k}) || + !constant_is(slice->input_value(3), {1}) || + (!constant_is(slice->input_value(4), {3}) && !constant_is(slice->input_value(4), {-1}))) { + return false; + } + slices.push_back(slice); + } + + // Remove GGML's leading singleton dimension while building the plugin pattern, + // then restore it on both outputs for the following GGML nodes. + auto axis0 = v0::Constant::create(ov::element::i64, ov::Shape{1}, {0}); + auto hidden = std::make_shared(mm->input_value(0), axis0); + auto routing = std::make_shared(hidden, mm->input_value(1), false, mm->get_transpose_b()); + auto probabilities = std::make_shared(routing, -1); + auto selected = std::make_shared(probabilities, + v0::Constant::create(ov::element::i64, ov::Shape{}, {k}), 2, + v11::TopK::Mode::MAX, v11::TopK::SortType::SORT_VALUES, tk->get_index_element_type(), false); + auto total = std::make_shared(selected->output(0), + v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); + auto normalized = std::make_shared(selected->output(0), total); + auto weights_out = std::make_shared(normalized, axis0); + auto ids_out = std::make_shared(selected->output(1), axis0); + ov::copy_runtime_info({mm, sm, tk, node(norm)}, {hidden, routing, probabilities, selected, total, normalized, weights_out, ids_out}); + ov::replace_node(node(norm), weights_out); + for (const auto & slice : slices) { + ov::replace_node(slice, ids_out); + } + return true; + }; + register_matcher(std::make_shared(norm, "ov::frontend::ggml::pass::FuseMoeRouter"), callback); +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.h b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.h new file mode 100644 index 0000000..49e01f1 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +class FuseMoeRouter : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseMoeRouter") + FuseMoeRouter(); +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp new file mode 100644 index 0000000..6fffd67 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp @@ -0,0 +1,283 @@ +#include "fuse_to_conv.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace opp = ov::pass::pattern; + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +FuseToConv::FuseToConv() { + const auto m_in0 = opp::any_input(); + const auto m_in1 = opp::any_input(); + const auto m_matmul = opp::wrap_type({m_in0, m_in1}); + + const auto callback = [=](opp::Matcher & m) { + auto matmul_node = ov::as_type_ptr(m.get_match_root()); + if (!matmul_node) { + return false; + } + + auto unwrap = [](ov::Output n) { + while (ov::is_type(n.get_node_shared_ptr()) || + ov::is_type(n.get_node_shared_ptr())) { + n = n.get_node_shared_ptr()->input_value(0); + } + return n; + }; + + auto get_im2col = [&](ov::Output trace) + -> std::pair, std::shared_ptr> { + auto t2 = ov::as_type_ptr(unwrap(std::move(trace)).get_node_shared_ptr()); + auto r1 = t2 ? ov::as_type_ptr(t2->get_input_node_shared_ptr(0)) : nullptr; + auto t1 = r1 ? ov::as_type_ptr(r1->get_input_node_shared_ptr(0)) : nullptr; + auto eip = + t1 ? ov::as_type_ptr(t1->get_input_node_shared_ptr(0)) : nullptr; + auto pad = eip ? ov::as_type_ptr(eip->get_input_node_shared_ptr(0)) : nullptr; + return {eip, pad}; + }; + + bool weight_is_in0 = true; + auto [eip, pad] = get_im2col(matmul_node->input_value(1)); + ov::Output w_trace = matmul_node->input_value(0); + if (!pad) { + std::tie(eip, pad) = get_im2col(matmul_node->input_value(0)); + w_trace = matmul_node->input_value(1); + weight_is_in0 = false; + if (!pad) { + return false; + } + } + + auto pb_const = ov::as_type_ptr(pad->get_input_node_shared_ptr(1)); + auto pe_const = ov::as_type_ptr(pad->get_input_node_shared_ptr(2)); + if (!pb_const || !pe_const) { + return false; + } + + const auto pb = pb_const->cast_vector(); + const auto pe = pe_const->cast_vector(); + if (pb.size() < 4 || pe.size() < 4) { + return false; + } + + auto image_input = pad->input_value(0); + auto image_shape = image_input.get_partial_shape(); + if (image_shape.rank() != 4 || image_shape[1].is_dynamic()) { + return false; + } + const size_t IC = static_cast(image_shape[1].get_length()); + + w_trace = unwrap(w_trace); + auto weight_pshape = w_trace.get_partial_shape(); + if (!weight_pshape.is_static()) { + return false; + } + + const auto & ws = weight_pshape.to_shape(); + const size_t KH = eip->get_sizes()[0]; + const size_t KW = eip->get_sizes()[1]; + const size_t kernel_spatial_ic = IC * KH * KW; + + size_t groups = 0; + if (IC == 1 && image_shape[0].is_static() && image_shape[2].is_static() && image_shape[3].is_static()) { + if ((ws.size() == 4 && ws[1] == 1 && ws[2] == KH && ws[3] == KW && ws[0] > 1) || + (ws.size() == 2 && ws[1] == KH * KW && ws[0] > 1) || + (ws.size() == 3 && ws[1] == 1 && ws[2] == KH * KW && ws[0] > 1)) { + groups = ws[0]; + } else if (ws.size() == 4 && ws[0] == 1 && ws[2] == 1 && ws[3] == KH * KW && ws[1] > 1) { + groups = ws[1]; + } + } + + const bool is_depthwise = groups > 1 && (image_shape[0].get_length() % groups == 0); + + ov::Output conv_out; + size_t OC = 0; + + if (is_depthwise) { + const size_t N = static_cast(image_shape[0].get_length()) / groups; + const size_t IH = static_cast(image_shape[2].get_length()); + const size_t IW = static_cast(image_shape[3].get_length()); + OC = groups; + + auto img_shape_const = register_new_node( + ov::element::i64, ov::Shape{4}, + std::vector{static_cast(N), static_cast(groups), static_cast(IH), + static_cast(IW)}); + ov::Output image_reshaped = + register_new_node(image_input, img_shape_const, false); + + const ov::Shape conv_w_shape = {groups, 1, 1, KH, KW}; + ov::Output weight_input; + if (auto weight_const = ov::as_type_ptr(w_trace.get_node_shared_ptr())) { + weight_input = register_new_node(weight_const->get_element_type(), conv_w_shape, + weight_const->get_data_ptr()); + } else { + auto shape_const = register_new_node( + ov::element::i64, ov::Shape{5}, + std::vector{static_cast(groups), 1, 1, static_cast(KH), + static_cast(KW)}); + weight_input = register_new_node(w_trace, shape_const, false); + } + + if (weight_input.get_element_type() != image_reshaped.get_element_type()) { + weight_input = register_new_node(weight_input, image_reshaped.get_element_type()); + } + + conv_out = register_new_node( + image_reshaped, weight_input, eip->get_strides(), ov::CoordinateDiff{pb[2], pb[3]}, + ov::CoordinateDiff{pe[2], pe[3]}, ov::Strides{eip->get_rates()[0], eip->get_rates()[1]}, + ov::op::PadType::EXPLICIT); + } else { + if ((ws.size() == 4 && ws[1] == IC && ws[2] == KH && ws[3] == KW) || + (ws.size() == 3 && ws[1] == IC && ws[2] == KW) || (ws.size() == 2 && ws[1] == kernel_spatial_ic)) { + OC = ws[0]; + } else if ((ws.size() == 4 && ws[0] == 1 && ws[2] == IC && ws[3] == KW) || + (ws.size() == 2 && ws[0] == kernel_spatial_ic)) { + OC = ws[1]; + } else if (ws.size() == 3 && ws[0] == 1 && ws[1] == IC && ws[2] == KW) { + OC = 1; + } else if (ws.size() == 4 && ws[3] == kernel_spatial_ic) { + OC = ws[2]; + } else if (ws.size() == 4 && ws[2] == kernel_spatial_ic) { + OC = ws[3]; + } else if (kernel_spatial_ic && ov::shape_size(ws) % kernel_spatial_ic == 0) { + OC = ov::shape_size(ws) / kernel_spatial_ic; + } else { + return false; + } + + const ov::Shape conv_w_shape = {OC, IC, KH, KW}; + + ov::Output weight_input; + if (auto weight_const = ov::as_type_ptr(w_trace.get_node_shared_ptr())) { + weight_input = register_new_node(weight_const->get_element_type(), conv_w_shape, + weight_const->get_data_ptr()); + } else { + auto shape_const = register_new_node( + ov::element::i64, ov::Shape{4}, + std::vector{static_cast(OC), static_cast(IC), static_cast(KH), + static_cast(KW)}); + weight_input = register_new_node(w_trace, shape_const, false); + } + + if (weight_input.get_element_type() != image_input.get_element_type()) { + weight_input = register_new_node(weight_input, image_input.get_element_type()); + } + + conv_out = register_new_node( + image_input, weight_input, eip->get_strides(), ov::CoordinateDiff{pb[2], pb[3]}, + ov::CoordinateDiff{pe[2], pe[3]}, ov::Strides{eip->get_rates()[0], eip->get_rates()[1]}, + ov::op::PadType::EXPLICIT); + } + + constexpr auto target_type = ov::element::f32; + if (conv_out.get_element_type() != target_type) { + conv_out = register_new_node(conv_out, target_type); + } + + std::shared_ptr add_node; + ov::Output bias_input; + + auto try_fuse_bias = [&](const std::shared_ptr & n) { + auto add = ov::as_type_ptr(n); + if (!add) { + return false; + } + for (size_t i = 0; i < 2; ++i) { + if (ov::is_type(add->get_input_node_shared_ptr(i))) { + bias_input = add->input_value(i); + add_node = add; + return true; + } + } + return false; + }; + + for (const auto & consumer : matmul_node->output(0).get_target_inputs()) { + auto n = consumer.get_node()->shared_from_this(); + if (try_fuse_bias(n)) { + break; + } + if (ov::is_type(n) || ov::is_type(n)) { + for (const auto & next : n->output(0).get_target_inputs()) { + if (try_fuse_bias(next.get_node()->shared_from_this())) { + break; + } + } + } + if (add_node) { + break; + } + } + + ov::Output final_out = conv_out; + std::shared_ptr target_node = matmul_node; + + if (add_node) { + ov::Output bias = bias_input; + if (bias.get_element_type() != target_type) { + bias = register_new_node(bias, target_type); + } + auto bias_shape = register_new_node( + ov::element::i64, ov::Shape{4}, std::vector{1, static_cast(OC), 1, 1}); + bias = register_new_node(bias, bias_shape, false); + final_out = register_new_node(conv_out, bias); + target_node = add_node; + } + + if (!is_depthwise) { + auto perm = register_new_node( + ov::element::i64, ov::Shape{4}, + weight_is_in0 ? std::vector{1, 0, 2, 3} : std::vector{0, 2, 3, 1}); + final_out = register_new_node(final_out, perm); + } + + auto orig_shape = target_node->get_output_partial_shape(0); + if (orig_shape.is_static() && final_out.get_partial_shape().is_static()) { + if (ov::shape_size(orig_shape.to_shape()) != ov::shape_size(final_out.get_shape())) { + return false; + } + } + if (orig_shape.is_static() && final_out.get_partial_shape() != orig_shape) { + auto shape_const = register_new_node(ov::element::i64, ov::Shape{orig_shape.size()}, + orig_shape.to_shape()); + final_out = register_new_node(final_out, shape_const, false); + } + + auto orig_type = target_node->get_output_element_type(0); + if (final_out.get_element_type() != orig_type) { + final_out = register_new_node(final_out, orig_type); + } + + final_out.get_node_shared_ptr()->set_friendly_name(target_node->get_friendly_name()); + ov::copy_runtime_info(m.get_matched_nodes(), final_out.get_node_shared_ptr()); + ov::replace_node(target_node, final_out.get_node_shared_ptr()); + + return true; + }; + + register_matcher(std::make_shared(m_matmul, "ov::frontend::ggml::pass::FuseToConv"), callback); +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/eliminate_zp.h b/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h similarity index 56% rename from ggml/src/ggml-openvino/openvino/pass/eliminate_zp.h rename to ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h index edd3cd7..feac14b 100644 --- a/ggml/src/ggml-openvino/openvino/pass/eliminate_zp.h +++ b/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h @@ -5,10 +5,10 @@ namespace frontend { namespace ggml { namespace pass { -class EliminateZeroPoints : public ov::pass::MatcherPass { +class FuseToConv : public ov::pass::MatcherPass { public: - OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::EliminateZeroPoints") - EliminateZeroPoints(); + OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseToConv") + FuseToConv(); }; } // namespace pass diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp new file mode 100644 index 0000000..04de2d0 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp @@ -0,0 +1,114 @@ +#include "kv_state_seq_axis.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +namespace { + +const std::vector & seq_axis_perm() { + // [1, seq, n_heads_kv, head_size] <-> [1, n_heads_kv, seq, head_size] + static const std::vector perm{0, 2, 1, 3}; + return perm; +} + +// True when the state still has the frontend's stateful KV layout, so the sequence axis +// can be moved: rank 4, batch and both head dims static, and seq the only dynamic dim, +// at dim 1. Any KV head count is fine. With a single head the rewrite is pure metadata +// ([1, seq, 1, head] and [1, 1, seq, head] are the same memory); with several heads it +// also drops the reader-side transpose of the whole accumulated state, which is where +// most of the gain comes from at depth. +bool can_move_seq_axis(const ov::PartialShape & shape) { + return shape.rank().is_static() && shape.rank().get_length() == 4 && shape[0].is_static() && + shape[1].is_dynamic() && shape[2].is_static() && shape[3].is_static(); +} + +std::shared_ptr match_kv_append(const std::shared_ptr & assign) { + auto concat = ov::as_type_ptr(assign->get_input_node_shared_ptr(0)); + if (!concat || concat->get_input_size() != 2 || concat->get_axis() != 1) { + return nullptr; + } + auto read_value = ov::as_type_ptr(concat->get_input_node_shared_ptr(0)); + if (!read_value || read_value->get_variable() != assign->get_variable()) { + return nullptr; + } + if (!can_move_seq_axis(read_value->get_output_partial_shape(0))) { + return nullptr; + } + return concat; +} + +} // namespace + +bool KVStateSeqAxis::run_on_model(const std::shared_ptr & model) { + std::vector> assigns; + for (const auto & op : model->get_ops()) { + if (auto assign = ov::as_type_ptr(op)) { + assigns.push_back(assign); + } + } + + bool changed = false; + for (const auto & assign : assigns) { + auto concat = match_kv_append(assign); + if (!concat) { + continue; + } + auto read_value = ov::as_type_ptr(concat->get_input_node_shared_ptr(0)); + + auto variable = read_value->get_variable(); + auto info = variable->get_info(); + const auto & shape = info.data_shape; + info.data_shape = ov::PartialShape{shape[0], shape[2], shape[1], shape[3]}; + variable->update(info); + read_value->validate_and_infer_types(); + + auto readers = concat->output(0).get_target_inputs(); + + auto new_rows = concat->input_value(1); + auto perm_in = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); + concat->set_argument(1, std::make_shared(new_rows, perm_in)); + concat->set_axis(2); + concat->validate_and_infer_types(); + + // Readers still expect seq at dim 1. A reader that is itself the inverse + // Transpose wanted seq at dim 2 all along, so drop it; give anything else the + // inverse Transpose so its input is unchanged. + for (const auto & reader : readers) { + auto * node = reader.get_node(); + if (ov::is_type(node)) { + continue; + } + bool dropped = false; + if (auto * transpose = ov::as_type(node)) { + auto order = ov::as_type_ptr(transpose->get_input_node_shared_ptr(1)); + if (order && order->cast_vector() == seq_axis_perm()) { + ov::replace_output_update_name(transpose->output(0), concat->output(0)); + dropped = true; + } + } + if (!dropped) { + auto perm_out = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); + reader.replace_source_output(std::make_shared(concat->output(0), perm_out)); + } + } + changed = true; + } + + return changed; +} + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h new file mode 100644 index 0000000..579022c --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h @@ -0,0 +1,24 @@ +#include "openvino/pass/pass.hpp" + +namespace ov { +namespace frontend { +namespace ggml { +namespace pass { + +// Moves the sequence axis of the stateful KV cache from dim 1 to dim 2, i.e. from +// [1, seq, n_heads_kv, head_size] to [1, n_heads_kv, seq, head_size], and updates the +// Concat that appends to it. Two wins: the GPU plugin only appends new tokens in place +// when the growing axis is a spatial axis, and the reader no longer has to transpose the +// whole accumulated state every token (that cost grows with context length, so it is the +// larger win at depth for a model with several KV heads). Only rewrites states that still +// match the frontend layout, so it no-ops if that layout ever changes. +class KVStateSeqAxis : public ov::pass::ModelPass { +public: + OPENVINO_MODEL_PASS_RTTI("ov::frontend::ggml::pass::KVStateSeqAxis") + bool run_on_model(const std::shared_ptr & model) override; +}; + +} // namespace pass +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h b/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h index b953856..c229e25 100644 --- a/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +++ b/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h @@ -1,8 +1,8 @@ #pragma once #include "mark_decompression_convert_constant_folding.h" -#include "openvino/pass/matcher_pass.hpp" #include "openvino/core/visibility.hpp" +#include "openvino/pass/matcher_pass.hpp" #ifdef OPENVINO_STATIC_LIBRARY # define TRANSFORMATIONS_API diff --git a/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h b/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h new file mode 100644 index 0000000..d51303d --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h @@ -0,0 +1,44 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// Local mirror of OpenVINO's ov::pass::MarkDequantization pass declaration. +// +// The pass body is provided by the linked libopenvino.so; only the declaration is needed here so +// we can register it directly in our own TranslateSession::apply_transformations (same approach as +// MarkCompressedFloatConstants's local mirror in mark_decompression_convert_constant_folding.h). This +// lets us mark our GatherMatmul dequantization chain with disable_constant_folding regardless of the +// CPU/GPU plugin's own is_decompression_multiply() consumer allowlist. +// The class layout must stay in sync with +// openvino/src/common/transformations/include/transformations/low_precision/mark_dequantization_subgraph.hpp + +#pragma once + +#include "openvino/core/type/element_type.hpp" +#include "openvino/core/visibility.hpp" +#include "openvino/pass/matcher_pass.hpp" + +#ifdef OPENVINO_STATIC_LIBRARY +# define TRANSFORMATIONS_API +#else +# ifdef IMPLEMENT_OPENVINO_API +# define TRANSFORMATIONS_API OPENVINO_CORE_EXPORTS +# else +# define TRANSFORMATIONS_API OPENVINO_CORE_IMPORTS +# endif // IMPLEMENT_OPENVINO_API +#endif // OPENVINO_STATIC_LIBRARY + +namespace ov { +namespace pass { + +class TRANSFORMATIONS_API MarkDequantization; + +} // namespace pass +} // namespace ov + +class ov::pass::MarkDequantization : public MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("MarkDequantization") + explicit MarkDequantization(const element::TypeVector & precisions, + bool fold_subtract_const = false, + bool fold_multiply_const = true); +}; diff --git a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp b/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp index 20a3a37..09c213f 100644 --- a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +++ b/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -26,7 +27,7 @@ SqueezeMatmul::SqueezeMatmul() { const auto callback = [=](ov::pass::pattern::Matcher & m) { const auto & pattern_map = m.get_pattern_value_map(); auto matmul_node = - std::dynamic_pointer_cast(pattern_map.at(m_matmul).get_node_shared_ptr()); + ov::as_type_ptr(pattern_map.at(m_matmul).get_node_shared_ptr()); auto act = pattern_map.at(m_act); auto wei = pattern_map.at(m_wei); auto act_shape = act.get_partial_shape(); diff --git a/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp b/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp new file mode 100644 index 0000000..f051891 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp @@ -0,0 +1,41 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +namespace ov { + +/** + * @brief Holds weightless caching attributes of a single constant. + * + * WeightlessCacheAttribute class represents runtime info attribute that holds + * the values of original size of the constant in bytes and the binary offset of the + * constant's data in the weights file used by the weightless caching mechanism. It's + * not copyable in case the data was changed (the original node was replaced by a new + * one produced during the tranformation pipeline) - in that case weightless caching + * can't be used for that constant. + */ +class OPENVINO_API WeightlessCacheAttribute : public RuntimeAttribute { +public: + OPENVINO_RTTI("WeightlessCacheAttribute", "0", RuntimeAttribute) + + WeightlessCacheAttribute() = delete; + + WeightlessCacheAttribute(size_t original_size, size_t bin_offset, ov::element::Type original_dtype) + : original_size(original_size), + bin_offset(bin_offset), + original_dtype(original_dtype) {} + + bool is_copyable() const override; + + size_t original_size; + size_t bin_offset; + ov::element::Type original_dtype; +}; + +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index 23a1dea..ec3596d 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -1,39 +1,60 @@ #include "translate_session.h" +#include "ggml-impl.h" +#include "ggml-openvino/ggml-openvino-extra.h" #include "ggml-openvino/openvino/node_context.h" #include "ggml-openvino/openvino/utils.h" #include "input_model.h" -#include "pass/eliminate_zp.h" +#include "pass/fuse_argsort_topk.h" +#include "pass/fuse_moe_router.h" +#include "pass/align_eltwise_ranks.h" +#include "pass/fuse_moe_compressed.h" +#include "pass/fuse_to_conv.h" +#include "pass/kv_state_seq_axis.h" #include "pass/mark_decompression_convert_constant_folding.h" +#include "pass/mark_dequantization_subgraph.h" #include "pass/squeeze_matmul.h" +#include "rt_info/weightless_caching_attributes.hpp" +#include #include #include #include #include #include +#include +#include +#include +#include #include #include #include +#include #include #include #include #include #include +#include +#include +#include #include #include #include #include #include +#include #include #include #include #include +#include #include #include #include #include -#include +#include +#include namespace ov { namespace frontend { @@ -43,6 +64,28 @@ using namespace ov::op; namespace { +std::shared_ptr create_parameter(const std::string & name, + const ModelInputInfo & input_info) { + auto param_node = std::make_shared(input_info.type, input_info.shape); + param_node->set_friendly_name(name); + param_node->output(0).get_tensor().set_names({name}); + return param_node; +} + +std::shared_ptr create_extra_input(const std::string & name, const ModelExtraInputInfo & input_info) { + if (input_info.is_parameter) { + auto param_node = std::make_shared(input_info.type, input_info.shape); + param_node->set_friendly_name(name); + param_node->output(0).get_tensor().set_names({name}); + return param_node; + } + + auto constant = std::make_shared(input_info.type, input_info.shape, + std::vector{input_info.value}); + constant->set_friendly_name(name); + return constant; +} + ov::pass::MakeStateful::ParamResPairs get_kv_param_res_pairs( const std::shared_ptr & model, const std::map & kv_param_res_names) { @@ -77,49 +120,113 @@ ov::pass::MakeStateful::ParamResPairs get_kv_param_res_pairs( return pairs; } -void add_sliced_mask(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { - - auto create_sliced_mask = [&](const std::string & mask_name, const std::string & sliced_name, bool is_static) { +void add_sliced_mask_stateful(TensorMap & tensor_map, bool imrope) { + auto create_sliced_mask = [&](const std::string & mask_name, const std::string & sliced_name) { if ((tensor_map.find(mask_name) != tensor_map.end()) && - (tensor_map.find("token_len_per_seq") != tensor_map.end())) { + (tensor_map.find("token_len_per_seq") != tensor_map.end()) && + (tensor_map.find("inp_pos") != tensor_map.end())) { auto token_len_per_seq = tensor_map.at("token_len_per_seq").get_node_shared_ptr(); auto mask = tensor_map.at(mask_name).get_node_shared_ptr(); - std::shared_ptr mask_sliced; - if (is_static) { - mask_sliced = mask; - } else if (ggml_model_decoder.is_stateful()) { - auto zero_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {0,0}); - auto one_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,1}); - auto zero_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto three_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - auto neg_one_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto axes = ov::op::v0::Constant::create(ov::element::i64, {2}, {-2,-1}); - auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); - auto gather_inp_pos = std::make_shared(inp_pos, neg_one_1d, three_1d); - auto reshaped_inp_pos = std::make_shared(gather_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false); - auto inp_pos_incremented = std::make_shared(reshaped_inp_pos, ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {1})); - auto stop = std::make_shared(ov::OutputVector{token_len_per_seq, std::make_shared(inp_pos_incremented, token_len_per_seq)}, 0); - mask_sliced = - std::make_shared(mask, zero_2d, stop, one_2d, axes); - mask_sliced = std::make_shared(mask_sliced, ov::element::f16); - mask_sliced->set_friendly_name(sliced_name); - } else { - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - mask_sliced = std::make_shared(mask, zero, token_len_per_seq, one, two); - mask_sliced = std::make_shared(mask_sliced, ov::element::f16); - mask_sliced->set_friendly_name(sliced_name); + std::shared_ptr mask_sliced = mask; + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + + auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); + // IMROPE's fourth position plane is zero for text; use the token's first plane. + ov::Output last_index = neg_one; + if (imrope) { + last_index = std::make_shared(token_len_per_seq, neg_one); } + auto last_inp_pos = std::make_shared(inp_pos, last_index, three); + auto last_inp_pos_1d = std::make_shared( + last_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false); + auto last_inp_pos_cvt = std::make_shared(last_inp_pos_1d, ov::element::i64); + auto last_inp_pos_inc = std::make_shared(last_inp_pos_cvt, one); + + mask_sliced = std::make_shared(mask, zero, last_inp_pos_inc, step, axes); + mask_sliced = std::make_shared(mask_sliced, ov::element::f16); + mask_sliced->set_friendly_name(sliced_name); + tensor_map.insert({sliced_name, mask_sliced->output(0)}); } }; - create_sliced_mask("self_kq_mask", "KQ_mask_sliced", ggml_model_decoder.is_static()); - create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced", ggml_model_decoder.is_static()); + create_sliced_mask("self_kq_mask", "KQ_mask_sliced"); + create_sliced_mask("KQ_mask", "KQ_mask_sliced"); + create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced"); +} + +// Rebuild the sliding-window mask from absolute positions. +// ggml caps self_kq_mask_swa at the size of its own SWA cache, but the stateful KV state is +// Concat-appended and grows without bound, so past that cap the two disagree on length and the +// mask add fails. A pure-Concat state is ordered by position, so positions can rebuild the mask. +// swa_window holds the real n_swa, read back from the ggml mask in ggml-decoder.cpp. +// No-op when the graph has no SWA mask, or when the window could not be read back. +void add_position_mask_stateful_swa(TensorMap & tensor_map) { + if (tensor_map.find("self_kq_mask_swa") == tensor_map.end() || tensor_map.find("inp_pos") == tensor_map.end() || + tensor_map.find("swa_window") == tensor_map.end()) { + return; + } + + auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); + + auto zero_i64 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto one_i64 = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + + auto query_pos = std::make_shared(inp_pos, ov::element::i64); + auto query_pos_1d = std::make_shared( + query_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}), false); + + auto last_pos = std::make_shared(inp_pos, neg_one, three); + auto last_pos_1d = std::make_shared(last_pos, one_i64, false); + auto last_pos_cvt = std::make_shared(last_pos_1d, ov::element::i64); + auto total_len = std::make_shared(last_pos_cvt, one_i64); + auto total_len_scalar = std::make_shared(total_len); + + auto cached_pos = std::make_shared( + ov::op::v0::Constant::create(ov::element::i64, {}, {0}), total_len_scalar, + ov::op::v0::Constant::create(ov::element::i64, {}, {1}), ov::element::i64); + + auto query_col = std::make_shared( + query_pos_1d, ov::op::v0::Constant::create(ov::element::i64, {2}, {-1, 1}), false); + auto cached_row = std::make_shared( + cached_pos, ov::op::v0::Constant::create(ov::element::i64, {2}, {1, -1}), false); + auto diff = std::make_shared(query_col, cached_row); + + auto swa_window = tensor_map.at("swa_window").get_node_shared_ptr(); + auto window = std::make_shared(swa_window, ov::element::i64); + auto causal_ok = std::make_shared(diff, zero_i64); + auto window_ok = std::make_shared(diff, window); + auto keep = std::make_shared(causal_ok, window_ok); + + auto zero_f = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); + auto neg_inf_f = ov::op::v0::Constant::create(ov::element::f32, {}, {-std::numeric_limits::infinity()}); + std::shared_ptr mask = std::make_shared(keep, zero_f, neg_inf_f); + + auto batch_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + mask = std::make_shared(mask, batch_axis); + mask = std::make_shared(mask, batch_axis); + mask = std::make_shared(mask, ov::element::f16); + mask->set_friendly_name("KQ_mask_swa_sliced"); + + tensor_map["KQ_mask_swa_sliced"] = mask->output(0); } void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { + // When ROPE ops in the graph have divergent op_params (e.g. gemma4's mixed + // SWA/non-SWA layers with different n_dims or freq_base), a shared sin/cos + // precompute cannot broadcast across every ROPE use. Skip it here and let + // translate_rope() build sin/cos per-op from its own op_params. + if (ggml_model_decoder.has_mixed_rope_params()) { + return; + } int32_t * rope_params = ggml_model_decoder.get_rope_params(); if (tensor_map.find("inp_pos") == tensor_map.end() || rope_params == nullptr) { return; @@ -142,8 +249,12 @@ void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) // Create common patterns void preprocess(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { - add_sliced_mask(tensor_map, ggml_model_decoder); - add_rope_sin_cos(tensor_map, ggml_model_decoder); + if (ggml_model_decoder.is_stateful()) { + add_sliced_mask_stateful(tensor_map, ggml_model_decoder.get_rope_params()[2] == GGML_ROPE_TYPE_IMROPE); + add_position_mask_stateful_swa(tensor_map); + } + // This optimization is error-prone + // add_rope_sin_cos(tensor_map, ggml_model_decoder); } } // namespace @@ -170,39 +281,49 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo auto tensor_map = std::make_shared(); std::shared_ptr resulting_model; - const auto & ggml_model = std::dynamic_pointer_cast(input_model); + const auto & ggml_model = ov::as_type_ptr(input_model); std::shared_ptr ggml_model_decoder = ggml_model->get_model_decoder(); for (const auto & it : ggml_model_decoder->get_model_inputs()) { - params.push_back(std::dynamic_pointer_cast(it.second)); - (*tensor_map)[it.first] = it.second; + auto param_node = create_parameter(it.first, it.second); + params.push_back(param_node); + (*tensor_map)[it.first] = param_node; } for (const auto & it : ggml_model_decoder->get_model_extra_inputs()) { - if (std::dynamic_pointer_cast(it.second)) { - params.push_back(std::dynamic_pointer_cast(it.second)); + auto input_node = create_extra_input(it.first, it.second); + if (it.second.is_parameter) { + params.push_back(ov::as_type_ptr(input_node)); } - (*tensor_map)[it.first] = it.second; + (*tensor_map)[it.first] = input_node; } for (const auto & it : ggml_model_decoder->get_model_weights()) { (*tensor_map)[it.first] = it.second; } - auto node_visitor = [&](std::shared_ptr decoder, int node_idx) { + auto translate_node = [&](const std::shared_ptr & decoder, int node_idx) { auto operation_type = decoder->get_op_type(node_idx); if (operation_type == "GGML_OP_NONE") { - return; + return ov::OutputVector{}; + } + + const auto & node_output_names = decoder->get_output_names(node_idx); + if (operation_type == "GGML_OP_VIEW" && decoder->get_op_case(node_idx) == 2 && node_output_names.size() == 1) { + auto direct_output = tensor_map->find(node_output_names[0]); + if (direct_output != tensor_map->end()) { + // GDN publishes its native attention/state outputs under the two GGML VIEW names. + // Keep those mappings instead of rebuilding slices of a packed temporary. + return ov::OutputVector{direct_output->second}; + } } - ov::OutputVector converted_outputs; auto it = m_translator_map.find(operation_type); FRONT_END_OP_CONVERSION_CHECK(it != m_translator_map.end(), "Translation for operation type ", operation_type, " is not implemented."); NodeContext node_context(decoder, tensor_map, node_idx, this); - converted_outputs = it->second(node_context); + ov::OutputVector converted_outputs = it->second(node_context); - const auto & node_output_names = decoder->get_output_names(node_idx); FRONT_END_OP_CONVERSION_CHECK(node_output_names.size() == converted_outputs.size(), "Number of ", operation_type, " outputs greater than number of converted outputs, which are ", node_output_names.size(), " and ", converted_outputs.size(), " respectively."); @@ -213,6 +334,46 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo (*tensor_map)[output_name] = converted_outputs[i]; } } + return converted_outputs; + }; + + // To handle cases like this + // 3: [ 18432, 1, 1, 1] RESHAPE cache_r_l0 (reshaped)#3 + // [ 18432, 1, 1, 1] 0: NONE cache_r_l0 + // 4: [ 0, 1, 1, 1] VIEW cache_r_l0 (reshaped) (view)#4 + // [ 18432, 1, 1, 1] 0: RESHAPE cache_r_l0 (reshaped)#3 + // 5: [ 0, 1, 1, 1] SCALE cache_r_l0 (reshaped) (view) (view)#5 + // [ 0, 1, 1, 1] 0: VIEW cache_r_l0 (reshaped) (view)#4 + // 6: [ 1, 1, 1, 1] VIEW (view)#6 + // [ 1, 1, 1, 1] 0: NONE leaf_5 + // 7: [ 18432, 1, 1, 1] GET_ROWS conv_states-0#7 + // [ 18432, 1, 1, 1] 0: RESHAPE cache_r_l0 (reshaped)#3 + // [ 1, 1, 1, 1] 1: VIEW (view)#6 + // The scale is in-place which modifies cache_r_l0 (reshaped)#3 + // The translation of scale overwrites cache_r in the tensor_map, + // but we also need to overwrite the old cache_r_l0 (reshaped)#3 + auto refresh_inplace_aliases = [&](const std::shared_ptr & decoder, int inplace_node_idx, + const std::string & view_src_name) { + for (int node_idx = 0; node_idx < inplace_node_idx; node_idx++) { + if (decoder->is_view_like_alias_of(node_idx, view_src_name)) { + translate_node(decoder, node_idx); + } + } + }; + + auto node_visitor = [&](const std::shared_ptr & decoder, int node_idx) { + auto converted_outputs = translate_node(decoder, node_idx); + if (converted_outputs.empty()) { + return; + } + const auto inplace_src = decoder->get_inplace_op_src(node_idx); + if (inplace_src.empty()) { + return; + } + if (converted_outputs[0].get_node_shared_ptr() != nullptr) { + (*tensor_map)[inplace_src] = converted_outputs[0]; + } + refresh_inplace_aliases(decoder, node_idx, inplace_src); }; if (!m_naive) { @@ -228,6 +389,46 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo results.push_back(result); } + // Debug-only hook: GGML_OPENVINO_DEBUG_NODE=,,... adds extra + // Result nodes for arbitrary intermediate tensors (looked up by name in + // tensor_map), on top of the real model outputs above. These debug + // Results are deliberately NOT added to ggml_decoder's model outputs, so + // the caller (ov_graph_compute_dynamic in utils.cpp) will not bind them + // to any ggml tensor buffer -- OpenVINO allocates its own tensor for + // them. This avoids the risk of reading a ggml buffer that has since + // been overwritten by a later in-place op (ggml aggressively reuses + // buffers), which can happen if trying to inspect an intermediate value + // via GGML_OPENVINO_DEBUG_OUTPUT by hacking it into a real output. + // + // tensor_map keys are usually the plain ggml tensor name (e.g. "embd"), + // but tensors that are recomputed multiple times in the same cgraph + // (GGML_TENSOR_FLAG_COMPUTE) are disambiguated with a "#" suffix + // (e.g. "cache_k_l0#4853", see get_tensor_ov_name()) which is not + // predictable ahead of time. To keep the env var usable, a requested + // name is matched either exactly, or as the "name" part before "#" of a + // suffixed key (first match wins; ambiguous requests should include the + // full "name#hash" form seen in a previous run's log/dump). + if (const char * debug_nodes = ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + std::stringstream ss(debug_nodes); + std::string name; + while (std::getline(ss, name, ',')) { + auto it = tensor_map->find(name); + if (it == tensor_map->end()) { + it = std::find_if(tensor_map->begin(), tensor_map->end(), [&](const auto & entry) { + return entry.first.compare(0, name.size(), name) == 0 && entry.first.size() > name.size() && + entry.first[name.size()] == '#'; + }); + } + if (it == tensor_map->end()) { + GGML_LOG_WARN("GGML_OPENVINO_DEBUG_NODE: node '%s' not found in tensor map, skipping\n", name.c_str()); + continue; + } + auto result = std::make_shared(it->second); + result->set_friendly_name("__debug_" + it->first); + results.push_back(result); + } + } + ov::ParameterVector used_params; for (const auto & param : params) { if (!param->output(0).get_target_inputs().empty()) { @@ -240,42 +441,94 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo resulting_model = std::make_shared(results, used_params); apply_transformations(resulting_model); + + // Set WeightlessCacheAttribute on large constants to avoid unnecessary memory copies + // in the NPUW plugin. Without this attribute, NPUW's LazyTensor constructor + // (lazy_tensor.cpp, op::Const::Const) will memcpy every constant "in case export + // occurs", doubling memory usage per compile_model call. + // + // The bin_offset field serves as a unique key (not a real file offset) — this is + // the same convention the GPU plugin uses for non-IR models (see + // Plugin::set_weightless_cache_attributes in intel_gpu/src/plugin/plugin.cpp). + // Each constant must have a distinct bin_offset, otherwise GPU's weightless cache + // import will map multiple constants to the same data. + // + // Small constants (< 16 elements) are excluded since they may be introduced by + // optimization patterns and the overhead is negligible. + // + // Note: use shape_size() rather than byte_size()/element_type().size() - GatherMatmul's default + // bias is a Constant(element::dynamic, Shape{0}), whose element_type().size() is 0 and would + // divide by zero. + size_t offset = 0; + for (auto & node : resulting_model->get_ordered_ops()) { + if (auto cnst = ov::as_type_ptr(node); cnst && ov::shape_size(cnst->get_shape()) >= 16) { + auto & rt_info = cnst->get_rt_info(); + if (rt_info.find(ov::WeightlessCacheAttribute::get_type_info_static()) == rt_info.end()) { + rt_info[ov::WeightlessCacheAttribute::get_type_info_static()] = + ov::WeightlessCacheAttribute(cnst->get_byte_size(), offset++, cnst->get_element_type()); + } + } + } return resulting_model; } std::shared_ptr TranslateSession::apply_transformations(std::shared_ptr model) { - auto ggml_model_decoder = std::dynamic_pointer_cast(m_input_model)->get_model_decoder(); + auto ggml_model_decoder = ov::as_type_ptr(m_input_model)->get_model_decoder(); { ov::pass::Manager manager; manager.set_per_pass_validation(true); manager.register_pass(); + // Marks the Convert/Subtract/Multiply nodes of our GatherMatmul dequantization chain + // (make_int4_weights/make_int8_weights, for_gather_matmul=true) with disable_constant_folding, + // so it survives ConstantFolding regardless of whether the target plugin's own + // is_decompression_multiply() recognizes GatherMatmul as a valid consumer. + manager.register_pass( + std::vector{ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}); + manager.register_pass(); + manager.register_pass(); + + // MOECompressed has no CPU plugin implementation, so enable it by default only on GPU. + // GGML_OPENVINO_MOE_OP=0 keeps the unfused GatherMatmul path for fallback/debugging. + if (ggml_openvino_is_gpu() && + ggml_openvino_getenv_int("GGML_OPENVINO_MOE_OP", 1) != 0) { + manager.register_pass(); + manager.register_pass(); + // Same fusion for models whose gate and up projections share one fused expert + // weight (gemma-4). It only matches that shape and only when the experts carry an + // integer zero point, so it is a no-op on the separate-gate/up models above. + manager.register_pass(); + } + + // Workaround for an OpenVINO GPU-plugin defect: an eltwise op whose operands differ in + // rank is computed wrongly once the plugin fuses it as a post-op into `rms`. gemma-4 + // dense under stateful execution adds a rank-3 residual to the rank-4 norm output and + // decodes garbage on GPU while CPU is correct. Equalising the ranks is a no-op for + // NUMPY broadcasting and makes the fused path correct. + // Remove once the plugin guards that fusion. Opt out with + // GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN=1. + if (ggml_openvino_is_gpu() && !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN")) { + manager.register_pass(); + } if (ggml_model_decoder->is_stateful()) { const auto kv_param_res_names = ggml_model_decoder->get_kv_param_res_names(); const auto kv_param_res_pairs = get_kv_param_res_pairs(model, kv_param_res_names); manager.register_pass(kv_param_res_pairs); + // Must run after MakeStateful, which is what creates the ReadValue/Assign pairs. + if (!ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT")) { + manager.register_pass(); + } } if (ggml_model_decoder->is_static()) { - manager.register_pass(); manager.register_pass(); } manager.run_passes(model); if (ggml_model_decoder->is_stateful()) { - auto output_names = ggml_model_decoder->get_model_output_names(); - std::map model_output_indexes; - for (size_t i=0; iget_output_size(); i++) { - auto output_friendly_name = model->output(i).get_node_shared_ptr()->get_friendly_name(); - auto output_id = model_output_indexes[output_friendly_name]; + for (size_t i = 0; i < model->get_output_size(); i++) { auto model_output_shape = model->output(i).get_partial_shape(); - auto decoder_output_shape = ggml_model_decoder->get_output_shape(output_id); - if (model_output_shape.rank().is_static() && decoder_output_shape.rank().is_static() - && model_output_shape.rank().get_length() + 1 == decoder_output_shape.rank().get_length() - && decoder_output_shape[0].is_static() && decoder_output_shape[0].get_length() == 1) { + if (model_output_shape.rank().is_static() && model_output_shape.rank().get_length() == 3) { ppp.output(i).postprocess().custom([](const ov::Output& node) { auto axes = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {0}); return std::make_shared(node, axes); diff --git a/ggml/src/ggml-openvino/openvino/translate_session.h b/ggml/src/ggml-openvino/openvino/translate_session.h index 56a14ae..675e632 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.h +++ b/ggml/src/ggml-openvino/openvino/translate_session.h @@ -9,16 +9,17 @@ namespace ggml { class TranslateSession { public: - TranslateSession(const frontend::InputModel::Ptr& input_model, - const std::unordered_map& translator_map, bool naive = false); + TranslateSession(const frontend::InputModel::Ptr & input_model, + const std::unordered_map & translator_map, + bool naive = false); std::shared_ptr get_converted_model(); - std::shared_ptr translate_graph(const frontend::InputModel::Ptr& input_model); + std::shared_ptr translate_graph(const frontend::InputModel::Ptr & input_model); private: std::shared_ptr apply_transformations(std::shared_ptr model); const frontend::InputModel::Ptr m_input_model; - const std::unordered_map& m_translator_map; + const std::unordered_map & m_translator_map; std::shared_ptr m_ov_model; bool m_naive; }; diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 65356a5..40ba938 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -1,7 +1,6 @@ #include "utils.h" -#include "ggml-impl.h" - +#include #include #include #include @@ -13,8 +12,11 @@ #include #include #include +#include #include #include +#include +#include #include #include #include @@ -24,13 +26,6 @@ namespace ov { namespace frontend { namespace ggml { -std::string getCurrentTime() { - std::time_t now = std::time(nullptr); - char buf[100]; - std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now)); - return buf; -} - void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs) { auto input_size = context.get_input_size(); FRONT_END_OP_CONVERSION_CHECK(input_size >= min_inputs, "Got less inputs than expected"); @@ -68,17 +63,17 @@ OutputVector rename_outputs_with_suffix(const OutputVector & outputs, const std: name += "_"; name += suffix; node->set_friendly_name(name); + // Uncomment to dump every node's inferred shape (used to hunt down dynamic dims on NPU). // std::cout << name << " " << output.get_partial_shape() << std::endl; } return outputs; } namespace { -ov::Output rope_yarn_ramp_mix(int n_dims, const float corr_dims[2], float ext_factor) { - int half_n_dims = n_dims / 2; - std::vector dim_ids_vec(half_n_dims); - std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0); - auto dim_ids = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, (size_t) half_n_dims}, dim_ids_vec); +ov::Output rope_yarn_ramp_mix(int num_elements, const float corr_dims[2], float ext_factor) { + std::vector dim_ids_vec(num_elements); + std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0.0f); + auto dim_ids = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, (size_t) num_elements}, dim_ids_vec); auto corr_low = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[0]}); auto corr_high = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[1]}); auto denom = std::make_shared( @@ -87,8 +82,11 @@ ov::Output rope_yarn_ramp_mix(int n_dims, const float corr_dims[2], fl auto ramp_y = std::make_shared(std::make_shared(dim_ids, corr_low), denom); auto ramp_clamped = std::make_shared(ramp_y, 0.0f, 1.0f); + // rope_yarn_ramp returns (1 - clamp(y)), so invert before scaling + auto one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f}); + auto ramp_inverted = std::make_shared(one, ramp_clamped); auto ext_factor_node = ov::op::v0::Constant::create(ov::element::f32, Shape{}, {ext_factor}); - auto ramp_mix = std::make_shared(ramp_clamped, ext_factor_node); + auto ramp_mix = std::make_shared(ramp_inverted, ext_factor_node); return ramp_mix; } @@ -115,13 +113,32 @@ void ggml_rope_yarn_corr_dims(int n_dims, std::pair, ov::Output> make_sin_cos(int32_t * rope_params, std::shared_ptr inp_pos, std::shared_ptr rope_freqs_weight, - bool stateful) { + int mode, + bool stateful, + int64_t head_dim) { + constexpr int TYPE_IMROPE = 2; + constexpr int TYPE_VISION = 3; + constexpr int TYPE_MROPE = 4; + + const bool is_imrope = (mode == TYPE_IMROPE); + const bool is_vision = (mode == TYPE_VISION); + const bool is_mrope = (mode == TYPE_MROPE); + const bool is_multi = is_imrope || is_vision || is_mrope; + if (stateful) { - inp_pos = std::make_shared(inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + inp_pos = + std::make_shared(inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); inp_pos = std::make_shared(inp_pos, ov::element::f32); auto pos_perm = std::make_shared(ov::element::i64, ov::Shape{3}, std::vector{2, 1, 0}); inp_pos = std::make_shared(inp_pos, pos_perm); + } else if (is_multi) { + inp_pos = std::make_shared(inp_pos, ov::element::f32); + auto pos_shape = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, {0, 0, 0, 4, -1}); + inp_pos = std::make_shared(inp_pos, pos_shape, true); + auto pos_transpose_shape = + std::make_shared(ov::element::i64, ov::Shape{5}, std::vector{0, 1, 2, 4, 3}); + inp_pos = std::make_shared(inp_pos, pos_transpose_shape); } else { inp_pos = std::make_shared(inp_pos, ov::element::f32); auto pos_perm = @@ -136,6 +153,7 @@ std::pair, ov::Output> make_sin_cos(int32_t * rope_params float beta_fast; float beta_slow; const int n_dims = rope_params[1]; + const size_t n_dims_half = n_dims >> 1; const int n_ctx_orig = rope_params[4]; memcpy(&freq_base, rope_params + 5, sizeof(float)); memcpy(&freq_scale, rope_params + 6, sizeof(float)); @@ -146,81 +164,824 @@ std::pair, ov::Output> make_sin_cos(int32_t * rope_params const float theta_scale = powf(freq_base, -2.0f / n_dims); - float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); - - std::vector factor(n_dims / 2); - factor[0] = 1.0f; - for (size_t i = 1; i < factor.size(); i++) { - factor[i] = theta_scale * factor[i - 1]; - } - - Output freq_factors; - if (stateful) { - freq_factors = - std::make_shared(ov::element::f32, ov::Shape{1, 1, factor.size()}, factor); - } else { - freq_factors = - std::make_shared(ov::element::f32, ov::Shape{1, 1, 1, factor.size()}, factor); - } - if (rope_freqs_weight) { - freq_factors = std::make_shared(freq_factors, rope_freqs_weight); - } - - auto theta_extrap = std::make_shared(freq_factors, inp_pos); - auto theta_interp = std::make_shared( - theta_extrap, ov::op::v0::Constant::create(ov::element::f32, {1}, {freq_scale})); - Output theta; float mscale = attn_factor; - if (ext_factor == 0.0f) { - theta = theta_interp; + if (is_multi) { + const size_t num_pairs = is_vision ? (n_dims > 0 ? (size_t) n_dims : (size_t) (head_dim / 2)) : n_dims_half; + int sections[4]; + memcpy(sections, rope_params + 11, sizeof(int) * 4); + int sect_dims = sections[0] + sections[1] + sections[2] + sections[3]; + if (sect_dims <= 0) { + sect_dims = num_pairs; + } + int sec_w = sections[0] + sections[1]; + int sec_e = sec_w + sections[2]; + std::vector gather_indices(num_pairs); + std::vector factor(num_pairs); + for (size_t j = 0; j < num_pairs; j++) { + int sector = j % sect_dims; + int plane = 0; + if (is_imrope) { + if (sector % 3 == 1 && sector < 3 * sections[1]) { + plane = 1; + } else if (sector % 3 == 2 && sector < 3 * sections[2]) { + plane = 2; + } else if (sector % 3 == 0 && sector < 3 * sections[0]) { + plane = 0; + } else { + plane = 3; + } + factor[j] = std::pow(theta_scale, j); + } else if (is_vision) { + int p_idx = 0; + if (sector < sections[0]) { + plane = 0; + p_idx = sector; + } else if (sector < sections[0] + sections[1]) { + plane = 1; + p_idx = sector - sections[0]; + } else if (sector < sec_w + sections[2]) { + plane = 2; + p_idx = sector - sec_w; + } else { + plane = 3; + p_idx = sector - sec_e; + } + factor[j] = std::pow(theta_scale, p_idx); + } else { + if (sector >= sections[0] && sector < sec_w) { + plane = 1; + } else if (sector >= sec_w && sector < sec_e) { + plane = 2; + } else if (sector >= sec_e) { + plane = 3; + } else { + plane = 0; + } + factor[j] = std::pow(theta_scale, j); + } + gather_indices[j] = plane; + } + auto gather_indices_const = + std::make_shared(ov::element::i64, ov::Shape{num_pairs}, gather_indices); + auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {4}); + inp_pos = std::make_shared(inp_pos, gather_indices_const, gather_axis); + Output factor_node = + std::make_shared(ov::element::f32, ov::Shape{num_pairs}, factor); + if (rope_freqs_weight) { + Output rope_factors = std::make_shared( + rope_freqs_weight, + ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) num_pairs}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {rope_freqs_weight->get_output_partial_shape(0).rank().get_length() - 1})); + rope_factors = std::make_shared( + rope_factors, + ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) num_pairs}), + false); + factor_node = std::make_shared(factor_node, rope_factors); + } + auto theta_extrap = std::make_shared(inp_pos, factor_node); + auto theta_interp = std::make_shared( + theta_extrap, ov::op::v0::Constant::create(ov::element::f32, {1}, {freq_scale})); + if (ext_factor == 0.0f) { + theta = theta_interp; + } else { + float corr_dims[2]; + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + auto ramp_mix = rope_yarn_ramp_mix(num_pairs, corr_dims, ext_factor); + Output one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f}); + auto one_minus_ramp = std::make_shared(one, ramp_mix); + theta = std::make_shared( + std::make_shared(theta_interp, one_minus_ramp), + std::make_shared(theta_extrap, ramp_mix)); + mscale *= (1.0f + 0.1f * std::log(1.0f / freq_scale)); + } } else { - auto ramp_mix = rope_yarn_ramp_mix(n_dims, corr_dims, ext_factor); - Output one; + std::vector factor(n_dims_half); + Output freq_factors; + float corr_dims[2]; + ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); + factor[0] = 1.0f; + for (size_t i = 1; i < factor.size(); i++) { + factor[i] = theta_scale * factor[i - 1]; + } if (stateful) { - one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1}, {1.0f}); + freq_factors = + std::make_shared(ov::element::f32, ov::Shape{1, 1, factor.size()}, factor); } else { - one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f}); + freq_factors = + std::make_shared(ov::element::f32, ov::Shape{1, 1, 1, factor.size()}, factor); + } + if (rope_freqs_weight) { + Output rope_factors = std::make_shared( + rope_freqs_weight, + ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) n_dims_half}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {rope_freqs_weight->get_output_partial_shape(0).rank().get_length() - 1})); + if (stateful) { + rope_factors = std::make_shared( + rope_factors, + ov::op::v0::Constant::create(ov::element::i64, {3}, {(int64_t) 1, (int64_t) 1, (int64_t) n_dims_half}), + false); + } else { + rope_factors = std::make_shared( + rope_factors, + ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) 1, (int64_t) 1, (int64_t) n_dims_half}), + false); + } + freq_factors = std::make_shared(freq_factors, rope_factors); } - auto one_minus_ramp = std::make_shared(one, ramp_mix); - theta = std::make_shared(std::make_shared(theta_interp, one_minus_ramp), + auto theta_extrap = std::make_shared(freq_factors, inp_pos); + auto theta_interp = std::make_shared( + theta_extrap, ov::op::v0::Constant::create(ov::element::f32, {1}, {freq_scale})); + + if (ext_factor == 0.0f) { + theta = theta_interp; + } else { + auto ramp_mix = rope_yarn_ramp_mix(n_dims_half, corr_dims, ext_factor); + Output one; + if (stateful) { + one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1}, {1.0f}); + } else { + one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f}); + } + auto one_minus_ramp = std::make_shared(one, ramp_mix); + + theta = + std::make_shared(std::make_shared(theta_interp, one_minus_ramp), std::make_shared(theta_extrap, ramp_mix)); - mscale *= (1.0f + 0.1f * std::log(1.0f / freq_scale)); + mscale *= (1.0f + 0.1f * std::log(1.0f / freq_scale)); + } } Output cos_theta = std::make_shared(theta); Output sin_theta = std::make_shared(theta); - auto mscale_node = ov::op::v0::Constant::create(ov::element::f32, Shape{}, {mscale}); + if (mscale != 1.0f) { + auto mscale_node = ov::op::v0::Constant::create(ov::element::f32, Shape{}, {mscale}); + + cos_theta = std::make_shared(cos_theta, mscale_node); + sin_theta = std::make_shared(sin_theta, mscale_node); + } - cos_theta = std::make_shared(cos_theta, mscale_node); - sin_theta = std::make_shared(sin_theta, mscale_node); return std::make_pair(sin_theta, cos_theta); } -ov::Output process_view_input(const NodeContext & context, int input_index, int slice_len) { - // Only works for VIEW operations that slice at the lowest dimension - // If the VIEW also reshape the result, `slice_len` should be provided +ov::Output process_view_input(const NodeContext & context, int input_index, int slice_len, int axis) { + // Only works for VIEW operations that does a non-strided slice with optinal reshape on the slice result. + // The function only does the slice part, the reshape (if any) should be handled by the caller. + // Default axis is -1, which means slicing the last dimension. + // If the VIEW reshapes the result, `slice_len` should be provided auto input = context.get_input(input_index); auto * op_params = (size_t *) context.get_input_op_params(input_index); - auto src1_stride = context.get_input_stride(input_index); + auto src_stride = context.get_input_stride(input_index); - int64_t split_addr = op_params[0] / src1_stride[3]; + int64_t slice_start = op_params[0] / src_stride[3]; if (slice_len == 0) { slice_len = context.get_input_shape(input_index)[3].get_length(); } - int64_t slice_end = split_addr + slice_len; + int64_t slice_end = slice_start + slice_len; - auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {split_addr}); + auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_start}); auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_end}); auto stride = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {context.is_stateful() ? 2 : 3}); + ov::Output axes; + if (axis == -1) { + axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {context.is_stateful() ? 2 : 3}); + } else { + axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {axis}); + } auto sliced = std::make_shared(input, begin, end, stride, axes); return sliced; } +ov::Output process_view_input_new(const NodeContext & context, int input_index) { + auto input = context.get_input(input_index); + + // Check if this input has view inputs + size_t view_input_size = context.get_view_input_size(input_index); + if (view_input_size == 0) { + // No view inputs, return the input as is + return input; + } + + // If translate_view already resolved this VIEW (produced a Slice), the input + // will already have the expected shape — skip re-slicing. + // + // Two notions of "matches" are accepted per axis: + // - both dims static and equal, OR + // - both dims dynamic. + // The dynamic case matters for the MoE expert-plane views: translate_view now emits a + // DYNAMIC-token slice (so the token dim is not frozen). An all-static-only check would + // see the dynamic token dim, decide the shapes "don't match", and fall through to + // re-slice/flatten the already-resolved view (a Reshape to the full flattened + // n_expert_used*n_embd tail, which then conflicts with the single-plane input). Treat a + // dynamic-vs-dynamic axis as matching so the already-resolved view is reused as-is. + // + // A third case matters for split-model MoE fragments: translate_view resolves the + // expert-plane view against the fragment's INPUT parameter. When the graph is split + // the token axis of that parameter may already be concrete (static n_tokens) even + // though get_view_input_ov_shape() still reports it as dynamic (-1). The resolved + // view is then static [1,1,n_tokens,n_embd] while `expected` is [1,1,?,n_embd]. + // An "expected dynamic, actual static" axis is a valid concretization of the SAME + // resolved view, so treat it as matching too. Falling through to process_single_view + // here would re-slice/re-flatten the already-resolved single-plane view against the + // recorded (multi-plane) source strides and emit a constant-target Reshape whose baked + // dims no longer divide the concretized input -> "dimensions do not evenly divide". + // A fourth case matters for stateful execution: `expected` comes from ggml metadata and is + // always GGML_MAX_DIMS ranks, but stateful drops the leading size-1 batch dim, so the + // resolved view is one rank lower. Compare the common trailing dims and require the + // leading expected dims we skip to be 1. Without this the rank test below never matches on + // the stateful path and every already-resolved MoE expert-plane view is re-sliced. + auto expected_ov_shape = context.get_view_input_ov_shape(input_index, 0); + auto actual_shape = input.get_partial_shape(); + if (expected_ov_shape.rank().is_static() && actual_shape.rank().is_static() && + expected_ov_shape.rank().get_length() >= actual_shape.rank().get_length()) { + const int64_t n_actual = actual_shape.rank().get_length(); + const int64_t shift = expected_ov_shape.rank().get_length() - n_actual; + bool shapes_match = true; + for (int64_t i = 0; i < shift; ++i) { + if (!expected_ov_shape[i].is_static() || expected_ov_shape[i].get_length() != 1) { + shapes_match = false; + break; + } + } + for (int64_t i = 0; i < n_actual && shapes_match; ++i) { + const auto & exp = expected_ov_shape[i + shift]; + const bool both_dynamic = exp.is_dynamic() && actual_shape[i].is_dynamic(); + const bool both_static_equal = + exp.is_static() && actual_shape[i].is_static() && exp == actual_shape[i]; + // expected dynamic, actual static: the resolved view already carries the + // concrete size for this fragment; reuse it rather than re-materializing. + const bool expected_dyn_actual_static = exp.is_dynamic() && actual_shape[i].is_static(); + if (!both_dynamic && !both_static_equal && !expected_dyn_actual_static) { + shapes_match = false; + break; + } + } + if (shapes_match) { + return input; + } + } + + // In static mode, use Split instead of Slice for single-dimension reductions. + // This ensures NPUW's FOLD doesn't parametrize per-layer slice indices (which + // would introduce dynamic shapes). A shared Split node sits outside the repeated + // subgraph boundary; each layer receives one of its output ports. + if (context.is_static() && view_input_size == 1) { + auto view_stride_v = context.get_view_input_stride(input_index, 0); + auto view_src_stride_v = context.get_view_input_src_stride(input_index, 0); + auto view_ggml_shape = context.get_view_input_ggml_shape(input_index, 0); + auto view_src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, 0); + auto view_offset = context.get_view_input_offset(input_index, 0); + auto view_src_offset = context.get_view_input_src_offset(input_index, 0); + + size_t ndims = view_ggml_shape.size(); + std::vector diff_dims; + if (view_src_ggml_shape.size() == ndims) { + for (size_t i = 0; i < ndims; ++i) { + if (view_ggml_shape[i] != view_src_ggml_shape[i]) { + diff_dims.push_back(static_cast(i)); + } + } + } + + if (diff_dims.size() == 1) { + int split_dim = diff_dims[0]; + int64_t num_splits = static_cast(view_src_ggml_shape[split_dim]); + int64_t chunk_size = static_cast(view_ggml_shape[split_dim]); + + // Only apply when slicing exactly 1 element from a multi-element dimension + if (chunk_size == 1 && num_splits > 1) { + // Check suffix strides match (dimensions after split_dim) + bool suffix_ok = view_stride_v.size() == view_src_stride_v.size(); + if (suffix_ok) { + for (size_t i = static_cast(split_dim) + 1; i < ndims; ++i) { + if (view_stride_v[i] != view_src_stride_v[i]) { + suffix_ok = false; + break; + } + } + } + + if (suffix_ok && view_src_stride_v[split_dim] > 0) { + size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; + int64_t split_index = static_cast(relative_offset / view_src_stride_v[split_dim]); + + if (split_index >= 0 && split_index < num_splits) { + auto src_node = input.get_node_shared_ptr(); + std::string rt_key = "split_dim_" + std::to_string(split_dim); + auto & rt_info = src_node->get_rt_info(); + + if (rt_info.find(rt_key) == rt_info.end()) { + auto axis_const = + ov::op::v0::Constant::create(ov::element::i64, {}, {static_cast(split_dim)}); + auto split_node = + std::make_shared(input, axis_const, static_cast(num_splits)); + split_node->set_friendly_name(src_node->get_friendly_name() + "_split"); + rt_info[rt_key] = split_node; + } + + auto split_node = rt_info[rt_key].as>(); + return split_node->output(static_cast(split_index)); + } + } + } + } + } + + // Lambda function to process a single view operation + auto process_single_view = + [](ov::Output current, size_t view_offset, const std::vector & view_stride, + const ov::Shape & view_ggml_shape, const ov::PartialShape & view_ov_shape, const std::string & view_name, + size_t view_src_offset, const std::vector & view_src_stride, const ov::Shape & view_src_ggml_shape, + const ov::PartialShape & view_src_ov_shape, const std::string & view_src_name) -> ov::Output { + // Stateful execution drops a leading size-1 axis, so `current` can be one rank lower + // than the ggml shape metadata (view_stride/view_ggml_shape, always GGML_MAX_DIMS + // entries) assumes. Shift any axis index built from that metadata down by the + // difference before using it as an OV Slice axis. + const auto current_rank = current.get_partial_shape().rank(); + const int axis_shift = current_rank.is_static() ? + static_cast(view_stride.size()) - static_cast(current_rank.get_length()) : + 0; + + auto build_reshape_pattern = [](const ov::PartialShape & target_ov_shape, + const ov::Shape & target_ggml_shape) -> std::vector { + const size_t ndims = target_ggml_shape.size(); + std::vector reshape_pattern(ndims); + size_t dynamic_dims = 0; + + if (target_ov_shape.rank().is_static() && + target_ov_shape.rank().get_length() == static_cast(ndims)) { + for (size_t i = 0; i < ndims; ++i) { + if (target_ov_shape[i].is_static()) { + reshape_pattern[i] = target_ov_shape[i].get_length(); + } else { + reshape_pattern[i] = -1; + ++dynamic_dims; + } + } + } else { + dynamic_dims = 2; + } + + if (dynamic_dims > 1) { + for (size_t i = 0; i < ndims; ++i) { + reshape_pattern[i] = static_cast(target_ggml_shape[i]); + } + } + + return reshape_pattern; + }; + + auto build_prefix_tail_reshape_pattern = [](const ov::PartialShape & target_ov_shape, + const ov::Shape & target_ggml_shape, size_t prefix_dims, + int64_t tail_dim) -> std::vector { + std::vector reshape_pattern(prefix_dims + 1); + size_t dynamic_dims = 0; + + if (target_ov_shape.rank().is_static() && + target_ov_shape.rank().get_length() == static_cast(target_ggml_shape.size())) { + for (size_t i = 0; i < prefix_dims; ++i) { + if (target_ov_shape[i].is_static()) { + reshape_pattern[i] = target_ov_shape[i].get_length(); + } else { + reshape_pattern[i] = -1; + ++dynamic_dims; + } + } + } else { + dynamic_dims = 2; + } + + if (dynamic_dims > 1) { + for (size_t i = 0; i < prefix_dims; ++i) { + reshape_pattern[i] = static_cast(target_ggml_shape[i]); + } + } + + reshape_pattern[prefix_dims] = tail_dim; + return reshape_pattern; + }; + + bool same_stride = view_stride.size() == view_src_stride.size(); + if (same_stride) { + for (size_t i = 0; i < view_stride.size(); ++i) { + if (view_stride[i] != view_src_stride[i]) { + same_stride = false; + break; + } + } + } + + bool same_ggml_shape = view_ggml_shape.size() == view_src_ggml_shape.size(); + if (same_ggml_shape) { + for (size_t i = 0; i < view_ggml_shape.size(); ++i) { + if (view_ggml_shape[i] != view_src_ggml_shape[i]) { + same_ggml_shape = false; + break; + } + } + } + + if (same_stride && same_ggml_shape) { + return current; + } + + if (same_stride) { + const size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; + const size_t ndims = view_stride.size(); + + std::vector diff_dims; + if (view_ggml_shape.size() == ndims && view_src_ggml_shape.size() == ndims) { + for (size_t i = 0; i < ndims; ++i) { + if (view_ggml_shape[i] != view_src_ggml_shape[i]) { + diff_dims.push_back(static_cast(i)); + } + } + } + + if (diff_dims.size() == 1) { + const int slice_dim = diff_dims[0]; + const int64_t dim_size = static_cast(view_src_ggml_shape[slice_dim]); + + if (view_stride[slice_dim] > 0 && relative_offset % view_stride[slice_dim] == 0) { + const int64_t begin_val = static_cast((relative_offset / view_stride[slice_dim]) % + static_cast(dim_size)); + const int64_t end_val = begin_val + static_cast(view_ggml_shape[slice_dim]); + + if (begin_val >= 0 && end_val <= dim_size) { + auto sliced = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim - axis_shift})); + + if (view_ov_shape.is_static()) { + auto reshaped = std::make_shared( + sliced, + ov::op::v0::Constant::create(ov::element::i64, {ndims}, view_ov_shape.to_shape()), + false); + reshaped->set_friendly_name(view_name); + return reshaped; + } + + sliced->set_friendly_name(view_name); + return sliced; + } + } + + int64_t tail_src_elems = 1; + int64_t tail_dst_elems = 1; + for (size_t i = slice_dim; i < ndims; ++i) { + tail_src_elems *= static_cast(view_src_ggml_shape[i]); + tail_dst_elems *= static_cast(view_ggml_shape[i]); + } + + const size_t elem_stride = view_stride[ndims - 1]; + int64_t tail_begin = 0; + if (elem_stride > 0) { + tail_begin = + static_cast((relative_offset / elem_stride) % static_cast(tail_src_elems)); + } + const int64_t tail_end = tail_begin + tail_dst_elems; + + if (tail_begin >= 0 && tail_end <= tail_src_elems) { + std::vector flat_shape; + flat_shape.reserve(slice_dim); + for (int i = 0; i < slice_dim; ++i) { + flat_shape.push_back(static_cast(view_src_ggml_shape[i])); + } + flat_shape.push_back(tail_src_elems); + const size_t flat_ndims = flat_shape.size(); + + auto flat = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {flat_ndims}, flat_shape), false); + + auto sliced = std::make_shared( + flat, ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_begin}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_end}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim})); + + if (view_ov_shape.is_static()) { + auto reshaped = std::make_shared( + sliced, ov::op::v0::Constant::create(ov::element::i64, {ndims}, view_ov_shape.to_shape()), + false); + reshaped->set_friendly_name(view_name); + return reshaped; + } + + sliced->set_friendly_name(view_name); + return sliced; + } + } + + std::vector begin(ndims, 0); + std::vector end(ndims, 0); + std::vector step(ndims, 1); + std::vector axes(ndims, 0); + + size_t remaining_offset = relative_offset; + for (size_t i = 0; i < ndims; ++i) { + axes[i] = static_cast(i); + if (view_stride[i] > 0) { + begin[i] = static_cast(remaining_offset / view_stride[i]); + remaining_offset %= view_stride[i]; + } + end[i] = begin[i] + static_cast(view_ggml_shape[i]); + } + + bool in_bounds = view_src_ggml_shape.size() == ndims && view_ggml_shape.size() == ndims; + if (in_bounds) { + for (size_t i = 0; i < ndims; ++i) { + if (end[i] > static_cast(view_src_ggml_shape[i])) { + in_bounds = false; + break; + } + } + } + + if (in_bounds && remaining_offset == 0) { + auto sliced = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {ndims}, begin), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, end), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, step), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, axes)); + + sliced->set_friendly_name(view_name); + return sliced; + } + } else { + bool same_rank = view_stride.size() == view_src_stride.size() && + view_ggml_shape.size() == view_src_ggml_shape.size() && + view_stride.size() == view_ggml_shape.size(); + const size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; + + if (same_rank) { + const size_t ndims = view_ggml_shape.size(); + std::vector diff_dims; + for (size_t i = 0; i < ndims; ++i) { + if (view_ggml_shape[i] != view_src_ggml_shape[i]) { + diff_dims.push_back(static_cast(i)); + } + } + + if (diff_dims.size() == 1) { + const size_t slice_dim = static_cast(diff_dims[0]); + bool suffix_stride_match = true; + for (size_t i = slice_dim + 1; i < ndims; ++i) { + if (view_stride[i] != view_src_stride[i]) { + suffix_stride_match = false; + break; + } + } + + if (suffix_stride_match && view_src_stride[slice_dim] > 0 && + relative_offset % view_src_stride[slice_dim] == 0) { + const int64_t begin_val = static_cast(relative_offset / view_src_stride[slice_dim]); + const int64_t end_val = begin_val + static_cast(view_ggml_shape[slice_dim]); + const int64_t dim_size = static_cast(view_src_ggml_shape[slice_dim]); + + if (begin_val >= 0 && end_val <= dim_size) { + auto sliced = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {static_cast(slice_dim)})); + sliced->set_friendly_name(view_name); + return sliced; + } + } + } + } + + size_t view_elems = 1; + size_t src_elems = 1; + if (same_rank) { + for (size_t i = 0; i < view_ggml_shape.size(); ++i) { + view_elems *= view_ggml_shape[i]; + src_elems *= view_src_ggml_shape[i]; + } + } + + bool same_num_elements = same_rank && view_elems == src_elems; + + if (same_rank && relative_offset == 0 && same_num_elements) { + auto reshape_pattern = build_reshape_pattern(view_ov_shape, view_ggml_shape); + + auto reshaped = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {reshape_pattern.size()}, reshape_pattern), + false); + reshaped->set_friendly_name(view_name); + return reshaped; + } + + if (same_rank) { + const size_t ndims = view_ggml_shape.size(); + + // Match views that can be expressed as a regular strided slice over the + // already reconstructed source tensor, e.g. offset on one axis plus step > 1 + // on another axis. + bool is_regular_slice = view_src_ggml_shape.size() == ndims; + std::vector begin(ndims, 0); + std::vector end(ndims, 0); + std::vector step(ndims, 1); + std::vector axes(ndims, 0); + size_t remaining_offset = relative_offset; + + if (is_regular_slice) { + for (size_t i = 0; i < ndims; ++i) { + axes[i] = static_cast(i); + + if (view_src_stride[i] == 0 || view_stride[i] == 0 || + view_stride[i] % view_src_stride[i] != 0) { + is_regular_slice = false; + break; + } + + step[i] = static_cast(view_stride[i] / view_src_stride[i]); + if (step[i] <= 0) { + is_regular_slice = false; + break; + } + + begin[i] = static_cast(remaining_offset / view_src_stride[i]); + remaining_offset %= view_src_stride[i]; + + if (view_ggml_shape[i] == 0) { + end[i] = begin[i]; + continue; + } + + end[i] = begin[i] + step[i] * static_cast(view_ggml_shape[i] - 1) + 1; + + if (begin[i] < 0 || end[i] > static_cast(view_src_ggml_shape[i])) { + is_regular_slice = false; + break; + } + } + } + + if (is_regular_slice && remaining_offset == 0) { + auto sliced = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {ndims}, begin), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, end), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, step), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, axes)); + + sliced->set_friendly_name(view_name); + return sliced; + } + + const size_t elem_stride = view_src_stride.back(); + const bool aligned_offset = elem_stride > 0 && relative_offset % elem_stride == 0; + + if (aligned_offset) { + size_t suffix_start = 0; + size_t expected_stride = elem_stride; + for (int i = static_cast(ndims) - 1; i >= 0; --i) { + if (view_stride[i] != expected_stride) { + suffix_start = static_cast(i + 1); + break; + } + expected_stride *= view_ggml_shape[i]; + } + + size_t prefix_elems = 1; + size_t suffix_elems = 1; + for (size_t i = 0; i < suffix_start; ++i) { + prefix_elems *= view_ggml_shape[i]; + } + for (size_t i = suffix_start; i < ndims; ++i) { + suffix_elems *= view_ggml_shape[i]; + } + + if (prefix_elems > 0 && src_elems % prefix_elems == 0) { + const size_t src_tail_elems = src_elems / prefix_elems; + const int64_t tail_begin = static_cast(relative_offset / elem_stride); + const int64_t tail_end = tail_begin + static_cast(suffix_elems); + + if (tail_begin >= 0 && tail_end <= static_cast(src_tail_elems)) { + auto prefix_tail_pattern = build_prefix_tail_reshape_pattern( + view_ov_shape, view_ggml_shape, suffix_start, static_cast(src_tail_elems)); + + auto prefix_tail = std::make_shared( + current, + ov::op::v0::Constant::create(ov::element::i64, {prefix_tail_pattern.size()}, + prefix_tail_pattern), + false); + + ov::Output selected = prefix_tail; + if (tail_begin != 0 || tail_end != static_cast(src_tail_elems)) { + selected = std::make_shared( + prefix_tail, ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_begin}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_end}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, + {static_cast(suffix_start)})); + } + + auto reshape_pattern = build_reshape_pattern(view_ov_shape, view_ggml_shape); + auto reshaped = std::make_shared( + selected, + ov::op::v0::Constant::create(ov::element::i64, {reshape_pattern.size()}, + reshape_pattern), + false); + reshaped->set_friendly_name(view_name); + return reshaped; + } + } + } + } + + return current; + } + + (void) view_name; + (void) view_src_ov_shape; + (void) view_src_name; + + return current; + }; + + // Special case: ggml collapses VIEW-of-VIEW chains so that `view_offs` is always an + // ABSOLUTE offset from the true root allocation, regardless of how many VIEW levels + // are in between (see ggml_new_tensor_impl). `src[0]` is still the immediate op-graph + // parent though, which can be a DIFFERENT (already narrowed) VIEW with the SAME ggml + // shape as this one but a different absolute offset -- e.g. a per-layer deepstack + // slice `view_2d(embd, n_embd, n_tokens, embd->nb[1], layer*n_embd*sizeof(float))` + // whose src[0] ("embd") is itself already a zero-offset VIEW of the true root (the + // padded embedding). Chaining through "embd" here would try to re-slice an already + // 2-narrowed tensor using a root-relative offset, going out of bounds and silently + // falling back to a no-op (returning the wrong, already-resolved sibling slice). + // Detect this (same shape as the immediate src, but different absolute offset) and + // re-slice directly from the untouched root using the innermost view's absolute + // offset against the ROOT's own shape/stride instead of chaining through src[0]. + { + auto innermost_offset = context.get_view_input_offset(input_index, 0); + auto innermost_src_offset = context.get_view_input_src_offset(input_index, 0); + auto innermost_shape = context.get_view_input_ggml_shape(input_index, 0); + auto innermost_src_shape = context.get_view_input_src_ggml_shape(input_index, 0); + if (innermost_offset != innermost_src_offset && innermost_shape == innermost_src_shape) { + size_t root_view_idx = view_input_size - 1; + auto root_ggml_shape = context.get_view_input_src_ggml_shape(input_index, root_view_idx); + auto root_stride = context.get_view_input_src_stride(input_index, root_view_idx); + auto root_offset = context.get_view_input_src_offset(input_index, root_view_idx); + auto root_ov_shape = context.get_view_input_src_ov_shape(input_index, root_view_idx); + auto root_name = context.get_view_input_src_name(input_index, root_view_idx); + auto innermost_stride = context.get_view_input_stride(input_index, 0); + auto innermost_ov_shape = context.get_view_input_ov_shape(input_index, 0); + auto innermost_name = context.get_view_input_name(input_index, 0); + + return process_single_view(input, innermost_offset, innermost_stride, innermost_shape, innermost_ov_shape, + innermost_name, root_offset, root_stride, root_ggml_shape, root_ov_shape, + root_name); + } + } + + // Process views from the base tensor (last) to the current view (first) + // Start with the base tensor + ov::Output current = input; + + // Process each view in reverse order (from base to current) + for (int view_idx = view_input_size - 1; view_idx >= 0; view_idx--) { + auto view_offset = context.get_view_input_offset(input_index, view_idx); + auto view_stride = context.get_view_input_stride(input_index, view_idx); + auto view_ggml_shape = context.get_view_input_ggml_shape(input_index, view_idx); + auto view_ov_shape = context.get_view_input_ov_shape(input_index, view_idx); + auto view_name = context.get_view_input_name(input_index, view_idx); + + // print view info + // std::cout << "View " << view_idx << ": name = " << view_name << ", offset = " << view_offset << ", stride = [" + // << view_stride[0] << "," << view_stride[1] << "," << view_stride[2] << "," << view_stride[3] + // << "], ggml shape = [" << view_ggml_shape[0] << "," << view_ggml_shape[1] << "," + // << view_ggml_shape[2] << "," << view_ggml_shape[3] << "], ov shape = " << view_ov_shape << std::endl; + + auto view_src_offset = context.get_view_input_src_offset(input_index, view_idx); + auto view_src_stride = context.get_view_input_src_stride(input_index, view_idx); + auto view_src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, view_idx); + auto view_src_ov_shape = context.get_view_input_src_ov_shape(input_index, view_idx); + auto view_src_name = context.get_view_input_src_name(input_index, view_idx); + // print source view info + // std::cout << "View " << view_idx << ": source name = " << view_src_name + // << ", source offset = " << view_src_offset << ", source stride = [" << view_src_stride[0] << "," + // << view_src_stride[1] << "," << view_src_stride[2] << "," << view_src_stride[3] + // << "], source ggml shape = [" << view_src_ggml_shape[0] << "," << view_src_ggml_shape[1] << "," + // << view_src_ggml_shape[2] << "," << view_src_ggml_shape[3] + // << "], source ov shape = " << view_src_ov_shape << std::endl; + + current = process_single_view(current, view_offset, view_stride, view_ggml_shape, view_ov_shape, view_name, + view_src_offset, view_src_stride, view_src_ggml_shape, view_src_ov_shape, + view_src_name); + } + + return current; +} + } // namespace ggml } // namespace frontend } // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/utils.h b/ggml/src/ggml-openvino/openvino/utils.h index 88dcad4..effa1f8 100644 --- a/ggml/src/ggml-openvino/openvino/utils.h +++ b/ggml/src/ggml-openvino/openvino/utils.h @@ -1,45 +1,37 @@ #pragma once +#include "node_context.h" + #include #include +#include #include #include #include -#include "node_context.h" - namespace ov { namespace frontend { namespace ggml { std::string getCurrentTime(); -void dump_ov_model(std::shared_ptr model); - -void num_inputs_check(const NodeContext& context, size_t min_inputs, size_t max_inputs); +void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs); int non_cont_dim(std::vector ne, std::vector nb); -template -std::vector argsort_descend(const std::vector& v) { +template std::vector argsort_descend(const std::vector & v) { std::vector idx(v.size()); std::iota(idx.begin(), idx.end(), 0); - std::sort(idx.begin(), idx.end(), [&v](int i1, int i2) { - return v[i1] > v[i2]; - }); + std::sort(idx.begin(), idx.end(), [&v](int i1, int i2) { return v[i1] > v[i2]; }); return idx; } -template -std::vector sorted_descend(std::vector v) { - std::sort(v.begin(), v.end(), [](T a, T b) { - return a > b; - }); +template std::vector sorted_descend(std::vector v) { + std::sort(v.begin(), v.end(), [](T a, T b) { return a > b; }); return v; } -template -bool is_permuted(const std::vector& strides) { +template bool is_permuted(const std::vector & strides) { for (size_t i = 0; i < strides.size() - 1; ++i) { if (strides[i] < strides[i + 1]) { return true; @@ -48,8 +40,7 @@ bool is_permuted(const std::vector& strides) { return false; } -template -std::vector permute(const std::vector& x, const std::vector& perm) { +template std::vector permute(const std::vector & x, const std::vector & perm) { std::vector result; result.reserve(perm.size()); for (int i : perm) { @@ -58,24 +49,50 @@ std::vector permute(const std::vector& x, const std::vector& perm) { return result; } -std::shared_ptr get_dimensions(const std::shared_ptr& shape, - const std::vector& dims); -std::shared_ptr get_dimensions(const std::shared_ptr& node, const std::vector& dims); +std::shared_ptr get_dimensions(const std::shared_ptr & shape, + const std::vector & dims); +std::shared_ptr get_dimensions(const std::shared_ptr & node, const std::vector & dims); -OutputVector rename_outputs_with_suffix(const OutputVector& outputs, const std::string& suffix); +OutputVector rename_outputs_with_suffix(const OutputVector & outputs, const std::string & suffix); -std::pair, ov::Output> make_sin_cos(int32_t* rope_params, +std::pair, ov::Output> make_sin_cos(int32_t * rope_params, std::shared_ptr inp_pos, std::shared_ptr rope_freqs_weight = nullptr, - bool stateful = false); + int mode = 0, + bool stateful = false, + int64_t head_dim = 0); + +ov::Output process_view_input(const NodeContext & context, int input_index, int slice_len = 0, int axis = -1); -ov::Output process_view_input(const NodeContext& context, int input_index, int slice_len = 0); +ov::Output process_view_input_new(const NodeContext & context, int input_index); namespace op { -template -OutputVector translate_1to1_match_2_inputs(const NodeContext& context) { +template OutputVector translate_1to1_match_2_inputs(const NodeContext & context) { num_inputs_check(context, 2, 2); - auto res = std::make_shared(context.get_input(0), context.get_input(1)); + auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + + auto output_type = context.get_output_type(); + if (input_0.get_element_type() != input_1.get_element_type()) { + if (input_0.get_element_type() != ov::element::f32) { + input_0 = std::make_shared(input_0, ov::element::f32); + } + if (input_1.get_element_type() != ov::element::f32) { + input_1 = std::make_shared(input_1, ov::element::f32); + } + } + + ov::Output res = std::make_shared(input_0, input_1); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + return rename_outputs_with_suffix({res}, context.get_name()); +} + +template OutputVector translate_1to1_match_1_input(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input); return rename_outputs_with_suffix({res}, context.get_name()); } } // namespace op diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 1b553a0..f07e180 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -2,8 +2,10 @@ #include "ggml-impl.h" #include "ggml-openvino-extra.h" +#include "ggml-openvino.h" #include "ggml-openvino/ggml-decoder.h" #include "ggml.h" +#include "model-cache.h" #include "openvino/frontend.h" #include "openvino/input_model.h" @@ -14,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -25,144 +29,1000 @@ #include #include #include +#include #include #include #include +#include +#include #include #include #include -// Suppress deprecation warning for ov::Tensor::data() -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +namespace { +// For a KV cache input, return an ov::Tensor sized to n_kv (== attention_size +// for that layer) instead of the fully-allocated ctx_per_seq. Pre-conditions: +// * non-static (CPU/GPU) backend, single sequence, seq_active_start == 0 +// * ggml KV layout is a contiguous [1, 1, ctx_per_seq, n_heads_kv*head_size] +// so the first n_kv rows are the live prefix and shrinking the ctx axis +// gives a valid tensor over the same host storage +// * not an SWA layer (ring cache): once the window has wrapped the first +// n_kv rows no longer contain the live prefix +// On any unmet pre-condition returns std::nullopt; the caller falls back to +// the full-size tensor. +std::optional try_make_kv_sliced_tensor(const std::shared_ptr & ggml_decoder, + const std::string & name, + const ggml_tensor * ggml_tensor) { + static const bool kv_slice_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE"); + if (kv_slice_disabled) { + return std::nullopt; + } + if (ggml_decoder->is_static() || ggml_decoder->is_stateful()) { + return std::nullopt; + } + if (ggml_tensor->op != GGML_OP_NONE || ggml_tensor->view_src != nullptr) { + return std::nullopt; + } + const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); + if (!GgmlOvDecoder::is_kvcache(ggml_tensor, op)) { + return std::nullopt; + } -enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) { - ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context; - try { - if (getenv("GGML_OPENVINO_DUMP_CGRAPH")) { - std::string filename = "cgraph_ov.txt"; - GgmlOvDecoder::dump_cgraph(cgraph, filename); + const auto & compute_params = ggml_decoder->get_compute_params(); + if (compute_params.n_seq_active != 1 || compute_params.seq_active_start != 0) { + return std::nullopt; + } + + int layer; + if (auto layer_opt = extract_layer_from_name(name); layer_opt.has_value()) { + layer = layer_opt.value(); + } else { + return std::nullopt; + } + + const bool is_swa = ggml_decoder->is_swa_layer(layer); + if (is_swa) { + return std::nullopt; + } + const int ctx_per_seq = ggml_decoder->get_ctx_per_seq(); + const int n_kv = compute_params.attention_size; + if (ctx_per_seq <= 0 || n_kv <= 0 || n_kv >= ctx_per_seq) { + return std::nullopt; + } + + ov::Shape full_shape = GgmlOvDecoder::get_shape(ggml_tensor); + if (full_shape.size() != 4 || full_shape[0] != 1 || full_shape[1] != 1 || + static_cast(full_shape[2]) != ctx_per_seq) { + return std::nullopt; + } + + ov::Shape sliced_shape = full_shape; + sliced_shape[2] = static_cast(n_kv); + + if (ggml_openvino_buffer_is_remote(ggml_tensor)) { + auto remote_context = ggml_openvino_get_remote_context(); + auto gpu_context = remote_context->as(); + return gpu_context.create_tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); + } + + return ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); +} + +static uint64_t ggml_openvino_model_cache_extra_cfg(bool stateful, bool recurrent) { + const char * manual_gqa_env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN"); + const bool manual_gqa_enabled = manual_gqa_env != nullptr ? + ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0 : + ggml_openvino_is_gpu(); + + uint64_t extra_cfg = 1; // Graph-ordinal port names (invalidate older disk-cache blobs). + extra_cfg = extra_cfg * 131 + (stateful ? (recurrent ? 2u : 1u) : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_reduce_compile_mem_enabled() ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE") ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (manual_gqa_enabled ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2 ? 1u : 0u); + return extra_cfg; +} + +std::map> get_weight_names(ggml_cgraph * cgraph) { + std::map> names; + for (const auto & name : GgmlOvDecoder::collect_weight_names(cgraph)) { + names[name] = nullptr; + } + return names; +} + +// A conservative, exact in-process key, evaluated only on a context-local cache +// miss. Include topology, layouts, op parameters, constant extra inputs and weight +// allocation identities. Never use a sampled weight hash or a graph name alone: +// different models can have identical topology. OV buffer IDs survive address reuse. +static std::string compiled_graph_key(const ggml_cgraph * graph, + const GgmlOvDecoder & decoder, + const std::string & device, + int prefill_chunk_size = 0, + bool disk_cache = false) { + std::string key; + auto append = [&key](const auto & value) { + key.append(reinterpret_cast(&value), sizeof(value)); + }; + auto append_string = [&](const std::string & value) { + append(value.size()); + key.append(value); + }; + append_string(device); + append(decoder.is_static()); + append(decoder.is_stateful()); + append(prefill_chunk_size); + bool has_weight_buffer_id = false; + std::unordered_map ids; + std::function visit = [&](const ggml_tensor * tensor) { + if (!tensor) { + append(size_t(0)); + return; } + auto inserted = ids.emplace(tensor, ids.size() + 1); + append(inserted.first->second); + if (!inserted.second) { + return; + } + append_string(tensor->name); + append(tensor->type); + append(tensor->op); + append(tensor->flags); + append(tensor->ne); + append(tensor->nb); + append(tensor->op_params); + append(tensor->view_offs); + const auto * base = tensor->view_src ? tensor->view_src : tensor; + const bool weight = base->buffer && base->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; + append(weight); + if (weight && !disk_cache) { + const size_t buffer_id = ggml_backend_openvino_buffer_get_ctx_id(base->buffer); + has_weight_buffer_id |= buffer_id != 0; + append(buffer_id); + append(tensor->data); + } + visit(tensor->view_src); + for (const auto * src : tensor->src) { + visit(src); + } + }; + append(graph->n_nodes); + for (int i = 0; i < graph->n_nodes; ++i) { + visit(graph->nodes[i]); + } + append(graph->n_leafs); + for (int i = 0; i < graph->n_leafs; ++i) { + visit(graph->leafs[i]); + } + for (const auto & input : decoder.get_model_extra_inputs()) { + append_string(input.first); + append_string(input.second.type.get_type_name()); + append(input.second.shape.size()); + for (auto dim : input.second.shape) { + append(dim); + } + append(input.second.is_parameter); + if (!input.second.is_parameter) { + append(input.second.value); + } + } + // Without an allocation generation, pointer reuse could select stale weights. + // Such graphs still get private requests; they simply do not share compilation. + return disk_cache || has_weight_buffer_id ? key : std::string{}; +} - const auto is_static = ggml_openvino_is_npu(); +static std::string dynamic_graph_signature(const ggml_cgraph * graph, + const GgmlOvDecoder & decoder, + const ModelParams & params) { + std::string key = "dynamic-1"; + auto append = [&key](const auto & value) { + key.append(reinterpret_cast(&value), sizeof(value)); + }; + auto append_string = [&](const std::string & value) { + append(value.size()); + key.append(value); + }; + graph_key graph_id(graph, true); + append(graph_id.n_nodes); + append(graph_id.n_leaves); + append_string(graph_id.first_node_name); + append_string(graph_id.last_node_name); + for (const auto & name : graph_id.input_srcs) { + append_string(name); + } + append(params.n_rs_slots); + append(params.has_rs_rollback); + append(params.mixed_rope_params); + append(params.is_cacheless_attn); + for (int layer : params.swa_layers) { + append(layer); + } + for (const auto & [layer, heads] : params.n_heads_kv_per_layer) { + append(layer); + append(heads); + } + for (const auto & [name, input] : decoder.get_model_inputs()) { + append_string(name); + append_string(input.type.get_type_name()); + } + for (const auto & [name, tensor] : decoder.get_model_outputs()) { + append_string(name); + append(tensor->type); + } + for (const auto & [name, input] : decoder.get_model_extra_inputs()) { + append_string(name); + append_string(input.type.get_type_name()); + append(input.is_parameter); + if (!input.is_parameter) { + append(input.value); + } + } + return key; +} - GGML_ASSERT(ctx->runtime_context != nullptr); - std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context); +static bool compiled_model_matches_graph(const ov::CompiledModel & model, + const std::vector & inputs, + const std::vector & outputs, + const GgmlOvDecoder & decoder) { + if (model.inputs().size() != inputs.size() || model.outputs().size() != outputs.size()) { + return false; + } + const auto & graph_inputs = decoder.get_model_inputs(); + const auto & extra_inputs = decoder.get_model_extra_inputs(); + const auto & graph_outputs = decoder.get_model_outputs(); + for (size_t i = 0; i < inputs.size(); ++i) { + const auto & port = model.input(i); + auto graph_it = graph_inputs.find(inputs[i]); + if (graph_it != graph_inputs.end()) { + if (port.get_element_type() != graph_it->second.type || + !port.get_partial_shape().compatible(graph_it->second.shape)) { + return false; + } + continue; + } + auto extra_it = extra_inputs.find(inputs[i]); + if (extra_it == extra_inputs.end() || port.get_element_type() != extra_it->second.type || + !port.get_partial_shape().compatible(ov::PartialShape(extra_it->second.shape))) { + return false; + } + } + for (size_t i = 0; i < outputs.size(); ++i) { + auto graph_it = graph_outputs.find(outputs[i]); + if (graph_it == graph_outputs.end()) { + if (outputs[i].compare(0, 8, "__debug_") != 0) { + return false; + } + continue; + } + const auto & port = model.output(i); + if (port.get_element_type() != GgmlOvDecoder::get_ov_type(graph_it->second) || + !port.get_partial_shape().compatible(ov::PartialShape(GgmlOvDecoder::get_shape(graph_it->second)))) { + return false; + } + } + return true; +} - return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx); - } catch (const ov::Exception & e) { - GGML_LOG_ERROR("GGML OpenVINO backend ov::Exception: %s\n", e.what()); - return GGML_STATUS_FAILED; - } catch (const std::exception & e) { - GGML_LOG_ERROR("GGML OpenVINO backend std::exception: %s\n", e.what()); - return GGML_STATUS_FAILED; - } catch (...) { - GGML_LOG_ERROR("GGML OpenVINO backend unknown exception\n"); - return GGML_STATUS_FAILED; +static std::string ov_profiling_csv_field(const std::string & value) { + std::string escaped = "\""; + for (char c : value) { + escaped += c; + if (c == '"') { + escaped += c; + } + } + escaped += '"'; + return escaped; +} + +static void dump_ov_profiling_info(const ov::InferRequest & infer_request) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") < 2) { + return; } + + static std::atomic inference_index{0}; + const uint64_t index = inference_index.fetch_add(1); + const std::string path = "openvino_profile_" + std::to_string(index) + ".csv"; + std::ofstream output(path, std::ios::trunc); + if (!output.is_open()) { + GGML_LOG_WARN("ggml-openvino: failed to write profiling data to %s\n", path.c_str()); + return; + } + + output << "status,real_time_us,cpu_time_us,start_time_us,node_type,node_name,exec_type\n"; + int64_t total_real_time = 0; + int64_t total_cpu_time = 0; + size_t executed_count = 0; + for (const auto & info : infer_request.get_profiling_info()) { + const char * status = "NOT_RUN"; + if (info.status == ov::ProfilingInfo::Status::EXECUTED) { + status = "EXECUTED"; + total_real_time += info.real_time.count(); + total_cpu_time += info.cpu_time.count(); + executed_count++; + } else if (info.status == ov::ProfilingInfo::Status::OPTIMIZED_OUT) { + status = "OPTIMIZED_OUT"; + } + + output << status << ',' << info.real_time.count() << ',' << info.cpu_time.count() << ',' + << info.start_time.count() << ',' << ov_profiling_csv_field(info.node_type) << ',' + << ov_profiling_csv_field(info.node_name) << ',' << ov_profiling_csv_field(info.exec_type) << '\n'; + } + + GGML_LOG_INFO("ggml-openvino: profile %s: %zu executed nodes, %.3f ms device, %.3f ms CPU\n", path.c_str(), + executed_count, total_real_time / 1000.0, total_cpu_time / 1000.0); } -ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, - std::shared_ptr infer_request, +static void reset_single_slot_recurrent_cache(ggml_cgraph * cgraph, + const ModelParams & model_params, + const ComputeParams & compute_params) { + if (model_params.n_rs_slots != 1 || compute_params.cache_rs_reset_len == 0) { + return; + } + GGML_ASSERT(compute_params.cache_rs_reset_idx == 0 && compute_params.cache_rs_reset_len == 1); + + std::set buffers; + for (int i = 0; i < cgraph->n_nodes; i++) { + auto * node = cgraph->nodes[i]; + if (node->op == GGML_OP_SCALE && GgmlOvDecoder::is_recurrent_cache(node->view_src) && + node->view_src->ne[1] == 1 && node->view_src->buffer != nullptr) { + buffers.insert(node->view_src->buffer); + } + } + for (auto * buffer : buffers) { + ggml_backend_buffer_clear(buffer, 0); + } +} + +ov::Tensor create_ov_output_tensor(const std::shared_ptr & ggml_decoder, + const std::shared_ptr & infer_request, int output_index, const ggml_tensor * ggml_tensor) { + if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, std::string(ggml_tensor->name), ggml_tensor)) { + return *sliced; + } + auto output_type = ggml_decoder->get_ov_type(ggml_tensor); ov::Shape output_shape; + void * output_data = ggml_tensor->data; if (ggml_decoder->is_static()) { output_shape = infer_request->get_output_tensor(output_index).get_shape(); } else { - output_shape = ggml_decoder->get_shape(ggml_tensor); + // For a CPY into a padded view_src (e.g. a padded KV cache buffer), the + // OV ScatterUpdate node outputs the full view_src shape, not the CPY node's + // own (smaller) shape. Using the CPY shape here causes set_output_tensor to + // fail with a shape-incompatibility error. Use view_src's shape and data + // pointer instead so the OV tensor matches the model output exactly. + if (ggml_tensor->op == GGML_OP_CPY && ggml_tensor->view_src != nullptr && + ggml_nbytes(ggml_tensor) != ggml_nbytes(ggml_tensor->view_src)) { + output_shape = GgmlOvDecoder::get_shape(ggml_tensor->view_src); + output_data = ggml_tensor->view_src->data; + } else { + output_shape = GgmlOvDecoder::get_shape(ggml_tensor); + } } - ov::Tensor output_tensor(output_type, output_shape, ggml_tensor->data); + // The sliced path above covers eligible KV outputs. This also covers recurrent state and full-size KV fallbacks. + if (!ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS") && !ggml_decoder->is_static() && + !ggml_decoder->is_splited_model() && ggml_openvino_buffer_is_remote(ggml_tensor) && + ggml_tensor->extra != nullptr) { + auto * extra_base = static_cast(ggml_tensor->extra); + if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { + auto * tensor_extra = static_cast(extra_base); + if (tensor_extra->tensor != nullptr && tensor_extra->tensor->get_element_type() == output_type && + tensor_extra->tensor->get_shape() == output_shape) { + return *tensor_extra->tensor; + } + } + } + + ov::Tensor output_tensor(output_type, output_shape, output_data); return output_tensor; } -enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr r_ctx) { +// Rewrite ggml's KV rows into a relayout state that keeps the sequence on dim 2. +// ggml stores [seq][n_heads_kv * head_size]; the state wants [1, n_heads_kv, seq, head_size], +// a different element order, so the rows are copied instead of reinterpreted. +ov::Tensor kv_rows_to_seq_axis_2(const ov::Tensor & kv_tensor, size_t n_heads_kv) { + const size_t rows = kv_tensor.get_shape()[2]; + const size_t head_size = kv_tensor.get_shape()[3] / n_heads_kv; + const size_t elem = kv_tensor.get_element_type().size(); + const size_t head_bytes = head_size * elem; + + ov::Tensor out(kv_tensor.get_element_type(), ov::Shape{1, n_heads_kv, rows, head_size}); + const auto * src = static_cast(kv_tensor.data()); + auto * dst = static_cast(out.data()); + for (size_t s = 0; s < rows; s++) { + for (size_t h = 0; h < n_heads_kv; h++) { + memcpy(dst + (h * rows + s) * head_bytes, src + (s * n_heads_kv + h) * head_bytes, head_bytes); + } + } + return out; +} + +template void set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols, T zero_value = T{}) { + for (size_t i = 0; i < rows; ++i) { + size_t diag_col = std::min(i, cols - 1); + matrix[i * cols + diag_col] = zero_value; + } +} + +ov::Tensor make_contiguous_split_input_tensor(const struct ggml_tensor * ggml_tensor, const ov::Shape & input_shape) { + const size_t element_size = ggml_type_size(ggml_tensor->type); + const size_t block_size = ggml_blck_size(ggml_tensor->type); + + GGML_ASSERT(block_size == 1 && "non-contiguous split inputs must be plain element types"); + + const struct ggml_tensor * source_tensor = ggml_tensor->view_src != nullptr ? ggml_tensor->view_src : ggml_tensor; + const size_t source_offset = ggml_tensor->view_src != nullptr ? ggml_tensor->view_offs : 0; + + std::vector source_data(ggml_nbytes(source_tensor)); + ggml_backend_tensor_get(source_tensor, source_data.data(), 0, source_data.size()); + + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + auto * dst = static_cast(input_tensor.data()); + size_t dst_offset = 0; + + for (size_t i3 = 0; i3 < static_cast(ggml_tensor->ne[3]); ++i3) { + for (size_t i2 = 0; i2 < static_cast(ggml_tensor->ne[2]); ++i2) { + for (size_t i1 = 0; i1 < static_cast(ggml_tensor->ne[1]); ++i1) { + for (size_t i0 = 0; i0 < static_cast(ggml_tensor->ne[0]); ++i0) { + const size_t src_offset = source_offset + i3 * ggml_tensor->nb[3] + i2 * ggml_tensor->nb[2] + + i1 * ggml_tensor->nb[1] + i0 * ggml_tensor->nb[0]; + std::memcpy(dst + dst_offset, source_data.data() + src_offset, element_size); + dst_offset += element_size; + } + } + } + } + + return input_tensor; +} + +ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr & ggml_decoder, const std::string & name) { + const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name); + + if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) { + return *sliced; + } + + if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) { + auto * extra_base = static_cast(ggml_tensor->extra); + if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { + // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); + auto * tensor_extra = static_cast(extra_base); + return *tensor_extra->tensor; + } + } + + // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str()); + auto * input_data = ggml_tensor->data; + ov::Shape input_shape; + if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) { + // This case is added to make test-backend-ops work + input_shape = GgmlOvDecoder::get_shape(ggml_tensor->view_src); + } else { + input_shape = GgmlOvDecoder::get_shape(ggml_tensor); + } + + if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) { + return make_contiguous_split_input_tensor(ggml_tensor, input_shape); + } + + auto input_tensor = ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape, input_data); + return input_tensor; +} + +ov::Tensor get_ov_input_tensor(const std::shared_ptr & ggml_decoder, const std::string & param_name) { + ov::Tensor input_tensor; + auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name); + if (extra_input != ggml_decoder->get_model_extra_inputs().end()) { + input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape); + *input_tensor.data() = extra_input->second.value; + } else { + input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name); + } + return input_tensor; +} + +ov::Tensor get_ov_input_tensor_static_decode(const std::shared_ptr & ggml_decoder, + const std::string & param_name) { + // NPU decoding stage + if (ggml_decoder->get_model_extra_inputs().count(param_name)) { + return get_ov_input_tensor(ggml_decoder, param_name); + } + const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); + const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); + + if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || + GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { + // IMROPE's inp_pos holds one value per t/h/w/e plane instead of a single position; + // with a single decode token the planes are still contiguous, so a flat copy works. + const int n_planes = GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ? GgmlOvDecoder::get_inp_pos_n_planes(op) : 1; + assert(ggml_tensor->ne[0] == n_planes); + ov::Shape input_shape = {1, 1, 1, (size_t) n_planes}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + std::memcpy(input_tensor.data(), ggml_tensor->data, n_planes * ggml_type_size(ggml_tensor->type)); + return input_tensor; + } + + if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { + ov::Shape input_shape = {1, 1, 1, 1}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + int32_t inp_out_id = *((int32_t *) ggml_tensor->data); + assert(ggml_tensor->ne[0] == 1); + assert(inp_out_id == 0); + *input_tensor.data() = inp_out_id; + return input_tensor; + } + + if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { + size_t context_size = ggml_decoder->get_ctx_size(); + if (ggml_tensor->type == GGML_TYPE_F16) { + std::vector padded_data = + pad_input(ggml_tensor, 1, context_size, GGML_FP32_TO_FP16(-INFINITY)); + ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, 1, context_size}); + std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); + return input_tensor; + } + + std::vector padded_data = pad_input(ggml_tensor, 1, context_size, -INFINITY); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size}); + auto * data_ptr = input_tensor.data(); + std::copy(padded_data.begin(), padded_data.begin() + context_size, data_ptr); + return input_tensor; + } + + return get_ov_input_tensor(ggml_decoder, param_name); +} + +ov::Tensor get_ov_input_tensor_static_prefill(const std::shared_ptr & ggml_decoder, + const std::string & param_name, + int chunk_index) { + // NPU prompt processing stage + const size_t input_len = ggml_decoder->get_input_len(); + const size_t chunk_size = ggml_decoder->m_prefill_chunk_size; + const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size); + const size_t chunk_pad_size = chunk_size - chunk_valid_size; + + if (param_name == "chunk_valid_len") { + ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); + *input_tensor.data() = (int64_t) chunk_valid_size; + return input_tensor; + } + if (chunk_index > 0 && param_name == "cache_rs_reset_len") { + // The recurrent-state clear belongs to the start of the sequence. Re-applying it on every + // chunk would wipe the state accumulated by the preceding chunks, so disable it (a zero + // length makes scale.cpp's keep-mask select every slot) after the first chunk. + ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); + *input_tensor.data() = 0; + return input_tensor; + } + if (ggml_decoder->get_model_extra_inputs().count(param_name)) { + return get_ov_input_tensor(ggml_decoder, param_name); + } + const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); + const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); + + if (GgmlOvDecoder::is_inp_pos(ggml_tensor, op) && GgmlOvDecoder::get_inp_pos_n_planes(op) > 1) { + // IMROPE: inp_pos stacks n_planes (t/h/w/e) position planes, each of length + // input_len; pad every plane independently so they stay aligned to chunk_size. + const int n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op); + const size_t element_size = ggml_type_size(ggml_tensor->type); + ov::Shape input_shape = {1, 1, 1, (size_t) n_planes * chunk_size}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + for (int p = 0; p < n_planes; p++) { + const char * src = + (const char *) ggml_tensor->data + (p * input_len + chunk_index * chunk_size) * element_size; + char * dst = (char *) input_tensor.data() + p * chunk_size * element_size; + std::memcpy(dst, src, chunk_valid_size * element_size); + if (chunk_pad_size > 0) { + if (ggml_tensor->type == GGML_TYPE_I32) { + int32_t last_value = *((const int32_t *) src + chunk_valid_size - 1); + int32_t * out = (int32_t *) dst; + std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); + } else if (ggml_tensor->type == GGML_TYPE_I64) { + int64_t last_value = *((const int64_t *) src + chunk_valid_size - 1); + int64_t * out = (int64_t *) dst; + std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); + } else { + throw std::runtime_error("Unexpected tensor type for " + param_name); + } + } + } + return input_tensor; + } + + if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || + GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { + ov::Shape input_shape = {1, 1, 1, chunk_size}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + // copy the chunk_index-th chunk from ggml_tensor + size_t element_size = ggml_type_size(ggml_tensor->type); + void * input_data = (char *) ggml_tensor->data + chunk_index * chunk_size * element_size; + std::memcpy(input_tensor.data(), input_data, chunk_valid_size * element_size); + // pad the rest with last_value + 1, so that kv's of padded positions are inserted + // to the next row after the valids row in the kvcache + if (chunk_pad_size > 0) { + if (ggml_tensor->type == GGML_TYPE_I32) { + int32_t last_value = + *((int32_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); + int32_t * output_data = input_tensor.data(); + std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); + } else if (ggml_tensor->type == GGML_TYPE_I64) { + int64_t last_value = + *((int64_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); + int64_t * output_data = input_tensor.data(); + std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); + } else { + throw std::runtime_error("Unexpected tensor type for " + param_name); + } + } + return input_tensor; + } + + if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { + size_t output_len = ggml_decoder->get_compute_params().output_len; + ov::Shape input_shape = {1, 1, 1, output_len}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + if (ggml_tensor->ne[0] == 0) { + *input_tensor.data() = 0; + } else { + auto * data_addr = input_tensor.data(); + for (size_t i = 0; i < output_len; i++) { + data_addr[i] = ((int32_t *) ggml_tensor->data)[i] % chunk_size; + } + } + return input_tensor; + } + + if (GgmlOvDecoder::is_inp_scale_rows(ggml_tensor, op)) { + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, 1}); + auto * dst = input_tensor.data(); + const auto * src = static_cast(ggml_tensor->data) + chunk_index * chunk_size; + std::copy(src, src + chunk_valid_size, dst); + std::fill(dst + chunk_valid_size, dst + chunk_size, 1.0f); + return input_tensor; + } + + if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) { + const size_t n_seqs = ggml_tensor->ne[1]; + const size_t src_stride = ggml_tensor->ne[0]; + const size_t copy_len = std::min(chunk_valid_size, src_stride - chunk_index * chunk_size); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, n_seqs, chunk_size}); + auto * dst = input_tensor.data(); + std::fill(dst, dst + n_seqs * chunk_size, 0.0f); + const auto * src = static_cast(ggml_tensor->data) + chunk_index * chunk_size; + for (size_t s = 0; s < n_seqs; s++) { + std::memcpy(dst + s * chunk_size, src + s * src_stride, copy_len * sizeof(float)); + } + return input_tensor; + } + + if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { + size_t cols = ggml_tensor->ne[0]; + size_t rows = ggml_tensor->ne[1]; + size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size); + size_t context_size = ggml_decoder->get_ctx_size(); + if (ggml_tensor->type == GGML_TYPE_F16) { + const auto * ggml_data = + static_cast(ggml_tensor->data) + chunk_index * chunk_size * cols; + std::vector padded_data = pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, + context_size, GGML_FP32_TO_FP16(-INFINITY)); + set_zero_diagonal(padded_data, chunk_size, context_size, GGML_FP32_TO_FP16(0.0f)); + ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, chunk_size, context_size}); + std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); + return input_tensor; + } + + const auto * ggml_data = static_cast(ggml_tensor->data) + chunk_index * chunk_size * cols; + std::vector padded_data = + pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY); + set_zero_diagonal(padded_data, chunk_size, context_size); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, context_size}); + auto * data_ptr = input_tensor.data(); + std::copy(padded_data.begin(), padded_data.begin() + chunk_size * context_size, data_ptr); + return input_tensor; + } + + return get_ov_input_tensor(ggml_decoder, param_name); +} + +enum ggml_status naive_compute(ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config, + ov_compiled_model_cache & cache) { + if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { + return GGML_STATUS_SUCCESS; + } + + std::unique_lock compile_lock(cache.mutex); + bool naive = true; + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); + auto decoder = std::make_shared(cgraph, model_weights); + auto input_model = std::make_shared(decoder); + auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { + ov::serialize(model, "IR_naive.xml"); + } + + std::shared_ptr infer_request; + auto remote_context = ggml_openvino_get_remote_context(); + ov::AnyMap compile_config = config; + if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { + // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved + compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::PERFORMANCE; + } else { + compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::ACCURACY; + } + if (remote_context.has_value()) { + infer_request = std::make_shared( + core.compile_model(model, remote_context.value(), compile_config).create_infer_request()); + } else { + infer_request = std::make_shared( + core.compile_model(model, device, compile_config).create_infer_request()); + } + std::vector input_names; + std::vector output_names; + for (const auto & param : model->get_parameters()) { + input_names.push_back(param->get_friendly_name()); + } + for (const auto & result : model->get_results()) { + output_names.push_back(result->get_friendly_name()); + } + // Destroy the frontend graph under the compilation lock as well: it can + // still own edges into the shared weight nodes. + model.reset(); + input_model.reset(); + decoder->clear_model_weights(); + model_weights.clear(); + compile_lock.unlock(); + + for (size_t i = 0; i < input_names.size(); i++) { + const auto & param_name = input_names[i]; + auto input_tensor = get_ov_input_tensor(decoder, param_name); + infer_request->set_input_tensor(i, input_tensor); + } + + // Use get_output_tensor + memcpy instead of set_output_tensor to avoid memory overwritten + // when i/o buffer overlaps, e.g. the cgraph is a single PERMUTE + + infer_request->infer(); + + for (size_t i = 0; i < output_names.size(); i++) { + auto output_tensor = infer_request->get_output_tensor(i); + const auto & model_outputs = decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(output_names[i]); + if (model_output_it == model_outputs.end()) { + // Debug-only output added via GGML_OPENVINO_DEBUG_NODE; nothing to copy into. + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + print_output_tensor_info(output_names[i], output_tensor, output_tensor.data()); + } + continue; + } + auto * ggml_tensor = model_output_it->second; + std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size()); + } + return GGML_STATUS_SUCCESS; +} + +enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::shared_ptr & r_ctx) { auto & core = ov_singleton_core(); const auto & config = ggml_openvino_get_compile_config(); - auto device = r_ctx->device; - bool stateful = r_ctx->stateful; + const auto & device = r_ctx->device; static auto is_static = false; - if (is_naive(cgraph)) { - return naive_compute(cgraph, core, device, config); - } + static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); auto start_time = ggml_time_us(); + // is_model_splitted is O(n_nodes^2) plus a create_weight_nodes scan and takes ~20 ms + // on a Llama-1B decode graph. It is called once per graph_compute invocation but the + // graph shape is identical across all decode steps, so memoize by graph_key: compute + // graph_key first (a few hundred us), and if the same key is already in decoder_cache + // we know the graph is not splitted (only not-splitted graphs get inserted there). + const int64_t cache_key_start_time = ggml_time_us(); + graph_key key(cgraph); + const int64_t cache_key_compute_time = ggml_time_us() - cache_key_start_time; + int64_t cache_lookup_time = 0; + bool key_seen = false; + if (!cache_disabled) { + const int64_t cache_lookup_start_time = ggml_time_us(); + std::lock_guard map_lock(r_ctx->ctx_mutex); + key_seen = r_ctx->decoder_cache.find(key) != r_ctx->decoder_cache.end(); + cache_lookup_time += ggml_time_us() - cache_lookup_start_time; + } + + const int64_t model_split_check_start_time = ggml_time_us(); + bool model_is_splitted = key_seen ? false : is_model_splitted(cgraph); + const int64_t model_split_check_time = ggml_time_us() - model_split_check_start_time; + + if (ggml_openvino_model_cache_only() && (model_is_splitted || is_naive(cgraph))) { + GGML_LOG_ERROR("ggml-openvino: cache-only mode requires a complete model graph\n"); + return GGML_STATUS_FAILED; + } + if (is_naive(cgraph)) { + if (!model_is_splitted) { + return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache); + } + } + std::shared_ptr ggml_decoder; std::shared_ptr infer_request; ModelParams m_params; ComputeParams c_params; + const int64_t graph_param_compute_start_time = ggml_time_us(); std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); - - graph_key key(cgraph); - bool cache_hit; + const int64_t graph_param_compute_time = ggml_time_us() - graph_param_compute_start_time; + + const bool cache_enabled = !model_is_splitted && !cache_disabled; + const bool stateful_recurrent = r_ctx->stateful && m_params.state_size >= 0; + const bool stateful_kv_only = r_ctx->stateful && !stateful_recurrent; + if (r_ctx->stateful && + (m_params.n_seq > 1 || c_params.n_seq_active > 1 || m_params.n_rs_slots > 1 || m_params.has_rs_rollback)) { + GGML_LOG_ERROR("OpenVINO stateful execution requires a single slot without recurrent rollback. Use -np 1.\n"); + return GGML_STATUS_FAILED; + } + if (stateful_recurrent && !cache_enabled) { + GGML_LOG_ERROR("OpenVINO recurrent stateful execution requires an unsplit graph with model caching enabled.\n"); + return GGML_STATUS_FAILED; + } + bool cache_hit = false; int64_t decoder_end_time; int64_t conversion_end_time; int64_t compile_end_time; int64_t infer_end_time; + int64_t ov_raw_infer_start; { - std::lock_guard lock(r_ctx->ov_compute_mutex); + std::shared_ptr entry; + ModelParams old_m_params; - auto it = r_ctx->decoder_cache.find(key); + const int64_t cache_lookup_start_time = ggml_time_us(); + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + auto it = r_ctx->decoder_cache.find(key); + cache_hit = it != r_ctx->decoder_cache.end(); + if (cache_hit) { + entry = it->second; + } else { + r_ctx->clear_caches_locked(); + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + r_ctx->decoder_cache[key] = entry; + } + } else { + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + cache_hit = false; + } + cache_lookup_time += ggml_time_us() - cache_lookup_start_time; + + std::lock_guard lock(*(entry->mutex)); + cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0; - cache_hit = it != r_ctx->decoder_cache.end(); - ModelParams old_m_params; if (cache_hit) { - ggml_decoder = it->second; + ggml_decoder = entry->ptr; old_m_params = ggml_decoder->get_model_params(); - cache_hit = old_m_params.can_reuse_dynamically(m_params); + if (!ggml_decoder->is_splited_model()) { + cache_hit = old_m_params.can_reuse_dynamically(m_params); + } } + std::vector ov_input_names; + std::vector ov_output_names; + if (cache_hit) { std::map> model_weights; ggml_decoder->set_compute_params(c_params); ggml_decoder->set_model_params(m_params); - if (old_m_params.kv_buffer_changed(m_params)) { + if (old_m_params.kv_buffer_changed(m_params) || !ggml_decoder->is_bound_to(cgraph)) { ggml_decoder->update_io(cgraph); } ggml_decoder->add_extra_inputs(); - infer_request = r_ctx->infer_request_cache.at(key); + { + std::lock_guard map_lock(r_ctx->ctx_mutex); + infer_request = r_ctx->infer_request_cache.at(key); + ov_input_names = r_ctx->ov_input_names_cache.at(key); + ov_output_names = r_ctx->ov_output_names_cache.at(key); + } - if (stateful) { + if (stateful_kv_only) { const auto * inp_pos = get_inp_pos_tensor(cgraph); int32_t * pos_data = (int32_t *) inp_pos->data; - auto pos_shape = ggml_decoder->get_shape(inp_pos); + auto pos_shape = GgmlOvDecoder::get_shape(inp_pos); if (pos_data[0] == 0) { infer_request->reset_state(); r_ctx->stateful_kv_size = pos_shape[3]; } else if (r_ctx->stateful_kv_size == static_cast(pos_data[0])) { r_ctx->stateful_kv_size += pos_shape[3]; } else { + const size_t pos_begin = static_cast(pos_data[0]); + const bool refill = pos_begin > r_ctx->stateful_kv_size; + + // A refill seeds the state from ggml's KV cache, so it needs that cache to be a + // plain prefix: cell i must hold position i. An SWA layer keeps only the last + // n_swa positions, so once a position leaves the window ggml drops it and the + // remaining cells shift - cell i stops holding position i. While every position + // is still inside the window nothing has been dropped and the refill is sound. + if (refill && !ggml_decoder->get_model_params().swa_layers.empty()) { + const int n_swa = ggml_decoder->get_compute_params().swa_window; + if (n_swa < 0 || static_cast(n_swa) < pos_begin) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: cannot resume at position %zu from a " + "state that holds %zu tokens, because the sliding-window layers keep only the last %d " + "positions. Run without GGML_OPENVINO_STATEFUL_EXECUTION.\n", + pos_begin, r_ctx->stateful_kv_size, n_swa); + return GGML_STATUS_FAILED; + } + } + + const bool relayout_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT"); + auto states = infer_request->query_state(); for (auto state : states) { auto state_tensor = state.get_state(); auto state_tensor_shape = state_tensor.get_shape(); - if (static_cast(pos_data[0]) > r_ctx->stateful_kv_size) { - std::string state_name; - try { - state_name = r_ctx->kv_state_input_name_map.at(state.get_name()); - } catch (...) { - GGML_LOG_ERROR("GGML OpenVINO backend stateful inference failed: no input found for the state\n"); + + std::string state_name; + if (auto it = r_ctx->kv_state_input_name_map.find(state.get_name()); + it != r_ctx->kv_state_input_name_map.end()) { + state_name = it->second; + } + + // Which axis holds the sequence: pass::KVStateSeqAxis moves it from dim 1 + // to dim 2. The head count is still needed below, because only a 1-head + // state stays byte-compatible with ggml's cache buffer. gemma-4 12B mixes + // 1-head full layers with 8-head sliding layers, so it is per state. + int n_heads_kv = ggml_decoder->get_model_params().n_heads_kv; + if (auto layer = extract_layer_from_name(state_name); layer.has_value()) { + n_heads_kv = ggml_decoder->get_n_heads_kv_for_layer(layer.value()); + } + const bool relayout_this_state = relayout_enabled; + const size_t seq_axis = relayout_this_state ? 2 : 1; + const size_t head_axis = seq_axis == 2 ? 1 : 2; + + if (refill) { + if (state_name.empty()) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: no input found for the state\n"); return GGML_STATUS_FAILED; } auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name); - kv_tensor.set_shape({state_tensor_shape[0], kv_tensor.get_shape()[2], - state_tensor_shape[2], state_tensor_shape[3]}); - state_tensor = kv_tensor; - state_tensor_shape = state_tensor.get_shape(); + if (relayout_this_state && n_heads_kv != 1) { + // several heads with seq on dim 2: not the same bytes as ggml's + // buffer, so the rows have to be copied into the new order + state_tensor = kv_rows_to_seq_axis_2(kv_tensor, (size_t) n_heads_kv); + } else { + ov::Shape refill_shape(4); + refill_shape[0] = state_tensor_shape[0]; + refill_shape[seq_axis] = kv_tensor.get_shape()[2]; + refill_shape[head_axis] = state_tensor_shape[head_axis]; + refill_shape[3] = state_tensor_shape[3]; + kv_tensor.set_shape(refill_shape); + state_tensor = kv_tensor; + } + state_tensor_shape = state_tensor.get_shape(); + } + // Only ever shrink to a prefix the source really has. Slicing past it used to + // surface as a bare ov::Exception from the ROI constructor. + if (state_tensor_shape[seq_axis] < pos_begin) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: state '%s' holds %zu tokens on axis " + "%zu, cannot resume at position %zu\n", + state.get_name().c_str(), state_tensor_shape[seq_axis], seq_axis, pos_begin); + return GGML_STATUS_FAILED; } ov::Coordinate begin = {0, 0, 0, 0}; - ov::Coordinate end = {state_tensor_shape[0], static_cast(pos_data[0]), - state_tensor_shape[2], state_tensor_shape[3]}; + ov::Coordinate end(state_tensor_shape.begin(), state_tensor_shape.end()); + end[seq_axis] = pos_begin; ov::Tensor new_state_tensor(state_tensor, begin, end); state.set_state(new_state_tensor); } - r_ctx->stateful_kv_size = pos_data[0] + 1; + r_ctx->stateful_kv_size = pos_begin + pos_shape[3]; } } @@ -170,122 +1030,406 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< conversion_end_time = decoder_end_time; compile_end_time = decoder_end_time; } else { - r_ctx->infer_request_cache.erase(key); + // Compilation can mutate shared weight nodes, so serialize cold paths. + // The lock is released before binding tensors or running inference. + auto shared_cache = r_ctx->compiled_cache; + std::unique_lock compile_lock(shared_cache->mutex); + auto weight_names = get_weight_names(cgraph); + ggml_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, + is_static, r_ctx->stateful, model_is_splitted); + const std::string model_cache_dir = ggml_openvino_model_cache_dir(); + const std::string exact_key = cache_enabled ? compiled_graph_key(cgraph, *ggml_decoder, device) : ""; + uint64_t model_fp = 0; + uint64_t exact_fp = 0; + std::string blob_path, manifest_path; + if (!model_cache_dir.empty() && !model_is_splitted) { + const uint64_t extra_cfg = + ggml_openvino_model_cache_extra_cfg(r_ctx->stateful, stateful_recurrent); + model_fp = + ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, 16, extra_cfg, + dynamic_graph_signature(cgraph, *ggml_decoder, m_params)); + exact_fp = + ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, 16, extra_cfg, + compiled_graph_key(cgraph, *ggml_decoder, device, 0, true)); + blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); + manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); + } + std::string shared_key = + cache_enabled ? (model_fp ? "dynamic:" + std::to_string(model_fp) : exact_key) : ""; + ov::CompiledModel shared_model; + bool imported = false; + auto shared_it = shared_cache->graphs.find(shared_key); + if (!shared_key.empty() && shared_it != shared_cache->graphs.end() && + (model_fp == 0 || compiled_model_matches_graph(shared_it->second.decode, shared_it->second.input_names, + shared_it->second.output_names, *ggml_decoder))) { + shared_model = shared_it->second.decode; + infer_request = std::make_shared(shared_model.create_infer_request()); + ov_input_names = shared_it->second.input_names; + ov_output_names = shared_it->second.output_names; + imported = true; + GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (dynamic)\n"); + } else if (model_fp && shared_it != shared_cache->graphs.end()) { + shared_key = exact_key; + shared_it = shared_cache->graphs.find(shared_key); + if (shared_it != shared_cache->graphs.end()) { + shared_model = shared_it->second.decode; + infer_request = std::make_shared(shared_model.create_infer_request()); + ov_input_names = shared_it->second.input_names; + ov_output_names = shared_it->second.output_names; + imported = true; + } + model_fp = exact_fp; + blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); + manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); + } + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache.erase(key); + } + + // Import standalone blobs with weights before graph conversion and compilation. + // Split models use the plugin cache instead. The decoder still maps graph I/O. + ov::AnyMap mc_config = config; + if (!model_cache_dir.empty()) { + mc_config.erase("CACHE_DIR"); + mc_config[ov::cache_mode.name()] = ov::CacheMode::OPTIMIZE_SPEED; + } + if (!imported && !model_cache_dir.empty() && !model_is_splitted) { + const uint64_t preferred_fp = model_fp; + bool dynamic_incompatible = preferred_fp == exact_fp; + for (int candidate = 0; candidate < 2; ++candidate) { + if (imported || (candidate == 1 && preferred_fp == exact_fp)) { + break; + } + const uint64_t candidate_fp = candidate == 0 ? preferred_fp : exact_fp; + const std::string candidate_blob = + ggml_openvino_model_cache_blob_path(model_cache_dir, candidate_fp); + const std::string candidate_manifest = + ggml_openvino_model_cache_manifest_path(model_cache_dir, candidate_fp); + std::ifstream blob_in(candidate_blob, std::ios::binary); + std::vector saved_inputs, saved_outputs; + if (!blob_in.is_open() || + !ggml_openvino_model_cache_verify_manifest(candidate_manifest, cgraph, candidate_fp, + saved_inputs, saved_outputs)) { + continue; + } + const int64_t import_start = ggml_time_us(); + try { + ov::CompiledModel cm; + auto remote_context = ggml_openvino_get_remote_context(); + if (remote_context.has_value()) { + cm = core.import_model(blob_in, remote_context.value(), mc_config); + } else { + cm = core.import_model(blob_in, device, mc_config); + } + // CPU serialization can rename Results that share a name with a Parameter. + if (!saved_inputs.empty() || !saved_outputs.empty()) { + ov_input_names = std::move(saved_inputs); + ov_output_names = std::move(saved_outputs); + } else { + for (const auto & p : cm.inputs()) { + ov_input_names.push_back(p.get_node()->get_friendly_name()); + } + for (const auto & o : cm.outputs()) { + ov_output_names.push_back(o.get_node()->get_friendly_name()); + } + } + if (!compiled_model_matches_graph(cm, ov_input_names, ov_output_names, *ggml_decoder)) { + dynamic_incompatible |= candidate == 0; + ov_input_names.clear(); + ov_output_names.clear(); + continue; + } + infer_request = std::make_shared(cm.create_infer_request()); + shared_model = cm; + entry->ptr = ggml_decoder; + imported = true; + model_fp = candidate_fp; + blob_path = candidate_blob; + manifest_path = candidate_manifest; + if (candidate_fp == exact_fp) { + shared_key = exact_key; + shared_it = shared_cache->graphs.find(shared_key); + } + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { + GGML_LOG_INFO(" - Model cache import time: %.3f ms \n", + (ggml_time_us() - import_start) / 1000.0); + } + GGML_LOG_INFO("ggml-openvino: model cache HIT %s\n", candidate_blob.c_str()); + } catch (const std::exception & e) { + GGML_LOG_WARN("ggml-openvino: model cache import failed: %s\n", e.what()); + dynamic_incompatible |= candidate == 0; + imported = false; + ov_input_names.clear(); + ov_output_names.clear(); + infer_request.reset(); + shared_model = {}; + } + } + if (!imported && dynamic_incompatible) { + model_fp = exact_fp; + shared_key = exact_key; + shared_it = shared_cache->graphs.find(shared_key); + blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); + manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); + } + } std::shared_ptr model; - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); + if (!imported && ggml_openvino_model_cache_only()) { + GGML_LOG_ERROR( + "ggml-openvino: missing or incompatible compiled model: %s; run once without " + "GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY\n", + blob_path.c_str()); + return GGML_STATUS_FAILED; + } + if (!imported && ggml_openvino_weight_buffers_released()) { + GGML_LOG_ERROR("ggml-openvino: cannot compile a new graph after releasing host weights\n"); + return GGML_STATUS_FAILED; + } + if (imported) { + decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); + } else { + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); + + ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, + r_ctx->stateful, model_is_splitted); + decoder_end_time = ggml_time_us(); + + auto input_model = std::make_shared(ggml_decoder); + model = ov::frontend::ggml::FrontEnd::convert(input_model); + ggml_decoder->clear_model_weights(); + conversion_end_time = ggml_time_us(); + + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { + char timestamped_filename[64]; + auto timestamp = (long long) ggml_time_us(); + snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp); + ov::serialize(model, timestamped_filename); + } - ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, stateful); - decoder_end_time = ggml_time_us(); + // Use the cache-stripped config when the frontend model cache is active, so + // the resulting CompiledModel can be exported and later re-imported. + const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config; + ov::CompiledModel compiled_model; + auto remote_context = ggml_openvino_get_remote_context(); + if (remote_context.has_value()) { + compiled_model = core.compile_model(model, remote_context.value(), compile_config); + } else { + compiled_model = core.compile_model(model, device, compile_config); + } + compile_end_time = ggml_time_us(); - auto input_model = std::make_shared(ggml_decoder); - model = ov::frontend::ggml::FrontEnd::convert(input_model); - ggml_decoder->clear_model_weights(); - conversion_end_time = ggml_time_us(); + for (const auto & ov_param : model->get_parameters()) { + ov_input_names.push_back(ov_param->get_friendly_name()); + } + for (const auto & ov_output : model->get_results()) { + ov_output_names.push_back(ov_output->get_friendly_name()); + } - if (getenv("GGML_OPENVINO_DUMP_IR")) { - char timestamped_filename[64]; - auto timestamp = (long long) ggml_time_us(); - snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp); - ov::serialize(model, timestamped_filename); - } + // Export to the frontend model cache for next time. Publish the blob first, + // then the manifest, so a cache hit only sees fully written artifacts. + if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) { + try { + const std::string blob_tmp = ggml_openvino_model_cache_temp_path(blob_path); + const std::string manifest_tmp = ggml_openvino_model_cache_temp_path(manifest_path); + if (ggml_openvino_model_cache_write_manifest(manifest_tmp, cgraph, model_fp, ov_input_names, + ov_output_names)) { + std::ofstream blob_out(blob_tmp, std::ios::binary | std::ios::trunc); + if (blob_out.is_open()) { + compiled_model.export_model(blob_out); + blob_out.close(); + if (blob_out.good()) { + if (std::rename(blob_tmp.c_str(), blob_path.c_str()) == 0 && + std::rename(manifest_tmp.c_str(), manifest_path.c_str()) == 0) { + GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str()); + } else { + std::remove(blob_tmp.c_str()); + std::remove(manifest_tmp.c_str()); + } + } else { + std::remove(blob_tmp.c_str()); + std::remove(manifest_tmp.c_str()); + } + } else { + std::remove(manifest_tmp.c_str()); + } + } + } catch (const std::exception & e) { + GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what()); + } + } - ov::CompiledModel compiled_model; - auto remote_context = ggml_openvino_get_remote_context(); - if (remote_context.has_value()) { - compiled_model = core.compile_model(model, remote_context.value(), config); - } else { - compiled_model = core.compile_model(model, device, config); - } - compile_end_time = ggml_time_us(); - infer_request = std::make_shared(compiled_model.create_infer_request()); - r_ctx->infer_request_cache[key] = infer_request; - r_ctx->decoder_cache[key] = ggml_decoder; + infer_request = std::make_shared(compiled_model.create_infer_request()); + shared_model = compiled_model; + entry->ptr = ggml_decoder; - std::vector ov_input_names; - std::vector ov_output_names; - for (const auto & ov_param : model->get_parameters()) { - ov_input_names.push_back(ov_param->get_friendly_name()); + } // end non-imported (compile) path + + entry->ptr = ggml_decoder; + if (!shared_key.empty() && shared_it == shared_cache->graphs.end()) { + shared_cache->graphs.emplace(shared_key, + ov_compiled_graph{shared_model, {}, ov_input_names, ov_output_names}); } - for (const auto & ov_output : model->get_results()) { - ov_output_names.push_back(ov_output->get_friendly_name()); + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache[key] = infer_request; + r_ctx->ov_input_names_cache[key] = ov_input_names; + r_ctx->ov_output_names_cache[key] = ov_output_names; } - r_ctx->ov_input_names_cache[key] = std::move(ov_input_names); - r_ctx->ov_output_names_cache[key] = std::move(ov_output_names); - if (stateful) { + if (stateful_kv_only && cache_enabled) { const auto * inp_pos = get_inp_pos_tensor(cgraph); - auto pos_shape = ggml_decoder->get_shape(inp_pos); + auto pos_shape = GgmlOvDecoder::get_shape(inp_pos); + // A freshly compiled model starts with an empty state, so it can only serve a + // sequence from its beginning. A non-zero start position means the KV history was + // built elsewhere (a restored ggml cache), which the state cannot adopt. + const int32_t pos_begin = ((int32_t *) inp_pos->data)[0]; + if (pos_begin != 0) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: a new model was compiled for a sequence that " + "starts at position %d, but its state is empty. Run without " + "GGML_OPENVINO_STATEFUL_EXECUTION.\n", + pos_begin); + return GGML_STATUS_FAILED; + } r_ctx->stateful_kv_size = pos_shape[3]; const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names(); - for (const auto& pair : kv_param_res_names) { - r_ctx->kv_state_input_name_map[pair.first+pair.second] = pair.first; + for (const auto & pair : kv_param_res_names) { + r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first; } } } - auto ov_input_names = r_ctx->ov_input_names_cache[key]; - auto ov_output_names = r_ctx->ov_output_names_cache[key]; + if (stateful_recurrent) { + const auto * inp_pos = get_inp_pos_tensor(cgraph); + const int32_t pos_begin = static_cast(inp_pos->data)[0]; + if (pos_begin == 0) { + infer_request->reset_state(); + } else if (!cache_hit || old_m_params.kv_buffer_changed(m_params) || c_params.cache_rs_reset_len > 0 || + pos_begin < 0 || static_cast(pos_begin) != r_ctx->stateful_kv_size) { + GGML_LOG_ERROR( + "OpenVINO recurrent stateful execution cannot restore or rewind a sequence. Restart at position 0 " + "or disable GGML_OPENVINO_STATEFUL_EXECUTION.\n"); + return GGML_STATUS_FAILED; + } + } else { + reset_single_slot_recurrent_cache(cgraph, m_params, c_params); + } for (size_t i = 0; i < ov_input_names.size(); i++) { - auto param_name = ov_input_names[i]; + const auto & param_name = ov_input_names[i]; auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name); infer_request->set_input_tensor(i, input_tensor); - if (getenv("GGML_OPENVINO_DEBUG_INPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { print_input_tensor_info(param_name, input_tensor); } } for (size_t i = 0; i < ov_output_names.size(); i++) { - auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names[i]); + // Debug-only outputs added via GGML_OPENVINO_DEBUG_NODE (see + // translate_session.cpp) have no corresponding ggml tensor; leave + // them unbound so OpenVINO allocates its own tensor for them, + // rather than aliasing a ggml buffer that may be overwritten by a + // later in-place op before we get to read it. + const auto & model_outputs = ggml_decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(ov_output_names[i]); + if (model_output_it == model_outputs.end()) { + continue; + } + auto * ggml_tensor = model_output_it->second; + if (ggml_nbytes(ggml_tensor) == 0) { + continue; + } auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); infer_request->set_output_tensor(i, output_tensor); } + ov_raw_infer_start = ggml_time_us(); infer_request->infer(); infer_end_time = ggml_time_us(); + if (stateful_recurrent) { + const auto * inp_pos = get_inp_pos_tensor(cgraph); + r_ctx->stateful_kv_size = + static_cast(inp_pos->data)[0] + get_inp_pos_n_tokens(cgraph, inp_pos); + } + dump_ov_profiling_info(*infer_request); - if (getenv("GGML_OPENVINO_DEBUG_OUTPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { for (size_t i = 0; i < ov_output_names.size(); i++) { const auto output_tensor = infer_request->get_output_tensor(i); print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data()); } } - if (getenv("GGML_OPENVINO_PROFILING")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); - GGML_LOG_INFO(" - Graph decoder time: %ld ms \n", (decoder_end_time - start_time) / 1000); + GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); + GGML_LOG_INFO(" - cache key compute time: %.3f ms \n", cache_key_compute_time / 1000.0); + GGML_LOG_INFO(" - model split check time: %.3f ms \n", model_split_check_time / 1000.0); + GGML_LOG_INFO(" - graph param compute time: %.3f ms \n", graph_param_compute_time / 1000.0); + GGML_LOG_INFO(" - cache lookup time: %.3f ms \n", cache_lookup_time / 1000.0); if (!cache_hit) { - GGML_LOG_INFO(" - Graph conversion time: %ld ms \n", (conversion_end_time - decoder_end_time) / 1000); - GGML_LOG_INFO(" - Graph compile time: %ld ms \n", (compile_end_time - conversion_end_time) / 1000); + GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", + (conversion_end_time - decoder_end_time) / 1000.0); + GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); } - GGML_LOG_INFO(" - Graph inference time: %ld ms \n", (infer_end_time - compile_end_time) / 1000); + GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); + GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", (infer_end_time - ov_raw_infer_start) / 1000.0); + } + } + + // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU): the plugin holds its own device copy of + // every weight after compile, so the host weight buffers can be dropped to reclaim + // RSS. Release only while holding the compilation mutex so another context cannot + // be reading host weights during conversion/compilation. Pin the shared compiled + // models across backend teardown; a later context can create its own request without + // reading the dropped pages. A new, uncached graph still fails fast above. + if (cache_hit && ggml_openvino_release_weights_enabled()) { + std::lock_guard compile_lock(r_ctx->compiled_cache->mutex); + if (!ggml_openvino_weight_buffers_released()) { + ggml_openvino_release_weight_buffers(); } } return GGML_STATUS_SUCCESS; } -enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr r_ctx) { +ov::AnyMap without_npuw(const ov::AnyMap & config) { + ov::AnyMap out; + for (const auto & kv : config) { + if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") { + continue; + } + out.insert(kv); + } + return out; +} + +enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared_ptr & r_ctx) { auto & core = ov_singleton_core(); auto get_prefill_chunk_size = [] { - const char * chunk_size_str = getenv("GGML_OPENVINO_PREFILL_CHUNK_SIZE"); - if (chunk_size_str && atoi(chunk_size_str) > 0) { - return atoi(chunk_size_str); - } - return 256; + static const int chunk_size = []() { + int env_prefill_chunk_size = ggml_openvino_getenv_int("GGML_OPENVINO_PREFILL_CHUNK_SIZE"); + return env_prefill_chunk_size > 0 ? env_prefill_chunk_size : 256; + }(); + return chunk_size; }; - static std::string device = "NPU"; + // Normally NPU, but honors GGML_OPENVINO_DEVICE so GGML_OPENVINO_FORCE_STATIC can run the + // static-shape path on CPU/GPU to isolate translation bugs from NPUW/NPU-driver issues. + static std::string device = ggml_openvino_get_device_name(); static auto is_static = true; static auto stateful = false; - static auto prefill_chunk_size = get_prefill_chunk_size(); + + auto prefill_chunk_size = get_prefill_chunk_size(); const auto & config = ggml_openvino_get_compile_config(); if (is_naive(cgraph)) { - return naive_compute(cgraph, core, device, config); + return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache); } auto start_time = ggml_time_us(); @@ -297,416 +1441,520 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrne[0]; + } graph_key key(cgraph); - bool cache_hit; + static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); + bool cache_hit = false; int64_t decoder_end_time; int64_t conversion_end_time; int64_t compile_end_time; int64_t infer_end_time; + int64_t ov_raw_infer_start; + int64_t ov_raw_infer_total = 0; - auto it = r_ctx->decoder_cache.find(key); - - cache_hit = it != r_ctx->decoder_cache.end(); + std::shared_ptr entry; ModelParams old_m_params; + + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + auto it = r_ctx->decoder_cache.find(key); + cache_hit = it != r_ctx->decoder_cache.end(); + if (cache_hit) { + entry = it->second; + } else { + r_ctx->clear_caches_locked(); + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + r_ctx->decoder_cache[key] = entry; + } + } else { + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + cache_hit = false; + } + + std::lock_guard lock(*(entry->mutex)); + cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0 && + r_ctx->infer_request_cache_prefill.count(key) != 0; + if (cache_hit) { - ggml_decoder = it->second; + ggml_decoder = entry->ptr; old_m_params = ggml_decoder->get_model_params(); cache_hit = old_m_params.can_reuse_statically(m_params); } + std::vector ov_input_names_local; + std::vector ov_output_names_local; + if (cache_hit) { std::map> model_weights; ggml_decoder->m_is_prefill = is_prefill; ggml_decoder->set_model_params(m_params); ggml_decoder->set_compute_params(c_params); - if (old_m_params.kv_buffer_changed(m_params)) { + if (old_m_params.kv_buffer_changed(m_params) || !ggml_decoder->is_bound_to(cgraph)) { ggml_decoder->update_io(cgraph); } ggml_decoder->add_extra_inputs(); - infer_request = is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key); + { + std::lock_guard map_lock(r_ctx->ctx_mutex); + infer_request = + is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key); + ov_input_names_local = r_ctx->ov_input_names_cache.at(key); + ov_output_names_local = r_ctx->ov_output_names_cache.at(key); + } decoder_end_time = ggml_time_us(); conversion_end_time = decoder_end_time; compile_end_time = decoder_end_time; } else { - r_ctx->infer_request_cache.erase(key); - r_ctx->infer_request_cache_prefill.erase(key); - - std::shared_ptr model; - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache.erase(key); + r_ctx->infer_request_cache_prefill.erase(key); + } - auto ggml_decoder_prefill = std::make_shared(cgraph, m_params, c_params, model_weights, - is_static, stateful, true, prefill_chunk_size); - auto ggml_decoder_decode = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, - stateful, false, prefill_chunk_size); - decoder_end_time = ggml_time_us(); + // Static execution shares a compiled prefill/decode pair. Each backend + // creates and retains its own requests for both phases. + auto shared_cache = r_ctx->compiled_cache; + std::unique_lock compile_lock(shared_cache->mutex); + auto weight_names = get_weight_names(cgraph); + auto local_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, is_static, + stateful, false, is_prefill, prefill_chunk_size); + const std::string shared_key = + cache_enabled ? compiled_graph_key(cgraph, *local_decoder, device, prefill_chunk_size) : ""; + auto shared_it = shared_cache->graphs.find(shared_key); + if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) { + auto & compiled = shared_it->second; + auto prefill_request = std::make_shared(compiled.prefill.create_infer_request()); + auto decode_request = no_kv_cache ? + prefill_request : + std::make_shared(compiled.decode.create_infer_request()); + ggml_decoder = local_decoder; + entry->ptr = ggml_decoder; + infer_request = is_prefill ? prefill_request : decode_request; + ov_input_names_local = compiled.input_names; + ov_output_names_local = compiled.output_names; + r_ctx->infer_request_cache_prefill[key] = prefill_request; + r_ctx->infer_request_cache[key] = decode_request; + r_ctx->ov_input_names_cache[key] = ov_input_names_local; + r_ctx->ov_output_names_cache[key] = ov_output_names_local; + decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); + GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n"); + } else { + std::shared_ptr model; + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - auto input_model_prefill = std::make_shared(ggml_decoder_prefill); - auto input_model_decode = std::make_shared(ggml_decoder_decode); + auto ggml_decoder_prefill = std::make_shared( + cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size); + auto ggml_decoder_decode = + no_kv_cache ? ggml_decoder_prefill : + std::make_shared(cgraph, m_params, c_params, model_weights, is_static, + stateful, false, false, prefill_chunk_size); + decoder_end_time = ggml_time_us(); - auto model_prefill = ov::frontend::ggml::FrontEnd::convert(input_model_prefill); - ggml_decoder_prefill->clear_model_weights(); - auto model_decode = ov::frontend::ggml::FrontEnd::convert(input_model_decode); - ggml_decoder_decode->clear_model_weights(); - conversion_end_time = ggml_time_us(); + const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR"); + const auto dump_ir_timestamp = static_cast(ggml_time_us()); + + auto build_static_model = [&core, &compile_config, dump_ir, dump_ir_timestamp]( + const std::shared_ptr & decoder, const char * tag, + std::shared_ptr & model, ov::CompiledModel & compiled_model, + std::shared_ptr & infer_request, + int64_t & local_conversion_end_time, int64_t & local_compile_end_time) { + auto input_model = std::make_shared(decoder); + model = ov::frontend::ggml::FrontEnd::convert(input_model); + decoder->clear_model_weights(); + local_conversion_end_time = ggml_time_us(); + + if (dump_ir) { + char timestamped_filename[64]; + snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag, + dump_ir_timestamp); + ov::serialize(model, timestamped_filename); + } - if (getenv("GGML_OPENVINO_DUMP_IR")) { - char timestamped_filename[64]; - auto timestamp = (long long) ggml_time_us(); - snprintf(timestamped_filename, sizeof(timestamped_filename), "model_prefill_%lld.xml", timestamp); - ov::serialize(model_prefill, timestamped_filename); - snprintf(timestamped_filename, sizeof(timestamped_filename), "model_decode_%lld.xml", timestamp); - ov::serialize(model_decode, timestamped_filename); - } + compiled_model = core.compile_model(model, device, compile_config); + infer_request = std::make_shared(compiled_model.create_infer_request()); + local_compile_end_time = ggml_time_us(); + }; + std::shared_ptr model_prefill; + std::shared_ptr model_decode; + ov::CompiledModel compiled_model_prefill; + ov::CompiledModel compiled_model_decode; + std::shared_ptr infer_request_prefill; + std::shared_ptr infer_request_decode; + int64_t prefill_conversion_end_time; + int64_t decode_conversion_end_time; + int64_t prefill_compile_end_time; + int64_t decode_compile_end_time; + build_static_model(ggml_decoder_prefill, "prefill", model_prefill, compiled_model_prefill, + infer_request_prefill, prefill_conversion_end_time, prefill_compile_end_time); + if (no_kv_cache) { + model_decode = model_prefill; + compiled_model_decode = compiled_model_prefill; + infer_request_decode = infer_request_prefill; + decode_conversion_end_time = prefill_conversion_end_time; + decode_compile_end_time = prefill_compile_end_time; + } else { + build_static_model(ggml_decoder_decode, "decode", model_decode, compiled_model_decode, + infer_request_decode, decode_conversion_end_time, decode_compile_end_time); + } + conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time); + compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time); - ov::CompiledModel compiled_model_prefill; - ov::CompiledModel compiled_model_decode; - auto remote_context = ggml_openvino_get_remote_context(); - if (remote_context.has_value()) { - compiled_model_prefill = core.compile_model(model_prefill, remote_context.value(), config); - compiled_model_decode = core.compile_model(model_decode, remote_context.value(), config); - } else { - compiled_model_prefill = core.compile_model(model_prefill, device, config); - compiled_model_decode = core.compile_model(model_decode, device, config); - } + model = is_prefill ? model_prefill : model_decode; + ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode; + infer_request = is_prefill ? infer_request_prefill : infer_request_decode; + entry->ptr = ggml_decoder; - r_ctx->infer_request_cache_prefill[key] = - std::make_shared(compiled_model_prefill.create_infer_request()); - r_ctx->infer_request_cache[key] = - std::make_shared(compiled_model_decode.create_infer_request()); - compile_end_time = ggml_time_us(); + for (const auto & ov_param : model->get_parameters()) { + ov_input_names_local.push_back(ov_param->get_friendly_name()); + } + for (const auto & ov_output : model->get_results()) { + ov_output_names_local.push_back(ov_output->get_friendly_name()); + } - model = is_prefill ? model_prefill : model_decode; - ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode; - infer_request = is_prefill ? r_ctx->infer_request_cache_prefill[key] : r_ctx->infer_request_cache[key]; - r_ctx->decoder_cache[key] = ggml_decoder; + if (!shared_key.empty()) { + shared_cache->graphs.emplace( + shared_key, ov_compiled_graph{compiled_model_decode, compiled_model_prefill, ov_input_names_local, + ov_output_names_local}); + } - std::vector ov_input_names; - std::vector ov_output_names; - for (const auto & ov_param : model->get_parameters()) { - ov_input_names.push_back(ov_param->get_friendly_name()); - } - for (const auto & ov_output : model->get_results()) { - ov_output_names.push_back(ov_output->get_friendly_name()); + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache_prefill[key] = infer_request_prefill; + r_ctx->infer_request_cache[key] = infer_request_decode; + r_ctx->ov_input_names_cache[key] = ov_input_names_local; + r_ctx->ov_output_names_cache[key] = ov_output_names_local; + } } - r_ctx->ov_input_names_cache[key] = std::move(ov_input_names); - r_ctx->ov_output_names_cache[key] = std::move(ov_output_names); } - auto ov_input_names = r_ctx->ov_input_names_cache[key]; - auto ov_output_names = r_ctx->ov_output_names_cache[key]; + reset_single_slot_recurrent_cache(cgraph, m_params, c_params); if (is_prefill) { - auto inp_len = inp_pos->ne[0]; + auto inp_len = get_inp_pos_n_tokens(cgraph, inp_pos); for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) { - for (size_t i = 0; i < ov_input_names.size(); i++) { - auto param_name = ov_input_names[i]; + for (size_t i = 0; i < ov_input_names_local.size(); i++) { + const auto & param_name = ov_input_names_local[i]; auto input_tensor = get_ov_input_tensor_static_prefill(ggml_decoder, param_name, chunk_index); infer_request->set_input_tensor(i, input_tensor); - if (getenv("GGML_OPENVINO_DEBUG_INPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { const auto input_tensor = infer_request->get_input_tensor(i); print_input_tensor_info(param_name, input_tensor); } } - for (size_t i = 0; i < ov_output_names.size(); i++) { - auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names[i]); + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto & model_outputs = ggml_decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(ov_output_names_local[i]); + if (model_output_it == model_outputs.end()) { + continue; + } + auto * ggml_tensor = model_output_it->second; + if (ggml_nbytes(ggml_tensor) == 0) { + // Zero-row in-place writeback (e.g. the empty s_copy defrag remainder). The OV + // Result is the full cache, so binding it over this 0-byte buffer overflows it. + continue; + } auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); infer_request->set_output_tensor(i, output_tensor); } + ov_raw_infer_start = ggml_time_us(); infer_request->infer(); + ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start; - if (getenv("GGML_OPENVINO_DEBUG_OUTPUT")) { - for (size_t i = 0; i < ov_output_names.size(); i++) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + for (size_t i = 0; i < ov_output_names_local.size(); i++) { const auto output_tensor = infer_request->get_output_tensor(i); - print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data()); + print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); } } } infer_end_time = ggml_time_us(); } else { - for (size_t i = 0; i < ov_input_names.size(); i++) { - auto param_name = ov_input_names[i]; + for (size_t i = 0; i < ov_input_names_local.size(); i++) { + const auto & param_name = ov_input_names_local[i]; auto input_tensor = get_ov_input_tensor_static_decode(ggml_decoder, param_name); infer_request->set_input_tensor(i, input_tensor); - if (getenv("GGML_OPENVINO_DEBUG_INPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { const auto input_tensor = infer_request->get_input_tensor(i); print_input_tensor_info(param_name, input_tensor); } } - for (size_t i = 0; i < ov_output_names.size(); i++) { - auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names[i]); + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto & model_outputs = ggml_decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(ov_output_names_local[i]); + if (model_output_it == model_outputs.end()) { + continue; + } + auto * ggml_tensor = model_output_it->second; + if (ggml_nbytes(ggml_tensor) == 0) { + continue; + } auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); infer_request->set_output_tensor(i, output_tensor); } + ov_raw_infer_start = ggml_time_us(); infer_request->infer(); infer_end_time = ggml_time_us(); + ov_raw_infer_total = infer_end_time - ov_raw_infer_start; - if (getenv("GGML_OPENVINO_DEBUG_OUTPUT")) { - for (size_t i = 0; i < ov_output_names.size(); i++) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + for (size_t i = 0; i < ov_output_names_local.size(); i++) { const auto output_tensor = infer_request->get_output_tensor(i); - print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data()); + print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); } } } - if (getenv("GGML_OPENVINO_PROFILING")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); - GGML_LOG_INFO(" - Graph decoder time: %ld ms \n", (decoder_end_time - start_time) / 1000); + GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); if (!cache_hit) { - GGML_LOG_INFO(" - Graph conversion time: %ld ms \n", (conversion_end_time - decoder_end_time) / 1000); - GGML_LOG_INFO(" - Graph compile time: %ld ms \n", (compile_end_time - conversion_end_time) / 1000); + GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", (conversion_end_time - decoder_end_time) / 1000.0); + GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); } - GGML_LOG_INFO(" - Graph inference time: %ld ms \n", (infer_end_time - compile_end_time) / 1000); + GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); + GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", ov_raw_infer_total / 1000.0); } return GGML_STATUS_SUCCESS; } +} // namespace -bool is_naive(ggml_cgraph * cgraph) { - constexpr int naive_graph_size_threshold = 20; - int count = 0; - for (int i = 0; i < cgraph->n_nodes; i++) { - if (cgraph->nodes[i]->op != GGML_OP_NONE) { - count++; - } +// Nodes on the unselected branches of ggml_build_forward_select() stay in the graph but must not be +// computed. Keep them out of the OV model, or their inputs become parameters with fixed shapes. +static ggml_cgraph * get_compute_graph(ggml_cgraph * cgraph, ov_runtime_context & r_ctx) { + auto is_skipped = [](const ggml_tensor * node) { + return node->op != GGML_OP_NONE && !(node->flags & GGML_TENSOR_FLAG_COMPUTE); + }; + if (std::none_of(cgraph->nodes, cgraph->nodes + cgraph->n_nodes, is_skipped)) { + return cgraph; } - return count < naive_graph_size_threshold; + auto & compute = r_ctx.compute_graphs[cgraph]; + compute.nodes.clear(); + std::copy_if(cgraph->nodes, cgraph->nodes + cgraph->n_nodes, std::back_inserter(compute.nodes), + [&](const ggml_tensor * node) { return !is_skipped(node); }); + compute.graph = *cgraph; + compute.graph.nodes = compute.nodes.data(); + compute.graph.n_nodes = (int) compute.nodes.size(); + return &compute.graph; } -enum ggml_status naive_compute(ggml_cgraph * cgraph, - ov::Core & core, - const std::string & device, - const ov::AnyMap & config) { - if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { - return GGML_STATUS_SUCCESS; - } +// Both execution paths use two cache levels: +// 1. Reuse this backend's decoder/request via graph_key and compatibility checks. +// 2. On a local miss, look up compiled_graph_key in the shared compilation cache, +// compile if needed, then create a private request from the compiled model. +// The shared lock covers compilation and frontend cleanup, never inference. +enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) { + ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context; + try { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_CGRAPH")) { + std::string filename = "cgraph_ov.txt"; + GgmlOvDecoder::dump_cgraph(cgraph, filename); + } - bool naive = true; - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); - auto decoder = std::make_shared(cgraph, model_weights); - auto input_model = std::make_shared(decoder); - auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); - if (getenv("GGML_OPENVINO_DUMP_IR")) { - ov::serialize(model, "IR_naive.xml"); - } + const auto is_static = ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC"); - std::shared_ptr infer_request; - auto remote_context = ggml_openvino_get_remote_context(); - if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { - // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved - core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::PERFORMANCE)); - } else { - core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY)); - } - if (remote_context.has_value()) { - infer_request = std::make_shared( - core.compile_model(model, remote_context.value(), config).create_infer_request()); - } else { - infer_request = - std::make_shared(core.compile_model(model, device, config).create_infer_request()); - } + GGML_ASSERT(ctx->runtime_context != nullptr); + std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context); + std::lock_guard execution_lock(r_ctx->execution_mutex); + cgraph = get_compute_graph(cgraph, *r_ctx); - auto ov_params = model->get_parameters(); - for (size_t i = 0; i < ov_params.size(); i++) { - auto param_name = ov_params[i]->get_friendly_name(); - auto input_tensor = get_ov_input_tensor(decoder, param_name); - infer_request->set_input_tensor(i, input_tensor); + return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx); + } catch (const ov::Exception & e) { + GGML_LOG_ERROR("GGML OpenVINO backend ov::Exception: %s\n", e.what()); + return GGML_STATUS_FAILED; + } catch (const std::exception & e) { + GGML_LOG_ERROR("GGML OpenVINO backend std::exception: %s\n", e.what()); + return GGML_STATUS_FAILED; + } catch (...) { + GGML_LOG_ERROR("GGML OpenVINO backend unknown exception\n"); + return GGML_STATUS_FAILED; } +} - auto ov_results = model->get_results(); - for (size_t i = 0; i < ov_results.size(); i++) { - auto * ggml_tensor = decoder->get_model_outputs().at(ov_results[i]->get_friendly_name()); - auto output_tensor = create_ov_output_tensor(decoder, infer_request, i, ggml_tensor); - infer_request->set_output_tensor(i, output_tensor); +// Detect whether a cgraph is a split subgraph or not. +// Step 1 compares each node's recorded use_count with actual fan-out references in node->src. +// Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split. +bool is_model_splitted(ggml_cgraph * cgraph) { + static const bool fallback_enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK") != 0; + if (!fallback_enabled) { + return false; } - infer_request->infer(); - return GGML_STATUS_SUCCESS; -} - -namespace { -ov::Tensor convert_ggml_input_to_ov(std::shared_ptr ggml_decoder, const std::string & name) { - const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name); + // Backend op tests execute each node through ggml_graph_view(), which preserves the original + // graph use_counts while exposing only one node. Treat those single-node views as regular + // naive graphs so intermediate ops do not look like split-model fragments. + if (cgraph->n_nodes <= 1 && cgraph->n_leafs == 0) { + return false; + } - if (ggml_tensor->extra != nullptr) { - // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); - auto * extra_base = static_cast(ggml_tensor->extra); - if (extra_base->type != ggml_openvino_extra_base::Type::TENSOR) { - throw std::runtime_error("ggml tensor extra is not of type TENSOR for input: " + name); + // check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false. + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + int use_count = cgraph->use_counts[ggml_hash_find(&cgraph->visited_hash_set, node)]; + // TODO: this is a workround for the tests case from llama.cpp, fix should from the root cause in the future. + if ((cgraph->n_nodes <= 1 && use_count == 0) || + (cgraph->n_nodes <= 1 && node->op == GGML_OP_VIEW && use_count == 1 && node->src[0] != nullptr && + node->src[0]->op == GGML_OP_NONE)) { + return false; + } + if (cgraph->n_nodes == 1 && + (cgraph->nodes[0]->op == GGML_OP_TRANSPOSE || cgraph->nodes[0]->op == GGML_OP_PERMUTE)) { + return false; + } + int input_use_count = 0; + for (int j = 0; j < cgraph->n_nodes; j++) { + ggml_tensor * other_node = cgraph->nodes[j]; + for (int k = 0; k < GGML_MAX_SRC; k++) { + if (other_node->src[k] == node) { + input_use_count++; + } + } + } + if (use_count != input_use_count && node->op != GGML_OP_NONE) { + return true; } - auto * tensor_extra = static_cast(extra_base); - return *tensor_extra->tensor; } - - // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str()); - auto * input_data = ggml_tensor->data; - ov::Shape input_shape; - if (ggml_tensor->op == GGML_OP_VIEW) { - // This case is added to make test-backend-ops work - input_shape = ggml_decoder->get_shape(ggml_tensor->view_src); + // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check. + // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM + // use the name-only collector (no weight extraction); otherwise keep the original + // behavior of building (naive) weight nodes and take their names. + std::set model_weights; + if (ggml_openvino_reduce_compile_mem_enabled()) { + model_weights = GgmlOvDecoder::collect_weight_names(cgraph); } else { - input_shape = ggml_decoder->get_shape(ggml_tensor); + for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) { + model_weights.insert(kv.first); + } } - auto input_tensor = ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape, input_data); - return input_tensor; -} -} // namespace - -ov::Tensor get_ov_input_tensor(std::shared_ptr ggml_decoder, const std::string & param_name) { - ov::Tensor input_tensor; - if (ggml_decoder->get_model_extra_inputs().find(param_name) != ggml_decoder->get_model_extra_inputs().end()) { - input_tensor = *ggml_decoder->get_model_extra_input_values().at(param_name); - } else { - input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name); + std::set model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes); + // leaf nodes + std::set model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs); + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + ggml_tensor * src = node->src[j]; + // the src is also not the model weights, we think the model is splitted. + // the src is also not in model leafs, we think the model is splitted. + if (src != nullptr && model_nodes.find(src) == model_nodes.end() && + model_weights.find(std::string(src->name)) == model_weights.end() && !model_leafs.empty() == false && + model_leafs.find(src) == model_leafs.end()) { + if (GgmlOvDecoder::is_inp_tok(src, node)) { + return false; + } + return true; + } + } } - return input_tensor; + return false; } -ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr ggml_decoder, - const std::string & param_name) { - // NPU decoding stage - const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); - const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); - - if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || - GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { - assert(ggml_tensor->ne[0] == 1); - ov::Shape input_shape = {1, 1, 1, 1}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - if (ggml_tensor->type == GGML_TYPE_I32) { - *input_tensor.data() = *((int32_t *) ggml_tensor->data); - } else if (ggml_tensor->type == GGML_TYPE_I64) { - *input_tensor.data() = *((int64_t *) ggml_tensor->data); - } else { - throw std::runtime_error("Unexpected tensor type for " + param_name); +bool is_naive(ggml_cgraph * cgraph) { + constexpr int naive_graph_size_threshold = 20; + int count = 0; + for (int i = 0; i < cgraph->n_nodes; i++) { + if (cgraph->nodes[i]->op != GGML_OP_NONE) { + count++; } - return input_tensor; } + return count < naive_graph_size_threshold; +} - if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { - ov::Shape input_shape = {1, 1, 1, 1}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - int32_t inp_out_id = *((int32_t *) ggml_tensor->data); - assert(ggml_tensor->ne[0] == 1); - assert(inp_out_id == 0); - *input_tensor.data() = inp_out_id; - return input_tensor; +size_t checksum(const void * data, size_t size) { + const uint8_t * bytes = static_cast(data); + size_t sum = 0; + for (size_t i = 0; i < size; ++i) { + sum += (uint8_t) i; + sum += bytes[i]; } + return sum; +} - if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { - size_t context_size = ggml_decoder->get_ctx_size(); - std::vector padded_data = pad_input(ggml_tensor, 1, context_size, -INFINITY); - ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size}); - auto * data_ptr = input_tensor.data(); - std::copy(padded_data.begin(), padded_data.begin() + context_size, data_ptr); - return input_tensor; +bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path) { + if (tensor == nullptr || tensor->data == nullptr) { + return false; } - return get_ov_input_tensor(ggml_decoder, param_name); -} - -ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr ggml_decoder, - const std::string & param_name, - int chunk_index) { - // NPU prompt processing stage - const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); - const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); + std::ofstream out(file_path); + if (!out.is_open()) { + return false; + } - const size_t input_len = ggml_decoder->get_input_len(); - const size_t chunk_size = ggml_decoder->m_prefill_chunk_size; - const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size); - const size_t chunk_pad_size = chunk_size - chunk_valid_size; + const size_t n = ggml_nelements(tensor); + out << "name: " << tensor->name << ", type: " << ggml_type_name(tensor->type) << ", shape: [" << tensor->ne[0] + << ", " << tensor->ne[1] << ", " << tensor->ne[2] << ", " << tensor->ne[3] << "]" << ", elements: " << n + << ", data:" << '\n'; - if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || - GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { - ov::Shape input_shape = {1, 1, 1, chunk_size}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - // copy the chunk_index-th chunk from ggml_tensor - size_t element_size = ggml_type_size(ggml_tensor->type); - void * input_data = (char *) ggml_tensor->data + chunk_index * chunk_size * element_size; - std::memcpy(input_tensor.data(), input_data, chunk_valid_size * element_size); - // pad the rest with last_value + 1, so that kv's of padded positions are inserted - // to the next row after the valids row in the kvcache - if (chunk_pad_size > 0) { - if (ggml_tensor->type == GGML_TYPE_I32) { - int32_t last_value = - *((int32_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); - int32_t * output_data = input_tensor.data(); - std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); - } else if (ggml_tensor->type == GGML_TYPE_I64) { - int64_t last_value = - *((int64_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); - int64_t * output_data = input_tensor.data(); - std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); - } else { - throw std::runtime_error("Unexpected tensor type for " + param_name); - } + switch (tensor->type) { + case GGML_TYPE_F32: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << data[i] << '\n'; } - return input_tensor; + break; } - - if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { - size_t output_len = ggml_decoder->get_compute_params().output_len; - ov::Shape input_shape = {1, 1, 1, output_len}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - if (ggml_tensor->ne[0] == 0) { - *input_tensor.data() = 0; - } else { - auto * data_addr = input_tensor.data(); - for (size_t i = 0; i < output_len; i++) { - data_addr[i] = ((int32_t *) ggml_tensor->data)[i] % chunk_size; - } + case GGML_TYPE_F16: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << ggml_fp16_to_fp32(data[i]) << '\n'; } - return input_tensor; + break; } - - if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { - size_t cols = ggml_tensor->ne[0]; - size_t rows = ggml_tensor->ne[1]; - float * ggml_data = (float *) ggml_tensor->data + chunk_index * chunk_size * cols; - size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size); - size_t context_size = ggml_decoder->get_ctx_size(); - std::vector padded_data = - pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY); - set_zero_diagonal(padded_data, chunk_size, context_size); - ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, context_size}); - auto * data_ptr = input_tensor.data(); - std::copy(padded_data.begin(), padded_data.begin() + chunk_size * context_size, data_ptr); - return input_tensor; + case GGML_TYPE_BF16: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << ggml_bf16_to_fp32(data[i]) << '\n'; + } + break; } - - return get_ov_input_tensor(ggml_decoder, param_name); -} - -size_t checksum(const void * data, size_t size) { - const uint8_t * bytes = static_cast(data); - size_t sum = 0; - for (size_t i = 0; i < size; ++i) { - sum += (uint8_t) i; - sum += bytes[i]; + case GGML_TYPE_I32: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << data[i] << '\n'; + } + break; } - return sum; + case GGML_TYPE_I64: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << data[i] << '\n'; + } + break; + } + default: + out << "unsupported tensor type for text dump" << '\n'; + return false; + } + + return true; } void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor) { std::cout << "Input name: " << name << ", Input shape: " << tensor.get_shape() << ", Address: " << tensor.data() - << std::endl; + << '\n'; switch (tensor.get_element_type()) { case ov::element::f32: { - if (name.find("self_kq_mask") == std::string::npos) { - std::cout << *(tensor.data()) << std::endl; + if (name.find("self_kq_mask") == std::string::npos && name.find("KQ_mask") == std::string::npos) { + std::cout << *(tensor.data()) << '\n'; } else { size_t rows = tensor.get_shape()[2]; size_t cols = tensor.get_shape()[3]; - auto * data = tensor.data(); + const float * data = tensor.data(); for (size_t i = 0; i < rows; ++i) { for (size_t j = 0; j < cols; ++j) { float val = data[i * cols + j]; @@ -716,26 +1964,26 @@ void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor std::cout << std::setw(5) << val; } } - std::cout << std::endl; + std::cout << '\n'; } } break; } case ov::element::f16: - std::cout << *(tensor.data()) << std::endl; + std::cout << *(tensor.data()) << '\n'; break; case ov::element::i32: for (size_t i = 0; i < tensor.get_size(); ++i) { - std::cout << tensor.data()[i] << " "; + std::cout << tensor.data()[i] << ' '; } - std::cout << std::endl; + std::cout << '\n'; break; case ov::element::i64: for (size_t i = 0; i < tensor.get_size(); ++i) { - std::cout << tensor.data()[i] << " "; + std::cout << tensor.data()[i] << ' '; } - std::cout << std::endl; + std::cout << '\n'; break; default: break; @@ -744,7 +1992,7 @@ void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor void print_output_tensor_info(const std::string & name, const ov::Tensor & tensor, const void * output_dst) { std::cout << "Output name: " << name << ", Output shape: " << tensor.get_shape() << ", Address: " << output_dst - << std::endl; + << '\n'; auto print_float_stats = [](const std::string & type_name, size_t size, auto get_value) { if (size == 0) { @@ -758,20 +2006,16 @@ void print_output_tensor_info(const std::string & name, const ov::Tensor & tenso for (size_t i = 1; i < size; ++i) { float v = get_value(i); - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } + min = std::min(v, min); + max = std::max(v, max); sum += v; } double mean = sum / size; std::cout << std::right << std::setw(6) << type_name << std::right << std::setw(12) << "First" << std::setw(12) - << "Min" << std::setw(12) << "Max" << std::setw(12) << "Mean" << std::endl; + << "Min" << std::setw(12) << "Max" << std::setw(12) << "Mean" << '\n'; std::cout << std::right << std::setw(6) << "" << std::right << std::setw(12) << first << std::setw(12) << min - << std::setw(12) << max << std::setw(12) << mean << std::endl; + << std::setw(12) << max << std::setw(12) << mean << '\n'; }; switch (tensor.get_element_type()) { @@ -792,13 +2036,6 @@ void print_output_tensor_info(const std::string & name, const ov::Tensor & tenso } } -void set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols) { - for (size_t i = 0; i < rows; ++i) { - size_t diag_col = std::min(i, cols - 1); - matrix[i * cols + diag_col] = 0.0f; - } -} - const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) { for (int i = 0; i < cgraph->n_nodes; ++i) { auto * op = cgraph->nodes[i]; @@ -816,8 +2053,22 @@ const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) { throw std::runtime_error("get_inp_pos_tensor: inp_pos not found in cgraph"); } -bool get_is_prefill(const ggml_tensor * inp_pos) { - return inp_pos->ne[0] > 1; +int64_t get_inp_pos_n_tokens(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) { + // IMROPE stacks n_planes (t/h/w/e) position planes into inp_pos, so ne[0] is + // n_planes * n_tokens. Callers that need a token count must divide the planes out. + int n_planes = 1; + for (int i = 0; i < cgraph->n_nodes; ++i) { + auto * op = cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC; ++j) { + if (op->src[j] == inp_pos) { + n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op); + break; + } + } + } + return inp_pos->ne[0] / n_planes; } -#pragma GCC diagnostic pop +bool get_is_prefill(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) { + return get_inp_pos_n_tokens(cgraph, inp_pos) > 1; +} diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 656573d..c34441f 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -1,72 +1,155 @@ -#include "ggml-backend-impl.h" #include "ggml-decoder.h" #include "ggml-impl.h" +#include "ggml-openvino-extra.h" -#include #include +#include +#include #include +#include #include #include #include #include +#include +#include #include +// Cache key for a translated/compiled graph. Node count plus the two end node names identify a +// graph during inference, where the same few graphs repeat for the whole session. The list of +// external input names below tells those apart more precisely, but it walks every node and every +// src slot, so it is built only when GGML_OPENVINO_FULL_GRAPH_KEY is set. Op tests run many small +// graphs that can share a node count and end names, and need the full key. struct graph_key { int n_nodes; + int n_leaves; std::string first_node_name; std::string last_node_name; + // Each entry: "::" + std::vector input_srcs; - graph_key(const ggml_cgraph * cgraph) : n_nodes(cgraph->n_nodes) { + graph_key(const ggml_cgraph * cgraph, bool include_inputs = false) : n_nodes(cgraph->n_nodes), n_leaves(cgraph->n_leafs) { if (n_nodes > 0) { first_node_name = cgraph->nodes[0]->name; last_node_name = cgraph->nodes[n_nodes - 1]->name; } + + static const bool full_key = ggml_openvino_getenv_int("GGML_OPENVINO_FULL_GRAPH_KEY") != 0; + if (!full_key && !include_inputs) { + return; + } + + std::unordered_set node_set; + node_set.reserve(cgraph->n_nodes); + for (int i = 0; i < cgraph->n_nodes; i++) { + node_set.insert(cgraph->nodes[i]); + } + + for (int node_idx = 0; node_idx < cgraph->n_nodes; node_idx++) { + const ggml_tensor * node = cgraph->nodes[node_idx]; + for (int src_idx = 0; src_idx < GGML_MAX_SRC; src_idx++) { + const ggml_tensor * src = node->src[src_idx]; + if (src == nullptr || src->buffer == nullptr || node_set.count(src) || src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + continue; + } + + input_srcs.push_back(std::to_string(node_idx) + ":" + std::to_string(src_idx) + ":" + + GgmlOvDecoder::compute_op_type(node)); + } + } } bool operator==(const graph_key & other) const { - return n_nodes == other.n_nodes && first_node_name == other.first_node_name && - last_node_name == other.last_node_name; + return n_nodes == other.n_nodes && n_leaves == other.n_leaves && + first_node_name == other.first_node_name && + last_node_name == other.last_node_name && input_srcs == other.input_srcs; } }; struct graph_key_hash { size_t operator()(const graph_key & key) const { - size_t h = std::hash{}(key.n_nodes); + size_t hash = std::hash{}(key.n_nodes); + hash ^= std::hash{}(key.n_leaves) + 0x9e3779b9 + (hash << 6) + (hash >> 2); if (key.n_nodes > 0) { - h ^= std::hash{}(key.first_node_name) + 0x9e3779b9 + (h << 6) + (h >> 2); - h ^= std::hash{}(key.last_node_name) + 0x9e3779b9 + (h << 6) + (h >> 2); + hash ^= std::hash{}(key.first_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + hash ^= std::hash{}(key.last_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + } + for (const auto & s : key.input_srcs) { + hash ^= std::hash{}(s) + 0x9e3779b9 + (hash << 6) + (hash >> 2); } - return h; + return hash; } }; +struct decoder_runtime_ctx { + decoder_runtime_ctx(std::shared_ptr mutex) : mutex(std::move(mutex)) {} + + std::shared_ptr mutex; + std::shared_ptr ptr; +}; + +struct ov_compiled_graph { + ov::CompiledModel decode; + ov::CompiledModel prefill; + std::vector input_names; + std::vector output_names; +}; + +// Only compilation and cache publication use this mutex. Requests, decoders and +// sequence state belong to individual backend contexts and never enter this cache. +struct ov_compiled_model_cache { + std::mutex mutex; + std::unordered_map graphs; + size_t backend_count = 0; +}; + +// Private to one backend instance. Only compiled_cache is shared with other +// instances; clearing these local caches cannot invalidate their requests. struct ov_runtime_context { - std::mutex ov_compute_mutex; + // Serializes calls on this backend only, not inference in other contexts. + std::mutex execution_mutex; + std::shared_ptr compiled_cache; + mutable std::mutex ctx_mutex; std::string device; bool stateful; - std::unordered_map, graph_key_hash> decoder_cache; + std::unordered_map, graph_key_hash> decoder_cache; std::unordered_map, graph_key_hash> infer_request_cache; std::unordered_map, graph_key_hash> infer_request_cache_prefill; std::unordered_map, graph_key_hash> ov_input_names_cache; std::unordered_map, graph_key_hash> ov_output_names_cache; - //TODO: Stateful is only supported for single request at a time. - // Simultanous stateful inference request support to be added. size_t stateful_kv_size; std::map kv_state_input_name_map; + // compute-only copies of graphs that carry unselected ggml_build_forward_select() branches + struct compute_graph { + ggml_cgraph graph; + std::vector nodes; + }; + std::unordered_map compute_graphs; + + ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0) {} + + void clear_caches_locked() { + decoder_cache.clear(); + infer_request_cache.clear(); + infer_request_cache_prefill.clear(); + ov_input_names_cache.clear(); + ov_output_names_cache.clear(); + kv_state_input_name_map.clear(); + stateful_kv_size = 0; + } - ov_runtime_context() : - device("CPU"), - stateful(false), - stateful_kv_size(0) {} + void clear_caches() { + std::lock_guard lock(ctx_mutex); + clear_caches_locked(); + } }; enum ggml_status ov_graph_compute(struct ggml_cgraph * cgraph, ggml_backend_t backend); -enum ggml_status ov_graph_compute_dynamic(struct ggml_cgraph * cgraph, std::shared_ptr r_ctx); -enum ggml_status ov_graph_compute_static(struct ggml_cgraph * cgraph, std::shared_ptr r_ctx); - size_t checksum(const void * data, size_t size); +bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path); + void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor); void print_output_tensor_info(const std::string & name, const ov::Tensor & tensor, const void * output_dst); @@ -97,27 +180,17 @@ std::vector pad_input(const ggml_tensor * tensor, size_t padded_rows, size_t padded_rows, padded_cols, pad_value); } -void set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols); - const ggml_tensor * get_inp_pos_tensor(struct ggml_cgraph * cgraph); -bool get_is_prefill(const ggml_tensor * inp_pos); - -ov::Tensor get_ov_input_tensor(std::shared_ptr ggml_decoder, const std::string & param_name); -ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr ggml_decoder, - const std::string & param_name); -ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr ggml_decoder, - const std::string & param_name, - int chunk_index); +int64_t get_inp_pos_n_tokens(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos); -ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, - std::shared_ptr infer_request, - int output_index, - const ggml_tensor * ggml_tensor); +bool get_is_prefill(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos); bool is_naive(struct ggml_cgraph * cgraph); -enum ggml_status naive_compute(struct ggml_cgraph * cgraph, - ov::Core & core, - const std::string & device, - const ov::AnyMap & config); +/** + * @brief Heuristically checks whether the given computation graph is a split-model fragment. + * @param cgraph Pointer to the GGML computation graph to analyze. + * @return true if the graph is identified as split; otherwise false. + */ +bool is_model_splitted(struct ggml_cgraph * cgraph); diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 15443aa..7750a72 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -13,6 +13,10 @@ #include // for qsort #include // for GGML_ASSERT +#ifdef GGML_USE_OPENMP +#include +#endif + #define GROUP_MAX_EPS 1e-15f #define GROUP_MAX_EPS_IQ3_XXS 1e-8f #define GROUP_MAX_EPS_IQ2_S 1e-8f @@ -67,6 +71,44 @@ void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_REST } } +void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k) { + static const int qk = QK2_0; + + assert(k % qk == 0); + + const int nb = k / qk; + + for (int i = 0; i < nb; i++) { + // Compute scale as max absolute value in the block + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + const float a = fabsf(x[i*qk + j]); + if (a > amax) amax = a; + } + const float d = amax; + const float id = d > 0.0f ? 1.0f / d : 0.0f; + + y[i].d = GGML_FP32_TO_FP16(d); + + // Clear quant bytes + for (int j = 0; j < qk / 4; ++j) { + y[i].qs[j] = 0; + } + + // Encode 2-bit values: round(w/d) clamped to [-1, 2], then add 1 + // 00 (-1) = -scale, 01 (0) = 0, 10 (+1) = +scale, 11 (+2) = 2*scale + for (int j = 0; j < qk; ++j) { + const float w = x[i*qk + j]; + int q = (int)roundf(w * id) + 1; + if (q < 0) q = 0; + if (q > 3) q = 3; + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + y[i].qs[byte_index] |= ((uint8_t)q << bit_offset); + } + } +} + // reference implementation for deterministic creation of model files void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k) { static const int qk = QK4_0; @@ -394,6 +436,26 @@ void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRI } } +void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + static const int qk = QK2_0; + + assert(k % qk == 0); + + const int nb = k / qk; + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(x[i].d); + + for (int j = 0; j < qk; ++j) { + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + const uint8_t q = (x[i].qs[byte_index] >> bit_offset) & 0x03; + // 00=-1, 01=0, 10=+1, 11=+2 + y[i*qk + j] = ((int)q - 1) * d; + } + } +} + void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { static const int qk = QK4_0; @@ -974,8 +1036,9 @@ static float make_qkx3_quants(int n, int nmax, const float * GGML_RESTRICT x, co iscale = (rmin + rdelta*is + nmax)/(max - min); float sum_l = 0, sum_l2 = 0, sum_xl = 0; for (int i = 0; i < n; ++i) { - int l = nearest_int(iscale*(x[i] - min)); - l = MAX(0, MIN(nmax, l)); + // min is the best fit so far and can be at or near max, so v can be inf, nan or out of range for nearest_int + const float v = iscale*(x[i] - min); + const int l = v > 0 ? nearest_int(MIN(v, nmax)) : 0; Laux[i] = l; float w = weights ? weights[i] : x[i]*x[i]; sum_l += w*l; @@ -2048,6 +2111,20 @@ size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * row_size; } +size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + if (!quant_weights) { + quantize_row_q2_0_ref(src, dst, (int64_t)nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_Q2_0, n_per_row); + } + size_t row_size = ggml_row_size(GGML_TYPE_Q2_0, n_per_row); + char * qrow = (char *)dst; + for (int64_t row = 0; row < nrow; ++row) { + quantize_row_q2_0_ref(src, (block_q2_0*)qrow, n_per_row); + src += n_per_row; + qrow += row_size; + } + return nrow * row_size; +} size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { if (!quant_weights) { @@ -3064,70 +3141,121 @@ void iq2xs_init_impl(enum ggml_type type) { } kmap_q2xs[index] = i; } - int8_t pos[8]; - int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); + // The neighbour search runs in three passes: + // 1. Parallel: for each i, qsort and count its neighbours into n_per_i, + // and reduce the totals (num_neighbors, num_not_in_map). + // 2. Serial: prefix-sum n_per_i into offsets[], so each i has a + // pre-assigned slice of kneighbors_q2xs to write into. + // 3. Parallel: redo the qsort and write each i's neighbour list at + // offsets[i]. + int * n_per_i = (int *)malloc(kmap_size*sizeof(int)); + GGML_ASSERT(n_per_i); int num_neighbors = 0, num_not_in_map = 0; - for (int i = 0; i < kmap_size; ++i) { - if (kmap_q2xs[i] >= 0) continue; - ++num_not_in_map; - for (int k = 0; k < 8; ++k) { - int l = (i >> 2*k) & 0x3; - pos[k] = 2*l + 1; - } - for (int j = 0; j < grid_size; ++j) { - const int8_t * pg = (const int8_t *)(kgrid_q2xs + j); - int d2 = 0; - for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); - dist2[2*j+0] = d2; - dist2[2*j+1] = j; - } - qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func); - int n = 0; int d2 = dist2[0]; - int nhave = 1; - for (int j = 0; j < grid_size; ++j) { - if (dist2[2*j] > d2) { - if (nhave == nwant) break; - d2 = dist2[2*j]; - ++nhave; - } - ++n; - } - num_neighbors += n; +#ifdef GGML_USE_OPENMP + #pragma omp parallel reduction(+:num_neighbors,num_not_in_map) +#endif + { + int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); + GGML_ASSERT(dist2); + int8_t pos[8]; + int i; +#ifdef GGML_USE_OPENMP + #pragma omp for schedule(dynamic, 64) +#endif + for (i = 0; i < kmap_size; ++i) { + if (kmap_q2xs[i] >= 0) { + n_per_i[i] = 0; + continue; + } + ++num_not_in_map; + for (int k = 0; k < 8; ++k) { + int l = (i >> 2*k) & 0x3; + pos[k] = 2*l + 1; + } + for (int j = 0; j < grid_size; ++j) { + const int8_t * pg = (const int8_t *)(kgrid_q2xs + j); + int d2 = 0; + for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); + dist2[2*j+0] = d2; + dist2[2*j+1] = j; + } + qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func); + int n = 0; int d2 = dist2[0]; + int nhave = 1; + for (int j = 0; j < grid_size; ++j) { + if (dist2[2*j] > d2) { + if (nhave == nwant) break; + d2 = dist2[2*j]; + ++nhave; + } + ++n; + } + n_per_i[i] = n; + num_neighbors += n; + } + free(dist2); } //printf("%s: %d neighbours in total\n", __func__, num_neighbors); kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t)); iq2_data[gindex].neighbours = kneighbors_q2xs; + + int * offsets = (int *)malloc(kmap_size*sizeof(int)); + GGML_ASSERT(offsets); int counter = 0; for (int i = 0; i < kmap_size; ++i) { - if (kmap_q2xs[i] >= 0) continue; - for (int k = 0; k < 8; ++k) { - int l = (i >> 2*k) & 0x3; - pos[k] = 2*l + 1; - } - for (int j = 0; j < grid_size; ++j) { - const int8_t * pg = (const int8_t *)(kgrid_q2xs + j); - int d2 = 0; - for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); - dist2[2*j+0] = d2; - dist2[2*j+1] = j; - } - qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func); - kmap_q2xs[i] = -(counter + 1); - int d2 = dist2[0]; - uint16_t * start = &kneighbors_q2xs[counter++]; - int n = 0, nhave = 1; - for (int j = 0; j < grid_size; ++j) { - if (dist2[2*j] > d2) { - if (nhave == nwant) break; - d2 = dist2[2*j]; - ++nhave; - } - kneighbors_q2xs[counter++] = dist2[2*j+1]; - ++n; - } - *start = n; - } - free(dist2); + if (kmap_q2xs[i] >= 0) { + offsets[i] = -1; + continue; + } + offsets[i] = counter; + counter += 1 + n_per_i[i]; + } + +#ifdef GGML_USE_OPENMP + #pragma omp parallel +#endif + { + int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); + GGML_ASSERT(dist2); + int8_t pos[8]; + int i; +#ifdef GGML_USE_OPENMP + #pragma omp for schedule(dynamic, 64) +#endif + for (i = 0; i < kmap_size; ++i) { + if (kmap_q2xs[i] >= 0) continue; + for (int k = 0; k < 8; ++k) { + int l = (i >> 2*k) & 0x3; + pos[k] = 2*l + 1; + } + for (int j = 0; j < grid_size; ++j) { + const int8_t * pg = (const int8_t *)(kgrid_q2xs + j); + int d2 = 0; + for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); + dist2[2*j+0] = d2; + dist2[2*j+1] = j; + } + qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func); + int local_counter = offsets[i]; + kmap_q2xs[i] = -(local_counter + 1); + int d2 = dist2[0]; + uint16_t * start = &kneighbors_q2xs[local_counter++]; + int n = 0, nhave = 1; + for (int j = 0; j < grid_size; ++j) { + if (dist2[2*j] > d2) { + if (nhave == nwant) break; + d2 = dist2[2*j]; + ++nhave; + } + kneighbors_q2xs[local_counter++] = dist2[2*j+1]; + ++n; + } + *start = n; + } + free(dist2); + } + free(offsets); + free(n_per_i); } void iq2xs_free_impl(enum ggml_type type) { @@ -3663,70 +3791,115 @@ void iq3xs_init_impl(int grid_size) { } kmap_q3xs[index] = i; } - int8_t pos[4]; - int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); + // See explanation of parallelism in iq2xs_init_impl + int * n_per_i = (int *)malloc(kmap_size*sizeof(int)); + GGML_ASSERT(n_per_i); int num_neighbors = 0, num_not_in_map = 0; - for (int i = 0; i < kmap_size; ++i) { - if (kmap_q3xs[i] >= 0) continue; - ++num_not_in_map; - for (int k = 0; k < 4; ++k) { - int l = (i >> 3*k) & 0x7; - pos[k] = 2*l + 1; - } - for (int j = 0; j < grid_size; ++j) { - const int8_t * pg = (const int8_t *)(kgrid_q3xs + j); - int d2 = 0; - for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); - dist2[2*j+0] = d2; - dist2[2*j+1] = j; - } - qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func); - int n = 0; int d2 = dist2[0]; - int nhave = 1; - for (int j = 0; j < grid_size; ++j) { - if (dist2[2*j] > d2) { - if (nhave == nwant) break; - d2 = dist2[2*j]; - ++nhave; - } - ++n; - } - num_neighbors += n; +#ifdef GGML_USE_OPENMP + #pragma omp parallel reduction(+:num_neighbors,num_not_in_map) +#endif + { + int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); + GGML_ASSERT(dist2); + int8_t pos[4]; + int i; +#ifdef GGML_USE_OPENMP + #pragma omp for schedule(dynamic, 64) +#endif + for (i = 0; i < kmap_size; ++i) { + if (kmap_q3xs[i] >= 0) { + n_per_i[i] = 0; + continue; + } + ++num_not_in_map; + for (int k = 0; k < 4; ++k) { + int l = (i >> 3*k) & 0x7; + pos[k] = 2*l + 1; + } + for (int j = 0; j < grid_size; ++j) { + const int8_t * pg = (const int8_t *)(kgrid_q3xs + j); + int d2 = 0; + for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); + dist2[2*j+0] = d2; + dist2[2*j+1] = j; + } + qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func); + int n = 0; int d2 = dist2[0]; + int nhave = 1; + for (int j = 0; j < grid_size; ++j) { + if (dist2[2*j] > d2) { + if (nhave == nwant) break; + d2 = dist2[2*j]; + ++nhave; + } + ++n; + } + n_per_i[i] = n; + num_neighbors += n; + } + free(dist2); } //printf("%s: %d neighbours in total\n", __func__, num_neighbors); kneighbors_q3xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t)); iq3_data[gindex].neighbours = kneighbors_q3xs; + + int * offsets = (int *)malloc(kmap_size*sizeof(int)); + GGML_ASSERT(offsets); int counter = 0; for (int i = 0; i < kmap_size; ++i) { - if (kmap_q3xs[i] >= 0) continue; - for (int k = 0; k < 4; ++k) { - int l = (i >> 3*k) & 0x7; - pos[k] = 2*l + 1; - } - for (int j = 0; j < grid_size; ++j) { - const int8_t * pg = (const int8_t *)(kgrid_q3xs + j); - int d2 = 0; - for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); - dist2[2*j+0] = d2; - dist2[2*j+1] = j; - } - qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func); - kmap_q3xs[i] = -(counter + 1); - int d2 = dist2[0]; - uint16_t * start = &kneighbors_q3xs[counter++]; - int n = 0, nhave = 1; - for (int j = 0; j < grid_size; ++j) { - if (dist2[2*j] > d2) { - if (nhave == nwant) break; - d2 = dist2[2*j]; - ++nhave; - } - kneighbors_q3xs[counter++] = dist2[2*j+1]; - ++n; - } - *start = n; - } - free(dist2); + if (kmap_q3xs[i] >= 0) { + offsets[i] = -1; + continue; + } + offsets[i] = counter; + counter += 1 + n_per_i[i]; + } + +#ifdef GGML_USE_OPENMP + #pragma omp parallel +#endif + { + int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); + GGML_ASSERT(dist2); + int8_t pos[4]; + int i; +#ifdef GGML_USE_OPENMP + #pragma omp for schedule(dynamic, 64) +#endif + for (i = 0; i < kmap_size; ++i) { + if (kmap_q3xs[i] >= 0) continue; + for (int k = 0; k < 4; ++k) { + int l = (i >> 3*k) & 0x7; + pos[k] = 2*l + 1; + } + for (int j = 0; j < grid_size; ++j) { + const int8_t * pg = (const int8_t *)(kgrid_q3xs + j); + int d2 = 0; + for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); + dist2[2*j+0] = d2; + dist2[2*j+1] = j; + } + qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func); + int local_counter = offsets[i]; + kmap_q3xs[i] = -(local_counter + 1); + int d2 = dist2[0]; + uint16_t * start = &kneighbors_q3xs[local_counter++]; + int n = 0, nhave = 1; + for (int j = 0; j < grid_size; ++j) { + if (dist2[2*j] > d2) { + if (nhave == nwant) break; + d2 = dist2[2*j]; + ++nhave; + } + kneighbors_q3xs[local_counter++] = dist2[2*j+1]; + ++n; + } + *start = n; + } + free(dist2); + } + free(offsets); + free(n_per_i); } void iq3xs_free_impl(int grid_size) { @@ -4599,80 +4772,51 @@ static void quantize_row_iq1_m_impl(const float * GGML_RESTRICT x, void * GGML_R // 1: +, - // 2: -, + // 3: -, - - for (int i1 = 0; i1 <= block_size; ++i1) { - for (int i2 = i1; i2 <= block_size; ++i2) { - memset(sumqx, 0, 4*sizeof(float)); - memset(sumq2, 0, 4*sizeof(float)); - for (int j = 0; j < i1; ++j) { - int i = idx[2*j]; - if (i < block_size/2) { - sumqx[0] += weight[i]*x_p[0]*xb[i]; - sumqx[1] += weight[i]*x_p[0]*xb[i]; - sumqx[2] += weight[i]*x_m[0]*xb[i]; - sumqx[3] += weight[i]*x_m[0]*xb[i]; - sumq2[0] += weight[i]*x_p[0]*x_p[0]; - sumq2[1] += weight[i]*x_p[0]*x_p[0]; - sumq2[2] += weight[i]*x_m[0]*x_m[0]; - sumq2[3] += weight[i]*x_m[0]*x_m[0]; - } else { - sumqx[0] += weight[i]*x_p[0]*xb[i]; - sumqx[2] += weight[i]*x_p[0]*xb[i]; - sumqx[1] += weight[i]*x_m[0]*xb[i]; - sumqx[3] += weight[i]*x_m[0]*xb[i]; - sumq2[0] += weight[i]*x_p[0]*x_p[0]; - sumq2[2] += weight[i]*x_p[0]*x_p[0]; - sumq2[1] += weight[i]*x_m[0]*x_m[0]; - sumq2[3] += weight[i]*x_m[0]*x_m[0]; - } - } - for (int j = i1; j < i2; ++j) { - int i = idx[2*j]; - if (i < block_size/2) { - sumqx[0] += weight[i]*x_p[1]*xb[i]; - sumqx[1] += weight[i]*x_p[1]*xb[i]; - sumqx[2] += weight[i]*x_m[1]*xb[i]; - sumqx[3] += weight[i]*x_m[1]*xb[i]; - sumq2[0] += weight[i]*x_p[1]*x_p[1]; - sumq2[1] += weight[i]*x_p[1]*x_p[1]; - sumq2[2] += weight[i]*x_m[1]*x_m[1]; - sumq2[3] += weight[i]*x_m[1]*x_m[1]; - } else { - sumqx[0] += weight[i]*x_p[1]*xb[i]; - sumqx[2] += weight[i]*x_p[1]*xb[i]; - sumqx[1] += weight[i]*x_m[1]*xb[i]; - sumqx[3] += weight[i]*x_m[1]*xb[i]; - sumq2[0] += weight[i]*x_p[1]*x_p[1]; - sumq2[2] += weight[i]*x_p[1]*x_p[1]; - sumq2[1] += weight[i]*x_m[1]*x_m[1]; - sumq2[3] += weight[i]*x_m[1]*x_m[1]; - } - } - for (int j = i2; j < block_size; ++j) { - int i = idx[2*j]; - if (i < block_size/2) { - sumqx[0] += weight[i]*x_p[2]*xb[i]; - sumqx[1] += weight[i]*x_p[2]*xb[i]; - sumqx[2] += weight[i]*x_m[2]*xb[i]; - sumqx[3] += weight[i]*x_m[2]*xb[i]; - sumq2[0] += weight[i]*x_p[2]*x_p[2]; - sumq2[1] += weight[i]*x_p[2]*x_p[2]; - sumq2[2] += weight[i]*x_m[2]*x_m[2]; - sumq2[3] += weight[i]*x_m[2]*x_m[2]; - } else { - sumqx[0] += weight[i]*x_p[2]*xb[i]; - sumqx[2] += weight[i]*x_p[2]*xb[i]; - sumqx[1] += weight[i]*x_m[2]*xb[i]; - sumqx[3] += weight[i]*x_m[2]*xb[i]; - sumq2[0] += weight[i]*x_p[2]*x_p[2]; - sumq2[2] += weight[i]*x_p[2]*x_p[2]; - sumq2[1] += weight[i]*x_m[2]*x_m[2]; - sumq2[3] += weight[i]*x_m[2]*x_m[2]; + // prefix sums are kept per half of the block because each half can use a different sign (x_p or x_m) + // since v[0]-v[1] = v[1]-v[2] = -1 for both x_p and x_m, the 3-group sum for a split collapses to T*v[2] - px[i1] - px[i2] + { + float px[2][IQ1M_BLOCK_SIZE+1]; + float pw[2][IQ1M_BLOCK_SIZE+1]; + px[0][0] = px[1][0] = 0; + pw[0][0] = pw[1][0] = 0; + for (int j = 0; j < block_size; ++j) { + const int i = idx[2*j]; + const int h = i < block_size/2 ? 0 : 1; + px[h][j+1] = px[h][j] + weight[i]*xb[i]; + px[1-h][j+1] = px[1-h][j]; + pw[h][j+1] = pw[h][j] + weight[i]; + pw[1-h][j+1] = pw[1-h][j]; + } + const float txs[2] = {px[0][block_size], px[1][block_size]}; // total weight*x per half + const float tws[2] = {pw[0][block_size], pw[1][block_size]}; // total weight per half + const float p2 = x_p[2], m2 = x_m[2]; + const float cp1 = x_p[0]*x_p[0] - x_p[1]*x_p[1]; + const float cp2 = x_p[1]*x_p[1] - x_p[2]*x_p[2]; + const float cm1 = x_m[0]*x_m[0] - x_m[1]*x_m[1]; + const float cm2 = x_m[1]*x_m[1] - x_m[2]*x_m[2]; + for (int i1 = 0; i1 <= block_size; ++i1) { + for (int i2 = i1; i2 <= block_size; ++i2) { + float qx_p[2], qx_m[2], q2_p[2], q2_m[2]; + for (int h = 0; h < 2; ++h) { + const float sx = px[h][i1] + px[h][i2]; + qx_p[h] = txs[h]*p2 - sx; + qx_m[h] = txs[h]*m2 - sx; + q2_p[h] = tws[h]*p2*p2 + pw[h][i1]*cp1 + pw[h][i2]*cp2; + q2_m[h] = tws[h]*m2*m2 + pw[h][i1]*cm1 + pw[h][i2]*cm2; } - } - for (int k = 0; k < 4; ++k) { - if (sumq2[k] > 0 && sumqx[k]*sumqx[k] > best_score*sumq2[k]) { - scale = sumqx[k]/sumq2[k]; best_score = scale*sumqx[k]; - besti1 = i1; besti2 = i2; best_k = k; + sumqx[0] = qx_p[0] + qx_p[1]; + sumqx[1] = qx_p[0] + qx_m[1]; + sumqx[2] = qx_m[0] + qx_p[1]; + sumqx[3] = qx_m[0] + qx_m[1]; + sumq2[0] = q2_p[0] + q2_p[1]; + sumq2[1] = q2_p[0] + q2_m[1]; + sumq2[2] = q2_m[0] + q2_p[1]; + sumq2[3] = q2_m[0] + q2_m[1]; + for (int k = 0; k < 4; ++k) { + if (sumq2[k] > 0 && sumqx[k]*sumqx[k] > best_score*sumq2[k]) { + scale = sumqx[k]/sumq2[k]; best_score = scale*sumqx[k]; + besti1 = i1; besti2 = i2; best_k = k; + } } } } @@ -5361,6 +5505,10 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte { VALIDATE_ROW_DATA_D_F16_IMPL(block_q1_0, data, nb); } break; + case GGML_TYPE_Q2_0: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_q2_0, data, nb); + } break; case GGML_TYPE_Q4_0: { VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb); diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index d56c86d..75188f1 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -15,6 +15,7 @@ extern "C" { // Quantization GGML_API void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k); @@ -43,6 +44,7 @@ GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_ // Dequantization GGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -93,6 +95,7 @@ GGML_API size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTR GGML_API size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); diff --git a/ggml/src/ggml-rpc/CMakeLists.txt b/ggml/src/ggml-rpc/CMakeLists.txt index 8671ce5..8976651 100644 --- a/ggml/src/ggml-rpc/CMakeLists.txt +++ b/ggml/src/ggml-rpc/CMakeLists.txt @@ -2,16 +2,26 @@ message(STATUS "Using RPC backend") ggml_add_backend_library(ggml-rpc ggml-rpc.cpp + log.cpp + transport.cpp ) if (WIN32) target_link_libraries(ggml-rpc PRIVATE ws2_32) endif() -# RDMA auto-detection (Linux only, requires libibverbs) -if (NOT WIN32 AND NOT APPLE) - find_library(IBVERBS_LIB ibverbs) - if (IBVERBS_LIB) +# RDMA auto-detection: Linux RoCE/IB via libibverbs, Apple RDMA-over-Thunderbolt via librdma +if (APPLE) + set(RDMA_LIB_NAME rdma) + set(RDMA_DESC "Apple RDMA-over-Thunderbolt, UC") +elseif (NOT WIN32) + set(RDMA_LIB_NAME ibverbs) + set(RDMA_DESC "auto-detected") +endif() + +if (RDMA_LIB_NAME) + find_library(RDMA_LIB ${RDMA_LIB_NAME}) + if (RDMA_LIB) option(GGML_RPC_RDMA "ggml: enable RDMA transport for RPC" ON) else() option(GGML_RPC_RDMA "ggml: enable RDMA transport for RPC" OFF) @@ -21,12 +31,22 @@ else() endif() if (GGML_RPC_RDMA) - if (NOT IBVERBS_LIB) - find_library(IBVERBS_LIB ibverbs REQUIRED) + if (NOT RDMA_LIB) + find_library(RDMA_LIB ${RDMA_LIB_NAME} REQUIRED) endif() target_compile_definitions(ggml-rpc PRIVATE GGML_RPC_RDMA) - target_link_libraries(ggml-rpc PRIVATE ${IBVERBS_LIB}) - message(STATUS " RDMA transport enabled (auto-detected)") + if (APPLE) + # librdma.dylib only exists on macOS 26.2 and later. Link it weakly so a build made + # where it exists still loads where it does not; checked at runtime before use + # but with BUILD_SHARED_LIBS=OFF ggml-rpc is a static archive and never links + # so the librdma symbols used by transport-apple.cpp stay undefined. + target_link_options(ggml-rpc PUBLIC "LINKER:-weak_library,${RDMA_LIB}") + target_compile_definitions(ggml-rpc PRIVATE GGML_RPC_RDMA_APPLE) + target_sources(ggml-rpc PRIVATE transport-apple.cpp) + else() + target_link_libraries(ggml-rpc PRIVATE ${RDMA_LIB}) + endif() + message(STATUS " RDMA transport enabled (${RDMA_DESC})") else() message(STATUS " RDMA transport disabled") endif() diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 017ef0a..49f1107 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -2,175 +2,31 @@ #include "ggml-impl.h" #include "ggml-backend-impl.h" #include "ggml-cpp.h" +#include "transport.h" +#include "log.h" #include +#include #include #include #include +#include #include +#include +#include +#include #include #include #include #include -#ifdef _WIN32 -# define WIN32_LEAN_AND_MEAN -# ifndef NOMINMAX -# define NOMINMAX -# endif -# include -# include -#else -# include -# include -# include -# include -# include -# include -# include -#endif #include #include #include #include - -#ifdef GGML_RPC_RDMA -# include -# include -# ifndef _WIN32 -# include -# endif -#endif // GGML_RPC_RDMA - -static const char * RPC_DEBUG = std::getenv("GGML_RPC_DEBUG"); - -#define LOG_DBG(...) \ - do { if (RPC_DEBUG) GGML_LOG_DEBUG(__VA_ARGS__); } while (0) - +#include namespace fs = std::filesystem; -static constexpr size_t MAX_CHUNK_SIZE = 1024ull * 1024ull * 1024ull; // 1 GiB - -#ifdef _WIN32 -typedef SOCKET sockfd_t; -using ssize_t = __int64; -#else -typedef int sockfd_t; -#endif - -// cross-platform socket - -#ifdef GGML_RPC_RDMA -static constexpr size_t RDMA_CHUNK = 256 * 1024; // 256 KiB per send/recv (fits default 8 MiB memlock) -static constexpr int RDMA_RX_DEPTH = 24; // pre-posted recv ring: 24 × 256 KiB = 6 MiB -static constexpr size_t RDMA_GID_SIZE = 16; // RoCE GID / IB GID is always 16 bytes -using rdma_gid_t = std::array; - -struct rdma_conn { - struct ibv_context * ctx = nullptr; - struct ibv_pd * pd = nullptr; - struct ibv_cq * scq = nullptr; // send completions - struct ibv_cq * rcq = nullptr; // recv completions - struct ibv_qp * qp = nullptr; - - void * tx_buf = nullptr; - struct ibv_mr * tx_mr = nullptr; - - void * rx_buf = nullptr; // RDMA_RX_DEPTH × RDMA_CHUNK contiguous - struct ibv_mr * rx_mr = nullptr; - int rx_head = 0; - - uint32_t max_inline = 0; - - uint8_t * rx_slot(int i) const { - return static_cast(rx_buf) + static_cast(i) * RDMA_CHUNK; - } - - bool post_rx(int i) { - struct ibv_sge sge = {}; - sge.addr = (uintptr_t)rx_slot(i); - sge.length = RDMA_CHUNK; - sge.lkey = rx_mr->lkey; - struct ibv_recv_wr wr = {}, * bad = nullptr; - wr.wr_id = (uint64_t)i; - wr.sg_list = &sge; - wr.num_sge = 1; - return ibv_post_recv(qp, &wr, &bad) == 0; - } - - ~rdma_conn() { - if (tx_mr) ibv_dereg_mr(tx_mr); - if (rx_mr) ibv_dereg_mr(rx_mr); - free(tx_buf); - free(rx_buf); - if (qp) ibv_destroy_qp(qp); - if (scq) ibv_destroy_cq(scq); - if (rcq) ibv_destroy_cq(rcq); - if (pd) ibv_dealloc_pd(pd); - if (ctx) ibv_close_device(ctx); - } -}; - -// Local RDMA parameters captured during the probe phase and later consumed -// by rdma_activate() after the remote side's caps arrive via HELLO. -struct rdma_local_info { - uint32_t qpn = 0; - uint32_t psn = 0; - uint8_t gid[RDMA_GID_SIZE] = {}; - uint8_t ib_port = 0; - int gid_idx = 0; - enum ibv_mtu path_mtu = IBV_MTU_1024; -}; -#endif // GGML_RPC_RDMA - -// conn_caps size for transport-agnostic capability exchange -static constexpr size_t RPC_CONN_CAPS_SIZE = 24; - -// conn_caps RDMA layout helper -#ifdef GGML_RPC_RDMA -struct rdma_caps { - uint32_t qpn; - uint32_t psn; - uint8_t gid[RDMA_GID_SIZE]; -}; -static_assert(sizeof(rdma_caps) == RPC_CONN_CAPS_SIZE, "rdma_caps must match conn_caps size"); -#endif // GGML_RPC_RDMA - -// Forward declarations for transport function pointers -struct socket_t; -static bool tcp_send_impl(socket_t * sock, const void * data, size_t size); -static bool tcp_recv_impl(socket_t * sock, void * data, size_t size); - -struct socket_t { - sockfd_t fd; - bool (*fn_send)(socket_t *, const void *, size_t) = tcp_send_impl; - bool (*fn_recv)(socket_t *, void *, size_t) = tcp_recv_impl; -#ifdef GGML_RPC_RDMA - std::unique_ptr rdma; - rdma_local_info rdma_local = {}; -#endif // GGML_RPC_RDMA - socket_t(sockfd_t fd) : fd(fd) {} - ~socket_t() { -#ifdef GGML_RPC_RDMA - rdma.reset(); -#endif // GGML_RPC_RDMA - LOG_DBG("[%s] closing socket %d\n", __func__, this->fd); -#ifdef _WIN32 - if (fd != INVALID_SOCKET) closesocket(this->fd); -#else - if (fd >= 0) close(this->fd); -#endif - } - - // Advertise local transport capabilities into conn_caps. - // May probe RDMA and store the probe on this socket for update_caps. - void get_caps(uint8_t * caps); - - // Activate transport upgrade based on remote conn_caps using the probe - // previously stored by get_caps. - void update_caps(const uint8_t * remote_caps); -}; - // macro for nicer error messages on server crash #define RPC_STATUS_ASSERT(x) if (!(x)) GGML_ABORT("Remote RPC server crashed or returned malformed response") @@ -192,7 +48,7 @@ struct rpc_tensor { uint64_t data; char name[GGML_MAX_NAME]; - char padding[4]; + int32_t use_count; }; static_assert(sizeof(rpc_tensor) % 8 == 0, "rpc_tensor size must be multiple of 8"); @@ -216,14 +72,55 @@ enum rpc_cmd { RPC_CMD_HELLO, RPC_CMD_DEVICE_COUNT, RPC_CMD_GRAPH_RECOMPUTE, + RPC_CMD_MEMSET_TENSOR, + RPC_CMD_SET_TENSOR_2D, + RPC_CMD_GET_TENSOR_2D, + RPC_CMD_COMM_INIT, + RPC_CMD_COMM_ALLREDUCE, + RPC_CMD_COMM_FREE, + RPC_CMD_NONE, RPC_CMD_COUNT, }; static_assert(RPC_CMD_HELLO == 14, "RPC_CMD_HELLO must be always 14"); +static const char * rpc_cmd_name(enum rpc_cmd cmd) { + switch (cmd) { + case RPC_CMD_ALLOC_BUFFER: return "ALLOC_BUFFER"; + case RPC_CMD_GET_ALIGNMENT: return "GET_ALIGNMENT"; + case RPC_CMD_GET_MAX_SIZE: return "GET_MAX_SIZE"; + case RPC_CMD_BUFFER_GET_BASE: return "BUFFER_GET_BASE"; + case RPC_CMD_FREE_BUFFER: return "FREE_BUFFER"; + case RPC_CMD_BUFFER_CLEAR: return "BUFFER_CLEAR"; + case RPC_CMD_SET_TENSOR: return "SET_TENSOR"; + case RPC_CMD_SET_TENSOR_HASH: return "SET_TENSOR_HASH"; + case RPC_CMD_GET_TENSOR: return "GET_TENSOR"; + case RPC_CMD_COPY_TENSOR: return "COPY_TENSOR"; + case RPC_CMD_GRAPH_COMPUTE: return "GRAPH_COMPUTE"; + case RPC_CMD_GET_DEVICE_MEMORY: return "GET_DEVICE_MEMORY"; + case RPC_CMD_INIT_TENSOR: return "INIT_TENSOR"; + case RPC_CMD_GET_ALLOC_SIZE: return "GET_ALLOC_SIZE"; + case RPC_CMD_HELLO: return "HELLO"; + case RPC_CMD_DEVICE_COUNT: return "DEVICE_COUNT"; + case RPC_CMD_GRAPH_RECOMPUTE: return "GRAPH_RECOMPUTE"; + case RPC_CMD_MEMSET_TENSOR: return "MEMSET_TENSOR"; + case RPC_CMD_SET_TENSOR_2D: return "SET_TENSOR_2D"; + case RPC_CMD_GET_TENSOR_2D: return "GET_TENSOR_2D"; + case RPC_CMD_COMM_INIT: return "COMM_INIT"; + case RPC_CMD_COMM_ALLREDUCE: return "COMM_ALLREDUCE"; + case RPC_CMD_COMM_FREE: return "COMM_FREE"; + case RPC_CMD_NONE: return "NONE"; + default: return "UNKNOWN"; + } +} + // Try RPC_CMD_SET_TENSOR_HASH first when data size is larger than this threshold const size_t HASH_THRESHOLD = 10 * 1024 * 1024; +// Maximum number of graphs cached per device; client and server must use the same value +// so that both sides clear their caches at the same point in the message stream +const size_t GRAPH_CACHE_MAX = 1024; + struct rpc_msg_hello_req { uint8_t conn_caps[RPC_CONN_CAPS_SIZE]; }; @@ -297,6 +194,13 @@ struct rpc_msg_buffer_clear_req { uint8_t value; }; +struct rpc_msg_memset_tensor_req { + rpc_tensor tensor; + uint64_t offset; + uint64_t size; + uint8_t value; +}; + struct rpc_msg_set_tensor_hash_req { rpc_tensor tensor; uint64_t offset; @@ -333,6 +237,36 @@ struct rpc_msg_get_device_memory_rsp { struct rpc_msg_graph_recompute_req { uint32_t device; + uint64_t uid; +}; + +struct rpc_msg_get_tensor_2d_req { + rpc_tensor tensor; + uint64_t offset; + uint64_t size; + uint64_t n_copies; + uint64_t stride; +}; + +struct rpc_msg_comm_init_req { + uint32_t device; + uint32_t rank; + uint32_t world; + uint32_t port; // rank 0: port to listen on; rank > 0: rank 0's comm port + char host[64]; // rank > 0: rank 0's host +}; + +struct rpc_msg_comm_init_rsp { + uint8_t ok; +}; + +struct rpc_msg_comm_allreduce_req { + uint32_t device; + rpc_tensor tensor; +}; + +struct rpc_msg_comm_free_req { + uint32_t device; }; #pragma pack(pop) @@ -344,6 +278,13 @@ static ggml_guid_t ggml_backend_rpc_guid() { return &guid; } +struct ggml_backend_rpc_device_context { + std::string endpoint; + uint32_t device; + std::string name; + std::string description; +}; + struct ggml_backend_rpc_buffer_type_context { std::string endpoint; uint32_t device; @@ -352,49 +293,25 @@ struct ggml_backend_rpc_buffer_type_context { size_t max_size; }; -struct graph_cache { - - bool is_cached(const ggml_cgraph * cgraph) { - if ((int)last_graph.size() != cgraph->n_nodes) { - return false; - } - for (int i = 0; i < cgraph->n_nodes; i++) { - if (memcmp(&last_graph[i], cgraph->nodes[i], sizeof(ggml_tensor)) != 0) { - return false; - } - } - return true; - } - - void add(const ggml_cgraph * cgraph) { - last_graph.resize(cgraph->n_nodes); - for (int i = 0; i < cgraph->n_nodes; i++) { - memcpy(&last_graph[i], cgraph->nodes[i], sizeof(ggml_tensor)); - } - } - - std::vector last_graph; -}; - +class rpc_dispatcher; struct ggml_backend_rpc_context { - std::string endpoint; - uint32_t device; - std::string name; - graph_cache gc; + std::shared_ptr dispatcher; + std::string endpoint; + uint32_t device; + std::string name; }; struct ggml_backend_rpc_buffer_context { - std::shared_ptr sock; - void * base_ptr; - uint64_t remote_ptr; + std::shared_ptr dispatcher; + void * base_ptr; + uint64_t remote_ptr; }; // RPC helper functions // Computes FNV-1a hash of the data -static uint64_t fnv_hash(const uint8_t * data, size_t len) { +static uint64_t fnv_hash(const uint8_t * data, size_t len, uint64_t hash = 0xcbf29ce484222325ULL) { const uint64_t fnv_prime = 0x100000001b3ULL; - uint64_t hash = 0xcbf29ce484222325ULL; for (size_t i = 0; i < len; ++i) { hash ^= data[i]; @@ -403,549 +320,39 @@ static uint64_t fnv_hash(const uint8_t * data, size_t len) { return hash; } -static std::shared_ptr make_socket(sockfd_t fd) { -#ifdef _WIN32 - if (fd == INVALID_SOCKET) { - return nullptr; - } -#else - if (fd < 0) { - return nullptr; - } -#endif - return std::make_shared(fd); -} - -static bool set_no_delay(sockfd_t sockfd) { - int flag = 1; - // set TCP_NODELAY to disable Nagle's algorithm - int ret = setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(int)); - return ret == 0; -} - -static bool set_reuse_addr(sockfd_t sockfd) { - int flag = 1; - int ret = setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof(int)); - return ret == 0; -} - -static std::shared_ptr socket_connect(const char * host, int port) { - struct sockaddr_in addr; - auto sockfd = socket(AF_INET, SOCK_STREAM, 0); - auto sock_ptr = make_socket(sockfd); - if (sock_ptr == nullptr) { - return nullptr; - } - if (!set_no_delay(sockfd)) { - GGML_LOG_ERROR("Failed to set TCP_NODELAY\n"); - return nullptr; - } - addr.sin_family = AF_INET; - addr.sin_port = htons(port); - struct hostent * server = gethostbyname(host); - if (server == NULL) { - GGML_LOG_ERROR("Cannot resolve host '%s'\n", host); - return nullptr; - } - memcpy(&addr.sin_addr.s_addr, server->h_addr, server->h_length); - if (connect(sock_ptr->fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { - return nullptr; - } - return sock_ptr; -} - -static std::shared_ptr socket_accept(sockfd_t srv_sockfd) { - auto client_socket_fd = accept(srv_sockfd, NULL, NULL); - auto client_socket = make_socket(client_socket_fd); - if (client_socket == nullptr) { - return nullptr; - } - if (!set_no_delay(client_socket_fd)) { - GGML_LOG_ERROR("Failed to set TCP_NODELAY\n"); - return nullptr; - } - return client_socket; -} - -static std::shared_ptr create_server_socket(const char * host, int port) { - auto sockfd = socket(AF_INET, SOCK_STREAM, 0); - auto sock = make_socket(sockfd); - if (sock == nullptr) { - return nullptr; - } - if (!set_reuse_addr(sockfd)) { - GGML_LOG_ERROR("Failed to set SO_REUSEADDR\n"); - return nullptr; - } - if (inet_addr(host) == INADDR_NONE) { - GGML_LOG_ERROR("Invalid host address: %s\n", host); - return nullptr; - } - struct sockaddr_in serv_addr; - serv_addr.sin_family = AF_INET; - serv_addr.sin_addr.s_addr = inet_addr(host); - serv_addr.sin_port = htons(port); - - if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { - return nullptr; - } - if (listen(sockfd, 1) < 0) { - return nullptr; - } - return sock; -} - -static bool send_data(sockfd_t sockfd, const void * data, size_t size) { - size_t bytes_sent = 0; - while (bytes_sent < size) { - size_t size_to_send = std::min(size - bytes_sent, MAX_CHUNK_SIZE); - ssize_t n = send(sockfd, (const char *)data + bytes_sent, size_to_send, 0); - if (n < 0) { - GGML_LOG_ERROR("send failed (bytes_sent=%zu, size_to_send=%zu)\n", - bytes_sent, size_to_send); - return false; - } - bytes_sent += (size_t)n; - } - return true; -} - -static bool recv_data(sockfd_t sockfd, void * data, size_t size) { - size_t bytes_recv = 0; - while (bytes_recv < size) { - size_t size_to_recv = std::min(size - bytes_recv, MAX_CHUNK_SIZE); - ssize_t n = recv(sockfd, (char *)data + bytes_recv, size_to_recv, 0); - if (n < 0) { - GGML_LOG_ERROR("recv failed (bytes_recv=%zu, size_to_recv=%zu)\n", - bytes_recv, size_to_recv); - return false; - } - if (n == 0) { - LOG_DBG("recv returned 0 (peer closed?)\n"); - return false; - } - bytes_recv += (size_t)n; - } - return true; -} - -// TCP transport implementations (for function-pointer dispatch) - -static bool tcp_send_impl(socket_t * sock, const void * data, size_t size) { - return send_data(sock->fd, data, size); -} - -static bool tcp_recv_impl(socket_t * sock, void * data, size_t size) { - return recv_data(sock->fd, data, size); -} - -// RDMA transport (performance-optimized, auto-negotiated) - -#ifdef GGML_RPC_RDMA - -static bool rdma_send_impl(socket_t * sock, const void * data, size_t size); -static bool rdma_recv_impl(socket_t * sock, void * data, size_t size); - -static inline bool tcp_peer_closed(int fd) { - if (fd < 0) return false; -#ifndef _WIN32 - struct pollfd pfd = { fd, POLLIN | POLLRDHUP, 0 }; - int r = poll(&pfd, 1, 0); - return r > 0 && (pfd.revents & (POLLHUP | POLLERR | POLLRDHUP)); -#else - return false; -#endif -} - -static inline bool rdma_poll(struct ibv_cq * cq, struct ibv_wc * wc, int tcp_fd) { - for (uint64_t s = 0; ; s++) { - int n = ibv_poll_cq(cq, 1, wc); - if (n > 0) { - if (wc->status != IBV_WC_SUCCESS) { - GGML_LOG_ERROR("RDMA CQ wc error: status=%d (%s) vendor_err=0x%x\n", - wc->status, ibv_wc_status_str(wc->status), wc->vendor_err); - } - return wc->status == IBV_WC_SUCCESS; - } - if (n < 0) return false; - if ((s & 0xFFFFF) == 0 && s > 0) { - if (tcp_peer_closed(tcp_fd)) { - return false; - } - } - } -} - -static bool rdma_send(rdma_conn * c, const void * data, size_t size, int tcp_fd) { - const uint8_t * src = (const uint8_t *)data; - size_t rem = size; - while (rem > 0) { - size_t chunk = std::min(rem, RDMA_CHUNK); - - struct ibv_sge sge = {}; - struct ibv_send_wr wr = {}, * bad = nullptr; - wr.opcode = IBV_WR_SEND; - wr.sg_list = &sge; - wr.num_sge = 1; - - if (chunk <= c->max_inline) { - sge.addr = (uintptr_t)src; - sge.length = chunk; - wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_INLINE; - } else { - memcpy(c->tx_buf, src, chunk); - sge.addr = (uintptr_t)c->tx_buf; - sge.length = chunk; - sge.lkey = c->tx_mr->lkey; - wr.send_flags = IBV_SEND_SIGNALED; - } - - if (ibv_post_send(c->qp, &wr, &bad) != 0) return false; - struct ibv_wc wc; - if (!rdma_poll(c->scq, &wc, tcp_fd)) return false; - - src += chunk; - rem -= chunk; - } - return true; -} - - -static bool rdma_recv(rdma_conn * c, void * data, size_t size, int tcp_fd) { - uint8_t * dst = (uint8_t *)data; - size_t rem = size; - while (rem > 0) { - struct ibv_wc wc; - if (!rdma_poll(c->rcq, &wc, tcp_fd)) return false; - - int slot = (int)wc.wr_id; - size_t got = wc.byte_len; - memcpy(dst, c->rx_slot(slot), got); - - if (!c->post_rx(slot)) return false; - - dst += got; - rem -= got; - } - return true; -} - -static bool rdma_send_impl(socket_t * sock, const void * data, size_t size) { - return rdma_send(sock->rdma.get(), data, size, sock->fd); -} - -static bool rdma_recv_impl(socket_t * sock, void * data, size_t size) { - return rdma_recv(sock->rdma.get(), data, size, sock->fd); -} - -// Build a RoCE GID-shaped 16-byte target from a TCP socket's local address. -// Used to match the socket's local IP against the kernel's GID table so that -// a single memcmp handles IPv4, IPv4-mapped IPv6, and native IPv6 uniformly: -// AF_INET -> ::ffff:a.b.c.d (bytes 10-11 = 0xff, last 4 = IPv4) -// AF_INET6 (IPv4-mapped) -> ::ffff:a.b.c.d (already in GID shape) -// AF_INET6 (native v6) -> the 16-byte IPv6 address as-is -// Returns std::nullopt on unsupported family or getsockname failure. -static std::optional rdma_build_target_gid(sockfd_t tcp_fd) { - sockaddr_storage addr = {}; - socklen_t addr_len = sizeof(addr); - if (getsockname(tcp_fd, reinterpret_cast(&addr), &addr_len) != 0) { - return std::nullopt; - } - rdma_gid_t target = {}; - if (addr.ss_family == AF_INET) { - const auto * a = reinterpret_cast(&addr); - target[10] = 0xff; - target[11] = 0xff; - memcpy(&target[12], &a->sin_addr, 4); - return target; - } - if (addr.ss_family == AF_INET6) { - const auto * a = reinterpret_cast(&addr); - memcpy(target.data(), &a->sin6_addr, RDMA_GID_SIZE); - return target; - } - return std::nullopt; -} - -static rdma_conn * rdma_probe(sockfd_t tcp_fd, rdma_local_info * out) { - const char * dev_env = std::getenv("GGML_RDMA_DEV"); - const char * gid_env = std::getenv("GGML_RDMA_GID"); - - auto target_gid = rdma_build_target_gid(tcp_fd); - if (!target_gid) { - return nullptr; - } - - const uint8_t ib_port = 1; - int num_devs = 0; - ibv_device ** devs = ibv_get_device_list(&num_devs); - if (!devs || num_devs == 0) return nullptr; - - ibv_context * ibctx = nullptr; - const char * matched_dev = nullptr; - int gid_idx = gid_env ? atoi(gid_env) : -1; - int gid_version = IBV_GID_TYPE_IB; // 0 = unknown/IB - - for (int d = 0; d < num_devs; d++) { - const char * dn = ibv_get_device_name(devs[d]); - if (dev_env && strcmp(dev_env, dn) != 0) continue; - - ibv_context * ctx = ibv_open_device(devs[d]); - if (!ctx) continue; - - ibv_port_attr pa; - if (ibv_query_port(ctx, ib_port, &pa) != 0) { ibv_close_device(ctx); continue; } - - int found_gid = gid_idx; - int found_version = IBV_GID_TYPE_IB; - if (found_gid < 0) { - // Find a GID on this port whose bytes equal the local TCP address - // (IPv4 or IPv6). Prefer RoCE v2 (UDP/IP, L3-routable) over v1 - // (raw Ethernet, same-L2 only) so silent hangs on L3-routed paths - // are avoided. ibv_query_gid_ex returns gid+type in one call. - int v2_idx = -1; - int v1_idx = -1; - for (int i = 0; i < pa.gid_tbl_len; i++) { - ibv_gid_entry entry = {}; - if (ibv_query_gid_ex(ctx, ib_port, i, &entry, 0) != 0) continue; - if (memcmp(entry.gid.raw, target_gid->data(), RDMA_GID_SIZE) != 0) continue; - if (entry.gid_type == IBV_GID_TYPE_ROCE_V2 && v2_idx < 0) { - v2_idx = i; - } else if (entry.gid_type == IBV_GID_TYPE_ROCE_V1 && v1_idx < 0) { - v1_idx = i; - } - } - if (v2_idx >= 0) { - found_gid = v2_idx; - found_version = IBV_GID_TYPE_ROCE_V2; - } else if (v1_idx >= 0) { - found_gid = v1_idx; - found_version = IBV_GID_TYPE_ROCE_V1; - } - } else { - // Explicit GID index from GGML_RDMA_GID — fetch its type for logging. - ibv_gid_entry entry = {}; - if (ibv_query_gid_ex(ctx, ib_port, found_gid, &entry, 0) == 0) { - found_version = entry.gid_type; - } - } - if (found_gid >= 0) { - ibctx = ctx; - gid_idx = found_gid; - gid_version = found_version; - matched_dev = dn; - out->path_mtu = pa.active_mtu; - break; - } - ibv_close_device(ctx); - } - ibv_free_device_list(devs); - if (!ibctx) return nullptr; - - out->ib_port = ib_port; - out->gid_idx = gid_idx; - - // unique_ptr owns ibctx and every subsequent resource via ~rdma_conn(), - // so each failure path is a plain `return nullptr;`. - auto c = std::make_unique(); - c->ctx = ibctx; - - c->pd = ibv_alloc_pd(ibctx); - if (!c->pd) return nullptr; - - c->scq = ibv_create_cq(ibctx, 16, nullptr, nullptr, 0); - c->rcq = ibv_create_cq(ibctx, RDMA_RX_DEPTH + 4, nullptr, nullptr, 0); - if (!c->scq || !c->rcq) return nullptr; - - ibv_qp_init_attr qia = {}; - qia.send_cq = c->scq; - qia.recv_cq = c->rcq; - qia.qp_type = IBV_QPT_RC; - qia.cap.max_send_wr = 4; - qia.cap.max_recv_wr = RDMA_RX_DEPTH + 4; - qia.cap.max_send_sge = 1; - qia.cap.max_recv_sge = 1; - qia.cap.max_inline_data = 256; - - c->qp = ibv_create_qp(c->pd, &qia); - if (!c->qp) return nullptr; - c->max_inline = qia.cap.max_inline_data; - - c->tx_buf = aligned_alloc(4096, RDMA_CHUNK); - c->rx_buf = aligned_alloc(4096, static_cast(RDMA_RX_DEPTH) * RDMA_CHUNK); - if (!c->tx_buf || !c->rx_buf) return nullptr; - - c->tx_mr = ibv_reg_mr(c->pd, c->tx_buf, RDMA_CHUNK, IBV_ACCESS_LOCAL_WRITE); - c->rx_mr = ibv_reg_mr(c->pd, c->rx_buf, static_cast(RDMA_RX_DEPTH) * RDMA_CHUNK, - IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); - if (!c->tx_mr || !c->rx_mr) return nullptr; - - ibv_gid local_gid; - if (ibv_query_gid(ibctx, ib_port, gid_idx, &local_gid) != 0) return nullptr; - - out->qpn = c->qp->qp_num; - out->psn = c->qp->qp_num & 0xffffff; - memcpy(out->gid, &local_gid, RDMA_GID_SIZE); - - const char * ver_str = ""; - if (gid_version == IBV_GID_TYPE_ROCE_V2) { - ver_str = " RoCEv2"; - } else if (gid_version == IBV_GID_TYPE_ROCE_V1) { - ver_str = " RoCEv1"; - } - GGML_LOG_INFO("RDMA probed: dev=%s gid=%d%s qpn=%u inline=%u\n", - matched_dev, gid_idx, ver_str, out->qpn, c->max_inline); - return c.release(); -} - -// Phase 2: Given remote QPN/PSN/GID, transition QP: RESET->INIT->pre-post->RTR->RTS. -// On success, the connection is live and ready for rdma_send/rdma_recv. -static bool rdma_activate(rdma_conn * c, const rdma_local_info * local, - uint32_t remote_qpn, uint32_t remote_psn, const uint8_t * remote_gid) { - // RESET -> INIT - { - struct ibv_qp_attr a = {}; - a.qp_state = IBV_QPS_INIT; - a.port_num = local->ib_port; - a.pkey_index = 0; - a.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE; - if (ibv_modify_qp(c->qp, &a, - IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS) != 0) { - return false; - } - } - - for (int i = 0; i < RDMA_RX_DEPTH; i++) { - if (!c->post_rx(i)) return false; - } - - // INIT -> RTR - { - struct ibv_qp_attr a = {}; - a.qp_state = IBV_QPS_RTR; - a.path_mtu = local->path_mtu; - a.dest_qp_num = remote_qpn; - a.rq_psn = remote_psn; - a.max_dest_rd_atomic = 1; - a.min_rnr_timer = 1; - a.ah_attr.is_global = 1; - memcpy(&a.ah_attr.grh.dgid, remote_gid, RDMA_GID_SIZE); - a.ah_attr.grh.hop_limit = 1; - a.ah_attr.grh.sgid_index = local->gid_idx; - a.ah_attr.dlid = 0; - a.ah_attr.port_num = local->ib_port; - if (ibv_modify_qp(c->qp, &a, - IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | - IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER) != 0) { - return false; - } - } - - // RTR -> RTS - { - struct ibv_qp_attr a = {}; - a.qp_state = IBV_QPS_RTS; - a.timeout = 14; - a.retry_cnt = 7; - a.rnr_retry = 7; - a.sq_psn = local->psn; - a.max_rd_atomic = 1; - if (ibv_modify_qp(c->qp, &a, - IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | - IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC) != 0) { - return false; - } - } - - GGML_LOG_INFO("RDMA activated: qpn=%u->%u mtu=%d rx_depth=%d\n", - local->qpn, remote_qpn, 128 << local->path_mtu, RDMA_RX_DEPTH); - return true; -} - -#endif // GGML_RPC_RDMA - -// --------------------------------------------------------------------------- -// socket_t transport capability methods -// --------------------------------------------------------------------------- - -void socket_t::get_caps(uint8_t * caps) { - memset(caps, 0, RPC_CONN_CAPS_SIZE); -#ifdef GGML_RPC_RDMA - rdma_local = {}; - rdma.reset(rdma_probe(fd, &rdma_local)); - if (rdma) { - rdma_caps rc = {}; - rc.qpn = rdma_local.qpn; - rc.psn = rdma_local.psn; - memcpy(rc.gid, rdma_local.gid, RDMA_GID_SIZE); - memcpy(caps, &rc, sizeof(rc)); - } -#endif // GGML_RPC_RDMA -} - -void socket_t::update_caps(const uint8_t * remote_caps) { -#ifdef GGML_RPC_RDMA - if (!rdma) { - return; - } - rdma_caps rc = {}; - memcpy(&rc, remote_caps, sizeof(rc)); - if (rc.qpn == 0) { - rdma.reset(); - return; - } - if (rdma_activate(rdma.get(), &rdma_local, rc.qpn, rc.psn, rc.gid)) { - fn_send = rdma_send_impl; - fn_recv = rdma_recv_impl; - } else { - GGML_LOG_ERROR("RDMA activate failed, staying on TCP\n"); - rdma.reset(); +static bool send_msg(socket_ptr sock, const void * msg, size_t msg_size) { + if (!sock->send_data(&msg_size, sizeof(msg_size))) { + return false; } -#else - (void)remote_caps; -#endif // GGML_RPC_RDMA -} - -// unified transport dispatch (via function pointers) - -static bool send_data(socket_t * sock, const void * data, size_t size) { - return sock->fn_send(sock, data, size); -} - -static bool recv_data(socket_t * sock, void * data, size_t size) { - return sock->fn_recv(sock, data, size); -} - -static bool send_msg(socket_t * sock, const void * msg, size_t msg_size) { - if (!send_data(sock, &msg_size, sizeof(msg_size))) { + if (!sock->send_data(msg, msg_size)) { return false; } - return send_data(sock, msg, msg_size); + return sock->flush(); } -static bool recv_msg(socket_t * sock, void * msg, size_t msg_size) { +static bool recv_msg(socket_ptr sock, void * msg, size_t msg_size) { uint64_t size; - if (!recv_data(sock, &size, sizeof(size))) { + if (!sock->recv_data(&size, sizeof(size))) { return false; } if (size != msg_size) { return false; } - return recv_data(sock, msg, msg_size); + return sock->recv_data(msg, msg_size); } -static bool recv_msg(socket_t * sock, std::vector & input) { +static bool recv_msg(socket_ptr sock, std::vector & input) { uint64_t size; - if (!recv_data(sock, &size, sizeof(size))) { + if (!sock->recv_data(&size, sizeof(size))) { return false; } try { input.resize(size); } catch (const std::bad_alloc & e) { - GGML_LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size); + LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size); return false; } - return recv_data(sock, input.data(), size); + return sock->recv_data(input.data(), size); } static bool parse_endpoint(const std::string & endpoint, std::string & host, int & port) { @@ -964,34 +371,34 @@ static bool parse_endpoint(const std::string & endpoint, std::string & host, int // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) | // No response -static bool send_rpc_cmd(const std::shared_ptr & sock, enum rpc_cmd cmd, const void * input, size_t input_size) { +static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) { uint8_t cmd_byte = cmd; - if (!send_data(sock.get(), &cmd_byte, sizeof(cmd_byte))) { + if (!sock->send_data(&cmd_byte, sizeof(cmd_byte))) { return false; } - if (!send_data(sock.get(), &input_size, sizeof(input_size))) { + if (!sock->send_data(&input_size, sizeof(input_size))) { return false; } - if (!send_data(sock.get(), input, input_size)) { + if (!sock->send_data(input, input_size)) { return false; } - return true; + return sock->flush(); } // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) | // RPC response: | response_size (8 bytes) | response_data (response_size bytes) | -static bool send_rpc_cmd(const std::shared_ptr & sock, enum rpc_cmd cmd, const void * input, size_t input_size, void * output, size_t output_size) { +static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, void * output, size_t output_size) { if (!send_rpc_cmd(sock, cmd, input, input_size)) { return false; } uint64_t out_size; - if (!recv_data(sock.get(), &out_size, sizeof(out_size))) { + if (!sock->recv_data(&out_size, sizeof(out_size))) { return false; } if (out_size != output_size) { return false; } - if (!recv_data(sock.get(), output, output_size)) { + if (!sock->recv_data(output, output_size)) { return false; } return true; @@ -999,6 +406,17 @@ static bool send_rpc_cmd(const std::shared_ptr & sock, enum rpc_cmd cm // RPC client-side implementation +// with busy spinning on, the dispatcher still blocks on its queue after this long without commands +static constexpr auto RPC_BUSY_SPIN_IDLE_TIME = std::chrono::milliseconds(100); + +static inline void rpc_cpu_relax() { +#if defined(__aarch64__) && (defined(__clang__) || defined(__GNUC__)) + __asm__ volatile("yield" ::: "memory"); +#else + std::this_thread::yield(); +#endif +} + // Performs HELLO handshake with transport auto-negotiation. // Advertises local capabilities via conn_caps; if the server responds with // matching capabilities, the socket is upgraded transparently. @@ -1012,63 +430,305 @@ static bool negotiate_hello(const std::shared_ptr & sock) { RPC_STATUS_ASSERT(status); if (response.major != RPC_PROTO_MAJOR_VERSION || response.minor > RPC_PROTO_MINOR_VERSION) { - GGML_LOG_ERROR("RPC server version mismatch: %d.%d.%d\n", + LOG_ERROR("RPC server version mismatch: %d.%d.%d\n", response.major, response.minor, response.patch); return false; } sock->update_caps(response.conn_caps); + LOG_DBG("[%s] handshake ok: server version %d.%d.%d, transport: %s\n", + __func__, response.major, response.minor, response.patch, sock->transport_name()); return true; } -static std::shared_ptr get_socket(const std::string & endpoint) { - static std::mutex mutex; - std::lock_guard lock(mutex); - static std::unordered_map> sockets; - static bool initialized = false; +template +class message_queue { +public: + message_queue() {} - auto it = sockets.find(endpoint); - if (it != sockets.end()) { - if (auto sock = it->second.lock()) { - return sock; + bool push(const T &value) { + std::unique_lock lock(mutex); + if (interrupted) { + return false; } + queue.push(value); + cvar.notify_all(); + return true; + } + + bool pop(T* out) { + std::unique_lock lock(mutex); + cvar.wait(lock, [this] { return !queue.empty() || interrupted; }); + if (interrupted) { + return false; + } + *out = queue.front(); + queue.pop(); + return true; } + + bool try_pop(T* out) { + std::unique_lock lock(mutex); + if (interrupted || queue.empty()) { + return false; + } + *out = queue.front(); + queue.pop(); + return true; + } + + void interrupt() { + std::unique_lock lock(mutex); + interrupted = true; + lock.unlock(); + cvar.notify_all(); + } + +private: + bool interrupted = false; + std::queue queue; + std::mutex mutex; + std::condition_variable cvar; +}; + +class rpc_dispatcher { +public: + rpc_dispatcher() { + } + + void send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size); + void send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, void * output, size_t output_size); + void send_async(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size); + void send_async(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, void * output, size_t output_size); + + ggml_backend_event_t event_new(ggml_backend_dev_t dev); + void event_free(ggml_backend_event_t event); + void event_synchronize(ggml_backend_event_t event); + void event_record(ggml_backend_event_t event); + void synchronize(); + void busy_spin_acquire(); + void busy_spin_release(); + void graph_compute(uint32_t device, const ggml_cgraph * cgraph); + + void start(const std::string & endpoint); + void work(); + + ~rpc_dispatcher(); + +private: + struct rpc_msg { + rpc_cmd cmd; + std::shared_ptr input; + size_t input_size; + void * output; + size_t output_size; + std::promise completion; + }; + using rpc_msg_ptr = std::shared_ptr; + using rpc_msg_queue = message_queue; + struct rpc_event { + rpc_msg_ptr msg; + std::shared_future sf; + }; + std::mutex graph_mutex; + std::unordered_map> graph_uids; + rpc_msg_queue queue; + socket_ptr sock; + std::atomic_uint busy_spin_users = 0; + std::atomic_bool running; + std::thread thread; +}; + +static void rpc_dispatcher_trampoline(rpc_dispatcher * dispatcher) +{ + dispatcher->work(); +} + +void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size) { + auto msg = std::make_shared(); + msg->cmd = cmd; + msg->input = input; + msg->input_size = input_size; + msg->output = nullptr; + msg->output_size = 0; + GGML_ASSERT(queue.push(msg)); + auto future = msg->completion.get_future(); + future.wait(); +} + +void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size) { + auto msg = std::make_shared(); + msg->cmd = cmd; + msg->input = input; + msg->input_size = input_size; + msg->output = nullptr; + msg->output_size = 0; + GGML_ASSERT(queue.push(msg)); +} + +void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, void * output, size_t output_size) { + auto msg = std::make_shared(); + msg->cmd = cmd; + msg->input = input; + msg->input_size = input_size; + msg->output = output; + msg->output_size = output_size; + GGML_ASSERT(queue.push(msg)); + auto future = msg->completion.get_future(); + future.wait(); +} + +void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr input, size_t input_size, void * output, size_t output_size) { + auto msg = std::make_shared(); + msg->cmd = cmd; + msg->input = input; + msg->input_size = input_size; + msg->output = output; + msg->output_size = output_size; + GGML_ASSERT(queue.push(msg)); +} + +ggml_backend_event_t rpc_dispatcher::event_new(ggml_backend_dev_t dev) { + rpc_event * ev = new rpc_event; + ev->msg = std::make_shared(); + ev->msg->cmd = RPC_CMD_NONE; + ev->sf = ev->msg->completion.get_future().share(); + GGML_ASSERT(queue.push(ev->msg)); + return new ggml_backend_event { + /* .device = */ dev, + /* .context = */ ev, + }; +} + +void rpc_dispatcher::event_free(ggml_backend_event_t event) { + rpc_event * ev = (rpc_event *)event->context; + delete ev; +} + +void rpc_dispatcher::event_synchronize(ggml_backend_event_t event) { + rpc_event * ev = (rpc_event *)event->context; + ev->sf.wait(); +} + +void rpc_dispatcher::event_record(ggml_backend_event_t event) { + rpc_event * ev = (rpc_event *)event->context; + ev->msg = std::make_shared(); + ev->msg->cmd = RPC_CMD_NONE; + ev->sf = ev->msg->completion.get_future().share(); + GGML_ASSERT(queue.push(ev->msg)); +} + +void rpc_dispatcher::synchronize() { + // to ensure all messages are processed, submit dummy message and wait for it to complete + auto msg = std::make_shared(); + msg->cmd = RPC_CMD_NONE; + GGML_ASSERT(queue.push(msg)); + msg->completion.get_future().wait(); +} + +void rpc_dispatcher::busy_spin_acquire() { + busy_spin_users.fetch_add(1, std::memory_order_relaxed); +} + +void rpc_dispatcher::busy_spin_release() { + const unsigned previous = busy_spin_users.fetch_sub(1, std::memory_order_relaxed); + GGML_ASSERT(previous > 0); +} + +void rpc_dispatcher::start(const std::string & endpoint) { std::string host; int port; if (!parse_endpoint(endpoint, host, port)) { - GGML_LOG_ERROR("Failed to parse endpoint: %s\n", endpoint.c_str()); - return nullptr; + GGML_ABORT("Failed to parse endpoint: %s\n", endpoint.c_str()); } - -#ifdef _WIN32 - if (!initialized) { - WSADATA wsaData; - int res = WSAStartup(MAKEWORD(2, 2), &wsaData); - if (res != 0) { - return nullptr; - } - initialized = true; + if (!rpc_transport_init()) { + GGML_ABORT("RPC transport initialization failed\n"); } -#else - GGML_UNUSED(initialized); -#endif - auto sock = socket_connect(host.c_str(), port); + + sock = socket_t::connect(host.c_str(), port); if (sock == nullptr) { - return nullptr; + GGML_ABORT("Failed to connect to %s\n", endpoint.c_str()); } if (!negotiate_hello(sock)) { - return nullptr; + GGML_ABORT("RPC handshake failed for %s\n", endpoint.c_str()); } LOG_DBG("[%s] connected to %s\n", __func__, endpoint.c_str()); - sockets[endpoint] = sock; - return sock; + running = true; + thread = std::thread(rpc_dispatcher_trampoline, this); +} + +void rpc_dispatcher::work() { + auto last_cmd = std::chrono::steady_clock::now(); + while (running) { + rpc_msg_ptr msg_ptr; + // spin only while commands keep coming, so an idle dispatcher does not keep a core busy + const bool spin = busy_spin_users.load(std::memory_order_relaxed) != 0 && + std::chrono::steady_clock::now() - last_cmd < RPC_BUSY_SPIN_IDLE_TIME; + if (spin) { + if (!queue.try_pop(&msg_ptr)) { + rpc_cpu_relax(); + continue; + } + } else if (!queue.pop(&msg_ptr)) { + break; + } + if (msg_ptr->cmd != RPC_CMD_NONE) { + LOG_DBG2("[%s] %s (in: %s, out: %s)\n", __func__, rpc_cmd_name(msg_ptr->cmd), + rpc_log_format_size(msg_ptr->input_size).c_str(), rpc_log_format_size(msg_ptr->output_size).c_str()); + const bool trace = rpc_log_debug_level() >= 3; + const auto t0 = std::chrono::steady_clock::now(); + if (msg_ptr->output) { + bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->output, msg_ptr->output_size); + RPC_STATUS_ASSERT(status); + } else { + bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size); + RPC_STATUS_ASSERT(status); + } + if (trace) { + const std::chrono::duration elapsed = std::chrono::steady_clock::now() - t0; + LOG_DBG3("[%s] %s done in %.3f ms\n", __func__, rpc_cmd_name(msg_ptr->cmd), elapsed.count()); + } + } else { + LOG_DBG3("[%s] sync barrier\n", __func__); + } + msg_ptr->completion.set_value(); + last_cmd = std::chrono::steady_clock::now(); + } +} + +rpc_dispatcher::~rpc_dispatcher() { + running = false; + queue.interrupt(); + sock = nullptr; + if (thread.joinable()) { + thread.join(); + } +} + +static std::shared_ptr get_dispatcher(const std::string & endpoint) { + static std::mutex mutex; + std::lock_guard lock(mutex); + static std::unordered_map> dispatchers; + + auto it = dispatchers.find(endpoint); + if (it != dispatchers.end()) { + if (auto dispatcher = it->second.lock()) { + return dispatcher; + } + } + + auto dispatcher = std::make_shared(); + dispatcher->start(endpoint); + dispatchers[endpoint] = dispatcher; + return dispatcher; } static void ggml_backend_rpc_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; - rpc_msg_free_buffer_req request = {ctx->remote_ptr}; - bool status = send_rpc_cmd(ctx->sock, RPC_CMD_FREE_BUFFER, &request, sizeof(request), nullptr, 0); - RPC_STATUS_ASSERT(status); + auto request = std::make_shared(); + request->remote_ptr = ctx->remote_ptr; + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request->remote_ptr); + ctx->dispatcher->send(RPC_CMD_FREE_BUFFER, request, sizeof(*request)); delete ctx; } @@ -1077,10 +737,11 @@ static void * ggml_backend_rpc_buffer_get_base(ggml_backend_buffer_t buffer) { if (ctx->base_ptr != nullptr) { return ctx->base_ptr; } - rpc_msg_buffer_get_base_req request = {ctx->remote_ptr}; + auto request = std::make_shared(); + request->remote_ptr = ctx->remote_ptr; + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, ctx->remote_ptr); rpc_msg_buffer_get_base_rsp response; - bool status = send_rpc_cmd(ctx->sock, RPC_CMD_BUFFER_GET_BASE, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + ctx->dispatcher->send(RPC_CMD_BUFFER_GET_BASE, request, sizeof(*request), &response, sizeof(response)); ctx->base_ptr = reinterpret_cast(response.base_ptr); return ctx->base_ptr; } @@ -1089,7 +750,7 @@ static bool ggml_backend_buffer_is_rpc(ggml_backend_buffer_t buffer) { return buffer->iface.free_buffer == ggml_backend_rpc_buffer_free_buffer; } -static rpc_tensor serialize_tensor(const ggml_tensor * tensor) { +static rpc_tensor serialize_tensor(const ggml_tensor * tensor, const rpc_dispatcher * dispatcher = nullptr) { rpc_tensor result; if (!tensor) { memset(&result, 0, sizeof(result)); @@ -1101,8 +762,14 @@ static rpc_tensor serialize_tensor(const ggml_tensor * tensor) { if (tensor->buffer && ggml_backend_buffer_is_rpc(tensor->buffer)) { ggml_backend_buffer_t buffer = tensor->buffer; ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; - result.buffer = ctx != nullptr ? ctx->remote_ptr : 0; - result.data = reinterpret_cast(tensor->data); + // ref: https://github.com/ggml-org/llama.cpp/pull/26500 + if (ctx != nullptr && (dispatcher == nullptr || ctx->dispatcher.get() == dispatcher)) { + result.buffer = ctx->remote_ptr; + result.data = reinterpret_cast(tensor->data); + } else { + result.buffer = 0; + result.data = 0; + } } else { result.buffer = 0; result.data = 0; @@ -1124,7 +791,7 @@ static rpc_tensor serialize_tensor(const ggml_tensor * tensor) { // Avoid sending uninitialized data over the wire memset(result.name, 0, sizeof(result.name)); - memset(result.padding, 0, sizeof(result.padding)); + result.use_count = 0; snprintf(result.name, GGML_MAX_NAME, "%s", tensor->name); return result; @@ -1137,50 +804,127 @@ static enum ggml_status ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_ // Due to bandwidth constraints, we only call the server init tensor functions if necessary. // In particular, only quantized tensors need padding if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr)) { - rpc_msg_init_tensor_req request; - - request.tensor = serialize_tensor(tensor); - - bool status = send_rpc_cmd(ctx->sock, RPC_CMD_INIT_TENSOR, &request, sizeof(request), nullptr, 0); - RPC_STATUS_ASSERT(status); + LOG_DBG2("[%s] tensor: %s\n", __func__, rpc_log_format_tensor(tensor).c_str()); + auto request = std::make_shared(); + request->tensor = serialize_tensor(tensor); + ctx->dispatcher->send(RPC_CMD_INIT_TENSOR, request, sizeof(*request)); } return GGML_STATUS_SUCCESS; } +static void ggml_backend_rpc_buffer_memset_tensor( + ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + auto request = std::make_shared(); + request->tensor = serialize_tensor(tensor); + request->offset = offset; + request->size = size; + request->value = value; + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s, value: %u\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str(), value); + ctx->dispatcher->send(RPC_CMD_MEMSET_TENSOR, request, sizeof(*request)); +} + +// input serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) +static std::shared_ptr serialize_set_tensor(const rpc_tensor & rpc_tensor, uint8_t cache_flag, uint64_t offset, const void * data, size_t size, size_t & input_size) { + input_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset) + size; + uint8_t * input = new uint8_t[input_size](); + uint8_t * p = input; + memcpy(p, &rpc_tensor, sizeof(rpc_tensor)); p += sizeof(rpc_tensor); + memcpy(p, &cache_flag, sizeof(cache_flag)); p += sizeof(cache_flag); + memcpy(p, &offset, sizeof(offset)); p += sizeof(offset); + memcpy(p, data, size); + return std::shared_ptr(input, std::default_delete()); +} + +// the hash cache is meant for weights, so that a model reload can skip re-sending them. +// compute-buffer inputs (the activations ggml_backend_sched copies between backends) must not +// take this path, otherwise with `rpc-server -c` every ubatch above the threshold is written +// to the cache directory and later served from there. +static bool rpc_use_hash_cache(const ggml_tensor * tensor, size_t size) { + return size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; +} + static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; rpc_tensor rpc_tensor = serialize_tensor(tensor); - if (size > HASH_THRESHOLD) { - rpc_msg_set_tensor_hash_req request; - request.tensor = rpc_tensor; - request.offset = offset; - request.hash = fnv_hash((const uint8_t*)data, size); + uint8_t cache_flag = 0; + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str()); + if (rpc_use_hash_cache(tensor, size)) { + auto request = std::make_shared(); + request->tensor = rpc_tensor; + request->offset = offset; + request->hash = fnv_hash((const uint8_t*)data, size); rpc_msg_set_tensor_hash_rsp response; - bool status = send_rpc_cmd(ctx->sock, RPC_CMD_SET_TENSOR_HASH, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response)); + LOG_DBG2("[%s] tensor: %s, hash: 0x%" PRIx64 ", cache: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), + request->hash, response.result ? "hit" : "miss"); if (response.result) { // the server has the same data, no need to send it return; } + // the server has no cache entry for this tensor - ask it to save one + cache_flag = 1; + } + size_t input_size; + auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input, input_size); +} + +static void ggml_backend_rpc_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + rpc_tensor rpc_tensor = serialize_tensor(tensor); + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s, n_copies: %zu, stride: %zu\n", __func__, + rpc_log_format_tensor(tensor).c_str(), offset, rpc_log_format_size(size).c_str(), n_copies, stride_tensor); + // input serialization format: | rpc_tensor | offset (8 bytes) | size (8 bytes) | n_copies (8 bytes) | stride (8 bytes) | data (size * n_copies bytes) | + size_t input_size = sizeof(rpc_tensor) + 4*sizeof(uint64_t) + size*n_copies; + uint8_t * input = new uint8_t[input_size](); + uint8_t * dest = input; + memcpy(dest, &rpc_tensor, sizeof(rpc_tensor)); + dest += sizeof(rpc_tensor); + uint64_t header[4] = { offset, size, n_copies, stride_tensor }; + memcpy(dest, header, sizeof(header)); + dest += sizeof(header); + for (size_t i = 0; i < n_copies; i++) { + memcpy(dest + i*size, (const char *)data + i*stride_data, size); + } + std::shared_ptr input_ptr(input, std::default_delete()); + ctx->dispatcher->send(RPC_CMD_SET_TENSOR_2D, input_ptr, input_size); +} + +static void ggml_backend_rpc_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s, n_copies: %zu, stride: %zu\n", __func__, + rpc_log_format_tensor(tensor).c_str(), offset, rpc_log_format_size(size).c_str(), n_copies, stride_tensor); + auto request = std::make_shared(); + request->tensor = serialize_tensor(tensor); + request->offset = offset; + request->size = size; + request->n_copies = n_copies; + request->stride = stride_tensor; + if (stride_data == size) { + ctx->dispatcher->send(RPC_CMD_GET_TENSOR_2D, request, sizeof(*request), data, size*n_copies); + } else { + std::vector packed(size*n_copies); + ctx->dispatcher->send(RPC_CMD_GET_TENSOR_2D, request, sizeof(*request), packed.data(), packed.size()); + for (size_t i = 0; i < n_copies; i++) { + memcpy((char *)data + i*stride_data, packed.data() + i*size, size); + } } - // input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) - size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size; - std::vector input(input_size, 0); - memcpy(input.data(), &rpc_tensor, sizeof(rpc_tensor)); - memcpy(input.data() + sizeof(rpc_tensor), &offset, sizeof(offset)); - memcpy(input.data() + sizeof(rpc_tensor) + sizeof(offset), data, size); - bool status = send_rpc_cmd(ctx->sock, RPC_CMD_SET_TENSOR, input.data(), input.size()); - RPC_STATUS_ASSERT(status); } static void ggml_backend_rpc_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; - rpc_msg_get_tensor_req request; - request.tensor = serialize_tensor(tensor); - request.offset = offset; - request.size = size; - bool status = send_rpc_cmd(ctx->sock, RPC_CMD_GET_TENSOR, &request, sizeof(request), data, size); - RPC_STATUS_ASSERT(status); + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str()); + auto request = std::make_shared(); + request->tensor = serialize_tensor(tensor); + request->offset = offset; + request->size = size; + ctx->dispatcher->send(RPC_CMD_GET_TENSOR, request, sizeof(*request), data, size); } static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { @@ -1190,16 +934,17 @@ static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con ggml_backend_rpc_buffer_context * src_ctx = (ggml_backend_rpc_buffer_context *)src_buffer->context; ggml_backend_buffer_t dst_buffer = dst->buffer; ggml_backend_rpc_buffer_context * dst_ctx = (ggml_backend_rpc_buffer_context *)dst_buffer->context; - if (src_ctx->sock != dst_ctx->sock) { + if (src_ctx->dispatcher != dst_ctx->dispatcher) { + LOG_DBG2("[%s] src and dst are on different servers, falling back to a host copy\n", __func__); return false; } ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; - rpc_msg_copy_tensor_req request; - request.src = serialize_tensor(src); - request.dst = serialize_tensor(dst); + LOG_DBG2("[%s] src: %s -> dst: %s\n", __func__, rpc_log_format_tensor(src).c_str(), rpc_log_format_tensor(dst).c_str()); + auto request = std::make_shared(); + request->src = serialize_tensor(src); + request->dst = serialize_tensor(dst); rpc_msg_copy_tensor_rsp response; - bool status = send_rpc_cmd(ctx->sock, RPC_CMD_COPY_TENSOR, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + ctx->dispatcher->send(RPC_CMD_COPY_TENSOR, request, sizeof(*request), &response, sizeof(response)); return response.result; } return false; @@ -1207,20 +952,22 @@ static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con static void ggml_backend_rpc_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; - rpc_msg_buffer_clear_req request = {ctx->remote_ptr, value}; - bool status = send_rpc_cmd(ctx->sock, RPC_CMD_BUFFER_CLEAR, &request, sizeof(request), nullptr, 0); - RPC_STATUS_ASSERT(status); + auto request = std::make_shared(); + request->remote_ptr = ctx->remote_ptr; + request->value = value; + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 ", value: %u\n", __func__, request->remote_ptr, request->value); + ctx->dispatcher->send(RPC_CMD_BUFFER_CLEAR, request, sizeof(*request)); } static ggml_backend_buffer_i ggml_backend_rpc_buffer_interface = { /* .free_buffer = */ ggml_backend_rpc_buffer_free_buffer, /* .get_base = */ ggml_backend_rpc_buffer_get_base, /* .init_tensor = */ ggml_backend_rpc_buffer_init_tensor, - /* .memset_tensor = */ NULL, + /* .memset_tensor = */ ggml_backend_rpc_buffer_memset_tensor, /* .set_tensor = */ ggml_backend_rpc_buffer_set_tensor, /* .get_tensor = */ ggml_backend_rpc_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, - /* .get_tensor_2d = */ NULL, + /* .set_tensor_2d = */ ggml_backend_rpc_buffer_set_tensor_2d, + /* .get_tensor_2d = */ ggml_backend_rpc_buffer_get_tensor_2d, /* .cpy_tensor = */ ggml_backend_rpc_buffer_cpy_tensor, /* .clear = */ ggml_backend_rpc_buffer_clear, /* .reset = */ NULL, @@ -1233,27 +980,35 @@ static const char * ggml_backend_rpc_buffer_type_name(ggml_backend_buffer_type_t static ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; - rpc_msg_alloc_buffer_req request = {buft_ctx->device, size}; + auto request = std::make_shared(); + request->device = buft_ctx->device; + request->size = size; rpc_msg_alloc_buffer_rsp response; - auto sock = get_socket(buft_ctx->endpoint); - bool status = send_rpc_cmd(sock, RPC_CMD_ALLOC_BUFFER, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + + auto dispatcher = get_dispatcher(buft_ctx->endpoint); + dispatcher->send(RPC_CMD_ALLOC_BUFFER, request, sizeof(*request), &response, sizeof(response)); if (response.remote_ptr != 0) { + LOG_DBG("[%s] endpoint: %s, device: %u, size: %s -> remote_ptr: 0x%" PRIx64 ", remote_size: %s\n", + __func__, buft_ctx->endpoint.c_str(), buft_ctx->device, rpc_log_format_size(request->size).c_str(), + response.remote_ptr, rpc_log_format_size(response.remote_size).c_str()); ggml_backend_buffer_t buffer = ggml_backend_buffer_init(buft, ggml_backend_rpc_buffer_interface, - new ggml_backend_rpc_buffer_context{sock, nullptr, response.remote_ptr}, + new ggml_backend_rpc_buffer_context{dispatcher, nullptr, response.remote_ptr}, response.remote_size); return buffer; } else { + LOG_WARN("[%s] endpoint: %s, device: %u, size: %s -> failed\n", + __func__, buft_ctx->endpoint.c_str(), buft_ctx->device, rpc_log_format_size(request->size).c_str()); return nullptr; } } -static size_t get_alignment(const std::shared_ptr & sock, uint32_t device) { - rpc_msg_get_alignment_req request = {device}; +static size_t get_alignment(const std::shared_ptr & dispatcher, uint32_t device) { + auto request = std::make_shared(); + request->device = device; rpc_msg_get_alignment_rsp response; - bool status = send_rpc_cmd(sock, RPC_CMD_GET_ALIGNMENT, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + dispatcher->send(RPC_CMD_GET_ALIGNMENT, request, sizeof(*request), &response, sizeof(response)); + LOG_DBG2("[%s] device: %u, alignment: %s\n", __func__, device, rpc_log_format_size(response.alignment).c_str()); return response.alignment; } @@ -1262,11 +1017,12 @@ static size_t ggml_backend_rpc_buffer_type_get_alignment(ggml_backend_buffer_typ return buft_ctx->alignment; } -static size_t get_max_size(const std::shared_ptr & sock, uint32_t device) { - rpc_msg_get_max_size_req request = {device}; +static size_t get_max_size(const std::shared_ptr & dispatcher, uint32_t device) { + auto request = std::make_shared(); + request->device = device; rpc_msg_get_max_size_rsp response; - bool status = send_rpc_cmd(sock, RPC_CMD_GET_MAX_SIZE, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + dispatcher->send(RPC_CMD_GET_MAX_SIZE, request, sizeof(*request), &response, sizeof(response)); + LOG_DBG2("[%s] device: %u, max_size: %s\n", __func__, device, rpc_log_format_size(response.max_size).c_str()); return response.max_size; } @@ -1282,44 +1038,99 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty // See comments in init_tensor. rpc_get |= ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr); - // ops that require additional memory for fleeting data on certain backends + // [TAG_ALLOC_SIZE_EXPAND] + // ops that may require additional memory for fleeting data on certain backends // ref: https://github.com/ggml-org/llama.cpp/pull/15966 - rpc_get |= tensor->op == GGML_OP_FLASH_ATTN_EXT; - rpc_get |= tensor->op == GGML_OP_MUL_MAT_ID; + rpc_get |= ggml_op_alloc_size_may_expand(tensor->op); if (rpc_get) { ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; - auto sock = get_socket(buft_ctx->endpoint); - rpc_msg_get_alloc_size_req request = { - /*.device =*/ buft_ctx->device, - /*.tensor =*/ serialize_tensor(tensor), - /*.srcs =*/ {}, + // the reported size must never be below ggml_nbytes: rpc_tensor stores nb[] as uint32_t, + // so a stride over 4 GiB is truncated on the wire and the remote size comes back too small + // TODO: change rpc_tensor nb to 64-bit int + const size_t min_size = ggml_nbytes(tensor); + + // Cache key for calls to read the alloc_size. + // We deliberately exclude src tensor dimensions from the key because: + // 1. For CPU backends, alloc_size = ggml_nbytes(output) regardless of src shapes + // 2. For GPU backends, the reservation graph uses max dimensions, so the + // cached value from reservation is always >= any subsequent request + // 3. Including src dims causes cache misses per-ubatch (e.g. growing KV cache) + // which blocks the main thread behind in-flight GRAPH_COMPUTE commands + struct alloc_size_cache_key { + uint32_t device; + uint32_t type; + uint32_t op; + int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; + uint32_t ne[GGML_MAX_DIMS]; + uint64_t nb[GGML_MAX_DIMS]; }; + alloc_size_cache_key key = {}; + key.device = buft_ctx->device; + key.type = tensor->type; + key.op = tensor->op; + memcpy(key.op_params, tensor->op_params, sizeof(key.op_params)); + for (int i = 0; i < GGML_MAX_DIMS; i++) { + key.ne[i] = (uint32_t)tensor->ne[i]; + key.nb[i] = (uint64_t)tensor->nb[i]; + } + + uint64_t cache_hash = fnv_hash((const uint8_t *)&key, sizeof(key)); + cache_hash = fnv_hash((const uint8_t *)buft_ctx->endpoint.data(), buft_ctx->endpoint.size(), cache_hash); + + // alloc sizes are immutable for a given tensor configuration + static std::mutex cache_mutex; + static std::unordered_map cache; + + { + std::lock_guard lock(cache_mutex); + auto it = cache.find(cache_hash); + if (it != cache.end()) { + LOG_DBG3("[%s] cache hit: %s -> %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), + rpc_log_format_size(it->second).c_str()); + return std::max(it->second, min_size); + } + } + + auto request = std::make_shared(); + request->device = buft_ctx->device; + request->tensor = serialize_tensor(tensor); + // .get_alloc_size could be a function of the tensor's srcs, so we must serialize them as well for (int i = 0; i < GGML_MAX_SRC; i++) { - request.srcs[i] = serialize_tensor(tensor->src[i]); + request->srcs[i] = serialize_tensor(tensor->src[i]); } - // TODO: cache the alloc responses to avoid extra RPC calls? + LOG_DBG2("[%s] cache miss: %s, querying server\n", __func__, rpc_log_format_tensor(tensor).c_str()); rpc_msg_get_alloc_size_rsp response; - bool status = send_rpc_cmd(sock, RPC_CMD_GET_ALLOC_SIZE, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + auto dispatcher = get_dispatcher(buft_ctx->endpoint); + dispatcher->send(RPC_CMD_GET_ALLOC_SIZE, request, sizeof(*request), &response, sizeof(response)); + + { + std::lock_guard lock(cache_mutex); + cache[cache_hash] = response.alloc_size; + } + + LOG_DBG2("[%s] %s -> alloc_size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), + rpc_log_format_size(response.alloc_size).c_str()); - return response.alloc_size; + return std::max(response.alloc_size, min_size); } return ggml_nbytes(tensor); } static ggml_backend_buffer_type_i ggml_backend_rpc_buffer_type_interface = { - /* .get_name = */ ggml_backend_rpc_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_rpc_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_rpc_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_rpc_get_max_size, - /* .get_alloc_size = */ ggml_backend_rpc_buffer_type_get_alloc_size, - /* .is_host = */ NULL, + /* .get_name = */ ggml_backend_rpc_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_rpc_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_rpc_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_rpc_get_max_size, + /* .get_alloc_size = */ ggml_backend_rpc_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ NULL, }; static const char * ggml_backend_rpc_name(ggml_backend_t backend) { @@ -1334,12 +1145,51 @@ static void ggml_backend_rpc_free(ggml_backend_t backend) { delete backend; } +static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context; + rpc_tensor rpc_tensor = serialize_tensor(tensor); + uint8_t cache_flag = 0; + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str()); + if (rpc_use_hash_cache(tensor, size)) { + auto request = std::make_shared(); + request->tensor = rpc_tensor; + request->offset = offset; + request->hash = fnv_hash((const uint8_t*)data, size); + rpc_msg_set_tensor_hash_rsp response; + // TODO: make this async + ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response)); + LOG_DBG2("[%s] tensor: %s, hash: 0x%" PRIx64 ", cache: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), + request->hash, response.result ? "hit" : "miss"); + if (response.result) { + // the server has the same data, no need to send it + return; + } + // the server has no cache entry for this tensor - ask it to save one + cache_flag = 1; + } + size_t input_size; + auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size); + ctx->dispatcher->send_async(RPC_CMD_SET_TENSOR, input, input_size); +} + +static void ggml_backend_rpc_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context; + LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset, + rpc_log_format_size(size).c_str()); + auto request = std::make_shared(); + request->tensor = serialize_tensor(tensor); + request->offset = offset; + request->size = size; + ctx->dispatcher->send_async(RPC_CMD_GET_TENSOR, request, sizeof(*request), data, size); +} + static void ggml_backend_rpc_synchronize(ggml_backend_t backend) { - GGML_UNUSED(backend); - // this is no-op because we don't have any async operations + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; + rpc_ctx->dispatcher->synchronize(); } -static void add_tensor(ggml_tensor * tensor, std::vector & tensors, std::unordered_set & visited) { +static void add_tensor(ggml_tensor * tensor, const ggml_cgraph * cgraph, const rpc_dispatcher * dispatcher, std::vector & tensors, std::unordered_set & visited) { if (tensor == nullptr) { return; } @@ -1348,27 +1198,34 @@ static void add_tensor(ggml_tensor * tensor, std::vector & tensors, } visited.insert(tensor); for (int i = 0; i < GGML_MAX_SRC; i++) { - add_tensor(tensor->src[i], tensors, visited); + add_tensor(tensor->src[i], cgraph, dispatcher, tensors, visited); + } + add_tensor(tensor->view_src, cgraph, dispatcher, tensors, visited); + rpc_tensor result = serialize_tensor(tensor, dispatcher); + const size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, tensor); + if (hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) { + result.use_count = cgraph->use_counts[hash_pos]; } - add_tensor(tensor->view_src, tensors, visited); - tensors.push_back(serialize_tensor(tensor)); + tensors.push_back(result); } -static void serialize_graph(uint32_t device, const ggml_cgraph * cgraph, std::vector & output) { +static uint8_t * serialize_graph(uint32_t device, const ggml_cgraph * cgraph, const rpc_dispatcher * dispatcher, size_t * output_size) { uint32_t n_nodes = cgraph->n_nodes; std::vector tensors; std::unordered_set visited; for (uint32_t i = 0; i < n_nodes; i++) { - add_tensor(cgraph->nodes[i], tensors, visited); + add_tensor(cgraph->nodes[i], cgraph, dispatcher, tensors, visited); } // serialization format: - // | device (4 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | + // | device (4 bytes) | uid (8 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | uint32_t n_tensors = tensors.size(); - int output_size = 2*sizeof(uint32_t) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t) + n_tensors * sizeof(rpc_tensor); - output.resize(output_size, 0); - uint8_t * dest = output.data(); + *output_size = 2*sizeof(uint32_t) + sizeof(uint64_t) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t) + n_tensors * sizeof(rpc_tensor); + uint8_t * output = new uint8_t[*output_size](); + uint8_t * dest = output; memcpy(dest, &device, sizeof(device)); dest += sizeof(device); + memcpy(dest, &cgraph->uid, sizeof(cgraph->uid)); + dest += sizeof(cgraph->uid); memcpy(dest, &n_nodes, sizeof(n_nodes)); dest += sizeof(n_nodes); for (uint32_t i = 0; i < n_nodes; i++) { @@ -1379,46 +1236,82 @@ static void serialize_graph(uint32_t device, const ggml_cgraph * cgraph, std::ve dest += sizeof(n_tensors); rpc_tensor * out_tensors = (rpc_tensor *)dest; memcpy(out_tensors, tensors.data(), n_tensors * sizeof(rpc_tensor)); + return output; } -static enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { - ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; - +void rpc_dispatcher::graph_compute(uint32_t device, const ggml_cgraph * cgraph) { + std::lock_guard lock(graph_mutex); GGML_ASSERT(cgraph->n_nodes > 0); - bool reuse = rpc_ctx->gc.is_cached(cgraph); + auto & device_graph_uids = graph_uids[device]; + bool reuse = cgraph->uid != 0 && device_graph_uids.count(cgraph->uid) > 0; + LOG_DBG("[%s] device: %u, uid: %" PRIu64 ", n_nodes: %u, reuse: %s\n", __func__, device, cgraph->uid, cgraph->n_nodes, reuse ? "yes" : "no"); if (reuse) { - rpc_msg_graph_recompute_req request; - request.device = rpc_ctx->device; - auto sock = get_socket(rpc_ctx->endpoint); - bool status = send_rpc_cmd(sock, RPC_CMD_GRAPH_RECOMPUTE, &request, sizeof(request)); - RPC_STATUS_ASSERT(status); + auto request = std::make_shared(); + request->device = device; + request->uid = cgraph->uid; + send_async(RPC_CMD_GRAPH_RECOMPUTE, request, sizeof(*request)); } else { - rpc_ctx->gc.add(cgraph); - std::vector input; - serialize_graph(rpc_ctx->device, cgraph, input); - auto sock = get_socket(rpc_ctx->endpoint); - bool status = send_rpc_cmd(sock, RPC_CMD_GRAPH_COMPUTE, input.data(), input.size()); - RPC_STATUS_ASSERT(status); + if (cgraph->uid != 0) { + if (device_graph_uids.size() >= GRAPH_CACHE_MAX) { + LOG_DBG("[%s] clearing graph cache for device %u (%zu entries)\n", + __func__, device, device_graph_uids.size()); + device_graph_uids.clear(); + } + device_graph_uids.insert(cgraph->uid); + } + size_t input_size = 0; + uint8_t * input = serialize_graph(device, cgraph, this, &input_size); + LOG_DBG2("[%s] serialized graph: %s\n", __func__, rpc_log_format_size(input_size).c_str()); + std::shared_ptr input_ptr(input, std::default_delete()); + send_async(RPC_CMD_GRAPH_COMPUTE, input_ptr, input_size); } +} + +static enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; + rpc_ctx->dispatcher->graph_compute(rpc_ctx->device, cgraph); return GGML_STATUS_SUCCESS; } +static void ggml_backend_rpc_event_record(ggml_backend_t backend, ggml_backend_event_t event) { + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; + rpc_ctx->dispatcher->event_record(event); +} + +static void ggml_backend_rpc_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { + // this is noop for RPC as we have a single stream + GGML_UNUSED(backend); + GGML_UNUSED(event); +} + +static void ggml_backend_rpc_set_tensor_2d_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + ggml_backend_tensor_set_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); + GGML_UNUSED(backend); +} + +static void ggml_backend_rpc_get_tensor_2d_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + ggml_backend_tensor_get_2d(tensor, data, offset, size, n_copies, stride_tensor, stride_data); + GGML_UNUSED(backend); +} + static ggml_backend_i ggml_backend_rpc_interface = { /* .get_name = */ ggml_backend_rpc_name, /* .free = */ ggml_backend_rpc_free, - /* .set_tensor_async = */ NULL, - /* .get_tensor_async = */ NULL, + /* .set_tensor_async = */ ggml_backend_rpc_set_tensor_async, + /* .get_tensor_async = */ ggml_backend_rpc_get_tensor_async, + /* .set_tensor_2d_async = */ ggml_backend_rpc_set_tensor_2d_async, + /* .get_tensor_2d_async = */ ggml_backend_rpc_get_tensor_2d_async, /* .cpy_tensor_async = */ NULL, - /* .get_tensor_2d_async = */ NULL, - /* .set_tensor_2d_async = */ NULL, /* .synchronize = */ ggml_backend_rpc_synchronize, /* .graph_plan_create = */ NULL, /* .graph_plan_free = */ NULL, /* .graph_plan_update = */ NULL, /* .graph_plan_compute = */ NULL, /* .graph_compute = */ ggml_backend_rpc_graph_compute, - /* .event_record = */ NULL, - /* .event_wait = */ NULL, + /* .event_record = */ ggml_backend_rpc_event_record, + /* .event_wait = */ ggml_backend_rpc_event_wait, /* .graph_optimize = */ NULL, }; @@ -1432,13 +1325,9 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u if (it != buft_map.end()) { return it->second; } - auto sock = get_socket(endpoint); - if (sock == nullptr) { - GGML_LOG_ERROR("Failed to connect to %s\n", endpoint); - return nullptr; - } - size_t alignment = get_alignment(sock, device); - size_t max_size = get_max_size(sock, device); + auto dispatcher = get_dispatcher(endpoint); + size_t alignment = get_alignment(dispatcher, device); + size_t max_size = get_max_size(dispatcher, device); ggml_backend_rpc_buffer_type_context * buft_ctx = new ggml_backend_rpc_buffer_type_context { /* .endpoint = */ endpoint, /* .device = */ device, @@ -1453,18 +1342,22 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u /* .context = */ buft_ctx }; buft_map[buft_name] = buft; + LOG_DBG("[%s] created %s (alignment: %s, max_size: %s)\n", __func__, buft_name.c_str(), + rpc_log_format_size(alignment).c_str(), rpc_log_format_size(max_size).c_str()); return buft; } ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device) { std::string dev_name = "RPC" + std::to_string(device) + "[" + std::string(endpoint) + "]"; + auto dispatcher = get_dispatcher(endpoint); ggml_backend_rpc_context * ctx = new ggml_backend_rpc_context { - /* .endpoint = */ endpoint, - /* .device = */ device, - /* .name = */ dev_name, - /* .gc = */ {}, + /* .dispatcher = */ dispatcher, + /* .endpoint = */ endpoint, + /* .device = */ device, + /* .name = */ dev_name, }; auto reg = ggml_backend_rpc_add_server(endpoint); + LOG_DBG("[%s] %s\n", __func__, dev_name.c_str()); ggml_backend_t backend = new ggml_backend { /* .guid = */ ggml_backend_rpc_guid(), /* .iface = */ ggml_backend_rpc_interface, @@ -1478,26 +1371,18 @@ bool ggml_backend_is_rpc(ggml_backend_t backend) { return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_rpc_guid()); } -static void get_device_memory(const std::shared_ptr & sock, uint32_t device, size_t * free, size_t * total) { - rpc_msg_get_device_memory_req request; - request.device = device; +void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total) { + auto dispatcher = get_dispatcher(endpoint); + auto request = std::make_shared(); + request->device = device; rpc_msg_get_device_memory_rsp response; - bool status = send_rpc_cmd(sock, RPC_CMD_GET_DEVICE_MEMORY, &request, sizeof(request), &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + dispatcher->send(RPC_CMD_GET_DEVICE_MEMORY, request, sizeof(*request), &response, sizeof(response)); + LOG_DBG2("[%s] device: %u, free: %s, total: %s\n", __func__, device, + rpc_log_format_size(response.free_mem).c_str(), rpc_log_format_size(response.total_mem).c_str()); *free = response.free_mem; *total = response.total_mem; } -void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total) { - auto sock = get_socket(endpoint); - if (sock == nullptr) { - *free = 0; - *total = 0; - return; - } - get_device_memory(sock, device, free, total); -} - // RPC server-side implementation class rpc_server { @@ -1505,6 +1390,7 @@ class rpc_server { rpc_server(std::vector all_backends, const char * cache_dir) : backends(std::move(all_backends)), cache_dir(cache_dir) { stored_graphs.resize(backends.size()); + comm_states.resize(backends.size()); } ~rpc_server(); @@ -1515,12 +1401,18 @@ class rpc_server { bool buffer_get_base(const rpc_msg_buffer_get_base_req & request, rpc_msg_buffer_get_base_rsp & response); bool free_buffer(const rpc_msg_free_buffer_req & request); bool buffer_clear(const rpc_msg_buffer_clear_req & request); + bool memset_tensor(const rpc_msg_memset_tensor_req & request); bool set_tensor(const std::vector & input); + bool set_tensor_2d(const std::vector & input); bool set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response); bool get_tensor(const rpc_msg_get_tensor_req & request, std::vector & response); + bool get_tensor_2d(const rpc_msg_get_tensor_2d_req & request, std::vector & response); bool copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response); bool graph_compute(const std::vector & input); bool graph_recompute(const rpc_msg_graph_recompute_req & request); + bool comm_init(const rpc_msg_comm_init_req & request, rpc_msg_comm_init_rsp & response); + bool comm_allreduce(const rpc_msg_comm_allreduce_req & request); + bool comm_free(const rpc_msg_comm_free_req & request); bool init_tensor(const rpc_msg_init_tensor_req & request); bool get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response); bool get_device_memory(const rpc_msg_get_device_memory_req & request, rpc_msg_get_device_memory_rsp & response); @@ -1531,6 +1423,7 @@ class rpc_server { }; private: + void sync_all_backends(); bool get_cached_file(uint64_t hash, std::vector & data); ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor); ggml_tensor * create_node(uint64_t id, @@ -1539,11 +1432,23 @@ class rpc_server { std::unordered_map & tensor_map); + // pairwise allreduce over a direct connection to the peer server + struct comm_state { + socket_ptr peer; + uint32_t rank = 0; + uint32_t world = 0; + ggml_backend_buffer_ptr scratch; + size_t scratch_size = 0; + std::vector send_buf; + std::vector recv_buf; + }; + std::vector backends; const char * cache_dir; std::unordered_set buffers; - // store the last computed graph for each backend - std::vector stored_graphs; + // computed graphs cached per backend, keyed by uid + std::vector> stored_graphs; + std::vector comm_states; }; void rpc_server::hello(rpc_msg_hello_rsp & response) { @@ -1571,7 +1476,7 @@ bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_ ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr) { - GGML_LOG_ERROR("Null tensor pointer passed to server get_alloc_size function.\n"); + LOG_ERROR("Null tensor pointer passed to server get_alloc_size function.\n"); return false; } for (int i = 0; i < GGML_MAX_SRC; i++) { @@ -1580,7 +1485,8 @@ bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_ } } - LOG_DBG("[%s] device: %d, buffer: %p, data: %p\n", __func__, dev_id, (void*)tensor->buffer, tensor->data); + LOG_DBG2("[%s] device: %u, tensor: %s, buffer: %p, data: %p\n", __func__, dev_id, rpc_log_format_tensor(tensor).c_str(), + (void*)tensor->buffer, tensor->data); if (tensor->buffer == nullptr) { //No buffer allocated. buft = ggml_backend_get_default_buffer_type(backends[dev_id]); @@ -1605,11 +1511,12 @@ bool rpc_server::alloc_buffer(const rpc_msg_alloc_buffer_req & request, rpc_msg_ if (buffer != nullptr) { response.remote_ptr = reinterpret_cast(buffer); response.remote_size = buffer->size; - LOG_DBG("[%s] device: %d, size: %" PRIu64 " -> remote_ptr: %" PRIx64 ", remote_size: %" PRIu64 "\n", - __func__, dev_id, request.size, response.remote_ptr, response.remote_size); + LOG_DBG("[%s] device: %u, size: %s -> remote_ptr: 0x%" PRIx64 ", remote_size: %s\n", + __func__, dev_id, rpc_log_format_size(request.size).c_str(), response.remote_ptr, + rpc_log_format_size(response.remote_size).c_str()); buffers.insert(buffer); } else { - LOG_DBG("[%s] device: %d, size: %" PRIu64 " -> failed\n", __func__, dev_id, request.size); + LOG_WARN("[%s] device: %u, size: %s -> failed\n", __func__, dev_id, rpc_log_format_size(request.size).c_str()); } return true; } @@ -1621,7 +1528,7 @@ bool rpc_server::get_alignment(const rpc_msg_get_alignment_req & request, rpc_ms } ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]); size_t alignment = ggml_backend_buft_get_alignment(buft); - LOG_DBG("[%s] device: %d, alignment: %lu\n", __func__, dev_id, alignment); + LOG_DBG2("[%s] device: %u, alignment: %s\n", __func__, dev_id, rpc_log_format_size(alignment).c_str()); response.alignment = alignment; return true; } @@ -1633,16 +1540,16 @@ bool rpc_server::get_max_size(const rpc_msg_get_max_size_req & request, rpc_msg_ } ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]); size_t max_size = ggml_backend_buft_get_max_size(buft); - LOG_DBG("[%s] device: %d, max_size: %lu\n", __func__, dev_id, max_size); + LOG_DBG2("[%s] device: %u, max_size: %s\n", __func__, dev_id, rpc_log_format_size(max_size).c_str()); response.max_size = max_size; return true; } bool rpc_server::buffer_get_base(const rpc_msg_buffer_get_base_req & request, rpc_msg_buffer_get_base_rsp & response) { - LOG_DBG("[%s] remote_ptr: %" PRIx64 "\n", __func__, request.remote_ptr); + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request.remote_ptr); ggml_backend_buffer_t buffer = reinterpret_cast(request.remote_ptr); if (buffers.find(buffer) == buffers.end()) { - GGML_LOG_ERROR("[%s] buffer not found\n", __func__); + LOG_ERROR("[%s] buffer not found\n", __func__); return false; } void * base = ggml_backend_buffer_get_base(buffer); @@ -1651,38 +1558,96 @@ bool rpc_server::buffer_get_base(const rpc_msg_buffer_get_base_req & request, rp } bool rpc_server::free_buffer(const rpc_msg_free_buffer_req & request) { - LOG_DBG("[%s] remote_ptr: %" PRIx64 "\n", __func__, request.remote_ptr); + sync_all_backends(); + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request.remote_ptr); ggml_backend_buffer_t buffer = reinterpret_cast(request.remote_ptr); if (buffers.find(buffer) == buffers.end()) { - GGML_LOG_ERROR("[%s] buffer not found\n", __func__); + LOG_ERROR("[%s] buffer not found\n", __func__); return false; } + // Discard all cached graphs to avoid use-after-free in graph_recompute, + // since their nodes may hold pointers to the buffer being freed. The client + // will have to re-serialize its graphs from now on. + for (auto & sgs : stored_graphs) { + for (auto & sg : sgs) { + sg.second.graph = nullptr; + } + } ggml_backend_buffer_free(buffer); buffers.erase(buffer); return true; } bool rpc_server::buffer_clear(const rpc_msg_buffer_clear_req & request) { - LOG_DBG("[%s] remote_ptr: %" PRIx64 ", value: %u\n", __func__, request.remote_ptr, request.value); + sync_all_backends(); + LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 ", value: %u\n", __func__, request.remote_ptr, request.value); ggml_backend_buffer_t buffer = reinterpret_cast(request.remote_ptr); if (buffers.find(buffer) == buffers.end()) { - GGML_LOG_ERROR("[%s] buffer not found\n", __func__); + LOG_ERROR("[%s] buffer not found\n", __func__); return false; } ggml_backend_buffer_clear(buffer, request.value); return true; } +bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) { + sync_all_backends(); + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_context * ctx = ctx_ptr.get(); + ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); + if (tensor == nullptr || tensor->buffer == nullptr) { + LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + + const uint64_t tensor_size = ggml_nbytes(tensor); + if (request.offset > tensor_size || request.size > tensor_size - request.offset) { + LOG_ERROR("[%s] tensor region (offset=%" PRIu64 ", size=%" PRIu64 ") out of tensor bounds [0, %" PRIu64 ")\n", + __func__, request.offset, request.size, tensor_size); + return false; + } + + const uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(tensor->buffer); + const uint64_t buffer_size = ggml_backend_buffer_get_size(tensor->buffer); + if (request.tensor.data < buffer_start) { + LOG_ERROR("[%s] tensor data before buffer start\n", __func__); + return false; + } + const uint64_t data_offset = request.tensor.data - buffer_start; + if (data_offset > buffer_size || + request.offset > buffer_size - data_offset || + request.size > buffer_size - data_offset - request.offset) { + LOG_ERROR("[%s] tensor region out of buffer bounds\n", __func__); + return false; + } + if (tensor->buffer->iface.memset_tensor == nullptr) { + LOG_ERROR("[%s] memset not implemented by backend buffer\n", __func__); + return false; + } + + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s, value: %u\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void *) tensor->buffer, tensor->data, request.offset, + rpc_log_format_size(request.size).c_str(), request.value); + ggml_backend_tensor_memset(tensor, request.value, request.offset, request.size); + return true; +} + ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor) { // Validate tensor type before using it if (tensor->type >= GGML_TYPE_COUNT) { - GGML_LOG_ERROR("[%s] invalid tensor type received: %u\n", __func__, tensor->type); + LOG_ERROR("[%s] invalid tensor type received: %u\n", __func__, tensor->type); return nullptr; } // Fix: Prevent division by zero if blck_size is 0 (e.g., deprecated types) if (ggml_blck_size((enum ggml_type)tensor->type) == 0) { - GGML_LOG_ERROR("[%s] invalid tensor type received (blck_size is 0): %u\n", __func__, tensor->type); + LOG_ERROR("[%s] invalid tensor type received (blck_size is 0): %u\n", __func__, tensor->type); return nullptr; } @@ -1691,7 +1656,7 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp // ggml_new_tensor_4d might fail if dimensions are invalid, although less likely to crash than invalid type if (result == nullptr) { - GGML_LOG_ERROR("[%s] ggml_new_tensor_4d failed for type %u\n", __func__, tensor->type); + LOG_ERROR("[%s] ggml_new_tensor_4d failed for type %u\n", __func__, tensor->type); return nullptr; } @@ -1703,13 +1668,20 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp result->buffer = nullptr; } - if (result->buffer) { + if (result->buffer && ggml_nelements(result) > 0) { // require that the tensor data does not go beyond the buffer end uint64_t tensor_size = (uint64_t) ggml_nbytes(result); uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(result->buffer); uint64_t buffer_size = (uint64_t) ggml_backend_buffer_get_size(result->buffer); - GGML_ASSERT(tensor->data + tensor_size >= tensor->data); // check for overflow - GGML_ASSERT(tensor->data >= buffer_start && tensor->data + tensor_size <= buffer_start + buffer_size); + if (tensor->data + tensor_size < tensor->data || + tensor->data < buffer_start || tensor->data + tensor_size > buffer_start + buffer_size) { + LOG_ERROR("[%s] tensor '%s' (op %s, type %s, ne [%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "]) " + "data [0x%" PRIx64 ", 0x%" PRIx64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n", + __func__, tensor->name, ggml_op_name((ggml_op) tensor->op), ggml_type_name(result->type), + result->ne[0], result->ne[1], result->ne[2], result->ne[3], + tensor->data, tensor->data + tensor_size, buffer_start, buffer_start + buffer_size); + return nullptr; + } } result->op = (ggml_op) tensor->op; @@ -1722,16 +1694,83 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp return result; } - -bool rpc_server::set_tensor(const std::vector & input) { - // serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) | - if (input.size() < sizeof(rpc_tensor) + sizeof(uint64_t)) { + +bool rpc_server::set_tensor(const std::vector & input) { + sync_all_backends(); + // serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) | + uint8_t cache_flag; + uint64_t offset; + const size_t header_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset); + if (input.size() < header_size) { + return false; + } + const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); + memcpy(&cache_flag, input.data() + sizeof(rpc_tensor), sizeof(cache_flag)); + memcpy(&offset, input.data() + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset)); + const size_t size = input.size() - header_size; + + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_context * ctx = ctx_ptr.get(); + ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor); + if (tensor == nullptr || tensor->buffer == nullptr) { + LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, offset, + rpc_log_format_size(size).c_str()); + + // sanitize tensor->data + { + const size_t p0 = (size_t) ggml_backend_buffer_get_base(tensor->buffer); + const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer); + + if (in_tensor->data + offset < p0 || in_tensor->data + offset >= p1 || size > (p1 - in_tensor->data - offset)) { + LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu) out of buffer bounds [0x%zx, 0x%zx)\n", + __func__, in_tensor->data, offset, size, p0, p1); + return false; + } + } + + const void * data = input.data() + header_size; + if (cache_dir && cache_flag) { + uint64_t hash = fnv_hash((const uint8_t*)data, size); + char hash_str[17]; + snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash); + // save to cache_dir/hash_str + fs::path cache_file = fs::path(cache_dir) / hash_str; + std::ofstream ofs(cache_file, std::ios::binary); + ofs.write((const char *)data, size); + LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.string().c_str()); + } + ggml_backend_tensor_set(tensor, data, offset, size); + return true; +} + +bool rpc_server::set_tensor_2d(const std::vector & input) { + sync_all_backends(); + // serialization format: | rpc_tensor | offset (8 bytes) | size (8 bytes) | n_copies (8 bytes) | stride (8 bytes) | data (size * n_copies bytes) | + if (input.size() < sizeof(rpc_tensor) + 4*sizeof(uint64_t)) { return false; } const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); - uint64_t offset; - memcpy(&offset, input.data() + sizeof(rpc_tensor), sizeof(offset)); - const size_t size = input.size() - sizeof(rpc_tensor) - sizeof(offset); + uint64_t header[4]; + memcpy(header, input.data() + sizeof(rpc_tensor), sizeof(header)); + const uint64_t offset = header[0]; + const uint64_t size = header[1]; + const uint64_t n_copies = header[2]; + const uint64_t stride = header[3]; + + const uint64_t data_size = input.size() - sizeof(rpc_tensor) - 4*sizeof(uint64_t); + if (n_copies == 0 || size == 0 || size > data_size / n_copies || size * n_copies != data_size) { + return false; + } struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), @@ -1743,35 +1782,36 @@ bool rpc_server::set_tensor(const std::vector & input) { ggml_context * ctx = ctx_ptr.get(); ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor); if (tensor == nullptr || tensor->buffer == nullptr) { - GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__); + LOG_ERROR("[%s] error deserializing tensor\n", __func__); return false; } - LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu\n", __func__, (void*)tensor->buffer, tensor->data, offset, size); + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s, n_copies: %" PRIu64 ", stride: %" PRIu64 "\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, offset, + rpc_log_format_size(size).c_str(), n_copies, stride); // sanitize tensor->data { - const size_t p0 = (size_t) ggml_backend_buffer_get_base(tensor->buffer); - const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer); + if (stride != 0 && n_copies - 1 > (UINT64_MAX - size) / stride) { + return false; + } + const uint64_t span = (n_copies - 1)*stride + size; + const uint64_t p0 = (uint64_t) ggml_backend_buffer_get_base(tensor->buffer); + const uint64_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer); - if (in_tensor->data + offset < p0 || in_tensor->data + offset >= p1 || size > (p1 - in_tensor->data - offset)) { - GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu) out of buffer bounds [0x%zx, 0x%zx)\n", - __func__, in_tensor->data, offset, size, p0, p1); + if (in_tensor->data < p0 || in_tensor->data > p1 || offset > p1 - in_tensor->data || span > p1 - in_tensor->data - offset) { + LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", span=%" PRIu64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n", + __func__, in_tensor->data, offset, span, p0, p1); + return false; + } + if (offset > ggml_nbytes(tensor) || span > ggml_nbytes(tensor) - offset) { + LOG_ERROR("[%s] tensor write region (offset=%" PRIu64 ", span=%" PRIu64 ") out of tensor bounds (%zu)\n", + __func__, offset, span, ggml_nbytes(tensor)); return false; } } - const void * data = input.data() + sizeof(rpc_tensor) + sizeof(offset); - if (cache_dir && size > HASH_THRESHOLD) { - uint64_t hash = fnv_hash((const uint8_t*)data, size); - char hash_str[17]; - snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash); - // save to cache_dir/hash_str - fs::path cache_file = fs::path(cache_dir) / hash_str; - std::ofstream ofs(cache_file, std::ios::binary); - ofs.write((const char *)data, size); - GGML_LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.c_str()); - } - ggml_backend_tensor_set(tensor, data, offset, size); + const void * data = input.data() + sizeof(rpc_tensor) + 4*sizeof(uint64_t); + ggml_backend_tensor_set_2d(tensor, data, offset, size, n_copies, stride, size); return true; } @@ -1797,12 +1837,7 @@ bool rpc_server::get_cached_file(uint64_t hash, std::vector & data) { bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response) { - std::vector cached_file; - if (!get_cached_file(request.hash, cached_file)) { - response.result = 0; - return true; - } - size_t size = cached_file.size(); + sync_all_backends(); struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -1813,11 +1848,21 @@ bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rp ggml_context * ctx = ctx_ptr.get(); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr || tensor->buffer == nullptr) { - GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__); + LOG_ERROR("[%s] error deserializing tensor\n", __func__); return false; } - LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu, hash: %" PRIx64 "\n", - __func__, (void*)tensor->buffer, tensor->data, request.offset, size, request.hash); + + std::vector cached_file; + if (!get_cached_file(request.hash, cached_file)) { + LOG_DBG2("[%s] tensor: %s, offset: %" PRIu64 ", hash: 0x%" PRIx64 ", cache miss\n", + __func__, rpc_log_format_tensor(tensor).c_str(), request.offset, request.hash); + response.result = 0; + return true; + } + size_t size = cached_file.size(); + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s, hash: 0x%" PRIx64 ", cache hit\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, request.offset, + rpc_log_format_size(size).c_str(), request.hash); // sanitize tensor->data { @@ -1827,7 +1872,7 @@ bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rp if (request.tensor.data + request.offset < p0 || request.tensor.data + request.offset >= p1 || size > (p1 - request.tensor.data - request.offset)) { - GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu, hash=0x%" PRIx64 ") out of buffer bounds [0x%zx, 0x%zx)\n", + LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu, hash=0x%" PRIx64 ") out of buffer bounds [0x%zx, 0x%zx)\n", __func__, request.tensor.data, request.offset, size, request.hash, p0, p1); return false; } @@ -1838,6 +1883,7 @@ bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rp } bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { + sync_all_backends(); struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -1848,24 +1894,25 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { ggml_context * ctx = ctx_ptr.get(); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr) { - GGML_LOG_ERROR("Null tensor pointer passed to server init_tensor function.\n"); + LOG_ERROR("Null tensor pointer passed to server init_tensor function.\n"); return false; } - LOG_DBG("[%s] buffer: %p, data: %p\n", __func__, (void*)tensor->buffer, tensor->data); + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p\n", __func__, rpc_log_format_tensor(tensor).c_str(), + (void*)tensor->buffer, tensor->data); // Call the backend's buffer_init_tensor function ggml_backend_buffer_t buffer = tensor->buffer; if (buffer && buffer->iface.init_tensor) { buffer->iface.init_tensor(buffer, tensor); } else { if (!buffer) { - GGML_LOG_ERROR("Tensor with null buffer passed to init_tensor function\n"); + LOG_ERROR("Tensor with null buffer passed to init_tensor function\n"); } } if (tensor->extra != nullptr) { // This pointer can either be passed around client/server, or probably better stored server-side and kept track of. // Currently unimplemented. - GGML_LOG_ERROR("tensor->extra populated by the backend, this is currently unsupported.\n"); + LOG_ERROR("tensor->extra populated by the backend, this is currently unsupported.\n"); return false; } @@ -1873,6 +1920,7 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) { } bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector & response) { + sync_all_backends(); struct ggml_init_params params { /*.mem_size =*/ ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -1883,10 +1931,12 @@ bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector< ggml_context * ctx = ctx_ptr.get(); ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); if (tensor == nullptr || tensor->buffer == nullptr) { - GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__); + LOG_ERROR("[%s] error deserializing tensor\n", __func__); return false; } - LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 "\n", __func__, (void*)tensor->buffer, tensor->data, request.offset, request.size); + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s\n", __func__, + rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, request.offset, + rpc_log_format_size(request.size).c_str()); // sanitize tensor->data { @@ -1896,7 +1946,7 @@ bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector< if (request.tensor.data + request.offset < p0 || request.tensor.data + request.offset >= p1 || request.size > (p1 - request.tensor.data - request.offset)) { - GGML_LOG_ERROR("[%s] requested tensor region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%" PRIu64 ") out of buffer bounds [0x%zx, 0x%zx)\n", + LOG_ERROR("[%s] requested tensor region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%" PRIu64 ") out of buffer bounds [0x%zx, 0x%zx)\n", __func__, request.tensor.data, request.offset, request.size, p0, p1); return false; } @@ -1907,7 +1957,57 @@ bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector< return true; } +bool rpc_server::get_tensor_2d(const rpc_msg_get_tensor_2d_req & request, std::vector & response) { + sync_all_backends(); + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_context * ctx = ctx_ptr.get(); + ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); + if (tensor == nullptr || tensor->buffer == nullptr) { + LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + LOG_DBG2("[%s] tensor: %s, buffer: %p, data: %p, offset: %" PRIu64 ", size: %s, n_copies: %" PRIu64 ", stride: %" PRIu64 "\n", + __func__, rpc_log_format_tensor(tensor).c_str(), (void*)tensor->buffer, tensor->data, request.offset, + rpc_log_format_size(request.size).c_str(), request.n_copies, request.stride); + + // sanitize tensor->data + { + if (request.n_copies == 0 || request.size == 0 || request.size > UINT64_MAX / request.n_copies) { + return false; + } + if (request.stride != 0 && request.n_copies - 1 > (UINT64_MAX - request.size) / request.stride) { + return false; + } + const uint64_t span = (request.n_copies - 1)*request.stride + request.size; + const uint64_t p0 = (uint64_t) ggml_backend_buffer_get_base(tensor->buffer); + const uint64_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer); + + if (request.tensor.data < p0 || request.tensor.data > p1 || request.offset > p1 - request.tensor.data || + span > p1 - request.tensor.data - request.offset) { + LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", span=%" PRIu64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n", + __func__, request.tensor.data, request.offset, span, p0, p1); + return false; + } + if (request.offset > ggml_nbytes(tensor) || span > ggml_nbytes(tensor) - request.offset) { + LOG_ERROR("[%s] tensor read region (offset=%" PRIu64 ", span=%" PRIu64 ") out of tensor bounds (%zu)\n", + __func__, request.offset, span, ggml_nbytes(tensor)); + return false; + } + } + + response.resize(request.size * request.n_copies, 0); + ggml_backend_tensor_get_2d(tensor, response.data(), request.offset, request.size, request.n_copies, request.stride, request.size); + return true; +} + bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response) { + sync_all_backends(); struct ggml_init_params params { /*.mem_size =*/ 2*ggml_tensor_overhead(), /*.mem_buffer =*/ NULL, @@ -1920,7 +2020,7 @@ bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_co ggml_tensor * src = deserialize_tensor(ctx, &request.src); ggml_tensor * dst = deserialize_tensor(ctx, &request.dst); if (src == nullptr || dst == nullptr || src->buffer == nullptr || dst->buffer == nullptr) { - GGML_LOG_ERROR("[%s] error deserializing tensors\n", __func__); + LOG_ERROR("[%s] error deserializing tensors\n", __func__); return false; } @@ -1930,7 +2030,7 @@ bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_co uint64_t dst_buf_sz = (uint64_t) ggml_backend_buffer_get_size(dst->buffer); if (dst_data + src_size > dst_base + dst_buf_sz) { - GGML_LOG_ERROR("[%s] out-of-bounds write in rpc_server::copy_tensor:\n" + LOG_ERROR("[%s] out-of-bounds write in rpc_server::copy_tensor:\n" " write range : [0x%" PRIx64 ", 0x%" PRIx64 "]\n" " buffer base: [0x%" PRIx64 ", 0x%" PRIx64 "]\n", __func__, @@ -1941,8 +2041,7 @@ bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_co return false; } - LOG_DBG("[%s] src->buffer: %p, dst->buffer: %p\n", - __func__, (void*) src->buffer, (void*) dst->buffer); + LOG_DBG2("[%s] src: %s -> dst: %s\n", __func__, rpc_log_format_tensor(src).c_str(), rpc_log_format_tensor(dst).c_str()); response.result = ggml_backend_buffer_copy_tensor(src, dst); return true; @@ -1967,7 +2066,7 @@ ggml_tensor * rpc_server::create_node(uint64_t id, return nullptr; } if (result->buffer == nullptr && result->data != nullptr) { - GGML_LOG_ERROR("[%s] invalid data ptr", __func__); + LOG_ERROR("[%s] invalid data ptr\n", __func__); return nullptr; } tensor_map[id] = result; @@ -1979,7 +2078,7 @@ ggml_tensor * rpc_server::create_node(uint64_t id, result->src[i] = create_node(tensor->src[i], ctx, tensor_ptrs, tensor_map); // If the recursive call failed for a non-zero ID, propagate the error if (result->src[i] == nullptr) { - GGML_LOG_ERROR("[%s] failed to create source node %d (src_id=%" PRIu64 ") for node id %" PRIu64 "\n", + LOG_ERROR("[%s] failed to create source node %d (src_id=%" PRIu64 ") for node id %" PRIu64 "\n", __func__, i, tensor->src[i], id); // Must return nullptr to signal failure up the call stack return nullptr; @@ -1994,7 +2093,7 @@ ggml_tensor * rpc_server::create_node(uint64_t id, result->view_src = create_node(tensor->view_src, ctx, tensor_ptrs, tensor_map); // If the recursive call failed for a non-zero ID, propagate the error if (result->view_src == nullptr) { - GGML_LOG_ERROR("[%s] failed to create view_src node (view_src_id=%" PRIu64 ") for node id %" PRIu64 "\n", + LOG_ERROR("[%s] failed to create view_src node (view_src_id=%" PRIu64 ") for node id %" PRIu64 "\n", __func__, tensor->view_src, id); // Must return nullptr to signal failure up the call stack return nullptr; @@ -2006,8 +2105,8 @@ ggml_tensor * rpc_server::create_node(uint64_t id, bool rpc_server::graph_compute(const std::vector & input) { // serialization format: - // | device (4 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | - if (input.size() < 2*sizeof(uint32_t)) { + // | device (4 bytes) | uid (8 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | + if (input.size() < 2*sizeof(uint32_t) + sizeof(uint64_t)) { return false; } const uint8_t * src = input.data(); @@ -2017,10 +2116,13 @@ bool rpc_server::graph_compute(const std::vector & input) { if (device >= backends.size()) { return false; } + uint64_t uid; + memcpy(&uid, src, sizeof(uid)); + src += sizeof(uid); uint32_t n_nodes; memcpy(&n_nodes, src, sizeof(n_nodes)); src += sizeof(n_nodes); - if (input.size() < 2*sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t)) { + if (input.size() < 2*sizeof(uint32_t) + sizeof(uint64_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t)) { return false; } const uint64_t * nodes = (const uint64_t *)src; @@ -2028,19 +2130,28 @@ bool rpc_server::graph_compute(const std::vector & input) { uint32_t n_tensors; memcpy(&n_tensors, src, sizeof(n_tensors)); src += sizeof(n_tensors); - if (input.size() < 2*sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t) + n_tensors*sizeof(rpc_tensor)) { + if (input.size() < 2*sizeof(uint32_t) + sizeof(uint64_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t) + n_tensors*sizeof(rpc_tensor)) { return false; } const rpc_tensor * tensors = (const rpc_tensor *)src; - LOG_DBG("[%s] device: %u, n_nodes: %u, n_tensors: %u\n", __func__, device, n_nodes, n_tensors); + LOG_DBG("[%s] device: %u, uid: %" PRIu64 ", n_nodes: %u, n_tensors: %u\n", __func__, device, uid, n_nodes, n_tensors); + + // graphs with uid == 0 are not cached, see GRAPH_CACHE_MAX for the eviction policy + if (uid != 0 && stored_graphs[device].size() >= GRAPH_CACHE_MAX) { + LOG_DBG("[%s] clearing graph cache for device %u (%zu entries)\n", + __func__, device, stored_graphs[device].size()); + stored_graphs[device].clear(); + } + stored_graph sg_tmp; + stored_graph & sg = uid != 0 ? stored_graphs[device][uid] : sg_tmp; size_t buf_size = ggml_tensor_overhead()*(n_nodes + n_tensors) + ggml_graph_overhead_custom(n_nodes, false); - if (stored_graphs[device].buffer.size() < buf_size) { - stored_graphs[device].buffer.resize(buf_size); + if (sg.buffer.size() < buf_size) { + sg.buffer.resize(buf_size); } struct ggml_init_params params = { /*.mem_size =*/ buf_size, - /*.mem_buffer =*/ stored_graphs[device].buffer.data(), + /*.mem_buffer =*/ sg.buffer.data(), /*.no_alloc =*/ true, }; ggml_context_ptr ctx_ptr { ggml_init(params) }; @@ -2055,22 +2166,32 @@ bool rpc_server::graph_compute(const std::vector & input) { } std::unordered_map tensor_map; tensor_map.reserve(n_nodes); + // the compute itself is asynchronous, so building the graph is the only server-side cost + // observable here - the round trip is timed on the client + const auto t_deserialize = std::chrono::steady_clock::now(); for (uint32_t i = 0; i < n_nodes; i++) { int64_t id; memcpy(&id, &nodes[i], sizeof(id)); graph->nodes[i] = create_node(id, ctx, tensor_ptrs, tensor_map); - // Check if create_node failed for a *non-zero* ID. // If id was 0, create_node returning nullptr is expected. // If id was non-zero and create_node returned nullptr, it indicates a deserialization error. if (graph->nodes[i] == nullptr && id != 0) { - GGML_LOG_ERROR("[%s] failed to create graph node %d (id=%" PRId64 ")\n", __func__, i, id); + LOG_ERROR("[%s] failed to create graph node %d (id=%" PRId64 ")\n", __func__, i, id); return false; } + if (graph->nodes[i] != nullptr) { + LOG_DBG3("[%s] node %u: id=%" PRId64 ", %s\n", __func__, i, id, rpc_log_format_tensor(graph->nodes[i]).c_str()); + const size_t hash_pos = ggml_hash_insert(&graph->visited_hash_set, graph->nodes[i]); + graph->use_counts[hash_pos] = tensor_ptrs.at(id)->use_count; + } } - ggml_status status = ggml_backend_graph_compute(backends[device], graph); + const std::chrono::duration deserialize_ms = std::chrono::steady_clock::now() - t_deserialize; + + ggml_status status = ggml_backend_graph_compute_async(backends[device], graph); GGML_ASSERT(status == GGML_STATUS_SUCCESS && "Unsuccessful graph computations are not supported with RPC"); - stored_graphs[device].graph = graph; + sg.graph = graph; + LOG_DBG3("[%s] device: %u, graph build: %.3f ms\n", __func__, device, deserialize_ms.count()); return true; } @@ -2079,16 +2200,216 @@ bool rpc_server::graph_recompute(const rpc_msg_graph_recompute_req & request) { if (device >= backends.size()) { return false; } - if (stored_graphs[device].graph == nullptr) { + auto it = stored_graphs[device].find(request.uid); + if (it == stored_graphs[device].end() || it->second.graph == nullptr) { + LOG_ERROR("[%s] device: %u, graph with uid %" PRIu64 " not found\n", __func__, device, request.uid); return false; } - ggml_cgraph * graph = stored_graphs[device].graph; - LOG_DBG("[%s] device: %u\n", __func__, device); - ggml_status status = ggml_backend_graph_compute(backends[device], graph); + ggml_cgraph * graph = it->second.graph; + LOG_DBG("[%s] device: %u, uid: %" PRIu64 "\n", __func__, device, request.uid); + ggml_status status = ggml_backend_graph_compute_async(backends[device], graph); GGML_ASSERT(status == GGML_STATUS_SUCCESS && "Unsuccessful graph computations are not supported with RPC"); return true; } +// graph compute is asynchronous; commands that read or write buffer data synchronize first +void rpc_server::sync_all_backends() { + for (ggml_backend_t backend : backends) { + ggml_backend_synchronize(backend); + } +} + +// The comm link between two servers uses the same caps negotiation as the client HELLO, +// so it gets the same transport upgrades (e.g. RDMA). +bool rpc_server::comm_init(const rpc_msg_comm_init_req & request, rpc_msg_comm_init_rsp & response) { + response.ok = 0; + if (request.device >= backends.size() || request.world != 2 || request.rank >= request.world) { + return true; + } + comm_state & state = comm_states[request.device]; + if (state.peer != nullptr) { + response.ok = 1; + return true; + } + uint8_t local_caps[RPC_CONN_CAPS_SIZE] = {}; + uint8_t remote_caps[RPC_CONN_CAPS_SIZE] = {}; + if (request.rank == 0) { + socket_ptr srv = socket_t::create_server("0.0.0.0", request.port); + if (srv == nullptr) { + LOG_ERROR("[%s] failed to listen on comm port %u\n", __func__, request.port); + return true; + } + state.peer = srv->accept(); + if (state.peer == nullptr) { + return true; + } + if (!state.peer->recv_data(remote_caps, sizeof(remote_caps))) { + state.peer = nullptr; + return true; + } + state.peer->get_caps(local_caps); + if (!state.peer->send_data(local_caps, sizeof(local_caps))) { + state.peer = nullptr; + return true; + } + state.peer->update_caps(remote_caps); + } else { + const std::string host(request.host, strnlen(request.host, sizeof(request.host))); + // rank 0 may not be listening yet, retry for a few seconds + for (int i = 0; i < 100 && state.peer == nullptr; i++) { + state.peer = socket_t::connect(host.c_str(), request.port); + if (state.peer == nullptr) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + } + if (state.peer == nullptr) { + LOG_ERROR("[%s] failed to connect to peer %s:%u\n", __func__, host.c_str(), request.port); + return true; + } + state.peer->get_caps(local_caps); + if (!state.peer->send_data(local_caps, sizeof(local_caps)) || + !state.peer->recv_data(remote_caps, sizeof(remote_caps))) { + state.peer = nullptr; + return true; + } + state.peer->update_caps(remote_caps); + } + state.rank = request.rank; + state.world = request.world; + LOG_INFO("[%s] device %u joined pairwise comm as rank %u\n", __func__, request.device, request.rank); + response.ok = 1; + return true; +} + +bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { + if (request.device >= backends.size()) { + return false; + } + comm_state & state = comm_states[request.device]; + if (state.peer == nullptr) { + LOG_ERROR("[%s] no communicator for device %u\n", __func__, request.device); + return false; + } + ggml_backend_t backend = backends[request.device]; + + size_t ctx_size = 16*ggml_tensor_overhead() + 2*ggml_graph_overhead_custom(8, false); + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_context * ctx = ctx_ptr.get(); + ggml_tensor * t_dst = deserialize_tensor(ctx, &request.tensor); + if (t_dst == nullptr || t_dst->buffer == nullptr) { + LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + const size_t nbytes = ggml_nbytes(t_dst); + const int64_t ne = ggml_nelements(t_dst); + if (nbytes == 0) { + return true; + } + // reduce large partials in bf16 to halve the wire bytes; small (decode-sized) ones + // stay f32 since the extra casts and sync cost more than the bytes saved + const bool wire_bf16 = t_dst->type == GGML_TYPE_F32 && ne >= 32768; + const size_t wire_bytes = wire_bf16 ? (size_t) ne*2 : nbytes; + const size_t need = wire_bf16 ? 2*nbytes : nbytes; + if (state.scratch_size < need) { + state.scratch.reset(ggml_backend_alloc_buffer(backend, need)); + state.scratch_size = need; + } + char * scratch_base = (char *) ggml_backend_buffer_get_base(state.scratch.get()); + state.send_buf.resize(wire_bytes); + state.recv_buf.resize(wire_bytes); + + auto new_scratch_tensor = [&](ggml_type type, size_t offset) { + ggml_tensor * t = ggml_new_tensor_4d(ctx, type, t_dst->ne[0], t_dst->ne[1], t_dst->ne[2], t_dst->ne[3]); + t->buffer = state.scratch.get(); + t->data = scratch_base + offset; + return t; + }; + auto new_cpy_node = [&](ggml_tensor * src, ggml_tensor * dst) { + ggml_tensor * t = ggml_new_tensor_4d(ctx, dst->type, dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]); + t->op = GGML_OP_CPY; + t->src[0] = src; + t->src[1] = dst; + t->buffer = dst->buffer; + t->data = dst->data; + t->flags |= GGML_TENSOR_FLAG_COMPUTE; + return t; + }; + auto compute_nodes = [&](ggml_tensor * n0, ggml_tensor * n1) { + ggml_cgraph * graph = ggml_new_graph_custom(ctx, 2, false); + graph->nodes[0] = n0; + graph->nodes[1] = n1; + graph->n_nodes = n1 != nullptr ? 2 : 1; + ggml_status status = ggml_backend_graph_compute_async(backend, graph); + GGML_ASSERT(status == GGML_STATUS_SUCCESS && "Unsuccessful graph computations are not supported with RPC"); + }; + + // wait for the pending subgraph that produced this partial + ggml_backend_synchronize(backend); + + ggml_tensor * t_wire_send = nullptr; + ggml_tensor * t_wire_recv = nullptr; + if (wire_bf16) { + t_wire_send = new_scratch_tensor(GGML_TYPE_BF16, 0); + t_wire_recv = new_scratch_tensor(GGML_TYPE_BF16, ne*2); + compute_nodes(new_cpy_node(t_dst, t_wire_send), nullptr); + ggml_backend_synchronize(backend); + ggml_backend_tensor_get(t_wire_send, state.send_buf.data(), 0, wire_bytes); + } else { + ggml_backend_tensor_get(t_dst, state.send_buf.data(), 0, wire_bytes); + } + + // rank 0 sends first, rank 1 receives first, so large payloads cannot deadlock + if (state.rank == 0) { + if (!state.peer->send_data(state.send_buf.data(), wire_bytes) || !state.peer->flush() || + !state.peer->recv_data(state.recv_buf.data(), wire_bytes)) { + return false; + } + } else { + if (!state.peer->recv_data(state.recv_buf.data(), wire_bytes) || + !state.peer->send_data(state.send_buf.data(), wire_bytes) || !state.peer->flush()) { + return false; + } + } + + ggml_tensor * t_peer = new_scratch_tensor(t_dst->type, wire_bf16 ? (size_t) ne*4 : 0); + ggml_tensor * t_cast = nullptr; + if (wire_bf16) { + ggml_backend_tensor_set(t_wire_recv, state.recv_buf.data(), 0, wire_bytes); + t_cast = new_cpy_node(t_wire_recv, t_peer); + } else { + ggml_backend_tensor_set(t_peer, state.recv_buf.data(), 0, wire_bytes); + } + + ggml_tensor * t_red = ggml_new_tensor_4d(ctx, t_dst->type, t_dst->ne[0], t_dst->ne[1], t_dst->ne[2], t_dst->ne[3]); + t_red->op = GGML_OP_ADD; + t_red->src[0] = t_dst; + t_red->src[1] = t_peer; + t_red->buffer = t_dst->buffer; + t_red->data = t_dst->data; + t_red->flags |= GGML_TENSOR_FLAG_COMPUTE; + + if (t_cast != nullptr) { + compute_nodes(t_cast, t_red); + } else { + compute_nodes(t_red, nullptr); + } + return true; +} + +bool rpc_server::comm_free(const rpc_msg_comm_free_req & request) { + if (request.device >= backends.size()) { + return false; + } + comm_states[request.device] = comm_state(); + return true; +} + bool rpc_server::get_device_memory(const rpc_msg_get_device_memory_req & request, rpc_msg_get_device_memory_rsp & response) { uint32_t dev_id = request.device; if (dev_id >= backends.size()) { @@ -2099,7 +2420,8 @@ bool rpc_server::get_device_memory(const rpc_msg_get_device_memory_req & request ggml_backend_dev_memory(dev, &free, &total); response.free_mem = free; response.total_mem = total; - LOG_DBG("[%s] device: %u, free_mem: %" PRIu64 ", total_mem: %" PRIu64 "\n", __func__, dev_id, response.free_mem, response.total_mem); + LOG_DBG2("[%s] device: %u, free: %s, total: %s\n", __func__, dev_id, + rpc_log_format_size(response.free_mem).c_str(), rpc_log_format_size(response.total_mem).c_str()); return true; } @@ -2110,170 +2432,174 @@ rpc_server::~rpc_server() { } static void rpc_serve_client(const std::vector & backends, const char * cache_dir, - socket_t * sockfd) { + socket_ptr sock) { rpc_server server(backends, cache_dir); uint8_t cmd; - if (!recv_data(sockfd, &cmd, 1)) { + if (!sock->recv_data(&cmd, 1)) { return; } if (cmd != RPC_CMD_HELLO) { - GGML_LOG_ERROR("Expected HELLO command, update client\n"); + LOG_ERROR("Expected HELLO command, update client\n"); return; } // Read input_size and validate protocol version uint64_t hello_input_size; - if (!recv_data(sockfd, &hello_input_size, sizeof(hello_input_size))) { + if (!sock->recv_data(&hello_input_size, sizeof(hello_input_size))) { return; } if (hello_input_size != sizeof(rpc_msg_hello_req)) { - GGML_LOG_ERROR("HELLO request size mismatch (%zu vs %zu) — client needs upgrade to protocol v%d.x\n", + LOG_ERROR("HELLO request size mismatch (%zu vs %zu) — client needs upgrade to protocol v%d.x\n", (size_t)hello_input_size, sizeof(rpc_msg_hello_req), RPC_PROTO_MAJOR_VERSION); return; } rpc_msg_hello_req req = {}; - if (!recv_data(sockfd, &req, sizeof(req))) { + if (!sock->recv_data(&req, sizeof(req))) { return; } rpc_msg_hello_rsp rsp = {}; server.hello(rsp); - // Advertise server transport capabilities based on client's caps - sockfd->get_caps(rsp.conn_caps); - - if (!send_msg(sockfd, &rsp, sizeof(rsp))) { + sock->get_caps(rsp.conn_caps); + if (!send_msg(sock, &rsp, sizeof(rsp))) { return; } // Activate transport upgrade using client's caps - sockfd->update_caps(req.conn_caps); + sock->update_caps(req.conn_caps); + LOG_DBG("[%s] client connected, transport: %s\n", __func__, sock->transport_name()); while (true) { - if (!recv_data(sockfd, &cmd, 1)) { + if (!sock->recv_data(&cmd, 1)) { break; } if (cmd >= RPC_CMD_COUNT) { // fail fast if the command is invalid - GGML_LOG_ERROR("Unknown command: %d\n", cmd); + LOG_ERROR("Unknown command: %d\n", cmd); break; } + LOG_DBG2("[%s] %s\n", __func__, rpc_cmd_name((enum rpc_cmd)cmd)); switch (cmd) { case RPC_CMD_HELLO: { // HELLO command is handled above return; } case RPC_CMD_DEVICE_COUNT: { - if (!recv_msg(sockfd, nullptr, 0)) { + if (!recv_msg(sock, nullptr, 0)) { return; } rpc_msg_device_count_rsp response; response.device_count = backends.size(); - if (!send_msg(sockfd, &response, sizeof(response))) { + if (!send_msg(sock, &response, sizeof(response))) { return; } break; } case RPC_CMD_ALLOC_BUFFER: { rpc_msg_alloc_buffer_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } rpc_msg_alloc_buffer_rsp response; if (!server.alloc_buffer(request, response)) { return; } - if (!send_msg(sockfd, &response, sizeof(response))) { + if (!send_msg(sock, &response, sizeof(response))) { return; } break; } case RPC_CMD_GET_ALLOC_SIZE: { rpc_msg_get_alloc_size_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } rpc_msg_get_alloc_size_rsp response; if (!server.get_alloc_size(request, response)) { return; } - if (!send_msg(sockfd, &response, sizeof(response))) { + if (!send_msg(sock, &response, sizeof(response))) { return; } break; } case RPC_CMD_GET_ALIGNMENT: { rpc_msg_get_alignment_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } rpc_msg_get_alignment_rsp response; if (!server.get_alignment(request, response)) { return; } - if (!send_msg(sockfd, &response, sizeof(response))) { + if (!send_msg(sock, &response, sizeof(response))) { return; } break; } case RPC_CMD_GET_MAX_SIZE: { rpc_msg_get_max_size_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } rpc_msg_get_max_size_rsp response; if (!server.get_max_size(request, response)) { return; } - if (!send_msg(sockfd, &response, sizeof(response))) { + if (!send_msg(sock, &response, sizeof(response))) { return; } break; } case RPC_CMD_BUFFER_GET_BASE: { rpc_msg_buffer_get_base_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } rpc_msg_buffer_get_base_rsp response; if (!server.buffer_get_base(request, response)) { return; } - if (!send_msg(sockfd, &response, sizeof(response))) { + if (!send_msg(sock, &response, sizeof(response))) { return; } break; } case RPC_CMD_FREE_BUFFER: { rpc_msg_free_buffer_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } if (!server.free_buffer(request)) { return; } - if (!send_msg(sockfd, nullptr, 0)) { - return; - } break; } case RPC_CMD_BUFFER_CLEAR: { rpc_msg_buffer_clear_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } if (!server.buffer_clear(request)) { return; } - if (!send_msg(sockfd, nullptr, 0)) { + break; + } + case RPC_CMD_MEMSET_TENSOR: { + rpc_msg_memset_tensor_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + if (!server.memset_tensor(request)) { return; } break; } case RPC_CMD_SET_TENSOR: { std::vector input; - if (!recv_msg(sockfd, input)) { + if (!recv_msg(sock, input)) { return; } if (!server.set_tensor(input)) { @@ -2281,64 +2607,85 @@ static void rpc_serve_client(const std::vector & backends, const } break; } + case RPC_CMD_SET_TENSOR_2D: { + std::vector input; + if (!recv_msg(sock, input)) { + return; + } + if (!server.set_tensor_2d(input)) { + return; + } + break; + } + case RPC_CMD_GET_TENSOR_2D: { + rpc_msg_get_tensor_2d_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + std::vector response; + if (!server.get_tensor_2d(request, response)) { + return; + } + if (!send_msg(sock, response.data(), response.size())) { + return; + } + break; + } case RPC_CMD_SET_TENSOR_HASH: { rpc_msg_set_tensor_hash_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } rpc_msg_set_tensor_hash_rsp response; if (!server.set_tensor_hash(request, response)) { return; } - if (!send_msg(sockfd, &response, sizeof(response))) { + if (!send_msg(sock, &response, sizeof(response))) { return; } break; } case RPC_CMD_INIT_TENSOR: { rpc_msg_init_tensor_req request; - if (!recv_msg(sockfd, &request,sizeof(request))) { + if (!recv_msg(sock, &request,sizeof(request))) { return; } if (!server.init_tensor(request)) { return; } - if (!send_msg(sockfd, nullptr, 0)) { - return; - } break; } case RPC_CMD_GET_TENSOR: { rpc_msg_get_tensor_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } std::vector response; if (!server.get_tensor(request, response)) { return; } - if (!send_msg(sockfd, response.data(), response.size())) { + if (!send_msg(sock, response.data(), response.size())) { return; } break; } case RPC_CMD_COPY_TENSOR: { rpc_msg_copy_tensor_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } rpc_msg_copy_tensor_rsp response; if (!server.copy_tensor(request, response)) { return; } - if (!send_msg(sockfd, &response, sizeof(response))) { + if (!send_msg(sock, &response, sizeof(response))) { return; } break; } case RPC_CMD_GRAPH_COMPUTE: { std::vector input; - if (!recv_msg(sockfd, input)) { + if (!recv_msg(sock, input)) { return; } if (!server.graph_compute(input)) { @@ -2348,7 +2695,7 @@ static void rpc_serve_client(const std::vector & backends, const } case RPC_CMD_GRAPH_RECOMPUTE: { rpc_msg_graph_recompute_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } if (!server.graph_recompute(request)) { @@ -2356,22 +2703,56 @@ static void rpc_serve_client(const std::vector & backends, const } break; } + case RPC_CMD_COMM_INIT: { + rpc_msg_comm_init_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + rpc_msg_comm_init_rsp response; + if (!server.comm_init(request, response)) { + return; + } + if (!send_msg(sock, &response, sizeof(response))) { + return; + } + break; + } + case RPC_CMD_COMM_ALLREDUCE: { + rpc_msg_comm_allreduce_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + if (!server.comm_allreduce(request)) { + return; + } + break; + } + case RPC_CMD_COMM_FREE: { + rpc_msg_comm_free_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + if (!server.comm_free(request)) { + return; + } + break; + } case RPC_CMD_GET_DEVICE_MEMORY: { rpc_msg_get_device_memory_req request; - if (!recv_msg(sockfd, &request, sizeof(request))) { + if (!recv_msg(sock, &request, sizeof(request))) { return; } rpc_msg_get_device_memory_rsp response; if (!server.get_device_memory(request, response)) { return; } - if (!send_msg(sockfd, &response, sizeof(response))) { + if (!send_msg(sock, &response, sizeof(response))) { return; } break; } default: { - GGML_LOG_ERROR("Unknown command: %d\n", cmd); + LOG_ERROR("Unhandled command: %s\n", rpc_cmd_name((enum rpc_cmd)cmd)); return; } } @@ -2381,26 +2762,26 @@ static void rpc_serve_client(const std::vector & backends, const void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir, size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices) { if (n_devices == 0 || devices == nullptr) { - fprintf(stderr, "Invalid arguments to ggml_backend_rpc_start_server\n"); + LOG_ERROR("Invalid arguments to ggml_backend_rpc_start_server\n"); return; } std::vector backends; - printf("Starting RPC server v%d.%d.%d\n", + LOG_INFO("Starting RPC server v%d.%d.%d\n", RPC_PROTO_MAJOR_VERSION, RPC_PROTO_MINOR_VERSION, RPC_PROTO_PATCH_VERSION); - printf(" endpoint : %s\n", endpoint); - printf(" local cache : %s\n", cache_dir ? cache_dir : "n/a"); - printf("Devices:\n"); + LOG_INFO(" endpoint : %s\n", endpoint); + LOG_INFO(" local cache : %s\n", cache_dir ? cache_dir : "n/a"); + LOG_INFO("Devices:\n"); for (size_t i = 0; i < n_devices; i++) { auto dev = devices[i]; size_t free, total; ggml_backend_dev_memory(dev, &free, &total); - printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), - total / 1024 / 1024, free / 1024 / 1024); + LOG_INFO(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), + total / 1024 / 1024, free / 1024 / 1024); auto backend = ggml_backend_dev_init(dev, nullptr); if (!backend) { - fprintf(stderr, "Failed to create backend for device %s\n", dev->iface.get_name(dev)); + LOG_ERROR("Failed to create backend for device %s\n", dev->iface.get_name(dev)); return; } backends.push_back(backend); @@ -2420,54 +2801,35 @@ void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir } #ifdef GGML_RPC_RDMA - printf(" transport : TCP (RDMA auto-negotiate enabled)\n"); + LOG_INFO(" transport : TCP (RDMA auto-negotiate enabled)\n"); #else - printf(" transport : TCP\n"); + LOG_INFO(" transport : TCP\n"); #endif // GGML_RPC_RDMA -#ifdef _WIN32 - { - WSADATA wsaData; - int res = WSAStartup(MAKEWORD(2, 2), &wsaData); - if (res != 0) { - fprintf(stderr, "WSAStartup failed: %d\n", res); - return; - } + if (!rpc_transport_init()) { + LOG_ERROR("Failed to initialize RPC transport\n"); + return; } -#endif - auto server_socket = create_server_socket(host.c_str(), port); + auto server_socket = socket_t::create_server(host.c_str(), port); if (server_socket == nullptr) { - fprintf(stderr, "Failed to create server socket\n"); + LOG_ERROR("Failed to create server socket\n"); return; } while (true) { - auto client_socket = socket_accept(server_socket->fd); + auto client_socket = server_socket->accept(); if (client_socket == nullptr) { - fprintf(stderr, "Failed to accept client connection\n"); + LOG_ERROR("Failed to accept client connection\n"); return; } - printf("Accepted client connection\n"); - fflush(stdout); - rpc_serve_client(backends, cache_dir, client_socket.get()); - printf("Client connection closed\n"); - fflush(stdout); - } -#ifdef _WIN32 - WSACleanup(); -#endif + LOG_INFO("Accepted client connection\n"); + rpc_serve_client(backends, cache_dir, client_socket); + LOG_INFO("Client connection closed\n"); + } + rpc_transport_shutdown(); for (auto backend : backends) { ggml_backend_free(backend); } } -// device interface - -struct ggml_backend_rpc_device_context { - std::string endpoint; - uint32_t device; - std::string name; - std::string description; -}; - static const char * ggml_backend_rpc_device_get_name(ggml_backend_dev_t dev) { ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; @@ -2499,10 +2861,11 @@ static void ggml_backend_rpc_device_get_props(ggml_backend_dev_t dev, struct ggm props->type = ggml_backend_rpc_device_get_type(dev); ggml_backend_rpc_device_get_memory(dev, &props->memory_free, &props->memory_total); props->caps = { - /* .async = */ false, + /* .async = */ true, /* .host_buffer = */ false, /* .buffer_from_host_ptr = */ false, - /* .events = */ false, + /* .events = */ true, + /* .mmap_support = */ true, }; } @@ -2538,6 +2901,24 @@ static bool ggml_backend_rpc_device_supports_buft(ggml_backend_dev_t dev, ggml_b return buft_ctx->endpoint == dev_ctx->endpoint && buft_ctx->device == dev_ctx->device; } +static ggml_backend_event_t ggml_backend_rpc_device_event_new(ggml_backend_dev_t dev) { + ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; + auto dispatcher = get_dispatcher(ctx->endpoint); + return dispatcher->event_new(dev); +} + +static void ggml_backend_rpc_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) { + ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; + auto dispatcher = get_dispatcher(ctx->endpoint); + dispatcher->event_free(event); +} + +static void ggml_backend_rpc_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) { + ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context; + auto dispatcher = get_dispatcher(ctx->endpoint); + dispatcher->event_synchronize(event); +} + static const struct ggml_backend_device_i ggml_backend_rpc_device_i = { /* .get_name = */ ggml_backend_rpc_device_get_name, /* .get_description = */ ggml_backend_rpc_device_get_description, @@ -2551,9 +2932,9 @@ static const struct ggml_backend_device_i ggml_backend_rpc_device_i = { /* .supports_op = */ ggml_backend_rpc_device_supports_op, /* .supports_buft = */ ggml_backend_rpc_device_supports_buft, /* .offload_op = */ NULL, - /* .event_new = */ NULL, - /* .event_free = */ NULL, - /* .event_synchronize = */ NULL, + /* .event_new = */ ggml_backend_rpc_device_event_new, + /* .event_free = */ ggml_backend_rpc_device_event_free, + /* .event_synchronize = */ ggml_backend_rpc_device_event_synchronize, }; // backend reg interface @@ -2583,6 +2964,137 @@ static ggml_backend_dev_t ggml_backend_rpc_reg_get_device(ggml_backend_reg_t reg } } +// Pairwise allreduce between two RPC servers over a direct server-to-server connection. +// The client only sends fire-and-forget COMM_ALLREDUCE commands; the tensor data is +// exchanged between the servers and never passes through the client. +struct ggml_backend_rpc_comm_context { + struct rank_info { + std::string endpoint; + uint32_t device; + std::shared_ptr dispatcher; + }; + std::vector ranks; +}; + +static void ggml_backend_rpc_comm_free(void * comm_ctx_v) { + ggml_backend_rpc_comm_context * comm_ctx = (ggml_backend_rpc_comm_context *) comm_ctx_v; + if (comm_ctx == nullptr) { + return; + } + for (const auto & rank : comm_ctx->ranks) { + auto request = std::make_shared(); + request->device = rank.device; + rank.dispatcher->send(RPC_CMD_COMM_FREE, request, sizeof(*request)); + rank.dispatcher->busy_spin_release(); + } + delete comm_ctx; +} + +static void * ggml_backend_rpc_comm_init(ggml_backend_t * backends, size_t n_backends) { + if (n_backends != 2 || std::getenv("GGML_RPC_NO_COMM") != nullptr) { + return nullptr; + } + std::vector ranks; + ranks.reserve(n_backends); + for (size_t i = 0; i < n_backends; i++) { + if (!ggml_backend_is_rpc(backends[i])) { + return nullptr; + } + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *) backends[i]->context; + // one rank per endpoint: a server processes its socket sequentially, so a second + // COMM_INIT on the same connection would deadlock behind the first + for (const auto & rank : ranks) { + if (rank.endpoint == rpc_ctx->endpoint) { + LOG_WARN("%s: multiple ranks on endpoint %s are not supported\n", __func__, rpc_ctx->endpoint.c_str()); + return nullptr; + } + } + ranks.push_back({rpc_ctx->endpoint, rpc_ctx->device, rpc_ctx->dispatcher}); + } + + // TODO: simplify this logic + // rank 1 connects to rank 0 on its serving host; endpoints must be mutually reachable + // (e.g. do not bind the servers to 127.0.0.1 when they run on different machines) + std::string host0; + int port0; + if (!parse_endpoint(ranks[0].endpoint, host0, port0)) { + return nullptr; + } + const uint32_t comm_port = (uint32_t) port0 + 1000; + if (host0.size() >= 64) { + return nullptr; + } + + for (const auto & rank : ranks) { + rank.dispatcher->busy_spin_acquire(); + } + + // Send all init requests before reading any response: rank 0 blocks in accept + // until rank 1 has connected. + std::vector responses(n_backends); + for (size_t i = 0; i < n_backends; i++) { + auto request = std::make_shared(); + request->device = ranks[i].device; + request->rank = (uint32_t) i; + request->world = (uint32_t) n_backends; + request->port = comm_port; + if (i > 0) { + memcpy(request->host, host0.c_str(), host0.size()); + } + ranks[i].dispatcher->send_async(RPC_CMD_COMM_INIT, request, sizeof(*request), &responses[i], sizeof(responses[i])); + } + for (size_t i = 0; i < n_backends; i++) { + ranks[i].dispatcher->synchronize(); + } + bool ok = true; + for (size_t i = 0; i < n_backends; i++) { + if (!responses[i].ok) { + LOG_WARN("%s: rank %zu (%s) failed to initialize\n", __func__, i, ranks[i].endpoint.c_str()); + ok = false; + } + } + if (!ok) { + for (const auto & rank : ranks) { + rank.dispatcher->busy_spin_release(); + } + return nullptr; + } + LOG_INFO("%s: pairwise communicator initialized (%s <-> %s)\n", __func__, + ranks[0].endpoint.c_str(), ranks[1].endpoint.c_str()); + return new ggml_backend_rpc_comm_context{std::move(ranks)}; +} + +static bool ggml_backend_rpc_comm_allreduce_tensor(void * comm_ctx_v, ggml_tensor ** tensors) { + ggml_backend_rpc_comm_context * comm_ctx = (ggml_backend_rpc_comm_context *) comm_ctx_v; + if (comm_ctx == nullptr) { + return false; + } + const size_t n_ranks = comm_ctx->ranks.size(); + const int64_t ne = ggml_nelements(tensors[0]); + if (ne == 0) { + return true; + } + for (size_t i = 0; i < n_ranks; i++) { + if (tensors[i] == nullptr || tensors[i]->type != GGML_TYPE_F32 || ggml_nelements(tensors[i]) != ne || + !ggml_is_contiguously_allocated(tensors[i]) || + tensors[i]->buffer == nullptr || !ggml_backend_buffer_is_rpc(tensors[i]->buffer)) { + return false; + } + // a rank with a disabled node has garbage in its partial and must contribute zeros, + // which only the fallback path handles + if ((tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + return false; + } + } + for (size_t i = 0; i < n_ranks; i++) { + auto request = std::make_shared(); + request->device = comm_ctx->ranks[i].device; + request->tensor = serialize_tensor(tensors[i]); + comm_ctx->ranks[i].dispatcher->send_async(RPC_CMD_COMM_ALLREDUCE, request, sizeof(*request)); + } + return true; +} + static void * ggml_backend_rpc_get_proc_address(ggml_backend_reg_t reg, const char * name) { if (std::strcmp(name, "ggml_backend_rpc_add_server") == 0) { return (void *)ggml_backend_rpc_add_server; @@ -2590,6 +3102,15 @@ static void * ggml_backend_rpc_get_proc_address(ggml_backend_reg_t reg, const ch if (std::strcmp(name, "ggml_backend_rpc_start_server") == 0) { return (void *)ggml_backend_rpc_start_server; } + if (std::strcmp(name, "ggml_backend_comm_init") == 0) { + return (void *)ggml_backend_rpc_comm_init; + } + if (std::strcmp(name, "ggml_backend_comm_free") == 0) { + return (void *)ggml_backend_rpc_comm_free; + } + if (std::strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { + return (void *)ggml_backend_rpc_comm_allreduce_tensor; + } return NULL; GGML_UNUSED(reg); @@ -2613,14 +3134,9 @@ ggml_backend_reg_t ggml_backend_rpc_reg(void) { } static uint32_t ggml_backend_rpc_get_device_count(const char * endpoint) { - auto sock = get_socket(endpoint); - if (sock == nullptr) { - GGML_LOG_ERROR("Failed to connect to %s\n", endpoint); - return 0; - } + auto dispatcher = get_dispatcher(endpoint); rpc_msg_device_count_rsp response; - bool status = send_rpc_cmd(sock, RPC_CMD_DEVICE_COUNT, nullptr, 0, &response, sizeof(response)); - RPC_STATUS_ASSERT(status); + dispatcher->send(RPC_CMD_DEVICE_COUNT, nullptr, 0, &response, sizeof(response)); return response.device_count; } @@ -2637,9 +3153,11 @@ ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint) { static uint32_t dev_id = 0; std::lock_guard lock(mutex); if (reg_map.find(endpoint) != reg_map.end()) { + LOG_DBG3("[%s] endpoint %s already registered\n", __func__, endpoint); return reg_map[endpoint]; } uint32_t dev_count = ggml_backend_rpc_get_device_count(endpoint); + LOG_DBG2("[%s] endpoint: %s, device count: %u\n", __func__, endpoint, dev_count); if (dev_count == 0) { return nullptr; } @@ -2649,10 +3167,10 @@ ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint) { std::string dev_name = "RPC" + std::to_string(dev_id); std::string dev_desc = std::string(endpoint); ggml_backend_rpc_device_context * dev_ctx = new ggml_backend_rpc_device_context { - /* .endpoint = */ endpoint, - /* .device = */ ind, - /* .name = */ dev_name, - /* .description = */ dev_desc + /* .endpoint = */ endpoint, + /* .device = */ ind, + /* .name = */ dev_name, + /* .description = */ dev_desc, }; ggml_backend_dev_t dev = new ggml_backend_device { diff --git a/ggml/src/ggml-rpc/log.cpp b/ggml/src/ggml-rpc/log.cpp new file mode 100644 index 0000000..89f0ce7 --- /dev/null +++ b/ggml/src/ggml-rpc/log.cpp @@ -0,0 +1,34 @@ +#include "log.h" + +#include +#include + +std::string rpc_log_format_size(size_t size) { + static const char * units[] = { "B", "KiB", "MiB", "GiB", "TiB", "PiB" }; + + double value = (double) size; + size_t unit = 0; + while (value >= 1024.0 && unit + 1 < sizeof(units) / sizeof(units[0])) { + value /= 1024.0; + unit++; + } + + char buf[32]; + if (unit == 0) { + snprintf(buf, sizeof(buf), "%zu B", size); + } else { + snprintf(buf, sizeof(buf), "%.2f %s", value, units[unit]); + } + return std::string(buf); +} + +std::string rpc_log_format_tensor(const ggml_tensor * tensor) { + char shape[64]; + snprintf(shape, sizeof(shape), "[%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "]", tensor->ne[0], tensor->ne[1], + tensor->ne[2], tensor->ne[3]); + + char buf[GGML_MAX_NAME + 192]; + snprintf(buf, sizeof(buf), "%s [%s, %s, %s]", ggml_get_name(tensor), ggml_op_name(tensor->op), + ggml_type_name(tensor->type), shape); + return std::string(buf); +} diff --git a/ggml/src/ggml-rpc/log.h b/ggml/src/ggml-rpc/log.h new file mode 100644 index 0000000..96a383d --- /dev/null +++ b/ggml/src/ggml-rpc/log.h @@ -0,0 +1,49 @@ +#pragma once + +// Log infrastructure for the RPC backend: LOG_* are unconditional, LOG_DBG* are gated by GGML_RPC_DEBUG + +#include "ggml-impl.h" + +#include +#include +#include + +// GGML_RPC_DEBUG verbosity: unset / 0 disabled, 1 connections and allocations, 2 per-command trace, +// 3 per-message detail. Non-numeric values fall back to 1 +static inline int rpc_log_debug_level() { + static const int level = [] { + const char * env = std::getenv("GGML_RPC_DEBUG"); + if (env == nullptr) { + return 0; + } + if (env[0] == '\0') { + return 1; + } + int value = 0; + for (const char * p = env; *p != '\0'; p++) { + if (*p < '0' || *p > '9') { + return 1; + } + value = value * 10 + (*p - '0'); + if (value > 3) { + value = 3; + } + } + return value; + }(); + return level; +} + +// Byte counts in binary units, e.g. 8208 -> "8.02 KiB". Sizes below 1 KiB stay in plain bytes +std::string rpc_log_format_size(size_t size); + +// Tensor identity: "blk.0.attn_q.weight [NONE, f16, [4096, 4096, 1, 1]]" +std::string rpc_log_format_tensor(const ggml_tensor * tensor); + +#define LOG_INFO(...) GGML_LOG_INFO(__VA_ARGS__) +#define LOG_WARN(...) GGML_LOG_WARN(__VA_ARGS__) +#define LOG_ERROR(...) GGML_LOG_ERROR(__VA_ARGS__) + +#define LOG_DBG(...) do { if (rpc_log_debug_level() >= 1) GGML_LOG_DEBUG(__VA_ARGS__); } while (0) +#define LOG_DBG2(...) do { if (rpc_log_debug_level() >= 2) GGML_LOG_DEBUG(__VA_ARGS__); } while (0) +#define LOG_DBG3(...) do { if (rpc_log_debug_level() >= 3) GGML_LOG_DEBUG(__VA_ARGS__); } while (0) diff --git a/ggml/src/ggml-rpc/transport-apple.cpp b/ggml/src/ggml-rpc/transport-apple.cpp new file mode 100644 index 0000000..18c4e7e --- /dev/null +++ b/ggml/src/ggml-rpc/transport-apple.cpp @@ -0,0 +1,486 @@ +#include "transport-apple.h" +#include "transport.h" +#include "log.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// Apple RDMA-over-Thunderbolt (see Apple TN3205). +// +// Apple's RDMA is quite different from what's supported in Linux - deserving of its own transport implementation. +// see https://developer.apple.com/documentation/technotes/tn3205-low-latency-communication-with-rdma-over-thunderbolt for details +// at a high level the main differences are: +// UC(unreliable connection) on Apple vs RC(reliable connection) QP transport types on Linux (though in practice UC on Apple is still lossless) +// fixed 128KiB stride on Apple vs variable chunk size on Linux +// relying on Apple's hardware credit based flow control vs RNR NAKs + retries on Linux +// +// on Apple a SEND and its corresponding RECV must cover the same number of 4 KiB Thunderbolt frames, +// so every SEND posts a whole 128KiB stride over the wire, even when partially filled. +// (In testing 128KiB was the best performing among 32, 64, 128, 256) + +//TODO: add mechanism similar to https://github.com/ggml-org/llama.cpp/pull/29440 to prevent idle CPU from spinning +static constexpr uint32_t RDMA_SEG_MAGIC = 0x52534547u; // "RSEG" +static constexpr int RDMA_NBUF = 16; // ring depth (frames per direction) +static constexpr size_t RDMA_FRAME = 4096; // Thunderbolt frame (fixed on Apple) +static constexpr size_t RDMA_STRIDE = 128 * 1024; // 32 Thunderbolt frames; NBUF x this = 2 MiB pinned per direction +static constexpr uint32_t RDMA_PSN = 0; // any value works if both sides match: UC has no retransmit +static constexpr size_t RDMA_GID_SIZE = 16; + +static_assert(RDMA_STRIDE % RDMA_FRAME == 0, "RDMA_STRIDE must be a whole number of frames"); +// TN3205 counts queue depth in Thunderbolt frames, not work requests. +static constexpr uint32_t RDMA_QP_WR = (uint32_t)RDMA_NBUF * (RDMA_STRIDE / RDMA_FRAME); +static constexpr uint64_t RDMA_RECV_WR = 1ull << 20; // wr_id bit tagging recv completions +static constexpr uint64_t RDMA_WR_IDX_MASK = 0xffff; // buffer index in the low bits of wr_id +static constexpr uint8_t RDMA_SYNC_READY = 0x2A; // readiness-handshake byte (peer activated) + +struct rdma_seg_hdr { + uint32_t magic; // RDMA_SEG_MAGIC; a mismatch means the stream desynced + uint32_t len; // payload bytes in this frame; the rest of the stride is padding +}; +static constexpr size_t RDMA_PAYLOAD = RDMA_STRIDE - sizeof(rdma_seg_hdr); + +struct apple_rdma_caps { + uint32_t qpn; + uint16_t lid; + uint16_t reserved; + uint8_t gid[RDMA_GID_SIZE]; +}; + +static_assert(sizeof(apple_rdma_caps) == RPC_CONN_CAPS_SIZE, "apple_rdma_caps must match conn_caps size"); + +struct apple_rdma::impl { + int fd = -1; // bootstrap TCP socket, kept as the liveness anchor + + struct ibv_context * ctx = nullptr; + struct ibv_pd * pd = nullptr; + struct ibv_cq * cq = nullptr; // one CQ for both directions; RDMA_RECV_WR tags recv completions + struct ibv_qp * qp = nullptr; + + uint8_t * send_mem = nullptr; + struct ibv_mr * send_mr = nullptr; + uint8_t * recv_mem = nullptr; + struct ibv_mr * recv_mr = nullptr; + + int send_busy[RDMA_NBUF] = {}; // 1 while this buffer has a send in flight + // completed recv frames, oldest first: ring index, bytes already handed to + // the reader, and total payload length + struct { int buf; uint32_t off; uint32_t len; } inq[RDMA_NBUF] = {}; + int inq_head = 0; + int inq_count = 0; + int pend_buf = -1; + uint32_t pend_len = 0; + bool broken = false; + + uint32_t qpn = 0; + uint8_t port = 0; + int gid_idx = 0; + enum ibv_mtu path_mtu = IBV_MTU_1024; + + int progress(); + bool acquire_pending(); + bool post_pending(); + + bool post_recv(int i) { + struct ibv_sge sge = {}; + sge.addr = (uintptr_t)(recv_mem + (size_t)i * RDMA_STRIDE); + sge.length = (uint32_t)RDMA_STRIDE; + sge.lkey = recv_mr->lkey; + struct ibv_recv_wr wr = {}, * bad = nullptr; + wr.wr_id = RDMA_RECV_WR | (uint64_t)i; + wr.sg_list = &sge; + wr.num_sge = 1; + return ibv_post_recv(qp, &wr, &bad) == 0; + } + + bool post_send(int i, size_t len) { + struct ibv_sge sge = {}; + sge.addr = (uintptr_t)(send_mem + (size_t)i * RDMA_STRIDE); + sge.length = (uint32_t)len; + sge.lkey = send_mr->lkey; + struct ibv_send_wr wr = {}, * bad = nullptr; + wr.wr_id = (uint64_t)i; + wr.sg_list = &sge; + wr.num_sge = 1; + wr.opcode = IBV_WR_SEND; + wr.send_flags = IBV_SEND_SIGNALED; + return ibv_post_send(qp, &wr, &bad) == 0; + } + + ~impl() { + broken = true; + // destroy the QP first: it can still write to the rings until it is gone. + // no IBV_QPS_ERR before it - Apple's provider then fails every region unmap. + if (qp) ibv_destroy_qp(qp); + if (send_mr) ibv_dereg_mr(send_mr); + if (recv_mr) ibv_dereg_mr(recv_mr); + free(send_mem); + free(recv_mem); + if (cq) ibv_destroy_cq(cq); + if (pd) ibv_dealloc_pd(pd); + if (ctx) ibv_close_device(ctx); + } +}; + +apple_rdma::apple_rdma(std::unique_ptr p) : pimpl(std::move(p)) {} + +apple_rdma::~apple_rdma() = default; + +bool apple_rdma::broken() const { + return pimpl->broken; +} + +// The readiness handshake below still runs over the bootstrap socket, one byte +// each way, before the transport is declared live. +static bool tcp_send_byte(int fd, uint8_t b) { + ssize_t n; + do { n = ::send(fd, &b, sizeof(b), 0); } while (n < 0 && errno == EINTR); + return n == sizeof(b); +} + +static bool tcp_recv_byte(int fd, uint8_t * b) { + ssize_t n; + do { n = ::recv(fd, b, sizeof(*b), 0); } while (n < 0 && errno == EINTR); + return n == (ssize_t)sizeof(*b); +} + +// Index of the GID on this port equal to the target, or -1. Thunderbolt GIDs are +// RoCEv2 IPv4-mapped (::ffff:a.b.c.d), so this matches the local TCP address. +static int rdma_match_gid(struct ibv_context * ctx, uint8_t port, int gid_tbl_len, + const uint8_t * target, union ibv_gid * out) { + for (int i = 0; i < gid_tbl_len; i++) { + union ibv_gid g; + if (ibv_query_gid(ctx, port, i, &g) != 0) continue; + if (memcmp(g.raw, target, RDMA_GID_SIZE) != 0) continue; + if (out) *out = g; + return i; + } + return -1; +} + +// First ACTIVE port on the device. Only a cabled, up Thunderbolt link reports +// ACTIVE, and it is not always port 1, so the port cannot be hardcoded the way +// the Linux path does. Returns 0 if none. +static uint8_t rdma_first_active_port(struct ibv_context * ctx, struct ibv_port_attr * out) { + struct ibv_device_attr da; + if (ibv_query_device(ctx, &da) != 0) return 0; + for (uint8_t p = 1; p <= da.phys_port_cnt; p++) { + struct ibv_port_attr pa; + if (ibv_query_port(ctx, p, &pa) != 0) continue; + if (pa.state == IBV_PORT_ACTIVE) { if (out) *out = pa; return p; } + } + return 0; +} + +// librdma.dylib is weak-linked, so its symbols are null when it is absent. Nothing may +// call one before this has returned true. +static bool rdma_library_present() { + static const bool present = [] { + void * handle = dlopen("/usr/lib/librdma.dylib", RTLD_LAZY); + if (handle == nullptr) { + return false; + } + dlclose(handle); + return true; + }(); + return present; +} + +// Called before the endpoints are exchanged: pick the local device facing this +// peer, create a UC QP and register the frame rings. RDMA is point-to-point, so +// the device is the one whose GID equals the bootstrap connection's local +// address, i.e. the one cabled to the peer. +std::unique_ptr apple_rdma::probe(int fd, const uint8_t * target_gid, uint8_t * caps) { + if (!rdma_library_present()) { + LOG_DBG2("[RDMA(Apple)] librdma.dylib not present, continuing with TCP\n"); + return nullptr; + } + int ndev = 0; + ibv_device ** devs = ibv_get_device_list(&ndev); + if (!devs) return nullptr; + + ibv_context * ctx = nullptr; + uint8_t port = 0; + struct ibv_port_attr pa = {}; + union ibv_gid gid = {}; + int gid_idx = -1; + std::string matched; + for (int d = 0; d < ndev; d++) { + ibv_context * c = ibv_open_device(devs[d]); + if (!c) continue; + struct ibv_port_attr p = {}; + uint8_t pt = rdma_first_active_port(c, &p); + int gi = pt ? rdma_match_gid(c, pt, p.gid_tbl_len, target_gid, &gid) : -1; + if (gi < 0) { ibv_close_device(c); continue; } + ctx = c; port = pt; pa = p; gid_idx = gi; + const char * name = ibv_get_device_name(devs[d]); + matched = name ? name : ""; + break; + } + ibv_free_device_list(devs); + if (!ctx) { + LOG_DBG2("[RDMA(Apple)] no RDMA device matched the connection address, continuing with TCP\n"); + return nullptr; + } + + std::unique_ptr c(new impl()); + c->fd = fd; + c->ctx = ctx; + c->port = port; + c->gid_idx = gid_idx; + c->path_mtu = pa.active_mtu; + + c->pd = ibv_alloc_pd(ctx); + if (!c->pd) return nullptr; + + c->cq = ibv_create_cq(ctx, 2 * RDMA_QP_WR + 1, nullptr, nullptr, 0); + if (!c->cq) return nullptr; + + ibv_qp_init_attr qia = {}; + qia.send_cq = c->cq; + qia.recv_cq = c->cq; + qia.qp_type = IBV_QPT_UC; + qia.cap.max_send_wr = RDMA_QP_WR; + qia.cap.max_recv_wr = RDMA_QP_WR; + qia.cap.max_send_sge = 1; + qia.cap.max_recv_sge = 1; + c->qp = ibv_create_qp(c->pd, &qia); + if (!c->qp) return nullptr; + + { + ibv_qp_attr a = {}; + a.qp_state = IBV_QPS_INIT; + a.pkey_index = 0; + a.port_num = port; + a.qp_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; + if (ibv_modify_qp(c->qp, &a, + IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS) != 0) { + return nullptr; + } + } + + long page = sysconf(_SC_PAGESIZE); + if (page <= 0) page = 4096; + const size_t ring_bytes = (size_t)RDMA_NBUF * RDMA_STRIDE; + if (posix_memalign((void **)&c->send_mem, (size_t)page, ring_bytes) != 0) c->send_mem = nullptr; + if (posix_memalign((void **)&c->recv_mem, (size_t)page, ring_bytes) != 0) c->recv_mem = nullptr; + if (!c->send_mem || !c->recv_mem) return nullptr; + + // Apple's provider rejects LOCAL_WRITE-only MRs even for two-sided SEND/RECV. + const int mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE; + c->send_mr = ibv_reg_mr(c->pd, c->send_mem, ring_bytes, mr_flags); + c->recv_mr = ibv_reg_mr(c->pd, c->recv_mem, ring_bytes, mr_flags); + if (!c->send_mr || !c->recv_mr) return nullptr; + + // Recvs are posted in activate() after the RTS transition, not here: Apple's + // provider rejects ibv_post_recv on a QP that has not reached RTS. + + c->qpn = c->qp->qp_num; + + apple_rdma_caps rc = {}; + rc.qpn = c->qpn; + rc.lid = pa.lid; + memcpy(rc.gid, gid.raw, RDMA_GID_SIZE); + memcpy(caps, &rc, sizeof(rc)); + + LOG_INFO("RDMA(Apple/UC) probed: dev=%s port=%u gid=%d qpn=%u lid=%u mtu=%d ring=%d x %zu KiB\n", + matched.c_str(), port, gid_idx, c->qpn, (unsigned)pa.lid, 128 << c->path_mtu, + RDMA_NBUF, RDMA_STRIDE / 1024); + return std::unique_ptr(new apple_rdma(std::move(c))); +} + +// Called once the peer's endpoint has arrived: INIT -> RTR -> RTS (UC: GID/GRH +// addressing, no timeout/retry/rnr/rd_atomic), then the readiness handshake. +bool apple_rdma::activate(const uint8_t * caps) { + impl * c = pimpl.get(); + + apple_rdma_caps rc = {}; + memcpy(&rc, caps, sizeof(rc)); + + bool ok = true; + { + ibv_qp_attr a = {}; + a.qp_state = IBV_QPS_RTR; + a.path_mtu = c->path_mtu; + a.rq_psn = RDMA_PSN; + a.dest_qp_num = rc.qpn; + a.ah_attr.is_global = 1; + a.ah_attr.port_num = c->port; + a.ah_attr.sl = 0; + a.ah_attr.src_path_bits = 0; + a.ah_attr.dlid = rc.lid; + a.ah_attr.grh.hop_limit = 1; + a.ah_attr.grh.sgid_index = (uint8_t)c->gid_idx; + memcpy(&a.ah_attr.grh.dgid, rc.gid, RDMA_GID_SIZE); + if (ibv_modify_qp(c->qp, &a, + IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN) != 0) { + LOG_ERROR("RDMA(Apple/UC) RTR failed: %s\n", strerror(errno)); + ok = false; + } + } + if (ok) { + ibv_qp_attr a = {}; + a.qp_state = IBV_QPS_RTS; + a.sq_psn = RDMA_PSN; + if (ibv_modify_qp(c->qp, &a, IBV_QP_STATE | IBV_QP_SQ_PSN) != 0) { + LOG_ERROR("RDMA(Apple/UC) RTS failed: %s\n", strerror(errno)); + ok = false; + } + } + + // Recvs are posted only now: the controller starts processing them at RTR. + for (int i = 0; ok && i < RDMA_NBUF; i++) { + if (!c->post_recv(i)) { + LOG_ERROR("RDMA(Apple/UC) post_recv %d/%d failed\n", i, RDMA_NBUF); + ok = false; + } + } + + // A queue pair processes receives only after RTR and the transitions above can + // fail on one side alone, so neither peer sends a frame until both report their + // recvs posted. + uint8_t peer_ready = 0; + if (!tcp_send_byte(c->fd, ok ? RDMA_SYNC_READY : 0) || !tcp_recv_byte(c->fd, &peer_ready)) { + return false; + } + if (!ok || peer_ready != RDMA_SYNC_READY) { + return false; + } + + LOG_INFO("RDMA(Apple/UC) activated: qpn=%u->%u mtu=%d rx_depth=%d\n", + c->qpn, rc.qpn, 128 << c->path_mtu, RDMA_NBUF); + return true; +} + +// Drain the CQ: release completed send buffers, queue completed recv frames for +// the reader. Returns the number of completions reaped, or -1 on error. +int apple_rdma::impl::progress() { + struct ibv_wc wc[RDMA_NBUF * 2]; + int n = ibv_poll_cq(cq, RDMA_NBUF * 2, wc); + if (n < 0) { LOG_ERROR("RDMA(Apple/UC) poll_cq failed\n"); broken = true; return -1; } + for (int j = 0; j < n; j++) { + uint64_t id = wc[j].wr_id; + bool is_recv = (id & RDMA_RECV_WR) != 0; + if (wc[j].status != IBV_WC_SUCCESS) { + LOG_ERROR("RDMA(Apple/UC) %s wc error: status=%d\n", is_recv ? "recv" : "send", wc[j].status); + broken = true; + return -1; + } + if (is_recv) { + int b = (int)(id & RDMA_WR_IDX_MASK); + const rdma_seg_hdr * h = (const rdma_seg_hdr *)(recv_mem + (size_t)b * RDMA_STRIDE); + if (h->magic != RDMA_SEG_MAGIC) { LOG_ERROR("RDMA(Apple/UC) bad frame magic\n"); broken = true; return -1; } + if (h->len > RDMA_PAYLOAD) { LOG_ERROR("RDMA(Apple/UC) frame len %u exceeds payload\n", h->len); broken = true; return -1; } + int slot = (inq_head + inq_count) % RDMA_NBUF; + inq[slot].buf = b; + inq[slot].off = 0; + inq[slot].len = h->len; + inq_count++; + } else { + send_busy[(int)(id & RDMA_WR_IDX_MASK)] = 0; + } + } + return n; +} + +// Reserve a free send buffer to coalesce into, waiting on progress if none free. +bool apple_rdma::impl::acquire_pending() { + if (pend_buf >= 0) return true; + for (;;) { + if (broken) return false; + for (int k = 0; k < RDMA_NBUF; k++) if (!send_busy[k]) { pend_buf = k; pend_len = 0; return true; } + if (progress() < 0) return false; + } +} + +// Post the pending frame. The whole STRIDE goes out even when only partly filled: +// TN3205 requires a SEND and its matching RECV to cover the same number of +// Thunderbolt frames, so a short send would fail the peer's receive. +bool apple_rdma::impl::post_pending() { + if (pend_buf < 0) return true; + int i = pend_buf; + rdma_seg_hdr * h = (rdma_seg_hdr *)(send_mem + (size_t)i * RDMA_STRIDE); + h->magic = RDMA_SEG_MAGIC; + h->len = pend_len; + if (!post_send(i, RDMA_STRIDE)) { broken = true; return false; } + send_busy[i] = 1; + pend_buf = -1; + pend_len = 0; + return true; +} + +// Coalescing write: append into the pending frame, posting a full frame when it +// fills. The trailing partial is posted by flush() at each message boundary. +bool apple_rdma::send(const void * data, size_t size) { + impl * c = pimpl.get(); + const uint8_t * p = (const uint8_t *)data; + while (size > 0) { + if (c->broken) return false; + if (!c->acquire_pending()) return false; + uint8_t * sb = c->send_mem + (size_t)c->pend_buf * RDMA_STRIDE; + size_t space = RDMA_PAYLOAD - c->pend_len; + size_t chunk = size < space ? size : space; + memcpy(sb + sizeof(rdma_seg_hdr) + c->pend_len, p, chunk); + c->pend_len += (uint32_t)chunk; + p += chunk; + size -= chunk; + if (c->pend_len == RDMA_PAYLOAD) { if (!c->post_pending()) return false; } + } + return true; +} + +bool apple_rdma::recv(void * data, size_t size) { + impl * c = pimpl.get(); + uint8_t * p = (uint8_t *)data; + if (!c->post_pending()) return false; // turnaround: flush the coalesced request + unsigned idle = 0; + while (size > 0) { + if (c->inq_count == 0) { + if (c->broken) return false; + int n = c->progress(); + if (n < 0) return false; + if (n == 0) { + // UC gives no disconnect notification, so the bootstrap TCP fd is + // the liveness anchor: nothing crosses it once RDMA is up, so any + // readability means the peer's FIN (macOS has no POLLRDHUP). + // Same idle interval as the Linux path. + if ((++idle & 0xFFFFF) == 0) { + struct pollfd pfd = { c->fd, POLLIN, 0 }; + if (poll(&pfd, 1, 0) > 0 && + (pfd.revents & (POLLIN | POLLHUP | POLLERR | POLLNVAL))) { + return false; + } + } + } else { + idle = 0; + } + continue; + } + idle = 0; + int slot = c->inq_head; + int b = c->inq[slot].buf; + uint32_t avail = c->inq[slot].len - c->inq[slot].off; + uint32_t take = (size < (size_t)avail) ? (uint32_t)size : avail; + memcpy(p, c->recv_mem + (size_t)b * RDMA_STRIDE + sizeof(rdma_seg_hdr) + c->inq[slot].off, take); + p += take; + size -= take; + c->inq[slot].off += take; + if (c->inq[slot].off == c->inq[slot].len) { + if (!c->post_recv(b)) { c->broken = true; return false; } + c->inq_head = (c->inq_head + 1) % RDMA_NBUF; + c->inq_count--; + } + } + return true; +} + +bool apple_rdma::flush() { + return pimpl->post_pending(); +} diff --git a/ggml/src/ggml-rpc/transport-apple.h b/ggml/src/ggml-rpc/transport-apple.h new file mode 100644 index 0000000..7968d38 --- /dev/null +++ b/ggml/src/ggml-rpc/transport-apple.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include + +struct apple_rdma { + // target_gid is 16 bytes in, caps is RPC_CONN_CAPS_SIZE bytes out. + static std::unique_ptr probe(int fd, const uint8_t * target_gid, uint8_t * caps); + ~apple_rdma(); + + // Peer endpoint from its caps, which must be non-zero: this blocks on a + // readiness handshake over fd that the peer only joins if it also has RDMA. + bool activate(const uint8_t * caps); + + bool send(const void * data, size_t size); + bool recv(void * data, size_t size); + // Post the trailing partial frame; must be called at every message boundary. + bool flush(); + // True once the connection has failed; the caller should drop the socket. + bool broken() const; + +private: + struct impl; + explicit apple_rdma(std::unique_ptr p); + std::unique_ptr pimpl; +}; diff --git a/ggml/src/ggml-rpc/transport.cpp b/ggml/src/ggml-rpc/transport.cpp new file mode 100644 index 0000000..d5670f3 --- /dev/null +++ b/ggml/src/ggml-rpc/transport.cpp @@ -0,0 +1,814 @@ +#include "transport.h" +#include "log.h" + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +# include +#else +# include +# include +# include +# include +# include +# include +# include +#endif +#include +#include +#include +#include + +#ifdef GGML_RPC_RDMA +# include +# include +# include +# include +# include +# ifndef _WIN32 +# include +# endif +# ifdef GGML_RPC_RDMA_APPLE +# include "transport-apple.h" +# endif +#endif // GGML_RPC_RDMA + +#ifdef _WIN32 +typedef SOCKET sockfd_t; +using ssize_t = __int64; +#else +typedef int sockfd_t; +#endif + +#ifdef GGML_RPC_RDMA +static constexpr size_t RDMA_GID_SIZE = 16; // RoCE GID / IB GID is always 16 bytes +using rdma_gid_t = std::array; +#endif // GGML_RPC_RDMA + +#if defined(GGML_RPC_RDMA) && !defined(GGML_RPC_RDMA_APPLE) +static constexpr size_t RDMA_CHUNK = 256 * 1024; // 256 KiB per send/recv (fits default 8 MiB memlock) +static constexpr int RDMA_RX_DEPTH = 24; // pre-posted recv ring: 24 × 256 KiB = 6 MiB +// keep polling the CQ for this long after the last activity, then sleep until the next completion +static constexpr auto RDMA_SPIN_TIME = std::chrono::milliseconds(100); + +struct rdma_conn { + struct ibv_context * ctx = nullptr; + struct ibv_pd * pd = nullptr; + struct ibv_cq * scq = nullptr; // send completions + struct ibv_cq * rcq = nullptr; // recv completions + struct ibv_qp * qp = nullptr; + struct ibv_comp_channel * ch = nullptr; // CQ events, so an idle connection can sleep instead of spinning + + std::chrono::steady_clock::time_point last_active; // last completion or posted send + + void * tx_buf = nullptr; + struct ibv_mr * tx_mr = nullptr; + + void * rx_buf = nullptr; // RDMA_RX_DEPTH × RDMA_CHUNK contiguous + struct ibv_mr * rx_mr = nullptr; + int rx_head = 0; + + uint32_t max_inline = 0; + + uint8_t * rx_slot(int i) const { + return static_cast(rx_buf) + static_cast(i) * RDMA_CHUNK; + } + + bool post_rx(int i) { + struct ibv_sge sge = {}; + sge.addr = (uintptr_t)rx_slot(i); + sge.length = RDMA_CHUNK; + sge.lkey = rx_mr->lkey; + struct ibv_recv_wr wr = {}, * bad = nullptr; + wr.wr_id = (uint64_t)i; + wr.sg_list = &sge; + wr.num_sge = 1; + return ibv_post_recv(qp, &wr, &bad) == 0; + } + + ~rdma_conn() { + if (tx_mr) ibv_dereg_mr(tx_mr); + if (rx_mr) ibv_dereg_mr(rx_mr); + free(tx_buf); + free(rx_buf); + if (qp) ibv_destroy_qp(qp); + if (scq) ibv_destroy_cq(scq); + if (rcq) ibv_destroy_cq(rcq); + if (ch) ibv_destroy_comp_channel(ch); + if (pd) ibv_dealloc_pd(pd); + if (ctx) ibv_close_device(ctx); + } +}; + +// Local RDMA parameters captured during the probe phase and later consumed +// by rdma_activate() after the remote side's caps arrive via HELLO. +struct rdma_local_info { + uint32_t qpn = 0; + uint32_t psn = 0; + uint8_t gid[RDMA_GID_SIZE] = {}; + uint8_t ib_port = 0; + int gid_idx = 0; + enum ibv_mtu path_mtu = IBV_MTU_1024; +}; + +struct rdma_caps { + uint32_t qpn; + uint32_t psn; + uint8_t gid[RDMA_GID_SIZE]; +}; + +static_assert(sizeof(rdma_caps) == RPC_CONN_CAPS_SIZE, "rdma_caps must match conn_caps size"); + +#endif // GGML_RPC_RDMA && !GGML_RPC_RDMA_APPLE + +struct socket_t::impl { + impl(sockfd_t fd) : use_rdma(false), fd(fd) {} + ~impl(); + bool send_data(const void * data, size_t size); + bool recv_data(void * data, size_t size); + bool flush(); + void get_caps(uint8_t * local_caps); + void update_caps(const uint8_t * remote_caps); + +#ifdef GGML_RPC_RDMA + std::optional rdma_build_target_gid(); + +# ifdef GGML_RPC_RDMA_APPLE + std::unique_ptr rdma; +# else + bool rdma_probe(); + bool rdma_send(const void * data, size_t size); + bool rdma_recv(void * data, size_t size); + bool tcp_peer_closed(); + bool rdma_activate(uint32_t remote_qpn, uint32_t remote_psn, const uint8_t * remote_gid); + bool rdma_poll(struct ibv_cq * cq, struct ibv_wc * wc); + bool rdma_wait_event(); + + std::unique_ptr rdma; + rdma_local_info rdma_local = {}; +# endif +#endif // GGML_RPC_RDMA + bool use_rdma; + sockfd_t fd; +}; + +socket_t::impl::~impl() { +#ifdef GGML_RPC_RDMA + rdma.reset(); +#endif // GGML_RPC_RDMA + LOG_DBG("[%s] closing socket %d\n", __func__, this->fd); +#ifdef _WIN32 + if (fd != INVALID_SOCKET) closesocket(this->fd); +#else + if (fd >= 0) close(this->fd); +#endif +} + +#ifdef GGML_RPC_RDMA + +// Build a RoCE GID-shaped 16-byte target from a TCP socket's local address. +// Used to match the socket's local IP against the kernel's GID table so that +// a single memcmp handles IPv4, IPv4-mapped IPv6, and native IPv6 uniformly: +// AF_INET -> ::ffff:a.b.c.d (bytes 10-11 = 0xff, last 4 = IPv4) +// AF_INET6 (IPv4-mapped) -> ::ffff:a.b.c.d (already in GID shape) +// AF_INET6 (native v6) -> the 16-byte IPv6 address as-is +// Returns std::nullopt on unsupported family or getsockname failure. +std::optional socket_t::impl::rdma_build_target_gid() { + sockaddr_storage addr = {}; + socklen_t addr_len = sizeof(addr); + if (getsockname(fd, reinterpret_cast(&addr), &addr_len) != 0) { + return std::nullopt; + } + rdma_gid_t target = {}; + if (addr.ss_family == AF_INET) { + const auto * a = reinterpret_cast(&addr); + target[10] = 0xff; + target[11] = 0xff; + memcpy(&target[12], &a->sin_addr, 4); + return target; + } + if (addr.ss_family == AF_INET6) { + const auto * a = reinterpret_cast(&addr); + memcpy(target.data(), &a->sin6_addr, RDMA_GID_SIZE); + return target; + } + return std::nullopt; +} + +#ifndef GGML_RPC_RDMA_APPLE + +bool socket_t::impl::tcp_peer_closed() { + if (fd < 0) return false; +#ifndef _WIN32 + struct pollfd pfd = { fd, POLLIN | POLLRDHUP, 0 }; + int r = poll(&pfd, 1, 0); + return r > 0 && (pfd.revents & (POLLHUP | POLLERR | POLLRDHUP)); +#else + return false; +#endif +} + +bool socket_t::impl::rdma_probe() { + const char * dev_env = std::getenv("GGML_RDMA_DEV"); + const char * gid_env = std::getenv("GGML_RDMA_GID"); + + auto target_gid = rdma_build_target_gid(); + if (!target_gid) { + return false; + } + + const uint8_t ib_port = 1; + int num_devs = 0; + ibv_device ** devs = ibv_get_device_list(&num_devs); + if (!devs || num_devs == 0) return false; + + ibv_context * ibctx = nullptr; + const char * matched_dev = nullptr; + int gid_idx = gid_env ? atoi(gid_env) : -1; + int gid_version = IBV_GID_TYPE_IB; // 0 = unknown/IB + + for (int d = 0; d < num_devs; d++) { + const char * dn = ibv_get_device_name(devs[d]); + if (dev_env && strcmp(dev_env, dn) != 0) continue; + + ibv_context * ctx = ibv_open_device(devs[d]); + if (!ctx) continue; + + ibv_port_attr pa; + if (ibv_query_port(ctx, ib_port, &pa) != 0) { ibv_close_device(ctx); continue; } + + int found_gid = gid_idx; + int found_version = IBV_GID_TYPE_IB; + if (found_gid < 0) { + // Find a GID on this port whose bytes equal the local TCP address + // (IPv4 or IPv6). Prefer RoCE v2 (UDP/IP, L3-routable) over v1 + // (raw Ethernet, same-L2 only) so silent hangs on L3-routed paths + // are avoided. ibv_query_gid_ex returns gid+type in one call. + int v2_idx = -1; + int v1_idx = -1; + for (int i = 0; i < pa.gid_tbl_len; i++) { + ibv_gid_entry entry = {}; + if (ibv_query_gid_ex(ctx, ib_port, i, &entry, 0) != 0) continue; + if (memcmp(entry.gid.raw, target_gid->data(), RDMA_GID_SIZE) != 0) continue; + if (entry.gid_type == IBV_GID_TYPE_ROCE_V2 && v2_idx < 0) { + v2_idx = i; + } else if (entry.gid_type == IBV_GID_TYPE_ROCE_V1 && v1_idx < 0) { + v1_idx = i; + } + } + if (v2_idx >= 0) { + found_gid = v2_idx; + found_version = IBV_GID_TYPE_ROCE_V2; + } else if (v1_idx >= 0) { + found_gid = v1_idx; + found_version = IBV_GID_TYPE_ROCE_V1; + } + } else { + // Explicit GID index from GGML_RDMA_GID — fetch its type for logging. + ibv_gid_entry entry = {}; + if (ibv_query_gid_ex(ctx, ib_port, found_gid, &entry, 0) == 0) { + found_version = entry.gid_type; + } + } + if (found_gid >= 0) { + ibctx = ctx; + gid_idx = found_gid; + gid_version = found_version; + matched_dev = dn; + rdma_local.path_mtu = pa.active_mtu; + break; + } + ibv_close_device(ctx); + } + ibv_free_device_list(devs); + if (!ibctx) return false; + + rdma_local.ib_port = ib_port; + rdma_local.gid_idx = gid_idx; + + rdma = std::make_unique(); + rdma->ctx = ibctx; + + rdma->pd = ibv_alloc_pd(ibctx); + if (!rdma->pd) return false; + + // without a completion channel rdma_poll() spins all the time, as before + rdma->ch = ibv_create_comp_channel(ibctx); + rdma->scq = ibv_create_cq(ibctx, 16, nullptr, rdma->ch, 0); + rdma->rcq = ibv_create_cq(ibctx, RDMA_RX_DEPTH + 4, nullptr, rdma->ch, 0); + if (!rdma->scq || !rdma->rcq) return false; + + ibv_qp_init_attr qia = {}; + qia.send_cq = rdma->scq; + qia.recv_cq = rdma->rcq; + qia.qp_type = IBV_QPT_RC; + qia.cap.max_send_wr = 4; + qia.cap.max_recv_wr = RDMA_RX_DEPTH + 4; + qia.cap.max_send_sge = 1; + qia.cap.max_recv_sge = 1; + qia.cap.max_inline_data = 256; + + rdma->qp = ibv_create_qp(rdma->pd, &qia); + if (!rdma->qp) return false; + rdma->max_inline = qia.cap.max_inline_data; + + rdma->tx_buf = aligned_alloc(4096, RDMA_CHUNK); + rdma->rx_buf = aligned_alloc(4096, static_cast(RDMA_RX_DEPTH) * RDMA_CHUNK); + if (!rdma->tx_buf || !rdma->rx_buf) return false; + + rdma->tx_mr = ibv_reg_mr(rdma->pd, rdma->tx_buf, RDMA_CHUNK, IBV_ACCESS_LOCAL_WRITE); + rdma->rx_mr = ibv_reg_mr(rdma->pd, rdma->rx_buf, static_cast(RDMA_RX_DEPTH) * RDMA_CHUNK, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + if (!rdma->tx_mr || !rdma->rx_mr) return false; + + ibv_gid local_gid; + if (ibv_query_gid(ibctx, ib_port, gid_idx, &local_gid) != 0) return false; + + rdma_local.qpn = rdma->qp->qp_num; + rdma_local.psn = rdma->qp->qp_num & 0xffffff; + memcpy(&rdma_local.gid, &local_gid, RDMA_GID_SIZE); + + const char * ver_str = ""; + if (gid_version == IBV_GID_TYPE_ROCE_V2) { + ver_str = " RoCEv2"; + } else if (gid_version == IBV_GID_TYPE_ROCE_V1) { + ver_str = " RoCEv1"; + } + LOG_INFO("RDMA probed: dev=%s gid=%d%s qpn=%u inline=%u\n", + matched_dev, gid_idx, ver_str, rdma_local.qpn, rdma->max_inline); + return true; +} + +// Phase 2: Given remote QPN/PSN/GID, transition QP: RESET->INIT->pre-post->RTR->RTS. +// On success, the connection is live and ready for rdma_send/rdma_recv. +bool socket_t::impl::rdma_activate(uint32_t remote_qpn, uint32_t remote_psn, const uint8_t * remote_gid) { + // RESET -> INIT + { + struct ibv_qp_attr a = {}; + a.qp_state = IBV_QPS_INIT; + a.port_num = rdma_local.ib_port; + a.pkey_index = 0; + a.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE; + if (ibv_modify_qp(rdma->qp, &a, + IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS) != 0) { + return false; + } + } + + for (int i = 0; i < RDMA_RX_DEPTH; i++) { + if (!rdma->post_rx(i)) return false; + } + + // INIT -> RTR + { + struct ibv_qp_attr a = {}; + a.qp_state = IBV_QPS_RTR; + a.path_mtu = rdma_local.path_mtu; + a.dest_qp_num = remote_qpn; + a.rq_psn = remote_psn; + a.max_dest_rd_atomic = 1; + a.min_rnr_timer = 1; + a.ah_attr.is_global = 1; + memcpy(&a.ah_attr.grh.dgid, remote_gid, RDMA_GID_SIZE); + a.ah_attr.grh.hop_limit = 1; + a.ah_attr.grh.sgid_index = rdma_local.gid_idx; + a.ah_attr.dlid = 0; + a.ah_attr.port_num = rdma_local.ib_port; + if (ibv_modify_qp(rdma->qp, &a, + IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | + IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER) != 0) { + return false; + } + } + + // RTR -> RTS + { + struct ibv_qp_attr a = {}; + a.qp_state = IBV_QPS_RTS; + a.timeout = 14; + a.retry_cnt = 7; + a.rnr_retry = 7; + a.sq_psn = rdma_local.psn; + a.max_rd_atomic = 1; + if (ibv_modify_qp(rdma->qp, &a, + IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | + IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC) != 0) { + return false; + } + } + + rdma->last_active = std::chrono::steady_clock::now(); + + LOG_INFO("RDMA activated: qpn=%u->%u mtu=%d rx_depth=%d\n", + rdma_local.qpn, remote_qpn, 128 << rdma_local.path_mtu, RDMA_RX_DEPTH); + return true; +} + +// Sleep until the completion channel has an event or the TCP peer closes. +bool socket_t::impl::rdma_wait_event() { + rdma_conn * c = rdma.get(); + // POLLHUP and POLLERR are always reported, the TCP socket carries no data after the RDMA upgrade + struct pollfd pfds[2] = { + { c->ch->fd, POLLIN, 0 }, + { fd, POLLRDHUP, 0 }, + }; + if (poll(pfds, 2, -1) < 0) { + return errno == EINTR; + } + if (pfds[1].revents & (POLLHUP | POLLERR | POLLRDHUP)) { + return false; + } + if (pfds[0].revents & POLLIN) { + struct ibv_cq * ev_cq = nullptr; + void * ev_ctx = nullptr; + if (ibv_get_cq_event(c->ch, &ev_cq, &ev_ctx) != 0) { + return false; + } + ibv_ack_cq_events(ev_cq, 1); + } + return true; +} + +bool socket_t::impl::rdma_poll(struct ibv_cq * cq, struct ibv_wc * wc) { + rdma_conn * c = rdma.get(); + bool armed = false; + for (uint64_t s = 1; ; s++) { + int n = ibv_poll_cq(cq, 1, wc); + if (n > 0) { + c->last_active = std::chrono::steady_clock::now(); + if (wc->status != IBV_WC_SUCCESS) { + LOG_ERROR("RDMA CQ wc error: status=%d (%s) vendor_err=0x%x\n", + wc->status, ibv_wc_status_str(wc->status), wc->vendor_err); + } + return wc->status == IBV_WC_SUCCESS; + } + if (n < 0) return false; + if (armed) { + // armed and still empty: sleep until the next completion + if (!rdma_wait_event()) { + return false; + } + armed = false; + continue; + } + // spin while the connection is busy, arm the CQ once it has been idle for RDMA_SPIN_TIME + // a completion that arrives before arming raises no event, so poll once more after arming + if (c->ch && (s & 0x3FF) == 0 && std::chrono::steady_clock::now() - c->last_active > RDMA_SPIN_TIME) { + if (ibv_req_notify_cq(cq, 0) != 0) { + return false; + } + armed = true; + continue; + } + if ((s & 0xFFFFF) == 0) { + if (tcp_peer_closed()) { + return false; + } + } + } +} + +bool socket_t::impl::rdma_send(const void * data, size_t size) { + rdma_conn * c = rdma.get(); + const uint8_t * src = (const uint8_t *)data; + size_t rem = size; + while (rem > 0) { + size_t chunk = std::min(rem, RDMA_CHUNK); + + struct ibv_sge sge = {}; + struct ibv_send_wr wr = {}, * bad = nullptr; + wr.opcode = IBV_WR_SEND; + wr.sg_list = &sge; + wr.num_sge = 1; + + if (chunk <= c->max_inline) { + sge.addr = (uintptr_t)src; + sge.length = chunk; + wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_INLINE; + } else { + memcpy(c->tx_buf, src, chunk); + sge.addr = (uintptr_t)c->tx_buf; + sge.length = chunk; + sge.lkey = c->tx_mr->lkey; + wr.send_flags = IBV_SEND_SIGNALED; + } + + if (ibv_post_send(c->qp, &wr, &bad) != 0) return false; + c->last_active = std::chrono::steady_clock::now(); + struct ibv_wc wc; + if (!rdma_poll(c->scq, &wc)) return false; + + src += chunk; + rem -= chunk; + } + return true; +} + +bool socket_t::impl::rdma_recv(void * data, size_t size) { + rdma_conn * c = rdma.get(); + uint8_t * dst = (uint8_t *)data; + size_t rem = size; + while (rem > 0) { + struct ibv_wc wc; + if (!rdma_poll(c->rcq, &wc)) return false; + + int slot = (int)wc.wr_id; + size_t got = wc.byte_len; + if (got > rem) { + LOG_ERROR("RDMA recv overrun: got=%zu rem=%zu\n", got, rem); + c->post_rx(slot); + return false; + } + memcpy(dst, c->rx_slot(slot), got); + + if (!c->post_rx(slot)) return false; + + dst += got; + rem -= got; + } + return true; +} + +#endif // !GGML_RPC_RDMA_APPLE (Linux RC transport) + +#endif // GGML_RPC_RDMA + +bool socket_t::impl::send_data(const void * data, size_t size) { + LOG_DBG3("[%s] transport: %s, size: %s\n", __func__, use_rdma ? "RDMA" : "TCP", rpc_log_format_size(size).c_str()); +#ifdef GGML_RPC_RDMA_APPLE + if (use_rdma) { + return rdma->send(data, size); + } +#elif defined(GGML_RPC_RDMA) + if (use_rdma) { + return rdma_send(data, size); + } +#endif + size_t bytes_sent = 0; + while (bytes_sent < size) { + size_t size_to_send = std::min(size - bytes_sent, MAX_CHUNK_SIZE); + ssize_t n = send(fd, (const char *)data + bytes_sent, size_to_send, 0); + if (n < 0) { + LOG_ERROR("send failed (bytes_sent=%zu, size_to_send=%zu)\n", + bytes_sent, size_to_send); + return false; + } + bytes_sent += (size_t)n; + } + return true; +} + +bool socket_t::impl::recv_data(void * data, size_t size) { + LOG_DBG3("[%s] transport: %s, size: %s\n", __func__, use_rdma ? "RDMA" : "TCP", rpc_log_format_size(size).c_str()); +#ifdef GGML_RPC_RDMA_APPLE + if (use_rdma) { + return rdma->recv(data, size); + } +#elif defined(GGML_RPC_RDMA) + if (use_rdma) { + return rdma_recv(data, size); + } +#endif + size_t bytes_recv = 0; + while (bytes_recv < size) { + size_t size_to_recv = std::min(size - bytes_recv, MAX_CHUNK_SIZE); + ssize_t n = recv(fd, (char *)data + bytes_recv, size_to_recv, 0); + if (n < 0) { + LOG_ERROR("recv failed (bytes_recv=%zu, size_to_recv=%zu)\n", + bytes_recv, size_to_recv); + return false; + } + if (n == 0) { + LOG_WARN("recv returned 0 (peer closed?)\n"); + return false; + } + bytes_recv += (size_t)n; + } + return true; +} + +void socket_t::impl::get_caps(uint8_t * local_caps) { + memset(local_caps, 0, RPC_CONN_CAPS_SIZE); +#ifdef GGML_RPC_RDMA + if (std::getenv("GGML_RPC_NO_RDMA")) { + return; + } +# ifdef GGML_RPC_RDMA_APPLE + auto target_gid = rdma_build_target_gid(); + if (target_gid) { + rdma = apple_rdma::probe(fd, target_gid->data(), local_caps); + } + if (!rdma) { + LOG_DBG2("[%s] RDMA probe failed, continuing with TCP\n", __func__); + } +# else + rdma_local = {}; + if (rdma_probe()) { + rdma_caps rc = {}; + rc.qpn = rdma_local.qpn; + rc.psn = rdma_local.psn; + memcpy(rc.gid, rdma_local.gid, RDMA_GID_SIZE); + memcpy(local_caps, &rc, sizeof(rc)); + } else { + rdma.reset(); + LOG_DBG2("[%s] RDMA probe failed, continuing with TCP\n", __func__); + } +# endif +#endif // GGML_RPC_RDMA +} + +void socket_t::impl::update_caps(const uint8_t * remote_caps) { +#ifdef GGML_RPC_RDMA + // a peer that has no RDMA advertises all-zero caps and takes no further part + // in the negotiation, so drop to TCP without reporting a failure + bool remote_rdma = false; + for (size_t i = 0; i < RPC_CONN_CAPS_SIZE; i++) { + remote_rdma |= remote_caps[i] != 0; + } + if (!rdma || !remote_rdma) { + if (rdma && !remote_rdma) { + LOG_WARN("[%s] peer does not support RDMA, continuing with TCP\n", __func__); + } + rdma.reset(); + return; + } +# ifdef GGML_RPC_RDMA_APPLE + bool activated = rdma->activate(remote_caps); +# else + rdma_caps rc = {}; + memcpy(&rc, remote_caps, sizeof(rc)); + bool activated = rdma_activate(rc.qpn, rc.psn, rc.gid); +# endif + if (activated) { + use_rdma = true; + } else { + LOG_WARN("RDMA activate failed, staying on TCP\n"); + rdma.reset(); + } +#else + (void)remote_caps; +#endif // GGML_RPC_RDMA +} + +bool socket_t::impl::flush() { +#ifdef GGML_RPC_RDMA_APPLE + if (use_rdma) { + return rdma->flush(); + } +#endif + return true; +} + +///////////////////////////////////////////////////////////////////////////// + +socket_t::socket_t(std::unique_ptr p) : pimpl(std::move(p)) {} + +socket_t::~socket_t() = default; + +bool socket_t::send_data(const void * data, size_t size) { + return pimpl->send_data(data, size); +} + +bool socket_t::recv_data(void * data, size_t size) { + return pimpl->recv_data(data, size); +} + +bool socket_t::flush() { + return pimpl->flush(); +} + +void socket_t::get_caps(uint8_t * local_caps) { + return pimpl->get_caps(local_caps); +} + +void socket_t::update_caps(const uint8_t * remote_caps) { + return pimpl->update_caps(remote_caps); +} + +const char * socket_t::transport_name() const { + return pimpl->use_rdma ? "RDMA" : "TCP"; +} + +static bool is_valid_fd(sockfd_t sockfd) { +#ifdef _WIN32 + return sockfd != INVALID_SOCKET; +#else + return sockfd >= 0; +#endif +} + +static bool set_no_delay(sockfd_t sockfd) { + int flag = 1; + // set TCP_NODELAY to disable Nagle's algorithm + int ret = setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(int)); + return ret == 0; +} + +static bool set_reuse_addr(sockfd_t sockfd) { + int flag = 1; + int ret = setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof(int)); + return ret == 0; +} + +socket_ptr socket_t::accept() { + auto client_socket_fd = ::accept(pimpl->fd, NULL, NULL); + if (!is_valid_fd(client_socket_fd)) { + return nullptr; + } + if (!set_no_delay(client_socket_fd)) { + LOG_ERROR("Failed to set TCP_NODELAY\n"); + return nullptr; + } + LOG_DBG("[%s] accepted client connection\n", __func__); + return socket_ptr(new socket_t(std::make_unique(client_socket_fd))); +} + +socket_ptr socket_t::create_server(const char * host, int port) { + auto sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (!is_valid_fd(sockfd)) { + return nullptr; + } + if (!set_reuse_addr(sockfd)) { + LOG_ERROR("Failed to set SO_REUSEADDR\n"); + return nullptr; + } + if (inet_addr(host) == INADDR_NONE) { + LOG_ERROR("Invalid host address: %s\n", host); + return nullptr; + } + struct sockaddr_in serv_addr; + serv_addr.sin_family = AF_INET; + serv_addr.sin_addr.s_addr = inet_addr(host); + serv_addr.sin_port = htons(port); + + if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + return nullptr; + } + if (listen(sockfd, 1) < 0) { + return nullptr; + } + LOG_DBG("[%s] listening on %s:%d\n", __func__, host, port); + return socket_ptr(new socket_t(std::make_unique(sockfd))); +} + +socket_ptr socket_t::connect(const char * host, int port) { + auto sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (!is_valid_fd(sockfd)) { + return nullptr; + } + if (!set_no_delay(sockfd)) { + LOG_ERROR("Failed to set TCP_NODELAY\n"); + return nullptr; + } + struct sockaddr_in addr; + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + struct hostent * server = gethostbyname(host); + if (server == NULL) { + LOG_ERROR("Cannot resolve host '%s'\n", host); + return nullptr; + } + memcpy(&addr.sin_addr.s_addr, server->h_addr, server->h_length); + if (::connect(sockfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + return nullptr; + } + LOG_DBG("[%s] connected to %s:%d\n", __func__, host, port); + return socket_ptr(new socket_t(std::make_unique(sockfd))); +} + +#ifdef _WIN32 +static std::mutex g_rpc_transport_mu; +static bool g_rpc_transport_wsa_started = false; +#endif + +bool rpc_transport_init() { +#ifdef _WIN32 + std::lock_guard lock(g_rpc_transport_mu); + if (g_rpc_transport_wsa_started) { + return true; + } + WSADATA wsaData; + int res = WSAStartup(MAKEWORD(2, 2), &wsaData); + if (res != 0) { + return false; + } + g_rpc_transport_wsa_started = true; + return true; +#else + return true; +#endif +} + +void rpc_transport_shutdown() { +#ifdef _WIN32 + std::lock_guard lock(g_rpc_transport_mu); + if (!g_rpc_transport_wsa_started) { + return; + } + WSACleanup(); + g_rpc_transport_wsa_started = false; +#endif +} diff --git a/ggml/src/ggml-rpc/transport.h b/ggml/src/ggml-rpc/transport.h new file mode 100644 index 0000000..59b8845 --- /dev/null +++ b/ggml/src/ggml-rpc/transport.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +struct socket_t; +typedef std::shared_ptr socket_ptr; + +static constexpr size_t MAX_CHUNK_SIZE = 1024ull * 1024ull * 1024ull; // 1 GiB +static constexpr size_t RPC_CONN_CAPS_SIZE = 24; + +struct socket_t { + ~socket_t(); + + bool send_data(const void * data, size_t size); + bool recv_data(void * data, size_t size); + // Must be called at every message boundary: the RDMA transport coalesces + // writes into fixed-size frames and posts the trailing partial frame only + // here. No-op on TCP. + bool flush(); + + socket_ptr accept(); + + void get_caps(uint8_t * local_caps); + void update_caps(const uint8_t * remote_caps); + const char * transport_name() const; + + static socket_ptr create_server(const char * host, int port); + static socket_ptr connect(const char * host, int port); + +private: + struct impl; + explicit socket_t(std::unique_ptr p); + std::unique_ptr pimpl; +}; + +bool rpc_transport_init(); +void rpc_transport_shutdown(); diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index 8e589fa..71c7673 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -39,6 +39,18 @@ if (WIN32) set(CMAKE_CXX_COMPILER "icx") set(CMAKE_CXX_COMPILER_ID "IntelLLVM") endif() + # Level Zero SDK path for Windows (only when GGML_SYCL_SUPPORT_LEVEL_ZERO_API is enabled) + if(GGML_SYCL_SUPPORT_LEVEL_ZERO_API) + if(DEFINED ENV{LEVEL_ZERO_V1_SDK_PATH}) + set(LEVEL_ZERO_V1_SDK_PATH $ENV{LEVEL_ZERO_V1_SDK_PATH}) + if(EXISTS "${LEVEL_ZERO_V1_SDK_PATH}") + target_include_directories(ggml-sycl SYSTEM PRIVATE "${LEVEL_ZERO_V1_SDK_PATH}/include") + set(LEVEL_ZERO_V1_SDK_LIB_PATH "${LEVEL_ZERO_V1_SDK_PATH}/lib") + else() + message(WARNING "LEVEL_ZERO_V1_SDK_PATH set but folder not found: ${LEVEL_ZERO_V1_SDK_PATH}") + endif() + endif() + endif() endif() macro(detect_and_find_package package_name) @@ -93,36 +105,91 @@ endif() target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing") +message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}") +if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API) + # Link against Level Zero loader for direct device memory allocation. + # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging + # in the xe kernel driver during multi-GPU inference. + find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include) + find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH) + if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR) + target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB}) + target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) + message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}") + message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}") + else() + message(WARNING "Level Zero loader or development headers not found, " + "Level Zero API support disabled. " + "Please install the Level Zero SDK/development package " + "to support Level Zero API features. " + "Level Zero API is not mandatory for SYCL backend, " + "but it is required by the special features for better " + "function & performance on Intel GPUs.") + endif() +endif() + # Link against oneDNN set(GGML_SYCL_DNNL 0) if(GGML_SYCL_DNN) - find_package(DNNL) - if(DNNL_FOUND) - if (NOT DEFINED DNNL_GPU_VENDOR) - # default to intel target - set(DNNL_GPU_VENDOR "INTEL") - if(NOT "${GGML_SYCL_TARGET}" STREQUAL "INTEL") - message(WARNING "oneDNN builds bundled with oneapi release only support INTEL target") - endif() + # oneAPI installs dnnl-config.cmake under a versioned path CMake does not + # search by default. When the user did not point CMake at it explicitly + # (DNNL_DIR), auto-detect the oneAPI-bundled GPU build via the DNNLROOT + # environment variable or the standard oneAPI install prefixes. A shell + # with setvars.sh sourced already exports CMAKE_PREFIX_PATH and finds it + # without any of this. + if(NOT DEFINED DNNL_DIR) + if(DEFINED ENV{DNNLROOT} AND EXISTS "$ENV{DNNLROOT}/lib/cmake/dnnl/dnnl-config.cmake") + set(DNNL_DIR "$ENV{DNNLROOT}/lib/cmake/dnnl") + elseif(EXISTS "/opt/intel/oneapi/dnnl/latest/lib/cmake/dnnl/dnnl-config.cmake") + set(DNNL_DIR "/opt/intel/oneapi/dnnl/latest/lib/cmake/dnnl") + elseif(EXISTS "/usr/intel/oneapi/dnnl/latest/lib/cmake/dnnl/dnnl-config.cmake") + set(DNNL_DIR "/usr/intel/oneapi/dnnl/latest/lib/cmake/dnnl") endif() - - # Verify oneDNN was compiled for the same target as llama - if("${GGML_SYCL_TARGET}" STREQUAL "${DNNL_GPU_VENDOR}") - target_link_libraries(ggml-sycl PRIVATE DNNL::dnnl) - set(GGML_SYCL_DNNL 1) - get_target_property(CONFIGS DNNL::dnnl IMPORTED_CONFIGURATIONS) - foreach(CONFIG ${CONFIGS}) - get_target_property(DNNL_LIB DNNL::dnnl IMPORTED_LOCATION_${CONFIG}) - message(STATUS "Found oneDNN: ${DNNL_LIB}") - endforeach() - else() - message(WARNING - "oneDNN must be compiled for the same target as llama.cpp. - llama.cpp: ${GGML_SYCL_TARGET}, oneDNN: ${DNNL_GPU_VENDOR}. - Disabling oneDNN support.") + if(DEFINED DNNL_DIR) + message(STATUS "oneDNN: auto-detected config at ${DNNL_DIR}") endif() + endif() + find_package(DNNL) + if(NOT DNNL_FOUND) + # GGML_SYCL_DNN defaults to ON, so a plain build used to continue + # silently without oneDNN: every flash-attention then runs the much + # slower MKL-FA path (20-40% prefill loss on long prompts). Fail + # loudly instead of producing a degraded binary. + message(FATAL_ERROR + "oneDNN (DNNL) was requested (GGML_SYCL_DNN=ON) but not found. " + "The SYCL backend needs the oneAPI-bundled GPU build: install " + "intel-oneapi-dnnl and intel-oneapi-dnnl-devel from the oneAPI " + "repo, source setvars.sh, or pass " + "-DDNNL_DIR=/lib/cmake/dnnl (or export DNNLROOT). " + "To build without oneDNN on purpose, pass -DGGML_SYCL_DNN=OFF.") + endif() + if(NOT DEFINED DNNL_GPU_VENDOR) + # A DNNL package that does not declare DNNL_GPU_VENDOR has no GPU/SYCL + # support compiled in (e.g. Debian/Ubuntu's CPU-only libdnnl-dev). + # The old code defaulted the vendor to INTEL here, which fake-passed + # the target check below and silently linked a CPU-only library into + # the SYCL backend. + message(FATAL_ERROR + "Found oneDNN at ${DNNL_DIR} but it does not declare " + "DNNL_GPU_VENDOR - it is a CPU-only build (Debian/Ubuntu " + "libdnnl-dev?). Install the oneAPI-bundled GPU build " + "(intel-oneapi-dnnl-devel) instead.") + endif() + + # Verify oneDNN was compiled for the same target as llama + if("${GGML_SYCL_TARGET}" STREQUAL "${DNNL_GPU_VENDOR}") + target_link_libraries(ggml-sycl PRIVATE DNNL::dnnl) + set(GGML_SYCL_DNNL 1) + get_target_property(CONFIGS DNNL::dnnl IMPORTED_CONFIGURATIONS) + foreach(CONFIG ${CONFIGS}) + get_target_property(DNNL_LIB DNNL::dnnl IMPORTED_LOCATION_${CONFIG}) + message(STATUS "Found oneDNN: ${DNNL_LIB}") + endforeach() else() - message(STATUS "oneDNN not found, disabling oneDNN support") + message(FATAL_ERROR + "oneDNN must be compiled for the same target as llama.cpp: " + "llama.cpp: ${GGML_SYCL_TARGET}, oneDNN: ${DNNL_GPU_VENDOR}. " + "Pass -DGGML_SYCL_DNN=OFF to build without oneDNN.") endif() else() message(STATUS "oneDNN support disabled by the user") @@ -135,7 +202,11 @@ endif() if (GGML_SYCL_TARGET STREQUAL "INTEL") add_compile_definitions(GGML_SYCL_WARP_SIZE=16) - target_link_options(ggml-sycl PRIVATE -Xs -ze-intel-greater-than-4GB-buffer-required) + if (NOT GGML_SYCL_DEVICE_ARCH) + target_link_options(ggml-sycl PRIVATE -Xs -ze-intel-greater-than-4GB-buffer-required) + else() + message(STATUS "Skipping -ze-intel-greater-than-4GB-buffer-required for spir64_gen AOT") + endif() # Link against Intel oneMKL if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") @@ -160,7 +231,49 @@ if (GGML_SYCL_HOST_MEM_FALLBACK) endif() if (GGML_SYCL_DEVICE_ARCH) - target_compile_options(ggml-sycl PRIVATE -Xsycl-target-backend --offload-arch=${GGML_SYCL_DEVICE_ARCH}) - target_link_options(ggml-sycl PRIVATE -Xsycl-target-backend --offload-arch=${GGML_SYCL_DEVICE_ARCH}) -endif() + message(STATUS "GGML_SYCL_DEVICE_ARCH=${GGML_SYCL_DEVICE_ARCH} (AOT via spir64_gen)") + target_compile_options( + ggml-sycl PRIVATE + -fsycl-targets=spir64_gen + "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\"" + ) + + # Pass through parallel job (process) count for parallelising the + # `llvm-foreach -- ocloc` invocation for compiling AOT device images. + include(ProcessorCount) + ProcessorCount(_ggml_sycl_nproc) + if (_ggml_sycl_nproc LESS 1) + set(_ggml_sycl_nproc 1) + endif() + set(GGML_SYCL_MAX_PARALLEL_LINK_JOBS ${_ggml_sycl_nproc} CACHE STRING + "Parallel ocloc jobs for spir64_gen AOT device-image lowering") + target_link_options( + ggml-sycl PRIVATE + -fsycl-targets=spir64_gen + "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\"" + -fsycl-max-parallel-link-jobs=${GGML_SYCL_MAX_PARALLEL_LINK_JOBS} + ) + # The XMX dequant-GEMM tiles need the sub-group size of the target: 8 on Xe-HPG (DG2, ARL-H), + # 16 on Xe-HPC and Xe2 or newer. ocloc fails on the other size, so build only the one that fits. + # 0 (unknown name, mixed list, or no XMX) builds no XMX tile and the path stays off. + set(_ggml_sycl_xmx_sg "") + string(TOLOWER "${GGML_SYCL_DEVICE_ARCH}" _ggml_sycl_archs) + string(REPLACE "," ";" _ggml_sycl_archs "${_ggml_sycl_archs}") + foreach(_arch IN LISTS _ggml_sycl_archs) + if (_arch MATCHES "^(dg2|acm|ats-m|arl-h|xe-hpg|12\\.5[567]\\.|12\\.74\\.)") + set(_sg 8) + elseif (_arch MATCHES "^(pvc|bmg|lnl|ptl|wcl|nvl|cri|xe2|xe3|xe-hpc|12\\.60\\.|20\\.|30\\.)") + set(_sg 16) + else() + set(_sg 0) + endif() + if (_ggml_sycl_xmx_sg STREQUAL "" OR _ggml_sycl_xmx_sg EQUAL _sg) + set(_ggml_sycl_xmx_sg ${_sg}) + else() + set(_ggml_sycl_xmx_sg 0) + endif() + endforeach() + message(STATUS "GGML_SYCL_DEVICE_ARCH: XMX dequant-GEMM sub-group size ${_ggml_sycl_xmx_sg} (0 = off)") + target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_XMX_AOT_SG=${_ggml_sycl_xmx_sg}) +endif() diff --git a/ggml/src/ggml-sycl/add-id.cpp b/ggml/src/ggml-sycl/add-id.cpp index e0adc4f..a790185 100644 --- a/ggml/src/ggml-sycl/add-id.cpp +++ b/ggml/src/ggml-sycl/add-id.cpp @@ -26,6 +26,15 @@ static void add_id_kernel( float* dst_row = (float*)((char*)dst + i1 * nb1 + i2 * nb2); const float* src0_row = (const float*)((const char*)src0 + i1 * nb01 + i2 * nb02); + + if (i11 == -1) { // skipped slot, add nothing + for (int64_t i0 = item_ct1.get_local_id(2); i0 < ne0; + i0 += item_ct1.get_local_range(2)) { + dst_row[i0] = src0_row[i0]; + } + return; + } + const float* src1_row = (const float*)((const char*)src1 + i11 * nb11); for (int64_t i0 = item_ct1.get_local_id(2); i0 < ne0; diff --git a/ggml/src/ggml-sycl/backend.hpp b/ggml/src/ggml-sycl/backend.hpp index a526d8e..ab80a2a 100644 --- a/ggml/src/ggml-sycl/backend.hpp +++ b/ggml/src/ggml-sycl/backend.hpp @@ -14,9 +14,11 @@ #define GGML_SYCL_BACKEND_HPP #include "binbcast.hpp" +#include "col2im-1d.hpp" #include "common.hpp" #include "concat.hpp" #include "conv.hpp" +#include "conv3d.hpp" #include "convert.hpp" #include "count-equal.hpp" #include "cpy.hpp" @@ -24,6 +26,7 @@ #include "dmmv.hpp" #include "element_wise.hpp" #include "fattn.hpp" +#include "fusion.hpp" #include "gated_delta_net.hpp" #include "gla.hpp" #include "im2col.hpp" @@ -40,6 +43,8 @@ #include "set_rows.hpp" #include "ssm_conv.hpp" #include "softmax.hpp" +#include "topk-moe.hpp" +#include "topk-radix.hpp" #include "tsembd.hpp" #include "upscale.hpp" #include "wkv.hpp" diff --git a/ggml/src/ggml-sycl/base.hpp b/ggml/src/ggml-sycl/base.hpp new file mode 100644 index 0000000..fe96c4a --- /dev/null +++ b/ggml/src/ggml-sycl/base.hpp @@ -0,0 +1,43 @@ +#ifndef GGML_SYCL_BASE_HPP +#define GGML_SYCL_BASE_HPP + +/** + * Module: base + * + * Description: + * Provides zero-dependency, foundational primitives, core abstractions, + * and low-level system interfaces. This module acts as the lowest layer + * of the architecture and is consumed globally across all subsystems. + * + * Constraints: + * - STRICTLY zero upstream dependencies (leaf module). + * - High stability and backward compatibility required. + */ + +#include + +extern int g_ggml_sycl_debug; +extern int g_ggml_sycl_dev_debug; + +#if defined(__clang__) && __has_builtin(__builtin_expect) +// Hint the optimizer to pipeline the more likely following instruction in branches +# define LIKELY(expr) __builtin_expect(expr, true) +# define UNLIKELY(expr) __builtin_expect(expr, false) +#else +# define LIKELY(expr) (expr) +# define UNLIKELY(expr) (expr) +#endif + +#define GGML_SYCL_DEBUG(...) \ + do { \ + if (UNLIKELY(g_ggml_sycl_debug)) \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) + +#define GGML_SYCL_DEV_DEBUG(...) \ + do { \ + if (UNLIKELY(g_ggml_sycl_dev_debug)) \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) + +#endif // GGML_SYCL_BASE_HPP diff --git a/ggml/src/ggml-sycl/binbcast.cpp b/ggml/src/ggml-sycl/binbcast.cpp index 92dd188..f2f7c4c 100644 --- a/ggml/src/ggml-sycl/binbcast.cpp +++ b/ggml/src/ggml-sycl/binbcast.cpp @@ -1,5 +1,6 @@ #include "binbcast.hpp" +#include #include #include #include @@ -287,6 +288,18 @@ inline void ggml_sycl_op_bin_bcast(ggml_backend_sycl_context & ctx, const ggml_t ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb0, nb1, nb2, nb3, ggml_is_contiguous(src0), ggml_is_contiguous(src1), ggml_is_permuted(src0), ggml_is_permuted(src1), main_stream); +#ifdef GGML_SYCL_HAS_BF16 + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { + op()((const sycl::ext::oneapi::bfloat16 *) src0->data, (const sycl::ext::oneapi::bfloat16 *) src1->data, + (sycl::ext::oneapi::bfloat16 *) dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, ne0, ne1, ne2, + ne3, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb0, nb1, nb2, nb3, ggml_is_contiguous(src0), + ggml_is_contiguous(src1), ggml_is_permuted(src0), ggml_is_permuted(src1), main_stream); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_BF16) { + op()((const sycl::ext::oneapi::bfloat16 *) src0->data, (const float *) src1->data, + (sycl::ext::oneapi::bfloat16 *) dst->data, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, ne0, ne1, ne2, + ne3, nb00, nb01, nb02, nb03, nb10, nb11, nb12, nb13, nb0, nb1, nb2, nb3, ggml_is_contiguous(src0), + ggml_is_contiguous(src1), ggml_is_permuted(src0), ggml_is_permuted(src1), main_stream); +#endif } else { fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__, ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type)); @@ -344,3 +357,294 @@ void ggml_sycl_repeat(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_op_repeat(ctx, dst); } +// fused ADD+ADD: dst = (src0 + src1) + src2. Same indexing as k_bin_bcast, so mixed +// types, broadcast, and non-contiguous layouts that add() already handles also fuse. +template +static void k_bin_bcast3(const src0_t * src0, const src1_t * src1, const src2_t * src2, dst_t * dst, + int ne0, int ne1, int ne2, int ne3, + int ne10, int ne11, int ne12, int ne13, + int ne20, int ne21, int ne22, int ne23, + int s1, int s2, int s3, + int s00, int s01, int s02, int s03, + int s10, int s11, int s12, int s13, + int s20, int s21, int s22, int s23, + const sycl::nd_item<3> & item_ct1) { + const int i0s = item_ct1.get_local_range(2) * item_ct1.get_group(2) + + item_ct1.get_local_id(2); + const int i1 = (item_ct1.get_local_range(1) * item_ct1.get_group(1) + + item_ct1.get_local_id(1)); + const int i2 = (item_ct1.get_local_range(0) * item_ct1.get_group(0) + + item_ct1.get_local_id(0)) / + ne3; + const int i3 = (item_ct1.get_local_range(0) * item_ct1.get_group(0) + + item_ct1.get_local_id(0)) % + ne3; + + if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) { + return; + } + + const int i11 = i1 % ne11; + const int i12 = i2 % ne12; + const int i13 = i3 % ne13; + const int i21 = i1 % ne21; + const int i22 = i2 % ne22; + const int i23 = i3 % ne23; + + const size_t i_src0 = i3 * s03 + i2 * s02 + i1 * s01; + const size_t i_src1 = i13 * s13 + i12 * s12 + i11 * s11; + const size_t i_src2 = i23 * s23 + i22 * s22 + i21 * s21; + const size_t i_dst = i3 * s3 + i2 * s2 + i1 * s1; + + const src0_t * src0_row = src0 + i_src0; + const src1_t * src1_row = src1 + i_src1; + const src2_t * src2_row = src2 + i_src2; + dst_t * dst_row = dst + i_dst; + + for (int i0 = i0s; i0 < ne0; + i0 += item_ct1.get_local_range(2) * item_ct1.get_group_range(2)) { + const int i10 = i0 % ne10; + const int i20 = i0 % ne20; + const float acc = bin_op((float) src0_row[i0 * s00], (float) src1_row[i10 * s10]); + dst_row[i0] = (dst_t) bin_op(acc, (float) src2_row[i20 * s20]); + } +} + +template +static void k_bin_bcast3_unravel(const src0_t * src0, const src1_t * src1, const src2_t * src2, dst_t * dst, + int ne0, int ne1, int ne2, int ne3, + int ne10, int ne11, int ne12, int ne13, + int ne20, int ne21, int ne22, int ne23, + int s1, int s2, int s3, + int s00, int s01, int s02, int s03, + int s10, int s11, int s12, int s13, + int s20, int s21, int s22, int s23, + const sycl::nd_item<3> & item_ct1) { + const int i = item_ct1.get_local_range(2) * item_ct1.get_group(2) + + item_ct1.get_local_id(2); + + const int i3 = i / (ne2 * ne1 * ne0); + const int i2 = (i / (ne1 * ne0)) % ne2; + const int i1 = (i / ne0) % ne1; + const int i0 = i % ne0; + + if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) { + return; + } + + const int i11 = i1 % ne11; + const int i12 = i2 % ne12; + const int i13 = i3 % ne13; + const int i21 = i1 % ne21; + const int i22 = i2 % ne22; + const int i23 = i3 % ne23; + + const size_t i_src0 = i3 * s03 + i2 * s02 + i1 * s01; + const size_t i_src1 = i13 * s13 + i12 * s12 + i11 * s11; + const size_t i_src2 = i23 * s23 + i22 * s22 + i21 * s21; + const size_t i_dst = i3 * s3 + i2 * s2 + i1 * s1; + + const int i10 = i0 % ne10; + const int i20 = i0 % ne20; + const float acc = bin_op((float) src0[i_src0 + i0 * s00], (float) src1[i_src1 + i10 * s10]); + dst[i_dst + i0] = (dst_t) bin_op(acc, (float) src2[i_src2 + i20 * s20]); +} + +template +static void launch_bin_bcast3(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, + const ggml_tensor * src2, ggml_tensor * dst) { + dpct::queue_ptr stream = ctx.stream(); + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + GGML_TENSOR_TERNARY_OP_LOCALS + + int nr1[4] = { (int) (ne10 / ne0), (int) (ne11 / ne1), (int) (ne12 / ne2), (int) (ne13 / ne3) }; + int nr2[4] = { (int) (ne20 / ne0), (int) (ne21 / ne1), (int) (ne22 / ne2), (int) (ne23 / ne3) }; + + int64_t cne[] = { ne0, ne1, ne2, ne3 }; + int64_t cne0[] = { ne00, ne01, ne02, ne03 }; + int64_t cne1[] = { ne10, ne11, ne12, ne13 }; + int64_t cne2[] = { ne20, ne21, ne22, ne23 }; + size_t cnb[] = { nb0, nb1, nb2, nb3 }; + size_t cnb0[] = { nb00, nb01, nb02, nb03 }; + size_t cnb1[] = { nb10, nb11, nb12, nb13 }; + size_t cnb2[] = { nb20, nb21, nb22, nb23 }; + + auto collapse = [](int64_t cne[]) { + cne[0] *= cne[1]; + cne[1] = cne[2]; + cne[2] = cne[3]; + cne[3] = 1; + }; + + auto collapse_nb = [](size_t cnb[], int64_t cne[]) { + cnb[1] *= cne[1]; + cnb[2] *= cne[2]; + cnb[3] *= cne[3]; + }; + + const bool can_collapse = ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(src2) && + !ggml_is_permuted(src0) && !ggml_is_permuted(src1) && !ggml_is_permuted(src2); + if (can_collapse) { + for (int i = 0; i < 4; i++) { + if (nr1[i] != 1 || nr2[i] != 1) { + break; + } + if (i > 0) { + collapse_nb(cnb, cne); + collapse_nb(cnb0, cne0); + collapse_nb(cnb1, cne1); + collapse_nb(cnb2, cne2); + collapse(cne); + collapse(cne0); + collapse(cne1); + collapse(cne2); + } + } + } + + { + int64_t ne0 = cne[0]; + int64_t ne1 = cne[1]; + int64_t ne2 = cne[2]; + int64_t ne3 = cne[3]; + + int64_t ne10 = cne1[0]; + int64_t ne11 = cne1[1]; + int64_t ne12 = cne1[2]; + int64_t ne13 = cne1[3]; + + int64_t ne20 = cne2[0]; + int64_t ne21 = cne2[1]; + int64_t ne22 = cne2[2]; + int64_t ne23 = cne2[3]; + + size_t s1 = cnb[1] / sizeof(dst_t); + size_t s2 = cnb[2] / sizeof(dst_t); + size_t s3 = cnb[3] / sizeof(dst_t); + + size_t s00 = cnb0[0] / sizeof(src0_t); + size_t s01 = cnb0[1] / sizeof(src0_t); + size_t s02 = cnb0[2] / sizeof(src0_t); + size_t s03 = cnb0[3] / sizeof(src0_t); + + size_t s10 = cnb1[0] / sizeof(src1_t); + size_t s11 = cnb1[1] / sizeof(src1_t); + size_t s12 = cnb1[2] / sizeof(src1_t); + size_t s13 = cnb1[3] / sizeof(src1_t); + + size_t s20 = cnb2[0] / sizeof(src2_t); + size_t s21 = cnb2[1] / sizeof(src2_t); + size_t s22 = cnb2[2] / sizeof(src2_t); + size_t s23 = cnb2[3] / sizeof(src2_t); + + GGML_ASSERT(cnb[0] % sizeof(dst_t) == 0 && cnb[1] % sizeof(dst_t) == 0 && cnb[2] % sizeof(dst_t) == 0 && + cnb[3] % sizeof(dst_t) == 0); + GGML_ASSERT(cnb0[0] % sizeof(src0_t) == 0 && cnb0[1] % sizeof(src0_t) == 0 && cnb0[2] % sizeof(src0_t) == 0 && + cnb0[3] % sizeof(src0_t) == 0); + GGML_ASSERT(cnb1[0] % sizeof(src1_t) == 0 && cnb1[1] % sizeof(src1_t) == 0 && cnb1[2] % sizeof(src1_t) == 0 && + cnb1[3] % sizeof(src1_t) == 0); + GGML_ASSERT(cnb2[0] % sizeof(src2_t) == 0 && cnb2[1] % sizeof(src2_t) == 0 && cnb2[2] % sizeof(src2_t) == 0 && + cnb2[3] % sizeof(src2_t) == 0); + + const src0_t * src0_dd = (const src0_t *) src0->data; + const src1_t * src1_dd = (const src1_t *) src1->data; + const src2_t * src2_dd = (const src2_t *) src2->data; + dst_t * dst_dd = (dst_t *) dst->data; + + const int block_size = 128; + int64_t hne0 = std::max(ne0 / 2LL, 1LL); + + sycl::range<3> block_dims(1, 1, 1); + block_dims[2] = std::min(hne0, block_size); + block_dims[1] = std::min(ne1, block_size / (unsigned int) block_dims[2]); + block_dims[0] = std::min(std::min(ne2 * ne3, + block_size / (unsigned int) block_dims[2] / + (unsigned int) block_dims[1]), + 64U); + + sycl::range<3> block_nums((ne2 * ne3 + block_dims[0] - 1) / block_dims[0], + (ne1 + block_dims[1] - 1) / block_dims[1], + (hne0 + block_dims[2] - 1) / block_dims[2]); + + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + + if (block_nums[0] > 65535) { + int block_num = (ne0 * ne1 * ne2 * ne3 + block_size - 1) / block_size; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, block_num) * sycl::range<3>(1, 1, block_size), + sycl::range<3>(1, 1, block_size)), + [=](sycl::nd_item<3> item_ct1) { + k_bin_bcast3_unravel(src0_dd, src1_dd, src2_dd, dst_dd, ne0, ne1, ne2, ne3, ne10, ne11, + ne12, ne13, ne20, ne21, ne22, ne23, s1, s2, s3, s00, s01, s02, s03, + s10, s11, s12, s13, s20, s21, s22, s23, item_ct1); + }); + } else { + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_bin_bcast3(src0_dd, src1_dd, src2_dd, dst_dd, ne0, ne1, ne2, ne3, ne10, + ne11, ne12, ne13, ne20, ne21, ne22, ne23, s1, s2, s3, s00, + s01, s02, s03, s10, s11, s12, s13, s20, s21, s22, s23, + item_ct1); + }); + } + } +} + +void ggml_sycl_op_add_add_fused(ggml_backend_sycl_context & ctx, ggml_tensor * add0, ggml_tensor * add1) { + const ggml_tensor * src0 = add0->src[0]; + const ggml_tensor * src1 = add0->src[1]; + const ggml_tensor * src2 = add1->src[1]; + ggml_tensor * dst = add1; + + GGML_ASSERT(add1->src[0] == add0); + GGML_ASSERT(ggml_sycl_add_kernel_supports(src0->type, src1->type, add0->type)); + GGML_ASSERT(ggml_sycl_add_kernel_supports(add0->type, src2->type, dst->type)); + + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F32) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && src2->type == GGML_TYPE_F16 && + dst->type == GGML_TYPE_F16) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F16) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && src2->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F16) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_F16 && + dst->type == GGML_TYPE_F16) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); + } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32 && src2->type == GGML_TYPE_I32 && + dst->type == GGML_TYPE_I32) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); + } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16 && src2->type == GGML_TYPE_I16 && + dst->type == GGML_TYPE_I16) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); +#ifdef GGML_SYCL_HAS_BF16 + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && src2->type == GGML_TYPE_BF16 && + dst->type == GGML_TYPE_BF16) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_BF16) { + launch_bin_bcast3( + ctx, src0, src1, src2, dst); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && src2->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_BF16) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_BF16 && + dst->type == GGML_TYPE_BF16) { + launch_bin_bcast3(ctx, src0, src1, src2, dst); +#endif + } else { + fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s, src2: %s\n", __func__, + ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type), + ggml_type_name(src2->type)); + GGML_ABORT("fatal error"); + } +} + diff --git a/ggml/src/ggml-sycl/binbcast.hpp b/ggml/src/ggml-sycl/binbcast.hpp index 9cce0f0..0e5a5ca 100644 --- a/ggml/src/ggml-sycl/binbcast.hpp +++ b/ggml/src/ggml-sycl/binbcast.hpp @@ -34,6 +34,36 @@ void ggml_sycl_div(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_repeat(ggml_backend_sycl_context & ctx, ggml_tensor * dst); +void ggml_sycl_op_add_add_fused(ggml_backend_sycl_context & ctx, ggml_tensor * add0, ggml_tensor * add1); + +// Type combinations the standalone SYCL add() kernel can run. Fused ADD+ADD +// uses the same set; anything else falls back to two add() launches. +inline bool ggml_sycl_add_kernel_supports(enum ggml_type src0, enum ggml_type src1, enum ggml_type dst) { + if (src0 == GGML_TYPE_F32 && src1 == GGML_TYPE_F32 && dst == GGML_TYPE_F32) { + return true; + } + if (src0 == GGML_TYPE_F16 && src1 == GGML_TYPE_F16 && dst == GGML_TYPE_F16) { + return true; + } + if (src0 == GGML_TYPE_F16 && src1 == GGML_TYPE_F32 && dst == GGML_TYPE_F16) { + return true; + } + if (src0 == GGML_TYPE_I32 && src1 == GGML_TYPE_I32 && dst == GGML_TYPE_I32) { + return true; + } + if (src0 == GGML_TYPE_I16 && src1 == GGML_TYPE_I16 && dst == GGML_TYPE_I16) { + return true; + } +#ifdef GGML_SYCL_HAS_BF16 + if (src0 == GGML_TYPE_BF16 && src1 == GGML_TYPE_BF16 && dst == GGML_TYPE_BF16) { + return true; + } + if (src0 == GGML_TYPE_BF16 && src1 == GGML_TYPE_F32 && dst == GGML_TYPE_BF16) { + return true; + } +#endif + return false; +} #endif //GGML_SYCL_BINBCAST_HPP diff --git a/ggml/src/ggml-sycl/col2im-1d.cpp b/ggml/src/ggml-sycl/col2im-1d.cpp new file mode 100644 index 0000000..c4f0900 --- /dev/null +++ b/ggml/src/ggml-sycl/col2im-1d.cpp @@ -0,0 +1,102 @@ +#include "col2im-1d.hpp" + +template +static void col2im_1d_sycl( + const T * col, + T * dst, + const int T_in, + const sycl::uint3 T_out_fd, + const int K, + const int K_OC, + const int32_t s0, + const int32_t p0, + const int total, + dpct::queue_ptr stream) { + + const uint32_t block_size = SYCL_COL2IM_1D_BLOCK_SIZE; + const uint32_t num_blocks = (uint32_t) ((total + block_size - 1) / block_size); + + stream->parallel_for( + sycl::nd_range<3>( + sycl::range<3>(1, 1, num_blocks * block_size), + sycl::range<3>(1, 1, block_size)), + [=](sycl::nd_item<3> item_ct1) { + const int idx = (int) item_ct1.get_global_id(2); + if (idx >= total) { + return; + } + + const sycl::uint2 qr = fast_div_modulo((uint32_t) idx, T_out_fd); + const int oc = (int) qr.x(); + const int t_out = (int) qr.y(); + const int t_abs = t_out + p0; + + int t_in_min = (t_abs - K + s0) / s0; + if (t_in_min < 0) { + t_in_min = 0; + } + int t_in_max = t_abs / s0; + if (t_in_max >= T_in) { + t_in_max = T_in - 1; + } + + float sum = 0.0f; + for (int t_in = t_in_min; t_in <= t_in_max; ++t_in) { + const int k = t_abs - t_in * s0; + sum += static_cast(col[(oc * K + k) + t_in * K_OC]); + } + + dst[idx] = static_cast(sum); + }); +} + +void ggml_sycl_op_col2im_1d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + + GGML_ASSERT(src0 != nullptr); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(src0->type == dst->type); + + const int32_t s0 = ((const int32_t *) dst->op_params)[0]; + const int32_t OC = ((const int32_t *) dst->op_params)[1]; + const int32_t p0 = ((const int32_t *) dst->op_params)[2]; + + const int K_OC = (int) src0->ne[0]; + const int T_in = (int) src0->ne[1]; + const int K = K_OC / OC; + const int T_out = (int) dst->ne[0]; + + GGML_ASSERT(OC > 0); + GGML_ASSERT(K_OC % OC == 0); + + const sycl::uint3 T_out_fd = init_fastdiv_values((uint32_t) T_out); + + const int total = T_out * OC; + + dpct::queue_ptr stream = ctx.stream(); + + switch (src0->type) { + case GGML_TYPE_F32: + col2im_1d_sycl( + (const float *) src0->data, + (float *) dst->data, + T_in, T_out_fd, K, K_OC, s0, p0, total, stream); + break; + case GGML_TYPE_F16: + col2im_1d_sycl( + (const sycl::half *) src0->data, + (sycl::half *) dst->data, + T_in, T_out_fd, K, K_OC, s0, p0, total, stream); + break; +#ifdef GGML_SYCL_HAS_BF16 + case GGML_TYPE_BF16: + col2im_1d_sycl( + (const sycl::ext::oneapi::bfloat16 *) src0->data, + (sycl::ext::oneapi::bfloat16 *) dst->data, + T_in, T_out_fd, K, K_OC, s0, p0, total, stream); + break; +#endif + default: + GGML_ABORT("col2im_1d: unsupported type %d", src0->type); + } +} diff --git a/ggml/src/ggml-sycl/col2im-1d.hpp b/ggml/src/ggml-sycl/col2im-1d.hpp new file mode 100644 index 0000000..cfb53da --- /dev/null +++ b/ggml/src/ggml-sycl/col2im-1d.hpp @@ -0,0 +1,8 @@ +#ifndef GGML_SYCL_COL2IM_1D_HPP +#define GGML_SYCL_COL2IM_1D_HPP + +#include "common.hpp" + +void ggml_sycl_op_col2im_1d(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_COL2IM_1D_HPP diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cpp index 05fd5ef..f01d0ff 100644 --- a/ggml/src/ggml-sycl/common.cpp +++ b/ggml/src/ggml-sycl/common.cpp @@ -11,6 +11,10 @@ // #include "common.hpp" +#include +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +#include +#endif #include "ggml-backend-impl.h" #include "ggml-impl.h" @@ -55,6 +59,20 @@ bool gpu_has_xmx(sycl::device &dev) { return dev.has(sycl::aspect::ext_intel_matrix); } +int ggml_sycl_get_env(const char *env_name, int default_val) { + char *user_device_string = getenv(env_name); + int user_number = default_val; + + unsigned n; + if (user_device_string != NULL && + sscanf(user_device_string, " %u", &n) == 1) { + user_number = (int)n; + } else { + user_number = default_val; + } + return user_number; +} + int64_t downsample_sycl_global_range(int64_t accumulate_block_num, int64_t block_size) { const int64_t max_range = std::numeric_limits::max(); int64_t sycl_down_blk_size = block_size; @@ -66,6 +84,69 @@ int64_t downsample_sycl_global_range(int64_t accumulate_block_num, int64_t block return sycl_down_blk_size; } +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +static bool ggml_sycl_use_level_zero_device_alloc(sycl::queue &q) { + return g_ggml_sycl_use_level_zero_api && + q.get_device().is_gpu() && + q.get_backend() == sycl::backend::ext_oneapi_level_zero; +} +#endif + +// Use Level Zero zeMemAllocDevice to avoid sycl::malloc_device triggering +// DMA-buf/TTM system RAM staging in the xe kernel driver during multi-GPU inference. +void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type type) { +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + if (ggml_sycl_use_level_zero_device_alloc(q)) { + void *ptr = nullptr; + auto ze_ctx = sycl::get_native(q.get_context()); + auto ze_dev = sycl::get_native(q.get_device()); +#ifdef ZE_RELAXED_ALLOCATION_LIMITS_EXP_NAME + ze_relaxed_allocation_limits_exp_desc_t relaxed_desc = { + ZE_STRUCTURE_TYPE_RELAXED_ALLOCATION_LIMITS_EXP_DESC, + nullptr, + ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE, + }; + ze_device_mem_alloc_desc_t alloc_desc = { + ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, + &relaxed_desc, + 0, + 0, + }; +#else + ze_device_mem_alloc_desc_t alloc_desc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, nullptr, 0, 0}; +#endif + // request SYCL_BUFFER_ALIGNMENT so tensor offsets aligned to it stay aligned in absolute terms + ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, SYCL_BUFFER_ALIGNMENT, ze_dev, &ptr); + if (r == ZE_RESULT_SUCCESS && ptr) { + ggml_sycl_memtrace_add(type, ptr, size); + return ptr; + } + ggml_sycl_memtrace_fail(type, size); + return nullptr; + } +#endif + void * ptr = sycl::malloc_device(size, q); + if (ptr == nullptr) { + ggml_sycl_memtrace_fail(type, size); + return nullptr; + } + ggml_sycl_memtrace_add(type, ptr, size); + return ptr; +} + +void ggml_sycl_free_device(void *ptr, sycl::queue &q) { + if (!ptr) return; + ggml_sycl_memtrace_del(ptr); +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + if (ggml_sycl_use_level_zero_device_alloc(q)) { + auto ze_ctx = sycl::get_native(q.get_context()); + zeMemFree(ze_ctx, ptr); + return; + } +#endif + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, q))); +} + void release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector streams) { for (int i = 0; i < ggml_sycl_info().device_count; ++i) { for (int64_t is = 0; is < GGML_SYCL_MAX_STREAMS; ++is) { @@ -75,8 +156,7 @@ void release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector str } if (extra->data_device[i] != nullptr && streams.size()>0) { ggml_sycl_set_device(i); - SYCL_CHECK( - CHECK_TRY_ERROR(sycl::free(extra->data_device[i], *(streams[i])))); + SYCL_CHECK(CHECK_TRY_ERROR(ggml_sycl_free_device(extra->data_device[i], *(streams[i])))); } } delete extra; diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index fd84c91..8f2f892 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -16,8 +16,12 @@ #include #include #include +#include #include +#include +#include +#include "base.hpp" #include "dpct/helper.hpp" #include "ggml.h" #include "ggml-impl.h" @@ -25,9 +29,21 @@ #include "presets.hpp" #include "type.hpp" #include "sycl_hw.hpp" +#include "memtrace.hpp" namespace syclexp = sycl::ext::oneapi::experimental; +#if defined(__INTEL_LLVM_COMPILER) && __has_include() + #include + #ifndef GGML_SYCL_HAS_BF16 + #define GGML_SYCL_HAS_BF16 + #endif +#endif + +#if defined(__INTEL_LLVM_COMPILER) + #define GGML_SYCL_DMMV_HAS_ESIMD +#endif + #if GGML_SYCL_DNNL #include "dnnl.hpp" #include "dnnl_sycl.hpp" @@ -37,6 +53,7 @@ namespace syclexp = sycl::ext::oneapi::experimental; #define GGML_COMMON_IMPL_SYCL #define SYCL_FLASH_ATTN //remove it to disable FLASH_ATTENTION in building. #define SYCL_FAST_FP16 //don't change. remove it will break fattn-tile.hpp building +#define GGML_SYCL_FA_ALL_QUANTS //define it to enable all quantization types in flash attention. undefine it to only support F16, Q4_0 and Q8_0 in flash attention. /* suppress warning spam */ #pragma clang diagnostic push @@ -50,25 +67,78 @@ void ggml_sycl_host_free(void* ptr); extern int g_ggml_sycl_debug; -extern int g_ggml_sycl_disable_optimize; +extern int g_ggml_sycl_enable_optimize; +extern int g_ggml_sycl_enable_fusion; +extern int g_ggml_sycl_enable_esimd; +extern int g_ggml_sycl_mmvq_wide; +extern int g_ggml_sycl_enable_xmx_esimd; +extern int g_ggml_sycl_xmx_max_cols; +extern int g_ggml_sycl_xmx_glu_max_cols; extern int g_ggml_sycl_prioritize_dmmv; -extern int g_ggml_sycl_enable_flash_attention; - -#if defined(__clang__) && __has_builtin(__builtin_expect) -// Hint the optimizer to pipeline the more likely following instruction in branches -# define LIKELY(expr) __builtin_expect(expr, true) -# define UNLIKELY(expr) __builtin_expect(expr, false) +// Which quantized weight formats may take the XMX dequant-GEMM paths. A bitmask rather than one +// flag per path, so a format can be enabled or measured on its own and adding a format is one bit. +enum ggml_sycl_xmx_gather_type { + GGML_SYCL_XMX_GATHER_IQ4_NL = 1 << 0, + GGML_SYCL_XMX_GATHER_IQ3_S = 1 << 1, + GGML_SYCL_XMX_GATHER_IQ4_XS = 1 << 2, + GGML_SYCL_XMX_GATHER_IQ3_XXS = 1 << 3, + GGML_SYCL_XMX_GATHER_IQ2_XXS = 1 << 4, + GGML_SYCL_XMX_GATHER_IQ2_XS = 1 << 5, + GGML_SYCL_XMX_GATHER_IQ2_S = 1 << 6, + GGML_SYCL_XMX_GATHER_IQ1_S = 1 << 7, + GGML_SYCL_XMX_GATHER_IQ1_M = 1 << 8, + GGML_SYCL_XMX_GATHER_Q8_0 = 1 << 9, + GGML_SYCL_XMX_GATHER_Q4_K = 1 << 10, + GGML_SYCL_XMX_GATHER_Q5_K = 1 << 11, + GGML_SYCL_XMX_GATHER_Q6_K = 1 << 12, +}; +static constexpr int GGML_SYCL_XMX_GATHER_TYPES_DEFAULT = ~0; +extern int g_ggml_sycl_xmx_gather_types; +// Which joint_matrix combinations the XMX dequant-GEMM paths may use, one bit each (see fused-gemm.cpp). +// GGML_SYCL_DYNAMIC_PRECISION picks the operand type, this mask the combinations of that type. +static constexpr int GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT = 0xff; +extern int g_ggml_sycl_xmx_gather_shapes; + +// GGML_SYCL_DYNAMIC_PRECISION: operand type of the XMX dequant-GEMM paths. F32 turns them off and +// keeps the library GEMM in f32. A src1 precision request of an op [TAG_GGML_PREC] is always met. +enum ggml_sycl_dynamic_precision { + GGML_SYCL_DYNAMIC_PRECISION_F16, + GGML_SYCL_DYNAMIC_PRECISION_BF16, + GGML_SYCL_DYNAMIC_PRECISION_TF32, + GGML_SYCL_DYNAMIC_PRECISION_F32, +}; +#ifdef GGML_SYCL_F16 +static constexpr int GGML_SYCL_DYNAMIC_PRECISION_DEFAULT = GGML_SYCL_DYNAMIC_PRECISION_F16; #else -# define LIKELY(expr) (expr) -# define UNLIKELY(expr) (expr) +static constexpr int GGML_SYCL_DYNAMIC_PRECISION_DEFAULT = GGML_SYCL_DYNAMIC_PRECISION_F32; #endif +extern int g_ggml_sycl_dynamic_precision; +// GGML_SYCL_DYNAMIC_REQUIRED_PRECISION: the XMX type an F32 src1 request may run on instead of f32 +// (TF32, or BF16 which also allows tf32). F32 (default): none. F16: src1 requests are ignored. +extern int g_ggml_sycl_dynamic_required_precision; + +// [TAG_GGML_PREC] src1 precision request of the MUL_MAT/MUL_MAT_ID op dst +static inline int32_t ggml_sycl_src1_prec(const ggml_tensor * dst) { + return g_ggml_sycl_dynamic_required_precision == GGML_SYCL_DYNAMIC_PRECISION_F16 ? GGML_PREC_UNDEFINED : + dst->op_params[3]; +} + +// [TAG_GGML_PREC] the library GEMM and dmmv may convert src1 of the MUL_MAT/MUL_MAT_ID op dst to f16 +static inline bool ggml_sycl_src1_f16_ok(const ggml_tensor * dst) { + const int32_t src1_prec = ggml_sycl_src1_prec(dst); + return g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 && + (src1_prec == GGML_PREC_UNDEFINED || src1_prec >= GGML_PREC_F16); +} +extern int g_ggml_sycl_enable_flash_attention; +extern int g_ggml_sycl_dev2dev_memcpy; +extern int g_ggml_sycl_fa_onednn; +extern int g_ggml_sycl_fa_onednn_max_kv; +extern int g_ggml_sycl_enable_mkl_fa; +extern int g_ggml_sycl_memtrace; +extern int g_ggml_sycl_memtrace_step; +extern int g_ggml_sycl_graph_eviction_timeout; -#define GGML_SYCL_DEBUG(...) \ - do { \ - if (UNLIKELY(g_ggml_sycl_debug)) \ - fprintf(stderr, __VA_ARGS__); \ - } while (0) #define CHECK_TRY_ERROR(expr) \ [&]() { \ @@ -117,6 +187,12 @@ enum ggml_sycl_backend_gpu_mode { SYCL_MUL_GPU_MODE }; +enum ggml_sycl_dev2dev_memcpy_mode { + DEV2DEV_MEMCPY_SYCL = 0, + DEV2DEV_MEMCPY_L0 = 1, + DEV2DEV_MEMCPY_FORWARD = 2 +}; + static_assert(sizeof(sycl::half) == sizeof(ggml_fp16_t), "wrong fp16 size"); static void crash() { @@ -158,6 +234,41 @@ typedef sycl::float2 dfloat2; #define MMVQ_MAX_BATCH_SIZE 8 +#if defined(__INTEL_LLVM_COMPILER) && __has_include() +# define GGML_SYCL_MMVQ_HAS_XMX +#endif // __INTEL_LLVM_COMPILER + +// default of the most columns the XMX mul_mat_vec_q handles (GGML_SYCL_XMX_MAX_COLS), wider batches use other kernels +#define GGML_SYCL_XMX_MAX_COLS_DEFAULT 80 + +// most columns the fused gate and up XMX kernel is built for, GGML_SYCL_XMX_GLU_MAX_COLS can only lower this limit +#define GGML_SYCL_XMX_GLU_MAX_COLS 16 + +static inline bool ggml_sycl_xmx_supports_type(ggml_type type) { + switch (type) { + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q8_0: + return true; + default: + return false; + } +} + +// the DPAS shapes and 2D block loads used by the XMX kernel are those of Xe2 +static inline bool ggml_sycl_xmx_supports_arch(gpu_arch arch) { + return arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31 || + arch == gpu_arch::intel_gpu_lnl_m; +} + +// fewest columns for which the XMX kernel is faster than the other kernels +static inline int ggml_sycl_xmx_min_cols(ggml_type type) { + return type == GGML_TYPE_Q2_K || type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q4_K ? 1 : 2; +} + static int g_all_sycl_device_count = -1; static bool g_ggml_backend_sycl_buffer_type_initialized = false; @@ -204,9 +315,11 @@ inline dpct::err0 ggml_sycl_set_device(const int device) try { ////////////////////// struct optimize_feature { bool reorder=false; + bool onednn_optimized_gemm=false; }; struct sycl_device_info { + bool active; //the device is activated/used in application. int cc; // compute capability int nsm; // number of streaming multiprocessors (CUDA) maps to the maximum // number of compute units on a SYCL device. @@ -216,13 +329,22 @@ struct sycl_device_info { int max_wg_per_cu; // max work groups per compute unit - refer to // cudaOccupancyMaxActiveBlocksPerMultiprocessor bool vmm; // virtual memory support + bool l0_device_type_valid; + bool l0_discrete_gpu; // Level Zero backend and not an integrated GPU + size_t vmm_granularity; // granularity of virtual memory size_t total_vram; - //sycl_hw_info hw_info; \\ device id and aarch, currently not used + sycl_hw_info hw_info; optimize_feature opt_feature; + bool usm_system_support; // support for USM system allocations +#ifdef GGML_SYCL_GRAPH + bool graph_support; // command graphs can be recorded and replayed + bool graph_update_support; // a finalized command graph can be updated +#endif }; struct ggml_sycl_device_info { + int active_device_count; //the number of active devices in the application int device_count; sycl_device_info devices[GGML_SYCL_MAX_DEVICES] = {}; @@ -230,9 +352,17 @@ struct ggml_sycl_device_info { std::array default_tensor_split = {}; int max_work_group_sizes[GGML_SYCL_MAX_DEVICES] = {0}; + + bool ext_oneapi_level_zero = true; // sycl::backend::ext_oneapi_level_zero used by all enumerated GPU devices }; -const ggml_sycl_device_info & ggml_sycl_info(); +ggml_sycl_device_info & ggml_sycl_info(); + +static inline bool ggml_sycl_xmx_enabled(int device) { + return g_ggml_sycl_enable_xmx_esimd && ggml_sycl_xmx_supports_arch(ggml_sycl_info().devices[device].hw_info.arch); +} + +static constexpr size_t SYCL_BUFFER_ALIGNMENT = 128; struct ggml_sycl_pool { virtual ~ggml_sycl_pool() = default; @@ -302,9 +432,51 @@ struct ggml_tensor_extra_gpu { optimize_feature optimized_feature; }; +extern int g_ggml_sycl_use_level_zero_api; +void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, + ggml_sycl_mem_type type = GGML_SYCL_MEM_DIRECT); +void ggml_sycl_free_device(void *ptr, sycl::queue &q); + void release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector streams={}); +struct mmid_row_mapping { + int32_t i1; + int32_t i2; +}; + +struct ggml_sycl_gg_tile { + int32_t expert; + int32_t n0; + int32_t n1; +}; + namespace sycl_ex = sycl::ext::oneapi::experimental; + +#ifdef GGML_SYCL_GRAPH +struct ggml_sycl_graph { + // src data/ne/nb are kept next to the node copy: the scheduler can hand back the same src + // pointer with different contents or shape, see https://github.com/ggml-org/llama.cpp/pull/21736 + struct node_properties { + ggml_tensor node; + void * node_src_data_ptrs[GGML_MAX_SRC]; + int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS]; + size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS]; + }; + + std::unique_ptr> exec_graph = nullptr; + std::vector node_props; + bool warmup_complete = false; + uint64_t uid = 0; + int64_t last_used_time = 0; + + // result of check_graph_compatibility() and the uid it was made for + bool compatible = false; + uint64_t compatible_uid = 0; +}; + +static_assert(std::is_trivial::value, "node_properties must be trivial"); +#endif + struct ggml_backend_sycl_context { int device; std::string name; @@ -373,32 +545,16 @@ struct ggml_backend_sycl_context { dnnl::stream stream_dnnl() { return stream_dnnl(device, 0); } - dnnl::memory get_scratchpad_mem(const dnnl::memory::desc & scratchpad_md, - const dnnl::engine & eng, const queue_ptr q) { - ggml_sycl_pool_alloc * pool; - auto it = scratchpad_map.find(q); - if (it == scratchpad_map.end()) { - scratchpad_map[q] = std::make_unique>(this->pool()); - pool = scratchpad_map[q].get(); - } else { - pool = it->second.get(); - } - - size_t scratchpad_size = scratchpad_md.get_size(); - if (scratchpad_size > pool->actual_size) { - pool->realloc(scratchpad_size); - } - void * mem_ptr = pool->get(); - return dnnl::memory(scratchpad_md, eng, mem_ptr); - } #endif // pool std::unique_ptr pools[GGML_SYCL_MAX_DEVICES]; - std::unordered_map>> scratchpad_map; std::unique_ptr host_pools[GGML_SYCL_MAX_DEVICES]; + std::vector mmid_row_mapping_host; + std::vector mmid_tile_schedule_host; + static std::unique_ptr new_pool_for_device(queue_ptr qptr, int device); static std::unique_ptr new_pool_for_host(queue_ptr qptr, int device); @@ -415,7 +571,36 @@ struct ggml_backend_sycl_context { } #ifdef GGML_SYCL_GRAPH - std::unique_ptr> exec_graph = nullptr; + // Map from first node pointer to graph - allows multiple graphs per context when the + // computation is split across CPU/GPU (e.g. with --n-cpu-moe) + std::unordered_map> sycl_graphs; + + int64_t last_graph_eviction_sweep = 0; + std::optional graph_compatible; + + ggml_sycl_graph * sycl_graph(const void * first_node_ptr) { + const int64_t time_now = ggml_time_us(); + const int64_t eviction_timeout_us = g_ggml_sycl_graph_eviction_timeout * 1'000'000LL; + + // sweep every half the eviction timeout, evicting graphs unused for >= the eviction timeout + if (time_now - last_graph_eviction_sweep >= eviction_timeout_us / 2) { + last_graph_eviction_sweep = time_now; + for (auto it = sycl_graphs.begin(); it != sycl_graphs.end(); ) { + if (time_now - it->second->last_used_time >= eviction_timeout_us) { + it = sycl_graphs.erase(it); + } else { + ++it; + } + } + } + + auto it = sycl_graphs.find(first_node_ptr); + if (it == sycl_graphs.end()) { + it = sycl_graphs.emplace(first_node_ptr, std::make_unique()).first; + } + it->second->last_used_time = time_now; + return it->second.get(); + } #endif ggml_sycl_pool & host_pool(int device) { @@ -611,6 +796,16 @@ constexpr size_t ceil_div(const size_t m, const size_t n) { bool gpu_has_xmx(sycl::device &dev); +#if GGML_SYCL_DNNL +// oneDNN builds JIT kernels only for some GPU architectures. On the rest it falls back to +// reference kernels, which are much slower than the SYCL kernels here. +inline bool ggml_sycl_dnnl_has_optimized_gemm(int device) { + return ggml_sycl_info().devices[device].opt_feature.onednn_optimized_gemm; +} +#endif + +int ggml_sycl_get_env(const char *env_name, int default_val); + template std::string debug_get_array_str(const std::string & prefix, const T array[N]) { if (LIKELY(!g_ggml_sycl_debug)) { return ""; @@ -967,9 +1162,20 @@ static T block_reduce(T val, T * shared_vals, int block_size_template) { } static __dpct_inline__ float ggml_sycl_ue4m3_to_fp32(uint8_t x) { - const uint32_t bits = x * (x != 0x7F && x != 0xFF); - const __nv_fp8_e4m3 xf = *reinterpret_cast(&bits); - return static_cast(xf) / 2; + // UE4M3 is unsigned: 4 exp bits (bias 7), 3 mantissa bits, no sign, no NaN. + // exp == 0xF is a valid exponent (256-448 range), not NaN. + if (x == 0 || x == 0x7F) { + return 0.0f; + } + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + float raw; + if (exp == 0) { + raw = man * (1.0f / 8.0f) * sycl::pow(2.0f, -6.0f); + } else { + raw = (1.0f + man / 8.0f) * sycl::pow(2.0f, (float) exp - 7.0f); + } + return raw * 0.5f; } #endif // GGML_SYCL_COMMON_HPP diff --git a/ggml/src/ggml-sycl/concat.cpp b/ggml/src/ggml-sycl/concat.cpp index d16215b..77a1741 100644 --- a/ggml/src/ggml-sycl/concat.cpp +++ b/ggml/src/ggml-sycl/concat.cpp @@ -10,6 +10,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // +#include "ggml.h" + #include "concat.hpp" static inline size_t elem_size(ggml_type t) { @@ -125,7 +127,15 @@ static void concat_T_sycl_non_cont( int64_t ne2, int64_t ne3, uint64_t nb0, uint64_t nb1, uint64_t nb2, uint64_t nb3, int32_t dim) { sycl::range<3> gridDim(ne3, ne2, ne1); - stream->parallel_for(sycl::nd_range<3>(gridDim, sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + + // Avoid oversubscribing device when there is not enough elements along the innermost dim to + // fill a full SYCL_CONCAT_BLOCK_SIZE. For larger # of elements, the full SYCL_CONCAT_BLOCK_SIZE + // is used. + const int64_t ne0_pad = GGML_PAD(ne0, WARP_SIZE); + const int64_t block_ne0 = ne0_pad < SYCL_CONCAT_BLOCK_SIZE ? ne0_pad : (int64_t) SYCL_CONCAT_BLOCK_SIZE; + sycl::range<3> blockDim(1, 1, block_ne0); + + stream->parallel_for(sycl::nd_range<3>(gridDim * blockDim, blockDim), [=](sycl::nd_item<3> item_ct1) { int64_t i3 = item_ct1.get_group(0); int64_t i2 = item_ct1.get_group(1); int64_t i1 = item_ct1.get_group(2); @@ -174,8 +184,8 @@ void concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { const size_t size0 = ggml_nbytes(src0); const size_t size1 = ggml_nbytes(src1); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0).wait())); - SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d + size0 / type_size, src1_d, size1).wait())); + SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); + SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d + size0 / type_size, src1_d, size1))); } } else { concat_T_sycl_non_cont(stream, (const char *) src0->data, (const char *) src1->data, (char *) dst->data, @@ -186,17 +196,103 @@ void concat_impl_sycl(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { } } +template +static void concat_impl_q_sycl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + queue_ptr stream = ctx.stream(); + + const int32_t dim = ((int32_t *) dst->op_params)[0]; + + GGML_ASSERT(src0->type == qtype); + GGML_ASSERT(src1->type == qtype); + GGML_ASSERT(dst->type == qtype); + GGML_ASSERT(src0->ne[0] % qk == 0); + GGML_ASSERT(src1->ne[0] % qk == 0); + GGML_ASSERT(dst->ne[0] % qk == 0); + + const int ne00_blk = src0->ne[0] / qk; + const int ne0_blk = dst->ne[0] / qk; + + if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) { + const block_t * src0_d = (const block_t *) src0->data; + const block_t * src1_d = (const block_t *) src1->data; + block_t * dst_d = (block_t *) dst->data; + const size_t type_size = sizeof(block_t); + + if (dim != 3) { + for (int i3 = 0; i3 < dst->ne[3]; i3++) { + concat_T_sycl( + src0_d + i3 * (src0->nb[3] / type_size), + src1_d + i3 * (src1->nb[3] / type_size), + dst_d + i3 * (dst->nb[3] / type_size), + ne00_blk, src0->ne[1], src0->ne[2], ne0_blk, + dst->ne[1], dst->ne[2], dim, stream); + } + } else { + const size_t size0 = ggml_nbytes(src0); + const size_t size1 = ggml_nbytes(src1); + + SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(dst_d, src0_d, size0))); + SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy((char *) dst_d + size0, src1_d, size1))); + } + } else { + concat_T_sycl_non_cont( + stream, (const char *) src0->data, (const char *) src1->data, + (char *) dst->data, + ne00_blk, src0->ne[1], src0->ne[2], src0->ne[3], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], + src1->ne[0] / qk, src1->ne[1], src1->ne[2], src1->ne[3], + src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], + ne0_blk, dst->ne[1], dst->ne[2], dst->ne[3], + dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3], dim); + } +} + void ggml_sycl_op_concat(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { switch (dst->type) { case GGML_TYPE_F32: concat_impl_sycl(ctx, dst); break; + case GGML_TYPE_F16: + concat_impl_sycl(ctx, dst); + break; +#ifdef GGML_SYCL_HAS_BF16 + case GGML_TYPE_BF16: + concat_impl_sycl(ctx, dst); + break; +#endif case GGML_TYPE_I32: concat_impl_sycl(ctx, dst); break; + case GGML_TYPE_I16: + concat_impl_sycl(ctx, dst); + break; + case GGML_TYPE_I64: + concat_impl_sycl(ctx, dst); + break; + case GGML_TYPE_I8: + concat_impl_sycl(ctx, dst); + break; + case GGML_TYPE_Q4_0: + concat_impl_q_sycl(ctx, dst); + break; + case GGML_TYPE_Q4_1: + concat_impl_q_sycl(ctx, dst); + break; + case GGML_TYPE_Q5_0: + concat_impl_q_sycl(ctx, dst); + break; + case GGML_TYPE_Q5_1: + concat_impl_q_sycl(ctx, dst); + break; + case GGML_TYPE_Q8_0: + concat_impl_q_sycl(ctx, dst); + break; default: - GGML_ASSERT(false && "ggml_sycl_op_concat: unsupported type"); - break; + fprintf(stderr, "%s: unsupported types: dst: %s\n", __func__, ggml_type_name(dst->type)); + GGML_ASSERT(false); } } diff --git a/ggml/src/ggml-sycl/conv2d-dw.cpp b/ggml/src/ggml-sycl/conv2d-dw.cpp new file mode 100644 index 0000000..8755a4c --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d-dw.cpp @@ -0,0 +1,170 @@ +#include "conv2d-dw.hpp" + +struct conv2d_dw_params { + int in_w, in_h; + int out_w, out_h; + int kernel_w, kernel_h; + int stride_x, stride_y; + int padding_x, padding_y; + int dilation_x, dilation_y; + int channels, batches; +}; + +struct conv2d_dw_kernel_bounds { + int y_min, y_max; + int x_min, x_max; +}; + +static inline conv2d_dw_kernel_bounds dw_calculate_kernel_bounds(int out_x, int out_y, + const conv2d_dw_params & p) { + conv2d_dw_kernel_bounds bounds; + bounds.y_min = sycl::max(0, (p.padding_y - out_y * p.stride_y + p.dilation_y - 1) / p.dilation_y); + bounds.y_max = sycl::min(p.kernel_h, + (p.in_h + p.padding_y - out_y * p.stride_y + p.dilation_y - 1) / p.dilation_y); + bounds.x_min = sycl::max(0, (p.padding_x - out_x * p.stride_x + p.dilation_x - 1) / p.dilation_x); + bounds.x_max = sycl::min(p.kernel_w, + (p.in_w + p.padding_x - out_x * p.stride_x + p.dilation_x - 1) / p.dilation_x); + return bounds; +} + +static inline int dw_calculate_input_coord(int out_coord, int kern_coord, int stride, int dilation, int padding) { + return out_coord * stride + kern_coord * dilation - padding; +} + +// whcn layout: input/output stored as [N, C, H, W] +struct dw_whcn_layout { + static int input_index(int n, int c, int y, int x, const conv2d_dw_params & p) { + return n * (p.channels * p.in_w * p.in_h) + c * p.in_w * p.in_h + y * p.in_w + x; + } + static int kernel_index(int c, int ky, int kx, const conv2d_dw_params & p) { + return c * p.kernel_h * p.kernel_w + ky * p.kernel_w + kx; + } + static int output_index(int n, int c, int y, int x, const conv2d_dw_params & p) { + return n * (p.channels * p.out_w * p.out_h) + c * p.out_w * p.out_h + y * p.out_w + x; + } + static void unpack_indices(int global_idx, const conv2d_dw_params & p, + int & n, int & c, int & out_y, int & out_x) { + out_x = global_idx % p.out_w; + out_y = (global_idx / p.out_w) % p.out_h; + c = (global_idx / (p.out_w * p.out_h)) % p.channels; + n = global_idx / (p.out_w * p.out_h * p.channels); + } +}; + +// cwhn layout: input/output stored as [N, H, W, C] +struct dw_cwhn_layout { + static int input_index(int n, int c, int y, int x, const conv2d_dw_params & p) { + return n * (p.channels * p.in_w * p.in_h) + (y * p.in_w + x) * p.channels + c; + } + static int kernel_index(int c, int ky, int kx, const conv2d_dw_params & p) { + return (ky * p.kernel_w + kx) * p.channels + c; + } + static int output_index(int n, int c, int y, int x, const conv2d_dw_params & p) { + return n * (p.channels * p.out_w * p.out_h) + y * (p.out_w * p.channels) + x * p.channels + c; + } + static void unpack_indices(int global_idx, const conv2d_dw_params & p, + int & n, int & c, int & out_y, int & out_x) { + c = global_idx % p.channels; + out_x = (global_idx / p.channels) % p.out_w; + out_y = (global_idx / (p.channels * p.out_w)) % p.out_h; + n = global_idx / (p.channels * p.out_w * p.out_h); + } +}; + +template +static void conv2d_dw_kernel(const float * input, const KernelT * kernel, float * output, + const conv2d_dw_params p, const sycl::nd_item<3> & item_ct1) { + const int global_idx = item_ct1.get_local_id(2) + + item_ct1.get_group(2) * item_ct1.get_local_range(2); + const int total_elements = p.batches * p.channels * p.out_h * p.out_w; + + if (global_idx >= total_elements) { + return; + } + + int n, c, out_y, out_x; + Layout::unpack_indices(global_idx, p, n, c, out_y, out_x); + + float acc = 0.0f; + const conv2d_dw_kernel_bounds bounds = dw_calculate_kernel_bounds(out_x, out_y, p); + + for (int ky = bounds.y_min; ky < bounds.y_max; ++ky) { + const int in_y = dw_calculate_input_coord(out_y, ky, p.stride_y, p.dilation_y, p.padding_y); + for (int kx = bounds.x_min; kx < bounds.x_max; ++kx) { + const int in_x = dw_calculate_input_coord(out_x, kx, p.stride_x, p.dilation_x, p.padding_x); + acc += input[Layout::input_index(n, c, in_y, in_x, p)] * + static_cast(kernel[Layout::kernel_index(c, ky, kx, p)]); + } + } + + output[Layout::output_index(n, c, out_y, out_x, p)] = acc; +} + +template +static void conv2d_dw_sycl(const float * x_d, const KernelT * w_d, float * y_d, + const conv2d_dw_params p, const queue_ptr & stream) { + const int total = p.batches * p.channels * p.out_h * p.out_w; + const int num_blocks = (total + SYCL_CONV2D_DW_BLOCK_SIZE - 1) / SYCL_CONV2D_DW_BLOCK_SIZE; + const sycl::range<3> block_dims(1, 1, SYCL_CONV2D_DW_BLOCK_SIZE); + const sycl::range<3> block_nums(1, 1, num_blocks); + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + conv2d_dw_kernel(x_d, w_d, y_d, p, item_ct1); + }); +} + +void ggml_sycl_op_conv2d_dw(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * kernel = dst->src[0]; + const ggml_tensor * input = dst->src[1]; + + GGML_ASSERT((kernel->type == GGML_TYPE_F32 || kernel->type == GGML_TYPE_F16) && + input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + + const float * x_d = (const float *) input->data; + float * y_d = (float *) dst->data; + + const int32_t * p = (const int32_t *) dst->op_params; + const int stride_x = p[0]; + const int stride_y = p[1]; + const int padding_x = p[2]; + const int padding_y = p[3]; + const int dilation_x = p[4]; + const int dilation_y = p[5]; + + const int in_w = input->ne[0]; + const int in_h = input->ne[1]; + const int kernel_w = kernel->ne[0]; + const int kernel_h = kernel->ne[1]; + const int out_w = dst->ne[0]; + const int out_h = dst->ne[1]; + const int channels = dst->ne[2]; + const int batches = dst->ne[3]; + + const conv2d_dw_params params = { in_w, in_h, out_w, out_h, kernel_w, kernel_h, + stride_x, stride_y, padding_x, padding_y, + dilation_x, dilation_y, channels, batches }; + + const queue_ptr stream = ctx.stream(); + + if (kernel->type == GGML_TYPE_F16) { + const sycl::half * w_d = (const sycl::half *) kernel->data; + if (ggml_is_contiguous(input)) { + conv2d_dw_sycl(x_d, w_d, y_d, params, stream); + } else if (ggml_is_contiguous_channels(input)) { + conv2d_dw_sycl(x_d, w_d, y_d, params, stream); + } else { + GGML_ABORT("Unsupported memory layout for conv2d_dw"); + } + } else { + const float * w_d = (const float *) kernel->data; + if (ggml_is_contiguous(input)) { + conv2d_dw_sycl(x_d, w_d, y_d, params, stream); + } else if (ggml_is_contiguous_channels(input)) { + conv2d_dw_sycl(x_d, w_d, y_d, params, stream); + } else { + GGML_ABORT("Unsupported memory layout for conv2d_dw"); + } + } +} diff --git a/ggml/src/ggml-sycl/conv2d-dw.hpp b/ggml/src/ggml-sycl/conv2d-dw.hpp new file mode 100644 index 0000000..5328922 --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d-dw.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_CONV2D_DW_HPP +#define GGML_SYCL_CONV2D_DW_HPP + +#include "common.hpp" + +#define SYCL_CONV2D_DW_BLOCK_SIZE 256 + +void ggml_sycl_op_conv2d_dw(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_CONV2D_DW_HPP diff --git a/ggml/src/ggml-sycl/conv2d-transpose.cpp b/ggml/src/ggml-sycl/conv2d-transpose.cpp new file mode 100644 index 0000000..07c325c --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d-transpose.cpp @@ -0,0 +1,125 @@ +#include "conv2d-transpose.hpp" +#include "convert.hpp" + +template +static void conv2d_transpose_kernel(const float * input, const kernel_t * kernel, float * output, + const int in_w, const int in_h, + const int out_w, const int out_h, + const int kernel_w, const int kernel_h, + const int stride, + const int c_in, const int c_out, const int batches, + const sycl::nd_item<3> & item_ct1) { + const int global_idx = item_ct1.get_local_id(2) + + item_ct1.get_group(2) * item_ct1.get_local_range(2); + const int total_elements = out_w * out_h * c_out * batches; + + if (global_idx >= total_elements) { + return; + } + + const int out_x = global_idx % out_w; + const int out_y = (global_idx / out_w) % out_h; + const int c_idx = (global_idx / (out_w * out_h)) % c_out; + const int n_idx = global_idx / (out_w * out_h * c_out); + + float acc = 0.0f; + + for (int c_in_idx = 0; c_in_idx < c_in; ++c_in_idx) { + for (int kh = 0; kh < kernel_h; ++kh) { + int in_y = out_y - kh; + if (in_y < 0 || in_y % stride) { + continue; + } + in_y /= stride; + if (in_y >= in_h) { + continue; + } + + for (int kw = 0; kw < kernel_w; ++kw) { + int in_x = out_x - kw; + if (in_x < 0 || in_x % stride) { + continue; + } + in_x /= stride; + if (in_x >= in_w) { + continue; + } + + const int input_idx = (in_w * in_h * c_in) * n_idx + (in_w * in_h) * c_in_idx + in_w * in_y + in_x; + const int kernel_idx = (kernel_h * kernel_w * c_out) * c_in_idx + (kernel_h * kernel_w) * c_idx + + kernel_w * kh + kw; + + acc += input[input_idx] * ggml_sycl_cast(kernel[kernel_idx]); + } + } + } + + output[(out_w * out_h * c_out) * n_idx + (out_w * out_h) * c_idx + out_w * out_y + out_x] = acc; +} + +template +static void conv2d_transpose_sycl(const float * input_d, const kernel_t * kernel_d, float * output_d, + const int in_w, const int in_h, + const int out_w, const int out_h, + const int kernel_w, const int kernel_h, + const int stride, + const int c_in, const int c_out, const int batches, + const queue_ptr & stream) { + const int total = out_w * out_h * c_out * batches; + const int num_blocks = (total + SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE - 1) / SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE; + const sycl::range<3> block_dims(1, 1, SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE); + const sycl::range<3> block_nums(1, 1, num_blocks); + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + conv2d_transpose_kernel(input_d, kernel_d, output_d, + in_w, in_h, out_w, out_h, kernel_w, kernel_h, + stride, c_in, c_out, batches, item_ct1); + }); +} + +// input: (W, H, C_in, N) +// kernel: (W, H, C_out, C_in) +// output: (W, H, C_out, N) +void ggml_sycl_op_conv2d_transpose(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * kernel = dst->src[0]; + const ggml_tensor * input = dst->src[1]; + + GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); + GGML_ASSERT(input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + + GGML_ASSERT(ggml_is_contiguous(input)); + GGML_ASSERT(ggml_is_contiguous(kernel)); + GGML_ASSERT(ggml_is_contiguous(dst)); + + const float * input_d = (const float *) input->data; + float * output_d = (float *) dst->data; + const void * kernel_d = kernel->data; + + const int input_w = input->ne[0]; + const int input_h = input->ne[1]; + const int channels_in = input->ne[2]; + const int batches = input->ne[3]; + const int output_w = dst->ne[0]; + const int output_h = dst->ne[1]; + const int channels_out = kernel->ne[2]; + const int kernel_w = kernel->ne[0]; + const int kernel_h = kernel->ne[1]; + const int stride = dst->op_params[0]; + + GGML_ASSERT(channels_in == kernel->ne[3]); + GGML_ASSERT(stride > 0); + + const queue_ptr stream = ctx.stream(); + + if (kernel->type == GGML_TYPE_F16) { + conv2d_transpose_sycl(input_d, (const sycl::half *) kernel_d, output_d, + input_w, input_h, output_w, output_h, kernel_w, kernel_h, + stride, channels_in, channels_out, batches, stream); + } else { + conv2d_transpose_sycl(input_d, (const float *) kernel_d, output_d, + input_w, input_h, output_w, output_h, kernel_w, kernel_h, + stride, channels_in, channels_out, batches, stream); + } +} diff --git a/ggml/src/ggml-sycl/conv2d-transpose.hpp b/ggml/src/ggml-sycl/conv2d-transpose.hpp new file mode 100644 index 0000000..ca06731 --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d-transpose.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_CONV2D_TRANSPOSE_HPP +#define GGML_SYCL_CONV2D_TRANSPOSE_HPP + +#include "common.hpp" + +#define SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE 256 + +void ggml_sycl_op_conv2d_transpose(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_CONV2D_TRANSPOSE_HPP diff --git a/ggml/src/ggml-sycl/conv2d.cpp b/ggml/src/ggml-sycl/conv2d.cpp new file mode 100644 index 0000000..d13ba97 --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d.cpp @@ -0,0 +1,171 @@ +#include "conv2d.hpp" +#include "convert.hpp" + +struct conv2d_params { + const int64_t IW, IH; + const int64_t OW, OH; + const int64_t KW, KH; + const int64_t ST_X, ST_Y; + const int64_t PD_X, PD_Y; + const int64_t DL_X, DL_Y; + const int64_t IC, OC; + const int64_t B; + const int64_t TOTAL; +}; + +struct conv2d_kernel_bounds { + int64_t y_min, y_max; + int64_t x_min, x_max; +}; + +static inline int64_t conv2d_max64(int64_t a, int64_t b) { + return (a > b) ? a : b; +} + +static inline int64_t conv2d_min64(int64_t a, int64_t b) { + return (a < b) ? a : b; +} + +static inline conv2d_kernel_bounds calculate_kernel_bounds(int64_t out_x, int64_t out_y, const conv2d_params & P) { + conv2d_kernel_bounds bounds; + bounds.y_min = conv2d_max64(0, (P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y); + bounds.y_max = conv2d_min64(P.KH, (P.IH + P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y); + bounds.x_min = conv2d_max64(0, (P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X); + bounds.x_max = conv2d_min64(P.KW, (P.IW + P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X); + return bounds; +} + +static inline int calculate_input_coord(int64_t out_coord, int64_t kern_coord, int64_t stride, + int64_t dilation, int64_t padding) { + return out_coord * stride + kern_coord * dilation - padding; +} + +// whcn layout helpers (matching ggml tensor memory order) +static inline int64_t whcn_input_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv2d_params & P) { + return n * (P.IC * P.IW * P.IH) + c * P.IW * P.IH + y * P.IW + x; +} + +static inline int64_t whcn_kernel_index(int64_t c_out, int64_t c_in, int64_t ky, int64_t kx, const conv2d_params & P) { + return c_out * (P.IC * P.KH * P.KW) + c_in * (P.KH * P.KW) + ky * P.KW + kx; +} + +static inline int64_t whcn_output_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv2d_params & P) { + return n * (P.OC * P.OW * P.OH) + c * P.OW * P.OH + y * P.OW + x; +} + +// cwhn layout helpers: buffers are [N][H][W][C] input and [KH][KW][IC][OC] kernel +static inline int64_t cwhn_input_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv2d_params & P) { + return ((n * P.IH + y) * P.IW + x) * P.IC + c; +} + +static inline int64_t cwhn_kernel_index(int64_t c_out, int64_t c_in, int64_t ky, int64_t kx, const conv2d_params & P) { + return ((ky * P.KW + kx) * P.IC + c_in) * P.OC + c_out; +} + +template +static void conv2d_kernel(const float * input, const T * kernel, float * output, + const conv2d_params P, const sycl::nd_item<3> & item_ct1) { + const int64_t global_idx = item_ct1.get_local_id(2) + + item_ct1.get_group(2) * item_ct1.get_local_range(2); + + if (global_idx >= P.TOTAL) { + return; + } + + const int64_t out_x = global_idx % P.OW; + const int64_t out_y = (global_idx / P.OW) % P.OH; + const int64_t c_out = (global_idx / (P.OW * P.OH)) % P.OC; + const int64_t n = global_idx / (P.OW * P.OH * P.OC); + + float acc = 0.0f; + + const conv2d_kernel_bounds bounds = calculate_kernel_bounds(out_x, out_y, P); + + for (int64_t c_in = 0; c_in < P.IC; ++c_in) { + for (int64_t ky = bounds.y_min; ky < bounds.y_max; ++ky) { + const int64_t in_y = calculate_input_coord(out_y, ky, P.ST_Y, P.DL_Y, P.PD_Y); + for (int64_t kx = bounds.x_min; kx < bounds.x_max; ++kx) { + const int64_t in_x = calculate_input_coord(out_x, kx, P.ST_X, P.DL_X, P.PD_X); + const float input_val = input[CWHN ? cwhn_input_index(n, c_in, in_y, in_x, P) : whcn_input_index(n, c_in, in_y, in_x, P)]; + const T kernel_val = kernel[CWHN ? cwhn_kernel_index(c_out, c_in, ky, kx, P) : whcn_kernel_index(c_out, c_in, ky, kx, P)]; + acc += input_val * ggml_sycl_cast(kernel_val); + } + } + } + + output[whcn_output_index(n, c_out, out_y, out_x, P)] = acc; +} + +template +static void conv2d_sycl(const float * X_D, const T * K_D, float * Y_D, + const conv2d_params P, const queue_ptr & stream) { + const int num_blocks = (P.TOTAL + SYCL_CONV2D_BLOCK_SIZE - 1) / SYCL_CONV2D_BLOCK_SIZE; + const sycl::range<3> block_dims(1, 1, SYCL_CONV2D_BLOCK_SIZE); + const sycl::range<3> block_nums(1, 1, num_blocks); + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + conv2d_kernel(X_D, K_D, Y_D, P, item_ct1); + }); +} + +void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * kernel = dst->src[0]; + const ggml_tensor * input = dst->src[1]; + const float * K_D = (const float *) kernel->data; + const float * X_D = (const float *) input->data; + float * Y_D = (float *) dst->data; + + // WHCN: contiguous [KW,KH,IC,OC] kernel and [W,H,C,N] input + // CWHN: channel-fastest buffer [N][H][W][C] input and [KH][KW][IC][OC] kernel (OC fastest) + const bool whcn = ggml_is_contiguous(kernel) && ggml_is_contiguous(input); + const bool cwhn = input->nb[2] == ggml_type_size(input->type) && + kernel->nb[3] == ggml_type_size(kernel->type); + GGML_ASSERT(whcn || cwhn); + const bool cwhn_layout = !whcn && cwhn; // prefer whcn when both match (degenerate size-1 dims) + + GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); + GGML_ASSERT(input->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + // same number of input channels + GGML_ASSERT(input->ne[2] == kernel->ne[2]); + + const queue_ptr stream = ctx.stream(); + + const int32_t * p = (const int32_t *) dst->op_params; + const int ST_X = p[0]; + const int ST_Y = p[1]; + const int PD_X = p[2]; + const int PD_Y = p[3]; + const int DL_X = p[4]; + const int DL_Y = p[5]; + + const int IW = input->ne[0]; + const int IH = input->ne[1]; + const int OW = dst->ne[0]; + const int OH = dst->ne[1]; + const int KW = kernel->ne[0]; + const int KH = kernel->ne[1]; + const int IC = input->ne[2]; + const int OC = kernel->ne[3]; + const int B = input->ne[3]; + + const int64_t total = (int64_t) B * OC * OH * OW; + const conv2d_params params = { IW, IH, OW, OH, KW, KH, ST_X, ST_Y, PD_X, PD_Y, DL_X, DL_Y, IC, OC, B, total }; + + if (cwhn_layout) { + if (kernel->type == GGML_TYPE_F16) { + conv2d_sycl(X_D, (const sycl::half *) K_D, Y_D, params, stream); + } else { + conv2d_sycl(X_D, K_D, Y_D, params, stream); + } + } else { + if (kernel->type == GGML_TYPE_F16) { + conv2d_sycl(X_D, (const sycl::half *) K_D, Y_D, params, stream); + } else { + conv2d_sycl(X_D, K_D, Y_D, params, stream); + } + } +} diff --git a/ggml/src/ggml-sycl/conv2d.hpp b/ggml/src/ggml-sycl/conv2d.hpp new file mode 100644 index 0000000..efd25ab --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_CONV2D_HPP +#define GGML_SYCL_CONV2D_HPP + +#include "common.hpp" + +#define SYCL_CONV2D_BLOCK_SIZE 256 + +void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_CONV2D_HPP diff --git a/ggml/src/ggml-sycl/conv3d.cpp b/ggml/src/ggml-sycl/conv3d.cpp new file mode 100644 index 0000000..3796562 --- /dev/null +++ b/ggml/src/ggml-sycl/conv3d.cpp @@ -0,0 +1,224 @@ +#include "conv3d.hpp" + +static inline int64_t ggml_sycl_conv3d_calc_patch_total(const ggml_tensor * dst, int32_t n) { + return (int64_t) n * dst->ne[0] * dst->ne[1] * dst->ne[2]; +} + +static inline int64_t ggml_sycl_conv3d_calc_knl_n_total(const ggml_tensor * src0, int32_t c) { + return (int64_t) src0->ne[0] * src0->ne[1] * src0->ne[2] * c; +} + +static inline void ggml_sycl_conv3d_write_output( + const ggml_tensor * dst, + const float * src, float * dst_data, + int64_t patch_total, int64_t oc, + int64_t dst_w, int64_t dst_h, int64_t dst_d, + dpct::queue_ptr stream) { + const int64_t dst_nb0 = dst->nb[0]; + const int64_t dst_nb1 = dst->nb[1]; + const int64_t dst_nb2 = dst->nb[2]; + const int64_t dst_nb3 = dst->nb[3]; + const int64_t total = patch_total * oc; + const int64_t block_size = 256; + const int64_t num_work_items = ((total + block_size - 1) / block_size) * block_size; + + stream->parallel_for(sycl::range<1>(num_work_items), [=](sycl::id<1> id) { + const int64_t i = id[0]; + if (i >= total) { + return; + } + + const int64_t patch_idx = i / oc; + const int64_t out_ch = i % oc; + const int64_t p_in_batch = patch_idx % (dst_w * dst_h * dst_d); + const int64_t batch_idx = patch_idx / (dst_w * dst_h * dst_d); + const int64_t dst_z = p_in_batch / (dst_w * dst_h); + const int64_t dst_y = (p_in_batch % (dst_w * dst_h)) / dst_w; + const int64_t dst_x = p_in_batch % dst_w; + const int64_t ocn_idx = batch_idx * oc + out_ch; + + const int64_t dst_offset = dst_x * dst_nb0 + dst_y * dst_nb1 + dst_z * dst_nb2 + ocn_idx * dst_nb3; + // `src` is a column-major (m x n) GEMM output where m == patch_total, n == oc. + // GEMM stores element (row, col) at index `row + col*m`, so compute index accordingly. + const int64_t src_index = patch_idx + out_ch * patch_total; + const float value = src[src_index]; + *(float *)((char *)dst_data + dst_offset) = value; + }); +} + +void ggml_sycl_op_conv_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); + + const int32_t * opts = (const int32_t *) dst->op_params; + const int32_t s0 = opts[0]; + const int32_t s1 = opts[1]; + const int32_t s2 = opts[2]; + const int32_t p0 = opts[3]; + const int32_t p1 = opts[4]; + const int32_t p2 = opts[5]; + const int32_t d0 = opts[6]; + const int32_t d1 = opts[7]; + const int32_t d2 = opts[8]; + const int32_t c = opts[9]; + const int32_t n = opts[10]; + const int32_t oc = opts[11]; + + const int64_t knl_w = src0->ne[0]; + const int64_t knl_h = src0->ne[1]; + const int64_t knl_d = src0->ne[2]; + + const int64_t patch_total = ggml_sycl_conv3d_calc_patch_total(dst, n); + const int64_t knl_n_total = ggml_sycl_conv3d_calc_knl_n_total(src0, c); + + const size_t kernel_type_size = ggml_element_size(src0); + + ggml_sycl_pool_alloc gemm_output(ctx.pool()); + gemm_output.alloc((size_t) patch_total * oc); + + ggml_tensor dst_mat = {}; + dst_mat.type = GGML_TYPE_F32; + dst_mat.ne[0] = patch_total; + dst_mat.ne[1] = oc; + dst_mat.ne[2] = 1; + dst_mat.ne[3] = 1; + dst_mat.nb[0] = sizeof(float); + dst_mat.nb[1] = dst_mat.nb[0] * dst_mat.ne[0]; + dst_mat.nb[2] = dst_mat.nb[1]; + dst_mat.nb[3] = dst_mat.nb[2]; + dst_mat.data = gemm_output.get(); + dst_mat.buffer = dst->buffer; + dst_mat.extra = dst->extra; + + dpct::queue_ptr stream = ctx.stream(); + + // allocate packed arrays: A_packed (k x m), B_packed (k x n) + ggml_sycl_pool_alloc A_packed_alloc(ctx.pool()); + ggml_sycl_pool_alloc B_packed_alloc(ctx.pool()); + A_packed_alloc.alloc((size_t) knl_n_total * patch_total); + B_packed_alloc.alloc((size_t) knl_n_total * oc); + + float * A_packed = A_packed_alloc.get(); + float * B_packed = B_packed_alloc.get(); + + const int m = (int) patch_total; + const int n_gemm = (int) oc; + const int k = (int) knl_n_total; + + // Combined kernel: im2col -> pack A, and pack B simultaneously + const char * src1_base = (const char *) src1->data; + const char * src0_base = (const char *) src0->data; + const int64_t src1_nb0 = src1->nb[0]; + const int64_t src1_nb1 = src1->nb[1]; + const int64_t src1_nb2 = src1->nb[2]; + const int64_t src1_nb3 = src1->nb[3]; + const int64_t src1_w = src1->ne[0]; + const int64_t src1_h = src1->ne[1]; + const int64_t src1_d = src1->ne[2]; + + const bool src0_is_f32 = (src0->type == GGML_TYPE_F32); + + // Compute correct strides for src0 as (knl_n_total, oc) matrix + const int64_t src0_packed_nb0 = kernel_type_size; + const int64_t src0_packed_nb1 = kernel_type_size * knl_n_total; + + const int64_t KW = knl_w; + const int64_t KH = knl_h; + const int64_t KD = knl_d; + const int64_t PW = dst->ne[0]; + const int64_t PH = dst->ne[1]; + const int64_t PD = dst->ne[2]; + + // Pack A (with inline im2col): for each (row, col) in k x m matrix + const int64_t A_total = (int64_t)k * m; + const int64_t A_block_size = 256; + const int64_t A_num_work = ((A_total + A_block_size - 1) / A_block_size) * A_block_size; + + stream->parallel_for(sycl::range<1>(A_num_work), [=](sycl::id<1> id) { + const int64_t t = id[0]; + if (t >= A_total) return; + + const int64_t row = t % k; + const int64_t col = t / k; + + // Inline im2col for this element + const int64_t k_index = row; + const int64_t patch_idx = col; + + const int64_t ic = k_index / (KD * KH * KW); + const int64_t rem = k_index - ic * (KD * KH * KW); + const int64_t kz = rem / (KH * KW); + const int64_t rem2 = rem - kz * (KH * KW); + const int64_t ky = rem2 / KW; + const int64_t kx = rem2 % KW; + + const int64_t p_in_batch = patch_idx % (PW * PH * PD); + const int64_t batch_idx = patch_idx / (PW * PH * PD); + const int64_t dst_z = p_in_batch / (PW * PH); + const int64_t dst_y = (p_in_batch % (PW * PH)) / PW; + const int64_t dst_x = p_in_batch % PW; + + const int64_t sx = dst_x * s0 + kx * d0 - p0; + const int64_t sy = dst_y * s1 + ky * d1 - p1; + const int64_t sz = dst_z * s2 + kz * d2 - p2; + + float val = 0.0f; + if (sx >= 0 && sx < src1_w && sy >= 0 && sy < src1_h && sz >= 0 && sz < src1_d) { + const int64_t channel_idx = batch_idx * c + ic; + const char * ptr = src1_base + sx * src1_nb0 + sy * src1_nb1 + sz * src1_nb2 + channel_idx * src1_nb3; + val = *(const float *) ptr; + } + A_packed[row + col * (int64_t)k] = val; + }); + + // Pack B: for each (row, col) in k x n_gemm matrix + const int64_t B_total = (int64_t)k * n_gemm; + const int64_t B_block_size = 256; + const int64_t B_num_work = ((B_total + B_block_size - 1) / B_block_size) * B_block_size; + + stream->parallel_for(sycl::range<1>(B_num_work), [=](sycl::id<1> id) { + const int64_t t = id[0]; + if (t >= B_total) return; + + const int64_t row = t % k; + const int64_t col = t / k; + const char * src_ptr = src0_base + row * src0_packed_nb0 + col * src0_packed_nb1; + float v; + if (src0_is_f32) { + v = *(const float *) src_ptr; + } else { + v = sycl::vec(*(const sycl::half *) src_ptr).convert()[0]; + } + B_packed[row + col * (int64_t)k] = v; + }); + + // GEMM: C = A^T * B where A is (k x m), B is (k x n), C is (m x n) + const float alpha = 1.0f; + const float beta = 0.0f; + const int lda = k; + const int ldb = k; + const int ldc = m; + + SYCL_CHECK(CHECK_TRY_ERROR(oneapi::mkl::blas::column_major::gemm( + *stream, oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, + m, n_gemm, k, + dpct::get_value(&alpha, *stream), + (const float *) A_packed, lda, + (const float *) B_packed, ldb, + dpct::get_value(&beta, *stream), + (float *) dst_mat.data, ldc))); + + const float * gemm_data = (const float *) dst_mat.data; + float * dst_data = (float *) dst->data; + + ggml_sycl_conv3d_write_output(dst, gemm_data, dst_data, patch_total, oc, + dst->ne[0], dst->ne[1], dst->ne[2], stream); +} diff --git a/ggml/src/ggml-sycl/conv3d.hpp b/ggml/src/ggml-sycl/conv3d.hpp new file mode 100644 index 0000000..5852f39 --- /dev/null +++ b/ggml/src/ggml-sycl/conv3d.hpp @@ -0,0 +1,8 @@ +#ifndef GGML_SYCL_CONV3D_HPP +#define GGML_SYCL_CONV3D_HPP + +#include "common.hpp" + +void ggml_sycl_op_conv_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_CONV3D_HPP diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index f3c521b..02b287e 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -2,13 +2,6 @@ #include "dequantize.hpp" #include "presets.hpp" -#if defined(__INTEL_LLVM_COMPILER) - #if __has_include() - #include - #define GGML_SYCL_HAS_BF16 - #endif -#endif - template static void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, const sycl::nd_item<3> &item_ct1) { @@ -83,6 +76,19 @@ static void dequantize_row_q2_K_sycl(const void *vx, dst_t *y, const int64_t k, #endif } +template +static void dequantize_row_q2_K_sycl_reorder(const void *vx, dst_t *y, const int64_t k, + dpct::queue_ptr stream) { + const int64_t nb = k / QK_K; + + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q2_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb); + }); +} + template static void dequantize_row_q3_K_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -114,6 +120,19 @@ static void dequantize_row_q3_K_sycl(const void *vx, dst_t *y, const int64_t k, #endif } +template +static void dequantize_row_q3_K_sycl_reorder(const void *vx, dst_t *y, const int64_t k, + dpct::queue_ptr stream) { + const int64_t nb = k / QK_K; + + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q3_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb); + }); +} + template static void dequantize_row_q4_0_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -139,16 +158,16 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); - int constexpr WARP_K = WARP_SIZE * QK4_0; - const int n_warp = (k + WARP_K - 1) / WARP_K; - GGML_ASSERT(k % 2 == 0); - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * - sycl::range<3>(1, 1, WARP_SIZE), - sycl::range<3>(1, 1, WARP_SIZE)), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + GGML_ASSERT(k % QK4_0 == 0); + const int64_t n_groups = ceil_div(k / 2, SYCL_DEQUANTIZE_BLOCK_SIZE); + // decrease global range when it exceeds the max int; the kernel loops to cover the rest + const int64_t wg_size = downsample_sycl_global_range(n_groups, SYCL_DEQUANTIZE_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_groups) * + sycl::range<3>(1, 1, wg_size), + sycl::range<3>(1, 1, wg_size)), + [=](sycl::nd_item<3> item_ct1) { dequantize_block_q4_0_reorder(vx, y, k, item_ct1); }); - } template @@ -158,16 +177,13 @@ static void dequantize_row_q8_0_sycl_reorder(const void *vx, dst_t *y, const int dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); - int constexpr WARP_K = WARP_SIZE * QK8_0; - const int n_warp = (k + WARP_K - 1) / WARP_K; + // flat range<1>, two pairs (4 elements) per item: no nd_range/group math, one vectorized store. + // k % QK8_0 == 0 implies k % 4 == 0, so the wide launch covers the row exactly (no tail). GGML_ASSERT(k % QK8_0 == 0); - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * - sycl::range<3>(1, 1, WARP_SIZE), - sycl::range<3>(1, 1, WARP_SIZE)), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ - dequantize_block_q8_0_reorder(vx, y, k, item_ct1); + stream->parallel_for(sycl::range<1>((size_t) (k / 4)), + [=](sycl::id<1> idx) { + dequantize_block_q8_0_reorder_wide(vx, y, k, idx); }); - } template @@ -193,20 +209,10 @@ template static void dequantize_row_q4_K_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { const int64_t nb = k / QK_K; - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor scale_local_acc(sycl::range<1>(12), cgh); - cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 32), - sycl::range<3>(1, 1, 32)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q4_K(vx, y, get_pointer(scale_local_acc), item_ct1); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) nb * SYCL_DEQK_WI_PER_BLOCK), + sycl::range<1>((size_t) SYCL_DEQK_WI_PER_BLOCK)), + [=](sycl::nd_item<1> it) { dequantize_block_q4_K_wide(vx, y, nb, it); }); } template @@ -222,7 +228,7 @@ static void dequantize_row_q4_K_sycl_reorder(const void * vx, dst_t * y, const i cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(global_size), sycl::range<1>(local_size)), [=](sycl::nd_item<1> item_ct1) { - dequantize_block_q4_K_reorder(vx, y, get_pointer(scale_local_acc), item_ct1, nb); + dequantize_block_q4_K_reorder(vx, y + item_ct1.get_group(0) * QK_K, item_ct1.get_group(0), get_pointer(scale_local_acc), item_ct1, nb); }); }); } @@ -231,32 +237,27 @@ template static void dequantize_row_q5_K_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { const int64_t nb = k / QK_K; -#if QK_K == 256 - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) nb * SYCL_DEQK_WI_PER_BLOCK), + sycl::range<1>((size_t) SYCL_DEQK_WI_PER_BLOCK)), + [=](sycl::nd_item<1> it) { dequantize_block_q5_K_wide(vx, y, nb, it); }); +} - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 64), - sycl::range<3>(1, 1, 64)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q5_K(vx, y, item_ct1); - }); - } -#else - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); +template +static void dequantize_row_q5_K_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) { + const int64_t nb = k / QK_K; - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 32), - sycl::range<3>(1, 1, 32)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q5_K(vx, y, item_ct1); - }); - } + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); -#endif + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor scale_local_acc(sycl::range<1>(K_SCALE_SIZE), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_q5_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), get_pointer(scale_local_acc), item_ct1, nb); + }); + }); } template @@ -299,7 +300,7 @@ static void dequantize_row_q6_K_sycl_reorder(const void * vx, dst_t * y, const i stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)), - [=](sycl::nd_item<3> item_ct1) { dequantize_block_q6_K_reorder(vx, y, item_ct1, nb); }); + [=](sycl::nd_item<3> item_ct1) { dequantize_block_q6_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb); }); } template @@ -427,6 +428,20 @@ static void dequantize_row_iq3_xxs_sycl(const void *vx, dst_t *y, const int64_t } } +template +static void dequantize_row_iq3_xxs_sycl_reorder(const void * vx, dst_t * y, const int64_t k, + dpct::queue_ptr stream) { + const int64_t nb = k / QK_K; + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 32), sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_iq3_xxs_reorder( + vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), item_ct1, nb, + iq3xxs_grid, ksigns_iq2xs, kmask_iq2xs); + }); +} + template static void dequantize_row_iq3_s_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -447,6 +462,19 @@ static void dequantize_row_iq3_s_sycl(const void *vx, dst_t *y, const int64_t k, } } +template +static void dequantize_row_iq3_s_sycl_reorder(const void * vx, dst_t * y, const int64_t k, + dpct::queue_ptr stream) { + const int64_t nb = k / QK_K; + dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 }); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 32), sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + dequantize_block_iq3_s_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), + item_ct1, nb, kmask_iq2xs, iq3s_grid); + }); +} + template static void dequantize_row_iq4_xs_sycl(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -501,6 +529,18 @@ static void dequantize_row_mxfp4_sycl(const void * vx, dst_t * y, const int64_t }); } +template +static void dequantize_row_mxfp4_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) { + GGML_ASSERT(k % QK_MXFP4 == 0); + const int n_warp = (k / QK_MXFP4 + WARP_SIZE - 1) / WARP_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * sycl::range<3>(1, 1, WARP_SIZE), + sycl::range<3>(1, 1, WARP_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_block_mxfp4_reorder(vx, y, k, item_ct1); + }); +} + template static void dequantize_row_nvfp4_sycl(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) { GGML_ASSERT(k % QK_NVFP4 == 0); @@ -617,42 +657,68 @@ static void convert_unary_sycl(const void * vx, dst_t * y, const int64_t k, dpct } -to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { +static bool is_reordered_src0(const ggml_tensor * dst) { + return dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder; +} + +template +static to_t_sycl_t ggml_get_dequant_sycl(ggml_type type, ggml_tensor * dst) { switch (type) { + case GGML_TYPE_Q1_0: + return dequantize_block_sycl; + case GGML_TYPE_Q2_0: + return dequantize_block_sycl; case GGML_TYPE_Q4_0: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q4_0_sycl_reorder; + } else if constexpr (std::is_same_v) { + return dequantize_row_q4_0_sycl; } else { - return dequantize_block_sycl; + return dequantize_block_sycl; } case GGML_TYPE_Q4_1: - return dequantize_block_sycl; + if constexpr (std::is_same_v) { + return dequantize_row_q4_1_sycl; + } else { + return dequantize_block_sycl; + } case GGML_TYPE_Q5_0: - return dequantize_block_sycl; + return dequantize_block_sycl; case GGML_TYPE_Q5_1: - return dequantize_block_sycl; + return dequantize_block_sycl; case GGML_TYPE_Q8_0: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q8_0_sycl_reorder; } else { - return dequantize_block_sycl; + return dequantize_block_sycl; } case GGML_TYPE_Q2_K: - return dequantize_row_q2_K_sycl; + if (is_reordered_src0(dst)) { + return dequantize_row_q2_K_sycl_reorder; + } else { + return dequantize_row_q2_K_sycl; + } case GGML_TYPE_Q3_K: - return dequantize_row_q3_K_sycl; + if (is_reordered_src0(dst)) { + return dequantize_row_q3_K_sycl_reorder; + } else { + return dequantize_row_q3_K_sycl; + } case GGML_TYPE_Q4_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q4_K_sycl_reorder; } else { return dequantize_row_q4_K_sycl; } case GGML_TYPE_Q5_K: - return dequantize_row_q5_K_sycl; + if (is_reordered_src0(dst)) { + return dequantize_row_q5_K_sycl_reorder; + } else { + return dequantize_row_q5_K_sycl; + } case GGML_TYPE_Q6_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (is_reordered_src0(dst)) { return dequantize_row_q6_K_sycl_reorder; } else { return dequantize_row_q6_K_sycl; @@ -668,104 +734,71 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { case GGML_TYPE_IQ2_S: return dequantize_row_iq2_s_sycl; case GGML_TYPE_IQ3_XXS: - return dequantize_row_iq3_xxs_sycl; + if (is_reordered_src0(dst)) { + return dequantize_row_iq3_xxs_sycl_reorder; + } else { + return dequantize_row_iq3_xxs_sycl; + } case GGML_TYPE_IQ3_S: - return dequantize_row_iq3_s_sycl; + if (is_reordered_src0(dst)) { + return dequantize_row_iq3_s_sycl_reorder; + } else { + return dequantize_row_iq3_s_sycl; + } case GGML_TYPE_IQ4_XS: return dequantize_row_iq4_xs_sycl; case GGML_TYPE_IQ4_NL: return dequantize_row_iq4_nl_sycl; case GGML_TYPE_MXFP4: + if (is_reordered_src0(dst)) { + return dequantize_row_mxfp4_sycl_reorder; + } return dequantize_row_mxfp4_sycl; case GGML_TYPE_NVFP4: return dequantize_row_nvfp4_sycl; case GGML_TYPE_F32: - return convert_unary_sycl; + if constexpr (std::is_same_v) { + return convert_unary_sycl; + } + break; + case GGML_TYPE_F16: + if constexpr (std::is_same_v) { + return convert_unary_sycl; + } + break; #ifdef GGML_SYCL_HAS_BF16 case GGML_TYPE_BF16: return convert_unary_sycl; #endif default: - GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type)); - return nullptr; + break; } + GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type)); +} + +to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { + return ggml_get_dequant_sycl(type, dst); } to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { + return ggml_get_dequant_sycl(type, dst); +} + + +#ifdef GGML_SYCL_HAS_BF16 +to_bf16_sycl_t ggml_get_to_bf16_sycl(ggml_type type, ggml_tensor * /*dst*/) { switch (type) { - case GGML_TYPE_Q4_0: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q4_0_sycl_reorder; - } else { - return dequantize_row_q4_0_sycl; - } - case GGML_TYPE_Q4_1: - return dequantize_row_q4_1_sycl; - case GGML_TYPE_Q5_0: - return dequantize_block_sycl; - case GGML_TYPE_Q5_1: - return dequantize_block_sycl; - case GGML_TYPE_Q8_0: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q8_0_sycl_reorder; - } else { - return dequantize_block_sycl; - } - case GGML_TYPE_Q2_K: - return dequantize_row_q2_K_sycl; - case GGML_TYPE_Q3_K: - return dequantize_row_q3_K_sycl; - case GGML_TYPE_Q4_K: - if (dst->src[0]->extra && - ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q4_K_sycl_reorder; - } else { - return dequantize_row_q4_K_sycl; - } - case GGML_TYPE_Q5_K: - return dequantize_row_q5_K_sycl; - case GGML_TYPE_Q6_K: - if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - return dequantize_row_q6_K_sycl_reorder; - } else { - return dequantize_row_q6_K_sycl; - } - case GGML_TYPE_IQ1_S: - return dequantize_row_iq1_s_sycl; - case GGML_TYPE_IQ1_M: - return dequantize_row_iq1_m_sycl; - case GGML_TYPE_IQ2_XXS: - return dequantize_row_iq2_xxs_sycl; - case GGML_TYPE_IQ2_XS: - return dequantize_row_iq2_xs_sycl; - case GGML_TYPE_IQ2_S: - return dequantize_row_iq2_s_sycl; - case GGML_TYPE_IQ3_XXS: - return dequantize_row_iq3_xxs_sycl; - case GGML_TYPE_IQ3_S: - return dequantize_row_iq3_s_sycl; - case GGML_TYPE_IQ4_XS: - return dequantize_row_iq4_xs_sycl; - case GGML_TYPE_IQ4_NL: - return dequantize_row_iq4_nl_sycl; - case GGML_TYPE_MXFP4: - return dequantize_row_mxfp4_sycl; - case GGML_TYPE_NVFP4: - return dequantize_row_nvfp4_sycl; + case GGML_TYPE_F32: + return convert_unary_sycl; case GGML_TYPE_F16: return convert_unary_sycl; -#ifdef GGML_SYCL_HAS_BF16 case GGML_TYPE_BF16: return convert_unary_sycl; -#endif default: GGML_ABORT("fatal error: unsupport data type=%s\n", ggml_type_name(type)); - return nullptr; } } - +#endif to_fp16_nc_sycl_t ggml_get_to_fp16_nc_sycl(ggml_type type) { switch (type) { @@ -775,6 +808,8 @@ to_fp16_nc_sycl_t ggml_get_to_fp16_nc_sycl(ggml_type type) { case GGML_TYPE_BF16: return convert_unary_nc_sycl; #endif + case GGML_TYPE_Q1_0: + return dequantize_block_nc_sycl; case GGML_TYPE_Q4_0: return dequantize_block_nc_sycl; case GGML_TYPE_Q4_1: diff --git a/ggml/src/ggml-sycl/convert.hpp b/ggml/src/ggml-sycl/convert.hpp index 6e621f2..8de79d1 100644 --- a/ggml/src/ggml-sycl/convert.hpp +++ b/ggml/src/ggml-sycl/convert.hpp @@ -23,6 +23,11 @@ typedef to_t_sycl_t to_fp16_sycl_t; to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst); to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor * dst); +#ifdef GGML_SYCL_HAS_BF16 +typedef to_t_sycl_t to_bf16_sycl_t; +to_bf16_sycl_t ggml_get_to_bf16_sycl(ggml_type type, ggml_tensor * dst); +#endif + // Nc = Non-contiguous template using to_t_nc_sycl_t = void (*)(const void * x, T * y, int64_t ne00, int64_t ne01, int64_t ne02, int64_t ne03, @@ -35,15 +40,19 @@ template inline dst_t ggml_sycl_cast(src_t x) { if constexpr (std::is_same_v) { return x; +#ifdef GGML_SYCL_HAS_BF16 } else if constexpr (std::is_same_v) { return sycl::ext::oneapi::bfloat16(float(x)); } else if constexpr (std::is_same_v) { return static_cast(x); +#endif } else if constexpr (std::is_same_v && std::is_same_v) { return x.template convert(); +#ifdef GGML_SYCL_HAS_BF16 } else if constexpr (std::is_same_v && std::is_same_v>) { return {x.x, x.y}; +#endif } else if constexpr(std::is_same_v) { return int32_t(x); } else { diff --git a/ggml/src/ggml-sycl/count-equal.cpp b/ggml/src/ggml-sycl/count-equal.cpp index 4580354..2012665 100644 --- a/ggml/src/ggml-sycl/count-equal.cpp +++ b/ggml/src/ggml-sycl/count-equal.cpp @@ -74,6 +74,5 @@ void ggml_sycl_count_equal(ggml_backend_sycl_context &ctx, ggml_tensor *dst) { } break; default: GGML_ASSERT(false); - break; } } diff --git a/ggml/src/ggml-sycl/cpy.cpp b/ggml/src/ggml-sycl/cpy.cpp index 9670955..8e1f1d5 100644 --- a/ggml/src/ggml-sycl/cpy.cpp +++ b/ggml/src/ggml-sycl/cpy.cpp @@ -1,13 +1,13 @@ #include "cpy.hpp" #include +#include #include "dequantize.hpp" #include "ggml-sycl/common.hpp" #include "ggml-sycl/presets.hpp" #include "ggml.h" - static void cpy_1_f32_f32(const char * cxi, char * cdsti) { const float * xi = (const float *) cxi; float * dsti = (float *) cdsti; @@ -50,6 +50,57 @@ static void cpy_1_i32_i32(const char * cxi, char * cdsti) { *dsti = *xi; } +static void cpy_1_f32_i32(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + int32_t * dsti = (int32_t *) cdsti; + + *dsti = (int32_t) *xi; +} + +static void cpy_1_i32_f32(const char * cxi, char * cdsti) { + const int32_t * xi = (const int32_t *) cxi; + float * dsti = (float *) cdsti; + + *dsti = (float) *xi; +} + +#ifdef GGML_SYCL_HAS_BF16 +static void cpy_1_f32_bf16(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + sycl::ext::oneapi::bfloat16 * dsti = (sycl::ext::oneapi::bfloat16 *) cdsti; + + *dsti = sycl::ext::oneapi::bfloat16(*xi); +} + +static void cpy_1_bf16_f32(const char * cxi, char * cdsti) { + const sycl::ext::oneapi::bfloat16 * xi = (const sycl::ext::oneapi::bfloat16 *) cxi; + float * dsti = (float *) cdsti; + + *dsti = static_cast(*xi); +} + +static void cpy_1_bf16_bf16(const char * cxi, char * cdsti) { + const sycl::ext::oneapi::bfloat16 * xi = (const sycl::ext::oneapi::bfloat16 *) cxi; + sycl::ext::oneapi::bfloat16 * dsti = (sycl::ext::oneapi::bfloat16 *) cdsti; + + *dsti = *xi; +} + +static void cpy_1_f16_bf16(const char * cxi, char * cdsti) { + const sycl::half * xi = (const sycl::half *) cxi; + sycl::ext::oneapi::bfloat16 * dsti = (sycl::ext::oneapi::bfloat16 *) cdsti; + + *dsti = sycl::ext::oneapi::bfloat16(static_cast(*xi)); +} + +static void cpy_1_bf16_f16(const char * cxi, char * cdsti) { + const sycl::ext::oneapi::bfloat16 * xi = (const sycl::ext::oneapi::bfloat16 *) cxi; + sycl::half * dsti = (sycl::half *) cdsti; + + *dsti = sycl::half(static_cast(*xi)); +} +#endif + template static void cpy_f32_f16(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, @@ -99,6 +150,20 @@ static void cpy_blck_q8_0_f32(const char * cxi, char * cdsti) { } } +static void cpy_blck_q2_0_f32(const char * cxi, char * cdsti) { + const block_q2_0 * xi = (const block_q2_0 *) cxi; + float * cdstf = (float *) cdsti; + + const float d = xi->d; + + for (int j = 0; j < QK2_0; ++j) { + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + const int q = (xi->qs[byte_index] >> bit_offset) & 0x3; + cdstf[j] = (float) (q - 1) * d; + } +} + template static void cpy_blck_q_f32(const char * cxi, char * cdsti) { @@ -204,7 +269,7 @@ static void ggml_cpy_f16_f32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -222,7 +287,7 @@ static void ggml_cpy_f32_f32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -240,21 +305,54 @@ static void ggml_cpy_f32_f16_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } } +static void ggml_cpy_f32_i32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + { + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); + } +} + +static void ggml_cpy_i32_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + { + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); + } +} + static void ggml_cpy_f32_q8_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { GGML_ASSERT(ne % QK8_0 == 0); - const int num_blocks = ne / QK8_0; - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + const int num_blocks = ceil_div(ne / QK8_0, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -264,22 +362,40 @@ static void ggml_cpy_q8_0_f32_sycl(const char * cx, char * cdst, const int ne, c const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ne; - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + GGML_ASSERT(ne % QK8_0 == 0); + const int num_blocks = ceil_div(ne / QK8_0, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } +static void ggml_cpy_q2_0_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK2_0 == 0); + const int num_blocks = ceil_div(ne / QK2_0, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cpy_q_f32(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, + ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + static void ggml_cpy_f32_q4_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { GGML_ASSERT(ne % QK4_0 == 0); - const int num_blocks = ne / QK4_0; - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + const int num_blocks = ceil_div(ne / QK4_0, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -289,9 +405,12 @@ static void ggml_cpy_q4_0_f32_sycl(const char * cx, char * cdst, const int ne, c const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ne; + GGML_ASSERT(ne % QK4_0 == 0); + const int num_blocks = ceil_div(ne / QK4_0, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK4_0>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -303,9 +422,10 @@ static void ggml_cpy_f32_q4_1_sycl(const char * cx, char * cdst, const int ne, c const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { GGML_ASSERT(ne % QK4_1 == 0); - const int num_blocks = ne / QK4_1; - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + const int num_blocks = ceil_div(ne / QK4_1, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -315,9 +435,12 @@ static void ggml_cpy_q4_1_f32_sycl(const char * cx, char * cdst, const int ne, c const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ne; + GGML_ASSERT(ne % QK4_1 == 0); + const int num_blocks = ceil_div(ne / QK4_1, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK4_1>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -329,9 +452,10 @@ static void ggml_cpy_f32_q5_0_sycl(const char * cx, char * cdst, const int ne, c const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { GGML_ASSERT(ne % QK5_0 == 0); - const int num_blocks = ne / QK5_0; - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + const int num_blocks = ceil_div(ne / QK5_0, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -341,9 +465,12 @@ static void ggml_cpy_q5_0_f32_sycl(const char * cx, char * cdst, const int ne, c const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ne; + GGML_ASSERT(ne % QK5_0 == 0); + const int num_blocks = ceil_div(ne / QK5_0, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK5_0>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -355,9 +482,10 @@ static void ggml_cpy_f32_q5_1_sycl(const char * cx, char * cdst, const int ne, c const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { GGML_ASSERT(ne % QK5_1 == 0); - const int num_blocks = ne / QK5_1; - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + const int num_blocks = ceil_div(ne / QK5_1, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -367,28 +495,323 @@ static void ggml_cpy_q5_1_f32_sycl(const char * cx, char * cdst, const int ne, c const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ne; + GGML_ASSERT(ne % QK5_1 == 0); + const int num_blocks = ceil_div(ne / QK5_1, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK5_1>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } +static void ggml_cpy_mxfp4_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_MXFP4 == 0); + const int num_blocks = ceil_div(ne / QK_MXFP4, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cpy_q_f32, QK_MXFP4>(cx, cdst, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + static void ggml_cpy_f32_iq4_nl_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { GGML_ASSERT(ne % QK4_NL == 0); - const int num_blocks = ne / QK4_NL; + const int num_blocks = ceil_div(ne / QK4_NL, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } +static void cpy_blck_f16_q4_0(const char * cxi, char * cdsti) { + const sycl::half * xi = (const sycl::half *) cxi; + float xf[QK4_0]; + + for (int j = 0; j < QK4_0; ++j) { + xf[j] = (float) xi[j]; + } + + cpy_blck_f32_q4_0((const char *) xf, cdsti); +} + +static void cpy_blck_f16_q4_1(const char * cxi, char * cdsti) { + const sycl::half * xi = (const sycl::half *) cxi; + float xf[QK4_1]; + + for (int j = 0; j < QK4_1; ++j) { + xf[j] = (float) xi[j]; + } + + cpy_blck_f32_q4_1((const char *) xf, cdsti); +} + +static void cpy_blck_f16_q5_0(const char * cxi, char * cdsti) { + const sycl::half * xi = (const sycl::half *) cxi; + float xf[QK5_0]; + + for (int j = 0; j < QK5_0; ++j) { + xf[j] = (float) xi[j]; + } + + cpy_blck_f32_q5_0((const char *) xf, cdsti); +} + +static void ggml_cpy_f16_q4_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK4_0 == 0); + const int num_blocks = ceil_div(ne / QK4_0, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_f16_q4_1_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK4_1 == 0); + const int num_blocks = ceil_div(ne / QK4_1, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_f16_q5_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK5_0 == 0); + const int num_blocks = ceil_div(ne / QK5_0, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, + ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static bool ggml_sycl_is_quantized_type(enum ggml_type type) { + switch (type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return true; + default: + return false; + } +} + +static bool ggml_sycl_can_quantize_rows_sycl(enum ggml_type type) { + switch (type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return true; + default: + return false; + } +} + +template +static inline float ggml_sycl_src_to_f32(const SrcScalar & x) { + return (float) x; +} + +#ifdef GGML_SYCL_HAS_BF16 +template <> +inline float ggml_sycl_src_to_f32(const sycl::ext::oneapi::bfloat16 & x) { + return static_cast(x); +} + +template <> +inline float ggml_sycl_src_to_f32(const ggml_bf16_t & x) { + union { + uint32_t u32; + float f32; + } value; + + value.u32 = (uint32_t) x.bits << 16; + return value.f32; +} +#endif + +template +static void ggml_sycl_quantize_rows_q(const char * cx, char * cdst, const int64_t ne, + const int64_t ne00, const int64_t ne01, const int64_t ne02, + const size_t nb00, const size_t nb01, const size_t nb02, const size_t nb03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, + const size_t nb10, const size_t nb11, const size_t nb12, const size_t nb13, + queue_ptr stream) { + GGML_ASSERT(ne % qk == 0); + GGML_ASSERT(ne00 % qk == 0); + + const int64_t total_blocks = ne / qk; + constexpr int block_size = 256; + const int64_t grid_size = ceil_div(total_blocks, (int64_t) block_size); + + stream->parallel_for(sycl::nd_range<1>(grid_size * block_size, block_size), + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const int64_t block_idx = item_ct1.get_global_linear_id(); + if (block_idx >= total_blocks) { + return; + } + + const int64_t i = block_idx * qk; + + const int64_t i03 = i / (ne00 * ne01 * ne02); + const int64_t i02 = (i - i03 * ne00 * ne01 * ne02) / (ne00 * ne01); + const int64_t i01 = (i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00) / ne00; + const int64_t i00 = i - i03 * ne00 * ne01 * ne02 - i02 * ne01 * ne00 - i01 * ne00; + const size_t x_offset = i00 * nb00 + i01 * nb01 + i02 * nb02 + i03 * nb03; + + const int64_t i13 = i / (ne10 * ne11 * ne12); + const int64_t i12 = (i - i13 * ne10 * ne11 * ne12) / (ne10 * ne11); + const int64_t i11 = (i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11) / ne10; + const int64_t i10 = i - i13 * ne10 * ne11 * ne12 - i12 * ne10 * ne11 - i11 * ne10; + const size_t dst_offset = (i10 / qk) * nb10 + i11 * nb11 + i12 * nb12 + i13 * nb13; + + float xf[qk]; + if (nb00 == sizeof(SrcScalar)) { + const SrcScalar * src_row = (const SrcScalar *) (cx + x_offset); + for (int j = 0; j < qk; ++j) { + xf[j] = ggml_sycl_src_to_f32(src_row[j]); + } + } else { + for (int j = 0; j < qk; ++j) { + const SrcScalar * src_val = (const SrcScalar *) (cx + x_offset + j * nb00); + xf[j] = ggml_sycl_src_to_f32(*src_val); + } + } + + quantize_block((const char *) xf, cdst + dst_offset); + }); +} + +template +static void ggml_sycl_quantize_rows_sycl(const char * cx, char * cdst, const ggml_tensor * src0, const ggml_tensor * src1, + const int64_t ne, const int64_t ne00, const int64_t ne01, const int64_t ne02, + const size_t nb00, const size_t nb01, const size_t nb02, const size_t nb03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, + const size_t nb11, const size_t nb12, const size_t nb13, queue_ptr stream) { + GGML_UNUSED(src0); + GGML_UNUSED(src1); + + switch (src1->type) { + case GGML_TYPE_Q8_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q1_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q2_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q5_1: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q5_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q4_1: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_Q4_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; + case GGML_TYPE_IQ4_NL: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, stream); + break; + case GGML_TYPE_MXFP4: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, stream); + break; + case GGML_TYPE_NVFP4: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, stream); + break; + default: + GGML_ABORT("unsupported quantized target type in sycl quantizer src1->type=%s\n", + ggml_type_name(src1->type)); + } +} + static void ggml_cpy_f16_f16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, @@ -400,7 +823,7 @@ static void ggml_cpy_f16_f16_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -419,7 +842,7 @@ static void ggml_cpy_i16_i16_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -438,7 +861,7 @@ static void ggml_cpy_i32_i32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -449,10 +872,12 @@ static void ggml_cpy_q8_0_q8_0(const char * cx, char * cdst, const int ne, const const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + GGML_ASSERT(ne % QK8_0 == 0); + const int num_blocks = ceil_div(ne / QK8_0, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -462,10 +887,12 @@ static void ggml_cpy_q5_0_q5_0(const char * cx, char * cdst, const int ne, const const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + GGML_ASSERT(ne % QK5_0 == 0); + const int num_blocks = ceil_div(ne / QK5_0, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -475,11 +902,13 @@ static void ggml_cpy_q5_1_q5_1(const char * cx, char * cdst, const int ne, const const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + GGML_ASSERT(ne % QK5_1 == 0); + const int num_blocks = ceil_div(ne / QK5_1, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -489,9 +918,11 @@ static void ggml_cpy_q4_0_q4_0(const char * cx, char * cdst, const int ne, const const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + GGML_ASSERT(ne % QK4_0 == 0); + const int num_blocks = ceil_div(ne / QK4_0, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -502,18 +933,330 @@ static void ggml_cpy_q4_1_q4_1(const char * cx, char * cdst, const int ne, const const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { - const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); - stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + GGML_ASSERT(ne % QK4_1 == 0); + const int num_blocks = ceil_div(ne / QK4_1, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } +static void ggml_cpy_q1_0_q1_0(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK1_0 == 0); + const int num_blocks = ceil_div(ne / QK1_0, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q2_0_q2_0(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK2_0 == 0); + const int num_blocks = ceil_div(ne / QK2_0, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_mxfp4_mxfp4(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_MXFP4 == 0); + const int num_blocks = ceil_div(ne / QK_MXFP4, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_nvfp4_nvfp4(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_NVFP4 == 0); + const int num_blocks = ceil_div(ne / QK_NVFP4, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q2_K_q2_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q3_K_q3_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q4_K_q4_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q5_K_q5_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_q6_K_q6_K(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq2_xxs_iq2_xxs(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq2_xs_iq2_xs(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq2_s_iq2_s(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq3_xxs_iq3_xxs(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq1_s_iq1_s(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq1_m_iq1_m(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq4_nl_iq4_nl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK4_NL == 0); + const int num_blocks = ceil_div(ne / QK4_NL, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq3_s_iq3_s(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_iq4_xs_iq4_xs(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + GGML_ASSERT(ne % QK_K == 0); + const int num_blocks = ceil_div(ne / QK_K, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + +#ifdef GGML_SYCL_HAS_BF16 +static void ggml_cpy_f32_bf16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_bf16_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_bf16_bf16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_f16_bf16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} + +static void ggml_cpy_bf16_f16_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = (ne + SYCL_CPY_BLOCK_SIZE - 1) / SYCL_CPY_BLOCK_SIZE; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, item_ct1); + }); +} +#endif + void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1) try { // Unlike other operators ggml_sycl_cpy takes 2 distinct tensors instead of a dst ggml_tensor and rely on its src field + GGML_SYCL_DEBUG("ggml_sycl_cpy: src0->type=%s, src1->type=%s\n", + ggml_type_name(src0->type), ggml_type_name(src1->type)); scope_op_debug_print scope_dbg_print(__func__, src1, /*num_src=*/0, debug_get_tensor_str("\tsrc0", src0)); const int64_t ne = ggml_nelements(src0); GGML_ASSERT(ne == ggml_nelements(src1)); + // the cpy kernels index in 32-bit + GGML_ASSERT(ne <= INT32_MAX); GGML_TENSOR_BINARY_OP_LOCALS01; @@ -525,12 +1268,31 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co if ((src0->type == src1->type) && (ggml_is_contiguous(src0) && ggml_is_contiguous(src1))) { GGML_SYCL_DEBUG("%s: memcpy path\n", __func__); main_stream->memcpy(src1_ddc, src0_ddc, ggml_nbytes(src0)); + } else if (src0->type == GGML_TYPE_F32 && ggml_sycl_is_quantized_type(src1->type)) { + GGML_ASSERT(ggml_sycl_can_quantize_rows_sycl(src1->type)); + ggml_sycl_quantize_rows_sycl(src0_ddc, src1_ddc, src0, src1, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && ggml_sycl_is_quantized_type(src1->type)) { + GGML_ASSERT(ggml_sycl_can_quantize_rows_sycl(src1->type)); + ggml_sycl_quantize_rows_sycl(src0_ddc, src1_ddc, src0, src1, ne, ne00, ne01, ne02, nb00, + nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, + main_stream); +#ifdef GGML_SYCL_HAS_BF16 + } else if (src0->type == GGML_TYPE_BF16 && ggml_sycl_is_quantized_type(src1->type)) { + GGML_ASSERT(ggml_sycl_can_quantize_rows_sycl(src1->type)); + ggml_sycl_quantize_rows_sycl(src0_ddc, src1_ddc, src0, src1, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, main_stream); +#endif } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) { ggml_cpy_f32_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) { ggml_cpy_f32_f16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32) { + ggml_cpy_f32_i32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) { ggml_cpy_f32_q8_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -546,12 +1308,24 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) { ggml_cpy_f16_f16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_Q4_0) { + ggml_cpy_f16_q4_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_Q4_1) { + ggml_cpy_f16_q4_1_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_Q5_0) { + ggml_cpy_f16_q5_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, + nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) { ggml_cpy_i16_i16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32) { ggml_cpy_i32_i32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_F32) { + ggml_cpy_i32_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_Q4_0 && src1->type == GGML_TYPE_F32) { ggml_cpy_q4_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -561,6 +1335,9 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co } else if (src0->type == GGML_TYPE_Q8_0 && src1->type == GGML_TYPE_F32) { ggml_cpy_q8_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q2_0 && src1->type == GGML_TYPE_F32) { + ggml_cpy_q2_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) { ggml_cpy_f32_q5_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -573,6 +1350,9 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co } else if (src0->type == GGML_TYPE_Q5_1 && src1->type == GGML_TYPE_F32) { ggml_cpy_q5_1_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_MXFP4 && src1->type == GGML_TYPE_F32) { + ggml_cpy_mxfp4_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) { ggml_cpy_f32_iq4_nl_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -586,6 +1366,59 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co ggml_cpy_q4_0_q4_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_Q4_1 && src1->type == GGML_TYPE_Q4_1) { ggml_cpy_q4_1_q4_1(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q1_0 && src1->type == GGML_TYPE_Q1_0) { + ggml_cpy_q1_0_q1_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q2_0 && src1->type == GGML_TYPE_Q2_0) { + ggml_cpy_q2_0_q2_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_MXFP4 && src1->type == GGML_TYPE_MXFP4) { + ggml_cpy_mxfp4_mxfp4(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_NVFP4 && src1->type == GGML_TYPE_NVFP4) { + ggml_cpy_nvfp4_nvfp4(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q2_K && src1->type == GGML_TYPE_Q2_K) { + ggml_cpy_q2_K_q2_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q3_K && src1->type == GGML_TYPE_Q3_K) { + ggml_cpy_q3_K_q3_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q4_K && src1->type == GGML_TYPE_Q4_K) { + ggml_cpy_q4_K_q4_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q5_K && src1->type == GGML_TYPE_Q5_K) { + ggml_cpy_q5_K_q5_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q6_K && src1->type == GGML_TYPE_Q6_K) { + ggml_cpy_q6_K_q6_K(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ2_XXS && src1->type == GGML_TYPE_IQ2_XXS) { + ggml_cpy_iq2_xxs_iq2_xxs(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ2_XS && src1->type == GGML_TYPE_IQ2_XS) { + ggml_cpy_iq2_xs_iq2_xs(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ2_S && src1->type == GGML_TYPE_IQ2_S) { + ggml_cpy_iq2_s_iq2_s(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ3_XXS && src1->type == GGML_TYPE_IQ3_XXS) { + ggml_cpy_iq3_xxs_iq3_xxs(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ1_S && src1->type == GGML_TYPE_IQ1_S) { + ggml_cpy_iq1_s_iq1_s(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ1_M && src1->type == GGML_TYPE_IQ1_M) { + ggml_cpy_iq1_m_iq1_m(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ4_NL && src1->type == GGML_TYPE_IQ4_NL) { + ggml_cpy_iq4_nl_iq4_nl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ3_S && src1->type == GGML_TYPE_IQ3_S) { + ggml_cpy_iq3_s_iq3_s(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_IQ4_XS && src1->type == GGML_TYPE_IQ4_XS) { + ggml_cpy_iq4_xs_iq4_xs(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); +#ifdef GGML_SYCL_HAS_BF16 + } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_BF16) { + ggml_cpy_f32_bf16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32) { + ggml_cpy_bf16_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16) { + ggml_cpy_bf16_bf16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_BF16) { + ggml_cpy_f16_bf16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F16) { + ggml_cpy_bf16_f16_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, + nb11, nb12, nb13, main_stream); +#endif } else { GGML_LOG_ERROR("%s: unsupported type combination (%s to %s)\n", __func__, ggml_type_name(src0->type), ggml_type_name(src1->type)); diff --git a/ggml/src/ggml-sycl/cpy.hpp b/ggml/src/ggml-sycl/cpy.hpp index 3c331f1..34bae1b 100644 --- a/ggml/src/ggml-sycl/cpy.hpp +++ b/ggml/src/ggml-sycl/cpy.hpp @@ -48,6 +48,320 @@ inline void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) { } } +inline void cpy_blck_f32_q1_0(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + block_q1_0 * dsti = (block_q1_0 *) cdsti; + + float sum_abs = 0.0f; + for (int j = 0; j < QK1_0; ++j) { + sum_abs += sycl::fabs((float) xi[j]); + } + + dsti->d = sum_abs / QK1_0; + + for (int j = 0; j < QK1_0 / 8; ++j) { + dsti->qs[j] = 0; + } + + for (int j = 0; j < QK1_0; ++j) { + if (xi[j] >= 0.0f) { + dsti->qs[j / 8] |= (1u << (j % 8)); + } + } +} + +inline int round_nearest_int(float x) { + return (int)(x >= 0.0f ? x + 0.5f : x - 0.5f); +} + +inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + block_q2_0 * dsti = (block_q2_0 *) cdsti; + + float amax = 0.0f; + + for (int j = 0; j < QK2_0; ++j) { + amax = sycl::fmax(amax, sycl::fabs((float) xi[j])); + } + + const float d = amax; + const float id = d > 0.0f ? 1.0f / d : 0.0f; + + dsti->d = d; + + for (int j = 0; j < QK2_0 / 4; ++j) { + dsti->qs[j] = 0; + } + + for (int j = 0; j < QK2_0; ++j) { + int q = round_nearest_int(xi[j] * id) + 1; + q = dpct::max(0, dpct::min(3, q)); + + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + dsti->qs[byte_index] |= (uint8_t) q << bit_offset; + } +} + +inline int best_index_mxfp4(const float x, const float e) { + int best_index = 0; + float best_err = sycl::fabs((float) (kvalues_mxfp4[0] * e - x)); + for (int i = 1; i < 16; ++i) { + const float err = sycl::fabs((float) (kvalues_mxfp4[i] * e - x)); + if (err < best_err) { + best_index = i; + best_err = err; + } + } + return best_index; +} + +inline int nearest_int_sycl(float x) { + const float val = x + 12582912.0f; + int i; + memcpy(&i, &val, sizeof(int)); + return (i & 0x007fffff) - 0x00400000; +} + +inline int nearest_int_ggml_sycl(float x) { + return (int) sycl::round((float) x); +} + +inline uint8_t clamp_u8(const int x, const int lo, const int hi) { + return (uint8_t) dpct::max(lo, dpct::min(hi, x)); +} + +inline int8_t clamp_i8(const int x, const int lo, const int hi) { + return (int8_t) dpct::max(lo, dpct::min(hi, x)); +} + +constexpr float GROUP_MAX_EPS_SYCL = 1e-15f; + +inline float make_qx_quants_sycl(int n, int nmax, const float * x, int8_t * L, int rmse_type, const float * qw) { + float max = 0.0f; + float amax = 0.0f; + for (int i = 0; i < n; ++i) { + const float ax = sycl::fabs(x[i]); + if (ax > amax) { + amax = ax; + max = x[i]; + } + } + if (amax < GROUP_MAX_EPS_SYCL) { + for (int i = 0; i < n; ++i) { + L[i] = 0; + } + return 0.0f; + } + + float iscale = -nmax / max; + if (rmse_type == 0) { + for (int i = 0; i < n; ++i) { + int l = nearest_int_ggml_sycl(iscale * x[i]); + L[i] = (int8_t) (nmax + dpct::max(-nmax, dpct::min(nmax - 1, l))); + } + return 1.0f / iscale; + } + + bool return_early = false; + if (rmse_type < 0) { + rmse_type = -rmse_type; + return_early = true; + } + + float sumlx = 0.0f; + float suml2 = 0.0f; + for (int i = 0; i < n; ++i) { + int l = nearest_int_ggml_sycl(iscale * x[i]); + l = dpct::max(-nmax, dpct::min(nmax - 1, l)); + L[i] = (int8_t) (l + nmax); + + const float w = qw ? qw[i] : (rmse_type == 1 ? x[i] * x[i] : + rmse_type == 2 ? 1.0f : rmse_type == 3 ? sycl::fabs(x[i]) : sycl::sqrt(sycl::fabs(x[i]))); + + sumlx += w * x[i] * l; + suml2 += w * l * l; + } + + float scale = suml2 ? sumlx / suml2 : 0.0f; + if (return_early) { + return suml2 > 0.0f ? 0.5f * (scale + 1.0f / iscale) : 1.0f / iscale; + } + + float best = scale * sumlx; + for (int is = -9; is <= 9; ++is) { + if (is == 0) { + continue; + } + iscale = -(nmax + 0.1f * is) / max; + sumlx = 0.0f; + suml2 = 0.0f; + for (int i = 0; i < n; ++i) { + int l = nearest_int_ggml_sycl(iscale * x[i]); + l = dpct::max(-nmax, dpct::min(nmax - 1, l)); + const float w = qw ? qw[i] : (rmse_type == 1 ? x[i] * x[i] : + rmse_type == 2 ? 1.0f : rmse_type == 3 ? sycl::fabs(x[i]) : sycl::sqrt(sycl::fabs(x[i]))); + sumlx += w * x[i] * l; + suml2 += w * l * l; + } + + if (suml2 > 0.0f && sumlx * sumlx > best * suml2) { + for (int i = 0; i < n; ++i) { + int l = nearest_int_ggml_sycl(iscale * x[i]); + L[i] = (int8_t) (nmax + dpct::max(-nmax, dpct::min(nmax - 1, l))); + } + scale = sumlx / suml2; + best = scale * sumlx; + } + } + + return scale; +} + +inline float make_q3_quants_sycl(int n, int nmax, const float * x, int8_t * L, bool do_rmse) { + float max = 0.0f; + float amax = 0.0f; + for (int i = 0; i < n; ++i) { + const float ax = sycl::fabs(x[i]); + if (ax > amax) { + amax = ax; + max = x[i]; + } + } + + if (amax < GROUP_MAX_EPS_SYCL) { + for (int i = 0; i < n; ++i) { + L[i] = 0; + } + return 0.0f; + } + + const float iscale = -nmax / max; + if (do_rmse) { + float sumlx = 0.0f; + float suml2 = 0.0f; + for (int i = 0; i < n; ++i) { + int l = nearest_int_ggml_sycl(iscale * x[i]); + l = dpct::max(-nmax, dpct::min(nmax - 1, l)); + L[i] = (int8_t) l; + const float w = x[i] * x[i]; + sumlx += w * x[i] * l; + suml2 += w * l * l; + } + + for (int itry = 0; itry < 5; ++itry) { + int n_changed = 0; + for (int i = 0; i < n; ++i) { + const float w = x[i] * x[i]; + float slx = sumlx - w * x[i] * L[i]; + if (slx > 0.0f) { + float sl2 = suml2 - w * L[i] * L[i]; + int new_l = nearest_int_ggml_sycl(x[i] * sl2 / slx); + new_l = dpct::max(-nmax, dpct::min(nmax - 1, new_l)); + if (new_l != L[i]) { + slx += w * x[i] * new_l; + sl2 += w * new_l * new_l; + if (sl2 > 0.0f && slx * slx * suml2 > sumlx * sumlx * sl2) { + L[i] = (int8_t) new_l; + sumlx = slx; + suml2 = sl2; + ++n_changed; + } + } + } + } + if (!n_changed) { + break; + } + } + + for (int i = 0; i < n; ++i) { + L[i] += nmax; + } + return suml2 > 0.0f ? sumlx / suml2 : 0.0f; + } + + for (int i = 0; i < n; ++i) { + int l = nearest_int_ggml_sycl(iscale * x[i]); + l = dpct::max(-nmax, dpct::min(nmax - 1, l)); + L[i] = (int8_t) (l + nmax); + } + + return 1.0f / iscale; +} + +inline void set_scale_min_k4(int j, uint8_t * q, uint8_t d, uint8_t m) { + if (j < 4) { + q[j] = (q[j] & 0xC0) | (d & 0x3F); + q[j + 4] = (q[j + 4] & 0xC0) | (m & 0x3F); + } else { + q[j + 4] = (d & 0x0F) | ((m & 0x0F) << 4); + q[j - 4] = (q[j - 4] & 0x3F) | ((d >> 4) << 6); + q[j - 0] = (q[j - 0] & 0x3F) | ((m >> 4) << 6); + } +} + +inline void get_scale_min_k4_local(int j, const uint8_t * q, uint8_t & d, uint8_t & m) { + if (j < 4) { + d = q[j] & 63; + m = q[j + 4] & 63; + } else { + d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); + } +} + +inline void cpy_blck_f32_mxfp4(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + block_mxfp4 * dsti = (block_mxfp4 *) cdsti; + + float amax = 0.0f; + for (int j = 0; j < QK_MXFP4; ++j) { + amax = sycl::fmax(amax, sycl::fabs((float) xi[j])); + } + + const uint8_t e = amax > 0.0f ? (uint8_t) (sycl::floor(sycl::log2(amax)) - 2 + 127) : 0; + const float d = GGML_E8M0_TO_FP32_HALF(e); + + dsti->e = e; + + for (int j = 0; j < QK_MXFP4 / 2; ++j) { + const uint8_t x0 = best_index_mxfp4(xi[0 + j], d); + const uint8_t x1 = best_index_mxfp4(xi[QK_MXFP4 / 2 + j], d); + + dsti->qs[j] = x0; + dsti->qs[j] |= x1 << 4; + } +} + +inline void cpy_blck_f32_nvfp4(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + block_nvfp4 * dsti = (block_nvfp4 *) cdsti; + + constexpr int n_sub = QK_NVFP4 / QK_NVFP4_SUB; + + for (int s = 0; s < n_sub; ++s) { + const float * xb = xi + s * QK_NVFP4_SUB; + + float amax = 0.0f; + for (int j = 0; j < QK_NVFP4_SUB; ++j) { + amax = sycl::fmax(amax, sycl::fabs((float) xb[j])); + } + + const uint8_t ue = ggml_fp32_to_ue4m3(amax / 6.0f); + dsti->d[s] = ue; + const float d = ggml_sycl_ue4m3_to_fp32(ue); + + for (int j = 0; j < QK_NVFP4_SUB / 2; ++j) { + const uint8_t x0 = best_index_mxfp4(xb[0 + j], d); + const uint8_t x1 = best_index_mxfp4(xb[QK_NVFP4_SUB / 2 + j], d); + + dsti->qs[s * (QK_NVFP4_SUB / 2) + j] = x0 | (x1 << 4); + } + } +} + + inline void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) { const float * xi = (const float *) cxi; block_q4_0 * dsti = (block_q4_0 *) cdsti; diff --git a/ggml/src/ggml-sycl/cross_entropy_loss.cpp b/ggml/src/ggml-sycl/cross_entropy_loss.cpp new file mode 100644 index 0000000..c9d12a5 --- /dev/null +++ b/ggml/src/ggml-sycl/cross_entropy_loss.cpp @@ -0,0 +1,255 @@ +#include "cross_entropy_loss.hpp" + +#include +#include + +template +static __dpct_inline__ void cross_entropy_loss_f32_kernel( + const float * __restrict__ logits, + const float * __restrict__ labels, + float * __restrict__ row_loss, + const int nclasses, + const int nrows, + float * __restrict__ smem, + const sycl::nd_item<3> & item) { + + const int row = item.get_group(2); + const int tid = item.get_local_id(2); + + logits += (int64_t) row * nclasses; + labels += (int64_t) row * nclasses; + + float max_logit = -INFINITY; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = logits[i]; + max_logit = sycl::fmax(max_logit, v); + if (has_shared) { + smem[i] = v; + } + } + max_logit = warp_reduce_max(max_logit); + + float sum_exp = 0.0f; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = has_shared ? smem[i] : logits[i]; + sum_exp += sycl::exp(v - max_logit); + } + sum_exp = warp_reduce_sum(sum_exp); + const float log_sum = sycl::log(sum_exp); + + float loss = 0.0f; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = has_shared ? smem[i] : logits[i]; + loss += (v - max_logit - log_sum) * labels[i]; + } + loss = -warp_reduce_sum(loss) / (float) nrows; + + if (tid == 0) { + row_loss[row] = loss; + } +} + +template +static __dpct_inline__ void cross_entropy_loss_back_f32_kernel( + const float * __restrict__ grad, + const float * __restrict__ logits, + const float * __restrict__ labels, + float * __restrict__ dst, + const int nclasses, + const int nrows, + float * __restrict__ smem, + const sycl::nd_item<3> & item) { + + const int row = item.get_group(2); + const int tid = item.get_local_id(2); + + logits += (int64_t) row * nclasses; + labels += (int64_t) row * nclasses; + dst += (int64_t) row * nclasses; + + float max_logit = -INFINITY; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = logits[i]; + max_logit = sycl::fmax(max_logit, v); + if (has_shared) { + smem[i] = v; + } + } + max_logit = warp_reduce_max(max_logit); + + float sum_exp = 0.0f; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float v = sycl::exp((has_shared ? smem[i] : logits[i]) - max_logit); + sum_exp += v; + if (has_shared) { + smem[i] = v; + } else { + dst[i] = v; + } + } + sum_exp = warp_reduce_sum(sum_exp); + const float inv_sum = 1.0f / sum_exp; + + const float d_by_nrows = grad[0] / (float) nrows; + for (int i = tid; i < nclasses; i += WARP_SIZE) { + const float sm_num = has_shared ? smem[i] : dst[i]; + dst[i] = (sm_num * inv_sum - labels[i]) * d_by_nrows; + } +} + +static void cross_entropy_reduce_rows( + ggml_backend_sycl_context & ctx, + const float * row_loss, + float * dst, + const int64_t nrows) { + if (nrows == 1) { + SYCL_CHECK(CHECK_TRY_ERROR( + ctx.stream()->memcpy(dst, row_loss, sizeof(float)))); + return; + } + + ggml_sycl_pool_alloc tmp_alloc(ctx.pool(), nrows); + float * tmp = tmp_alloc.get(); + SYCL_CHECK(CHECK_TRY_ERROR( + ctx.stream()->memcpy(tmp, row_loss, nrows * sizeof(float)))); + + int64_t cur = nrows; + while (cur > 1) { + const int64_t out = (cur + WARP_SIZE - 1) / WARP_SIZE; + const sycl::range<3> block(1, 1, WARP_SIZE); + const sycl::range<3> grid(1, 1, out); + ctx.stream()->parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const int row = item.get_group(2); + const int tid = item.get_local_id(2); + const int64_t i = (int64_t) row * WARP_SIZE + tid; + float v = i < cur ? tmp[i] : 0.0f; + v = warp_reduce_sum(v); + if (tid == 0) { + tmp[row] = v; + } + }); + cur = out; + } + + SYCL_CHECK(CHECK_TRY_ERROR( + ctx.stream()->memcpy(dst, tmp, sizeof(float)))); +} + +void ggml_sycl_cross_entropy_loss(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_are_same_shape(src0, src1)); + GGML_ASSERT(ggml_is_scalar(dst)); + + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const int64_t nclasses = src0->ne[0]; + const int64_t nrows = ggml_nrows(src0); + + const float * logits_d = (const float *) src0->data; + const float * labels_d = (const float *) src1->data; + float * dst_d = (float *) dst->data; + + ggml_sycl_pool_alloc row_loss_alloc(ctx.pool(), nrows); + float * row_loss = row_loss_alloc.get(); + + const sycl::range<3> block(1, 1, WARP_SIZE); + const sycl::range<3> grid(1, 1, nrows); + const size_t nbytes_shared = (size_t) nclasses * sizeof(float); + const size_t smpbo = ggml_sycl_info().devices[ctx.device].smpbo; + + if (nbytes_shared <= smpbo) { + ctx.stream()->submit([&](sycl::handler & cgh) { + sycl::local_accessor smem(sycl::range<1>(nclasses), cgh); + cgh.parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cross_entropy_loss_f32_kernel( + logits_d, labels_d, row_loss, + (int) nclasses, (int) nrows, + get_pointer(smem), item); + }); + }); + } else { + ctx.stream()->parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cross_entropy_loss_f32_kernel( + logits_d, labels_d, row_loss, + (int) nclasses, (int) nrows, + nullptr, item); + }); + } + + cross_entropy_reduce_rows(ctx, row_loss, dst_d, nrows); +} + +void ggml_sycl_cross_entropy_loss_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3); + + const ggml_tensor * grad = dst->src[0]; + const ggml_tensor * src0f = dst->src[1]; + const ggml_tensor * src1f = dst->src[2]; + + GGML_ASSERT(grad->type == GGML_TYPE_F32); + GGML_ASSERT(src0f->type == GGML_TYPE_F32); + GGML_ASSERT(src1f->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_ASSERT(ggml_is_scalar(grad)); + GGML_ASSERT(ggml_is_contiguous(grad)); + GGML_ASSERT(ggml_is_contiguous(src0f)); + GGML_ASSERT(ggml_is_contiguous(src1f)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_are_same_shape(src0f, src1f)); + GGML_ASSERT(ggml_are_same_shape(src0f, dst)); + + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const int64_t nclasses = src0f->ne[0]; + const int64_t nrows = ggml_nrows(src0f); + + const float * grad_d = (const float *) grad->data; + const float * logits_d = (const float *) src0f->data; + const float * labels_d = (const float *) src1f->data; + float * dst_d = (float *) dst->data; + + const sycl::range<3> block(1, 1, WARP_SIZE); + const sycl::range<3> grid(1, 1, nrows); + const size_t nbytes_shared = (size_t) nclasses * sizeof(float); + const size_t smpbo = ggml_sycl_info().devices[ctx.device].smpbo; + + if (nbytes_shared <= smpbo) { + ctx.stream()->submit([&](sycl::handler & cgh) { + sycl::local_accessor smem(sycl::range<1>(nclasses), cgh); + cgh.parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cross_entropy_loss_back_f32_kernel( + grad_d, logits_d, labels_d, dst_d, + (int) nclasses, (int) nrows, + get_pointer(smem), item); + }); + }); + } else { + ctx.stream()->parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cross_entropy_loss_back_f32_kernel( + grad_d, logits_d, labels_d, dst_d, + (int) nclasses, (int) nrows, + nullptr, item); + }); + } +} diff --git a/ggml/src/ggml-sycl/cross_entropy_loss.hpp b/ggml/src/ggml-sycl/cross_entropy_loss.hpp new file mode 100644 index 0000000..3f1cb81 --- /dev/null +++ b/ggml/src/ggml-sycl/cross_entropy_loss.hpp @@ -0,0 +1,7 @@ +#pragma once + +#include "common.hpp" + +void ggml_sycl_cross_entropy_loss(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +void ggml_sycl_cross_entropy_loss_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/cumsum.cpp b/ggml/src/ggml-sycl/cumsum.cpp new file mode 100644 index 0000000..c1c5fe4 --- /dev/null +++ b/ggml/src/ggml-sycl/cumsum.cpp @@ -0,0 +1,148 @@ +#include "cumsum.hpp" +#include "common.hpp" + +#include + +#define SYCL_CUMSUM_BLOCK_SIZE 256 + +static __dpct_inline__ float warp_prefix_inclusive_sum_f32(float x, const sycl::nd_item<3> & item) { + return sycl::inclusive_scan_over_group(item.get_sub_group(), x, sycl::plus()); +} + +static void cumsum_f32_kernel( + const float * __restrict__ src, float * __restrict__ dst, + const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03, + const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t d1, const int64_t d2, const int64_t d3, + const sycl::nd_item<3> & item, float * smem) { + + const int tid = item.get_local_id(2); + const int block_size = item.get_local_range(2); + const int lane = tid % WARP_SIZE; + const int warp = tid / WARP_SIZE; + const int warps_per_block = block_size / WARP_SIZE; + + float * s_vals = smem; + float * s_warp_sums = smem + block_size; + float * s_carry = smem + block_size + warps_per_block; + + if (tid == 0) { + s_carry[0] = 0.0f; + } + item.barrier(sycl::access::fence_space::local_space); + + const int64_t i3 = item.get_group(0); + const int64_t i2 = item.get_group(1); + const int64_t i1 = item.get_group(2); + if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) { + return; + } + + const float * src_row = src + i1 * s01 + i2 * s02 + i3 * s03; + float * dst_row = dst + i1 * d1 + i2 * d2 + i3 * d3; + + constexpr int num_unroll = 4; + float temp[num_unroll]; + + for (int64_t i = 0; i < ne00; i += num_unroll * block_size) { + int64_t idx = i + tid * num_unroll; + + temp[0] = (idx < ne00 ? src_row[idx] : 0.0f); +#pragma unroll + for (int j = 1; j < num_unroll; j++) { + temp[j] = temp[j - 1]; + if (idx + j < ne00) { + temp[j] += src_row[idx + j]; + } + } + + float val = (idx < ne00) ? temp[num_unroll - 1] : 0.0f; + + val = warp_prefix_inclusive_sum_f32(val, item); + s_vals[tid] = val; + + if (lane == WARP_SIZE - 1) { + s_warp_sums[warp] = val; + } + item.barrier(sycl::access::fence_space::local_space); + + if (warp == 0) { + float w = (tid < warps_per_block) ? s_warp_sums[tid] : 0.0f; + float inc = warp_prefix_inclusive_sum_f32(w, item); + if (tid < warps_per_block) { + s_warp_sums[tid] = inc - w; + } + if (tid == warps_per_block - 1) { + s_carry[1] = inc; + } + } + item.barrier(sycl::access::fence_space::local_space); + + float carry = s_carry[0]; + float final_offset = s_vals[tid] + s_warp_sums[warp] + carry - temp[num_unroll - 1]; + +#pragma unroll + for (int j = 0; j < num_unroll; j++) { + if (idx + j < ne00) { + dst_row[idx + j] = temp[j] + final_offset; + } + } + + item.barrier(sycl::access::fence_space::local_space); + + if (tid == 0) { + s_carry[0] += s_carry[1]; + } + } +} + +inline void ggml_sycl_op_cumsum(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + dpct::queue_ptr stream = ctx.stream(); + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const float * src_d = static_cast(src0->data); + float * dst_d = static_cast(dst->data); + + const int64_t ne00 = src0->ne[0]; + const int64_t ne01 = src0->ne[1]; + const int64_t ne02 = src0->ne[2]; + const int64_t ne03 = src0->ne[3]; + + const size_t ts = sizeof(float); + const int64_t s01 = src0->nb[1] / ts; + const int64_t s02 = src0->nb[2] / ts; + const int64_t s03 = src0->nb[3] / ts; + const int64_t d1 = dst->nb[1] / ts; + const int64_t d2 = dst->nb[2] / ts; + const int64_t d3 = dst->nb[3] / ts; + + const int num_warps = (ne00 + WARP_SIZE - 1) / WARP_SIZE; + int block_size = num_warps * WARP_SIZE; + block_size = std::min(block_size, SYCL_CUMSUM_BLOCK_SIZE); + const int warps_per_block = block_size / WARP_SIZE; + const int smem_size = block_size + warps_per_block + 2; + + const sycl::range<3> grid(ne03, ne02, ne01); + const sycl::range<3> block(1, 1, block_size); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor smem_acc(sycl::range<1>(smem_size), cgh); + cgh.parallel_for( + sycl::nd_range<3>(grid * block, block), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cumsum_f32_kernel(src_d, dst_d, ne00, ne01, ne02, ne03, + s01, s02, s03, d1, d2, d3, + item, get_pointer(smem_acc)); + }); + }); +} + +void ggml_sycl_cumsum(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); + ggml_sycl_op_cumsum(ctx, dst); +} diff --git a/ggml/src/ggml-sycl/cumsum.hpp b/ggml/src/ggml-sycl/cumsum.hpp new file mode 100644 index 0000000..f1a5644 --- /dev/null +++ b/ggml/src/ggml-sycl/cumsum.hpp @@ -0,0 +1,5 @@ +#pragma once + +#include "common.hpp" + +void ggml_sycl_cumsum(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index 19fa886..ae709ef 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -19,6 +19,33 @@ typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, dfloat2 & v); typedef void (*dequantize_kernel_t_reorder)(const void *d, const int64_t ib, const void *qs, const int iqs, dfloat2 &v); +typedef void (*dequantize_kernel_f32_t)(const void * vx, const int64_t ib, const int iqs, float & v0, float & v1); + +#if QK_K == 256 +static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m); +#endif + +static __dpct_inline__ void dequantize_q2_0(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { + const block_q2_0 * x = (const block_q2_0 *) vx; + + const dfloat d = x[ib].d; + + const int byte_idx = iqs / 4; + const int shift = (iqs % 4) * 2; + const uint8_t vui = x[ib].qs[byte_idx]; + + v.x() = (vui >> shift) & 3; + v.y() = (vui >> (shift + 2)) & 3; + +#ifdef GGML_SYCL_F16 + v.s0() = ((dfloat)v.s0() - 1.0f) * d; + v.s1() = ((dfloat)v.s1() - 1.0f) * d; +#else + v.x() = ((dfloat)v.x() - 1.0f) * d; + v.y() = ((dfloat)v.y() - 1.0f) * d; +#endif // GGML_SYCL_F16 +} static __dpct_inline__ void dequantize_q4_0(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { @@ -66,6 +93,36 @@ static __dpct_inline__ void dequantize_q4_0_reorder(const void *d_ptr, const int #endif // GGML_SYCL_F16 } +static __dpct_inline__ void dequantize_q1_0_reorder(const void *d_ptr, const int64_t ib, const void *qs, + const int iqs, dfloat2 &v) { + // Q1_0 reorder layout: scale values followed by quantized bits + const dfloat d = (const dfloat)*((const sycl::half*)d_ptr+ib); + + const int bit_index_0 = iqs + 0; + const int bit_index_1 = iqs + 1; + + const int bit_0 = (*((const uint8_t *)qs + bit_index_0 / 8) >> (bit_index_0 % 8)) & 1; + const int bit_1 = (*((const uint8_t *)qs + bit_index_1 / 8) >> (bit_index_1 % 8)) & 1; + + v.x() = (2 * bit_0 - 1) * d; + v.y() = (2 * bit_1 - 1) * d; +} + +static __dpct_inline__ void dequantize_q1_0(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { + const block_q1_0 * x = (const block_q1_0 *) vx; + const dfloat d = x[ib].d; + + const int bit_index_0 = iqs + 0; + const int bit_index_1 = iqs + 1; + + const int bit_0 = (x[ib].qs[bit_index_0 / 8] >> (bit_index_0 % 8)) & 1; + const int bit_1 = (x[ib].qs[bit_index_1 / 8] >> (bit_index_1 % 8)) & 1; + + v.x() = (2 * bit_0 - 1) * d; + v.y() = (2 * bit_1 - 1) * d; +} + static __dpct_inline__ void dequantize_q4_1(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { const block_q4_1 * x = (const block_q4_1 *) vx; @@ -90,6 +147,557 @@ static __dpct_inline__ void dequantize_q4_1(const void *vx, const int64_t ib, #endif // GGML_SYCL_F16 } +static __dpct_inline__ void dequantize_q4_K(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_q4_K * x = (const block_q4_K *) vx; + const sycl::half2 dm = x[ib].dm; + const float dall = dm[0]; + const float dmin = dm[1]; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int il = idx / 64; + const int in = idx % 64; + const int is = 2 * il + (in >= 32 ? 1 : 0); + const int off = in & 31; + const int qsi = 32 * il + off; + + uint8_t sc; + uint8_t m; + get_scale_min_k4(is, x[ib].scales, sc, m); + + const uint8_t q = x[ib].qs[qsi]; + const uint8_t qv = (in >= 32) ? (q >> 4) : (q & 0xF); + return sycl::fma((dfloat) qv, (dfloat) (dall * sc), (dfloat) (-dmin * m)); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("Q4_K dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_q4_K_f32(const void *vx, const int64_t ib, + const int iqs, float &v0, float &v1) { +#if QK_K == 256 + const block_q4_K * x = (const block_q4_K *) vx; + const sycl::half2 dm = x[ib].dm; + const float dall = dm[0]; + const float dmin = dm[1]; + + auto dequantize_one = [&](const int idx) -> float { + const int il = idx / 64; + const int in = idx % 64; + const int is = 2 * il + (in >= 32 ? 1 : 0); + const int qsi = 32 * il + (in & 31); + + uint8_t sc; + uint8_t m; + get_scale_min_k4(is, x[ib].scales, sc, m); + + const float d = dall * sc; + const float mn = dmin * m; + const uint8_t q = x[ib].qs[qsi]; + const uint8_t qv = (in >= 32) ? (q >> 4) : (q & 0xF); + + return d * qv - mn; + }; + + v0 = dequantize_one(iqs + 0); + v1 = dequantize_one(iqs + 1); +#else + GGML_ABORT("Q4_K dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_q2_K(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_q2_K * x = (const block_q2_K *) vx; + const float dall = x[ib].dm[0]; + const float dmin = x[ib].dm[1]; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int n = idx / 128; + const int r = idx % 128; + const int g = r / 32; + const int l = r % 32; + const int is = 8 * n + l / 16; + + const uint8_t q = x[ib].qs[32 * n + l]; + const uint8_t sc = x[ib].scales[is + 2 * g]; + const float d = dall * (sc & 0xF); + const float m = dmin * (sc >> 4); + + return (dfloat) d * (dfloat) ((q >> (2 * g)) & 3) - (dfloat) m; + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("Q2_K dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_q2_K_f32(const void *vx, const int64_t ib, + const int iqs, float &v0, float &v1) { +#if QK_K == 256 + const block_q2_K * x = (const block_q2_K *) vx; + const float dall = x[ib].dm[0]; + const float dmin = x[ib].dm[1]; + + auto dequantize_one = [&](const int idx) -> float { + const int n = idx / 128; + const int r = idx % 128; + const int g = r / 32; + const int l = r % 32; + const int is = 8 * n + l / 16; + + const uint8_t q = x[ib].qs[32 * n + l]; + const uint8_t sc = x[ib].scales[is + 2 * g]; + const float d = dall * (sc & 0xF); + const float m = dmin * (sc >> 4); + + return d * ((q >> (2 * g)) & 3) - m; + }; + + v0 = dequantize_one(iqs + 0); + v1 = dequantize_one(iqs + 1); +#else + GGML_ABORT("Q2_K dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_q3_K(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_q3_K * x = (const block_q3_K *) vx; + const float d_all = x[ib].d; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int n = idx / 128; + const int r = idx % 128; + const int j = r / 32; + const int l = r % 32; + + const int is0 = l / 16; + const int is = 8 * n + 2 * j + is0; + const int shift = 2 * j; + const uint8_t m = 1 << (4 * n + j); + + const int8_t us = is < 4 ? (x[ib].scales[is - 0] & 0xF) | (((x[ib].scales[is + 8] >> 0) & 3) << 4) : + is < 8 ? (x[ib].scales[is - 0] & 0xF) | (((x[ib].scales[is + 4] >> 2) & 3) << 4) : + is < 12 ? (x[ib].scales[is - 8] >> 4) | (((x[ib].scales[is + 0] >> 4) & 3) << 4) : + (x[ib].scales[is - 8] >> 4) | (((x[ib].scales[is - 4] >> 6) & 3) << 4); + + const float dl = d_all * (us - 32); + const uint8_t q = x[ib].qs[32 * n + l]; + const uint8_t h = x[ib].hmask[l]; + const int8_t qv = ((q >> shift) & 3) - ((h & m) ? 0 : 4); + + return (dfloat) (dl * qv); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("Q3_K dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_q5_K(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_q5_K * x = (const block_q5_K *) vx; + const float dall = x[ib].dm[0]; + const float dmin = x[ib].dm[1]; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int il = idx / 64; + const int in = idx % 64; + const int is = 2 * il + (in >= 32 ? 1 : 0); + const int ir = (in & 31) / 2; + const int iq = in & 1; + + const uint8_t q = x[ib].qs[32 * il + 2 * ir + iq]; + const uint8_t h = x[ib].qh[2 * ir + iq]; + const uint8_t qv = (in >= 32) ? (q >> 4) : (q & 0xF); + + uint8_t sc; + uint8_t m; + get_scale_min_k4(is, x[ib].scales, sc, m); + + const float d = dall * sc; + const float mn = dmin * m; + const uint8_t hm = 1 << (2 * il + (in >= 32 ? 1 : 0)); + + return sycl::fma((dfloat) (qv + ((h & hm) ? 16 : 0)), (dfloat) d, (dfloat) (-mn)); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("Q5_K dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_q5_K_f32(const void *vx, const int64_t ib, + const int iqs, float &v0, float &v1) { +#if QK_K == 256 + const block_q5_K * x = (const block_q5_K *) vx; + const float dall = x[ib].dm[0]; + const float dmin = x[ib].dm[1]; + + auto dequantize_one = [&](const int idx) -> float { + const int il = idx / 64; + const int in = idx % 64; + const int is = 2 * il + (in >= 32 ? 1 : 0); + const int ir = (in & 31) / 2; + const int iq = in & 1; + + const uint8_t q = x[ib].qs[32 * il + 2 * ir + iq]; + const uint8_t h = x[ib].qh[2 * ir + iq]; + const uint8_t qv = (in >= 32) ? (q >> 4) : (q & 0xF); + + uint8_t sc; + uint8_t m; + get_scale_min_k4(is, x[ib].scales, sc, m); + + const float d = dall * sc; + const float mn = dmin * m; + const uint8_t hm = 1 << (2 * il + (in >= 32 ? 1 : 0)); + + return (qv + ((h & hm) ? 16 : 0)) * d - mn; + }; + + v0 = dequantize_one(iqs + 0); + v1 = dequantize_one(iqs + 1); +#else + GGML_ABORT("Q5_K dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_q6_K(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_q6_K * x = (const block_q6_K *) vx; + const float d = x[ib].d; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int ip = idx / 128; + const int in = idx % 128; + const int il = in & 31; + const int ig = in / 32; + const int is = 8 * ip + il / 16; + + const uint8_t ql0 = x[ib].ql[64 * ip + il]; + const uint8_t ql1 = x[ib].ql[64 * ip + il + 32]; + const uint8_t qh = x[ib].qh[32 * ip + il]; + const int8_t * sc = x[ib].scales + is; + + uint8_t qv; + int8_t scale; + if (ig == 0) { + qv = (ql0 & 0xF) | (((qh >> 0) & 3) << 4); + scale = sc[0]; + } else if (ig == 1) { + qv = (ql1 & 0xF) | (((qh >> 2) & 3) << 4); + scale = sc[2]; + } else if (ig == 2) { + qv = (ql0 >> 4) | (((qh >> 4) & 3) << 4); + scale = sc[4]; + } else { + qv = (ql1 >> 4) | (((qh >> 6) & 3) << 4); + scale = sc[6]; + } + + return (dfloat) (d * scale * ((int8_t) qv - 32)); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("Q6_K dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_mxfp4(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { + const block_mxfp4 * x = (const block_mxfp4 *) vx; + const float d = ggml_sycl_e8m0_to_fp32(x[ib].e); + const uint8_t q = x[ib].qs[iqs]; + + v.x() = d * kvalues_mxfp4[q & 0xF] * 0.5f; + v.y() = d * kvalues_mxfp4[q >> 4] * 0.5f; +} + +static __dpct_inline__ void dequantize_nvfp4(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { + const block_nvfp4 & xb = ((const block_nvfp4 *) vx)[ib]; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int sub = idx / QK_NVFP4_SUB; + const int j = idx % QK_NVFP4_SUB; + const int jh = j % (QK_NVFP4_SUB / 2); + + const float d = ggml_sycl_ue4m3_to_fp32(xb.d[sub]); + const uint8_t q = xb.qs[sub * (QK_NVFP4_SUB / 2) + jh]; + const uint8_t qv = (j < (QK_NVFP4_SUB / 2)) ? (q & 0x0F) : (q >> 4); + + return d * kvalues_mxfp4[qv]; + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +} + +static __dpct_inline__ void dequantize_iq2_xxs(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_iq2_xxs * x = (const block_iq2_xxs *) vx; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int ib8 = idx / 32; + const int r = idx % 32; + const int il = r / 8; + const int j = r % 8; + + const uint16_t * q2 = x[ib].qs + 4 * ib8; + const uint8_t * aux8 = (const uint8_t *) q2; + const uint8_t * grid = (const uint8_t *) (iq2xxs_grid + aux8[il]); + const uint32_t aux32 = q2[2] | (q2[3] << 16); + const float d = (float) x[ib].d * (0.5f + (aux32 >> 28)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127]; + + return d * grid[j] * ((signs & kmask_iq2xs[j]) ? -1.f : 1.f); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("IQ2_XXS dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_iq2_xs(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_iq2_xs * x = (const block_iq2_xs *) vx; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int ib8 = idx / 32; + const int r = idx % 32; + const int il = r / 8; + const int j = r % 8; + + const uint16_t * q2 = x[ib].qs + 4 * ib8; + const uint8_t * grid = (const uint8_t *) (iq2xs_grid + (q2[il] & 511)); + const float d = (float) x[ib].d * (0.5f + ((x[ib].scales[ib8] >> (4 * (il / 2))) & 0xf)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; + + return d * grid[j] * ((signs & kmask_iq2xs[j]) ? -1.f : 1.f); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("IQ2_XS dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_iq2_s(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_iq2_s * x = (const block_iq2_s *) vx; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int ib8 = idx / 32; + const int r = idx % 32; + const int il = r / 8; + const int j = r % 8; + + const uint16_t grid_id = x[ib].qs[4 * ib8 + il] | ((x[ib].qh[ib8] << (8 - 2 * il)) & 0x300); + const uint8_t * grid = (const uint8_t *) (iq2s_grid + grid_id); + const float d = (float) x[ib].d * (0.5f + ((x[ib].scales[ib8] >> (4 * (il / 2))) & 0xf)) * 0.25f; + const uint8_t signs = x[ib].qs[QK_K / 8 + 4 * ib8 + il]; + + return d * grid[j] * ((signs & kmask_iq2xs[j]) ? -1.f : 1.f); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("IQ2_S dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_iq3_xxs(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_iq3_xxs * x = (const block_iq3_xxs *) vx; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int ib8 = idx / 32; + const int r = idx % 32; + const int il = r / 8; + const int j = r % 8; + + const uint8_t * q3 = x[ib].qs + 8 * ib8; + const uint16_t * gas = (const uint16_t *) (x[ib].qs + QK_K / 4) + 2 * ib8; + const uint8_t * grid1 = (const uint8_t *) (iq3xxs_grid + q3[2 * il + 0]); + const uint8_t * grid2 = (const uint8_t *) (iq3xxs_grid + q3[2 * il + 1]); + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float d = (float) x[ib].d * (0.5f + (aux32 >> 28)) * 0.5f; + const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127]; + + if (j < 4) { + return d * grid1[j] * ((signs & kmask_iq2xs[j + 0]) ? -1.f : 1.f); + } + return d * grid2[j - 4] * ((signs & kmask_iq2xs[j + 0]) ? -1.f : 1.f); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("IQ3_XXS dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_iq3_s(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_iq3_s * x = (const block_iq3_s *) vx; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int ib8 = idx / 32; + const int r = idx % 32; + const int il = r / 8; + const int j = r % 8; + + const uint8_t * qs = x[ib].qs + 8 * ib8; + const uint16_t grid1_id = qs[2 * il + 0] | ((x[ib].qh[ib8] << (8 - 2 * il)) & 256); + const uint16_t grid2_id = qs[2 * il + 1] | ((x[ib].qh[ib8] << (7 - 2 * il)) & 256); + const uint8_t * grid1 = (const uint8_t *) (iq3s_grid + grid1_id); + const uint8_t * grid2 = (const uint8_t *) (iq3s_grid + grid2_id); + const float d = (float) x[ib].d * (1 + 2 * ((x[ib].scales[ib8 / 2] >> (4 * (ib8 % 2))) & 0xf)); + const uint8_t signs = x[ib].signs[4 * ib8 + il]; + + if (j < 4) { + return d * grid1[j] * ((signs & kmask_iq2xs[j + 0]) ? -1.f : 1.f); + } + return d * grid2[j - 4] * ((signs & kmask_iq2xs[j + 0]) ? -1.f : 1.f); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("IQ3_S dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_iq1_s(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_iq1_s * x = (const block_iq1_s *) vx; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int ib8 = idx / 32; + const int r = idx % 32; + const int il = r / 8; + const int j = r % 8; + + const float delta = (x[ib].qh[ib8] & 0x8000) ? (-1.f - IQ1S_DELTA) : (-1.f + IQ1S_DELTA); + const float d = (float) x[ib].d * (2 * ((x[ib].qh[ib8] >> 12) & 7) + 1); + const uint16_t grid_id = x[ib].qs[4 * ib8 + il] | (((x[ib].qh[ib8] >> (3 * il)) & 7) << 8); + const uint32_t g = iq1s_grid_gpu[grid_id]; + const int8_t qv = (j < 4) ? ((g >> (8 * j)) & 0x0F) : ((g >> (8 * (j - 4) + 4)) & 0x0F); + + return d * (qv + delta); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("IQ1_S dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_iq1_m(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_iq1_m * x = (const block_iq1_m *) vx; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int ib8 = idx / 32; + const int r = idx % 32; + const int il = r / 8; + const int j = r % 8; + + const uint16_t * sc = (const uint16_t *) x[ib].scales; + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); + + const int ib16 = 2 * ib8 + il / 2; + const float d = (float) scale.f16 * (2 * ((sc[ib16 / 4] >> (3 * (ib16 % 4))) & 0x7) + 1); + + const uint8_t qh = x[ib].qh[2 * ib8 + il / 2]; + const float delta = (qh & (0x08 << (4 * (il % 2)))) ? (-1.f - IQ1M_DELTA) : (-1.f + IQ1M_DELTA); + + const uint16_t grid_id = x[ib].qs[4 * ib8 + il] | (((qh >> (4 * (il % 2))) & 7) << 8); + const uint32_t g = iq1s_grid_gpu[grid_id]; + const int8_t qv = (j < 4) ? ((g >> (8 * j)) & 0x0F) : ((g >> (8 * (j - 4) + 4)) & 0x0F); + + return d * (qv + delta); + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("IQ1_M dequantize not supported for QK_K != 256"); +#endif +} + +static __dpct_inline__ void dequantize_iq4_nl(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { + const block_iq4_nl * x = (const block_iq4_nl *) vx; + const float d = (float) x[ib].d; + + auto dequantize_one = [&](const int idx) -> dfloat { + if (idx < 16) { + return d * kvalues_iq4nl[x[ib].qs[idx] & 0xF]; + } + return d * kvalues_iq4nl[x[ib].qs[idx - 16] >> 4]; + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +} + +static __dpct_inline__ void dequantize_iq4_xs(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { +#if QK_K == 256 + const block_iq4_xs * x = (const block_iq4_xs *) vx; + + auto dequantize_one = [&](const int idx) -> dfloat { + const int ib8 = idx / 32; + const int r = idx % 32; + const int byte_idx = (r < 16) ? r : (r - 16); + const uint8_t q = x[ib].qs[16 * ib8 + byte_idx]; + const uint8_t qv = (r < 16) ? (q & 0x0F) : (q >> 4); + + const float d = (float) x[ib].d * ((((x[ib].scales_l[ib8 / 2] >> (4 * (ib8 % 2))) & 0xf) | + (((x[ib].scales_h >> (2 * ib8)) & 3) << 4)) - 32); + return d * kvalues_iq4nl[qv]; + }; + + v.x() = dequantize_one(iqs + 0); + v.y() = dequantize_one(iqs + 1); +#else + GGML_ABORT("IQ4_XS dequantize not supported for QK_K != 256"); +#endif +} + static __dpct_inline__ void dequantize_q5_0(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { const block_q5_0 * x = (const block_q5_0 *) vx; @@ -210,61 +818,63 @@ static void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restri } template -static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32, +static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, const sycl::nd_item<3> &item_ct1) { - - const int64_t i = item_ct1.get_group(2); - auto k=nb32; - // assume 32 threads - const int64_t tid = item_ct1.get_local_id(2); - const int lane_ib = i * WARP_SIZE + tid; - - if (lane_ib >= k / QK4_0) { - return; - } - - dst_t * y_ptr = yy + lane_ib * QK4_0; - - auto qs = (const uint8_t*)vx + lane_ib * QK4_0 / 2; - auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k / 2) + lane_ib; - - const float d = float(*s_ptr); - -#pragma unroll - for (int l = 0; l < QK4_0 / 2; ++l) { - int vq = qs[l]; - y_ptr[l + 0] = d * ((vq & 0xF) - 8); - y_ptr[l + 16] = d * ((vq >> 4) - 8); + // one thread per qs byte, like dequantize_block; the reorder layout makes the byte reads contiguous + // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX + // float math, not dequantize_q4_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output + const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k/2); + const int64_t stride = item_ct1.get_local_range(2) * item_ct1.get_group_range(2); + + for (int64_t iq = item_ct1.get_global_id(2); iq < k/2; iq += stride) { + const int64_t ib = iq / (QK4_0/2); + const int64_t iqs = iq % (QK4_0/2); + const int vq = ((const uint8_t *) vx)[iq]; + const float d = float(d_ptr[ib]); + + dst_t * y = yy + ib*QK4_0 + iqs; + y[0] = d * ((vq & 0xF) - 8); + y[QK4_0/2] = d * ((vq >> 4) - 8); } - } // Dequantize Q8_0 from reorder layout: [all qs (k bytes)][all d values] -// Each thread handles one block of QK8_0 elements. template static void dequantize_block_q8_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, const sycl::nd_item<3> &item_ct1) { - - const int64_t i = item_ct1.get_group(2); - const int64_t tid = item_ct1.get_local_id(2); - const int lane_ib = i * WARP_SIZE + tid; - - if (lane_ib >= k / QK8_0) { - return; + // two values per thread, like dequantize_block; the reorder layout keeps all qs contiguous + // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX + // float math, not dequantize_q8_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output + const int8_t * qs = (const int8_t *) vx; + const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k); + const int64_t stride = 2 * item_ct1.get_local_range(2) * item_ct1.get_group_range(2); + + for (int64_t i = 2 * item_ct1.get_global_id(2); i < k; i += stride) { + const float d = float(d_ptr[i / QK8_0]); + + yy[i + 0] = d * qs[i + 0]; + yy[i + 1] = d * qs[i + 1]; } +} - dst_t * y_ptr = yy + lane_ib * QK8_0; +template +static void dequantize_block_q8_0_reorder_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, + const sycl::id<1> idx) { + const int64_t i = 4 * (int64_t) idx[0]; - auto qs = (const int8_t*)vx + lane_ib * QK8_0; - auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k) + lane_ib; + const int8_t * qs = (const int8_t *) vx; + const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k); - const float d = float(*s_ptr); + const float d0 = float(d_ptr[i / QK8_0]); + const float d1 = float(d_ptr[(i + 2) / QK8_0]); -#pragma unroll - for (int l = 0; l < QK8_0; ++l) { - y_ptr[l] = d * qs[l]; - } + sycl::vec p; + p[0] = d0 * (float) qs[i + 0]; + p[1] = d0 * (float) qs[i + 1]; + p[2] = d1 * (float) qs[i + 2]; + p[3] = d1 * (float) qs[i + 3]; + *reinterpret_cast *>(yy + i) = p.convert(); } template @@ -335,6 +945,47 @@ static void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restri } +template +static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, + const sycl::nd_item<3> & item_ct1, int64_t n_blocks) { +#if QK_K == 256 + if (i >= n_blocks) { + return; + } + + const uint8_t * base = static_cast(vx); + const size_t qs_offset = i * (QK_K / 4); + const size_t scales_offset = n_blocks * (QK_K / 4) + i * (QK_K / 16); + const size_t dm_offset = n_blocks * (QK_K / 4) + n_blocks * (QK_K / 16) + i * sizeof(ggml_half2); + + const uint8_t * qs = base + qs_offset; + const uint8_t * scales = base + scales_offset; + const ggml_half2 * dm = reinterpret_cast(base + dm_offset); + + const int64_t tid = item_ct1.get_local_id(2); + const int64_t n = tid / 32; + const int64_t l = tid - 32 * n; + const int64_t is = 8 * n + l / 16; + + const uint8_t q = qs[32 * n + l]; + dst_t * y = yy + 128 * n; + + const float dall = (*dm)[0]; + const float dmin = (*dm)[1]; + y[l+ 0] = dall * (scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (scales[is+0] >> 4); + y[l+32] = dall * (scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (scales[is+2] >> 4); + y[l+64] = dall * (scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (scales[is+4] >> 4); + y[l+96] = dall * (scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (scales[is+6] >> 4); +#else + GGML_UNUSED(vx); + GGML_UNUSED(yy); + GGML_UNUSED(i); + GGML_UNUSED(item_ct1); + GGML_UNUSED(n_blocks); + GGML_ABORT("Q2_K reorder dequantize not supported for QK_K != 256"); +#endif +} + template static void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy, const sycl::nd_item<3> &item_ct1) { @@ -390,6 +1041,63 @@ static void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restri } +template +static void dequantize_block_q3_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, + const sycl::nd_item<3> & item_ct1, int64_t n_blocks) { +#if QK_K == 256 + if (i >= n_blocks) { + return; + } + + const uint8_t * base = static_cast(vx); + const size_t qs_offset = i * (QK_K / 4); + const size_t hmask_offset = n_blocks * (QK_K / 4) + i * (QK_K / 8); + const size_t scales_offset = n_blocks * (QK_K / 4) + n_blocks * (QK_K / 8) + i * 12; + const size_t d_offset = n_blocks * (QK_K / 4) + n_blocks * (QK_K / 8) + n_blocks * 12 + + i * sizeof(ggml_half); + + const uint8_t * qs = base + qs_offset; + const uint8_t * hmask = base + hmask_offset; + const uint8_t * scales = base + scales_offset; + const float d_all = static_cast(*reinterpret_cast(base + d_offset)); + + const int64_t r = item_ct1.get_local_id(2) / 4; + const int64_t tid = r / 2; + const int64_t is0 = r % 2; + const int64_t l0 = 16 * is0 + 4 * (item_ct1.get_local_id(2) % 4); + const int64_t n = tid / 4; + const int64_t j = tid - 4 * n; + const int64_t is = 8 * n + 2 * j + is0; + const int shift = 2 * j; + uint8_t m = 1 << (4 * n + j); + + uint8_t us = is < 4 + ? (scales[is - 0] & 0xF) | (((scales[is + 8] >> 0) & 3) << 4) + : is < 8 + ? (scales[is - 0] & 0xF) | (((scales[is + 4] >> 2) & 3) << 4) + : is < 12 + ? (scales[is - 8] >> 4) | (((scales[is + 0] >> 4) & 3) << 4) + : (scales[is - 8] >> 4) | (((scales[is - 4] >> 6) & 3) << 4); + + const float dl = d_all * (us - 32); + + dst_t * y = yy + 128 * n + 32 * j; + const uint8_t * q = qs + 32 * n; + const uint8_t * hm = hmask; + + for (int l = l0; l < l0 + 4; ++l) { + y[l] = dl * ((int8_t) ((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)); + } +#else + GGML_UNUSED(vx); + GGML_UNUSED(yy); + GGML_UNUSED(i); + GGML_UNUSED(item_ct1); + GGML_UNUSED(n_blocks); + GGML_ABORT("Q3_K reorder dequantize not supported for QK_K != 256"); +#endif +} + #if QK_K == 256 static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) { if (j < 4) { @@ -402,6 +1110,53 @@ static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8 } #endif +// Work-items that cover one 256-weight K-quant block: four 64-element chunks, 8 work-items +// per chunk, 4 qs bytes per work-item. +static constexpr int SYCL_DEQK_WI_PER_BLOCK = 32; + +template +static inline void dequantize_q4_K_wide_one(dst_t * __restrict__ y, const uint8_t * __restrict__ qs, + const uint8_t * __restrict__ scales, const float dall, + const float dmin, const int tid) { + const int il = tid >> 3; // 64-element chunk, 0..3 + const int ir = (tid & 7) * 4; // 4 qs bytes within the chunk + + uint8_t sc, m; + get_scale_min_k4(2 * il + 0, scales, sc, m); + const float d1 = dall * sc; + const float m1 = dmin * m; + get_scale_min_k4(2 * il + 1, scales, sc, m); + const float d2 = dall * sc; + const float m2 = dmin * m; + + const sycl::vec q = vec_aligned_load(qs + 32 * il + ir); + + sycl::vec lo, hi; +#pragma unroll + for (int l = 0; l < 4; ++l) { + lo[l] = d1 * (q[l] & 0xF) - m1; + hi[l] = d2 * (q[l] >> 4) - m2; + } + dst_t * yb = y + 64 * il + ir; + *reinterpret_cast *>(yb + 0) = lo; + *reinterpret_cast *>(yb + 32) = hi; +} + +template +static void dequantize_block_q4_K_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb, + const sycl::nd_item<1> & it) { + static_assert(QK_K == 256, "wide K-quant dequant assumes QK_K == 256"); + const int64_t g = it.get_global_id(0); + const int64_t b = g / SYCL_DEQK_WI_PER_BLOCK; + const int tid = (int) (g % SYCL_DEQK_WI_PER_BLOCK); + if (b >= nb) { + return; + } + const block_q4_K * x = (const block_q4_K *) vx + b; + dequantize_q4_K_wide_one(yy + b * QK_K, x->qs, x->scales, (float) x->dm[0], (float) x->dm[1], tid); +} + + template inline void dequantize_q4_K_common(dst_t * __restrict__ y, const uint8_t * __restrict__ qs_ptr, const float dall, const float dmin, uint8_t * __restrict__ scales_local, int il, int ir) { @@ -424,50 +1179,14 @@ inline void dequantize_q4_K_common(dst_t * __restrict__ y, const uint8_t * __res } } -template -static void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy, - uint8_t* scales_local, const sycl::nd_item<3> &item_ct1) { - const block_q4_K * x = (const block_q4_K *) vx; - - const int64_t i = item_ct1.get_group(2); - -#if QK_K == 256 - const int64_t tid = item_ct1.get_local_id(2); - const int64_t il = tid / 8; - const int64_t ir = tid % 8; - - dst_t * y = yy + i * QK_K + 64 * il + 4 * ir; - - const sycl::half2 dm = x[i].dm; - const float dall = dm[0]; - const float dmin = dm[1]; - - if (tid < 12) { - scales_local[tid] = x[i].scales[tid]; - } - - item_ct1.barrier(sycl::access::fence_space::local_space); - dequantize_q4_K_common(y, x[i].qs, dall, dmin, scales_local, il, ir); -#else - const int64_t tid = item_ct1.get_local_id(2); - const uint8_t * q = x[i].qs; - dst_t * y = yy + i*QK_K; - const float d = (float)x[i].dm[0]; - const float m = (float)x[i].dm[1]; - y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4); - y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >> 4) - m * (x[i].scales[1] >> 4); -#endif -} - template -static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, uint8_t * scales_local, - const sycl::nd_item<1> & item_ct1, int64_t nb) { - const int64_t i = item_ct1.get_group(0); // block index +static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, + uint8_t * scales_local, const sycl::nd_item<1> & item_ct1, int64_t nb) { const int64_t tid = item_ct1.get_local_id(0); // thread index within block const int64_t il = tid / 8; const int64_t ir = tid % 8; - dst_t * y = yy + i * QK_K + 64 * il + 4 * ir; + dst_t * y = yy + 64 * il + 4 * ir; const uint8_t * base = static_cast(vx); const size_t qs_offset = i * (QK_K / 2); @@ -489,51 +1208,106 @@ static void dequantize_block_q4_K_reorder(const void * __restrict__ vx, dst_t * dequantize_q4_K_common(y, qs_ptr, dall, dmin, scales_local, il, ir); } -template -static void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy, - const sycl::nd_item<3> &item_ct1) { - const block_q5_K * x = (const block_q5_K *) vx; +template +static inline void dequantize_q5_K_wide_one(dst_t * __restrict__ y, const uint8_t * __restrict__ qs, + const uint8_t * __restrict__ qh_ptr, + const uint8_t * __restrict__ scales, const float dall, + const float dmin, const int tid) { + const int il = tid >> 3; // 64-element chunk, 0..3 + const int ir = (tid & 7) * 4; // 4 qs bytes within the chunk; qh holds the 5th bit - const int64_t i = item_ct1.get_group(2); + uint8_t sc, m; + get_scale_min_k4(2 * il + 0, scales, sc, m); + const float d1 = dall * sc; + const float m1 = dmin * m; + get_scale_min_k4(2 * il + 1, scales, sc, m); + const float d2 = dall * sc; + const float m2 = dmin * m; + + const sycl::vec q = vec_aligned_load(qs + 32 * il + ir); + const sycl::vec h = vec_aligned_load(qh_ptr + ir); + + const uint8_t hm1 = 1 << (2 * il); + const uint8_t hm2 = hm1 << 1; + + sycl::vec lo, hi; +#pragma unroll + for (int l = 0; l < 4; ++l) { + lo[l] = d1 * ((q[l] & 0xF) + ((h[l] & hm1) ? 16 : 0)) - m1; + hi[l] = d2 * ((q[l] >> 4) + ((h[l] & hm2) ? 16 : 0)) - m2; + } + dst_t * yb = y + 64 * il + ir; + *reinterpret_cast *>(yb + 0) = lo; + *reinterpret_cast *>(yb + 32) = hi; +} + +template +static void dequantize_block_q5_K_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb, + const sycl::nd_item<1> & it) { + static_assert(QK_K == 256, "wide K-quant dequant assumes QK_K == 256"); + const int64_t g = it.get_global_id(0); + const int64_t b = g / SYCL_DEQK_WI_PER_BLOCK; + const int tid = (int) (g % SYCL_DEQK_WI_PER_BLOCK); + if (b >= nb) { + return; + } + const block_q5_K * x = (const block_q5_K *) vx + b; + dequantize_q5_K_wide_one(yy + b * QK_K, x->qs, x->qh, x->scales, (float) x->dm[0], (float) x->dm[1], tid); +} + +template +static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t ib, + uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t n_blocks) { #if QK_K == 256 - // assume 64 threads - this is very slightly better than the one below + // assume 64 threads const int64_t tid = item_ct1.get_local_id(2); - const int64_t il = tid/16; // il is in 0...3 - const int64_t ir = tid%16; // ir is in 0...15 - const int64_t is = 2*il; // is is in 0...6 + const int64_t il = tid / 16; // 0...3 + const int64_t ir = tid % 16; // 0...15 + const int64_t is = 2 * il; + + dst_t * y = yy + 64 * il + 2 * ir; + + const uint8_t * base = static_cast(vx); - dst_t * y = yy + i*QK_K + 64*il + 2*ir; + // Reordered layout: [qs (QK_K/2 per block)] [qh (QK_K/8 per block)] [scales (K_SCALE_SIZE per block)] [dm (half2 per block)] + const size_t qs_offset = ib * (QK_K / 2); + const size_t qh_offset = n_blocks * (QK_K / 2) + ib * (QK_K / 8); + const size_t scales_offset = n_blocks * (QK_K / 2) + n_blocks * (QK_K / 8) + ib * K_SCALE_SIZE; + const size_t dm_offset = n_blocks * (QK_K / 2) + n_blocks * (QK_K / 8) + n_blocks * K_SCALE_SIZE + ib * sizeof(ggml_half2); - const float dall = x[i].dm[0]; - const float dmin = x[i].dm[1]; + const uint8_t * qs_ptr = base + qs_offset; + const uint8_t * qh_ptr = base + qh_offset; + const uint8_t * scales_ptr = base + scales_offset; + const ggml_half2 dm_values = *reinterpret_cast(base + dm_offset); - const uint8_t * ql = x[i].qs + 32*il + 2*ir; - const uint8_t * qh = x[i].qh + 2*ir; + const float dall = dm_values.x(); + const float dmin = dm_values.y(); + + const uint8_t * ql = qs_ptr + 32 * il + 2 * ir; + const uint8_t * qh = qh_ptr + 2 * ir; + + if (tid < K_SCALE_SIZE) { + scales_local[tid] = scales_ptr[tid]; + } + + item_ct1.barrier(sycl::access::fence_space::local_space); uint8_t sc, m; - get_scale_min_k4(is + 0, x[i].scales, sc, m); + get_scale_min_k4(is + 0, scales_local, sc, m); const float d1 = dall * sc; const float m1 = dmin * m; - get_scale_min_k4(is + 1, x[i].scales, sc, m); + get_scale_min_k4(is + 1, scales_local, sc, m); const float d2 = dall * sc; const float m2 = dmin * m; - uint8_t hm = 1 << (2*il); + uint8_t hm = 1 << (2 * il); y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1; y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1; hm <<= 1; y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2; y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2; #else - const int64_t tid = item_ct1.get_local_id(2); - const uint8_t q = x[i].qs[tid]; - const int64_t im = tid/8; // 0...3 - const int64_t in = tid%8; // 0...7 - const int64_t is = tid/16; // 0 or 1 - const uint8_t h = x[i].qh[in] >> im; - const float d = x[i].d; - dst_t * y = yy + i*QK_K + tid; - y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16)); - y[32] = d * x[i].scales[is+2] * ((q >> 4) - ((h >> 4) & 1 ? 0 : 16)); + GGML_UNUSED(ib); GGML_UNUSED(tid); GGML_UNUSED(yy); GGML_UNUSED(scales_local); GGML_UNUSED(n_blocks); + GGML_ABORT("Q5_K reorder dequantize not supported for QK_K != 256"); #endif } @@ -584,9 +1358,8 @@ static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restri } template -static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, +static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t ib, const sycl::nd_item<3> & item_ct1, int64_t n_blocks) { - const int64_t ib = item_ct1.get_group(2); const int64_t tid = item_ct1.get_local_id(2); const int64_t ip = tid / 32; // ip is 0 or 1 @@ -603,7 +1376,7 @@ static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * const uint8_t * scales_ptr = base_ptr + base_scales_offset; const ggml_half * d = (const ggml_half *) (base_ptr + base_d_offset) + ib; - dst_t * y = yy + ib * QK_K + 128 * ip + il; + dst_t * y = yy + 128 * ip + il; const uint8_t * ql = ql_ptr + 64 * ip + il; const uint8_t qh = *(qh_ptr + 32 * ip + il); @@ -727,6 +1500,45 @@ static void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __res } +template +static void dequantize_block_iq3_xxs_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, + const sycl::nd_item<3> & item_ct1, const int64_t nb, + const uint32_t * iq3xxs_grid, + const uint8_t * ksigns_iq2xs, + const uint8_t * kmask_iq2xs) { + const int64_t tid = item_ct1.get_local_id(2); +#if QK_K == 256 + const int64_t il = tid / 8; + const int64_t ib = tid % 8; + const uint8_t * base = static_cast(vx); + const uint8_t * qs = base + i * (3 * QK_K / 8); + const ggml_half * d = reinterpret_cast(base + nb * (3 * QK_K / 8)); + const uint8_t * q3 = qs + 8 * ib; + const uint16_t * gas = reinterpret_cast(qs + QK_K / 4) + 2 * ib; + const uint8_t * grid1 = reinterpret_cast(iq3xxs_grid + q3[2*il+0]); + const uint8_t * grid2 = reinterpret_cast(iq3xxs_grid + q3[2*il+1]); + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float dl = (float) d[i] * (0.5f + (aux32 >> 28)) * 0.5f; + const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127]; + dst_t * y = yy + 32 * ib + 8 * il; +#pragma unroll + for (int j = 0; j < 4; ++j) { + y[j+0] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); + y[j+4] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + } +#else + GGML_UNUSED(vx); + GGML_UNUSED(yy); + GGML_UNUSED(i); + GGML_UNUSED(item_ct1); + GGML_UNUSED(nb); + GGML_UNUSED(iq3xxs_grid); + GGML_UNUSED(ksigns_iq2xs); + GGML_UNUSED(kmask_iq2xs); + GGML_ABORT("IQ3_XXS reorder dequantization requires QK_K == 256"); +#endif +} + template __dpct_inline__ static void dequantize_block_iq3_s(const void *__restrict__ vx, dst_t *__restrict__ yy, @@ -757,6 +1569,48 @@ dequantize_block_iq3_s(const void *__restrict__ vx, dst_t *__restrict__ yy, } +template +__dpct_inline__ static void +dequantize_block_iq3_s_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t i, + const sycl::nd_item<3> & item_ct1, const int64_t nb, + const uint8_t * kmask_iq2xs, const uint32_t * iq3s_grid) { + const int64_t tid = item_ct1.get_local_id(2); +#if QK_K == 256 + const int64_t il = tid / 8; + const int64_t ib = tid % 8; + const uint8_t * base = static_cast(vx); + const uint8_t * qs = base + i * (QK_K / 4); + const uint8_t * qh = base + nb * (QK_K / 4) + i * (QK_K / 32); + const uint8_t * signs_base = base + nb * (QK_K / 4 + QK_K / 32) + i * (QK_K / 8); + const uint8_t * metadata = base + nb * (QK_K / 4 + QK_K / 32 + QK_K / 8) + + i * (sizeof(ggml_half) + IQ3S_N_SCALE); + const ggml_half d = *reinterpret_cast(metadata); + const uint8_t * scales = metadata + sizeof(ggml_half); + const uint8_t * qs_ib = qs + 8 * ib; + const uint8_t * grid1 = reinterpret_cast( + iq3s_grid + (qs_ib[2*il+0] | ((qh[ib] << (8 - 2*il)) & 256))); + const uint8_t * grid2 = reinterpret_cast( + iq3s_grid + (qs_ib[2*il+1] | ((qh[ib] << (7 - 2*il)) & 256))); + const float dl = (float) d * (1 + 2 * ((scales[ib/2] >> (4 * (ib%2))) & 0xf)); + const uint8_t signs = signs_base[4 * ib + il]; + dst_t * y = yy + 32 * ib + 8 * il; +#pragma unroll + for (int j = 0; j < 4; ++j) { + y[j+0] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); + y[j+4] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); + } +#else + GGML_UNUSED(vx); + GGML_UNUSED(yy); + GGML_UNUSED(i); + GGML_UNUSED(item_ct1); + GGML_UNUSED(nb); + GGML_UNUSED(kmask_iq2xs); + GGML_UNUSED(iq3s_grid); + GGML_ABORT("IQ3_S reorder dequantization requires QK_K == 256"); +#endif +} + template __dpct_inline__ static void dequantize_block_iq1_s(const void *__restrict__ vx, dst_t *__restrict__ yy, @@ -883,6 +1737,26 @@ static void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restr } } +// Reordered MXFP4 ([qs...][e...], see ggml_sycl_reordered::block_q_t): one work-item per block. +template +static void dequantize_block_mxfp4_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, + const sycl::nd_item<3> & item_ct1) { + const int64_t ib = (int64_t) item_ct1.get_group(2) * WARP_SIZE + item_ct1.get_local_id(2); + if (ib >= k / QK_MXFP4) { + return; + } + + const uint8_t * qs = (const uint8_t *) vx + ib * (QK_MXFP4 / 2); + const float d = ggml_sycl_e8m0_to_fp32(((const uint8_t *) vx)[k / 2 + ib]) * 0.5f; + dst_t * y = yy + ib * QK_MXFP4; + +#pragma unroll + for (int j = 0; j < QK_MXFP4 / 2; ++j) { + y[j] = d * kvalues_mxfp4[qs[j] & 0xf]; + y[j + QK_MXFP4 / 2] = d * kvalues_mxfp4[qs[j] >> 4]; + } +} + template static void dequantize_block_nvfp4( diff --git a/ggml/src/ggml-sycl/diag.cpp b/ggml/src/ggml-sycl/diag.cpp new file mode 100644 index 0000000..c4264fe --- /dev/null +++ b/ggml/src/ggml-sycl/diag.cpp @@ -0,0 +1,67 @@ +#include "diag.hpp" +#include "common.hpp" + +#define SYCL_DIAG_BLOCK_SIZE 256 + +template +static void diag_kernel(T * __restrict__ dst, const T * __restrict__ src, + const int64_t ne0, const int64_t ne1, + const int64_t ne2, const int64_t ne3, + const int64_t total_elements, + const sycl::nd_item<1> & item) { + const int64_t i = item.get_global_id(0); + if (i >= total_elements) { + return; + } + + const int64_t i0 = i % ne0; + const int64_t i1 = (i / ne0) % ne1; + const int64_t i2 = (i / (ne0 * ne1)) % ne2; + const int64_t i3 = i / (ne0 * ne1 * ne2); + + const int64_t dst_idx = ((i3 * ne2 + i2) * ne1 + i1) * ne0 + i0; + + if (i0 == i1) { + const int64_t batch_idx = i3 * ne2 + i2; + dst[dst_idx] = src[batch_idx * ne0 + i0]; + } else { + dst[dst_idx] = T(0); + } + + (void)ne3; +} + +inline void ggml_sycl_op_diag(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(src0->ne[1] == 1); + + dpct::queue_ptr stream = ctx.stream(); + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const void * src0_d = src0->data; + void * dst_d = dst->data; + + const int64_t ne0 = dst->ne[0]; + const int64_t ne1 = dst->ne[1]; + const int64_t ne2 = dst->ne[2]; + const int64_t ne3 = dst->ne[3]; + const int64_t n_elems = ggml_nelements(dst); + const int64_t num_blocks = (n_elems + SYCL_DIAG_BLOCK_SIZE - 1) / SYCL_DIAG_BLOCK_SIZE; + + GGML_ASSERT(dst->type == GGML_TYPE_F32); + stream->parallel_for( + sycl::nd_range<1>(num_blocks * SYCL_DIAG_BLOCK_SIZE, SYCL_DIAG_BLOCK_SIZE), + [=](sycl::nd_item<1> item) { + diag_kernel(static_cast(dst_d), + static_cast(src0_d), + ne0, ne1, ne2, ne3, n_elems, item); + }); +} + +void ggml_sycl_diag(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); + ggml_sycl_op_diag(ctx, dst); +} diff --git a/ggml/src/ggml-sycl/diag.hpp b/ggml/src/ggml-sycl/diag.hpp new file mode 100644 index 0000000..20d7ce4 --- /dev/null +++ b/ggml/src/ggml-sycl/diag.hpp @@ -0,0 +1,5 @@ +#pragma once + +#include "common.hpp" + +void ggml_sycl_diag(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index 5577bf7..d720bc1 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -3,6 +3,15 @@ #include "dequantize.hpp" #include "presets.hpp" +#if defined(__INTEL_LLVM_COMPILER) + #if __has_include() + #include + #define GGML_SYCL_DMMV_HAS_BF16 + #endif + #include + #include "esimd.hpp" +#endif + static void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){ const sycl::half *x = (const sycl::half *)vx; @@ -11,6 +20,16 @@ static void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat v.y() = x[ib + iqs + 1]; } +#ifdef GGML_SYCL_DMMV_HAS_BF16 +static void convert_bf16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){ + const sycl::ext::oneapi::bfloat16 *x = (const sycl::ext::oneapi::bfloat16 *)vx; + + // automatic bfloat16 -> float type cast if dfloat == float + v.x() = x[ib + iqs + 0]; + v.y() = x[ib + iqs + 1]; +} +#endif + static void convert_f32(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){ const float * x = (const float *) vx; @@ -46,6 +65,9 @@ static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * for (int i = 0; i < ncols; i += iter_stride) { const int col = i + vals_per_iter*tid; + if (col >= ncols) { + continue; + } const int ib = (row*ncols + col)/qk; // x block index const int iqs = (col%qk)/qr; // x quant index const int iybs = col - col%qk; // y block start index @@ -217,13 +239,28 @@ static void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y, } } -/* -DPCT1110:4: The total declared local variable size in device function -dequantize_mul_mat_vec_q2_k exceeds 128 bytes and may cause high register -pressure. Consult with your hardware vendor to find the total register size -available and adjust the code, or use smaller sub-group size to avoid high -register pressure. -*/ +#ifdef GGML_SYCL_DMMV_HAS_BF16 +static void convert_mul_mat_vec_bf16_sycl(const void *vx, const dfloat *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + // The qk=1 kernel iterates with stride 2*GGML_SYCL_DMMV_X, so ncols must be a + // multiple of that — not just GGML_SYCL_DMMV_X — to avoid out-of-bounds reads. + GGML_ASSERT(ncols % (2*GGML_SYCL_DMMV_X) == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec<1, 1, convert_bf16>(vx, y, dst, ncols, + nrows, item_ct1); + }); + } +} +#endif + static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, const float *__restrict__ yy, float *__restrict__ dst, @@ -234,7 +271,7 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); - if (row > nrows) return; + if (row >= nrows) return; const int num_blocks_per_row = ncols / QK_K; const int ib0 = row*num_blocks_per_row; @@ -245,19 +282,15 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, #if QK_K == 256 const int tid = - item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...15 + item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15 const int ix = item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1 const int step = 16/K_QUANTS_PER_ITERATION; - const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... - const int in = tid - step*im; // 0...15 or 0...7 + const int in = tid % step; // 0...15 or 0...7 const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2 - const int q_offset = 32*im + l0; - const int s_offset = 8*im; - const int y_offset = 128*im + l0; uint32_t aux[4]; const uint8_t * d = (const uint8_t *)aux; @@ -265,33 +298,39 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { - const float * y = yy + i * QK_K + y_offset; - const uint8_t * q = x[i].qs + q_offset; - const float dall = x[i].dm[0]; const float dmin = x[i].dm[1]; - const uint32_t * a = (const uint32_t *)(x[i].scales + s_offset); - aux[0] = a[0] & 0x0f0f0f0f; - aux[1] = a[1] & 0x0f0f0f0f; - aux[2] = (a[0] >> 4) & 0x0f0f0f0f; - aux[3] = (a[1] >> 4) & 0x0f0f0f0f; - - float sum1 = 0, sum2 = 0; - for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) { - sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3) - + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3) - + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3) - + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3) - + y[l+16] * d[1] * ((q[l+16] >> 0) & 3) - + y[l+48] * d[3] * ((q[l+16] >> 2) & 3) - + y[l+80] * d[5] * ((q[l+16] >> 4) & 3) - +y[l+112] * d[7] * ((q[l+16] >> 6) & 3); - sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6] - + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7]; + for (int im = 0; im < 2; ++im) { + const int q_offset = 32*im + l0; + const int s_offset = 8*im; + const int y_offset = 128*im + l0; + + const float * y = yy + i * QK_K + y_offset; + const uint8_t * q = x[i].qs + q_offset; + + const uint32_t * a = (const uint32_t *)(x[i].scales + s_offset); + aux[0] = a[0] & 0x0f0f0f0f; + aux[1] = a[1] & 0x0f0f0f0f; + aux[2] = (a[0] >> 4) & 0x0f0f0f0f; + aux[3] = (a[1] >> 4) & 0x0f0f0f0f; + + float sum1 = 0, sum2 = 0; + for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) { + sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3) + + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3) + + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3) + + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3) + + y[l+16] * d[1] * ((q[l+16] >> 0) & 3) + + y[l+48] * d[3] * ((q[l+16] >> 2) & 3) + + y[l+80] * d[5] * ((q[l+16] >> 4) & 3) + +y[l+112] * d[7] * ((q[l+16] >> 6) & 3); + sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6] + + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7]; + } + tmp += dall * sum1 - dmin * sum2; } - tmp += dall * sum1 - dmin * sum2; } #else @@ -333,7 +372,105 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + tmp += + dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + } + + if (item_ct1.get_local_id(2) == 0) { + dst[row] = tmp; + } +} + +static void dequantize_mul_mat_vec_q2_k_reorder(const void *__restrict__ vx, + const float *__restrict__ yy, + float *__restrict__ dst, + const int ncols, int nrows, + const sycl::nd_item<3> &item_ct1) { + + static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION"); + + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + if (row >= nrows) return; + + const int num_blocks_per_row = ncols / QK_K; + const int ib0 = row*num_blocks_per_row; + + // SOA base pointers for the reordered layout: + // [qs: nb * (QK_K/4)] [scales: nb * (QK_K/16)] [dm: nb * sizeof(half2)] + const int nb = nrows * num_blocks_per_row; + const uint8_t * qs_base = (const uint8_t *)vx; + const uint8_t * scales_base = qs_base + (size_t)nb * (QK_K / 4); + const sycl::half2 * dm_base = (const sycl::half2 *)(scales_base + (size_t)nb * (QK_K / 16)); + + float tmp = 0; // partial sum for thread in warp + +#if QK_K == 256 + const int tid = + item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15 + const int ix = + item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1 + + const int step = 16/K_QUANTS_PER_ITERATION; + + const int in = tid % step; // 0...15 or 0...7 + + const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2 + + uint32_t aux[4]; + const uint8_t * d = (const uint8_t *)aux; + const uint8_t * m = (const uint8_t *)(aux + 2); + + for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { + const int bi = ib0 + i; + + const sycl::half2 dm_val = dm_base[bi]; + const float dall = dm_val[0]; + const float dmin = dm_val[1]; + + for (int im = 0; im < 2; ++im) { + const int q_offset = 32*im + l0; + const int s_offset = 8*im; + const int y_offset = 128*im + l0; + + const float * y = yy + i * QK_K + y_offset; + const uint8_t * q = qs_base + bi * (QK_K / 4) + q_offset; + + const uint32_t * a = (const uint32_t *)(scales_base + bi * (QK_K / 16) + s_offset); + aux[0] = a[0] & 0x0f0f0f0f; + aux[1] = a[1] & 0x0f0f0f0f; + aux[2] = (a[0] >> 4) & 0x0f0f0f0f; + aux[3] = (a[1] >> 4) & 0x0f0f0f0f; + + float sum1 = 0, sum2 = 0; + for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) { + sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3) + + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3) + + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3) + + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3) + + y[l+16] * d[1] * ((q[l+16] >> 0) & 3) + + y[l+48] * d[3] * ((q[l+16] >> 2) & 3) + + y[l+80] * d[5] * ((q[l+16] >> 4) & 3) + +y[l+112] * d[7] * ((q[l+16] >> 6) & 3); + sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6] + + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7]; + + } + tmp += dall * sum1 - dmin * sum2; + } + } +#else + GGML_UNUSED(vx); + GGML_UNUSED(yy); + GGML_UNUSED(ncols); + GGML_UNUSED(item_ct1); + GGML_ABORT("Q2_K reorder DMMV not supported for QK_K != 256"); +#endif + + // sum up partial sums and write back result +#pragma unroll + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -343,13 +480,6 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, } } -/* -DPCT1110:5: The total declared local variable size in device function -dequantize_mul_mat_vec_q3_k exceeds 128 bytes and may cause high register -pressure. Consult with your hardware vendor to find the total register size -available and adjust the code, or use smaller sub-group size to avoid high -register pressure. -*/ static void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx, const float *__restrict__ yy, float *__restrict__ dst, @@ -358,7 +488,7 @@ static void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx, const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); - if (row > nrows) return; + if (row >= nrows) return; const int num_blocks_per_row = ncols / QK_K; const int ib0 = row*num_blocks_per_row; @@ -373,52 +503,52 @@ static void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx, const uint16_t kmask2 = 0x0f0f; const int tid = - item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...16 + item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15 const int ix = item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1 const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop const int step = 16/K_QUANTS_PER_ITERATION; - const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... - const int in = tid - step*im; // 0....15 or 0...7 - - const uint8_t m = 1 << (4*im); + const int in = tid % step; // 0...15 or 0...7 const int l0 = n*in; // 0...15 or 0...14 in steps of 2 - const int q_offset = 32*im + l0; - const int y_offset = 128*im + l0; uint16_t utmp[4]; const int8_t * s = (const int8_t *)utmp; - const uint16_t s_shift = 4*im; - for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { - const float * y = yy + i * QK_K + y_offset; - const uint8_t * q = x[i].qs + q_offset; const uint8_t * h = x[i].hmask + l0; - - const uint16_t * a = (const uint16_t *)x[i].scales; - utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4); - utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4); - utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4); - utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4); - const float d = x[i].d; - float sum = 0; - for (int l = 0; l < n; ++l) { - sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4)) - + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4)) - + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4)) - + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4)); - sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4)) - + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4)) - + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4)) - + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4)); + for (int im = 0; im < 2; ++im) { + const int q_offset = 32*im + l0; + const int y_offset = 128*im + l0; + const uint16_t s_shift = 4*im; + const uint8_t m = 1 << (4*im); + + const float * y = yy + i * QK_K + y_offset; + const uint8_t * q = x[i].qs + q_offset; + + const uint16_t * a = (const uint16_t *)x[i].scales; + utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4); + utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4); + utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4); + utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4); + + float sum = 0; + for (int l = 0; l < n; ++l) { + sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4)) + + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4)) + + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4)) + + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4)); + sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4)) + + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4)) + + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4)) + + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4)); + } + tmp += d * sum; } - tmp += d * sum; } #else @@ -452,7 +582,105 @@ static void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + tmp += + dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + } + + if (item_ct1.get_local_id(2) == 0) { + dst[row] = tmp; + } +} + +static void dequantize_mul_mat_vec_q3_k_reorder(const void *__restrict__ vx, + const float *__restrict__ yy, + float *__restrict__ dst, + const int ncols, int nrows, + const sycl::nd_item<3> &item_ct1) { + + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + if (row >= nrows) return; + + const int num_blocks_per_row = ncols / QK_K; + const int ib0 = row*num_blocks_per_row; + + // SOA base pointers for the reordered layout: + // [qs: nb * (QK_K/4)] [hmask: nb * (QK_K/8)] [scales: nb * 12] [d: nb * sizeof(half)] + const int nb = nrows * num_blocks_per_row; + const uint8_t * qs_base = (const uint8_t *)vx; + const uint8_t * hmask_base = qs_base + (size_t)nb * (QK_K / 4); + const uint8_t * scales_base = hmask_base + (size_t)nb * (QK_K / 8); + const sycl::half * d_base = (const sycl::half *)(scales_base + (size_t)nb * 12); + + float tmp = 0; // partial sum for thread in warp + +#if QK_K == 256 + + const uint16_t kmask1 = 0x0303; + const uint16_t kmask2 = 0x0f0f; + + const int tid = + item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15 + const int ix = + item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1 + + const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop + const int step = 16/K_QUANTS_PER_ITERATION; + const int in = tid % step; // 0...15 or 0...7 + + const int l0 = n*in; // 0...15 or 0...14 in steps of 2 + + uint16_t utmp[4]; + const int8_t * s = (const int8_t *)utmp; + + for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { + const int bi = ib0 + i; + + const uint8_t * h = hmask_base + bi * (QK_K / 8) + l0; + + const float d = d_base[bi]; + + for (int im = 0; im < 2; ++im) { + const int q_offset = 32*im + l0; + const int y_offset = 128*im + l0; + const uint16_t s_shift = 4*im; + const uint8_t m = 1 << (4*im); + + const float * y = yy + i * QK_K + y_offset; + const uint8_t * q = qs_base + bi * (QK_K / 4) + q_offset; + + const uint16_t * a = (const uint16_t *)(scales_base + bi * 12); + utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4); + utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4); + utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4); + utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4); + + float sum = 0; + for (int l = 0; l < n; ++l) { + sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4)) + + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4)) + + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4)) + + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4)); + sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4)) + + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4)) + + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4)) + + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4)); + } + tmp += d * sum; + } + } +#else + GGML_UNUSED(vx); + GGML_UNUSED(yy); + GGML_UNUSED(ncols); + GGML_UNUSED(item_ct1); + GGML_ABORT("Q3_K reorder DMMV not supported for QK_K != 256"); +#endif + + // sum up partial sums and write back result +#pragma unroll + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -462,13 +690,6 @@ static void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx, } } -/* -DPCT1110:6: The total declared local variable size in device function -dequantize_mul_mat_vec_q4_k exceeds 128 bytes and may cause high register -pressure. Consult with your hardware vendor to find the total register size -available and adjust the code, or use smaller sub-group size to avoid high -register pressure. -*/ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx, const float *__restrict__ yy, float *__restrict__ dst, @@ -477,7 +698,7 @@ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx, const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); - if (row > nrows) return; + if (row >= nrows) return; const int num_blocks_per_row = ncols / QK_K; const int ib0 = row*num_blocks_per_row; @@ -489,22 +710,19 @@ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx, const uint16_t kmask3 = 0xc0c0; const int tid = - item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...16 + item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15 const int ix = item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1 const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4 - const int il = tid/step; // 0...3 - const int ir = tid - step*il; // 0...7 or 0...3 + const int il_base = tid/step; // 0 or 1 (was 0...3) + const int ir = tid - step*il_base; // 0...7 or 0...3 const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4 - const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 - const int in = il%2; + const int in = il_base%2; const int l0 = n*(2*ir + in); - const int q_offset = 32*im + l0; - const int y_offset = 64*im + l0; uint16_t aux[4]; const uint8_t * sc = (const uint8_t *)aux; @@ -521,55 +739,60 @@ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx, for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { - const float * y1 = yy + i*QK_K + y_offset; - const float * y2 = y1 + 128; - const float dall = x[i].dm[0]; const float dmin = x[i].dm[1]; - const uint16_t * a = (const uint16_t *)x[i].scales; - aux[0] = a[im+0] & kmask1; - aux[1] = a[im+2] & kmask1; - aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2); - aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2); + for (int im = 0; im < 2; ++im) { + const int q_offset = 32*im + l0; + const int y_offset = 64*im + l0; + + const float * y1 = yy + i*QK_K + y_offset; + const float * y2 = y1 + 128; + + const uint16_t * a = (const uint16_t *)x[i].scales; + aux[0] = a[im+0] & kmask1; + aux[1] = a[im+2] & kmask1; + aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2); + aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2); #if K_QUANTS_PER_ITERATION == 2 - const uint32_t * q1 = (const uint32_t *)(x[i].qs + q_offset); - const uint32_t * q2 = q1 + 16; - - q32[0] = q1[0] & 0x0f0f0f0f; - q32[1] = q1[0] & 0xf0f0f0f0; - q32[2] = q2[0] & 0x0f0f0f0f; - q32[3] = q2[0] & 0xf0f0f0f0; - - sycl::float4 s = {0.f, 0.f, 0.f, 0.f}; - float smin = 0; - for (int l = 0; l < 4; ++l) { - s.x() += y1[l] * q4[l + 0]; s.y() += y1[l + 32] * q4[l + 4]; - s.z() += y2[l] * q4[l + 8]; s.w() += y2[l + 32] * q4[l + 12]; - smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; - } - tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f / 16.f + - s.z() * sc[4] + s.w() * sc[5] * 1.f / 16.f) - - dmin * smin; + const uint32_t * q1 = (const uint32_t *)(x[i].qs + q_offset); + const uint32_t * q2 = q1 + 16; + + q32[0] = q1[0] & 0x0f0f0f0f; + q32[1] = q1[0] & 0xf0f0f0f0; + q32[2] = q2[0] & 0x0f0f0f0f; + q32[3] = q2[0] & 0xf0f0f0f0; + + sycl::float4 s = {0.f, 0.f, 0.f, 0.f}; + float smin = 0; + for (int l = 0; l < 4; ++l) { + s.x() += y1[l] * q4[l + 0]; s.y() += y1[l + 32] * q4[l + 4]; + s.z() += y2[l] * q4[l + 8]; s.w() += y2[l + 32] * q4[l + 12]; + smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; + } + tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f / 16.f + + s.z() * sc[4] + s.w() * sc[5] * 1.f / 16.f) - + dmin * smin; #else - const uint16_t * q1 = (const uint16_t *)(x[i].qs + q_offset); - const uint16_t * q2 = q1 + 32; - - q16[0] = q1[0] & 0x0f0f; - q16[1] = q1[0] & 0xf0f0; - q16[2] = q2[0] & 0x0f0f; - q16[3] = q2[0] & 0xf0f0; - - float4 s = {0.f, 0.f, 0.f, 0.f}; - float smin = 0; - for (int l = 0; l < 2; ++l) { - s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2]; - s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6]; - smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; - } - tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin; + const uint16_t * q1 = (const uint16_t *)(x[i].qs + q_offset); + const uint16_t * q2 = q1 + 32; + + q16[0] = q1[0] & 0x0f0f; + q16[1] = q1[0] & 0xf0f0; + q16[2] = q2[0] & 0x0f0f; + q16[3] = q2[0] & 0xf0f0; + + sycl::float4 s = {0.f, 0.f, 0.f, 0.f}; + float smin = 0; + for (int l = 0; l < 2; ++l) { + s.x() += y1[l] * q4[l+0]; s.y() += y1[l+32] * q4[l+2]; + s.z() += y2[l] * q4[l+4]; s.w() += y2[l+32] * q4[l+6]; + smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; + } + tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f/16.f + s.z() * sc[4] + s.w() * sc[5] * 1.f/16.f) - dmin * smin; #endif + } } #else @@ -605,7 +828,7 @@ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -623,7 +846,7 @@ static void dequantize_mul_mat_vec_q4_k_reorder(const void *__restrict__ vx, const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); - if (row > nrows) return; + if (row >= nrows) return; const int num_blocks_per_row = ncols / QK_K; const int ib0 = row*num_blocks_per_row; @@ -640,22 +863,19 @@ static void dequantize_mul_mat_vec_q4_k_reorder(const void *__restrict__ vx, const uint16_t kmask3 = 0xc0c0; const int tid = - item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...16 + item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15 const int ix = item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1 const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4 - const int il = tid/step; // 0...3 - const int ir = tid - step*il; // 0...7 or 0...3 + const int il_base = tid/step; // 0 or 1 (was 0...3) + const int ir = tid - step*il_base; // 0...7 or 0...3 const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4 - const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 - const int in = il%2; + const int in = il_base%2; const int l0 = n*(2*ir + in); - const int q_offset = 32*im + l0; - const int y_offset = 64*im + l0; uint16_t aux[4]; const uint8_t * sc = (const uint8_t *)aux; @@ -673,56 +893,61 @@ static void dequantize_mul_mat_vec_q4_k_reorder(const void *__restrict__ vx, for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { const int bi = ib0 + i; - const float * y1 = yy + i*QK_K + y_offset; - const float * y2 = y1 + 128; - const sycl::half2 dm_val = dm_base[bi]; const float dall = dm_val[0]; const float dmin = dm_val[1]; - const uint16_t * a = (const uint16_t *)(scales_base + bi * K_SCALE_SIZE); - aux[0] = a[im+0] & kmask1; - aux[1] = a[im+2] & kmask1; - aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2); - aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2); + for (int im = 0; im < 2; ++im) { + const int q_offset = 32*im + l0; + const int y_offset = 64*im + l0; + + const float * y1 = yy + i*QK_K + y_offset; + const float * y2 = y1 + 128; + + const uint16_t * a = (const uint16_t *)(scales_base + bi * K_SCALE_SIZE); + aux[0] = a[im+0] & kmask1; + aux[1] = a[im+2] & kmask1; + aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2); + aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2); #if K_QUANTS_PER_ITERATION == 2 - const uint32_t * q1 = (const uint32_t *)(qs_base + bi * (QK_K / 2) + q_offset); - const uint32_t * q2 = q1 + 16; - - q32[0] = q1[0] & 0x0f0f0f0f; - q32[1] = q1[0] & 0xf0f0f0f0; - q32[2] = q2[0] & 0x0f0f0f0f; - q32[3] = q2[0] & 0xf0f0f0f0; - - sycl::float4 s = {0.f, 0.f, 0.f, 0.f}; - float smin = 0; - for (int l = 0; l < 4; ++l) { - s.x() += y1[l] * q4[l + 0]; s.y() += y1[l + 32] * q4[l + 4]; - s.z() += y2[l] * q4[l + 8]; s.w() += y2[l + 32] * q4[l + 12]; - smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; - } - tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f / 16.f + - s.z() * sc[4] + s.w() * sc[5] * 1.f / 16.f) - - dmin * smin; + const uint32_t * q1 = (const uint32_t *)(qs_base + bi * (QK_K / 2) + q_offset); + const uint32_t * q2 = q1 + 16; + + q32[0] = q1[0] & 0x0f0f0f0f; + q32[1] = q1[0] & 0xf0f0f0f0; + q32[2] = q2[0] & 0x0f0f0f0f; + q32[3] = q2[0] & 0xf0f0f0f0; + + sycl::float4 s = {0.f, 0.f, 0.f, 0.f}; + float smin = 0; + for (int l = 0; l < 4; ++l) { + s.x() += y1[l] * q4[l + 0]; s.y() += y1[l + 32] * q4[l + 4]; + s.z() += y2[l] * q4[l + 8]; s.w() += y2[l + 32] * q4[l + 12]; + smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; + } + tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f / 16.f + + s.z() * sc[4] + s.w() * sc[5] * 1.f / 16.f) - + dmin * smin; #else - const uint16_t * q1 = (const uint16_t *)(qs_base + bi * (QK_K / 2) + q_offset); - const uint16_t * q2 = q1 + 32; - - q16[0] = q1[0] & 0x0f0f; - q16[1] = q1[0] & 0xf0f0; - q16[2] = q2[0] & 0x0f0f; - q16[3] = q2[0] & 0xf0f0; - - float4 s = {0.f, 0.f, 0.f, 0.f}; - float smin = 0; - for (int l = 0; l < 2; ++l) { - s.x += y1[l] * q4[l+0]; s.y += y1[l+32] * q4[l+2]; - s.z += y2[l] * q4[l+4]; s.w += y2[l+32] * q4[l+6]; - smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; - } - tmp += dall * (s.x * sc[0] + s.y * sc[1] * 1.f/16.f + s.z * sc[4] + s.w * sc[5] * 1.f/16.f) - dmin * smin; + const uint16_t * q1 = (const uint16_t *)(qs_base + bi * (QK_K / 2) + q_offset); + const uint16_t * q2 = q1 + 32; + + q16[0] = q1[0] & 0x0f0f; + q16[1] = q1[0] & 0xf0f0; + q16[2] = q2[0] & 0x0f0f; + q16[3] = q2[0] & 0xf0f0; + + sycl::float4 s = {0.f, 0.f, 0.f, 0.f}; + float smin = 0; + for (int l = 0; l < 2; ++l) { + s.x() += y1[l] * q4[l+0]; s.y() += y1[l+32] * q4[l+2]; + s.z() += y2[l] * q4[l+4]; s.w() += y2[l+32] * q4[l+6]; + smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7]; + } + tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f/16.f + s.z() * sc[4] + s.w() * sc[5] * 1.f/16.f) - dmin * smin; #endif + } } #else @@ -761,7 +986,7 @@ static void dequantize_mul_mat_vec_q4_k_reorder(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -771,20 +996,15 @@ static void dequantize_mul_mat_vec_q4_k_reorder(const void *__restrict__ vx, } } -/* -DPCT1110:7: The total declared local variable size in device function -dequantize_mul_mat_vec_q5_k exceeds 128 bytes and may cause high register -pressure. Consult with your hardware vendor to find the total register size -available and adjust the code, or use smaller sub-group size to avoid high -register pressure. -*/ static void dequantize_mul_mat_vec_q5_k(const void *__restrict__ vx, const float *__restrict__ yy, float *__restrict__ dst, - const int ncols, + const int ncols, int nrows, const sycl::nd_item<3> &item_ct1) { - const int row = item_ct1.get_group(2); + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + if (row >= nrows) return; const int num_blocks_per_row = ncols / QK_K; const int ib0 = row*num_blocks_per_row; @@ -797,22 +1017,16 @@ static void dequantize_mul_mat_vec_q5_k(const void *__restrict__ vx, const uint16_t kmask2 = 0x0f0f; const uint16_t kmask3 = 0xc0c0; - const int tid = item_ct1.get_local_id(2) / 2; // 0...15 + const int tid = item_ct1.get_local_id(2) / 2; // 0...7 const int ix = item_ct1.get_local_id(2) % 2; - const int il = tid/4; // 0...3 - const int ir = tid - 4*il;// 0...3 + const int il_base = tid/4; // 0 or 1 (was 0...3) + const int ir = tid - 4*il_base;// 0...3 const int n = 2; - const int im = il/2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 - const int in = il%2; + const int in = il_base%2; const int l0 = n*(2*ir + in); - const int q_offset = 32*im + l0; - const int y_offset = 64*im + l0; - - const uint8_t hm1 = 1 << (2*im); - const uint8_t hm2 = hm1 << 4; uint16_t aux[4]; const uint8_t * sc = (const uint8_t *)aux; @@ -822,51 +1036,59 @@ static void dequantize_mul_mat_vec_q5_k(const void *__restrict__ vx, for (int i = ix; i < num_blocks_per_row; i += 2) { - const uint8_t * ql1 = x[i].qs + q_offset; const uint8_t * qh = x[i].qh + l0; - const float * y1 = yy + i*QK_K + y_offset; - const float * y2 = y1 + 128; - const float dall = x[i].dm[0]; const float dmin = x[i].dm[1]; - const uint16_t * a = (const uint16_t *)x[i].scales; - aux[0] = a[im+0] & kmask1; - aux[1] = a[im+2] & kmask1; - aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2); - aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2); - - sycl::float4 sum = {0.f, 0.f, 0.f, 0.f}; - float smin = 0; - const uint16_t * q1 = (const uint16_t *)ql1; - const uint16_t * q2 = q1 + 32; - q16[0] = q1[0] & 0x0f0f; - q16[1] = q1[8] & 0x0f0f; - q16[2] = (q1[0] >> 4) & 0x0f0f; - q16[3] = (q1[8] >> 4) & 0x0f0f; - q16[4] = q2[0] & 0x0f0f; - q16[5] = q2[8] & 0x0f0f; - q16[6] = (q2[0] >> 4) & 0x0f0f; - q16[7] = (q2[8] >> 4) & 0x0f0f; - for (int l = 0; l < n; ++l) { - sum.x() += - y1[l + 0] * (q4[l + 0] + (qh[l + 0] & (hm1 << 0) ? 16 : 0)) + - y1[l + 16] * (q4[l + 2] + (qh[l + 16] & (hm1 << 0) ? 16 : 0)); - sum.y() += - y1[l + 32] * (q4[l + 4] + (qh[l + 0] & (hm1 << 1) ? 16 : 0)) + - y1[l + 48] * (q4[l + 6] + (qh[l + 16] & (hm1 << 1) ? 16 : 0)); - sum.z() += - y2[l + 0] * (q4[l + 8] + (qh[l + 0] & (hm2 << 0) ? 16 : 0)) + - y2[l + 16] * (q4[l + 10] + (qh[l + 16] & (hm2 << 0) ? 16 : 0)); - sum.w() += - y2[l + 32] * (q4[l + 12] + (qh[l + 0] & (hm2 << 1) ? 16 : 0)) + - y2[l + 48] * (q4[l + 14] + (qh[l + 16] & (hm2 << 1) ? 16 : 0)); - smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3] - + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7]; + for (int im = 0; im < 2; ++im) { + const int q_offset = 32*im + l0; + const int y_offset = 64*im + l0; + + const uint8_t hm1 = 1 << (2*im); + const uint8_t hm2 = hm1 << 4; + + const uint8_t * ql1 = x[i].qs + q_offset; + const float * y1 = yy + i*QK_K + y_offset; + const float * y2 = y1 + 128; + + const uint16_t * a = (const uint16_t *)x[i].scales; + aux[0] = a[im+0] & kmask1; + aux[1] = a[im+2] & kmask1; + aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2); + aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2); + + sycl::float4 sum = {0.f, 0.f, 0.f, 0.f}; + float smin = 0; + const uint16_t * q1 = (const uint16_t *)ql1; + const uint16_t * q2 = q1 + 32; + q16[0] = q1[0] & 0x0f0f; + q16[1] = q1[8] & 0x0f0f; + q16[2] = (q1[0] >> 4) & 0x0f0f; + q16[3] = (q1[8] >> 4) & 0x0f0f; + q16[4] = q2[0] & 0x0f0f; + q16[5] = q2[8] & 0x0f0f; + q16[6] = (q2[0] >> 4) & 0x0f0f; + q16[7] = (q2[8] >> 4) & 0x0f0f; + for (int l = 0; l < n; ++l) { + sum.x() += + y1[l + 0] * (q4[l + 0] + (qh[l + 0] & (hm1 << 0) ? 16 : 0)) + + y1[l + 16] * (q4[l + 2] + (qh[l + 16] & (hm1 << 0) ? 16 : 0)); + sum.y() += + y1[l + 32] * (q4[l + 4] + (qh[l + 0] & (hm1 << 1) ? 16 : 0)) + + y1[l + 48] * (q4[l + 6] + (qh[l + 16] & (hm1 << 1) ? 16 : 0)); + sum.z() += + y2[l + 0] * (q4[l + 8] + (qh[l + 0] & (hm2 << 0) ? 16 : 0)) + + y2[l + 16] * (q4[l + 10] + (qh[l + 16] & (hm2 << 0) ? 16 : 0)); + sum.w() += + y2[l + 32] * (q4[l + 12] + (qh[l + 0] & (hm2 << 1) ? 16 : 0)) + + y2[l + 48] * (q4[l + 14] + (qh[l + 16] & (hm2 << 1) ? 16 : 0)); + smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3] + + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7]; + } + tmp += dall * (sum.x() * sc[0] + sum.y() * sc[1] + sum.z() * sc[4] + + sum.w() * sc[5]) - + dmin * smin; } - tmp += dall * (sum.x() * sc[0] + sum.y() * sc[1] + sum.z() * sc[4] + - sum.w() * sc[5]) - - dmin * smin; } #else @@ -895,7 +1117,125 @@ static void dequantize_mul_mat_vec_q5_k(const void *__restrict__ vx, // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + tmp += + dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); + } + + if (item_ct1.get_local_id(2) == 0) { + dst[row] = tmp; + } +} + +static void dequantize_mul_mat_vec_q5_k_reorder(const void *__restrict__ vx, + const float *__restrict__ yy, + float *__restrict__ dst, + const int ncols, int nrows, + const sycl::nd_item<3> &item_ct1) { + + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + if (row >= nrows) return; + const int num_blocks_per_row = ncols / QK_K; + const int ib0 = row*num_blocks_per_row; + + // SOA base pointers for the reordered layout: + // [qs: nb * QK_K/2] [qh: nb * QK_K/8] [scales: nb * K_SCALE_SIZE] [dm: nb * sizeof(half2)] + const int nb = nrows * num_blocks_per_row; + const uint8_t * qs_base = (const uint8_t *)vx; + const uint8_t * qh_base = qs_base + (size_t)nb * (QK_K / 2); + const uint8_t * scales_base = qh_base + (size_t)nb * (QK_K / 8); + const sycl::half2 * dm_base = (const sycl::half2 *)(scales_base + (size_t)nb * K_SCALE_SIZE); + + float tmp = 0; // partial sum for thread in warp + +#if QK_K == 256 + const uint16_t kmask1 = 0x3f3f; + const uint16_t kmask2 = 0x0f0f; + const uint16_t kmask3 = 0xc0c0; + + const int tid = item_ct1.get_local_id(2) / 2; // 0...15 + const int ix = item_ct1.get_local_id(2) % 2; + + const int il_base = tid/4; // 0...3 + const int ir = tid - 4*il_base;// 0...3 + const int n = 2; + + const int in = il_base%2; + + const int l0 = n*(2*ir + in); + + uint16_t aux[4]; + const uint8_t * sc = (const uint8_t *)aux; + + uint16_t q16[8]; + const uint8_t * q4 = (const uint8_t *)q16; + + for (int i = ix; i < num_blocks_per_row; i += 2) { + const int bi = ib0 + i; + + const uint8_t * qh = qh_base + bi * (QK_K / 8) + l0; + const sycl::half2 dm_val = dm_base[bi]; + const float dall = dm_val[0]; + const float dmin = dm_val[1]; + + for (int im = 0; im < 2; ++im) { + const int q_offset = 32*im + l0; + const int y_offset = 64*im + l0; + + const uint8_t hm1 = 1 << (2*im); + const uint8_t hm2 = hm1 << 4; + + const uint8_t * ql1 = qs_base + bi * (QK_K / 2) + q_offset; + const float * y1 = yy + i*QK_K + y_offset; + const float * y2 = y1 + 128; + + const uint16_t * a = (const uint16_t *)(scales_base + bi * K_SCALE_SIZE); + aux[0] = a[im+0] & kmask1; + aux[1] = a[im+2] & kmask1; + aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2); + aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2); + + sycl::float4 sum = {0.f, 0.f, 0.f, 0.f}; + float smin = 0; + const uint16_t * q1 = (const uint16_t *)ql1; + const uint16_t * q2 = q1 + 32; + q16[0] = q1[0] & 0x0f0f; + q16[1] = q1[8] & 0x0f0f; + q16[2] = (q1[0] >> 4) & 0x0f0f; + q16[3] = (q1[8] >> 4) & 0x0f0f; + q16[4] = q2[0] & 0x0f0f; + q16[5] = q2[8] & 0x0f0f; + q16[6] = (q2[0] >> 4) & 0x0f0f; + q16[7] = (q2[8] >> 4) & 0x0f0f; + for (int l = 0; l < n; ++l) { + sum.x() += + y1[l + 0] * (q4[l + 0] + (qh[l + 0] & (hm1 << 0) ? 16 : 0)) + + y1[l + 16] * (q4[l + 2] + (qh[l + 16] & (hm1 << 0) ? 16 : 0)); + sum.y() += + y1[l + 32] * (q4[l + 4] + (qh[l + 0] & (hm1 << 1) ? 16 : 0)) + + y1[l + 48] * (q4[l + 6] + (qh[l + 16] & (hm1 << 1) ? 16 : 0)); + sum.z() += + y2[l + 0] * (q4[l + 8] + (qh[l + 0] & (hm2 << 0) ? 16 : 0)) + + y2[l + 16] * (q4[l + 10] + (qh[l + 16] & (hm2 << 0) ? 16 : 0)); + sum.w() += + y2[l + 32] * (q4[l + 12] + (qh[l + 0] & (hm2 << 1) ? 16 : 0)) + + y2[l + 48] * (q4[l + 14] + (qh[l + 16] & (hm2 << 1) ? 16 : 0)); + smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3] + + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7]; + } + tmp += dall * (sum.x() * sc[0] + sum.y() * sc[1] + sum.z() * sc[4] + + sum.w() * sc[5]) - + dmin * smin; + } + } +#else + // The reordered Q5_K layout is only produced for QK_K == 256. +#endif + + // sum up partial sums and write back result +#pragma unroll + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -912,7 +1252,7 @@ static void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const floa const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); - if (row > nrows) return; + if (row >= nrows) return; const int num_blocks_per_row = ncols / QK_K; const int ib0 = row*num_blocks_per_row; @@ -922,14 +1262,13 @@ static void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const floa #if QK_K == 256 const int tid = - item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...16 + item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15 const int ix = item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0, 1 const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8 - const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... - const int in = tid - step*im; // 0...15 or 0...7 + const int in = tid % step; // 0...15 or 0...7 #if K_QUANTS_PER_ITERATION == 1 const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 @@ -938,42 +1277,45 @@ static void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const floa const int l0 = 4 * in; // 0, 4, 8, ..., 28 const int is = in / 4; #endif - const int ql_offset = 64*im + l0; - const int qh_offset = 32*im + l0; - const int s_offset = 8*im + is; - const int y_offset = 128*im + l0; float tmp = 0; // partial sum for thread in warp for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { - const float * y = yy + i * QK_K + y_offset; - const uint8_t * ql = x[i].ql + ql_offset; - const uint8_t * qh = x[i].qh + qh_offset; - const int8_t * s = x[i].scales + s_offset; - const float d = x[i].d; + for (int im = 0; im < 2; ++im) { + const int ql_offset = 64*im + l0; + const int qh_offset = 32*im + l0; + const int s_offset = 8*im + is; + const int y_offset = 128*im + l0; + + const float * y = yy + i * QK_K + y_offset; + const uint8_t * ql = x[i].ql + ql_offset; + const uint8_t * qh = x[i].qh + qh_offset; + const int8_t * s = x[i].scales + s_offset; + #if K_QUANTS_PER_ITERATION == 1 - float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32) - + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32) - + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32) - + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32) - + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32) - + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32) - + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32) - +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32); - tmp += sum; + float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32) + + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32) + + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32) + + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32) + + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32) + + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32) + + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32) + +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32); + tmp += sum; #else - float sum = 0; - for (int l = 0; l < 4; ++l) { - sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32) - + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32) - + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32) - + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32); - } - tmp += sum; + float sum = 0; + for (int l = 0; l < 4; ++l) { + sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32) + + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32) + + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32) + + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32); + } + tmp += sum; #endif + } } @@ -1010,7 +1352,7 @@ static void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const floa // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -1027,7 +1369,7 @@ static void dequantize_mul_mat_vec_q6_k_reorder(const void * __restrict__ vx, co const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); - if (row > nrows) return; + if (row >= nrows) return; const int num_blocks_per_row = ncols / QK_K; const int ib0 = row*num_blocks_per_row; @@ -1043,14 +1385,13 @@ static void dequantize_mul_mat_vec_q6_k_reorder(const void * __restrict__ vx, co #if QK_K == 256 const int tid = - item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...16 + item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15 const int ix = item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0, 1 const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8 - const int im = tid/step; // 0 or 1. 0 computes 0..., 1 computes 128... - const int in = tid - step*im; // 0...15 or 0...7 + const int in = tid % step; // 0...15 or 0...7 #if K_QUANTS_PER_ITERATION == 1 const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 @@ -1059,43 +1400,46 @@ static void dequantize_mul_mat_vec_q6_k_reorder(const void * __restrict__ vx, co const int l0 = 4 * in; // 0, 4, 8, ..., 28 const int is = in / 4; #endif - const int ql_offset = 64*im + l0; - const int qh_offset = 32*im + l0; - const int s_offset = 8*im + is; - const int y_offset = 128*im + l0; float tmp = 0; // partial sum for thread in warp for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) { const int bi = ib0 + i; - const float * y = yy + i * QK_K + y_offset; - const uint8_t * ql = ql_base + bi * (QK_K / 2) + ql_offset; - const uint8_t * qh = qh_base + bi * (QK_K / 4) + qh_offset; - const int8_t * s = scales_base + bi * (QK_K / 16) + s_offset; - const float d = d_base[bi]; + for (int im = 0; im < 2; ++im) { + const int ql_offset = 64*im + l0; + const int qh_offset = 32*im + l0; + const int s_offset = 8*im + is; + const int y_offset = 128*im + l0; + + const float * y = yy + i * QK_K + y_offset; + const uint8_t * ql = ql_base + bi * (QK_K / 2) + ql_offset; + const uint8_t * qh = qh_base + bi * (QK_K / 4) + qh_offset; + const int8_t * s = scales_base + bi * (QK_K / 16) + s_offset; + #if K_QUANTS_PER_ITERATION == 1 - float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32) - + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32) - + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32) - + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32) - + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32) - + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32) - + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32) - +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32); - tmp += sum; + float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32) + + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32) + + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32) + + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32) + + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32) + + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32) + + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32) + +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32); + tmp += sum; #else - float sum = 0; - for (int l = 0; l < 4; ++l) { - sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32) - + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32) - + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32) - + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32); - } - tmp += sum; + float sum = 0; + for (int l = 0; l < 4; ++l) { + sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32) + + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32) + + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32) + + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32); + } + tmp += sum; #endif + } } @@ -1133,7 +1477,7 @@ static void dequantize_mul_mat_vec_q6_k_reorder(const void * __restrict__ vx, co // sum up partial sums and write back result #pragma unroll - for (int mask = QK_WARP_SIZE / 2; mask > 0; mask >>= 1) { + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask); } @@ -1188,6 +1532,50 @@ static void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y, } } +static void dequantize_mul_mat_vec_q1_0_sycl_reorder(const void *vx, const dfloat *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec_reorder( + vx, y, dst, ncols, nrows, item_ct1); + }); + } +} + +static void dequantize_mul_mat_vec_q1_0_sycl(const void *vx, const dfloat *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec( + vx, y, dst, ncols, nrows, item_ct1); + }); + } +} + static void dequantize_mul_mat_vec_q4_1_sycl(const void *vx, const dfloat *y, float *dst, const int ncols, const int nrows, @@ -1377,14 +1765,30 @@ static void dequantize_mul_mat_vec_q2_K_sycl(const void *vx, const float *y, const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2 const int block_num_y = (nrows + ny - 1) / ny; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec_q2_k(vx, y, dst, ncols, nrows, item_ct1); }); } +static void dequantize_mul_mat_vec_q2_K_sycl_reorder(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int ny = 2 / K_QUANTS_PER_ITERATION; + const int block_num_y = (nrows + ny - 1) / ny; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec_q2_k_reorder(vx, y, dst, ncols, nrows, item_ct1); + }); +} + static void dequantize_mul_mat_vec_q3_K_sycl(const void *vx, const float *y, float *dst, const int ncols, const int nrows, @@ -1393,14 +1797,30 @@ static void dequantize_mul_mat_vec_q3_K_sycl(const void *vx, const float *y, const int ny = 2 / K_QUANTS_PER_ITERATION; const int block_num_y = (nrows + ny - 1) / ny; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec_q3_k(vx, y, dst, ncols, nrows, item_ct1); }); } +static void dequantize_mul_mat_vec_q3_K_sycl_reorder(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int ny = 2 / K_QUANTS_PER_ITERATION; + const int block_num_y = (nrows + ny - 1) / ny; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec_q3_k_reorder(vx, y, dst, ncols, nrows, item_ct1); + }); +} + static void dequantize_mul_mat_vec_q4_K_sycl(const void *vx, const float *y, float *dst, const int ncols, const int nrows, @@ -1409,10 +1829,10 @@ static void dequantize_mul_mat_vec_q4_K_sycl(const void *vx, const float *y, const int ny = 2 / K_QUANTS_PER_ITERATION; const int block_num_y = (nrows + ny - 1) / ny; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec_q4_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1422,11 +1842,14 @@ static void dequantize_mul_mat_vec_q5_K_sycl(const void *vx, const float *y, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); - const sycl::range<3> block_dims(1, 1, QK_WARP_SIZE); + const int ny = 2 / K_QUANTS_PER_ITERATION; + const int block_num_y = (nrows + ny - 1) / ny; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, nrows) * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { - dequantize_mul_mat_vec_q5_k(vx, y, dst, ncols, item_ct1); + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec_q5_k(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1438,14 +1861,393 @@ static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y, const int ny = 2 / K_QUANTS_PER_ITERATION; const int block_num_y = (nrows + ny - 1) / ny; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec_q6_k(vx, y, dst, ncols, nrows, item_ct1); }); } +#ifdef GGML_SYCL_DMMV_HAS_ESIMD +using ggml_sycl_esimd::GGML_SYCL_DMMV_ESIMD_WG_SIZE; + +// policies for dequantize_mul_mat_vec_reorder_esimd: row mapping and what to do with the two sums +// plain mat-vec: a work-group owns two consecutive rows of one matrix +struct esimd_dmmv_row_pair_policy { + const void * vx; + float * dst; + + ESIMD_INLINE const void * w_a() const { return vx; } + ESIMD_INLINE const void * w_b() const { return vx; } + + ESIMD_INLINE int row_a(int g) const { return g * 2; } + ESIMD_INLINE int row_b(int g) const { return g * 2 + 1; } + ESIMD_INLINE bool has_b(int g, int nrows) const { return row_b(g) < nrows; } + + ESIMD_INLINE void epilogue(int g, int nrows, float sum_a, float sum_b) const { + dst[row_a(g)] = sum_a; + if (has_b(g, nrows)) { + dst[row_b(g)] = sum_b; + } + } +}; + +// fused gate+up+GLU: a work-group owns one row of both matrices; a is the gate, b the up +struct esimd_dmmv_glu_policy { + const void * vgate; + const void * vup; + float * dst; + ggml_glu_op glu_op; + + ESIMD_INLINE const void * w_a() const { return vgate; } + ESIMD_INLINE const void * w_b() const { return vup; } + + ESIMD_INLINE int row_a(int g) const { return g; } + ESIMD_INLINE int row_b(int g) const { return g; } + ESIMD_INLINE bool has_b(int, int) const { return true; } // up always contributes + + ESIMD_INLINE void epilogue(int g, int, float gate, float up) const { + using namespace sycl::ext::intel::esimd; + + // SwiGLU: silu(gate)*up = gate*sigmoid(gate)*up. GEGLU's tanh form reduces + // to the same shape via tanh(y) = 2*sigmoid(2y) - 1, so only the sigmoid + // argument differs. sigmoid via exp2 (width-1 simd is a degenerate case for + // the EM pipe, so broadcast to 16 lanes); lane 0 holds the result. + const float GELU_COEF_A = 0.044715f; + const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; + const float NEG_LOG2E = -1.44269504088896341f; + + const float arg = (glu_op == GGML_GLU_OP_GEGLU) ? + 2.0f * SQRT_2_OVER_PI * gate * (1.0f + GELU_COEF_A * gate * gate) : + gate; + + simd t = arg * NEG_LOG2E; + simd e = exp2(t); + dst[g] = gate / (1.0f + e[0]) * up; + } +}; + +// generic reordered dequantize-matvec: each work-group owns a pair of rows chosen by the policy +// and updates one 32-wide accumulator per row +template +ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd( + const Policy & policy, const float * y, + const int ncols, const int nrows, + sycl::local_accessor lmem, + const sycl::nd_item<1> & it) { + using namespace sycl::ext::intel::esimd; + using traits = ggml_sycl_esimd::esimd_reorder_q_traits; + + const int num_blocks_per_row = ncols / QK_K; + const size_t nb = (size_t) nrows * num_blocks_per_row; + const auto pa = traits::make_ptrs(policy.w_a(), nb); + const auto pb = traits::make_ptrs(policy.w_b(), nb); + + const int tid = it.get_local_id(0); + const int g = it.get_group(0); + const int row_a = policy.row_a(g); + const int row_b = policy.row_b(g); + const bool has_b = policy.has_b(g, nrows); + + // one 32-wide accumulator per row (small footprint, no spill) + simd acc_a = 0.0f; + simd acc_b = 0.0f; + + for (int ib = tid; ib < num_blocks_per_row; ib += GGML_SYCL_DMMV_ESIMD_WG_SIZE) { + simd y_vec = block_load(y + (size_t) ib * QK_K); + + const size_t bi_a = (size_t) row_a * num_blocks_per_row + ib; + const size_t bi_b = (size_t) row_b * num_blocks_per_row + ib; + + traits::mac_pair(pa, bi_a, pb, bi_b, has_b, y_vec, acc_a, acc_b); + } + + lmem[tid * 2 + 0] = reduce(acc_a, std::plus<>{}); + lmem[tid * 2 + 1] = reduce(acc_b, std::plus<>{}); + it.barrier(sycl::access::fence_space::local_space); + + if (tid == 0) { + float sum_a = 0.0f; + float sum_b = 0.0f; + for (int p = 0; p < GGML_SYCL_DMMV_ESIMD_WG_SIZE; ++p) { + sum_a += lmem[p * 2 + 0]; + sum_b += lmem[p * 2 + 1]; + } + + policy.epilogue(g, nrows, sum_a, sum_b); + } +} + +static void dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int workgroups = (nrows + 1) / 2; + stream->submit([&](sycl::handler &h) { + sycl::local_accessor lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_reorder_esimd( + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); + }); + }); +} + +static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int workgroups = (nrows + 1) / 2; + stream->submit([&](sycl::handler &h) { + sycl::local_accessor lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_reorder_esimd( + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); + }); + }); +} + +static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int workgroups = (nrows + 1) / 2; + stream->submit([&](sycl::handler &h) { + sycl::local_accessor lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_reorder_esimd( + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); + }); + }); +} + +static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int workgroups = (nrows + 1) / 2; + stream->submit([&](sycl::handler &h) { + sycl::local_accessor lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_reorder_esimd( + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); + }); + }); +} + +static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int workgroups = (nrows + 1) / 2; + stream->submit([&](sycl::handler &h) { + sycl::local_accessor lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_reorder_esimd( + esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it); + }); + }); +} + +// Q8_0 SOA reorder layout: [qs: nb*QK8_0] [d: nb*sizeof(half)]. +// Process eight blocks per stripe and process remaining blocks one at a time. +template +ESIMD_INLINE void q8_0_mac_stripe( + const int8_t * qs_a, const int8_t * qs_b, + const sycl::half * d_a, const sycl::half * d_b, bool has_b, + sycl::ext::intel::esimd::simd & y_vec, + sycl::ext::intel::esimd::simd & acc_a, + sycl::ext::intel::esimd::simd & acc_b) { + using namespace sycl::ext::intel::esimd; + + simd qa = block_load(qs_a); + simd qb = 0; + // Scale rows can be only 2-byte aligned when nblk_row is odd. + simd da = block_load(d_a, element_aligned_tag{}); + simd db = 0; + if (has_b) { + qb = block_load(qs_b); + db = block_load(d_b, element_aligned_tag{}); + } + + simd da_f = convert(da); + simd db_f = convert(db); + +#pragma unroll + for (int s = 0; s < NBLK; ++s) { + simd y_s = y_vec.template select<32, 1>(s * 32); + simd qa_s = qa.template select<32, 1>(s * 32); + simd qb_s = qb.template select<32, 1>(s * 32); + const float sa = da_f[s]; + const float sb = db_f[s]; + acc_a += y_s * (convert(qa_s) * sa); + acc_b += y_s * (convert(qb_s) * sb); + } +} + +template +ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd( + const Policy & policy, const float * y, + const int ncols, const int nrows, + sycl::local_accessor lmem, + const sycl::nd_item<1> & it) { + using namespace sycl::ext::intel::esimd; + + constexpr int STRIPE = 8; + + const int nblk_row = ncols / QK8_0; + const size_t nb = (size_t) nrows * nblk_row; + const int8_t * qs_a = (const int8_t *) policy.w_a(); + const int8_t * qs_b = (const int8_t *) policy.w_b(); + const sycl::half * d_a = (const sycl::half *) (qs_a + nb * QK8_0); + const sycl::half * d_b = (const sycl::half *) (qs_b + nb * QK8_0); + + const int tid = it.get_local_id(0); + const int g = it.get_group(0); + const bool has_b = policy.has_b(g, nrows); + + const size_t base_a = (size_t) policy.row_a(g) * nblk_row; + const size_t base_b = (size_t) policy.row_b(g) * nblk_row; + + simd acc_a = 0.0f; + simd acc_b = 0.0f; + + // Each thread processes one contiguous stripe. + int ib = 0; + for (; ib + WG * STRIPE <= nblk_row; ib += WG * STRIPE) { + const int b = ib + tid * STRIPE; + simd y_vec = block_load(y + (size_t) b * QK8_0); + q8_0_mac_stripe(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0, + d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b); + } + + // Distribute remaining blocks across the work-group. + for (int b = ib + tid; b < nblk_row; b += WG) { + simd y_vec = block_load(y + (size_t) b * QK8_0); + q8_0_mac_stripe<1>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0, + d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b); + } + + lmem[tid * 2 + 0] = reduce(acc_a, std::plus<>{}); + lmem[tid * 2 + 1] = reduce(acc_b, std::plus<>{}); + it.barrier(sycl::access::fence_space::local_space); + + if (tid == 0) { + float sum_a = 0.0f; + float sum_b = 0.0f; + for (int p = 0; p < WG; ++p) { + sum_a += lmem[p * 2 + 0]; + sum_b += lmem[p * 2 + 1]; + } + + policy.epilogue(g, nrows, sum_a, sum_b); + } +} + +template +static void q8_0_esimd_launch(const Policy & policy, const float * y, const int ncols, const int nrows, + const int workgroups, dpct::queue_ptr stream) { + stream->submit([&](sycl::handler & h) { + sycl::local_accessor lmem(sycl::range<1>(WG * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * WG), sycl::range<1>(WG)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_q8_0_reorder_esimd(policy, y, ncols, nrows, lmem, it); + }); + }); +} + +// Scale the work-group with the number of blocks per row. +template +static void q8_0_esimd_launch_scaled(const Policy & policy, const float * y, const int ncols, const int nrows, + const int workgroups, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK8_0 == 0); + + const int nblk_row = ncols / QK8_0; + if (nblk_row >= 64) { + q8_0_esimd_launch<8>(policy, y, ncols, nrows, workgroups, stream); + } else if (nblk_row >= 32) { + q8_0_esimd_launch<4>(policy, y, ncols, nrows, workgroups, stream); + } else if (nblk_row >= 16) { + q8_0_esimd_launch<2>(policy, y, ncols, nrows, workgroups, stream); + } else { + q8_0_esimd_launch<1>(policy, y, ncols, nrows, workgroups, stream); + } +} + +static void dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + q8_0_esimd_launch_scaled(esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, (nrows + 1) / 2, stream); +} + +// ESIMD fused gate+up+GLU on reordered K-quant SOA weights: one output row (gate +// and up) per work-group, two independent 32-wide accumulators per super-block. +// Reuses the DMMV kernels' dequant+MAC primitive (esimd_reorder_q_traits::mac_pair). +template +static void dequantize_mul_mat_vec_glu_reorder_esimd(const void * vx, const void * vgate, + const float * y, float * dst, + const int ncols, const int nrows, + const ggml_glu_op glu_op, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int workgroups = nrows; // one output row (gate+up) per work-group + stream->submit([&](sycl::handler & h) { + sycl::local_accessor lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h); + h.parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)), + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + dequantize_mul_mat_vec_reorder_esimd( + esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, lmem, it); + }); + }); +} + +bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(enum ggml_type src0_type, enum ggml_glu_op glu_op, + const void * vx, const void * vgate, const float * y, + float * dst, int ncols, int nrows, dpct::queue_ptr stream) { + switch (src0_type) { + case GGML_TYPE_Q2_K: + dequantize_mul_mat_vec_glu_reorder_esimd(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q3_K: + dequantize_mul_mat_vec_glu_reorder_esimd(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q4_K: + dequantize_mul_mat_vec_glu_reorder_esimd(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q5_K: + dequantize_mul_mat_vec_glu_reorder_esimd(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q6_K: + dequantize_mul_mat_vec_glu_reorder_esimd(vx, vgate, y, dst, ncols, nrows, glu_op, stream); + return true; + case GGML_TYPE_Q8_0: + q8_0_esimd_launch_scaled(esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, nrows, stream); + return true; + default: + return false; + } +} + +#endif // GGML_SYCL_DMMV_HAS_ESIMD + static void dequantize_mul_mat_vec_q4_K_sycl_reorder(const void *vx, const float *y, float *dst, const int ncols, const int nrows, @@ -1454,14 +2256,30 @@ static void dequantize_mul_mat_vec_q4_K_sycl_reorder(const void *vx, const float const int ny = 2 / K_QUANTS_PER_ITERATION; const int block_num_y = (nrows + ny - 1) / ny; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec_q4_k_reorder(vx, y, dst, ncols, nrows, item_ct1); }); } +static void dequantize_mul_mat_vec_q5_K_sycl_reorder(const void *vx, const float *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int ny = 2 / K_QUANTS_PER_ITERATION; + const int block_num_y = (nrows + ny - 1) / ny; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec_q5_k_reorder(vx, y, dst, ncols, nrows, item_ct1); + }); +} + static void dequantize_mul_mat_vec_q6_K_sycl_reorder(const void *vx, const float *y, float *dst, const int ncols, const int nrows, @@ -1470,10 +2288,10 @@ static void dequantize_mul_mat_vec_q6_K_sycl_reorder(const void *vx, const float const int ny = 2 / K_QUANTS_PER_ITERATION; const int block_num_y = (nrows + ny - 1) / ny; const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, ny, QK_WARP_SIZE); + const sycl::range<3> block_dims(1, ny, WARP_SIZE); stream->parallel_for( sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(QK_WARP_SIZE)]] { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { dequantize_mul_mat_vec_q6_k_reorder(vx, y, dst, ncols, nrows, item_ct1); }); } @@ -1494,10 +2312,21 @@ void ggml_sycl_op_dequantize_mul_mat_vec( ggml_sycl_pool_alloc src1_dfloat_a(ctx.pool()); sycl::half *src1_dfloat = nullptr; // dfloat == half +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + // The ESIMD Q8_0 kernel reads F32 activations. + const bool q8_0_esimd = src0->type == GGML_TYPE_Q8_0 && g_ggml_sycl_enable_esimd && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra) && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder; +#else + const bool q8_0_esimd = false; +#endif + bool src1_convert_f16 = + src0->type == GGML_TYPE_Q1_0 || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 || src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 || - src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16; + (src0->type == GGML_TYPE_Q8_0 && !q8_0_esimd) || src0->type == GGML_TYPE_F16 || + src0->type == GGML_TYPE_BF16; if (src1_convert_f16) { scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2, @@ -1512,6 +2341,14 @@ void ggml_sycl_op_dequantize_mul_mat_vec( #endif // GGML_SYCL_F16 switch (src0->type) { + case GGML_TYPE_Q1_0: + if ((ggml_tensor_extra_gpu*)dst->src[0]->extra && + ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { + dequantize_mul_mat_vec_q1_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } else { + dequantize_mul_mat_vec_q1_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } + break; case GGML_TYPE_Q4_0: if ((ggml_tensor_extra_gpu*)dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { @@ -1532,32 +2369,94 @@ void ggml_sycl_op_dequantize_mul_mat_vec( case GGML_TYPE_Q8_0: if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - dequantize_mul_mat_vec_q8_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + if (g_ggml_sycl_enable_esimd) { + dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } else +#endif + { + dequantize_mul_mat_vec_q8_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } } else { dequantize_mul_mat_vec_q8_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_Q2_K: - dequantize_mul_mat_vec_q2_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + if (g_ggml_sycl_enable_esimd) { + dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + else +#endif + { + dequantize_mul_mat_vec_q2_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + } else { + dequantize_mul_mat_vec_q2_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } break; case GGML_TYPE_Q3_K: - dequantize_mul_mat_vec_q3_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + if (g_ggml_sycl_enable_esimd) { + dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + else +#endif + { + dequantize_mul_mat_vec_q3_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + } else { + dequantize_mul_mat_vec_q3_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } break; case GGML_TYPE_Q4_K: if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - dequantize_mul_mat_vec_q4_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + if (g_ggml_sycl_enable_esimd) { + dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + else +#endif + { + dequantize_mul_mat_vec_q4_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } } else { dequantize_mul_mat_vec_q4_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } break; case GGML_TYPE_Q5_K: - dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + if (g_ggml_sycl_enable_esimd) { + dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + else +#endif + { + dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + } else { + dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } break; case GGML_TYPE_Q6_K: if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { - dequantize_mul_mat_vec_q6_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + if (g_ggml_sycl_enable_esimd) { + dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } + else +#endif + { + dequantize_mul_mat_vec_q6_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); + } } else { dequantize_mul_mat_vec_q6_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream); } @@ -1565,6 +2464,11 @@ void ggml_sycl_op_dequantize_mul_mat_vec( case GGML_TYPE_F16: convert_mul_mat_vec_f16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); break; +#ifdef GGML_SYCL_DMMV_HAS_BF16 + case GGML_TYPE_BF16: + convert_mul_mat_vec_bf16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + break; +#endif default: printf("ggml_sycl_op_dequantize_mul_mat_vec unsupported GGML_TYPE %d\n", src0->type); GGML_ABORT("fatal error"); diff --git a/ggml/src/ggml-sycl/dmmv.hpp b/ggml/src/ggml-sycl/dmmv.hpp index bd83735..318e348 100644 --- a/ggml/src/ggml-sycl/dmmv.hpp +++ b/ggml/src/ggml-sycl/dmmv.hpp @@ -24,4 +24,13 @@ void ggml_sycl_op_dequantize_mul_mat_vec( const int64_t src1_ncols, const int64_t src1_padded_row_size, const dpct::queue_ptr &stream); +// Fused dense-FFN mat-vec: writes glu(gate . y, up . y) for same-type reordered +// weights (Q2_K..Q6_K, Q8_0) and a single F32 activation column, using the ESIMD DMMV kernels. +// vx is the up weight, vgate the gate weight. Only defined when GGML_SYCL_DMMV_HAS_ESIMD; +// returns false if the type is unhandled. +bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd( + enum ggml_type src0_type, enum ggml_glu_op glu_op, + const void * vx, const void * vgate, const float * y, + float * dst, int ncols, int nrows, dpct::queue_ptr stream); + #endif // GGML_SYCL_DMMV_HPP diff --git a/ggml/src/ggml-sycl/dpct/helper.hpp b/ggml/src/ggml-sycl/dpct/helper.hpp index 791d3ca..fd3b965 100644 --- a/ggml/src/ggml-sycl/dpct/helper.hpp +++ b/ggml/src/ggml-sycl/dpct/helper.hpp @@ -13,14 +13,14 @@ #ifndef GGML_SYCL_DPCT_HELPER_HPP #define GGML_SYCL_DPCT_HELPER_HPP +#include +#include +#include + #include #include #include -#include - -#include "ggml.h" - #if defined(__linux__) #include #elif defined(_WIN64) @@ -43,6 +43,7 @@ #include #endif + #define DPCT_COMPATIBILITY_TEMP (900) #if defined(_MSC_VER) @@ -59,6 +60,13 @@ #define __dpct_noinline__ __attribute__((noinline)) #endif +#define DPCT_UNUSED(x) (void)(x) + +[[noreturn]] inline void _abort(const char * str) { + std::cerr << str << std::endl; + std::abort(); +} + inline std::string get_device_type_name(const sycl::device &Device) { auto DeviceType = Device.get_info(); switch (DeviceType) { @@ -1017,7 +1025,7 @@ namespace dpct if (backend == "opencl:cpu") return 4; if (backend == "opencl:acc") return 5; printf("convert_backend_index: can't handle backend=%s\n", backend.c_str()); - GGML_ABORT("fatal error"); + _abort("fatal error"); } static bool compare_backend(std::string &backend1, std::string &backend2) { return convert_backend_index(backend1) < convert_backend_index(backend2); @@ -1426,7 +1434,7 @@ namespace dpct if (!size) return sycl::event{}; return q.memcpy(to_ptr, from_ptr, size, dep_events); - GGML_UNUSED(direction); + DPCT_UNUSED(direction); } // Get actual copy range and make sure it will not exceed range. @@ -2092,7 +2100,7 @@ namespace dpct if (!size) return sycl::event{}; return q.memcpy(to_ptr, from_ptr, size, dep_events); - GGML_UNUSED(direction); + DPCT_UNUSED(direction); } // Get actual copy range and make sure it will not exceed range. @@ -3151,7 +3159,7 @@ namespace dpct _local_mem_size = local_mem_size; - }; + } static inline std::mutex kernel_function_ptr_map_mutex; public: diff --git a/ggml/src/ggml-sycl/dsv4-hc.cpp b/ggml/src/ggml-sycl/dsv4-hc.cpp new file mode 100644 index 0000000..337f4af --- /dev/null +++ b/ggml/src/ggml-sycl/dsv4-hc.cpp @@ -0,0 +1,338 @@ +#include "ggml-impl.h" +#include "dsv4-hc.hpp" + +#include +#include + +static constexpr int DSV4_HC = 4; + +// tunable: one work-item per (embedding element, token) +static constexpr int dsv4_hc_pre_block_size = 256; + +// gated: the weight is a per-element gate [n_embd, hc, n_tokens] passed through a sigmoid. +// otherwise it is one weight per (stream, token). +template +static void dsv4_hc_pre_f32_sycl( + const float * x, const float * weights, float * dst, + int64_t n_embd, int64_t hc, int64_t n_tokens, + int64_t sx0, int64_t sx1, int64_t sx2, + int64_t sw0, int64_t sw1, int64_t sw2, + int64_t sd0, int64_t sd1, + float scale, + queue_ptr stream) { + const int64_t nr = n_embd * n_tokens; + const int64_t num_blocks = (nr + dsv4_hc_pre_block_size - 1) / dsv4_hc_pre_block_size; + + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(num_blocks * dsv4_hc_pre_block_size), + sycl::range<1>(dsv4_hc_pre_block_size)), + [=](sycl::nd_item<1> item) { + const int64_t ir = item.get_global_id(0); + if (ir >= nr) { + return; + } + + const int64_t i0 = ir % n_embd; + const int64_t it = ir / n_embd; + + float sum = 0.0f; + for (int64_t ih = 0; ih < hc; ++ih) { + const float xv = x[i0*sx0 + ih*sx1 + it*sx2]; + float wv; + if constexpr (gated) { + const float gv = weights[i0*sw0 + ih*sw1 + it*sw2]; + wv = 1.0f / (1.0f + sycl::exp(-gv)); + } else { + wv = weights[ih*sw0 + it*sw1]; + } + sum += xv * wv; + } + + dst[i0*sd0 + it*sd1] = scale * sum; + }); +} + +static void dsv4_hc_comb_norm_cols(float * comb, float eps) { + for (int idst = 0; idst < DSV4_HC; ++idst) { + float sum = eps; + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + sum += comb[idst + DSV4_HC*isrc]; + } + + const float inv_sum = 1.0f / sum; + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + comb[idst + DSV4_HC*isrc] *= inv_sum; + } + } +} + +static void dsv4_hc_comb_norm_rows(float * comb, float eps) { + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + float sum = eps; + for (int idst = 0; idst < DSV4_HC; ++idst) { + sum += comb[idst + DSV4_HC*isrc]; + } + + const float inv_sum = 1.0f / sum; + for (int idst = 0; idst < DSV4_HC; ++idst) { + comb[idst + DSV4_HC*isrc] *= inv_sum; + } + } +} + +static void dsv4_hc_comb_f32_sycl( + const float * mixes, + const float * scale, + const float * base, + float * dst, + int64_t n_tokens, + int64_t sm0, + int64_t sm1, + int64_t ss0, + int64_t sb0, + int64_t sd0, + int64_t sd1, + int64_t sd2, + float eps, + int32_t n_iter, + queue_ptr stream) { + constexpr int comb_offset = 2*DSV4_HC; + + const int64_t block_size = 256; + const int64_t num_blocks = (n_tokens + block_size - 1) / block_size; + + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(num_blocks * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item_ct1) { + const int64_t it = item_ct1.get_global_id(0); + + if (it >= n_tokens) { + return; + } + + const float scale_comb = scale[2*ss0]; + float comb[DSV4_HC*DSV4_HC]; + + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + float max = -INFINITY; + for (int idst = 0; idst < DSV4_HC; ++idst) { + const int idx = idst + DSV4_HC*isrc; + const float v = mixes[(comb_offset + idx)*sm0 + it*sm1] * scale_comb + base[(comb_offset + idx)*sb0]; + comb[idx] = v; + max = fmaxf(max, v); + } + + float sum = 0.0f; + for (int idst = 0; idst < DSV4_HC; ++idst) { + const int idx = idst + DSV4_HC*isrc; + const float v = expf(comb[idx] - max); + comb[idx] = v; + sum += v; + } + + const float inv_sum = 1.0f / sum; + for (int idst = 0; idst < DSV4_HC; ++idst) { + const int idx = idst + DSV4_HC*isrc; + comb[idx] = comb[idx] * inv_sum + eps; + } + } + + dsv4_hc_comb_norm_cols(comb, eps); + for (int32_t i = 1; i < n_iter; ++i) { + dsv4_hc_comb_norm_rows(comb, eps); + dsv4_hc_comb_norm_cols(comb, eps); + } + + for (int isrc = 0; isrc < DSV4_HC; ++isrc) { + for (int idst = 0; idst < DSV4_HC; ++idst) { + const int idx = idst + DSV4_HC*isrc; + dst[idst*sd0 + isrc*sd1 + it*sd2] = comb[idx]; + } + } + }); +} + +// tunable: one work-item per (embedding element, stream, token) +static constexpr int dsv4_hc_post_block_size = 256; + +// comb == nullptr is identity mixing: each destination stream keeps its own residual +// instead of summing across the streams. +template +static void dsv4_hc_post_f32_sycl( + const float * x, const float * residual, const float * post, const float * comb, float * dst, + int64_t n_embd, int64_t hc, int64_t n_tokens, + int64_t sx0, int64_t sx1, + int64_t sr0, int64_t sr1, int64_t sr2, + int64_t sp0, int64_t sp1, + int64_t sc0, int64_t sc1, int64_t sc2, + int64_t sd0, int64_t sd1, int64_t sd2, + queue_ptr stream) { + const int64_t nr = n_embd * hc * n_tokens; + const int64_t block_size = dsv4_hc_post_block_size; + const int64_t num_blocks = (nr + block_size - 1) / block_size; + + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(num_blocks * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item) { + const int64_t ir = item.get_global_id(0); + if (ir >= nr) { + return; + } + + const int64_t i0 = ir % n_embd; + const int64_t idst = (ir / n_embd) % hc; + const int64_t it = ir / (n_embd * hc); + + float sum = x[i0*sx0 + it*sx1] * post[idst*sp0 + it*sp1]; + if constexpr (has_comb) { + for (int64_t isrc = 0; isrc < hc; ++isrc) { + sum += residual[i0*sr0 + isrc*sr1 + it*sr2] * comb[idst*sc0 + isrc*sc1 + it*sc2]; + } + } else { + sum += residual[i0*sr0 + idst*sr1 + it*sr2]; + } + + dst[i0*sd0 + idst*sd1 + it*sd2] = sum; + }); +} + +void ggml_sycl_op_dsv4_hc_pre(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + const ggml_tensor * x = dst->src[0]; + const ggml_tensor * weights = dst->src[1]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(weights->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_TENSOR_LOCALS(size_t, nbx, x, nb); + GGML_TENSOR_LOCALS(size_t, nbw, weights, nb); + GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const int64_t n_embd = x->ne[0]; + const int64_t hc = x->ne[1]; + const int64_t n_tokens = x->ne[2]; + + const float scale = ggml_get_op_params_f32(dst, 0); + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + + queue_ptr stream = ctx.stream(); + + if (gated) { + GGML_ASSERT(weights->ne[0] == n_embd); + GGML_ASSERT(weights->ne[1] == hc); + GGML_ASSERT(weights->ne[2] == n_tokens); + dsv4_hc_pre_f32_sycl( + (const float *) x->data, (const float *) weights->data, (float *) dst->data, + n_embd, hc, n_tokens, + nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float), + nbw0 / sizeof(float), nbw1 / sizeof(float), nbw2 / sizeof(float), + nbd0 / sizeof(float), nbd1 / sizeof(float), + scale, stream); + } else { + GGML_ASSERT(weights->ne[0] == hc); + GGML_ASSERT(weights->ne[1] == n_tokens); + dsv4_hc_pre_f32_sycl( + (const float *) x->data, (const float *) weights->data, (float *) dst->data, + n_embd, hc, n_tokens, + nbx0 / sizeof(float), nbx1 / sizeof(float), nbx2 / sizeof(float), + nbw0 / sizeof(float), nbw1 / sizeof(float), /*sw2=*/ 0, + nbd0 / sizeof(float), nbd1 / sizeof(float), + scale, stream); + } +} + +void ggml_sycl_op_dsv4_hc_comb(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3); + + const ggml_tensor * mixes = dst->src[0]; + const ggml_tensor * scale = dst->src[1]; + const ggml_tensor * base = dst->src[2]; + + GGML_ASSERT(mixes->type == GGML_TYPE_F32); + GGML_ASSERT(scale->type == GGML_TYPE_F32); + GGML_ASSERT(base->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + constexpr int64_t hc_mix_dim = (2 + DSV4_HC)*DSV4_HC; + + GGML_ASSERT(mixes->ne[0] == hc_mix_dim); + GGML_ASSERT(dst->ne[0] == DSV4_HC); + GGML_ASSERT(dst->ne[1] == DSV4_HC); + GGML_ASSERT(dst->ne[2] == mixes->ne[1]); + GGML_ASSERT(scale->ne[0] >= 3); + GGML_ASSERT(base->ne[0] == hc_mix_dim); + + GGML_TENSOR_LOCALS(size_t, nbm, mixes, nb); + GGML_TENSOR_LOCALS(size_t, nbs, scale, nb); + GGML_TENSOR_LOCALS(size_t, nbb, base, nb); + GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + const int64_t n_tokens = mixes->ne[1]; + const float eps = ggml_get_op_params_f32(dst, 0); + const int32_t n_iter = ggml_get_op_params_i32(dst, 1); + + queue_ptr stream = ctx.stream(); + + dsv4_hc_comb_f32_sycl( + (const float *) mixes->data, (const float *) scale->data, (const float *) base->data, (float *) dst->data, + n_tokens, + nbm0 / sizeof(float), nbm1 / sizeof(float), + nbs0 / sizeof(float), + nbb0 / sizeof(float), + nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float), + eps, n_iter, stream); +} + +void ggml_sycl_op_dsv4_hc_post(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/4); + const ggml_tensor * x = dst->src[0]; + const ggml_tensor * residual = dst->src[1]; + const ggml_tensor * post = dst->src[2]; + const ggml_tensor * comb = dst->src[3]; + + GGML_ASSERT(x->type == GGML_TYPE_F32); + GGML_ASSERT(residual->type == GGML_TYPE_F32); + GGML_ASSERT(post->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_TENSOR_LOCALS(size_t, nbx, x, nb); + GGML_TENSOR_LOCALS(size_t, nbr, residual, nb); + GGML_TENSOR_LOCALS(size_t, nbp, post, nb); + GGML_TENSOR_LOCALS(size_t, nbd, dst, nb); + + size_t nbc0 = 0; + size_t nbc1 = 0; + size_t nbc2 = 0; + if (comb) { + GGML_ASSERT(comb->type == GGML_TYPE_F32); + nbc0 = comb->nb[0]; + nbc1 = comb->nb[1]; + nbc2 = comb->nb[2]; + } + + const int64_t n_embd = x->ne[0]; + const int64_t n_tokens = x->ne[1]; + const int64_t hc = residual->ne[1]; + + queue_ptr stream = ctx.stream(); + + const auto launch = [&](auto has_comb) { + dsv4_hc_post_f32_sycl( + (const float *) x->data, (const float *) residual->data, + (const float *) post->data, comb ? (const float *) comb->data : nullptr, (float *) dst->data, + n_embd, hc, n_tokens, + nbx0 / sizeof(float), nbx1 / sizeof(float), + nbr0 / sizeof(float), nbr1 / sizeof(float), nbr2 / sizeof(float), + nbp0 / sizeof(float), nbp1 / sizeof(float), + nbc0 / sizeof(float), nbc1 / sizeof(float), nbc2 / sizeof(float), + nbd0 / sizeof(float), nbd1 / sizeof(float), nbd2 / sizeof(float), + stream); + }; + + if (comb) { + launch(std::true_type{}); + } else { + launch(std::false_type{}); + } +} diff --git a/ggml/src/ggml-sycl/dsv4-hc.hpp b/ggml/src/ggml-sycl/dsv4-hc.hpp new file mode 100644 index 0000000..330518d --- /dev/null +++ b/ggml/src/ggml-sycl/dsv4-hc.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_DSV4_HC_HPP +#define GGML_SYCL_DSV4_HC_HPP + +#include "common.hpp" + +void ggml_sycl_op_dsv4_hc_pre(ggml_backend_sycl_context & ctx, ggml_tensor * dst); +void ggml_sycl_op_dsv4_hc_comb(ggml_backend_sycl_context & ctx, ggml_tensor * dst); +void ggml_sycl_op_dsv4_hc_post(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_DSV4_HC_HPP diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cpp index 249e80c..c7fe70a 100644 --- a/ggml/src/ggml-sycl/element_wise.cpp +++ b/ggml/src/ggml-sycl/element_wise.cpp @@ -9,9 +9,12 @@ #define SYCL_LOCAL_ID_CALC(ITEM, IDX) \ (ITEM.get_local_range(IDX) * ITEM.get_group(IDX) + ITEM.get_local_id(IDX)) -static void acc_f32(const float * x, const float * y, float * dst, const int64_t ne, - const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, - const int64_t s11, const int64_t s12, const int64_t s13, const int64_t offset) { +static void acc_f32(const char * x, const char * y, float * dst, const int64_t ne, + const int64_t ne0, const int64_t ne1, const int64_t ne2, + const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, + const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, + const int64_t s11, const int64_t s12, const int64_t s13, const int64_t offset) { auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>(); const int64_t i = SYCL_LOCAL_ID_CALC(item_ct1, 2); @@ -30,9 +33,18 @@ static void acc_f32(const float * x, const float * y, float * dst, const int64_t tmp -= i11 * s11; const int64_t i10 = tmp; - float val = x[i]; + int64_t tmp_dst = i; + const int64_t i3 = tmp_dst / (ne2*ne1*ne0); + tmp_dst -= i3 * (ne2*ne1*ne0); + const int64_t i2 = tmp_dst / (ne1*ne0); + tmp_dst -= i2 * (ne1*ne0); + const int64_t i1 = tmp_dst / ne0; + tmp_dst -= i1 * ne0; + const int64_t i0 = tmp_dst; + + float val = *(const float *) (x + i0*nb00 + i1*nb01 + i2*nb02 + i3*nb03); if (src1_idx >= 0 && i10 < ne10 && i11 < ne11 && i12 < ne12 && i13 < ne13) { - val += y[((i13*ne12 + i12) * ne11 + i11) * ne10 + i10]; + val += *(const float *) (y + i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13); } dst[i] = val; } @@ -43,85 +55,115 @@ static __dpct_inline__ T op_sgn(T x) { return x > static_cast(0.f) ? static_cast(1.f) : ((x < static_cast(0.f) ? static_cast(-1.f) : static_cast(0.f))); } -template -static __dpct_inline__ T op_abs(T x) { - return sycl::fabs(x); -} template -static __dpct_inline__ T op_elu(T x) { - return (x > static_cast(0.f)) ? x : sycl::expm1(x); +static __dpct_inline__ T op_abs(T x) { + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::fabs(x); // or experimental namespace if needed + } else { + return sycl::fabs(x); + } } template -static __dpct_inline__ T op_gelu(T x) { - const T GELU_COEF_A = static_cast(0.044715f); - const T SQRT_2_OVER_PI = static_cast(0.79788456080286535587989211986876f); - return static_cast(0.5f) * x * - (static_cast(1.0f) + - sycl::tanh(SQRT_2_OVER_PI * x * (static_cast(1.0f) + GELU_COEF_A * x * x))); +static __dpct_inline__ T op_expm1(T x) { + if constexpr (std::is_same_v) { + return static_cast( + sycl::expm1(static_cast(x)) + ); + } else { + return sycl::expm1(x); + } } template -static __dpct_inline__ T op_silu(T x) { - return x / (static_cast(1.0f) + sycl::native::exp(-x)); +static __dpct_inline__ T op_elu(T x) { + return (x > static_cast(0.f)) ? x : op_expm1(x); } template -static __dpct_inline__ T op_gelu_quick(T x) { - const T GELU_QUICK_COEF_LOCAL = static_cast(-1.702f); - return x * (static_cast(1.0f) / (static_cast(1.0f) + sycl::native::exp(GELU_QUICK_COEF_LOCAL * x))); +static __dpct_inline__ T op_erf(T x) { + if constexpr (std::is_same_v) { + return static_cast( + sycl::erf(static_cast(x)) + ); + } else { + return sycl::erf(x); + } } template static __dpct_inline__ T op_gelu_erf(T x) { const T SQRT_2_INV = static_cast(0.70710678118654752440084436210484f); - return static_cast(0.5f) * x * (static_cast(1.0f) + sycl::erf(x * SQRT_2_INV)); + return static_cast(0.5f) * x * (static_cast(1.0f) + op_erf(x * SQRT_2_INV)); } template -static __dpct_inline__ T op_tanh(T x) { - return sycl::tanh(x); +static __dpct_inline__ T op_gelu_quick(T x) { + const T GELU_QUICK_COEF_LOCAL = static_cast(-1.702f); + return x * (static_cast(1.0f) / (static_cast(1.0f) + op_exp(GELU_QUICK_COEF_LOCAL * x))); } template static __dpct_inline__ T op_relu(T x) { - return sycl::fmax(x, static_cast(0)); + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::fmax(x, static_cast(0)); + } else { + return sycl::fmax(x, static_cast(0)); + } } template static __dpct_inline__ T op_sigmoid(T x) { - return static_cast(1.0f) / (static_cast(1.0f) + sycl::native::exp(-x)); + return static_cast(1.0f) / (static_cast(1.0f) + op_exp(-x)); } template static __dpct_inline__ T op_sqrt(T x) { - return sycl::sqrt(x); + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::sqrt(x); + } else { + return sycl::sqrt(x); + } } template static __dpct_inline__ T op_sin(T x) { - return sycl::sin(x); + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::sin(x); + } else { + return sycl::sin(x); + } } template static __dpct_inline__ T op_cos(T x) { - return sycl::cos(x); + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::cos(x); + } else { + return sycl::cos(x); + } } template static __dpct_inline__ T op_hardsigmoid(T x) { - return sycl::fmin(static_cast(1.0f), sycl::fmax(static_cast(0.0f), (x + static_cast(3.0f)) / static_cast(6.0f))); + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::fmin( + static_cast(1.0f), sycl::ext::oneapi::experimental::fmax( + static_cast(0.0f), (x + static_cast(3.0f)) / static_cast(6.0f))); + } else { + return sycl::fmin(static_cast(1.0f), + sycl::fmax(static_cast(0.0f), (x + static_cast(3.0f)) / static_cast(6.0f))); + } } template static __dpct_inline__ T op_hardswish(T x) { - return x * sycl::fmin(static_cast(1.0f), sycl::fmax(static_cast(0.0f), (x + static_cast(3.0f)) / static_cast(6.0f))); -} - -template -static __dpct_inline__ T op_exp(T x) { - return sycl::exp(x); + if constexpr (std::is_same_v) { + return x * sycl::ext::oneapi::experimental::fmin(static_cast(1.0f), sycl::ext::oneapi::experimental::fmax(static_cast(0.0f), (x + static_cast(3.0f)) / static_cast(6.0f))); + } else { + return x * sycl::fmin(static_cast(1.0f), sycl::fmax(static_cast(0.0f), (x + static_cast(3.0f)) / static_cast(6.0f))); + } } template @@ -129,13 +171,17 @@ static __dpct_inline__ T op_log(T x) { if (x <= static_cast(0)) { return neg_infinity(); } - return sycl::log(x); + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::log(x); + } else { + return sycl::log(x); + } } template static __dpct_inline__ T op_softplus(T x) { const float xf = (float) x; - const float ax = sycl::fabs(xf); + const float ax = op_abs(xf); const float m = sycl::fmax(xf, 0.0f); const float y = m + sycl::log1p(sycl::exp(-ax)); return (T) y; @@ -154,8 +200,25 @@ static __dpct_inline__ T op_step(T x) { template static __dpct_inline__ T op_leaky_relu(T x, float negative_slope) { T neg_slope_T = static_cast(negative_slope); - return sycl::fmax(x, static_cast(0)) + + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::fmax(x, static_cast(0)) + + sycl::ext::oneapi::experimental::fmin(x, static_cast(0.0f)) * neg_slope_T; + + } else { + return sycl::fmax(x, static_cast(0)) + sycl::fmin(x, static_cast(0.0f)) * neg_slope_T; + } +} + +template +static __dpct_inline__ T op_xielu(T x, float alpha_n, float alpha_p, float beta, float eps) { + const float xi = static_cast(x); + const float gate_pos = (xi > 0.0f); + const float y_pos = alpha_p * xi * xi + beta * xi; + const float min_v_eps = sycl::fmin(xi, eps); + const float y_neg = (sycl::expm1(min_v_eps) - xi) * alpha_n + beta * xi; + const float out = gate_pos * y_pos + (1.0f - gate_pos) * y_neg; + return static_cast(out); } template @@ -170,22 +233,47 @@ static __dpct_inline__ T op_clamp(T x, float min_val, float max_val) { template static __dpct_inline__ T op_floor(T x) { - return sycl::floor(x); + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::floor(x); + } else { + return sycl::floor(x); + } } template static __dpct_inline__ T op_ceil(T x) { - return sycl::ceil(x); + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::ceil(x); + } else { + return sycl::ceil(x); + } } template static __dpct_inline__ T op_round(T x) { - return sycl::round(x); + if constexpr (std::is_same_v) { + return static_cast( + sycl::round(static_cast(x)) + ); + } else { + return sycl::round(x); + } } template static __dpct_inline__ T op_trunc(T x) { - return sycl::trunc(x); + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::trunc(x); + } else { + return sycl::trunc(x); + } +} + +template +static void unary_op_flat_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> & item_ct1, F func) { + SYCL_GLOBAL_ID_LOOP(k, item_ct1) { + dst[i] = func(x[i]); + } } template @@ -193,24 +281,31 @@ static void unary_op_generic_kernel( const T * x, T * dst, const int k, - const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, + const sycl::uint3 ne0_fd, const sycl::uint3 ne1_fd, const sycl::uint3 ne2_fd, const size_t nb0, const size_t nb1, const size_t nb2, const size_t nb3, const size_t nbd0, const size_t nbd1, const size_t nbd2, const size_t nbd3, const sycl::nd_item<1> & item_ct1, F func) { - (void) ne3; + // 32-bit index math: k is int, so every logical index fits u32. 64-bit integer div/mod is + // emulated on Xe and dominates this kernel otherwise, and even the 32-bit divide is worth + // avoiding -- the divisors are launch-invariant, so the magic numbers are precomputed + // host-side and each division becomes a multiply-high plus a shift. + // Byte offsets are widened back to size_t only for the final address math. SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t i0 = i % ne0; - const int64_t i1 = (i / ne0) % ne1; - const int64_t i2 = (i / (ne0*ne1)) % ne2; - const int64_t i3 = i / (ne0*ne1*ne2); + sycl::uint2 dm = fast_div_modulo((uint32_t) i, ne0_fd); + const uint32_t i0 = dm.y(); + dm = fast_div_modulo(dm.x(), ne1_fd); + const uint32_t i1 = dm.y(); + dm = fast_div_modulo(dm.x(), ne2_fd); + const uint32_t i2 = dm.y(); + const uint32_t i3 = dm.x(); const char * src_base = (const char *) x; char * dst_base = (char *) dst; - const T * srcp = (const T *)(src_base + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3 ); - T * dstp = (T *)(dst_base + i0*nbd0 + i1*nbd1 + i2*nbd2 + i3*nbd3); + const T * srcp = (const T *)(src_base + (size_t) i0*nb0 + (size_t) i1*nb1 + (size_t) i2*nb2 + (size_t) i3*nb3 ); + T * dstp = (T *)(dst_base + (size_t) i0*nbd0 + (size_t) i1*nbd1 + (size_t) i2*nbd2 + (size_t) i3*nbd3); *dstp = func(*srcp); } @@ -253,23 +348,23 @@ static void unary_op_leaky_relu_kernel(const T * x, T * dst, const int k, float } template -static void unary_op_sqr_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) { +static void unary_op_xielu_kernel(const T * x, T * dst, const int k, float alpha_n, float alpha_p, float beta, float eps, const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = op_sqr(x[i]); + dst[i] = op_xielu(x[i], alpha_n, alpha_p, beta, eps); } } template -static void unary_op_clamp_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1, float min_val, float max_val) { +static void unary_op_sqr_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = op_clamp(x[i], min_val, max_val); + dst[i] = op_sqr(x[i]); } } template -static void unary_op_floor_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) { +static void unary_op_clamp_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1, float min_val, float max_val) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = op_floor(x[i]); + dst[i] = op_clamp(x[i], min_val, max_val); } } @@ -281,82 +376,141 @@ static void unary_op_ceil_kernel(const T * x, T * dst, const int k, const sycl:: } template -static void unary_op_round_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) { +static void clamp(const T * x, T * dst, const float min, const float max, const int k, + const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = op_round(x[i]); + dst[i] = x[i] < static_cast(min) ? static_cast(min) : (x[i] > static_cast(max) ? static_cast(max) : x[i]); } } -template -static void unary_op_trunc_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> &item_ct1) { +template +static void unary_gated_op_flat_kernel(const T * x, const T * g, T * dst, const uint64_t k, const sycl::nd_item<1> & item_ct1, F func) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = op_trunc(x[i]); + dst[i] = func(x[i]) * g[i]; } } -template -static void clamp(const T * x, T * dst, const float min, const float max, const int k, - const sycl::nd_item<1> &item_ct1) { +template +static void unary_gated_op_generic_kernel( + const T * x, + const T * g, + T * dst, + const uint64_t k, + const sycl::uint3 n_fd, + const uint64_t o0, + const uint64_t o1, + const sycl::nd_item<1> & item_ct1, + F func) { + + // rows of n columns at strides o0 and o1: two halves of one fused tensor, or two tensors SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - dst[i] = x[i] < static_cast(min) ? static_cast(min) : (x[i] > static_cast(max) ? static_cast(max) : x[i]); + const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd); + const int64_t j0 = rc.x() * o0 + rc.y(); + const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y(); + dst[i] = func(x[j0]) * g[j1]; } } -template -static void gated_op_fused_geglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { +// Fused UNARY + MUL. Unlike the gated ops above, `x` and `g` are separate tensors of the +// same shape; `o0`/`o1` are their row strides in elements, so a half-view needs no repack. +// `dst` is contiguous and indexed flat. Math is done in f32, as the CPU and CUDA references do. +template +static void unary_mul_flat_kernel(const T * x, const T * g, T * dst, const int64_t k, const sycl::nd_item<1> &item_ct1, F op) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); - dst[i] = op_gelu(x[j0]) * g[j1]; + dst[i] = (T) (op((float) x[i]) * (float) g[i]); } } -template -static void gated_op_fused_reglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { +template +static void unary_mul_strided_kernel(const T * x, const T * g, T * dst, const int64_t k, const sycl::uint3 n_fd, const int64_t o0, const int64_t o1, const sycl::nd_item<1> &item_ct1, F op) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); - dst[i] = op_relu(x[j0]) * g[j1]; + const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd); + const int64_t j0 = rc.x() * o0 + rc.y(); + const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y(); + dst[i] = (T) (op((float) x[j0]) * (float) g[j1]); } } -template -static void gated_op_fused_swiglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { - SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); - dst[i] = op_silu(x[j0]) * g[j1]; +template +static void unary_mul_sycl(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, queue_ptr main_stream, F op) { + const size_t num_blocks = ceil_div((size_t) k, (size_t) SYCL_GLU_BLOCK_SIZE); + const sycl::nd_range<1> range(num_blocks * sycl::range<1>(SYCL_GLU_BLOCK_SIZE), sycl::range<1>(SYCL_GLU_BLOCK_SIZE)); + + // o0 == o1 == n makes (i/n)*o0 + (i%n) == i, so the strided kernel degenerates to the flat one + if (o0 == n && o1 == n) { + main_stream->parallel_for(range, [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + unary_mul_flat_kernel(x, g, dst, k, item_ct1, op); + }); + return; } + + // 32-bit fastdiv, exact only below 2^31; ggml_sycl_can_fuse() already declined past that + GGML_ASSERT(k < ((int64_t) 1 << 31)); + const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n); + main_stream->parallel_for(range, [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + unary_mul_strided_kernel(x, g, dst, k, n_fd, o0, o1, item_ct1, op); + }); } -template -static void gated_op_fused_geglu_erf(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { +// ADD(bias) + UNARY + MUL(scale) with both broadcast over dim 0, the delta-net alpha gate: +// dst[i] = op(a[i] + bias[i % ne0]) * scale[i % ne0]. k == ne0 makes that the flat index. +template +static void add_unary_mul_flat_kernel(const float * a, const float * bias, const float * scale, float * dst, + const int64_t k, const sycl::nd_item<1> &item_ct1, F op) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); - dst[i] = op_gelu_erf(x[j0]) * g[j1]; + dst[i] = op(a[i] + bias[i]) * scale[i]; } } -template -static void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { +template +static void add_unary_mul_bcast_kernel(const float * a, const float * bias, const float * scale, float * dst, + const int64_t k, const sycl::uint3 ne0_fd, const sycl::nd_item<1> &item_ct1, F op) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); - dst[i] = op_gelu_quick(x[j0]) * g[j1]; + const uint32_t h = fastmodulo((uint32_t) i, ne0_fd); + dst[i] = op(a[i] + bias[h]) * scale[h]; } } +template +static void add_unary_mul_sycl(const float * a, const float * bias, const float * scale, float * dst, + const int64_t k, const int64_t ne0, queue_ptr main_stream, F op) { + const size_t num_blocks = ceil_div((size_t) k, (size_t) SYCL_GLU_BLOCK_SIZE); + const sycl::nd_range<1> range(num_blocks * sycl::range<1>(SYCL_GLU_BLOCK_SIZE), sycl::range<1>(SYCL_GLU_BLOCK_SIZE)); + + if (k == ne0) { + main_stream->parallel_for(range, [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + add_unary_mul_flat_kernel(a, bias, scale, dst, k, item_ct1, op); + }); + return; + } + + // 32-bit fastdiv, exact only below 2^31; ggml_sycl_can_fuse() already declined past that + GGML_ASSERT(k < ((int64_t) 1 << 31)); + const sycl::uint3 ne0_fd = init_fastdiv_values((uint32_t) ne0); + main_stream->parallel_for(range, [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + add_unary_mul_bcast_kernel(a, bias, scale, dst, k, ne0_fd, item_ct1, op); + }); +} + namespace ggml_sycl_detail { -static void acc_f32_sycl(const float *x, const float *y, float *dst, - const int64_t n_elements, const int64_t ne10, const int64_t ne11, - const int64_t ne12, const int64_t ne13, const int64_t s1, const int64_t s2, const int64_t s3, +static void acc_f32_sycl(const char *x, const char *y, float *dst, + const int64_t n_elements, + const int64_t ne0, const int64_t ne1, const int64_t ne2, + const int64_t nb00, const int64_t nb01, const int64_t nb02, const int64_t nb03, + const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13, + const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, + const int64_t s1, const int64_t s2, const int64_t s3, const int64_t offset, queue_ptr stream) { const int num_blocks = (n_elements + SYCL_ACC_BLOCK_SIZE - 1) / SYCL_ACC_BLOCK_SIZE; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_ACC_BLOCK_SIZE)), - [=](sycl::nd_item<3> /*item_ct1*/) { - acc_f32(x, y, dst, n_elements, ne10, ne11, ne12, ne13, s1, s2, s3, offset); + [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + acc_f32(x, y, dst, n_elements, + ne0, ne1, ne2, + nb00, nb01, nb02, nb03, + ne10, ne11, ne12, ne13, + nb10, nb11, nb12, nb13, + s1, s2, s3, offset); }); } @@ -370,8 +524,8 @@ static void arange_kernel(T * dst, const int k, T start, T step, template static inline void dispatch_ggml_sycl_op_unary(ggml_backend_sycl_context & ctx, ggml_tensor * dst, KernelInvoker kernel_invoker, Args&&... args) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16); - GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); + GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16 || dst->src[0]->type == GGML_TYPE_BF16); + GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16); GGML_ASSERT(dst->src[0]->type == dst->type); dpct::queue_ptr main_stream = ctx.stream(); @@ -383,6 +537,14 @@ static inline void dispatch_ggml_sycl_op_unary(ggml_backend_sycl_context & ctx, kernel_invoker(data_pts.src, data_pts.dst, (int)ggml_nelements(dst->src[0]), main_stream, std::forward(args)...); break; } +#ifdef GGML_SYCL_HAS_BF16 + case GGML_TYPE_BF16: + { + auto data_pts = cast_data(dst); + kernel_invoker(data_pts.src, data_pts.dst, (int)ggml_nelements(dst->src[0]), main_stream, std::forward(args)...); + break; + } +#endif case GGML_TYPE_F32: { auto data_pts = cast_data(dst); @@ -408,6 +570,10 @@ static inline void dispatch_ggml_sycl_op_fused_glu(ggml_backend_sycl_context & c GGML_ASSERT(dst->ne[0] == nc); GGML_ASSERT(ggml_is_contiguous_1(dst->src[0])); GGML_ASSERT(ggml_is_contiguous(dst)); + // The fused GLU kernels index with 32-bit fastdiv, which is exact only for indices below + // 2^31. A dst that large is ~8 GB at f32, and the grid sizing already narrows to 32 bits, + // so assert the bound rather than carry a second code path for it. + GGML_ASSERT(ggml_nelements(dst) < ((int64_t) 1 << 31)); const int32_t swapped = ((const int32_t *) dst->op_params)[1]; void * src0_d = src0->data; void * src1_d = src1 ? src1->data : src0->data; @@ -476,7 +642,6 @@ static inline void ggml_sycl_op_unary( const int64_t ne0 = dst->ne[0]; const int64_t ne1 = dst->ne[1]; const int64_t ne2 = dst->ne[2]; - const int64_t ne3 = dst->ne[3]; const size_t nb0 = src0->nb[0]; const size_t nb1 = src0->nb[1]; @@ -488,24 +653,71 @@ static inline void ggml_sycl_op_unary( const size_t nbd2 = dst->nb[2]; const size_t nbd3 = dst->nb[3]; + // Hot unary ops (FFN/GDN silu, sigmoid, ...) run on contiguous tensors; + // skip the strided index math entirely for them. + const bool contiguous = ggml_is_contiguous(src0) && ggml_is_contiguous(dst); + ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst, [=](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) { const int num_blocks = ceil_div(k_elements, 256); - stream->parallel_for( - sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256), - sycl::range<1>(256)), - [=](sycl::nd_item<1> item_ct1) { - unary_op_generic_kernel( - src, dst_ptr, k_elements, - ne0, ne1, ne2, ne3, - nb0, nb1, nb2, nb3, - nbd0, nbd1, nbd2, nbd3, - item_ct1, - func - ); - }); + if (contiguous) { + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256), + sycl::range<1>(256)), + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + unary_op_flat_kernel(src, dst_ptr, k_elements, item_ct1, func); + }); + } else { + // Launch-invariant divisors: compute the magic numbers once on the host so the + // kernel never issues an integer divide. Only the strided path needs them. + const sycl::uint3 ne0_fd = init_fastdiv_values((uint32_t) ne0); + const sycl::uint3 ne1_fd = init_fastdiv_values((uint32_t) ne1); + const sycl::uint3 ne2_fd = init_fastdiv_values((uint32_t) ne2); + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256), + sycl::range<1>(256)), + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + unary_op_generic_kernel( + src, dst_ptr, k_elements, + ne0_fd, ne1_fd, ne2_fd, + nb0, nb1, nb2, nb3, + nbd0, nbd1, nbd2, nbd3, + item_ct1, + func + ); + }); + } + }); +} + +template +static inline void ggml_sycl_op_unary_gated( + ggml_backend_sycl_context & ctx, ggml_tensor * dst, F func) { + + dispatch_ggml_sycl_op_fused_glu(ctx, dst, + [func](const auto * x_ptr, const auto * g_ptr, auto * dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { + + const uint32_t num_blocks = (uint32_t) ceil_div(k, SYCL_GLU_BLOCK_SIZE); + const sycl::nd_range<1> launch_range(num_blocks * sycl::range<1>(SYCL_GLU_BLOCK_SIZE), + sycl::range<1>(SYCL_GLU_BLOCK_SIZE)); + + // o0 == n and o1 == n make the index math the identity, so index flat + // note: not ggml_is_contiguous - a fused [gate|up] src0 is contiguous with o0 == 2n + if (o0 == n && o1 == n) { + main_stream->parallel_for(launch_range, + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + unary_gated_op_flat_kernel(x_ptr, g_ptr, dst_ptr, k, item_ct1, func); + }); + } else { + // launch-invariant divisor, and only this path needs it + const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n); + main_stream->parallel_for(launch_range, + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + unary_gated_op_generic_kernel(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1, func); + }); + } }); } @@ -524,7 +736,7 @@ static inline void ggml_sycl_op_arange(ggml_backend_sycl_context & ctx, ggml_ten stream->parallel_for( sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_ARANGE_BLOCK_SIZE), sycl::range<1>(SYCL_ARANGE_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) { + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { arange_kernel(dst_ptr, k, start, step, item_ct1); }); } @@ -605,6 +817,12 @@ static inline void ggml_sycl_op_exp(ggml_backend_sycl_context & ctx, ggml_tensor }); } +static inline void ggml_sycl_op_expm1(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) { + return op_expm1(x); + }); +} + static inline void ggml_sycl_op_log(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst, [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) { @@ -612,7 +830,7 @@ static inline void ggml_sycl_op_log(ggml_backend_sycl_context & ctx, ggml_tensor stream->parallel_for( sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_EXP_BLOCK_SIZE), sycl::range<1>(SYCL_EXP_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) { + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { unary_op_log_kernel(src, dst_ptr, k_elements, item_ct1); }); }); @@ -650,7 +868,7 @@ static inline void ggml_sycl_op_sqrt(ggml_backend_sycl_context & ctx, ggml_tenso stream->parallel_for( sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SQRT_BLOCK_SIZE), sycl::range<1>(SYCL_SQRT_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) { + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { unary_op_sqrt_kernel(src, dst_ptr, k_elements, item_ct1); }); }); @@ -663,7 +881,7 @@ static inline void ggml_sycl_op_sin(ggml_backend_sycl_context & ctx, ggml_tensor stream->parallel_for( sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SIN_BLOCK_SIZE), sycl::range<1>(SYCL_SIN_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) { + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { unary_op_sin_kernel(src, dst_ptr, k_elements, item_ct1); }); }); @@ -676,7 +894,7 @@ static inline void ggml_sycl_op_cos(ggml_backend_sycl_context & ctx, ggml_tensor stream->parallel_for( sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SIN_BLOCK_SIZE), sycl::range<1>(SYCL_SIN_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) { + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { unary_op_cos_kernel(src, dst_ptr, k_elements, item_ct1); }); }); @@ -691,7 +909,7 @@ static inline void ggml_sycl_op_leaky_relu(ggml_backend_sycl_context & ctx, ggml stream->parallel_for( sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_RELU_BLOCK_SIZE), sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) { + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { unary_op_leaky_relu_kernel(src, dst_ptr, k_elements, slope, item_ct1); }); }, negative_slope); @@ -704,7 +922,7 @@ static inline void ggml_sycl_op_sqr(ggml_backend_sycl_context & ctx, ggml_tensor stream->parallel_for( sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_SQR_BLOCK_SIZE), sycl::range<1>(SYCL_SQR_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) { + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { unary_op_sqr_kernel(src, dst_ptr, k_elements, item_ct1); }); }); @@ -721,23 +939,33 @@ static inline void ggml_sycl_op_clamp(ggml_backend_sycl_context & ctx, ggml_tens stream->parallel_for( sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE), sycl::range<1>(SYCL_CLAMP_BLOCK_SIZE)), - [=](sycl::nd_item<1> item_ct1) { + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { clamp(src, dst_ptr, min_arg, max_arg, k_elements, item_ct1); }); }, min_val, max_val); } -static inline void ggml_sycl_op_floor(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { +static inline void ggml_sycl_op_xielu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const float alpha_n = ggml_get_op_params_f32(dst, 1); + const float alpha_p = ggml_get_op_params_f32(dst, 2); + const float beta = ggml_get_op_params_f32(dst, 3); + const float eps = ggml_get_op_params_f32(dst, 4); ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst, - [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) { - const int num_blocks = ceil_div(k_elements, 256); + [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream, float alpha_n_arg, float alpha_p_arg, float beta_arg, float eps_arg) { + const int num_blocks = ceil_div(k_elements, SYCL_RELU_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256), - sycl::range<1>(256)), - [=](sycl::nd_item<1> item_ct1) { - unary_op_floor_kernel(src, dst_ptr, k_elements, item_ct1); + sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(SYCL_RELU_BLOCK_SIZE), + sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + unary_op_xielu_kernel(src, dst_ptr, k_elements, alpha_n_arg, alpha_p_arg, beta_arg, eps_arg, item_ct1); }); - }); + }, alpha_n, alpha_p, beta, eps); +} + +static inline void ggml_sycl_op_floor(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) { + return op_floor(x); + }); } static inline void ggml_sycl_op_ceil(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -747,37 +975,23 @@ static inline void ggml_sycl_op_ceil(ggml_backend_sycl_context & ctx, ggml_tenso } static inline void ggml_sycl_op_round(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst, - [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) { - const int num_blocks = ceil_div(k_elements, 256); - stream->parallel_for( - sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256), - sycl::range<1>(256)), - [=](sycl::nd_item<1> item_ct1) { - unary_op_round_kernel(src, dst_ptr, k_elements, item_ct1); - }); - }); + ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) { + return op_round(x); + }); } static inline void ggml_sycl_op_trunc(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst, - [](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) { - const int num_blocks = ceil_div(k_elements, 256); - stream->parallel_for( - sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256), - sycl::range<1>(256)), - [=](sycl::nd_item<1> item_ct1) { - unary_op_trunc_kernel(src, dst_ptr, k_elements, item_ct1); - }); - }); + ggml_sycl_detail::ggml_sycl_op_unary(ctx, dst, [](auto x) { + return op_trunc(x); + }); } static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; - const float * src0_d = (const float *) src0->data; - const float * src1_d = (const float *) src1->data; + const char * src0_d = (const char *) src0->data; + const char * src1_d = (const char *) src1->data; float * dst_d = (float *) dst->data; dpct::queue_ptr stream = ctx.stream(); @@ -786,51 +1000,126 @@ static inline void ggml_sycl_op_acc(ggml_backend_sycl_context & ctx, ggml_tensor GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(src1)); GGML_ASSERT(dst->nb[0] == ggml_element_size(dst)); GGML_ASSERT(ggml_is_contiguously_allocated(dst)); + GGML_ASSERT(ggml_are_same_shape(src0, dst)); - const int64_t s1 = dst->op_params[0] / sizeof(float); - const int64_t s2 = dst->op_params[1] / sizeof(float); - const int64_t s3 = dst->op_params[2] / sizeof(float); - const int64_t offset = dst->op_params[3] / sizeof(float); + const int64_t s1 = (int64_t) ((const int32_t *) dst->op_params)[0] / (int64_t) sizeof(float); + const int64_t s2 = (int64_t) ((const int32_t *) dst->op_params)[1] / (int64_t) sizeof(float); + const int64_t s3 = (int64_t) ((const int32_t *) dst->op_params)[2] / (int64_t) sizeof(float); + const int64_t offset = (int64_t) ((const int32_t *) dst->op_params)[3] / (int64_t) sizeof(float); ggml_sycl_detail::acc_f32_sycl(src0_d, src1_d, dst_d, ggml_nelements(dst), + dst->ne[0], dst->ne[1], dst->ne[2], + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], + src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], s1, s2, s3, offset, stream); } static inline void ggml_sycl_op_geglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, - [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { - const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE); - main_stream->parallel_for( - sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) { - gated_op_fused_geglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); - }); - }); + ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) { + return op_gelu(x); + }); } static inline void ggml_sycl_op_reglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, - [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { - const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_RELU_BLOCK_SIZE); // Using RELU block size for reglu - main_stream->parallel_for( - sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) { - gated_op_fused_reglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); - }); - }); + ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) { + return op_relu(x); + }); } static inline void ggml_sycl_op_swiglu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, - [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { - const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_SILU_BLOCK_SIZE); // Using SILU block size for swiglu - main_stream->parallel_for( - sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) { - gated_op_fused_swiglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); - }); - }); + ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) { + return op_silu(x); + }); +} + +// Hands `launch` the functor for the unary op of a fused unary chain. Anything else +// ggml_sycl_can_fuse() has already declined, so the default is a dispatcher bug. +template +static void dispatch_fused_unary_op(ggml_unary_op uop, F && launch) { + switch (uop) { + case GGML_UNARY_OP_SILU: launch([](float v) { return op_silu(v); }); break; + case GGML_UNARY_OP_SIGMOID: launch([](float v) { return op_sigmoid(v); }); break; + case GGML_UNARY_OP_SOFTPLUS: launch([](float v) { return op_softplus(v); }); break; + default: + GGML_ABORT("fused unary chain: unsupported unary op %s", ggml_unary_op_name(uop)); + } +} + +// dst = op(unary_node->src[0]) * other, written straight to the MUL output, saving the +// standalone unary launch. Preconditions come from ggml_sycl_can_fuse(); re-asserted here. +void ggml_sycl_op_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node) { + scope_op_debug_print scope_dbg_print(__func__, mul_node, /*num_src=*/2); + + const ggml_tensor * x = unary_node->src[0]; + const ggml_tensor * g = (mul_node->src[0] == unary_node) ? mul_node->src[1] : mul_node->src[0]; + + // g is picked by elimination; ggml_can_fuse()'s single-use rule rules out MUL(unary, unary) + GGML_ASSERT(g != unary_node); + GGML_ASSERT(x->type == g->type && x->type == mul_node->type); + GGML_ASSERT(ggml_are_same_shape(x, g) && ggml_are_same_shape(x, mul_node)); + GGML_ASSERT(ggml_is_contiguous_1(x) && ggml_is_contiguous_1(g)); + // dst is indexed flat + GGML_ASSERT(ggml_is_contiguous(mul_node)); + + queue_ptr main_stream = ctx.stream(); + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const int64_t k = ggml_nelements(mul_node); + const int64_t n = mul_node->ne[0]; + + const auto dispatch_type = [&](auto op) { + switch (mul_node->type) { + case GGML_TYPE_F32: + unary_mul_sycl((const float *) x->data, (const float *) g->data, (float *) mul_node->data, + k, n, x->nb[1] / sizeof(float), g->nb[1] / sizeof(float), main_stream, op); + break; + case GGML_TYPE_F16: + unary_mul_sycl((const sycl::half *) x->data, (const sycl::half *) g->data, (sycl::half *) mul_node->data, + k, n, x->nb[1] / sizeof(sycl::half), g->nb[1] / sizeof(sycl::half), main_stream, op); + break; + default: + GGML_ABORT("fused unary+mul: unsupported type %s", ggml_type_name(mul_node->type)); + } + }; + + dispatch_fused_unary_op(ggml_get_unary_op(unary_node), dispatch_type); +} + +// dst = op(a + bias) * scale for an ADD + UNARY + MUL chain whose bias and scale broadcast +// over dim 0. Preconditions come from ggml_sycl_can_fuse(); re-asserted here. +void ggml_sycl_op_add_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * add_node, + ggml_tensor * unary_node, ggml_tensor * mul_node) { + // the dst-arity convention the other fusions follow; a and bias live on add_node + scope_op_debug_print scope_dbg_print(__func__, mul_node, /*num_src=*/2); + + const ggml_tensor * a = add_node->src[0]; + const ggml_tensor * bias = add_node->src[1]; + const ggml_tensor * scale = (mul_node->src[0] == unary_node) ? mul_node->src[1] : mul_node->src[0]; + + // scale is picked by elimination; ggml_can_fuse()'s single-use rule rules out MUL(unary, unary) + GGML_ASSERT(scale != unary_node); + GGML_ASSERT(a->type == GGML_TYPE_F32 && bias->type == GGML_TYPE_F32); + GGML_ASSERT(scale->type == GGML_TYPE_F32 && mul_node->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_are_same_shape(a, mul_node)); + // a and dst are indexed flat + GGML_ASSERT(ggml_is_contiguous(a) && ggml_is_contiguous(mul_node)); + // bias and scale are one contiguous ne0-length row each, broadcast over the outer dims + GGML_ASSERT(bias->ne[0] == a->ne[0] && scale->ne[0] == a->ne[0]); + GGML_ASSERT(ggml_nrows(bias) == 1 && ggml_nrows(scale) == 1); + GGML_ASSERT(ggml_is_contiguous(bias) && ggml_is_contiguous(scale)); + + queue_ptr main_stream = ctx.stream(); + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + const auto dispatch_op = [&](auto op) { + add_unary_mul_sycl((const float *) a->data, (const float *) bias->data, (const float *) scale->data, + (float *) mul_node->data, ggml_nelements(mul_node), mul_node->ne[0], main_stream, op); + }; + + dispatch_fused_unary_op(ggml_get_unary_op(unary_node), dispatch_op); } __dpct_inline__ float ggml_sycl_op_swiglu_oai_single(float x, float g, float alpha = 1.702f, float limit = 7.0f) { @@ -842,7 +1131,6 @@ __dpct_inline__ float ggml_sycl_op_swiglu_oai_single(float x, float g, float alp return out_glu; } - template static void swiglu_oai_kernel(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, @@ -876,12 +1164,12 @@ static void swiglu_oai_sycl(const T * x, const int64_t num_blocks = (k + SYCL_GLU_BLOCK_SIZE - 1) / SYCL_GLU_BLOCK_SIZE; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { swiglu_oai_kernel(x, g, dst, k, n, o0, o1, alpha, limit, item_ct1); }); } -void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { +static void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; void * src0_d = src0->data; @@ -925,26 +1213,112 @@ void ggml_sycl_op_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) swiglu_oai_sycl(src0_p, src1_p, (float *)dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), src1_o / sizeof(float), alpha, limit, stream); } +template +static void swiglu_clamp_kernel(const T * gate, + const T * up, + T * dst, + const int64_t k, + const int64_t n, + const int64_t o0, + const int64_t o1, + float limit, + sycl::nd_item<3> item_ct1) { + const int64_t i = int64_t(item_ct1.get_local_range(2)) * item_ct1.get_group(2) + item_ct1.get_local_id(2); + + if (i >= k) { + return; + } + + const int64_t j0 = (i / n) * o0 + (i % n); + const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); + + const float gate_value = sycl::fmin((float) gate[j0], limit); + const float up_value = sycl::fmax(sycl::fmin((float) up[j1], limit), -limit); + dst[i] = (T) (gate_value / (1.0f + sycl::native::exp(-gate_value)) * up_value); +} + +template +static void swiglu_clamp_sycl(const T * gate, + const T * up, + T * dst, + const int64_t k, + const int64_t n, + const int64_t o0, + const int64_t o1, + float limit, + dpct::queue_ptr stream) { + const int64_t num_blocks = (k + SYCL_GLU_BLOCK_SIZE - 1) / SYCL_GLU_BLOCK_SIZE; + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE), + sycl::range<3>(1, 1, SYCL_GLU_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + swiglu_clamp_kernel(gate, up, dst, k, n, o0, o1, limit, item_ct1); + }); +} + +static void ggml_sycl_op_swiglu_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + void * src0_d = src0->data; + void * src1_d = src1 ? src1->data : src0->data; + const int64_t src0_o = src0->nb[1]; + const int64_t src1_o = src1 ? src1->nb[1] : src0->nb[1]; + void * dst_d = dst->data; + const int64_t nc = src1 ? src0->ne[0] : src0->ne[0] / 2; + dpct::queue_ptr stream = ctx.stream(); + + GGML_ASSERT(ggml_is_contiguous_1(src0)); + GGML_ASSERT(src0->nb[0] == ggml_element_size(src0)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + GGML_ASSERT(src0->type == dst->type); + GGML_ASSERT(dst->ne[0] == nc); + GGML_ASSERT(ggml_nrows(dst) == ggml_nrows(src0)); + + if (src1) { + GGML_ASSERT(ggml_is_contiguous_1(src1)); + GGML_ASSERT(src1->nb[0] == ggml_element_size(src1)); + GGML_ASSERT(src1->ne[0] == nc); + GGML_ASSERT(src0->type == src1->type); + } + + const int32_t swapped = ggml_get_op_params_i32(dst, 1); + const float limit = ggml_get_op_params_f32(dst, 3); + + if (src0->type == GGML_TYPE_F16) { + sycl::half * src0_p = (sycl::half *) src0_d; + sycl::half * src1_p = (sycl::half *) src1_d; + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + swiglu_clamp_sycl(src0_p, src1_p, (sycl::half *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(sycl::half), + src1_o / sizeof(sycl::half), limit, stream); + } else { + float * src0_p = (float *) src0_d; + float * src1_p = (float *) src1_d; + + if (!src1) { + src0_p += swapped ? nc : 0; + src1_p += swapped ? 0 : nc; + } + + swiglu_clamp_sycl(src0_p, src1_p, (float *) dst_d, ggml_nelements(dst), nc, src0_o / sizeof(float), + src1_o / sizeof(float), limit, stream); + } +} + static inline void ggml_sycl_op_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, - [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { - const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE); - main_stream->parallel_for( - sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) { - gated_op_fused_geglu_erf(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); - }); - }); + ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) { + return op_gelu_erf(x); + }); } static inline void ggml_sycl_op_geglu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, - [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { - const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE); - main_stream->parallel_for( - sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) { - gated_op_fused_geglu_quick(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); - }); - }); + ggml_sycl_detail::ggml_sycl_op_unary_gated(ctx, dst, [](auto x) { + return op_gelu_quick(x); + }); } @@ -1018,6 +1392,11 @@ void ggml_sycl_exp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_op_exp(ctx, dst); } +void ggml_sycl_expm1(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); + ggml_sycl_op_expm1(ctx, dst); +} + void ggml_sycl_log(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); ggml_sycl_op_log(ctx, dst); @@ -1053,6 +1432,11 @@ void ggml_sycl_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_op_clamp(ctx, dst); } +void ggml_sycl_xielu(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); + ggml_sycl_op_xielu(ctx, dst); +} + void ggml_sycl_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); ggml_sycl_op_sgn(ctx, dst); @@ -1088,6 +1472,11 @@ void ggml_sycl_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_op_swiglu_oai(ctx, dst); } +void ggml_sycl_swiglu_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); + ggml_sycl_op_swiglu_clamp(ctx, dst); +} + void ggml_sycl_geglu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); ggml_sycl_op_geglu_erf(ctx, dst); diff --git a/ggml/src/ggml-sycl/element_wise.hpp b/ggml/src/ggml-sycl/element_wise.hpp index 9971321..0a5bfa9 100644 --- a/ggml/src/ggml-sycl/element_wise.hpp +++ b/ggml/src/ggml-sycl/element_wise.hpp @@ -28,6 +28,39 @@ typed_data cast_data(ggml_tensor * dst) { const float GELU_QUICK_COEF = -1.702f; +// Single-element activations, shared with the mat-vec kernels that fuse a GLU epilogue +// (mmvq.cpp), so both apply the same formula. +template static __dpct_inline__ T op_tanh(T x) { + if constexpr (std::is_same_v) { +#if defined(__INTEL_LLVM_COMPILER) && (__INTEL_LLVM_COMPILER >= 20260000) + return sycl::ext::oneapi::experimental::tanh(x); +#else + return static_cast(sycl::tanh(static_cast(x))); +#endif + } else { + return sycl::tanh(x); + } +} + +template static __dpct_inline__ T op_gelu(T x) { + const T GELU_COEF_A = static_cast(0.044715f); + const T SQRT_2_OVER_PI = static_cast(0.79788456080286535587989211986876f); + return static_cast(0.5f) * x * + (static_cast(1.0f) + + op_tanh(SQRT_2_OVER_PI * x * (static_cast(1.0f) + GELU_COEF_A * x * x))); +} + +template static __dpct_inline__ T op_exp(T x) { + if constexpr (std::is_same_v) { + return sycl::ext::oneapi::experimental::exp(x); + } else { + return sycl::exp(x); + } +} + +template static __dpct_inline__ T op_silu(T x) { + return x / (static_cast(1.0f) + op_exp(-x)); +} void ggml_sycl_sqrt(ggml_backend_sycl_context & ctx, ggml_tensor * dst); @@ -44,6 +77,7 @@ void ggml_sycl_silu(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_gelu_quick(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_swiglu_oai(ggml_backend_sycl_context & ctx, ggml_tensor * dst); +void ggml_sycl_swiglu_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_gelu_erf(ggml_backend_sycl_context & ctx, ggml_tensor * dst); @@ -59,6 +93,8 @@ void ggml_sycl_hardswish(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_exp(ggml_backend_sycl_context & ctx, ggml_tensor * dst); +void ggml_sycl_expm1(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + void ggml_sycl_log(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_softplus(ggml_backend_sycl_context & ctx, ggml_tensor * dst); @@ -73,6 +109,8 @@ void ggml_sycl_sqr(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_clamp(ggml_backend_sycl_context & ctx, ggml_tensor * dst); +void ggml_sycl_xielu(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + void ggml_sycl_sgn(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_abs(ggml_backend_sycl_context & ctx, ggml_tensor * dst); @@ -91,4 +129,12 @@ void ggml_sycl_trunc(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_arange(ggml_backend_sycl_context & ctx, ggml_tensor * dst); +// fused UNARY(silu|sigmoid|softplus) + MUL; see ggml_sycl_can_fuse() for the accepted shapes +void ggml_sycl_op_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node); + +// fused f32 ADD + UNARY(silu|sigmoid|softplus) + MUL with the bias and the scale broadcast +// over dim 0; see ggml_sycl_can_fuse() for the accepted shapes +void ggml_sycl_op_add_unary_mul_fused(ggml_backend_sycl_context & ctx, ggml_tensor * add_node, + ggml_tensor * unary_node, ggml_tensor * mul_node); + #endif // GGML_SYCL_ELEMENTWISE_HPP diff --git a/ggml/src/ggml-sycl/esimd.hpp b/ggml/src/ggml-sycl/esimd.hpp new file mode 100644 index 0000000..85cc93e --- /dev/null +++ b/ggml/src/ggml-sycl/esimd.hpp @@ -0,0 +1,1399 @@ +#ifndef GGML_SYCL_ESIMD_HPP +#define GGML_SYCL_ESIMD_HPP + +#include +#include +#include + +#include "common.hpp" + +namespace ggml_sycl_esimd { + +constexpr int GGML_SYCL_DMMV_ESIMD_WG_SIZE = 4; + +// +// Shared ESIMD building blocks for the reordered K-quant dequantize-matvec +// kernels. +// +// The reordered K-quant ESIMD matvec kernels share one skeleton: per super-block, +// load a 256-float activation slice, load one weight block, dequantize it into 8 +// chunks of 32 and MAC each chunk against the matching activation slice, then +// reduce and run a lane-0 epilogue. +// +// Each K-quant kernel emits exactly 8 chunks of 32 mapping to activation slices +// 0..7, so the per-block work is captured by esimd_reorder_q_traits::mac_pair, +// which dequantizes two weight blocks and MACs both against a shared activation +// vector with the two FMA chains interleaved (co-scheduled to hide FMA latency). +// The "pair" is the (row0,row1) row pair owned by one work-group, so the +// layout+dequant is written once per quant type here. +// + +template struct esimd_reorder_q_traits; + +// build a 32-lane vector whose low 16 lanes are `lo` and high 16 are `hi` +// (a super-chunk splits into two 16-wide halves with distinct scale/min codes). +static ESIMD_INLINE sycl::ext::intel::esimd::simd splat_lo_hi(float lo, float hi) { + using namespace sycl::ext::intel::esimd; + simd v; + v.select<16, 1>(0) = lo; + v.select<16, 1>(16) = hi; + return v; +} + +// unpack one block of Q4_K/Q5_K scale/min codes (get_scale_min_k4 layout) into 8 +// float scales (dall * sc) and 8 float mins (-dmin * m); the min carries the +// negation so the dequant epilogue adds. +static ESIMD_INLINE void unpack_scale_min_k4( + sycl::ext::intel::esimd::simd scales, float dall, float dmin, + sycl::ext::intel::esimd::simd & scale_f, + sycl::ext::intel::esimd::simd & min_f) { + using namespace sycl::ext::intel::esimd; + simd sc = 0; + simd m = 0; + simd scale_lo = scales.select<4, 1>(0); + simd min_lo = scales.select<4, 1>(4); + simd hi_bits = scales.select<4, 1>(8); + sc.select<4, 1>(0) = scale_lo & simd(0x3F); + sc.select<4, 1>(4) = (hi_bits & simd(0x0F)) | + ((scale_lo >> simd(6)) << simd(4)); + m.select<4, 1>(0) = min_lo & simd(0x3F); + m.select<4, 1>(4) = (hi_bits >> simd(4)) | + ((min_lo >> simd(6)) << simd(4)); + scale_f = convert(sc) * dall; + min_f = convert(m) * (-dmin); +} + +// --------------------------------------------------------------------------- +// Q2_K, SOA reorder layout produced by reorder_qw_q2_k: +// [qs: nb*(QK_K/4)] [scales: nb*(QK_K/16)] [dm: nb*sizeof(half2)] +// with nb = nrows*num_blocks_per_row. +// +// 2 bits per weight. The 8 output chunks of 32 (matching dequantize_row_q2_K) +// map to super-chunk s (0..7): byte base 32*(s/4) into the 64-byte qs array, +// bit shift 2*(s%4); the low 16 lanes use scales[2s], the high 16 use +// scales[2s+1], with dl = d*(sc & 0xF), ml = dmin*(sc >> 4), deq = dl*q - ml. +// --------------------------------------------------------------------------- +template <> struct esimd_reorder_q_traits { + struct ptrs { + const uint8_t * qs; + const uint8_t * scales; + const sycl::half * dm; + }; + + static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) { + const uint8_t * qs = (const uint8_t *) vx; + const uint8_t * scales = qs + nb * (QK_K / 4); + const sycl::half * dm = (const sycl::half *) (scales + nb * (QK_K / 16)); + return { qs, scales, dm }; + } + + static ESIMD_INLINE void mac_pair( + const ptrs & pa, size_t bia, + const ptrs & pb, size_t bib, bool has_b, + sycl::ext::intel::esimd::simd & y_vec, + sycl::ext::intel::esimd::simd & acc_a, + sycl::ext::intel::esimd::simd & acc_b) { + using namespace sycl::ext::intel::esimd; + + simd qs_a = block_load(pa.qs + bia * (QK_K / 4)); + simd qs_b = 0; + simd scales_a = block_load(pa.scales + bia * (QK_K / 16)); + simd scales_b = 0; + + const float dall_a = (float) pa.dm[bia * 2 + 0]; + const float dmin_a = (float) pa.dm[bia * 2 + 1]; + float dall_b = 0.0f; + float dmin_b = 0.0f; + if (has_b) { + qs_b = block_load(pb.qs + bib * (QK_K / 4)); + scales_b = block_load(pb.scales + bib * (QK_K / 16)); + dall_b = (float) pb.dm[bib * 2 + 0]; + dmin_b = (float) pb.dm[bib * 2 + 1]; + } + + // per-chunk scale (d * (sc & 0xF)) and min (-dmin * (sc >> 4)), all 16 codes; + // min carries the negation so the dequant epilogue adds (matches Q4_K/Q5_K) + simd scale_f_a = convert(scales_a & simd(0x0F)) * dall_a; + simd min_f_a = convert(scales_a >> simd(4)) * (-dmin_a); + simd scale_f_b = convert(scales_b & simd(0x0F)) * dall_b; + simd min_f_b = convert(scales_b >> simd(4)) * (-dmin_b); + +#pragma unroll + for (int s = 0; s < 8; ++s) { + const int byte_base = 32 * (s / 4); + const uint8_t shift = (uint8_t) (2 * (s % 4)); + simd y_s = y_vec.select<32, 1>(s * 32); + + simd qa = (qs_a.select<32, 1>(byte_base) >> shift) & simd(3); + simd qb = (qs_b.select<32, 1>(byte_base) >> shift) & simd(3); + + const float scale_a_lo = scale_f_a[2 * s + 0]; + const float scale_a_hi = scale_f_a[2 * s + 1]; + const float min_a_lo = min_f_a[2 * s + 0]; + const float min_a_hi = min_f_a[2 * s + 1]; + const float scale_b_lo = scale_f_b[2 * s + 0]; + const float scale_b_hi = scale_f_b[2 * s + 1]; + const float min_b_lo = min_f_b[2 * s + 0]; + const float min_b_hi = min_f_b[2 * s + 1]; + + simd scale_vec_a = splat_lo_hi(scale_a_lo, scale_a_hi); + simd min_vec_a = splat_lo_hi(min_a_lo, min_a_hi); + simd scale_vec_b = splat_lo_hi(scale_b_lo, scale_b_hi); + simd min_vec_b = splat_lo_hi(min_b_lo, min_b_hi); + + simd deq_a = convert(qa) * scale_vec_a + min_vec_a; + simd deq_b = convert(qb) * scale_vec_b + min_vec_b; + + acc_a += y_s * deq_a; + acc_b += y_s * deq_b; + } + } +}; + +// --------------------------------------------------------------------------- +// Q3_K, SOA reorder layout produced by reorder_qw_q3_k: +// [qs: nb*(QK_K/4)] [hmask: nb*(QK_K/8)] [scales: nb*12] [d: nb*sizeof(half)] +// with nb = nrows*num_blocks_per_row. Single super-block scale d, no dmin. +// +// 3 bits per weight: 2 low bits in qs, 1 high bit in hmask. The 8 output chunks +// of 32 (matching dequantize_row_q3_K) map to super-chunk s (0..7): byte base +// 32*(s/4) into the 64-byte qs array, bit shift 2*(s%4); the low 16 lanes use +// scale code 2s, the high 16 use 2s+1. hmask is a 32-byte array (like Q5_K's +// qh) where chunk s uses bit s of the same 32 bytes, but INVERTED: the value is +// (q & 3) - (hmask_bit_set ? 0 : 4), i.e. (q & 3) + 4*bit - 4. +// +// The 16 6-bit scale codes are packed into 12 bytes (get_scale_min layout for +// Q3_K): low nibbles from bytes 0..7, high 2 bits from bytes 8..11 shifted by +// 0/2/4/6; the dequant scale is d * (code - 32). +// --------------------------------------------------------------------------- +template <> struct esimd_reorder_q_traits { + struct ptrs { + const uint8_t * qs; + const uint8_t * hmask; + const uint8_t * scales; + const sycl::half * d; + }; + + static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) { + const uint8_t * qs = (const uint8_t *) vx; + const uint8_t * hmask = qs + nb * (QK_K / 4); + const uint8_t * scales = hmask + nb * (QK_K / 8); + const sycl::half * d = (const sycl::half *) (scales + nb * 12); + return { qs, hmask, scales, d }; + } + + // unpack the 12 packed bytes into 16 6-bit scale codes (dequantize_row_q3_K + // aux layout), returned as float scale = d * (code - 32). + // done with wide (8/16-lane) ops rather than four 4-lane groups. + static ESIMD_INLINE sycl::ext::intel::esimd::simd unpack_scales( + sycl::ext::intel::esimd::simd in, float d) { + using namespace sycl::ext::intel::esimd; + + // low 6-bit part: codes 0..7 = low nibble of bytes 0..7, + // codes 8..15 = high nibble of bytes 0..7 + simd lo8 = in.select<8, 1>(0); + simd code; + code.select<8, 1>(0) = lo8 & simd(0x0F); + code.select<8, 1>(8) = lo8 >> simd(4); + + // high 2-bit part: bytes 8..11 replicated 4x, group g (0..3) shifted 2*g + simd hib; + hib.select<4, 1>(0) = in.select<4, 1>(8); + hib.select<4, 1>(4) = in.select<4, 1>(8); + hib.select<4, 1>(8) = in.select<4, 1>(8); + hib.select<4, 1>(12) = in.select<4, 1>(8); + simd hshift; + hshift.select<4, 1>(0) = 0; + hshift.select<4, 1>(4) = 2; + hshift.select<4, 1>(8) = 4; + hshift.select<4, 1>(12) = 6; + hib = (hib >> hshift) & simd(0x03); + + code = code | (hib << simd(4)); + return (convert(code) - 32.0f) * d; + } + + static ESIMD_INLINE void mac_pair( + const ptrs & pa, size_t bia, + const ptrs & pb, size_t bib, bool has_b, + sycl::ext::intel::esimd::simd & y_vec, + sycl::ext::intel::esimd::simd & acc_a, + sycl::ext::intel::esimd::simd & acc_b) { + using namespace sycl::ext::intel::esimd; + + simd qs_a = block_load(pa.qs + bia * (QK_K / 4)); + simd qs_b = 0; + simd hmask_a = block_load(pa.hmask + bia * (QK_K / 8)); + simd hmask_b = 0; + simd scales_a = block_load(pa.scales + bia * 12); + simd scales_b = 0; + + const float d_a = (float) pa.d[bia]; + float d_b = 0.0f; + if (has_b) { + qs_b = block_load(pb.qs + bib * (QK_K / 4)); + hmask_b = block_load(pb.hmask + bib * (QK_K / 8)); + scales_b = block_load(pb.scales + bib * 12); + d_b = (float) pb.d[bib]; + } + + simd scale_f_a = unpack_scales(scales_a, d_a); + simd scale_f_b = unpack_scales(scales_b, d_b); + +#pragma unroll + for (int s = 0; s < 8; ++s) { + const int byte_base = 32 * (s / 4); + const uint8_t shift = (uint8_t) (2 * (s % 4)); + simd y_s = y_vec.select<32, 1>(s * 32); + + // 2 low bits from qs, high bit from hmask (bit s of the same 32 bytes); + // value = (q & 3) + 4*bit - 4 (inverted hmask: subtract 4 when bit clear). + // merge in the integer domain: q3 = (q & 3) | (bit << 2) in {0..7}, + // then a single convert + subtract yields q3 - 4 (one convert, not two) + simd q3_a = convert( + (qs_a.select<32, 1>(byte_base) >> shift) & simd(3)); + q3_a |= convert( + ((hmask_a >> simd((uint8_t) s)) & simd(1)) << simd(2)); + simd q3_b = convert( + (qs_b.select<32, 1>(byte_base) >> shift) & simd(3)); + q3_b |= convert( + ((hmask_b >> simd((uint8_t) s)) & simd(1)) << simd(2)); + + simd qf_a = convert(q3_a) - 4.0f; + simd qf_b = convert(q3_b) - 4.0f; + + const float scale_a_lo = scale_f_a[2 * s + 0]; + const float scale_a_hi = scale_f_a[2 * s + 1]; + const float scale_b_lo = scale_f_b[2 * s + 0]; + const float scale_b_hi = scale_f_b[2 * s + 1]; + + simd scale_vec_a = splat_lo_hi(scale_a_lo, scale_a_hi); + simd scale_vec_b = splat_lo_hi(scale_b_lo, scale_b_hi); + + simd deq_a = qf_a * scale_vec_a; + simd deq_b = qf_b * scale_vec_b; + + acc_a += y_s * deq_a; + acc_b += y_s * deq_b; + } + } +}; + +// --------------------------------------------------------------------------- +// Q4_K, SOA reorder layout produced by reorder_qw_q4_k: +// [qs: nb*(QK_K/2)] [scales: nb*K_SCALE_SIZE] [dm: nb*sizeof(half2)] +// with nb = nrows*num_blocks_per_row. +// --------------------------------------------------------------------------- +template <> struct esimd_reorder_q_traits { + struct ptrs { + const uint8_t * qs; + const uint8_t * scales; + const sycl::half * dm; + }; + + static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) { + const uint8_t * qs = (const uint8_t *) vx; + const uint8_t * scales = qs + nb * (QK_K / 2); + const sycl::half * dm = (const sycl::half *) (scales + nb * K_SCALE_SIZE); + return { qs, scales, dm }; + } + + static ESIMD_INLINE void mac_pair( + const ptrs & pa, size_t bia, + const ptrs & pb, size_t bib, bool has_b, + sycl::ext::intel::esimd::simd & y_vec, + sycl::ext::intel::esimd::simd & acc_a, + sycl::ext::intel::esimd::simd & acc_b) { + using namespace sycl::ext::intel::esimd; + + simd qs_a = block_load(pa.qs + bia * (QK_K / 2)); + simd qs_b = 0; + simd scales_a = block_load(pa.scales + bia * K_SCALE_SIZE); + simd scales_b = 0; + + const float dall_a = (float) pa.dm[bia * 2 + 0]; + const float dmin_a = (float) pa.dm[bia * 2 + 1]; + float dall_b = 0.0f; + float dmin_b = 0.0f; + if (has_b) { + qs_b = block_load(pb.qs + bib * (QK_K / 2)); + scales_b = block_load(pb.scales + bib * K_SCALE_SIZE); + dall_b = (float) pb.dm[bib * 2 + 0]; + dmin_b = (float) pb.dm[bib * 2 + 1]; + } + + simd scale_f_a, min_f_a, scale_f_b, min_f_b; + unpack_scale_min_k4(scales_a, dall_a, dmin_a, scale_f_a, min_f_a); + unpack_scale_min_k4(scales_b, dall_b, dmin_b, scale_f_b, min_f_b); + + simd qs_lo_a = qs_a & simd(0x0F); + simd qs_hi_a = qs_a >> simd(4); + simd qs_lo_b = qs_b & simd(0x0F); + simd qs_hi_b = qs_b >> simd(4); + +#pragma unroll + for (int sb = 0; sb < 8; sb += 2) { + const int q_offset = sb * 16; + simd y_lo = y_vec.select<32, 1>(sb * 32); + simd y_hi = y_vec.select<32, 1>((sb + 1) * 32); + + const float scale_a_lo = scale_f_a[sb]; + const float scale_a_hi = scale_f_a[sb + 1]; + const float min_a_lo = min_f_a[sb]; + const float min_a_hi = min_f_a[sb + 1]; + const float scale_b_lo = scale_f_b[sb]; + const float scale_b_hi = scale_f_b[sb + 1]; + const float min_b_lo = min_f_b[sb]; + const float min_b_hi = min_f_b[sb + 1]; + + simd qa_lo = qs_lo_a.select<32, 1>(q_offset); + simd qa_hi = qs_hi_a.select<32, 1>(q_offset); + simd qb_lo = qs_lo_b.select<32, 1>(q_offset); + simd qb_hi = qs_hi_b.select<32, 1>(q_offset); + + simd deq_a_lo = convert(qa_lo) * scale_a_lo + min_a_lo; + simd deq_a_hi = convert(qa_hi) * scale_a_hi + min_a_hi; + simd deq_b_lo = convert(qb_lo) * scale_b_lo + min_b_lo; + simd deq_b_hi = convert(qb_hi) * scale_b_hi + min_b_hi; + + acc_a += y_lo * deq_a_lo; + acc_b += y_lo * deq_b_lo; + acc_a += y_hi * deq_a_hi; + acc_b += y_hi * deq_b_hi; + } + } +}; + +// --------------------------------------------------------------------------- +// Q5_K, SOA reorder layout produced by reorder_qw_q5_k: +// [qs: nb*(QK_K/2)] [qh: nb*(QK_K/8)] [scales: nb*K_SCALE_SIZE] [dm: nb*sizeof(half2)] +// with nb = nrows*num_blocks_per_row. +// +// Identical to Q4_K except each 4-bit quant gains a 5th (high) bit from qh: +// output chunk c (0..7) adds 16 when bit c of qh[l] is set, where qh[l] indexes +// the same 32 bytes for every chunk (matches dequantize_row_q5_K). +// --------------------------------------------------------------------------- +template <> struct esimd_reorder_q_traits { + struct ptrs { + const uint8_t * qs; + const uint8_t * qh; + const uint8_t * scales; + const sycl::half * dm; + }; + + static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) { + const uint8_t * qs = (const uint8_t *) vx; + const uint8_t * qh = qs + nb * (QK_K / 2); + const uint8_t * scales = qh + nb * (QK_K / 8); + const sycl::half * dm = (const sycl::half *) (scales + nb * K_SCALE_SIZE); + return { qs, qh, scales, dm }; + } + + // extract bit `bit` (0..7) of each lane and move it to bit position 4, + // e.g. for the 4-bit base quant's 5th (high) bit. `bit` is always a + // compile-time-known unrolled loop constant at call sites, so this folds + // to a single mask (bit==4), mask+left-shift (bit<4), or mask+right-shift + // (bit>4) instead of the shift+mask+shift a naive `(qh>>bit & 1) << 4` emits. + static ESIMD_INLINE sycl::ext::intel::esimd::simd extract_bit_to_pos4( + sycl::ext::intel::esimd::simd qh, int bit) { + using namespace sycl::ext::intel::esimd; + simd masked = convert(qh & simd((uint8_t) (1u << bit))); + if (bit < 4) { + return masked << simd((uint16_t) (4 - bit)); + } else if (bit > 4) { + return masked >> simd((uint16_t) (bit - 4)); + } + return masked; + } + + static ESIMD_INLINE void mac_pair( + const ptrs & pa, size_t bia, + const ptrs & pb, size_t bib, bool has_b, + sycl::ext::intel::esimd::simd & y_vec, + sycl::ext::intel::esimd::simd & acc_a, + sycl::ext::intel::esimd::simd & acc_b) { + using namespace sycl::ext::intel::esimd; + + simd qs_a = block_load(pa.qs + bia * (QK_K / 2)); + simd qs_b = 0; + simd qh_a = block_load(pa.qh + bia * (QK_K / 8)); + simd qh_b = 0; + simd scales_a = block_load(pa.scales + bia * K_SCALE_SIZE); + simd scales_b = 0; + + const float dall_a = (float) pa.dm[bia * 2 + 0]; + const float dmin_a = (float) pa.dm[bia * 2 + 1]; + float dall_b = 0.0f; + float dmin_b = 0.0f; + if (has_b) { + qs_b = block_load(pb.qs + bib * (QK_K / 2)); + qh_b = block_load(pb.qh + bib * (QK_K / 8)); + scales_b = block_load(pb.scales + bib * K_SCALE_SIZE); + dall_b = (float) pb.dm[bib * 2 + 0]; + dmin_b = (float) pb.dm[bib * 2 + 1]; + } + + simd scale_f_a, min_f_a, scale_f_b, min_f_b; + unpack_scale_min_k4(scales_a, dall_a, dmin_a, scale_f_a, min_f_a); + unpack_scale_min_k4(scales_b, dall_b, dmin_b, scale_f_b, min_f_b); + + simd qs_lo_a = qs_a & simd(0x0F); + simd qs_hi_a = qs_a >> simd(4); + simd qs_lo_b = qs_b & simd(0x0F); + simd qs_hi_b = qs_b >> simd(4); + +#pragma unroll + for (int sb = 0; sb < 8; sb += 2) { + const int q_offset = sb * 16; + simd y_lo = y_vec.select<32, 1>(sb * 32); + simd y_hi = y_vec.select<32, 1>((sb + 1) * 32); + + const float scale_a_lo = scale_f_a[sb]; + const float scale_a_hi = scale_f_a[sb + 1]; + const float min_a_lo = min_f_a[sb]; + const float min_a_hi = min_f_a[sb + 1]; + const float scale_b_lo = scale_f_b[sb]; + const float scale_b_hi = scale_f_b[sb + 1]; + const float min_b_lo = min_f_b[sb]; + const float min_b_hi = min_f_b[sb + 1]; + + simd qa_lo_u8 = qs_lo_a.select<32, 1>(q_offset); + simd qa_hi_u8 = qs_hi_a.select<32, 1>(q_offset); + simd qb_lo_u8 = qs_lo_b.select<32, 1>(q_offset); + simd qb_hi_u8 = qs_hi_b.select<32, 1>(q_offset); + simd qa_lo = convert(qa_lo_u8); + simd qa_hi = convert(qa_hi_u8); + simd qb_lo = convert(qb_lo_u8); + simd qb_hi = convert(qb_hi_u8); + + // add the 5th bit: chunk sb uses qh bit sb, chunk sb+1 uses qh bit sb+1; + // qh always indexes the same 32 bytes regardless of chunk + qa_lo += extract_bit_to_pos4(qh_a, sb); + qa_hi += extract_bit_to_pos4(qh_a, sb + 1); + qb_lo += extract_bit_to_pos4(qh_b, sb); + qb_hi += extract_bit_to_pos4(qh_b, sb + 1); + + simd deq_a_lo = convert(qa_lo) * scale_a_lo + min_a_lo; + simd deq_a_hi = convert(qa_hi) * scale_a_hi + min_a_hi; + simd deq_b_lo = convert(qb_lo) * scale_b_lo + min_b_lo; + simd deq_b_hi = convert(qb_hi) * scale_b_hi + min_b_hi; + + acc_a += y_lo * deq_a_lo; + acc_b += y_lo * deq_b_lo; + acc_a += y_hi * deq_a_hi; + acc_b += y_hi * deq_b_hi; + } + } +}; + +// --------------------------------------------------------------------------- +// Q6_K, SOA reorder layout: +// [ql: nb*(QK_K/2)] [qh: nb*(QK_K/4)] [scales(int8): nb*(QK_K/16)] [d: nb*half] +// --------------------------------------------------------------------------- +template <> struct esimd_reorder_q_traits { + struct ptrs { + const uint8_t * ql; + const uint8_t * qh; + const int8_t * scales; + const sycl::half * d; + }; + + static ESIMD_INLINE ptrs make_ptrs(const void * vx, size_t nb) { + const uint8_t * ql = (const uint8_t *) vx; + const uint8_t * qh = ql + nb * (QK_K / 2); + const int8_t * scales = (const int8_t *) (qh + nb * (QK_K / 4)); + const sycl::half * d = (const sycl::half *) (scales + nb * (QK_K / 16)); + return { ql, qh, scales, d }; + } + + static ESIMD_INLINE void mac_pair( + const ptrs & pa, size_t bia, + const ptrs & pb, size_t bib, bool has_b, + sycl::ext::intel::esimd::simd & y_vec, + sycl::ext::intel::esimd::simd & acc_a, + sycl::ext::intel::esimd::simd & acc_b) { + using namespace sycl::ext::intel::esimd; + + simd ql_a = block_load(pa.ql + bia * (QK_K / 2)); + simd ql_b = 0; + simd qh_a = block_load(pa.qh + bia * (QK_K / 4)); + simd qh_b = 0; + simd scales_a = block_load(pa.scales + bia * (QK_K / 16)); + simd scales_b = 0; + + const float d_a = (float) pa.d[bia]; + float d_b = 0.0f; + if (has_b) { + ql_b = block_load(pb.ql + bib * (QK_K / 2)); + qh_b = block_load(pb.qh + bib * (QK_K / 4)); + scales_b = block_load(pb.scales + bib * (QK_K / 16)); + d_b = (float) pb.d[bib]; + } + + simd sc_a = convert(scales_a); + simd sc_b = convert(scales_b); + +#pragma unroll + for (int im = 0; im < 2; ++im) { + simd ql_lo_a = ql_a.select<32, 1>(64 * im); + simd ql_hi_a = ql_a.select<32, 1>(64 * im + 32); + simd qh_bits_a = qh_a.select<32, 1>(32 * im); + simd ql_lo_b = ql_b.select<32, 1>(64 * im); + simd ql_hi_b = ql_b.select<32, 1>(64 * im + 32); + simd qh_bits_b = qh_b.select<32, 1>(32 * im); + + // reconstruct each 32-wide 6-bit group (matches dequantize_row_q6_K) +#pragma unroll + for (int g = 0; g < 4; ++g) { + simd y_g = y_vec.select<32, 1>(32 * (4 * im + g)); + + const float scale_a_lo = sc_a[8 * im + 2 * g + 0] * d_a; + const float scale_a_hi = sc_a[8 * im + 2 * g + 1] * d_a; + const float scale_b_lo = sc_b[8 * im + 2 * g + 0] * d_b; + const float scale_b_hi = sc_b[8 * im + 2 * g + 1] * d_b; + + simd scale_vec_a = splat_lo_hi(scale_a_lo, scale_a_hi); + simd scale_vec_b = splat_lo_hi(scale_b_lo, scale_b_hi); + + simd qa; + simd qb; + switch (g) { + case 0: + qa = (ql_lo_a & simd(0x0F)) | ((qh_bits_a & simd(0x03)) << simd(4)); + qb = (ql_lo_b & simd(0x0F)) | ((qh_bits_b & simd(0x03)) << simd(4)); + break; + case 1: + qa = (ql_hi_a & simd(0x0F)) | ((qh_bits_a & simd(0x0C)) << simd(2)); + qb = (ql_hi_b & simd(0x0F)) | ((qh_bits_b & simd(0x0C)) << simd(2)); + break; + case 2: + qa = (ql_lo_a >> simd(4)) | (qh_bits_a & simd(0x30)); + qb = (ql_lo_b >> simd(4)) | (qh_bits_b & simd(0x30)); + break; + default: + qa = (ql_hi_a >> simd(4)) | ((qh_bits_a & simd(0xC0)) >> simd(2)); + qb = (ql_hi_b >> simd(4)) | ((qh_bits_b & simd(0xC0)) >> simd(2)); + break; + } + + simd deq_a = (convert(qa) - 32.0f) * scale_vec_a; + simd deq_b = (convert(qb) - 32.0f) * scale_vec_b; + + acc_a += y_g * deq_a; + acc_b += y_g * deq_b; + } + } + } +}; + +// +// Multi-column mul_mat_vec for reordered K-quant weights on the XMX engines (int8 DPAS). +// A thread owns 16 weight rows (B operand, one dword is 4 weights of a row), the q8_1 activation +// columns are the A operand. A super-block is 8 groups of 32, run as 2 parts of 4 groups. A quant +// type supplies its loads and unpack as a traits struct. +// +using sycl::ext::intel::esimd::simd; +constexpr int XMX_ROWS = 16; // weight rows per thread (DPAS N) +constexpr int XMX_CHUNK = 64; // widest single launch, more columns run as repeated launches +constexpr int XMX_FUSED_CHUNK = GGML_SYCL_XMX_GLU_MAX_COLS; // widest launch of the fused gate and up kernel + +// threads that split K for one tile, sized so that the shared memory stays small +template constexpr int xmx_n_split() { + return NC <= 8 ? 32 : (NC == 16 ? 16 : 8); +} + +inline bool xmx_supported(const void * vx, const int ncols) { + // 2D block loads need a 64 byte aligned base + return ncols % QK_K == 0 && (uintptr_t) vx % 64 == 0; +} + +// transposed 2D block load of 16 rows, dword (kd, n) lands at [kd * 16 + n] +// 2D block loads need rows of at least 64 bytes, shorter rows (SR) are read row by row +template +ESIMD_INLINE simd xmx_load_t(const void * base, unsigned pitch, unsigned h, int x, int y) { + using namespace sycl::ext::intel::esimd; + if constexpr (SR) { + simd r; + for (int n = 0; n < XMX_ROWS; ++n) { + const unsigned row = y + n < (int) h ? y + n : h; + const char * p = (const char *) base + (size_t) row * (pitch + 1); + r.template select(n) = block_load((const uint32_t *) p + x); + } + return r; + } else { + return load_2d((const uint32_t *) base, pitch, h, pitch, x, y); + } +} + +// per byte (q - zero) as int8 for q in 0..127, no borrow between bytes +ESIMD_INLINE simd xmx_bytes_sub(simd x, uint32_t zero_x4) { + return ((x | 0x80808080u) - zero_x4) ^ 0x80808080u; +} + +// How the scales of a quant type enter the DPAS epilogue of a group of 32 (see xmx_group) +enum class xmx_scale_kind { + split, // two scales per group, one per 16 wide half: the weights are masked to one half for each DPAS + min, // one scale and one min per group: a second DPAS sums the activations for the min term + min16, // a scale and a min per 16 wide half: the scales are folded into the DPAS operands + plain, // one scale per group +}; + +// What every quant type states about itself. nc_wide is the tile width from which it takes the large register file. +template struct xmx_traits_base { + static constexpr bool short_rows = SR; + static constexpr int nc_wide = NC_WIDE; + static constexpr bool has_min = KIND == xmx_scale_kind::min; + static constexpr bool plain = KIND == xmx_scale_kind::plain; + static constexpr bool min16 = KIND == xmx_scale_kind::min16; +}; + +// The part of a context every quant type shares: the last row (the clamp of 2D loads), this thread's first row and +// the number of super-blocks per row. A type adds its array pointers and 2D load pitches. +struct xmx_ctx_base { + unsigned h; + int row0; + int bpr; +}; + +ESIMD_INLINE void xmx_ctx_init(xmx_ctx_base & c, int nrows, int bpr, int row0) { + c.h = nrows - 1; + c.row0 = row0; + c.bpr = bpr; +} + +// pitch of a 2D load over an array with `bytes` bytes per super-block +constexpr unsigned xmx_pitch(int bpr, int bytes) { + return bpr * bytes - 1; +} + +// per group row scales of the 16 rows, one vector per group +template struct xmx_row_scales { + simd v[N]; + + ESIMD_INLINE simd get(int i) { return v[i]; } +}; + +// the 12 packed scale bytes of a super-block (Q3_K, Q4_K, Q5_K) of the 16 rows as three dwords +ESIMD_INLINE void xmx_load_sc12(const uint8_t * sc, + const simd & bi, + simd & w0, + simd & w1, + simd & w2) { + using namespace sycl::ext::intel::esimd; + w0 = gather((const uint32_t *) sc, bi * 12); + w1 = gather((const uint32_t *) sc, bi * 12 + 4); + w2 = gather((const uint32_t *) sc, bi * 12 + 8); +} + +// d and dmin (a half2 per super-block, Q2_K, Q4_K, Q5_K) of the 16 rows +ESIMD_INLINE void xmx_load_dm(const sycl::half * dm, + const simd & bi, + simd & d, + simd & dmin) { + using namespace sycl::ext::intel::esimd; + d = convert(gather(dm, bi * (2 * (uint32_t) sizeof(sycl::half)))); + dmin = convert( + gather(dm, bi * (2 * (uint32_t) sizeof(sycl::half)) + (uint32_t) sizeof(sycl::half))); +} + +// Q6_K reorder layout: [ql: nb*128] [qh: nb*64] [scales int8: nb*16] [d: nb*2] +template struct xmx_traits_q6_k : xmx_traits_base { + struct ctx : xmx_ctx_base { + const uint8_t * ql; + const uint8_t * qh; + const int8_t * scales; + const sycl::half * dh; + unsigned pl, ph, ps; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + const size_t nb = (size_t) nrows * bpr; + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.ql = (const uint8_t *) vx; + c.qh = c.ql + nb * (QK_K / 2); + c.scales = (const int8_t *) (c.qh + nb * (QK_K / 4)); + c.dh = (const sycl::half *) (c.scales + nb * (QK_K / 16)); + c.pl = xmx_pitch(bpr, QK_K / 2); + c.ph = xmx_pitch(bpr, QK_K / 4); + c.ps = xmx_pitch(bpr, QK_K / 16); + return c; + } + + // row scales of the 16 scale groups of a super-block + struct scales_t { + simd sc8; + simd dv; + + ESIMD_INLINE simd get(int gr) { + simd s = sc8.template select<16, 4>(64 * (gr / 4) + (gr % 4)); + return sycl::ext::intel::esimd::convert(s) * dv; + } + }; + + // rows: clamped row index of each of the 16 lanes + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + s.dv = convert(gather(c.dh, (rows * c.bpr + sb) * (uint32_t) sizeof(sycl::half))); + simd raw = xmx_load_t<4, SR>(c.scales, c.ps, c.h, sb * 4, c.row0); + s.sc8 = raw.template bit_cast_view(); + return s; + } + + struct part_t { + simd a, b, h; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; + p.a = xmx_load_t<8, SR>(c.ql, c.pl, c.h, sb * 32 + part * 16, c.row0); + p.b = xmx_load_t<8, SR>(c.ql, c.pl, c.h, sb * 32 + part * 16 + 8, c.row0); + p.h = xmx_load_t<8, SR>(c.qh, c.ph, c.h, sb * 16 + part * 8, c.row0); + return p; + } + + static ESIMD_INLINE simd unpack(const part_t & p, int g, int /*part*/) { + simd qs = (g & 1) ? p.b : p.a; + simd x; + if (g == 0) { + x = (qs & 0x0F0F0F0Fu) | ((p.h << 4) & 0x30303030u); + } else if (g == 1) { + x = (qs & 0x0F0F0F0Fu) | ((p.h << 2) & 0x30303030u); + } else if (g == 2) { + x = ((qs >> 4) & 0x0F0F0F0Fu) | (p.h & 0x30303030u); + } else { + x = ((qs >> 4) & 0x0F0F0F0Fu) | ((p.h >> 2) & 0x30303030u); + } + return xmx_bytes_sub(x, 0x20202020u); + } +}; + +// Q3_K reorder layout: [qs: nb*64] [hmask: nb*32] [scales: nb*12] [d: nb*2] +// chunk s of 32: 2 bits from qs byte 32*(s/4)+l at shift 2*(s%4), high bit is hmask[l] bit s, value is q - 4 +template struct xmx_traits_q3_k : xmx_traits_base { + struct ctx : xmx_ctx_base { + const uint8_t * qs; + const uint8_t * hm; + const uint8_t * sc; + const sycl::half * dh; + unsigned pq, ph; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + const size_t nb = (size_t) nrows * bpr; + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.qs = (const uint8_t *) vx; + c.hm = c.qs + nb * (QK_K / 4); + c.sc = c.hm + nb * (QK_K / 8); + c.dh = (const sycl::half *) (c.sc + nb * 12); + c.pq = xmx_pitch(bpr, QK_K / 4); + c.ph = xmx_pitch(bpr, QK_K / 8); + return c; + } + + using scales_t = xmx_row_scales<16>; + + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + simd bi = rows * c.bpr + sb; + simd w0, w1, w2; + xmx_load_sc12(c.sc, bi, w0, w1, w2); + simd dv = convert(gather(c.dh, bi * (uint32_t) sizeof(sycl::half))); +#pragma unroll + for (int i = 0; i < 16; ++i) { + const simd lo = (i % 8) < 4 ? w0 : w1; + const int bl = i % 4; + simd nib = (lo >> (8 * bl + (i < 8 ? 0 : 4))) & 0xFu; + simd hi = ((w2 >> (8 * bl + 2 * (i / 4))) & 3u) << 4; + simd code = nib | hi; + s.v[i] = convert(code - 32) * dv; + } + return s; + } + + struct part_t { + simd q, h; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; + p.q = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 16 + part * 8, c.row0); + p.h = xmx_load_t<8, SR>(c.hm, c.ph, c.h, sb * 8, c.row0); + return p; + } + + static ESIMD_INLINE simd unpack(const part_t & p, int g, int part) { + const int s = 4 * part + g; + simd x = ((p.q >> (2 * g)) & 0x03030303u) | (((p.h >> s) & 0x01010101u) << 2); + return xmx_bytes_sub(x, 0x04040404u); + } +}; + +// Q4_K and Q5_K (QH) reorder layout: [qs: nb*128] ([qh: nb*32]) [scales: nb*12] [dm: nb*4] +// group j of 32: nibble of qs byte 32*(j/2)+l (low for even j), Q5_K adds 16 * bit j of qh[l] +// value is dall * sc_j * q - dmin * m_j, one scale and one min per group +template struct xmx_traits_q45_k : xmx_traits_base { + struct ctx : xmx_ctx_base { + const uint8_t * qs; + const uint8_t * qh; + const uint8_t * sc; + const sycl::half * dm; + unsigned pq, ph; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + const size_t nb = (size_t) nrows * bpr; + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.qs = (const uint8_t *) vx; + c.qh = c.qs + nb * (QK_K / 2); + c.sc = QH ? c.qh + nb * (QK_K / 8) : c.qh; + c.dm = (const sycl::half *) (c.sc + nb * 12); + c.pq = xmx_pitch(bpr, QK_K / 2); + c.ph = xmx_pitch(bpr, QK_K / 8); + return c; + } + + struct scales_t { + xmx_row_scales<8> d; // dall * scale of each group + xmx_row_scales<8> m; // dmin * min of each group + }; + + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + simd bi = rows * c.bpr + sb; + simd w0, w1, w2; + xmx_load_sc12(c.sc, bi, w0, w1, w2); + simd d, dmin; + xmx_load_dm(c.dm, bi, d, dmin); +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int jj = j % 4; + simd sc, m; + if (j < 4) { + sc = (w0 >> (8 * jj)) & 63u; + m = (w1 >> (8 * jj)) & 63u; + } else { + sc = ((w2 >> (8 * jj)) & 0xFu) | (((w0 >> (8 * jj + 6)) & 3u) << 4); + m = ((w2 >> (8 * jj + 4)) & 0xFu) | (((w1 >> (8 * jj + 6)) & 3u) << 4); + } + s.d.v[j] = convert(simd(sc)) * d; + s.m.v[j] = convert(simd(m)) * dmin; + } + return s; + } + + struct part_t { + simd a, b, h; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; + p.a = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 32 + part * 16, c.row0); + p.b = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 32 + part * 16 + 8, c.row0); + if constexpr (QH) { + p.h = xmx_load_t<8, SR>(c.qh, c.ph, c.h, sb * 8, c.row0); + } + return p; + } + + static ESIMD_INLINE simd unpack(const part_t & p, int g, int part) { + simd qs = (g < 2) ? p.a : p.b; + simd x = (qs >> (4 * (g & 1))) & 0x0F0F0F0Fu; + if constexpr (QH) { + x |= ((p.h >> (4 * part + g)) & 0x01010101u) << 4; + } + return x; + } +}; + +// Q2_K reorder layout: [qs: nb*64] [scales: nb*16] [dm: nb*4] +// chunk s of 32: 2 bits from qs byte 32*(s/4)+l at shift 2*(s%4), each 16 wide half has a 4 bit scale and min +// value is d * (code & 15) * q - dmin * (code >> 4) +template struct xmx_traits_q2_k : xmx_traits_base { + struct ctx : xmx_ctx_base { + const uint8_t * qs; + const uint8_t * sc; + const sycl::half * dm; + unsigned pq, ps; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + const size_t nb = (size_t) nrows * bpr; + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.qs = (const uint8_t *) vx; + c.sc = c.qs + nb * (QK_K / 4); + c.dm = (const sycl::half *) (c.sc + nb * (QK_K / 16)); + c.pq = xmx_pitch(bpr, QK_K / 4); + c.ps = xmx_pitch(bpr, QK_K / 16); + return c; + } + + struct scales_t { + simd code; // 4 bit scale (low) and min (high) of each 16 wide half + simd d, dmin; + + ESIMD_INLINE simd byte(int gr) { return code.template select<16, 4>(64 * (gr / 4) + (gr % 4)); } + }; + + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + simd bi = rows * c.bpr + sb; + simd raw = xmx_load_t<4, SR>(c.sc, c.ps, c.h, sb * 4, c.row0); + s.code = raw.template bit_cast_view(); + xmx_load_dm(c.dm, bi, s.d, s.dmin); + return s; + } + + struct part_t { + simd q; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; + p.q = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 16 + part * 8, c.row0); + return p; + } + + static ESIMD_INLINE simd unpack(const part_t & p, int g, int /*part*/) { + return (p.q >> (2 * g)) & 0x03030303u; + } +}; + +// Q8_0 reorder layout: [qs: nrows*K bytes] [d: one half per 32 weights] +// the int8 weights feed DPAS as they are, a group is one block with one scale +template struct xmx_traits_q8_0 : xmx_traits_base { + struct ctx : xmx_ctx_base { + const uint8_t * qs; + const sycl::half * dh; + unsigned pq; + }; + + static ESIMD_INLINE ctx make(const void * vx, int nrows, int bpr, int row0) { + ctx c; + xmx_ctx_init(c, nrows, bpr, row0); + c.qs = (const uint8_t *) vx; + c.dh = (const sycl::half *) (c.qs + (size_t) nrows * bpr * QK_K); + c.pq = xmx_pitch(bpr, QK_K); + return c; + } + + struct scales_t { + xmx_row_scales<8> d; // scale of each of the 8 blocks of a super-block + }; + + static ESIMD_INLINE scales_t load_scales(const ctx & c, int sb, const simd & rows) { + using namespace sycl::ext::intel::esimd; + scales_t s; + simd blk = (rows * c.bpr + sb) * (QK_K / QK8_0); +#pragma unroll + for (int j = 0; j < 8; ++j) { + s.d.v[j] = convert(gather(c.dh, (blk + j) * (uint32_t) sizeof(sycl::half))); + } + return s; + } + + struct part_t { + simd q[4]; + }; + + static ESIMD_INLINE part_t load_part(const ctx & c, int sb, int part) { + part_t p; +#pragma unroll + for (int g = 0; g < 4; ++g) { + p.q[g] = xmx_load_t<8, SR>(c.qs, c.pq, c.h, sb * 64 + 8 * (4 * part + g), c.row0); + } + return p; + } + + static ESIMD_INLINE simd unpack(const part_t & p, int g, int /*part*/) { return p.q[g]; } +}; + +template using xmx_traits_q4_k = xmx_traits_q45_k; +template using xmx_traits_q5_k = xmx_traits_q45_k; + +// registers per thread, the large file halves the resident threads but avoids spills in wide tiles, the fused +// kernel keeps two sets of weights and scales and always takes it +template constexpr int xmx_grf() { + return FUSED || NC >= T::nc_wide ? 256 : 128; +} + +// One DPAS of a block of RC columns: the int32 products of the weights B and the activations A, as float and scaled +// per weight row. Every quant type sums one or two of these per group of 32. +template +ESIMD_INLINE simd xmx_term(const simd & B, + const simd & A, + const simd & s) { + using namespace sycl::ext::intel::esimd; + namespace xmx = sycl::ext::intel::esimd::xmx; + + simd C = xmx::dpas<8, RC, int>(B, A); + return convert(C) * s.template replicate(); +} + +// one group of 32 of one weight set, accumulated into Cf for all the columns of the tile +template +ESIMD_INLINE void xmx_group(simd & Cf, + const typename T::part_t & pd, + typename T::scales_t & sc, + A2T & A2, + D8T & d8f, + simd & Bl, + simd & Bh, + const simd & Ol, + const int g, + const int part) { + using namespace sycl::ext::intel::esimd; + + constexpr int NT = XMX_ROWS; + constexpr int RC = NC < 8 ? NC : 8; + constexpr int NB = NC / RC; + constexpr bool split = !T::has_min && !T::plain && !T::min16; // two scales per group of 32 + constexpr bool pair = (RC * 32) % 64 == 0; // a block of RC rows fills whole registers + + const int gi = 4 * part + g; + + simd x = T::unpack(pd, g, part); + simd B = x.template bit_cast_view(); + if constexpr (split) { + Bl.template select<256, 1>(0) = B.template select<256, 1>(0); + Bh.template select<256, 1>(256) = B.template select<256, 1>(256); + } + + simd rd; + simd rm; + simd rs0; + simd rs1; + + // Q2_K: scale and min of each 16 wide half are 4 bit codes, the weight is d * scl * q - dmin * mc. scl * q + // is at most 45, so a dword multiply keeps the bytes apart and the scaled weights go to DPAS as int8. The + // min term sums a * mc over K, the codes mc take the place of the weights in a second DPAS. + simd Bs; + simd Bm; + if constexpr (T::min16) { + const simd c0 = sc.byte(2 * gi); + const simd c1 = sc.byte(2 * gi + 1); + simd sl0 = convert(simd(c0 & (uint8_t) 15)); + simd sl1 = convert(simd(c1 & (uint8_t) 15)); + simd mc0 = convert(simd(c0 >> (uint8_t) 4)) * 0x01010101u; + simd mc1 = convert(simd(c1 >> (uint8_t) 4)) * 0x01010101u; + simd ws; + simd wm; + ws.template select<64, 1>(0) = x.template select<64, 1>(0) * sl0.template replicate<4>(); + ws.template select<64, 1>(64) = x.template select<64, 1>(64) * sl1.template replicate<4>(); + wm.template select<64, 1>(0) = mc0.template replicate<4>(); + wm.template select<64, 1>(64) = mc1.template replicate<4>(); + Bs = ws.template bit_cast_view(); + Bm = wm.template bit_cast_view(); + } + if constexpr (T::plain) { + rd = sc.d.get(gi); + } else if constexpr (T::has_min) { + rd = sc.d.get(gi); + rm = sc.m.get(gi); + } else if constexpr (T::min16) { + // scales are applied through Bs and Bm + } else { + rs0 = sc.get(2 * gi); + rs1 = sc.get(2 * gi + 1); + } + +#pragma unroll + for (int b = 0; b < NB; ++b) { + simd A = + A2[b].template select(pair ? (g % 2) * RC * 8 : 0).template bit_cast_view(); + simd d8r = + d8f[b].template select(gi).template replicate_vs_w_hs(0); + auto Cb = Cf.template select(b * RC * NT); + + simd z; + if constexpr (T::plain) { + z = xmx_term(B, A, rd); + } else if constexpr (T::has_min) { + // the second DPAS (all ones against the activations) is the exact integer sum of the activations + z = xmx_term(B, A, rd) - xmx_term(Ol, A, rm); + } else if constexpr (T::min16) { + z = xmx_term(Bs, A, sc.d) - xmx_term(Bm, A, sc.dmin); + } else { + z = xmx_term(Bl, A, rs0) + xmx_term(Bh, A, rs1); + } + Cb += z * d8r; + } +} + +// activation of the GLU applied to the gate, uniform across the launch +template ESIMD_INLINE simd xmx_glu_act(const simd & gate, const int glu_op) { + using namespace sycl::ext::intel::esimd; + if (glu_op == GGML_GLU_OP_SWIGLU) { + return gate / (1.0f + exp(-gate)); + } + // GEGLU, the tanh is written with exp + simd a = 0.79788456f * gate * (1.0f + 0.044715f * gate * gate); + simd t = 1.0f - 2.0f / (exp(2.0f * a) + 1.0f); + return 0.5f * gate * (1.0f + t); +} + +// activations come in through 2D block loads, one for the (d, s) pairs of a super-block (s is unused) and one per two +// groups for the quants, the DPAS A operand needs no repacking. Groups with two scales (16 wide) mask the +// weights instead, zeros in B drop the other half of K. +// KS is the most threads that split K for one tile, the launch may use fewer. +// FUSED computes up * act(gate) from the weights vx (up) and vg (gate), reading the activations once. +template +ESIMD_INLINE void xmx_mul_mat(const void * vx, + const void * vg, + const char * vy, + float * dst, + const int ncols, + const int nrows, + const int ncv, + const int stride_y_bytes, + const int ldd, + const int glu_op, + const int tile, + const int lid, + const int ks) { + using namespace sycl::ext::intel::esimd; + + constexpr int NT = XMX_ROWS; + constexpr int RC = NC < 8 ? NC : 8; + constexpr int NB = NC / RC; + constexpr int N_ACC = FUSED ? 2 : 1; + constexpr bool pair = (RC * 32) % 64 == 0; // a block of RC rows fills whole registers + + if constexpr (KS > 1) { + slm_init(); + } + + const int bpr = ncols / QK_K; + const int row0 = tile * NT; + const auto c = T::make(vx, nrows, bpr, row0); + const auto cg = T::make(FUSED ? vg : vx, nrows, bpr, row0); + + simd rows; +#pragma unroll + for (int n = 0; n < NT; ++n) { + rows[n] = row0 + n < nrows ? row0 + n : nrows - 1; + } + + const uint32_t * yq = (const uint32_t *) vy; + const uint32_t * yd = (const uint32_t *) (vy + ncols); + const unsigned yqw = ncols - 1; + const unsigned ydw = ncols / QK8_1 * sizeof(sycl::half2) - 1; + const unsigned yh = ncv - 1; + const unsigned yp = stride_y_bytes - 1; + + simd Cf = 0.0f; + simd Cfg = 0.0f; // gate, only used when fused + simd Bl = int8_t(0); + simd Bh = int8_t(0); + simd Ol = int8_t(0); + if constexpr (T::has_min) { + Ol = int8_t(1); + } + + for (int sb = lid; sb < bpr; sb += ks) { + auto sc = T::load_scales(c, sb, rows); + typename T::scales_t scg; + if constexpr (FUSED) { + scg = T::load_scales(cg, sb, rows); + } + + // d of the 8 groups of the super-block, [column * 8 + group] + simd d8f[NB]; +#pragma unroll + for (int b = 0; b < NB; ++b) { + simd ds = load_2d(yd, ydw, yh, yp, sb * 8, b * RC); + simd dh = ds.template bit_cast_view(); + d8f[b] = convert(simd(dh.template select<8 * RC, 2>(0))); + } + +#pragma unroll + for (int part = 0; part < 2; ++part) { + const auto pd = T::load_part(c, sb, part); + typename T::part_t pdg; + if constexpr (FUSED) { + pdg = T::load_part(cg, sb, part); + } + + simd A2[NB]; // quants of two groups + +#pragma unroll + for (int g = 0; g < 4; ++g) { + const int gi = 4 * part + g; + if constexpr (pair) { + if (g % 2 == 0) { +#pragma unroll + for (int b = 0; b < NB; ++b) { + A2[b] = + load_2d(yq, yqw, yh, yp, sb * 64 + gi * 8, b * RC); + } + } + } else { +#pragma unroll + for (int b = 0; b < NB; ++b) { + A2[b].template select(0) = + load_2d(yq, yqw, yh, yp, sb * 64 + gi * 8, b * RC); + } + } + + xmx_group(Cf, pd, sc, A2, d8f, Bl, Bh, Ol, g, part); + if constexpr (FUSED) { + xmx_group(Cfg, pdg, scg, A2, d8f, Bl, Bh, Ol, g, part); + } + } + } + } + + if constexpr (KS > 1) { + slm_block_store(lid * NC * NT * sizeof(float), Cf); + if constexpr (FUSED) { + slm_block_store((KS + lid) * NC * NT * sizeof(float), Cfg); + } + barrier(); + if (lid != 0) { + return; + } + Cf = 0.0f; + Cfg = 0.0f; +#pragma unroll 4 + for (int j = 0; j < ks; ++j) { + Cf += slm_block_load(j * NC * NT * sizeof(float)); + if constexpr (FUSED) { + Cfg += slm_block_load((KS + j) * NC * NT * sizeof(float)); + } + } + } + + if constexpr (FUSED) { + Cf *= xmx_glu_act(Cfg, glu_op); + } + + if (row0 + NT <= nrows && ldd % 4 == 0 && (uintptr_t) dst % 16 == 0) { +#pragma unroll + for (int m = 0; m < NC; ++m) { + if (NC <= 8 || m < ncv) { + block_store(dst + (size_t) m * ldd + row0, Cf.template select(m * NT)); + } + } + } else { + // tail tile or unaligned output + simd lane(0, 1); + simd_mask in_rows = (lane + row0) < (uint32_t) nrows; +#pragma unroll + for (int m = 0; m < NC; ++m) { + if (NC <= 8 || m < ncv) { + scatter(dst + (size_t) m * ldd + row0, lane * (uint32_t) sizeof(float), + Cf.template select(m * NT), in_rows); + } + } + } +} + +// KS_MAX bounds the threads that split K for one tile, the launch uses as few as keep every thread to the same +// number of super-blocks as the busiest one +template +static void xmx_mul_mat_launch_ks(const void * vx, + const void * vg, + const char * vy, + float * dst, + const int ncols, + const int nrows, + const int ncv, + const int stride_y_bytes, + const int ldd, + const int glu_op, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const size_t n_tiles = (nrows + XMX_ROWS - 1) / XMX_ROWS; + const int bpr = ncols / QK_K; + const int rounds = (bpr + KS_MAX - 1) / KS_MAX; + const int ks = (bpr + rounds - 1) / rounds; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(n_tiles * ks), sycl::range<1>(ks)), + sycl::ext::oneapi::experimental::properties{ + sycl::ext::intel::experimental::grf_size()> }, + [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { + xmx_mul_mat(vx, vg, vy, dst, ncols, nrows, ncv, stride_y_bytes, ldd, + glu_op, (int) it.get_group(0), (int) it.get_local_id(0), + ks); + }); + }); +} + +template +static void xmx_mul_mat_launch(const void * vx, + const void * vg, + const char * vy, + float * dst, + const int ncols, + const int nrows, + const int ncv, + const int stride_y_bytes, + const int ldd, + const int glu_op, + dpct::queue_ptr stream) { + // the fused kernel keeps two sets of accumulators in shared memory, half the threads for the same size + constexpr int ks_max = xmx_n_split() / (FUSED ? 2 : 1); + xmx_mul_mat_launch_ks(vx, vg, vy, dst, ncols, nrows, ncv, stride_y_bytes, ldd, glu_op, + stream); +} + +// FUSED: vx and vg are the up and gate weights, dst gets up * act(gate) with glu_op a ggml_glu_op +template +static void xmx_mul_mat_ncols(const void * vx, + const void * vg, + const char * vy, + float * dst, + const int ncols, + const int nrows, + const int ncols_dst, + const int stride_y_bytes, + const int ldd, + const int glu_op, + dpct::queue_ptr stream) { + constexpr int chunk_max = FUSED ? XMX_FUSED_CHUNK : XMX_CHUNK; + for (int c0 = 0; c0 < ncols_dst; c0 += chunk_max) { + const int n = std::min(chunk_max, ncols_dst - c0); + const char * y = vy + (size_t) c0 * stride_y_bytes; + float * d = dst + (size_t) c0 * ldd; + const int nc = n > 32 ? 64 : (n > 16 ? 32 : (n > 8 ? 16 : n)); + switch (nc) { +#define XMX_CASE(N) \ + case N: \ + xmx_mul_mat_launch(vx, vg, y, d, ncols, nrows, n, stride_y_bytes, ldd, glu_op, stream); \ + break + XMX_CASE(1); + XMX_CASE(2); + XMX_CASE(3); + XMX_CASE(4); + XMX_CASE(5); + XMX_CASE(6); + XMX_CASE(7); + XMX_CASE(8); +#undef XMX_CASE + // short row variants are for tests, not built for the wide tiles +#define XMX_CASE_WIDE(N) \ + case N: \ + if constexpr (!T::short_rows && (!FUSED || N <= XMX_FUSED_CHUNK)) { \ + xmx_mul_mat_launch(vx, vg, y, d, ncols, nrows, n, stride_y_bytes, ldd, glu_op, stream); \ + } \ + break + XMX_CASE_WIDE(16); + XMX_CASE_WIDE(32); + XMX_CASE_WIDE(64); +#undef XMX_CASE_WIDE + default: + GGML_ABORT("unsupported ncols_dst=%d for XMX MMV", n); + } + } +} + +} // namespace ggml_sycl_esimd + +#endif // GGML_SYCL_ESIMD_HPP diff --git a/ggml/src/ggml-sycl/fattn-common.hpp b/ggml/src/ggml-sycl/fattn-common.hpp index ed00d03..c373263 100644 --- a/ggml/src/ggml-sycl/fattn-common.hpp +++ b/ggml/src/ggml-sycl/fattn-common.hpp @@ -1,10 +1,12 @@ #pragma once #include +#include #include "dpct/helper.hpp" #include "common.hpp" #include "convert.hpp" #include "vecdotq.hpp" +#include "fattn.hpp" #include "ggml.h" @@ -830,7 +832,30 @@ static void flash_attn_combine_results(const float * __restrict__ VKQ_parts, dst[tid] = VKQ_numerator / VKQ_denominator; } -template +// Carries the sub-group size (and grf_size_automatic when use_large_grf) as +// kernel properties via get(properties_tag), in place of the +// reqd_sub_group_size attribute and the deprecated parallel_for overload. +template +struct props_kernel { + KernelFunc kernel; + + void operator()(sycl::nd_item<3> item) const { + kernel(item); + } + + auto get(sycl::ext::oneapi::experimental::properties_tag) const { + if constexpr (UseGrf) { + return sycl::ext::oneapi::experimental::properties{ + sycl::ext::intel::experimental::grf_size_automatic, + sycl::ext::oneapi::experimental::sub_group_size }; + } else { + return sycl::ext::oneapi::experimental::properties{ + sycl::ext::oneapi::experimental::sub_group_size }; + } + } +}; + +template static void lauch_kernel( dpct::dim3 group_range, dpct::dim3 local_range, @@ -874,23 +899,28 @@ static void lauch_kernel( const int32_t nb32, const int64_t nb33) { GGML_UNUSED(local_mem_size); + + const auto rng = sycl::nd_range<3>( + static_cast>(group_range * local_range), + static_cast>(local_range)); + + const auto kernel = [=](sycl::nd_item<3> item_ct1) { + GGML_UNUSED(item_ct1); + fattn_kernel(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, + max_bias, m0, m1, n_head_log2, logit_softcap, ne00, + ne01, ne02, ne03, nb01, nb02, nb03, ne10, ne11, + ne12, ne13, nb11, nb12, nb13, nb21, nb22, nb23, + ne31, ne32, ne33, nb31, nb32, nb33); + }; + q->submit([&](sycl::handler &cgh) { - cgh.parallel_for( - sycl::nd_range<3>( - static_cast>(group_range * local_range), - static_cast>(local_range)), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] { - GGML_UNUSED(item_ct1); - fattn_kernel(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale, - max_bias, m0, m1, n_head_log2, logit_softcap, ne00, - ne01, ne02, ne03, nb01, nb02, nb03, ne10, ne11, - ne12, ne13, nb11, nb12, nb13, nb21, nb22, nb23, - ne31, ne32, ne33, nb31, nb32, nb33); - }); + // grf_size_automatic lets the compiler use the large register file when the kernel needs it. grf_size<256> would say the same but is undefined on devices that do not have it. + cgh.parallel_for(rng, props_kernel{kernel}); }); } -template +// use_large_grf lets the compiler choose the large register file. Opt in where spilling limits the kernel, not thread occupancy. +template void launch_fattn( ggml_backend_sycl_context & ctx, ggml_tensor * dst, const int nwarps, const size_t nbytes_shared, const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k) { @@ -922,8 +952,7 @@ void launch_fattn( const int id = ggml_sycl_get_device(); const int nsm = ggml_sycl_info().devices[id].nsm; - ggml_sycl_pool_alloc K_f16(pool); - ggml_sycl_pool_alloc V_f16(pool); + const ggml_sycl_fattn_extra extra = ggml_sycl_fattn_get_extra(dst); ggml_sycl_pool_alloc KV_max(pool); ggml_sycl_pool_alloc dst_tmp(pool); ggml_sycl_pool_alloc dst_tmp_meta(pool); @@ -942,10 +971,11 @@ void launch_fattn( const size_t bs = ggml_blck_size(K->type); const size_t ts = ggml_type_size(K->type); - K_f16.alloc(ggml_nelements(K)); + GGML_ASSERT(extra.K_buffer_ptr); + sycl::half * K_f16_ptr = (sycl::half *) extra.K_buffer_ptr; if (ggml_is_contiguously_allocated(K)) { to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(K->type, dst); - to_fp16(K_data, K_f16.ptr, ggml_nelements(K), main_stream); + to_fp16(K_data, K_f16_ptr, ggml_nelements(K), main_stream); nb11 = nb11 * bs * sizeof(sycl::half) / ts; nb12 = nb12 * bs * sizeof(sycl::half) / ts; @@ -956,13 +986,13 @@ void launch_fattn( const int64_t s01 = nb11 / ts; const int64_t s02 = nb12 / ts; const int64_t s03 = nb13 / ts; - to_fp16(K_data, K_f16.ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], s01, s02, s03, main_stream); + to_fp16(K_data, K_f16_ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], s01, s02, s03, main_stream); nb11 = K->ne[0] * sizeof(sycl::half); nb12 = K->ne[1] * nb11; nb13 = K->ne[2] * nb12; } - K_data = (char *) K_f16.ptr; + K_data = (char *) K_f16_ptr; } if (need_f16_V && V->type != GGML_TYPE_F16) { @@ -975,11 +1005,12 @@ void launch_fattn( const size_t bs = ggml_blck_size(V->type); const size_t ts = ggml_type_size(V->type); - V_f16.alloc(ggml_nelements(V)); + GGML_ASSERT(extra.V_buffer_ptr); + sycl::half * V_f16_ptr = (sycl::half *) extra.V_buffer_ptr; if (ggml_is_contiguously_allocated(V)) { to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(V->type, dst); - to_fp16(V_data, V_f16.ptr, ggml_nelements(V), main_stream); - V_data = (char *) V_f16.ptr; + to_fp16(V_data, V_f16_ptr, ggml_nelements(V), main_stream); + V_data = (char *) V_f16_ptr; nb21 = nb21 * bs * sizeof(sycl::half) / ts; nb22 = nb22 * bs * sizeof(sycl::half) / ts; @@ -990,13 +1021,13 @@ void launch_fattn( const int64_t s01 = nb21 / ts; const int64_t s02 = nb22 / ts; const int64_t s03 = nb23 / ts; - to_fp16(V_data, V_f16.ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], s01, s02, s03, main_stream); + to_fp16(V_data, V_f16_ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], s01, s02, s03, main_stream); nb21 = V->ne[0] * sizeof(sycl::half); nb22 = V->ne[1] * nb21; nb23 = V->ne[2] * nb22; } - V_data = (char *) V_f16.ptr; + V_data = (char *) V_f16_ptr; } } @@ -1029,7 +1060,7 @@ void launch_fattn( auto KV_max_ptr_ct1 = KV_max.ptr; cgh.parallel_for(sycl::nd_range<3>(blocks_num_KV_max * block_dim_KV_max, block_dim_KV_max), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] { GGML_UNUSED(item_ct1); flash_attn_mask_to_KV_max( mask_data_ct0, KV_max_ptr_ct1, iter_k, s31, s33, @@ -1122,7 +1153,7 @@ void launch_fattn( GGML_ASSERT(block_dim.x % warp_size == 0); - lauch_kernel( + lauch_kernel( blocks_num, block_dim, main_stream, (unsigned int) nbytes_shared, (const char *) Q->data, K_data, V_data, mask ? ((const char *) mask->data) : nullptr, sinks ? ((const char *) sinks->data) : nullptr, KV_max.ptr, !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, (sycl::float2 *)dst_tmp_meta.ptr, scale, max_bias, m0, m1, @@ -1147,7 +1178,7 @@ void launch_fattn( auto K_ne_ct6 = K->ne[2]; cgh.parallel_for(sycl::nd_range<3>(blocks_num_combine * block_dim_combine, block_dim_combine), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] { GGML_UNUSED(item_ct1); flash_attn_stream_k_fixup(KQV_data_ct0, dst_tmp_meta_ptr_ct1, Q_ne_ct2, Q_ne_ct3, Q_ne_ct4, @@ -1167,7 +1198,7 @@ void launch_fattn( auto KQV_data_ct2 = (float *) KQV->data; cgh.parallel_for(sycl::nd_range<3>(blocks_num_combine * block_dim_combine, block_dim_combine), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(warp_size)]] { GGML_UNUSED(item_ct1); flash_attn_combine_results( dst_tmp_ptr_ct0, dst_tmp_meta_ptr_ct1, KQV_data_ct2, parallel_blocks, diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cpp new file mode 100644 index 0000000..497815b --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-mkl.cpp @@ -0,0 +1,691 @@ +// Flash attention via oneMKL GEMM (XMX-accelerated). +// Uses column_major::gemm for Q*K^T and S*V matmuls +// with an online softmax SYCL kernel. +// +// All GQA query heads sharing a KV head are batched into single +// GEMM calls, amortizing MKL launch overhead across K and V reuse. +// + +#include "common.hpp" +#include "fattn-common.hpp" +#include "convert.hpp" +#include "fattn.hpp" + +#include +#include +#include +#include + +#define MKL_FA_CHUNK_SIZE_KV 8192 + +// Number of query rows processed per tile. The score buffers (KQ_f32, S_f16) +// are sized q_tile_rows * chunk_size, so this bounds their footprint +// regardless of batch size (n_query_rows = n_queries * gqa_ratio). A typical +// single-ubatch prefill (e.g. ubatch 1024 * gqa 8 = 8192 rows) is exactly one +// tile, so it runs with no extra iterations. Larger batches tile and stay +// bounded. Override with GGML_SYCL_MKL_FA_Q_TILE. +#define MKL_FA_Q_TILE 8192 + +#define MKL_FA_WG_SIZE 256 + +using oneapi::mkl::transpose; +using oneapi::mkl::blas::column_major::gemm; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// Pack all GQA Q heads for one KV head into fp16, applying q_scale. +// Launches one kernel per GQA group — each kernel copies exactly +// n_queries * DKQ elements using the per-group dst offset and +// per-head source stride. +static void mkl_fa_pack_q_fp16( + dpct::queue_ptr stream, + sycl::half * __restrict dst, + const float * __restrict q_src, + int n_queries, int DKQ, + int gqa_ratio, int kvh_base_head, + float q_scale, int64_t q_row_stride, int64_t q_head_stride, + int64_t wg_size) { + + for (int iqg = 0; iqg < gqa_ratio; iqg++) { + int iqh = kvh_base_head + iqg; + sycl::half * dst_g = dst + (int64_t)iqg * n_queries * DKQ; + + const int64_t n_elem = (int64_t)n_queries * DKQ; + const int64_t wg = ((n_elem + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int64_t e = item.get_global_id(0); + if (e >= n_elem) return; + + int64_t q = e / DKQ; + int64_t d = e - q * DKQ; + + // Stride-aware source offset: handles permuted, + // sliced, or contiguous Q tensor layouts. + int64_t src_off = d + + q * q_row_stride + + (int64_t)iqh * q_head_stride; + + dst_g[e] = sycl::half( + q_src[src_off] * q_scale); + }); + }); + } +} + +// Zero-initialize the online softmax state arrays. +// KQ_max → -inf, KQ_sum → 0, VKQ_accum → 0. +// Merged into one kernel to avoid per-array launch overhead. +static void mkl_fa_init_softmax_state( + dpct::queue_ptr stream, + float * kmax, float * ksum, float * vacc, + int n_query_rows, int DV, int64_t wg_size) { + + const float neg_inf = -1e30f; + const int64_t n_maxsum = n_query_rows; + const int64_t n_vacc = (int64_t)n_query_rows * DV; + const int64_t total = (n_vacc > n_maxsum) ? n_vacc : n_maxsum; + const int64_t wg = ((total + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int64_t i = item.get_global_id(0); + if (i < n_maxsum) { + kmax[i] = neg_inf; + ksum[i] = 0.0f; + } + if (i < n_vacc) { + vacc[i] = 0.0f; + } + }); + }); +} + +// Online softmax over one KV chunk for a tile of GQA query rows. +// The tile spans absolute rows [q0, q0 + q_rows). Score buffers +// (KQ_f32/S_f16) are indexed RELATIVE to the tile; the persistent state +// (VKQ_accum/KQ_max/KQ_sum) and mask are indexed by ABSOLUTE row. +// One WORK-GROUP per query row (local size = wg_size): work-items stride +// over the chunk so adjacent items touch adjacent elements (coalesced), +// the row max/sum come from group reductions, and the DV-long VKQ +// rescale is spread across the items. Item 0 is the sole writer of +// KQ_max/KQ_sum; its writes are ordered after every other item's reads +// by the second group reduction (a collective). Per-element math is +// identical to the original one-item-per-row kernel: softcap before +// mask, native::exp, -1e30 sentinel, half-precision S. Only the float +// summation order differs (tree vs serial), i.e. last-ulp level. +static void mkl_fa_online_softmax_chunk( + dpct::queue_ptr stream, + float * __restrict KQ_f32, + sycl::half * __restrict S_f16, + float * __restrict KQ_max, + float * __restrict KQ_sum, + float * __restrict VKQ_accum, + int q0, int q_rows, int n_queries, int DV, + int chunk_size, int chunk_start, + int kvh_head, + const sycl::half * mask_data, int64_t mask_head_stride, + int64_t mask_row_stride, int mask_n_heads, + float logit_softcap, int64_t wg_size, + const sycl::event & dep_event) { + + // One work-group per query row: exactly q_rows groups of wg_size + // items. q_rows * wg_size is already a multiple of wg_size, so unlike + // the one-item-per-row kernels there is no round-up / tail guard. + const int64_t wg = q_rows * wg_size; + const int local_size = (int) wg_size; // stride in the loops below + stream->submit([&](sycl::handler & cgh) { + cgh.depends_on(dep_event); + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + const int local_id = (int)item.get_local_id(0); + const int row = (int)item.get_group(0); // tile-relative + const int jc_abs = q0 + row; + const int gqa_group = jc_abs / n_queries; + const int q_row = jc_abs % n_queries; + // Score buffers are tile-local (relative index). + const float * __restrict KQ_row = KQ_f32 + + row * (int64_t)chunk_size; + sycl::half * __restrict S_row = S_f16 + + row * (int64_t)chunk_size; + // Persistent accumulator is full-sized (absolute index). + float * __restrict vkq = VKQ_accum + + jc_abs * (int64_t)DV; + const sycl::half * mask_h = nullptr; + int64_t m_stride = 0; + if (mask_data) { + int m_head = (mask_n_heads > 1) + ? (kvh_head + gqa_group) : 0; + mask_h = mask_data + (int64_t)m_head * mask_head_stride; + m_stride = mask_row_stride; + } + // Score at chunk offset i — original per-element math. + auto score = [&](int i) { + float s = KQ_row[i]; + if (logit_softcap != 0.0f) { + s = logit_softcap * sycl::tanh(s); + } + if (mask_h) { + s += (float)mask_h[q_row * m_stride + + (chunk_start + i)]; + } + return s; + }; + // Pass 1: strided (coalesced) row-wise local maximum. + float local_max = -1e30f; + for (int i = local_id; i < chunk_size; i += local_size) { + float s = score(i); + if (s > local_max) local_max = s; + } + const float final_local_max = sycl::reduce_over_group( + item.get_group(), local_max, sycl::maximum()); + // Rescale previous accumulator by exp(old_max - new_max) + float old_max = KQ_max[jc_abs]; + float new_max = (old_max > final_local_max) ? old_max : final_local_max; + float rescale = (old_max < -1e29f) ? 1.0f + : sycl::native::exp(old_max - new_max); + for (int v = local_id; v < DV; v += local_size) { + vkq[v] *= rescale; + } + // Pass 2: softmax numerators, strided; S row written once. + float local_sum = 0.0f; + for (int i = local_id; i < chunk_size; i += local_size) { + float s = score(i); + float val = sycl::native::exp(s - new_max); + S_row[i] = sycl::half(val); + local_sum += val; + } + const float total_sum = sycl::reduce_over_group( + item.get_group(), local_sum, sycl::plus()); + if (local_id == 0) { + KQ_sum[jc_abs] = KQ_sum[jc_abs] * rescale + total_sum; + KQ_max[jc_abs] = new_max; + } + }); + }); +} + +// Write one GQA group's normalized output to its destination head. +static void mkl_fa_normalize_head( + dpct::queue_ptr stream, + float * __restrict dst_batch, + const float * __restrict VKQ_accum, + const float * __restrict KQ_sum, + int iqh, int n_queries, int DV, int n_q_heads, + int64_t src_offset, int64_t wg_size) { + + const int64_t wg = ((n_queries + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int jc = item.get_global_id(0); + if (jc >= n_queries) return; + + int ksum_idx = (int)(src_offset / DV) + jc; + float inv_sum = 1.0f / KQ_sum[ksum_idx]; + const float * __restrict src = VKQ_accum + + src_offset + jc * (int64_t)DV; + // Interleaved dst layout (matching TILE): + // rows alternate between heads, then increment query. + // offset = (query * n_q_heads + head) * DV + float * __restrict dst_row = dst_batch + + ((int64_t)jc * n_q_heads + iqh) * (int64_t)DV; + + for (int v = 0; v < DV; v++) { + dst_row[v] = src[v] * inv_sum; + } + }); + }); +} + +// --------------------------------------------------------------------------- +// Per-chunk dequant +// +// Rather than dequantizing all of K/V up front (footprint scales with +// context), we dequant one KV-head chunk at a time into a dense +// [this_chunk x D] fp16 buffer (row-major, lda = D). The source address of +// element (head=ikvh, row=chunk_start+r, col=c) decomposes into independent +// linear terms head_off(ikvh) + row_off(chunk_start) + (r,c), so slicing a +// chunk is a clean pointer offset in every layout case. The true-Gemma- +// interleave vs padded-seq-view distinction is resolved once when the +// descriptor is built; slicing does not reintroduce it. +// --------------------------------------------------------------------------- +enum mkl_fa_kv_desc_mode { + MKL_FA_KV_MODE_F16_DENSE = 0, + MKL_FA_KV_MODE_F16_INTERLEAVED = 1, + MKL_FA_KV_MODE_QUANT_CONTIG = 2, + MKL_FA_KV_MODE_QUANT_NC = 3, +}; + +struct mkl_fa_kv_desc { + const char * data = nullptr; + ggml_type type = GGML_TYPE_F16; + int64_t D = 0; // ne[0] + int64_t nb1 = 0; // byte stride, seq dim + int64_t nb2 = 0; // byte stride, head dim + mkl_fa_kv_desc_mode mode = MKL_FA_KV_MODE_F16_DENSE; + int64_t ts = 0; // type size (mode 3 base offset) + int64_t s01 = 0; // nc row stride in blocks (mode 3) + int64_t s02 = 0; // nc head stride in blocks (mode 3) +}; + +static mkl_fa_kv_desc mkl_fa_make_desc(const ggml_tensor * T, bool interleaved, int n_kv_heads) { + mkl_fa_kv_desc d; + d.data = (const char *)T->data; + d.type = T->type; + d.D = T->ne[0]; + d.nb1 = (int64_t)T->nb[1]; + d.nb2 = (int64_t)T->nb[2]; + d.ts = (int64_t)ggml_type_size(T->type); + + if (T->type == GGML_TYPE_F16) { + // MLA's V cache is a 512-wide view of 576-wide K rows. Treat any + // padded row stride as strided even when there is only one KV head. + d.mode = interleaved || d.nb1 != d.D * (int64_t)sizeof(sycl::half) + ? MKL_FA_KV_MODE_F16_INTERLEAVED : MKL_FA_KV_MODE_F16_DENSE; + } else if (ggml_is_contiguously_allocated(T) && !interleaved) { + d.mode = MKL_FA_KV_MODE_QUANT_CONTIG; + } else { + d.mode = MKL_FA_KV_MODE_QUANT_NC; + const int64_t bs = (int64_t)ggml_blck_size(T->type); + const int64_t blk_per_row = T->ne[0] / bs; + // True Gemma interleave packs heads within a row (nb[2] < ne[1]*nb[1]) + // → reconstruct physical strides. Padded seq-views (nb[2] > ne[1]*nb[1]) + // already have correct physical strides. + const bool gemma = interleaved && + ((int64_t)T->nb[2] < (int64_t)T->ne[1] * (int64_t)T->nb[1]); + if (gemma) { + d.s01 = (int64_t)n_kv_heads * blk_per_row; + d.s02 = blk_per_row; + } else { + d.s01 = d.nb1 / d.ts; + d.s02 = d.nb2 / d.ts; + } + } + return d; +} + +// Dequant one KV-head chunk into a dense [this_chunk x D] fp16 buffer. +static void mkl_fa_dequant_chunk( + dpct::queue_ptr stream, const mkl_fa_kv_desc & d, ggml_tensor * dst_ctx, + sycl::half * out, int ikvh, int chunk_start, int this_chunk) { + + const int64_t D = d.D; + switch (d.mode) { + case MKL_FA_KV_MODE_F16_DENSE: { + const char * base = d.data + (int64_t)ikvh * d.nb2 + + (int64_t)chunk_start * d.nb1; + stream->memcpy(out, base, (size_t)this_chunk * D * sizeof(sycl::half)); + break; + } + case MKL_FA_KV_MODE_F16_INTERLEAVED: { + const char * base = d.data + (int64_t)ikvh * d.nb2 + + (int64_t)chunk_start * d.nb1; + const int64_t row_halfs = d.nb1 / (int64_t)sizeof(sycl::half); + const sycl::half * src = (const sycl::half *)base; + stream->parallel_for( + sycl::range<2>((size_t)this_chunk, (size_t)D), + [=](sycl::item<2> it) { + int64_t r = it.get_id(0); + int64_t c = it.get_id(1); + out[r * D + c] = src[r * row_halfs + c]; + }); + break; + } + case MKL_FA_KV_MODE_QUANT_CONTIG: { + const char * base = d.data + (int64_t)ikvh * d.nb2 + + (int64_t)chunk_start * d.nb1; + to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(d.type, dst_ctx); + to_fp16(base, out, (int64_t)this_chunk * D, stream); + break; + } + default: { // MKL_FA_KV_MODE_QUANT_NC + to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(d.type); + const int64_t base_blocks = (int64_t)ikvh * d.s02 + + (int64_t)chunk_start * d.s01; + const char * base = d.data + base_blocks * d.ts; + // ne02 = ne03 = 1 → s02/s03 inert; head+chunk offset carried by base. + to_fp16(base, out, D, this_chunk, 1, 1, d.s01, d.s02, d.s02, stream); + break; + } + } +} + +// --------------------------------------------------------------------------- +// MKL Flash Attention orchestrator +// +// Pipeline: dequantize K/V → for each KV head: +// pack GQA Q heads → MKL GEMM KQ → online softmax → +// MKL GEMM VKQ → accumulate → normalize → scatter to dst +// --------------------------------------------------------------------------- +void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + ggml_tensor * KQV = dst; + + GGML_ASSERT(Q->type == GGML_TYPE_F32); + GGML_ASSERT(KQV->type == GGML_TYPE_F32); + + // --- Op params --- + float scale = 1.0f, max_bias = 0.0f, logit_softcap = 0.0f; + memcpy(&scale, (const float *)KQV->op_params + 0, sizeof(float)); + memcpy(&max_bias, (const float *)KQV->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *)KQV->op_params + 2, sizeof(float)); + + const float q_scale = scale; + + // --- Dimensions --- + const int DKQ = (int)K->ne[0]; + const int DV = (int)V->ne[0]; + const int n_queries = (int)Q->ne[1]; + const int n_q_heads = (int)Q->ne[2]; + const int n_kv_heads = (int)K->ne[2]; + const int n_batch = (int)Q->ne[3]; + const int n_kv = (int)K->ne[1]; + const int gqa_ratio = n_q_heads / n_kv_heads; + const int n_query_rows = n_queries * gqa_ratio; + + GGML_ASSERT(n_q_heads % n_kv_heads == 0); + GGML_ASSERT(max_bias == 0.0f); // ALiBi not supported + GGML_ASSERT(Q->ne[3] == K->ne[3] || K->ne[3] == 1); + + const int chunk_size = std::min(MKL_FA_CHUNK_SIZE_KV, n_kv); + + // Query rows are processed in tiles of q_tile_rows so the score buffers + // (KQ_f32/S_f16 = q_tile_rows * chunk_size) stay bounded regardless of + // batch size. n_query_rows <= Q_TILE is a single tile (no extra work). + static int q_tile_env = ggml_sycl_get_env("GGML_SYCL_MKL_FA_Q_TILE", MKL_FA_Q_TILE); + const int q_tile_rows = std::max(1, std::min(q_tile_env, n_query_rows)); + + const int64_t wg_size = MKL_FA_WG_SIZE; + + // --- Debug output (gated by GGML_SYCL_MKL_FA_DEBUG=1) --- + static int mkl_call_count = 0; + mkl_call_count++; + static int mkl_debug = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DEBUG", 0); + const bool do_print = (mkl_debug == 1); + + const int64_t q_row_stride = Q->nb[1] / sizeof(float); + const int64_t q_head_stride = Q->nb[2] / sizeof(float); + + // Alias the dequantized buffers only when K and V expose the same values. + // MLA V is a narrower view of K and needs its own strided dequantization. + const bool V_is_K_view = V->ne[0] == K->ne[0] && V->view_src + && (V->view_src == K || (V->view_src == K->view_src + && V->view_offs == K->view_offs)); + + // Early interleaved detection for debug output. + // True interleaved detection happens after dequant (nb12_fp16 == nb11_fp16), + // but we can pre-detect on the original tensor strides. + const bool k_early_interleaved = + ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]); + const bool v_early_interleaved = + !V_is_K_view && ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]); + + if (do_print) { + GGML_LOG_INFO("[MKL-FA] #%d D=%d DV=%d n_q=%d n_kv=%d " + "n_qh=%d n_kvh=%d gqa=%d batch=%d K=%s V=%s " + "chunk=%d buf=%.1fMB%s%s\n", + mkl_call_count, DKQ, DV, n_queries, n_kv, + n_q_heads, n_kv_heads, gqa_ratio, n_batch, + ggml_type_name(K->type), ggml_type_name(V->type), + chunk_size, + (double)((int64_t)n_query_rows * chunk_size * sizeof(float)) + / (1024.0 * 1024.0), + k_early_interleaved ? " K_ILV" : "", + v_early_interleaved ? " V_ILV" : ""); + GGML_LOG_INFO("[MKL-FA] #%d Q-nb1=%lld Q-nb2=%lld " + "q_rs=%lld q_hs=%lld dst_rs=%lld dst_hs=%lld\n", + mkl_call_count, + (long long)Q->nb[1], (long long)Q->nb[2], + (long long)q_row_stride, (long long)q_head_stride, + (long long)(KQV->nb[1] / sizeof(float)), + (long long)(KQV->nb[2] / sizeof(float))); + } + + // --- Stream and allocators --- + dpct::queue_ptr stream = ctx.stream(); + +#define MKL_TAKE_TIME(t0) auto t0 = std::chrono::steady_clock::now() +#define MKL_ACCUM(acc, t0) do { if (do_print) { \ + acc += (int64_t)std::chrono::duration_cast \ + (std::chrono::steady_clock::now() - (t0)).count(); \ +} } while(0) + + int64_t gemm_kq_time_us = 0; + int64_t gemm_vkq_time_us = 0; + int64_t softmax_time_us = 0; + int64_t dequant_time_us = 0; + + MKL_TAKE_TIME(t_deq); + + // --- K/V dequant descriptors --- + // Dequant is done per-chunk inside the KV loop (footprint independent of + // context). Output is always dense row-major fp16 [this_chunk x D], lda=D. + // Interleaved detection: ne[1]*nb[1] != nb[2] means heads are interleaved. + const bool k_interleaved = + ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]) && K->ne[2] > 1; + const bool v_interleaved = + ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]) && V->ne[2] > 1; + + const mkl_fa_kv_desc K_desc = mkl_fa_make_desc(K, k_interleaved, n_kv_heads); + const mkl_fa_kv_desc V_desc = V_is_K_view + ? K_desc : mkl_fa_make_desc(V, v_interleaved, n_kv_heads); + + MKL_ACCUM(dequant_time_us, t_deq); + + // --- Resolve mask pointers --- + int64_t mask_head_stride = 0; + int64_t mask_row_stride = 0; + int mask_n_heads = 0; + + if (mask) { + // Use actual fp16 device size (2 bytes), NOT sizeof(sycl::half) + // which may be 4 on the host in oneAPI. + mask_head_stride = mask->nb[2] / 2; + mask_row_stride = mask->nb[1] / 2; + mask_n_heads = (int)mask->ne[2]; + } + + // --- Allocate intermediates from pool --- + ggml_sycl_pool & pool = ctx.pool(); + + ggml_sycl_pool_alloc KQ_f32(pool); // [q_tile_rows x chunk] + ggml_sycl_pool_alloc S_f16(pool); // [q_tile_rows x chunk] + ggml_sycl_pool_alloc VKQ_chunk(pool); // [q_tile_rows x DV] + ggml_sycl_pool_alloc VKQ_accum(pool); // [n_query_rows x DV] (full) + ggml_sycl_pool_alloc KQ_max(pool); // [n_query_rows] (full) + ggml_sycl_pool_alloc KQ_sum(pool); // [n_query_rows] (full) + ggml_sycl_pool_alloc Q_head_f16(pool); // [n_query_rows x DKQ] (full) + ggml_sycl_pool_alloc K_chunk_f16(pool); // [chunk x DKQ] (per-chunk dequant) + ggml_sycl_pool_alloc V_chunk_f16(pool); // [chunk x DV] (per-chunk dequant) + + KQ_f32.alloc((size_t)q_tile_rows * chunk_size); + S_f16.alloc((size_t)q_tile_rows * chunk_size); + VKQ_chunk.alloc((size_t)q_tile_rows * DV); + VKQ_accum.alloc((size_t)n_query_rows * DV); + KQ_max.alloc(n_query_rows); + KQ_sum.alloc(n_query_rows); + Q_head_f16.alloc((size_t)n_query_rows * DKQ); + K_chunk_f16.alloc((size_t)chunk_size * DKQ); + + sycl::half * V_chunk_f16_ptr; + if (V_is_K_view) { + V_chunk_f16_ptr = K_chunk_f16.ptr; // V aliases K (DV == DKQ) + } else { + V_chunk_f16.alloc((size_t)chunk_size * DV); + V_chunk_f16_ptr = V_chunk_f16.ptr; + } + + sycl::half * Q_head_f16_ptr = Q_head_f16.ptr; + float * KQ_f32_ptr = KQ_f32.ptr; + sycl::half * S_f16_ptr = S_f16.ptr; + float * VKQ_chunk_ptr = VKQ_chunk.ptr; + float * VKQ_accum_ptr = VKQ_accum.ptr; + float * KQ_max_ptr = KQ_max.ptr; + float * KQ_sum_ptr = KQ_sum.ptr; + sycl::half * K_chunk_f16_ptr = K_chunk_f16.ptr; + + const float alpha = 1.0f; + const float beta = 0.0f; + + for (int ib = 0; ib < n_batch; ib++) { + const float * Q_batch = (const float *)Q->data + + ib * (Q->nb[3] / sizeof(float)); + float * dst_batch = (float *)KQV->data + + ib * (KQV->nb[3] / sizeof(float)); + + const sycl::half * mask_batch = nullptr; + if (mask) { + int m_batch = (mask->ne[3] > 1) ? ib : 0; + mask_batch = (const sycl::half *)mask->data + + m_batch * (mask->nb[3] / 2); // 2 = actual fp16 device size + } + + for (int ikvh = 0; ikvh < n_kv_heads; ikvh++) { + int kvh_base_head = ikvh * gqa_ratio; + + // 1. Pack all GQA Q heads into fp16 (full n_query_rows) + mkl_fa_pack_q_fp16(stream, + Q_head_f16_ptr, Q_batch, + n_queries, DKQ, + gqa_ratio, kvh_base_head, + q_scale, q_row_stride, q_head_stride, wg_size); + + // 2. Initialize softmax state (full n_query_rows) + mkl_fa_init_softmax_state(stream, + KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr, + n_query_rows, DV, wg_size); + + // 3. KV chunk loop (OUTER): dequant each chunk once, then tile queries. + for (int chunk_start = 0; chunk_start < n_kv; chunk_start += chunk_size) { + int this_chunk = std::min(chunk_size, n_kv - chunk_start); + + // 3a. Dequant this KV chunk to dense fp16 (once per chunk) + { + MKL_TAKE_TIME(t0); + mkl_fa_dequant_chunk(stream, K_desc, KQV, + K_chunk_f16_ptr, ikvh, chunk_start, this_chunk); + if (!V_is_K_view) { + mkl_fa_dequant_chunk(stream, V_desc, KQV, + V_chunk_f16_ptr, ikvh, chunk_start, this_chunk); + } + MKL_ACCUM(dequant_time_us, t0); + } + + // 3b. Query tile loop (INNER) — bounds KQ_f32/S_f16 footprint. + for (int q0 = 0; q0 < n_query_rows; q0 += q_tile_rows) { + int q_rows = std::min(q_tile_rows, n_query_rows - q0); + sycl::event kq_done, vkq_done; + + // GEMM: KQ = Q_tile × K_chunk^T + { + MKL_TAKE_TIME(t0); + kq_done = gemm(*stream, + transpose::trans, transpose::nontrans, + this_chunk, q_rows, DKQ, + alpha, + K_chunk_f16_ptr, DKQ, + Q_head_f16_ptr + (int64_t)q0 * DKQ, DKQ, + beta, + KQ_f32_ptr, this_chunk, + std::vector{*stream->ext_oneapi_get_last_event()}); + MKL_ACCUM(gemm_kq_time_us, t0); + } + // Online softmax over this chunk for this query tile + { + MKL_TAKE_TIME(t0); + mkl_fa_online_softmax_chunk(stream, + KQ_f32_ptr, S_f16_ptr, + KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr, + q0, q_rows, n_queries, DV, + this_chunk, chunk_start, + kvh_base_head, + mask_batch, mask_head_stride, + mask_row_stride, mask_n_heads, + logit_softcap, wg_size, kq_done); + MKL_ACCUM(softmax_time_us, t0); + } + + // GEMM: VKQ_chunk = S × V_chunk + { + MKL_TAKE_TIME(t0); + vkq_done = gemm(*stream, + transpose::nontrans, transpose::nontrans, + DV, q_rows, this_chunk, + alpha, + V_chunk_f16_ptr, DV, + S_f16_ptr, this_chunk, + beta, + VKQ_chunk_ptr, DV, + std::vector{*stream->ext_oneapi_get_last_event()}); + MKL_ACCUM(gemm_vkq_time_us, t0); + } + // VKQ_accum[q0..] += VKQ_chunk + { + const int64_t n_total = (int64_t)q_rows * DV; + const int64_t wg = ((n_total + wg_size - 1) / wg_size) + * wg_size; + float * accum = VKQ_accum_ptr + (int64_t)q0 * DV; + stream->submit([&](sycl::handler & cgh) { + cgh.depends_on(vkq_done); + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int64_t i = item.get_global_id(0); + if (i < n_total) { + accum[i] += VKQ_chunk_ptr[i]; + } + }); + }); + } + } + } + + // 4. Normalize and scatter each GQA head to dst + for (int iqg = 0; iqg < gqa_ratio; iqg++) { + int iqh = kvh_base_head + iqg; + int64_t src_offset = (int64_t)iqg * n_queries * DV; + mkl_fa_normalize_head(stream, + dst_batch, VKQ_accum_ptr, KQ_sum_ptr, + iqh, n_queries, DV, n_q_heads, + src_offset, wg_size); + } + } + } + +#undef MKL_TAKE_TIME +#undef MKL_ACCUM + + if (do_print) { + const int64_t v_chunk_elems = V_is_K_view ? 0 : (int64_t)chunk_size * DV; + double total_mb = (double)( + (int64_t)q_tile_rows * chunk_size * sizeof(float) // KQ_f32 + + (int64_t)q_tile_rows * chunk_size * sizeof(sycl::half) // S_f16 + + (int64_t)q_tile_rows * DV * sizeof(float) // VKQ_chunk + + (int64_t)n_query_rows * DV * sizeof(float) // VKQ_accum + + (int64_t)n_query_rows * sizeof(float) // KQ_max + + (int64_t)n_query_rows * sizeof(float) // KQ_sum + + (int64_t)n_query_rows * DKQ * sizeof(sycl::half) // Q_head_f16 + + (int64_t)chunk_size * DKQ * sizeof(sycl::half) // K_chunk_f16 + + v_chunk_elems * (int64_t)sizeof(sycl::half) // V_chunk_f16 + ) / (1024.0 * 1024.0); + GGML_LOG_INFO("[MKL-FA] #%d n_kv=%d n_q=%d q_tile=%d time_us: " + "dequant=%lld GEMM_KQ=%lld softmax=%lld GEMM_VKQ=%lld " + "buf_mb=%.1f\n", + mkl_call_count, n_kv, n_queries, q_tile_rows, + (long long)dequant_time_us, + (long long)gemm_kq_time_us, + (long long)softmax_time_us, + (long long)gemm_vkq_time_us, + total_mb); + } +} diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp new file mode 100644 index 0000000..6e03094 --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-onednn.cpp @@ -0,0 +1,454 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fattn-onednn.hpp" +#include "fattn-tile.hpp" +#include "convert.hpp" + +// set minimum query length to treat as prefill (32) +#define GGML_SYCL_FA_ONEDNN_MIN_Q 32 + +bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * V) { + if (K->type != GGML_TYPE_F16 || V->type != GGML_TYPE_F16) { + return false; + } + auto bindable = [](const ggml_tensor * t) { + return t->nb[0] == sizeof(sycl::half) && t->nb[1] % sizeof(sycl::half) == 0 && + t->nb[2] % sizeof(sycl::half) == 0 && t->nb[3] % sizeof(sycl::half) == 0; + }; + return bindable(K) && bindable(V); +} + +bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit) { +#if !GGML_SYCL_DNNL + GGML_UNUSED(dst); + GGML_UNUSED(use_shape_limit); + return false; +#else + if (!g_ggml_sycl_fa_onednn) { + return false; + } + // without a fused SDPA kernel oneDNN falls back to the same matmul, so reuse the check + if (!ggml_sycl_dnnl_has_optimized_gemm(ggml_sycl_get_device())) { + return false; + } + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + + // F16 KV: native SDPA at any KV length. + // Non-F16: dequant to F16 then SDPA at prefill lengths. Only the + // standard quantized KV cache types (Q4_0-Q8_0) and F32 are accepted + // because their to_fp16_sycl conversion is verified. BF16 and IQ* + // are excluded: BF16 needs a strided conversion kernel that does not + // exist yet; IQ types are model-weight-only quants with no dequant + // registration and are never used as KV caches. + if (K->type != GGML_TYPE_F16 || V->type != GGML_TYPE_F16) { + auto kt = K->type, vt = V->type; + bool k_ok = kt == GGML_TYPE_F32 || kt == GGML_TYPE_Q4_0 || kt == GGML_TYPE_Q4_1 || + kt == GGML_TYPE_Q5_0 || kt == GGML_TYPE_Q5_1 || kt == GGML_TYPE_Q8_0; + bool v_ok = vt == GGML_TYPE_F32 || vt == GGML_TYPE_Q4_0 || vt == GGML_TYPE_Q4_1 || + vt == GGML_TYPE_Q5_0 || vt == GGML_TYPE_Q5_1 || vt == GGML_TYPE_Q8_0; + if (!k_ok || !v_ok) { + return false; + } + if (use_shape_limit && (Q->ne[1] < 32 || K->ne[1] < 1024)) { + return false; + } + for (const ggml_tensor * t : {K, V}) { + if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) { + return false; + } + } + } + // This is the improved SPDA gate. Rather than gating Alchemist GPUs from all SPDA features, we instead target only the failing shapes. + // If the GPU being assessed isn't in the grouping below, it has full access to all SPDA shapes. Otherwise, if it's an Alchemist GPU, we block only the shapes with head sizes that fail. + // It is much easier to compare the device to a small list of failing cases than to define all the passing ones. + const gpu_arch arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch; + bool support_spda = !(arch == gpu_arch::intel_gpu_dg2_g10 || + arch == gpu_arch::intel_gpu_dg2_g11 || + arch == gpu_arch::intel_gpu_dg2_g12); + + if (!support_spda && K->ne[0] == 64) { + return false; + } + // Optional KV-length ceiling (GGML_SYCL_FA_ONEDNN_MAX_KV, 0 = unlimited). Escape hatch: + // very long sequences make the fused SDPA slow enough to risk the xe driver watchdog on + // some stacks; past the cap we fall back to the native FA kernel instead. + if (g_ggml_sycl_fa_onednn_max_kv > 0 && K->ne[1] > g_ggml_sycl_fa_onednn_max_kv) { + return false; + } + // gate for the following cases + // 1. if the oneDNN graph Add node has no input --> skip + // 2. types other than f16 need different logical_tensor declaration + // 3. the mask must be shape [1, 1, q, seq] + // 4. sinks: excludes attention sink (Xiao et al., 2024) that can't be modeled by oneDNN graph + if (!mask || mask->type != GGML_TYPE_F16 || mask->ne[2] != 1 || mask->ne[3] != 1 || sinks) { + return false; + } + float max_bias = 0.0f, logit_softcap = 0.0f; + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + if (max_bias != 0.0f || logit_softcap != 0.0f) { + return false; + } + // K and V must share head_dim: the SDPA graph uses a single `d` for both. + const int64_t d = K->ne[0]; + if (V->ne[0] != d || Q->ne[3] != 1) { + return false; + } + // GQA must divide evenly. + if (K->ne[2] == 0 || Q->ne[2] % K->ne[2] != 0) { + return false; + } + // Prefill only. + if (use_shape_limit && Q->ne[1] < GGML_SYCL_FA_ONEDNN_MIN_Q) { + return false; + } + return true; +#endif +} + +#if GGML_SYCL_DNNL + +#include "dnnl.hpp" +#include "dnnl_sycl.hpp" +#include "oneapi/dnnl/dnnl_graph.hpp" // graph API lives only under oneapi/dnnl/, not at the include root + +using namespace dnnl; +using namespace dnnl::graph; + +// strided src (f16 or f32) -> contiguous f16 [ne0,ne1,ne2,ne3] (ne0 innermost). nb* are BYTE strides. +template +static void cont_to_f16_sycl(const char * src, sycl::half * dst, + int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, + size_t nb1, size_t nb2, size_t nb3, dpct::queue_ptr stream) { + const int64_t n = ne0 * ne1 * ne2 * ne3; + stream->parallel_for(sycl::range<1>(n), [=](sycl::id<1> ix) { + const int64_t gid = ix[0]; + int64_t i = gid; + const int64_t i0 = i % ne0; i /= ne0; + const int64_t i1 = i % ne1; i /= ne1; + const int64_t i2 = i % ne2; const int64_t i3 = i / ne2; + const src_t * p = (const src_t *) (src + i1 * nb1 + i2 * nb2 + i3 * nb3) + i0; + dst[gid] = (sycl::half) (*p); + }); +} + +// oneDNN SDPA out (f16 contiguous [mb,H,q,d]) -> ggml dst (f32 [head_dim,H,n_tok,mb], contiguous). +static void permute_sdpa_out_sycl(const sycl::half * out, float * dst, + int64_t mb, int64_t H, int64_t q, int64_t d, dpct::queue_ptr stream) { + const int64_t n = mb * H * q * d; + stream->parallel_for(sycl::range<1>(n), [=](sycl::id<1> ix) { + const int64_t gid = ix[0]; + int64_t i = gid; + const int64_t e = i % d; i /= d; + const int64_t t = i % q; i /= q; + const int64_t h = i % H; const int64_t b = i / H; + dst[e + h * d + t * d * H + b * d * H * q] = (float) out[gid]; + }); +} + +struct sdpa_partition { + compiled_partition cp; + std::vector ins; + logical_tensor out; + size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0; + bool ok = false; +}; + +// Build + compile the contiguous-input GQA SDPA graph (MatMul->Divide->Add->SoftMax->MatMul), f32 out. +// Mirrors the hardware-verified scratch/onednn_sdpa_probe.cpp build_gqa (partitions=1, sdp_primitive_kernel_t). +static sdpa_partition build_sdpa(const engine & eng, int H, int Hkv, int q, int seq, int d, + const std::array & k_str, const std::array & v_str) try { + using ltype = logical_tensor::layout_type; + using dt = logical_tensor::data_type; + using ldims = logical_tensor::dims; + const dt fi = dt::f32, t = dt::f16; + const int rep = H / Hkv; + const ldims q_sz = {1, Hkv, rep, q, d}, kv_sz = {1, Hkv, 1, seq, d}, s_sz = {1, Hkv, rep, q, seq}, + sc = {1, 1, 1, 1, 1}, msk = {1, 1, 1, q, seq}, o_sz = {1, Hkv, rep, q, d}; + const ldims k_st(k_str.begin(), k_str.end()), v_st(v_str.begin(), v_str.end()); + int64_t id = 0; + sdpa_partition E; + + auto query = logical_tensor(id++, t, q_sz, ltype::strided); + auto key = logical_tensor(id++, t, kv_sz, k_st); + auto score = logical_tensor(id++, fi, s_sz, ltype::strided); + auto bmm1 = op(id++, op::kind::MatMul, "bmm1"); + bmm1.set_attr(op::attr::transpose_b, true); // key is [.., seq, d] + bmm1.add_inputs({query, key}); bmm1.add_outputs({score}); + + auto scale = logical_tensor(id++, t, sc, ltype::strided); + auto scaled = logical_tensor(id++, fi, s_sz, ltype::strided); + auto sdiv = op(id++, op::kind::Divide, "scale_div"); // score / (1/kq_scale) == score * kq_scale + sdiv.add_inputs({score, scale}); sdiv.add_outputs({scaled}); + + auto mask = logical_tensor(id++, t, msk, ltype::strided); + auto masked = logical_tensor(id++, fi, s_sz, ltype::strided); + auto madd = op(id++, op::kind::Add, "mask_add"); + madd.add_inputs({scaled, mask}); madd.add_outputs({masked}); + + auto probs = logical_tensor(id++, t, s_sz, ltype::strided); + auto smax = op(id++, op::kind::SoftMax, "softmax"); + smax.set_attr(op::attr::axis, -1); + smax.set_attr(op::attr::mode, "inf_as_zero"); + smax.add_inputs({masked}); smax.add_outputs({probs}); + + auto value = logical_tensor(id++, t, kv_sz, v_st); + // f16 output is REQUIRED to hit sdp_primitive_kernel_t (the systolic micro-kernel); an f32 output + // falls to larger_partition_kernel_t which materializes N^2 (confirmed: scratch/onednn_sdpa_kernel_probe.cpp). + // converted to the f32 ggml dst in the permute below. + auto output = logical_tensor(id++, t, o_sz, ltype::strided); // f16 contiguous [mb,Hkv,rep,q,d] + auto bmm2 = op(id++, op::kind::MatMul, "bmm2"); + bmm2.add_inputs({probs, value}); bmm2.add_outputs({output}); + + dnnl::graph::graph g(eng.get_kind()); + g.add_op(bmm1); g.add_op(sdiv); g.add_op(madd); g.add_op(smax); g.add_op(bmm2); + g.finalize(); + + auto parts = g.get_partitions(); + if (parts.size() != 1 || !parts[0].is_supported()) { + GGML_LOG_WARN("%s: oneDNN did not fuse the SDPA graph; falling back to TILE kernel\n", __func__); + return E; // ok stays false -> caller falls back to TILE + } + E.ins = parts[0].get_input_ports(); + E.out = parts[0].get_output_ports()[0]; + E.cp = parts[0].compile(E.ins, {E.out}, eng); + E.out = E.cp.query_logical_tensor(E.out.get_id()); + E.id_q = query.get_id(); E.id_k = key.get_id(); E.id_v = value.get_id(); + E.id_scale = scale.get_id(); E.id_mask = mask.get_id(); + E.ok = true; + return E; +} +catch (const std::exception & e) { + // compile() can reject a stride set the partitioner never inspects; memoise the failure so the + // fallback costs one build rather than one per call. + GGML_LOG_WARN("%s: oneDNN SDPA partition build failed (%s); falling back to TILE kernel\n", __func__, e.what()); + return {}; +} + +void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tensor * dst) try { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + + const int64_t d = K->ne[0]; // head_dim + const int64_t seq = K->ne[1]; // n_kv + const int64_t Hkv = K->ne[2]; // n_head_kv + const int64_t H = Q->ne[2]; // n_head + const int64_t q = Q->ne[1]; // n_tok + const int64_t mb = Q->ne[3]; // batch (== 1, gated) + + float kq_scale = 1.0f; + memcpy(&kq_scale, (const float *) dst->op_params + 0, sizeof(float)); + + dpct::queue_ptr stream = ctx.stream(); + dnnl::engine eng = ctx.engine_dnnl(stream); + dnnl::stream strm = ctx.stream_dnnl(stream); + + const ggml_sycl_fattn_extra extra = ggml_sycl_fattn_get_extra(dst); + + // Q: always f32 -- copy to dense f16. + std::optional> Qf_pool; + sycl::half * Qf_ptr = (sycl::half *) extra.Q_buffer_ptr; + if (!Qf_ptr) { + Qf_pool.emplace(ctx.pool(), (size_t) H * q * d); + Qf_ptr = Qf_pool->get(); + } + cont_to_f16_sycl((const char *) Q->data, Qf_ptr, d, q, H, mb, Q->nb[1], Q->nb[2], Q->nb[3], stream); + + // K/V: bind the f16 cache in place. llama.cpp permutes it to [token][head][dim], so its head + // plane is strided rather than dense, which is what an explicit stride vector expresses. + // Quantized and f32 KV still stage a dense copy -- the layout the k_str/v_str defaults describe. + sycl::half * K_ptr = nullptr; + sycl::half * V_ptr = nullptr; + std::array k_str{ Hkv * seq * d, seq * d, seq * d, d, 1 }; + std::array v_str = k_str; + std::optional> Kf_pool; + std::optional> Vf_pool; + // Helper: hand out reserved space, or fall back to the pool. + auto stage_k = [&](size_t n) { if (extra.K_buffer_ptr) { return (sycl::half *) extra.K_buffer_ptr; } + Kf_pool.emplace(ctx.pool(), n); return Kf_pool->get(); }; + auto stage_v = [&](size_t n) { if (extra.V_buffer_ptr) { return (sycl::half *) extra.V_buffer_ptr; } + Vf_pool.emplace(ctx.pool(), n); return Vf_pool->get(); }; + + auto elem_strides = [](const ggml_tensor * t) { + const int64_t s1 = (int64_t) (t->nb[1] / t->nb[0]); + const int64_t s2 = (int64_t) (t->nb[2] / t->nb[0]); + const int64_t s3 = (int64_t) (t->nb[3] / t->nb[0]); + // dims are {mb=1, Hkv, rep=1, seq, d}; the size-1 dims at 0 and 2 never advance an address. + return std::array{ s3, s2, s2, s1, 1 }; + }; + + if (ggml_sycl_fattn_onednn_binds_kv(K, V)) { + K_ptr = (sycl::half *) K->data; + V_ptr = (sycl::half *) V->data; + k_str = elem_strides(K); + v_str = elem_strides(V); + } else if (K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16) { + K_ptr = stage_k((size_t) Hkv * seq * d); + V_ptr = stage_v((size_t) Hkv * seq * d); + cont_to_f16_sycl((const char *) K->data, K_ptr, d, seq, Hkv, mb, K->nb[1], K->nb[2], K->nb[3], stream); + cont_to_f16_sycl((const char *) V->data, V_ptr, d, seq, Hkv, mb, V->nb[1], V->nb[2], V->nb[3], stream); + } else if (ggml_is_quantized(K->type)) { + // Quantized K/V: dequant to dense F16 using pool, same lifetime as F16 path. + K_ptr = stage_k((size_t) ggml_nelements(K)); + { + const char * K_data = (const char *)K->data; + const bool k_non_dense = ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]) && K->ne[2] > 1; + const bool k_gemma = k_non_dense && + ((int64_t)K->nb[2] < (int64_t)K->ne[1] * (int64_t)K->nb[1]); + if (ggml_is_contiguously_allocated(K) && !k_non_dense) { + to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(K->type, dst); + to_fp16(K_data, K_ptr, ggml_nelements(K), stream); + } else { + const size_t bs = ggml_blck_size(K->type); + const size_t ts = ggml_type_size(K->type); + to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(K->type); + int64_t s01, s02, s03; + if (k_gemma) { + const int64_t blk_per_row = (int64_t)K->ne[0] / bs; + s01 = (int64_t)Hkv * blk_per_row; + s02 = blk_per_row; + s03 = (int64_t)K->ne[1] * s01; + } else { + s01 = (int64_t)K->nb[1] / ts; + s02 = (int64_t)K->nb[2] / ts; + s03 = (int64_t)K->nb[3] / ts; + } + to_fp16(K_data, K_ptr, + K->ne[0], K->ne[1], K->ne[2], K->ne[3], + s01, s02, s03, stream); + } + } + // Quantized V: always dequant separately. Even when K and V share + // the same underlying allocation (V is a view of K with the same + // data pointer), their logical values differ because the quantized + // elements at different positions/offsets represent different K/V + // data. Master's F16 path also never aliases K and V. + V_ptr = stage_v((size_t) ggml_nelements(V)); + { + const char * V_data = (const char *)V->data; + const bool v_non_dense = ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]) && V->ne[2] > 1; + const bool v_gemma = v_non_dense && + ((int64_t)V->nb[2] < (int64_t)V->ne[1] * (int64_t)V->nb[1]); + if (ggml_is_contiguously_allocated(V) && !v_non_dense) { + to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(V->type, dst); + to_fp16(V_data, V_ptr, ggml_nelements(V), stream); + } else { + const size_t bs = ggml_blck_size(V->type); + const size_t ts = ggml_type_size(V->type); + to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(V->type); + int64_t s01, s02, s03; + if (v_gemma) { + const int64_t blk_per_row = (int64_t)V->ne[0] / bs; + s01 = (int64_t)V->ne[2] * blk_per_row; + s02 = blk_per_row; + s03 = (int64_t)V->ne[1] * s01; + } else { + s01 = (int64_t)V->nb[1] / ts; + s02 = (int64_t)V->nb[2] / ts; + s03 = (int64_t)V->nb[3] / ts; + } + to_fp16(V_data, V_ptr, + V->ne[0], V->ne[1], V->ne[2], V->ne[3], + s01, s02, s03, stream); + } + } + } else { + // F32: strided copy to dense F16 via cont_to_f16_sycl. + K_ptr = stage_k((size_t) ggml_nelements(K)); + cont_to_f16_sycl((const char *) K->data, K_ptr, K->ne[0], K->ne[1], K->ne[2], K->ne[3], + K->nb[1], K->nb[2], K->nb[3], stream); + V_ptr = stage_v((size_t) ggml_nelements(V)); + cont_to_f16_sycl((const char *) V->data, V_ptr, V->ne[0], V->ne[1], V->ne[2], V->ne[3], + V->nb[1], V->nb[2], V->nb[3], stream); + } + + // divide-by-(1/scale) reproduces ggml's score *= kq_scale on the proven probe graph. + // + // The scale must not be uploaded with an async memcpy from a stack local: on the in-order + // queue that copy waits behind the K/V staging kernels, and once those take long enough + // (n_kv >= ~26k on B70) the host frame is recycled before the copy runs, feeding the SDPA a + // garbage scale (output collapses to a repeated token). Write the scalar from a kernel + // instead -- the value is captured into the command, so no host memory has to outlive the + // call, and the enqueue stays async. + const sycl::half scale_h = (sycl::half) (1.0f / kq_scale); + std::optional> scbuf; + sycl::half * scale_dev = (sycl::half *) extra.scale_buffer_ptr; + if (!scale_dev) { + scbuf.emplace(ctx.pool(), 1); + scale_dev = scbuf->get(); + } + stream->single_task([=]() { *scale_dev = scale_h; }); + + // f16 contiguous SDPA out [mb,H,q,d] + std::optional> outf_pool; + sycl::half * outf_ptr = (sycl::half *) extra.out_buffer_ptr; + if (!outf_ptr) { + outf_pool.emplace(ctx.pool(), (size_t) H * q * d); + outf_ptr = outf_pool->get(); + } + + // compile once per (device, shape, KV strides), reuse across layers/calls. Stride 2 always + // repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ. + static std::unordered_map cache; + char keyb[256]; + snprintf(keyb, sizeof(keyb), "%d:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", ggml_sycl_get_device(), + (long long) H, (long long) Hkv, (long long) q, (long long) seq, (long long) d, + (long long) k_str[0], (long long) k_str[1], (long long) k_str[3], + (long long) v_str[0], (long long) v_str[1], (long long) v_str[3]); + auto it = cache.find(keyb); + if (it == cache.end()) { + it = cache.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first; + } + sdpa_partition & E = it->second; + if (!E.ok) { + // oneDNN can decline a shape or a stride set that _supported() never sees; build_sdpa warns per key. + ggml_sycl_flash_attn_ext_tile(ctx, dst); + return; + } + + auto id2ptr = [&](size_t r) -> void * { + if (r == E.id_q) return Qf_ptr; + if (r == E.id_k) return K_ptr; + if (r == E.id_v) return V_ptr; + if (r == E.id_scale) return scale_dev; + if (r == E.id_mask) return (void *) mask->data; + return nullptr; + }; + std::vector ti; + ti.reserve(E.ins.size()); + for (auto & lt : E.ins) { + ti.emplace_back(lt, eng, id2ptr(lt.get_id())); + } + tensor to(E.out, eng, outf_ptr); + E.cp.execute(strm, ti, {to}); + + permute_sdpa_out_sycl(outf_ptr, (float *) dst->data, mb, H, q, d, stream); + // Single device needs no sync: the dnnl stream wraps this same in-order queue, so the SDPA + // serializes with the staging kernels before it and the permute/pool reuse after it. The + // garbage output formerly blamed on the missing sync here was the scale use-after-return + // fixed above. Keep the conservative wait for multi-GPU, where other devices' streams can + // race the pool: + if (ggml_sycl_info().device_count > 1) { + stream->wait_and_throw(); + } +} +catch (const std::exception & e) { + // any oneDNN/SYCL failure is non-fatal: fall back to the existing kernel (strictly additive). + GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what()); + ggml_sycl_flash_attn_ext_tile(ctx, dst); +} + +#endif // GGML_SYCL_DNNL diff --git a/ggml/src/ggml-sycl/fattn-onednn.hpp b/ggml/src/ggml-sycl/fattn-onednn.hpp new file mode 100644 index 0000000..9669d1b --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-onednn.hpp @@ -0,0 +1,18 @@ +#ifndef GGML_SYCL_FATTN_ONEDNN_HPP +#define GGML_SYCL_FATTN_ONEDNN_HPP + +#include "common.hpp" + +// Static-only check: fused-XMX oneDNN Graph SDPA path==flash-attn op +// (f16 KV, no softcap/ALiBi, single stream, tuned head_dim, prefill-sized q.) +bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit = true); + +// True when the oneDNN path binds an F16 KV cache in place instead of staging a dense copy of +// it. Depends only on the types and strides of K and V, so the answer holds for every call. +bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * V); + +// Run flash attention through oneDNN's fused xmx SDPA +// execute the cached SDPA partition, write the f32 dst. Falls back to the TILE kernel on any failure. +void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_FATTN_ONEDNN_HPP diff --git a/ggml/src/ggml-sycl/fattn-sparse.cpp b/ggml/src/ggml-sycl/fattn-sparse.cpp new file mode 100644 index 0000000..0c56d43 --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-sparse.cpp @@ -0,0 +1,268 @@ +#include "fattn.hpp" +#include "fattn-sparse.hpp" + +#include +#include +#include + +static constexpr int64_t SPARSE_FA_PAD = 256; +static constexpr int64_t SPARSE_FA_MIN_RATIO = 2; + +extern int g_ggml_sycl_enable_sparse_fa; +extern int g_ggml_sycl_debug_sparse_fa; +extern int g_ggml_sycl_sparse_fa_margin; + +static int sparse_fa_enabled(void) { + return g_ggml_sycl_enable_sparse_fa; +} + +static int sparse_fa_debug(void) { + return g_ggml_sycl_debug_sparse_fa; +} + +// slack above n_kv_max; callers may exceed the hint by a few always-attended positions +static int sparse_fa_margin(void) { + return g_ggml_sycl_sparse_fa_margin; +} + +// Unordered output is fine: softmax over the selected set is permutation invariant. +static void sparse_fa_compact_mask(sycl::queue * stream, + const sycl::half * __restrict__ mask, + int32_t * __restrict__ indices, + int32_t * __restrict__ count, + const int64_t n_kv, + const int64_t n_kv_g) { + constexpr size_t WG = 256; + const size_t global = (size_t) GGML_PAD(n_kv, (int64_t) WG); + + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(global), sycl::range<1>(WG)), + [=](sycl::nd_item<1> item) { + const int64_t i = (int64_t) item.get_global_id(0); + if (i >= n_kv || !sycl::isfinite((float) mask[i])) { + return; + } + + sycl::atomic_ref ctr(*count); + + const int32_t pos = ctr.fetch_add(1); + if (pos < (int32_t) n_kv_g) { + indices[pos] = (int32_t) i; + } + }); +} + +// Rows along ne[0] are contiguous for every type used as a KV cache, so this is +// a plain byte copy and needs no per-type code. Padding slots are zeroed. +static void sparse_fa_gather_rows(sycl::queue * stream, + const uint8_t * __restrict__ src, + uint8_t * __restrict__ dst, + const int32_t * __restrict__ indices, + const int32_t * __restrict__ count, + const size_t row_size, + const size_t src_nb1, + const size_t src_nb2, + const int64_t n_kv_g, + const int64_t n_head) { + GGML_ASSERT(row_size % sizeof(uint32_t) == 0); + const size_t words = row_size / sizeof(uint32_t); + + stream->parallel_for( + sycl::range<3>((size_t) n_head, (size_t) n_kv_g, words), + [=](sycl::id<3> id) { + const int64_t h = (int64_t) id[0]; + const int64_t slot = (int64_t) id[1]; + const size_t w = id[2]; + + uint32_t * dst_row = + (uint32_t *) (dst + ((size_t) (h * n_kv_g + slot)) * row_size); + + if (slot >= (int64_t) *count) { + dst_row[w] = 0; + return; + } + + const uint32_t * src_row = + (const uint32_t *) (src + (size_t) indices[slot] * src_nb1 + + (size_t) h * src_nb2); + dst_row[w] = src_row[w]; + }); +} + +static void sparse_fa_gather_mask(sycl::queue * stream, + const sycl::half * __restrict__ mask, + sycl::half * __restrict__ mask_g, + const int32_t * __restrict__ indices, + const int32_t * __restrict__ count, + const int64_t n_kv_g, + const int64_t n_rows, + const size_t mask_s1) { + stream->parallel_for( + sycl::range<2>((size_t) n_rows, (size_t) n_kv_g), + [=](sycl::id<2> id) { + const int64_t r = (int64_t) id[0]; + const int64_t slot = (int64_t) id[1]; + + sycl::half v = sycl::half(-INFINITY); + if (slot < (int64_t) *count) { + v = mask[(size_t) r * mask_s1 + (size_t) indices[slot]]; + } + mask_g[(size_t) r * n_kv_g + slot] = v; + }); +} + +static bool sparse_fa_applicable(const ggml_tensor * dst, int64_t & n_kv_g_out) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + + if (!Q || !K || !V || !mask) { + return false; + } + + const int32_t n_kv_max = ggml_get_op_params_i32(dst, 4); + if (n_kv_max <= 0) { + return false; + } + + float max_bias = 0.0f; + float logit_softcap = 0.0f; + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float)); + if (max_bias != 0.0f || logit_softcap != 0.0f) { + return false; + } + + // single-token decode only; prefill amortises the scan already + if (Q->ne[1] != 1) { + return false; + } + if (K->ne[3] != 1 || V->ne[3] != 1 || mask->ne[2] != 1 || mask->ne[3] != 1) { + return false; + } + if (mask->type != GGML_TYPE_F16 || mask->ne[0] < K->ne[1]) { + return false; + } + if (K->ne[2] != V->ne[2]) { + return false; + } + + // nb[1] may stride over heads (interleaved cache); only ne[0] must be contiguous + if (K->nb[0] != ggml_type_size(K->type) || V->nb[0] != ggml_type_size(V->type)) { + return false; + } + + const size_t k_row = ggml_row_size(K->type, K->ne[0]); + const size_t v_row = ggml_row_size(V->type, V->ne[0]); + if (k_row % sizeof(uint32_t) || v_row % sizeof(uint32_t)) { + return false; + } + + const int64_t n_kv_g = GGML_PAD((int64_t) n_kv_max + sparse_fa_margin(), SPARSE_FA_PAD); + if (n_kv_g * SPARSE_FA_MIN_RATIO > K->ne[1]) { + return false; + } + + n_kv_g_out = n_kv_g; + return true; +} + +bool ggml_sycl_flash_attn_ext_sparse(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + int64_t n_kv_g = 0; + if (!sparse_fa_enabled() || !sparse_fa_applicable(dst, n_kv_g)) { + return false; + } + + ggml_tensor * K = dst->src[1]; + ggml_tensor * V = dst->src[2]; + ggml_tensor * mask = dst->src[3]; + + const int64_t n_kv = K->ne[1]; + const int64_t n_head_k = K->ne[2]; + const int64_t n_rows_m = mask->ne[1]; + + const size_t k_row = ggml_row_size(K->type, K->ne[0]); + const size_t v_row = ggml_row_size(V->type, V->ne[0]); + + dpct::queue_ptr stream = ctx.stream(); + + ggml_sycl_pool_alloc idx_alloc(ctx.pool(), (size_t) n_kv_g); + ggml_sycl_pool_alloc cnt_alloc(ctx.pool(), 1); + ggml_sycl_pool_alloc k_alloc(ctx.pool(), (size_t) n_head_k * n_kv_g * k_row); + ggml_sycl_pool_alloc v_alloc(ctx.pool(), (size_t) n_head_k * n_kv_g * v_row); + ggml_sycl_pool_alloc m_alloc(ctx.pool(), (size_t) n_rows_m * n_kv_g); + + int32_t * d_idx = idx_alloc.get(); + int32_t * d_cnt = cnt_alloc.get(); + uint8_t * d_K = k_alloc.get(); + uint8_t * d_V = v_alloc.get(); + sycl::half * d_mask = m_alloc.get(); + + // reset with a kernel: on Level Zero a queue memset is not always ordered before the next kernel + stream->single_task([=]() { *d_cnt = 0; }); + + sparse_fa_compact_mask(stream, (const sycl::half *) mask->data, + d_idx, d_cnt, n_kv, n_kv_g); + + sparse_fa_gather_rows(stream, (const uint8_t *) K->data, d_K, d_idx, d_cnt, + k_row, K->nb[1], K->nb[2], n_kv_g, n_head_k); + + sparse_fa_gather_rows(stream, (const uint8_t *) V->data, d_V, d_idx, d_cnt, + v_row, V->nb[1], V->nb[2], n_kv_g, n_head_k); + + sparse_fa_gather_mask(stream, (const sycl::half *) mask->data, d_mask, + d_idx, d_cnt, n_kv_g, n_rows_m, + mask->nb[1] / sizeof(sycl::half)); + + if (sparse_fa_debug() && stream->ext_oneapi_get_state() != sycl::ext::oneapi::experimental::queue_state::recording) { + int32_t h_cnt = 0; + SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(&h_cnt, d_cnt, sizeof(int32_t)))); + SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); + fprintf(stderr, "[FA-SPARSE] n_kv=%lld n_kv_max=%d n_kv_g=%lld finite=%d%s\n", + (long long) n_kv, ggml_get_op_params_i32(dst, 4), + (long long) n_kv_g, (int) h_cnt, + h_cnt > (int32_t) n_kv_g ? " OVERFLOW" : ""); + } + + // shallow copies retargeted at the gathered buffers; kernels are unchanged + ggml_tensor K_g = *K; + K_g.data = d_K; + K_g.ne[1] = n_kv_g; + K_g.nb[1] = k_row; + K_g.nb[2] = (size_t) n_kv_g * k_row; + K_g.nb[3] = (size_t) n_head_k * n_kv_g * k_row; + K_g.view_src = nullptr; + K_g.view_offs = 0; + + ggml_tensor V_g = *V; + V_g.data = d_V; + V_g.ne[1] = n_kv_g; + V_g.nb[1] = v_row; + V_g.nb[2] = (size_t) n_kv_g * v_row; + V_g.nb[3] = (size_t) V->ne[2] * n_kv_g * v_row; + V_g.view_src = nullptr; + V_g.view_offs = 0; + + ggml_tensor M_g = *mask; + M_g.data = d_mask; + M_g.ne[0] = n_kv_g; + M_g.nb[1] = (size_t) n_kv_g * sizeof(sycl::half); + M_g.nb[2] = M_g.nb[1] * mask->ne[1]; + M_g.nb[3] = M_g.nb[2]; + M_g.view_src = nullptr; + M_g.view_offs = 0; + + ggml_tensor dst_g = *dst; + dst_g.src[1] = &K_g; + dst_g.src[2] = &V_g; + dst_g.src[3] = &M_g; + dst_g.op_params[4] = 0; // avoid re-entering this path + + ggml_sycl_flash_attn_ext(ctx, &dst_g); + + return true; +} diff --git a/ggml/src/ggml-sycl/fattn-sparse.hpp b/ggml/src/ggml-sycl/fattn-sparse.hpp new file mode 100644 index 0000000..98b06bd --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-sparse.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_FATTN_SPARSE_HPP +#define GGML_SYCL_FATTN_SPARSE_HPP + +#include "common.hpp" + +// Gather the K/V rows selected by a sparse mask and re-dispatch the dense +// kernels onto them. Returns false if the caller should use the dense path. +bool ggml_sycl_flash_attn_ext_sparse(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_FATTN_SPARSE_HPP diff --git a/ggml/src/ggml-sycl/fattn-tile.cpp b/ggml/src/ggml-sycl/fattn-tile.cpp index 9449d75..db03026 100644 --- a/ggml/src/ggml-sycl/fattn-tile.cpp +++ b/ggml/src/ggml-sycl/fattn-tile.cpp @@ -54,6 +54,6 @@ void ggml_sycl_flash_attn_ext_tile(ggml_backend_sycl_context & ctx, ggml_tensor } break; default: { GGML_ABORT("Unsupported head size"); - } break; + } } } diff --git a/ggml/src/ggml-sycl/fattn-tile.hpp b/ggml/src/ggml-sycl/fattn-tile.hpp index 9ba5296..dcdcad8 100644 --- a/ggml/src/ggml-sycl/fattn-tile.hpp +++ b/ggml/src/ggml-sycl/fattn-tile.hpp @@ -1173,6 +1173,10 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_sycl_context & ctx, ggm launch_fattn_tile_switch_ncols1(ctx, dst); return; } + if (use_gqa_opt && gqa_ratio % 8 == 0) { + launch_fattn_tile_switch_ncols1(ctx, dst); + return; + } if (use_gqa_opt && gqa_ratio % 4 == 0) { launch_fattn_tile_switch_ncols1(ctx, dst); return; diff --git a/ggml/src/ggml-sycl/fattn-vec.hpp b/ggml/src/ggml-sycl/fattn-vec.hpp index 8031acf..5e40c88 100644 --- a/ggml/src/ggml-sycl/fattn-vec.hpp +++ b/ggml/src/ggml-sycl/fattn-vec.hpp @@ -15,13 +15,11 @@ namespace syclex = sycl::ext::oneapi::experimental; -static int ggml_sycl_fattn_vec_get_nthreads_host(const int cc) { - return 128; - GGML_UNUSED(cc); -} - -static constexpr int ggml_sycl_fattn_vec_get_nthreads_device() { - return 128; +static int ggml_sycl_fattn_vec_get_nthreads_device(gpu_arch arch) { + // Xe2 (Battlemage, Lunar Lake) runs the flash-attention vec kernel best with a 256-thread work group. + return (arch == gpu_arch::intel_gpu_bmg_g21 || + arch == gpu_arch::intel_gpu_bmg_g31 || + arch == gpu_arch::intel_gpu_lnl_m) ? 256 : 128; } // Currenlty llvm with the amdgcn target dose not support unrolling loops @@ -36,7 +34,8 @@ template // D == head size + int warp_size, + int nthreads> // D == head size static void flash_attn_ext_vec(const char* __restrict__ Q, const char* __restrict__ K, const char* __restrict__ V, @@ -74,6 +73,7 @@ static void flash_attn_ext_vec(const char* __restrict__ Q, const int32_t nb31, const int32_t nb32, const int64_t nb33) { + #ifdef SYCL_FLASH_ATTN // Skip unused kernel variants for faster compilation: @@ -99,7 +99,6 @@ static void flash_attn_ext_vec(const char* __restrict__ Q, constexpr int nthreads_KQ_q = (D/4 < warp_size ? D/4 : warp_size); constexpr int nthreads_V_q = (D/4 < warp_size ? D/4 : warp_size); - constexpr int nthreads = ggml_sycl_fattn_vec_get_nthreads_device(); constexpr int nthreads_KQ = type_K == GGML_TYPE_F16 ? 128 / cpy_nb : nthreads_KQ_q; constexpr int nthreads_V = type_V == GGML_TYPE_F16 ? 128 / cpy_nb : nthreads_V_q; @@ -471,7 +470,6 @@ static void flash_attn_ext_vec(const char* __restrict__ Q, } } - item_ct1.barrier(sycl::access::fence_space::local_space); #pragma unroll @@ -581,23 +579,38 @@ static void flash_attn_ext_vec(const char* __restrict__ Q, #endif // __clang__ + template void ggml_sycl_flash_attn_ext_vec_case_impl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - const int warp_size = WARP_16_SIZE; //better performance than WARP_32_SIZE - - const int cc = ggml_sycl_info().devices[ggml_sycl_get_device()].cc; - - const int nthreads = ggml_sycl_fattn_vec_get_nthreads_host(cc); - const int nwarps = nthreads / warp_size; + constexpr int warp_size = WARP_16_SIZE; //better performance than WARP_32_SIZE const bool need_f16_K = type_K == GGML_TYPE_F16; const bool need_f16_V = type_V == GGML_TYPE_F16; constexpr size_t nbytes_shared = 0; + const auto arch = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.arch; + // D=512 does not fit the default register file; it spills up to 343 bytes per thread, against at most 57 for D <= 256. This kernel is decode only, so thread occupancy is not the limit. It is 1.9x faster at every KV depth on Battlemage. + constexpr bool use_large_grf = D >= 512; + + const int nthreads = ggml_sycl_fattn_vec_get_nthreads_device(arch); + if constexpr (D <= 256) { + if (nthreads == 256) { + constexpr int nthreads_hw = 256; + constexpr int nwarps = nthreads_hw / warp_size; + launch_fattn, warp_size, use_large_grf>( + ctx, dst, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false); + return; + } + } + + constexpr int nthreads_hw = 128; + constexpr int nwarps = nthreads_hw / warp_size; launch_fattn, warp_size>( + use_logit_softcap, warp_size, nthreads_hw>, warp_size, use_large_grf>( ctx, dst, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false); } diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index 7c6e611..fef6e8d 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -18,7 +18,8 @@ #include "fattn-tile.hpp" #include "fattn-vec.hpp" #include "fattn.hpp" - +#include "fattn-onednn.hpp" +#include "fattn-sparse.hpp" #define FATTN_VEC_CASE(D, type_K, type_V) \ { \ @@ -96,11 +97,13 @@ static void ggml_sycl_flash_attn_ext_vec(ggml_backend_sycl_context & ctx, ggml_t enum best_fattn_kernel { BEST_FATTN_KERNEL_NONE = 0, BEST_FATTN_KERNEL_VEC = 100, + BEST_FATTN_KERNEL_ONEDNN = 150, // oneDNN SDPA: native F16 (PR #25222) BEST_FATTN_KERNEL_TILE = 200, + BEST_FATTN_KERNEL_MKL = 300, }; + static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const ggml_tensor * dst) { - GGML_UNUSED(device); #ifndef SYCL_FLASH_ATTN GGML_UNUSED(dst); return BEST_FATTN_KERNEL_NONE; @@ -113,6 +116,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; const int gqa_ratio = Q->ne[2] / K->ne[2]; GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); @@ -120,7 +124,60 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const float max_bias = 0.0f; memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); + float logit_softcap = 0.0f; + memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); + bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; + + // XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16). + // ONEDNN requires min 32 query tokens — short-circuit decode to avoid + // calling _supported() on every decode FA call. + if (Q->ne[1] >= 32 + && ggml_sycl_flash_attn_ext_onednn_supported(dst)) { + return BEST_FATTN_KERNEL_ONEDNN; + } + + // MKL path: XMX-accelerated GEMM for prompt processing (all KV cache types). + // The MKL kernel converts non-F16 K/V to F16 via to_fp16_sycl before GEMM, + // so quantized, F16, BF16, and F32 caches all benefit from XMX acceleration. + // Activates automatically when flash-attn is enabled (--flash-attn on or -fa) + // and n_kv >= 1024. Falls through to TILE/VEC for ALiBi, logit softcap, + // and mismatched batch dimensions (unsupported by the MKL kernel). + // Set GGML_SYCL_ENABLE_MKL_FA=0 to force TILE/VEC path for A/B testing. + // Example: GGML_SYCL_ENABLE_MKL_FA=0 llama-cli -m model.gguf -fa -ngl 99 ... + // Note: MKL GEMM calls are incompatible with SYCL graph capture replay. + const bool standard_shape = Q->ne[0] >= 64 && Q->ne[0] <= 512 && + Q->ne[0] % 64 == 0 && Q->ne[0] == V->ne[0]; + // GLM-4.7 Flash's MLA shape is already expressible by the MKL pipeline: + // KQ GEMM uses DKQ=576 while VKQ and output use DV=512. Keep this narrow + // until other mismatched K/V shapes have independent correctness data. + const bool glm_mla_shape = Q->ne[0] == 576 && K->ne[0] == 576 && + V->ne[0] == 512 && gqa_ratio == 20 && + K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16; + + if (g_ggml_sycl_enable_mkl_fa == 1 && mask && !sinks && gqa_ratio >= 2 && + (standard_shape || glm_mla_shape) && + Q->ne[1] >= 32 && K->ne[1] >= 1024 && + max_bias == 0.0f && logit_softcap == 0.0f && + (Q->ne[3] == K->ne[3] || K->ne[3] == 1)) { + // F16 K/V strides must be a multiple of ne[0]*2 (the natural row size + // in bytes). This passes both dense (nb1 == ne0*2) and interleaved + // (nb1 == H * ne0*2). Only pathological test strides like nb1=32 or + // nb1=75 for ne0=40 fall through to TILE. + bool kv_strides_ok = true; + for (const ggml_tensor * t : {K, V}) { + const bool glm_v_stride = glm_mla_shape && t == V && + V->view_src && V->nb[1] == K->nb[1]; + if (!glm_v_stride && t->type == GGML_TYPE_F16 && + t->nb[1] % (t->ne[0] * 2) != 0) { + kv_strides_ok = false; + break; + } + } + if (kv_strides_ok) { + return BEST_FATTN_KERNEL_MKL; + } + } for (const ggml_tensor * t : {Q, K, V, mask}) { if (t == nullptr || ggml_is_quantized(t->type)) { continue; @@ -168,6 +225,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const switch (K->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: + case GGML_TYPE_BF16: break; case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -186,10 +244,17 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_NONE; } - // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes: - const bool can_use_vector_kernel = Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0; + // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes. + // BF16 is excluded: the VEC kernel has no BF16 template (it needs GGML_SYCL_FA_ALL_QUANTS for non-F16/Q4_0/Q8_0). + const bool has_bf16 = (K->type == GGML_TYPE_BF16 || V->type == GGML_TYPE_BF16); + const bool can_use_vector_kernel = Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0 + && !has_bf16; - // Todo: Use the XMX kernel if possible: + // Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly + // additive -- taken only when statically supported, otherwise falls through to VEC/TILE below. + if (ggml_sycl_flash_attn_ext_onednn_supported(dst)) { + return BEST_FATTN_KERNEL_ONEDNN; + } // If there are no tensor cores available, use the generic tile kernel: if (can_use_vector_kernel) { @@ -201,6 +266,11 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const } } else { if (Q->ne[1] <= 2) { + // TILE is faster for quantized KV decode on Xe2 (BMG); keep VEC on untested archs + const gpu_arch arch = ggml_sycl_info().devices[device].hw_info.arch; + if (arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31) { + return BEST_FATTN_KERNEL_TILE; + } return BEST_FATTN_KERNEL_VEC; } } @@ -210,18 +280,207 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_set_device(ctx.device); - switch (ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst)) { + + ggml_tensor * Q = dst->src[0]; + ggml_tensor * K = dst->src[1]; + ggml_tensor * V = dst->src[2]; + if (K && Q && K->ne[0] != Q->ne[0] && Q->ne[0] > 0 && K->ne[0] % Q->ne[0] == 0) { + const int64_t d = Q->ne[0]; + const int64_t n_heads = (K->ne[0] * K->ne[2]) / d; + K->ne[0] = d; + K->ne[2] = n_heads; + } + if (V && Q && V->ne[0] != Q->ne[0] && Q->ne[0] > 0 && V->ne[0] % Q->ne[0] == 0) { + const int64_t d = Q->ne[0]; + const int64_t n_heads = (V->ne[0] * V->ne[2]) / d; + V->ne[0] = d; + V->ne[2] = n_heads; + } + + // sparse nodes are gathered down to n_kv_max rows and re-dispatched here + if (ggml_sycl_flash_attn_ext_sparse(ctx, dst)) { + return; + } + + // n_kv watchdog: log when n_kv differs from the last FA call with + // the same D — helps detect cache-truncation issues. + static int nkv_debug = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DEBUG", 0); + if (nkv_debug == 1) { + const ggml_tensor * K_dbg = dst->src[1]; + const ggml_tensor * V_dbg = dst->src[2]; + static int64_t last_nkv_d256 = 0, last_nkv_d512 = 0; + static int fa_call_seq = 0; + fa_call_seq++; + int64_t cur_nkv = K_dbg->ne[1]; + int Dk = (int)K_dbg->ne[0]; + const char * kname = "TILE"; + best_fattn_kernel k = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); + if (k == BEST_FATTN_KERNEL_MKL) kname = "MKL"; + if (k == BEST_FATTN_KERNEL_ONEDNN) kname = "ONEDNN"; + if (k == BEST_FATTN_KERNEL_VEC) kname = "VEC"; + int64_t delta = 0; + if (Dk == 256) { + delta = cur_nkv - last_nkv_d256; + last_nkv_d256 = cur_nkv; + } else if (Dk == 512) { + delta = cur_nkv - last_nkv_d512; + last_nkv_d512 = cur_nkv; + } + GGML_LOG_INFO("[FA-DISP] #%d %s D=%d n_kv=%lld delta=%lld " + "V_ne1=%lld\n", + fa_call_seq, kname, Dk, + (long long)cur_nkv, (long long)delta, + (long long)V_dbg->ne[1]); + } + + const best_fattn_kernel fk = ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst); + switch (fk) { case BEST_FATTN_KERNEL_NONE: GGML_ABORT("Not support Flash-Attention"); + case BEST_FATTN_KERNEL_ONEDNN: + // guarded: ggml_sycl_flash_attn_ext_onednn() is only defined under GGML_SYCL_DNNL; + // the reference must be compiled out here or the GGML_SYCL_DNNL=0 build fails to link. +#if GGML_SYCL_DNNL + ggml_sycl_flash_attn_ext_onednn(ctx, dst); +#endif + break; case BEST_FATTN_KERNEL_TILE: ggml_sycl_flash_attn_ext_tile(ctx, dst); break; case BEST_FATTN_KERNEL_VEC: ggml_sycl_flash_attn_ext_vec(ctx, dst); break; + case BEST_FATTN_KERNEL_MKL: + ggml_sycl_flash_attn_ext_mkl(ctx, dst); + break; + } + + // --- Output fingerprint (GGML_SYCL_MKL_FA_DIAG=1) --- + // Copy first 64 float output values to host for fingerprinting. + // Compare MKL vs TILE (GGML_SYCL_ENABLE_MKL_FA=0) to detect divergence. + // Only fingerprints the first 6 FA calls with n_kv >= 1024. + static int fa_diag = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DIAG", 0); + static int fa_diag_count = 0; + if (fa_diag == 1 && fa_diag_count < 6) { + const ggml_tensor * K_diag = dst->src[1]; + const ggml_tensor * V_diag = dst->src[2]; + const ggml_tensor * Q_diag = dst->src[0]; + dpct::queue_ptr q = ctx.stream(); + if (K_diag->ne[1] >= 1024 && q->ext_oneapi_get_state() != sycl::ext::oneapi::experimental::queue_state::recording) { + fa_diag_count++; + float diag_buf[64]; + q->memcpy(diag_buf, dst->data, 64 * sizeof(float)); + q->wait(); + const char * kname = "???"; + best_fattn_kernel kb = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); + if (kb == BEST_FATTN_KERNEL_ONEDNN) kname = "ONEDNN"; + if (kb == BEST_FATTN_KERNEL_MKL) kname = "MKL"; + if (kb == BEST_FATTN_KERNEL_TILE) kname = "TILE"; + if (kb == BEST_FATTN_KERNEL_VEC) kname = "VEC"; + GGML_LOG_INFO("[FA-DIAG] #%d %s D=%d n_kv=%lld n_q=%lld " + "n_qh=%lld n_kvh=%lld K=%s V=%s " + "nb1=%zu nb2=%zu first 64 floats:\n", + fa_diag_count, kname, + (int)K_diag->ne[0], (long long)K_diag->ne[1], + (long long)Q_diag->ne[1], + (long long)Q_diag->ne[2], (long long)K_diag->ne[2], + ggml_type_name(K_diag->type), + ggml_type_name(V_diag->type), + K_diag->nb[1], K_diag->nb[2]); + for (int i = 0; i < 64; i += 8) { + GGML_LOG_INFO(" [%2d] %08x %08x %08x %08x %08x %08x %08x %08x\n", + i, + *(unsigned *)&diag_buf[i+0], *(unsigned *)&diag_buf[i+1], + *(unsigned *)&diag_buf[i+2], *(unsigned *)&diag_buf[i+3], + *(unsigned *)&diag_buf[i+4], *(unsigned *)&diag_buf[i+5], + *(unsigned *)&diag_buf[i+6], *(unsigned *)&diag_buf[i+7]); + } + } } + } bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst) { return ggml_sycl_get_best_fattn_kernel(device, dst) != BEST_FATTN_KERNEL_NONE; } + +bool ggml_sycl_flash_attn_ext_needs_sync(int device, const ggml_tensor * dst) { + return ggml_sycl_get_best_fattn_kernel(device, dst) == BEST_FATTN_KERNEL_MKL; +} + +bool ggml_sycl_flash_attn_ext_uses_onednn(int device, const ggml_tensor * dst) { + return ggml_sycl_get_best_fattn_kernel(device, dst) == BEST_FATTN_KERNEL_ONEDNN; +} + +static uintptr_t ggml_sycl_fattn_reserve_halves(ggml_sycl_fattn_extra & extra, size_t n_halves) { + if (n_halves == 0) { + return 0; + } + extra.end = GGML_PAD(extra.end, SYCL_BUFFER_ALIGNMENT); + const uintptr_t block = extra.end; + extra.end += n_halves * sizeof(sycl::half); + return block; +} + +ggml_sycl_fattn_extra ggml_sycl_fattn_get_extra(const ggml_tensor * dst) { + ggml_sycl_fattn_extra extra; + + extra.end = (uintptr_t) dst->data + ggml_nbytes(dst); + + if (dst->op != GGML_OP_FLASH_ATTN_EXT) { + return extra; + } + + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + if (!Q || !K || !V) { + return extra; + } + + const int64_t d = K->ne[0]; + const int64_t H = Q->ne[2]; + const int64_t q = Q->ne[1]; + + // calculate the worst-case memory consumption across all kernels + const bool onednn_supported = ggml_sycl_flash_attn_ext_onednn_supported(dst, /* use_shape_limit */ false); + + const bool tile_needs_K = K->type != GGML_TYPE_F16; + const bool tile_needs_V = V->type != GGML_TYPE_F16; + + const bool V_is_K_view = V->view_src && + (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs)); + + size_t need_K = 0, need_V = 0, need_Q = 0, need_out = 0, need_scale = 0; + if (onednn_supported) { + need_Q = (size_t) H * q * d; + need_out = (size_t) H * q * d; + need_scale = 1; + // an f16 cache is bound in place, so it needs no staging copy + if (!ggml_sycl_fattn_onednn_binds_kv(K, V)) { + need_K = (size_t) ggml_nelements(K); + need_V = (size_t) ggml_nelements(V); + } + } + if (tile_needs_K) { + need_K = std::max(need_K, (size_t) ggml_nelements(K)); + } + if (tile_needs_V) { + need_V = std::max(need_V, (size_t) ggml_nelements(V)); + } + + extra.Q_buffer_ptr = ggml_sycl_fattn_reserve_halves(extra, need_Q); + extra.K_buffer_ptr = ggml_sycl_fattn_reserve_halves(extra, need_K); + extra.V_buffer_ptr = (V_is_K_view && !onednn_supported && need_V) + ? extra.K_buffer_ptr + : ggml_sycl_fattn_reserve_halves(extra, need_V); + extra.scale_buffer_ptr = ggml_sycl_fattn_reserve_halves(extra, need_scale); + extra.out_buffer_ptr = ggml_sycl_fattn_reserve_halves(extra, need_out); + + return extra; +} + +size_t ggml_sycl_flash_attn_ext_get_alloc_size(const ggml_tensor * dst) { + const ggml_sycl_fattn_extra extra = ggml_sycl_fattn_get_extra(dst); + return (size_t) (extra.end - (uintptr_t) dst->data); +} diff --git a/ggml/src/ggml-sycl/fattn.hpp b/ggml/src/ggml-sycl/fattn.hpp index f2a8ffc..8ba6342 100644 --- a/ggml/src/ggml-sycl/fattn.hpp +++ b/ggml/src/ggml-sycl/fattn.hpp @@ -19,4 +19,30 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst); +// Reports if flash attention runs this node with the oneMKL kernel, which waits on the host +bool ggml_sycl_flash_attn_ext_needs_sync(int device, const ggml_tensor * dst); + +// Reports if flash attention runs this node with the oneDNN kernel +bool ggml_sycl_flash_attn_ext_uses_onednn(int device, const ggml_tensor * dst); + +// Scratch that flash attention needs beyond the output tensor +struct ggml_sycl_fattn_extra { + uintptr_t K_buffer_ptr = 0; // F16 copy of the K cache + uintptr_t V_buffer_ptr = 0; // F16 copy of the V cache + uintptr_t Q_buffer_ptr = 0; // dense F16 copy of Q, oneDNN only + uintptr_t scale_buffer_ptr = 0; // the softmax scale as an F16 scalar, oneDNN only + uintptr_t out_buffer_ptr = 0; // F16 SDPA output before conversion to F32, oneDNN only + uintptr_t end = 0; // one past the last reserved byte; sizes the allocation +}; + +// ggml_sycl_fattn_get_extra() is the single source of truth for the layout: it both sizes +// the reservation and hands out the pointers, so the two cannot disagree. +// Each field is the address of one reserved block, or 0 if that block was not reserved, +// in which case the caller allocates from the scratch pool instead. +ggml_sycl_fattn_extra ggml_sycl_fattn_get_extra(const ggml_tensor * dst); + +size_t ggml_sycl_flash_attn_ext_get_alloc_size(const ggml_tensor * dst); + +void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + #endif // GGML_SYCL_FATTN_HPP diff --git a/ggml/src/ggml-sycl/fill.cpp b/ggml/src/ggml-sycl/fill.cpp new file mode 100644 index 0000000..28e618e --- /dev/null +++ b/ggml/src/ggml-sycl/fill.cpp @@ -0,0 +1,55 @@ +#include "fill.hpp" +#include "common.hpp" + +#define SYCL_FILL_BLOCK_SIZE 256 + +template +static void fill_kernel(T * dst, const int64_t k, const T value, + const sycl::nd_item<1> & item) { + const int64_t i = (int64_t)item.get_global_id(0); + if (i >= k) { + return; + } + dst[i] = value; +} + +inline void ggml_sycl_op_fill(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + GGML_ASSERT(ggml_is_contiguous(dst)); + + dpct::queue_ptr stream = ctx.stream(); + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + float value; + memcpy(&value, dst->op_params, sizeof(float)); + + const int64_t k = ggml_nelements(dst); + const int64_t num_blocks = (k + SYCL_FILL_BLOCK_SIZE - 1) / SYCL_FILL_BLOCK_SIZE; + void * dst_d = dst->data; + + switch (dst->type) { + case GGML_TYPE_F32: + stream->parallel_for( + sycl::nd_range<1>(num_blocks * SYCL_FILL_BLOCK_SIZE, SYCL_FILL_BLOCK_SIZE), + [=](sycl::nd_item<1> item) { + fill_kernel(static_cast(dst_d), k, value, item); + }); + break; + case GGML_TYPE_F16: + { + sycl::half h_value = sycl::half(value); + stream->parallel_for( + sycl::nd_range<1>(num_blocks * SYCL_FILL_BLOCK_SIZE, SYCL_FILL_BLOCK_SIZE), + [=](sycl::nd_item<1> item) { + fill_kernel(static_cast(dst_d), k, h_value, item); + }); + } + break; + default: + GGML_ABORT("unsupported type"); + } +} + +void ggml_sycl_fill(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/0); + ggml_sycl_op_fill(ctx, dst); +} diff --git a/ggml/src/ggml-sycl/fill.hpp b/ggml/src/ggml-sycl/fill.hpp new file mode 100644 index 0000000..b2adb94 --- /dev/null +++ b/ggml/src/ggml-sycl/fill.hpp @@ -0,0 +1,5 @@ +#pragma once + +#include "common.hpp" + +void ggml_sycl_fill(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/fused-gemm.cpp b/ggml/src/ggml-sycl/fused-gemm.cpp new file mode 100644 index 0000000..3024f13 --- /dev/null +++ b/ggml/src/ggml-sycl/fused-gemm.cpp @@ -0,0 +1,1099 @@ +#include "fused-gemm.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace mx = sycl::ext::oneapi::experimental::matrix; + +// FG_ / fg_ is short for fused GEMM: the weights are dequantized inside the GEMM, into the XMX tiles. + +// A k step is one 32-value weight sub-block; iq3_s and the other superblock formats split their +// superblock into steps of this width. The sub-groups of a work-group each walk their own K range +// and are summed at the end. +static constexpr int FG_BK = QK4_NL; +static constexpr int FG_KSPLIT = 4; + +// Element traits of one joint_matrix operand type. The A stage and the B pack compute in f32 and +// convert once, in registers, when they write the element, so any type costs the same one pass. +// store: storage in SLM (A) and in the packed B buffer +// mtype: matrix_type in matrix_combinations +// mode: GGML_SYCL_DYNAMIC_PRECISION value that selects this type +// src: ggml type that needs no conversion into this type (GGML_TYPE_COUNT: none) +// slow: XMX throughput class, 0 is fastest. f16 and bf16 share the DPAS rate; tf32 does half the +// K per instruction. B60, Qwen3-30B-A3B pp512: f16 1108, bf16 1000, tf32 751 t/s +template struct fg_elem; + +template <> struct fg_elem { + using store = sycl::half; + using pair = sycl::half2; + static constexpr mx::matrix_type mtype = mx::matrix_type::fp16; + static constexpr int mode = GGML_SYCL_DYNAMIC_PRECISION_F16; + static constexpr ggml_type src = GGML_TYPE_F16; + static constexpr int mant = 10; + static constexpr int slow = 0; + static store cvt(float x) { return (store) x; } + static pair make(float x, float y) { return pair((store) x, (store) y); } +}; + +// tf32 rounds to nearest even with plain bit ops: round_to_tf32 needs a SPIR-V extension that the +// DG2 AOT target rejects +static inline uint32_t fg_round_bits(float x, int drop) { + const uint32_t u = sycl::bit_cast(x); + if ((u & 0x7f800000u) == 0x7f800000u) { + return (u & 0x7fffffu) ? u | (1u << drop) : u; // nan stays nan + } + return u + ((1u << (drop - 1)) - 1) + ((u >> drop) & 1); +} + +struct alignas(4) fg_bf16x2 { + sycl::ext::oneapi::bfloat16 x, y; +}; + +template <> struct fg_elem { + using store = sycl::ext::oneapi::bfloat16; + using pair = fg_bf16x2; + static constexpr mx::matrix_type mtype = mx::matrix_type::bf16; + static constexpr int mode = GGML_SYCL_DYNAMIC_PRECISION_BF16; + static constexpr ggml_type src = GGML_TYPE_BF16; + static constexpr int mant = 7; + static constexpr int slow = 0; + static store cvt(float x) { return store(x); } + static pair make(float x, float y) { return { cvt(x), cvt(y) }; } +}; + +// tf32 keeps f32 range and f16 mantissa, in f32 storage +template <> struct fg_elem { + using store = float; + using pair = sycl::float2; + static constexpr mx::matrix_type mtype = mx::matrix_type::tf32; + static constexpr int mode = GGML_SYCL_DYNAMIC_PRECISION_TF32; + static constexpr ggml_type src = GGML_TYPE_COUNT; + static constexpr int mant = 10; + static constexpr int slow = 1; + static store cvt(float x) { return sycl::bit_cast(fg_round_bits(x, 13) & ~0x1fffu); } + static pair make(float x, float y) { return pair(cvt(x), cvt(y)); } +}; + +// One joint_matrix combination (A type, B type, TM x TN x TK, sub-group size; C and D are f32) and +// the tiling built on it. A sub-group owns SG_ROWS rows of A (at least 16) and BN (at least 32) +// columns of B. A and B may differ: the device lists the pairs it supports. +template struct fg_combo { + using ta = TA; + using tb = TB; + using EA = fg_elem; + using EB = fg_elem; + using tsa = typename EA::store; + using tsb = typename EB::store; + static constexpr int TM = TM_; + static constexpr int TN = TN_; + static constexpr int TK = TK_; + static constexpr int SG = SG_; + static constexpr int VNNI = 4 / sizeof(tsb); // K rows of B packed in one 32-bit word + static constexpr int SG_ROWS = TM > 16 ? TM : 16; + static constexpr int RPL = SG_ROWS / SG; // A rows one lane decodes per k step + static constexpr int MT = SG_ROWS / TM; + static constexpr int BN = TN > 32 ? TN : 32; + static constexpr int NT = BN / TN; + static constexpr int WG_SIZE = FG_KSPLIT * SG; + static constexpr mx::layout b_layout = VNNI == 1 ? mx::layout::row_major : mx::layout::ext_intel_packed; + // a 64-wide N is mostly padding here and a 32x64 f32 accumulator needs 128 registers per lane, + // so it spills: 13x slower on B60 + static constexpr bool efficient = TN <= 32; + static_assert(SG_ROWS % SG == 0 && SG_ROWS % TM == 0 && BN % TN == 0 && FG_BK % TK == 0, "bad tile"); + static_assert(BN <= GGML_SYCL_FG_MAX_N, "header gate must cover the tile width"); +}; + +using fg_half = sycl::half; +using fg_bf16 = sycl::ext::oneapi::bfloat16; +using fg_tf32 = mx::precision::tf32; + +// One bit of GGML_SYCL_XMX_GATHER_SHAPES per combination. Only combinations some device lists in +// matrix_combinations are built (appendix of sycl_ext_oneapi_matrix and the runtime's own list). +template static void fg_visit_combo(int idx, F && f); +static constexpr int FG_N_COMBOS = 8; + +// A spir64_gen AOT build (GGML_SYCL_XMX_AOT_SG) drops the combinations of the other sub-group size +// entirely: ocloc rejects even an empty kernel that asks for a sub-group size it lacks. +template static constexpr bool fg_listed() { +#if defined(GGML_SYCL_XMX_AOT_SG) + return SG == GGML_SYCL_XMX_AOT_SG; +#else + return true; +#endif +} + +template static void fg_call_combo(F && f) { + if constexpr (fg_listed()) { + f(S{}); + } +} + +template static void fg_visit_combo(int idx, F && f) { + switch (idx) { + case 0: fg_call_combo>(f); break; // Xe2, Xe3, Xe-HPC + case 1: fg_call_combo>(f); break; // Xe2, Xe3, Xe-HPC + case 2: fg_call_combo>(f); break; // Xe2, Xe3, Xe-HPC + case 3: fg_call_combo>(f); break; // Xe2, Xe3, Xe-HPC + case 4: fg_call_combo>(f); break; // Xe-HPG (Arc A), ARL-H + case 5: fg_call_combo>(f); break; // Xe2, Xe3, Xe-HPC + case 6: fg_call_combo>(f); break; // Xe2, Xe3, Xe-HPC + case 7: fg_call_combo>(f); break; // Xe-HPG (Arc A), ARL-H + default: GGML_ABORT("bad XMX combination %d", idx); + } +} + +// AOT with -fsycl-targets=intel_gpu_*: compile each tile body only for targets with its sub-group +// size, since IGC fails on the other ones. A JIT build keeps them all, but each combination lands in +// its own device image (joint_matrix is an optional kernel feature) and only a combination the +// device reports is launched, so the runtime never asks IGC for the others. +#if defined(__SYCL_DEVICE_ONLY__) +# if __SYCL_TARGET_INTEL_GPU_ACM_G10__ || __SYCL_TARGET_INTEL_GPU_ACM_G11__ || __SYCL_TARGET_INTEL_GPU_ACM_G12__ || \ + __SYCL_TARGET_INTEL_GPU_ARL_H__ +# define FG_AOT_SG 8 +# elif __SYCL_TARGET_INTEL_GPU_PVC__ || __SYCL_TARGET_INTEL_GPU_PVC_VG__ || __SYCL_TARGET_INTEL_GPU_BMG_G21__ || \ + __SYCL_TARGET_INTEL_GPU_BMG_G31__ || __SYCL_TARGET_INTEL_GPU_LNL_M__ || __SYCL_TARGET_INTEL_GPU_PTL_H__ || \ + __SYCL_TARGET_INTEL_GPU_PTL_U__ || __SYCL_TARGET_INTEL_GPU_WCL__ || __SYCL_TARGET_INTEL_GPU_NVL_S__ || \ + __SYCL_TARGET_INTEL_GPU_NVL_U__ || __SYCL_TARGET_INTEL_GPU_NVL_P__ +# define FG_AOT_SG 16 +# elif __SYCL_TARGET_INTEL_GPU_TGLLP__ || __SYCL_TARGET_INTEL_GPU_RKL__ || __SYCL_TARGET_INTEL_GPU_ADL_S__ || \ + __SYCL_TARGET_INTEL_GPU_ADL_P__ || __SYCL_TARGET_INTEL_GPU_ADL_N__ || __SYCL_TARGET_INTEL_GPU_DG1__ || \ + __SYCL_TARGET_INTEL_GPU_MTL_U__ || __SYCL_TARGET_INTEL_GPU_MTL_H__ +# define FG_AOT_SG 0 // no XMX +# endif +#endif + +template static constexpr bool fg_built() { +#if defined(FG_AOT_SG) + return SG == FG_AOT_SG; +#else + return true; +#endif +} + +// Upper bound on the tile count when total_rows rows are routed to n_as experts: the worst case gives +// each expert one row and fills whole tiles with the rest. The bound depends only on the shape, not +// on the routing, so the pool reuses one buffer every ubatch instead of keeping one per size seen. +static constexpr int64_t grouped_gemm_max_tiles(int64_t total_rows, int64_t n_as, int64_t BN) { + return total_rows <= n_as ? total_rows : n_as + (total_rows - n_as) / BN; +} +// Tiles do not cross experts, so the bound is not ceil(total_rows / BN): 34 rows over 2 experts with +// BN = 16 split 17 + 17 need 2 + 2 tiles, where the ceil gives 3. +static_assert(grouped_gemm_max_tiles(34, 2, 16) == 4); + +// the device lists S with an f32 accumulator and output +template static bool fg_device_has_combo(const std::vector & combinations) { + for (const auto & c : combinations) { + if (c.atype == S::EA::mtype && c.btype == S::EB::mtype && c.ctype == mx::matrix_type::fp32 && + c.dtype == mx::matrix_type::fp32 && + (c.max_msize >= (size_t) S::TM || c.msize == (size_t) S::TM) && + (c.max_nsize >= (size_t) S::TN || c.nsize == (size_t) S::TN) && + (c.max_ksize >= (size_t) S::TK || c.ksize == (size_t) S::TK)) { + return true; + } + } + return false; +} + +template static const char * fg_type_name() { + return std::is_same_v ? "f16" : std::is_same_v ? "bf16" : "tf32"; +} + +static std::string fg_combo_name(int idx) { + std::string name; + fg_visit_combo(idx, [&](auto s) { + using S = decltype(s); + name = std::string(fg_type_name()) + "x" + fg_type_name() + " " + + std::to_string(S::TM) + "x" + std::to_string(S::TN) + "x" + std::to_string(S::TK) + " sg" + + std::to_string(S::SG); + }); + return name; +} + +// Combinations this build has kernels for and the device lists, one bit each. Cached per device: +// on a mixed box the first caller's verdict is not the others'. +static int fg_device_combos(const sycl::device & dev) { + static std::mutex mtx; + static std::unordered_map known; + std::lock_guard lock(mtx); + const auto it = known.find(dev); + if (it != known.end()) { + return it->second; + } + int available = 0; + try { + const auto combinations = dev.get_info(); + const auto sg_sizes = dev.get_info(); + for (int idx = 0; idx < FG_N_COMBOS; ++idx) { + fg_visit_combo(idx, [&](auto s) { + using S = decltype(s); + const bool sg_ok = std::find(sg_sizes.begin(), sg_sizes.end(), (size_t) S::SG) != sg_sizes.end(); + if (sg_ok && fg_device_has_combo(combinations)) { + available |= 1 << idx; + } + }); + } + } catch (const sycl::exception &) { + available = 0; + } + GGML_LOG_INFO("%s: %s: XMX dequant-GEMM combinations available 0x%x, allowed 0x%x\n", __func__, + dev.get_info().c_str(), available, g_ggml_sycl_xmx_gather_shapes); + known.emplace(dev, available); + return available; +} + +// Rank of combination S for a src1 of type src1_type, lower is better. Order: +// 1. throughput: a tile that does not spill, then the fastest type class of A and B +// 2. B type equal to the src1 type, so the pack is a plain copy +// 3. B at least as precise as f16 +// 4. the device's native DPAS tile (8 x SG x 32 bytes of K), then the largest M x K +// A costs nothing to convert: the A stage emits any type at the same cost. +template static int64_t fg_rank(ggml_type src1_type) { + const int64_t spills = !S::efficient; + const int64_t slow = std::max(S::EA::slow, S::EB::slow); + const int64_t convert = S::EB::src != src1_type; + const int64_t lossy = S::EB::mant < 10; + const int64_t foreign = !(S::TM == 8 && S::TN == S::SG); + const int64_t mk = 1024 - S::TM * S::TK; + return ((((spills * 2 + slow) * 2 + convert) * 2 + lossy) * 2 + foreign) * 2048 + mk; +} + +// whether XMX operands of type mode (a GGML_SYCL_DYNAMIC_PRECISION value) meet the src1 request +// [TAG_GGML_PREC]. f16 lacks the f32 range that BF16 and F32 ask for; an F32 request goes only as far +// down as GGML_SYCL_DYNAMIC_REQUIRED_PRECISION allows. +static bool fg_mode_meets(int mode, int32_t src1_prec) { + if (src1_prec == GGML_PREC_UNDEFINED || src1_prec >= GGML_PREC_F16) { + return true; + } + if (src1_prec >= GGML_PREC_BF16) { + return mode != GGML_SYCL_DYNAMIC_PRECISION_F16; + } + switch (g_ggml_sycl_dynamic_required_precision) { + case GGML_SYCL_DYNAMIC_PRECISION_TF32: return mode == GGML_SYCL_DYNAMIC_PRECISION_TF32; + case GGML_SYCL_DYNAMIC_PRECISION_BF16: return mode != GGML_SYCL_DYNAMIC_PRECISION_F16; + default: return false; + } +} + +// Best allowed combination for this call, or -1 if none. The type is GGML_SYCL_DYNAMIC_PRECISION if it +// meets the src1 request; if not, bf16 then tf32 for a BF16 request (fastest first) and tf32 then bf16 +// for an F32 request (most mantissa first). +static int fg_pick_combo(dpct::queue_ptr stream, ggml_type src1_type, int32_t src1_prec) { + const sycl::device dev = stream->get_device(); + const int allowed = fg_device_combos(dev) & g_ggml_sycl_xmx_gather_shapes; + const bool f32_req = src1_prec != GGML_PREC_UNDEFINED && src1_prec < GGML_PREC_BF16; + const int modes[] = { + g_ggml_sycl_dynamic_precision, + f32_req ? GGML_SYCL_DYNAMIC_PRECISION_TF32 : GGML_SYCL_DYNAMIC_PRECISION_BF16, + f32_req ? GGML_SYCL_DYNAMIC_PRECISION_BF16 : GGML_SYCL_DYNAMIC_PRECISION_TF32, + }; + int best = -1; + int64_t best_rank = 0; + for (int i = 0; i < 3 && best < 0; ++i) { + const int mode = modes[i]; + if (!fg_mode_meets(mode, src1_prec)) { + continue; + } + for (int idx = 0; idx < FG_N_COMBOS; ++idx) { + if (!(allowed & (1 << idx))) { + continue; + } + fg_visit_combo(idx, [&](auto s) { + using S = decltype(s); + if (S::EA::mode != mode || S::EB::mode != mode) { + return; + } + const int64_t rank = fg_rank(src1_type); + if (best < 0 || rank < best_rank) { + best = idx; + best_rank = rank; + } + }); + } + } + // log each distinct decision once + static std::mutex mtx; + static std::set> seen; + std::lock_guard lock(mtx); + if (seen.emplace(std::hash{}(dev), (int) src1_type, src1_prec, best).second) { + GGML_LOG_INFO("%s: src1 %s, src1 prec %d -> %s\n", __func__, ggml_type_name(src1_type), src1_prec, + best >= 0 ? fg_combo_name(best).c_str() : "none (library GEMM)"); + } + return best; +} + +// src1 [N][K] -> packed [K/V][Npad][V] so B tiles load straight from global memory +template +static void fused_gemm_pack_b(const T_src * y, typename E::store * packed, int N, int Npad, int K, + dpct::queue_ptr stream) { + constexpr int V = 4 / sizeof(typename E::store); + const int kqs = K / V; + stream->parallel_for(sycl::range<1>((size_t) Npad * kqs), [=](sycl::id<1> id) { + const int idx = id[0]; + const int n = idx / kqs; + const int kq = idx - n * kqs; + typename E::store vals[V] = {}; + if (n < N) { + const T_src * src = y + (size_t) n * K + V * kq; +#pragma unroll + for (int v = 0; v < V; ++v) { + vals[v] = E::cvt((float) src[v]); + } + } + typename E::store * out = packed + ((size_t) kq * Npad + n) * V; +#pragma unroll + for (int v = 0; v < V; ++v) { + out[v] = vals[v]; + } + }); +} + +// A stage: one lane owns one row and decodes FG_BK values of it per k step, with every scale +// folded into the value so the mad below sees plain A elements. One overload per weight format. +template +static __dpct_inline__ void fg_stage_a(const block_iq4_nl * __restrict__ xrow, const int kb, typename E::pair * a) { + const block_iq4_nl blk = xrow[kb]; + const float d = (float) blk.d; +#pragma unroll + for (int j = 0; j < QK4_NL / 2; j += 2) { + const uint8_t q0 = blk.qs[j]; + const uint8_t q1 = blk.qs[j + 1]; + a[j / 2] = E::make(d * kvalues_iq4nl[q0 & 0xf], d * kvalues_iq4nl[q1 & 0xf]); + a[j / 2 + 8] = E::make(d * kvalues_iq4nl[q0 >> 4], d * kvalues_iq4nl[q1 >> 4]); + } +} + +// iq3_s: k step kb is sub-block kb % 8 of superblock kb / 8. The superblock is 110 bytes, so read +// only the fields of that sub-block instead of copying the block. Same decode as +// dequantize_block_iq3_s: grid entries are taken as dwords and the sign bit is a plain shift. +template +static __dpct_inline__ void fg_stage_a(const block_iq3_s * __restrict__ xrow, const int kb, typename E::pair * a) { + static_assert(QK_K == 256, "the iq3_s A stage assumes 8 sub-blocks per superblock"); + const block_iq3_s * blk = xrow + kb / (QK_K / 32); + const int ib8 = kb % (QK_K / 32); + const uint8_t * qs = blk->qs + 8 * ib8; + const int qh = blk->qh[ib8]; + const float d = (float) blk->d * (1 + 2 * ((blk->scales[ib8 / 2] >> (4 * (ib8 % 2))) & 0xf)); +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint32_t grid1 = iq3s_grid[qs[2 * il + 0] | ((qh << (8 - 2 * il)) & 256)]; + const uint32_t grid2 = iq3s_grid[qs[2 * il + 1] | ((qh << (7 - 2 * il)) & 256)]; + const int signs = blk->signs[4 * ib8 + il]; +#pragma unroll + for (int j = 0; j < 2; ++j) { + const float g1a = (float) ((grid1 >> (16 * j + 0)) & 0xff); + const float g1b = (float) ((grid1 >> (16 * j + 8)) & 0xff); + const float g2a = (float) ((grid2 >> (16 * j + 0)) & 0xff); + const float g2b = (float) ((grid2 >> (16 * j + 8)) & 0xff); + const int s = 2 * j; + a[4 * il + j] = E::make(d * ((signs & (1 << (s + 0))) ? -g1a : g1a), + d * ((signs & (1 << (s + 1))) ? -g1b : g1b)); + a[4 * il + j + 2] = E::make(d * ((signs & (1 << (s + 4))) ? -g2a : g2a), + d * ((signs & (1 << (s + 5))) ? -g2b : g2b)); + } + } +} + +// values per stored block, so a row of K values is K/qk blocks +template struct fg_block_traits; +template <> struct fg_block_traits { static constexpr int qk = QK4_NL; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK8_0; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; +template <> struct fg_block_traits { static constexpr int qk = QK_K; }; + +// The A stages below are the dequantize_block_iq* kernels rewritten for one k step. There a +// work-item handled one quarter (il) of one 32-wide sub-block (ib); here one lane produces the +// whole step, so il becomes a loop and ib is kb inside the superblock. Each quarter yields 8 +// consecutive values, i.e. 4 pairs at a[4*il], so nothing larger than 8 floats is ever live. +#define FG_SUPERBLOCK(T) \ + static_assert(QK_K == 256, "the " #T " A stage assumes 8 sub-blocks per superblock"); \ + const T * blk = xrow + kb / (QK_K / 32); \ + const int ib = kb % (QK_K / 32) + +template +static __dpct_inline__ void fg_pack_quarter(const float * __restrict__ t, typename E::pair * a, int il) { +#pragma unroll + for (int j = 0; j < 4; ++j) { + a[4 * il + j] = E::make(t[2 * j], t[2 * j + 1]); + } +} + +template +static __dpct_inline__ void fg_stage_a(const block_iq4_xs * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq4_xs); + // low nibbles fill the first half of the step, high nibbles the second, so the two halves + // land at a[0..7] and a[8..15] and no quarter loop is needed + const float d = (float) blk->d * + ((((blk->scales_l[ib / 2] >> (4 * (ib % 2))) & 0xf) | (((blk->scales_h >> (2 * ib)) & 3) << 4)) - 32); + const uint8_t * q4 = blk->qs + 16 * ib; +#pragma unroll + for (int j = 0; j < 8; ++j) { + a[j] = E::make(d * kvalues_iq4nl[q4[2 * j] & 0xf], d * kvalues_iq4nl[q4[2 * j + 1] & 0xf]); + a[8 + j] = E::make(d * kvalues_iq4nl[q4[2 * j] >> 4], d * kvalues_iq4nl[q4[2 * j + 1] >> 4]); + } +} + +template +static __dpct_inline__ void fg_stage_a(const block_iq3_xxs * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq3_xxs); + const uint8_t * q3 = blk->qs + 8 * ib; + const uint16_t * gas = (const uint16_t *) (blk->qs + QK_K / 4) + 2 * ib; + const uint32_t aux32 = gas[0] | (gas[1] << 16); + const float d = (float) blk->d * (0.5f + (aux32 >> 28)) * 0.5f; +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint8_t * grid1 = (const uint8_t *) (iq3xxs_grid + q3[2 * il + 0]); + const uint8_t * grid2 = (const uint8_t *) (iq3xxs_grid + q3[2 * il + 1]); + const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127]; + float t[8]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + t[j + 0] = d * grid1[j] * (signs & kmask_iq2xs[j + 0] ? -1.f : 1.f); + t[j + 4] = d * grid2[j] * (signs & kmask_iq2xs[j + 4] ? -1.f : 1.f); + } + fg_pack_quarter(t, a, il); + } +} + +template +static __dpct_inline__ void fg_stage_a(const block_iq2_xxs * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq2_xxs); + const uint16_t * q2 = blk->qs + 4 * ib; + const uint8_t * aux8 = (const uint8_t *) q2; + const uint32_t aux32 = q2[2] | (q2[3] << 16); + const float d = (float) blk->d * (0.5f + (aux32 >> 28)) * 0.25f; +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint8_t * grid = (const uint8_t *) (iq2xxs_grid + aux8[il]); + const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127]; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + fg_pack_quarter(t, a, il); + } +} + +template +static __dpct_inline__ void fg_stage_a(const block_iq2_xs * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq2_xs); + const uint16_t * q2 = blk->qs + 4 * ib; +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint8_t * grid = (const uint8_t *) (iq2xs_grid + (q2[il] & 511)); + const float d = (float) blk->d * (0.5f + ((blk->scales[ib] >> (4 * (il / 2))) & 0xf)) * 0.25f; + const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + fg_pack_quarter(t, a, il); + } +} + +template +static __dpct_inline__ void fg_stage_a(const block_iq2_s * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq2_s); +#pragma unroll + for (int il = 0; il < 4; ++il) { + const uint8_t * grid = + (const uint8_t *) (iq2s_grid + (blk->qs[4 * ib + il] | ((blk->qh[ib] << (8 - 2 * il)) & 0x300))); + const float d = (float) blk->d * (0.5f + ((blk->scales[ib] >> (4 * (il / 2))) & 0xf)) * 0.25f; + const uint8_t signs = blk->qs[QK_K / 8 + 4 * ib + il]; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); + } + fg_pack_quarter(t, a, il); + } +} + +template +static __dpct_inline__ void fg_stage_a(const block_iq1_s * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq1_s); + const float delta = blk->qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA; + const float d = (float) blk->d * (2 * ((blk->qh[ib] >> 12) & 7) + 1); +#pragma unroll + for (int il = 0; il < 4; ++il) { + uint32_t grid32[2]; + const int8_t * q = (const int8_t *) grid32; + grid32[0] = iq1s_grid_gpu[blk->qs[4 * ib + il] | (((blk->qh[ib] >> (3 * il)) & 7) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * (q[j] + delta); + } + fg_pack_quarter(t, a, il); + } +} + +template +static __dpct_inline__ void fg_stage_a(const block_iq1_m * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_iq1_m); + const uint16_t * sc = (const uint16_t *) blk->scales; + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); +#pragma unroll + for (int il = 0; il < 4; ++il) { + const int ib16 = 2 * ib + il / 2; + const float d = (float) scale.f16 * (2 * ((sc[ib16 / 4] >> (3 * (ib16 % 4))) & 0x7) + 1); + const float delta = blk->qh[2 * ib + il / 2] & (0x08 << (4 * (il % 2))) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA; + uint32_t grid32[2]; + const int8_t * q = (const int8_t *) grid32; + grid32[0] = iq1s_grid_gpu[blk->qs[4 * ib + il] | + (((blk->qh[2 * ib + il / 2] >> (4 * (il % 2))) & 7) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + float t[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + t[j] = d * (q[j] + delta); + } + fg_pack_quarter(t, a, il); + } +} + + +// q8_0 and the k-quants. Each decode takes the fields of one 32-value sub-block, so the canonical +// layout and the reorder (SoA) layout of reorder_qw() share it and differ only in where the fields +// live, as in dequantize.hpp. +template +static __dpct_inline__ void fg_decode_q8_0(const int8_t * __restrict__ qs, const float d, typename E::pair * a) { +#pragma unroll + for (int j = 0; j < QK8_0 / 2; ++j) { + a[j] = E::make(d * qs[2 * j], d * qs[2 * j + 1]); + } +} + +// same unpack as get_scale_min_k4() in dequantize.hpp +static __dpct_inline__ void fg_scale_min_k4(const int j, const uint8_t * __restrict__ q, uint8_t & d, uint8_t & m) { + if (j < 4) { + d = q[j] & 63; + m = q[j + 4] & 63; + } else { + d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4); + } +} + +// q4_K sub-block ib (0..7) uses scale/min pair ib and the low (even ib) or high (odd ib) nibbles of +// qs[32 * (ib / 2) ...], as in dequantize_row_q4_K +template +static __dpct_inline__ void fg_decode_q4_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ scales, + const sycl::half2 dm, const int ib, typename E::pair * a) { + uint8_t sc, mb; + fg_scale_min_k4(ib, scales, sc, mb); + const float d = (float) dm[0] * sc; + const float m = (float) dm[1] * mb; + const uint8_t * q = qs + 32 * (ib / 2); + const int shift = 4 * (ib % 2); +#pragma unroll + for (int j = 0; j < 16; ++j) { + a[j] = E::make(d * ((q[2 * j] >> shift) & 0xF) - m, d * ((q[2 * j + 1] >> shift) & 0xF) - m); + } +} + +// q5_K: q4_K plus one high bit per value, bit ib of qh[l] +template +static __dpct_inline__ void fg_decode_q5_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ qh, + const uint8_t * __restrict__ scales, const sycl::half2 dm, const int ib, + typename E::pair * a) { + uint8_t sc, mb; + fg_scale_min_k4(ib, scales, sc, mb); + const float d = (float) dm[0] * sc; + const float m = (float) dm[1] * mb; + const uint8_t * q = qs + 32 * (ib / 2); + const int shift = 4 * (ib % 2); +#pragma unroll + for (int j = 0; j < 16; ++j) { + const int l = 2 * j; + a[j] = E::make(d * (((q[l] >> shift) & 0xF) | (((qh[l] >> ib) & 1) << 4)) - m, + d * (((q[l + 1] >> shift) & 0xF) | (((qh[l + 1] >> ib) & 1) << 4)) - m); + } +} + +// q6_K: sub-block ib is quarter r = ib % 4 of half h = ib / 4. The half selects ql + 64h, qh + 32h +// and scales + 8h; the quarter selects ql + 32(r & 1), the ql nibble r / 2, the qh bit pair r and +// scales + 2r, as in dequantize_row_q6_K. The scale changes at value 16 of the sub-block. +template +static __dpct_inline__ void fg_decode_q6_K(const uint8_t * __restrict__ ql, const uint8_t * __restrict__ qh, + const int8_t * __restrict__ scales, const float d, const int ib, + typename E::pair * a) { + const int h = ib / 4; + const int r = ib % 4; + const uint8_t * q = ql + 64 * h + 32 * (r & 1); + const uint8_t * hb = qh + 32 * h; + const int8_t * sc = scales + 8 * h + 2 * r; +#pragma unroll + for (int j = 0; j < 16; ++j) { + const int l = 2 * j; + const float dl = d * sc[j / 8]; + const int q0 = (((q[l] >> (4 * (r / 2))) & 0xF) | (((hb[l] >> (2 * r)) & 3) << 4)) - 32; + const int q1 = (((q[l + 1] >> (4 * (r / 2))) & 0xF) | (((hb[l + 1] >> (2 * r)) & 3) << 4)) - 32; + a[j] = E::make(dl * q0, dl * q1); + } +} + +template +static __dpct_inline__ void fg_stage_a(const block_q8_0 * __restrict__ xrow, const int kb, typename E::pair * a) { + fg_decode_q8_0(xrow[kb].qs, (float) xrow[kb].d, a); +} + +template +static __dpct_inline__ void fg_stage_a(const block_q4_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q4_K); + fg_decode_q4_K(blk->qs, blk->scales, blk->dm, ib, a); +} + +template +static __dpct_inline__ void fg_stage_a(const block_q5_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q5_K); + fg_decode_q5_K(blk->qs, blk->qh, blk->scales, blk->dm, ib, a); +} + +template +static __dpct_inline__ void fg_stage_a(const block_q6_K * __restrict__ xrow, const int kb, typename E::pair * a) { + FG_SUPERBLOCK(block_q6_K); + fg_decode_q6_K(blk->ql, blk->qh, blk->scales, (float) blk->d, ib, a); +} + +// Reorder (SoA) layout: each block field is one stream over the nblocks of the matrix (of the expert +// slice for MUL_MAT_ID), in the order reorder_qw() writes them. Block ib holds k step kb. +template struct fg_soa { + static constexpr bool supported = false; +}; + +template <> struct fg_soa { + static constexpr bool supported = true; + // [qs][d] + template + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int, + typename E::pair * a) { + const float d = (float) ((const sycl::half *) (x + nblocks * QK8_0))[ib]; + fg_decode_q8_0((const int8_t *) x + ib * QK8_0, d, a); + } +}; + +template <> struct fg_soa { + static constexpr bool supported = true; + // [qs][scales][dm] + template + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * scales = x + nblocks * (QK_K / 2); + const sycl::half2 dm = ((const sycl::half2 *) (scales + nblocks * K_SCALE_SIZE))[ib]; + fg_decode_q4_K(x + ib * (QK_K / 2), scales + ib * K_SCALE_SIZE, dm, kb % (QK_K / 32), a); + } +}; + +template <> struct fg_soa { + static constexpr bool supported = true; + // [qs][qh][scales][dm] + template + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * qh = x + nblocks * (QK_K / 2); + const uint8_t * scales = qh + nblocks * (QK_K / 8); + const sycl::half2 dm = ((const sycl::half2 *) (scales + nblocks * K_SCALE_SIZE))[ib]; + fg_decode_q5_K(x + ib * (QK_K / 2), qh + ib * (QK_K / 8), scales + ib * K_SCALE_SIZE, dm, + kb % (QK_K / 32), a); + } +}; + +template <> struct fg_soa { + static constexpr bool supported = true; + // [ql][qh][scales][d] + template + static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb, + typename E::pair * a) { + const uint8_t * qh = x + nblocks * (QK_K / 2); + const uint8_t * scales = qh + nblocks * (QK_K / 4); + const float d = (float) ((const sycl::half *) (scales + nblocks * (QK_K / 16)))[ib]; + fg_decode_q6_K(x + ib * (QK_K / 2), qh + ib * (QK_K / 4), (const int8_t *) scales + ib * (QK_K / 16), d, + kb % (QK_K / 32), a); + } +}; + + +// one SG_ROWS x BN output tile: B columns [b0, b0 + BN) of packed_b go to dst columns [n0, n1), +// n1 - n0 <= BN +template +static void fused_dequant_gemm_tile( + const block_q_t * __restrict__ x, + const typename S::tsb * __restrict__ packed_b, + float * __restrict__ dst, + const int M, const int Npad, const int K, const int ldd, + const int b0, const int n0, const int n1, + sycl::local_accessor tile_a, + sycl::local_accessor tile_c, + const sycl::nd_item<2> & item) { + if constexpr (fg_built()) { + using EA = typename S::EA; + using TA = typename S::ta; + using TB = typename S::tb; + const auto sg = item.get_sub_group(); + const int sg_id = sg.get_group_id()[0]; + const int lane = sg.get_local_id()[0]; + const int m0 = item.get_group(1) * S::SG_ROWS; + const int nstep = K / FG_BK; + const int a_base = sg_id * S::SG_ROWS * FG_BK; + const int c_base = sg_id * S::SG_ROWS * S::BN; + + mx::joint_matrix acc[S::MT][S::NT]; +#pragma unroll + for (int mt = 0; mt < S::MT; ++mt) { +#pragma unroll + for (int nt = 0; nt < S::NT; ++nt) { + mx::joint_matrix_fill(sg, acc[mt][nt], 0.0f); + } + } + + // lane decodes rows lane, lane + SG, ... of the sub-group's SG_ROWS + constexpr int KPB = fg_block_traits::qk / FG_BK; // k steps per block + const size_t bpr = K / fg_block_traits::qk; + const size_t nblocks = (size_t) M * bpr; + size_t row_blk[S::RPL]; + bool row_ok[S::RPL]; + typename EA::pair * a[S::RPL]; +#pragma unroll + for (int r = 0; r < S::RPL; ++r) { + const int row = m0 + r * S::SG + lane; + row_ok[r] = row < M; + row_blk[r] = (size_t) (row_ok[r] ? row : 0) * bpr; + a[r] = (typename EA::pair *) &tile_a[a_base + (r * S::SG + lane) * FG_BK]; + } + + const auto b_ptr = sycl::address_space_cast(packed_b); + const int b_stride = Npad * S::VNNI; + + const int kb_begin = (sg_id * nstep) / FG_KSPLIT; + const int kb_end = ((sg_id + 1) * nstep) / FG_KSPLIT; + for (int kb = kb_begin; kb < kb_end; ++kb) { +#pragma unroll + for (int r = 0; r < S::RPL; ++r) { + if (row_ok[r]) { + if constexpr (reordered) { + fg_soa::template stage((const uint8_t *) x, nblocks, row_blk[r] + kb / KPB, kb, + a[r]); + } else { + fg_stage_a(x + row_blk[r], kb, a[r]); + } + } else { +#pragma unroll + for (int j = 0; j < FG_BK / 2; ++j) { + a[r][j] = EA::make(0.0f, 0.0f); + } + } + } + sycl::group_barrier(sg); + +#pragma unroll + for (int kt = 0; kt < FG_BK / S::TK; ++kt) { + const int kq0 = (kb * FG_BK + kt * S::TK) / S::VNNI; + mx::joint_matrix sub_b[S::NT]; +#pragma unroll + for (int nt = 0; nt < S::NT; ++nt) { + mx::joint_matrix_load(sg, sub_b[nt], b_ptr + (size_t) kq0 * b_stride + (b0 + nt * S::TN) * S::VNNI, b_stride); + } +#pragma unroll + for (int mt = 0; mt < S::MT; ++mt) { + mx::joint_matrix sub_a; + mx::joint_matrix_load(sg, sub_a, + tile_a.template get_multi_ptr() + a_base + (mt * S::TM) * FG_BK + kt * S::TK, + FG_BK); +#pragma unroll + for (int nt = 0; nt < S::NT; ++nt) { + mx::joint_matrix_mad(sg, acc[mt][nt], sub_a, sub_b[nt], acc[mt][nt]); + } + } + } + // the next step overwrites tile_a + sycl::group_barrier(sg); + } + +#pragma unroll + for (int mt = 0; mt < S::MT; ++mt) { +#pragma unroll + for (int nt = 0; nt < S::NT; ++nt) { + mx::joint_matrix_store(sg, acc[mt][nt], + tile_c.template get_multi_ptr() + c_base + (mt * S::TM) * S::BN + nt * S::TN, + S::BN, mx::layout::row_major); + } + } + sycl::group_barrier(item.get_group()); + + // sum the K splits; consecutive lanes write consecutive rows of one dst column + for (int idx = item.get_local_linear_id(); idx < S::SG_ROWS * S::BN; idx += S::WG_SIZE) { + const int r = idx % S::SG_ROWS; + const int c = idx / S::SG_ROWS; + const int m = m0 + r; + const int n = n0 + c; + if (m < M && n < n1) { + float sum = 0.0f; +#pragma unroll + for (int s = 0; s < FG_KSPLIT; ++s) { + sum += tile_c[s * S::SG_ROWS * S::BN + r * S::BN + c]; + } + dst[(size_t) n * ldd + m] = sum; + } + } + } +} + +template +static void fused_dequant_gemm_launch(const void * src0, const typename S::tsb * packed, float * dst, const int M, + const int N, const int Npad, const int K, const int ldd, + const int64_t groups_n, const int64_t groups_m, dpct::queue_ptr stream) { + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor tile_a(FG_KSPLIT * S::SG_ROWS * FG_BK, cgh); + sycl::local_accessor tile_c(FG_KSPLIT * S::SG_ROWS * S::BN, cgh); + cgh.parallel_for( + sycl::nd_range<2>(sycl::range<2>(groups_n, groups_m * S::WG_SIZE), sycl::range<2>(1, S::WG_SIZE)), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(S::SG)]] { + const int n0 = item.get_group(0) * S::BN; + fused_dequant_gemm_tile((const block_q_t *) src0, packed, dst, M, Npad, K, + ldd, n0, n0, N, tile_a, tile_c, item); + }); + }); +} + +// grouped: work-group (t, mt) is tile t of the schedule; its B columns sit at t * BN +template +static void grouped_dequant_gemm_launch(const char * src0_dd, const size_t expert_stride, + const ggml_sycl_gg_tile * tiles_ptr, const typename S::tsb * packed, float * dst, + const int M, const int Npad, const int K, const int64_t n_tiles, + const int64_t groups_m, dpct::queue_ptr stream) { + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor tile_a(FG_KSPLIT * S::SG_ROWS * FG_BK, cgh); + sycl::local_accessor tile_c(FG_KSPLIT * S::SG_ROWS * S::BN, cgh); + cgh.parallel_for( + sycl::nd_range<2>(sycl::range<2>(n_tiles, groups_m * S::WG_SIZE), sycl::range<2>(1, S::WG_SIZE)), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(S::SG)]] { + const int t = item.get_group(0); + const ggml_sycl_gg_tile tile = tiles_ptr[t]; + const block_q_t * x = (const block_q_t *) (src0_dd + (size_t) tile.expert * expert_stride); + fused_dequant_gemm_tile(x, packed, dst, M, Npad, K, M, t * S::BN, tile.n0, + tile.n1, tile_a, tile_c, item); + }); + }); +} + +// src1 f32 rows -> packed [K/V][n_tiles*BN][V], tile t holds its rows [n0, n1) at columns t*BN.., +// zero past n1. The column runs fastest so a sub-group writes one contiguous run. +template +static void grouped_gemm_pack_b(const float * y, typename S::tsb * packed, const ggml_sycl_gg_tile * tiles, int Npad, + int K, dpct::queue_ptr stream) { + using E = typename S::EB; + constexpr int V = S::VNNI; + const int kqs = K / V; + stream->parallel_for(sycl::range<1>((size_t) Npad * kqs), [=](sycl::id<1> id) { + const size_t idx = id[0]; + const int kq = idx / Npad; + const int n = idx - (size_t) kq * Npad; + const ggml_sycl_gg_tile tile = tiles[n / S::BN]; + const int row = tile.n0 + n % S::BN; + // one guarded load run per work-item, as a per-element select costs ~1.5% prefill + typename S::tsb vals[V] = {}; + if (row < tile.n1) { + const float * src = y + (size_t) row * K + V * kq; +#pragma unroll + for (int v = 0; v < V; ++v) { + vals[v] = E::cvt(src[v]); + } + } + typename S::tsb * out = packed + ((size_t) kq * Npad + n) * V; +#pragma unroll + for (int v = 0; v < V; ++v) { + out[v] = vals[v]; + } + }); +} + +// q8_0 and the k-quants take only the grouped path. The plain kernel decodes A again for every BN columns +// of a dense batch, and for these formats that costs more than the one dequantization of the library GEMM. +template static constexpr bool fg_plain_ok() { + return !std::is_same_v && !std::is_same_v && !std::is_same_v && + !std::is_same_v; +} + +template struct fg_tag { + using type = T; + static constexpr bool reordered = R; +}; + +// calls f(fg_tag{}) for the weight format and layout; false if it has no A stage +template static bool fg_visit_layout(bool reordered, F && f) { + if (!reordered) { + f(fg_tag{}); + return true; + } + if constexpr (fg_soa::supported) { + f(fg_tag{}); + return true; + } + return false; +} + +template static bool fg_visit_type(ggml_type type, bool reordered, F && f) { + switch (type) { + case GGML_TYPE_IQ4_NL: return fg_visit_layout(reordered, f); + case GGML_TYPE_IQ3_S: return fg_visit_layout(reordered, f); + case GGML_TYPE_IQ4_XS: return fg_visit_layout(reordered, f); + case GGML_TYPE_IQ3_XXS: return fg_visit_layout(reordered, f); + case GGML_TYPE_IQ2_XXS: return fg_visit_layout(reordered, f); + case GGML_TYPE_IQ2_XS: return fg_visit_layout(reordered, f); + case GGML_TYPE_IQ2_S: return fg_visit_layout(reordered, f); + case GGML_TYPE_IQ1_S: return fg_visit_layout(reordered, f); + case GGML_TYPE_IQ1_M: return fg_visit_layout(reordered, f); + case GGML_TYPE_Q8_0: return fg_visit_layout(reordered, f); + case GGML_TYPE_Q4_K: return fg_visit_layout(reordered, f); + case GGML_TYPE_Q5_K: return fg_visit_layout(reordered, f); + case GGML_TYPE_Q6_K: return fg_visit_layout(reordered, f); + default: return false; + } +} + +template +static bool fg_fused_run(ggml_type src0_type, bool src0_reordered, const void * src0, const void * src1, ggml_type src1_type, + float * dst, int64_t M, int64_t N, int64_t K, int64_t ldd, ggml_sycl_pool & pool, + dpct::queue_ptr stream) { + const int64_t groups_n = (N + S::BN - 1) / S::BN; + const int64_t groups_m = (M + S::SG_ROWS - 1) / S::SG_ROWS; + const int Npad = (int) (groups_n * S::BN); + + // src1 is read in its own type: one pass, converted in registers only if B differs + ggml_sycl_pool_alloc packed_b(pool, (size_t) K * Npad); + if (src1_type == GGML_TYPE_F16) { + fused_gemm_pack_b((const sycl::half *) src1, packed_b.get(), (int) N, Npad, (int) K, stream); + } else if (src1_type == GGML_TYPE_BF16) { + fused_gemm_pack_b((const fg_bf16 *) src1, packed_b.get(), (int) N, Npad, (int) K, stream); + } else { + fused_gemm_pack_b((const float *) src1, packed_b.get(), (int) N, Npad, (int) K, stream); + } + + const typename S::tsb * packed = packed_b.get(); + return fg_visit_type(src0_type, src0_reordered, [&](auto tag) { + using block_q_t = typename decltype(tag)::type; + if constexpr (fg_plain_ok()) { + fused_dequant_gemm_launch(src0, packed, dst, (int) M, (int) N, + Npad, (int) K, (int) ldd, groups_n, + groups_m, stream); + } + }); +} + +bool ggml_sycl_fused_dequant_gemm(ggml_type src0_type, bool src0_reordered, const void * src0, const void * src1, ggml_type src1_type, + int32_t src1_prec, float * dst, int64_t M, int64_t N, int64_t K, int64_t ldd, + ggml_sycl_pool & pool, dpct::queue_ptr stream) { + // every BN columns dequantize A again, so wide N is left to the library GEMM. + // src0_reordered: the persistent weight reorder may have been installed on this + // tensor by an earlier small-batch mul_mat; the A stage then has to decode the + // SoA layout. Types without an SoA stage (fg_soa::supported == false) + // decline via fg_visit_layout and the caller falls through to the + // reorder-aware conversion route. + bool plain_ok = false; + fg_visit_type(src0_type, src0_reordered, [&](auto tag) { plain_ok = fg_plain_ok(); }); + if (g_ggml_sycl_dynamic_precision == GGML_SYCL_DYNAMIC_PRECISION_F32 || + !ggml_sycl_xmx_gather_type_enabled(src0_type) || !plain_ok) { + return false; + } + if (src1_type != GGML_TYPE_F32 && src1_type != GGML_TYPE_F16 && src1_type != GGML_TYPE_BF16) { + return false; + } + if (!ggml_sycl_fused_dequant_gemm_shape_ok(src0_type, M, N, K, ldd)) { + return false; + } + const int combo = fg_pick_combo(stream, src1_type, src1_prec); + if (combo < 0) { + return false; + } + bool launched = false; + fg_visit_combo(combo, [&](auto s) { + launched = fg_fused_run(src0_type, src0_reordered, src0, src1, src1_type, dst, M, N, K, ldd, pool, stream); + }); + return launched; +} + +template +static bool fg_grouped_run(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, + const float * src1, float * dst, const int64_t * expert_row_offsets, int64_t n_as, int64_t M, + int64_t K, std::vector & tiles, ggml_sycl_pool & pool, + dpct::queue_ptr stream) { + // the host knows every slice, so it lays out the work-groups: no search on the device + tiles.clear(); + for (int64_t e = 0; e < n_as; ++e) { + const int64_t end = expert_row_offsets[e + 1]; + for (int64_t n0 = expert_row_offsets[e]; n0 < end; n0 += S::BN) { + tiles.push_back({ (int32_t) e, (int32_t) n0, (int32_t) std::min(n0 + S::BN, end) }); + } + } + const int64_t n_tiles = tiles.size(); + const int64_t groups_m = (M + S::SG_ROWS - 1) / S::SG_ROWS; + const int Npad = (int) (n_tiles * S::BN); + + const int64_t max_tiles = grouped_gemm_max_tiles(expert_row_offsets[n_as], n_as, S::BN); + GGML_ASSERT(n_tiles <= max_tiles); + ggml_sycl_pool_alloc tiles_dev(pool, max_tiles); + SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(tiles_dev.get(), tiles.data(), n_tiles * sizeof(ggml_sycl_gg_tile)))); + + ggml_sycl_pool_alloc packed_b(pool, (size_t) K * max_tiles * S::BN); + grouped_gemm_pack_b(src1, packed_b.get(), tiles_dev.get(), Npad, (int) K, stream); + + const typename S::tsb * packed = packed_b.get(); + const ggml_sycl_gg_tile * tiles_ptr = tiles_dev.get(); + const char * src0_dd = (const char *) src0_base; + return fg_visit_type(src0_type, reordered, [&](auto tag) { + using T = decltype(tag); + grouped_dequant_gemm_launch(src0_dd, expert_stride, tiles_ptr, packed, dst, + (int) M, Npad, (int) K, n_tiles, groups_m, + stream); + }); +} + +bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, + const float * src1, int32_t src1_prec, float * dst, + const int64_t * expert_row_offsets, int64_t n_as, int64_t M, int64_t K, + int64_t total_rows, std::vector & tiles, + ggml_sycl_pool & pool, dpct::queue_ptr stream) { + int64_t n_active = 0; + for (int64_t e = 0; e < n_as; ++e) { + n_active += expert_row_offsets[e + 1] > expert_row_offsets[e]; + } + if (g_ggml_sycl_dynamic_precision == GGML_SYCL_DYNAMIC_PRECISION_F32 || + !ggml_sycl_xmx_gather_type_enabled(src0_type) || !fg_visit_type(src0_type, reordered, [](auto) {})) { + return false; + } + if (!ggml_sycl_grouped_dequant_gemm_shape_ok(src0_type, M, K, total_rows, n_active)) { + return false; + } + const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec); + if (combo < 0) { + return false; + } + bool launched = false; + fg_visit_combo(combo, [&](auto s) { + launched = fg_grouped_run(src0_type, reordered, src0_base, expert_stride, src1, dst, + expert_row_offsets, n_as, M, K, tiles, pool, stream); + }); + return launched; +} diff --git a/ggml/src/ggml-sycl/fused-gemm.hpp b/ggml/src/ggml-sycl/fused-gemm.hpp new file mode 100644 index 0000000..226676d --- /dev/null +++ b/ggml/src/ggml-sycl/fused-gemm.hpp @@ -0,0 +1,88 @@ +#ifndef GGML_SYCL_FUSED_GEMM_HPP +#define GGML_SYCL_FUSED_GEMM_HPP + +#include "common.hpp" + + +// Shape and type gates for the kernels below. Device capability is separate: it needs a queue to ask. +static constexpr int GGML_SYCL_FG_MAX_N = 64; // widest N taken; each shape covers it in BN-wide tiles + +// weight formats the fused A stage decodes; K must cover whole stored blocks +constexpr bool ggml_sycl_fused_dequant_gemm_type_ok(ggml_type src0_type, int64_t K) { + // iq4_nl and q8_0 store 32 values per block; every other format here is a 256-value superblock + // that the A stage walks in steps of 32, so K must cover whole superblocks. + if (src0_type == GGML_TYPE_IQ4_NL || src0_type == GGML_TYPE_Q8_0) { + return K % 32 == 0; + } + const bool superblock = + src0_type == GGML_TYPE_Q4_K || + src0_type == GGML_TYPE_Q5_K || + src0_type == GGML_TYPE_Q6_K || + src0_type == GGML_TYPE_IQ3_S || + src0_type == GGML_TYPE_IQ4_XS || + src0_type == GGML_TYPE_IQ3_XXS || + src0_type == GGML_TYPE_IQ2_XXS || + src0_type == GGML_TYPE_IQ2_XS || + src0_type == GGML_TYPE_IQ2_S || + src0_type == GGML_TYPE_IQ1_S || + src0_type == GGML_TYPE_IQ1_M; + return superblock && QK_K == 256 && K % QK_K == 0; +} + +constexpr bool ggml_sycl_fused_dequant_gemm_shape_ok(ggml_type src0_type, int64_t M, int64_t N, int64_t K, + int64_t ldd) { + return ggml_sycl_fused_dequant_gemm_type_ok(src0_type, K) && M > 0 && N > 0 && K > 0 && + N <= GGML_SYCL_FG_MAX_N && + M <= INT32_MAX && N <= INT32_MAX && K <= INT32_MAX && ldd <= INT32_MAX; +} + +// grouped variant: the per-expert fused kernel is only worth it while each expert is narrow, +// so wider average slices are left to the per-expert library GEMM loop +constexpr bool ggml_sycl_grouped_dequant_gemm_shape_ok(ggml_type src0_type, int64_t M, int64_t K, + int64_t total_rows, int64_t n_active) { + return ggml_sycl_fused_dequant_gemm_shape_ok(src0_type, M, 1, K, M) && total_rows > 0 && + total_rows <= INT32_MAX && total_rows <= n_active * GGML_SYCL_FG_MAX_N; +} + +// Runtime type gate, kept out of the constexpr predicates above so those stay pure. +inline bool ggml_sycl_xmx_gather_type_enabled(ggml_type src0_type) { + switch (src0_type) { + case GGML_TYPE_IQ4_NL: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ4_NL ) != 0; + case GGML_TYPE_IQ3_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ3_S ) != 0; + case GGML_TYPE_IQ4_XS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ4_XS ) != 0; + case GGML_TYPE_IQ3_XXS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ3_XXS ) != 0; + case GGML_TYPE_IQ2_XXS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ2_XXS ) != 0; + case GGML_TYPE_IQ2_XS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ2_XS ) != 0; + case GGML_TYPE_IQ2_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ2_S ) != 0; + case GGML_TYPE_IQ1_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_S ) != 0; + case GGML_TYPE_IQ1_M: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_M ) != 0; + case GGML_TYPE_Q8_0: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q8_0 ) != 0; + case GGML_TYPE_Q4_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q4_K ) != 0; + case GGML_TYPE_Q5_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q5_K ) != 0; + case GGML_TYPE_Q6_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q6_K ) != 0; + default: return false; + } +} + +// dst[n*ldd + m] = sum_k dequant(src0)[m*K + k] * src1[n*K + k], src1 is F32, F16 or BF16. +// The XMX combination is picked per call from the src1 type and its precision request src1_prec +// (op_params[3], [TAG_GGML_PREC]); the accumulator is f32, which meets any request. +// q8_0 and the k-quants are not handled here, only in the grouped path below. +// Returns false when the case is not handled (type, device, precision, or shape). +bool ggml_sycl_fused_dequant_gemm(ggml_type src0_type, bool src0_reordered, const void * src0, const void * src1, ggml_type src1_type, + int32_t src1_prec, float * dst, int64_t M, int64_t N, int64_t K, int64_t ldd, + ggml_sycl_pool & pool, dpct::queue_ptr stream); + +// One launch for every expert of a MUL_MAT_ID: rows of src1/dst are grouped by expert, expert e +// owns rows [expert_row_offsets[e], expert_row_offsets[e+1]) and reads its weights at +// src0_base + e*expert_stride. tiles is host scratch that must stay alive until the queue drains. +// reordered: each expert slice is in the reorder (SoA) layout of reorder_qw(). +// dst[n*M + m] = sum_k dequant(src0_e)[m*K + k] * src1[n*K + k] +// Returns false when the case is not handled (type, layout, device, precision, or shape). +bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride, + const float * src1, int32_t src1_prec, float * dst, + const int64_t * expert_row_offsets, int64_t n_as, int64_t M, int64_t K, + int64_t total_rows, std::vector & tiles, + ggml_sycl_pool & pool, dpct::queue_ptr stream); + +#endif // GGML_SYCL_FUSED_GEMM_HPP diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp new file mode 100644 index 0000000..9b55382 --- /dev/null +++ b/ggml/src/ggml-sycl/fusion.cpp @@ -0,0 +1,351 @@ +#include "fusion.hpp" +#include "binbcast.hpp" + +#include + +// mul_mat(gate) + mul_mat(up) + GLU: graph shape and tensor properties only. Backend state +// (weight layout, split buffers, DMMV) is checked by ggml_sycl_op_mul_mat_glu_fused(). +static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const ggml_tensor * up, + const ggml_tensor * glu) { + // the fused epilogue implements these two; the rest fall back to the standalone GLU kernels + const ggml_glu_op glu_op = ggml_get_glu_op(glu); + if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { + return false; + } + + // the kernel always treats src[0] as the activated operand and src[1] as the multiplier + if (ggml_get_op_params_i32(glu, 1) /* swapped */) { + return false; + } + + const ggml_tensor * wu = up->src[0]; + const ggml_tensor * wg = gate->src[0]; + const ggml_tensor * act = up->src[1]; + + // one activation and one output indexing must serve both weights; the block types + // may differ, since the plain-layout fused kernel runs each operand's own vec_dot + // (different types then imply different byte strides, so only the shape must agree) + if (!ggml_are_same_shape(wu, wg)) { + return false; + } + if (act != gate->src[1]) { + return false; + } + + // fused K-quant GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type + // q4_K, the plain-layout kernel covers q5_K / iq4_xs pairs incl. mixed gate/up types + const bool reorder_pair = wu->type == GGML_TYPE_Q4_K && wg->type == GGML_TYPE_Q4_K; + const bool plain_pair = (wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_IQ4_XS) && + (wg->type == GGML_TYPE_Q5_K || wg->type == GGML_TYPE_IQ4_XS); +#ifdef GGML_SYCL_MMVQ_HAS_XMX + // same-type pairs of the types of the XMX kernels, which also take a few more columns + // (the XMX kernels walk whole QK_K super-blocks, so they need rows of a multiple of QK_K) + const bool xmx_pair = wu->type == wg->type && ggml_sycl_xmx_supports_type(wu->type) && wu->ne[0] % QK_K == 0; +#else + const bool xmx_pair = false; +#endif // GGML_SYCL_MMVQ_HAS_XMX + // the ESIMD kernel covers same-type q2_K..q6_K and q8_0, single column only (as unfused DMMV) +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + const bool esimd_pair = g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 && + (wu->type == GGML_TYPE_Q2_K || wu->type == GGML_TYPE_Q3_K || wu->type == GGML_TYPE_Q4_K || + wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_Q6_K || wu->type == GGML_TYPE_Q8_0); +#else + const bool esimd_pair = false; +#endif + // q8_0 rows are walked in 32-wide blocks, so they only need to be a multiple of QK8_0 + const int64_t row_align = esimd_pair && wu->type == GGML_TYPE_Q8_0 ? QK8_0 : QK_K; + if ((!reorder_pair && !plain_pair && !xmx_pair && !esimd_pair) || wu->ne[0] % row_align != 0) { + return false; + } + + // one 2D reorder-layout matrix in, a plain column stride out: no broadcast or padding + if (!ggml_is_contiguous(wu) || !ggml_is_contiguous(wg) || !ggml_is_contiguous(act) || + !ggml_is_contiguous(glu)) { + return false; + } + if (act->type != GGML_TYPE_F32 || glu->type != GGML_TYPE_F32) { + return false; + } + if (act->ne[2] != 1 || act->ne[3] != 1 || wu->ne[2] != 1 || wu->ne[3] != 1) { + return false; + } + // the kernel writes rows [0, wu->ne[1]) of each glu column, strided by glu->ne[0] + if (glu->ne[0] != wu->ne[1] || glu->ne[1] != act->ne[1]) { + return false; + } + // mat-vec only: one column per decoded token, up to the batch the reorder kernels cover + const int max_cols = xmx_pair ? std::max(MMVQ_MAX_BATCH_SIZE, g_ggml_sycl_xmx_glu_max_cols) : MMVQ_MAX_BATCH_SIZE; + if (act->ne[1] > max_cols) { + return false; + } + + return true; +} + +// the unary ops the fused unary chains in element_wise.cpp have a functor for +static bool ggml_sycl_fused_unary_has_kernel(ggml_unary_op unary_op) { + return unary_op == GGML_UNARY_OP_SILU || unary_op == GGML_UNARY_OP_SIGMOID || + unary_op == GGML_UNARY_OP_SOFTPLUS; +} + +bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializer_list ops, + std::initializer_list unary_ops) { +#ifndef NDEBUG + const size_t num_unary = std::count(ops.begin(), ops.end(), GGML_OP_UNARY); + GGML_ASSERT(unary_ops.size() == num_unary); +#endif + + if (!g_ggml_sycl_enable_fusion) { + return false; + } + + // gate and up are siblings, not a chain, so ggml_can_fuse cannot express this: use the + // subgraph form with the GLU as the only materialised output. + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_MUL_MAT && ops.begin()[1] == GGML_OP_MUL_MAT && + ops.begin()[2] == GGML_OP_GLU) { + if (!ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx + 2 })) { + return false; + } + + const ggml_tensor * glu = cgraph->nodes[node_idx + 2]; + const ggml_tensor * gate = glu->src[0]; + const ggml_tensor * up = glu->src[1]; + + // don't assume which of the two mat-muls is the gate; infer it from the GLU's operands + const bool ok = (gate == cgraph->nodes[node_idx] && up == cgraph->nodes[node_idx + 1]) || + (gate == cgraph->nodes[node_idx + 1] && up == cgraph->nodes[node_idx]); + if (!ok) { + return false; + } + + return ggml_sycl_should_fuse_mul_mat_glu(gate, up, glu); + } + + if (!ggml_can_fuse(cgraph, node_idx, ops)) { + return false; + } + + if ((ops.size() == 2 || ops.size() == 3) && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { + if (ops.size() == 3 && ops.begin()[2] != GGML_OP_ADD) { + return false; + } + + const ggml_tensor * rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + const ggml_tensor * add = ops.size() == 3 ? cgraph->nodes[node_idx + 2] : nullptr; + + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + + if (mul->src[0]->type != GGML_TYPE_F32 || + mul->src[1]->type != GGML_TYPE_F32 || + mul->type != GGML_TYPE_F32) { + return false; + } + + // if rms norm is the B operand, then we don't handle broadcast + if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) { + return false; + } + + const ggml_tensor * mul_w = (mul->src[0] == rms_norm) ? mul->src[1] : mul->src[0]; + // the fused kernel indexes the weight as mul[col], so it must span ncols contiguously + if (mul_w->ne[0] != rms_norm->ne[0] || mul_w->nb[0] != ggml_type_size(mul_w->type)) { + return false; + } + + if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) { + return false; + } + + if (add != nullptr) { + if (add->src[0]->type != GGML_TYPE_F32 || + add->src[1]->type != GGML_TYPE_F32 || + add->type != GGML_TYPE_F32) { + return false; + } + + // the fused kernel indexes the residual as add[col] and does not broadcast it + const ggml_tensor * add_w = (add->src[0] == mul) ? add->src[1] : add->src[0]; + if (!ggml_are_same_shape(add_w, add)) { + return false; + } + + if (!ggml_is_contiguous(add->src[0]) || !ggml_is_contiguous_rows(add->src[1])) { + return false; + } + } + + return true; + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_ADD) { + const ggml_tensor * add0 = cgraph->nodes[node_idx]; + const ggml_tensor * add1 = cgraph->nodes[node_idx + 1]; + // ggml_can_fuse already guarantees add1 consumes add0 and that add0 has a single use. + // Keep the CUDA association: the running sum is src0 of the next ADD so the fused + // float fold matches two sequential add() launches. + if (add1->src[0] != add0) { + return false; + } + + const ggml_tensor * c = add1->src[1]; + if (!ggml_sycl_add_kernel_supports(add0->src[0]->type, add0->src[1]->type, add0->type) || + !ggml_sycl_add_kernel_supports(add0->type, c->type, add1->type)) { + return false; + } + + return true; + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL && + unary_ops.size() == 1) { + const ggml_tensor * unary = cgraph->nodes[node_idx]; + const ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + + const ggml_unary_op unary_op = ggml_get_unary_op(unary); + if (unary_op != unary_ops.begin()[0]) { + return false; + } + + if (!ggml_sycl_fused_unary_has_kernel(unary_op)) { + return false; + } + + if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) { + return false; + } + + const ggml_tensor * other = (mul->src[0] == unary) ? mul->src[1] : mul->src[0]; + if (other->type != unary->type) { + return false; + } + + // one row stride per source comes from nb[1], so rows must be contiguous and equally + // shaped; the destination is written flat, so it must be fully contiguous + if (!ggml_is_contiguous_1(unary->src[0]) || !ggml_is_contiguous_1(other) || + !ggml_are_same_shape(other, unary) || !ggml_is_contiguous(mul)) { + return false; + } + + // the 32-bit fastdiv is inexact past 2^31; decline, the unfused path handles it + if (ggml_nelements(mul) >= ((int64_t) 1 << 31)) { + return false; + } + + return true; + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_SSM_CONV && ops.begin()[1] == GGML_OP_UNARY && + unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_SILU) { + const ggml_tensor * ssm_conv = cgraph->nodes[node_idx]; + const ggml_tensor * silu = cgraph->nodes[node_idx + 1]; + + if (ggml_get_unary_op(silu) != unary_ops.begin()[0]) { + return false; + } + if (ssm_conv->type != GGML_TYPE_F32 || silu->type != GGML_TYPE_F32) { + return false; + } + // the fused kernel writes the SiLU output with dense strides, so it must be contiguous + if (!ggml_is_contiguous(silu)) { + return false; + } + + return true; + } + + // ADD(bias) + UNARY + MUL(scale): the delta-net alpha gate, softplus(alpha + dt) * a. + // The broadcast is what stops the same-shape UNARY + MUL branch above firing past one token. + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_UNARY && + ops.begin()[2] == GGML_OP_MUL && unary_ops.size() == 1) { + const ggml_tensor * add = cgraph->nodes[node_idx]; + const ggml_tensor * unary = cgraph->nodes[node_idx + 1]; + const ggml_tensor * mul = cgraph->nodes[node_idx + 2]; + + const ggml_unary_op unary_op = ggml_get_unary_op(unary); + if (unary_op != unary_ops.begin()[0]) { + return false; + } + + if (!ggml_sycl_fused_unary_has_kernel(unary_op)) { + return false; + } + + // ggml_can_fuse() has already pinned the chain: unary consumes add, mul consumes + // unary, add and unary have one use each, and all three have the same shape + const ggml_tensor * a = add->src[0]; + const ggml_tensor * bias = add->src[1]; + const ggml_tensor * scale = (mul->src[0] == unary) ? mul->src[1] : mul->src[0]; + + if (a->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 || + scale->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { + return false; + } + + // the activation and the destination are indexed flat + if (!ggml_is_contiguous(a) || !ggml_is_contiguous(mul) || !ggml_are_same_shape(a, mul)) { + return false; + } + + // the kernel reads the bias and the scale as v[col], so each must be a single + // contiguous row spanning ne0 + if (bias->ne[0] != a->ne[0] || scale->ne[0] != a->ne[0] || + ggml_nrows(bias) != 1 || ggml_nrows(scale) != 1 || + !ggml_is_contiguous(bias) || !ggml_is_contiguous(scale)) { + return false; + } + + // the 32-bit fastdiv is inexact past 2^31; decline, the unfused path handles it + if (ggml_nelements(mul) >= ((int64_t) 1 << 31)) { + return false; + } + + return true; + } + + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_SSM_CONV && ops.begin()[1] == GGML_OP_ADD && + ops.begin()[2] == GGML_OP_UNARY && unary_ops.size() == 1 && unary_ops.begin()[0] == GGML_UNARY_OP_SILU) { + const ggml_tensor * ssm_conv = cgraph->nodes[node_idx]; + const ggml_tensor * add = cgraph->nodes[node_idx + 1]; + const ggml_tensor * silu = cgraph->nodes[node_idx + 2]; + + if (ggml_get_unary_op(silu) != unary_ops.begin()[0]) { + return false; + } + if (ssm_conv->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 || silu->type != GGML_TYPE_F32) { + return false; + } + // the fused kernel writes the SiLU output with dense strides, so it must be contiguous + if (!ggml_is_contiguous(silu)) { + return false; + } + + // ADD must consume ssm_conv's output and broadcast a 1-D channel-wise bias + const ggml_tensor * bias = (add->src[0] == ssm_conv) ? add->src[1] : add->src[0]; + if (bias->type != GGML_TYPE_F32 || !ggml_is_contiguous(bias)) { + return false; + } + if (ggml_nelements(bias) != ssm_conv->ne[0] || bias->ne[0] != ssm_conv->ne[0]) { + return false; + } + + return true; + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_SCALE) { + const ggml_tensor * rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor * scale = cgraph->nodes[node_idx + 1]; + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + if (scale->src[0]->type != GGML_TYPE_F32 || scale->type != GGML_TYPE_F32) { + return false; + } + // the fused kernel reads/writes rows flat like the unfused pair + if (!ggml_is_contiguous_rows(rms_norm) || !ggml_is_contiguous_rows(scale)) { + return false; + } + return true; + } + return false; +} diff --git a/ggml/src/ggml-sycl/fusion.hpp b/ggml/src/ggml-sycl/fusion.hpp new file mode 100644 index 0000000..94e7408 --- /dev/null +++ b/ggml/src/ggml-sycl/fusion.hpp @@ -0,0 +1,17 @@ +#ifndef GGML_SYCL_FUSION_HPP +#define GGML_SYCL_FUSION_HPP + +#include + +#include "common.hpp" + +// Backend-side fusability test. `ops` names a candidate op sequence starting at cgraph node +// `node_idx`, and `unary_ops` the GGML_UNARY_OP each GGML_OP_UNARY in `ops` must carry, in +// order; the result is true only if ggml considers that subgraph fusable *and* the SYCL +// kernel which would service it accepts the tensors involved (types, shapes, contiguity). +// +// Lives in its own translation unit because it grows a branch per supported op sequence. +bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializer_list ops, + std::initializer_list unary_ops); + +#endif // GGML_SYCL_FUSION_HPP diff --git a/ggml/src/ggml-sycl/fwht.cpp b/ggml/src/ggml-sycl/fwht.cpp new file mode 100644 index 0000000..5af549c --- /dev/null +++ b/ggml/src/ggml-sycl/fwht.cpp @@ -0,0 +1,416 @@ +#include "fwht.hpp" + +#include +#define P 1.0f +#define N -1.0f + +// constant Hadamard matrix via Paley I construction +static constexpr float H12[12][12] = { + { P, P, P, P, P, P, P, P, P, P, P, P }, + { P, N, P, N, P, P, P, N, N, N, P, N }, + { P, N, N, P, N, P, P, P, N, N, N, P }, + { P, P, N, N, P, N, P, P, P, N, N, N }, + { P, N, P, N, N, P, N, P, P, P, N, N }, + { P, N, N, P, N, N, P, N, P, P, P, N }, + { P, N, N, N, P, N, N, P, N, P, P, P }, + { P, P, N, N, N, P, N, N, P, N, P, P }, + { P, P, P, N, N, N, P, N, N, P, N, P }, + { P, P, P, P, N, N, N, P, N, N, P, N }, + { P, N, P, P, P, N, N, N, P, N, N, P }, + { P, P, N, P, P, P, N, N, N, P, N, N } +}; + +static constexpr float H20[20][20] = { + { P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P }, + { P, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N }, + { P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P }, + { P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P }, + { P, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N }, + { P, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N, N }, + { P, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N, N }, + { P, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P, N }, + { P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N, P }, + { P, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P, N }, + { P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N, P }, + { P, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P, N }, + { P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P, P }, + { P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P, P }, + { P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P, P }, + { P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N, P }, + { P, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N, N }, + { P, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P, N }, + { P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N, P }, + { P, P, N, N, P, P, P, P, N, P, N, P, N, N, N, N, P, P, N, N } +}; + +#undef P +#undef N + +template +static void fwht_kernel(const T * __restrict__ src, float * __restrict__ dst, const int64_t n_rows, + const float scale, const sycl::nd_item<2> & item) { + const sycl::sub_group sg = item.get_sub_group(); + + const int64_t r = item.get_global_id(0); + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + constexpr int el_w = N / WARP_SIZE; + static_assert(el_w >= 1 && N % WARP_SIZE == 0, "row must be a whole number of sub-group widths"); + + float reg[el_w]; + const int lane = sg.get_local_linear_id(); + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + reg[i] = static_cast(src[i * WARP_SIZE + lane]) * scale; + } + + // Butterflies inside the sub-group. The partner of a lane with bit h clear is the + // lower index of the pair, so it takes the sum and the upper takes lower - upper. +#pragma unroll + for (int h = 1; h < WARP_SIZE; h *= 2) { +#pragma unroll + for (int j = 0; j < el_w; ++j) { + const float val = reg[j]; + const float val2 = dpct::permute_sub_group_by_xor(sg, val, h, WARP_SIZE); + + reg[j] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + + // Butterflies across registers: h is a multiple of WARP_SIZE, so the partner of + // element i*WARP_SIZE + lane lives in reg[i + h/WARP_SIZE] on the same lane. +#pragma unroll + for (int h = WARP_SIZE; h < N; h *= 2) { + const int step = h / WARP_SIZE; +#pragma unroll + for (int j = 0; j < el_w; j += 2 * step) { +#pragma unroll + for (int k = 0; k < step; ++k) { + const float x = reg[j + k]; + const float y = reg[j + k + step]; + + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + dst[i * WARP_SIZE + lane] = reg[i]; + } +} + +template +static void launch_fwht(const T * src, float * dst, const int64_t n_rows, const float scale, + dpct::queue_ptr stream) { + constexpr int rows_per_block = 4; + + const int64_t num_blocks = (n_rows + rows_per_block - 1) / rows_per_block; + + // dim 1 is the fastest-varying, so a sub-group is exactly one row's WARP_SIZE lanes. + const sycl::range<2> global(num_blocks * rows_per_block, WARP_SIZE); + const sycl::range<2> local(rows_per_block, WARP_SIZE); + + stream->parallel_for(sycl::nd_range<2>(global, local), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + fwht_kernel(src, dst, n_rows, scale, item); + }); +} + +// Wide blocks: one row per work-group instead of per sub-group, so each work-item +// keeps N/NT values rather than N/WARP_SIZE. Butterflies below the sub-group width +// still shuffle; those up to NT go through work-group local memory; the rest stay +// in registers. +template +static void fwht_kernel_wide(const T * __restrict__ src, + float * __restrict__ dst, + const int64_t n_rows, + const float scale, + const sycl::nd_item<2> & item, + float * smem) { + const int64_t r = item.get_global_id(0); + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + constexpr int el_w = N / NT; + static_assert(el_w >= 1 && N % NT == 0, "row must be a whole number of work-group widths"); + + const int tid = item.get_local_id(1); + + float reg[el_w]; +#pragma unroll + for (int i = 0; i < el_w; ++i) { + reg[i] = static_cast(src[i * NT + tid]) * scale; + } + + const sycl::sub_group sg = item.get_sub_group(); + const int lane = sg.get_local_linear_id(); + + // Butterflies inside the sub-group, same pattern as the narrow kernel. +#pragma unroll + for (int h = 1; h < WARP_SIZE; h *= 2) { +#pragma unroll + for (int j = 0; j < el_w; ++j) { + const float val = reg[j]; + const float val2 = dpct::permute_sub_group_by_xor(sg, val, h, WARP_SIZE); + + reg[j] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + + // Butterflies from the sub-group width up to NT: the partner lane is outside + // this sub-group, so it goes through work-group local memory instead of a shuffle. + for (int h = WARP_SIZE; h < NT; h *= 2) { +#pragma unroll + for (int j = 0; j < el_w; ++j) { + smem[j * NT + tid] = reg[j]; + } + item.barrier(sycl::access::fence_space::local_space); +#pragma unroll + for (int j = 0; j < el_w; ++j) { + const float val = reg[j]; + const float val2 = smem[j * NT + (tid ^ h)]; + reg[j] = (tid & h) == 0 ? val + val2 : val2 - val; + } + item.barrier(sycl::access::fence_space::local_space); + } + + // Butterflies across registers: h is a multiple of NT, so the partner of element + // i*NT + tid lives in reg[i + h/NT] on the same work-item. + for (int h = NT; h < N; h *= 2) { + const int step = h / NT; + for (int j = 0; j < el_w; j += 2 * step) { + for (int k = 0; k < step; ++k) { + const float x = reg[j + k]; + const float y = reg[j + k + step]; + + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + dst[i * NT + tid] = reg[i]; + } +} + +template +static void launch_fwht_wide(const T * src, + float * dst, + const int64_t n_rows, + const float scale, + dpct::queue_ptr stream) { + const sycl::range<2> global(n_rows, NT); + const sycl::range<2> local(1, NT); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor smem(sycl::range<1>(N), cgh); + cgh.parallel_for(sycl::nd_range<2>(global, local), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + fwht_kernel_wide(src, dst, n_rows, scale, item, get_pointer(smem)); + }); + }); +} + +template +static void kronecker_kernel(const T * __restrict__ src, + float * __restrict__ dst, + const int64_t n_rows, + const float scale, + const sycl::nd_item<2> & item) { + static_assert(m == 12 || m == 20, "block size has to be 12 or 20."); + + const sycl::sub_group sg = item.get_sub_group(); + + const int64_t r = item.get_global_id(0); + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + constexpr int blocks_per_group = N / m; + constexpr int el_w = blocks_per_group / WARP_SIZE; + static_assert(el_w >= 1 && blocks_per_group % WARP_SIZE == 0, "blocks_per_group must be a multiple of WARP_SIZE"); + float reg[el_w * m]; + const int lane = sg.get_local_linear_id(); + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + const int b_idx = i * WARP_SIZE + lane; + +#pragma unroll + for (int j = 0; j < m; ++j) { + reg[i * m + j] = static_cast(src[b_idx * m + j]) * scale; + } + } + +#pragma unroll + for (int b = 0; b < el_w; ++b) { + float z[m] = { 0.0f }; + +#pragma unroll + for (int i = 0; i < m; ++i) { +#pragma unroll + for (int j = 0; j < m; ++j) { + const float h = (m == 12 ? H12[j][i] : H20[j][i]); + z[i] += reg[b * m + j] * h; + } + } + +#pragma unroll + for (int i = 0; i < m; ++i) { + reg[b * m + i] = z[i]; + } + } + +#pragma unroll + for (int h = 1; h < WARP_SIZE; h *= 2) { +#pragma unroll + for (int j = 0; j < el_w; ++j) { +#pragma unroll + for (int k = 0; k < m; ++k) { + const float val = reg[j * m + k]; + const float val2 = dpct::permute_sub_group_by_xor(sg, val, h, WARP_SIZE); + + reg[j * m + k] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + } + +#pragma unroll + for (int h = WARP_SIZE; h < blocks_per_group; h *= 2) { + const int step = h / WARP_SIZE; +#pragma unroll + for (int j = 0; j < el_w; j += 2 * step) { +#pragma unroll + for (int s = 0; s < step; ++s) { +#pragma unroll + for (int k = 0; k < m; ++k) { + const float x = reg[(j + s) * m + k]; + const float y = reg[(j + s + step) * m + k]; + + reg[(j + s) * m + k] = x + y; + reg[(j + s + step) * m + k] = x - y; + } + } + } + } + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + const int b_idx = i * WARP_SIZE + lane; +#pragma unroll + for (int k = 0; k < m; ++k) { + dst[b_idx * m + k] = reg[i * m + k]; + } + } +} + +template +static void launch_kronecker(const T * src, + float * dst, + const int64_t n_rows, + const float scale, + dpct::queue_ptr stream) { + constexpr int rows_per_block = 4; + + const int64_t num_blocks = (n_rows + rows_per_block - 1) / rows_per_block; + + // dim 1 is the fastest-varying, so a sub-group is exactly one row's WARP_SIZE lanes. + const sycl::range<2> global(num_blocks * rows_per_block, WARP_SIZE); + const sycl::range<2> local(rows_per_block, WARP_SIZE); + + stream->parallel_for(sycl::nd_range<2>(global, local), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + kronecker_kernel(src, dst, n_rows, scale, item); + }); +} + +template +static bool ggml_sycl_op_fwht_impl(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { + const int n = (int) src->ne[0]; + const int64_t rows = ggml_nrows(src); + + const T * src_d = (const T *) src->data; + float * dst_d = (float *) dst->data; + dpct::queue_ptr stream = ctx.stream(); + + const float scale = 1.0f / std::sqrt((float) n); + + switch (n) { + case 64: + launch_fwht<64>(src_d, dst_d, rows, scale, stream); + return true; + case 128: + launch_fwht<128>(src_d, dst_d, rows, scale, stream); + return true; + case 256: + launch_fwht<256>(src_d, dst_d, rows, scale, stream); + return true; + case 512: + launch_fwht<512>(src_d, dst_d, rows, scale, stream); + return true; + case 384: + launch_kronecker<384, 12>(src_d, dst_d, rows, scale, stream); + return true; + case 768: + launch_kronecker<768, 12>(src_d, dst_d, rows, scale, stream); + return true; + case 640: + launch_kronecker<640, 20>(src_d, dst_d, rows, scale, stream); + return true; + case 1280: + launch_kronecker<1280, 20>(src_d, dst_d, rows, scale, stream); + return true; + case 1024: + launch_fwht_wide<1024, 256>(src_d, dst_d, rows, scale, stream); + return true; + case 2048: + launch_fwht_wide<2048, 256>(src_d, dst_d, rows, scale, stream); + return true; + case 4096: + launch_fwht_wide<4096, 256>(src_d, dst_d, rows, scale, stream); + return true; + case 8192: + launch_fwht_wide<8192, 256>(src_d, dst_d, rows, scale, stream); + return true; + default: + return false; + } +} + +bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { + if (dst->type != GGML_TYPE_F32) { + return false; + } + if (!ggml_are_same_shape(src, dst)) { + return false; + } + if (!ggml_is_contiguous(src) || !ggml_is_contiguous(dst)) { + return false; + } + + switch (src->type) { + case GGML_TYPE_F32: + return ggml_sycl_op_fwht_impl(ctx, src, dst); + case GGML_TYPE_F16: + return ggml_sycl_op_fwht_impl(ctx, src, dst); + default: + return false; + } +} diff --git a/ggml/src/ggml-sycl/fwht.hpp b/ggml/src/ggml-sycl/fwht.hpp new file mode 100644 index 0000000..cd238cf --- /dev/null +++ b/ggml/src/ggml-sycl/fwht.hpp @@ -0,0 +1,12 @@ +#ifndef GGML_SYCL_FWHT_HPP +#define GGML_SYCL_FWHT_HPP + +#include "common.hpp" + +// Fast Walsh-Hadamard transform, the fast path for a MUL_MAT whose src0 ggml has +// tagged GGML_HINT_SRC0_IS_HADAMARD. src0 is not read at all. Returns false if the +// shape is not one this can serve, in which case the caller must fall through to the +// ordinary mat-mul dispatch. +bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst); + +#endif // GGML_SYCL_FWHT_HPP diff --git a/ggml/src/ggml-sycl/gated_delta_net.cpp b/ggml/src/ggml-sycl/gated_delta_net.cpp index ebc5875..8468bbf 100644 --- a/ggml/src/ggml-sycl/gated_delta_net.cpp +++ b/ggml/src/ggml-sycl/gated_delta_net.cpp @@ -6,7 +6,7 @@ #include -template +template void gated_delta_net_sycl(const float * q, const float * k, const float * v, @@ -14,9 +14,9 @@ void gated_delta_net_sycl(const float * q, const float * beta, const float * curr_state, float * dst, + float * state, int64_t H, int64_t n_tokens, - int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, @@ -28,7 +28,9 @@ void gated_delta_net_sycl(const float * q, int64_t sb3, const sycl::uint3 neqk1_magic, const sycl::uint3 rq3_magic, - float scale) { + float scale, + int64_t state_slot_stride, + int K) { auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>(); const uint32_t h_idx = item_ct1.get_group(2); const uint32_t sequence = item_ct1.get_group(1); @@ -39,13 +41,14 @@ void gated_delta_net_sycl(const float * q, const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic); const uint32_t iq3 = fastdiv(sequence, rq3_magic); - const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs; float * attn_data = dst; - float * state = dst + attn_score_elems; - const int64_t state_offset = (sequence * H + h_idx) * S_v * S_v; - state += state_offset; - curr_state += state_offset; + // input state holds s0 only [S_v, S_v, H, n_seqs] — seq stride is D = H * S_v * S_v. + // output state layout (per-slot D * n_seqs) — same per-(seq,head) offset as before. + const int64_t state_in_offset = sequence * H * S_v * S_v + h_idx * S_v * S_v; + const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v; + state += state_out_offset; + curr_state += state_in_offset + col * S_v; attn_data += (sequence * n_tokens * H + h_idx) * S_v; constexpr int warp_size = ggml_sycl_get_physical_warp_size() < S_v ? ggml_sycl_get_physical_warp_size() : S_v; @@ -55,9 +58,12 @@ void gated_delta_net_sycl(const float * q, #pragma unroll for (int r = 0; r < rows_per_lane; r++) { const int i = r * warp_size + lane; - s_shard[r] = curr_state[col * S_v + i]; + s_shard[r] = curr_state[i]; } + // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. + // When n_tokens < K only slots 0..n_tokens-1 are written; older slots are caller-owned. + for (int t = 0; t < n_tokens; t++) { const float * q_t = q + iq3 * sq3 + t * sq2 + iq1 * sq1; const float * k_t = k + iq3 * sq3 + t * sq2 + iq1 * sq1; @@ -131,17 +137,32 @@ void gated_delta_net_sycl(const float * q, } attn_data += S_v * H; - } + // Write state back to global memory + if constexpr (keep_rs_t) { + const int target_slot = (int) n_tokens - 1 - t; + if (target_slot >= 0 && target_slot < K) { + float * curr_state = state + target_slot * state_slot_stride; #pragma unroll - for (int r = 0; r < rows_per_lane; r++) { - const int i = r * warp_size + lane; - state[col * S_v + i] = s_shard[r]; + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * warp_size + lane; + curr_state[col * S_v + i] = s_shard[r]; + } + } + } + } + + if constexpr (!keep_rs_t) { +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * warp_size + lane; + state[col * S_v + i] = s_shard[r]; + } } } -template +template static void launch_gated_delta_net(const float * q_d, const float * k_d, const float * v_d, @@ -149,6 +170,7 @@ static void launch_gated_delta_net(const float * q_d, const float * b_d, const float * s_d, float * dst_d, + float * state_d, int64_t S_v, int64_t H, int64_t n_tokens, @@ -165,6 +187,8 @@ static void launch_gated_delta_net(const float * q_d, int64_t neqk1, int64_t rq3, float scale, + int64_t state_slot_stride, + int K, dpct::queue_ptr stream) { //TODO: Add chunked kernel for even faster pre-fill const int warp_size = ggml_sycl_info().devices[ggml_sycl_get_device()].warp_size; @@ -182,9 +206,9 @@ static void launch_gated_delta_net(const float * q_d, constexpr int sv = 16; stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims), [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, n_tokens, - n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, - sb3, neqk1_magic, rq3_magic, scale); + gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, + sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, + sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); }); } break; @@ -193,9 +217,9 @@ static void launch_gated_delta_net(const float * q_d, constexpr int sv = 32; stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims), [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, n_tokens, - n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, - sb3, neqk1_magic, rq3_magic, scale); + gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, + sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, + sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); }); } break; @@ -204,9 +228,9 @@ static void launch_gated_delta_net(const float * q_d, constexpr int sv = 64; stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims), [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - gated_delta_net_sycl( - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, n_tokens, n_seqs, sq1, sq2, - sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale); + gated_delta_net_sycl( + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, + sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); }); } break; @@ -216,9 +240,9 @@ static void launch_gated_delta_net(const float * q_d, constexpr int sv = 128; stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims), [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - gated_delta_net_sycl( - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, n_tokens, n_seqs, sq1, sq2, - sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale); + gated_delta_net_sycl( + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, + sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); }); } break; @@ -229,7 +253,8 @@ static void launch_gated_delta_net(const float * q_d, } } -void ggml_sycl_op_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { +static void ggml_sycl_op_gated_delta_net_impl(ggml_backend_sycl_context & ctx, ggml_tensor * dst, + const ggml_sycl_gated_delta_net_fused_cache * cache) { ggml_tensor * src_q = dst->src[0]; ggml_tensor * src_k = dst->src[1]; ggml_tensor * src_v = dst->src[2]; @@ -290,18 +315,52 @@ void ggml_sycl_op_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dpct::queue_ptr stream = ctx.stream(); + // K (snapshot slot count) is an op param; state holds s0 only [S_v, S_v, H, n_seqs]. + const int K = ggml_get_op_params_i32(dst, 0); + const bool keep_rs = K > 1; + + // recurrent state -> dst tail (after attention scores), or the cache when fusing + float * state_d = dst_d + S_v * H * n_tokens * n_seqs; + int64_t state_slot_stride = S_v * S_v * H * n_seqs; + if (cache != nullptr) { + state_d = cache->data; + state_slot_stride = cache->slot_stride; + } + if (kda) { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, - S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, stream); + if (keep_rs) { + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, + S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + } else { + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, + S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + } } else { - launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, - S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, stream); + if (keep_rs) { + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, + S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + } else { + launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, + S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + } } } +void ggml_sycl_op_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + ggml_sycl_op_gated_delta_net_impl(ctx, dst, nullptr); +} + void ggml_sycl_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/6); ggml_sycl_op_gated_delta_net(ctx, dst); } + +void ggml_sycl_op_gated_delta_net_fused_cache(ggml_backend_sycl_context & ctx, ggml_tensor * dst, + ggml_sycl_gated_delta_net_fused_cache cache) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/6); + ggml_sycl_op_gated_delta_net_impl(ctx, dst, &cache); +} diff --git a/ggml/src/ggml-sycl/gated_delta_net.hpp b/ggml/src/ggml-sycl/gated_delta_net.hpp index a3308ee..7903b8e 100644 --- a/ggml/src/ggml-sycl/gated_delta_net.hpp +++ b/ggml/src/ggml-sycl/gated_delta_net.hpp @@ -5,4 +5,15 @@ #include "common.hpp" #include "ggml.h" +// fused-kernel recurrent-state output; strides in elements (per-seq stride is always D, set in-kernel) +struct ggml_sycl_gated_delta_net_fused_cache { + float * data; // rollback slot 0 + int64_t slot_stride; // between rollback slots (0 when K==1) +}; + +void ggml_sycl_op_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst); void ggml_sycl_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +// same op, but writes the snapshot(s) into the cache instead of dst (see ggml_sycl_try_gdn_cache_fusion) +void ggml_sycl_op_gated_delta_net_fused_cache(ggml_backend_sycl_context & ctx, ggml_tensor * dst, + ggml_sycl_gated_delta_net_fused_cache cache); diff --git a/ggml/src/ggml-sycl/gemm.hpp b/ggml/src/ggml-sycl/gemm.hpp index dcf6c7a..81bc5c2 100644 --- a/ggml/src/ggml-sycl/gemm.hpp +++ b/ggml/src/ggml-sycl/gemm.hpp @@ -29,6 +29,9 @@ class DnnlGemmWrapper { static constexpr dt to_dt() { if constexpr (std::is_same_v) return dt::f32; else if constexpr (std::is_same_v) return dt::f16; +#ifdef GGML_SYCL_HAS_BF16 + else if constexpr (std::is_same_v) return dt::bf16; +#endif else static_assert(0); } @@ -63,8 +66,10 @@ class DnnlGemmWrapper { auto matmul_pd = dnnl::matmul::primitive_desc(eng, a_in_md, b_in_md, c_md, primitive_attr); auto c_mem = dnnl::memory(matmul_pd.dst_desc(), eng, c); - auto scratchpad_md = matmul_pd.scratchpad_desc(); - auto scratchpad_mem = ctx.get_scratchpad_mem(scratchpad_md, eng, q); + const auto scratchpad_md = matmul_pd.scratchpad_desc(); + ggml_sycl_pool_alloc scratchpad(ctx.pool()); + void * scratchpad_ptr = scratchpad_md.get_size() > 0 ? scratchpad.alloc(scratchpad_md.get_size()) : nullptr; + auto scratchpad_mem = dnnl::memory(scratchpad_md, eng, scratchpad_ptr); auto matmul_prim = dnnl::matmul(matmul_pd); diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp index 03f8dd9..36f6fcd 100644 --- a/ggml/src/ggml-sycl/getrows.cpp +++ b/ggml/src/ggml-sycl/getrows.cpp @@ -60,6 +60,50 @@ static void k_get_rows( dst_row[iybs + iqs + y_offset] = v.y(); } +template +static void k_get_rows_f32( + const void * src0, const int32_t * src1, dst_t * dst, + int64_t ne00, + int64_t ne12, + size_t s1, size_t s2, size_t s3, + size_t nb01, size_t nb02, size_t nb03, + size_t s10, size_t s11, size_t s12, + const sycl::nd_item<3> &item_ct1) { + + const int i00 = (item_ct1.get_group(2) * item_ct1.get_local_range(2) + + item_ct1.get_local_id(2)) * + 2; + const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) + + item_ct1.get_local_id(1); + const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) / + ne12; + const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) % + ne12; + + if (i00 >= ne00) { + return; + } + + const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; + + dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; + const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03; + + const int ib = i00/qk; + const int iqs = (i00%qk)/qr; + const int iybs = i00 - i00%qk; + const int y_offset = qr == 1 ? 1 : qk/2; + + float v0; + float v1; + dequantize_kernel(src0_row, ib, iqs, v0, v1); + + dst_row[iybs + iqs + 0] = (dst_t) v0; + dst_row[iybs + iqs + y_offset] = (dst_t) v1; +} + template static void k_get_rows_float( const src0_t * src0, const int32_t * src1, dst_t * dst, @@ -93,6 +137,47 @@ static void k_get_rows_float( dst_row[i00] = src0_row[i00]; } +// src0 is in reorder layout: all qs of the tensor, then all d +template +static void k_get_rows_reorder( + const void * src0_qs, const void * src0_d, const int32_t * src1, dst_t * dst, + int64_t ne00, int64_t ne12, + size_t s1, size_t s2, size_t s3, + size_t s10, size_t s11, size_t s12, + const sycl::nd_item<3> &item_ct1) { + + const int i00 = (item_ct1.get_group(2) * item_ct1.get_local_range(2) + + item_ct1.get_local_id(2)) * + 2; + const int i10 = item_ct1.get_local_range(1) * item_ct1.get_group(1) + + item_ct1.get_local_id(1); + const int i11 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) / + ne12; + const int i12 = (item_ct1.get_group(0) * item_ct1.get_local_range(0) + + item_ct1.get_local_id(0)) % + ne12; + + if (i00 >= ne00) { + return; + } + + const int i01 = src1[i10*s10 + i11*s11 + i12*s12]; + + dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3; + + const int64_t ib = (int64_t) i01*(ne00/qk) + i00/qk; // block index in the whole tensor + const int iqs = (i00%qk)/qr; + const int iybs = i00 - i00%qk; + const int y_offset = qr == 1 ? 1 : qk/2; + + dfloat2 v; + dequantize_kernel(src0_d, ib, src0_qs, ib*(qk/qr) + iqs, v); + + dst_row[iybs + iqs + 0] = v.x(); + dst_row[iybs + iqs + y_offset] = v.y(); +} + template static void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const void *src0_dd, @@ -129,11 +214,177 @@ static void get_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor *sr GGML_UNUSED(ctx); } -template +template +static void get_rows_sycl_reorder(const ggml_tensor *src0, const ggml_tensor *src1, + ggml_tensor *dst, const void *src0_dd, + const int32_t *src1_dd, float *dst_dd, + queue_ptr stream) { + + GGML_TENSOR_BINARY_OP_LOCALS + + // reorder is done only on 2D weights + GGML_ASSERT(ne02 == 1 && ne03 == 1); + GGML_ASSERT(ne00 % 2 == 0); + + const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); + const int block_num_x = (ne00 + 2*SYCL_GET_ROWS_BLOCK_SIZE - 1) / (2*SYCL_GET_ROWS_BLOCK_SIZE); + const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x); + + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + + const void * src0_d = (const char *) src0_dd + ne00*ne01/qr; + + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_get_rows_reorder( + src0_dd, src0_d, src1_dd, dst_dd, ne00, ne12, s1, s2, + s3, s10, s11, s12, item_ct1); + }); + + GGML_UNUSED(src0); +} + +// K-quants in reorder layout: one work-group dequantizes one QK_K block of one row +template +static void get_rows_sycl_reorder_k(const ggml_tensor *src0, const ggml_tensor *src1, + ggml_tensor *dst, const int32_t *src1_dd, float *dst_dd, + queue_ptr stream, F dequantize_block) { + + GGML_TENSOR_BINARY_OP_LOCALS + + // reorder is done only on 2D weights + GGML_ASSERT(ne02 == 1 && ne03 == 1); + GGML_ASSERT(ne00 % QK_K == 0); + + const int64_t nblk = ne00 / QK_K; + const int64_t nb = nblk * ne01; // blocks in the whole tensor + const int64_t ngroups = ne10 * ne11 * ne12 * nblk; + + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor scales_local(sycl::range<1>(nlocal), cgh); + + auto kernel = [=](sycl::nd_item item_ct1) { + const int64_t g = item_ct1.get_group(dims - 1); + const int64_t b = g % nblk; + const int64_t r = g / nblk; + const int64_t i10 = r % ne10; + const int64_t i11 = (r / ne10) % ne11; + const int64_t i12 = r / (ne10 * ne11); + + const int64_t i01 = src1_dd[i10*s10 + i11*s11 + i12*s12]; + + float * y = dst_dd + i10*s1 + i11*s2 + i12*s3 + b*QK_K; + dequantize_block(y, i01*nblk + b, get_pointer(scales_local), item_ct1, nb); + }; + + if constexpr (dims == 1) { + cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>(ngroups * nthreads), sycl::range<1>(nthreads)), kernel); + } else { + cgh.parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, ngroups * nthreads), sycl::range<3>(1, 1, nthreads)), kernel); + } + }); + + GGML_UNUSED(src0); +} + +// IQ3_S / IQ3_XXS in the persistent reorder layout: one work-group of 32 threads +// dequantizes one QK_K block of one requested row, reusing the reorder-aware +// block dequantizers that the dequantize-based mul_mat path already uses. +template +static void get_rows_sycl_reorder_iq3(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst, + const int32_t * src1_dd, float * dst_dd, + F dequantize_block) { + GGML_TENSOR_BINARY_OP_LOCALS + + // the persistent reorder is only installed for 2D weights + GGML_ASSERT(ne02 == 1 && ne03 == 1); + GGML_ASSERT(ne00 % QK_K == 0); + + const int64_t nblk = ne00 / QK_K; + const int64_t nb = nblk * ne01; // blocks in the whole tensor + const int64_t ngroups = ne10 * ne11 * ne12 * nblk; + + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + + queue_ptr stream = ctx.stream(); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, ngroups * 32), sycl::range<3>(1, 1, 32)), + [=](sycl::nd_item<3> item_ct1) { + const int64_t g = item_ct1.get_group(2); + const int64_t b = g % nblk; + const int64_t r = g / nblk; + const int64_t i10 = r % ne10; + const int64_t i11 = (r / ne10) % ne11; + const int64_t i12 = r / (ne10 * ne11); + + const int64_t i01 = src1_dd[i10*s10 + i11*s11 + i12*s12]; + + float * y = dst_dd + i10*s1 + i11*s2 + i12*s3 + b*QK_K; + dequantize_block(y, i01*nblk + b, item_ct1, nb); + }); + + GGML_UNUSED(dst); +} + +template +static void get_rows_sycl_f32(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, + ggml_tensor *dst, const void *src0_dd, + const int32_t *src1_dd, float *dst_dd, + queue_ptr stream) { + + GGML_TENSOR_BINARY_OP_LOCALS + + const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); + const int block_num_x = (ne00 + 2*SYCL_GET_ROWS_BLOCK_SIZE - 1) / (2*SYCL_GET_ROWS_BLOCK_SIZE); + const sycl::range<3> block_nums(ne11 * ne12, ne10, block_num_x); + + const size_t s1 = nb1 / ggml_element_size(dst); + const size_t s2 = nb2 / ggml_element_size(dst); + const size_t s3 = nb3 / ggml_element_size(dst); + + const size_t s10 = nb10 / ggml_element_size(src1); + const size_t s11 = nb11 / ggml_element_size(src1); + const size_t s12 = nb12 / ggml_element_size(src1); + + GGML_ASSERT(ne00 % 2 == 0); + + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_get_rows_f32( + src0_dd, src1_dd, dst_dd, ne00, ne12, s1, s2, + s3, nb01, nb02, nb03, s10, s11, s12, item_ct1); + }); + + GGML_UNUSED(dst); + GGML_UNUSED(ctx); +} + +template static void get_rows_sycl_float(ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const src0_t *src0_dd, const int32_t *src1_dd, - float *dst_dd, queue_ptr stream) { + dst_t *dst_dd, queue_ptr stream) { GGML_TENSOR_BINARY_OP_LOCALS @@ -168,26 +419,214 @@ static void get_rows_sycl_float(ggml_backend_sycl_context & ctx, const ggml_tens GGML_UNUSED(ctx); } +template +static void k_get_rows_back_float(const src0_t * src0, const int32_t * src1, float * dst, + const int64_t ncols, const int64_t nrows_grad_10, const int64_t nrows_grad_11, const int64_t nrows_dst, + const size_t s01, const size_t s02, + const size_t s10, const size_t s11, + const size_t s1, + const int64_t block_num_y, + const sycl::nd_item<3> & item_ct1) { + const int64_t col = item_ct1.get_group(2) * item_ct1.get_local_range(2) + item_ct1.get_local_id(2); + if (col >= ncols) { + return; + } + + // block_num_y is clamped, so stride over destination rows like CUDA k_get_rows_back_float + for (int64_t dst_row = item_ct1.get_group(1); dst_row < nrows_dst; dst_row += block_num_y) { + float sum = 0.0f; + + const int64_t nrows_grad_total = nrows_grad_10 * nrows_grad_11; + for (int64_t i = 0; i < nrows_grad_total; ++i) { + const int64_t i10 = i % nrows_grad_10; + const int64_t i11 = i / nrows_grad_10; + if (src1[i10*s10 + i11*s11] != dst_row) { + continue; + } + sum += (float) src0[col + i10*s01 + i11*s02]; + } + + dst[col + dst_row*s1] = sum; + } +} + +template +static void get_rows_back_sycl_float(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, + const ggml_tensor * src1, ggml_tensor * dst, + const src0_t * src0_dd, const int32_t * src1_dd, + float * dst_dd, queue_ptr stream) { + + GGML_TENSOR_BINARY_OP_LOCALS + + GGML_ASSERT(ne02*ne03 == 1); + GGML_ASSERT(ne12*ne13 == 1); + GGML_ASSERT(ne2*ne3 == 1); + GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type)); + GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type)); + GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type)); + + const int64_t ncols = ne00; + const int64_t nrows_grad_10 = ne10; + const int64_t nrows_grad_11 = ne11; + const int64_t nrows_dst = ne1; + + const size_t s01 = nb01 / sizeof(src0_t); + const size_t s02 = nb02 / sizeof(src0_t); + + const size_t s10 = nb10 / sizeof(int32_t); + const size_t s11 = nb11 / sizeof(int32_t); + + const size_t s1 = nb1 / sizeof(float); + + const sycl::range<3> block_dims(1, 1, SYCL_GET_ROWS_BLOCK_SIZE); + const int64_t block_num_x = (ncols + SYCL_GET_ROWS_BLOCK_SIZE - 1) / SYCL_GET_ROWS_BLOCK_SIZE; + const int64_t block_num_y = std::min(nrows_dst, (int64_t) UINT16_MAX); + const sycl::range<3> block_nums(1, block_num_y, block_num_x); + + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_get_rows_back_float(src0_dd, src1_dd, dst_dd, + ncols, nrows_grad_10, nrows_grad_11, nrows_dst, + s01, s02, s10, s11, s1, + block_num_y, item_ct1); + }); + + GGML_UNUSED(src0); + GGML_UNUSED(src1); + GGML_UNUSED(dst); + GGML_UNUSED(ctx); +} + void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { GGML_ASSERT(dst->src[1]->type == GGML_TYPE_I32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_I32 ); GGML_ASSERT(dst->src[0]->nb[0] == ggml_type_size(dst->src[0]->type)); GGML_ASSERT(dst->src[1]->nb[0] == ggml_type_size(dst->src[1]->type)); GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type)); const int32_t * src1_i32 = (const int32_t *) dst->src[1]->data; + // a weight shared with MUL_MAT (e.g. tied embeddings) can be in the + // persistent reorder layout (incl. IQ3_S / IQ3_XXS since the SYCL IQ3 + // reorder work) + const bool src0_reordered = dst->src[0]->extra && + ((const ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder; /* TODO: Refactor and remove duplicates */ switch (dst->src[0]->type) { case GGML_TYPE_F16: get_rows_sycl_float(ctx, dst->src[0], dst->src[1], dst, (const sycl::half *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; + case GGML_TYPE_BF16: + get_rows_sycl_float(ctx, dst->src[0], dst->src[1], dst, (const sycl::ext::oneapi::bfloat16 *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; case GGML_TYPE_F32: get_rows_sycl_float(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; + case GGML_TYPE_I32: + get_rows_sycl_float(ctx, dst->src[0], dst->src[1], dst, (const int32_t *)dst->src[0]->data, + src1_i32, (int32_t *)dst->data, ctx.stream()); + break; + case GGML_TYPE_Q1_0: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_MXFP4: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_NVFP4: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_IQ2_XXS: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_IQ2_XS: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_IQ2_S: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_IQ3_XXS: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_iq3(ctx, dst->src[0], dst->src[1], dst, src1_i32, (float *) dst->data, + [=](float * y, int64_t iblk, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_iq3_xxs_reorder(vx, y, iblk, item_ct1, nb, + iq3xxs_grid, ksigns_iq2xs, kmask_iq2xs); + }); + break; + } + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_IQ1_S: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_IQ1_M: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_IQ3_S: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_iq3(ctx, dst->src[0], dst->src[1], dst, src1_i32, (float *) dst->data, + [=](float * y, int64_t iblk, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_iq3_s_reorder(vx, y, iblk, item_ct1, nb, + kmask_iq2xs, iq3s_grid); + }); + break; + } + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_IQ4_NL: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_IQ4_XS: + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_Q2_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<3, 64, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_q2_K_reorder(vx, y, ib, item_ct1, nb); + GGML_UNUSED(scales_local); + }); + break; + } + get_rows_sycl_f32(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_Q3_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<3, 64, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_q3_K_reorder(vx, y, ib, item_ct1, nb); + GGML_UNUSED(scales_local); + }); + break; + } + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; case GGML_TYPE_Q4_0: + if (src0_reordered) { + get_rows_sycl_reorder(dst->src[0], dst->src[1], dst, dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + } get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; @@ -195,6 +634,18 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; + case GGML_TYPE_Q4_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<1, 32, K_SCALE_SIZE>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<1> & item_ct1, int64_t nb) { + dequantize_block_q4_K_reorder(vx, y, ib, scales_local, item_ct1, nb); + }); + break; + } + get_rows_sycl_f32(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; case GGML_TYPE_Q5_0: get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); @@ -203,7 +654,37 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; + case GGML_TYPE_Q5_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<3, 64, K_SCALE_SIZE>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_q5_K_reorder(vx, y, ib, scales_local, item_ct1, nb); + }); + break; + } + get_rows_sycl_f32(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + case GGML_TYPE_Q6_K: + if (src0_reordered) { + const void * vx = dst->src[0]->data; + get_rows_sycl_reorder_k<3, 64, 1>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(), + [=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) { + dequantize_block_q6_K_reorder(vx, y, ib, item_ct1, nb); + GGML_UNUSED(scales_local); + }); + break; + } + get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; case GGML_TYPE_Q8_0: + if (src0_reordered) { + get_rows_sycl_reorder(dst->src[0], dst->src[1], dst, dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; + } get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; @@ -213,3 +694,29 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { GGML_ABORT("fatal error"); } } + +void ggml_sycl_op_get_rows_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); + GGML_ASSERT(src1->type == GGML_TYPE_I32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + GGML_ASSERT(ggml_is_contiguous(dst)); + + switch (src0->type) { + case GGML_TYPE_F16: + get_rows_back_sycl_float(ctx, src0, src1, dst, (const sycl::half *) src0->data, + (const int32_t *) src1->data, (float *) dst->data, + ctx.stream()); + break; + case GGML_TYPE_F32: + get_rows_back_sycl_float(ctx, src0, src1, dst, (const float *) src0->data, + (const int32_t *) src1->data, (float *) dst->data, + ctx.stream()); + break; + default: + GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0->type)); + } +} diff --git a/ggml/src/ggml-sycl/getrows.hpp b/ggml/src/ggml-sycl/getrows.hpp index 1c560cd..0388e6c 100644 --- a/ggml/src/ggml-sycl/getrows.hpp +++ b/ggml/src/ggml-sycl/getrows.hpp @@ -16,5 +16,6 @@ #include "common.hpp" void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor *dst); +void ggml_sycl_op_get_rows_back(ggml_backend_sycl_context & ctx, ggml_tensor *dst); #endif // GGML_SYCL_GETROWS_HPP diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index c02a41a..ae0f291 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -11,18 +11,22 @@ // #include +#include #include #include +#include #include #include #include #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -30,9 +34,26 @@ #include #include +#include +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +#include +#include +#endif #if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC # include #endif +#if SYCL_EXT_ONEAPI_VIRTUAL_MEM +# include +# include +# define GGML_SYCL_SUPPORT_VMM +#endif +// beginning in oneDNN 3.12.3, "Enabled matmul and inner product primitives compatibility with SYCL Graph native recording mode on Intel GPUs." +#if GGML_SYCL_DNNL +# include "dnnl_version.h" +# if (DNNL_VERSION_MAJOR > 3 || (DNNL_VERSION_MAJOR == 3 && (DNNL_VERSION_MINOR > 12 || (DNNL_VERSION_MINOR == 12 && DNNL_VERSION_PATCH >= 3)))) +# define GGML_SYCL_GRAPH_DNNL_SUPPORTED +# endif +#endif #include #include "ggml.h" @@ -44,32 +65,142 @@ #include "ggml-sycl/backend.hpp" #include "ggml-sycl/common.hpp" #include "ggml-sycl/element_wise.hpp" +#include "ggml-sycl/fwht.hpp" +#include "ggml-sycl/fused-gemm.hpp" #include "ggml-sycl/gemm.hpp" #include "ggml-sycl/getrows.hpp" +#include "ggml-sycl/mem.hpp" #include "ggml-sycl/norm.hpp" #include "ggml-sycl/presets.hpp" #include "ggml-sycl/quantize.hpp" #include "ggml-sycl/repeat_back.hpp" #include "ggml-sycl/set_rows.hpp" #include "ggml-sycl/set.hpp" +#include "ggml-sycl/dsv4-hc.hpp" +#include "ggml-sycl/lightning-indexer.hpp" +#include "ggml-sycl/conv2d.hpp" +#include "ggml-sycl/conv2d-dw.hpp" +#include "ggml-sycl/conv2d-transpose.hpp" #include "ggml-sycl/ssm_conv.hpp" #include "ggml-sycl/sycl_hw.hpp" - +#include "ggml-sycl/ssm_scan.hpp" +#include "ggml-sycl/fill.hpp" +#include "ggml-sycl/cumsum.hpp" +#include "ggml-sycl/diag.hpp" +#include "ggml-sycl/opt-step.hpp" +#include "ggml-sycl/solve_tri.hpp" +#include "ggml-sycl/gated_delta_net.hpp" +#include "ggml-sycl/pool.hpp" +#include "ggml-sycl/cross_entropy_loss.hpp" + +#define MEM_SIZE_2M 0x00200000 +#define MEM_SIZE_1G 0x40000000 static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; -int g_ggml_sycl_disable_optimize = 0; -int g_ggml_sycl_disable_graph = 0; -int g_ggml_sycl_disable_dnn = 0; +int g_ggml_sycl_dev_debug = 0; +int g_ggml_sycl_enable_optimize = 1; +int g_ggml_sycl_enable_graph = 0; +int g_ggml_sycl_enable_dnn = 1; +int g_ggml_sycl_fa_onednn = 1; +int g_ggml_sycl_fa_onednn_max_kv = 0; +int g_ggml_sycl_enable_mkl_fa = 1; +int g_ggml_sycl_memtrace = 0; +int g_ggml_sycl_memtrace_step = 64; +int g_ggml_sycl_enable_vmm = 1; +int g_ggml_sycl_enable_fusion = 1; +int g_ggml_sycl_enable_esimd = 1; +int g_ggml_sycl_mmvq_wide = 1; +int g_ggml_sycl_enable_xmx_esimd = 1; +int g_ggml_sycl_xmx_max_cols = GGML_SYCL_XMX_MAX_COLS_DEFAULT; +int g_ggml_sycl_xmx_glu_max_cols = GGML_SYCL_XMX_GLU_MAX_COLS; int g_ggml_sycl_prioritize_dmmv = 0; +int g_ggml_sycl_xmx_gather_types = GGML_SYCL_XMX_GATHER_TYPES_DEFAULT; +int g_ggml_sycl_xmx_gather_shapes = GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT; +int g_ggml_sycl_dynamic_precision = GGML_SYCL_DYNAMIC_PRECISION_DEFAULT; +int g_ggml_sycl_dynamic_required_precision = GGML_SYCL_DYNAMIC_PRECISION_F32; +static const char * ggml_sycl_dynamic_precision_names[] = { "F16", "BF16", "TF32", "F32" }; + +// value of a GGML_SYCL_DYNAMIC_PRECISION-style variable; def if unset or invalid +static int ggml_sycl_get_env_precision(const char * name, int def) { + const char * env = getenv(name); + if (!env) { + return def; + } + std::string mode(env); + for (char & c : mode) { + c = (char) std::toupper((unsigned char) c); + } + for (int i = GGML_SYCL_DYNAMIC_PRECISION_F16; i <= GGML_SYCL_DYNAMIC_PRECISION_F32; i++) { + if (mode == ggml_sycl_dynamic_precision_names[i]) { + return i; + } + } + GGML_LOG_WARN("%s: unknown %s=%s, using %s\n", __func__, name, env, ggml_sycl_dynamic_precision_names[def]); + return def; +} int g_ggml_sycl_use_async_mem_op = 0; +int g_ggml_sycl_use_async_mem_op_requested = 1; +int g_ggml_sycl_use_level_zero_api = 0; int g_ggml_sycl_enable_flash_attention = 1; +int g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; +int g_ggml_sycl_usm_system = 0; +int g_ggml_sycl_enable_host_pinned_mem = 1; +int g_ggml_sycl_host_pinned_mem_2g = 0; +int g_ggml_sycl_upload_staging_slots = 4; +int g_ggml_sycl_get_mem_api = MEMORY_API_TYPE_LEVEL_ZERO; +int g_ggml_sycl_enable_sparse_fa = 0; +int g_ggml_sycl_debug_sparse_fa = 0; +int g_ggml_sycl_sparse_fa_margin = 256; +int g_ggml_sycl_graph_eviction_timeout = 10; + +#if GGML_SYCL_DNNL +// ask oneDNN which matmul it picks for a small f16 problem +static bool ggml_sycl_dnnl_detect_optimized_gemm(int device) { + using dt = dnnl::memory::data_type; + + try { + const dnnl::memory::dims dims = { 1, 64, 64 }; + const dnnl::memory::dims strides = { 64 * 64, 64, 1 }; + + dnnl::primitive_attr attr; + attr.set_scratchpad_mode(dnnl::scratchpad_mode::user); + + const auto & q = dpct::dev_mgr::instance().get_device(device).default_queue(); + const auto eng = dnnl::sycl_interop::make_engine(q.get_device(), q.get_context()); + const auto pd = dnnl::matmul::primitive_desc(eng, + dnnl::memory::desc(dims, dt::f16, strides), + dnnl::memory::desc(dims, dt::f16, strides), + dnnl::memory::desc(dims, dt::f32, strides), attr); + + const std::string impl = pd.impl_info_str(); + if (impl.find("ref") == std::string::npos) { + return true; + } + + GGML_LOG_WARN("%s: oneDNN has no optimized matmul for device %d (picks %s), using SYCL kernels\n", + __func__, device, impl.c_str()); + } catch (const std::exception & e) { + GGML_LOG_WARN("%s: oneDNN matmul probe failed on device %d (%s), using SYCL kernels\n", + __func__, device, e.what()); + } + return false; +} +#endif static ggml_sycl_device_info ggml_sycl_init() { + GGML_SYCL_DEBUG("[SYCL] call ggml_sycl_init\n"); ggml_sycl_device_info info = {}; - info.device_count = dpct::dev_mgr::instance().device_count(); + // Do not hard crash when there exists no SYCL devices. + // We want to allow the user to use non-SYCL tools when SYCL is compiled (such as llama-quantize) + try { + info.device_count = dpct::dev_mgr::instance().device_count(); + } catch (sycl::exception const &exc) { + GGML_LOG_INFO("%s: no SYCL device available: %s\n", __func__, exc.what()); + info.device_count = 0; + } if (info.device_count == 0) { GGML_LOG_ERROR("%s: failed to initialize: %s\n", GGML_SYCL_NAME, __func__); return info; @@ -85,13 +216,30 @@ static ggml_sycl_device_info ggml_sycl_init() { // GGML_LOG_INFO("%s: SYCL_USE_XMX: no\n", __func__); // #endif for (int i = 0; i < info.device_count; ++i) { - info.devices[i].vmm = 0; dpct::device_info prop; - sycl::device device = dpct::dev_mgr::instance().get_device(i); + auto & device = dpct::dev_mgr::instance().get_device(i); SYCL_CHECK(CHECK_TRY_ERROR(dpct::get_device_info( prop, device))); +#if !defined(GGML_SYCL_SUPPORT_VMM) + info.devices[i].vmm = 0; +#else + info.devices[i].vmm = device.has(sycl::aspect::ext_oneapi_virtual_mem); + if (info.devices[i].vmm) { + // NB: SYCL's get_mem_granularity always returns the _minimum_ granularity, + // but the L0 API requires a larger page size for allocs above 2 MiB and + // rejects non-multiples with UR_RESULT_ERROR_INVALID_VALUE [sic]. + // Here we clamp it to 2 MiB for simplicity, but other devices may require + // calling zeVirtualMemQueryPageSize or yet unexposed public API. + const size_t physical_page = 2ull << 20; // 2 MiB + info.devices[i].vmm_granularity = std::max( + sycl::ext::oneapi::experimental::get_mem_granularity( + device, sycl::context(device)), + physical_page); + } +#endif + info.default_tensor_split[i] = total_vram; total_vram += prop.get_global_mem_size(); @@ -99,21 +247,57 @@ static ggml_sycl_device_info ggml_sycl_init() { 100 * prop.get_major_version() + 10 * prop.get_minor_version(); info.devices[i].nsm = prop.get_max_compute_units() / 16; //16: Number of Xe Cores info.devices[i].opt_feature.reorder = device.ext_oneapi_architecture_is(syclex::arch_category::intel_gpu); +#if GGML_SYCL_DNNL + info.devices[i].opt_feature.onednn_optimized_gemm = ggml_sycl_dnnl_detect_optimized_gemm(i); +#endif info.devices[i].smpbo = prop.get_local_mem_size(); info.devices[i].warp_size = WARP_SIZE; + info.devices[i].usm_system_support = device.has(sycl::aspect::usm_system_allocations); + +#ifdef GGML_SYCL_GRAPH + info.devices[i].graph_support = device.has(sycl::aspect::ext_oneapi_limited_graph); + info.devices[i].graph_update_support = device.has(sycl::aspect::ext_oneapi_graph); +#endif info.max_work_group_sizes[i] = prop.get_max_work_group_size(); info.devices[i].max_wg_per_cu = info.max_work_group_sizes[i] / prop.get_max_compute_units(); + info.devices[i].hw_info = get_device_hw_info(&device); + + // Only check GPU devices; CPU devices use OpenCL and would otherwise + // disable Level Zero for the GPUs on systems without ONEAPI_DEVICE_SELECTOR set. + if (device.is_gpu() && device.default_queue().get_backend() != sycl::backend::ext_oneapi_level_zero) { + GGML_LOG_WARN("SYCL GPU device %d does not use Level Zero backend, disabling Level Zero memory API\n", i); + info.ext_oneapi_level_zero = false; + } +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + if (info.ext_oneapi_level_zero && device.is_gpu() && device.default_queue().get_backend() == sycl::backend::ext_oneapi_level_zero) { + ze_device_handle_t ze_dev = sycl::get_native(device.default_queue().get_device()); + ze_device_properties_t props = {}; + props.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + ze_result_t r = zeDeviceGetProperties(ze_dev, &props); + if (r == ZE_RESULT_SUCCESS) { + info.devices[i].l0_device_type_valid = true; + info.devices[i].l0_discrete_gpu = !(props.flags & ZE_DEVICE_PROPERTY_FLAG_INTEGRATED); + } + } +#endif } for (int id = 0; id < info.device_count; ++id) { info.default_tensor_split[id] /= total_vram; } + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + //update g_ggml_sycl_use_level_zero_api according to the device support + g_ggml_sycl_use_level_zero_api = + info.ext_oneapi_level_zero && g_ggml_sycl_use_level_zero_api; +#endif + return info; } -const ggml_sycl_device_info & ggml_sycl_info() { +ggml_sycl_device_info & ggml_sycl_info() { static ggml_sycl_device_info info = ggml_sycl_init(); return info; } @@ -194,75 +378,176 @@ void ggml_backend_sycl_print_sycl_devices() { print_device_opt_feature(device_count); } -static inline int get_sycl_env(const char *env_name, int default_val) { - char *user_device_string = getenv(env_name); - int user_number = default_val; - - unsigned n; - if (user_device_string != NULL && - sscanf(user_device_string, " %u", &n) == 1) { - user_number = (int)n; +static const char* dev2dev_int2str(int dev2dev) { + if (dev2dev == DEV2DEV_MEMCPY_SYCL) { + return "SYCL API"; + } else if (dev2dev == DEV2DEV_MEMCPY_L0) { + return "Level Zero API"; + } else if (dev2dev == DEV2DEV_MEMCPY_FORWARD) { + return "Host Forward"; } else { - user_number = default_val; + return "Unknown"; } - return user_number; +} + +/* +* There are several entry APIs to be called as first function in SYCL backend in different cases. +* It's the first internal function to be called by them in SYCL backend. +* This function is used to do initialize work for the SYCL backend and set the global variables. +*/ +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +static ze_result_t init_zes() { + ze_result_t res = zesInit(0); + if (res != ZE_RESULT_SUCCESS) { + GGML_SYCL_DEBUG("Warning: [%s] zesInit failed with code %d. Sysman free-memory query be unavailable.\n", + __func__, (int) res); + } + return res; +} + +static ze_result_t get_zes_init_res() { + static ze_result_t zes_init_res = init_zes(); + GGML_SYCL_DEBUG("[SYCL] call %s: zesInit result: %d\n", __func__, (int) zes_init_res); + return zes_init_res; +} +#endif + +static void initialize_sycl_begining() { +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + //must be called in initialization stage, before any other Level Zero API calls + GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); + get_zes_init_res(); +#endif } static void ggml_check_sycl() try { + GGML_SYCL_DEBUG("[SYCL] ggml_check_sycl()\n"); static bool initialized = false; if (!initialized) { - g_ggml_sycl_debug = get_sycl_env("GGML_SYCL_DEBUG", 0); - g_ggml_sycl_disable_optimize = get_sycl_env("GGML_SYCL_DISABLE_OPT", 0); - g_ggml_sycl_disable_graph = get_sycl_env("GGML_SYCL_DISABLE_GRAPH", 1); - g_ggml_sycl_disable_dnn = get_sycl_env("GGML_SYCL_DISABLE_DNN", 0); - g_ggml_sycl_prioritize_dmmv = get_sycl_env("GGML_SYCL_PRIORITIZE_DMMV", 0); + initialize_sycl_begining(); + + g_ggml_sycl_debug = ggml_sycl_get_env("GGML_SYCL_DEBUG", 0); + g_ggml_sycl_dev_debug = ggml_sycl_get_env("GGML_SYCL_DEV_DEBUG", 0); + g_ggml_sycl_enable_optimize = ggml_sycl_get_env("GGML_SYCL_ENABLE_OPT", 1); + g_ggml_sycl_enable_graph = ggml_sycl_get_env("GGML_SYCL_ENABLE_GRAPH", 0); + g_ggml_sycl_enable_dnn = ggml_sycl_get_env("GGML_SYCL_ENABLE_DNN", 1); + g_ggml_sycl_fa_onednn = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN", 1); + g_ggml_sycl_fa_onednn_max_kv = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN_MAX_KV", 0); + g_ggml_sycl_enable_mkl_fa = ggml_sycl_get_env("GGML_SYCL_ENABLE_MKL_FA", 1); + g_ggml_sycl_memtrace = ggml_sycl_get_env("GGML_SYCL_MEMTRACE", 0); + g_ggml_sycl_memtrace_step = ggml_sycl_get_env("GGML_SYCL_MEMTRACE_STEP", 64); + g_ggml_sycl_enable_vmm = ggml_sycl_get_env("GGML_SYCL_ENABLE_VMM", 1); + g_ggml_sycl_enable_fusion = ggml_sycl_get_env("GGML_SYCL_ENABLE_FUSION", 1); + g_ggml_sycl_enable_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_ESIMD", 1); + g_ggml_sycl_mmvq_wide = ggml_sycl_get_env("GGML_SYCL_MMVQ_WIDE", 1); + g_ggml_sycl_enable_xmx_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_XMX_ESIMD", 1); + g_ggml_sycl_xmx_max_cols = std::max(0, ggml_sycl_get_env("GGML_SYCL_XMX_MAX_COLS", GGML_SYCL_XMX_MAX_COLS_DEFAULT)); + g_ggml_sycl_xmx_glu_max_cols = + std::clamp(ggml_sycl_get_env("GGML_SYCL_XMX_GLU_MAX_COLS", GGML_SYCL_XMX_GLU_MAX_COLS), 0, GGML_SYCL_XMX_GLU_MAX_COLS); + g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0); + g_ggml_sycl_graph_eviction_timeout = ggml_sycl_get_env("GGML_SYCL_GRAPH_EVICTION_TIMEOUT", 10); + g_ggml_sycl_xmx_gather_types = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_TYPES", GGML_SYCL_XMX_GATHER_TYPES_DEFAULT); + g_ggml_sycl_xmx_gather_shapes = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_SHAPES", GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT); + g_ggml_sycl_dynamic_precision = + ggml_sycl_get_env_precision("GGML_SYCL_DYNAMIC_PRECISION", GGML_SYCL_DYNAMIC_PRECISION_DEFAULT); + g_ggml_sycl_dynamic_required_precision = + ggml_sycl_get_env_precision("GGML_SYCL_DYNAMIC_REQUIRED_PRECISION", GGML_SYCL_DYNAMIC_PRECISION_F32); + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + g_ggml_sycl_use_level_zero_api = ggml_sycl_get_env("GGML_SYCL_USE_LEVEL_ZERO_API", 1); +#else + g_ggml_sycl_use_level_zero_api = 0; +#endif + g_ggml_sycl_dev2dev_memcpy = ggml_sycl_get_env("GGML_SYCL_DEV2DEV_MEMCPY", DEV2DEV_MEMCPY_SYCL); + g_ggml_sycl_get_mem_api = ggml_sycl_get_env("GGML_SYCL_GET_MEM_API", MEMORY_API_TYPE_LEVEL_ZERO); + if (g_ggml_sycl_use_level_zero_api == 0) { + g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; + g_ggml_sycl_get_mem_api = MEMORY_API_TYPE_SYCL; + } #ifdef SYCL_FLASH_ATTN - g_ggml_sycl_enable_flash_attention = get_sycl_env("GGML_SYCL_ENABLE_FLASH_ATTN", 1); + g_ggml_sycl_enable_flash_attention = ggml_sycl_get_env("GGML_SYCL_ENABLE_FLASH_ATTN", 1); #else g_ggml_sycl_enable_flash_attention = 0; #endif + g_ggml_sycl_usm_system = ggml_sycl_get_env("GGML_SYCL_USM_SYSTEM", 0); + g_ggml_sycl_enable_host_pinned_mem = + ggml_sycl_get_env("GGML_SYCL_ENABLE_HOST_PINNED_MEM", 1); + + g_ggml_sycl_host_pinned_mem_2g = + ggml_sycl_get_env("GGML_SYCL_HOST_PINNED_MEM_2G", 0) & g_ggml_sycl_enable_host_pinned_mem; + g_ggml_sycl_upload_staging_slots = std::max(0, ggml_sycl_get_env("GGML_SYCL_UPLOAD_STAGING_SLOTS", 4)); + + g_ggml_sycl_enable_sparse_fa = ggml_sycl_get_env("GGML_SYCL_SPARSE_FA", 0); + g_ggml_sycl_debug_sparse_fa = ggml_sycl_get_env("GGML_SYCL_SPARSE_FA_DEBUG", 0); + g_ggml_sycl_sparse_fa_margin = ggml_sycl_get_env("GGML_SYCL_SPARSE_FA_MARGIN", 256); + GGML_SYCL_DEBUG("[SYCL] call ggml_check_sycl\n"); GGML_LOG_INFO("Build with Macros:\n"); -#if defined(GGML_SYCL_FORCE_MMQ) - GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: yes\n"); +#if defined(GGML_SYCL_DNNL) + GGML_LOG_INFO(" GGML_SYCL_DNNL: yes\n"); #else - GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: no\n"); + GGML_LOG_INFO(" GGML_SYCL_DNNL: no\n"); #endif + #if defined(GGML_SYCL_F16) GGML_LOG_INFO(" GGML_SYCL_F16: yes\n"); #else GGML_LOG_INFO(" GGML_SYCL_F16: no\n"); #endif + +#if defined(GGML_SYCL_FORCE_MMQ) + GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: yes\n"); +#else + GGML_LOG_INFO(" GGML_SYCL_FORCE_MMQ: no\n"); +#endif + #if defined(GGML_SYCL_GRAPH) GGML_LOG_INFO(" GGML_SYCL_GRAPH: yes\n"); #else GGML_LOG_INFO(" GGML_SYCL_GRAPH: no\n"); #endif -#if defined(GGML_SYCL_DNNL) - GGML_LOG_INFO(" GGML_SYCL_DNNL: yes\n"); + +#if defined(GGML_SYCL_SUPPORT_LEVEL_ZERO_API) + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_LEVEL_ZERO_API: yes\n"); #else - GGML_LOG_INFO(" GGML_SYCL_DNNL: no\n"); + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_LEVEL_ZERO_API: no\n"); +#endif +#if defined(GGML_SYCL_SUPPORT_VMM) + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_VMM: yes\n"); +#else + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_VMM: no\n"); #endif + //Print the running environment variables for SYCL backend GGML_LOG_INFO("Running with Environment Variables:\n"); GGML_LOG_INFO(" GGML_SYCL_DEBUG: %d\n", g_ggml_sycl_debug); - GGML_LOG_INFO(" GGML_SYCL_DISABLE_OPT: %d\n", g_ggml_sycl_disable_optimize); -#ifdef GGML_SYCL_GRAPH - GGML_LOG_INFO(" GGML_SYCL_DISABLE_GRAPH: %d\n", g_ggml_sycl_disable_graph); + GGML_LOG_INFO(" GGML_SYCL_DEV_DEBUG: %d\n", g_ggml_sycl_dev_debug); + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d (%s)\n", g_ggml_sycl_dev2dev_memcpy, dev2dev_int2str(g_ggml_sycl_dev2dev_memcpy)); + GGML_LOG_INFO(" GGML_SYCL_GET_MEM_API: %d (%s)\n", g_ggml_sycl_get_mem_api, mem_api_int2str(g_ggml_sycl_get_mem_api)); #else - GGML_LOG_INFO(" GGML_SYCL_DISABLE_GRAPH: graph disabled by compile flag\n"); + GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d (%s), enable to SYCL API since missing GGML_SYCL_SUPPORT_LEVEL_ZERO_API\n", + g_ggml_sycl_dev2dev_memcpy, dev2dev_int2str(g_ggml_sycl_dev2dev_memcpy)); + GGML_LOG_INFO(" GGML_SYCL_GET_MEM_API: %d (%s), enable to SYCL API since missing GGML_SYCL_SUPPORT_LEVEL_ZERO_API\n", + g_ggml_sycl_get_mem_api, mem_api_int2str(g_ggml_sycl_get_mem_api)); #endif -#if GGML_SYCL_DNNL - GGML_LOG_INFO(" GGML_SYCL_DISABLE_DNN: %d\n", g_ggml_sycl_disable_dnn); + +#if defined(GGML_SYCL_DNNL) + GGML_LOG_INFO(" GGML_SYCL_ENABLE_DNN: %d\n", g_ggml_sycl_enable_dnn); + GGML_LOG_INFO(" GGML_SYCL_FA_ONEDNN: %d\n", g_ggml_sycl_fa_onednn); #else - GGML_LOG_INFO(" GGML_SYCL_DISABLE_DNN: DNN disabled by compile flag\n"); + GGML_LOG_INFO(" GGML_SYCL_ENABLE_DNN: DNN disabled by compile flag\n"); + GGML_LOG_INFO(" GGML_SYCL_FA_ONEDNN: %d\n", g_ggml_sycl_fa_onednn); #endif - GGML_LOG_INFO(" GGML_SYCL_PRIORITIZE_DMMV: %d\n", g_ggml_sycl_prioritize_dmmv); - + GGML_LOG_INFO(" GGML_SYCL_FA_ONEDNN_MAX_KV: %d\n", g_ggml_sycl_fa_onednn_max_kv); + GGML_LOG_INFO(" GGML_SYCL_ENABLE_MKL_FA: %d\n", g_ggml_sycl_enable_mkl_fa); + GGML_LOG_INFO(" GGML_SYCL_MEMTRACE: %d\n", g_ggml_sycl_memtrace); + GGML_LOG_INFO(" GGML_SYCL_MEMTRACE_STEP: %d\n", g_ggml_sycl_memtrace_step); #ifdef SYCL_FLASH_ATTN GGML_LOG_INFO(" GGML_SYCL_ENABLE_FLASH_ATTN: %d\n", g_ggml_sycl_enable_flash_attention); #else @@ -270,6 +555,61 @@ static void ggml_check_sycl() try { g_ggml_sycl_enable_flash_attention); #endif +#ifdef GGML_SYCL_GRAPH + GGML_LOG_INFO(" GGML_SYCL_ENABLE_GRAPH: %d\n", g_ggml_sycl_enable_graph); + GGML_LOG_INFO(" GGML_SYCL_GRAPH_EVICTION_TIMEOUT: %d\n", g_ggml_sycl_graph_eviction_timeout); +#else + GGML_LOG_INFO(" GGML_SYCL_ENABLE_GRAPH: graph disabled by compile flag\n"); +#endif + + GGML_LOG_INFO(" GGML_SYCL_ENABLE_OPT: %d\n", g_ggml_sycl_enable_optimize); + GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_TYPES: %d\n", g_ggml_sycl_xmx_gather_types); + GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_SHAPES: %d\n", g_ggml_sycl_xmx_gather_shapes); + GGML_LOG_INFO(" GGML_SYCL_DYNAMIC_PRECISION: %s\n", + ggml_sycl_dynamic_precision_names[g_ggml_sycl_dynamic_precision]); + GGML_LOG_INFO(" GGML_SYCL_DYNAMIC_REQUIRED_PRECISION: %s\n", + ggml_sycl_dynamic_precision_names[g_ggml_sycl_dynamic_required_precision]); + +#if defined(GGML_SYCL_SUPPORT_VMM) + GGML_LOG_INFO(" GGML_SYCL_ENABLE_VMM: %d\n", g_ggml_sycl_enable_vmm); +#else + GGML_LOG_INFO(" GGML_SYCL_ENABLE_VMM: virtual memory extension is not available\n"); +#endif + + GGML_LOG_INFO(" GGML_SYCL_ENABLE_FUSION: %d\n", g_ggml_sycl_enable_fusion); + GGML_LOG_INFO(" GGML_SYCL_UPLOAD_STAGING_SLOTS: %d\n", g_ggml_sycl_upload_staging_slots); + +#if defined(__INTEL_LLVM_COMPILER) + GGML_LOG_INFO(" GGML_SYCL_ENABLE_ESIMD: %d\n", g_ggml_sycl_enable_esimd); +#else + GGML_LOG_INFO(" GGML_SYCL_ENABLE_ESIMD: %d disabled by compile flag\n", g_ggml_sycl_enable_esimd); +#endif + + GGML_LOG_INFO(" GGML_SYCL_MMVQ_WIDE: %d\n", g_ggml_sycl_mmvq_wide); + + GGML_LOG_INFO(" GGML_SYCL_ENABLE_XMX_ESIMD: %d\n", g_ggml_sycl_enable_xmx_esimd); + GGML_LOG_INFO(" GGML_SYCL_XMX_MAX_COLS: %d\n", g_ggml_sycl_xmx_max_cols); + GGML_LOG_INFO(" GGML_SYCL_XMX_GLU_MAX_COLS: %d\n", g_ggml_sycl_xmx_glu_max_cols); + + GGML_LOG_INFO(" GGML_SYCL_PRIORITIZE_DMMV: %d\n", g_ggml_sycl_prioritize_dmmv); + + g_ggml_sycl_use_async_mem_op_requested = ggml_sycl_get_env("GGML_SYCL_USE_ASYNC_MEM_OP", 1); + GGML_LOG_INFO(" GGML_SYCL_USE_ASYNC_MEM_OP: %d\n", g_ggml_sycl_use_async_mem_op_requested); + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + GGML_LOG_INFO(" GGML_SYCL_USE_LEVEL_ZERO_API: %d\n", g_ggml_sycl_use_level_zero_api); +#else + GGML_LOG_INFO(" GGML_SYCL_USE_LEVEL_ZERO_API: Disable Level Zero API usage by compile flag\n"); +#endif + + GGML_LOG_INFO(" GGML_SYCL_USM_SYSTEM: %d\n", g_ggml_sycl_usm_system); + GGML_LOG_INFO(" GGML_SYCL_ENABLE_HOST_PINNED_MEM: %d\n", g_ggml_sycl_enable_host_pinned_mem); + GGML_LOG_INFO(" GGML_SYCL_HOST_PINNED_MEM_2G: %d\n", g_ggml_sycl_host_pinned_mem_2g); + + GGML_LOG_INFO(" GGML_SYCL_SPARSE_FA: %d\n", g_ggml_sycl_enable_sparse_fa); + GGML_LOG_INFO(" GGML_SYCL_SPARSE_FA_DEBUG: %d\n", g_ggml_sycl_debug_sparse_fa); + GGML_LOG_INFO(" GGML_SYCL_SPARSE_FA_MARGIN: %d\n", g_ggml_sycl_sparse_fa_margin); + /* NOT REMOVE, keep it for next optimize for XMX. #if defined(SYCL_USE_XMX) fprintf(stderr, "%s: SYCL_USE_XMX: yes\n", __func__); @@ -277,11 +617,20 @@ static void ggml_check_sycl() try { fprintf(stderr, "%s: SYCL_USE_XMX: no\n", __func__); #endif */ - // Currently, we only use async malloc / free when graphs are enabled as it is required for the calls to be - // properly recorded. As this SYCL extension matures it may be beneficial to enable as the default path and in - // other places. -#if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC - g_ggml_sycl_use_async_mem_op = !g_ggml_sycl_disable_graph; +#ifdef GGML_SYCL_GRAPH + if (g_ggml_sycl_enable_graph) { + for (unsigned int i = 0; i < dpct::dev_mgr::instance().device_count(); ++i) { + if (!dpct::dev_mgr::instance().get_device(i).has(sycl::aspect::ext_oneapi_limited_graph)) { + GGML_LOG_WARN("SYCL device %u does not support SYCL graphs\n", i); + } + } + } + + // Async USM allocation/free is also useful outside the graph path: it avoids the host waits in the reorder + // staging path while preserving queue ordering semantics. Graph support still depends on the extension being + // available, but it no longer needs to control the non-graph fast path. +#if SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC + g_ggml_sycl_use_async_mem_op = g_ggml_sycl_use_async_mem_op_requested || g_ggml_sycl_enable_graph; if (g_ggml_sycl_use_async_mem_op) { for (unsigned int i = 0; i < dpct::dev_mgr::instance().device_count(); ++i) { if (!dpct::dev_mgr::instance().get_device(i).has(sycl::aspect::ext_oneapi_async_memory_alloc)) { @@ -290,6 +639,7 @@ static void ggml_check_sycl() try { } } } +#endif #endif if (CHECK_TRY_ERROR(g_all_sycl_device_count = dpct::dev_mgr::instance().device_count()) != 0) { @@ -345,27 +695,65 @@ catch (sycl::exception const &exc) { std::exit(1); } +inline void free_aligned_mem_host(void * memblock) { +#ifdef _WIN32 + _aligned_free(memblock); +#else + free(memblock); +#endif +} + // sycl buffer struct ggml_backend_sycl_buffer_context { + // pinned staging for uploads; the host fills one slot while the previous one transfers + static constexpr size_t staging_slot_size = 8*1024*1024; + + struct host_staging { + void * data = nullptr; + std::vector events; + std::vector submitted; + int next = 0; + }; + int device; void * dev_ptr = nullptr; queue_ptr stream; std::string name; optimize_feature opt_feature; std::vector tensor_extras; + host_staging staging; + bool is_usm_system; - ggml_backend_sycl_buffer_context(int device, void * dev_ptr, queue_ptr stream) : - device(device), dev_ptr(dev_ptr), stream(stream) { + ggml_backend_sycl_buffer_context(int device, void * dev_ptr, queue_ptr stream, bool is_usm_system) : + device(device), dev_ptr(dev_ptr), stream(stream), is_usm_system(is_usm_system) { check_allow_gpu_index(device); name = (GGML_SYCL_NAME + std::to_string(device)); opt_feature = ggml_sycl_info().devices[device].opt_feature; } + // waits for every queued upload, then releases the pinned block + void drop_host_staging() { + for (size_t i = 0; i < staging.submitted.size(); ++i) { + if (staging.submitted[i]) { + staging.events[i].wait_and_throw(); + staging.submitted[i] = false; + } + } + if (staging.data != nullptr) { + sycl::free(staging.data, *stream); + staging.data = nullptr; + } + } + ~ggml_backend_sycl_buffer_context() { + drop_host_staging(); if (dev_ptr != nullptr) { ggml_sycl_set_device(device); - SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(dev_ptr, *stream))); + if (is_usm_system) + free_aligned_mem_host(dev_ptr); + else + SYCL_CHECK(CHECK_TRY_ERROR(ggml_sycl_free_device(dev_ptr, *stream))); } //release extra used by tensors @@ -412,13 +800,19 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, return GGML_STATUS_SUCCESS; } - if (!g_ggml_sycl_disable_optimize) { + if (g_ggml_sycl_enable_optimize) { // set reorder extra buffer based on supported type switch (tensor->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: - case GGML_TYPE_Q6_K:{ + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_MXFP4:{ ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; tensor->extra = extra; ctx->tensor_extras.push_back(extra); @@ -448,6 +842,19 @@ catch (sycl::exception const &exc) { std::exit(1); } +// allocates the pinned upload staging on first use; returns false if staging is disabled or the allocation failed +static bool ggml_sycl_buffer_init_staging(ggml_backend_sycl_buffer_context * ctx) { + const int n_slots = g_ggml_sycl_upload_staging_slots; + if (n_slots > 0 && ctx->staging.data == nullptr) { + ctx->staging.data = sycl::malloc_host(n_slots * ctx->staging_slot_size, *ctx->stream); + if (ctx->staging.data != nullptr) { + ctx->staging.events.resize(n_slots); + ctx->staging.submitted.assign(n_slots, false); + } + } + return ctx->staging.data != nullptr; +} + static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor *tensor, const void *data, size_t offset, @@ -457,6 +864,32 @@ static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, GGML_SYCL_DEBUG(" size=%zu offset=%zu\n", size, offset); ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; ggml_sycl_set_device(ctx->device); + + // copy through pinned memory so the device never reads mmap()ed pages directly + // chunks pipeline on the in-order compute queue, so no drain per tensor is needed + if (ggml_sycl_buffer_init_staging(ctx)) { + queue_ptr stream = ctx->stream; + char * dst = (char *) tensor->data + offset; + const char * src = (const char *) data; + size_t remaining = size; + while (remaining > 0) { + const size_t chunk = std::min(remaining, ctx->staging_slot_size); + const int slot = ctx->staging.next; + ctx->staging.next = (ctx->staging.next + 1) % (int) ctx->staging.submitted.size(); + if (ctx->staging.submitted[slot]) { + ctx->staging.events[slot].wait_and_throw(); + } + void * stage = (char *) ctx->staging.data + slot * ctx->staging_slot_size; + memcpy(stage, src, chunk); + ctx->staging.events[slot] = stream->memcpy(dst, stage, chunk); + ctx->staging.submitted[slot] = true; + src += chunk; + dst += chunk; + remaining -= chunk; + } + return; + } + auto stream = &(dpct::dev_mgr::instance().get_device(ctx->device).default_queue()); SYCL_CHECK(CHECK_TRY_ERROR(dpct::dev_mgr::instance().get_device(ctx->device).queues_wait_and_throw())); #ifndef _WIN32 @@ -476,6 +909,49 @@ catch (sycl::exception const &exc) { std::exit(1); } +// Copy n_copies rows in one call. -sm tensor uploads split weights this way; the generic fallback does one set_tensor per row. +static void ggml_backend_sycl_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, + const void * data, size_t offset, size_t size, + size_t n_copies, size_t stride_tensor, + size_t stride_data) try { + ggml_backend_sycl_buffer_context * ctx = (ggml_backend_sycl_buffer_context *) buffer->context; + const char * src = (const char *) data; + ggml_sycl_set_device(ctx->device); + + if (stride_tensor != size || size > ctx->staging_slot_size || !ggml_sycl_buffer_init_staging(ctx)) { + // dst rows not contiguous, row larger than a staging slot, or no staging: same as the generic fallback + for (size_t i = 0; i < n_copies; ++i) { + ggml_backend_sycl_buffer_set_tensor(buffer, tensor, src + i * stride_data, offset + i * stride_tensor, size); + } + return; + } + + // dst rows are contiguous: pack as many rows as fit into a staging slot and copy each slot with one memcpy + // a slot is only waited on when it is reused, so packing the next slot overlaps with the previous copies + queue_ptr stream = ctx->stream; + char * dst = (char *) tensor->data + offset; + const size_t rows_per_slot = ctx->staging_slot_size / size; + for (size_t i0 = 0; i0 < n_copies; i0 += rows_per_slot) { + const size_t n = std::min(rows_per_slot, n_copies - i0); + const int slot = ctx->staging.next; + ctx->staging.next = (ctx->staging.next + 1) % (int) ctx->staging.submitted.size(); + if (ctx->staging.submitted[slot]) { + ctx->staging.events[slot].wait_and_throw(); + } + char * stage = (char *) ctx->staging.data + slot * ctx->staging_slot_size; + for (size_t i = 0; i < n; ++i) { + memcpy(stage + i * size, src + (i0 + i) * stride_data, size); + } + ctx->staging.events[slot] = stream->memcpy(dst + i0 * size, stage, n * size); + ctx->staging.submitted[slot] = true; + } +} +catch (sycl::exception const &exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ + << ", line:" << __LINE__ << std::endl; + std::exit(1); +} + static void ggml_backend_sycl_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor *tensor, void *data, size_t offset, @@ -498,14 +974,70 @@ catch (sycl::exception const &exc) { std::exit(1); } -static void dev2dev_memcpy(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst, - const void *ptr_src, size_t size) { +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +static bool ggml_sycl_is_l0_discrete_gpu(int device) { + return ggml_sycl_info().devices[device].l0_discrete_gpu; +} +#endif + +static void memcpy_host_forward(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst, + const void *ptr_src, size_t size) { char *host_buf = (char *)malloc(size); + if (host_buf == nullptr) { + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory for host staging\n", __func__, size); + GGML_ABORT("fatal error"); + } q_src.memcpy(host_buf, (const char *)ptr_src, size).wait(); q_dst.memcpy((char *)ptr_dst, host_buf, size).wait(); free(host_buf); } +static void dev2dev_memcpy(int device_dst, sycl::queue &q_dst, int device_src, sycl::queue &q_src, void *ptr_dst, + const void *ptr_src, size_t size) { + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + if (g_ggml_sycl_dev2dev_memcpy == DEV2DEV_MEMCPY_L0) { + // Use Level Zero direct copy for dGPU-to-dGPU transfers. + const bool l0_copy_supported = + ggml_sycl_is_l0_discrete_gpu(device_dst) && ggml_sycl_is_l0_discrete_gpu(device_src); + if (g_ggml_sycl_use_level_zero_api && l0_copy_supported) { + auto ze_ctx = sycl::get_native(q_dst.get_context()); + auto ze_dev = sycl::get_native(q_dst.get_device()); + ze_command_queue_desc_t cq_desc = {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, nullptr, 0, 0, + 0, ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS, ZE_COMMAND_QUEUE_PRIORITY_NORMAL}; + ze_command_list_handle_t cl; + ze_result_t r = zeCommandListCreateImmediate(ze_ctx, ze_dev, &cq_desc, &cl); + if (r == ZE_RESULT_SUCCESS) { + GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by L0\n"); + r = zeCommandListAppendMemoryCopy(cl, ptr_dst, ptr_src, size, nullptr, 0, nullptr); + zeCommandListDestroy(cl); + if (r == ZE_RESULT_SUCCESS) { + return; + } + } + } + } +#endif + + if (g_ggml_sycl_dev2dev_memcpy == DEV2DEV_MEMCPY_SYCL) { + if (q_dst.get_device().ext_oneapi_can_access_peer(q_src.get_device(), + sycl::ext::oneapi::peer_access::access_supported)) { + GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by SYCL\n"); + q_dst.get_device().ext_oneapi_enable_peer_access(q_src.get_device()); + SYCL_CHECK(CHECK_TRY_ERROR(q_dst.memcpy(ptr_dst, ptr_src, size).wait())); + return; + } + } + + // Host-staged copy + if(g_ggml_sycl_dev2dev_memcpy == DEV2DEV_MEMCPY_FORWARD) { + GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by host forward for setting GGML_SYCL_DEV2DEV_MEMCPY=2\n"); + } else { + GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by host forward for SYCL/L0 fallback\n"); + } + memcpy_host_forward(q_dst, q_src, ptr_dst, ptr_src, size); +} + static bool ggml_backend_sycl_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor *src, @@ -546,7 +1078,7 @@ ggml_backend_sycl_buffer_cpy_tensor(ggml_backend_buffer_t buffer, size_t size = ggml_nbytes(src); //todo. it's dirty solutino to walkaroud known issue:device2device cross GPUs. - dev2dev_memcpy(*stream_dst, *stream_src, dst->data, src->data, size); + dev2dev_memcpy(dst_ctx->device, *stream_dst, src_ctx->device, *stream_src, dst->data, src->data, size); //todo, it's known issue：error in device2device cross GPUs. reused when the issue is fixed. DON"T remove #if 0 @@ -638,7 +1170,7 @@ static const ggml_backend_buffer_i ggml_backend_sycl_buffer_interface = { /* .memset_tensor = */ ggml_backend_sycl_buffer_memset_tensor, /* .set_tensor = */ ggml_backend_sycl_buffer_set_tensor, /* .get_tensor = */ ggml_backend_sycl_buffer_get_tensor, - /* .set_tensor_2d = */ NULL, + /* .set_tensor_2d = */ ggml_backend_sycl_buffer_set_tensor_2d, /* .get_tensor_2d = */ NULL, /* .cpy_tensor = */ ggml_backend_sycl_buffer_cpy_tensor, /* .clear = */ ggml_backend_sycl_buffer_clear, @@ -660,22 +1192,61 @@ static const char * ggml_backend_sycl_buffer_type_get_name(ggml_backend_buffer_t return ctx->name.c_str(); } +static bool check_usm_system(int device, size_t size) { + bool use_usm_system = g_ggml_sycl_usm_system && size >= ((size_t)4 * MEM_SIZE_1G); + + if (use_usm_system && !ggml_sycl_info().devices[device].usm_system_support) { + GGML_LOG_INFO("Device does not support USM system allocations\n"); + use_usm_system = false; + } + + return use_usm_system; +} + +inline void * aligned_malloc_host(size_t alignment, size_t size) { +#ifdef _WIN32 + return _aligned_malloc(size, alignment); +#else + return aligned_alloc(alignment, size); +#endif +} + static ggml_backend_buffer_t ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) try { + GGML_SYCL_DEBUG("[SYCL] call %s: size=%zu\n", __func__, size); + ggml_check_sycl(); + ggml_backend_sycl_buffer_type_context * buft_ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; ggml_sycl_set_device(buft_ctx->device); const queue_ptr stream = buft_ctx->stream; size = std::max(size, (size_t)1); // syclMalloc returns null for size 0 + /* + Alignment below ensures best performance. While in theory it could lead to + wasting memory, this is acceptable because in practice only few buffers are + allocated and even less exceed the minimum size accepted here for USM system + allocations. + */ + size_t alignment = MEM_SIZE_2M; + size_t aligned_size = ((size + alignment - 1) / alignment) * alignment; + bool use_usm_system = check_usm_system(buft_ctx->device, aligned_size); void * dev_ptr; - SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)sycl::malloc_device( - size, *stream))); - if (!dev_ptr) { - GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device\n", __func__, size); - return nullptr; + if (use_usm_system) { + GGML_SYCL_DEBUG("[SYCL] allocating %zu Bytes with USM system\n", size); + dev_ptr = (void *)aligned_malloc_host(alignment, aligned_size); + if (!dev_ptr) { + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size); + return nullptr; + } + } else { + SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)ggml_sycl_malloc_device(size, *stream, GGML_SYCL_MEM_BUFFER))); + if (!dev_ptr) { + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device\n", __func__, size); + return nullptr; + } } - ggml_backend_sycl_buffer_context * ctx = new ggml_backend_sycl_buffer_context(buft_ctx->device, dev_ptr, buft_ctx->stream); + ggml_backend_sycl_buffer_context * ctx = new ggml_backend_sycl_buffer_context(buft_ctx->device, dev_ptr, buft_ctx->stream, use_usm_system); return ggml_backend_buffer_init(buft, ggml_backend_sycl_buffer_interface, ctx, size); } catch (sycl::exception const &exc) { @@ -685,18 +1256,39 @@ catch (sycl::exception const &exc) { } static size_t ggml_backend_sycl_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return 128; + return SYCL_BUFFER_ALIGNMENT; GGML_UNUSED(buft); } +static bool is_bmg_g31_arch(int device) { + return ggml_sycl_info().devices[device].hw_info.arch == gpu_arch::intel_gpu_bmg_g31; +} + static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { - return dpct::get_current_device().get_max_mem_alloc_size(); + ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; + int device = ctx->device; + // use the buft device, the current device may differ on multi-GPU systems + size_t max_alloc_size = dpct::get_device(device).get_max_mem_alloc_size(); + if (g_ggml_sycl_host_pinned_mem_2g) { + return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024); + } else { + if(is_bmg_g31_arch(device)) { + //Todo, it's workaround for BMG-G31, which has a known issue with large allocations. + //The max alloc size is reduced to 60% of the reported max alloc size. + //remove it after https://github.com/intel/compute-runtime/issues/998 is fixed. + max_alloc_size = max_alloc_size*0.6; + } + return max_alloc_size; + } GGML_UNUSED(buft); } static size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { - size_t size = ggml_nbytes(tensor); + // Reserve the additional scratch so it's visible to the graph allocator + size_t size = tensor->op == GGML_OP_FLASH_ATTN_EXT + ? ggml_sycl_flash_attn_ext_get_alloc_size(tensor) + : ggml_nbytes(tensor); int64_t ne0 = tensor->ne[0]; if (ggml_is_quantized(tensor->type)) { @@ -711,12 +1303,14 @@ static size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_t } static const ggml_backend_buffer_type_i ggml_backend_sycl_buffer_type_interface = { - /* .get_name = */ ggml_backend_sycl_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_sycl_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_sycl_buffer_type_get_alignment, - /* .get_max_size = */ ggml_backend_sycl_buffer_type_get_max_size, - /* .get_alloc_size = */ ggml_backend_sycl_buffer_type_get_alloc_size, - /* .is_host = */ NULL, + /* .get_name = */ ggml_backend_sycl_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_sycl_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_sycl_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_sycl_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_sycl_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ NULL, }; ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device) { @@ -793,6 +1387,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= VER_GEN9 ? 128 : 64; @@ -892,7 +1487,12 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{}; ctx->tensor_extras.push_back(extra); - ctx->streams.push_back(&(dpct::get_current_device().default_queue())); + // consumers index the queues by device id, so size the vector once with one queue per device + if (ctx->streams.empty()) { + for (int i = 0; i < ggml_sycl_info().device_count; ++i) { + ctx->streams.push_back(&(dpct::get_device(i).default_queue())); + } + } for (int i = 0; i < ggml_sycl_info().device_count; ++i) { int64_t row_low, row_high; @@ -911,21 +1511,13 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer, size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); } - // FIXME: do not crash if SYCL Buffer alloc fails - // currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first ggml_sycl_set_device(i); const queue_ptr stream = ctx->streams[i]; char * buf; - /* - DPCT1009:208: SYCL uses exceptions to report errors and does not use the - error codes. The original code was commented out and a warning string - was inserted. You need to rewrite this code. - */ - SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)sycl::malloc_device( - size, *stream))); + SYCL_CHECK(CHECK_TRY_ERROR(buf = (char *)ggml_sycl_malloc_device(size, *stream, GGML_SYCL_MEM_BUFFER))); if (!buf) { char err_buf[1024]; - snprintf(err_buf, 1023, "%s: can't allocate %lu Bytes of memory on device\n", __func__, size); + snprintf(err_buf, 1023, "%s: can't allocate %zu Bytes of memory on device\n", __func__, size); throw std::runtime_error(err_buf); } // set padding to 0 to avoid possible NaN values @@ -1116,7 +1708,7 @@ static ggml_backend_buffer_t ggml_backend_sycl_split_buffer_type_alloc_buffer(gg } static size_t ggml_backend_sycl_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { - return 128; + return SYCL_BUFFER_ALIGNMENT; GGML_UNUSED(buft); } @@ -1154,19 +1746,24 @@ static bool ggml_backend_sycl_split_buffer_type_is_host(ggml_backend_buffer_type } static ggml_backend_buffer_type_i ggml_backend_sycl_split_buffer_type_interface = { - /* .get_name = */ ggml_backend_sycl_split_buffer_type_get_name, - /* .alloc_buffer = */ ggml_backend_sycl_split_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_sycl_split_buffer_type_get_alignment, - /* .get_max_size = */ NULL, // defaults to SIZE_MAX - /* .get_alloc_size = */ ggml_backend_sycl_split_buffer_type_get_alloc_size, - /* .is_host = */ ggml_backend_sycl_split_buffer_type_is_host, + /* .get_name = */ ggml_backend_sycl_split_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_sycl_split_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_sycl_split_buffer_type_get_alignment, + /* .get_max_size = */ NULL, // defaults to SIZE_MAX + /* .get_alloc_size = */ ggml_backend_sycl_split_buffer_type_get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_sycl_split_buffer_type_is_host, }; -ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split) { +ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(int main_device, const float * tensor_split) { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_split_buffer_type\n"); + + GGML_UNUSED(main_device); + static std::mutex mutex; std::lock_guard lock(mutex); - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_split_buffer_type\n"); ggml_check_sycl(); // FIXME: this is not thread safe static std::map, struct ggml_backend_buffer_type> buft_map; @@ -1204,34 +1801,60 @@ ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * ten // host buffer type +struct ggml_backend_sycl_device_context { + int device; + std::string name; + std::string description; + int op_offload_min_batch_size; +}; + static const char * ggml_backend_sycl_host_buffer_type_name(ggml_backend_buffer_type_t buft) { return GGML_SYCL_NAME "_Host"; GGML_UNUSED(buft); } -inline void * aligned_malloc_host(size_t alignment, size_t size) { -#ifdef _WIN32 - return _aligned_malloc(size, alignment); -#else - return aligned_alloc(alignment, size); -#endif +static int ggml_backend_sycl_host_buffer_type_device(ggml_backend_buffer_type_t buft) { + return static_cast(buft->device->context)->device; } -inline void free_aligned_mem_host(void * memblock) { -#ifdef _WIN32 - _aligned_free(memblock); -#else - free(memblock); -#endif +//host pinned memory +static void * ggml_backend_sycl_host_malloc(int device, size_t size) { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_host_malloc of size %.2f MiB on device %d\n", size / 1024.0 / 1024.0, device); + void * ptr = nullptr; + try { + ggml_check_sycl(); + // USM host memory is page-locked and device-accessible by construction + auto & q = dpct::dev_mgr::instance().get_device(device).default_queue(); + ptr = sycl::malloc_host(size, q, sycl::property_list{}); + } catch (...) { + ptr = nullptr; + } + if (ptr == nullptr) { + GGML_LOG_WARN("%s: failed to allocate %.2f MiB of pinned memory\n", __func__, + size / 1024.0 / 1024.0); + } + + return ptr; } static void ggml_backend_sycl_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { - free_aligned_mem_host((void *)buffer->context); + if (buffer->context == nullptr) { + return; + } + if (g_ggml_sycl_enable_host_pinned_mem) { + const int device = ggml_backend_sycl_host_buffer_type_device(buffer->buft); + auto & q = dpct::dev_mgr::instance().get_device(device).default_queue(); + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(buffer->context, q))); + } else { + free_aligned_mem_host((void *) buffer->context); + } } static ggml_backend_buffer_t ggml_backend_sycl_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { - void * ptr = aligned_malloc_host(TENSOR_ALIGNMENT, size); + void * ptr = g_ggml_sycl_enable_host_pinned_mem ? + ggml_backend_sycl_host_malloc(ggml_backend_sycl_host_buffer_type_device(buft), size) : + aligned_malloc_host(TENSOR_ALIGNMENT, size); if (ptr == nullptr) { // fallback to cpu buffer return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size); @@ -1245,29 +1868,61 @@ static ggml_backend_buffer_t ggml_backend_sycl_host_buffer_type_alloc_buffer(ggm return buffer; } -ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() { - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_host_buffer_type\n"); - static struct ggml_backend_buffer_type ggml_backend_sycl_buffer_type_host = { - /* .iface = */ { - /* .get_name = */ ggml_backend_sycl_host_buffer_type_name, - /* .alloc_buffer = */ ggml_backend_sycl_host_buffer_type_alloc_buffer, - /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, - /* .get_max_size = */ NULL, // TODO: return device.maxBufferLength - /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, - /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, - }, - /* .device = */ ggml_backend_reg_dev_get(ggml_backend_sycl_reg(), 0), - /* .context = */ nullptr, - }; +static size_t ggml_backend_sycl_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { - return &ggml_backend_sycl_buffer_type_host; + if (g_ggml_sycl_enable_host_pinned_mem) { + const int device = ggml_backend_sycl_host_buffer_type_device(buft); + size_t max_alloc_size = dpct::dev_mgr::instance().get_device(device).get_max_mem_alloc_size(); + if (g_ggml_sycl_host_pinned_mem_2g) { + return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024); + } else { + return max_alloc_size; + } + } else { + return SIZE_MAX; + } } -// buffer pool for sycl (legacy) -struct ggml_sycl_pool_leg : public ggml_sycl_pool { - static const int MAX_SYCL_BUFFERS = 256; - - int device; +static ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type_for_device(int device) { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_host_buffer_type_for_device on device %d\n", device); + + // the vector is never resized after this, so the returned pointers stay valid + static std::vector buffer_types_host = [] { + std::vector bufts(ggml_backend_sycl_get_device_count()); + for (size_t i = 0; i < bufts.size(); i++) { + bufts[i] = { + /* .iface = */ { + /* .get_name = */ ggml_backend_sycl_host_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_sycl_host_buffer_type_alloc_buffer, + /* .alloc_buffer_n = */ NULL, + /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment, + /* .get_max_size = */ ggml_backend_sycl_host_buffer_type_get_max_size, + /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size, + /* .get_alloc_size_n = */ NULL, + /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host, + }, + /* .device = */ ggml_backend_reg_dev_get(ggml_backend_sycl_reg(), i), + /* .context = */ nullptr, + }; + } + return bufts; + }(); + + GGML_ASSERT(device >= 0 && device < (int) buffer_types_host.size()); + + return &buffer_types_host[device]; +} + +// TODO: this function is unused and is a temporary hack to avoid breaking changes +ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type() { + return ggml_backend_sycl_host_buffer_type_for_device(0); +} + +// buffer pool for sycl (legacy) +struct ggml_sycl_pool_leg : public ggml_sycl_pool { + static const int MAX_SYCL_BUFFERS = 256; + + int device; queue_ptr qptr; struct ggml_sycl_buffer { void * ptr = nullptr; @@ -1280,16 +1935,53 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool { explicit ggml_sycl_pool_leg(queue_ptr qptr_, int device_) : device(device_), qptr(qptr_) {} ~ggml_sycl_pool_leg() { +#ifdef DEBUG_SYCL_POOL + int n_cached = 0; + size_t bytes_cached = 0; + for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) { + if (buffer_pool[i].ptr != nullptr) { + ++n_cached; + bytes_cached += buffer_pool[i].size; + } + } + GGML_LOG_INFO("%s: %d buffers, cached = %.2f MiB\n", __func__, + n_cached, bytes_cached / 1024.0 / 1024.0); + const auto slots = format_slots_in_alloc_order(); + if (!slots.empty()) { + GGML_LOG_INFO("%s: slots MiB: %s\n", __func__, slots.c_str()); + } +#endif + for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) { ggml_sycl_buffer & b = buffer_pool[i]; if (b.ptr != nullptr) { - SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr))); + SYCL_CHECK(CHECK_TRY_ERROR(ggml_sycl_free_device(b.ptr, *qptr))); pool_size -= b.size; } } GGML_ASSERT(pool_size == 0); } +#ifdef DEBUG_SYCL_POOL + std::string format_slots_in_alloc_order() const { + std::string line; + char buf[32]; + bool first = true; + for (int i = 0; i < MAX_SYCL_BUFFERS; ++i) { + if (buffer_pool[i].ptr == nullptr) { + continue; + } + if (!first) { + line += '/'; + } + first = false; + snprintf(buf, sizeof(buf), "%.2f", buffer_pool[i].size / 1024.0 / 1024.0); + line += buf; + } + return line; + } +#endif + void * alloc(size_t size, size_t * actual_size) override { #ifdef DEBUG_sycl_MALLOC int nnz = 0; @@ -1331,11 +2023,9 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool { void * ptr; size_t look_ahead_size = (size_t) (1.05 * size); - SYCL_CHECK( - CHECK_TRY_ERROR(ptr = (void *)sycl::malloc_device( - look_ahead_size, *qptr))); + SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *)ggml_sycl_malloc_device(look_ahead_size, *qptr, GGML_SYCL_MEM_POOL_LEG))); if (!ptr) { - GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device/GPU\n", __func__, look_ahead_size); + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on device/GPU\n", __func__, look_ahead_size); return nullptr; } @@ -1347,7 +2037,7 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool { (uint32_t)(max_size/1024/1024), (uint32_t)(g_sycl_pool_size[id]/1024/1024), (uint32_t)(size/1024/1024)); #endif - // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg look_ahead_size=%lu, return %p\n", look_ahead_size, ptr); + // GGML_SYCL_DEBUG("ggml_sycl_pool_malloc_leg look_ahead_size=%zu, return %p\n", look_ahead_size, ptr); return ptr; } @@ -1361,17 +2051,138 @@ struct ggml_sycl_pool_leg : public ggml_sycl_pool { } } GGML_LOG_WARN("WARNING: sycl buffer pool full, increase MAX_sycl_BUFFERS\n"); - SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr))); + SYCL_CHECK(CHECK_TRY_ERROR(ggml_sycl_free_device(ptr, *qptr))); pool_size -= size; } }; +// pool with virtual memory management +#if defined(GGML_SYCL_SUPPORT_VMM) +struct ggml_sycl_pool_vmm : public ggml_sycl_pool { + static const size_t SYCL_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB + + int device; + sycl::context ctx; + sycl::device dev; + + uintptr_t pool_addr = 0; + size_t pool_used = 0; + size_t pool_size = 0; + size_t granularity; + + // physical_mem owns the commits (unlike cuMemMap) + struct mapping { + sycl::ext::oneapi::experimental::physical_mem phys; + void * map_ptr; + }; + std::vector mappings; + + explicit ggml_sycl_pool_vmm(queue_ptr qptr_, int device_) : + device(device_), + ctx(qptr_->get_context()), + dev(qptr_->get_device()), + granularity(ggml_sycl_info().devices[device_].vmm_granularity) { + } + + ~ggml_sycl_pool_vmm() { + if (pool_addr == 0) { + return; + } + + // Per spec, unmap must (a) match the exact (ptr, size) of an earlier + // physical_mem::map() call and (b) precede destruction of the + // physical_mem objects (their dtors won't unmap). + for (auto & m : mappings) { + SYCL_CHECK(CHECK_TRY_ERROR(sycl::ext::oneapi::experimental::unmap( + m.map_ptr, m.phys.size(), ctx))); + } + SYCL_CHECK(CHECK_TRY_ERROR(sycl::ext::oneapi::experimental::free_virtual_mem( + pool_addr, SYCL_POOL_VMM_MAX_SIZE, ctx))); + } + + void * alloc(size_t size, size_t * actual_size) override { + // round up the allocation size to the alignment to ensure that all allocations are aligned for all data types + size = GGML_PAD(size, SYCL_BUFFER_ALIGNMENT); + + size_t avail = pool_size - pool_used; + + if (size > avail) { + // round up to the next multiple of the granularity + size_t reserve_size = GGML_PAD(size - avail, granularity); + + GGML_ASSERT(pool_size + reserve_size <= SYCL_POOL_VMM_MAX_SIZE); + + if (ggml_sycl_memtrace_enabled()) { + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " pool_vmm[%d] committing %5zu MiB (pool %5zu -> %5zu MiB)\n", + device, reserve_size / (1024 * 1024), pool_size / (1024 * 1024), + (pool_size + reserve_size) / (1024 * 1024)); + ggml_sycl_memtrace_report("before pool_vmm commit"); + } + + // allocate more physical memory + std::optional phys; + SYCL_CHECK(CHECK_TRY_ERROR(phys.emplace(dev, ctx, reserve_size))); + + // reserve virtual address space (if not already reserved) + if (pool_addr == 0) { + SYCL_CHECK(CHECK_TRY_ERROR( + pool_addr = sycl::ext::oneapi::experimental::reserve_virtual_mem( + SYCL_POOL_VMM_MAX_SIZE, ctx))); + } + + // map at the end of the pool + void * map_ptr = nullptr; + SYCL_CHECK(CHECK_TRY_ERROR( + map_ptr = phys->map(pool_addr + pool_size, reserve_size, + sycl::ext::oneapi::experimental::address_access_mode::read_write))); + + // stash these so we could unmap this exact range in dtor + mappings.push_back({ + std::move(*phys), + map_ptr, + }); + + // add to the pool + pool_size += reserve_size; + ggml_sycl_memtrace_add(GGML_SYCL_MEM_POOL_VMM, map_ptr, reserve_size); + +#ifdef DEBUG_SYCL_MALLOC + GGML_LOG_INFO("sycl pool[%d]: size increased to %llu MB (reserved %llu MB)\n", + device, (unsigned long long) (pool_size/1024/1024), + (unsigned long long) (reserve_size/1024/1024)); +#endif + } + + GGML_ASSERT(pool_addr != 0); + + void * ptr = reinterpret_cast(pool_addr + pool_used); + *actual_size = size; + pool_used += size; + +#ifdef DEBUG_SYCL_MALLOC + GGML_LOG_INFO("sycl pool[%d]: allocated %llu bytes at %p\n", device, (unsigned long long) size, ptr); +#endif + + return ptr; + } + + void free(void * ptr, size_t size) override { +#ifdef DEBUG_SYCL_MALLOC + GGML_LOG_INFO("sycl pool[%d]: freed %llu bytes at %p\n", device, (unsigned long long) size, ptr); +#endif + + pool_used -= size; + + // all deallocations must be in reverse order of the allocations + GGML_ASSERT(ptr == reinterpret_cast(pool_addr + pool_used)); + } +}; +#endif // defined(GGML_SYCL_SUPPORT_VMM) + struct ggml_sycl_pool_host : public ggml_sycl_pool { queue_ptr qptr; int device; - inline static int counter{ 0 }; - struct ggml_sycl_buffer { void * ptr = nullptr; size_t size = 0; @@ -1381,6 +2192,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { static constexpr int MAX_POOL_SIZE{ 64 }; std::vector buffer_pool = std::vector(MAX_POOL_SIZE); size_t pool_size = 0; + int counter = 0; explicit ggml_sycl_pool_host(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {} @@ -1394,41 +2206,55 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { b.size = 0; } } - counter = 0; } void * alloc(size_t size, size_t * actual_size) override { if (counter == MAX_POOL_SIZE) { - ggml_sycl_buffer b = buffer_pool[0]; - void * ptr = b.ptr; - *actual_size = b.size; - counter = 1; - return ptr; - } - ggml_sycl_buffer & b = buffer_pool[counter]; - - if (b.ptr == nullptr) { - void * ptr; - + // the pool wrapped around, no slot is reserved for this allocation + void * ptr = nullptr; SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr))); if (!ptr) { - GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on host\n", __func__, size); + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size); return nullptr; } - pool_size += size; + pool_size += size; *actual_size = size; - counter = counter + 1; + counter = 0; return ptr; - } else { - ++counter; - b.size = size; - return b.ptr; } + ggml_sycl_buffer & b = buffer_pool[counter]; + ++counter; + + if (b.ptr != nullptr) { + if (b.size >= size) { + // hand out the cached buffer, free() puts it back into the pool + void * ptr = b.ptr; + *actual_size = b.size; + b.ptr = nullptr; + b.size = 0; + return ptr; + } + // the cached buffer is too small, replace it + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr))); + pool_size -= b.size; + b.ptr = nullptr; + b.size = 0; + } + + void * ptr; + SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr))); + if (!ptr) { + GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size); + return nullptr; + } + pool_size += size; + *actual_size = size; + return ptr; } void free(void * ptr, size_t size) override { // if the pool is not completed add the pointer to it in place of the first nullptr found. - // Otherwise do nothing, pointers will be freed once the pool is deallocated. + // Otherwise free the buffer, all slots are cached. for (int i = 0; i < MAX_POOL_SIZE; ++i) { ggml_sycl_buffer & b = buffer_pool[i]; if (b.ptr == nullptr) { @@ -1437,6 +2263,8 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool { return; } } + SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr))); + pool_size -= size; } }; @@ -1446,16 +2274,14 @@ std::unique_ptr ggml_backend_sycl_context::new_pool_for_host(que } std::unique_ptr ggml_backend_sycl_context::new_pool_for_device(queue_ptr qptr, int device) { - // TBD: NO VMM support - // if (ggml_sycl_info().devices[device].vmm) { - // return std::unique_ptr(new ggml_sycl_pool_vmm(device)); - // } - return std::unique_ptr(new ggml_sycl_pool_leg(qptr, device)); +#if defined(GGML_SYCL_SUPPORT_VMM) + if (g_ggml_sycl_enable_vmm && ggml_sycl_info().devices[device].vmm) { + return std::unique_ptr(new ggml_sycl_pool_vmm(qptr, device)); + } +#endif // defined(GGML_SYCL_SUPPORT_VMM) + return std::unique_ptr(new ggml_sycl_pool_leg(qptr, device)); } -// TBD pool with virtual memory management -// struct ggml_sycl_pool_vmm : public ggml_sycl_pool - /// kernels typedef void (*ggml_sycl_op_mul_mat_t)( ggml_backend_sycl_context & ctx, @@ -1698,69 +2524,6 @@ static void scale_f32(const float * x, float * dst, const float scale, const flo } -template -static void pool2d_nchw_kernel( - const int ih, const int iw, const int oh, const int ow, - const int kh, const int kw, const int sh, const int sw, - const int ph, const int pw, const int parallel_elements, - const Ti* src, To* dst, const enum ggml_op_pool op, - const sycl::nd_item<3> &item_ct1) { - int idx = item_ct1.get_local_id(2) + - item_ct1.get_group(2) * item_ct1.get_local_range(2); - if (idx >= parallel_elements) { - return; - } - - const int I_HW = ih * iw; - const int O_HW = oh * ow; - const int nc = idx / O_HW; - const int cur_oh = idx % O_HW / ow; - const int cur_ow = idx % O_HW % ow; - const Ti* i_ptr = src + nc * I_HW; - To* o_ptr = dst + nc * O_HW; - const int start_h = cur_oh * sh - ph; - const int bh = sycl::max(0, start_h); - const int eh = sycl::min(ih, start_h + kh); - const int start_w = cur_ow * sw - pw; - const int bw = sycl::max(0, start_w); - const int ew = sycl::min(iw, start_w + kw); - - To res = 0; - - switch (op) { - case GGML_OP_POOL_AVG: res = 0; break; - case GGML_OP_POOL_MAX: res = -FLT_MAX; break; - default: - res = (To) sycl::nan(uint32_t(0)); - break; - } - - for (int i = bh; i < eh; i += 1) { - for (int j = bw; j < ew; j += 1) { -#if DPCT_COMPATIBILITY_TEMP >= 350 - /* - DPCT1098:106: The '*' expression is used instead of the __ldg - call. These two expressions do not provide the exact same - functionality. Check the generated code for potential precision - and/or performance issues. - */ - Ti cur = *(i_ptr + i * iw + j); -#else - Ti cur = i_ptr[i * iw + j]; -#endif - switch (op) { - case GGML_OP_POOL_AVG: res += (cur / (kh * kw)); break; - case GGML_OP_POOL_MAX: res = sycl::max(res, (To)cur); break; - default: - res = (To) sycl::nan(uint32_t(0)); - break; - } - } - } - o_ptr[cur_oh * ow + cur_ow] = res; -} - - static void ggml_mul_mat_p021_f16_f32_sycl(const void *vx, const float *y, float *dst, const int ncols_x, const int nrows_x, @@ -1838,11 +2601,148 @@ static int next_power_of_2(int x) { return n; } +static void init_argsort_indices_padded( + int * idx, + const int nrows, + const int ncols_pad, + const sycl::nd_item<1> & item_ct1) { + const size_t gid = item_ct1.get_local_range(0) * item_ct1.get_group(0) + item_ct1.get_local_id(0); + const size_t total = (size_t) nrows * (size_t) ncols_pad; + + if (gid >= total) { + return; + } + + idx[gid] = (int) (gid % (size_t) ncols_pad); +} + +template +static void argsort_f32_i32_global_pass(const float * x, + int * idx, + const int ncols, + const int nrows, + const int ncols_pad, + const int j, + const int k, + const sycl::nd_item<1> & item_ct1) { + const size_t gid = item_ct1.get_local_range(0) * item_ct1.get_group(0) + item_ct1.get_local_id(0); + const size_t total = (size_t) nrows * (size_t) ncols_pad; + + if (gid >= total) { + return; + } + + const int row = (int) (gid / (size_t) ncols_pad); + const int col = (int) (gid % (size_t) ncols_pad); + const int ixj = col ^ j; + + if (ixj <= col || ixj >= ncols_pad) { + return; + } + + const size_t base = (size_t) row * (size_t) ncols_pad; + const size_t pos_a = base + (size_t) col; + const size_t pos_b = base + (size_t) ixj; + + const int a = idx[pos_a]; + const int b = idx[pos_b]; + + bool do_swap = false; + + if ((col & k) == 0) { + if (a >= ncols || + (b < ncols && + (order == GGML_SORT_ORDER_ASC ? + x[(size_t) row * (size_t) ncols + (size_t) a] > x[(size_t) row * (size_t) ncols + (size_t) b] : + x[(size_t) row * (size_t) ncols + (size_t) a] < x[(size_t) row * (size_t) ncols + (size_t) b]))) { + do_swap = true; + } + } else { + if (b >= ncols || + (a < ncols && + (order == GGML_SORT_ORDER_ASC ? + x[(size_t) row * (size_t) ncols + (size_t) a] < x[(size_t) row * (size_t) ncols + (size_t) b] : + x[(size_t) row * (size_t) ncols + (size_t) a] > x[(size_t) row * (size_t) ncols + (size_t) b]))) { + do_swap = true; + } + } + + if (do_swap) { + idx[pos_a] = b; + idx[pos_b] = a; + } +} + +static void copy_argsort_indices_unpadded(const int * idx_padded, + int * dst, + const int nrows, + const int ncols, + const int ncols_pad, + const sycl::nd_item<1> & item_ct1) { + const size_t gid = item_ct1.get_local_range(0) * item_ct1.get_group(0) + item_ct1.get_local_id(0); + const size_t total = (size_t) nrows * (size_t) ncols; + + if (gid >= total) { + return; + } + + const int row = (int) (gid / (size_t) ncols); + const int col = (int) (gid % (size_t) ncols); + + dst[(size_t) row * (size_t) ncols + (size_t) col] = idx_padded[(size_t) row * (size_t) ncols_pad + (size_t) col]; +} + static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols, const int nrows, ggml_sort_order order, - queue_ptr stream, int device) { + queue_ptr stream, int device, ggml_sycl_pool & pool) { // bitonic sort requires ncols to be power of 2 const int ncols_pad = next_power_of_2(ncols); + const size_t shared_mem = (size_t) ncols_pad * sizeof(int); + const size_t smpbo = ggml_sycl_info().devices[device].smpbo; + + if (shared_mem > smpbo) { + ggml_sycl_pool_alloc idx_padded_alloc(pool, (size_t) nrows * (size_t) ncols_pad); + int * idx_padded = idx_padded_alloc.get(); + + constexpr size_t block_size = 256; + const size_t total_padded = (size_t) nrows * (size_t) ncols_pad; + const size_t nblocks_padded = (total_padded + block_size - 1) / block_size; + + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(nblocks_padded * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item_ct1) { init_argsort_indices_padded(idx_padded, nrows, ncols_pad, item_ct1); }); + + for (int k = 2; k <= ncols_pad; k *= 2) { + for (int j = k / 2; j > 0; j /= 2) { + if (order == GGML_SORT_ORDER_ASC) { + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(nblocks_padded * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item_ct1) { + argsort_f32_i32_global_pass(x, idx_padded, ncols, nrows, ncols_pad, j, + k, item_ct1); + }); + } else if (order == GGML_SORT_ORDER_DESC) { + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(nblocks_padded * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item_ct1) { + argsort_f32_i32_global_pass(x, idx_padded, ncols, nrows, ncols_pad, j, + k, item_ct1); + }); + } else { + GGML_ABORT("invalid sort order"); + } + } + } + + const size_t total = (size_t) nrows * (size_t) ncols; + const size_t nblocks = (total + block_size - 1) / block_size; + stream->parallel_for(sycl::nd_range<1>(sycl::range<1>(nblocks * block_size), sycl::range<1>(block_size)), + [=](sycl::nd_item<1> item_ct1) { + copy_argsort_indices_unpadded(idx_padded, dst, nrows, ncols, ncols_pad, item_ct1); + }); + + return; + } int nth = 1; int max_block_size = ggml_sycl_info().max_work_group_sizes[device]; @@ -1855,8 +2755,6 @@ static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols, const sycl::range<3> block_dims(1, 1, nth); const sycl::range<3> block_nums(1, nrows, 1); - const size_t shared_mem = ncols_pad * sizeof(int); - GGML_ASSERT(shared_mem<=ggml_sycl_info().devices[device].smpbo); if (order == GGML_SORT_ORDER_ASC) { stream->submit([&](sycl::handler &cgh) { @@ -1893,7 +2791,138 @@ static void argsort_f32_i32_sycl(const float *x, int *dst, const int ncols, } } +// Scan and block merge, shared by every launch shape below so a partitioned row uses the +// same insertion order as an unpartitioned one. +// +// src_map != nullptr: report src_map[col] instead of col, so a merge pass can carry the +// original column index through. +// out_vals != nullptr: also emit the k winning values, for a later merge pass. +// swap01: emit in the output order the single-pass path uses. +static void top_k_scan_merge_f32( + const float * src_vals, + const int32_t * src_map, + const int begin, + const int end, + const int k, + const int block_size, + float * shared_vals, + int * shared_idx, + float * out_vals, + int32_t * out_idx, + const bool swap01, + const sycl::nd_item<1> & item_ct1 +) { + const int tid = item_ct1.get_local_id(0); + + // The running top-k lives in SLM (shared local memory) rather than a private array: + // an array indexed by a runtime position cannot be register-allocated, so a private + // one lands in scratch, i.e. device memory, and insertion is this kernel's dominant + // cost. + // + // Lane-strided (lv[i * block_size]) rather than lane-blocked (lv[i]) so a given i is + // contiguous across lanes; a k-strided layout would put every lane of a shift step in + // the same SLM bank. + float * lv = shared_vals + tid; + int * li = shared_idx + tid; + + for (int i = 0; i < k; i++) { + lv[i * block_size] = -FLT_MAX; + li[i * block_size] = -1; + } + + // The k-th best, cached in a register. The reject test is taken for the large + // majority of elements scanned, and in that case touches no memory. + float kth = -FLT_MAX; + + for (int col = begin + tid; col < end; col += block_size) { + float val = src_vals[col]; + + if (val > kth) { + int pos = k - 1; + while (pos > 0 && val > lv[(pos - 1) * block_size]) { + pos--; + } + + for (int i = k - 1; i > pos; i--) { + lv[i * block_size] = lv[(i - 1) * block_size]; + li[i * block_size] = li[(i - 1) * block_size]; + } + lv[pos * block_size] = val; + li[pos * block_size] = src_map ? src_map[col] : col; + + kth = lv[(k - 1) * block_size]; + } + } + + item_ct1.barrier(sycl::access::fence_space::local_space); + + if (tid != 0) { + return; + } + + // Same treatment for the merge accumulator, past the per-lane region. + float * fv = shared_vals + (size_t) k * block_size; + int * fi = shared_idx + (size_t) k * block_size; + + for (int i = 0; i < k; i++) { + fv[i] = -FLT_MAX; + fi[i] = -1; + } + + float fkth = -FLT_MAX; + + // Candidates are visited in the same (t, i) order as before, so tie-breaking is + // unchanged. + for (int t = 0; t < block_size; t++) { + for (int i = 0; i < k; i++) { + float val = shared_vals[i * block_size + t]; + + if (val <= fkth) { + // Lane t's list is sorted descending, so once one of its entries loses + // to the k-th best, every later entry loses too. fkth only rises, so + // that stays true for the rest of the merge. This turns the merge from + // block_size*k steps into roughly block_size plus the candidates + // accepted. + break; + } + + int idx = shared_idx[i * block_size + t]; + + int pos = k - 1; + while (pos > 0 && val > fv[pos - 1]) { + pos--; + } + + for (int j = k - 1; j > pos; j--) { + fv[j] = fv[j - 1]; + fi[j] = fi[j - 1]; + } + fv[pos] = val; + fi[pos] = idx; + + fkth = fv[k - 1]; + } + } + + if (out_vals) { + for (int i = 0; i < k; i++) { + out_vals[i] = fv[i]; + } + } + + for (int i = 0; i < k; i++) { + out_idx[i] = fi[i]; + } + + if (swap01 && k > 1) { + int32_t temp = out_idx[0]; + out_idx[0] = out_idx[1]; + out_idx[1] = temp; + } +} + static void top_k_f32_sycl( + ggml_backend_sycl_context & ctx, const float * src, int32_t * dst_indices, const int64_t ncols, @@ -1901,98 +2930,107 @@ static void top_k_f32_sycl( const int k, dpct::queue_ptr main_stream ) { - const int block_size = 128; + // A row is scanned by exactly one work-group, so a vocabulary-sized row leaves the + // rest of the device idle. What the scan is short of is memory requests in flight, + // not bandwidth or per-request latency, so lanes in flight is the lever: split the + // row across independent work-groups, have each emit its partition's top-k, and + // merge those nsplit*k candidates in a second launch. + // + // split_block trades parallelism against SLM residency. Its cost is + // (split_block + 1) * k * 8 bytes of SLM per group, so at the k <= 32 ceiling 128 + // lanes need about 33 KB, which leaves a single resident group per Xe-core. Revisit + // if the supported k ever grows. + constexpr int split_block = 128; + constexpr int max_splits = 128; + constexpr int min_cols = 8192; + + int nsplit = 1; + if (ncols >= min_cols) { + // A partition is then always >= split_block = 128 columns, hence always more than + // the k <= 32 ceiling, so no pass is ever padded with -FLT_MAX sentinels. + const int64_t want = ncols / split_block; + nsplit = (int) (want > max_splits ? max_splits : want); + } - const sycl::range<1> block_dims(block_size); - const sycl::range<1> grid_dims(nrows); + if (nsplit > 1) { + const int nchunk = (int) ((ncols + nsplit - 1) / nsplit); + const size_t ncand = (size_t) nrows * nsplit * k; - main_stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor shared_vals(sycl::range<1>(block_size * k), cgh); - sycl::local_accessor shared_idx(sycl::range<1>(block_size * k), cgh); + ggml_sycl_pool_alloc part_vals(ctx.pool(), ncand); + ggml_sycl_pool_alloc part_idx(ctx.pool(), ncand); - cgh.parallel_for( - sycl::nd_range<1>(grid_dims * block_dims, block_dims), - [=](sycl::nd_item<1> item_ct1) { - const int row = item_ct1.get_group(0); - const int tid = item_ct1.get_local_id(0); + float * pv = part_vals.get(); + int32_t * pi = part_idx.get(); - if (row >= nrows) return; + const sycl::range<1> block_dims(split_block); - const float * src_row = src + row * ncols; - int32_t * dst_idx_row = dst_indices + row * k; + main_stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor shared_vals(sycl::range<1>((split_block + 1) * k), cgh); + sycl::local_accessor shared_idx(sycl::range<1>((split_block + 1) * k), cgh); - float local_vals[32]; - int local_idx[32]; + cgh.parallel_for( + sycl::nd_range<1>(sycl::range<1>(nrows * nsplit) * block_dims, block_dims), + [=](sycl::nd_item<1> item_ct1) { + const int grp = item_ct1.get_group(0); + const int row = grp / nsplit; + const int part = grp % nsplit; + + const int begin = part * nchunk; + int end = begin + nchunk; + if (end > (int) ncols) { + end = (int) ncols; + } - for (int i = 0; i < k; i++) { - local_vals[i] = -FLT_MAX; - local_idx[i] = -1; - } + top_k_scan_merge_f32( + src + (int64_t) row * ncols, nullptr, begin, end, k, split_block, + shared_vals.get_multi_ptr().get(), + shared_idx.get_multi_ptr().get(), + pv + (size_t) grp * k, pi + (size_t) grp * k, false, item_ct1); + }); + }); - for (int col = tid; col < ncols; col += block_size) { - float val = src_row[col]; + main_stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor shared_vals(sycl::range<1>((split_block + 1) * k), cgh); + sycl::local_accessor shared_idx(sycl::range<1>((split_block + 1) * k), cgh); - if (val > local_vals[k-1]) { - int pos = k - 1; - while (pos > 0 && val > local_vals[pos - 1]) { - pos--; - } + cgh.parallel_for( + sycl::nd_range<1>(sycl::range<1>(nrows) * block_dims, block_dims), + [=](sycl::nd_item<1> item_ct1) { + const int row = item_ct1.get_group(0); + const size_t off = (size_t) row * nsplit * k; + + top_k_scan_merge_f32( + pv + off, pi + off, 0, nsplit * k, k, split_block, + shared_vals.get_multi_ptr().get(), + shared_idx.get_multi_ptr().get(), + nullptr, dst_indices + (int64_t) row * k, true, item_ct1); + }); + }); - for (int i = k - 1; i > pos; i--) { - local_vals[i] = local_vals[i - 1]; - local_idx[i] = local_idx[i - 1]; - } - local_vals[pos] = val; - local_idx[pos] = col; - } - } + return; + } - for (int i = 0; i < k; i++) { - shared_vals[tid * k + i] = local_vals[i]; - shared_idx[tid * k + i] = local_idx[i]; - } - item_ct1.barrier(sycl::access::fence_space::local_space); + const int block_size = 128; - if (tid == 0) { - float final_vals[32]; - int final_idx[32]; + const sycl::range<1> block_dims(block_size); + const sycl::range<1> grid_dims(nrows); - for (int i = 0; i < k; i++) { - final_vals[i] = -FLT_MAX; - final_idx[i] = -1; - } + main_stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor shared_vals(sycl::range<1>((block_size + 1) * k), cgh); + sycl::local_accessor shared_idx(sycl::range<1>((block_size + 1) * k), cgh); - for (int t = 0; t < block_size; t++) { - for (int i = 0; i < k; i++) { - float val = shared_vals[t * k + i]; - int idx = shared_idx[t * k + i]; - - if (val > final_vals[k-1]) { - int pos = k - 1; - while (pos > 0 && val > final_vals[pos - 1]) { - pos--; - } - - for (int j = k - 1; j > pos; j--) { - final_vals[j] = final_vals[j - 1]; - final_idx[j] = final_idx[j - 1]; - } - final_vals[pos] = val; - final_idx[pos] = idx; - } - } - } + cgh.parallel_for( + sycl::nd_range<1>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<1> item_ct1) { + const int row = item_ct1.get_group(0); - for (int i = 0; i < k; i++) { - dst_idx_row[i] = final_idx[i]; - } + if (row >= nrows) return; - if (k > 1) { - int32_t temp = dst_idx_row[0]; - dst_idx_row[0] = dst_idx_row[1]; - dst_idx_row[1] = temp; - } - } + top_k_scan_merge_f32( + src + (int64_t) row * ncols, nullptr, 0, (int) ncols, k, block_size, + shared_vals.get_multi_ptr().get(), + shared_idx.get_multi_ptr().get(), + nullptr, dst_indices + (int64_t) row * k, true, item_ct1); }); }); } @@ -2144,6 +3182,11 @@ catch (sycl::exception const &exc) { std::exit(1); } +#if GGML_SYCL_DNNL +// F32 GEMMs below this size (row_diff * src1_ncols * ne10) call oneMKL directly instead of oneDNN +static constexpr int64_t SYCL_F32_DNNL_MIN_GEMM_FLOPS = 256 * 256 * 256; +#endif + inline void ggml_sycl_op_mul_mat_sycl( ggml_backend_sycl_context & ctx, const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, @@ -2176,8 +3219,65 @@ inline void ggml_sycl_op_mul_mat_sycl( #else bool use_fp16 = false; #endif + +#if GGML_SYCL_DNNL && defined(GGML_SYCL_HAS_BF16) + // Fast path for bf16 src0 + if (src0->type == GGML_TYPE_BF16 && g_ggml_sycl_enable_dnn && + ggml_sycl_dnnl_has_optimized_gemm(ggml_sycl_get_device()) && ggml_is_contiguous(src0) && + row_diff == src0->ne[1]) { + using bf16_t = sycl::ext::oneapi::bfloat16; + ggml_sycl_pool_alloc src1_as_bf16(ctx.pool(), src1_ncols*ne10); + if (src1->type != GGML_TYPE_BF16) { + const to_bf16_sycl_t to_bf16_sycl = ggml_get_to_bf16_sycl(src1->type, dst); + GGML_ASSERT(to_bf16_sycl != nullptr); + to_bf16_sycl(src1_ddf_i, src1_as_bf16.get(), src1_ncols*ne10, stream); + } else { + stream->memcpy(src1_as_bf16.get(), src1_ddf_i, src1_ncols*ne10*sizeof(bf16_t)); + } + DnnlGemmWrapper::row_gemm(ctx, row_diff, src1_ncols, ne10, + src0_dd_i, DnnlGemmWrapper::to_dt(), + src1_as_bf16.get(), DnnlGemmWrapper::to_dt(), + dst_dd_i, DnnlGemmWrapper::to_dt(), stream); + GGML_UNUSED(dst); + GGML_UNUSED(src1_ddq_i); + GGML_UNUSED(src1_padded_row_size); + return; + } +#endif + + // dequantize inside the GEMM instead of writing the f16 weights out and reading them back; src1 + // goes in its own type, so there is no separate conversion pass. The A stage + // decodes the weight bytes in place, so an installed persistent reorder must be + // visible to it: pass the flag through; types without an SoA A-stage decline + // inside and this falls through to the reorder-aware conversion route below. + bool src0_reordered = false; + if (const auto * src0_extra = static_cast(src0->extra)) { + src0_reordered = src0_extra->optimized_feature.reorder; + } + if (ggml_is_quantized(src0->type) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && + ggml_sycl_fused_dequant_gemm(src0->type, src0_reordered, src0_dd_i, src1_ddf_i, src1->type, ggml_sycl_src1_prec(dst), dst_dd_i, + row_diff, src1_ncols, ne10, ldc, ctx.pool(), stream)) { + return; + } + + // the f16 route converts src1 to f16 [TAG_GGML_PREC] + use_fp16 = use_fp16 && ggml_sycl_src1_f16_ok(dst); if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && use_fp16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) { + ggml_sycl_pool_alloc src1_as_f16(ctx.pool()); + if (src1->type != GGML_TYPE_F16) { + scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2, + " : converting src1 to fp16"); + const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst); + GGML_ASSERT(to_fp16_sycl != nullptr); + size_t ne = src1_ncols*ne10; + src1_as_f16.alloc(ne); + to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream); + } + const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16 + ? (const sycl::half *)src1_ddf_i + : src1_as_f16.get(); + ggml_sycl_pool_alloc src0_as_f16(ctx.pool()); if (src0->type != GGML_TYPE_F16) { scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2, @@ -2192,22 +3292,8 @@ inline void ggml_sycl_op_mul_mat_sycl( ? (const sycl::half *)src0_dd_i : src0_as_f16.get(); - ggml_sycl_pool_alloc src1_as_f16(ctx.pool()); - if (src1->type != GGML_TYPE_F16) { - scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2, - " : converting src1 to fp16"); - const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst); - GGML_ASSERT(to_fp16_sycl != nullptr); - size_t ne = src1_ncols*ne10; - src1_as_f16.alloc(ne); - to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream); - } - const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16 - ? (const sycl::half *)src1->data + src1_padded_row_size - : src1_as_f16.get(); - #if GGML_SYCL_DNNL - if (!g_ggml_sycl_disable_dnn) { + if (g_ggml_sycl_enable_dnn && ggml_sycl_dnnl_has_optimized_gemm(ggml_sycl_get_device())) { DnnlGemmWrapper::row_gemm(ctx,row_diff, src1_ncols , ne10, src0_ptr, DnnlGemmWrapper::to_dt(), src1_ptr, DnnlGemmWrapper::to_dt(), dst_dd_i, DnnlGemmWrapper::to_dt(), stream); @@ -2215,21 +3301,15 @@ inline void ggml_sycl_op_mul_mat_sycl( else #endif { - ggml_sycl_pool_alloc dst_f16(ctx.pool(), row_diff * src1_ncols); - - const sycl::half alpha_f16 = 1.0f; - const sycl::half beta_f16 = 0.0f; + const float alpha = 1.0f; + const float beta = 0.0f; SYCL_CHECK(CHECK_TRY_ERROR(dpct::gemm( *stream, oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, row_diff, src1_ncols, ne10, - &alpha_f16, src0_ptr, dpct::library_data_t::real_half, ne00, - src1_ptr, dpct::library_data_t::real_half, ne10, &beta_f16, - dst_f16.get(), dpct::library_data_t::real_half, ldc, - dpct::library_data_t::real_half))); - scope_op_debug_print scope_dbg_print(__func__, "/to_fp32_sycl", dst, /*num_src=*/2, - " : converting dst to fp32"); - const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16, dst); - to_fp32_sycl(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream); + &alpha, src0_ptr, dpct::library_data_t::real_half, ne00, + src1_ptr, dpct::library_data_t::real_half, ne10, &beta, + dst_dd_i, dpct::library_data_t::real_float, ldc, + dpct::library_data_t::real_float))); } } else { ggml_sycl_pool_alloc src0_ddq_as_f32(ctx.pool()); @@ -2253,21 +3333,25 @@ inline void ggml_sycl_op_mul_mat_sycl( const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get(); const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get(); + { #if GGML_SYCL_DNNL - if (!g_ggml_sycl_disable_dnn) { - DnnlGemmWrapper::row_gemm(ctx, row_diff, src1_ncols, ne10, src0_ddf_i, - DnnlGemmWrapper::to_dt(), src1_ddf1_i, DnnlGemmWrapper::to_dt(), - dst_dd_i, DnnlGemmWrapper::to_dt(), stream); - } - else + const int64_t gemm_flops = (int64_t)row_diff * src1_ncols * ne10; + const bool use_mkl_direct = gemm_flops < SYCL_F32_DNNL_MIN_GEMM_FLOPS; + if (g_ggml_sycl_enable_dnn && !use_mkl_direct && ggml_sycl_dnnl_has_optimized_gemm(ggml_sycl_get_device())) { + DnnlGemmWrapper::row_gemm(ctx, row_diff, src1_ncols, ne10, src0_ddf_i, + DnnlGemmWrapper::to_dt(), src1_ddf1_i, DnnlGemmWrapper::to_dt(), + dst_dd_i, DnnlGemmWrapper::to_dt(), stream); + } + else #endif - { - const float alpha = 1.0f; - const float beta = 0.0f; - SYCL_CHECK(CHECK_TRY_ERROR(oneapi::mkl::blas::column_major::gemm( - *stream, oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, row_diff, - src1_ncols, ne10, dpct::get_value(&alpha, *stream), src0_ddf_i, ne00, src1_ddf1_i, ne10, - dpct::get_value(&beta, *stream), dst_dd_i, ldc))); + { + const float alpha = 1.0f; + const float beta = 0.0f; + SYCL_CHECK(CHECK_TRY_ERROR(oneapi::mkl::blas::column_major::gemm( + *stream, oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, row_diff, + src1_ncols, ne10, dpct::get_value(&alpha, *stream), src0_ddf_i, ne00, src1_ddf1_i, ne10, + dpct::get_value(&beta, *stream), dst_dd_i, ldc))); + } } } GGML_UNUSED(dst); @@ -2280,45 +3364,6 @@ catch (sycl::exception const &exc) { std::exit(1); } -static void ggml_sycl_op_pool2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); - dpct::queue_ptr main_stream = ctx.stream(); - SYCL_CHECK(ggml_sycl_set_device(ctx.device)); - const float * src0_dd = static_cast(dst->src[0]->data); - float * dst_dd = static_cast(dst->data); - - const int32_t * opts = (const int32_t *)dst->op_params; - enum ggml_op_pool op = static_cast(opts[0]); - const int k0 = opts[1]; - const int k1 = opts[2]; - const int s0 = opts[3]; - const int s1 = opts[4]; - const int p0 = opts[5]; - const int p1 = opts[6]; - - const int64_t IH = dst->src[0]->ne[1]; - const int64_t IW = dst->src[0]->ne[0]; - - const int64_t N = dst->ne[3]; - const int64_t OC = dst->ne[2]; - const int64_t OH = dst->ne[1]; - const int64_t OW = dst->ne[0]; - - const int parallel_elements = N * OC * OH * OW; - const int num_blocks = (parallel_elements + SYCL_POOL2D_BLOCK_SIZE - 1) / SYCL_POOL2D_BLOCK_SIZE; - sycl::range<3> block_nums(1, 1, num_blocks); - main_stream->parallel_for( - sycl::nd_range<3>(block_nums * - sycl::range<3>(1, 1, SYCL_IM2COL_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_IM2COL_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { - pool2d_nchw_kernel(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, - parallel_elements, src0_dd, dst_dd, op, - item_ct1); - }); -} - inline void ggml_sycl_op_sum(ggml_backend_sycl_context & ctx, ggml_tensor *dst) { GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); @@ -2328,6 +3373,8 @@ inline void ggml_sycl_op_sum(ggml_backend_sycl_context & ctx, ggml_tensor *dst) float * dst_dd = static_cast(dst->data); const int64_t ne = ggml_nelements(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ne <= INT32_MAX); sum_rows_f32_sycl(src0_dd, dst_dd, ne, 1, main_stream); } @@ -2342,6 +3389,8 @@ inline void ggml_sycl_op_sum_rows(ggml_backend_sycl_context & ctx, ggml_tensor * const int64_t ncols = dst->src[0]->ne[0]; const int64_t nrows = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ncols * nrows <= INT32_MAX); sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); } @@ -2358,6 +3407,8 @@ inline void ggml_sycl_op_mean(ggml_backend_sycl_context & ctx, ggml_tensor * dst const int64_t ncols = dst->src[0]->ne[0]; const int64_t nrows = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ncols * nrows <= INT32_MAX); sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); @@ -2385,7 +3436,7 @@ inline void ggml_sycl_op_argsort(ggml_backend_sycl_context & ctx, ggml_tensor * enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0]; argsort_f32_i32_sycl(src0_dd, (int *)dst_dd, ncols, nrows, order, - main_stream, ctx.device); + main_stream, ctx.device, ctx.pool()); } static void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -2406,10 +3457,14 @@ static void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * ds const int64_t ncols = src0->ne[0]; const int64_t nrows = ggml_nrows(src0); - GGML_ASSERT(k > 0 && k <= 32); + GGML_ASSERT(k > 0); GGML_ASSERT(k <= ncols); - top_k_f32_sycl(src0_dd, dst_dd, ncols, nrows, k, main_stream); + if (k <= SYCL_TOP_K_SCAN_MERGE_MAX_K) { + top_k_f32_sycl(ctx, src0_dd, dst_dd, ncols, nrows, k, main_stream); + } else { + ggml_sycl_top_k_radix(ctx, src0_dd, dst_dd, ncols, nrows, k, main_stream); + } } inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -2423,6 +3478,8 @@ inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * d const int64_t ncols = dst->src[0]->ne[0]; const int64_t nrows = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ncols * nrows <= INT32_MAX); argmax_f32_i32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream); } @@ -2438,6 +3495,8 @@ inline void ggml_sycl_op_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_ten const int64_t ne00 = dst->src[0]->ne[0]; const int64_t ne01 = dst->src[0]->ne[1]; const int nrows0 = ggml_nrows(dst->src[0]); + // the kernel indexes in 32-bit + GGML_ASSERT(ne00 * nrows0 <= INT32_MAX); const int n_past = ((int32_t *) dst->op_params)[0]; @@ -2516,6 +3575,9 @@ inline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * ds memcpy(&scale, (float *) dst->op_params + 0, sizeof(float)); memcpy(&bias, (float *) dst->op_params + 1, sizeof(float)); + // the kernel indexes in 32-bit + GGML_ASSERT(ggml_nelements(dst->src[0]) <= INT32_MAX); + scale_f32_sycl(src0_dd, dst_dd, scale, bias, ggml_nelements(dst->src[0]), main_stream); /* DPCT1010:87: SYCL uses exceptions to report errors and does not use the @@ -2524,7 +3586,7 @@ inline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * ds SYCL_CHECK(0); } -static void ggml_sycl_set_peer_access(const int n_tokens, int main_device) { +static void ggml_sycl_set_peer_access(const int n_tokens, [[maybe_unused]] int main_device) { static bool peer_access_enabled = false; const bool enable_peer_access = n_tokens <= GGML_SYCL_PEER_MAX_BATCH_SIZE; @@ -2585,7 +3647,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten GGML_ASSERT(!ggml_backend_buffer_is_sycl_split(dst->buffer)); GGML_ASSERT(!ggml_backend_buffer_is_sycl_split(src1->buffer)); - GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1)); GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0); @@ -2597,7 +3658,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten const size_t q8_1_bs = QK8_1; ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra; - ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra; const bool src0_is_contiguous = ggml_is_contiguous(src0); const bool src1_is_contiguous = ggml_is_contiguous(src1); @@ -2679,7 +3739,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten queue_ptr stream = ctx.stream(i, 0); if (src0_is_contiguous) { - dev[i].src0_dd = (char *) src0->data; + // a split tensor keeps its rows in the per-device slices of the tensor extra + dev[i].src0_dd = split ? (char *) src0_extra->data_device[i] : (char *) src0->data; } else { dev[i].src0_dd = dev[i].src0_dd_alloc.alloc(ctx.pool(i), ggml_nbytes(src0)); } @@ -2753,7 +3814,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten // for split tensors the data begins at i0 == i0_offset_low char * src0_dd_i = dev[i].src0_dd + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs; - float * src1_ddf_i = dev[i].src1_ddf + (i0*ne11 + src1_col_0) * ne10; + float * src1_ddf_i = (float *) ((char *) dev[i].src1_ddf + + (i0*ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type)); char * src1_ddq_i = dev[i].src1_ddq + src1_ddq_i_offset; float * dst_dd_i = dev[i].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff); @@ -2774,12 +3836,13 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten src1_ncols * src1_padded_col_size * q8_1_ts / q8_1_bs) .wait())); } else { - float * src1_ddf_i_source = (float *) src1_extra->data_device[ctx.device]; - src1_ddf_i_source += (i0 * ne11 + src1_col_0) * ne10; + // src1 is not split, on the main device it lives at dev[ctx.device].src1_ddf + const char * src1_ddf_i_source = (const char *) dev[ctx.device].src1_ddf + + (i0 * ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type); SYCL_CHECK( - CHECK_TRY_ERROR(dev2dev_memcpy(*stream, *main_stream, src1_ddf_i, src1_ddf_i_source, - src1_ncols * ne10 * sizeof(float)))); + CHECK_TRY_ERROR(dev2dev_memcpy(i, *stream, ctx.device, *main_stream, src1_ddf_i, src1_ddf_i_source, + src1_ncols * ne10 * ggml_type_size(src1->type)))); } } } else { @@ -2885,6 +3948,11 @@ static void ggml_sycl_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * ds ggml_sycl_op_get_rows(ctx, dst); } +static void ggml_sycl_get_rows_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + ggml_sycl_op_get_rows_back(ctx, dst); +} + static void ggml_sycl_norm(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); ggml_sycl_op_norm(ctx, dst); @@ -3028,7 +4096,9 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons float * dst_ddf = static_cast(dst->data); const sycl::half * src1_f16 = static_cast(src1->data); +#if GGML_SYCL_DNNL const size_t type_size_src0 = ggml_type_size(src0->type); +#endif const size_t type_size_src1 = ggml_type_size(src1->type); bool is_src0_cont_2 = ggml_is_contiguous_2(src0); @@ -3045,6 +4115,7 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_nc_sycl", dst, /*num_src=*/2, " : converting src1 to fp16"); +#if GGML_SYCL_DNNL // iterate tensor dims and find the slowest moving dim and stride int last_dim=0; int last_str=0; @@ -3064,7 +4135,6 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons } } -#if GGML_SYCL_DNNL // oneDNN handles strided data and does not need overhead of ggml_get_to_fp16_nc_sycl const int64_t ne_src1 = src1->nb[last_str] * src1->ne[last_dim] / type_size_src1; src1_f16_alloc.alloc(ne_src1); @@ -3112,7 +4182,7 @@ static void ggml_sycl_mul_mat_batched_sycl(ggml_backend_sycl_context & ctx, cons const int64_t r3 = ne13 / ne03; #if GGML_SYCL_DNNL - if (!g_ggml_sycl_disable_dnn) { + if (g_ggml_sycl_enable_dnn && ggml_sycl_dnnl_has_optimized_gemm(ggml_sycl_get_device())) { int64_t str_a0 = nb00 / type_size_src0; int64_t str_a1 = nb01 / type_size_src0; int64_t str_a2 = nb02 / type_size_src0; @@ -3268,10 +4338,14 @@ inline bool ggml_sycl_supports_mmq(enum ggml_type type) { inline bool ggml_sycl_supports_reorder_mul_mat_sycl(enum ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: return true; + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: return !g_ggml_sycl_prioritize_dmmv; default: @@ -3281,8 +4355,14 @@ inline bool ggml_sycl_supports_reorder_mul_mat_sycl(enum ggml_type type) { inline bool ggml_sycl_supports_reorder_dmmv(enum ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: return true; default: return false; @@ -3291,18 +4371,44 @@ inline bool ggml_sycl_supports_reorder_dmmv(enum ggml_type type) { inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + return true; + default: + return false; + } +} + +static bool ggml_sycl_supports_reorder_esimd(enum ggml_type type) { +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + switch (type) { + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: + case GGML_TYPE_Q8_0: return true; default: return false; } +#else + GGML_UNUSED(type); + return false; +#endif } static bool ggml_sycl_supports_dmmv(enum ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -3314,6 +4420,37 @@ static bool ggml_sycl_supports_dmmv(enum ggml_type type) { case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_F16: + case GGML_TYPE_BF16: + return true; + default: + return false; + } +} + +static bool ggml_sycl_supports_mmvq_id(enum ggml_type type) { + switch (type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: return true; default: return false; @@ -3325,19 +4462,30 @@ static inline void * sycl_ext_malloc_device(dpct::queue_ptr stream, size_t size) bool use_async = g_ggml_sycl_use_async_mem_op; #if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC if (use_async) { - return syclex::async_malloc(*stream, sycl::usm::alloc::device, size); + // async_malloc throws on out-of-memory; return nullptr like the sync path so callers can fall back. + void * ptr = nullptr; + try { + ptr = syclex::async_malloc(*stream, sycl::usm::alloc::device, size); + } catch (const sycl::exception & e) { + GGML_LOG_WARN("%s: async_malloc of %zu bytes failed: %s\n", __func__, size, e.what()); + ggml_sycl_memtrace_fail(GGML_SYCL_MEM_ASYNC, size); + return nullptr; + } + ggml_sycl_memtrace_add(GGML_SYCL_MEM_ASYNC, ptr, size); + return ptr; } #else // If async allocation extension is not available, use_async should always be false. GGML_ASSERT(!use_async); #endif - return sycl::malloc(size, *stream, sycl::usm::alloc::device); + return ggml_sycl_malloc_device(size, *stream); } static inline void sycl_ext_free(dpct::queue_ptr stream, void * ptr) { bool use_async = g_ggml_sycl_use_async_mem_op; #if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC if (use_async) { + ggml_sycl_memtrace_del(ptr); syclex::async_free(*stream, ptr); return; } @@ -3345,7 +4493,7 @@ static inline void sycl_ext_free(dpct::queue_ptr stream, void * ptr) { // If async allocation extension is not available, use_async should always be false. GGML_ASSERT(!use_async); #endif - sycl::free(ptr, *stream); + ggml_sycl_free_device(ptr, *stream); } // RAII wrapper for temporary reorder buffers with optional host memory fallback. @@ -3389,206 +4537,516 @@ struct sycl_reorder_temp_buffer { bool host_fallback = false; }; -static bool reorder_qw_q4_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, - dpct::queue_ptr stream) { - sycl_reorder_temp_buffer tmp(stream, size); +// Reorder a 2D tensor of quantized blocks in place into [head of every block][tail of every block] without a +// full-size temporary copy. The head is the contiguous `head_bytes` at `head_offset` inside each block. Heads only +// move towards the front, so they are packed in chunks that never overwrite unread blocks. The tail overlaps +// blocks that are not packed yet, so `write_tail(x, ib, tail, nblocks)` first writes every block's remaining fields +// (in their final layout, relative to the start of the tail) into a side buffer that is copied to the end afterwards. +template +static bool reorder_qw_planar(const char * name, uint8_t * data_device, size_t size, size_t head_offset, + size_t head_bytes, tail_fn_t write_tail, dpct::queue_ptr stream) { + GGML_ASSERT(size % sizeof(block_t) == 0); + GGML_ASSERT(head_offset + head_bytes <= sizeof(block_t)); + + const int64_t nblocks = size / sizeof(block_t); + const size_t tail_bytes = (sizeof(block_t) - head_bytes) * nblocks; + + sycl_reorder_temp_buffer tail_tmp(stream, tail_bytes); + if (!tail_tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", name, tail_bytes); + return false; + } + uint8_t * tail_buf = static_cast(tail_tmp.ptr); + + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_blocks = std::max(1, std::min(nblocks, max_chunk_bytes / sizeof(block_t))); + sycl_reorder_temp_buffer tmp(stream, chunk_blocks * sizeof(block_t)); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", name, + (size_t) chunk_blocks * sizeof(block_t)); return false; } uint8_t * tmp_buf = static_cast(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); + sycl::event event = stream->parallel_for(nblocks, [=](auto i) { + const int64_t ib = i; + write_tail(((const block_t *) data_device)[ib], ib, tail_buf, nblocks); + }); if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); + event.wait_and_throw(); } - GGML_ASSERT((size % sizeof(block_q4_0) == 0)); - GGML_ASSERT((offset % sizeof(block_q4_0) == 0)); - int offset_blks = offset / sizeof(block_q4_0); - auto qs_ptr = data_device + offset_blks * QK4_0 / 2; - auto d_ptr = (sycl::half*)(qs_ptr + ncols * nrows / 2) + offset_blks; + for (int64_t b0 = 0; b0 < nblocks; b0 += chunk_blocks) { + const int64_t n_chunk = std::min(chunk_blocks, nblocks - b0); - auto reorder_event = stream->parallel_for( - size / sizeof(block_q4_0), - [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - const block_q4_0* x = (const block_q4_0*)tmp_buf; - const int ib = i; + SYCL_CHECK(CHECK_TRY_ERROR(event = stream->memcpy(tmp_buf, data_device + b0 * sizeof(block_t), + n_chunk * sizeof(block_t)))); + if (!g_ggml_sycl_use_async_mem_op) { + event.wait(); + } - for (int j = 0; j < QK4_0/2; j ++) - { - *(qs_ptr + ib * QK4_0 / 2 + j) = x[ib].qs[j]; + uint8_t * head_ptr = data_device + b0 * head_bytes; + event = stream->parallel_for(n_chunk, [=](auto i) { + const int64_t ib = i; + const uint8_t * src = tmp_buf + ib * sizeof(block_t) + head_offset; + for (size_t j = 0; j < head_bytes; ++j) { + head_ptr[ib * head_bytes + j] = src[j]; } - *(d_ptr + ib) = x[ib].d; }); + if (!g_ggml_sycl_use_async_mem_op) { + event.wait_and_throw(); + } + } + + SYCL_CHECK(CHECK_TRY_ERROR(event = stream->memcpy(data_device + head_bytes * nblocks, tail_buf, tail_bytes))); if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); + event.wait(); } return true; } +static bool reorder_qw_q4_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, + dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + GGML_ASSERT((size_t) ncols * nrows / 2 == size / sizeof(block_q4_0) * (QK4_0 / 2)); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q4_0, qs), QK4_0 / 2, + [](const block_q4_0 & x, int64_t ib, uint8_t * tail, int64_t) { + ((sycl::half *) tail)[ib] = x.d; + }, stream); +} + static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, dpct::queue_ptr stream) { - sycl_reorder_temp_buffer tmp(stream, size); + GGML_ASSERT(offset == 0); + GGML_ASSERT((size_t) ncols * nrows == size / sizeof(block_q8_0) * QK8_0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q8_0, qs), QK8_0, + [](const block_q8_0 & x, int64_t ib, uint8_t * tail, int64_t) { + ((sycl::half *) tail)[ib] = x.d; + }, stream); +} + +static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q4_K, qs), QK_K / 2, + [](const block_q4_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * scales = tail; + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales[ib * K_SCALE_SIZE + j] = x.scales[j]; + } + ((sycl::half2 *) (scales + K_SCALE_SIZE * nblocks))[ib] = x.dm; + }, stream); +} + +// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout, a few experts at a time. +static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0); + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max(1, std::min(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); + + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - GGML_ASSERT((size % sizeof(block_q8_0) == 0)); - GGML_ASSERT((offset % sizeof(block_q8_0) == 0)); - int offset_blks = offset / sizeof(block_q8_0); - auto qs_ptr = data_device + offset_blks * QK8_0; - auto d_ptr = (sycl::half*)(qs_ptr + ncols * nrows) + offset_blks; + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } - auto reorder_event = stream->parallel_for( - size / sizeof(block_q8_0), - [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - const block_q8_0* x = (const block_q8_0*)tmp_buf; - const int ib = i; + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q8_0 * x = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - for (int j = 0; j < QK8_0; j++) - { - *((int8_t*)qs_ptr + ib * QK8_0 + j) = x[ib].qs[j]; + auto * qs_ptr = base; + auto * d_ptr = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0); + + for (int j = 0; j < QK8_0; ++j) { + *((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j]; } - *(d_ptr + ib) = x[ib].d; + d_ptr[ib] = x[ib].d; }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); + } } return true; } -static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q4_K) == 0); - GGML_ASSERT(offset % sizeof(block_q4_K) == 0); - - const int nblocks = size / sizeof(block_q4_K); +// Reorder each expert slice into a self-contained SoA layout. +// Like the MXFP4 version below, this goes a few experts at a time through a small temporary: +// a whole-tensor temporary (GBs for large MoE models) can exceed the VRAM left on a nearly full card. +static bool reorder_qw_q4_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + GGML_ASSERT(expert_bytes % sizeof(block_q4_K) == 0); + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q4_K)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max(1, std::min(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, size); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); - } + // the queue is in-order: each chunk's copy into tmp_buf waits for the previous chunk's kernel + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - auto * qs_ptr = data_device; - auto * scales_ptr = qs_ptr + QK_K / 2 * nblocks; - auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * nblocks); + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q4_K * x = (const block_q4_K *) tmp_buf; - const int ib = i; + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q4_K * x = (const block_q4_K *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; - for (int j = 0; j < QK_K / 2; ++j) { - qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; - } + auto * qs_ptr = base; + auto * scales_ptr = qs_ptr + QK_K / 2 * blocks_per_expert; + auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); - for (int j = 0; j < K_SCALE_SIZE; ++j) { - scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + for (int j = 0; j < QK_K / 2; ++j) { + qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; + } + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + } + dm_ptr[ib] = x[ib].dm; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } - - dm_ptr[ib] = x[ib].dm; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); } return true; } -static bool reorder_qw_q6_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { - GGML_ASSERT(size % sizeof(block_q6_K) == 0); - GGML_ASSERT(offset % sizeof(block_q6_K) == 0); - - const int nblocks = size / sizeof(block_q6_K); +// Reorder each Q5_K expert slice into [qs][qh][scales][dm], a few experts at a time (see Q4_K above). +static bool reorder_qw_q5_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + GGML_ASSERT(expert_bytes % sizeof(block_q5_K) == 0); + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q5_K)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max(1, std::min(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - sycl_reorder_temp_buffer tmp(stream, size); + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); if (!tmp) { - GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); return false; } uint8_t * tmp_buf = static_cast(tmp.ptr); - sycl::event copy_event; - SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); - if (!g_ggml_sycl_use_async_mem_op) { - copy_event.wait(); + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; + + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } + + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q5_K * x = (const block_q5_K *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; + + auto * qs_ptr = base; + auto * qh_ptr = qs_ptr + (QK_K / 2) * blocks_per_expert; + auto * scales_ptr = qh_ptr + (QK_K / 8) * blocks_per_expert; + auto * dm_ptr = (sycl::half2 *) (scales_ptr + K_SCALE_SIZE * blocks_per_expert); + + for (int j = 0; j < QK_K / 2; ++j) { + qs_ptr[ib * (QK_K / 2) + j] = x[ib].qs[j]; + } + for (int j = 0; j < QK_K / 8; ++j) { + qh_ptr[ib * (QK_K / 8) + j] = x[ib].qh[j]; + } + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales_ptr[ib * K_SCALE_SIZE + j] = x[ib].scales[j]; + } + dm_ptr[ib] = x[ib].dm; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); + } } + return true; +} - auto * ql_ptr = data_device; - auto * qh_ptr = ql_ptr + (QK_K / 2) * nblocks; - auto * scales_ptr = qh_ptr + (QK_K / 4) * nblocks; - sycl::half * dm_ptr = (sycl::half *) (scales_ptr + (QK_K / 16) * nblocks); +// Reorder each Q6_K expert slice into [ql][qh][scales][d], a few experts at a time (see Q4_K above). +static bool reorder_qw_q6_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + GGML_ASSERT(expert_bytes % sizeof(block_q6_K) == 0); + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q6_K)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max(1, std::min(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); - auto reorder_event = stream->parallel_for(nblocks, [=](auto i) { - const block_q6_K * x = (const block_q6_K *) tmp_buf; - const int ib = i; + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); + if (!tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); + return false; + } + uint8_t * tmp_buf = static_cast(tmp.ptr); - const uint8_t * ql = x[ib].ql; - const uint8_t * qh = x[ib].qh; - uint8_t * base_ql_ptr = ql_ptr + (QK_K / 2) * ib; - uint8_t * base_qh_ptr = qh_ptr + (QK_K / 4) * ib; - uint8_t * base_scales_ptr = scales_ptr + (QK_K / 16) * ib; + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; - for (int j = 0; j < QK_K / 2; ++j) { - base_ql_ptr[j] = ql[j]; + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); } - for (int j = 0; j < QK_K / 4; ++j) { - base_qh_ptr[j] = qh[j]; + + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q6_K * x = (const block_q6_K *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; + + auto * ql_ptr = base; + auto * qh_ptr = ql_ptr + (QK_K / 2) * blocks_per_expert; + auto * scales_ptr = qh_ptr + (QK_K / 4) * blocks_per_expert; + auto * d_ptr = (sycl::half *) (scales_ptr + (QK_K / 16) * blocks_per_expert); + + for (int j = 0; j < QK_K / 2; ++j) { + ql_ptr[ib * (QK_K / 2) + j] = x[ib].ql[j]; + } + for (int j = 0; j < QK_K / 4; ++j) { + qh_ptr[ib * (QK_K / 4) + j] = x[ib].qh[j]; + } + for (int j = 0; j < QK_K / 16; ++j) { + scales_ptr[ib * (QK_K / 16) + j] = x[ib].scales[j]; + } + d_ptr[ib] = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); } + } + return true; +} - for (int j = 0; j < QK_K / 16; ++j) { - base_scales_ptr[j] = x[ib].scales[j]; +// Reorder each MXFP4 expert slice into [qs][e]: 16-byte nibble blocks, then one E8M0 byte per block. +// Experts are self-contained, so the tensor is reordered a few experts at a time through a small +// temporary: a whole-tensor temporary (hundreds of MB) can exceed the VRAM left on a nearly full card, +// and on Windows the driver then pages device memory out to host RAM instead of failing. +static bool reorder_qw_mxfp4_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + GGML_ASSERT(expert_bytes % sizeof(block_mxfp4) == 0); + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_mxfp4)); + const size_t max_chunk_bytes = 32u << 20; + const int64_t chunk_experts = std::max(1, std::min(n_expert, (int64_t) (max_chunk_bytes / expert_bytes))); + + sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes); + if (!tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, + (size_t) chunk_experts * expert_bytes); + return false; + } + uint8_t * tmp_buf = static_cast(tmp.ptr); + + // the queue is in-order: each chunk's copy into tmp_buf waits for the previous chunk's kernel + for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) { + const int64_t n_chunk = std::min(chunk_experts, n_expert - e0); + uint8_t * chunk = data_device + (size_t) e0 * expert_bytes; + + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); } - dm_ptr[ib] = x[ib].d; - }); - if (!g_ggml_sycl_use_async_mem_op) { - reorder_event.wait_and_throw(); + const int total_blocks = blocks_per_expert * (int) n_chunk; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_mxfp4 * x = (const block_mxfp4 *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = chunk + (size_t) e * expert_bytes; + + uint8_t * qs_ptr = base; + uint8_t * e_ptr = qs_ptr + (QK_MXFP4 / 2) * (size_t) blocks_per_expert; + + for (int j = 0; j < QK_MXFP4 / 2; ++j) { + qs_ptr[(size_t) ib * (QK_MXFP4 / 2) + j] = x[ib].qs[j]; + } + e_ptr[ib] = x[ib].e; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); + } } return true; } +static bool reorder_qw_q2_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q2_K, qs), QK_K / 4, + [](const block_q2_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * scales = tail; + for (int j = 0; j < QK_K / 16; ++j) { + scales[ib * (QK_K / 16) + j] = x.scales[j]; + } + ((sycl::half2 *) (scales + (QK_K / 16) * nblocks))[ib] = x.dm; + }, stream); +} + +static bool reorder_qw_q3_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q3_K, qs), QK_K / 4, + [](const block_q3_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * hmask = tail; + uint8_t * scales = hmask + (QK_K / 8) * nblocks; + for (int j = 0; j < QK_K / 8; ++j) { + hmask[ib * (QK_K / 8) + j] = x.hmask[j]; + } + for (int j = 0; j < 12; ++j) { + scales[ib * 12 + j] = x.scales[j]; + } + ((sycl::half *) (scales + 12 * nblocks))[ib] = x.d; + }, stream); +} + +static bool reorder_qw_q5_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q5_K, qs), QK_K / 2, + [](const block_q5_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * qh = tail; + uint8_t * scales = qh + (QK_K / 8) * nblocks; + for (int j = 0; j < QK_K / 8; ++j) { + qh[ib * (QK_K / 8) + j] = x.qh[j]; + } + for (int j = 0; j < K_SCALE_SIZE; ++j) { + scales[ib * K_SCALE_SIZE + j] = x.scales[j]; + } + ((sycl::half2 *) (scales + K_SCALE_SIZE * nblocks))[ib] = x.dm; + }, stream); +} + +static bool reorder_qw_q6_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { + GGML_ASSERT(offset == 0); + return reorder_qw_planar(__func__, data_device, size, offsetof(block_q6_K, ql), QK_K / 2, + [](const block_q6_K & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * qh = tail; + uint8_t * scales = qh + (QK_K / 4) * nblocks; + for (int j = 0; j < QK_K / 4; ++j) { + qh[ib * (QK_K / 4) + j] = x.qh[j]; + } + for (int j = 0; j < QK_K / 16; ++j) { + scales[ib * (QK_K / 16) + j] = x.scales[j]; + } + ((sycl::half *) (scales + (QK_K / 16) * nblocks))[ib] = x.d; + }, stream); +} + +static bool reorder_qw_iq3_s(uint8_t * data_device, size_t size, dpct::queue_ptr stream) { + return reorder_qw_planar(__func__, data_device, size, offsetof(block_iq3_s, qs), QK_K / 4, + [](const block_iq3_s & x, int64_t ib, uint8_t * tail, int64_t nblocks) { + uint8_t * qh = tail; + uint8_t * signs = qh + (QK_K / 32) * nblocks; + uint8_t * metadata = signs + (QK_K / 8) * nblocks + ib * (sizeof(ggml_half) + IQ3S_N_SCALE); + for (int j = 0; j < QK_K / 32; ++j) { + qh[ib * (QK_K / 32) + j] = x.qh[j]; + } + for (int j = 0; j < QK_K / 8; ++j) { + signs[ib * (QK_K / 8) + j] = x.signs[j]; + } + *reinterpret_cast(metadata) = x.d; + for (int j = 0; j < IQ3S_N_SCALE; ++j) { + metadata[sizeof(ggml_half) + j] = x.scales[j]; + } + }, stream); +} + +static bool reorder_qw_iq3_xxs(uint8_t * data_device, size_t size, dpct::queue_ptr stream) { + return reorder_qw_planar(__func__, data_device, size, offsetof(block_iq3_xxs, qs), 3 * QK_K / 8, + [](const block_iq3_xxs & x, int64_t ib, uint8_t * tail, int64_t) { + reinterpret_cast(tail)[ib] = x.d; + }, stream); +} + static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { uint8_t * data_device = (uint8_t *) src0->data; size_t ncols = src0->ne[0]; size_t nrows = src0->ne[1]; size_t size = ggml_nbytes(src0); + // MoE expert weights are addressed per expert via nb[2], so each slice must + // remain self-contained after reorder. + if (src0->ne[2] > 1) { + GGML_ASSERT((size_t) size == (size_t) src0->ne[2] * src0->nb[2]); + switch (src0->type) { + case GGML_TYPE_Q8_0: + return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream); + case GGML_TYPE_Q4_K: + return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream); + case GGML_TYPE_Q5_K: + return reorder_qw_q5_k_moe(data_device, src0->nb[2], src0->ne[2], stream); + case GGML_TYPE_Q6_K: + return reorder_qw_q6_k_moe(data_device, src0->nb[2], src0->ne[2], stream); + case GGML_TYPE_MXFP4: + return reorder_qw_mxfp4_moe(data_device, src0->nb[2], src0->ne[2], stream); + default: + return false; + } + } + switch (src0->type) { case GGML_TYPE_Q4_0: return reorder_qw_q4_0(data_device, ncols, nrows, size, 0, stream); case GGML_TYPE_Q8_0: return reorder_qw_q8_0(data_device, ncols, nrows, size, 0, stream); + case GGML_TYPE_Q2_K: + return reorder_qw_q2_k(data_device, size, 0, stream); + case GGML_TYPE_Q3_K: + return reorder_qw_q3_k(data_device, size, 0, stream); case GGML_TYPE_Q4_K: return reorder_qw_q4_k(data_device, size, 0, stream); + case GGML_TYPE_Q5_K: + return reorder_qw_q5_k(data_device, size, 0, stream); case GGML_TYPE_Q6_K: return reorder_qw_q6_k(data_device, size, 0, stream); + case GGML_TYPE_IQ3_XXS: + return reorder_qw_iq3_xxs(data_device, size, stream); + case GGML_TYPE_IQ3_S: + return reorder_qw_iq3_s(data_device, size, stream); default: - GGML_ABORT("reorder_qw() called with unsupported type"); return false; } } static bool should_reorder_tensor(ggml_backend_sycl_context& ctx, const ggml_tensor * dst) { - return !g_ggml_sycl_disable_optimize && //allow optimize, controlled by $GGML_SYCL_DISABLE_OPT - ctx.opt_feature.reorder && //allow this device due to good perf, skip the devices with bad perf. - dst->op == GGML_OP_MUL_MAT && //limit to some supported cases of Q4_0, to do for more cases. - dst->src[1]->ne[1]==1 && dst->src[1]->ne[2]==1 && dst->src[1]->ne[3]==1; + return g_ggml_sycl_enable_optimize && //allow optimize, controlled by $GGML_SYCL_ENABLE_OPT + ctx.opt_feature.reorder && //allow this device due to good perf, skip the devices with bad perf. + dst->op == GGML_OP_MUL_MAT && //limit to some supported cases of Q4_0, to do for more cases. + // ne[1] <= 8 so multi-column decode (spec / MTP verify) also bootstraps the reorder; + // all reorderable types have a _switch_ncols kernel. + dst->src[1]->ne[1] <= 8 && dst->src[1]->ne[2]==1 && dst->src[1]->ne[3]==1; } static void opt_for_reorder(ggml_backend_sycl_context * ctx, const ggml_tensor * src0, const ggml_tensor * /* src1 */, @@ -3625,19 +5083,87 @@ static void opt_for_reorder(ggml_backend_sycl_context * ctx, const ggml_tensor * } } - -static bool can_use_dequantize_mul_mat_vec(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - return ggml_sycl_supports_dmmv(src0->type) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && - src0->ne[0] % GGML_SYCL_DMMV_X == 0 && src1->ne[1] == 1; -} - -static bool can_use_mul_mat_vec_q(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +// Lazily reorder supported MoE expert weights once their fused path is used. +static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tensor * src0) { + if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) { + return; + } + if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q4_K && + src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) { + return; + } + // The MXFP4 reorder kernels use 8-byte vector loads, so every expert slice must stay aligned. + if (src0->type == GGML_TYPE_MXFP4 && (src0->nb[2] % 16 != 0 || (uintptr_t) src0->data % 16 != 0)) { + return; + } + ggml_tensor_extra_gpu * extra = static_cast(src0->extra); + if (!extra || extra->optimized_feature.reorder) { + return; + } + if (reorder_qw(src0, ctx->stream())) { + extra->optimized_feature.reorder = true; + } +} + + +static bool can_use_dequantize_mul_mat_vec(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + // The F16/BF16 qk=1 kernel iterates with stride 2*DMMV_X, requiring ne[0] to be + // a multiple of 2*DMMV_X. Quantized types use block-structured kernels that only + // need ne[0] % DMMV_X == 0. + const int64_t dmmv_x_required = (src0->type == GGML_TYPE_BF16 || src0->type == GGML_TYPE_F16) ? + 2*GGML_SYCL_DMMV_X : GGML_SYCL_DMMV_X; + return ggml_sycl_supports_dmmv(src0->type) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && + src0->ne[0] % dmmv_x_required == 0 && src1->ne[1] == 1; +} + +// reordered weights of the types in ggml_sycl_xmx_supports_type() take the XMX kernel for wider batches +static bool can_use_xmx_batch(int device, const ggml_tensor * src0, const ggml_tensor * src1) { +#ifdef GGML_SYCL_MMVQ_HAS_XMX + const auto * extra = static_cast(src0->extra); + return ggml_sycl_xmx_enabled(device) && ggml_sycl_xmx_supports_type(src0->type) && extra && + extra->optimized_feature.reorder && src1->ne[1] <= g_ggml_sycl_xmx_max_cols && src1->ne[2] == 1 && + src1->ne[3] == 1; +#else + GGML_UNUSED(device); + GGML_UNUSED(src0); + GGML_UNUSED(src1); + return false; +#endif // GGML_SYCL_MMVQ_HAS_XMX +} + +static bool can_use_mul_mat_vec_q(int device, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { return ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && - src1->ne[1] <= MMVQ_MAX_BATCH_SIZE; + (src1->ne[1] <= MMVQ_MAX_BATCH_SIZE || can_use_xmx_batch(device, src0, src1)); +} + +static bool can_use_mul_mat_q(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + // mmvq and mmq need the __dp4a instruction which is available for gen12+ + // Workaround in https://github.com/ggml-org/llama.cpp/commit/95f84d5ce8b449a9b16009434aca800df504a02e + if (!ggml_sycl_supports_mmq(src0->type) || src0->type == GGML_TYPE_IQ2_XXS) { + return false; + } +#ifdef SYCL_USE_XMX + if (src1->ne[1] > MMQ_MAX_BATCH_SIZE) { + return false; + } +#endif // SYCL_USE_XMX + return src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; } static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + // Handle HADAMARAD hint given from further up the pipeline and pass it to the correct + // kernel. + // + // The op check is not redundant: this backend also routes MUL_MAT_ID through here with a + // stack copy of dst, which carries MUL_MAT_ID's own op_params. ggml_mul_mat_set_hint() + // asserts GGML_OP_MUL_MAT for the same reason. + if (dst->op == GGML_OP_MUL_MAT && ggml_get_op_params_i32(dst, 1) == GGML_HINT_SRC0_IS_HADAMARD && + ggml_sycl_op_fwht(ctx, src1, dst)) { + return; + } + const bool split = ggml_backend_buffer_is_sycl_split(src0->buffer); int64_t min_compute_capability = INT_MAX; @@ -3661,26 +5187,34 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor // check data types and tensor shapes for custom matrix multiplication kernels: bool use_dequantize_mul_mat_vec = can_use_dequantize_mul_mat_vec(src0, src1, dst); + if (src1->ne[1] == 1 && ggml_sycl_xmx_min_cols(src0->type) == 1 && can_use_xmx_batch(ctx.device, src0, src1)) { + use_dequantize_mul_mat_vec = false; + } +#ifdef GGML_SYCL_F16 + // dmmv may convert src1 to f16 in this build [TAG_GGML_PREC] + use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && ggml_sycl_src1_f16_ok(dst); +#endif - bool use_mul_mat_vec_q = can_use_mul_mat_vec_q(src0, src1, dst); - - bool use_mul_mat_q = ggml_sycl_supports_mmq(src0->type) - && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32; + bool use_mul_mat_vec_q = can_use_mul_mat_vec_q(ctx.device, src0, src1, dst); + bool use_mul_mat_q = can_use_mul_mat_q(src0, src1, dst); - // mmvq and mmq need the __dp4a instruction which is available for gen12+ - // Workaround in https://github.com/ggml-org/llama.cpp/commit/95f84d5ce8b449a9b16009434aca800df504a02e - use_mul_mat_q = use_mul_mat_q && (src0->type != GGML_TYPE_IQ2_XXS); -#ifdef SYCL_USE_XMX - use_mul_mat_q = use_mul_mat_q && (src1->ne[1] <= MMQ_MAX_BATCH_SIZE); -#endif // SYCL_USE_XMX + // When reorder is enabled, both ESIMD, MMVQ and DMMV kernels may be used. For + // best performance use ESIMD when supported, followed by MMVQ, and finally DMMV. + // But the reordered ESIMD path cannot be used without reordered MMVQ. A later + // multi-token call (ne[1] in 2..8) will take the MMVQ path and it would read the + // reordered bytes as if they were still the unreordered layout. - // Dispatch becomes obscure with the reorder, MMVQ when the reorder optimization - // is enabled takes precedence over DMMV, the current if-else implementation - // requires disabling DMMV if both conditions are met if (!g_ggml_sycl_prioritize_dmmv && ((should_reorder_tensor(ctx, dst) && ggml_sycl_supports_reorder_mmvq(src0->type)))) { - use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && !use_mul_mat_vec_q; + bool use = g_ggml_sycl_enable_esimd && ggml_sycl_supports_reorder_esimd(src0->type); + // Arc770 get benefit with Q4_0 by skipping MMVQ path + if (!(ggml_sycl_info().devices[ctx.device].hw_info.arch == + gpu_arch::intel_gpu_acm_g10 && + src0->type == GGML_TYPE_Q4_0)) { + use = use || !use_mul_mat_vec_q; + } + use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && use; } if (!split && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) { @@ -3717,43 +5251,303 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor } } +// {mul_mat(gate), mul_mat(up), GLU} over the standard (non-reorder) weight layout, +// for quant pairs the reorder kernel does not cover (mixed gate/up types, e.g. UD-Q4_K_XL's +// iq4_xs gate + q5_K up). Two launches replace five: one shared q8_1 quantization and one +// dual-GEMV+GLU. +static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, ggml_tensor * glu, + ggml_tensor * gate, ggml_tensor * up, const ggml_tensor * wu, + const ggml_tensor * wg, const ggml_tensor * act) { + // weights already migrated to the reorder layout would be misread by the plain kernel + const auto * extra_u = static_cast(wu->extra); + const auto * extra_g = static_cast(wg->extra); + if ((extra_u && extra_u->optimized_feature.reorder) || (extra_g && extra_g->optimized_feature.reorder)) { + return false; + } + + // the plain kernel only covers q5_K / iq4_xs pairs: bail before quantizing the activation + const auto plain_type = [](ggml_type t) { return t == GGML_TYPE_Q5_K || t == GGML_TYPE_IQ4_XS; }; + if (!plain_type(wu->type) || !plain_type(wg->type)) { + return false; + } + + // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises + scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (plain layout)"); + + const int64_t ne00 = wu->ne[0]; + const int64_t ne11 = act->ne[1]; + + const queue_ptr stream = ctx.stream(); + const int src1_padded_cols = GGML_PAD((int) ne00, MATRIX_ROW_PADDING); + + ggml_sycl_pool_alloc src1_q8_alloc(ctx.pool(), + (size_t) ne11 * src1_padded_cols * sizeof(block_q8_1) / QK8_1); + char * src1_ddq = src1_q8_alloc.get(); + + quantize_row_q8_1_sycl((const float *) act->data, src1_ddq, (int) ne00, (int) ne11, + src1_padded_cols, stream); + + return ggml_sycl_mul_mat_vec_q_glu_plain(wg->type, wu->type, ggml_get_glu_op(glu), wg->data, wu->data, + src1_ddq, (float *) glu->data, (int) ne00, (int) wu->ne[1], + (int) ne11, + /*stride_col_y=*/src1_padded_cols / QK8_1, + /*stride_col_dst=*/(int) glu->ne[0], stream); + GGML_UNUSED(gate); +} + +// Gate and up weights of one type on the XMX engines: one launch reads the activations once and applies the GLU. +// Both weights must already be in the reorder layout. Returns false if the kernel declined. +static bool ggml_sycl_mul_mat_glu_xmx(ggml_backend_sycl_context & ctx, + ggml_tensor * glu, + ggml_tensor * up, + const ggml_tensor * wu, + const ggml_tensor * wg, + const ggml_tensor * act) { + const int64_t ne00 = wu->ne[0]; + const int64_t ne11 = act->ne[1]; + + const queue_ptr stream = ctx.stream(); + const int src1_padded_cols = GGML_PAD((int) ne00, MATRIX_ROW_PADDING); + const int stride_y_bytes = src1_padded_cols * (int) sizeof(block_q8_1) / QK8_1; + + // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises + scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (XMX)"); + + ggml_sycl_pool_alloc src1_q8_alloc(ctx.pool(), (size_t) ne11 * stride_y_bytes); + char * src1_ddq = src1_q8_alloc.get(); + + quantize_row_q8_1_sycl((const float *) act->data, src1_ddq, (int) ne00, (int) ne11, + src1_padded_cols, stream); + + return ggml_sycl_mul_mat_vec_q_glu_xmx(ctx.device, wu->type, ggml_get_glu_op(glu), wu->data, wg->data, src1_ddq, + (float *) glu->data, (int) ne00, (int) wu->ne[1], (int) ne11, stride_y_bytes, + /*stride_col_dst=*/(int) glu->ne[0], stream); +} + +#ifdef GGML_SYCL_DMMV_HAS_ESIMD +// {mul_mat(gate), mul_mat(up), GLU} on the reorder layout via the ESIMD DMMV kernel, for +// same-type K-quant or Q8_0 pairs and a single activation column. It reads the F32 activation +// directly, so one launch replaces three and no q8_1 quantization is needed. +static bool ggml_sycl_mul_mat_glu_dmmv_esimd(ggml_backend_sycl_context & ctx, ggml_tensor * glu, + ggml_tensor * gate, ggml_tensor * up, const ggml_tensor * wu, + const ggml_tensor * wg, const ggml_tensor * act) { + // the kernel reads the activation while writing the output; glu may reuse the dead activation's memory + const char * act_begin = (const char *) act->data; + const char * glu_begin = (const char *) glu->data; + if (act_begin < glu_begin + ggml_nbytes(glu) && glu_begin < act_begin + ggml_nbytes(act)) { + return false; + } + + // install the reorder (SoA) layout the kernel needs, as the unfused DMMV path would + opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::DMMV); + opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::DMMV); + + const auto * extra_u = static_cast(wu->extra); + const auto * extra_g = static_cast(wg->extra); + if (!extra_u || !extra_g || !extra_u->optimized_feature.reorder || !extra_g->optimized_feature.reorder) { + return false; + } + + // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises + scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (ESIMD)"); + + return ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(wu->type, ggml_get_glu_op(glu), wu->data, wg->data, + (const float *) act->data, (float *) glu->data, + (int) wu->ne[0], (int) wu->ne[1], ctx.stream()); +} +#endif + +// Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx. +// Returns false if it declined, in which case the caller runs the three nodes normally. +static bool ggml_sycl_mul_mat_glu_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) { + if (!ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) { + return false; + } + + ggml_tensor * glu = cgraph->nodes[node_idx + 2]; + ggml_tensor * gate = glu->src[0]; + ggml_tensor * up = glu->src[1]; + const ggml_tensor * wu = up->src[0]; + const ggml_tensor * wg = gate->src[0]; + const ggml_tensor * act = up->src[1]; + + // this writes glu->data directly rather than the per-device row slices that + // ggml_sycl_op_mul_mat() stitches back together, so it cannot serve split weights + if (ggml_backend_buffer_is_sycl_split(wu->buffer) || ggml_backend_buffer_is_sycl_split(wg->buffer)) { + return false; + } + +#ifdef GGML_SYCL_DMMV_HAS_ESIMD + // single-column same-type K-quant or Q8_0 pairs take the ESIMD kernel, as the unfused mul-mats + // would via DMMV; if the reorder layout could not be installed, fall through to mmvq + if (g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 && + ggml_sycl_supports_reorder_esimd(wu->type) && + ggml_sycl_mul_mat_glu_dmmv_esimd(ctx, glu, gate, up, wu, wg, act)) { + return true; + } +#endif + + // with DMMV prioritised the unfused path would not have gone through mmvq at all + if (g_ggml_sycl_prioritize_dmmv) { + return false; + } + + // gate and up of one type the XMX kernel handles, it needs the reorder layout like the unfused mmvq path + if (wg->type == wu->type && gate->src[1] == act && act->ne[1] >= ggml_sycl_xmx_min_cols(wu->type) && + act->ne[1] <= g_ggml_sycl_xmx_glu_max_cols) { + opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::MMVQ); + opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::MMVQ); + if (can_use_xmx_batch(ctx.device, wu, act) && can_use_xmx_batch(ctx.device, wg, act) && + ggml_sycl_mul_mat_glu_xmx(ctx, glu, up, wu, wg, act)) { + return true; + } + } + + // the kernels below take up to MMVQ_MAX_BATCH_SIZE columns, a wider batch got here for the XMX kernel + if (act->ne[1] > MMVQ_MAX_BATCH_SIZE) { + return false; + } + + // quant pairs the reorder kernel does not serve (mixed gate/up types, q5_K off BMG) take the + // standard-layout fused path instead; same-type q4_K / q5_K keep the reorder path below + const bool reorder_pair = wg->type == wu->type && + (wu->type == GGML_TYPE_Q4_K || (wu->type == GGML_TYPE_Q5_K && ggml_sycl_q5_k_mmvq_reuse(ctx.device))); + if (!reorder_pair) { + const auto is_plain_type = [](ggml_type type) { + return type == GGML_TYPE_Q5_K || type == GGML_TYPE_IQ4_XS; + }; + if (!is_plain_type(wu->type) || !is_plain_type(wg->type)) { + return false; + } + return ggml_sycl_mul_mat_glu_mmvq_plain(ctx, glu, gate, up, wu, wg, act); + } + + // past 5 columns the two unfused q5_K GEMVs are faster than the fused kernel + if (wu->type == GGML_TYPE_Q5_K && act->ne[1] > 5) { + return false; + } + + // install the reorder (SoA) layout the fused kernel needs, as the unfused mmvq path would; + // a no-op once done. after the bail checks so a declined op does not pay for it. + opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::MMVQ); + opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::MMVQ); + + const auto * extra_u = static_cast(wu->extra); + const auto * extra_g = static_cast(wg->extra); + if (!extra_u || !extra_g || !extra_u->optimized_feature.reorder || !extra_g->optimized_feature.reorder) { + return false; + } + + // log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises + scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU"); + + const int64_t ne00 = wu->ne[0]; + const int64_t ne11 = act->ne[1]; + + const queue_ptr stream = ctx.stream(); + const int src1_padded_cols = GGML_PAD((int) ne00, MATRIX_ROW_PADDING); + + // one activation, quantized once and fully consumed into src1_ddq before the GEMV on this + // in-order queue, so glu->data aliasing the dead activation needs no memory-range check + ggml_sycl_pool_alloc src1_q8_alloc(ctx.pool(), + (size_t) ne11 * src1_padded_cols * sizeof(block_q8_1) / QK8_1); + char * src1_ddq = src1_q8_alloc.get(); + + quantize_row_q8_1_sycl((const float *) act->data, src1_ddq, (int) ne00, (int) ne11, + src1_padded_cols, stream); + + return ggml_sycl_mul_mat_vec_q_glu_reorder(wu->type, ggml_get_glu_op(glu), wu->data, wg->data, src1_ddq, + (float *) glu->data, (int) ne00, (int) wu->ne[1], (int) ne11, + /*stride_col_y_bytes=*/src1_padded_cols * (int) sizeof(block_q8_1) / + QK8_1, + /*stride_col_dst=*/(int) glu->ne[0], stream); +} + +// Batch the run of consecutive L2_NORM siblings starting at node_idx into one launch. +// Returns the number of extra graph nodes consumed, or 0 if the run is shorter than two +// (the caller then runs the norm through the per-tensor kernel). +static int ggml_sycl_l2_norm_batch_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) { + const ggml_tensor * node = cgraph->nodes[node_idx]; + if (ggml_sycl_info().device_count != 1 || node->type != GGML_TYPE_F32 || + node->src[0]->type != GGML_TYPE_F32 || node->src[0]->ne[0] >= 1024) { + return 0; + } + + ggml_tensor * batch[GGML_SYCL_L2_BATCH_MAX]; + int count = 0; + int last = node_idx; + float eps0; + memcpy(&eps0, node->op_params, sizeof(float)); + + // Conservative aliasing test: the batched norms run concurrently in one kernel, + // so none may read what another writes, and none may write where another writes. + auto overlaps = [](const ggml_tensor * a, const ggml_tensor * b) { + const char * ab = (const char *) a->data; + const char * bb = (const char *) b->data; + return ab < bb + ggml_nbytes(b) && bb < ab + ggml_nbytes(a); + }; + + for (int j = node_idx; j < cgraph->n_nodes && count < GGML_SYCL_L2_BATCH_MAX; ++j) { + ggml_tensor * nj = cgraph->nodes[j]; + if (ggml_is_empty(nj) || nj->op == GGML_OP_RESHAPE || nj->op == GGML_OP_TRANSPOSE || + nj->op == GGML_OP_VIEW || nj->op == GGML_OP_PERMUTE || nj->op == GGML_OP_NONE || + (nj->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; // not a launch; cannot break a run of adjacent norms + } + if (nj->op != GGML_OP_L2_NORM || nj->type != GGML_TYPE_F32 || + nj->src[0]->type != GGML_TYPE_F32 || !ggml_are_same_shape(nj, node) || + !ggml_are_same_shape(nj->src[0], node->src[0])) { + break; // any other launch ends the run + } + bool same_nb = true; + for (int d = 0; d < GGML_MAX_DIMS; ++d) { + if (nj->nb[d] != node->nb[d] || nj->src[0]->nb[d] != node->src[0]->nb[d]) { + same_nb = false; + break; + } + } + if (!same_nb) { + break; // one nb[] stride set is shared by the whole batch + } + float epsj; + memcpy(&epsj, nj->op_params, sizeof(float)); + if (epsj != eps0) { + break; // eps mismatch ends the run + } + bool indep = true; + for (int k = 0; k < count; ++k) { + if (overlaps(nj->src[0], batch[k]) || overlaps(nj, batch[k])) { + indep = false; + break; + } + } + if (!indep) { + break; // an overlapping tensor would race inside one launch + } + batch[count++] = nj; + last = j; + } + if (count < 2) { + return 0; // a lone norm falls through to the per-tensor kernel + } + ggml_sycl_l2_norm_batch(ctx, batch, count); + return last - node_idx; +} -struct mmid_row_mapping { - int32_t i1; - int32_t i2; -}; __dpct_inline__ static void k_copy_src1_to_contiguous( const char *__restrict__ src1_original, char *__restrict__ src1_contiguous, - int *__restrict__ cur_src1_row, mmid_row_mapping *__restrict__ row_mapping, - const char *__restrict ids, int64_t i02, size_t ids_nb1, size_t ids_nb0, + const mmid_row_mapping *__restrict__ row_mapping, int64_t ne11, int64_t ne10, size_t nb11, size_t nb12, - const sycl::nd_item<3> &item_ct1, int &src1_row) { - int32_t iid1 = item_ct1.get_group(2); - int32_t id = item_ct1.get_group(1); - - const int32_t row_id_i = *(const int32_t *) (ids + iid1*ids_nb1 + id*ids_nb0); + const sycl::nd_item<3> &item_ct1) { + const int32_t src1_row = item_ct1.get_group(2); - if (row_id_i != i02) { - return; - } + const int32_t iid1 = row_mapping[src1_row].i2; + const int32_t id = row_mapping[src1_row].i1; const int64_t i11 = id % ne11; const int64_t i12 = iid1; - if (item_ct1.get_local_id(2) == 0) { - src1_row = - dpct::atomic_fetch_add( - cur_src1_row, 1); - row_mapping[src1_row] = {id, iid1}; - } - /* - DPCT1065:194: Consider replacing sycl::nd_item::barrier() with - sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better - performance if there is no access to global memory. - */ - item_ct1.barrier(); - const float * src1_row_original = (const float *)(src1_original + i11*nb11 + i12*nb12); float * src1_row_contiguous = (float *)(src1_contiguous + src1_row*nb11); @@ -3783,6 +5577,154 @@ __dpct_inline__ static void k_copy_dst_from_contiguous( } } +__dpct_inline__ static void k_zero_dst_rows( + char *__restrict__ dst_original, + const mmid_row_mapping *__restrict__ row_mapping, int64_t ne0, size_t nb1, + size_t nb2, const sycl::nd_item<3> &item_ct1) { + int32_t i = item_ct1.get_group(2); + + const int32_t i1 = row_mapping[i].i1; + const int32_t i2 = row_mapping[i].i2; + + float * dst_row_original = (float *)(dst_original + i1*nb1 + i2*nb2); + + for (int j = item_ct1.get_local_id(2); j < ne0; + j += item_ct1.get_local_range(2)) { + dst_row_original[j] = 0.0f; + } +} + +// returns true when ggml_sycl_mul_mat_id takes the fallback path that requires stream synchronization +static bool ggml_sycl_mul_mat_id_needs_sync(const ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * ids = dst->src[2]; + + const int64_t ne10 = src1->ne[0]; + const int64_t ne11 = src1->ne[1]; + const int64_t ne12 = src1->ne[2]; + + if (ne12 != 1) return true; + if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) return true; + if (ne10 != src0->ne[0] || ne10 % QK8_1 != 0) return true; + if (!ggml_is_contiguous(src1)) return true; + + const int64_t n_ids_per_group = ids->ne[0]; + if (ids->ne[1] != 1) return true; + if (ne11 != 1 && ne11 != n_ids_per_group) return true; + + return !ggml_sycl_supports_mmvq_id(src0->type); +} + +// Fused MoE TG fast path. Returns false to fall back to the per-expert loop below. +static bool ggml_sycl_mul_mat_id_mmvq_fused( + ggml_backend_sycl_context & ctx, const ggml_tensor * src0, + const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) +{ + if (ggml_sycl_mul_mat_id_needs_sync(dst)) return false; + + const int64_t ne10 = src1->ne[0]; + const int64_t ne11 = src1->ne[1]; + + const int64_t n_ids_per_group = ids->ne[0]; + + const queue_ptr stream = ctx.stream(); + const int src1_padded_cols = GGML_PAD((int) ne10, MATRIX_ROW_PADDING); + const int n_experts_used = (int) n_ids_per_group; + const int nrows = (int) src0->ne[1]; + + // Lazily reorder supported expert weights into a per-expert SoA layout, then run the reorder + // GEMV. Placed after the bail checks so a non-dispatchable op does not pay the reorder cost. + opt_for_reorder_id(&ctx, src0); + const ggml_tensor_extra_gpu * src0_extra = + static_cast(src0->extra); + const bool use_reorder = src0_extra && src0_extra->optimized_feature.reorder; + + ggml_sycl_pool_alloc src1_q8_alloc(ctx.pool(), + (size_t) ne11 * src1_padded_cols * sizeof(block_q8_1) / QK8_1); + char * src1_ddq = src1_q8_alloc.get(); + if (use_reorder) { + quantize_row_q8_1_sycl( + (const float *) src1->data, src1_ddq, (int) ne10, (int) ne11, + src1_padded_cols, stream); + } else { + quantize_row_q8_1_sycl( + (const float *) src1->data, src1_ddq, (int) ne10, (int) ne11, + src1_padded_cols, stream); + } + + const size_t bytes_per_qrow = (size_t) src1_padded_cols * sizeof(block_q8_1) / QK8_1; + const size_t src1_row_stride = (ne11 == 1) ? 0 : bytes_per_qrow; + + if (use_reorder) { + return ggml_sycl_mul_mat_vec_q_id_reorder( + src0->type, src0->data, src1_ddq, (const int32_t *) ids->data, + (float *) dst->data, (int) ne10, nrows, n_experts_used, + /*expert_weight_stride=*/ src0->nb[2], + /*dst_row_stride=*/ dst->nb[1], + src1_row_stride, stream); + } + return ggml_sycl_mul_mat_vec_q_id( + src0->type, src0->data, src1_ddq, (const int32_t *) ids->data, + (float *) dst->data, (int) ne10, nrows, n_experts_used, + /*expert_weight_stride=*/ src0->nb[2], + /*dst_row_stride=*/ dst->nb[1], + src1_row_stride, stream); +} + +// counting sort of the routed rows by expert id (row_id_i, as chosen by the router): +// builds a projection of a memory layout where each expert's slice is contiguous +static void mmid_counting_sort_rows( + const ggml_tensor * ids, const char * ids_host, + int64_t n_ids, int64_t n_as, int64_t n_routed_rows, + std::vector & expert_counts, + std::vector & expert_row_offsets, + std::vector & routed_row_src, + std::vector & skipped_row_dst) { + + // frequencies: how many routed rows each expert "owns" + expert_counts.assign(n_as, 0); + for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) { + for (int64_t id = 0; id < n_ids; id++) { + const int32_t row_id_i = *(const int32_t *) (ids_host + iid1*ids->nb[1] + id*ids->nb[0]); + GGML_ASSERT(row_id_i == -1 || (row_id_i >= 0 && row_id_i < n_as)); + if (row_id_i == -1) { + continue; + } + expert_counts[row_id_i]++; + } + } + + // where each expert's slice starts (row indices) and the previous ends + expert_row_offsets.assign(n_as + 1, 0); + for (int64_t i02 = 0; i02 < n_as; i02++) { + expert_row_offsets[i02 + 1] = expert_row_offsets[i02] + expert_counts[i02]; + } + + std::vector expert_row_next = expert_row_offsets; + routed_row_src.resize(n_routed_rows); + skipped_row_dst.clear(); + for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) { + for (int64_t id = 0; id < n_ids; id++) { + const int32_t row_id_i = *(const int32_t *) (ids_host + iid1*ids->nb[1] + id*ids->nb[0]); + GGML_ASSERT(row_id_i == -1 || (row_id_i >= 0 && row_id_i < n_as)); + + // a skipped slot has no expert, its dst row is zeroed instead + if (row_id_i == -1) { + skipped_row_dst.push_back({(int32_t) id, (int32_t) iid1}); + continue; + } + + // find and validate the next free row for a given expert (row_id_i) + const int64_t routed_row = expert_row_next[row_id_i]++; + GGML_ASSERT(routed_row >= expert_row_offsets[row_id_i]); + GGML_ASSERT(routed_row < expert_row_offsets[row_id_i + 1]); + routed_row_src[routed_row] = {(int32_t) id, (int32_t) iid1}; + } + } + routed_row_src.resize(expert_row_offsets[n_as]); +} + static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, ggml_tensor *dst) try { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3); @@ -3798,11 +5740,27 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, const int64_t n_as = ne02; const int64_t n_ids = ids->ne[0]; + if (ne12 == 1) { + if (ggml_sycl_mul_mat_id_mmvq_fused(ctx, src0, src1, ids, dst)) { + return; + } + } + GGML_ASSERT(ggml_sycl_mul_mat_id_needs_sync(dst)); + + // The per-expert loop below reads the experts in whatever layout they have: reorder MXFP4 here as well, so prompt processing does not depend on a single-token decode having run first. + if (src0->type == GGML_TYPE_MXFP4) { + opt_for_reorder_id(&ctx, src0); + } + std::vector ids_host(ggml_nbytes(ids)); const char * ids_dev = (const char *) ids->data; SYCL_CHECK(CHECK_TRY_ERROR( stream->memcpy(ids_host.data(), ids_dev, ggml_nbytes(ids)))); + + // also ensures ctx.mmid_row_mapping_host and ctx.mmid_tile_schedule_host are drained before we + // refill them: the grouped GEMM enqueues an async copy out of the tile schedule, so removing + // this wait would let the next node overwrite a buffer the device is still reading SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); ggml_tensor src0_row = *src0; @@ -3832,7 +5790,7 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) { for (int64_t id = 0; id < n_ids; id++) { const int32_t i02 = *(const int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]); - GGML_ASSERT(i02 >= 0 && i02 < n_as); + GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as)); const int64_t i11 = id % ne11; const int64_t i12 = iid1; @@ -3840,6 +5798,12 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, const int64_t i1 = id; const int64_t i2 = i12; + if (i02 == -1) { // skipped slot + SYCL_CHECK(CHECK_TRY_ERROR( + stream->memset(dst_original + i1*nb1 + i2*nb2, 0, ne0*sizeof(float)))); + continue; + } + src0_row.data = src0_original + i02*nb02; src1_row.data = src1_original + i11*nb11 + i12*nb12; dst_row.data = dst_original + i1*nb1 + i2*nb2; @@ -3848,105 +5812,139 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, } } } else { - ggml_sycl_pool_alloc src1_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(src1)); - ggml_sycl_pool_alloc dst_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(dst)); + const int64_t n_routed_rows = ids->ne[1] * n_ids; + ggml_sycl_pool_alloc src1_contiguous(ctx.pool(), sizeof(float)*n_routed_rows*ne10); + ggml_sycl_pool_alloc dst_contiguous(ctx.pool(), sizeof(float)*n_routed_rows*ne0); src1_row.data = src1_contiguous.get(); dst_row.data = dst_contiguous.get(); - for (int64_t i02 = 0; i02 < n_as; i02++) { - int64_t num_src1_rows = 0; - for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) { - for (int64_t id = 0; id < n_ids; id++) { - const int32_t row_id_i = *(const int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]); - - GGML_ASSERT(row_id_i >= 0 && row_id_i < n_as); + // how many "owned" routed rows to pass to each expert + std::vector expert_row_counts; + // where each expert's slice starts and the previous ends (row indices, right-exclusive) + std::vector expert_row_offsets; + // the sources (slot/token pairs) of contiguous rows to guide k_copy_src1_to_contiguous + std::vector & routed_row_src = ctx.mmid_row_mapping_host; + // the slot/token pairs of the skipped slots, whose dst rows are zeroed + std::vector skipped_row_dst; - if (row_id_i != i02) { - continue; - } + mmid_counting_sort_rows(ids, ids_host.data(), n_ids, n_as, n_routed_rows, + expert_row_counts, expert_row_offsets, routed_row_src, skipped_row_dst); - num_src1_rows++; - } - } + const int64_t n_valid_rows = (int64_t) routed_row_src.size(); - if (num_src1_rows == 0) { - continue; - } + ggml_sycl_pool_alloc dev_row_mapping(ctx.pool(), n_routed_rows); + SYCL_CHECK(CHECK_TRY_ERROR( + stream->memcpy(dev_row_mapping.get(), routed_row_src.data(), n_valid_rows*sizeof(mmid_row_mapping)))); + const unsigned int max_work_group_size = ggml_sycl_info().max_work_group_sizes[ctx.device]; + assert(max_work_group_size % (WARP_SIZE * WARP_SIZE) == 0); - ggml_sycl_pool_alloc dev_cur_src1_row(ctx.pool(), 1); - ggml_sycl_pool_alloc dev_row_mapping(ctx.pool(), num_src1_rows); + if (!skipped_row_dst.empty()) { + ggml_sycl_pool_alloc dev_skipped(ctx.pool(), skipped_row_dst.size()); SYCL_CHECK(CHECK_TRY_ERROR( - stream->memset(dev_cur_src1_row.get(), 0, sizeof(int)))); + stream->memcpy(dev_skipped.get(), skipped_row_dst.data(), skipped_row_dst.size()*sizeof(mmid_row_mapping)))); + + sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne0, max_work_group_size)); + sycl::range<3> grid_dims(1, 1, skipped_row_dst.size()); + stream->submit([&](sycl::handler &cgh) { + mmid_row_mapping *__restrict dev_skipped_get = dev_skipped.get(); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_zero_dst_rows(dst_original, dev_skipped_get, ne0, nb1, nb2, item_ct1); + }); + }); + SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); + } - const unsigned int max_work_group_size = ggml_sycl_info().max_work_group_sizes[ctx.device]; - assert(max_work_group_size % (WARP_SIZE * WARP_SIZE) == 0); + if (n_valid_rows > 0) { + sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne10, max_work_group_size)); + sycl::range<3> grid_dims(1, 1, n_valid_rows); + stream->submit([&](sycl::handler &cgh) { + char *__restrict src1_contiguous_get = + src1_contiguous.get(); + mmid_row_mapping *__restrict dev_row_mapping_get = + dev_row_mapping.get(); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_copy_src1_to_contiguous( + src1_original, src1_contiguous_get, + dev_row_mapping_get, + ne11, ne10, nb11, nb12, + item_ct1); + }); + }); + } - { - sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne10, max_work_group_size)); - sycl::range<3> grid_dims(1, n_ids, ids->ne[1]); - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor src1_row_acc(cgh); - - char *__restrict src1_contiguous_get = - src1_contiguous.get(); - int *__restrict dev_cur_src1_row_get = - dev_cur_src1_row.get(); - mmid_row_mapping *__restrict dev_row_mapping_get = - dev_row_mapping.get(); - size_t ids_nb_ct6 = ids->nb[1]; - size_t ids_nb_ct7 = ids->nb[0]; - - cgh.parallel_for( - sycl::nd_range<3>(grid_dims * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - k_copy_src1_to_contiguous( - src1_original, src1_contiguous_get, - dev_cur_src1_row_get, - dev_row_mapping_get, ids_dev, i02, - ids_nb_ct6, ids_nb_ct7, ne11, ne10, nb11, nb12, - item_ct1, src1_row_acc); - }); - }); + bool grouped = false; + if (ggml_is_contiguous(src0) && src1->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F32 && nb11 == sizeof(float)*ne10 && nb1 == sizeof(float)*ne0) { + // the grouped GEMM reads the reorder (SoA) layout faster, and the first decode step installs it + // anyway: install it here already, so prefill does not depend on whether a decode ran before + if (g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 && + ggml_sycl_xmx_gather_type_enabled(src0->type)) { + opt_for_reorder_id(&ctx, src0); + } + const bool src0_reordered = + src0->extra && ((const ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + grouped = ggml_sycl_grouped_dequant_gemm(src0->type, src0_reordered, src0_original, nb02, + (const float *) src1_contiguous.get(), ggml_sycl_src1_prec(dst), + (float *) dst_contiguous.get(), + expert_row_offsets.data(), n_as, ne01, ne10, n_routed_rows, + ctx.mmid_tile_schedule_host, ctx.pool(), stream); + } + + for (int64_t i02 = 0; i02 < n_as && !grouped; i02++) { + const int64_t num_src1_rows = expert_row_counts[i02]; + + if (num_src1_rows == 0) { + continue; } + const int64_t expert_row_offset = expert_row_offsets[i02]; + src0_row.data = src0_original + i02*nb02; GGML_ASSERT(nb11 == sizeof(float)*ne10); GGML_ASSERT(nb1 == sizeof(float)*ne0); + src1_row.data = src1_contiguous.get() + expert_row_offset*nb11; src1_row.ne[1] = num_src1_rows; src1_row.nb[1] = nb11; src1_row.nb[2] = num_src1_rows*nb11; src1_row.nb[3] = num_src1_rows*nb11; + dst_row.data = dst_contiguous.get() + expert_row_offset*nb1; dst_row.ne[1] = num_src1_rows; dst_row.nb[1] = nb1; dst_row.nb[2] = num_src1_rows*nb1; dst_row.nb[3] = num_src1_rows*nb1; ggml_sycl_mul_mat(ctx, &src0_row, &src1_row, &dst_row); + } - { - sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne0, max_work_group_size)); - sycl::range<3> grid_dims(1, 1, num_src1_rows); - stream->submit([&](sycl::handler &cgh) { - const char *__restrict dst_contiguous_get = - dst_contiguous.get(); - const mmid_row_mapping *__restrict dev_row_mapping_get = - dev_row_mapping.get(); - - cgh.parallel_for( - sycl::nd_range<3>(grid_dims * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - k_copy_dst_from_contiguous(dst_original, - dst_contiguous_get, - dev_row_mapping_get, - ne0, nb1, nb2, item_ct1); - }); - }); - } + if (n_valid_rows > 0) { + sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne0, max_work_group_size)); + sycl::range<3> grid_dims(1, 1, n_valid_rows); + stream->submit([&](sycl::handler &cgh) { + const char *__restrict dst_contiguous_get = + dst_contiguous.get(); + const mmid_row_mapping *__restrict dev_row_mapping_get = + dev_row_mapping.get(); + + cgh.parallel_for( + sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + k_copy_dst_from_contiguous(dst_original, + dst_contiguous_get, + dev_row_mapping_get, + ne0, nb1, nb2, item_ct1); + }); + }); } } } @@ -3971,11 +5969,31 @@ static void ggml_sycl_pool2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) ggml_sycl_op_pool2d(ctx, dst); } +static void ggml_sycl_pool1d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); + ggml_sycl_op_pool1d(ctx, dst); +} + static void ggml_sycl_im2col(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); ggml_sycl_op_im2col(ctx, dst); } +static void ggml_sycl_im2col_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + ggml_sycl_op_im2col_3d(ctx, dst); +} + +static void ggml_sycl_col2im_1d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); + ggml_sycl_op_col2im_1d(ctx, dst); +} + +static void ggml_sycl_conv_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + ggml_sycl_op_conv_3d(ctx, dst); +} + static void ggml_sycl_sum(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); GGML_ASSERT(ggml_is_contiguous(dst->src[0])); @@ -4029,6 +6047,7 @@ catch (sycl::exception const &exc) { } static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tensor * dst) try { + GGML_SYCL_DEBUG("[SYCL] ggml_sycl_compute_forward: dst=%s, op=%s\n", dst->name, ggml_op_name(dst->op)); if (!g_sycl_loaded) return false; if (dst->src[0] != nullptr && ggml_backend_buffer_is_sycl_split(dst->src[0]->buffer)) { @@ -4039,9 +6058,21 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_ARGMAX: ggml_sycl_argmax(ctx, dst); break; + case GGML_OP_CONV_2D: + ggml_sycl_op_conv2d(ctx, dst); + break; + case GGML_OP_CONV_2D_DW: + ggml_sycl_op_conv2d_dw(ctx, dst); + break; + case GGML_OP_CONV_3D: + ggml_sycl_conv_3d(ctx, dst); + break; case GGML_OP_CONV_TRANSPOSE_1D: ggml_sycl_op_conv_transpose_1d(ctx, dst); break; + case GGML_OP_CONV_TRANSPOSE_2D: + ggml_sycl_op_conv2d_transpose(ctx, dst); + break; case GGML_OP_REPEAT: ggml_sycl_repeat(ctx, dst); break; @@ -4051,12 +6082,27 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_GET_ROWS: ggml_sycl_get_rows(ctx, dst); break; + case GGML_OP_GET_ROWS_BACK: + ggml_sycl_get_rows_back(ctx, dst); + break; case GGML_OP_SET: ggml_sycl_op_set(ctx, dst); break; case GGML_OP_SET_ROWS: ggml_sycl_op_set_rows(ctx, dst); break; + case GGML_OP_DSV4_HC_PRE: + ggml_sycl_op_dsv4_hc_pre(ctx, dst); + break; + case GGML_OP_DSV4_HC_COMB: + ggml_sycl_op_dsv4_hc_comb(ctx, dst); + break; + case GGML_OP_DSV4_HC_POST: + ggml_sycl_op_dsv4_hc_post(ctx, dst); + break; + case GGML_OP_LIGHTNING_INDEXER: + ggml_sycl_op_lightning_indexer(ctx, dst); + break; case GGML_OP_DUP: ggml_sycl_dup(ctx, dst); break; @@ -4123,6 +6169,9 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_UNARY_OP_EXP: ggml_sycl_exp(ctx, dst); break; + case GGML_UNARY_OP_EXPM1: + ggml_sycl_expm1(ctx, dst); + break; case GGML_UNARY_OP_SOFTPLUS: ggml_sycl_softplus(ctx, dst); break; @@ -4135,6 +6184,9 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_UNARY_OP_ELU: ggml_sycl_elu(ctx, dst); break; + case GGML_UNARY_OP_XIELU: + ggml_sycl_xielu(ctx, dst); + break; case GGML_UNARY_OP_FLOOR: ggml_sycl_floor(ctx, dst); break; @@ -4165,6 +6217,9 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_GLU_OP_SWIGLU_OAI: ggml_sycl_swiglu_oai(ctx, dst); break; + case GGML_GLU_OP_SWIGLU_CLAMP: + ggml_sycl_swiglu_clamp(ctx, dst); + break; case GGML_GLU_OP_GEGLU_ERF: ggml_sycl_geglu_erf(ctx, dst); break; @@ -4264,6 +6319,12 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_SOFT_MAX_BACK: ggml_sycl_op_soft_max_back(ctx, dst); break; + case GGML_OP_CROSS_ENTROPY_LOSS: + ggml_sycl_cross_entropy_loss(ctx, dst); + break; + case GGML_OP_CROSS_ENTROPY_LOSS_BACK: + ggml_sycl_cross_entropy_loss_back(ctx, dst); + break; case GGML_OP_ROPE: ggml_sycl_rope(ctx, dst); break; @@ -4273,9 +6334,18 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_IM2COL: ggml_sycl_im2col(ctx, dst); break; + case GGML_OP_IM2COL_3D: + ggml_sycl_im2col_3d(ctx, dst); + break; + case GGML_OP_COL2IM_1D: + ggml_sycl_col2im_1d(ctx, dst); + break; case GGML_OP_POOL_2D: ggml_sycl_pool2d(ctx, dst); break; + case GGML_OP_POOL_1D: + ggml_sycl_pool1d(ctx, dst); + break; case GGML_OP_SUM: ggml_sycl_sum(ctx, dst); break; @@ -4306,9 +6376,30 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_GATED_DELTA_NET: ggml_sycl_gated_delta_net(ctx, dst); break; + case GGML_OP_OPT_STEP_ADAMW: + ggml_sycl_opt_step_adamw(ctx, dst); + break; + case GGML_OP_OPT_STEP_SGD: + ggml_sycl_opt_step_sgd(ctx, dst); + break; case GGML_OP_SSM_CONV: ggml_sycl_ssm_conv(ctx, dst); break; + case GGML_OP_SSM_SCAN: + ggml_sycl_ssm_scan(ctx, dst); + break; + case GGML_OP_FILL: + ggml_sycl_fill(ctx, dst); + break; + case GGML_OP_CUMSUM: + ggml_sycl_cumsum(ctx, dst); + break; + case GGML_OP_DIAG: + ggml_sycl_diag(ctx, dst); + break; + case GGML_OP_SOLVE_TRI: + ggml_sycl_solve_tri(ctx, dst); + break; case GGML_OP_ROLL: ggml_sycl_roll(ctx, dst); break; @@ -4343,18 +6434,33 @@ catch (sycl::exception const &exc) { std::exit(1); } -void ggml_backend_sycl_get_device_memory(int device, size_t *free, - size_t *total) try { - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_get_device_memory\n"); - ggml_sycl_set_device(device); +static bool sycl_get_mem_info(int device, size_t * free, size_t * total) { + GGML_SYCL_DEBUG("[SYCL] [%s] g_ggml_sycl_get_mem_api=%d\n", + __func__, g_ggml_sycl_get_mem_api); - SYCL_CHECK(CHECK_TRY_ERROR( - dpct::dev_mgr::instance().get_device(device).get_memory_info(*free, *total))); + MemoryAPIType mem_api_type = MemoryAPIType::MEMORY_API_TYPE_SYCL; + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + mem_api_type = get_zes_init_res() == ZE_RESULT_SUCCESS ? + (MemoryAPIType) g_ggml_sycl_get_mem_api : MemoryAPIType::MEMORY_API_TYPE_SYCL; +#else + mem_api_type = MemoryAPIType::MEMORY_API_TYPE_SYCL; +#endif + bool res = get_memory_size(dpct::dev_mgr::instance().get_device(device), + *free, *total, mem_api_type); + GGML_SYCL_DEBUG("[SYCL] [%s] total = %zu free = %zu\n", __func__, *total, *free); + return res; } -catch (sycl::exception const &exc) { - std::cerr << exc.what() << "Exception caught at file:" << __FILE__ - << ", line:" << __LINE__ << std::endl; - std::exit(1); + +void ggml_backend_sycl_get_device_memory(int device, size_t * free, size_t * total) try { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_get_device_memory\n"); + if (!sycl_get_mem_info(device, free, total)) { + GGML_ABORT("[%s] failed to get device memory size", __func__); + } + ggml_sycl_memtrace_report_device("device memory query", device, *free, *total); +} catch (const sycl::exception & exc) { + std::cerr << exc.what() << "Exception caught at file:" << __FILE__ << ", line:" << __LINE__ << std::endl; + std::exit(1); } //////////////////////////////////////////////////////////////////////////////// @@ -4461,17 +6567,99 @@ catch (sycl::exception const &exc) { std::exit(1); } -static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { - ggml_sycl_set_main_device(sycl_ctx->device); +static bool ggml_sycl_is_view_or_noop(const ggml_tensor * t) { + return ggml_is_empty(t) || t->op == GGML_OP_RESHAPE || t->op == GGML_OP_TRANSPOSE || + t->op == GGML_OP_VIEW || t->op == GGML_OP_PERMUTE || t->op == GGML_OP_NONE; +} + +// match gated_delta_net + the strided cpy that scatters its state snapshots into the cache +// (slot i -> rollback group i, slot 0 newest), so the kernel can write them and skip the cpy. +// returns the number of following nodes to skip (0 = no fusion) +// ported from ggml_cuda_try_gdn_cache_fusion - pure graph inspection, backend-agnostic +static int ggml_sycl_try_gdn_cache_fusion(const ggml_cgraph * cgraph, int node_idx, + ggml_sycl_gated_delta_net_fused_cache & fused_state_cpy) { + if (!g_ggml_sycl_enable_fusion) { + return 0; + } + + const ggml_tensor * gdn = cgraph->nodes[node_idx]; + // the kernel skips the snapshot tail, so the gdn output must not be a graph output, and the cpy + // found below is taken to be its only reader, as it is in every graph that builds this op + if (gdn->op != GGML_OP_GATED_DELTA_NET || gdn->type != GGML_TYPE_F32 || + (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + const ggml_tensor * src_v = gdn->src[2]; + const int64_t S_v = src_v->ne[0]; + const int64_t H = src_v->ne[1]; + const int64_t n_tokens = src_v->ne[2]; + const int64_t n_seqs = src_v->ne[3]; + const int64_t D = S_v * S_v * H; + const int64_t K = ggml_get_op_params_i32(gdn, 0); // snapshot slot count + const int64_t n_written = std::min(n_tokens, K); // newest n_written slots are written + + // snapshot tail starts right after the attention scores + const size_t tail_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + + // the cpy must be the first node the compute loop below runs, so nothing can read the cache first. + // skip exactly what that loop skips: views, no-ops, and nodes the graph does not compute. + const ggml_tensor * cpy = nullptr; + int skip = 0; + for (int j = node_idx + 1; j < cgraph->n_nodes && cpy == nullptr; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_sycl_is_view_or_noop(n) || (n->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; + } + if (n->op != GGML_OP_CPY || (n->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + cpy = n; + skip = j - node_idx; + } + if (cpy == nullptr) { + return 0; + } + + const ggml_tensor * src = cpy->src[0]; // view of the gdn snapshot tail + const ggml_tensor * dst = cpy->src[1]; // cache view the kernel writes to + + // src must be this gdn's snapshot tail (contiguous, at the tail offset) + if (src->op != GGML_OP_VIEW || src->view_src != gdn || src->view_offs != tail_off || + !ggml_is_contiguous(src)) { + return 0; + } + + // dst is the [D, n_seqs, n_written] cache view, with the per-seq stride D that the kernel assumes. + // ggml_cpy pins src to the same element count, so src needs no shape check of its own. + const std::array expected_ne = { D, n_seqs, n_written, 1 }; + if (dst->op != GGML_OP_VIEW || dst->type != GGML_TYPE_F32 || dst->data == nullptr || + !std::equal(expected_ne.begin(), expected_ne.end(), dst->ne) || + dst->nb[0] != ggml_type_size(GGML_TYPE_F32) || + dst->nb[1] != (size_t) ggml_row_size(GGML_TYPE_F32, D)) { + return 0; + } + + fused_state_cpy.data = (float *) dst->data; // rollback group 0 (newest) + fused_state_cpy.slot_stride = K > 1 ? (int64_t) (dst->nb[2] / sizeof(float)) : 0; + return skip; +} + +static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_sycl_is_view_or_noop(node)) { continue; } if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { continue; } + + const int nodes_to_skip = ggml_sycl_fuse(*sycl_ctx, cgraph, i); + if (nodes_to_skip != 0) { + i += nodes_to_skip; + continue; + } #ifndef NDEBUG assert(node->buffer->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device)); for (int j = 0; j < GGML_MAX_SRC; j++) { @@ -4480,6 +6668,89 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc } } #endif + // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache + if (node->op == GGML_OP_GATED_DELTA_NET) { + ggml_sycl_gated_delta_net_fused_cache fused_state_cpy; + const int gdn_nodes_to_skip = ggml_sycl_try_gdn_cache_fusion(cgraph, i, fused_state_cpy); + if (gdn_nodes_to_skip > 0) { + ggml_sycl_op_gated_delta_net_fused_cache(*sycl_ctx, node, fused_state_cpy); + i += gdn_nodes_to_skip; + continue; + } + } + if (node->op == GGML_OP_RMS_NORM && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) { + ggml_sycl_op_rms_norm_fused_add(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]); + i += 2; + continue; + } + if (node->op == GGML_OP_RMS_NORM && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL }, {})) { + ggml_sycl_op_rms_norm_fused(*sycl_ctx, node, cgraph->nodes[i + 1]); + i++; + continue; + } + // qwen35 GDN l2 norms are emitted as rms_norm + scalar scale (models.h + // build_gdn_l2_norm), which the rms_norm+mul fusion above cannot match + if (node->op == GGML_OP_RMS_NORM && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_SCALE }, {})) { + ggml_sycl_op_rms_norm_scale_fused(*sycl_ctx, node, cgraph->nodes[i + 1]); + i++; + continue; + } + if (node->op == GGML_OP_ADD && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_ADD }, {})) { + ggml_sycl_op_add_add_fused(*sycl_ctx, node, cgraph->nodes[i + 1]); + i++; + continue; + } + if (node->op == GGML_OP_UNARY && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { ggml_get_unary_op(node) })) { + ggml_sycl_op_unary_mul_fused(*sycl_ctx, node, cgraph->nodes[i + 1]); + i++; + continue; + } + // ADD(bias) + UNARY + MUL(scale) with both broadcast over dim 0, the form the branch + // above cannot take; ggml_get_unary_op() asserts, so check the op first. + if (node->op == GGML_OP_ADD && i + 2 < cgraph->n_nodes && + cgraph->nodes[i + 1]->op == GGML_OP_UNARY && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_MUL }, + { ggml_get_unary_op(cgraph->nodes[i + 1]) })) { + ggml_sycl_op_add_unary_mul_fused(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]); + i += 2; + continue; + } + + // Batch consecutive independent same-shape F32 L2_NORM siblings (the GDN q/k + // norms) into one launch; sources are strided views of the fused qkv buffer, so + // the scan skips the interleaved view nodes instead of breaking on them. + if (node->op == GGML_OP_L2_NORM) { + const int l2_batch_skip = ggml_sycl_l2_norm_batch_fused(*sycl_ctx, cgraph, i); + if (l2_batch_skip > 0) { + i += l2_batch_skip; + continue; + } + } + + if (node->op == GGML_OP_SSM_CONV && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_SSM_CONV, GGML_OP_ADD, GGML_OP_UNARY }, { GGML_UNARY_OP_SILU })) { + ggml_sycl_ssm_conv_fused(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]); + i += 2; + continue; + } + + if (node->op == GGML_OP_SSM_CONV && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_SSM_CONV, GGML_OP_UNARY }, { GGML_UNARY_OP_SILU })) { + ggml_sycl_ssm_conv_fused(*sycl_ctx, node, nullptr, cgraph->nodes[i + 1]); + i++; + continue; + } + + if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_fused(*sycl_ctx, cgraph, i)) { + i += 2; + continue; + } + bool ok = ggml_sycl_compute_forward(*sycl_ctx, node); if (!ok) { GGML_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); @@ -4489,42 +6760,167 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc } #ifdef GGML_SYCL_GRAPH -static bool check_graph_compatibility(ggml_cgraph * cgraph) { - if (ggml_sycl_info().device_count > 1) { +static const void * ggml_sycl_graph_get_key(ggml_cgraph * cgraph) { + return cgraph->nodes[0]; +} + +static bool ggml_sycl_graph_update_required(ggml_sycl_graph * graph, ggml_cgraph * cgraph) { + bool res = false; + + if (cgraph->uid != 0 && cgraph->uid == graph->uid) { + GGML_SYCL_DEBUG("[SYCL-GRAPH] graph id %" PRIu64 " reused\n", cgraph->uid); + GGML_ASSERT((int) graph->node_props.size() == cgraph->n_nodes); + return false; + } + + graph->uid = cgraph->uid; + + if ((int) graph->node_props.size() != cgraph->n_nodes) { + res = true; + graph->node_props.resize(cgraph->n_nodes); + } + + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_sycl_graph::node_properties prop = {}; + memcpy(&prop.node, cgraph->nodes[i], sizeof(ggml_tensor)); + + for (int j = 0; j < GGML_MAX_SRC; ++j) { + if (cgraph->nodes[i]->src[j]) { + prop.node_src_data_ptrs[j] = cgraph->nodes[i]->src[j]->data; + memcpy(prop.node_src_ne[j], cgraph->nodes[i]->src[j]->ne, sizeof(prop.node_src_ne[j])); + memcpy(prop.node_src_nb[j], cgraph->nodes[i]->src[j]->nb, sizeof(prop.node_src_nb[j])); + } + } + + if (res || memcmp(&graph->node_props[i], &prop, sizeof(prop)) != 0) { + graph->node_props[i] = prop; + res = true; + } + } + + return res; +} + +// Reports if ggml_sycl_mul_mat() gives this node to oneDNN. oneDNN chains the submission on events +// made before recording started, which SYCL graphs do not allow. +static bool mul_mat_uses_onednn(int device, ggml_tensor * dst) { +#if GGML_SYCL_DNNL + if (!g_ggml_sycl_enable_dnn) { + return false; + } + + ggml_tensor * src0 = dst->src[0]; + ggml_tensor * src1 = dst->src[1]; + + // the branch order below follows the dispatch in ggml_sycl_mul_mat(); the F16 branches end in + // a vector kernel or the batched oneMKL path, neither of which uses oneDNN + if (!ggml_backend_buffer_is_sycl_split(src0->buffer) && src0->type == GGML_TYPE_F16) { + if (ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) { + return false; + } + if (!ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1 && src1->ne[3] == 1) { + return false; + } + if (!ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2] * src1->ne[3] > 1) { + return false; + } + } + + // ggml_sycl_op_mul_mat_sycl() is the last resort of the dispatch and the only user of oneDNN. + // The reorder decision that ggml_sycl_mul_mat() makes between DMMV and MMVQ is left out here: + // neither uses a library. + if (can_use_dequantize_mul_mat_vec(src0, src1, dst) || can_use_mul_mat_vec_q(device, src0, src1, dst) || + can_use_mul_mat_q(src0, src1, dst)) { + return false; + } + + // the branch order below follows ggml_sycl_op_mul_mat_sycl(); graphs are single device only, so + // row_diff == src0->ne[1] and src1_ncols == src1->ne[1] there +#if defined(GGML_SYCL_HAS_BF16) + if (src0->type == GGML_TYPE_BF16 && ggml_is_contiguous(src0)) { + return true; + } +#endif +#ifdef GGML_SYCL_F16 + if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && + dst->op_params[0] == GGML_PREC_DEFAULT) { + return true; + } +#endif + // the F32 branch only uses oneDNN for large GEMMs and calls oneMKL directly otherwise + const int64_t gemm_flops = src0->ne[1] * src1->ne[1] * src0->ne[0]; + return gemm_flops >= SYCL_F32_DNNL_MIN_GEMM_FLOPS; +#else + GGML_UNUSED(device); + GGML_UNUSED(dst); + return false; +#endif +} + +static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgraph * cgraph) { + if (ggml_sycl_info().active_device_count > 1) { // A sycl_ex::command_graph object can only be created for a single device - GGML_LOG_INFO("%s: disabling SYCL graphs due to multiple devices\n", __func__); + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to multiple devices\n", __func__); return false; } for (int i = 0; i < cgraph->n_nodes; i++) { - const ggml_op node_op = cgraph->nodes[i]->op; + ggml_tensor * node = cgraph->nodes[i]; + + // skip the nodes that ggml_backend_sycl_graph_compute_impl() does not run + if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0 || ggml_sycl_is_view_or_noop(node)) { + continue; + } + + const ggml_op node_op = node->op; + switch (node_op) { default: break; - case GGML_OP_CONCAT: - // ggml_sycl_op_concat() does a blocking host wait after memcpy operations, - // but wait() can't be called on the events returned by a queue recording - // to a graph. - [[fallthrough]]; case GGML_OP_MUL_MAT_ID: - // ggml_sycl_mul_mat_id() does a blocking host wait on the sycl queue after - // submitting a memcpy operation, but wait() can't be called on a queue that - // is recording to a graph. - GGML_LOG_INFO("%s: disabling SYCL graphs due to unsupported node type %s\n", __func__, - ggml_op_name(node_op)); - return false; + // The ggml_sycl_mul_mat_id() fallback path does a blocking host wait on the sycl + // queue after submitting a memcpy operation, but wait() can't be called on a queue + // that is recording to a graph. + if (ggml_sycl_mul_mat_id_needs_sync(node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to host wait in %s\n", __func__, + ggml_op_name(node_op)); + return false; + } + break; + case GGML_OP_FLASH_ATTN_EXT: + if (ggml_sycl_flash_attn_ext_needs_sync(ctx->device, node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to host wait in %s\n", __func__, + ggml_op_name(node_op)); + return false; + } +#ifndef GGML_SYCL_GRAPH_DNNL_SUPPORTED + if (ggml_sycl_flash_attn_ext_uses_onednn(ctx->device, node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to %s using oneDNN\n", __func__, + ggml_op_name(node_op)); + return false; + } +#endif + break; case GGML_OP_MUL_MAT: // We cannot use graphs with ggml_sycl_mul_mat() when SYCL async memory allocation extensions are not available, // as SYCL malloc / free and host wait calls are not supported when recording to a graph which are all present // in reordering. if (!g_ggml_sycl_use_async_mem_op) { - GGML_LOG_INFO( + GGML_LOG_DEBUG( "%s: disabling SYCL graphs due to unsupported node type when using a compiler without the " "oneAPI async memory allocation extension " "%s\n", __func__, ggml_op_name(node_op)); return false; } +#ifndef GGML_SYCL_GRAPH_DNNL_SUPPORTED + if (mul_mat_uses_onednn(ctx->device, node)) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to %s using oneDNN\n", __func__, + ggml_op_name(node_op)); + return false; + } +#endif + break; } } return true; @@ -4534,41 +6930,102 @@ static bool check_graph_compatibility(ggml_cgraph * cgraph) { static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { auto * sycl_ctx = static_cast(backend->context); + // set the device on every call: a graph replay does not go through + // ggml_backend_sycl_graph_compute_impl() + ggml_sycl_set_main_device(sycl_ctx->device); + #ifdef GGML_SYCL_GRAPH - bool use_sycl_graph = !g_ggml_sycl_disable_graph && check_graph_compatibility(cgraph); - if (use_sycl_graph) { - const bool graph_support = dpct::get_device(sycl_ctx->device).has(sycl::aspect::ext_oneapi_limited_graph); - if (!graph_support) { - GGML_SYCL_DEBUG("[SYCL-GRAPH] can not use graphs on device:%d\n", sycl_ctx->device); - ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); - return GGML_STATUS_SUCCESS; - } + bool use_sycl_graph = false; + ggml_sycl_graph * graph = nullptr; + + if (g_ggml_sycl_enable_graph) { + if (!ggml_sycl_info().devices[sycl_ctx->device].graph_support) { + // the device does not change, so report this once + if (sycl_ctx->graph_compatible != false) { + GGML_LOG_DEBUG("%s: disabling SYCL graphs due to unsupported graph feature on device\n", __func__); + GGML_LOG_INFO("%s: disabling SYCL graphs\n", __func__); + sycl_ctx->graph_compatible = false; + } + } else { + graph = sycl_ctx->sycl_graph(ggml_sycl_graph_get_key(cgraph)); - sycl_ex::command_graph model_sycl_graph(*(sycl_ctx->stream()), {sycl_ex::property::graph::assume_buffer_outlives_graph{}}); + // The nodes of a cgraph do not change while its uid does not, so the compatibility scan + // runs again only when the uid changes. A uid of 0 means it is not set. + if (cgraph->uid == 0 || cgraph->uid != graph->compatible_uid) { + graph->compatible_uid = cgraph->uid; + graph->compatible = check_graph_compatibility(sycl_ctx, cgraph); - model_sycl_graph.begin_recording(*(sycl_ctx->stream())); - ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); - model_sycl_graph.end_recording(); - - const bool graph_update_support = dpct::get_device(sycl_ctx->device).has(sycl::aspect::ext_oneapi_graph); - if (!sycl_ctx->exec_graph || !graph_update_support) { - auto exec_graph = graph_update_support ? model_sycl_graph.finalize(sycl_ex::property::graph::updatable{}) : - model_sycl_graph.finalize(); - sycl_ctx->exec_graph = std::make_unique< - sycl_ex::command_graph>(exec_graph); + if (sycl_ctx->graph_compatible != graph->compatible) { + if (graph->compatible) { + GGML_LOG_INFO("%s: enabling SYCL graphs\n", __func__); + } else { + GGML_LOG_INFO("%s: disabling SYCL graphs\n", __func__); + } + sycl_ctx->graph_compatible = graph->compatible; + } + } + + use_sycl_graph = graph->compatible; + } + } + if (use_sycl_graph) { + const bool properties_changed = ggml_sycl_graph_update_required(graph, cgraph); + bool replay_graph = false; + bool sycl_graph_update_required = false; + + if (!graph->warmup_complete) { + // Warmup: need at least 2 calls with no property change on the 2nd call. + if (!properties_changed) { + graph->warmup_complete = true; + replay_graph = true; + sycl_graph_update_required = true; + GGML_SYCL_DEBUG("[SYCL-GRAPH] warmup complete\n"); + } } else { - try { - sycl_ctx->exec_graph->update(model_sycl_graph); - GGML_SYCL_DEBUG("[SYCL-GRAPH] update success\n"); - } catch (sycl::exception const & e) { - GGML_SYCL_DEBUG("[SYCL-GRAPH] Exception when updating graph, %s\n", e.what()); - auto exec_graph = model_sycl_graph.finalize({sycl_ex::property::graph::updatable{}}); - sycl_ctx->exec_graph = std::make_unique< - sycl_ex::command_graph>(exec_graph); + // Post-warmup: replay graph while properties remain stable. + if (properties_changed) { + graph->warmup_complete = false; + GGML_SYCL_DEBUG("[SYCL-GRAPH] warmup reset\n"); + } else { + replay_graph = true; + sycl_graph_update_required = graph->exec_graph == nullptr; } } - sycl_ctx->stream()->ext_oneapi_graph(*(sycl_ctx->exec_graph)); + if (replay_graph) { + if (sycl_graph_update_required) { + const bool graph_update_support = ggml_sycl_info().devices[sycl_ctx->device].graph_update_support; + + sycl_ex::command_graph model_sycl_graph(*(sycl_ctx->stream()), {sycl_ex::property::graph::assume_buffer_outlives_graph{}}); + + model_sycl_graph.begin_recording(*(sycl_ctx->stream())); + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + model_sycl_graph.end_recording(); + + if (!graph->exec_graph || !graph_update_support) { + auto exec_graph = graph_update_support ? model_sycl_graph.finalize(sycl_ex::property::graph::updatable{}) : + model_sycl_graph.finalize(); + graph->exec_graph = std::make_unique< + sycl_ex::command_graph>(exec_graph); + } else { + try { + graph->exec_graph->update(model_sycl_graph); + GGML_SYCL_DEBUG("[SYCL-GRAPH] update success\n"); + } catch (sycl::exception const & e) { + // the executable graph cannot be updated due to violated constraints, + // so finalize the recorded graph again + GGML_SYCL_DEBUG("[SYCL-GRAPH] Exception when updating graph, %s\n", e.what()); + auto exec_graph = model_sycl_graph.finalize({sycl_ex::property::graph::updatable{}}); + graph->exec_graph = std::make_unique< + sycl_ex::command_graph>(exec_graph); + } + } + } + + sycl_ctx->stream()->ext_oneapi_graph(*(graph->exec_graph)); + } else { + ggml_backend_sycl_graph_compute_impl(sycl_ctx, cgraph); + } } else #endif { @@ -4615,8 +7072,8 @@ static ggml_backend_i ggml_backend_sycl_interface = { /* .free = */ ggml_backend_sycl_free, /* .set_tensor_async = */ ggml_backend_sycl_set_tensor_async, /* .get_tensor_async = */ ggml_backend_sycl_get_tensor_async, - /* .get_tensor_2d_async = */ NULL, /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, /* .cpy_tensor_async = */ NULL, // ggml_backend_sycl_cpy_tensor_async, // // TODO: update for the new // interface @@ -4640,20 +7097,13 @@ bool ggml_backend_is_sycl(ggml_backend_t backend) { return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_sycl_guid()); } -int ggml_backend_sycl_get_device_count() { +int ggml_backend_sycl_get_device_count(void) { return ggml_sycl_info().device_count; } // backend device -struct ggml_backend_sycl_device_context { - int device; - std::string name; - std::string description; - int op_offload_min_batch_size; -}; - static const char * ggml_backend_sycl_device_get_name(ggml_backend_dev_t dev) { ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *)dev->context; return ctx->name.c_str(); @@ -4665,14 +7115,21 @@ static const char * ggml_backend_sycl_device_get_description(ggml_backend_dev_t } static void ggml_backend_sycl_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { - ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *)dev->context; - ggml_sycl_set_device(ctx->device); - SYCL_CHECK(CHECK_TRY_ERROR( - dpct::dev_mgr::instance().get_device(ctx->device).get_memory_info(*free, *total))); + GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); + ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *) dev->context; + if (!sycl_get_mem_info(ctx->device, free, total)) { + GGML_ABORT("[%s] failed to get device memory size", __func__); + } + GGML_SYCL_DEBUG("[SYCL] call %s total %zu free %zu\n", __func__, *total, *free); + ggml_sycl_memtrace_report_device("device memory query (dev)", ctx->device, *free, *total); } static enum ggml_backend_dev_type ggml_backend_sycl_device_get_type(ggml_backend_dev_t dev) { - GGML_UNUSED(dev); + ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *)dev->context; + const sycl_device_info & info = ggml_sycl_info().devices[ctx->device]; + if (info.l0_device_type_valid && !info.l0_discrete_gpu) { + return GGML_BACKEND_DEVICE_TYPE_IGPU; + } return GGML_BACKEND_DEVICE_TYPE_GPU; } @@ -4694,6 +7151,7 @@ static void ggml_backend_sycl_device_get_props(ggml_backend_dev_t dev, ggml_back /* .host_buffer = */ host_buffer, /* .buffer_from_host_ptr = */ false, /* .events = */ events, + /* .mmap_support = */ true, }; } @@ -4709,8 +7167,8 @@ static ggml_backend_buffer_type_t ggml_backend_sycl_device_get_buffer_type(ggml_ } static ggml_backend_buffer_type_t ggml_backend_sycl_device_get_host_buffer_type(ggml_backend_dev_t dev) { - GGML_UNUSED(dev); - return ggml_backend_sycl_host_buffer_type(); + ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *) dev->context; + return ggml_backend_sycl_host_buffer_type_for_device(ctx->device); } static ggml_backend_buffer_t ggml_backend_sycl_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { @@ -4721,7 +7179,7 @@ static ggml_backend_buffer_t ggml_backend_sycl_device_buffer_from_host_ptr(ggml_ return nullptr; } -static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { +static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { ggml_backend_sycl_device_context *sycl_ctx = (ggml_backend_sycl_device_context *)dev->context; int device = sycl_ctx->device; @@ -4735,6 +7193,15 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g } return false; } + case GGML_OP_CONV_2D: + // WHCN: contiguous [KW,KH,IC,OC] kernel and [W,H,C,N] input + // CWHN: channel-fastest input (C fastest) and kernel with OC fastest + return (ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) || + (op->src[1]->nb[2] == ggml_type_size(op->src[1]->type) && + op->src[0]->nb[3] == ggml_type_size(op->src[0]->type)); + case GGML_OP_CONV_2D_DW: + case GGML_OP_CONV_TRANSPOSE_2D: + return true; case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { case GGML_UNARY_OP_SGN: @@ -4751,18 +7218,16 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_UNARY_OP_GELU_QUICK: case GGML_UNARY_OP_GELU_ERF: case GGML_UNARY_OP_EXP: + case GGML_UNARY_OP_EXPM1: case GGML_UNARY_OP_SOFTPLUS: case GGML_UNARY_OP_ELU: + case GGML_UNARY_OP_XIELU: case GGML_UNARY_OP_CEIL: return true; case GGML_UNARY_OP_FLOOR: case GGML_UNARY_OP_ROUND: case GGML_UNARY_OP_TRUNC: -#if defined (GGML_SYCL_F16) - return ggml_is_contiguous(op->src[0]) && (op->type == op->src[0]->type); -#else - return ggml_is_contiguous(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) && (op->type == op->src[0]->type); -#endif + return true; default: return false; } @@ -4774,30 +7239,25 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_GLU_OP_SWIGLU_OAI: case GGML_GLU_OP_GEGLU_ERF: case GGML_GLU_OP_GEGLU_QUICK: - return ggml_is_contiguous_1(op->src[0]); + case GGML_GLU_OP_SWIGLU_CLAMP: + // the fused GLU kernels only have f32/f16 instantiations + return ggml_is_contiguous_1(op->src[0]) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16); default: return false; } - break; case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: { struct ggml_tensor * a = op->src[0]; struct ggml_tensor * b = op->src[1]; - // disable Q1_0 until implementation - if (a->type == GGML_TYPE_Q1_0 || b->type == GGML_TYPE_Q1_0) { - return false; - } - if (a->ne[3] != b->ne[3]) { return false; } ggml_type src0_type = op->src[0]->type; - - // TODO: The configuration below needs more work to be supported with oneDNN if (ggml_is_permuted(a) && !ggml_is_contiguous(a) && a->ne[2] > 1 && a->ne[3] > 1 && src0_type == GGML_TYPE_F16) { @@ -4809,25 +7269,59 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g a->ne[0] > 128 && a->ne[2] == 1 && src0_type == GGML_TYPE_F16) { return false; } + + if (src0_type == GGML_TYPE_TQ2_0 || src0_type == GGML_TYPE_TQ1_0) { + return false; + } + return true; } case GGML_OP_OUT_PROD: - return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->ne[2] == 1 && op->ne[3] == 1; + return op->type == GGML_TYPE_F32 && + (op->src[0]->type == GGML_TYPE_F32 || + (op->src[0]->type == GGML_TYPE_Q1_0 && op->src[0]->ne[2] == op->src[1]->ne[2] && + op->src[0]->ne[3] == op->src[1]->ne[3])) && + op->src[1]->type == GGML_TYPE_F32; case GGML_OP_GET_ROWS: { switch (op->src[0]->type) { + case GGML_TYPE_I32: case GGML_TYPE_F16: + case GGML_TYPE_BF16: case GGML_TYPE_F32: + case GGML_TYPE_Q1_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: + case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: case GGML_TYPE_Q8_0: return true; default: return false; } } + case GGML_OP_GET_ROWS_BACK: + // return true; + return op->type == GGML_TYPE_F32 && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + op->src[1]->type == GGML_TYPE_I32 && + op->ne[2] == 1 && op->ne[3] == 1; case GGML_OP_SET: return (op->type == GGML_TYPE_F32) && (op->src[0] && op->src[1]) && @@ -4836,80 +7330,140 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_SET_ROWS: { - return ((op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || - op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q5_0 || - op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_IQ4_NL) && - (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32)); + if (op->type == GGML_TYPE_TQ2_0 || op->type == GGML_TYPE_TQ1_0) { + return false; + } + auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || + op->src[0]->type == GGML_TYPE_BF16) && + (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); + return res; } - break; + case GGML_OP_DSV4_HC_PRE: + return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32; + case GGML_OP_DSV4_HC_COMB: + return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_DSV4_HC_POST: + return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && + op->src[2]->type == GGML_TYPE_F32 && + (op->src[3] == nullptr || op->src[3]->type == GGML_TYPE_F32) && + op->type == GGML_TYPE_F32; + case GGML_OP_LIGHTNING_INDEXER: + return op->src[0]->type == GGML_TYPE_F32 && + (op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32 || + op->src[1]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_Q8_0 || + op->src[1]->type == GGML_TYPE_Q5_1 || op->src[1]->type == GGML_TYPE_Q5_0 || + op->src[1]->type == GGML_TYPE_Q4_1 || op->src[1]->type == GGML_TYPE_Q4_0 || + op->src[1]->type == GGML_TYPE_IQ4_NL) && + op->src[2]->type == GGML_TYPE_F32 && + op->src[3]->type == GGML_TYPE_F16 && + op->type == GGML_TYPE_F32 && + op->src[0]->ne[0] == WARP_SIZE * 8; case GGML_OP_CPY: { ggml_type src0_type = op->src[0]->type; ggml_type src1_type = op->src[1]->type; - if (src0_type == src1_type && (ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) && src0_type != GGML_TYPE_BF16) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q8_0) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_0) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_1) { - return true; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) { - return true; - } - if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_Q8_0 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_Q4_0 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_Q4_1 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_0) { - return true; - } - if (src0_type == GGML_TYPE_Q5_0 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_1) { - return true; - } - if (src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_F32) { - return true; - } - if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_IQ4_NL) { - return true; + + if (src0_type == GGML_TYPE_F16) { + if (src1_type == GGML_TYPE_Q2_K || + src1_type == GGML_TYPE_Q3_K || + src1_type == GGML_TYPE_Q4_K || + src1_type == GGML_TYPE_Q5_K || + src1_type == GGML_TYPE_Q6_K || + src1_type == GGML_TYPE_IQ2_XXS || + src1_type == GGML_TYPE_IQ2_XS || + src1_type == GGML_TYPE_IQ2_S || + src1_type == GGML_TYPE_IQ3_XXS || + src1_type == GGML_TYPE_IQ1_S || + src1_type == GGML_TYPE_IQ1_M || + src1_type == GGML_TYPE_IQ3_S || + src1_type == GGML_TYPE_IQ4_XS) { + return false; + } } - if(src0_type == GGML_TYPE_Q8_0 && src1_type == GGML_TYPE_Q8_0) { - return true; + + if (src0_type == GGML_TYPE_BF16) { + if (src1_type == GGML_TYPE_Q4_0 || //big error in ut + src1_type == GGML_TYPE_Q4_1 || //big error in ut + src1_type == GGML_TYPE_Q8_0 || //big error in ut + src1_type == GGML_TYPE_Q2_K || + src1_type == GGML_TYPE_Q3_K || + src1_type == GGML_TYPE_Q4_K || + src1_type == GGML_TYPE_Q5_K || + src1_type == GGML_TYPE_Q6_K || + src1_type == GGML_TYPE_IQ2_XXS || + src1_type == GGML_TYPE_IQ2_XS || + src1_type == GGML_TYPE_IQ2_S || + src1_type == GGML_TYPE_IQ3_XXS || + src1_type == GGML_TYPE_IQ1_S || + src1_type == GGML_TYPE_IQ1_M || + src1_type == GGML_TYPE_IQ3_S || + src1_type == GGML_TYPE_IQ4_XS) { + return false; + } } - if(src0_type == GGML_TYPE_Q5_0 && src1_type == GGML_TYPE_Q5_0) { - return true; + + if (src0_type == GGML_TYPE_F32) { + if (src1_type == GGML_TYPE_Q2_K || + src1_type == GGML_TYPE_Q3_K || + src1_type == GGML_TYPE_Q4_K || + src1_type == GGML_TYPE_Q5_K || + src1_type == GGML_TYPE_Q6_K || + src1_type == GGML_TYPE_IQ2_XXS || + src1_type == GGML_TYPE_IQ2_XS || + src1_type == GGML_TYPE_IQ2_S || + src1_type == GGML_TYPE_IQ3_XXS || + src1_type == GGML_TYPE_IQ1_S || + src1_type == GGML_TYPE_IQ1_M || + src1_type == GGML_TYPE_IQ3_S || + src1_type == GGML_TYPE_IQ4_XS) { + return false; + } } - if(src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_Q5_1) { - return true; + + if (src1_type == GGML_TYPE_F32) { + if (src0_type == GGML_TYPE_Q1_0 || + src0_type == GGML_TYPE_NVFP4 || + src0_type == GGML_TYPE_Q2_K || + src0_type == GGML_TYPE_Q3_K || + src0_type == GGML_TYPE_Q4_K || + src0_type == GGML_TYPE_Q5_K || + src0_type == GGML_TYPE_Q6_K || + src0_type == GGML_TYPE_IQ2_XXS || + src0_type == GGML_TYPE_IQ2_XS || + src0_type == GGML_TYPE_IQ2_S || + src0_type == GGML_TYPE_IQ3_XXS || + src0_type == GGML_TYPE_IQ1_S || + src0_type == GGML_TYPE_IQ1_M || + src0_type == GGML_TYPE_IQ3_S || + src0_type == GGML_TYPE_IQ4_NL || + src0_type == GGML_TYPE_IQ4_XS + ) { + return false; + } } - if(src0_type == GGML_TYPE_Q4_0 && src1_type == GGML_TYPE_Q4_0) { - return true; + + if (src0_type == src1_type) { + if (src1_type == GGML_TYPE_IQ2_XXS || + src1_type == GGML_TYPE_IQ2_XS || + src1_type == GGML_TYPE_IQ2_S || + src1_type == GGML_TYPE_IQ3_XXS || + src1_type == GGML_TYPE_IQ3_S || + src1_type == GGML_TYPE_IQ1_S || + src1_type == GGML_TYPE_IQ1_M || + src1_type == GGML_TYPE_TQ2_0 || + src1_type == GGML_TYPE_TQ1_0) { + return false; + } } - if(src0_type == GGML_TYPE_Q4_1 && src1_type == GGML_TYPE_Q4_1) { - return true; + + if (src0_type == GGML_TYPE_TQ2_0 || src1_type == GGML_TYPE_TQ2_0 || + src0_type == GGML_TYPE_TQ1_0 || src1_type == GGML_TYPE_TQ1_0) { + return false; } - return false; + + return true; } case GGML_OP_REPEAT_BACK: { @@ -4941,11 +7495,6 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_COS: case GGML_OP_CLAMP: case GGML_OP_LOG: -#if defined (GGML_SYCL_F16) - return ((op->type == GGML_TYPE_F32 || op->type == GGML_SYCL_F16) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_SYCL_F16) && (op->type == op->src[0]->type)); -#else - return (op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32) && (op->type == op->src[0]->type); -#endif case GGML_OP_NORM: case GGML_OP_L2_NORM: case GGML_OP_GROUP_NORM: @@ -4954,9 +7503,10 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_RMS_NORM_BACK: return ggml_is_contiguous(op->src[0]); case GGML_OP_SCALE: - return true; + // the scale kernel is f32 only + return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_CONT: - return op->src[0]->type != GGML_TYPE_BF16; + return true; case GGML_OP_TRI: { const ggml_tensor * src0 = op->src[0]; @@ -4976,15 +7526,36 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_ROPE: case GGML_OP_ROPE_BACK: case GGML_OP_IM2COL: + case GGML_OP_IM2COL_3D: case GGML_OP_UPSCALE: return true; + case GGML_OP_COL2IM_1D: + return ggml_is_contiguous(op->src[0]) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 +#ifdef GGML_SYCL_HAS_BF16 + || op->type == GGML_TYPE_BF16 +#endif + ) && + op->src[0]->type == op->type; + case GGML_OP_CONV_3D: { + const int32_t * opts = (const int32_t *) op->op_params; + // zero kernel volume, channels or output makes the im2col GEMM degenerate (k == 0 or m == 0) + // and MKL BLAS rejects it; let those fall back to the CPU backend + const int64_t knl_n_total = op->src[0]->ne[0] * op->src[0]->ne[1] * op->src[0]->ne[2] * opts[9]; + const int64_t patch_total = op->ne[0] * op->ne[1] * op->ne[2] * opts[10]; + return op->type == GGML_TYPE_F32 && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + op->src[1]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op->src[1]) && + knl_n_total > 0 && patch_total > 0; + } case GGML_OP_SUM: case GGML_OP_SUM_ROWS: case GGML_OP_MEAN: return ggml_is_contiguous(op->src[0]); case GGML_OP_ARGSORT: - return op->src[0]->ne[0] * sizeof(int) <= - ggml_sycl_info().devices[device].smpbo; + return true; case GGML_OP_TOP_K: { const ggml_tensor * src0 = op->src[0]; const int k = op->ne[0]; @@ -4992,24 +7563,25 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g op->type == GGML_TYPE_I32 && src0->type == GGML_TYPE_F32 && ggml_is_contiguous(src0) && - k > 0 && k <= 32; + k > 0 && k <= src0->ne[0]; } case GGML_OP_POOL_2D: - return true; + case GGML_OP_POOL_1D: case GGML_OP_ACC: - return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); + return true; case GGML_OP_PAD: - // TODO: add circular padding support for syscl, see https://github.com/ggml-org/llama.cpp/pull/16985 if (ggml_get_op_params_i32(op, 8) != 0) { return false; } - return ggml_is_contiguous(op->src[0]); + return true; case GGML_OP_LEAKY_RELU: case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_RWKV_WKV6: case GGML_OP_RWKV_WKV7: case GGML_OP_GATED_LINEAR_ATTN: case GGML_OP_GATED_DELTA_NET: + case GGML_OP_OPT_STEP_ADAMW: + case GGML_OP_OPT_STEP_SGD: return true; case GGML_OP_SSM_CONV: return op->type == GGML_TYPE_F32 && @@ -5019,6 +7591,23 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g return op->type == GGML_TYPE_F32; case GGML_OP_ARANGE: return op->type == GGML_TYPE_F32; + case GGML_OP_SSM_SCAN: + if (op->src[3]->ne[0] == 1) { + // Mamba2 + // (kernel only supports (d_state == 128 || d_state == 256) && d_head % WARP_SIZE == 0) + return (op->src[0]->ne[0] == 128 || op->src[0]->ne[0] == 256) && op->src[0]->ne[1] % WARP_SIZE == 0; + } else { + // TODO Mamba-1 not yet ported to SYCL + return false; + } + case GGML_OP_FILL: + case GGML_OP_CUMSUM: + case GGML_OP_DIAG: + case GGML_OP_CROSS_ENTROPY_LOSS: + case GGML_OP_CROSS_ENTROPY_LOSS_BACK: + return true; + case GGML_OP_SOLVE_TRI: + return op->src[0]->ne[0] <= SYCL_SOLVE_TRI_MAX_N && op->src[1]->ne[0] <= SYCL_SOLVE_TRI_MAX_K; case GGML_OP_FLASH_ATTN_EXT: return ggml_sycl_flash_attn_ext_supported(device, op); default: @@ -5028,6 +7617,13 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g GGML_UNUSED(dev); } +static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + bool res = do_ggml_backend_sycl_device_supports_op(dev, op); + GGML_SYCL_DEBUG("[SYCL] call %s op->op=%s op->type=%s -> %s\n", __func__, ggml_op_name(op->op), + ggml_type_name(op->type), res ? "true" : "false"); + return res; +} + static bool ggml_backend_sycl_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { if (buft->iface.get_name != ggml_backend_sycl_buffer_type_get_name) { return false; @@ -5143,6 +7739,277 @@ static ggml_backend_dev_t ggml_backend_sycl_reg_get_device(ggml_backend_reg_t re return ctx->devices[index]; } +// ========================================================================== +// Tensor parallelism (--split-mode tensor) for the SYCL backend. +// +// The meta-backend invokes these three entry points via get_proc_address: +// * ggml_backend_sycl_comm_init - one-time per-graph setup +// * ggml_backend_sycl_comm_allreduce_tensor - per-allreduce step +// * ggml_backend_sycl_comm_free - tear-down +// +// For N=2 (dual-GPU), this is a degenerate ring allreduce with dual paths +// chosen by tensor size: +// +// * Small (nelem < 32K): FP32 direct memcpy + per-device ADD +// kernel. The kernel depends_on() its corresponding memcpy event +// so it doesn't read partial data. Both devices run in parallel. +// +// * Large (nelem >= 32K): BF16-compressed. Each device compresses +// its FP32 partial to BF16 locally, cross-device memcpys +// to the peer (half the PCI bandwidth), where it is decompressed +// and added into the local FP32 partial. 6 SYCL submissions per +// allreduce (2 compress + 2 memcpy + 2 decompress-add) vs the +// 4 for the small path, but the bandwidth saving > 6 GB/s PCIe x 2 +// dominates for larger tensors. +// +// Storage: A persistent uint8_t buffer per device, sized to +// 4 * nelem bytes. Both paths reinterpret the same bytes (small path +// as nelem floats; large path as outbox + inbox = 2*nelem uint16_t +// each, using the full 4*nelem byte budget either way). Single +// alloc+free per device keeps the SYCL pool's strict-LIFO invariant +// trivial. +// +// For non-(N=2 FP32 contiguous) cases, comm_init or comm_allreduce_tensor +// returns null/false, causing the meta-backend to use its generic +// butterfly all-reduce fallback. +// ========================================================================== + +struct ggml_backend_sycl_comm_context { + std::vector backends; + // ONE persistent per-device byte buffer, 4*nelem bytes. Both the + // FP32 small-tensor path and the BF16 large-tensor path share it + // by reinterpreting. + std::unique_ptr> buf0; + std::unique_ptr> buf1; + int64_t buf_nelem = 0; + + // pinned host staging per device context, reused across calls (queues are in-order) + uint8_t * host_out[2] = { nullptr, nullptr }; + uint8_t * host_in[2] = { nullptr, nullptr }; + size_t host_bytes = 0; + + void free_host() { + for (int i = 0; i < 2; ++i) { + const sycl::context sctx = ((ggml_backend_sycl_context *) backends[i]->context)->stream()->get_context(); + if (host_out[i]) { sycl::free(host_out[i], sctx); host_out[i] = nullptr; } + if (host_in[i]) { sycl::free(host_in[i], sctx); host_in[i] = nullptr; } + } + host_bytes = 0; + } +}; + +// dst0 <- src1 and dst1 <- src0 through pinned host memory, with a single host wait +static void ggml_sycl_comm_exchange(ggml_backend_sycl_comm_context * comm, queue_ptr q0, queue_ptr q1, + void * dst0, const void * src1, void * dst1, const void * src0, + size_t nbytes) { + if (comm->host_bytes < nbytes) { + q0->wait(); + q1->wait(); + comm->free_host(); + comm->host_out[0] = sycl::malloc_host(nbytes, q0->get_context()); + comm->host_in[0] = sycl::malloc_host(nbytes, q0->get_context()); + comm->host_out[1] = sycl::malloc_host(nbytes, q1->get_context()); + comm->host_in[1] = sycl::malloc_host(nbytes, q1->get_context()); + GGML_ASSERT(comm->host_out[0] && comm->host_in[0] && comm->host_out[1] && comm->host_in[1]); + comm->host_bytes = nbytes; + } + + sycl::event e0 = q0->memcpy(comm->host_out[0], src0, nbytes); + sycl::event e1 = q1->memcpy(comm->host_out[1], src1, nbytes); + e0.wait(); + e1.wait(); + std::memcpy(comm->host_in[0], comm->host_out[1], nbytes); + std::memcpy(comm->host_in[1], comm->host_out[0], nbytes); + q0->memcpy(dst0, comm->host_in[0], nbytes); + q1->memcpy(dst1, comm->host_in[1], nbytes); +} + +void * ggml_backend_sycl_comm_init(ggml_backend_t * backends, size_t n_backends) try { + for (size_t i = 0; i < n_backends; ++i) { + if (!ggml_backend_is_sycl(backends[i])) { + return nullptr; + } + } + + // Initial version: N=2 only. For N!=2, returning null makes the + // meta-backend skip this backend-specific allreduce entirely. + if (n_backends != 2) { + return nullptr; + } + + auto * ctx = new ggml_backend_sycl_comm_context; + ctx->backends.assign(backends, backends + n_backends); + auto * sctx0 = (ggml_backend_sycl_context *) backends[0]->context; + auto * sctx1 = (ggml_backend_sycl_context *) backends[1]->context; + ctx->buf0 = std::make_unique>(sctx0->pool()); + ctx->buf1 = std::make_unique>(sctx1->pool()); + return ctx; +} +catch (const sycl::exception &) { return nullptr; } +catch (...) { return nullptr; } + +void ggml_backend_sycl_comm_free(void * comm_ctx_v) { + auto * comm_ctx = static_cast(comm_ctx_v); + if (comm_ctx == nullptr) { + return; + } + + // Sync both per-device queues so the pool_alloc destructors don't + // return memory still in use by the last kernel. + if (comm_ctx->backends.size() == 2) { + auto * sctx0 = (ggml_backend_sycl_context *) comm_ctx->backends[0]->context; + auto * sctx1 = (ggml_backend_sycl_context *) comm_ctx->backends[1]->context; + try { + sctx0->stream()->wait(); + sctx1->stream()->wait(); + comm_ctx->free_host(); + } catch (...) { /* best effort during shutdown */ } + } + + delete comm_ctx; +} + +bool ggml_backend_sycl_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) try { + if (comm_ctx_v == nullptr) { + return false; + } + + auto * comm_ctx = static_cast(comm_ctx_v); + const size_t n_backends = comm_ctx->backends.size(); + + // Fast path: N=2, F32/F16, contiguous, matching shapes. + if (n_backends != 2) { + return false; + } + // Accept F32 or F16 inputs natively (types must match). F16 takes the + // direct 2-byte memcpy + add path below; other types return false so the + // meta-backend uses its generic all-reduce. + if (tensors[0]->type != tensors[1]->type) { + return false; + } + if (tensors[0]->type != GGML_TYPE_F32 && tensors[0]->type != GGML_TYPE_F16) { + return false; + } + if (!ggml_is_contiguous(tensors[0]) || !ggml_is_contiguous(tensors[1])) { + return false; + } + if (ggml_nelements(tensors[0]) != ggml_nelements(tensors[1])) { + return false; + } + + const int64_t nelem = ggml_nelements(tensors[0]); + const size_t nbytes = ggml_nbytes(tensors[0]); + if (nelem == 0) { + return true; + } + + auto * ctx0 = (ggml_backend_sycl_context *) comm_ctx->backends[0]->context; + auto * ctx1 = (ggml_backend_sycl_context *) comm_ctx->backends[1]->context; + queue_ptr q0 = ctx0->stream(); + queue_ptr q1 = ctx1->stream(); + + // Grow per-device byte buffers if needed (4 * nelem bytes each). + if (comm_ctx->buf_nelem < nelem) { + comm_ctx->buf0->realloc(nelem * 4); + comm_ctx->buf1->realloc(nelem * 4); + comm_ctx->buf_nelem = nelem; + } + uint8_t * buf0 = comm_ctx->buf0->get(); + uint8_t * buf1 = comm_ctx->buf1->get(); + + // F16 native path: direct 2-byte cross-device copy + add, skipping the + // F32 round-trip the meta-backend fallback would force. The devices are in separate SYCL + // contexts (a raw peer-USM q->memcpy would be a silent no-op), so copies go through host memory. + if (tensors[0]->type == GGML_TYPE_F16) { + sycl::half * f16_out0 = (sycl::half *) tensors[0]->data; + sycl::half * f16_out1 = (sycl::half *) tensors[1]->data; + sycl::half * f16_tmp0 = (sycl::half *) buf0; + sycl::half * f16_tmp1 = (sycl::half *) buf1; + + ggml_sycl_comm_exchange(comm_ctx, q0, q1, f16_tmp0, tensors[1]->data, f16_tmp1, tensors[0]->data, nbytes); + + q0->submit([&](sycl::handler & h) { + h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { + f16_out0[i] = (sycl::half) ((float) f16_out0[i] + (float) f16_tmp0[i]); + }); + }); + q1->submit([&](sycl::handler & h) { + h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { + f16_out1[i] = (sycl::half) ((float) f16_out1[i] + (float) f16_tmp1[i]); + }); + }); + return true; + } + + float * out0 = (float *) tensors[0]->data; + float * out1 = (float *) tensors[1]->data; + + // BF16 threshold: above this, the PCIe savings from halving the + // cross-device bytes outweigh the 2 extra compress kernels. + // Below: stay on the FP32 fast path. Threshold mirrors the CUDA + // NCCL allreduce pattern for n_backends=2. + static constexpr int64_t BF16_THRESHOLD = 32768; + + if (nelem < BF16_THRESHOLD) { + // FP32 small path: 4 SYCL submissions per allreduce. + float * tmp0 = (float *) buf0; + float * tmp1 = (float *) buf1; + + // separate SYCL contexts: a raw peer-USM q->memcpy is a silent no-op, so stage through host memory + ggml_sycl_comm_exchange(comm_ctx, q0, q1, tmp0, tensors[1]->data, tmp1, tensors[0]->data, nbytes); + + q0->submit([&](sycl::handler & h) { + h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { + out0[i] += tmp0[i]; + }); + }); + q1->submit([&](sycl::handler & h) { + h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { + out1[i] += tmp1[i]; + }); + }); + return true; + } + + // BF16 large path: 6 SYCL submissions per allreduce, but the + // cross-device memcpy is HALF the bytes. Pure bit-shift + // conversion (no rounding) — matches ggml's truncating fp32->bf16. + uint16_t * outbox0 = (uint16_t *) buf0; + uint16_t * inbox0 = outbox0 + nelem; + uint16_t * outbox1 = (uint16_t *) buf1; + uint16_t * inbox1 = outbox1 + nelem; + + // Phase A: compress each device's local partial in parallel. + q0->parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { + outbox0[i] = (uint16_t) (sycl::bit_cast(out0[i]) >> 16); + }); + + q1->parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { + outbox1[i] = (uint16_t) (sycl::bit_cast(out1[i]) >> 16); + }); + + // Phase B: exchange the compressed bytes (in-order queues keep this after the compress kernels). + const size_t bf16_bytes = nelem * sizeof(uint16_t); + ggml_sycl_comm_exchange(comm_ctx, q0, q1, inbox0, outbox1, inbox1, outbox0, bf16_bytes); + + // Phase C: decompress + add into local FP32 partial. + q0->submit([&](sycl::handler & h) { + h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { + out0[i] += sycl::bit_cast(((uint32_t) inbox0[i]) << 16); + }); + }); + + q1->submit([&](sycl::handler & h) { + h.parallel_for(sycl::range<1>(nelem), [=](sycl::id<1> i) { + out1[i] += sycl::bit_cast(((uint32_t) inbox1[i]) << 16); + }); + }); + + return true; +} +catch (const sycl::exception &) { return false; } +catch (...) { return false; } + static void *ggml_backend_sycl_reg_get_proc_address(ggml_backend_reg_t reg, const char *name) { GGML_UNUSED(reg); @@ -5150,6 +8017,17 @@ static void *ggml_backend_sycl_reg_get_proc_address(ggml_backend_reg_t reg, cons return (void *)ggml_backend_sycl_split_buffer_type; } + // Tensor parallelism (--split-mode tensor) entry points. + if (strcmp(name, "ggml_backend_comm_init") == 0) { + return (void *)ggml_backend_sycl_comm_init; + } + if (strcmp(name, "ggml_backend_comm_free") == 0) { + return (void *)ggml_backend_sycl_comm_free; + } + if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { + return (void *)ggml_backend_sycl_comm_allreduce_tensor; + } + // SYCL doesn't support registering host memory, left here for reference // "ggml_backend_register_host_buffer" // "ggml_backend_unregister_host_buffer" @@ -5168,6 +8046,7 @@ static const ggml_backend_reg_i ggml_backend_sycl_reg_interface = { // backend registry ggml_backend_reg_t ggml_backend_sycl_reg() { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_reg\n"); static ggml_backend_reg reg; static bool initialized = false; @@ -5175,6 +8054,7 @@ ggml_backend_reg_t ggml_backend_sycl_reg() { static std::mutex mutex; std::lock_guard lock(mutex); if (!initialized) { + ggml_check_sycl(); ggml_backend_sycl_reg_context * ctx = new ggml_backend_sycl_reg_context; const int min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32; @@ -5213,8 +8093,19 @@ ggml_backend_reg_t ggml_backend_sycl_reg() { return ® } +static void set_device_active(int device) { + ggml_sycl_device_info & info = ggml_sycl_info(); + info.devices[device].active = true; + int cnt = 0; + for (int i = 0; i < info.device_count; i++) { + if (info.devices[i].active) + cnt++; + } + info.active_device_count = cnt; +} + ggml_backend_t ggml_backend_sycl_init(int device) { - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_init\n"); + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_init. device=%d\n", device); ggml_check_sycl(); check_allow_gpu_index(device); @@ -5225,6 +8116,7 @@ ggml_backend_t ggml_backend_sycl_init(int device) { return nullptr; }; + set_device_active(device); ggml_backend_t sycl_backend = new ggml_backend { /* .guid = */ ggml_backend_sycl_guid(), /* .iface = */ ggml_backend_sycl_interface, diff --git a/ggml/src/ggml-sycl/gla.cpp b/ggml/src/ggml-sycl/gla.cpp index 879184f..3794834 100644 --- a/ggml/src/ggml-sycl/gla.cpp +++ b/ggml/src/ggml-sycl/gla.cpp @@ -1,6 +1,7 @@ #include #include "common.hpp" +#include "gla.hpp" template static void gated_linear_attn_f32_kernel(const dpct::queue_ptr stream, u_int B, u_int T, u_int C, u_int H, float scale, diff --git a/ggml/src/ggml-sycl/im2col.cpp b/ggml/src/ggml-sycl/im2col.cpp index 6d75d34..e666167 100644 --- a/ggml/src/ggml-sycl/im2col.cpp +++ b/ggml/src/ggml-sycl/im2col.cpp @@ -1,6 +1,6 @@ // // MIT license -// Copyright (C) 2024 Intel Corporation +// Copyright (C) 2026 Intel Corporation // SPDX-License-Identifier: MIT // @@ -12,125 +12,389 @@ #include "im2col.hpp" -#include -#include // For std::is_same_v - -#include "ggml.h" +#define MAX_GRIDDIM_Z 65535 template -static void im2col_kernel(const float * x, T * dst, int64_t batch_offset, int64_t offset_delta, int64_t IC, int64_t IW, - int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, int64_t pelements, int64_t CHW, - int s0, int s1, int p0, int p1, int d0, int d1, const sycl::nd_item<3> & item_ct1) { - const int64_t work_group_size = item_ct1.get_local_range(2); - const int64_t global_id = item_ct1.get_local_id(2) + (work_group_size * item_ct1.get_group(2)); - - // make each work-item deal with more elements since sycl global range can not exceed max int - for (int64_t i = global_id; i < pelements; i += (work_group_size * item_ct1.get_group_range(2))) { - const int64_t ksize = OW * KH; - const int64_t kx = i / ksize; - const int64_t kd = kx * ksize; - const int64_t ky = (i - kd) / OW; - const int64_t ix = i % OW; - - const int64_t oh = item_ct1.get_group(1); - const int64_t batch = item_ct1.get_group(0) / IC; - const int64_t ic = item_ct1.get_group(0) % IC; - - const int64_t iiw = (ix * s0) + (kx * d0) - p0; - const int64_t iih = (oh * s1) + (ky * d1) - p1; - - const int64_t offset_dst = (((batch * OH + oh) * OW + ix) * CHW) + (ic * (KW * KH) + ky * KW + kx); - - const int64_t offset_src_base = (ic * offset_delta) + (batch * batch_offset); - const int64_t offset_src = offset_src_base + (iih * IW) + iiw; - - const bool out_of_bounds = (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW); - const float src_val = out_of_bounds ? 0.0f : x[offset_src]; - - if constexpr (std::is_same_v) { - dst[offset_dst] = sycl::half(src_val); - } else if constexpr (std::is_same_v) { - dst[offset_dst] = src_val; - } +static void im2col_kernel( + const float * x, T * dst, + int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, + int64_t IC_IH_IW, int64_t IH_IW, int64_t N_OH, int64_t KH_KW, int64_t IC_KH_KW, + int s0, int s1, int p0, int p1, int d0, int d1) { + auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>(); + const int64_t i = item_ct1.get_local_id(2) + item_ct1.get_group(2) * item_ct1.get_local_range(2); + if (i >= IC_KH_KW) { + return; } -} -template -static void im2col_sycl_internal(const float * x, T * dst, int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, - int64_t KH, int64_t IC, int64_t batch, int64_t batch_offset, int64_t offset_delta, - int s0, int s1, int p0, int p1, int d0, int d1, queue_ptr stream) { - const int64_t parallel_elements = OW * KW * KH; - const int64_t num_blocks = (parallel_elements + SYCL_IM2COL_BLOCK_SIZE - 1) / SYCL_IM2COL_BLOCK_SIZE; + const int64_t iic = i / (KH_KW); + const int64_t rem = i - iic * KH_KW; + const int64_t ikh = rem / KW; + const int64_t ikw = rem - ikh * KW; - // decrease global range when it exceeds the max int - int64_t local_size = downsample_sycl_global_range(batch * IC * OH * num_blocks, SYCL_IM2COL_BLOCK_SIZE); + const int64_t iow = item_ct1.get_group(1); + for (int64_t iz = item_ct1.get_group(0); iz < N_OH; iz += MAX_GRIDDIM_Z) { + const int64_t in = iz / OH; + const int64_t ioh = iz - in * OH; - sycl::range<3> block_nums(batch * IC, OH, num_blocks); - sycl::range<3> local_range(1, 1, local_size); + const int64_t iiw = iow * s0 + ikw * d0 - p0; + const int64_t iih = ioh * s1 + ikh * d1 - p1; - const int64_t CHW = IC * KH * KW; + const int64_t offset_dst = + ((in * OH + ioh) * OW + iow) * IC_KH_KW + iic * KH_KW + ikh * KW + ikw; - stream->parallel_for(sycl::nd_range<3>(block_nums * local_range, local_range), [=](sycl::nd_item<3> item_ct1) { - im2col_kernel(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, CHW, s0, s1, - p0, p1, d0, d1, item_ct1); - }); + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) { + dst[offset_dst] = 0.0f; + } else { + const int64_t offset_src = iic * IC_IH_IW + in * IH_IW; + dst[offset_dst] = x[offset_src + iih * IW + iiw]; + } + } + + GGML_UNUSED(IC); + GGML_UNUSED(KH); } -static void im2col_sycl_f16(const float * x, sycl::half * dst, int64_t IW, int64_t IH, int64_t OW, int64_t OH, - int64_t KW, int64_t KH, int64_t IC, int64_t batch, int64_t batch_offset, - int64_t offset_delta, int s0, int s1, int p0, int p1, int d0, int d1, queue_ptr stream) { - if (!stream->get_device().has(sycl::aspect::fp16)) { - throw sycl::exception(sycl::make_error_code(sycl::errc::kernel_not_supported), - "Device does not support half precision (fp16) operations!"); - } - im2col_sycl_internal(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, - p1, d0, d1, stream); +// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW] +template +static void im2col_sycl(const float * x, + T * dst, + int64_t IW, + int64_t IH, + int64_t OW, + int64_t OH, + int64_t KW, + int64_t KH, + int64_t IC, + int64_t N, + int64_t IC_IH_IW, + int64_t IH_IW, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, + dpct::queue_ptr stream) { + const int64_t IC_KH_KW = IC * KH * KW; + const int64_t num_blocks = (IC_KH_KW + SYCL_IM2COL_BLOCK_SIZE - 1) / SYCL_IM2COL_BLOCK_SIZE; + const int64_t N_OH = N * OH; + const int64_t KH_KW = KW*KH; + dpct::dim3 block_nums(num_blocks, OW, MIN(N_OH, MAX_GRIDDIM_Z)); + /* + DPCT1049:73: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. + */ + stream->parallel_for(sycl::nd_range<3>(block_nums * sycl::range<3>(1, 1, MIN(IC_KH_KW, SYCL_IM2COL_BLOCK_SIZE)), + sycl::range<3>(1, 1, MIN(IC_KH_KW, SYCL_IM2COL_BLOCK_SIZE))), + [=](sycl::nd_item<3>) { + im2col_kernel(x, dst, IC, IW, IH, OH, OW, KW, KH, IC_IH_IW, IH_IW, N_OH, KH_KW, IC_KH_KW, + s0, s1, p0, p1, d0, d1); + }); +} + +static void im2col_sycl_f16(const float * x, + sycl::half * dst, + int64_t IW, + int64_t IH, + int64_t OW, + int64_t OH, + int64_t KW, + int64_t KH, + int64_t IC, + int64_t N, + int64_t IC_IH_IW, + int64_t IH_IW, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, + dpct::queue_ptr stream) { + im2col_sycl(x, dst, IW, IH, OW, OH, KW, KH, IC, N, IC_IH_IW, IH_IW, s0, s1, p0, p1, d0, d1, stream); } -static void im2col_sycl_f32(const float * x, float * dst, int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, - int64_t KH, int64_t IC, int64_t batch, int64_t batch_offset, int64_t offset_delta, int s0, - int s1, int p0, int p1, int d0, int d1, queue_ptr stream) { - im2col_sycl_internal(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, - d0, d1, stream); +static void im2col_sycl_f32(const float * x, + float * dst, + int64_t IW, + int64_t IH, + int64_t OW, + int64_t OH, + int64_t KW, + int64_t KH, + int64_t IC, + int64_t N, + int64_t IC_IH_IW, + int64_t IH_IW, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, + dpct::queue_ptr stream) { + im2col_sycl(x, dst, IW, IH, OW, OH, KW, KH, IC, N, IC_IH_IW, IH_IW, s0, s1, p0, p1, d0, d1, stream); } void ggml_sycl_op_im2col(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; + const float * src1_d = (const float *)src1->data; + float * dst_d = (float *)dst->data; + dpct::queue_ptr stream = ctx.stream(); GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); - const int32_t s0 = ((const int32_t *) (dst->op_params))[0]; - const int32_t s1 = ((const int32_t *) (dst->op_params))[1]; - const int32_t p0 = ((const int32_t *) (dst->op_params))[2]; - const int32_t p1 = ((const int32_t *) (dst->op_params))[3]; - const int32_t d0 = ((const int32_t *) (dst->op_params))[4]; - const int32_t d1 = ((const int32_t *) (dst->op_params))[5]; + const int32_t s0 = ((const int32_t*)(dst->op_params))[0]; + const int32_t s1 = ((const int32_t*)(dst->op_params))[1]; + const int32_t p0 = ((const int32_t*)(dst->op_params))[2]; + const int32_t p1 = ((const int32_t*)(dst->op_params))[3]; + const int32_t d0 = ((const int32_t*)(dst->op_params))[4]; + const int32_t d1 = ((const int32_t*)(dst->op_params))[5]; - const bool is_2D = ((const int32_t *) (dst->op_params))[6] == 1; + const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1; const int64_t IC = src1->ne[is_2D ? 2 : 1]; const int64_t IH = is_2D ? src1->ne[1] : 1; - const int64_t IW = src1->ne[0]; + const int64_t IW = src1->ne[0]; const int64_t KH = is_2D ? src0->ne[1] : 1; - const int64_t KW = src0->ne[0]; + const int64_t KW = src0->ne[0]; const int64_t OH = is_2D ? dst->ne[2] : 1; - const int64_t OW = dst->ne[1]; + const int64_t OW = dst->ne[1]; + + const int64_t IC_IH_IW = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32 + const int64_t N = src1->ne[is_2D ? 3 : 2]; + const int64_t IH_IW = src1->nb[is_2D ? 3 : 2] / 4; // nb is byte offset, src is type float32 + + if(dst->type == GGML_TYPE_F16) { + im2col_sycl_f16(src1_d, (sycl::half *) dst_d, IW, IH, OW, OH, KW, KH, IC, N, IC_IH_IW, IH_IW, s0, s1, p0, p1, + d0, d1, stream); + } else { + im2col_sycl_f32(src1_d, (float *) dst_d, IW, IH, OW, OH, KW, KH, IC, N, IC_IH_IW, IH_IW, s0, s1, p0, p1, d0, d1, stream); + } +} + +// [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW] +template +static void im2col_3d_kernel( + const float * src, T * dst, + int64_t N, int64_t IC, int64_t ID, int64_t IH, int64_t IW, int64_t OC, + int64_t KD, int64_t KH, int64_t KW, int64_t OD, int64_t OH, int64_t OW, + int64_t OH_OW, int64_t KD_KH_KW, int64_t ID_IH_IW, int64_t KH_KW, int64_t IH_IW, int64_t IC_ID_IH_IW, + int64_t IC_KD_KH_KW, int64_t OW_KD_KH_KW, int64_t OD_OH_OW_IC_KD_KH_KW, int64_t OH_OW_IC_KD_KH_KW, + int64_t OW_IC_KD_KH_KW, int64_t N_OD_OH, int64_t OD_OH, + int64_t stride_q, int64_t stride_z, int64_t stride_y, int64_t stride_x, + int s0, int s1, int s2, int p0, int p1, int p2, int d0, int d1, int d2) { + auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>(); + const int64_t i = item_ct1.get_local_id(2) + item_ct1.get_group(2) * item_ct1.get_local_range(2); + if (i >= IC_KD_KH_KW) { + return; + } + GGML_UNUSED(N); GGML_UNUSED(OC); GGML_UNUSED(OH_OW); GGML_UNUSED(OD); GGML_UNUSED(OW); GGML_UNUSED(KD); GGML_UNUSED(KH); + GGML_UNUSED(ID_IH_IW); GGML_UNUSED(IH_IW); GGML_UNUSED(IC_ID_IH_IW); GGML_UNUSED(OW_KD_KH_KW); + + const int64_t iic = i / KD_KH_KW; + const int64_t ikd = (i - iic * KD_KH_KW) / KH_KW; + const int64_t ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / KW; + const int64_t ikw = i % KW; + + const int64_t iow = item_ct1.get_group(1); + for (int64_t iz = item_ct1.get_group(0); iz < N_OD_OH; iz += MAX_GRIDDIM_Z) { + const int64_t in = iz / OD_OH; + const int64_t iod = (iz - in*OD_OH) / OH; + const int64_t ioh = iz % OH; + + const int64_t iiw = iow * s0 + ikw * d0 - p0; + const int64_t iih = ioh * s1 + ikh * d1 - p1; + const int64_t iid = iod * s2 + ikd * d2 - p2; + + const int64_t offset_dst = in*OD_OH_OW_IC_KD_KH_KW + iod*OH_OW_IC_KD_KH_KW + ioh*OW_IC_KD_KH_KW + iow*IC_KD_KH_KW + iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw; + + if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) { + dst[offset_dst] = 0.0f; + } else { + const int64_t offset_src = ((in * IC + iic) * stride_q) + (iid * stride_z) + (iih * stride_y) + (iiw * stride_x); + dst[offset_dst] = src[offset_src]; + } + } +} + +// [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW] +template +static void im2col_3d_sycl(const float * src, + T * dst, + int64_t N, + int64_t IC, + int64_t ID, + int64_t IH, + int64_t IW, + int64_t OC, + int64_t KD, + int64_t KH, + int64_t KW, + int64_t OD, + int64_t OH, + int64_t OW, + int64_t stride_q, + int64_t stride_z, + int64_t stride_y, + int64_t stride_x, + int s0, + int s1, + int s2, + int p0, + int p1, + int p2, + int d0, + int d1, + int d2, + dpct::queue_ptr stream) { + const int64_t OH_OW = OH*OW; + const int64_t KD_KH_KW = KD*KH*KW; + const int64_t ID_IH_IW = ID*IH*IW; + const int64_t KH_KW = KH*KW; + const int64_t IH_IW = IH*IW; + const int64_t IC_KD_KH_KW = IC*KD*KH*KW; + const int64_t OW_KD_KH_KW = OW*KD*KH*KW; + const int64_t N_OD_OH = N*OD*OH; + const int64_t OD_OH = OD*OH; + const int64_t IC_ID_IH_IW = IC*ID*IH*IW; + const int64_t OD_OH_OW_IC_KD_KH_KW = OD*OH*OW*IC*KD*KH*KW; + const int64_t OH_OW_IC_KD_KH_KW = OH*OW*IC*KD*KH*KW; + const int64_t OW_IC_KD_KH_KW = OW*IC*KD*KH*KW; + const int64_t num_blocks = (IC_KD_KH_KW + SYCL_IM2COL_BLOCK_SIZE - 1) / SYCL_IM2COL_BLOCK_SIZE; + dpct::dim3 block_nums(num_blocks, OW, MIN(N_OD_OH, MAX_GRIDDIM_Z)); + /* + DPCT1049:74: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. + */ + stream->parallel_for(sycl::nd_range<3>(block_nums * sycl::range<3>(1, 1, MIN(IC_KD_KH_KW, SYCL_IM2COL_BLOCK_SIZE)), + sycl::range<3>(1, 1, MIN(IC_KD_KH_KW, SYCL_IM2COL_BLOCK_SIZE))), + [=](sycl::nd_item<3>) { + im2col_3d_kernel(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW, OH_OW, KD_KH_KW, + ID_IH_IW, KH_KW, IH_IW, IC_ID_IH_IW, IC_KD_KH_KW, OW_KD_KH_KW, + OD_OH_OW_IC_KD_KH_KW, OH_OW_IC_KD_KH_KW, OW_IC_KD_KH_KW, N_OD_OH, OD_OH, + stride_q, stride_z, stride_y, stride_x, s0, s1, s2, p0, p1, p2, d0, d1, + d2); + }); +} + +static void im2col_3d_sycl_f16(const float * src, + sycl::half * dst, + int64_t N, + int64_t IC, + int64_t ID, + int64_t IH, + int64_t IW, + int64_t OC, + int64_t KD, + int64_t KH, + int64_t KW, + int64_t OD, + int64_t OH, + int64_t OW, + int64_t stride_q, + int64_t stride_z, + int64_t stride_y, + int64_t stride_x, + int s0, + int s1, + int s2, + int p0, + int p1, + int p2, + int d0, + int d1, + int d2, + dpct::queue_ptr stream) { + im2col_3d_sycl(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW, stride_q, stride_z, stride_y, + stride_x, s0, s1, s2, p0, p1, p2, d0, d1, d2, stream); +} + +static void im2col_3d_sycl_f32(const float * src, + float * dst, + int64_t N, + int64_t IC, + int64_t ID, + int64_t IH, + int64_t IW, + int64_t OC, + int64_t KD, + int64_t KH, + int64_t KW, + int64_t OD, + int64_t OH, + int64_t OW, + int64_t stride_q, + int64_t stride_z, + int64_t stride_y, + int64_t stride_x, + int s0, + int s1, + int s2, + int p0, + int p1, + int p2, + int d0, + int d1, + int d2, + dpct::queue_ptr stream) { + im2col_3d_sycl(src, dst, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW, + stride_q, stride_z, stride_y, stride_x, + s0, s1, s2, p0, p1, p2, d0, d1, d2, stream); +} + +void ggml_sycl_op_im2col_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + const float * src1_d = (const float *)src1->data; + float * dst_d = (float *)dst->data; + dpct::queue_ptr stream = ctx.stream(); + + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32); + + GGML_TENSOR_BINARY_OP_LOCALS + + const int32_t s0 = ((const int32_t *)(dst->op_params))[0]; + const int32_t s1 = ((const int32_t *)(dst->op_params))[1]; + const int32_t s2 = ((const int32_t *)(dst->op_params))[2]; + const int32_t p0 = ((const int32_t *)(dst->op_params))[3]; + const int32_t p1 = ((const int32_t *)(dst->op_params))[4]; + const int32_t p2 = ((const int32_t *)(dst->op_params))[5]; + const int32_t d0 = ((const int32_t *)(dst->op_params))[6]; + const int32_t d1 = ((const int32_t *)(dst->op_params))[7]; + const int32_t d2 = ((const int32_t *)(dst->op_params))[8]; + const int32_t IC = ((const int32_t *)(dst->op_params))[9]; + + const int64_t N = ne13 / IC; + const int64_t ID = ne12; + const int64_t IH = ne11; + const int64_t IW = ne10; + + const int64_t OC = ne03 / IC; + const int64_t KD = ne02; + const int64_t KH = ne01; + const int64_t KW = ne00; - const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / sizeof(float); - const int64_t batch = src1->ne[is_2D ? 3 : 2]; - const size_t batch_offset = src1->nb[is_2D ? 3 : 2] / sizeof(float); + const int64_t OD = ne3 / N; + const int64_t OH = ne2; + const int64_t OW = ne1; - queue_ptr stream = ctx.stream(); + const size_t es = ggml_element_size(src1); + const int64_t stride_x = src1->nb[0] / es; + const int64_t stride_y = src1->nb[1] / es; + const int64_t stride_z = src1->nb[2] / es; + const int64_t stride_q = src1->nb[3] / es; - if (dst->type == GGML_TYPE_F16) { - im2col_sycl_f16((const float *) src1->data, (sycl::half *) dst->data, IW, IH, OW, OH, KW, KH, IC, batch, - batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream); + if(dst->type == GGML_TYPE_F16) { + im2col_3d_sycl_f16(src1_d, (sycl::half *) dst_d, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW, + stride_q, stride_z, stride_y, stride_x, + s0, s1, s2, p0, p1, p2, d0, d1, d2, stream); } else { - im2col_sycl_f32((const float *) src1->data, (float *) dst->data, IW, IH, OW, OH, KW, KH, IC, batch, - batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream); + im2col_3d_sycl_f32(src1_d, (float *) dst_d, N, IC, ID, IH, IW, OC, KD, KH, KW, OD, OH, OW, + stride_q, stride_z, stride_y, stride_x, + s0, s1, s2, p0, p1, p2, d0, d1, d2, stream); } } diff --git a/ggml/src/ggml-sycl/im2col.hpp b/ggml/src/ggml-sycl/im2col.hpp index dbbb248..c9eba49 100644 --- a/ggml/src/ggml-sycl/im2col.hpp +++ b/ggml/src/ggml-sycl/im2col.hpp @@ -1,6 +1,6 @@ // // MIT license -// Copyright (C) 2024 Intel Corporation +// Copyright (C) 2026 Intel Corporation // SPDX-License-Identifier: MIT // @@ -15,7 +15,7 @@ #include "common.hpp" -void ggml_sycl_op_im2col( - ggml_backend_sycl_context & ctx, ggml_tensor *dst); +void ggml_sycl_op_im2col(ggml_backend_sycl_context & ctx, ggml_tensor * dst); +void ggml_sycl_op_im2col_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst); #endif // GGML_SYCL_IM2COL_HPP diff --git a/ggml/src/ggml-sycl/lightning-indexer.cpp b/ggml/src/ggml-sycl/lightning-indexer.cpp new file mode 100644 index 0000000..823b713 --- /dev/null +++ b/ggml/src/ggml-sycl/lightning-indexer.cpp @@ -0,0 +1,197 @@ +#include "lightning-indexer.hpp" +#include "dequantize.hpp" + +static void lightning_indexer_f32_sycl( + const char * q, const char * k, const char * w, const char * m, float * dst, + int64_t n_embd, int64_t n_head, int64_t n_batch, int64_t n_stream, int64_t n_kv, + int64_t nem3, + int64_t nbq1, int64_t nbq2, int64_t nbq3, + int64_t nbk2, int64_t nbk3, + int64_t nbw1, int64_t nbw3, + int64_t nbm1, int64_t nbm3, + int64_t nb1, int64_t nb3, + ggml_type k_type, + queue_ptr stream) { + + constexpr int64_t LANES = WARP_SIZE; + constexpr int64_t ELEMS_PER_LANE = 8; + constexpr int64_t ROWS_PER_BLOCK = 4; + constexpr int64_t BLOCK_SIZE = ROWS_PER_BLOCK * LANES; + + const int64_t n_rows = n_batch * n_stream * n_kv; + const int64_t n_blocks = (n_rows + ROWS_PER_BLOCK - 1) / ROWS_PER_BLOCK; + + stream->parallel_for( + sycl::nd_range<1>( + sycl::range<1>(n_blocks * BLOCK_SIZE), + sycl::range<1>(BLOCK_SIZE)), + [=](sycl::nd_item<1> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const int64_t ir = item.get_global_id(0); + const int64_t lane = ir % LANES; + const int64_t row = ir / LANES; + if (row >= n_rows) { + return; + } + + const int64_t i_bs = row / n_kv; + const int64_t i_kv = row % n_kv; + const int64_t i_batch = i_bs / n_stream; + const int64_t i_stream = i_bs % n_stream; + + // load K row slice into registers (row is contiguous, nbk0 == type size) + const char * k_base = k + i_kv*nbk2 + i_stream*nbk3; + float k_local[ELEMS_PER_LANE]; + if (k_type == GGML_TYPE_F16) { + const sycl::half * k_row = (const sycl::half *) k_base; +#pragma unroll + for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) { + k_local[j] = static_cast(k_row[lane*ELEMS_PER_LANE + j]); + } + } else if (k_type == GGML_TYPE_F32) { + const float * k_row = (const float *) k_base; +#pragma unroll + for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) { + k_local[j] = k_row[lane*ELEMS_PER_LANE + j]; + } + } else { + const int64_t lane_base = lane * ELEMS_PER_LANE; + switch (k_type) { + case GGML_TYPE_BF16: { + const sycl::ext::oneapi::bfloat16 * k_row = (const sycl::ext::oneapi::bfloat16 *) k_base; +#pragma unroll + for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) { + k_local[j] = static_cast(k_row[lane_base + j]); + } + } break; + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: { +#pragma unroll + for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) { + const int64_t idx = lane_base + j; + const int64_t ib = idx / QK4_0; + const int iqs = idx % (QK4_0/2); + dfloat2 kv; + if (k_type == GGML_TYPE_Q4_0) { + dequantize_q4_0(k_base, ib, iqs, kv); + } else if (k_type == GGML_TYPE_Q4_1) { + dequantize_q4_1(k_base, ib, iqs, kv); + } else if (k_type == GGML_TYPE_Q5_0) { + dequantize_q5_0(k_base, ib, iqs, kv); + } else { + dequantize_q5_1(k_base, ib, iqs, kv); + } + k_local[j] = (idx % QK4_0) < (QK4_0/2) ? static_cast(kv.x()) : static_cast(kv.y()); + } + } break; + case GGML_TYPE_Q8_0: { +#pragma unroll + for (int64_t pair = 0; pair < ELEMS_PER_LANE / 2; ++pair) { + const int64_t elem0 = lane_base + 2 * pair; + dfloat2 kv; + dequantize_q8_0(k_base, elem0 / QK8_0, elem0 % QK8_0, kv); + k_local[2 * pair + 0] = static_cast(kv.x()); + k_local[2 * pair + 1] = static_cast(kv.y()); + } + } break; + case GGML_TYPE_IQ4_NL: { +#pragma unroll + for (int64_t pair = 0; pair < ELEMS_PER_LANE / 2; ++pair) { + const int64_t elem0 = lane_base + 2 * pair; + dfloat2 kv; + dequantize_iq4_nl(k_base, elem0 / QK4_NL, elem0 % QK4_NL, kv); + k_local[2 * pair + 0] = static_cast(kv.x()); + k_local[2 * pair + 1] = static_cast(kv.y()); + } + } break; + default: +#pragma unroll + for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) { + k_local[j] = 0.0f; + } + break; + } + } + + const char * q_base = q + i_batch*nbq2 + i_stream*nbq3; + const float * w_base = (const float *) (w + i_batch*nbw1 + i_stream*nbw3); + + float score = 0.0f; + for (int64_t h = 0; h < n_head; ++h) { + const float * q_row = (const float *) (q_base + h*nbq1); + float dot = 0.0f; +#pragma unroll + for (int64_t j = 0; j < ELEMS_PER_LANE; ++j) { + const int64_t i = lane*ELEMS_PER_LANE + j; + if (i < n_embd) { + dot += q_row[i] * k_local[j]; + } + } + dot = sycl::reduce_over_group(item.get_sub_group(), dot, sycl::plus()); + if (lane == 0) { + score += sycl::max(dot, 0.0f) * w_base[h]; + } + } + + if (lane == 0) { + const sycl::half * m_base = (const sycl::half *) (m + i_batch*nbm1 + (i_stream % nem3)*nbm3); + // flat-index store: storing through a strided base pointer + // hangs/misroutes writes on this stack when n_batch*n_stream > 1 + const int64_t dst_idx = i_kv + i_batch*(nb1/sizeof(float)) + i_stream*(nb3/sizeof(float)); + dst[dst_idx] = score + static_cast(m_base[i_kv]); + } + }); +} + +void ggml_sycl_op_lightning_indexer(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/4); + const ggml_tensor * q = dst->src[0]; + const ggml_tensor * k = dst->src[1]; + const ggml_tensor * w = dst->src[2]; // weights + const ggml_tensor * m = dst->src[3]; // mask + + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT( q->type == GGML_TYPE_F32); + GGML_ASSERT( w->type == GGML_TYPE_F32); + GGML_ASSERT( m->type == GGML_TYPE_F16); + GGML_ASSERT(k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_F32 || k->type == GGML_TYPE_BF16 || + k->type == GGML_TYPE_Q8_0 || k->type == GGML_TYPE_Q5_1 || k->type == GGML_TYPE_Q5_0 || + k->type == GGML_TYPE_Q4_1 || k->type == GGML_TYPE_Q4_0 || k->type == GGML_TYPE_IQ4_NL); + + GGML_TENSOR_LOCALS(int64_t, neq, q, ne); + GGML_TENSOR_LOCALS(size_t, nbq, q, nb); + GGML_TENSOR_LOCALS(int64_t, nek, k, ne); + GGML_TENSOR_LOCALS(size_t, nbk, k, nb); + GGML_TENSOR_LOCALS(size_t, nbw, w, nb); + GGML_TENSOR_LOCALS(int64_t, nem, m, ne); + GGML_TENSOR_LOCALS(size_t, nbm, m, nb); + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne); + GGML_TENSOR_LOCALS(size_t, nb, dst, nb); + + // input rows must be contiguous + GGML_ASSERT(nbq0 == ggml_type_size(q->type)); + GGML_ASSERT(nbk0 == ggml_type_size(k->type)); + GGML_ASSERT(nbm0 == ggml_type_size(m->type)); + GGML_ASSERT(nb0 == ggml_type_size(dst->type)); + + const int64_t n_embd = neq0; + const int64_t n_head = neq1; + const int64_t n_batch = neq2; + const int64_t n_stream = neq3; + const int64_t n_kv = nek2; + + GGML_ASSERT(n_embd == WARP_SIZE * 8); + + lightning_indexer_f32_sycl( + (const char *) q->data, (const char *) k->data, + (const char *) w->data, (const char *) m->data, (float *) dst->data, + n_embd, n_head, n_batch, n_stream, n_kv, nem3, + nbq1, nbq2, nbq3, + nbk2, nbk3, + nbw1, nbw3, + nbm1, nbm3, + nb1, nb3, + k->type, + ctx.stream()); +} diff --git a/ggml/src/ggml-sycl/lightning-indexer.hpp b/ggml/src/ggml-sycl/lightning-indexer.hpp new file mode 100644 index 0000000..0b88c41 --- /dev/null +++ b/ggml/src/ggml-sycl/lightning-indexer.hpp @@ -0,0 +1,8 @@ +#ifndef GGML_SYCL_LIGHTNING_INDEXER_HPP +#define GGML_SYCL_LIGHTNING_INDEXER_HPP + +#include "common.hpp" + +void ggml_sycl_op_lightning_indexer(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_LIGHTNING_INDEXER_HPP diff --git a/ggml/src/ggml-sycl/mem.cpp b/ggml/src/ggml-sycl/mem.cpp new file mode 100644 index 0000000..8a1c326 --- /dev/null +++ b/ggml/src/ggml-sycl/mem.cpp @@ -0,0 +1,151 @@ +#include +#include + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +#include +#include +#endif + +#include +#include +#include + +#include "base.hpp" +#include "mem.hpp" + +const char * mem_api_int2str(int mem_api) { + if (mem_api == MEMORY_API_TYPE_SYCL) { + return "SYCL API"; + } else if (mem_api == MEMORY_API_TYPE_LEVEL_ZERO) { + return "Level Zero API"; + } else { + return "Unknown"; + } +} + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +/* +* Depend on to call zesInit(0) before any other Level Zero API calls, otherwise the Level Zero API calls may fail. +*/ +static bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { + GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using Level Zero API.\n", __func__); + + free_bytes = 0; + total_bytes = 0; + + uint32_t module_count = 0; + +#if defined(SYCL_EXT_ONEAPI_BACKEND_LEVEL_ZERO) + constexpr sycl::backend kL0Backend = sycl::backend::ext_oneapi_level_zero; +#else + constexpr sycl::backend kL0Backend = sycl::backend::level_zero; +#endif + + try { + + if (dev.get_platform().get_backend() != kL0Backend) { + GGML_SYCL_DEBUG("Device backend is not Level Zero.\n"); + return false; + } + + ze_device_handle_t ze_dev = sycl::get_native(dev); + if (ze_dev == nullptr) { + GGML_SYCL_DEBUG("Level Zero device handle is null.\n"); + return false; + } + + ze_result_t r = zesDeviceEnumMemoryModules(ze_dev, &module_count, nullptr); + if (r != ZE_RESULT_SUCCESS || module_count == 0) { + GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules.\n"); + return false; + } + + std::vector modules(module_count); + r = zesDeviceEnumMemoryModules(ze_dev, &module_count, modules.data()); + if (r != ZE_RESULT_SUCCESS || module_count == 0) { + GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules.\n"); + return false; + } + + for (uint32_t i = 0; i < module_count; ++i) { + zes_mem_state_t state = {}; + state.stype = ZES_STRUCTURE_TYPE_MEM_STATE; + state.pNext = nullptr; + + r = zesMemoryGetState(modules[i], &state); + if (r != ZE_RESULT_SUCCESS) { + continue; + } + + free_bytes += state.free; + total_bytes += state.size; + } + + if (total_bytes == 0) { + GGML_SYCL_DEBUG("Level Zero memory query returned zero total bytes.\n"); + return false; + } + return total_bytes >= free_bytes; + + } catch (const sycl::exception & e) { + GGML_SYCL_DEBUG("Level Zero memory query failed: %s\n", e.what()); + return false; + } +} +#endif + +static bool get_memory_size_by_sycl_api(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { + GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using SYCL API.\n", __func__); + total_bytes = dev.get_info(); + +#if (defined(__SYCL_COMPILER_VERSION) && __SYCL_COMPILER_VERSION >= 20221105) + if (dev.has(sycl::aspect::ext_intel_free_memory)) { + try { + GGML_SYCL_DEBUG("Querying free memory using SYCL aspect::ext_intel_free_memory.\n"); + free_bytes = dev.get_info(); + return true; + } catch (const sycl::exception &) { + GGML_SYCL_DEBUG( + "Failed to query free memory using SYCL aspect::ext_intel_free_memory.\n"); + return false; + } + } else { + GGML_SYCL_DEBUG( + "Device does not support SYCL aspect::ext_intel_free_memory.\n"); + } +#else + GGML_SYCL_DEBUG("SYCL Compiler version is older than 20221105.\n"); +#endif + return false; +} + +bool get_memory_size(sycl::device dev, size_t & free_bytes, size_t & total_bytes, MemoryAPIType api_type) { + + GGML_SYCL_DEBUG("[%s]GPU Name: %s\n", __func__, + dev.get_info().c_str()); + GGML_SYCL_DEBUG("[%s]GPU Vendor: %s\n", __func__, + dev.get_info().c_str()); + + if (api_type == MEMORY_API_TYPE_LEVEL_ZERO) { +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + GGML_SYCL_DEBUG("[%s] Querying free memory using Level Zero API.\n", __func__); + if (query_free_memory_by_ze(dev, free_bytes, total_bytes)) { + return true; + } + //fallback to SYCL API if Level Zero API fails + GGML_SYCL_DEBUG("[%s] Falling back to SYCL API for memory query.\n", __func__); +#endif + } + + //MEMORY_API_TYPE_SYCL + if(get_memory_size_by_sycl_api(dev, free_bytes, total_bytes)){ + return true; + } + + //Todo, fallback to other methods to get free memory size, such as using OS-specific APIs (e.g., /proc/meminfo on Linux, GlobalMemoryStatusEx on Windows, etc.) + GGML_SYCL_DEBUG( + "[%s] Can't get free mem size by Level Zero and SYCL API. Using total memory as free memory.\n", __func__); + free_bytes = total_bytes; + + return true; +} diff --git a/ggml/src/ggml-sycl/mem.hpp b/ggml/src/ggml-sycl/mem.hpp new file mode 100644 index 0000000..b3e45cf --- /dev/null +++ b/ggml/src/ggml-sycl/mem.hpp @@ -0,0 +1,16 @@ +#ifndef GGML_SYCL_MEM_HPP +#define GGML_SYCL_MEM_HPP + +#include + +enum MemoryAPIType { + MEMORY_API_TYPE_LEVEL_ZERO = 0, + MEMORY_API_TYPE_SYCL = 1, +}; + +const char* mem_api_int2str(int mem_api); + +bool get_memory_size(sycl::device dev, size_t & free_bytes, size_t & total_bytes, + MemoryAPIType api_type); + +#endif // GGML_SYCL_MEM_HPP diff --git a/ggml/src/ggml-sycl/memtrace.cpp b/ggml/src/ggml-sycl/memtrace.cpp new file mode 100644 index 0000000..ecd82ff --- /dev/null +++ b/ggml/src/ggml-sycl/memtrace.cpp @@ -0,0 +1,193 @@ +#include "memtrace.hpp" + +#include "common.hpp" +#include "ggml-impl.h" + +#include +#include +#include + +constexpr size_t MIB = 1024 * 1024; + +static const char * mem_type_name(ggml_sycl_mem_type type) { + switch (type) { + case GGML_SYCL_MEM_BUFFER: return "buffer"; + case GGML_SYCL_MEM_POOL_LEG: return "pool_leg"; + case GGML_SYCL_MEM_POOL_VMM: return "pool_vmm"; + case GGML_SYCL_MEM_ASYNC: return "async"; + case GGML_SYCL_MEM_DIRECT: return "direct"; + default: GGML_ABORT("[%s] The type value %d is not supported\n", __func__, (int) type); + } +} + +struct mem_tracker { + std::mutex mutex; + std::unordered_map> live_by_ptr; + size_t live[GGML_SYCL_MEM_TYPE_COUNT] = {}; + size_t peak[GGML_SYCL_MEM_TYPE_COUNT] = {}; + size_t total_live = 0; + size_t total_peak = 0; + size_t last_logged_peak = 0; +}; + +static mem_tracker & get_tracker() { + static mem_tracker t; + return t; +} + +static size_t step_bytes() { + const int mib = g_ggml_sycl_memtrace_step > 0 ? g_ggml_sycl_memtrace_step : 64; + return (size_t) mib * MIB; +} + +static void report_sites_locked() { + mem_tracker & t = get_tracker(); + for (int i = 0; i < GGML_SYCL_MEM_TYPE_COUNT; i++) { + if (t.peak[i] == 0) { + continue; + } + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %-9s allocated %5zu MiB, peak %5zu MiB\n", + mem_type_name((ggml_sycl_mem_type) i), t.live[i] / MIB, t.peak[i] / MIB); + } +} + +static void report_locked(const char * tag) { + mem_tracker & t = get_tracker(); + + const size_t allocated = t.total_live / MIB; + const size_t buffers = t.live[GGML_SYCL_MEM_BUFFER] / MIB; + + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %s: allocated %5zu MiB (buffers %5zu + scratch %5zu)," + " peak %5zu MiB\n", + tag, allocated, buffers, allocated - buffers, t.total_peak / MIB); + report_sites_locked(); +} + +static void log_event_locked(const char * op, ggml_sycl_mem_type type, const void * ptr, size_t bytes) { + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " allocated %5zu MiB %-5s %-9s %9.3f MiB ptr=%p\n", + get_tracker().total_live / MIB, op, mem_type_name(type), + (double) bytes / MIB, ptr); +} + +bool ggml_sycl_memtrace_enabled() { + return g_ggml_sycl_memtrace > 0; +} + +void ggml_sycl_memtrace_add(ggml_sycl_mem_type type, const void * ptr, size_t bytes) { + if (!ggml_sycl_memtrace_enabled()) { + return; + } + GGML_ASSERT(ptr != nullptr); + GGML_ASSERT(bytes != 0); + + mem_tracker & t = get_tracker(); + std::lock_guard lock(t.mutex); + + auto it = t.live_by_ptr.find(ptr); + if (it != t.live_by_ptr.end()) { + t.live[it->second.first] -= it->second.second; + t.total_live -= it->second.second; + } + + t.live_by_ptr[ptr] = { type, bytes }; + t.live[type] += bytes; + t.total_live += bytes; + + if (t.live[type] > t.peak[type]) { + t.peak[type] = t.live[type]; + } + if (t.total_live > t.total_peak) { + t.total_peak = t.total_live; + } + + if (g_ggml_sycl_memtrace >= 2) { + log_event_locked("alloc", type, ptr, bytes); + } + + static const size_t step = step_bytes(); + if (t.total_peak >= t.last_logged_peak + step) { + t.last_logged_peak = t.total_peak; + char tag[96]; + std::snprintf(tag, sizeof(tag), "peak grew (+%zu MiB from %s)", bytes / MIB, + mem_type_name(type)); + report_locked(tag); + } +} + +void ggml_sycl_memtrace_del(const void * ptr) { + if (!ggml_sycl_memtrace_enabled() || ptr == nullptr) { + return; + } + mem_tracker & t = get_tracker(); + std::lock_guard lock(t.mutex); + + auto it = t.live_by_ptr.find(ptr); + if (it == t.live_by_ptr.end()) { + return; + } + const ggml_sycl_mem_type type = it->second.first; + const size_t bytes = it->second.second; + t.live[type] -= bytes; + t.total_live -= bytes; + t.live_by_ptr.erase(it); + + if (g_ggml_sycl_memtrace >= 2) { + log_event_locked("free", type, ptr, bytes); + } +} + +void ggml_sycl_memtrace_fail(ggml_sycl_mem_type type, size_t bytes) { + GGML_LOG_ERROR(GGML_SYCL_MEMTRACE_TAG " alloc FAILED: %9.3f MiB %s\n", + (double) bytes / MIB, mem_type_name(type)); + if (!ggml_sycl_memtrace_enabled()) { + return; + } + mem_tracker & t = get_tracker(); + std::lock_guard lock(t.mutex); + report_locked("at allocation failure"); +} + +void ggml_sycl_memtrace_report(const char * tag) { + if (!ggml_sycl_memtrace_enabled()) { + return; + } + mem_tracker & t = get_tracker(); + std::lock_guard lock(t.mutex); + report_locked(tag); +} + +static bool device_memory_is_dedicated(int device) { + if (device < 0 || device >= ggml_sycl_info().device_count) { + return false; + } + const sycl_device_info & info = ggml_sycl_info().devices[device]; + return info.l0_device_type_valid && info.l0_discrete_gpu; +} + +void ggml_sycl_memtrace_report_device(const char * tag, int device, size_t dev_free, size_t dev_total) { + if (!ggml_sycl_memtrace_enabled()) { + return; + } + mem_tracker & t = get_tracker(); + std::lock_guard lock(t.mutex); + + const size_t in_use = dev_total > dev_free ? dev_total - dev_free : 0; + const size_t total = dev_total / MIB; + const size_t freed = dev_free / MIB; + const size_t allocated = t.total_live / MIB; + const size_t buffers = t.live[GGML_SYCL_MEM_BUFFER] / MIB; + const size_t peak = t.total_peak / MIB; + + if (in_use >= t.total_live && device_memory_is_dedicated(device) && total >= freed + allocated) { + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %s: total %5zu MiB = free %5zu + allocated %5zu" + " (buffers %5zu + scratch %5zu) + other %5zu, peak %5zu MiB\n", + tag, total, freed, allocated, buffers, allocated - buffers, + total - freed - allocated, peak); + } else { + GGML_LOG_INFO(GGML_SYCL_MEMTRACE_TAG " %s: total %5zu MiB, free %5zu, in use %5zu;" + " allocated %5zu (buffers %5zu + scratch %5zu), peak %5zu MiB\n", + tag, total, freed, in_use / MIB, allocated, buffers, + allocated - buffers, peak); + } + report_sites_locked(); +} diff --git a/ggml/src/ggml-sycl/memtrace.hpp b/ggml/src/ggml-sycl/memtrace.hpp new file mode 100644 index 0000000..c7da47f --- /dev/null +++ b/ggml/src/ggml-sycl/memtrace.hpp @@ -0,0 +1,27 @@ +#ifndef GGML_SYCL_MEMTRACE_HPP +#define GGML_SYCL_MEMTRACE_HPP + +#include + +#define GGML_SYCL_MEMTRACE_TAG "[SYCL-MEMTRACE]" + +enum ggml_sycl_mem_type { + GGML_SYCL_MEM_BUFFER = 0, + GGML_SYCL_MEM_POOL_LEG, + GGML_SYCL_MEM_POOL_VMM, + GGML_SYCL_MEM_ASYNC, + GGML_SYCL_MEM_DIRECT, + + GGML_SYCL_MEM_TYPE_COUNT, +}; + +bool ggml_sycl_memtrace_enabled(); + +void ggml_sycl_memtrace_add(ggml_sycl_mem_type type, const void * ptr, size_t bytes); +void ggml_sycl_memtrace_del(const void * ptr); + +void ggml_sycl_memtrace_report(const char * tag); +void ggml_sycl_memtrace_report_device(const char * tag, int device, size_t dev_free, size_t dev_total); +void ggml_sycl_memtrace_fail(ggml_sycl_mem_type type, size_t bytes); + +#endif // GGML_SYCL_MEMTRACE_HPP diff --git a/ggml/src/ggml-sycl/mmq.cpp b/ggml/src/ggml-sycl/mmq.cpp index ffb272a..19ff067 100644 --- a/ggml/src/ggml-sycl/mmq.cpp +++ b/ggml/src/ggml-sycl/mmq.cpp @@ -1807,76 +1807,44 @@ static void ggml_mul_mat_q4_0_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:20: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_qs_q4_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_d_q4_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_0) + mmq_y / QI4_0), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_0( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q4_0_acc_ct1), - get_pointer(tile_x_d_q4_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_qs_q4_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor tile_x_d_q4_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI4_0) + mmq_y / QI4_0), + cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q4_0( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_qs_q4_0_acc_ct1), + get_pointer(tile_x_d_q4_0_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:21: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_qs_q4_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_d_q4_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_0) + mmq_y / QI4_0), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_0( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q4_0_acc_ct1), - get_pointer(tile_x_d_q4_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -1922,76 +1890,44 @@ static void ggml_mul_mat_q4_1_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:22: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_qs_q4_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + +mmq_y), cgh); - sycl::local_accessor tile_x_dm_q4_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_1) + mmq_y / QI4_1), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_1( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q4_1_acc_ct1), - get_pointer(tile_x_dm_q4_1_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_qs_q4_1_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + +mmq_y), cgh); + sycl::local_accessor tile_x_dm_q4_1_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI4_1) + mmq_y / QI4_1), + cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q4_1( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_qs_q4_1_acc_ct1), + get_pointer(tile_x_dm_q4_1_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:23: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_qs_q4_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + +mmq_y), cgh); - sycl::local_accessor tile_x_dm_q4_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_1) + mmq_y / QI4_1), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_1( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q4_1_acc_ct1), - get_pointer(tile_x_dm_q4_1_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2037,76 +1973,44 @@ static void ggml_mul_mat_q5_0_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:24: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q5_0_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_d_q5_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_0) + mmq_y / QI5_0), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_0( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_0_acc_ct1), - get_pointer(tile_x_d_q5_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_ql_q5_0_acc_ct1( + sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor tile_x_d_q5_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI5_0) + mmq_y / QI5_0), + cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q5_0( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q5_0_acc_ct1), + get_pointer(tile_x_d_q5_0_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:25: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q5_0_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_d_q5_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_0) + mmq_y / QI5_0), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_0( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_0_acc_ct1), - get_pointer(tile_x_d_q5_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2152,76 +2056,44 @@ static void ggml_mul_mat_q5_1_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:26: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q5_1_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q5_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_1) + mmq_y / QI5_1), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_1( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_1_acc_ct1), - get_pointer(tile_x_dm_q5_1_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_ql_q5_1_acc_ct1( + sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor tile_x_dm_q5_1_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI5_1) + mmq_y / QI5_1), + cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q5_1( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q5_1_acc_ct1), + get_pointer(tile_x_dm_q5_1_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:27: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q5_1_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q5_1_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_1) + mmq_y / QI5_1), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_1( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_1_acc_ct1), - get_pointer(tile_x_dm_q5_1_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2267,76 +2139,44 @@ static void ggml_mul_mat_q8_0_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:28: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_qs_q8_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_d_q8_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI8_0) + mmq_y / QI8_0), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q8_0( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q8_0_acc_ct1), - get_pointer(tile_x_d_q8_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_qs_q8_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor tile_x_d_q8_0_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI8_0) + mmq_y / QI8_0), + cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q8_0( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_qs_q8_0_acc_ct1), + get_pointer(tile_x_d_q8_0_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:29: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_qs_q8_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_d_q8_0_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI8_0) + mmq_y / QI8_0), - cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q8_0( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_qs_q8_0_acc_ct1), - get_pointer(tile_x_d_q8_0_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2382,82 +2222,47 @@ static void ggml_mul_mat_q2_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:30: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI2_K) + mmq_y / QI2_K), - cgh); - sycl::local_accessor tile_x_sc_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q2_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q2_K_acc_ct1), - get_pointer(tile_x_dm_q2_K_acc_ct1), - get_pointer(tile_x_sc_q2_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_ql_q2_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor tile_x_dm_q2_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI2_K) + mmq_y / QI2_K), + cgh); + sycl::local_accessor tile_x_sc_q2_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q2_K( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q2_K_acc_ct1), + get_pointer(tile_x_dm_q2_K_acc_ct1), + get_pointer(tile_x_sc_q2_K_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:31: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI2_K) + mmq_y / QI2_K), - cgh); - sycl::local_accessor tile_x_sc_q2_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q2_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q2_K_acc_ct1), - get_pointer(tile_x_dm_q2_K_acc_ct1), - get_pointer(tile_x_sc_q2_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2505,88 +2310,50 @@ static void ggml_mul_mat_q3_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:32: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI3_K) + mmq_y / QI3_K), - cgh); - sycl::local_accessor tile_x_qh_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 2) + mmq_y / 2), cgh); - sycl::local_accessor tile_x_sc_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q3_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q3_K_acc_ct1), - get_pointer(tile_x_dm_q3_K_acc_ct1), - get_pointer(tile_x_qh_q3_K_acc_ct1), - get_pointer(tile_x_sc_q3_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_ql_q3_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor tile_x_dm_q3_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI3_K) + mmq_y / QI3_K), + cgh); + sycl::local_accessor tile_x_qh_q3_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 2) + mmq_y / 2), cgh); + sycl::local_accessor tile_x_sc_q3_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q3_K( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q3_K_acc_ct1), + get_pointer(tile_x_dm_q3_K_acc_ct1), + get_pointer(tile_x_qh_q3_K_acc_ct1), + get_pointer(tile_x_sc_q3_K_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:33: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI3_K) + mmq_y / QI3_K), - cgh); - sycl::local_accessor tile_x_qh_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 2) + mmq_y / 2), cgh); - sycl::local_accessor tile_x_sc_q3_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 4) + mmq_y / 4), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q3_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q3_K_acc_ct1), - get_pointer(tile_x_dm_q3_K_acc_ct1), - get_pointer(tile_x_qh_q3_K_acc_ct1), - get_pointer(tile_x_sc_q3_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } #endif } @@ -2633,82 +2400,47 @@ static void ggml_mul_mat_q4_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:34: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_K) + mmq_y / QI4_K), - cgh); - sycl::local_accessor tile_x_sc_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q4_K_acc_ct1), - get_pointer(tile_x_dm_q4_K_acc_ct1), - get_pointer(tile_x_sc_q4_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_ql_q4_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor tile_x_dm_q4_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI4_K) + mmq_y / QI4_K), + cgh); + sycl::local_accessor tile_x_sc_q4_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q4_K( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q4_K_acc_ct1), + get_pointer(tile_x_dm_q4_K_acc_ct1), + get_pointer(tile_x_sc_q4_K_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:35: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI4_K) + mmq_y / QI4_K), - cgh); - sycl::local_accessor tile_x_sc_q4_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q4_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q4_K_acc_ct1), - get_pointer(tile_x_dm_q4_K_acc_ct1), - get_pointer(tile_x_sc_q4_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2754,82 +2486,47 @@ static void ggml_mul_mat_q5_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:36: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_K) + mmq_y / QI5_K), - cgh); - sycl::local_accessor tile_x_sc_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_K_acc_ct1), - get_pointer(tile_x_dm_q5_K_acc_ct1), - get_pointer(tile_x_sc_q5_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_ql_q5_K_acc_ct1( + sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor tile_x_dm_q5_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI5_K) + mmq_y / QI5_K), + cgh); + sycl::local_accessor tile_x_sc_q5_K_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q5_K( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_q5_K_acc_ct1), + get_pointer(tile_x_dm_q5_K_acc_ct1), + get_pointer(tile_x_sc_q5_K_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:37: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI5_K) + mmq_y / QI5_K), - cgh); - sycl::local_accessor tile_x_sc_q5_K_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q5_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_q5_K_acc_ct1), - get_pointer(tile_x_dm_q5_K_acc_ct1), - get_pointer(tile_x_sc_q5_K_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { @@ -2875,82 +2572,47 @@ static void ggml_mul_mat_q6_K_q8_1_sycl(const void *vx, const void *vy, const sycl::range<3> block_nums(1, block_num_y, block_num_x); const sycl::range<3> block_dims(1, nwarps, WARP_SIZE); - if (nrows_x % mmq_y == 0) { - const bool need_check = false; + const auto launch = [&](auto need_check) { /* DPCT1049:38: The work-group size passed to the SYCL kernel may exceed the limit. To get the device limit, query info::device::max_work_group_size. Adjust the work-group size if needed. */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI6_K) + mmq_y / QI6_K), - cgh); - sycl::local_accessor tile_x_sc_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q6_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_acc_ct1), - get_pointer(tile_x_dm_acc_ct1), - get_pointer(tile_x_sc_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->submit([&](sycl::handler &cgh) { + sycl::local_accessor tile_x_ql_acc_ct1( + sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); + sycl::local_accessor tile_x_dm_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / QI6_K) + mmq_y / QI6_K), + cgh); + sycl::local_accessor tile_x_sc_acc_ct1( + sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); + sycl::local_accessor tile_y_qs_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE), cgh); + sycl::local_accessor tile_y_ds_acc_ct1( + sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); + + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + mul_mat_q6_K( + vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, + nrows_dst, item_ct1, + get_pointer(tile_x_ql_acc_ct1), + get_pointer(tile_x_dm_acc_ct1), + get_pointer(tile_x_sc_acc_ct1), + get_pointer(tile_y_qs_acc_ct1), + get_pointer(tile_y_ds_acc_ct1)); + }); + }); + }; + + if (nrows_x % mmq_y == 0) { + launch(std::false_type{}); } else { - const bool need_check = true; - /* - DPCT1049:39: The work-group size passed to the SYCL kernel may exceed - the limit. To get the device limit, query - info::device::max_work_group_size. Adjust the work-group size if needed. - */ - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->submit([&](sycl::handler &cgh) { - sycl::local_accessor tile_x_ql_acc_ct1( - sycl::range<1>(mmq_y * (2 * WARP_SIZE) + mmq_y), cgh); - sycl::local_accessor tile_x_dm_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / QI6_K) + mmq_y / QI6_K), - cgh); - sycl::local_accessor tile_x_sc_acc_ct1( - sycl::range<1>(mmq_y * (WARP_SIZE / 8) + mmq_y / 8), cgh); - sycl::local_accessor tile_y_qs_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE), cgh); - sycl::local_accessor tile_y_ds_acc_ct1( - sycl::range<1>(mmq_x * WARP_SIZE / QI8_1), cgh); - - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) { - mul_mat_q6_K( - vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, - nrows_dst, item_ct1, - get_pointer(tile_x_ql_acc_ct1), - get_pointer(tile_x_dm_acc_ct1), - get_pointer(tile_x_sc_acc_ct1), - get_pointer(tile_y_qs_acc_ct1), - get_pointer(tile_y_ds_acc_ct1)); - }); - }); - } + launch(std::true_type{}); } } catch (sycl::exception const &exc) { diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index af22b98..c385381 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -2,9 +2,65 @@ #include "ggml.h" #include "common.hpp" +#include "element_wise.hpp" #include "quants.hpp" #include "vecdotq.hpp" +#ifdef GGML_SYCL_MMVQ_HAS_XMX +# include "esimd.hpp" +#endif // GGML_SYCL_MMVQ_HAS_XMX + +// vec_dot_q_sycl_t adapters for the IQ vec_dots that take their codebook tables as extra +// arguments: bind the constant tables here (as vec_dot_iq2_s_q8_1 / vec_dot_iq1_m_q8_1 already do +// internally) so they can be used as template arguments of mul_mat_vec_q_moe. +static __dpct_inline__ float vec_dot_iq2_xxs_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq2_xxs_q8_1(vbq, bq8_1, iqs, iq2xxs_grid, ksigns_iq2xs, kmask_iq2xs); +} + +static __dpct_inline__ float vec_dot_iq2_xs_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq2_xs_q8_1(vbq, bq8_1, iqs, iq2xs_grid); +} + +static __dpct_inline__ float vec_dot_iq3_xxs_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq3_xxs_q8_1(vbq, bq8_1, iqs, iq3xxs_grid); +} + +static __dpct_inline__ float vec_dot_iq3_s_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq3_s_q8_1(vbq, bq8_1, iqs, iq3s_grid); +} + +static __dpct_inline__ float vec_dot_iq1_s_q8_1_moe(const void * __restrict__ vbq, + const block_q8_1 * __restrict__ bq8_1, + const int & iqs) { + return vec_dot_iq1_s_q8_1(vbq, bq8_1, iqs, iq1s_grid_gpu); +} + +// Minimum weight-row count at which the Q4_K multi-column MMVQ kernel handles two output rows per +// subgroup (rows_per_sg == 2) instead of one, when ncols_dst == 2. +// +// Pairing rows lets a subgroup load each activation block once and apply it to two rows, at the cost +// of halving the number of subgroups in the launch. With only two destination columns there is too +// little work per row to hide that loss of parallelism, so pairing only pays off once there are +// enough rows to keep the device occupied. This is a measured performance crossover, not a +// correctness or hardware limit - both variants compute the same result for any nrows. +// +// Derived on Intel Arc Pro B70 with `test-backend-ops perf -o MUL_MAT` (Q4_K, ncols_dst == 2), +// sweeping nrows over 5120..6912 at ncols 17408 and 19968: one row per subgroup was up to 9% faster +// below the crossover, two rows per subgroup 8-15% faster above it, and the crossover fell inside +// (6144, 6272] for both ncols with no measurable ncols dependence. A later 32-row granularity sweep +// narrowed it to (6144, 6176], so 6272 is a conservative gate rather than the exact crossover. +// ncols_dst >= 3 amortizes the activation loads over more columns and is faster with two rows at +// every row count, so it does not consult this threshold. +static constexpr int Q4_K_MMVQ_ROW_PAIR_MIN_NROWS = 6272; + template static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, const sycl::nd_item<3> & nd_item) { @@ -56,6 +112,155 @@ static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __r } } +// With has_fusion, `vgate` is a second weight matrix sharing vx's shape, stride and reorder +// layout: one pass computes both row dot products and the epilogue writes glu(gate, up). +template ::value> +static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void * __restrict__ vgate, + const void * __restrict__ vy, float * __restrict__ dst, const int ncols, + const int nrows, const int stride_col_y_bytes, const int stride_col_dst, + const ggml_glu_op glu_op, const sycl::nd_item<3> & nd_item) { + using block_type = ggml_sycl_reordered::block_q_t; + using block_traits = typename block_type::traits; + + const auto sg = nd_item.get_sub_group(); + const int sg_range = sg.get_group_linear_range(); + const int workgroup_id = nd_item.get_group_linear_id(); + const int sg_id = sg.get_group_linear_id(); + const int row0 = (workgroup_id * sg_range + sg_id) * rows_per_sg; + + // row is sub-group uniform, so this retires whole sub-groups and the collectives below + // stay convergent + if (row0 >= nrows) { + return; + } + + static_assert(rows_per_sg == 1 || + reorder_vec_dot_shared_activations::value); + + const int blocks_per_row = ncols / block_traits::qk; + constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi); + constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq; + const int nblocks = nrows * (ncols / block_traits::qk); + + static_assert(blocks_per_subgroup > 0); + static_assert(block_elements_per_subgroup > 0); + + float partial_sum[ncols_dst][rows_per_sg] = {}; + // sized 1 rather than 0 when unused: zero-length arrays are not standard C++, and the + // array is dead and eliminated in that case + [[maybe_unused]] float partial_gate[has_fusion ? ncols_dst : 1][has_fusion ? rows_per_sg : 1] = {}; + for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) { + const int iby = i * block_type::block_to_q8_1_ratio(); + +#pragma unroll + for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) { + const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup); + + if constexpr (rows_per_sg > 1) { + typename reorder_vec_dot_q_sycl::weights wx[rows_per_sg]; + [[maybe_unused]] typename reorder_vec_dot_q_sycl::weights wg[rows_per_sg]; +#pragma unroll + for (int r = 0; r < rows_per_sg; ++r) { + const int row = sycl::min(row0 + r, nrows - 1); + const int ibx = row * blocks_per_row + i; + const auto bx_offset = block_type::get_block_offset(ibx, nblocks); + const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); + wx[r] = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs); + if constexpr (has_fusion) { + wg[r] = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs); + } + } +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs); +#pragma unroll + for (int r = 0; r < rows_per_sg; ++r) { + partial_sum[j][r] += reorder_vec_dot_q_sycl::apply(wx[r], a); + if constexpr (has_fusion) { + partial_gate[j][r] += reorder_vec_dot_q_sycl::apply(wg[r], a); + } + } + } + } else if constexpr (shared_weights) { + const int ibx = row0 * blocks_per_row + i; + const auto bx_offset = block_type::get_block_offset(ibx, nblocks); + const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); + const auto wx = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs); + if constexpr (has_fusion) { + const auto wg = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs); + +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + + // up and gate share the activation, so load it once and apply it twice + const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs); + + partial_sum[j][0] += reorder_vec_dot_q_sycl::apply(wx, a); + partial_gate[j][0] += reorder_vec_dot_q_sycl::apply(wg, a); + } + } else { +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + + partial_sum[j][0] += reorder_vec_dot_q_sycl::dot(wx, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + } + } + } else { + const int ibx = row0 * blocks_per_row + i; + const auto bx_offset = block_type::get_block_offset(ibx, nblocks); + const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + + partial_sum[j][0] += + reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + + if constexpr (has_fusion) { + partial_gate[j][0] += + reorder_vec_dot_q_sycl()(vgate, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + } + } + } + } + } + +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { +#pragma unroll + for (int r = 0; r < rows_per_sg; ++r) { + float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j][r], std::plus<>()); + + if constexpr (has_fusion) { + const float gate = sycl::reduce_over_group(nd_item.get_sub_group(), partial_gate[j][r], std::plus<>()); + + // uniform across the launch; the launcher only instantiates SWIGLU and GEGLU + sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate); + } + + if (sg.leader() && row0 + r < nrows) { + dst[j * stride_col_dst + row0 + r] = sum; + } + } + } +} + template static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, const sycl::nd_item<3> & item_ct1) { @@ -100,6 +305,109 @@ static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict_ } } +template +static void mul_mat_vec_q_ncols( + const void * __restrict__ vx, + const void * __restrict__ vy, + float * __restrict__ dst, + const int ncols, + const int nrows, + const int stride_col_y, + const int stride_col_dst, + const sycl::nd_item<3> & item_ct1) { + + const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + + item_ct1.get_local_id(1); + + if (row >= nrows) { + return; + } + + const int blocks_per_row = ncols / qk; + constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi; + + // partial sums: one per output column + float tmp[ncols_dst] = {0.0f}; + + const block_q_t * x = (const block_q_t *) vx; + const block_q8_1 * y = (const block_q8_1 *) vy; + + for (int i = item_ct1.get_local_id(2) / (qi / vdr); + i < blocks_per_row; + i += blocks_per_warp) { + + const int ibx = row * blocks_per_row + i; + const int iby = i * (qk / QK8_1); + + // read weight block once, dot against all columns + for (size_t elem = 0; elem < qi / vdr; elem += WARP_SIZE) { + const int iqs = elem + vdr * (item_ct1.get_local_id(2) % (qi / vdr)); + +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + tmp[j] += vec_dot_q_sycl(&x[ibx], &y[j * stride_col_y + iby], iqs); + } + } + } + + // reduce within subgroup +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { +#pragma unroll + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + tmp[j] += dpct::permute_sub_group_by_xor( + item_ct1.get_sub_group(), tmp[j], mask); + } + } + + if (item_ct1.get_local_id(2) == 0) { +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + dst[j * stride_col_dst + row] = tmp[j]; + } + } +} + +template +static void launch_mul_mat_vec_q(const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % assert_qk == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + const auto launch = [&](auto ncols_tag) { + constexpr int N = decltype(ncols_tag)::value; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + if constexpr (N == 1) { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1); + } else { + mul_mat_vec_q_ncols( + vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); + } + }); + }); + }; + + switch (ncols_dst) { + case 1: launch(std::integral_constant{}); break; + case 2: launch(std::integral_constant{}); break; + case 3: launch(std::integral_constant{}); break; + case 4: launch(std::integral_constant{}); break; + case 5: launch(std::integral_constant{}); break; + case 6: launch(std::integral_constant{}); break; + case 7: launch(std::integral_constant{}); break; + case 8: launch(std::integral_constant{}); break; + default: GGML_ABORT("unsupported ncols_dst=%d", ncols_dst); + } +} + template static void mul_mat_vec_q_iq2_xxs_q8_1(const void *__restrict__ vx, const void *__restrict__ vy, @@ -182,7 +490,7 @@ static void mul_mat_vec_q_iq2_xs_q8_1(const void *__restrict__ vx, (item_ct1.get_local_id(2) % (qi / vdr)); // x block quant index when casting the quants to int - tmp += vec_dot_iq2_xs_q8_1(&x[ibx], &y[iby], iqs, iq2xs_grid, ksigns64); + tmp += vec_dot_iq2_xs_q8_1(&x[ibx], &y[iby], iqs, iq2xs_grid); } // sum up partial sums and write back result @@ -278,7 +586,7 @@ static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx, (item_ct1.get_local_id(2) % (qi / vdr)); // x block quant index when casting the quants to int - tmp += vec_dot_iq3_xxs_q8_1(&x[ibx], &y[iby], iqs, iq3xxs_grid, ksigns64); + tmp += vec_dot_iq3_xxs_q8_1(&x[ibx], &y[iby], iqs, iq3xxs_grid); } // sum up partial sums and write back result @@ -293,6 +601,11 @@ static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx, } } +static __dpct_inline__ float vec_dot_iq3_s_q8_1_adapter( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { + return vec_dot_iq3_s_q8_1(vbq, bq8_1, iqs, iq3s_grid); +} + template static void mul_mat_vec_q_iq3_s_q8_1(const void *__restrict__ vx, const void *__restrict__ vy, @@ -537,15 +850,14 @@ static void mul_mat_vec_q_iq4_xs_q8_1(const void *__restrict__ vx, static void reorder_mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK4_0 == 0); - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y); - constexpr size_t num_subgroups = 16; - GGML_ASSERT(block_num_y % num_subgroups == 0); - - const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, (block_num_y * WARP_SIZE)); - const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel. + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size), + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, nd_item); @@ -553,144 +865,199 @@ static void reorder_mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, }); } -static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, - dpct::queue_ptr stream) { +template +static void reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK4_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); - { - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_q4_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q4_0 reorder multi-col MMVQ", ncols_dst); + } +} + +static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, + dpct::queue_ptr stream) { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} + + +static void mul_mat_vec_q4_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q4_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK4_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} - stream->submit([&](sycl::handler &cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); +static void mul_mat_vec_q4_1_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q4_1 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_MXFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} - { - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); + +static void mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for MXFP4 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_nvfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_NVFP4 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} - { - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); + +static void mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for NVFP4 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} - stream->submit([&](sycl::handler &cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); +static void mul_mat_vec_q5_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q5_0 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK5_1 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} - stream->submit([&](sycl::handler &cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); +static void mul_mat_vec_q5_1_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q5_1 multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q8_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK8_0 == 0); - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y); - constexpr size_t num_subgroups = 16; - GGML_ASSERT(block_num_y % num_subgroups == 0); + // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel. + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); - const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, (block_num_y * WARP_SIZE)); - const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + if (g_ggml_sycl_mmvq_wide) { + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder(vx, vy, dst, ncols, nrows, nd_item); + }); + }); + return; + } stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size), + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, nd_item); @@ -698,115 +1065,381 @@ static void reorder_mul_mat_vec_q8_0_q8_1_sycl(const void * vx, const void * vy, }); } -static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { +template +static void reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK8_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); - stream->submit([&](sycl::handler &cgh) { + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); +static void reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_q8_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q8_0 reorder multi-col MMVQ", ncols_dst); } } -static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; +// MXFP4 reorder GEMV. Only MoE expert slices are reordered (opt_for_reorder_id); these dense entry +// points serve per-expert ggml_sycl_mul_mat calls from multi-token MUL_MAT_ID after that reorder. +static void reorder_mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_MXFP4 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { - - stream->submit([&](sycl::handler &cgh) { + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); - } + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, + nd_item); + }); + }); } -static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, - float *dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; +template +static void reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_MXFP4 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); - stream->submit([&](sycl::handler &cgh) { + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); +static void reorder_mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_mxfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for MXFP4 reorder multi-col MMVQ", ncols_dst); } } -static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, +static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} + + +static void mul_mat_vec_q8_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q8_0 multi-col MMVQ", ncols_dst); + } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); +} + + +static void mul_mat_vec_q1_0_q8_1_sycl(const void * vx, const void * vy, + float * dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} + + +static void mul_mat_vec_q1_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q1_0 multi-col MMVQ", ncols_dst); + } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); +} + + +static void mul_mat_vec_q2_0_q8_1_sycl(const void * vx, const void * vy, + float * dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} + + +static void mul_mat_vec_q2_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q2_0 multi-col MMVQ", ncols_dst); + } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); +} + + +static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} + + +static void mul_mat_vec_q2_K_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q2_K multi-col MMVQ", ncols_dst); + } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); +} + + +static void reorder_mul_mat_vec_q2_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + + // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel. + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); - stream->submit([&](sycl::handler &cgh) { + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, + nd_item); + }); + }); +} - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); +template +static void reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_q2_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K reorder multi-col MMVQ", ncols_dst); + } +} + +static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} + + +static void reorder_mul_mat_vec_q3_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + + // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel. + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, + nd_item); + }); + }); +} + +template +static void reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_q3_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q3_K reorder multi-col MMVQ", ncols_dst); + } +} + +static void mul_mat_vec_q3_K_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q3_K multi-col MMVQ", ncols_dst); + } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); +} + + + +static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} + + +static void mul_mat_vec_q4_K_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q4_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y); - constexpr size_t num_subgroups = 16; - GGML_ASSERT(block_num_y % num_subgroups == 0); - - const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, block_num_y * WARP_SIZE); - const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel. + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size), + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, nd_item); @@ -814,74 +1447,248 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy, }); } +template +static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst, + /*has_fusion=*/ false, rows_per_sg>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +template +static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + constexpr int rows_per_sg = ncols_dst >= 3 && ncols_dst <= 4 ? 2 : 1; + reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); +} + +static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_q4_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: + if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) { + reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } else { + reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 1>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } + break; + case 3: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q4_K reorder multi-col MMVQ", ncols_dst); + } +} static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} + + +static void mul_mat_vec_q5_K_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q5_K multi-col MMVQ", ncols_dst); + } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); +} + + +static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); - stream->submit([&](sycl::handler &cgh) { + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>(vx, vy, dst, ncols, + nrows, nd_item); + }); + }); +} - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); +template +static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols_impl( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst, + /*has_fusion=*/ false, rows_per_sg, shared_weights>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +template +static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ggml_sycl_q5_k_mmvq_reuse(ggml_sycl_get_device())) { + constexpr int rows_per_sg = ncols_dst >= 3 ? 2 : 1; + reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols_impl(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } else { + reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols_impl(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } +} + +static void reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_q5_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q5_K reorder multi-col MMVQ", ncols_dst); } } static void reorder_mul_mat_vec_q6_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y); - constexpr size_t num_subgroups = 16; - GGML_ASSERT(block_num_y % num_subgroups == 0); + // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel. + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); - const sycl::range<3> global_size(1, GGML_SYCL_MMV_Y, block_num_y * WARP_SIZE); - const sycl::range<3> workgroup_size(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<3>(global_size, workgroup_size), + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, nd_item); }); }); } + +template +static void reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_q6_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q6_K reorder multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - { + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, 1, 0, 0, stream); +} - stream->submit([&](sycl::handler &cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) - [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); +static void mul_mat_vec_q6_K_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ncols_dst < 1 || ncols_dst > 8) { + GGML_ABORT("unsupported ncols_dst=%d for Q6_K multi-col MMVQ", ncols_dst); } + launch_mul_mat_vec_q( + vx, vy, dst, ncols, nrows, ncols_dst, stride_col_y, stride_col_dst, stream); } + static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -968,6 +1775,67 @@ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy, } } +template +static void reorder_mul_mat_vec_iq3_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S); + GGML_ASSERT(ncols % QK_K == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>( + vx, vy, dst, ncols, nrows, nd_item); + }); + }); +} + +template +static void reorder_mul_mat_vec_iq3_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + static_assert(type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S); + GGML_ASSERT(ncols % QK_K == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +template +static void reorder_mul_mat_vec_iq3_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_iq3_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_iq3_q8_1_sycl_ncols(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for reordered IQ3 multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -990,6 +1858,67 @@ static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy, } } +template +static void mul_mat_vec_iq3_s_q8_1_sycl_ncols(const void * vx, + const void * vy, + float * dst, + const int ncols, + const int nrows, + const int stride_col_y, + const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_ncols( + vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); + }); + }); +} + +static void mul_mat_vec_iq3_s_q8_1_sycl_switch_ncols(const void * vx, + const void * vy, + float * dst, + const int ncols, + const int nrows, + const int ncols_dst, + const int stride_col_y, + const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: + mul_mat_vec_iq3_s_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); + break; + case 2: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 3: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 4: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 5: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 6: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 7: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + case 8: + mul_mat_vec_iq3_s_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); + break; + default: + GGML_ABORT("unsupported ncols_dst=%d for IQ3_S multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_iq1_s_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -1077,6 +2006,203 @@ static void mul_mat_vec_iq4_xs_q8_1_sycl(const void *vx, const void *vy, } } +template +static void mul_mat_vec_iq4_xs_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_K == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_ncols( + vx, vy, dst, ncols, nrows, + stride_col_y, stride_col_dst, item_ct1); + }); + }); +} + +static void mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: mul_mat_vec_iq4_xs_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 3: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 4: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 5: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 6: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 7: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 8: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for IQ4_XS multi-col MMVQ", ncols_dst); + } +} + +#ifdef GGML_SYCL_MMVQ_HAS_XMX +// vgate is the gate weight of the fused gate and up kernel, nullptr for a plain mat-vec +template